diff --git a/core/http/react-ui/e2e/connector-fixtures.js b/core/http/react-ui/e2e/connector-fixtures.js index 6261d91ad3..477fbb5df6 100644 --- a/core/http/react-ui/e2e/connector-fixtures.js +++ b/core/http/react-ui/e2e/connector-fixtures.js @@ -29,7 +29,7 @@ export const DECLARED = { export const CAPABILITIES = [ { name: 'files.upload', title: 'Receive files', description: 'The agent may read files that senders attach to a message.', sensitive: false, usual: true }, { name: 'memory.write', title: 'Save to memory', description: 'The agent may remember what a sender asks it to remember.', sensitive: false, usual: true }, - { name: 'proposals.confirm', title: 'Confirm changes from the chat', description: 'A sender can confirm a change that the agent proposed, by answering in the same chat.', sensitive: false, usual: true }, + { name: 'proposals.confirm', title: 'Confirm changes from the chat', description: 'A sender can confirm a trigger or a change that the agent proposed (a memory note, the agent rules, a skill, a page or the dashboard) by answering yes or no in the same chat. Changes to the rules, skills, pages and dashboard need a paired sender.', sensitive: false, usual: true }, { name: 'talkback.channels', title: 'Post to approved channels', description: 'The agent may write to the approved channels when a sender asks.', sensitive: false, usual: true }, { name: 'talkback.new_channel', title: 'Post to a new channel', description: 'The agent may write to a channel that is not approved yet.', sensitive: true, usual: false }, { name: 'tools.write', title: 'Change things outside LocalAI', description: 'An add-on tool may change something with the credential of the binding.', sensitive: false, usual: false }, diff --git a/core/services/studio/changes_test.go b/core/services/studio/changes_test.go index b18ebf008c..6c8350ea01 100644 --- a/core/services/studio/changes_test.go +++ b/core/services/studio/changes_test.go @@ -95,6 +95,15 @@ func expectProposeConfirmRevert(r *trigRig, send func(string), wantKind, wantID res := r.h.ResultsFor("propose_change") Expect(res[0].Err).To(BeNil(), "result = %+v", res) Expect(res[0].Out).To(ContainSubstring("Nothing is saved yet")) + if wantKind == "" { + // A chat the person watches in Studio keeps the Studio wording; a + // connector chat is told the host asked in the chat (connectorchange_test.go). + Expect(res[0].Out).To(ContainSubstring("confirm it in Studio")) + Expect(res[0].Out).NotTo(ContainSubstring("yes or no")) + } else { + Expect(res[0].Out).To(ContainSubstring("the person answers there with yes or no")) + Expect(res[0].Out).NotTo(ContainSubstring("Studio")) + } p := onlyProposal(r) Expect(p.Status).To(Equal(contract.ProposalPending)) wantSource := p.Source @@ -380,11 +389,15 @@ var _ = Describe("change proposals", func() { r := newConnRig(sh, connStore(), bot, changeScript("channel", "- home: Rome\n"), nil) r.grantMemory() b := r.bind(studio.ConnectorInput{Allow: []string{"11"}}) - n := 0 + n, want := 0, 0 send := func(text string) { n++ + want++ + if text == "remember" { + want++ // the host also writes the proposal in the chat + } bot.say(n, 11, "Ada", "ada", text) - waitFor("the answer", func() bool { return len(bot.answers()) == n }) + waitFor("the answer", func() bool { return len(bot.answers()) == want }) } expectProposeConfirmRevert(r.trigRig, send, session.OriginConnector, b.ID, channelFile) }) @@ -407,6 +420,9 @@ var _ = Describe("change proposals", func() { n := 0 send := func(text string) { n++ + if text == "remember" { + n++ // the host also writes the proposal in the chat + } f.Push(slackApp, slacktest.Mention("C1", "U1", "UBOT", text, "")) waitFor("the answer", func() bool { return posted() == n }) } diff --git a/core/services/studio/changetools.go b/core/services/studio/changetools.go index a30e5d132b..81a8ade4d0 100644 --- a/core/services/studio/changetools.go +++ b/core/services/studio/changetools.go @@ -24,7 +24,7 @@ const ( const proposeChangeDescription = "Ask to change something that lasts: a note in the user's memory, the profile of this agent, a skill, a page or the Home dashboard. " + "Use it when the user asks you to remember, change or forget something, gives a rule you must keep, or asks for a skill, a page or a dashboard change. " + - "Unless the result says the note was saved, it changes nothing by itself: the owner must confirm in Studio, and the proposal expires after a day. " + + "Unless the result says the note was saved, it changes nothing by itself: the person must confirm it (the result says how), and the proposal expires after a day. " + "Never say that something is saved until the result says so. " + "content is the full new text of the file, not a patch: read the current file first and keep what stays. " + "kind is memory (a Markdown file under Memory/), profile (the agent's standing rules, at most 4 KiB, scope agent), " + @@ -171,7 +171,7 @@ func (h *Changes) proposeTool(ctx context.Context, c builtins.Chat, raw json.Raw } args.Content = string(b) } - in := ProposeInput{User: c.Actor.Principal, Agent: c.AgentID, Source: sourceOf(c), + in := ProposeInput{User: c.Actor.Principal, Agent: c.AgentID, Source: h.sourceOf(c), Kind: args.Kind, Scope: args.Scope, Target: args.Target, Content: args.Content, Summary: args.Summary} // How a memory note is saved is the person's choice (see changemode.go). // Every other kind always asks. @@ -197,6 +197,11 @@ func (h *Changes) proposeTool(ctx context.Context, c builtins.Chat, raw json.Raw // The card lets the interface show Confirm and Dismiss on this tool line, // also after a reload. session.SetToolCard(ctx, session.ToolCard{Kind: ChangeProposalCardKind, ID: p.ID}) + if c.Origin == session.OriginConnector { + // The person reads the chat of the messaging app, not Studio: the host + // writes the question there and the person answers it there. + return h.connectorProposed(ctx, c, p, autoNote), nil + } return fmt.Sprintf("Proposal %s stored: %s (%s, %s). Nothing is saved yet: the owner must confirm it in Studio, and it expires in 24 hours. Do not say it is saved.%s", p.ID, p.Summary, p.Kind, p.Scope, autoNote), nil } diff --git a/core/services/studio/connectorchange.go b/core/services/studio/connectorchange.go new file mode 100644 index 0000000000..797b78459d --- /dev/null +++ b/core/services/studio/connectorchange.go @@ -0,0 +1,387 @@ +package studio + +import ( + "context" + "crypto/sha256" + "fmt" + "slices" + "strings" + "time" + "unicode/utf8" + + "github.com/mudler/LocalAI/core/services/studio/builtins" + "github.com/mudler/LocalAI/core/services/studio/connectors" + "github.com/mudler/LocalAI/core/services/studio/proposals" + "github.com/mudler/LocalAI/core/services/studio/session" + "github.com/mudler/LocalAI/core/services/studio/triggers" + "github.com/mudler/LocalAI/pkg/addons/contract" +) + +// A change proposal that a connector chat made (a note, a profile, a skill, a +// page, the dashboard) is shown in that chat by the host, and the sender +// answers it there with a short word, exactly as for a trigger. The host reads +// the answer before any model turn, so the text never reaches the model. + +// Sentences the bot says about a change proposal. +const ( + sentChangeNoRight = "Only the owner can confirm this." + sentChangeNotYou = "Only the person who asked can confirm this." + sentChangeNothing = "There is nothing here to undo." + sentChangeCannot = "I could not undo it: %s" + + // undoWord is the one word that takes back a change that was just applied + // from the chat. A bare "no" or "cancel" after a save does not: it may + // mean anything. + undoWord = "undo" + + // changeExcerptRunes bounds the text of a proposal that is written to a chat. + changeExcerptRunes = 500 + // changeExcerptLines bounds the lines of it. + changeExcerptLines = 8 +) + +// ChangeCode is the short code of a change proposal, the same on every +// frontend because it is derived from the ID of the proposal. +func ChangeCode(id string) string { + sum := sha256.Sum256([]byte(id)) + var b [codeLen]byte + for i := range b { + b[i] = codeAlphabet[int(sum[i])%len(codeAlphabet)] + } + return string(b[:]) +} + +// kindWords says what a kind of change is, for a person who reads it in a chat. +func kindWords(k contract.ChangeKind) string { + switch k { + case contract.ChangeMemory: + return "a note in the memory" + case contract.ChangeProfile: + return "a change of the rules of this agent" + case contract.ChangeSkill: + return "a skill" + case contract.ChangePage: + return "a page" + case contract.ChangeDashboard: + return "a change of the Home dashboard" + } + return "a change" +} + +// scopeWords says where a change applies. +func scopeWords(s contract.ChangeScope) string { + switch s { + case contract.ScopeAgent: + return "this agent only" + case contract.ScopeChannel: + return "this conversation only" + } + return "every chat of the owner" +} + +// addedLines returns the non-empty lines of next that cur does not have, in +// order. For a text that only removes lines it is empty. +func addedLines(cur, next string) []string { + have := map[string]bool{} + for _, l := range strings.Split(cur, "\n") { + have[strings.TrimSpace(l)] = true + } + var out []string + for _, l := range strings.Split(next, "\n") { + if t := strings.TrimSpace(l); t != "" && !have[t] { + out = append(out, t) + } + } + return out +} + +// excerptOf shows the lines, at most changeExcerptLines and changeExcerptRunes. +func excerptOf(lines []string) string { + more := false + if len(lines) > changeExcerptLines { + lines, more = lines[:changeExcerptLines], true + } + s := "+ " + strings.Join(lines, "\n+ ") + if utf8.RuneCountInString(s) > changeExcerptRunes { + s, more = string([]rune(s)[:changeExcerptRunes]), true + } + if more { + s += "…" + } + return s +} + +// proposalChatText is what the host writes in the chat where a change was +// proposed: what will be saved, and how to answer. The model's words are +// quoted, never trusted: mentions that notify everyone are broken. +func (h *Changes) proposalChatText(ctx context.Context, p contract.ChangeProposal) string { + var shown string + if kind, ok := h.kinds.Lookup(string(p.Kind)); ok { + switch p.Kind { + case contract.ChangePage: + shown = "a page with its files" + default: + if st, err := h.snapshot(ctx, p.Owner, kind, p.Target, p.Content); err == nil { + if add := addedLines(st.current, p.Content); len(add) > 0 { + shown = excerptOf(add) + } else { + shown = "(it only removes lines)" + } + } + } + } + var b strings.Builder + fmt.Fprintf(&b, "I can save %s (%s): %s", kindWords(p.Kind), scopeWords(p.Scope), p.Summary) + if shown != "" { + b.WriteString("\n" + shown) + } + code := ChangeCode(p.ID) + fmt.Fprintf(&b, "\nNothing is saved until you answer. Reply \"yes\" to save it or \"no\" to drop it, here, within 24 hours. "+ + "If more than one waits, reply \"confirm %s\" or \"cancel %s\".", code, code) + return neutralMentions.ReplaceAllString(b.String(), "${1}​${2}") +} + +// connectorSender is the one sender whose message made the turn of the chat, or +// "" when several did or none is known. +func (h *Changes) connectorSender(chat string) string { + if g, ok := h.s.access.get(chat); ok && len(g.Senders) == 1 { + return g.Senders[0] + } + return "" +} + +// sourceOf says where a change of the chat c comes from. A connector chat also +// names the sender, so that only that sender can answer in a shared chat. +func (h *Changes) sourceOf(c builtins.Chat) contract.ChangeSource { + src := sourceOf(c) + if c.Origin == session.OriginConnector { + src.Sender = h.connectorSender(c.Actor.ChatID) + } + return src +} + +// senderMayConfirm says whether the sender who made a proposal could also +// confirm it. A change of a kind that changes what the agent may do needs a +// paired sender, as the gate decides. +func (h *Changes) senderMayConfirm(ctx context.Context, c builtins.Chat, p contract.ChangeProposal) bool { + if ok, _ := c.Holds(CapProposalsConfirm); !ok || p.Source.Sender == "" { + return false + } + if p.Kind == contract.ChangeMemory { + return true + } + return h.s.allowCap(ctx, CapabilityRequest{Capability: CapProposalsConfirm, Binding: c.OriginID, Owner: p.Owner, Agent: c.AgentID, + Sender: p.Source.Sender, Kind: string(p.Kind), Sensitive: true}) +} + +// connectorProposed writes the proposal in the chat of the sender and words the +// result for the model. The model is told what the host did, so it never sends +// the person to Studio when they can answer where they are. +func (h *Changes) connectorProposed(ctx context.Context, c builtins.Chat, p contract.ChangeProposal, autoNote string) string { + head := fmt.Sprintf("Proposal %s stored: %s (%s, %s). Nothing is saved yet.", p.ID, p.Summary, p.Kind, p.Scope) + conns := h.s.connectors + if !h.senderMayConfirm(ctx, c, p) { + return head + " The person cannot confirm changes from this chat, so it waits for the owner in Studio and expires in 24 hours. " + + "Tell them it is not saved and that the owner decides. Do not say it is saved." + autoNote + } + chat, thread, ok := "", "", false + if conns != nil { + chat, thread, ok = conns.PlaceOf(ctx, c.OriginID, c.Actor.ChatID) + } + if ok { + if _, err := conns.Post(ctx, Post{Binding: c.OriginID, Chat: chat, Thread: thread, Text: h.proposalChatText(ctx, p)}); err == nil { + return head + " The host wrote the proposal in this chat, with what will be saved; the person answers there with yes or no, and it expires in 24 hours. " + + "Say in one short sentence that you asked; do not repeat the text and do not say it is saved." + autoNote + } else { + h.s.log.Warn("studio: could not write a change proposal to its chat", "proposal", p.ID, "error", err) + } + } + return head + " The host could not write the proposal in this chat, so it waits for the owner in Studio and expires in 24 hours. " + + "Tell the person it is not saved yet. Do not say it is saved." + autoNote +} + +// hereChanges lists the proposals of the binding that were made in the place +// where m is written, newest first, in every state. Where a chat is written is +// the platform chat and thread of its last message, as for a trigger. +func (c *Connectors) hereChanges(ctx context.Context, b connectors.Binding, m contract.ConnectorMessage) []contract.ChangeProposal { + ch := c.svc.changes + if ch == nil { + return nil + } + all, err := ch.List(ctx, b.Owner) + if err != nil { + return nil + } + var out []contract.ChangeProposal + for _, p := range all { + if p.Source.Kind != "connector" || p.Source.ID != b.ID || p.Source.Chat == "" { + continue + } + if chat, thread, ok := c.PlaceOf(ctx, b.ID, p.Source.Chat); !ok || chat != m.Chat || thread != m.Thread { + continue + } + out = append(out, p) + } + return out +} + +// mayAnswerChange asks the capability gate whether the sender may answer the +// proposal. A kind that changes what the agent may do is sensitive: only a +// paired sender passes. +func (c *Connectors) mayAnswerChange(ctx context.Context, b connectors.Binding, m contract.ConnectorMessage, p contract.ChangeProposal) bool { + return c.svc.allowCap(ctx, CapabilityRequest{Capability: CapProposalsConfirm, Binding: b.ID, Owner: b.Owner, Agent: b.AgentID, + Sender: m.SenderID, Username: m.SenderUsername, Roles: m.Roles, Kind: string(p.Kind), Sensitive: p.Kind != contract.ChangeMemory}) +} + +// answerChange confirms or drops the waiting proposal p for the sender of m. A +// sender who may not gets one short sentence; the proposal keeps waiting. +func (c *Connectors) answerChange(ctx context.Context, b connectors.Binding, m contract.ConnectorMessage, p contract.ChangeProposal, verb string, bare bool) (string, bool) { + ch := c.svc.changes + detail := "by text from sender " + m.SenderID + if !c.mayAnswerChange(ctx, b, m, p) { + ch.audit(ctx, proposals.ActionRefuse, p, 0, "not allowed; "+detail) + if bare && p.Source.Sender == m.SenderID { + // The sender who asked cannot confirm, and the host wrote nothing + // to answer: an ordinary "ok" is just a message. + return "", false + } + return sentChangeNoRight, true + } + // In a chat of two the other listed sender may answer; in a shared chat + // only the sender whose message made the proposal. + if !m.Direct && p.Source.Sender != "" && p.Source.Sender != m.SenderID { + ch.audit(ctx, proposals.ActionRefuse, p, 0, "not the sender who asked; "+detail) + return sentChangeNotYou, true + } + if verb == "cancel" { + if _, err := ch.Dismiss(ctx, b.Owner, p.ID); err != nil { + return fmt.Sprintf(sentProposalFailed, changeErr(err)), true + } + return sentProposalCancel, true + } + done, err := ch.Confirm(ctx, b.Owner, p.ID) + if err != nil { + ch.audit(ctx, proposals.ActionRefuse, p, 0, "confirm failed; "+detail) + return fmt.Sprintf(sentProposalFailed, changeErr(err)), true + } + return fmt.Sprintf("%s: %s. Reply \"undo\" within %d minutes to take it back.", savedWord(done.Kind), done.Summary, int(cancelWindow/time.Minute)), true +} + +func savedWord(k contract.ChangeKind) string { + if k == contract.ChangeMemory { + return "Saved" + } + return "Done" +} + +// undoChange takes back the change that was applied last in this place, by the +// word "undo", when the sender may answer proposals here. +func (c *Connectors) undoChange(ctx context.Context, b connectors.Binding, m contract.ConnectorMessage, here []contract.ChangeProposal) (string, bool) { + ch := c.svc.changes + now := c.o.Clock() + var last *contract.ChangeProposal + for i, p := range here { + if p.Status == contract.ProposalApplied && !p.Auto && now.Sub(p.Answered) < cancelWindow && + (last == nil || p.Answered.After(last.Answered)) { + last = &here[i] + } + } + if last == nil { + return "", false + } + p := *last + if !c.mayAnswerChange(ctx, b, m, p) { + ch.audit(ctx, proposals.ActionRefuse, p, 0, "undo not allowed; by text from sender "+m.SenderID) + return sentChangeNoRight, true + } + if !m.Direct && p.Source.Sender != "" && p.Source.Sender != m.SenderID { + return sentChangeNotYou, true + } + if _, err := ch.Revert(ctx, b.Owner, p.ID); err != nil { + return fmt.Sprintf(sentChangeCannot, changeErr(err)), true + } + return fmt.Sprintf("Removed: %s.", p.Summary), true +} + +// answerChanges handles a short answer when change proposals wait in this +// place or one was just applied. handled false leaves the message to the +// trigger path and then to the agent. +func (c *Connectors) answerChanges(ctx context.Context, b connectors.Binding, m contract.ConnectorMessage, verb, code string, isUndo bool) (string, bool) { + here := c.hereChanges(ctx, b, m) + if len(here) == 0 { + return "", false + } + now := c.o.Clock() + var waiting, lapsed []contract.ChangeProposal + for _, p := range here { + switch { + case p.Status == contract.ProposalPending && now.Before(p.Expires): + waiting = append(waiting, p) + case (p.Status == contract.ProposalExpired || p.Status == contract.ProposalPending) && now.Sub(p.Expires) < lapsedWindow: + lapsed = append(lapsed, p) + } + } + // The newest waits first in the list. + slices.SortFunc(waiting, func(a, z contract.ChangeProposal) int { return z.Created.Compare(a.Created) }) + if code != "" { + for _, p := range append(slices.Clone(waiting), lapsed...) { + if ChangeCode(p.ID) != code { + continue + } + if !slices.ContainsFunc(waiting, func(w contract.ChangeProposal) bool { return w.ID == p.ID }) { + if !c.mayAnswerChange(ctx, b, m, p) { + return "", false + } + return sentProposalGone, true + } + return c.answerChange(ctx, b, m, p, verb, false) + } + return "", false + } + if len(waiting) == 0 { + if isUndo { + return c.undoChange(ctx, b, m, here) + } + if len(lapsed) > 0 && c.mayAnswerChange(ctx, b, m, lapsed[0]) { + return sentProposalGone, true + } + return "", false + } + // Several things can wait here, triggers too: a bare word answers only when + // exactly one does. + others := c.svc.triggers.waitingHere(ctx, PhraseFrom{Binding: b.ID, Owner: b.Owner, Chat: m.Chat, Thread: m.Thread}) + if len(waiting) == 1 && len(others) == 0 { + return c.answerChange(ctx, b, m, waiting[0], verb, true) + } + if !c.mayAnswerChange(ctx, b, m, waiting[0]) { + return "", false + } + var sb strings.Builder + sb.WriteString("More than one is waiting here. Say which, for example \"confirm CODE\" or \"cancel CODE\":") + for _, p := range waiting { + fmt.Fprintf(&sb, "\n%s: %q", ChangeCode(p.ID), p.Summary) + } + for _, t := range others { + fmt.Fprintf(&sb, "\n%s: %q", t.Route.Code, t.Name) + } + return sb.String(), true +} + +// waitingHere lists the trigger proposals that wait in the place of the sender. +func (h *Triggers) waitingHere(ctx context.Context, from PhraseFrom) []triggers.Trigger { + if h == nil { + return nil + } + all, err := h.o.Store.ListTriggers(ctx, principal(from.Owner)) + if err != nil { + return nil + } + now := h.o.Clock() + var out []triggers.Trigger + for _, t := range all { + if t.Pending() && !t.Expired(now) && t.Route.Binding == from.Binding && sameThread(t, from) { + out = append(out, t) + } + } + return out +} diff --git a/core/services/studio/connectorchange_test.go b/core/services/studio/connectorchange_test.go new file mode 100644 index 0000000000..49240d6cfb --- /dev/null +++ b/core/services/studio/connectorchange_test.go @@ -0,0 +1,398 @@ +package studio_test + +import ( + "context" + "strings" + "time" + + "github.com/mudler/LocalAI/addons/telegram/transport" + "github.com/mudler/LocalAI/core/services/studio" + "github.com/mudler/LocalAI/core/services/studio/connectors" + "github.com/mudler/LocalAI/core/services/studio/proposals" + "github.com/mudler/LocalAI/core/services/studio/studiotest" + "github.com/mudler/LocalAI/core/services/studio/workspace" + "github.com/mudler/LocalAI/pkg/addons/contract" + "github.com/mudler/LocalAI/tests/studiobot" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "github.com/onsi/gomega/types" +) + +var _ = DescribeTable("a short answer to a proposal", + func(text, verb, code string, ok bool) { + v, c, got := studio.ParseReply(text) + Expect(got).To(Equal(ok)) + Expect(v).To(Equal(verb)) + Expect(c).To(Equal(code)) + }, + Entry("yes", "yes", "confirm", "", true), + Entry("with marks and capitals", " Yes! ", "confirm", "", true), + Entry("confirm", "confirm", "confirm", "", true), + Entry("ok to apply", "ok", "confirm", "", true), + Entry("Italian", "sì", "confirm", "", true), + Entry("a thumb", "👍", "confirm", "", true), + Entry("no", "no", "cancel", "", true), + Entry("dismiss with the word cancel", "cancel", "cancel", "", true), + Entry("Italian no", "annulla", "cancel", "", true), + Entry("a code, lower case", "yes 7k2q", "confirm", "7K2Q", true), + Entry("confirm and a code", "confirm 7K2Q", "confirm", "7K2Q", true), + Entry("cancel and a code", "cancel 7K2Q", "cancel", "7K2Q", true), + Entry("a sentence is not an answer", "yes please save that and also the other thing", "", "", false), + Entry("another word", "maybe", "", "", false), + Entry("empty", " ", "", "", false), +) + +// Change proposals of a connector chat are shown in the chat and answered +// there, before any model turn. +var _ = Describe("change proposals in a connector chat", func() { + ctx := context.Background() + const ( + listed = "11" // on the allowlist, not paired + paired = "12" // paired + ) + var ( + sh *shared + bot *scriptedBot + st connectors.Store + props proposals.Store + r *connRig + b connectors.Binding + ) + script := func(_ workspace.Actor, text string) []studiotest.ToolStep { + kind := func(name string) []studiotest.ToolStep { + for _, c := range grantCases { + if c.args["kind"] == name { + return []studiotest.ToolStep{{Tool: "propose_change", Args: argsJSON(c.args)}, {Say: "I asked."}} + } + } + return nil + } + switch text { + case "note": + return []studiotest.ToolStep{{Tool: "propose_change", Args: proposeArgs("memory", "user", "index.md", "- standup at 09:30\n", "Standup time")}, {Say: "I asked."}} + case "note two": + return []studiotest.ToolStep{{Tool: "propose_change", Args: proposeArgs("memory", "user", "food.md", "- likes tea @channel\n", "Drinks")}, {Say: "I asked."}} + case "profile", "skill", "page", "dashboard": + return kind(text) + } + return []studiotest.ToolStep{{Say: "model"}} + } + allCaps := []string{contract.CapMemoryWrite, contract.CapProfileWrite, contract.CapSkillsWrite, contract.CapPagesWrite, contract.CapDashboardWrite, contract.CapProposalsConfirm} + + newTransport := func() *studiobot.BotTransport { + addon := studiobot.Telegram(&transport.API{BaseURL: bot.srv.URL, Client: bot.srv.Client()}) + addon.Kind.Permissions = allCaps + return studiobot.NewTransport(addon) + } + setupWith := func(caps []string) { + sh, bot, st = changeShared(grantAll), newScriptedBot(), connStore() + var err error + props, err = proposals.NewJSON("") + Expect(err).NotTo(HaveOccurred()) + prev := sh.deps + sh.deps = func(d *studio.Deps) { prev(d); d.Proposals = props } + r = newConnRigWith(sh, st, newTransport(), script, func(context.Context) bool { return false }) + r.bot, r.untrusted = bot, true + r.svc.SetCapabilityGate(r.conns.CapabilityGate()) + r.grantBuiltins(contract.CapMemoryWrite, contract.CapProfileWrite, contract.CapSkillsWrite, contract.CapPagesWrite, contract.CapDashboardWrite) + b = r.bind(studio.ConnectorInput{Allow: []string{listed, paired, "-100"}, Caps: caps}) + got, err := r.store.GetBinding(ctx, b.ID) + Expect(err).NotTo(HaveOccurred()) + got.Paired = append(got.Paired, connectors.PairedSender{SenderID: paired, At: sh.clock.Now()}) + Expect(r.store.PutBinding(ctx, got)).To(Succeed()) + b = got + } + setup := func() { setupWith(allCaps) } + + // say writes in a private chat and waits for n messages of the bot. + say := func(via *connRig, sender, text string, n int) []string { + before := len(bot.answers()) + Expect(via.conns.Ingest(ctx, "telegram", contract.ConnectorInbound{BindingID: b.ID, ConnectorMessage: contract.ConnectorMessage{ + Chat: sender, Direct: true, SenderID: sender, SenderName: "S", Text: text}})).To(Succeed()) + waitFor("the answers", func() bool { return len(bot.answers()) >= before+n }) + sh.clock.Advance(15 * time.Second) + return bot.answers()[before:] + } + // sayIn writes in a group. + sayIn := func(sender, text string, n int) []string { + before := len(bot.answers()) + Expect(r.conns.Ingest(ctx, "telegram", contract.ConnectorInbound{BindingID: b.ID, ConnectorMessage: contract.ConnectorMessage{ + Chat: "-100", Mention: true, SenderID: sender, SenderName: "S", Text: text}})).To(Succeed()) + waitFor("the answers", func() bool { return len(bot.answers()) >= before+n }) + sh.clock.Advance(15 * time.Second) + return bot.answers()[before:] + } + modelTurns := func() int { + chats, err := r.svc.List(ctx, alice, 50) + Expect(err).NotTo(HaveOccurred()) + n := 0 + for _, c := range chats { + n += len(r.h.Sent(c.ID)) + } + return n + } + all := func() []contract.ChangeProposal { + l, err := r.svc.Changes().List(ctx, alice) + Expect(err).NotTo(HaveOccurred()) + return l + } + only := func() contract.ChangeProposal { + l := all() + Expect(l).To(HaveLen(1)) + return l[0] + } + haveStatus := func(s contract.ProposalStatus) types.GomegaMatcher { + return WithTransform(func(p contract.ChangeProposal) contract.ProposalStatus { return p.Status }, Equal(s)) + } + lastResult := func() string { + res := r.h.ResultsFor("propose_change") + Expect(res).NotTo(BeEmpty()) + Expect(res[len(res)-1].Err).NotTo(HaveOccurred()) + return res[len(res)-1].Out + } + + Describe("a memory note from a private chat", func() { + It("is shown with what will be saved, confirmed by yes without a model turn, and undone by undo", func() { + setup() + out := say(r, listed, "note", 2) + Expect(out[0]).To(ContainSubstring("I can save a note in the memory (every chat of the owner): Standup time")) + Expect(out[0]).To(ContainSubstring("+ - standup at 09:30")) + Expect(out[0]).To(ContainSubstring(`Reply "yes" to save it or "no" to drop it`)) + p := only() + Expect(out[0]).To(ContainSubstring(`"confirm ` + studio.ChangeCode(p.ID) + `"`)) + Expect(p.Source.Sender).To(Equal(listed)) + Expect(out[1]).To(Equal("I asked.")) + // The model is told what the host did, and is not sent to Studio. + res := lastResult() + Expect(res).To(ContainSubstring("The host wrote the proposal in this chat")) + Expect(res).To(ContainSubstring("yes or no")) + Expect(res).NotTo(ContainSubstring("Studio")) + _, found := readHome(r.trigRig, p.Target) + Expect(found).To(BeFalse(), "nothing is written before the answer") + + turns := modelTurns() + out = say(r, listed, "yes", 1) + Expect(out[0]).To(Equal(`Saved: Standup time. Reply "undo" within 30 minutes to take it back.`)) + Expect(modelTurns()).To(Equal(turns), "the model never read the answer") + Expect(only()).To(haveStatus(contract.ProposalApplied)) + text, found := readHome(r.trigRig, p.Target) + Expect(found).To(BeTrue()) + Expect(text).To(ContainSubstring("standup at 09:30")) + + out = say(r, listed, "undo", 1) + Expect(out[0]).To(Equal("Removed: Standup time.")) + Expect(only()).To(haveStatus(contract.ProposalReverted)) + _, found = readHome(r.trigRig, p.Target) + Expect(found).To(BeFalse()) + Expect(modelTurns()).To(Equal(turns)) + + // The answer is used once: a second yes is an ordinary message. + out = say(r, listed, "yes", 1) + Expect(out[0]).To(Equal("model")) + }) + + It("breaks the mentions of everyone in the text it writes", func() { + setup() + out := say(r, listed, "note two", 2) + Expect(out[0]).NotTo(ContainSubstring("@channel")) + Expect(out[0]).To(ContainSubstring("tea")) + }) + + It("is dropped by no", func() { + setup() + say(r, listed, "note", 2) + out := say(r, listed, "no", 1) + Expect(out[0]).To(Equal("Cancelled.")) + Expect(only()).To(haveStatus(contract.ProposalDismissed)) + }) + + It("is confirmed by a word with its code", func() { + setup() + say(r, listed, "note", 2) + p := only() + out := say(r, listed, "Confirm "+strings.ToLower(studio.ChangeCode(p.ID)), 1) + Expect(out[0]).To(HavePrefix("Saved:")) + }) + + It("does not undo on a plain no after the save", func() { + setup() + say(r, listed, "note", 2) + say(r, listed, "yes", 1) + out := say(r, listed, "no", 1) + Expect(out[0]).To(Equal("model")) + Expect(only()).To(haveStatus(contract.ProposalApplied)) + }) + + It("lets undo through to the model when nothing was saved from the chat", func() { + setup() + out := say(r, listed, "undo", 1) + Expect(out[0]).To(Equal("model")) + }) + }) + + DescribeTable("a proposal of each kind is confirmed in the chat", + func(kind, wantText, wantDone string) { + setup() + out := say(r, paired, kind, 2) + Expect(out[0]).To(ContainSubstring(wantText)) + p := only() + Expect(p.Kind).To(Equal(contract.ChangeKind(kind))) + Expect(lastResult()).NotTo(ContainSubstring("Studio")) + turns := modelTurns() + out = say(r, paired, "yes", 1) + Expect(out[0]).To(HavePrefix(wantDone)) + Expect(modelTurns()).To(Equal(turns)) + Expect(only()).To(haveStatus(contract.ProposalApplied)) + _, found := readHome(r.trigRig, p.Target+pageIndex(kind)) + Expect(found).To(BeTrue()) + }, + Entry("profile", "profile", "I can save a change of the rules of this agent (this agent only)", "Done:"), + Entry("skill", "skill", "I can save a skill (every chat of the owner)", "Done:"), + Entry("page", "page", "I can save a page (every chat of the owner)", "Done:"), + Entry("dashboard", "dashboard", "I can save a change of the Home dashboard", "Done:"), + ) + + Describe("who may answer", func() { + It("refuses a proposal of a sensitive kind to a sender who is only on the allowlist", func() { + setup() + sayIn(paired, "profile", 2) + Expect(only().Source.Sender).To(Equal(paired)) + out := sayIn(listed, "yes", 1) + Expect(out[0]).To(Equal("Only the owner can confirm this.")) + Expect(only()).To(haveStatus(contract.ProposalPending)) + }) + + It("holds a note of a shared chat for the sender who asked", func() { + setup() + sayIn(listed, "note", 2) + out := sayIn(paired, "yes", 1) + Expect(out[0]).To(Equal("Only the person who asked can confirm this.")) + Expect(only()).To(haveStatus(contract.ProposalPending)) + out = sayIn(listed, "yes", 1) + Expect(out[0]).To(HavePrefix("Saved:")) + }) + + It("lets another trusted sender answer in a chat of two", func() { + setup() + // Two accounts that write in the same private place. + ingest := func(sender, text string) { + Expect(r.conns.Ingest(ctx, "telegram", contract.ConnectorInbound{BindingID: b.ID, ConnectorMessage: contract.ConnectorMessage{ + Chat: "500", Direct: true, SenderID: sender, SenderName: "S", Text: text}})).To(Succeed()) + } + ingest(listed, "note") + waitFor("the proposal", func() bool { return len(bot.answers()) >= 2 }) + ingest(paired, "yes") + waitFor("the answer", func() bool { return len(bot.answers()) >= 3 }) + Expect(bot.answers()[2]).To(HavePrefix("Saved:")) + }) + + It("tells the model that the owner decides when the sender cannot confirm, writes nothing, and passes an ordinary yes on", func() { + setupWith([]string{contract.CapMemoryWrite}) + out := say(r, listed, "note", 1) + Expect(out[0]).To(Equal("I asked."), "the host wrote no proposal") + res := lastResult() + Expect(res).To(ContainSubstring("cannot confirm changes from this chat")) + Expect(res).To(ContainSubstring("waits for the owner in Studio")) + p := only() + turns := modelTurns() + out = say(r, listed, "yes", 1) + Expect(out[0]).To(Equal("model")) + Expect(modelTurns()).To(Equal(turns + 1)) + Expect(only()).To(haveStatus(contract.ProposalPending)) + // A word with the code is answered with a refusal. + out = say(r, listed, "confirm "+studio.ChangeCode(p.ID), 1) + Expect(out[0]).To(Equal("Only the owner can confirm this.")) + Expect(only()).To(haveStatus(contract.ProposalPending)) + }) + }) + + Describe("time", func() { + It("says that the proposal expired, soon after, and then lets the word through", func() { + setup() + say(r, listed, "note", 2) + sh.clock.Advance(contract.ChangeExpiry + time.Minute) + out := say(r, listed, "yes", 1) + Expect(out[0]).To(Equal("That proposal expired. Ask again if you still want it.")) + Expect(only()).To(haveStatus(contract.ProposalExpired)) + _, found := readHome(r.trigRig, only().Target) + Expect(found).To(BeFalse()) + sh.clock.Advance(2 * time.Hour) + out = say(r, listed, "yes", 1) + Expect(out[0]).To(Equal("model")) + }) + + It("does not undo after the window", func() { + setup() + say(r, listed, "note", 2) + say(r, listed, "yes", 1) + sh.clock.Advance(31 * time.Minute) + out := say(r, listed, "undo", 1) + Expect(out[0]).To(Equal("model")) + Expect(only()).To(haveStatus(contract.ProposalApplied)) + }) + }) + + Describe("two proposals at once", func() { + It("asks which, and a code answers one", func() { + setup() + say(r, listed, "note", 2) + say(r, listed, "note two", 2) + ps := all() + Expect(ps).To(HaveLen(2)) + turns := modelTurns() + out := say(r, listed, "yes", 1) + Expect(out[0]).To(ContainSubstring("More than one is waiting here.")) + for _, p := range ps { + Expect(out[0]).To(ContainSubstring(studio.ChangeCode(p.ID) + ": ")) + } + Expect(modelTurns()).To(Equal(turns)) + for _, p := range all() { + Expect(p).To(haveStatus(contract.ProposalPending)) + } + var food contract.ChangeProposal + for _, p := range ps { + if strings.HasSuffix(p.Target, "food.md") { + food = p + } + } + out = say(r, listed, "confirm "+studio.ChangeCode(food.ID), 1) + Expect(out[0]).To(HavePrefix("Saved: Drinks")) + for _, p := range all() { + if p.ID == food.ID { + Expect(p).To(haveStatus(contract.ProposalApplied)) + } else { + Expect(p).To(haveStatus(contract.ProposalPending)) + } + } + // With one left, a bare word answers it. + out = say(r, listed, "no", 1) + Expect(out[0]).To(Equal("Cancelled.")) + }) + }) + + Describe("another frontend", func() { + It("answers a proposal that the first frontend stored, once", func() { + setup() + say(r, listed, "note", 2) + p := only() + second := newConnRigWith(sh, st, newTransport(), script, func(context.Context) bool { return false }) + second.svc.SetCapabilityGate(second.conns.CapabilityGate()) + out := say(second, listed, "yes", 1) + Expect(out[0]).To(HavePrefix("Saved:")) + got, err := second.svc.Changes().Get(ctx, alice, p.ID) + Expect(err).NotTo(HaveOccurred()) + Expect(got).To(haveStatus(contract.ProposalApplied)) + Expect(only()).To(haveStatus(contract.ProposalApplied)) + out = say(r, listed, "yes", 1) + Expect(out[0]).To(Equal("model"), "the answer is used once") + }) + }) +}) + +// pageIndex is the file of a page to look for after it is saved. +func pageIndex(kind string) string { + if kind == "page" { + return "/index.html" + } + return "" +} diff --git a/core/services/studio/connectorconfirm.go b/core/services/studio/connectorconfirm.go index 6031642e6e..5dad7c6872 100644 --- a/core/services/studio/connectorconfirm.go +++ b/core/services/studio/connectorconfirm.go @@ -15,12 +15,21 @@ import ( // read or fetched can reach this path. Who may answer is decided by the // capability gate, not here. func (c *Connectors) answerPhrase(ctx context.Context, b connectors.Binding, m contract.ConnectorMessage, say func(string)) bool { - h := c.svc.triggers - if h == nil { + verb, code, ok := ParseReply(m.Text) + isUndo := normalizeReply(m.Text) == undoWord + if !ok { return false } - verb, code, ok := ParseReply(m.Text) - if !ok { + // Change proposals (a note, a profile, a skill, a page, the dashboard) are + // looked at first, since a bare word with a change waiting must not be + // taken by a trigger alone. They return handled false when nothing waits + // here, or when the code is not theirs. + if reply, handled := c.answerChanges(ctx, b, m, verb, code, isUndo); handled { + say(reply) + return true + } + h := c.svc.triggers + if h == nil { return false } reply, handled := h.AnswerByPhrase(ctx, PhraseFrom{Binding: b.ID, Owner: b.Owner, Agent: b.AgentID, diff --git a/core/services/studio/proposals/proposalstest/suite.go b/core/services/studio/proposals/proposalstest/suite.go index eb96d986b2..6cf17ce594 100644 --- a/core/services/studio/proposals/proposalstest/suite.go +++ b/core/services/studio/proposals/proposalstest/suite.go @@ -26,7 +26,7 @@ var t0 = time.Date(2026, 11, 3, 9, 0, 0, 0, time.UTC) func Sample(id, owner, agent string, age time.Duration) contract.ChangeProposal { return contract.ChangeProposal{V: contract.ChangeProposalVersion, ID: id, Owner: owner, Agent: agent, Kind: contract.ChangeMemory, Scope: contract.ScopeUser, Target: "Memory/index.md", Summary: "a note", Content: "- home: Rome\n", BaseHash: "abc", - Source: contract.ChangeSource{Kind: "connector", ID: "b1", Chat: "c1"}, Status: contract.ProposalPending, + Source: contract.ChangeSource{Kind: "connector", ID: "b1", Chat: "c1", Thread: "t1", Sender: "s1"}, Status: contract.ProposalPending, Created: t0.Add(age), Expires: t0.Add(age + contract.ChangeExpiry)} } diff --git a/docs/content/features/chats-connectors.md b/docs/content/features/chats-connectors.md index 5ea96a8ed9..f4e7937c88 100644 --- a/docs/content/features/chats-connectors.md +++ b/docs/content/features/chats-connectors.md @@ -625,7 +625,7 @@ owner paired with this bot"). The model is told to relay it and not to say that no such tool. Today this gates the agent's change proposals (`memory.write` for a memory note, `profile.write` for a profile), files sent to the bot (`files.upload`), the trigger tools -(`triggers.propose`), and the confirm phrase (`proposals.confirm`). Messages from a +(`triggers.propose`), and the confirm phrase (`proposals.confirm`), which answers trigger proposals and change proposals alike. Messages from a sender who may not send files get a short refusal and no turn. A binding that was stored before this check has no capabilities. Enable what you want @@ -735,6 +735,42 @@ The preview writes nothing and records no decision. It does not take a token and reply never has one. It uses the same sign-in, Connectors feature and routing as the other connector routes, and the connection of another user is a 404. +## Confirming a change in the chat + +When the agent proposes a change that lasts (a memory note, a change of its rules, a +skill, a page or the Home dashboard, with `propose_change`) in a chat from a messaging +app, LocalAI writes the question in that chat. The model does not write it, and it does +not send the person to Studio. Messaging chats always ask first; this only gives the +answer a place. + +- **The question.** For example: `I can save a note in the memory (every chat of the + owner): Standup time`, then the lines that will be added (at most 8 lines or 500 + characters), then `Reply "yes" to save it or "no" to drop it, here, within 24 hours. If + more than one waits, reply "confirm CODE" or "cancel CODE".` The code has four + characters and is the same on every frontend. Mentions that notify everyone are broken. +- **The answer.** A short word as the whole message, with the same words and languages as + for triggers: `yes`, `ok`, `confirm` to save, `no`, `cancel` to drop. LocalAI reads it + before any model turn and the model never sees it. The reply says what happened, for + example `Saved: Standup time. Reply "undo" within 30 minutes to take it back.` +- **Undo.** The word `undo`, alone, within 30 minutes of a save that was made from the + chat, restores the earlier text (`Removed: ...`). Any other word does not undo. +- **Several waiting.** A bare word answers only when exactly one proposal (a change or a + trigger) waits in that chat and thread. Otherwise the bot lists the codes and the + person writes `confirm CODE` or `cancel CODE`. +- **Who may answer.** A sender who holds `proposals.confirm` on the binding (a sender on + the allowlist by ID or name, a paired sender, or a role with the grant). A change of + the rules of the agent, a skill, a page or the dashboard needs a paired sender. In a + shared chat only the sender whose message made the proposal can answer; in a private + chat any trusted sender of that chat can. Another sender gets one sentence, `Only the + owner can confirm this.` or `Only the person who asked can confirm this.`, and the + proposal keeps waiting. When the sender who asked cannot confirm, the bot writes no + question and the proposal waits in Studio for the owner; the model is told so. +- **Time.** A proposal lapses after 24 hours. For 15 minutes after that, a bare word is + answered with `That proposal expired. Ask again if you still want it.`; later it is + an ordinary message. +- **Studio.** The same proposal shows as a card in Studio; whichever answer comes first + wins and the other finds it closed. Every answer is in the audit of changes. + ## Triggers from a connector chat When the binding holds the capability `triggers.propose`, the agent can propose a @@ -753,7 +789,8 @@ trigger stays in Studio. Nothing runs before the owner confirms. `confirm CODE` or `cancel CODE`. Any longer message goes to the agent as usual. It needs the capability `proposals.confirm` for that sender (a sender on the allowlist by ID or name, a paired sender, or a role with the grant); a message from any other sender, or text the agent read, never answers - a proposal. A code is used once, and a proposal lapses after 24 hours. Studio always + a proposal. The same words confirm a change of the agent (see + [Confirming a change in the chat](#confirming-a-change-in-the-chat)). A code is used once, and a proposal lapses after 24 hours. Studio always works as well. - **Permissions are in the text.** The proposal says what a run may do with nobody present, in plain words: "Permissions: read only", or the `act.*` classes it carries, diff --git a/docs/content/features/short-memory.md b/docs/content/features/short-memory.md index 83b2ecdc24..1c349f6cc1 100644 --- a/docs/content/features/short-memory.md +++ b/docs/content/features/short-memory.md @@ -59,6 +59,12 @@ whatever you choose: - Chats that come from a messaging app always ask first. The person who wrote the message is not always you, so a message from outside can never save to your memory without your confirmation. Scheduled and background runs cannot save notes at all. + The bot writes the question in the chat ("I can save a note ... Reply yes to save it or + no to drop it"), and a sender whom you trust with the permission **Confirm changes from + the chat** answers there with `yes` or `no`, within 24 hours. `undo` takes a saved note + back for 30 minutes. Another sender, or one without that permission, cannot answer: the + note waits in Studio for you. See + [Confirming a change in the chat](/features/chats-connectors/#confirming-a-change-in-the-chat). - A saved agent cannot widen your choice. If you choose to be asked, it asks, even when you let it write to memory. - A reply saves at most 5 notes at once. A long note, or one that looks like it holds a diff --git a/pkg/addons/contract/changeproposal.go b/pkg/addons/contract/changeproposal.go index 853c27413f..7b0fd18a9c 100644 --- a/pkg/addons/contract/changeproposal.go +++ b/pkg/addons/contract/changeproposal.go @@ -86,6 +86,10 @@ type ChangeSource struct { ID string `json:"id,omitempty"` Chat string `json:"chat,omitempty"` Thread string `json:"thread,omitempty"` + // Sender is the platform ID of the sender whose message made a connector + // chat propose this change. The host keeps it so that, in a shared chat, + // only that sender can answer the proposal there. + Sender string `json:"sender,omitempty"` } // ChangeProposal is one proposed change. Content is the full new text of the diff --git a/pkg/addons/contract/permissions.go b/pkg/addons/contract/permissions.go index 1973b07d50..888d38a7ed 100644 --- a/pkg/addons/contract/permissions.go +++ b/pkg/addons/contract/permissions.go @@ -83,7 +83,7 @@ type CapabilityInfo struct { // capabilityText is the title and description of each capability. Titles are // short verbs; descriptions say what changes and who can cause it. var capabilityText = map[string][2]string{ - CapProposalsConfirm: {"Confirm changes from the chat", "A sender can confirm a change that the agent proposed, such as a memory or a trigger, by answering in the same chat."}, + CapProposalsConfirm: {"Confirm changes from the chat", "A sender can confirm a trigger or a change that the agent proposed (a memory note, the agent rules, a skill, a page or the dashboard) by answering yes or no in the same chat. Changes to the rules, skills, pages and dashboard need a paired sender."}, CapMemoryWrite: {"Save to memory", "The agent may remember what a sender asks it to remember."}, CapFilesUpload: {"Receive files", "The agent may read files that senders attach to a message."}, CapTriggersPropose: {"Propose triggers", "The agent may propose a timer or an event trigger when a sender asks for one."}, diff --git a/swagger/docs.go b/swagger/docs.go index d80c75f857..aca692e9bc 100644 --- a/swagger/docs.go +++ b/swagger/docs.go @@ -16695,6 +16695,10 @@ const docTemplate = `{ "kind": { "type": "string" }, + "sender": { + "description": "Sender is the platform ID of the sender whose message made a connector\nchat propose this change. The host keeps it so that, in a shared chat,\nonly that sender can answer the proposal there.", + "type": "string" + }, "thread": { "type": "string" } diff --git a/swagger/swagger.json b/swagger/swagger.json index 82f25e197f..3ad02ee0e1 100644 --- a/swagger/swagger.json +++ b/swagger/swagger.json @@ -16692,6 +16692,10 @@ "kind": { "type": "string" }, + "sender": { + "description": "Sender is the platform ID of the sender whose message made a connector\nchat propose this change. The host keeps it so that, in a shared chat,\nonly that sender can answer the proposal there.", + "type": "string" + }, "thread": { "type": "string" } diff --git a/swagger/swagger.yaml b/swagger/swagger.yaml index bb722715f9..fcb17664a5 100644 --- a/swagger/swagger.yaml +++ b/swagger/swagger.yaml @@ -907,6 +907,12 @@ definitions: type: string kind: type: string + sender: + description: |- + Sender is the platform ID of the sender whose message made a connector + chat propose this change. The host keeps it so that, in a shared chat, + only that sender can answer the proposal there. + type: string thread: type: string type: object