diff --git a/core/services/studio/connectoraccess.go b/core/services/studio/connectoraccess.go index 202353db8f..52564f584e 100644 --- a/core/services/studio/connectoraccess.go +++ b/core/services/studio/connectoraccess.go @@ -513,6 +513,26 @@ func (s *Service) messagingFacts(chat string) (direct, paired bool) { return g.Direct, g.Paired } +// NotesReadCapability names the decision that says whether a sender's turn +// reads the owner's personal notes. It is not on the closed list of +// capabilities: the owner cannot tick it, and no grant gives it. A paired +// sender holds it and nobody else (see Service.personalNotesOpen); the +// notes of the conversation itself are read whatever the answer. +const NotesReadCapability = "notes.read" + +// notesDecision is the verdict of the read rule for a kind of sender, on the +// same terms as the turn applies it. +func notesDecision(b connectors.Binding, s AccessSubject, now time.Time) Decision { + if b.CredentialsOnly() { + return refusal(b, NotesReadCapability, CodeNoAgent, "this connection has no agent: it holds credentials only", nil) + } + if s.SenderID != "" && b.IsPaired(s.SenderID, now) { + return Decision{Capability: NotesReadCapability, Allowed: true, Via: "paired"} + } + return refusal(b, NotesReadCapability, CodeSenderNeedsPairing, + "the personal notes of the owner are read only for a sender that the owner paired with this bot", nil) +} + // AccessSubjectView is what one kind of sender may do on a binding. type AccessSubjectView struct { // ID is "listed" (a sender on the allowlist by ID or name), "anyone" (a @@ -520,7 +540,8 @@ type AccessSubjectView struct { // "role:". ID string `json:"id"` Title string `json:"title"` - // Decisions has one entry for each capability of the closed list. + // Decisions has one entry for each capability of the closed list, then + // the one of NotesReadCapability. Decisions []Decision `json:"decisions"` } @@ -646,6 +667,7 @@ func (c *Connectors) accessOf(ctx context.Context, b connectors.Binding, only st for _, name := range contract.ConnectorCapabilityNames() { v.Decisions = append(v.Decisions, Authorize(bind, kind, s, name, now, c.offReason(ctx, b, name))) } + v.Decisions = append(v.Decisions, notesDecision(bind, s, now)) return v } want := func(id string) bool { diff --git a/core/services/studio/connectorreasons.go b/core/services/studio/connectorreasons.go index 180c35a5b2..e2233de128 100644 --- a/core/services/studio/connectorreasons.go +++ b/core/services/studio/connectorreasons.go @@ -204,6 +204,11 @@ var capabilityWords = map[string]capabilityPhrase{ // actionOf is the verb phrase of a capability, "do this" for a name that is // not on the list. func actionOf(capability string) string { + // The read rule of the personal notes is not a capability of the closed + // list, but it is refused with the same texts. + if capability == NotesReadCapability { + return "read the owner's personal notes" + } if w, ok := capabilityWords[capability]; ok { return w.Action } diff --git a/core/services/studio/connectors_enforce_test.go b/core/services/studio/connectors_enforce_test.go index 499f30d197..683f8baf3b 100644 --- a/core/services/studio/connectors_enforce_test.go +++ b/core/services/studio/connectors_enforce_test.go @@ -154,7 +154,7 @@ var _ = Describe("permissions in the turns of a connector chat", func() { allowed := map[string][]string{} for _, s := range view.Subjects { for _, d := range s.Decisions { - if d.Allowed { + if d.Allowed && d.Capability != studio.NotesReadCapability { allowed[s.ID] = append(allowed[s.ID], d.Capability) } } diff --git a/core/services/studio/connectors_reasons_test.go b/core/services/studio/connectors_reasons_test.go index bf98152d25..d4247d7400 100644 --- a/core/services/studio/connectors_reasons_test.go +++ b/core/services/studio/connectors_reasons_test.go @@ -61,6 +61,22 @@ var _ = Describe("why a capability is blocked on a connection", func() { } const propose = contract.CapTriggersPropose + It("says that the owner's personal notes are read only for a paired sender, in the saved view and the preview", func() { + setup(contract.CapMemoryWrite) + for name, v := range map[string]studio.AccessView{"saved": saved(), "preview": preview(studio.AccessDraft{})} { + Expect(decision(v, "paired", studio.NotesReadCapability).Allowed).To(BeTrue(), name) + for _, kind := range []string{"listed", "anyone", "role:writer"} { + d := decision(v, kind, studio.NotesReadCapability) + Expect(d.Allowed).To(BeFalse(), name+" "+kind) + Expect(d.Code).To(Equal(studio.CodeSenderNeedsPairing), name+" "+kind) + Expect(d.BlockedBy).To(Equal(studio.BySender)) + Expect(d.Fix).To(Equal(&studio.Fix{Kind: studio.FixPairSender, Target: "pairing"})) + Expect(d.Message).To(Equal("This person needs to be paired with this bot before the bot can read the owner's personal notes for them.")) + Expect(d.Say()).To(Equal("You need to be paired with this bot by the owner before it can read the owner's personal notes for you.")) + } + } + }) + It("names the connection when the owner did not tick the permission", func() { setup(contract.CapMemoryWrite) d := decision(saved(), "listed", propose) diff --git a/core/services/studio/contextturn.go b/core/services/studio/contextturn.go index 1adea08307..4dd0d32234 100644 --- a/core/services/studio/contextturn.go +++ b/core/services/studio/contextturn.go @@ -81,7 +81,11 @@ func (s *Service) contextTurnFor(ctx context.Context, rec session.ChatRecord, to } sub := contextSubject(rec, saveToolOf(tools)) sub.OmitEmpty = omitEmpty - profile, index := s.ctxfiles.Profile(ctx, sub), s.ctxfiles.Block(ctx, sub, extra...) + var profile contextfiles.Block + if s.personalNotesOpen(rec) { + profile = s.ctxfiles.Profile(ctx, sub) + } + index := s.ctxfiles.Block(ctx, sub, extra...) return joinBlocks(profile.Text, index.Text), profile.Key + index.Key } diff --git a/core/services/studio/contextturn_internal_test.go b/core/services/studio/contextturn_internal_test.go index 27433a0edd..0ffa751a57 100644 --- a/core/services/studio/contextturn_internal_test.go +++ b/core/services/studio/contextturn_internal_test.go @@ -434,6 +434,8 @@ var _ = Describe("context files in the prompt of a chat", func() { _, err = svc.Changes().Confirm(ctx, "alice", p.ID) Expect(err).NotTo(HaveOccurred(), scope) } + // The owner's notes are read only for a paired sender. + svc.access.set(rec.ID, turnGrant{Binding: "b-tg", Kind: "telegram", Senders: []string{"owner"}, Direct: true, Paired: true}) x, err := svc.ChatAccess(ctx, "alice", rec.ID) Expect(err).NotTo(HaveOccurred()) tt, err := svc.chatTools(ctx, x) diff --git a/core/services/studio/memory.go b/core/services/studio/memory.go index da5ec1c525..f9511f99ad 100644 --- a/core/services/studio/memory.go +++ b/core/services/studio/memory.go @@ -306,6 +306,31 @@ func (s *Service) memoryReads(ctx context.Context, rec session.ChatRecord, sc *s return sc == nil || sc.memory != "" || sc.addons[MemoryAddonID] } +// personalNotesOpen says whether a chat may read the owner's personal notes +// (the user and agent scopes of the notes, and the agent profile). A chat +// outside a messaging app always may. A messaging chat may only when the +// sender is paired with the bot; an allowlisted sender who is not paired, a +// sender let in through a channel, a turn with no known sender and a grant +// that expired all read none. The notes of the conversation itself (the +// channel scope) are not personal notes and are never held back here. This is +// the read side of the rule that memoryChatScopes enforces on writes. +func (s *Service) personalNotesOpen(rec session.ChatRecord) bool { + if rec.Origin != session.OriginConnector { + return true + } + _, paired := s.messagingFacts(rec.ID) + return paired +} + +// withoutPersonalNotes drops the user and agent index files of the notes from +// a list of sources, for the ids that serve the notes. The channel files and +// the sources of other add-ons stay. +func withoutPersonalNotes(sources []contextfiles.Source, ids ...string) []contextfiles.Source { + return slices.DeleteFunc(slices.Clone(sources), func(src contextfiles.Source) bool { + return src.Scope != contextfiles.ScopeChannel && slices.Contains(ids, src.ID) + }) +} + // agentMemory is the choice of notes of a saved agent: one of the agents.Memory // values, or empty for a chat without an agent, an agent that cannot be read // and an agent saved before the choice ("as before"). diff --git a/core/services/studio/memory_paired_internal_test.go b/core/services/studio/memory_paired_internal_test.go new file mode 100644 index 0000000000..84192fdbef --- /dev/null +++ b/core/services/studio/memory_paired_internal_test.go @@ -0,0 +1,266 @@ +package studio + +import ( + "context" + "encoding/json" + "path/filepath" + "strings" + "sync" + "time" + + "github.com/mudler/LocalAI/core/services/studio/agents" + "github.com/mudler/LocalAI/core/services/studio/builtinskills" + "github.com/mudler/LocalAI/core/services/studio/connectors" + "github.com/mudler/LocalAI/core/services/studio/contextfiles" + "github.com/mudler/LocalAI/core/services/studio/files" + "github.com/mudler/LocalAI/core/services/studio/proposals" + "github.com/mudler/LocalAI/core/services/studio/provider" + "github.com/mudler/LocalAI/core/services/studio/session" + "github.com/mudler/LocalAI/core/services/studio/store" + "github.com/mudler/LocalAI/core/services/studio/workspace" + "github.com/mudler/LocalAI/core/services/studio/workspace/blob" + "github.com/mudler/LocalAI/core/services/studio/workspace/gormstore" + "github.com/mudler/LocalAI/pkg/addons/contract" + "github.com/mudler/LocalAI/pkg/sqlitedb" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "gorm.io/gorm" +) + +// The personal notes of the owner in a messaging chat: the user and agent +// notes (and the agent profile) are read into the turn only for a sender who is +// paired with the bot. The notes of the conversation itself are read for every +// sender. Both the built-in provider and an add-on served through the provider +// slot follow the rule. +var _ = Describe("the personal notes in a messaging chat", func() { + const ( + userNote, agentNote, channelNote, profileRule = "lives in Rome", "keeps answers terse", "this group plays chess", "always greet in Latin" + other = "stranger" + ) + now := time.Unix(1_800_000_000, 0) + + // The four kinds of sender. The access chain gives a turn Paired for the + // owner (who pairs with their own bot) and for a paired sender, and not for + // the others, whatever way they were let in. Each row builds its grant from + // a binding, so the flag is the one IsPaired says. + type sender struct { + name string + bind connectors.Binding + id string + opened bool + } + senders := []sender{ + {"the owner", connectors.Binding{ID: "b1", Allow: []string{"owner"}, Paired: []connectors.PairedSender{{SenderID: "owner", At: now}}}, "owner", true}, + {"a paired sender", connectors.Binding{ID: "b1", Allow: []string{"pal"}, Paired: []connectors.PairedSender{{SenderID: "pal", At: now}}}, "pal", true}, + {"an allowlisted sender who is not paired", connectors.Binding{ID: "b1", Allow: []string{"friend"}}, "friend", false}, + {"anyone, in an open channel", connectors.Binding{ID: "b1", AcceptAny: true}, other, false}, + } + // What each choice of the agent reads when the sender is open. + choices := []struct { + name, choice string + user, agent bool + }{ + {"own", agents.MemoryOwn, false, true}, + {"mine", agents.MemoryMine, true, false}, + {"mine_and_own", agents.MemoryMineAndOwn, true, true}, + } + + grantOf := func(s sender) turnGrant { + return turnGrant{Binding: s.bind.ID, Kind: "chatty", Senders: []string{s.id}, Direct: true, Paired: s.bind.IsPaired(s.id, now)} + } + + Describe("the built-in provider", func() { + var ( + ctx context.Context + svc *Service + f *files.Service + ) + home := session.HomePath("alice") + put := func(rel, text string) { + GinkgoHelper() + _, err := f.Store().Put(ctx, f.Actor("alice"), home+"/"+rel, strings.NewReader(text), workspace.PutOptions{Overwrite: true}) + Expect(err).NotTo(HaveOccurred()) + } + BeforeEach(func() { + ctx = context.Background() + dsn := "file:" + filepath.Join(GinkgoT().TempDir(), "ws.db") + "?_busy_timeout=5000&_txlock=immediate" + db, err := sqlitedb.Open(dsn, &gorm.Config{}) + Expect(err).NotTo(HaveOccurred()) + blobs, _ := blob.NewFS(filepath.Join(GinkgoT().TempDir(), "blobs")) + ws, err := gormstore.New(db, blobs, gormstore.Options{Admin: files.AdminPrincipal}) + Expect(err).NotTo(HaveOccurred()) + f = files.New(ws) + Expect(f.SyncBuiltin(ctx, builtinskills.FS())).To(Succeed()) + st, _ := store.NewJSON("") + ag, _ := agents.NewJSON("") + svc, err = New(Deps{Store: st, Provider: provider.New(), StateDir: GinkgoT().TempDir(), Files: f, Agents: ag, AddonTools: &memBroker{}, + Clock: func() time.Time { return now }, + ProposalGate: proposals.GateFunc(func(context.Context, proposals.Request) error { return nil })}) + Expect(err).NotTo(HaveOccurred()) + DeferCleanup(func() { + _ = svc.Close(context.Background()) + if raw, err := db.DB(); err == nil { + _ = raw.Close() + } + }) + Expect(f.EnsureHome(ctx, "alice")).To(Succeed()) + put("Memory/index.md", "- home: "+userNote+" -> home.md\n") + }) + + // turnFor starts a messaging chat of the agent for a sender and builds + // its turn. + turnFor := func(a agents.Agent, s sender) session.TurnTools { + GinkgoHelper() + rec, err := svc.CreateAs(ctx, "alice", CreateInput{Model: "m", AgentID: a.ID}, Origin{Kind: session.OriginConnector, ID: "b1"}) + Expect(err).NotTo(HaveOccurred()) + dir := proposals.ChannelDir(contract.ChangeSource{ID: rec.OriginID, Chat: rec.ID}) + put("Memory/channels/"+dir+"/index.md", "- game: "+channelNote+" -> game.md\n") + svc.access.set(rec.ID, grantOf(s)) + x, err := svc.ChatAccess(ctx, "alice", rec.ID) + Expect(err).NotTo(HaveOccurred()) + tt, err := svc.chatTools(session.WithTurnMessage(ctx, "hello"), x) + Expect(err).NotTo(HaveOccurred()) + return tt + } + savedAgent := func(choice string) agents.Agent { + GinkgoHelper() + a, err := svc.SaveAgentWithFolder(ctx, "alice", agents.Agent{Name: "A-" + choice, Model: "m", Memory: choice}, nil) + Expect(err).NotTo(HaveOccurred()) + put("Memory/agents/"+a.ID+"/index.md", "- style: "+agentNote+" -> style.md\n") + put(contextfiles.ProfilePath(a.ID), profileRule+"\n") + return a + } + contextRead := func(tt session.TurnTools) string { + for _, t := range tt.Tools { + if t.Name == contextfiles.ToolName { + return string(t.Schema) + } + } + return "" + } + + for _, c := range choices { + for _, s := range senders { + It("reads the right notes for "+s.name+" with the choice "+c.name, func() { + tt := turnFor(savedAgent(c.choice), s) + wantUser, wantAgent := s.opened && c.user, s.opened && c.agent + Expect(strings.Contains(tt.Prompt, userNote)).To(Equal(wantUser), "the notes of the owner") + Expect(strings.Contains(tt.Prompt, agentNote)).To(Equal(wantAgent), "the notes of the agent") + Expect(strings.Contains(tt.Prompt, profileRule)).To(Equal(s.opened), "the profile of the agent") + Expect(tt.Prompt).To(ContainSubstring(channelNote), "the notes of the conversation are for every sender") + schema := contextRead(tt) + Expect(schema).NotTo(BeEmpty(), "the conversation notes can be read") + Expect(strings.Contains(schema, `"user"`)).To(Equal(wantUser)) + Expect(strings.Contains(schema, `"agent"`)).To(Equal(wantAgent)) + }) + } + } + + It("reads nothing of the owner when there is no grant (an expired or missing sender)", func() { + a := savedAgent(agents.MemoryMineAndOwn) + rec, err := svc.CreateAs(ctx, "alice", CreateInput{Model: "m", AgentID: a.ID}, Origin{Kind: session.OriginConnector, ID: "b1"}) + Expect(err).NotTo(HaveOccurred()) + x, err := svc.ChatAccess(ctx, "alice", rec.ID) + Expect(err).NotTo(HaveOccurred()) + tt, err := svc.chatTools(session.WithTurnMessage(ctx, "hello"), x) + Expect(err).NotTo(HaveOccurred()) + Expect(tt.Prompt).NotTo(ContainSubstring(userNote)) + Expect(tt.Prompt).NotTo(ContainSubstring(agentNote)) + }) + + It("leaves a chat outside a messaging app unchanged", func() { + a := savedAgent(agents.MemoryMineAndOwn) + rec, err := svc.CreateAs(ctx, "alice", CreateInput{Model: "m", AgentID: a.ID}, Origin{}) + Expect(err).NotTo(HaveOccurred()) + x, err := svc.ChatAccess(ctx, "alice", rec.ID) + Expect(err).NotTo(HaveOccurred()) + tt, err := svc.chatTools(session.WithTurnMessage(ctx, "hello"), x) + Expect(err).NotTo(HaveOccurred()) + Expect(tt.Prompt).To(ContainSubstring(userNote)) + Expect(tt.Prompt).To(ContainSubstring(agentNote)) + }) + }) + + Describe("an add-on served through the provider slot", func() { + var ( + ctx context.Context + svc *Service + host *callHost + stored map[string]string + smu sync.Mutex + ) + BeforeEach(func() { + ctx = context.Background() + stored = map[string]string{} + dsn := "file:" + filepath.Join(GinkgoT().TempDir(), "ws.db") + "?_busy_timeout=5000&_txlock=immediate" + db, err := sqlitedb.Open(dsn, &gorm.Config{}) + Expect(err).NotTo(HaveOccurred()) + blobs, _ := blob.NewFS(filepath.Join(GinkgoT().TempDir(), "blobs")) + ws, err := gormstore.New(db, blobs, gormstore.Options{Admin: files.AdminPrincipal}) + Expect(err).NotTo(HaveOccurred()) + f := files.New(ws) + Expect(f.SyncBuiltin(ctx, builtinskills.FS())).To(Succeed()) + st, _ := store.NewJSON("") + ag, _ := agents.NewJSON("") + host = newCallHost() + host.on("context", func(map[string]any) (string, error) { return result(map[string]any{"text": "- likes tea"}), nil }) + svc, err = New(Deps{Store: st, Provider: provider.New(), StateDir: GinkgoT().TempDir(), Files: f, Agents: ag, AddonTools: host, + Clock: func() time.Time { return now }, + SetUserSetting: func(_ context.Context, user, addon, key, value string) error { + smu.Lock() + defer smu.Unlock() + stored[user+"|"+addon+"/"+key] = value + return nil + }, + UserSetting: func(_ context.Context, user, addon, key string) (string, bool, error) { + smu.Lock() + defer smu.Unlock() + v, ok := stored[user+"|"+addon+"/"+key] + return v, ok, nil + }, + ProposalGate: proposals.GateFunc(func(context.Context, proposals.Request) error { return nil })}) + Expect(err).NotTo(HaveOccurred()) + DeferCleanup(func() { + _ = svc.Close(context.Background()) + if raw, err := db.DB(); err == nil { + _ = raw.Close() + } + }) + Expect(f.EnsureHome(ctx, "alice")).To(Succeed()) + _, err = svc.SelectProvider(ctx, "alice", MemorySlot, callProviderID) + Expect(err).NotTo(HaveOccurred()) + }) + + for _, c := range choices { + for _, s := range senders { + It("asks the add-on for the right scopes for "+s.name+" with the choice "+c.name, func() { + a, err := svc.SaveAgentWithFolder(ctx, "alice", agents.Agent{Name: "A-" + c.choice, Model: "m", Memory: c.choice}, nil) + Expect(err).NotTo(HaveOccurred()) + rec, err := svc.CreateAs(ctx, "alice", CreateInput{Model: "m", AgentID: a.ID}, Origin{Kind: session.OriginConnector, ID: "b1"}) + Expect(err).NotTo(HaveOccurred()) + svc.access.set(rec.ID, grantOf(s)) + x, err := svc.ChatAccess(ctx, "alice", rec.ID) + Expect(err).NotTo(HaveOccurred()) + _, err = svc.chatTools(session.WithTurnMessage(ctx, "hello"), x) + Expect(err).NotTo(HaveOccurred()) + want := []string{"channel"} + if s.opened { + want = nil + if c.user { + want = append(want, "user") + } + if c.agent { + want = append(want, "agent") + } + want = append(want, "channel") + } + Expect(host.count("context")).To(Equal(1)) + raw, _ := json.Marshal(host.sentTo("context")["scopes"]) + var got []string + Expect(json.Unmarshal(raw, &got)).To(Succeed()) + Expect(got).To(ConsistOf(want)) + }) + } + } + }) +}) diff --git a/core/services/studio/memoryprovider.go b/core/services/studio/memoryprovider.go index 5773709129..bf8229d036 100644 --- a/core/services/studio/memoryprovider.go +++ b/core/services/studio/memoryprovider.go @@ -9,6 +9,7 @@ import ( "errors" "io" "path" + "slices" "strings" "github.com/mudler/LocalAI/core/services/studio/agents" @@ -103,6 +104,11 @@ func (s *Service) providerContext(ctx context.Context, rec session.ChatRecord, s scopes = append(scopes, string(contract.ScopeChannel)) } scopes = scopesOfChoice(scopes, choice, rec.AgentID != "") + if !s.personalNotesOpen(rec) { + // Only the notes of this conversation: the provider is not asked for + // the owner's. + scopes = slices.DeleteFunc(scopes, func(sc string) bool { return sc != string(contract.ScopeChannel) }) + } if len(scopes) == 0 { return "", "" } diff --git a/core/services/studio/subagent.go b/core/services/studio/subagent.go index 83e37d6f40..58007b8770 100644 --- a/core/services/studio/subagent.go +++ b/core/services/studio/subagent.go @@ -162,6 +162,11 @@ func (s *Service) chatTools(ctx context.Context, x session.Access) (session.Turn // This chat turned the notes off: the provider's files stay out. extra = slices.DeleteFunc(extra, func(src contextfiles.Source) bool { return src.ID == prov.ID() }) } + if !s.personalNotesOpen(rec) { + // A messaging sender who is not paired reads the notes of the + // conversation only, never the owner's. + extra = withoutPersonalNotes(extra, MemoryAddonID, prov.ID()) + } notes, notesKey = s.contextTurnFor(ctx, rec, tools, extra, !(legacy.Installed && legacy.Running)) tools = append(tools, s.contextTools(rec, tools, extra)...) // A provider that serves the notes answers for this message: its text diff --git a/docs/content/features/chats-connectors.md b/docs/content/features/chats-connectors.md index 1d267a27e1..8cbccf4daa 100644 --- a/docs/content/features/chats-connectors.md +++ b/docs/content/features/chats-connectors.md @@ -859,6 +859,11 @@ a page can turn into a sentence and a link, in the first place the check failed: } ``` +Each kind of sender has one more entry, `notes.read`. It is not a permission that you can tick: +the owner's personal notes (the user and agent notes and the agent profile) are read into a +turn only for a paired sender, so a listed sender who is not paired, an `anyone` sender and a +role sender get `sender_needs_pairing`. The notes of the conversation itself are read for all. + An allowed capability has `allowed: true` and `via` (`paired`, `allowlist` or `role:`) and none of the other fields. `reason` stays the English line for logs. diff --git a/docs/content/features/short-memory.md b/docs/content/features/short-memory.md index 167bd78945..c48eb987d7 100644 --- a/docs/content/features/short-memory.md +++ b/docs/content/features/short-memory.md @@ -190,6 +190,13 @@ memory choice allows that scope, and only after that sender confirms it: a note scope never saves at once. Under **Always confirm** only the paired sender who asked can confirm a note, in a private chat too. An agent whose memory is **None** saves nothing. +Reading follows the same rule. A turn from a messaging chat reads your notes, the agent's +notes and the agent's profile only when the sender is paired with the bot. A listed sender who +is not paired, and anyone let in through a channel, get the notes of that conversation only. +This holds for the built-in notes and for an add-on that serves the notes. If no sender is +known for the turn, it counts as unpaired. The connection page shows it as the line +`notes.read` of each kind of sender. + ## Memory for each agent A saved agent has its own choice of notes. You set it in the agent form, in the