diff --git a/core/services/studio/actorscan_test.go b/core/services/studio/actorscan_test.go index 0b46fc9770..99576dd4ef 100644 --- a/core/services/studio/actorscan_test.go +++ b/core/services/studio/actorscan_test.go @@ -43,6 +43,7 @@ var actorAllowlist = map[string]string{ "skillsvc.go:Service.chatSkillSet": "reads the index of the skills the chat may use", "loadskill.go:Service.loadSkill": "reads only skills the agent grants, by name", "chatfiles.go:Service.ResolveAttachments": "the attachments a person sees in the transcript are a user surface", + "contextturn.go:homeReader.ReadFile": "the host reads the user's notes at paths the loader builds from the chat record, never from the model; the chat's fence is not widened", "service.go:New": "the administrator of the change feed and the trigger watcher", "../../application/studio_files.go:Application.initStudioTextIndex": "the administrator of the text index", } diff --git a/core/services/studio/contextturn.go b/core/services/studio/contextturn.go new file mode 100644 index 0000000000..f5123bde61 --- /dev/null +++ b/core/services/studio/contextturn.go @@ -0,0 +1,99 @@ +package studio + +import ( + "context" + "encoding/json" + "errors" + "io" + + "github.com/mudler/LocalAI/core/services/studio/contextfiles" + "github.com/mudler/LocalAI/core/services/studio/files" + "github.com/mudler/LocalAI/core/services/studio/session" + "github.com/mudler/LocalAI/core/services/studio/workspace" +) + +// homeReader reads the files of a user's home as the user, outside the fence +// of any chat. The loader names the path, never the model, so a scoped chat +// reads its own notes without gaining any access to the files. +type homeReader struct{ f *files.Service } + +func (h homeReader) ReadFile(ctx context.Context, principal, rel string, limit int64) ([]byte, error) { + if !h.f.Available() { + return nil, contextfiles.ErrNotExist + } + abs, err := h.f.Abs(principal, rel) + if err != nil { + return nil, err + } + r, node, err := h.f.Store().Open(ctx, h.f.Actor(principal), abs) + if errors.Is(err, workspace.ErrNotFound) { + return nil, contextfiles.ErrNotExist + } + if err != nil { + return nil, err + } + defer func() { _ = r.Close() }() + if node.Kind != workspace.KindFile { + return nil, contextfiles.ErrNotExist + } + return io.ReadAll(io.LimitReader(r, limit)) +} + +// ContextSources returns the registry of context files. A component that wants +// the host to load a small index of the user's files into the prompt registers +// a source here; nothing else is needed, and the source applies to every chat +// whatever started it. +func (s *Service) ContextSources() *contextfiles.Registry { return s.ctxfiles.Registry() } + +// SetContextWriter plugs in the path that saves notes. Without one the prompt +// does not mention saving. +func (s *Service) SetContextWriter(w contextfiles.Writer) { s.ctxfiles.SetWriter(w) } + +// contextSubject says whose notes a chat reads. Everything comes from the chat +// record: the model names a topic, never an agent or a channel. +func contextSubject(rec session.ChatRecord) contextfiles.Subject { + sub := contextfiles.Subject{Principal: rec.Principal, AgentID: rec.AgentID} + if rec.Origin == session.OriginConnector { + sub.Channel = rec.OriginID + } + return sub +} + +// contextTurn returns the agent profile and the context indexes for the prompt +// of a turn, and a key that changes when either does. Both texts are the same +// at each turn until a file changes, so the transcript does not grow and the +// harness is restarted only for a real change. +func (s *Service) contextTurn(ctx context.Context, rec session.ChatRecord) (text, key string) { + if !s.d.Files.Available() { + return "", "" + } + sub := contextSubject(rec) + profile, index := s.ctxfiles.Profile(ctx, sub), s.ctxfiles.Block(ctx, sub) + return joinBlocks(profile.Text, index.Text), profile.Key + index.Key +} + +// contextTools is memory_read, while a source is registered. +func (s *Service) contextTools(rec session.ChatRecord) []session.Tool { + if !s.d.Files.Available() || len(s.ctxfiles.Registry().Sources()) == 0 { + return nil + } + sub := contextSubject(rec) + return []session.Tool{{ + Name: contextfiles.ToolName, + Description: contextfiles.ToolDescription, + Schema: json.RawMessage(`{"type":"object","properties":{` + + `"topic":{"type":"string","description":"The topic name or file name from a line of the index."},` + + `"scope":{"type":"string","enum":["user","agent","channel"],"description":"Where to look. Omit to search the channel, then the agent, then the user."}},` + + `"required":["topic"]}`), + Run: func(ctx context.Context, raw json.RawMessage) (string, error) { + var args struct { + Topic string `json:"topic"` + Scope string `json:"scope"` + } + if err := json.Unmarshal(raw, &args); err != nil { + return "", err + } + return s.ctxfiles.Detail(ctx, sub, contextfiles.Scope(args.Scope), args.Topic) + }, + }} +} diff --git a/core/services/studio/contextturn_internal_test.go b/core/services/studio/contextturn_internal_test.go new file mode 100644 index 0000000000..7779b3154e --- /dev/null +++ b/core/services/studio/contextturn_internal_test.go @@ -0,0 +1,283 @@ +package studio + +import ( + "context" + "encoding/json" + "path/filepath" + "strings" + + "github.com/mudler/LocalAI/core/services/studio/agents" + "github.com/mudler/LocalAI/core/services/studio/connectors" + "github.com/mudler/LocalAI/core/services/studio/contextfiles" + "github.com/mudler/LocalAI/core/services/studio/files" + "github.com/mudler/LocalAI/core/services/studio/provider" + "github.com/mudler/LocalAI/core/services/studio/session" + "github.com/mudler/LocalAI/core/services/studio/store" + "github.com/mudler/LocalAI/core/services/studio/workspace" + "github.com/mudler/LocalAI/core/services/studio/workspace/blob" + "github.com/mudler/LocalAI/core/services/studio/workspace/gormstore" + "github.com/mudler/LocalAI/pkg/sqlitedb" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "gorm.io/gorm" +) + +type saver struct{} + +func (saver) SaveTool() string { return "propose_change" } + +var _ = Describe("context files in the prompt of a chat", func() { + var ( + ctx context.Context + svc *Service + f *files.Service + cs connectors.Store + ) + home := session.HomePath("alice") + put := func(rel, text string) { + GinkgoHelper() + _, err := f.Store().Put(ctx, f.Actor("alice"), home+"/"+rel, strings.NewReader(text), workspace.PutOptions{Overwrite: true}) + Expect(err).NotTo(HaveOccurred()) + } + register := func() { + reg := svc.ContextSources() + Expect(reg.Register(contextfiles.Source{ID: "notes-user", Scope: contextfiles.ScopeUser, Path: "Memory/index.md"})).To(Succeed()) + Expect(reg.Register(contextfiles.Source{ID: "notes-agent", Scope: contextfiles.ScopeAgent, Path: "Memory/agents/{agent}/index.md"})).To(Succeed()) + Expect(reg.Register(contextfiles.Source{ID: "notes-channel", Scope: contextfiles.ScopeChannel, Path: "Memory/channels/{channel}/index.md"})).To(Succeed()) + } + bind := func(id, kind string) { + GinkgoHelper() + Expect(cs.PutBinding(ctx, connectors.Binding{ID: id, Owner: "alice", Kind: kind, AddonID: kind + "-addon"})).To(Succeed()) + } + turn := func(in CreateInput, origin Origin) (session.TurnTools, session.Access) { + GinkgoHelper() + rec, err := svc.CreateAs(ctx, "alice", in, origin) + Expect(err).NotTo(HaveOccurred()) + x, err := svc.ChatAccess(ctx, "alice", rec.ID) + Expect(err).NotTo(HaveOccurred()) + tt, err := svc.chatTools(ctx, x) + Expect(err).NotTo(HaveOccurred()) + return tt, x + } + tool := func(tt session.TurnTools, name string) *session.Tool { + for i := range tt.Tools { + if tt.Tools[i].Name == name { + return &tt.Tools[i] + } + } + return nil + } + read := func(tt session.TurnTools, args string) (string, error) { + GinkgoHelper() + mr := tool(tt, contextfiles.ToolName) + Expect(mr).NotTo(BeNil(), "the chat has memory_read") + return mr.Run(ctx, json.RawMessage(args)) + } + + BeforeEach(func() { + ctx = context.Background() + dsn := "file:" + filepath.Join(GinkgoT().TempDir(), "ws.db") + "?_busy_timeout=5000&_txlock=immediate" + db, err := sqlitedb.Open(dsn, &gorm.Config{}) + Expect(err).NotTo(HaveOccurred()) + blobs, _ := blob.NewFS(filepath.Join(GinkgoT().TempDir(), "blobs")) + ws, err := gormstore.New(db, blobs, gormstore.Options{Admin: files.AdminPrincipal}) + Expect(err).NotTo(HaveOccurred()) + f = files.New(ws) + st, _ := store.NewJSON("") + ag, _ := agents.NewJSON("") + svc, err = New(Deps{Store: st, Provider: provider.New(), StateDir: GinkgoT().TempDir(), Files: f, Agents: ag}) + Expect(err).NotTo(HaveOccurred()) + DeferCleanup(func() { + _ = svc.Close(context.Background()) + if raw, err := db.DB(); err == nil { + _ = raw.Close() + } + }) + Expect(f.EnsureHome(ctx, "alice")).To(Succeed()) + cs, err = connectors.NewJSON("") + Expect(err).NotTo(HaveOccurred()) + c, err := NewConnectors(svc, ConnectorOptions{Store: cs, Transport: titleTransport{}}) + Expect(err).NotTo(HaveOccurred()) + svc.SetConnectors(c) + }) + + It("adds nothing and no tool while no source is registered", func() { + put("Memory/index.md", "- home: Rome\n") + tt, _ := turn(CreateInput{Model: "m"}, Origin{}) + Expect(tt.Prompt).NotTo(ContainSubstring(contextfiles.Header)) + Expect(tool(tt, contextfiles.ToolName)).To(BeNil()) + }) + + Describe("a plain chat", func() { + BeforeEach(register) + + It("loads the user index under the header and offers memory_read", func() { + put("Memory/index.md", "- home: lives in Rome -> home.md\n") + put("Memory/home.md", "Rome, since 2020.") + tt, _ := turn(CreateInput{Model: "m"}, Origin{}) + Expect(tt.Prompt).To(ContainSubstring(contextfiles.Header)) + Expect(tt.Prompt).To(ContainSubstring("lives in Rome")) + out, err := read(tt, `{"topic":"home"}`) + Expect(err).NotTo(HaveOccurred()) + Expect(out).To(ContainSubstring("Rome, since 2020.")) + Expect(tool(tt, contextfiles.ToolName).Description).To(Equal(contextfiles.ToolDescription)) + }) + + It("keeps the prompt and the key the same from one turn to the next, and the note free of it", func() { + put("Memory/index.md", "- home: lives in Rome\n") + rec, err := svc.Create(ctx, "alice", CreateInput{Model: "m"}) + Expect(err).NotTo(HaveOccurred()) + x, _ := svc.ChatAccess(ctx, "alice", rec.ID) + first, err := svc.chatTools(ctx, x) + Expect(err).NotTo(HaveOccurred()) + second, err := svc.chatTools(ctx, x) + Expect(err).NotTo(HaveOccurred()) + Expect(second.Prompt).To(Equal(first.Prompt)) + Expect(second.RefreshKey).To(Equal(first.RefreshKey)) + Expect(first.Note).NotTo(ContainSubstring("Rome")) + Expect(strings.Count(first.Prompt, contextfiles.Header)).To(Equal(1)) + put("Memory/index.md", "- home: lives in Milan\n") + third, err := svc.chatTools(ctx, x) + Expect(err).NotTo(HaveOccurred()) + Expect(third.RefreshKey).NotTo(Equal(first.RefreshKey), "a changed index restarts the harness") + }) + + It("tells the model how to save once a writer is plugged in", func() { + svc.SetContextWriter(saver{}) + tt, _ := turn(CreateInput{Model: "m"}, Origin{}) + Expect(tt.Prompt).To(ContainSubstring("Save new facts with propose_change.")) + }) + + It("adds a bounded amount even when every file is at its cap", func() { + base, _ := turn(CreateInput{Model: "m"}, Origin{}) + big := strings.Repeat("- t: "+strings.Repeat("w", 90)+"\n", 100) + put("Memory/index.md", big) + put("Memory/agents/x/index.md", big) + tt, _ := turn(CreateInput{Model: "m"}, Origin{}) + Expect(len(tt.Prompt) - len(base.Prompt)).To(BeNumerically("<=", contextfiles.MaxTotalBytes+1024)) + }) + }) + + Describe("the same loader for chats of any origin", func() { + BeforeEach(func() { + register() + put("Memory/index.md", "- home: lives in Rome\n") + bind("b-slack", "slack") + bind("b-tg", "telegram") + put("Memory/channels/"+contextfiles.ChannelKey("b-slack")+"/index.md", "- team: SLACKNOTE\n") + put("Memory/channels/"+contextfiles.ChannelKey("b-tg")+"/index.md", "- team: TGNOTE\n") + put("Memory/channels/"+contextfiles.ChannelKey("b-slack")+"/team.md", "slack detail") + put("Memory/channels/"+contextfiles.ChannelKey("b-tg")+"/team.md", "telegram detail") + }) + + It("gives a Slack-origin chat the user index and its own channel", func() { + tt, _ := turn(CreateInput{Model: "m"}, Origin{Kind: session.OriginConnector, ID: "b-slack"}) + Expect(tt.Prompt).To(ContainSubstring("lives in Rome")) + Expect(tt.Prompt).To(ContainSubstring("SLACKNOTE")) + Expect(tt.Prompt).NotTo(ContainSubstring("TGNOTE")) + out, err := read(tt, `{"topic":"team","scope":"channel"}`) + Expect(err).NotTo(HaveOccurred()) + Expect(out).To(ContainSubstring("slack detail")) + }) + + It("gives a Telegram-origin chat the user index and its own channel", func() { + tt, _ := turn(CreateInput{Model: "m"}, Origin{Kind: session.OriginConnector, ID: "b-tg"}) + Expect(tt.Prompt).To(ContainSubstring("lives in Rome")) + Expect(tt.Prompt).To(ContainSubstring("TGNOTE")) + Expect(tt.Prompt).NotTo(ContainSubstring("SLACKNOTE")) + out, err := read(tt, `{"topic":"team"}`) + Expect(err).NotTo(HaveOccurred()) + Expect(out).To(ContainSubstring("telegram detail")) + Expect(out).NotTo(ContainSubstring("slack")) + }) + + It("gives a plain chat the user index and no channel", func() { + tt, _ := turn(CreateInput{Model: "m"}, Origin{}) + Expect(tt.Prompt).To(ContainSubstring("lives in Rome")) + Expect(tt.Prompt).NotTo(ContainSubstring("SLACKNOTE")) + Expect(tt.Prompt).NotTo(ContainSubstring("TGNOTE")) + _, err := read(tt, `{"topic":"team"}`) + Expect(err).To(MatchError(contextfiles.ErrNoTopic)) + }) + + It("builds the same shape of block for the three", func() { + shape := func(p string) string { + i := strings.Index(p, contextfiles.Header) + Expect(i).To(BeNumerically(">=", 0)) + return p[i : i+len(contextfiles.Header)] + } + a, _ := turn(CreateInput{Model: "m"}, Origin{}) + b, _ := turn(CreateInput{Model: "m"}, Origin{Kind: session.OriginConnector, ID: "b-slack"}) + c, _ := turn(CreateInput{Model: "m"}, Origin{Kind: session.OriginConnector, ID: "b-tg"}) + Expect(shape(a.Prompt)).To(Equal(shape(b.Prompt))) + Expect(shape(b.Prompt)).To(Equal(shape(c.Prompt))) + }) + }) + + Describe("a scoped agent", func() { + var mine, other agents.Agent + BeforeEach(func() { + register() + _, err := f.Store().Mkdir(ctx, f.Actor("alice"), home+"/Clients/Acme") + Expect(err).NotTo(HaveOccurred()) + mine, err = svc.SaveAgentWithFolder(ctx, "alice", agents.Agent{Name: "Mine", Model: "m", Instructions: "INSTR-MINE"}, + &AgentFolderWrite{Path: "/Clients/Acme", Mode: agents.FolderWrite}) + Expect(err).NotTo(HaveOccurred()) + other, err = svc.SaveAgentWithFolder(ctx, "alice", agents.Agent{Name: "Other", Model: "m"}, nil) + Expect(err).NotTo(HaveOccurred()) + put("Memory/index.md", "- home: lives in Rome -> home.md\n") + put("Memory/home.md", "user detail") + put("Memory/agents/"+mine.ID+"/index.md", "- style: MYSTYLE -> style.md\n") + put("Memory/agents/"+mine.ID+"/style.md", "my style detail") + put("Memory/agents/"+other.ID+"/index.md", "- style: OTHERSTYLE\n") + put("Memory/agents/"+other.ID+"/style.md", "other style detail") + put(contextfiles.ProfilePath(mine.ID), "PROFILE-MINE: answer in French.") + }) + + It("reads its own index and the user index, and nothing of another agent", func() { + tt, x := turn(CreateInput{Model: "m", AgentID: mine.ID}, Origin{}) + Expect(x.Scoped()).To(BeTrue()) + Expect(tt.Prompt).To(ContainSubstring("lives in Rome")) + Expect(tt.Prompt).To(ContainSubstring("MYSTYLE")) + Expect(tt.Prompt).NotTo(ContainSubstring("OTHERSTYLE")) + out, err := read(tt, `{"topic":"style","scope":"agent"}`) + Expect(err).NotTo(HaveOccurred()) + Expect(out).To(ContainSubstring("my style detail")) + Expect(out).NotTo(ContainSubstring("other")) + out, err = read(tt, `{"topic":"home"}`) + Expect(err).NotTo(HaveOccurred()) + Expect(out).To(ContainSubstring("user detail")) + }) + + It("gains no access to the files: the fence still refuses the notes and the profile", func() { + _, x := turn(CreateInput{Model: "m", AgentID: mine.ID}, Origin{}) + for _, p := range []string{home + "/Memory/index.md", home + "/" + contextfiles.ProfilePath(mine.ID)} { + _, _, err := f.Store().Open(ctx, x.Actor(), p) + Expect(err).To(HaveOccurred(), p) + _, err = f.Store().Put(ctx, x.Actor(), p, strings.NewReader("x"), workspace.PutOptions{Overwrite: true}) + Expect(err).To(HaveOccurred(), p) + } + }) + + It("puts the profile after the instructions and before the index, and bounds it", func() { + tt, _ := turn(CreateInput{Model: "m", AgentID: mine.ID}, Origin{}) + i, p, h := strings.Index(tt.Prompt, "INSTR-MINE"), strings.Index(tt.Prompt, "PROFILE-MINE"), strings.Index(tt.Prompt, contextfiles.Header) + Expect(i).To(BeNumerically(">=", 0)) + Expect(p).To(BeNumerically(">", i)) + Expect(h).To(BeNumerically(">", p)) + put(contextfiles.ProfilePath(mine.ID), strings.Repeat("rule ", 5000)) + big, _ := turn(CreateInput{Model: "m", AgentID: mine.ID}, Origin{}) + Expect(strings.Count(big.Prompt, "rule ")).To(BeNumerically("<=", contextfiles.MaxProfileBytes/5)) + }) + + It("reads no profile for a chat without an agent", func() { + tt, _ := turn(CreateInput{Model: "m"}, Origin{}) + Expect(tt.Prompt).NotTo(ContainSubstring("PROFILE-MINE")) + }) + + It("leaves the profile out of the prompt when the file is absent", func() { + tt, _ := turn(CreateInput{Model: "m", AgentID: other.ID}, Origin{}) + Expect(tt.Prompt).NotTo(ContainSubstring("Agent profile")) + }) + }) +}) diff --git a/core/services/studio/service.go b/core/services/studio/service.go index 557a60be13..b101ca5909 100644 --- a/core/services/studio/service.go +++ b/core/services/studio/service.go @@ -20,6 +20,7 @@ import ( mcpendpoint "github.com/mudler/LocalAI/core/http/endpoints/mcp" "github.com/mudler/LocalAI/core/services/studio/agents" "github.com/mudler/LocalAI/core/services/studio/builtins" + "github.com/mudler/LocalAI/core/services/studio/contextfiles" "github.com/mudler/LocalAI/core/services/studio/egress" "github.com/mudler/LocalAI/core/services/studio/egress/audit" "github.com/mudler/LocalAI/core/services/studio/files" @@ -308,10 +309,11 @@ type Service struct { admit sync.Mutex // Sub-agents; see subagent.go. - triggers *Triggers // set once by SetTriggers - loops *Loops // set once by SetLoops - jobs *Jobs // set once by SetJobs - connectors *Connectors // set once by SetConnectors + triggers *Triggers // set once by SetTriggers + loops *Loops // set once by SetLoops + jobs *Jobs // set once by SetJobs + connectors *Connectors // set once by SetConnectors + ctxfiles *contextfiles.Loader conversion ConversionSource // set once by SetConversion videos map[string][]string // videos for the next model request of a turn, by conversation @@ -421,6 +423,7 @@ func New(d Deps) (*Service, error) { if err := s.registerToolGroups(); err != nil { return nil, err } + s.ctxfiles = contextfiles.NewLoader(contextfiles.NewRegistry(), homeReader{f: d.Files}, log) s.shared = sharedfolders.New(sharedfolders.Options{Files: d.Files, Consents: d.PageConsents, Audit: d.SharedFolderAudit, Users: s.folderUsers()}) // A chat mounts the shared folders of its owner at /workspace/shared-. mgr, err := session.New(session.Options{Store: d.Store, Workspace: d.Files.Store(), Harness: h, StateDir: d.StateDir, Inputs: s.inputs, ChatTools: s.chatTools, diff --git a/core/services/studio/subagent.go b/core/services/studio/subagent.go index c2048b2beb..3ad88816dd 100644 --- a/core/services/studio/subagent.go +++ b/core/services/studio/subagent.go @@ -130,8 +130,16 @@ func (s *Service) chatTools(ctx context.Context, x session.Access) (session.Turn if !sub { persona, personaKey = s.WellKnown().TurnPrompt(ctx, a.Principal) } - if persona != "" { - prompt = strings.TrimSpace(persona + "\n\n" + prompt) + // The agent's profile and the index of the user's notes follow the persona. + // The first is the user's standing rules for this agent; the second points + // the model at memory_read. + var notes, notesKey string + if !sub { + notes, notesKey = s.contextTurn(ctx, rec) + tools = append(tools, s.contextTools(rec)...) + } + if lead := joinBlocks(persona, notes); lead != "" { + prompt = strings.TrimSpace(lead + "\n\n" + prompt) } if sc != nil { // Granted skills the user lacks come back from skillTurn as a note of @@ -233,7 +241,7 @@ func (s *Service) chatTools(ctx context.Context, x session.Access) (session.Turn // load_skill learns from the final list whether the chat can read a // skill's files and run what a skill needs. tools = s.fitLoadSkill(a, tools) - return session.TurnTools{Tools: tools, Prompt: prompt, Note: note, WorkFolder: work, RefreshKey: key + "\x00" + rec.Settings.SystemPrompt + "\x00" + personaKey}, nil + return session.TurnTools{Tools: tools, Prompt: prompt, Note: note, WorkFolder: work, RefreshKey: key + "\x00" + rec.Settings.SystemPrompt + "\x00" + personaKey + "\x00" + notesKey}, nil } // spawnTool is spawn_agent for the chat of x: a generic sub-agent gets