From cfcef8346c16d44727ac375a3779e27b42ddb4f5 Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Tue, 3 Nov 2026 13:50:08 +0000 Subject: [PATCH] feat(studio): load the context files that add-ons declare A turn now asks the add-on broker which running add-ons declare context files, and loads their index files with the loader that already reads registered sources. The same rules as for add-on skills apply: a stopped add-on, one the chat switched off, one the chat's agent does not grant, and one that cannot work for the user bring nothing. memory_read serves the topics of those sources. The Memory page lists the files of any installed add-on, running or not, because the files belong to the person. The channel notes folder of a chat is now the one propose_change writes for the same conversation: the binding and a hash of the chat. A note the person confirmed is read back by the conversation that proposed it. Before, the loader built the folder from the binding alone, so the two never met. The prompt names the save tool only in a chat that has it, so a chat that switched propose_change off is not told to call it. Assisted-by: Claude:claude-sonnet-5-5 Signed-off-by: Ettore Di Giacinto --- core/application/addon_gateway.go | 35 ++++ core/services/addons/apps/apps.go | 5 + core/services/addons/tools/adapter.go | 32 ++++ .../services/addons/tools/definitions_test.go | 16 ++ core/services/studio/addoncontext.go | 105 +++++++++++ .../studio/contextfiles/contextfiles_test.go | 72 ++++++++ core/services/studio/contextfiles/loader.go | 74 ++++++-- core/services/studio/contextfiles/source.go | 14 +- core/services/studio/contextlist.go | 2 +- core/services/studio/contextturn.go | 38 ++-- .../studio/contextturn_internal_test.go | 171 +++++++++++++++++- core/services/studio/subagent.go | 5 +- scripts/addons/suites.json | 1 + swagger/docs.go | 23 +++ swagger/swagger.json | 23 +++ swagger/swagger.yaml | 23 +++ 16 files changed, 601 insertions(+), 38 deletions(-) create mode 100644 core/services/studio/addoncontext.go diff --git a/core/application/addon_gateway.go b/core/application/addon_gateway.go index f1323591b4..4903537755 100644 --- a/core/application/addon_gateway.go +++ b/core/application/addon_gateway.go @@ -591,6 +591,15 @@ func (l lateStudioAddons) AddonSkills(ctx context.Context, user string) ([]studi return nil, nil } +// AddonContextFiles forwards to the broker when it knows the context files of +// the add-ons. +func (l lateStudioAddons) AddonContextFiles(ctx context.Context, user string) ([]studio.AddonContextSet, error) { + if g, ok := l.a.StudioAddonTools().(studio.AddonContextLister); ok { + return g.AddonContextFiles(ctx, user) + } + return nil, nil +} + // Isolation forwards to the broker when it can say how commands are isolated. func (l lateStudioAddons) Isolation(ctx context.Context, user, addonID string) string { if g, ok := l.a.StudioAddonTools().(studio.CommandIsolation); ok { @@ -805,6 +814,32 @@ func (s *studioAddons) AddonSkills(ctx context.Context, user string) ([]studio.A return out, nil } +// AddonContextFiles lists the add-ons that declare context files and that the +// user can see, less those whose host operations need a feature the user +// lacks: the same rule as their tools and skills. +func (s *studioAddons) AddonContextFiles(ctx context.Context, user string) ([]studio.AddonContextSet, error) { + actor, err := s.actor(user) + if err != nil { + return nil, nil + } + list, err := s.adapter.ContextFiles(ctx, actor) + if errors.Is(err, gateway.ErrForbidden) { + return nil, nil + } + if err != nil { + return nil, err + } + denied := addonsWithoutFeature(s.hostOps, s.has, actor) + out := make([]studio.AddonContextSet, 0, len(list)) + for _, a := range list { + if denied[a.AddonID] { + continue + } + out = append(out, studio.AddonContextSet{AddonID: a.AddonID, Files: a.Files, Running: a.Running}) + } + return out, nil +} + // AddonShadowers lists the add-ons that replace a built-in chat tool and that // the user can see, in any state, less those whose host operations need a // feature the user lacks: the same rule as their tools. diff --git a/core/services/addons/apps/apps.go b/core/services/addons/apps/apps.go index f4eb1af63c..d3a1b80a37 100644 --- a/core/services/addons/apps/apps.go +++ b/core/services/addons/apps/apps.go @@ -124,6 +124,10 @@ type App struct { Skills []contract.SkillDefinition `json:"skills,omitempty"` // SkillShadows lists the built-in skills the add-on replaces (xlsx, say). SkillShadows []string `json:"skillShadows,omitempty"` + // ContextFiles are the index files the add-on asks the host to load into + // the prompt of a chat, from the verified manifest. The host reads them as + // the person; the add-on is not called. + ContextFiles []contract.ContextFile `json:"contextFiles,omitempty"` // UI describes what the shell loads for this app. It is present only for // an enabled addon with a UI whose installed package has a sha256 digest. UI *UIView `json:"ui,omitempty"` @@ -266,6 +270,7 @@ func build(s Snapshot, admin bool) App { a.Resources = resourceTools(m) a.Skills = slices.Clone(m.Capabilities.SkillDefinitions) a.SkillShadows = contract.ShadowedSkills(m.Capabilities.SkillDefinitions) + a.ContextFiles = slices.Clone(m.Capabilities.ContextFiles) if u := m.Capabilities.UI; u != nil { a.HasUI = true if u.Title != nil { diff --git a/core/services/addons/tools/adapter.go b/core/services/addons/tools/adapter.go index 424627ffbc..0ea657a61c 100644 --- a/core/services/addons/tools/adapter.go +++ b/core/services/addons/tools/adapter.go @@ -455,6 +455,38 @@ func (t *ToolAdapter) Skills(ctx context.Context, actor contract.Actor) ([]Addon return out, nil } +// AddonContextFiles is the context files one add-on declares. +type AddonContextFiles struct { + AddonID string + Files []contract.ContextFile + // Running is true when the add-on may serve now: enabled and running, as + // Discover requires for tools. + Running bool +} + +// ContextFiles lists the installed add-ons that declare context files and that +// the actor can see, running or not. The host reads the files itself; a chat +// loads those of a running add-on, and the Memory page lists them for any. +func (t *ToolAdapter) ContextFiles(ctx context.Context, actor contract.Actor) ([]AddonContextFiles, error) { + if err := checkActor(actor); err != nil { + return nil, err + } + list, err := t.backend.Apps(ctx, actor) + if err != nil { + return nil, gateway.ClassifyCheck(err) + } + var out []AddonContextFiles + for _, a := range list.Apps { + if len(a.ContextFiles) == 0 { + continue + } + running := a.Enabled && a.State == "running" && a.Generation > 0 + out = append(out, AddonContextFiles{AddonID: a.ID, Files: slices.Clone(a.ContextFiles), Running: running}) + } + slices.SortFunc(out, func(x, y AddonContextFiles) int { return strings.Compare(x.AddonID, y.AddonID) }) + return out, nil +} + // AddonShadows is one installed add-on that replaces built-in chat tools, as // the actor sees it, running or not. type AddonShadows struct { diff --git a/core/services/addons/tools/definitions_test.go b/core/services/addons/tools/definitions_test.go index ec950778ae..b14606f7e5 100644 --- a/core/services/addons/tools/definitions_test.go +++ b/core/services/addons/tools/definitions_test.go @@ -243,6 +243,22 @@ var _ = Describe("The add-ons that replace a built-in chat tool", func() { }) }) +var _ = Describe("The add-ons that declare context files", func() { + ctx := context.Background() + It("are listed in any state, with their files and whether they run", func() { + files := []contract.ContextFile{{Scope: contract.ScopeUser, Path: "Memory/index.md"}} + running := apps.App{ID: "memory", Generation: 1, Enabled: true, State: "running", ContextFiles: files} + stopped := apps.App{ID: "aaa-notes", Generation: 1, Enabled: true, State: "stopped", ContextFiles: files} + plain := apps.App{ID: "plain", Generation: 1, Enabled: true, State: "running"} + list, err := tools.NewChatToolAdapter(&fake{list: apps.List{Apps: []apps.App{running, stopped, plain}}}).ContextFiles(ctx, member) + Expect(err).NotTo(HaveOccurred()) + Expect(list).To(Equal([]tools.AddonContextFiles{ + {AddonID: "aaa-notes", Files: files, Running: false}, + {AddonID: "memory", Files: files, Running: true}, + })) + }) +}) + var _ = Describe("typed execution status model exclusion", func() { It("keeps declared uiOnly execution status tools out of generic adapters", func() { app := apps.App{ID: "renamed", Generation: 1, Enabled: true, State: "running", Tools: []string{"health"}, ToolDefinitions: []contract.ToolDefinition{{Name: "health", Kind: "read", UIOnly: true, Description: "Observe execution.", InputSchema: json.RawMessage(`{"type":"object","additionalProperties":false}`)}}} diff --git a/core/services/studio/addoncontext.go b/core/services/studio/addoncontext.go new file mode 100644 index 0000000000..b0fb4532d5 --- /dev/null +++ b/core/services/studio/addoncontext.go @@ -0,0 +1,105 @@ +package studio + +import ( + "context" + "slices" + "sort" + + "github.com/mudler/LocalAI/core/services/studio/contextfiles" + "github.com/mudler/LocalAI/pkg/addons/contract" +) + +// Add-on context files. An add-on can declare small index files of the user's +// home (capabilities.contextFiles in its manifest). The host loads them into +// the prompt of a chat and serves their topics with memory_read, with the same +// loader as any other source. The add-on is not called: it only names paths, +// and the host reads them as the user. +// +// A turn decides which add-ons contribute, as it does for their skills: a +// stopped add-on, one the chat switched off and one the chat's agent does not +// grant contribute none. The files stay where they are while the add-on is +// stopped or removed; they belong to the user. +// +// Distributed mode: the files are in the workspace store that every frontend +// reads, and which add-ons run is read per turn from the broker. Nothing is +// cached here. + +// AddonContextSet is the context files one add-on declares, as the chats see +// them. +type AddonContextSet struct { + AddonID string + Files []contract.ContextFile + // Running is true when the add-on can serve now: installed, enabled and + // running. + Running bool +} + +// AddonContextLister is implemented by a broker that knows the context files +// of the add-ons. It lists the installed add-ons that declare some and that +// the user can see, with whether each one runs. It is optional; a broker +// without it gives no add-on context files. +type AddonContextLister interface { + AddonContextFiles(ctx context.Context, user string) ([]AddonContextSet, error) +} + +// sources turns the declaration of an add-on into loader sources. A file the +// loader would refuse is left out. +func (set AddonContextSet) sources() []contextfiles.Source { + var out []contextfiles.Source + for _, f := range set.Files { + src := contextfiles.Source{ID: set.AddonID, Scope: contextfiles.Scope(f.Scope), Path: f.Path, Cap: f.Cap} + if src.Validate() == nil { + out = append(out, src) + } + } + return out +} + +func (s *Service) addonContextSets(ctx context.Context, user string) []AddonContextSet { + if s.d.AddonTools == nil { + return nil + } + lister, ok := s.d.AddonTools.(AddonContextLister) + if !ok { + return nil + } + sets, err := lister.AddonContextFiles(ctx, user) + if err != nil { + s.log.Warn("studio: add-on context files unavailable", "user", user, "error", err) + return nil + } + sort.Slice(sets, func(i, j int) bool { return sets[i].AddonID < sets[j].AddonID }) + return sets +} + +// addonContextSources returns the sources the add-ons bring to a turn: those +// of the add-ons that run, that the chat did not switch off and, for a chat +// with an agent, that the agent grants. +func (s *Service) addonContextSources(ctx context.Context, user string, addonsOff []string, sc *scope) []contextfiles.Source { + if !s.d.Files.Available() { + return nil + } + var out []contextfiles.Source + gate, _ := s.d.AddonTools.(AddonGate) + for _, set := range s.addonContextSets(ctx, user) { + if !set.Running || slices.Contains(addonsOff, set.AddonID) || (sc != nil && !sc.addons[set.AddonID]) { + continue + } + if gate != nil && gate.Unavailable(ctx, user, set.AddonID, true) != "" { + continue + } + out = append(out, set.sources()...) + } + return out +} + +// allAddonContextSources returns the sources of every add-on the user can see, +// running or not: the files are the user's, and the Memory page lists them +// whether or not the add-on runs now. +func (s *Service) allAddonContextSources(ctx context.Context, user string) []contextfiles.Source { + var out []contextfiles.Source + for _, set := range s.addonContextSets(ctx, user) { + out = append(out, set.sources()...) + } + return out +} diff --git a/core/services/studio/contextfiles/contextfiles_test.go b/core/services/studio/contextfiles/contextfiles_test.go index 78cdf0c0ce..a75ce54266 100644 --- a/core/services/studio/contextfiles/contextfiles_test.go +++ b/core/services/studio/contextfiles/contextfiles_test.go @@ -379,3 +379,75 @@ var _ = Describe("the write limits", func() { Expect(over.Current).To(Equal("old")) }) }) + +var _ = Describe("sources a turn brings with it", func() { + var ( + ctx context.Context + rd *memReader + ld *contextfiles.Loader + ) + user := contextfiles.Source{ID: "memory", Scope: contextfiles.ScopeUser, Path: "Memory/index.md"} + agent := contextfiles.Source{ID: "memory", Scope: contextfiles.ScopeAgent, Path: "Memory/agents/{agent}/index.md"} + channel := contextfiles.Source{ID: "memory", Scope: contextfiles.ScopeChannel, Path: "Memory/channels/{channel}/index.md"} + BeforeEach(func() { + ctx = context.Background() + rd = &memReader{files: map[string]string{ + "alice:Memory/index.md": "- home: lives in Rome -> home.md\n", + "alice:Memory/agents/ag1/index.md": "- style: short\n", + "alice:Memory/channels/room-1a2b/index.md": "- team: blue\n", + "alice:Memory/home.md": "Rome since 2020.", + "alice:Memory/channels/room-1a2b/team.md": "Blue team.", + }} + ld = contextfiles.NewLoader(contextfiles.NewRegistry(), rd, nil) + }) + + It("loads them with no registered source and keeps the scope order", func() { + sub := contextfiles.Subject{Principal: "alice", AgentID: "ag1", ChannelDir: "room-1a2b"} + b := ld.Block(ctx, sub, channel, user, agent) + u, a, c := strings.Index(b.Text, "lives in Rome"), strings.Index(b.Text, "short"), strings.Index(b.Text, "blue") + Expect(u).To(BeNumerically(">", 0)) + Expect(a).To(BeNumerically(">", u)) + Expect(c).To(BeNumerically(">", a)) + }) + + It("adds nothing when the turn brings none and none is registered", func() { + Expect(ld.Block(ctx, contextfiles.Subject{Principal: "alice"}).Text).To(BeEmpty()) + }) + + It("reads one file once when a registered source and a brought source name it", func() { + Expect(ld.Registry().Register(contextfiles.Source{ID: "host", Scope: contextfiles.ScopeUser, Path: "Memory/index.md"})).To(Succeed()) + ld.Block(ctx, contextfiles.Subject{Principal: "alice"}, user) + Expect(rd.reads).To(Equal([]string{"alice:Memory/index.md"})) + }) + + It("uses the channel folder the host names and never builds one from the model", func() { + sub := contextfiles.Subject{Principal: "alice", ChannelDir: "room-1a2b"} + Expect(ld.Block(ctx, sub, channel).Text).To(ContainSubstring("blue")) + out, err := ld.Detail(ctx, sub, contextfiles.ScopeChannel, "team", channel) + Expect(err).NotTo(HaveOccurred()) + Expect(out).To(ContainSubstring("Blue team.")) + for _, bad := range []string{"../x", "a/b", "", "a b", strings.Repeat("a", 90)} { + Expect(ld.Block(ctx, contextfiles.Subject{Principal: "alice", ChannelDir: bad}, channel).Text).NotTo(ContainSubstring("channel notes"), bad) + } + }) + + It("reads a topic of a brought source", func() { + out, err := ld.Detail(ctx, contextfiles.Subject{Principal: "alice"}, "", "home", user) + Expect(err).NotTo(HaveOccurred()) + Expect(out).To(ContainSubstring("Rome since 2020.")) + _, err = ld.Detail(ctx, contextfiles.Subject{Principal: "alice"}, "", "home") + Expect(err).To(MatchError(contextfiles.ErrNoTopic)) + }) + + It("names the save tool of the chat, and only when the chat has it", func() { + sub := contextfiles.Subject{Principal: "alice"} + Expect(ld.Block(ctx, sub, user).Text).NotTo(ContainSubstring("Save new facts")) + sub.SaveTool = "propose_change" + Expect(ld.Block(ctx, sub, user).Text).To(ContainSubstring("Save new facts with propose_change.")) + }) + + It("exports the check a source passes before it is used", func() { + Expect(user.Validate()).To(Succeed()) + Expect(contextfiles.Source{ID: "x", Scope: contextfiles.ScopeUser, Path: "../x.md"}.Validate()).To(MatchError(contextfiles.ErrInvalidSource)) + }) +}) diff --git a/core/services/studio/contextfiles/loader.go b/core/services/studio/contextfiles/loader.go index bef219f788..3b328fe255 100644 --- a/core/services/studio/contextfiles/loader.go +++ b/core/services/studio/contextfiles/loader.go @@ -66,6 +66,14 @@ type Subject struct { // (the ID of a connector binding), if any. The loader turns it into a path // segment with ChannelKey. Channel string + // ChannelDir is the folder name of the channel's notes, when the host + // already has it. It wins over Channel. The host builds it from the chat, + // never from the model; the loader accepts a plain name only. + ChannelDir string + // SaveTool is the tool this chat has to save a note, if any. It wins over + // the Writer, which knows the host but not the chat: a chat whose agent + // cannot propose changes must not be told to. + SaveTool string } // Block is text for the prompt of a turn and a key that changes with it. @@ -88,6 +96,8 @@ var ( segment = regexp.MustCompile(`^[A-Za-z0-9][A-Za-z0-9_-]{0,63}$`) slugBad = regexp.MustCompile(`[^a-z0-9]+`) topicName = segment + // channelDirName is the folder name a host gives a channel. + channelDirName = regexp.MustCompile(`^[A-Za-z0-9][A-Za-z0-9_-]{0,79}$`) ) // ChannelKey turns the identifier of a channel into a safe folder name: a short @@ -138,7 +148,10 @@ func (l *Loader) SetWriter(w Writer) { l.mu.Unlock() } -func (l *Loader) saveTool() string { +func (l *Loader) saveTool(sub Subject) string { + if sub.SaveTool != "" { + return sub.SaveTool + } l.mu.RLock() defer l.mu.RUnlock() if l.wr == nil { @@ -159,25 +172,62 @@ func (s Source) resolve(sub Subject) (string, bool) { } return strings.ReplaceAll(s.Path, "{agent}", sub.AgentID), true case ScopeChannel: - if sub.Channel == "" { - return "", false + switch { + case sub.ChannelDir != "": + if !channelDirName.MatchString(sub.ChannelDir) { + return "", false + } + return strings.ReplaceAll(s.Path, "{channel}", sub.ChannelDir), true + case sub.Channel != "": + return strings.ReplaceAll(s.Path, "{channel}", ChannelKey(sub.Channel)), true } - return strings.ReplaceAll(s.Path, "{channel}", ChannelKey(sub.Channel)), true + return "", false } return "", false } -// Block returns the header and the indexes that apply to the chat. It is empty -// when no source is registered. A file that cannot be read is left out: a turn -// never fails for it. The text is the same at each turn until a file changes. -func (l *Loader) Block(ctx context.Context, sub Subject) Block { - sources := l.reg.Sources() +// sourcesFor returns the registered sources and the sources the turn brings, +// widest scope first. A source the registry would refuse is left out, and so is +// a second source for the same file, so a file is read once and the budget is +// not spent twice on it. At most MaxSources are kept. +func (l *Loader) sourcesFor(extra []Source) []Source { + all := l.reg.Sources() + if len(extra) == 0 { + return all + } + seen := make(map[string]bool, len(all)+len(extra)) + for _, s := range all { + seen[string(s.Scope)+" "+s.Path] = true + } + for _, s := range extra { + key := string(s.Scope) + " " + s.Path + if seen[key] || s.validate() != nil || len(all) >= MaxSources { + continue + } + seen[key] = true + all = append(all, s) + } + sortSources(all) + return all +} + +// Sources returns the registered sources and extra, widest scope first. A +// source the registry would refuse, or that repeats a file, is left out. +func (l *Loader) Sources(extra ...Source) []Source { return l.sourcesFor(extra) } + +// Block returns the header and the indexes that apply to the chat: those of +// the registered sources and those of extra, the sources this turn brings. It +// is empty when there is no source. A file that cannot be read is left out: a +// turn never fails for it. The text is the same at each turn until a file +// changes. +func (l *Loader) Block(ctx context.Context, sub Subject, extra ...Source) Block { + sources := l.sourcesFor(extra) if len(sources) == 0 { return Block{} } var sb strings.Builder sb.WriteString(Header) - if tool := l.saveTool(); tool != "" { + if tool := l.saveTool(sub); tool != "" { sb.WriteString(" Save new facts with " + tool + ".") } budget := MaxTotalBytes @@ -280,12 +330,12 @@ func clean(s string) string { // the user. The path comes from the registry and the subject, so the model can // only name a topic: it cannot leave the folder of an index, and it cannot name // another agent or channel. -func (l *Loader) Detail(ctx context.Context, sub Subject, scope Scope, topic string) (string, error) { +func (l *Loader) Detail(ctx context.Context, sub Subject, scope Scope, topic string, extra ...Source) (string, error) { topic = strings.TrimSuffix(strings.TrimSpace(topic), ".md") if !topicName.MatchString(topic) || (scope != "" && !slices.Contains(scopeOrder, scope)) { return "", ErrBadTopic } - sources := l.reg.Sources() + sources := l.sourcesFor(extra) slices.Reverse(sources) for _, src := range sources { if scope != "" && src.Scope != scope { diff --git a/core/services/studio/contextfiles/source.go b/core/services/studio/contextfiles/source.go index 45ab677b9b..525725a3fc 100644 --- a/core/services/studio/contextfiles/source.go +++ b/core/services/studio/contextfiles/source.go @@ -71,6 +71,10 @@ func (s Source) limit() int { var placeholder = regexp.MustCompile(`\{[^{}]*\}`) +// Validate reports why a registry would refuse the source. A caller that builds +// sources for one turn, instead of registering them, checks them with it. +func (s Source) Validate() error { return s.validate() } + func (s Source) validate() error { bad := func(format string, args ...any) error { return fmt.Errorf("%w: %s", ErrInvalidSource, fmt.Sprintf(format, args...)) @@ -147,8 +151,12 @@ func (r *Registry) Sources() []Source { r.mu.RLock() out := slices.Clone(r.sources) r.mu.RUnlock() - slices.SortStableFunc(out, func(a, b Source) int { - return slices.Index(scopeOrder, a.Scope) - slices.Index(scopeOrder, b.Scope) - }) + sortSources(out) return out } + +func sortSources(s []Source) { + slices.SortStableFunc(s, func(a, b Source) int { + return slices.Index(scopeOrder, a.Scope) - slices.Index(scopeOrder, b.Scope) + }) +} diff --git a/core/services/studio/contextlist.go b/core/services/studio/contextlist.go index 93c9d82ab7..25f433d7f8 100644 --- a/core/services/studio/contextlist.go +++ b/core/services/studio/contextlist.go @@ -69,7 +69,7 @@ func (s *Service) ListContextFiles(ctx context.Context, user string) (ContextFil return out, nil } principal := s.Principal(user) - for _, src := range s.ctxfiles.Registry().Sources() { + for _, src := range s.ctxfiles.Sources(s.allAddonContextSources(ctx, user)...) { limit := src.Cap if limit == 0 { limit = contextfiles.MaxIndexBytes diff --git a/core/services/studio/contextturn.go b/core/services/studio/contextturn.go index f5123bde61..9d327aa8a3 100644 --- a/core/services/studio/contextturn.go +++ b/core/services/studio/contextturn.go @@ -8,8 +8,10 @@ import ( "github.com/mudler/LocalAI/core/services/studio/contextfiles" "github.com/mudler/LocalAI/core/services/studio/files" + "github.com/mudler/LocalAI/core/services/studio/proposals" "github.com/mudler/LocalAI/core/services/studio/session" "github.com/mudler/LocalAI/core/services/studio/workspace" + "github.com/mudler/LocalAI/pkg/addons/contract" ) // homeReader reads the files of a user's home as the user, outside the fence @@ -50,11 +52,14 @@ func (s *Service) ContextSources() *contextfiles.Registry { return s.ctxfiles.Re func (s *Service) SetContextWriter(w contextfiles.Writer) { s.ctxfiles.SetWriter(w) } // contextSubject says whose notes a chat reads. Everything comes from the chat -// record: the model names a topic, never an agent or a channel. -func contextSubject(rec session.ChatRecord) contextfiles.Subject { - sub := contextfiles.Subject{Principal: rec.Principal, AgentID: rec.AgentID} +// record: the model names a topic, never an agent or a channel. The channel +// folder is the one propose_change writes for the same chat, so a confirmed +// note is read back by the conversation that proposed it. saveTool is the tool +// the chat has to save a note, or empty. +func contextSubject(rec session.ChatRecord, saveTool string) contextfiles.Subject { + sub := contextfiles.Subject{Principal: rec.Principal, AgentID: rec.AgentID, SaveTool: saveTool} if rec.Origin == session.OriginConnector { - sub.Channel = rec.OriginID + sub.ChannelDir = proposals.ChannelDir(contract.ChangeSource{ID: rec.OriginID, Chat: rec.ID}) } return sub } @@ -63,21 +68,30 @@ func contextSubject(rec session.ChatRecord) contextfiles.Subject { // of a turn, and a key that changes when either does. Both texts are the same // at each turn until a file changes, so the transcript does not grow and the // harness is restarted only for a real change. -func (s *Service) contextTurn(ctx context.Context, rec session.ChatRecord) (text, key string) { +func (s *Service) contextTurn(ctx context.Context, rec session.ChatRecord, tools []session.Tool, extra []contextfiles.Source) (text, key string) { if !s.d.Files.Available() { return "", "" } - sub := contextSubject(rec) - profile, index := s.ctxfiles.Profile(ctx, sub), s.ctxfiles.Block(ctx, sub) + sub := contextSubject(rec, saveToolOf(tools)) + profile, index := s.ctxfiles.Profile(ctx, sub), s.ctxfiles.Block(ctx, sub, extra...) return joinBlocks(profile.Text, index.Text), profile.Key + index.Key } -// contextTools is memory_read, while a source is registered. -func (s *Service) contextTools(rec session.ChatRecord) []session.Tool { - if !s.d.Files.Available() || len(s.ctxfiles.Registry().Sources()) == 0 { +// saveToolOf is the tool a chat has to save a note, or empty. The prompt names +// it only when the chat has it. +func saveToolOf(tools []session.Tool) string { + if hasTool(tools, proposeChangeName) { + return proposeChangeName + } + return "" +} + +// contextTools is memory_read, while a source applies to the chat. +func (s *Service) contextTools(rec session.ChatRecord, tools []session.Tool, extra []contextfiles.Source) []session.Tool { + if !s.d.Files.Available() || len(s.ctxfiles.Sources(extra...)) == 0 { return nil } - sub := contextSubject(rec) + sub := contextSubject(rec, saveToolOf(tools)) return []session.Tool{{ Name: contextfiles.ToolName, Description: contextfiles.ToolDescription, @@ -93,7 +107,7 @@ func (s *Service) contextTools(rec session.ChatRecord) []session.Tool { if err := json.Unmarshal(raw, &args); err != nil { return "", err } - return s.ctxfiles.Detail(ctx, sub, contextfiles.Scope(args.Scope), args.Topic) + return s.ctxfiles.Detail(ctx, sub, contextfiles.Scope(args.Scope), args.Topic, extra...) }, }} } diff --git a/core/services/studio/contextturn_internal_test.go b/core/services/studio/contextturn_internal_test.go index 3d4c209558..8b737b6e26 100644 --- a/core/services/studio/contextturn_internal_test.go +++ b/core/services/studio/contextturn_internal_test.go @@ -5,17 +5,21 @@ import ( "encoding/json" "path/filepath" "strings" + "sync" "github.com/mudler/LocalAI/core/services/studio/agents" + "github.com/mudler/LocalAI/core/services/studio/builtins" "github.com/mudler/LocalAI/core/services/studio/connectors" "github.com/mudler/LocalAI/core/services/studio/contextfiles" "github.com/mudler/LocalAI/core/services/studio/files" + "github.com/mudler/LocalAI/core/services/studio/proposals" "github.com/mudler/LocalAI/core/services/studio/provider" "github.com/mudler/LocalAI/core/services/studio/session" "github.com/mudler/LocalAI/core/services/studio/store" "github.com/mudler/LocalAI/core/services/studio/workspace" "github.com/mudler/LocalAI/core/services/studio/workspace/blob" "github.com/mudler/LocalAI/core/services/studio/workspace/gormstore" + "github.com/mudler/LocalAI/pkg/addons/contract" "github.com/mudler/LocalAI/pkg/sqlitedb" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" @@ -24,6 +28,34 @@ import ( type saver struct{} +// ctxBroker stands in for the add-on broker: the add-ons that declare context +// files. +type ctxBroker struct { + mu sync.Mutex + sets []AddonContextSet + reasons map[string]string +} + +func (b *ctxBroker) Tools(context.Context, string) ([]AddonTool, error) { return nil, nil } +func (b *ctxBroker) Call(context.Context, string, string, string, json.RawMessage) (string, error) { + return "", nil +} +func (b *ctxBroker) AddonContextFiles(context.Context, string) ([]AddonContextSet, error) { + b.mu.Lock() + defer b.mu.Unlock() + return append([]AddonContextSet(nil), b.sets...), nil +} +func (b *ctxBroker) Unavailable(_ context.Context, _, addon string, _ bool) string { + b.mu.Lock() + defer b.mu.Unlock() + return b.reasons[addon] +} +func (b *ctxBroker) set(sets ...AddonContextSet) { + b.mu.Lock() + defer b.mu.Unlock() + b.sets = sets +} + func (saver) SaveTool() string { return "propose_change" } var _ = Describe("context files in the prompt of a chat", func() { @@ -32,6 +64,7 @@ var _ = Describe("context files in the prompt of a chat", func() { svc *Service f *files.Service cs connectors.Store + br *ctxBroker ) home := session.HomePath("alice") put := func(rel, text string) { @@ -85,7 +118,8 @@ var _ = Describe("context files in the prompt of a chat", func() { f = files.New(ws) st, _ := store.NewJSON("") ag, _ := agents.NewJSON("") - svc, err = New(Deps{Store: st, Provider: provider.New(), StateDir: GinkgoT().TempDir(), Files: f, Agents: ag}) + br = &ctxBroker{} + svc, err = New(Deps{Store: st, Provider: provider.New(), StateDir: GinkgoT().TempDir(), Files: f, Agents: ag, AddonTools: br}) Expect(err).NotTo(HaveOccurred()) DeferCleanup(func() { _ = svc.Close(context.Background()) @@ -202,29 +236,44 @@ var _ = Describe("context files in the prompt of a chat", func() { }) Describe("the same loader for chats of any origin", func() { + // The channel folder is the one propose_change writes: the binding and a + // hash of the chat, so a confirmed note is read back by its conversation. + dirOf := func(rec session.ChatRecord) string { + return proposals.ChannelDir(contract.ChangeSource{ID: rec.OriginID, Chat: rec.ID}) + } + // connectorTurn creates a chat of the binding and puts its channel notes + // before the tools are built. + connectorTurn := func(binding, note, detail string) (session.TurnTools, session.ChatRecord) { + GinkgoHelper() + rec, err := svc.CreateAs(ctx, "alice", CreateInput{Model: "m"}, Origin{Kind: session.OriginConnector, ID: binding}) + Expect(err).NotTo(HaveOccurred()) + put("Memory/channels/"+dirOf(rec)+"/index.md", note) + put("Memory/channels/"+dirOf(rec)+"/team.md", detail) + x, err := svc.ChatAccess(ctx, "alice", rec.ID) + Expect(err).NotTo(HaveOccurred()) + tt, err := svc.chatTools(ctx, x) + Expect(err).NotTo(HaveOccurred()) + return tt, rec + } BeforeEach(func() { register() put("Memory/index.md", "- home: lives in Rome\n") bind("b-slack", "slack") bind("b-tg", "telegram") - put("Memory/channels/"+contextfiles.ChannelKey("b-slack")+"/index.md", "- team: SLACKNOTE\n") - put("Memory/channels/"+contextfiles.ChannelKey("b-tg")+"/index.md", "- team: TGNOTE\n") - put("Memory/channels/"+contextfiles.ChannelKey("b-slack")+"/team.md", "slack detail") - put("Memory/channels/"+contextfiles.ChannelKey("b-tg")+"/team.md", "telegram detail") }) It("gives a Slack-origin chat the user index and its own channel", func() { - tt, _ := turn(CreateInput{Model: "m"}, Origin{Kind: session.OriginConnector, ID: "b-slack"}) + tt, _ := connectorTurn("b-slack", "- team: SLACKNOTE\n", "slack detail") Expect(tt.Prompt).To(ContainSubstring("lives in Rome")) Expect(tt.Prompt).To(ContainSubstring("SLACKNOTE")) - Expect(tt.Prompt).NotTo(ContainSubstring("TGNOTE")) out, err := read(tt, `{"topic":"team","scope":"channel"}`) Expect(err).NotTo(HaveOccurred()) Expect(out).To(ContainSubstring("slack detail")) }) It("gives a Telegram-origin chat the user index and its own channel", func() { - tt, _ := turn(CreateInput{Model: "m"}, Origin{Kind: session.OriginConnector, ID: "b-tg"}) + connectorTurn("b-slack", "- team: SLACKNOTE\n", "slack detail") + tt, _ := connectorTurn("b-tg", "- team: TGNOTE\n", "telegram detail") Expect(tt.Prompt).To(ContainSubstring("lives in Rome")) Expect(tt.Prompt).To(ContainSubstring("TGNOTE")) Expect(tt.Prompt).NotTo(ContainSubstring("SLACKNOTE")) @@ -234,6 +283,28 @@ var _ = Describe("context files in the prompt of a chat", func() { Expect(out).NotTo(ContainSubstring("slack")) }) + It("keeps the notes of two conversations of one binding apart", func() { + connectorTurn("b-tg", "- team: FIRSTROOM\n", "first detail") + tt, _ := connectorTurn("b-tg", "- team: SECONDROOM\n", "second detail") + Expect(tt.Prompt).To(ContainSubstring("SECONDROOM")) + Expect(tt.Prompt).NotTo(ContainSubstring("FIRSTROOM")) + }) + + It("reads back the file a confirmed channel proposal writes", func() { + rec, err := svc.CreateAs(ctx, "alice", CreateInput{Model: "m"}, Origin{Kind: session.OriginConnector, ID: "b-tg"}) + Expect(err).NotTo(HaveOccurred()) + p, err := svc.Changes().Propose(ctx, ProposeInput{User: "alice", Source: contract.ChangeSource{Kind: session.OriginConnector, ID: "b-tg", Chat: rec.ID}, + Kind: "memory", Scope: "channel", Target: "index.md", Content: "- pets: the dog is Rex\n", Summary: "remember the dog"}) + Expect(err).NotTo(HaveOccurred()) + _, err = svc.Changes().Confirm(ctx, "alice", p.ID) + Expect(err).NotTo(HaveOccurred()) + x, err := svc.ChatAccess(ctx, "alice", rec.ID) + Expect(err).NotTo(HaveOccurred()) + tt, err := svc.chatTools(ctx, x) + Expect(err).NotTo(HaveOccurred()) + Expect(tt.Prompt).To(ContainSubstring("the dog is Rex")) + }) + It("gives a plain chat the user index and no channel", func() { tt, _ := turn(CreateInput{Model: "m"}, Origin{}) Expect(tt.Prompt).To(ContainSubstring("lives in Rome")) @@ -257,6 +328,90 @@ var _ = Describe("context files in the prompt of a chat", func() { }) }) + Describe("files an add-on declares", func() { + memory := func(running bool) AddonContextSet { + return AddonContextSet{AddonID: "memory", Running: running, Files: []contract.ContextFile{ + {Scope: contract.ScopeUser, Path: "Memory/index.md"}, + {Scope: contract.ScopeAgent, Path: "Memory/agents/{agent}/index.md"}, + {Scope: contract.ScopeChannel, Path: "Memory/channels/{channel}/index.md"}, + }} + } + BeforeEach(func() { + put("Memory/index.md", "- home: lives in Rome -> home.md\n") + put("Memory/home.md", "user detail") + }) + + It("loads the index and serves memory_read while the add-on runs", func() { + br.set(memory(true)) + tt, _ := turn(CreateInput{Model: "m"}, Origin{}) + Expect(tt.Prompt).To(ContainSubstring(contextfiles.Header)) + Expect(tt.Prompt).To(ContainSubstring("lives in Rome")) + out, err := read(tt, `{"topic":"home"}`) + Expect(err).NotTo(HaveOccurred()) + Expect(out).To(ContainSubstring("user detail")) + }) + + It("adds nothing when the add-on is stopped, absent, switched off or unavailable", func() { + for name, setup := range map[string]func() CreateInput{ + "stopped": func() CreateInput { br.set(memory(false)); return CreateInput{Model: "m"} }, + "absent": func() CreateInput { br.set(); return CreateInput{Model: "m"} }, + "switched off": func() CreateInput { + br.set(memory(true)) + return CreateInput{Model: "m", Switches: Switches{AddonsOff: []string{"memory"}}} + }, + "unavailable": func() CreateInput { + br.set(memory(true)) + br.reasons = map[string]string{"memory": "files are off"} + return CreateInput{Model: "m"} + }, + } { + in := setup() + tt, _ := turn(in, Origin{}) + Expect(tt.Prompt).NotTo(ContainSubstring(contextfiles.Header), name) + Expect(tool(tt, contextfiles.ToolName)).To(BeNil(), name) + br.reasons = nil + } + }) + + It("names propose_change only in a chat that has it", func() { + br.set(memory(true)) + tt, _ := turn(CreateInput{Model: "m"}, Origin{}) + Expect(tool(tt, "propose_change")).NotTo(BeNil()) + Expect(tt.Prompt).To(ContainSubstring("Save new facts with propose_change.")) + off, _ := turn(CreateInput{Model: "m", Switches: Switches{BuiltinsOff: []string{string(builtins.GroupChanges)}}}, Origin{}) + Expect(tool(off, "propose_change")).To(BeNil()) + Expect(off.Prompt).To(ContainSubstring(contextfiles.Header)) + Expect(off.Prompt).NotTo(ContainSubstring("Save new facts")) + }) + + It("reaches a saved agent only when the agent grants the add-on", func() { + br.set(memory(true)) + plain, err := svc.SaveAgentWithFolder(ctx, "alice", agents.Agent{Name: "Plain", Model: "m"}, nil) + Expect(err).NotTo(HaveOccurred()) + granted, err := svc.SaveAgentWithFolder(ctx, "alice", agents.Agent{Name: "Granted", Model: "m", Addons: []string{"memory"}}, nil) + Expect(err).NotTo(HaveOccurred()) + put("Memory/agents/"+granted.ID+"/index.md", "- style: GRANTEDSTYLE\n") + no, _ := turn(CreateInput{Model: "m", AgentID: plain.ID}, Origin{}) + Expect(no.Prompt).NotTo(ContainSubstring(contextfiles.Header)) + yes, _ := turn(CreateInput{Model: "m", AgentID: granted.ID}, Origin{}) + Expect(yes.Prompt).To(ContainSubstring("GRANTEDSTYLE")) + }) + + It("lists the files of a stopped add-on on the Memory page", func() { + br.set(memory(false)) + list, err := svc.ListContextFiles(ctx, "alice") + Expect(err).NotTo(HaveOccurred()) + Expect(list.Groups).To(HaveLen(1)) + Expect(list.Groups[0].Source).To(Equal("memory")) + }) + + It("refuses a declaration the loader would refuse", func() { + br.set(AddonContextSet{AddonID: "memory", Running: true, Files: []contract.ContextFile{{Scope: contract.ScopeUser, Path: "../secrets.md"}}}) + tt, _ := turn(CreateInput{Model: "m"}, Origin{}) + Expect(tt.Prompt).NotTo(ContainSubstring(contextfiles.Header)) + }) + }) + Describe("a scoped agent", func() { var mine, other agents.Agent BeforeEach(func() { diff --git a/core/services/studio/subagent.go b/core/services/studio/subagent.go index f03a2b9e0f..c002f7097b 100644 --- a/core/services/studio/subagent.go +++ b/core/services/studio/subagent.go @@ -138,8 +138,9 @@ func (s *Service) chatTools(ctx context.Context, x session.Access) (session.Turn // the model at memory_read. var notes, notesKey string if !sub { - notes, notesKey = s.contextTurn(ctx, rec) - tools = append(tools, s.contextTools(rec)...) + extra := s.addonContextSources(ctx, a.Principal, rec.AddonsOff, sc) + notes, notesKey = s.contextTurn(ctx, rec, tools, extra) + tools = append(tools, s.contextTools(rec, tools, extra)...) } if lead := joinBlocks(persona, notes); lead != "" { prompt = strings.TrimSpace(lead + "\n\n" + prompt) diff --git a/scripts/addons/suites.json b/scripts/addons/suites.json index 00cf15b780..cdb0c48c53 100644 --- a/scripts/addons/suites.json +++ b/scripts/addons/suites.json @@ -3500,6 +3500,7 @@ "go: is unavailable when the owner cannot act, and never falls back to a read or a proposal", "go: keeps the argument rules: an identity argument is refused before anything runs", "go: are listed in any state, with the names they replace and whether they run", + "go: are listed in any state, with their files and whether they run", "go: keeps its namespaced name and carries the built-in it shadows, only for a replaceable name", "go: names the replacement string for legacy arguments at the chat boundary without sending them", "go: synthesizes exactly one strict host read, without enumerating resources at discovery", diff --git a/swagger/docs.go b/swagger/docs.go index e5840042dc..5a860ef96d 100644 --- a/swagger/docs.go +++ b/swagger/docs.go @@ -14902,6 +14902,13 @@ const docTemplate = `{ } ] }, + "contextFiles": { + "description": "ContextFiles are the index files the add-on asks the host to load into\nthe prompt of a chat, from the verified manifest. The host reads them as\nthe person; the add-on is not called.", + "type": "array", + "items": { + "$ref": "#/definitions/contract.ContextFile" + } + }, "description": { "type": "string", "example": "Keep notes." @@ -15621,6 +15628,22 @@ const docTemplate = `{ } } }, + "contract.ContextFile": { + "type": "object", + "properties": { + "cap": { + "description": "Cap is the most bytes of the index the prompt carries. Zero means\nMaxContextFileBytes.", + "type": "integer" + }, + "path": { + "description": "Path is relative to the user's home, under ContextFilesRoot, and ends\nin .md.", + "type": "string" + }, + "scope": { + "$ref": "#/definitions/contract.ChangeScope" + } + } + }, "contract.ExecutionDescriptor": { "type": "object", "properties": { diff --git a/swagger/swagger.json b/swagger/swagger.json index a712454ca9..0235805603 100644 --- a/swagger/swagger.json +++ b/swagger/swagger.json @@ -14899,6 +14899,13 @@ } ] }, + "contextFiles": { + "description": "ContextFiles are the index files the add-on asks the host to load into\nthe prompt of a chat, from the verified manifest. The host reads them as\nthe person; the add-on is not called.", + "type": "array", + "items": { + "$ref": "#/definitions/contract.ContextFile" + } + }, "description": { "type": "string", "example": "Keep notes." @@ -15618,6 +15625,22 @@ } } }, + "contract.ContextFile": { + "type": "object", + "properties": { + "cap": { + "description": "Cap is the most bytes of the index the prompt carries. Zero means\nMaxContextFileBytes.", + "type": "integer" + }, + "path": { + "description": "Path is relative to the user's home, under ContextFilesRoot, and ends\nin .md.", + "type": "string" + }, + "scope": { + "$ref": "#/definitions/contract.ChangeScope" + } + } + }, "contract.ExecutionDescriptor": { "type": "object", "properties": { diff --git a/swagger/swagger.yaml b/swagger/swagger.yaml index 893251e8bc..30fd35644d 100644 --- a/swagger/swagger.yaml +++ b/swagger/swagger.yaml @@ -179,6 +179,14 @@ definitions: allOf: - $ref: '#/definitions/apps.AdminView' description: Admin is present only for administrators. + contextFiles: + description: |- + ContextFiles are the index files the add-on asks the host to load into + the prompt of a chat, from the verified manifest. The host reads them as + the person; the add-on is not called. + items: + $ref: '#/definitions/contract.ContextFile' + type: array description: example: Keep notes. type: string @@ -768,6 +776,21 @@ definitions: thread: type: string type: object + contract.ContextFile: + properties: + cap: + description: |- + Cap is the most bytes of the index the prompt carries. Zero means + MaxContextFileBytes. + type: integer + path: + description: |- + Path is relative to the user's home, under ContextFilesRoot, and ends + in .md. + type: string + scope: + $ref: '#/definitions/contract.ChangeScope' + type: object contract.ExecutionDescriptor: properties: class: