feat(studio): load context files and memory_read in every chat

Each turn the chat prompt gets the agent profile and the indexes of the
registered context sources, after the persona and before the skill index.
The scope of a source is derived from the chat record: the saved agent, and
the binding when a connector started the chat. The text is the same at each
turn, so the transcript does not grow, and it feeds the refresh key so the
harness restarts only when a file changes.

memory_read reads a topic next to an index as the user, outside the chat
fence. A scoped agent reads its own notes and the user notes and gains no
access to the files. The service exposes ContextSources and
SetContextWriter so other parts of the host can register a source or plug in
the save path.

Assisted-by: Claude Code:claude-sonnet-5-5
Signed-off-by: Ettore Di Giacinto <mudler@localai.io>
This commit is contained in:
Ettore Di Giacinto committed 2026-11-03 09:28:57 +00:00
1 parent b64b35b5a4
commit 9ee1407989
5 files changed
+401 -7

No files matched your search

+1
View File
@@ -43,6 +43,7 @@ var actorAllowlist = map[string]string{
"skillsvc.go:Service.chatSkillSet": "reads the index of the skills the chat may use",
"loadskill.go:Service.loadSkill": "reads only skills the agent grants, by name",
"chatfiles.go:Service.ResolveAttachments": "the attachments a person sees in the transcript are a user surface",
"contextturn.go:homeReader.ReadFile": "the host reads the user's notes at paths the loader builds from the chat record, never from the model; the chat's fence is not widened",
"service.go:New": "the administrator of the change feed and the trigger watcher",
"../../application/studio_files.go:Application.initStudioTextIndex": "the administrator of the text index",
}
+99
View File
@@ -0,0 +1,99 @@
package studio
import (
"context"
"encoding/json"
"errors"
"io"
"github.com/mudler/LocalAI/core/services/studio/contextfiles"
"github.com/mudler/LocalAI/core/services/studio/files"
"github.com/mudler/LocalAI/core/services/studio/session"
"github.com/mudler/LocalAI/core/services/studio/workspace"
)
// homeReader reads the files of a user's home as the user, outside the fence
// of any chat. The loader names the path, never the model, so a scoped chat
// reads its own notes without gaining any access to the files.
type homeReader struct{ f *files.Service }
func (h homeReader) ReadFile(ctx context.Context, principal, rel string, limit int64) ([]byte, error) {
if !h.f.Available() {
return nil, contextfiles.ErrNotExist
}
abs, err := h.f.Abs(principal, rel)
if err != nil {
return nil, err
}
r, node, err := h.f.Store().Open(ctx, h.f.Actor(principal), abs)
if errors.Is(err, workspace.ErrNotFound) {
return nil, contextfiles.ErrNotExist
}
if err != nil {
return nil, err
}
defer func() { _ = r.Close() }()
if node.Kind != workspace.KindFile {
return nil, contextfiles.ErrNotExist
}
return io.ReadAll(io.LimitReader(r, limit))
}
// ContextSources returns the registry of context files. A component that wants
// the host to load a small index of the user's files into the prompt registers
// a source here; nothing else is needed, and the source applies to every chat
// whatever started it.
func (s *Service) ContextSources() *contextfiles.Registry { return s.ctxfiles.Registry() }
// SetContextWriter plugs in the path that saves notes. Without one the prompt
// does not mention saving.
func (s *Service) SetContextWriter(w contextfiles.Writer) { s.ctxfiles.SetWriter(w) }
// contextSubject says whose notes a chat reads. Everything comes from the chat
// record: the model names a topic, never an agent or a channel.
func contextSubject(rec session.ChatRecord) contextfiles.Subject {
sub := contextfiles.Subject{Principal: rec.Principal, AgentID: rec.AgentID}
if rec.Origin == session.OriginConnector {
sub.Channel = rec.OriginID
}
return sub
}
// contextTurn returns the agent profile and the context indexes for the prompt
// of a turn, and a key that changes when either does. Both texts are the same
// at each turn until a file changes, so the transcript does not grow and the
// harness is restarted only for a real change.
func (s *Service) contextTurn(ctx context.Context, rec session.ChatRecord) (text, key string) {
if !s.d.Files.Available() {
return "", ""
}
sub := contextSubject(rec)
profile, index := s.ctxfiles.Profile(ctx, sub), s.ctxfiles.Block(ctx, sub)
return joinBlocks(profile.Text, index.Text), profile.Key + index.Key
}
// contextTools is memory_read, while a source is registered.
func (s *Service) contextTools(rec session.ChatRecord) []session.Tool {
if !s.d.Files.Available() || len(s.ctxfiles.Registry().Sources()) == 0 {
return nil
}
sub := contextSubject(rec)
return []session.Tool{{
Name: contextfiles.ToolName,
Description: contextfiles.ToolDescription,
Schema: json.RawMessage(`{"type":"object","properties":{` +
`"topic":{"type":"string","description":"The topic name or file name from a line of the index."},` +
`"scope":{"type":"string","enum":["user","agent","channel"],"description":"Where to look. Omit to search the channel, then the agent, then the user."}},` +
`"required":["topic"]}`),
Run: func(ctx context.Context, raw json.RawMessage) (string, error) {
var args struct {
Topic string `json:"topic"`
Scope string `json:"scope"`
}
if err := json.Unmarshal(raw, &args); err != nil {
return "", err
}
return s.ctxfiles.Detail(ctx, sub, contextfiles.Scope(args.Scope), args.Topic)
},
}}
}
@@ -0,0 +1,283 @@
package studio
import (
"context"
"encoding/json"
"path/filepath"
"strings"
"github.com/mudler/LocalAI/core/services/studio/agents"
"github.com/mudler/LocalAI/core/services/studio/connectors"
"github.com/mudler/LocalAI/core/services/studio/contextfiles"
"github.com/mudler/LocalAI/core/services/studio/files"
"github.com/mudler/LocalAI/core/services/studio/provider"
"github.com/mudler/LocalAI/core/services/studio/session"
"github.com/mudler/LocalAI/core/services/studio/store"
"github.com/mudler/LocalAI/core/services/studio/workspace"
"github.com/mudler/LocalAI/core/services/studio/workspace/blob"
"github.com/mudler/LocalAI/core/services/studio/workspace/gormstore"
"github.com/mudler/LocalAI/pkg/sqlitedb"
. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
"gorm.io/gorm"
)
type saver struct{}
func (saver) SaveTool() string { return "propose_change" }
var _ = Describe("context files in the prompt of a chat", func() {
var (
ctx context.Context
svc *Service
f *files.Service
cs connectors.Store
)
home := session.HomePath("alice")
put := func(rel, text string) {
GinkgoHelper()
_, err := f.Store().Put(ctx, f.Actor("alice"), home+"/"+rel, strings.NewReader(text), workspace.PutOptions{Overwrite: true})
Expect(err).NotTo(HaveOccurred())
}
register := func() {
reg := svc.ContextSources()
Expect(reg.Register(contextfiles.Source{ID: "notes-user", Scope: contextfiles.ScopeUser, Path: "Memory/index.md"})).To(Succeed())
Expect(reg.Register(contextfiles.Source{ID: "notes-agent", Scope: contextfiles.ScopeAgent, Path: "Memory/agents/{agent}/index.md"})).To(Succeed())
Expect(reg.Register(contextfiles.Source{ID: "notes-channel", Scope: contextfiles.ScopeChannel, Path: "Memory/channels/{channel}/index.md"})).To(Succeed())
}
bind := func(id, kind string) {
GinkgoHelper()
Expect(cs.PutBinding(ctx, connectors.Binding{ID: id, Owner: "alice", Kind: kind, AddonID: kind + "-addon"})).To(Succeed())
}
turn := func(in CreateInput, origin Origin) (session.TurnTools, session.Access) {
GinkgoHelper()
rec, err := svc.CreateAs(ctx, "alice", in, origin)
Expect(err).NotTo(HaveOccurred())
x, err := svc.ChatAccess(ctx, "alice", rec.ID)
Expect(err).NotTo(HaveOccurred())
tt, err := svc.chatTools(ctx, x)
Expect(err).NotTo(HaveOccurred())
return tt, x
}
tool := func(tt session.TurnTools, name string) *session.Tool {
for i := range tt.Tools {
if tt.Tools[i].Name == name {
return &tt.Tools[i]
}
}
return nil
}
read := func(tt session.TurnTools, args string) (string, error) {
GinkgoHelper()
mr := tool(tt, contextfiles.ToolName)
Expect(mr).NotTo(BeNil(), "the chat has memory_read")
return mr.Run(ctx, json.RawMessage(args))
}
BeforeEach(func() {
ctx = context.Background()
dsn := "file:" + filepath.Join(GinkgoT().TempDir(), "ws.db") + "?_busy_timeout=5000&_txlock=immediate"
db, err := sqlitedb.Open(dsn, &gorm.Config{})
Expect(err).NotTo(HaveOccurred())
blobs, _ := blob.NewFS(filepath.Join(GinkgoT().TempDir(), "blobs"))
ws, err := gormstore.New(db, blobs, gormstore.Options{Admin: files.AdminPrincipal})
Expect(err).NotTo(HaveOccurred())
f = files.New(ws)
st, _ := store.NewJSON("")
ag, _ := agents.NewJSON("")
svc, err = New(Deps{Store: st, Provider: provider.New(), StateDir: GinkgoT().TempDir(), Files: f, Agents: ag})
Expect(err).NotTo(HaveOccurred())
DeferCleanup(func() {
_ = svc.Close(context.Background())
if raw, err := db.DB(); err == nil {
_ = raw.Close()
}
})
Expect(f.EnsureHome(ctx, "alice")).To(Succeed())
cs, err = connectors.NewJSON("")
Expect(err).NotTo(HaveOccurred())
c, err := NewConnectors(svc, ConnectorOptions{Store: cs, Transport: titleTransport{}})
Expect(err).NotTo(HaveOccurred())
svc.SetConnectors(c)
})
It("adds nothing and no tool while no source is registered", func() {
put("Memory/index.md", "- home: Rome\n")
tt, _ := turn(CreateInput{Model: "m"}, Origin{})
Expect(tt.Prompt).NotTo(ContainSubstring(contextfiles.Header))
Expect(tool(tt, contextfiles.ToolName)).To(BeNil())
})
Describe("a plain chat", func() {
BeforeEach(register)
It("loads the user index under the header and offers memory_read", func() {
put("Memory/index.md", "- home: lives in Rome -> home.md\n")
put("Memory/home.md", "Rome, since 2020.")
tt, _ := turn(CreateInput{Model: "m"}, Origin{})
Expect(tt.Prompt).To(ContainSubstring(contextfiles.Header))
Expect(tt.Prompt).To(ContainSubstring("lives in Rome"))
out, err := read(tt, `{"topic":"home"}`)
Expect(err).NotTo(HaveOccurred())
Expect(out).To(ContainSubstring("Rome, since 2020."))
Expect(tool(tt, contextfiles.ToolName).Description).To(Equal(contextfiles.ToolDescription))
})
It("keeps the prompt and the key the same from one turn to the next, and the note free of it", func() {
put("Memory/index.md", "- home: lives in Rome\n")
rec, err := svc.Create(ctx, "alice", CreateInput{Model: "m"})
Expect(err).NotTo(HaveOccurred())
x, _ := svc.ChatAccess(ctx, "alice", rec.ID)
first, err := svc.chatTools(ctx, x)
Expect(err).NotTo(HaveOccurred())
second, err := svc.chatTools(ctx, x)
Expect(err).NotTo(HaveOccurred())
Expect(second.Prompt).To(Equal(first.Prompt))
Expect(second.RefreshKey).To(Equal(first.RefreshKey))
Expect(first.Note).NotTo(ContainSubstring("Rome"))
Expect(strings.Count(first.Prompt, contextfiles.Header)).To(Equal(1))
put("Memory/index.md", "- home: lives in Milan\n")
third, err := svc.chatTools(ctx, x)
Expect(err).NotTo(HaveOccurred())
Expect(third.RefreshKey).NotTo(Equal(first.RefreshKey), "a changed index restarts the harness")
})
It("tells the model how to save once a writer is plugged in", func() {
svc.SetContextWriter(saver{})
tt, _ := turn(CreateInput{Model: "m"}, Origin{})
Expect(tt.Prompt).To(ContainSubstring("Save new facts with propose_change."))
})
It("adds a bounded amount even when every file is at its cap", func() {
base, _ := turn(CreateInput{Model: "m"}, Origin{})
big := strings.Repeat("- t: "+strings.Repeat("w", 90)+"\n", 100)
put("Memory/index.md", big)
put("Memory/agents/x/index.md", big)
tt, _ := turn(CreateInput{Model: "m"}, Origin{})
Expect(len(tt.Prompt) - len(base.Prompt)).To(BeNumerically("<=", contextfiles.MaxTotalBytes+1024))
})
})
Describe("the same loader for chats of any origin", func() {
BeforeEach(func() {
register()
put("Memory/index.md", "- home: lives in Rome\n")
bind("b-slack", "slack")
bind("b-tg", "telegram")
put("Memory/channels/"+contextfiles.ChannelKey("b-slack")+"/index.md", "- team: SLACKNOTE\n")
put("Memory/channels/"+contextfiles.ChannelKey("b-tg")+"/index.md", "- team: TGNOTE\n")
put("Memory/channels/"+contextfiles.ChannelKey("b-slack")+"/team.md", "slack detail")
put("Memory/channels/"+contextfiles.ChannelKey("b-tg")+"/team.md", "telegram detail")
})
It("gives a Slack-origin chat the user index and its own channel", func() {
tt, _ := turn(CreateInput{Model: "m"}, Origin{Kind: session.OriginConnector, ID: "b-slack"})
Expect(tt.Prompt).To(ContainSubstring("lives in Rome"))
Expect(tt.Prompt).To(ContainSubstring("SLACKNOTE"))
Expect(tt.Prompt).NotTo(ContainSubstring("TGNOTE"))
out, err := read(tt, `{"topic":"team","scope":"channel"}`)
Expect(err).NotTo(HaveOccurred())
Expect(out).To(ContainSubstring("slack detail"))
})
It("gives a Telegram-origin chat the user index and its own channel", func() {
tt, _ := turn(CreateInput{Model: "m"}, Origin{Kind: session.OriginConnector, ID: "b-tg"})
Expect(tt.Prompt).To(ContainSubstring("lives in Rome"))
Expect(tt.Prompt).To(ContainSubstring("TGNOTE"))
Expect(tt.Prompt).NotTo(ContainSubstring("SLACKNOTE"))
out, err := read(tt, `{"topic":"team"}`)
Expect(err).NotTo(HaveOccurred())
Expect(out).To(ContainSubstring("telegram detail"))
Expect(out).NotTo(ContainSubstring("slack"))
})
It("gives a plain chat the user index and no channel", func() {
tt, _ := turn(CreateInput{Model: "m"}, Origin{})
Expect(tt.Prompt).To(ContainSubstring("lives in Rome"))
Expect(tt.Prompt).NotTo(ContainSubstring("SLACKNOTE"))
Expect(tt.Prompt).NotTo(ContainSubstring("TGNOTE"))
_, err := read(tt, `{"topic":"team"}`)
Expect(err).To(MatchError(contextfiles.ErrNoTopic))
})
It("builds the same shape of block for the three", func() {
shape := func(p string) string {
i := strings.Index(p, contextfiles.Header)
Expect(i).To(BeNumerically(">=", 0))
return p[i : i+len(contextfiles.Header)]
}
a, _ := turn(CreateInput{Model: "m"}, Origin{})
b, _ := turn(CreateInput{Model: "m"}, Origin{Kind: session.OriginConnector, ID: "b-slack"})
c, _ := turn(CreateInput{Model: "m"}, Origin{Kind: session.OriginConnector, ID: "b-tg"})
Expect(shape(a.Prompt)).To(Equal(shape(b.Prompt)))
Expect(shape(b.Prompt)).To(Equal(shape(c.Prompt)))
})
})
Describe("a scoped agent", func() {
var mine, other agents.Agent
BeforeEach(func() {
register()
_, err := f.Store().Mkdir(ctx, f.Actor("alice"), home+"/Clients/Acme")
Expect(err).NotTo(HaveOccurred())
mine, err = svc.SaveAgentWithFolder(ctx, "alice", agents.Agent{Name: "Mine", Model: "m", Instructions: "INSTR-MINE"},
&AgentFolderWrite{Path: "/Clients/Acme", Mode: agents.FolderWrite})
Expect(err).NotTo(HaveOccurred())
other, err = svc.SaveAgentWithFolder(ctx, "alice", agents.Agent{Name: "Other", Model: "m"}, nil)
Expect(err).NotTo(HaveOccurred())
put("Memory/index.md", "- home: lives in Rome -> home.md\n")
put("Memory/home.md", "user detail")
put("Memory/agents/"+mine.ID+"/index.md", "- style: MYSTYLE -> style.md\n")
put("Memory/agents/"+mine.ID+"/style.md", "my style detail")
put("Memory/agents/"+other.ID+"/index.md", "- style: OTHERSTYLE\n")
put("Memory/agents/"+other.ID+"/style.md", "other style detail")
put(contextfiles.ProfilePath(mine.ID), "PROFILE-MINE: answer in French.")
})
It("reads its own index and the user index, and nothing of another agent", func() {
tt, x := turn(CreateInput{Model: "m", AgentID: mine.ID}, Origin{})
Expect(x.Scoped()).To(BeTrue())
Expect(tt.Prompt).To(ContainSubstring("lives in Rome"))
Expect(tt.Prompt).To(ContainSubstring("MYSTYLE"))
Expect(tt.Prompt).NotTo(ContainSubstring("OTHERSTYLE"))
out, err := read(tt, `{"topic":"style","scope":"agent"}`)
Expect(err).NotTo(HaveOccurred())
Expect(out).To(ContainSubstring("my style detail"))
Expect(out).NotTo(ContainSubstring("other"))
out, err = read(tt, `{"topic":"home"}`)
Expect(err).NotTo(HaveOccurred())
Expect(out).To(ContainSubstring("user detail"))
})
It("gains no access to the files: the fence still refuses the notes and the profile", func() {
_, x := turn(CreateInput{Model: "m", AgentID: mine.ID}, Origin{})
for _, p := range []string{home + "/Memory/index.md", home + "/" + contextfiles.ProfilePath(mine.ID)} {
_, _, err := f.Store().Open(ctx, x.Actor(), p)
Expect(err).To(HaveOccurred(), p)
_, err = f.Store().Put(ctx, x.Actor(), p, strings.NewReader("x"), workspace.PutOptions{Overwrite: true})
Expect(err).To(HaveOccurred(), p)
}
})
It("puts the profile after the instructions and before the index, and bounds it", func() {
tt, _ := turn(CreateInput{Model: "m", AgentID: mine.ID}, Origin{})
i, p, h := strings.Index(tt.Prompt, "INSTR-MINE"), strings.Index(tt.Prompt, "PROFILE-MINE"), strings.Index(tt.Prompt, contextfiles.Header)
Expect(i).To(BeNumerically(">=", 0))
Expect(p).To(BeNumerically(">", i))
Expect(h).To(BeNumerically(">", p))
put(contextfiles.ProfilePath(mine.ID), strings.Repeat("rule ", 5000))
big, _ := turn(CreateInput{Model: "m", AgentID: mine.ID}, Origin{})
Expect(strings.Count(big.Prompt, "rule ")).To(BeNumerically("<=", contextfiles.MaxProfileBytes/5))
})
It("reads no profile for a chat without an agent", func() {
tt, _ := turn(CreateInput{Model: "m"}, Origin{})
Expect(tt.Prompt).NotTo(ContainSubstring("PROFILE-MINE"))
})
It("leaves the profile out of the prompt when the file is absent", func() {
tt, _ := turn(CreateInput{Model: "m", AgentID: other.ID}, Origin{})
Expect(tt.Prompt).NotTo(ContainSubstring("Agent profile"))
})
})
})
+7 -4
View File
@@ -20,6 +20,7 @@ import (
mcpendpoint "github.com/mudler/LocalAI/core/http/endpoints/mcp"
"github.com/mudler/LocalAI/core/services/studio/agents"
"github.com/mudler/LocalAI/core/services/studio/builtins"
"github.com/mudler/LocalAI/core/services/studio/contextfiles"
"github.com/mudler/LocalAI/core/services/studio/egress"
"github.com/mudler/LocalAI/core/services/studio/egress/audit"
"github.com/mudler/LocalAI/core/services/studio/files"
@@ -308,10 +309,11 @@ type Service struct {
admit sync.Mutex
// Sub-agents; see subagent.go.
triggers *Triggers // set once by SetTriggers
loops *Loops // set once by SetLoops
jobs *Jobs // set once by SetJobs
connectors *Connectors // set once by SetConnectors
triggers *Triggers // set once by SetTriggers
loops *Loops // set once by SetLoops
jobs *Jobs // set once by SetJobs
connectors *Connectors // set once by SetConnectors
ctxfiles *contextfiles.Loader
conversion ConversionSource // set once by SetConversion
videos map[string][]string // videos for the next model request of a turn, by conversation
@@ -421,6 +423,7 @@ func New(d Deps) (*Service, error) {
if err := s.registerToolGroups(); err != nil {
return nil, err
}
s.ctxfiles = contextfiles.NewLoader(contextfiles.NewRegistry(), homeReader{f: d.Files}, log)
s.shared = sharedfolders.New(sharedfolders.Options{Files: d.Files, Consents: d.PageConsents, Audit: d.SharedFolderAudit, Users: s.folderUsers()})
// A chat mounts the shared folders of its owner at /workspace/shared-<name>.
mgr, err := session.New(session.Options{Store: d.Store, Workspace: d.Files.Store(), Harness: h, StateDir: d.StateDir, Inputs: s.inputs, ChatTools: s.chatTools,
+11 -3
View File
@@ -130,8 +130,16 @@ func (s *Service) chatTools(ctx context.Context, x session.Access) (session.Turn
if !sub {
persona, personaKey = s.WellKnown().TurnPrompt(ctx, a.Principal)
}
if persona != "" {
prompt = strings.TrimSpace(persona + "\n\n" + prompt)
// The agent's profile and the index of the user's notes follow the persona.
// The first is the user's standing rules for this agent; the second points
// the model at memory_read.
var notes, notesKey string
if !sub {
notes, notesKey = s.contextTurn(ctx, rec)
tools = append(tools, s.contextTools(rec)...)
}
if lead := joinBlocks(persona, notes); lead != "" {
prompt = strings.TrimSpace(lead + "\n\n" + prompt)
}
if sc != nil {
// Granted skills the user lacks come back from skillTurn as a note of
@@ -233,7 +241,7 @@ func (s *Service) chatTools(ctx context.Context, x session.Access) (session.Turn
// load_skill learns from the final list whether the chat can read a
// skill's files and run what a skill needs.
tools = s.fitLoadSkill(a, tools)
return session.TurnTools{Tools: tools, Prompt: prompt, Note: note, WorkFolder: work, RefreshKey: key + "\x00" + rec.Settings.SystemPrompt + "\x00" + personaKey}, nil
return session.TurnTools{Tools: tools, Prompt: prompt, Note: note, WorkFolder: work, RefreshKey: key + "\x00" + rec.Settings.SystemPrompt + "\x00" + personaKey + "\x00" + notesKey}, nil
}
// spawnTool is spawn_agent for the chat of x: a generic sub-agent gets