feat(studio): let the agent attach an existing file in a connector chat

The host sent only the files a turn made or changed. A person who asked
in a messaging app for a file that already existed got nothing.

Add the attach_file tool. It is offered only in a chat that a connector
started whose binding sends files, so a chat in Studio has no such tool.
It takes a path as the file tools show it, checks the file against the
chat's access (a scoped agent reaches only its folders), the type
allowlist, the size limit and the count, and marks it. When it refuses, the
result says why in plain words so the model can tell the person.

sendFiles now sends the files the turn made, then the attached ones, each
path once and at most five in all. It reads them with the chat's access
resolved at send time. Delivery is the existing send_file op, so every
connector add-on that implements it gets the feature with no code of its
own. A failed send says "I could not send X" and adds the reason the add-on
reports, cut to one short line.

The marks are kept in the memory of the process, like the other
per-turn state of the connector manager.

Assisted-by: Claude Code:claude-sonnet-5-5
Signed-off-by: Ettore Di Giacinto <mudler@localai.io>
This commit is contained in:
Ettore Di Giacinto committed 2026-11-03 09:33:24 +00:00
1 parent ef79f5336d
commit eca78c3f97
6 files changed
+434 -12

No files matched your search

+115
View File
@@ -0,0 +1,115 @@
package studio
import (
"context"
"encoding/json"
"errors"
"fmt"
"path"
"slices"
"github.com/mudler/LocalAI/core/services/studio/session"
"github.com/mudler/LocalAI/core/services/studio/workspace"
"github.com/mudler/LocalAI/pkg/addons/contract"
)
const attachFileName = "attach_file"
const attachFileDescription = "Give the person a file that already exists in your files: it is sent to them with your answer. " +
"Files you make or change in a turn are attached without this call; use it for a file from an earlier turn or from a folder you were given. " +
"path is the file's path as your file tools show it. A file that cannot be attached comes back with the reason: tell the person plainly."
const attachFileSchema = `{"type":"object","properties":{"path":{"type":"string","description":"The path of the file, such as /workspace/chat/report.docx."}},"required":["path"]}`
// attachTools returns attach_file for a chat that a connector started whose
// binding sends files. Any other chat has no such tool: a person in Studio
// sees the files of the chat there. The tool only marks the file; sendFiles
// reads and sends it after the answer, through the send_file op that any
// connector add-on serves.
func (s *Service) attachTools(ctx context.Context, x session.Access, rec session.ChatRecord) []session.Tool {
c := s.connectors
if c == nil || rec.Origin != session.OriginConnector || rec.Kind == session.KindSubAgent || !s.d.Files.Available() {
return nil
}
b, err := c.o.Store.GetBinding(ctx, rec.OriginID)
if err != nil || !b.SendsFiles() {
return nil
}
return []session.Tool{{
Name: attachFileName, Description: attachFileDescription, Schema: json.RawMessage(attachFileSchema),
Run: func(ctx context.Context, raw json.RawMessage) (string, error) {
return c.attachFile(ctx, x, raw)
},
}}
}
// attachFile checks the file against the chat's access and the limits of a
// sender, and marks it for the answer. The check is repeated when the file is
// sent, because the file or the access can change in between; this one lets
// the model tell the person at once why a file cannot go.
func (c *Connectors) attachFile(ctx context.Context, x session.Access, raw json.RawMessage) (string, error) {
in, err := session.DecodeArgs[struct{ Path string }](raw)
if err != nil {
return "", err
}
t, err := x.Place().Resolve(in.Path)
if err != nil {
return "", notFound(err, in.Path)
}
n, err := c.svc.d.Files.Store().Stat(ctx, x.Actor(), t.Abs)
if err != nil {
return "", notFound(err, in.Path)
}
name := contract.SafeFileName(n.Name)
switch {
case n.Kind == workspace.KindDir:
return "", fmt.Errorf("%s is a folder, not a file; attach the files in it one by one", in.Path)
case !outboundAllowed(name):
return "", fmt.Errorf("%s is not sent: files of this type (%s) are not sent here. Documents, data, pictures, sound, video and zip files are", name, path.Ext(name))
case n.Size == 0:
return "", fmt.Errorf("%s is empty, so there is nothing to send", name)
case n.Size > contract.MaxConnectorFileBytes:
return "", fmt.Errorf("%s is too large to send here (%d bytes; the limit is %d MB). Tell the person; offer to split it, to make a smaller file, or to paste the content", name, n.Size, contract.MaxConnectorFileBytes>>20)
}
key := key(x.Actor().Principal, x.Actor().ChatID)
c.amu.Lock()
defer c.amu.Unlock()
marked := c.attached[key]
if slices.Contains(marked, t.Abs) {
return name + " is attached already.", nil
}
if len(marked) >= maxFilesPerTurn {
return "", fmt.Errorf("at most %d files go with one answer, and %d are marked already; tell the person which one is left out", maxFilesPerTurn, len(marked))
}
if c.attached == nil {
c.attached = map[string][]string{}
}
c.attached[key] = append(marked, t.Abs)
return name + " is attached: it is sent to the person after your answer. Do not give them a path.", nil
}
// takeAttached returns the files that the chat's turn marked, in the order of
// the calls, and forgets them.
func (c *Connectors) takeAttached(principal, chatID string) []string {
c.amu.Lock()
defer c.amu.Unlock()
k := key(principal, chatID)
out := c.attached[k]
delete(c.attached, k)
return out
}
// outboundAllowed says whether a file of this name goes to a sender.
func outboundAllowed(name string) bool {
_, ok := outboundType(name)
return ok
}
// notFound tells the model that a path is not a file it can reach. A path
// outside the chat's access and a path that does not exist look the same.
func notFound(err error, p string) error {
if errors.Is(err, workspace.ErrNotFound) || errors.Is(err, workspace.ErrReadOnly) {
return fmt.Errorf("%s was not found", p)
}
return fmt.Errorf("%s cannot be attached: %w", p, err)
}
+63 -12
View File
@@ -92,8 +92,36 @@ func (c *Connectors) turnFiles(ctx context.Context, b connectors.Binding, x sess
return out
}
// sendFiles hands the files of a turn to the sender after the answer. A file
// that cannot go is named in one plain message, with the reason.
// outFile is a file on its way to a sender. made says that the turn made or
// changed it, so the sender may be told "I made"; else the agent attached it.
type outFile struct {
path, name string
made bool
}
// outgoing lists what goes with the answer: the files the turn made or
// changed, then the files the agent attached, each path once.
func (c *Connectors) outgoing(ctx context.Context, b connectors.Binding, x session.Access, chatID string, turn int, since time.Time) []outFile {
var out []outFile
seen := map[string]bool{}
for _, f := range c.turnFiles(ctx, b, x, chatID, turn, since) {
if !seen[f.Path] {
seen[f.Path] = true
out = append(out, outFile{path: f.Path, name: contract.SafeFileName(f.Name), made: true})
}
}
for _, p := range c.takeAttached(x.Actor().Principal, chatID) {
if !seen[p] {
seen[p] = true
out = append(out, outFile{path: p, name: contract.SafeFileName(path.Base(p))})
}
}
return out
}
// sendFiles hands the files of a turn to the sender after the answer: those
// the turn made and those the agent attached. A file that cannot go is named
// in one plain message, with the reason.
func (c *Connectors) sendFiles(b connectors.Binding, token string, m contract.ConnectorMessage, chatID string, turn int, since time.Time) {
if !b.SendsFiles() {
return
@@ -101,47 +129,70 @@ func (c *Connectors) sendFiles(b connectors.Binding, token string, m contract.Co
ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute)
defer cancel()
// The files go with the chat's access, resolved now: a scoped agent's
// reply sends nothing from outside its folders.
// reply sends nothing from outside its folders, whatever was marked.
x, err := c.svc.ChatAccess(ctx, b.Owner, chatID)
if err != nil {
c.log.Warn("studio: the files of a connector turn are not sent", "binding", b.ID, "error", err)
return
}
files := c.turnFiles(ctx, b, x, chatID, turn, since)
files := c.outgoing(ctx, b, x, chatID, turn, since)
var notes []string
for i, f := range files {
if i == maxFilesPerTurn {
notes = append(notes, fmt.Sprintf("%d more files are in the chat's files.", len(files)-i))
break
}
name := contract.SafeFileName(f.Name)
mimeType, ok := outboundType(name)
mimeType, ok := outboundType(f.name)
if !ok {
notes = append(notes, fmt.Sprintf("I made %s, but files of this type are not sent here.", name))
if f.made {
notes = append(notes, fmt.Sprintf("I made %s, but files of this type are not sent here.", f.name))
} else {
notes = append(notes, fmt.Sprintf("%s was not sent: files of this type are not sent here.", f.name))
}
continue
}
data, err := c.svc.readChatFile(ctx, x, f.Path, contract.MaxConnectorFileBytes)
data, err := c.svc.readChatFile(ctx, x, f.path, contract.MaxConnectorFileBytes)
switch {
case errors.Is(err, workspace.ErrNotFound):
continue // gone since, or outside the chat's access: not named
case errors.Is(err, errFileTooLarge):
notes = append(notes, fmt.Sprintf("%s is too large to send here (the limit is 1 MB). It is in the chat's files.", name))
notes = append(notes, fmt.Sprintf("%s is too large to send here (the limit is 1 MB). It is in the chat's files.", f.name))
continue
case err != nil:
c.log.Warn("studio: could not read a file for a connector", "binding", b.ID, "error", err)
notes = append(notes, fmt.Sprintf("I could not read %s.", name))
notes = append(notes, fmt.Sprintf("I could not read %s.", f.name))
continue
case len(data) == 0:
continue // an empty file says nothing, and platforms refuse it
}
err = c.send(ctx, b, contract.ConnectorRequest{Op: contract.ConnectorOpSendFile, Token: token, Chat: m.Chat, Thread: m.Thread,
File: &contract.ConnectorFile{Name: name, MIME: mimeType, Data: data}})
File: &contract.ConnectorFile{Name: f.name, MIME: mimeType, Data: data}})
if err != nil {
c.log.Warn("studio: could not send a file to a connector", "binding", b.ID, "kind", b.Kind, "error", err)
notes = append(notes, fmt.Sprintf("I could not send %s.", name))
notes = append(notes, fmt.Sprintf("I could not send %s%s.", f.name, sendReason(err)))
}
}
if len(notes) > 0 {
c.say(b, token, m.Chat, m.Thread, strings.Join(notes, "\n"))
}
}
// maxReasonRunes bounds the platform's reason in a message to the sender.
const maxReasonRunes = 160
// sendReason is ": <why>" when the add-on says why the platform refused a
// file (a missing permission, say), cut to one short line, and "" otherwise.
func sendReason(err error) string {
var ce *contract.ConnectorError
if !errors.As(err, &ce) {
return ""
}
why := strings.Join(strings.Fields(ce.Message), " ")
if r := []rune(why); len(r) > maxReasonRunes {
why = string(r[:maxReasonRunes]) + "…"
}
if why == "" {
return ""
}
return ": " + why
}
+3
View File
@@ -138,6 +138,9 @@ type Connectors struct {
emu sync.Mutex
nextEdit map[string]time.Time // by binding: when its next edit may go
amu sync.Mutex
attached map[string][]string // by chat: files that attach_file marked for the answer of the turn
wake chan struct{}
done chan struct{}
once sync.Once
@@ -0,0 +1,249 @@
package studio_test
import (
"context"
"net/http"
"strings"
"time"
"github.com/mudler/LocalAI/addons/slack/slacktest"
slack "github.com/mudler/LocalAI/addons/slack/transport"
"github.com/mudler/LocalAI/core/services/studio"
"github.com/mudler/LocalAI/core/services/studio/agents"
"github.com/mudler/LocalAI/core/services/studio/connectors"
"github.com/mudler/LocalAI/core/services/studio/session"
"github.com/mudler/LocalAI/core/services/studio/studiotest"
"github.com/mudler/LocalAI/core/services/studio/workspace"
"github.com/mudler/LocalAI/tests/studiobot"
. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
)
// attach_file hands a file that already exists to the person who talks to a
// connector chat. Delivery is the typed send_file op, so the specs drive it
// through two add-ons (Telegram and Slack) and through a plain chat.
// attachScript returns the steps by a word of the message: "make" writes a
// report, the other words attach what the spec names.
func attachScript(steps map[string][]studiotest.ToolStep) func(workspace.Actor, string) []studiotest.ToolStep {
return func(_ workspace.Actor, text string) []studiotest.ToolStep {
for k, s := range steps {
if strings.Contains(text, k) {
return s
}
}
return []studiotest.ToolStep{{Say: "ok"}}
}
}
func attach(path string) studiotest.ToolStep {
return studiotest.ToolStep{Tool: "attach_file", Args: `{"path":"` + path + `"}`}
}
func write(path, body string) studiotest.ToolStep {
return studiotest.ToolStep{Tool: "write_file", Args: `{"path":"` + path + `","content":"` + body + `"}`}
}
var _ = Describe("attach_file in a connector chat", func() {
var (
sh *shared
bot *scriptedBot
)
BeforeEach(func() { sh, bot = newShared(), newScriptedBot() })
// telegram binds the sender 11 over the scripted bot.
telegram := func(steps map[string][]studiotest.ToolStep, in studio.ConnectorInput) *connRig {
r := newConnRig(sh, connStore(), bot, attachScript(steps), nil)
in.Allow, in.Reply = []string{"11"}, connectors.ReplyOff
r.bind(in)
return r
}
ask := func(id int, text string) { bot.say(id, 11, "Ada", "ada", text) }
answered := func(n int) { waitFor("the answer", func() bool { return len(bot.answers()) >= n }) }
It("sends a file that an earlier turn made", func() {
r := telegram(map[string][]studiotest.ToolStep{
"make": {write("/out/report.csv", "a,b"), {Say: "Made."}},
"again": {attach("/out/report.csv"), {Say: "Here it is."}},
}, studio.ConnectorInput{})
ask(1, "make")
waitFor("the first file", func() bool { return len(bot.sentFiles()) == 1 })
answered(1)
ask(2, "again")
waitFor("the second file", func() bool { return len(bot.sentFiles()) == 2 })
answered(2)
f := bot.sentFiles()[1]
Expect([]string{f.Method, f.Name, string(f.Data)}).To(Equal([]string{"sendDocument", "report.csv", "a,b"}))
res := r.h.ResultsFor("attach_file")
Expect(res).To(HaveLen(1))
Expect(res[0].Err).NotTo(HaveOccurred())
Expect(res[0].Out).To(ContainSubstring("report.csv"))
Expect(bot.answers()[1]).To(Equal("Here it is."), "the answer comes before the file, and no note follows")
})
It("sends a file once when it was made and attached in the same turn", func() {
telegram(map[string][]studiotest.ToolStep{
"make": {write("/out/report.csv", "a,b"), attach("/out/report.csv"), attach("/out/report.csv"), {Say: "Made."}},
}, studio.ConnectorInput{})
ask(1, "make")
waitFor("the file", func() bool { return len(bot.sentFiles()) >= 1 })
answered(1)
time.Sleep(300 * time.Millisecond)
Expect(bot.sentFiles()).To(HaveLen(1))
})
It("tells the model why a file cannot be attached", func() {
big := strings.Repeat("x", 1<<20+10)
steps := []studiotest.ToolStep{
write("/out/big.csv", big), write("/out/tool.exe", "MZ"), write("/out/empty.txt", ""),
attach("/out/missing.csv"), attach("/out/big.csv"), attach("/out/tool.exe"), attach("/out"),
{Say: "Could not."},
}
r := telegram(map[string][]studiotest.ToolStep{"files": steps}, studio.ConnectorInput{})
ask(1, "files")
answered(2) // the answer and the note about the files of the turn
res := r.h.ResultsFor("attach_file")
Expect(res).To(HaveLen(4))
for _, x := range res {
Expect(x.Err).To(HaveOccurred())
}
Expect(res[0].Err.Error()).To(ContainSubstring("not found"))
Expect(res[1].Err.Error()).To(ContainSubstring("too large"))
Expect(res[1].Err.Error()).To(ContainSubstring("1 MB"))
Expect(res[2].Err.Error()).To(ContainSubstring("not sent"))
Expect(res[3].Err.Error()).To(ContainSubstring("folder"))
Expect(bot.sentFiles()).To(BeEmpty())
})
It("counts the files of a turn, those made and those attached", func() {
var steps []studiotest.ToolStep
for _, n := range []string{"a", "b", "c", "d", "e", "f"} {
steps = append(steps, write("/out/"+n+".txt", n))
}
telegram(map[string][]studiotest.ToolStep{
"make": append(steps, studiotest.ToolStep{Say: "Made."}),
}, studio.ConnectorInput{})
ask(1, "make")
waitFor("five files", func() bool { return len(bot.sentFiles()) == 5 })
answered(2)
Expect(bot.answers()[1]).To(ContainSubstring("1 more file"))
})
It("limits the files that one answer attaches, in one stable order", func() {
r := telegram(map[string][]studiotest.ToolStep{
"make": {write("/out/a.txt", "a"), write("/out/b.txt", "b"), write("/out/c.txt", "c"), write("/out/d.txt", "d"),
write("/out/e.txt", "e"), write("/out/f.txt", "f"), {Say: "Made."}},
"all": {attach("/out/b.txt"), attach("/out/a.txt"), attach("/out/c.txt"), attach("/out/d.txt"), attach("/out/e.txt"), attach("/out/f.txt"), {Say: "Sent."}},
}, studio.ConnectorInput{})
ask(1, "make")
waitFor("the first five", func() bool { return len(bot.sentFiles()) == 5 })
answered(2)
ask(2, "all")
waitFor("five more", func() bool { return len(bot.sentFiles()) == 10 })
res := r.h.ResultsFor("attach_file")
Expect(res).To(HaveLen(6))
Expect(res[5].Err).To(HaveOccurred())
Expect(res[5].Err.Error()).To(ContainSubstring("5 files"))
var names []string
for _, f := range bot.sentFiles()[5:] {
names = append(names, f.Name)
}
Expect(names).To(Equal([]string{"b.txt", "a.txt", "c.txt", "d.txt", "e.txt"}), "the order of the calls")
})
It("names a file that the platform refused, with its reason", func() {
bot.setFail("sendDocument", func() (int, string) {
return http.StatusBadRequest, `{"ok":false,"error_code":400,"description":"Bad Request: wrong file identifier"}`
})
telegram(map[string][]studiotest.ToolStep{
"make": {write("/out/report.csv", "a"), attach("/out/report.csv"), {Say: "Done."}},
}, studio.ConnectorInput{})
ask(1, "make")
waitFor("the note", func() bool { return len(bot.answers()) == 2 })
Expect(bot.answers()[1]).To(ContainSubstring("I could not send report.csv"))
Expect(bot.answers()[1]).To(ContainSubstring("wrong file identifier"))
})
It("is offered only when the binding sends files", func() {
script := map[string][]studiotest.ToolStep{}
r := telegram(script, studio.ConnectorInput{})
ask(1, "hi")
answered(1)
chats, _ := r.svc.List(context.Background(), alice, 10)
Expect(chats).To(HaveLen(1))
Expect(r.h.ToolNames(chats[0].ID)).To(ContainElement("attach_file"))
sh2, bot2 := newShared(), newScriptedBot()
r2 := newConnRig(sh2, connStore(), bot2, attachScript(script), nil)
r2.bind(studio.ConnectorInput{Allow: []string{"11"}, Reply: connectors.ReplyOff, SendFiles: &filesOff})
bot2.say(1, 11, "Ada", "ada", "hi")
waitFor("the answer", func() bool { return len(bot2.answers()) >= 1 })
chats, _ = r2.svc.List(context.Background(), alice, 10)
Expect(chats).To(HaveLen(1))
Expect(r2.h.ToolNames(chats[0].ID)).NotTo(ContainElement("attach_file"))
})
It("does not exist in a chat that a person opens in Studio", func() {
r := newConnRig(sh, connStore(), bot, attachScript(nil), nil)
id := create(r.svc, alice)
sent(r.svc, alice, id, "hi")
idle(r.svc, alice, id)
Expect(r.h.ToolNames(id)).NotTo(ContainElement("attach_file"))
})
Describe("with Slack", func() {
It("sends a file that an earlier turn made through the same send_file op", func() {
f := slacktest.New()
DeferCleanup(f.Close)
f.AddApp(slacktest.App{Bot: slackBot, AppToken: slackApp, UserID: "UBOT"})
f.AddUser("U1", "ada", "Ada")
rt := slack.New(&slack.API{BaseURL: f.URL, FilesHost: f.FilesHost})
DeferCleanup(rt.Close)
script := attachScript(map[string][]studiotest.ToolStep{
"make": {write("/out/notes.md", "# notes"), {Say: "Made."}},
"again": {attach("/out/notes.md"), {Say: "Here."}},
})
r := newConnRigWith(newShared(), connStore(), studiobot.NewTransport(studiobot.Slack(rt)), script, nil)
r.bind(studio.ConnectorInput{Kind: "slack", Token: slackBot + " " + slackApp, Allow: []string{"U1"}})
waitFor("the Socket Mode link", func() bool { return f.Connected(slackApp) })
f.Push(slackApp, slacktest.DM("D1", "U1", "make"))
waitFor("the first upload", func() bool { return len(f.Uploads(slackBot)) == 1 })
f.Push(slackApp, slacktest.DM("D1", "U1", "again"))
waitFor("the second upload", func() bool { return len(f.Uploads(slackBot)) == 2 })
up := f.Uploads(slackBot)[1]
Expect([]string{up.Name, up.Channel, string(up.Data)}).To(Equal([]string{"notes.md", "D1", "# notes"}))
})
})
Describe("with a scoped agent", func() {
It("attaches only inside the agent's folder", func() {
ctx := context.Background()
Expect(sh.files.EnsureHome(ctx, alice)).To(Succeed())
home := session.HomePath(alice)
for _, d := range []string{"/Projects", "/Projects/Acme", "/Secret"} {
_, err := sh.files.Store().Mkdir(ctx, sh.files.Actor(alice), home+d)
Expect(err).NotTo(HaveOccurred())
}
_, err := sh.files.Store().Put(ctx, sh.files.Actor(alice), home+"/Secret/plan.csv", strings.NewReader("OUTSIDE-SECRET-MARKER"), workspace.PutOptions{Overwrite: true})
Expect(err).NotTo(HaveOccurred())
r := newConnRig(sh, connStore(), bot, attachScript(map[string][]studiotest.ToolStep{
"steal": {attach("/Secret/plan.csv"), attach("/workspace/home/Secret/plan.csv"), attach("../Secret/plan.csv"), {Say: "No."}},
}), nil)
ag, err := r.svc.SaveAgentWithFolder(ctx, alice, agents.Agent{Name: "Support", Model: "m"},
&studio.AgentFolderWrite{Path: "/Projects/Acme", Mode: agents.FolderWrite})
Expect(err).NotTo(HaveOccurred())
r.agent = ag
r.bind(studio.ConnectorInput{Allow: []string{"11"}, Reply: connectors.ReplyOff})
ask(1, "steal")
answered(1)
time.Sleep(300 * time.Millisecond)
res := r.h.ResultsFor("attach_file")
Expect(res).To(HaveLen(3))
for _, x := range res {
Expect(x.Err).To(HaveOccurred())
Expect(x.Err.Error()).NotTo(ContainSubstring("OUTSIDE-SECRET-MARKER"))
}
Expect(bot.sentFiles()).To(BeEmpty())
})
})
})
+1
View File
@@ -292,6 +292,7 @@ func newConnRigOpts(sh *shared, store connectors.Store, tr *studiobot.BotTranspo
c, err := studio.NewConnectors(r.svc, o)
Expect(err).NotTo(HaveOccurred())
DeferCleanup(c.Close)
r.svc.SetConnectors(c) // as the application does: the chats see their bindings
ctx, cancel := context.WithCancel(context.Background())
DeferCleanup(cancel)
go c.Run(ctx)
+3
View File
@@ -108,6 +108,9 @@ func (s *Service) chatTools(ctx context.Context, x session.Access) (session.Turn
fa := fx.Actor()
nudge := s.newRulesNudge(ctx, x, rec)
tools = append(tools, nudge.wrap(s.fileNudgeTools(fx, s.announceFileChanges(fa, s.skillHintTools(fa, s.binaryHintTools(fa, cmds.State == CommandsAvailable, dropHidden(s.d.Files.Tools(fx), hidden))))))...)
if !sub {
tools = append(tools, s.attachTools(ctx, fx, rec)...)
}
if s.d.Media != nil {
tools = append(tools, s.skillHintTools(fa, dropHidden(s.d.Media.Tools(s.d.Files, x, rec.Model), hidden))...)
}