fix(studio): tell the model of a connector chat that files are attached

A chat started from a messaging app has its answer read in that app, and
the host attaches the files a turn makes to it. The model did not know
this: it said it could not send files and quoted a sandbox path.

Add a short block to the turn prompt of connector chats. It names the
platform, says that files saved in the chat folder are attached after the
answer, gives the limits the host enforces, and says that files are not
sent when the binding turns them off. If the binding cannot be read the
file part is left out. Other chats are unchanged.

Assisted-by: Claude Code:claude-sonnet-5-5
Signed-off-by: Ettore Di Giacinto <mudler@localai.io>

core/services/studio/connectorturn.go

@@ -0,0 +1,51 @@
+package studio
+
+import (
+	"context"
+	"fmt"
+	"strings"
+
+	"github.com/mudler/LocalAI/core/services/studio/session"
+	"github.com/mudler/LocalAI/pkg/addons/contract"
+)
+
+// connectorBlock tells the model of a chat that a messaging app started where
+// its answer goes. The person reads it there, not in Studio, and the host
+// attaches the files a turn makes (sendFiles), so the model must not say it
+// cannot send a file or quote a path of the sandbox. It is a block of the turn
+// prompt, not a note: a note is cut at 400 bytes, and the text of a block is
+// the same at each turn, so the harness is not restarted and the transcript
+// does not grow. Other chats get nothing. canWrite says whether the chat can
+// make a file at all.
+func (s *Service) connectorBlock(ctx context.Context, rec session.ChatRecord, canWrite bool) string {
+	if rec.Origin != session.OriginConnector {
+		return ""
+	}
+	platform := "a messaging app"
+	sends, known := false, false
+	if c := s.connectors; c != nil {
+		if b, err := c.o.Store.GetBinding(ctx, rec.OriginID); err == nil {
+			sends, known = b.SendsFiles(), true
+			platform = b.Kind
+			if kinds, err := c.o.Transport.Kinds(ctx); err == nil {
+				if k, ok := kindFor(kinds, b); ok && k.Title != "" {
+					platform = k.Title
+				}
+			}
+		}
+	}
+	var sb strings.Builder
+	fmt.Fprintf(&sb, "This chat is on %s. The person reads your answer there, not in Studio, and cannot open sandbox paths.", platform)
+	switch {
+	case !canWrite:
+	case !known:
+		sb.WriteString(" Never tell them a file path.")
+	case !sends:
+		sb.WriteString(" Files are NOT sent to them: paste short content in the answer, or say you cannot attach a file. Never tell them a file path.")
+	default:
+		fmt.Fprintf(&sb, " Files you save in the chat folder during a turn are attached to your answer automatically after it. Name the file and say it is attached; never say you cannot send files, and never tell them a path."+
+			" Up to %d files per turn, %d MB each; documents, data, images, audio, video and zip only, no scripts or HTML. Keep files small, and say so if one cannot be attached.",
+			maxFilesPerTurn, contract.MaxConnectorFileBytes>>20)
+	}
+	return sb.String()
+}

core/services/studio/connectorturn_internal_test.go

@@ -0,0 +1,90 @@
+package studio
+
+import (
+	"context"
+	"strings"
+
+	"github.com/mudler/LocalAI/core/services/studio/connectors"
+	"github.com/mudler/LocalAI/core/services/studio/session"
+	"github.com/mudler/LocalAI/pkg/addons/contract"
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+)
+
+// titleTransport serves one kind and answers no call.
+type titleTransport struct{}
+
+func (titleTransport) Kinds(context.Context) ([]ConnectorKind, error) {
+	return []ConnectorKind{{AddonID: "slack-addon", ConnectorKind: contract.ConnectorKind{ID: "slack", Title: "Slack"}}}, nil
+}
+
+func (titleTransport) Call(context.Context, ConnectorCallScope, contract.ConnectorRequest) (contract.ConnectorResponse, error) {
+	return contract.ConnectorResponse{}, nil
+}
+
+var _ = Describe("the turn block of a connector chat", func() {
+	var (
+		ctx context.Context
+		svc *Service
+		st  connectors.Store
+	)
+	BeforeEach(func() {
+		ctx = context.Background()
+		svc, _ = newSkillService(&skillBroker{})
+		var err error
+		st, err = connectors.NewJSON("")
+		Expect(err).NotTo(HaveOccurred())
+		c, err := NewConnectors(svc, ConnectorOptions{Store: st, Transport: titleTransport{}})
+		Expect(err).NotTo(HaveOccurred())
+		svc.SetConnectors(c)
+	})
+
+	// prompt builds the host prompt of two turns of a chat and returns both.
+	prompt := func(origin Origin) (string, string) {
+		rec, err := svc.CreateAs(ctx, "alice", CreateInput{Model: "m"}, origin)
+		Expect(err).NotTo(HaveOccurred())
+		x, err := svc.ChatAccess(ctx, "alice", rec.ID)
+		Expect(err).NotTo(HaveOccurred())
+		first, err := svc.chatTools(ctx, x)
+		Expect(err).NotTo(HaveOccurred())
+		second, err := svc.chatTools(ctx, x)
+		Expect(err).NotTo(HaveOccurred())
+		return first.Prompt, second.Prompt
+	}
+	bind := func(id string, filesOff bool) {
+		Expect(st.PutBinding(ctx, connectors.Binding{ID: id, Owner: "alice", Kind: "slack", AddonID: "slack-addon", FilesOff: filesOff})).To(Succeed())
+	}
+
+	It("tells the model that files are attached and what the limits are", func() {
+		bind("b1", false)
+		p1, p2 := prompt(Origin{Kind: session.OriginConnector, ID: "b1"})
+		Expect(p1).To(ContainSubstring("This chat is on Slack."))
+		Expect(p1).To(ContainSubstring("attached to your answer automatically"))
+		Expect(p1).To(ContainSubstring("Up to 5 files per turn, 1 MB each"))
+		Expect(p1).NotTo(ContainSubstring("NOT sent"))
+		Expect(strings.Count(p2, "This chat is on")).To(Equal(1), "a second turn does not repeat the block")
+		Expect(p2).To(Equal(p1))
+		Expect(len(svc.connectorBlock(ctx, session.ChatRecord{Origin: session.OriginConnector, OriginID: "b1"}, true))).To(BeNumerically("<", 700))
+	})
+
+	It("says that files are not sent when the binding turns them off", func() {
+		bind("b2", true)
+		p, _ := prompt(Origin{Kind: session.OriginConnector, ID: "b2"})
+		Expect(p).To(ContainSubstring("This chat is on Slack."))
+		Expect(p).To(ContainSubstring("Files are NOT sent"))
+		Expect(p).NotTo(ContainSubstring("attached to your answer automatically"))
+	})
+
+	It("leaves out the file part when the binding cannot be read", func() {
+		p, _ := prompt(Origin{Kind: session.OriginConnector, ID: "gone"})
+		Expect(p).To(ContainSubstring("This chat is on a messaging app."))
+		Expect(p).NotTo(ContainSubstring("attached to your answer"))
+		Expect(p).NotTo(ContainSubstring("NOT sent"))
+	})
+
+	It("adds nothing to a chat a person opens in Studio", func() {
+		bind("b3", false)
+		p, _ := prompt(Origin{})
+		Expect(p).NotTo(ContainSubstring("This chat is on"))
+	})
+})

core/services/studio/subagent.go

@@ -210,7 +210,7 @@ func (s *Service) chatTools(ctx context.Context, x session.Access) (session.Turn
 		// at its end, and the model must not lose the goal.
 		goal = goalBlock(rec.Goal)
 	}
-	prompt = joinBlocks(head, goal, webBlock(tools), prompt)
+	prompt = joinBlocks(head, goal, s.connectorBlock(ctx, rec, canWrite), webBlock(tools), prompt)
 	if sp := strings.TrimSpace(rec.Settings.SystemPrompt); sp != "" {
 		// The chat's own instructions come first: the prompt is cut at its end.
 		prompt = strings.TrimSpace(sp + "\n\n" + prompt)