fix(studio): tell the model what to do with a binary file

read_file and edit_file answered "X.docx is not a text file (43963
bytes)" and gave the model nothing to go on.

Make the error typed and let the chat's tool list add the hint. If the
chat has a skill for the file type (docx, xlsx, pptx, pdf), the error
names it. Otherwise it suggests a script run with the commands tool, with
a library for office types, when the chat can run commands. A chat with
neither is told that no tool opens the file. The skills are resolved only
when a read fails.

Assisted-by: Claude Code:claude-sonnet-5-5
Signed-off-by: Ettore Di Giacinto <mudler@localai.io>

core/services/studio/binaryhint.go

@@ -0,0 +1,78 @@
+package studio
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"path"
+	"strings"
+
+	"github.com/mudler/LocalAI/core/services/studio/files"
+	"github.com/mudler/LocalAI/core/services/studio/session"
+	"github.com/mudler/LocalAI/core/services/studio/workspace"
+)
+
+// officeKinds are the binary document types a chat meets most: the words a
+// skill of the type has in its name, and a library that reads the type in a
+// script.
+var officeKinds = map[string]struct {
+	skills  []string
+	library string
+}{
+	".docx": {[]string{"docx", "word"}, "python-docx"},
+	".xlsx": {[]string{"xlsx", "excel"}, "openpyxl"},
+	".pptx": {[]string{"pptx", "powerpoint"}, "python-pptx"},
+	".pdf":  {[]string{"pdf"}, "pypdf"},
+}
+
+// binaryHintTools wraps the tools that read a file as text: when one meets a
+// binary file, the error says what the chat can do with it. It names the
+// skill that matches the file type when the chat has one, and a script when
+// the chat can run commands. The skills are resolved only when a read fails.
+func (s *Service) binaryHintTools(a workspace.Actor, canRun bool, tools []session.Tool) []session.Tool {
+	for i, tool := range tools {
+		switch tool.Name {
+		case "read_file", "edit_file":
+		default:
+			continue
+		}
+		run := tool.Run
+		tools[i].Run = func(ctx context.Context, args json.RawMessage) (string, error) {
+			out, err := run(ctx, args)
+			var nt *files.NotTextError
+			if !errors.As(err, &nt) {
+				return out, err
+			}
+			return out, errors.New(s.binaryAdvice(ctx, a, canRun, nt))
+		}
+	}
+	return tools
+}
+
+// binaryAdvice is the error for a binary file, short and true for the tools
+// the chat has.
+func (s *Service) binaryAdvice(ctx context.Context, a workspace.Actor, canRun bool, nt *files.NotTextError) string {
+	ext := strings.ToLower(path.Ext(nt.Name))
+	msg := fmt.Sprintf("%s is a binary file (%d bytes), not text.", nt.Name, nt.Size)
+	kind, office := officeKinds[ext]
+	if office {
+		if list, _, err := s.Skills(ctx, a.Principal, a.ChatID); err == nil {
+			for _, sk := range list {
+				name := strings.ToLower(sk.Name)
+				for _, w := range kind.skills {
+					if strings.Contains(name, w) {
+						return msg + fmt.Sprintf(" Use the %s skill for %s files (load_skill).", sk.Name, ext)
+					}
+				}
+			}
+		}
+	}
+	switch {
+	case canRun && office:
+		return msg + fmt.Sprintf(" Run a script with the commands tool instead, for example with %s.", kind.library)
+	case canRun:
+		return msg + " Run a script with the commands tool instead."
+	}
+	return msg + " This chat has no tool that opens it."
+}

core/services/studio/binaryhint_internal_test.go

@@ -0,0 +1,53 @@
+package studio
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"testing/fstest"
+
+	"github.com/mudler/LocalAI/core/services/studio/files"
+	"github.com/mudler/LocalAI/core/services/studio/session"
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+)
+
+var _ = Describe("hints on binary files", func() {
+	// advice runs read_file on a file that is binary, as the file tool does, in
+	// a chat with the skills and the commands the spec chooses.
+	advice := func(name string, canRun bool, skillNames ...string) string {
+		GinkgoHelper()
+		ctx := context.Background()
+		svc, f := newSkillService(&skillBroker{})
+		sk := fstest.MapFS{}
+		for _, n := range skillNames {
+			sk[n+"/SKILL.md"] = &fstest.MapFile{Data: md(n, "A skill.")}
+		}
+		Expect(f.SyncBuiltin(ctx, sk)).To(Succeed())
+		binary := func(context.Context, json.RawMessage) (string, error) {
+			return "", &files.NotTextError{Name: name, Size: 43963}
+		}
+		tools := svc.binaryHintTools(f.Actor("alice"), canRun, []session.Tool{{Name: "read_file", Run: binary}, {Name: "list_files", Run: binary}})
+		_, err := tools[0].Run(ctx, json.RawMessage(`{"path":"/x"}`))
+		Expect(err).To(HaveOccurred())
+		_, err = tools[1].Run(ctx, json.RawMessage(`{"path":"/x"}`))
+		var nt *files.NotTextError
+		Expect(errors.As(err, &nt)).To(BeTrue(), "a tool that is not a text reader is left alone")
+		_, err = tools[0].Run(ctx, json.RawMessage(`{"path":"/x"}`))
+		return err.Error()
+	}
+
+	It("names the skill of the file type when the chat has one", func() {
+		got := advice("report.docx", true, "docx", "xlsx")
+		Expect(got).To(Equal("report.docx is a binary file (43963 bytes), not text. Use the docx skill for .docx files (load_skill)."))
+	})
+
+	It("suggests a script when there is no skill and the chat can run commands", func() {
+		Expect(advice("report.docx", true, "xlsx")).To(HaveSuffix("Run a script with the commands tool instead, for example with python-docx."))
+		Expect(advice("data.bin", true)).To(HaveSuffix("Run a script with the commands tool instead."))
+	})
+
+	It("says so when the chat has neither", func() {
+		Expect(advice("report.pdf", false)).To(HaveSuffix("This chat has no tool that opens it."))
+	})
+})

core/services/studio/files/tools.go

@@ -469,11 +469,22 @@ func (s *Service) readText(ctx context.Context, a workspace.Actor, abs string, m
 		return "", n, err
 	}
 	if !isText(data) || !utf8.Valid(data) {
-		return "", n, fmt.Errorf("%s is not a text file (%d bytes)", n.Name, n.Size)
+		return "", n, &NotTextError{Name: n.Name, Size: n.Size}
 	}
 	return string(data), n, nil
 }
 
+// NotTextError means a text tool met a binary file. The host adds a hint about
+// the tools the chat has, so the model knows what to do instead.
+type NotTextError struct {
+	Name string
+	Size int64
+}
+
+func (e *NotTextError) Error() string {
+	return fmt.Sprintf("%s is not a text file (%d bytes)", e.Name, e.Size)
+}
+
 // put writes content as one change set of the chat.
 func (s *Service) put(ctx context.Context, a workspace.Actor, abs, content string, o workspace.PutOptions, verb string) (workspace.Node, error) {
 	tx, err := s.ws.Begin(ctx, a, verb+lastName(abs))

core/services/studio/subagent.go

@@ -107,7 +107,7 @@ func (s *Service) chatTools(ctx context.Context, x session.Access) (session.Turn
 	}
 	fa := fx.Actor()
 	nudge := s.newRulesNudge(ctx, x, rec)
-	tools = append(tools, nudge.wrap(s.fileNudgeTools(fx, s.announceFileChanges(fa, s.skillHintTools(fa, dropHidden(s.d.Files.Tools(fx), hidden)))))...)
+	tools = append(tools, nudge.wrap(s.fileNudgeTools(fx, s.announceFileChanges(fa, s.skillHintTools(fa, s.binaryHintTools(fa, cmds.State == CommandsAvailable, dropHidden(s.d.Files.Tools(fx), hidden))))))...)
 	if s.d.Media != nil {
 		tools = append(tools, s.skillHintTools(fa, dropHidden(s.d.Media.Tools(s.d.Files, x, rec.Model), hidden))...)
 	}