diff --git a/core/services/studio/binaryhint.go b/core/services/studio/binaryhint.go new file mode 100644 index 000000000..bb1a60b26 --- /dev/null +++ b/core/services/studio/binaryhint.go @@ -0,0 +1,78 @@ +package studio + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "path" + "strings" + + "github.com/mudler/LocalAI/core/services/studio/files" + "github.com/mudler/LocalAI/core/services/studio/session" + "github.com/mudler/LocalAI/core/services/studio/workspace" +) + +// officeKinds are the binary document types a chat meets most: the words a +// skill of the type has in its name, and a library that reads the type in a +// script. +var officeKinds = map[string]struct { + skills []string + library string +}{ + ".docx": {[]string{"docx", "word"}, "python-docx"}, + ".xlsx": {[]string{"xlsx", "excel"}, "openpyxl"}, + ".pptx": {[]string{"pptx", "powerpoint"}, "python-pptx"}, + ".pdf": {[]string{"pdf"}, "pypdf"}, +} + +// binaryHintTools wraps the tools that read a file as text: when one meets a +// binary file, the error says what the chat can do with it. It names the +// skill that matches the file type when the chat has one, and a script when +// the chat can run commands. The skills are resolved only when a read fails. +func (s *Service) binaryHintTools(a workspace.Actor, canRun bool, tools []session.Tool) []session.Tool { + for i, tool := range tools { + switch tool.Name { + case "read_file", "edit_file": + default: + continue + } + run := tool.Run + tools[i].Run = func(ctx context.Context, args json.RawMessage) (string, error) { + out, err := run(ctx, args) + var nt *files.NotTextError + if !errors.As(err, &nt) { + return out, err + } + return out, errors.New(s.binaryAdvice(ctx, a, canRun, nt)) + } + } + return tools +} + +// binaryAdvice is the error for a binary file, short and true for the tools +// the chat has. +func (s *Service) binaryAdvice(ctx context.Context, a workspace.Actor, canRun bool, nt *files.NotTextError) string { + ext := strings.ToLower(path.Ext(nt.Name)) + msg := fmt.Sprintf("%s is a binary file (%d bytes), not text.", nt.Name, nt.Size) + kind, office := officeKinds[ext] + if office { + if list, _, err := s.Skills(ctx, a.Principal, a.ChatID); err == nil { + for _, sk := range list { + name := strings.ToLower(sk.Name) + for _, w := range kind.skills { + if strings.Contains(name, w) { + return msg + fmt.Sprintf(" Use the %s skill for %s files (load_skill).", sk.Name, ext) + } + } + } + } + } + switch { + case canRun && office: + return msg + fmt.Sprintf(" Run a script with the commands tool instead, for example with %s.", kind.library) + case canRun: + return msg + " Run a script with the commands tool instead." + } + return msg + " This chat has no tool that opens it." +} diff --git a/core/services/studio/binaryhint_internal_test.go b/core/services/studio/binaryhint_internal_test.go new file mode 100644 index 000000000..a21885bcf --- /dev/null +++ b/core/services/studio/binaryhint_internal_test.go @@ -0,0 +1,53 @@ +package studio + +import ( + "context" + "encoding/json" + "errors" + "testing/fstest" + + "github.com/mudler/LocalAI/core/services/studio/files" + "github.com/mudler/LocalAI/core/services/studio/session" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("hints on binary files", func() { + // advice runs read_file on a file that is binary, as the file tool does, in + // a chat with the skills and the commands the spec chooses. + advice := func(name string, canRun bool, skillNames ...string) string { + GinkgoHelper() + ctx := context.Background() + svc, f := newSkillService(&skillBroker{}) + sk := fstest.MapFS{} + for _, n := range skillNames { + sk[n+"/SKILL.md"] = &fstest.MapFile{Data: md(n, "A skill.")} + } + Expect(f.SyncBuiltin(ctx, sk)).To(Succeed()) + binary := func(context.Context, json.RawMessage) (string, error) { + return "", &files.NotTextError{Name: name, Size: 43963} + } + tools := svc.binaryHintTools(f.Actor("alice"), canRun, []session.Tool{{Name: "read_file", Run: binary}, {Name: "list_files", Run: binary}}) + _, err := tools[0].Run(ctx, json.RawMessage(`{"path":"/x"}`)) + Expect(err).To(HaveOccurred()) + _, err = tools[1].Run(ctx, json.RawMessage(`{"path":"/x"}`)) + var nt *files.NotTextError + Expect(errors.As(err, &nt)).To(BeTrue(), "a tool that is not a text reader is left alone") + _, err = tools[0].Run(ctx, json.RawMessage(`{"path":"/x"}`)) + return err.Error() + } + + It("names the skill of the file type when the chat has one", func() { + got := advice("report.docx", true, "docx", "xlsx") + Expect(got).To(Equal("report.docx is a binary file (43963 bytes), not text. Use the docx skill for .docx files (load_skill).")) + }) + + It("suggests a script when there is no skill and the chat can run commands", func() { + Expect(advice("report.docx", true, "xlsx")).To(HaveSuffix("Run a script with the commands tool instead, for example with python-docx.")) + Expect(advice("data.bin", true)).To(HaveSuffix("Run a script with the commands tool instead.")) + }) + + It("says so when the chat has neither", func() { + Expect(advice("report.pdf", false)).To(HaveSuffix("This chat has no tool that opens it.")) + }) +}) diff --git a/core/services/studio/files/tools.go b/core/services/studio/files/tools.go index bba26ee63..ef09bd0ac 100644 --- a/core/services/studio/files/tools.go +++ b/core/services/studio/files/tools.go @@ -469,11 +469,22 @@ func (s *Service) readText(ctx context.Context, a workspace.Actor, abs string, m return "", n, err } if !isText(data) || !utf8.Valid(data) { - return "", n, fmt.Errorf("%s is not a text file (%d bytes)", n.Name, n.Size) + return "", n, &NotTextError{Name: n.Name, Size: n.Size} } return string(data), n, nil } +// NotTextError means a text tool met a binary file. The host adds a hint about +// the tools the chat has, so the model knows what to do instead. +type NotTextError struct { + Name string + Size int64 +} + +func (e *NotTextError) Error() string { + return fmt.Sprintf("%s is not a text file (%d bytes)", e.Name, e.Size) +} + // put writes content as one change set of the chat. func (s *Service) put(ctx context.Context, a workspace.Actor, abs, content string, o workspace.PutOptions, verb string) (workspace.Node, error) { tx, err := s.ws.Begin(ctx, a, verb+lastName(abs)) diff --git a/core/services/studio/subagent.go b/core/services/studio/subagent.go index 3ad88816d..7b4a686c1 100644 --- a/core/services/studio/subagent.go +++ b/core/services/studio/subagent.go @@ -107,7 +107,7 @@ func (s *Service) chatTools(ctx context.Context, x session.Access) (session.Turn } fa := fx.Actor() nudge := s.newRulesNudge(ctx, x, rec) - tools = append(tools, nudge.wrap(s.fileNudgeTools(fx, s.announceFileChanges(fa, s.skillHintTools(fa, dropHidden(s.d.Files.Tools(fx), hidden)))))...) + tools = append(tools, nudge.wrap(s.fileNudgeTools(fx, s.announceFileChanges(fa, s.skillHintTools(fa, s.binaryHintTools(fa, cmds.State == CommandsAvailable, dropHidden(s.d.Files.Tools(fx), hidden))))))...) if s.d.Media != nil { tools = append(tools, s.skillHintTools(fa, dropHidden(s.d.Media.Tools(s.d.Files, x, rec.Model), hidden))...) }