diff --git a/core/services/studio/providerguard/guard_suite_test.go b/core/services/studio/providerguard/guard_suite_test.go new file mode 100644 index 0000000000..d20418f56e --- /dev/null +++ b/core/services/studio/providerguard/guard_suite_test.go @@ -0,0 +1,13 @@ +package providerguard_test + +import ( + "testing" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +func TestProviderGuard(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "provider guard suite") +} diff --git a/core/services/studio/providerguard/guard_test.go b/core/services/studio/providerguard/guard_test.go new file mode 100644 index 0000000000..557256aff4 --- /dev/null +++ b/core/services/studio/providerguard/guard_test.go @@ -0,0 +1,126 @@ +package providerguard_test + +// The test keeps core free of the names of shipped add-ons that provide a +// generic role: the text that a person or a model reads must come from the +// provider's declared title, or be generic. + +import ( + "go/ast" + "go/parser" + "go/token" + "os" + "path/filepath" + "regexp" + "strconv" + "strings" + + "github.com/mudler/LocalAI/core/services/studio/providerguard" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +// matcher finds a name as a whole word, case sensitive: "Sandbox" is the +// add-on, "sandbox" the ordinary noun for an isolated environment. +func matcher(names []string) *regexp.Regexp { + quoted := make([]string, len(names)) + for i, n := range names { + quoted[i] = regexp.QuoteMeta(n) + } + return regexp.MustCompile(`(^|[^\w-])(` + strings.Join(quoted, "|") + `)($|[^\w-])`) +} + +// allowed lists the string literals that may name a provider, by the file +// they are in and the text, with the reason. Each one must still be found. +var allowed = []struct{ file, text, why string }{ + {"core/http/endpoints/localai/api_instructions.go", "Telegram connectors", "the one-time conversion of the legacy Telegram connectors"}, + {"core/schema/chats.go", `example:"Sandbox"`, "a sample value in the API documentation"}, + {"core/services/addons/tools/consumers.go", "such as Telegram", "an example of a connector platform in a note for add-on authors"}, + {"core/services/routing/piipattern/builtins.go", "JSON Web Token", "the name of a token format"}, + {"core/services/studio/webtools.go", "Web search", "the label of the built-in web search tool"}, + {"core/services/studio/webtools.go", "Web:", "the built-in Web tool group, the web as a noun"}, + {"core/services/studio/connectorsvc.go", "Telegram", "one-time migration of the legacy Telegram connector settings"}, + {"core/services/studio/convert/run.go", "Telegram", "one-time migration of the legacy Telegram bindings"}, + {"core/cli/run.go", "Telegram, Slack", "help text that gives the connector platforms as examples"}, + {"core/http/auth/features.go", "Telegram bots", "the name of a feature permission; the platform is the content"}, + {"core/services/routing/piipattern/builtins.go", "Slack token", "a secret pattern: Slack is the issuer of the token, not a provider"}, +} + +var _ = Describe("the strings of core", func() { + It("reads the providers from the manifests of addons/", func() { + names, err := providerguard.Names() + Expect(err).NotTo(HaveOccurred()) + Expect(len(names)).To(BeNumerically(">=", 4), "expected the provider add-ons of addons/") + }) + + It("matches a name as a whole word and never the ordinary noun", func() { + re := matcher([]string{"Sandbox", "Simple Office"}) + Expect(re.MatchString("Install the Sandbox to isolate")).To(BeTrue()) + Expect(re.MatchString("Open Simple Office")).To(BeTrue()) + Expect(re.MatchString("curl works in the sandbox")).To(BeFalse()) + Expect(re.MatchString("a Sandboxed frame")).To(BeFalse()) + }) + + It("names no provider add-on in a user-facing or model-facing string outside the allow-list", func() { + root, err := providerguard.RepoRoot() + Expect(err).NotTo(HaveOccurred()) + names, err := providerguard.Names() + Expect(err).NotTo(HaveOccurred()) + re := matcher(names) + used := map[int]bool{} + var found []string + for _, dir := range []string{"core", "pkg"} { + err := filepath.WalkDir(filepath.Join(root, dir), func(path string, d os.DirEntry, err error) error { + if err != nil { + return err + } + if d.IsDir() { + // The sandbox backend is the add-on's own code that lives + // in core for now; the other trees are not ours. + switch d.Name() { + case "node_modules", "vendor", "testdata", "sandbox", "react-ui": + return filepath.SkipDir + } + return nil + } + if !strings.HasSuffix(path, ".go") || strings.HasSuffix(path, "_test.go") { + return nil + } + fset := token.NewFileSet() + f, perr := parser.ParseFile(fset, path, nil, parser.SkipObjectResolution) + if perr != nil { + return nil + } + rel, _ := filepath.Rel(root, path) + rel = filepath.ToSlash(rel) + ast.Inspect(f, func(n ast.Node) bool { + lit, ok := n.(*ast.BasicLit) + if !ok || lit.Kind != token.STRING { + return true + } + s, uerr := strconv.Unquote(lit.Value) + if uerr != nil { + s = lit.Value + } + hit := re.FindStringSubmatch(s) + if hit == nil { + return true + } + for i, a := range allowed { + if a.file == rel && strings.Contains(s, a.text) { + used[i] = true + return true + } + } + found = append(found, rel+":"+strconv.Itoa(fset.Position(lit.Pos()).Line)+": "+hit[2]+": "+strconv.Quote(s)) + return true + }) + return nil + }) + Expect(err).NotTo(HaveOccurred()) + } + Expect(found).To(BeEmpty(), "build the text from the provider's declared title or word it for any provider") + for i, a := range allowed { + Expect(used[i]).To(BeTrue(), "allow-list entry %s %q (%s) no longer matches; remove it", a.file, a.text, a.why) + } + }) +}) diff --git a/core/services/studio/providerguard/names.go b/core/services/studio/providerguard/names.go new file mode 100644 index 0000000000..d69f12f4ed --- /dev/null +++ b/core/services/studio/providerguard/names.go @@ -0,0 +1,97 @@ +// Package providerguard reads the shipped add-ons so tests can check that core +// names none of the ones that provide a generic role. Only tests import it. +package providerguard + +import ( + "encoding/json" + "fmt" + "os" + "path/filepath" + "sort" + "strings" +) + +// RepoRoot is the module root, found from this package's directory. +func RepoRoot() (string, error) { + dir, err := os.Getwd() + if err != nil { + return "", err + } + for { + if _, err := os.Stat(filepath.Join(dir, "go.mod")); err == nil { + return dir, nil + } + parent := filepath.Dir(dir) + if parent == dir { + return "", fmt.Errorf("go.mod not found above %s", dir) + } + dir = parent + } +} + +type manifest struct { + ID string `json:"id"` + Capabilities struct { + ToolDefinitions []struct{ Shadows string } `json:"toolDefinitions"` + SkillDefinitions []struct{ Shadows string } `json:"skillDefinitions"` + Connector json.RawMessage `json:"connector"` + ExecutionProv json.RawMessage `json:"executionProvider"` + UI struct { + Title json.RawMessage `json:"title"` + } `json:"ui"` + } `json:"capabilities"` +} + +// Names reads addons/*/manifest.json and returns the names of the +// add-ons that provide a generic role (they shadow a built-in tool or skill, +// serve as a connector, or declare the execution provider): the capitalised +// id and every declared title. The other add-ons (voice, face, tables ...) +// are ordinary words of the model features that core owns. +func Names() ([]string, error) { + root, err := RepoRoot() + if err != nil { + return nil, err + } + files, err := filepath.Glob(filepath.Join(root, "addons", "*", "manifest.json")) + if err != nil || len(files) == 0 { + return nil, fmt.Errorf("no add-on manifests found: %v", err) + } + set := map[string]bool{} + for _, f := range files { + raw, err := os.ReadFile(f) + if err != nil { + return nil, err + } + var m manifest + if json.Unmarshal(raw, &m) != nil || m.ID == "" { + continue + } + role := len(m.Capabilities.Connector) > 0 && string(m.Capabilities.Connector) != "null" + role = role || (len(m.Capabilities.ExecutionProv) > 0 && string(m.Capabilities.ExecutionProv) != "null") + for _, d := range m.Capabilities.ToolDefinitions { + role = role || d.Shadows != "" + } + for _, d := range m.Capabilities.SkillDefinitions { + role = role || d.Shadows != "" + } + if !role { + continue + } + set[strings.ToUpper(m.ID[:1])+m.ID[1:]] = true + var title string + var titles map[string]string + if json.Unmarshal(m.Capabilities.UI.Title, &title) == nil && title != "" { + set[title] = true + } else if json.Unmarshal(m.Capabilities.UI.Title, &titles) == nil { + for _, v := range titles { + set[v] = true + } + } + } + var out []string + for n := range set { + out = append(out, n) + } + sort.Strings(out) + return out, nil +} diff --git a/core/services/studio/providerstrings_internal_test.go b/core/services/studio/providerstrings_internal_test.go new file mode 100644 index 0000000000..e83814d45d --- /dev/null +++ b/core/services/studio/providerstrings_internal_test.go @@ -0,0 +1,57 @@ +package studio + +import ( + "regexp" + "strings" + + "github.com/mudler/LocalAI/core/services/studio/egress" + "github.com/mudler/LocalAI/core/services/studio/providerguard" + "github.com/mudler/LocalAI/core/services/studio/session" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +// The text that the prompt builders and the Tools menu reasons produce names +// the add-on that runs commands by the title it declares, never by a name that +// core knows. The names come from addons/*/manifest.json at run time. +var _ = Describe("the prompts that talk about the add-on that runs commands", func() { + It("name no shipped provider, with a declared title or without one", func() { + names, err := providerguard.Names() + Expect(err).NotTo(HaveOccurred()) + var words []string + for _, n := range names { + // A short name can be an ordinary word (web), so it is checked as a title only. + if len(n) >= 6 || strings.ContainsAny(n, "- ") { + words = append(words, regexp.QuoteMeta(strings.ToLower(n))) + } + } + re := regexp.MustCompile(`(^|[^\w-])(` + strings.Join(words, "|") + `)($|[^\w-])`) + + var texts []string + for _, base := range []Commands{{Title: "Zeta Runner"}, {}} { + for _, st := range []CommandState{CommandsAvailable, CommandsOff, CommandsUnavailable, CommandsNone, CommandsSwitchedOff} { + for _, iso := range []string{"isolated", "host", ""} { + c := base + c.State, c.Isolation, c.Tool, c.Reason, c.Consent = st, iso, "bash", "stopped", "ask" + texts = append(texts, shellReason(c, ""), commandsBlock(c, true, true, true, session.Place{})) + } + } + } + for _, p := range []egress.Preset{egress.PresetOff, egress.PresetRegistries, egress.PresetAll} { + for _, flag := range []bool{true, false} { + reg := egress.BuiltinRegistries() + texts = append(texts, egress.Describe(p, reg, flag), egress.DescribeLimit(p, egress.PresetOff, reg, flag), egress.DescribeLimit(p, egress.PresetRegistries, reg, flag), egress.Hint(p, flag)) + } + } + texts = append(texts, scopedHostBashReason, hostBashDescription) + for _, text := range texts { + Expect(re.FindStringSubmatch(strings.ToLower(text))).To(BeNil(), "a prompt names a shipped add-on: %q", text) + } + }) + + It("carries the declared title, or says generically what runs commands", func() { + Expect(shellReason(Commands{State: CommandsOff, Title: "Zeta Runner"}, "")).To(ContainSubstring("the Zeta Runner add-on")) + Expect(shellReason(Commands{State: CommandsUnavailable, Title: "Zeta Runner", Reason: "no vm"}, "")).To(ContainSubstring("the Zeta Runner add-on cannot run commands here: no vm")) + Expect(shellReason(Commands{State: CommandsOff}, "")).To(ContainSubstring("the add-on that runs commands")) + }) +})