test(studio): fail when core names an add-on that provides a role

Read addons/*/manifest.json at run time and find the add-ons that shadow a
built-in tool or skill, serve as a connector or declare the execution
provider. One spec scans the string literals of core and pkg for their
names; another builds the command prompts and the network text for every
state and checks that none names a shipped provider. A short allow-list
holds the strings that name a platform or a token format as content.

Assisted-by: Claude Code:claude-sonnet-5-5
Signed-off-by: Ettore Di Giacinto <mudler@localai.io>

core/services/studio/providerguard/guard_suite_test.go

@@ -0,0 +1,13 @@
+package providerguard_test
+
+import (
+	"testing"
+
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+)
+
+func TestProviderGuard(t *testing.T) {
+	RegisterFailHandler(Fail)
+	RunSpecs(t, "provider guard suite")
+}

core/services/studio/providerguard/guard_test.go

@@ -0,0 +1,126 @@
+package providerguard_test
+
+// The test keeps core free of the names of shipped add-ons that provide a
+// generic role: the text that a person or a model reads must come from the
+// provider's declared title, or be generic.
+
+import (
+	"go/ast"
+	"go/parser"
+	"go/token"
+	"os"
+	"path/filepath"
+	"regexp"
+	"strconv"
+	"strings"
+
+	"github.com/mudler/LocalAI/core/services/studio/providerguard"
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+)
+
+// matcher finds a name as a whole word, case sensitive: "Sandbox" is the
+// add-on, "sandbox" the ordinary noun for an isolated environment.
+func matcher(names []string) *regexp.Regexp {
+	quoted := make([]string, len(names))
+	for i, n := range names {
+		quoted[i] = regexp.QuoteMeta(n)
+	}
+	return regexp.MustCompile(`(^|[^\w-])(` + strings.Join(quoted, "|") + `)($|[^\w-])`)
+}
+
+// allowed lists the string literals that may name a provider, by the file
+// they are in and the text, with the reason. Each one must still be found.
+var allowed = []struct{ file, text, why string }{
+	{"core/http/endpoints/localai/api_instructions.go", "Telegram connectors", "the one-time conversion of the legacy Telegram connectors"},
+	{"core/schema/chats.go", `example:"Sandbox"`, "a sample value in the API documentation"},
+	{"core/services/addons/tools/consumers.go", "such as Telegram", "an example of a connector platform in a note for add-on authors"},
+	{"core/services/routing/piipattern/builtins.go", "JSON Web Token", "the name of a token format"},
+	{"core/services/studio/webtools.go", "Web search", "the label of the built-in web search tool"},
+	{"core/services/studio/webtools.go", "Web:", "the built-in Web tool group, the web as a noun"},
+	{"core/services/studio/connectorsvc.go", "Telegram", "one-time migration of the legacy Telegram connector settings"},
+	{"core/services/studio/convert/run.go", "Telegram", "one-time migration of the legacy Telegram bindings"},
+	{"core/cli/run.go", "Telegram, Slack", "help text that gives the connector platforms as examples"},
+	{"core/http/auth/features.go", "Telegram bots", "the name of a feature permission; the platform is the content"},
+	{"core/services/routing/piipattern/builtins.go", "Slack token", "a secret pattern: Slack is the issuer of the token, not a provider"},
+}
+
+var _ = Describe("the strings of core", func() {
+	It("reads the providers from the manifests of addons/", func() {
+		names, err := providerguard.Names()
+		Expect(err).NotTo(HaveOccurred())
+		Expect(len(names)).To(BeNumerically(">=", 4), "expected the provider add-ons of addons/")
+	})
+
+	It("matches a name as a whole word and never the ordinary noun", func() {
+		re := matcher([]string{"Sandbox", "Simple Office"})
+		Expect(re.MatchString("Install the Sandbox to isolate")).To(BeTrue())
+		Expect(re.MatchString("Open Simple Office")).To(BeTrue())
+		Expect(re.MatchString("curl works in the sandbox")).To(BeFalse())
+		Expect(re.MatchString("a Sandboxed frame")).To(BeFalse())
+	})
+
+	It("names no provider add-on in a user-facing or model-facing string outside the allow-list", func() {
+		root, err := providerguard.RepoRoot()
+		Expect(err).NotTo(HaveOccurred())
+		names, err := providerguard.Names()
+		Expect(err).NotTo(HaveOccurred())
+		re := matcher(names)
+		used := map[int]bool{}
+		var found []string
+		for _, dir := range []string{"core", "pkg"} {
+			err := filepath.WalkDir(filepath.Join(root, dir), func(path string, d os.DirEntry, err error) error {
+				if err != nil {
+					return err
+				}
+				if d.IsDir() {
+					// The sandbox backend is the add-on's own code that lives
+					// in core for now; the other trees are not ours.
+					switch d.Name() {
+					case "node_modules", "vendor", "testdata", "sandbox", "react-ui":
+						return filepath.SkipDir
+					}
+					return nil
+				}
+				if !strings.HasSuffix(path, ".go") || strings.HasSuffix(path, "_test.go") {
+					return nil
+				}
+				fset := token.NewFileSet()
+				f, perr := parser.ParseFile(fset, path, nil, parser.SkipObjectResolution)
+				if perr != nil {
+					return nil
+				}
+				rel, _ := filepath.Rel(root, path)
+				rel = filepath.ToSlash(rel)
+				ast.Inspect(f, func(n ast.Node) bool {
+					lit, ok := n.(*ast.BasicLit)
+					if !ok || lit.Kind != token.STRING {
+						return true
+					}
+					s, uerr := strconv.Unquote(lit.Value)
+					if uerr != nil {
+						s = lit.Value
+					}
+					hit := re.FindStringSubmatch(s)
+					if hit == nil {
+						return true
+					}
+					for i, a := range allowed {
+						if a.file == rel && strings.Contains(s, a.text) {
+							used[i] = true
+							return true
+						}
+					}
+					found = append(found, rel+":"+strconv.Itoa(fset.Position(lit.Pos()).Line)+": "+hit[2]+": "+strconv.Quote(s))
+					return true
+				})
+				return nil
+			})
+			Expect(err).NotTo(HaveOccurred())
+		}
+		Expect(found).To(BeEmpty(), "build the text from the provider's declared title or word it for any provider")
+		for i, a := range allowed {
+			Expect(used[i]).To(BeTrue(), "allow-list entry %s %q (%s) no longer matches; remove it", a.file, a.text, a.why)
+		}
+	})
+})

core/services/studio/providerguard/names.go

@@ -0,0 +1,97 @@
+// Package providerguard reads the shipped add-ons so tests can check that core
+// names none of the ones that provide a generic role. Only tests import it.
+package providerguard
+
+import (
+	"encoding/json"
+	"fmt"
+	"os"
+	"path/filepath"
+	"sort"
+	"strings"
+)
+
+// RepoRoot is the module root, found from this package's directory.
+func RepoRoot() (string, error) {
+	dir, err := os.Getwd()
+	if err != nil {
+		return "", err
+	}
+	for {
+		if _, err := os.Stat(filepath.Join(dir, "go.mod")); err == nil {
+			return dir, nil
+		}
+		parent := filepath.Dir(dir)
+		if parent == dir {
+			return "", fmt.Errorf("go.mod not found above %s", dir)
+		}
+		dir = parent
+	}
+}
+
+type manifest struct {
+	ID           string `json:"id"`
+	Capabilities struct {
+		ToolDefinitions  []struct{ Shadows string } `json:"toolDefinitions"`
+		SkillDefinitions []struct{ Shadows string } `json:"skillDefinitions"`
+		Connector        json.RawMessage            `json:"connector"`
+		ExecutionProv    json.RawMessage            `json:"executionProvider"`
+		UI               struct {
+			Title json.RawMessage `json:"title"`
+		} `json:"ui"`
+	} `json:"capabilities"`
+}
+
+// Names reads addons/*/manifest.json and returns the names of the
+// add-ons that provide a generic role (they shadow a built-in tool or skill,
+// serve as a connector, or declare the execution provider): the capitalised
+// id and every declared title. The other add-ons (voice, face, tables ...)
+// are ordinary words of the model features that core owns.
+func Names() ([]string, error) {
+	root, err := RepoRoot()
+	if err != nil {
+		return nil, err
+	}
+	files, err := filepath.Glob(filepath.Join(root, "addons", "*", "manifest.json"))
+	if err != nil || len(files) == 0 {
+		return nil, fmt.Errorf("no add-on manifests found: %v", err)
+	}
+	set := map[string]bool{}
+	for _, f := range files {
+		raw, err := os.ReadFile(f)
+		if err != nil {
+			return nil, err
+		}
+		var m manifest
+		if json.Unmarshal(raw, &m) != nil || m.ID == "" {
+			continue
+		}
+		role := len(m.Capabilities.Connector) > 0 && string(m.Capabilities.Connector) != "null"
+		role = role || (len(m.Capabilities.ExecutionProv) > 0 && string(m.Capabilities.ExecutionProv) != "null")
+		for _, d := range m.Capabilities.ToolDefinitions {
+			role = role || d.Shadows != ""
+		}
+		for _, d := range m.Capabilities.SkillDefinitions {
+			role = role || d.Shadows != ""
+		}
+		if !role {
+			continue
+		}
+		set[strings.ToUpper(m.ID[:1])+m.ID[1:]] = true
+		var title string
+		var titles map[string]string
+		if json.Unmarshal(m.Capabilities.UI.Title, &title) == nil && title != "" {
+			set[title] = true
+		} else if json.Unmarshal(m.Capabilities.UI.Title, &titles) == nil {
+			for _, v := range titles {
+				set[v] = true
+			}
+		}
+	}
+	var out []string
+	for n := range set {
+		out = append(out, n)
+	}
+	sort.Strings(out)
+	return out, nil
+}

core/services/studio/providerstrings_internal_test.go

@@ -0,0 +1,57 @@
+package studio
+
+import (
+	"regexp"
+	"strings"
+
+	"github.com/mudler/LocalAI/core/services/studio/egress"
+	"github.com/mudler/LocalAI/core/services/studio/providerguard"
+	"github.com/mudler/LocalAI/core/services/studio/session"
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+)
+
+// The text that the prompt builders and the Tools menu reasons produce names
+// the add-on that runs commands by the title it declares, never by a name that
+// core knows. The names come from addons/*/manifest.json at run time.
+var _ = Describe("the prompts that talk about the add-on that runs commands", func() {
+	It("name no shipped provider, with a declared title or without one", func() {
+		names, err := providerguard.Names()
+		Expect(err).NotTo(HaveOccurred())
+		var words []string
+		for _, n := range names {
+			// A short name can be an ordinary word (web), so it is checked as a title only.
+			if len(n) >= 6 || strings.ContainsAny(n, "- ") {
+				words = append(words, regexp.QuoteMeta(strings.ToLower(n)))
+			}
+		}
+		re := regexp.MustCompile(`(^|[^\w-])(` + strings.Join(words, "|") + `)($|[^\w-])`)
+
+		var texts []string
+		for _, base := range []Commands{{Title: "Zeta Runner"}, {}} {
+			for _, st := range []CommandState{CommandsAvailable, CommandsOff, CommandsUnavailable, CommandsNone, CommandsSwitchedOff} {
+				for _, iso := range []string{"isolated", "host", ""} {
+					c := base
+					c.State, c.Isolation, c.Tool, c.Reason, c.Consent = st, iso, "bash", "stopped", "ask"
+					texts = append(texts, shellReason(c, ""), commandsBlock(c, true, true, true, session.Place{}))
+				}
+			}
+		}
+		for _, p := range []egress.Preset{egress.PresetOff, egress.PresetRegistries, egress.PresetAll} {
+			for _, flag := range []bool{true, false} {
+				reg := egress.BuiltinRegistries()
+				texts = append(texts, egress.Describe(p, reg, flag), egress.DescribeLimit(p, egress.PresetOff, reg, flag), egress.DescribeLimit(p, egress.PresetRegistries, reg, flag), egress.Hint(p, flag))
+			}
+		}
+		texts = append(texts, scopedHostBashReason, hostBashDescription)
+		for _, text := range texts {
+			Expect(re.FindStringSubmatch(strings.ToLower(text))).To(BeNil(), "a prompt names a shipped add-on: %q", text)
+		}
+	})
+
+	It("carries the declared title, or says generically what runs commands", func() {
+		Expect(shellReason(Commands{State: CommandsOff, Title: "Zeta Runner"}, "")).To(ContainSubstring("the Zeta Runner add-on"))
+		Expect(shellReason(Commands{State: CommandsUnavailable, Title: "Zeta Runner", Reason: "no vm"}, "")).To(ContainSubstring("the Zeta Runner add-on cannot run commands here: no vm"))
+		Expect(shellReason(Commands{State: CommandsOff}, "")).To(ContainSubstring("the add-on that runs commands"))
+	})
+})