feat(addons): let a manifest declare the context files the host loads

An add-on that keeps small notes for the person can now name the index
files that the host puts in the prompt of a chat. The declaration is
capabilities.contextFiles: at most one file for each scope (user, agent,
channel), each a path under Memory/ that ends in .md, with an optional
byte cap that can only lower the host's 2 KiB cap.

The field is new, so a manifest that uses it must accept manifest API 2.
The add-on only names paths. The host reads them as the person, so the
declaration gives the add-on no access to files.

Assisted-by: Claude:claude-sonnet-5-5
Signed-off-by: Ettore Di Giacinto <mudler@localai.io>

pkg/addons/contract/README.md

@@ -61,3 +61,13 @@ FLAG_DIARIZATION FLAG_SOUND_CLASSIFICATION FLAG_REALTIME_AUDIO FLAG_SCORE
 FLAG_DEPTH FLAG_TOKEN_CLASSIFY FLAG_3D FLAG_3D_ANIMATION FLAG_DECISIONS
 FLAG_UPSCALE FLAG_MOTION FLAG_FORECAST
 ```
+
+## Context files
+
+`capabilities.contextFiles` names up to three small index files (one for each scope: user, agent, channel) that the host loads into the prompt of a chat. It needs manifest api 2.
+
+```json
+{"contextFiles":[{"scope":"user","path":"Memory/index.md"},{"scope":"agent","path":"Memory/agents/{agent}/index.md","cap":1024}]}
+```
+
+Every path lies under `Memory/`, ends in `.md` and holds no parent segment. An agent path holds `{agent}` once and a channel path holds `{channel}` once; the host fills them from the chat, never from the model. `cap` lowers the 2 KiB index cap; it cannot raise it. The add-on declares and the host reads: the add-on gains no access to the files.

pkg/addons/contract/contextfiles.go

@@ -0,0 +1,125 @@
+// SPDX-License-Identifier: MIT
+package contract
+
+import (
+	"fmt"
+	"regexp"
+	"strings"
+	"unicode"
+)
+
+// MinContextFilesAPI is the first manifest API version that knows the context
+// files field. A manifest that declares one must accept it.
+const MinContextFilesAPI uint64 = 2
+
+// Limits of the context files an add-on declares.
+const (
+	// MaxContextFiles is the most files one add-on declares: one for each
+	// scope.
+	MaxContextFiles = 3
+	// MaxContextFileBytes is the most bytes of one index the prompt carries.
+	// It is the cap of the host; a smaller Cap lowers it.
+	MaxContextFileBytes = 2 << 10
+	// ContextFilesRoot is the folder of the user's home that every declared
+	// file lies in. An add-on cannot name any other part of the home.
+	ContextFilesRoot = "Memory/"
+	// maxContextPath bounds the path of a declared file.
+	maxContextPath = 200
+)
+
+// ContextFile names one small index file that the host loads into the prompt
+// of a chat, under a fixed header that asks the model to read the notes the
+// index points to. The add-on declares where its files are; the host reads
+// them as the user, caps them and builds the prompt. The add-on is not asked
+// at each turn and holds no authority over the files: it names a path under
+// ContextFilesRoot and nothing else.
+//
+// A user file applies to every chat of the person. An agent file holds the
+// placeholder {agent} once and applies to the chats of one saved agent. A
+// channel file holds {channel} once and applies to the chats that one
+// connector conversation started. The host fills a placeholder from the chat,
+// never from the model.
+type ContextFile struct {
+	Scope ChangeScope `json:"scope"`
+	// Path is relative to the user's home, under ContextFilesRoot, and ends
+	// in .md.
+	Path string `json:"path"`
+	// Cap is the most bytes of the index the prompt carries. Zero means
+	// MaxContextFileBytes.
+	Cap int `json:"cap,omitempty"`
+}
+
+var contextPlaceholder = regexp.MustCompile(`\{[^{}]*\}`)
+
+// placeholderOf is the one placeholder a scope's path holds, or "".
+func placeholderOf(s ChangeScope) string {
+	switch s {
+	case ScopeAgent:
+		return "{agent}"
+	case ScopeChannel:
+		return "{channel}"
+	}
+	return ""
+}
+
+func (f ContextFile) validate() error {
+	bad := func(format string, a ...any) error {
+		return fmt.Errorf("context file %q: %s", f.Path, fmt.Sprintf(format, a...))
+	}
+	if !ValidChangeScope(string(f.Scope)) {
+		return bad("the scope must be user, agent or channel")
+	}
+	if f.Cap < 0 || f.Cap > MaxContextFileBytes {
+		return bad("the cap must be 0 to %d bytes", MaxContextFileBytes)
+	}
+	p := f.Path
+	if len(p) > maxContextPath || !strings.HasPrefix(p, ContextFilesRoot) || !strings.HasSuffix(p, ".md") {
+		return bad("the path must lie under %s, end in .md and have at most %d bytes", ContextFilesRoot, maxContextPath)
+	}
+	found := contextPlaceholder.FindAllString(p, -1)
+	if want := placeholderOf(f.Scope); (want == "" && len(found) > 0) || (want != "" && (len(found) != 1 || found[0] != want)) {
+		return bad("a %s path holds %s once and no other placeholder", f.Scope, placeholderOrNone(want))
+	}
+	rest := contextPlaceholder.ReplaceAllString(p, "")
+	if strings.ContainsAny(rest, `{}\`) || strings.IndexFunc(rest, unicode.IsControl) >= 0 {
+		return bad("the path holds a character that is not allowed")
+	}
+	for seg := range strings.SplitSeq(p, "/") {
+		if seg == "" || seg == "." || seg == ".." {
+			return bad("the path holds an empty or parent segment")
+		}
+	}
+	return nil
+}
+
+func placeholderOrNone(p string) string {
+	if p == "" {
+		return "no placeholder"
+	}
+	return p
+}
+
+// validateContextFiles checks the declaration against the manifest.
+func validateContextFiles(m Manifest) error {
+	files := m.Capabilities.ContextFiles
+	if len(files) == 0 {
+		return nil
+	}
+	if m.API.Max < MinContextFilesAPI {
+		return fmt.Errorf("context files need api %d", MinContextFilesAPI)
+	}
+	if len(files) > MaxContextFiles {
+		return fmt.Errorf("too many context files: at most %d", MaxContextFiles)
+	}
+	seen := map[ChangeScope]bool{}
+	for _, f := range files {
+		if err := f.validate(); err != nil {
+			return err
+		}
+		if seen[f.Scope] {
+			return fmt.Errorf("context files: a second file for the %s scope", f.Scope)
+		}
+		seen[f.Scope] = true
+	}
+	return nil
+}

pkg/addons/contract/contextfiles_test.go

@@ -0,0 +1,84 @@
+// SPDX-License-Identifier: MIT
+package contract
+
+import (
+	"encoding/json"
+	"strings"
+
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+)
+
+var _ = Describe("The context files declaration", func() {
+	notes := []ContextFile{
+		{Scope: ScopeUser, Path: "Memory/index.md"},
+		{Scope: ScopeAgent, Path: "Memory/agents/{agent}/index.md", Cap: 1024},
+		{Scope: ScopeChannel, Path: "Memory/channels/{channel}/index.md"},
+	}
+	withFiles := func(files []ContextFile) Manifest {
+		m, err := ParseManifest(fixture())
+		Expect(err).NotTo(HaveOccurred())
+		m.API = VersionRange{Min: 2, Max: 2}
+		m.Capabilities.ContextFiles = files
+		return m
+	}
+	host := func() HostCompatibility {
+		h := compatible()
+		h.API, h.APIMin = HostAPIVersion, HostAPIMin
+		return h
+	}
+
+	It("is valid with one file for each scope", func() {
+		Expect(ValidateManifest(withFiles(notes), host())).To(Succeed())
+	})
+
+	It("needs a manifest that accepts API 2 to declare context files", func() {
+		m := withFiles(notes)
+		m.API = VersionRange{Min: 1, Max: 1}
+		Expect(ValidateManifest(m, host())).To(MatchError(ContainSubstring("api 2")))
+	})
+
+	It("refuses a path outside the Memory folder, an unsafe path and a wrong placeholder", func() {
+		for _, bad := range []ContextFile{
+			{Scope: ScopeUser, Path: "Documents/index.md"},
+			{Scope: ScopeUser, Path: "Memory/../Documents/index.md"},
+			{Scope: ScopeUser, Path: "/Memory/index.md"},
+			{Scope: ScopeUser, Path: "Memory/index.txt"},
+			{Scope: ScopeUser, Path: "Memory/{agent}/index.md"},
+			{Scope: ScopeUser, Path: "Memory//index.md"},
+			{Scope: ScopeUser, Path: "Memory\\index.md"},
+			{Scope: ScopeUser, Path: "Memory/in\x00dex.md"},
+			{Scope: ScopeAgent, Path: "Memory/agents/index.md"},
+			{Scope: ScopeAgent, Path: "Memory/agents/{channel}/index.md"},
+			{Scope: ScopeChannel, Path: "Memory/{channel}/{channel}/index.md"},
+			{Scope: ScopeChannel, Path: "Memory/{x}/{channel}/index.md"},
+			{Scope: ScopeUser, Path: "Memory/" + strings.Repeat("a", 200) + ".md"},
+			{Scope: "everyone", Path: "Memory/index.md"},
+			{Scope: "", Path: "Memory/index.md"},
+		} {
+			Expect(ValidateManifest(withFiles([]ContextFile{bad}), host())).NotTo(Succeed(), "%+v", bad)
+		}
+	})
+
+	It("refuses a cap that is negative or above the host's index cap", func() {
+		for _, c := range []int{-1, MaxContextFileBytes + 1} {
+			Expect(ValidateManifest(withFiles([]ContextFile{{Scope: ScopeUser, Path: "Memory/index.md", Cap: c}}), host())).NotTo(Succeed())
+		}
+		Expect(ValidateManifest(withFiles([]ContextFile{{Scope: ScopeUser, Path: "Memory/index.md", Cap: MaxContextFileBytes}}), host())).To(Succeed())
+	})
+
+	It("refuses two files for one scope and more files than scopes", func() {
+		twice := []ContextFile{{Scope: ScopeUser, Path: "Memory/index.md"}, {Scope: ScopeUser, Path: "Memory/other.md"}}
+		Expect(ValidateManifest(withFiles(twice), host())).NotTo(Succeed())
+		many := append(append([]ContextFile{}, notes...), ContextFile{Scope: ScopeUser, Path: "Memory/x.md"})
+		Expect(ValidateManifest(withFiles(many), host())).NotTo(Succeed())
+	})
+
+	It("reads capabilities.contextFiles from JSON and refuses an unknown field in it", func() {
+		var c Capabilities
+		Expect(json.Unmarshal([]byte(`{"service":true,"contextFiles":[{"scope":"user","path":"Memory/index.md","cap":2048}]}`), &c)).To(Succeed())
+		Expect(c.ContextFiles).To(Equal([]ContextFile{{Scope: ScopeUser, Path: "Memory/index.md", Cap: 2048}}))
+		Expect(json.Unmarshal([]byte(`{"service":true,"contextFiles":[{"scope":"user","path":"Memory/index.md","all":true}]}`), &c)).NotTo(Succeed())
+		Expect(json.Unmarshal([]byte(`{"service":true,"ContextFiles":[]}`), &c)).NotTo(Succeed())
+	})
+})

pkg/addons/contract/exactkeys.go

@@ -42,7 +42,7 @@ func rejectExtensionAliases(b []byte, keys ...string) error {
 }
 
 func (c *Capabilities) UnmarshalJSON(b []byte) error {
-	if err := rejectExtensionAliases(b, "executionProvider", "hostOperations", "network"); err != nil {
+	if err := rejectExtensionAliases(b, "executionProvider", "hostOperations", "network", "contextFiles"); err != nil {
 		return err
 	}
 	type wire Capabilities

pkg/addons/contract/manifest.go

@@ -134,6 +134,10 @@ type Capabilities struct {
 	// Network declares the network policy of the commands the add-on runs
 	// (see network.go). It attaches the network-policy host control.
 	Network *NetworkCapability `json:"network,omitempty"`
+	// ContextFiles name small index files of the user's home that the host
+	// loads into the prompt of a chat (see contextfiles.go). They need
+	// manifest API 2.
+	ContextFiles []ContextFile `json:"contextFiles,omitempty"`
 }
 
 // ResourceTools maps the host's UI-only resource management operations to
@@ -397,6 +401,9 @@ func ValidateManifest(m Manifest, h HostCompatibility) error {
 	if e := c.Network.validate(m); e != nil {
 		return e
 	}
+	if e := validateContextFiles(m); e != nil {
+		return e
+	}
 	if e := validateResourceTools(c); e != nil {
 		return e
 	}

scripts/addons/suites.json

@@ -414,6 +414,12 @@
           "needs the egress check among the host operations",
           "refuses an unknown kind, a bad title and a bad or repeated preset",
           "reads capabilities.network from JSON and refuses an unknown field in it",
+          "is valid with one file for each scope",
+          "needs a manifest that accepts API 2 to declare context files",
+          "refuses a path outside the Memory folder, an unsafe path and a wrong placeholder",
+          "refuses a cap that is negative or above the host's index cap",
+          "refuses two files for one scope and more files than scopes",
+          "reads capabilities.contextFiles from JSON and refuses an unknown field in it",
           "is drawn for a declared capability that holds the grant, with no bash tool",
           "is not drawn for a declared capability without the grant",
           "is drawn for a manifest that declares no network and replaces bash, as before",