From 484813f856edbef7de70619a1a9aeba66e4626be Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Wed, 21 Oct 2026 17:51:09 +0000 Subject: [PATCH] feat(addons): add a kind to library entries and read unknown kinds tolerantly A library entry now has an optional kind: addon (the default when it is absent), backend or model. Every index written before kinds existed stays valid and its entries are add-ons. A reader keeps the entries of the kinds it knows and skips the others without reading their fields, so a newer kind never makes an older host refuse the whole index. The refresh result and the library state report how many entries were skipped and of which kinds, and the refresh logs a warning. A kind that is not a plain kind name still makes the index malformed. Search lists add-ons unless the caller asks for a kind, and Resolve returns add-ons only, so an add-on install cannot be pointed at a backend image. A library policy may name the signer with an anchored identityRegex instead of an exact identity, so builds from master and from tags both verify. An add-on install refuses a policy that carries one. Assisted-by: Claude Code:claude-sonnet-5-5 go Signed-off-by: Ettore Di Giacinto --- core/services/addons/catalog/catalog.go | 58 ++++-- core/services/addons/catalog/index.go | 66 +++++-- core/services/addons/catalog/kind.go | 90 +++++++++ core/services/addons/catalog/kind_test.go | 181 ++++++++++++++++++ core/services/addons/catalog/sources.go | 121 +++++++++--- core/services/addons/gateway/management.go | 11 +- core/services/addons/packages/install.go | 5 + core/services/addons/packages/install_test.go | 13 ++ 8 files changed, 486 insertions(+), 59 deletions(-) create mode 100644 core/services/addons/catalog/kind.go create mode 100644 core/services/addons/catalog/kind_test.go diff --git a/core/services/addons/catalog/catalog.go b/core/services/addons/catalog/catalog.go index 28dcbc784..d09ebdad2 100644 --- a/core/services/addons/catalog/catalog.go +++ b/core/services/addons/catalog/catalog.go @@ -24,6 +24,10 @@ type VerificationPolicy struct { RequireSignature bool `json:"requireSignature"` Issuer string `json:"issuer,omitempty"` Identity string `json:"identity,omitempty"` + // IdentityRegex is an anchored pattern for the signer identity, used + // instead of Identity so that master and tagged builds both verify. Only a + // library that serves backends may use it; an add-on install refuses it. + IdentityRegex string `json:"identityRegex,omitempty"` } // EntryUI is optional display metadata a catalog may offer for an entry. It is @@ -59,21 +63,30 @@ type Requirements struct { } type Release struct { + // Kind says what the entry installs: addon (the default when absent), + // backend or model. See ReadIndex for how a reader treats a kind it does + // not know. + Kind Kind `json:"kind,omitempty"` Channel string `json:"channel,omitempty"` AllowLoopbackHTTP bool `json:"allowLoopbackHTTP,omitempty"` // Selection is stamped by Resolve, not trusted from remote metadata. - Selection string `json:"selection,omitempty"` - SourceID string `json:"sourceId,omitempty"` - SourceURL string `json:"sourceUrl,omitempty"` - AddonID string `json:"addonId"` - Name string `json:"name"` - Description string `json:"description,omitempty"` - Version string `json:"version"` - Artifact string `json:"artifact"` - Digest string `json:"digest"` - OS string `json:"os"` - Arch string `json:"arch"` - Policy VerificationPolicy `json:"policy"` + Selection string `json:"selection,omitempty"` + SourceID string `json:"sourceId,omitempty"` + SourceURL string `json:"sourceUrl,omitempty"` + // AddonID is the identifier of the entry for every kind. For a backend it + // is the backend name, which is also its directory name. + AddonID string `json:"addonId"` + Name string `json:"name"` + Description string `json:"description,omitempty"` + Version string `json:"version"` + Artifact string `json:"artifact"` + Digest string `json:"digest"` + // Alias is the name a model may use to ask for this backend. Only a + // backend entry may set it. + Alias string `json:"alias,omitempty"` + OS string `json:"os"` + Arch string `json:"arch"` + Policy VerificationPolicy `json:"policy"` // UI is the optional display metadata of the entry. It is comparable on // purpose: a release is compared with its trusted resolution at staging. UI EntryUI `json:"ui,omitzero"` @@ -81,6 +94,9 @@ type Release struct { Requires Requirements `json:"requires,omitzero"` } type Query struct { + // Kind limits the listing. Empty lists add-ons only; "all" lists every + // kind this host reads. + Kind Kind Text, SourceID string Offset, Limit int } @@ -119,7 +135,7 @@ func (c Catalog) Search(ctx context.Context, q Query) (Page, error) { if c.Sources == nil { return Page{}, ErrUnavailable } - if q.Offset < 0 || q.Limit < 0 || q.Limit > 100 { + if q.Offset < 0 || q.Limit < 0 || q.Limit > 100 || !validQueryKind(q.Kind) { return Page{}, ErrInvalid } if q.Limit == 0 { @@ -142,7 +158,7 @@ func (c Catalog) Search(ctx context.Context, q Query) (Page, error) { return Page{}, ErrRefresh } for _, r := range cache.releases { - if r.OS != targetOS || r.Arch != targetArch { + if r.OS != targetOS || r.Arch != targetArch || !q.matchesKind(r) { continue } if strings.Contains(strings.ToLower(r.Name+" "+r.AddonID+" "+r.Description), strings.ToLower(q.Text)) { @@ -158,9 +174,17 @@ func (c Catalog) Search(ctx context.Context, q Query) (Page, error) { return p, nil } func releaseKey(r Release) string { - return r.SourceID + "/" + r.AddonID + "/" + r.Version + "/" + r.OS + "/" + r.Arch + return r.SourceID + "/" + string(r.EntryKind()) + "/" + r.AddonID + "/" + r.Version + "/" + r.OS + "/" + r.Arch } + +// Resolve finds one add-on release. A backend release is never returned here, +// so an add-on install cannot be pointed at a backend image. func (c Catalog) Resolve(ctx context.Context, sourceID, addonID, version string) (Release, error) { + return c.ResolveKind(ctx, KindAddon, sourceID, addonID, version) +} + +// ResolveKind finds one release of the given kind. +func (c Catalog) ResolveKind(ctx context.Context, kind Kind, sourceID, addonID, version string) (Release, error) { if e := ctx.Err(); e != nil { return Release{}, e } @@ -184,7 +208,7 @@ func (c Catalog) Resolve(ctx context.Context, sourceID, addonID, version string) targetOS, targetArch := c.target() var found *Release for _, r := range cache.releases { - if r.AddonID == addonID && (r.Version == version || (r.Channel != "" && r.Channel == version)) && r.OS == targetOS && r.Arch == targetArch { + if r.EntryKind() == kind && r.AddonID == addonID && (r.Version == version || (r.Channel != "" && r.Channel == version)) && r.OS == targetOS && r.Arch == targetArch { if found != nil { return Release{}, ErrInvalid } @@ -226,7 +250,7 @@ func (c Catalog) Releases(ctx context.Context, sourceID, addonID string) ([]Rele } out := []Release{} for _, r := range cache.releases { - if r.AddonID == addonID { + if r.EntryKind() == KindAddon && r.AddonID == addonID { out = append(out, r) } } diff --git a/core/services/addons/catalog/index.go b/core/services/addons/catalog/index.go index d8f98ceac..64a6d87f0 100644 --- a/core/services/addons/catalog/index.go +++ b/core/services/addons/catalog/index.go @@ -6,6 +6,7 @@ import ( "encoding/json" "errors" "io" + "slices" "gopkg.in/yaml.v3" ) @@ -29,40 +30,77 @@ const ( // The YAML is converted to plain values and read through the JSON field names // of Release, so one set of names and one strict decoder serve the whole type. func DecodeIndex(data []byte) ([]Release, error) { + res, err := ReadIndex(data, ReadableKinds...) + if err != nil { + return nil, err + } + return res.Releases, nil +} + +// ReadIndex reads a library index the way DecodeIndex does and applies the +// reader's kinds. An entry carries an optional "kind". A reader keeps the +// entries of the kinds it is given and skips every other entry, without +// reading it any further and without refusing the index; the skipped entries +// come back in the result so the caller can report them. Without kinds the +// reader is an add-on reader. A kind that is not plain text, or not a +// well-formed kind name, makes the index malformed and it is refused. +func ReadIndex(data []byte, known ...Kind) (IndexResult, error) { + if len(known) == 0 { + known = []Kind{KindAddon} + } if len(data) > MaxIndexBytes { - return nil, errors.New("index is too large") + return IndexResult{}, errors.New("index is too large") } dec := yaml.NewDecoder(bytes.NewReader(data)) var doc yaml.Node if err := dec.Decode(&doc); err != nil { - return nil, err + return IndexResult{}, err } if err := dec.Decode(new(yaml.Node)); err != io.EOF { - return nil, errors.New("index holds more than one document") + return IndexResult{}, errors.New("index holds more than one document") } if doc.Kind != yaml.DocumentNode || len(doc.Content) != 1 || doc.Content[0].Kind != yaml.SequenceNode { - return nil, errors.New("index must be a list of releases") + return IndexResult{}, errors.New("index must be a list of releases") } list := doc.Content[0] if list.Style&yaml.FlowStyle != 0 && len(list.Content) > 0 { - return nil, errors.New("index must be a YAML block list, not JSON") + return IndexResult{}, errors.New("index must be a YAML block list, not JSON") } if len(list.Content) > MaxIndexReleases { - return nil, errors.New("index has too many releases") + return IndexResult{}, errors.New("index has too many releases") } - var plain any + var plain []any if err := list.Decode(&plain); err != nil { - return nil, err + return IndexResult{}, err } - b, err := json.Marshal(plain) + res := IndexResult{} + kept := make([]any, 0, len(plain)) + for i, item := range plain { + entry, ok := item.(map[string]any) + if !ok { + return IndexResult{}, errors.New("index entry must be a mapping") + } + if raw, present := entry["kind"]; present { + kind, ok := raw.(string) + if !ok || !kindPattern.MatchString(kind) { + return IndexResult{}, errors.New("index entry has a malformed kind") + } + if !slices.Contains(known, Kind(kind)) { + id, _ := entry["addonId"].(string) + res.Skipped = append(res.Skipped, SkippedEntry{Position: i, Kind: kind, ID: id}) + continue + } + } + kept = append(kept, entry) + } + b, err := json.Marshal(kept) if err != nil { - return nil, err + return IndexResult{}, err } - var releases []Release - if err := decode(b, &releases); err != nil { - return nil, err + if err := decode(b, &res.Releases); err != nil { + return IndexResult{}, err } - return releases, nil + return res, nil } // EncodeIndex writes releases as a library index. It is the inverse of diff --git a/core/services/addons/catalog/kind.go b/core/services/addons/catalog/kind.go new file mode 100644 index 000000000..fde991f2c --- /dev/null +++ b/core/services/addons/catalog/kind.go @@ -0,0 +1,90 @@ +// SPDX-License-Identifier: MIT +package catalog + +import ( + "regexp" + "slices" + "sort" +) + +// Kind says what an index entry installs. One entry schema serves every kind; +// the kind decides which host component owns the install. +type Kind string + +const ( + // KindAddon is a supervised add-on package. It is the default: an entry + // without a kind is an add-on, so every index written before kinds existed + // stays valid. + KindAddon Kind = "addon" + // KindBackend is a gRPC backend image installed into the backends directory. + KindBackend Kind = "backend" + // KindModel is reserved by the schema. No reader installs it yet, so a + // reader skips it like any other kind it does not know. + KindModel Kind = "model" +) + +// kindPattern is the grammar of a kind name. A future kind must fit it to be +// skipped; anything else is a malformed entry and the index is refused. +var kindPattern = regexp.MustCompile(`^[a-z][a-z0-9-]{0,31}$`) + +// ReadableKinds are the kinds this host installs. +var ReadableKinds = []Kind{KindAddon, KindBackend} + +// EntryKind is the kind of the release, with the default applied. +func (r Release) EntryKind() Kind { + if r.Kind == "" { + return KindAddon + } + return r.Kind +} + +// SkippedEntry is an index entry a reader left out because it does not know +// its kind. The entry is not validated further: a newer kind may carry fields +// this reader has never seen. +type SkippedEntry struct { + // Position is the zero-based place of the entry in the index list. + Position int `json:"position"` + Kind string `json:"kind"` + // ID is the entry's addonId when it is plain text, to help the report. + ID string `json:"id,omitempty"` +} + +// IndexResult is what a reader made of one index: the entries it understands +// and a record of those it skipped. +type IndexResult struct { + Releases []Release + Skipped []SkippedEntry +} + +// SkippedKinds lists the distinct kinds that were skipped, sorted. +func (r IndexResult) SkippedKinds() []string { + out := []string{} + for _, s := range r.Skipped { + if !slices.Contains(out, s.Kind) { + out = append(out, s.Kind) + } + } + sort.Strings(out) + return out +} + +// Query kinds. The empty value lists add-ons only, so every caller written +// before kinds existed keeps seeing what it saw; QueryAllKinds lists every +// kind this host reads. +const QueryAllKinds Kind = "all" + +func (q Query) matchesKind(r Release) bool { + switch q.Kind { + case "", KindAddon: + return r.EntryKind() == KindAddon + case QueryAllKinds: + return true + default: + return r.EntryKind() == q.Kind + } +} + +// validQueryKind accepts the kinds a caller may ask for. +func validQueryKind(k Kind) bool { + return k == "" || k == QueryAllKinds || slices.Contains(ReadableKinds, k) +} diff --git a/core/services/addons/catalog/kind_test.go b/core/services/addons/catalog/kind_test.go new file mode 100644 index 000000000..c83b23fcd --- /dev/null +++ b/core/services/addons/catalog/kind_test.go @@ -0,0 +1,181 @@ +// SPDX-License-Identifier: MIT +package catalog + +import ( + "net/http" + "strings" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +const kindDigest = "sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" + +const oldShapeIndex = `- addonId: tables + name: Tables + version: 1.0.0 + artifact: oci://registry.example/tables + digest: ` + yamlDigest + ` + os: linux + arch: amd64 +` + +const backendEntry = `- kind: backend + addonId: cpu-local-store + alias: local-store + name: Local store (CPU) + version: 0.0.0-latest.bbbbbbbbbbbb + artifact: oci://registry.example/backends:latest-cpu-local-store + digest: ` + kindDigest + ` + os: linux + arch: amd64 +` + +const unknownKindEntry = `- kind: gizmo + addonId: widget + futureField: 7 + nested: {a: [1, 2]} +` + +var _ = Describe("entry kinds", func() { + It("reads an index written before kinds existed, and the entry is an add-on", func() { + res, err := ReadIndex([]byte(oldShapeIndex), KindAddon) + Expect(err).NotTo(HaveOccurred()) + Expect(res.Skipped).To(BeEmpty()) + Expect(res.Releases).To(HaveLen(1)) + Expect(res.Releases[0].Kind).To(BeEmpty()) + Expect(res.Releases[0].EntryKind()).To(Equal(KindAddon)) + releases, err := DecodeIndex([]byte(oldShapeIndex)) + Expect(err).NotTo(HaveOccurred()) + Expect(releases).To(HaveLen(1)) + }) + + It("reads a backend entry on a reader that knows backends", func() { + res, err := ReadIndex([]byte(oldShapeIndex+backendEntry), KindAddon, KindBackend) + Expect(err).NotTo(HaveOccurred()) + Expect(res.Skipped).To(BeEmpty()) + Expect(res.Releases).To(HaveLen(2)) + Expect(res.Releases[1].EntryKind()).To(Equal(KindBackend)) + Expect(res.Releases[1].Alias).To(Equal("local-store")) + // The reader of this host knows backends too. + releases, err := DecodeIndex([]byte(oldShapeIndex + backendEntry)) + Expect(err).NotTo(HaveOccurred()) + Expect(releases).To(HaveLen(2)) + }) + + It("skips a backend entry on an add-on only reader and says so", func() { + res, err := ReadIndex([]byte(oldShapeIndex+backendEntry), KindAddon) + Expect(err).NotTo(HaveOccurred()) + Expect(res.Releases).To(HaveLen(1)) + Expect(res.Skipped).To(Equal([]SkippedEntry{{Position: 1, Kind: "backend", ID: "cpu-local-store"}})) + }) + + It("skips an unknown kind without reading its fields, and keeps the rest", func() { + res, err := ReadIndex([]byte(oldShapeIndex+unknownKindEntry+backendEntry), KindAddon, KindBackend) + Expect(err).NotTo(HaveOccurred()) + Expect(res.Releases).To(HaveLen(2)) + Expect(res.Skipped).To(Equal([]SkippedEntry{{Position: 1, Kind: "gizmo", ID: "widget"}})) + Expect(res.SkippedKinds()).To(Equal([]string{"gizmo"})) + releases, err := DecodeIndex([]byte(unknownKindEntry)) + Expect(err).NotTo(HaveOccurred()) + Expect(releases).To(BeEmpty()) + }) + + It("skips the reserved model kind in this phase", func() { + res, err := ReadIndex([]byte("- kind: model\n addonId: m\n files: []\n"), ReadableKinds...) + Expect(err).NotTo(HaveOccurred()) + Expect(res.Releases).To(BeEmpty()) + Expect(res.SkippedKinds()).To(Equal([]string{"model"})) + }) + + It("still refuses an unknown field on an entry of a known kind", func() { + _, err := ReadIndex([]byte(backendEntry+" surprise: 1\n"), ReadableKinds...) + Expect(err).To(HaveOccurred()) + }) + + It("refuses a malformed kind", func() { + for _, bad := range []string{"kind: 3", "kind: \"\"", "kind: Backend", "kind: [backend]", "kind: " + strings.Repeat("a", 40)} { + _, err := ReadIndex([]byte("- "+bad+"\n addonId: x\n"), ReadableKinds...) + Expect(err).To(HaveOccurred(), bad) + } + }) +}) + +var _ = Describe("kinds in a library", func() { + refresh := func(body string, policy VerificationPolicy) (RefreshResult, *Sources, error) { + server := library(func(w http.ResponseWriter, _ *http.Request) { _, _ = w.Write([]byte(body)) }) + s := setup() + Expect(s.Save(ctx, admin, Source{ID: "lib", Name: "lib", URL: server.URL, Enabled: true, Policy: policy})).To(Succeed()) + res, err := s.Refresh(ctx, "lib") + return res, s, err + } + + It("refreshes an index with an unknown kind and reports what it skipped", func() { + res, s, err := refresh(oldShapeIndex+unknownKindEntry+backendEntry, VerificationPolicy{}) + Expect(err).NotTo(HaveOccurred()) + Expect(res.Count).To(Equal(2)) + Expect(res.Skipped).To(Equal(1)) + Expect(res.SkippedKinds).To(Equal([]string{"gizmo"})) + views, err := s.Views(ctx, admin) + Expect(err).NotTo(HaveOccurred()) + Expect(views[0].Refresh.Skipped).To(Equal(1)) + }) + + It("lists add-ons by default and backends on request", func() { + _, s, err := refresh(oldShapeIndex+backendEntry, VerificationPolicy{}) + Expect(err).NotTo(HaveOccurred()) + c := Catalog{Sources: s, TargetOS: "linux", TargetArch: "amd64"} + page, err := c.Search(ctx, Query{}) + Expect(err).NotTo(HaveOccurred()) + Expect(page.Releases).To(HaveLen(1)) + Expect(page.Releases[0].AddonID).To(Equal("tables")) + page, err = c.Search(ctx, Query{Kind: KindBackend}) + Expect(err).NotTo(HaveOccurred()) + Expect(page.Releases).To(HaveLen(1)) + Expect(page.Releases[0].AddonID).To(Equal("cpu-local-store")) + page, err = c.Search(ctx, Query{Kind: QueryAllKinds}) + Expect(err).NotTo(HaveOccurred()) + Expect(page.Total).To(Equal(2)) + _, err = c.Search(ctx, Query{Kind: "model"}) + Expect(err).To(MatchError(ErrInvalid)) + }) + + It("resolves a backend only as a backend", func() { + _, s, err := refresh(backendEntry, VerificationPolicy{}) + Expect(err).NotTo(HaveOccurred()) + c := Catalog{Sources: s, TargetOS: "linux", TargetArch: "amd64"} + _, err = c.Resolve(ctx, "lib", "cpu-local-store", "0.0.0-latest.bbbbbbbbbbbb") + Expect(err).To(MatchError(ErrNotFound)) + r, err := c.ResolveKind(ctx, KindBackend, "lib", "cpu-local-store", "0.0.0-latest.bbbbbbbbbbbb") + Expect(err).NotTo(HaveOccurred()) + Expect(r.EntryKind()).To(Equal(KindBackend)) + }) + + It("refuses entries that mix the fields of the kinds", func() { + for name, body := range map[string]string{ + "backend with an https artifact": strings.Replace(backendEntry, "oci://registry.example", "https://registry.example", 1), + "backend with a main page": backendEntry + " ui:\n mainPage: x\n", + "backend name with a slash": strings.Replace(backendEntry, "addonId: cpu-local-store", "addonId: a/b", 1), + "backend name with dots": strings.Replace(backendEntry, "addonId: cpu-local-store", "addonId: a..b", 1), + "add-on with an alias": oldShapeIndex + " alias: x\n", + } { + _, _, err := refresh(body, VerificationPolicy{}) + Expect(err).To(HaveOccurred(), name) + } + }) + + It("accepts an anchored identity pattern and refuses a loose one", func() { + good := VerificationPolicy{RequireSignature: true, Issuer: "https://issuer.example", IdentityRegex: `^https://github\.com/o/r/\.github/workflows/w\.yml@refs/(heads/master|tags/.+)$`} + Expect(validPolicy(good)).To(BeTrue()) + for name, p := range map[string]VerificationPolicy{ + "no anchors": {RequireSignature: true, Issuer: "i", IdentityRegex: `https://x/.*`}, + "alternation": {RequireSignature: true, Issuer: "i", IdentityRegex: `^a|b$`}, + "does not compile": {RequireSignature: true, Issuer: "i", IdentityRegex: `^(a$`}, + "both forms": {RequireSignature: true, Issuer: "i", Identity: "x", IdentityRegex: `^a$`}, + "no issuer": {RequireSignature: true, IdentityRegex: `^a$`}, + "unused pattern": {IdentityRegex: `^a$`}, + } { + Expect(validPolicy(p)).To(BeFalse(), name) + } + }) +}) diff --git a/core/services/addons/catalog/sources.go b/core/services/addons/catalog/sources.go index cad4b6036..ae6b74636 100644 --- a/core/services/addons/catalog/sources.go +++ b/core/services/addons/catalog/sources.go @@ -14,12 +14,15 @@ import ( "os" "path/filepath" "regexp" + "regexp/syntax" "sort" + "strings" "sync" "time" "github.com/mudler/LocalAI/pkg/addons/contract" "github.com/mudler/LocalAI/pkg/httpclient" + "github.com/mudler/xlog" ) // AdminCheck must revalidate current identity, credential and administrator role @@ -38,10 +41,15 @@ type Source struct { Removed bool `json:"removed,omitempty"` } type RefreshResult struct { - SourceID string `json:"sourceId"` - Count int `json:"count"` - CheckedAt time.Time `json:"checkedAt"` - LastSuccess time.Time `json:"lastSuccess"` + SourceID string `json:"sourceId"` + Count int `json:"count"` + // Skipped counts the entries left out because their kind is not one this + // host reads, and SkippedKinds names those kinds. A library that serves a + // newer kind still refreshes; this is how the owner learns of the gap. + Skipped int `json:"skipped,omitempty"` + SkippedKinds []string `json:"skippedKinds,omitempty"` + CheckedAt time.Time `json:"checkedAt"` + LastSuccess time.Time `json:"lastSuccess"` } // RefreshState is what the owner last learned from fetching one library. It is @@ -50,6 +58,9 @@ type RefreshState struct { CheckedAt time.Time `json:"checkedAt"` LastSuccess time.Time `json:"lastSuccess"` Count int `json:"count"` + // Skipped and SkippedKinds are as in RefreshResult. + Skipped int `json:"skipped,omitempty"` + SkippedKinds []string `json:"skippedKinds,omitempty"` // Failed is true when the latest fetch failed; the previous releases are // not served in that case. Failed bool `json:"failed"` @@ -89,7 +100,39 @@ var versionPattern = regexp.MustCompile(`^[0-9]+\.[0-9]+\.[0-9]+(?:-[a-z][a-z0-9 func validSource(s Source) bool { _, e := safeURL(s.URL) - return idPattern.MatchString(s.ID) && len(s.Name) > 0 && len(s.Name) <= 256 && e == nil && (!s.Policy.RequireSignature || (s.Policy.Identity != "" && s.Policy.Issuer != "")) + return idPattern.MatchString(s.ID) && len(s.Name) > 0 && len(s.Name) <= 256 && e == nil && validPolicy(s.Policy) +} + +// validPolicy accepts a policy that asks for no signature, or one with an +// issuer and exactly one identity: an exact string or an anchored pattern. +// Without a signature requirement the identity fields must be empty, so a +// policy never looks stricter than it is. +func validPolicy(p VerificationPolicy) bool { + if !p.RequireSignature { + return p.IdentityRegex == "" + } + if p.Issuer == "" || (p.Identity == "") == (p.IdentityRegex == "") { + return false + } + if p.IdentityRegex == "" { + return true + } + return anchoredPattern(p.IdentityRegex) +} + +// anchoredPattern reports whether the pattern is one expression that must match +// the whole identity: a concatenation that starts at the beginning of the text +// and ends at its end. A top-level alternation such as "^a|b$" is refused, +// because one of its branches would match a part of the identity only. +func anchoredPattern(p string) bool { + if len(p) > 512 { + return false + } + re, e := syntax.Parse(p, syntax.Perl) + if e != nil || re.Op != syntax.OpConcat || len(re.Sub) < 2 { + return false + } + return re.Sub[0].Op == syntax.OpBeginText && re.Sub[len(re.Sub)-1].Op == syntax.OpEndText } func NewSources(path string, admin AdminCheck, credentials CredentialResolver) (*Sources, error) { s := &Sources{path: path, admin: admin, credentials: credentials, sources: map[string]Source{}, cache: map[string]cached{}} @@ -151,7 +194,7 @@ func (s *Sources) Views(ctx context.Context, actor contract.Actor) ([]SourceView for _, src := range list { v := SourceView{Source: src} if c, ok := s.cache[src.ID]; ok { - v.Refresh = &RefreshState{CheckedAt: c.result.CheckedAt, LastSuccess: c.result.LastSuccess, Count: c.result.Count, Failed: c.failed} + v.Refresh = &RefreshState{CheckedAt: c.result.CheckedAt, LastSuccess: c.result.LastSuccess, Count: c.result.Count, Skipped: c.result.Skipped, SkippedKinds: c.result.SkippedKinds, Failed: c.failed} } out = append(out, v) } @@ -176,7 +219,7 @@ func (s *Sources) State(sourceID string) (RefreshState, bool) { if !ok { return RefreshState{}, false } - return RefreshState{CheckedAt: c.result.CheckedAt, LastSuccess: c.result.LastSuccess, Count: c.result.Count, Failed: c.failed}, true + return RefreshState{CheckedAt: c.result.CheckedAt, LastSuccess: c.result.LastSuccess, Count: c.result.Count, Skipped: c.result.Skipped, SkippedKinds: c.result.SkippedKinds, Failed: c.failed}, true } // Get returns a value copy only after authoritative administrator validation. @@ -294,7 +337,7 @@ func (s *Sources) Refresh(ctx context.Context, sourceID string) (RefreshResult, } old := s.cache[sourceID] result := RefreshResult{SourceID: sourceID, CheckedAt: time.Now().UTC(), LastSuccess: old.result.LastSuccess} - releases, e := s.fetch(ctx, src) + releases, skipped, e := s.fetch(ctx, src) if e != nil { s.cache[sourceID] = cached{result: result, failed: true} if errors.Is(e, ErrCredential) { @@ -303,14 +346,20 @@ func (s *Sources) Refresh(ctx context.Context, sourceID string) (RefreshResult, return result, ErrRefresh } result.Count = len(releases) + result.Skipped = len(skipped.Skipped) + if result.Skipped > 0 { + result.SkippedKinds = skipped.SkippedKinds() + xlog.Warn("library index has entries of a kind this host does not read; they were skipped", + "source", sourceID, "skipped", result.Skipped, "kinds", result.SkippedKinds) + } result.LastSuccess = result.CheckedAt s.cache[sourceID] = cached{releases: releases, result: result} return result, nil } -func (s *Sources) fetch(ctx context.Context, src Source) ([]Release, error) { +func (s *Sources) fetch(ctx context.Context, src Source) ([]Release, IndexResult, error) { req, e := http.NewRequestWithContext(ctx, http.MethodGet, src.URL, nil) if e != nil { - return nil, ErrRefresh + return nil, IndexResult{}, ErrRefresh } if src.AuthReference != "" { c, e := s.credentials.Resolve(ctx, src.AuthReference, src.URL) @@ -320,37 +369,38 @@ func (s *Sources) fetch(ctx context.Context, src Source) ([]Release, error) { if e != nil { // A refresh that failed for want of a credential is still a // failed refresh; the class lets a caller say which one. - return nil, fmt.Errorf("%w: %w", ErrRefresh, ErrCredential) + return nil, IndexResult{}, fmt.Errorf("%w: %w", ErrRefresh, ErrCredential) } } response, e := httpclient.NewWithTimeout(30 * time.Second).Do(req) if e != nil { - return nil, ErrRefresh + return nil, IndexResult{}, ErrRefresh } defer func() { _ = response.Body.Close() }() if response.StatusCode != http.StatusOK { - return nil, ErrRefresh + return nil, IndexResult{}, ErrRefresh } data, e := io.ReadAll(io.LimitReader(response.Body, MaxIndexBytes+1)) if e != nil || len(data) > MaxIndexBytes { - return nil, ErrRefresh + return nil, IndexResult{}, ErrRefresh } - releases, e := DecodeIndex(data) + indexed, e := ReadIndex(data, ReadableKinds...) + releases := indexed.Releases if e != nil || releases == nil { - return nil, ErrRefresh + return nil, IndexResult{}, ErrRefresh } seen := map[string]bool{} for i := range releases { r := &releases[i] u, e := url.Parse(r.Artifact) - if e != nil || !validArtifactScheme(u, src.AllowLoopbackHTTP) || u.Host == "" || u.Path == "" || u.User != nil || u.RawQuery != "" || u.Fragment != "" || !contract.ValidAddonID(r.AddonID) || !versionPattern.MatchString(r.Version) || !digestPattern.MatchString(r.Digest) || r.Name == "" || !idPattern.MatchString(r.OS) || !idPattern.MatchString(r.Arch) { - return nil, ErrRefresh + if e != nil || !validArtifactScheme(u, src.AllowLoopbackHTTP) || u.Host == "" || u.Path == "" || u.User != nil || u.RawQuery != "" || u.Fragment != "" || !validEntryID(*r) || !versionPattern.MatchString(r.Version) || !digestPattern.MatchString(r.Digest) || r.Name == "" || !idPattern.MatchString(r.OS) || !idPattern.MatchString(r.Arch) { + return nil, IndexResult{}, ErrRefresh } if r.Channel != "" && (!channelPattern.MatchString(r.Channel) || versionPattern.MatchString(r.Channel)) { - return nil, ErrRefresh + return nil, IndexResult{}, ErrRefresh } - if !validEntryUI(r.UI) || !validRequirements(r.Requires) { - return nil, ErrRefresh + if !validKindFields(*r) || !validEntryUI(r.UI) || !validRequirements(r.Requires) { + return nil, IndexResult{}, ErrRefresh } r.Selection = "" r.SourceID = src.ID @@ -359,11 +409,36 @@ func (s *Sources) fetch(ctx context.Context, src Source) ([]Release, error) { r.Policy = src.Policy key := releaseKey(*r) if seen[key] { - return nil, ErrRefresh + return nil, IndexResult{}, ErrRefresh } seen[key] = true } - return releases, nil + return releases, indexed, nil +} + +// backendNamePattern is a backend name: it is also a directory name, so it +// holds no separator and cannot start with a dot. +var backendNamePattern = regexp.MustCompile(`^[a-z0-9][a-z0-9._-]{0,127}$`) + +// validEntryID applies the identifier rule of the entry's kind. +func validEntryID(r Release) bool { + if r.EntryKind() == KindBackend { + return backendNamePattern.MatchString(r.AddonID) && !strings.Contains(r.AddonID, "..") + } + return contract.ValidAddonID(r.AddonID) +} + +// validKindFields refuses fields that belong to another kind. A backend is an +// image for the backends directory: it has no main page or icon, and it is +// always an OCI image, pinned by digest. Only a backend may set an alias. +func validKindFields(r Release) bool { + if r.EntryKind() != KindBackend { + return r.Alias == "" + } + if !strings.HasPrefix(r.Artifact, "oci://") || r.UI.MainPage != "" || r.UI.Icon != "" || r.UI.LaunchMode != "" || r.Requires != (Requirements{}) { + return false + } + return r.Alias == "" || (backendNamePattern.MatchString(r.Alias) && !strings.Contains(r.Alias, "..")) } // validEntryUI applies to catalog display hints the same rules the manifest diff --git a/core/services/addons/gateway/management.go b/core/services/addons/gateway/management.go index 5727be57e..95a9b7d43 100644 --- a/core/services/addons/gateway/management.go +++ b/core/services/addons/gateway/management.go @@ -23,15 +23,16 @@ func (g *Gateway) management(ctx context.Context, actor contract.Actor, request switch request.Action { case "catalog-search": var args struct { - Text string `json:"text"` - SourceID string `json:"sourceId"` - Offset int `json:"offset"` - Limit int `json:"limit"` + Text string `json:"text"` + Kind catalog.Kind `json:"kind"` + SourceID string `json:"sourceId"` + Offset int `json:"offset"` + Limit int `json:"limit"` } if decode(bytes.NewReader(request.Arguments), &args) != nil { return contract.Result{}, ErrForbidden } - out, err = g.Manager.CatalogSearch(ctx, actor, catalog.Query{Text: args.Text, SourceID: args.SourceID, Offset: args.Offset, Limit: args.Limit}) + out, err = g.Manager.CatalogSearch(ctx, actor, catalog.Query{Kind: args.Kind, Text: args.Text, SourceID: args.SourceID, Offset: args.Offset, Limit: args.Limit}) case "catalog-details": var args struct { SourceID string `json:"sourceId"` diff --git a/core/services/addons/packages/install.go b/core/services/addons/packages/install.go index a0a30869d..b047bf777 100644 --- a/core/services/addons/packages/install.go +++ b/core/services/addons/packages/install.go @@ -70,6 +70,11 @@ func (i *Installer) Stage(ctx context.Context, r catalog.Release, progress func( } // Do not accept a caller's substituted artifact, policy, source or digest. r.Selection = selection + // An add-on library names its signer exactly. A pattern is for backend + // sources only, so an add-on install refuses a policy that carries one. + if r.Policy.IdentityRegex != "" || r.EntryKind() != catalog.KindAddon { + return out, ErrPackage + } if r != trusted || r.Digest == "" || r.OS != i.Host.OS || r.Arch != i.Host.Arch { return out, ErrPackage } diff --git a/core/services/addons/packages/install_test.go b/core/services/addons/packages/install_test.go index afde5c883..a66b445b2 100644 --- a/core/services/addons/packages/install_test.go +++ b/core/services/addons/packages/install_test.go @@ -602,3 +602,16 @@ var _ = Describe("trusted signed staging", func() { Expect(external.Load()).To(BeZero()) }) }) + +var _ = Describe("signer identity pattern", func() { + It("is for backend sources only: an add-on install refuses it", func() { + i, r := fixture([]byte("x"), func(_ *catalog.Release, s *catalog.Source) { + s.Policy = catalog.VerificationPolicy{RequireSignature: true, Issuer: "https://issuer.invalid", IdentityRegex: `^builder@example\.invalid$`} + }) + _, e := i.Stage(context.Background(), r, nil) + Expect(e).To(MatchError(ErrPackage)) + entries, e := os.ReadDir(i.Root) + Expect(e).NotTo(HaveOccurred()) + Expect(entries).To(BeEmpty()) + }) +})