diff --git a/core/services/addons/catalog/catalog.go b/core/services/addons/catalog/catalog.go index 28dcbc784..d09ebdad2 100644 --- a/core/services/addons/catalog/catalog.go +++ b/core/services/addons/catalog/catalog.go @@ -24,6 +24,10 @@ type VerificationPolicy struct { RequireSignature bool `json:"requireSignature"` Issuer string `json:"issuer,omitempty"` Identity string `json:"identity,omitempty"` + // IdentityRegex is an anchored pattern for the signer identity, used + // instead of Identity so that master and tagged builds both verify. Only a + // library that serves backends may use it; an add-on install refuses it. + IdentityRegex string `json:"identityRegex,omitempty"` } // EntryUI is optional display metadata a catalog may offer for an entry. It is @@ -59,21 +63,30 @@ type Requirements struct { } type Release struct { + // Kind says what the entry installs: addon (the default when absent), + // backend or model. See ReadIndex for how a reader treats a kind it does + // not know. + Kind Kind `json:"kind,omitempty"` Channel string `json:"channel,omitempty"` AllowLoopbackHTTP bool `json:"allowLoopbackHTTP,omitempty"` // Selection is stamped by Resolve, not trusted from remote metadata. - Selection string `json:"selection,omitempty"` - SourceID string `json:"sourceId,omitempty"` - SourceURL string `json:"sourceUrl,omitempty"` - AddonID string `json:"addonId"` - Name string `json:"name"` - Description string `json:"description,omitempty"` - Version string `json:"version"` - Artifact string `json:"artifact"` - Digest string `json:"digest"` - OS string `json:"os"` - Arch string `json:"arch"` - Policy VerificationPolicy `json:"policy"` + Selection string `json:"selection,omitempty"` + SourceID string `json:"sourceId,omitempty"` + SourceURL string `json:"sourceUrl,omitempty"` + // AddonID is the identifier of the entry for every kind. For a backend it + // is the backend name, which is also its directory name. + AddonID string `json:"addonId"` + Name string `json:"name"` + Description string `json:"description,omitempty"` + Version string `json:"version"` + Artifact string `json:"artifact"` + Digest string `json:"digest"` + // Alias is the name a model may use to ask for this backend. Only a + // backend entry may set it. + Alias string `json:"alias,omitempty"` + OS string `json:"os"` + Arch string `json:"arch"` + Policy VerificationPolicy `json:"policy"` // UI is the optional display metadata of the entry. It is comparable on // purpose: a release is compared with its trusted resolution at staging. UI EntryUI `json:"ui,omitzero"` @@ -81,6 +94,9 @@ type Release struct { Requires Requirements `json:"requires,omitzero"` } type Query struct { + // Kind limits the listing. Empty lists add-ons only; "all" lists every + // kind this host reads. + Kind Kind Text, SourceID string Offset, Limit int } @@ -119,7 +135,7 @@ func (c Catalog) Search(ctx context.Context, q Query) (Page, error) { if c.Sources == nil { return Page{}, ErrUnavailable } - if q.Offset < 0 || q.Limit < 0 || q.Limit > 100 { + if q.Offset < 0 || q.Limit < 0 || q.Limit > 100 || !validQueryKind(q.Kind) { return Page{}, ErrInvalid } if q.Limit == 0 { @@ -142,7 +158,7 @@ func (c Catalog) Search(ctx context.Context, q Query) (Page, error) { return Page{}, ErrRefresh } for _, r := range cache.releases { - if r.OS != targetOS || r.Arch != targetArch { + if r.OS != targetOS || r.Arch != targetArch || !q.matchesKind(r) { continue } if strings.Contains(strings.ToLower(r.Name+" "+r.AddonID+" "+r.Description), strings.ToLower(q.Text)) { @@ -158,9 +174,17 @@ func (c Catalog) Search(ctx context.Context, q Query) (Page, error) { return p, nil } func releaseKey(r Release) string { - return r.SourceID + "/" + r.AddonID + "/" + r.Version + "/" + r.OS + "/" + r.Arch + return r.SourceID + "/" + string(r.EntryKind()) + "/" + r.AddonID + "/" + r.Version + "/" + r.OS + "/" + r.Arch } + +// Resolve finds one add-on release. A backend release is never returned here, +// so an add-on install cannot be pointed at a backend image. func (c Catalog) Resolve(ctx context.Context, sourceID, addonID, version string) (Release, error) { + return c.ResolveKind(ctx, KindAddon, sourceID, addonID, version) +} + +// ResolveKind finds one release of the given kind. +func (c Catalog) ResolveKind(ctx context.Context, kind Kind, sourceID, addonID, version string) (Release, error) { if e := ctx.Err(); e != nil { return Release{}, e } @@ -184,7 +208,7 @@ func (c Catalog) Resolve(ctx context.Context, sourceID, addonID, version string) targetOS, targetArch := c.target() var found *Release for _, r := range cache.releases { - if r.AddonID == addonID && (r.Version == version || (r.Channel != "" && r.Channel == version)) && r.OS == targetOS && r.Arch == targetArch { + if r.EntryKind() == kind && r.AddonID == addonID && (r.Version == version || (r.Channel != "" && r.Channel == version)) && r.OS == targetOS && r.Arch == targetArch { if found != nil { return Release{}, ErrInvalid } @@ -226,7 +250,7 @@ func (c Catalog) Releases(ctx context.Context, sourceID, addonID string) ([]Rele } out := []Release{} for _, r := range cache.releases { - if r.AddonID == addonID { + if r.EntryKind() == KindAddon && r.AddonID == addonID { out = append(out, r) } } diff --git a/core/services/addons/catalog/index.go b/core/services/addons/catalog/index.go index d8f98ceac..64a6d87f0 100644 --- a/core/services/addons/catalog/index.go +++ b/core/services/addons/catalog/index.go @@ -6,6 +6,7 @@ import ( "encoding/json" "errors" "io" + "slices" "gopkg.in/yaml.v3" ) @@ -29,40 +30,77 @@ const ( // The YAML is converted to plain values and read through the JSON field names // of Release, so one set of names and one strict decoder serve the whole type. func DecodeIndex(data []byte) ([]Release, error) { + res, err := ReadIndex(data, ReadableKinds...) + if err != nil { + return nil, err + } + return res.Releases, nil +} + +// ReadIndex reads a library index the way DecodeIndex does and applies the +// reader's kinds. An entry carries an optional "kind". A reader keeps the +// entries of the kinds it is given and skips every other entry, without +// reading it any further and without refusing the index; the skipped entries +// come back in the result so the caller can report them. Without kinds the +// reader is an add-on reader. A kind that is not plain text, or not a +// well-formed kind name, makes the index malformed and it is refused. +func ReadIndex(data []byte, known ...Kind) (IndexResult, error) { + if len(known) == 0 { + known = []Kind{KindAddon} + } if len(data) > MaxIndexBytes { - return nil, errors.New("index is too large") + return IndexResult{}, errors.New("index is too large") } dec := yaml.NewDecoder(bytes.NewReader(data)) var doc yaml.Node if err := dec.Decode(&doc); err != nil { - return nil, err + return IndexResult{}, err } if err := dec.Decode(new(yaml.Node)); err != io.EOF { - return nil, errors.New("index holds more than one document") + return IndexResult{}, errors.New("index holds more than one document") } if doc.Kind != yaml.DocumentNode || len(doc.Content) != 1 || doc.Content[0].Kind != yaml.SequenceNode { - return nil, errors.New("index must be a list of releases") + return IndexResult{}, errors.New("index must be a list of releases") } list := doc.Content[0] if list.Style&yaml.FlowStyle != 0 && len(list.Content) > 0 { - return nil, errors.New("index must be a YAML block list, not JSON") + return IndexResult{}, errors.New("index must be a YAML block list, not JSON") } if len(list.Content) > MaxIndexReleases { - return nil, errors.New("index has too many releases") + return IndexResult{}, errors.New("index has too many releases") } - var plain any + var plain []any if err := list.Decode(&plain); err != nil { - return nil, err + return IndexResult{}, err } - b, err := json.Marshal(plain) + res := IndexResult{} + kept := make([]any, 0, len(plain)) + for i, item := range plain { + entry, ok := item.(map[string]any) + if !ok { + return IndexResult{}, errors.New("index entry must be a mapping") + } + if raw, present := entry["kind"]; present { + kind, ok := raw.(string) + if !ok || !kindPattern.MatchString(kind) { + return IndexResult{}, errors.New("index entry has a malformed kind") + } + if !slices.Contains(known, Kind(kind)) { + id, _ := entry["addonId"].(string) + res.Skipped = append(res.Skipped, SkippedEntry{Position: i, Kind: kind, ID: id}) + continue + } + } + kept = append(kept, entry) + } + b, err := json.Marshal(kept) if err != nil { - return nil, err + return IndexResult{}, err } - var releases []Release - if err := decode(b, &releases); err != nil { - return nil, err + if err := decode(b, &res.Releases); err != nil { + return IndexResult{}, err } - return releases, nil + return res, nil } // EncodeIndex writes releases as a library index. It is the inverse of diff --git a/core/services/addons/catalog/kind.go b/core/services/addons/catalog/kind.go new file mode 100644 index 000000000..fde991f2c --- /dev/null +++ b/core/services/addons/catalog/kind.go @@ -0,0 +1,90 @@ +// SPDX-License-Identifier: MIT +package catalog + +import ( + "regexp" + "slices" + "sort" +) + +// Kind says what an index entry installs. One entry schema serves every kind; +// the kind decides which host component owns the install. +type Kind string + +const ( + // KindAddon is a supervised add-on package. It is the default: an entry + // without a kind is an add-on, so every index written before kinds existed + // stays valid. + KindAddon Kind = "addon" + // KindBackend is a gRPC backend image installed into the backends directory. + KindBackend Kind = "backend" + // KindModel is reserved by the schema. No reader installs it yet, so a + // reader skips it like any other kind it does not know. + KindModel Kind = "model" +) + +// kindPattern is the grammar of a kind name. A future kind must fit it to be +// skipped; anything else is a malformed entry and the index is refused. +var kindPattern = regexp.MustCompile(`^[a-z][a-z0-9-]{0,31}$`) + +// ReadableKinds are the kinds this host installs. +var ReadableKinds = []Kind{KindAddon, KindBackend} + +// EntryKind is the kind of the release, with the default applied. +func (r Release) EntryKind() Kind { + if r.Kind == "" { + return KindAddon + } + return r.Kind +} + +// SkippedEntry is an index entry a reader left out because it does not know +// its kind. The entry is not validated further: a newer kind may carry fields +// this reader has never seen. +type SkippedEntry struct { + // Position is the zero-based place of the entry in the index list. + Position int `json:"position"` + Kind string `json:"kind"` + // ID is the entry's addonId when it is plain text, to help the report. + ID string `json:"id,omitempty"` +} + +// IndexResult is what a reader made of one index: the entries it understands +// and a record of those it skipped. +type IndexResult struct { + Releases []Release + Skipped []SkippedEntry +} + +// SkippedKinds lists the distinct kinds that were skipped, sorted. +func (r IndexResult) SkippedKinds() []string { + out := []string{} + for _, s := range r.Skipped { + if !slices.Contains(out, s.Kind) { + out = append(out, s.Kind) + } + } + sort.Strings(out) + return out +} + +// Query kinds. The empty value lists add-ons only, so every caller written +// before kinds existed keeps seeing what it saw; QueryAllKinds lists every +// kind this host reads. +const QueryAllKinds Kind = "all" + +func (q Query) matchesKind(r Release) bool { + switch q.Kind { + case "", KindAddon: + return r.EntryKind() == KindAddon + case QueryAllKinds: + return true + default: + return r.EntryKind() == q.Kind + } +} + +// validQueryKind accepts the kinds a caller may ask for. +func validQueryKind(k Kind) bool { + return k == "" || k == QueryAllKinds || slices.Contains(ReadableKinds, k) +} diff --git a/core/services/addons/catalog/kind_test.go b/core/services/addons/catalog/kind_test.go new file mode 100644 index 000000000..c83b23fcd --- /dev/null +++ b/core/services/addons/catalog/kind_test.go @@ -0,0 +1,181 @@ +// SPDX-License-Identifier: MIT +package catalog + +import ( + "net/http" + "strings" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +const kindDigest = "sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" + +const oldShapeIndex = `- addonId: tables + name: Tables + version: 1.0.0 + artifact: oci://registry.example/tables + digest: ` + yamlDigest + ` + os: linux + arch: amd64 +` + +const backendEntry = `- kind: backend + addonId: cpu-local-store + alias: local-store + name: Local store (CPU) + version: 0.0.0-latest.bbbbbbbbbbbb + artifact: oci://registry.example/backends:latest-cpu-local-store + digest: ` + kindDigest + ` + os: linux + arch: amd64 +` + +const unknownKindEntry = `- kind: gizmo + addonId: widget + futureField: 7 + nested: {a: [1, 2]} +` + +var _ = Describe("entry kinds", func() { + It("reads an index written before kinds existed, and the entry is an add-on", func() { + res, err := ReadIndex([]byte(oldShapeIndex), KindAddon) + Expect(err).NotTo(HaveOccurred()) + Expect(res.Skipped).To(BeEmpty()) + Expect(res.Releases).To(HaveLen(1)) + Expect(res.Releases[0].Kind).To(BeEmpty()) + Expect(res.Releases[0].EntryKind()).To(Equal(KindAddon)) + releases, err := DecodeIndex([]byte(oldShapeIndex)) + Expect(err).NotTo(HaveOccurred()) + Expect(releases).To(HaveLen(1)) + }) + + It("reads a backend entry on a reader that knows backends", func() { + res, err := ReadIndex([]byte(oldShapeIndex+backendEntry), KindAddon, KindBackend) + Expect(err).NotTo(HaveOccurred()) + Expect(res.Skipped).To(BeEmpty()) + Expect(res.Releases).To(HaveLen(2)) + Expect(res.Releases[1].EntryKind()).To(Equal(KindBackend)) + Expect(res.Releases[1].Alias).To(Equal("local-store")) + // The reader of this host knows backends too. + releases, err := DecodeIndex([]byte(oldShapeIndex + backendEntry)) + Expect(err).NotTo(HaveOccurred()) + Expect(releases).To(HaveLen(2)) + }) + + It("skips a backend entry on an add-on only reader and says so", func() { + res, err := ReadIndex([]byte(oldShapeIndex+backendEntry), KindAddon) + Expect(err).NotTo(HaveOccurred()) + Expect(res.Releases).To(HaveLen(1)) + Expect(res.Skipped).To(Equal([]SkippedEntry{{Position: 1, Kind: "backend", ID: "cpu-local-store"}})) + }) + + It("skips an unknown kind without reading its fields, and keeps the rest", func() { + res, err := ReadIndex([]byte(oldShapeIndex+unknownKindEntry+backendEntry), KindAddon, KindBackend) + Expect(err).NotTo(HaveOccurred()) + Expect(res.Releases).To(HaveLen(2)) + Expect(res.Skipped).To(Equal([]SkippedEntry{{Position: 1, Kind: "gizmo", ID: "widget"}})) + Expect(res.SkippedKinds()).To(Equal([]string{"gizmo"})) + releases, err := DecodeIndex([]byte(unknownKindEntry)) + Expect(err).NotTo(HaveOccurred()) + Expect(releases).To(BeEmpty()) + }) + + It("skips the reserved model kind in this phase", func() { + res, err := ReadIndex([]byte("- kind: model\n addonId: m\n files: []\n"), ReadableKinds...) + Expect(err).NotTo(HaveOccurred()) + Expect(res.Releases).To(BeEmpty()) + Expect(res.SkippedKinds()).To(Equal([]string{"model"})) + }) + + It("still refuses an unknown field on an entry of a known kind", func() { + _, err := ReadIndex([]byte(backendEntry+" surprise: 1\n"), ReadableKinds...) + Expect(err).To(HaveOccurred()) + }) + + It("refuses a malformed kind", func() { + for _, bad := range []string{"kind: 3", "kind: \"\"", "kind: Backend", "kind: [backend]", "kind: " + strings.Repeat("a", 40)} { + _, err := ReadIndex([]byte("- "+bad+"\n addonId: x\n"), ReadableKinds...) + Expect(err).To(HaveOccurred(), bad) + } + }) +}) + +var _ = Describe("kinds in a library", func() { + refresh := func(body string, policy VerificationPolicy) (RefreshResult, *Sources, error) { + server := library(func(w http.ResponseWriter, _ *http.Request) { _, _ = w.Write([]byte(body)) }) + s := setup() + Expect(s.Save(ctx, admin, Source{ID: "lib", Name: "lib", URL: server.URL, Enabled: true, Policy: policy})).To(Succeed()) + res, err := s.Refresh(ctx, "lib") + return res, s, err + } + + It("refreshes an index with an unknown kind and reports what it skipped", func() { + res, s, err := refresh(oldShapeIndex+unknownKindEntry+backendEntry, VerificationPolicy{}) + Expect(err).NotTo(HaveOccurred()) + Expect(res.Count).To(Equal(2)) + Expect(res.Skipped).To(Equal(1)) + Expect(res.SkippedKinds).To(Equal([]string{"gizmo"})) + views, err := s.Views(ctx, admin) + Expect(err).NotTo(HaveOccurred()) + Expect(views[0].Refresh.Skipped).To(Equal(1)) + }) + + It("lists add-ons by default and backends on request", func() { + _, s, err := refresh(oldShapeIndex+backendEntry, VerificationPolicy{}) + Expect(err).NotTo(HaveOccurred()) + c := Catalog{Sources: s, TargetOS: "linux", TargetArch: "amd64"} + page, err := c.Search(ctx, Query{}) + Expect(err).NotTo(HaveOccurred()) + Expect(page.Releases).To(HaveLen(1)) + Expect(page.Releases[0].AddonID).To(Equal("tables")) + page, err = c.Search(ctx, Query{Kind: KindBackend}) + Expect(err).NotTo(HaveOccurred()) + Expect(page.Releases).To(HaveLen(1)) + Expect(page.Releases[0].AddonID).To(Equal("cpu-local-store")) + page, err = c.Search(ctx, Query{Kind: QueryAllKinds}) + Expect(err).NotTo(HaveOccurred()) + Expect(page.Total).To(Equal(2)) + _, err = c.Search(ctx, Query{Kind: "model"}) + Expect(err).To(MatchError(ErrInvalid)) + }) + + It("resolves a backend only as a backend", func() { + _, s, err := refresh(backendEntry, VerificationPolicy{}) + Expect(err).NotTo(HaveOccurred()) + c := Catalog{Sources: s, TargetOS: "linux", TargetArch: "amd64"} + _, err = c.Resolve(ctx, "lib", "cpu-local-store", "0.0.0-latest.bbbbbbbbbbbb") + Expect(err).To(MatchError(ErrNotFound)) + r, err := c.ResolveKind(ctx, KindBackend, "lib", "cpu-local-store", "0.0.0-latest.bbbbbbbbbbbb") + Expect(err).NotTo(HaveOccurred()) + Expect(r.EntryKind()).To(Equal(KindBackend)) + }) + + It("refuses entries that mix the fields of the kinds", func() { + for name, body := range map[string]string{ + "backend with an https artifact": strings.Replace(backendEntry, "oci://registry.example", "https://registry.example", 1), + "backend with a main page": backendEntry + " ui:\n mainPage: x\n", + "backend name with a slash": strings.Replace(backendEntry, "addonId: cpu-local-store", "addonId: a/b", 1), + "backend name with dots": strings.Replace(backendEntry, "addonId: cpu-local-store", "addonId: a..b", 1), + "add-on with an alias": oldShapeIndex + " alias: x\n", + } { + _, _, err := refresh(body, VerificationPolicy{}) + Expect(err).To(HaveOccurred(), name) + } + }) + + It("accepts an anchored identity pattern and refuses a loose one", func() { + good := VerificationPolicy{RequireSignature: true, Issuer: "https://issuer.example", IdentityRegex: `^https://github\.com/o/r/\.github/workflows/w\.yml@refs/(heads/master|tags/.+)$`} + Expect(validPolicy(good)).To(BeTrue()) + for name, p := range map[string]VerificationPolicy{ + "no anchors": {RequireSignature: true, Issuer: "i", IdentityRegex: `https://x/.*`}, + "alternation": {RequireSignature: true, Issuer: "i", IdentityRegex: `^a|b$`}, + "does not compile": {RequireSignature: true, Issuer: "i", IdentityRegex: `^(a$`}, + "both forms": {RequireSignature: true, Issuer: "i", Identity: "x", IdentityRegex: `^a$`}, + "no issuer": {RequireSignature: true, IdentityRegex: `^a$`}, + "unused pattern": {IdentityRegex: `^a$`}, + } { + Expect(validPolicy(p)).To(BeFalse(), name) + } + }) +}) diff --git a/core/services/addons/catalog/sources.go b/core/services/addons/catalog/sources.go index cad4b6036..ae6b74636 100644 --- a/core/services/addons/catalog/sources.go +++ b/core/services/addons/catalog/sources.go @@ -14,12 +14,15 @@ import ( "os" "path/filepath" "regexp" + "regexp/syntax" "sort" + "strings" "sync" "time" "github.com/mudler/LocalAI/pkg/addons/contract" "github.com/mudler/LocalAI/pkg/httpclient" + "github.com/mudler/xlog" ) // AdminCheck must revalidate current identity, credential and administrator role @@ -38,10 +41,15 @@ type Source struct { Removed bool `json:"removed,omitempty"` } type RefreshResult struct { - SourceID string `json:"sourceId"` - Count int `json:"count"` - CheckedAt time.Time `json:"checkedAt"` - LastSuccess time.Time `json:"lastSuccess"` + SourceID string `json:"sourceId"` + Count int `json:"count"` + // Skipped counts the entries left out because their kind is not one this + // host reads, and SkippedKinds names those kinds. A library that serves a + // newer kind still refreshes; this is how the owner learns of the gap. + Skipped int `json:"skipped,omitempty"` + SkippedKinds []string `json:"skippedKinds,omitempty"` + CheckedAt time.Time `json:"checkedAt"` + LastSuccess time.Time `json:"lastSuccess"` } // RefreshState is what the owner last learned from fetching one library. It is @@ -50,6 +58,9 @@ type RefreshState struct { CheckedAt time.Time `json:"checkedAt"` LastSuccess time.Time `json:"lastSuccess"` Count int `json:"count"` + // Skipped and SkippedKinds are as in RefreshResult. + Skipped int `json:"skipped,omitempty"` + SkippedKinds []string `json:"skippedKinds,omitempty"` // Failed is true when the latest fetch failed; the previous releases are // not served in that case. Failed bool `json:"failed"` @@ -89,7 +100,39 @@ var versionPattern = regexp.MustCompile(`^[0-9]+\.[0-9]+\.[0-9]+(?:-[a-z][a-z0-9 func validSource(s Source) bool { _, e := safeURL(s.URL) - return idPattern.MatchString(s.ID) && len(s.Name) > 0 && len(s.Name) <= 256 && e == nil && (!s.Policy.RequireSignature || (s.Policy.Identity != "" && s.Policy.Issuer != "")) + return idPattern.MatchString(s.ID) && len(s.Name) > 0 && len(s.Name) <= 256 && e == nil && validPolicy(s.Policy) +} + +// validPolicy accepts a policy that asks for no signature, or one with an +// issuer and exactly one identity: an exact string or an anchored pattern. +// Without a signature requirement the identity fields must be empty, so a +// policy never looks stricter than it is. +func validPolicy(p VerificationPolicy) bool { + if !p.RequireSignature { + return p.IdentityRegex == "" + } + if p.Issuer == "" || (p.Identity == "") == (p.IdentityRegex == "") { + return false + } + if p.IdentityRegex == "" { + return true + } + return anchoredPattern(p.IdentityRegex) +} + +// anchoredPattern reports whether the pattern is one expression that must match +// the whole identity: a concatenation that starts at the beginning of the text +// and ends at its end. A top-level alternation such as "^a|b$" is refused, +// because one of its branches would match a part of the identity only. +func anchoredPattern(p string) bool { + if len(p) > 512 { + return false + } + re, e := syntax.Parse(p, syntax.Perl) + if e != nil || re.Op != syntax.OpConcat || len(re.Sub) < 2 { + return false + } + return re.Sub[0].Op == syntax.OpBeginText && re.Sub[len(re.Sub)-1].Op == syntax.OpEndText } func NewSources(path string, admin AdminCheck, credentials CredentialResolver) (*Sources, error) { s := &Sources{path: path, admin: admin, credentials: credentials, sources: map[string]Source{}, cache: map[string]cached{}} @@ -151,7 +194,7 @@ func (s *Sources) Views(ctx context.Context, actor contract.Actor) ([]SourceView for _, src := range list { v := SourceView{Source: src} if c, ok := s.cache[src.ID]; ok { - v.Refresh = &RefreshState{CheckedAt: c.result.CheckedAt, LastSuccess: c.result.LastSuccess, Count: c.result.Count, Failed: c.failed} + v.Refresh = &RefreshState{CheckedAt: c.result.CheckedAt, LastSuccess: c.result.LastSuccess, Count: c.result.Count, Skipped: c.result.Skipped, SkippedKinds: c.result.SkippedKinds, Failed: c.failed} } out = append(out, v) } @@ -176,7 +219,7 @@ func (s *Sources) State(sourceID string) (RefreshState, bool) { if !ok { return RefreshState{}, false } - return RefreshState{CheckedAt: c.result.CheckedAt, LastSuccess: c.result.LastSuccess, Count: c.result.Count, Failed: c.failed}, true + return RefreshState{CheckedAt: c.result.CheckedAt, LastSuccess: c.result.LastSuccess, Count: c.result.Count, Skipped: c.result.Skipped, SkippedKinds: c.result.SkippedKinds, Failed: c.failed}, true } // Get returns a value copy only after authoritative administrator validation. @@ -294,7 +337,7 @@ func (s *Sources) Refresh(ctx context.Context, sourceID string) (RefreshResult, } old := s.cache[sourceID] result := RefreshResult{SourceID: sourceID, CheckedAt: time.Now().UTC(), LastSuccess: old.result.LastSuccess} - releases, e := s.fetch(ctx, src) + releases, skipped, e := s.fetch(ctx, src) if e != nil { s.cache[sourceID] = cached{result: result, failed: true} if errors.Is(e, ErrCredential) { @@ -303,14 +346,20 @@ func (s *Sources) Refresh(ctx context.Context, sourceID string) (RefreshResult, return result, ErrRefresh } result.Count = len(releases) + result.Skipped = len(skipped.Skipped) + if result.Skipped > 0 { + result.SkippedKinds = skipped.SkippedKinds() + xlog.Warn("library index has entries of a kind this host does not read; they were skipped", + "source", sourceID, "skipped", result.Skipped, "kinds", result.SkippedKinds) + } result.LastSuccess = result.CheckedAt s.cache[sourceID] = cached{releases: releases, result: result} return result, nil } -func (s *Sources) fetch(ctx context.Context, src Source) ([]Release, error) { +func (s *Sources) fetch(ctx context.Context, src Source) ([]Release, IndexResult, error) { req, e := http.NewRequestWithContext(ctx, http.MethodGet, src.URL, nil) if e != nil { - return nil, ErrRefresh + return nil, IndexResult{}, ErrRefresh } if src.AuthReference != "" { c, e := s.credentials.Resolve(ctx, src.AuthReference, src.URL) @@ -320,37 +369,38 @@ func (s *Sources) fetch(ctx context.Context, src Source) ([]Release, error) { if e != nil { // A refresh that failed for want of a credential is still a // failed refresh; the class lets a caller say which one. - return nil, fmt.Errorf("%w: %w", ErrRefresh, ErrCredential) + return nil, IndexResult{}, fmt.Errorf("%w: %w", ErrRefresh, ErrCredential) } } response, e := httpclient.NewWithTimeout(30 * time.Second).Do(req) if e != nil { - return nil, ErrRefresh + return nil, IndexResult{}, ErrRefresh } defer func() { _ = response.Body.Close() }() if response.StatusCode != http.StatusOK { - return nil, ErrRefresh + return nil, IndexResult{}, ErrRefresh } data, e := io.ReadAll(io.LimitReader(response.Body, MaxIndexBytes+1)) if e != nil || len(data) > MaxIndexBytes { - return nil, ErrRefresh + return nil, IndexResult{}, ErrRefresh } - releases, e := DecodeIndex(data) + indexed, e := ReadIndex(data, ReadableKinds...) + releases := indexed.Releases if e != nil || releases == nil { - return nil, ErrRefresh + return nil, IndexResult{}, ErrRefresh } seen := map[string]bool{} for i := range releases { r := &releases[i] u, e := url.Parse(r.Artifact) - if e != nil || !validArtifactScheme(u, src.AllowLoopbackHTTP) || u.Host == "" || u.Path == "" || u.User != nil || u.RawQuery != "" || u.Fragment != "" || !contract.ValidAddonID(r.AddonID) || !versionPattern.MatchString(r.Version) || !digestPattern.MatchString(r.Digest) || r.Name == "" || !idPattern.MatchString(r.OS) || !idPattern.MatchString(r.Arch) { - return nil, ErrRefresh + if e != nil || !validArtifactScheme(u, src.AllowLoopbackHTTP) || u.Host == "" || u.Path == "" || u.User != nil || u.RawQuery != "" || u.Fragment != "" || !validEntryID(*r) || !versionPattern.MatchString(r.Version) || !digestPattern.MatchString(r.Digest) || r.Name == "" || !idPattern.MatchString(r.OS) || !idPattern.MatchString(r.Arch) { + return nil, IndexResult{}, ErrRefresh } if r.Channel != "" && (!channelPattern.MatchString(r.Channel) || versionPattern.MatchString(r.Channel)) { - return nil, ErrRefresh + return nil, IndexResult{}, ErrRefresh } - if !validEntryUI(r.UI) || !validRequirements(r.Requires) { - return nil, ErrRefresh + if !validKindFields(*r) || !validEntryUI(r.UI) || !validRequirements(r.Requires) { + return nil, IndexResult{}, ErrRefresh } r.Selection = "" r.SourceID = src.ID @@ -359,11 +409,36 @@ func (s *Sources) fetch(ctx context.Context, src Source) ([]Release, error) { r.Policy = src.Policy key := releaseKey(*r) if seen[key] { - return nil, ErrRefresh + return nil, IndexResult{}, ErrRefresh } seen[key] = true } - return releases, nil + return releases, indexed, nil +} + +// backendNamePattern is a backend name: it is also a directory name, so it +// holds no separator and cannot start with a dot. +var backendNamePattern = regexp.MustCompile(`^[a-z0-9][a-z0-9._-]{0,127}$`) + +// validEntryID applies the identifier rule of the entry's kind. +func validEntryID(r Release) bool { + if r.EntryKind() == KindBackend { + return backendNamePattern.MatchString(r.AddonID) && !strings.Contains(r.AddonID, "..") + } + return contract.ValidAddonID(r.AddonID) +} + +// validKindFields refuses fields that belong to another kind. A backend is an +// image for the backends directory: it has no main page or icon, and it is +// always an OCI image, pinned by digest. Only a backend may set an alias. +func validKindFields(r Release) bool { + if r.EntryKind() != KindBackend { + return r.Alias == "" + } + if !strings.HasPrefix(r.Artifact, "oci://") || r.UI.MainPage != "" || r.UI.Icon != "" || r.UI.LaunchMode != "" || r.Requires != (Requirements{}) { + return false + } + return r.Alias == "" || (backendNamePattern.MatchString(r.Alias) && !strings.Contains(r.Alias, "..")) } // validEntryUI applies to catalog display hints the same rules the manifest diff --git a/core/services/addons/gateway/management.go b/core/services/addons/gateway/management.go index 5727be57e..95a9b7d43 100644 --- a/core/services/addons/gateway/management.go +++ b/core/services/addons/gateway/management.go @@ -23,15 +23,16 @@ func (g *Gateway) management(ctx context.Context, actor contract.Actor, request switch request.Action { case "catalog-search": var args struct { - Text string `json:"text"` - SourceID string `json:"sourceId"` - Offset int `json:"offset"` - Limit int `json:"limit"` + Text string `json:"text"` + Kind catalog.Kind `json:"kind"` + SourceID string `json:"sourceId"` + Offset int `json:"offset"` + Limit int `json:"limit"` } if decode(bytes.NewReader(request.Arguments), &args) != nil { return contract.Result{}, ErrForbidden } - out, err = g.Manager.CatalogSearch(ctx, actor, catalog.Query{Text: args.Text, SourceID: args.SourceID, Offset: args.Offset, Limit: args.Limit}) + out, err = g.Manager.CatalogSearch(ctx, actor, catalog.Query{Kind: args.Kind, Text: args.Text, SourceID: args.SourceID, Offset: args.Offset, Limit: args.Limit}) case "catalog-details": var args struct { SourceID string `json:"sourceId"` diff --git a/core/services/addons/packages/install.go b/core/services/addons/packages/install.go index a0a30869d..b047bf777 100644 --- a/core/services/addons/packages/install.go +++ b/core/services/addons/packages/install.go @@ -70,6 +70,11 @@ func (i *Installer) Stage(ctx context.Context, r catalog.Release, progress func( } // Do not accept a caller's substituted artifact, policy, source or digest. r.Selection = selection + // An add-on library names its signer exactly. A pattern is for backend + // sources only, so an add-on install refuses a policy that carries one. + if r.Policy.IdentityRegex != "" || r.EntryKind() != catalog.KindAddon { + return out, ErrPackage + } if r != trusted || r.Digest == "" || r.OS != i.Host.OS || r.Arch != i.Host.Arch { return out, ErrPackage } diff --git a/core/services/addons/packages/install_test.go b/core/services/addons/packages/install_test.go index afde5c883..a66b445b2 100644 --- a/core/services/addons/packages/install_test.go +++ b/core/services/addons/packages/install_test.go @@ -602,3 +602,16 @@ var _ = Describe("trusted signed staging", func() { Expect(external.Load()).To(BeZero()) }) }) + +var _ = Describe("signer identity pattern", func() { + It("is for backend sources only: an add-on install refuses it", func() { + i, r := fixture([]byte("x"), func(_ *catalog.Release, s *catalog.Source) { + s.Policy = catalog.VerificationPolicy{RequireSignature: true, Issuer: "https://issuer.invalid", IdentityRegex: `^builder@example\.invalid$`} + }) + _, e := i.Stage(context.Background(), r, nil) + Expect(e).To(MatchError(ErrPackage)) + entries, e := os.ReadDir(i.Root) + Expect(e).NotTo(HaveOccurred()) + Expect(entries).To(BeEmpty()) + }) +})