From f8b7576dc7dac7d156940b435322964cb819b9ae Mon Sep 17 00:00:00 2001 From: mudler-agent Date: Sat, 24 Oct 2026 22:04:56 +0200 Subject: [PATCH] feat(addons): install development add-ons by tag, as backends do (#12883) A development add-on entry now names a moving tag, sets mutable and has no digest, the same shape a development backend entry has. A release entry stays pinned by digest. One helper, catalog.Pin, turns a mutable entry into a digest at install: it resolves the tag, checks the digest form and builds the version from the first 12 hex digits. The backend library installer and the add-on installer both call it. The add-on installer then verifies the signature against the resolved digest and records that digest in the receipt. Kind, platform and install label checks stay in each caller. The official index lists the development entries as -development, which the publish workflow already moves in its promote job. The catalog command renders the development channel as mutable entries. The update check reports no state for a mutable entry until it resolves the tag too. Assisted-by: Claude:claude-sonnet-5-5 [Claude Code] Signed-off-by: Ettore Di Giacinto Co-authored-by: Ettore Di Giacinto --- .github/workflows/addons-package.yml | 5 +- core/services/addons/backendlib/install.go | 17 +-- core/services/addons/catalog/backend_entry.go | 8 +- .../addons/catalog/backend_entry_test.go | 19 +++- core/services/addons/catalog/mutable.go | 67 +++++++++++ core/services/addons/catalog/mutable_test.go | 55 +++++++++ core/services/addons/catalog/official_test.go | 9 +- core/services/addons/packages/install.go | 26 ++++- core/services/addons/packages/install_test.go | 106 ++++++++++++++++++ core/services/addons/updates/checker.go | 6 + docs/content/features/addons.md | 7 ++ gallery/addons/index.yaml | 49 ++++---- pkg/addons/packaging/catalog.go | 30 ++++- pkg/addons/packaging/packaging_test.go | 22 ++++ 14 files changed, 375 insertions(+), 51 deletions(-) create mode 100644 core/services/addons/catalog/mutable.go create mode 100644 core/services/addons/catalog/mutable_test.go diff --git a/.github/workflows/addons-package.yml b/.github/workflows/addons-package.yml index 33c5bf53f..1c868d75d 100644 --- a/.github/workflows/addons-package.yml +++ b/.github/workflows/addons-package.yml @@ -18,7 +18,10 @@ name: 'add-on packages' # value of ADDON_OCI_REPOSITORY (the add-on registry organization). Each # add-on is a set of tags of that repository, prefixed with its id: # -sha- for a commit, - for a release and -# -development or -latest for a channel alias. Signatures and digests +# -development or -latest for a channel alias. The development alias is +# the tag a development library entry names (mutable, no digest); the install +# resolves it to a digest, as for a development backend, so it moves only in +# the promote job below. Signatures and digests # are per image, so add-ons never collide. A publishing event fails for an # add-on until one of its platforms is enabled. # diff --git a/core/services/addons/backendlib/install.go b/core/services/addons/backendlib/install.go index 6be3bc655..560d12f77 100644 --- a/core/services/addons/backendlib/install.go +++ b/core/services/addons/backendlib/install.go @@ -256,7 +256,7 @@ var digestForm = regexp.MustCompile(`^sha256:[a-f0-9]{64}$`) // the same pinned path: the signature is verified against that digest and the // metadata records it, so an update check can compare it later. func (i *Installer) plan(ctx context.Context, r catalog.Release) (*gallery.GalleryBackend, error) { - if r.EntryKind() != catalog.KindBackend || !strings.HasPrefix(r.Artifact, "oci://") || (r.Mutable != (r.Digest == "")) || (!r.Mutable && !digestForm.MatchString(r.Digest)) { + if r.EntryKind() != catalog.KindBackend { return nil, ErrInvalid } target, arch := i.Catalog.Target() @@ -266,19 +266,12 @@ func (i *Installer) plan(ctx context.Context, r catalog.Release) (*gallery.Galle if i.RequireIntegrity && !r.Policy.RequireSignature { return nil, fmt.Errorf("%w (library %q)", ErrIntegrity, r.SourceID) } - ref, err := name.ParseReference(strings.TrimPrefix(r.Artifact, "oci://")) + // The shared pin: a mutable entry's tag is resolved to a digest here. + pin, err := catalog.Pin(ctx, r, i.digestOf) if err != nil { - return nil, ErrInvalid - } - digest, version := r.Digest, r.Version - if r.Mutable { - d, err := i.digestOf(ctx, ref.Name(), r.OS+"/"+r.Arch) - if err != nil || !digestForm.MatchString(d) { - return nil, fmt.Errorf("%w: resolving %s: %v", ErrInvalid, ref.Name(), err) - } - digest = d - version = r.Version + "." + strings.TrimPrefix(d, "sha256:")[:12] + return nil, fmt.Errorf("%w: %v", ErrInvalid, err) } + ref, digest, version := pin.Ref, pin.Digest, pin.Version gb := &gallery.GalleryBackend{ URI: "oci://" + ref.Context().Digest(digest).Name(), Version: version, diff --git a/core/services/addons/catalog/backend_entry.go b/core/services/addons/catalog/backend_entry.go index bd9bd65f6..aac4a943c 100644 --- a/core/services/addons/catalog/backend_entry.go +++ b/core/services/addons/catalog/backend_entry.go @@ -32,7 +32,7 @@ var capabilityPattern = regexp.MustCompile(`^[a-z][a-z0-9-]{0,63}$`) // backend may set. An add-on entry that does is refused. func (r Release) hasBackendOnlyFields() bool { return r.License != "" || len(r.URLs) > 0 || len(r.Tags) > 0 || r.Icon != "" || - len(r.Mirrors) > 0 || len(r.Capabilities) > 0 || r.Mutable || r.SchemaVersion != 0 + len(r.Mirrors) > 0 || len(r.Capabilities) > 0 || r.SchemaVersion != 0 } // validFetchedRelease is the check a library applies to every entry it keeps. @@ -60,7 +60,11 @@ func validFetchedRelease(r Release, allowLoopback bool) bool { if r.Mutable != (r.Digest == "") || (r.Digest != "" && !digestPattern.MatchString(r.Digest)) { return false } - if r.Mutable && r.EntryKind() != KindBackend { + // A mutable entry is an OCI tag: an HTTP archive has nothing to resolve. + if r.Mutable && r.EntryKind() != KindBackend && r.EntryKind() != KindAddon { + return false + } + if r.Mutable && !strings.HasPrefix(r.Artifact, "oci://") { return false } } diff --git a/core/services/addons/catalog/backend_entry_test.go b/core/services/addons/catalog/backend_entry_test.go index ceadae7b6..dab8706cc 100644 --- a/core/services/addons/catalog/backend_entry_test.go +++ b/core/services/addons/catalog/backend_entry_test.go @@ -99,10 +99,10 @@ var _ = Describe("the backend body of a library entry", func() { Expect(err).To(HaveOccurred()) }) - It("refuses a license, urls, tags, icon, mirrors, capabilities or mutable flag on an add-on", func() { + It("refuses a license, urls, tags, icon, mirrors or capabilities on an add-on", func() { for _, extra := range []string{ " license: mit\n", " urls: [https://example.com]\n", " tags: [a]\n", " icon: https://example.com/i.png\n", - " mirrors: [oci://hub.example/x:y]\n", " capabilities: {default: x}\n", " mutable: true\n", " schemaVersion: 1\n", + " mirrors: [oci://hub.example/x:y]\n", " capabilities: {default: x}\n", " schemaVersion: 1\n", } { _, _, err := read(oldShapeIndex + extra) Expect(err).To(HaveOccurred(), extra) @@ -272,9 +272,22 @@ var _ = Describe("the backend body of a library entry", func() { _, _, err = read(strings.Replace(mutableEntry, " mutable: true\n", "", 1)) Expect(err).To(HaveOccurred()) }) - It("is refused on an add-on", func() { + It("is accepted on an add-on without a digest, as on a backend", func() { + entry := strings.Replace(oldShapeIndex, " digest: "+yamlDigest+"\n", " mutable: true\n", 1) + _, s, err := read(entry) + Expect(err).NotTo(HaveOccurred()) + r, err := Catalog{Sources: s, TargetOS: "linux", TargetArch: "amd64"}.Resolve(ctx, "lib", "tables", "1.0.0") + Expect(err).NotTo(HaveOccurred()) + Expect(r.Mutable).To(BeTrue()) + Expect(r.Digest).To(BeEmpty()) + }) + It("is refused on an add-on with a digest, without a digest unless mutable, or on an archive", func() { _, _, err := read(oldShapeIndex + " mutable: true\n") Expect(err).To(HaveOccurred()) + _, _, err = read(strings.Replace(oldShapeIndex, " digest: "+yamlDigest+"\n", "", 1)) + Expect(err).To(HaveOccurred()) + _, _, err = read(strings.Replace(strings.Replace(oldShapeIndex, " digest: "+yamlDigest+"\n", " mutable: true\n", 1), "oci://registry.example/tables", "https://registry.example/tables.tar", 1)) + Expect(err).To(HaveOccurred()) }) }) diff --git a/core/services/addons/catalog/mutable.go b/core/services/addons/catalog/mutable.go new file mode 100644 index 000000000..241528a8d --- /dev/null +++ b/core/services/addons/catalog/mutable.go @@ -0,0 +1,67 @@ +// SPDX-License-Identifier: MIT +package catalog + +import ( + "context" + "fmt" + "strings" + + "github.com/google/go-containerregistry/pkg/name" + "github.com/mudler/LocalAI/pkg/oci" +) + +// DigestLookup resolves an image reference to the manifest digest for a +// platform. Tests replace it; the default asks the registry. +type DigestLookup func(ctx context.Context, image, platform string) (string, error) + +// Pinned is an entry artifact fixed to one digest. +type Pinned struct { + // Ref is the parsed artifact of the entry. + Ref name.Reference + // Digest is the sha256 digest the install uses. It is the digest of the + // entry, or for a mutable entry the digest its tag named at install. + Digest string + // Version is the version to record. A mutable entry gets the first 12 hex + // digits of the digest appended, so an update check can tell two builds of + // one tag apart. + Version string +} + +// ByDigest is the image reference of the pin: the repository of the artifact +// with the digest, never the tag. +func (p Pinned) ByDigest() string { return p.Ref.Context().Digest(p.Digest).Name() } + +// Pin fixes an OCI entry to a digest. This is the one place where a mutable +// entry (a development image) is turned into a pinned one, for backends and +// for add-ons alike: the tag is resolved to a digest here, at install, and the +// caller then verifies the signature against Pinned.Digest and records it, so +// the install goes through the same path as a release entry. +// +// An entry that is not mutable needs a well-formed digest. A mutable entry has +// none. A nil lookup asks the registry. What differs by kind (the entry kind, +// the platform, the label of the install) stays with the caller. +func Pin(ctx context.Context, r Release, lookup DigestLookup) (Pinned, error) { + if !strings.HasPrefix(r.Artifact, "oci://") || r.Mutable != (r.Digest == "") || (!r.Mutable && !digestPattern.MatchString(r.Digest)) { + return Pinned{}, fmt.Errorf("entry %q: not an OCI artifact pinned by a digest or marked mutable", r.ID) + } + ref, err := name.ParseReference(strings.TrimPrefix(r.Artifact, "oci://")) + if err != nil { + return Pinned{}, fmt.Errorf("entry %q: %w", r.ID, err) + } + p := Pinned{Ref: ref, Digest: r.Digest, Version: r.Version} + if !r.Mutable { + return p, nil + } + if lookup == nil { + lookup = func(_ context.Context, image, platform string) (string, error) { + return oci.GetImageDigest(image, platform, nil, nil) + } + } + d, err := lookup(ctx, ref.Name(), r.OS+"/"+r.Arch) + if err != nil || !digestPattern.MatchString(d) { + return Pinned{}, fmt.Errorf("resolving %s: %v", ref.Name(), err) + } + p.Digest = d + p.Version = r.Version + "." + strings.TrimPrefix(d, "sha256:")[:12] + return p, nil +} diff --git a/core/services/addons/catalog/mutable_test.go b/core/services/addons/catalog/mutable_test.go new file mode 100644 index 000000000..cddbaf0e7 --- /dev/null +++ b/core/services/addons/catalog/mutable_test.go @@ -0,0 +1,55 @@ +// SPDX-License-Identifier: MIT +package catalog + +import ( + "context" + "errors" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("Pin, the one place a mutable entry becomes a digest", func() { + const resolved = "sha256:cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc" + mutable := Release{ID: "demo", Version: "0.0.0-development", Artifact: "oci://quay.example/acme/addons:demo-development", Mutable: true, OS: "linux", Arch: "amd64"} + + It("resolves the tag at install and builds the version from the digest", func() { + var asked string + p, err := Pin(context.Background(), mutable, func(_ context.Context, image, platform string) (string, error) { + asked = image + " " + platform + return resolved, nil + }) + Expect(err).NotTo(HaveOccurred()) + Expect(asked).To(Equal("quay.example/acme/addons:demo-development linux/amd64")) + Expect(p.Digest).To(Equal(resolved)) + Expect(p.Version).To(Equal("0.0.0-development.cccccccccccc")) + Expect(p.ByDigest()).To(Equal("quay.example/acme/addons@" + resolved)) + }) + It("refuses a tag that cannot be resolved or a answer that is not a digest", func() { + _, err := Pin(context.Background(), mutable, func(context.Context, string, string) (string, error) { return "", errors.New("down") }) + Expect(err).To(HaveOccurred()) + _, err = Pin(context.Background(), mutable, func(context.Context, string, string) (string, error) { return "latest", nil }) + Expect(err).To(HaveOccurred()) + }) + It("keeps a pinned entry as it is and never looks it up", func() { + r := mutable + r.Mutable, r.Digest = false, resolved + p, err := Pin(context.Background(), r, func(context.Context, string, string) (string, error) { + Fail("a pinned entry needs no lookup") + return "", nil + }) + Expect(err).NotTo(HaveOccurred()) + Expect(p.Digest).To(Equal(resolved)) + Expect(p.Version).To(Equal(r.Version)) + }) + It("refuses an entry that is neither mutable nor pinned, and a mutable one that has a digest", func() { + r := mutable + r.Mutable = false + _, err := Pin(context.Background(), r, nil) + Expect(err).To(HaveOccurred()) + r = mutable + r.Digest = resolved + _, err = Pin(context.Background(), r, nil) + Expect(err).To(HaveOccurred()) + }) +}) diff --git a/core/services/addons/catalog/official_test.go b/core/services/addons/catalog/official_test.go index 4ed9282b0..dff91b08c 100644 --- a/core/services/addons/catalog/official_test.go +++ b/core/services/addons/catalog/official_test.go @@ -78,7 +78,14 @@ var _ = Describe("official library index", func() { ids := []string{} for _, r := range releases { ids = append(ids, r.ID) - Expect(digestPattern.MatchString(r.Digest)).To(BeTrue(), r.ID) + // A development entry is mutable and has no digest; any other is pinned. + if r.Mutable { + Expect(r.Digest).To(BeEmpty(), r.ID) + Expect(r.Channel).To(Equal("development"), r.ID) + Expect(r.Artifact).To(HaveSuffix(":"+r.ID+"-development"), r.ID) + } else { + Expect(digestPattern.MatchString(r.Digest)).To(BeTrue(), r.ID) + } Expect(versionPattern.MatchString(r.Version)).To(BeTrue(), r.ID) Expect(r.Artifact).To(HavePrefix("oci://"), r.ID) } diff --git a/core/services/addons/packages/install.go b/core/services/addons/packages/install.go index 0038926dd..f7408f461 100644 --- a/core/services/addons/packages/install.go +++ b/core/services/addons/packages/install.go @@ -39,9 +39,12 @@ type Installer struct { Root string Catalog catalog.Catalog Host contract.HostCompatibility - MaxBytes int64 - MaxFiles int - writeFile func(string, os.FileMode) (fileWriter, error) + // Lookup resolves the tag of a mutable entry to a digest. Nil asks the + // registry; tests replace it. + Lookup catalog.DigestLookup + MaxBytes int64 + MaxFiles int + writeFile func(string, os.FileMode) (fileWriter, error) } func (i *Installer) byteLimit() int64 { @@ -75,9 +78,20 @@ func (i *Installer) Stage(ctx context.Context, r catalog.Release, progress func( if r.Policy.IdentityRegex != "" || r.EntryKind() != catalog.KindAddon { return out, ErrPackage } - if !r.Equal(trusted) || r.Digest == "" || r.OS != i.Host.OS || r.Arch != i.Host.Arch { + if !r.Equal(trusted) || (r.Digest == "" && !r.Mutable) || r.OS != i.Host.OS || r.Arch != i.Host.Arch { return out, ErrPackage } + // A mutable entry (a development image) has no digest in the library. The + // same helper as for backends resolves its tag to a digest here; from then + // on the install is the pinned path and the receipt records that digest. + // Only an OCI artifact can be mutable. + if r.Mutable { + pin, e := catalog.Pin(ctx, r, i.Lookup) + if e != nil { + return out, ErrPackage + } + r.Digest = pin.Digest + } root, e := filepath.Abs(i.Root) if e != nil || i.Root == "" { return out, ErrPackage @@ -217,7 +231,9 @@ func (i *Installer) Stage(ctx context.Context, r catalog.Release, progress func( if e = contract.ValidateManifest(m, i.Host); e != nil { return out, e } - if m.ID != r.ID || m.Version != r.Version { + // A mutable entry names a moving tag, so the version the library shows is a + // label; the package says which build it is, and the digest pins it. + if m.ID != r.ID || (!r.Mutable && m.Version != r.Version) { return out, ErrPackage } assets := []string{m.Executable} diff --git a/core/services/addons/packages/install_test.go b/core/services/addons/packages/install_test.go index 383bfabba..73f46be37 100644 --- a/core/services/addons/packages/install_test.go +++ b/core/services/addons/packages/install_test.go @@ -7,6 +7,7 @@ import ( "context" "crypto/sha256" "encoding/json" + "errors" "fmt" "github.com/google/go-containerregistry/pkg/name" "github.com/google/go-containerregistry/pkg/registry" @@ -656,3 +657,108 @@ var _ = Describe("signer identity pattern", func() { Expect(entries).To(BeEmpty()) }) }) + +var _ = Describe("a mutable development add-on", func() { + // sign publishes a signature of the given digest for the image at artifact. + sign := func(authority *signing.Authority, artifact, signedDigest string) { + ref, e := name.ParseReference(strings.TrimPrefix(artifact, "oci://")) + Expect(e).NotTo(HaveOccurred()) + blob, e := authority.Bundle(signedDigest[7:], "builder@example.invalid", "https://issuer.invalid") + Expect(e).NotTo(HaveOccurred()) + img, e := mutate.AppendLayers(empty.Image, static.NewLayer(blob, types.MediaType("application/vnd.dev.sigstore.bundle.v0.3+json"))) + Expect(e).NotTo(HaveOccurred()) + hash, e := img.Digest() + Expect(e).NotTo(HaveOccurred()) + size, e := img.Size() + Expect(e).NotTo(HaveOccurred()) + index := mutate.AppendManifests(empty.Index, mutate.IndexAddendum{Add: img, Descriptor: v1.Descriptor{Digest: hash, Size: size, MediaType: types.DockerManifestSchema2, ArtifactType: "application/vnd.dev.sigstore.bundle.v0.3+json"}}) + Expect(remote.WriteIndex(ref.Context().Tag(strings.Replace(signedDigest, ":", "-", 1)), index)).To(Succeed()) + } + mutable := func(artifact string, signed bool) func(*catalog.Release, *catalog.Source) { + return func(r *catalog.Release, s *catalog.Source) { + r.Artifact, r.Digest, r.Mutable = artifact, "", true + if signed { + s.Policy = catalog.VerificationPolicy{RequireSignature: true, Issuer: "https://issuer.invalid", Identity: "builder@example.invalid"} + } + } + } + var m []byte + BeforeEach(func() { + var e error + m, e = os.ReadFile("../../../../tests/addons/fixtures/packages/manifest.json") + Expect(e).NotTo(HaveOccurred()) + }) + + It("installs by a digest resolved at install, and the receipt records it", func() { + data := archive(nil, m) + artifact, digest, _ := localOCI(data) + i, r := fixture(data, mutable(artifact, false)) + var asked []string + i.Lookup = func(_ context.Context, image, platform string) (string, error) { + asked = append(asked, image+" "+platform) + return digest, nil + } + staged, e := i.Stage(context.Background(), r, nil) + Expect(e).NotTo(HaveOccurred()) + receipt, e := i.Commit(context.Background(), staged) + Expect(e).NotTo(HaveOccurred()) + Expect(asked).To(Equal([]string{strings.TrimPrefix(artifact, "oci://") + " linux/amd64"})) + Expect(receipt.Digest).To(Equal(digest)) + Expect(receipt.Reference).To(Equal(artifact)) + }) + + It("checks the signature against the resolved digest", func() { + authority, e := signing.New() + Expect(e).NotTo(HaveOccurred()) + data := archive(nil, m) + for _, scenario := range []string{"valid", "other digest"} { + artifact, digest, _ := localOCI(data) + signed := digest + if scenario == "other digest" { + signed = "sha256:" + strings.Repeat("a", 64) + } + sign(authority, artifact, signed) + i, r := fixture(data, mutable(artifact, true)) + i.SignatureTrust = authority + i.Lookup = func(context.Context, string, string) (string, error) { return digest, nil } + staged, e := i.Stage(context.Background(), r, nil) + if scenario == "valid" { + Expect(e).NotTo(HaveOccurred()) + receipt, e := i.Commit(context.Background(), staged) + Expect(e).NotTo(HaveOccurred()) + Expect(receipt.Verification).To(Equal("sha256+sigstore")) + Expect(receipt.Digest).To(Equal(digest)) + } else { + Expect(e).To(MatchError(ErrPackage), scenario) + } + } + }) + + It("is refused when the tag cannot be resolved or resolves to something that is not the image", func() { + data := archive(nil, m) + artifact, _, _ := localOCI(data) + for _, answer := range []func(context.Context, string, string) (string, error){ + func(context.Context, string, string) (string, error) { return "", errors.New("registry down") }, + func(context.Context, string, string) (string, error) { return "latest", nil }, + func(context.Context, string, string) (string, error) { return "sha256:" + strings.Repeat("b", 64), nil }, + } { + i, r := fixture(data, mutable(artifact, false)) + i.Lookup = answer + _, e := i.Stage(context.Background(), r, nil) + Expect(e).To(HaveOccurred()) + } + }) + + It("refuses an entry that has no digest and is not mutable", func() { + data := archive(nil, m) + artifact, _, _ := localOCI(data) + i, r := fixture(data, func(r *catalog.Release, _ *catalog.Source) { r.Artifact = artifact }) + r.Digest = "" + i.Lookup = func(context.Context, string, string) (string, error) { + Fail("only a mutable entry is resolved") + return "", nil + } + _, e := i.Stage(context.Background(), r, nil) + Expect(e).To(HaveOccurred()) + }) +}) diff --git a/core/services/addons/updates/checker.go b/core/services/addons/updates/checker.go index 0f70c427f..da662c667 100644 --- a/core/services/addons/updates/checker.go +++ b/core/services/addons/updates/checker.go @@ -427,6 +427,12 @@ func (c *Checker) moving(ctx context.Context, in Installation, st Status, select st.State, st.Error = CheckFailed, classOf(e) return Result{Status: st} } + // A mutable entry has no digest to compare: its tag is resolved at install. + // Until the check resolves it too, report nothing rather than a false update. + if rel.Mutable { + st.State = Unknown + return Result{Status: st} + } st.CandidateVersion, st.CandidateDigest = rel.Version, rel.Digest if rel.Digest == in.Receipt.Digest { st.State = UpToDate diff --git a/docs/content/features/addons.md b/docs/content/features/addons.md index da5841e2e..0523f76ed 100644 --- a/docs/content/features/addons.md +++ b/docs/content/features/addons.md @@ -3242,6 +3242,13 @@ add-on is a set of tags of that repository, and every tag starts with the add-on build of one commit is `oci://quay.io/go-skynet/local-ai-addons:fine-tuning-sha-99d1dca2c869`. +A release entry of the official library is pinned by digest. A development entry is +mutable, as a development backend is: it names the moving tag +(`oci://quay.io/go-skynet/local-ai-addons:fine-tuning-development`), sets `mutable: true` +and has no digest. At install LocalAI resolves the tag to a digest, verifies the signature +against that digest, installs by it and records it in the receipt. A mutable entry must be +an OCI artifact. The update check does not compare a mutable entry yet. + Set `ADDON_OCI_REPOSITORY` to `quay.io/go-skynet/local-ai-addons`. The value has no tag and no digest. A value that is a backend image repository makes the add-on tools refuse it. `addonpkg target --repository --addon --tag ` prints the diff --git a/gallery/addons/index.yaml b/gallery/addons/index.yaml index 374d19306..c518b1786 100644 --- a/gallery/addons/index.yaml +++ b/gallery/addons/index.yaml @@ -1,15 +1,18 @@ # Official add-on library: the first-party add-ons, one entry for each package. # LocalAI registers this file as its default library (the raw URL of this file # on the master branch). The entries are rendered by the add-on package -# workflow with `addonpkg catalog`, so the digests pin the published images: -# regenerate this file from the workflow output, never edit a digest by hand. +# workflow with `addonpkg catalog`: regenerate this file from the workflow +# output, never edit a digest by hand. A release entry is pinned by digest. A +# development entry is mutable: it keeps the moving tag -development, has +# no digest, and the install resolves the tag to a digest, verifies the +# signature against it and records it, as for a development backend. - id: "fine-tuning" name: Fine-tuning description: "Train an adapter on your own data, then export it as a model you can chat with." - version: "0.0.0-dev.99d1dca2c869" + version: "0.0.0-development" channel: "development" - artifact: "oci://quay.io/go-skynet/local-ai-addons:fine-tuning-sha-99d1dca2c869" - digest: "sha256:9e2168ea73c32427e9f7d3c071d2dd36ca37877a78dd3bdbacebd8b9fe6e270a" + artifact: "oci://quay.io/go-skynet/local-ai-addons:fine-tuning-development" + mutable: true os: "linux" arch: "amd64" policy: @@ -24,10 +27,10 @@ - id: "quantization" name: Quantization description: "Shrink a model to a smaller quantized file, then import it so you can chat with it." - version: "0.0.0-dev.99d1dca2c869" + version: "0.0.0-development" channel: "development" - artifact: "oci://quay.io/go-skynet/local-ai-addons:quantization-sha-99d1dca2c869" - digest: "sha256:d1dd5b3cfc7e96cac2825cd8aaf86ebacb49d686f433f1969ad149928d0858ad" + artifact: "oci://quay.io/go-skynet/local-ai-addons:quantization-development" + mutable: true os: "linux" arch: "amd64" policy: @@ -42,10 +45,10 @@ - id: "voice" name: Voice description: "Recognise who is speaking, keep the voices that text-to-speech can use, and speak text with them." - version: "0.0.0-dev.50f0f3de26d2" + version: "0.0.0-development" channel: "development" - artifact: "oci://quay.io/go-skynet/local-ai-addons:voice-sha-50f0f3de26d2" - digest: "sha256:1343e18fe7b05e6d65da57f6f86ebf75f2861f24ab029a9179935c98fb613583" + artifact: "oci://quay.io/go-skynet/local-ai-addons:voice-development" + mutable: true os: "linux" arch: "amd64" policy: @@ -60,10 +63,10 @@ - id: "face" name: Face description: "Find out whose face is in a photo, or whether two photos show the same person." - version: "0.0.0-dev.99d1dca2c869" + version: "0.0.0-development" channel: "development" - artifact: "oci://quay.io/go-skynet/local-ai-addons:face-sha-99d1dca2c869" - digest: "sha256:a48ad28afc98b10f952e330243487a0fd3ab2aa4187478e8768712f6b77e86f6" + artifact: "oci://quay.io/go-skynet/local-ai-addons:face-development" + mutable: true os: "linux" arch: "amd64" policy: @@ -78,10 +81,10 @@ - id: "studio" name: Studio description: "Make images, video, 3D models, speech, sound and transformed audio with the models you have installed, and keep what you make." - version: "0.0.0-dev.422af83405a3" + version: "0.0.0-development" channel: "development" - artifact: "oci://quay.io/go-skynet/local-ai-addons:studio-sha-422af83405a3" - digest: "sha256:1b06153044ba49ce235a56b0739376b75079f38b939ee62c9f3f6f308ddb8073" + artifact: "oci://quay.io/go-skynet/local-ai-addons:studio-development" + mutable: true os: "linux" arch: "amd64" policy: @@ -96,10 +99,10 @@ - id: "talk" name: Talk description: "Have a spoken conversation with a model: your voice goes in, the answer is spoken back." - version: "0.0.0-dev.422af83405a3" + version: "0.0.0-development" channel: "development" - artifact: "oci://quay.io/go-skynet/local-ai-addons:talk-sha-422af83405a3" - digest: "sha256:678569227df72b597f40859a3830bb3a8a17ec02a19050ad71c416b91f4e7951" + artifact: "oci://quay.io/go-skynet/local-ai-addons:talk-development" + mutable: true os: "linux" arch: "amd64" policy: @@ -114,10 +117,10 @@ - id: "tables" name: Tables description: "Databases of tables you own or share: typed rows, filters, schema and sharing." - version: "0.0.0-dev.4edf30cda051" + version: "0.0.0-development" channel: "development" - artifact: "oci://quay.io/go-skynet/local-ai-addons:tables-sha-4edf30cda051" - digest: "sha256:97863f9c6e848e50cb11617aa1e02d69716e7003d11585388ab5e2547e39d795" + artifact: "oci://quay.io/go-skynet/local-ai-addons:tables-development" + mutable: true os: "linux" arch: "amd64" policy: diff --git a/pkg/addons/packaging/catalog.go b/pkg/addons/packaging/catalog.go index abd6d3913..ef265cc95 100644 --- a/pkg/addons/packaging/catalog.go +++ b/pkg/addons/packaging/catalog.go @@ -30,6 +30,7 @@ type catalogRelease struct { Channel string `json:"channel"` Artifact string `json:"artifact"` Digest string `json:"digest"` + Mutable bool `json:"mutable"` OS string `json:"os"` Arch string `json:"arch"` Policy struct { @@ -151,14 +152,24 @@ func cmdCatalog(args []string) error { description = ui.Description.Default() } } + // The development channel is mutable, like a development backend: the + // entry names the moving tag -development and has no digest. + mutable := *channel == aliasDevelopment + entryVersion, entryTag := m.Version, *tag + if mutable { + entryVersion, entryTag = "0.0.0-development", aliasDevelopment + } values := map[string]string{ - "addon": v.Manifest.ID, "version": m.Version, "tag": addonTagOrEmpty(v.Manifest.ID, *tag), "channel": *channel, + "addon": v.Manifest.ID, "version": entryVersion, "tag": addonTagOrEmpty(v.Manifest.ID, entryTag), "channel": *channel, "repository": repo.Name(), "digest": d.Digest.String(), "os": d.Platform.OS, "arch": d.Platform.Architecture, "issuer": OIDCIssuer, "identity": identity, "title": title, "description": description, "main_page": mainPage, "icon": icon, } text := string(tmpl) + if mutable { + text = strings.Replace(text, "digest: \"{{digest}}\"", "mutable: true", 1) + } for k, val := range values { text = strings.ReplaceAll(text, "{{"+k+"}}", jsonString(val)) } @@ -171,11 +182,14 @@ func cmdCatalog(args []string) error { return fmt.Errorf("rendered entry: %v", err) } r := parsed[0] - regTag, err := addonTag(v.Manifest.ID, *tag) + regTag, err := addonTag(v.Manifest.ID, entryTag) if err != nil { return err } - if err := checkRelease(r, m.Version, d.Digest.String(), repo.Name(), regTag); err != nil { + if mutable != r.Mutable { + return errors.New("the development channel renders a mutable entry and no other channel does") + } + if err := checkRelease(r, entryVersion, d.Digest.String(), repo.Name(), regTag); err != nil { return err } releases = append(releases, r) @@ -230,9 +244,17 @@ func readCatalogIndex(b []byte) ([]catalogRelease, error) { // add-on workflow. func checkRelease(r catalogRelease, version, digest, repository, tag string) error { switch { + case r.Mutable: + // A development entry names the moving tag and carries no digest; the + // install resolves the tag, as it does for a development backend. + if r.Digest != "" { + return errors.New("a mutable entry must not carry a digest") + } case r.Digest != digest || !digestRE.MatchString(r.Digest): return errors.New("entry digest differs from the verified image digest") - case r.Version != version: + } + switch { + case !r.Mutable && r.Version != version: return errors.New("entry version differs from the package version") case r.Artifact != "oci://"+repository+":"+tag: return fmt.Errorf("entry artifact %q must be oci://%s:%s", r.Artifact, repository, tag) diff --git a/pkg/addons/packaging/packaging_test.go b/pkg/addons/packaging/packaging_test.go index a51a35fa7..778003018 100644 --- a/pkg/addons/packaging/packaging_test.go +++ b/pkg/addons/packaging/packaging_test.go @@ -864,6 +864,28 @@ var _ = Describe("catalog template", func() { Expect(entries[0].Digest).NotTo(Equal(entries[1].Digest)) Expect(entries[0].Arch).NotTo(Equal(entries[1].Arch)) }) + It("renders the development channel as mutable entries: the moving tag, no digest", func() { + reg := newRegistry() + p := newProject(both) + m := p.merged("0.0.0-dev.0123456789ab", rev, nil) + repoName := reg.repo("addons/tables") + out := filepath.Join(GinkgoT().TempDir(), "catalog.yaml") + _, err := run("catalog", "--config", p.config, "--template", "../../../gallery/addons/tables.yaml.tmpl", "--merged", m, "--repository", repoName, "--tag", "sha-0123456789ab", "--channel", "development", "--owner-repo", "example/localai", "--signer-ref", "refs/heads/master", "--out", out) + Expect(err).NotTo(HaveOccurred()) + b, err := os.ReadFile(out) + Expect(err).NotTo(HaveOccurred()) + Expect(string(b)).NotTo(ContainSubstring("digest:")) + entries, err := readCatalogIndex(b) + Expect(err).NotTo(HaveOccurred()) + Expect(entries).To(HaveLen(2)) + for _, e := range entries { + Expect(e.Mutable).To(BeTrue()) + Expect(e.Digest).To(BeEmpty()) + Expect(e.Artifact).To(Equal("oci://" + repoName + ":tables-development")) + Expect(e.Channel).To(Equal("development")) + Expect(e.Policy.RequireSignature).To(BeTrue()) + } + }) It("renders an entry for every first-party add-on from its own template", func() { for _, id := range []string{"tables", "voice", "face", "fine-tuning", "quantization", "studio", "talk"} { p := newProjectFor(id, both)