diff --git a/.github/workflows/addons-package.yml b/.github/workflows/addons-package.yml index 33c5bf53f..1c868d75d 100644 --- a/.github/workflows/addons-package.yml +++ b/.github/workflows/addons-package.yml @@ -18,7 +18,10 @@ name: 'add-on packages' # value of ADDON_OCI_REPOSITORY (the add-on registry organization). Each # add-on is a set of tags of that repository, prefixed with its id: # -sha- for a commit, - for a release and -# -development or -latest for a channel alias. Signatures and digests +# -development or -latest for a channel alias. The development alias is +# the tag a development library entry names (mutable, no digest); the install +# resolves it to a digest, as for a development backend, so it moves only in +# the promote job below. Signatures and digests # are per image, so add-ons never collide. A publishing event fails for an # add-on until one of its platforms is enabled. # diff --git a/core/services/addons/backendlib/install.go b/core/services/addons/backendlib/install.go index 6be3bc655..560d12f77 100644 --- a/core/services/addons/backendlib/install.go +++ b/core/services/addons/backendlib/install.go @@ -256,7 +256,7 @@ var digestForm = regexp.MustCompile(`^sha256:[a-f0-9]{64}$`) // the same pinned path: the signature is verified against that digest and the // metadata records it, so an update check can compare it later. func (i *Installer) plan(ctx context.Context, r catalog.Release) (*gallery.GalleryBackend, error) { - if r.EntryKind() != catalog.KindBackend || !strings.HasPrefix(r.Artifact, "oci://") || (r.Mutable != (r.Digest == "")) || (!r.Mutable && !digestForm.MatchString(r.Digest)) { + if r.EntryKind() != catalog.KindBackend { return nil, ErrInvalid } target, arch := i.Catalog.Target() @@ -266,19 +266,12 @@ func (i *Installer) plan(ctx context.Context, r catalog.Release) (*gallery.Galle if i.RequireIntegrity && !r.Policy.RequireSignature { return nil, fmt.Errorf("%w (library %q)", ErrIntegrity, r.SourceID) } - ref, err := name.ParseReference(strings.TrimPrefix(r.Artifact, "oci://")) + // The shared pin: a mutable entry's tag is resolved to a digest here. + pin, err := catalog.Pin(ctx, r, i.digestOf) if err != nil { - return nil, ErrInvalid - } - digest, version := r.Digest, r.Version - if r.Mutable { - d, err := i.digestOf(ctx, ref.Name(), r.OS+"/"+r.Arch) - if err != nil || !digestForm.MatchString(d) { - return nil, fmt.Errorf("%w: resolving %s: %v", ErrInvalid, ref.Name(), err) - } - digest = d - version = r.Version + "." + strings.TrimPrefix(d, "sha256:")[:12] + return nil, fmt.Errorf("%w: %v", ErrInvalid, err) } + ref, digest, version := pin.Ref, pin.Digest, pin.Version gb := &gallery.GalleryBackend{ URI: "oci://" + ref.Context().Digest(digest).Name(), Version: version, diff --git a/core/services/addons/catalog/backend_entry.go b/core/services/addons/catalog/backend_entry.go index bd9bd65f6..aac4a943c 100644 --- a/core/services/addons/catalog/backend_entry.go +++ b/core/services/addons/catalog/backend_entry.go @@ -32,7 +32,7 @@ var capabilityPattern = regexp.MustCompile(`^[a-z][a-z0-9-]{0,63}$`) // backend may set. An add-on entry that does is refused. func (r Release) hasBackendOnlyFields() bool { return r.License != "" || len(r.URLs) > 0 || len(r.Tags) > 0 || r.Icon != "" || - len(r.Mirrors) > 0 || len(r.Capabilities) > 0 || r.Mutable || r.SchemaVersion != 0 + len(r.Mirrors) > 0 || len(r.Capabilities) > 0 || r.SchemaVersion != 0 } // validFetchedRelease is the check a library applies to every entry it keeps. @@ -60,7 +60,11 @@ func validFetchedRelease(r Release, allowLoopback bool) bool { if r.Mutable != (r.Digest == "") || (r.Digest != "" && !digestPattern.MatchString(r.Digest)) { return false } - if r.Mutable && r.EntryKind() != KindBackend { + // A mutable entry is an OCI tag: an HTTP archive has nothing to resolve. + if r.Mutable && r.EntryKind() != KindBackend && r.EntryKind() != KindAddon { + return false + } + if r.Mutable && !strings.HasPrefix(r.Artifact, "oci://") { return false } } diff --git a/core/services/addons/catalog/backend_entry_test.go b/core/services/addons/catalog/backend_entry_test.go index ceadae7b6..dab8706cc 100644 --- a/core/services/addons/catalog/backend_entry_test.go +++ b/core/services/addons/catalog/backend_entry_test.go @@ -99,10 +99,10 @@ var _ = Describe("the backend body of a library entry", func() { Expect(err).To(HaveOccurred()) }) - It("refuses a license, urls, tags, icon, mirrors, capabilities or mutable flag on an add-on", func() { + It("refuses a license, urls, tags, icon, mirrors or capabilities on an add-on", func() { for _, extra := range []string{ " license: mit\n", " urls: [https://example.com]\n", " tags: [a]\n", " icon: https://example.com/i.png\n", - " mirrors: [oci://hub.example/x:y]\n", " capabilities: {default: x}\n", " mutable: true\n", " schemaVersion: 1\n", + " mirrors: [oci://hub.example/x:y]\n", " capabilities: {default: x}\n", " schemaVersion: 1\n", } { _, _, err := read(oldShapeIndex + extra) Expect(err).To(HaveOccurred(), extra) @@ -272,9 +272,22 @@ var _ = Describe("the backend body of a library entry", func() { _, _, err = read(strings.Replace(mutableEntry, " mutable: true\n", "", 1)) Expect(err).To(HaveOccurred()) }) - It("is refused on an add-on", func() { + It("is accepted on an add-on without a digest, as on a backend", func() { + entry := strings.Replace(oldShapeIndex, " digest: "+yamlDigest+"\n", " mutable: true\n", 1) + _, s, err := read(entry) + Expect(err).NotTo(HaveOccurred()) + r, err := Catalog{Sources: s, TargetOS: "linux", TargetArch: "amd64"}.Resolve(ctx, "lib", "tables", "1.0.0") + Expect(err).NotTo(HaveOccurred()) + Expect(r.Mutable).To(BeTrue()) + Expect(r.Digest).To(BeEmpty()) + }) + It("is refused on an add-on with a digest, without a digest unless mutable, or on an archive", func() { _, _, err := read(oldShapeIndex + " mutable: true\n") Expect(err).To(HaveOccurred()) + _, _, err = read(strings.Replace(oldShapeIndex, " digest: "+yamlDigest+"\n", "", 1)) + Expect(err).To(HaveOccurred()) + _, _, err = read(strings.Replace(strings.Replace(oldShapeIndex, " digest: "+yamlDigest+"\n", " mutable: true\n", 1), "oci://registry.example/tables", "https://registry.example/tables.tar", 1)) + Expect(err).To(HaveOccurred()) }) }) diff --git a/core/services/addons/catalog/mutable.go b/core/services/addons/catalog/mutable.go new file mode 100644 index 000000000..241528a8d --- /dev/null +++ b/core/services/addons/catalog/mutable.go @@ -0,0 +1,67 @@ +// SPDX-License-Identifier: MIT +package catalog + +import ( + "context" + "fmt" + "strings" + + "github.com/google/go-containerregistry/pkg/name" + "github.com/mudler/LocalAI/pkg/oci" +) + +// DigestLookup resolves an image reference to the manifest digest for a +// platform. Tests replace it; the default asks the registry. +type DigestLookup func(ctx context.Context, image, platform string) (string, error) + +// Pinned is an entry artifact fixed to one digest. +type Pinned struct { + // Ref is the parsed artifact of the entry. + Ref name.Reference + // Digest is the sha256 digest the install uses. It is the digest of the + // entry, or for a mutable entry the digest its tag named at install. + Digest string + // Version is the version to record. A mutable entry gets the first 12 hex + // digits of the digest appended, so an update check can tell two builds of + // one tag apart. + Version string +} + +// ByDigest is the image reference of the pin: the repository of the artifact +// with the digest, never the tag. +func (p Pinned) ByDigest() string { return p.Ref.Context().Digest(p.Digest).Name() } + +// Pin fixes an OCI entry to a digest. This is the one place where a mutable +// entry (a development image) is turned into a pinned one, for backends and +// for add-ons alike: the tag is resolved to a digest here, at install, and the +// caller then verifies the signature against Pinned.Digest and records it, so +// the install goes through the same path as a release entry. +// +// An entry that is not mutable needs a well-formed digest. A mutable entry has +// none. A nil lookup asks the registry. What differs by kind (the entry kind, +// the platform, the label of the install) stays with the caller. +func Pin(ctx context.Context, r Release, lookup DigestLookup) (Pinned, error) { + if !strings.HasPrefix(r.Artifact, "oci://") || r.Mutable != (r.Digest == "") || (!r.Mutable && !digestPattern.MatchString(r.Digest)) { + return Pinned{}, fmt.Errorf("entry %q: not an OCI artifact pinned by a digest or marked mutable", r.ID) + } + ref, err := name.ParseReference(strings.TrimPrefix(r.Artifact, "oci://")) + if err != nil { + return Pinned{}, fmt.Errorf("entry %q: %w", r.ID, err) + } + p := Pinned{Ref: ref, Digest: r.Digest, Version: r.Version} + if !r.Mutable { + return p, nil + } + if lookup == nil { + lookup = func(_ context.Context, image, platform string) (string, error) { + return oci.GetImageDigest(image, platform, nil, nil) + } + } + d, err := lookup(ctx, ref.Name(), r.OS+"/"+r.Arch) + if err != nil || !digestPattern.MatchString(d) { + return Pinned{}, fmt.Errorf("resolving %s: %v", ref.Name(), err) + } + p.Digest = d + p.Version = r.Version + "." + strings.TrimPrefix(d, "sha256:")[:12] + return p, nil +} diff --git a/core/services/addons/catalog/mutable_test.go b/core/services/addons/catalog/mutable_test.go new file mode 100644 index 000000000..cddbaf0e7 --- /dev/null +++ b/core/services/addons/catalog/mutable_test.go @@ -0,0 +1,55 @@ +// SPDX-License-Identifier: MIT +package catalog + +import ( + "context" + "errors" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("Pin, the one place a mutable entry becomes a digest", func() { + const resolved = "sha256:cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc" + mutable := Release{ID: "demo", Version: "0.0.0-development", Artifact: "oci://quay.example/acme/addons:demo-development", Mutable: true, OS: "linux", Arch: "amd64"} + + It("resolves the tag at install and builds the version from the digest", func() { + var asked string + p, err := Pin(context.Background(), mutable, func(_ context.Context, image, platform string) (string, error) { + asked = image + " " + platform + return resolved, nil + }) + Expect(err).NotTo(HaveOccurred()) + Expect(asked).To(Equal("quay.example/acme/addons:demo-development linux/amd64")) + Expect(p.Digest).To(Equal(resolved)) + Expect(p.Version).To(Equal("0.0.0-development.cccccccccccc")) + Expect(p.ByDigest()).To(Equal("quay.example/acme/addons@" + resolved)) + }) + It("refuses a tag that cannot be resolved or a answer that is not a digest", func() { + _, err := Pin(context.Background(), mutable, func(context.Context, string, string) (string, error) { return "", errors.New("down") }) + Expect(err).To(HaveOccurred()) + _, err = Pin(context.Background(), mutable, func(context.Context, string, string) (string, error) { return "latest", nil }) + Expect(err).To(HaveOccurred()) + }) + It("keeps a pinned entry as it is and never looks it up", func() { + r := mutable + r.Mutable, r.Digest = false, resolved + p, err := Pin(context.Background(), r, func(context.Context, string, string) (string, error) { + Fail("a pinned entry needs no lookup") + return "", nil + }) + Expect(err).NotTo(HaveOccurred()) + Expect(p.Digest).To(Equal(resolved)) + Expect(p.Version).To(Equal(r.Version)) + }) + It("refuses an entry that is neither mutable nor pinned, and a mutable one that has a digest", func() { + r := mutable + r.Mutable = false + _, err := Pin(context.Background(), r, nil) + Expect(err).To(HaveOccurred()) + r = mutable + r.Digest = resolved + _, err = Pin(context.Background(), r, nil) + Expect(err).To(HaveOccurred()) + }) +}) diff --git a/core/services/addons/catalog/official_test.go b/core/services/addons/catalog/official_test.go index 4ed9282b0..dff91b08c 100644 --- a/core/services/addons/catalog/official_test.go +++ b/core/services/addons/catalog/official_test.go @@ -78,7 +78,14 @@ var _ = Describe("official library index", func() { ids := []string{} for _, r := range releases { ids = append(ids, r.ID) - Expect(digestPattern.MatchString(r.Digest)).To(BeTrue(), r.ID) + // A development entry is mutable and has no digest; any other is pinned. + if r.Mutable { + Expect(r.Digest).To(BeEmpty(), r.ID) + Expect(r.Channel).To(Equal("development"), r.ID) + Expect(r.Artifact).To(HaveSuffix(":"+r.ID+"-development"), r.ID) + } else { + Expect(digestPattern.MatchString(r.Digest)).To(BeTrue(), r.ID) + } Expect(versionPattern.MatchString(r.Version)).To(BeTrue(), r.ID) Expect(r.Artifact).To(HavePrefix("oci://"), r.ID) } diff --git a/core/services/addons/packages/install.go b/core/services/addons/packages/install.go index 0038926dd..f7408f461 100644 --- a/core/services/addons/packages/install.go +++ b/core/services/addons/packages/install.go @@ -39,9 +39,12 @@ type Installer struct { Root string Catalog catalog.Catalog Host contract.HostCompatibility - MaxBytes int64 - MaxFiles int - writeFile func(string, os.FileMode) (fileWriter, error) + // Lookup resolves the tag of a mutable entry to a digest. Nil asks the + // registry; tests replace it. + Lookup catalog.DigestLookup + MaxBytes int64 + MaxFiles int + writeFile func(string, os.FileMode) (fileWriter, error) } func (i *Installer) byteLimit() int64 { @@ -75,9 +78,20 @@ func (i *Installer) Stage(ctx context.Context, r catalog.Release, progress func( if r.Policy.IdentityRegex != "" || r.EntryKind() != catalog.KindAddon { return out, ErrPackage } - if !r.Equal(trusted) || r.Digest == "" || r.OS != i.Host.OS || r.Arch != i.Host.Arch { + if !r.Equal(trusted) || (r.Digest == "" && !r.Mutable) || r.OS != i.Host.OS || r.Arch != i.Host.Arch { return out, ErrPackage } + // A mutable entry (a development image) has no digest in the library. The + // same helper as for backends resolves its tag to a digest here; from then + // on the install is the pinned path and the receipt records that digest. + // Only an OCI artifact can be mutable. + if r.Mutable { + pin, e := catalog.Pin(ctx, r, i.Lookup) + if e != nil { + return out, ErrPackage + } + r.Digest = pin.Digest + } root, e := filepath.Abs(i.Root) if e != nil || i.Root == "" { return out, ErrPackage @@ -217,7 +231,9 @@ func (i *Installer) Stage(ctx context.Context, r catalog.Release, progress func( if e = contract.ValidateManifest(m, i.Host); e != nil { return out, e } - if m.ID != r.ID || m.Version != r.Version { + // A mutable entry names a moving tag, so the version the library shows is a + // label; the package says which build it is, and the digest pins it. + if m.ID != r.ID || (!r.Mutable && m.Version != r.Version) { return out, ErrPackage } assets := []string{m.Executable} diff --git a/core/services/addons/packages/install_test.go b/core/services/addons/packages/install_test.go index 383bfabba..73f46be37 100644 --- a/core/services/addons/packages/install_test.go +++ b/core/services/addons/packages/install_test.go @@ -7,6 +7,7 @@ import ( "context" "crypto/sha256" "encoding/json" + "errors" "fmt" "github.com/google/go-containerregistry/pkg/name" "github.com/google/go-containerregistry/pkg/registry" @@ -656,3 +657,108 @@ var _ = Describe("signer identity pattern", func() { Expect(entries).To(BeEmpty()) }) }) + +var _ = Describe("a mutable development add-on", func() { + // sign publishes a signature of the given digest for the image at artifact. + sign := func(authority *signing.Authority, artifact, signedDigest string) { + ref, e := name.ParseReference(strings.TrimPrefix(artifact, "oci://")) + Expect(e).NotTo(HaveOccurred()) + blob, e := authority.Bundle(signedDigest[7:], "builder@example.invalid", "https://issuer.invalid") + Expect(e).NotTo(HaveOccurred()) + img, e := mutate.AppendLayers(empty.Image, static.NewLayer(blob, types.MediaType("application/vnd.dev.sigstore.bundle.v0.3+json"))) + Expect(e).NotTo(HaveOccurred()) + hash, e := img.Digest() + Expect(e).NotTo(HaveOccurred()) + size, e := img.Size() + Expect(e).NotTo(HaveOccurred()) + index := mutate.AppendManifests(empty.Index, mutate.IndexAddendum{Add: img, Descriptor: v1.Descriptor{Digest: hash, Size: size, MediaType: types.DockerManifestSchema2, ArtifactType: "application/vnd.dev.sigstore.bundle.v0.3+json"}}) + Expect(remote.WriteIndex(ref.Context().Tag(strings.Replace(signedDigest, ":", "-", 1)), index)).To(Succeed()) + } + mutable := func(artifact string, signed bool) func(*catalog.Release, *catalog.Source) { + return func(r *catalog.Release, s *catalog.Source) { + r.Artifact, r.Digest, r.Mutable = artifact, "", true + if signed { + s.Policy = catalog.VerificationPolicy{RequireSignature: true, Issuer: "https://issuer.invalid", Identity: "builder@example.invalid"} + } + } + } + var m []byte + BeforeEach(func() { + var e error + m, e = os.ReadFile("../../../../tests/addons/fixtures/packages/manifest.json") + Expect(e).NotTo(HaveOccurred()) + }) + + It("installs by a digest resolved at install, and the receipt records it", func() { + data := archive(nil, m) + artifact, digest, _ := localOCI(data) + i, r := fixture(data, mutable(artifact, false)) + var asked []string + i.Lookup = func(_ context.Context, image, platform string) (string, error) { + asked = append(asked, image+" "+platform) + return digest, nil + } + staged, e := i.Stage(context.Background(), r, nil) + Expect(e).NotTo(HaveOccurred()) + receipt, e := i.Commit(context.Background(), staged) + Expect(e).NotTo(HaveOccurred()) + Expect(asked).To(Equal([]string{strings.TrimPrefix(artifact, "oci://") + " linux/amd64"})) + Expect(receipt.Digest).To(Equal(digest)) + Expect(receipt.Reference).To(Equal(artifact)) + }) + + It("checks the signature against the resolved digest", func() { + authority, e := signing.New() + Expect(e).NotTo(HaveOccurred()) + data := archive(nil, m) + for _, scenario := range []string{"valid", "other digest"} { + artifact, digest, _ := localOCI(data) + signed := digest + if scenario == "other digest" { + signed = "sha256:" + strings.Repeat("a", 64) + } + sign(authority, artifact, signed) + i, r := fixture(data, mutable(artifact, true)) + i.SignatureTrust = authority + i.Lookup = func(context.Context, string, string) (string, error) { return digest, nil } + staged, e := i.Stage(context.Background(), r, nil) + if scenario == "valid" { + Expect(e).NotTo(HaveOccurred()) + receipt, e := i.Commit(context.Background(), staged) + Expect(e).NotTo(HaveOccurred()) + Expect(receipt.Verification).To(Equal("sha256+sigstore")) + Expect(receipt.Digest).To(Equal(digest)) + } else { + Expect(e).To(MatchError(ErrPackage), scenario) + } + } + }) + + It("is refused when the tag cannot be resolved or resolves to something that is not the image", func() { + data := archive(nil, m) + artifact, _, _ := localOCI(data) + for _, answer := range []func(context.Context, string, string) (string, error){ + func(context.Context, string, string) (string, error) { return "", errors.New("registry down") }, + func(context.Context, string, string) (string, error) { return "latest", nil }, + func(context.Context, string, string) (string, error) { return "sha256:" + strings.Repeat("b", 64), nil }, + } { + i, r := fixture(data, mutable(artifact, false)) + i.Lookup = answer + _, e := i.Stage(context.Background(), r, nil) + Expect(e).To(HaveOccurred()) + } + }) + + It("refuses an entry that has no digest and is not mutable", func() { + data := archive(nil, m) + artifact, _, _ := localOCI(data) + i, r := fixture(data, func(r *catalog.Release, _ *catalog.Source) { r.Artifact = artifact }) + r.Digest = "" + i.Lookup = func(context.Context, string, string) (string, error) { + Fail("only a mutable entry is resolved") + return "", nil + } + _, e := i.Stage(context.Background(), r, nil) + Expect(e).To(HaveOccurred()) + }) +}) diff --git a/core/services/addons/updates/checker.go b/core/services/addons/updates/checker.go index 0f70c427f..da662c667 100644 --- a/core/services/addons/updates/checker.go +++ b/core/services/addons/updates/checker.go @@ -427,6 +427,12 @@ func (c *Checker) moving(ctx context.Context, in Installation, st Status, select st.State, st.Error = CheckFailed, classOf(e) return Result{Status: st} } + // A mutable entry has no digest to compare: its tag is resolved at install. + // Until the check resolves it too, report nothing rather than a false update. + if rel.Mutable { + st.State = Unknown + return Result{Status: st} + } st.CandidateVersion, st.CandidateDigest = rel.Version, rel.Digest if rel.Digest == in.Receipt.Digest { st.State = UpToDate diff --git a/docs/content/features/addons.md b/docs/content/features/addons.md index da5841e2e..0523f76ed 100644 --- a/docs/content/features/addons.md +++ b/docs/content/features/addons.md @@ -3242,6 +3242,13 @@ add-on is a set of tags of that repository, and every tag starts with the add-on build of one commit is `oci://quay.io/go-skynet/local-ai-addons:fine-tuning-sha-99d1dca2c869`. +A release entry of the official library is pinned by digest. A development entry is +mutable, as a development backend is: it names the moving tag +(`oci://quay.io/go-skynet/local-ai-addons:fine-tuning-development`), sets `mutable: true` +and has no digest. At install LocalAI resolves the tag to a digest, verifies the signature +against that digest, installs by it and records it in the receipt. A mutable entry must be +an OCI artifact. The update check does not compare a mutable entry yet. + Set `ADDON_OCI_REPOSITORY` to `quay.io/go-skynet/local-ai-addons`. The value has no tag and no digest. A value that is a backend image repository makes the add-on tools refuse it. `addonpkg target --repository --addon --tag ` prints the diff --git a/gallery/addons/index.yaml b/gallery/addons/index.yaml index 374d19306..c518b1786 100644 --- a/gallery/addons/index.yaml +++ b/gallery/addons/index.yaml @@ -1,15 +1,18 @@ # Official add-on library: the first-party add-ons, one entry for each package. # LocalAI registers this file as its default library (the raw URL of this file # on the master branch). The entries are rendered by the add-on package -# workflow with `addonpkg catalog`, so the digests pin the published images: -# regenerate this file from the workflow output, never edit a digest by hand. +# workflow with `addonpkg catalog`: regenerate this file from the workflow +# output, never edit a digest by hand. A release entry is pinned by digest. A +# development entry is mutable: it keeps the moving tag -development, has +# no digest, and the install resolves the tag to a digest, verifies the +# signature against it and records it, as for a development backend. - id: "fine-tuning" name: Fine-tuning description: "Train an adapter on your own data, then export it as a model you can chat with." - version: "0.0.0-dev.99d1dca2c869" + version: "0.0.0-development" channel: "development" - artifact: "oci://quay.io/go-skynet/local-ai-addons:fine-tuning-sha-99d1dca2c869" - digest: "sha256:9e2168ea73c32427e9f7d3c071d2dd36ca37877a78dd3bdbacebd8b9fe6e270a" + artifact: "oci://quay.io/go-skynet/local-ai-addons:fine-tuning-development" + mutable: true os: "linux" arch: "amd64" policy: @@ -24,10 +27,10 @@ - id: "quantization" name: Quantization description: "Shrink a model to a smaller quantized file, then import it so you can chat with it." - version: "0.0.0-dev.99d1dca2c869" + version: "0.0.0-development" channel: "development" - artifact: "oci://quay.io/go-skynet/local-ai-addons:quantization-sha-99d1dca2c869" - digest: "sha256:d1dd5b3cfc7e96cac2825cd8aaf86ebacb49d686f433f1969ad149928d0858ad" + artifact: "oci://quay.io/go-skynet/local-ai-addons:quantization-development" + mutable: true os: "linux" arch: "amd64" policy: @@ -42,10 +45,10 @@ - id: "voice" name: Voice description: "Recognise who is speaking, keep the voices that text-to-speech can use, and speak text with them." - version: "0.0.0-dev.50f0f3de26d2" + version: "0.0.0-development" channel: "development" - artifact: "oci://quay.io/go-skynet/local-ai-addons:voice-sha-50f0f3de26d2" - digest: "sha256:1343e18fe7b05e6d65da57f6f86ebf75f2861f24ab029a9179935c98fb613583" + artifact: "oci://quay.io/go-skynet/local-ai-addons:voice-development" + mutable: true os: "linux" arch: "amd64" policy: @@ -60,10 +63,10 @@ - id: "face" name: Face description: "Find out whose face is in a photo, or whether two photos show the same person." - version: "0.0.0-dev.99d1dca2c869" + version: "0.0.0-development" channel: "development" - artifact: "oci://quay.io/go-skynet/local-ai-addons:face-sha-99d1dca2c869" - digest: "sha256:a48ad28afc98b10f952e330243487a0fd3ab2aa4187478e8768712f6b77e86f6" + artifact: "oci://quay.io/go-skynet/local-ai-addons:face-development" + mutable: true os: "linux" arch: "amd64" policy: @@ -78,10 +81,10 @@ - id: "studio" name: Studio description: "Make images, video, 3D models, speech, sound and transformed audio with the models you have installed, and keep what you make." - version: "0.0.0-dev.422af83405a3" + version: "0.0.0-development" channel: "development" - artifact: "oci://quay.io/go-skynet/local-ai-addons:studio-sha-422af83405a3" - digest: "sha256:1b06153044ba49ce235a56b0739376b75079f38b939ee62c9f3f6f308ddb8073" + artifact: "oci://quay.io/go-skynet/local-ai-addons:studio-development" + mutable: true os: "linux" arch: "amd64" policy: @@ -96,10 +99,10 @@ - id: "talk" name: Talk description: "Have a spoken conversation with a model: your voice goes in, the answer is spoken back." - version: "0.0.0-dev.422af83405a3" + version: "0.0.0-development" channel: "development" - artifact: "oci://quay.io/go-skynet/local-ai-addons:talk-sha-422af83405a3" - digest: "sha256:678569227df72b597f40859a3830bb3a8a17ec02a19050ad71c416b91f4e7951" + artifact: "oci://quay.io/go-skynet/local-ai-addons:talk-development" + mutable: true os: "linux" arch: "amd64" policy: @@ -114,10 +117,10 @@ - id: "tables" name: Tables description: "Databases of tables you own or share: typed rows, filters, schema and sharing." - version: "0.0.0-dev.4edf30cda051" + version: "0.0.0-development" channel: "development" - artifact: "oci://quay.io/go-skynet/local-ai-addons:tables-sha-4edf30cda051" - digest: "sha256:97863f9c6e848e50cb11617aa1e02d69716e7003d11585388ab5e2547e39d795" + artifact: "oci://quay.io/go-skynet/local-ai-addons:tables-development" + mutable: true os: "linux" arch: "amd64" policy: diff --git a/pkg/addons/packaging/catalog.go b/pkg/addons/packaging/catalog.go index abd6d3913..ef265cc95 100644 --- a/pkg/addons/packaging/catalog.go +++ b/pkg/addons/packaging/catalog.go @@ -30,6 +30,7 @@ type catalogRelease struct { Channel string `json:"channel"` Artifact string `json:"artifact"` Digest string `json:"digest"` + Mutable bool `json:"mutable"` OS string `json:"os"` Arch string `json:"arch"` Policy struct { @@ -151,14 +152,24 @@ func cmdCatalog(args []string) error { description = ui.Description.Default() } } + // The development channel is mutable, like a development backend: the + // entry names the moving tag -development and has no digest. + mutable := *channel == aliasDevelopment + entryVersion, entryTag := m.Version, *tag + if mutable { + entryVersion, entryTag = "0.0.0-development", aliasDevelopment + } values := map[string]string{ - "addon": v.Manifest.ID, "version": m.Version, "tag": addonTagOrEmpty(v.Manifest.ID, *tag), "channel": *channel, + "addon": v.Manifest.ID, "version": entryVersion, "tag": addonTagOrEmpty(v.Manifest.ID, entryTag), "channel": *channel, "repository": repo.Name(), "digest": d.Digest.String(), "os": d.Platform.OS, "arch": d.Platform.Architecture, "issuer": OIDCIssuer, "identity": identity, "title": title, "description": description, "main_page": mainPage, "icon": icon, } text := string(tmpl) + if mutable { + text = strings.Replace(text, "digest: \"{{digest}}\"", "mutable: true", 1) + } for k, val := range values { text = strings.ReplaceAll(text, "{{"+k+"}}", jsonString(val)) } @@ -171,11 +182,14 @@ func cmdCatalog(args []string) error { return fmt.Errorf("rendered entry: %v", err) } r := parsed[0] - regTag, err := addonTag(v.Manifest.ID, *tag) + regTag, err := addonTag(v.Manifest.ID, entryTag) if err != nil { return err } - if err := checkRelease(r, m.Version, d.Digest.String(), repo.Name(), regTag); err != nil { + if mutable != r.Mutable { + return errors.New("the development channel renders a mutable entry and no other channel does") + } + if err := checkRelease(r, entryVersion, d.Digest.String(), repo.Name(), regTag); err != nil { return err } releases = append(releases, r) @@ -230,9 +244,17 @@ func readCatalogIndex(b []byte) ([]catalogRelease, error) { // add-on workflow. func checkRelease(r catalogRelease, version, digest, repository, tag string) error { switch { + case r.Mutable: + // A development entry names the moving tag and carries no digest; the + // install resolves the tag, as it does for a development backend. + if r.Digest != "" { + return errors.New("a mutable entry must not carry a digest") + } case r.Digest != digest || !digestRE.MatchString(r.Digest): return errors.New("entry digest differs from the verified image digest") - case r.Version != version: + } + switch { + case !r.Mutable && r.Version != version: return errors.New("entry version differs from the package version") case r.Artifact != "oci://"+repository+":"+tag: return fmt.Errorf("entry artifact %q must be oci://%s:%s", r.Artifact, repository, tag) diff --git a/pkg/addons/packaging/packaging_test.go b/pkg/addons/packaging/packaging_test.go index a51a35fa7..778003018 100644 --- a/pkg/addons/packaging/packaging_test.go +++ b/pkg/addons/packaging/packaging_test.go @@ -864,6 +864,28 @@ var _ = Describe("catalog template", func() { Expect(entries[0].Digest).NotTo(Equal(entries[1].Digest)) Expect(entries[0].Arch).NotTo(Equal(entries[1].Arch)) }) + It("renders the development channel as mutable entries: the moving tag, no digest", func() { + reg := newRegistry() + p := newProject(both) + m := p.merged("0.0.0-dev.0123456789ab", rev, nil) + repoName := reg.repo("addons/tables") + out := filepath.Join(GinkgoT().TempDir(), "catalog.yaml") + _, err := run("catalog", "--config", p.config, "--template", "../../../gallery/addons/tables.yaml.tmpl", "--merged", m, "--repository", repoName, "--tag", "sha-0123456789ab", "--channel", "development", "--owner-repo", "example/localai", "--signer-ref", "refs/heads/master", "--out", out) + Expect(err).NotTo(HaveOccurred()) + b, err := os.ReadFile(out) + Expect(err).NotTo(HaveOccurred()) + Expect(string(b)).NotTo(ContainSubstring("digest:")) + entries, err := readCatalogIndex(b) + Expect(err).NotTo(HaveOccurred()) + Expect(entries).To(HaveLen(2)) + for _, e := range entries { + Expect(e.Mutable).To(BeTrue()) + Expect(e.Digest).To(BeEmpty()) + Expect(e.Artifact).To(Equal("oci://" + repoName + ":tables-development")) + Expect(e.Channel).To(Equal("development")) + Expect(e.Policy.RequireSignature).To(BeTrue()) + } + }) It("renders an entry for every first-party add-on from its own template", func() { for _, id := range []string{"tables", "voice", "face", "fine-tuning", "quantization", "studio", "talk"} { p := newProjectFor(id, both)