mirror of
https://github.com/mudler/LocalAI.git
synced 2026-10-28 00:59:28 -04:00
feat(addons): install development add-ons by tag, as backends do (#12883)
A development add-on entry now names a moving tag, sets mutable and has no digest, the same shape a development backend entry has. A release entry stays pinned by digest. One helper, catalog.Pin, turns a mutable entry into a digest at install: it resolves the tag, checks the digest form and builds the version from the first 12 hex digits. The backend library installer and the add-on installer both call it. The add-on installer then verifies the signature against the resolved digest and records that digest in the receipt. Kind, platform and install label checks stay in each caller. The official index lists the development entries as <id>-development, which the publish workflow already moves in its promote job. The catalog command renders the development channel as mutable entries. The update check reports no state for a mutable entry until it resolves the tag too. Assisted-by: Claude:claude-sonnet-5-5 [Claude Code] Signed-off-by: Ettore Di Giacinto <mudler@localai.io> Co-authored-by: Ettore Di Giacinto <mudler@localai.io>
This commit is contained in:
1 parent
b8f5453cc1
commit
f8b7576dc7
14 files changed
+375
-51
No files matched your search
@@ -18,7 +18,10 @@ name: 'add-on packages'
|
||||
# value of ADDON_OCI_REPOSITORY (the add-on registry organization). Each
|
||||
# add-on is a set of tags of that repository, prefixed with its id:
|
||||
# <id>-sha-<short sha> for a commit, <id>-<version> for a release and
|
||||
# <id>-development or <id>-latest for a channel alias. Signatures and digests
|
||||
# <id>-development or <id>-latest for a channel alias. The development alias is
|
||||
# the tag a development library entry names (mutable, no digest); the install
|
||||
# resolves it to a digest, as for a development backend, so it moves only in
|
||||
# the promote job below. Signatures and digests
|
||||
# are per image, so add-ons never collide. A publishing event fails for an
|
||||
# add-on until one of its platforms is enabled.
|
||||
#
|
||||
|
||||
@@ -256,7 +256,7 @@ var digestForm = regexp.MustCompile(`^sha256:[a-f0-9]{64}$`)
|
||||
// the same pinned path: the signature is verified against that digest and the
|
||||
// metadata records it, so an update check can compare it later.
|
||||
func (i *Installer) plan(ctx context.Context, r catalog.Release) (*gallery.GalleryBackend, error) {
|
||||
if r.EntryKind() != catalog.KindBackend || !strings.HasPrefix(r.Artifact, "oci://") || (r.Mutable != (r.Digest == "")) || (!r.Mutable && !digestForm.MatchString(r.Digest)) {
|
||||
if r.EntryKind() != catalog.KindBackend {
|
||||
return nil, ErrInvalid
|
||||
}
|
||||
target, arch := i.Catalog.Target()
|
||||
@@ -266,19 +266,12 @@ func (i *Installer) plan(ctx context.Context, r catalog.Release) (*gallery.Galle
|
||||
if i.RequireIntegrity && !r.Policy.RequireSignature {
|
||||
return nil, fmt.Errorf("%w (library %q)", ErrIntegrity, r.SourceID)
|
||||
}
|
||||
ref, err := name.ParseReference(strings.TrimPrefix(r.Artifact, "oci://"))
|
||||
// The shared pin: a mutable entry's tag is resolved to a digest here.
|
||||
pin, err := catalog.Pin(ctx, r, i.digestOf)
|
||||
if err != nil {
|
||||
return nil, ErrInvalid
|
||||
}
|
||||
digest, version := r.Digest, r.Version
|
||||
if r.Mutable {
|
||||
d, err := i.digestOf(ctx, ref.Name(), r.OS+"/"+r.Arch)
|
||||
if err != nil || !digestForm.MatchString(d) {
|
||||
return nil, fmt.Errorf("%w: resolving %s: %v", ErrInvalid, ref.Name(), err)
|
||||
}
|
||||
digest = d
|
||||
version = r.Version + "." + strings.TrimPrefix(d, "sha256:")[:12]
|
||||
return nil, fmt.Errorf("%w: %v", ErrInvalid, err)
|
||||
}
|
||||
ref, digest, version := pin.Ref, pin.Digest, pin.Version
|
||||
gb := &gallery.GalleryBackend{
|
||||
URI: "oci://" + ref.Context().Digest(digest).Name(),
|
||||
Version: version,
|
||||
|
||||
@@ -32,7 +32,7 @@ var capabilityPattern = regexp.MustCompile(`^[a-z][a-z0-9-]{0,63}$`)
|
||||
// backend may set. An add-on entry that does is refused.
|
||||
func (r Release) hasBackendOnlyFields() bool {
|
||||
return r.License != "" || len(r.URLs) > 0 || len(r.Tags) > 0 || r.Icon != "" ||
|
||||
len(r.Mirrors) > 0 || len(r.Capabilities) > 0 || r.Mutable || r.SchemaVersion != 0
|
||||
len(r.Mirrors) > 0 || len(r.Capabilities) > 0 || r.SchemaVersion != 0
|
||||
}
|
||||
|
||||
// validFetchedRelease is the check a library applies to every entry it keeps.
|
||||
@@ -60,7 +60,11 @@ func validFetchedRelease(r Release, allowLoopback bool) bool {
|
||||
if r.Mutable != (r.Digest == "") || (r.Digest != "" && !digestPattern.MatchString(r.Digest)) {
|
||||
return false
|
||||
}
|
||||
if r.Mutable && r.EntryKind() != KindBackend {
|
||||
// A mutable entry is an OCI tag: an HTTP archive has nothing to resolve.
|
||||
if r.Mutable && r.EntryKind() != KindBackend && r.EntryKind() != KindAddon {
|
||||
return false
|
||||
}
|
||||
if r.Mutable && !strings.HasPrefix(r.Artifact, "oci://") {
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
@@ -99,10 +99,10 @@ var _ = Describe("the backend body of a library entry", func() {
|
||||
Expect(err).To(HaveOccurred())
|
||||
})
|
||||
|
||||
It("refuses a license, urls, tags, icon, mirrors, capabilities or mutable flag on an add-on", func() {
|
||||
It("refuses a license, urls, tags, icon, mirrors or capabilities on an add-on", func() {
|
||||
for _, extra := range []string{
|
||||
" license: mit\n", " urls: [https://example.com]\n", " tags: [a]\n", " icon: https://example.com/i.png\n",
|
||||
" mirrors: [oci://hub.example/x:y]\n", " capabilities: {default: x}\n", " mutable: true\n", " schemaVersion: 1\n",
|
||||
" mirrors: [oci://hub.example/x:y]\n", " capabilities: {default: x}\n", " schemaVersion: 1\n",
|
||||
} {
|
||||
_, _, err := read(oldShapeIndex + extra)
|
||||
Expect(err).To(HaveOccurred(), extra)
|
||||
@@ -272,9 +272,22 @@ var _ = Describe("the backend body of a library entry", func() {
|
||||
_, _, err = read(strings.Replace(mutableEntry, " mutable: true\n", "", 1))
|
||||
Expect(err).To(HaveOccurred())
|
||||
})
|
||||
It("is refused on an add-on", func() {
|
||||
It("is accepted on an add-on without a digest, as on a backend", func() {
|
||||
entry := strings.Replace(oldShapeIndex, " digest: "+yamlDigest+"\n", " mutable: true\n", 1)
|
||||
_, s, err := read(entry)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
r, err := Catalog{Sources: s, TargetOS: "linux", TargetArch: "amd64"}.Resolve(ctx, "lib", "tables", "1.0.0")
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
Expect(r.Mutable).To(BeTrue())
|
||||
Expect(r.Digest).To(BeEmpty())
|
||||
})
|
||||
It("is refused on an add-on with a digest, without a digest unless mutable, or on an archive", func() {
|
||||
_, _, err := read(oldShapeIndex + " mutable: true\n")
|
||||
Expect(err).To(HaveOccurred())
|
||||
_, _, err = read(strings.Replace(oldShapeIndex, " digest: "+yamlDigest+"\n", "", 1))
|
||||
Expect(err).To(HaveOccurred())
|
||||
_, _, err = read(strings.Replace(strings.Replace(oldShapeIndex, " digest: "+yamlDigest+"\n", " mutable: true\n", 1), "oci://registry.example/tables", "https://registry.example/tables.tar", 1))
|
||||
Expect(err).To(HaveOccurred())
|
||||
})
|
||||
})
|
||||
|
||||
|
||||
@@ -0,0 +1,67 @@
|
||||
// SPDX-License-Identifier: MIT
|
||||
package catalog
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
"github.com/google/go-containerregistry/pkg/name"
|
||||
"github.com/mudler/LocalAI/pkg/oci"
|
||||
)
|
||||
|
||||
// DigestLookup resolves an image reference to the manifest digest for a
|
||||
// platform. Tests replace it; the default asks the registry.
|
||||
type DigestLookup func(ctx context.Context, image, platform string) (string, error)
|
||||
|
||||
// Pinned is an entry artifact fixed to one digest.
|
||||
type Pinned struct {
|
||||
// Ref is the parsed artifact of the entry.
|
||||
Ref name.Reference
|
||||
// Digest is the sha256 digest the install uses. It is the digest of the
|
||||
// entry, or for a mutable entry the digest its tag named at install.
|
||||
Digest string
|
||||
// Version is the version to record. A mutable entry gets the first 12 hex
|
||||
// digits of the digest appended, so an update check can tell two builds of
|
||||
// one tag apart.
|
||||
Version string
|
||||
}
|
||||
|
||||
// ByDigest is the image reference of the pin: the repository of the artifact
|
||||
// with the digest, never the tag.
|
||||
func (p Pinned) ByDigest() string { return p.Ref.Context().Digest(p.Digest).Name() }
|
||||
|
||||
// Pin fixes an OCI entry to a digest. This is the one place where a mutable
|
||||
// entry (a development image) is turned into a pinned one, for backends and
|
||||
// for add-ons alike: the tag is resolved to a digest here, at install, and the
|
||||
// caller then verifies the signature against Pinned.Digest and records it, so
|
||||
// the install goes through the same path as a release entry.
|
||||
//
|
||||
// An entry that is not mutable needs a well-formed digest. A mutable entry has
|
||||
// none. A nil lookup asks the registry. What differs by kind (the entry kind,
|
||||
// the platform, the label of the install) stays with the caller.
|
||||
func Pin(ctx context.Context, r Release, lookup DigestLookup) (Pinned, error) {
|
||||
if !strings.HasPrefix(r.Artifact, "oci://") || r.Mutable != (r.Digest == "") || (!r.Mutable && !digestPattern.MatchString(r.Digest)) {
|
||||
return Pinned{}, fmt.Errorf("entry %q: not an OCI artifact pinned by a digest or marked mutable", r.ID)
|
||||
}
|
||||
ref, err := name.ParseReference(strings.TrimPrefix(r.Artifact, "oci://"))
|
||||
if err != nil {
|
||||
return Pinned{}, fmt.Errorf("entry %q: %w", r.ID, err)
|
||||
}
|
||||
p := Pinned{Ref: ref, Digest: r.Digest, Version: r.Version}
|
||||
if !r.Mutable {
|
||||
return p, nil
|
||||
}
|
||||
if lookup == nil {
|
||||
lookup = func(_ context.Context, image, platform string) (string, error) {
|
||||
return oci.GetImageDigest(image, platform, nil, nil)
|
||||
}
|
||||
}
|
||||
d, err := lookup(ctx, ref.Name(), r.OS+"/"+r.Arch)
|
||||
if err != nil || !digestPattern.MatchString(d) {
|
||||
return Pinned{}, fmt.Errorf("resolving %s: %v", ref.Name(), err)
|
||||
}
|
||||
p.Digest = d
|
||||
p.Version = r.Version + "." + strings.TrimPrefix(d, "sha256:")[:12]
|
||||
return p, nil
|
||||
}
|
||||
@@ -0,0 +1,55 @@
|
||||
// SPDX-License-Identifier: MIT
|
||||
package catalog
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
|
||||
. "github.com/onsi/ginkgo/v2"
|
||||
. "github.com/onsi/gomega"
|
||||
)
|
||||
|
||||
var _ = Describe("Pin, the one place a mutable entry becomes a digest", func() {
|
||||
const resolved = "sha256:cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc"
|
||||
mutable := Release{ID: "demo", Version: "0.0.0-development", Artifact: "oci://quay.example/acme/addons:demo-development", Mutable: true, OS: "linux", Arch: "amd64"}
|
||||
|
||||
It("resolves the tag at install and builds the version from the digest", func() {
|
||||
var asked string
|
||||
p, err := Pin(context.Background(), mutable, func(_ context.Context, image, platform string) (string, error) {
|
||||
asked = image + " " + platform
|
||||
return resolved, nil
|
||||
})
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
Expect(asked).To(Equal("quay.example/acme/addons:demo-development linux/amd64"))
|
||||
Expect(p.Digest).To(Equal(resolved))
|
||||
Expect(p.Version).To(Equal("0.0.0-development.cccccccccccc"))
|
||||
Expect(p.ByDigest()).To(Equal("quay.example/acme/addons@" + resolved))
|
||||
})
|
||||
It("refuses a tag that cannot be resolved or a answer that is not a digest", func() {
|
||||
_, err := Pin(context.Background(), mutable, func(context.Context, string, string) (string, error) { return "", errors.New("down") })
|
||||
Expect(err).To(HaveOccurred())
|
||||
_, err = Pin(context.Background(), mutable, func(context.Context, string, string) (string, error) { return "latest", nil })
|
||||
Expect(err).To(HaveOccurred())
|
||||
})
|
||||
It("keeps a pinned entry as it is and never looks it up", func() {
|
||||
r := mutable
|
||||
r.Mutable, r.Digest = false, resolved
|
||||
p, err := Pin(context.Background(), r, func(context.Context, string, string) (string, error) {
|
||||
Fail("a pinned entry needs no lookup")
|
||||
return "", nil
|
||||
})
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
Expect(p.Digest).To(Equal(resolved))
|
||||
Expect(p.Version).To(Equal(r.Version))
|
||||
})
|
||||
It("refuses an entry that is neither mutable nor pinned, and a mutable one that has a digest", func() {
|
||||
r := mutable
|
||||
r.Mutable = false
|
||||
_, err := Pin(context.Background(), r, nil)
|
||||
Expect(err).To(HaveOccurred())
|
||||
r = mutable
|
||||
r.Digest = resolved
|
||||
_, err = Pin(context.Background(), r, nil)
|
||||
Expect(err).To(HaveOccurred())
|
||||
})
|
||||
})
|
||||
@@ -78,7 +78,14 @@ var _ = Describe("official library index", func() {
|
||||
ids := []string{}
|
||||
for _, r := range releases {
|
||||
ids = append(ids, r.ID)
|
||||
Expect(digestPattern.MatchString(r.Digest)).To(BeTrue(), r.ID)
|
||||
// A development entry is mutable and has no digest; any other is pinned.
|
||||
if r.Mutable {
|
||||
Expect(r.Digest).To(BeEmpty(), r.ID)
|
||||
Expect(r.Channel).To(Equal("development"), r.ID)
|
||||
Expect(r.Artifact).To(HaveSuffix(":"+r.ID+"-development"), r.ID)
|
||||
} else {
|
||||
Expect(digestPattern.MatchString(r.Digest)).To(BeTrue(), r.ID)
|
||||
}
|
||||
Expect(versionPattern.MatchString(r.Version)).To(BeTrue(), r.ID)
|
||||
Expect(r.Artifact).To(HavePrefix("oci://"), r.ID)
|
||||
}
|
||||
|
||||
@@ -39,9 +39,12 @@ type Installer struct {
|
||||
Root string
|
||||
Catalog catalog.Catalog
|
||||
Host contract.HostCompatibility
|
||||
MaxBytes int64
|
||||
MaxFiles int
|
||||
writeFile func(string, os.FileMode) (fileWriter, error)
|
||||
// Lookup resolves the tag of a mutable entry to a digest. Nil asks the
|
||||
// registry; tests replace it.
|
||||
Lookup catalog.DigestLookup
|
||||
MaxBytes int64
|
||||
MaxFiles int
|
||||
writeFile func(string, os.FileMode) (fileWriter, error)
|
||||
}
|
||||
|
||||
func (i *Installer) byteLimit() int64 {
|
||||
@@ -75,9 +78,20 @@ func (i *Installer) Stage(ctx context.Context, r catalog.Release, progress func(
|
||||
if r.Policy.IdentityRegex != "" || r.EntryKind() != catalog.KindAddon {
|
||||
return out, ErrPackage
|
||||
}
|
||||
if !r.Equal(trusted) || r.Digest == "" || r.OS != i.Host.OS || r.Arch != i.Host.Arch {
|
||||
if !r.Equal(trusted) || (r.Digest == "" && !r.Mutable) || r.OS != i.Host.OS || r.Arch != i.Host.Arch {
|
||||
return out, ErrPackage
|
||||
}
|
||||
// A mutable entry (a development image) has no digest in the library. The
|
||||
// same helper as for backends resolves its tag to a digest here; from then
|
||||
// on the install is the pinned path and the receipt records that digest.
|
||||
// Only an OCI artifact can be mutable.
|
||||
if r.Mutable {
|
||||
pin, e := catalog.Pin(ctx, r, i.Lookup)
|
||||
if e != nil {
|
||||
return out, ErrPackage
|
||||
}
|
||||
r.Digest = pin.Digest
|
||||
}
|
||||
root, e := filepath.Abs(i.Root)
|
||||
if e != nil || i.Root == "" {
|
||||
return out, ErrPackage
|
||||
@@ -217,7 +231,9 @@ func (i *Installer) Stage(ctx context.Context, r catalog.Release, progress func(
|
||||
if e = contract.ValidateManifest(m, i.Host); e != nil {
|
||||
return out, e
|
||||
}
|
||||
if m.ID != r.ID || m.Version != r.Version {
|
||||
// A mutable entry names a moving tag, so the version the library shows is a
|
||||
// label; the package says which build it is, and the digest pins it.
|
||||
if m.ID != r.ID || (!r.Mutable && m.Version != r.Version) {
|
||||
return out, ErrPackage
|
||||
}
|
||||
assets := []string{m.Executable}
|
||||
|
||||
@@ -7,6 +7,7 @@ import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"github.com/google/go-containerregistry/pkg/name"
|
||||
"github.com/google/go-containerregistry/pkg/registry"
|
||||
@@ -656,3 +657,108 @@ var _ = Describe("signer identity pattern", func() {
|
||||
Expect(entries).To(BeEmpty())
|
||||
})
|
||||
})
|
||||
|
||||
var _ = Describe("a mutable development add-on", func() {
|
||||
// sign publishes a signature of the given digest for the image at artifact.
|
||||
sign := func(authority *signing.Authority, artifact, signedDigest string) {
|
||||
ref, e := name.ParseReference(strings.TrimPrefix(artifact, "oci://"))
|
||||
Expect(e).NotTo(HaveOccurred())
|
||||
blob, e := authority.Bundle(signedDigest[7:], "builder@example.invalid", "https://issuer.invalid")
|
||||
Expect(e).NotTo(HaveOccurred())
|
||||
img, e := mutate.AppendLayers(empty.Image, static.NewLayer(blob, types.MediaType("application/vnd.dev.sigstore.bundle.v0.3+json")))
|
||||
Expect(e).NotTo(HaveOccurred())
|
||||
hash, e := img.Digest()
|
||||
Expect(e).NotTo(HaveOccurred())
|
||||
size, e := img.Size()
|
||||
Expect(e).NotTo(HaveOccurred())
|
||||
index := mutate.AppendManifests(empty.Index, mutate.IndexAddendum{Add: img, Descriptor: v1.Descriptor{Digest: hash, Size: size, MediaType: types.DockerManifestSchema2, ArtifactType: "application/vnd.dev.sigstore.bundle.v0.3+json"}})
|
||||
Expect(remote.WriteIndex(ref.Context().Tag(strings.Replace(signedDigest, ":", "-", 1)), index)).To(Succeed())
|
||||
}
|
||||
mutable := func(artifact string, signed bool) func(*catalog.Release, *catalog.Source) {
|
||||
return func(r *catalog.Release, s *catalog.Source) {
|
||||
r.Artifact, r.Digest, r.Mutable = artifact, "", true
|
||||
if signed {
|
||||
s.Policy = catalog.VerificationPolicy{RequireSignature: true, Issuer: "https://issuer.invalid", Identity: "builder@example.invalid"}
|
||||
}
|
||||
}
|
||||
}
|
||||
var m []byte
|
||||
BeforeEach(func() {
|
||||
var e error
|
||||
m, e = os.ReadFile("../../../../tests/addons/fixtures/packages/manifest.json")
|
||||
Expect(e).NotTo(HaveOccurred())
|
||||
})
|
||||
|
||||
It("installs by a digest resolved at install, and the receipt records it", func() {
|
||||
data := archive(nil, m)
|
||||
artifact, digest, _ := localOCI(data)
|
||||
i, r := fixture(data, mutable(artifact, false))
|
||||
var asked []string
|
||||
i.Lookup = func(_ context.Context, image, platform string) (string, error) {
|
||||
asked = append(asked, image+" "+platform)
|
||||
return digest, nil
|
||||
}
|
||||
staged, e := i.Stage(context.Background(), r, nil)
|
||||
Expect(e).NotTo(HaveOccurred())
|
||||
receipt, e := i.Commit(context.Background(), staged)
|
||||
Expect(e).NotTo(HaveOccurred())
|
||||
Expect(asked).To(Equal([]string{strings.TrimPrefix(artifact, "oci://") + " linux/amd64"}))
|
||||
Expect(receipt.Digest).To(Equal(digest))
|
||||
Expect(receipt.Reference).To(Equal(artifact))
|
||||
})
|
||||
|
||||
It("checks the signature against the resolved digest", func() {
|
||||
authority, e := signing.New()
|
||||
Expect(e).NotTo(HaveOccurred())
|
||||
data := archive(nil, m)
|
||||
for _, scenario := range []string{"valid", "other digest"} {
|
||||
artifact, digest, _ := localOCI(data)
|
||||
signed := digest
|
||||
if scenario == "other digest" {
|
||||
signed = "sha256:" + strings.Repeat("a", 64)
|
||||
}
|
||||
sign(authority, artifact, signed)
|
||||
i, r := fixture(data, mutable(artifact, true))
|
||||
i.SignatureTrust = authority
|
||||
i.Lookup = func(context.Context, string, string) (string, error) { return digest, nil }
|
||||
staged, e := i.Stage(context.Background(), r, nil)
|
||||
if scenario == "valid" {
|
||||
Expect(e).NotTo(HaveOccurred())
|
||||
receipt, e := i.Commit(context.Background(), staged)
|
||||
Expect(e).NotTo(HaveOccurred())
|
||||
Expect(receipt.Verification).To(Equal("sha256+sigstore"))
|
||||
Expect(receipt.Digest).To(Equal(digest))
|
||||
} else {
|
||||
Expect(e).To(MatchError(ErrPackage), scenario)
|
||||
}
|
||||
}
|
||||
})
|
||||
|
||||
It("is refused when the tag cannot be resolved or resolves to something that is not the image", func() {
|
||||
data := archive(nil, m)
|
||||
artifact, _, _ := localOCI(data)
|
||||
for _, answer := range []func(context.Context, string, string) (string, error){
|
||||
func(context.Context, string, string) (string, error) { return "", errors.New("registry down") },
|
||||
func(context.Context, string, string) (string, error) { return "latest", nil },
|
||||
func(context.Context, string, string) (string, error) { return "sha256:" + strings.Repeat("b", 64), nil },
|
||||
} {
|
||||
i, r := fixture(data, mutable(artifact, false))
|
||||
i.Lookup = answer
|
||||
_, e := i.Stage(context.Background(), r, nil)
|
||||
Expect(e).To(HaveOccurred())
|
||||
}
|
||||
})
|
||||
|
||||
It("refuses an entry that has no digest and is not mutable", func() {
|
||||
data := archive(nil, m)
|
||||
artifact, _, _ := localOCI(data)
|
||||
i, r := fixture(data, func(r *catalog.Release, _ *catalog.Source) { r.Artifact = artifact })
|
||||
r.Digest = ""
|
||||
i.Lookup = func(context.Context, string, string) (string, error) {
|
||||
Fail("only a mutable entry is resolved")
|
||||
return "", nil
|
||||
}
|
||||
_, e := i.Stage(context.Background(), r, nil)
|
||||
Expect(e).To(HaveOccurred())
|
||||
})
|
||||
})
|
||||
@@ -427,6 +427,12 @@ func (c *Checker) moving(ctx context.Context, in Installation, st Status, select
|
||||
st.State, st.Error = CheckFailed, classOf(e)
|
||||
return Result{Status: st}
|
||||
}
|
||||
// A mutable entry has no digest to compare: its tag is resolved at install.
|
||||
// Until the check resolves it too, report nothing rather than a false update.
|
||||
if rel.Mutable {
|
||||
st.State = Unknown
|
||||
return Result{Status: st}
|
||||
}
|
||||
st.CandidateVersion, st.CandidateDigest = rel.Version, rel.Digest
|
||||
if rel.Digest == in.Receipt.Digest {
|
||||
st.State = UpToDate
|
||||
|
||||
@@ -3242,6 +3242,13 @@ add-on is a set of tags of that repository, and every tag starts with the add-on
|
||||
build of one commit is
|
||||
`oci://quay.io/go-skynet/local-ai-addons:fine-tuning-sha-99d1dca2c869`.
|
||||
|
||||
A release entry of the official library is pinned by digest. A development entry is
|
||||
mutable, as a development backend is: it names the moving tag
|
||||
(`oci://quay.io/go-skynet/local-ai-addons:fine-tuning-development`), sets `mutable: true`
|
||||
and has no digest. At install LocalAI resolves the tag to a digest, verifies the signature
|
||||
against that digest, installs by it and records it in the receipt. A mutable entry must be
|
||||
an OCI artifact. The update check does not compare a mutable entry yet.
|
||||
|
||||
Set `ADDON_OCI_REPOSITORY` to `quay.io/go-skynet/local-ai-addons`. The value has no tag
|
||||
and no digest. A value that is a backend image repository makes the add-on tools refuse
|
||||
it. `addonpkg target --repository <repository> --addon <id> --tag <tag>` prints the
|
||||
|
||||
+26
-23
@@ -1,15 +1,18 @@
|
||||
# Official add-on library: the first-party add-ons, one entry for each package.
|
||||
# LocalAI registers this file as its default library (the raw URL of this file
|
||||
# on the master branch). The entries are rendered by the add-on package
|
||||
# workflow with `addonpkg catalog`, so the digests pin the published images:
|
||||
# regenerate this file from the workflow output, never edit a digest by hand.
|
||||
# workflow with `addonpkg catalog`: regenerate this file from the workflow
|
||||
# output, never edit a digest by hand. A release entry is pinned by digest. A
|
||||
# development entry is mutable: it keeps the moving tag <id>-development, has
|
||||
# no digest, and the install resolves the tag to a digest, verifies the
|
||||
# signature against it and records it, as for a development backend.
|
||||
- id: "fine-tuning"
|
||||
name: Fine-tuning
|
||||
description: "Train an adapter on your own data, then export it as a model you can chat with."
|
||||
version: "0.0.0-dev.99d1dca2c869"
|
||||
version: "0.0.0-development"
|
||||
channel: "development"
|
||||
artifact: "oci://quay.io/go-skynet/local-ai-addons:fine-tuning-sha-99d1dca2c869"
|
||||
digest: "sha256:9e2168ea73c32427e9f7d3c071d2dd36ca37877a78dd3bdbacebd8b9fe6e270a"
|
||||
artifact: "oci://quay.io/go-skynet/local-ai-addons:fine-tuning-development"
|
||||
mutable: true
|
||||
os: "linux"
|
||||
arch: "amd64"
|
||||
policy:
|
||||
@@ -24,10 +27,10 @@
|
||||
- id: "quantization"
|
||||
name: Quantization
|
||||
description: "Shrink a model to a smaller quantized file, then import it so you can chat with it."
|
||||
version: "0.0.0-dev.99d1dca2c869"
|
||||
version: "0.0.0-development"
|
||||
channel: "development"
|
||||
artifact: "oci://quay.io/go-skynet/local-ai-addons:quantization-sha-99d1dca2c869"
|
||||
digest: "sha256:d1dd5b3cfc7e96cac2825cd8aaf86ebacb49d686f433f1969ad149928d0858ad"
|
||||
artifact: "oci://quay.io/go-skynet/local-ai-addons:quantization-development"
|
||||
mutable: true
|
||||
os: "linux"
|
||||
arch: "amd64"
|
||||
policy:
|
||||
@@ -42,10 +45,10 @@
|
||||
- id: "voice"
|
||||
name: Voice
|
||||
description: "Recognise who is speaking, keep the voices that text-to-speech can use, and speak text with them."
|
||||
version: "0.0.0-dev.50f0f3de26d2"
|
||||
version: "0.0.0-development"
|
||||
channel: "development"
|
||||
artifact: "oci://quay.io/go-skynet/local-ai-addons:voice-sha-50f0f3de26d2"
|
||||
digest: "sha256:1343e18fe7b05e6d65da57f6f86ebf75f2861f24ab029a9179935c98fb613583"
|
||||
artifact: "oci://quay.io/go-skynet/local-ai-addons:voice-development"
|
||||
mutable: true
|
||||
os: "linux"
|
||||
arch: "amd64"
|
||||
policy:
|
||||
@@ -60,10 +63,10 @@
|
||||
- id: "face"
|
||||
name: Face
|
||||
description: "Find out whose face is in a photo, or whether two photos show the same person."
|
||||
version: "0.0.0-dev.99d1dca2c869"
|
||||
version: "0.0.0-development"
|
||||
channel: "development"
|
||||
artifact: "oci://quay.io/go-skynet/local-ai-addons:face-sha-99d1dca2c869"
|
||||
digest: "sha256:a48ad28afc98b10f952e330243487a0fd3ab2aa4187478e8768712f6b77e86f6"
|
||||
artifact: "oci://quay.io/go-skynet/local-ai-addons:face-development"
|
||||
mutable: true
|
||||
os: "linux"
|
||||
arch: "amd64"
|
||||
policy:
|
||||
@@ -78,10 +81,10 @@
|
||||
- id: "studio"
|
||||
name: Studio
|
||||
description: "Make images, video, 3D models, speech, sound and transformed audio with the models you have installed, and keep what you make."
|
||||
version: "0.0.0-dev.422af83405a3"
|
||||
version: "0.0.0-development"
|
||||
channel: "development"
|
||||
artifact: "oci://quay.io/go-skynet/local-ai-addons:studio-sha-422af83405a3"
|
||||
digest: "sha256:1b06153044ba49ce235a56b0739376b75079f38b939ee62c9f3f6f308ddb8073"
|
||||
artifact: "oci://quay.io/go-skynet/local-ai-addons:studio-development"
|
||||
mutable: true
|
||||
os: "linux"
|
||||
arch: "amd64"
|
||||
policy:
|
||||
@@ -96,10 +99,10 @@
|
||||
- id: "talk"
|
||||
name: Talk
|
||||
description: "Have a spoken conversation with a model: your voice goes in, the answer is spoken back."
|
||||
version: "0.0.0-dev.422af83405a3"
|
||||
version: "0.0.0-development"
|
||||
channel: "development"
|
||||
artifact: "oci://quay.io/go-skynet/local-ai-addons:talk-sha-422af83405a3"
|
||||
digest: "sha256:678569227df72b597f40859a3830bb3a8a17ec02a19050ad71c416b91f4e7951"
|
||||
artifact: "oci://quay.io/go-skynet/local-ai-addons:talk-development"
|
||||
mutable: true
|
||||
os: "linux"
|
||||
arch: "amd64"
|
||||
policy:
|
||||
@@ -114,10 +117,10 @@
|
||||
- id: "tables"
|
||||
name: Tables
|
||||
description: "Databases of tables you own or share: typed rows, filters, schema and sharing."
|
||||
version: "0.0.0-dev.4edf30cda051"
|
||||
version: "0.0.0-development"
|
||||
channel: "development"
|
||||
artifact: "oci://quay.io/go-skynet/local-ai-addons:tables-sha-4edf30cda051"
|
||||
digest: "sha256:97863f9c6e848e50cb11617aa1e02d69716e7003d11585388ab5e2547e39d795"
|
||||
artifact: "oci://quay.io/go-skynet/local-ai-addons:tables-development"
|
||||
mutable: true
|
||||
os: "linux"
|
||||
arch: "amd64"
|
||||
policy:
|
||||
|
||||
@@ -30,6 +30,7 @@ type catalogRelease struct {
|
||||
Channel string `json:"channel"`
|
||||
Artifact string `json:"artifact"`
|
||||
Digest string `json:"digest"`
|
||||
Mutable bool `json:"mutable"`
|
||||
OS string `json:"os"`
|
||||
Arch string `json:"arch"`
|
||||
Policy struct {
|
||||
@@ -151,14 +152,24 @@ func cmdCatalog(args []string) error {
|
||||
description = ui.Description.Default()
|
||||
}
|
||||
}
|
||||
// The development channel is mutable, like a development backend: the
|
||||
// entry names the moving tag <id>-development and has no digest.
|
||||
mutable := *channel == aliasDevelopment
|
||||
entryVersion, entryTag := m.Version, *tag
|
||||
if mutable {
|
||||
entryVersion, entryTag = "0.0.0-development", aliasDevelopment
|
||||
}
|
||||
values := map[string]string{
|
||||
"addon": v.Manifest.ID, "version": m.Version, "tag": addonTagOrEmpty(v.Manifest.ID, *tag), "channel": *channel,
|
||||
"addon": v.Manifest.ID, "version": entryVersion, "tag": addonTagOrEmpty(v.Manifest.ID, entryTag), "channel": *channel,
|
||||
"repository": repo.Name(), "digest": d.Digest.String(),
|
||||
"os": d.Platform.OS, "arch": d.Platform.Architecture,
|
||||
"issuer": OIDCIssuer, "identity": identity,
|
||||
"title": title, "description": description, "main_page": mainPage, "icon": icon,
|
||||
}
|
||||
text := string(tmpl)
|
||||
if mutable {
|
||||
text = strings.Replace(text, "digest: \"{{digest}}\"", "mutable: true", 1)
|
||||
}
|
||||
for k, val := range values {
|
||||
text = strings.ReplaceAll(text, "{{"+k+"}}", jsonString(val))
|
||||
}
|
||||
@@ -171,11 +182,14 @@ func cmdCatalog(args []string) error {
|
||||
return fmt.Errorf("rendered entry: %v", err)
|
||||
}
|
||||
r := parsed[0]
|
||||
regTag, err := addonTag(v.Manifest.ID, *tag)
|
||||
regTag, err := addonTag(v.Manifest.ID, entryTag)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := checkRelease(r, m.Version, d.Digest.String(), repo.Name(), regTag); err != nil {
|
||||
if mutable != r.Mutable {
|
||||
return errors.New("the development channel renders a mutable entry and no other channel does")
|
||||
}
|
||||
if err := checkRelease(r, entryVersion, d.Digest.String(), repo.Name(), regTag); err != nil {
|
||||
return err
|
||||
}
|
||||
releases = append(releases, r)
|
||||
@@ -230,9 +244,17 @@ func readCatalogIndex(b []byte) ([]catalogRelease, error) {
|
||||
// add-on workflow.
|
||||
func checkRelease(r catalogRelease, version, digest, repository, tag string) error {
|
||||
switch {
|
||||
case r.Mutable:
|
||||
// A development entry names the moving tag and carries no digest; the
|
||||
// install resolves the tag, as it does for a development backend.
|
||||
if r.Digest != "" {
|
||||
return errors.New("a mutable entry must not carry a digest")
|
||||
}
|
||||
case r.Digest != digest || !digestRE.MatchString(r.Digest):
|
||||
return errors.New("entry digest differs from the verified image digest")
|
||||
case r.Version != version:
|
||||
}
|
||||
switch {
|
||||
case !r.Mutable && r.Version != version:
|
||||
return errors.New("entry version differs from the package version")
|
||||
case r.Artifact != "oci://"+repository+":"+tag:
|
||||
return fmt.Errorf("entry artifact %q must be oci://%s:%s", r.Artifact, repository, tag)
|
||||
|
||||
@@ -864,6 +864,28 @@ var _ = Describe("catalog template", func() {
|
||||
Expect(entries[0].Digest).NotTo(Equal(entries[1].Digest))
|
||||
Expect(entries[0].Arch).NotTo(Equal(entries[1].Arch))
|
||||
})
|
||||
It("renders the development channel as mutable entries: the moving tag, no digest", func() {
|
||||
reg := newRegistry()
|
||||
p := newProject(both)
|
||||
m := p.merged("0.0.0-dev.0123456789ab", rev, nil)
|
||||
repoName := reg.repo("addons/tables")
|
||||
out := filepath.Join(GinkgoT().TempDir(), "catalog.yaml")
|
||||
_, err := run("catalog", "--config", p.config, "--template", "../../../gallery/addons/tables.yaml.tmpl", "--merged", m, "--repository", repoName, "--tag", "sha-0123456789ab", "--channel", "development", "--owner-repo", "example/localai", "--signer-ref", "refs/heads/master", "--out", out)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
b, err := os.ReadFile(out)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
Expect(string(b)).NotTo(ContainSubstring("digest:"))
|
||||
entries, err := readCatalogIndex(b)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
Expect(entries).To(HaveLen(2))
|
||||
for _, e := range entries {
|
||||
Expect(e.Mutable).To(BeTrue())
|
||||
Expect(e.Digest).To(BeEmpty())
|
||||
Expect(e.Artifact).To(Equal("oci://" + repoName + ":tables-development"))
|
||||
Expect(e.Channel).To(Equal("development"))
|
||||
Expect(e.Policy.RequireSignature).To(BeTrue())
|
||||
}
|
||||
})
|
||||
It("renders an entry for every first-party add-on from its own template", func() {
|
||||
for _, id := range []string{"tables", "voice", "face", "fine-tuning", "quantization", "studio", "talk"} {
|
||||
p := newProjectFor(id, both)
|
||||
|
||||
Reference in new issue
Block a user