diff --git a/.github/workflows/gallery_publish.yml b/.github/workflows/gallery_publish.yml new file mode 100644 index 000000000..c27b3d282 --- /dev/null +++ b/.github/workflows/gallery_publish.yml @@ -0,0 +1,78 @@ +name: Publish official OCI galleries + +on: + push: + branches: [master] + paths: + - 'gallery/**' + - 'backend/index.yaml' + - 'scripts/build/gallery/**' + - '.github/workflows/gallery_publish.yml' + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: publish-official-galleries + cancel-in-progress: false + +jobs: + publish: + if: github.repository == 'mudler/LocalAI' && github.ref == 'refs/heads/master' + runs-on: ubuntu-latest + permissions: + contents: read + id-token: write + env: + COSIGN_EXPERIMENTAL: '1' + GALLERY_REPOSITORY: quay.io/go-skynet/local-ai-backends + strategy: + matrix: + include: + - source: gallery + tag: gallery-models + - source: backend + tag: gallery-backends + steps: + - uses: actions/checkout@v7 + - uses: actions/setup-go@v6 + with: + go-version-file: go.mod + - name: Test and package gallery + env: + GALLERY_SOURCE: ${{ matrix.source }} + run: | + go test ./scripts/build/gallery -count=1 + go run ./scripts/build/gallery . "$GALLERY_SOURCE" "$RUNNER_TEMP/gallery" + - uses: oras-project/setup-oras@v1 + with: + version: '1.3.0' + - uses: sigstore/cosign-installer@v3 + with: + cosign-release: 'v2.6.5' + - name: Login to Quay.io + uses: docker/login-action@v4 + with: + registry: quay.io + username: ${{ secrets.LOCALAI_REGISTRY_USERNAME }} + password: ${{ secrets.LOCALAI_REGISTRY_PASSWORD }} + - name: Publish and sign gallery + shell: bash + env: + GALLERY_TAG: ${{ matrix.tag }} + run: | + set -euo pipefail + cd "$RUNNER_TEMP/gallery" + files=() + while IFS= read -r -d '' file; do + files+=("${file#./}:application/yaml") + done < <(find . -type f -print0 | sort -z) + # Publish an immutable revision, then expose latest only after signing. + ref="$GALLERY_REPOSITORY:$GALLERY_TAG-$GITHUB_SHA" + oras push --artifact-type application/vnd.localai.gallery.v1 \ + --format json "$ref" "${files[@]}" > "$RUNNER_TEMP/push.json" + digest=$(jq -er '.digest' "$RUNNER_TEMP/push.json") + cosign sign --yes --new-bundle-format \ + --registry-referrers-mode=oci-1-1 "$GALLERY_REPOSITORY@$digest" + oras tag "$GALLERY_REPOSITORY@$digest" "$GALLERY_TAG" diff --git a/core/config/gallery.go b/core/config/gallery.go index e22cbc94f..3f6b31ab9 100644 --- a/core/config/gallery.go +++ b/core/config/gallery.go @@ -47,10 +47,13 @@ type Gallery struct { // fallback for availability, not a load-balancing pool: the primary is // always preferred, and a mirror is only consulted after the one before // it fails. Any URI the gallery loader understands works here - // (https://, github:, file://). + // (https://, github:, file://, oci://). Mirrors []string `json:"mirrors,omitempty" yaml:"mirrors,omitempty"` Name string `json:"name" yaml:"name"` Verification *GalleryVerification `json:"verification,omitempty" yaml:"verification,omitempty"` + // ArtifactVerification overrides Verification only for the gallery OCI artifact. + // Backend images keep their separate Verification policy. + ArtifactVerification *GalleryVerification `json:"artifact_verification,omitempty" yaml:"artifact_verification,omitempty"` } // Equal reports whether two gallery entries describe the same gallery. @@ -68,6 +71,13 @@ func (g Gallery) Equal(other Gallery) bool { if !slices.Equal(g.Mirrors, other.Mirrors) { return false } + if g.ArtifactVerification == nil || other.ArtifactVerification == nil { + if g.ArtifactVerification != other.ArtifactVerification { + return false + } + } else if *g.ArtifactVerification != *other.ArtifactVerification { + return false + } if g.Verification == nil || other.Verification == nil { return g.Verification == other.Verification } diff --git a/core/config/gallery_test.go b/core/config/gallery_test.go index 71f4f3a36..7e1848a36 100644 --- a/core/config/gallery_test.go +++ b/core/config/gallery_test.go @@ -179,3 +179,24 @@ var _ = Describe("GalleryVerification", func() { Expect(g[0].Verification.SourceRepository).To(Equal("https://github.com/acme/gallery")) }) }) + +var _ = Describe("Gallery artifact verification", func() { + It("compares artifact policies by value and preserves them in JSON and YAML", func() { + a := config.Gallery{Name: "gallery", ArtifactVerification: &config.GalleryVerification{Identity: "gallery-workflow"}} + b := config.Gallery{Name: "gallery", ArtifactVerification: &config.GalleryVerification{Identity: "gallery-workflow"}} + Expect(a.Equal(b)).To(BeTrue()) + b.ArtifactVerification.Identity = "another-workflow" + Expect(a.Equal(b)).To(BeFalse()) + b.ArtifactVerification = nil + Expect(a.Equal(b)).To(BeFalse()) + raw, err := json.Marshal(a) + Expect(err).ToNot(HaveOccurred()) + Expect(json.Unmarshal(raw, &b)).To(Succeed()) + Expect(a.Equal(b)).To(BeTrue()) + raw, err = yaml.Marshal(a) + Expect(err).ToNot(HaveOccurred()) + b = config.Gallery{} + Expect(yaml.Unmarshal(raw, &b)).To(Succeed()) + Expect(a.Equal(b)).To(BeTrue()) + }) +}) diff --git a/core/config/runtime_settings_startup.go b/core/config/runtime_settings_startup.go index 9808c877b..e5d7e2a46 100644 --- a/core/config/runtime_settings_startup.go +++ b/core/config/runtime_settings_startup.go @@ -17,8 +17,8 @@ import ( // a caching mirror of the files below. The GitHub URI stays as a mirror so an // install still resolves its gallery unchanged whenever the primary is // unreachable - see the fallback chain in core/gallery/gallery_mirrors.go. -const DefaultGalleriesJSON = `[{"name":"localai", "url":"https://index.localai.io/models", "mirrors":["github:mudler/LocalAI/gallery/index.yaml@master"]}]` -const DefaultBackendGalleriesJSON = `[{"name":"localai", "url":"https://index.localai.io/backends", "mirrors":["github:mudler/LocalAI/backend/index.yaml@master"]}]` +const DefaultGalleriesJSON = `[{"name":"localai","url":"https://index.localai.io/models","mirrors":["github:mudler/LocalAI/gallery/index.yaml@master","oci://quay.io/go-skynet/local-ai-backends:gallery-models"],"artifact_verification":{"issuer":"https://token.actions.githubusercontent.com","identity":"https://github.com/mudler/LocalAI/.github/workflows/gallery_publish.yml@refs/heads/master"}}]` +const DefaultBackendGalleriesJSON = `[{"name":"localai","url":"https://index.localai.io/backends","mirrors":["github:mudler/LocalAI/backend/index.yaml@master","oci://quay.io/go-skynet/local-ai-backends:gallery-backends"],"artifact_verification":{"issuer":"https://token.actions.githubusercontent.com","identity":"https://github.com/mudler/LocalAI/.github/workflows/gallery_publish.yml@refs/heads/master"}}]` func mustGalleries(jsonList string) []Gallery { var g []Gallery diff --git a/core/config/runtime_settings_startup_test.go b/core/config/runtime_settings_startup_test.go index 410e457d6..d49c53ac4 100644 --- a/core/config/runtime_settings_startup_test.go +++ b/core/config/runtime_settings_startup_test.go @@ -10,22 +10,22 @@ import ( ) var _ = Describe("default galleries", func() { - It("serves the model gallery from index.localai.io with GitHub as a mirror", func() { + It("serves the model gallery from index.localai.io with GitHub then OCI as mirrors", func() { var galleries []config.Gallery Expect(json.Unmarshal([]byte(config.DefaultGalleriesJSON), &galleries)).To(Succeed()) Expect(galleries).To(HaveLen(1)) Expect(galleries[0].Name).To(Equal("localai")) Expect(galleries[0].URL).To(Equal("https://index.localai.io/models")) - Expect(galleries[0].Mirrors).To(Equal([]string{"github:mudler/LocalAI/gallery/index.yaml@master"})) + Expect(galleries[0].Mirrors).To(Equal([]string{"github:mudler/LocalAI/gallery/index.yaml@master", "oci://quay.io/go-skynet/local-ai-backends:gallery-models"})) }) - It("serves the backend gallery from index.localai.io with GitHub as a mirror", func() { + It("serves the backend gallery from index.localai.io with GitHub then OCI as mirrors", func() { var galleries []config.Gallery Expect(json.Unmarshal([]byte(config.DefaultBackendGalleriesJSON), &galleries)).To(Succeed()) Expect(galleries).To(HaveLen(1)) Expect(galleries[0].Name).To(Equal("localai")) Expect(galleries[0].URL).To(Equal("https://index.localai.io/backends")) - Expect(galleries[0].Mirrors).To(Equal([]string{"github:mudler/LocalAI/backend/index.yaml@master"})) + Expect(galleries[0].Mirrors).To(Equal([]string{"github:mudler/LocalAI/backend/index.yaml@master", "oci://quay.io/go-skynet/local-ai-backends:gallery-backends"})) }) // The mirror is the whole reason this default is safe to ship: if @@ -37,6 +37,10 @@ var _ = Describe("default galleries", func() { Expect(json.Unmarshal([]byte(raw), &galleries)).To(Succeed()) for _, g := range galleries { Expect(g.Mirrors).ToNot(BeEmpty(), "default %q has no mirror", g.Name) + Expect(g.ArtifactVerification).ToNot(BeNil()) + Expect(g.ArtifactVerification.Identity).To(Equal("https://github.com/mudler/LocalAI/.github/workflows/gallery_publish.yml@refs/heads/master")) + Expect(g.ArtifactVerification.Issuer).To(Equal("https://token.actions.githubusercontent.com")) + Expect(g.Verification).To(BeNil(), "gallery policy must not change backend image trust") } } }) diff --git a/core/gallery/entry_url.go b/core/gallery/entry_url.go index 184ce2dd9..49f4f720d 100644 --- a/core/gallery/entry_url.go +++ b/core/gallery/entry_url.go @@ -42,7 +42,7 @@ func ociGalleryRoot(g config.Gallery, basePath string) string { if !looksLikeOCIGallery(candidate) { continue } - dir := ociGalleryCacheDir(basePath, candidate, g.Verification) + dir := ociGalleryCacheDir(basePath, candidate, galleryArtifactPolicy(g)) if dir == "" { continue } diff --git a/core/gallery/gallery.go b/core/gallery/gallery.go index bd054b0ff..c7ebd7bcc 100644 --- a/core/gallery/gallery.go +++ b/core/gallery/gallery.go @@ -634,7 +634,7 @@ var galleryCache = xsync.NewSyncedMap[string, galleryCacheEntry]() // would also point relative entry urls at an unpacked tree the new policy has // not produced yet, so they could not be installed. func galleryIndexCacheKey(g config.Gallery) string { - return g.Name + "-" + galleryCacheName(g.URL, g.Verification) + return g.Name + "-" + galleryCacheName(g.URL, galleryArtifactPolicy(g)) } func getGalleryElements[T GalleryElement](gallery config.Gallery, basePath string, requireIntegrity bool, isInstalledCallback func(T) bool) ([]T, error) { diff --git a/core/gallery/gallery_mirrors.go b/core/gallery/gallery_mirrors.go index 4058c0a45..083512417 100644 --- a/core/gallery/gallery_mirrors.go +++ b/core/gallery/gallery_mirrors.go @@ -144,7 +144,7 @@ func indexCachePolicy(g config.Gallery) *config.GalleryVerification { if !looksLikeOCIGallery(g.URL) { return nil } - return g.Verification + return galleryArtifactPolicy(g) } // verifiableCandidates drops the candidates that cannot answer for a signed @@ -156,7 +156,7 @@ func indexCachePolicy(g config.Gallery) *config.GalleryVerification { // at, and after a refusal it would turn "this artifact is not trusted" into // "use this other, unchecked copy instead". func verifiableCandidates(g config.Gallery, candidates []string, requireIntegrity bool) []string { - if !looksLikeOCIGallery(g.URL) || (g.Verification == nil && !requireIntegrity) { + if !looksLikeOCIGallery(g.URL) || (galleryArtifactPolicy(g) == nil && !requireIntegrity) { return candidates } out := make([]string, 0, len(candidates)) diff --git a/core/gallery/gallery_oci.go b/core/gallery/gallery_oci.go index 19d6d16c0..cbd649229 100644 --- a/core/gallery/gallery_oci.go +++ b/core/gallery/gallery_oci.go @@ -151,17 +151,18 @@ func readCachedOCIGallery(cacheDir string) ([]byte, bool) { // later fetch served would hand the user a truncated gallery with no sign that // anything went wrong. func fetchOCIGalleryIndex(ctx context.Context, g config.Gallery, candidate, basePath string, requireIntegrity bool) ([]byte, error) { + policy := galleryArtifactPolicy(g) // Checked before the cache: a copy unpacked while strict integrity was // off was never verified, and turning strict integrity on must not keep // serving it for the rest of its TTL. - if g.Verification == nil && requireIntegrity { + if policy == nil && requireIntegrity { return nil, &galleryVerificationError{ strict: true, - err: fmt.Errorf("no verification policy is set for %q (set verification: in the gallery configuration or disable --require-backend-integrity)", candidate), + err: fmt.Errorf("no verification policy is set for %q (set artifact_verification: in the gallery configuration or disable --require-backend-integrity)", candidate), } } - cacheDir := ociGalleryCacheDir(basePath, candidate, g.Verification) + cacheDir := ociGalleryCacheDir(basePath, candidate, policy) if cacheDir == "" { return nil, fmt.Errorf("gallery %q needs an absolute models directory to cache %q", g.Name, candidate) } @@ -171,7 +172,7 @@ func fetchOCIGalleryIndex(ctx context.Context, g config.Gallery, candidate, base pullRef := downloader.URI(candidate).OCIReference() - if g.Verification != nil { + if policy != nil { // Resolve first, verify the digest, then pull that same digest. // Nothing has been fetched at this point beyond the manifest, so a // policy failure leaves no content anywhere. @@ -179,7 +180,7 @@ func fetchOCIGalleryIndex(ctx context.Context, g config.Gallery, candidate, base if err != nil { return nil, err } - if err := verifyGalleryArtifact(ctx, g.Verification, digestRef); err != nil { + if err := verifyGalleryArtifact(ctx, policy, digestRef); err != nil { // Only a decision about the artifact is a refusal. The // verifier also reaches the Sigstore TUF mirror and the // registry, and a timeout or a 5xx there says nothing about @@ -239,3 +240,10 @@ func fetchOCIGalleryIndex(ctx context.Context, g config.Gallery, candidate, base return body, nil } + +func galleryArtifactPolicy(g config.Gallery) *config.GalleryVerification { + if g.ArtifactVerification != nil { + return g.ArtifactVerification + } + return g.Verification +} diff --git a/core/gallery/gallery_oci_test.go b/core/gallery/gallery_oci_test.go index 5672da123..4c038b5a2 100644 --- a/core/gallery/gallery_oci_test.go +++ b/core/gallery/gallery_oci_test.go @@ -229,6 +229,21 @@ var _ = Describe("oci:// galleries", func() { }) }) + It("uses the artifact policy without replacing backend image verification", func() { + srv, _, _ := ociRegistry() + url := pushGalleryArtifact(srv.URL, "galleries/separate-policy", galleryArtifactType, []ociGalleryFile{{title: "index.yaml", body: "- name: demo\n"}}) + backendPolicy := &config.GalleryVerification{Identity: "backend-workflow"} + artifactPolicy := &config.GalleryVerification{Identity: "gallery-workflow"} + var seen *config.GalleryVerification + stubGalleryVerifier(func(_ context.Context, policy *config.GalleryVerification, _ string) error { seen = policy; return nil }) + g := config.Gallery{URL: srv.URL + "/unavailable", Mirrors: []string{srv.URL + "/also-unavailable", url}, Name: "separate", Verification: backendPolicy, ArtifactVerification: artifactPolicy} + _, source, err := fetchGalleryIndex(context.Background(), g, tempModelsDir(), true) + Expect(source).To(Equal(url)) + Expect(err).ToNot(HaveOccurred()) + Expect(seen).To(Equal(artifactPolicy)) + Expect(g.Verification).To(Equal(backendPolicy)) + }) + It("refuses an unsigned gallery in strict integrity mode", func() { srv, _, blobs := ociRegistry() url := pushGalleryArtifact(srv.URL, "galleries/strict", galleryArtifactType, []ociGalleryFile{ diff --git a/docs/content/features/backends.md b/docs/content/features/backends.md index 69ae0cf0f..c3fd369c5 100644 --- a/docs/content/features/backends.md +++ b/docs/content/features/backends.md @@ -82,6 +82,8 @@ tags: ### Verifying OCI Backends +The default backend gallery tries `https://index.localai.io/backends`, then `github:mudler/LocalAI/backend/index.yaml@master`, then `oci://quay.io/go-skynet/local-ai-backends:gallery-backends`. The OCI fallback is signed by `gallery_publish.yml`. Its `artifact_verification` policy applies only to the gallery artifact; `verification` continues to control backend image signatures. Existing custom gallery lists are not changed. See [gallery publishing]({{% relref "features/model-gallery#official-gallery-publishing" %}}) for details. + Backend galleries can require keyless Sigstore signatures for every OCI image they provide. Add a `verification` policy to the gallery configuration, then enable strict integrity mode: diff --git a/docs/content/features/model-gallery.md b/docs/content/features/model-gallery.md index 770428d57..bddadfe14 100644 --- a/docs/content/features/model-gallery.md +++ b/docs/content/features/model-gallery.md @@ -74,7 +74,7 @@ To use a gallery that needs authentication, such as a private GitHub repository A gallery entry can declare a `mirrors` list of alternative locations for the same index file. Mirrors exist for availability, not for load balancing: LocalAI always prefers the `url`, and only falls back to the mirrors, in the order you listed them, when the one before it cannot be fetched. If the primary works, the mirrors are never contacted. -Mirrors accept any URI the gallery loader understands — `https://`, `github:`, `huggingface://` (also `hf://` and `hf.co/`), and `file://` — and the same rules apply to them as to a primary URL, so a `file://` mirror must still live inside your models directory. +Mirrors accept any URI the gallery loader understands — `https://`, `github:`, `huggingface://` (also `hf://` and `hf.co/`), `file://`, and `oci://` — and the same rules apply to them as to a primary URL, so a `file://` mirror must still live inside your models directory. ```json GALLERIES=[{"name":"localai", "url":"https://example.org/gallery/index.yaml", "mirrors":["github:mudler/LocalAI/gallery/index.yaml@master"]}] @@ -128,10 +128,10 @@ A relative `url` cannot leave the gallery root. An entry that tries to climb out ### Signature verification -An `oci://` gallery can be signed, and LocalAI verifies the signature before it unpacks anything. Add a `verification` block with the Fulcio issuer and the signing identity, in the same form the [backend galleries]({{%relref "features/backends#verifying-oci-backends" %}}) use: +An `oci://` gallery can be signed, and LocalAI verifies the signature before it unpacks anything. Add an `artifact_verification` block with the Fulcio issuer and the signing identity, in the same form the [backend galleries]({{%relref "features/backends#verifying-oci-backends" %}}) use: ```json -GALLERIES=[{"name":"premium","url":"oci://quay.io/acme/gallery:latest","verification":{"issuer":"https://token.actions.githubusercontent.com","identity_regex":"^https://github\\.com/acme/gallery/\\.github/workflows/publish\\.yml@refs/tags/.+$"}}] +GALLERIES=[{"name":"premium","url":"oci://quay.io/acme/gallery:latest","artifact_verification":{"issuer":"https://token.actions.githubusercontent.com","identity_regex":"^https://github\\.com/acme/gallery/\\.github/workflows/publish\\.yml@refs/tags/.+$"}}] ``` The tag is resolved to a digest, the signature is checked against that digest, and the same digest is then pulled. A gallery that fails verification is never written to the cache, so no unverified file reaches your disk. The optional `not_before` RFC3339 value revokes signatures logged before that time, exactly as it does for backends. @@ -150,9 +150,17 @@ With strict integrity on (`--require-backend-integrity` or `LOCALAI_REQUIRE_BACK The optional `source_repository` value works the same for `oci://` galleries as it does for backends: it pins the repository the signature was made for when a shared reusable workflow does the signing. See [Verifying OCI Backends]({{%relref "features/backends#verifying-oci-backends" %}}). {{% notice warning %}} -With `--require-backend-integrity` (`LOCALAI_REQUIRE_BACKEND_INTEGRITY=1`), an `oci://` gallery that has no `verification` block is refused when the models are listed, not only when one is installed. Add a `verification` block to every `oci://` gallery before you turn strict integrity on, or the galleries without one stop listing. An `oci://` gallery without a policy still lists outside strict mode, with a warning in the log. +`artifact_verification` applies only to the gallery artifact. Backend image signatures use `verification`. For compatibility, the artifact loader uses `verification` when `artifact_verification` is absent. Set both fields when the gallery and its backend images have different signing identities. + +With `--require-backend-integrity` (`LOCALAI_REQUIRE_BACKEND_INTEGRITY=1`), an `oci://` gallery with neither policy is refused when the models are listed, not only when one is installed. An `oci://` gallery without a policy still lists outside strict mode, with a warning in the log. {{% /notice %}} +### Official gallery publishing + +The `gallery_publish.yml` workflow publishes both official galleries on relevant changes to `master`, or through a manual dispatch on `master`. It uses the existing `LOCALAI_REGISTRY_USERNAME` and `LOCALAI_REGISTRY_PASSWORD` secrets. It reuses the public backend repository `go-skynet/local-ai-backends`. The `gallery-models` and `gallery-backends` tags move only after their artifact digest has been signed. Revision tags include the source commit SHA. + +To prepare the same files locally, run `go run ./scripts/build/gallery . gallery /tmp/model-gallery` or use `backend` as the source directory. The helper rewrites repository-local base configuration URLs to artifact-relative paths and copies the files. The published artifact type is `application/vnd.localai.gallery.v1`; each file is a separate layer with its relative path as its title. + ### Private registries A gallery in a private registry needs a credentials entry that matches the registry, the same entry an image pull from it would use: @@ -184,10 +192,10 @@ GALLERIES=[{"name":"", "url":"