From c48a230c23d32a6a245df36f67b12cba91c17620 Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Sun, 20 Sep 2026 21:38:09 +0000 Subject: [PATCH] fix(ci): sign backends in the format we verify Published backend images carry legacy simplesigning signatures: config application/vnd.dev.cosign.artifact.sig.v1+json, layer application/vnd.dev.cosign.simplesigning.v1+json, attached as referrers with no artifactType. pkg/oci/cosignverify accepts only new-format Sigstore bundles and refuses the legacy format on purpose, so LOCALAI_REQUIRE_BACKEND_INTEGRITY and any gallery verification: block reject our own backends. The cause is one missing flag. backend-signing.md has prescribed `--new-bundle-format` all along; backend_merge.yml never passed it, and the comment beside the cosign pin claims v2.4.1 emits the current bundle format by default, which it does not. The flag arrives in v2.5.0, so the pin moves to v2.6.5 and both sign calls pass it. Verified against a published image today (quay.io/go-skynet/local-ai-backends:latest-gpu-nvidia-cuda-12-llama-cpp): both referrers are simplesigning, so nothing verifies. Signed-off-by: Ettore Di Giacinto Assisted-by: Claude:claude-opus-5 [Claude Code] --- .agents/backend-signing.md | 14 ++++++++++---- .github/workflows/backend_merge.yml | 13 ++++++++----- 2 files changed, 18 insertions(+), 9 deletions(-) diff --git a/.agents/backend-signing.md b/.agents/backend-signing.md index 5edec5891..8eaa0ff5a 100644 --- a/.agents/backend-signing.md +++ b/.agents/backend-signing.md @@ -49,12 +49,18 @@ cosign sign --yes --recursive \ Sign by digest, never by tag — signing by tag binds the signature to whatever the tag points at *now*, and a subsequent tag push orphans it. -`--registry-referrers-mode=oci-1-1` is still gated behind -`COSIGN_EXPERIMENTAL=1` in cosign v2.4.x (set at the job env level in -`backend_merge.yml`). Re-evaluate when bumping the pinned cosign release -— newer versions are expected to graduate this flag and the env var can +`--registry-referrers-mode=oci-1-1` is gated behind +`COSIGN_EXPERIMENTAL=1` (set at the job env level in +`backend_merge.yml`). Re-evaluate when bumping the pinned cosign release: +newer versions are expected to graduate this flag and the env var can then be dropped. +`--new-bundle-format` needs cosign v2.5.0 or newer, which is why +`backend_merge.yml` pins v2.6.5. Without the flag cosign writes the legacy +simplesigning format instead, and `pkg/oci/cosignverify` refuses it on +purpose, so an older cosign silently publishes signatures this project +cannot verify. Check both the flag and the pinned version together. + `backend_build_darwin.yml` builds and pushes single-arch darwin images that bypass the manifest-list merge. If/when those entries get a gallery `verification:` policy, the equivalent cosign step has to land there diff --git a/.github/workflows/backend_merge.yml b/.github/workflows/backend_merge.yml index ccc073d10..9aa996011 100644 --- a/.github/workflows/backend_merge.yml +++ b/.github/workflows/backend_merge.yml @@ -40,8 +40,8 @@ jobs: id-token: write env: quay_username: ${{ secrets.quayUsername }} - # cosign v2.4.x still gates --registry-referrers-mode=oci-1-1 behind - # this flag. Without it, signing fails with: + # cosign gates --registry-referrers-mode=oci-1-1 behind this flag. + # Without it, signing fails with: # invalid argument "oci-1-1" for "--registry-referrers-mode" flag: # in order to use mode "oci-1-1", you must set COSIGN_EXPERIMENTAL=1 COSIGN_EXPERIMENTAL: '1' @@ -71,13 +71,14 @@ jobs: # cosign signs each pushed manifest list with --recursive so the # index and every per-arch entry get an attached Sigstore bundle. - # Cosign v2.4.1 emits the current bundle format by default; the - # verifier discovers those bundles through OCI 1.1 referrers. + # The version floor is what makes --new-bundle-format available: it + # arrives in cosign v2.5.0, and without it cosign writes the legacy + # simplesigning format, which our own verifier refuses. - name: Install cosign if: github.event_name != 'pull_request' uses: sigstore/cosign-installer@v3 with: - cosign-release: 'v2.4.1' + cosign-release: 'v2.6.5' - name: Login to DockerHub if: github.event_name != 'pull_request' @@ -159,6 +160,7 @@ jobs: # manifest before checking signatures need the per-arch # signatures, not just the list-level one. cosign sign --yes --recursive \ + --new-bundle-format \ --registry-referrers-mode=oci-1-1 \ "quay.io/go-skynet/local-ai-backends@${digest}" @@ -185,6 +187,7 @@ jobs: ' <<< "$DOCKER_METADATA_OUTPUT_JSON") digest=$(docker buildx imagetools inspect "$first_tag" --format '{{.Manifest.Digest}}') cosign sign --yes --recursive \ + --new-bundle-format \ --registry-referrers-mode=oci-1-1 \ "localai/localai-backends@${digest}"