From 6bfe7f8c05289a35834881d68e90fa06176b9bc0 Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Wed, 13 May 2026 21:36:28 +0000 Subject: [PATCH] ci(image-merge): apply the keepalive+ci-cache source fix to image_merge MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Mirror of 8521af14 (which fixed backend_merge.yml) for image_merge.yml. Today's master-push run 25823024353 failed the gpu-vulkan-image-merge job with the exact same error pattern the backend merge had on v4.2.2: ERROR: quay.io/go-skynet/local-ai@sha256:68b22611...: not found Same root cause: image_build.yml pushes the per-arch manifest to quay.io/go-skynet/local-ai with push-by-digest=true (no tag), then the merge runs minutes-to-hours later, by which time quay's per-repo manifest GC has reaped the untagged digest from local-ai. The blob still lives in quay's storage but local-ai@ no longer resolves. Three matching edits: 1. image_build.yml: anchor each per-arch digest into ci-cache immediately after the push, reusing .github/scripts/anchor-digest-in-cache.sh with SOURCE_IMAGE=quay.io/go-skynet/local-ai and TAG_SUFFIX defaulting to "-core" for the core image (matches the artifact-name convention). 2. image_merge.yml: change the quay merge source from local-ai@ to ci-cache@. Same correctness argument as backend_merge.yml — the manifest content is alive in ci-cache; buildx imagetools create republishes it into local-ai and writes the user-facing manifest list pointing at it. End state in local-ai is self-contained. 3. image_merge.yml: add a sparse `actions/checkout@v6` (only .github/scripts) so cleanup-keepalive-tags.sh is available, plus the cleanup step itself with TAG_SUFFIX matching the anchor's "-core" placeholder. v4.2.3's image.yml run completed successfully (~50 min between push and merge — beat quay's GC). This commit closes the race for future releases and master pushes regardless of run length. Assisted-by: Claude:claude-opus-4-7 Signed-off-by: Ettore Di Giacinto --- .github/workflows/image_build.yml | 13 +++++++++++++ .github/workflows/image_merge.yml | 27 ++++++++++++++++++++++++++- 2 files changed, 39 insertions(+), 1 deletion(-) diff --git a/.github/workflows/image_build.yml b/.github/workflows/image_build.yml index 18a18f525..9b9683b32 100644 --- a/.github/workflows/image_build.yml +++ b/.github/workflows/image_build.yml @@ -185,6 +185,19 @@ jobs: digest="${{ steps.build.outputs.digest }}" touch "/tmp/digests/${digest#sha256:}" + # See .github/scripts/anchor-digest-in-cache.sh for why this is needed + # and how it interacts with image_merge.yml's cleanup step. Mirrors the + # same anchor in backend_build.yml — quay's per-repo manifest GC reaps + # untagged manifests in local-ai before the merge runs. + - name: Anchor digest in ci-cache so quay GC won't reap before merge + if: github.event_name != 'pull_request' + env: + TAG_SUFFIX: ${{ inputs.tag-suffix == '' && '-core' || inputs.tag-suffix }} + PLATFORM_TAG: ${{ inputs.platform-tag || 'single' }} + DIGEST: ${{ steps.build.outputs.digest }} + SOURCE_IMAGE: quay.io/go-skynet/local-ai + run: .github/scripts/anchor-digest-in-cache.sh + - name: Upload digest artifact if: github.event_name != 'pull_request' uses: actions/upload-artifact@v7 diff --git a/.github/workflows/image_merge.yml b/.github/workflows/image_merge.yml index 958910519..85315fe58 100644 --- a/.github/workflows/image_merge.yml +++ b/.github/workflows/image_merge.yml @@ -33,6 +33,15 @@ jobs: env: quay_username: ${{ secrets.quayUsername }} steps: + # Sparse checkout: needed for .github/scripts/ (the keepalive cleanup + # script). Skips the rest of the source tree. + - name: Checkout (.github/scripts only) + uses: actions/checkout@v6 + with: + sparse-checkout: | + .github/scripts + sparse-checkout-cone-mode: false + - name: Download digests uses: actions/download-artifact@v8 with: @@ -72,6 +81,13 @@ jobs: latest=${{ inputs.tag-latest }} suffix=${{ inputs.tag-suffix }},onlatest=true + # Source from ci-cache, not local-ai. See backend_merge.yml for the + # detailed rationale — quay's manifest GC is per-repository, so the + # untagged digest in local-ai gets reaped while the same content lives + # tagged under ci-cache (anchored by image_build.yml). buildx imagetools + # create copies the manifest into local-ai (blobs already cross-mounted) + # and publishes the manifest list with user-facing tags. End state in + # local-ai is self-contained; no embedded reference to ci-cache. - name: Create manifest list and push (quay) working-directory: /tmp/digests run: | @@ -82,7 +98,7 @@ jobs: else # shellcheck disable=SC2086 docker buildx imagetools create $tags \ - $(printf 'quay.io/go-skynet/local-ai@sha256:%s ' *) + $(printf 'quay.io/go-skynet/ci-cache@sha256:%s ' *) fi - name: Create manifest list and push (dockerhub) @@ -107,6 +123,15 @@ jobs: docker buildx imagetools inspect "$first_tag" fi + # See .github/scripts/cleanup-keepalive-tags.sh for the best-effort + # semantics — fails soft when the registry credential isn't OAuth-scoped. + - name: Cleanup keepalive tags in ci-cache + if: github.event_name != 'pull_request' && success() + env: + TAG_SUFFIX: ${{ inputs.tag-suffix == '' && '-core' || inputs.tag-suffix }} + QUAY_TOKEN: ${{ secrets.quayPassword }} + run: .github/scripts/cleanup-keepalive-tags.sh + - name: Job summary run: | set -euo pipefail