ci: tag every backend digest, including singletons

backend_build.yml pushes by canonical digest only (push-by-digest=true,
no tags applied at build time). User-facing tagging happens in
backend_merge.yml's `imagetools create` step. Before this commit,
scripts/changed-backends.js emitted a merge entry only for tag-suffixes
with 2+ legs, so every single-arch backend (CUDA/ROCm/Intel Python
images, vLLM, sglang, transformers, diffusers, ...) pushed its digest
untagged and stayed that way until quay's GC reaped it. Symptom: tag
releases shipped multi-arch backends tagged correctly, but no
v<X>-gpu-nvidia-cuda-12-vllm (or any singleton variant) ever appeared
in the registry.

Changes:

- scripts/changed-backends.js drops the `group.length < 2` skip and
  emits two merge matrices, one per arch class, so each downstream
  merge job can `needs:` only its corresponding build matrix.
- backend.yml splits backend-merge-jobs into multiarch and singlearch
  variants. The split preserves PR #9746's fix: slow singlearch CUDA
  builds (~6h) must not gate multiarch merges, or quay's GC reaps the
  multiarch per-arch digests before they're tagged.
- backend_pr.yml mirrors the split.
- backend_build.yml renames the digest artifact from
  `digests<suffix>-<platform-tag>` to
  `digests<suffix>--<platform-tag-or-"single">`. The `--` separator
  prevents the merge-side glob from over-matching sibling backends
  whose tag-suffix is a prefix of ours (e.g. -cpu-vllm vs
  -cpu-vllm-omni, -cpu-mlx vs -cpu-mlx-audio); the `single` placeholder
  keeps the name well-formed when platform-tag is empty.
- backend_merge.yml updates the download pattern to match.

Verified locally: a tag-push event now expands to 36 multiarch merge
entries (= 72 builds / 2 legs) and 199 singlearch merge entries (one
per singleton, including -gpu-nvidia-cuda-12-vllm at index 24).

Assisted-by: Claude:claude-opus-4-7
Signed-off-by: Ettore Di Giacinto <mudler@localai.io>

.github/workflows/backend.yml

@@ -35,11 +35,13 @@ jobs:
       matrix-singlearch: ${{ steps.set-matrix.outputs['matrix-singlearch'] }}
       matrix-multiarch: ${{ steps.set-matrix.outputs['matrix-multiarch'] }}
       matrix-darwin: ${{ steps.set-matrix.outputs['matrix-darwin'] }}
-      merge-matrix: ${{ steps.set-matrix.outputs['merge-matrix'] }}
+      merge-matrix-multiarch: ${{ steps.set-matrix.outputs['merge-matrix-multiarch'] }}
+      merge-matrix-singlearch: ${{ steps.set-matrix.outputs['merge-matrix-singlearch'] }}
       has-backends-singlearch: ${{ steps.set-matrix.outputs['has-backends-singlearch'] }}
       has-backends-multiarch: ${{ steps.set-matrix.outputs['has-backends-multiarch'] }}
       has-backends-darwin: ${{ steps.set-matrix.outputs['has-backends-darwin'] }}
-      has-merges: ${{ steps.set-matrix.outputs['has-merges'] }}
+      has-merges-multiarch: ${{ steps.set-matrix.outputs['has-merges-multiarch'] }}
+      has-merges-singlearch: ${{ steps.set-matrix.outputs['has-merges-singlearch'] }}
     steps:
       - name: Checkout repository
         uses: actions/checkout@v6
@@ -138,15 +140,21 @@ jobs:
       max-parallel: 8
       matrix: ${{ fromJson(needs.generate-matrix.outputs['matrix-singlearch']) }}
 
-  # Merge per-arch digests into manifest lists. Depends ONLY on
-  # backend-jobs-multiarch — single-arch builds are independent and slow.
-  # Without this split, a 6h CUDA-12 single-arch job would gate the merge,
-  # leaving multi-arch digests untagged on quay long enough for quay's
-  # garbage collector to reap them and the merge step to fail with
-  # "manifest not found".
-  backend-merge-jobs:
+  # Apply tags to per-arch digests via `imagetools create`. Split into two
+  # jobs that mirror the build split so each merge waits ONLY on its
+  # corresponding build matrix:
+  #
+  #   - backend-merge-jobs-multiarch  needs backend-jobs-multiarch  (~2-3h)
+  #   - backend-merge-jobs-singlearch needs backend-jobs-singlearch (up to ~6h)
+  #
+  # If a single shared merge job depended on both, slow CUDA singlearch
+  # builds would block multiarch merges long enough for quay's GC to reap
+  # the multiarch per-arch digests (the bug fixed by PR #9746). Singletons
+  # also need a merge step because backend_build.yml pushes by canonical
+  # digest only — no tags are applied at build time.
+  backend-merge-jobs-multiarch:
     needs: [generate-matrix, backend-jobs-multiarch]
-    if: needs.generate-matrix.outputs['has-merges'] == 'true'
+    if: needs.generate-matrix.outputs['has-merges-multiarch'] == 'true'
     uses: ./.github/workflows/backend_merge.yml
     with:
       tag-latest: ${{ matrix.tag-latest }}
@@ -158,7 +166,23 @@ jobs:
       quayPassword: ${{ secrets.LOCALAI_REGISTRY_PASSWORD }}
     strategy:
       fail-fast: false
-      matrix: ${{ fromJson(needs.generate-matrix.outputs['merge-matrix']) }}
+      matrix: ${{ fromJson(needs.generate-matrix.outputs['merge-matrix-multiarch']) }}
+
+  backend-merge-jobs-singlearch:
+    needs: [generate-matrix, backend-jobs-singlearch]
+    if: needs.generate-matrix.outputs['has-merges-singlearch'] == 'true'
+    uses: ./.github/workflows/backend_merge.yml
+    with:
+      tag-latest: ${{ matrix.tag-latest }}
+      tag-suffix: ${{ matrix.tag-suffix }}
+    secrets:
+      dockerUsername: ${{ secrets.DOCKERHUB_USERNAME }}
+      dockerPassword: ${{ secrets.DOCKERHUB_PASSWORD }}
+      quayUsername: ${{ secrets.LOCALAI_REGISTRY_USERNAME }}
+      quayPassword: ${{ secrets.LOCALAI_REGISTRY_PASSWORD }}
+    strategy:
+      fail-fast: false
+      matrix: ${{ fromJson(needs.generate-matrix.outputs['merge-matrix-singlearch']) }}
 
   backend-jobs-darwin:
     needs: generate-matrix