From 96392f3af01be5fe4f00cb3aeb8050906d006ab1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Deluan=20Quint=C3=A3o?= <deluan@navidrome.org>
Date: Wed, 3 Dec 2025 18:24:11 -0500
Subject: [PATCH] ci: improve docker manifest push reliability and isolation
 (#4764)

* ci: improve docker manifest push reliability and isolation

Split Docker manifest push into separate GHCR and Docker Hub jobs to improve pipeline reliability and resilience:

- Separated push-manifest job into push-manifest-ghcr and push-manifest-dockerhub for independent execution
- Filter tags per registry using jq to prevent cross-registry push attempts
- Add automatic retry logic (3 attempts with 30s delay) for Docker Hub push using nick-fields/retry action
- Make Docker Hub job continue-on-error to prevent Docker Hub intermittent failures from failing the entire pipeline
- Add dedicated cleanup-digests job that only requires GHCR job success
- GHCR is now the critical path and will fail the pipeline if it fails, while Docker Hub failures are tolerated with retries

This addresses the recurring 400 Bad Request errors from Docker Hub registry that were causing pipeline failures even when ghcr.io push succeeded.

* fix(ci): use ghcr.io as source for docker hub manifest creation

The docker buildx imagetools create command needs to reference the source images from where they exist (ghcr.io) rather than from Docker Hub. The digests uploaded during the build step are stored on ghcr.io, so we need to pull from there and tag to Docker Hub.

* fix(ci): simplify Docker manifest push job names for clarity

* fix(ci): add permissions for Docker manifest push jobs

* fix(ci): update permissions for GHCR manifest push to write

* fix(ci): update Docker Hub image tagging in manifest creation

* fix(ci): update permissions for GHCR manifest push to read contents and write packages

* Revert "fix(ci): update Docker Hub image tagging in manifest creation"

This reverts commit b5f04d9c8b40a9f7d9c5952e5ca57c42dadfc20a.
---
 .github/workflows/pipeline.yml | 64 ++++++++++++++++++++++++++--------
 1 file changed, 50 insertions(+), 14 deletions(-)

diff --git a/.github/workflows/pipeline.yml b/.github/workflows/pipeline.yml
index 3352cfa4b..43c39f4df 100644
--- a/.github/workflows/pipeline.yml
+++ b/.github/workflows/pipeline.yml
@@ -256,8 +256,11 @@ jobs:
           if-no-files-found: error
           retention-days: 1
 
-  push-manifest:
-    name: Push Docker manifest
+  push-manifest-ghcr:
+    name: Push to GHCR
+    permissions:
+      contents: read
+      packages: write
     runs-on: ubuntu-latest
     needs: [build, check-push-enabled]
     if: needs.check-push-enabled.outputs.is_enabled == 'true'
@@ -278,32 +281,65 @@ jobs:
         id: docker
         with:
           github_token: ${{ secrets.GITHUB_TOKEN }}
-          hub_repository: ${{ vars.DOCKER_HUB_REPO }}
-          hub_username: ${{ secrets.DOCKER_HUB_USERNAME }}
-          hub_password: ${{ secrets.DOCKER_HUB_PASSWORD }}
 
       - name: Create manifest list and push to ghcr.io
         working-directory: /tmp/digests
         run: |
-          docker buildx imagetools create $(jq -cr '.tags | map("-t " + .) | join(" ")' <<< "$DOCKER_METADATA_OUTPUT_JSON") \
+          docker buildx imagetools create $(jq -cr '.tags | map(select(startswith("ghcr.io"))) | map("-t " + .) | join(" ")' <<< "$DOCKER_METADATA_OUTPUT_JSON") \
             $(printf '${{ env.REGISTRY_IMAGE }}@sha256:%s ' *)          
 
-      - name: Create manifest list and push to Docker Hub
-        working-directory: /tmp/digests
-        if: vars.DOCKER_HUB_REPO != ''
-        run: |
-          docker buildx imagetools create $(jq -cr '.tags | map("-t " + .) | join(" ")' <<< "$DOCKER_METADATA_OUTPUT_JSON") \
-            $(printf '${{ vars.DOCKER_HUB_REPO }}@sha256:%s ' *)          
-
       - name: Inspect image in ghcr.io
         run: |
           docker buildx imagetools inspect ${{ env.REGISTRY_IMAGE }}:${{ steps.docker.outputs.version }}
 
+  push-manifest-dockerhub:
+    name: Push to Docker Hub
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+    needs: [build, check-push-enabled]
+    if: needs.check-push-enabled.outputs.is_enabled == 'true' && vars.DOCKER_HUB_REPO != ''
+    continue-on-error: true
+    steps:
+      - uses: actions/checkout@v6
+
+      - name: Download digests
+        uses: actions/download-artifact@v6
+        with:
+          path: /tmp/digests
+          pattern: digests-*
+          merge-multiple: true
+
+      - name: Prepare Docker Buildx
+        uses: ./.github/actions/prepare-docker
+        id: docker
+        with:
+          github_token: ${{ secrets.GITHUB_TOKEN }}
+          hub_repository: ${{ vars.DOCKER_HUB_REPO }}
+          hub_username: ${{ secrets.DOCKER_HUB_USERNAME }}
+          hub_password: ${{ secrets.DOCKER_HUB_PASSWORD }}
+
+      - name: Create manifest list and push to Docker Hub
+        uses: nick-fields/retry@v3
+        with:
+          timeout_minutes: 5
+          max_attempts: 3
+          retry_wait_seconds: 30
+          command: |
+            cd /tmp/digests
+            docker buildx imagetools create $(jq -cr '.tags | map(select(startswith("ghcr.io") | not)) | map("-t " + .) | join(" ")' <<< "$DOCKER_METADATA_OUTPUT_JSON") \
+              $(printf 'ghcr.io/${{ github.repository }}@sha256:%s ' *)
+
       - name: Inspect image in Docker Hub
-        if: vars.DOCKER_HUB_REPO != ''
         run: |
           docker buildx imagetools inspect ${{ vars.DOCKER_HUB_REPO }}:${{ steps.docker.outputs.version }}
 
+  cleanup-digests:
+    name: Cleanup digest artifacts
+    runs-on: ubuntu-latest
+    needs: [push-manifest-ghcr, push-manifest-dockerhub]
+    if: always() && needs.push-manifest-ghcr.result == 'success'
+    steps:
       - name: Delete unnecessary digest artifacts
         env:
           GH_TOKEN: ${{ github.token }}