diff --git a/.github/workflows/docker_build.yml b/.github/workflows/docker_build.yml
index 501b175234..8c16e75aad 100644
--- a/.github/workflows/docker_build.yml
+++ b/.github/workflows/docker_build.yml
@@ -45,6 +45,10 @@ jobs:
     outputs:
       digest: ${{ steps.docker_variant.outputs.digest }}
     runs-on: ${{ inputs.runs-on }}
+    env:
+      # Only cache on default branch (develop).
+      # 'docker/' actions don't support separate Restore/Save actions, so we have to disable caching entirely on PRs/merge_queue.
+      USE_CACHE: ${{ github.ref_name == github.event.repository.default_branch }}
     steps:
       - name: Checkout code
         uses: actions/checkout@v7
@@ -59,11 +63,12 @@ jobs:
       - name: Set up QEMU
         uses: docker/setup-qemu-action@v4
         with:
-          cache-image: true
+          cache-image: ${{ env.USE_CACHE }}
 
       - name: Docker setup
         uses: docker/setup-buildx-action@v4
         with:
+          cache-binary: ${{ env.USE_CACHE }}
           # Add Google/Amazon DockerHub mirrors to buildkit config
           # https://docs.docker.com/build/ci/github-actions/configure-builder/#registry-mirror
           buildkitd-config-inline: |