From f0864a47da35a9d9417323d0bbd53f6d76e3676a Mon Sep 17 00:00:00 2001
From: Ettore Di Giacinto <mudler@localai.io>
Date: Tue, 3 Nov 2026 20:51:26 +0000
Subject: [PATCH] ci: cancel superseded master runs of test workflows

Many pushes to master left over a hundred runs queued, because every
push used its own concurrency group. Group push runs by ref and cancel
the in-flight run on master for the test, lint and E2E workflows. PR,
tag, schedule and dispatch runs keep groups that a master push cannot
cancel.

The intel cache job pushes one fixed tag, so it now keeps at most one
pending run per event and never cancels a running build.

image.yml and backend.yml are unchanged: they build from the diff of a
single push, so replacing a queued run could skip a code commit. Document
the policy in .agents/ci-caching.md.

Assisted-by: Claude:claude-sonnet-5-5 python3
---
 .agents/ci-caching.md                       | 20 ++++++++++++++++++++
 .github/workflows/build-test.yaml           | 10 ++++++++--
 .github/workflows/ci-tools-tests.yaml       | 10 ++++++++++
 .github/workflows/generate_intel_image.yaml |  9 +++++++--
 .github/workflows/lint.yml                  | 10 ++++++++--
 .github/workflows/realtime-conformance.yml  | 10 ++++++++--
 .github/workflows/test-extra.yml            | 10 ++++++++--
 .github/workflows/test.yml                  | 10 ++++++++--
 .github/workflows/tests-aio.yml             | 10 ++++++++--
 .github/workflows/tests-e2e.yml             | 10 ++++++++--
 .github/workflows/tests-pii-ner-e2e.yml     | 10 ++++++++--
 .github/workflows/tests-ui-e2e.yml          | 10 ++++++++--
 12 files changed, 109 insertions(+), 20 deletions(-)

diff --git a/.agents/ci-caching.md b/.agents/ci-caching.md
index 53eec1778c..dc497201fd 100644
--- a/.agents/ci-caching.md
+++ b/.agents/ci-caching.md
@@ -289,6 +289,26 @@ concurrency:
 - **PR events** group by PR number → newer pushes to the same PR cancel old runs (intended).
 - **Push events** group by `github.sha` → each master commit gets its own run; rapid-fire merges don't cancel each other (this was a real issue prior — two master pushes 11 seconds apart would cancel the first's CI).
 
+### Test, lint and E2E workflows cancel superseded master runs
+
+`test.yml`, `lint.yml`, `build-test.yaml`, `test-extra.yml`, `tests-e2e.yml`, `tests-ui-e2e.yml`, `tests-aio.yml`, `tests-pii-ner-e2e.yml`, `realtime-conformance.yml` and `ci-tools-tests.yaml` use this instead:
+
+```yaml
+concurrency:
+  group: ci-<workflow>-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.sha }}-${{ github.repository }}
+  cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && github.ref == 'refs/heads/master') }}
+```
+
+- A newer push to master cancels the older in-flight run of the same workflow. Tests are cumulative, so the newest commit is the one worth testing. This stops a burst of merges from leaving hundreds of queued runs.
+- Tag pushes group by their own tag ref, and schedule and `workflow_dispatch` runs group by SHA, so a master push never cancels them.
+- The cost: the run of an older commit shows as cancelled, so a failure that the newer commit also contains is reported by the newer run, not the older one.
+
+### Image workflows are deliberately not cancelled
+
+`image.yml` and `backend.yml` keep the per-SHA group. Both decide what to build from the diff of one push (`event.before` to `sha`): the `changes` job in `image.yml`, `scripts/changed-backends.js` in `backend.yml`. If a queued run were replaced by a newer one, the newer run would only see its own diff. A code commit followed by a gallery-only commit would then build nothing. Do not add a pending-replacement group to them until the diff base is the last commit that was actually built.
+
+`generate_intel_image.yaml` pushes one fixed tag, so its group is `intel-cache-<event>-<repo>` with `cancel-in-progress: false`: a running build finishes, and a newer pending run replaces an older pending one.
+
 ## Self-warming, no separate populator
 
 There is no cron job that pre-warms the BuildKit cache for individual backends. The production builds *are* the populators. The first master build of a given matrix entry pays the cold cost; subsequent same-entry master builds reuse everything that hasn't changed (apt installs, gRPC compile in the variant `builder-fromsource` stage or skipped entirely when consuming `base-grpc-*`, Python wheel installs, etc.). The base-images workflow's weekly cron is the closest thing to a populator and only refreshes the prebuilt builder bases.
diff --git a/.github/workflows/build-test.yaml b/.github/workflows/build-test.yaml
index 9c34121f93..a4612d75f8 100644
--- a/.github/workflows/build-test.yaml
+++ b/.github/workflows/build-test.yaml
@@ -1,3 +1,9 @@
+# Concurrency policy: a newer push to master cancels the in-flight run of this
+# workflow on master, because the newer commit contains the older one and the
+# result that matters is the newest. PR runs group by PR number (as before),
+# tag pushes group by their own tag ref, and schedule/dispatch runs group by
+# commit SHA, so none of those is ever cancelled by a master push.
+# See .agents/ci-caching.md, section "Concurrency".
 name: Build test
 
 on:
@@ -44,8 +50,8 @@ on:
       - 'Entitlements.plist'
 
 concurrency:
-  group: ci-build-test-${{ github.event.pull_request.number || github.sha }}-${{ github.repository }}
-  cancel-in-progress: ${{ github.event_name == 'pull_request' }}
+  group: ci-build-test-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.sha }}-${{ github.repository }}
+  cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && github.ref == 'refs/heads/master') }}
 
 jobs:
   build-test:
diff --git a/.github/workflows/ci-tools-tests.yaml b/.github/workflows/ci-tools-tests.yaml
index 604426253e..1d880e7680 100644
--- a/.github/workflows/ci-tools-tests.yaml
+++ b/.github/workflows/ci-tools-tests.yaml
@@ -1,3 +1,9 @@
+# Concurrency policy: a newer push to master cancels the in-flight run of this
+# workflow on master, because the newer commit contains the older one and the
+# result that matters is the newest. PR runs group by PR number (as before),
+# tag pushes group by their own tag ref, and schedule/dispatch runs group by
+# commit SHA, so none of those is ever cancelled by a master push.
+# See .agents/ci-caching.md, section "Concurrency".
 ---
 # The packages under .github/ci/ are invisible to `go list ./...`, so neither
 # `make lint` nor the repository test run ever touches them. Their specs are
@@ -13,6 +19,10 @@ on:
       - master
     paths:
       - '.github/ci/**'
+concurrency:
+  group: ci-tools-tests-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.sha }}-${{ github.repository }}
+  cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && github.ref == 'refs/heads/master') }}
+
 jobs:
   ci-tools:
     name: 'Test the .github/ci generators'
diff --git a/.github/workflows/generate_intel_image.yaml b/.github/workflows/generate_intel_image.yaml
index 22627387f3..33b652cdfa 100644
--- a/.github/workflows/generate_intel_image.yaml
+++ b/.github/workflows/generate_intel_image.yaml
@@ -1,3 +1,8 @@
+# Concurrency policy: this job pushes one fixed tag, so only the newest build
+# matters. A running build is never cancelled (cancel-in-progress is false);
+# GitHub keeps at most one pending run per group and replaces an older pending
+# run with a newer one. Push and workflow_dispatch use separate groups, so a
+# manual run is never replaced by a push.
 name: 'generate and publish intel docker caches'
 
 on:
@@ -7,8 +12,8 @@ on:
       - master
 
 concurrency:
-  group: intel-cache-${{ github.event.pull_request.number || github.sha }}-${{ github.repository }}
-  cancel-in-progress: ${{ github.event_name == 'pull_request' }}
+  group: intel-cache-${{ github.event_name }}-${{ github.repository }}
+  cancel-in-progress: false
 
 jobs:
   generate_caches:
diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml
index 2f2e0e8ec2..0fe1ae3823 100644
--- a/.github/workflows/lint.yml
+++ b/.github/workflows/lint.yml
@@ -1,4 +1,10 @@
 ---
+# Concurrency policy: a newer push to master cancels the in-flight run of this
+# workflow on master, because the newer commit contains the older one and the
+# result that matters is the newest. PR runs group by PR number (as before),
+# tag pushes group by their own tag ref, and schedule/dispatch runs group by
+# commit SHA, so none of those is ever cancelled by a master push.
+# See .agents/ci-caching.md, section "Concurrency".
 name: 'lint'
 
 on:
@@ -16,8 +22,8 @@ on:
       - master
 
 concurrency:
-  group: ci-lint-${{ github.event.pull_request.number || github.sha }}-${{ github.repository }}
-  cancel-in-progress: ${{ github.event_name == 'pull_request' }}
+  group: ci-lint-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.sha }}-${{ github.repository }}
+  cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && github.ref == 'refs/heads/master') }}
 
 jobs:
   golangci-lint:
diff --git a/.github/workflows/realtime-conformance.yml b/.github/workflows/realtime-conformance.yml
index 20a92f35ac..1c0b94c283 100644
--- a/.github/workflows/realtime-conformance.yml
+++ b/.github/workflows/realtime-conformance.yml
@@ -1,4 +1,10 @@
 ---
+# Concurrency policy: a newer push to master cancels the in-flight run of this
+# workflow on master, because the newer commit contains the older one and the
+# result that matters is the newest. PR runs group by PR number (as before),
+# tag pushes group by their own tag ref, and schedule/dispatch runs group by
+# commit SHA, so none of those is ever cancelled by a master push.
+# See .agents/ci-caching.md, section "Concurrency".
 name: 'realtime-conformance'
 
 # Verifies the realtime state-machine implementations conform to their formal
@@ -36,8 +42,8 @@ on:
       - 'scripts/realtime-conformance.sh'
 
 concurrency:
-  group: realtime-conformance-${{ github.event.pull_request.number || github.sha }}-${{ github.repository }}
-  cancel-in-progress: ${{ github.event_name == 'pull_request' }}
+  group: realtime-conformance-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.sha }}-${{ github.repository }}
+  cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && github.ref == 'refs/heads/master') }}
 
 jobs:
   conformance:
diff --git a/.github/workflows/test-extra.yml b/.github/workflows/test-extra.yml
index 892b34100c..4e94dcb322 100644
--- a/.github/workflows/test-extra.yml
+++ b/.github/workflows/test-extra.yml
@@ -1,4 +1,10 @@
 ---
+# Concurrency policy: a newer push to master cancels the in-flight run of this
+# workflow on master, because the newer commit contains the older one and the
+# result that matters is the newest. PR runs group by PR number (as before),
+# tag pushes group by their own tag ref, and schedule/dispatch runs group by
+# commit SHA, so none of those is ever cancelled by a master push.
+# See .agents/ci-caching.md, section "Concurrency".
 name: 'Tests extras backends'
 
 on:
@@ -10,8 +16,8 @@ on:
       - '*'
 
 concurrency:
-  group: ci-tests-extra-${{ github.event.pull_request.number || github.sha }}-${{ github.repository }}
-  cancel-in-progress: ${{ github.event_name == 'pull_request' }}
+  group: ci-tests-extra-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.sha }}-${{ github.repository }}
+  cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && github.ref == 'refs/heads/master') }}
 
 jobs:
   detect-changes:
diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index bf8d694db6..eab2e29a27 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -1,4 +1,10 @@
 ---
+# Concurrency policy: a newer push to master cancels the in-flight run of this
+# workflow on master, because the newer commit contains the older one and the
+# result that matters is the newest. PR runs group by PR number (as before),
+# tag pushes group by their own tag ref, and schedule/dispatch runs group by
+# commit SHA, so none of those is ever cancelled by a master push.
+# See .agents/ci-caching.md, section "Concurrency".
 name: 'tests'
 
 on:
@@ -67,8 +73,8 @@ on:
   workflow_dispatch:
 
 concurrency:
-  group: ci-tests-${{ github.event.pull_request.number || github.sha }}-${{ github.repository }}
-  cancel-in-progress: ${{ github.event_name == 'pull_request' }}
+  group: ci-tests-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.sha }}-${{ github.repository }}
+  cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && github.ref == 'refs/heads/master') }}
 
 jobs:
   tests-linux:
diff --git a/.github/workflows/tests-aio.yml b/.github/workflows/tests-aio.yml
index 738f96dacc..a09391a70c 100644
--- a/.github/workflows/tests-aio.yml
+++ b/.github/workflows/tests-aio.yml
@@ -1,4 +1,10 @@
 ---
+# Concurrency policy: a newer push to master cancels the in-flight run of this
+# workflow on master, because the newer commit contains the older one and the
+# result that matters is the newest. PR runs group by PR number (as before),
+# tag pushes group by their own tag ref, and schedule/dispatch runs group by
+# commit SHA, so none of those is ever cancelled by a master push.
+# See .agents/ci-caching.md, section "Concurrency".
 name: 'tests-aio'
 
 # Runs the all-in-one (AIO) Docker image with real backends + real models.
@@ -22,8 +28,8 @@ on:
       - '*'
 
 concurrency:
-  group: ci-tests-aio-${{ github.event.pull_request.number || github.sha }}-${{ github.repository }}
-  cancel-in-progress: ${{ github.event_name == 'pull_request' }}
+  group: ci-tests-aio-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.sha }}-${{ github.repository }}
+  cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && github.ref == 'refs/heads/master') }}
 
 jobs:
   tests-aio:
diff --git a/.github/workflows/tests-e2e.yml b/.github/workflows/tests-e2e.yml
index 030e170a7f..3da43c8f1c 100644
--- a/.github/workflows/tests-e2e.yml
+++ b/.github/workflows/tests-e2e.yml
@@ -1,4 +1,10 @@
 ---
+# Concurrency policy: a newer push to master cancels the in-flight run of this
+# workflow on master, because the newer commit contains the older one and the
+# result that matters is the newest. PR runs group by PR number (as before),
+# tag pushes group by their own tag ref, and schedule/dispatch runs group by
+# commit SHA, so none of those is ever cancelled by a master push.
+# See .agents/ci-caching.md, section "Concurrency".
 name: 'E2E Backend Tests'
 
 on:
@@ -46,8 +52,8 @@ on:
       - '.github/workflows/tests-e2e.yml'
 
 concurrency:
-  group: ci-tests-e2e-backend-${{ github.event.pull_request.number || github.sha }}-${{ github.repository }}
-  cancel-in-progress: ${{ github.event_name == 'pull_request' }}
+  group: ci-tests-e2e-backend-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.sha }}-${{ github.repository }}
+  cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && github.ref == 'refs/heads/master') }}
 
 jobs:
   tests-e2e-backend:
diff --git a/.github/workflows/tests-pii-ner-e2e.yml b/.github/workflows/tests-pii-ner-e2e.yml
index c9ab182f26..5caba27696 100644
--- a/.github/workflows/tests-pii-ner-e2e.yml
+++ b/.github/workflows/tests-pii-ner-e2e.yml
@@ -1,4 +1,10 @@
 ---
+# Concurrency policy: a newer push to master cancels the in-flight run of this
+# workflow on master, because the newer commit contains the older one and the
+# result that matters is the newest. PR runs group by PR number (as before),
+# tag pushes group by their own tag ref, and schedule/dispatch runs group by
+# commit SHA, so none of those is ever cancelled by a master push.
+# See .agents/ci-caching.md, section "Concurrency".
 name: 'PII NER tier E2E (live GGUF, CPU)'
 
 # Runs the real privacy-filter GGUF NER tier end-to-end on CPU — the gap the
@@ -46,8 +52,8 @@ on:
       - '.github/workflows/tests-pii-ner-e2e.yml'
 
 concurrency:
-  group: ci-tests-pii-ner-e2e-${{ github.event.pull_request.number || github.sha }}-${{ github.repository }}
-  cancel-in-progress: ${{ github.event_name == 'pull_request' }}
+  group: ci-tests-pii-ner-e2e-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.sha }}-${{ github.repository }}
+  cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && github.ref == 'refs/heads/master') }}
 
 jobs:
   tests-pii-ner-e2e:
diff --git a/.github/workflows/tests-ui-e2e.yml b/.github/workflows/tests-ui-e2e.yml
index e26221b17b..c795f9c941 100644
--- a/.github/workflows/tests-ui-e2e.yml
+++ b/.github/workflows/tests-ui-e2e.yml
@@ -1,4 +1,10 @@
 ---
+# Concurrency policy: a newer push to master cancels the in-flight run of this
+# workflow on master, because the newer commit contains the older one and the
+# result that matters is the newest. PR runs group by PR number (as before),
+# tag pushes group by their own tag ref, and schedule/dispatch runs group by
+# commit SHA, so none of those is ever cancelled by a master push.
+# See .agents/ci-caching.md, section "Concurrency".
 name: 'UI E2E Tests'
 
 on:
@@ -45,8 +51,8 @@ on:
       - '.github/actions/**'
 
 concurrency:
-  group: ci-tests-ui-e2e-${{ github.event.pull_request.number || github.sha }}-${{ github.repository }}
-  cancel-in-progress: ${{ github.event_name == 'pull_request' }}
+  group: ci-tests-ui-e2e-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.sha }}-${{ github.repository }}
+  cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && github.ref == 'refs/heads/master') }}
 
 jobs:
   tests-ui-e2e: