From b4e16b963ffec22398a4c32d8aea82f51a1ac428 Mon Sep 17 00:00:00 2001 From: localai-org-maint-bot Date: Tue, 1 Sep 2026 08:55:35 +0200 Subject: [PATCH] fix(ci): stop the e2e teardown from failing a green suite (#11816) * fix(ci): remove the e2e container before removing its image `docker stop` returns as soon as the container exits, but the daemon reaps a `--rm` container asynchronously after that. The `docker rmi localai-tests` that follows teardown-e2e then loses the race against the reaper and fails with "conflict: ... is using its referenced image", so make exits 1 and the job goes red after every spec has passed. This is why the E2E Backend Tests job fails at random across pull requests. Runs 33435319093, 33435332991, 33412165884 and 33444669207 all report "SUCCESS! -- 235 Passed | 0 Failed" and then die in teardown. `docker rm -f` is synchronous, so the image reference is gone before teardown-e2e returns. It also covers the case where no container is running, which `docker stop` could not because it rejects an empty argument list. Signed-off-by: Ettore Di Giacinto Assisted-by: Claude:claude-opus-5 [Claude Code] * fix(ci): open a tmate session only when a PR asks for one The tmate step runs on every failure and then holds the runner until GitHub cancels the job at the 6 hour limit. A one second cleanup race in the e2e teardown therefore costs a whole ubuntu-latest slot. The recent run list is full of 6h, 7h and 12h cancelled runs for that reason. The step now needs the `ci-debug` label on the pull request, so a session opens when somebody wants to debug and never otherwise. The 30 minute step timeout caps the cost when the label is left behind. Signed-off-by: Ettore Di Giacinto Assisted-by: Claude:claude-opus-5 [Claude Code] --------- Signed-off-by: Ettore Di Giacinto Co-authored-by: Ettore Di Giacinto --- .github/workflows/test.yml | 14 ++++++++++++-- .github/workflows/tests-aio.yml | 7 ++++++- .github/workflows/tests-e2e.yml | 7 ++++++- .github/workflows/tests-pii-ner-e2e.yml | 7 ++++++- .github/workflows/tests-ui-e2e.yml | 7 ++++++- Makefile | 10 +++++++++- 6 files changed, 45 insertions(+), 7 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 7929f6b65..74d57567f 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -80,8 +80,13 @@ jobs: coverage/coverage.out coverage/coverage.html if-no-files-found: ignore + # tmate keeps the runner busy until the 6 hour job limit, so a single + # failure costs a whole runner slot. Only open a session when someone + # asked for one by labelling the pull request `ci-debug`, and cap the + # session so a forgotten label cannot idle a runner either. - name: Setup tmate session if tests fail - if: ${{ failure() }} + if: ${{ failure() && contains(github.event.pull_request.labels.*.name, 'ci-debug') }} + timeout-minutes: 30 uses: mxschmitt/action-tmate@v3.23 with: detached: true @@ -125,8 +130,13 @@ jobs: export PATH="/opt/homebrew/opt/make/libexec/gnubin:$PATH" PATH="$PATH:$HOME/go/bin" make protogen-go PATH="$PATH:$HOME/go/bin" BUILD_TYPE="GITHUB_CI_HAS_BROKEN_METAL" CMAKE_ARGS="-DGGML_F16C=OFF -DGGML_AVX512=OFF -DGGML_AVX2=OFF -DGGML_FMA=OFF" make --jobs 4 --output-sync=target test + # tmate keeps the runner busy until the 6 hour job limit, so a single + # failure costs a whole runner slot. Only open a session when someone + # asked for one by labelling the pull request `ci-debug`, and cap the + # session so a forgotten label cannot idle a runner either. - name: Setup tmate session if tests fail - if: ${{ failure() }} + if: ${{ failure() && contains(github.event.pull_request.labels.*.name, 'ci-debug') }} + timeout-minutes: 30 uses: mxschmitt/action-tmate@v3.23 with: detached: true diff --git a/.github/workflows/tests-aio.yml b/.github/workflows/tests-aio.yml index f8d3d34f0..738f96dac 100644 --- a/.github/workflows/tests-aio.yml +++ b/.github/workflows/tests-aio.yml @@ -77,8 +77,13 @@ jobs: - name: Test run: | PATH="$PATH:$HOME/go/bin" make backends/local-store backends/silero-vad backends/llama-cpp backends/whisper backends/piper backends/stablediffusion-ggml docker-build-e2e e2e-aio + # tmate keeps the runner busy until the 6 hour job limit, so a single + # failure costs a whole runner slot. Only open a session when someone + # asked for one by labelling the pull request `ci-debug`, and cap the + # session so a forgotten label cannot idle a runner either. - name: Setup tmate session if tests fail - if: ${{ failure() }} + if: ${{ failure() && contains(github.event.pull_request.labels.*.name, 'ci-debug') }} + timeout-minutes: 30 uses: mxschmitt/action-tmate@v3.23 with: detached: true diff --git a/.github/workflows/tests-e2e.yml b/.github/workflows/tests-e2e.yml index 3c1cb711c..b5c84d2a8 100644 --- a/.github/workflows/tests-e2e.yml +++ b/.github/workflows/tests-e2e.yml @@ -63,8 +63,13 @@ jobs: - name: Test Backend E2E run: | PATH="$PATH:$HOME/go/bin" make build-mock-backend test-e2e + # tmate keeps the runner busy until the 6 hour job limit, so a single + # failure costs a whole runner slot. Only open a session when someone + # asked for one by labelling the pull request `ci-debug`, and cap the + # session so a forgotten label cannot idle a runner either. - name: Setup tmate session if tests fail - if: ${{ failure() }} + if: ${{ failure() && contains(github.event.pull_request.labels.*.name, 'ci-debug') }} + timeout-minutes: 30 uses: mxschmitt/action-tmate@v3.23 with: detached: true diff --git a/.github/workflows/tests-pii-ner-e2e.yml b/.github/workflows/tests-pii-ner-e2e.yml index 800f67190..c9ab182f2 100644 --- a/.github/workflows/tests-pii-ner-e2e.yml +++ b/.github/workflows/tests-pii-ner-e2e.yml @@ -88,8 +88,13 @@ jobs: # CPU and runs the token_classify capability spec (byte-offset contract). - name: Run live PII NER backend E2E run: PATH="$PATH:$HOME/go/bin" make test-extra-backend-privacy-filter + # tmate keeps the runner busy until the 6 hour job limit, so a single + # failure costs a whole runner slot. Only open a session when someone + # asked for one by labelling the pull request `ci-debug`, and cap the + # session so a forgotten label cannot idle a runner either. - name: Setup tmate session if tests fail - if: ${{ failure() }} + if: ${{ failure() && contains(github.event.pull_request.labels.*.name, 'ci-debug') }} + timeout-minutes: 30 uses: mxschmitt/action-tmate@v3.23 with: detached: true diff --git a/.github/workflows/tests-ui-e2e.yml b/.github/workflows/tests-ui-e2e.yml index 1a170fcf3..6d45c8e9f 100644 --- a/.github/workflows/tests-ui-e2e.yml +++ b/.github/workflows/tests-ui-e2e.yml @@ -75,8 +75,13 @@ jobs: path: core/http/react-ui/coverage/ if-no-files-found: ignore retention-days: 7 + # tmate keeps the runner busy until the 6 hour job limit, so a single + # failure costs a whole runner slot. Only open a session when someone + # asked for one by labelling the pull request `ci-debug`, and cap the + # session so a forgotten label cannot idle a runner either. - name: Setup tmate session if tests fail - if: ${{ failure() }} + if: ${{ failure() && contains(github.event.pull_request.labels.*.name, 'ci-debug') }} + timeout-minutes: 30 uses: mxschmitt/action-tmate@v3.23 with: detached: true diff --git a/Makefile b/Makefile index ebedb2c98..cf3248c3b 100644 --- a/Makefile +++ b/Makefile @@ -393,9 +393,17 @@ test-e2e: build-mock-backend build-cloud-proxy-backend prepare-e2e run-e2e-image $(MAKE) teardown-e2e docker rmi localai-tests +# `docker stop` returns as soon as the container exits, but Docker reaps a +# `--rm` container asynchronously after that. The `docker rmi localai-tests` in +# test-e2e then loses the race against the reaper and fails on a still +# referenced image, turning a green suite red. Removing the container ourselves +# is synchronous, so the image reference is gone before we return. It also +# covers the case where nothing is running, which `docker stop` could not +# because it rejects an empty argument list. teardown-e2e: rm -rf $(TEST_DIR) || true - docker stop $$(docker ps -q --filter ancestor=localai-tests) + @CONTAINERS=$$(docker ps -aq --filter ancestor=localai-tests 2>/dev/null); \ + if [ -n "$$CONTAINERS" ]; then docker rm -f $$CONTAINERS || true; fi ######################################################## ## Integration and unit tests