ci(distributed): run the distributed e2e suite on PRs

The suite has never run in CI, so 239 specs across 32 files were verified only
by hand. Path-filtered to distributed code, advisory until it earns a track
record, and with flake retries at 1 rather than 5 so nondeterminism surfaces
instead of being retried away.

Assisted-by: Claude Opus 5 [claude-code]
Signed-off-by: Ettore Di Giacinto <mudler@localai.io>

.github/workflows/tests-e2e-distributed.yml

@@ -0,0 +1,73 @@
+---
+name: 'E2E Distributed Tests'
+
+on:
+  pull_request:
+    paths:
+      - 'core/services/nodes/**'
+      - 'core/services/worker/**'
+      - 'core/services/messaging/**'
+      - 'core/services/syncstate/**'
+      - 'core/services/jobs/**'
+      - 'core/services/agents/**'
+      - 'core/services/agentpool/**'
+      - 'core/services/galleryop/**'
+      - 'core/http/routes/nodes.go'
+      - 'core/http/endpoints/localai/nodes.go'
+      - 'core/http/endpoints/openresponses/**'
+      - 'core/application/distributed.go'
+      - 'core/config/distributed_config.go'
+      - 'pkg/natsauth/**'
+      - 'tests/e2e/distributed/**'
+      - '.github/workflows/tests-e2e-distributed.yml'
+      - 'Makefile'
+  push:
+    branches:
+      - master
+
+concurrency:
+  group: ci-tests-e2e-distributed-${{ github.event.pull_request.number || github.sha }}-${{ github.repository }}
+  cancel-in-progress: ${{ github.event_name == 'pull_request' }}
+
+jobs:
+  tests-e2e-distributed:
+    runs-on: ubuntu-latest
+    # Advisory while the suite builds a track record. Flip to a required check
+    # only after it has run clean for two weeks; a heavy suite made required on
+    # day one gets disabled instead of fixed.
+    continue-on-error: true
+    timeout-minutes: 45
+    steps:
+      - name: Clone
+        uses: actions/checkout@v7
+        with:
+          submodules: true
+      - name: Configure apt mirror on runner
+        uses: ./.github/actions/configure-apt-mirror
+      - name: Setup Go
+        uses: actions/setup-go@v5
+        with:
+          go-version: '1.25.x'
+          cache: false
+      - name: Dependencies
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y build-essential libopus-dev
+      - name: Proto Dependencies
+        run: |
+          curl -L -s https://github.com/protocolbuffers/protobuf/releases/download/v26.1/protoc-26.1-linux-x86_64.zip -o protoc.zip && \
+          unzip -j -d /usr/local/bin protoc.zip bin/protoc && \
+          rm protoc.zip
+          go install google.golang.org/protobuf/cmd/protoc-gen-go@v1.34.2
+          go install google.golang.org/grpc/cmd/protoc-gen-go-grpc@1958fcbe2ca8bd93af633f11e97d44e567e945af
+          PATH="$PATH:$HOME/go/bin" make protogen-go
+      - name: Pre-pull test images
+        # Pulling here rather than inside the suite keeps container-start timing
+        # out of the spec timeouts and makes a registry outage read as a
+        # setup failure instead of a test failure.
+        run: |
+          docker pull postgres:16-alpine
+          docker pull nats:2-alpine
+      - name: Distributed E2E
+        run: |
+          PATH="$PATH:$HOME/go/bin" make test-e2e-distributed

Makefile

@@ -340,12 +340,18 @@ run-e2e-aio: protogen-go
 	@echo 'Running e2e AIO tests'
 	$(GOCMD) run github.com/onsi/ginkgo/v2/ginkgo --flake-attempts $(TEST_FLAKES) -v -r ./tests/e2e-aio
 
+# Flake retries for the distributed suite. Defaults to 1, unlike TEST_FLAKES:
+# this suite exists to catch nondeterministic cluster behaviour, and retrying
+# hides exactly the failures it is meant to surface. Raise it locally if you are
+# bisecting something unrelated.
+DISTRIBUTED_TEST_FLAKES?=1
+
 # Distributed architecture e2e (PostgreSQL + NATS via testcontainers).
 # Includes NatsJWT specs (JWT-enabled NATS). Requires Docker.
 # VLLMMultinode is excluded here; use test-e2e-vllm-multinode for that.
 test-e2e-distributed: protogen-go
 	@echo 'Running distributed e2e tests (label Distributed, incl. NatsJWT)'
-	$(GOCMD) run github.com/onsi/ginkgo/v2/ginkgo --label-filter='Distributed && !VLLMMultinode' --flake-attempts $(TEST_FLAKES) -v -r ./tests/e2e/distributed
+	$(GOCMD) run github.com/onsi/ginkgo/v2/ginkgo --label-filter='Distributed && !VLLMMultinode && !Cluster' --flake-attempts $(DISTRIBUTED_TEST_FLAKES) --timeout=40m -v -r ./tests/e2e/distributed
 
 # vLLM multi-node DP smoke (CPU). Builds local-ai:tests and the
 # cpu-vllm backend from the current working tree, then drives a