From 3257fc5cd866bdd9c391b16c4bedb4361326ff80 Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Mon, 31 Aug 2026 10:49:54 +0000 Subject: [PATCH] ci(distributed): run the distributed e2e suite on PRs The suite has never run in CI, so 239 specs across 32 files were verified only by hand. Path-filtered to distributed code, advisory until it earns a track record, and with flake retries at 1 rather than 5 so nondeterminism surfaces instead of being retried away. Assisted-by: Claude Opus 5 [claude-code] Signed-off-by: Ettore Di Giacinto --- .github/workflows/tests-e2e-distributed.yml | 73 +++++++++++++++++++++ Makefile | 8 ++- 2 files changed, 80 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/tests-e2e-distributed.yml diff --git a/.github/workflows/tests-e2e-distributed.yml b/.github/workflows/tests-e2e-distributed.yml new file mode 100644 index 000000000..4afc76dc0 --- /dev/null +++ b/.github/workflows/tests-e2e-distributed.yml @@ -0,0 +1,73 @@ +--- +name: 'E2E Distributed Tests' + +on: + pull_request: + paths: + - 'core/services/nodes/**' + - 'core/services/worker/**' + - 'core/services/messaging/**' + - 'core/services/syncstate/**' + - 'core/services/jobs/**' + - 'core/services/agents/**' + - 'core/services/agentpool/**' + - 'core/services/galleryop/**' + - 'core/http/routes/nodes.go' + - 'core/http/endpoints/localai/nodes.go' + - 'core/http/endpoints/openresponses/**' + - 'core/application/distributed.go' + - 'core/config/distributed_config.go' + - 'pkg/natsauth/**' + - 'tests/e2e/distributed/**' + - '.github/workflows/tests-e2e-distributed.yml' + - 'Makefile' + push: + branches: + - master + +concurrency: + group: ci-tests-e2e-distributed-${{ github.event.pull_request.number || github.sha }}-${{ github.repository }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + +jobs: + tests-e2e-distributed: + runs-on: ubuntu-latest + # Advisory while the suite builds a track record. Flip to a required check + # only after it has run clean for two weeks; a heavy suite made required on + # day one gets disabled instead of fixed. + continue-on-error: true + timeout-minutes: 45 + steps: + - name: Clone + uses: actions/checkout@v7 + with: + submodules: true + - name: Configure apt mirror on runner + uses: ./.github/actions/configure-apt-mirror + - name: Setup Go + uses: actions/setup-go@v5 + with: + go-version: '1.25.x' + cache: false + - name: Dependencies + run: | + sudo apt-get update + sudo apt-get install -y build-essential libopus-dev + - name: Proto Dependencies + run: | + curl -L -s https://github.com/protocolbuffers/protobuf/releases/download/v26.1/protoc-26.1-linux-x86_64.zip -o protoc.zip && \ + unzip -j -d /usr/local/bin protoc.zip bin/protoc && \ + rm protoc.zip + go install google.golang.org/protobuf/cmd/protoc-gen-go@v1.34.2 + go install google.golang.org/grpc/cmd/protoc-gen-go-grpc@1958fcbe2ca8bd93af633f11e97d44e567e945af + PATH="$PATH:$HOME/go/bin" make protogen-go + - name: Pre-pull test images + # Pulling here rather than inside the suite keeps container-start timing + # out of the spec timeouts and makes a registry outage read as a + # setup failure instead of a test failure. + run: | + docker pull postgres:16-alpine + docker pull nats:2-alpine + - name: Distributed E2E + run: | + PATH="$PATH:$HOME/go/bin" make test-e2e-distributed diff --git a/Makefile b/Makefile index ebedb2c98..3df884de8 100644 --- a/Makefile +++ b/Makefile @@ -340,12 +340,18 @@ run-e2e-aio: protogen-go @echo 'Running e2e AIO tests' $(GOCMD) run github.com/onsi/ginkgo/v2/ginkgo --flake-attempts $(TEST_FLAKES) -v -r ./tests/e2e-aio +# Flake retries for the distributed suite. Defaults to 1, unlike TEST_FLAKES: +# this suite exists to catch nondeterministic cluster behaviour, and retrying +# hides exactly the failures it is meant to surface. Raise it locally if you are +# bisecting something unrelated. +DISTRIBUTED_TEST_FLAKES?=1 + # Distributed architecture e2e (PostgreSQL + NATS via testcontainers). # Includes NatsJWT specs (JWT-enabled NATS). Requires Docker. # VLLMMultinode is excluded here; use test-e2e-vllm-multinode for that. test-e2e-distributed: protogen-go @echo 'Running distributed e2e tests (label Distributed, incl. NatsJWT)' - $(GOCMD) run github.com/onsi/ginkgo/v2/ginkgo --label-filter='Distributed && !VLLMMultinode' --flake-attempts $(TEST_FLAKES) -v -r ./tests/e2e/distributed + $(GOCMD) run github.com/onsi/ginkgo/v2/ginkgo --label-filter='Distributed && !VLLMMultinode && !Cluster' --flake-attempts $(DISTRIBUTED_TEST_FLAKES) --timeout=40m -v -r ./tests/e2e/distributed # vLLM multi-node DP smoke (CPU). Builds local-ai:tests and the # cpu-vllm backend from the current working tree, then drives a