test: make coverage failures observable

Keep per-root logs, reject concurrent coverage runs, and avoid relying on /bin/sleep in the worker timeout test.

Assisted-by: Codex:gpt-5
Signed-off-by: Richard Palethorpe <io@richiejp.com>

.agents/building-and-testing.md

@@ -21,6 +21,7 @@ Let's say the user wants to build a particular backend for a given platform. For
 The core Go suites (`./pkg`, `./core`, plus the in-process integration suite `./tests/e2e`) are covered by a **strict, monotonic coverage ratchet**:
 
 - `make test-coverage` — runs the suites with `covermode=atomic` instrumentation and writes a merged profile to `coverage/coverage.out`. Uses the same prerequisites as `make test`.
+  - Verbose Ginkgo output is written to `coverage/logs/<root>.log`, with the prior run retained as `<root>.log.previous`. The terminal prints one status line per root and a short failure extract. If any suite fails, no merged profile is produced and the percentage ratchet is explicitly not run. A lock under `coverage/` rejects concurrent runs, which would otherwise corrupt their shared profiles and logs.
   - **`--coverpkg` (`COVERAGE_COVERPKG = core/...,pkg/...`):** coverage is attributed to the core+pkg packages, not just the package under test. This is what lets the in-process `tests/e2e` suite (which drives the real HTTP server over loopback via `application.New`) credit the `core/http/endpoints/...` handlers it exercises — folding it in roughly doubled endpoint coverage (e.g. `endpoints/openai` 13.6% → 52%). The denominator is therefore *all* of `core`+`pkg` (minus generated proto, dropped via `COVERAGE_EXCLUDE_RE`), so the number isn't comparable to a plain per-package figure.
   - **Integration suites (`COVERAGE_E2E_ROOTS = ./tests/e2e`)** run non-recursively (excludes `tests/e2e/distributed`, which needs containers) with `--label-filter=!real-models` (those need a downloaded model) against the mock backend built by `prepare-test`. `tests/integration` is deliberately excluded — it needs `make backends/local-store`, which the coverage CI job doesn't build.
   - **Flake note:** folding integration tests into a *strict* gate means a hard e2e failure (or a spec that silently stops running) can fail the coverage gate, not just the test. `--flake-attempts` absorbs transient retryable failures; covermode=atomic keeps line coverage deterministic otherwise.

Makefile

@@ -227,7 +227,7 @@ test-ci-scripts:
 ## --fail-fast so a single failure doesn't truncate the coverage number, and
 ## uses covermode=atomic so the result is deterministic. Prints the total.
 test-coverage: prepare-test
-	@echo 'Running tests with coverage'
+	@echo 'Running tests with coverage (test failures stop before the percentage ratchet)'
 	GINKGO_TAGS="$(COVERAGE_TAGS)" \
 	COVERAGE_COVERPKG="$(COVERAGE_COVERPKG)" \
 	COVERAGE_E2E_ROOTS="$(COVERAGE_E2E_ROOTS)" \
@@ -250,6 +250,7 @@ test-coverage-baseline: test-coverage
 ## run-to-run jitter from the in-process tests/e2e suite folded in via
 ## --coverpkg (timing-dependent which handler lines execute).
 test-coverage-check: test-coverage
+	@echo 'Running coverage percentage ratchet'
 	@scripts/coverage-check.sh $(COVERAGE_PROFILE) $(COVERAGE_BASELINE)
 
 ########################################################

core/services/worker/free_timeout_test.go

@@ -6,6 +6,8 @@ import (
 	"os"
 	"strconv"
 	"syscall"
+	"testing"
+	"time"
 
 	process "github.com/mudler/go-processmanager"
 	gogrpc "google.golang.org/grpc"
@@ -16,6 +18,19 @@ import (
 	. "github.com/onsi/gomega"
 )
 
+// TestWorkerFixtureProcess turns the current test binary into a portable
+// long-running child for the process-stop assertions below. Using the test
+// binary avoids assuming Unix utilities live at paths such as /bin/sleep,
+// which is not true in Nix environments.
+func TestWorkerFixtureProcess(t *testing.T) {
+	if os.Getenv("LOCALAI_WORKER_FIXTURE_PROCESS") != "1" {
+		return
+	}
+	for {
+		time.Sleep(time.Hour)
+	}
+}
+
 // pidAlive probes the OS directly for a process ID. The supervisor's own
 // liveness helpers all go through go-processmanager's pidfile, which Stop
 // deletes as part of releasing the handle, so they report "not alive" even if
@@ -82,10 +97,13 @@ var _ = Describe("Stopping a backend whose Free never returns", func() {
 		// actually dead afterwards, not merely that Stop() returned. It
 		// outlives every timeout below, so if it is gone at the end it is
 		// because the supervisor signalled it.
+		executable, err := os.Executable()
+		Expect(err).ToNot(HaveOccurred())
 		proc = process.New(
 			process.WithTemporaryStateDir(),
-			process.WithName("/bin/sleep"),
-			process.WithArgs("300"),
+			process.WithName(executable),
+			process.WithArgs("-test.run=^TestWorkerFixtureProcess$"),
+			process.WithEnvironment(append(os.Environ(), "LOCALAI_WORKER_FIXTURE_PROCESS=1")...),
 		)
 		Expect(proc.Run()).To(Succeed())
 

scripts/run-coverage.sh

@@ -22,6 +22,10 @@
 #   COVERAGE_EXCLUDE_RE  egrep pattern of profile lines to drop before merging,
 #                        e.g. generated protobuf (grpc/proto/.*\.pb\.go).
 #
+# Verbose Ginkgo output is retained in OUTPUT_DIR/logs. The previous run's log
+# for each root is kept with a .previous suffix, so a noisy failure remains
+# available without flooding the commit-hook output.
+#
 # Why one ginkgo invocation per root: passing several recursive roots to a
 # single ginkgo run only merges ONE root's coverprofile into --output-dir
 # (verified ginkgo 2.29.0) — the rest are silently dropped. So each root runs
@@ -41,10 +45,25 @@ shift 3
 unit_roots="$*" # space-free tokens (./pkg ./core)
 
 mkdir -p "$out_dir"
+lock_dir="$out_dir/.run-coverage.lock"
+if ! mkdir "$lock_dir" 2>/dev/null; then
+	echo "run-coverage: another coverage run is using $out_dir" >&2
+	echo "run-coverage: wait for it to finish; if none is running, remove stale lock $lock_dir" >&2
+	exit 2
+fi
+cleanup() {
+	rmdir "$lock_dir" 2>/dev/null || :
+}
+trap cleanup EXIT
+trap 'exit 130' HUP INT TERM
+
+log_dir="$out_dir/logs"
+mkdir -p "$log_dir"
 # Clear per-root profiles from a previous run: the merge collects them by glob,
 # so a stale profile (e.g. from a root that failed to rebuild this run) must not
 # leak into the merged result.
 rm -f "$out_dir"/cover-*.out
+rm -f "$merged"
 fail=0
 
 # Common optional flags go into "$@"; unquoted ${VAR:+...} would word-split a
@@ -59,26 +78,72 @@ profile_name() {
 	printf 'cover-%s.out' "$(printf '%s' "$1" | sed 's#[./][./]*#_#g; s#^_##; s#_$##')"
 }
 
+log_name() {
+	printf '%s.log' "$(printf '%s' "$1" | sed 's#[./][./]*#_#g; s#^_##; s#_$##')"
+}
+
+rotate_log() {
+	log="$1"
+	if [ -f "$log" ]; then
+		mv -f "$log" "$log.previous"
+	fi
+}
+
+report_failure() {
+	root="$1"
+	log="$2"
+	echo "run-coverage: FAIL — tests under coverage failed for $root" >&2
+	echo "run-coverage: full output: $log" >&2
+	echo "run-coverage: relevant tail:" >&2
+	# Keep the terminal useful even when Ginkgo emits thousands of verbose lines.
+	# The complete log remains available when this short extract is insufficient.
+	summary="$(grep -E 'Summarizing|\[FAIL(ED)?\]|FAIL!|--- FAIL:|Test Suite Failed|could not finalize|Status code: 429|HTTP 429|rate limit|timed out|panic:|fork/exec|no such file or directory|Expected.*(but got|success)' "$log" \
+		| tail -n 30)"
+	if [ -n "$summary" ]; then
+		printf '%s\n' "$summary" >&2
+	else
+		tail -n 30 "$log" >&2
+	fi
+}
+
 # Unit/suite roots: recursive.
 for root in $unit_roots; do
 	base="$(profile_name "$root")"
+	log="$log_dir/$(log_name "$root")"
+	rotate_log "$log"
+	echo "run-coverage: testing $root (full output: $log)"
 	go run github.com/onsi/ginkgo/v2/ginkgo --flake-attempts "$flakes" -v -r "$@" \
-		--cover --covermode=atomic --coverprofile="$base" --output-dir="$out_dir" "$root" || fail=1
+		--cover --covermode=atomic --coverprofile="$base" --output-dir="$out_dir" "$root" >"$log" 2>&1 \
+		&& echo "run-coverage: PASS — $root" \
+		|| { fail=1; report_failure "$root" "$log"; }
 done
 
 # In-process integration roots: NON-recursive + optional label filter.
 for root in ${COVERAGE_E2E_ROOTS:-}; do
 	base="$(profile_name "$root")"
+	log="$log_dir/$(log_name "$root")"
+	rotate_log "$log"
+	echo "run-coverage: testing $root (full output: $log)"
 	if [ -n "${COVERAGE_E2E_LABELS:-}" ]; then
 		go run github.com/onsi/ginkgo/v2/ginkgo --flake-attempts "$flakes" -v "$@" \
 			--label-filter="$COVERAGE_E2E_LABELS" \
-			--cover --covermode=atomic --coverprofile="$base" --output-dir="$out_dir" "$root" || fail=1
+			--cover --covermode=atomic --coverprofile="$base" --output-dir="$out_dir" "$root" >"$log" 2>&1 \
+			&& echo "run-coverage: PASS — $root" \
+			|| { fail=1; report_failure "$root" "$log"; }
 	else
 		go run github.com/onsi/ginkgo/v2/ginkgo --flake-attempts "$flakes" -v "$@" \
-			--cover --covermode=atomic --coverprofile="$base" --output-dir="$out_dir" "$root" || fail=1
+			--cover --covermode=atomic --coverprofile="$base" --output-dir="$out_dir" "$root" >"$log" 2>&1 \
+			&& echo "run-coverage: PASS — $root" \
+			|| { fail=1; report_failure "$root" "$log"; }
 	fi
 done
 
+if [ "$fail" -ne 0 ]; then
+	echo "run-coverage: FAILED — one or more test suites failed; no merged profile was produced." >&2
+	echo "run-coverage: the coverage percentage ratchet was not run." >&2
+	exit "$fail"
+fi
+
 # Collect the per-root profiles by glob (space-safe, no list to track).
 set -- "$out_dir"/cover-*.out
 if [ ! -e "$1" ]; then
@@ -98,4 +163,4 @@ fi
 	' "$@"
 } > "$merged"
 
-exit "$fail"
+echo "run-coverage: all test suites passed; merged profile: $merged"