diff --git a/.agents/building-and-testing.md b/.agents/building-and-testing.md index 021d555ec..05cc9c2f6 100644 --- a/.agents/building-and-testing.md +++ b/.agents/building-and-testing.md @@ -21,6 +21,7 @@ Let's say the user wants to build a particular backend for a given platform. For The core Go suites (`./pkg`, `./core`, plus the in-process integration suite `./tests/e2e`) are covered by a **strict, monotonic coverage ratchet**: - `make test-coverage` — runs the suites with `covermode=atomic` instrumentation and writes a merged profile to `coverage/coverage.out`. Uses the same prerequisites as `make test`. + - Verbose Ginkgo output is written to `coverage/logs/.log`, with the prior run retained as `.log.previous`. The terminal prints one status line per root and a short failure extract. If any suite fails, no merged profile is produced and the percentage ratchet is explicitly not run. A lock under `coverage/` rejects concurrent runs, which would otherwise corrupt their shared profiles and logs. - **`--coverpkg` (`COVERAGE_COVERPKG = core/...,pkg/...`):** coverage is attributed to the core+pkg packages, not just the package under test. This is what lets the in-process `tests/e2e` suite (which drives the real HTTP server over loopback via `application.New`) credit the `core/http/endpoints/...` handlers it exercises — folding it in roughly doubled endpoint coverage (e.g. `endpoints/openai` 13.6% → 52%). The denominator is therefore *all* of `core`+`pkg` (minus generated proto, dropped via `COVERAGE_EXCLUDE_RE`), so the number isn't comparable to a plain per-package figure. - **Integration suites (`COVERAGE_E2E_ROOTS = ./tests/e2e`)** run non-recursively (excludes `tests/e2e/distributed`, which needs containers) with `--label-filter=!real-models` (those need a downloaded model) against the mock backend built by `prepare-test`. `tests/integration` is deliberately excluded — it needs `make backends/local-store`, which the coverage CI job doesn't build. - **Flake note:** folding integration tests into a *strict* gate means a hard e2e failure (or a spec that silently stops running) can fail the coverage gate, not just the test. `--flake-attempts` absorbs transient retryable failures; covermode=atomic keeps line coverage deterministic otherwise. diff --git a/Makefile b/Makefile index 81b00d7c8..b39926f8c 100644 --- a/Makefile +++ b/Makefile @@ -227,7 +227,7 @@ test-ci-scripts: ## --fail-fast so a single failure doesn't truncate the coverage number, and ## uses covermode=atomic so the result is deterministic. Prints the total. test-coverage: prepare-test - @echo 'Running tests with coverage' + @echo 'Running tests with coverage (test failures stop before the percentage ratchet)' GINKGO_TAGS="$(COVERAGE_TAGS)" \ COVERAGE_COVERPKG="$(COVERAGE_COVERPKG)" \ COVERAGE_E2E_ROOTS="$(COVERAGE_E2E_ROOTS)" \ @@ -250,6 +250,7 @@ test-coverage-baseline: test-coverage ## run-to-run jitter from the in-process tests/e2e suite folded in via ## --coverpkg (timing-dependent which handler lines execute). test-coverage-check: test-coverage + @echo 'Running coverage percentage ratchet' @scripts/coverage-check.sh $(COVERAGE_PROFILE) $(COVERAGE_BASELINE) ######################################################## diff --git a/core/services/worker/free_timeout_test.go b/core/services/worker/free_timeout_test.go index 4f1b6346e..3ecdea2fb 100644 --- a/core/services/worker/free_timeout_test.go +++ b/core/services/worker/free_timeout_test.go @@ -6,6 +6,8 @@ import ( "os" "strconv" "syscall" + "testing" + "time" process "github.com/mudler/go-processmanager" gogrpc "google.golang.org/grpc" @@ -16,6 +18,19 @@ import ( . "github.com/onsi/gomega" ) +// TestWorkerFixtureProcess turns the current test binary into a portable +// long-running child for the process-stop assertions below. Using the test +// binary avoids assuming Unix utilities live at paths such as /bin/sleep, +// which is not true in Nix environments. +func TestWorkerFixtureProcess(t *testing.T) { + if os.Getenv("LOCALAI_WORKER_FIXTURE_PROCESS") != "1" { + return + } + for { + time.Sleep(time.Hour) + } +} + // pidAlive probes the OS directly for a process ID. The supervisor's own // liveness helpers all go through go-processmanager's pidfile, which Stop // deletes as part of releasing the handle, so they report "not alive" even if @@ -82,10 +97,13 @@ var _ = Describe("Stopping a backend whose Free never returns", func() { // actually dead afterwards, not merely that Stop() returned. It // outlives every timeout below, so if it is gone at the end it is // because the supervisor signalled it. + executable, err := os.Executable() + Expect(err).ToNot(HaveOccurred()) proc = process.New( process.WithTemporaryStateDir(), - process.WithName("/bin/sleep"), - process.WithArgs("300"), + process.WithName(executable), + process.WithArgs("-test.run=^TestWorkerFixtureProcess$"), + process.WithEnvironment(append(os.Environ(), "LOCALAI_WORKER_FIXTURE_PROCESS=1")...), ) Expect(proc.Run()).To(Succeed()) diff --git a/scripts/run-coverage.sh b/scripts/run-coverage.sh index 430ca1f11..88120bb58 100755 --- a/scripts/run-coverage.sh +++ b/scripts/run-coverage.sh @@ -22,6 +22,10 @@ # COVERAGE_EXCLUDE_RE egrep pattern of profile lines to drop before merging, # e.g. generated protobuf (grpc/proto/.*\.pb\.go). # +# Verbose Ginkgo output is retained in OUTPUT_DIR/logs. The previous run's log +# for each root is kept with a .previous suffix, so a noisy failure remains +# available without flooding the commit-hook output. +# # Why one ginkgo invocation per root: passing several recursive roots to a # single ginkgo run only merges ONE root's coverprofile into --output-dir # (verified ginkgo 2.29.0) — the rest are silently dropped. So each root runs @@ -41,10 +45,25 @@ shift 3 unit_roots="$*" # space-free tokens (./pkg ./core) mkdir -p "$out_dir" +lock_dir="$out_dir/.run-coverage.lock" +if ! mkdir "$lock_dir" 2>/dev/null; then + echo "run-coverage: another coverage run is using $out_dir" >&2 + echo "run-coverage: wait for it to finish; if none is running, remove stale lock $lock_dir" >&2 + exit 2 +fi +cleanup() { + rmdir "$lock_dir" 2>/dev/null || : +} +trap cleanup EXIT +trap 'exit 130' HUP INT TERM + +log_dir="$out_dir/logs" +mkdir -p "$log_dir" # Clear per-root profiles from a previous run: the merge collects them by glob, # so a stale profile (e.g. from a root that failed to rebuild this run) must not # leak into the merged result. rm -f "$out_dir"/cover-*.out +rm -f "$merged" fail=0 # Common optional flags go into "$@"; unquoted ${VAR:+...} would word-split a @@ -59,26 +78,72 @@ profile_name() { printf 'cover-%s.out' "$(printf '%s' "$1" | sed 's#[./][./]*#_#g; s#^_##; s#_$##')" } +log_name() { + printf '%s.log' "$(printf '%s' "$1" | sed 's#[./][./]*#_#g; s#^_##; s#_$##')" +} + +rotate_log() { + log="$1" + if [ -f "$log" ]; then + mv -f "$log" "$log.previous" + fi +} + +report_failure() { + root="$1" + log="$2" + echo "run-coverage: FAIL — tests under coverage failed for $root" >&2 + echo "run-coverage: full output: $log" >&2 + echo "run-coverage: relevant tail:" >&2 + # Keep the terminal useful even when Ginkgo emits thousands of verbose lines. + # The complete log remains available when this short extract is insufficient. + summary="$(grep -E 'Summarizing|\[FAIL(ED)?\]|FAIL!|--- FAIL:|Test Suite Failed|could not finalize|Status code: 429|HTTP 429|rate limit|timed out|panic:|fork/exec|no such file or directory|Expected.*(but got|success)' "$log" \ + | tail -n 30)" + if [ -n "$summary" ]; then + printf '%s\n' "$summary" >&2 + else + tail -n 30 "$log" >&2 + fi +} + # Unit/suite roots: recursive. for root in $unit_roots; do base="$(profile_name "$root")" + log="$log_dir/$(log_name "$root")" + rotate_log "$log" + echo "run-coverage: testing $root (full output: $log)" go run github.com/onsi/ginkgo/v2/ginkgo --flake-attempts "$flakes" -v -r "$@" \ - --cover --covermode=atomic --coverprofile="$base" --output-dir="$out_dir" "$root" || fail=1 + --cover --covermode=atomic --coverprofile="$base" --output-dir="$out_dir" "$root" >"$log" 2>&1 \ + && echo "run-coverage: PASS — $root" \ + || { fail=1; report_failure "$root" "$log"; } done # In-process integration roots: NON-recursive + optional label filter. for root in ${COVERAGE_E2E_ROOTS:-}; do base="$(profile_name "$root")" + log="$log_dir/$(log_name "$root")" + rotate_log "$log" + echo "run-coverage: testing $root (full output: $log)" if [ -n "${COVERAGE_E2E_LABELS:-}" ]; then go run github.com/onsi/ginkgo/v2/ginkgo --flake-attempts "$flakes" -v "$@" \ --label-filter="$COVERAGE_E2E_LABELS" \ - --cover --covermode=atomic --coverprofile="$base" --output-dir="$out_dir" "$root" || fail=1 + --cover --covermode=atomic --coverprofile="$base" --output-dir="$out_dir" "$root" >"$log" 2>&1 \ + && echo "run-coverage: PASS — $root" \ + || { fail=1; report_failure "$root" "$log"; } else go run github.com/onsi/ginkgo/v2/ginkgo --flake-attempts "$flakes" -v "$@" \ - --cover --covermode=atomic --coverprofile="$base" --output-dir="$out_dir" "$root" || fail=1 + --cover --covermode=atomic --coverprofile="$base" --output-dir="$out_dir" "$root" >"$log" 2>&1 \ + && echo "run-coverage: PASS — $root" \ + || { fail=1; report_failure "$root" "$log"; } fi done +if [ "$fail" -ne 0 ]; then + echo "run-coverage: FAILED — one or more test suites failed; no merged profile was produced." >&2 + echo "run-coverage: the coverage percentage ratchet was not run." >&2 + exit "$fail" +fi + # Collect the per-root profiles by glob (space-safe, no list to track). set -- "$out_dir"/cover-*.out if [ ! -e "$1" ]; then @@ -98,4 +163,4 @@ fi ' "$@" } > "$merged" -exit "$fail" +echo "run-coverage: all test suites passed; merged profile: $merged"