LocalAI/scripts/run-coverage.sh

#!/usr/bin/env sh
#
# run-coverage.sh OUTPUT_DIR MERGED_PROFILE FLAKE_ATTEMPTS UNIT_ROOT [UNIT_ROOT...]
#
# Runs the unit/suite tests under each UNIT_ROOT (recursively) plus the
# in-process integration suites in $COVERAGE_E2E_ROOTS, all instrumented to
# attribute coverage to $COVERAGE_COVERPKG, and merges everything into
# MERGED_PROFILE.
#
# Environment:
#   GINKGO_TAGS          build tags, e.g. "debug auth"
#   COVERAGE_COVERPKG    packages coverage is attributed to (comma-separated
#                        go patterns). Required so the in-process e2e suites
#                        credit the core/http handlers they drive over HTTP —
#                        with default per-package cover they'd only credit the
#                        e2e_test package itself.
#   COVERAGE_E2E_ROOTS   space-separated in-process integration suites, run
#                        NON-recursively (so tests/e2e/distributed, which needs
#                        containers, is excluded) with --label-filter.
#   COVERAGE_E2E_LABELS  ginkgo --label-filter for the e2e roots, e.g.
#                        "!real-models" (those specs need a downloaded model).
#   COVERAGE_EXCLUDE_RE  egrep pattern of profile lines to drop before merging,
#                        e.g. generated protobuf (grpc/proto/.*\.pb\.go).
#   COVERAGE_PROCS       parallel Ginkgo processes; 0 lets Ginkgo detect CPUs.
#   COVERAGE_SUITE_TIMEOUT maximum duration of each recursive root (default 5m).
#   COVERAGE_PROGRESS_AFTER emit diagnostics when a spec is slow (default 30s).
#
# Verbose Ginkgo output is retained in OUTPUT_DIR/logs. The previous run's log
# for each root is kept with a .previous suffix, so a noisy failure remains
# available without flooding the commit-hook output.
#
# Why one ginkgo invocation per root: passing several recursive roots to a
# single ginkgo run only merges ONE root's coverprofile into --output-dir
# (verified ginkgo 2.29.0) — the rest are silently dropped. So each root runs
# separately. Because --coverpkg makes the same package appear in multiple
# root profiles, the merge SUMS per-block hit counts (a plain concat would
# duplicate blocks and miscount); summing is valid for covermode=atomic.
#
# POSIX sh (no bash): no arrays. Roots are space-free tokens iterated with
# word-splitting; the only flag that can contain a space (--tags="debug auth")
# is held in the positional parameters ("$@") so its quoting survives.
set -u

out_dir="${1:?usage: run-coverage.sh OUTPUT_DIR MERGED_PROFILE FLAKES UNIT_ROOT...}"
merged="${2:?missing MERGED_PROFILE}"
flakes="${3:?missing FLAKE_ATTEMPTS}"
shift 3
unit_roots="$*" # space-free tokens (./pkg ./core)

mkdir -p "$out_dir"
lock_dir="$out_dir/.run-coverage.lock"
if ! mkdir "$lock_dir" 2>/dev/null; then
	echo "run-coverage: another coverage run is using $out_dir" >&2
	echo "run-coverage: wait for it to finish; if none is running, remove stale lock $lock_dir" >&2
	exit 2
fi
cleanup() {
	for root in $unit_roots ${COVERAGE_E2E_ROOTS:-}; do
		# --keep-separate-coverprofiles leaves Go's package test binaries behind.
		# They are deterministic runner artifacts, never source inputs.
		find "$root" -type f -name '*.test' -delete 2>/dev/null || :
	done
	rmdir "$lock_dir" 2>/dev/null || :
}
trap cleanup EXIT
trap 'exit 130' HUP INT TERM

log_dir="$out_dir/logs"
mkdir -p "$log_dir"
# Clear per-root profiles from a previous run: the merge collects them by glob,
# so a stale profile (e.g. from a root that failed to rebuild this run) must not
# leak into the merged result.
rm -f "$out_dir"/cover-*.out
rm -f "$out_dir"/*_cover-*.out
rm -f "$merged"
fail=0
timings="$out_dir/timings.tsv"
: > "$timings"
run_started="$(date +%s)"

procs="${COVERAGE_PROCS:-0}"
suite_timeout="${COVERAGE_SUITE_TIMEOUT:-5m}"
progress_after="${COVERAGE_PROGRESS_AFTER:-30s}"
parallel_flags="-p --keep-going --timeout=$suite_timeout --poll-progress-after=$progress_after --poll-progress-interval=10s"
if [ "$procs" -gt 0 ] 2>/dev/null; then
	parallel_flags="$parallel_flags --procs=$procs --compilers=$procs"
fi

# Common optional flags go into "$@"; unquoted ${VAR:+...} would word-split a
# --tags value that contains a space. The unit roots were captured above, so
# overwriting the positional parameters here is safe.
set --
[ -n "${GINKGO_TAGS:-}" ] && set -- "$@" "--tags=$GINKGO_TAGS"
[ -n "${COVERAGE_COVERPKG:-}" ] && set -- "$@" "--coverpkg=$COVERAGE_COVERPKG"

# cover-<root>.out, with path separators flattened (POSIX BRE, no \+).
profile_name() {
	printf 'cover-%s.out' "$(printf '%s' "$1" | sed 's#[./][./]*#_#g; s#^_##; s#_$##')"
}

log_name() {
	printf '%s.log' "$(printf '%s' "$1" | sed 's#[./][./]*#_#g; s#^_##; s#_$##')"
}

rotate_log() {
	log="$1"
	if [ -f "$log" ]; then
		mv -f "$log" "$log.previous"
	fi
}

# Ginkgo's recursive-run merger can fail after every suite has passed when a
# large --coverpkg run produces many profiles. Keep its profiles separate and
# merge them here using the same block-summing rule as the cross-root merge.
consolidate_root_profiles() {
	base="$1"
	set -- "$out_dir"/*_"$base"
	if [ ! -e "$1" ]; then
		echo "run-coverage: no per-package profiles produced for $base" >&2
		return 1
	fi
	tmp="$out_dir/.${base}.tmp"
	{
		echo "mode: atomic"
		awk '
			/^mode:/ { next }
			{ stmts[$1] = $2; cnt[$1] += $3 }
			END { for (k in stmts) print k, stmts[k], cnt[k] }
		' "$@"
	} > "$tmp"
	mv "$tmp" "$out_dir/$base"
	rm -f "$@"
}

report_failure() {
	root="$1"
	log="$2"
	echo "run-coverage: FAIL — tests under coverage failed for $root" >&2
	echo "run-coverage: full output: $log" >&2
	echo "run-coverage: relevant tail:" >&2
	# Keep the terminal useful even when Ginkgo emits thousands of verbose lines.
	# The complete log remains available when this short extract is insufficient.
	summary="$(grep -E 'Summarizing|\[FAIL(ED)?\]|FAIL!|--- FAIL:|Test Suite Failed|could not finalize|Status code: 429|HTTP 429|rate limit|timed out|panic:|fork/exec|no such file or directory|Expected.*(but got|success)' "$log" \
		| tail -n 30)"
	if [ -n "$summary" ]; then
		printf '%s\n' "$summary" >&2
	else
		tail -n 30 "$log" >&2
	fi
}

record_timing() {
	root="$1"
	started="$2"
	elapsed="$(( $(date +%s) - started ))"
	printf '%s\t%s\n' "$root" "$elapsed" >> "$timings"
	echo "run-coverage: TIMING — $root ${elapsed}s"
}

print_timing_summary() {
	echo "run-coverage: wall-clock summary"
	sort -t "$(printf '\t')" -k2,2nr "$timings" | awk -F '\t' '{ printf "  %5ss  %s\n", $2, $1 }'
	echo "  $(( $(date +%s) - run_started ))s  total"
	echo "run-coverage: slowest specs/hooks taking at least ${COVERAGE_SLOW_SPEC_THRESHOLD:-3}s (up to ${COVERAGE_SLOW_SPEC_LIMIT:-25} per root)"
	found=0
	while IFS="$(printf '\t')" read -r root elapsed; do
		log="$log_dir/$(log_name "$root")"
		entries="$(scripts/summarize-ginkgo-waits.sh "${COVERAGE_SLOW_SPEC_THRESHOLD:-3}" "$root" "$log" "${COVERAGE_SLOW_SPEC_LIMIT:-25}")"
		if [ -n "$entries" ]; then
			printf '%s\n' "$entries"
			found=1
		fi
	done < "$timings"
	[ "$found" -eq 1 ] || echo "  none"
}

# Unit/suite roots: recursive.
for root in $unit_roots; do
	base="$(profile_name "$root")"
	log="$log_dir/$(log_name "$root")"
	rotate_log "$log"
	echo "run-coverage: testing $root (full output: $log)"
	started="$(date +%s)"
	# parallel_flags is intentionally word-split: it contains CLI arguments only.
	# shellcheck disable=SC2086
	go run github.com/onsi/ginkgo/v2/ginkgo $parallel_flags --keep-separate-coverprofiles --flake-attempts "$flakes" -v -r "$@" \
		--cover --covermode=atomic --coverprofile="$base" --output-dir="$out_dir" "$root" >"$log" 2>&1 \
		&& consolidate_root_profiles "$base" \
		&& echo "run-coverage: PASS — $root" \
		|| { fail=1; report_failure "$root" "$log"; }
	record_timing "$root" "$started"
done

# In-process integration roots: NON-recursive + optional label filter.
for root in ${COVERAGE_E2E_ROOTS:-}; do
	base="$(profile_name "$root")"
	log="$log_dir/$(log_name "$root")"
	rotate_log "$log"
	echo "run-coverage: testing $root (full output: $log)"
	started="$(date +%s)"
	if [ -n "${COVERAGE_E2E_LABELS:-}" ]; then
		# shellcheck disable=SC2086
		go run github.com/onsi/ginkgo/v2/ginkgo $parallel_flags --keep-separate-coverprofiles --flake-attempts "$flakes" -v "$@" \
			--label-filter="$COVERAGE_E2E_LABELS" \
			--cover --covermode=atomic --coverprofile="$base" --output-dir="$out_dir" "$root" >"$log" 2>&1 \
			&& consolidate_root_profiles "$base" \
			&& echo "run-coverage: PASS — $root" \
			|| { fail=1; report_failure "$root" "$log"; }
	else
		# shellcheck disable=SC2086
		go run github.com/onsi/ginkgo/v2/ginkgo $parallel_flags --keep-separate-coverprofiles --flake-attempts "$flakes" -v "$@" \
			--cover --covermode=atomic --coverprofile="$base" --output-dir="$out_dir" "$root" >"$log" 2>&1 \
			&& consolidate_root_profiles "$base" \
			&& echo "run-coverage: PASS — $root" \
			|| { fail=1; report_failure "$root" "$log"; }
	fi
	record_timing "$root" "$started"
done

print_timing_summary

if [ "$fail" -ne 0 ]; then
	echo "run-coverage: FAILED — one or more test suites failed; no merged profile was produced." >&2
	echo "run-coverage: the coverage percentage ratchet was not run." >&2
	exit "$fail"
fi

# Collect the per-root profiles by glob (space-safe, no list to track).
set -- "$out_dir"/cover-*.out
if [ ! -e "$1" ]; then
	echo "run-coverage: no coverprofiles produced" >&2
	exit 2
fi

# Merge: drop the per-file mode headers and any excluded (generated) lines,
# then sum hit counts per identical block, emitting a single mode header.
{
	echo "mode: atomic"
	awk -v exre="${COVERAGE_EXCLUDE_RE:-}" '
		/^mode:/ { next }
		exre != "" && $1 ~ exre { next }
		{ stmts[$1] = $2; cnt[$1] += $3 }
		END { for (k in stmts) print k, stmts[k], cnt[k] }
	' "$@"
} > "$merged"

echo "run-coverage: all test suites passed; merged profile: $merged"