From bd4d52653037ee79ffcdfaa71babdd09b999fbf4 Mon Sep 17 00:00:00 2001 From: Zoltan Kochan Date: Mon, 21 Sep 2026 20:54:20 +0200 Subject: [PATCH] fix(ci): compare integrated benchmarks on the same runner (#15228) Compare integrated benchmark samples against main measured on the same runner. Keep absolute minimum timings in Bencher for historical tracking without letting cross-machine historical alerts overwrite the PR regression check. Require at least nine successful samples per target. Fail when the median slowdown and retained sample separation both exceed 5% of main's median or 2 ms, whichever is larger. Trim up to 10% from each tail. Report other median slowdowns above that tolerance as inconclusive. This conservative noise guard is not a statistical confidence interval; smaller or noisy regressions can pass, and sequential measurements remain vulnerable to runner drift. Run the comparison from trusted default-branch code in the privileged comment workflow, publish the report before enforcing the result, and preserve the existing check name on the triggering PR commit. Keep the peer-heavy Rust-versus-TypeScript assertion unchanged. Cover regressions, shared machine slowdowns, outliers, invalid data, all scenario and engine combinations, command names, CLI exit codes, and the duration-dependent tolerance with Node tests. Keep the explanation in the existing benchmark documentation. --- .../scripts/compare-integrated-benchmarks.mjs | 82 +++++++++++++ .../compare-integrated-benchmarks.test.mjs | 116 ++++++++++++++++++ .../pacquet-integrated-benchmark-comment.yml | 75 +++++++++-- .../pacquet-integrated-benchmark.yml | 42 +++---- pnpm/CONTRIBUTING.md | 37 ++++++ 5 files changed, 320 insertions(+), 32 deletions(-) create mode 100644 .github/scripts/compare-integrated-benchmarks.mjs create mode 100644 .github/scripts/compare-integrated-benchmarks.test.mjs diff --git a/.github/scripts/compare-integrated-benchmarks.mjs b/.github/scripts/compare-integrated-benchmarks.mjs new file mode 100644 index 0000000000..75d454ea27 --- /dev/null +++ b/.github/scripts/compare-integrated-benchmarks.mjs @@ -0,0 +1,82 @@ +import { readFile } from 'node:fs/promises' +import { join, resolve } from 'node:path' +import { pathToFileURL } from 'node:url' + +export const scenarios = [ + 'ISOLATED_FRESH_RESTORE_COLD_CACHE_COLD_STORE', + 'ISOLATED_FRESH_RESTORE_HOT_CACHE_HOT_STORE', + 'ISOLATED_REPEAT_INSTALL_HOT_CACHE_HOT_STORE', + 'ISOLATED_REPEAT_INSTALL_COLD_CACHE_HOT_STORE', + 'ISOLATED_FRESH_INSTALL_COLD_CACHE_COLD_STORE', + 'ISOLATED_FRESH_INSTALL_HOT_CACHE_HOT_STORE', + 'ISOLATED_FRESH_INSTALL_COLD_CACHE_HOT_STORE', + 'ISOLATED_FRESH_RESOLVE_HOT_CACHE_OFFLINE', + 'ISOLATED_PEER_HEAVY_RESOLVE_HOT_CACHE_OFFLINE', + 'ISOLATED_LINKED_WORKSPACE_RESOLVE_HOT_CACHE_OFFLINE', + 'ISOLATED_FRESH_RESTORE_COLD_CACHE_COLD_STORE_COLD_PNPR', + 'GVS_FRESH_RESTORE_HOT_CACHE_HOT_STORE', +] + +export function compare (report, engine) { + const head = samples(report, `${engine}@HEAD`) + const base = samples(report, `${engine}@main`) + const headMedian = median(head) + const baseMedian = median(base) + const tolerance = Math.max(baseMedian * 0.05, 0.002) + const ratio = headMedian / baseMedian + // Trim at most 10% of each tail. This is an empirical noise guard, not a + // confidence interval: hyperfine measures commands sequentially. + const headLow = head[Math.floor(head.length / 10)] + const baseHigh = base[base.length - 1 - Math.floor(base.length / 10)] + const status = headMedian <= baseMedian + tolerance ? 'Within tolerance' : headLow > baseHigh + tolerance + ? 'Regression' : 'Inconclusive (overlapping samples)' + return { head: headMedian, base: baseMedian, ratio, tolerance, status } +} + +function samples (report, name) { + const matches = report.results.filter(result => (result.command_name ?? result.command) === name) + if (matches.length !== 1) throw new Error(`Expected one result for ${name}`) + const { times, exit_codes: exitCodes } = matches[0] + if (!Array.isArray(times) || times.length < 9 || times.some(time => !Number.isFinite(time) || time <= 0)) { + throw new Error(`Expected at least 9 positive finite samples for ${name}`) + } + if (!Array.isArray(exitCodes) || exitCodes.length !== times.length || exitCodes.some(code => code !== 0)) { + throw new Error(`Missing or unsuccessful exit codes for ${name}`) + } + return [...times].sort((a, b) => a - b) +} + +function median (values) { + const middle = Math.floor(values.length / 2) + return values.length % 2 ? values[middle] : (values[middle - 1] + values[middle]) / 2 +} + +export async function renderComparison (directory) { + const lines = [ + '### Same-run regression check', + '', + 'HEAD and main are measured on the same runner. The tolerance is 5% of the main median or 2 ms, whichever is larger. A regression requires both the median slowdown and the gap between the fastest retained HEAD sample and slowest retained main sample to exceed this tolerance, after trimming up to 10% from each tail. This conservative noise guard is not a statistical confidence interval.', + '', + 'Inconclusive rows do not fail the check. They remain visible for performance review. Bencher retains absolute minimum timings for historical tracking; historical alerts do not gate this PR. Sequential measurements can still be affected by runner drift.', + '', + '| Scenario | Engine | main median | HEAD median | Change | Tolerance | Result |', + '| --- | --- | ---: | ---: | ---: | ---: | --- |', + ] + let failed = false + for (const scenario of scenarios) { + const report = JSON.parse(await readFile(join(directory, `BENCHMARK_REPORT_${scenario}.json`), 'utf8')) + const engines = scenario.includes('PEER_HEAVY') || scenario.includes('LINKED_WORKSPACE') ? ['pacquet'] : ['pacquet', 'pnpr'] + for (const engine of engines) { + const result = compare(report, engine) + failed ||= result.status === 'Regression' + lines.push(`| ${scenario.toLowerCase().replaceAll('_', '-')} | ${engine} | ${(result.base * 1000).toFixed(2)} ms | ${(result.head * 1000).toFixed(2)} ms | ${((result.ratio - 1) * 100).toFixed(1)}% | ${(result.tolerance * 1000).toFixed(2)} ms | ${result.status} |`) + } + } + return { markdown: lines.join('\n') + '\n', failed } +} + +if (process.argv[1] && import.meta.url === pathToFileURL(resolve(process.argv[1])).href) { + const { markdown, failed } = await renderComparison(process.argv[2]) + console.log(markdown) + process.exitCode = failed ? 1 : 0 +} diff --git a/.github/scripts/compare-integrated-benchmarks.test.mjs b/.github/scripts/compare-integrated-benchmarks.test.mjs new file mode 100644 index 0000000000..3a9c91f9ca --- /dev/null +++ b/.github/scripts/compare-integrated-benchmarks.test.mjs @@ -0,0 +1,116 @@ +import assert from 'node:assert/strict' +import { spawnSync } from 'node:child_process' +import { mkdtemp, readFile, rm, writeFile } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { fileURLToPath } from 'node:url' +import { test } from 'node:test' +import { compare, renderComparison, scenarios } from './compare-integrated-benchmarks.mjs' + +function command (name, times) { + return { command: name, times, exit_codes: times.map(() => 0) } +} + +function report (head, base) { + return { results: [command('pacquet@HEAD', head), command('pacquet@main', base)] } +} + +const stable = Array(10).fill(1) + +test('a slower machine affects both revisions without causing an alert', () => { + assert.equal(compare(report(stable.map(x => x * 3.1), stable.map(x => x * 3)), 'pacquet').status, 'Within tolerance') +}) + +test('a consistent regression fails, an improvement passes', () => { + assert.equal(compare(report(Array(9).fill(1.3), Array(9).fill(1)), 'pacquet').status, 'Regression') + assert.equal(compare(report(stable, stable.map(x => x * 1.3)), 'pacquet').status, 'Within tolerance') +}) + +test('multi-second scenarios catch consistent slowdowns below 20%', () => { + const result = compare(report(Array(10).fill(3.18), Array(10).fill(3)), 'pacquet') + assert.equal(result.status, 'Regression') + assert.equal(result.tolerance, 3 * 0.05) +}) + +test('short scenarios tolerate up to 2 ms but still catch larger slowdowns', () => { + const base = Array(10).fill(0.008) + assert.equal(compare(report(Array(10).fill(0.009), base), 'pacquet').status, 'Within tolerance') + assert.equal(compare(report(Array(10).fill(0.010), base), 'pacquet').status, 'Within tolerance') + const result = compare(report(Array(10).fill(0.011), base), 'pacquet') + assert.equal(result.status, 'Regression') + assert.equal(result.tolerance, 0.002) +}) + +test('exactly 5% is tolerated when the relative threshold is larger', () => { + assert.equal(compare(report(Array(10).fill(1.05), stable), 'pacquet').status, 'Within tolerance') +}) + +test('a noisy slowdown is explicitly inconclusive', () => { + assert.equal(compare(report([1, 1, 1.3, 1.3, 1.3, 1.3, 1.4, 1.4, 2, 3], stable), 'pacquet').status, 'Inconclusive (overlapping samples)') +}) + +test('isolated outliers neither create nor conceal a regression', () => { + assert.equal(compare(report([...stable.slice(1), 10], stable), 'pacquet').status, 'Within tolerance') + assert.equal(compare(report([0.5, ...Array(9).fill(1.3)], [...stable.slice(1), 10]), 'pacquet').status, 'Regression') +}) + +test('missing, duplicate, failed, undersampled and invalid data fail closed', () => { + const valid = report(stable, stable) + const invalid = [ + { results: [] }, + { results: [...valid.results, valid.results[0]] }, + report(stable.slice(0, 8), stable), + report([...stable.slice(1), NaN], stable), + report([...stable.slice(1), 0], stable), + { results: [{ ...valid.results[0], exit_codes: [0] }, valid.results[1]] }, + { results: [{ ...valid.results[0], exit_codes: [1, ...Array(9).fill(0)] }, valid.results[1]] }, + ] + for (const input of invalid) assert.throws(() => compare(input, 'pacquet')) +}) + +test('all scenarios and both engines are checked, and missing reports fail', async () => { + const directory = await mkdtemp(join(tmpdir(), 'benchmark-comparison-')) + try { + for (const scenario of scenarios) { + const input = report(stable, stable) + if (!scenario.includes('PEER_HEAVY') && !scenario.includes('LINKED_WORKSPACE')) { + input.results.push(command('pnpr@HEAD', Array(10).fill(1.4)), command('pnpr@main', stable)) + } + await writeFile(join(directory, `BENCHMARK_REPORT_${scenario}.json`), JSON.stringify(input)) + } + const result = await renderComparison(directory) + assert.equal(result.failed, true) + assert.equal(result.markdown.match(/\| Regression \|/g).length, 10) + assert.equal(result.markdown.match(/\| Within tolerance \|/g).length, 12) + const cli = new URL('./compare-integrated-benchmarks.mjs', import.meta.url) + const run = spawnSync(process.execPath, [fileURLToPath(cli), directory], { encoding: 'utf8' }) + assert.equal(run.status, 1) + assert.match(run.stdout, /Regression/) + assert.equal(run.stderr, '') + for (const scenario of scenarios) { + const input = { results: ['pacquet', 'pnpr'].flatMap(engine => [command(`${engine}@HEAD`, stable), command(`${engine}@main`, stable)]) } + await writeFile(join(directory, `BENCHMARK_REPORT_${scenario}.json`), JSON.stringify(input)) + } + assert.equal(spawnSync(process.execPath, [fileURLToPath(cli), directory]).status, 0) + await rm(join(directory, `BENCHMARK_REPORT_${scenarios[0]}.json`)) + await assert.rejects(renderComparison(directory), /ENOENT/) + } finally { + await rm(directory, { recursive: true, force: true }) + } +}) + +test('comparison and Bencher uploads cover every matrix scenario', async () => { + const workflow = await readFile(new URL('../workflows/pacquet-integrated-benchmark.yml', import.meta.url), 'utf8') + const tags = [...workflow.matchAll(/'([A-Z_]+):[a-z][^']+'/g)].map(match => match[1]) + const matrix = workflow.match(/ scenario:\n((?: - [^\n]+\n)+)/)[1] + const matrixTags = [...matrix.matchAll(/- ([a-z][\w.-]+)/g)].map(match => match[1] + .replace('-linker.', '_').replaceAll(/[.-]/g, '_').toUpperCase()) + assert.deepEqual([...new Set(tags)].sort(), [...matrixTags].sort()) + assert.deepEqual([...scenarios].sort(), [...matrixTags].sort()) +}) + +test('hyperfine command names identify targets when commands are shell scripts', () => { + const input = report(stable, stable) + input.results = input.results.map(result => ({ ...result, command_name: result.command, command: '/tmp/run.sh' })) + assert.equal(compare(input, 'pacquet').status, 'Within tolerance') +}) diff --git a/.github/workflows/pacquet-integrated-benchmark-comment.yml b/.github/workflows/pacquet-integrated-benchmark-comment.yml index 1290b2b450..336b7dd6c1 100644 --- a/.github/workflows/pacquet-integrated-benchmark-comment.yml +++ b/.github/workflows/pacquet-integrated-benchmark-comment.yml @@ -101,6 +101,15 @@ jobs: fi echo "pr=$pr" >> "$GITHUB_OUTPUT" + - name: Checkout trusted comparison script + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + # workflow_run's SHA belongs to the default branch, never the PR. + ref: ${{ github.sha }} + persist-credentials: false + sparse-checkout: .github/scripts + path: trusted-tools + - name: Download benchmark report # `run-id` + `github-token` lets v8 pull artifacts from a different # workflow run, which is exactly the workflow_run pattern. @@ -111,6 +120,25 @@ jobs: run-id: ${{ github.event.workflow_run.id }} github-token: ${{ secrets.GITHUB_TOKEN }} + - name: Compare HEAD with main from the same runner + id: comparison + continue-on-error: true + shell: bash + run: | + node trusted-tools/.github/scripts/compare-integrated-benchmarks.mjs benchmark-artifact > comparison.md + + - name: Add comparison to report + if: always() && (steps.comparison.outcome == 'success' || steps.comparison.outcome == 'failure') + shell: bash + run: | + if [ ! -s comparison.md ]; then + echo '### Same-run regression check' > comparison.md + echo 'Comparison failed because benchmark data was missing or invalid. See the workflow log.' >> comparison.md + fi + cat comparison.md >> "$GITHUB_STEP_SUMMARY" + cat comparison.md benchmark-artifact/SUMMARY.md > combined-summary.md + mv combined-summary.md benchmark-artifact/SUMMARY.md + - name: Find existing comment # Match on the literal substring the report header carries. # Updating the heading text in `integrated-benchmark.yml` (the @@ -141,9 +169,8 @@ jobs: # workflow_run runs in the base-repo privilege context, so # BENCHER_API_TOKEN is available even for fork PRs. PR number # comes from the trusted workflow_run payload (resolved in - # `meta` above), never from the artifact body. `--ci-number` - # is required here because Bencher can't infer the PR from a - # workflow_run event the way it does on pull_request. + # `meta` above), never from the artifact body. Historical results + # are retained without creating a separate historical-threshold check. # # One benchmark, two testbeds: `pacquet` (direct install) and # `pnpr` (the accelerated path). Each uploads from its own @@ -151,7 +178,6 @@ jobs: if: hashFiles('benchmark-artifact/bencher-results.json') != '' env: BENCHER_API_TOKEN: ${{ secrets.BENCHER_API_TOKEN }} - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} PR_NUMBER: ${{ steps.meta.outputs.pr }} shell: bash run: | @@ -159,9 +185,6 @@ jobs: echo "::notice::BENCHER_API_TOKEN not set, skipping Bencher upload" exit 0 fi - # `--start-point-clone-thresholds` so the PR branch inherits the - # threshold configured on main; `--err` so the action surfaces - # a regression on the PR check. upload() { local testbed=$1 file=$2 if [ ! -f "$file" ]; then @@ -175,11 +198,41 @@ jobs: --start-point main \ --start-point-reset \ --start-point-clone-thresholds \ - --err \ --adapter json \ - --file "$file" \ - --ci-number "$PR_NUMBER" \ - --github-actions "$GITHUB_TOKEN" + --file "$file" } upload pacquet benchmark-artifact/bencher-results.json upload pnpr benchmark-artifact/bencher-results-pnpr.json + + - name: Publish regression check on the PR commit + if: always() && steps.meta.outputs.pr != '' + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + HEAD_SHA: ${{ github.event.workflow_run.head_sha }} + REPO: ${{ github.repository }} + OUTCOME: ${{ steps.comparison.outcome }} + RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + shell: bash + run: | + conclusion=failure + if [ "$OUTCOME" = success ]; then + conclusion=success + fi + if [ ! -s comparison.md ]; then + echo 'Benchmark comparison could not complete. See the workflow log.' > comparison.md + fi + # Preserve the existing required check name while replacing its + # historical threshold with the same-run comparison. + jq -n --arg sha "$HEAD_SHA" --arg conclusion "$conclusion" \ + --arg url "$RUN_URL" --rawfile summary comparison.md \ + --arg name "Bencher Report (pnpm's project)" \ + '{name: $name, head_sha: $sha, status: "completed", conclusion: $conclusion, + details_url: $url, output: {title: "Same-run benchmark comparison", summary: $summary}}' \ + > check.json + gh api --method POST "repos/$REPO/check-runs" --input check.json + + - name: Enforce same-run regression check + if: always() && steps.comparison.outcome != 'success' + run: | + echo '::error::Same-run benchmark comparison failed. See the comparison report and workflow log.' + exit 1 diff --git a/.github/workflows/pacquet-integrated-benchmark.yml b/.github/workflows/pacquet-integrated-benchmark.yml index 358f7bfa59..9904886d74 100644 --- a/.github/workflows/pacquet-integrated-benchmark.yml +++ b/.github/workflows/pacquet-integrated-benchmark.yml @@ -28,6 +28,8 @@ on: - '.github/actions/binstall/**' - '.github/actions/install-crates/**' - '.github/workflows/pacquet-integrated-benchmark.yml' + - '.github/workflows/pacquet-integrated-benchmark-comment.yml' + - '.github/scripts/compare-integrated-benchmarks*.mjs' pull_request: types: [opened, synchronize] paths: @@ -46,6 +48,8 @@ on: - '.github/actions/binstall/**' - '.github/actions/install-crates/**' - '.github/workflows/pacquet-integrated-benchmark.yml' + - '.github/workflows/pacquet-integrated-benchmark-comment.yml' + - '.github/scripts/compare-integrated-benchmarks*.mjs' concurrency: group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }} @@ -679,6 +683,9 @@ jobs: ref: ${{ needs.revisions.outputs.head_sha }} persist-credentials: false + - name: Test benchmark comparison + run: node --test .github/scripts/compare-integrated-benchmarks.test.mjs + - name: Download scenario results uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: @@ -702,7 +709,7 @@ jobs: echo echo 'Regular scenarios report direct and pnpr installs. The peer-heavy resolver scenario compares current Rust, main Rust, and TypeScript pnpm; the linked-workspace scenario compares current Rust against main Rust. Bencher consumes pacquet@HEAD and pnpr@HEAD.' echo - echo 'The tables below show mean ± σ; Bencher thresholds on the **minimum** latency, which is far less perturbed by shared-runner contention (noise only adds time).' + echo 'The tables below show mean ± σ. Bencher tracks absolute minimum latency; PR regression checks compare HEAD and main samples from the same runner. Historical timing alerts do not gate PRs.' echo echo '### Scenario: Isolated linker: fresh restore, cold cache + cold store' echo @@ -858,19 +865,9 @@ jobs: # scenarios contribute both testbeds; the client-only peer-heavy case # contributes only `pacquet@HEAD`. # - # We report each scenario's *minimum* latency as the metric value, so - # Bencher's threshold tracks the min rather than the mean. On a shared - # CI runner, contention only ever *adds* time, so the minimum is the run - # least perturbed by a noisy neighbour and the truest signal of a real - # code regression — the mean drifts run-to-run with whatever else the - # machine is doing (the latency-free offline scenario is pure CPU/disk - # and swung ~30% between runs, tripping a spurious alert on a control - # run whose HEAD and main binaries were byte-identical). Because the - # minimum only needs one uncontended run, hyperfine's default sampling - # suffices for regular scenarios. The peer-heavy case is noisier and - # fixes its own count instead. The human-readable - # comment still reports mean ± σ; only the alerting signal moves to the - # min. + # Keep absolute minimum latency for historical continuity in Bencher. + # The PR gate uses same-run sample distributions in the comment + # workflow; historical timings vary with the machine running them. # # BMF rather than the `shell_hyperfine` adapter with a doctored `mean`: # this reports the min honestly as the value instead of relabelling it. @@ -969,7 +966,6 @@ jobs: if: github.event_name == 'push' || github.event_name == 'workflow_dispatch' env: BENCHER_API_TOKEN: ${{ secrets.BENCHER_API_TOKEN }} - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} REF_NAME: ${{ github.ref_name }} shell: bash run: | @@ -980,10 +976,8 @@ jobs: # One run is one benchmark, two testbeds: `pacquet` (direct # install, all scenarios) and `pnpr` (accelerated path, hot # scenarios). Upload each from its own bencher-results file. - # `--start-point-clone-thresholds` so the forked branch inherits - # the threshold configured on main; `--err` so the workflow fails - # when a sample breaches the upper boundary. Main pushes skip - # both — by then the regression has already landed. + # Historical alerts remain in Bencher. Only same-run comparisons + # gate branches, since absolute times vary between runners. upload() { local testbed=$1 file=$2 if [ ! -f "$file" ]; then @@ -995,7 +989,6 @@ jobs: --testbed "$testbed" --adapter json --file "$file" - --github-actions "$GITHUB_TOKEN" ) if [ "$REF_NAME" = "main" ]; then args+=(--branch main) @@ -1005,7 +998,6 @@ jobs: --start-point main --start-point-reset --start-point-clone-thresholds - --err ) fi bencher run "${args[@]}" @@ -1013,6 +1005,14 @@ jobs: upload pacquet bench-work-env/bencher-results.json upload pnpr bench-work-env/bencher-results-pnpr.json + - name: Check same-run regressions on dispatched branches + if: github.event_name == 'workflow_dispatch' && github.ref_name != 'main' + shell: bash + run: | + node .github/scripts/compare-integrated-benchmarks.mjs bench-work-env > comparison.md || status=$? + cat comparison.md >> "$GITHUB_STEP_SUMMARY" + exit "${status:-0}" + - name: Re-raise the peer-heavy gate # Deferred from the peer-heavy step so its cross-engine assertion # cannot abort the job before the run's data points are published. diff --git a/pnpm/CONTRIBUTING.md b/pnpm/CONTRIBUTING.md index d12b62e01e..7101ee5709 100644 --- a/pnpm/CONTRIBUTING.md +++ b/pnpm/CONTRIBUTING.md @@ -205,3 +205,40 @@ just integrated-benchmark --scenario=isolated-linker.fresh-restore.cold-cache.co # See more options just integrated-benchmark --help ``` + +### CI regression checks + +Each PR benchmark scenario measures HEAD and main on the same runner. The check +uses a tolerance of **5% of main's median or 2 ms, whichever is larger**. That is +2 ms for a 10 ms baseline and 150 ms for a 3 second baseline. + +A regression requires both the median slowdown and the gap between the fastest +retained HEAD sample and slowest retained main sample to exceed that tolerance. +The comparison trims `floor(sample count / 10)` samples from each tail and +requires at least nine successful samples per target. Missing or invalid data +fails the check. Both pacquet and pnpr are checked, except in the two client-only +resolver scenarios. + +Slowdowns above the tolerance without this sample separation are reported as +inconclusive and do not fail CI. This conservative noise guard is not a +statistical confidence interval; sequential measurements remain vulnerable to +runner drift. All timings remain available for performance review. + +Bencher retains absolute minimum timings for historical tracking. Historical +alerts do not gate PRs, since their baselines may come from faster machines. +The privileged comment workflow runs the comparator from the default branch, +treats PR artifacts only as data, posts the report before enforcing the result, +and preserves the `Bencher Report (pnpm's project)` check name. Non-main manual +dispatches use the same comparison. The peer-heavy Rust-versus-TypeScript +speedup assertion remains active. + +To replay a downloaded `integrated-benchmark-report-ubuntu-latest` artifact, +run these commands from the repository root: + +```sh +node .github/scripts/compare-integrated-benchmarks.mjs /path/to/artifact +node --test .github/scripts/compare-integrated-benchmarks.test.mjs +``` + +Adding a scenario requires updating the comparison script's scenario list and +the workflow. The tests check that both cover the benchmark matrix.