pnpm/benchmarks/bench.sh

#!/bin/bash
set -euo pipefail

# Thin wrapper around `pacquet/tasks/integrated-benchmark`. Builds two
# pnpm revisions (the current branch and `main`) and runs hyperfine for
# each of the six scenarios that used to live in this script.
#
# Scenarios, registry choice, and runner behaviour are preserved exactly
# as before; the orchestration logic is shared with the pacquet bench.
#
# Prerequisites: cargo, hyperfine, pnpm, node, git.
#
# Env vars: WARMUP (default 1), RUNS (default 10).
#
# Usage: ./benchmarks/bench.sh

REPO_ROOT="$(cd "$(dirname "$0")/.." && pwd)"
FIXTURE_DIR="$REPO_ROOT/benchmarks/fixture"
WARMUP="${WARMUP:-1}"
RUNS="${RUNS:-10}"
BENCH_DIR="$(mktemp -d "${TMPDIR:-/tmp}/pnpm-bench.XXXXXX")"

for tool in cargo hyperfine pnpm node git; do
  if ! command -v "$tool" >/dev/null; then
    echo "error: $tool not on PATH" >&2
    exit 1
  fi
done

echo "── Building integrated-benchmark ──"
cargo build --release --bin=integrated-benchmark --manifest-path "$REPO_ROOT/Cargo.toml"
BIN="$REPO_ROOT/target/release/integrated-benchmark"

# Ensure `pnpm@main` resolves locally. `actions/checkout` only creates a
# local ref for the branch it checked out; on a workflow_dispatch run from
# a non-main branch (or after the optional PR-head checkout in
# `benchmark.yml`) there's no `refs/heads/main` for `git rev-parse` to
# hit. Skip the fetch entirely when the local ref already exists, and
# let the fetch surface its real error if it fails.
if ! git -C "$REPO_ROOT" rev-parse --verify --quiet refs/heads/main >/dev/null; then
  echo "── Fetching main into local ref ──"
  git -C "$REPO_ROOT" fetch --no-tags origin main:main
fi

# Scenario list: `slug:Display label`. The slug matches the
# orchestrator's `--scenario` value (the clap-derived kebab-case name
# from `BenchmarkScenario`). All six start with `node_modules` wiped
# — "Fresh" names that target state. "Isolated linker" names the
# `nodeLinker` mode; alternatives (`hoisted`, `pnp`) and populated-
# node_modules counterparts are reserved for future scenarios.
SCENARIOS=(
  "isolated-linker.fresh-restore.hot-cache.hot-store:Isolated linker: fresh restore, hot cache + hot store"
  "isolated-linker.fresh-add-dep.hot-cache.hot-store:Isolated linker: fresh add new dep, hot cache + hot store"
  "isolated-linker.fresh-install.hot-cache.hot-store:Isolated linker: fresh install, hot cache + hot store"
  "isolated-linker.fresh-restore.cold-cache.cold-store:Isolated linker: fresh restore, cold cache + cold store"
  "isolated-linker.fresh-install.cold-cache.cold-store:Isolated linker: fresh install, cold cache + cold store"
  "gvs-linker.fresh-restore.hot-cache.hot-store:GVS linker: fresh restore, hot cache + hot store"
)

# Pre-build both revisions once. Subsequent scenario invocations still
# re-enter the orchestrator's build step (sync_bench_repo + pnpm install
# + pnpm run compile), but `pnpm install` is a no-op on the populated
# node_modules and `tsgo --build` is incremental. `pnpm run bundle`
# (which produces pnpm/dist/pnpm.mjs) does run each time and is not
# incremental — accepted overhead in exchange for keeping the build
# path in one consistent place across pacquet and pnpm benches.
echo "── Pre-building pnpm revisions ──"
"$BIN" \
  --pnpm-repository "$REPO_ROOT" \
  --work-env "$BENCH_DIR/work-env" \
  --build-only \
  pnpm@HEAD pnpm@main

# Pull mean ± stddev for each variant out of a hyperfine JSON into one
# table cell. Falls back to "n/a" if jq isn't on PATH, the file is
# missing, or the target isn't present in the JSON.
read_cell() {
  local target=$1
  local json=$2
  if ! command -v jq >/dev/null; then
    echo "n/a"
    return
  fi
  jq -r --arg t "$target" '
    [.results[] | select(.command == $t)
      | "\((.mean*1000|round)/1000)s ± \((.stddev*1000|round)/1000)s"]
    | first // "n/a"
  ' "$json" 2>/dev/null || echo "n/a"
}

results_md="$BENCH_DIR/results.md"
{
  echo "# Benchmark Results"
  echo
  echo "| # | Scenario | main | HEAD |"
  echo "|---|---|---|---|"
} > "$results_md"

i=1
for entry in "${SCENARIOS[@]}"; do
  scenario="${entry%%:*}"
  label="${entry#*:}"

  echo ""
  echo "━━━ Benchmark $i: $label ━━━"

  "$BIN" \
    --scenario "$scenario" \
    --registry npm \
    --pnpm-repository "$REPO_ROOT" \
    --fixture-dir "$FIXTURE_DIR" \
    --work-env "$BENCH_DIR/work-env" \
    --warmup "$WARMUP" \
    --runs "$RUNS" \
    --ignore-failure \
    pnpm@main pnpm@HEAD

  cp "$BENCH_DIR/work-env/BENCHMARK_REPORT.md"   "$BENCH_DIR/${scenario}.md"
  cp "$BENCH_DIR/work-env/BENCHMARK_REPORT.json" "$BENCH_DIR/${scenario}.json"

  main_cell=$(read_cell "pnpm@main" "$BENCH_DIR/${scenario}.json")
  head_cell=$(read_cell "pnpm@HEAD" "$BENCH_DIR/${scenario}.json")
  echo "| $i | $label | $main_cell | $head_cell |" >> "$results_md"

  i=$((i + 1))
done

# Combine the per-scenario hyperfine JSONs into one Bencher-shaped
# report. Keep only the @HEAD result from each scenario and rename
# `.command` to the scenario name so Bencher's shell_hyperfine adapter
# names the benchmark after the scenario instead of `pnpm@HEAD`.
if command -v jq >/dev/null; then
  bencher_inputs=()
  for entry in "${SCENARIOS[@]}"; do
    scenario="${entry%%:*}"
    jq --arg s "$scenario" \
      '.results |= [.[] | select(.command == "pnpm@HEAD") | .command = $s]' \
      "$BENCH_DIR/${scenario}.json" > "$BENCH_DIR/${scenario}-bencher.json"
    bencher_inputs+=("$BENCH_DIR/${scenario}-bencher.json")
  done
  jq -s '{results: map(.results) | add}' \
    "${bencher_inputs[@]}" > "$BENCH_DIR/bencher-results.json"
else
  echo "warning: jq not on PATH; skipping bencher-results.json generation" >&2
fi

echo
echo "━━━ Results ━━━"
cat "$results_md"
echo
echo "Results saved to: $results_md"
echo "Temp directory kept at: $BENCH_DIR"
echo "Remove with: rm -rf $BENCH_DIR"