Zoltan Kochan
28bb2a339b
zizmor's self-repository audit flags every `uses: ./.github/...` reference now that GitHub has a dedicated `$/` form. The `$/` form resolves against the workflow's own commit rather than the runtime checkout, so it cannot load an action cloned into the workspace at runtime, and GitHub treats it as pinned. Rewrites all 22 references (18 local actions, 4 reusable workflow calls). With the `$/` form the runner downloads the whole repository as an action archive at job setup, and that download fails on any broken symlink in the tree. The four broken symlinks were all test fixtures: the directory-fetcher and cafs tests now copy their fixture into a temp dir and create the broken symlink there, and the has-not-outdated-deps fixture drops two dangling node_modules links that `pnpm outdated` never followed. pnpm's GitHub Actions dependency discovery only followed `./` references into local actions and reusable workflows; both stacks now follow `$/` too.
212 lines
7.0 KiB
YAML
212 lines
7.0 KiB
YAML
name: Benchmarks
|
|
|
|
on:
|
|
workflow_dispatch:
|
|
inputs:
|
|
pr_number:
|
|
description: 'PR number to benchmark (works with fork PRs too)'
|
|
required: false
|
|
default: ''
|
|
type: string
|
|
runs:
|
|
description: 'Number of benchmark runs per scenario'
|
|
required: false
|
|
default: '10'
|
|
type: string
|
|
warmup:
|
|
description: 'Number of warmup runs before timing'
|
|
required: false
|
|
default: '1'
|
|
type: string
|
|
push:
|
|
# Build Bencher's continuous baseline for the `pnpm` testbed.
|
|
# Every merge to main that touches the TypeScript stack re-runs the
|
|
# bench so PR comparisons have an up-to-date reference; cancel-in-progress
|
|
# stays off below so we never throw away a partial run.
|
|
branches: [main]
|
|
paths:
|
|
- 'pnpm11/**'
|
|
- 'package.json'
|
|
- 'pnpm-workspace.yaml'
|
|
- 'pnpm-lock.yaml'
|
|
- '.github/workflows/benchmark.yml'
|
|
|
|
permissions:
|
|
contents: read
|
|
pull-requests: write
|
|
checks: write
|
|
|
|
# Don't cancel-in-progress — killing a bench mid-run wastes a long
|
|
# CI job and produces no usable data.
|
|
concurrency:
|
|
group: benchmark-${{ inputs.pr_number || github.ref }}
|
|
cancel-in-progress: false
|
|
|
|
jobs:
|
|
benchmark:
|
|
name: Run Benchmarks
|
|
runs-on: blacksmith-8vcpu-ubuntu-2404
|
|
timeout-minutes: 180
|
|
|
|
steps:
|
|
- name: Checkout
|
|
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
|
with:
|
|
fetch-depth: 0
|
|
persist-credentials: false
|
|
|
|
- name: Checkout PR head
|
|
if: inputs.pr_number != ''
|
|
env:
|
|
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
|
PR_NUMBER: ${{ inputs.pr_number }}
|
|
run: |
|
|
echo "Fetching PR #$PR_NUMBER head..."
|
|
git fetch origin "refs/pull/${PR_NUMBER}/head:refs/remotes/origin/pr-${PR_NUMBER}"
|
|
git checkout "origin/pr-${PR_NUMBER}"
|
|
echo "Checked out PR #$PR_NUMBER at $(git rev-parse --short HEAD)"
|
|
|
|
- name: Install pnpm and Node
|
|
uses: pnpm/setup@703c52620218391530e48b9e8870d5c0082e1b9b # v2.1.0
|
|
with:
|
|
runtime: node@26.8.1
|
|
|
|
- name: Install Rust Toolchain
|
|
uses: $/.github/actions/rustup
|
|
with:
|
|
shared-key: pnpm-benchmark
|
|
|
|
- name: Install hyperfine
|
|
uses: $/.github/actions/binstall
|
|
with:
|
|
packages: hyperfine@1.18.0
|
|
|
|
- name: Run benchmarks
|
|
id: bench
|
|
run: |
|
|
set -o pipefail
|
|
./pnpm11/benchmarks/bench.sh 2>&1 | tee bench-output.txt
|
|
BENCH_DIR=$(grep "Temp directory kept at:" bench-output.txt | sed 's/Temp directory kept at: //')
|
|
echo "bench_dir=$BENCH_DIR" >> "$GITHUB_OUTPUT"
|
|
env:
|
|
RUNS: ${{ inputs.runs }}
|
|
WARMUP: ${{ inputs.warmup }}
|
|
|
|
- name: Install Bencher CLI
|
|
if: steps.bench.outputs.bench_dir != ''
|
|
uses: bencherdev/bencher@8d75325c3bc59403a2186a056b472c4f49d42838 # v0.6.12
|
|
|
|
- name: Upload results to Bencher
|
|
if: steps.bench.outputs.bench_dir != ''
|
|
env:
|
|
BENCHER_API_TOKEN: ${{ secrets.BENCHER_API_TOKEN }}
|
|
BENCH_DIR: ${{ steps.bench.outputs.bench_dir }}
|
|
EVENT_NAME: ${{ github.event_name }}
|
|
INPUT_PR_NUMBER: ${{ inputs.pr_number }}
|
|
REF_NAME: ${{ github.ref_name }}
|
|
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
|
run: |
|
|
if [ -z "${BENCHER_API_TOKEN:-}" ]; then
|
|
echo "::notice::BENCHER_API_TOKEN not set, skipping Bencher upload"
|
|
exit 0
|
|
fi
|
|
if [ ! -f "$BENCH_DIR/bencher-results.json" ]; then
|
|
echo "::warning::bencher-results.json not found, skipping upload"
|
|
exit 0
|
|
fi
|
|
|
|
# `bencher run --file` takes hyperfine JSON via the
|
|
# shell_hyperfine adapter. Branch policy:
|
|
# - push to main → record into the `main` branch (baseline)
|
|
# - workflow_dispatch with pr_number → record into `pr/<n>`,
|
|
# forked from main at the latest baseline
|
|
# - workflow_dispatch without pr_number → record into the
|
|
# ref's branch name (e.g. a feature branch), forked from main
|
|
args=(
|
|
--project pnpm
|
|
--testbed pnpm
|
|
--adapter shell_hyperfine
|
|
--file "$BENCH_DIR/bencher-results.json"
|
|
--github-actions "$GITHUB_TOKEN"
|
|
)
|
|
# `--start-point-clone-thresholds` so the forked branch inherits
|
|
# the threshold configured on main; `--err` so the workflow fails
|
|
# when a sample breaches the upper boundary. Main pushes skip
|
|
# both — by then the regression has already landed.
|
|
if [ "$EVENT_NAME" = "push" ] || [ "$REF_NAME" = "main" ]; then
|
|
args+=(--branch main)
|
|
elif [ -n "$INPUT_PR_NUMBER" ]; then
|
|
args+=(
|
|
--branch "pr/$INPUT_PR_NUMBER"
|
|
--start-point main
|
|
--start-point-reset
|
|
--start-point-clone-thresholds
|
|
--err
|
|
)
|
|
else
|
|
args+=(
|
|
--branch "$REF_NAME"
|
|
--start-point main
|
|
--start-point-reset
|
|
--start-point-clone-thresholds
|
|
--err
|
|
)
|
|
fi
|
|
bencher run "${args[@]}"
|
|
|
|
- name: Comment on PR
|
|
if: steps.bench.outputs.bench_dir != ''
|
|
env:
|
|
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
|
BENCH_DIR: ${{ steps.bench.outputs.bench_dir }}
|
|
INPUT_PR_NUMBER: ${{ inputs.pr_number }}
|
|
REF_NAME: ${{ github.ref_name }}
|
|
RUN_ID: ${{ github.run_id }}
|
|
SERVER_URL: ${{ github.server_url }}
|
|
REPO: ${{ github.repository }}
|
|
RUNS: ${{ inputs.runs }}
|
|
ACTOR: ${{ github.actor }}
|
|
run: |
|
|
RESULTS_FILE="$BENCH_DIR/results.md"
|
|
|
|
if [ ! -f "$RESULTS_FILE" ]; then
|
|
echo "::warning::Results file not found at $RESULTS_FILE"
|
|
exit 0
|
|
fi
|
|
|
|
echo "--- Benchmark Results ---"
|
|
cat "$RESULTS_FILE"
|
|
echo "-------------------------"
|
|
|
|
if [ -n "$INPUT_PR_NUMBER" ]; then
|
|
PR_NUMBER="$INPUT_PR_NUMBER"
|
|
else
|
|
PR_NUMBER=$(gh pr list --head "$REF_NAME" --json number --jq '.[0].number' 2>/dev/null || echo "")
|
|
fi
|
|
|
|
if [ -z "$PR_NUMBER" ]; then
|
|
echo "::notice::No open PR found for branch $REF_NAME. Results printed above."
|
|
exit 0
|
|
fi
|
|
|
|
MARKER="<!-- pnpm-benchmark-results -->"
|
|
{
|
|
echo "$MARKER"
|
|
cat "$RESULTS_FILE"
|
|
echo ""
|
|
echo "_Run [${RUN_ID}](${SERVER_URL}/${REPO}/actions/runs/${RUN_ID}) · ${RUNS} runs per scenario · triggered by @${ACTOR}_"
|
|
} > /tmp/comment-body.md
|
|
|
|
COMMENT_ID=$(gh api "repos/${REPO}/issues/${PR_NUMBER}/comments" \
|
|
--jq "[.[] | select(.body | startswith(\"$MARKER\"))] | .[0].id // empty" 2>/dev/null || echo "")
|
|
|
|
if [ -n "$COMMENT_ID" ]; then
|
|
echo "Updating existing benchmark comment $COMMENT_ID on PR #$PR_NUMBER"
|
|
gh api "repos/${REPO}/issues/comments/${COMMENT_ID}" \
|
|
-X PATCH \
|
|
-F "body=@/tmp/comment-body.md"
|
|
else
|
|
echo "Creating new benchmark comment on PR #$PR_NUMBER"
|
|
gh pr comment "$PR_NUMBER" --body-file /tmp/comment-body.md
|
|
fi
|