pnpm/.github/workflows/benchmark.yml

name: Benchmarks

on:
  workflow_dispatch:
    inputs:
      pr_number:
        description: 'PR number to benchmark (works with fork PRs too)'
        required: false
        default: ''
        type: string
      runs:
        description: 'Number of benchmark runs per scenario'
        required: false
        default: '10'
        type: string
      warmup:
        description: 'Number of warmup runs before timing'
        required: false
        default: '1'
        type: string
  push:
    # Build Bencher's continuous baseline for the `pnpm` testbed.
    # Every merge to main that touches the TypeScript stack re-runs the
    # bench so PR comparisons have an up-to-date reference; cancel-in-progress
    # stays off below so we never throw away a partial run.
    branches: [main]
    paths:
      - 'pnpm11/**'
      - 'package.json'
      - 'pnpm-workspace.yaml'
      - '.config/pnpm/**'
      - 'pnpm-lock.yaml'
      - '.github/workflows/benchmark.yml'

permissions:
  contents: read
  pull-requests: write
  checks: write

# Don't cancel-in-progress — killing a bench mid-run wastes a long
# CI job and produces no usable data.
concurrency:
  group: benchmark-${{ inputs.pr_number || github.ref }}
  cancel-in-progress: false

jobs:
  benchmark:
    name: Run Benchmarks
    runs-on: blacksmith-8vcpu-ubuntu-2404
    timeout-minutes: 180

    steps:
    - name: Checkout
      uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
      with:
        fetch-depth: 0
        persist-credentials: false

    - name: Checkout PR head
      if: inputs.pr_number != ''
      env:
        GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
        PR_NUMBER: ${{ inputs.pr_number }}
      run: |
        echo "Fetching PR #$PR_NUMBER head..."
        git fetch origin "refs/pull/${PR_NUMBER}/head:refs/remotes/origin/pr-${PR_NUMBER}"
        git checkout "origin/pr-${PR_NUMBER}"
        echo "Checked out PR #$PR_NUMBER at $(git rev-parse --short HEAD)"

    - name: Install Rust Toolchain
      uses: $/.github/actions/rustup
      with:
        shared-key: pnpm-benchmark

    - name: Install pnpm and Node
      uses: pnpm/setup@a0881977cd8ea73b1ed212236c99ca10dcc80cf4 # v3.1.1
      with:
        runtime: node@26.12.1

    - name: Install hyperfine
      uses: $/.github/actions/binstall
      with:
        packages: hyperfine@1.18.0

    - name: Run benchmarks
      id: bench
      run: |
        set -o pipefail
        ./pnpm11/benchmarks/bench.sh 2>&1 | tee bench-output.txt
        BENCH_DIR=$(grep "Temp directory kept at:" bench-output.txt | sed 's/Temp directory kept at: //')
        echo "bench_dir=$BENCH_DIR" >> "$GITHUB_OUTPUT"
      env:
        RUNS: ${{ inputs.runs }}
        WARMUP: ${{ inputs.warmup }}

    - name: Install Bencher CLI
      if: steps.bench.outputs.bench_dir != ''
      uses: bencherdev/bencher@f80847436d7157936b40525db4c2a1064124f04a # v0.6.13

    - name: Upload results to Bencher
      if: steps.bench.outputs.bench_dir != ''
      env:
        BENCHER_API_TOKEN: ${{ secrets.BENCHER_API_TOKEN }}
        BENCH_DIR: ${{ steps.bench.outputs.bench_dir }}
        EVENT_NAME: ${{ github.event_name }}
        INPUT_PR_NUMBER: ${{ inputs.pr_number }}
        REF_NAME: ${{ github.ref_name }}
        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
      run: |
        if [ -z "${BENCHER_API_TOKEN:-}" ]; then
          echo "::notice::BENCHER_API_TOKEN not set, skipping Bencher upload"
          exit 0
        fi
        if [ ! -f "$BENCH_DIR/bencher-results.json" ]; then
          echo "::warning::bencher-results.json not found, skipping upload"
          exit 0
        fi

        # `bencher run --file` takes hyperfine JSON via the
        # shell_hyperfine adapter. Branch policy:
        # - push to main → record into the `main` branch (baseline)
        # - workflow_dispatch with pr_number → record into `pr/<n>`,
        #   forked from main at the latest baseline
        # - workflow_dispatch without pr_number → record into the
        #   ref's branch name (e.g. a feature branch), forked from main
        args=(
          --project pnpm
          --testbed pnpm
          --adapter shell_hyperfine
          --file "$BENCH_DIR/bencher-results.json"
          --github-actions "$GITHUB_TOKEN"
        )
        # `--start-point-clone-thresholds` so the forked branch inherits
        # the threshold configured on main; `--err` so the workflow fails
        # when a sample breaches the upper boundary. Main pushes skip
        # both — by then the regression has already landed.
        if [ "$EVENT_NAME" = "push" ] || [ "$REF_NAME" = "main" ]; then
          args+=(--branch main)
        elif [ -n "$INPUT_PR_NUMBER" ]; then
          args+=(
            --branch "pr/$INPUT_PR_NUMBER"
            --start-point main
            --start-point-reset
            --start-point-clone-thresholds
            --err
          )
        else
          args+=(
            --branch "$REF_NAME"
            --start-point main
            --start-point-reset
            --start-point-clone-thresholds
            --err
          )
        fi
        bencher run "${args[@]}"

    - name: Comment on PR
      if: steps.bench.outputs.bench_dir != ''
      env:
        GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
        BENCH_DIR: ${{ steps.bench.outputs.bench_dir }}
        INPUT_PR_NUMBER: ${{ inputs.pr_number }}
        REF_NAME: ${{ github.ref_name }}
        RUN_ID: ${{ github.run_id }}
        SERVER_URL: ${{ github.server_url }}
        REPO: ${{ github.repository }}
        RUNS: ${{ inputs.runs }}
        ACTOR: ${{ github.actor }}
      run: |
        RESULTS_FILE="$BENCH_DIR/results.md"

        if [ ! -f "$RESULTS_FILE" ]; then
          echo "::warning::Results file not found at $RESULTS_FILE"
          exit 0
        fi

        echo "--- Benchmark Results ---"
        cat "$RESULTS_FILE"
        echo "-------------------------"

        if [ -n "$INPUT_PR_NUMBER" ]; then
          PR_NUMBER="$INPUT_PR_NUMBER"
        else
          PR_NUMBER=$(gh pr list --head "$REF_NAME" --json number --jq '.[0].number' 2>/dev/null || echo "")
        fi

        if [ -z "$PR_NUMBER" ]; then
          echo "::notice::No open PR found for branch $REF_NAME. Results printed above."
          exit 0
        fi

        MARKER="<!-- pnpm-benchmark-results -->"
        {
          echo "$MARKER"
          cat "$RESULTS_FILE"
          echo ""
          echo "_Run [${RUN_ID}](${SERVER_URL}/${REPO}/actions/runs/${RUN_ID}) · ${RUNS} runs per scenario · triggered by @${ACTOR}_"
        } > /tmp/comment-body.md

        COMMENT_ID=$(gh api "repos/${REPO}/issues/${PR_NUMBER}/comments" \
          --jq "[.[] | select(.body | startswith(\"$MARKER\"))] | .[0].id // empty" 2>/dev/null || echo "")

        if [ -n "$COMMENT_ID" ]; then
          echo "Updating existing benchmark comment $COMMENT_ID on PR #$PR_NUMBER"
          gh api "repos/${REPO}/issues/comments/${COMMENT_ID}" \
            -X PATCH \
            -F "body=@/tmp/comment-body.md"
        else
          echo "Creating new benchmark comment on PR #$PR_NUMBER"
          gh pr comment "$PR_NUMBER" --body-file /tmp/comment-body.md
        fi