name: Benchmarks on: workflow_dispatch: inputs: pr_number: description: 'PR number to benchmark (works with fork PRs too)' required: false default: '' type: string runs: description: 'Number of benchmark runs per scenario' required: false default: '10' type: string warmup: description: 'Number of warmup runs before timing' required: false default: '1' type: string permissions: contents: read pull-requests: write jobs: benchmark: name: Run Benchmarks runs-on: ubuntu-latest timeout-minutes: 180 steps: - name: Checkout uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: fetch-depth: 0 - name: Checkout PR head if: inputs.pr_number != '' env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | PR_NUMBER="${{ inputs.pr_number }}" echo "Fetching PR #$PR_NUMBER head..." git fetch origin "refs/pull/${PR_NUMBER}/head:refs/remotes/origin/pr-${PR_NUMBER}" git checkout "origin/pr-${PR_NUMBER}" echo "Checked out PR #$PR_NUMBER at $(git rev-parse --short HEAD)" - name: Install pnpm uses: pnpm/action-setup@6e7bdbda5fe05107efc88b23b7ed00aa05f84ca0 with: standalone: true - name: Setup Node run: pnpm runtime -g set node 25.7.0 timeout-minutes: 2 - name: Install hyperfine run: | wget -q https://github.com/sharkdp/hyperfine/releases/download/v1.18.0/hyperfine_1.18.0_amd64.deb sudo dpkg -i hyperfine_1.18.0_amd64.deb - name: Install dependencies run: pnpm install timeout-minutes: 5 - name: Compile run: pnpm run compile - name: Run benchmarks id: bench continue-on-error: true run: | set -o pipefail ./benchmarks/bench.sh 2>&1 | tee bench-output.txt BENCH_DIR=$(grep "Temp directory kept at:" bench-output.txt | sed 's/Temp directory kept at: //') echo "bench_dir=$BENCH_DIR" >> "$GITHUB_OUTPUT" env: RUNS: ${{ inputs.runs }} WARMUP: ${{ inputs.warmup }} - name: Comment on PR if: steps.bench.outputs.bench_dir != '' env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | BENCH_DIR="${{ steps.bench.outputs.bench_dir }}" RESULTS_FILE="$BENCH_DIR/results.md" if [ ! -f "$RESULTS_FILE" ]; then echo "::warning::Results file not found at $RESULTS_FILE" exit 0 fi echo "--- Benchmark Results ---" cat "$RESULTS_FILE" echo "-------------------------" if [ -n "${{ inputs.pr_number }}" ]; then PR_NUMBER="${{ inputs.pr_number }}" else PR_NUMBER=$(gh pr list --head "${{ github.ref_name }}" --json number --jq '.[0].number' 2>/dev/null || echo "") fi if [ -z "$PR_NUMBER" ]; then echo "::notice::No open PR found for branch ${{ github.ref_name }}. Results printed above." exit 0 fi MARKER="" { echo "$MARKER" cat "$RESULTS_FILE" echo "" echo "_Run [${{ github.run_id }}](${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}) · ${{ inputs.runs }} runs per scenario · triggered by @${{ github.actor }}_" } > /tmp/comment-body.md COMMENT_ID=$(gh api "repos/${{ github.repository }}/issues/${PR_NUMBER}/comments" \ --jq "[.[] | select(.body | startswith(\"$MARKER\"))] | .[0].id // empty" 2>/dev/null || echo "") if [ -n "$COMMENT_ID" ]; then echo "Updating existing benchmark comment $COMMENT_ID on PR #$PR_NUMBER" gh api "repos/${{ github.repository }}/issues/comments/${COMMENT_ID}" \ -X PATCH \ -F "body=@/tmp/comment-body.md" else echo "Creating new benchmark comment on PR #$PR_NUMBER" gh pr comment "$PR_NUMBER" --body-file /tmp/comment-body.md fi