diff --git a/.github/scripts/bench-output-path.mjs b/.github/scripts/bench-output-path.mjs new file mode 100644 index 0000000000..49a2c4b4e2 --- /dev/null +++ b/.github/scripts/bench-output-path.mjs @@ -0,0 +1,67 @@ +import { existsSync, lstatSync, mkdirSync, realpathSync } from 'node:fs' +import { dirname, isAbsolute, relative, resolve, sep } from 'node:path' + +export function resolveBenchOutputPath (output) { + const benchDir = resolve('.bench') + const outputPath = resolve(output) + const relativePath = relative(benchDir, outputPath) + if (relativePath === '' || isOutside(relativePath)) { + throw new Error(`Output path must be under .bench/: ${output}`) + } + ensureBenchDir(benchDir, output) + assertNotSymlink(outputPath, output) + assertNoSymlinkAncestors(benchDir, dirname(outputPath), output) + + const canonicalBenchDir = realpathSync(benchDir) + const canonicalParent = realpathSync(nearestExistingAncestor(dirname(outputPath))) + if (!isSameOrChild(canonicalBenchDir, canonicalParent)) { + throw new Error(`Output path must be under .bench/: ${output}`) + } + return outputPath +} + +function ensureBenchDir (benchDir, output) { + if (!existsSync(benchDir)) { + mkdirSync(benchDir) + return + } + const stats = lstatSync(benchDir) + if (stats.isSymbolicLink() || !stats.isDirectory()) { + throw new Error(`Output path must be under .bench/: ${output}`) + } +} + +function assertNoSymlinkAncestors (benchDir, outputParent, output) { + const relativeParent = relative(benchDir, outputParent) + if (relativeParent === '') return + + let current = benchDir + for (const segment of relativeParent.split(sep)) { + current = resolve(current, segment) + assertNotSymlink(current, output) + } +} + +function assertNotSymlink (path, output) { + if (existsSync(path) && lstatSync(path).isSymbolicLink()) { + throw new Error(`Output path must be under .bench/: ${output}`) + } +} + +function nearestExistingAncestor (path) { + let current = path + while (!existsSync(current)) { + const parent = dirname(current) + if (parent === current) return current + current = parent + } + return current +} + +function isSameOrChild (base, target) { + return !isOutside(relative(base, target)) +} + +function isOutside (relativePath) { + return relativePath === '..' || relativePath.startsWith(`..${sep}`) || isAbsolute(relativePath) +} diff --git a/.github/scripts/bencher-result-from-pnpm-summary.mjs b/.github/scripts/bencher-result-from-pnpm-summary.mjs new file mode 100644 index 0000000000..bf814acb3e --- /dev/null +++ b/.github/scripts/bencher-result-from-pnpm-summary.mjs @@ -0,0 +1,78 @@ +#!/usr/bin/env node +import { mkdir, readFile, writeFile } from 'node:fs/promises' +import { dirname, resolve } from 'node:path' +import { resolveBenchOutputPath } from './bench-output-path.mjs' + +const { name, output, packageDir, summary } = parseArgs(process.argv.slice(2)) +const summaryPath = resolve(summary) +const outputPath = resolveBenchOutputPath(output) +const packagePath = resolve(packageDir) +const { executionStatus } = JSON.parse(await readFile(summaryPath, 'utf8')) + +const entry = executionStatus?.[packagePath] +if (entry == null) { + console.error(`No execution summary entry found for ${packagePath}`) + process.exit(1) +} +if (entry.status !== 'passed') { + console.error(`Execution summary entry for ${packagePath} has status ${entry.status}`) + process.exit(1) +} +if (typeof entry.duration !== 'number') { + console.error(`Execution summary entry for ${packagePath} does not include a duration`) + process.exit(1) +} + +const durationSeconds = entry.duration / 1000 + +await mkdir(dirname(outputPath), { recursive: true }) +await writeFile(outputPath, JSON.stringify({ + results: [ + { + command: name, + mean: durationSeconds, + stddev: 0, + median: durationSeconds, + user: 0, + system: 0, + min: durationSeconds, + max: durationSeconds, + times: [durationSeconds], + exit_codes: [0], + }, + ], +}, null, 2) + '\n') + +function parseArgs (args) { + let name + let output + let packageDir + let summary = 'pnpm-exec-summary.json' + + for (let i = 0; i < args.length; i++) { + const arg = args[i] + if (arg === '--name') { + name = args[++i] + } else if (arg === '--output') { + output = args[++i] + } else if (arg === '--package-dir') { + packageDir = args[++i] + } else if (arg === '--summary') { + summary = args[++i] + } else { + usage(`unknown argument: ${arg}`) + } + } + + if (!name) usage('missing --name') + if (!output) usage('missing --output') + if (!packageDir) usage('missing --package-dir') + + return { name, output, packageDir, summary } +} + +function usage (message) { + console.error(message) + console.error('Usage: bencher-result-from-pnpm-summary.mjs --name --output --package-dir [--summary ]') + process.exit(1) +} diff --git a/.github/scripts/measure-command.mjs b/.github/scripts/measure-command.mjs new file mode 100644 index 0000000000..d813e37468 --- /dev/null +++ b/.github/scripts/measure-command.mjs @@ -0,0 +1,98 @@ +#!/usr/bin/env node +import { mkdir, writeFile } from 'node:fs/promises' +import { dirname } from 'node:path' +import { spawn } from 'node:child_process' +import { performance } from 'node:perf_hooks' +import { resolveBenchOutputPath } from './bench-output-path.mjs' + +const { name, output, command } = parseArgs(process.argv.slice(2)) +const outputPath = resolveBenchOutputPath(output) + +await mkdir(dirname(outputPath), { recursive: true }) + +const startedAt = performance.now() +const exitCode = await runCommand(command) +const durationSeconds = (performance.now() - startedAt) / 1000 + +await writeFile(outputPath, JSON.stringify({ + results: [ + { + command: name, + mean: durationSeconds, + stddev: 0, + median: durationSeconds, + user: 0, + system: 0, + min: durationSeconds, + max: durationSeconds, + times: [durationSeconds], + exit_codes: [exitCode], + }, + ], +}, null, 2) + '\n') + +process.exitCode = exitCode + +function parseArgs (args) { + let name + let output + const commandIndex = args.indexOf('--') + + if (commandIndex === -1) { + usage('missing command separator: --') + } + + for (let i = 0; i < commandIndex; i++) { + const arg = args[i] + if (arg === '--name') { + name = args[++i] + } else if (arg === '--output') { + output = args[++i] + } else { + usage(`unknown argument: ${arg}`) + } + } + + const command = args.slice(commandIndex + 1) + if (!name) usage('missing --name') + if (!output) usage('missing --output') + if (command.length === 0) usage('missing command') + + return { name, output, command } +} + +function usage (message) { + console.error(message) + console.error('Usage: measure-command.mjs --name --output -- [args...]') + process.exit(1) +} + +function runCommand ([command, ...args]) { + return new Promise((resolve) => { + const shell = process.platform === 'win32' + if (shell) { + validateWindowsShellArgs([command, ...args]) + } + const child = spawn(command, args, { shell, stdio: 'inherit' }) + child.on('error', (err) => { + console.error(err) + resolve(1) + }) + child.on('close', (code, signal) => { + if (signal) { + console.error(`Command terminated by signal ${signal}`) + resolve(1) + } else { + resolve(code ?? 1) + } + }) + }) +} + +function validateWindowsShellArgs (args) { + for (const arg of args) { + if (/[&|<>^%\r\n]/.test(arg)) { + throw new Error(`Cannot run command with Windows shell metacharacters: ${arg}`) + } + } +} diff --git a/.github/scripts/merge-bencher-results.mjs b/.github/scripts/merge-bencher-results.mjs new file mode 100644 index 0000000000..636b230f91 --- /dev/null +++ b/.github/scripts/merge-bencher-results.mjs @@ -0,0 +1,53 @@ +#!/usr/bin/env node +import { mkdir, readFile, writeFile } from 'node:fs/promises' +import { dirname } from 'node:path' +import { resolveBenchOutputPath } from './bench-output-path.mjs' + +const { output, files } = parseArgs(process.argv.slice(2)) +const outputPath = resolveBenchOutputPath(output) +const results = [] + +for (const file of files) { + const report = JSON.parse(await readFile(file, 'utf8')) + if (Array.isArray(report.results)) { + results.push(...report.results) + } +} + +if (results.length === 0) { + console.error('No Bencher results found to merge') + process.exit(1) +} + +await mkdir(dirname(outputPath), { recursive: true }) +await writeFile(outputPath, JSON.stringify({ results }, null, 2) + '\n') + +function parseArgs (args) { + let output + const filesIndex = args.indexOf('--') + + if (filesIndex === -1) { + usage('missing file separator: --') + } + + for (let i = 0; i < filesIndex; i++) { + const arg = args[i] + if (arg === '--output') { + output = args[++i] + } else { + usage(`unknown argument: ${arg}`) + } + } + + const files = args.slice(filesIndex + 1) + if (!output) usage('missing --output') + if (files.length === 0) usage('missing files') + + return { output, files } +} + +function usage (message) { + console.error(message) + console.error('Usage: merge-bencher-results.mjs --output -- [file...]') + process.exit(1) +} diff --git a/.github/workflows/ci-performance-bencher-upload.yml b/.github/workflows/ci-performance-bencher-upload.yml new file mode 100644 index 0000000000..ddf84b2d28 --- /dev/null +++ b/.github/workflows/ci-performance-bencher-upload.yml @@ -0,0 +1,223 @@ +name: CI Performance Bencher Upload + +on: + # zizmor: ignore[dangerous-triggers] + # The CI jobs produce untrusted measurement artifacts without access to + # BENCHER_API_TOKEN. This workflow runs from the default branch via + # workflow_run, validates artifact metadata, then performs the authenticated + # upload from trusted workflow code. + workflow_run: + workflows: ["CI", "Pacquet CI"] + types: [completed] + +permissions: + contents: read + actions: read + +jobs: + upload: + name: Upload CI performance results + runs-on: ubuntu-latest + if: >- + github.event.workflow_run.conclusion == 'success' && + github.event.workflow_run.head_repository.full_name == github.repository + steps: + - name: Resolve Bencher branch + id: meta + env: + EVENT_NAME: ${{ github.event.workflow_run.event }} + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + HEAD_BRANCH: ${{ github.event.workflow_run.head_branch }} + HEAD_SHA: ${{ github.event.workflow_run.head_sha }} + REPO: ${{ github.repository }} + TRIGGER_PR: ${{ github.event.workflow_run.pull_requests[0].number }} + shell: bash + run: | + set -euo pipefail + if [ "$EVENT_NAME" = "pull_request" ]; then + if [[ -n "${TRIGGER_PR:-}" ]]; then + pr="$TRIGGER_PR" + else + pr=$( + gh api "repos/$REPO/pulls?state=open&per_page=100" --paginate --jq '.[]' \ + | jq -s --arg sha "$HEAD_SHA" \ + '[.[] | select(.head.sha == $sha) | .number] | unique' + ) + count=$(echo "$pr" | jq 'length') + if [[ "$count" == "0" ]]; then + echo "::error::no open PR matches head_sha=$HEAD_SHA in $REPO" + exit 1 + fi + if [[ "$count" != "1" ]]; then + echo "::error::ambiguous head_sha=$HEAD_SHA matches $count PRs: $pr" + exit 1 + fi + pr=$(echo "$pr" | jq '.[0]') + fi + if ! [[ "$pr" =~ ^[1-9][0-9]*$ ]]; then + echo "::error::resolved PR number is not a positive integer: '$pr'" + exit 1 + fi + echo "branch=pr/$pr" >> "$GITHUB_OUTPUT" + echo "start_point=true" >> "$GITHUB_OUTPUT" + else + if [[ -z "${HEAD_BRANCH:-}" ]]; then + echo "::error::workflow_run head_branch is empty" + exit 1 + fi + echo "branch=$HEAD_BRANCH" >> "$GITHUB_OUTPUT" + if [ "$HEAD_BRANCH" = "main" ]; then + echo "start_point=false" >> "$GITHUB_OUTPUT" + else + echo "start_point=true" >> "$GITHUB_OUTPUT" + fi + fi + + - name: Count CI performance artifacts + id: artifacts + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + REPO: ${{ github.repository }} + RUN_ID: ${{ github.event.workflow_run.id }} + shell: bash + run: | + set -euo pipefail + count=$( + gh api "repos/$REPO/actions/runs/$RUN_ID/artifacts?per_page=100" --paginate \ + --jq '[.artifacts[] | select(.expired == false and (.name | startswith("ci-performance-")))] | length' \ + | jq -s 'add' + ) + echo "count=$count" >> "$GITHUB_OUTPUT" + if [ "$count" = "0" ]; then + echo "::notice::No CI performance artifacts found, skipping Bencher upload" + fi + + - name: Download CI performance artifacts + if: steps.artifacts.outputs.count != '0' + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + REPO: ${{ github.repository }} + RUN_ID: ${{ github.event.workflow_run.id }} + shell: bash + run: | + gh run download "$RUN_ID" \ + --repo "$REPO" \ + --pattern 'ci-performance-*' \ + --dir ci-performance-artifacts + + - name: Install Bencher CLI + if: steps.artifacts.outputs.count != '0' + uses: bencherdev/bencher@50fb1e138651a46d2fb704fab1adab38c181552e # v0.6.6 + + - name: Upload results to Bencher + if: steps.artifacts.outputs.count != '0' + env: + BENCHER_API_TOKEN: ${{ secrets.BENCHER_API_TOKEN }} + BENCHER_BRANCH: ${{ steps.meta.outputs.branch }} + BENCHER_START_POINT: ${{ steps.meta.outputs.start_point }} + shell: bash + run: | + set -euo pipefail + if [ -z "${BENCHER_API_TOKEN:-}" ]; then + echo "::notice::BENCHER_API_TOKEN not set, skipping Bencher upload" + exit 0 + fi + + upload() { + local testbed=$1 file=$2 + local rust_testbed_regex='^(pacquet|pnpr)\.(ubuntu|windows|macos)$' + local pnpm_testbed_regex='^pnpm\.(ubuntu|windows)\.node(22|24|26)$' + if [[ ! "$testbed" =~ $rust_testbed_regex && ! "$testbed" =~ $pnpm_testbed_regex ]]; then + echo "::error::unexpected testbed '$testbed'" + exit 1 + fi + if [ ! -f "$file" ]; then + echo "::error::$file not found for $testbed" + exit 1 + fi + + local args=( + --project pnpm-ci-performance + --testbed "$testbed" + --adapter shell_hyperfine + --file "$file" + --branch "$BENCHER_BRANCH" + ) + if [ "$BENCHER_START_POINT" = "true" ]; then + args+=( + --start-point main + --start-point-reset + --start-point-clone-thresholds + ) + fi + bencher run "${args[@]}" + } + + shopt -s nullglob + metadata_files=(ci-performance-artifacts/ci-performance-*/metadata.json) + if [ "${#metadata_files[@]}" -eq 0 ]; then + echo "::error::No metadata.json files found in CI performance artifacts" + exit 1 + fi + + for metadata in "${metadata_files[@]}"; do + dir=$(dirname "$metadata") + kind=$(jq -r '.kind // empty' "$metadata") + case "$kind" in + pnpm) + testbed=$(jq -r '.testbed // empty' "$metadata") + upload "$testbed" "$dir/results.json" + ;; + rust) + if ! reports_json=$(jq -ce '.reports | arrays' "$metadata"); then + echo "::error::missing or invalid rust reports array in $metadata" + exit 1 + fi + reports_count=$(jq 'length' <<< "$reports_json") + if [ "$reports_count" -ne 2 ]; then + echo "::error::expected exactly 2 rust reports in $metadata" + exit 1 + fi + + seen_pacquet=false + seen_pnpr=false + while IFS= read -r report; do + testbed=$(jq -r '.testbed // empty' <<< "$report") + file=$(jq -r '.file // empty' <<< "$report") + case "$file" in + pacquet-tests-all.json) + if [ "$seen_pacquet" = "true" ]; then + echo "::error::duplicate result file '$file' in $metadata" + exit 1 + fi + seen_pacquet=true + ;; + pnpr-tests-all.json) + if [ "$seen_pnpr" = "true" ]; then + echo "::error::duplicate result file '$file' in $metadata" + exit 1 + fi + seen_pnpr=true + ;; + *) + echo "::error::unexpected result file '$file'" + exit 1 + ;; + esac + upload "$testbed" "$dir/$file" + done < <(jq -c '.[]' <<< "$reports_json") + if [ "$seen_pacquet" != "true" ]; then + echo "::error::missing result file 'pacquet-tests-all.json' in $metadata" + exit 1 + fi + if [ "$seen_pnpr" != "true" ]; then + echo "::error::missing result file 'pnpr-tests-all.json' in $metadata" + exit 1 + fi + ;; + *) + echo "::error::unexpected artifact kind '$kind' in $metadata" + exit 1 + ;; + esac + done diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b96225fa82..ef1bdc1572 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -27,7 +27,11 @@ jobs: - name: Lint run: pn lint - name: Package compiled artifacts - run: tar -czf compiled.tar.gz --exclude='node_modules' $(find . -type d -name lib -not -path '*/node_modules/*') $(find . -name 'tsconfig.tsbuildinfo' -not -path '*/node_modules/*') pnpm/dist + shell: bash + run: | + mapfile -d '' -t lib_dirs < <(find . -type d -name lib -not -path '*/node_modules/*' -print0) + mapfile -d '' -t tsbuildinfo_files < <(find . -name 'tsconfig.tsbuildinfo' -not -path '*/node_modules/*' -print0) + tar -czf compiled.tar.gz --exclude='node_modules' "${lib_dirs[@]}" "${tsbuildinfo_files[@]}" pnpm/dist - name: Upload compiled artifacts uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: diff --git a/.github/workflows/pacquet-ci.yml b/.github/workflows/pacquet-ci.yml index 6215e31e44..d53c717929 100644 --- a/.github/workflows/pacquet-ci.yml +++ b/.github/workflows/pacquet-ci.yml @@ -117,7 +117,7 @@ jobs: with: tool: cargo-nextest - - name: Test + - name: Test pacquet shell: bash run: | # removing env vars is a temporary workaround for unit tests in pacquet relying on external environment @@ -125,7 +125,58 @@ jobs: unset PNPM_HOME unset XDG_DATA_HOME - just test + node .github/scripts/measure-command.mjs \ + --name tests.all \ + --output .bench/pacquet-tests-all.json \ + -- just test-pacquet + + - name: Test pnpr + shell: bash + run: | + # removing env vars is a temporary workaround for unit tests in pacquet relying on external environment + # this should be removed in the future + unset PNPM_HOME + unset XDG_DATA_HOME + + node .github/scripts/measure-command.mjs \ + --name tests.all \ + --output .bench/pnpr-tests-all.json \ + -- just test-pnpr + + - name: Stage Bencher test durations + env: + TESTBED_OS: ${{ matrix.os }} + shell: bash + run: | + os="${TESTBED_OS%-latest}" + artifact_dir=.bench/artifact/rust + mkdir -p "$artifact_dir" + cp .bench/pacquet-tests-all.json "$artifact_dir/pacquet-tests-all.json" + cp .bench/pnpr-tests-all.json "$artifact_dir/pnpr-tests-all.json" + + cat > "$artifact_dir/metadata.json" <> "$GITHUB_OUTPUT" - echo "scope=all" >> "$GITHUB_OUTPUT" + { + echo "script=ci:test-all" + echo "scope=all" + echo "full_tests=true" + echo "benchmark=tests.all" + } >> "$GITHUB_OUTPUT" else git remote set-branches --add origin main && git fetch origin main --depth=1 if [ -n "$(git diff --name-only origin/main HEAD -- pnpm-workspace.yaml)" ]; then - echo "script=ci:test-all" >> "$GITHUB_OUTPUT" - echo "scope=all — pnpm-workspace.yaml modified" >> "$GITHUB_OUTPUT" + { + echo "script=ci:test-all" + echo "scope=all — pnpm-workspace.yaml modified" + echo "full_tests=true" + echo "benchmark=tests.all" + } >> "$GITHUB_OUTPUT" else - echo "script=ci:test-branch" >> "$GITHUB_OUTPUT" - echo "scope=affected packages" >> "$GITHUB_OUTPUT" + { + echo "script=ci:test-branch" + echo "scope=affected packages" + echo "full_tests=false" + echo "benchmark=tests.affected" + } >> "$GITHUB_OUTPUT" fi fi - name: Run tests (${{ steps.test-scope.outputs.scope }}) timeout-minutes: 70 shell: bash env: + BENCHMARK: ${{ steps.test-scope.outputs.benchmark }} PNPM_WORKERS: 3 TEST_SCRIPT: ${{ steps.test-scope.outputs.script }} - run: pn run "$TEST_SCRIPT" + run: | + node .github/scripts/measure-command.mjs \ + --name "$BENCHMARK" \ + --output ".bench/${BENCHMARK}.json" \ + -- pn run "$TEST_SCRIPT" + - name: Extract pnpm CLI e2e test duration + if: steps.test-scope.outputs.full_tests == 'true' + shell: bash + run: | + node .github/scripts/bencher-result-from-pnpm-summary.mjs \ + --name tests.cli \ + --output .bench/tests.cli.json \ + --package-dir pnpm + - name: Stage Bencher test durations + if: steps.test-scope.outputs.full_tests == 'true' + env: + NODE_VERSION: ${{ inputs.node }} + PLATFORM: ${{ inputs.platform }} + shell: bash + run: | + platform_slug=${PLATFORM%-latest} + node_major=${NODE_VERSION%%.*} + case "$platform_slug" in + ubuntu|windows) ;; + *) + echo "::error::Unsupported platform '$platform_slug' for pnpm benchmarks; expected ubuntu or windows" + exit 1 + ;; + esac + case "$node_major" in + 22|24|26) ;; + *) + echo "::error::Unsupported Node major '$node_major' for pnpm benchmarks; expected 22, 24, or 26" + exit 1 + ;; + esac + artifact_dir=.bench/artifact/pnpm + mkdir -p "$artifact_dir" + + node .github/scripts/merge-bencher-results.mjs \ + --output "$artifact_dir/results.json" \ + -- .bench/tests.all.json .bench/tests.cli.json + + cat > "$artifact_dir/metadata.json" <