diff --git a/.github/scripts/collect-failed-tests.js b/.github/scripts/collect-failed-tests.js deleted file mode 100644 index c9fac81392..0000000000 --- a/.github/scripts/collect-failed-tests.js +++ /dev/null @@ -1,45 +0,0 @@ -#!/usr/bin/env node -// Walks Playwright JSON reporter output (one or more files) and writes a -// deduped list of "file:line" locations for every test that ended up -// "unexpected" (failed after all in-job retries), so a follow-up job can -// re-run just those tests single-threaded. -// -// Usage: node collect-failed-tests.js - -const fs = require('fs'); -const path = require('path'); - -const [, , reportsDir, outFile] = process.argv; - -if (!reportsDir || !outFile) { - console.error('Usage: collect-failed-tests.js '); - process.exit(1); -} - -const failed = new Set(); - -function walkSuite(suite) { - for (const spec of suite.specs || []) { - const isFailed = (spec.tests || []).some(test => test.status === 'unexpected'); - if (isFailed) { - failed.add(`${spec.file}:${spec.line}`); - } - } - for (const child of suite.suites || []) { - walkSuite(child); - } -} - -const files = fs.existsSync(reportsDir) - ? fs.readdirSync(reportsDir).filter(f => f.endsWith('.json')) - : []; - -for (const file of files) { - const report = JSON.parse(fs.readFileSync(path.join(reportsDir, file), 'utf8')); - for (const suite of report.suites || []) { - walkSuite(suite); - } -} - -fs.writeFileSync(outFile, Array.from(failed).sort().join('\n')); -console.log(`Found ${failed.size} failed test file(s) across ${files.length} report(s).`); diff --git a/.github/scripts/stitch-rerun-attempts.js b/.github/scripts/stitch-rerun-attempts.js deleted file mode 100644 index 63ae8bae6c..0000000000 --- a/.github/scripts/stitch-rerun-attempts.js +++ /dev/null @@ -1,182 +0,0 @@ -#!/usr/bin/env node -// Splices sequential-test's rerun attempts into the shard blob report that -// originally ran each test, as additional (higher-numbered) retries of the -// same testId, instead of leaving them in a separate blob-report-rerun.zip. -// -// Playwright's `merge-reports` salts any testId that reappears in a second -// blob (see IdsPatcher._mapTestId in playwright/lib/reporters/merge.js) — -// it assumes blobs cover disjoint test sets, as true shards do. Because -// blob-report-rerun.zip re-runs tests that already exist in one of the -// shard blobs, the merged report used to show that test twice instead of -// once with an extra "Retry #N" attempt. -// -// Usage: node stitch-rerun-attempts.js -// Mutates blob-report-.zip files in place and deletes blob-report-rerun.zip -// once every rerun test has been folded into its origin shard. - -const fs = require('fs'); -const path = require('path'); -const { execFileSync } = require('child_process'); - -const [, , dir] = process.argv; -if (!dir) { - console.error('Usage: stitch-rerun-attempts.js '); - process.exit(1); -} - -const rerunZip = path.join(dir, 'blob-report-rerun.zip'); -if (!fs.existsSync(rerunZip)) { - console.log('No blob-report-rerun.zip found, nothing to stitch.'); - process.exit(0); -} - -const shardZips = fs.readdirSync(dir) - .filter(f => /^blob-report-\d+\.zip$/.test(f)) - .map(f => ({ file: f, zipPath: path.join(dir, f) })); - -if (shardZips.length === 0) { - console.log('No shard blob-report-.zip files found, nothing to stitch.'); - process.exit(0); -} - -const workDir = path.join(dir, '.stitch-work'); -fs.rmSync(workDir, { recursive: true, force: true }); -fs.mkdirSync(workDir, { recursive: true }); - -function unzip(zipPath, destDir) { - fs.mkdirSync(destDir, { recursive: true }); - execFileSync('unzip', ['-q', '-o', zipPath, '-d', destDir]); -} - -function rezip(srcDir, zipPath) { - fs.rmSync(zipPath, { force: true }); - const entries = fs.readdirSync(srcDir); - // -D: no directory entries. Playwright's merge-reports (extractAndParseReports - // in playwright/lib/reporters/merge.js) reads every zip entry as a file, so a - // bare "resources/" directory entry makes it crash with EISDIR. - execFileSync('zip', ['-q', '-r', '-D', path.resolve(zipPath), ...entries], { cwd: srcDir }); -} - -function findJsonl(dir) { - const file = fs.readdirSync(dir).find(f => f.endsWith('.jsonl')); - return file ? path.join(dir, file) : null; -} - -function readJsonl(file) { - if (!file || !fs.existsSync(file)) return []; - return fs.readFileSync(file, 'utf8') - .split('\n') - .filter(Boolean) - .map(line => JSON.parse(line)); -} - -function writeJsonl(file, events) { - fs.writeFileSync(file, events.map(e => JSON.stringify(e) + '\n').join('')); -} - -function testIdOf(event) { - switch (event.method) { - case 'onTestBegin': - case 'onStepBegin': - case 'onStepEnd': - case 'onAttach': - case 'onStdIO': - return event.params.testId; - case 'onTestEnd': - return event.params.test.testId; - default: - return undefined; - } -} - -// --- Extract the rerun blob and split its events into per-test attempts --- -const rerunDir = path.join(workDir, 'rerun'); -unzip(rerunZip, rerunDir); -const rerunEvents = readJsonl(findJsonl(rerunDir)); - -// testId -> array of attempts, each attempt an ordered array of events -// starting with its onTestBegin. -const rerunAttemptsByTestId = new Map(); -for (const event of rerunEvents) { - const testId = testIdOf(event); - if (testId === undefined) - continue; - let attempts = rerunAttemptsByTestId.get(testId); - if (!attempts) - rerunAttemptsByTestId.set(testId, attempts = []); - if (event.method === 'onTestBegin') - attempts.push([event]); - else - attempts[attempts.length - 1].push(event); -} - -if (rerunAttemptsByTestId.size === 0) { - console.log('Rerun blob contains no test events, nothing to stitch.'); - fs.rmSync(workDir, { recursive: true, force: true }); - process.exit(0); -} - -// --- Extract every shard blob once, so we can find which shard originally -// ran each rerun testId without needing to know that ahead of time. --- -const shards = shardZips.map(({ file, zipPath }) => { - const shardDir = path.join(workDir, file.replace(/\.zip$/, '')); - unzip(zipPath, shardDir); - const jsonlPath = findJsonl(shardDir); - const events = readJsonl(jsonlPath); - const maxRetryByTestId = new Map(); - for (const event of events) { - if (event.method !== 'onTestBegin') - continue; - const { testId, result } = event.params; - if (!maxRetryByTestId.has(testId) || result.retry > maxRetryByTestId.get(testId)) - maxRetryByTestId.set(testId, result.retry); - } - return { zipPath, shardDir, jsonlPath, events, maxRetryByTestId, changed: false }; -}); - -const unmatched = []; - -for (const [testId, attempts] of rerunAttemptsByTestId) { - const shard = shards.find(s => s.maxRetryByTestId.has(testId)); - if (!shard) { - unmatched.push(testId); - continue; - } - let nextRetry = shard.maxRetryByTestId.get(testId) + 1; - for (const attempt of attempts) { - for (const event of attempt) { - if (event.method === 'onTestBegin') { - event.params.result.retry = nextRetry; - } else if (event.method === 'onAttach') { - for (const attachment of event.params.attachments || []) { - if (!attachment.path) - continue; - const src = path.join(rerunDir, attachment.path); - const dest = path.join(shard.shardDir, attachment.path); - fs.mkdirSync(path.dirname(dest), { recursive: true }); - fs.copyFileSync(src, dest); - } - } - } - shard.events.push(...attempt); - nextRetry++; - } - shard.changed = true; -} - -for (const shard of shards) { - if (!shard.changed) - continue; - writeJsonl(shard.jsonlPath, shard.events); - rezip(shard.shardDir, shard.zipPath); - console.log(`Stitched rerun attempt(s) into ${path.basename(shard.zipPath)}`); -} - -if (unmatched.length > 0) { - console.warn(`Warning: ${unmatched.length} rerun test(s) had no matching shard, keeping blob-report-rerun.zip as a fallback: ${unmatched.join(', ')}`); -} else { - fs.rmSync(rerunZip); - console.log('Removed blob-report-rerun.zip (fully folded into shard blobs).'); -} - -fs.rmSync(workDir, { recursive: true, force: true }); diff --git a/.github/workflows/insomnia-test.yml b/.github/workflows/insomnia-test.yml index 222d0a5f11..e7090ffefd 100644 --- a/.github/workflows/insomnia-test.yml +++ b/.github/workflows/insomnia-test.yml @@ -88,15 +88,13 @@ jobs: # --------------------------------------------------------------------------- # Run the whole suite across N shards in parallel. Each shard restores the # cached binary and runs a slice of the suite, retrying failures in-process - # (see playwright.config.ts `retries`). Each shard also emits a JSON report - # so collect-failed-tests can find whatever is still failing afterwards. + # (see playwright.config.ts `retries`). # --------------------------------------------------------------------------- parallel-test: name: Parallel Test needs: build timeout-minutes: 60 runs-on: ubuntu-latest - continue-on-error: true permissions: contents: read packages: read @@ -141,10 +139,9 @@ jobs: working-directory: packages/insomnia-test env: INSOMNIA_BINARY: ${{ github.workspace }}/packages/insomnia/dist/linux-unpacked/insomnia - PLAYWRIGHT_JSON_OUTPUT_NAME: results-${{ matrix.shardIndex }}.json PLAYWRIGHT_BLOB_OUTPUT_NAME: blob-report-${{ matrix.shardIndex }}.zip COVERAGE: 'true' - run: xvfb-run --auto-servernum npx playwright test --reporter=list,html,json,blob --shard=${{ matrix.shardIndex }}/${{ matrix.shardTotal }} + run: xvfb-run --auto-servernum npx playwright test --reporter=list,html,blob --shard=${{ matrix.shardIndex }}/${{ matrix.shardTotal }} - name: Upload coverage (raw) if: ${{ !cancelled() }} @@ -164,15 +161,6 @@ jobs: retention-days: 7 if-no-files-found: ignore - - name: Upload JSON results - uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6 - if: ${{ !cancelled() }} - with: - name: playwright-json-${{ matrix.shardIndex }} - path: packages/insomnia-test/results-${{ matrix.shardIndex }}.json - retention-days: 1 - if-no-files-found: ignore - - name: Upload blob report uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6 if: ${{ !cancelled() }} @@ -183,151 +171,21 @@ jobs: if-no-files-found: ignore # --------------------------------------------------------------------------- - # Look at every shard's JSON report to find tests still failing after - # Playwright's own in-job retries, then re-run just those, single-threaded. - # Running them in isolation (one worker, no sharding) rules out - # cross-test/cross-shard interference as the cause before treating a - # failure as real. Every step after "Collect failed tests" is gated on - # there being failures, so a clean run costs only the collect step itself. - # --------------------------------------------------------------------------- - sequential-test: - name: Rerun Failed Test - needs: parallel-test - if: ${{ !cancelled() }} - timeout-minutes: 30 - runs-on: ubuntu-latest - continue-on-error: true - permissions: - contents: read - packages: read - outputs: - has_failures: ${{ steps.collect.outputs.has_failures }} - passed: ${{ steps.outcome.outputs.passed }} - steps: - - name: Checkout repo - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5 - - - name: Download JSON results - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 - with: - pattern: playwright-json-* - path: json-reports - merge-multiple: true - - - name: Collect failed tests - id: collect - run: | - node .github/scripts/collect-failed-tests.js json-reports packages/insomnia-test/failed-tests.txt - if [ -s packages/insomnia-test/failed-tests.txt ]; then - echo "has_failures=true" >> "$GITHUB_OUTPUT" - else - echo "has_failures=false" >> "$GITHUB_OUTPUT" - fi - - - name: Setup Node.js - if: steps.collect.outputs.has_failures == 'true' - uses: actions/setup-node@a0853c24544627f65ddf259abe73b1d18a591444 # v5 - with: - node-version: '24' - - - name: Restore Insomnia binary - if: steps.collect.outputs.has_failures == 'true' - uses: actions/cache/restore@caa296126883cff596d87d8935842f9db880ef25 # v5 - with: - path: packages/insomnia/dist/linux-unpacked - key: insomnia-linux-unpacked-${{ github.sha }} - fail-on-cache-miss: true - - - name: Install test dependencies - if: steps.collect.outputs.has_failures == 'true' - run: | - sed -i 's/patch-package && npm run verify-bundle-plugins -w insomnia && npm run install-libcurl-electron/patch-package/' package.json - npm ci - env: - NODE_AUTH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - - name: Install Electron system dependencies - if: steps.collect.outputs.has_failures == 'true' - working-directory: packages/insomnia-test - run: npx playwright install-deps chromium - - - name: Install ffmpeg (required for video recording) - if: steps.collect.outputs.has_failures == 'true' - working-directory: packages/insomnia-test - run: npx playwright install ffmpeg - - - name: Run Playwright tests (failed tests, single worker) - id: run_tests - if: steps.collect.outputs.has_failures == 'true' - working-directory: packages/insomnia-test - env: - INSOMNIA_BINARY: ${{ github.workspace }}/packages/insomnia/dist/linux-unpacked/insomnia - PLAYWRIGHT_BLOB_OUTPUT_NAME: blob-report-rerun.zip - COVERAGE: 'true' - run: xvfb-run --auto-servernum npx playwright test --reporter=list,html,blob --workers=1 --retries=0 $(cat failed-tests.txt) - - - name: Record test outcome - id: outcome - if: ${{ steps.collect.outputs.has_failures == 'true' && always() }} - run: echo "passed=${{ steps.run_tests.outcome == 'success' }}" >> "$GITHUB_OUTPUT" - - - name: Upload coverage (raw) - if: ${{ steps.collect.outputs.has_failures == 'true' && !cancelled() }} - uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6 - with: - # Feeds insomnia-test-result's "Merge coverage reports" step (its - # download pattern is "coverage-raw-*") so the rerun's coverage is - # folded into the final merged coverage report too. - name: coverage-raw-rerun - path: packages/insomnia-test/coverage/raw/ - retention-days: 1 - if-no-files-found: ignore - - - name: Upload Playwright report - uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6 - if: ${{ steps.collect.outputs.has_failures == 'true' && !cancelled() }} - with: - # Distinct from merge-coverage's "playwright-report" (the full - # merged suite report) — this one only covers the tests re-run - # here. - name: playwright-report-rerun - path: packages/insomnia-test/playwright-report/ - retention-days: 7 - if-no-files-found: ignore - - - name: Upload blob report - uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6 - if: ${{ steps.collect.outputs.has_failures == 'true' && !cancelled() }} - with: - # Feeds insomnia-test-result's "Merge Playwright reports" step so - # the rerun's outcome (pass or fail) is reflected in the final - # merged suite report, not just this job's own HTML report above. - name: blob-report-rerun - path: packages/insomnia-test/blob-report/blob-report-rerun.zip - retention-days: 1 - if-no-files-found: ignore - - # --------------------------------------------------------------------------- - # Every prior job sets continue-on-error: true so a failure never blocks - # this job from running. This is the one job that actually gates the - # workflow: pass if Parallel Test had no failures, otherwise pass only if - # Failed Test Review's re-run of the failures succeeded. - # - # It also merges every shard's raw V8 coverage data (collected on both - # develop pushes and PRs — see parallel-test's COVERAGE env), plus - # sequential-test's rerun coverage when it ran (coverage-raw-rerun), into - # one coverage report, and every shard's blob report into one full-suite - # Playwright report — both run on every trigger this workflow has (push - # and pull_request are the only two, so no `if` gate is needed on either). - # Each merge step is its own `continue-on-error: true` so it never affects - # the pass/fail gate below. By the time this job runs, sequential-test is - # already done consuming the per-shard artifacts, so it finishes by - # deleting every per-shard intermediate (coverage-raw-*, playwright-report-*, - # playwright-json-*, blob-report-*), leaving only the merged artifact(s). + # This job doesn't gate the workflow — parallel-test (no continue-on-error) + # already fails the run on its own if any shard fails. This job only + # merges every shard's raw V8 coverage data (collected on both develop + # pushes and PRs — see parallel-test's COVERAGE env) into one coverage + # report, and every shard's blob report into one full-suite Playwright + # report — both run on every trigger this workflow has (push and + # pull_request are the only two, so no `if` gate is needed on either). + # Each merge step is its own `continue-on-error: true` so a merge failure + # never fails this job. It finishes by deleting every per-shard + # intermediate (coverage-raw-*, playwright-report-*, blob-report-*), + # leaving only the merged artifact(s). # --------------------------------------------------------------------------- insomnia-test-result: name: Insomnia Test Result - needs: [parallel-test, sequential-test] + needs: parallel-test if: ${{ !cancelled() }} runs-on: ubuntu-latest permissions: @@ -357,8 +215,8 @@ jobs: NODE_AUTH_TOKEN: ${{ secrets.GITHUB_TOKEN }} # Coverage raw data can embed `file:` sourcemap URLs pointing into this - # build output (see parallel-test/sequential-test's own restore step). - # Without it here, sourcemap resolution silently fails for those + # build output (see parallel-test's own restore step). Without it + # here, sourcemap resolution silently fails for those # scripts during the merge below, deflating line/statement/branch # coverage relative to what each shard reports locally. - name: Restore Insomnia binary @@ -397,16 +255,6 @@ jobs: pattern: blob-report-* merge-multiple: true - # blob-report-rerun.zip re-runs tests that already have a (failing) - # result in one of the shard blobs above. Left as-is, merge-reports - # would show that test twice instead of once with an extra retry - # attempt (see the script for why). This splices the rerun's result - # into its origin shard blob as the next retry and removes - # blob-report-rerun.zip, so it never reaches merge-reports. - - name: Stitch rerun attempts into origin shard blobs - continue-on-error: true - run: node .github/scripts/stitch-rerun-attempts.js all-blob-reports - - name: Merge Playwright reports continue-on-error: true working-directory: packages/insomnia-test @@ -420,10 +268,9 @@ jobs: path: packages/insomnia-test/playwright-report/ retention-days: 30 - # coverage-raw-*/playwright-report-([0-9]+|rerun)/playwright-json-*/ - # blob-report-([0-9]+|rerun) were only a transport into this job — now - # that they're merged, delete them so only the two artifacts uploaded - # above remain. + # coverage-raw-*/playwright-report-[0-9]+/blob-report-[0-9]+ were only a + # transport into this job — now that they're merged, delete them so + # only the two artifacts uploaded above remain. - name: Delete per-shard intermediate artifacts continue-on-error: true env: @@ -431,21 +278,8 @@ jobs: run: | ids=$(gh api --paginate \ "/repos/${{ github.repository }}/actions/runs/${{ github.run_id }}/artifacts" \ - --jq '.artifacts[] | select(.name | test("^playwright-report-([0-9]+|rerun)$|^playwright-json-[0-9]+$|^coverage-raw-([0-9]+|rerun)$|^blob-report-([0-9]+|rerun)$")) | .id') + --jq '.artifacts[] | select(.name | test("^playwright-report-[0-9]+$|^coverage-raw-[0-9]+$|^blob-report-[0-9]+$")) | .id') for id in $ids; do echo "Deleting intermediate artifact $id" gh api -X DELETE "/repos/${{ github.repository }}/actions/artifacts/$id" done - - - name: Evaluate result - run: | - if [ "${{ needs.sequential-test.outputs.has_failures }}" != "true" ]; then - echo "Parallel Test passed." - exit 0 - fi - if [ "${{ needs.sequential-test.outputs.passed }}" == "true" ]; then - echo "Parallel Test had failures, but Rerun Failed Test re-ran them successfully." - exit 0 - fi - echo "Parallel Test had failures, and Rerun Failed Test failed to re-run them successfully." - exit 1