From fbb662f230405871380c8a382f0824e244b4ea56 Mon Sep 17 00:00:00 2001 From: yaruk-byte Date: Fri, 9 Oct 2026 15:40:52 -0700 Subject: [PATCH] .github: run the Windows tests nightly (#21731) Updates #21489 Signed-off-by: Yaruk Asghar --- .../actions/windows-slack-payload/action.yml | 114 +++++++++++++++ .github/workflows/test.yml | 100 ++----------- .github/workflows/windows-nightly.yml | 131 ++++++++++++++++++ 3 files changed, 259 insertions(+), 86 deletions(-) create mode 100644 .github/actions/windows-slack-payload/action.yml create mode 100644 .github/workflows/windows-nightly.yml diff --git a/.github/actions/windows-slack-payload/action.yml b/.github/actions/windows-slack-payload/action.yml new file mode 100644 index 000000000..2a779af8f --- /dev/null +++ b/.github/actions/windows-slack-payload/action.yml @@ -0,0 +1,114 @@ +name: windows-slack-payload +description: Build a Slack message from the windows-integration-results.json files under results/ + +inputs: + result: + description: Result of the job that ran the tests + required: true + title: + description: Message title, shown before the result + required: true + header: + description: First line of the message, in Slack mrkdwn + required: true + status-label: + description: Label for an extra line with the result, left out when empty + required: false + default: "" + tests-label: + description: Label for the line with the test counts + required: true + expected-files: + description: Number of result files the run should have uploaded + required: true + +runs: + using: composite + steps: + - name: Build Slack payload + shell: bash + env: + RESULT: ${{ inputs.result }} + # Title and header can come from a fork PR, so they only reach the script through env. + TITLE: ${{ inputs.title }} + HEADER: ${{ inputs.header }} + STATUS_LABEL: ${{ inputs.status-label }} + TESTS_LABEL: ${{ inputs.tests-label }} + EXPECTED_FILES: ${{ inputs.expected-files }} + RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + run: | + # Read each artifact separately so one unreadable file can't discard the others. + rows='[]'; unreadable=0; reported=0 + while IFS= read -r -d '' f; do + reported=$((reported + 1)) + [ -s "$f" ] || continue + case "$(jq -r 'type' "$f" 2>/dev/null)" in + array) rows=$(printf '%s\n%s' "$rows" "$(jq -c . "$f")" | jq -s 'add') ;; + null) ;; # ran none of the filtered package + *) unreadable=$((unreadable + 1)) ;; + esac + done < <(find results -name windows-integration-results.json -print0 2>/dev/null) + # Dedupe so a re-shard reporting a test twice can't double every count. + rows=$(printf '%s' "$rows" | jq '[ .[] | objects ] | unique_by([.Package, .Test])') + count() { printf '%s' "$rows" | jq "[ (. // [])[] | select(.Outcome == \"$1\") ] | length"; } + retried=$(count retried) + passed=$(( $(count pass) + retried )) + failed=$(count fail) + skipped=$(count skip) + executed=$((passed + failed)) + if [ "$executed" -gt 0 ]; then pct=$(( (passed * 100) / executed )); else pct=0; fi + # Names behind the retried count, capped so one bad run can't flood the message. + flaky=$(printf '%s' "$rows" | jq -r ' + [ .[] | select(.Outcome == "retried") | (.Package | sub("^tailscale\\.com/"; "")) + "." + .Test ] + | sort + | if length > 5 then (.[:5] | join(", ")) + ", and \(length - 5) more" else join(", ") end') + + case "$RESULT" in + success) emoji=":white_check_mark:"; color="good" ;; + failure) emoji=":x:"; color="danger" ;; + cancelled) emoji=":no_entry_sign:"; color="warning" ;; + skipped) emoji=":fast_forward:"; color="warning" ;; + *) emoji=":question:"; color="warning" ;; + esac + + tests="${passed} passed, ${failed} failed, ${skipped} skipped" + if [ "$executed" -gt 0 ]; then + tests="${tests} (${pct}%)" + fi + # Anything that makes the numbers incomplete or overstated is said out loud. + caveats="" + if [ "$unreadable" -gt 0 ]; then + caveats="${caveats} — ${unreadable} result file(s) unreadable" + fi + if [ "$reported" -lt "$EXPECTED_FILES" ]; then + caveats="${caveats} — only ${reported} of ${EXPECTED_FILES} result files reported" + fi + # Retries get their own line below, so they don't warn the count line. + if [ "$executed" -eq 0 ]; then + caveats="${caveats} — no tests executed" + fi + if [ "$RESULT" = "failure" ] && [ "$failed" -eq 0 ]; then + caveats="${caveats} — results may be partial (build error or timeout)" + fi + tests="${tests}${caveats}" + if [ "$failed" -gt 0 ]; then tests_emoji=":x:" + elif [ -n "$caveats" ]; then tests_emoji=":warning:" + else tests_emoji=":white_check_mark:"; fi + + jq -n --arg emoji "$emoji" --arg status "$RESULT" --arg tests_emoji "$tests_emoji" \ + --arg tests "$tests" --arg flaky "$flaky" --arg retried "$retried" \ + --arg title "$TITLE" --arg header "$HEADER" --arg status_label "$STATUS_LABEL" \ + --arg tests_label "$TESTS_LABEL" --arg color "$color" --arg run_url "$RUN_URL" \ + '{attachments: [{ + title: ($emoji + " " + $title + ": " + $status), + title_link: $run_url, + text: ( + $header + + "\n\n" + (if $status_label == "" then "" else $emoji + " " + $status_label + ": " + $status + "\n" end) + + $tests_emoji + " " + $tests_label + ": " + $tests + + (if $flaky == "" then "" else "\n ↳ :warning: Flaky Tests (" + $retried + " passed on retry): " + $flaky end) + + "\n\n<" + $run_url + "|View run details>" + ), + color: $color, + mrkdwn_in: ["text"] + }]}' > payload.json diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 3e877922f..066a99200 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -862,6 +862,11 @@ jobs: - windows runs-on: ubuntu-24.04 steps: + - name: checkout + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + path: src + persist-credentials: false - name: Download Windows integration results uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: @@ -869,92 +874,15 @@ jobs: path: results - name: Build Slack payload id: payload - env: - WINDOWS_RESULT: ${{ needs.windows.result }} - PR_TITLE: ${{ github.event.pull_request.title }} - PR_NUMBER: ${{ github.event.pull_request.number }} - PR_URL: ${{ github.event.pull_request.html_url }} - # head.ref is untrusted on fork PRs; keep it in env and encode via jq --arg, never run: text. - HEAD_REF: ${{ github.event.pull_request.head.ref }} - RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} - # Non-bench legs of the windows matrix; a missing artifact means a shard never reported. - EXPECTED_SHARDS: "2" - run: | - # Read each artifact separately so one unreadable file can't discard the others. - rows='[]'; unreadable=0; reported=0 - while IFS= read -r -d '' f; do - reported=$((reported + 1)) - [ -s "$f" ] || continue - case "$(jq -r 'type' "$f" 2>/dev/null)" in - array) rows=$(printf '%s\n%s' "$rows" "$(jq -c . "$f")" | jq -s 'add') ;; - null) ;; # shard ran none of the filtered package - *) unreadable=$((unreadable + 1)) ;; - esac - done < <(find results -name windows-integration-results.json -print0 2>/dev/null) - # Dedupe so a re-shard reporting a test twice can't double every count. - rows=$(printf '%s' "$rows" | jq '[ .[] | objects ] | unique_by([.Package, .Test])') - count() { printf '%s' "$rows" | jq "[ (. // [])[] | select(.Outcome == \"$1\") ] | length"; } - retried=$(count retried) - passed=$(( $(count pass) + retried )) - failed=$(count fail) - skipped=$(count skip) - executed=$((passed + failed)) - if [ "$executed" -gt 0 ]; then pct=$(( (passed * 100) / executed )); else pct=0; fi - # Names behind the retried count, capped so one bad run can't flood the message. - flaky=$(printf '%s' "$rows" | jq -r ' - [ .[] | select(.Outcome == "retried") | (.Package | sub("^tailscale\\.com/"; "")) + "." + .Test ] - | sort - | if length > 5 then (.[:5] | join(", ")) + ", and \(length - 5) more" else join(", ") end') - - case "$WINDOWS_RESULT" in - success) emoji=":white_check_mark:"; color="good" ;; - failure) emoji=":x:"; color="danger" ;; - cancelled) emoji=":no_entry_sign:"; color="warning" ;; - skipped) emoji=":fast_forward:"; color="warning" ;; - *) emoji=":question:"; color="warning" ;; - esac - - integration="${passed} passed, ${failed} failed, ${skipped} skipped" - if [ "$executed" -gt 0 ]; then - integration="${integration} (${pct}%)" - fi - # Anything that makes the numbers incomplete or overstated is said out loud. - caveats="" - if [ "$unreadable" -gt 0 ]; then - caveats="${caveats} — ${unreadable} shard artifact(s) unreadable" - fi - if [ "$reported" -lt "$EXPECTED_SHARDS" ]; then - caveats="${caveats} — only ${reported} of ${EXPECTED_SHARDS} shards reported" - fi - # Retries get their own line below, so they don't warn the count line. - if [ "$executed" -eq 0 ]; then - caveats="${caveats} — no tests executed" - fi - if [ "$WINDOWS_RESULT" = "failure" ] && [ "$failed" -eq 0 ]; then - caveats="${caveats} — results may be partial (build error or timeout)" - fi - integration="${integration}${caveats}" - if [ "$failed" -gt 0 ]; then int_emoji=":x:" - elif [ -n "$caveats" ]; then int_emoji=":warning:" - else int_emoji=":white_check_mark:"; fi - - jq -n --arg emoji "$emoji" --arg status "$WINDOWS_RESULT" --arg int_emoji "$int_emoji" \ - --arg integration "$integration" --arg flaky "$flaky" --arg retried "$retried" \ - --arg branch "$HEAD_REF" --arg title "$PR_TITLE" --arg pr_url "$PR_URL" --arg pr_number "$PR_NUMBER" \ - --arg color "$color" --arg run_url "$RUN_URL" \ - '{attachments: [{ - title: ($emoji + " " + $title + ": " + $status), - title_link: $run_url, - text: ( - "<" + $pr_url + "|PR #" + $pr_number + "> · `" + $branch + "`" - + "\n\n" + $emoji + " Windows CI: " + $status - + "\n" + $int_emoji + " Windows Integration Tests: " + $integration - + (if $flaky == "" then "" else "\n ↳ :warning: Flaky Tests (" + $retried + " passed on retry): " + $flaky end) - + "\n\n<" + $run_url + "|View run details>" - ), - color: $color, - mrkdwn_in: ["text"] - }]}' > payload.json + uses: ./src/.github/actions/windows-slack-payload + with: + result: ${{ needs.windows.result }} + title: ${{ github.event.pull_request.title }} + header: "<${{ github.event.pull_request.html_url }}|PR #${{ github.event.pull_request.number }}> · `${{ github.event.pull_request.head.ref }}`" + status-label: Windows CI + tests-label: Windows Integration Tests + # One result file per non-bench leg of the windows matrix. + expected-files: "2" - name: Send Slack notification env: WEBHOOK: ${{ secrets.WINDOWS_SLACK_WEBHOOK_URL }} diff --git a/.github/workflows/windows-nightly.yml b/.github/workflows/windows-nightly.yml new file mode 100644 index 000000000..ac6ba0367 --- /dev/null +++ b/.github/workflows/windows-nightly.yml @@ -0,0 +1,131 @@ +# Runs the whole tstest/integration suite against the Windows service three times a night. +name: windows-nightly + +on: + workflow_dispatch: + + schedule: + # Three runs, 20 minutes apart and off the hour, to catch flaky tests. + - cron: "13,33,53 7 * * *" + + pull_request: + paths: + - ".github/workflows/windows-nightly.yml" + +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} + cancel-in-progress: true + +env: + GOMODCACHE: ${{ github.workspace }}/gomodcache + CMD_GO_USE_GIT_HASH: "true" + +permissions: {} + +jobs: + nightly: + permissions: + id-token: write # request the identity JWT that authenticates to cigocached + contents: read # for actions/checkout + runs-on: ci-windows-github-1 + name: Windows nightly (${{ matrix.suite }}) + timeout-minutes: 30 + strategy: + fail-fast: false + matrix: + include: + # One row per suite, so smoke and regression suites can be added as rows later. + - suite: integration + packages: ./tstest/integration + results_pkg: tailscale.com/tstest/integration + flags: --run-windows-service-tests + steps: + - name: checkout + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + path: ${{ github.workspace }}/src + persist-credentials: false + + - name: Compute Go module cache key + id: hash + shell: bash + run: echo "key=gomod-cross3-${{ hashFiles('src/go.mod', 'src/go.sum') }}" >> "$GITHUB_OUTPUT" + + - name: Restore Go module cache + uses: actions/cache/restore@668228422ae6a00e4ad889ee87cd7109ec5666a7 # v5.0.4 + with: + path: gomodcache + key: ${{ steps.hash.outputs.key }} + enableCrossOsArchive: true + + - name: Set up cigocacher + uses: ./src/.github/actions/go-cache + with: + checkout-path: ${{ github.workspace }}/src + cache-dir: ${{ github.workspace }}/cigocacher + cigocached-url: ${{ vars.CIGOCACHED_AZURE_URL }} + cigocached-host: ${{ vars.CIGOCACHED_AZURE_HOST }} + + - name: test + shell: bash + working-directory: src + # -count=1: a cached pass is never retried, so flakes can't be reported. + run: ./tool/go run ./cmd/testwrapper $PACKAGES -count=1 $FLAGS + env: + PACKAGES: ${{ matrix.packages }} + FLAGS: ${{ matrix.flags }} + NOPWSHDEBUG: "true" # to quiet tool/gocross/gocross-wrapper.ps1 in CI + TS_TESTWRAPPER_RESULTS_SUMMARY: "1" + TS_TESTWRAPPER_RESULTS_PKG: ${{ matrix.results_pkg }} + TS_TESTWRAPPER_RESULTS_JSON: ${{ github.workspace }}/windows-integration-results.json + + - name: Upload results + if: always() + uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 + with: + name: windows-nightly-${{ matrix.suite }}-results + path: ${{ github.workspace }}/windows-integration-results.json + if-no-files-found: ignore + retention-days: 90 + + notify: + if: always() + needs: + - nightly + permissions: + contents: read # for actions/checkout + runs-on: ubuntu-24.04 + steps: + - name: checkout + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + path: ${{ github.workspace }}/src + persist-credentials: false + - name: Download results + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + # A pattern, not a name, so a run that never uploaded still gets reported. + pattern: windows-nightly-*-results + path: results + - name: Build Slack payload + id: payload + if: always() + uses: ./src/.github/actions/windows-slack-payload + with: + result: ${{ needs.nightly.result }} + title: Windows Nightly Tests + header: "*${{ github.ref_name }}*" + tests-label: smoke and integration tests + # One result file per suite row. + expected-files: "1" + - name: Send Slack notification + env: + WEBHOOK: ${{ secrets.WINDOWS_SLACK_WEBHOOK_URL }} + # Posts on cancel too, so a missing message always means something went wrong. + if: always() && env.WEBHOOK != '' && github.event_name != 'pull_request' + continue-on-error: true + uses: slackapi/slack-github-action@91efab103c0de0a537f72a35f6b8cda0ee76bf0a # v2.1.1 + with: + webhook: ${{ env.WEBHOOK }} + webhook-type: incoming-webhook + payload-file-path: payload.json