.github: run the Windows tests nightly (#21731)

Updates #21489 Signed-off-by: Yaruk Asghar <yaruk@tailscale.com>
2026-11-06 09:33:17 -05:00 · 2026-10-09 15:40:52 -07:00
1 parent 5b226c7d1d
commit fbb662f230
@@ -0,0 +1,114 @@
+name: windows-slack-payload
+description: Build a Slack message from the windows-integration-results.json files under results/
+
+inputs:
+  result:
+    description: Result of the job that ran the tests
+    required: true
+  title:
+    description: Message title, shown before the result
+    required: true
+  header:
+    description: First line of the message, in Slack mrkdwn
+    required: true
+  status-label:
+    description: Label for an extra line with the result, left out when empty
+    required: false
+    default: ""
+  tests-label:
+    description: Label for the line with the test counts
+    required: true
+  expected-files:
+    description: Number of result files the run should have uploaded
+    required: true
+
+runs:
+  using: composite
+  steps:
+    - name: Build Slack payload
+      shell: bash
+      env:
+        RESULT: ${{ inputs.result }}
+        # Title and header can come from a fork PR, so they only reach the script through env.
+        TITLE: ${{ inputs.title }}
+        HEADER: ${{ inputs.header }}
+        STATUS_LABEL: ${{ inputs.status-label }}
+        TESTS_LABEL: ${{ inputs.tests-label }}
+        EXPECTED_FILES: ${{ inputs.expected-files }}
+        RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
+      run: |
+        # Read each artifact separately so one unreadable file can't discard the others.
+        rows='[]'; unreadable=0; reported=0
+        while IFS= read -r -d '' f; do
+          reported=$((reported + 1))
+          [ -s "$f" ] || continue
+          case "$(jq -r 'type' "$f" 2>/dev/null)" in
+            array) rows=$(printf '%s\n%s' "$rows" "$(jq -c . "$f")" | jq -s 'add') ;;
+            null)  ;; # ran none of the filtered package
+            *)     unreadable=$((unreadable + 1)) ;;
+          esac
+        done < <(find results -name windows-integration-results.json -print0 2>/dev/null)
+        # Dedupe so a re-shard reporting a test twice can't double every count.
+        rows=$(printf '%s' "$rows" | jq '[ .[] | objects ] | unique_by([.Package, .Test])')
+        count() { printf '%s' "$rows" | jq "[ (. // [])[] | select(.Outcome == \"$1\") ] | length"; }
+        retried=$(count retried)
+        passed=$(( $(count pass) + retried ))
+        failed=$(count fail)
+        skipped=$(count skip)
+        executed=$((passed + failed))
+        if [ "$executed" -gt 0 ]; then pct=$(( (passed * 100) / executed )); else pct=0; fi
+        # Names behind the retried count, capped so one bad run can't flood the message.
+        flaky=$(printf '%s' "$rows" | jq -r '
+          [ .[] | select(.Outcome == "retried") | (.Package | sub("^tailscale\\.com/"; "")) + "." + .Test ]
+          | sort
+          | if length > 5 then (.[:5] | join(", ")) + ", and \(length - 5) more" else join(", ") end')
+
+        case "$RESULT" in
+          success)   emoji=":white_check_mark:"; color="good" ;;
+          failure)   emoji=":x:";                color="danger" ;;
+          cancelled) emoji=":no_entry_sign:";    color="warning" ;;
+          skipped)   emoji=":fast_forward:";     color="warning" ;;
+          *)         emoji=":question:";         color="warning" ;;
+        esac
+
+        tests="${passed} passed, ${failed} failed, ${skipped} skipped"
+        if [ "$executed" -gt 0 ]; then
+          tests="${tests} (${pct}%)"
+        fi
+        # Anything that makes the numbers incomplete or overstated is said out loud.
+        caveats=""
+        if [ "$unreadable" -gt 0 ]; then
+          caveats="${caveats} — ${unreadable} result file(s) unreadable"
+        fi
+        if [ "$reported" -lt "$EXPECTED_FILES" ]; then
+          caveats="${caveats} — only ${reported} of ${EXPECTED_FILES} result files reported"
+        fi
+        # Retries get their own line below, so they don't warn the count line.
+        if [ "$executed" -eq 0 ]; then
+          caveats="${caveats} — no tests executed"
+        fi
+        if [ "$RESULT" = "failure" ] && [ "$failed" -eq 0 ]; then
+          caveats="${caveats} — results may be partial (build error or timeout)"
+        fi
+        tests="${tests}${caveats}"
+        if [ "$failed" -gt 0 ]; then tests_emoji=":x:"
+        elif [ -n "$caveats" ]; then tests_emoji=":warning:"
+        else tests_emoji=":white_check_mark:"; fi
+
+        jq -n --arg emoji "$emoji" --arg status "$RESULT" --arg tests_emoji "$tests_emoji" \
+          --arg tests "$tests" --arg flaky "$flaky" --arg retried "$retried" \
+          --arg title "$TITLE" --arg header "$HEADER" --arg status_label "$STATUS_LABEL" \
+          --arg tests_label "$TESTS_LABEL" --arg color "$color" --arg run_url "$RUN_URL" \
+          '{attachments: [{
+            title: ($emoji + " " + $title + ": " + $status),
+            title_link: $run_url,
+            text: (
+              $header
+              + "\n\n" + (if $status_label == "" then "" else $emoji + " " + $status_label + ": " + $status + "\n" end)
+              + $tests_emoji + " " + $tests_label + ": " + $tests
+              + (if $flaky == "" then "" else "\n    ↳ :warning: Flaky Tests (" + $retried + " passed on retry): " + $flaky end)
+              + "\n\n<" + $run_url + "|View run details>"
+            ),
+            color: $color,
+            mrkdwn_in: ["text"]
+          }]}' > payload.json
@@ -862,6 +862,11 @@ jobs:
      - windows
    runs-on: ubuntu-24.04
    steps:
+    - name: checkout
+      uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+      with:
+        path: src
+        persist-credentials: false
    - name: Download Windows integration results
      uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
      with:
@@ -869,92 +874,15 @@ jobs:
        path: results
    - name: Build Slack payload
      id: payload
-      env:
-        WINDOWS_RESULT: ${{ needs.windows.result }}
-        PR_TITLE: ${{ github.event.pull_request.title }}
-        PR_NUMBER: ${{ github.event.pull_request.number }}
-        PR_URL: ${{ github.event.pull_request.html_url }}
-        # head.ref is untrusted on fork PRs; keep it in env and encode via jq --arg, never run: text.
-        HEAD_REF: ${{ github.event.pull_request.head.ref }}
-        RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
-        # Non-bench legs of the windows matrix; a missing artifact means a shard never reported.
-        EXPECTED_SHARDS: "2"
-      run: |
-        # Read each artifact separately so one unreadable file can't discard the others.
-        rows='[]'; unreadable=0; reported=0
-        while IFS= read -r -d '' f; do
-          reported=$((reported + 1))
-          [ -s "$f" ] || continue
-          case "$(jq -r 'type' "$f" 2>/dev/null)" in
-            array) rows=$(printf '%s\n%s' "$rows" "$(jq -c . "$f")" | jq -s 'add') ;;
-            null)  ;; # shard ran none of the filtered package
-            *)     unreadable=$((unreadable + 1)) ;;
-          esac
-        done < <(find results -name windows-integration-results.json -print0 2>/dev/null)
-        # Dedupe so a re-shard reporting a test twice can't double every count.
-        rows=$(printf '%s' "$rows" | jq '[ .[] | objects ] | unique_by([.Package, .Test])')
-        count() { printf '%s' "$rows" | jq "[ (. // [])[] | select(.Outcome == \"$1\") ] | length"; }
-        retried=$(count retried)
-        passed=$(( $(count pass) + retried ))
-        failed=$(count fail)
-        skipped=$(count skip)
-        executed=$((passed + failed))
-        if [ "$executed" -gt 0 ]; then pct=$(( (passed * 100) / executed )); else pct=0; fi
-        # Names behind the retried count, capped so one bad run can't flood the message.
-        flaky=$(printf '%s' "$rows" | jq -r '
-          [ .[] | select(.Outcome == "retried") | (.Package | sub("^tailscale\\.com/"; "")) + "." + .Test ]
-          | sort
-          | if length > 5 then (.[:5] | join(", ")) + ", and \(length - 5) more" else join(", ") end')
-
-        case "$WINDOWS_RESULT" in
-          success)   emoji=":white_check_mark:"; color="good" ;;
-          failure)   emoji=":x:";                color="danger" ;;
-          cancelled) emoji=":no_entry_sign:";    color="warning" ;;
-          skipped)   emoji=":fast_forward:";     color="warning" ;;
-          *)         emoji=":question:";         color="warning" ;;
-        esac
-
-        integration="${passed} passed, ${failed} failed, ${skipped} skipped"
-        if [ "$executed" -gt 0 ]; then
-          integration="${integration} (${pct}%)"
-        fi
-        # Anything that makes the numbers incomplete or overstated is said out loud.
-        caveats=""
-        if [ "$unreadable" -gt 0 ]; then
-          caveats="${caveats} — ${unreadable} shard artifact(s) unreadable"
-        fi
-        if [ "$reported" -lt "$EXPECTED_SHARDS" ]; then
-          caveats="${caveats} — only ${reported} of ${EXPECTED_SHARDS} shards reported"
-        fi
-        # Retries get their own line below, so they don't warn the count line.
-        if [ "$executed" -eq 0 ]; then
-          caveats="${caveats} — no tests executed"
-        fi
-        if [ "$WINDOWS_RESULT" = "failure" ] && [ "$failed" -eq 0 ]; then
-          caveats="${caveats} — results may be partial (build error or timeout)"
-        fi
-        integration="${integration}${caveats}"
-        if [ "$failed" -gt 0 ]; then int_emoji=":x:"
-        elif [ -n "$caveats" ]; then int_emoji=":warning:"
-        else int_emoji=":white_check_mark:"; fi
-
-        jq -n --arg emoji "$emoji" --arg status "$WINDOWS_RESULT" --arg int_emoji "$int_emoji" \
-          --arg integration "$integration" --arg flaky "$flaky" --arg retried "$retried" \
-          --arg branch "$HEAD_REF" --arg title "$PR_TITLE" --arg pr_url "$PR_URL" --arg pr_number "$PR_NUMBER" \
-          --arg color "$color" --arg run_url "$RUN_URL" \
-          '{attachments: [{
-            title: ($emoji + " " + $title + ": " + $status),
-            title_link: $run_url,
-            text: (
-              "<" + $pr_url + "|PR #" + $pr_number + "> · `" + $branch + "`"
-              + "\n\n" + $emoji + " Windows CI: " + $status
-              + "\n" + $int_emoji + " Windows Integration Tests: " + $integration
-              + (if $flaky == "" then "" else "\n    ↳ :warning: Flaky Tests (" + $retried + " passed on retry): " + $flaky end)
-              + "\n\n<" + $run_url + "|View run details>"
-            ),
-            color: $color,
-            mrkdwn_in: ["text"]
-          }]}' > payload.json
+      uses: ./src/.github/actions/windows-slack-payload
+      with:
+        result: ${{ needs.windows.result }}
+        title: ${{ github.event.pull_request.title }}
+        header: "<${{ github.event.pull_request.html_url }}|PR #${{ github.event.pull_request.number }}> · `${{ github.event.pull_request.head.ref }}`"
+        status-label: Windows CI
+        tests-label: Windows Integration Tests
+        # One result file per non-bench leg of the windows matrix.
+        expected-files: "2"
    - name: Send Slack notification
      env:
        WEBHOOK: ${{ secrets.WINDOWS_SLACK_WEBHOOK_URL }}
@@ -0,0 +1,131 @@
+# Runs the whole tstest/integration suite against the Windows service three times a night.
+name: windows-nightly
+
+on:
+  workflow_dispatch:
+
+  schedule:
+    # Three runs, 20 minutes apart and off the hour, to catch flaky tests.
+    - cron: "13,33,53 7 * * *"
+
+  pull_request:
+    paths:
+      - ".github/workflows/windows-nightly.yml"
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
+  cancel-in-progress: true
+
+env:
+  GOMODCACHE: ${{ github.workspace }}/gomodcache
+  CMD_GO_USE_GIT_HASH: "true"
+
+permissions: {}
+
+jobs:
+  nightly:
+    permissions:
+      id-token: write # request the identity JWT that authenticates to cigocached
+      contents: read # for actions/checkout
+    runs-on: ci-windows-github-1
+    name: Windows nightly (${{ matrix.suite }})
+    timeout-minutes: 30
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          # One row per suite, so smoke and regression suites can be added as rows later.
+          - suite: integration
+            packages: ./tstest/integration
+            results_pkg: tailscale.com/tstest/integration
+            flags: --run-windows-service-tests
+    steps:
+    - name: checkout
+      uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+      with:
+        path: ${{ github.workspace }}/src
+        persist-credentials: false
+
+    - name: Compute Go module cache key
+      id: hash
+      shell: bash
+      run: echo "key=gomod-cross3-${{ hashFiles('src/go.mod', 'src/go.sum') }}" >> "$GITHUB_OUTPUT"
+
+    - name: Restore Go module cache
+      uses: actions/cache/restore@668228422ae6a00e4ad889ee87cd7109ec5666a7 # v5.0.4
+      with:
+        path: gomodcache
+        key: ${{ steps.hash.outputs.key }}
+        enableCrossOsArchive: true
+
+    - name: Set up cigocacher
+      uses: ./src/.github/actions/go-cache
+      with:
+        checkout-path: ${{ github.workspace }}/src
+        cache-dir: ${{ github.workspace }}/cigocacher
+        cigocached-url: ${{ vars.CIGOCACHED_AZURE_URL }}
+        cigocached-host: ${{ vars.CIGOCACHED_AZURE_HOST }}
+
+    - name: test
+      shell: bash
+      working-directory: src
+      # -count=1: a cached pass is never retried, so flakes can't be reported.
+      run: ./tool/go run ./cmd/testwrapper $PACKAGES -count=1 $FLAGS
+      env:
+        PACKAGES: ${{ matrix.packages }}
+        FLAGS: ${{ matrix.flags }}
+        NOPWSHDEBUG: "true" # to quiet tool/gocross/gocross-wrapper.ps1 in CI
+        TS_TESTWRAPPER_RESULTS_SUMMARY: "1"
+        TS_TESTWRAPPER_RESULTS_PKG: ${{ matrix.results_pkg }}
+        TS_TESTWRAPPER_RESULTS_JSON: ${{ github.workspace }}/windows-integration-results.json
+
+    - name: Upload results
+      if: always()
+      uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0
+      with:
+        name: windows-nightly-${{ matrix.suite }}-results
+        path: ${{ github.workspace }}/windows-integration-results.json
+        if-no-files-found: ignore
+        retention-days: 90
+
+  notify:
+    if: always()
+    needs:
+      - nightly
+    permissions:
+      contents: read # for actions/checkout
+    runs-on: ubuntu-24.04
+    steps:
+    - name: checkout
+      uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+      with:
+        path: ${{ github.workspace }}/src
+        persist-credentials: false
+    - name: Download results
+      uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
+      with:
+        # A pattern, not a name, so a run that never uploaded still gets reported.
+        pattern: windows-nightly-*-results
+        path: results
+    - name: Build Slack payload
+      id: payload
+      if: always()
+      uses: ./src/.github/actions/windows-slack-payload
+      with:
+        result: ${{ needs.nightly.result }}
+        title: Windows Nightly Tests
+        header: "*${{ github.ref_name }}*"
+        tests-label: smoke and integration tests
+        # One result file per suite row.
+        expected-files: "1"
+    - name: Send Slack notification
+      env:
+        WEBHOOK: ${{ secrets.WINDOWS_SLACK_WEBHOOK_URL }}
+      # Posts on cancel too, so a missing message always means something went wrong.
+      if: always() && env.WEBHOOK != '' && github.event_name != 'pull_request'
+      continue-on-error: true
+      uses: slackapi/slack-github-action@91efab103c0de0a537f72a35f6b8cda0ee76bf0a # v2.1.1
+      with:
+        webhook: ${{ env.WEBHOOK }}
+        webhook-type: incoming-webhook
+        payload-file-path: payload.json