cmd/testwrapper: add Windows integration results panel + Slack notification (#20811)

Updates #20464

Signed-off-by: Yaruk Asghar <yaruk@tailscale.com>

.github/workflows/test.yml

@@ -267,6 +267,17 @@ jobs:
       run: ./tool/go run ./cmd/testwrapper sharded:${{ matrix.shard }}
       env:
         NOPWSHDEBUG: "true" # to quiet tool/gocross/gocross-wrapper.ps1 in CI
+        TS_TESTWRAPPER_RESULTS_SUMMARY: "1"
+        TS_TESTWRAPPER_RESULTS_PKG: tailscale.com/tstest/integration
+        TS_TESTWRAPPER_RESULTS_JSON: ${{ github.workspace }}/windows-integration-results.json
+
+    - name: Upload Windows integration results
+      if: always() && matrix.key != 'win-bench'
+      uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0
+      with:
+        name: windows-integration-results-${{ matrix.key }}
+        path: ${{ github.workspace }}/windows-integration-results.json
+        if-no-files-found: ignore
 
     - name: bench all
       shell: bash
@@ -925,6 +936,78 @@ jobs:
             }]
           }
 
+  notify_windows:
+    if: always()
+    needs:
+      - windows
+    runs-on: ubuntu-24.04
+    steps:
+    - name: Download Windows integration results
+      uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
+      with:
+        pattern: windows-integration-results-*
+        path: results
+    - name: Build Slack payload
+      id: payload
+      env:
+        WINDOWS_RESULT: ${{ needs.windows.result }}
+        PR_TITLE: ${{ github.event.pull_request.title }}
+        PR_NUMBER: ${{ github.event.pull_request.number }}
+        PR_URL: ${{ github.event.pull_request.html_url }}
+        # head.ref is untrusted on fork PRs; keep it in env and encode via jq --arg, never run: text.
+        HEAD_REF: ${{ github.event.pull_request.head.ref }}
+        RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
+      run: |
+        rows=$(find results -name windows-integration-results.json -print0 \
+          | xargs -0 jq -s 'add // []' 2>/dev/null || echo '[]')
+        count() { printf '%s' "$rows" | jq "[ (. // [])[] | select(.Outcome == \"$1\") ] | length"; }
+        passed=$(( $(count pass) + $(count retried) ))
+        failed=$(count fail)
+        skipped=$(count skip)
+        total=$((passed + failed + skipped))
+        executed=$((passed + failed))
+        if [ "$executed" -gt 0 ]; then pct=$(( (passed * 100) / executed )); else pct=0; fi
+
+        case "$WINDOWS_RESULT" in
+          success)   emoji=":white_check_mark:"; color="good" ;;
+          failure)   emoji=":x:";                color="danger" ;;
+          cancelled) emoji=":no_entry_sign:";    color="warning" ;;
+          skipped)   emoji=":fast_forward:";     color="warning" ;;
+          *)         emoji=":question:";         color="warning" ;;
+        esac
+
+        integration="${passed} passed, ${failed} failed, ${skipped} skipped (${pct}%)"
+        if { [ "$WINDOWS_RESULT" = "failure" ] && [ "$failed" -eq 0 ]; } || [ "$total" -eq 0 ]; then
+          int_emoji=":warning:"; integration="${integration} — results may be partial (build error or timeout)"
+        elif [ "$failed" -gt 0 ]; then int_emoji=":x:"; else int_emoji=":white_check_mark:"; fi
+
+        jq -n --arg emoji "$emoji" --arg status "$WINDOWS_RESULT" --arg int_emoji "$int_emoji" \
+          --arg integration "$integration" \
+          --arg branch "$HEAD_REF" --arg title "$PR_TITLE" --arg pr_url "$PR_URL" --arg pr_number "$PR_NUMBER" \
+          --arg color "$color" --arg run_url "$RUN_URL" \
+          '{attachments: [{
+            title: ($emoji + " " + $title + ": " + $status),
+            title_link: $run_url,
+            text: (
+              "<" + $pr_url + "|PR #" + $pr_number + "> · `" + $branch + "`"
+              + "\n\n" + $emoji + " Windows CI: " + $status
+              + "\n" + $int_emoji + " Windows Integration Tests: " + $integration
+              + "\n\n<" + $run_url + "|View run details>"
+            ),
+            color: $color,
+            mrkdwn_in: ["text"]
+          }]}' > payload.json
+    - name: Send Slack notification
+      env:
+        WEBHOOK: ${{ secrets.WINDOWS_SLACK_WEBHOOK_URL }}
+      if: env.WEBHOOK != '' && github.event_name == 'pull_request' && needs.windows.result != 'cancelled'
+      continue-on-error: true
+      uses: slackapi/slack-github-action@91efab103c0de0a537f72a35f6b8cda0ee76bf0a # v2.1.1
+      with:
+        webhook: ${{ env.WEBHOOK }}
+        webhook-type: incoming-webhook
+        payload-file-path: payload.json
+
   merge_blocker:
     if: always()
     runs-on: ubuntu-24.04

cmd/testwrapper/testwrapper.go

@@ -698,6 +698,76 @@ func writeFlakeSummary(path string, flaky []*failedTest, repo string) {
 	}
 }
 
+// writeResultsSummary renders the per-test results panel and JSON for the Windows results feature.
+func writeResultsSummary(summaryPath, jsonPath, pkgOnly string, results map[string]testOutcome, retried map[string]bool, pkgFatal bool) {
+	type result struct {
+		Package string
+		Test    string
+		Outcome string
+	}
+	var rows []result
+	for key, outcome := range results {
+		pkg, test, _ := strings.Cut(key, "\t")
+		if pkgOnly != "" && pkg != pkgOnly {
+			continue
+		}
+		out := string(outcome)
+		if outcome == outcomeFail && retried[key] {
+			out = "retried"
+		}
+		rows = append(rows, result{Package: pkg, Test: test, Outcome: out})
+	}
+	slices.SortFunc(rows, func(a, b result) int {
+		return strings.Compare(a.Test, b.Test)
+	})
+
+	if jsonPath != "" {
+		if j, err := json.Marshal(rows); err != nil {
+			log.Printf("testwrapper: marshaling results JSON: %v", err)
+		} else if err := os.WriteFile(jsonPath, j, 0o644); err != nil {
+			log.Printf("testwrapper: writing results JSON %s: %v", jsonPath, err)
+		}
+	}
+
+	f, err := os.OpenFile(summaryPath, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0o644)
+	if err != nil {
+		log.Printf("testwrapper: opening summary file %s: %v", summaryPath, err)
+		return
+	}
+	defer f.Close()
+
+	title := "Windows integration test results"
+	if pkgOnly == "" {
+		title = "Test results"
+	}
+	fmt.Fprintf(f, "\n### %s\n\n", title)
+	if len(rows) == 0 {
+		fmt.Fprintln(f, "_No tests ran._")
+		return
+	}
+	var pass, fail, skip int
+	fmt.Fprintln(f, "| Result | Test |")
+	fmt.Fprintln(f, "|--------|------|")
+	for _, r := range rows {
+		var icon string
+		switch r.Outcome {
+		case "pass":
+			icon, pass = "✅", pass+1
+		case "retried":
+			icon, pass = "✅ (retried)", pass+1
+		case "skip":
+			icon, skip = "⚠️", skip+1
+		default:
+			icon, fail = "❌", fail+1
+		}
+		fmt.Fprintf(f, "| %s | `%s` |\n", icon, r.Test)
+	}
+	fmt.Fprintf(f, "\n**%d passed, %d failed, %d skipped**\n", pass, fail, skip)
+	if pkgFatal {
+		fmt.Fprintln(f, "\n_⚠️ A package did not complete (build error or timeout); results may be partial._")
+	}
+}
+
 // buildPackageTests groups failedTests by package into the wire format
 // flakeapp expects.
 //
@@ -786,6 +856,9 @@ func main() {
 	// First pass: run every package once, collect failed tests for retry.
 	var failed []*failedTest
 	var pkgFatal bool // a package produced a non-test fatal (build error, etc.)
+
+	resultsSummary := os.Getenv("TS_TESTWRAPPER_RESULTS_SUMMARY") != ""
+	allResults := map[string]testOutcome{}
 	for _, pkgPattern := range packages {
 		pt := &packageTests{Pattern: pkgPattern}
 		ch := make(chan *testAttempt)
@@ -828,6 +901,9 @@ func main() {
 				printPkgOutcome(tr.pkg, tr.outcome, tr.cached, tr.end.Sub(tr.start))
 				continue
 			}
+			if resultsSummary && tr.testName != "" {
+				allResults[tr.pkg+"\t"+tr.testName] = tr.outcome
+			}
 			if testingVerbose || tr.outcome == outcomeFail {
 				io.Copy(os.Stdout, &tr.logs)
 			}
@@ -880,6 +956,13 @@ func main() {
 	}
 	if path := os.Getenv("GITHUB_STEP_SUMMARY"); path != "" {
 		writeFlakeSummary(path, flaky, repo)
+		if resultsSummary {
+			retried := map[string]bool{}
+			for _, ft := range flaky {
+				retried[ft.pkg+"\t"+ft.testName] = true
+			}
+			writeResultsSummary(path, os.Getenv("TS_TESTWRAPPER_RESULTS_JSON"), os.Getenv("TS_TESTWRAPPER_RESULTS_PKG"), allResults, retried, pkgFatal)
+		}
 	}
 	if len(permanent) > 0 {
 		j, _ := json.Marshal(buildPackageTests(permanent, ""))

cmd/testwrapper/testwrapper_test.go

@@ -32,10 +32,15 @@ func cmdTestwrapper(t *testing.T, args ...string) *exec.Cmd {
 	cmd := exec.Command(buildPath, args...)
 	// Tests of testwrapper run with a small per-test budget so they don't
 	// take 10 minutes when checking permanent-failure behavior.
+	// Clear the summary env vars so a child testwrapper doesn't write stray output.
 	cmd.Env = append(os.Environ(),
 		"TS_TESTWRAPPER_BUDGET=2s",
 		"TS_TESTWRAPPER_MIN_RETRIES=2",
 		"GITHUB_REPOSITORY=tailscale/tailscale",
+		"GITHUB_STEP_SUMMARY=",
+		"TS_TESTWRAPPER_RESULTS_SUMMARY=",
+		"TS_TESTWRAPPER_RESULTS_JSON=",
+		"TS_TESTWRAPPER_RESULTS_PKG=",
 	)
 	return cmd
 }
@@ -505,3 +510,68 @@ func errExitCode(err error) (int, bool) {
 	}
 	return 0, false
 }
+
+// TestResultsSummary checks the Windows results panel/JSON is written only when gated on.
+func TestResultsSummary(t *testing.T) {
+	t.Parallel()
+
+	testfile := filepath.Join(t.TempDir(), "results_test.go")
+	code := []byte(`package results_test
+
+import "testing"
+
+func TestPass(t *testing.T) {}
+
+func TestSkip(t *testing.T) { t.Skip("nope") }
+
+func TestSub(t *testing.T) {
+	t.Run("a", func(t *testing.T) {})
+	t.Run("b", func(t *testing.T) { t.Skip() })
+}
+`)
+	if err := os.WriteFile(testfile, code, 0o644); err != nil {
+		t.Fatalf("writing package: %s", err)
+	}
+
+	run := func(t *testing.T, gate bool) (summary, jsonOut string) {
+		t.Helper()
+		dir := t.TempDir()
+		summaryPath := filepath.Join(dir, "summary.md")
+		jsonPath := filepath.Join(dir, "results.json")
+		cmd := cmdTestwrapper(t, testfile)
+		cmd.Env = append(cmd.Env, "GITHUB_STEP_SUMMARY="+summaryPath)
+		summaryVal, jsonVal, pkgVal := "", "", ""
+		if gate {
+			summaryVal, jsonVal, pkgVal = "1", jsonPath, testfile
+		}
+		cmd.Env = append(cmd.Env,
+			"TS_TESTWRAPPER_RESULTS_SUMMARY="+summaryVal,
+			"TS_TESTWRAPPER_RESULTS_JSON="+jsonVal,
+			"TS_TESTWRAPPER_RESULTS_PKG="+pkgVal,
+		)
+		if out, err := cmd.CombinedOutput(); err != nil {
+			t.Fatalf("testwrapper: %v\n%s", err, out)
+		}
+		s, _ := os.ReadFile(summaryPath)
+		j, _ := os.ReadFile(jsonPath)
+		return string(s), string(j)
+	}
+
+	summary, jsonOut := run(t, true)
+	for _, want := range []string{"Windows integration test results", "`TestPass`", "`TestSkip`", "`TestSub`", "2 passed, 0 failed, 1 skipped"} {
+		if !strings.Contains(summary, want) {
+			t.Errorf("summary missing %q; got:\n%s", want, summary)
+		}
+	}
+	if !strings.Contains(jsonOut, `"Test":"TestSkip"`) || !strings.Contains(jsonOut, `"Outcome":"skip"`) {
+		t.Errorf("results JSON missing skip entry; got:\n%s", jsonOut)
+	}
+
+	summaryOff, jsonOff := run(t, false)
+	if strings.Contains(summaryOff, "integration test results") {
+		t.Errorf("results panel written when gate unset:\n%s", summaryOff)
+	}
+	if jsonOff != "" {
+		t.Errorf("results JSON written when gate unset:\n%s", jsonOff)
+	}
+}