From bf18243bf7343560758ecd3832995cababbd3ca5 Mon Sep 17 00:00:00 2001 From: yaruk-byte Date: Wed, 19 Aug 2026 12:55:12 -0700 Subject: [PATCH] cmd/testwrapper: add Windows integration results panel + Slack notification (#20811) Updates #20464 Signed-off-by: Yaruk Asghar --- .github/workflows/test.yml | 83 +++++++++++++++++++++++++++++ cmd/testwrapper/testwrapper.go | 83 +++++++++++++++++++++++++++++ cmd/testwrapper/testwrapper_test.go | 70 ++++++++++++++++++++++++ 3 files changed, 236 insertions(+) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index a5952f720..9b1cde3da 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -267,6 +267,17 @@ jobs: run: ./tool/go run ./cmd/testwrapper sharded:${{ matrix.shard }} env: NOPWSHDEBUG: "true" # to quiet tool/gocross/gocross-wrapper.ps1 in CI + TS_TESTWRAPPER_RESULTS_SUMMARY: "1" + TS_TESTWRAPPER_RESULTS_PKG: tailscale.com/tstest/integration + TS_TESTWRAPPER_RESULTS_JSON: ${{ github.workspace }}/windows-integration-results.json + + - name: Upload Windows integration results + if: always() && matrix.key != 'win-bench' + uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 + with: + name: windows-integration-results-${{ matrix.key }} + path: ${{ github.workspace }}/windows-integration-results.json + if-no-files-found: ignore - name: bench all shell: bash @@ -925,6 +936,78 @@ jobs: }] } + notify_windows: + if: always() + needs: + - windows + runs-on: ubuntu-24.04 + steps: + - name: Download Windows integration results + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + pattern: windows-integration-results-* + path: results + - name: Build Slack payload + id: payload + env: + WINDOWS_RESULT: ${{ needs.windows.result }} + PR_TITLE: ${{ github.event.pull_request.title }} + PR_NUMBER: ${{ github.event.pull_request.number }} + PR_URL: ${{ github.event.pull_request.html_url }} + # head.ref is untrusted on fork PRs; keep it in env and encode via jq --arg, never run: text. + HEAD_REF: ${{ github.event.pull_request.head.ref }} + RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + run: | + rows=$(find results -name windows-integration-results.json -print0 \ + | xargs -0 jq -s 'add // []' 2>/dev/null || echo '[]') + count() { printf '%s' "$rows" | jq "[ (. // [])[] | select(.Outcome == \"$1\") ] | length"; } + passed=$(( $(count pass) + $(count retried) )) + failed=$(count fail) + skipped=$(count skip) + total=$((passed + failed + skipped)) + executed=$((passed + failed)) + if [ "$executed" -gt 0 ]; then pct=$(( (passed * 100) / executed )); else pct=0; fi + + case "$WINDOWS_RESULT" in + success) emoji=":white_check_mark:"; color="good" ;; + failure) emoji=":x:"; color="danger" ;; + cancelled) emoji=":no_entry_sign:"; color="warning" ;; + skipped) emoji=":fast_forward:"; color="warning" ;; + *) emoji=":question:"; color="warning" ;; + esac + + integration="${passed} passed, ${failed} failed, ${skipped} skipped (${pct}%)" + if { [ "$WINDOWS_RESULT" = "failure" ] && [ "$failed" -eq 0 ]; } || [ "$total" -eq 0 ]; then + int_emoji=":warning:"; integration="${integration} — results may be partial (build error or timeout)" + elif [ "$failed" -gt 0 ]; then int_emoji=":x:"; else int_emoji=":white_check_mark:"; fi + + jq -n --arg emoji "$emoji" --arg status "$WINDOWS_RESULT" --arg int_emoji "$int_emoji" \ + --arg integration "$integration" \ + --arg branch "$HEAD_REF" --arg title "$PR_TITLE" --arg pr_url "$PR_URL" --arg pr_number "$PR_NUMBER" \ + --arg color "$color" --arg run_url "$RUN_URL" \ + '{attachments: [{ + title: ($emoji + " " + $title + ": " + $status), + title_link: $run_url, + text: ( + "<" + $pr_url + "|PR #" + $pr_number + "> · `" + $branch + "`" + + "\n\n" + $emoji + " Windows CI: " + $status + + "\n" + $int_emoji + " Windows Integration Tests: " + $integration + + "\n\n<" + $run_url + "|View run details>" + ), + color: $color, + mrkdwn_in: ["text"] + }]}' > payload.json + - name: Send Slack notification + env: + WEBHOOK: ${{ secrets.WINDOWS_SLACK_WEBHOOK_URL }} + if: env.WEBHOOK != '' && github.event_name == 'pull_request' && needs.windows.result != 'cancelled' + continue-on-error: true + uses: slackapi/slack-github-action@91efab103c0de0a537f72a35f6b8cda0ee76bf0a # v2.1.1 + with: + webhook: ${{ env.WEBHOOK }} + webhook-type: incoming-webhook + payload-file-path: payload.json + merge_blocker: if: always() runs-on: ubuntu-24.04 diff --git a/cmd/testwrapper/testwrapper.go b/cmd/testwrapper/testwrapper.go index d1977691b..373835fd1 100644 --- a/cmd/testwrapper/testwrapper.go +++ b/cmd/testwrapper/testwrapper.go @@ -698,6 +698,76 @@ func writeFlakeSummary(path string, flaky []*failedTest, repo string) { } } +// writeResultsSummary renders the per-test results panel and JSON for the Windows results feature. +func writeResultsSummary(summaryPath, jsonPath, pkgOnly string, results map[string]testOutcome, retried map[string]bool, pkgFatal bool) { + type result struct { + Package string + Test string + Outcome string + } + var rows []result + for key, outcome := range results { + pkg, test, _ := strings.Cut(key, "\t") + if pkgOnly != "" && pkg != pkgOnly { + continue + } + out := string(outcome) + if outcome == outcomeFail && retried[key] { + out = "retried" + } + rows = append(rows, result{Package: pkg, Test: test, Outcome: out}) + } + slices.SortFunc(rows, func(a, b result) int { + return strings.Compare(a.Test, b.Test) + }) + + if jsonPath != "" { + if j, err := json.Marshal(rows); err != nil { + log.Printf("testwrapper: marshaling results JSON: %v", err) + } else if err := os.WriteFile(jsonPath, j, 0o644); err != nil { + log.Printf("testwrapper: writing results JSON %s: %v", jsonPath, err) + } + } + + f, err := os.OpenFile(summaryPath, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0o644) + if err != nil { + log.Printf("testwrapper: opening summary file %s: %v", summaryPath, err) + return + } + defer f.Close() + + title := "Windows integration test results" + if pkgOnly == "" { + title = "Test results" + } + fmt.Fprintf(f, "\n### %s\n\n", title) + if len(rows) == 0 { + fmt.Fprintln(f, "_No tests ran._") + return + } + var pass, fail, skip int + fmt.Fprintln(f, "| Result | Test |") + fmt.Fprintln(f, "|--------|------|") + for _, r := range rows { + var icon string + switch r.Outcome { + case "pass": + icon, pass = "✅", pass+1 + case "retried": + icon, pass = "✅ (retried)", pass+1 + case "skip": + icon, skip = "⚠️", skip+1 + default: + icon, fail = "❌", fail+1 + } + fmt.Fprintf(f, "| %s | `%s` |\n", icon, r.Test) + } + fmt.Fprintf(f, "\n**%d passed, %d failed, %d skipped**\n", pass, fail, skip) + if pkgFatal { + fmt.Fprintln(f, "\n_⚠️ A package did not complete (build error or timeout); results may be partial._") + } +} + // buildPackageTests groups failedTests by package into the wire format // flakeapp expects. // @@ -786,6 +856,9 @@ func main() { // First pass: run every package once, collect failed tests for retry. var failed []*failedTest var pkgFatal bool // a package produced a non-test fatal (build error, etc.) + + resultsSummary := os.Getenv("TS_TESTWRAPPER_RESULTS_SUMMARY") != "" + allResults := map[string]testOutcome{} for _, pkgPattern := range packages { pt := &packageTests{Pattern: pkgPattern} ch := make(chan *testAttempt) @@ -828,6 +901,9 @@ func main() { printPkgOutcome(tr.pkg, tr.outcome, tr.cached, tr.end.Sub(tr.start)) continue } + if resultsSummary && tr.testName != "" { + allResults[tr.pkg+"\t"+tr.testName] = tr.outcome + } if testingVerbose || tr.outcome == outcomeFail { io.Copy(os.Stdout, &tr.logs) } @@ -880,6 +956,13 @@ func main() { } if path := os.Getenv("GITHUB_STEP_SUMMARY"); path != "" { writeFlakeSummary(path, flaky, repo) + if resultsSummary { + retried := map[string]bool{} + for _, ft := range flaky { + retried[ft.pkg+"\t"+ft.testName] = true + } + writeResultsSummary(path, os.Getenv("TS_TESTWRAPPER_RESULTS_JSON"), os.Getenv("TS_TESTWRAPPER_RESULTS_PKG"), allResults, retried, pkgFatal) + } } if len(permanent) > 0 { j, _ := json.Marshal(buildPackageTests(permanent, "")) diff --git a/cmd/testwrapper/testwrapper_test.go b/cmd/testwrapper/testwrapper_test.go index 7dbc4e57b..263c3d237 100644 --- a/cmd/testwrapper/testwrapper_test.go +++ b/cmd/testwrapper/testwrapper_test.go @@ -32,10 +32,15 @@ func cmdTestwrapper(t *testing.T, args ...string) *exec.Cmd { cmd := exec.Command(buildPath, args...) // Tests of testwrapper run with a small per-test budget so they don't // take 10 minutes when checking permanent-failure behavior. + // Clear the summary env vars so a child testwrapper doesn't write stray output. cmd.Env = append(os.Environ(), "TS_TESTWRAPPER_BUDGET=2s", "TS_TESTWRAPPER_MIN_RETRIES=2", "GITHUB_REPOSITORY=tailscale/tailscale", + "GITHUB_STEP_SUMMARY=", + "TS_TESTWRAPPER_RESULTS_SUMMARY=", + "TS_TESTWRAPPER_RESULTS_JSON=", + "TS_TESTWRAPPER_RESULTS_PKG=", ) return cmd } @@ -505,3 +510,68 @@ func errExitCode(err error) (int, bool) { } return 0, false } + +// TestResultsSummary checks the Windows results panel/JSON is written only when gated on. +func TestResultsSummary(t *testing.T) { + t.Parallel() + + testfile := filepath.Join(t.TempDir(), "results_test.go") + code := []byte(`package results_test + +import "testing" + +func TestPass(t *testing.T) {} + +func TestSkip(t *testing.T) { t.Skip("nope") } + +func TestSub(t *testing.T) { + t.Run("a", func(t *testing.T) {}) + t.Run("b", func(t *testing.T) { t.Skip() }) +} +`) + if err := os.WriteFile(testfile, code, 0o644); err != nil { + t.Fatalf("writing package: %s", err) + } + + run := func(t *testing.T, gate bool) (summary, jsonOut string) { + t.Helper() + dir := t.TempDir() + summaryPath := filepath.Join(dir, "summary.md") + jsonPath := filepath.Join(dir, "results.json") + cmd := cmdTestwrapper(t, testfile) + cmd.Env = append(cmd.Env, "GITHUB_STEP_SUMMARY="+summaryPath) + summaryVal, jsonVal, pkgVal := "", "", "" + if gate { + summaryVal, jsonVal, pkgVal = "1", jsonPath, testfile + } + cmd.Env = append(cmd.Env, + "TS_TESTWRAPPER_RESULTS_SUMMARY="+summaryVal, + "TS_TESTWRAPPER_RESULTS_JSON="+jsonVal, + "TS_TESTWRAPPER_RESULTS_PKG="+pkgVal, + ) + if out, err := cmd.CombinedOutput(); err != nil { + t.Fatalf("testwrapper: %v\n%s", err, out) + } + s, _ := os.ReadFile(summaryPath) + j, _ := os.ReadFile(jsonPath) + return string(s), string(j) + } + + summary, jsonOut := run(t, true) + for _, want := range []string{"Windows integration test results", "`TestPass`", "`TestSkip`", "`TestSub`", "2 passed, 0 failed, 1 skipped"} { + if !strings.Contains(summary, want) { + t.Errorf("summary missing %q; got:\n%s", want, summary) + } + } + if !strings.Contains(jsonOut, `"Test":"TestSkip"`) || !strings.Contains(jsonOut, `"Outcome":"skip"`) { + t.Errorf("results JSON missing skip entry; got:\n%s", jsonOut) + } + + summaryOff, jsonOff := run(t, false) + if strings.Contains(summaryOff, "integration test results") { + t.Errorf("results panel written when gate unset:\n%s", summaryOff) + } + if jsonOff != "" { + t.Errorf("results JSON written when gate unset:\n%s", jsonOff) + } +}