diff --git a/testsuite/fleettest.py b/testsuite/fleettest.py index e2deef36..95757a39 100755 --- a/testsuite/fleettest.py +++ b/testsuite/fleettest.py @@ -1170,6 +1170,23 @@ def main() -> int: # The expected-skip lists travel with the suite, so read workflows from the # tree that provides the tests. WORKFLOWS = TESTSUITE_REPO / ".github" / "workflows" + + # A tree that is OLDER than the suite being run against it -- a backport + # branch under --testsuite-repo -- cannot pass tests for fixes and features + # it does not carry, and cannot build the suite's newer unit-test helpers. + # It declares those in its own testsuite/skiplist/backport.txt, which is + # read from the BUILT tree, not the suite tree, because only the built tree + # knows what it lacks. The names are excluded outright (RSYNC_EXCLUDE) + # rather than declared as expected skips: some of them fail rather than + # skip, and an expected-skip list cannot describe a failure. + bp = REPO / "testsuite" / "skiplist" / "backport.txt" + if bp.is_file(): + names = [ln.split("#", 1)[0].strip() for ln in bp.read_text().splitlines()] + names = [x for x in names if x] + if names: + SKIP_CSV = ",".join(x for x in ([SKIP_CSV] + names) if x) + print(f"[backport] excluding {len(names)} test(s) declared in " + f"{bp.relative_to(REPO)}") if not args.cleanup: # The Python test suite (runtests.py + testsuite/) comes from # TESTSUITE_REPO, so that is where runtests.py must live. The build tree diff --git a/testsuite/skiplist-spec_test.py b/testsuite/skiplist-spec_test.py index d90d7d92..2fe0dde2 100644 --- a/testsuite/skiplist-spec_test.py +++ b/testsuite/skiplist-spec_test.py @@ -110,7 +110,15 @@ if got.startswith('exit:') or not got: # --- the committed lists --------------------------------------------------- -lists = sorted((SRC / 'testsuite' / 'skiplist').glob('*.txt')) +# backport.txt is not an expected-skip list: it is an EXCLUDE list that a +# backport branch ships to say which of this suite's tests it cannot run, and +# fleettest reads it from the BUILT tree rather than from any workflow. It only +# appears here when a backport tree has been overlaid with this suite, so it is +# exempt from the checks below that assume a workflow points at the file. +BACKPORT_LIST = 'backport.txt' + +lists = sorted(p for p in (SRC / 'testsuite' / 'skiplist').glob('*.txt') + if p.name != BACKPORT_LIST) if len(lists) < 5: test_fail(f'expected the per-platform skip lists, found {lists}') for path in lists: diff --git a/testsuite/skiplist/README.md b/testsuite/skiplist/README.md index 59c175ad..954dee71 100644 --- a/testsuite/skiplist/README.md +++ b/testsuite/skiplist/README.md @@ -67,3 +67,29 @@ target with `"protocols": [29]` is only pinned if its workflow actually has a `make check29` step composing `proto29.txt`. Today only the Ubuntu workflows do; a macOS or Cygwin target set to run protocol 29 gets no expected-skip oracle for that pass rather than a wrong one. + +## `backport.txt` — a different thing in the same directory + +A backport branch (`v3.4.1-sec-patches3`, `v3.2.7-sec-patches3`) is tested with +a NEWER suite than it shipped with, via +`fleettest.py --repo --testsuite-repo <3.5.0>`. Such a tree cannot +pass tests for fixes it does not carry, and cannot build unit-test helpers its +Makefile has never heard of. + +Those branches each carry their own `testsuite/skiplist/backport.txt`. It is +**not** an expected-skip list: + +* the other files here are `RSYNC_EXPECT_SKIPPED` oracles — "these should skip, + tell me if that changes"; +* `backport.txt` is an `RSYNC_EXCLUDE` list — "do not run these at all". + +It has to be an exclusion because some of the tests *fail* rather than skip on +an older tree, and an expected-skip list cannot describe a failure. + +`fleettest.py` reads it from the tree being BUILT (`--repo`), not from the tree +providing the suite, because only the built tree knows what it lacks. The +overlay that puts a newer `testsuite/` onto an older tree is a merge with no +delete, so a file that exists only on the backport survives it. + +`skiplist-spec` exempts this name from the rule that every committed list must +be referenced by a workflow: nothing references it, by design.