From 2f9bbe835c0fee46a0948b9ff381edbd23befea6 Mon Sep 17 00:00:00 2001 From: Andrew Tridgell Date: Fri, 31 Jul 2026 13:39:14 +1000 Subject: [PATCH] testsuite: keep the expected-skip lists in files, not in the workflow line Every branch that added a test which skips somewhere had to edit RSYNC_EXPECT_SKIPPED, a single ~3 KB YAML line duplicated across seven workflow steps -- so two such branches always conflicted, and the conflict was in the one format git cannot merge. The lists move to testsuite/skiplist/*.txt, one name per line with the reason as a comment, and RSYNC_EXPECT_SKIPPED takes @FILE entries which runtests.py expands (relative to srcdir, so out-of-tree builds work). Several compose, which lets the 46 names common to Linux/macOS/Cygwin live in one file: adding a require_tcp test now edits one line of common.txt instead of three lists in three files. Lists must be sorted, duplicate-free, and name real tests, and an unreadable or malformed list is a hard error -- it must never degrade to "expect no skips", which would silently disarm the oracle on that job. fleettest passes the spec through to the remote runtests.py, which expands it against the tree that was staged there. The oracle itself is unchanged. Verified on Linux by running the full suite in all three lanes (check, check30, check29) against the new files: same expected sets, all green. --- .github/workflows/almalinux-8-build.yml | 2 +- .github/workflows/cygwin-build.yml | 2 +- .github/workflows/macos-build.yml | 2 +- .github/workflows/ubuntu-22.04-build.yml | 6 +- .github/workflows/ubuntu-build.yml | 6 +- runtests.py | 68 +++++++++++++++++- testsuite/README.md | 17 +++-- testsuite/fleettest.py | 12 ++-- testsuite/skiplist-spec_test.py | 92 ++++++++++++++++++++++++ testsuite/skiplist/README.md | 55 ++++++++++++++ testsuite/skiplist/common.txt | 59 +++++++++++++++ testsuite/skiplist/cygwin.txt | 65 +++++++++++++++++ testsuite/skiplist/linux.txt | 10 +++ testsuite/skiplist/macos.txt | 32 +++++++++ testsuite/skiplist/proto29.txt | 16 +++++ 15 files changed, 425 insertions(+), 19 deletions(-) create mode 100644 testsuite/skiplist-spec_test.py create mode 100644 testsuite/skiplist/README.md create mode 100644 testsuite/skiplist/common.txt create mode 100644 testsuite/skiplist/cygwin.txt create mode 100644 testsuite/skiplist/linux.txt create mode 100644 testsuite/skiplist/macos.txt create mode 100644 testsuite/skiplist/proto29.txt diff --git a/.github/workflows/almalinux-8-build.yml b/.github/workflows/almalinux-8-build.yml index 9abb1110..7a9ab776 100644 --- a/.github/workflows/almalinux-8-build.yml +++ b/.github/workflows/almalinux-8-build.yml @@ -65,7 +65,7 @@ jobs: # crtimes-not-supported skip matches the other Linux jobs; # daemon-chroot-acl and proxy-response-line-too-long skip because # the default (secure) transport opens no listening socket. - run: RSYNC_EXPECT_SKIPPED=checksum-zero-blocklen,chroot-basis-forge-inner-module,crtimes,daemon-access-ip,daemon-argv-limit,daemon-chroot,daemon-chroot-acl,daemon-deny-dns-failopen,daemon-dot-file-force-wipe,daemon-http-proxy,daemon-leaf-type-race-fchmod,daemon-module-options,daemon-namecvt-empty-response,daemon-namecvt-newline-token,daemon-proxy-protocol,daemon-refuse-compress-threads-alias,daemon-standalone-detach,daemon-symlink-escape-matrix,daemon-zstd-thread-exhaustion,exclude-implied-trailing-backslash,inband-modname-leak,io-noop-flood-recursion,io-nosend-flood-recursion,io-readargs-argv-nullwrite,ki62-io-error-mask,macos-setgid-ordinary-mode-regression,malicious-dot-dir-delete-scope,malicious-dot-file-delete-scope,malicious-server-partial-basis-symlink-overwrite,match-append-empty-nullmap,match-want-i-nolen,partial-protected-regular-retry-policy,peer-legacy-implied-delete-scope,proto-cleared-dirflist,proto-cleared-ndx,proto-hlink-flag-oob,proto-hlink-gnum,proto-msg-info-assert,proto-sender-selftest,proto-subflist-freed,proxy-connect-request-too-long,proxy-protocol-trusted-peer,proxy-response-header-too-long,proxy-response-line-too-long,readonly-partial-abort-mode-regression,recv-discard-nullderef,recv-generator-acl-leak,uidlist-id0-name-leak,variety-symlink-traversal,xattr-wire-cap,xrsync make check + run: RSYNC_EXPECT_SKIPPED=@testsuite/skiplist/common.txt,@testsuite/skiplist/linux.txt make check - name: check (TCP daemon transport) # Second run exercising the real loopback-TCP daemon path. run: ./runtests.py --rsync-bin="$PWD/rsync" --use-tcp -j 8 diff --git a/.github/workflows/cygwin-build.yml b/.github/workflows/cygwin-build.yml index 373fdf4a..1a72ad4f 100644 --- a/.github/workflows/cygwin-build.yml +++ b/.github/workflows/cygwin-build.yml @@ -47,7 +47,7 @@ jobs: # package installed above), verified on a real Cygwin host. The real # chown/devices tests still skip (need root/mknod), as do the # RESOLVE_BENEATH symlink-race tests. - run: bash -c 'RSYNC_EXPECT_SKIPPED=acl-symlink-race,acls,acls-default,acls-depth,acls-unpinnable,backup-acl-xattr-cache,backup-dir-symlink-race,basis-xname-traversal,batch-file-symlink,checksum-zero-blocklen,chown,chroot-alt-dest-inner-module,chroot-basis-forge-inner-module,chroot-copy-dest-inner-module,chroot-link-dest-inner-module,chroot-receiver-write-inner-module,chroot-special-inner-module,copy-dest-symlink-readleak,copy-xattrs-symlink-race,daemon-access-ip,daemon-argv-limit,daemon-auth-group,daemon-chroot,daemon-chroot-acl,daemon-chroot-munge-default,daemon-config-symlink,daemon-deny-dns-failopen,daemon-dot-file-force-wipe,daemon-http-proxy,daemon-leaf-type-race-fchmod,daemon-max-alloc-zero,daemon-module-chdir-symlink,daemon-module-options,daemon-module-private-parent,daemon-namecvt-empty-response,daemon-namecvt-newline-token,daemon-proxy-protocol,daemon-refuse-compress-threads-alias,daemon-secrets-file-symlink,daemon-standalone-detach,daemon-symlink-escape-matrix,daemon-zstd-thread-exhaustion,devices,dir-sgid,early-input-symlink,exclude-implied-trailing-backslash,excludefrom-symlink,fake-super-acl-xattr,fake-super-backup-fifo-regression,files-from-leak,filter-leak,filter-merge-symlink,inband-modname-leak,insecure-links-admin-optout,io-noop-flood-recursion,io-nosend-flood-recursion,io-readargs-argv-nullwrite,ki62-io-error-mask,log-file-symlink,macos-setgid-ordinary-mode-regression,malicious-dot-dir-delete-scope,malicious-dot-file-delete-scope,malicious-server-partial-basis-symlink-overwrite,match-append-empty-nullmap,match-want-i-nolen,msg-io-timeout-overflow,nondaemon-symlink-race,nonroot-restrictive-perms,open-noatime,operator-path-backup-rmdir,operator-path-backup-symlink,operator-path-insecure-links-daemon,partial-protected-regular-retry-linux,partial-protected-regular-retry-policy,password-file-symlink,peer-legacy-implied-delete-scope,protected-regular,proto-cleared-dirflist,proto-cleared-ndx,proto-hlink-flag-oob,proto-hlink-gnum,proto-msg-info-assert,proto-sender-selftest,proto-subflist-freed,proxy-connect-request-too-long,proxy-protocol-trusted-peer,proxy-response-header-too-long,proxy-response-line-too-long,recv-discard-nullderef,recv-generator-acl-leak,rename-mixed-parent-transfer,rrsync-sender-leaf-flip,rrsync-sender-parent-pin,rrsync-symlink,sender-remove-source-root-anchor,simd-checksum,source-change-size-continues,symlink-dest-backupdir,symlink-exclude-xattr,symlink-race-dest,symlink-race-relative-dest,temp-dir-symlink-injection,uidlist-id0-name-leak,variety-symlink-traversal,xattr-wire-cap,xrsync make check' + run: bash -c 'RSYNC_EXPECT_SKIPPED=@testsuite/skiplist/common.txt,@testsuite/skiplist/cygwin.txt make check' - name: check (TCP daemon transport) # Second run with daemon tests over a real loopback rsyncd; the default # 'make check' above uses the secure stdio-pipe transport. diff --git a/.github/workflows/macos-build.yml b/.github/workflows/macos-build.yml index 77d21cf4..88080e97 100644 --- a/.github/workflows/macos-build.yml +++ b/.github/workflows/macos-build.yml @@ -47,7 +47,7 @@ jobs: # chown-fake / devices-fake / xattrs / xattrs-hlink now RUN on macOS # (rsyncfns.py drives xattrs via the `xattr` command), verified on a # real macOS host, so they're no longer in the skip set. - run: sudo RSYNC_EXPECT_SKIPPED=acl-symlink-race,acls-default,acls-depth,acls-unpinnable,backup-acl-xattr-cache,backup-crossdev-copy,checksum-zero-blocklen,chmod-temp-dir,chroot-basis-forge-inner-module,copy-xattrs-symlink-race,daemon-access-ip,daemon-argv-limit,daemon-auth-group,daemon-chroot,daemon-chroot-acl,daemon-deny-dns-failopen,daemon-dot-file-force-wipe,daemon-http-proxy,daemon-leaf-type-race-fchmod,daemon-max-alloc-zero,daemon-module-options,daemon-namecvt-empty-response,daemon-namecvt-newline-token,daemon-proxy-protocol,daemon-refuse-compress-threads-alias,daemon-standalone-detach,daemon-symlink-escape-matrix,daemon-zstd-thread-exhaustion,dir-sgid,exclude-implied-trailing-backslash,fake-super-acl-xattr,inband-modname-leak,io-noop-flood-recursion,io-nosend-flood-recursion,io-readargs-argv-nullwrite,ki62-io-error-mask,macos-setgid-ordinary-mode-regression,malicious-dot-dir-delete-scope,malicious-dot-file-delete-scope,malicious-server-partial-basis-symlink-overwrite,match-append-empty-nullmap,match-want-i-nolen,open-noatime,partial-protected-regular-retry-linux,peer-legacy-implied-delete-scope,preallocate,protected-regular,proto-cleared-dirflist,proto-cleared-ndx,proto-hlink-flag-oob,proto-hlink-gnum,proto-msg-info-assert,proto-sender-selftest,proto-subflist-freed,proxy-connect-request-too-long,proxy-protocol-trusted-peer,proxy-response-header-too-long,proxy-response-line-too-long,readonly-partial-abort-mode-regression,recv-discard-nullderef,recv-generator-acl-leak,rrsync-sender-leaf-flip,rrsync-sender-parent-pin,rrsync-symlink,sender-remove-source-root-anchor,simd-checksum,source-change-size-continues,sparse,symlink-exclude-xattr,uidlist-id0-name-leak,variety-symlink-traversal,xattr-wire-cap,xrsync make check + run: sudo RSYNC_EXPECT_SKIPPED=@testsuite/skiplist/common.txt,@testsuite/skiplist/macos.txt make check - name: check (TCP daemon transport) # Second run with daemon tests over a real loopback rsyncd; the default # 'make check' above uses the secure stdio-pipe transport. diff --git a/.github/workflows/ubuntu-22.04-build.yml b/.github/workflows/ubuntu-22.04-build.yml index 6966dfcc..8c110d94 100644 --- a/.github/workflows/ubuntu-22.04-build.yml +++ b/.github/workflows/ubuntu-22.04-build.yml @@ -42,11 +42,11 @@ jobs: - name: info run: rsync --version - name: check - run: sudo RSYNC_EXPECT_SKIPPED=checksum-zero-blocklen,chroot-basis-forge-inner-module,crtimes,daemon-access-ip,daemon-argv-limit,daemon-chroot,daemon-chroot-acl,daemon-deny-dns-failopen,daemon-dot-file-force-wipe,daemon-http-proxy,daemon-leaf-type-race-fchmod,daemon-module-options,daemon-namecvt-empty-response,daemon-namecvt-newline-token,daemon-proxy-protocol,daemon-refuse-compress-threads-alias,daemon-standalone-detach,daemon-symlink-escape-matrix,daemon-zstd-thread-exhaustion,exclude-implied-trailing-backslash,inband-modname-leak,io-noop-flood-recursion,io-nosend-flood-recursion,io-readargs-argv-nullwrite,ki62-io-error-mask,macos-setgid-ordinary-mode-regression,malicious-dot-dir-delete-scope,malicious-dot-file-delete-scope,malicious-server-partial-basis-symlink-overwrite,match-append-empty-nullmap,match-want-i-nolen,partial-protected-regular-retry-policy,peer-legacy-implied-delete-scope,proto-cleared-dirflist,proto-cleared-ndx,proto-hlink-flag-oob,proto-hlink-gnum,proto-msg-info-assert,proto-sender-selftest,proto-subflist-freed,proxy-connect-request-too-long,proxy-protocol-trusted-peer,proxy-response-header-too-long,proxy-response-line-too-long,readonly-partial-abort-mode-regression,recv-discard-nullderef,recv-generator-acl-leak,uidlist-id0-name-leak,variety-symlink-traversal,xattr-wire-cap,xrsync make check + run: sudo RSYNC_EXPECT_SKIPPED=@testsuite/skiplist/common.txt,@testsuite/skiplist/linux.txt make check - name: check30 - run: sudo RSYNC_EXPECT_SKIPPED=checksum-zero-blocklen,chroot-basis-forge-inner-module,crtimes,daemon-access-ip,daemon-argv-limit,daemon-chroot,daemon-chroot-acl,daemon-deny-dns-failopen,daemon-dot-file-force-wipe,daemon-http-proxy,daemon-leaf-type-race-fchmod,daemon-module-options,daemon-namecvt-empty-response,daemon-namecvt-newline-token,daemon-proxy-protocol,daemon-refuse-compress-threads-alias,daemon-standalone-detach,daemon-symlink-escape-matrix,daemon-zstd-thread-exhaustion,exclude-implied-trailing-backslash,inband-modname-leak,io-noop-flood-recursion,io-nosend-flood-recursion,io-readargs-argv-nullwrite,ki62-io-error-mask,macos-setgid-ordinary-mode-regression,malicious-dot-dir-delete-scope,malicious-dot-file-delete-scope,malicious-server-partial-basis-symlink-overwrite,match-append-empty-nullmap,match-want-i-nolen,partial-protected-regular-retry-policy,peer-legacy-implied-delete-scope,proto-cleared-dirflist,proto-cleared-ndx,proto-hlink-flag-oob,proto-hlink-gnum,proto-msg-info-assert,proto-sender-selftest,proto-subflist-freed,proxy-connect-request-too-long,proxy-protocol-trusted-peer,proxy-response-header-too-long,proxy-response-line-too-long,readonly-partial-abort-mode-regression,recv-discard-nullderef,recv-generator-acl-leak,uidlist-id0-name-leak,variety-symlink-traversal,xattr-wire-cap,xrsync make check30 + run: sudo RSYNC_EXPECT_SKIPPED=@testsuite/skiplist/common.txt,@testsuite/skiplist/linux.txt make check30 - name: check29 - run: sudo RSYNC_EXPECT_SKIPPED=acl-symlink-race,acls-unpinnable,checksum-zero-blocklen,chroot-basis-forge-inner-module,crtimes,daemon-access-ip,daemon-argv-limit,daemon-chroot,daemon-chroot-acl,daemon-copylinks-parent-target-regression,daemon-deny-dns-failopen,daemon-dot-file-force-wipe,daemon-http-proxy,daemon-leaf-type-race-fchmod,daemon-module-options,daemon-namecvt-empty-response,daemon-namecvt-newline-token,daemon-proxy-protocol,daemon-refuse-compress-threads-alias,daemon-standalone-detach,daemon-symlink-escape-matrix,daemon-zstd-thread-exhaustion,exclude-implied-trailing-backslash,inband-modname-leak,io-noop-flood-recursion,io-nosend-flood-recursion,io-readargs-argv-nullwrite,ki62-io-error-mask,macos-setgid-ordinary-mode-regression,malicious-dot-dir-delete-scope,malicious-dot-file-delete-scope,malicious-server-partial-basis-symlink-overwrite,match-append-empty-nullmap,match-want-i-nolen,partial-protected-regular-retry-linux,partial-protected-regular-retry-policy,peer-legacy-implied-delete-scope,proto-cleared-dirflist,proto-cleared-ndx,proto-hlink-flag-oob,proto-hlink-gnum,proto-msg-info-assert,proto-sender-selftest,proto-subflist-freed,proxy-connect-request-too-long,proxy-protocol-trusted-peer,proxy-response-header-too-long,proxy-response-line-too-long,readonly-partial-abort-mode-regression,recv-discard-nullderef,recv-generator-acl-leak,scanner-batch-flag-mismatch,symlink-exclude-xattr,uidlist-id0-name-leak,variety-symlink-traversal,xattr-wire-cap,xrsync make check29 + run: sudo RSYNC_EXPECT_SKIPPED=@testsuite/skiplist/common.txt,@testsuite/skiplist/linux.txt,@testsuite/skiplist/proto29.txt make check29 - name: check (TCP daemon transport) # Second run with daemon tests over a real loopback rsyncd; the default # 'make check' above uses the secure stdio-pipe transport. diff --git a/.github/workflows/ubuntu-build.yml b/.github/workflows/ubuntu-build.yml index a76a1a23..5fec4c88 100644 --- a/.github/workflows/ubuntu-build.yml +++ b/.github/workflows/ubuntu-build.yml @@ -38,11 +38,11 @@ jobs: - name: info run: rsync --version - name: check - run: sudo RSYNC_EXPECT_SKIPPED=checksum-zero-blocklen,chroot-basis-forge-inner-module,crtimes,daemon-access-ip,daemon-argv-limit,daemon-chroot,daemon-chroot-acl,daemon-deny-dns-failopen,daemon-dot-file-force-wipe,daemon-http-proxy,daemon-leaf-type-race-fchmod,daemon-module-options,daemon-namecvt-empty-response,daemon-namecvt-newline-token,daemon-proxy-protocol,daemon-refuse-compress-threads-alias,daemon-standalone-detach,daemon-symlink-escape-matrix,daemon-zstd-thread-exhaustion,exclude-implied-trailing-backslash,inband-modname-leak,io-noop-flood-recursion,io-nosend-flood-recursion,io-readargs-argv-nullwrite,ki62-io-error-mask,macos-setgid-ordinary-mode-regression,malicious-dot-dir-delete-scope,malicious-dot-file-delete-scope,malicious-server-partial-basis-symlink-overwrite,match-append-empty-nullmap,match-want-i-nolen,partial-protected-regular-retry-policy,peer-legacy-implied-delete-scope,proto-cleared-dirflist,proto-cleared-ndx,proto-hlink-flag-oob,proto-hlink-gnum,proto-msg-info-assert,proto-sender-selftest,proto-subflist-freed,proxy-connect-request-too-long,proxy-protocol-trusted-peer,proxy-response-header-too-long,proxy-response-line-too-long,readonly-partial-abort-mode-regression,recv-discard-nullderef,recv-generator-acl-leak,uidlist-id0-name-leak,variety-symlink-traversal,xattr-wire-cap,xrsync make check + run: sudo RSYNC_EXPECT_SKIPPED=@testsuite/skiplist/common.txt,@testsuite/skiplist/linux.txt make check - name: check30 - run: sudo RSYNC_EXPECT_SKIPPED=checksum-zero-blocklen,chroot-basis-forge-inner-module,crtimes,daemon-access-ip,daemon-argv-limit,daemon-chroot,daemon-chroot-acl,daemon-deny-dns-failopen,daemon-dot-file-force-wipe,daemon-http-proxy,daemon-leaf-type-race-fchmod,daemon-module-options,daemon-namecvt-empty-response,daemon-namecvt-newline-token,daemon-proxy-protocol,daemon-refuse-compress-threads-alias,daemon-standalone-detach,daemon-symlink-escape-matrix,daemon-zstd-thread-exhaustion,exclude-implied-trailing-backslash,inband-modname-leak,io-noop-flood-recursion,io-nosend-flood-recursion,io-readargs-argv-nullwrite,ki62-io-error-mask,macos-setgid-ordinary-mode-regression,malicious-dot-dir-delete-scope,malicious-dot-file-delete-scope,malicious-server-partial-basis-symlink-overwrite,match-append-empty-nullmap,match-want-i-nolen,partial-protected-regular-retry-policy,peer-legacy-implied-delete-scope,proto-cleared-dirflist,proto-cleared-ndx,proto-hlink-flag-oob,proto-hlink-gnum,proto-msg-info-assert,proto-sender-selftest,proto-subflist-freed,proxy-connect-request-too-long,proxy-protocol-trusted-peer,proxy-response-header-too-long,proxy-response-line-too-long,readonly-partial-abort-mode-regression,recv-discard-nullderef,recv-generator-acl-leak,uidlist-id0-name-leak,variety-symlink-traversal,xattr-wire-cap,xrsync make check30 + run: sudo RSYNC_EXPECT_SKIPPED=@testsuite/skiplist/common.txt,@testsuite/skiplist/linux.txt make check30 - name: check29 - run: sudo RSYNC_EXPECT_SKIPPED=acl-symlink-race,acls-unpinnable,checksum-zero-blocklen,chroot-basis-forge-inner-module,crtimes,daemon-access-ip,daemon-argv-limit,daemon-chroot,daemon-chroot-acl,daemon-copylinks-parent-target-regression,daemon-deny-dns-failopen,daemon-dot-file-force-wipe,daemon-http-proxy,daemon-leaf-type-race-fchmod,daemon-module-options,daemon-namecvt-empty-response,daemon-namecvt-newline-token,daemon-proxy-protocol,daemon-refuse-compress-threads-alias,daemon-standalone-detach,daemon-symlink-escape-matrix,daemon-zstd-thread-exhaustion,exclude-implied-trailing-backslash,inband-modname-leak,io-noop-flood-recursion,io-nosend-flood-recursion,io-readargs-argv-nullwrite,ki62-io-error-mask,macos-setgid-ordinary-mode-regression,malicious-dot-dir-delete-scope,malicious-dot-file-delete-scope,malicious-server-partial-basis-symlink-overwrite,match-append-empty-nullmap,match-want-i-nolen,partial-protected-regular-retry-linux,partial-protected-regular-retry-policy,peer-legacy-implied-delete-scope,proto-cleared-dirflist,proto-cleared-ndx,proto-hlink-flag-oob,proto-hlink-gnum,proto-msg-info-assert,proto-sender-selftest,proto-subflist-freed,proxy-connect-request-too-long,proxy-protocol-trusted-peer,proxy-response-header-too-long,proxy-response-line-too-long,readonly-partial-abort-mode-regression,recv-discard-nullderef,recv-generator-acl-leak,scanner-batch-flag-mismatch,symlink-exclude-xattr,uidlist-id0-name-leak,variety-symlink-traversal,xattr-wire-cap,xrsync make check29 + run: sudo RSYNC_EXPECT_SKIPPED=@testsuite/skiplist/common.txt,@testsuite/skiplist/linux.txt,@testsuite/skiplist/proto29.txt make check29 - name: check (TCP daemon transport) # Second run with daemon tests over a real loopback rsyncd. The default # 'make check' above uses the secure stdio-pipe transport (no listening diff --git a/runtests.py b/runtests.py index 55542554..e3aca8fa 100755 --- a/runtests.py +++ b/runtests.py @@ -81,7 +81,11 @@ def parse_args(): p.add_argument('--protocol', type=int, default=None, metavar='VER', help='Force protocol version (adds --protocol=VER to rsync)') p.add_argument('--expect-skipped', default=None, metavar='LIST', - help='Comma-separated list of expected-skipped tests') + help='Comma-separated list of expected-skipped tests. An ' + '@FILE entry reads a skip list (one test per line, ' + '"#" comments); relative paths resolve against srcdir ' + 'and several may be composed, e.g. ' + '@testsuite/skiplist/linux.txt,@testsuite/skiplist/proto29.txt') p.add_argument('--expect-result', default=None, metavar='FILE', help='Path to an expected-outcome manifest (one ' '" " per line). When ' @@ -271,6 +275,66 @@ def parse_expect_result(path): return expect +def expand_skip_spec(spec, srcdir, suitedir): + """Expand an RSYNC_EXPECT_SKIPPED spec into a normalised csv. + + The spec is a comma-separated list of test names and/or '@FILE' skip-list + references. A skip-list file holds one test name per line ('#' starts a + comment; blank lines are ignored), which is what keeps two branches from + colliding: adding a test edits one line of one file rather than a shared + 3 KB csv. Several may be composed, e.g. + RSYNC_EXPECT_SKIPPED=@testsuite/skiplist/linux.txt,@.../proto29.txt + Relative paths resolve against srcdir (not the cwd) so out-of-tree builds + and `make installcheck` work. + + Entries must name a real test, and each file must be sorted and free of + duplicates: unsorted files defeat the point (everyone appends to the same + last line), and a stale name would otherwise fail as a skip mismatch far + from its cause. Exits 2 on any of those. + """ + def die(msg): + sys.stderr.write(msg + '\n') + sys.exit(Exit.ERROR) + + names = [] + for tok in (t.strip() for t in spec.split(',')): + if not tok: + continue + if not tok.startswith('@'): + names.append((tok, 'RSYNC_EXPECT_SKIPPED')) + continue + path = tok[1:] + if not os.path.isabs(path): + path = os.path.join(srcdir, path) + try: + with open(path) as f: + lines = f.readlines() + except OSError as e: + die(f'{tok}: cannot read skip list: {e}') + prev = None + for lineno, raw in enumerate(lines, 1): + name = raw.split('#', 1)[0].strip() + if not name: + continue + where = f'{path}:{lineno}' + if len(name.split()) != 1: + die(f'{where}: expected one test name per line, got: {raw.rstrip()}') + if prev is not None and name <= prev: + die(f'{where}: skip lists must be sorted and duplicate-free ' + f'({name!r} follows {prev!r})') + prev = name + names.append((name, where)) + + seen = {} + for name, where in names: + if name in seen: + continue + seen[name] = where + if not os.path.exists(os.path.join(suitedir, name + '_test.py')): + die(f'{where}: no such test: {name}') + return ','.join(sorted(seen)) + + def outcome_of(result): """Map a per-test exit code to an outcome string.""" if result == Exit.PASS: @@ -449,6 +513,8 @@ def main(): rsync_bin2 = os.path.abspath(rsync_bin2) suitedir = os.path.join(srcdir, 'testsuite') + if args.expect_skipped != 'IGNORE': + args.expect_skipped = expand_skip_spec(args.expect_skipped, srcdir, suitedir) scratchbase = os.path.join(os.environ.get('scratchbase', tooldir), 'testtmp') os.makedirs(scratchbase, exist_ok=True) diff --git a/testsuite/README.md b/testsuite/README.md index 4dc94d8c..f84e89c0 100644 --- a/testsuite/README.md +++ b/testsuite/README.md @@ -128,7 +128,12 @@ precondition and otherwise `SKIP` — read the individual test scripts for detai **Skip enforcement:** on a full run, set `RSYNC_EXPECT_SKIPPED=a,b,c` (or `--expect-skipped a,b,c`) and the run fails if the set of skipped tests does not -match. This is how the CI workflows pin each platform's expected skip set. +match. This is how the CI workflows pin each platform's expected skip set. An +`@FILE` entry reads a skip list (one test per line) instead, and several may be +composed: the workflows use +`@testsuite/skiplist/common.txt,@testsuite/skiplist/linux.txt`. Keeping the +lists one-name-per-line is what stops two branches that each add a skipping test +from conflicting -- see `testsuite/skiplist/README.md`. ### Scratch dirs and debugging @@ -182,10 +187,12 @@ fleet-config change. A target with `"protocols": [30, 29]` runs one extra stdio-pipe pass per listed version, each forcing that older wire version with `runtests --protocol=N` — the -fleet analogue of a workflow's `check30`/`check29` steps. The passes reuse the -same parsed `RSYNC_EXPECT_SKIPPED` list as the pipe run and show up as `protoNN` -columns in the report (and `--timing` breakdown). Targets that don't set -`protocols` show `-` there. +fleet analogue of a workflow's `check30`/`check29` steps. Each pass takes the +`RSYNC_EXPECT_SKIPPED` spec from the workflow's own `check30`/`check29` step, so +a lane with extra protocol-gated skips (`check29` adds +`@testsuite/skiplist/proto29.txt`) is enforced correctly. They show up as +`protoNN` columns in the report (and `--timing` breakdown); targets that don't +set `protocols` show `-` there. Run it from inside a checkout (it builds the current directory's HEAD; use `--repo PATH` for another tree): diff --git a/testsuite/fleettest.py b/testsuite/fleettest.py index 3c184487..0d02a915 100755 --- a/testsuite/fleettest.py +++ b/testsuite/fleettest.py @@ -13,8 +13,8 @@ list (matching the workflows), so only test FAILs matter there. A target may also list older "protocols" (e.g. [30, 29]) in the fleet config: each runs as an extra stdio-pipe pass with runtests --protocol=N (the fleet -analogue of a workflow's check30/check29 steps), using the same parsed skip list -as the pipe run, and shows up as a protoNN column in the report. +analogue of a workflow's check30/check29 steps), using that step's own parsed +skip list, and shows up as a protoNN column in the report. The fleet -- which machines, how to reach and build each -- is read from a JSON config: ~/.fleettest.json if present, else fleettest.json next to this script, @@ -261,12 +261,16 @@ def push_argv(target: Target, staging: str) -> list[str]: # --------------------------------------------------------------------------- def parse_workflow_skip(workflow: str, make_target: str = "check") -> str | None: - """Return the literal RSYNC_EXPECT_SKIPPED csv for the given `make ` + """Return the literal RSYNC_EXPECT_SKIPPED spec for the given `make ` step (check / check30 / check29), or None if that step leaves it unset. The protocol passes have their own check30/check29 lines (e.g. an xattr/ACL test that runs at proto 30 but skips at 29), so they must be parsed separately from the plain pipe `make check`. The trailing '? tolerates a `bash -c '... make - check'` wrapper (e.g. Cygwin).""" + check'` wrapper (e.g. Cygwin). + + The spec is passed through to the remote runtests.py verbatim; @FILE entries + (testsuite/skiplist/*.txt) are expanded there, against the staged tree, so + the list always matches the tests that shipped with it.""" path = WORKFLOWS / workflow try: text = path.read_text() diff --git a/testsuite/skiplist-spec_test.py b/testsuite/skiplist-spec_test.py new file mode 100644 index 00000000..6f6b9dd8 --- /dev/null +++ b/testsuite/skiplist-spec_test.py @@ -0,0 +1,92 @@ +#!/usr/bin/env python3 +# runtests.py's RSYNC_EXPECT_SKIPPED parser. The per-platform expected-skip +# lists live in testsuite/skiplist/*.txt and are referenced from the workflows +# as @FILE, so a bug here silently weakens the skip oracle on every CI job (an +# unreadable list must never read as "expect no skips"). Also checks that the +# committed lists themselves parse, are sorted, and name real tests. + +import importlib.util + +from rsyncfns import SCRATCHDIR, SRCDIR, test_fail + +spec = importlib.util.spec_from_file_location('runtests', SRCDIR / 'runtests.py') +runtests = importlib.util.module_from_spec(spec) +spec.loader.exec_module(runtests) + +SUITE = str(SRCDIR / 'testsuite') + + +def expand(text_spec): + """expand_skip_spec, but reporting a SystemExit as a string.""" + try: + return runtests.expand_skip_spec(text_spec, str(SRCDIR), SUITE) + except SystemExit as e: + return f'exit:{e.code}' + + +def write(name, body): + p = SCRATCHDIR / name + p.write_text(body) + return '@' + str(p) + + +# --- a bad spec must be a hard error, never a silently empty expectation ---- + +for what, arg in ( + ('missing file', '@testsuite/skiplist/does-not-exist.txt'), + ('unknown test name', 'no-such-test-here'), + ('unsorted list', write('unsorted.txt', 'zebra_z\nacls\n')), + ('duplicate entry', write('dup.txt', 'acls\nacls\n')), + ('two names on a line', write('twonames.txt', 'acls sparse\n')), + ('stale name in a list', write('stale.txt', 'gone-away-test\n')), +): + got = expand(arg) + if got != f'exit:{runtests.Exit.ERROR}': + test_fail(f'{what}: expected a hard error, got {got!r}') + +# --- good specs ------------------------------------------------------------ + +one = write('one.txt', '# comment\n\nacls # trailing comment\nsparse\n') +two = write('two.txt', 'devices\nsparse\n') + +if expand(one) != 'acls,sparse': + test_fail(f'@FILE expansion: got {expand(one)!r}') +if expand(f'{one},{two}') != 'acls,devices,sparse': + test_fail(f'composing two lists: got {expand(f"{one},{two}")!r}') +if expand(f'{one},crtimes') != 'acls,crtimes,sparse': + test_fail(f'mixing a bare name with @FILE: got {expand(f"{one},crtimes")!r}') +if expand('') != '': + test_fail('an empty spec must expand to an empty set') + +# A relative @FILE resolves against srcdir, not the cwd: `make check` in an +# out-of-tree build directory would otherwise not find the lists. +rel = expand('@testsuite/skiplist/common.txt') +if 'exit:' in rel or not rel: + test_fail(f'relative @FILE did not resolve against srcdir: {rel!r}') + +# --- the committed lists --------------------------------------------------- + +lists = sorted((SRCDIR / 'testsuite' / 'skiplist').glob('*.txt')) +if len(lists) < 4: + test_fail(f'expected the per-platform skip lists, found {lists}') +for path in lists: + got = expand('@' + str(path)) + if got.startswith('exit:'): + test_fail(f'{path.name} does not parse (rerun to see the diagnostic)') + +# The workflows must reference lists that exist and parse. Failing here means +# a workflow points at a list that was renamed or removed. +wf = sorted((SRCDIR / '.github' / 'workflows').glob('*.yml')) +refs = 0 +for path in wf: + for line in path.read_text().splitlines(): + if 'RSYNC_EXPECT_SKIPPED=' not in line: + continue + arg = line.split('RSYNC_EXPECT_SKIPPED=', 1)[1].split()[0] + got = expand(arg) + if got.startswith('exit:'): + test_fail(f'{path.name}: unusable RSYNC_EXPECT_SKIPPED spec {arg!r}') + refs += 1 +if wf and refs == 0: + test_fail('no workflow references RSYNC_EXPECT_SKIPPED any more') +print(f'ok: {len(lists)} skip lists, {refs} workflow references') diff --git a/testsuite/skiplist/README.md b/testsuite/skiplist/README.md new file mode 100644 index 00000000..4b205173 --- /dev/null +++ b/testsuite/skiplist/README.md @@ -0,0 +1,55 @@ +# Expected-skip lists + +`runtests.py` checks, on a full run, that the set of tests which skipped is +*exactly* the set that was expected. That oracle is what stops a test from +quietly turning into a permanent no-op: if a test starts skipping (a probe +regresses, a helper goes missing) the run fails instead of reporting green. + +The expected set is passed in `RSYNC_EXPECT_SKIPPED`. It used to be one huge +comma-separated line per CI job, duplicated across seven workflow steps, which +meant every branch that added a skipping test edited the same line in the same +files and conflicted with every other such branch. The lists now live here, one +test name per line, and a workflow step references them: + +```yaml +run: RSYNC_EXPECT_SKIPPED=@testsuite/skiplist/common.txt,@testsuite/skiplist/linux.txt make check +``` + +Adding a test therefore touches one line of one file, and two branches adding +different tests merge cleanly. + +## Files + +| file | contents | +| --- | --- | +| `common.txt` | skipped on every platform that runs the oracle — mostly `require_tcp` / `require_asan` tests, which the default stdio-pipe `make check` cannot satisfy | +| `linux.txt` | Linux-only additions | +| `macos.txt` | macOS-only additions | +| `cygwin.txt` | Cygwin-only additions | +| `proto29.txt` | additions for a `--protocol=29` run, on any platform | + +Compose them with commas; the result is the union, so listing a test twice is +harmless. Plain test names may be mixed in with `@FILE` entries. + +## Format rules + +One test name per line. `#` starts a comment; a trailing `# reason` is +encouraged — it is the only place the reason for the expectation is recorded. +Each file must be **sorted and duplicate-free**, and every name must match a +real `testsuite/_test.py`; `runtests.py` errors out otherwise. Sorting is +not cosmetic: it is what makes two independent additions land on different +lines. + +Relative `@FILE` paths resolve against `srcdir`, so out-of-tree builds and +`make installcheck` work. + +## Changing a list + +If a test newly skips on a platform, prefer fixing the test so it does not skip. +When the skip is legitimate, add the name to the narrowest file that fits, with +a reason. Do not paper over a mismatch by adding a name you cannot explain — an +unexpected skip is usually a real regression in that test's setup. + +`testsuite/fleettest.py` reads these same lists (through each target's +workflow), and merges per-box `expect_skip_extra` from `fleettest.json` on top +for facts that are true of one machine rather than one platform. diff --git a/testsuite/skiplist/common.txt b/testsuite/skiplist/common.txt new file mode 100644 index 00000000..ea4329fd --- /dev/null +++ b/testsuite/skiplist/common.txt @@ -0,0 +1,59 @@ +# Tests expected to SKIP. One name per line, '#' starts a comment; the file +# must stay sorted and duplicate-free (runtests.py enforces both). Referenced +# from a workflow as RSYNC_EXPECT_SKIPPED=@testsuite/skiplist/[,@...]. +# See testsuite/skiplist/README.md. +# +# Skipped on every platform that runs the oracle (Linux, macOS, Cygwin). +# Mostly tests that need the real-TCP transport (require_tcp) or an ASan +# build (require_asan), neither of which the default `make check` provides. +# Reasons below are the ones this box reported; another platform may skip +# the same test earlier for a different reason. + +checksum-zero-blocklen # the pure-Python receiver needs a real TCP socket; run with --use-tcp +chroot-basis-forge-inner-module # the pure-Python sender needs a real TCP daemon; run with --use-tcp +daemon-access-ip # hosts allow/deny address matching needs a real TCP peer +daemon-argv-limit # raw malicious daemon client needs a real TCP daemon; run with --use-tcp +daemon-chroot # daemon chroot path needs the real start_daemon socket flow +daemon-chroot-acl # needs a real TCP peer address for reverse-DNS hostname ACL; run with --use-tcp +daemon-deny-dns-failopen # hosts deny DNS-failure matching needs a real TCP peer +daemon-dot-file-force-wipe # the malicious daemon client needs a real TCP socket; run with --use-tcp +daemon-http-proxy # RSYNC_PROXY uses open_socket_out(), needs a real TCP connect +daemon-leaf-type-race-fchmod # leaf-type authority proof currently requires Darwin +daemon-module-options # motd/socket-options need a real socket +daemon-namecvt-empty-response # namecvt-empty-response test must run as a non-root user (sender uid/gid must differ from 0 to expose the bug) +daemon-namecvt-newline-token # the pure-Python sender needs a real TCP daemon; run with --use-tcp +daemon-proxy-protocol # PROXY-protocol header is read from a real TCP socket; run with --use-tcp +daemon-refuse-compress-threads-alias # the malicious receiver needs a real TCP daemon; run with --use-tcp +daemon-standalone-detach # standalone detaching daemon opens a real loopback listener; run with --use-tcp +daemon-symlink-escape-matrix # the daemon symlink-resolution flow needs a real TCP peer +daemon-zstd-thread-exhaustion # the malicious receiver needs a real TCP daemon; run with --use-tcp +exclude-implied-trailing-backslash # a real network rsync daemon is needed (am_server==0 per-module); run with --use-tcp +inband-modname-leak # the daemon @ERROR handshake needs a real TCP peer +io-noop-flood-recursion # the pure-Python client needs a real TCP daemon; run with --use-tcp +io-nosend-flood-recursion # the pure-Python client needs a real TCP daemon; run with --use-tcp +io-readargs-argv-nullwrite # the pure-Python client needs a real TCP daemon; run with --use-tcp +ki62-io-error-mask # needs --use-tcp to kill the daemon (sender) mid-transfer +macos-setgid-ordinary-mode-regression # macOS-specific fchmodat setgid behavior +malicious-dot-dir-delete-scope # the malicious sender needs a real TCP socket; run with --use-tcp +malicious-dot-file-delete-scope # the malicious sender needs a real TCP socket; run with --use-tcp +malicious-server-partial-basis-symlink-overwrite # the malicious sender needs a real TCP socket; run with --use-tcp +match-append-empty-nullmap # the pure-Python generator needs a real TCP daemon; run with --use-tcp +match-want-i-nolen # the pure-Python generator needs a real TCP daemon; run with --use-tcp +peer-legacy-implied-delete-scope # the malicious protocol-29 server needs TCP; run with --use-tcp +proto-cleared-dirflist # the pure-Python sender needs a real TCP daemon; run with --use-tcp +proto-cleared-ndx # the pure-Python sender needs a real TCP daemon; run with --use-tcp +proto-hlink-flag-oob # the pure-Python sender needs a real TCP daemon; run with --use-tcp +proto-hlink-gnum # the pure-Python sender needs a real TCP daemon; run with --use-tcp +proto-msg-info-assert # the pure-Python sender needs a real TCP daemon; run with --use-tcp +proto-sender-selftest # the pure-Python sender needs a real TCP daemon; run with --use-tcp +proto-subflist-freed # the pure-Python sender needs a real TCP daemon; run with --use-tcp +proxy-connect-request-too-long # fake-proxy listener needs a real TCP socket; run with --use-tcp +proxy-protocol-trusted-peer # PROXY protocol peer policy needs a real TCP daemon; run with --use-tcp +proxy-response-header-too-long # fake-proxy listener needs a real TCP socket; run with --use-tcp +proxy-response-line-too-long # fake-proxy listener needs a real TCP socket; run with --use-tcp +recv-discard-nullderef # root bypasses DAC: the unwritable dest dir wouldn't make the receiver's mkstemp fail, so the discard path (and the bug) is never reached +recv-generator-acl-leak # KI-23 real_sx ACL leak is only observable under AddressSanitizer/LSan +uidlist-id0-name-leak # KI-25 id-0 name leak is only observable under AddressSanitizer/LSan +variety-symlink-traversal # no old peer selected (RSYNC_PEER == RSYNC); nothing to compare for symlink-traversal divergence +xattr-wire-cap # the pure-Python daemon needs a real TCP socket; run with --use-tcp +xrsync # xrsync speaks the daemon protocol over a raw TCP socket; run with --use-tcp diff --git a/testsuite/skiplist/cygwin.txt b/testsuite/skiplist/cygwin.txt new file mode 100644 index 00000000..7b29c797 --- /dev/null +++ b/testsuite/skiplist/cygwin.txt @@ -0,0 +1,65 @@ +# Tests expected to SKIP. One name per line, '#' starts a comment; the file +# must stay sorted and duplicate-free (runtests.py enforces both). Referenced +# from a workflow as RSYNC_EXPECT_SKIPPED=@testsuite/skiplist/[,@...]. +# See testsuite/skiplist/README.md. +# +# Cygwin-only additions to common.txt. + +acl-symlink-race +acls +acls-default +acls-depth +acls-unpinnable +backup-acl-xattr-cache +backup-dir-symlink-race +basis-xname-traversal +batch-file-symlink +chown +chroot-alt-dest-inner-module +chroot-copy-dest-inner-module +chroot-link-dest-inner-module +chroot-receiver-write-inner-module +chroot-special-inner-module +copy-dest-symlink-readleak +copy-xattrs-symlink-race +daemon-auth-group +daemon-chroot-munge-default +daemon-config-symlink +daemon-max-alloc-zero +daemon-module-chdir-symlink +daemon-module-private-parent +daemon-secrets-file-symlink +devices +dir-sgid +early-input-symlink +excludefrom-symlink +fake-super-acl-xattr +fake-super-backup-fifo-regression +files-from-leak +filter-leak +filter-merge-symlink +insecure-links-admin-optout +log-file-symlink +msg-io-timeout-overflow +nondaemon-symlink-race +nonroot-restrictive-perms +open-noatime +operator-path-backup-rmdir +operator-path-backup-symlink +operator-path-insecure-links-daemon +partial-protected-regular-retry-linux +partial-protected-regular-retry-policy # deterministic partial EACCES recovery uses dyld interposing +password-file-symlink +protected-regular +rename-mixed-parent-transfer +rrsync-sender-leaf-flip +rrsync-sender-parent-pin +rrsync-symlink +sender-remove-source-root-anchor +simd-checksum +source-change-size-continues +symlink-dest-backupdir +symlink-exclude-xattr +symlink-race-dest +symlink-race-relative-dest +temp-dir-symlink-injection diff --git a/testsuite/skiplist/linux.txt b/testsuite/skiplist/linux.txt new file mode 100644 index 00000000..b89100d2 --- /dev/null +++ b/testsuite/skiplist/linux.txt @@ -0,0 +1,10 @@ +# Tests expected to SKIP. One name per line, '#' starts a comment; the file +# must stay sorted and duplicate-free (runtests.py enforces both). Referenced +# from a workflow as RSYNC_EXPECT_SKIPPED=@testsuite/skiplist/[,@...]. +# See testsuite/skiplist/README.md. +# +# Linux-only additions to common.txt. + +crtimes # Rsync is configured without crtimes support +partial-protected-regular-retry-policy # deterministic partial EACCES recovery uses dyld interposing +readonly-partial-abort-mode-regression # diff --git a/testsuite/skiplist/macos.txt b/testsuite/skiplist/macos.txt new file mode 100644 index 00000000..0fae0d8b --- /dev/null +++ b/testsuite/skiplist/macos.txt @@ -0,0 +1,32 @@ +# Tests expected to SKIP. One name per line, '#' starts a comment; the file +# must stay sorted and duplicate-free (runtests.py enforces both). Referenced +# from a workflow as RSYNC_EXPECT_SKIPPED=@testsuite/skiplist/[,@...]. +# See testsuite/skiplist/README.md. +# +# macOS-only additions to common.txt. + +acl-symlink-race +acls-default +acls-depth +acls-unpinnable +backup-acl-xattr-cache +backup-crossdev-copy +chmod-temp-dir +copy-xattrs-symlink-race +daemon-auth-group +daemon-max-alloc-zero +dir-sgid +fake-super-acl-xattr +open-noatime +partial-protected-regular-retry-linux +preallocate +protected-regular +readonly-partial-abort-mode-regression # +rrsync-sender-leaf-flip +rrsync-sender-parent-pin +rrsync-symlink +sender-remove-source-root-anchor +simd-checksum +source-change-size-continues +sparse +symlink-exclude-xattr diff --git a/testsuite/skiplist/proto29.txt b/testsuite/skiplist/proto29.txt new file mode 100644 index 00000000..2ec82b85 --- /dev/null +++ b/testsuite/skiplist/proto29.txt @@ -0,0 +1,16 @@ +# Tests expected to SKIP. One name per line, '#' starts a comment; the file +# must stay sorted and duplicate-free (runtests.py enforces both). Referenced +# from a workflow as RSYNC_EXPECT_SKIPPED=@testsuite/skiplist/[,@...]. +# See testsuite/skiplist/README.md. +# +# Additions for a --protocol=29 run (make check29), on top of the platform +# files. These tests transfer something the 29 wire format cannot carry +# (ACLs, xattrs) or drive a helper that speaks 30+, so they gate on the +# protocol regardless of platform. + +acl-symlink-race # ACL transfer requires protocol 30+ (negotiated 29) +acls-unpinnable # ACL transfer requires protocol 30+ (negotiated 29) +daemon-copylinks-parent-target-regression # the stdio_daemon client speaks protocol 30 (forced 29) +partial-protected-regular-retry-linux # one-inplace partial staging needs protocol >= 30 (forced 29) +scanner-batch-flag-mismatch # xattrs (-X) need protocol 30+ +symlink-exclude-xattr # xattr (-X) transfer requires protocol 30+ (negotiated 29)