diff --git a/.github/workflows/almalinux-8-build.yml b/.github/workflows/almalinux-8-build.yml index 9abb1110..7a9ab776 100644 --- a/.github/workflows/almalinux-8-build.yml +++ b/.github/workflows/almalinux-8-build.yml @@ -65,7 +65,7 @@ jobs: # crtimes-not-supported skip matches the other Linux jobs; # daemon-chroot-acl and proxy-response-line-too-long skip because # the default (secure) transport opens no listening socket. - run: RSYNC_EXPECT_SKIPPED=checksum-zero-blocklen,chroot-basis-forge-inner-module,crtimes,daemon-access-ip,daemon-argv-limit,daemon-chroot,daemon-chroot-acl,daemon-deny-dns-failopen,daemon-dot-file-force-wipe,daemon-http-proxy,daemon-leaf-type-race-fchmod,daemon-module-options,daemon-namecvt-empty-response,daemon-namecvt-newline-token,daemon-proxy-protocol,daemon-refuse-compress-threads-alias,daemon-standalone-detach,daemon-symlink-escape-matrix,daemon-zstd-thread-exhaustion,exclude-implied-trailing-backslash,inband-modname-leak,io-noop-flood-recursion,io-nosend-flood-recursion,io-readargs-argv-nullwrite,ki62-io-error-mask,macos-setgid-ordinary-mode-regression,malicious-dot-dir-delete-scope,malicious-dot-file-delete-scope,malicious-server-partial-basis-symlink-overwrite,match-append-empty-nullmap,match-want-i-nolen,partial-protected-regular-retry-policy,peer-legacy-implied-delete-scope,proto-cleared-dirflist,proto-cleared-ndx,proto-hlink-flag-oob,proto-hlink-gnum,proto-msg-info-assert,proto-sender-selftest,proto-subflist-freed,proxy-connect-request-too-long,proxy-protocol-trusted-peer,proxy-response-header-too-long,proxy-response-line-too-long,readonly-partial-abort-mode-regression,recv-discard-nullderef,recv-generator-acl-leak,uidlist-id0-name-leak,variety-symlink-traversal,xattr-wire-cap,xrsync make check + run: RSYNC_EXPECT_SKIPPED=@testsuite/skiplist/common.txt,@testsuite/skiplist/linux.txt make check - name: check (TCP daemon transport) # Second run exercising the real loopback-TCP daemon path. run: ./runtests.py --rsync-bin="$PWD/rsync" --use-tcp -j 8 diff --git a/.github/workflows/cygwin-build.yml b/.github/workflows/cygwin-build.yml index 373fdf4a..1a72ad4f 100644 --- a/.github/workflows/cygwin-build.yml +++ b/.github/workflows/cygwin-build.yml @@ -47,7 +47,7 @@ jobs: # package installed above), verified on a real Cygwin host. The real # chown/devices tests still skip (need root/mknod), as do the # RESOLVE_BENEATH symlink-race tests. - run: bash -c 'RSYNC_EXPECT_SKIPPED=acl-symlink-race,acls,acls-default,acls-depth,acls-unpinnable,backup-acl-xattr-cache,backup-dir-symlink-race,basis-xname-traversal,batch-file-symlink,checksum-zero-blocklen,chown,chroot-alt-dest-inner-module,chroot-basis-forge-inner-module,chroot-copy-dest-inner-module,chroot-link-dest-inner-module,chroot-receiver-write-inner-module,chroot-special-inner-module,copy-dest-symlink-readleak,copy-xattrs-symlink-race,daemon-access-ip,daemon-argv-limit,daemon-auth-group,daemon-chroot,daemon-chroot-acl,daemon-chroot-munge-default,daemon-config-symlink,daemon-deny-dns-failopen,daemon-dot-file-force-wipe,daemon-http-proxy,daemon-leaf-type-race-fchmod,daemon-max-alloc-zero,daemon-module-chdir-symlink,daemon-module-options,daemon-module-private-parent,daemon-namecvt-empty-response,daemon-namecvt-newline-token,daemon-proxy-protocol,daemon-refuse-compress-threads-alias,daemon-secrets-file-symlink,daemon-standalone-detach,daemon-symlink-escape-matrix,daemon-zstd-thread-exhaustion,devices,dir-sgid,early-input-symlink,exclude-implied-trailing-backslash,excludefrom-symlink,fake-super-acl-xattr,fake-super-backup-fifo-regression,files-from-leak,filter-leak,filter-merge-symlink,inband-modname-leak,insecure-links-admin-optout,io-noop-flood-recursion,io-nosend-flood-recursion,io-readargs-argv-nullwrite,ki62-io-error-mask,log-file-symlink,macos-setgid-ordinary-mode-regression,malicious-dot-dir-delete-scope,malicious-dot-file-delete-scope,malicious-server-partial-basis-symlink-overwrite,match-append-empty-nullmap,match-want-i-nolen,msg-io-timeout-overflow,nondaemon-symlink-race,nonroot-restrictive-perms,open-noatime,operator-path-backup-rmdir,operator-path-backup-symlink,operator-path-insecure-links-daemon,partial-protected-regular-retry-linux,partial-protected-regular-retry-policy,password-file-symlink,peer-legacy-implied-delete-scope,protected-regular,proto-cleared-dirflist,proto-cleared-ndx,proto-hlink-flag-oob,proto-hlink-gnum,proto-msg-info-assert,proto-sender-selftest,proto-subflist-freed,proxy-connect-request-too-long,proxy-protocol-trusted-peer,proxy-response-header-too-long,proxy-response-line-too-long,recv-discard-nullderef,recv-generator-acl-leak,rename-mixed-parent-transfer,rrsync-sender-leaf-flip,rrsync-sender-parent-pin,rrsync-symlink,sender-remove-source-root-anchor,simd-checksum,source-change-size-continues,symlink-dest-backupdir,symlink-exclude-xattr,symlink-race-dest,symlink-race-relative-dest,temp-dir-symlink-injection,uidlist-id0-name-leak,variety-symlink-traversal,xattr-wire-cap,xrsync make check' + run: bash -c 'RSYNC_EXPECT_SKIPPED=@testsuite/skiplist/common.txt,@testsuite/skiplist/cygwin.txt make check' - name: check (TCP daemon transport) # Second run with daemon tests over a real loopback rsyncd; the default # 'make check' above uses the secure stdio-pipe transport. diff --git a/.github/workflows/macos-build.yml b/.github/workflows/macos-build.yml index 77d21cf4..88080e97 100644 --- a/.github/workflows/macos-build.yml +++ b/.github/workflows/macos-build.yml @@ -47,7 +47,7 @@ jobs: # chown-fake / devices-fake / xattrs / xattrs-hlink now RUN on macOS # (rsyncfns.py drives xattrs via the `xattr` command), verified on a # real macOS host, so they're no longer in the skip set. - run: sudo RSYNC_EXPECT_SKIPPED=acl-symlink-race,acls-default,acls-depth,acls-unpinnable,backup-acl-xattr-cache,backup-crossdev-copy,checksum-zero-blocklen,chmod-temp-dir,chroot-basis-forge-inner-module,copy-xattrs-symlink-race,daemon-access-ip,daemon-argv-limit,daemon-auth-group,daemon-chroot,daemon-chroot-acl,daemon-deny-dns-failopen,daemon-dot-file-force-wipe,daemon-http-proxy,daemon-leaf-type-race-fchmod,daemon-max-alloc-zero,daemon-module-options,daemon-namecvt-empty-response,daemon-namecvt-newline-token,daemon-proxy-protocol,daemon-refuse-compress-threads-alias,daemon-standalone-detach,daemon-symlink-escape-matrix,daemon-zstd-thread-exhaustion,dir-sgid,exclude-implied-trailing-backslash,fake-super-acl-xattr,inband-modname-leak,io-noop-flood-recursion,io-nosend-flood-recursion,io-readargs-argv-nullwrite,ki62-io-error-mask,macos-setgid-ordinary-mode-regression,malicious-dot-dir-delete-scope,malicious-dot-file-delete-scope,malicious-server-partial-basis-symlink-overwrite,match-append-empty-nullmap,match-want-i-nolen,open-noatime,partial-protected-regular-retry-linux,peer-legacy-implied-delete-scope,preallocate,protected-regular,proto-cleared-dirflist,proto-cleared-ndx,proto-hlink-flag-oob,proto-hlink-gnum,proto-msg-info-assert,proto-sender-selftest,proto-subflist-freed,proxy-connect-request-too-long,proxy-protocol-trusted-peer,proxy-response-header-too-long,proxy-response-line-too-long,readonly-partial-abort-mode-regression,recv-discard-nullderef,recv-generator-acl-leak,rrsync-sender-leaf-flip,rrsync-sender-parent-pin,rrsync-symlink,sender-remove-source-root-anchor,simd-checksum,source-change-size-continues,sparse,symlink-exclude-xattr,uidlist-id0-name-leak,variety-symlink-traversal,xattr-wire-cap,xrsync make check + run: sudo RSYNC_EXPECT_SKIPPED=@testsuite/skiplist/common.txt,@testsuite/skiplist/macos.txt make check - name: check (TCP daemon transport) # Second run with daemon tests over a real loopback rsyncd; the default # 'make check' above uses the secure stdio-pipe transport. diff --git a/.github/workflows/ubuntu-22.04-build.yml b/.github/workflows/ubuntu-22.04-build.yml index 6966dfcc..8c110d94 100644 --- a/.github/workflows/ubuntu-22.04-build.yml +++ b/.github/workflows/ubuntu-22.04-build.yml @@ -42,11 +42,11 @@ jobs: - name: info run: rsync --version - name: check - run: sudo RSYNC_EXPECT_SKIPPED=checksum-zero-blocklen,chroot-basis-forge-inner-module,crtimes,daemon-access-ip,daemon-argv-limit,daemon-chroot,daemon-chroot-acl,daemon-deny-dns-failopen,daemon-dot-file-force-wipe,daemon-http-proxy,daemon-leaf-type-race-fchmod,daemon-module-options,daemon-namecvt-empty-response,daemon-namecvt-newline-token,daemon-proxy-protocol,daemon-refuse-compress-threads-alias,daemon-standalone-detach,daemon-symlink-escape-matrix,daemon-zstd-thread-exhaustion,exclude-implied-trailing-backslash,inband-modname-leak,io-noop-flood-recursion,io-nosend-flood-recursion,io-readargs-argv-nullwrite,ki62-io-error-mask,macos-setgid-ordinary-mode-regression,malicious-dot-dir-delete-scope,malicious-dot-file-delete-scope,malicious-server-partial-basis-symlink-overwrite,match-append-empty-nullmap,match-want-i-nolen,partial-protected-regular-retry-policy,peer-legacy-implied-delete-scope,proto-cleared-dirflist,proto-cleared-ndx,proto-hlink-flag-oob,proto-hlink-gnum,proto-msg-info-assert,proto-sender-selftest,proto-subflist-freed,proxy-connect-request-too-long,proxy-protocol-trusted-peer,proxy-response-header-too-long,proxy-response-line-too-long,readonly-partial-abort-mode-regression,recv-discard-nullderef,recv-generator-acl-leak,uidlist-id0-name-leak,variety-symlink-traversal,xattr-wire-cap,xrsync make check + run: sudo RSYNC_EXPECT_SKIPPED=@testsuite/skiplist/common.txt,@testsuite/skiplist/linux.txt make check - name: check30 - run: sudo RSYNC_EXPECT_SKIPPED=checksum-zero-blocklen,chroot-basis-forge-inner-module,crtimes,daemon-access-ip,daemon-argv-limit,daemon-chroot,daemon-chroot-acl,daemon-deny-dns-failopen,daemon-dot-file-force-wipe,daemon-http-proxy,daemon-leaf-type-race-fchmod,daemon-module-options,daemon-namecvt-empty-response,daemon-namecvt-newline-token,daemon-proxy-protocol,daemon-refuse-compress-threads-alias,daemon-standalone-detach,daemon-symlink-escape-matrix,daemon-zstd-thread-exhaustion,exclude-implied-trailing-backslash,inband-modname-leak,io-noop-flood-recursion,io-nosend-flood-recursion,io-readargs-argv-nullwrite,ki62-io-error-mask,macos-setgid-ordinary-mode-regression,malicious-dot-dir-delete-scope,malicious-dot-file-delete-scope,malicious-server-partial-basis-symlink-overwrite,match-append-empty-nullmap,match-want-i-nolen,partial-protected-regular-retry-policy,peer-legacy-implied-delete-scope,proto-cleared-dirflist,proto-cleared-ndx,proto-hlink-flag-oob,proto-hlink-gnum,proto-msg-info-assert,proto-sender-selftest,proto-subflist-freed,proxy-connect-request-too-long,proxy-protocol-trusted-peer,proxy-response-header-too-long,proxy-response-line-too-long,readonly-partial-abort-mode-regression,recv-discard-nullderef,recv-generator-acl-leak,uidlist-id0-name-leak,variety-symlink-traversal,xattr-wire-cap,xrsync make check30 + run: sudo RSYNC_EXPECT_SKIPPED=@testsuite/skiplist/common.txt,@testsuite/skiplist/linux.txt make check30 - name: check29 - run: sudo RSYNC_EXPECT_SKIPPED=acl-symlink-race,acls-unpinnable,checksum-zero-blocklen,chroot-basis-forge-inner-module,crtimes,daemon-access-ip,daemon-argv-limit,daemon-chroot,daemon-chroot-acl,daemon-copylinks-parent-target-regression,daemon-deny-dns-failopen,daemon-dot-file-force-wipe,daemon-http-proxy,daemon-leaf-type-race-fchmod,daemon-module-options,daemon-namecvt-empty-response,daemon-namecvt-newline-token,daemon-proxy-protocol,daemon-refuse-compress-threads-alias,daemon-standalone-detach,daemon-symlink-escape-matrix,daemon-zstd-thread-exhaustion,exclude-implied-trailing-backslash,inband-modname-leak,io-noop-flood-recursion,io-nosend-flood-recursion,io-readargs-argv-nullwrite,ki62-io-error-mask,macos-setgid-ordinary-mode-regression,malicious-dot-dir-delete-scope,malicious-dot-file-delete-scope,malicious-server-partial-basis-symlink-overwrite,match-append-empty-nullmap,match-want-i-nolen,partial-protected-regular-retry-linux,partial-protected-regular-retry-policy,peer-legacy-implied-delete-scope,proto-cleared-dirflist,proto-cleared-ndx,proto-hlink-flag-oob,proto-hlink-gnum,proto-msg-info-assert,proto-sender-selftest,proto-subflist-freed,proxy-connect-request-too-long,proxy-protocol-trusted-peer,proxy-response-header-too-long,proxy-response-line-too-long,readonly-partial-abort-mode-regression,recv-discard-nullderef,recv-generator-acl-leak,scanner-batch-flag-mismatch,symlink-exclude-xattr,uidlist-id0-name-leak,variety-symlink-traversal,xattr-wire-cap,xrsync make check29 + run: sudo RSYNC_EXPECT_SKIPPED=@testsuite/skiplist/common.txt,@testsuite/skiplist/linux.txt,@testsuite/skiplist/proto29.txt make check29 - name: check (TCP daemon transport) # Second run with daemon tests over a real loopback rsyncd; the default # 'make check' above uses the secure stdio-pipe transport. diff --git a/.github/workflows/ubuntu-build.yml b/.github/workflows/ubuntu-build.yml index a76a1a23..5fec4c88 100644 --- a/.github/workflows/ubuntu-build.yml +++ b/.github/workflows/ubuntu-build.yml @@ -38,11 +38,11 @@ jobs: - name: info run: rsync --version - name: check - run: sudo RSYNC_EXPECT_SKIPPED=checksum-zero-blocklen,chroot-basis-forge-inner-module,crtimes,daemon-access-ip,daemon-argv-limit,daemon-chroot,daemon-chroot-acl,daemon-deny-dns-failopen,daemon-dot-file-force-wipe,daemon-http-proxy,daemon-leaf-type-race-fchmod,daemon-module-options,daemon-namecvt-empty-response,daemon-namecvt-newline-token,daemon-proxy-protocol,daemon-refuse-compress-threads-alias,daemon-standalone-detach,daemon-symlink-escape-matrix,daemon-zstd-thread-exhaustion,exclude-implied-trailing-backslash,inband-modname-leak,io-noop-flood-recursion,io-nosend-flood-recursion,io-readargs-argv-nullwrite,ki62-io-error-mask,macos-setgid-ordinary-mode-regression,malicious-dot-dir-delete-scope,malicious-dot-file-delete-scope,malicious-server-partial-basis-symlink-overwrite,match-append-empty-nullmap,match-want-i-nolen,partial-protected-regular-retry-policy,peer-legacy-implied-delete-scope,proto-cleared-dirflist,proto-cleared-ndx,proto-hlink-flag-oob,proto-hlink-gnum,proto-msg-info-assert,proto-sender-selftest,proto-subflist-freed,proxy-connect-request-too-long,proxy-protocol-trusted-peer,proxy-response-header-too-long,proxy-response-line-too-long,readonly-partial-abort-mode-regression,recv-discard-nullderef,recv-generator-acl-leak,uidlist-id0-name-leak,variety-symlink-traversal,xattr-wire-cap,xrsync make check + run: sudo RSYNC_EXPECT_SKIPPED=@testsuite/skiplist/common.txt,@testsuite/skiplist/linux.txt make check - name: check30 - run: sudo RSYNC_EXPECT_SKIPPED=checksum-zero-blocklen,chroot-basis-forge-inner-module,crtimes,daemon-access-ip,daemon-argv-limit,daemon-chroot,daemon-chroot-acl,daemon-deny-dns-failopen,daemon-dot-file-force-wipe,daemon-http-proxy,daemon-leaf-type-race-fchmod,daemon-module-options,daemon-namecvt-empty-response,daemon-namecvt-newline-token,daemon-proxy-protocol,daemon-refuse-compress-threads-alias,daemon-standalone-detach,daemon-symlink-escape-matrix,daemon-zstd-thread-exhaustion,exclude-implied-trailing-backslash,inband-modname-leak,io-noop-flood-recursion,io-nosend-flood-recursion,io-readargs-argv-nullwrite,ki62-io-error-mask,macos-setgid-ordinary-mode-regression,malicious-dot-dir-delete-scope,malicious-dot-file-delete-scope,malicious-server-partial-basis-symlink-overwrite,match-append-empty-nullmap,match-want-i-nolen,partial-protected-regular-retry-policy,peer-legacy-implied-delete-scope,proto-cleared-dirflist,proto-cleared-ndx,proto-hlink-flag-oob,proto-hlink-gnum,proto-msg-info-assert,proto-sender-selftest,proto-subflist-freed,proxy-connect-request-too-long,proxy-protocol-trusted-peer,proxy-response-header-too-long,proxy-response-line-too-long,readonly-partial-abort-mode-regression,recv-discard-nullderef,recv-generator-acl-leak,uidlist-id0-name-leak,variety-symlink-traversal,xattr-wire-cap,xrsync make check30 + run: sudo RSYNC_EXPECT_SKIPPED=@testsuite/skiplist/common.txt,@testsuite/skiplist/linux.txt make check30 - name: check29 - run: sudo RSYNC_EXPECT_SKIPPED=acl-symlink-race,acls-unpinnable,checksum-zero-blocklen,chroot-basis-forge-inner-module,crtimes,daemon-access-ip,daemon-argv-limit,daemon-chroot,daemon-chroot-acl,daemon-copylinks-parent-target-regression,daemon-deny-dns-failopen,daemon-dot-file-force-wipe,daemon-http-proxy,daemon-leaf-type-race-fchmod,daemon-module-options,daemon-namecvt-empty-response,daemon-namecvt-newline-token,daemon-proxy-protocol,daemon-refuse-compress-threads-alias,daemon-standalone-detach,daemon-symlink-escape-matrix,daemon-zstd-thread-exhaustion,exclude-implied-trailing-backslash,inband-modname-leak,io-noop-flood-recursion,io-nosend-flood-recursion,io-readargs-argv-nullwrite,ki62-io-error-mask,macos-setgid-ordinary-mode-regression,malicious-dot-dir-delete-scope,malicious-dot-file-delete-scope,malicious-server-partial-basis-symlink-overwrite,match-append-empty-nullmap,match-want-i-nolen,partial-protected-regular-retry-linux,partial-protected-regular-retry-policy,peer-legacy-implied-delete-scope,proto-cleared-dirflist,proto-cleared-ndx,proto-hlink-flag-oob,proto-hlink-gnum,proto-msg-info-assert,proto-sender-selftest,proto-subflist-freed,proxy-connect-request-too-long,proxy-protocol-trusted-peer,proxy-response-header-too-long,proxy-response-line-too-long,readonly-partial-abort-mode-regression,recv-discard-nullderef,recv-generator-acl-leak,scanner-batch-flag-mismatch,symlink-exclude-xattr,uidlist-id0-name-leak,variety-symlink-traversal,xattr-wire-cap,xrsync make check29 + run: sudo RSYNC_EXPECT_SKIPPED=@testsuite/skiplist/common.txt,@testsuite/skiplist/linux.txt,@testsuite/skiplist/proto29.txt make check29 - name: check (TCP daemon transport) # Second run with daemon tests over a real loopback rsyncd. The default # 'make check' above uses the secure stdio-pipe transport (no listening diff --git a/runtests.py b/runtests.py index 55542554..e3aca8fa 100755 --- a/runtests.py +++ b/runtests.py @@ -81,7 +81,11 @@ def parse_args(): p.add_argument('--protocol', type=int, default=None, metavar='VER', help='Force protocol version (adds --protocol=VER to rsync)') p.add_argument('--expect-skipped', default=None, metavar='LIST', - help='Comma-separated list of expected-skipped tests') + help='Comma-separated list of expected-skipped tests. An ' + '@FILE entry reads a skip list (one test per line, ' + '"#" comments); relative paths resolve against srcdir ' + 'and several may be composed, e.g. ' + '@testsuite/skiplist/linux.txt,@testsuite/skiplist/proto29.txt') p.add_argument('--expect-result', default=None, metavar='FILE', help='Path to an expected-outcome manifest (one ' '" " per line). When ' @@ -271,6 +275,66 @@ def parse_expect_result(path): return expect +def expand_skip_spec(spec, srcdir, suitedir): + """Expand an RSYNC_EXPECT_SKIPPED spec into a normalised csv. + + The spec is a comma-separated list of test names and/or '@FILE' skip-list + references. A skip-list file holds one test name per line ('#' starts a + comment; blank lines are ignored), which is what keeps two branches from + colliding: adding a test edits one line of one file rather than a shared + 3 KB csv. Several may be composed, e.g. + RSYNC_EXPECT_SKIPPED=@testsuite/skiplist/linux.txt,@.../proto29.txt + Relative paths resolve against srcdir (not the cwd) so out-of-tree builds + and `make installcheck` work. + + Entries must name a real test, and each file must be sorted and free of + duplicates: unsorted files defeat the point (everyone appends to the same + last line), and a stale name would otherwise fail as a skip mismatch far + from its cause. Exits 2 on any of those. + """ + def die(msg): + sys.stderr.write(msg + '\n') + sys.exit(Exit.ERROR) + + names = [] + for tok in (t.strip() for t in spec.split(',')): + if not tok: + continue + if not tok.startswith('@'): + names.append((tok, 'RSYNC_EXPECT_SKIPPED')) + continue + path = tok[1:] + if not os.path.isabs(path): + path = os.path.join(srcdir, path) + try: + with open(path) as f: + lines = f.readlines() + except OSError as e: + die(f'{tok}: cannot read skip list: {e}') + prev = None + for lineno, raw in enumerate(lines, 1): + name = raw.split('#', 1)[0].strip() + if not name: + continue + where = f'{path}:{lineno}' + if len(name.split()) != 1: + die(f'{where}: expected one test name per line, got: {raw.rstrip()}') + if prev is not None and name <= prev: + die(f'{where}: skip lists must be sorted and duplicate-free ' + f'({name!r} follows {prev!r})') + prev = name + names.append((name, where)) + + seen = {} + for name, where in names: + if name in seen: + continue + seen[name] = where + if not os.path.exists(os.path.join(suitedir, name + '_test.py')): + die(f'{where}: no such test: {name}') + return ','.join(sorted(seen)) + + def outcome_of(result): """Map a per-test exit code to an outcome string.""" if result == Exit.PASS: @@ -449,6 +513,8 @@ def main(): rsync_bin2 = os.path.abspath(rsync_bin2) suitedir = os.path.join(srcdir, 'testsuite') + if args.expect_skipped != 'IGNORE': + args.expect_skipped = expand_skip_spec(args.expect_skipped, srcdir, suitedir) scratchbase = os.path.join(os.environ.get('scratchbase', tooldir), 'testtmp') os.makedirs(scratchbase, exist_ok=True) diff --git a/testsuite/README.md b/testsuite/README.md index 4dc94d8c..f84e89c0 100644 --- a/testsuite/README.md +++ b/testsuite/README.md @@ -128,7 +128,12 @@ precondition and otherwise `SKIP` — read the individual test scripts for detai **Skip enforcement:** on a full run, set `RSYNC_EXPECT_SKIPPED=a,b,c` (or `--expect-skipped a,b,c`) and the run fails if the set of skipped tests does not -match. This is how the CI workflows pin each platform's expected skip set. +match. This is how the CI workflows pin each platform's expected skip set. An +`@FILE` entry reads a skip list (one test per line) instead, and several may be +composed: the workflows use +`@testsuite/skiplist/common.txt,@testsuite/skiplist/linux.txt`. Keeping the +lists one-name-per-line is what stops two branches that each add a skipping test +from conflicting -- see `testsuite/skiplist/README.md`. ### Scratch dirs and debugging @@ -182,10 +187,12 @@ fleet-config change. A target with `"protocols": [30, 29]` runs one extra stdio-pipe pass per listed version, each forcing that older wire version with `runtests --protocol=N` — the -fleet analogue of a workflow's `check30`/`check29` steps. The passes reuse the -same parsed `RSYNC_EXPECT_SKIPPED` list as the pipe run and show up as `protoNN` -columns in the report (and `--timing` breakdown). Targets that don't set -`protocols` show `-` there. +fleet analogue of a workflow's `check30`/`check29` steps. Each pass takes the +`RSYNC_EXPECT_SKIPPED` spec from the workflow's own `check30`/`check29` step, so +a lane with extra protocol-gated skips (`check29` adds +`@testsuite/skiplist/proto29.txt`) is enforced correctly. They show up as +`protoNN` columns in the report (and `--timing` breakdown); targets that don't +set `protocols` show `-` there. Run it from inside a checkout (it builds the current directory's HEAD; use `--repo PATH` for another tree): diff --git a/testsuite/fleettest.py b/testsuite/fleettest.py index 3c184487..0d02a915 100755 --- a/testsuite/fleettest.py +++ b/testsuite/fleettest.py @@ -13,8 +13,8 @@ list (matching the workflows), so only test FAILs matter there. A target may also list older "protocols" (e.g. [30, 29]) in the fleet config: each runs as an extra stdio-pipe pass with runtests --protocol=N (the fleet -analogue of a workflow's check30/check29 steps), using the same parsed skip list -as the pipe run, and shows up as a protoNN column in the report. +analogue of a workflow's check30/check29 steps), using that step's own parsed +skip list, and shows up as a protoNN column in the report. The fleet -- which machines, how to reach and build each -- is read from a JSON config: ~/.fleettest.json if present, else fleettest.json next to this script, @@ -261,12 +261,16 @@ def push_argv(target: Target, staging: str) -> list[str]: # --------------------------------------------------------------------------- def parse_workflow_skip(workflow: str, make_target: str = "check") -> str | None: - """Return the literal RSYNC_EXPECT_SKIPPED csv for the given `make ` + """Return the literal RSYNC_EXPECT_SKIPPED spec for the given `make ` step (check / check30 / check29), or None if that step leaves it unset. The protocol passes have their own check30/check29 lines (e.g. an xattr/ACL test that runs at proto 30 but skips at 29), so they must be parsed separately from the plain pipe `make check`. The trailing '? tolerates a `bash -c '... make - check'` wrapper (e.g. Cygwin).""" + check'` wrapper (e.g. Cygwin). + + The spec is passed through to the remote runtests.py verbatim; @FILE entries + (testsuite/skiplist/*.txt) are expanded there, against the staged tree, so + the list always matches the tests that shipped with it.""" path = WORKFLOWS / workflow try: text = path.read_text() diff --git a/testsuite/skiplist-spec_test.py b/testsuite/skiplist-spec_test.py new file mode 100644 index 00000000..6f6b9dd8 --- /dev/null +++ b/testsuite/skiplist-spec_test.py @@ -0,0 +1,92 @@ +#!/usr/bin/env python3 +# runtests.py's RSYNC_EXPECT_SKIPPED parser. The per-platform expected-skip +# lists live in testsuite/skiplist/*.txt and are referenced from the workflows +# as @FILE, so a bug here silently weakens the skip oracle on every CI job (an +# unreadable list must never read as "expect no skips"). Also checks that the +# committed lists themselves parse, are sorted, and name real tests. + +import importlib.util + +from rsyncfns import SCRATCHDIR, SRCDIR, test_fail + +spec = importlib.util.spec_from_file_location('runtests', SRCDIR / 'runtests.py') +runtests = importlib.util.module_from_spec(spec) +spec.loader.exec_module(runtests) + +SUITE = str(SRCDIR / 'testsuite') + + +def expand(text_spec): + """expand_skip_spec, but reporting a SystemExit as a string.""" + try: + return runtests.expand_skip_spec(text_spec, str(SRCDIR), SUITE) + except SystemExit as e: + return f'exit:{e.code}' + + +def write(name, body): + p = SCRATCHDIR / name + p.write_text(body) + return '@' + str(p) + + +# --- a bad spec must be a hard error, never a silently empty expectation ---- + +for what, arg in ( + ('missing file', '@testsuite/skiplist/does-not-exist.txt'), + ('unknown test name', 'no-such-test-here'), + ('unsorted list', write('unsorted.txt', 'zebra_z\nacls\n')), + ('duplicate entry', write('dup.txt', 'acls\nacls\n')), + ('two names on a line', write('twonames.txt', 'acls sparse\n')), + ('stale name in a list', write('stale.txt', 'gone-away-test\n')), +): + got = expand(arg) + if got != f'exit:{runtests.Exit.ERROR}': + test_fail(f'{what}: expected a hard error, got {got!r}') + +# --- good specs ------------------------------------------------------------ + +one = write('one.txt', '# comment\n\nacls # trailing comment\nsparse\n') +two = write('two.txt', 'devices\nsparse\n') + +if expand(one) != 'acls,sparse': + test_fail(f'@FILE expansion: got {expand(one)!r}') +if expand(f'{one},{two}') != 'acls,devices,sparse': + test_fail(f'composing two lists: got {expand(f"{one},{two}")!r}') +if expand(f'{one},crtimes') != 'acls,crtimes,sparse': + test_fail(f'mixing a bare name with @FILE: got {expand(f"{one},crtimes")!r}') +if expand('') != '': + test_fail('an empty spec must expand to an empty set') + +# A relative @FILE resolves against srcdir, not the cwd: `make check` in an +# out-of-tree build directory would otherwise not find the lists. +rel = expand('@testsuite/skiplist/common.txt') +if 'exit:' in rel or not rel: + test_fail(f'relative @FILE did not resolve against srcdir: {rel!r}') + +# --- the committed lists --------------------------------------------------- + +lists = sorted((SRCDIR / 'testsuite' / 'skiplist').glob('*.txt')) +if len(lists) < 4: + test_fail(f'expected the per-platform skip lists, found {lists}') +for path in lists: + got = expand('@' + str(path)) + if got.startswith('exit:'): + test_fail(f'{path.name} does not parse (rerun to see the diagnostic)') + +# The workflows must reference lists that exist and parse. Failing here means +# a workflow points at a list that was renamed or removed. +wf = sorted((SRCDIR / '.github' / 'workflows').glob('*.yml')) +refs = 0 +for path in wf: + for line in path.read_text().splitlines(): + if 'RSYNC_EXPECT_SKIPPED=' not in line: + continue + arg = line.split('RSYNC_EXPECT_SKIPPED=', 1)[1].split()[0] + got = expand(arg) + if got.startswith('exit:'): + test_fail(f'{path.name}: unusable RSYNC_EXPECT_SKIPPED spec {arg!r}') + refs += 1 +if wf and refs == 0: + test_fail('no workflow references RSYNC_EXPECT_SKIPPED any more') +print(f'ok: {len(lists)} skip lists, {refs} workflow references') diff --git a/testsuite/skiplist/README.md b/testsuite/skiplist/README.md new file mode 100644 index 00000000..4b205173 --- /dev/null +++ b/testsuite/skiplist/README.md @@ -0,0 +1,55 @@ +# Expected-skip lists + +`runtests.py` checks, on a full run, that the set of tests which skipped is +*exactly* the set that was expected. That oracle is what stops a test from +quietly turning into a permanent no-op: if a test starts skipping (a probe +regresses, a helper goes missing) the run fails instead of reporting green. + +The expected set is passed in `RSYNC_EXPECT_SKIPPED`. It used to be one huge +comma-separated line per CI job, duplicated across seven workflow steps, which +meant every branch that added a skipping test edited the same line in the same +files and conflicted with every other such branch. The lists now live here, one +test name per line, and a workflow step references them: + +```yaml +run: RSYNC_EXPECT_SKIPPED=@testsuite/skiplist/common.txt,@testsuite/skiplist/linux.txt make check +``` + +Adding a test therefore touches one line of one file, and two branches adding +different tests merge cleanly. + +## Files + +| file | contents | +| --- | --- | +| `common.txt` | skipped on every platform that runs the oracle — mostly `require_tcp` / `require_asan` tests, which the default stdio-pipe `make check` cannot satisfy | +| `linux.txt` | Linux-only additions | +| `macos.txt` | macOS-only additions | +| `cygwin.txt` | Cygwin-only additions | +| `proto29.txt` | additions for a `--protocol=29` run, on any platform | + +Compose them with commas; the result is the union, so listing a test twice is +harmless. Plain test names may be mixed in with `@FILE` entries. + +## Format rules + +One test name per line. `#` starts a comment; a trailing `# reason` is +encouraged — it is the only place the reason for the expectation is recorded. +Each file must be **sorted and duplicate-free**, and every name must match a +real `testsuite/_test.py`; `runtests.py` errors out otherwise. Sorting is +not cosmetic: it is what makes two independent additions land on different +lines. + +Relative `@FILE` paths resolve against `srcdir`, so out-of-tree builds and +`make installcheck` work. + +## Changing a list + +If a test newly skips on a platform, prefer fixing the test so it does not skip. +When the skip is legitimate, add the name to the narrowest file that fits, with +a reason. Do not paper over a mismatch by adding a name you cannot explain — an +unexpected skip is usually a real regression in that test's setup. + +`testsuite/fleettest.py` reads these same lists (through each target's +workflow), and merges per-box `expect_skip_extra` from `fleettest.json` on top +for facts that are true of one machine rather than one platform. diff --git a/testsuite/skiplist/common.txt b/testsuite/skiplist/common.txt new file mode 100644 index 00000000..ea4329fd --- /dev/null +++ b/testsuite/skiplist/common.txt @@ -0,0 +1,59 @@ +# Tests expected to SKIP. One name per line, '#' starts a comment; the file +# must stay sorted and duplicate-free (runtests.py enforces both). Referenced +# from a workflow as RSYNC_EXPECT_SKIPPED=@testsuite/skiplist/[,@...]. +# See testsuite/skiplist/README.md. +# +# Skipped on every platform that runs the oracle (Linux, macOS, Cygwin). +# Mostly tests that need the real-TCP transport (require_tcp) or an ASan +# build (require_asan), neither of which the default `make check` provides. +# Reasons below are the ones this box reported; another platform may skip +# the same test earlier for a different reason. + +checksum-zero-blocklen # the pure-Python receiver needs a real TCP socket; run with --use-tcp +chroot-basis-forge-inner-module # the pure-Python sender needs a real TCP daemon; run with --use-tcp +daemon-access-ip # hosts allow/deny address matching needs a real TCP peer +daemon-argv-limit # raw malicious daemon client needs a real TCP daemon; run with --use-tcp +daemon-chroot # daemon chroot path needs the real start_daemon socket flow +daemon-chroot-acl # needs a real TCP peer address for reverse-DNS hostname ACL; run with --use-tcp +daemon-deny-dns-failopen # hosts deny DNS-failure matching needs a real TCP peer +daemon-dot-file-force-wipe # the malicious daemon client needs a real TCP socket; run with --use-tcp +daemon-http-proxy # RSYNC_PROXY uses open_socket_out(), needs a real TCP connect +daemon-leaf-type-race-fchmod # leaf-type authority proof currently requires Darwin +daemon-module-options # motd/socket-options need a real socket +daemon-namecvt-empty-response # namecvt-empty-response test must run as a non-root user (sender uid/gid must differ from 0 to expose the bug) +daemon-namecvt-newline-token # the pure-Python sender needs a real TCP daemon; run with --use-tcp +daemon-proxy-protocol # PROXY-protocol header is read from a real TCP socket; run with --use-tcp +daemon-refuse-compress-threads-alias # the malicious receiver needs a real TCP daemon; run with --use-tcp +daemon-standalone-detach # standalone detaching daemon opens a real loopback listener; run with --use-tcp +daemon-symlink-escape-matrix # the daemon symlink-resolution flow needs a real TCP peer +daemon-zstd-thread-exhaustion # the malicious receiver needs a real TCP daemon; run with --use-tcp +exclude-implied-trailing-backslash # a real network rsync daemon is needed (am_server==0 per-module); run with --use-tcp +inband-modname-leak # the daemon @ERROR handshake needs a real TCP peer +io-noop-flood-recursion # the pure-Python client needs a real TCP daemon; run with --use-tcp +io-nosend-flood-recursion # the pure-Python client needs a real TCP daemon; run with --use-tcp +io-readargs-argv-nullwrite # the pure-Python client needs a real TCP daemon; run with --use-tcp +ki62-io-error-mask # needs --use-tcp to kill the daemon (sender) mid-transfer +macos-setgid-ordinary-mode-regression # macOS-specific fchmodat setgid behavior +malicious-dot-dir-delete-scope # the malicious sender needs a real TCP socket; run with --use-tcp +malicious-dot-file-delete-scope # the malicious sender needs a real TCP socket; run with --use-tcp +malicious-server-partial-basis-symlink-overwrite # the malicious sender needs a real TCP socket; run with --use-tcp +match-append-empty-nullmap # the pure-Python generator needs a real TCP daemon; run with --use-tcp +match-want-i-nolen # the pure-Python generator needs a real TCP daemon; run with --use-tcp +peer-legacy-implied-delete-scope # the malicious protocol-29 server needs TCP; run with --use-tcp +proto-cleared-dirflist # the pure-Python sender needs a real TCP daemon; run with --use-tcp +proto-cleared-ndx # the pure-Python sender needs a real TCP daemon; run with --use-tcp +proto-hlink-flag-oob # the pure-Python sender needs a real TCP daemon; run with --use-tcp +proto-hlink-gnum # the pure-Python sender needs a real TCP daemon; run with --use-tcp +proto-msg-info-assert # the pure-Python sender needs a real TCP daemon; run with --use-tcp +proto-sender-selftest # the pure-Python sender needs a real TCP daemon; run with --use-tcp +proto-subflist-freed # the pure-Python sender needs a real TCP daemon; run with --use-tcp +proxy-connect-request-too-long # fake-proxy listener needs a real TCP socket; run with --use-tcp +proxy-protocol-trusted-peer # PROXY protocol peer policy needs a real TCP daemon; run with --use-tcp +proxy-response-header-too-long # fake-proxy listener needs a real TCP socket; run with --use-tcp +proxy-response-line-too-long # fake-proxy listener needs a real TCP socket; run with --use-tcp +recv-discard-nullderef # root bypasses DAC: the unwritable dest dir wouldn't make the receiver's mkstemp fail, so the discard path (and the bug) is never reached +recv-generator-acl-leak # KI-23 real_sx ACL leak is only observable under AddressSanitizer/LSan +uidlist-id0-name-leak # KI-25 id-0 name leak is only observable under AddressSanitizer/LSan +variety-symlink-traversal # no old peer selected (RSYNC_PEER == RSYNC); nothing to compare for symlink-traversal divergence +xattr-wire-cap # the pure-Python daemon needs a real TCP socket; run with --use-tcp +xrsync # xrsync speaks the daemon protocol over a raw TCP socket; run with --use-tcp diff --git a/testsuite/skiplist/cygwin.txt b/testsuite/skiplist/cygwin.txt new file mode 100644 index 00000000..7b29c797 --- /dev/null +++ b/testsuite/skiplist/cygwin.txt @@ -0,0 +1,65 @@ +# Tests expected to SKIP. One name per line, '#' starts a comment; the file +# must stay sorted and duplicate-free (runtests.py enforces both). Referenced +# from a workflow as RSYNC_EXPECT_SKIPPED=@testsuite/skiplist/[,@...]. +# See testsuite/skiplist/README.md. +# +# Cygwin-only additions to common.txt. + +acl-symlink-race +acls +acls-default +acls-depth +acls-unpinnable +backup-acl-xattr-cache +backup-dir-symlink-race +basis-xname-traversal +batch-file-symlink +chown +chroot-alt-dest-inner-module +chroot-copy-dest-inner-module +chroot-link-dest-inner-module +chroot-receiver-write-inner-module +chroot-special-inner-module +copy-dest-symlink-readleak +copy-xattrs-symlink-race +daemon-auth-group +daemon-chroot-munge-default +daemon-config-symlink +daemon-max-alloc-zero +daemon-module-chdir-symlink +daemon-module-private-parent +daemon-secrets-file-symlink +devices +dir-sgid +early-input-symlink +excludefrom-symlink +fake-super-acl-xattr +fake-super-backup-fifo-regression +files-from-leak +filter-leak +filter-merge-symlink +insecure-links-admin-optout +log-file-symlink +msg-io-timeout-overflow +nondaemon-symlink-race +nonroot-restrictive-perms +open-noatime +operator-path-backup-rmdir +operator-path-backup-symlink +operator-path-insecure-links-daemon +partial-protected-regular-retry-linux +partial-protected-regular-retry-policy # deterministic partial EACCES recovery uses dyld interposing +password-file-symlink +protected-regular +rename-mixed-parent-transfer +rrsync-sender-leaf-flip +rrsync-sender-parent-pin +rrsync-symlink +sender-remove-source-root-anchor +simd-checksum +source-change-size-continues +symlink-dest-backupdir +symlink-exclude-xattr +symlink-race-dest +symlink-race-relative-dest +temp-dir-symlink-injection diff --git a/testsuite/skiplist/linux.txt b/testsuite/skiplist/linux.txt new file mode 100644 index 00000000..b89100d2 --- /dev/null +++ b/testsuite/skiplist/linux.txt @@ -0,0 +1,10 @@ +# Tests expected to SKIP. One name per line, '#' starts a comment; the file +# must stay sorted and duplicate-free (runtests.py enforces both). Referenced +# from a workflow as RSYNC_EXPECT_SKIPPED=@testsuite/skiplist/[,@...]. +# See testsuite/skiplist/README.md. +# +# Linux-only additions to common.txt. + +crtimes # Rsync is configured without crtimes support +partial-protected-regular-retry-policy # deterministic partial EACCES recovery uses dyld interposing +readonly-partial-abort-mode-regression # diff --git a/testsuite/skiplist/macos.txt b/testsuite/skiplist/macos.txt new file mode 100644 index 00000000..0fae0d8b --- /dev/null +++ b/testsuite/skiplist/macos.txt @@ -0,0 +1,32 @@ +# Tests expected to SKIP. One name per line, '#' starts a comment; the file +# must stay sorted and duplicate-free (runtests.py enforces both). Referenced +# from a workflow as RSYNC_EXPECT_SKIPPED=@testsuite/skiplist/[,@...]. +# See testsuite/skiplist/README.md. +# +# macOS-only additions to common.txt. + +acl-symlink-race +acls-default +acls-depth +acls-unpinnable +backup-acl-xattr-cache +backup-crossdev-copy +chmod-temp-dir +copy-xattrs-symlink-race +daemon-auth-group +daemon-max-alloc-zero +dir-sgid +fake-super-acl-xattr +open-noatime +partial-protected-regular-retry-linux +preallocate +protected-regular +readonly-partial-abort-mode-regression # +rrsync-sender-leaf-flip +rrsync-sender-parent-pin +rrsync-symlink +sender-remove-source-root-anchor +simd-checksum +source-change-size-continues +sparse +symlink-exclude-xattr diff --git a/testsuite/skiplist/proto29.txt b/testsuite/skiplist/proto29.txt new file mode 100644 index 00000000..2ec82b85 --- /dev/null +++ b/testsuite/skiplist/proto29.txt @@ -0,0 +1,16 @@ +# Tests expected to SKIP. One name per line, '#' starts a comment; the file +# must stay sorted and duplicate-free (runtests.py enforces both). Referenced +# from a workflow as RSYNC_EXPECT_SKIPPED=@testsuite/skiplist/[,@...]. +# See testsuite/skiplist/README.md. +# +# Additions for a --protocol=29 run (make check29), on top of the platform +# files. These tests transfer something the 29 wire format cannot carry +# (ACLs, xattrs) or drive a helper that speaks 30+, so they gate on the +# protocol regardless of platform. + +acl-symlink-race # ACL transfer requires protocol 30+ (negotiated 29) +acls-unpinnable # ACL transfer requires protocol 30+ (negotiated 29) +daemon-copylinks-parent-target-regression # the stdio_daemon client speaks protocol 30 (forced 29) +partial-protected-regular-retry-linux # one-inplace partial staging needs protocol >= 30 (forced 29) +scanner-batch-flag-mismatch # xattrs (-X) need protocol 30+ +symlink-exclude-xattr # xattr (-X) transfer requires protocol 30+ (negotiated 29)