diff --git a/.github/workflows/almalinux-8-build.yml b/.github/workflows/almalinux-8-build.yml index 9d7ea782..e269a3e7 100644 --- a/.github/workflows/almalinux-8-build.yml +++ b/.github/workflows/almalinux-8-build.yml @@ -59,8 +59,13 @@ jobs: run: ./rsync --version - name: check # In the container we already run as root, so no sudo. The - # crtimes-not-supported skip matches the other Linux jobs. - run: RSYNC_EXPECT_SKIPPED=crtimes make check + # crtimes-not-supported skip matches the other Linux jobs; + # daemon-chroot-acl and proxy-response-line-too-long skip because + # the default (secure) transport opens no listening socket. + run: RSYNC_EXPECT_SKIPPED=crtimes,daemon-chroot-acl,proxy-response-line-too-long make check + - name: check (TCP daemon transport) + # Second run exercising the real loopback-TCP daemon path. + run: ./runtests.py --rsync-bin=`pwd`/rsync --use-tcp -j 8 - name: ssl file list run: ./rsync-ssl --no-motd download.samba.org::rsyncftp/ || true - name: save artifact diff --git a/.github/workflows/cygwin-build.yml b/.github/workflows/cygwin-build.yml index fe5a5c42..7f766f07 100644 --- a/.github/workflows/cygwin-build.yml +++ b/.github/workflows/cygwin-build.yml @@ -39,7 +39,11 @@ jobs: - name: info run: bash -c '/usr/local/bin/rsync --version' - name: check - run: bash -c 'RSYNC_EXPECT_SKIPPED=acls-default,acls,bare-do-open-symlink-race,chdir-symlink-race,chown,daemon-chroot-acl,devices,dir-sgid,open-noatime,protected-regular,sender-flist-symlink-leak,simd-checksum,symlink-dirlink-basis make check' + run: bash -c 'RSYNC_EXPECT_SKIPPED=acls-default,acls,bare-do-open-symlink-race,chdir-symlink-race,chown,daemon-chroot-acl,devices,dir-sgid,open-noatime,protected-regular,proxy-response-line-too-long,sender-flist-symlink-leak,simd-checksum,symlink-dirlink-basis make check' + - name: check (TCP daemon transport) + # Second run with daemon tests over a real loopback rsyncd; the default + # 'make check' above uses the secure stdio-pipe transport. + run: bash -c './runtests.py --rsync-bin=`pwd`/rsync.exe --use-tcp -j 8' - name: ssl file list run: bash -c 'PATH="/usr/local/bin:$PATH" rsync-ssl --no-motd download.samba.org::rsyncftp/ || true' - name: save artifact diff --git a/.github/workflows/freebsd-build.yml b/.github/workflows/freebsd-build.yml index 79633ad1..0d2b3627 100644 --- a/.github/workflows/freebsd-build.yml +++ b/.github/workflows/freebsd-build.yml @@ -35,6 +35,7 @@ jobs: make ./rsync --version make check + ./runtests.py --rsync-bin=`pwd`/rsync --use-tcp -j 8 ./rsync-ssl --no-motd download.samba.org::rsyncftp/ || true - name: save artifact uses: actions/upload-artifact@v4 diff --git a/.github/workflows/macos-build.yml b/.github/workflows/macos-build.yml index a127526e..19ef8a75 100644 --- a/.github/workflows/macos-build.yml +++ b/.github/workflows/macos-build.yml @@ -41,7 +41,11 @@ jobs: - name: info run: rsync --version - name: check - run: sudo RSYNC_EXPECT_SKIPPED=acls-default,chmod-temp-dir,chown-fake,daemon-chroot-acl,devices-fake,dir-sgid,open-noatime,protected-regular,simd-checksum,xattrs-hlink,xattrs make check + run: sudo RSYNC_EXPECT_SKIPPED=acls-default,chmod-temp-dir,chown-fake,daemon-chroot-acl,devices-fake,dir-sgid,open-noatime,protected-regular,proxy-response-line-too-long,simd-checksum,xattrs-hlink,xattrs make check + - name: check (TCP daemon transport) + # Second run with daemon tests over a real loopback rsyncd; the default + # 'make check' above uses the secure stdio-pipe transport. + run: sudo ./runtests.py --rsync-bin=`pwd`/rsync --use-tcp -j 8 - name: ssl file list run: rsync-ssl --no-motd download.samba.org::rsyncftp/ || true - name: save artifact diff --git a/.github/workflows/netbsd-build.yml b/.github/workflows/netbsd-build.yml index 770d7124..3acc2340 100644 --- a/.github/workflows/netbsd-build.yml +++ b/.github/workflows/netbsd-build.yml @@ -36,6 +36,7 @@ jobs: make ./rsync --version make check + ./runtests.py --rsync-bin=`pwd`/rsync --use-tcp -j 8 ./rsync-ssl --no-motd download.samba.org::rsyncftp/ || true - name: save artifact uses: actions/upload-artifact@v4 diff --git a/.github/workflows/openbsd-build.yml b/.github/workflows/openbsd-build.yml index 749724cd..3d83ab1a 100644 --- a/.github/workflows/openbsd-build.yml +++ b/.github/workflows/openbsd-build.yml @@ -37,6 +37,7 @@ jobs: make ./rsync --version make check + ./runtests.py --rsync-bin=`pwd`/rsync --use-tcp -j 8 ./rsync-ssl --no-motd download.samba.org::rsyncftp/ || true - name: save artifact uses: actions/upload-artifact@v4 diff --git a/.github/workflows/solaris-build.yml b/.github/workflows/solaris-build.yml index e41e002d..c7867f90 100644 --- a/.github/workflows/solaris-build.yml +++ b/.github/workflows/solaris-build.yml @@ -35,6 +35,7 @@ jobs: make ./rsync --version make check + ./runtests.py --rsync-bin=`pwd`/rsync --use-tcp -j 8 ./rsync-ssl --no-motd download.samba.org::rsyncftp/ || true - name: save artifact uses: actions/upload-artifact@v4 diff --git a/.github/workflows/ubuntu-22.04-build.yml b/.github/workflows/ubuntu-22.04-build.yml index 0e608279..154b1b90 100644 --- a/.github/workflows/ubuntu-22.04-build.yml +++ b/.github/workflows/ubuntu-22.04-build.yml @@ -39,11 +39,15 @@ jobs: - name: info run: rsync --version - name: check - run: sudo RSYNC_EXPECT_SKIPPED=crtimes make check + run: sudo RSYNC_EXPECT_SKIPPED=crtimes,daemon-chroot-acl,proxy-response-line-too-long make check - name: check30 - run: sudo RSYNC_EXPECT_SKIPPED=crtimes make check30 + run: sudo RSYNC_EXPECT_SKIPPED=crtimes,daemon-chroot-acl,proxy-response-line-too-long make check30 - name: check29 - run: sudo RSYNC_EXPECT_SKIPPED=crtimes make check29 + run: sudo RSYNC_EXPECT_SKIPPED=crtimes,daemon-chroot-acl,proxy-response-line-too-long make check29 + - name: check (TCP daemon transport) + # Second run with daemon tests over a real loopback rsyncd; the default + # 'make check' above uses the secure stdio-pipe transport. + run: sudo ./runtests.py --rsync-bin=`pwd`/rsync --use-tcp -j 8 - name: ssl file list run: rsync-ssl --no-motd download.samba.org::rsyncftp/ || true - name: save artifact diff --git a/.github/workflows/ubuntu-build.yml b/.github/workflows/ubuntu-build.yml index 5efadce5..5fe6cca4 100644 --- a/.github/workflows/ubuntu-build.yml +++ b/.github/workflows/ubuntu-build.yml @@ -35,11 +35,17 @@ jobs: - name: info run: rsync --version - name: check - run: sudo RSYNC_EXPECT_SKIPPED=crtimes make check + run: sudo RSYNC_EXPECT_SKIPPED=crtimes,daemon-chroot-acl,proxy-response-line-too-long make check - name: check30 - run: sudo RSYNC_EXPECT_SKIPPED=crtimes make check30 + run: sudo RSYNC_EXPECT_SKIPPED=crtimes,daemon-chroot-acl,proxy-response-line-too-long make check30 - name: check29 - run: sudo RSYNC_EXPECT_SKIPPED=crtimes make check29 + run: sudo RSYNC_EXPECT_SKIPPED=crtimes,daemon-chroot-acl,proxy-response-line-too-long make check29 + - name: check (TCP daemon transport) + # Second run with daemon tests over a real loopback rsyncd. The default + # 'make check' above uses the secure stdio-pipe transport (no listening + # sockets); this run exercises the real TCP accept/auth path. Skip-set + # is env-dependent here (chroot-acl), so leave RSYNC_EXPECT_SKIPPED unset. + run: sudo ./runtests.py --rsync-bin=`pwd`/rsync --use-tcp -j 8 - name: ssl file list run: rsync-ssl --no-motd download.samba.org::rsyncftp/ || true - name: save artifact diff --git a/Makefile.in b/Makefile.in index 451ff5c6..af9fbfb2 100644 --- a/Makefile.in +++ b/Makefile.in @@ -320,17 +320,21 @@ test: check # catch Bash-isms earlier even if we're running on GNU. Of course, we # might lose in the future where POSIX diverges from old sh. +# `make check` runs tests in parallel by default. Override with +# `make check CHECK_J=1` (serial) or any other value. +CHECK_J = 8 + .PHONY: check check: all $(CHECK_PROGS) $(CHECK_SYMLINKS) - $(srcdir)/runtests.py --rsync-bin=`pwd`/rsync$(EXEEXT) + $(srcdir)/runtests.py --rsync-bin=`pwd`/rsync$(EXEEXT) -j $(CHECK_J) .PHONY: check29 check29: all $(CHECK_PROGS) $(CHECK_SYMLINKS) - $(srcdir)/runtests.py --rsync-bin=`pwd`/rsync$(EXEEXT) --protocol=29 + $(srcdir)/runtests.py --rsync-bin=`pwd`/rsync$(EXEEXT) -j $(CHECK_J) --protocol=29 .PHONY: check30 check30: all $(CHECK_PROGS) $(CHECK_SYMLINKS) - $(srcdir)/runtests.py --rsync-bin=`pwd`/rsync$(EXEEXT) --protocol=30 + $(srcdir)/runtests.py --rsync-bin=`pwd`/rsync$(EXEEXT) -j $(CHECK_J) --protocol=30 wildtest.o: wildtest.c t_stub.o lib/wildmatch.c rsync.h config.h wildtest$(EXEEXT): wildtest.o lib/compat.o lib/snprintf.o @BUILD_POPT@ @@ -362,7 +366,7 @@ testsuite/exclude-lsh_test.py: .PHONY: installcheck installcheck: $(CHECK_PROGS) $(CHECK_SYMLINKS) - $(srcdir)/runtests.py --rsync-bin="$(bindir)/rsync$(EXEEXT)" --srcdir="$(srcdir)" --tooldir=`pwd` + $(srcdir)/runtests.py --rsync-bin="$(bindir)/rsync$(EXEEXT)" --srcdir="$(srcdir)" --tooldir=`pwd` -j $(CHECK_J) # TODO: Add 'dist' target; need to know which files will be included diff --git a/runtests.py b/runtests.py index feb05d62..2259aee1 100755 --- a/runtests.py +++ b/runtests.py @@ -61,6 +61,12 @@ def parse_args(): help='Force protocol version (adds --protocol=VER to rsync)') p.add_argument('--expect-skipped', default=None, metavar='LIST', help='Comma-separated list of expected-skipped tests') + p.add_argument('--use-tcp', action='store_true', + help='Run daemon tests against a real rsyncd bound to ' + '127.0.0.1 (non-default). The default is the secure ' + 'stdio-pipe transport, which opens no listening ' + 'socket; --use-tcp exposes a loopback port for the ' + 'duration of each daemon test.') return p.parse_args() @@ -366,6 +372,7 @@ def main(): print(f' valgrind=enabled (logs in valgrind.*.log)') if args.parallel > 1: print(f' parallel={args.parallel}') + print(f' daemon_transport={"tcp (loopback)" if args.use_tcp else "pipe (secure default)"}') print(f' scratchbase={scratchbase}') # Build base environment for test scripts @@ -393,6 +400,10 @@ def main(): 'HOME': scratchbase, 'PYTHONPATH': pythonpath, }) + if args.use_tcp: + # Opt-in: daemon tests start a real rsyncd on a claimed loopback port. + # Default (unset) keeps the secure stdio-pipe transport. + base_env['RSYNC_TEST_USE_TCP'] = '1' for k, v in shconfig.items(): if v: base_env[k] = v diff --git a/testsuite/alt-dest-symlink-race_test.py b/testsuite/alt-dest-symlink-race_test.py index 9045fa6e..fc83c576 100644 --- a/testsuite/alt-dest-symlink-race_test.py +++ b/testsuite/alt-dest-symlink-race_test.py @@ -20,12 +20,15 @@ import os import subprocess from rsyncfns import ( - RSYNC, SCRATCHDIR, + SCRATCHDIR, rsync_argv, get_testuid, get_rootuid, get_rootgid, - rmtree, test_fail, + rmtree, start_test_daemon, test_fail, ) +DAEMON_PORT = 12882 + + mod = SCRATCHDIR / 'module' outside = SCRATCHDIR / 'outside' src_dir = SCRATCHDIR / 'src_files' @@ -69,17 +72,15 @@ log file = {SCRATCHDIR}/rsyncd.log read only = no """) -env = os.environ.copy() -env['RSYNC_CONNECT_PROG'] = f"{RSYNC} --config={conf} --daemon" +url = start_test_daemon(conf, DAEMON_PORT) # Push directly into the module root: pushing into a destination subdir # would make the receiver chdir into it before resolving --link-dest, # making "cd" resolve in the wrong CWD and masking the bug. subprocess.run( rsync_argv('-rtp', '--link-dest=cd', - f'{src_dir}/', 'rsync://localhost/upload/'), + f'{src_dir}/', f'{url}upload/'), stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, - env=env, ) target = mod / 'target.txt' diff --git a/testsuite/bare-do-open-symlink-race_test.py b/testsuite/bare-do-open-symlink-race_test.py index d232204b..deff0731 100644 --- a/testsuite/bare-do-open-symlink-race_test.py +++ b/testsuite/bare-do-open-symlink-race_test.py @@ -14,12 +14,15 @@ import stat import subprocess from rsyncfns import ( - RSYNC, SCRATCHDIR, + SCRATCHDIR, get_rootgid, get_rootuid, get_testuid, - rmtree, rsync_argv, test_fail, test_skipped, + rmtree, rsync_argv, start_test_daemon, test_fail, test_skipped, ) +DAEMON_PORT = 12884 + + if platform.system() in ('SunOS', 'OpenBSD', 'NetBSD') or platform.system().startswith('CYGWIN'): test_skipped( f"secure_relative_open relies on RESOLVE_BENEATH-equivalent kernel " @@ -66,27 +69,32 @@ if my_uid != root_uid: gid_line = '#' + gid_line -def write_conf(module_name: str, fake_super: bool = False) -> None: - extra = " fake super = yes\n" if fake_super else "" - conf.write_text(f"""\ +# All three scenarios use the same daemon -- they just target a different +# module. Write both modules up-front so the daemon doesn't need to be +# restarted between scenarios. +conf.write_text(f"""\ use chroot = no {uid_line} {gid_line} log file = {SCRATCHDIR}/rsyncd.log -[{module_name}] +[upload] path = {mod} use chroot = no read only = no -{extra}""") + +[upload_fake] + path = {mod} + use chroot = no + read only = no + fake super = yes +""") +daemon_url = start_test_daemon(conf, DAEMON_PORT).rstrip('/') def run_attack(args): - env = os.environ.copy() - env['RSYNC_CONNECT_PROG'] = f"{RSYNC} --config={conf} --daemon" subprocess.run( rsync_argv(*args), stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, - env=env, ) @@ -97,10 +105,9 @@ os.chmod(mod / 'target.txt', 0o666) (src / 'target.txt').write_text("NEW_DATA_FROM_SENDER\n") os.chmod(src / 'target.txt', 0o644) -write_conf('upload') run_attack([ '--inplace', '--backup', '--backup-dir=cd', - f'{src}/target.txt', 'rsync://localhost/upload/target.txt', + f'{src}/target.txt', f'{daemon_url}/upload/target.txt', ]) verify_outside_unchanged("3b inplace+backup-dir=cd") @@ -110,8 +117,7 @@ setup() (src / 'cd').mkdir() os.symlink('/etc/passwd', src / 'cd' / 'sym') -write_conf('upload_fake', fake_super=True) -run_attack(['-rl', f'{src}/', 'rsync://localhost/upload_fake/']) +run_attack(['-rl', f'{src}/', f'{daemon_url}/upload_fake/']) verify_outside_unchanged_or_absent("3c-symlink fake-super symlink push", "sym") @@ -126,6 +132,5 @@ except OSError: if not stat.S_ISFIFO((src / 'cd' / 'fifo').stat().st_mode): test_skipped("mkfifo unavailable; cannot exercise 3c-mknod") -write_conf('upload_fake', fake_super=True) -run_attack(['-rD', f'{src}/', 'rsync://localhost/upload_fake/']) +run_attack(['-rD', f'{src}/', f'{daemon_url}/upload_fake/']) verify_outside_unchanged_or_absent("3c-mknod fake-super FIFO push", "fifo") diff --git a/testsuite/batch-mode_test.py b/testsuite/batch-mode_test.py index 7cd9e79d..3c1a6827 100644 --- a/testsuite/batch-mode_test.py +++ b/testsuite/batch-mode_test.py @@ -9,12 +9,14 @@ import shutil import subprocess from rsyncfns import ( - CHKDIR, FROMDIR, RSYNC, SCRATCHDIR, TMPDIR, TODIR, + CHKDIR, FROMDIR, SCRATCHDIR, TMPDIR, TODIR, build_rsyncd_conf, checkit, hands_setup, rmtree, - run_rsync, test_fail, + run_rsync, start_test_daemon, test_fail, ) +DAEMON_PORT = 12874 + conf = build_rsyncd_conf() hands_setup() @@ -44,12 +46,14 @@ rmtree(TODIR) print("Test --read-batch:") checkit(['-av', '--read-batch=BATCH', str(TODIR)], FROMDIR, TODIR) -# Daemon variants. RSYNC_CONNECT_PROG plumbs an in-process daemon. -os.environ['RSYNC_CONNECT_PROG'] = f"{RSYNC} --config={conf} --daemon" +# Daemon variants: pipe transport by default, real loopback rsyncd under +# --use-tcp. +url = start_test_daemon(conf, DAEMON_PORT) rmtree(TODIR) print("Test daemon sender --write-batch:") -checkit(['-av', '--write-batch=BATCH', 'rsync://localhost/test-from/', str(TODIR)], +checkit(['-av', '--write-batch=BATCH', + f'{url}test-from/', str(TODIR)], CHKDIR, TODIR, allowed_codes=(0, 23)) rmtree(TODIR) @@ -84,7 +88,7 @@ ignore23 = SCRATCHDIR / 'ignore23' from rsyncfns import rsync_argv proc = subprocess.run( [str(ignore23), *rsync_argv('-av', '--write-batch=BATCH', - f'{FROMDIR}/', 'rsync://localhost/test-to')], + f'{FROMDIR}/', f'{url}test-to')], ) if proc.returncode != 0: test_fail(f"daemon recv --write-batch exited {proc.returncode}") diff --git a/testsuite/chdir-symlink-race_test.py b/testsuite/chdir-symlink-race_test.py index f2495f7e..80b314c0 100644 --- a/testsuite/chdir-symlink-race_test.py +++ b/testsuite/chdir-symlink-race_test.py @@ -13,12 +13,15 @@ import platform import subprocess from rsyncfns import ( - RSYNC, SCRATCHDIR, + SCRATCHDIR, get_rootgid, get_rootuid, get_testuid, - make_data_file, rmtree, rsync_argv, test_fail, test_skipped, + make_data_file, rmtree, rsync_argv, start_test_daemon, + test_fail, test_skipped, ) +DAEMON_PORT = 12885 + if platform.system() in ('SunOS', 'OpenBSD', 'NetBSD') or platform.system().startswith('CYGWIN'): test_skipped( f"secure chdir relies on RESOLVE_BENEATH-equivalent kernel " @@ -79,14 +82,14 @@ def verify_unchanged(label: str) -> None: test_fail(f"{label}: outside file content changed (write escape)") +url = start_test_daemon(conf, DAEMON_PORT) + + def run_attack(label: str, *args) -> None: reset_outside() - env = os.environ.copy() - env['RSYNC_CONNECT_PROG'] = f"{RSYNC} --config={conf} --daemon" subprocess.run( rsync_argv(*args), stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, - env=env, ) verify_unchanged(label) @@ -96,24 +99,24 @@ def run_attack(label: str, *args) -> None: run_attack("single-file --size-only", '-tp', '--size-only', f'{src}/target.txt', - 'rsync://localhost/upload/subdir/target.txt') + f'{url}upload/subdir/target.txt') # 2. -r push INTO the symlinked subdir -- receiver chdir's into "subdir", # follows the symlink, ends up in outside. run_attack("-r --size-only into subdir/", '-rtp', '--size-only', f'{src}/subdir/', - 'rsync://localhost/upload/subdir/') + f'{url}upload/subdir/') # 3. Same but with delta+rename (read-disclosure + write-escape together). run_attack("-r without --size-only into subdir/", '-rtp', f'{src}/subdir/', - 'rsync://localhost/upload/subdir/') + f'{url}upload/subdir/') # 4. -r into the module root -- already covered by the original CVE fix; # regression-check. run_attack("-r --size-only into upload/ root", '-rtp', '--size-only', f'{src}/', - 'rsync://localhost/upload/') + f'{url}upload/') diff --git a/testsuite/chmod-option_test.py b/testsuite/chmod-option_test.py index 7ca2e292..b16e7816 100644 --- a/testsuite/chmod-option_test.py +++ b/testsuite/chmod-option_test.py @@ -10,11 +10,15 @@ import os import shutil from rsyncfns import ( - FROMDIR, RSYNC, SCRATCHDIR, TODIR, - build_rsyncd_conf, checkit, makepath, rmtree, run_rsync, + FROMDIR, SCRATCHDIR, TODIR, + build_rsyncd_conf, checkit, makepath, rmtree, + run_rsync, start_test_daemon, ) +DAEMON_PORT = 12875 + + checkdir = SCRATCHDIR / 'check' FROMDIR.mkdir(parents=True, exist_ok=True) @@ -78,10 +82,11 @@ with open(conf, 'a') as f: \tincoming chmod = Fo-x """) -os.environ['RSYNC_CONNECT_PROG'] = f"{RSYNC} --config={conf} --daemon" +url = start_test_daemon(conf, DAEMON_PORT) rmtree(TODIR) makepath(TODIR) -checkit(['-avv', '--no-perms', f'{FROMDIR}/', 'localhost::test-incoming-chmod/'], +checkit(['-avv', '--no-perms', f'{FROMDIR}/', + f'{url}test-incoming-chmod/'], checkdir, TODIR, allowed_codes=(0, 23)) diff --git a/testsuite/copy-dest-source-symlink_test.py b/testsuite/copy-dest-source-symlink_test.py index f3c6b09f..d1f44542 100644 --- a/testsuite/copy-dest-source-symlink_test.py +++ b/testsuite/copy-dest-source-symlink_test.py @@ -18,12 +18,15 @@ import os import subprocess from rsyncfns import ( - RSYNC, SCRATCHDIR, + SCRATCHDIR, rsync_argv, get_testuid, get_rootuid, get_rootgid, - rmtree, test_fail, + rmtree, start_test_daemon, test_fail, ) +DAEMON_PORT = 12883 + + mod = SCRATCHDIR / 'module' outside = SCRATCHDIR / 'outside' src_dir = SCRATCHDIR / 'src_files' @@ -64,14 +67,12 @@ log file = {SCRATCHDIR}/rsyncd.log read only = no """) -env = os.environ.copy() -env['RSYNC_CONNECT_PROG'] = f"{RSYNC} --config={conf} --daemon" +url = start_test_daemon(conf, DAEMON_PORT) subprocess.run( rsync_argv('-rtp', '--copy-dest=cd', - f'{src_dir}/', 'rsync://localhost/upload/'), + f'{src_dir}/', f'{url}upload/'), stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, - env=env, ) target = mod / 'target.txt' diff --git a/testsuite/daemon-chroot-acl_test.py b/testsuite/daemon-chroot-acl_test.py index 3fa5e2c3..f477145e 100644 --- a/testsuite/daemon-chroot-acl_test.py +++ b/testsuite/daemon-chroot-acl_test.py @@ -15,11 +15,19 @@ import subprocess import sys from rsyncfns import ( - RSYNC, SCRATCHDIR, TODIR, - rmtree, rsync_argv, test_fail, test_skipped, + SCRATCHDIR, TODIR, + require_tcp, rmtree, rsync_argv, start_test_daemon, test_fail, test_skipped, ) +DAEMON_PORT = 12878 + +# This test fundamentally needs a real TCP peer address: the daemon reverse- +# resolves the connecting IP for a hostname-based "hosts deny" ACL check. +# The stdio-pipe transport has no peer IP, so only run under --use-tcp. +require_tcp("needs a real TCP peer address for reverse-DNS hostname ACL; " + "run with --use-tcp") + if platform.system() != 'Linux': test_skipped("test is Linux-specific (uses chroot+unshare)") @@ -99,11 +107,12 @@ def run_check(label: str) -> bool: rmtree(TODIR) TODIR.mkdir() - env = os.environ.copy() - env['RSYNC_CONNECT_PROG'] = f"{RSYNC} --config={conf} --daemon" + # rsyncd re-reads its config file on each accepted connection, so + # rewriting `conf` between scenarios is enough -- we keep the one + # daemon for both. proc = subprocess.run( - rsync_argv('-av', 'localhost::chrootmod/', f'{TODIR}/'), - capture_output=True, text=True, env=env, + rsync_argv('-av', f'{url}chrootmod/', f'{TODIR}/'), + capture_output=True, text=True, ) out = proc.stdout + proc.stderr @@ -117,8 +126,12 @@ def run_check(label: str) -> bool: return '@ERROR' in out and 'access denied' in out -# Scenario A: global reverse lookup. Covered by b6abdb4c. +# Spin up the daemon once; we'll rewrite `conf` between scenarios and rely +# on rsyncd's per-connection re-read of the config file. write_conf('yes', 'yes') +url = start_test_daemon(conf, DAEMON_PORT) + +# Scenario A: global reverse lookup. Covered by b6abdb4c. if not run_check("Scenario A (global reverse lookup = yes)"): test_fail("Scenario A: hostname deny rule was bypassed") diff --git a/testsuite/daemon-gzip-download_test.py b/testsuite/daemon-gzip-download_test.py index adf21ae0..4550863a 100644 --- a/testsuite/daemon-gzip-download_test.py +++ b/testsuite/daemon-gzip-download_test.py @@ -1,28 +1,29 @@ #!/usr/bin/env python3 # Python rewrite of testsuite/daemon-gzip-download.test. # -# Download a file tree over a compressed connection from an in-process -# rsyncd (via RSYNC_CONNECT_PROG). Exercises (exorcises?) a bug in -# 2.5.3 that mis-handled doubly-compressed transfers. - -import os +# Download a file tree over a compressed connection from a test daemon. +# Exercises (exorcises?) a bug in 2.5.3 that mis-handled doubly-compressed +# transfers. Uses the secure stdio-pipe transport by default; --use-tcp +# runs it against a real loopback rsyncd. from rsyncfns import ( - CHKDIR, FROMDIR, RSYNC, TODIR, - build_rsyncd_conf, checkit, hands_setup, run_rsync, + CHKDIR, FROMDIR, TODIR, + build_rsyncd_conf, checkit, hands_setup, run_rsync, start_test_daemon, ) -conf = build_rsyncd_conf() -os.environ['RSYNC_CONNECT_PROG'] = f"{RSYNC} --config={conf} --daemon" +DAEMON_PORT = 12879 +conf = build_rsyncd_conf() hands_setup() # chkdir: vanilla copy minus the daemon's global "foobar.baz" exclude. run_rsync('-av', '--exclude=foobar.baz', f'{FROMDIR}/', f'{CHKDIR}/') +url = start_test_daemon(conf, DAEMON_PORT) + checkit( - ['-avvvvzz', 'localhost::test-from/', f'{TODIR}/'], + ['-avvvvzz', f'{url}test-from/', f'{TODIR}/'], CHKDIR, TODIR, allowed_codes=(0, 23), ) diff --git a/testsuite/daemon-gzip-upload_test.py b/testsuite/daemon-gzip-upload_test.py index f5fd85f7..64922a14 100644 --- a/testsuite/daemon-gzip-upload_test.py +++ b/testsuite/daemon-gzip-upload_test.py @@ -1,29 +1,29 @@ #!/usr/bin/env python3 # Python rewrite of testsuite/daemon-gzip-upload.test. # -# Upload a file tree over a compressed connection to an in-process -# rsyncd (via RSYNC_CONNECT_PROG). Exercises (exorcises?) a bug in -# 2.5.3 that mis-handled doubly-compressed transfers. - -import os +# Upload a file tree over a compressed connection to a test daemon. +# Exercises (exorcises?) a bug in 2.5.3 that mis-handled doubly-compressed +# transfers. Uses the secure stdio-pipe transport by default; --use-tcp +# runs it against a real loopback rsyncd. from rsyncfns import ( - CHKDIR, FROMDIR, RSYNC, SCRATCHDIR, TODIR, - build_rsyncd_conf, checkit, hands_setup, run_rsync, + CHKDIR, FROMDIR, TODIR, + build_rsyncd_conf, checkit, hands_setup, run_rsync, start_test_daemon, ) -conf = build_rsyncd_conf() -os.environ['RSYNC_CONNECT_PROG'] = f"{RSYNC} --config={conf} --daemon" +DAEMON_PORT = 12880 +conf = build_rsyncd_conf() hands_setup() # chkdir: vanilla copy minus the daemon's global "foobar.baz" exclude. run_rsync('-av', '--exclude=foobar.baz', f'{FROMDIR}/', f'{CHKDIR}/') -ignore23 = str(SCRATCHDIR / 'ignore23') +url = start_test_daemon(conf, DAEMON_PORT) + checkit( - ['-avvvvzz', f'{FROMDIR}/', 'localhost::test-to/'], + ['-avvvvzz', f'{FROMDIR}/', f'{url}test-to/'], CHKDIR, TODIR, allowed_codes=(0, 23), ) diff --git a/testsuite/daemon-refuse-compress_test.py b/testsuite/daemon-refuse-compress_test.py index 29b28bb7..83af1729 100644 --- a/testsuite/daemon-refuse-compress_test.py +++ b/testsuite/daemon-refuse-compress_test.py @@ -5,16 +5,17 @@ # reject clients that ask for compression, and still serve the same # transfer when the client does not. -import os import subprocess from rsyncfns import ( - CHKDIR, FROMDIR, RSYNC, SCRATCHDIR, TODIR, + CHKDIR, FROMDIR, SCRATCHDIR, TODIR, build_rsyncd_conf, checkit, hands_setup, rmtree, - rsync_argv, run_rsync, test_fail, + rsync_argv, run_rsync, start_test_daemon, test_fail, ) +DAEMON_PORT = 12876 + conf = build_rsyncd_conf() # Append an extra module that refuses --compress (-z). with open(conf, 'a') as f: @@ -25,15 +26,15 @@ with open(conf, 'a') as f: \trefuse options = compress """) -os.environ['RSYNC_CONNECT_PROG'] = f"{RSYNC} --config={conf} --daemon" - hands_setup() run_rsync('-av', '--exclude=foobar.baz', f'{FROMDIR}/', f'{CHKDIR}/') +url = start_test_daemon(conf, DAEMON_PORT) + 'no-compress/' + # A compressed transfer must be refused. errlog = SCRATCHDIR / 'refuse.err' proc = subprocess.run( - rsync_argv('-avz', 'localhost::no-compress/', f'{TODIR}/'), + rsync_argv('-avz', url, f'{TODIR}/'), stdout=subprocess.DEVNULL, stderr=subprocess.PIPE, text=True, ) errlog.write_text(proc.stderr) @@ -47,5 +48,5 @@ if '--compress' not in proc.stderr: # The same transfer without -z must succeed. rmtree(TODIR) TODIR.mkdir() -checkit(['-av', 'localhost::no-compress/', f'{TODIR}/'], CHKDIR, TODIR, +checkit(['-av', url, f'{TODIR}/'], CHKDIR, TODIR, allowed_codes=(0, 23)) diff --git a/testsuite/daemon_test.py b/testsuite/daemon_test.py index 41de280a..2dcee1a1 100644 --- a/testsuite/daemon_test.py +++ b/testsuite/daemon_test.py @@ -13,10 +13,13 @@ import subprocess from rsyncfns import ( CHKFILE, FROMDIR, OUTFILE, RSYNC, SCRATCHDIR, SRCDIR, TODIR, build_rsyncd_conf, get_rootuid, get_testuid, makepath, - rsync_argv, run_rsync, test_fail, + rsync_argv, run_rsync, start_test_daemon, test_fail, ) +DAEMON_PORT = 12877 + + SSH = f"{SRCDIR / 'support' / 'lsh.sh'} --no-cd" # Replacements that hide the variable parts of `rsync -r` listings: tabs/ @@ -84,16 +87,18 @@ if expected_modules not in out: test_fail("module list via lsh.sh did not contain the expected modules") print('====') -# Same module list via RSYNC_CONNECT_PROG -- the same daemon, no remote shell. -os.environ['RSYNC_CONNECT_PROG'] = f"{RSYNC} --config={conf} --daemon" -out = run_and_check(['-v', 'localhost::'], expected_modules, "module list via daemon") +# Same module list via the test daemon (pipe transport by default; real +# loopback rsyncd under --use-tcp). +daemon_url = start_test_daemon(conf, DAEMON_PORT).rstrip('/') + +out = run_and_check(['-v', f'{daemon_url}/'], expected_modules, "module list via daemon") if expected_modules not in out: test_fail("module list via daemon did not contain the expected modules") print('====') # test-hidden: a recursive listing of the module, with file/dir/date # columns normalised so the diff is content-only. -out = run_and_check(['-r', 'localhost::test-hidden'], "", "test-hidden listing") +out = run_and_check(['-r', f'{daemon_url}/test-hidden'], "", "test-hidden listing") normalised = normalise(out) expected_hidden = """\ drwxr-xr-x DIR ####/##/## ##:##:## . @@ -112,7 +117,7 @@ for path in ('bar', 'bar/two', 'bar/baz', 'bar/baz/three', 'foo', 'foo/one'): test_fail(f"test-hidden listing missing path {path!r}") # test-from/f* glob: only the foo subtree. -out = run_and_check(['-r', 'localhost::test-from/f*'], "", "test-from glob") +out = run_and_check(['-r', f'{daemon_url}/test-from/f*'], "", "test-from glob") normalised = normalise(out) for path in ('foo', 'foo/one'): if path not in normalised: @@ -125,7 +130,7 @@ if 'bar' in normalised: # atimes-format variant -- only if rsync was built with atimes support. vv = run_rsync('-VV', check=True, capture_output=True) if '"atimes": true' in vv.stdout: - out = run_and_check(['-rU', 'localhost::test-from/f*'], "", "test-from glob with -U") + out = run_and_check(['-rU', f'{daemon_url}/test-from/f*'], "", "test-from glob with -U") normalised = normalise(out) for path in ('foo', 'foo/one'): if path not in normalised: diff --git a/testsuite/proxy-response-line-too-long_test.py b/testsuite/proxy-response-line-too-long_test.py index af4a79f3..a1e7aa66 100644 --- a/testsuite/proxy-response-line-too-long_test.py +++ b/testsuite/proxy-response-line-too-long_test.py @@ -15,9 +15,16 @@ import sys import threading import time -from rsyncfns import SCRATCHDIR, claim_ports, rsync_argv, test_fail, test_skipped +from rsyncfns import ( + SCRATCHDIR, claim_ports, require_tcp, rsync_argv, test_fail, test_skipped, +) +# This test has no stdio-pipe equivalent: it binds a real loopback socket to +# stand in for a malicious HTTP proxy. Honour the secure default (no listening +# sockets) by only running it under --use-tcp. +require_tcp("fake-proxy listener needs a real TCP socket; run with --use-tcp") + if shutil.which('python3') is None: test_skipped("python3 not available") diff --git a/testsuite/rsyncfns.py b/testsuite/rsyncfns.py index 224f6388..95029ff1 100644 --- a/testsuite/rsyncfns.py +++ b/testsuite/rsyncfns.py @@ -17,12 +17,15 @@ Conventions matching the shell harness: from __future__ import annotations +import atexit import fcntl import os import shlex import shutil +import socket as _socket import subprocess import sys +import time from pathlib import Path @@ -55,6 +58,12 @@ RSYNC = _required('RSYNC') # full command line, possibly with valgrind/p # assign to rsyncfns.TLS_ARGS before calling checkit / rsync_ls_lR. TLS_ARGS = os.environ.get('TLS_ARGS', '') +# Daemon-mode transport. The DEFAULT is the secure stdio-pipe mechanism +# (RSYNC_CONNECT_PROG), which opens no listening socket at all. The runner +# sets RSYNC_TEST_USE_TCP=1 only when invoked with --use-tcp, which switches +# daemon tests to a real rsyncd bound to loopback (see start_test_daemon). +USE_TCP = os.environ.get('RSYNC_TEST_USE_TCP') == '1' + # Mnemonics for rsync's itemize-changes (-i / -ii) format: # all_plus -> +++++++++ every attribute changed (an additive create) # allspace -> every attribute unchanged @@ -148,6 +157,124 @@ def claim_ports(*ports: int) -> 'None': fcntl.lockf(_port_lock_fd, fcntl.LOCK_EX, 1, port) +# --- standalone rsyncd helpers --------------------------------------------- + +def _set_pdeathsig() -> 'None': + """Linux: ask the kernel to send SIGTERM to us if our parent dies. + A no-op on every other platform. Used as preexec_fn so a kill -9 of + the test process doesn't strand the rsyncd we spawned.""" + if not sys.platform.startswith('linux'): + return + try: + import ctypes + libc = ctypes.CDLL('libc.so.6', use_errno=True) + PR_SET_PDEATHSIG = 1 + libc.prctl(PR_SET_PDEATHSIG, 15, 0, 0, 0) # 15 == SIGTERM + except OSError: + pass + + +def _stop_rsyncd(proc) -> 'None': + if proc.poll() is not None: + return + try: + proc.terminate() + proc.wait(timeout=2) + except subprocess.TimeoutExpired: + try: + proc.kill() + proc.wait(timeout=1) + except (subprocess.TimeoutExpired, OSError): + pass + + +def start_rsyncd(conf_path, port: int) -> 'subprocess.Popen': + """Spawn `rsync --daemon --no-detach --address=127.0.0.1 --port=N + --config=conf` and return the Popen handle after the port is accepting + connections. + + The daemon is bound to LOOPBACK ONLY (--address=127.0.0.1): without it, + rsync --daemon binds 0.0.0.0 and the test modules would be reachable from + the whole LAN. The daemon is killed automatically when this Python + process exits (atexit). On Linux, the kernel also signals SIGTERM to the + daemon if the parent dies abruptly (PR_SET_PDEATHSIG), so a SIGKILL on + the test process doesn't strand the daemon either. The caller is expected + to have already claim_ports()'d `port`. + + This is only ever reached from start_test_daemon() in --use-tcp mode; the + default (pipe) mode never starts a listening daemon. + """ + argv = shlex.split(RSYNC) + [ + '--daemon', '--no-detach', + '--address=127.0.0.1', + f'--port={port}', + f'--config={conf_path}', + ] + proc = subprocess.Popen( + argv, + stdin=subprocess.DEVNULL, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + preexec_fn=_set_pdeathsig, + ) + atexit.register(_stop_rsyncd, proc) + + deadline = time.monotonic() + 10 + last_err = None + while time.monotonic() < deadline: + if proc.poll() is not None: + test_fail( + f"rsyncd exited before listening on port {port} " + f"(status={proc.returncode})" + ) + try: + with _socket.create_connection(('127.0.0.1', port), timeout=0.5): + return proc + except OSError as e: + last_err = e + time.sleep(0.05) + + _stop_rsyncd(proc) + test_fail(f"rsyncd never listened on 127.0.0.1:{port}: {last_err}") + + +def start_test_daemon(conf_path, port: int) -> str: + """Bring up the test daemon and return a URL prefix for client commands. + + This is the single seam every daemon test uses. The transport depends on + the mode the runner selected: + + * DEFAULT (secure) -- no TCP socket at all. Sets RSYNC_CONNECT_PROG so + the rsync client forks the daemon over a private stdio pipe. Returns + 'rsync://localhost/'. Another local user can't reach it; nothing is + listening. + + * --use-tcp -- starts a real rsyncd bound to 127.0.0.1 on the given + claim_ports()-reserved port. Returns 'rsync://localhost:PORT/'. Bound + to loopback so off-host/LAN access is impossible; a same-host user + could still connect during the test window, which is the documented, + accepted cost of explicitly opting into TCP. + + Build URLs as f"{prefix}module/path". `port` is only used (and claimed) + in --use-tcp mode. + """ + if USE_TCP: + claim_ports(port) + start_rsyncd(conf_path, port) + return f'rsync://localhost:{port}/' + os.environ['RSYNC_CONNECT_PROG'] = f'{RSYNC} --config={conf_path} --daemon' + return 'rsync://localhost/' + + +def require_tcp(reason: str) -> 'None': + """Skip the test (exit 77) unless we're in --use-tcp mode. For tests that + fundamentally need a real listening socket / TCP peer and have no secure + pipe equivalent (the fake-proxy listener; the reverse-DNS hostname-ACL + daemon test).""" + if not USE_TCP: + test_skipped(reason) + + def rsync_argv(*args: str) -> list: """Return the argv for invoking rsync with the given extra arguments. @@ -269,8 +396,6 @@ def build_rsyncd_conf() -> 'Path': pidfile = SCRATCHDIR / 'rsyncd.pid' logfile = SCRATCHDIR / 'rsyncd.log' - hostname = subprocess.check_output(['uname', '-n'], text=True).strip() - my_uid = get_testuid() root_uid = get_rootuid() root_gid = get_rootgid() @@ -289,7 +414,10 @@ def build_rsyncd_conf() -> 'Path': pid file = {pidfile} use chroot = no munge symlinks = no -hosts allow = localhost 127.0.0.0/24 192.168.0.0/16 10.0.0.0/8 {hostname} +# Loopback only. In --use-tcp mode the daemon is also bound to 127.0.0.1 +# (start_rsyncd passes --address), so this is belt-and-suspenders; in the +# default pipe mode there is no socket to guard at all. +hosts allow = localhost 127.0.0.0/8 log file = {logfile} transfer logging = yes # We don't define log format here so the test-hidden module defaults diff --git a/testsuite/sender-flist-symlink-leak_test.py b/testsuite/sender-flist-symlink-leak_test.py index afa2ead6..67fc2f11 100644 --- a/testsuite/sender-flist-symlink-leak_test.py +++ b/testsuite/sender-flist-symlink-leak_test.py @@ -15,12 +15,15 @@ import platform import subprocess from rsyncfns import ( - RSYNC, SCRATCHDIR, + SCRATCHDIR, rsync_argv, get_testuid, get_rootuid, get_rootgid, - rmtree, test_fail, test_skipped, + rmtree, start_test_daemon, test_fail, test_skipped, ) +DAEMON_PORT = 12881 + + # Platforms without RESOLVE_BENEATH equivalents fall back to a per- # component walk that this test is not in scope for. if platform.system() in ('SunOS', 'OpenBSD', 'NetBSD') or platform.system().startswith('CYGWIN'): @@ -66,12 +69,11 @@ log file = {SCRATCHDIR}/rsyncd.log read only = no """) -env = os.environ.copy() -env['RSYNC_CONNECT_PROG'] = f"{RSYNC} --config={conf} --daemon" +url = start_test_daemon(conf, DAEMON_PORT) proc = subprocess.run( - rsync_argv('-nrv', 'rsync://localhost/upload/cd/', f'{SCRATCHDIR}/dst/'), - capture_output=True, text=True, env=env, + rsync_argv('-nrv', f'{url}upload/cd/', f'{SCRATCHDIR}/dst/'), + capture_output=True, text=True, ) listfile.write_text(proc.stdout + proc.stderr)