"""Shared helpers for rsync's Python test scripts. This is the Python counterpart of testsuite/rsync.fns. It exposes only what the Python-rewritten tests actually need; grow it as more shell tests are ported. Conventions matching the shell harness: * Exit codes (see the Exit enum): 0=pass, 1=fail, 2=error, 77=skip, 78=xfail. * The runner sets these environment variables before invoking each test: scratchdir per-test scratch directory srcdir rsync source directory TOOLDIR build directory (holds the rsync binary and helpers) RSYNC the rsync command line (may include valgrind / --protocol=N) TLS_ARGS extra arguments to pass to the 'tls' helper suitedir this directory (testsuite/) """ from __future__ import annotations import atexit import fcntl import filecmp import os import platform import shlex import shutil import socket as _socket import stat import subprocess import sys import time from pathlib import Path from exitcodes import Exit # re-exported: tests may `from rsyncfns import Exit` # --- environment ----------------------------------------------------------- def _required(name: str) -> str: v = os.environ.get(name) if not v: sys.stderr.write( f"rsyncfns: required environment variable {name} is not set; " "run this test via runtests.py rather than directly.\n" ) sys.exit(Exit.ERROR) return v SCRATCHDIR = Path(_required('scratchdir')) SRCDIR = Path(_required('srcdir')) TOOLDIR = Path(_required('TOOLDIR')) SUITEDIR = Path(os.environ.get('suitedir', SRCDIR / 'testsuite')) # rsync.fns set `umask 022` for every shell test, so the suite's expected # file/dir modes are computed against that baseline. Mirror it here so the # Python tests are deterministic regardless of the caller's ambient umask # (e.g. a CI runner with umask 077) -- several permission tests depend on # newly-created dirs being 0755. Individual tests may still narrow it (e.g. # chmod-option uses 002 for its --chmod comparison). os.umask(0o022) # rsync.fns overrides HOME to $scratchdir; tests that exercise ssh-style # transfers with no path component (e.g. localhost: at end of args) rely on # HOME pointing at the per-test scratch dir. os.environ['HOME'] = str(SCRATCHDIR) RSYNC = _required('RSYNC') # full command line, possibly with valgrind/protocol # The "peer" rsync command -- used for the SERVER side of two-sided transfers # (the daemon process; the remote-shell --rsync-path target). The runner sets # RSYNC_PEER to a second binary when invoked with --rsync-bin2, letting a run # mix two rsync versions over the wire. When no second binary was selected, # RSYNC_PEER == RSYNC, so every consumer below behaves exactly as before and # single-binary runs are unchanged. Use .get (not _required) so a test invoked # by hand without the runner still works. RSYNC_PEER = os.environ.get('RSYNC_PEER', RSYNC) # TLS_ARGS controls how the 'tls' helper formats listings (e.g. --atimes, # -l, -L). Tests that exercise non-default rsync features (atimes, etc.) # assign to rsyncfns.TLS_ARGS before calling checkit / rsync_ls_lR. TLS_ARGS = os.environ.get('TLS_ARGS', '') # Daemon-mode transport. The DEFAULT is the secure stdio-pipe mechanism # (RSYNC_CONNECT_PROG), which opens no listening socket at all. The runner # sets RSYNC_TEST_USE_TCP=1 only when invoked with --use-tcp, which switches # daemon tests to a real rsyncd bound to loopback (see start_test_daemon). USE_TCP = os.environ.get('RSYNC_TEST_USE_TCP') == '1' # Mnemonics for rsync's itemize-changes (-i / -ii) format: # all_plus -> +++++++++ every attribute changed (an additive create) # allspace -> every attribute unchanged # dots -> ..... trailing dots after the change columns all_plus = '+++++++++' allspace = ' ' dots = '.....' # The "$tmpdir/from", "$tmpdir/to", "$tmpdir/chk" layout from rsync.fns. TMPDIR = SCRATCHDIR FROMDIR = SCRATCHDIR / 'from' TODIR = SCRATCHDIR / 'to' CHKDIR = SCRATCHDIR / 'chk' CHKFILE = SCRATCHDIR / 'rsync.chk' OUTFILE = SCRATCHDIR / 'rsync.out' # --- result reporting ------------------------------------------------------ def test_fail(msg: str) -> 'None': sys.stderr.write(msg.rstrip() + '\n') sys.exit(Exit.FAIL) def test_skipped(msg: str) -> 'None': sys.stderr.write(msg.rstrip() + '\n') (TMPDIR / 'whyskipped').write_text(msg.rstrip() + '\n') sys.exit(Exit.SKIP) def test_xfail(msg: str) -> 'None': sys.stderr.write(msg.rstrip() + '\n') sys.exit(Exit.XFAIL) # --- rsync invocation ------------------------------------------------------ # --- TCP port coordination across parallel tests --------------------------- _PORT_LOCK_PATH = '/tmp/rsync_test.lck' _port_lock_fd = None def _open_lock_file() -> int: """Open (or create) the host-wide port-lock file, defending against a local attacker who pre-plants the well-known /tmp path. CI runs some tests under sudo, so we must never let root open/chmod an attacker- controlled target. Strategy: * Try an O_EXCL|O_CREAT create. If we win, the file is brand-new, regular, owned by us and nlink==1 -- the ONLY case where we widen the mode to 0o666 (so a second user sharing the lock can open it RDWR; the create mode is otherwise narrowed by umask). * If it already exists, open it WITHOUT O_CREAT, WITHOUT chmod, and with O_NOFOLLOW so a planted symlink fails (ELOOP) rather than being followed. Then require a pristine regular file with nlink==1, rejecting a hard link to some other (e.g. root-owned 0600) file -- O_NOFOLLOW alone does not catch hard links. """ nofollow = getattr(os, 'O_NOFOLLOW', 0) # Path 1: we create it ourselves, exclusively. try: fd = os.open(_PORT_LOCK_PATH, os.O_CREAT | os.O_EXCL | os.O_RDWR | nofollow, 0o666) except FileExistsError: fd = None if fd is not None: try: os.fchmod(fd, 0o666) # we own this fresh file; undo umask except OSError: pass return fd # Path 2: it already exists -- open without creating or chmod'ing. try: fd = os.open(_PORT_LOCK_PATH, os.O_RDWR | nofollow) except OSError as e: test_fail(f"cannot open lock file {_PORT_LOCK_PATH}: {e} " "(refusing to follow a symlink -- possible tampering)") st = os.fstat(fd) if not stat.S_ISREG(st.st_mode) or st.st_nlink != 1: os.close(fd) test_fail(f"lock file {_PORT_LOCK_PATH} is not a pristine regular " f"file (type/nlink check failed -- possible tampering)") return fd def _probe_bindable(port: int) -> 'None': """Confirm `port` is actually free once we hold its claim_ports() lock. The byte-range lock only coordinates *live* test drivers, and the kernel releases it the instant the holding process dies -- even if that driver left an orphaned daemon still bound to the port. That happens when a run is SIGKILLed (or its ssh drops) on a platform with no parent-death backstop: rsyncfns only arms PR_SET_PDEATHSIG, which is Linux-only, so on the BSDs/Solaris/macOS a killed fleettest run can strand its rsyncd, which then squats the fixed test port forever. A later run wins the (now-free) lock but the socket is still taken, and the daemon dies with a cryptic "bind() failed: Address already in use" / the client "did not see server greeting". So actually try to bind it. SO_REUSEADDR is used so a port merely in TIME_WAIT (recently and cleanly closed) is NOT a false positive; only a live bound/listening socket -- a real squatter -- makes the bind fail, and then we stop here with an actionable message instead of failing obscurely later. The probe socket is closed immediately, freeing the port for the daemon that is about to bind it. """ s = _socket.socket(_socket.AF_INET, _socket.SOCK_STREAM) s.setsockopt(_socket.SOL_SOCKET, _socket.SO_REUSEADDR, 1) try: s.bind(('127.0.0.1', port)) except OSError as e: test_fail( f"port {port} was claimed for this run but something is still bound " f"to 127.0.0.1:{port} ({e.strerror}). The claim_ports() lock only " "serializes live test runs, so a still-bound port almost always " "means an orphaned 'rsync --daemon' from a previously killed run " f"(find it with `fstat | grep {port}` / `netstat -an | grep {port}` " "and kill it, or run `fleettest.py --cleanup`), then retry.") finally: s.close() def claim_ports(*ports: int) -> 'None': """Reserve the given TCP port numbers for the rest of this process. Uses POSIX byte-range locks on /tmp/rsync_test.lck (one byte per port, offset = port number) so that any number of tests can run in parallel without colliding on a port: if another test has already claimed any of the requested ports the call blocks until that test exits. The kernel drops POSIX advisory locks automatically when the holding process terminates, so a crashed test releases its ports without manual cleanup. Ports are claimed in sorted order, so two callers that ask for the same set in different orders can't deadlock against each other. Call once near the top of any test that binds to a specific TCP port, BEFORE the bind: from rsyncfns import claim_ports claim_ports(12873) listener = socket.socket(...) listener.bind(('127.0.0.1', 12873)) The lock file lives in /tmp so it's shared across all rsync test processes on the host. Ports outside the claim_ports() ecosystem are not protected -- nothing stops an unrelated process from binding the port. For the rsync testsuite that's fine; we just need to avoid collisions between concurrent test scripts. """ global _port_lock_fd if _port_lock_fd is None: _port_lock_fd = _open_lock_file() for port in sorted(ports): # F_SETLKW via fcntl.lockf(LOCK_EX, length, start): exclusive # byte-range lock on byte `port`, blocking until acquired. fcntl.lockf(_port_lock_fd, fcntl.LOCK_EX, 1, port) # The lock only proves no other live test run owns the port; an orphaned # daemon from a killed run can still squat it (see _probe_bindable). _probe_bindable(port) # --- standalone rsyncd helpers --------------------------------------------- def _set_pdeathsig() -> 'None': """Linux: ask the kernel to send SIGTERM to us if our parent dies. A no-op on every other platform. Used as preexec_fn so a kill -9 of the test process doesn't strand the rsyncd we spawned.""" if not sys.platform.startswith('linux'): return try: import ctypes libc = ctypes.CDLL('libc.so.6', use_errno=True) PR_SET_PDEATHSIG = 1 libc.prctl(PR_SET_PDEATHSIG, 15, 0, 0, 0) # 15 == SIGTERM except OSError: pass def _stop_rsyncd(proc) -> 'None': if proc.poll() is not None: return try: proc.terminate() proc.wait(timeout=2) except subprocess.TimeoutExpired: try: proc.kill() proc.wait(timeout=1) except (subprocess.TimeoutExpired, OSError): pass def start_rsyncd(conf_path, port: int, rsync_cmd: str = None) -> 'subprocess.Popen': """Spawn `rsync --daemon --no-detach --address=127.0.0.1 --port=N --config=conf` and return the Popen handle after the port is accepting connections. The daemon is bound to LOOPBACK ONLY (--address=127.0.0.1): without it, rsync --daemon binds 0.0.0.0 and the test modules would be reachable from the whole LAN. The daemon is killed automatically when this Python process exits (atexit). On Linux, the kernel also signals SIGTERM to the daemon if the parent dies abruptly (PR_SET_PDEATHSIG), so a SIGKILL on the test process doesn't strand the daemon either. The caller is expected to have already claim_ports()'d `port`. rsync_cmd selects the binary to run as the daemon; it defaults to RSYNC_PEER (the peer side of a two-sided run), so ordinary daemon tests get current-client <-> peer-daemon. The reverse-direction test passes rsync_cmd=RSYNC to put the current build on the daemon side and drive with the old client. This is only ever reached from start_test_daemon() in --use-tcp mode; the default (pipe) mode never starts a listening daemon. """ argv = shlex.split(rsync_cmd or RSYNC_PEER) + [ '--daemon', '--no-detach', '--address=127.0.0.1', f'--port={port}', f'--config={conf_path}', ] proc = subprocess.Popen( argv, stdin=subprocess.DEVNULL, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, preexec_fn=_set_pdeathsig, ) atexit.register(_stop_rsyncd, proc) deadline = time.monotonic() + 10 last_err = None while time.monotonic() < deadline: if proc.poll() is not None: test_fail( f"rsyncd exited before listening on port {port} " f"(status={proc.returncode})" ) try: with _socket.create_connection(('127.0.0.1', port), timeout=0.5): return proc except OSError as e: last_err = e time.sleep(0.05) _stop_rsyncd(proc) test_fail(f"rsyncd never listened on 127.0.0.1:{port}: {last_err}") def start_test_daemon(conf_path, port: int, rsync_cmd: str = None) -> str: """Bring up the test daemon and return a URL prefix for client commands. rsync_cmd selects the daemon-side binary (default RSYNC_PEER); pass rsync_cmd=RSYNC for the reverse-direction test (current daemon, old client). This is the single seam every daemon test uses. The transport depends on the mode the runner selected: * DEFAULT (secure) -- no TCP socket at all. Sets RSYNC_CONNECT_PROG so the rsync client forks the daemon over a private stdio pipe. Returns 'rsync://localhost/'. Another local user can't reach it; nothing is listening. * --use-tcp -- starts a real rsyncd bound to 127.0.0.1 on the given claim_ports()-reserved port. Returns 'rsync://localhost:PORT/'. Bound to loopback so off-host/LAN access is impossible; a same-host user could still connect during the test window, which is the documented, accepted cost of explicitly opting into TCP. Build URLs as f"{prefix}module/path". `port` is only used (and claimed) in --use-tcp mode. """ daemon_cmd = rsync_cmd or RSYNC_PEER if USE_TCP: claim_ports(port) start_rsyncd(conf_path, port, daemon_cmd) return f'rsync://localhost:{port}/' os.environ['RSYNC_CONNECT_PROG'] = f'{daemon_cmd} --config={conf_path} --daemon' return 'rsync://localhost/' def require_tcp(reason: str) -> 'None': """Skip the test (exit 77) unless we're in --use-tcp mode. For tests that fundamentally need a real listening socket / TCP peer and have no secure pipe equivalent (the fake-proxy listener; the reverse-DNS hostname-ACL daemon test).""" if not USE_TCP: test_skipped(reason) def rsync_argv(*args: str) -> list: """Return the argv for invoking rsync with the given extra arguments. RSYNC may be a multi-word command (e.g. 'valgrind ... /build/rsync'); we shlex-split it so subprocess sees a proper argv list. Each *args entry is appended verbatim, so callers should pass tokens already split (no embedded option/value joined by spaces). """ return shlex.split(RSYNC) + list(args) def forced_protocol(): """The protocol version pinned via --protocol=N in the RSYNC command, or None when the run isn't pinning one (so the binary negotiates its newest). Protocol-sensitive tests use this to gate sub-cases -- e.g. the split between --append and --append-verify only exists at protocol >= 30; at protocol 29 plain --append behaves like the old verifying append.""" import re m = re.search(r'--protocol[ =](\d+)', RSYNC) return int(m.group(1)) if m else None def run_rsync(*args: str, check: bool = True, capture_output: bool = False) -> subprocess.CompletedProcess: """Run rsync with the given arguments. By default, stdout/stderr inherit (so the runner captures them in the per-test log). Set capture_output=True if the test needs to inspect the output. If check is True (the default), a non-zero exit calls test_fail() with the rsync command line. """ argv = rsync_argv(*args) if capture_output: proc = subprocess.run(argv, capture_output=True, text=True) else: proc = subprocess.run(argv) if check and proc.returncode != 0: test_fail(f"rsync exited {proc.returncode}: {' '.join(argv)}") return proc # --- filesystem helpers ---------------------------------------------------- def makepath(*paths) -> 'None': """Equivalent of rsync.fns makepath: mkdir -p, but for multiple paths.""" for p in paths: os.makedirs(p, exist_ok=True) def rmtree(path) -> 'None': """Remove a tree if it exists, ignoring missing entries.""" p = Path(path) if p.exists() or p.is_symlink(): shutil.rmtree(p, ignore_errors=True) def is_a_link(path) -> bool: """True if 'path' is a symbolic link (dangling or not).""" return os.path.islink(path) def cp_p(src, dst) -> 'None': """Equivalent of rsync.fns cp_p: copy preserving mode + timestamps.""" shutil.copy2(src, dst) def make_data_file(path, size: int) -> 'None': """Equivalent of rsync.fns make_data_file: create `path` with `size` bytes of non-trivial content suitable for rsync's delta algorithm. Prefers /dev/urandom for speed. Falls back to a deterministic LCG seeded from PID and the destination path so successive calls produce distinct content -- matching the shell helper. """ path = str(path) if os.path.exists('/dev/urandom'): try: with open('/dev/urandom', 'rb') as src, open(path, 'wb') as dst: remaining = size while remaining: chunk = src.read(min(remaining, 1 << 16)) if not chunk: break dst.write(chunk) remaining -= len(chunk) if remaining == 0: return except OSError: pass # Fallback: BSD-LCG to printable-ASCII (33..126), so output stays # exactly `size` bytes regardless of awk/utf8 quirks the shell # version worked around. path_seed = int.from_bytes(path.encode(), 'big') & 0xFFFFFFFF state = (os.getpid() + path_seed) % 2147483648 with open(path, 'wb') as f: out = bytearray(size) for i in range(size): state = (state * 1103515245 + 12345) % 2147483648 out[i] = ((state >> 16) % 94) + 33 f.write(bytes(out)) def make_text_file(path, lines: int = 100) -> 'None': """Write a predictable, self-contained text file of `lines` lines. This replaces the old habit of capturing `ls -l /etc` / `ls -l /bin` (falling back to `ls /`) into the test tree. Those tied the fixtures to the host filesystem layout: the directories are absent or unreadable on Android/Termux and other minimal environments, where `ls /` fails outright, and the captured content was never reproducible. The output here is deterministic and depends on nothing outside the suite, so every platform builds the identical fixture. """ content = ''.join( "line %06d the quick brown fox jumps over the lazy dog %d %d\n" % (i, (i * 31) % 97, (i * 131) % 89) for i in range(1, lines + 1) ) with open(str(path), 'w') as f: f.write(content) def get_testuid() -> int: return os.getuid() def get_rootuid() -> int: return 0 def get_rootgid() -> int: return 0 def build_rsyncd_conf() -> 'Path': """Equivalent of rsync.fns build_rsyncd_conf. Writes $scratchdir/test-rsyncd.conf with the four standard modules (test-from, test-to, test-scratch, test-hidden) and a $scratchdir/ ignore23 wrapper that propagates rsync's exit status except for code 23 (vanished/missing source files), which it eats so that the surrounding test can tolerate the partial-transfer case. Returns the path to the config file. Tests typically follow up by setting RSYNC_CONNECT_PROG so rsync forks an in-tree daemon instead of contacting one over the network. """ conf = SCRATCHDIR / 'test-rsyncd.conf' pidfile = SCRATCHDIR / 'rsyncd.pid' logfile = SCRATCHDIR / 'rsyncd.log' my_uid = get_testuid() root_uid = get_rootuid() root_gid = get_rootgid() if my_uid != root_uid: # Non-root cannot specify uid/gid in rsyncd.conf. uid_line = f"#uid = {root_uid}" gid_line = f"#gid = {root_gid}" else: uid_line = f"uid = {root_uid}" gid_line = f"gid = {root_gid}" conf.write_text(f"""\ # rsyncd configuration file autogenerated by rsyncfns.build_rsyncd_conf pid file = {pidfile} use chroot = no munge symlinks = no # Loopback only. In --use-tcp mode the daemon is also bound to 127.0.0.1 # (start_rsyncd passes --address), so this is belt-and-suspenders; in the # default pipe mode there is no socket to guard at all. hosts allow = localhost 127.0.0.0/8 log file = {logfile} transfer logging = yes # We don't define log format here so the test-hidden module defaults # to the internal static string (since we had a crash trying to tweak it). exclude = ? foobar.baz max verbosity = 4 {uid_line} {gid_line} [test-from] \tpath = {FROMDIR} \tlog format = %i %h [%a] %m (%u) %l %f%L \tread only = yes \tcomment = r/o [test-to] \tpath = {TODIR} \tlog format = %i %h [%a] %m (%u) %l %f%L \tread only = no \tcomment = r/w [test-scratch] \tpath = {SCRATCHDIR} \tlog format = %i %h [%a] %m (%u) %l %f%L \tread only = no [test-hidden] \tpath = {FROMDIR} \tlist = no """) ignore23 = SCRATCHDIR / 'ignore23' ignore23.write_text( '#!/bin/sh\n' 'if "${@}"; then exit; fi\n' 'ret=$?\n' 'if test $ret = 23; then exit; fi\n' 'exit $ret\n' ) ignore23.chmod(0o755) return conf def rsync_getgroups() -> list: """List of group ids the test user is a member of, via the getgroups test helper binary. Mirrors rsync.fns rsync_getgroups.""" out = subprocess.check_output([str(TOOLDIR / 'getgroups')], text=True) return out.split() # --- extended attributes (per-OS surface) ---------------------------------- # Mirrors the per-OS xset/xls/RSYNC_PREFIX/RUSR logic from the old # testsuite/rsync.fns + xattrs.test so the xattr / fake-super tests run on # Linux, macOS and FreeBSD (not just Linux). Test attributes use literal # names ("user.foo" etc., exactly as the shell did on every platform); only # rsync's own fake-super attribute name (RSYNC_PREFIX, used for the # "%stat" attr) and the special "equal" attr (RUSR) vary by OS. _SYSTEM = platform.system() # Cygwin reports "CYGWIN_NT-10.0-..." and uses Linux-style user.* xattrs # (rsync builds there with HAVE_LINUX_XATTRS), but CPython on Cygwin lacks # os.*xattr, so we drive the getfattr/setfattr CLIs there instead. _CYGWIN = _SYSTEM.startswith('CYGWIN') # Platforms whose user xattrs live in the "user." namespace encoded in the # attribute name (Linux and Cygwin). macOS/FreeBSD carry the namespace out # of band and a literal "user." prefix is actually rejected there. _LINUX_NS = _SYSTEM == 'Linux' or _CYGWIN # Test attribute names are LOGICAL (un-prefixed, e.g. "foo", "rsync.%stat"); # _xattr_full() adds the "user." prefix on the Linux-namespace platforms. # RSYNC_PREFIX is the logical name of rsync's own fake-super attr ("rsync" # -> "rsync.%stat", and "user.rsync.%stat" on Linux/Cygwin). RUSR is the # prefix for the test's "equal" attr; macOS and Solaris use "rsync.nonuser" # to stay clear of rsync's reserved "rsync.*" space. RSYNC_PREFIX = 'rsync' RUSR = 'rsync.nonuser' if _SYSTEM in ('Darwin', 'SunOS') else 'rsync' def _xattr_full(name: str) -> str: """Map a logical user-xattr name to the on-disk name for this OS.""" return ('user.' + name) if _LINUX_NS else name def xattrs_supported() -> bool: """True if this rsync was built with xattr support AND this platform has a way for the tests to set/list user xattrs.""" vv = run_rsync('-VV', check=True, capture_output=True).stdout if '"xattrs": true' not in vv: return False if _SYSTEM == 'Linux': return hasattr(os, 'setxattr') if _CYGWIN: return shutil.which('setfattr') is not None if _SYSTEM == 'Darwin': return shutil.which('xattr') is not None if _SYSTEM == 'FreeBSD': return shutil.which('setextattr') is not None if _SYSTEM == 'SunOS': return shutil.which('runat') is not None return False # NetBSD/etc.: not yet ported def xattr_set(name: str, value: str, *paths) -> 'None': """Set the user-namespace xattr `name` (logical) = `value` on each path.""" full = _xattr_full(name) for p in paths: p = str(p) if _SYSTEM == 'Linux': os.setxattr(p, full.encode(), value.encode()) elif _CYGWIN: subprocess.run(['setfattr', '-n', full, '-v', value, p], check=True) elif _SYSTEM == 'Darwin': subprocess.run(['xattr', '-w', full, value, p], check=True) elif _SYSTEM == 'FreeBSD': subprocess.run(['setextattr', '-h', 'user', full, value, p], check=True) elif _SYSTEM == 'SunOS': # Solaris extended attributes are a per-file namespace; runat # cd's into it and runs a shell that reads the script on stdin # (the -c form mangles args). Pass name/value via the environment # to dodge quoting; printf writes the value with no trailing # newline, matching the byte-exact value other platforms store. subprocess.run( ['runat', p, '/bin/sh'], input='printf %s "$XVAL" > "$XNAME"\n', text=True, env={**os.environ, 'XNAME': full, 'XVAL': value}, check=True) else: raise NotImplementedError(f"xattr_set on {_SYSTEM}") def xattr_dump(*paths) -> str: """Return a deterministic name=value dump of the user xattrs on `paths`, for comparing a source tree against its rsync'd copy. The format only needs to be self-consistent on a given OS (we never compare across OSes), mirroring the per-OS xls() in the old xattrs.test.""" if _SYSTEM == 'Linux': # Read xattrs natively (symmetric with the os.setxattr used in # xattr_set) so the suite needs no external getfattr. The attr # package's CLI tools are frequently absent -- on Android/Termux # and minimal CI images -- even when the filesystem itself supports # user xattrs, in which case shelling out to getfattr would crash # the test instead of exercising it. The output mimics "getfattr # -d": a "# file:" header then sorted name="value" lines, files # with no user xattrs omitted. out = [] for p in paths: sp = str(p) names = sorted(n for n in os.listxattr(sp) if n.startswith('user.')) if not names: continue out.append(f'# file: {sp}\n') for n in names: v = os.getxattr(sp, n).decode('utf-8', 'surrogateescape') out.append(f'{n}="{v}"\n') out.append('\n') return ''.join(out) if _CYGWIN: # Python on Cygwin lacks os.*xattr, so use the CLI there. return subprocess.check_output( ['getfattr', '-d', *(str(p) for p in paths)], text=True) if _SYSTEM == 'Darwin': out = [] for p in paths: t = subprocess.check_output(['xattr', '-l', str(p)], text=True) out.append('\n'.join(ln.lstrip(' \t') for ln in t.splitlines())) out.append('\n') return ''.join(out) if _SYSTEM == 'FreeBSD': out = [] for p in paths: names = subprocess.check_output( ['lsextattr', '-q', '-h', 'user', str(p)], text=True).split() for n in sorted(names): out.append(subprocess.check_output( ['getextattr', '-h', 'user', n, str(p)], text=True)) return ''.join(out) if _SYSTEM == 'SunOS': # List the file's extended-attribute namespace via runat (script on # stdin), skipping the always-present SUNWattr_* system attrs, and # dump name=value (sorted glob order; $(cat) drops a trailing newline). script = ('for x in *; do case "$x" in SUNWattr_*) continue;; esac; ' 'printf "%s=%s\\n" "$x" "$(cat "$x")"; done\n') out = [] for p in paths: out.append(subprocess.run( ['runat', str(p), '/bin/sh'], input=script, capture_output=True, text=True, check=True).stdout) return ''.join(out) raise NotImplementedError(f"xattr_dump on {_SYSTEM}") def runtest(label: str, fn, *args, **kwargs): """Run a sub-test step with an echoed label, like rsync.fns runtest. The shell helper does `Test $1: $2 ... done.` -- this prints a similar banner and propagates exceptions (which surface as a failing test). """ print(f"Test {label}: ", end="", flush=True) fn(*args, **kwargs) print("done.") def cp_touch(src, dst) -> 'None': """Equivalent of rsync.fns cp_touch: copy preserving timestamps, then forcibly re-touch both source and destination to identical times. On some filesystems cp rounds microsecond timestamps on the destination; rsync.fns works around this by then `touch -r dst src dst`. Here we set both src and dst to dst's mtime/atime after the copy, so a diff of the tls output (which prints times) sees identical entries on both sides. """ shutil.copy2(src, dst) if os.path.isdir(dst): dst = os.path.join(dst, os.path.basename(src)) st = os.stat(dst, follow_symlinks=False) os.utime(src, ns=(st.st_atime_ns, st.st_mtime_ns), follow_symlinks=False) os.utime(dst, ns=(st.st_atime_ns, st.st_mtime_ns), follow_symlinks=False) def build_symlinks() -> 'None': """Equivalent of rsync.fns build_symlinks: a set of canonical relative, absolute, dangling and unsafe symlinks under FROMDIR for symlink tests. """ FROMDIR.mkdir(parents=True, exist_ok=True) (FROMDIR / 'referent').write_text( subprocess.check_output(['date'], text=True) ) os.symlink('referent', FROMDIR / 'relative') os.symlink(str(FROMDIR / 'referent'), FROMDIR / 'absolute') os.symlink('nonexistent', FROMDIR / 'dangling') os.symlink(str(SRCDIR / 'rsync.c'), FROMDIR / 'unsafe') def hands_setup() -> 'None': """Populate FROMDIR with a varied tree of files and directories for the canonical 'hands' transfer test. All content is generated from within the suite (srcdir contents plus make_text_file output) so the fixture is self-contained and reproducible on every platform. """ rmtree(FROMDIR) rmtree(TODIR) TMPDIR.mkdir(parents=True, exist_ok=True) FROMDIR.mkdir(parents=True, exist_ok=True) TODIR.mkdir(parents=True, exist_ok=True) (FROMDIR / 'empty').touch() (FROMDIR / 'emptydir').mkdir(exist_ok=True) # File list of srcdir contents, generated through the tls helper so it # matches the format the rest of the suite uses. (FROMDIR / 'filelist').write_text(rsync_ls_lR(SRCDIR)) # The shell test uses `echo -n` semantics; write_text without a trailing # newline is the cleanest equivalent. (FROMDIR / 'nolf').write_text("This file has no trailing lf") old_umask = os.umask(0) try: os.symlink('nolf', FROMDIR / 'nolf-symlink') finally: os.umask(old_umask) # Concatenate all *.c files in srcdir into a single 'text' file. text = bytearray() for c in sorted(SRCDIR.glob('*.c')): text.extend(c.read_bytes()) (FROMDIR / 'text').write_bytes(bytes(text)) (FROMDIR / 'dir').mkdir(exist_ok=True) shutil.copy(FROMDIR / 'text', FROMDIR / 'dir') (FROMDIR / 'dir' / 'subdir').mkdir(exist_ok=True) (FROMDIR / 'dir' / 'subdir' / 'foobar.baz').write_text("some data\n") (FROMDIR / 'dir' / 'subdir' / 'subsubdir').mkdir(exist_ok=True) # Predictable, self-contained fixture files (the names etc-ltr-list / # bin-lt-list are kept because other tests reference them by name). make_text_file(FROMDIR / 'dir' / 'subdir' / 'subsubdir' / 'etc-ltr-list', 120) (FROMDIR / 'dir' / 'subdir' / 'subsubdir2').mkdir(exist_ok=True) make_text_file(FROMDIR / 'dir' / 'subdir' / 'subsubdir2' / 'bin-lt-list', 200) # --- listing / verification ------------------------------------------------ def rsync_ls_lR(directory) -> str: """Equivalent of rsync.fns rsync_ls_lR: print a sorted ls-style listing of `directory`, pruning .git / auto-build-save / testtmp subtrees, using the project's `tls` helper so the output format matches the rest of the suite. """ cmd = ( "find . -name .git -prune -o -name auto-build-save -prune " "-o -name testtmp -prune -o -print | sort | sed 's/ /\\\\ /g' | " f"xargs '{TOOLDIR}/tls' {TLS_ARGS}" ) proc = subprocess.run(['sh', '-c', cmd], capture_output=True, text=True, cwd=str(directory)) return proc.stdout def checkit(args, expected_dir, actual_dir, skip_file_diff: bool = False, allowed_codes=(0,)) -> 'None': """Run rsync with `args` (a list of extra rsync arguments) and then verify two things: 1. The tls-formatted listings of `expected_dir` and `actual_dir` are identical. 2. (Unless skip_file_diff) diff -r against the two trees reports no differences. `allowed_codes` is the tuple of exit codes treated as success. Pass (0, 23) for daemon-mode transfers that may report partial- transfer codes even when the listings still match. Calls test_fail() on any mismatch. Mirrors the rsync.fns checkit shell helper; callers pass rsync arguments as a Python list rather than as a pre-quoted command string, which avoids the shell-quoting gymnastics that the shell version needed. """ expected_dir = str(expected_dir) actual_dir = str(actual_dir) failed = [] # If TLS_ARGS asks for atimes, the listing must be captured BEFORE the # rsync run because diff'ing files afterwards updates their atimes. ls_from = None if '--atimes' in TLS_ARGS: ls_from = rsync_ls_lR(expected_dir) print(f"Running: rsync {' '.join(args)}") proc = subprocess.run(rsync_argv(*args)) if proc.returncode not in allowed_codes: failed.append(f"status={proc.returncode}") if ls_from is None: ls_from = rsync_ls_lR(expected_dir) ls_to = rsync_ls_lR(actual_dir) print("-------------") print("check how the directory listings compare with diff:") print() if ls_from != ls_to: ls_from_path = TMPDIR / 'ls-from' ls_to_path = TMPDIR / 'ls-to' ls_from_path.write_text(ls_from) ls_to_path.write_text(ls_to) diff = subprocess.run( ['diff', '-u', str(ls_from_path), str(ls_to_path)], capture_output=True, text=True, ) sys.stdout.write(diff.stdout) failed.append("dir-diff") print("-------------") print("check how the files compare with diff:") print() if skip_file_diff: print(" === Skipping (as directed) ===") else: diff = subprocess.run(['diff', '-r', '-u', expected_dir, actual_dir]) if diff.returncode != 0: failed.append("file-diff") print("-------------") if failed: test_fail("Failed: " + " ".join(failed)) def verify_dirs(expected_dir, actual_dir, skip_file_diff: bool = False, label: str = '') -> 'None': """Verify two directory trees match: identical tls listings and (unless skip_file_diff) identical file contents. Same comparison logic as checkit() but with no rsync invocation -- useful when the rsync that produced `actual_dir` had to be driven manually so that its output could be captured for inspection.""" expected_dir = str(expected_dir) actual_dir = str(actual_dir) tag = f"{label}: " if label else "" ls_expected = rsync_ls_lR(expected_dir) ls_actual = rsync_ls_lR(actual_dir) if ls_expected != ls_actual: ls_expected_path = TMPDIR / 'ls-from' ls_actual_path = TMPDIR / 'ls-to' ls_expected_path.write_text(ls_expected) ls_actual_path.write_text(ls_actual) diff = subprocess.run( ['diff', '-u', str(ls_expected_path), str(ls_actual_path)], capture_output=True, text=True, ) sys.stdout.write(diff.stdout) test_fail(f"{tag}directory listings differ between " f"{expected_dir} and {actual_dir}") if not skip_file_diff: diff = subprocess.run(['diff', '-r', '-u', expected_dir, actual_dir]) if diff.returncode != 0: test_fail(f"{tag}file content differs between " f"{expected_dir} and {actual_dir}") def v_filt(text: str) -> str: """Strip the boilerplate lines rsync emits at -v / -vv so callers can diff only the file/directory change lines. Mirrors rsync.fns v_filt: delete the build/progress banners, then everything from the first blank line to end-of-text.""" out = [] skip_prefix = ( 'building file list ', 'sending incremental file list', 'created directory ', 'total: ', 'client charset: ', 'server charset: ', ) for line in text.splitlines(): if line == '': break if line.startswith(skip_prefix): continue if line == 'done': continue if line.endswith(' --whole-file'): continue out.append(line) return '\n'.join(out) + ('\n' if out else '') def checkdiff(args, expected: str, *, filter=None, allowed_codes=(0,), direct: bool = False) -> 'None': """Run a command, capture its stdout, optionally pipe through `filter`, then compare to `expected`. Mirrors rsync.fns checkdiff/checkdiff2. args is normally a list of rsync arguments -- the rsync binary is prepended via rsync_argv. Pass direct=True to run `args` as a literal command (used by tests that drive a wrapper such as BATCH.sh). """ if direct: argv = list(args) label = ' '.join(argv) else: argv = rsync_argv(*args) label = 'rsync ' + ' '.join(args) print(f"Running: {label}") proc = subprocess.run(argv, capture_output=True, text=True) stdout = proc.stdout if proc.stderr: sys.stderr.write(proc.stderr) sys.stdout.write(stdout) failed = [] if proc.returncode not in allowed_codes: failed.append(f"status={proc.returncode}") if filter is not None: stdout = filter(stdout) if stdout != expected: from difflib import unified_diff diff = unified_diff( expected.splitlines(keepends=True), stdout.splitlines(keepends=True), fromfile='expected', tofile='got', ) sys.stdout.write(''.join(diff)) failed.append("output differs") if failed: test_fail("Failed: " + " ".join(failed)) def check_perms(path, expected: str) -> 'None': """Verify that the 9-char rwx permission string of `path` matches `expected` (e.g. 'rwx------'). Calls test_fail() on mismatch.""" mode = os.stat(path, follow_symlinks=False).st_mode bits = [ (0o400, 'r'), (0o200, 'w'), (0o100, 'x'), (0o040, 'r'), (0o020, 'w'), (0o010, 'x'), (0o004, 'r'), (0o002, 'w'), (0o001, 'x'), ] chars = [c if mode & bit else '-' for bit, c in bits] # Layer the setuid/setgid/sticky bits over x as the long-listing format does. if mode & 0o4000: chars[2] = 's' if mode & 0o100 else 'S' if mode & 0o2000: chars[5] = 's' if mode & 0o010 else 'S' if mode & 0o1000: chars[8] = 't' if mode & 0o001 else 'T' perms = ''.join(chars) if perms != expected: print(f"permissions: {perms} on {path}") print(f"should be: {expected}") test_fail(f"check_perms failed for {path}") # --- depth / cross-dir coverage helpers ------------------------------------ # Added for the option-coverage expansion (see testsuite/COVERAGE.md). # The path-handling restructure changes how parent components resolve, so its # bugs surface only at DEPTH and across directory boundaries -- these helpers # build trees with an entry at every level and assert the concrete property an # option controls (not just dest == src). def make_tree(root, depth: int = 3, *, data: bool = False, content_lines: int = 20, data_size: int = 4096, dirname: str = 'd', leaf: str = 'f'): """Create a layered directory tree with one regular file at every level. For depth=3 under `root`: root/f0 root/d1/f1 root/d1/d2/f2 root/d1/d2/d3/f3 so an option's effect can be checked at the tree root AND >=3 levels deep (the parent-component resolution the path restructure rewrites). Returns (dirs, files): `dirs` the created subdirectories outermost-first, `files` the regular files shallow-first. Content is deterministic (make_text_file) unless data=True (make_data_file, delta-friendly). """ root = Path(root) root.mkdir(parents=True, exist_ok=True) dirs = [] files = [] cur = root for level in range(depth + 1): f = cur / f'{leaf}{level}' if data: make_data_file(f, data_size) else: make_text_file(f, content_lines) files.append(f) if level < depth: cur = cur / f'{dirname}{level + 1}' cur.mkdir(exist_ok=True) dirs.append(cur) return dirs, files def walk_files(root) -> list: """Every regular (non-symlink) file under `root`, sorted, recursively. For asserting a per-entry property holds at every depth.""" root = Path(root) return sorted(p for p in root.rglob('*') if p.is_file() and not p.is_symlink()) def walk_dirs(root) -> list: """Every subdirectory under `root`, sorted, recursively.""" root = Path(root) return sorted(p for p in root.rglob('*') if p.is_dir() and not p.is_symlink()) def _tag(label: str) -> str: return f"{label}: " if label else "" def assert_same(a, b, label: str = '') -> 'None': """Fail unless files `a` and `b` have byte-identical content.""" if not filecmp.cmp(str(a), str(b), shallow=False): test_fail(f"{_tag(label)}content differs between {a} and {b}") def assert_mode(path, expected_octal: int, label: str = '') -> 'None': """Fail unless the permission bits (low 12) of `path` equal expected_octal (pass an int like 0o644). Does not follow symlinks.""" mode = stat.S_IMODE(os.stat(path, follow_symlinks=False).st_mode) if mode != expected_octal: test_fail(f"{_tag(label)}mode {mode:04o} != expected " f"{expected_octal:04o} on {path}") def assert_mtime_close(a, b, tol: float = 1.0, label: str = '') -> 'None': """Fail unless the mtimes of `a` and `b` are within `tol` seconds. `b` may be a number (an explicit epoch mtime) instead of a path.""" ma = os.stat(a, follow_symlinks=False).st_mtime mb = b if isinstance(b, (int, float)) else os.stat( b, follow_symlinks=False).st_mtime if abs(ma - mb) > tol: test_fail(f"{_tag(label)}mtime {ma} vs {mb} differ by > {tol}s " f"(checking {a})") def assert_is_symlink(path, target: str = None, label: str = '') -> 'None': """Fail unless `path` is a symlink (optionally pointing exactly at `target`).""" if not os.path.islink(path): test_fail(f"{_tag(label)}{path} is not a symlink") if target is not None: actual = os.readlink(path) if actual != target: test_fail(f"{_tag(label)}{path} -> {actual!r}, " f"expected {target!r}") def assert_hardlinked(a, b, label: str = '') -> 'None': """Fail unless `a` and `b` are the same inode (a hard link / --link-dest result).""" sa = os.stat(a, follow_symlinks=False) sb = os.stat(b, follow_symlinks=False) if (sa.st_dev, sa.st_ino) != (sb.st_dev, sb.st_ino): test_fail(f"{_tag(label)}{a} and {b} are not hard-linked " f"(ino {sa.st_ino} vs {sb.st_ino})") def assert_not_hardlinked(a, b, label: str = '') -> 'None': """Fail if `a` and `b` share an inode (e.g. --copy-dest must copy, not link).""" sa = os.stat(a, follow_symlinks=False) sb = os.stat(b, follow_symlinks=False) if (sa.st_dev, sa.st_ino) == (sb.st_dev, sb.st_ino): test_fail(f"{_tag(label)}{a} and {b} unexpectedly share " f"inode {sa.st_ino}") def assert_exists(path, label: str = '') -> 'None': """Fail unless `path` exists (a symlink counts even if dangling).""" if not os.path.lexists(path): test_fail(f"{_tag(label)}{path} does not exist") def assert_not_exists(path, label: str = '') -> 'None': """Fail if `path` exists (a dangling symlink counts as existing).""" if os.path.lexists(path): test_fail(f"{_tag(label)}{path} exists but should not") _rb_cache = None def resolve_beneath_supported() -> bool: """True if this rsync can FOLLOW an in-tree directory symlink under its secure resolver -- i.e. update a file through a dir-symlink on the receiver (--keep-dirlinks; issue #715). False wherever the portable per-component O_NOFOLLOW fallback is the active resolver: a platform with no kernel "beneath" primitive, Linux < 5.6, a seccomp-blocked openat2, or a --disable-openat2 build. There the delta update through the symlinked directory fails verification. Probed functionally (an initial transfer plus a delta update through a dir-symlink) so it tracks the actual binary rather than a platform name, and cached.""" global _rb_cache if _rb_cache is not None: return _rb_cache probe = SCRATCHDIR / '.rb_probe' rmtree(probe) (probe / 'home' / 'real').mkdir(parents=True) os.symlink('real', probe / 'home' / 'link') (probe / 'src' / 'link').mkdir(parents=True) f = probe / 'src' / 'link' / 'f' make_data_file(f, 40000) def push(): subprocess.run( rsync_argv('-KRl', '--no-whole-file', 'link/f', f"{probe / 'home'}/"), cwd=str(probe / 'src'), stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) push() with open(f, 'ab') as fh: # size change -> forces a delta update fh.write(b'appended tail for delta\n') push() dst = probe / 'home' / 'real' / 'f' _rb_cache = dst.is_file() and filecmp.cmp(str(f), str(dst), shallow=False) rmtree(probe) return _rb_cache def write_daemon_conf(modules, globals=None, *, name: str = 'test-rsyncd.conf') -> 'Path': """Write a custom rsyncd.conf for daemon-parameter tests. `modules` is a list of (module_name, {param: value}) pairs; `globals` an optional dict of global parameters that override the minimal defaults (pid file / use chroot=no / hosts allow / log file / max verbosity). Mirrors build_rsyncd_conf()'s root-aware uid/gid handling (only emitted when running as root) and writes the same `ignore23` wrapper, but lets a test set arbitrary parameters/modules beyond the fixed four. Returns the config path; pair with start_test_daemon(). """ conf = SCRATCHDIR / name pidfile = SCRATCHDIR / 'rsyncd.pid' logfile = SCRATCHDIR / 'rsyncd.log' g = { 'pid file': str(pidfile), 'use chroot': 'no', 'hosts allow': 'localhost 127.0.0.0/8', 'log file': str(logfile), 'max verbosity': '4', } if globals: g.update(globals) if get_testuid() == get_rootuid(): g.setdefault('uid', str(get_rootuid())) g.setdefault('gid', str(get_rootgid())) else: # Non-root cannot set uid/gid in rsyncd.conf. g.pop('uid', None) g.pop('gid', None) lines = ['# autogenerated by rsyncfns.write_daemon_conf', ''] lines += [f'{k} = {v}' for k, v in g.items()] lines.append('') for mod_name, params in modules: lines.append(f'[{mod_name}]') lines += [f'\t{k} = {v}' for k, v in params.items()] lines.append('') conf.write_text('\n'.join(lines) + '\n') ignore23 = SCRATCHDIR / 'ignore23' if not ignore23.exists(): ignore23.write_text( '#!/bin/sh\n' 'if "${@}"; then exit; fi\n' 'ret=$?\n' 'if test $ret = 23; then exit; fi\n' 'exit $ret\n' ) ignore23.chmod(0o755) return conf