From 1346bc623f06fb8f2525c8f651d6bb7ac19bafdc Mon Sep 17 00:00:00 2001 From: Andrew Tridgell Date: Sun, 21 Jun 2026 15:16:08 +1000 Subject: [PATCH] testsuite: shared harness, helpers and unit-test programs The t_rename_secure / t_symlink_secure / t_acl unit harnesses (C), the rsyncfns.py helper library (daemon fixtures, symlink matrix, tree compare, xattr/ACL drivers), runtests.py, and the rsync_proto/xrsync/cmptree/mkvariety helper scripts that the security tests build on. --- runtests.py | 62 +- t_acl.c | 532 +++++++++++++++++ t_chmod_secure.c | 105 ++-- t_rename_secure.c | 206 +++++++ t_stub.c | 9 +- t_symlink_secure.c | 144 +++++ testsuite/COVERAGE.md | 2 +- testsuite/README.md | 33 ++ testsuite/cmptree.py | 82 +++ testsuite/mkvariety.py | 126 +++++ testsuite/rsync_proto.py | 1073 +++++++++++++++++++++++++++++++++++ testsuite/rsyncfns.py | 1164 ++++++++++++++++++++++++++++++++++++-- testsuite/xrsync.py | 164 ++++++ 13 files changed, 3566 insertions(+), 136 deletions(-) create mode 100644 t_acl.c create mode 100644 t_rename_secure.c create mode 100644 t_symlink_secure.c create mode 100755 testsuite/cmptree.py create mode 100755 testsuite/mkvariety.py create mode 100644 testsuite/rsync_proto.py create mode 100755 testsuite/xrsync.py diff --git a/runtests.py b/runtests.py index 809276e3..5acad404 100755 --- a/runtests.py +++ b/runtests.py @@ -27,6 +27,7 @@ import concurrent.futures import fnmatch import glob import os +import signal import subprocess import sys import threading @@ -164,11 +165,30 @@ def get_testuser(): return os.environ.get('LOGNAME', os.environ.get('USER', 'UNKNOWN')) +def _move_aside(path): + """Rename an un-removable directory to a unique sibling so its name is free. + + A rename-storm symlink-race test can corrupt a directory on some filesystems + (OpenBSD FFS soft-updates can leave an "empty" dir that still reports + ENOTEMPTY/EPERM and only fsck clears). `rm -rf` then can't remove it, but + renaming the top dir aside succeeds even with a corrupted descendant, freeing + the original name for a clean scratchdir.""" + n = 0 + while os.path.exists(f"{path}.corrupt.{os.getpid()}.{n}"): + n += 1 + try: + os.rename(path, f"{path}.corrupt.{os.getpid()}.{n}") + except OSError: + pass + + def prep_scratch(scratchdir, srcdir, tooldir, setfacl_nodef): """Prepare a scratch directory for a test.""" if os.path.isdir(scratchdir): subprocess.run(['chmod', '-R', 'u+rwX', scratchdir], capture_output=True) subprocess.run(['rm', '-rf', scratchdir], capture_output=True) + if os.path.isdir(scratchdir): + _move_aside(scratchdir) # rm -rf left corrupted debris; don't inherit it os.makedirs(scratchdir, exist_ok=True) if setfacl_nodef: subprocess.run(setfacl_nodef + [scratchdir], capture_output=True) @@ -319,18 +339,36 @@ def run_one_test(testscript, testbase, scratchdir, base_env, timeout, cmd = ['sh', '-e', testscript] logfile = os.path.join(scratchdir, 'test.log') - try: - with open(logfile, 'w') as log: - proc = subprocess.run( - cmd, - stdout=log, stderr=subprocess.STDOUT, - env=env, timeout=timeout, - cwd=env.get('TOOLDIR', '.') - ) - result = proc.returncode - except subprocess.TimeoutExpired: - result = 1 - with open(logfile, 'a') as log: + with open(logfile, 'w') as log: + # start_new_session: run the test driver as its own session/group leader + # so the daemon, clients and flipper it spawns inherit that group. A + # timeout then killpg's the whole tree (not just the driver), and the + # lock-file sweep can reap a SIGKILLed run's stranded group the same way. + proc = subprocess.Popen( + cmd, + stdout=log, stderr=subprocess.STDOUT, + env=env, cwd=env.get('TOOLDIR', '.'), + start_new_session=True, + ) + try: + result = proc.wait(timeout=timeout) + except subprocess.TimeoutExpired: + # Reap the whole session group, but only if the driver really is its + # own group leader (start_new_session took) and that group isn't ours + # -- killpg of our own group would take down the runner. + try: + pgid = os.getpgid(proc.pid) + except OSError: + pgid = -1 + if pgid == proc.pid and pgid != os.getpgrp(): + try: + os.killpg(pgid, signal.SIGKILL) + except OSError: + proc.kill() + else: + proc.kill() + proc.wait() + result = 1 log.write(f"\nTIMEOUT: test took over {timeout} seconds\n") # Build output text diff --git a/t_acl.c b/t_acl.c new file mode 100644 index 00000000..d18a6516 --- /dev/null +++ b/t_acl.c @@ -0,0 +1,532 @@ +/* + * Unit test for lib/acl.c. + * + * Validates the fd- and at-based POSIX ACL get/set/delete in lib/acl.c against + * the system libacl, used here as an oracle: we set an ACL via one and read it + * back via the other (both directions), round-trip through lib/acl.c, and check + * the default-ACL and delete paths. We deliberately do NOT try to reproduce + * libacl's symlink-following races -- we only compare functional behaviour. + * + * Not linked into rsync itself. Exits 0 if all checks pass, 1 on any failure, + * 77 to skip (built without SUPPORT_ACL_FD, no libacl, or a scratch filesystem + * without ACL support). + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + */ + +#include "rsync.h" +#include "lib/acl.h" + +#include + +#ifndef SUPPORT_ACL_FD + +int main(int argc, char *argv[]) +{ + (void)argc; + (void)argv; + fprintf(stderr, "t_acl: built without SUPPORT_ACL_FD -- skipping\n"); + return 77; +} + +#else + +#include +#include +#include +#include +#include +#include +#include +#include +#ifdef HAVE_ACL_LIBACL_H +#include /* acl_get_perm() */ +#endif + +#define MAX_ENT 16 + +static int errs = 0; +static const char *scratch; + +static void ok(int cond, const char *label) +{ + if (cond) + fprintf(stderr, "OK %s\n", label); + else { + fprintf(stderr, "FAIL %s\n", label); + errs++; + } +} + +static void dump_entries(const char *pfx, const rsync_acl_ent *e, int n) +{ + int i; + for (i = 0; i < n; i++) + fprintf(stderr, " %s tag=0x%02x perm=%o id=%u\n", + pfx, e[i].tag, e[i].perm, + e[i].id == RACL_UNDEFINED_ID ? (unsigned)-1 : e[i].id); +} + +static int ent_cmp(const void *a, const void *b) +{ + const rsync_acl_ent *x = a, *y = b; + if (x->tag != y->tag) + return x->tag < y->tag ? -1 : 1; + if (x->id != y->id) + return x->id < y->id ? -1 : 1; + return 0; +} + +static void canon(rsync_acl_ent *e, int n) +{ + if (n > 1) + qsort(e, n, sizeof e[0], ent_cmp); +} + +static int entries_equal(rsync_acl_ent *a, int na, rsync_acl_ent *b, int nb) +{ + int i; + canon(a, na); + canon(b, nb); + if (na != nb) + return 0; + for (i = 0; i < na; i++) { + if (a[i].tag != b[i].tag || a[i].perm != b[i].perm) + return 0; + if ((a[i].tag == RACL_USER || a[i].tag == RACL_GROUP) + && a[i].id != b[i].id) + return 0; + } + return 1; +} + +/* === libacl oracle helpers === */ + +static uint16_t racl_from_tag(acl_tag_t tag) +{ + switch (tag) { + case ACL_USER_OBJ: return RACL_USER_OBJ; + case ACL_USER: return RACL_USER; + case ACL_GROUP_OBJ: return RACL_GROUP_OBJ; + case ACL_GROUP: return RACL_GROUP; + case ACL_MASK: return RACL_MASK; + case ACL_OTHER: return RACL_OTHER; + } + return 0; +} + +static acl_tag_t tag_from_racl(uint16_t tag) +{ + switch (tag) { + case RACL_USER_OBJ: return ACL_USER_OBJ; + case RACL_USER: return ACL_USER; + case RACL_GROUP_OBJ: return ACL_GROUP_OBJ; + case RACL_GROUP: return ACL_GROUP; + case RACL_MASK: return ACL_MASK; + case RACL_OTHER: return ACL_OTHER; + } + return 0; +} + +/* Convert a libacl acl_t to our neutral entry array. Returns count or -1. */ +static int libacl_to_entries(acl_t acl, rsync_acl_ent *out, int max) +{ + acl_entry_t e; + int n = 0, r; + + for (r = acl_get_entry(acl, ACL_FIRST_ENTRY, &e); r == 1; + r = acl_get_entry(acl, ACL_NEXT_ENTRY, &e)) { + acl_tag_t tag; + acl_permset_t ps; + uint16_t perm = 0; + + if (n >= max) + return -1; + if (acl_get_tag_type(e, &tag) != 0 || acl_get_permset(e, &ps) != 0) + return -1; + if (acl_get_perm(ps, ACL_READ) > 0) + perm |= 4; + if (acl_get_perm(ps, ACL_WRITE) > 0) + perm |= 2; + if (acl_get_perm(ps, ACL_EXECUTE) > 0) + perm |= 1; + out[n].tag = racl_from_tag(tag); + out[n].perm = perm; + out[n].id = RACL_UNDEFINED_ID; + if (tag == ACL_USER || tag == ACL_GROUP) { + void *q = acl_get_qualifier(e); + if (q) { + out[n].id = *(id_t *)q; + acl_free(q); + } + } + n++; + } + if (r < 0) + return -1; + canon(out, n); + return n; +} + +/* Build a libacl acl_t from our neutral entries. */ +static acl_t entries_to_libacl(const rsync_acl_ent *ents, int n) +{ + acl_t acl = acl_init(n); + int i; + + if (!acl) + return NULL; + for (i = 0; i < n; i++) { + acl_entry_t e; + acl_permset_t ps; + if (acl_create_entry(&acl, &e) != 0) + goto fail; + if (acl_set_tag_type(e, tag_from_racl(ents[i].tag)) != 0) + goto fail; + if (acl_get_permset(e, &ps) != 0 || acl_clear_perms(ps) != 0) + goto fail; + if ((ents[i].perm & 4) && acl_add_perm(ps, ACL_READ) != 0) + goto fail; + if ((ents[i].perm & 2) && acl_add_perm(ps, ACL_WRITE) != 0) + goto fail; + if ((ents[i].perm & 1) && acl_add_perm(ps, ACL_EXECUTE) != 0) + goto fail; + if (acl_set_permset(e, ps) != 0) + goto fail; + if (ents[i].tag == RACL_USER || ents[i].tag == RACL_GROUP) { + id_t id = ents[i].id; + if (acl_set_qualifier(e, &id) != 0) + goto fail; + } + } + return acl; + fail: + acl_free(acl); + return NULL; +} + +static int libacl_get(const char *path, acl_type_t type, rsync_acl_ent *out, int max) +{ + acl_t acl = acl_get_file(path, type); + int n; + if (!acl) + return -1; + n = libacl_to_entries(acl, out, max); + acl_free(acl); + return n; +} + +static int libacl_set(const char *path, acl_type_t type, const rsync_acl_ent *ents, int n) +{ + acl_t acl = entries_to_libacl(ents, n); + int rc; + if (!acl) + return -1; + rc = acl_set_file(path, type, acl); + acl_free(acl); + return rc; +} + +/* === scratch helpers === */ + +static char *acl_path(const char *name) +{ + static char buf[4096]; + snprintf(buf, sizeof buf, "%s/%s", scratch, name); + return buf; +} + +static int acl_mkfile(const char *name) +{ + char *p = acl_path(name); + int fd = open(p, O_WRONLY | O_CREAT | O_TRUNC, 0644); + if (fd >= 0) + close(fd); + return fd < 0 ? -1 : 0; +} + +static int acl_mkdir(const char *name) +{ + return mkdir(acl_path(name), 0755); +} + +/* Open a held fd the way set_file_attrs does (REG/DIR, NOFOLLOW). */ +static int open_held(const char *name) +{ + return open(acl_path(name), O_RDONLY | O_NOFOLLOW | O_NONBLOCK | O_CLOEXEC); +} + +/* === comparison tests for one ACL shape === */ + +static void cmp_via_fd(const char *name, int want_default, + const rsync_acl_ent *ents, int n, const char *tag) +{ + char label[256]; + char *path = acl_path(name); + acl_type_t type = want_default ? ACL_TYPE_DEFAULT : ACL_TYPE_ACCESS; + rsync_acl_ent got[MAX_ENT], oracle[MAX_ENT]; + rsync_acl_ent *lib_ents = NULL; + int gc = 0, oc, fd; + + /* 1. set via lib (fd) -> read via libacl */ + fd = open_held(name); + snprintf(label, sizeof label, "%s: open held fd", tag); + ok(fd >= 0, label); + if (fd < 0) + return; + snprintf(label, sizeof label, "%s: xacl_set_fd", tag); + ok(xacl_set_fd(fd, want_default, ents, n) == 0, label); + oc = libacl_get(path, type, oracle, MAX_ENT); + memcpy(got, ents, n * sizeof ents[0]); + snprintf(label, sizeof label, "%s: libacl reads back what xacl_set_fd wrote", tag); + if (!entries_equal(got, n, oracle, oc)) { + dump_entries("set ", got, n); + dump_entries("got ", oracle, oc < 0 ? 0 : oc); + } + ok(oc == n && entries_equal(got, n, oracle, oc), label); + + /* 2. set via libacl -> read via lib (fd) */ + snprintf(label, sizeof label, "%s: libacl_set", tag); + ok(libacl_set(path, type, ents, n) == 0, label); + snprintf(label, sizeof label, "%s: xacl_get_fd reads back what libacl wrote", tag); + if (xacl_get_fd(fd, want_default, &lib_ents, &gc) == 0) { + memcpy(got, ents, n * sizeof ents[0]); + if (!entries_equal(got, n, lib_ents, gc)) { + dump_entries("set ", got, n); + dump_entries("got ", lib_ents, gc); + } + ok(gc == n && entries_equal(got, n, lib_ents, gc), label); + } else + ok(0, label); + if (lib_ents) + free(lib_ents); + lib_ents = NULL; + + /* 3. round-trip lib set -> lib get */ + snprintf(label, sizeof label, "%s: xacl_set_fd/xacl_get_fd round-trip", tag); + if (xacl_set_fd(fd, want_default, ents, n) == 0 + && xacl_get_fd(fd, want_default, &lib_ents, &gc) == 0) { + memcpy(got, ents, n * sizeof ents[0]); + ok(gc == n && entries_equal(got, n, lib_ents, gc), label); + } else + ok(0, label); + if (lib_ents) + free(lib_ents); + + close(fd); +} + +static void cmp_via_at(const char *name, int want_default, + const rsync_acl_ent *ents, int n, const char *tag) +{ + char label[256]; + char *path = acl_path(name); + acl_type_t type = want_default ? ACL_TYPE_DEFAULT : ACL_TYPE_ACCESS; + rsync_acl_ent got[MAX_ENT], oracle[MAX_ENT]; + rsync_acl_ent *lib_ents = NULL; + int gc = 0, oc, dirfd; + + dirfd = open(scratch, O_RDONLY | O_DIRECTORY | O_CLOEXEC); + snprintf(label, sizeof label, "%s: open scratch dirfd", tag); + ok(dirfd >= 0, label); + if (dirfd < 0) + return; + + snprintf(label, sizeof label, "%s: xacl_set_at", tag); + ok(xacl_set_at(dirfd, name, want_default, ents, n) == 0, label); + oc = libacl_get(path, type, oracle, MAX_ENT); + memcpy(got, ents, n * sizeof ents[0]); + snprintf(label, sizeof label, "%s: libacl reads back what xacl_set_at wrote", tag); + ok(oc == n && entries_equal(got, n, oracle, oc), label); + + snprintf(label, sizeof label, "%s: libacl_set", tag); + ok(libacl_set(path, type, ents, n) == 0, label); + snprintf(label, sizeof label, "%s: xacl_get_at reads back what libacl wrote", tag); + if (xacl_get_at(dirfd, name, want_default, &lib_ents, &gc) == 0) { + memcpy(got, ents, n * sizeof ents[0]); + ok(gc == n && entries_equal(got, n, lib_ents, gc), label); + } else + ok(0, label); + if (lib_ents) + free(lib_ents); + + close(dirfd); +} + +/* === scenarios === */ + +#define U(p) { RACL_USER_OBJ, p, RACL_UNDEFINED_ID } +#define G(p) { RACL_GROUP_OBJ, p, RACL_UNDEFINED_ID } +#define M(p) { RACL_MASK, p, RACL_UNDEFINED_ID } +#define O(p) { RACL_OTHER, p, RACL_UNDEFINED_ID } +#define NU(i,p) { RACL_USER, p, i } +#define NG(i,p) { RACL_GROUP, p, i } + +static const rsync_acl_ent A1[] = { U(6), NU(12345,4), G(4), M(6), O(4) }; +static const rsync_acl_ent A2[] = { U(7), NU(1000,6), NU(2000,5), G(5), NG(100,4), NG(200,1), M(7), O(0) }; +static const rsync_acl_ent A3[] = { U(7), NU(4000000000U,5), G(5), M(7), O(4) }; +static const rsync_acl_ent A4[] = { U(7), NU(0,0), G(0), M(4), O(0) }; + +static const rsync_acl_ent D1[] = { U(7), G(5), O(5) }; +static const rsync_acl_ent D2[] = { U(7), NU(1000,6), G(5), NG(100,4), M(6), O(0) }; + +#define NELEM(a) ((int)(sizeof (a) / sizeof (a)[0])) + +static void run_access(const char *base, const rsync_acl_ent *e, int n, const char *tag) +{ + char fdname[128], atname[128]; + char fdtag[160], attag[160]; + + snprintf(fdname, sizeof fdname, "%s_fd", base); + snprintf(atname, sizeof atname, "%s_at", base); + snprintf(fdtag, sizeof fdtag, "access %s [fd]", tag); + snprintf(attag, sizeof attag, "access %s [at]", tag); + + if (acl_mkfile(fdname) == 0) + cmp_via_fd(fdname, 0, e, n, fdtag); + if (xacl_at_available()) { + if (acl_mkfile(atname) == 0) + cmp_via_at(atname, 0, e, n, attag); + } +} + +static void run_default(const char *base, const rsync_acl_ent *e, int n, const char *tag) +{ + char fdname[128], atname[128]; + char fdtag[160], attag[160]; + + snprintf(fdname, sizeof fdname, "%s_fd", base); + snprintf(atname, sizeof atname, "%s_at", base); + snprintf(fdtag, sizeof fdtag, "default %s [fd]", tag); + snprintf(attag, sizeof attag, "default %s [at]", tag); + + if (acl_mkdir(fdname) == 0) + cmp_via_fd(fdname, 1, e, n, fdtag); + if (xacl_at_available()) { + if (acl_mkdir(atname) == 0) + cmp_via_at(atname, 1, e, n, attag); + } +} + +/* delete of a default ACL + the "no explicit ACL" / errno behaviour */ +static void run_misc(void) +{ + rsync_acl_ent *ents = NULL; + int n = 0, fd; + char *p; + + /* default-ACL delete */ + if (acl_mkdir("deldir") == 0) { + fd = open_held("deldir"); + ok(fd >= 0, "misc: open deldir fd"); + if (fd >= 0) { + ok(xacl_set_fd(fd, 1, D2, NELEM(D2)) == 0, "misc: set default to delete"); + ok(xacl_del_default_fd(fd) == 0, "misc: xacl_del_default_fd"); + /* libacl returns an empty (0-entry) default ACL after delete */ + { + acl_t a = acl_get_file(acl_path("deldir"), ACL_TYPE_DEFAULT); + int cnt = -1; + if (a) { + acl_entry_t e; + cnt = acl_get_entry(a, ACL_FIRST_ENTRY, &e); + acl_free(a); + } + ok(cnt == 0, "misc: default ACL is empty after delete"); + } + /* deleting again is still success */ + ok(xacl_del_default_fd(fd) == 0, "misc: xacl_del_default_fd idempotent"); + close(fd); + } + } + + /* a freshly-created file has no explicit access ACL xattr -> count 0 */ + if (acl_mkfile("plainfile") == 0) { + fd = open_held("plainfile"); + ok(fd >= 0, "misc: open plainfile fd"); + if (fd >= 0) { + int rc = xacl_get_fd(fd, 0, &ents, &n); + ok(rc == 0 && n == 0, "misc: xacl_get_fd on mode-only file -> no entries"); + if (ents) + free(ents); + ents = NULL; + /* no default ACL on a regular file -> empty */ + rc = xacl_get_fd(fd, 1, &ents, &n); + ok(rc == 0 && n == 0, "misc: xacl_get_fd default on regular file -> no entries"); + if (ents) + free(ents); + close(fd); + } + } + + /* at-variant: leaf NOFOLLOW must not touch a symlink target */ + if (xacl_at_available()) { + int dirfd; + acl_mkfile("nofollow_target"); + p = acl_path("nofollow_link"); + unlink(p); + if (symlink("nofollow_target", p) == 0 + && (dirfd = open(scratch, O_RDONLY | O_DIRECTORY | O_CLOEXEC)) >= 0) { + rsync_acl_ent before[MAX_ENT], after[MAX_ENT]; + int bc, ac; + /* give the target a distinctive ACL via libacl */ + libacl_set(acl_path("nofollow_target"), ACL_TYPE_ACCESS, A1, NELEM(A1)); + bc = libacl_get(acl_path("nofollow_target"), ACL_TYPE_ACCESS, before, MAX_ENT); + /* attempt to set through the symlink leaf with NOFOLLOW: must fail */ + errno = 0; + ok(xacl_set_at(dirfd, "nofollow_link", 0, A2, NELEM(A2)) != 0, + "misc: xacl_set_at on symlink leaf is refused"); + ac = libacl_get(acl_path("nofollow_target"), ACL_TYPE_ACCESS, after, MAX_ENT); + ok(bc == ac && entries_equal(before, bc, after, ac), + "misc: symlink target ACL unchanged by NOFOLLOW set"); + close(dirfd); + } + } +} + +int main(int argc, char *argv[]) +{ + char *p; + + if (argc > 1) + scratch = argv[1]; + else + scratch = "/tmp"; + + /* Probe filesystem ACL support: set a trivial ACL via libacl. */ + if (acl_mkfile(".acl_probe") != 0) { + fprintf(stderr, "t_acl: cannot create scratch file in %s\n", scratch); + return 77; + } + p = acl_path(".acl_probe"); + if (libacl_set(p, ACL_TYPE_ACCESS, A1, NELEM(A1)) != 0) { + if (errno == EOPNOTSUPP || errno == ENOTSUP || errno == ENOSYS) { + fprintf(stderr, "t_acl: %s has no ACL support -- skipping\n", scratch); + unlink(p); + return 77; + } + fprintf(stderr, "t_acl: probe acl_set_file failed: %s\n", strerror(errno)); + unlink(p); + return 77; + } + unlink(p); + + fprintf(stderr, "t_acl: scratch=%s, setxattrat=%s\n", + scratch, xacl_at_available() ? "yes" : "no"); + + run_access("a1", A1, NELEM(A1), "named-user+mask"); + run_access("a2", A2, NELEM(A2), "multi-named+mask"); + run_access("a3", A3, NELEM(A3), "large-uid"); + run_access("a4", A4, NELEM(A4), "zero-perms"); + + run_default("d1", D1, NELEM(D1), "minimal"); + run_default("d2", D2, NELEM(D2), "named+mask"); + + run_misc(); + + fprintf(stderr, "t_acl: %d failure(s)\n", errs); + return errs ? 1 : 0; +} + +#endif /* SUPPORT_ACL_FD */ diff --git a/t_chmod_secure.c b/t_chmod_secure.c index b99655a4..bb6d2dd2 100644 --- a/t_chmod_secure.c +++ b/t_chmod_secure.c @@ -17,11 +17,6 @@ #include -#if defined(__linux__) && defined(HAVE_OPENAT2) -#include -#include -#endif - int dry_run = 0; int am_root = 0; int am_sender = 0; @@ -35,42 +30,18 @@ short info_levels[COUNT_INFO], debug_levels[COUNT_DEBUG]; static int errs = 0; -/* Probe the running kernel for the RESOLVE_BENEATH-equivalent confinement - * that secure_relative_open() prefers over the per-component O_NOFOLLOW - * walk. Returns 1 if either openat2(RESOLVE_BENEATH) on Linux 5.6+ or - * openat(O_RESOLVE_BENEATH) on FreeBSD 13+ / macOS 15+ is honoured by - * the running kernel, 0 otherwise. The probe opens "." (a directory - * the helper has just chdir'd into) so it can't fail for any reason - * other than the kernel rejecting the requested confinement flag. */ -static int kernel_resolve_beneath_supported(void) + +/* Does do_chmod_at()'s leaf handling refuse to follow a symlink at the final + * component? Yes wherever AT_SYMLINK_NOFOLLOW exists; otherwise the wrapper + * falls back to a following fchmodat() (documented limitation). Mirrors the + * #ifdef ladder in do_fchmodat_nofollow. */ +static int leaf_chmod_nofollow_supported(void) { -#if (defined(__linux__) && defined(HAVE_OPENAT2)) || defined(O_RESOLVE_BENEATH) - int fd; -#endif -#if defined(__linux__) && defined(HAVE_OPENAT2) - if (openat2_usable()) { - struct open_how how; - memset(&how, 0, sizeof how); - how.flags = O_RDONLY | O_DIRECTORY; - how.resolve = RESOLVE_BENEATH | RESOLVE_NO_MAGICLINKS; - fd = syscall(SYS_openat2, AT_FDCWD, ".", &how, sizeof how); - if (fd >= 0) { - close(fd); - return 1; - } - /* ENOSYS = kernel < 5.6 or openat2 seccomp-blocked. Fall through to the O_RESOLVE_BENEATH - * probe in case we're a Linux build running on a kernel that - * gained O_RESOLVE_BENEATH via some out-of-tree backport. */ - } -#endif -#ifdef O_RESOLVE_BENEATH - fd = openat(AT_FDCWD, ".", O_RDONLY | O_DIRECTORY | O_RESOLVE_BENEATH); - if (fd >= 0) { - close(fd); - return 1; - } -#endif +#if defined AT_SYMLINK_NOFOLLOW + return 1; +#else return 0; +#endif } static void check(const char *label, int actual_rc, int expect_ok, @@ -78,7 +49,9 @@ static void check(const char *label, int actual_rc, int expect_ok, { struct stat st; int got_ok = (actual_rc == 0); - if (got_ok != expect_ok) { + /* expect_ok < 0: rc is platform-dependent, don't assert it (leaf-symlink + * scenario); only the target-mode check below is portable. */ + if (expect_ok >= 0 && got_ok != expect_ok) { fprintf(stderr, "FAIL [%s]: rc=%d errno=%d (%s), expected %s\n", label, actual_rc, errno, strerror(errno), expect_ok ? "success" : "rejection"); @@ -130,35 +103,18 @@ int main(int argc, char **argv) * files to mode 0600 so we have a clean baseline to compare. */ - /* Scenario A: legitimate parent dir-symlink. + /* Scenario A: legitimate parent dir-symlink within the tree. * - * On platforms whose kernel offers RESOLVE_BENEATH-equivalent - * confinement (Linux 5.6+ openat2, FreeBSD 13+ / macOS 15+ - * O_RESOLVE_BENEATH), the within-tree symlink is followed and - * the chmod must succeed. - * - * On platforms that fall back to the per-component O_NOFOLLOW - * walk (OpenBSD, NetBSD, Solaris, older Cygwin, HPE NonStop, - * and pre-5.6 Linux), every symlink is rejected -- including - * this legitimate one. That's a real platform limitation (the - * same one that causes the #715 regression there) and the - * expected outcome is rejection. - * - * Detect at runtime and expect accordingly. The other three - * scenarios behave identically on both code paths and need no - * adjustment. */ - int kernel_has_rb = kernel_resolve_beneath_supported(); - fprintf(stderr, "INFO: kernel RESOLVE_BENEATH-equivalent confinement: %s\n", - kernel_has_rb ? "available" : "not available (per-component fallback)"); - + * The within-tree symlink is followed and the chmod must succeed on + * every platform: the kernel RESOLVE_BENEATH paths (Linux 5.6+ openat2, + * FreeBSD 13+ / macOS 15+ O_RESOLVE_BENEATH) and, since the #715/-K + * fallback fix, the per-component O_NOFOLLOW walk too (OpenBSD, NetBSD, + * Solaris, older Cygwin, HPE NonStop, pre-5.6 Linux) -- which now follows + * an in-tree directory symlink whose target is relative and ".."-free. + * Escapes are still rejected on both paths (Scenario B). */ int rc = do_chmod_at("inside_link/sentinel", 0640); - if (kernel_has_rb) { - check("A: legit dir-symlink within tree (kernel confined)", - rc, 1, "realdir/sentinel", 0640); - } else { - check("A: legit dir-symlink within tree (per-component fallback rejects)", - rc, 0, "realdir/sentinel", 0600); - } + check("A: legit dir-symlink within tree (followed)", + rc, 1, "realdir/sentinel", 0640); /* Scenario B: parent symlink escapes the tree -- chmod must be * rejected and the outside file's mode must be unchanged. */ @@ -179,6 +135,21 @@ int main(int argc, char **argv) check("D: top-level file, no parent component", rc, 1, "topfile", 0640); + /* Scenario E: the LEAF component is an escaping symlink -- the chmod- + * specific TOCTOU, distinct from the parent races A/B. realdir is a real + * directory, isolating the O_NOFOLLOW leaf guard. rc is platform-dependent + * (refused on Linux, lchmod-the-symlink on *BSD/macOS), so assert only that + * the outside target's mode is unchanged. */ + if (leaf_chmod_nofollow_supported()) { + rc = do_chmod_at("realdir/leaflink", 0666); + check("E: leaf component is an escaping symlink (must not be followed)", + rc, -1, "../trap/sentinel", 0600); + } else { + fprintf(stderr, "INFO: leaf-nofollow chmod unsupported here; " + "do_chmod_at follows a leaf symlink (documented limitation), " + "skipping scenario E\n"); + } + if (errs) fprintf(stderr, "%d failure(s)\n", errs); return errs ? 1 : 0; diff --git a/t_rename_secure.c b/t_rename_secure.c new file mode 100644 index 00000000..79684a00 --- /dev/null +++ b/t_rename_secure.c @@ -0,0 +1,206 @@ +/* + * Test harness for do_rename_at(): a mixed top-level/slashed rename must still + * resolve the slashed side's parent under secure_relative_open() rather than + * fall back to plain rename(). Not linked into rsync. GPL version 2. + */ + +#include "rsync.h" + +#include + +int dry_run = 0; +int am_root = 0; +int am_sender = 0; +int read_only = 0; +int list_only = 0; +int copy_links = 0; +int copy_unsafe_links = 0; +extern int am_daemon, am_chrooted; + +short info_levels[COUNT_INFO], debug_levels[COUNT_DEBUG]; + +static int errs = 0; + +#ifdef AT_FDCWD +/* The 3.4.3 bug: if either side has no slash the whole op fell back to plain + * rename(), leaving the slashed side's parent outside secure_relative_open(). */ +static int vulnerable_mixed_rename_at(const char *old_path, const char *new_path) +{ + const char *old_slash, *new_slash; + + if (!old_path || !*old_path || *old_path == '/' + || !new_path || !*new_path || *new_path == '/') + return do_rename(old_path, new_path); + + old_slash = strrchr(old_path, '/'); + new_slash = strrchr(new_path, '/'); + if (!old_slash || !new_slash) + return do_rename(old_path, new_path); + + return do_rename_at(old_path, new_path); +} +#endif + +static void check_exists(const char *label, const char *path, int expect_exists) +{ + int exists = access(path, F_OK) == 0; + + if (exists != expect_exists) { + fprintf(stderr, "FAIL [%s]: %s %s, expected %s\n", + label, path, exists ? "exists" : "does not exist", + expect_exists ? "exists" : "does not exist"); + errs++; + return; + } + fprintf(stderr, "OK [%s]: %s %s\n", + label, path, exists ? "exists" : "does not exist"); +} + +static void check_rename(const char *label, const char *old_path, + const char *new_path, int expect_ok) +{ + int rc; + int got_ok; + int saved_errno; + + errno = 0; + rc = do_rename_at(old_path, new_path); + saved_errno = errno; + got_ok = rc == 0; + + if (got_ok != expect_ok) { + fprintf(stderr, "FAIL [%s]: rename %s -> %s rc=%d errno=%d (%s), expected %s\n", + label, old_path, new_path, rc, saved_errno, + strerror(saved_errno), expect_ok ? "success" : "rejection"); + errs++; + return; + } + fprintf(stderr, "OK [%s]: rename %s -> %s %s\n", + label, old_path, new_path, expect_ok ? "succeeded" : "rejected"); +} + +static void check_vulnerable_rename(const char *label, const char *old_path, + const char *new_path) +{ +#ifdef AT_FDCWD + int rc; + int saved_errno; + + errno = 0; + rc = vulnerable_mixed_rename_at(old_path, new_path); + saved_errno = errno; + + if (rc != 0) { + fprintf(stderr, "FAIL [%s]: vulnerable rename %s -> %s rc=%d errno=%d (%s), expected escape\n", + label, old_path, new_path, rc, saved_errno, + strerror(saved_errno)); + errs++; + return; + } + fprintf(stderr, "OK [%s]: vulnerable rename %s -> %s escaped\n", + label, old_path, new_path); +#else + fprintf(stderr, "SKIP [%s]: AT_FDCWD not available\n", label); +#endif +} + +static int run_escape_poc(const char *module_dir) +{ +#ifndef AT_FDCWD + fprintf(stderr, "SKIP: AT_FDCWD not available\n"); + return 77; +#else + if (chdir(module_dir) < 0) { + perror("chdir"); + return 2; + } + + am_daemon = 1; + am_chrooted = 0; + + check_vulnerable_rename("P1: 3.4.3-style top-level source to escaping destination parent", + "poc-top-to-escape", "escape_link/vuln-created"); + check_exists("P1 source consumed", "poc-top-to-escape", 0); + check_exists("P1 outside destination created", "../trap/vuln-created", 1); + + check_vulnerable_rename("P2: 3.4.3-style escaping source parent to top-level destination", + "escape_link/poc-outside-source", "vuln-stolen"); + check_exists("P2 outside source consumed", "../trap/poc-outside-source", 0); + check_exists("P2 destination created in module", "vuln-stolen", 1); + + check_rename("P3: fixed top-level source to escaping destination parent", + "fixed-top-to-escape", "escape_link/fixed-created", 0); + check_exists("P3 source preserved", "fixed-top-to-escape", 1); + check_exists("P3 outside destination absent", "../trap/fixed-created", 0); + + check_rename("P4: fixed escaping source parent to top-level destination", + "escape_link/fixed-outside-source", "fixed-stolen", 0); + check_exists("P4 outside source preserved", "../trap/fixed-outside-source", 1); + check_exists("P4 destination absent", "fixed-stolen", 0); + + if (errs) + fprintf(stderr, "%d failure(s)\n", errs); + return errs ? 1 : 0; +#endif +} + +int main(int argc, char **argv) +{ + if (argc == 3 && strcmp(argv[1], "--poc") == 0) + return run_escape_poc(argv[2]); + + if (argc != 2) { + fprintf(stderr, "usage: %s [--poc] \n", argv[0]); + return 2; + } + +#ifndef AT_FDCWD + fprintf(stderr, "SKIP: AT_FDCWD not available\n"); + return 77; +#else + if (chdir(argv[1]) < 0) { + perror("chdir"); + return 2; + } + + am_daemon = 1; + am_chrooted = 0; + + /* Plain mixed paths must keep working. */ + check_rename("A: top-level source to slashed destination", + "top-to-dir", "realdir/top-to-dir", 1); + check_exists("A source consumed", "top-to-dir", 0); + check_exists("A destination created", "realdir/top-to-dir", 1); + + check_rename("B: slashed source to top-level destination", + "realdir/dir-to-top", "dir-to-top", 1); + check_exists("B source consumed", "realdir/dir-to-top", 0); + check_exists("B destination created", "dir-to-top", 1); + + /* A slashed destination parent that escapes the module must be rejected. */ + check_rename("C: top-level source to escaping destination parent", + "top-to-escape", "escape_link/new-outside", 0); + check_exists("C source preserved", "top-to-escape", 1); + check_exists("C outside destination absent", "../trap/new-outside", 0); + + /* A slashed source parent that escapes the module must be rejected too. */ + check_rename("D: escaping source parent to top-level destination", + "escape_link/outside-source", "stolen-from-outside", 0); + check_exists("D outside source preserved", "../trap/outside-source", 1); + check_exists("D destination absent", "stolen-from-outside", 0); + + check_rename("E: shared slashed parent", + "realdir/same-old", "realdir/same-new", 1); + check_exists("E source consumed", "realdir/same-old", 0); + check_exists("E destination created", "realdir/same-new", 1); + + check_rename("F: top-level source to top-level destination", + "top-old", "top-new", 1); + check_exists("F source consumed", "top-old", 0); + check_exists("F destination created", "top-new", 1); + + if (errs) + fprintf(stderr, "%d failure(s)\n", errs); + return errs ? 1 : 0; +#endif +} diff --git a/t_stub.c b/t_stub.c index 2b99e74d..7877e241 100644 --- a/t_stub.c +++ b/t_stub.c @@ -25,6 +25,7 @@ int do_fsync = 0; int inplace = 0; int am_daemon = 0; int am_chrooted = 0; +int insecure_links = 0; int modify_window = 0; int preallocate_files = 0; int sparse_files = 0; @@ -45,6 +46,7 @@ size_t max_alloc = (size_t)-1; /* test helpers are not memory-constrained; * hits at its first my_strdup() call. */ char *partial_dir; char *module_dir; +int module_dirfd = -1; /* curr_dir[]/curr_dir_len (read by secure_relative_open) are defined in * syscall.c, which every helper links -- no stub needed here. */ filter_rule_list daemon_filter_list; @@ -82,7 +84,7 @@ filter_rule_list daemon_filter_list; return 0; } - int copy_xattrs(UNUSED(const char *source), UNUSED(const char *dest)) + int copy_xattrs(UNUSED(const char *source), UNUSED(const char *dest), UNUSED(int dest_fd)) { return -1; } @@ -107,6 +109,11 @@ filter_rule_list daemon_filter_list; return 0; } + BOOL lp_insecure_links(UNUSED(int mod)) +{ + return 0; +} + const char *who_am_i(void) { return "tester"; diff --git a/t_symlink_secure.c b/t_symlink_secure.c new file mode 100644 index 00000000..41cb3a02 --- /dev/null +++ b/t_symlink_secure.c @@ -0,0 +1,144 @@ +/* + * Test harness for the fake-super branches of do_symlink_at()/do_mknod_at(). + * Fake-super stores a symlink/device as a placeholder file, so the create + * resolves the final component; the no-slash branch used to fall back to + * do_symlink()/do_mknod(), whose plain open() followed a planted basename + * symlink and escaped the module. Checks the fixed wrappers refuse it; + * --poc shows the old fallback escaping. Not linked into rsync. GPL version 2. + */ + +#include "rsync.h" + +#include + +/* The symlink placeholder (and thus this escape) exists only where symlink + * xattrs are unavailable -- the same guard do_symlink() uses. Elsewhere + * symlink() fails EEXIST on a planted link, so only the device path applies. */ +#if defined SUPPORT_LINKS && (defined NO_SYMLINK_XATTRS || defined NO_SYMLINK_USER_XATTRS) +#define TEST_SYMLINK_PLACEHOLDER 1 +#endif + +int dry_run = 0; +int am_root = -1; /* --fake-super */ +int am_sender = 0; +int read_only = 0; +int list_only = 0; +int copy_links = 0; +int copy_unsafe_links = 0; +extern int am_daemon, am_chrooted; + +short info_levels[COUNT_INFO], debug_levels[COUNT_DEBUG]; + +static int errs = 0; + +static void check_preserved(const char *label, const char *victim, const char *want) +{ + char buf[256]; + int fd = open(victim, O_RDONLY); + ssize_t n = fd >= 0 ? read(fd, buf, sizeof buf - 1) : -1; + + if (fd >= 0) + close(fd); + if (n < 0) + n = 0; + buf[n] = '\0'; + if (n > 0 && buf[n-1] == '\n') + buf[n-1] = '\0'; + + if (strcmp(buf, want) != 0) { + fprintf(stderr, "FAIL [%s]: victim %s = \"%s\", expected \"%s\" " + "(basename symlink was followed -> module escape)\n", + label, victim, buf, want); + errs++; + return; + } + fprintf(stderr, "OK [%s]: victim %s preserved\n", label, victim); +} + +static void check_clobbered(const char *label, const char *victim, const char *unwanted) +{ + char buf[256]; + int fd = open(victim, O_RDONLY); + ssize_t n = fd >= 0 ? read(fd, buf, sizeof buf - 1) : -1; + + if (fd >= 0) + close(fd); + if (n < 0) + n = 0; + buf[n] = '\0'; + if (n > 0 && buf[n-1] == '\n') + buf[n-1] = '\0'; + + if (strcmp(buf, unwanted) != 0) { + fprintf(stderr, "FAIL [%s]: victim %s = \"%s\", expected the escape to write \"%s\"\n", + label, victim, buf, unwanted); + errs++; + return; + } + fprintf(stderr, "OK [%s]: victim %s clobbered as expected (escape demonstrated)\n", + label, victim); +} + +int main(int argc, char **argv) +{ +#ifndef AT_FDCWD + fprintf(stderr, "SKIP: AT_FDCWD not available\n"); + return 77; +#else + int poc = 0; + const char *moddir; + + if (argc == 3 && strcmp(argv[1], "--poc") == 0) { + poc = 1; + moddir = argv[2]; + } else if (argc == 2) { + moddir = argv[1]; + } else { + fprintf(stderr, "usage: %s [--poc] \n", argv[0]); + return 2; + } + + if (chdir(moddir) < 0) { + perror("chdir"); + return 2; + } + + am_daemon = 1; + am_chrooted = 0; + am_root = -1; /* fake-super: symlinks/devices stored as files */ + + if (poc) { + /* Pre-fix fallback: a no-slash path went to do_symlink()/do_mknod(), + * which open() the basename without O_NOFOLLOW. */ +#ifdef TEST_SYMLINK_PLACEHOLDER + do_symlink("VULN_SYM_PAYLOAD", "sympath"); + check_clobbered("poc do_symlink bare", "../outside/secret_sym", + "VULN_SYM_PAYLOAD"); +#endif + do_mknod("nodpath", S_IFCHR | 0600, 0); + check_clobbered("poc do_mknod bare", "../outside/secret_nod", ""); + return errs ? 1 : 0; + } + + /* Fixed wrappers: a bare-path basename symlink must not be followed; + * the victim outside the module stays untouched. */ +#ifdef TEST_SYMLINK_PLACEHOLDER + do_symlink_at("FIXED_SYM_PAYLOAD", "sympath"); + check_preserved("do_symlink_at bare", "../outside/secret_sym", "VICTIM_SYM"); + + /* Slashed path for parity (already protected before the fix). */ + do_symlink_at("FIXED_SYM_PAYLOAD", "sub/sympath2"); + check_preserved("do_symlink_at slashed", "../outside/secret_sym2", "VICTIM_SYM2"); +#endif + + do_mknod_at("nodpath", S_IFCHR | 0600, 0); + check_preserved("do_mknod_at bare", "../outside/secret_nod", "VICTIM_NOD"); + + do_mknod_at("sub/nodpath2", S_IFCHR | 0600, 0); + check_preserved("do_mknod_at slashed", "../outside/secret_nod2", "VICTIM_NOD2"); + + if (errs) + fprintf(stderr, "%d failure(s)\n", errs); + return errs ? 1 : 0; +#endif +} diff --git a/testsuite/COVERAGE.md b/testsuite/COVERAGE.md index 6f6e37cf..48dd50d6 100644 --- a/testsuite/COVERAGE.md +++ b/testsuite/COVERAGE.md @@ -90,7 +90,7 @@ Status legend: ✓ property asserted · `~` shallow / by an existing ported test | -x, --one-file-system | — | — | — | ✗ (needs a mount boundary) | | --preallocate / --fsync | — | — | — | ✗ | | -B, --block-size | — | — | — | ✗ | -| --max-alloc | — | — | — | ✗ | +| --max-alloc | max-alloc-zero-rejected*new*, daemon-max-alloc-zero*new* | — | — | ✓ zero rejected locally and when forwarded to the daemon (needs an old client) | ### Filtering | option | test(s) | depth | x-dir | notes / gap | diff --git a/testsuite/README.md b/testsuite/README.md index 4883a5e8..4dc94d8c 100644 --- a/testsuite/README.md +++ b/testsuite/README.md @@ -9,6 +9,39 @@ is gone). Shared helpers live in `testsuite/rsyncfns.py`. A handful of C helper programs (`tls`, `getgroups`, `trimslash`, …) are built alongside `rsync` and used by some tests. Coverage notes are in [COVERAGE.md](COVERAGE.md). +## Writing tests + +Favour readability — a test is also documentation of the behaviour it pins, so +prefer clarity over cleverness: + +* When a test writes an `rsyncd.conf`, write it as a triple-quoted f-string so + the actual config is readable top-to-bottom, with module parameters indented + with plain spaces. Don't build it from adjacent string literals full of `\n` + (and `\t`) escapes. The daemon's parser accepts space-indented parameters. +* Better still, use the structured helpers in `rsyncfns.py` when a stock config + will do: `write_daemon_conf(modules, globals)` (per-test modules/params) or + `build_rsyncd_conf()` (the four standard modules). They also handle the + root-only `uid`/`gid` lines for you (needed so a `use chroot = no` daemon run + as root can read a root-owned module). +* For config that varies (e.g. those root-only `uid`/`gid` lines), interpolate a + single optional block that expands when needed and is an empty string + otherwise, rather than splicing pieces together: + + ```python + root = get_testuid() == get_rootuid() + ids = f"uid = {get_rootuid()}\ngid = {get_rootgid()}" if root else "" + conf.write_text(f"""\ + pid file = {base}/rsyncd.pid + use chroot = no + {ids} + log file = {base}/rsyncd.log + + [m] + path = {mod} + read only = yes + """) + ``` + ## Running the tests ### Via make diff --git a/testsuite/cmptree.py b/testsuite/cmptree.py new file mode 100755 index 00000000..5ae6962b --- /dev/null +++ b/testsuite/cmptree.py @@ -0,0 +1,82 @@ +#!/usr/bin/env python3 +"""Compare two directory trees and print any differences. + +Reuses rsyncfns.compare_trees(), so it is special-file aware (never +opens a fifo/socket/device as a stream) and reports differences in: the tls +listing (type, mode, owner, size, mtime, symlink target for every inode), +regular-file contents, user xattrs, POSIX ACLs, and hard-link grouping. + +Works on any two trees, not just ones built by mkvariety.py. Exit status is 0 +when the trees match, 1 when they differ. + +Examples: + testsuite/cmptree.py /tmp/vt/transfer_root /tmp/copy + testsuite/cmptree.py --no-acls --no-xattrs treeA treeB +""" + +import argparse +import os +import sys + +_HERE = os.path.dirname(os.path.abspath(__file__)) + + +def _resolve_tooldir(opt): + """Directory holding the `tls` helper (needed for the listing comparison).""" + for d in (opt, os.path.dirname(_HERE), os.getcwd()): + if d and os.path.exists(os.path.join(d, 'tls')): + return d + return None + + +def main(): + ap = argparse.ArgumentParser( + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter) + ap.add_argument('tree_a') + ap.add_argument('tree_b') + ap.add_argument('--no-xattrs', dest='xattrs', action='store_false', + help='skip user-xattr comparison') + ap.add_argument('--no-acls', dest='acls', action='store_false', + help='skip POSIX ACL comparison') + ap.add_argument('--tooldir', default=None, + help='directory holding the tls helper ' + '(default: the build tree, else cwd)') + ap.add_argument('-q', '--quiet', action='store_true', + help='print nothing; only set the exit status') + args = ap.parse_args() + + for t in (args.tree_a, args.tree_b): + if not os.path.isdir(t): + ap.error(f"not a directory: {t}") + + tooldir = _resolve_tooldir(args.tooldir) + if tooldir is None: + ap.error("cannot find the 'tls' helper (needed for the listing " + "comparison); build it with `make check-progs` or pass " + "--tooldir DIR") + + # rsyncfns reads these at import time; compare_trees only needs + # TOOLDIR (for tls). scratchdir must be an existing dir but is unused here. + import tempfile + os.environ.setdefault('scratchdir', tempfile.gettempdir()) + os.environ.setdefault('srcdir', os.getcwd()) + os.environ.setdefault('TOOLDIR', tooldir) + os.environ.setdefault('RSYNC', 'rsync') + sys.path.insert(0, _HERE) + import rsyncfns as R + + diffs = R.compare_trees(args.tree_a, args.tree_b, + with_acls=args.acls, with_xattrs=args.xattrs) + if diffs: + if not args.quiet: + print(f"trees DIFFER ({len(diffs)} difference(s)):") + print('\n'.join(diffs)) + return 1 + if not args.quiet: + print("trees are identical") + return 0 + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/testsuite/mkvariety.py b/testsuite/mkvariety.py new file mode 100755 index 00000000..dd824778 --- /dev/null +++ b/testsuite/mkvariety.py @@ -0,0 +1,126 @@ +#!/usr/bin/env python3 +"""Generate a "variety tree" for manual inspection and ad-hoc rsync testing. + +Builds exactly the tree that rsyncfns.make_variety_tree() produces for the +regression suite (every inode type rsync handles, with heavy symlink, perm, +xattr, ACL, hard-link and -- as root -- ownership coverage), so you can drive +rsync at it by hand. Transfer the transfer_root/ subdir; the escape/ links +deliberately point up into the sibling above/ tree. + +Examples: + testsuite/mkvariety.py /tmp/vt + testsuite/mkvariety.py /tmp/vt --list + sudo testsuite/mkvariety.py /tmp/vt # adds device nodes + owners + testsuite/mkvariety.py /tmp/vt --no-acls --depth 4 + + rsync -aHAX --specials --devices /tmp/vt/transfer_root/ /tmp/copy/ + rsync -a --safe-links /tmp/vt/transfer_root/ /tmp/copy2/ +""" + +import argparse +import os +import stat +import sys +from pathlib import Path + +_HERE = os.path.dirname(os.path.abspath(__file__)) + + +def _resolve_rsync(opt): + """Find an rsync binary to probe xattr/ACL capability with.""" + if opt: + return os.path.abspath(opt) + cand = os.path.join(os.path.dirname(_HERE), 'rsync') # build-tree ./rsync + if os.path.exists(cand): + return cand + from shutil import which + return which('rsync') or 'rsync' + + +def _detect(flag, prober, default): + if flag is not None: + return flag + try: + return prober() + except Exception: + return default + + +def list_tree(root): + rootp = Path(root) + for dp, dns, fns in os.walk(root): # followlinks=False + dns.sort() + for name in sorted(dns + fns): + p = Path(dp) / name + m = p.lstat().st_mode + tgt = ' -> ' + os.readlink(p) if stat.S_ISLNK(m) else '' + print(f"{stat.filemode(m)} {p.relative_to(rootp)}{tgt}") + + +def main(): + ap = argparse.ArgumentParser( + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter) + ap.add_argument('target', help='directory to create (REMOVED first if it exists)') + ap.add_argument('--depth', type=int, default=8, help='backbone depth (default 8)') + ap.add_argument('--seed', default='0x5A17', + help='int seed for the fixed choices (default 0x5A17)') + ap.add_argument('--rsync', default=None, + help='rsync binary for xattr/ACL capability probes ' + '(default: ./rsync in the build tree, else PATH rsync)') + ap.add_argument('--list', action='store_true', + help='print a recursive listing of the tree afterwards') + for cap, helptext in (('xattrs', 'user xattrs'), ('acls', 'POSIX ACLs'), + ('devices', 'char/block device nodes (needs root)'), + ('owners', 'mixed uid/gid (needs root)')): + g = ap.add_mutually_exclusive_group() + g.add_argument(f'--{cap}', dest=cap, action='store_true', default=None, + help=f'force {helptext} on') + g.add_argument(f'--no-{cap}', dest=cap, action='store_false', + help=f'force {helptext} off') + args = ap.parse_args() + + target = Path(args.target).resolve() + for bad in (Path('/'), Path.home(), Path.cwd()): + if target == bad: + ap.error(f"refusing to use {target} as the target (it is wiped first)") + + rsync = _resolve_rsync(args.rsync) + + # rsyncfns reads these at import time; set them before importing it. + os.environ.setdefault('scratchdir', str(target.parent)) + os.environ.setdefault('srcdir', os.getcwd()) + os.environ.setdefault('TOOLDIR', os.path.dirname(rsync) or os.getcwd()) + os.environ.setdefault('RSYNC', rsync) + sys.path.insert(0, _HERE) + import rsyncfns as R + + caps = dict( + with_xattrs=_detect(args.xattrs, R.xattrs_supported, False), + with_acls=_detect(args.acls, R.acls_supported, False), + with_devices=args.devices if args.devices is not None + else R.devices_supported(), + with_owners=args.owners if args.owners is not None + else R.owners_supported(), + ) + + info = R.make_variety_tree(target, depth=args.depth, + seed=int(args.seed, 0), **caps) + + tr = info['transfer_root'] + print(f"created {sum(info['counts'].values())} entries under {target}") + print(f" counts: {info['counts']}") + print(f" caps: xattrs={caps['with_xattrs']} acls={caps['with_acls']} " + f"devices={caps['with_devices']} owners={caps['with_owners']}") + if not caps['with_devices'] or not caps['with_owners']: + print(" (run as root to add device nodes and mixed ownership)") + print(f"\ntransfer this: {tr}/") + print(f"e.g. rsync -aHAX --specials --devices {tr}/ /tmp/copy/") + + if args.list: + print() + list_tree(target) + + +if __name__ == '__main__': + main() diff --git a/testsuite/rsync_proto.py b/testsuite/rsync_proto.py new file mode 100644 index 00000000..7d77c4eb --- /dev/null +++ b/testsuite/rsync_proto.py @@ -0,0 +1,1073 @@ +#!/usr/bin/env python3 +"""Minimal pure-Python implementation of the rsync *client/sender* side of the +daemon protocol -- enough to push a (possibly malformed) file list to a real +rsync daemon receiver over a TCP socket. + +Why this exists +--------------- +Several rsync security fixes are for crashes the receiver hits while parsing a +file list that no cooperating sender would ever produce. The historical way to +test them was build_patched_rsync(): copy the source, string-replace a line in +the sender, recompile. That is slow (a full rsync rebuild per test) and brittle +(it breaks when the patched line is refactored). This module replaces that with +a declarative wire-format speaker: encode a normal file entry, then flip one +field. + +Design notes / scope +-------------------- +* It is pinned to protocol 30 and advertises NO optional client capabilities + (no 'i' => no inc_recurse, no 'v' => byte/short flist flags and no string + negotiation), which keeps the encoding surface small. Per send_file_entry() + / setup_protocol() in the C source. If the on-wire protocol changes, the + constants and encoders here must be regenerated -- this is deliberately a + second implementation kept tiny for that reason. +* The integer encoders are byte-for-byte ports of io.c (write_varint / + write_varlong / read_varint / read_int), so the bytes match the C exactly. +* It is structured as a fuzzing substrate: FileEntry holds each field + separately and every field (or the whole entry) can be overridden with raw + bytes, so a future fuzzer can mutate one field at a time. + +This is test-only code; it is not built or shipped. +""" + +import socket +import struct + +# --------------------------------------------------------------------------- +# Protocol constants (mirror rsync.h) +# --------------------------------------------------------------------------- +MPLEX_BASE = 7 +MSG_DATA = 0 # FNONE-based data channel +MSG_ERROR_XFER = 1 # FERROR_XFER +MSG_INFO = 2 # FINFO +MSG_ERROR = 3 # FERROR +MSG_WARNING = 4 # FWARNING +MSG_DELETED = 101 # deleted a file on the receiving side +MSG_NO_SEND = 102 # sender failed to open a file we wanted + +# XMIT_* file-entry flags (rsync.h) +XMIT_TOP_DIR = 1 << 0 +XMIT_SAME_MODE = 1 << 1 +XMIT_EXTENDED_FLAGS = 1 << 2 +XMIT_SAME_UID = 1 << 3 +XMIT_SAME_GID = 1 << 4 +XMIT_SAME_NAME = 1 << 5 +XMIT_LONG_NAME = 1 << 6 +XMIT_SAME_TIME = 1 << 7 +XMIT_SAME_RDEV_MAJOR = 1 << 8 +XMIT_NO_CONTENT_DIR = 1 << 8 +XMIT_HLINKED = 1 << 9 +XMIT_USER_NAME_FOLLOWS = 1 << 10 +XMIT_GROUP_NAME_FOLLOWS = 1 << 11 +XMIT_HLINK_FIRST = 1 << 12 +XMIT_MOD_NSEC = 1 << 13 + +# from sys/stat.h +S_IFREG = 0o100000 +S_IFDIR = 0o040000 +S_IFLNK = 0o120000 +S_IFMT = 0o170000 + + +def S_ISREG(m): + return (m & S_IFMT) == S_IFREG + + +def S_ISDIR(m): + return (m & S_IFMT) == S_IFDIR + + +def S_ISLNK(m): + return (m & S_IFMT) == S_IFLNK + + +DEFAULT_PROTOCOL = 30 + +NDX_DONE = -1 +NDX_FLIST_EOF = -2 +NDX_FLIST_OFFSET = -101 + +# Transfer-phase iflags (rsync.h) and basis-type tags. +ITEM_BASIS_TYPE_FOLLOWS = 1 << 11 +ITEM_XNAME_FOLLOWS = 1 << 12 +ITEM_IS_NEW = 1 << 13 +ITEM_TRANSFER = 1 << 15 +FNAMECMP_FNAME = 0x80 +FNAMECMP_FUZZY = 0x83 + +CHUNK_SIZE = 32 * 1024 + +# --------------------------------------------------------------------------- +# Integer encoders -- exact ports of io.c +# --------------------------------------------------------------------------- + +def w_byte(x): + return bytes([x & 0xFF]) + + +def w_shortint(x): + return struct.pack('> 8) & 0xFF + b[3] = (v >> 16) & 0xFF + b[4] = (v >> 24) & 0xFF + cnt = 4 + while cnt > 1 and b[cnt] == 0: + cnt -= 1 + bit = 1 << (7 - cnt + 1) + if b[cnt] >= bit: + cnt += 1 + b[0] = (~(bit - 1)) & 0xFF + elif cnt > 1: + b[0] = (b[cnt] | ((~(bit * 2 - 1)) & 0xFF)) & 0xFF + else: + b[0] = b[1] + return bytes(b[:cnt]) + + +def w_varlong(x, min_bytes): + """Port of io.c write_varlong().""" + b = bytearray(9) + v = x & ((1 << 64) - 1) + for i in range(8): + b[1 + i] = (v >> (8 * i)) & 0xFF + cnt = 8 + while cnt > min_bytes and b[cnt] == 0: + cnt -= 1 + bit = 1 << (7 - cnt + min_bytes) + if b[cnt] >= bit: + cnt += 1 + b[0] = (~(bit - 1)) & 0xFF + elif cnt > min_bytes: + b[0] = (b[cnt] | ((~(bit * 2 - 1)) & 0xFF)) & 0xFF + else: + b[0] = b[cnt] + return bytes(b[:cnt]) + + +def w_varint30(x, protocol=DEFAULT_PROTOCOL): + # write_varint30(): varint at proto >= 30, plain int below. + return w_varint(x) if protocol >= 30 else w_int(x) + + +def w_varlong30(x, min_bytes, protocol=DEFAULT_PROTOCOL): + return w_varlong(x, min_bytes) if protocol >= 30 else w_int(x) + + +_INT_BYTE_EXTRA = ([0] * 32) + ([1] * 16) + ([2] * 8) + ([3] * 4) + ([4] * 2) + [5, 6] + + +def to_wire_mode(mode): + # rsync.h to_wire_mode(): identity on Linux (S_ISLNK already 0120000 etc). + return mode + + +def w_sum_head(count, blength, s2length, remainder): + """io.c write_sum_head() at protocol >= 27: 4 ints.""" + return w_int(count) + w_int(blength) + w_int(s2length) + w_int(remainder) + + +def get_checksum1(buf): + """Port of checksum.c get_checksum1() (CHAR_OFFSET == 0): rsync's rolling + block checksum (sum1). `buf` is bytes; the bytes are signed-char and the + accumulators are uint32 (wrapping), matching the C exactly.""" + if isinstance(buf, str): + buf = buf.encode() + sb = [c - 256 if c >= 128 else c for c in buf] + n = len(sb) + s1 = s2 = 0 + i = 0 + while i < n - 4: + s2 = (s2 + 4 * (s1 + sb[i]) + 3 * sb[i + 1] + 2 * sb[i + 2] + sb[i + 3]) & 0xFFFFFFFF + s1 = (s1 + sb[i] + sb[i + 1] + sb[i + 2] + sb[i + 3]) & 0xFFFFFFFF + i += 4 + while i < n: + s1 = (s1 + sb[i]) & 0xFFFFFFFF + s2 = (s2 + s1) & 0xFFFFFFFF + i += 1 + return ((s1 & 0xFFFF) + ((s2 & 0xFFFF) << 16)) & 0xFFFFFFFF + + +def w_vstring(s): + """io.c write_vstring(): a 1- or 2-byte length prefix then the bytes.""" + if isinstance(s, str): + s = s.encode() + n = len(s) + if n > 0x7F: + return bytes([n // 0x100 + 0x80, n & 0xFF]) + s + return bytes([n]) + s + + +_PERM_BITS = [ + (S_IFDIR, 'd'), (S_IFLNK, 'l'), (0o020000, 'c'), (0o060000, 'b'), + (0o010000, 'p'), (0o140000, 's'), +] + + +def sort_key(entry): + """Approximate rsync's f_name_cmp ordering: bytewise on the name, with a + directory keyed as if it had a trailing '/' so it sorts immediately before + its contents (and a file foo.txt before a dir foo). Good for flat lists and + simple trees; not the full path-state machine.""" + return entry.name + (b'/' if entry.is_dir else b'') + + +def sort_entries(entries): + """rsync sorts the received file list before indexing it, so the transfer + ndx is the sorted position, not the wire position. Return entries in that + order.""" + return sorted(entries, key=sort_key) + + +def mode_to_perms(mode): + """An ls-style permission string, e.g. '-rw-r--r--' / 'drwxr-xr-x'.""" + typ = '-' + for bits, ch in _PERM_BITS: + if (mode & S_IFMT) == bits: + typ = ch + break + out = [typ] + for who in (6, 3, 0): + out.append('r' if mode & (4 << who) else '-') + out.append('w' if mode & (2 << who) else '-') + out.append('x' if mode & (1 << who) else '-') + return ''.join(out) + + +def xattr_list_wire(items): + """xattrs.c send_xattr() wire bytes for a NEW xattr list -- the form a + sender appends to a file-list entry under -X. The leading 0 (ndx+1 with + ndx=-1) means 'literal data follows'. `items` is a list of (name, + datum_len, datum): `name` includes any namespace prefix and a trailing NUL, + `datum_len` is the declared value length (need not match `datum`), and + `datum` is the literal trailing bytes. A peer exercising the receiver's + datum_len cap can pass an empty `datum` since receive_xattr() validates + datum_len before it reads the value.""" + out = bytearray(w_varint(0) + w_varint(len(items))) + for name, datum_len, datum in items: + if isinstance(name, str): + name = name.encode() + out += w_varint(len(name)) + w_varint(datum_len) + name + datum + return bytes(out) + + +# --------------------------------------------------------------------------- +# File-list entry -- declarative, every field overridable for fuzzing +# --------------------------------------------------------------------------- + +class FileEntry: + """One flat (non-inc-recurse) file-list entry. + + Set the fields you care about; encode() emits the exact wire bytes for + protocol 30 with byte/short flags. For malformed-input testing, set + extra_flags to OR bits into the computed xflags, or set raw= to emit a + fully hand-built byte string instead. + """ + + def __init__(self, name, *, mode=S_IFREG | 0o644, length=0, modtime=1700000000, + csum=None, extra_flags=0, protocol=DEFAULT_PROTOCOL, raw=None, + hlink_ndx=None, uid=None, user_name=None): + self.name = name.encode() if isinstance(name, str) else name + self.mode = mode + self.length = length + self.modtime = modtime + self.csum = csum # bytes; appended when the daemon has -c + self.extra_flags = extra_flags + self.protocol = protocol + self.raw = raw + # A non-first hard-link entry: sets XMIT_HLINKED (without HLINK_FIRST) + # and carries this gnum (first_hlink_ndx) as a varint after the name. + self.hlink_ndx = hlink_ndx + # Owner: when uid is set the entry drops XMIT_SAME_UID and carries the + # uid varint (the receiver reads it under preserve_uid). When user_name + # is also set it adds XMIT_USER_NAME_FOLLOWS + a byte-counted name, which + # the daemon feeds to its name converter (recv_user_name). + self.uid = uid + self.user_name = (user_name.encode() if isinstance(user_name, str) + else user_name) + + def encode(self): + if self.raw is not None: + return self.raw + + is_reg = (self.mode & 0o170000) == S_IFREG + is_dir = (self.mode & 0o170000) == S_IFDIR + + # We never preserve uid/gid and always emit a full (non-abbreviated) + # entry, so the only "same as previous" flags are UID/GID. + xflags = XMIT_SAME_UID | XMIT_SAME_GID + xflags |= self.extra_flags + if self.hlink_ndx is not None: + xflags |= XMIT_HLINKED + if self.uid is not None: + xflags &= ~XMIT_SAME_UID + if self.user_name is not None: + xflags |= XMIT_USER_NAME_FOLLOWS + + out = bytearray() + + # --- flags byte/short (proto >= 28, non-varint path) --- + if not xflags and not is_dir: + xflags |= XMIT_TOP_DIR + if (xflags & 0xFF00) or not xflags: + xflags |= XMIT_EXTENDED_FLAGS + out += w_shortint(xflags) + else: + out += w_byte(xflags) + + # --- name (no prefix compression: l1 == 0) --- + l2 = len(self.name) + if xflags & XMIT_LONG_NAME: + out += w_varint30(l2, self.protocol) + else: + out += w_byte(l2) + out += self.name + + # A non-first hard-link entry (XMIT_HLINKED set, HLINK_FIRST unset) + # carries its gnum here, per send_file_entry(). A HLINK_FIRST entry + # carries no gnum (BITS_SETnUNSET is false in recv_file_entry), which is + # why 0004 uses extra_flags rather than hlink_ndx. We only use small + # gnums (< this flist's ndx_start), so the full length/mode fields + # follow; a gnum >= ndx_start would abbreviate the entry (goto the_end). + if self.hlink_ndx is not None: + out += w_varint(self.hlink_ndx) + + out += w_varlong30(self.length, 3, self.protocol) + if not (xflags & XMIT_SAME_TIME): + out += w_varlong(self.modtime, 4) if self.protocol >= 30 else w_int(self.modtime) + if not (xflags & XMIT_SAME_MODE): + out += w_int(to_wire_mode(self.mode)) + + # Owner (after mode, per recv_file_entry): uid varint, then -- under + # XMIT_USER_NAME_FOLLOWS -- a byte-counted user name for the converter. + if self.uid is not None: + out += w_varint(self.uid) + if self.user_name is not None: + out += w_byte(len(self.user_name)) + self.user_name + + # checksum trailer (only when the receiver runs with -c / always_checksum) + if self.csum is not None: + out += self.csum + + return bytes(out) + + +def end_of_flist(io_error=0, protocol=DEFAULT_PROTOCOL): + """Trailing marker after the last entry. In byte-flags mode (no + CF_VARINT_FLIST_FLAGS, i.e. no 'v' advertised) recv_file_list() ends the + list on a single 0 flag byte and reads NO io_error after it -- the io_error + varint exists only in the varint-flags path. Sending an extra byte here + leaves a stray 0x00 that the receiver then reads as NDX_DONE (freeing the + flist), which broke inc_recurse sub-flist sequencing.""" + return w_byte(0) + + +# NOTE: a plain push to a daemon module (no --delete, no --prune-empty-dirs) +# exchanges NO filter list -- recv_filter_list() only reads from the wire when +# `am_sender || receiver_wants_list`, both false for the receiver here. So the +# file list starts immediately after the checksum seed; do not send a filter +# list or its first byte is consumed as a 0 (end-of-list) flag. + + +# --------------------------------------------------------------------------- +# Daemon client / sender +# --------------------------------------------------------------------------- + +class ProtocolError(Exception): + pass + + +class ParsedEntry: + """A file-list entry decoded from the wire (the read side of FileEntry).""" + + __slots__ = ('name', 'mode', 'length', 'mtime', 'link_target') + + def __init__(self, name, mode, length, mtime, link_target=None): + self.name = name # bytes, module-relative + self.mode = mode + self.length = length + self.mtime = mtime + self.link_target = link_target # bytes for a symlink, else None + + @property + def is_reg(self): + return S_ISREG(self.mode) + + @property + def is_dir(self): + return S_ISDIR(self.mode) + + @property + def is_link(self): + return S_ISLNK(self.mode) + + def __repr__(self): + return f"ParsedEntry({self.name!r}, mode={self.mode:o}, length={self.length})" + + +class DaemonClient: + """A client that connects to an rsync daemon: runs the @RSYNCD handshake + + protocol setup, then drives either role depending on the server args -- push + (send a file list + deltas, as a sender) or pull/list (receive + parse the + file list, request and receive files, as a receiver). The protocol steps + are small overridable methods so a test can swap one behaviour (e.g. the sum + header or an flist entry) while reusing the rest -- see xrsync.py.""" + + def __init__(self, host, port, timeout=10): + self.sock = socket.create_connection((host, port), timeout=timeout) + self.sock.settimeout(timeout) + self.protocol = DEFAULT_PROTOCOL + self.compat_flags = None + self.seed = None + self.xfer_sum_len = 16 # md5 at proto 30 (no string negotiation) + self._rbuf = b'' + self._ndx_prev_positive = -1 # write_ndx delta state (proto >= 30) + self._ndx_prev_negative = 1 + self._mux_in = b'' # de-multiplexed sender input (data channel) + self._r_ndx_prev_positive = -1 # read_ndx delta state + self._r_ndx_prev_negative = 1 + self.messages = [] # (tag, payload) of non-data frames seen + + # -- low-level socket I/O -------------------------------------------- + def _recv_exact(self, n): + data = b'' + while len(data) < n: + if self._rbuf: + take = self._rbuf[:n - len(data)] + self._rbuf = self._rbuf[len(take):] + data += take + continue + chunk = self.sock.recv(n - len(data)) + if not chunk: + raise ProtocolError(f"EOF after {len(data)}/{n} bytes") + data += chunk + return data + + def _readline(self): + line = b'' + while not line.endswith(b'\n'): + if self._rbuf: + c, self._rbuf = self._rbuf[:1], self._rbuf[1:] + else: + c = self.sock.recv(1) + if not c: + raise ProtocolError("EOF reading a line") + line += c + return line.decode('latin-1').rstrip('\n') + + def _r_int(self): + return _s32(int.from_bytes(self._recv_exact(4), 'little')) + + def _r_varint(self): + ch = self._recv_exact(1)[0] + extra = _INT_BYTE_EXTRA[ch >> 2] + b = bytearray(5) + if extra: + bit = 1 << (8 - extra) + b[0:extra] = self._recv_exact(extra) + b[extra] = ch & (bit - 1) + else: + b[0] = ch + return b[0] | (b[1] << 8) | (b[2] << 16) | (b[3] << 24) + + # -- de-multiplexed input (the generator's stream) ------------------- + def _read_data(self, n): + """Read n bytes from the de-multiplexed sender input, dispatching any + non-MSG_DATA frames (MSG_INFO/MSG_ERROR/keepalive/redo/...) into + self.messages and discarding them.""" + while len(self._mux_in) < n: + val = int.from_bytes(self._recv_exact(4), 'little') + tag = (val >> 24) - MPLEX_BASE + ln = val & 0xFFFFFF + payload = self._recv_exact(ln) if ln else b'' + if tag == MSG_DATA: + self._mux_in += payload + else: + self.messages.append((tag, payload)) + out, self._mux_in = self._mux_in[:n], self._mux_in[n:] + return out + + def r_int(self): + return _s32(int.from_bytes(self._read_data(4), 'little')) + + def r_byte(self): + return self._read_data(1)[0] + + def r_shortint(self): + return int.from_bytes(self._read_data(2), 'little') + + def r_buf(self, n): + return self._read_data(n) + + def r_vstring(self): + n = self._read_data(1)[0] + if n & 0x80: + n = (n & 0x7F) * 0x100 + self._read_data(1)[0] + return self._read_data(n) if n else b'' + + def r_ndx(self): + """Port of io.c read_ndx() at protocol >= 30.""" + b0 = self._read_data(1)[0] + if b0 == 0xFF: + b0 = self._read_data(1)[0] + neg = True + elif b0 == 0: + return NDX_DONE + else: + neg = False + prev = self._r_ndx_prev_negative if neg else self._r_ndx_prev_positive + if b0 == 0xFE: + b = self._read_data(2) + if b[0] & 0x80: + rest = self._read_data(2) + num = ((b[0] & 0x7F) << 24) | b[1] | (rest[0] << 8) | (rest[1] << 16) + else: + num = (b[0] << 8) + b[1] + prev + else: + num = b0 + prev + if neg: + self._r_ndx_prev_negative = num + return -num + self._r_ndx_prev_positive = num + return num + + def r_sum_head(self): + """io.c read_sum_head(): count, blength, s2length, remainder.""" + return (self.r_int(), self.r_int(), self.r_int(), self.r_int()) + + def r_varint(self): + """io.c read_varint() on the de-multiplexed stream.""" + ch = self._read_data(1)[0] + extra = _INT_BYTE_EXTRA[ch >> 2] + b = bytearray(5) + if extra: + bit = 1 << (8 - extra) + b[0:extra] = self._read_data(extra) + b[extra] = ch & (bit - 1) + else: + b[0] = ch + return b[0] | (b[1] << 8) | (b[2] << 16) | (b[3] << 24) + + def r_varlong(self, min_bytes): + """io.c read_varlong() on the de-multiplexed stream.""" + b2 = self._read_data(min_bytes) + u = bytearray(9) + u[0:min_bytes - 1] = b2[1:min_bytes] + extra = _INT_BYTE_EXTRA[b2[0] >> 2] + if extra: + bit = 1 << (8 - extra) + u[min_bytes - 1:min_bytes - 1 + extra] = self._read_data(extra) + u[min_bytes + extra - 1] = b2[0] & (bit - 1) + else: + u[min_bytes - 1] = b2[0] + x = 0 + for i in range(8): + x |= u[i] << (8 * i) + return x + + def r_varint30(self): + return self.r_varint() if self.protocol >= 30 else self.r_int() + + def r_varlong30(self, min_bytes): + return self.r_varlong(min_bytes) if self.protocol >= 30 else self.r_int() + + # -- handshake + setup ------------------------------------------------ + def handshake(self, module, server_args, greeting_version=30): + """Run the @RSYNCD handshake for `module` and send `server_args` + (the daemon-side argv). Returns once protocol setup is done and the + stream is multiplexed in both directions.""" + greeting = self._readline() + if not greeting.startswith('@RSYNCD:'): + raise ProtocolError(f"bad greeting: {greeting!r}") + # Our greeting: claim greeting_version so the daemon negotiates down to + # it; <=31 means we needn't send a digest list. + self._send_raw(f"@RSYNCD: {greeting_version}.0\n".encode()) + self._send_raw(module.encode() + b"\n") + resp = self._readline() + if 'OK' not in resp: + raise ProtocolError(f"daemon did not send OK (got {resp!r}); " + "module may require auth or be unknown") + # Server-side argv, NUL-terminated, with a trailing empty arg. + payload = b''.join(a.encode() + b"\0" for a in server_args) + b"\0" + self._send_raw(payload) + # setup_protocol (proto >= 30, daemon side): it skips the binary version + # exchange (remote_protocol is already set from the greeting), writes + # compat_flags (varint) and -- since we didn't advertise 'v' -- skips + # string negotiation, then writes the checksum seed. All still raw. + self.compat_flags = self._r_varint() + self.seed = self._r_int() + # Multiplexing is now active in both directions. + + # -- multiplexed output ---------------------------------------------- + def _send_raw(self, data): + self.sock.sendall(data) + + def _frame(self, code, payload): + hdr = struct.pack('= 30: NDX_DONE -> a single + 0 byte; negatives -> a leading 0xFF then a delta against prev_negative; + non-negatives -> a delta against prev_positive.""" + if ndx == NDX_DONE: + return b'\x00' + b = bytearray() + if ndx >= 0: + diff = ndx - self._ndx_prev_positive + self._ndx_prev_positive = ndx + absndx = ndx + else: + b.append(0xFF) + absndx = -ndx + diff = absndx - self._ndx_prev_negative + self._ndx_prev_negative = absndx + if 0 < diff < 0xFE: + b.append(diff) + elif diff < 0 or diff > 0x7FFF: + b += bytes([0xFE, ((absndx >> 24) | 0x80) & 0xFF, absndx & 0xFF, + (absndx >> 8) & 0xFF, (absndx >> 16) & 0xFF]) + else: + b += bytes([0xFE, (diff >> 8) & 0xFF, diff & 0xFF]) + return bytes(b) + + def send_transfer_ndx(self, ndx, iflags=0): + """Transfer-phase token: a write_ndx() index followed by the shortint + iflags read by read_ndx_and_attrs() (protocol >= 29).""" + self.send_data(self.w_ndx(ndx) + w_shortint(iflags)) + + def send_ndx_done(self): + """Send NDX_DONE (a single 0 byte in the ndx stream).""" + self.send_data(self.w_ndx(NDX_DONE)) + + def send_subflist_marker(self, dir_ndx): + """Announce an inc_recurse sub-flist for dir_ndx: write_ndx of + NDX_FLIST_OFFSET - dir_ndx (a negative index).""" + self.send_data(self.w_ndx(NDX_FLIST_OFFSET - dir_ndx)) + + def run_forged_transfer(self, forged_type, xname, literal_tail=b'', + file_csum_len=16, max_phase=2): + """Drive the sender side of the transfer phase, FORGING fnamecmp_type on + every file the generator requests (the chroot-basis attack): for each + request, read its iflags / basis-type / xname / sum header + block sums, + then send back the ndx with our forged basis type + xname, the echoed sum + header, a delta that MATCHES block 0 of the (forged) basis plus the given + literal tail, and a deliberately wrong whole-file checksum (so the + receiver -- if it opened the forged basis -- reconstructs the wrong bytes + and logs 'failed verification'). Loops through both transfer phases (the + checksum-mismatch redo) until NDX_DONE has advanced past max_phase. + + Stops early (returns) if the receiver drops the connection mid-transfer + -- e.g. a confined receiver that refuses the forged basis and exits with + 'got a block match with no basis file'.""" + try: + self._forged_transfer_loop(forged_type, xname, literal_tail, + file_csum_len, max_phase) + except (ProtocolError, socket.timeout, OSError): + pass + + def _forged_transfer_loop(self, forged_type, xname, literal_tail, + file_csum_len, max_phase): + phase = 0 + while True: + ndx = self.r_ndx() + if ndx == NDX_DONE: + self.send_data(self.w_ndx(NDX_DONE)) + phase += 1 + if phase > max_phase: + break + continue + iflags = self.r_shortint() + if iflags & ITEM_BASIS_TYPE_FOLLOWS: + self.r_byte() + if iflags & ITEM_XNAME_FOLLOWS: + self.r_vstring() + count, blength, s2length, remainder = self.r_sum_head() + for _ in range(count): + self.r_int() # block weak checksum + self.r_buf(s2length) # block strong checksum + out = bytearray() + out += self.w_ndx(ndx) + out += w_shortint(iflags | ITEM_BASIS_TYPE_FOLLOWS | ITEM_XNAME_FOLLOWS) + out += w_byte(forged_type) + out += w_vstring(xname) + out += w_sum_head(count, blength, s2length, remainder) + if count > 0: + out += w_int(-1) # match block 0 of the (forged) basis + if literal_tail: + out += w_int(len(literal_tail)) + literal_tail + out += w_int(0) # end-of-file token + out += b'\x00' * file_csum_len # wrong whole-file checksum + self.send_data(bytes(out)) + + # -- receive side (pull / list) -------------------------------------- + def recv_flist(self, preserve_links=True): + """Send the (empty) filter list the sender expects, then read + parse + the daemon's file list. Returns a list of ParsedEntry. Decodes the + -lt subset (no uid/gid/devices); override for more. This is a natural + hook point: a test can subclass and tamper with what it returns.""" + self.send_data(w_int(0)) # empty filter list (terminator) + entries = [] + prev_name = b'' + prev_mode = 0 + prev_mtime = 0 + while True: + flags = self.r_byte() + if flags & XMIT_EXTENDED_FLAGS: + flags |= self.r_byte() << 8 + if flags == 0: + break # end of list (no io_error at p30) + l1 = self.r_byte() if flags & XMIT_SAME_NAME else 0 + l2 = self.r_varint30() if flags & XMIT_LONG_NAME else self.r_byte() + name = prev_name[:l1] + self.r_buf(l2) + prev_name = name + length = self.r_varlong30(3) + if not (flags & XMIT_SAME_TIME): + prev_mtime = self.r_varlong30(4) + mtime = prev_mtime + if flags & XMIT_MOD_NSEC: + self.r_varint() # nsec, ignored + if not (flags & XMIT_SAME_MODE): + prev_mode = self.r_int() + mode = prev_mode + link_target = None + if preserve_links and S_ISLNK(mode): + link_target = self.r_buf(self.r_varint30()) + entries.append(ParsedEntry(name, mode, length, mtime, link_target)) + return entries + + def make_request(self, ndx): + """Generator request for a whole-file transfer of `ndx`: the ndx, the + item flags, and a count=0 sum header (no local basis). Overridable + hook -- a test can return a malformed request here.""" + return (self.w_ndx(ndx) + w_shortint(ITEM_TRANSFER) + + w_sum_head(0, 0, 0, 0)) + + def recv_file_transfer(self, ndx): + """Read one file the sender sends in reply to make_request(): item + flags, optional basis type / xname, the echoed sum header, the literal + token stream, and the whole-file checksum. Returns the file bytes.""" + iflags = self.r_shortint() + if iflags & ITEM_BASIS_TYPE_FOLLOWS: + self.r_byte() + if iflags & ITEM_XNAME_FOLLOWS: + self.r_vstring() + self.r_sum_head() # count=0 for a whole-file pull + data = bytearray() + while True: + tok = self.r_int() + if tok == 0: + break # end of file + if tok > 0: + data += self.r_buf(tok) # literal chunk + else: + raise ProtocolError(f"unexpected block match token {tok} " + "(no basis was offered)") + self.r_buf(self.xfer_sum_len) # whole-file checksum (unverified) + return bytes(data) + + def pull(self, dest_dir, verbose=False, preserve_times=True, + preserve_perms=True): + """Receive the file list and materialise it under dest_dir: make + directories, create symlinks, and download regular files (whole-file + requests). Returns the parsed file list.""" + import os + # rsync indexes the file list by SORTED order, so the transfer ndx is + # the sorted position (both peers sort identically). + entries = sort_entries(self.recv_flist()) + reg = [] # (ndx, entry, path) to fetch + for ndx, e in enumerate(entries): + rel = e.name.decode('utf-8', 'surrogateescape') + if rel in ('.', ''): + continue + path = os.path.join(dest_dir, rel) + if e.is_dir: + os.makedirs(path, exist_ok=True) + elif e.is_link and e.link_target is not None: + tgt = e.link_target.decode('utf-8', 'surrogateescape') + if os.path.lexists(path): + os.unlink(path) + os.symlink(tgt, path) + elif e.is_reg: + reg.append((ndx, e, path)) + if verbose: + print(rel) + # Phase 1: send all whole-file requests, then NDX_DONE. + for ndx, e, path in reg: + self.send_data(self.make_request(ndx)) + self.send_data(self.w_ndx(NDX_DONE)) + # Read replies, mirroring the sender's per-phase NDX_DONE handshake. + got = {} + phase = 1 + while True: + ndx = self.r_ndx() + if ndx == NDX_DONE: + phase += 1 + if phase > 2: + break + self.send_data(self.w_ndx(NDX_DONE)) # phase 2: no redos + continue + got[ndx] = self.recv_file_transfer(ndx) + for ndx, e, path in reg: + if ndx not in got: + continue + with open(path, 'wb') as fh: + fh.write(got[ndx]) + if preserve_perms: + os.chmod(path, e.mode & 0o7777) + if preserve_times: + os.utime(path, (e.mtime, e.mtime)) + return entries + + def finish_no_transfer(self): + """Walk the per-phase NDX_DONE handshake with no files requested -- used + after --list-only so the daemon shuts down cleanly.""" + self.send_data(self.w_ndx(NDX_DONE)) + phase = 1 + try: + while True: + if self.r_ndx() == NDX_DONE: + phase += 1 + if phase > 2: + break + self.send_data(self.w_ndx(NDX_DONE)) + except (ProtocolError, socket.timeout, OSError): + pass + + # -- send side (basic push) ------------------------------------------ + def push(self, files, modtime=1700000000): + """Basic upload: `files` is a list of (name, content) regular files. + Send the file list, then satisfy the receiver's whole-file requests with + the literal content + its md5 (the proto-30 whole-file digest is plain + md5, no seed). Overridable pieces: make_file_token_stream().""" + import hashlib + names = [n.encode() if isinstance(n, str) else n for n, _ in files] + entries = [FileEntry(n, mode=S_IFREG | 0o644, length=len(c), + modtime=modtime, protocol=self.protocol) + for n, (_, c) in zip(names, files)] + self.send_flat_flist(entries) + # The receiver requests by sorted ndx; map it back to our content. + order = sorted(range(len(files)), key=lambda i: names[i]) + content_by_ndx = {ndx: files[i][1] for ndx, i in enumerate(order)} + phase = 0 + while True: + ndx = self.r_ndx() + if ndx == NDX_DONE: + self.send_data(self.w_ndx(NDX_DONE)) + phase += 1 + if phase > 2: + break + continue + iflags = self.r_shortint() + if iflags & ITEM_BASIS_TYPE_FOLLOWS: + self.r_byte() + if iflags & ITEM_XNAME_FOLLOWS: + self.r_vstring() + count, blength, s2length, remainder = self.r_sum_head() + for _ in range(count): + self.r_int() + self.r_buf(s2length) + content = content_by_ndx.get(ndx, b'') + out = bytearray() + out += self.w_ndx(ndx) + out += w_shortint(ITEM_TRANSFER) + out += w_sum_head(0, 0, 0, 0) + out += self.make_file_token_stream(content) + out += hashlib.md5(content).digest() + self.send_data(bytes(out)) + + def make_file_token_stream(self, content): + """Whole-file literal token stream (token.c simple_send_token, no -z): + CHUNK_SIZE-sized literal runs then a 0 end token. Overridable hook.""" + out = bytearray() + for off in range(0, len(content), CHUNK_SIZE): + chunk = content[off:off + CHUNK_SIZE] + out += w_int(len(chunk)) + chunk + out += w_int(0) + return bytes(out) + + def drain(self, timeout=3.0): + """Read whatever the daemon sends back until EOF/timeout. Returns the + raw bytes (mux frames undecoded); used only to detect a dropped + connection (crash) vs an orderly close.""" + self.sock.settimeout(timeout) + out = b'' + try: + while True: + chunk = self.sock.recv(65536) + if not chunk: + break + out += chunk + except socket.timeout: + pass + return out + + def close(self): + try: + self.sock.close() + except OSError: + pass + + +# Back-compat alias: the existing security tests speak of a "DaemonSender". +DaemonSender = DaemonClient + + +class DaemonReceiver: + """The SERVER side of the daemon protocol -- the inverse of DaemonSender. + Accepts a real rsync client and runs the @RSYNCD handshake + protocol-30 + setup as the daemon. The @RSYNCD handshake is direction-agnostic, so after + setup this can drive either role: when the client PUSHES (client = sender) + we act as the receiver/generator and send transfer requests (e.g. malformed + sum headers); when the client PULLS (client = receiver) we act as the sender + and send a file list (e.g. one carrying an oversized xattr datum). Both are + things rsync_proto's client/sender role cannot do. + + The two stream directions are independent, so we never need to parse the + client's file list: we just send a request for an index the client's flist + is known to contain (index 0 for a single pushed file) and let the client's + send_files() read it. After sending we drain the client's stream so its + flist writes don't block while it reaches the request and errors out.""" + + def __init__(self, sock, greeting_version=30, protocol=DEFAULT_PROTOCOL): + self.sock = sock + self.protocol = protocol + self.greeting_version = greeting_version + self._rbuf = b'' + self._ndx_prev_positive = -1 + self._ndx_prev_negative = 1 + + # -- low-level I/O (mirrors DaemonSender) ----------------------------- + def _send_raw(self, data): + self.sock.sendall(data) + + def _recv_exact(self, n): + data = b'' + while len(data) < n: + if self._rbuf: + take = self._rbuf[:n - len(data)] + self._rbuf = self._rbuf[len(take):] + data += take + continue + chunk = self.sock.recv(n - len(data)) + if not chunk: + raise ProtocolError(f"EOF after {len(data)}/{n} bytes") + data += chunk + return data + + def _readline(self): + line = b'' + while not line.endswith(b'\n'): + if self._rbuf: + c, self._rbuf = self._rbuf[:1], self._rbuf[1:] + else: + c = self.sock.recv(1) + if not c: + raise ProtocolError("EOF reading a line") + line += c + return line.decode('latin-1').rstrip('\n') + + def _frame(self, code, payload): + hdr = struct.pack('= 30). + if ndx == NDX_DONE: + return b'\x00' + b = bytearray() + if ndx >= 0: + diff = ndx - self._ndx_prev_positive + self._ndx_prev_positive = ndx + absndx = ndx + else: + b.append(0xFF) + absndx = -ndx + diff = absndx - self._ndx_prev_negative + self._ndx_prev_negative = absndx + if 0 < diff < 0xFE: + b.append(diff) + elif diff < 0 or diff > 0x7FFF: + b += bytes([0xFE, ((absndx >> 24) | 0x80) & 0xFF, absndx & 0xFF, + (absndx >> 8) & 0xFF, (absndx >> 16) & 0xFF]) + else: + b += bytes([0xFE, (diff >> 8) & 0xFF, diff & 0xFF]) + return bytes(b) + + # -- handshake (server side) ----------------------------------------- + def handshake(self, compat_flags=0, seed=0): + """Send the daemon greeting, read the client's greeting + module line, + send '@RSYNCD: OK', then write the protocol-30 setup (compat flags + + checksum seed). We don't read the client's NUL-separated args: it sends + them after reading our OK and reads our compat/seed afterwards, and the + directions are independent.""" + self._send_raw(f"@RSYNCD: {self.greeting_version}.0\n".encode()) + self._readline() # client greeting (version[, digests]) + self._readline() # module name + self._send_raw(b"@RSYNCD: OK\n") + self._send_raw(w_varint(compat_flags) + w_int(seed)) + # Multiplexing is now active in both directions. + + # -- generator requests ---------------------------------------------- + def send_sum_request(self, ndx, count, blength, s2length, remainder, + iflags=ITEM_TRANSFER): + """As the generator, request a transfer of file `ndx` with the given + (possibly malformed) sum header -- ndx + iflags + write_sum_head(), the + bytes the client's send_files()/receive_sums() read.""" + buf = (self.w_ndx(ndx) + w_shortint(iflags) + + w_sum_head(count, blength, s2length, remainder)) + self.send_data(buf) + + def drain(self, timeout=5.0): + """Read and discard the client's stream until it closes (it pushes its + file list, reaches our request, errors, and disconnects).""" + self.sock.settimeout(timeout) + try: + while self.sock.recv(65536): + pass + except (OSError, ProtocolError): + pass + + def close(self): + try: + self.sock.close() + except OSError: + pass diff --git a/testsuite/rsyncfns.py b/testsuite/rsyncfns.py index 2e00abb3..5dcd571a 100644 --- a/testsuite/rsyncfns.py +++ b/testsuite/rsyncfns.py @@ -24,8 +24,10 @@ import os import platform import shlex import shutil +import signal import socket as _socket import stat +import struct import subprocess import sys import time @@ -75,6 +77,19 @@ RSYNC = _required('RSYNC') # full command line, possibly with valgrind/p # by hand without the runner still works. RSYNC_PEER = os.environ.get('RSYNC_PEER', RSYNC) + +def _under_valgrind(): + """True when the runner wrapped rsync in valgrind (runtests.py --valgrind). + + Match the wrapper's program name (first token of RSYNC or RSYNC_PEER), not a + bare 'valgrind' substring, so an rsync path that merely contains the word + does not false-trigger. + """ + for cmd in (RSYNC, RSYNC_PEER): + if os.path.basename(shlex.split(cmd)[0]) == 'valgrind': + return True + return False + # TLS_ARGS controls how the 'tls' helper formats listings (e.g. --atimes, # -l, -L). Tests that exercise non-default rsync features (atimes, etc.) # assign to rsyncfns.TLS_ARGS before calling checkit / rsync_ls_lR. @@ -86,6 +101,10 @@ TLS_ARGS = os.environ.get('TLS_ARGS', '') # daemon tests to a real rsyncd bound to loopback (see start_test_daemon). USE_TCP = os.environ.get('RSYNC_TEST_USE_TCP') == '1' +# Budget (seconds) a TOCTOU symlink-race test may spend trying to win its race +# before giving up. Set by runtests.py --race-timeout (default 5). +RACE_TIMEOUT = float(os.environ.get('race_timeout', '5')) + # Mnemonics for rsync's itemize-changes (-i / -ii) format: # all_plus -> +++++++++ every attribute changed (an additive create) # allspace -> every attribute unchanged @@ -127,6 +146,25 @@ def test_xfail(msg: str) -> 'None': _PORT_LOCK_PATH = '/tmp/rsync_test.lck' _port_lock_fd = None +_reaped_stale = False + +# The lock file doubles as a registry of the rsyncd pid bound to each port, so a +# later run that wins the (orphan-released) lock can find and reap a daemon a +# SIGKILLed run stranded. The byte-range LOCKS sit at offsets 0..65535 (one byte +# per port number); the pid RECORDS sit in a separate region past them, one +# native-endian int32 (a pid_t) per port, written/read only while holding that +# port's lock so they're never raced. The file is host-local, so native endian is +# fine; an all-zero record (a sparse/older lock file) reads back as pid 0. +_PORT_PID_BASE = 1 << 16 # past every possible port lock byte (port < 65536) +_PORT_PID_REC = 8 # two native-endian int32 per port: (pgid, pid) + +# Bytes 0..3 hold a magic identifying the lock-file layout. A fresh (all-zero) +# file gets it written under the byte-0 lock; a non-zero value that doesn't match +# means a stale file from an incompatible testsuite layout -- we error rather +# than misread the (pgid, pid) records. Bytes 0..3 also sit in the port-lock byte +# region, but ports 0..3 are never test ports so the overlap is harmless. Fixed +# arbitrary value; bump it on any on-disk layout change. +_LOCK_MAGIC = 0x9d4f2b8a def _open_lock_file() -> int: @@ -159,6 +197,7 @@ def _open_lock_file() -> int: os.fchmod(fd, 0o666) # we own this fresh file; undo umask except OSError: pass + _check_or_write_magic(fd) return fd # Path 2: it already exists -- open without creating or chmod'ing. @@ -172,10 +211,184 @@ def _open_lock_file() -> int: os.close(fd) test_fail(f"lock file {_PORT_LOCK_PATH} is not a pristine regular " f"file (type/nlink check failed -- possible tampering)") + _check_or_write_magic(fd) return fd -def _probe_bindable(port: int) -> 'None': +def _check_or_write_magic(fd: int) -> 'None': + """Validate (or stamp) the layout-version magic in bytes 0..3. + + Serialise on the byte-0 lock (also port 0's lock byte, never a test port) so + two starting runs don't race the stamp. An all-zero header is a fresh file -- + write the magic. A non-zero header that doesn't match means a stale lock file + from an incompatible testsuite layout (e.g. the old 4-byte pid records); error + out so we never misread its records as (pgid, pid).""" + fcntl.lockf(fd, fcntl.LOCK_EX, 4, 0) + try: + rec = os.pread(fd, 4, 0) + cur = struct.unpack('=I', rec)[0] if len(rec) == 4 else 0 + if cur == 0: + os.pwrite(fd, struct.pack('=I', _LOCK_MAGIC), 0) + elif cur != _LOCK_MAGIC: + os.close(fd) + test_fail(f"lock file {_PORT_LOCK_PATH} has layout magic " + f"{cur:#010x}, expected {_LOCK_MAGIC:#010x} -- a stale file " + "from an incompatible testsuite. Remove it and retry.") + except (OSError, struct.error): + pass + finally: + try: + fcntl.lockf(fd, fcntl.LOCK_UN, 4, 0) + except OSError: + pass + + +def _record_port_proc(port: int, pgid: int, pid: int) -> 'None': + """Record (or clear, with pgid==pid==0) the test process group and rsyncd pid + bound to `port`. The caller holds the port's lock. The pgid reaps the whole + test (daemon + clients + flipper) with one killpg; the pid is the recycle + guard (_pid_is_rsync) so we only kill a group still running our rsync.""" + if _port_lock_fd is None: + return + try: + os.pwrite(_port_lock_fd, struct.pack('=ii', pgid, pid), + _PORT_PID_BASE + port * _PORT_PID_REC) + except (OSError, struct.error): + pass + + +def _read_port_proc(port: int) -> 'tuple': + """Read the recorded (pgid, pid) for `port`, or (0, 0) if none. Caller holds + the lock. + + Normalises a pid <= 1 to (0, 0): a record holding 0 / negative / garbage must + NEVER be treated as a real pid (os.kill/os.killpg of 0 or -N would signal a + whole process group). Only pid > 1 is a candidate, and _pid_is_rsync() still + verifies it before any kill.""" + if _port_lock_fd is None: + return (0, 0) + try: + rec = os.pread(_port_lock_fd, _PORT_PID_REC, + _PORT_PID_BASE + port * _PORT_PID_REC) + if len(rec) != _PORT_PID_REC: + return (0, 0) + pgid, pid = struct.unpack('=ii', rec) + except (OSError, struct.error): + return (0, 0) + return (pgid, pid) if pid > 1 else (0, 0) + + +def _pid_is_rsync(pid: int) -> bool: + """True if `pid` is a live process whose command is rsync. Guards against a + recycled pid before we kill it. Tries `ps -p N -o comm=` (precise, Linux/BSD/ + Solaris/macOS) and falls back to plain `ps -p N` (Cygwin's ps rejects -o but + still prints the command). If neither confirms it, return False (leave the + process alone).""" + if pid <= 1: + return False # 0/-N would make os.kill signal a whole process group + if pid == os.getpid(): + return False # never signal ourselves + try: + os.kill(pid, 0) + except OSError: + return False + for argv in (['ps', '-p', str(pid), '-o', 'comm='], ['ps', '-p', str(pid)]): + try: + r = subprocess.run(argv, stdout=subprocess.PIPE, + stderr=subprocess.DEVNULL, text=True, timeout=5) + except (OSError, subprocess.SubprocessError): + return False + if r.returncode == 0: # ps understood the form -> answer is definitive + return 'rsync' in r.stdout + return False # no ps form worked -> don't kill + + +def _reap_group(pgid: int, pid: int) -> bool: + """Kill the test's whole process group (daemon + its clients + flipper) when + `pid` is still a live rsync -- the portable recycle guard, so we only ever + signal a group still running our rsync. killpg(pgid) sweeps the group in one + shot (the test driver runs in its own session, so the group is exactly that + test's tree); if the pgid is unusable, fall back to killing the daemon pid + alone. Returns True if it signalled something.""" + if not _pid_is_rsync(pid): + return False + try: + if pgid > 1 and pgid != os.getpgrp(): + os.killpg(pgid, signal.SIGKILL) + else: + os.kill(pid, signal.SIGKILL) + except OSError: + try: + os.kill(pid, signal.SIGKILL) + except OSError: + return False + return True + + +def _reap_orphan_daemon(port: int) -> bool: + """Kill an orphaned test process group squatting `port`, if we can identify it. + + We hold the claim_ports() exclusive lock for `port`, so nothing we coordinate + with owns it -- a still-bound port is an orphan a SIGKILLed run stranded (off + Linux there's no PR_SET_PDEATHSIG backstop, so its --no-detach rsyncd outlives + the test). start_rsyncd recorded that test's (pgid, pid); if the pid is still a + live rsync, killpg the group. Returns True if it signalled something (caller + re-probes the bind). Pure os/ps calls -> every platform.""" + pgid, pid = _read_port_proc(port) + if not _reap_group(pgid, pid): + return False + _record_port_proc(port, 0, 0) + time.sleep(0.2) # let the kernel release the socket before the re-probe + return True + + +def _reap_stale_daemons() -> 'None': + """Intra-run sweep: kill every orphaned test rsyncd recorded in the lock file + whose port-lock is free (no live test owns it), and clear its record. + + _reap_orphan_daemon() only fires when a NEW test claims the *same* port an + orphan still squats; a daemon a SIGKILLed/timed-out test stranded on a port + nothing else re-claims would otherwise linger for the whole run (off Linux + there's no PR_SET_PDEATHSIG backstop), accumulating and exhausting ports until + a later race test wedges. This sweeps the whole pid registry so each test + process reaps the leaks left by earlier ones. + + Run once per test process at the first claim_ports(), BEFORE this process has + recorded any daemon of its own, so it never kills our own rsyncd. A port a + live concurrent test holds keeps its byte-lock, so LOCK_NB skips it; only a + free-locked port with a recorded live rsync pid is a genuine orphan.""" + if _port_lock_fd is None: + return + try: + size = os.fstat(_port_lock_fd).st_size + except OSError: + return + if size <= _PORT_PID_BASE: + return + try: + region = os.pread(_port_lock_fd, size - _PORT_PID_BASE, _PORT_PID_BASE) + except OSError: + return + for port in range(min(len(region) // _PORT_PID_REC, 65536)): + pgid, pid = struct.unpack('=ii', region[port*_PORT_PID_REC:(port+1)*_PORT_PID_REC]) + if pid <= 1: + continue + # Grab the port's byte-lock non-blocking: success => no live test owns it. + try: + fcntl.lockf(_port_lock_fd, fcntl.LOCK_EX | fcntl.LOCK_NB, 1, port) + except OSError: + continue # a live test holds it -- not an orphan, leave it alone + try: + if _reap_group(pgid, pid): + _record_port_proc(port, 0, 0) + finally: + try: + fcntl.lockf(_port_lock_fd, fcntl.LOCK_UN, 1, port) + except OSError: + pass + + +def _probe_bindable(port: int, _reaped: bool = False) -> 'None': """Confirm `port` is actually free once we hold its claim_ports() lock. The byte-range lock only coordinates *live* test drivers, and the kernel @@ -184,31 +397,36 @@ def _probe_bindable(port: int) -> 'None': SIGKILLed (or its ssh drops) on a platform with no parent-death backstop: rsyncfns only arms PR_SET_PDEATHSIG, which is Linux-only, so on the BSDs/Solaris/macOS a killed fleettest run can strand its rsyncd, which then - squats the fixed test port forever. A later run wins the (now-free) lock but - the socket is still taken, and the daemon dies with a cryptic "bind() failed: - Address already in use" / the client "did not see server greeting". + squats the fixed test port. Because we recorded that rsyncd's pid in the lock + file (and hold the lock now, proving it's not a live run), we can reap it and + retry rather than failing -- see _reap_orphan_daemon. So actually try to bind it. SO_REUSEADDR is used so a port merely in TIME_WAIT (recently and cleanly closed) is NOT a false positive; only a - live bound/listening socket -- a real squatter -- makes the bind fail, and - then we stop here with an actionable message instead of failing obscurely - later. The probe socket is closed immediately, freeing the port for the - daemon that is about to bind it. + live bound/listening socket -- a real squatter -- makes the bind fail. The + probe socket is closed immediately, freeing the port for the daemon that is + about to bind it. """ s = _socket.socket(_socket.AF_INET, _socket.SOCK_STREAM) s.setsockopt(_socket.SOL_SOCKET, _socket.SO_REUSEADDR, 1) try: s.bind(('127.0.0.1', port)) + return except OSError as e: - test_fail( - f"port {port} was claimed for this run but something is still bound " - f"to 127.0.0.1:{port} ({e.strerror}). The claim_ports() lock only " - "serializes live test runs, so a still-bound port almost always " - "means an orphaned 'rsync --daemon' from a previously killed run " - f"(find it with `fstat | grep {port}` / `netstat -an | grep {port}` " - "and kill it, or run `fleettest.py --cleanup`), then retry.") + err = e finally: s.close() + # Bound by a squatter. If it's our own stranded orphan, kill it and retry once. + if not _reaped and _reap_orphan_daemon(port): + _probe_bindable(port, _reaped=True) + return + test_fail( + f"port {port} was claimed for this run but something is still bound " + f"to 127.0.0.1:{port} ({err.strerror}). The claim_ports() lock only " + "serializes live test runs, so a still-bound port almost always " + "means an orphaned 'rsync --daemon' from a previously killed run " + f"(find it with `fstat | grep {port}` / `netstat -an | grep {port}` " + "and kill it, or run `fleettest.py --cleanup`), then retry.") def claim_ports(*ports: int) -> 'None': @@ -239,9 +457,14 @@ def claim_ports(*ports: int) -> 'None': port. For the rsync testsuite that's fine; we just need to avoid collisions between concurrent test scripts. """ - global _port_lock_fd + global _port_lock_fd, _reaped_stale if _port_lock_fd is None: _port_lock_fd = _open_lock_file() + if not _reaped_stale: + # Intra-run cleanup: reap any daemon an earlier test in this run stranded, + # BEFORE we record one of our own. Once per process is enough. + _reaped_stale = True + _reap_stale_daemons() for port in sorted(ports): # F_SETLKW via fcntl.lockf(LOCK_EX, length, start): exclusive # byte-range lock on byte `port`, blocking until acquired. @@ -282,6 +505,14 @@ def _stop_rsyncd(proc) -> 'None': pass +def _cleanup_rsyncd(proc, port: int) -> 'None': + """atexit handler: stop the daemon and clear its pid slot. A clean exit thus + leaves no orphan to reap; only a SIGKILL (which skips atexit) leaves the slot + set -- exactly the case _reap_orphan_daemon() needs it for.""" + _stop_rsyncd(proc) + _record_port_proc(port, 0, 0) + + def start_rsyncd(conf_path, port: int, rsync_cmd: str = None) -> 'subprocess.Popen': """Spawn `rsync --daemon --no-detach --address=127.0.0.1 --port=N --config=conf` and return the Popen handle after the port is accepting @@ -317,7 +548,12 @@ def start_rsyncd(conf_path, port: int, rsync_cmd: str = None) -> 'subprocess.Pop stderr=subprocess.DEVNULL, preexec_fn=_set_pdeathsig, ) - atexit.register(_stop_rsyncd, proc) + # Record this test's process group (os.getpgrp() -- the daemon and its clients + # and flipper all live in the per-test session runtests.py started) together + # with this --no-detach rsyncd's pid, while we still hold the port's lock, so a + # later test/run can killpg the whole stranded tree (see _reap_orphan_daemon). + _record_port_proc(port, os.getpgrp(), proc.pid) + atexit.register(_cleanup_rsyncd, proc, port) deadline = time.monotonic() + 10 last_err = None @@ -379,6 +615,25 @@ def require_tcp(reason: str) -> 'None': test_skipped(reason) +def require_asan(reason: str, which: str = None) -> 'None': + """Skip the test (exit 77) unless the rsync binary is AddressSanitizer- + instrumented. `which` defaults to the daemon/peer command (RSYNC_PEER); + pass RSYNC to check the client side. Detection runs the binary with + ASAN_OPTIONS=help=1, which makes an instrumented binary print the ASan + flag help banner to stderr.""" + cmd = shlex.split(which or RSYNC_PEER) + try: + r = subprocess.run(cmd + ['--version'], + env={**os.environ, 'ASAN_OPTIONS': 'help=1'}, + stdout=subprocess.DEVNULL, stderr=subprocess.PIPE, + timeout=15) + except Exception: + test_skipped(reason) + return + if b'AddressSanitizer' not in r.stderr: + test_skipped(reason) + + def rsync_argv(*args: str) -> list: """Return the argv for invoking rsync with the given extra arguments. @@ -390,6 +645,40 @@ def rsync_argv(*args: str) -> list: return shlex.split(RSYNC) + list(args) +import functools as _functools + + +@_functools.lru_cache(maxsize=64) +def rsync_supports(flag: str) -> bool: + """Does the configured rsync binary accept ``flag``? + + Probes by invoking ``rsync --version`` and checking the exit code + + stderr. C rsync accepts every flag we'd care about and exits 0 before + --version prints; other implementations (gokrazy/rsync, openrsync) reject + unsupported flags with "unknown option" / "unrecognized option" / + "no such option" and a non-zero exit. + + Used by tests that want to *optionally* pass a hardening flag like + `--no-inc-recursive` (only meaningful where the implementation has + incremental recursion to disable). When the probe is inconclusive (e.g. + timeout) the helper returns True so tests fall back to today's C-rsync + behaviour. + """ + try: + r = subprocess.run(rsync_argv(flag, '--version'), + capture_output=True, text=True, timeout=5) + except (subprocess.TimeoutExpired, OSError): + return True + if r.returncode == 0: + return True + stderr = (r.stderr or '').lower() + for marker in ('unknown option', 'unrecognized option', 'no such option'): + if marker in stderr: + return False + # Non-zero exit but no recognizable "unknown" marker -- assume supported. + return True + + def forced_protocol(): """The protocol version pinned via --protocol=N in the RSYNC command, or None when the run isn't pinning one (so the binary negotiates its newest). @@ -440,6 +729,52 @@ def is_a_link(path) -> bool: return os.path.islink(path) +def start_path_flipper(name_a, name_b): + """Spawn a separate PROCESS that repeatedly swaps two sibling paths + name_a <-> name_b in a tight rename loop, for TOCTOU symlink-race tests: + point one at a real directory and the other at a symlink so the shared name + keeps flipping between a directory and a symlink under a running rsync. + + A separate process (not a thread) is used deliberately: a Python thread + contends with the test's own loop for the GIL and flips far too slowly to + win the race. The swap is three renames via a scratch name in the same + directory, so the shared name is absent only for the brief instant between + two renames (rsync just gets ENOENT and retries). + + The caller should stop it with stop_flipper(), but the flipper also + self-terminates: it exits when its parent (the test process) goes away -- + os.getppid() changes once the test is reaped -- and after a hard deadline as a + backstop. Without this, a test killed before its stop_flipper() finally (a + timeout, a crash) would leak an orphan that keeps renaming paths in the shared + scratch and poisons later tests on the same box. os.getppid() is POSIX, so + this is portable across the fleet. + + Returns a subprocess.Popen; the caller must stop it with stop_flipper().""" + code = ( + "import os, sys, time\n" + "a, b = sys.argv[1], sys.argv[2]\n" + "tmp = a + '.flip'\n" + "parent = os.getppid()\n" + "deadline = time.monotonic() + 300\n" + "while os.getppid() == parent and time.monotonic() < deadline:\n" + " try:\n" + " os.rename(a, tmp); os.rename(b, a); os.rename(tmp, b)\n" + " except OSError:\n" + " pass\n" + ) + return subprocess.Popen([sys.executable, '-c', code, str(name_a), str(name_b)]) + + +def stop_flipper(proc): + """Stop a start_path_flipper() process.""" + proc.terminate() + try: + proc.wait(timeout=2) + except subprocess.TimeoutExpired: + proc.kill() + proc.wait() + + def cp_p(src, dst) -> 'None': """Equivalent of rsync.fns cp_p: copy preserving mode + timestamps.""" shutil.copy2(src, dst) @@ -1186,46 +1521,31 @@ def assert_not_exists(path, label: str = '') -> 'None': test_fail(f"{_tag(label)}{path} exists but should not") -_rb_cache = None +_psf_cache = None -def resolve_beneath_supported() -> bool: - """True if this rsync can FOLLOW an in-tree directory symlink under its - secure resolver -- i.e. update a file through a dir-symlink on the receiver - (--keep-dirlinks; issue #715). - - False wherever the portable per-component O_NOFOLLOW fallback is the active - resolver: a platform with no kernel "beneath" primitive, Linux < 5.6, a - seccomp-blocked openat2, or a --disable-openat2 build. There the delta - update through the symlinked directory fails verification. Probed - functionally (an initial transfer plus a delta update through a dir-symlink) - so it tracks the actual binary rather than a platform name, and cached.""" - global _rb_cache - if _rb_cache is not None: - return _rb_cache - probe = SCRATCHDIR / '.rb_probe' - rmtree(probe) - (probe / 'home' / 'real').mkdir(parents=True) - os.symlink('real', probe / 'home' / 'link') - (probe / 'src' / 'link').mkdir(parents=True) - f = probe / 'src' / 'link' / 'f' - make_data_file(f, 40000) - - def push(): - subprocess.run( - rsync_argv('-KRl', '--no-whole-file', 'link/f', - f"{probe / 'home'}/"), - cwd=str(probe / 'src'), - stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) - - push() - with open(f, 'ab') as fh: # size change -> forces a delta update - fh.write(b'appended tail for delta\n') - push() - dst = probe / 'home' / 'real' / 'f' - _rb_cache = dst.is_file() and filecmp.cmp(str(f), str(dst), shallow=False) - rmtree(probe) - return _rb_cache +def proc_self_fd_pins() -> bool: + """True iff /proc/self/fd/N is a Linux-style magic symlink whose readlink + yields the open file's real path -- the primitive rrsync's realpath-vs-exec + inode-pin relies on. macOS/BSD lack the directory; Solaris HAS /proc/self/fd + but its entries are not such symlinks. Mirrors rrsync's own HAVE_PROC_SELF_FD + probe so the rrsync race test runs only where the protection actually exists + (it falls through unpinned, by design, elsewhere). Cached.""" + global _psf_cache + if _psf_cache is not None: + return _psf_cache + try: + fd = os.open('/', os.O_RDONLY) + except OSError: + _psf_cache = False + return _psf_cache + try: + _psf_cache = (os.readlink('/proc/self/fd/%d' % fd) == '/') + except OSError: + _psf_cache = False + finally: + os.close(fd) + return _psf_cache def write_daemon_conf(modules, globals=None, *, @@ -1282,3 +1602,737 @@ def write_daemon_conf(modules, globals=None, *, ignore23.chmod(0o755) return conf + + +# --- security regression helpers ------------------------------------------- + +def expect_fail(argv, text, env=None, cwd=None): + proc = subprocess.run(argv, stdout=subprocess.PIPE, stderr=subprocess.PIPE, + text=True, env=env, cwd=cwd) + out = (proc.stdout or '') + (proc.stderr or '') + if proc.returncode == 0: + test_fail(f"command unexpectedly succeeded: {argv!r}\n{out}") + if text not in out: + test_fail(f"expected {text!r} in command output:\n{out}") + return proc + + +def patched_rrsync(workdir, rsync_path=None): + # The stub rsync just has to exec successfully; the BSDs keep true(1) in + # /usr/bin, not /bin, so resolve it on PATH rather than hard-coding /bin/true. + if rsync_path is None: + rsync_path = shutil.which('true') or '/usr/bin/true' + src = SRCDIR / 'support' / 'rrsync' + dst = Path(workdir) / 'rrsync-under-test' + dst.write_text(src.read_text().replace( + "RSYNC = '/usr/bin/rsync'", + f"RSYNC = {rsync_path!r}", + 1, + )) + dst.chmod(0o755) + return dst + + +def run_rrsync_denied(command, expected): + base = SCRATCHDIR / expected.replace(' ', '_').replace('/', '_') + base.mkdir(parents=True, exist_ok=True) + restricted = base / 'restricted' + restricted.mkdir(exist_ok=True) + rrsync = patched_rrsync(base) + env = {**os.environ, 'SSH_ORIGINAL_COMMAND': command} + expect_fail([str(rrsync), '-ro', '-no-lock', str(restricted)], expected, env=env) + + +def make_proxy_server(port, response): + listener = _socket.socket(_socket.AF_INET, _socket.SOCK_STREAM) + listener.setsockopt(_socket.SOL_SOCKET, _socket.SO_REUSEADDR, 1) + listener.bind(('127.0.0.1', port)) + listener.listen(1) + + def serve(): + conn, _ = listener.accept() + try: + conn.recv(65536) + conn.sendall(response) + finally: + try: + conn.close() + finally: + listener.close() + + import threading + t = threading.Thread(target=serve) + t.daemon = True + t.start() + return t + + +def run_proxy_probe(port, host, expected): + env = {**os.environ, 'RSYNC_PROXY': f'127.0.0.1:{port}'} + proc = subprocess.run( + rsync_argv(f'rsync://{host}/mod/', str(SCRATCHDIR / 'proxy-out')), + stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, env=env) + out = (proc.stdout or '') + (proc.stderr or '') + if proc.returncode == 0: + test_fail(f"proxy probe unexpectedly succeeded:\n{out}") + if expected not in out: + test_fail(f"expected {expected!r} in proxy probe output:\n{out}") + return proc + + +def setup_chroot_inner(name): + if get_testuid() != get_rootuid(): + test_skipped("chroot /./ module regression requires root") + if _under_valgrind(): + # The daemon's per-connection child chroots into the module, after + # which valgrind can no longer create its absolute --log-file %p path + # and the child dies (the transfer then resets) -- skip under valgrind. + test_skipped("daemon chroot prevents valgrind from writing its per-process log") + base = SCRATCHDIR / name + outer = base / 'outer' + inner = outer / 'inner' + outside = outer / 'outside' + src = base / 'src' + rmtree(base) + makepath(inner, outside, src) + os.symlink('../outside', inner / 'linkparent') + conf = write_daemon_conf([ + ('mod', {'path': str(outer) + '/./inner', 'read only': 'no', + 'use chroot': 'yes', 'munge symlinks': 'no'}), + ], name=f'{name}.conf') + url = start_test_daemon(conf, 12940 + (abs(hash(name)) % 200)) + return base, inner, outside, src, url + + +def run_checked(argv): + proc = subprocess.run(argv, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) + return proc, (proc.stdout or '') + (proc.stderr or '') + + +def build_patched_rsync(name, replacements): + # Cygwin can't reliably rebuild a single patched unit from the copied tree: + # make leaves the copied object in place (coarse NTFS mtimes) so the patch is + # silently absent, and forcing the rebuild trips gcc-13's -fno-common link + # errors against the prebuilt objects. The malicious-peer behaviour these + # tests simulate is platform-independent and is exercised on every POSIX + # target, so skip the unbuildable simulation here rather than misreport it. + if sys.platform == 'cygwin' or platform.system().startswith('CYGWIN'): + test_skipped(f"{name}: build_patched_rsync is unreliable on Cygwin " + "(prebuilt-object staleness / -fno-common relink); the " + "patched-peer fix is validated on the POSIX targets") + if not (SRCDIR / 'Makefile').is_file(): + test_skipped(f"{name}: needs a configured rsync source tree with a Makefile") + if not shutil.which('make'): + test_skipped(f"{name}: make(1) not on PATH") + if not shutil.which('gcc') and not shutil.which('cc'): + test_skipped(f"{name}: no C compiler on PATH") + + work = SCRATCHDIR / name + rmtree(work) + shutil.copytree( + SRCDIR, work, symlinks=True, + ignore=shutil.ignore_patterns( + 'testtmp', '.git', 'auto-build-save', 'autom4te.cache', '__pycache__')) + + for relpath, old, new in replacements: + path = work / relpath + text = path.read_text() + if old not in text: + test_skipped(f"{name}: could not find patch target in {relpath}: {old!r}") + path.write_text(text.replace(old, new, 1)) + + env = {**os.environ, 'CCACHE_DISABLE': '1'} + build = subprocess.run(['make', '-j2', 'rsync'], cwd=str(work), env=env, + stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True) + rsync = work / 'rsync' + if build.returncode != 0 or not rsync.is_file() or not os.access(rsync, os.X_OK): + test_skipped( + f"{name}: patched rsync build failed (rc={build.returncode}). " + "Tail of build output:\n" + '\n'.join(build.stdout.splitlines()[-20:])) + return rsync + + +# --- operator-supplied-path symlink policy matrix -------------------------- +# +# Policy: an operator-supplied path follows a symlink at any component iff that +# symlink is owned by uid 0 or the running euid, and refuses one owned by any +# other uid -- for absolute and relative paths alike. --insecure-links is a +# LOCAL opt-out that restores legacy following (a daemon never honours it; that +# is covered by a separate daemon test). run_symlink_matrix() drives one +# path-taking option through {cross-uid vs same-uid} x {absolute vs relative} x +# {symlink-at-leaf vs symlink-in-a-parent-component} x {--insecure-links off/on} +# and asserts each cell against the policy: FOLLOW iff (insecure or same-uid). + +def find_attacker_uid(): + """An untrusted uid (not 0, not the euid) for a cross-uid plant, else None.""" + import pwd + for nm in ('nobody', 'nfsnobody', 'daemon'): + try: + u = pwd.getpwnam(nm).pw_uid + except KeyError: + continue + if u != 0 and u != os.geteuid(): + return u + return None + + +def run_symlink_matrix(option, case, *, paths=('abs', 'rel'), + wheres=('leaf', 'parent'), label=''): + """Run `case(ctx)` over the operator-path symlink matrix; assert the policy. + + `case(ctx)` plants the option's path symlink (per ctx.where/ctx.abspath), + runs rsync (honouring ctx.insecure), and returns True if the symlink was + FOLLOWED (the op escaped to ctx.outside / read an out-of-tree object). + ctx carries: base, outside, plant (fresh Paths); owner ('cross'|'self'); + att_uid; abspath ('abs'|'rel'); where ('leaf'|'parent'); insecure (bool); + and plant_link(at, target) which symlinks target->at and lchowns it to the + attacker uid when owner=='cross' (otherwise it stays euid-owned). + + Cross-uid cells need root (to own a symlink by a foreign uid) and are + skipped otherwise; same-uid cells run at any uid. + """ + import re + import types + tag = option + (f' [{label}]' if label else '') + euid = os.geteuid() + att = find_attacker_uid() if euid == 0 else None + slug = re.sub(r'[^a-z0-9]+', '-', tag.lower()).strip('-') + + for abspath in paths: + for where in wheres: + for insecure in (False, True): + owners = ('self', 'cross') if att is not None else ('self',) + for owner in owners: + base = SCRATCHDIR / (f"{slug}-{owner}-{abspath}-{where}-" + + ('ins' if insecure else 'safe')) + rmtree(base) + base.mkdir(parents=True) + ctx = types.SimpleNamespace( + base=base, outside=base / 'outside', plant=base / 'plant', + owner=owner, att_uid=att, abspath=abspath, where=where, + insecure=insecure) + ctx.outside.mkdir() + ctx.plant.mkdir() + + def plant_link(at, target, _c=ctx): + os.symlink(target, at) + if _c.owner == 'cross': + os.lchown(at, _c.att_uid, _c.att_uid) + ctx.plant_link = plant_link + + followed = bool(case(ctx)) + expect = insecure or owner == 'self' + cell = f"{abspath} {where} {'insecure' if insecure else 'safe'}" + if followed and not expect: + test_fail( + f"{tag}: CROSS-UID {cell}: the planted symlink was " + "FOLLOWED (op escaped to outside/). An operator path " + "must refuse a symlink not owned by uid 0 or the euid.") + if not followed and expect: + why = ("--insecure-links did not restore symlink following" + if insecure else + "the operator's OWN (euid-owned) symlink was refused") + test_fail(f"{tag}: {('CROSS' if owner=='cross' else 'SAME')}" + f"-UID {cell}: {why}.") + if att is None and euid != 0: + print(f"{tag}: same-uid cells confirmed; cross-uid cells need root (skipped)") + + +def plant_operator_symlink(ctx, rel_anchor, kind='dir'): + """Plant this cell's option-path symlink and return (option_value, escape). + + option_value is what to feed the option: an absolute path, or a name + relative to rel_anchor (the directory the option resolves a relative value + against -- e.g. the destination dir for --backup-dir/--link-dest, or the cwd + for --temp-dir). escape is the out-of-tree object the operation acts on IF + the symlink is followed. + + kind='dir' (a directory option, e.g. --backup-dir/--temp-dir/--link-dest): + leaf -> the symlink itself is the dir; escape = ctx.outside. + parent -> a parent component is the symlink; escape = ctx.outside/'sub'. + kind='file' (a file option, e.g. --log-file/--files-from/--write-batch): + leaf -> the symlink targets the out-of-tree victim file directly. + parent -> a parent component is the symlink; the leaf name is appended. + escape = ctx.outside/'victim' either way. + """ + base = ctx.plant if ctx.abspath == 'abs' else rel_anchor + if kind == 'file': + victim = ctx.outside / 'victim' + if ctx.where == 'leaf': + link = base / 'osl' + ctx.plant_link(link, victim) + return (str(link) if ctx.abspath == 'abs' else 'osl'), victim + link = base / 'opd' + ctx.plant_link(link, ctx.outside) + return ((str(link / 'victim') if ctx.abspath == 'abs' else 'opd/victim'), + victim) + if ctx.where == 'leaf': + link = base / 'osl' + ctx.plant_link(link, ctx.outside) + return (str(link) if ctx.abspath == 'abs' else 'osl'), ctx.outside + link = base / 'opd' + ctx.plant_link(link, ctx.outside) + return ((str(link / 'sub') if ctx.abspath == 'abs' else 'opd/sub'), + ctx.outside / 'sub') + + +# --- variety tree (cross-version regression coverage) ---------------------- +# A "variety tree" exercises every inode type rsync handles (dirs, regular +# files, symlinks, fifos, sockets, char/block devices) with a spread of +# permissions, xattrs, ACLs and (as root) ownership, plus heavy symlink +# coverage: links to each type, links that escape a transfer root via ../.., +# absolute links, and links whose intermediate components transit outside the +# tree. It is the source for differential tests that assert the current binary +# produces the same destination tree as an old release (see variety_test.py). + +def acls_supported() -> bool: + """True if this rsync was built with ACL support AND this platform has a + usable setfacl/getfacl (or macOS chmod +a). Mirrors xattrs_supported().""" + vv = run_rsync('-VV', check=True, capture_output=True).stdout + if '"ACLs": true' not in vv: + return False + if _SYSTEM in ('Linux', 'FreeBSD') or _CYGWIN: + return (shutil.which('setfacl') is not None + and shutil.which('getfacl') is not None) + if _SYSTEM == 'Darwin': + return shutil.which('chmod') is not None + return False + + +def devices_supported() -> bool: + """True if device nodes can be created here: euid==0 AND os.mknod exists + (mknod of S_IFCHR/S_IFBLK needs CAP_MKNOD, i.e. root).""" + return os.geteuid() == 0 and hasattr(os, 'mknod') + + +def owners_supported() -> bool: + """True if the builder may assign mixed uid/gid (euid==0).""" + return os.geteuid() == 0 + + +def make_fifo(path) -> 'None': + """Create a FIFO (named pipe) at `path`.""" + os.mkfifo(str(path)) + + +def make_socket(path) -> 'None': + """Create a UNIX-domain socket inode at `path`. + + The AF_UNIX sun_path is capped at ~108 bytes, which a depth-8 absolute path + can overflow, so we chdir to the parent and bind the bare (short) basename, + restoring the cwd in a finally. The bound inode persists as an S_IFSOCK on + disk after the socket is closed.""" + path = Path(path) + old = os.getcwd() + s = _socket.socket(_socket.AF_UNIX, _socket.SOCK_STREAM) + try: + os.chdir(path.parent) + s.bind(path.name) + finally: + s.close() + os.chdir(old) + + +def make_device(path, kind: str, major: int, minor: int, + mode: int = 0o644) -> 'None': + """Create a char ('c') or block ('b') device node. Caller must gate on + devices_supported() -- unprivileged this raises PermissionError.""" + fmt = stat.S_IFCHR if kind == 'c' else stat.S_IFBLK + os.mknod(str(path), mode | fmt, os.makedev(major, minor)) + + +def _acl_env() -> dict: + """Environment for getfacl/setfacl: scrub POSIXLY_CORRECT (which alters + some getfacl builds' flag/header semantics) and pin LC_ALL=C.""" + env = dict(os.environ) + env.pop('POSIXLY_CORRECT', None) + env['LC_ALL'] = 'C' + return env + + +def acl_set(spec: str, path) -> bool: + """Apply one ACL entry `spec` (e.g. 'u:0:rwx', 'g:0:r-x', 'd:u:0:rwx' for a + directory default entry) to `path` via setfacl. Returns True on success, + False if the filesystem rejects ACLs (EOPNOTSUPP) so the caller degrades + gracefully. macOS is not driven here (returns False).""" + if not (_SYSTEM in ('Linux', 'FreeBSD') or _CYGWIN): + return False + proc = subprocess.run(['setfacl', '-m', spec, str(path)], + capture_output=True, text=True, env=_acl_env()) + return proc.returncode == 0 + + +def _acl_sig(path) -> str: + """Path-free signature of a node's ACL entries, for comparing two trees. + Strips getfacl's comment header (which embeds the path/owner) and sorts the + entries so the result depends only on the access/default ACL, not on where + the file lives. Empty when ACLs aren't readable here.""" + if not (_SYSTEM in ('Linux', 'FreeBSD') or _CYGWIN): + return '' + try: + out = subprocess.run(['getfacl', str(path)], capture_output=True, + text=True, env=_acl_env()).stdout + except OSError: + return '' + return ';'.join(sorted(l for l in out.splitlines() + if l.strip() and not l.startswith('#'))) + + +def _xattr_sig(path) -> str: + """Path-free signature of a node's user xattrs, for comparing two trees. + Native on Linux (symmetric with xattr_set); getfattr with the '# file:' + header stripped on Cygwin. Returns '' elsewhere -- xattr fidelity is gated + on the Linux CI run, and tls still compares structure on every platform.""" + p = str(path) + if _SYSTEM == 'Linux': + try: + names = sorted(n for n in os.listxattr(p, follow_symlinks=False) + if n.startswith('user.')) + except OSError: + return '' + out = [] + for n in names: + try: + v = os.getxattr(p, n, follow_symlinks=False) + except OSError: + continue + out.append(n + '=' + v.decode('utf-8', 'surrogateescape')) + return ';'.join(out) + if _CYGWIN: + try: + d = subprocess.check_output( + ['getfattr', '--no-dereference', '-d', p], + text=True, stderr=subprocess.DEVNULL) + except (subprocess.CalledProcessError, OSError): + return '' + return ';'.join(sorted(l for l in d.splitlines() + if l and not l.startswith('# file:'))) + return '' + + +def _variety_fill(path, size: int, key: str) -> 'None': + """Write `size` bytes of deterministic, non-trivial content derived from + `key`, so a variety tree is byte-reproducible across separate builds.""" + import hashlib + buf = bytearray() + i = 0 + while len(buf) < size: + buf += hashlib.sha256(f'{key}:{i}'.encode()).digest() + i += 1 + with open(str(path), 'wb') as f: + f.write(bytes(buf[:size])) + + +def _all_entries(root) -> list: + """Every entry under `root` (the root dir, real subdirs, files, specials, + and symlinks themselves) visited exactly once and WITHOUT following any + symlink. For chown/utime finalisation that must not escape the tree.""" + root = Path(root) + res = [] + for dp, dns, fns in os.walk(root): # followlinks=False + d = Path(dp) + res.append(d) + for n in fns: + res.append(d / n) + for n in dns: + sub = d / n + if sub.is_symlink(): # os.walk won't recurse into it + res.append(sub) + return res + + +def make_variety_tree(root, *, depth: int = 8, with_acls=None, with_xattrs=None, + with_devices=None, with_owners=None, + seed: int = 0x5A17) -> dict: + """Build a deterministic 'variety tree' rooted at `root`. + + Capability args default to None => auto-detect (xattrs_supported() etc.). + Tests pass EXPLICIT bools so the current- and old-binary source trees are + built with identical capabilities, keeping the differential comparison + apples-to-apples. Re-runnable: rmtree(root) first; `seed` drives only fixed + choices (no time/pid randomness) so two calls yield identical trees. + + Layout (caller transfers root/transfer_root/): + root/above/ real nodes ABOVE the transfer root (escape targets) + root/transfer_root d0..d{depth-1} backbone; at each level a bouquet of + every type + a symlink to every type; plus abs_links/ + (absolute links) and escape/ (../.. links that leave + the transfer root, one per above-root type). + + Returns {'transfer_root': Path, 'above_targets': {type: Path}, + 'counts': {type: n}}. + """ + root = Path(root) + rmtree(root) + if with_xattrs is None: + with_xattrs = xattrs_supported() + if with_acls is None: + with_acls = acls_supported() + if with_devices is None: + with_devices = devices_supported() + if with_owners is None: + with_owners = owners_supported() + + root.mkdir(parents=True) + above = root / 'above' + above.mkdir() + troot = root / 'transfer_root' + troot.mkdir() + + counts = {} + def bump(t): + counts[t] = counts.get(t, 0) + 1 + + perm_cycle = [0o400, 0o640, 0o644, 0o600, 0o755] + + def reg(p, size, mode=0o644): + _variety_fill(p, size, f'{seed:x}:{os.path.relpath(p, root)}') + os.chmod(p, mode) + bump('file') + return p + + def mkdir1(p, mode=None): + p.mkdir() + if mode is not None: + os.chmod(p, mode) + bump('dir') + return p + + def lnk(target, p): + os.symlink(target, p) + bump('symlink') + return p + + # --- above-root real targets (escape/ links point here) --- + above_targets = {} + above_targets['dir'] = mkdir1(above / 'a_dir') + reg(above / 'a_dir' / 'inner', 256) + above_targets['file'] = reg(above / 'a_file', 8192) + above_targets['fifo'] = above / 'a_fifo'; make_fifo(above_targets['fifo']); bump('fifo') + above_targets['sock'] = above / 'a_sock'; make_socket(above_targets['sock']); bump('socket') + if with_devices: + above_targets['dev'] = above / 'a_dev_c' + make_device(above_targets['dev'], 'c', 1, 3); bump('device') + make_device(above / 'a_dev_b', 'b', 7, 0); bump('device') + above_targets['link'] = lnk('a_file', above / 'a_link') + + # --- depth backbone with a bouquet at each level --- + cur = troot + for n in range(depth): + reg(cur / f'f{n}', 1024 * (n + 1)) + if n % 2 == 0: # hard-link coverage for -H + os.link(cur / f'f{n}', cur / f'hl{n}') + bump('hardlink') + reg(cur / f'perm{n}', 700, perm_cycle[(seed + n) % len(perm_cycle)]) + reg(cur / f'setuid{n}', 512, 0o4755) + mkdir1(cur / f'setgid{n}', 0o2775) + mkdir1(cur / f'sticky{n}', 0o1777) + for k in range(3): # widen toward ~200 entries + reg(cur / f'g{n}_{k}', 300) + make_fifo(cur / f'fifo{n}'); bump('fifo') + make_socket(cur / f'sk{n}'); bump('socket') + if with_devices: + make_device(cur / f'cdev{n}', 'c', 1, 5); bump('device') + make_device(cur / f'bdev{n}', 'b', 7, n); bump('device') + # symlink bouquet: one link to each type present at this level + lnk(f'f{n}', cur / f'ln2file{n}') + lnk(f'fifo{n}', cur / f'ln2fifo{n}') + lnk(f'sk{n}', cur / f'ln2sock{n}') + if with_devices: + lnk(f'cdev{n}', cur / f'ln2dev{n}') + lnk(f'ln2file{n}', cur / f'ln2ln{n}') # link to a symlink + lnk(f'nonexistent_{n}', cur / f'dangling{n}') + if n < depth - 1: + nxt = mkdir1(cur / f'd{n + 1}') + lnk(f'd{n + 1}', cur / f'ln2dir{n}') # link to a directory + cur = nxt + + # --- absolute-path links (targets encode the source scratch path) --- + absd = mkdir1(troot / 'abs_links') + lnk(str((troot / 'f0').resolve()), absd / 'abs_file') + lnk(str(above_targets['file'].resolve()), absd / 'abs_above') + + # --- escaping (unsafe) links: inside the transfer root, target outside --- + escd = mkdir1(troot / 'escape') + lnk('../../above/a_dir', escd / 'esc_dir') + lnk('../../above/a_file', escd / 'esc_file') + lnk('../../above/a_fifo', escd / 'esc_fifo') + lnk('../../above/a_sock', escd / 'esc_sock') + if with_devices: + lnk('../../above/a_dev_c', escd / 'esc_dev') + lnk('../../above/a_link', escd / 'esc_link') + # intermediate component transits OUTSIDE the whole tree, then back in + lnk(f'../../../{root.name}/above/a_file', escd / 'esc_deep') + + # --- xattrs on dirs + regular files (symlink xattrs are unsupported on + # Linux, so skip them) --- + if with_xattrs: + for p in walk_dirs(troot) + walk_files(troot): + try: + xattr_set('variety', os.path.basename(str(p)), p) + except OSError: + pass + + # --- ACLs on a deterministic subset --- + if with_acls: + dirs = walk_dirs(troot) + files = walk_files(troot) + for i, d in enumerate(dirs): + if i % 3 == 0: + acl_set('u:0:rwx', d) + if i % 5 == 0: + acl_set('d:u:0:rwx', d) # directory default entry + for i, f in enumerate(files): + if i % 4 == 0: + acl_set('g:0:r-x', f) + + entries = sorted(_all_entries(root), key=lambda x: str(x)) + + # --- mixed ownership (root only), including symlinks --- + if with_owners: + idset = [(0, 0), (1, 1), (2, 2)] + for i, p in enumerate(entries): + uid, gid = idset[(seed + i) % len(idset)] + try: + os.chown(str(p), uid, gid, follow_symlinks=False) + except OSError: + pass + + # --- deterministic, varied mtimes (last, so nothing resets them); makes + # the tls listing reproducible across separate builds --- + base = 1_000_000_000 + for i, p in enumerate(entries): + t = base + (i * 7) % 1_000_000 + try: + os.utime(str(p), (t, t), follow_symlinks=False) + except (OSError, NotImplementedError, ValueError): + if not os.path.islink(str(p)): + try: + os.utime(str(p), (t, t)) + except OSError: + pass + + return {'transfer_root': troot, 'above_targets': above_targets, + 'counts': counts} + + +def _rel_nonlink_entries(root) -> list: + """Relative paths of every non-symlink entry (real dirs + non-symlink + files/specials) under `root`, sorted. Used to compare per-entry metadata + without following or descending into symlinks.""" + root = Path(root) + res = [] + for dirpath, _dirnames, filenames in os.walk(root): # followlinks=False + d = Path(dirpath) + if d != root: + res.append(d.relative_to(root)) + for fn in filenames: + fp = d / fn + if not fp.is_symlink(): + res.append(fp.relative_to(root)) + return sorted(res, key=lambda p: str(p)) + + +def _safe_walk_files(root) -> list: + """walk_files() variant that tolerates unreadable directories (skips them + instead of raising), for comparing trees whose mixed/foreign ownership can + leave some entries inaccessible to the current user.""" + root = Path(root) + res = [] + for dp, _dns, fns in os.walk(root): # onerror=None -> unreadable dirs skipped + d = Path(dp) + for n in fns: + p = d / n + try: + if p.is_file() and not p.is_symlink(): + res.append(p) + except OSError: + continue + return sorted(res, key=lambda x: str(x)) + + +def compare_trees(a, b, label: str = '', *, + with_acls: bool = True, + with_xattrs: bool = True) -> list: + """Compare two trees WITHOUT ever opening a fifo/socket/device as a + stream. Returns a list of human-readable difference strings ([] == match); + the caller decides whether a difference is a fail or an xfail. + + Checks: (1) the tls listings (type+mode+owner+size+mtime+symlink target for + every inode); (2) byte-equality of regular files by relative path; (3) user + xattrs; (4) POSIX ACLs. Never uses `diff -r` (it blocks on specials).""" + a = Path(a) + b = Path(b) + pre = f"{label}: " if label else "" + diffs = [] + + la = rsync_ls_lR(a) + lb = rsync_ls_lR(b) + if la != lb: + import difflib + ud = ''.join(difflib.unified_diff( + la.splitlines(keepends=True), lb.splitlines(keepends=True), + fromfile=f'{a} (tls)', tofile=f'{b} (tls)')) + diffs.append(f"{pre}tls listings differ:\n{ud}") + + files_a = sorted(p.relative_to(a) for p in _safe_walk_files(a)) + files_b = sorted(p.relative_to(b) for p in _safe_walk_files(b)) + set_b = set(files_b) + if set(files_a) != set_b: + only_a = sorted(str(p) for p in set(files_a) - set_b) + only_b = sorted(str(p) for p in set_b - set(files_a)) + diffs.append(f"{pre}regular-file set differs: " + f"only in a={only_a} only in b={only_b}") + for rel in files_a: + if rel not in set_b: + continue + try: + same = filecmp.cmp(str(a / rel), str(b / rel), shallow=False) + except OSError as e: + diffs.append(f"{pre}cannot compare contents of {rel} " + f"(permission denied?): {e}") + continue + if not same: + diffs.append(f"{pre}content differs: {rel}") + + # hard-link grouping (catches an -H divergence the tls listing can't show) + def _hl_groups(rootp): + from collections import defaultdict + ino = defaultdict(list) + for p in _safe_walk_files(rootp): + try: + st = p.stat() + except OSError: + continue + if st.st_nlink > 1: + ino[(st.st_dev, st.st_ino)].append(str(p.relative_to(rootp))) + return sorted(tuple(sorted(v)) for v in ino.values() if len(v) > 1) + ga, gb = _hl_groups(a), _hl_groups(b) + if ga != gb: + diffs.append(f"{pre}hard-link grouping differs: a={ga} b={gb}") + + if with_xattrs or with_acls: + for rel in _rel_nonlink_entries(a): + pa = a / rel + pb = b / rel + if not pb.exists(): + continue + if with_xattrs: + xa, xb = _xattr_sig(pa), _xattr_sig(pb) + if xa != xb: + diffs.append(f"{pre}xattr differs: {rel} " + f"(a={xa!r} b={xb!r})") + if with_acls: + aa, ab = _acl_sig(pa), _acl_sig(pb) + if aa != ab: + diffs.append(f"{pre}ACL differs: {rel} " + f"(a={aa!r} b={ab!r})") + + return diffs + + +def assert_trees_equal(a, b, label: str = '', **kwargs) -> 'None': + """compare_trees(); test_fail() on any difference.""" + diffs = compare_trees(a, b, label, **kwargs) + if diffs: + test_fail('\n'.join(diffs)) diff --git a/testsuite/xrsync.py b/testsuite/xrsync.py new file mode 100755 index 00000000..f2a5bc25 --- /dev/null +++ b/testsuite/xrsync.py @@ -0,0 +1,164 @@ +#!/usr/bin/env python3 +"""xrsync.py -- a small, runnable rsync client built on rsync_proto.py. + +It speaks the rsync daemon protocol (host::module/path or rsync://host/module/ +path) and supports three operations against a real rsyncd: + + xrsync.py [opts] host::module/path # list (like rsync's listing) + xrsync.py [opts] host::module/glob localdir # pull (download) + xrsync.py [opts] localfile... host::mod/ # push (upload, regular files) + +This is deliberately a *subset* of rsync -- enough to be useful for protocol +development and as a test harness, not a drop-in replacement. Supported flags: +-a (= -rlpt), -r, -l, -t, -p, -v, --list-only, --port=N. + +Hookability: rsync_proto's DaemonClient exposes the protocol steps as small +overridable methods (recv_flist / make_request / recv_file_transfer / +make_file_token_stream / ...). main() takes a `client_factory`, so a test can +pass a DaemonClient subclass that tampers with one step while reusing all of +xrsync's machinery. See xrsync_test.py. +""" + +import argparse +import os +import sys + +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) +import rsync_proto as rp # noqa: E402 + +DEFAULT_PORT = 873 +CAPS = 'e.LsfxCIu' # the -e capability marker rsync_proto negotiates at p30 + + +def parse_remote(spec): + """Return (host, port, path) for a daemon spec, or None if `spec` is local. + `path` keeps the module as its first component (what the daemon expects).""" + if spec.startswith('rsync://'): + rest = spec[len('rsync://'):] + hostport, _, path = rest.partition('/') + host, _, port = hostport.partition(':') + return host, int(port) if port else DEFAULT_PORT, path + if '::' in spec: + host, _, path = spec.partition('::') + return host, None, path + return None + + +def server_opts(args, listing): + """Build the daemon-side short-option string from the parsed flags.""" + flags = '' + if args.recurse: + flags += 'r' + if args.links: + flags += 'l' + if args.perms: + flags += 'p' + if args.times: + flags += 't' + if listing and not args.recurse: + flags += 'd' # list a directory's contents without recursing + return '-' + flags + CAPS + + +def do_list(host, port, module, path, args, factory): + c = factory(host, port) + c.handshake(module, ['--server', '--sender', server_opts(args, True), + '.', path], greeting_version=30) + entries = rp.sort_entries(c.recv_flist(preserve_links=args.links)) + for e in entries: + tgt = '' + if e.is_link and e.link_target is not None: + tgt = ' -> ' + e.link_target.decode('utf-8', 'surrogateescape') + print('%s %15d %s%s' % (rp.mode_to_perms(e.mode), e.length, + e.name.decode('utf-8', 'surrogateescape'), tgt)) + c.finish_no_transfer() + c.drain(timeout=1.0) + c.close() + return 0 + + +def do_pull(host, port, module, path, dest, args, factory): + c = factory(host, port) + c.handshake(module, ['--server', '--sender', server_opts(args, False), + '.', path], greeting_version=30) + c.pull(dest, verbose=args.verbose, preserve_times=args.times, + preserve_perms=args.perms) + c.drain(timeout=1.0) + c.close() + return 0 + + +def do_push(srcs, host, port, module, path, args, factory): + files = [] + for s in srcs: + if os.path.isfile(s): + with open(s, 'rb') as fh: + files.append((os.path.basename(s), fh.read())) + elif os.path.isdir(s): + for root, _dirs, names in os.walk(s): + for nm in names: + full = os.path.join(root, nm) + if os.path.isfile(full) and not os.path.islink(full): + rel = os.path.relpath(full, s) + with open(full, 'rb') as fh: + files.append((rel, fh.read())) + c = factory(host, port) + c.handshake(module, ['--server', server_opts(args, False), '.', path], + greeting_version=30) + c.push(files) + if args.verbose: + for name, _ in files: + print(name) + c.drain(timeout=1.0) + c.close() + return 0 + + +def main(argv=None, client_factory=rp.DaemonClient): + p = argparse.ArgumentParser(prog='xrsync.py', add_help=True) + p.add_argument('-a', '--archive', action='store_true', help='= -rlpt') + p.add_argument('-r', '--recursive', dest='recurse', action='store_true') + p.add_argument('-l', '--links', action='store_true') + p.add_argument('-p', '--perms', action='store_true') + p.add_argument('-t', '--times', action='store_true') + p.add_argument('-v', '--verbose', action='store_true') + p.add_argument('--list-only', action='store_true') + p.add_argument('--port', type=int, default=None) + p.add_argument('paths', nargs='+') + args = p.parse_args(argv) + if args.archive: + args.recurse = args.links = args.perms = args.times = True + + paths = args.paths + remotes = [parse_remote(x) for x in paths] + + # List: a single remote arg (or --list-only with no local dest). + if args.list_only or (len(paths) == 1 and remotes[0] is not None): + host, port, path = remotes[0] + port = args.port or port or DEFAULT_PORT + module = path.split('/')[0] + return do_list(host, port, module, path, args, client_factory) + + if len(paths) < 2: + p.error('need a source and a destination') + *srcs, dest = paths + *src_remotes, dest_remote = remotes + + if dest_remote is not None and all(r is None for r in src_remotes): + host, port, path = dest_remote + port = args.port or port or DEFAULT_PORT + module = path.split('/')[0] + return do_push(srcs, host, port, module, path, args, client_factory) + + if len(srcs) == 1 and src_remotes[0] is not None and dest_remote is None: + host, port, path = src_remotes[0] + port = args.port or port or DEFAULT_PORT + module = path.split('/')[0] + return do_pull(host, port, module, path, dest, args, client_factory) + + p.error('unsupported source/destination combination (one side must be ' + 'host::module/path and the other local)') + + +if __name__ == '__main__': + sys.exit(main())