Compare commits

..

1 Commits

Author SHA1 Message Date
Will Sarg
6143dba73f testsuite: fix executability test skip on FreeBSD (EFTYPE)
FreeBSD and OpenBSD return EFTYPE (errno 79) when chmod-ing a sticky bit
onto a regular file as non-root, rather than EPERM/EACCES. Catch OSError
and check errno against the expected skip set so the test skips correctly
on those platforms instead of erroring out.

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
2026-06-11 01:02:42 -04:00
56 changed files with 1214 additions and 2753 deletions

View File

@@ -43,10 +43,8 @@ jobs:
# (rsyncfns.py drives xattrs via getfattr/setfattr from the `attr`
# package installed above), verified on a real Cygwin host. The real
# chown/devices tests still skip (need root/mknod), as do the
# RESOLVE_BENEATH symlink-race tests. symlink-dirlink-basis also now
# RUNS (the #915 non-daemon basis open uses a plain do_open, restoring
# following an in-tree dir-symlink basis without RESOLVE_BENEATH).
run: bash -c 'RSYNC_EXPECT_SKIPPED=acls-default,acls-depth,acls,bare-do-open-symlink-race,chdir-symlink-race,chown,daemon-access-ip,daemon-chroot-acl,devices,dir-sgid,open-noatime,protected-regular,proxy-response-line-too-long,sender-flist-symlink-leak,simd-checksum make check'
# RESOLVE_BENEATH symlink-race tests.
run: bash -c 'RSYNC_EXPECT_SKIPPED=acls-default,acls-depth,acls,bare-do-open-symlink-race,chdir-symlink-race,chown,daemon-access-ip,daemon-chroot-acl,devices,dir-sgid,open-noatime,protected-regular,proxy-response-line-too-long,sender-flist-symlink-leak,simd-checksum,symlink-dirlink-basis make check'
- name: check (TCP daemon transport)
# Second run with daemon tests over a real loopback rsyncd; the default
# 'make check' above uses the secure stdio-pipe transport.

View File

@@ -1,64 +0,0 @@
name: Test fleettest harness
# Bitrot check for testsuite/fleettest.py (the developer fleet CI harness).
# fleettest is meant to be run by developers on a modern Ubuntu box, so this
# job runs only on ubuntu-latest: it stands up a one-host "fleet" of two
# targets that both ssh to localhost and runs a real fleettest pass against it.
# It does not run on the BSD/Solaris/macOS/Cygwin matrix.
on:
push:
branches: [ master ]
paths:
- 'testsuite/fleettest.py'
- '.github/workflows/fleettest.yml'
pull_request:
branches: [ master ]
paths:
- 'testsuite/fleettest.py'
- '.github/workflows/fleettest.yml'
workflow_dispatch:
schedule:
- cron: '17 7 * * 1'
jobs:
fleettest:
runs-on: ubuntu-latest
name: fleettest against localhost
steps:
- uses: actions/checkout@v4
with:
fetch-depth: 0
- name: prep
run: |
sudo apt-get update
sudo apt-get install -y gcc g++ gawk autoconf automake \
acl libacl1-dev attr libattr1-dev liblz4-dev libzstd-dev libxxhash-dev \
python3-cmarkgfm openssl rsync openssh-server
- name: set up ssh to localhost
run: |
mkdir -p ~/.ssh && chmod 700 ~/.ssh
ssh-keygen -t ed25519 -N '' -f ~/.ssh/id_ed25519
cat ~/.ssh/id_ed25519.pub >> ~/.ssh/authorized_keys
chmod 600 ~/.ssh/authorized_keys
sudo systemctl start ssh || sudo service ssh start
# fleettest connects with `ssh -o BatchMode=yes localhost`, which won't
# answer a host-key prompt -- so pre-trust localhost in known_hosts.
ssh-keyscan -H localhost 127.0.0.1 >> ~/.ssh/known_hosts 2>/dev/null
ssh -o BatchMode=yes -o ConnectTimeout=15 localhost 'echo ssh-to-localhost-ok'
- name: write localhost fleet config
run: |
cat > fleettest-ci.json <<'EOF'
{ "targets": [
{ "name": "local-a", "ssh_host": "localhost", "workflow": "none.yml",
"configure_flags": [], "builddir": "rsync-citest-a", "privilege": "sudo" },
{ "name": "local-b", "ssh_host": "localhost", "workflow": "none.yml",
"configure_flags": [], "builddir": "rsync-citest-b", "privilege": "sudo" }
] }
EOF
- name: fleettest --list (config sanity)
run: python3 testsuite/fleettest.py --fleet fleettest-ci.json --list
- name: run fleettest against localhost
# Two targets both on localhost exercise the parallel multi-target path
# and the per-run dir / port isolation; exit 0 iff every cell is OK.
run: python3 testsuite/fleettest.py --fleet fleettest-ci.json --timing

1
.gitignore vendored
View File

@@ -52,7 +52,6 @@ aclocal.m4
/testsuite/chown-fake.test
/testsuite/devices-fake.test
/testsuite/xattrs-hlink.test
/testsuite/fleettest.json
/patches
/patches.gen
/build

View File

@@ -44,7 +44,7 @@ LIBOBJ=lib/wildmatch.o lib/compat.o lib/snprintf.o lib/mdfour.o lib/md5.o \
zlib_OBJS=zlib/deflate.o zlib/inffast.o zlib/inflate.o zlib/inftrees.o \
zlib/trees.o zlib/zutil.o zlib/adler32.o zlib/compress.o zlib/crc32.o
OBJS1=flist.o rsync.o generator.o receiver.o cleanup.o sender.o exclude.o \
util1.o util2.o main.o checksum.o match.o syscall.o android.o log.o backup.o delete.o
util1.o util2.o main.o checksum.o match.o syscall.o log.o backup.o delete.o
OBJS2=options.o io.o compat.o hlink.o token.o uidlist.o socket.o hashtable.o \
usage.o fileio.o batch.o clientname.o chmod.o acls.o xattrs.o
OBJS3=progress.o pipe.o @MD5_ASM@ @ROLL_SIMD@ @ROLL_ASM@
@@ -53,7 +53,7 @@ popt_OBJS= popt/popt.o popt/poptconfig.o \
popt/popthelp.o popt/poptparse.o popt/poptint.o
OBJS=$(OBJS1) $(OBJS2) $(OBJS3) $(DAEMON_OBJ) $(LIBOBJ) @BUILD_ZLIB@ @BUILD_POPT@
TLS_OBJ = tls.o syscall.o android.o util2.o t_stub.o lib/compat.o lib/snprintf.o lib/permstring.o lib/sysxattrs.o @BUILD_POPT@
TLS_OBJ = tls.o syscall.o util2.o t_stub.o lib/compat.o lib/snprintf.o lib/permstring.o lib/sysxattrs.o @BUILD_POPT@
# Programs we must have to run the test cases
CHECK_PROGS = rsync$(EXEEXT) tls$(EXEEXT) getgroups$(EXEEXT) getfsdev$(EXEEXT) \
@@ -84,19 +84,12 @@ install: all
$(INSTALLCMD) -m 755 $(srcdir)/rsync-ssl $(DESTDIR)$(bindir)
-$(MKDIR_P) $(DESTDIR)$(mandir)/man1
-$(MKDIR_P) $(DESTDIR)$(mandir)/man5
for fn in rsync.1 rsync-ssl.1; do \
if test -f $$fn; then $(INSTALLMAN) -m 644 $$fn $(DESTDIR)$(mandir)/man1; \
elif test -f $(srcdir)/$$fn; then $(INSTALLMAN) -m 644 $(srcdir)/$$fn $(DESTDIR)$(mandir)/man1; fi; \
done
for fn in rsyncd.conf.5; do \
if test -f $$fn; then $(INSTALLMAN) -m 644 $$fn $(DESTDIR)$(mandir)/man5; \
elif test -f $(srcdir)/$$fn; then $(INSTALLMAN) -m 644 $(srcdir)/$$fn $(DESTDIR)$(mandir)/man5; fi; \
done
if test -f rsync.1; then $(INSTALLMAN) -m 644 rsync.1 $(DESTDIR)$(mandir)/man1; fi
if test -f rsync-ssl.1; then $(INSTALLMAN) -m 644 rsync-ssl.1 $(DESTDIR)$(mandir)/man1; fi
if test -f rsyncd.conf.5; then $(INSTALLMAN) -m 644 rsyncd.conf.5 $(DESTDIR)$(mandir)/man5; fi
if test "$(with_rrsync)" = yes; then \
$(INSTALLCMD) -m 755 rrsync $(DESTDIR)$(bindir); \
fn=rrsync.1; \
if test -f $$fn; then $(INSTALLMAN) -m 644 $$fn $(DESTDIR)$(mandir)/man1; \
elif test -f $(srcdir)/$$fn; then $(INSTALLMAN) -m 644 $(srcdir)/$$fn $(DESTDIR)$(mandir)/man1; fi; \
if test -f rrsync.1; then $(INSTALLMAN) -m 644 rrsync.1 $(DESTDIR)$(mandir)/man1; fi; \
fi
install-ssl-daemon: stunnel-rsyncd.conf
@@ -179,19 +172,19 @@ getgroups$(EXEEXT): getgroups.o
getfsdev$(EXEEXT): getfsdev.o
$(CC) $(CFLAGS) $(LDFLAGS) -o $@ getfsdev.o $(LIBS)
TRIMSLASH_OBJ = trimslash.o syscall.o android.o util2.o t_stub.o lib/compat.o lib/snprintf.o
TRIMSLASH_OBJ = trimslash.o syscall.o util2.o t_stub.o lib/compat.o lib/snprintf.o
trimslash$(EXEEXT): $(TRIMSLASH_OBJ)
$(CC) $(CFLAGS) $(LDFLAGS) -o $@ $(TRIMSLASH_OBJ) $(LIBS)
T_UNSAFE_OBJ = t_unsafe.o syscall.o android.o util1.o util2.o t_stub.o lib/compat.o lib/snprintf.o lib/wildmatch.o
T_UNSAFE_OBJ = t_unsafe.o syscall.o util1.o util2.o t_stub.o lib/compat.o lib/snprintf.o lib/wildmatch.o
t_unsafe$(EXEEXT): $(T_UNSAFE_OBJ)
$(CC) $(CFLAGS) $(LDFLAGS) -o $@ $(T_UNSAFE_OBJ) $(LIBS)
T_CHMOD_SECURE_OBJ = t_chmod_secure.o syscall.o android.o util1.o util2.o t_stub.o lib/compat.o lib/snprintf.o lib/wildmatch.o lib/permstring.o
T_CHMOD_SECURE_OBJ = t_chmod_secure.o syscall.o util1.o util2.o t_stub.o lib/compat.o lib/snprintf.o lib/wildmatch.o lib/permstring.o
t_chmod_secure$(EXEEXT): $(T_CHMOD_SECURE_OBJ)
$(CC) $(CFLAGS) $(LDFLAGS) -o $@ $(T_CHMOD_SECURE_OBJ) $(LIBS)
T_SECURE_RELPATH_OBJ = t_secure_relpath.o syscall.o android.o util1.o util2.o t_stub.o lib/compat.o lib/snprintf.o lib/wildmatch.o lib/permstring.o
T_SECURE_RELPATH_OBJ = t_secure_relpath.o syscall.o util1.o util2.o t_stub.o lib/compat.o lib/snprintf.o lib/wildmatch.o lib/permstring.o
t_secure_relpath$(EXEEXT): $(T_SECURE_RELPATH_OBJ)
$(CC) $(CFLAGS) $(LDFLAGS) -o $@ $(T_SECURE_RELPATH_OBJ) $(LIBS)

11
TODO
View File

@@ -15,6 +15,7 @@ Create more granular verbosity 2003/05/15
DOCUMENTATION --------------------------------------------------------
Keep list of open issues and todos on the web site
Perhaps redo manual as SGML
LOGGING --------------------------------------------------------------
Memory accounting
@@ -212,6 +213,16 @@ DOCUMENTATION --------------------------------------------------------
Keep list of open issues and todos on the web site
-- --
Perhaps redo manual as SGML
The man page is getting rather large, and there is more information
that ought to be added.
TexInfo source is probably a dying format.
Linuxdoc looks like the most likely contender. I know DocBook is
favoured by some people, but it's so bloody verbose, even with emacs
support.

View File

@@ -1,82 +0,0 @@
/*
* Android-specific helpers.
*
* openat2() usability probe
* -------------------------
* openat2(2) is invoked directly via syscall() because the C library lacked a
* wrapper for it for years. Under a seccomp filter that uses
* SECCOMP_RET_TRAP -- as the Android application sandbox does -- a disallowed
* syscall raises SIGSYS and *kills the process* rather than failing with
* ENOSYS, so inspecting errno after the call is too late. We therefore probe
* openat2() once, behind a temporary SIGSYS handler, so a trapped syscall is
* caught and secure_relative_open_linux() can fall back to the portable
* per-component O_NOFOLLOW resolver instead of the whole process dying.
*
* This is only needed on Android, so the probe body is compiled only there.
* __ANDROID__ is defined by Bionic's headers and reflects the *target*, not
* the build host: it is set both for NDK cross-compiles (from a Linux/macOS
* host) and for native Termux builds, and is unset on every other platform.
* That makes it a reliable compile-time switch for cross builds -- there is
* nothing to detect in configure. Everywhere else openat2() is never
* seccomp-trapped to SIGSYS (a missing syscall simply returns ENOSYS), so
* openat2_usable() collapses to a constant 1 with no run-time cost.
*/
#include "rsync.h"
#if defined(__ANDROID__) && defined(HAVE_OPENAT2)
#include <setjmp.h>
#include <sys/syscall.h>
#include <linux/openat2.h>
static sigjmp_buf openat2_probe_env;
static void openat2_probe_handler(int signo)
{
(void)signo;
siglongjmp(openat2_probe_env, 1);
}
#endif
int openat2_usable(void)
{
#if defined(__ANDROID__) && defined(HAVE_OPENAT2)
static int cached = -1;
struct sigaction sa, old_sa;
if (cached >= 0)
return cached;
memset(&sa, 0, sizeof sa);
sa.sa_handler = openat2_probe_handler;
sigemptyset(&sa.sa_mask);
if (sigaction(SIGSYS, &sa, &old_sa) != 0)
return cached = 0;
if (sigsetjmp(openat2_probe_env, 1) != 0) {
/* SIGSYS delivered: openat2 is blocked by a seccomp filter. */
cached = 0;
} else {
struct open_how how;
int fd;
memset(&how, 0, sizeof how);
how.flags = O_RDONLY | O_DIRECTORY;
how.resolve = RESOLVE_BENEATH | RESOLVE_NO_MAGICLINKS;
fd = syscall(SYS_openat2, AT_FDCWD, ".", &how, sizeof how);
if (fd >= 0)
close(fd);
/* Usable only if the probe actually succeeded. Any failure --
* ENOSYS (kernel < 5.6), a seccomp SECCOMP_RET_ERRNO denial
* (EPERM/EACCES), or EINVAL (RESOLVE_BENEATH unsupported) --
* means we must fall back to the portable O_NOFOLLOW walk. */
cached = fd >= 0;
}
sigaction(SIGSYS, &old_sa, NULL);
return cached;
#else
return 1;
#endif
}

95
chmod.c
View File

@@ -29,7 +29,7 @@ extern mode_t orig_umask;
struct chmod_mode_struct {
struct chmod_mode_struct *next;
int ModeAND, ModeOR, ModeCOPY_SRC, ModeCOPY_DST, ModeCOPY_AND, ModeOP;
int ModeAND, ModeOR;
char flags;
};
@@ -43,20 +43,6 @@ struct chmod_mode_struct {
#define STATE_2ND_HALF 2
#define STATE_OCTAL_NUM 3
static int mode_dest_special_bits(int where)
{
int bits = 0;
if (where & 0100)
bits |= S_ISUID;
if (where & 0010)
bits |= S_ISGID;
if (where & 0001)
bits |= S_ISVTX;
return bits;
}
/* Parse a chmod-style argument, and break it down into one or more AND/OR
* pairs in a linked list. We return a pointer to new items on success
* (appending the items to the specified list), or NULL on error. */
@@ -64,13 +50,13 @@ struct chmod_mode_struct *parse_chmod(const char *modestr,
struct chmod_mode_struct **root_mode_ptr)
{
int state = STATE_1ST_HALF;
int where = 0, what = 0, op = 0, topbits = 0, topoct = 0, flags = 0, copybits = 0;
int where = 0, what = 0, op = 0, topbits = 0, topoct = 0, flags = 0;
struct chmod_mode_struct *first_mode = NULL, *curr_mode = NULL,
*prev_mode = NULL;
while (state != STATE_ERROR) {
if (!*modestr || *modestr == ',') {
int bits, where_specified;
int bits;
if (!op) {
state = STATE_ERROR;
@@ -84,10 +70,9 @@ struct chmod_mode_struct *parse_chmod(const char *modestr,
first_mode = curr_mode;
curr_mode->next = NULL;
where_specified = where;
if (where) {
if (where)
bits = where * what;
} else {
else {
where = 0111;
bits = (where * what) & ~orig_umask;
}
@@ -96,35 +81,18 @@ struct chmod_mode_struct *parse_chmod(const char *modestr,
case CHMOD_ADD:
curr_mode->ModeAND = CHMOD_BITS;
curr_mode->ModeOR = bits + topoct;
curr_mode->ModeCOPY_SRC = copybits;
curr_mode->ModeCOPY_DST = where;
curr_mode->ModeCOPY_AND = where_specified ? CHMOD_BITS : ~orig_umask;
curr_mode->ModeOP = op;
break;
case CHMOD_SUB:
curr_mode->ModeAND = CHMOD_BITS - bits - topoct;
curr_mode->ModeOR = 0;
curr_mode->ModeCOPY_SRC = copybits;
curr_mode->ModeCOPY_DST = where;
curr_mode->ModeCOPY_AND = where_specified ? CHMOD_BITS : ~orig_umask;
curr_mode->ModeOP = op;
break;
case CHMOD_EQ:
curr_mode->ModeAND = CHMOD_BITS - (where * 7) - (topoct ? topbits : 0)
- (copybits ? mode_dest_special_bits(where) : 0);
curr_mode->ModeAND = CHMOD_BITS - (where * 7) - (topoct ? topbits : 0);
curr_mode->ModeOR = bits + topoct;
curr_mode->ModeCOPY_SRC = copybits;
curr_mode->ModeCOPY_DST = where;
curr_mode->ModeCOPY_AND = where_specified ? CHMOD_BITS : ~orig_umask;
curr_mode->ModeOP = op;
break;
case CHMOD_SET:
curr_mode->ModeAND = 0;
curr_mode->ModeOR = bits;
curr_mode->ModeCOPY_SRC = 0;
curr_mode->ModeCOPY_DST = 0;
curr_mode->ModeCOPY_AND = CHMOD_BITS;
curr_mode->ModeOP = op;
break;
}
@@ -135,7 +103,7 @@ struct chmod_mode_struct *parse_chmod(const char *modestr,
modestr++;
state = STATE_1ST_HALF;
where = what = op = topoct = topbits = flags = copybits = 0;
where = what = op = topoct = topbits = flags = 0;
}
switch (state) {
@@ -191,53 +159,26 @@ struct chmod_mode_struct *parse_chmod(const char *modestr,
case STATE_2ND_HALF:
switch (*modestr) {
case 'r':
if (copybits)
state = STATE_ERROR;
what |= 4;
break;
case 'w':
if (copybits)
state = STATE_ERROR;
what |= 2;
break;
case 'X':
if (copybits)
state = STATE_ERROR;
flags |= FLAG_X_KEEP;
/* FALL THROUGH */
case 'x':
if (copybits)
state = STATE_ERROR;
what |= 1;
break;
case 's':
if (copybits)
state = STATE_ERROR;
if (topbits)
topoct |= topbits;
else
topoct = 04000;
break;
case 't':
if (copybits)
state = STATE_ERROR;
topoct |= 01000;
break;
case 'u':
if (what || topoct || copybits)
state = STATE_ERROR;
copybits = 0100;
break;
case 'g':
if (what || topoct || copybits)
state = STATE_ERROR;
copybits = 0010;
break;
case 'o':
if (what || topoct || copybits)
state = STATE_ERROR;
copybits = 0001;
break;
default:
state = STATE_ERROR;
break;
@@ -271,20 +212,6 @@ struct chmod_mode_struct *parse_chmod(const char *modestr,
return first_mode;
}
static int mode_copy_bits(int mode, int copy_src, int copy_dst, int copy_and)
{
int copy_bits = 0;
if (copy_src & 0100)
copy_bits |= (mode >> 6) & 7;
if (copy_src & 0010)
copy_bits |= (mode >> 3) & 7;
if (copy_src & 0001)
copy_bits |= mode & 7;
return (copy_dst * copy_bits) & copy_and;
}
/* Takes an existing file permission and a list of AND/OR changes, and
* create a new permissions. */
@@ -292,25 +219,17 @@ int tweak_mode(int mode, struct chmod_mode_struct *chmod_modes)
{
int IsX = mode & 0111;
int NonPerm = mode & ~CHMOD_BITS;
int copy_bits;
for ( ; chmod_modes; chmod_modes = chmod_modes->next) {
if ((chmod_modes->flags & FLAG_DIRS_ONLY) && !S_ISDIR(NonPerm))
continue;
if ((chmod_modes->flags & FLAG_FILES_ONLY) && S_ISDIR(NonPerm))
continue;
copy_bits = mode_copy_bits(mode, chmod_modes->ModeCOPY_SRC,
chmod_modes->ModeCOPY_DST,
chmod_modes->ModeCOPY_AND);
mode &= chmod_modes->ModeAND;
if ((chmod_modes->flags & FLAG_X_KEEP) && !IsX && !S_ISDIR(NonPerm))
mode |= chmod_modes->ModeOR & ~0111;
else
mode |= chmod_modes->ModeOR;
if (chmod_modes->ModeOP == CHMOD_SUB)
mode &= CHMOD_BITS - copy_bits;
else
mode |= copy_bits;
}
return mode | NonPerm;

View File

@@ -1070,7 +1070,7 @@ static int rsync_module(int f_in, int f_out, int i, const char *addr, const char
io_printf(f_out, "@RSYNCD: OK\n");
read_args(f_in, name, line, sizeof line, rl_nulls, 1, &argv, &argc, &request);
read_args(f_in, name, line, sizeof line, rl_nulls, &argv, &argc, &request);
orig_argv = argv;
save_munge_symlinks = munge_symlinks;
@@ -1080,7 +1080,7 @@ static int rsync_module(int f_in, int f_out, int i, const char *addr, const char
if (protect_args && ret) {
orig_early_argv = orig_argv;
protect_args = 2;
read_args(f_in, name, line, sizeof line, 1, 0, &argv, &argc, &request);
read_args(f_in, name, line, sizeof line, 1, &argv, &argc, &request);
orig_argv = argv;
ret = parse_arguments(&argc, (const char ***) &argv);
} else

View File

@@ -103,6 +103,10 @@ dnl (and coverage-counted) without needing a pre-5.6 kernel. Behaviour-neutral
dnl by default (the knob only REMOVES a tier when explicitly disabled).
AC_ARG_ENABLE(openat2,
AS_HELP_STRING([--disable-openat2],[do not use Linux openat2(RESOLVE_BENEATH); force the portable resolver (for exercising the fallback tier)]))
if test x"$enable_openat2" != x"no"; then
AC_DEFINE([HAVE_OPENAT2], 1,
[Define to use Linux openat2(RESOLVE_BENEATH) in secure_relative_open where available.])
fi
AC_MSG_CHECKING([if md2man can create manpages])
if test x"$ac_cv_path_PYTHON3" = x; then
@@ -353,28 +357,6 @@ AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[ ]], [[return 0;]])],
CFLAGS="$OLD_CFLAGS"
AC_SUBST(NOEXECSTACK)
dnl We need both the SYS_openat2 syscall number and <linux/openat2.h> (for
dnl struct open_how / RESOLVE_BENEATH); some setups have one without the other.
AC_CACHE_CHECK([for openat2],rsync_cv_HAVE_OPENAT2,[
AC_COMPILE_IFELSE([
AC_LANG_PROGRAM([[
#include <sys/syscall.h>
#include <linux/openat2.h>
]], [[
struct open_how how;
how.resolve = RESOLVE_BENEATH;
return SYS_openat2 + (int)how.resolve;
]])
],
[rsync_cv_HAVE_OPENAT2=yes], [rsync_cv_HAVE_OPENAT2=no])
])
if test x"$enable_openat2" != x"no"; then
if test x"$rsync_cv_HAVE_OPENAT2" = x"yes"; then
AC_DEFINE([HAVE_OPENAT2], 1,
[Define to use Linux openat2(RESOLVE_BENEATH) in secure_relative_open where available.])
fi
fi
# arrgh. libc in some old debian version screwed up the largefile
# stuff, getting byte range locking wrong
AC_CACHE_CHECK([for broken largefile support],rsync_cv_HAVE_BROKEN_LARGEFILE,[
@@ -432,17 +414,21 @@ AS_HELP_STRING([--disable-ipv6],[disable to omit ipv6 support]),
;;
esac ],
AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[
AC_RUN_IFELSE([AC_LANG_SOURCE([[ /* AF_INET6 availability check */
#include <stdlib.h>
#include <sys/types.h>
#include <sys/socket.h>
#include <netinet/in.h>
]], [[
struct sockaddr_in6 sa6;
(void)sa6;
(void)AF_INET6;
int main()
{
if (socket(AF_INET6, SOCK_STREAM, 0) < 0)
exit(1);
else
exit(0);
}
]])],
[AC_MSG_RESULT(yes)
AC_DEFINE(INET6, 1, [true if you have IPv6])],
AC_DEFINE(INET6, 1, true if you have IPv6)],
[AC_MSG_RESULT(no)],
[AC_MSG_RESULT(no)]
))
@@ -928,7 +914,7 @@ AC_FUNC_UTIME_NULL
AC_FUNC_ALLOCA
AC_CHECK_FUNCS(waitpid wait4 getcwd chown chmod lchmod mknod mkfifo \
fchmod fstat ftruncate strchr readlink link utime utimes lutimes strftime \
chflags getattrlist mktime innetgr linkat mknodat mkfifoat \
chflags getattrlist mktime innetgr linkat \
memmove lchown vsnprintf snprintf vasprintf asprintf setsid strpbrk \
strlcat strlcpy stpcpy strtol mallinfo mallinfo2 getgroups setgroups geteuid getegid \
setlocale setmode open64 lseek64 mkstemp64 mtrace va_copy __va_copy \

20
doc/README-SGML Normal file
View File

@@ -0,0 +1,20 @@
Handling the rsync SGML documentation
rsync documentation is now primarily in Docbook format. Docbook is an
SGML/XML documentation format that is becoming standard on free
operating systems. It's also used for Samba documentation.
The SGML files are source code that can be translated into various
useful output formats, primarily PDF, HTML, Postscript and plain text.
To do this transformation on Debian, you should install the
docbook-utils package. Having done that, you can say
docbook2pdf rsync.sgml
and so on.
On other systems you probably need James Clark's "sp" and "JadeTeX"
packages. Work it out for yourself and send a note to the mailing
list.

42
doc/profile.txt Normal file
View File

@@ -0,0 +1,42 @@
Notes on rsync profiling
strlcpy is hot:
0.00 0.00 1/7735635 push_dir [68]
0.00 0.00 1/7735635 pop_dir [71]
0.00 0.00 1/7735635 send_file_list [15]
0.01 0.00 18857/7735635 send_files [4]
0.04 0.00 129260/7735635 send_file_entry [18]
0.04 0.00 129260/7735635 make_file [20]
0.04 0.00 141666/7735635 send_directory <cycle 1> [36]
2.29 0.00 7316589/7735635 f_name [13]
[14] 11.7 2.42 0.00 7735635 strlcpy [14]
Here's the top few functions:
46.23 9.57 9.57 13160929 0.00 0.00 mdfour64
14.78 12.63 3.06 13160929 0.00 0.00 copy64
11.69 15.05 2.42 7735635 0.00 0.00 strlcpy
10.05 17.13 2.08 41438 0.05 0.38 sum_update
4.11 17.98 0.85 13159996 0.00 0.00 mdfour_update
1.50 18.29 0.31 file_compare
1.45 18.59 0.30 129261 0.00 0.01 send_file_entry
1.23 18.84 0.26 2557585 0.00 0.00 f_name
1.11 19.07 0.23 1483750 0.00 0.00 u_strcmp
1.11 19.30 0.23 118129 0.00 0.00 writefd_unbuffered
0.92 19.50 0.19 1085011 0.00 0.00 writefd
0.43 19.59 0.09 156987 0.00 0.00 read_timeout
0.43 19.68 0.09 129261 0.00 0.00 clean_fname
0.39 19.75 0.08 32887 0.00 0.38 matched
0.34 19.82 0.07 1 70.00 16293.92 send_files
0.29 19.89 0.06 129260 0.00 0.00 make_file
0.29 19.95 0.06 75430 0.00 0.00 read_unbuffered
mdfour could perhaps be made faster:
/* NOTE: This code makes no attempt to be fast! */
There might be an optimized version somewhere that we can borrow.

351
doc/rsync.sgml Normal file
View File

@@ -0,0 +1,351 @@
<!DOCTYPE book PUBLIC "-//OASIS//DTD DocBook V4.1//EN">
<book id="rsync">
<bookinfo>
<title>rsync</title>
<copyright>
<year>1996 -- 2002</year>
<holder>Martin Pool</holder>
<holder>Andrew Tridgell</holder>
</copyright>
<author>
<firstname>Martin</firstname>
<surname>Pool</surname>
</author>
</bookinfo>
<chapter>
<title>Introduction</title>
<para>rsync is a flexible program for efficiently copying files or
directory trees.
<para>rsync has many options to select which files will be copied
and how they are to be transferred. It may be used as an
alternative to ftp, http, scp or rcp.
<para>The rsync remote-update protocol allows rsync to transfer just
the differences between two sets of files across the network link,
using an efficient checksum-search algorithm described in the
technical report that accompanies this package.</para>
<para>Some of the additional features of rsync are:</para>
<itemizedlist>
<listitem>
<para>support for copying links, devices, owners, groups and
permissions
</para>
</listitem>
<listitem>
<para>
exclude and exclude-from options similar to GNU tar
</para>
</listitem>
<listitem>
<para>
a CVS exclude mode for ignoring the same files that CVS would ignore
</listitem>
<listitem>
<para>
can use any transparent remote shell, including rsh or ssh
</listitem>
<listitem>
<para>
does not require root privileges
</listitem>
<listitem>
<para>
pipelining of file transfers to minimize latency costs
</listitem>
<listitem>
<para>
support for anonymous or authenticated rsync servers (ideal for
mirroring)
</para>
</listitem>
</itemizedlist>
</chapter>
<chapter>
<title>Using rsync</title>
<section>
<title>
Introductory example
</title>
<para>
Probably the most common case of rsync usage is to copy files
to or from a remote machine using
<application>ssh</application> as a network transport. In
this situation rsync is a good alternative to
<application>scp</application>.
</para>
<para>
The most commonly used arguments for rsync are
</para>
<variablelist>
<varlistentry>
<term><option>-v</option></term>
<listitem>
<para>Be verbose. Primarily, display the name of each file as it is copied.</para>
</listitem>
</varlistentry>
<varlistentry>
<term><option>-a</option></term>
<listitem>
<para>
Reproduce the structure and attributes of the origin files as exactly
as possible: this includes copying subdirectories, symlinks, special
files, ownership and permissions. (@xref{Attributes to
copy}.)
</para>
</listitem>
</varlistentry>
</variablelist>
<para><option>-v </option>
<para><option>-z</option>
Compress network traffic, using a modified version of the
@command{zlib} library.</para>
<para><option>-P</option>
Display a progress indicator while files are transferred. This should
normally be omitted if rsync is not run on a terminal.
</para>
</section>
<section>
<title>Local and remote</title>
<para>There are six different ways of using rsync. They
are:</para>
<!-- one of (CALLOUTLIST GLOSSLIST ITEMIZEDLIST ORDEREDLIST SEGMENTEDLIST SIMPLELIST VARIABLELIST CAUTION IMPORTANT NOTE TIP WARNING LITERALLAYOUT PROGRAMLISTING PROGRAMLISTINGCO SCREEN SCREENCO SCREENSHOT SYNOPSIS CMDSYNOPSIS FUNCSYNOPSIS CLASSSYNOPSIS FIELDSYNOPSIS CONSTRUCTORSYNOPSIS DESTRUCTORSYNOPSIS METHODSYNOPSIS FORMALPARA PARA SIMPARA ADDRESS BLOCKQUOTE GRAPHIC GRAPHICCO MEDIAOBJECT MEDIAOBJECTCO INFORMALEQUATION INFORMALEXAMPLE INFORMALFIGURE INFORMALTABLE EQUATION EXAMPLE FIGURE TABLE MSGSET PROCEDURE SIDEBAR QANDASET ANCHOR BRIDGEHEAD REMARK HIGHLIGHTS ABSTRACT AUTHORBLURB EPIGRAPH INDEXTERM REFENTRY SECTION) -->
<orderedlist>
<listitem>
<para>
for copying local files. This is invoked when neither
source nor destination path contains a @code{:} separator
<listitem>
<para>
for copying from the local machine to a remote machine using
a remote shell program as the transport (such as rsh or
ssh). This is invoked when the destination path contains a
single @code{:} separator.
<listitem>
<para>
for copying from a remote machine to the local machine
using a remote shell program. This is invoked when the source
contains a @code{:} separator.
<listitem>
<para>
for copying from a remote rsync server to the local
machine. This is invoked when the source path contains a @code{::}
separator or a @code{rsync://} URL.
<listitem>
<para>
for copying from the local machine to a remote rsync
server. This is invoked when the destination path contains a @code{::}
separator.
<listitem>
<para>
for listing files on a remote machine. This is done the
same way as rsync transfers except that you leave off the
local destination.
</listitem>
</orderedlist>
<para>
Note that in all cases (other than listing) at least one of the source
and destination paths must be local.
<para>
Any one invocation of rsync makes a copy in a single direction. rsync
currently has no equivalent of @command{ftp}'s interactive mode.
@cindex @sc{nfs}
@cindex network filesystems
@cindex remote filesystems
<para>
rsync's network protocol is generally faster at copying files than
network filesystems such as @sc{nfs} or @sc{cifs}. It is better to
run rsync on the file server either as a daemon or over ssh than
running rsync giving the network directory.
</para>
</section>
</chapter>
<chapter>
<title>Frequently asked questions</title>
<!-- one of (CALLOUTLIST GLOSSLIST ITEMIZEDLIST ORDEREDLIST SEGMENTEDLIST SIMPLELIST VARIABLELIST CAUTION IMPORTANT NOTE TIP WARNING LITERALLAYOUT PROGRAMLISTING PROGRAMLISTINGCO SCREEN SCREENCO SCREENSHOT SYNOPSIS CMDSYNOPSIS FUNCSYNOPSIS CLASSSYNOPSIS FIELDSYNOPSIS CONSTRUCTORSYNOPSIS DESTRUCTORSYNOPSIS METHODSYNOPSIS FORMALPARA PARA SIMPARA ADDRESS BLOCKQUOTE GRAPHIC GRAPHICCO MEDIAOBJECT MEDIAOBJECTCO INFORMALEQUATION INFORMALEXAMPLE INFORMALFIGURE INFORMALTABLE EQUATION EXAMPLE FIGURE TABLE MSGSET PROCEDURE SIDEBAR QANDASET ANCHOR BRIDGEHEAD REMARK HIGHLIGHTS ABSTRACT AUTHORBLURB EPIGRAPH INDEXTERM SECTION SIMPLESECT REFENTRY SECT1) -->
<qandaset>
<!-- one of (QANDADIV QANDAENTRY) -->
<qandaentry>
<question>
<!-- one of (CALLOUTLIST GLOSSLIST ITEMIZEDLIST ORDEREDLIST
SEGMENTEDLIST SIMPLELIST VARIABLELIST CAUTION IMPORTANT NOTE
TIP WARNING LITERALLAYOUT PROGRAMLISTING PROGRAMLISTINGCO
SCREEN SCREENCO SCREENSHOT SYNOPSIS CMDSYNOPSIS FUNCSYNOPSIS
CLASSSYNOPSIS FIELDSYNOPSIS CONSTRUCTORSYNOPSIS
DESTRUCTORSYNOPSIS METHODSYNOPSIS FORMALPARA PARA SIMPARA
ADDRESS BLOCKQUOTE GRAPHIC GRAPHICCO MEDIAOBJECT
MEDIAOBJECTCO INFORMALEQUATION INFORMALEXAMPLE
INFORMALFIGURE INFORMALTABLE EQUATION EXAMPLE FIGURE TABLE
PROCEDURE ANCHOR BRIDGEHEAD REMARK HIGHLIGHTS INDEXTERM) -->
<para>Are there mailing lists for rsync?
</question>
<answer>
<para>Yes, and you can subscribe and unsubscribe through a
web interface at
<ulink
url="http://lists.samba.org/">http://lists.samba.org/</ulink>
</para>
<para>
If you are having trouble with the mailing list, please
send mail to the administrator
<email>rsync-admin@lists.samba.org</email>
not to the list itself.
</para>
<para>
The mailing list archives are searchable. Use
<ulink url="http://google.com/">Google</ulink> and prepend
the search with <userinput>site:lists.samba.org
rsync</userinput>, plus relevant keywords.
</para>
</answer>
</qandaentry>
<qandaentry>
<question>
<para>
Why is rsync so much bigger when I build it with
<command>gcc</command>?
</para>
</question>
<answer>
<para>
On gcc, rsync builds by default with debug symbols
included. If you strip both executables, they should end
up about the same size. (Use <command>make
install-strip</command>.)
</para>
</answer>
</qandaentry>
<qandaentry>
<question>
<para>Is rsync useful for a single large file like an ISO image?</para>
</question>
<answer>
<para>
Yes, but note the following:
<para>
Background: A common use of rsync is to update a file (or set of files) in one location from a more
correct or up-to-date copy in another location, taking advantage of portions of the files that are
identical to speed up the process. (Note that rsync will transfer a file in its entirety if no copy
exists at the destination.)
<para>
(This discussion is written in terms of updating a local copy of a file from a correct file in a
remote location, although rsync can work in either direction.)
<para>
The file to be updated (the local file) must be in a destination directory that has enough space for
two copies of the file. (In addition, keep an extra copy of the file to be updated in a different
location for safety -- see the discussion (below) about rsync's behavior when the rsync process is
interrupted before completion.)
<para>
The local file must have the same name as the remote file being sync'd to (I think?). If you are
trying to upgrade an iso from, for example, beta1 to beta2, rename the local file to the same name
as the beta2 file. *(This is a useful thing to do -- only the changed portions will be
transmitted.)*
<para>
The extra copy of the local file kept in a different location is because of rsync's behavior if
interrupted before completion:
<para>
* If you specify the --partial option and rsync is interrupted, rsync will save the partially
rsync'd file and throw away the original local copy. (The partially rsync'd file is correct but
truncated.) If rsync is restarted, it will not have a local copy of the file to check for duplicate
blocks beyond the section of the file that has already been rsync'd, thus the remainder of the rsync
process will be a "pure transfer" of the file rather than taking advantage of the rsync algorithm.
<para>
* If you don't specify the --partial option and rsync is interrupted, rsync will throw away the
partially rsync'd file, and, when rsync is restarted starts the rsync process over from the
beginning.
<para>
Which of these is most desirable depends on the degree of commonality between the local and remote
copies of the file *and how much progress was made before the interruption*.
<para>
The ideal approach after an interruption would be to create a new file by taking the original file
and deleting a portion equal in size to the portion already rsync'd and then appending *the
remaining* portion to the portion of the file that has already been rsync'd. (There has been some
discussion about creating an option to do this automatically.)
The --compare-dest option is useful when transferring multiple files, but is of no benefit in
transferring a single file. (AFAIK)
*Other potentially useful information can be found at:
-[3]http://twiki.org/cgi-bin/view/Wikilearn/RsyncingALargeFile
This answer, formatted with "real" bullets, can be found at:
-[4]http://twiki.org/cgi-bin/view/Wikilearn/RsyncingALargeFileFAQ*
</para>
</answer>
</qandaentry>
</qandaset>
</chapter>
<appendix>
<title>Other Resources</title>
<para><ulink url="http://www.ccp14.ac.uk/ccp14admin/rsync/"></ulink></para>
</appendix>
</book>

54
flist.c
View File

@@ -132,18 +132,6 @@ static int64 tmp_dev = -1, tmp_ino;
#endif
static char tmp_sum[MAX_DIGEST_LEN];
#ifdef ST_MTIME_NSEC
/* Return st_mtim nsec if it is in the wire-valid range, else 0. */
static inline uint32 wire_mtime_nsec_from_stat(const STRUCT_STAT *stp)
{
unsigned long nsec = (unsigned long)stp->ST_MTIME_NSEC;
if (nsec > MAX_WIRE_NSEC)
return 0;
return (uint32)nsec;
}
#endif
static char empty_sum[MAX_DIGEST_LEN];
static int flist_count_offset; /* for --delete --progress */
static int show_filelist_progress;
@@ -877,18 +865,13 @@ static struct file_struct *recv_file_entry(int f, struct file_list *flist, int x
mode = from_wire_mode(read_int(f));
/* Reject modes whose type bits are not one of the standard
* file types; otherwise garbage mode values propagate through
* the file-type checks below unpredictably. mode 0 is the one
* legitimate exception: --delete-missing-args (missing_args==2)
* sends a missing arg as a mode-0 entry (IS_MISSING_FILE), the
* generator's delete signal (#910). */
if (mode != 0 || missing_args != 2) {
if (!S_ISREG(mode) && !S_ISDIR(mode) && !S_ISLNK(mode)
&& !S_ISCHR(mode) && !S_ISBLK(mode)
&& !S_ISFIFO(mode) && !S_ISSOCK(mode)) {
* the file-type checks below unpredictably. */
if (!S_ISREG(mode) && !S_ISDIR(mode) && !S_ISLNK(mode)
&& !S_ISCHR(mode) && !S_ISBLK(mode)
&& !S_ISFIFO(mode) && !S_ISSOCK(mode)) {
rprintf(FERROR, "invalid file mode 0%o for %s [%s]\n",
(unsigned)mode, lastname, who_am_i());
exit_cleanup(RERR_PROTOCOL);
}
}
}
if (atimes_ndx && !S_ISDIR(mode) && !(xflags & XMIT_SAME_ATIME)) {
@@ -1267,7 +1250,7 @@ struct file_struct *make_file(const char *fname, struct file_list *flist,
int extra_len = file_extra_cnt * EXTRA_LEN;
const char *basename;
alloc_pool_t *pool;
STRUCT_STAT st = {0};
STRUCT_STAT st;
char *bp;
if (strlcpy(thisname, fname, sizeof thisname) >= sizeof thisname) {
@@ -1429,12 +1412,8 @@ struct file_struct *make_file(const char *fname, struct file_list *flist,
}
#ifdef ST_MTIME_NSEC
{
uint32 nsec = wire_mtime_nsec_from_stat(&st);
if (nsec && protocol_version >= 31)
extra_len += EXTRA_LEN;
}
if (st.ST_MTIME_NSEC && protocol_version >= 31)
extra_len += EXTRA_LEN;
#endif
#if SIZEOF_CAPITAL_OFF_T >= 8
if (st.st_size > 0xFFFFFFFFu && S_ISREG(st.st_mode))
@@ -1489,13 +1468,9 @@ struct file_struct *make_file(const char *fname, struct file_list *flist,
file->flags = flags;
file->modtime = st.st_mtime;
#ifdef ST_MTIME_NSEC
{
uint32 nsec = wire_mtime_nsec_from_stat(&st);
if (nsec && protocol_version >= 31) {
file->flags |= FLAG_MOD_NSEC;
F_MOD_NSEC(file) = nsec;
}
if (st.ST_MTIME_NSEC && protocol_version >= 31) {
file->flags |= FLAG_MOD_NSEC;
F_MOD_NSEC(file) = st.ST_MTIME_NSEC;
}
#endif
file->len32 = (uint32)st.st_size;
@@ -2095,9 +2070,10 @@ static void send1extra(int f, struct file_struct *file, struct file_list *flist)
}
if (name_type != NORMAL_NAME) {
STRUCT_STAT st = {0};
if (name_type != MISSING_NAME && link_stat(fbuf, &st, 1) != 0) {
STRUCT_STAT st;
if (name_type == MISSING_NAME)
memset(&st, 0, sizeof st);
else if (link_stat(fbuf, &st, 1) != 0) {
interpret_stat_error(fbuf, True);
continue;
}
@@ -2229,7 +2205,7 @@ struct file_list *send_file_list(int f, int argc, char *argv[])
static const char *lastdir;
static int lastdir_len = -1;
int len, dirlen;
STRUCT_STAT st = {0};
STRUCT_STAT st;
char *p, *dir;
struct file_list *flist;
struct timeval start_tv, end_tv;

View File

@@ -66,7 +66,6 @@ extern int inplace;
extern int append_mode;
extern int make_backups;
extern int csum_length;
extern int xfer_sum_len;
extern int ignore_times;
extern int size_only;
extern OFF_T max_size;
@@ -698,11 +697,6 @@ static void sum_sizes_sqroot(struct sum_struct *sum, int64 len)
{
int32 blength;
int s2length;
/* The strong sum can be no longer than the negotiated checksum digest:
* a short checksum (e.g. xxh64 = 8 bytes, when xxh128/xxh3 are absent)
* makes xfer_sum_len < SUM_LENGTH, and the sender rejects an s2length
* larger than xfer_sum_len (io.c). */
int max_s2length = MIN(SUM_LENGTH, xfer_sum_len);
int64 l;
if (len < 0) {
@@ -737,7 +731,7 @@ static void sum_sizes_sqroot(struct sum_struct *sum, int64 len)
if (protocol_version < 27) {
s2length = csum_length;
} else if (csum_length == SUM_LENGTH) {
s2length = max_s2length;
s2length = SUM_LENGTH;
} else {
int32 c;
int b = BLOCKSUM_BIAS;
@@ -746,7 +740,7 @@ static void sum_sizes_sqroot(struct sum_struct *sum, int64 len)
/* add a bit, subtract rollsum, round up. */
s2length = (b + 1 - 32 + 7) / 8; /* --optimize in compiler-- */
s2length = MAX(s2length, csum_length);
s2length = MIN(s2length, max_s2length);
s2length = MIN(s2length, SUM_LENGTH);
}
sum->flength = len;
@@ -1718,8 +1712,7 @@ static void recv_generator(char *fname, struct file_struct *file, int ndx,
goto cleanup;
}
if (update_only > 0 && statret == 0 && stype == ftype
&& file->modtime - sx.st.st_mtime < modify_window) {
if (update_only > 0 && statret == 0 && file->modtime - sx.st.st_mtime < modify_window) {
if (INFO_GTE(SKIP, 1))
rprintf(FINFO, "%s is newer\n", fname);
#ifdef SUPPORT_HARD_LINKS
@@ -2391,7 +2384,7 @@ void generate_files(int f_out, const char *local_name)
write_ndx(f_out, NDX_DONE);
if (protocol_version >= 31 && EARLY_DELETE_DONE_MSG()) {
if (delete_mode || force_delete || read_batch)
if ((INFO_GTE(STATS, 2) && (delete_mode || force_delete)) || read_batch)
write_del_stats(f_out);
if (EARLY_DELAY_DONE_MSG()) /* Can't send this before delay */
write_ndx(f_out, NDX_DONE);
@@ -2436,7 +2429,7 @@ void generate_files(int f_out, const char *local_name)
if (protocol_version >= 31) {
if (!EARLY_DELETE_DONE_MSG()) {
if (delete_mode || force_delete || read_batch)
if (INFO_GTE(STATS, 2) || read_batch)
write_del_stats(f_out);
write_ndx(f_out, NDX_DONE);
}

20
io.c
View File

@@ -1292,21 +1292,8 @@ int read_line(int fd, char *buf, size_t bufsiz, int flags)
return s - buf;
}
/* Reverse safe_arg()'s backslash escaping of a daemon option arg, the way a
* remote shell un-escapes args for the ssh transport. In place; \X -> X. */
static void unbackslash_arg(char *s)
{
char *f = s, *t = s;
while (*f) {
if (*f == '\\' && f[1])
f++;
*t++ = *f++;
}
*t = '\0';
}
void read_args(int f_in, char *mod_name, char *buf, size_t bufsiz, int rl_nulls,
int unescape, char ***argv_p, int *argc_p, char **request_p)
char ***argv_p, int *argc_p, char **request_p)
{
int maxargs = MAX_ARGS;
int dot_pos = 0, argc = 0, request_len = 0;
@@ -1348,11 +1335,6 @@ void read_args(int f_in, char *mod_name, char *buf, size_t bufsiz, int rl_nulls,
glob_expand(buf, &argv, &argc, &maxargs);
} else {
p = strdup(buf);
/* An option arg the client escaped with safe_arg() (no
* remote shell un-escapes it for a daemon). File args
* after the dot are handled by glob_expand() below. */
if (unescape)
unbackslash_arg(p);
argv[argc++] = p;
if (*p == '.' && p[1] == '\0')
dot_pos = argc;

13
main.c
View File

@@ -832,16 +832,7 @@ static char *get_local_name(struct file_list *flist, char *dest_path)
dest_path = "/";
*cp = '\0';
if (dry_run && mkpath_dest_arg && do_stat(dest_path, &st) < 0) {
/* --mkpath would have created this parent dir, but a dry run did
* not, so don't chdir into it; flag the destination as not yet
* present (as the dir-creation path above does) so the generator
* doesn't try to compare against the missing tree (#880). Only
* the missing-parent case is touched, so an ordinary file-to-file
* dry run still itemizes against an existing destination. */
dry_run++;
change_dir(dest_path, CD_SKIP_CHDIR);
} else if (!change_dir(dest_path, CD_NORMAL)) {
if (!change_dir(dest_path, CD_NORMAL)) {
rsyserr(FERROR, errno, "change_dir#3 %s failed",
full_fname(dest_path));
exit_cleanup(RERR_FILESELECT);
@@ -1854,7 +1845,7 @@ int main(int argc,char *argv[])
if (am_server && protect_args) {
char buf[MAXPATHLEN];
protect_args = 2;
read_args(STDIN_FILENO, NULL, buf, sizeof buf, 1, 0, &argv, &argc, NULL);
read_args(STDIN_FILENO, NULL, buf, sizeof buf, 1, &argv, &argc, NULL);
if (!parse_arguments(&argc, (const char ***) &argv)) {
option_error();
exit_cleanup(RERR_SYNTAX);

View File

@@ -15,7 +15,7 @@ if [ ! -f "$flagfile" ]; then
if "$srcdir/md-convert" --test "$srcdir/rsync-ssl.1.md" >/dev/null 2>&1; then
touch $flagfile
else
outname=`basename "$inname" .md`
outname=`echo "$inname" | sed 's/\.md$//'`
if [ -f "$outname" ]; then
exit 0
elif [ -f "$srcdir/$outname" ]; then

View File

@@ -99,27 +99,6 @@ static int updating_basis_or_equiv;
* Anything else is a straight pass-through that preserves the strict contract. */
static int secure_basis_open(const char *basedir, const char *relpath, int flags, mode_t mode)
{
extern int am_daemon, am_chrooted;
/* The confined resolver is only needed for the sanitizing daemon
* (am_daemon && !am_chrooted, i.e. use_secure_symlinks). Local /
* remote-shell mode has no module boundary, and "use chroot = yes" makes
* the kernel root the boundary, so there an alt-dest basis like
* --link-dest=../01 must resolve against the cwd as a bare open did before
* the hardening (confining it would reject the legitimate sibling "..",
* #915). */
if (!am_daemon || am_chrooted) {
if (basedir) {
char fullpath[MAXPATHLEN];
if (pathjoin(fullpath, sizeof fullpath, basedir, relpath) >= sizeof fullpath) {
errno = ENAMETOOLONG;
return -1;
}
return do_open(fullpath, flags, mode);
}
return do_open(relpath, flags, mode);
}
if (!basedir && relpath && *relpath == '/') {
const char *slash = strrchr(relpath, '/');
const char *leaf = slash + 1;
@@ -880,7 +859,7 @@ int recv_files(int f_in, int f_out, char *local_name)
basedir = basis_dir[0];
fnamecmp = fname;
fnamecmp_type = FNAMECMP_BASIS_DIR_LOW;
fd1 = secure_basis_open(basedir, fnamecmp, O_RDONLY, 0);
fd1 = secure_relative_open(basedir, fnamecmp, O_RDONLY, 0);
}
}
@@ -963,40 +942,11 @@ int recv_files(int f_in, int f_out, char *local_name)
if (fd2 == -1 && errno == EACCES) {
/* Maybe the error was due to protected_regular setting? */
if (use_secure_symlinks)
fd2 = secure_relative_open(NULL, fnametmp, O_WRONLY, 0600);
fd2 = secure_relative_open(NULL, fname, O_WRONLY, 0600);
else
fd2 = do_open(fnametmp, O_WRONLY, 0600);
fd2 = do_open(fname, O_WRONLY, 0600);
}
#endif
if (fd2 == -1 && errno == EACCES) {
/* A read-only existing file: make it writable, then retry
* (its mode is restored after the transfer). On a
* non-chroot daemon fchmod() a no-follow fd rather than
* chmod the path, so a symlink raced into fnametmp can't
* redirect the chmod (do_chmod_at follows the final link). */
int errno_save = errno, chmod_ok;
if (use_secure_symlinks) {
#ifdef O_NOFOLLOW
int cfd = secure_relative_open(NULL, fnametmp, O_RDONLY|O_NOFOLLOW, 0);
chmod_ok = cfd != -1 && fchmod(cfd, 0600) == 0;
if (cfd != -1)
close(cfd);
#else
/* Without O_NOFOLLOW the resolver's oldest fallback would
* follow a raced symlink, so fail closed rather than
* chmod through it. */
chmod_ok = 0;
#endif
} else
chmod_ok = do_chmod_at(fnametmp, 0600) == 0;
if (chmod_ok) {
if (use_secure_symlinks)
fd2 = secure_relative_open(NULL, fnametmp, O_WRONLY, 0600);
else
fd2 = do_open(fnametmp, O_WRONLY, 0600);
} else
errno = errno_save;
}
if (fd2 == -1) {
rsyserr(FERROR_XFER, errno, "open %s failed",
full_fname(fnametmp));

View File

@@ -1513,16 +1513,6 @@ expand it.
> --chmod=D2775,F664
Symbolic permission-copy modes are also allowed, such as `g=u`, `o=g` or
`g-o`. A permission-copy item may copy from one class only (`u`, `g` or
`o`) and cannot be combined with `rwxXst` permission letters in the same
item. Use comma-separated items when you need both behaviours, such as
`g=o,o=`.
A permission-copy `=` item also clears the special bit for each destination
class it updates (`u` clears setuid, `g` clears setgid, and `o` clears
sticky), matching GNU **chmod** behaviour.
It is also legal to specify multiple `--chmod` options, as each additional
option is just appended to the list of changes to make.
@@ -2400,9 +2390,7 @@ expand it.
The filenames that are read from the FILE are all relative to the source
dir -- any leading slashes are removed and no ".." references are allowed
to go higher than the source dir. Blank entries are ignored, as are
whole-entry comments that start with '`;`' or '`#`'. For example, take
this command:
to go higher than the source dir. For example, take this command:
> rsync -a --files-from=/tmp/foo /usr remote:/backup
@@ -3028,10 +3016,6 @@ expand it.
> --usermap=:nobody --groupmap=*:nobody
An empty **FROM** value matches only sender-side IDs that have no name. It
is not a wildcard for named users or groups; use "`*`" when you want to map
every sender-side name.
When the [`--numeric-ids`](#opt) option is used, the sender does not send any
names, so all the IDs are treated as having an empty name. This means that
you will need to specify numeric **FROM** values if you want to map these
@@ -3717,9 +3701,9 @@ expand it.
also the [`--only-write-batch`](#opt) option.
This option overrides the negotiated checksum & compress lists and always
negotiates a choice based on old-school md5/md4/zlib choices. This means
batch mode is not compatible with newer compression choices such as zstd or
lz4.
negotiates a choice based on old-school md5/md4/zlib choices. If you want
a more modern choice, use the [`--checksum-choice`](#opt) (`--cc`) and/or
[`--compress-choice`](#opt) (`--zc`) options.
0. `--only-write-batch=FILE`

467
rsync3.txt Normal file
View File

@@ -0,0 +1,467 @@
-*- indented-text -*-
Notes towards a new version of rsync
Martin Pool <mbp@samba.org>, September 2001.
Good things about the current implementation:
- Widely known and adopted.
- Fast/efficient, especially for moderately small sets of files over
slow links (transoceanic or modem.)
- Fairly reliable.
- The choice of running over a plain TCP socket or tunneling over
ssh.
- rsync operations are idempotent: you can always run the same
command twice to make sure it worked properly without any fear.
(Are there any exceptions?)
- Small changes to files cause small deltas.
- There is a way to evolve the protocol to some extent.
- rdiff and rsync --write-batch allow generation of standalone patch
sets. rsync+ is pretty cheesy, though. xdelta seems cleaner.
- Process triangle is creative, but seems to provoke OS bugs.
- "Morning-after property": you don't need to know anything on the
local machine about the state of the remote machine, or about
transfers that have been done in the past.
- You can easily push or pull simply by switching the order of
files.
- The "modules" system has some neat features compared to
e.g. Apache's per-directory configuration. In particular, because
you can set a userid and chroot directory, there is strong
protection between different modules. I haven't seen any calls
for a more flexible system.
Bad things about the current implementation:
- Persistent and hard-to-diagnose hang bugs remain
- Protocol is sketchily documented, tied to this implementation, and
hard to modify/extend
- Both the program and the protocol assume a single non-interactive
one-way transfer
- A list of all files are held in memory for the entire transfer,
which cripples scalability to large file trees
- Opening a new socket for every operation causes problems,
especially when running over SSH with password authentication.
- Renamed files are not handled: the old file is removed, and the
new file created from scratch.
- The versioning approach assumes that future versions of the
program know about all previous versions, and will do the right
thing.
- People always get confused about ':' vs '::'
- Error messages can be cryptic.
- Default behaviour is not intuitive: in too many cases rsync will
happily do nothing. Perhaps -a should be the default?
- People get confused by trailing slashes, though it's hard to think
of another reasonable way to make this necessary distinction
between a directory and its contents.
Protocol philosophy:
*The* big difference between protocols like HTTP, FTP, and NFS is
that their fundamental operations are "read this file", "delete
this file", and "make this directory", whereas rsync is "make this
directory like this one".
Questionable features:
These are neat, but not necessarily clean or worth preserving.
- The remote rsync can be wrapped by some other program, such as in
tridge's rsync-mail scripts. The general feature of sending and
retrieving mail over rsync is good, but this is perhaps not the
right way to implement it.
Desirable features:
These don't really require architectural changes; they're just
something to keep in mind.
- Synchronize ACLs and extended attributes
- Anonymous servers should be efficient
- Code should be portable to non-UNIX systems
- Should be possible to document the protocol in RFC form
- --dry-run option
- IPv6 support. Pretty straightforward.
- Allow the basis and destination files to be different. For
example, you could use this when you have a CD-ROM and want to
download an updated image onto a hard drive.
- Efficiently interrupt and restart a transfer. We can write a
checkpoint file that says where we're up to in the filesystem.
Alternatively, as long as transfers are idempotent, we can just
restart the whole thing. [NFSv4]
- Scripting support.
- Propagate atimes and do not modify them. This is very ugly on
Unix. It might be better to try to add O_NOATIME to kernels, and
call that.
- Unicode. Probably just use UTF-8 for everything.
- Open authentication system. Can we use PAM? Is SASL an adequate
mapping of PAM to the network, or useful in some other way?
- Resume interrupted transfers without the --partial flag. We need
to leave the temporary file behind, and then know to use it. This
leaves a risk of large temporary files accumulating, which is not
good. Perhaps it should be off by default.
- tcpwrappers support. Should be trivial; can already be done
through tcpd or inetd.
- Socks support built in. It's not clear this is any better than
just linking against the socks library, though.
- When run over SSH, invoke with predictable command-line arguments,
so that people can restrict what commands sshd will run. (Is this
really required?)
- Comparison mode: give a list of which files are new, gone, or
different. Set return code depending on whether anything has
changed.
- Internationalized messages (gettext?)
- Optionally use real regexps rather than globs?
- Show overall progress. Pretty hard to do, especially if we insist
on not scanning the directory tree up front.
Regression testing:
- Support automatic testing.
- Have hard internal timeouts against hangs.
- Be deterministic.
- Measure performance.
Hard links:
At the moment, we can recreate hard links, but it's a bit
inefficient: it depends on holding a list of all files in the tree.
Every time we see a file with a linkcount >1, we need to search for
another known name that has the same (fsid,inum) tuple. We could do
that more efficiently by keeping a list of only files with
linkcount>1, and removing files from that list as all their names
become known.
Command-line options:
We have rather a lot at the moment. We might get more if the tool
becomes more flexible. Do we need a .rc or configuration file?
That wouldn't really fit with its pattern of use: cp and tar don't
have them, though ssh does.
Scripting issues:
- Perhaps support multiple scripting languages: candidates include
Perl, Python, Tcl, Scheme (guile?), sh, ...
- Simply running a subprocess and looking at its stdout/exit code
might be sufficient, though it could also be pretty slow if it's
called often.
- There are security issues about running remote code, at least if
it's not running in the users own account. So we can either
disallow it, or use some kind of sandbox system.
- Python is a good language, but the syntax is not so good for
giving small fragments on the command line.
- Tcl is broken Lisp.
- Lots of sysadmins know Perl, though Perl can give some bizarre or
confusing errors. The built in stat operators and regexps might
be useful.
- Sadly probably not enough people know Scheme.
- sh is hard to embed.
Scripting hooks:
- Whether to transfer a file
- What basis file to use
- Logging
- Whether to allow transfers (for public servers)
- Authentication
- Locking
- Cache
- Generating backup path/name.
- Post-processing of backups, e.g. to do compression.
- After transfer, before replacement: so that we can spit out a diff
of what was changed, or kick off some kind of reconciliation
process.
VFS:
Rather than talking straight to the filesystem, rsyncd talks through
an internal API. Samba has one. Is it useful?
- Could be a tidy way to implement cached signatures.
- Keep files compressed on disk?
Interactive interface:
- Something like ncFTP, or integration into GNOME-vfs. Probably
hold a single socket connection open.
- Can either call us as a separate process, or as a library.
- The standalone process needs to produce output in a form easily
digestible by a calling program, like the --emacs feature some
have. Same goes for output: rpm outputs a series of hash symbols,
which are easier for a GUI to handle than "\r30% complete"
strings.
- Yow! emacs support. (You could probably build that already, of
course.) I'd like to be able to write a simple script on a remote
machine that rsyncs it to my workstation, edits it there, then
pushes it back up.
Pie-in-the-sky features:
These might have a severe impact on the protocol, and are not
clearly in our core requirements. It looks like in many of them
having scripting hooks will allow us
- Transport over UDP multicast. The hard part is handling multiple
destinations which have different basis files. We can look at
multicast-TFTP for inspiration.
- Conflict resolution. Possibly general scripting support will be
sufficient.
- Integrate with locking. It's hard to see a good general solution,
because Unix systems have several locking mechanisms, and grabbing
the lock from programs that don't expect it could cause deadlocks,
timeouts, or other problems. Scripting support might help.
- Replicate in place, rather than to a temporary file. This is
dangerous in the case of interruption, and it also means that the
delta can't refer to blocks that have already been overwritten.
On the other hand we could semi-trivially do this at first by
simply generating a delta with no copy instructions.
- Replicate block devices. Most of the difficulties here are to do
with replication in place, though on some systems we will also
have to do I/O on block boundaries.
- Peer to peer features. Flavour of the year. Can we think about
ways for clients to smoothly and voluntarily become servers for
content they receive?
- Imagine a situation where the destination has a much faster link
to the cloud than the source. In this case, Mojo Nation downloads
interleaved blocks from several slower servers. The general
situation might be a way for a master rsync process to farm out
tasks to several subjobs. In this particular case they'd need
different sockets. This might be related to multicast.
Unlikely features:
- Allow remote source and destination. If this can be cleanly
designed into the protocol, perhaps with the remote machine acting
as a kind of echo, then it's good. It's uncommon enough that we
don't want to shape the whole protocol around it, though.
In fact, in a triangle of machines there are two possibilities:
all traffic passes from remote1 to remote2 through local, or local
just sets up the transfer and then remote1 talks to remote2. FTP
supports the second but it's not clearly good. There are some
security problems with being able to instruct one machine to open
a connection to another.
In favour of evolving the protocol:
- Keeping compatibility with existing rsync servers will help with
adoption and testing.
- We should at the very least be able to fall back to the new
protocol.
- Error handling is not so good.
In favour of using a new protocol:
- Maintaining compatibility might soak up development time that
would better go into improving a new protocol.
- If we start from scratch, it can be documented as we go, and we
can avoid design decisions that make the protocol complex or
implementation-bound.
Error handling:
- Errors should come back reliably, and be clearly associated with
the particular file that caused the problem.
- Some errors ought to cause the whole transfer to abort; some are
just warnings. If any errors have occurred, then rsync ought to
return an error.
Concurrency:
- We want to keep the CPU, filesystem, and network as full as
possible as much of the time as possible.
- We can do nonblocking network IO, but not so for disk.
- It makes sense to on the destination be generating signatures and
applying patches at the same time.
- Can structure this with nonblocking, threads, separate processes,
etc.
Uses:
- Mirroring software distributions:
- Synchronizing laptop and desktop
- NFS filesystem migration/replication. See
http://www.ietf.org/proceedings/00jul/00july-133.htm#P24510_1276764
- Sync with PDA
- Network backup systems
- CVS filemover
Conflict resolution:
- Requires application-specific knowledge. We want to provide
policy, rather than mechanism.
- Possibly allowing two-way migration across a single connection
would be useful.
Moved files:
- There's no trivial way to detect renamed files, especially if they
move between directories.
- If we had a picture of the remote directory from last time on
either machine, then the inode numbers might give us a hint about
files which may have been renamed.
- Files that are renamed and not modified can be detected by
examining the directory listing, looking for files with the same
size/date as the origin.
Filesystem migration:
NFSv4 probably wants to migrate file locks, but that's not really
our problem.
Atomic updates:
The NFSv4 working group wants atomic migration. Most of the
responsibility for this lies on the NFS server or OS.
If migrating a whole tree, then we could do a nearly-atomic rename
at the end. This ties in to having separate basis and destination
files.
There's no way in Unix to replace a whole set of files atomically.
However, if we get them all onto the destination machine and then do
the updates quickly it would greatly reduce the window.
Scalability:
We should aim to work well on machines in use in a year or two.
That probably means transfers of many millions of files in one
batch, and gigabytes or terabytes of data.
For argument's sake: at the low end, we want to sync ten files for a
total of 10kb across a 1kB/s link. At the high end, we want to sync
1e9 files for 1TB of data across a 1GB/s link.
On the whole CPU usage is not normally a limiting factor, if only
because running over SSH burns a lot of cycles on encryption.
Perhaps have resource throttling without relying on rlimit.
Streaming:
A big attraction of rsync is that there are few round-trip delays:
basically only one to get started, and then everything is
pipelined. This is a problem with FTP, and NFS (at least up to
v3). NFSv4 can pipeline operations, but building on that is
probably a bit complicated.
Related work:
- mirror.pl
- ProFTPd
- Apache
- BitTorrent -- p2p mirroring
http://bitconjurer.org/BitTorrent/

View File

@@ -56,13 +56,6 @@ You can launch it either via inetd, as a stand-alone daemon, or from an rsync
client via a remote shell. If run as a stand-alone daemon then just run the
command "`rsync --daemon`" from a suitable startup script.
Systems using systemd can use the example unit files in the source tree's
`packaging/systemd` directory. The `rsync.service` file runs a stand-alone
daemon using `rsync --daemon --no-detach`, while `rsync.socket` and
`rsync@.service` show a socket-activated setup for incoming connections. These
files may need local adjustment to match your installed rsync path, packaging
layout, and module policy.
When run via inetd you should add a line like this to /etc/services:
> rsync 873/tcp

26
rsyncsh.txt Normal file
View File

@@ -0,0 +1,26 @@
rsyncsh
Copyright (C) 2001 by Martin Pool
This is a quick hack to build an interactive shell around rsync, the
same way we have the ftp, lftp and ncftp programs for the FTP
protocol. The key application for this is connecting to a public
rsync server, such as rsync.kernel.org, change down through and list
directories, and finally pull down the file you want.
rsync is somewhat ill-at-ease as an interactive operation, since every
network connection is used to carry out exactly one operation. rsync
kind of "forks across the network" passing the options and filenames
to operate upon, and the connection is closed when the transfer is
complete. (This might be fixed in the future, either by adapting the
current protocol to allow chained operations over a single socket, or
by writing a new protocol that better supports interactive use.)
So, rsyncsh runs a new rsync command and opens a new socket for every
(network-based) command you type.
This has two consequences. Firstly, there is more command latency
than is really desirable. More seriously, if the connection cannot be
done automatically, because for example it uses SSH with a password,
then you will need to enter the password every time. We might even
fix this in the future, though, by having a way to automatically feed
the password to SSH if it's entered once.

View File

@@ -31,11 +31,6 @@ import subprocess
import sys
import threading
# Share the test exit-code enum with the test helpers. exitcodes.py lives in
# testsuite/ (next to this script); it has no import-time side effects.
sys.path.insert(0, os.path.join(os.path.dirname(os.path.abspath(__file__)), 'testsuite'))
from exitcodes import Exit
def parse_args():
p = argparse.ArgumentParser(description='Run rsync test suite')
@@ -63,9 +58,6 @@ def parse_args():
help='Stop after first test failure')
p.add_argument('--timeout', type=int, default=300, metavar='SECS',
help='Per-test timeout in seconds (default: 300)')
p.add_argument('--race-timeout', type=float, default=5.0, metavar='SECS',
help='Budget (seconds) a TOCTOU symlink-race test may spend '
'trying to win its race before concluding (default: 5)')
p.add_argument('--rsync-bin', default=None, metavar='PATH',
help='Path to rsync binary (default: ./rsync)')
p.add_argument('--rsync-bin2', default=None, metavar='PATH',
@@ -250,18 +242,18 @@ def parse_expect_result(path):
f"{path}:{lineno}: expected '<testname> "
f"<{'|'.join(_VALID_OUTCOMES)}>', got: {raw.rstrip()}\n"
)
sys.exit(Exit.ERROR)
sys.exit(2)
expect[fields[0]] = fields[1]
return expect
def outcome_of(result):
"""Map a per-test exit code to an outcome string."""
if result == Exit.PASS:
if result == 0:
return 'pass'
if result == Exit.SKIP:
if result == 77:
return 'skip'
if result == Exit.XFAIL:
if result == 78:
return 'xfail'
return 'fail'
@@ -329,7 +321,7 @@ def run_one_test(testscript, testbase, scratchdir, base_env, timeout,
# Build output text
output_parts = []
show_log = always_log or (result not in (Exit.PASS, Exit.SKIP, Exit.XFAIL))
show_log = always_log or (result not in (0, 77, 78))
if show_log:
output_parts.append(f'----- {testbase} log follows')
try:
@@ -346,9 +338,9 @@ def run_one_test(testscript, testbase, scratchdir, base_env, timeout,
output_parts.append(f'----- {testbase} rsyncd.log ends')
skipped_reason = ''
if result == Exit.PASS:
if result == 0:
output_parts.append(f'PASS {testbase}')
elif result == Exit.SKIP:
elif result == 77:
whyfile = os.path.join(scratchdir, 'whyskipped')
try:
with open(whyfile) as f:
@@ -356,7 +348,7 @@ def run_one_test(testscript, testbase, scratchdir, base_env, timeout,
except FileNotFoundError:
pass
output_parts.append(f'SKIP {testbase} ({skipped_reason})')
elif result == Exit.XFAIL:
elif result == 78:
output_parts.append(f'XFAIL {testbase}')
else:
output_parts.append(f'FAIL {testbase}')
@@ -415,13 +407,13 @@ def main():
if not os.path.isfile(rsync_bin):
sys.stderr.write(f"rsync_bin {rsync_bin} is not a file\n")
sys.exit(Exit.ERROR)
sys.exit(2)
if not os.path.isfile(rsync_bin2):
sys.stderr.write(f"rsync_bin2 {rsync_bin2} is not a file\n")
sys.exit(Exit.ERROR)
sys.exit(2)
if not os.path.isdir(srcdir):
sys.stderr.write(f"srcdir {srcdir} is not a directory\n")
sys.exit(Exit.ERROR)
sys.exit(2)
# Helper programs the test scripts invoke directly. Missing any of these
# would cause many tests to fail with confusing "not found" errors, so
@@ -438,7 +430,7 @@ def main():
f"Build them with: make {' '.join(missing)}\n"
f"or run the full test target: make check\n"
)
sys.exit(Exit.ERROR)
sys.exit(2)
testuser = get_testuser()
@@ -483,7 +475,6 @@ def main():
'scratchbase': scratchbase,
'suitedir': suitedir,
'TESTRUN_TIMEOUT': str(args.timeout),
'race_timeout': str(args.race_timeout),
'HOME': scratchbase,
'PYTHONPATH': pythonpath,
})
@@ -544,40 +535,34 @@ def main():
passed = 0
failed = 0
skipped = 0
xfailed = 0
skipped_list = []
outcomes = {} # testbase -> actual outcome string ('pass'/'skip'/'fail'/'xfail')
def process_result(tr):
"""Process a TestResult and update counters. Returns True if the test
should count as a failure for --stop-on-fail purposes."""
nonlocal passed, failed, skipped, xfailed
nonlocal passed, failed, skipped
with _print_lock:
if tr.output:
print(tr.output)
scratchdir = os.path.join(scratchbase, tr.testbase)
oc = outcome_of(tr.result)
outcomes[tr.testbase] = oc
if tr.result == Exit.PASS:
if tr.result == 0:
passed += 1
elif tr.result == Exit.SKIP:
elif tr.result == 77:
skipped_list.append(tr.testbase)
skipped += 1
elif tr.result == Exit.XFAIL:
# XFAIL: an expected failure (a known, documented residual the test
# asserts against). Reported distinctly but does NOT fail the suite;
# when the underlying issue is fixed the test returns 0 instead.
xfailed += 1
else:
failed += 1
if tr.result in (Exit.PASS, Exit.SKIP, Exit.XFAIL) and not args.preserve_scratch \
if tr.result in (0, 77) and not args.preserve_scratch \
and os.path.isdir(scratchdir):
subprocess.run(['rm', '-rf', scratchdir], capture_output=True)
# With a manifest, only a mismatch is a "failure" (an expected fail is
# fine); without one, any non-pass/non-skip/non-xfail result is a failure.
# fine); without one, any non-pass/non-skip result is a failure.
if expect is not None:
return mismatch(tr.testbase, oc)
return tr.result not in (Exit.PASS, Exit.SKIP, Exit.XFAIL)
return tr.result not in (0, 77)
if args.parallel > 1:
# Parallel execution
@@ -639,8 +624,6 @@ def main():
print(f' {passed} passed')
if failed > 0:
print(f' {failed} failed')
if xfailed > 0:
print(f' {xfailed} xfailed (expected)')
if skipped > 0:
print(f' {skipped} skipped')
if vg_errors > 0:

View File

@@ -362,7 +362,6 @@ void send_files(int f_in, int f_out)
* Reconstruct the full path relative to module_dir
* from F_PATHNAME (path) and f_name (fname). */
char secure_path[MAXPATHLEN];
const char *relp;
int slen = snprintf(secure_path, sizeof secure_path, "%s%s%s", path, slash, fname);
if (slen >= (int)sizeof secure_path) {
io_error |= IOERR_GENERAL;
@@ -372,13 +371,7 @@ void send_files(int f_in, int f_out)
send_msg_int(MSG_NO_SEND, ndx);
continue;
}
/* A module with `path = /` makes F_PATHNAME absolute, so the
* joined path starts with '/'; strip leading slashes to a
* module-relative path that secure_relative_open accepts (#897). */
relp = secure_path;
while (*relp == '/')
relp++;
fd = secure_relative_open(module_dir, relp, O_RDONLY, 0);
fd = secure_relative_open(module_dir, secure_path, O_RDONLY, 0);
} else {
fd = do_open_checklinks(fname);
}

View File

@@ -302,12 +302,12 @@ def validated_arg(opt, arg, typ=3, wild=False):
if arg.startswith('./'):
arg = arg[1:]
arg = arg.replace('//', '/')
is_absolute_arg = args.absolute and opt == 'arg' and args.dir != '/' and (arg == args.dir or arg.startswith(args.dir_slash))
if not is_absolute_arg:
arg = arg.lstrip('/')
arg = arg.lstrip('/')
if args.dir != '/':
if HAS_DOT_DOT_RE.search(arg):
die("do not use .. in", opt, "(anchor the path at the root of your restricted dir)")
if arg.startswith('/'):
arg = args.dir + arg
if wild:
got = glob.glob(arg)
@@ -328,15 +328,12 @@ def validated_arg(opt, arg, typ=3, wild=False):
arg = arg[:-2]
real_arg = os.path.realpath(arg)
if arg != real_arg and not real_arg.startswith(args.dir_slash):
if not (is_absolute_arg and real_arg == args.dir):
die('unsafe arg:', orig_arg, [arg, real_arg])
die('unsafe arg:', orig_arg, [arg, real_arg])
if arg_has_trailing_slash:
arg += '/'
elif arg_has_trailing_slash_dot:
arg += '/.'
if is_absolute_arg and arg == args.dir:
arg = '.'
elif opt == 'arg' and arg.startswith(args.dir_slash):
if opt == 'arg' and arg.startswith(args.dir_slash):
arg = arg[args.dir_slash_len:]
if arg == '':
arg = '.'
@@ -375,7 +372,6 @@ if __name__ == '__main__':
only_group.add_argument('-ro', action='store_true', help="Allow only reading from the DIR. Implies -no-del and -no-lock.")
only_group.add_argument('-wo', action='store_true', help="Allow only writing to the DIR.")
arg_parser.add_argument('-munge', action='store_true', help="Enable rsync's --munge-links on the server side.")
arg_parser.add_argument('-absolute', action='store_true', help="Allow transfer args to use absolute server paths under DIR.")
arg_parser.add_argument('-no-del', action='store_true', help="Disable rsync's --delete* and --remove* options.")
arg_parser.add_argument('-no-lock', action='store_true', help="Avoid the single-run (per-user) lock check.")
arg_parser.add_argument('-no-overwrite', action='store_true', help="Prevent overwriting existing files by enforcing --ignore-existing")

View File

@@ -5,7 +5,7 @@ rrsync - a script to setup restricted rsync users via ssh logins
## SYNOPSIS
```
rrsync [-ro|-wo] [-munge] [-absolute] [-no-del] [-no-lock] [-no-overwrite] DIR
rrsync [-ro|-wo] [-munge] [-no-del] [-no-lock] [-no-overwrite] DIR
```
The single non-option argument specifies the restricted _DIR_ to use. It can be
@@ -77,12 +77,6 @@ The remainder of this manpage is dedicated to using the rrsync script.
Enable rsync's [`--munge-links`](rsync.1#opt) on the server side.
0. `-absolute`
Allow file-transfer arguments to name the restricted directory using its
absolute server path. For example, with `rrsync -absolute /path/to/root`,
the transfer arg `/path/to/root/dir1` is accepted as an alias for `dir1`.
0. `-no-del`
Disable rsync's `--delete*` and `--remove*` options.

114
syscall.c
View File

@@ -536,9 +536,7 @@ int do_mknod(const char *pathname, mode_t mode, dev_t dev)
*/
int do_mknod_at(const char *pathname, mode_t mode, dev_t dev)
{
/* HAVE_MKNODAT: older Darwin declares AT_FDCWD but not mknodat(), so
* the at-variant won't build there; fall back to do_mknod() (#896). */
#if defined(AT_FDCWD) && defined(HAVE_MKNODAT)
#ifdef AT_FDCWD
extern int am_daemon, am_chrooted;
char dirpath[MAXPATHLEN];
const char *bname;
@@ -600,7 +598,7 @@ int do_mknod_at(const char *pathname, mode_t mode, dev_t dev)
return ret;
}
#if !defined MKNOD_CREATES_FIFOS && defined HAVE_MKFIFO && defined HAVE_MKFIFOAT
#if !defined MKNOD_CREATES_FIFOS && defined HAVE_MKFIFO
if (S_ISFIFO(mode))
ret = mkfifoat(dfd, bname, mode);
else
@@ -1708,19 +1706,6 @@ static int path_has_dotdot_component(const char *path)
}
#if defined(__linux__) && defined(HAVE_OPENAT2)
/* openat2(RESOLVE_BENEATH) via the raw syscall, gated on openat2_usable() so a
* seccomp filter that traps openat2 with SIGSYS (e.g. the Android sandbox)
* makes us report ENOSYS and fall back rather than killing the process. Only
* the openat2 call is gated here; a plain openat() is always safe to attempt. */
static int openat2_beneath(int dirfd, const char *path, const struct open_how *how)
{
if (!openat2_usable()) {
errno = ENOSYS;
return -1;
}
return syscall(SYS_openat2, dirfd, path, how, sizeof *how);
}
static int secure_relative_open_linux(const char *basedir, const char *relpath, int flags, mode_t mode)
{
struct open_how how;
@@ -1749,12 +1734,12 @@ static int secure_relative_open_linux(const char *basedir, const char *relpath,
memset(&bhow, 0, sizeof bhow);
bhow.flags = O_RDONLY | O_DIRECTORY;
bhow.resolve = RESOLVE_BENEATH | RESOLVE_NO_MAGICLINKS;
dirfd = openat2_beneath(AT_FDCWD, basedir, &bhow);
dirfd = syscall(SYS_openat2, AT_FDCWD, basedir, &bhow, sizeof bhow);
if (dirfd == -1)
return -1;
}
retfd = openat2_beneath(dirfd, relpath, &how);
retfd = syscall(SYS_openat2, dirfd, relpath, &how, sizeof how);
if (dirfd != AT_FDCWD)
close(dirfd);
@@ -1795,68 +1780,13 @@ static int secure_relative_open_resolve_beneath(const char *basedir, const char
}
#endif
/* The logical current directory (maintained by change_dir() in util1.c).
* Defined here -- rather than in util1.c -- so the test helpers that link
* syscall.o but not util1.o (tls, trimslash) get the definition without a
* weak-symbol fallback, which is not portable to PE/COFF targets (Cygwin). */
char curr_dir[MAXPATHLEN];
unsigned int curr_dir_len;
int secure_relative_open(const char *basedir, const char *relpath, int flags, mode_t mode)
{
extern int am_daemon, am_chrooted;
extern char *module_dir;
extern unsigned int module_dirlen;
char modrel_buf[MAXPATHLEN];
int reanchored = 0;
if (!relpath || relpath[0] == '/') {
// must be a relative path
errno = EINVAL;
return -1;
}
/* Sanitizing daemon only (am_daemon && !am_chrooted). Here we have chdir'd
* into a sub-dir of the module (the transfer destination), so a relative
* alt-dest like "../01" may legitimately climb to a sibling that is still
* inside the module (#915). Confining beneath the cwd would reject that
* climb. Re-anchor at the module root -- the real trust boundary -- by
* prefixing the cwd's module-relative path (from rsync's logical curr_dir[],
* a guaranteed lexical prefix of module_dir, unlike getcwd()) and resolving
* beneath module_dir; RESOLVE_BENEATH then allows in-module climbs and still
* rejects escapes. Only for paths that contain "..". module_dirlen is 0 for
* a `path = /` module (clientserver.c), so we gate on module_dir, not its
* length, to cover that case too -- the prefix check below treats
* module_dirlen 0 as "module root is /". */
if (am_daemon && !am_chrooted
&& module_dir && module_dir[0] == '/'
&& (basedir == NULL || basedir[0] != '/')
&& (path_has_dotdot_component(relpath)
|| (basedir && path_has_dotdot_component(basedir)))) {
const char *p;
int n;
if (curr_dir_len >= module_dirlen
&& strncmp(curr_dir, module_dir, module_dirlen) == 0
&& (curr_dir[module_dirlen] == '\0' || curr_dir[module_dirlen] == '/')) {
for (p = curr_dir + module_dirlen; *p == '/'; p++) {}
if (basedir)
n = snprintf(modrel_buf, sizeof modrel_buf, "%s%s%s/%s",
p, *p ? "/" : "", basedir, relpath);
else
n = snprintf(modrel_buf, sizeof modrel_buf, "%s%s%s",
p, *p ? "/" : "", relpath);
if (n < 0 || n >= (int)sizeof modrel_buf) {
errno = ENAMETOOLONG;
return -1;
}
basedir = module_dir; /* absolute, operator-trusted anchor */
relpath = modrel_buf;
reanchored = 1;
}
/* else: cwd not under module root as expected -- fall through to the
* front-door rejection below (fail safe). */
}
/* Reject any path with a literal ".." component (bare "..",
* "../foo", "foo/..", "foo/../bar", "subdir/.."). The previous
* substring-based check caught only "../" prefix and "/../"
@@ -1865,19 +1795,14 @@ int secure_relative_open(const char *basedir, const char *relpath, int flags, mo
* and pre-5.6 Linux. RESOLVE_BENEATH on Linux/FreeBSD/macOS
* catches some of these in-kernel with EXDEV, but the front
* door must reject them consistently with EINVAL across all
* platforms so callers can rely on the validation. Skipped for a
* re-anchored path: its ".." is deliberate, stays within the module,
* and is adjudicated by RESOLVE_BENEATH below (the portable fallback
* re-rejects it -- see there). */
if (!reanchored) {
if (path_has_dotdot_component(relpath)) {
errno = EINVAL;
return -1;
}
if (basedir && basedir[0] != '/' && path_has_dotdot_component(basedir)) {
errno = EINVAL;
return -1;
}
* platforms so callers can rely on the validation. */
if (path_has_dotdot_component(relpath)) {
errno = EINVAL;
return -1;
}
if (basedir && basedir[0] != '/' && path_has_dotdot_component(basedir)) {
errno = EINVAL;
return -1;
}
#if defined(__linux__) && defined(HAVE_OPENAT2)
@@ -1896,21 +1821,6 @@ int secure_relative_open(const char *basedir, const char *relpath, int flags, mo
return secure_relative_open_resolve_beneath(basedir, relpath, flags, mode);
#endif
/* Portable fallback only (no kernel RESOLVE_BENEATH): the per-component
* O_NOFOLLOW walk below can't adjudicate ".." safely, so reject it here --
* even for a re-anchored path. This re-breaks --link-dest=../01 on
* openat2/O_RESOLVE_BENEATH-less platforms (NetBSD/OpenBSD/Solaris/Cygwin/
* pre-5.6 Linux), trading function for safety; on the kernel paths above
* RESOLVE_BENEATH already allowed the in-module climb. */
if (path_has_dotdot_component(relpath)) {
errno = EINVAL;
return -1;
}
if (basedir && basedir[0] != '/' && path_has_dotdot_component(basedir)) {
errno = EINVAL;
return -1;
}
#if !defined(O_NOFOLLOW) || !defined(O_DIRECTORY) || !defined(AT_FDCWD)
// really old system, all we can do is live with the risks
if (!basedir) {

View File

@@ -17,7 +17,7 @@
#include <sys/stat.h>
#if defined(__linux__) && defined(HAVE_OPENAT2)
#ifdef __linux__
#include <sys/syscall.h>
#include <linux/openat2.h>
#endif
@@ -44,11 +44,9 @@ static int errs = 0;
* other than the kernel rejecting the requested confinement flag. */
static int kernel_resolve_beneath_supported(void)
{
#if (defined(__linux__) && defined(HAVE_OPENAT2)) || defined(O_RESOLVE_BENEATH)
int fd;
#endif
#if defined(__linux__) && defined(HAVE_OPENAT2)
if (openat2_usable()) {
#ifdef __linux__
{
struct open_how how;
memset(&how, 0, sizeof how);
how.flags = O_RDONLY | O_DIRECTORY;
@@ -58,7 +56,7 @@ static int kernel_resolve_beneath_supported(void)
close(fd);
return 1;
}
/* ENOSYS = kernel < 5.6 or openat2 seccomp-blocked. Fall through to the O_RESOLVE_BENEATH
/* ENOSYS = kernel < 5.6. Fall through to the O_RESOLVE_BENEATH
* probe in case we're a Linux build running on a kernel that
* gained O_RESOLVE_BENEATH via some out-of-tree backport. */
}

View File

@@ -45,8 +45,6 @@ size_t max_alloc = (size_t)-1; /* test helpers are not memory-constrained;
* hits at its first my_strdup() call. */
char *partial_dir;
char *module_dir;
/* curr_dir[]/curr_dir_len (read by secure_relative_open) are defined in
* syscall.c, which every helper links -- no stub needed here. */
filter_rule_list daemon_filter_list;
void rprintf(UNUSED(enum logcode code), const char *format, ...)

133
testhelp/maketree.py Normal file
View File

@@ -0,0 +1,133 @@
#!/usr/bin/env python2
# Copyright (C) 2002 by Martin Pool <mbp@samba.org>
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License version
# 2 as published by the Free Software Foundation.
#
# This program is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
# Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with this program; if not, write to the Free Software
# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
# Populate a tree with pseudo-randomly distributed files to test
# rsync.
from __future__ import generators
import random, string, os, os.path
nfiles = 10000
depth = 5
n_children = 20
n_files = 20
n_symlinks = 10
name_chars = string.digits + string.letters
abuffer = 'a' * 1024
def random_name_chars():
a = ""
for i in range(10):
a = a + random.choice(name_chars)
return a
def generate_names():
n = 0
while 1:
yield "%05d_%s" % (n, random_name_chars())
n += 1
class TreeBuilder:
def __init__(self):
self.n_children = 20
self.n_files = 100
self.total_entries = 100000 # long(1e8)
self.actual_size = 0
self.name_gen = generate_names()
self.all_files = []
self.all_dirs = []
self.all_symlinks = []
def random_size(self):
return random.lognormvariate(4, 4)
def random_symlink_target(self):
what = random.choice(['directory', 'file', 'symlink', 'none'])
try:
if what == 'directory':
return random.choice(self.all_dirs)
elif what == 'file':
return random.choice(self.all_files)
elif what == 'symlink':
return random.choice(self.all_symlinks)
elif what == 'none':
return self.name_gen.next()
except IndexError:
return self.name_gen.next()
def can_continue(self):
self.total_entries -= 1
return self.total_entries > 0
def build_tree(self, prefix, depth):
"""Generate a breadth-first tree"""
for count, function in [[n_files, self.make_file],
[n_children, self.make_child_recurse],
[n_symlinks, self.make_symlink]]:
for i in range(count):
if not self.can_continue():
return
name = os.path.join(prefix, self.name_gen.next())
function(name, depth)
def print_summary(self):
print "total bytes: %d" % self.actual_size
def make_child_recurse(self, dname, depth):
if depth > 1:
self.make_dir(dname)
self.build_tree(dname, depth-1)
def make_dir(self, dname, depth='ignore'):
print "%s/" % (dname)
os.mkdir(dname)
self.all_dirs.append(dname)
def make_symlink(self, lname, depth='ignore'):
print "%s -> %s" % (lname, self.random_symlink_target())
def make_file(self, fname, depth='ignore'):
size = long(self.random_size())
print "%-70s %d" % (fname, size)
f = open(fname, 'w')
f.truncate(size)
self.fill_file(f, size)
self.all_files.append(fname)
self.actual_size += size
def fill_file(self, f, size):
while size > 0:
f.write(abuffer[:size])
size -= len(abuffer)
tb = TreeBuilder()
tb.build_tree('/tmp/foo', 3)
tb.print_summary()

View File

@@ -1,186 +0,0 @@
# rsync testsuite
This directory holds rsync's automated regression tests. Ideally every code
change or bug fix comes with a test that would have caught the problem.
The tests are Python scripts named `testsuite/*_test.py`, driven by the
`runtests.py` harness at the top of the tree (the old shell-based `runtests.sh`
is gone). Shared helpers live in `testsuite/rsyncfns.py`. A handful of C helper
programs (`tls`, `getgroups`, `trimslash`, …) are built alongside `rsync` and
used by some tests. Coverage notes are in [COVERAGE.md](COVERAGE.md).
## Running the tests
### Via make
Run from the build directory:
- **`make check`** — build the helper programs and run the whole suite in
parallel (`CHECK_J`, default 8) against the just-built `./rsync`. You do **not**
need `make install` first; indeed you generally should not install before
testing. Use `make check CHECK_J=1` to run serially.
- **`make check29`** / **`make check30`** — the same, forcing protocol version 29
or 30.
- **`make installcheck`** — run the suite against the *installed* binary (e.g.
`/usr/local/bin/rsync`). Per the GNU standards this does not search `$PATH`.
Handy for testing a distribution build.
- **`make check-progs`** — (re)build just the C helper programs the tests need,
without running anything.
- **`make coverage`** / **`coverage-tcp`** / **`coverage-all`** — generate an HTML
coverage report (needs `./configure --enable-coverage` and `gcovr`);
`coverage-all` merges runs across protocol versions and the tcp transport.
### Via runtests.py directly
`make check` just drives `runtests.py`; run it directly for finer control. It
defaults `--rsync-bin` to `./rsync`, so run it from the build directory (or pass
`--rsync-bin` / `--tooldir`):
```sh
./runtests.py # all tests
./runtests.py chmod-temp-dir # a single test by name
./runtests.py 'xattr*' # a glob of test names
```
Useful options:
- `-j N`, `--parallel N` — run up to N tests at once
- `--use-tcp` — run daemon tests against a real `rsyncd` on `127.0.0.1` (the
default runs them over a stdio pipe). **Read the security warning below before
using this on a shared machine.**
- `--protocol VER` — force a protocol version
- `--preserve-scratch` — keep each test's scratch dir afterwards
- `--log-level N`, `--always-log` — more verbose output / show logs for passing tests too
- `--stop-on-fail` — stop after the first failure
- `--timeout SECS` — per-test timeout (default 300)
- `--valgrind`, `--valgrind-opts OPTS` — run rsync under valgrind
- `--rsync-bin PATH`, `--tooldir DIR`, `--srcdir DIR` — locate the binary / build / source dirs
- `--expect-skipped LIST` — see skip enforcement below
### Security warning: `--use-tcp`
> **⚠️ Do not use `--use-tcp` on a machine with untrusted local users.**
>
> `--use-tcp` starts a real `rsync` daemon listening on a loopback TCP port
> (`127.0.0.1` / `::1`) and **deliberately configures insecure test scenarios**
> (daemon modules without authentication, unsafe options enabled, etc.). Loopback
> addresses are reachable by *every* local user, so for as long as the tests run,
> any other user on the machine can connect to that daemon and exploit those
> deliberately-insecure modules — potentially reading or writing files with the
> privileges of the user running the tests (which is **root** if you run the suite
> as root).
>
> Only run `--use-tcp` where there are **no possible local users who might try to
> exploit it** — a single-user workstation or a dedicated, isolated CI machine.
> The default stdio-pipe transport carries no such risk: it talks to the daemon
> over a private pipe with nothing listening on the network, so prefer it on any
> shared or multi-user host.
### Results and exit codes
Each test prints one result line — `PASS`, `FAIL`, `ERROR`, `SKIP` (with a
reason), or `XFAIL` (an expected failure) — and the run ends with a
`passed / failed / skipped` summary. Per-test exit-code convention:
| code | meaning |
|------|---------|
| 0 | pass |
| 1 | fail |
| 2 | error |
| 77 | skip |
| 78 | xfail |
`runtests.py` exits non-zero if any test fails. Some tests need root or another
precondition and otherwise `SKIP` — read the individual test scripts for details.
**Skip enforcement:** on a full run, set `RSYNC_EXPECT_SKIPPED=a,b,c` (or
`--expect-skipped a,b,c`) and the run fails if the set of skipped tests does not
match. This is how the CI workflows pin each platform's expected skip set.
### Scratch dirs and debugging
Each test runs in `testtmp/<name>/`. On failure the scratch directory is left in
place (also `--preserve-scratch`); including its logs in a bug report is helpful.
### Preconditions
You need `python3`, `/bin/sh`, and the normal build toolchain. The ACL/xattr
tests need the `acl` and `attr` tools (`getfacl`/`setfacl`,
`getfattr`/`setfattr`) and skip if they are absent. Some tests need root.
These tests also run in CI via GitHub Actions (see `.github/workflows/`).
## Fleet testing (fleettest.py)
`testsuite/fleettest.py` builds the committed HEAD of an rsync checkout on a
fleet of remote machines over ssh and runs the suite under both transports
(stdio-pipe and `--use-tcp`) in parallel, reporting only the *unexpected*
results. It is a fast local pre-flight for the GitHub CI matrix: each target
mirrors a `.github/workflows/*.yml` job — its configure flags, and the
`RSYNC_EXPECT_SKIPPED` list parsed straight from the workflow.
Because every run includes a `--use-tcp` pass, the fleet stands up the insecure
loopback test daemon on each target — so only point it at machines with **no
untrusted local users** (see the [security warning](#security-warning---use-tcp)
above).
The fleet — which machines, and how to reach and build on each — is described in
a JSON file. Copy the bundled example (it is git-ignored) and edit it for your
hosts:
```sh
cp testsuite/fleettest.json.example testsuite/fleettest.json # then edit
# (or symlink it, or point elsewhere with --fleet PATH)
```
The config is looked up in order: `~/.fleettest.json` first, then
`testsuite/fleettest.json`, unless overridden with `--fleet PATH`.
Each entry names an ssh host (`null` to run locally), the workflow it mirrors,
and its configure flags, plus optional per-target settings (`make`, `privilege`,
`env_prefix`, …). See the comments in `fleettest.json.example`.
A target with `"nonroot": true` does an extra pass, after the main (root) run,
that reruns the privilege-sensitive tests as the unprivileged ssh user. Which
tests those are is **not** listed in the fleet config — a test opts in by
setting a module-level `fleet_nonroot = True`, so the set is maintained in the
test files and new privilege-sensitive tests join automatically with no
fleet-config change.
A target with `"protocols": [30, 29]` runs one extra stdio-pipe pass per listed
version, each forcing that older wire version with `runtests --protocol=N` — the
fleet analogue of a workflow's `check30`/`check29` steps. The passes reuse the
same parsed `RSYNC_EXPECT_SKIPPED` list as the pipe run and show up as `protoNN`
columns in the report (and `--timing` breakdown). Targets that don't set
`protocols` show `-` there.
Run it from inside a checkout (it builds the current directory's HEAD; use
`--repo PATH` for another tree):
```sh
python3 testsuite/fleettest.py # whole fleet, both transports
python3 testsuite/fleettest.py --list # list configured targets
python3 testsuite/fleettest.py --targets NAME[,NAME]
python3 testsuite/fleettest.py --fleet other.json --transport pipe
python3 testsuite/fleettest.py --timing # per-target wall-clock breakdown
```
`--timing` adds a per-target breakdown after the report — total wall-clock plus
the push / build / pipe / tcp / protoNN / nonroot phases, sorted slowest-first. Targets
run in parallel, so the whole run is gated by the slowest one; the phase columns
show whether that target's hold-up is the push, the build, or a test pass.
Each run gets its own randomly-named build dir on every target
(`<builddir>-<run_id>`), so two or three runs can share the same fleet without
interfering. The dir is removed when the run ends — on success or failure, and
best-effort on Ctrl-C/kill; pass `--keep` to retain it for inspection. A hard
kill (`SIGKILL`), or a signal arriving mid-push, can leave a stray
`<builddir>-<id>` behind; sweep leftovers with
`python3 testsuite/fleettest.py --cleanup` (scope it with `--targets`, and only
run it when no other fleet runs are active, since it removes *all* matching run
dirs on the selected targets).
Each target must be provisioned with the build toolchain its workflow installs
(autoconf, automake, a C compiler, perl, a python3 markdown module such as
cmarkgfm or commonmark unless the flags pass `--disable-md2man`, and the dev
libraries its configure flags enable). A missing piece shows up as `BUILD-FAIL`.

View File

@@ -0,0 +1,28 @@
automatic testsuite for rsync -*- text -*-
We're trying to develop some more substantial tests to prevent rsync
regressions. Ideally, all code changes or bug reports would come with
an appropriate test suite.
You can run these tests by typing "make check" in the build directory.
The tests will run using the rsync binary in the build directory, so
you do not need to do "make install" first. Indeed, you probably
should not install rsync before running the tests.
If you instead type "make installcheck" then the suite will test the
rsync binary from its installed location (e.g. /usr/local/bin/rsync).
You can use this to test a distribution build, or perhaps to run a new
test suite against an old version of rsync. Note that in accordance
with the GNU Standards, installcheck does not look for rsync on the
path.
If the tests pass, you should see a report to that effect. Some tests
require being root or some other precondition, and so will normally not
be checked -- look at the test scripts for more information.
If the tests fail, you will see rather more output. The scratch
directory will remain in the build directory. It would be useful if
you could include the log messages when reporting a failure.
These tests also run automatically on the build farm, and you can see
the results on http://build.samba.org/.

View File

@@ -1,52 +0,0 @@
#!/usr/bin/env python3
"""Regression: a short transfer checksum must not over-state the block s2length.
A full-checksum (--append-verify redo) pass computes the strong block sum length
(s2length). The generator used to cap it at SUM_LENGTH (16), the legacy MD4/MD5
digest size, regardless of the negotiated algorithm. Since the sum2 array holds
xfer_sum_len-byte elements and the sender rejects an s2length larger than
xfer_sum_len, a sub-16-byte transfer checksum -- xxh64 (8 bytes), which is what
rsync negotiates when the build's libxxhash lacks xxh128/xxh3 (e.g. Ubuntu
20.04) -- made the sender die with "Invalid checksum length 16 [sender]"
(protocol incompatibility, code 2).
Forcing --checksum-choice=xxh64 reproduces it on any build that has xxhash, so
this guards the fix without needing an old-libxxhash host. Skipped where xxh64
is unavailable (a build without xxhash).
"""
import json
from rsyncfns import (
FROMDIR, TODIR, assert_same, make_data_file, rmtree, run_rsync,
test_skipped,
)
vv = json.loads(run_rsync('-VV', check=True, capture_output=True).stdout)
if 'xxh64' not in vv.get('checksum_list', []):
test_skipped("xxh64 not in this build's checksum list (no xxhash)")
src, dst = FROMDIR, TODIR
rmtree(src)
rmtree(dst)
src.mkdir(parents=True)
dst.mkdir(parents=True)
# Source longer than the destination so --append has bytes to add; the dest is a
# *corrupted* prefix so --append-verify's whole-file check fails and the file is
# redone with a full checksum -- the csum_length == SUM_LENGTH path that emitted
# the over-long s2length.
make_data_file(src / 'f', 40000)
full = (src / 'f').read_bytes()
prefix = bytearray(full[:20000])
prefix[0:64] = b'\x00' * 64
(dst / 'f').write_bytes(bytes(prefix))
# --no-whole-file forces the delta/checksum path regardless of local-vs-remote.
# run_rsync(check=True) fails the test on the non-zero exit the bug produced.
run_rsync('-a', '--append-verify', '--checksum-choice=xxh64', '--no-whole-file',
f'{src}/', f'{dst}/')
assert_same(dst / 'f', src / 'f', label='append-verify xxh64 redo')
print("append-shortsum: --append-verify with an 8-byte (xxh64) checksum no "
"longer overflows the block s2length")

View File

@@ -11,8 +11,8 @@ import shutil
from rsyncfns import (
FROMDIR, SCRATCHDIR, TODIR,
build_rsyncd_conf, check_perms, checkit, makepath, rmtree,
run_rsync, start_test_daemon, test_fail,
build_rsyncd_conf, checkit, makepath, rmtree,
run_rsync, start_test_daemon,
)
@@ -62,37 +62,6 @@ for d in (checkdir, checkdir / 'dir1', checkdir / 'dir2'):
checkit(['-avv', '--chmod', 'ug-s,a+rX,D+w', f'{FROMDIR}/', f'{TODIR}/'],
checkdir, TODIR)
def check_permcopy(chmod_arg, start_mode, expected, is_dir=False):
rmtree(FROMDIR)
rmtree(TODIR)
makepath(FROMDIR)
permcopy = FROMDIR / 'permcopy'
if is_dir:
permcopy.mkdir()
else:
permcopy.write_text('permcopy\n')
os.chmod(permcopy, start_mode)
run_rsync('-avv', f'--chmod={chmod_arg}', f'{FROMDIR}/', f'{TODIR}/')
check_perms(TODIR / 'permcopy', expected)
# Exercise chmod(1)-style permission copies.
check_permcopy('g=o,o=', 0o647, 'rw-rwx---')
check_permcopy('g=u', 0o741, 'rwxrwx--x')
check_permcopy('g-o', 0o775, 'rwx-w-r-x')
check_permcopy('u=g', 0o4755, 'r-xr-xr-x')
check_permcopy('g=u', 0o2755, 'rwxrwxr-x')
check_permcopy('o=u', 0o1750, 'rwxr-xrwx', is_dir=True)
rmtree(FROMDIR)
rmtree(TODIR)
makepath(FROMDIR)
(FROMDIR / 'permcopy').write_text('permcopy\n')
proc = run_rsync('-avv', '--chmod=g=ur', f'{FROMDIR}/', f'{TODIR}/',
check=False, capture_output=True)
if proc.returncode == 0:
test_fail('--chmod=g=ur was not rejected')
# Now exercise the F-only chmod path.
rmtree(FROMDIR)
rmtree(checkdir)

View File

@@ -1,72 +0,0 @@
#!/usr/bin/env python3
# Regression test for issue #951.
#
# When rsync is built against a system zlib (no bundled Z_INSERT_ONLY
# extension), send_deflated_token() falls back to Z_SYNC_FLUSH to add a
# matched block to the compressor history -- but Z_SYNC_FLUSH emits a flush
# block into a fixed-size obuf. A large incompressible matched block
# overflowed obuf and aborted the transfer with
# "deflate on token returned 0 (N bytes left)" at token.c
# The fix loops, discarding the (never-sent) output, until the input is
# consumed. A bundled-zlib build emits no output here, so this test passes
# on either build; it is RED only on a pre-fix system-zlib build.
#
# The matched-block insert path needs all of: --compress-choice=zlib (the
# only method that feeds matched blocks into the deflate history), a large
# --block-size so a single matched token exceeds obuf, incompressible
# (random) data, and a delta over a real connection (compression is skipped
# for purely local transfers). We assert the upload SUCCEEDS *and* the
# result matches the source, so the fix is verified correct, not merely
# non-crashing.
import filecmp
import shutil
import subprocess
from rsyncfns import (
SCRATCHDIR, make_data_file, makepath, rmtree, rsync_argv,
start_test_daemon, test_fail, write_daemon_conf,
)
DAEMON_PORT = 12922
SIZE = 8 * 1024 * 1024 # enough blocks to exercise many inserts
# 65535 (0xffff) is a single insert fragment larger than the deflate output
# buffer (AVAIL_OUT_SIZE(CHUNK_SIZE) ~= 32816). It exercises both failure
# modes of the pre-fix code: the obuf overflow abort, and -- once that is
# loop-drained -- pending insert output left in the stream that leaks into
# the next send. A block that splits into chunks ending in a tiny fragment
# (e.g. 131072 = 65535+65535+2) would hide the pending case.
BLOCK = 65535
moddir = SCRATCHDIR / 'zmod'
srcdir = SCRATCHDIR / 'zsrc'
rmtree(moddir)
rmtree(srcdir)
makepath(moddir)
makepath(srcdir)
# Source is incompressible. The basis (already in the module) is the same
# data with a few bytes changed in one block, so every other 128KB block
# matches exactly and is sent as a token -> the deflate insert path.
make_data_file(srcdir / 'big.dat', SIZE)
shutil.copy(srcdir / 'big.dat', moddir / 'big.dat')
with open(srcdir / 'big.dat', 'r+b') as f:
f.seek(SIZE // 2 + 1000)
f.write(b'\x00' * 32)
conf = write_daemon_conf([('zmod', {'path': str(moddir), 'read only': 'no'})])
url = start_test_daemon(conf, DAEMON_PORT) + 'zmod/'
# -I forces the delta even though the basis has the same size (otherwise the
# quick check skips the file and the matched-block insert path never runs).
proc = subprocess.run(
rsync_argv('-zI', '--compress-choice=zlib', '--no-whole-file',
f'--block-size={BLOCK}', str(srcdir / 'big.dat'), url),
stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True)
if proc.returncode != 0:
print(proc.stdout)
test_fail(f"zlib delta upload failed (rc={proc.returncode}); "
"regression of #951 deflate-token overflow")
if not filecmp.cmp(srcdir / 'big.dat', moddir / 'big.dat', shallow=False):
test_fail("uploaded file differs from source -- zlib delta corruption")

View File

@@ -1,48 +0,0 @@
#!/usr/bin/env python3
"""Daemon upload delete stats report deleted files."""
import subprocess
from rsyncfns import (
FROMDIR, TODIR,
build_rsyncd_conf, forced_protocol, makepath, rmtree, rsync_argv,
start_test_daemon, test_fail,
)
DAEMON_PORT = 12899
src = FROMDIR
dst = TODIR
rmtree(src)
rmtree(dst)
makepath(src, dst)
(src / 'keep.txt').write_text("keep\n")
(dst / 'keep.txt').write_text("keep\n")
(dst / 'delete.txt').write_text("delete\n")
url = start_test_daemon(build_rsyncd_conf(), DAEMON_PORT)
proc = subprocess.run(
rsync_argv('-a', '--delete', '-i', '--stats', f'{src}/', f'{url}test-to/'),
capture_output=True,
text=True,
)
out = proc.stdout + proc.stderr
print(out)
if proc.returncode != 0:
test_fail(f"daemon upload delete run exited {proc.returncode}")
if '*deleting delete.txt' not in out:
test_fail(f"daemon upload did not itemize the deleted file:\n{out}")
# The delete-stats summary line is only sent to the client at protocol >= 31
# (the NDX_DEL_STATS message); an older client can't receive the count, so
# only assert it when the protocol isn't pinned below 31.
pv = forced_protocol()
if pv is None or pv >= 31:
if 'Number of deleted files: 1 (reg: 1)' not in out:
test_fail(f"daemon upload did not report the deleted file in stats:\n{out}")

View File

@@ -1,87 +0,0 @@
#!/usr/bin/env python3
# Regression test for issue #829.
#
# Without --secluded-args the client's safe_arg() backslash-escapes wildcard
# chars in option values, so --chown / --groupmap=*:GROUP is sent to a daemon
# as --groupmap=\*:GROUP. A daemon has no shell to strip the backslash, and
# read_args() used to store option args verbatim, so the receiver saw the
# literal "\*", the wildcard never matched, and the map was ignored (the
# module's configured gid won instead). The fix un-backslashes daemon option
# args.
#
# We run it both ways:
# * default args -- the '*' is safe_arg-escaped and the daemon must
# un-backslash it (the path the fix repairs);
# * --secluded-args -- the '*' is sent raw over the protected channel and
# read with unescape=0, so it must keep working too
# (a guard that the fix didn't disturb that path).
#
# No root needed: a non-root receiver can chgrp(2) to a group the test user
# belongs to, so we map every source group to a second such group and check
# the wildcard took effect.
import os
import subprocess
from rsyncfns import (
SCRATCHDIR, makepath, rmtree, rsync_argv, start_test_daemon,
test_fail, test_skipped, write_daemon_conf,
)
DAEMON_PORT = 12923
# Two distinct groups to map between. As root (the usual CI case) we can
# chgrp(2) to any gid, so take two distinct named groups from the group
# database; a non-root user can only chgrp to groups it belongs to, so use those
# (skip if it is in fewer than two).
if os.geteuid() == 0:
import grp
usable = []
for gr in grp.getgrall():
if gr.gr_gid not in usable:
usable.append(gr.gr_gid)
if len(usable) < 2:
test_skipped("need >=2 groups defined on the system")
else:
usable = []
for g in [os.getgid()] + list(os.getgroups()):
if g not in usable:
usable.append(g)
if len(usable) < 2:
test_skipped("need >=2 groups the test user belongs to")
src_gid, dst_gid = usable[0], usable[1]
moddir = SCRATCHDIR / 'gmod'
srcdir = SCRATCHDIR / 'gsrc'
makepath(moddir)
conf = write_daemon_conf([('gmod', {'path': str(moddir), 'read only': 'no'})])
url = start_test_daemon(conf, DAEMON_PORT) + 'gmod/'
def check(label, *extra_opts):
rmtree(moddir)
rmtree(srcdir)
makepath(moddir)
makepath(srcdir)
f = srcdir / 'f.dat'
f.write_text("hi\n")
os.chown(f, -1, src_gid) # source group differs from the map target
# A --chown-style wildcard map sent to a daemon: the '*' must survive as a
# wildcard so every source group is remapped to dst_gid.
proc = subprocess.run(
rsync_argv('-rg', *extra_opts, f'--groupmap=*:{dst_gid}', str(f), url),
stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True)
if proc.returncode != 0:
print(proc.stdout)
test_fail(f"[{label}] groupmap upload failed (rc={proc.returncode})")
got = os.stat(moddir / 'f.dat').st_gid
if got != dst_gid:
test_fail(f"[{label}] --groupmap='*:{dst_gid}' wildcard ignored over "
f"daemon: got gid {got}, expected {dst_gid} (regression of #829)")
check('default-args')
check('secluded-args', '--secluded-args')

View File

@@ -1,79 +0,0 @@
#!/usr/bin/env python3
# Functional regression: a daemon module with `path = /` (use chroot = no)
# cannot send ANY file in 3.4.3 -- every read fails with "Invalid argument (22)".
#
# Reported as #897 ("Regression in 3.4.3, Invalid argument (22) on all file
# reads when using native protocol"). Works in 3.4.2, works over a remote
# shell; only the native (daemon) protocol with an absolute module path breaks.
#
# Root cause: the 3.4.3 symlink-race hardening routes the sender's file open
# through secure_relative_open(module_dir, secure_path, ...) (sender.c), where
# secure_path = F_PATHNAME + "/" + f_name.
# With `path = /` the module-relative F_PATHNAME is itself ABSOLUTE, so
# secure_path starts with '/'. secure_relative_open()'s front door rejects any
# absolute relpath with EINVAL *before* it ever calls openat2 (matching the
# reporter's strace: the file is stat'd and access()'d but never opened). The
# generator then reports "send_files failed to open ...: Invalid argument (22)"
# and the whole transfer ends in code 23.
#
# This is a pure functional regression (no attacker, no symlink): XFAIL until
# the sender open is made to tolerate an absolute module-root path (the
# accompanying sender.c fix). Runs at any uid.
import subprocess
from rsyncfns import (
SCRATCHDIR, makepath, rmtree, rsync_argv, start_test_daemon, test_fail,
write_daemon_conf,
)
DAEMON_PORT = 12897
# A small source tree under the scratch dir: a file at the served-subdir root
# and one nested deeper (the bug fails on EVERY file, regardless of depth).
served = SCRATCHDIR / 'served'
dst = SCRATCHDIR / 'pulldst'
rmtree(served)
rmtree(dst)
makepath(served / 'sub')
makepath(dst)
(served / 'README').write_text("readme-contents\n")
(served / 'sub' / 'deep.txt').write_text("deep-contents\n")
# Module rooted at the filesystem root, exactly like the report (path = /,
# use chroot = no). We then request the served subtree by its absolute path
# with the leading '/' stripped, so the daemon serves $served from "/".
conf = write_daemon_conf([
('root', {'path': '/', 'read only': 'yes'}),
])
url = start_test_daemon(conf, DAEMON_PORT)
served_rel = str(served).lstrip('/') # e.g. tmp/.../served
proc = subprocess.run(
rsync_argv('-a', f'{url}root/{served_rel}/', f'{dst}/'),
stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True,
)
out = proc.stdout or ''
print(out)
# Bug present: the sender refuses to open the files with EINVAL(22).
if 'Invalid argument (22)' in out or ('failed to open' in out and proc.returncode != 0):
from rsyncfns import test_xfail
test_xfail(
"#897: daemon module `path = /` (use chroot = no) cannot send files -- "
"`send_files failed to open ...: Invalid argument (22)`. The sender's "
"secure_relative_open(module_dir, secure_path) gets an ABSOLUTE "
"secure_path (F_PATHNAME is absolute when path=/) and the front door "
"rejects absolute relpaths with EINVAL before any openat2. To be closed "
"by letting the sender open succeed for an absolute module-root path.")
# Bug fixed (or never present): the files transfer intact.
if proc.returncode != 0:
test_fail(f"daemon pull failed unexpectedly (rc={proc.returncode}); "
f"not the #897 EINVAL symptom:\n{out}")
for rel in ('README', 'sub/deep.txt'):
got = dst / rel
if not got.is_file():
test_fail(f"daemon pull did not deliver {rel} (dst={dst})")
if got.read_text() != (served / rel).read_text():
test_fail(f"delivered {rel} content differs from source")

View File

@@ -6,10 +6,6 @@
# atimes-format variant. We avoid actually starting a listening server
# by using RSYNC_CONNECT_PROG to spawn the daemon as a child of rsync.
# Rerun under the fleet harness's non-root pass (testsuite/fleettest.py): a
# non-root rsyncd emits different uid/gid config, so exercise that path too.
fleet_nonroot = True
import os
import subprocess

View File

@@ -1,80 +0,0 @@
#!/usr/bin/env python3
# Functional regression: --delete-missing-args with --files-from aborts the
# transfer with "invalid file mode 00 ... protocol incompatibility (code 2)"
# instead of deleting the entries that are missing on the sender.
#
# Reported as #910 ("Security fix in flist.c breaks --delete-missing-args with
# --files-from").
#
# Root cause: for a --files-from entry that does not exist on the sender,
# --delete-missing-args==2 deliberately sends a "missing" file entry with
# mode == 0 (the generator's signal to delete it on the receiver). The 3.4.x
# security mode-validation added to recv_file_entry() (flist.c) rejects mode 0
# as an invalid file type BEFORE the generator can act on it, so the receiver
# bails out with a protocol error and nothing is deleted. Works in 3.4.1.
#
# Two scenarios, since a missing FILE and a missing DIRECTORY are sent as
# distinct mode-0 entries:
# * a regular file present on the receiver but absent on the sender, and
# * a directory present on the receiver but absent on the sender,
# both named in --files-from. Both must be deleted on the receiver.
#
# XFAIL until recv_file_entry() accepts the missing-args mode-0 entry again
# (the accompanying flist.c fix). Runs at any uid.
import subprocess
from rsyncfns import (
SCRATCHDIR, makepath, rmtree, rsync_argv, start_test_daemon, test_fail,
test_xfail, write_daemon_conf,
)
DAEMON_PORT = 12910
mod = SCRATCHDIR / 'recvmod910' # daemon receive module
src = SCRATCHDIR / 'src910'
rmtree(mod)
rmtree(src)
makepath(mod / 'ghostdir', src)
(src / 'keep.txt').write_text("keep-me\n") # present on sender
(mod / 'keep.txt').write_text("stale\n") # will be updated
(mod / 'ghost.txt').write_text("delete-me-file\n") # absent on sender -> delete
(mod / 'ghostdir' / 'inner').write_text("delete-me-dir\n") # absent on sender -> delete
# --files-from lists one present file plus the two entries that are missing on
# the sender (a file and a directory) -- those become mode-0 "missing" entries.
flist = SCRATCHDIR / 'files910.lst'
flist.write_text("keep.txt\nghost.txt\nghostdir\n")
conf = write_daemon_conf([
('recv', {'path': str(mod), 'read only': 'no'}),
])
url = start_test_daemon(conf, DAEMON_PORT)
proc = subprocess.run(
rsync_argv('-a', '--delete', '--delete-missing-args',
f'--files-from={flist}', f'{src}/', f'{url}recv/'),
stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True)
out = proc.stdout or ''
print(out)
# Bug present: the receiver rejects the mode-0 missing-args entry.
if 'invalid file mode' in out or (proc.returncode == 2 and (mod / 'ghost.txt').exists()):
test_xfail(
"#910: --delete-missing-args with --files-from aborts with "
"`invalid file mode 00 ... protocol incompatibility (code 2)`. The "
"sender sends mode-0 entries for the missing args (the delete signal), "
"but recv_file_entry()'s 3.4.x mode-validation rejects mode 0 before the "
"generator can delete them. To be closed by accepting the "
"missing-args mode-0 entry in recv_file_entry().")
# Bug fixed (or absent): both missing args were deleted, the present file kept.
if proc.returncode != 0:
test_fail(f"transfer failed unexpectedly (rc={proc.returncode}); "
f"not the #910 mode-00 symptom:\n{out}")
if (mod / 'ghost.txt').exists():
test_fail("missing-arg file ghost.txt was not deleted on the receiver")
if (mod / 'ghostdir').exists():
test_fail("missing-arg directory ghostdir was not deleted on the receiver")
if not (mod / 'keep.txt').is_file() or (mod / 'keep.txt').read_text() != "keep-me\n":
test_fail("present file keep.txt was not delivered/updated correctly")

View File

@@ -5,6 +5,7 @@
# from source to destination (other permission changes ignored), while a
# normal copy without -E should leave the destination permissions alone.
import errno
import os
from rsyncfns import FROMDIR, TODIR, check_perms, run_rsync, test_skipped
@@ -15,11 +16,14 @@ FROMDIR.mkdir(parents=True, exist_ok=True)
(FROMDIR / '2').write_text("#!/bin/sh\necho 'Program Two!'\n")
# Setuid-and-rwx for owner, nothing else. Some platforms reject 1700 for
# non-root callers (no permission to set sticky); the shell test treats
# that case as a skip.
# non-root callers (no permission to set sticky); FreeBSD rejects it with
# EFTYPE rather than EPERM. Only skip on those; re-raise anything unexpected.
_STICKY_SKIP_ERRNOS = {errno.EPERM, errno.EACCES, getattr(errno, 'EFTYPE', None)}
try:
os.chmod(FROMDIR / '1', 0o1700)
except PermissionError:
except OSError as e:
if e.errno not in _STICKY_SKIP_ERRNOS:
raise
test_skipped("Can't chmod")
os.chmod(FROMDIR / '2', 0o600)

View File

@@ -1,17 +0,0 @@
"""Exit codes a test reports to runtests.py (autotools test convention).
Shared by runtests.py (the harness, which reads these from each test) and
rsyncfns.py (the helpers, which exit with them) so the 0/1/2/77/78 values are
named in exactly one place. This module has no import-time side effects, so
runtests.py can import it without pulling in rsyncfns's environment checks.
"""
import enum
class Exit(enum.IntEnum):
PASS = 0
FAIL = 1
ERROR = 2 # the test could not run (e.g. missing environment)
SKIP = 77
XFAIL = 78 # expected failure: a known, documented residual

View File

@@ -1,62 +0,0 @@
#!/usr/bin/env python3
# Regression test for issue #880 (and the dry-run itemize regression that the
# first proposed fix, PR #952, would have introduced).
#
# (1) Copying file-to-file with --mkpath and --dry-run used to abort with
# "change_dir#3 ... failed", because make_path() only *reports* (does not
# create) directories in a dry run, so the later chdir found no parent.
#
# (2) The fix must stay scoped to the missing-parent case: a plain
# file-to-file --dry-run onto an *existing*, differing destination must
# still itemize the real change, not report the file as brand new (PR #952
# bumped dry_run unconditionally, which broke this).
#
# In both cases a "--dry-run -i" must produce the same itemized output as the
# real run. Based on the test from PR #952 by Stiliyan Tonev.
import os
import subprocess
from rsyncfns import SCRATCHDIR, makepath, rmtree, rsync_argv, test_fail
def itemize(*args):
p = subprocess.run(rsync_argv('-ai', *args), capture_output=True, text=True)
return p.returncode, p.stdout + p.stderr
# (1) --mkpath file-to-file: the dry run must succeed and match the real run.
mk = SCRATCHDIR / 'mk'
rmtree(mk)
makepath(mk / 'from')
(mk / 'from' / 'src').write_text("payload\n")
drc, dry = itemize('--dry-run', '--mkpath',
str(mk / 'from' / 'src'), str(mk / 'dndir' / 'dst'))
rc, real = itemize('--mkpath', str(mk / 'from' / 'src'), str(mk / 'rdir' / 'dst'))
if drc != 0:
print(dry)
test_fail("--mkpath file-to-file --dry-run failed (#880)")
if not (mk / 'rdir' / 'dst').exists():
test_fail("--mkpath real run did not create the file")
if dry.replace('dndir', 'X') != real.replace('rdir', 'X'):
test_fail(f"--mkpath dry-run output differs from the real run:\n"
f" dry : {dry!r}\n real: {real!r}")
# (2) Plain file-to-file onto an existing, differing destination: the dry run
# must itemize the same change as the real run (a/dst and b/dst share the
# basename 'dst', so the itemized lines are directly comparable).
ex = SCRATCHDIR / 'ex'
rmtree(ex)
makepath(ex / 'a')
makepath(ex / 'b')
(ex / 'src').write_text("brand new content\n")
for d in ('a', 'b'):
(ex / d / 'dst').write_text("old\n")
os.utime(ex / d / 'dst', (0, 0)) # make size + mtime differ
_, dry2 = itemize('--dry-run', str(ex / 'src'), str(ex / 'a' / 'dst'))
_, real2 = itemize(str(ex / 'src'), str(ex / 'b' / 'dst'))
if dry2 != real2:
test_fail(f"file-to-file --dry-run misreports an existing destination:\n"
f" dry : {dry2!r}\n real: {real2!r}")

View File

@@ -44,32 +44,6 @@ for rel in listed:
for rel in unlisted:
assert_not_exists(TODIR / rel, label=f'--from0 excluded {rel}')
# --- comments: line mode and --from0 both ignore them -----------------------
rmtree(TODIR)
(src / '#ignored').write_text('hash ignored\n')
(src / ';ignored').write_text('semi ignored\n')
commented = SCRATCHDIR / 'files-commented.lst'
commented.write_text('\n'.join(['', ';ignored', '#ignored', *listed]) + '\n')
run_rsync('-a', f'--files-from={commented}', f'{src}/', f'{TODIR}/')
for rel in listed:
assert_same(TODIR / rel, src / rel, label=f'--files-from comment list {rel}')
for rel in unlisted:
assert_not_exists(TODIR / rel, label=f'--files-from comment list excluded {rel}')
for rel in ['#ignored', ';ignored']:
assert_not_exists(TODIR / rel, label=f'--files-from comment list skipped {rel}')
rmtree(TODIR)
comments0 = SCRATCHDIR / 'files-comments0.lst'
comments0.write_bytes(
b'\0;ignored\0#ignored\0' + b'\0'.join(p.encode() for p in listed) + b'\0')
run_rsync('-a', '--from0', f'--files-from={comments0}', f'{src}/', f'{TODIR}/')
for rel in listed:
assert_same(TODIR / rel, src / rel, label=f'--from0 comment list {rel}')
for rel in unlisted:
assert_not_exists(TODIR / rel, label=f'--from0 comment list excluded {rel}')
for rel in ['#ignored', ';ignored']:
assert_not_exists(TODIR / rel, label=f'--from0 comment list skipped {rel}')
# --- --exclude-from drops matching files at depth ---------------------------
seed()
(src / 'a.skip').write_text('s\n')

View File

@@ -1,106 +0,0 @@
{
"_comment": [
"Example fleet definition for testsuite/fleettest.py -- this is one",
"maintainer's setup. Copy (or symlink) this file to testsuite/fleettest.json",
"and edit it for your own machines, or point at another file with --fleet PATH.",
"fleettest.json is git-ignored; this .example is the committed template.",
"",
"Each object under \"targets\" maps to fields of the Target dataclass in",
"fleettest.py. Required: name, ssh_host (null = run locally), workflow",
"(a file under .github/workflows, whose configure flags and RSYNC_EXPECT_SKIPPED",
"this target mirrors), configure_flags. Optional (with defaults): make (\"make\"),",
"python (\"python3\"), rsync_bin (\"rsync\"; \"rsync.exe\" on Cygwin), privilege",
"(\"root\" | \"sudo\" | \"user\"), pipe_jobs/tcp_jobs (8), builddir (\"rsync-citest\",",
"relative to the remote $HOME), env_prefix, configure_pre, nonroot, protocols.",
"",
"nonroot: true reruns -- as the non-root ssh user, after the sudo runs -- the",
"tests that declare `fleet_nonroot = True` at module level (so the set is",
"maintained in the test files, not here).",
"",
"protocols: [30, 29] adds one extra stdio-pipe test pass per listed version,",
"each run with runtests --protocol=N (the fleet analogue of a workflow's",
"check30/check29 steps) and shown as a protoNN column. Keys starting with",
"\"_\" are comments. See testsuite/README.md."
],
"targets": [
{
"name": "freebsd",
"ssh_host": "root@freebsd",
"workflow": "freebsd-build.yml",
"make": "gmake",
"configure_flags": ["--with-rrsync", "--disable-zstd", "--disable-md2man",
"--disable-xxhash", "--disable-lz4"]
},
{
"name": "solaris",
"ssh_host": "root@solaris",
"workflow": "solaris-build.yml",
"make": "gmake",
"configure_flags": ["--with-rrsync", "--disable-zstd", "--disable-md2man",
"--disable-xxhash", "--disable-lz4"]
},
{
"name": "openbsd",
"ssh_host": "root@openbsd",
"workflow": "openbsd-build.yml",
"make": "gmake",
"configure_pre": "export AUTOCONF_VERSION=2.71 AUTOMAKE_VERSION=1.16;",
"tcp_jobs": 2,
"configure_flags": ["--with-rrsync", "--disable-zstd", "--disable-md2man",
"--disable-xxhash", "--disable-lz4"]
},
{
"name": "netbsd",
"ssh_host": "root@netbsd",
"workflow": "netbsd-build.yml",
"make": "gmake",
"configure_flags": ["--with-rrsync", "--disable-zstd", "--disable-md2man",
"--disable-xxhash", "--disable-lz4"]
},
{
"_comment": "Ubuntu 20.04 older-LTS backport coverage on a root@ box; no 20.04 runner image exists so it mirrors the 22.04 workflow.",
"name": "ubuntu-2004",
"ssh_host": "root@ubuntu-2004",
"workflow": "ubuntu-22.04-build.yml",
"configure_flags": ["--with-rrsync"]
},
{
"_comment": "Builds unprivileged (like a CI runner) and runs the suite via sudo; the nonroot pass reruns the privilege-sensitive tests as the ssh user.",
"name": "ubuntu-2204",
"ssh_host": "runner@ubuntu-2204",
"workflow": "ubuntu-22.04-build.yml",
"privilege": "sudo",
"nonroot": true,
"configure_flags": ["--with-rrsync"]
},
{
"_comment": "Modern Ubuntu (mirrors ubuntu-build.yml). protocols: [30, 29] also runs the workflow's check30/check29 passes as extra stdio-pipe runs.",
"name": "ubuntu-2604",
"ssh_host": "runner@ubuntu-2604",
"workflow": "ubuntu-build.yml",
"privilege": "sudo",
"nonroot": true,
"protocols": [30, 29],
"configure_flags": ["--with-rrsync"]
},
{
"_comment": "macOS: brew is not on the non-interactive ssh PATH, so put it on PATH for the whole build and pass brew include/lib dirs to configure.",
"name": "mac2",
"ssh_host": "runner@mac2",
"workflow": "macos-build.yml",
"privilege": "sudo",
"env_prefix": "export PATH=/opt/homebrew/bin:/usr/local/bin:$PATH",
"configure_pre": "CPPFLAGS=\"-I$(brew --prefix)/include -I$(brew --prefix openssl)/include\"; LDFLAGS=\"-L$(brew --prefix)/lib -L$(brew --prefix openssl)/lib\"; export CPPFLAGS LDFLAGS;",
"configure_flags": ["--with-rrsync"]
},
{
"_comment": "Cygwin: non-root plain user (no sudo), binary is rsync.exe.",
"name": "cygwin",
"ssh_host": "win11",
"workflow": "cygwin-build.yml",
"rsync_bin": "rsync.exe",
"privilege": "user",
"configure_flags": ["--with-rrsync"]
}
]
}

View File

@@ -1,868 +0,0 @@
#!/usr/bin/env python3
"""Fleet CI harness for rsync.
Builds the committed HEAD of an rsync checkout on a fleet of remote machines
(over ssh), runs the test suite under both transports (default stdio-pipe and
--use-tcp) in parallel, and prints one report of only the UNEXPECTED results --
a fast local pre-flight for the GitHub CI matrix.
Each target maps 1:1 to a .github/workflows/*.yml job: the per-target configure
flags mirror that workflow, and the pipe-run RSYNC_EXPECT_SKIPPED list is PARSED
from the workflow (not hardcoded). The --use-tcp run never sets an expected-skip
list (matching the workflows), so only test FAILs matter there.
A target may also list older "protocols" (e.g. [30, 29]) in the fleet config:
each runs as an extra stdio-pipe pass with runtests --protocol=N (the fleet
analogue of a workflow's check30/check29 steps), using the same parsed skip list
as the pipe run, and shows up as a protoNN column in the report.
The fleet -- which machines, how to reach and build each -- is read from a JSON
config: ~/.fleettest.json if present, else fleettest.json next to this script,
or --fleet PATH. Copy the bundled fleettest.json.example to either location (or
symlink it) and edit for your own hosts; see testsuite/README.md and the
comments in fleettest.json.example.
Source = `git archive HEAD` of the rsync tree (the current directory, or --repo
PATH) -- source-only, no .o/binaries are ever pushed.
Every run uses its own randomly-named build directory on each target
(<builddir>-<run_id>), so two or three fleettest runs can share the same fleet
without interfering: each pushes, builds and tests in isolation. The run dir is
removed when the run ends -- on success or failure, and best-effort on
Ctrl-C/kill (pass --keep to retain it for inspection). A run that is hard-killed
(SIGKILL), or signalled mid-push, or whose ssh dies during cleanup can leave a
stray <builddir>-<id> behind; sweep those with `fleettest.py --cleanup`
(optionally scoped with --targets). Because each
run starts from a fresh dir, every build is a full configure + build.
PROVISIONING: each target must have the build toolchain its workflow's prepare
step installs -- the target regenerates its own configure/proto.h/man pages, so
it needs autoconf+automake, perl, a python3 markdown lib (cmarkgfm or commonmark)
unless its flags pass --disable-md2man, and the dev libraries for whatever its
configure flags enable (e.g. --with-rrsync needs openssl/xxhash/zstd/lz4 headers).
A missing piece shows up as BUILD-FAIL with configure's own "you need X" hint.
Per-target "privilege" (set in the JSON) controls how the suite runs: "root"
(already root -- run directly), "sudo" (build unprivileged, run the suite via
sudo to match a CI runner), or "user" (run directly as a plain non-root user). A
target with "nonroot": true additionally reruns -- as the (non-root) ssh user,
after the sudo runs -- every test that declares `fleet_nonroot = True` at module
level, so privilege-sensitive tests opt in from the test file itself with no
fleet-config edit when new ones are added.
Usage (run from inside an rsync checkout, or pass --repo):
python3 testsuite/fleettest.py # whole fleet, both transports
python3 testsuite/fleettest.py --targets cygwin,freebsd
python3 testsuite/fleettest.py --transport pipe
python3 testsuite/fleettest.py --keep # keep run dirs for inspection
python3 testsuite/fleettest.py --cleanup # sweep stray run dirs, exit
python3 testsuite/fleettest.py --fleet my-fleet.json --list
Exit 0 iff every selected (target x transport) cell is OK.
"""
from __future__ import annotations
import argparse
import atexit
import concurrent.futures
import dataclasses
import json
import os
import re
import secrets
import signal
import subprocess
import sys
import tempfile
import threading
import time
from pathlib import Path
# Set from --repo in main() (default: cwd). The harness builds whatever rsync
# source tree these point at, so it must be run from inside an rsync checkout
# or given --repo PATH.
REPO = Path.cwd()
WORKFLOWS = REPO / ".github" / "workflows"
# Fleet config (overridable with --fleet): ~/.fleettest.json is tried first, then
# fleettest.json next to this script. The example template sits next to the
# script too.
HOME_CONFIG = Path.home() / ".fleettest.json"
SCRIPT_CONFIG = Path(__file__).resolve().parent / "fleettest.json"
DEFAULT_CONFIGS = [HOME_CONFIG, SCRIPT_CONFIG]
EXAMPLE_CONFIG = SCRIPT_CONFIG.with_name(SCRIPT_CONFIG.name + ".example")
# The pushed tree is source-only (git archive). Each target regenerates its own
# build files, so --delete must NOT prune them: we exclude everything `make`
# produces (autotools output, proto.h, man pages, config.h/Makefile, *.o, the
# binaries) plus test artifacts a prior sudo run left root-owned (testtmp,
# __pycache__, *.pyc -- which a non-root --delete can't unlink). Excluded paths
# are protected from --delete, so each target keeps its native build state for
# incremental rebuilds. `configure` itself is committed, so it is NOT excluded.
PUSH_EXCLUDES = [
".git", "config.h", "config.status", "config.log", "Makefile", "shconfig",
"configure.sh", "config.h.in", "aclocal.m4", "proto.h", "git-version.h",
"/rsync.1", "/rsync-ssl.1", "/rsyncd.conf.5", "/rrsync.1",
"*.o", "*.exe", "__pycache__", "*.pyc", "/testtmp",
"/rsync", "/tls", "/getgroups", "/getfsdev", "/trimslash", "/wildtest",
"/testrun", "/simdtest", "/t_unsafe", "/t_chmod_secure", "/t_rename_secure",
"/t_symlink_secure", "/t_secure_relpath",
]
@dataclasses.dataclass
class Target:
name: str
ssh_host: str | None # null in JSON => run locally
workflow: str # filename under .github/workflows
configure_flags: list[str]
make: str = "make" # e.g. "gmake" on the BSDs/Solaris
env_prefix: str = "" # exported before configure AND make (e.g. PATH)
configure_pre: str = "" # shell run before ./configure (env exports, brew)
python: str = "python3"
rsync_bin: str = "rsync" # "rsync.exe" on Cygwin
privilege: str = "root" # "root" (already root) | "sudo" | "user" (plain, no sudo)
pipe_jobs: int = 8
tcp_jobs: int = 8
# Base build-dir name (relative to remote $HOME; absolute for local). A
# per-run random suffix is appended (-> <builddir>-<run_id>) so concurrent
# fleettest runs don't share a tree; --cleanup sweeps leftover <builddir>-*.
builddir: str = "rsync-citest"
# When true, after the sudo runs, additionally run -- as the (non-root) ssh
# user -- every test that declares `fleet_nonroot = True` (see
# discover_nonroot_tests). Mirrors a workflow's non-root check step.
nonroot: bool = False
# Older protocol versions to additionally exercise, each as a separate
# stdio-pipe pass with runtests --protocol=N (the fleet analogue of a
# workflow's check30/check29 steps). e.g. [30, 29]. Empty => proto pass off.
protocols: list[int] = dataclasses.field(default_factory=list)
def load_fleet(path: Path) -> list[Target]:
"""Load the fleet from a JSON file of the shape {"targets": [ {...}, ... ]}.
Each entry's keys are Target fields; keys starting with "_" are treated as
comments and ignored (both at top level and per target). Validation errors
name the offending target so a typo is easy to find."""
try:
data = json.loads(path.read_text())
except OSError as e:
sys.exit(f"cannot read fleet config {path}: {e}")
except json.JSONDecodeError as e:
sys.exit(f"invalid JSON in {path}: {e}")
if not isinstance(data, dict) or not isinstance(data.get("targets"), list):
sys.exit(f'{path}: expected a JSON object with a "targets" array')
fields = {f.name for f in dataclasses.fields(Target)}
fleet: list[Target] = []
for i, entry in enumerate(data["targets"]):
if not isinstance(entry, dict):
sys.exit(f"{path}: targets[{i}] is not an object")
entry = {k: v for k, v in entry.items() if not k.startswith("_")}
who = entry.get("name", f"targets[{i}]")
bad = set(entry) - fields
if bad:
sys.exit(f"{path}: target {who!r} has unknown key(s): "
f"{', '.join(sorted(bad))}")
try:
fleet.append(Target(**entry))
except TypeError as e:
sys.exit(f"{path}: target {who!r}: {e}")
if not fleet:
sys.exit(f"{path}: no targets defined")
return fleet
# ---------------------------------------------------------------------------
# command execution (ssh for remote, local shell when ssh_host is null)
# ---------------------------------------------------------------------------
@dataclasses.dataclass
class CmdResult:
rc: int
out: str # combined stdout + stderr
timed_out: bool = False
def run_on(target: Target, script: str, timeout: int) -> CmdResult:
"""Run a /bin/sh script on the target. Remote via ssh, else local."""
if target.ssh_host:
argv = ["ssh", "-o", "BatchMode=yes", "-o", "ConnectTimeout=15",
target.ssh_host, script]
else:
argv = ["/bin/sh", "-c", script]
try:
p = subprocess.run(argv, capture_output=True, text=True, timeout=timeout)
return CmdResult(p.returncode, (p.stdout or "") + (p.stderr or ""))
except subprocess.TimeoutExpired as e:
out = (e.stdout or b"") + (e.stderr or b"")
if isinstance(out, bytes):
out = out.decode(errors="replace")
return CmdResult(124, out, timed_out=True)
except FileNotFoundError as e:
return CmdResult(127, str(e))
def push_argv(target: Target, staging: str) -> list[str]:
# -rlpgoD = -a without -t: do NOT preserve mtimes. The host clock can be
# hours AHEAD of a target, so preserved (commit-time) mtimes land "in the
# future" there and rsync's `Makefile: Makefile.in config.status` rule
# triggers a config.status/autoconf regeneration storm. Letting files take
# the target's own clock avoids that. --checksum keeps the transfer
# incremental despite the unstable mtimes (decide by content, not size+time).
args = ["rsync", "-rlpgoD", "--checksum", "--delete"]
for ex in PUSH_EXCLUDES:
args.append(f"--exclude={ex}")
dst = f"{target.ssh_host}:{target.builddir}/" if target.ssh_host \
else f"{target.builddir}/"
args += [f"{staging}/", dst]
return args
# ---------------------------------------------------------------------------
# workflow skip-list parsing
# ---------------------------------------------------------------------------
# The trailing '? tolerates a `bash -c '... make check'` wrapper (e.g. Cygwin).
_SKIP_RE = re.compile(r"RSYNC_EXPECT_SKIPPED=(\S+)\s+make\s+check'?\s*$", re.M)
def parse_workflow_skip(workflow: str) -> str | None:
"""Return the literal RSYNC_EXPECT_SKIPPED csv for the `make check` step, or
None if the workflow leaves it unset."""
path = WORKFLOWS / workflow
try:
text = path.read_text()
except OSError:
return None
m = _SKIP_RE.search(text)
return m.group(1) if m else None
# ---------------------------------------------------------------------------
# non-root test discovery
# ---------------------------------------------------------------------------
# A test opts into the fleet's extra non-root pass by setting a module-level
# `fleet_nonroot = True`. We read it with a text scan rather than importing the
# module (test files execute their body on import), so a new privilege-sensitive
# test joins the pass just by carrying the marker -- no fleet-config edit needed.
_NONROOT_RE = re.compile(r"^[ \t]*fleet_nonroot[ \t]*=[ \t]*True\b", re.M)
def discover_nonroot_tests(testsuite_dir: Path) -> list[str]:
"""Return the names (without the _test.py suffix) of the tests under
testsuite_dir that declare `fleet_nonroot = True`."""
names = []
for p in sorted(testsuite_dir.glob("*_test.py")):
try:
if _NONROOT_RE.search(p.read_text(errors="replace")):
names.append(p.name[: -len("_test.py")])
except OSError:
continue
return names
# ---------------------------------------------------------------------------
# remote script builders
# ---------------------------------------------------------------------------
def build_script(t: Target) -> str:
flags = " ".join(t.configure_flags)
# configure only when not yet configured (keeps incremental builds fast);
# --clean wipes the builddir beforehand so Makefile is absent -> reconfigure.
pre = f'{t.env_prefix}\n' if t.env_prefix else ''
return (
f'cd {t.builddir} || exit 3\n'
f'{pre}'
f'if [ ! -f Makefile ]; then {t.configure_pre} ./configure {flags} || exit 4; fi\n'
f'{t.make} -j{t.pipe_jobs} check-progs || exit 5\n'
)
def test_script(t: Target, transport: str, skip_csv: str | None, jobs: int,
protocol: int | None = None) -> str:
rb = f'--rsync-bin="$PWD/{t.rsync_bin}"'
tcp = " --use-tcp" if transport == "tcp" else ""
# protocol forces an older wire version (mirrors `make check30`/`check29`).
proto = f" --protocol={protocol}" if protocol is not None else ""
# PYTHONDONTWRITEBYTECODE: don't drop root-owned __pycache__/*.pyc into the
# tree (a sudo run would, breaking the next non-root push --delete).
env = "PYTHONDONTWRITEBYTECODE=1 "
if skip_csv:
env += f"RSYNC_EXPECT_SKIPPED={skip_csv} "
runtests = f'{t.python} runtests.py {rb}{tcp}{proto} -j {jobs}'
# env_prefix (e.g. a brew PATH) must reach the test too: some tests build a
# helper binary on the fly (a test may invoke `make`, which needs gawk etc.),
# so the build tools must be on PATH at test time.
pre = f'{t.env_prefix}; ' if t.env_prefix else ''
if t.privilege == "sudo":
# -n: never prompt (capture_output has no TTY -- a prompt would hang
# the whole timeout). Targets need passwordless sudo or a fresh
# `sudo -v`. env keeps the vars (and PATH) across the sudo boundary.
path_pass = 'PATH="$PATH" ' if t.env_prefix else ''
cmd = f"{pre}sudo -n env {path_pass}{env}{runtests}"
else:
cmd = pre + env + runtests
return f'cd {t.builddir} || exit 3\n{cmd}\n'
def nonroot_test_script(t: Target, names: list[str]) -> str:
"""Run the given tests as the (non-root) ssh user -- the fleet analogue of a
workflow's non-root check step. Explicit test names make runtests.py
full_run False, so no RSYNC_EXPECT_SKIPPED is involved; only FAILs matter.
The prior sudo pipe/tcp runs left testtmp root-owned, so clear it (via sudo)
before the non-root run recreates it."""
pre = f'{t.env_prefix}; ' if t.env_prefix else ''
runtests = (f'PYTHONDONTWRITEBYTECODE=1 {t.python} runtests.py '
f'--rsync-bin="$PWD/{t.rsync_bin}" {" ".join(names)}')
return (f'cd {t.builddir} || exit 3\n'
f'sudo -n rm -rf testtmp\n'
f'{pre}{runtests}\n')
# ---------------------------------------------------------------------------
# runtests.py output parsing
# ---------------------------------------------------------------------------
RE_RESULT = re.compile(r"^(PASS|FAIL|ERROR|XFAIL|SKIP)\s+(\S+)", re.M)
RE_COUNT = re.compile(r"^\s+(\d+)\s+(passed|failed|xfailed|skipped)\b", re.M)
RE_SKIP_HDR = re.compile(r"^----- skipped results:", re.M)
RE_SKIP_EXP = re.compile(r"^\s+expected:\s*(.*)$", re.M)
RE_SKIP_GOT = re.compile(r"^\s+got:\s*(.*)$", re.M)
def _csv_set(s: str) -> set[str]:
return {x for x in s.strip().split(",") if x}
@dataclasses.dataclass
class TransportResult:
transport: str
exit_code: int
timed_out: bool
counts: dict[str, int]
failed: list[str]
skip_checked: bool
skip_expected: set[str]
skip_got: set[str]
raw: str
@property
def skip_mismatch(self) -> bool:
return self.skip_checked and self.skip_expected != self.skip_got
@property
def ok(self) -> bool:
return (not self.timed_out and self.exit_code == 0
and not self.failed and not self.skip_mismatch)
def parse_transport(transport: str, r: CmdResult, skip_checked: bool) -> TransportResult:
counts = {"passed": 0, "failed": 0, "xfailed": 0, "skipped": 0}
for m in RE_COUNT.finditer(r.out):
counts[m.group(2)] = int(m.group(1))
failed = [m.group(2) for m in RE_RESULT.finditer(r.out)
if m.group(1) in ("FAIL", "ERROR")]
exp = got = set()
if skip_checked and RE_SKIP_HDR.search(r.out):
em = RE_SKIP_EXP.search(r.out)
gm = RE_SKIP_GOT.search(r.out)
exp = _csv_set(em.group(1)) if em else set()
got = _csv_set(gm.group(1)) if gm else set()
return TransportResult(transport, r.rc, r.timed_out, counts, failed,
skip_checked, exp, got, r.out)
@dataclasses.dataclass
class TargetResult:
target: str
reachable: bool = True
pushed: bool = True
build_ok: bool = True
error: str = ""
build_log: str = ""
transports: dict[str, TransportResult] = dataclasses.field(default_factory=dict)
# Wall-clock seconds per phase (push/build/pipe/tcp/nonroot) plus "total";
# populated for --timing. Phases run sequentially, so they sum to the total.
timings: dict[str, float] = dataclasses.field(default_factory=dict)
# ---------------------------------------------------------------------------
# per-target worker
# ---------------------------------------------------------------------------
_print_lock = threading.Lock()
def log(msg: str) -> None:
with _print_lock:
print(msg, flush=True)
def run_target(t: Target, args, staging: str) -> TargetResult:
res = TargetResult(t.name)
log(f"[{t.name}] start")
started = time.monotonic()
if t.ssh_host:
ping = run_on(t, "echo ok", timeout=25)
if ping.rc != 0:
res.reachable = False
res.error = f"ssh unreachable (rc={ping.rc}): {ping.out.strip()[:200]}"
log(f"[{t.name}] UNREACHABLE")
return res
# Always push: the run dir is freshly named per run, so there is no prior
# tree to reuse -- every run is a full configure + build.
t0 = time.monotonic()
push = subprocess.run(push_argv(t, staging),
capture_output=True, text=True, timeout=600)
res.timings["push"] = time.monotonic() - t0
if push.returncode != 0:
res.pushed = False
res.error = f"push failed (rc={push.returncode}): {push.stderr.strip()[:300]}"
log(f"[{t.name}] PUSH-FAIL")
return res
t0 = time.monotonic()
b = run_on(t, build_script(t), timeout=1200)
res.timings["build"] = time.monotonic() - t0
res.build_ok = b.rc == 0
res.build_log = b.out
if not res.build_ok:
log(f"[{t.name}] BUILD-FAIL")
return res
for transport in args.transports:
skip_csv = parse_workflow_skip(t.workflow) if transport == "pipe" else None
jobs = (args.jobs if args.jobs else
(t.tcp_jobs if transport == "tcp" else t.pipe_jobs))
cmd = test_script(t, transport, skip_csv, jobs)
t0 = time.monotonic()
r = run_on(t, cmd, timeout=2400)
res.timings[transport] = time.monotonic() - t0
res.transports[transport] = parse_transport(transport, r, skip_csv is not None)
log(f"[{t.name}] {transport} done "
f"({'ok' if res.transports[transport].ok else 'ISSUE'})")
# Extra older-protocol passes (mirroring the workflow's check30/check29
# steps): same stdio-pipe transport and skip list as `make check`, but with
# runtests --protocol=N forcing an older wire version. Only targets that list
# `protocols` opt in; skipped under --transport tcp (these are pipe runs).
if t.protocols and "pipe" in args.transports:
skip_csv = parse_workflow_skip(t.workflow)
jobs = args.jobs if args.jobs else t.pipe_jobs
for proto in t.protocols:
label = f"proto{proto}"
cmd = test_script(t, "pipe", skip_csv, jobs, protocol=proto)
t0 = time.monotonic()
r = run_on(t, cmd, timeout=2400)
res.timings[label] = time.monotonic() - t0
res.transports[label] = parse_transport(label, r, skip_csv is not None)
log(f"[{t.name}] {label} done "
f"({'ok' if res.transports[label].ok else 'ISSUE'})")
# Extra non-root pass (after the sudo runs) for targets that opt in, running
# the tests that declare `fleet_nonroot = True` (discovered in main()).
if t.nonroot and args.nonroot_tests:
t0 = time.monotonic()
r = run_on(t, nonroot_test_script(t, args.nonroot_tests), timeout=2400)
res.timings["nonroot"] = time.monotonic() - t0
res.transports["nonroot"] = parse_transport("nonroot", r, skip_checked=False)
log(f"[{t.name}] nonroot done "
f"({'ok' if res.transports['nonroot'].ok else 'ISSUE'})")
res.timings["total"] = time.monotonic() - started
return res
# ---------------------------------------------------------------------------
# reporting
# ---------------------------------------------------------------------------
def cell_status(res: TargetResult, transport: str) -> str:
if not res.reachable:
return "UNREACHABLE"
if not res.pushed:
return "PUSH-FAIL"
if not res.build_ok:
return "BUILD-FAIL"
tr = res.transports.get(transport)
if tr is None:
return "-"
if tr.timed_out:
return "TIMEOUT"
if tr.failed:
return f"FAIL({len(tr.failed)})"
if tr.skip_mismatch:
return "SKIP-MISMATCH"
if tr.exit_code != 0:
return f"EXIT({tr.exit_code})"
return "OK"
def print_report(results: list[TargetResult], args, fleet: list[Target]) -> bool:
by_name = {t.name: t for t in fleet}
order = {t.name: i for i, t in enumerate(fleet)}
results.sort(key=lambda r: order.get(r.target, 99))
# protoNN columns appear only when some target ran that older-protocol pass;
# the 'nonroot' column only when some target ran a non-root pass. Targets
# without a given pass show "-" there (a neutral N/A, not a failure).
transports = list(args.transports)
protos = {k for r in results for k in r.transports if k.startswith("proto")}
# highest protocol first (proto30 before proto29), matching check30/check29.
transports += sorted(protos, key=lambda c: int(c[len("proto"):]), reverse=True)
if any("nonroot" in r.transports for r in results):
transports.append("nonroot")
ts = time.strftime("%Y-%m-%d %H:%M")
print("\n" + "=" * 64)
print(f"rsync fleet CI — branch {current_branch()}{ts}")
print(f"source: HEAD run: {args.run_id} "
f"transports: {','.join(args.transports)}")
print("(A target's pipe skip-set is only enforced when its workflow sets "
"RSYNC_EXPECT_SKIPPED; otherwise only FAILs matter. The 'nonroot' "
"column runs the privilege-sensitive tests as the unprivileged user; "
"'-' = N/A.)")
print("=" * 64)
width = max(len(t) for t in order) + 2
header = "TARGET".ljust(width) + "".join(tr.upper().ljust(16) for tr in transports)
print(header)
all_ok = True
for res in results:
row = res.target.ljust(width)
for transport in transports:
st = cell_status(res, transport)
if st not in ("OK", "-"): # "-" = N/A (e.g. no nonroot pass)
all_ok = False
row += st.ljust(16)
# data-driven row notes: local target, or a target with a distinct tcp -j
t = by_name.get(res.target)
notes = []
if t is not None:
if t.ssh_host is None:
notes.append("(local)")
if "tcp" in transports and t.tcp_jobs != t.pipe_jobs:
notes.append(f"(tcp -j{t.tcp_jobs})")
print(row + " ".join(notes))
# detail section: only the unexpected cells
details: list[str] = []
for res in results:
if not res.reachable:
details.append(f"{res.target} — UNREACHABLE: {res.error}")
continue
if not res.pushed:
details.append(f"{res.target} — PUSH-FAIL: {res.error}")
continue
if not res.build_ok:
tail = "\n ".join(res.build_log.strip().splitlines()[-20:])
details.append(f"{res.target} — BUILD-FAIL:\n {tail}")
continue
for transport in transports:
tr = res.transports.get(transport)
if tr is None or tr.ok:
continue
if tr.timed_out:
details.append(f"{res.target} / {transport} — TIMEOUT")
if tr.failed:
details.append(f"{res.target} / {transport}{len(tr.failed)} failed:\n "
+ " ".join(tr.failed))
if tr.skip_mismatch:
extra = tr.skip_got - tr.skip_expected
missing = tr.skip_expected - tr.skip_got
diff = []
if extra:
diff.append(f"unexpected skips: {','.join(sorted(extra))}")
if missing:
diff.append(f"expected-but-ran: {','.join(sorted(missing))}")
details.append(f"{res.target} / {transport} — skip mismatch ("
+ "; ".join(diff) + ")\n"
f" expected: {','.join(sorted(tr.skip_expected))}\n"
f" got: {','.join(sorted(tr.skip_got))}")
elif not tr.failed and not tr.timed_out and tr.exit_code != 0:
details.append(f"{res.target} / {transport} — runtests exit {tr.exit_code}")
# Exclude N/A ("-") cells (e.g. the nonroot column for targets that don't
# run a non-root pass) from the OK/not-OK tally.
statuses = [cell_status(res, transport)
for res in results for transport in transports]
cells = sum(1 for s in statuses if s != "-")
ok_cells = sum(1 for s in statuses if s == "OK")
print("=" * 64)
if details:
print("==== UNEXPECTED RESULTS ====")
for d in details:
print(d)
print("=" * 64)
print(f"{len(results)} targets x {len(transports)} transports = {cells} cells: "
f"{ok_cells} OK, {cells - ok_cells} not OK")
return all_ok
# Phase columns for --timing, in execution order (push -> build -> tests).
_TIMING_PHASES = ("push", "build", "pipe", "tcp", "nonroot")
def _fmt_dur(s: float) -> str:
if s < 60:
return f"{s:.0f}s"
m, sec = divmod(int(round(s)), 60)
return f"{m}m{sec:02d}s"
def print_timing(results: list[TargetResult]) -> None:
"""Per-target wall-clock breakdown, slowest first. Targets run in parallel,
so the whole run is gated by the slowest one -- that's the hold-up; the
phase columns show whether it's push, build or the test passes."""
timed = [r for r in results if r.timings]
if not timed:
return
# Insert any protoNN phases (highest first) just before nonroot, in run order.
protos = sorted({k for r in timed for k in r.timings if k.startswith("proto")},
key=lambda c: int(c[len("proto"):]), reverse=True)
order = [p for p in _TIMING_PHASES if p != "nonroot"] + protos + ["nonroot"]
phases = [p for p in order if any(p in r.timings for r in timed)]
def total(r: TargetResult) -> float:
# Failed-early targets have no "total"; sum the phases they did reach.
return r.timings.get("total") or sum(r.timings.get(p, 0.0) for p in phases)
timed.sort(key=total, reverse=True)
width = max([len("TARGET")] + [len(r.target) for r in timed]) + 2
print("\n==== TIMING (slowest target first) ====")
print("TARGET".ljust(width) + "TOTAL".ljust(9)
+ "".join(p.upper().ljust(9) for p in phases))
for r in timed:
row = r.target.ljust(width) + _fmt_dur(total(r)).ljust(9)
for p in phases:
v = r.timings.get(p)
row += (_fmt_dur(v) if v is not None else "-").ljust(9)
print(row)
slow = timed[0]
print(f"hold-up: {slow.target} at {_fmt_dur(total(slow))} gates the run "
"(targets run in parallel)")
def current_branch() -> str:
try:
return subprocess.run(["git", "-C", str(REPO), "rev-parse",
"--abbrev-ref", "HEAD"],
capture_output=True, text=True).stdout.strip() or "?"
except Exception:
return "?"
# ---------------------------------------------------------------------------
# run-dir cleanup
# ---------------------------------------------------------------------------
# Targets whose per-run dir (t.builddir, already suffixed with the run_id) this
# process must remove on exit. Populated in main() once the run_id is applied.
_cleanup_targets: list[Target] = []
_cleanup_lock = threading.Lock()
_cleanup_done = False
def _unsafe_builddir(path: str) -> bool:
"""True if `path` is too broad to feed to `rm -rf` -- empty, root, $HOME, or
an absolute path sitting directly under / (e.g. /tmp). A real run dir is
always nested deeper, so this rejects an obvious builddir misconfiguration
before any destructive command is built."""
p = (path or "").rstrip("/")
if p in ("", "/", "~") or os.path.expanduser(p) == os.path.expanduser("~"):
return True
return os.path.isabs(p) and os.path.dirname(p) == "/"
def cleanup_run() -> None:
"""Best-effort `rm -rf` of this run's dir on every chosen target. Idempotent
(atexit + a signal handler may both call it). Each target removes only its
own <base>-<run_id> dir, so a concurrent run's dir is never touched."""
global _cleanup_done
with _cleanup_lock:
if _cleanup_done or not _cleanup_targets:
return
_cleanup_done = True
targets = list(_cleanup_targets)
for t in targets:
if _unsafe_builddir(t.builddir):
continue
run_on(t, f'rm -rf -- {t.builddir}', timeout=60)
def _on_signal(signum, frame):
cleanup_run()
# Skip atexit/thread-join: worker threads' ssh calls can't be cancelled and
# would otherwise block exit until they return. The remote build/test simply
# errors out now that its dir is gone.
os._exit(130 if signum == signal.SIGINT else 143)
def cleanup_remnants(targets: list[Target]) -> int:
"""--cleanup mode: remove every <base>-* run dir on each target, reporting
what each removed. Returns a process exit code. Only suffixed run dirs are
swept -- a bare <base> is left alone."""
rc = 0
for t in targets:
base = t.builddir
if _unsafe_builddir(base):
log(f"[{t.name}] skipped (unsafe builddir {base!r})")
continue
# Echo each match before removing it so the harness can report what
# went; an unmatched glob stays literal and is skipped by the -e test.
script = (f'set -e\n'
f'for d in {base}-*; do\n'
f' [ -e "$d" ] || continue\n'
f' echo "$d"\n'
f' rm -rf -- "$d"\n'
f'done\n')
r = run_on(t, script, timeout=120)
removed = [ln for ln in r.out.splitlines() if ln.strip()]
if r.rc != 0:
rc = 1
log(f"[{t.name}] cleanup error (rc={r.rc}): {r.out.strip()[:200]}")
elif removed:
log(f"[{t.name}] removed: {' '.join(removed)}")
else:
log(f"[{t.name}] nothing to remove")
return rc
# ---------------------------------------------------------------------------
# main
# ---------------------------------------------------------------------------
def main() -> int:
ap = argparse.ArgumentParser(description="Fleet CI harness for rsync.")
ap.add_argument("--targets", help="comma-separated subset (default: all)")
ap.add_argument("--transport", choices=["pipe", "tcp", "both"], default="both")
ap.add_argument("--keep", action="store_true",
help="keep each run's build dir (default: remove it at exit)")
ap.add_argument("--cleanup", action="store_true",
help="remove stray <builddir>-* run dirs on the targets, then exit")
ap.add_argument("--jobs", type=int, help="override -j for both transports")
ap.add_argument("--timing", action="store_true",
help="report per-target wall-clock (push/build/test) to find "
"the slowest target")
ap.add_argument("--repo", help="rsync source tree to build (default: cwd)")
ap.add_argument("--fleet", help="fleet config JSON (default: ~/.fleettest.json, "
"else fleettest.json next to this script)")
ap.add_argument("--list", action="store_true", help="list targets and exit")
args = ap.parse_args()
global REPO, WORKFLOWS
REPO = Path(args.repo).resolve() if args.repo else Path.cwd()
WORKFLOWS = REPO / ".github" / "workflows"
if not args.cleanup and not (REPO / "runtests.py").is_file():
print(f"{REPO} is not an rsync source tree (no runtests.py); "
f"run from inside a checkout or pass --repo", file=sys.stderr)
return 2
if args.fleet:
config_path = Path(args.fleet).resolve()
if not config_path.exists():
print(f"no fleet config at {config_path}", file=sys.stderr)
return 2
else:
config_path = next((p for p in DEFAULT_CONFIGS if p.exists()), None)
if config_path is None:
tried = " or ".join(str(p) for p in DEFAULT_CONFIGS)
print(f"no fleet config found (looked for {tried})\n"
f"copy {EXAMPLE_CONFIG} to {SCRIPT_CONFIG} or {HOME_CONFIG} "
f"(or pass --fleet PATH)", file=sys.stderr)
return 2
fleet = load_fleet(config_path)
if args.list:
for t in fleet:
host = t.ssh_host or "(local)"
skip = parse_workflow_skip(t.workflow)
proto = (",".join(f"proto{p}" for p in t.protocols)
if t.protocols else "none")
print(f"{t.name:12} {host:18} {t.make:6} "
f"pipe-skip={'set' if skip else 'unset'} protocols={proto}")
return 0
chosen = fleet
if args.targets:
want = [s.strip() for s in args.targets.split(",") if s.strip()]
by_name = {t.name: t for t in fleet}
bad = [w for w in want if w not in by_name]
if bad:
print(f"unknown target(s): {', '.join(bad)}", file=sys.stderr)
print(f"known: {', '.join(by_name)}", file=sys.stderr)
return 2
chosen = [by_name[w] for w in want]
if args.cleanup:
# Sweep every <builddir>-* run dir on the selected targets. NB: this
# also removes dirs belonging to runs that are still in progress, so
# only run it when no other fleettest runs are active (or scope with
# --targets).
return cleanup_remnants(chosen)
args.transports = ["pipe", "tcp"] if args.transport == "both" else [args.transport]
# Give this run its own build dir on every target so concurrent runs don't
# collide: <builddir>-<run_id>. The base name is the prefix --cleanup globs.
args.run_id = secrets.token_hex(3)
for t in chosen:
t.builddir = f"{t.builddir}-{args.run_id}"
log(f"run {args.run_id}: build dir <target>:{chosen[0].builddir} "
f"(removed at exit; --keep to retain)")
# Remove each run dir when we exit -- success or failure, and best-effort on
# Ctrl-C/kill (a signal mid-push may still leave a remnant). SIGKILL can't be
# caught; `fleettest.py --cleanup` sweeps any such remnant.
if not args.keep:
_cleanup_targets.extend(chosen)
atexit.register(cleanup_run)
signal.signal(signal.SIGINT, _on_signal)
signal.signal(signal.SIGTERM, _on_signal)
# Stage committed HEAD (source-only). Each target regenerates its own
# build files with its own toolchain -- exactly like the CI jobs, which
# install autotools / python-markdown / dev-libs in their prepare step.
# (Pushing locally-generated files instead fights rsync's Makefile
# maintainer rules: a target with a different autoconf version sees
# "configure.sh has CHANGED" and errors.) So each target must be
# provisioned like its workflow -- see the module docstring.
staging = tempfile.mkdtemp(prefix="rsync-fleettest-stage.")
try:
ar = subprocess.run(f"git -C {REPO} archive HEAD | tar -x -C {staging}",
shell=True, capture_output=True, text=True)
if ar.returncode != 0:
print(f"git archive failed: {ar.stderr}", file=sys.stderr)
return 2
# Tests that opt into the non-root pass (same for every target).
args.nonroot_tests = discover_nonroot_tests(Path(staging) / "testsuite")
results: list[TargetResult] = []
with concurrent.futures.ThreadPoolExecutor(max_workers=len(chosen)) as ex:
futs = {ex.submit(run_target, t, args, staging): t for t in chosen}
for fut in concurrent.futures.as_completed(futs):
t = futs[fut]
try:
results.append(fut.result())
except Exception as e: # never let one target kill the run
r = TargetResult(t.name)
r.reachable = False
r.error = f"harness exception: {e!r}"
results.append(r)
finally:
subprocess.run(["rm", "-rf", staging])
all_ok = print_report(results, args, fleet)
if args.timing:
print_timing(results)
return 0 if all_ok else 1
if __name__ == "__main__":
sys.exit(main())

View File

@@ -1,67 +0,0 @@
#!/usr/bin/env python3
# Security guard for the #915 re-anchor: a daemon receiver must NOT honour an
# alt-basis dir whose `..` climbs OUT of the module.
#
# Honouring a relative --link-dest=../01 again (#915) deliberately re-permits an
# in-module `..` climb (dest 00 -> sibling basis 01). This test pins the other
# side of that boundary: a client-supplied --link-dest=../../OUTSIDE that points
# at a file OUTSIDE the module root must be refused, so the basis is never used
# and the dest file is re-transferred rather than hard-linked to the outside
# file (which would be an info-leak / cross-module hard-link).
#
# The re-anchor confines resolution beneath module_dir with RESOLVE_BENEATH, so
# the escaping climb is rejected in-kernel; on platforms without
# openat2/O_RESOLVE_BENEATH the portable resolver rejects the `..` outright.
# Either way the escape is blocked, so this test must PASS on every platform.
# Runs at any uid.
import shutil
import subprocess
from rsyncfns import (
SCRATCHDIR, make_data_file, makepath, rmtree, rsync_argv, start_test_daemon,
test_fail, write_daemon_conf,
)
DAEMON_PORT = 12916
DATA_SIZE = 40000
mod = SCRATCHDIR / 'escmod' # daemon module root (holds dest 00)
src = SCRATCHDIR / 'escsrc'
outside = SCRATCHDIR / 'OUTSIDE' # sibling of the module root -- OUTSIDE it
for d in (mod, src, outside):
rmtree(d)
makepath(mod / '00', src, outside)
# Source file, plus a byte-identical secret OUTSIDE the module with the same
# name/size/mtime (so a followed basis would quick-check as a match).
make_data_file(src / 'f.dat', DATA_SIZE)
shutil.copy2(src / 'f.dat', outside / 'f.dat')
conf = write_daemon_conf([
('bak', {'path': str(mod), 'read only': 'no'}),
])
url = start_test_daemon(conf, DAEMON_PORT)
# Dest is bak/00 (cwd = module/00). --link-dest=../../OUTSIDE climbs
# module/00 -> module -> SCRATCHDIR/OUTSIDE, i.e. out of the module.
proc = subprocess.run(
rsync_argv('-a', '--link-dest=../../OUTSIDE', f'{src}/', f'{url}bak/00/'),
stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True)
out = proc.stdout or ''
if proc.returncode not in (0, 23): # 23: a basis rejection is non-fatal here
test_fail(f"escape push failed unexpectedly (rc={proc.returncode}):\n{out}")
dest = mod / '00' / 'f.dat'
secret = outside / 'f.dat'
if not dest.is_file():
test_fail(f"destination file missing ({dest})")
ds, ss = dest.stat(), secret.stat()
if (ds.st_dev, ds.st_ino) == (ss.st_dev, ss.st_ino):
test_fail(
"MODULE ESCAPE: the dest was hard-linked to a file OUTSIDE the module "
f"via --link-dest=../../OUTSIDE -- the confined resolver let a `..` "
f"climb escape the module root.\n{out}")
# Escape blocked: the basis was refused, so the file was re-transferred and the
# dest is its own inode, not the outside secret's.

View File

@@ -1,63 +0,0 @@
#!/usr/bin/env python3
# Functional regression: a relative --link-dest=../sibling against a daemon
# module with `path = /` (the intersection of #897 and #915).
#
# #915 re-anchors the receiver's basis open at the module root so an in-module
# "../01" climb is honoured. The gate keyed on a nonzero module_dirlen, but a
# `path = /` module has module_dirlen == 0 (clientserver.c), so the re-anchor
# was skipped there and --link-dest=../01 was silently ignored (every file
# re-transferred) even though plain #915 modules were fixed.
#
# Like link-dest-relative-basis this XFAILs on platforms without
# openat2/O_RESOLVE_BENEATH (the portable resolver rejects the '..' for safety);
# it flips to PASS where the kernel can adjudicate the in-module climb. Runs at
# any uid.
import shutil
import subprocess
from rsyncfns import (
SCRATCHDIR, make_data_file, makepath, rmtree, rsync_argv, start_test_daemon,
test_fail, test_xfail, write_daemon_conf,
)
DAEMON_PORT = 12931
DATA_SIZE = 40000
# dest 00 and basis 01 live side by side under `base`; the module is rooted at
# "/", so the served subtree is addressed by its absolute path minus the leading
# slash, and --link-dest=../01 climbs dest 00 -> sibling 01 (both inside /).
base = SCRATCHDIR / 'bakroot'
src = SCRATCHDIR / 'srcroot'
rmtree(base)
rmtree(src)
makepath(base / '01', src)
make_data_file(src / 'f.dat', DATA_SIZE)
shutil.copy2(src / 'f.dat', base / '01' / 'f.dat')
conf = write_daemon_conf([
('root', {'path': '/', 'read only': 'no'}),
])
url = start_test_daemon(conf, DAEMON_PORT)
base_rel = str(base).lstrip('/') # address `base` via the path=/ module
rmtree(base / '00')
proc = subprocess.run(
rsync_argv('-a', '--link-dest=../01', f'{src}/', f'{url}root/{base_rel}/00/'),
stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True)
out = proc.stdout or ''
if proc.returncode not in (0, 23): # 23: no-RESOLVE_BENEATH platforms reject the basis
test_fail(f"path=/ --link-dest push failed unexpectedly (rc={proc.returncode}):\n{out}")
dest = base / '00' / 'f.dat'
basis = base / '01' / 'f.dat'
if not dest.is_file():
test_fail(f"destination file missing ({dest})")
ds, bs = dest.stat(), basis.stat()
if (ds.st_dev, ds.st_ino) != (bs.st_dev, bs.st_ino):
test_xfail(
"#915 (path=/ case): a `path = /` daemon module ignored --link-dest=../01 "
"(module_dirlen==0 skipped the re-anchor) -- the file was re-transferred "
"instead of hard-linked. Honoured once the re-anchor covers path=/.")
# Honoured: the dest is hard-linked to the in-module sibling basis.

View File

@@ -1,121 +0,0 @@
#!/usr/bin/env python3
# Functional regression: a RELATIVE alt-basis dir (--link-dest / --copy-dest /
# --compare-dest = ../sibling) is silently ignored by a daemon receiver, so the
# basis is never used -- every file is re-transferred instead of hard-linked /
# copied / skipped. No error is printed; backups silently stop de-duplicating.
#
# Reported as #915 ("Security fix breaks --link-dest via rsync daemon": a
# `use chroot = no` daemon with `--link-dest=../01` re-transfers everything and
# fills the backup disk). The closely-related #928 is the same family over a
# remote shell with a relative `--link-dest=../snap.1`.
#
# Root cause: the 3.4.x symlink-race hardening resolves the receiver's basis
# through the confined resolver, which rejects the `..` that climbs from the
# destination (00) to its sibling basis (01); no basis is found, so the file is
# treated as new. Works in 3.4.1 (basis honoured).
#
# We exercise all three alt-basis forms because they are NOT obviously identical
# even though they share check_alt_basis_dirs():
# * --link-dest=../01 : the matched file must be HARD-LINKED to the basis.
# * --copy-dest=../01 : the matched file is COPIED from the basis, so its
# data is NOT sent over the wire (literal data ~ 0).
# * --compare-dest=../01 : a matched file is skipped entirely -- NOT created
# in the destination at all.
# Each signal cleanly separates "basis honoured" (fixed/3.4.1) from "basis
# ignored" (the regression).
#
# XFAIL until a relative alt-basis dir is honoured by a sanitize_paths receiver
# again (the accompanying syscall.c/receiver.c fix; cf. upstream PR #930). On
# platforms without openat2/O_RESOLVE_BENEATH the portable resolver still
# rejects the '..' for safety, so this stays XFAIL there. Runs at any uid.
import re
import subprocess
from rsyncfns import (
SCRATCHDIR, make_data_file, makepath, rmtree, rsync_argv, start_test_daemon,
test_fail, test_xfail, write_daemon_conf,
)
DAEMON_PORT = 12915
DATA_SIZE = 40000
mod = SCRATCHDIR / 'bakmod' # daemon module root: holds basis 01 and dest 00
src = SCRATCHDIR / 'src915'
rmtree(mod)
rmtree(src)
makepath(mod / '01', src)
make_data_file(src / 'f.dat', DATA_SIZE)
# Basis 01 holds a byte-identical copy of the file (same name/size/mtime so the
# quick-check treats it as a match and the basis is eligible).
import shutil
shutil.copy2(src / 'f.dat', mod / '01' / 'f.dat')
conf = write_daemon_conf([
('bak', {'path': str(mod), 'read only': 'no'}),
])
url = start_test_daemon(conf, DAEMON_PORT)
def push(opt):
"""Fresh dest 00, push src/ into bak/00/ with the given alt-basis option.
Returns (rc, stdout)."""
rmtree(mod / '00')
proc = subprocess.run(
rsync_argv('-a', '--stats', opt, f'{src}/', f'{url}bak/00/'),
stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True)
return proc.returncode, (proc.stdout or '')
def same_inode(a, b):
sa, sb = a.stat(), b.stat()
return (sa.st_dev, sa.st_ino) == (sb.st_dev, sb.st_ino)
def literal_bytes(out):
m = re.search(r'Literal data:\s*([\d,]+)', out)
return int(m.group(1).replace(',', '')) if m else -1
regressions = []
basis = mod / '01' / 'f.dat'
# --- 1. --link-dest=../01 : matched file must be hard-linked to the basis ----
rc, out = push('--link-dest=../01')
if rc not in (0, 23): # 23: no-RESOLVE_BENEATH platforms reject the basis
test_fail(f"--link-dest push failed unexpectedly (rc={rc}):\n{out}")
dest = mod / '00' / 'f.dat'
if not dest.is_file():
test_fail(f"--link-dest: destination file missing ({dest})")
if not same_inode(dest, basis):
regressions.append("--link-dest=../01 did not hard-link to the basis "
"(file re-transferred)")
# --- 2. --copy-dest=../01 : matched file copied locally, NOT sent on the wire -
rc, out = push('--copy-dest=../01')
if rc not in (0, 23): # 23: no-RESOLVE_BENEATH platforms reject the basis
test_fail(f"--copy-dest push failed unexpectedly (rc={rc}):\n{out}")
dest = mod / '00' / 'f.dat'
if not dest.is_file():
test_fail(f"--copy-dest: destination file missing ({dest})")
lit = literal_bytes(out)
if lit > DATA_SIZE // 2:
regressions.append(f"--copy-dest=../01 re-sent the data over the wire "
f"(Literal data={lit}, basis not used)")
# --- 3. --compare-dest=../01 : matched file skipped, NOT created in dest ------
rc, out = push('--compare-dest=../01')
if rc not in (0, 23): # 23: no-RESOLVE_BENEATH platforms reject the basis
test_fail(f"--compare-dest push failed unexpectedly (rc={rc}):\n{out}")
if (mod / '00' / 'f.dat').is_file():
regressions.append("--compare-dest=../01 created the file in the dest "
"(basis not matched, so the file was transferred)")
if regressions:
test_xfail(
"#915: a daemon receiver ignored a RELATIVE alt-basis dir (../01); the "
"confined path resolver rejects the `..` climb to the sibling basis so "
"the basis is never used:\n - " + "\n - ".join(regressions) +
"\nTo be closed by honouring a relative alt-basis dir on a "
"sanitize_paths receiver again (cf. PR #930).")
# No regressions -> all three relative alt-basis forms honoured the basis.

View File

@@ -7,11 +7,6 @@ covered too. As a normal user we can still remap the group to a secondary group
we belong to; the uid side then needs root and is skipped.
"""
# Rerun under the fleet harness's non-root pass (testsuite/fleettest.py): the uid
# remap only runs as root, so a non-root run exercises the group-only path too.
fleet_nonroot = True
import grp
import os
from rsyncfns import (
@@ -46,13 +41,6 @@ def assert_all(entries, *, gid=None, uid=None, label=''):
test_fail(f"{label}: owner of {rel} is {st.st_uid}, expected {uid}")
try:
grp.getgrgid(prim)
prim_has_name = True
except KeyError:
prim_has_name = False
if is_root:
# Root may assign any numeric id (it need not exist); pick targets that
# differ from the source's ids so the remap is observable.
@@ -63,20 +51,6 @@ if is_root:
run_rsync('-a', f'--groupmap={prim}:{target_gid}', f'{src}/', f'{TODIR}/')
assert_all(entries, gid=target_gid, label='--groupmap (root)')
entries = seed()
run_rsync('-a', f'--groupmap=*:{target_gid}', f'{src}/', f'{TODIR}/')
assert_all(entries, gid=target_gid, label='--groupmap wildcard (root)')
if prim_has_name:
entries = seed()
run_rsync('-a', f'--groupmap=:{target_gid}', f'{src}/', f'{TODIR}/')
assert_all(entries, gid=prim, label='--groupmap empty named group (root)')
entries = seed()
run_rsync('-a', '--numeric-ids', f'--groupmap=:{target_gid}',
f'{src}/', f'{TODIR}/')
assert_all(entries, gid=target_gid, label='--groupmap empty nameless group (root)')
entries = seed()
run_rsync('-a', f'--chown=:{target_gid}', f'{src}/', f'{TODIR}/')
assert_all(entries, gid=target_gid, label='--chown group (root)')
@@ -101,19 +75,6 @@ else:
run_rsync('-a', f'--groupmap={prim}:{sec}', f'{src}/', f'{TODIR}/')
assert_all(entries, gid=sec, label='--groupmap')
entries = seed()
run_rsync('-a', f'--groupmap=*:{sec}', f'{src}/', f'{TODIR}/')
assert_all(entries, gid=sec, label='--groupmap wildcard')
if prim_has_name:
entries = seed()
run_rsync('-a', f'--groupmap=:{sec}', f'{src}/', f'{TODIR}/')
assert_all(entries, gid=prim, label='--groupmap empty named group')
entries = seed()
run_rsync('-a', '--numeric-ids', f'--groupmap=:{sec}', f'{src}/', f'{TODIR}/')
assert_all(entries, gid=sec, label='--groupmap empty nameless group')
entries = seed()
run_rsync('-a', f'--chown=:{sec}', f'{src}/', f'{TODIR}/')
assert_all(entries, gid=sec, label='--chown group')

View File

@@ -1,70 +0,0 @@
#!/usr/bin/env python3
#
# Test that --partial and --delay-updates work as expected when then
# permissions of the destination file prevent writing to it.
import os
from pathlib import Path
import shutil
import subprocess
import sys
import tempfile
from rsyncfns import make_data_file, cp_p, makepath, checkit, RSYNC, TMPDIR, get_testuid, get_rootuid
BASEDIR = TMPDIR
FROMDIR = BASEDIR / 'from'
TODIR = BASEDIR / 'to'
makepath(FROMDIR)
makepath(TODIR)
makepath(FROMDIR)
make_data_file(FROMDIR / 'some_file', 1 * 1024 * 1024)
os.chmod(FROMDIR / 'some_file', 0o444)
makepath(TODIR / '.~tmp~')
os.chmod(TODIR / '.~tmp~', 0o700)
cp_p(FROMDIR / 'some_file', TODIR / '.~tmp~' / 'some_file')
is_root = get_testuid() == get_rootuid()
# As root the read-only dest temp wouldn't deny the write (root bypasses DAC),
# so the EACCES path under test never fires. On Linux we can drop
# CAP_DAC_OVERRIDE with setpriv inside a private mount namespace to force it;
# where that isn't possible -- non-Linux, Python < 3.12, no mount privilege, or
# a build dir the cap-dropped root can't even traverse (owned by an
# unprivileged user with restrictive perms, e.g. a CI tree owned by the ssh
# user at 0700) -- just run as root: the transfer still succeeds, it merely
# doesn't exercise the chmod-retry path here (non-root runs do).
_cwd_st = os.stat(os.getcwd())
_cwd_traversable = ((_cwd_st.st_uid == 0 and _cwd_st.st_mode & 0o100)
or _cwd_st.st_mode & 0o001)
if (is_root and sys.platform == 'linux' and hasattr(os, 'unshare')
and shutil.which('setpriv') and _cwd_traversable):
try:
cwd = Path(os.getcwd())
chown_target = None
for p in reversed(cwd.parents):
st = p.stat()
if not (st.st_uid == 0 or st.st_mode & 0o005):
chown_target = p
break
if chown_target is not None:
os.unshare(os.CLONE_NEWNS)
subprocess.run(['mount', '--make-rprivate', '/'], check=True)
tempdir = tempfile.mkdtemp()
subprocess.run(['mount', '--bind', cwd, tempdir], check=True)
subprocess.run(['mount', '-t', 'tmpfs', '-o', 'mode=0755', 'tmpfs', chown_target], check=True)
makepath(cwd)
subprocess.run(['mount', '--bind', tempdir, cwd], check=True)
subprocess.run(['umount', tempdir], check=True)
os.rmdir(tempdir)
import rsyncfns
rsyncfns.RSYNC = "setpriv --inh-caps -all --bounding-set -all " + RSYNC
except (OSError, subprocess.CalledProcessError):
pass # mount namespace denied (unprivileged container) -- run as root
checkit(['-avv', '--partial', '--delay-updates', f'{FROMDIR}/', f'{TODIR}/'], FROMDIR, TODIR)

View File

@@ -5,7 +5,7 @@ the Python-rewritten tests actually need; grow it as more shell tests are
ported.
Conventions matching the shell harness:
* Exit codes (see the Exit enum): 0=pass, 1=fail, 2=error, 77=skip, 78=xfail.
* Exit 0 = pass, 1 = fail, 77 = skip, 78 = xfail.
* The runner sets these environment variables before invoking each test:
scratchdir per-test scratch directory
srcdir rsync source directory
@@ -31,8 +31,6 @@ import sys
import time
from pathlib import Path
from exitcodes import Exit # re-exported: tests may `from rsyncfns import Exit`
# --- environment -----------------------------------------------------------
@@ -43,7 +41,7 @@ def _required(name: str) -> str:
f"rsyncfns: required environment variable {name} is not set; "
"run this test via runtests.py rather than directly.\n"
)
sys.exit(Exit.ERROR)
sys.exit(2)
return v
@@ -107,18 +105,18 @@ OUTFILE = SCRATCHDIR / 'rsync.out'
def test_fail(msg: str) -> 'None':
sys.stderr.write(msg.rstrip() + '\n')
sys.exit(Exit.FAIL)
sys.exit(1)
def test_skipped(msg: str) -> 'None':
sys.stderr.write(msg.rstrip() + '\n')
(TMPDIR / 'whyskipped').write_text(msg.rstrip() + '\n')
sys.exit(Exit.SKIP)
sys.exit(77)
def test_xfail(msg: str) -> 'None':
sys.stderr.write(msg.rstrip() + '\n')
sys.exit(Exit.XFAIL)
sys.exit(78)
# --- rsync invocation ------------------------------------------------------

View File

@@ -40,22 +40,6 @@ os.utime(TODIR / deep, (st.st_atime, st.st_mtime - 100)) # dest mtime older
run_rsync('-a', '-u', f'{src}/', f'{TODIR}/')
assert_same(TODIR / deep, src / deep, label='-u updated an older dest file')
# A newer destination symlink is still replaced by a source regular file
# because a file-format difference is always important enough to update.
rmtree(src)
rmtree(TODIR)
makepath(src, TODIR)
(src / 'foo').write_text("regular source file\n")
os.symlink('/should/not/exist', TODIR / 'foo')
st = os.stat(src / 'foo')
os.utime(TODIR / 'foo', (st.st_atime, st.st_mtime + 100),
follow_symlinks=False)
run_rsync('-a', '-u', f'{src}/', f'{TODIR}/')
if os.path.islink(TODIR / 'foo'):
test_fail("-u skipped a source file over a newer destination symlink")
assert_same(TODIR / 'foo', src / 'foo',
label='-u replaced a newer dest symlink with a regular file')
# --- --force replaces a non-empty dest directory with a file at depth -------
rmtree(src)
rmtree(TODIR)

31
token.c
View File

@@ -481,29 +481,14 @@ send_deflated_token(int f, int32 token, struct map_struct *buf, OFF_T offset, in
tx_strm.avail_in = n1;
if (protocol_version >= 31) /* Newer protocols avoid a data-duplicating bug */
offset += n1;
/* With our bundled zlib's Z_INSERT_ONLY this produces no
* output and consumes the input in one call. A build
* against a system zlib lacks Z_INSERT_ONLY and falls back
* to Z_SYNC_FLUSH (see top of file), which emits a flush
* block we discard -- and for an incompressible token that
* block can exceed obuf. Loop, resetting the output buffer,
* until all the input is consumed so a large token can't
* overflow obuf and abort the transfer (#951). Drain until
* avail_out != 0 too: a full output buffer can leave pending
* bytes that would otherwise leak into the next real deflate
* send and corrupt the stream (same condition as the data loop
* above). The discarded output is not sent: the receiver
* rebuilds the matching history itself in see_deflate_token(). */
do {
tx_strm.next_out = (Bytef *) obuf;
tx_strm.avail_out = AVAIL_OUT_SIZE(CHUNK_SIZE);
r = deflate(&tx_strm, Z_INSERT_ONLY);
if (r != Z_OK) {
rprintf(FERROR, "deflate on token returned %d (%d bytes left)\n",
r, tx_strm.avail_in);
exit_cleanup(RERR_STREAMIO);
}
} while (tx_strm.avail_in != 0 || tx_strm.avail_out == 0);
tx_strm.next_out = (Bytef *) obuf;
tx_strm.avail_out = AVAIL_OUT_SIZE(CHUNK_SIZE);
r = deflate(&tx_strm, Z_INSERT_ONLY);
if (r != Z_OK || tx_strm.avail_in != 0) {
rprintf(FERROR, "deflate on token returned %d (%d bytes left)\n",
r, tx_strm.avail_in);
exit_cleanup(RERR_STREAMIO);
}
} while (toklen > 0);
}
}

View File

@@ -41,8 +41,8 @@ extern filter_rule_list daemon_filter_list;
int sanitize_paths = 0;
extern char curr_dir[MAXPATHLEN]; /* defined in syscall.c */
extern unsigned int curr_dir_len;
char curr_dir[MAXPATHLEN];
unsigned int curr_dir_len;
int curr_dir_depth; /* This is only set for a sanitizing daemon. */
/* Set a fd into nonblocking mode. */
@@ -1788,6 +1788,8 @@ void *expand_item_list(item_list *lp, size_t item_size, const char *desc, int in
new_ptr == lp->items ? " not" : "");
}
memset((char *)new_ptr + lp->malloced * item_size, 0,
(expand_size - lp->malloced) * item_size);
lp->items = new_ptr;
lp->malloced = expand_size;
}

View File

@@ -79,9 +79,7 @@ void *my_alloc(void *ptr, size_t num, size_t size, const char *file, int line)
who_am_i(), do_big_num(max_alloc, 0, NULL), src_file(file), line);
exit_cleanup(RERR_MALLOC);
}
if (!ptr)
ptr = malloc(num * size);
else if (ptr == do_calloc)
if (!ptr || ptr == do_calloc)
ptr = calloc(num, size);
else
ptr = realloc(ptr, num * size);