Preparing for release of 3.2.1

- Mention the -VV behavior.
- Mention how the protect-args default is presented in -V list.

.github/workflows/ccpp.yml

@@ -18,7 +18,7 @@ jobs:
     - name: prepare-source
       run: ./prepare-source
     - name: configure
-      run: ./configure --with-included-popt --with-included-zlib --enable-simd
+      run: ./configure --with-included-popt --with-included-zlib
     - name: make
       run: make
     - name: version-summary

.gitignore

@@ -19,6 +19,8 @@ aclocal.m4
 /rsync*.5
 /rsync*.html
 /help-rsync*.h
+/default-cvsignore.h
+/default-dont-compress.h
 /.md2man-works
 /autom4te*.cache
 /confdefs.h

Makefile.in

@@ -10,6 +10,7 @@ mandir=@mandir@
 
 LIBS=@LIBS@
 CC=@CC@
+AWK=@AWK@
 CFLAGS=@CFLAGS@
 CPPFLAGS=@CPPFLAGS@
 CXX=@CXX@
@@ -31,12 +32,13 @@ VERSION=@RSYNC_VERSION@
 .SUFFIXES:
 .SUFFIXES: .c .o
 
-SIMD_x86_64=simd-checksum-x86_64.o lib/md5-asm-x86_64.o
+SIMD_x86_64=simd-checksum-x86_64.o
+ASM_x86_64=lib/md5-asm-x86_64.o
 
 GENFILES=configure.sh aclocal.m4 config.h.in proto.h proto.h-tstamp rsync.1 rsync.1.html \
 	 rsync-ssl.1 rsync-ssl.1.html rsyncd.conf.5 rsyncd.conf.5.html
 HEADERS=byteorder.h config.h errcode.h proto.h rsync.h ifuncs.h itypes.h inums.h \
-	lib/pool_alloc.h
+	lib/pool_alloc.h lib/mdigest.h lib/md-defines.h
 LIBOBJ=lib/wildmatch.o lib/compat.o lib/snprintf.o lib/mdfour.o lib/md5.o \
 	lib/permstring.o lib/pool_alloc.o lib/sysacls.o lib/sysxattrs.o @LIBOBJS@
 zlib_OBJS=zlib/deflate.o zlib/inffast.o zlib/inflate.o zlib/inftrees.o \
@@ -45,7 +47,7 @@ OBJS1=flist.o rsync.o generator.o receiver.o cleanup.o sender.o exclude.o \
 	util.o util2.o main.o checksum.o match.o syscall.o log.o backup.o delete.o
 OBJS2=options.o io.o compat.o hlink.o token.o uidlist.o socket.o hashtable.o \
 	fileio.o batch.o clientname.o chmod.o acls.o xattrs.o
-OBJS3=progress.o pipe.o
+OBJS3=progress.o pipe.o @ASM@
 DAEMON_OBJ = params.o loadparm.o clientserver.o access.o connection.o authenticate.o
 popt_OBJS=popt/findme.o  popt/popt.o  popt/poptconfig.o \
 	popt/popthelp.o popt/poptparse.o
@@ -99,12 +101,18 @@ rsync$(EXEEXT): $(OBJS)
 
 $(OBJS): $(HEADERS)
 $(CHECK_OBJS): $(HEADERS)
+tls.o xattrs.o: lib/sysxattrs.h
 options.o: latest-year.h help-rsync.h help-rsyncd.h
+exclude.o: default-cvsignore.h
+loadparm.o: default-dont-compress.h
 
 flist.o: rounding.h
 
+default-cvsignore.h default-dont-compress.h: rsync.1.md define-from-md.awk
+	$(AWK) -f $(srcdir)/define-from-md.awk -v hfile=$@ $(srcdir)/rsync.1.md
+
 help-rsync.h help-rsyncd.h: rsync.1.md help-from-md.awk
-	awk -f $(srcdir)/help-from-md.awk -v helpfile=$@ $(srcdir)/rsync.1.md
+	$(AWK) -f $(srcdir)/help-from-md.awk -v hfile=$@ $(srcdir)/rsync.1.md
 
 rounding.h: rounding.c rsync.h proto.h
 	@for r in 0 1 3; do \
@@ -125,10 +133,10 @@ rounding.h: rounding.c rsync.h proto.h
 	@rm -f rounding.out
 
 simd-checksum-x86_64.o: simd-checksum-x86_64.cpp
-	$(CXX) $(CXXFLAGS) $(CPPFLAGS) -c -o $@ $(srcdir)/simd-checksum-x86_64.cpp
+	@$(srcdir)/cmdormsg disable-simd $(CXX) -I. $(CXXFLAGS) $(CPPFLAGS) -c -o $@ $(srcdir)/simd-checksum-x86_64.cpp
 
-lib/md5-asm-x86_64.o: lib/md5-asm-x86_64.s
-	$(CC) -c -o $@ $(srcdir)/lib/md5-asm-x86_64.s
+lib/md5-asm-x86_64.o: lib/md5-asm-x86_64.S config.h lib/md-defines.h
+	@$(srcdir)/cmdormsg disable-asm $(CC) -I. @NOEXECSTACK@ -c -o $@ $(srcdir)/lib/md5-asm-x86_64.S
 
 tls$(EXEEXT): $(TLS_OBJ)
 	$(CC) $(CFLAGS) $(LDFLAGS) -o $@ $(TLS_OBJ) $(LIBS)
@@ -220,7 +228,7 @@ proto.h: proto.h-tstamp
 	@if test -f proto.h; then :; else cp -p $(srcdir)/proto.h .; fi
 
 proto.h-tstamp: $(srcdir)/*.c $(srcdir)/lib/compat.c config.h
-	awk -f $(srcdir)/mkproto.awk $(srcdir)/*.c $(srcdir)/lib/compat.c
+	$(AWK) -f $(srcdir)/mkproto.awk $(srcdir)/*.c $(srcdir)/lib/compat.c
 
 .PHONY: man
 man: rsync.1 rsync-ssl.1 rsyncd.conf.5

README.md

@@ -65,7 +65,7 @@ WEB SITE
 
 The main rsync web site is here:
 
->  http://rsync.samba.org/
+>  https://rsync.samba.org/
 
 You'll find a FAQ list, downloads, resources, HTML versions of the
 manpages, etc.
@@ -80,7 +80,7 @@ list, and there is also an announcement-only mailing list for those that
 want official announcements.  See the mailing-list page for full
 details:
 
->  http://rsync.samba.org/lists.html
+>  https://rsync.samba.org/lists.html
 
 
 BUG REPORTS
@@ -88,14 +88,14 @@ BUG REPORTS
 
 To visit this web page for full the details on bug reporting:
 
->  http://rsync.samba.org/bugzilla.html
+>  https://rsync.samba.org/bugtracking.html
 
 That page contains links to the current bug list, and information on how
 to report a bug well.  You might also like to try searching the Internet
 for the error message you've received, or looking in the mailing list
 archives at:
 
->  http://mail-archive.com/rsync@lists.samba.org/
+>  https://mail-archive.com/rsync@lists.samba.org/
 
 To send a bug report, follow the instructions on the bug-tracking
 page of the web site.
@@ -112,38 +112,31 @@ is hosted on github and on samba's site.  Feel free to access it here:
 
 >  https://github.com/WayneD/rsync
 
-or clone it from its samba repo:
+A backup git repo is available on the samba site:
 
 >     git clone git://git.samba.org/rsync.git
 
 See the download page for full details on all the ways to grab the
 source:
 
->  http://rsync.samba.org/download.html
+>  https://rsync.samba.org/download.html
 
 
 COPYRIGHT
 ---------
 
 Rsync was originally written by Andrew Tridgell and is currently
-maintained by Wayne Davison.   It has been improved by many developers
+maintained by Wayne Davison.  It has been improved by many developers
 from around the world.
 
 Rsync may be used, modified and redistributed only under the terms of
 the GNU General Public License, found in the file COPYING in this
 distribution, or at:
 
->  http://www.fsf.org/licenses/gpl.html
+>  https://www.fsf.org/licenses/gpl.html
 
 
 AVAILABILITY
 ------------
 
-The main web site for rsync is http://rsync.samba.org/
-
-The main ftp site is ftp://rsync.samba.org/pub/rsync/
-
-This is also available as rsync://download.samba.org/rsyncftp/ if you
-connect via ssl. Use the `rsync-ssl` script if you have it, otherwise
-connect to the rsync server via a normal rsync command and it will
-output some instructions for how to connect.
+The main web site for rsync is https://rsync.samba.org/

TODO

@@ -49,7 +49,7 @@ Create test makefile target for some tests
 
 RELATED PROJECTS -----------------------------------------------------
 rsyncsh
-http://rsync.samba.org/rsync-and-debian/
+https://rsync.samba.org/rsync-and-debian/
 rsyncable gzip patch
 rsyncsplit as alternative to real integration with gzip?
 reverse rsync over HTTP Range
@@ -66,8 +66,8 @@ Use chroot only if supported
   If running as non-root, then don't fail, just give a warning.
   (There was a thread about this a while ago?)
 
-    http://lists.samba.org/pipermail/rsync/2001-August/thread.html
-    http://lists.samba.org/pipermail/rsync/2001-September/thread.html
+    https://lists.samba.org/pipermail/rsync/2001-August/thread.html
+    https://lists.samba.org/pipermail/rsync/2001-September/thread.html
 
                       --          --
 
@@ -204,7 +204,7 @@ Create more granular verbosity					2003/05/15
   fine grained selection of statistical reporting and what
   actions are logged.
 
-  http://lists.samba.org/archive/rsync/2003-May/006059.html
+  https://lists.samba.org/archive/rsync/2003-May/006059.html
 
                       --          --
 
@@ -287,7 +287,7 @@ Perhaps flush stdout like syslog
 
   Perhaps flush stdout after each filename, so that people trying to
   monitor progress in a log file can do so more easily.  See
-  http://bugs.debian.org/cgi-bin/bugreport.cgi?bug=48108
+  https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=48108
 
                       --          --
 
@@ -495,7 +495,7 @@ rsyncsh
                       --          --
 
 
-http://rsync.samba.org/rsync-and-debian/
+https://rsync.samba.org/rsync-and-debian/
 
 
                       --          --

clientname.c

@@ -481,7 +481,7 @@ static int valid_ipaddr(const char *s)
 
 		for (count = 0; count < 8; count++) {
 			if (!*s)
-				return saw_double_colon && count < 7;
+				return saw_double_colon;
 
 			if (strchr(s, ':') == NULL && strchr(s, '.') != NULL) {
 				if ((!saw_double_colon && count != 6) || (saw_double_colon && count > 6))

clientserver.c

@@ -54,6 +54,7 @@ extern char *config_file;
 extern char *logfile_format;
 extern char *files_from;
 extern char *tmpdir;
+extern char *early_input_file;
 extern struct chmod_mode_struct *chmod_modes;
 extern filter_rule_list daemon_filter_list;
 #ifdef ICONV_OPTION
@@ -67,8 +68,13 @@ char *auth_user;
 int read_only = 0;
 int module_id = -1;
 int pid_file_fd = -1;
+int early_input_len = 0;
+char *early_input = NULL;
 struct chmod_mode_struct *daemon_chmod_modes;
 
+#define EARLY_INPUT_CMD "#early_input="
+#define EARLY_INPUT_CMDLEN (sizeof EARLY_INPUT_CMD - 1)
+
 /* module_dirlen is the length of the module_dir string when in daemon
  * mode and module_dir is not "/"; otherwise 0.  (Note that a chroot-
  * enabled module can have a non-"/" module_dir these days.) */
@@ -144,14 +150,12 @@ static int exchange_protocols(int f_in, int f_out, char *buf, size_t bufsiz, int
 #else
 	int our_sub = 0;
 #endif
-	char *motd;
 
 	io_printf(f_out, "@RSYNCD: %d.%d\n", protocol_version, our_sub);
-
 	if (!am_client) {
-		motd = lp_motd_file();
+		char *motd = lp_motd_file();
 		if (motd && *motd) {
-			FILE *f = fopen(motd,"r");
+			FILE *f = fopen(motd, "r");
 			while (f && !feof(f)) {
 				int len = fread(buf, 1, bufsiz - 1, f);
 				if (len > 0)
@@ -245,6 +249,36 @@ int start_inband_exchange(int f_in, int f_out, const char *user, int argc, char
 	if (exchange_protocols(f_in, f_out, line, sizeof line, 1) < 0)
 		return -1;
 
+	if (early_input_file) {
+		STRUCT_STAT st;
+		FILE *f = fopen(early_input_file, "rb");
+		if (!f || do_fstat(fileno(f), &st) < 0) {
+			rsyserr(FERROR, errno, "failed to open %s", early_input_file);
+			return -1;
+		}
+		early_input_len = st.st_size;
+		if (early_input_len > (int)sizeof line) {
+			rprintf(FERROR, "%s is > %d bytes.\n", early_input_file, (int)sizeof line);
+			return -1;
+		}
+		if (early_input_len > 0) {
+			io_printf(f_out, EARLY_INPUT_CMD "%d\n", early_input_len);
+			while (early_input_len > 0) {
+				int len;
+				if (feof(f)) {
+					rprintf(FERROR, "Early EOF in %s\n", early_input_file);
+					return -1;
+				}
+				len = fread(line, 1, early_input_len, f);
+				if (len > 0) {
+					write_buf(f_out, line, len);
+					early_input_len -= len;
+				}
+			}
+		}
+		fclose(f);
+	}
+
 	/* set daemon_over_rsh to false since we need to build the
 	 * true set of args passed through the rsh/ssh connection;
 	 * this is a no-op for direct-socket-connection mode */
@@ -394,10 +428,10 @@ void set_env_num(const char *var, long num)
 /* Used for both early exec & pre-xfer exec */
 static pid_t start_pre_exec(const char *cmd, int *arg_fd_ptr, int *error_fd_ptr)
 {
-	int arg_fds[2], error_fds[2], arg_fd, error_fd;
+	int arg_fds[2], error_fds[2], arg_fd;
 	pid_t pid;
 
-	if ((error_fd_ptr && pipe(error_fds) < 0) || (arg_fd_ptr && pipe(arg_fds) < 0) || (pid = fork()) < 0)
+	if ((error_fd_ptr && pipe(error_fds) < 0) || pipe(arg_fds) < 0 || (pid = fork()) < 0)
 		return (pid_t)-1;
 
 	if (pid == 0) {
@@ -406,38 +440,36 @@ static pid_t start_pre_exec(const char *cmd, int *arg_fd_ptr, int *error_fd_ptr)
 
 		if (error_fd_ptr) {
 			close(error_fds[0]);
-			error_fd = error_fds[1];
-			set_blocking(error_fd);
+			set_blocking(error_fds[1]);
 		}
 
-		if (arg_fd_ptr) {
-			close(arg_fds[1]);
-			arg_fd = arg_fds[0];
-			set_blocking(arg_fd);
+		close(arg_fds[1]);
+		arg_fd = arg_fds[0];
+		set_blocking(arg_fd);
 
+		len = read_arg_from_pipe(arg_fd, buf, BIGPATHBUFLEN);
+		if (len <= 0)
+			_exit(1);
+		set_env_str("RSYNC_REQUEST", buf);
+
+		for (j = 0; ; j++) {
+			char *p;
 			len = read_arg_from_pipe(arg_fd, buf, BIGPATHBUFLEN);
-			if (len <= 0)
+			if (len <= 0) {
+				if (!len)
+					break;
 				_exit(1);
-			set_env_str("RSYNC_REQUEST", buf);
-
-			for (j = 0; ; j++) {
-				char *p;
-				len = read_arg_from_pipe(arg_fd, buf, BIGPATHBUFLEN);
-				if (len <= 0) {
-					if (!len)
-						break;
-					_exit(1);
-				}
-				if (asprintf(&p, "RSYNC_ARG%d=%s", j, buf) >= 0)
-					putenv(p);
 			}
-			close(arg_fd);
+			if (asprintf(&p, "RSYNC_ARG%d=%s", j, buf) >= 0)
+				putenv(p);
 		}
 
+		dup2(arg_fd, STDIN_FILENO);
+		close(arg_fd);
+
 		if (error_fd_ptr) {
-			close(STDIN_FILENO);
-			dup2(error_fd, STDOUT_FILENO);
-			close(error_fd);
+			dup2(error_fds[1], STDOUT_FILENO);
+			close(error_fds[1]);
 		}
 
 		status = shell_exec(cmd);
@@ -449,20 +481,18 @@ static pid_t start_pre_exec(const char *cmd, int *arg_fd_ptr, int *error_fd_ptr)
 
 	if (error_fd_ptr) {
 		close(error_fds[1]);
-		error_fd = *error_fd_ptr = error_fds[0];
-		set_blocking(error_fd);
+		*error_fd_ptr = error_fds[0];
+		set_blocking(error_fds[0]);
 	}
 
-	if (arg_fd_ptr) {
-		close(arg_fds[0]);
-		arg_fd = *arg_fd_ptr = arg_fds[1];
-		set_blocking(arg_fd);
-	}
+	close(arg_fds[0]);
+	arg_fd = *arg_fd_ptr = arg_fds[1];
+	set_blocking(arg_fd);
 
 	return pid;
 }
 
-static void write_pre_exec_args(int write_fd, char *request, char **early_argv, char **argv)
+static void write_pre_exec_args(int write_fd, char *request, char **early_argv, char **argv, int am_early)
 {
 	int j = 0;
 
@@ -475,10 +505,15 @@ static void write_pre_exec_args(int write_fd, char *request, char **early_argv,
 			write_buf(write_fd, *early_argv, strlen(*early_argv)+1);
 		j = 1; /* Skip arg0 name in argv. */
 	}
-	for ( ; argv[j]; j++)
-		write_buf(write_fd, argv[j], strlen(argv[j])+1);
+	if (argv) {
+		for ( ; argv[j]; j++)
+			write_buf(write_fd, argv[j], strlen(argv[j])+1);
+	}
 	write_byte(write_fd, 0);
 
+	if (am_early && early_input_len)
+		write_buf(write_fd, early_input, early_input_len);
+
 	close(write_fd);
 }
 
@@ -812,12 +847,14 @@ static int rsync_module(int f_in, int f_out, int i, const char *addr, const char
 		/* For early exec, fork a child process to run the indicated
 		 * command and wait for it to exit. */
 		if (*lp_early_exec(i)) {
-			pid_t pid = start_pre_exec(lp_early_exec(i), NULL, NULL);
+			int arg_fd;
+			pid_t pid = start_pre_exec(lp_early_exec(i), &arg_fd, NULL);
 			if (pid == (pid_t)-1) {
 				rsyserr(FLOG, errno, "early exec preparation failed");
 				io_printf(f_out, "@ERROR: early exec preparation failed\n");
 				return -1;
 			}
+			write_pre_exec_args(arg_fd, NULL, NULL, NULL, 1);
 			if (finish_pre_exec("early exec", pid, -1) != NULL) {
 				rsyserr(FLOG, errno, "early exec failed");
 				io_printf(f_out, "@ERROR: early exec failed\n");
@@ -839,6 +876,11 @@ static int rsync_module(int f_in, int f_out, int i, const char *addr, const char
 	}
 #endif
 
+	if (early_input) {
+		free(early_input);
+		early_input = NULL;
+	}
+
 	if (use_chroot) {
 		/*
 		 * XXX: The 'use chroot' flag is a fairly reliable
@@ -953,7 +995,7 @@ static int rsync_module(int f_in, int f_out, int i, const char *addr, const char
 		msgs2stderr = 0; /* A non-rsh-run daemon doesn't have stderr for msgs. */
 
 	if (pre_exec_pid) {
-		write_pre_exec_args(pre_exec_arg_fd, request, orig_early_argv, orig_argv);
+		write_pre_exec_args(pre_exec_arg_fd, request, orig_early_argv, orig_argv, 0);
 		err_msg = finish_pre_exec("pre-xfer exec", pre_exec_pid, pre_exec_error_fd);
 	}
 
@@ -1185,6 +1227,20 @@ int start_daemon(int f_in, int f_out)
 	if (!read_line_old(f_in, line, sizeof line, 0))
 		return -1;
 
+	if (strncmp(line, EARLY_INPUT_CMD, EARLY_INPUT_CMDLEN) == 0) {
+		early_input_len = strtol(line + EARLY_INPUT_CMDLEN, NULL, 10);
+		if (early_input_len <= 0 || early_input_len > BIGPATHBUFLEN) {
+			io_printf(f_out, "@ERROR: invalid early_input length\n");
+			return -1;
+		}
+		if (!(early_input = new_array(char, early_input_len)))
+			out_of_memory("exchange_protocols");
+		read_buf(f_in, early_input, early_input_len);
+
+		if (!read_line_old(f_in, line, sizeof line, 0))
+			return -1;
+	}
+
 	if (!*line || strcmp(line, "#list") == 0) {
 		rprintf(FLOG, "module-list request from %s (%s)\n",
 			host, addr);

cmdormsg

@@ -0,0 +1,11 @@
+#!/bin/sh
+
+srcdir=`dirname $0`
+opt="$1"
+shift
+
+echo "$*"
+if ! "${@}"; then
+    echo "If you can't fix the issue, re-run $srcdir/configure with --$opt."
+    exit 1
+fi

configure.ac

@@ -1,6 +1,6 @@
 dnl Process this file with autoconf to produce a configure script.
 
-AC_INIT([rsync],[3.2.0pre3],[http://rsync.samba.org/bugzilla.html])
+AC_INIT([rsync],[3.2.1],[https://rsync.samba.org/bugtracking.html])
 
 AC_CONFIG_MACRO_DIR([m4])
 AC_CONFIG_SRCDIR([byteorder.h])
@@ -42,6 +42,7 @@ dnl Checks for programs.
 AC_PROG_CC
 AC_PROG_CPP
 AC_PROG_CXX
+AC_PROG_AWK
 AC_PROG_EGREP
 AC_PROG_INSTALL
 AC_PROG_MKDIR_P
@@ -68,7 +69,7 @@ if test x"$ac_cv_path_PYTHON3" = x; then
     AC_MSG_RESULT(no - python3 not found)
     md2man_works=no
 else
-    md2man_out=`"$srcdir/md2man" --test "$srcdir/rsync.1.md" 2>&1`
+    md2man_out=`"$srcdir/md2man" --test "$srcdir/rsync-ssl.1.md" 2>&1`
     if test $? = 0; then
 	AC_MSG_RESULT(yes)
 	md2man_works=yes
@@ -197,35 +198,104 @@ SIMD=
 
 AC_MSG_CHECKING([whether to enable SIMD optimizations])
 AC_ARG_ENABLE(simd,
-    AS_HELP_STRING([--enable-simd],[enable SIMD optimizations (requires g++)]))
+    AS_HELP_STRING([--disable-simd],[disable SIMD optimizations (requires c++)]))
 
-if test x"$enable_simd" = x"yes"; then
-    # For x86-64 SIMD, g++ is also required
+# Clag is crashing with -g -O2, so we'll get rid of -g for now.
+CXXFLAGS=`echo "$CXXFLAGS" | sed 's/-g //'`
+
+if test x"$enable_simd" != x"no"; then
+    # For x86-64 SIMD, g++ >=5 or clang++ >=7 is required
     if test x"$build_cpu" = x"x86_64"; then
-	if test x"$CXX" = x"g++"; then
-	    # AC_MSG_RESULT() called below
-	    SIMD="$SIMD x86_64"
-	else
-	    AC_MSG_RESULT(no)
-	    AC_MSG_ERROR(Failed to find g++ for SIMD speedups.
+	AC_LANG(C++)
+	AC_RUN_IFELSE([AC_LANG_PROGRAM([[#include <stdio.h>
+#include <immintrin.h>
+__attribute__ ((target("default"))) int test_ssse3(int x) { return x; }
+__attribute__ ((target("default"))) int test_sse2(int x) { return x; }
+__attribute__ ((target("default"))) int test_avx2(int x) { return x; }
+__attribute__ ((target("ssse3"))) int test_ssse3(int x) { return x; }
+__attribute__ ((target("sse2"))) int test_sse2(int x) { return x; }
+__attribute__ ((target("avx2"))) int test_avx2(int x) { return x; }
+typedef long long __m128i_u __attribute__((__vector_size__(16), __may_alias__, __aligned__(1)));
+typedef long long __m256i_u __attribute__((__vector_size__(32), __may_alias__, __aligned__(1)));
+__attribute__ ((target("default"))) void more_testing(char* buf, int len) { }
+__attribute__ ((target("ssse3"))) void more_testing(char* buf, int len)
+{
+    int i;
+    for (i = 0; i < (len-32); i+=32) {
+	__m128i in8_1, in8_2;
+	in8_1 = _mm_lddqu_si128((__m128i_u*)&buf[i]);
+	in8_2 = _mm_lddqu_si128((__m128i_u*)&buf[i + 16]);
+    }
+}
+]], [[if (test_ssse3(42) != 42 || test_sse2(42) != 42 || test_avx2(42) != 42) exit(1);]])],[CXX_OK=yes],[CXX_OK=no])
+	AC_LANG(C)
+	if test x"$CXX_OK" = x"yes"; then
+	    # AC_MSG_RESULT() is called below.
+	    SIMD="x86_64"
+	elif test x"$enable_simd" = x"yes"; then
+	    AC_MSG_RESULT(error)
+	    AC_MSG_ERROR(The SIMD compilation test failed.
 Omit --enable-simd to continue without it.)
 	fi
+    elif test x"$enable_simd" = x"yes"; then
+        AC_MSG_RESULT(unavailable)
+        AC_MSG_ERROR(The SIMD optimizations are currently x86_64 only.
+Omit --enable-simd to continue without it.)
     fi
 fi
 
 if test x"$SIMD" != x""; then
-    SIMD=`echo "$SIMD" | sed -e 's/^ *//'`
     AC_MSG_RESULT([yes ($SIMD)])
     AC_DEFINE(HAVE_SIMD, 1, [Define to 1 to enable SIMD optimizations])
-    SIMD=`echo "$SIMD" | sed -e 's/[[^ ]]\+/$(SIMD_&)/g'`
-    # We only use g++ for its target attribute dispatching, disable unneeded bulky features
+    SIMD='$(SIMD_'"$SIMD)"
+    # We only use c++ for its target attribute dispatching, disable unneeded bulky features
     CXXFLAGS="$CXXFLAGS -fno-exceptions -fno-rtti"
+    # Apple often has "g++" as a symlink for clang. Try to find out the truth.
+    CXX_VERSION=`$CXX --version 2>/dev/null | head -n 2`
+    case "$CXX_VERSION" in
+    *clang*) CXXFLAGS="$CXXFLAGS -fno-slp-vectorize" ;; # avoid a performance hit
+    esac
 else
     AC_MSG_RESULT(no)
 fi
 
 AC_SUBST(SIMD)
 
+AC_MSG_CHECKING([if assembler accepts noexecstack])
+OLD_CFLAGS="$CFLAGS"
+CFLAGS="$CFLAGS -Wa,--noexecstack"
+AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[ ]], [[return 0;]])],
+[ NOEXECSTACK='-Wa,--noexecstack' ; AC_MSG_RESULT(yes) ],
+[ NOEXECSTACK='' ; AC_MSG_RESULT(no) ])
+CFLAGS="$OLD_CFLAGS"
+AC_SUBST(NOEXECSTACK)
+
+ASM=
+
+AC_MSG_CHECKING([whether to enable ASM optimizations])
+AC_ARG_ENABLE(asm,
+    AS_HELP_STRING([--disable-asm],[disable ASM optimizations]))
+
+if test x"$enable_asm" != x"no"; then
+    if test x"$build_cpu" = x"x86_64"; then
+	ASM="$build_cpu"
+    elif test x"$enable_asm" = x"yes"; then
+        AC_MSG_RESULT(unavailable)
+        AC_MSG_ERROR(The ASM optimizations are currently x86_64 only.
+Omit --enable-asm to continue without it.)
+    fi
+fi
+
+if test x"$ASM" != x""; then
+    AC_MSG_RESULT([yes ($ASM)])
+    AC_DEFINE(HAVE_ASM, 1, [Define to 1 to enable ASM optimizations])
+    ASM='$(ASM_'"$ASM)"
+else
+    AC_MSG_RESULT(no)
+fi
+
+AC_SUBST(ASM)
+
 # arrgh. libc in some old debian version screwed up the largefile
 # stuff, getting byte range locking wrong
 AC_CACHE_CHECK([for broken largefile support],rsync_cv_HAVE_BROKEN_LARGEFILE,[
@@ -630,7 +700,7 @@ size_t iconv();
 #endif
 ]], [[]])],[am_cv_proto_iconv_arg1=""],[am_cv_proto_iconv_arg1="const"])
       am_cv_proto_iconv="extern size_t iconv (iconv_t cd, $am_cv_proto_iconv_arg1 char * *inbuf, size_t *inbytesleft, char * *outbuf, size_t *outbytesleft);"])
-    am_cv_proto_iconv=`echo "[$]am_cv_proto_iconv" | tr -s ' ' | sed -e 's/( /(/'`
+    am_cv_proto_iconv=`echo "[$]am_cv_proto_iconv" | tr -s ' ' | sed 's/( /(/'`
 AC_MSG_RESULT([$]{ac_t:-
          }[$]am_cv_proto_iconv)
 AC_DEFINE_UNQUOTED(ICONV_CONST, $am_cv_proto_iconv_arg1,

define-from-md.awk

@@ -0,0 +1,41 @@
+#!/usr/bin/awk -f
+
+# The caller must pass args: -v hfile=NAME rsync.1.md
+
+BEGIN {
+    heading = "/* DO NOT EDIT THIS FILE!  It is auto-generated from a list of values in " ARGV[1] "! */"
+    if (hfile ~ /compress/) {
+	define = "#define DEFAULT_DONT_COMPRESS"
+	prefix = "*."
+    } else {
+	define = "#define DEFAULT_CVSIGNORE"
+	prefix = ""
+    }
+    value_list = ""
+}
+
+/^    > [^ ]+$/ {
+    gsub(/`/, "")
+    if (value_list != "") value_list = value_list " "
+    value_list = value_list prefix $2
+    next
+}
+
+value_list ~ /\.gz / && hfile ~ /compress/ {
+    exit
+}
+
+value_list ~ /SCCS / && hfile ~ /cvsignore/ {
+    exit
+}
+
+value_list = ""
+
+END {
+    if (value_list != "")
+	print heading "\n\n" define " \"" value_list "\"" > hfile
+    else {
+	print "Failed to find a value list in " ARGV[1] " for " hfile
+	exit 1
+    }
+}

exclude.c

@@ -21,6 +21,7 @@
  */
 
 #include "rsync.h"
+#include "default-cvsignore.h"
 
 extern int am_server;
 extern int am_sender;
@@ -1051,16 +1052,6 @@ static filter_rule *parse_rule_tok(const char **rulestr_ptr,
 	return rule;
 }
 
-static char default_cvsignore[] =
-	/* These default ignored items come from the CVS manual. */
-	"RCS SCCS CVS CVS.adm RCSLOG cvslog.* tags TAGS"
-	" .make.state .nse_depinfo *~ #* .#* ,* _$* *$"
-	" *.old *.bak *.BAK *.orig *.rej .del-*"
-	" *.a *.olb *.o *.obj *.so *.exe"
-	" *.Z *.elc *.ln core"
-	/* The rest we added to suit ourself. */
-	" .svn/ .git/ .hg/ .bzr/";
-
 static void get_cvs_excludes(uint32 rflags)
 {
 	static int initialized = 0;
@@ -1070,7 +1061,7 @@ static void get_cvs_excludes(uint32 rflags)
 		return;
 	initialized = 1;
 
-	parse_filter_str(&cvs_filter_list, default_cvsignore,
+	parse_filter_str(&cvs_filter_list, DEFAULT_CVSIGNORE,
 			 rule_template(rflags | (protocol_version >= 30 ? FILTRULE_PERISHABLE : 0)),
 			 0);
 

help-from-md.awk

@@ -1,10 +1,10 @@
 #!/usr/bin/awk -f
 
-# The caller must set -v helpfile=help-NAME.h and pass arg NAME.NUM.md
+# The caller must pass args: -v hfile=help-NAME.h NAME.NUM.md
 
 BEGIN {
     heading = "/* DO NOT EDIT THIS FILE!  It is auto-generated from the option list in " ARGV[1] "! */"
-    findcomment = helpfile
+    findcomment = hfile
     sub("\\.", "\\.", findcomment)
     findcomment = "\\[comment\\].*" findcomment
     backtick_cnt = 0
@@ -32,9 +32,9 @@ $0 ~ findcomment {
 
 END {
     if (foundcomment && backtick_cnt > 1)
-	print heading "\n" prints > helpfile
+	print heading "\n" prints > hfile
     else {
-	print "Failed to find " helpfile " section in " ARGV[1]
+	print "Failed to find " hfile " section in " ARGV[1]
 	exit 1
     }
 }

lib/md-defines.h

@@ -0,0 +1,15 @@
+/* Keep this simple so both C and ASM can use it */
+
+#define MD4_DIGEST_LEN 16
+#define MD5_DIGEST_LEN 16
+#define MAX_DIGEST_LEN MD5_DIGEST_LEN
+
+#define CSUM_CHUNK 64
+
+#define CSUM_NONE 0
+#define CSUM_MD4_ARCHAIC 1
+#define CSUM_MD4_BUSTED 2
+#define CSUM_MD4_OLD 3
+#define CSUM_MD4 4
+#define CSUM_MD5 5
+#define CSUM_XXH64 6

lib/md5-asm-x86_64.S

@@ -24,11 +24,15 @@
  * show any significant difference in performance, though.
  */
 
+#include "config.h"
+#include "md-defines.h"
+
+#if !defined USE_OPENSSL && CSUM_CHUNK == 64
+
 .text
 .align 16
 
 .globl md5_process_asm
-.type md5_process_asm,@function
 md5_process_asm:
 	push	%rbp
 	push	%rbx
@@ -689,5 +693,5 @@ md5_process_asm:
 	pop	%rbx
 	pop	%rbp
 	ret
-.L_md5_process_asm_end:
-.size md5_process_asm,.L_md5_process_asm_end-md5_process_asm
+
+#endif /* !USE_OPENSSL ... */

lib/md5.c

@@ -148,7 +148,7 @@ static void md5_process(md_context *ctx, const uchar data[CSUM_CHUNK])
 	ctx->D += D;
 }
 
-#if defined(HAVE_SIMD) && (CSUM_CHUNK == 64)
+#if defined HAVE_ASM && CSUM_CHUNK == 64
 extern void md5_process_asm(md_context *ctx, const void *data, size_t num);
 #endif
 
@@ -176,7 +176,7 @@ void md5_update(md_context *ctx, const uchar *input, uint32 length)
 		left = 0;
 	}
 
-#if defined(HAVE_SIMD) && (CSUM_CHUNK == 64)
+#if defined HAVE_ASM && CSUM_CHUNK == 64
 	if (length >= CSUM_CHUNK) {
 		uint32 chunks = length / CSUM_CHUNK;
 		md5_process_asm(ctx, input, chunks);

lib/mdigest.h

@@ -4,20 +4,7 @@
 #include "openssl/md4.h"
 #include "openssl/md5.h"
 #endif
-
-#define MD4_DIGEST_LEN 16
-#define MD5_DIGEST_LEN 16
-#define MAX_DIGEST_LEN MD5_DIGEST_LEN
-
-#define CSUM_CHUNK 64
-
-#define CSUM_NONE 0
-#define CSUM_MD4_ARCHAIC 1
-#define CSUM_MD4_BUSTED 2
-#define CSUM_MD4_OLD 3
-#define CSUM_MD4 4
-#define CSUM_MD5 5
-#define CSUM_XXH64 6
+#include "md-defines.h"
 
 typedef struct {
 	uint32 A, B, C, D;

lib/sysxattrs.h

@@ -1,9 +1,9 @@
 #ifdef SUPPORT_XATTRS
 
-#if defined HAVE_ATTR_XATTR_H
-#include <attr/xattr.h>
-#elif defined HAVE_SYS_XATTR_H
+#if defined HAVE_SYS_XATTR_H
 #include <sys/xattr.h>
+#elif defined HAVE_ATTR_XATTR_H
+#include <attr/xattr.h>
 #elif defined HAVE_SYS_EXTATTR_H
 #include <sys/extattr.h>
 #endif

loadparm.c

@@ -42,6 +42,7 @@
 
 #include "rsync.h"
 #include "itypes.h"
+#include "default-dont-compress.h"
 
 extern item_list dparam_list;
 
@@ -52,11 +53,6 @@ extern item_list dparam_list;
 #define LOG_DAEMON 0
 #endif
 
-#define DEFAULT_DONT_COMPRESS "*.gz *.zip *.z *.rpm *.deb *.iso *.bz2" \
-	" *.t[gb]z *.7z *.mp[34] *.mov *.avi *.ogg *.jpg *.jpeg *.png" \
-	" *.lzo *.rzip *.lzma *.rar *.ace *.gpg *.xz *.txz *.lz *.tlz" \
-	" *.ogv *.web[mp] *.squashfs"
-
 /* the following are used by loadparm for option lists */
 typedef enum {
 	P_BOOL, P_BOOLREV, P_CHAR, P_INTEGER,

md2man

@@ -35,6 +35,7 @@ body, b, strong, u {
 code {
   font-family: 'Roboto Mono', monospace;
   font-weight: bold;
+  white-space: pre;
 }
 pre code {
   display: block;
@@ -64,7 +65,9 @@ MAN_END = """\
 
 NORM_FONT = ('\1', r"\fP")
 BOLD_FONT = ('\2', r"\fB")
-ULIN_FONT = ('\3', r"\fI")
+UNDR_FONT = ('\3', r"\fI")
+NBR_DASH = ('\4', r"\-")
+NBR_SPACE = ('\xa0', r"\ ")
 
 md_parser = None
 
@@ -80,8 +83,9 @@ def main():
     fi.title = fi.prog + '(' + fi.sect + ') man page'
     fi.mtime = 0
 
-    if os.path.lexists(fi.srcdir + '.git'):
-        fi.mtime = int(subprocess.check_output('git log -1 --format=%at'.split()))
+    git_dir = fi.srcdir + '.git'
+    if os.path.lexists(git_dir):
+        fi.mtime = int(subprocess.check_output(['git', '--git-dir', git_dir, 'log', '-1', '--format=%at']))
 
     env_subs = { 'prefix': os.environ.get('RSYNC_OVERRIDE_PREFIX', None) }
 
@@ -89,24 +93,24 @@ def main():
         env_subs['VERSION'] = '1.0.0'
         env_subs['libdir'] = '/usr'
     else:
-        for fn in 'NEWS.md Makefile'.split():
+        for fn in (fi.srcdir + 'NEWS.md', 'Makefile'):
             try:
-                st = os.lstat(fi.srcdir + fn)
+                st = os.lstat(fn)
             except:
                 die('Failed to find', fi.srcdir + fn)
             if not fi.mtime:
                 fi.mtime = st.st_mtime
 
-        with open(fi.srcdir + 'Makefile', 'r', encoding='utf-8') as fh:
+        with open('Makefile', 'r', encoding='utf-8') as fh:
             for line in fh:
                 m = re.match(r'^(\w+)=(.+)', line)
                 if not m:
                     continue
-                var, val = (m[1], m[2])
+                var, val = (m.group(1), m.group(2))
                 if var == 'prefix' and env_subs[var] is not None:
                     continue
                 while re.search(r'\$\{', val):
-                    val = re.sub(r'\$\{(\w+)\}', lambda m: env_subs[m[1]], val)
+                    val = re.sub(r'\$\{(\w+)\}', lambda m: env_subs[m.group(1)], val)
                 env_subs[var] = val
                 if var == 'VERSION':
                     break
@@ -212,7 +216,7 @@ class HtmlToManPage(HTMLParser):
             st.txt += BOLD_FONT[0]
         elif tag == 'em' or  tag == 'i':
             tag = 'u' # Change it into underline to be more like the man page
-            st.txt += ULIN_FONT[0]
+            st.txt += UNDR_FONT[0]
         elif tag == 'ol':
             start = 1
             for var, val in attrs_list:
@@ -305,15 +309,21 @@ class HtmlToManPage(HTMLParser):
             st.at_first_tag_in_dd = True
 
 
-    def handle_data(self, data):
+    def handle_data(self, txt):
         st = self.state
         if args.debug:
-            self.output_debug('DATA', (data,))
-        if st.in_code:
-            data = re.sub(r'\s', '\xa0', data) # nbsp in non-pre code
-        data = re.sub(r'\s--\s', '\xa0-- ', data)
-        st.html_out.append(htmlify(data))
-        st.txt += data
+            self.output_debug('DATA', (txt,))
+        if st.in_pre:
+            html = htmlify(txt)
+        else:
+            txt = re.sub(r'\s--(\s)', NBR_SPACE[0] + r'--\1', txt).replace('--', NBR_DASH[0]*2)
+            txt = re.sub(r'(^|\W)-', r'\1' + NBR_DASH[0], txt)
+            html = htmlify(txt)
+            if st.in_code:
+                txt = re.sub(r'\s', NBR_SPACE[0], txt)
+                html = html.replace(NBR_DASH[0], '-').replace(NBR_SPACE[0], ' ') # <code> is non-breaking in CSS
+        st.html_out.append(html.replace(NBR_SPACE[0], '&nbsp;').replace(NBR_DASH[0], '-&#8288;'))
+        st.txt += txt
 
 
     def output_debug(self, event, extra):
@@ -331,17 +341,15 @@ class HtmlToManPage(HTMLParser):
 
 def manify(txt):
     return re.sub(r"^(['.])", r'\&\1', txt.replace('\\', '\\\\')
-            .replace("\xa0", r'\ ') # non-breaking space
-            .replace('--', r'\-\-') # non-breaking double dash
+            .replace(NBR_SPACE[0], NBR_SPACE[1])
+            .replace(NBR_DASH[0], NBR_DASH[1])
             .replace(NORM_FONT[0], NORM_FONT[1])
             .replace(BOLD_FONT[0], BOLD_FONT[1])
-            .replace(ULIN_FONT[0], ULIN_FONT[1]), flags=re.M)
+            .replace(UNDR_FONT[0], UNDR_FONT[1]), flags=re.M)
 
 
 def htmlify(txt):
-    return re.sub(r'(\W)-', r'\1&#8209;',
-            txt.replace('&', '&amp;').replace('<', '&lt;').replace('>', '&gt;').replace('"', '&quot;')
-            .replace('--', '&#8209;&#8209;').replace("\xa0-", '&nbsp;&#8209;').replace("\xa0", '&nbsp;'))
+    return txt.replace('&', '&amp;').replace('<', '&lt;').replace('>', '&gt;').replace('"', '&quot;')
 
 
 def warn(*msg):

options.c

@@ -170,6 +170,7 @@ char *logfile_name = NULL;
 char *logfile_format = NULL;
 char *stdout_format = NULL;
 char *password_file = NULL;
+char *early_input_file = NULL;
 char *rsync_path = RSYNC_PATH;
 char *backup_dir = NULL;
 char backup_dir_buf[MAXPATHLEN];
@@ -180,12 +181,17 @@ int rsync_port = 0;
 int alt_dest_type = 0;
 int basis_dir_cnt = 0;
 
+static int version_opt_cnt = 0;
 static int remote_option_alloc = 0;
 int remote_option_cnt = 0;
 const char **remote_options = NULL;
 const char *checksum_choice = NULL;
 const char *compress_choice = NULL;
 
+#ifndef __APPLE__ /* Do we need a configure check for this? */
+#define SUPPORT_ATIMES 1
+#endif
+
 int quiet = 0;
 int output_motd = 1;
 int log_before_transfer = 0;
@@ -602,6 +608,11 @@ static void print_capabilities(enum logcode f)
 #endif
 			"IPv6",
 
+#ifndef SUPPORT_ATIMES
+		"no "
+#endif
+			"atimes",
+
 		"batchfiles",
 
 #ifndef HAVE_FTRUNCATE
@@ -624,6 +635,13 @@ static void print_capabilities(enum logcode f)
 #endif
 			"xattrs",
 
+#ifdef RSYNC_USE_PROTECTED_ARGS
+		"default "
+#else
+		"optional "
+#endif
+			"protect-args",
+
 #ifndef ICONV_OPTION
 		"no "
 #endif
@@ -639,14 +657,38 @@ static void print_capabilities(enum logcode f)
 #endif
 			"prealloc",
 
+	"*" /* All options after this point are hidden w/o -V -V */
 #ifndef HAVE_SIMD
 		"no "
 #endif
 			"SIMD",
 
+#ifndef HAVE_ASM
+		"no "
+#endif
+			"asm",
+
+#ifndef USE_OPENSSL
+		"no "
+#endif
+			"openssl-crypto",
+
 		NULL
 	};
 
+	for (line_len = 0, j = 0; ; j++) {
+		char *cap = capabilities[j];
+		if (!cap)
+			break;
+		if (*cap == '*') {
+			if (version_opt_cnt >= 2)
+				capabilities[j]++;
+			else
+				capabilities[j] = NULL;
+			break;
+		}
+	}
+
 	for (line_len = 0, j = 0; ; j++) {
 		char *cap = capabilities[j];
 		int cap_len = cap ? strlen(cap) : 1000;
@@ -672,7 +714,7 @@ static void print_rsync_version(enum logcode f)
 		RSYNC_NAME, RSYNC_VERSION, PROTOCOL_VERSION, subprotocol);
 
 	rprintf(f, "Copyright (C) 1996-" LATEST_YEAR " by Andrew Tridgell, Wayne Davison, and others.\n");
-	rprintf(f, "Web site: http://rsync.samba.org/\n");
+	rprintf(f, "Web site: https://rsync.samba.org/\n");
 
 	rprintf(f, "Capabilities:\n");
 	print_capabilities(f);
@@ -729,7 +771,7 @@ void usage(enum logcode F)
   rprintf(F,"\n");
   rprintf(F,"Use \"rsync --daemon --help\" to see the daemon-mode command-line options.\n");
   rprintf(F,"Please see the rsync(1) and rsyncd.conf(5) man pages for full documentation.\n");
-  rprintf(F,"See http://rsync.samba.org/ for updates, bug reports, and answers\n");
+  rprintf(F,"See https://rsync.samba.org/ for updates, bug reports, and answers\n");
 }
 
 enum {OPT_SERVER = 1000, OPT_DAEMON, OPT_SENDER, OPT_EXCLUDE, OPT_EXCLUDE_FROM,
@@ -903,6 +945,7 @@ static struct poptOption long_options[] = {
   {"zc",               0,  POPT_ARG_STRING, &compress_choice, 0, 0, 0 },
   {"skip-compress",    0,  POPT_ARG_STRING, &skip_compress, 0, 0, 0 },
   {"compress-level",   0,  POPT_ARG_INT,    &do_compression_level, 0, 0, 0 },
+  {"zl",               0,  POPT_ARG_INT,    &do_compression_level, 0, 0, 0 },
   {0,                 'P', POPT_ARG_NONE,   0, 'P', 0, 0 },
   {"progress",         0,  POPT_ARG_VAL,    &do_progress, 1, 0, 0 },
   {"no-progress",      0,  POPT_ARG_VAL,    &do_progress, 0, 0, 0 },
@@ -962,6 +1005,7 @@ static struct poptOption long_options[] = {
   {"port",             0,  POPT_ARG_INT,    &rsync_port, 0, 0, 0 },
   {"sockopts",         0,  POPT_ARG_STRING, &sockopts, 0, 0, 0 },
   {"password-file",    0,  POPT_ARG_STRING, &password_file, 0, 0, 0 },
+  {"early-input",      0,  POPT_ARG_STRING, &early_input_file, 0, 0, 0 },
   {"blocking-io",      0,  POPT_ARG_VAL,    &blocking_io, 1, 0, 0 },
   {"no-blocking-io",   0,  POPT_ARG_VAL,    &blocking_io, 0, 0, 0 },
   {"outbuf",           0,  POPT_ARG_STRING, &outbuf_mode, 0, 0, 0 },
@@ -1143,6 +1187,9 @@ static void set_refuse_options(void)
 		parse_one_refuse_match(0, "log-file*", list_end);
 	}
 
+#ifndef SUPPORT_ATIMES
+	parse_one_refuse_match(0, "atimes", list_end);
+#endif
 #ifndef SUPPORT_HARD_LINKS
 	parse_one_refuse_match(0, "link-dest", list_end);
 #endif
@@ -1361,8 +1408,8 @@ int parse_arguments(int *argc_p, const char ***argv_p)
 
 		switch (opt) {
 		case 'V':
-			print_rsync_version(FINFO);
-			exit_cleanup(0);
+			version_opt_cnt++;
+			break;
 
 		case OPT_SERVER:
 			if (!am_server) {
@@ -1848,6 +1895,11 @@ int parse_arguments(int *argc_p, const char ***argv_p)
 		}
 	}
 
+	if (version_opt_cnt) {
+		print_rsync_version(FINFO);
+		exit_cleanup(0);
+	}
+
 	if (protect_args < 0) {
 		if (am_server)
 			protect_args = 0;
@@ -2171,6 +2223,7 @@ int parse_arguments(int *argc_p, const char ***argv_p)
 	}
 	if (backup_dir) {
 		size_t len;
+		make_backups = 1; /* --backup-dir implies --backup */
 		while (*backup_dir == '.' && backup_dir[1] == '/')
 			backup_dir += 2;
 		if (*backup_dir == '.' && backup_dir[1] == '\0')

packaging/lsb/rsync.spec

@@ -1,14 +1,14 @@
 Summary: A fast, versatile, remote (and local) file-copying tool
 Name: rsync
-Version: 3.2.0
-%define fullversion %{version}pre3
-Release: 0.1.pre3
-%define srcdir src-previews
+Version: 3.2.1
+%define fullversion %{version}
+Release: 1
+%define srcdir src
 Group: Applications/Internet
 License: GPL
-Source0: http://rsync.samba.org/ftp/rsync/%{srcdir}/rsync-%{fullversion}.tar.gz
-#Source1: http://rsync.samba.org/ftp/rsync/%{srcdir}/rsync-patches-%{fullversion}.tar.gz
-URL: http://rsync.samba.org/
+Source0: https://rsync.samba.org/ftp/rsync/%{srcdir}/rsync-%{fullversion}.tar.gz
+#Source1: https://rsync.samba.org/ftp/rsync/%{srcdir}/rsync-patches-%{fullversion}.tar.gz
+URL: https://rsync.samba.org/
 
 Prefix: %{_prefix}
 BuildRoot: /var/tmp/%{name}-root
@@ -66,7 +66,7 @@ rm -rf $RPM_BUILD_ROOT
 
 %files
 %defattr(-,root,root)
-%doc COPYING NEWS.md OLDNEWS.md README.md support/ tech_report.tex
+%doc COPYING NEWS.md README.md support/ tech_report.tex
 %config(noreplace) /etc/xinetd.d/rsync
 %{_prefix}/bin/rsync
 %{_prefix}/bin/rsync-ssl
@@ -79,8 +79,8 @@ rm -rf $RPM_BUILD_ROOT
 %dir /etc/rsync-ssl/certs
 
 %changelog
-* Wed Jun 17 2020 Wayne Davison <wayne@opencoder.net>
-Released 3.2.0pre3.
+* Mon Jun 22 2020 Wayne Davison <wayne@opencoder.net>
+Released 3.2.1.
 
 * Fri Mar 21 2008 Wayne Davison <wayne@opencoder.net>
 Added installation of /etc/xinetd.d/rsync file and some commented-out

packaging/nightly-rsync

@@ -1,106 +0,0 @@
-#!/usr/bin/env -S python3 -B
-
-# This script expects the directory ~/samba-rsync-ftp to exist and to be a
-# copy of the /home/ftp/pub/rsync dir on samba.org.  It also requires a
-# git checkout of rsync (feel free to use your normal rsync build dir as
-# long as it doesn't have any uncommitted changes).
-#
-# If this is run with -tu, it will make an updated "nightly" tar file in
-# the nightly dir.  It will also remove any old tar files, regenerate the
-# HTML man pages in the nightly dir, and then rsync the changes to the
-# samba.org server.
-
-import os, sys, re, argparse, glob
-from datetime import datetime, timezone
-from getpass import getpass
-
-sys.path = ['packaging'] + sys.path
-
-from pkglib import *
-
-# Where the local copy of /home/ftp/pub/rsync/dev/nightly should be updated.
-dest = os.environ['HOME'] + '/samba-rsync-ftp/dev/nightly'
-samba_host = os.environ['SAMBA_HOST']
-nightly_symlink = f"{dest}/rsync-HEAD.tar.gz"
-
-def main():
-    now = datetime.now(timezone.utc)
-    name = now.strftime('rsync-HEAD-%Y%m%d-%H%MGMT')
-    ztoday = now.strftime('%d %b %Y')
-    today = ztoday.lstrip('0')
-    gen_target = 'gensend' if args.upload else 'gen'
-
-    if not os.path.isdir(dest):
-        die("$dest does not exist")
-    if not os.path.isdir('.git'):
-        die("There is no .git dir in the current directory.")
-    if not os.path.exists('rsyncd.conf.5.md'):
-        die("There is no rsync checkout in the current directory.")
-
-    mandate_gensend_hook()
-
-    if args.make_tar:
-        check_git_state('master')
-        cmd_chk(['touch', 'NEWS.md'])
-        cmd_chk(['make', gen_target])
-        cmd_chk(['rsync', '-a', *glob.glob('*.[1-9].html'), dest])
-
-        gen_files = get_gen_files()
-
-        confversion = get_configure_version()
-
-        # All version values are strings!
-        last_version, last_protocol_version = get_OLDNEWS_version_info()
-        protocol_version, subprotocol_version = get_protocol_versions()
-
-        if 'dev' in confversion or 'pre' in confversion:
-            if last_protocol_version != protocol_version:
-                if subprotocol_version == '0':
-                    die("SUBPROTOCOL_VERSION must not be 0 for a non-final release with a changed PROTOCOL_VERSION.")
-            elif subprotocol_version != '0':
-                die("SUBPROTOCOL_VERSION must be 0 when the PROTOCOL_VERSION hasn't changed from the last release.")
-        elif subprotocol_version != '0':
-            die("SUBPROTOCOL_VERSION must be 0 for a final release.")
-
-        name_slash = name + '/'
-        tar_name = f"{name}.tar.gz"
-
-        print('Creating', tar_name)
-
-        cmd_chk(['rsync', '-a', *gen_files, name_slash])
-        cmd_chk(f"git archive --format=tar --prefix={name}/ HEAD | tar xf -")
-        cmd_chk(['support/git-set-file-times', '--quiet', '--prefix', name_slash])
-        cmd_chk(['fakeroot', 'tar', 'czf', os.path.join(dest, tar_name), name])
-        cmd_chk(['rm', '-rf', name])
-
-        if os.path.lexists(nightly_symlink):
-            os.unlink(nightly_symlink)
-        os.symlink(tar_name, nightly_symlink)
-
-    os.chdir(dest)
-
-    tar_files = list(reversed(sorted(glob.glob('rsync-HEAD-*'))))
-    if len(tar_files) > 10:
-        for fn in tar_files[10:]:
-            print('Removing', fn)
-            os.unlink(fn)
-
-    cmd_run('ls -ltr'.split())
-
-    if args.upload:
-        cmd = 'rsync -aivHP --delete-after'.split()
-        partial_dir = os.environ.get('RSYNC_PARTIAL_DIR', None)
-        if partial_dir:
-            cmd.append('-fR ' + partial_dir)
-        cmd_chk([*cmd, '.', f"{samba_host}:/home/ftp/pub/rsync/dev/nightly"])
-
-
-if __name__ == '__main__':
-    parser = argparse.ArgumentParser(description='A helper script for "nightly" tar files.', add_help=False)
-    parser.add_argument('--make-tar', '-t', action='store_true', help=f"Create a new tar file in {dest}.")
-    parser.add_argument('--upload', '-u', action='store_true', help="Upload the revised nightly dir to {samba_host}.")
-    parser.add_argument("--help", "-h", action="help", help="Output this help message and exit.")
-    args = parser.parse_args()
-    main()
-
-# vim: sw=4 et

packaging/pkglib.py

@@ -211,15 +211,15 @@ def get_configure_version():
     die("Unable to find AC_INIT with version in configure.ac")
 
 
-def get_OLDNEWS_version_info():
+def get_NEWS_version_info():
     rel_re = re.compile(r'^\| \d{2} \w{3} \d{4}\s+\|\s+(?P<ver>\d+\.\d+\.\d+)\s+\|\s+(?P<pdate>\d{2} \w{3} \d{4}\s+)?\|\s+(?P<pver>\d+)\s+\|')
     last_version = last_protocol_version = None
     pdate = { }
 
-    with open('OLDNEWS.md', 'r', encoding='utf-8') as fh:
+    with open('NEWS.md', 'r', encoding='utf-8') as fh:
         for line in fh:
-            if not last_version:
-                m = re.search(r'(\d+\.\d+\.\d+)', line)
+            if not last_version: # Find the first non-dev|pre version with a release date.
+                m = re.search(r'rsync (\d+\.\d+\.\d+) .*\d\d\d\d', line)
                 if m:
                     last_version = m[1]
             m = rel_re.match(line)

packaging/release-rsync

@@ -55,7 +55,7 @@ def main():
     confversion = get_configure_version()
 
     # All version values are strings!
-    lastversion, last_protocol_version = get_OLDNEWS_version_info()
+    lastversion, last_protocol_version = get_NEWS_version_info()
     protocol_version, subprotocol_version = get_protocol_versions()
 
     version = confversion
@@ -153,7 +153,7 @@ release is "{release}"
 About to:
     - tweak SUBPROTOCOL_VERSION in rsync.h, if needed
     - tweak the version in configure.ac and the spec files
-    - tweak NEWS.md and OLDNEWS.md to ensure header values are correct
+    - tweak NEWS.md to ensure header values are correct
     - generate configure.sh, config.h.in, and proto.h
     - page through the differences
 """)
@@ -167,7 +167,7 @@ About to:
         '%define srcdir': srcdir,
         }
 
-    tweak_files = 'configure.ac rsync.h NEWS.md OLDNEWS.md'.split()
+    tweak_files = 'configure.ac rsync.h NEWS.md'.split()
     tweak_files += glob.glob('packaging/*.spec')
     tweak_files += glob.glob('packaging/*/*.spec')
 
@@ -188,16 +188,14 @@ About to:
             repl = lambda m: m[1] + ' ' + '0' if not pre or proto_changed != 'changed' else 1 if m[2] == '0' else m[2]
             txt = replace_or_die(x_re, repl, txt, f"Unable to find SUBPROTOCOL_VERSION define in {fn}")
         elif fn == 'NEWS.md':
-            x_re = re.compile(
-                    r'^(# NEWS for rsync %s )(\(UNRELEASED\))\s*(\n\nProtocol: )(\d+) (\([^)]+\))\n' % re.escape(finalversion),
-                    re.I)
-            repl = lambda m: m[1] + (m[2] if pre else f"({today})") + m[3] + f"{protocol_version} ({proto_changed})\n"
-            msg = (f"The first 3 lines of {fn} are not in the right format.  They must be:\n"
-                    + f"# NEWS for rsync {finalversion} (UNRELEASED)\n\n"
-                    + f"Protocol: {protocol_version} ({proto_changed})")
-            txt = replace_or_die(x_re, repl, txt, msg)
-        elif fn == 'OLDNEWS.md':
             efv = re.escape(finalversion)
+            x_re = re.compile(r'^<.+>\s+# NEWS for rsync %s \(UNRELEASED\)\s+Protocol: .+\n' % efv)
+            rel_day = 'UNRELEASED' if pre else today
+            repl = (f'<a name="{finalversion}"></a>\n\n# NEWS for rsync {finalversion} ({rel_day})\n\n'
+                + f"Protocol: {protocol_version} ({proto_changed})\n")
+            good_top = re.sub(r'\(.*?\)', '(UNRELEASED)', repl, 1)
+            msg = f"The top lines of {fn} are not in the right format.  It should be:\n" + good_top
+            txt = replace_or_die(x_re, repl, txt, msg)
             x_re = re.compile(r'^(\| )(\S{2} \S{3} \d{4})(\s+\|\s+%s\s+\| ).{11}(\s+\| )\S{2}(\s+\|+)$' % efv, re.M)
             repl = lambda m: m[1] + (m[2] if pre else ztoday) + m[3] + proto_change_date + m[4] + protocol_version + m[5]
             txt = replace_or_die(x_re, repl, txt, f'Unable to find "| ?? ??? {year} | {finalversion} | ... |" line in {fn}')
@@ -220,7 +218,6 @@ About to:
     srctar_file = f"{dest}/{srcdir}/{srctar_name}"
     pattar_file = f"{dest}/{srcdir}/{pattar_name}"
     diff_file = f"{dest}/{srcdiffdir}/{diff_name}"
-    news_file = f"{dest}/{srcdir}/{rsync_ver}-NEWS.md"
     lasttar_file = f"{dest}/{lastsrcdir}/{rsync_lastver}.tar.gz"
 
     print(f"""\
@@ -273,7 +270,7 @@ About to:
     - create release tar, "{srctar_name}"
     - generate {rsync_ver}/patches/* files
     - create patches tar, "{pattar_name}"
-    - update top-level README.md, *NEWS.md, TODO, and ChangeLog
+    - update top-level README.md, NEWS.md, TODO, and ChangeLog
     - update top-level rsync*.html manpages
     - gpg-sign the release files
     - update hard-linked top-level release files{skipping}
@@ -330,17 +327,11 @@ About to:
     shutil.rmtree(rsync_ver)
 
     print(f"Updating the other files in {dest} ...")
-    md_files = 'README.md NEWS.md OLDNEWS.md'.split()
+    md_files = 'README.md NEWS.md'.split()
     html_files = [ fn for fn in gen_files if fn.endswith('.html') ]
     cmd_chk(['rsync', '-a', *md_files, *html_files, dest])
     cmd_chk(["packaging/md2html"] + [ dest +'/'+ fn for fn in md_files ])
 
-    for topfn, verfn in (('NEWS.md', news_file), ('NEWS.html', news_file.replace('.md', '.html'))):
-        topfn = dest + '/' + topfn
-        if os.path.lexists(verfn):
-            os.unlink(verfn)
-        os.link(topfn, verfn)
-
     cmd_chk(f"git log --name-status | gzip -9 >{dest}/ChangeLog.gz")
 
     for fn in (srctar_file, pattar_file, diff_file):
@@ -352,14 +343,13 @@ About to:
             die("gpg signing failed")
 
     if not pre:
-        for find in f'{dest}/rsync-*.gz {dest}/rsync-*.asc {dest}/rsync-*-NEWS.md {dest}/src-previews/rsync-*diffs.gz*'.split():
+        for find in f'{dest}/rsync-*.gz {dest}/rsync-*.asc {dest}/src-previews/rsync-*diffs.gz*'.split():
             for fn in glob.glob(find):
                 os.unlink(fn)
         top_link = [
                 srctar_file, f"{srctar_file}.asc",
                 pattar_file, f"{pattar_file}.asc",
                 diff_file, f"{diff_file}.asc",
-                news_file,
                 ]
         for fn in top_link:
             os.link(fn, re.sub(r'/src(-\w+)?/', '/', fn))

packaging/systemd/rsync.service

@@ -2,6 +2,7 @@
 Description=fast remote file copy program daemon
 ConditionPathExists=/etc/rsyncd.conf
 After=network.target
+Documentation=man:rsync(1) man:rsyncd.conf(5)
 
 [Service]
 ExecStart=/usr/bin/rsync --daemon --no-detach

prepare-source

@@ -25,15 +25,13 @@ for action in "${@}"; do
     build|make)
 	(cd $dir && make -f prepare-source.mak)
 	;;
-    fetch)
+    fetch|fetchgen)
 	$dir/rsync-ssl -iip --no-motd rsync://download.samba.org/rsyncftp/generated-files/'[ca]*' $dir
-	if ! perl --version >/dev/null 2>/dev/null; then
-	    $dir/rsync-ssl -iip --no-motd rsync://download.samba.org/rsyncftp/generated-files/'p*' .
+	if test "$action" = fetchgen; then
+	    $dir/rsync-ssl -iip --no-motd rsync://download.samba.org/rsyncftp/generated-files/'[^ca]*' .
 	fi
-	;;
-    fetchgen)
-	$dir/rsync-ssl -iip --no-motd rsync://download.samba.org/rsyncftp/generated-files/'[ca]*' $dir
-	$dir/rsync-ssl -iip --no-motd rsync://download.samba.org/rsyncftp/generated-files/'[^ca]*' .
+	sleep 1 # The following files need to be newer than aclocal.m4
+	touch configure.sh config.h.in
 	;;
     fetchSRC)
 	./rsync-ssl -iipr --no-motd --exclude=/.git/ rsync://download.samba.org/ftp/pub/unpacked/rsync/ .

rounding.c

@@ -24,11 +24,11 @@
 
 struct test {
 	union file_extras extras[ARRAY_LEN];
-	struct file_struct file;
+	int64 test;
 };
 
 #define ACTUAL_SIZE	SIZEOF(struct test)
-#define EXPECTED_SIZE	(SIZEOF(union file_extras) * ARRAY_LEN + SIZEOF(struct file_struct))
+#define EXPECTED_SIZE	(SIZEOF(union file_extras) * ARRAY_LEN + SIZEOF(int64))
 
  int main(UNUSED(int argc), UNUSED(char *argv[]))
 {

rsync-ssl.1.md

@@ -13,6 +13,13 @@ rsync-ssl [--type=SSL_TYPE] RSYNC_ARGS
 The rsync-ssl script helps you to run an rsync copy to/from an rsync daemon
 that requires ssl connections.
 
+The script requires that you specify an rsync-daemon arg in the style of either
+`hostname::` (with 2 colons) or `rsync://hostname/`.  The default port used for
+connecting is 874 (one higher than the normal 873) unless overridden in the
+environment.  You can specify an overriding port via `--port` or by including
+it in the normal spot in the URL format, though both of those require your
+rsync version to be at least 3.2.0.
+
 # OPTIONS
 
 If the **first** arg is a `--type=SSL_TYPE` option, the script will only use
@@ -23,7 +30,7 @@ option must specify one of `openssl` or `stunnel`.  The equal sign is
 required for this particular option.
 
 All the other options are passed through to the rsync command, so consult the
-**rsync** manpage for more information on how it works.
+**rsync**(1) manpage for more information on how it works.
 
 # ENVIRONMENT VARIABLES
 
@@ -53,9 +60,13 @@ The ssl helper scripts are affected by the following environment variables:
 
 # EXAMPLES
 
->     rsync-ssl -aiv example.com::src/ dest
+>     rsync-ssl -aiv example.com::mod/ dest
 
->     rsync-ssl --type=openssl -aiv example.com::src/ dest
+>     rsync-ssl --type=openssl -aiv example.com::mod/ dest
+
+>     rsync-ssl -aiv --port 9874 example.com::mod/ dest
+
+>     rsync-ssl -aiv rsync://example.com:9874/mod/ dest
 
 # SEE ALSO
 
@@ -76,7 +87,7 @@ exported RSYNC_SSL_TYPE environment variable to make its use the default.
 
 # BUGS
 
-Please report bugs! See the web site at <http://rsync.samba.org/>.
+Please report bugs! See the web site at <https://rsync.samba.org/>.
 
 # VERSION
 
@@ -87,7 +98,7 @@ This man page is current for version @VERSION@ of rsync.
 rsync is distributed under the GNU General Public License.  See the file
 COPYING for details.
 
-A web site is available at <http://rsync.samba.org/>.  The site includes an
+A web site is available at <https://rsync.samba.org/>.  The site includes an
 FAQ-O-Matic which may cover questions unanswered by this manual page.
 
 # AUTHOR
@@ -95,4 +106,4 @@ FAQ-O-Matic which may cover questions unanswered by this manual page.
 This manpage was written by Wayne Davison.
 
 Mailing lists for support and development are available at
-<http://lists.samba.org/>.
+<https://lists.samba.org/>.

rsync.1.md

@@ -381,7 +381,7 @@ detailed description below for a complete description.
 --write-devices          write to devices as files (implies --inplace)
 --dry-run, -n            perform a trial run with no changes made
 --whole-file, -W         copy files whole (w/o delta-xfer algorithm)
---checksum-choice=STR    choose the checksum algorithms
+--checksum-choice=STR    choose the checksum algorithm (aka --cc)
 --one-file-system, -x    don't cross filesystem boundaries
 --block-size=SIZE, -B    force a fixed checksum block-size
 --rsh=COMMAND, -e        specify the remote shell to use
@@ -422,7 +422,8 @@ detailed description below for a complete description.
 --copy-dest=DIR          ... and include copies of unchanged files
 --link-dest=DIR          hardlink to files in DIR when unchanged
 --compress, -z           compress file data during the transfer
---compress-level=NUM     explicitly set compression level
+--compress-choice=STR    choose the compression algorithm (aka --zc)
+--compress-level=NUM     explicitly set compression level (aka --zl)
 --skip-compress=LIST     skip compressing files with suffix in LIST
 --cvs-exclude, -C        auto-ignore files in the same way CVS does
 --filter=RULE, -f        add a file-filtering RULE
@@ -452,6 +453,7 @@ detailed description below for a complete description.
 --log-file=FILE          log what we're doing to the specified FILE
 --log-file-format=FMT    log updates using the specified FMT
 --password-file=FILE     read daemon-access password from FILE
+--early-input=FILE       use FILE for daemon's early exec input
 --list-only              list the files instead of copying them
 --bwlimit=RATE           limit socket I/O bandwidth
 --write-batch=FILE       write a batched update to FILE
@@ -498,7 +500,7 @@ parameter, the parameter is only listed after the long variant, even though it
 must also be specified for the short.  When specifying a parameter, you can
 either use the form `--option=param` or replace the '=' with whitespace.  The
 parameter may need to be quoted in some manner for it to survive the shell's
-command-line parsing.  Keep in mind that a leading tilde (\~) in a filename is
+command-line parsing.  Keep in mind that a leading tilde (`~`) in a filename is
 substituted by your shell, so `--option=~/foo` will not change the tilde into
 your home directory (remove the '=' for that).
 
@@ -518,6 +520,9 @@ your home directory (remove the '=' for that).
     list of compression algorithms, a list of compiled-in capabilities, a link
     to the rsync web site, and some license/copyright info.
 
+    Repeat the option (`-VV`) to include some optimization info at the end of
+    the capabilities list.
+
 0.  `--verbose`, `-v`
 
     This option increases the amount of information you are given during the
@@ -640,7 +645,7 @@ your home directory (remove the '=' for that).
     after using another mirroring system which may not preserve timestamps
     exactly.
 
-0.  `--modify-window`, `-@`
+0.  `--modify-window=NUM`, `-@`
 
     When comparing two timestamps, rsync treats the timestamps as being equal
     if they differ by no more than the modify-window value.  The default is 0,
@@ -838,7 +843,7 @@ your home directory (remove the '=' for that).
 
 0.  `--backup-dir=DIR`
 
-    In combination with the `--backup` option, this tells rsync to store all
+    This implies the `--backup` option, and tells rsync to store all
     backups in the specified directory on the receiving side.  This can be used
     for incremental backups.  You can additionally specify a backup suffix
     using the `--suffix` option (otherwise the files backed up in the specified
@@ -1221,7 +1226,7 @@ your home directory (remove the '=' for that).
     those used by `--fake-super`) unless you repeat the option (e.g. `-XX`).
     This "copy all xattrs" mode cannot be used with `--fake-super`.
 
-0.  `--chmod`
+0.  `--chmod=CHMOD`
 
     This option tells rsync to apply one or more comma-separated "chmod" modes
     to the permission of the files in the transfer.  The resulting value is
@@ -1746,7 +1751,7 @@ your home directory (remove the '=' for that).
 
     Note that rsync versions prior to 3.1.0 did not allow `--min-size=0`.
 
-0.  `--block-size=BLOCKSIZE`, `-B`
+0.  `--block-size=SIZE`, `-B`
 
     This forces the block size used in rsync's delta-transfer algorithm to a
     fixed value.  It is normally selected based on the size of each file being
@@ -1852,6 +1857,8 @@ your home directory (remove the '=' for that).
     The exclude list is initialized to exclude the following items (these
     initial items are marked as perishable -- see the FILTER RULES section):
 
+    [comment]: # (This list gets used for the default-cvsignore.h file.)
+
     > `RCS`
     > `SCCS`
     > `CVS`
@@ -2066,6 +2073,9 @@ your home directory (remove the '=' for that).
 
     Rsync can also be configured (at build time) to have this option enabled by
     default (with is overridden by both the environment and the command-line).
+    Run `rsync -V` to check if this is the case, as it will display "default
+    protect-args" or "optional protect-args" depending on how it was compiled.
+
     This option will eventually become a new default setting at some
     as-yet-undetermined point in the future.
 
@@ -2302,11 +2312,39 @@ your home directory (remove the '=' for that).
     enough to handle a compression negotiation list, the list is silently
     ignored unless it contains the string "FAIL".
 
-0.  `--compress-level=NUM`
+0.  `--compress-level=NUM`, `--zl=NUM`
 
-    Explicitly set the compression level to use (see `--compress`) instead of
-    letting it default.  If NUM is non-zero, the `--compress` option is
-    implied.
+    Explicitly set the compression level to use (see `--compress`, `-z`)
+    instead of letting it default.  The `--compress` option is implied as long
+    as the level chosen is not a "don't compress" level for the compression
+    algorithm that is in effect (e.g. zlib compression treats level 0 as
+    "off").
+
+    The level values vary depending on the checksum in effect.  Because rsync
+    will negotiate a checksum choice by default when the remote rsync is new
+    enough, it can be good to combine this option with a `--compress-choice`
+    (`--zc`) option unless you're sure of the choice in effect.  For example:
+
+    >     rsync -aiv --zc=zstd --zl=22 host:src/ dest/
+
+    For zlib & zlibx compression the valid values are from 1 to 9 with 6 being
+    the default.  Specifying 0 turns compression off, and specifying -1 chooses
+    the default of 6.
+
+    For zstd compression the valid values are from -131072 to 22 with 3 being
+    the default. Specifying 0 chooses the default of 3.
+
+    For lz4 compression there are no levels, so the value is always 0.
+
+    If you specify a too-large or too-small value, the number is silently
+    limited to a valid value.  This allows you to specify something like
+    `--zl=999999999` and be assured that you'll end up with the maximum
+    compression level no matter what algorithm was chosen.
+
+    If you want to know the compression level that is in effect, specify
+    `--debug=nstr` to see the "negotiated string" results.  This will report
+    something like "`Client compress: zstd (level 3)`" (along with the checksum
+    choice in effect).
 
 0.  `--skip-compress=LIST`
 
@@ -2318,14 +2356,14 @@ your home directory (remove the '=' for that).
     possible.
 
     The **LIST** should be one or more file suffixes (without the dot) separated
-    by slashes (/).  You may specify an empty string to indicate that no files
+    by slashes (`/`).  You may specify an empty string to indicate that no files
     should be skipped.
 
     Simple character-class matching is supported: each must consist of a list
     of letters inside the square brackets (e.g. no special classes, such as
     "[:alpha:]", are supported, and '-' has no special meaning).
 
-    The characters asterisk (\*) and question-mark (?) have no special meaning.
+    The characters asterisk (`*`) and question-mark (`?`) have no special meaning.
 
     Here's an example that specifies 6 suffixes to skip (since 1 of the 5 rules
     matches 2 suffixes):
@@ -2335,38 +2373,68 @@ your home directory (remove the '=' for that).
     The default file suffixes in the skip-compress list in this version of
     rsync are:
 
+    [comment]: # (This list gets used for the default-dont-compress.h file.)
+
     > 7z
     > ace
+    > apk
     > avi
     > bz2
     > deb
+    > flac
     > gpg
     > gz
     > iso
+    > jar
     > jpeg
     > jpg
     > lz
+    > lz4
     > lzma
     > lzo
+    > mkv
     > mov
     > mp3
     > mp4
+    > odb
+    > odf
+    > odg
+    > odi
+    > odm
+    > odp
+    > ods
+    > odt
     > ogg
     > ogv
+    > opus
+    > otg
+    > oth
+    > otp
+    > ots
+    > ott
+    > oxt
     > png
     > rar
     > rpm
+    > rz
     > rzip
     > squashfs
+    > sxc
+    > sxd
+    > sxg
+    > sxm
+    > sxw
     > tbz
     > tgz
     > tlz
     > txz
+    > tzo
     > webm
     > webp
     > xz
     > z
     > zip
+    > zst
 
     This list will be replaced by your `--skip-compress` list in all but one
     situation: a copy from a daemon rsync will add your skipped suffixes to its
@@ -2399,7 +2467,7 @@ your home directory (remove the '=' for that).
     You may specify usernames or user IDs for the **FROM** and **TO** values,
     and the **FROM** value may also be a wild-card string, which will be
     matched against the sender's names (wild-cards do NOT match against ID
-    numbers, though see below for why a '\*' matches everything).  You may
+    numbers, though see below for why a '`*`' matches everything).  You may
     instead specify a range of ID numbers via an inclusive range: LOW-HIGH.
     For example:
 
@@ -2417,7 +2485,7 @@ your home directory (remove the '=' for that).
 
     Any IDs that do not have a name on the sending side are treated as having
     an empty name for the purpose of matching.  This allows them to be matched
-    via a "\*" or using an empty name.  For instance:
+    via a "`*`" or using an empty name.  For instance:
 
     >     --usermap=:nobody --groupmap=*:nobody
 
@@ -2444,19 +2512,19 @@ your home directory (remove the '=' for that).
     If you specify "`--chown=foo:bar`", this is exactly the same as specifying
     "`--usermap=*:foo --groupmap=*:bar`", only easier.
 
-0.  `--timeout=TIMEOUT`
+0.  `--timeout=SECONDS`
 
     This option allows you to set a maximum I/O timeout in seconds.  If no data
     is transferred for the specified time then rsync will exit.  The default is
     0, which means no timeout.
 
-0.  `--contimeout`
+0.  `--contimeout=SECONDS`
 
     This option allows you to set the amount of time that rsync will wait for
     its connection to an rsync daemon to succeed.  If the timeout is reached,
     rsync exits with an error.
 
-0.  `--address`
+0.  `--address=ADDRESS`
 
     By default rsync will bind to the wildcard address when connecting to an
     rsync daemon.  The `--address` option allows you to specify a specific IP
@@ -2471,15 +2539,16 @@ your home directory (remove the '=' for that).
     the port as a part of the URL).  See also this option in the `--daemon`
     mode section.
 
-0.  `--sockopts`
+0.  `--sockopts=OPTIONS`
 
     This option can provide endless fun for people who like to tune their
     systems to the utmost degree.  You can set all sorts of socket options
     which may make transfers faster (or slower!).  Read the man page for the
     `setsockopt()` system call for details on some of the options you may be
     able to set.  By default no special socket options are set.  This only
-    affects direct socket connections to a remote rsync daemon.  This option
-    also exists in the `--daemon` mode section.
+    affects direct socket connections to a remote rsync daemon.
+
+    This option also exists in the `--daemon` mode section.
 
 0.  `--blocking-io`
 
@@ -2686,7 +2755,7 @@ your home directory (remove the '=' for that).
 
     The escape idiom that started in 2.6.7 is to output a literal backslash
     (`\`) and a hash (`#`), followed by exactly 3 octal digits.  For example, a
-    newline would output as "`\\#012`".  A literal backslash that is in a
+    newline would output as "`\#012`".  A literal backslash that is in a
     filename is not escaped unless it is followed by a hash and 3 digits (0-9).
 
 0.  `--human-readable`, `-h`
@@ -2923,7 +2992,7 @@ your home directory (remove the '=' for that).
 
     CAUTION: sending SIGVTALRM to an older rsync (pre-3.2.0) will kill it.
 
-0. `--password-file=FILE`
+0.  `--password-file=FILE`
 
     This option allows you to provide a password for accessing an rsync daemon
     via a file or via standard input if **FILE** is `-`.  The file should
@@ -2938,7 +3007,16 @@ your home directory (remove the '=' for that).
     authentication (i.e. if you have also specified a password in the daemon's
     config file).
 
-0. `--list-only`
+0.  `--early-input=FILE`
+
+    This option allows rsync to send up to 5K of data to the "early exec"
+    script on its stdin.  One possible use of this data is to give the script a
+    secret that can be used to mount an encrypted filesystem (which you should
+    unmount in the the "post-xfer exec" script).
+
+    The daemon must be at least version 3.2.1.
+
+0.  `--list-only`
 
     This option will cause the source files to be listed instead of
     transferred.  This option is inferred if there is a single source arg and
@@ -2967,7 +3045,7 @@ your home directory (remove the '=' for that).
     need to expand a directory's content), or turn on recursion and exclude the
     content of subdirectories: `-r --exclude='/*/*'`.
 
-0. `--bwlimit=RATE`
+0.  `--bwlimit=RATE`
 
     This option allows you to specify the maximum transfer rate for the data
     sent over the socket, specified in units per second.  The RATE value can be
@@ -2993,13 +3071,13 @@ your home directory (remove the '=' for that).
     buffered, while other can show up as very slow when the flushing of the
     output buffer occurs.  This may be fixed in a future version.
 
-0. `--write-batch=FILE`
+0.  `--write-batch=FILE`
 
     Record a file that can later be applied to another identical destination
     with `--read-batch`.  See the "BATCH MODE" section for details, and also
     the `--only-write-batch` option.
 
-0. `--only-write-batch=FILE`
+0.  `--only-write-batch=FILE`
 
     Works like `--write-batch`, except that no updates are made on the
     destination system when creating the batch.  This lets you transport the
@@ -3018,13 +3096,13 @@ your home directory (remove the '=' for that).
     into the batch file without having to flow over the wire to the receiver
     (when pulling, the sender is remote, and thus can't write the batch).
 
-0. `--read-batch=FILE`
+0.  `--read-batch=FILE`
 
     Apply all of the changes stored in FILE, a file previously generated by
     `--write-batch`.  If _FILE_ is `-`, the batch data will be read from
     standard input. See the "BATCH MODE" section for details.
 
-0. `--protocol=NUM`
+0.  `--protocol=NUM`
 
     Force an older protocol version to be used.  This is useful for creating a
     batch file that is compatible with an older version of rsync.  For
@@ -3034,7 +3112,7 @@ your home directory (remove the '=' for that).
     protocol version to be used in the batch file (assuming you can't upgrade
     the rsync on the reading system).
 
-0. `--iconv=CONVERT_SPEC`
+0.  `--iconv=CONVERT_SPEC`
 
     Rsync can convert filenames between character sets using this option.
     Using a CONVERT_SPEC of "." tells rsync to look up the default
@@ -3066,7 +3144,7 @@ your home directory (remove the '=' for that).
     free to specify just the local charset for a daemon transfer (e.g.
     `--iconv=utf8`).
 
-0. `--ipv4`, `-4` or `--ipv6`, `-6`
+0.  `--ipv4`, `-4` or `--ipv6`, `-6`
 
     Tells rsync to prefer IPv4/IPv6 when creating sockets or running ssh.  This
     affects sockets that rsync has direct control over, such as the outgoing
@@ -3082,9 +3160,7 @@ your home directory (remove the '=' for that).
     have no effect.  The `rsync -V` output will contain "`no IPv6`" if is the
     case.
 
-    See also these options in the `--daemon` mode section.
-
-0. `--checksum-seed=NUM`
+0.  `--checksum-seed=NUM`
 
     Set the checksum seed to the integer NUM.  This 4 byte checksum seed is
     included in each block and MD4 file checksum calculation (the more modern
@@ -3099,7 +3175,7 @@ your home directory (remove the '=' for that).
 
 The options allowed when starting an rsync daemon are as follows:
 
-0. `--daemon`
+0.  `--daemon`
 
     This tells rsync that it is to run as a daemon.  The daemon you start
     running may be accessed using an rsync client using the `host::module` or
@@ -3111,7 +3187,7 @@ The options allowed when starting an rsync daemon are as follows:
     each connect made by a client and respond to requests accordingly.  See the
     **rsyncd.conf**(5) man page for more details.
 
-0. `--address`
+0.  `--address=ADDRESS`
 
     By default rsync will bind to the wildcard address when run as a daemon
     with the `--daemon` option.  The `--address` option allows you to specify a
@@ -3119,14 +3195,14 @@ The options allowed when starting an rsync daemon are as follows:
     possible in conjunction with the `--config` option.  See also the "address"
     global option in the rsyncd.conf manpage.
 
-0. `--bwlimit=RATE`
+0.  `--bwlimit=RATE`
 
     This option allows you to specify the maximum transfer rate for the data
     the daemon sends over the socket.  The client can still specify a smaller
     `--bwlimit` value, but no larger value will be allowed.  See the client
     version of this option (above) for some extra details.
 
-0. `--config=FILE`
+0.  `--config=FILE`
 
     This specifies an alternate config file than the default.  This is only
     relevant when `--daemon` is specified.  The default is /etc/rsyncd.conf
@@ -3134,7 +3210,7 @@ The options allowed when starting an rsync daemon are as follows:
     user is not the super-user; in that case the default is rsyncd.conf in the
     current directory (typically $HOME).
 
-0. `--dparam=OVERRIDE`, `-M`
+0.  `--dparam=OVERRIDE`, `-M`
 
     This option can be used to set a daemon-config parameter when starting up
     rsync in daemon mode.  It is equivalent to adding the parameter at the end
@@ -3144,7 +3220,7 @@ The options allowed when starting an rsync daemon are as follows:
 
     >     rsync --daemon -M pidfile=/path/rsync.pid
 
-0. `--no-detach`
+0.  `--no-detach`
 
     When running as a daemon, this option instructs rsync to not detach itself
     and become a background process.  This option is required when running as a
@@ -3153,37 +3229,37 @@ The options allowed when starting an rsync daemon are as follows:
     `--no-detach` is also recommended when rsync is run under a debugger.  This
     option has no effect if rsync is run from inetd or sshd.
 
-0. `--port=PORT`
+0.  `--port=PORT`
 
     This specifies an alternate TCP port number for the daemon to listen on
     rather than the default of 873.  See also the "port" global option in the
     rsyncd.conf manpage.
 
-0. `--log-file=FILE`
+0.  `--log-file=FILE`
 
     This option tells the rsync daemon to use the given log-file name instead
     of using the "`log file`" setting in the config file.
 
-0. `--log-file-format=FORMAT`
+0.  `--log-file-format=FORMAT`
 
     This option tells the rsync daemon to use the given FORMAT string instead
     of using the "`log format`" setting in the config file.  It also enables
     "`transfer logging`" unless the string is empty, in which case transfer
     logging is turned off.
 
-0. `--sockopts`
+0.  `--sockopts`
 
     This overrides the `socket options` setting in the rsyncd.conf file and has
     the same syntax.
 
-0. `--verbose`, `-v`
+0.  `--verbose`, `-v`
 
     This option increases the amount of information the daemon logs during its
     startup phase.  After the client connects, the daemon's verbosity level
     will be controlled by the options that the client used and the
     "`max verbosity`" setting in the module's config section.
 
-0. `--ipv4`, `-4` or `--ipv6`, `-6`
+0.  `--ipv4`, `-4` or `--ipv6`, `-6`
 
     Tells rsync to prefer IPv4/IPv6 when creating the incoming sockets that the
     rsync daemon will use to listen for connections.  One of these options may
@@ -3198,7 +3274,7 @@ The options allowed when starting an rsync daemon are as follows:
     have no effect.  The `rsync -V` output will contain "`no IPv6`" if is the
     case.
 
-0. `--help`, `-h`
+0.  `--help`, `-h`
 
     When specified after `--daemon`, print a short help page describing the
     options available for starting an rsync daemon.
@@ -3267,9 +3343,9 @@ include/exclude rules each specify a pattern that is matched against the names
 of the files that are going to be transferred.  These patterns can take several
 forms:
 
-- if the pattern starts with a / then it is anchored to a particular spot in
+- if the pattern starts with a `/` then it is anchored to a particular spot in
   the hierarchy of files, otherwise it is matched against the end of the
-  pathname.  This is similar to a leading ^ in regular expressions.  Thus
+  pathname.  This is similar to a leading `^` in regular expressions.  Thus
   `/foo` would match a name of "foo" at either the "root of the transfer" (for
   a global rule) or in the merge-file's directory (for a per-directory rule).
   An unqualified `foo` would match a name of "foo" anywhere in the tree because
@@ -3279,24 +3355,24 @@ forms:
   was found within a directory named "sub".  See the section on ANCHORING
   INCLUDE/EXCLUDE PATTERNS for a full discussion of how to specify a pattern
   that matches at the root of the transfer.
-- if the pattern ends with a / then it will only match a directory, not a
+- if the pattern ends with a `/` then it will only match a directory, not a
   regular file, symlink, or device.
 - rsync chooses between doing a simple string match and wildcard matching by
   checking if the pattern contains one of these three wildcard characters:
   '`*`', '`?`', and '`[`' .
 - a '`*`' matches any path component, but it stops at slashes.
 - use '`**`' to match anything, including slashes.
-- a '?' matches any character except a slash (/).
-- a '[' introduces a character class, such as [a-z] or [[:alpha:]].
+- a '`?`' matches any character except a slash (`/`).
+- a '`[`' introduces a character class, such as `[a-z]` or `[[:alpha:]]`.
 - in a wildcard pattern, a backslash can be used to escape a wildcard
   character, but it is matched literally when no wildcards are present.  This
   means that there is an extra level of backslash removal when a pattern
   contains wildcard characters compared to a pattern that has none.  e.g. if
   you add a wildcard to "`foo\bar`" (which matches the backslash) you would
   need to use "`foo\\bar*`" to avoid the "`\b`" becoming just "b".
-- if the pattern contains a / (not counting a trailing /) or a "`**`", then it
+- if the pattern contains a `/` (not counting a trailing /) or a "`**`", then it
   is matched against the full pathname, including any leading directories.  If
-  the pattern doesn't contain a / or a "`**`", then it is matched only against
+  the pattern doesn't contain a `/` or a "`**`", then it is matched only against
   the final component of the filename. (Remember that the algorithm is applied
   recursively so "full filename" can actually be any portion of a path from the
   starting directory on down.)
@@ -3311,20 +3387,20 @@ include/exclude patterns are applied recursively to the pathname of each node
 in the filesystem's tree (those inside the transfer).  The exclude patterns
 short-circuit the directory traversal stage as rsync finds the files to send.
 
-For instance, to include "/foo/bar/baz", the directories "/foo" and "/foo/bar"
+For instance, to include "`/foo/bar/baz`", the directories "`/foo`" and "`/foo/bar`"
 must not be excluded.  Excluding one of those parent directories prevents the
 examination of its content, cutting off rsync's recursion into those paths and
-rendering the include for "/foo/bar/baz" ineffectual (since rsync can't match
+rendering the include for "`/foo/bar/baz`" ineffectual (since rsync can't match
 something it never sees in the cut-off section of the directory hierarchy).
 
-The concept path exclusion is particularly important when using a trailing '\*'
+The concept path exclusion is particularly important when using a trailing '`*`'
 rule.  For instance, this won't work:
 
 >     + /some/path/this-file-will-not-be-found
 >     + /file-is-included
 >     - *
 
-This fails because the parent directory "some" is excluded by the '\*' rule, so
+This fails because the parent directory "some" is excluded by the '`*`' rule, so
 rsync never visits any of the files in the "some" or "some/path" directories.
 One solution is to ask for all directories in the hierarchy to be included by
 using a single rule: "`+ */`" (put it somewhere before the "`- *`" rule), and
@@ -3885,7 +3961,7 @@ values
 
 see also the comments on the `--delete` option
 
-Please report bugs! See the web site at <http://rsync.samba.org/>.
+Please report bugs! See the web site at <https://rsync.samba.org/>.
 
 # VERSION
 
@@ -3905,11 +3981,9 @@ that can be used with a restricted ssh login.
 rsync is distributed under the GNU General Public License.  See the file
 COPYING for details.
 
-A web site is available at <http://rsync.samba.org/>.  The site includes an
+A web site is available at <https://rsync.samba.org/>.  The site includes an
 FAQ-O-Matic which may cover questions unanswered by this manual page.
 
-The primary ftp site for rsync is <ftp://rsync.samba.org/pub/rsync>
-
 We would be delighted to hear from you if you like this program.  Please
 contact the mailing-list at <rsync@lists.samba.org>.
 
@@ -3932,4 +4006,4 @@ people have later contributed to it. It is currently maintained by Wayne
 Davison.
 
 Mailing lists for support and development are available at
-<http://lists.samba.org/>.
+<https://lists.samba.org/>.

rsync3.txt

@@ -396,7 +396,7 @@ Conflict resolution:
     would be useful.
 
 
-Moved files: <http://rsync.samba.org/cgi-bin/rsync.fom?file=44>
+Moved files:
 
   - There's no trivial way to detect renamed files, especially if they
     move between directories.
@@ -457,13 +457,11 @@ Streaming:
 
 Related work:
 
-  - mirror.pl http://freshmeat.net/project/mirror/
+  - mirror.pl
 
   - ProFTPd
 
   - Apache
 
-  - http://freshmeat.net/search/?site=Freshmeat&q=mirror&section=projects
-
   - BitTorrent -- p2p mirroring
     http://bitconjurer.org/BitTorrent/

rsyncd.conf.5.md

@@ -957,7 +957,9 @@ the values of parameters.  See the GLOBAL PARAMETERS section for more details.
     request is known except for the module name.  This helper script can be
     used to setup a disk mount or decrypt some data into a module dir, but you
     may need to use `lock file` and `max connections` to avoid concurrency
-    issues.
+    issues.  If the client rsync specified the `--early-input=FILE` option, it
+    can send up to about 5K of data to the stdin of the early script.  The
+    stdin will otherwise be empty.
 
     Note that the `post-xfer exec` command is still run even if one of the
     other scripts returns an error code. The `pre-xfer exec` command will _not_
@@ -1173,7 +1175,7 @@ The /etc/rsyncd.secrets file would look something like this:
 # BUGS
 
 Please report bugs! The rsync bug tracking system is online at
-<http://rsync.samba.org/>.
+<https://rsync.samba.org/>.
 
 # VERSION
 
@@ -1186,7 +1188,7 @@ COPYING for details.
 
 The primary ftp site for rsync is <ftp://rsync.samba.org/pub/rsync>
 
-A web site is available at <http://rsync.samba.org/>.
+A web site is available at <https://rsync.samba.org/>.
 
 We would be delighted to hear from you if you like this program.
 
@@ -1204,4 +1206,4 @@ rsync was written by Andrew Tridgell and Paul Mackerras.  Many people have
 later contributed to it.
 
 Mailing lists for support and development are available at
-<http://lists.samba.org/>.
+<https://lists.samba.org/>.

simd-checksum-x86_64.cpp

@@ -45,9 +45,10 @@
  * the available xmm registers, this optimized version may not be faster than
  * the pure C version anyway. Note that all x86-64 CPUs support at least SSE2.
  *
- * This file is compiled using GCC 4.8+'s C++ front end to allow the use of
- * the target attribute, selecting the fastest code path based on runtime
- * detection of CPU capabilities.
+ * This file is compiled using GCC 4.8+/clang 6+'s C++ front end to allow the
+ * use of the target attribute, selecting the fastest code path based on
+ * dispatch priority (GCC 5) or runtime detection of CPU capabilities (GCC 6+).
+ * GCC 4.x are not supported to ease configure.ac logic.
  */
 
 #ifdef __x86_64__
@@ -59,73 +60,34 @@
 
 #include <immintrin.h>
 
-/* Compatibility functions to let our SSSE3 algorithm run on SSE2 */
+/* Some clang versions don't like it when you use static with multi-versioned functions: linker errors */
+#ifdef __clang__
+#define MVSTATIC
+#else
+#define MVSTATIC static
+#endif
 
-__attribute__ ((target("sse2"))) static inline __m128i sse_interleave_odd_epi16(__m128i a, __m128i b)
-{
-    return _mm_packs_epi32(
-        _mm_srai_epi32(a, 16),
-        _mm_srai_epi32(b, 16)
-    );
-}
+// Missing from the headers on gcc 6 and older, clang 8 and older
+typedef long long __m128i_u __attribute__((__vector_size__(16), __may_alias__, __aligned__(1)));
+typedef long long __m256i_u __attribute__((__vector_size__(32), __may_alias__, __aligned__(1)));
 
-__attribute__ ((target("sse2"))) static inline __m128i sse_interleave_even_epi16(__m128i a, __m128i b)
-{
-    return sse_interleave_odd_epi16(
-        _mm_slli_si128(a, 2),
-        _mm_slli_si128(b, 2)
-    );
-}
+/* Compatibility macros to let our SSSE3 algorithm run with only SSE2.
+   These used to be neat individual functions with target attributes switching between SSE2 and SSSE3 implementations
+   as needed, but though this works perfectly with GCC, clang fails to inline those properly leading to a near 50%
+   performance drop - combined with static and inline modifiers gets you linker errors and even compiler crashes...
+*/
 
-__attribute__ ((target("sse2"))) static inline __m128i sse_mulu_odd_epi8(__m128i a, __m128i b)
-{
-    return _mm_mullo_epi16(
-        _mm_srli_epi16(a, 8),
-        _mm_srai_epi16(b, 8)
-    );
-}
+#define SSE2_INTERLEAVE_ODD_EPI16(a, b) _mm_packs_epi32(_mm_srai_epi32(a, 16), _mm_srai_epi32(b, 16))
+#define SSE2_INTERLEAVE_EVEN_EPI16(a, b) SSE2_INTERLEAVE_ODD_EPI16(_mm_slli_si128(a, 2), _mm_slli_si128(b, 2))
+#define SSE2_MULU_ODD_EPI8(a, b) _mm_mullo_epi16(_mm_srli_epi16(a, 8), _mm_srai_epi16(b, 8))
+#define SSE2_MULU_EVEN_EPI8(a, b) _mm_mullo_epi16(_mm_and_si128(a, _mm_set1_epi16(0xFF)), _mm_srai_epi16(_mm_slli_si128(b, 1), 8))
 
-__attribute__ ((target("sse2"))) static inline __m128i sse_mulu_even_epi8(__m128i a, __m128i b)
-{
-    return _mm_mullo_epi16(
-        _mm_and_si128(a, _mm_set1_epi16(0xFF)),
-        _mm_srai_epi16(_mm_slli_si128(b, 1), 8)
-    );
-}
+#define SSE2_HADDS_EPI16(a, b) _mm_adds_epi16(SSE2_INTERLEAVE_EVEN_EPI16(a, b), SSE2_INTERLEAVE_ODD_EPI16(a, b))
+#define SSE2_MADDUBS_EPI16(a, b) _mm_adds_epi16(SSE2_MULU_EVEN_EPI8(a, b), SSE2_MULU_ODD_EPI8(a, b))
 
-__attribute__ ((target("sse2"))) static inline __m128i sse_hadds_epi16(__m128i a, __m128i b)
-{
-    return _mm_adds_epi16(
-        sse_interleave_even_epi16(a, b),
-        sse_interleave_odd_epi16(a, b)
-    );
-}
-
-__attribute__ ((target("ssse3"))) static inline __m128i sse_hadds_epi16(__m128i a, __m128i b)
-{
-    return _mm_hadds_epi16(a, b);
-}
-
-__attribute__ ((target("sse2"))) static inline __m128i sse_maddubs_epi16(__m128i a, __m128i b)
-{
-    return _mm_adds_epi16(
-        sse_mulu_even_epi8(a, b),
-        sse_mulu_odd_epi8(a, b)
-    );
-}
-
-__attribute__ ((target("ssse3"))) static inline __m128i sse_maddubs_epi16(__m128i a, __m128i b)
-{
-    return _mm_maddubs_epi16(a, b);
-}
-
-/* These don't actually get called, but we need to define them. */
-__attribute__ ((target("default"))) static inline __m128i sse_interleave_odd_epi16(__m128i a, __m128i b) { return a; }
-__attribute__ ((target("default"))) static inline __m128i sse_interleave_even_epi16(__m128i a, __m128i b) { return a; }
-__attribute__ ((target("default"))) static inline __m128i sse_mulu_odd_epi8(__m128i a, __m128i b) { return a; }
-__attribute__ ((target("default"))) static inline __m128i sse_mulu_even_epi8(__m128i a, __m128i b) { return a; }
-__attribute__ ((target("default"))) static inline __m128i sse_hadds_epi16(__m128i a, __m128i b) { return a; }
-__attribute__ ((target("default"))) static inline __m128i sse_maddubs_epi16(__m128i a, __m128i b) { return a; }
+__attribute__ ((target("default"))) MVSTATIC int32 get_checksum1_avx2_64(schar* buf, int32 len, int32 i, uint32* ps1, uint32* ps2) { return i; }
+__attribute__ ((target("default"))) MVSTATIC int32 get_checksum1_ssse3_32(schar* buf, int32 len, int32 i, uint32* ps1, uint32* ps2) { return i; }
+__attribute__ ((target("default"))) MVSTATIC int32 get_checksum1_sse2_32(schar* buf, int32 len, int32 i, uint32* ps1, uint32* ps2) { return i; }
 
 /*
   Original loop per 4 bytes:
@@ -146,12 +108,7 @@ __attribute__ ((target("default"))) static inline __m128i sse_maddubs_epi16(__m1
     s1 += (uint32)(t1[0] + t1[1] + t1[2] + t1[3] + t1[4] + t1[5] + t1[6] + t1[7]) +
           32*CHAR_OFFSET;
  */
-/*
-  Both sse2 and ssse3 targets must be specified here or we lose (a lot) of
-  performance, possibly due to not unrolling+inlining the called targeted
-  functions.
- */
-__attribute__ ((target("sse2", "ssse3"))) static int32 get_checksum1_sse2_32(schar* buf, int32 len, int32 i, uint32* ps1, uint32* ps2)
+__attribute__ ((target("ssse3"))) MVSTATIC int32 get_checksum1_ssse3_32(schar* buf, int32 len, int32 i, uint32* ps1, uint32* ps2)
 {
     if (len > 32) {
         int aligned = ((uintptr_t)buf & 15) == 0;
@@ -167,16 +124,11 @@ __attribute__ ((target("sse2", "ssse3"))) static int32 get_checksum1_sse2_32(sch
 
         for (; i < (len-32); i+=32) {
             // Load ... 2*[int8*16]
-            // SSSE3 has _mm_lqqdu_si128, but this requires another
-            // target function for each SSE2 and SSSE3 loads. For reasons
-            // unknown (to me) we lose about 10% performance on some CPUs if
-            // we do that right here. We just use _mm_loadu_si128 as for all
-            // but a handful of specific old CPUs they are synonymous, and
-            // take the 1-5% hit on those specific CPUs where it isn't.
             __m128i in8_1, in8_2;
             if (!aligned) {
-                in8_1 = _mm_loadu_si128((__m128i_u*)&buf[i]);
-                in8_2 = _mm_loadu_si128((__m128i_u*)&buf[i + 16]);
+                // Synonymous with _mm_loadu_si128 on all but a handful of old CPUs
+                in8_1 = _mm_lddqu_si128((__m128i_u*)&buf[i]);
+                in8_2 = _mm_lddqu_si128((__m128i_u*)&buf[i + 16]);
             } else {
                 in8_1 = _mm_load_si128((__m128i_u*)&buf[i]);
                 in8_2 = _mm_load_si128((__m128i_u*)&buf[i + 16]);
@@ -185,13 +137,13 @@ __attribute__ ((target("sse2", "ssse3"))) static int32 get_checksum1_sse2_32(sch
             // (1*buf[i] + 1*buf[i+1]), (1*buf[i+2], 1*buf[i+3]), ... 2*[int16*8]
             // Fastest, even though multiply by 1
             __m128i mul_one = _mm_set1_epi8(1);
-            __m128i add16_1 = sse_maddubs_epi16(mul_one, in8_1);
-            __m128i add16_2 = sse_maddubs_epi16(mul_one, in8_2);
+            __m128i add16_1 = _mm_maddubs_epi16(mul_one, in8_1);
+            __m128i add16_2 = _mm_maddubs_epi16(mul_one, in8_2);
 
             // (4*buf[i] + 3*buf[i+1]), (2*buf[i+2], buf[i+3]), ... 2*[int16*8]
             __m128i mul_const = _mm_set1_epi32(4 + (3 << 8) + (2 << 16) + (1 << 24));
-            __m128i mul_add16_1 = sse_maddubs_epi16(mul_const, in8_1);
-            __m128i mul_add16_2 = sse_maddubs_epi16(mul_const, in8_2);
+            __m128i mul_add16_1 = _mm_maddubs_epi16(mul_const, in8_1);
+            __m128i mul_add16_2 = _mm_maddubs_epi16(mul_const, in8_2);
 
             // s2 += 32*s1
             ss2 = _mm_add_epi32(ss2, _mm_slli_epi32(ss1, 5));
@@ -224,7 +176,111 @@ __attribute__ ((target("sse2", "ssse3"))) static int32 get_checksum1_sse2_32(sch
             // [t1[0] + t1[1], t1[2] + t1[3] ...] [int16*8]
             // We could've combined this with generating sum_add32 above and
             // save an instruction but benchmarking shows that as being slower
-            __m128i add16 = sse_hadds_epi16(add16_1, add16_2);
+            __m128i add16 = _mm_hadds_epi16(add16_1, add16_2);
+
+            // [t1[0], t1[1], ...] -> [t1[0]*28 + t1[1]*24, ...] [int32*4]
+            __m128i mul32 = _mm_madd_epi16(add16, mul_t1);
+
+            // [sum(mul32), X, X, X] [int32*4]; faster than multiple _mm_hadd_epi32
+            mul32 = _mm_add_epi32(mul32, _mm_srli_si128(mul32, 4));
+            mul32 = _mm_add_epi32(mul32, _mm_srli_si128(mul32, 8));
+
+            // s2 += 28*t1[0] + 24*t1[1] + 20*t1[2] + 16*t1[3] + 12*t1[4] + 8*t1[5] + 4*t1[6]
+            ss2 = _mm_add_epi32(ss2, mul32);
+
+#if CHAR_OFFSET != 0
+            // s1 += 32*CHAR_OFFSET
+            __m128i char_offset_multiplier = _mm_set1_epi32(32 * CHAR_OFFSET);
+            ss1 = _mm_add_epi32(ss1, char_offset_multiplier);
+
+            // s2 += 528*CHAR_OFFSET
+            char_offset_multiplier = _mm_set1_epi32(528 * CHAR_OFFSET);
+            ss2 = _mm_add_epi32(ss2, char_offset_multiplier);
+#endif
+        }
+
+        _mm_store_si128((__m128i_u*)x, ss1);
+        *ps1 = x[0];
+        _mm_store_si128((__m128i_u*)x, ss2);
+        *ps2 = x[0];
+    }
+    return i;
+}
+
+/*
+  Same as SSSE3 version, but using macros defined above to emulate SSSE3 calls that are not available with SSE2.
+  For GCC-only the SSE2 and SSSE3 versions could be a single function calling other functions with the right
+  target attributes to emulate SSSE3 calls on SSE2 if needed, but clang doesn't inline those properly leading
+  to a near 50% performance drop.
+ */
+__attribute__ ((target("sse2"))) MVSTATIC int32 get_checksum1_sse2_32(schar* buf, int32 len, int32 i, uint32* ps1, uint32* ps2)
+{
+    if (len > 32) {
+        int aligned = ((uintptr_t)buf & 15) == 0;
+
+        uint32 x[4] = {0};
+        x[0] = *ps1;
+        __m128i ss1 = _mm_loadu_si128((__m128i_u*)x);
+        x[0] = *ps2;
+        __m128i ss2 = _mm_loadu_si128((__m128i_u*)x);
+
+        const int16 mul_t1_buf[8] = {28, 24, 20, 16, 12, 8, 4, 0};
+        __m128i mul_t1 = _mm_loadu_si128((__m128i_u*)mul_t1_buf);
+
+        for (; i < (len-32); i+=32) {
+            // Load ... 2*[int8*16]
+            __m128i in8_1, in8_2;
+            if (!aligned) {
+                in8_1 = _mm_loadu_si128((__m128i_u*)&buf[i]);
+                in8_2 = _mm_loadu_si128((__m128i_u*)&buf[i + 16]);
+            } else {
+                in8_1 = _mm_load_si128((__m128i_u*)&buf[i]);
+                in8_2 = _mm_load_si128((__m128i_u*)&buf[i + 16]);
+            }
+
+            // (1*buf[i] + 1*buf[i+1]), (1*buf[i+2], 1*buf[i+3]), ... 2*[int16*8]
+            // Fastest, even though multiply by 1
+            __m128i mul_one = _mm_set1_epi8(1);
+            __m128i add16_1 = SSE2_MADDUBS_EPI16(mul_one, in8_1);
+            __m128i add16_2 = SSE2_MADDUBS_EPI16(mul_one, in8_2);
+
+            // (4*buf[i] + 3*buf[i+1]), (2*buf[i+2], buf[i+3]), ... 2*[int16*8]
+            __m128i mul_const = _mm_set1_epi32(4 + (3 << 8) + (2 << 16) + (1 << 24));
+            __m128i mul_add16_1 = SSE2_MADDUBS_EPI16(mul_const, in8_1);
+            __m128i mul_add16_2 = SSE2_MADDUBS_EPI16(mul_const, in8_2);
+
+            // s2 += 32*s1
+            ss2 = _mm_add_epi32(ss2, _mm_slli_epi32(ss1, 5));
+
+            // [sum(t1[0]..t1[7]), X, X, X] [int32*4]; faster than multiple _mm_hadds_epi16
+            // Shifting left, then shifting right again and shuffling (rather than just
+            // shifting right as with mul32 below) to cheaply end up with the correct sign
+            // extension as we go from int16 to int32.
+            __m128i sum_add32 = _mm_add_epi16(add16_1, add16_2);
+            sum_add32 = _mm_add_epi16(sum_add32, _mm_slli_si128(sum_add32, 2));
+            sum_add32 = _mm_add_epi16(sum_add32, _mm_slli_si128(sum_add32, 4));
+            sum_add32 = _mm_add_epi16(sum_add32, _mm_slli_si128(sum_add32, 8));
+            sum_add32 = _mm_srai_epi32(sum_add32, 16);
+            sum_add32 = _mm_shuffle_epi32(sum_add32, 3);
+
+            // [sum(t2[0]..t2[7]), X, X, X] [int32*4]; faster than multiple _mm_hadds_epi16
+            __m128i sum_mul_add32 = _mm_add_epi16(mul_add16_1, mul_add16_2);
+            sum_mul_add32 = _mm_add_epi16(sum_mul_add32, _mm_slli_si128(sum_mul_add32, 2));
+            sum_mul_add32 = _mm_add_epi16(sum_mul_add32, _mm_slli_si128(sum_mul_add32, 4));
+            sum_mul_add32 = _mm_add_epi16(sum_mul_add32, _mm_slli_si128(sum_mul_add32, 8));
+            sum_mul_add32 = _mm_srai_epi32(sum_mul_add32, 16);
+            sum_mul_add32 = _mm_shuffle_epi32(sum_mul_add32, 3);
+
+            // s1 += t1[0] + t1[1] + t1[2] + t1[3] + t1[4] + t1[5] + t1[6] + t1[7]
+            ss1 = _mm_add_epi32(ss1, sum_add32);
+
+            // s2 += t2[0] + t2[1] + t2[2] + t2[3] + t2[4] + t2[5] + t2[6] + t2[7]
+            ss2 = _mm_add_epi32(ss2, sum_mul_add32);
+
+            // [t1[0] + t1[1], t1[2] + t1[3] ...] [int16*8]
+            // We could've combined this with generating sum_add32 above and
+            // save an instruction but benchmarking shows that as being slower
+            __m128i add16 = SSE2_HADDS_EPI16(add16_1, add16_2);
 
             // [t1[0], t1[1], ...] -> [t1[0]*28 + t1[1]*24, ...] [int32*4]
             __m128i mul32 = _mm_madd_epi16(add16, mul_t1);
@@ -270,7 +326,7 @@ __attribute__ ((target("sse2", "ssse3"))) static int32 get_checksum1_sse2_32(sch
     s1 += (uint32)(t1[0] + t1[1] + t1[2] + t1[3] + t1[4] + t1[5] + t1[6] + t1[7] + t1[8] + t1[9] + t1[10] + t1[11] + t1[12] + t1[13] + t1[14] + t1[15]) +
           64*CHAR_OFFSET;
  */
-__attribute__ ((target("avx2"))) static int32 get_checksum1_avx2_64(schar* buf, int32 len, int32 i, uint32* ps1, uint32* ps2)
+__attribute__ ((target("avx2"))) MVSTATIC int32 get_checksum1_avx2_64(schar* buf, int32 len, int32 i, uint32* ps1, uint32* ps2)
 {
     if (len > 64) {
         // Instructions reshuffled compared to SSE2 for slightly better performance
@@ -377,17 +433,7 @@ __attribute__ ((target("avx2"))) static int32 get_checksum1_avx2_64(schar* buf,
     return i;
 }
 
-__attribute__ ((target("default"))) static int32 get_checksum1_avx2_64(schar* buf, int32 len, int32 i, uint32* ps1, uint32* ps2)
-{
-    return i;
-}
-
-__attribute__ ((target("default"))) static int32 get_checksum1_sse2_32(schar* buf, int32 len, int32 i, uint32* ps1, uint32* ps2)
-{
-    return i;
-}
-
-static inline int32 get_checksum1_default_1(schar* buf, int32 len, int32 i, uint32* ps1, uint32* ps2)
+static int32 get_checksum1_default_1(schar* buf, int32 len, int32 i, uint32* ps1, uint32* ps2)
 {
     uint32 s1 = *ps1;
     uint32 s2 = *ps2;
@@ -403,9 +449,10 @@ static inline int32 get_checksum1_default_1(schar* buf, int32 len, int32 i, uint
     return i;
 }
 
-extern "C" {
-
-uint32 get_checksum1(char *buf1, int32 len)
+/* With GCC 10 putting this implementation inside 'extern "C"' causes an
+   assembler error. That worked fine on GCC 5-9 and clang 6-10...
+  */
+static inline uint32 get_checksum1_cpp(char *buf1, int32 len)
 {
     int32 i = 0;
     uint32 s1 = 0;
@@ -414,7 +461,10 @@ uint32 get_checksum1(char *buf1, int32 len)
     // multiples of 64 bytes using AVX2 (if available)
     i = get_checksum1_avx2_64((schar*)buf1, len, i, &s1, &s2);
 
-    // multiples of 32 bytes using SSE2/SSSE3 (if available)
+    // multiples of 32 bytes using SSSE3 (if available)
+    i = get_checksum1_ssse3_32((schar*)buf1, len, i, &s1, &s2);
+
+    // multiples of 32 bytes using SSE2 (if available)
     i = get_checksum1_sse2_32((schar*)buf1, len, i, &s1, &s2);
 
     // whatever is left
@@ -423,7 +473,70 @@ uint32 get_checksum1(char *buf1, int32 len)
     return (s1 & 0xffff) + (s2 << 16);
 }
 
-} // "C"
+extern "C" {
+
+uint32 get_checksum1(char *buf1, int32 len)
+{
+    return get_checksum1_cpp(buf1, len);
+}
+
+} // extern "C"
+
+#ifdef BENCHMARK_SIMD_CHECKSUM1
+#pragma clang optimize off
+#pragma GCC push_options
+#pragma GCC optimize ("O0")
+
+#define ROUNDS 1024
+#define BLOCK_LEN 1024*1024
+
+#ifndef CLOCK_MONOTONIC_RAW
+#define CLOCK_MONOTONIC_RAW CLOCK_MONOTONIC
+#endif
+
+static void benchmark(const char* desc, int32 (*func)(schar* buf, int32 len, int32 i, uint32* ps1, uint32* ps2), schar* buf, int32 len) {
+    struct timespec start, end;
+    uint64_t us;
+    uint32_t cs, s1, s2;
+    int i, next;
+
+    clock_gettime(CLOCK_MONOTONIC_RAW, &start);
+    for (i = 0; i < ROUNDS; i++) {
+        s1 = s2 = 0;
+        next = func((schar*)buf, len, 0, &s1, &s2);
+        get_checksum1_default_1((schar*)buf, len, next, &s1, &s2);
+    }
+    clock_gettime(CLOCK_MONOTONIC_RAW, &end);
+    us = next == 0 ? 0 : (end.tv_sec - start.tv_sec) * 1000000 + (end.tv_nsec - start.tv_nsec) / 1000;
+    cs = next == 0 ? 0 : (s1 & 0xffff) + (s2 << 16);
+    printf("%-5s :: %5.0f MB/s :: %08x\n", desc, us ? (float)(len / (1024 * 1024) * ROUNDS) / ((float)us / 1000000.0f) : 0, cs);
+}
+
+static int32 get_checksum1_auto(schar* buf, int32 len, int32 i, uint32* ps1, uint32* ps2) {
+    uint32 cs = get_checksum1((char*)buf, len);
+    *ps1 = cs & 0xffff;
+    *ps2 = cs >> 16;
+    return len;
+}
+
+int main() {
+    int i;
+    unsigned char* buf = (unsigned char*)malloc(BLOCK_LEN);
+    for (i = 0; i < BLOCK_LEN; i++) buf[i] = (i + (i % 3) + (i % 11)) % 256;
+
+    benchmark("Auto", get_checksum1_auto, (schar*)buf, BLOCK_LEN);
+    benchmark("Raw-C", get_checksum1_default_1, (schar*)buf, BLOCK_LEN);
+    benchmark("SSE2", get_checksum1_sse2_32, (schar*)buf, BLOCK_LEN);
+    benchmark("SSSE3", get_checksum1_ssse3_32, (schar*)buf, BLOCK_LEN);
+    benchmark("AVX2", get_checksum1_avx2_64, (schar*)buf, BLOCK_LEN);
+
+    free(buf);
+    return 0;
+}
+
+#pragma GCC pop_options
+#pragma clang optimize on
+#endif /* BENCHMARK_SIMD_CHECKSUM1 */
 
 #endif /* HAVE_SIMD */
 #endif /* __cplusplus */

testsuite/atimes.test

@@ -4,6 +4,8 @@
 
 . "$suitedir/rsync.fns"
 
+$RSYNC --version | grep "[, ] atimes" >/dev/null || test_skipped "Rsync is configured without atimes support"
+
 mkdir "$fromdir"
 
 touch "$fromdir/foo"

token.c

@@ -1135,30 +1135,23 @@ void send_token(int f, int32 token, struct map_struct *buf, OFF_T offset,
  */
 int32 recv_token(int f, char **data)
 {
-	int tok;
-
 	switch (do_compression) {
 	case CPRES_NONE:
-		tok = simple_recv_token(f,data);
-		break;
+		return simple_recv_token(f,data);
 	case CPRES_ZLIB:
 	case CPRES_ZLIBX:
-		tok = recv_deflated_token(f, data);
-		break;
+		return recv_deflated_token(f, data);
 #ifdef SUPPORT_ZSTD
 	case CPRES_ZSTD:
-		tok = recv_zstd_token(f, data);
-		break;
+		return recv_zstd_token(f, data);
 #endif
 #ifdef SUPPORT_LZ4
 	case CPRES_LZ4:
-		tok = recv_compressed_token(f, data);
-		break;
+		return recv_compressed_token(f, data);
 #endif
 	default:
 		assert(0);
 	}
-	return tok;
 }
 
 /*

uidlist.c

@@ -527,14 +527,14 @@ const char *getallgroups(uid_t uid, item_list *gid_list)
 		return "getpwuid failed";
 
 	gid_list->count = 0; /* We're overwriting any items in the list */
-	EXPAND_ITEM_LIST(gid_list, gid_t, 32);
+	(void)EXPAND_ITEM_LIST(gid_list, gid_t, 32);
 	size = gid_list->malloced;
 
 	/* Get all the process's groups, with the pw_gid group first. */
 	if (getgrouplist(pw->pw_name, pw->pw_gid, gid_list->items, &size) < 0) {
 		if (size > (int)gid_list->malloced) {
 			gid_list->count = gid_list->malloced;
-			EXPAND_ITEM_LIST(gid_list, gid_t, size);
+			(void)EXPAND_ITEM_LIST(gid_list, gid_t, size);
 			if (getgrouplist(pw->pw_name, pw->pw_gid, gid_list->items, &size) < 0)
 				size = -1;
 		} else
@@ -553,7 +553,7 @@ const char *getallgroups(uid_t uid, item_list *gid_list)
 				break;
 		}
 		if (j == size) { /* The default group wasn't found! */
-			EXPAND_ITEM_LIST(gid_list, gid_t, size+1);
+			(void)EXPAND_ITEM_LIST(gid_list, gid_t, size+1);
 			gid_array = gid_list->items;
 		}
 		gid_array[j] = gid_array[0];