Preparing for release of 3.2.0

xattr headers have been provided by glibc (at least on Linux/glibc)
for many years now. Reorder the inclusion of xattr headers to
attempt compatibility/legacy after the common case.
This prevents the warning without changing compatibility to
non-glibc systems.

* Add dependency on lib/sysxattrs.h header in Makefile

Co-authored-by: Wayne Davison <wayne@opencoder.net>

.github/workflows/ccpp.yml

@@ -18,7 +18,7 @@ jobs:
     - name: prepare-source
       run: ./prepare-source
     - name: configure
-      run: ./configure --with-included-popt --with-protected-args --with-included-zlib --enable-simd
+      run: ./configure --with-included-popt --with-included-zlib
     - name: make
       run: make
     - name: version-summary

.gitignore

@@ -19,6 +19,8 @@ aclocal.m4
 /rsync*.5
 /rsync*.html
 /help-rsync*.h
+/default-cvsignore.h
+/default-dont-compress.h
 /.md2man-works
 /autom4te*.cache
 /confdefs.h

Makefile.in

@@ -10,6 +10,7 @@ mandir=@mandir@
 
 LIBS=@LIBS@
 CC=@CC@
+AWK=@AWK@
 CFLAGS=@CFLAGS@
 CPPFLAGS=@CPPFLAGS@
 CXX=@CXX@
@@ -69,7 +70,7 @@ CHECK_OBJS=tls.o testrun.o getgroups.o getfsdev.o t_stub.o t_unsafe.o trimslash.
 @OBJ_RESTORE@
 
 .PHONY: all
-all: Makefile rsync$(EXEEXT) stunnel-rsyncd.conf man
+all: Makefile rsync$(EXEEXT) stunnel-rsyncd.conf @MAKE_MAN@
 
 .PHONY: install
 install: all
@@ -99,12 +100,18 @@ rsync$(EXEEXT): $(OBJS)
 
 $(OBJS): $(HEADERS)
 $(CHECK_OBJS): $(HEADERS)
+tls.o xattrs.o: lib/sysxattrs.h
 options.o: latest-year.h help-rsync.h help-rsyncd.h
+exclude.o: default-cvsignore.h
+loadparm.o: default-dont-compress.h
 
 flist.o: rounding.h
 
-help-rsync.h help-rsyncd.h: rsync.1.md
+default-cvsignore.h default-dont-compress.h: rsync.1.md define-from-md.awk
-	./help-from-md "$(srcdir)/$<" $@
+	$(AWK) -f $(srcdir)/define-from-md.awk -v hfile=$@ $(srcdir)/rsync.1.md
+
+help-rsync.h help-rsyncd.h: rsync.1.md help-from-md.awk
+	$(AWK) -f $(srcdir)/help-from-md.awk -v hfile=$@ $(srcdir)/rsync.1.md
 
 rounding.h: rounding.c rsync.h proto.h
 	@for r in 0 1 3; do \
@@ -125,10 +132,10 @@ rounding.h: rounding.c rsync.h proto.h
 	@rm -f rounding.out
 
 simd-checksum-x86_64.o: simd-checksum-x86_64.cpp
-	$(CXX) $(CXXFLAGS) $(CPPFLAGS) -c -o $@ $<
+	$(CXX) $(CXXFLAGS) $(CPPFLAGS) -c -o $@ $(srcdir)/simd-checksum-x86_64.cpp
 
 lib/md5-asm-x86_64.o: lib/md5-asm-x86_64.s
-	$(CC) -c -o $@ $<
+	$(CC) -c -o $@ $(srcdir)/lib/md5-asm-x86_64.s
 
 tls$(EXEEXT): $(TLS_OBJ)
 	$(CC) $(CFLAGS) $(LDFLAGS) -o $@ $(TLS_OBJ) $(LIBS)
@@ -220,7 +227,7 @@ proto.h: proto.h-tstamp
 	@if test -f proto.h; then :; else cp -p $(srcdir)/proto.h .; fi
 
 proto.h-tstamp: $(srcdir)/*.c $(srcdir)/lib/compat.c config.h
-	awk -f $(srcdir)/mkproto.awk $(srcdir)/*.c $(srcdir)/lib/compat.c
+	$(AWK) -f $(srcdir)/mkproto.awk $(srcdir)/*.c $(srcdir)/lib/compat.c
 
 .PHONY: man
 man: rsync.1 rsync-ssl.1 rsyncd.conf.5

NEWS.md

@@ -1,4 +1,4 @@
-# NEWS for rsync 3.2.0 (UNRELEASED)
+# NEWS for rsync 3.2.0 (19 Jun 2020)
 
 Protocol: 31 (unchanged)
 
@@ -72,7 +72,8 @@ Protocol: 31 (unchanged)
    strings (for checksums and compression).  The first level just outputs the
    result of each negotiation on the client, level 2 outputs the values of the
    strings that were sent to and received from the server, and level 3 outputs
-   all those values on the server side too (when given the debug option).
+   all those values on the server side too (when the server was given the debug
+   option).
 
  - The --debug=OPTS command-line option is no longer auto-forwarded to the
    remote rsync which allows for the client and server to have different levels
@@ -104,7 +105,9 @@ Protocol: 31 (unchanged)
 
  - Added negated matching to the daemon's `refuse options` setting by using
    match strings that start with a `!` (such as `!compress*`).  This lets you
-   refuse all options except for a particular approved list, for example.
+   refuse all options except for a particular approved list, for example.  It
+   also lets rsync refuse certain options by default (such as `write-devices`)
+   while allowing the config to override that, as desired.
 
  - Added the `early exec` daemon parameter that runs a script before the
    transfer parameters are known, allowing some early setup based on module
@@ -120,7 +123,7 @@ Protocol: 31 (unchanged)
    copy.  This requires both sender & receiver to be at least v3.2.0.
 
  - Added support for `RSYNC_SHELL` & `RSYNC_NO_XFER_EXEC` environment variables
-   that affect the pre-xfer exec and post-xfer exec rsync daemon options.
+   that affect the early, pre-xfer, and post-xfer exec rsync daemon parameters.
 
  - Optimize the `--fuzzy --fuzzy` heuristic to avoid the fuzzy directory scan
    until all other basis-file options are exhausted (such as `--link-dest`).
@@ -129,23 +132,28 @@ Protocol: 31 (unchanged)
    transfer exited with an error when possible (i.e. if it is the sender).
 
  - The daemon now locks its pid file (when configured to use one) so that it
-   will not fail to start when the file exists and it is unlocked.
+   will not fail to start when the file exists but no daemon is running.
 
  - Various man page improvements, including some html representations (that
    aren't installed by default).
 
  - Made -V the short option for --version and improved its information.
 
- - Forward -4 or -6 to the ssh command, making it easier to type than
+ - Pass the -4 or -6 option to the ssh command, making it easier to type than
    `--rsh='ssh -4'` (or -6).
 
+ - Added example config for rsyncd SSL proxy configs to rsyncd.conf.
+
+ - More errors messages now mention if the error is coming from the sender or
+   the receiver.
+
 ### PACKAGING RELATED:
 
- - Add installed binary: /usr/bin/rsync-sll
+ - Add installed binary: /usr/bin/rsync-ssl
 
  - Add installed man page: /usr/man/man1/rsync-ssl.1
 
- - Tweak auxilliary doc file names, such as: README.md, INSTALL.md, NEWS.md, &
+ - Tweak auxiliary doc file names, such as: README.md, INSTALL.md, NEWS.md, &
    OLDNEWS.md.
 
  - The rsync-ssl script wants to run openssl or stunnel4, so consider adding a
@@ -162,11 +170,16 @@ Protocol: 31 (unchanged)
    algorithms, extra checksum algorithms, and allow use of openssl's crypto
    lib for (potentially) faster MD4/MD5 checksums.
 
- - Add _build_ dependency for g++ to enable the SIMD checksum optimizations.
+ - Add _build_ dependency for g++ or clang++ on x86_64 systems to enable the
+   SIMD checksum optimizations.
 
  - Add _build_ dependency for _either_ python3-cmarkcfm or python3-commonmark
-   to allow for patching of man pages or building a git release.  Note that
+   to allow for patching of man pages or building a git release.  This is not
-   cmarkcfm is faster than commonmark, but they generate the same data.
+   required for a release-tar build, since it comes with pre-built man pages.
+   Note that cmarkcfm is faster than commonmark, but they generate the same
+   data.  The commonmark dependency is easiest to install since it's native
+   python, and can be installed via `pip3 install --user commonmark` if you
+   want to just install it for the build user (or omit `--user`).
 
  - Remove yodl _build_ dependency (if it was even listed before).
 

OLDNEWS.md

@@ -3794,7 +3794,7 @@ Protocol: 25 (changed)
 
 | RELEASE DATE | VER.   | DATE OF COMMIT\* | PROTOCOL    |
 |--------------|--------|------------------|-------------|
-| ?? Jun 2020  | 3.2.0  |                  | 31          |
+| 19 Jun 2020  | 3.2.0  |                  | 31          |
 | 28 Jan 2018  | 3.1.3  |                  | 31          |
 | 21 Dec 2015  | 3.1.2  |                  | 31          |
 | 22 Jun 2014  | 3.1.1  |                  | 31          |

batch.c

@@ -271,7 +271,7 @@ void write_batch_shell_file(void)
 
 	/* We need to make sure that any protocol-based or negotiated choices get accurately
 	 * reflected in the options we save AND that we avoid any need for --read-batch to
-	 * do a string-based negotation (since we don't write them into the file). */
+	 * do a string-based negotiation (since we don't write them into the file). */
 	if (do_compression)
 		err |= write_opt("--compress-choice", compress_choice);
 	if (strchr(checksum_choice, ',') || xfersum_type != parse_csum_name(NULL, -1))

clientname.c

@@ -388,7 +388,7 @@ static int compare_addrinfo_sockaddr(const struct addrinfo *ai, const struct soc
 		sin1 = (const struct sockaddr_in6 *) ss;
 		sin2 = (const struct sockaddr_in6 *) ai->ai_addr;
 
-		if (ai->ai_addrlen < sizeof (struct sockaddr_in6)) {
+		if (ai->ai_addrlen < (int)sizeof (struct sockaddr_in6)) {
 			rprintf(FLOG, "%s: too short sockaddr_in6; length=%d\n",
 				fn, (int)ai->ai_addrlen);
 			return 1;

clientserver.c

@@ -394,7 +394,7 @@ void set_env_num(const char *var, long num)
 /* Used for both early exec & pre-xfer exec */
 static pid_t start_pre_exec(const char *cmd, int *arg_fd_ptr, int *error_fd_ptr)
 {
-	int arg_fds[2], error_fds[2], arg_fd, error_fd;
+	int arg_fds[2], error_fds[2], arg_fd;
 	pid_t pid;
 
 	if ((error_fd_ptr && pipe(error_fds) < 0) || (arg_fd_ptr && pipe(arg_fds) < 0) || (pid = fork()) < 0)
@@ -406,8 +406,7 @@ static pid_t start_pre_exec(const char *cmd, int *arg_fd_ptr, int *error_fd_ptr)
 
 		if (error_fd_ptr) {
 			close(error_fds[0]);
-			error_fd = error_fds[1];
+			set_blocking(error_fds[1]);
-			set_blocking(error_fd);
 		}
 
 		if (arg_fd_ptr) {
@@ -436,8 +435,8 @@ static pid_t start_pre_exec(const char *cmd, int *arg_fd_ptr, int *error_fd_ptr)
 
 		if (error_fd_ptr) {
 			close(STDIN_FILENO);
-			dup2(error_fd, STDOUT_FILENO);
+			dup2(error_fds[1], STDOUT_FILENO);
-			close(error_fd);
+			close(error_fds[1]);
 		}
 
 		status = shell_exec(cmd);
@@ -449,8 +448,8 @@ static pid_t start_pre_exec(const char *cmd, int *arg_fd_ptr, int *error_fd_ptr)
 
 	if (error_fd_ptr) {
 		close(error_fds[1]);
-		error_fd = *error_fd_ptr = error_fds[0];
+		*error_fd_ptr = error_fds[0];
-		set_blocking(error_fd);
+		set_blocking(error_fds[0]);
 	}
 
 	if (arg_fd_ptr) {

compat.c

@@ -394,7 +394,7 @@ static void send_negotiate_str(int f_out, struct name_num_obj *nno, const char *
 
 	if (!do_negotiated_strings) {
 		if (!am_server && fail_if_empty) {
-			rprintf(FERROR, "Remote rsync is too old for %s negotation\n", nno->type);
+			rprintf(FERROR, "Remote rsync is too old for %s negotiation\n", nno->type);
 			exit_cleanup(RERR_UNSUPPORTED);
 		}
 		return;

configure.ac

@@ -1,6 +1,6 @@
 dnl Process this file with autoconf to produce a configure script.
 
-AC_INIT([rsync],[3.2.0pre2],[http://rsync.samba.org/bugzilla.html])
+AC_INIT([rsync],[3.2.0],[http://rsync.samba.org/bugzilla.html])
 
 AC_CONFIG_MACRO_DIR([m4])
 AC_CONFIG_SRCDIR([byteorder.h])
@@ -42,12 +42,14 @@ dnl Checks for programs.
 AC_PROG_CC
 AC_PROG_CPP
 AC_PROG_CXX
+AC_PROG_AWK
 AC_PROG_EGREP
 AC_PROG_INSTALL
 AC_PROG_MKDIR_P
 AC_PROG_CC_STDC
 AC_SUBST(SHELL)
 AC_PATH_PROG([PERL], [perl])
+AC_PATH_PROG([PYTHON3], [python3])
 
 AC_DEFINE([_GNU_SOURCE], 1,
           [Define _GNU_SOURCE so that we get all necessary prototypes])
@@ -62,6 +64,40 @@ if test x"$enable_profile" = x"yes"; then
 	CFLAGS="$CFLAGS -pg"
 fi
 
+AC_MSG_CHECKING([if md2man can create man pages])
+if test x"$ac_cv_path_PYTHON3" = x; then
+    AC_MSG_RESULT(no - python3 not found)
+    md2man_works=no
+else
+    md2man_out=`"$srcdir/md2man" --test "$srcdir/rsync.1.md" 2>&1`
+    if test $? = 0; then
+	AC_MSG_RESULT(yes)
+	md2man_works=yes
+    else
+	AC_MSG_RESULT(no)
+	md2man_works=no
+	echo "$md2man_out"
+    fi
+fi
+
+AC_MSG_CHECKING([if we require man-page building])
+AC_ARG_ENABLE([md2man],
+	AS_HELP_STRING([--disable-md2man],[disable md2man for man page creation]))
+if test x"$enable_md2man" != x"no"; then
+    if test -f "$srcdir/rsync.1"; then
+	AC_MSG_RESULT(optional)
+    else
+	AC_MSG_RESULT(required)
+	if test x"$md2man_works" = x"no"; then
+	    AC_MSG_ERROR(You need python3 and the cmarkgfm OR commonmark python3 lib in order to build man pages.
+You can specify --disable-md2man if you want to skip building them.)
+	fi
+    fi
+    MAKE_MAN=man
+else
+    AC_MSG_RESULT(no)
+    MAKE_MAN=''
+fi
 
 # Specifically, this turns on panic_action handling.
 AC_ARG_ENABLE(maintainer-mode,
@@ -70,7 +106,6 @@ if test x"$enable_maintainer_mode" = x"yes"; then
 	CFLAGS="$CFLAGS -DMAINTAINER_MODE"
 fi
 
-
 # This is needed for our included version of popt.  Kind of silly, but
 # I don't want our version too far out of sync.
 CFLAGS="$CFLAGS -DHAVE_CONFIG_H"
@@ -163,31 +198,64 @@ SIMD=
 
 AC_MSG_CHECKING([whether to enable SIMD optimizations])
 AC_ARG_ENABLE(simd,
-    AS_HELP_STRING([--disable-simd],[disable SIMD optimizations (requires g++)]))
+    AS_HELP_STRING([--disable-simd],[disable SIMD optimizations (requires c++)]))
 
 if test x"$enable_simd" != x"no"; then
-    # For x86-64 SIMD, g++ is also required
+    # For x86-64 SIMD, g++ >=5 or clang++ >=7 is required
-    if test x"$build_cpu" = x"x86_64" && test x"$CXX" = x"g++"; then
+    if test x"$build_cpu" = x"x86_64"; then
-        SIMD="$SIMD x86_64"
+	CXX_OK=
+	if test x"$CXX" != x""; then
+	    CXX_VERSION=`$CXX --version 2>/dev/null | head -n 1`
+	    case "$CXX_VERSION" in
+	    g++*)
+		CXX_VERSION=`$CXX -dumpversion | sed 's/\..*//g'`
+		if test "$CXX_VERSION" -ge "5"; then
+		    CXX_OK=yes
+		fi
+		;;
+	    *clang*)
+		# $CXX -dumpversion would have been ideal, but is broken on older clang
+		CXX_VERSION=`echo "$CXX_VERSION" | sed 's/.*version //g' | sed 's/\..*//g'`
+		if test "$CXX_VERSION" -ge "7"; then
+		    CXX_OK=yes
+		fi
+		;;
+	    *)
+		CXX_VERSION='Unknown'
+		;;
+	    esac
+	else
+	    CXX='No c++'
+	    CXX_VERSION='Unknown'
+	fi
+	if test x"$CXX_OK" = x"yes"; then
+	    # AC_MSG_RESULT() is called below.
+	    SIMD="$SIMD x86_64"
+	else
+	    AC_MSG_RESULT(error)
+	    AC_MSG_ERROR([Failed to find g++ >=5 or clang++ >=7 for SIMD optimizations.
+Specify --disable-simd to continue without it. ($CXX, $CXX_VERSION)])
+	fi
+    elif test x"$enable_simd" = x"yes"; then
+        AC_MSG_RESULT(unavailable)
+        AC_MSG_ERROR(The SIMD optimizations are currently x86_64 only.
+Omit --enable-simd to continue without it.)
     fi
 fi
 
 if test x"$SIMD" != x""; then
-    SIMD=`echo "$SIMD" | sed -e 's/^ *//'`
+    SIMD=`echo "$SIMD" | sed 's/^ *//'`
     AC_MSG_RESULT([yes ($SIMD)])
-	AC_DEFINE(HAVE_SIMD, 1, [Define to 1 to enable SIMD optimizations])
+    AC_DEFINE(HAVE_SIMD, 1, [Define to 1 to enable SIMD optimizations])
-    SIMD=`echo "$SIMD" | sed -e 's/[[^ ]]\+/$(SIMD_&)/g'`
+    SIMD=`echo "\\\$(SIMD_$SIMD)" | sed 's/ /) $(SIMD_/g'`
+    # We only use c++ for its target attribute dispatching, disable unneeded bulky features
+    CXXFLAGS="$CXXFLAGS -fno-exceptions -fno-rtti"
 else
     AC_MSG_RESULT(no)
 fi
 
 AC_SUBST(SIMD)
 
-# We only use g++ for its target attribute dispatching, disable unneeded bulky features
-if test x"$CXXOBJ" != x""; then
-    CXXFLAGS="$CXXFLAGS -fno-exceptions -fno-rtti"
-fi
-
 # arrgh. libc in some old debian version screwed up the largefile
 # stuff, getting byte range locking wrong
 AC_CACHE_CHECK([for broken largefile support],rsync_cv_HAVE_BROKEN_LARGEFILE,[
@@ -382,9 +450,18 @@ AC_ARG_ENABLE([openssl],
 	AS_HELP_STRING([--disable-openssl],[disable openssl crypto library]))
 AH_TEMPLATE([USE_OPENSSL],
 [Undefine if you do not want to use openssl crypto library.  By default this is defined.])
-if test x"$enable_openssl" != x"no" && test x"$ac_cv_header_openssl_md4_h" = x"yes" && test x"$ac_cv_header_openssl_md5_h" = x"yes"; then
+if test x"$enable_openssl" != x"no"; then
-    AC_MSG_RESULT(yes)
+    if test x"$ac_cv_header_openssl_md4_h" = x"yes" && test x"$ac_cv_header_openssl_md5_h" = x"yes"; then
-    AC_SEARCH_LIBS(MD5_Init, crypto, [AC_DEFINE(USE_OPENSSL)])
+      AC_MSG_RESULT(yes)
+      AC_SEARCH_LIBS(MD5_Init, crypto,
+          [AC_DEFINE(USE_OPENSSL)],
+          [AC_MSG_ERROR(Failed to find MD5_Init function in openssl crypto lib.
+Use --disable-openssl to continue without openssl crypto lib support.)])
+    else
+        AC_MSG_RESULT(no)
+        AC_MSG_ERROR(Failed to find openssl/md4.h and openssl/md5.h for openssl crypto lib support.
+Use --disable-openssl to continue without it.)
+    fi
 else
     AC_MSG_RESULT(no)
 fi
@@ -394,9 +471,18 @@ AC_ARG_ENABLE([xxhash],
 	AS_HELP_STRING([--disable-xxhash],[disable xxhash checksums]))
 AH_TEMPLATE([SUPPORT_XXHASH],
 [Undefine if you do not want xxhash checksums.  By default this is defined.])
-if test x"$enable_xxhash" != x"no" && test x"$ac_cv_header_xxhash_h" = x"yes"; then
+if test x"$enable_xxhash" != x"no"; then
-    AC_MSG_RESULT(yes)
+    if test x"$ac_cv_header_xxhash_h" = x"yes"; then
-    AC_SEARCH_LIBS(XXH64_createState, xxhash, [AC_DEFINE(SUPPORT_XXHASH)])
+	AC_MSG_RESULT(yes)
+	AC_SEARCH_LIBS(XXH64_createState, xxhash,
+	    [AC_DEFINE(SUPPORT_XXHASH)],
+	    [AC_MSG_ERROR(Failed to find XXH64_createState function in xxhash lib.
+Use --disable-xxhash to continue without xxhash checksums.)])
+    else
+	AC_MSG_RESULT(no)
+	AC_MSG_ERROR(Failed to find xxhash.h for xxhash checksum support.
+Use --disable-xxhash to continue without it.)
+    fi
 else
     AC_MSG_RESULT(no)
 fi
@@ -406,9 +492,18 @@ AC_ARG_ENABLE([zstd],
         AC_HELP_STRING([--disable-zstd], [disable zstd compression]))
 AH_TEMPLATE([SUPPORT_ZSTD],
 [Undefine if you do not want zstd compression.  By default this is defined.])
-if test x"$enable_zstd" != x"no" && test x"$ac_cv_header_zstd_h" = x"yes"; then
+if test x"$enable_zstd" != x"no"; then
-    AC_MSG_RESULT(yes)
+    if test x"$ac_cv_header_zstd_h" = x"yes"; then
-    AC_SEARCH_LIBS(ZSTD_minCLevel, zstd, [AC_DEFINE(SUPPORT_ZSTD)])
+	AC_MSG_RESULT(yes)
+	AC_SEARCH_LIBS(ZSTD_minCLevel, zstd,
+	    [AC_DEFINE(SUPPORT_ZSTD)],
+	    [AC_MSG_ERROR(Failed to find ZSTD_minCLevel function in zstd lib.
+Use --disable-zstd to continue without zstd compression.)])
+    else
+	AC_MSG_RESULT(no)
+	AC_MSG_ERROR(Failed to find zstd.h for zstd compression support.
+Use --disable-zstd to continue without it.)
+    fi
 else
     AC_MSG_RESULT(no)
 fi
@@ -418,9 +513,18 @@ AC_ARG_ENABLE([lz4],
         AC_HELP_STRING([--disable-lz4], [disable LZ4 compression]))
 AH_TEMPLATE([SUPPORT_LZ4],
 [Undefine if you do not want LZ4 compression.  By default this is defined.])
-if test x"$enable_lz4" != x"no" && test x"$ac_cv_header_lz4_h" = x"yes"; then
+if test x"$enable_lz4" != x"no"; then
-    AC_MSG_RESULT(yes)
+    if test x"$ac_cv_header_lz4_h" = x"yes"; then
-    AC_SEARCH_LIBS(LZ4_compress_default, lz4, [AC_DEFINE(SUPPORT_LZ4)])
+	AC_MSG_RESULT(yes)
+	AC_SEARCH_LIBS(LZ4_compress_default, lz4,
+	    [AC_DEFINE(SUPPORT_LZ4)],
+	    [AC_MSG_ERROR(Failed to find LZ4_compress_default function in lz4 lib.
+Use --disable-lz4 to continue without lz4 compression.)])
+    else
+	AC_MSG_RESULT(no)
+	AC_MSG_ERROR(Failed to find lz4.h for lz4 compression support.
+Use --disable-lz4 to continue without it.)
+    fi
 else
     AC_MSG_RESULT(no)
 fi
@@ -556,7 +660,7 @@ size_t iconv();
 #endif
 ]], [[]])],[am_cv_proto_iconv_arg1=""],[am_cv_proto_iconv_arg1="const"])
       am_cv_proto_iconv="extern size_t iconv (iconv_t cd, $am_cv_proto_iconv_arg1 char * *inbuf, size_t *inbytesleft, char * *outbuf, size_t *outbytesleft);"])
-    am_cv_proto_iconv=`echo "[$]am_cv_proto_iconv" | tr -s ' ' | sed -e 's/( /(/'`
+    am_cv_proto_iconv=`echo "[$]am_cv_proto_iconv" | tr -s ' ' | sed 's/( /(/'`
 AC_MSG_RESULT([$]{ac_t:-
          }[$]am_cv_proto_iconv)
 AC_DEFINE_UNQUOTED(ICONV_CONST, $am_cv_proto_iconv_arg1,
@@ -763,7 +867,7 @@ AC_ARG_ENABLE(iconv,
     AS_HELP_STRING([--disable-iconv],[disable rsync's --iconv option]),
     [], [enable_iconv=$enable_iconv_open])
 AH_TEMPLATE([ICONV_OPTION],
-[Define if you want the --iconv option.  Specifing a value will set the
+[Define if you want the --iconv option.  Specifying a value will set the
 default iconv setting (a NULL means no --iconv processing by default).])
 if test x"$enable_iconv" != x"no"; then
 	if test x"$enable_iconv" = x"yes"; then

define-from-md.awk

@@ -0,0 +1,41 @@
+#!/usr/bin/awk -f
+
+# The caller must pass args: -v hfile=NAME rsync.1.md
+
+BEGIN {
+    heading = "/* DO NOT EDIT THIS FILE!  It is auto-generated from a list of values in " ARGV[1] "! */"
+    if (hfile ~ /compress/) {
+	define = "#define DEFAULT_DONT_COMPRESS"
+	prefix = "*."
+    } else {
+	define = "#define DEFAULT_CVSIGNORE"
+	prefix = ""
+    }
+    value_list = ""
+}
+
+/^    > [^ ]+$/ {
+    gsub(/`/, "")
+    if (value_list != "") value_list = value_list " "
+    value_list = value_list prefix $2
+    next
+}
+
+value_list ~ /\.gz / && hfile ~ /compress/ {
+    exit
+}
+
+value_list ~ /SCCS / && hfile ~ /cvsignore/ {
+    exit
+}
+
+value_list = ""
+
+END {
+    if (value_list != "")
+	print heading "\n\n" define " \"" value_list "\"" > hfile
+    else {
+	print "Failed to find a value list in " ARGV[1] " for " hfile
+	exit 1
+    }
+}

exclude.c

@@ -21,6 +21,7 @@
  */
 
 #include "rsync.h"
+#include "default-cvsignore.h"
 
 extern int am_server;
 extern int am_sender;
@@ -1051,16 +1052,6 @@ static filter_rule *parse_rule_tok(const char **rulestr_ptr,
 	return rule;
 }
 
-static char default_cvsignore[] =
-	/* These default ignored items come from the CVS manual. */
-	"RCS SCCS CVS CVS.adm RCSLOG cvslog.* tags TAGS"
-	" .make.state .nse_depinfo *~ #* .#* ,* _$* *$"
-	" *.old *.bak *.BAK *.orig *.rej .del-*"
-	" *.a *.olb *.o *.obj *.so *.exe"
-	" *.Z *.elc *.ln core"
-	/* The rest we added to suit ourself. */
-	" .svn/ .git/ .hg/ .bzr/";
-
 static void get_cvs_excludes(uint32 rflags)
 {
 	static int initialized = 0;
@@ -1070,7 +1061,7 @@ static void get_cvs_excludes(uint32 rflags)
 		return;
 	initialized = 1;
 
-	parse_filter_str(&cvs_filter_list, default_cvsignore,
+	parse_filter_str(&cvs_filter_list, DEFAULT_CVSIGNORE,
 			 rule_template(rflags | (protocol_version >= 30 ? FILTRULE_PERISHABLE : 0)),
 			 0);
 

help-from-md

@@ -1,33 +0,0 @@
-#!/bin/bash
-
-if [[ "$#" != 2 ]]; then
-    echo "Usage: $0 MD_FILE HELP_FILE.h"
-    exit 1
-fi
-
-mdfile="$1"
-helpfile="$2"
-newfile="$helpfile.new"
-findfile="${helpfile/./\\.}"
-
-sed -e '1,/^\[comment\].*'"$findfile"'/d' \
-    -e '1,/^```/d' \
-    -e '/^```/,$d' \
-    -e 's/"/\\"/g' \
-    -e 's/^/  rprintf(F,"/' \
-    -e 's/$/\\n");/' \
-    <"$mdfile" >"$newfile"
-
-if [[ ! -s "$newfile" ]]; then
-    rm "$newfile"
-    echo "Discarding empty output for $helpfile file from $mdfile"
-    exit 1
-fi
-
-(cat <<EOT
-/* DO NOT EDIT THIS FILE!  It is auto-generated from the option list in $mdfile! */
- 
-EOT
-cat "$newfile"
-) >"$helpfile"
-rm "$newfile"

help-from-md.awk

@@ -0,0 +1,40 @@
+#!/usr/bin/awk -f
+
+# The caller must pass args: -v hfile=help-NAME.h NAME.NUM.md
+
+BEGIN {
+    heading = "/* DO NOT EDIT THIS FILE!  It is auto-generated from the option list in " ARGV[1] "! */"
+    findcomment = hfile
+    sub("\\.", "\\.", findcomment)
+    findcomment = "\\[comment\\].*" findcomment
+    backtick_cnt = 0
+    prints = ""
+}
+
+/^```/ {
+    backtick_cnt++
+    next
+}
+
+foundcomment {
+    if (backtick_cnt > 1) exit
+    if (backtick_cnt == 1) {
+	gsub(/"/, "\\\"")
+	prints = prints "\n  rprintf(F,\"" $0 "\\n\");"
+    }
+    next
+}
+
+$0 ~ findcomment {
+    foundcomment = 1
+    backtick_cnt = 0
+}
+
+END {
+    if (foundcomment && backtick_cnt > 1)
+	print heading "\n" prints > hfile
+    else {
+	print "Failed to find " hfile " section in " ARGV[1]
+	exit 1
+    }
+}

lib/sysxattrs.h

@@ -1,9 +1,9 @@
 #ifdef SUPPORT_XATTRS
 
-#if defined HAVE_ATTR_XATTR_H
+#if defined HAVE_SYS_XATTR_H
-#include <attr/xattr.h>
-#elif defined HAVE_SYS_XATTR_H
 #include <sys/xattr.h>
+#elif defined HAVE_ATTR_XATTR_H
+#include <attr/xattr.h>
 #elif defined HAVE_SYS_EXTATTR_H
 #include <sys/extattr.h>
 #endif

loadparm.c

@@ -42,6 +42,7 @@
 
 #include "rsync.h"
 #include "itypes.h"
+#include "default-dont-compress.h"
 
 extern item_list dparam_list;
 
@@ -52,11 +53,6 @@ extern item_list dparam_list;
 #define LOG_DAEMON 0
 #endif
 
-#define DEFAULT_DONT_COMPRESS "*.gz *.zip *.z *.rpm *.deb *.iso *.bz2" \
-	" *.t[gb]z *.7z *.mp[34] *.mov *.avi *.ogg *.jpg *.jpeg *.png" \
-	" *.lzo *.rzip *.lzma *.rar *.ace *.gpg *.xz *.txz *.lz *.tlz" \
-	" *.ogv *.web[mp] *.squashfs"
-
 /* the following are used by loadparm for option lists */
 typedef enum {
 	P_BOOL, P_BOOLREV, P_CHAR, P_INTEGER,

md2man

@@ -1,4 +1,4 @@
-#!/usr/bin/python3
+#!/usr/bin/env python3
 
 # This script takes a manpage written in markdown and turns it into an html web
 # page and a nroff man page.  The input file must have the name of the program
@@ -35,6 +35,7 @@ body, b, strong, u {
 code {
   font-family: 'Roboto Mono', monospace;
   font-weight: bold;
+  white-space: pre;
 }
 pre code {
   display: block;
@@ -64,7 +65,9 @@ MAN_END = """\
 
 NORM_FONT = ('\1', r"\fP")
 BOLD_FONT = ('\2', r"\fB")
-ULIN_FONT = ('\3', r"\fI")
+UNDR_FONT = ('\3', r"\fI")
+NBR_DASH = ('\4', r"\-")
+NBR_SPACE = ('\xa0', r"\ ")
 
 md_parser = None
 
@@ -78,46 +81,49 @@ def main():
         fi.srcdir = './'
 
     fi.title = fi.prog + '(' + fi.sect + ') man page'
-    fi.mtime = None
+    fi.mtime = 0
 
     if os.path.lexists(fi.srcdir + '.git'):
         fi.mtime = int(subprocess.check_output('git log -1 --format=%at'.split()))
 
-    chk_files = 'NEWS.md Makefile'.split()
-    for fn in chk_files:
-        try:
-            st = os.lstat(fi.srcdir + fn)
-        except:
-            die('Failed to find', fi.srcdir + fn)
-        if not fi.mtime:
-            fi.mtime = st.st_mtime
-
-    fi.date = time.strftime('%d %b %Y', time.localtime(fi.mtime))
-
     env_subs = { 'prefix': os.environ.get('RSYNC_OVERRIDE_PREFIX', None) }
 
-    with open(fi.srcdir + 'Makefile', 'r', encoding='utf-8') as fh:
+    if args.test:
-        for line in fh:
+        env_subs['VERSION'] = '1.0.0'
-            m = re.match(r'^(\w+)=(.+)', line)
+        env_subs['libdir'] = '/usr'
-            if not m:
+    else:
-                continue
+        for fn in 'NEWS.md Makefile'.split():
-            var, val = (m[1], m[2])
+            try:
-            if var == 'prefix' and env_subs[var] is not None:
+                st = os.lstat(fi.srcdir + fn)
-                continue
+            except:
-            while re.search(r'\$\{', val):
+                die('Failed to find', fi.srcdir + fn)
-                val = re.sub(r'\$\{(\w+)\}', lambda m: env_subs[m[1]], val)
+            if not fi.mtime:
-            env_subs[var] = val
+                fi.mtime = st.st_mtime
-            if var == 'VERSION':
+
-                break
+        with open(fi.srcdir + 'Makefile', 'r', encoding='utf-8') as fh:
+            for line in fh:
+                m = re.match(r'^(\w+)=(.+)', line)
+                if not m:
+                    continue
+                var, val = (m.group(1), m.group(2))
+                if var == 'prefix' and env_subs[var] is not None:
+                    continue
+                while re.search(r'\$\{', val):
+                    val = re.sub(r'\$\{(\w+)\}', lambda m: env_subs[m.group(1)], val)
+                env_subs[var] = val
+                if var == 'VERSION':
+                    break
 
     with open(fi.fn, 'r', encoding='utf-8') as fh:
         txt = fh.read()
 
     txt = re.sub(r'@VERSION@', env_subs['VERSION'], txt)
     txt = re.sub(r'@LIBDIR@', env_subs['libdir'], txt)
+
     fi.html_in = md_parser(txt)
     txt = None
 
+    fi.date = time.strftime('%d %b %Y', time.localtime(fi.mtime))
     fi.man_headings = (fi.prog, fi.sect, fi.date, fi.prog + ' ' + env_subs['VERSION'])
 
     HtmlToManPage(fi)
@@ -181,6 +187,8 @@ class HtmlToManPage(HTMLParser):
                     tag = 'dt'
                 else:
                     st.html_out.append('<dt>')
+            elif tag == 'p':
+                st.at_first_tag_in_dd = True # Kluge to suppress a .P at the start of an li.
             st.at_first_tag_in_li = False
         if tag == 'p':
             if not st.at_first_tag_in_dd:
@@ -207,7 +215,7 @@ class HtmlToManPage(HTMLParser):
             st.txt += BOLD_FONT[0]
         elif tag == 'em' or  tag == 'i':
             tag = 'u' # Change it into underline to be more like the man page
-            st.txt += ULIN_FONT[0]
+            st.txt += UNDR_FONT[0]
         elif tag == 'ol':
             start = 1
             for var, val in attrs_list:
@@ -300,15 +308,21 @@ class HtmlToManPage(HTMLParser):
             st.at_first_tag_in_dd = True
 
 
-    def handle_data(self, data):
+    def handle_data(self, txt):
         st = self.state
         if args.debug:
-            self.output_debug('DATA', (data,))
+            self.output_debug('DATA', (txt,))
-        if st.in_code:
+        if st.in_pre:
-            data = re.sub(r'\s', '\xa0', data) # nbsp in non-pre code
+            html = htmlify(txt)
-        data = re.sub(r'\s--\s', '\xa0-- ', data)
+        else:
-        st.html_out.append(htmlify(data))
+            txt = re.sub(r'\s--(\s)', NBR_SPACE[0] + r'--\1', txt).replace('--', NBR_DASH[0]*2)
-        st.txt += data
+            txt = re.sub(r'(^|\W)-', r'\1' + NBR_DASH[0], txt)
+            html = htmlify(txt)
+            if st.in_code:
+                txt = re.sub(r'\s', NBR_SPACE[0], txt)
+                html = html.replace(NBR_DASH[0], '-').replace(NBR_SPACE[0], ' ') # <code> is non-breaking in CSS
+        st.html_out.append(html.replace(NBR_SPACE[0], '&nbsp;').replace(NBR_DASH[0], '-&#8288;'))
+        st.txt += txt
 
 
     def output_debug(self, event, extra):
@@ -326,17 +340,15 @@ class HtmlToManPage(HTMLParser):
 
 def manify(txt):
     return re.sub(r"^(['.])", r'\&\1', txt.replace('\\', '\\\\')
-            .replace("\xa0", r'\ ') # non-breaking space
+            .replace(NBR_SPACE[0], NBR_SPACE[1])
-            .replace('--', r'\-\-') # non-breaking double dash
+            .replace(NBR_DASH[0], NBR_DASH[1])
             .replace(NORM_FONT[0], NORM_FONT[1])
             .replace(BOLD_FONT[0], BOLD_FONT[1])
-            .replace(ULIN_FONT[0], ULIN_FONT[1]), flags=re.M)
+            .replace(UNDR_FONT[0], UNDR_FONT[1]), flags=re.M)
 
 
 def htmlify(txt):
-    return re.sub(r'(\W)-', r'\1&#8209;',
+    return txt.replace('&', '&amp;').replace('<', '&lt;').replace('>', '&gt;').replace('"', '&quot;')
-            txt.replace('&', '&amp;').replace('<', '&lt;').replace('>', '&gt;').replace('"', '&quot;')
-            .replace('--', '&#8209;&#8209;').replace("\xa0-", '&nbsp;&#8209;').replace("\xa0", '&nbsp;'))
 
 
 def warn(*msg):

options.c

@@ -200,8 +200,11 @@ int list_only = 0;
 char *batch_name = NULL;
 
 int need_unsorted_flist = 0;
+char *iconv_opt =
 #ifdef ICONV_OPTION
-char *iconv_opt = ICONV_OPTION;
+		ICONV_OPTION;
+#else
+		NULL;
 #endif
 
 struct chmod_mode_struct *chmod_modes = NULL;
@@ -308,9 +311,7 @@ static int refused_partial, refused_progress, refused_delete_before;
 static int refused_delete_during;
 static int refused_inplace, refused_no_iconv;
 static BOOL usermap_via_chown, groupmap_via_chown;
-#ifdef HAVE_SETVBUF
 static char *outbuf_mode;
-#endif
 static char *bwlimit_arg, *max_size_arg, *min_size_arg;
 static char tmp_partialdir[] = ".~tmp~";
 

packaging/branch-from-patch

@@ -1,4 +1,4 @@
-#!/usr/bin/python3 -B
+#!/usr/bin/env -S python3 -B
 
 # This script turns one or more diff files in the patches dir (which is
 # expected to be a checkout of the rsync-patches git repo) into a branch

packaging/cull_options

@@ -1,4 +1,4 @@
-#!/usr/bin/perl
+#!/usr/bin/env perl
 # This script outputs some perl code that parses all possible options
 # that the code in options.c might send to the server.  This perl code
 # is included in the rrsync script.

packaging/lsb/rsync.spec

@@ -1,9 +1,9 @@
 Summary: A fast, versatile, remote (and local) file-copying tool
 Name: rsync
 Version: 3.2.0
-%define fullversion %{version}pre2
+%define fullversion %{version}
-Release: 0.1.pre2
+Release: 1
-%define srcdir src-previews
+%define srcdir src
 Group: Applications/Internet
 License: GPL
 Source0: http://rsync.samba.org/ftp/rsync/%{srcdir}/rsync-%{fullversion}.tar.gz
@@ -79,9 +79,9 @@ rm -rf $RPM_BUILD_ROOT
 %dir /etc/rsync-ssl/certs
 
 %changelog
-* Mon Jun 15 2020 Wayne Davison <wayned@samba.org>
+* Fri Jun 19 2020 Wayne Davison <wayne@opencoder.net>
-Released 3.2.0pre2.
+Released 3.2.0.
 
-* Fri Mar 21 2008 Wayne Davison <wayned@samba.org>
+* Fri Mar 21 2008 Wayne Davison <wayne@opencoder.net>
 Added installation of /etc/xinetd.d/rsync file and some commented-out
 lines that demonstrate how to use the rsync-patches tar file.

packaging/md2html

@@ -1,10 +1,10 @@
-#!/usr/bin/python3
+#!/usr/bin/env python3
 
 # Copyright (C) 2020 Wayne Davison
 #
 # This program is freely redistributable.
 
-import re, argparse
+import os, re, argparse
 
 HTML_START = """\
 <html><head>
@@ -63,6 +63,9 @@ def main():
         html = html.replace('--', '&#8209;&#8209;').replace("\xa0-", '&nbsp;&#8209;').replace("\xa0", '&nbsp;')
         html = re.sub(r'(\W)-', r'\1&#8209;', html)
 
+        if os.path.lexists(htfn):
+            os.unlink(htfn)
+
         with open(htfn, 'w', encoding='utf-8') as fh:
             fh.write(HTML_START % title)
             fh.write(html)

packaging/nightly-rsync

@@ -1,4 +1,4 @@
-#!/usr/bin/python3 -B
+#!/usr/bin/env -S python3 -B
 
 # This script expects the directory ~/samba-rsync-ftp to exist and to be a
 # copy of the /home/ftp/pub/rsync dir on samba.org.  It also requires a

packaging/patch-update

@@ -1,4 +1,4 @@
-#!/usr/bin/python3 -B
+#!/usr/bin/env -S python3 -B
 
 # This script is used to turn one or more of the "patch/BASE/*" branches
 # into one or more diffs in the "patches" directory.  Pass the option

packaging/release-rsync

@@ -1,4 +1,4 @@
-#!/usr/bin/python3 -B
+#!/usr/bin/env -S python3 -B
 
 # This script expects the directory ~/samba-rsync-ftp to exist and to be a
 # copy of the /home/ftp/pub/rsync dir on samba.org.  When the script is done,

packaging/systemd/rsync.service

@@ -1,9 +1,30 @@
 [Unit]
 Description=fast remote file copy program daemon
 ConditionPathExists=/etc/rsyncd.conf
+After=network.target
 
 [Service]
 ExecStart=/usr/bin/rsync --daemon --no-detach
+RestartSec=1
+
+# Citing README.md:
+#
+#   [...] Using ssh is recommended for its security features.
+#
+#   Alternatively, rsync can run in `daemon' mode, listening on a socket.
+#   This is generally used for public file distribution, [...]
+#
+# So let's assume some extra security is more than welcome here. We do full
+# system protection (which makes it read-only) and hide users' homes and
+# devices. To override these defaults, it's best to do so in the drop-in
+# directory, often done via `systemctl edit rsync.service`. The file needs
+# just the bare minimum of the right [heading] and override values.
+# See systemd.unit(5) and search for "drop-in" for full details.
+
+ProtectSystem=full
+ProtectHome=on
+PrivateDevices=on
+NoNewPrivileges=on
 
 [Install]
 WantedBy=multi-user.target

packaging/systemd/rsync.socket

@@ -0,0 +1,10 @@
+[Unit]
+Description=socket for fast remote file copy program daemon
+Conflicts=rsync.service
+
+[Socket]
+ListenStream=873
+Accept=true
+
+[Install]
+WantedBy=sockets.target

packaging/systemd/rsync@.service

@@ -0,0 +1,28 @@
+[Unit]
+Description=fast remote file copy program daemon
+After=network.target
+
+[Service]
+ExecStart=-/usr/bin/rsync --daemon
+StandardInput=socket
+StandardOutput=inherit
+StandardError=journal
+
+# Citing README.md:
+#
+#   [...] Using ssh is recommended for its security features.
+#
+#   Alternatively, rsync can run in `daemon' mode, listening on a socket.
+#   This is generally used for public file distribution, [...]
+#
+# So let's assume some extra security is more than welcome here. We do full
+# system protection (which makes it read-only) and hide users' homes and
+# devices. To override these defaults, it's best to do so in the drop-in
+# directory, often done via `systemctl edit rsync@.service`. The file needs
+# just the bare minimum of the right [heading] and override values.
+# See systemd.unit(5) and search for "drop-in" for full details.
+
+ProtectSystem=full
+ProtectHome=on
+PrivateDevices=on
+NoNewPrivileges=on

packaging/var-checker

@@ -1,4 +1,4 @@
-#!/usr/bin/python3 -B
+#!/usr/bin/env -S python3 -B
 
 # This script checks the *.c files for extraneous "extern" variables,
 # for vars that are defined but not used, and for inconsistent array

packaging/year-tweak

@@ -1,4 +1,4 @@
-#!/usr/bin/python3
+#!/usr/bin/env python3
 
 # This uses the output from "support/git-set-file-times --list" to discern
 # the last-modified year of each *.c & *.h file and updates the copyright

rounding.c

@@ -24,11 +24,11 @@
 
 struct test {
 	union file_extras extras[ARRAY_LEN];
-	struct file_struct file;
+	int64 test;
 };
 
 #define ACTUAL_SIZE	SIZEOF(struct test)
-#define EXPECTED_SIZE	(SIZEOF(union file_extras) * ARRAY_LEN + SIZEOF(struct file_struct))
+#define EXPECTED_SIZE	(SIZEOF(union file_extras) * ARRAY_LEN + SIZEOF(int64))
 
  int main(UNUSED(int argc), UNUSED(char *argv[]))
 {

rsync-ssl

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 
 # This script uses openssl, gnutls, or stunnel to secure an rsync daemon connection.
 

rsync-ssl.1.md

@@ -13,6 +13,13 @@ rsync-ssl [--type=SSL_TYPE] RSYNC_ARGS
 The rsync-ssl script helps you to run an rsync copy to/from an rsync daemon
 that requires ssl connections.
 
+The script requires that you specify an rsync-daemon arg in the style of either
+`hostname::` (with 2 colons) or `rsync://hostname/`.  The default port used for
+connecting is 874 (one higher than the normal 873) unless overridden in the
+environment.  You can specify an overriding port via `--port` or by including
+it in the normal spot in the URL format, though both of those require your
+rsync version to be at least 3.2.0.
+
 # OPTIONS
 
 If the **first** arg is a `--type=SSL_TYPE` option, the script will only use
@@ -23,7 +30,7 @@ option must specify one of `openssl` or `stunnel`.  The equal sign is
 required for this particular option.
 
 All the other options are passed through to the rsync command, so consult the
-**rsync** manpage for more information on how it works.
+**rsync**(1) manpage for more information on how it works.
 
 # ENVIRONMENT VARIABLES
 
@@ -53,9 +60,13 @@ The ssl helper scripts are affected by the following environment variables:
 
 # EXAMPLES
 
->     rsync-ssl -aiv example.com::src/ dest
+>     rsync-ssl -aiv example.com::mod/ dest
 
->     rsync-ssl --type=openssl -aiv example.com::src/ dest
+>     rsync-ssl --type=openssl -aiv example.com::mod/ dest
+
+>     rsync-ssl -aiv --port 9874 example.com::mod/ dest
+
+>     rsync-ssl -aiv rsync://example.com:9874/mod/ dest
 
 # SEE ALSO
 

rsync.1.md

@@ -498,7 +498,7 @@ parameter, the parameter is only listed after the long variant, even though it
 must also be specified for the short.  When specifying a parameter, you can
 either use the form `--option=param` or replace the '=' with whitespace.  The
 parameter may need to be quoted in some manner for it to survive the shell's
-command-line parsing.  Keep in mind that a leading tilde (\~) in a filename is
+command-line parsing.  Keep in mind that a leading tilde (`~`) in a filename is
 substituted by your shell, so `--option=~/foo` will not change the tilde into
 your home directory (remove the '=' for that).
 
@@ -595,9 +595,9 @@ your home directory (remove the '=' for that).
     but it can delay messages behind a slew of data.
 
     One case where this is helpful is when sending really large files, since
-    errors that happen on a remote receiver tend to get delayed until afer the
+    errors that happen on a remote receiver tend to get delayed until after the
-    file's data is fully sent.  It is also helpful for debugging, since it helps
+    file's data is fully sent.  It is also helpful for debugging, since it
-    to avoid overpopulating the protocol data with extra message data.
+    helps to avoid overpopulating the protocol data with extra message data.
 
     The option does not affect the remote side of a transfer without using
     `--remote-option` -- e.g. `-M--msgs2stderr` or `{-M,}--msgs2stderr`.
@@ -1482,13 +1482,13 @@ your home directory (remove the '=' for that).
     the `--checksum` option cannot be used.
 
     The "auto" option is the default, where rsync bases its algorithm choice on
-    a negotation between the client and the server as follows:
+    a negotiation between the client and the server as follows:
 
     If both the client and the server are at least version 3.2.0, they will
     exchange a list of checksum names and choose the first one in the list that
     they have in common.  This typically means that they will choose xxh64 if
     they both support it and fall back to MD5.  If one side of the transfer is
-    not new enough to support this checksum negotation, then a value is chosen
+    not new enough to support this checksum negotiation, then a value is chosen
     based on the protocol version (which chooses between MD5 and various
     flavors of MD4 based on protocol age).
 
@@ -1852,6 +1852,8 @@ your home directory (remove the '=' for that).
     The exclude list is initialized to exclude the following items (these
     initial items are marked as perishable -- see the FILTER RULES section):
 
+    [comment]: # (This list gets used for the default-cvsignore.h file.)
+
     > `RCS`
     > `SCCS`
     > `CVS`
@@ -2294,7 +2296,7 @@ your home directory (remove the '=' for that).
 
     This option implies `--compress` unless "none" was specified.
 
-    You can also override the compression negotation using the
+    You can also override the compression negotiation using the
     RSYNC_COMPRESS_LIST environment variable by setting it to a space-separated
     list of compression names that you consider acceptable.  If no common
     compress choice is found, the client exits with an error.  It ignores
@@ -2318,14 +2320,14 @@ your home directory (remove the '=' for that).
     possible.
 
     The **LIST** should be one or more file suffixes (without the dot) separated
-    by slashes (/).  You may specify an empty string to indicate that no files
+    by slashes (`/`).  You may specify an empty string to indicate that no files
     should be skipped.
 
     Simple character-class matching is supported: each must consist of a list
     of letters inside the square brackets (e.g. no special classes, such as
     "[:alpha:]", are supported, and '-' has no special meaning).
 
-    The characters asterisk (\*) and question-mark (?) have no special meaning.
+    The characters asterisk (`*`) and question-mark (`?`) have no special meaning.
 
     Here's an example that specifies 6 suffixes to skip (since 1 of the 5 rules
     matches 2 suffixes):
@@ -2335,38 +2337,68 @@ your home directory (remove the '=' for that).
     The default file suffixes in the skip-compress list in this version of
     rsync are:
 
+    [comment]: # (This list gets used for the default-dont-compress.h file.)
+
     > 7z
     > ace
+    > apk
     > avi
     > bz2
     > deb
+    > flac
     > gpg
     > gz
     > iso
+    > jar
     > jpeg
     > jpg
     > lz
+    > lz4
     > lzma
     > lzo
+    > mkv
     > mov
     > mp3
     > mp4
+    > odb
+    > odf
+    > odg
+    > odi
+    > odm
+    > odp
+    > ods
+    > odt
     > ogg
     > ogv
+    > opus
+    > otg
+    > oth
+    > otp
+    > ots
+    > ott
+    > oxt
     > png
     > rar
     > rpm
+    > rz
     > rzip
     > squashfs
+    > sxc
+    > sxd
+    > sxg
+    > sxm
+    > sxw
     > tbz
     > tgz
     > tlz
     > txz
+    > tzo
     > webm
     > webp
     > xz
     > z
     > zip
+    > zst
 
     This list will be replaced by your `--skip-compress` list in all but one
     situation: a copy from a daemon rsync will add your skipped suffixes to its
@@ -2399,7 +2431,7 @@ your home directory (remove the '=' for that).
     You may specify usernames or user IDs for the **FROM** and **TO** values,
     and the **FROM** value may also be a wild-card string, which will be
     matched against the sender's names (wild-cards do NOT match against ID
-    numbers, though see below for why a '\*' matches everything).  You may
+    numbers, though see below for why a '`*`' matches everything).  You may
     instead specify a range of ID numbers via an inclusive range: LOW-HIGH.
     For example:
 
@@ -2417,7 +2449,7 @@ your home directory (remove the '=' for that).
 
     Any IDs that do not have a name on the sending side are treated as having
     an empty name for the purpose of matching.  This allows them to be matched
-    via a "\*" or using an empty name.  For instance:
+    via a "`*`" or using an empty name.  For instance:
 
     >     --usermap=:nobody --groupmap=*:nobody
 
@@ -2478,8 +2510,9 @@ your home directory (remove the '=' for that).
     which may make transfers faster (or slower!).  Read the man page for the
     `setsockopt()` system call for details on some of the options you may be
     able to set.  By default no special socket options are set.  This only
-    affects direct socket connections to a remote rsync daemon.  This option
+    affects direct socket connections to a remote rsync daemon.
-    also exists in the `--daemon` mode section.
+
+    This option also exists in the `--daemon` mode section.
 
 0.  `--blocking-io`
 
@@ -2686,7 +2719,7 @@ your home directory (remove the '=' for that).
 
     The escape idiom that started in 2.6.7 is to output a literal backslash
     (`\`) and a hash (`#`), followed by exactly 3 octal digits.  For example, a
-    newline would output as "`\\#012`".  A literal backslash that is in a
+    newline would output as "`\#012`".  A literal backslash that is in a
     filename is not escaped unless it is followed by a hash and 3 digits (0-9).
 
 0.  `--human-readable`, `-h`
@@ -3070,7 +3103,7 @@ your home directory (remove the '=' for that).
 
     Tells rsync to prefer IPv4/IPv6 when creating sockets or running ssh.  This
     affects sockets that rsync has direct control over, such as the outgoing
-    socket when directly contacting an rsync daemon, as well as the forwaring
+    socket when directly contacting an rsync daemon, as well as the forwarding
     of the `-4` or `-6` option to ssh when rsync can deduce that ssh is being
     used as the remote shell.  For other remote shells you'll need to specify
     the "`--rsh SHELL -4`" option directly (or whatever ipv4/ipv6 hint options
@@ -3082,8 +3115,6 @@ your home directory (remove the '=' for that).
     have no effect.  The `rsync -V` output will contain "`no IPv6`" if is the
     case.
 
-    See also these options in the `--daemon` mode section.
-
 0. `--checksum-seed=NUM`
 
     Set the checksum seed to the integer NUM.  This 4 byte checksum seed is
@@ -3267,9 +3298,9 @@ include/exclude rules each specify a pattern that is matched against the names
 of the files that are going to be transferred.  These patterns can take several
 forms:
 
-- if the pattern starts with a / then it is anchored to a particular spot in
+- if the pattern starts with a `/` then it is anchored to a particular spot in
   the hierarchy of files, otherwise it is matched against the end of the
-  pathname.  This is similar to a leading ^ in regular expressions.  Thus
+  pathname.  This is similar to a leading `^` in regular expressions.  Thus
   `/foo` would match a name of "foo" at either the "root of the transfer" (for
   a global rule) or in the merge-file's directory (for a per-directory rule).
   An unqualified `foo` would match a name of "foo" anywhere in the tree because
@@ -3279,24 +3310,24 @@ forms:
   was found within a directory named "sub".  See the section on ANCHORING
   INCLUDE/EXCLUDE PATTERNS for a full discussion of how to specify a pattern
   that matches at the root of the transfer.
-- if the pattern ends with a / then it will only match a directory, not a
+- if the pattern ends with a `/` then it will only match a directory, not a
   regular file, symlink, or device.
 - rsync chooses between doing a simple string match and wildcard matching by
   checking if the pattern contains one of these three wildcard characters:
   '`*`', '`?`', and '`[`' .
 - a '`*`' matches any path component, but it stops at slashes.
 - use '`**`' to match anything, including slashes.
-- a '?' matches any character except a slash (/).
+- a '`?`' matches any character except a slash (`/`).
-- a '[' introduces a character class, such as [a-z] or [[:alpha:]].
+- a '`[`' introduces a character class, such as `[a-z]` or `[[:alpha:]]`.
 - in a wildcard pattern, a backslash can be used to escape a wildcard
   character, but it is matched literally when no wildcards are present.  This
   means that there is an extra level of backslash removal when a pattern
   contains wildcard characters compared to a pattern that has none.  e.g. if
   you add a wildcard to "`foo\bar`" (which matches the backslash) you would
   need to use "`foo\\bar*`" to avoid the "`\b`" becoming just "b".
-- if the pattern contains a / (not counting a trailing /) or a "`**`", then it
+- if the pattern contains a `/` (not counting a trailing /) or a "`**`", then it
   is matched against the full pathname, including any leading directories.  If
-  the pattern doesn't contain a / or a "`**`", then it is matched only against
+  the pattern doesn't contain a `/` or a "`**`", then it is matched only against
   the final component of the filename. (Remember that the algorithm is applied
   recursively so "full filename" can actually be any portion of a path from the
   starting directory on down.)
@@ -3311,20 +3342,20 @@ include/exclude patterns are applied recursively to the pathname of each node
 in the filesystem's tree (those inside the transfer).  The exclude patterns
 short-circuit the directory traversal stage as rsync finds the files to send.
 
-For instance, to include "/foo/bar/baz", the directories "/foo" and "/foo/bar"
+For instance, to include "`/foo/bar/baz`", the directories "`/foo`" and "`/foo/bar`"
 must not be excluded.  Excluding one of those parent directories prevents the
 examination of its content, cutting off rsync's recursion into those paths and
-rendering the include for "/foo/bar/baz" ineffectual (since rsync can't match
+rendering the include for "`/foo/bar/baz`" ineffectual (since rsync can't match
 something it never sees in the cut-off section of the directory hierarchy).
 
-The concept path exclusion is particularly important when using a trailing '\*'
+The concept path exclusion is particularly important when using a trailing '`*`'
 rule.  For instance, this won't work:
 
 >     + /some/path/this-file-will-not-be-found
 >     + /file-is-included
 >     - *
 
-This fails because the parent directory "some" is excluded by the '\*' rule, so
+This fails because the parent directory "some" is excluded by the '`*`' rule, so
 rsync never visits any of the files in the "some" or "some/path" directories.
 One solution is to ask for all directories in the hierarchy to be included by
 using a single rule: "`+ */`" (put it somewhere before the "`- *`" rule), and

rsync.c

@@ -57,12 +57,6 @@ extern struct chmod_mode_struct *daemon_chmod_modes;
 extern char *iconv_opt;
 #endif
 
-#ifdef ICONV_CONST
-iconv_t ic_chck = (iconv_t)-1;
-# ifdef ICONV_OPTION
-iconv_t ic_send = (iconv_t)-1, ic_recv = (iconv_t)-1;
-# endif
-
 #define UPDATED_OWNER (1<<0)
 #define UPDATED_GROUP (1<<1)
 #define UPDATED_MTIME (1<<2)
@@ -72,6 +66,12 @@ iconv_t ic_send = (iconv_t)-1, ic_recv = (iconv_t)-1;
 
 #define UPDATED_TIMES (UPDATED_MTIME|UPDATED_ATIME)
 
+#ifdef ICONV_CONST
+iconv_t ic_chck = (iconv_t)-1;
+# ifdef ICONV_OPTION
+iconv_t ic_send = (iconv_t)-1, ic_recv = (iconv_t)-1;
+# endif
+
 static const char *default_charset(void)
 {
 # if defined HAVE_LIBCHARSET_H && defined HAVE_LOCALE_CHARSET

rsyncd.conf.5.md

@@ -1077,7 +1077,6 @@ encryption.
 - You should limit the access to the backend-rsyncd port to only allow the
   proxy to connect.  If it is on the same host as the proxy, then configuring
   it to only listen on localhost is a good idea.
-
 - You should consider turning on the `proxy protocol` parameter if your proxy
   supports sending that information.  The examples below assume that this is
   enabled.

simd-checksum-x86_64.cpp

@@ -45,9 +45,10 @@
  * the available xmm registers, this optimized version may not be faster than
  * the pure C version anyway. Note that all x86-64 CPUs support at least SSE2.
  *
- * This file is compiled using GCC 4.8+'s C++ front end to allow the use of
+ * This file is compiled using GCC 4.8+/clang 6+'s C++ front end to allow the
- * the target attribute, selecting the fastest code path based on runtime
+ * use of the target attribute, selecting the fastest code path based on
- * detection of CPU capabilities.
+ * dispatch priority (GCC 5) or runtime detection of CPU capabilities (GCC 6+).
+ * GCC 4.x are not supported to ease configure.ac logic.
  */
 
 #ifdef __x86_64__
@@ -59,73 +60,34 @@
 
 #include <immintrin.h>
 
-/* Compatibility functions to let our SSSE3 algorithm run on SSE2 */
+/* Some clang versions don't like it when you use static with multi-versioned functions: linker errors */
+#ifdef __clang__
+#define MVSTATIC
+#else
+#define MVSTATIC static
+#endif
 
-__attribute__ ((target("sse2"))) static inline __m128i sse_interleave_odd_epi16(__m128i a, __m128i b)
+// Missing from the headers on gcc 6 and older, clang 8 and older
-{
+typedef long long __m128i_u __attribute__((__vector_size__(16), __may_alias__, __aligned__(1)));
-    return _mm_packs_epi32(
+typedef long long __m256i_u __attribute__((__vector_size__(32), __may_alias__, __aligned__(1)));
-        _mm_srai_epi32(a, 16),
-        _mm_srai_epi32(b, 16)
-    );
-}
 
-__attribute__ ((target("sse2"))) static inline __m128i sse_interleave_even_epi16(__m128i a, __m128i b)
+/* Compatibility macros to let our SSSE3 algorithm run with only SSE2.
-{
+   These used to be neat individual functions with target attributes switching between SSE2 and SSSE3 implementations
-    return sse_interleave_odd_epi16(
+   as needed, but though this works perfectly with GCC, clang fails to inline those properly leading to a near 50%
-        _mm_slli_si128(a, 2),
+   performance drop - combined with static and inline modifiers gets you linker errors and even compiler crashes...
-        _mm_slli_si128(b, 2)
+*/
-    );
-}
 
-__attribute__ ((target("sse2"))) static inline __m128i sse_mulu_odd_epi8(__m128i a, __m128i b)
+#define SSE2_INTERLEAVE_ODD_EPI16(a, b) _mm_packs_epi32(_mm_srai_epi32(a, 16), _mm_srai_epi32(b, 16))
-{
+#define SSE2_INTERLEAVE_EVEN_EPI16(a, b) SSE2_INTERLEAVE_ODD_EPI16(_mm_slli_si128(a, 2), _mm_slli_si128(b, 2))
-    return _mm_mullo_epi16(
+#define SSE2_MULU_ODD_EPI8(a, b) _mm_mullo_epi16(_mm_srli_epi16(a, 8), _mm_srai_epi16(b, 8))
-        _mm_srli_epi16(a, 8),
+#define SSE2_MULU_EVEN_EPI8(a, b) _mm_mullo_epi16(_mm_and_si128(a, _mm_set1_epi16(0xFF)), _mm_srai_epi16(_mm_slli_si128(b, 1), 8))
-        _mm_srai_epi16(b, 8)
-    );
-}
 
-__attribute__ ((target("sse2"))) static inline __m128i sse_mulu_even_epi8(__m128i a, __m128i b)
+#define SSE2_HADDS_EPI16(a, b) _mm_adds_epi16(SSE2_INTERLEAVE_EVEN_EPI16(a, b), SSE2_INTERLEAVE_ODD_EPI16(a, b))
-{
+#define SSE2_MADDUBS_EPI16(a, b) _mm_adds_epi16(SSE2_MULU_EVEN_EPI8(a, b), SSE2_MULU_ODD_EPI8(a, b))
-    return _mm_mullo_epi16(
-        _mm_and_si128(a, _mm_set1_epi16(0xFF)),
-        _mm_srai_epi16(_mm_slli_si128(b, 1), 8)
-    );
-}
 
-__attribute__ ((target("sse2"))) static inline __m128i sse_hadds_epi16(__m128i a, __m128i b)
+__attribute__ ((target("default"))) MVSTATIC int32 get_checksum1_avx2_64(schar* buf, int32 len, int32 i, uint32* ps1, uint32* ps2) { return i; }
-{
+__attribute__ ((target("default"))) MVSTATIC int32 get_checksum1_ssse3_32(schar* buf, int32 len, int32 i, uint32* ps1, uint32* ps2) { return i; }
-    return _mm_adds_epi16(
+__attribute__ ((target("default"))) MVSTATIC int32 get_checksum1_sse2_32(schar* buf, int32 len, int32 i, uint32* ps1, uint32* ps2) { return i; }
-        sse_interleave_even_epi16(a, b),
-        sse_interleave_odd_epi16(a, b)
-    );
-}
-
-__attribute__ ((target("ssse3"))) static inline __m128i sse_hadds_epi16(__m128i a, __m128i b)
-{
-    return _mm_hadds_epi16(a, b);
-}
-
-__attribute__ ((target("sse2"))) static inline __m128i sse_maddubs_epi16(__m128i a, __m128i b)
-{
-    return _mm_adds_epi16(
-        sse_mulu_even_epi8(a, b),
-        sse_mulu_odd_epi8(a, b)
-    );
-}
-
-__attribute__ ((target("ssse3"))) static inline __m128i sse_maddubs_epi16(__m128i a, __m128i b)
-{
-    return _mm_maddubs_epi16(a, b);
-}
-
-/* These don't actually get called, but we need to define them. */
-__attribute__ ((target("default"))) static inline __m128i sse_interleave_odd_epi16(__m128i a, __m128i b) { return a; }
-__attribute__ ((target("default"))) static inline __m128i sse_interleave_even_epi16(__m128i a, __m128i b) { return a; }
-__attribute__ ((target("default"))) static inline __m128i sse_mulu_odd_epi8(__m128i a, __m128i b) { return a; }
-__attribute__ ((target("default"))) static inline __m128i sse_mulu_even_epi8(__m128i a, __m128i b) { return a; }
-__attribute__ ((target("default"))) static inline __m128i sse_hadds_epi16(__m128i a, __m128i b) { return a; }
-__attribute__ ((target("default"))) static inline __m128i sse_maddubs_epi16(__m128i a, __m128i b) { return a; }
 
 /*
   Original loop per 4 bytes:
@@ -146,12 +108,7 @@ __attribute__ ((target("default"))) static inline __m128i sse_maddubs_epi16(__m1
     s1 += (uint32)(t1[0] + t1[1] + t1[2] + t1[3] + t1[4] + t1[5] + t1[6] + t1[7]) +
           32*CHAR_OFFSET;
  */
-/*
+__attribute__ ((target("ssse3"))) MVSTATIC int32 get_checksum1_ssse3_32(schar* buf, int32 len, int32 i, uint32* ps1, uint32* ps2)
-  Both sse2 and ssse3 targets must be specified here or we lose (a lot) of
-  performance, possibly due to not unrolling+inlining the called targeted
-  functions.
- */
-__attribute__ ((target("sse2", "ssse3"))) static int32 get_checksum1_sse2_32(schar* buf, int32 len, int32 i, uint32* ps1, uint32* ps2)
 {
     if (len > 32) {
         int aligned = ((uintptr_t)buf & 15) == 0;
@@ -167,16 +124,11 @@ __attribute__ ((target("sse2", "ssse3"))) static int32 get_checksum1_sse2_32(sch
 
         for (; i < (len-32); i+=32) {
             // Load ... 2*[int8*16]
-            // SSSE3 has _mm_lqqdu_si128, but this requires another
-            // target function for each SSE2 and SSSE3 loads. For reasons
-            // unknown (to me) we lose about 10% performance on some CPUs if
-            // we do that right here. We just use _mm_loadu_si128 as for all
-            // but a handful of specific old CPUs they are synonymous, and
-            // take the 1-5% hit on those specific CPUs where it isn't.
             __m128i in8_1, in8_2;
             if (!aligned) {
-                in8_1 = _mm_loadu_si128((__m128i_u*)&buf[i]);
+                // Synonymous with _mm_loadu_si128 on all but a handful of old CPUs
-                in8_2 = _mm_loadu_si128((__m128i_u*)&buf[i + 16]);
+                in8_1 = _mm_lddqu_si128((__m128i_u*)&buf[i]);
+                in8_2 = _mm_lddqu_si128((__m128i_u*)&buf[i + 16]);
             } else {
                 in8_1 = _mm_load_si128((__m128i_u*)&buf[i]);
                 in8_2 = _mm_load_si128((__m128i_u*)&buf[i + 16]);
@@ -185,13 +137,13 @@ __attribute__ ((target("sse2", "ssse3"))) static int32 get_checksum1_sse2_32(sch
             // (1*buf[i] + 1*buf[i+1]), (1*buf[i+2], 1*buf[i+3]), ... 2*[int16*8]
             // Fastest, even though multiply by 1
             __m128i mul_one = _mm_set1_epi8(1);
-            __m128i add16_1 = sse_maddubs_epi16(mul_one, in8_1);
+            __m128i add16_1 = _mm_maddubs_epi16(mul_one, in8_1);
-            __m128i add16_2 = sse_maddubs_epi16(mul_one, in8_2);
+            __m128i add16_2 = _mm_maddubs_epi16(mul_one, in8_2);
 
             // (4*buf[i] + 3*buf[i+1]), (2*buf[i+2], buf[i+3]), ... 2*[int16*8]
             __m128i mul_const = _mm_set1_epi32(4 + (3 << 8) + (2 << 16) + (1 << 24));
-            __m128i mul_add16_1 = sse_maddubs_epi16(mul_const, in8_1);
+            __m128i mul_add16_1 = _mm_maddubs_epi16(mul_const, in8_1);
-            __m128i mul_add16_2 = sse_maddubs_epi16(mul_const, in8_2);
+            __m128i mul_add16_2 = _mm_maddubs_epi16(mul_const, in8_2);
 
             // s2 += 32*s1
             ss2 = _mm_add_epi32(ss2, _mm_slli_epi32(ss1, 5));
@@ -224,7 +176,111 @@ __attribute__ ((target("sse2", "ssse3"))) static int32 get_checksum1_sse2_32(sch
             // [t1[0] + t1[1], t1[2] + t1[3] ...] [int16*8]
             // We could've combined this with generating sum_add32 above and
             // save an instruction but benchmarking shows that as being slower
-            __m128i add16 = sse_hadds_epi16(add16_1, add16_2);
+            __m128i add16 = _mm_hadds_epi16(add16_1, add16_2);
+
+            // [t1[0], t1[1], ...] -> [t1[0]*28 + t1[1]*24, ...] [int32*4]
+            __m128i mul32 = _mm_madd_epi16(add16, mul_t1);
+
+            // [sum(mul32), X, X, X] [int32*4]; faster than multiple _mm_hadd_epi32
+            mul32 = _mm_add_epi32(mul32, _mm_srli_si128(mul32, 4));
+            mul32 = _mm_add_epi32(mul32, _mm_srli_si128(mul32, 8));
+
+            // s2 += 28*t1[0] + 24*t1[1] + 20*t1[2] + 16*t1[3] + 12*t1[4] + 8*t1[5] + 4*t1[6]
+            ss2 = _mm_add_epi32(ss2, mul32);
+
+#if CHAR_OFFSET != 0
+            // s1 += 32*CHAR_OFFSET
+            __m128i char_offset_multiplier = _mm_set1_epi32(32 * CHAR_OFFSET);
+            ss1 = _mm_add_epi32(ss1, char_offset_multiplier);
+
+            // s2 += 528*CHAR_OFFSET
+            char_offset_multiplier = _mm_set1_epi32(528 * CHAR_OFFSET);
+            ss2 = _mm_add_epi32(ss2, char_offset_multiplier);
+#endif
+        }
+
+        _mm_store_si128((__m128i_u*)x, ss1);
+        *ps1 = x[0];
+        _mm_store_si128((__m128i_u*)x, ss2);
+        *ps2 = x[0];
+    }
+    return i;
+}
+
+/*
+  Same as SSSE3 version, but using macros defined above to emulate SSSE3 calls that are not available with SSE2.
+  For GCC-only the SSE2 and SSSE3 versions could be a single function calling other functions with the right
+  target attributes to emulate SSSE3 calls on SSE2 if needed, but clang doesn't inline those properly leading
+  to a near 50% performance drop.
+ */
+__attribute__ ((target("sse2"))) MVSTATIC int32 get_checksum1_sse2_32(schar* buf, int32 len, int32 i, uint32* ps1, uint32* ps2)
+{
+    if (len > 32) {
+        int aligned = ((uintptr_t)buf & 15) == 0;
+
+        uint32 x[4] = {0};
+        x[0] = *ps1;
+        __m128i ss1 = _mm_loadu_si128((__m128i_u*)x);
+        x[0] = *ps2;
+        __m128i ss2 = _mm_loadu_si128((__m128i_u*)x);
+
+        const int16 mul_t1_buf[8] = {28, 24, 20, 16, 12, 8, 4, 0};
+        __m128i mul_t1 = _mm_loadu_si128((__m128i_u*)mul_t1_buf);
+
+        for (; i < (len-32); i+=32) {
+            // Load ... 2*[int8*16]
+            __m128i in8_1, in8_2;
+            if (!aligned) {
+                in8_1 = _mm_loadu_si128((__m128i_u*)&buf[i]);
+                in8_2 = _mm_loadu_si128((__m128i_u*)&buf[i + 16]);
+            } else {
+                in8_1 = _mm_load_si128((__m128i_u*)&buf[i]);
+                in8_2 = _mm_load_si128((__m128i_u*)&buf[i + 16]);
+            }
+
+            // (1*buf[i] + 1*buf[i+1]), (1*buf[i+2], 1*buf[i+3]), ... 2*[int16*8]
+            // Fastest, even though multiply by 1
+            __m128i mul_one = _mm_set1_epi8(1);
+            __m128i add16_1 = SSE2_MADDUBS_EPI16(mul_one, in8_1);
+            __m128i add16_2 = SSE2_MADDUBS_EPI16(mul_one, in8_2);
+
+            // (4*buf[i] + 3*buf[i+1]), (2*buf[i+2], buf[i+3]), ... 2*[int16*8]
+            __m128i mul_const = _mm_set1_epi32(4 + (3 << 8) + (2 << 16) + (1 << 24));
+            __m128i mul_add16_1 = SSE2_MADDUBS_EPI16(mul_const, in8_1);
+            __m128i mul_add16_2 = SSE2_MADDUBS_EPI16(mul_const, in8_2);
+
+            // s2 += 32*s1
+            ss2 = _mm_add_epi32(ss2, _mm_slli_epi32(ss1, 5));
+
+            // [sum(t1[0]..t1[7]), X, X, X] [int32*4]; faster than multiple _mm_hadds_epi16
+            // Shifting left, then shifting right again and shuffling (rather than just
+            // shifting right as with mul32 below) to cheaply end up with the correct sign
+            // extension as we go from int16 to int32.
+            __m128i sum_add32 = _mm_add_epi16(add16_1, add16_2);
+            sum_add32 = _mm_add_epi16(sum_add32, _mm_slli_si128(sum_add32, 2));
+            sum_add32 = _mm_add_epi16(sum_add32, _mm_slli_si128(sum_add32, 4));
+            sum_add32 = _mm_add_epi16(sum_add32, _mm_slli_si128(sum_add32, 8));
+            sum_add32 = _mm_srai_epi32(sum_add32, 16);
+            sum_add32 = _mm_shuffle_epi32(sum_add32, 3);
+
+            // [sum(t2[0]..t2[7]), X, X, X] [int32*4]; faster than multiple _mm_hadds_epi16
+            __m128i sum_mul_add32 = _mm_add_epi16(mul_add16_1, mul_add16_2);
+            sum_mul_add32 = _mm_add_epi16(sum_mul_add32, _mm_slli_si128(sum_mul_add32, 2));
+            sum_mul_add32 = _mm_add_epi16(sum_mul_add32, _mm_slli_si128(sum_mul_add32, 4));
+            sum_mul_add32 = _mm_add_epi16(sum_mul_add32, _mm_slli_si128(sum_mul_add32, 8));
+            sum_mul_add32 = _mm_srai_epi32(sum_mul_add32, 16);
+            sum_mul_add32 = _mm_shuffle_epi32(sum_mul_add32, 3);
+
+            // s1 += t1[0] + t1[1] + t1[2] + t1[3] + t1[4] + t1[5] + t1[6] + t1[7]
+            ss1 = _mm_add_epi32(ss1, sum_add32);
+
+            // s2 += t2[0] + t2[1] + t2[2] + t2[3] + t2[4] + t2[5] + t2[6] + t2[7]
+            ss2 = _mm_add_epi32(ss2, sum_mul_add32);
+
+            // [t1[0] + t1[1], t1[2] + t1[3] ...] [int16*8]
+            // We could've combined this with generating sum_add32 above and
+            // save an instruction but benchmarking shows that as being slower
+            __m128i add16 = SSE2_HADDS_EPI16(add16_1, add16_2);
 
             // [t1[0], t1[1], ...] -> [t1[0]*28 + t1[1]*24, ...] [int32*4]
             __m128i mul32 = _mm_madd_epi16(add16, mul_t1);
@@ -270,7 +326,7 @@ __attribute__ ((target("sse2", "ssse3"))) static int32 get_checksum1_sse2_32(sch
     s1 += (uint32)(t1[0] + t1[1] + t1[2] + t1[3] + t1[4] + t1[5] + t1[6] + t1[7] + t1[8] + t1[9] + t1[10] + t1[11] + t1[12] + t1[13] + t1[14] + t1[15]) +
           64*CHAR_OFFSET;
  */
-__attribute__ ((target("avx2"))) static int32 get_checksum1_avx2_64(schar* buf, int32 len, int32 i, uint32* ps1, uint32* ps2)
+__attribute__ ((target("avx2"))) MVSTATIC int32 get_checksum1_avx2_64(schar* buf, int32 len, int32 i, uint32* ps1, uint32* ps2)
 {
     if (len > 64) {
         // Instructions reshuffled compared to SSE2 for slightly better performance
@@ -377,17 +433,7 @@ __attribute__ ((target("avx2"))) static int32 get_checksum1_avx2_64(schar* buf,
     return i;
 }
 
-__attribute__ ((target("default"))) static int32 get_checksum1_avx2_64(schar* buf, int32 len, int32 i, uint32* ps1, uint32* ps2)
+static int32 get_checksum1_default_1(schar* buf, int32 len, int32 i, uint32* ps1, uint32* ps2)
-{
-    return i;
-}
-
-__attribute__ ((target("default"))) static int32 get_checksum1_sse2_32(schar* buf, int32 len, int32 i, uint32* ps1, uint32* ps2)
-{
-    return i;
-}
-
-static inline int32 get_checksum1_default_1(schar* buf, int32 len, int32 i, uint32* ps1, uint32* ps2)
 {
     uint32 s1 = *ps1;
     uint32 s2 = *ps2;
@@ -403,9 +449,10 @@ static inline int32 get_checksum1_default_1(schar* buf, int32 len, int32 i, uint
     return i;
 }
 
-extern "C" {
+/* With GCC 10 putting this implementation inside 'extern "C"' causes an
-
+   assembler error. That worked fine on GCC 5-9 and clang 6-10...
-uint32 get_checksum1(char *buf1, int32 len)
+  */
+static inline uint32 get_checksum1_cpp(char *buf1, int32 len)
 {
     int32 i = 0;
     uint32 s1 = 0;
@@ -414,7 +461,10 @@ uint32 get_checksum1(char *buf1, int32 len)
     // multiples of 64 bytes using AVX2 (if available)
     i = get_checksum1_avx2_64((schar*)buf1, len, i, &s1, &s2);
 
-    // multiples of 32 bytes using SSE2/SSSE3 (if available)
+    // multiples of 32 bytes using SSSE3 (if available)
+    i = get_checksum1_ssse3_32((schar*)buf1, len, i, &s1, &s2);
+
+    // multiples of 32 bytes using SSE2 (if available)
     i = get_checksum1_sse2_32((schar*)buf1, len, i, &s1, &s2);
 
     // whatever is left
@@ -423,7 +473,70 @@ uint32 get_checksum1(char *buf1, int32 len)
     return (s1 & 0xffff) + (s2 << 16);
 }
 
-} // "C"
+extern "C" {
+
+uint32 get_checksum1(char *buf1, int32 len)
+{
+    return get_checksum1_cpp(buf1, len);
+}
+
+} // extern "C"
+
+#ifdef BENCHMARK_SIMD_CHECKSUM1
+#pragma clang optimize off
+#pragma GCC push_options
+#pragma GCC optimize ("O0")
+
+#define ROUNDS 1024
+#define BLOCK_LEN 1024*1024
+
+#ifndef CLOCK_MONOTONIC_RAW
+#define CLOCK_MONOTONIC_RAW CLOCK_MONOTONIC
+#endif
+
+static void benchmark(const char* desc, int32 (*func)(schar* buf, int32 len, int32 i, uint32* ps1, uint32* ps2), schar* buf, int32 len) {
+    struct timespec start, end;
+    uint64_t us;
+    uint32_t cs, s1, s2;
+    int i, next;
+
+    clock_gettime(CLOCK_MONOTONIC_RAW, &start);
+    for (i = 0; i < ROUNDS; i++) {
+        s1 = s2 = 0;
+        next = func((schar*)buf, len, 0, &s1, &s2);
+        get_checksum1_default_1((schar*)buf, len, next, &s1, &s2);
+    }
+    clock_gettime(CLOCK_MONOTONIC_RAW, &end);
+    us = next == 0 ? 0 : (end.tv_sec - start.tv_sec) * 1000000 + (end.tv_nsec - start.tv_nsec) / 1000;
+    cs = next == 0 ? 0 : (s1 & 0xffff) + (s2 << 16);
+    printf("%-5s :: %5.0f MB/s :: %08x\n", desc, us ? (float)(len / (1024 * 1024) * ROUNDS) / ((float)us / 1000000.0f) : 0, cs);
+}
+
+static int32 get_checksum1_auto(schar* buf, int32 len, int32 i, uint32* ps1, uint32* ps2) {
+    uint32 cs = get_checksum1((char*)buf, len);
+    *ps1 = cs & 0xffff;
+    *ps2 = cs >> 16;
+    return len;
+}
+
+int main() {
+    int i;
+    unsigned char* buf = (unsigned char*)malloc(BLOCK_LEN);
+    for (i = 0; i < BLOCK_LEN; i++) buf[i] = (i + (i % 3) + (i % 11)) % 256;
+
+    benchmark("Auto", get_checksum1_auto, (schar*)buf, BLOCK_LEN);
+    benchmark("Raw-C", get_checksum1_default_1, (schar*)buf, BLOCK_LEN);
+    benchmark("SSE2", get_checksum1_sse2_32, (schar*)buf, BLOCK_LEN);
+    benchmark("SSSE3", get_checksum1_ssse3_32, (schar*)buf, BLOCK_LEN);
+    benchmark("AVX2", get_checksum1_avx2_64, (schar*)buf, BLOCK_LEN);
+
+    free(buf);
+    return 0;
+}
+
+#pragma GCC pop_options
+#pragma clang optimize on
+#endif /* BENCHMARK_SIMD_CHECKSUM1 */
 
 #endif /* HAVE_SIMD */
 #endif /* __cplusplus */

support/atomic-rsync

@@ -1,4 +1,4 @@
-#!/usr/bin/perl
+#!/usr/bin/env perl
 #
 # This script lets you update a hierarchy of files in an atomic way by
 # first creating a new hierarchy using rsync's --link-dest option, and

support/cvs2includes

@@ -1,4 +1,4 @@
-#!/usr/bin/perl
+#!/usr/bin/env perl
 #
 # This script finds all CVS/Entries files in the current directory and below
 # and creates a local .cvsinclude file with non-inherited rules including each

support/deny-rsync

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 # Send an error message via the rsync-protocol to a non-daemon client rsync.
 #
 # Usage:  deny-rsync "message"

support/file-attr-restore

@@ -1,4 +1,4 @@
-#!/usr/bin/perl
+#!/usr/bin/env perl
 # This script will parse the output of "find ARG [ARG...] -ls" and
 # apply (at your discretion) the permissions, owner, and group info
 # it reads onto any existing files and dirs (it doesn't try to affect

support/files-to-excludes

@@ -1,4 +1,4 @@
-#!/usr/bin/perl
+#!/usr/bin/env perl
 # This script takes an input of filenames and outputs a set of
 # include/exclude directives that can be used by rsync to copy
 # just the indicated files using an --exclude-from=FILE option.

support/git-set-file-times

@@ -1,4 +1,4 @@
-#!/usr/bin/python3
+#!/usr/bin/env python3
 
 import os, re, argparse, subprocess
 from datetime import datetime

support/instant-rsyncd

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 
 # instant-rsyncd lets you quickly set up and start a simple, unprivileged rsync
 # daemon with a single module in the current directory.  I've found it

support/logfilter

@@ -1,4 +1,4 @@
-#!/usr/bin/perl
+#!/usr/bin/env perl
 # Filter the rsync daemon log messages by module name.  The log file can be
 # in either syslog format or rsync's own log-file format.  Note that the
 # MODULE_NAME parameter is used in a regular-expression match in order to

support/lsh

@@ -1,4 +1,4 @@
-#!/usr/bin/perl
+#!/usr/bin/env perl
 # This script can be used as a "remote shell" command that is only
 # capable of pretending to connect to "localhost".  This is useful
 # for testing or for running a local copy where the sender and the

support/mapfrom

@@ -1,4 +1,4 @@
-#!/usr/bin/perl
+#!/usr/bin/env perl
 # This helper script makes it easy to use a passwd or group file to map
 # values in a LOCAL transfer.  For instance, if you mount a backup that
 # does not have the same passwd setup as the local machine, you can do

support/mapto

@@ -1,4 +1,4 @@
-#!/usr/bin/perl
+#!/usr/bin/env perl
 # This helper script makes it easy to use a passwd or group file to map
 # values in a LOCAL transfer.  For instance, if you mount a backup that
 # does not have the same passwd setup as the local machine, you can do

support/mnt-excl

@@ -1,4 +1,4 @@
-#!/usr/bin/perl
+#!/usr/bin/env perl
 # This script takes a command-line arg of a source directory
 # that will be passed to rsync, and generates a set of excludes
 # that will exclude all mount points from the list.  This is

support/munge-symlinks

@@ -1,4 +1,4 @@
-#!/usr/bin/perl
+#!/usr/bin/env perl
 # This script will either prefix all symlink values with the string
 # "/rsyncd-munged/" or remove that prefix.
 

support/rrsync

@@ -1,8 +1,8 @@
-#!/usr/bin/perl
+#!/usr/bin/env perl
 # Name: /usr/local/bin/rrsync (should also have a symlink in /usr/bin)
 # Purpose: Restricts rsync to subdirectory declared in .ssh/authorized_keys
 # Author: Joe Smith <js-cgi@inwap.com> 30-Sep-2004
-# Modified by: Wayne Davison <wayned@samba.org>
+# Modified by: Wayne Davison <wayne@opencoder.net>
 use strict;
 
 use Socket;

support/rsync-no-vanished

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 
 IGNOREEXIT=24
 IGNOREOUT='^(file has vanished: |rsync warning: some files vanished before they could be transferred)'

support/rsync-slash-strip

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 # This script can be used as an rsync command-line filter that strips a single
 # trailing slash from each arg.  That treats "src/" the same as "src", thus
 # you need to use "src/." or "src//" for just the contents of the "src" dir.

support/rsyncstats

@@ -1,4 +1,4 @@
-#!/usr/bin/perl
+#!/usr/bin/env perl
 #
 # This script parses the default logfile format produced by rsync when running
 # as a daemon with transfer logging enabled.  It also parses a slightly tweaked

testhelp/maketree.py

@@ -1,4 +1,4 @@
-#! /usr/bin/python2.2
+#!/usr/bin/env python2
 
 # Copyright (C) 2002 by Martin Pool <mbp@samba.org>
 

token.c

@@ -1135,30 +1135,23 @@ void send_token(int f, int32 token, struct map_struct *buf, OFF_T offset,
  */
 int32 recv_token(int f, char **data)
 {
-	int tok;
-
 	switch (do_compression) {
 	case CPRES_NONE:
-		tok = simple_recv_token(f,data);
+		return simple_recv_token(f,data);
-		break;
 	case CPRES_ZLIB:
 	case CPRES_ZLIBX:
-		tok = recv_deflated_token(f, data);
+		return recv_deflated_token(f, data);
-		break;
 #ifdef SUPPORT_ZSTD
 	case CPRES_ZSTD:
-		tok = recv_zstd_token(f, data);
+		return recv_zstd_token(f, data);
-		break;
 #endif
 #ifdef SUPPORT_LZ4
 	case CPRES_LZ4:
-		tok = recv_compressed_token(f, data);
+		return recv_compressed_token(f, data);
-		break;
 #endif
 	default:
 		assert(0);
 	}
-	return tok;
 }
 
 /*

uidlist.c

@@ -527,14 +527,14 @@ const char *getallgroups(uid_t uid, item_list *gid_list)
 		return "getpwuid failed";
 
 	gid_list->count = 0; /* We're overwriting any items in the list */
-	EXPAND_ITEM_LIST(gid_list, gid_t, 32);
+	(void)EXPAND_ITEM_LIST(gid_list, gid_t, 32);
 	size = gid_list->malloced;
 
 	/* Get all the process's groups, with the pw_gid group first. */
 	if (getgrouplist(pw->pw_name, pw->pw_gid, gid_list->items, &size) < 0) {
 		if (size > (int)gid_list->malloced) {
 			gid_list->count = gid_list->malloced;
-			EXPAND_ITEM_LIST(gid_list, gid_t, size);
+			(void)EXPAND_ITEM_LIST(gid_list, gid_t, size);
 			if (getgrouplist(pw->pw_name, pw->pw_gid, gid_list->items, &size) < 0)
 				size = -1;
 		} else
@@ -553,7 +553,7 @@ const char *getallgroups(uid_t uid, item_list *gid_list)
 				break;
 		}
 		if (j == size) { /* The default group wasn't found! */
-			EXPAND_ITEM_LIST(gid_list, gid_t, size+1);
+			(void)EXPAND_ITEM_LIST(gid_list, gid_t, size+1);
 			gid_array = gid_list->items;
 		}
 		gid_array[j] = gid_array[0];