Preparing for release of 3.2.2

The code now derives all the struct defines, default value assignments,
parser-param defines, and lp_foo() accessor functions from a single list
of daemon parameters.

.github/workflows/ccpp.yml

@@ -1,4 +1,4 @@
-name: C CI
+name: build
 
 on:
   push:
@@ -18,7 +18,7 @@ jobs:
     - name: prepare-source
       run: ./prepare-source
     - name: configure
-      run: ./configure --with-included-popt --with-protected-args --with-included-zlib --enable-simd
+      run: ./configure --with-included-popt --with-included-zlib
     - name: make
       run: make
     - name: version-summary

.gitignore

@@ -19,6 +19,9 @@ aclocal.m4
 /rsync*.5
 /rsync*.html
 /help-rsync*.h
+/default-cvsignore.h
+/default-dont-compress.h
+/daemon-parm.h
 /.md2man-works
 /autom4te*.cache
 /confdefs.h
@@ -46,5 +49,7 @@ aclocal.m4
 /testsuite/devices-fake.test
 /testsuite/xattrs-hlink.test
 /patches
-/SaVeDiR
+/patches.gen
+/build
+/auto-build-save
 .deps

Makefile.in

@@ -1,5 +1,4 @@
-# Makefile for rsync. This is processed by configure to produce the final
-# Makefile
+# The Makefile for rsync (configure creates it from Makefile.in).
 
 prefix=@prefix@
 datarootdir=@datarootdir@
@@ -10,6 +9,7 @@ mandir=@mandir@
 
 LIBS=@LIBS@
 CC=@CC@
+AWK=@AWK@
 CFLAGS=@CFLAGS@
 CPPFLAGS=@CPPFLAGS@
 CXX=@CXX@
@@ -26,17 +26,16 @@ MKDIR_P=@MKDIR_P@
 VPATH=$(srcdir)
 SHELL=/bin/sh
 
-VERSION=@RSYNC_VERSION@
-
 .SUFFIXES:
 .SUFFIXES: .c .o
 
-SIMD_x86_64=simd-checksum-x86_64.o lib/md5-asm-x86_64.o
+SIMD_x86_64=simd-checksum-x86_64.o
+ASM_x86_64=lib/md5-asm-x86_64.o
 
 GENFILES=configure.sh aclocal.m4 config.h.in proto.h proto.h-tstamp rsync.1 rsync.1.html \
 	 rsync-ssl.1 rsync-ssl.1.html rsyncd.conf.5 rsyncd.conf.5.html
 HEADERS=byteorder.h config.h errcode.h proto.h rsync.h ifuncs.h itypes.h inums.h \
-	lib/pool_alloc.h
+	lib/pool_alloc.h lib/mdigest.h lib/md-defines.h version.h
 LIBOBJ=lib/wildmatch.o lib/compat.o lib/snprintf.o lib/mdfour.o lib/md5.o \
 	lib/permstring.o lib/pool_alloc.o lib/sysacls.o lib/sysxattrs.o @LIBOBJS@
 zlib_OBJS=zlib/deflate.o zlib/inffast.o zlib/inflate.o zlib/inftrees.o \
@@ -45,7 +44,7 @@ OBJS1=flist.o rsync.o generator.o receiver.o cleanup.o sender.o exclude.o \
 	util.o util2.o main.o checksum.o match.o syscall.o log.o backup.o delete.o
 OBJS2=options.o io.o compat.o hlink.o token.o uidlist.o socket.o hashtable.o \
 	fileio.o batch.o clientname.o chmod.o acls.o xattrs.o
-OBJS3=progress.o pipe.o
+OBJS3=progress.o pipe.o @ASM@
 DAEMON_OBJ = params.o loadparm.o clientserver.o access.o connection.o authenticate.o
 popt_OBJS=popt/findme.o  popt/popt.o  popt/poptconfig.o \
 	popt/popthelp.o popt/poptparse.o
@@ -69,13 +68,13 @@ CHECK_OBJS=tls.o testrun.o getgroups.o getfsdev.o t_stub.o t_unsafe.o trimslash.
 @OBJ_RESTORE@
 
 .PHONY: all
-all: Makefile rsync$(EXEEXT) stunnel-rsyncd.conf man
+all: Makefile rsync$(EXEEXT) stunnel-rsyncd.conf @MAKE_MAN@
 
 .PHONY: install
 install: all
 	-${MKDIR_P} ${DESTDIR}${bindir}
 	${INSTALLCMD} ${INSTALL_STRIP} -m 755 rsync$(EXEEXT) ${DESTDIR}${bindir}
-	${INSTALLCMD} -m 755 rsync-ssl ${DESTDIR}${bindir}
+	${INSTALLCMD} -m 755 $(srcdir)/rsync-ssl ${DESTDIR}${bindir}
 	-${MKDIR_P} ${DESTDIR}${mandir}/man1
 	-${MKDIR_P} ${DESTDIR}${mandir}/man5
 	if test -f rsync.1; then ${INSTALLMAN} -m 644 rsync.1 ${DESTDIR}${mandir}/man1; fi
@@ -99,19 +98,21 @@ rsync$(EXEEXT): $(OBJS)
 
 $(OBJS): $(HEADERS)
 $(CHECK_OBJS): $(HEADERS)
+tls.o xattrs.o: lib/sysxattrs.h
 options.o: latest-year.h help-rsync.h help-rsyncd.h
+exclude.o: default-cvsignore.h
+loadparm.o: default-dont-compress.h daemon-parm.h
 
 flist.o: rounding.h
 
-help-rsync.h help-rsyncd.h: rsync.1.md
-	@sed -e '1,/^\[comment\].*$@/d' \
-	    -e '1,/^```/d' \
-	    -e '/^```/,$$d' \
-	    -e 's/"/\\"/g' \
-	    -e 's/^/  rprintf(F,"/' \
-	    -e 's/$$/\\n");/' \
-	    <"$(srcdir)/$<" >$@
-	@if ! test -s $@; then rm -f $@ ; echo "The Makefile generated an empty file: $@" ; exit 1 ; fi
+default-cvsignore.h default-dont-compress.h: rsync.1.md define-from-md.awk
+	$(AWK) -f $(srcdir)/define-from-md.awk -v hfile=$@ $(srcdir)/rsync.1.md
+
+help-rsync.h help-rsyncd.h: rsync.1.md help-from-md.awk
+	$(AWK) -f $(srcdir)/help-from-md.awk -v hfile=$@ $(srcdir)/rsync.1.md
+
+daemon-parm.h: daemon-parm.txt daemon-parm.awk
+	$(AWK) -f $(srcdir)/daemon-parm.awk $(srcdir)/daemon-parm.txt
 
 rounding.h: rounding.c rsync.h proto.h
 	@for r in 0 1 3; do \
@@ -132,10 +133,10 @@ rounding.h: rounding.c rsync.h proto.h
 	@rm -f rounding.out
 
 simd-checksum-x86_64.o: simd-checksum-x86_64.cpp
-	$(CXX) $(CXXFLAGS) $(CPPFLAGS) -c -o $@ $<
+	@$(srcdir)/cmdormsg disable-simd $(CXX) -I. $(CXXFLAGS) $(CPPFLAGS) -c -o $@ $(srcdir)/simd-checksum-x86_64.cpp
 
-lib/md5-asm-x86_64.o: lib/md5-asm-x86_64.s
-	$(CC) -c -o $@ $<
+lib/md5-asm-x86_64.o: lib/md5-asm-x86_64.S config.h lib/md-defines.h
+	@$(srcdir)/cmdormsg disable-asm $(CC) -I. @NOEXECSTACK@ -c -o $@ $(srcdir)/lib/md5-asm-x86_64.S
 
 tls$(EXEEXT): $(TLS_OBJ)
 	$(CC) $(CFLAGS) $(LDFLAGS) -o $@ $(TLS_OBJ) $(LIBS)
@@ -202,6 +203,10 @@ reconfigure: configure.sh
 	./config.status --recheck
 	./config.status
 
+.PHONY: restatus
+restatus:
+	./config.status
+
 Makefile: Makefile.in config.status configure.sh config.h.in
 	@if test -f Makefile; then cp -p Makefile Makefile.old; else touch Makefile.old; fi
 	@./config.status
@@ -226,19 +231,19 @@ proto: proto.h-tstamp
 proto.h: proto.h-tstamp
 	@if test -f proto.h; then :; else cp -p $(srcdir)/proto.h .; fi
 
-proto.h-tstamp: $(srcdir)/*.c $(srcdir)/lib/compat.c config.h
-	awk -f $(srcdir)/mkproto.awk $(srcdir)/*.c $(srcdir)/lib/compat.c
+proto.h-tstamp: $(srcdir)/*.c $(srcdir)/lib/compat.c config.h daemon-parm.h
+	$(AWK) -f $(srcdir)/mkproto.awk $(srcdir)/*.c $(srcdir)/lib/compat.c daemon-parm.h
 
 .PHONY: man
 man: rsync.1 rsync-ssl.1 rsyncd.conf.5
 
-rsync.1: rsync.1.md md2man NEWS.md Makefile
+rsync.1: rsync.1.md md2man version.h Makefile
 	@$(srcdir)/maybe-make-man $(srcdir) rsync.1.md
 
-rsync-ssl.1: rsync-ssl.1.md md2man NEWS.md Makefile
+rsync-ssl.1: rsync-ssl.1.md md2man version.h Makefile
 	@$(srcdir)/maybe-make-man $(srcdir) rsync-ssl.1.md
 
-rsyncd.conf.5: rsyncd.conf.5.md md2man NEWS.md Makefile
+rsyncd.conf.5: rsyncd.conf.5.md md2man version.h Makefile
 	@$(srcdir)/maybe-make-man $(srcdir) rsyncd.conf.5.md
 
 .PHONY: clean

README.md

@@ -65,7 +65,7 @@ WEB SITE
 
 The main rsync web site is here:
 
->  http://rsync.samba.org/
+>  https://rsync.samba.org/
 
 You'll find a FAQ list, downloads, resources, HTML versions of the
 manpages, etc.
@@ -80,7 +80,7 @@ list, and there is also an announcement-only mailing list for those that
 want official announcements.  See the mailing-list page for full
 details:
 
->  http://rsync.samba.org/lists.html
+>  https://rsync.samba.org/lists.html
 
 
 BUG REPORTS
@@ -88,14 +88,14 @@ BUG REPORTS
 
 To visit this web page for full the details on bug reporting:
 
->  http://rsync.samba.org/bugzilla.html
+>  https://rsync.samba.org/bugtracking.html
 
 That page contains links to the current bug list, and information on how
 to report a bug well.  You might also like to try searching the Internet
 for the error message you've received, or looking in the mailing list
 archives at:
 
->  http://mail-archive.com/rsync@lists.samba.org/
+>  https://mail-archive.com/rsync@lists.samba.org/
 
 To send a bug report, follow the instructions on the bug-tracking
 page of the web site.
@@ -112,38 +112,31 @@ is hosted on github and on samba's site.  Feel free to access it here:
 
 >  https://github.com/WayneD/rsync
 
-or clone it from its samba repo:
+A backup git repo is available on the samba site:
 
-    git clone git://git.samba.org/rsync.git
+>     git clone git://git.samba.org/rsync.git
 
 See the download page for full details on all the ways to grab the
 source:
 
->  http://rsync.samba.org/download.html
+>  https://rsync.samba.org/download.html
 
 
 COPYRIGHT
 ---------
 
 Rsync was originally written by Andrew Tridgell and is currently
-maintained by Wayne Davison.   It has been improved by many developers
+maintained by Wayne Davison.  It has been improved by many developers
 from around the world.
 
 Rsync may be used, modified and redistributed only under the terms of
 the GNU General Public License, found in the file COPYING in this
 distribution, or at:
 
->  http://www.fsf.org/licenses/gpl.html
+>  https://www.fsf.org/licenses/gpl.html
 
 
 AVAILABILITY
 ------------
 
-The main web site for rsync is http://rsync.samba.org/
-
-The main ftp site is ftp://rsync.samba.org/pub/rsync/
-
-This is also available as rsync://download.samba.org/rsyncftp/ if you
-connect via ssl. Use the `rsync-ssl` script if you have it, otherwise
-connect to the rsync server via a normal rsync command and it will
-output some instructions for how to connect.
+The main web site for rsync is https://rsync.samba.org/

TODO

@@ -49,7 +49,7 @@ Create test makefile target for some tests
 
 RELATED PROJECTS -----------------------------------------------------
 rsyncsh
-http://rsync.samba.org/rsync-and-debian/
+https://rsync.samba.org/rsync-and-debian/
 rsyncable gzip patch
 rsyncsplit as alternative to real integration with gzip?
 reverse rsync over HTTP Range
@@ -66,8 +66,8 @@ Use chroot only if supported
   If running as non-root, then don't fail, just give a warning.
   (There was a thread about this a while ago?)
 
-    http://lists.samba.org/pipermail/rsync/2001-August/thread.html
-    http://lists.samba.org/pipermail/rsync/2001-September/thread.html
+    https://lists.samba.org/pipermail/rsync/2001-August/thread.html
+    https://lists.samba.org/pipermail/rsync/2001-September/thread.html
 
                       --          --
 
@@ -204,7 +204,7 @@ Create more granular verbosity					2003/05/15
   fine grained selection of statistical reporting and what
   actions are logged.
 
-  http://lists.samba.org/archive/rsync/2003-May/006059.html
+  https://lists.samba.org/archive/rsync/2003-May/006059.html
 
                       --          --
 
@@ -287,7 +287,7 @@ Perhaps flush stdout like syslog
 
   Perhaps flush stdout after each filename, so that people trying to
   monitor progress in a log file can do so more easily.  See
-  http://bugs.debian.org/cgi-bin/bugreport.cgi?bug=48108
+  https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=48108
 
                       --          --
 
@@ -495,7 +495,7 @@ rsyncsh
                       --          --
 
 
-http://rsync.samba.org/rsync-and-debian/
+https://rsync.samba.org/rsync-and-debian/
 
 
                       --          --

access.c

@@ -19,6 +19,7 @@
  */
 
 #include "rsync.h"
+#include "ifuncs.h"
 
 static int allow_forward_dns;
 
@@ -52,10 +53,8 @@ static int match_hostname(const char **host_ptr, const char *addr, const char *t
 		if (strcmp(addr, inet_ntoa(*(struct in_addr*)(hp->h_addr_list[i]))) == 0) {
 			/* If reverse lookups are off, we'll use the conf-specified
 			 * hostname in preference to UNDETERMINED. */
-			if (host == undetermined_hostname) {
-				if (!(*host_ptr = strdup(tok)))
-					*host_ptr = undetermined_hostname;
-			}
+			if (host == undetermined_hostname)
+				*host_ptr = strdup(tok);
 			return 1;
 		}
 	}
@@ -241,9 +240,6 @@ static int access_match(const char *list, const char *addr, const char **host_pt
 	char *tok;
 	char *list2 = strdup(list);
 
-	if (!list2)
-		out_of_memory("access_match");
-
 	strlower(list2);
 
 	for (tok = strtok(list2, " ,\t"); tok; tok = strtok(NULL, " ,\t")) {

acls.c

@@ -168,8 +168,6 @@ static rsync_acl *create_racl(void)
 {
 	rsync_acl *racl = new(rsync_acl);
 
-	if (!racl)
-		out_of_memory("create_racl");
 	*racl = empty_rsync_acl;
 
 	return racl;
@@ -335,8 +333,7 @@ static BOOL unpack_smb_acl(SMB_ACL_T sacl, rsync_acl *racl)
 			qsort(temp_ida_list.items, temp_ida_list.count, sizeof (id_access), id_access_sorter);
 		}
 #endif
-		if (!(racl->names.idas = new_array(id_access, temp_ida_list.count)))
-			out_of_memory("unpack_smb_acl");
+		racl->names.idas = new_array(id_access, temp_ida_list.count);
 		memcpy(racl->names.idas, temp_ida_list.items, temp_ida_list.count * sizeof (id_access));
 	} else
 		racl->names.idas = NULL;
@@ -505,9 +502,7 @@ static int get_rsync_acl(const char *fname, rsync_acl *racl,
 
 		if (cnt) {
 			char *bp = buf + 4*4;
-			id_access *ida;
-			if (!(ida = racl->names.idas = new_array(id_access, cnt)))
-				out_of_memory("get_rsync_acl");
+			id_access *ida = racl->names.idas = new_array(id_access, cnt);
 			racl->names.count = cnt;
 			for ( ; cnt--; ida++, bp += 4+4) {
 				ida->id = IVAL(bp, 0);
@@ -703,12 +698,7 @@ static uchar recv_ida_entries(int f, ida_entries *ent)
 	uchar computed_mask_bits = 0;
 	int i, count = read_varint(f);
 
-	if (count) {
-		if (!(ent->idas = new_array(id_access, count)))
-			out_of_memory("recv_ida_entries");
-	} else
-		ent->idas = NULL;
-
+	ent->idas = count ? new_array(id_access, count) : NULL;
 	ent->count = count;
 
 	for (i = 0; i < count; i++) {

authenticate.c

@@ -20,6 +20,7 @@
 
 #include "rsync.h"
 #include "itypes.h"
+#include "ifuncs.h"
 
 extern int read_only;
 extern char *password_file;
@@ -250,8 +251,7 @@ char *auth_server(int f_in, int f_out, int module, const char *host,
 	}
 	*pass++ = '\0';
 
-	if (!(users = strdup(users)))
-		out_of_memory("auth_server");
+	users = strdup(users);
 
 	for (tok = strtok(users, " ,\t"); tok; tok = strtok(NULL, " ,\t")) {
 		char *opts;
@@ -287,8 +287,7 @@ char *auth_server(int f_in, int f_out, int module, const char *host,
 				else {
 					gid_t *gid_array = gid_list.items;
 					auth_uid_groups_cnt = gid_list.count;
-					if ((auth_uid_groups = new_array(char *, auth_uid_groups_cnt)) == NULL)
-						out_of_memory("auth_server");
+					auth_uid_groups = new_array(char *, auth_uid_groups_cnt);
 					for (j = 0; j < auth_uid_groups_cnt; j++)
 						auth_uid_groups[j] = gid_to_group(gid_array[j]);
 				}

batch.c

@@ -38,13 +38,10 @@ extern int do_compression;
 extern int inplace;
 extern int append_mode;
 extern int write_batch;
-extern int xfersum_type;
 extern int protocol_version;
 extern int raw_argc, cooked_argc;
 extern char **raw_argv, **cooked_argv;
 extern char *batch_name;
-extern const char *checksum_choice;
-extern const char *compress_choice;
 #ifdef ICONV_OPTION
 extern char *iconv_opt;
 #endif
@@ -269,14 +266,6 @@ void write_batch_shell_file(void)
 			err |= write_opt("--exclude-from", "-");
 	}
 
-	/* We need to make sure that any protocol-based or negotiated choices get accurately
-	 * reflected in the options we save AND that we avoid any need for --read-batch to
-	 * do a string-based negotation (since we don't write them into the file). */
-	if (do_compression)
-		err |= write_opt("--compress-choice", compress_choice);
-	if (strchr(checksum_choice, ',') || xfersum_type != parse_csum_name(NULL, -1))
-		err |= write_opt("--checksum-choice", checksum_choice);
-
 	/* Elide the filename args from the option list, but scan for them in reverse. */
 	for (i = raw_argc-1, j = cooked_argc-1; i > 0 && j >= 0; i--) {
 		if (strcmp(raw_argv[i], cooked_argv[j]) == 0) {

checksum.c

@@ -27,8 +27,12 @@
  */
 
 #include "rsync.h"
+
 #ifdef SUPPORT_XXHASH
 #include "xxhash.h"
+# if XXH_VERSION_NUMBER >= 800
+#  define SUPPORT_XXH3 1
+# endif
 #endif
 
 extern int am_server;
@@ -40,6 +44,10 @@ extern const char *checksum_choice;
 
 struct name_num_obj valid_checksums = {
 	"checksum", NULL, NULL, 0, 0, {
+#ifdef SUPPORT_XXH3
+		{ CSUM_XXH3_128, "xxh128", NULL },
+		{ CSUM_XXH3_64, "xxh3", NULL },
+#endif
 #ifdef SUPPORT_XXHASH
 		{ CSUM_XXH64, "xxh64", NULL },
 		{ CSUM_XXH64, "xxhash", NULL },
@@ -85,7 +93,7 @@ static const char *checksum_name(int num)
 {
 	struct name_num_item *nni = get_nni_by_num(&valid_checksums, num);
 
-	return nni ? nni->name : num < CSUM_MD4 ? "MD4" : "UNKNOWN";
+	return nni ? nni->name : num < CSUM_MD4 ? "md4" : "UNKNOWN";
 }
 
 void parse_checksum_choice(int final_call)
@@ -99,6 +107,8 @@ void parse_checksum_choice(int final_call)
 			checksum_type = parse_csum_name(cp+1, -1);
 		} else
 			xfersum_type = checksum_type = parse_csum_name(checksum_choice, -1);
+		if (am_server && checksum_choice)
+			validate_choice_vs_env(NSTR_CHECKSUM, xfersum_type, checksum_type);
 	}
 
 	if (xfersum_type == CSUM_NONE)
@@ -133,10 +143,11 @@ int csum_len_for_type(int cst, BOOL flist_csum)
 		return MD4_DIGEST_LEN;
 	  case CSUM_MD5:
 		return MD5_DIGEST_LEN;
-#ifdef SUPPORT_XXHASH
 	  case CSUM_XXH64:
+	  case CSUM_XXH3_64:
 		return 64/8;
-#endif
+	  case CSUM_XXH3_128:
+		return 128/8;
 	  default: /* paranoia to prevent missing case values */
 		exit_cleanup(RERR_UNSUPPORTED);
 	}
@@ -158,10 +169,10 @@ int canonical_checksum(int csum_type)
 	  case CSUM_MD4:
 	  case CSUM_MD5:
 		return -1;
-#ifdef SUPPORT_XXHASH
 	  case CSUM_XXH64:
+	  case CSUM_XXH3_64:
+	  case CSUM_XXH3_128:
 		return 1;
-#endif
 	  default: /* paranoia to prevent missing case values */
 		exit_cleanup(RERR_UNSUPPORTED);
 	}
@@ -198,6 +209,17 @@ void get_checksum2(char *buf, int32 len, char *sum)
 	  case CSUM_XXH64:
 		SIVAL64(sum, 0, XXH64(buf, len, checksum_seed));
 		break;
+#endif
+#ifdef SUPPORT_XXH3
+	  case CSUM_XXH3_64:
+		SIVAL64(sum, 0, XXH3_64bits_withSeed(buf, len, checksum_seed));
+		break;
+	  case CSUM_XXH3_128: {
+		XXH128_hash_t digest = XXH3_128bits_withSeed(buf, len, checksum_seed);
+		SIVAL64(sum, 0, digest.low64);
+		SIVAL64(sum, 8, digest.high64);
+		break;
+	  }
 #endif
 	  case CSUM_MD5: {
 		MD5_CTX m5;
@@ -249,8 +271,6 @@ void get_checksum2(char *buf, int32 len, char *sum)
 				free(buf1);
 			buf1 = new_array(char, len+4);
 			len1 = len;
-			if (!buf1)
-				out_of_memory("get_checksum2");
 		}
 
 		memcpy(buf1, buf, len);
@@ -313,6 +333,45 @@ void file_checksum(const char *fname, const STRUCT_STAT *st_p, char *sum)
 		SIVAL64(sum, 0, XXH64_digest(state));
 		break;
 	  }
+#endif
+#ifdef SUPPORT_XXH3
+	  case CSUM_XXH3_64: {
+		static XXH3_state_t* state = NULL;
+		if (!state && !(state = XXH3_createState()))
+			out_of_memory("file_checksum");
+
+		XXH3_64bits_reset(state);
+
+		for (i = 0; i + CHUNK_SIZE <= len; i += CHUNK_SIZE)
+			XXH3_64bits_update(state, (uchar *)map_ptr(buf, i, CHUNK_SIZE), CHUNK_SIZE);
+
+		remainder = (int32)(len - i);
+		if (remainder > 0)
+			XXH3_64bits_update(state, (uchar *)map_ptr(buf, i, remainder), remainder);
+
+		SIVAL64(sum, 0, XXH3_64bits_digest(state));
+		break;
+	  }
+	  case CSUM_XXH3_128: {
+		XXH128_hash_t digest;
+		static XXH3_state_t* state = NULL;
+		if (!state && !(state = XXH3_createState()))
+			out_of_memory("file_checksum");
+
+		XXH3_128bits_reset(state);
+
+		for (i = 0; i + CHUNK_SIZE <= len; i += CHUNK_SIZE)
+			XXH3_128bits_update(state, (uchar *)map_ptr(buf, i, CHUNK_SIZE), CHUNK_SIZE);
+
+		remainder = (int32)(len - i);
+		if (remainder > 0)
+			XXH3_128bits_update(state, (uchar *)map_ptr(buf, i, remainder), remainder);
+
+		digest = XXH3_128bits_digest(state);
+		SIVAL64(sum, 0, digest.low64);
+		SIVAL64(sum, 8, digest.high64);
+		break;
+	  }
 #endif
 	  case CSUM_MD5: {
 		MD5_CTX m5;
@@ -389,6 +448,9 @@ static union {
 #ifdef SUPPORT_XXHASH
 static XXH64_state_t* xxh64_state;
 #endif
+#ifdef SUPPORT_XXH3
+static XXH3_state_t* xxh3_state;
+#endif
 static int cursum_type;
 
 void sum_init(int csum_type, int seed)
@@ -406,6 +468,18 @@ void sum_init(int csum_type, int seed)
 			out_of_memory("sum_init");
 		XXH64_reset(xxh64_state, 0);
 		break;
+#endif
+#ifdef SUPPORT_XXH3
+	  case CSUM_XXH3_64:
+		if (!xxh3_state && !(xxh3_state = XXH3_createState()))
+			out_of_memory("sum_init");
+		XXH3_64bits_reset(xxh3_state);
+		break;
+	  case CSUM_XXH3_128:
+		if (!xxh3_state && !(xxh3_state = XXH3_createState()))
+			out_of_memory("sum_init");
+		XXH3_128bits_reset(xxh3_state);
+		break;
 #endif
 	  case CSUM_MD5:
 		MD5_Init(&ctx.m5);
@@ -448,6 +522,14 @@ void sum_update(const char *p, int32 len)
 	  case CSUM_XXH64:
 		XXH64_update(xxh64_state, p, len);
 		break;
+#endif
+#ifdef SUPPORT_XXH3
+	  case CSUM_XXH3_64:
+		XXH3_64bits_update(xxh3_state, p, len);
+		break;
+	  case CSUM_XXH3_128:
+		XXH3_128bits_update(xxh3_state, p, len);
+		break;
 #endif
 	  case CSUM_MD5:
 		MD5_Update(&ctx.m5, (uchar *)p, len);
@@ -502,6 +584,17 @@ int sum_end(char *sum)
 	  case CSUM_XXH64:
 		SIVAL64(sum, 0, XXH64_digest(xxh64_state));
 		break;
+#endif
+#ifdef SUPPORT_XXH3
+	  case CSUM_XXH3_64:
+		SIVAL64(sum, 0, XXH3_64bits_digest(xxh3_state));
+		break;
+	  case CSUM_XXH3_128: {
+		XXH128_hash_t digest = XXH3_128bits_digest(xxh3_state);
+		SIVAL64(sum, 0, digest.low64);
+		SIVAL64(sum, 8, digest.high64);
+		break;
+	  }
 #endif
 	  case CSUM_MD5:
 		MD5_Final((uchar *)sum, &ctx.m5);

cleanup.c

@@ -221,8 +221,13 @@ NORETURN void _exit_cleanup(int code, const char *file, int line)
 		/* If line < 0, this exit is after a MSG_ERROR_EXIT event, so
 		 * we don't want to output a duplicate error. */
 		if ((exit_code && line > 0)
-		 || am_daemon || (logfile_name && (am_server || !INFO_GTE(STATS, 1))))
+		 || am_daemon || (logfile_name && (am_server || !INFO_GTE(STATS, 1)))) {
+#ifdef HAVE_USLEEP /* A tiny delay just in case both sender & receiver are sending a msg at the same time. */
+			if (am_server && exit_code)
+				usleep(50);
+#endif
 			log_exit(exit_code, exit_file, exit_line);
+		}
 
 #include "case_N.h"
 		switch_step++;

clientname.c

@@ -156,8 +156,8 @@ char *client_name(const char *ipaddr)
 }
 
 
-/* Try to read an haproxy header (V1 or V2). Returns 1 on success or 0 on failure. */
-int read_haproxy_header(int fd)
+/* Try to read an proxy protocol header (V1 or V2). Returns 1 on success or 0 on failure. */
+int read_proxy_protocol_header(int fd)
 {
 	union {
 		struct {
@@ -388,7 +388,7 @@ static int compare_addrinfo_sockaddr(const struct addrinfo *ai, const struct soc
 		sin1 = (const struct sockaddr_in6 *) ss;
 		sin2 = (const struct sockaddr_in6 *) ai->ai_addr;
 
-		if (ai->ai_addrlen < sizeof (struct sockaddr_in6)) {
+		if (ai->ai_addrlen < (int)sizeof (struct sockaddr_in6)) {
 			rprintf(FLOG, "%s: too short sockaddr_in6; length=%d\n",
 				fn, (int)ai->ai_addrlen);
 			return 1;
@@ -481,7 +481,7 @@ static int valid_ipaddr(const char *s)
 
 		for (count = 0; count < 8; count++) {
 			if (!*s)
-				return saw_double_colon && count < 7;
+				return saw_double_colon;
 
 			if (strchr(s, ':') == NULL && strchr(s, '.') != NULL) {
 				if ((!saw_double_colon && count != 6) || (saw_double_colon && count > 6))

clientserver.c

@@ -38,6 +38,7 @@ extern int preserve_xattrs;
 extern int kluge_around_eof;
 extern int daemon_over_rsh;
 extern int munge_symlinks;
+extern int open_noatime;
 extern int sanitize_paths;
 extern int numeric_ids;
 extern int filesfrom_fd;
@@ -54,6 +55,7 @@ extern char *config_file;
 extern char *logfile_format;
 extern char *files_from;
 extern char *tmpdir;
+extern char *early_input_file;
 extern struct chmod_mode_struct *chmod_modes;
 extern filter_rule_list daemon_filter_list;
 #ifdef ICONV_OPTION
@@ -67,8 +69,13 @@ char *auth_user;
 int read_only = 0;
 int module_id = -1;
 int pid_file_fd = -1;
+int early_input_len = 0;
+char *early_input = NULL;
 struct chmod_mode_struct *daemon_chmod_modes;
 
+#define EARLY_INPUT_CMD "#early_input="
+#define EARLY_INPUT_CMDLEN (sizeof EARLY_INPUT_CMD - 1)
+
 /* module_dirlen is the length of the module_dir string when in daemon
  * mode and module_dir is not "/"; otherwise 0.  (Note that a chroot-
  * enabled module can have a non-"/" module_dir these days.) */
@@ -144,14 +151,12 @@ static int exchange_protocols(int f_in, int f_out, char *buf, size_t bufsiz, int
 #else
 	int our_sub = 0;
 #endif
-	char *motd;
 
 	io_printf(f_out, "@RSYNCD: %d.%d\n", protocol_version, our_sub);
-
 	if (!am_client) {
-		motd = lp_motd_file();
+		char *motd = lp_motd_file();
 		if (motd && *motd) {
-			FILE *f = fopen(motd,"r");
+			FILE *f = fopen(motd, "r");
 			while (f && !feof(f)) {
 				int len = fread(buf, 1, bufsiz - 1, f);
 				if (len > 0)
@@ -231,8 +236,7 @@ int start_inband_exchange(int f_in, int f_out, const char *user, int argc, char
 	else
 		modlen = p - *argv;
 
-	if (!(modname = new_array(char, modlen+1+1))) /* room for '/' & '\0' */
-		out_of_memory("start_inband_exchange");
+	modname = new_array(char, modlen+1+1); /* room for '/' & '\0' */
 	strlcpy(modname, *argv, modlen + 1);
 	modname[modlen] = '/';
 	modname[modlen+1] = '\0';
@@ -245,6 +249,36 @@ int start_inband_exchange(int f_in, int f_out, const char *user, int argc, char
 	if (exchange_protocols(f_in, f_out, line, sizeof line, 1) < 0)
 		return -1;
 
+	if (early_input_file) {
+		STRUCT_STAT st;
+		FILE *f = fopen(early_input_file, "rb");
+		if (!f || do_fstat(fileno(f), &st) < 0) {
+			rsyserr(FERROR, errno, "failed to open %s", early_input_file);
+			return -1;
+		}
+		early_input_len = st.st_size;
+		if (early_input_len > (int)sizeof line) {
+			rprintf(FERROR, "%s is > %d bytes.\n", early_input_file, (int)sizeof line);
+			return -1;
+		}
+		if (early_input_len > 0) {
+			io_printf(f_out, EARLY_INPUT_CMD "%d\n", early_input_len);
+			while (early_input_len > 0) {
+				int len;
+				if (feof(f)) {
+					rprintf(FERROR, "Early EOF in %s\n", early_input_file);
+					return -1;
+				}
+				len = fread(line, 1, early_input_len, f);
+				if (len > 0) {
+					write_buf(f_out, line, len);
+					early_input_len -= len;
+				}
+			}
+		}
+		fclose(f);
+	}
+
 	/* set daemon_over_rsh to false since we need to build the
 	 * true set of args passed through the rsh/ssh connection;
 	 * this is a no-op for direct-socket-connection mode */
@@ -394,10 +428,10 @@ void set_env_num(const char *var, long num)
 /* Used for both early exec & pre-xfer exec */
 static pid_t start_pre_exec(const char *cmd, int *arg_fd_ptr, int *error_fd_ptr)
 {
-	int arg_fds[2], error_fds[2], arg_fd, error_fd;
+	int arg_fds[2], error_fds[2], arg_fd;
 	pid_t pid;
 
-	if ((error_fd_ptr && pipe(error_fds) < 0) || (arg_fd_ptr && pipe(arg_fds) < 0) || (pid = fork()) < 0)
+	if ((error_fd_ptr && pipe(error_fds) < 0) || pipe(arg_fds) < 0 || (pid = fork()) < 0)
 		return (pid_t)-1;
 
 	if (pid == 0) {
@@ -406,38 +440,36 @@ static pid_t start_pre_exec(const char *cmd, int *arg_fd_ptr, int *error_fd_ptr)
 
 		if (error_fd_ptr) {
 			close(error_fds[0]);
-			error_fd = error_fds[1];
-			set_blocking(error_fd);
+			set_blocking(error_fds[1]);
 		}
 
-		if (arg_fd_ptr) {
-			close(arg_fds[1]);
-			arg_fd = arg_fds[0];
-			set_blocking(arg_fd);
+		close(arg_fds[1]);
+		arg_fd = arg_fds[0];
+		set_blocking(arg_fd);
 
+		len = read_arg_from_pipe(arg_fd, buf, BIGPATHBUFLEN);
+		if (len <= 0)
+			_exit(1);
+		set_env_str("RSYNC_REQUEST", buf);
+
+		for (j = 0; ; j++) {
+			char *p;
 			len = read_arg_from_pipe(arg_fd, buf, BIGPATHBUFLEN);
-			if (len <= 0)
+			if (len <= 0) {
+				if (!len)
+					break;
 				_exit(1);
-			set_env_str("RSYNC_REQUEST", buf);
-
-			for (j = 0; ; j++) {
-				char *p;
-				len = read_arg_from_pipe(arg_fd, buf, BIGPATHBUFLEN);
-				if (len <= 0) {
-					if (!len)
-						break;
-					_exit(1);
-				}
-				if (asprintf(&p, "RSYNC_ARG%d=%s", j, buf) >= 0)
-					putenv(p);
 			}
-			close(arg_fd);
+			if (asprintf(&p, "RSYNC_ARG%d=%s", j, buf) >= 0)
+				putenv(p);
 		}
 
+		dup2(arg_fd, STDIN_FILENO);
+		close(arg_fd);
+
 		if (error_fd_ptr) {
-			close(STDIN_FILENO);
-			dup2(error_fd, STDOUT_FILENO);
-			close(error_fd);
+			dup2(error_fds[1], STDOUT_FILENO);
+			close(error_fds[1]);
 		}
 
 		status = shell_exec(cmd);
@@ -449,20 +481,18 @@ static pid_t start_pre_exec(const char *cmd, int *arg_fd_ptr, int *error_fd_ptr)
 
 	if (error_fd_ptr) {
 		close(error_fds[1]);
-		error_fd = *error_fd_ptr = error_fds[0];
-		set_blocking(error_fd);
+		*error_fd_ptr = error_fds[0];
+		set_blocking(error_fds[0]);
 	}
 
-	if (arg_fd_ptr) {
-		close(arg_fds[0]);
-		arg_fd = *arg_fd_ptr = arg_fds[1];
-		set_blocking(arg_fd);
-	}
+	close(arg_fds[0]);
+	arg_fd = *arg_fd_ptr = arg_fds[1];
+	set_blocking(arg_fd);
 
 	return pid;
 }
 
-static void write_pre_exec_args(int write_fd, char *request, char **early_argv, char **argv)
+static void write_pre_exec_args(int write_fd, char *request, char **early_argv, char **argv, int am_early)
 {
 	int j = 0;
 
@@ -475,10 +505,15 @@ static void write_pre_exec_args(int write_fd, char *request, char **early_argv,
 			write_buf(write_fd, *early_argv, strlen(*early_argv)+1);
 		j = 1; /* Skip arg0 name in argv. */
 	}
-	for ( ; argv[j]; j++)
-		write_buf(write_fd, argv[j], strlen(argv[j])+1);
+	if (argv) {
+		for ( ; argv[j]; j++)
+			write_buf(write_fd, argv[j], strlen(argv[j])+1);
+	}
 	write_byte(write_fd, 0);
 
+	if (am_early && early_input_len)
+		write_buf(write_fd, early_input, early_input_len);
+
 	close(write_fd);
 }
 
@@ -661,8 +696,8 @@ static int rsync_module(int f_in, int f_out, int i, const char *addr, const char
 
 	module_id = i;
 
-	if (lp_transfer_logging(i) && !logfile_format)
-		logfile_format = lp_log_format(i);
+	if (lp_transfer_logging(module_id) && !logfile_format)
+		logfile_format = lp_log_format(module_id);
 	if (log_format_has(logfile_format, 'i'))
 		logfile_format_has_i = 1;
 	if (logfile_format_has_i || log_format_has(logfile_format, 'o'))
@@ -671,7 +706,7 @@ static int rsync_module(int f_in, int f_out, int i, const char *addr, const char
 	uid = MY_UID();
 	am_root = (uid == 0);
 
-	p = *lp_uid(i) ? lp_uid(i) : am_root ? NOBODY_USER : NULL;
+	p = *lp_uid(module_id) ? lp_uid(module_id) : am_root ? NOBODY_USER : NULL;
 	if (p) {
 		if (!user_to_uid(p, &uid, True)) {
 			rprintf(FLOG, "Invalid uid %s\n", p);
@@ -682,7 +717,7 @@ static int rsync_module(int f_in, int f_out, int i, const char *addr, const char
 	} else
 		set_uid = 0;
 
-	p = *lp_gid(i) ? conf_strtok(lp_gid(i)) : NULL;
+	p = *lp_gid(module_id) ? conf_strtok(lp_gid(module_id)) : NULL;
 	if (p) {
 		/* The "*" gid must be the first item in the list. */
 		if (strcmp(p, "*") == 0) {
@@ -715,7 +750,7 @@ static int rsync_module(int f_in, int f_out, int i, const char *addr, const char
 			return -1;
 	}
 
-	module_dir = lp_path(i);
+	module_dir = lp_path(module_id);
 	if (*module_dir == '\0') {
 		rprintf(FLOG, "No path specified for module %s\n", name);
 		io_printf(f_out, "@ERROR: no path setting.\n");
@@ -752,38 +787,38 @@ static int rsync_module(int f_in, int f_out, int i, const char *addr, const char
 	} else
 		set_filter_dir(module_dir, module_dirlen);
 
-	p = lp_filter(i);
+	p = lp_filter(module_id);
 	parse_filter_str(&daemon_filter_list, p, rule_template(FILTRULE_WORD_SPLIT),
 		XFLG_ABS_IF_SLASH | XFLG_DIR2WILD3);
 
-	p = lp_include_from(i);
+	p = lp_include_from(module_id);
 	parse_filter_file(&daemon_filter_list, p, rule_template(FILTRULE_INCLUDE),
 		XFLG_ABS_IF_SLASH | XFLG_DIR2WILD3 | XFLG_OLD_PREFIXES | XFLG_FATAL_ERRORS);
 
-	p = lp_include(i);
+	p = lp_include(module_id);
 	parse_filter_str(&daemon_filter_list, p,
 		rule_template(FILTRULE_INCLUDE | FILTRULE_WORD_SPLIT),
 		XFLG_ABS_IF_SLASH | XFLG_DIR2WILD3 | XFLG_OLD_PREFIXES);
 
-	p = lp_exclude_from(i);
+	p = lp_exclude_from(module_id);
 	parse_filter_file(&daemon_filter_list, p, rule_template(0),
 		XFLG_ABS_IF_SLASH | XFLG_DIR2WILD3 | XFLG_OLD_PREFIXES | XFLG_FATAL_ERRORS);
 
-	p = lp_exclude(i);
+	p = lp_exclude(module_id);
 	parse_filter_str(&daemon_filter_list, p, rule_template(FILTRULE_WORD_SPLIT),
 		XFLG_ABS_IF_SLASH | XFLG_DIR2WILD3 | XFLG_OLD_PREFIXES);
 
 	log_init(1);
 
 #ifdef HAVE_PUTENV
-	if ((*lp_early_exec(i) || *lp_prexfer_exec(i) || *lp_postxfer_exec(i))
+	if ((*lp_early_exec(module_id) || *lp_prexfer_exec(module_id) || *lp_postxfer_exec(module_id))
 	 && !getenv("RSYNC_NO_XFER_EXEC")) {
 		set_env_num("RSYNC_PID", (long)getpid());
 
 		/* For post-xfer exec, fork a new process to run the rsync
 		 * daemon while this process waits for the exit status and
 		 * runs the indicated command at that point. */
-		if (*lp_postxfer_exec(i)) {
+		if (*lp_postxfer_exec(module_id)) {
 			pid_t pid = fork();
 			if (pid < 0) {
 				rsyserr(FLOG, errno, "fork failed");
@@ -803,7 +838,7 @@ static int rsync_module(int f_in, int f_out, int i, const char *addr, const char
 				else
 					status = -1;
 				set_env_num("RSYNC_EXIT_STATUS", status);
-				if (shell_exec(lp_postxfer_exec(i)) < 0)
+				if (shell_exec(lp_postxfer_exec(module_id)) < 0)
 					status = -1;
 				_exit(status);
 			}
@@ -811,13 +846,15 @@ static int rsync_module(int f_in, int f_out, int i, const char *addr, const char
 
 		/* For early exec, fork a child process to run the indicated
 		 * command and wait for it to exit. */
-		if (*lp_early_exec(i)) {
-			pid_t pid = start_pre_exec(lp_early_exec(i), NULL, NULL);
+		if (*lp_early_exec(module_id)) {
+			int arg_fd;
+			pid_t pid = start_pre_exec(lp_early_exec(module_id), &arg_fd, NULL);
 			if (pid == (pid_t)-1) {
 				rsyserr(FLOG, errno, "early exec preparation failed");
 				io_printf(f_out, "@ERROR: early exec preparation failed\n");
 				return -1;
 			}
+			write_pre_exec_args(arg_fd, NULL, NULL, NULL, 1);
 			if (finish_pre_exec("early exec", pid, -1) != NULL) {
 				rsyserr(FLOG, errno, "early exec failed");
 				io_printf(f_out, "@ERROR: early exec failed\n");
@@ -828,8 +865,8 @@ static int rsync_module(int f_in, int f_out, int i, const char *addr, const char
 		/* For pre-xfer exec, fork a child process to run the indicated
 		 * command, though it first waits for the parent process to
 		 * send us the user's request via a pipe. */
-		if (*lp_prexfer_exec(i)) {
-			pre_exec_pid = start_pre_exec(lp_prexfer_exec(i), &pre_exec_arg_fd, &pre_exec_error_fd);
+		if (*lp_prexfer_exec(module_id)) {
+			pre_exec_pid = start_pre_exec(lp_prexfer_exec(module_id), &pre_exec_arg_fd, &pre_exec_error_fd);
 			if (pre_exec_pid == (pid_t)-1) {
 				rsyserr(FLOG, errno, "pre-xfer exec preparation failed");
 				io_printf(f_out, "@ERROR: pre-xfer exec preparation failed\n");
@@ -839,6 +876,11 @@ static int rsync_module(int f_in, int f_out, int i, const char *addr, const char
 	}
 #endif
 
+	if (early_input) {
+		free(early_input);
+		early_input = NULL;
+	}
+
 	if (use_chroot) {
 		/*
 		 * XXX: The 'use chroot' flag is a fairly reliable
@@ -865,7 +907,7 @@ static int rsync_module(int f_in, int f_out, int i, const char *addr, const char
 	if (module_dirlen || (!use_chroot && !*lp_daemon_chroot()))
 		sanitize_paths = 1;
 
-	if ((munge_symlinks = lp_munge_symlinks(i)) < 0)
+	if ((munge_symlinks = lp_munge_symlinks(module_id)) < 0)
 		munge_symlinks = !use_chroot || module_dirlen;
 	if (munge_symlinks) {
 		STRUCT_STAT st;
@@ -920,8 +962,8 @@ static int rsync_module(int f_in, int f_out, int i, const char *addr, const char
 		am_root = (our_uid == 0);
 	}
 
-	if (lp_temp_dir(i) && *lp_temp_dir(i)) {
-		tmpdir = lp_temp_dir(i);
+	if (lp_temp_dir(module_id) && *lp_temp_dir(module_id)) {
+		tmpdir = lp_temp_dir(module_id);
 		if (strlen(tmpdir) >= MAXPATHLEN - 10) {
 			rprintf(FLOG,
 				"the 'temp dir' value for %s is WAY too long -- ignoring.\n",
@@ -948,12 +990,17 @@ static int rsync_module(int f_in, int f_out, int i, const char *addr, const char
 	} else
 		orig_early_argv = NULL;
 
+	/* The default is to use the user's setting unless the module sets True or False. */
+	if (lp_open_noatime(module_id) >= 0)
+		open_noatime = lp_open_noatime(module_id);
+
 	munge_symlinks = save_munge_symlinks; /* The client mustn't control this. */
+
 	if (am_daemon > 0)
 		msgs2stderr = 0; /* A non-rsh-run daemon doesn't have stderr for msgs. */
 
 	if (pre_exec_pid) {
-		write_pre_exec_args(pre_exec_arg_fd, request, orig_early_argv, orig_argv);
+		write_pre_exec_args(pre_exec_arg_fd, request, orig_early_argv, orig_argv, 0);
 		err_msg = finish_pre_exec("pre-xfer exec", pre_exec_pid, pre_exec_error_fd);
 	}
 
@@ -967,7 +1014,7 @@ static int rsync_module(int f_in, int f_out, int i, const char *addr, const char
 	if (write_batch < 0)
 		dry_run = 1;
 
-	if (lp_fake_super(i)) {
+	if (lp_fake_super(module_id)) {
 		if (preserve_xattrs > 1)
 			preserve_xattrs = 1;
 		am_root = -1;
@@ -992,7 +1039,7 @@ static int rsync_module(int f_in, int f_out, int i, const char *addr, const char
 
 #ifndef DEBUG
 	/* don't allow the logs to be flooded too fast */
-	limit_output_verbosity(lp_max_verbosity(i));
+	limit_output_verbosity(lp_max_verbosity(module_id));
 #endif
 
 	if (protocol_version < 23 && (protocol_version == 22 || am_sender))
@@ -1053,20 +1100,20 @@ static int rsync_module(int f_in, int f_out, int i, const char *addr, const char
 #endif
 
 	if (!numeric_ids
-	 && (use_chroot ? lp_numeric_ids(i) != False : lp_numeric_ids(i) == True))
+	 && (use_chroot ? lp_numeric_ids(module_id) != False : lp_numeric_ids(module_id) == True))
 		numeric_ids = -1; /* Set --numeric-ids w/o breaking protocol. */
 
-	if (lp_timeout(i) && (!io_timeout || lp_timeout(i) < io_timeout))
-		set_io_timeout(lp_timeout(i));
+	if (lp_timeout(module_id) && (!io_timeout || lp_timeout(module_id) < io_timeout))
+		set_io_timeout(lp_timeout(module_id));
 
 	/* If we have some incoming/outgoing chmod changes, append them to
 	 * any user-specified changes (making our changes have priority).
 	 * We also get a pointer to just our changes so that a receiver
 	 * process can use them separately if --perms wasn't specified. */
 	if (am_sender)
-		p = lp_outgoing_chmod(i);
+		p = lp_outgoing_chmod(module_id);
 	else
-		p = lp_incoming_chmod(i);
+		p = lp_incoming_chmod(module_id);
 	if (*p && !(daemon_chmod_modes = parse_chmod(p, &chmod_modes))) {
 		rprintf(FLOG, "Invalid \"%sing chmod\" directive: %s\n",
 			am_sender ? "outgo" : "incom", p);
@@ -1130,7 +1177,7 @@ int start_daemon(int f_in, int f_out)
 	if (!load_config(0))
 		exit_cleanup(RERR_SYNTAX);
 
-	if (lp_haproxy_header() && !read_haproxy_header(f_in))
+	if (lp_proxy_protocol() && !read_proxy_protocol_header(f_in))
 		return -1;
 
 	p = lp_daemon_chroot();
@@ -1185,6 +1232,19 @@ int start_daemon(int f_in, int f_out)
 	if (!read_line_old(f_in, line, sizeof line, 0))
 		return -1;
 
+	if (strncmp(line, EARLY_INPUT_CMD, EARLY_INPUT_CMDLEN) == 0) {
+		early_input_len = strtol(line + EARLY_INPUT_CMDLEN, NULL, 10);
+		if (early_input_len <= 0 || early_input_len > BIGPATHBUFLEN) {
+			io_printf(f_out, "@ERROR: invalid early_input length\n");
+			return -1;
+		}
+		early_input = new_array(char, early_input_len);
+		read_buf(f_in, early_input, early_input_len);
+
+		if (!read_line_old(f_in, line, sizeof line, 0))
+			return -1;
+	}
+
 	if (!*line || strcmp(line, "#list") == 0) {
 		rprintf(FLOG, "module-list request from %s (%s)\n",
 			host, addr);

cmdormsg

@@ -0,0 +1,11 @@
+#!/bin/sh
+
+srcdir=`dirname $0`
+opt="$1"
+shift
+
+echo "$*"
+if ! "${@}"; then
+    echo "If you can't fix the issue, re-run $srcdir/configure with --$opt."
+    exit 1
+fi

compat.c

@@ -20,6 +20,7 @@
  */
 
 #include "rsync.h"
+#include "itypes.h"
 
 extern int am_server;
 extern int am_sender;
@@ -173,6 +174,8 @@ void parse_compress_choice(int final_call)
 			exit_cleanup(RERR_UNSUPPORTED);
 		}
 		do_compression = nni->num;
+		if (am_server)
+			validate_choice_vs_env(NSTR_COMPRESS, do_compression, -1);
 	} else if (do_compression)
 		do_compression = CPRES_ZLIB;
 	else
@@ -241,8 +244,7 @@ static void init_nno_saw(struct name_num_obj *nno, int val)
 	}
 
 	if (!nno->saw) {
-		if (!(nno->saw = new_array0(uchar, nno->saw_len)))
-			out_of_memory("init_nno_saw");
+		nno->saw = new_array0(uchar, nno->saw_len);
 
 		/* We'll take this opportunity to make sure that the main_name values are set right. */
 		for (cnt = 1, nni = nno->list; nni->name; nni++, cnt++) {
@@ -261,10 +263,14 @@ static void init_nno_saw(struct name_num_obj *nno, int val)
 static int parse_nni_str(struct name_num_obj *nno, const char *from, char *tobuf, int tobuf_len)
 {
 	char *to = tobuf, *tok = NULL;
-	int cnt = 0;
+	int saw_tok = 0, cnt = 0;
 
 	while (1) {
-		if (*from == ' ' || !*from) {
+		int at_space = isSpace(from);
+		char ch = *from++;
+		if (ch == '&')
+			ch = '\0';
+		if (!ch || at_space) {
 			if (tok) {
 				struct name_num_item *nni = get_nni_by_name(nno, tok, to - tok);
 				if (nni && !nno->saw[nni->num]) {
@@ -278,9 +284,10 @@ static int parse_nni_str(struct name_num_obj *nno, const char *from, char *tobuf
 					}
 				} else
 					to = tok - (tok != tobuf);
+				saw_tok = 1;
 				tok = NULL;
 			}
-			if (!*from++)
+			if (!ch)
 				break;
 			continue;
 		}
@@ -293,13 +300,19 @@ static int parse_nni_str(struct name_num_obj *nno, const char *from, char *tobuf
 			to = tok - (tok != tobuf);
 			break;
 		}
-		*to++ = *from++;
+		*to++ = ch;
 	}
 	*to = '\0';
 
+	if (saw_tok && to == tobuf)
+		return strlcpy(tobuf, "INVALID", MAX_NSTR_STRLEN);
+
 	return to - tobuf;
 }
 
+/* This routine is always called with a tmpbuf of MAX_NSTR_STRLEN length, but the
+ * buffer may be pre-populated with a "len" length string to use OR a len of -1
+ * to tell us to read a string from the fd. */
 static void recv_negotiate_str(int f_in, struct name_num_obj *nno, char *tmpbuf, int len)
 {
 	struct name_num_item *ret = NULL;
@@ -315,17 +328,26 @@ static void recv_negotiate_str(int f_in, struct name_num_obj *nno, char *tmpbuf,
 	}
 
 	if (len > 0) {
+		struct name_num_item *nni;
 		int best = nno->saw_len; /* We want best == 1 from the client list, so start with a big number. */
-		char *tok;
-		if (am_server)
-			init_nno_saw(nno, 1); /* Since we're parsing client names, anything we parse first is #1. */
-		for (tok = strtok(tmpbuf, " \t"); tok; tok = strtok(NULL, " \t")) {
-			struct name_num_item *nni = get_nni_by_name(nno, tok, -1);
+		char *space, *tok = tmpbuf;
+		while (tok) {
+			while (*tok == ' ') tok++; /* Should be unneeded... */
+			if (!*tok)
+				break;
+			if ((space = strchr(tok, ' ')) != NULL)
+				*space = '\0';
+			nni = get_nni_by_name(nno, tok, -1);
+			if (space) {
+				*space = ' ';
+				tok = space + 1;
+			} else
+				tok = NULL;
 			if (!nni || !nno->saw[nni->num] || best <= nno->saw[nni->num])
 				continue;
 			ret = nni;
 			best = nno->saw[nni->num];
-			if (best == 1)
+			if (best == 1 || am_server) /* The server side stops at the first acceptable client choice */
 				break;
 		}
 		if (ret) {
@@ -337,15 +359,84 @@ static void recv_negotiate_str(int f_in, struct name_num_obj *nno, char *tmpbuf,
 		}
 	}
 
-	if (!am_server)
-		rprintf(FERROR, "Failed to negotiate a common %s\n", nno->type);
+	if (!am_server || !do_negotiated_strings) {
+		char *cp = tmpbuf;
+		int j;
+		rprintf(FERROR, "Failed to negotiate a %s choice.\n", nno->type);
+		rprintf(FERROR, "%s list: %s\n", am_server ? "Client" : "Server", tmpbuf);
+		/* Recreate our original list from the saw values. This can't overflow our huge
+		 * buffer because we don't have enough valid entries to get anywhere close. */
+		for (j = 1, *cp = '\0'; j <= nno->saw_len; j++) {
+			struct name_num_item *nni;
+			for (nni = nno->list; nni->name; nni++) {
+				if (nno->saw[nni->num] == j) {
+					*cp++ = ' ';
+					cp += strlcpy(cp, nni->name, MAX_NSTR_STRLEN - (cp - tmpbuf));
+					break;
+				}
+			}
+		}
+		if (!*tmpbuf)
+			strlcpy(cp, " INVALID", MAX_NSTR_STRLEN);
+		rprintf(FERROR, "%s list:%s\n", am_server ? "Server" : "Client", tmpbuf);
+	}
+
 	exit_cleanup(RERR_UNSUPPORTED);
 }
 
+static const char *getenv_nstr(int ntype)
+{
+	const char *env_str = getenv(ntype == NSTR_COMPRESS ? "RSYNC_COMPRESS_LIST" : "RSYNC_CHECKSUM_LIST");
+
+	/* When writing a batch file, we always negotiate an old-style choice. */
+	if (write_batch) 
+		env_str = ntype == NSTR_COMPRESS ? "zlib" : protocol_version >= 30 ? "md5" : "md4";
+
+	if (am_server && env_str) {
+		char *cp = strchr(env_str, '&');
+		if (cp)
+			env_str = cp + 1;
+	}
+
+	return env_str;
+}
+
+void validate_choice_vs_env(int ntype, int num1, int num2)
+{
+	struct name_num_obj *nno = ntype == NSTR_COMPRESS ? &valid_compressions : &valid_checksums;
+	const char *list_str = getenv_nstr(ntype);
+	char tmpbuf[MAX_NSTR_STRLEN];
+
+	if (!list_str)
+		return;
+
+	while (isSpace(list_str)) list_str++;
+
+	if (!*list_str)
+		return;
+
+	init_nno_saw(nno, 0);
+	parse_nni_str(nno, list_str, tmpbuf, MAX_NSTR_STRLEN);
+
+	if (ntype == NSTR_CHECKSUM) /* If "md4" is in the env list, all the old MD4 choices are OK too. */
+		nno->saw[CSUM_MD4_ARCHAIC] = nno->saw[CSUM_MD4_BUSTED] = nno->saw[CSUM_MD4_OLD] = nno->saw[CSUM_MD4];
+
+	if (!nno->saw[num1] || (num2 >= 0 && !nno->saw[num2])) {
+		rprintf(FERROR, "Your --%s-choice value (%s) was refused by the server.\n", 
+			ntype == NSTR_COMPRESS ? "compress" : "checksum",
+			ntype == NSTR_COMPRESS ? compress_choice : checksum_choice);
+		exit_cleanup(RERR_UNSUPPORTED);
+	}
+
+	free(nno->saw);
+	nno->saw = NULL;
+}
+
 /* The saw buffer is initialized and used to store ordinal values from 1 to N
  * for the order of the args in the array.  If dup_markup == '\0', duplicates
  * are removed otherwise the char is prefixed to the duplicate term and, if it
- * is an opening paren/bracket/brace, the matching closing char is suffixed. */
+ * is an opening paren/bracket/brace, the matching closing char is suffixed.
+ * "none" is removed on the client side unless dup_markup != '\0'. */
 int get_default_nno_list(struct name_num_obj *nno, char *to_buf, int to_buf_len, char dup_markup)
 {
 	struct name_num_item *nni;
@@ -367,6 +458,8 @@ int get_default_nno_list(struct name_num_obj *nno, char *to_buf, int to_buf_len,
 				continue;
 			delim = dup_markup;
 		}
+		if (nni->num == 0 && !am_server && !dup_markup)
+			continue;
 		if (len)
 			to_buf[len++]= ' ';
 		if (delim) {
@@ -386,25 +479,15 @@ int get_default_nno_list(struct name_num_obj *nno, char *to_buf, int to_buf_len,
 	return len;
 }
 
-static void send_negotiate_str(int f_out, struct name_num_obj *nno, const char *env_name)
+static void send_negotiate_str(int f_out, struct name_num_obj *nno, int ntype)
 {
 	char tmpbuf[MAX_NSTR_STRLEN];
-	const char *list_str = getenv(env_name);
-	int len, fail_if_empty = list_str && strstr(list_str, "FAIL");
+	const char *list_str = getenv_nstr(ntype);
+	int len;
 
-	if (!do_negotiated_strings) {
-		if (!am_server && fail_if_empty) {
-			rprintf(FERROR, "Remote rsync is too old for %s negotation\n", nno->type);
-			exit_cleanup(RERR_UNSUPPORTED);
-		}
-		return;
-	}
-
-	if (list_str && *list_str && (!am_server || local_server)) {
+	if (list_str && *list_str) {
 		init_nno_saw(nno, 0);
 		len = parse_nni_str(nno, list_str, tmpbuf, MAX_NSTR_STRLEN);
-		if (fail_if_empty && !len)
-			len = strlcpy(tmpbuf, "FAIL", MAX_NSTR_STRLEN);
 		list_str = tmpbuf;
 	} else
 		list_str = NULL;
@@ -419,15 +502,10 @@ static void send_negotiate_str(int f_out, struct name_num_obj *nno, const char *
 			rprintf(FINFO, "Client %s list (on client): %s\n", nno->type, tmpbuf);
 	}
 
-	if (local_server) {
-		/* A local server doesn't bother to send/recv the strings, it just constructs
-		 * and parses the same string on both sides. */
-		recv_negotiate_str(-1, nno, tmpbuf, len);
-	} else {
-		/* Each side sends their list of valid names to the other side and then both sides
-		 * pick the first name in the client's list that is also in the server's list. */
+	/* Each side sends their list of valid names to the other side and then both sides
+	 * pick the first name in the client's list that is also in the server's list. */
+	if (do_negotiated_strings)
 		write_vstring(f_out, tmpbuf, len);
-	}
 }
 
 static void negotiate_the_strings(int f_in, int f_out)
@@ -435,20 +513,35 @@ static void negotiate_the_strings(int f_in, int f_out)
 	/* We send all the negotiation strings before we start to read them to help avoid a slow startup. */
 
 	if (!checksum_choice)
-		send_negotiate_str(f_out, &valid_checksums, "RSYNC_CHECKSUM_LIST");
+		send_negotiate_str(f_out, &valid_checksums, NSTR_CHECKSUM);
 
 	if (do_compression && !compress_choice)
-		send_negotiate_str(f_out, &valid_compressions, "RSYNC_COMPRESS_LIST");
+		send_negotiate_str(f_out, &valid_compressions, NSTR_COMPRESS);
 
 	if (valid_checksums.saw) {
 		char tmpbuf[MAX_NSTR_STRLEN];
-		recv_negotiate_str(f_in, &valid_checksums, tmpbuf, -1);
+		int len;
+		if (do_negotiated_strings)
+			len = -1;
+		else
+			len = strlcpy(tmpbuf, protocol_version >= 30 ? "md5" : "md4", MAX_NSTR_STRLEN);
+		recv_negotiate_str(f_in, &valid_checksums, tmpbuf, len);
 	}
 
 	if (valid_compressions.saw) {
 		char tmpbuf[MAX_NSTR_STRLEN];
-		recv_negotiate_str(f_in, &valid_compressions, tmpbuf, -1);
+		int len;
+		if (do_negotiated_strings)
+			len = -1;
+		else
+			len = strlcpy(tmpbuf, "zlib", MAX_NSTR_STRLEN);
+		recv_negotiate_str(f_in, &valid_compressions, tmpbuf, len);
 	}
+
+	/* If the other side is too old to negotiate, the above steps just made sure that
+	 * the env didn't disallow the old algorithm. Mark things as non-negotiated. */
+	if (!do_negotiated_strings)
+		valid_checksums.negotiated_name = valid_compressions.negotiated_name = NULL;
 }
 
 void setup_protocol(int f_out,int f_in)
@@ -602,10 +695,8 @@ void setup_protocol(int f_out,int f_in)
 			if (local_server || strchr(client_info, 'I') != NULL)
 				compat_flags |= CF_INPLACE_PARTIAL_DIR;
 			if (local_server || strchr(client_info, 'v') != NULL) {
-				if (!write_batch || protocol_version >= 30) {
-					do_negotiated_strings = 1;
-					compat_flags |= CF_VARINT_FLIST_FLAGS;
-				}
+				do_negotiated_strings = 1;
+				compat_flags |= CF_VARINT_FLIST_FLAGS;
 			}
 			if (strchr(client_info, 'V') != NULL) { /* Support a pre-release 'V' that got superseded */
 				if (!write_batch)
@@ -654,6 +745,9 @@ void setup_protocol(int f_out,int f_in)
 #endif
 	}
 
+	if (read_batch)
+		do_negotiated_strings = 0;
+
 	if (need_unsorted_flist && (!am_sender || inc_recurse))
 		unsort_ndx = ++file_extra_cnt;
 

configure

@@ -4,24 +4,24 @@
 # then transfer control to the configure.sh script to do the real work.
 
 dir=`dirname $0`
-realconfigure="$dir/configure.sh"
+if test x"$dir" = x; then
+    dir=.
+fi
 
-if test ! -f "$realconfigure"; then
-    if test -f "$HOME/build_farm/build_test.fns"; then
-	# Test the included popt
-	set -- --with-included-popt "${@}"
-	# Allow the build farm to grab latest files via rsync.
-	actions='build fetch'
-    else
-	actions='build'
+if test "$dir" = '.'; then
+    branch=`packaging/prep-auto-dir` || exit 1
+    if test x"$branch" != x; then
+	cd build || exit 1
+	dir=..
     fi
-    if "$dir/prepare-source" $actions; then
-	:
-    else
+fi
+
+if test ! -f configure.sh; then
+    if ! "$dir/prepare-source" build; then
 	echo 'Failed to build configure.sh and/or config.h.in -- giving up.' >&2
-	rm -f "$realconfigure"
+	rm -f configure.sh
 	exit 1
     fi
 fi
 
-exec "$realconfigure" "${@}"
+exec ./configure.sh --srcdir="$dir" "${@}"

configure.ac

@@ -1,16 +1,15 @@
 dnl Process this file with autoconf to produce a configure script.
 
-AC_INIT([rsync],[3.2.0pre1],[http://rsync.samba.org/bugzilla.html])
+AC_INIT([rsync],[ ],[https://rsync.samba.org/bugtracking.html])
 
 AC_CONFIG_MACRO_DIR([m4])
 AC_CONFIG_SRCDIR([byteorder.h])
 AC_CONFIG_HEADER(config.h)
 AC_PREREQ([2.69])
 
-AC_SUBST(RSYNC_VERSION, $PACKAGE_VERSION)
-AC_MSG_NOTICE([Configuring rsync $PACKAGE_VERSION])
+PACKAGE_VERSION=`sed 's/.*"\(.*\)".*/\1/' <$srcdir/version.h`
 
-AC_DEFINE_UNQUOTED(RSYNC_VERSION, ["$PACKAGE_VERSION"], [rsync release version])
+AC_MSG_NOTICE([Configuring rsync $PACKAGE_VERSION])
 
 LDFLAGS=${LDFLAGS-""}
 
@@ -42,12 +41,14 @@ dnl Checks for programs.
 AC_PROG_CC
 AC_PROG_CPP
 AC_PROG_CXX
+AC_PROG_AWK
 AC_PROG_EGREP
 AC_PROG_INSTALL
 AC_PROG_MKDIR_P
 AC_PROG_CC_STDC
 AC_SUBST(SHELL)
 AC_PATH_PROG([PERL], [perl])
+AC_PATH_PROG([PYTHON3], [python3])
 
 AC_DEFINE([_GNU_SOURCE], 1,
           [Define _GNU_SOURCE so that we get all necessary prototypes])
@@ -62,6 +63,40 @@ if test x"$enable_profile" = x"yes"; then
 	CFLAGS="$CFLAGS -pg"
 fi
 
+AC_MSG_CHECKING([if md2man can create man pages])
+if test x"$ac_cv_path_PYTHON3" = x; then
+    AC_MSG_RESULT(no - python3 not found)
+    md2man_works=no
+else
+    md2man_out=`"$srcdir/md2man" --test "$srcdir/rsync-ssl.1.md" 2>&1`
+    if test $? = 0; then
+	AC_MSG_RESULT(yes)
+	md2man_works=yes
+    else
+	AC_MSG_RESULT(no)
+	md2man_works=no
+	echo "$md2man_out"
+    fi
+fi
+
+AC_MSG_CHECKING([if we require man-page building])
+AC_ARG_ENABLE([md2man],
+	AS_HELP_STRING([--disable-md2man],[disable md2man for man page creation]))
+if test x"$enable_md2man" != x"no"; then
+    if test -f "$srcdir/rsync.1"; then
+	AC_MSG_RESULT(optional)
+    else
+	AC_MSG_RESULT(required)
+	if test x"$md2man_works" = x"no"; then
+	    AC_MSG_ERROR(You need python3 and the cmarkgfm OR commonmark python3 lib in order to build man pages.
+You can specify --disable-md2man if you want to skip building them.)
+	fi
+    fi
+    MAKE_MAN=man
+else
+    AC_MSG_RESULT(no)
+    MAKE_MAN=''
+fi
 
 # Specifically, this turns on panic_action handling.
 AC_ARG_ENABLE(maintainer-mode,
@@ -70,7 +105,6 @@ if test x"$enable_maintainer_mode" = x"yes"; then
 	CFLAGS="$CFLAGS -DMAINTAINER_MODE"
 fi
 
-
 # This is needed for our included version of popt.  Kind of silly, but
 # I don't want our version too far out of sync.
 CFLAGS="$CFLAGS -DHAVE_CONFIG_H"
@@ -163,31 +197,104 @@ SIMD=
 
 AC_MSG_CHECKING([whether to enable SIMD optimizations])
 AC_ARG_ENABLE(simd,
-    AS_HELP_STRING([--disable-simd],[disable SIMD optimizations (requires g++)]))
+    AS_HELP_STRING([--disable-simd],[disable SIMD optimizations (requires c++)]))
+
+# Clag is crashing with -g -O2, so we'll get rid of -g for now.
+CXXFLAGS=`echo "$CXXFLAGS" | sed 's/-g //'`
 
 if test x"$enable_simd" != x"no"; then
-    # For x86-64 SIMD, g++ is also required
-    if test x"$build_cpu" = x"x86_64" && test x"$CXX" = x"g++"; then
-        SIMD="$SIMD x86_64"
+    # For x86-64 SIMD, g++ >=5 or clang++ >=7 is required
+    if test x"$build_cpu" = x"x86_64"; then
+	AC_LANG(C++)
+	AC_RUN_IFELSE([AC_LANG_PROGRAM([[#include <stdio.h>
+#include <immintrin.h>
+__attribute__ ((target("default"))) int test_ssse3(int x) { return x; }
+__attribute__ ((target("default"))) int test_sse2(int x) { return x; }
+__attribute__ ((target("default"))) int test_avx2(int x) { return x; }
+__attribute__ ((target("ssse3"))) int test_ssse3(int x) { return x; }
+__attribute__ ((target("sse2"))) int test_sse2(int x) { return x; }
+__attribute__ ((target("avx2"))) int test_avx2(int x) { return x; }
+typedef long long __m128i_u __attribute__((__vector_size__(16), __may_alias__, __aligned__(1)));
+typedef long long __m256i_u __attribute__((__vector_size__(32), __may_alias__, __aligned__(1)));
+__attribute__ ((target("default"))) void more_testing(char* buf, int len) { }
+__attribute__ ((target("ssse3"))) void more_testing(char* buf, int len)
+{
+    int i;
+    for (i = 0; i < (len-32); i+=32) {
+	__m128i in8_1, in8_2;
+	in8_1 = _mm_lddqu_si128((__m128i_u*)&buf[i]);
+	in8_2 = _mm_lddqu_si128((__m128i_u*)&buf[i + 16]);
+    }
+}
+]], [[if (test_ssse3(42) != 42 || test_sse2(42) != 42 || test_avx2(42) != 42) exit(1);]])],[CXX_OK=yes],[CXX_OK=no])
+	AC_LANG(C)
+	if test x"$CXX_OK" = x"yes"; then
+	    # AC_MSG_RESULT() is called below.
+	    SIMD="x86_64"
+	elif test x"$enable_simd" = x"yes"; then
+	    AC_MSG_RESULT(error)
+	    AC_MSG_ERROR(The SIMD compilation test failed.
+Omit --enable-simd to continue without it.)
+	fi
+    elif test x"$enable_simd" = x"yes"; then
+        AC_MSG_RESULT(unavailable)
+        AC_MSG_ERROR(The SIMD optimizations are currently x86_64 only.
+Omit --enable-simd to continue without it.)
     fi
 fi
 
 if test x"$SIMD" != x""; then
-    SIMD=`echo "$SIMD" | sed -e 's/^ *//'`
     AC_MSG_RESULT([yes ($SIMD)])
-	AC_DEFINE(HAVE_SIMD, 1, [Define to 1 to enable SIMD optimizations])
-    SIMD=`echo "$SIMD" | sed -e 's/[[^ ]]\+/$(SIMD_&)/g'`
+    AC_DEFINE(HAVE_SIMD, 1, [Define to 1 to enable SIMD optimizations])
+    SIMD='$(SIMD_'"$SIMD)"
+    # We only use c++ for its target attribute dispatching, disable unneeded bulky features
+    CXXFLAGS="$CXXFLAGS -fno-exceptions -fno-rtti"
+    # Apple often has "g++" as a symlink for clang. Try to find out the truth.
+    CXX_VERSION=`$CXX --version 2>/dev/null | head -n 2`
+    case "$CXX_VERSION" in
+    *clang*) CXXFLAGS="$CXXFLAGS -fno-slp-vectorize" ;; # avoid a performance hit
+    esac
 else
     AC_MSG_RESULT(no)
 fi
 
 AC_SUBST(SIMD)
 
-# We only use g++ for its target attribute dispatching, disable unneeded bulky features
-if test x"$CXXOBJ" != x""; then
-    CXXFLAGS="$CXXFLAGS -fno-exceptions -fno-rtti"
+AC_MSG_CHECKING([if assembler accepts noexecstack])
+OLD_CFLAGS="$CFLAGS"
+CFLAGS="$CFLAGS -Wa,--noexecstack"
+AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[ ]], [[return 0;]])],
+[ NOEXECSTACK='-Wa,--noexecstack' ; AC_MSG_RESULT(yes) ],
+[ NOEXECSTACK='' ; AC_MSG_RESULT(no) ])
+CFLAGS="$OLD_CFLAGS"
+AC_SUBST(NOEXECSTACK)
+
+ASM=
+
+AC_MSG_CHECKING([whether to enable ASM optimizations])
+AC_ARG_ENABLE(asm,
+    AS_HELP_STRING([--disable-asm],[disable ASM optimizations]))
+
+if test x"$enable_asm" != x"no"; then
+    if test x"$build_cpu" = x"x86_64"; then
+	ASM="$build_cpu"
+    elif test x"$enable_asm" = x"yes"; then
+        AC_MSG_RESULT(unavailable)
+        AC_MSG_ERROR(The ASM optimizations are currently x86_64 only.
+Omit --enable-asm to continue without it.)
+    fi
 fi
 
+if test x"$ASM" != x""; then
+    AC_MSG_RESULT([yes ($ASM)])
+    AC_DEFINE(HAVE_ASM, 1, [Define to 1 to enable ASM optimizations])
+    ASM='$(ASM_'"$ASM)"
+else
+    AC_MSG_RESULT(no)
+fi
+
+AC_SUBST(ASM)
+
 # arrgh. libc in some old debian version screwed up the largefile
 # stuff, getting byte range locking wrong
 AC_CACHE_CHECK([for broken largefile support],rsync_cv_HAVE_BROKEN_LARGEFILE,[
@@ -382,9 +489,18 @@ AC_ARG_ENABLE([openssl],
 	AS_HELP_STRING([--disable-openssl],[disable openssl crypto library]))
 AH_TEMPLATE([USE_OPENSSL],
 [Undefine if you do not want to use openssl crypto library.  By default this is defined.])
-if test x"$enable_openssl" != x"no" && test x"$ac_cv_header_openssl_md4_h" = x"yes" && test x"$ac_cv_header_openssl_md5_h" = x"yes"; then
-    AC_MSG_RESULT(yes)
-    AC_SEARCH_LIBS(MD5_Init, crypto, [AC_DEFINE(USE_OPENSSL)])
+if test x"$enable_openssl" != x"no"; then
+    if test x"$ac_cv_header_openssl_md4_h" = x"yes" && test x"$ac_cv_header_openssl_md5_h" = x"yes"; then
+      AC_MSG_RESULT(yes)
+      AC_SEARCH_LIBS(MD5_Init, crypto,
+          [AC_DEFINE(USE_OPENSSL)],
+          [AC_MSG_ERROR(Failed to find MD5_Init function in openssl crypto lib.
+Use --disable-openssl to continue without openssl crypto lib support.)])
+    else
+        AC_MSG_RESULT(no)
+        AC_MSG_ERROR(Failed to find openssl/md4.h and openssl/md5.h for openssl crypto lib support.
+Use --disable-openssl to continue without it.)
+    fi
 else
     AC_MSG_RESULT(no)
 fi
@@ -394,9 +510,18 @@ AC_ARG_ENABLE([xxhash],
 	AS_HELP_STRING([--disable-xxhash],[disable xxhash checksums]))
 AH_TEMPLATE([SUPPORT_XXHASH],
 [Undefine if you do not want xxhash checksums.  By default this is defined.])
-if test x"$enable_xxhash" != x"no" && test x"$ac_cv_header_xxhash_h" = x"yes"; then
-    AC_MSG_RESULT(yes)
-    AC_SEARCH_LIBS(XXH64_createState, xxhash, [AC_DEFINE(SUPPORT_XXHASH)])
+if test x"$enable_xxhash" != x"no"; then
+    if test x"$ac_cv_header_xxhash_h" = x"yes"; then
+	AC_MSG_RESULT(yes)
+	AC_SEARCH_LIBS(XXH64_createState, xxhash,
+	    [AC_DEFINE(SUPPORT_XXHASH)],
+	    [AC_MSG_ERROR(Failed to find XXH64_createState function in xxhash lib.
+Use --disable-xxhash to continue without xxhash checksums.)])
+    else
+	AC_MSG_RESULT(no)
+	AC_MSG_ERROR(Failed to find xxhash.h for xxhash checksum support.
+Use --disable-xxhash to continue without it.)
+    fi
 else
     AC_MSG_RESULT(no)
 fi
@@ -406,9 +531,18 @@ AC_ARG_ENABLE([zstd],
         AC_HELP_STRING([--disable-zstd], [disable zstd compression]))
 AH_TEMPLATE([SUPPORT_ZSTD],
 [Undefine if you do not want zstd compression.  By default this is defined.])
-if test x"$enable_zstd" != x"no" && test x"$ac_cv_header_zstd_h" = x"yes"; then
-    AC_MSG_RESULT(yes)
-    AC_SEARCH_LIBS(ZSTD_minCLevel, zstd, [AC_DEFINE(SUPPORT_ZSTD)])
+if test x"$enable_zstd" != x"no"; then
+    if test x"$ac_cv_header_zstd_h" = x"yes"; then
+	AC_MSG_RESULT(yes)
+	AC_SEARCH_LIBS(ZSTD_minCLevel, zstd,
+	    [AC_DEFINE(SUPPORT_ZSTD)],
+	    [AC_MSG_ERROR(Failed to find ZSTD_minCLevel function in zstd lib.
+Use --disable-zstd to continue without zstd compression.)])
+    else
+	AC_MSG_RESULT(no)
+	AC_MSG_ERROR(Failed to find zstd.h for zstd compression support.
+Use --disable-zstd to continue without it.)
+    fi
 else
     AC_MSG_RESULT(no)
 fi
@@ -418,9 +552,18 @@ AC_ARG_ENABLE([lz4],
         AC_HELP_STRING([--disable-lz4], [disable LZ4 compression]))
 AH_TEMPLATE([SUPPORT_LZ4],
 [Undefine if you do not want LZ4 compression.  By default this is defined.])
-if test x"$enable_lz4" != x"no" && test x"$ac_cv_header_lz4_h" = x"yes"; then
-    AC_MSG_RESULT(yes)
-    AC_SEARCH_LIBS(LZ4_compress_default, lz4, [AC_DEFINE(SUPPORT_LZ4)])
+if test x"$enable_lz4" != x"no"; then
+    if test x"$ac_cv_header_lz4_h" = x"yes"; then
+	AC_MSG_RESULT(yes)
+	AC_SEARCH_LIBS(LZ4_compress_default, lz4,
+	    [AC_DEFINE(SUPPORT_LZ4)],
+	    [AC_MSG_ERROR(Failed to find LZ4_compress_default function in lz4 lib.
+Use --disable-lz4 to continue without lz4 compression.)])
+    else
+	AC_MSG_RESULT(no)
+	AC_MSG_ERROR(Failed to find lz4.h for lz4 compression support.
+Use --disable-lz4 to continue without it.)
+    fi
 else
     AC_MSG_RESULT(no)
 fi
@@ -556,7 +699,7 @@ size_t iconv();
 #endif
 ]], [[]])],[am_cv_proto_iconv_arg1=""],[am_cv_proto_iconv_arg1="const"])
       am_cv_proto_iconv="extern size_t iconv (iconv_t cd, $am_cv_proto_iconv_arg1 char * *inbuf, size_t *inbytesleft, char * *outbuf, size_t *outbytesleft);"])
-    am_cv_proto_iconv=`echo "[$]am_cv_proto_iconv" | tr -s ' ' | sed -e 's/( /(/'`
+    am_cv_proto_iconv=`echo "[$]am_cv_proto_iconv" | tr -s ' ' | sed 's/( /(/'`
 AC_MSG_RESULT([$]{ac_t:-
          }[$]am_cv_proto_iconv)
 AC_DEFINE_UNQUOTED(ICONV_CONST, $am_cv_proto_iconv_arg1,
@@ -763,7 +906,7 @@ AC_ARG_ENABLE(iconv,
     AS_HELP_STRING([--disable-iconv],[disable rsync's --iconv option]),
     [], [enable_iconv=$enable_iconv_open])
 AH_TEMPLATE([ICONV_OPTION],
-[Define if you want the --iconv option.  Specifing a value will set the
+[Define if you want the --iconv option.  Specifying a value will set the
 default iconv setting (a NULL means no --iconv processing by default).])
 if test x"$enable_iconv" != x"no"; then
 	if test x"$enable_iconv" = x"yes"; then
@@ -1194,5 +1337,5 @@ AC_CONFIG_FILES([Makefile lib/dummy zlib/dummy popt/dummy shconfig])
 AC_OUTPUT
 
 AC_MSG_RESULT()
-AC_MSG_RESULT([    rsync ${RSYNC_VERSION} configuration successful])
+AC_MSG_RESULT([    rsync $PACKAGE_VERSION configuration successful])
 AC_MSG_RESULT()

daemon-parm.awk

@@ -0,0 +1,114 @@
+#!/usr/bin/awk -f
+
+# The caller must pass arg: daemon-parm.txt
+# The resulting code is output into daemon-parm.h
+
+BEGIN {
+    heading = "/* DO NOT EDIT THIS FILE!  It is auto-generated from a list of values in " ARGV[1] "! */\n\n"
+    sect = psect = defines = accessors = prior_ptype = ""
+    parms = "\nstatic struct parm_struct parm_table[] = {"
+    comment_fmt = "\n/********** %s **********/\n"
+    tdstruct = "typedef struct {"
+}
+
+/^\s*$/ { next }
+/^#/ { next }
+
+/^Globals:/ {
+    if (defines != "") {
+	print "The Globals section must come first!"
+	defines = ""
+	exit
+    }
+    defines = tdstruct
+    values = "\nstatic const all_vars Defaults = {\n    { /* Globals: */\n"
+    exps = exp_values = sprintf(comment_fmt, "EXP")
+    sect = "GLOBAL"
+    psect = ", P_GLOBAL, &Vars.g."
+    next
+}
+
+/^Locals:/ {
+    if (sect == "") {
+	print "The Locals section must come after the Globals!"
+	exit
+    }
+    defines = defines exps "} global_vars;\n\n" tdstruct
+    values = values exp_values "\n    }, { /* Locals: */\n"
+    exps = exp_values = sprintf(comment_fmt, "EXP")
+    sect = "LOCAL"
+    psect = ", P_LOCAL, &Vars.l."
+    next
+}
+
+/^(STRING|CHAR|PATH|INTEGER|ENUM|OCTAL|BOOL|BOOLREV|BOOL3)[ \t]/ {
+    ptype = $1
+    name = $2
+    $1 = $2 = ""
+    sub(/^[ \t]+/, "")
+
+    if (ptype != prior_ptype) {
+	comment = sprintf(comment_fmt, ptype)
+	defines = defines comment
+	values = values comment
+	parms = parms "\n"
+	accessors = accessors "\n"
+	prior_ptype = ptype
+    }
+
+    if (ptype == "STRING" || ptype == "PATH") {
+	atype = "STRING"
+	vtype = "char*"
+    } else if (ptype ~ /BOOL/) {
+	atype = vtype = "BOOL"
+    } else if (ptype == "CHAR") {
+	atype = "CHAR"
+	vtype = "char"
+    } else {
+	atype = "INTEGER"
+	vtype = "int"
+    }
+
+    # The name might be var_name|public_name
+    pubname = name
+    sub(/\|.*/, "", name)
+    sub(/.*\|/, "", pubname)
+    gsub(/_/, " ", pubname)
+    gsub(/-/, "", name)
+
+    if (ptype == "ENUM")
+	enum = "enum_" name
+    else
+	enum = "NULL"
+
+    defines = defines "\t" vtype " " name ";\n"
+    values = values "\t" $0 ", /* " name " */\n"
+    parms = parms " {\"" pubname "\", P_" ptype psect name ", " enum ", 0},\n"
+    accessors = accessors "FN_" sect "_" atype "(lp_" name ", " name ")\n"
+
+    if (vtype == "char*") {
+	exps = exps "\tBOOL " name "_EXP;\n"
+	exp_values = exp_values "\tFalse, /* " name "_EXP */\n"
+    }
+
+    next
+}
+
+/./ {
+    print "Extraneous line:" $0
+    defines = ""
+    exit
+}
+
+END {
+    if (sect != "" && defines != "") {
+	defines = defines exps "} local_vars;\n\n"
+	defines = defines tdstruct "\n\tglobal_vars g;\n\tlocal_vars l;\n} all_vars;\n"
+	values = values exp_values "\n    }\n};\n\nstatic all_vars Vars;\n"
+	parms = parms "\n {NULL, P_BOOL, P_NONE, NULL, NULL, 0}\n};\n"
+	print heading defines values parms accessors > "daemon-parm.h"
+    } else {
+	print "Failed to parse the data in " ARGV[1]
+	exit 1
+    }
+}

daemon-parm.txt

@@ -0,0 +1,67 @@
+Globals: ================================================================
+
+STRING	bind_address|address	NULL
+STRING	daemon_chroot		NULL
+STRING	daemon_gid		NULL
+STRING	daemon_uid		NULL
+STRING	motd_file		NULL
+STRING	pid_file		NULL
+STRING	socket_options		NULL
+
+INTEGER	listen_backlog		5
+INTEGER	rsync_port|port		0
+
+BOOL	proxy_protocol		False
+
+Locals: =================================================================
+
+STRING	auth_users		NULL
+STRING	charset			NULL
+STRING	comment			NULL
+STRING	dont_compress		DEFAULT_DONT_COMPRESS
+STRING	early_exec		NULL
+STRING	exclude			NULL
+STRING	exclude_from		NULL
+STRING	filter			NULL
+STRING	gid			NULL
+STRING	hosts_allow		NULL
+STRING	hosts_deny		NULL
+STRING	include			NULL
+STRING	include_from		NULL
+STRING	incoming_chmod		NULL
+STRING	lock_file		DEFAULT_LOCK_FILE
+STRING	log_file		NULL
+STRING	log_format		"%o %h [%a] %m (%u) %f %l"
+STRING	name			NULL
+STRING	outgoing_chmod		NULL
+STRING	post-xfer_exec		NULL
+STRING	pre-xfer_exec		NULL
+STRING	refuse_options		NULL
+STRING	secrets_file		NULL
+STRING	syslog_tag		"rsyncd"
+STRING	uid			NULL
+
+PATH	path			NULL
+PATH	temp_dir		NULL
+
+INTEGER	max_connections		0
+INTEGER	max_verbosity		1
+INTEGER	timeout			0
+
+ENUM	syslog_facility		LOG_DAEMON
+
+BOOL	fake_super		False
+BOOL	forward_lookup		True
+BOOL	ignore_errors		False
+BOOL	ignore_nonreadable	False
+BOOL	list			True
+BOOL	read_only		True
+BOOL	reverse_lookup		True
+BOOL	strict_modes		True
+BOOL	transfer_logging	False
+BOOL	use_chroot		True
+BOOL	write_only		False
+
+BOOL3	munge_symlinks		Unset
+BOOL3	numeric_ids		Unset
+BOOL3	open_noatime		Unset

define-from-md.awk

@@ -0,0 +1,41 @@
+#!/usr/bin/awk -f
+
+# The caller must pass args: -v hfile=NAME rsync.1.md
+
+BEGIN {
+    heading = "/* DO NOT EDIT THIS FILE!  It is auto-generated from a list of values in " ARGV[1] "! */"
+    if (hfile ~ /compress/) {
+	define = "#define DEFAULT_DONT_COMPRESS"
+	prefix = "*."
+    } else {
+	define = "#define DEFAULT_CVSIGNORE"
+	prefix = ""
+    }
+    value_list = ""
+}
+
+/^    > [^ ]+$/ {
+    gsub(/`/, "")
+    if (value_list != "") value_list = value_list " "
+    value_list = value_list prefix $2
+    next
+}
+
+value_list ~ /\.gz / && hfile ~ /compress/ {
+    exit
+}
+
+value_list ~ /SCCS / && hfile ~ /cvsignore/ {
+    exit
+}
+
+value_list = ""
+
+END {
+    if (value_list != "")
+	print heading "\n\n" define " \"" value_list "\"" > hfile
+    else {
+	print "Failed to find a value list in " ARGV[1] " for " hfile
+	exit 1
+    }
+}

exclude.c

@@ -21,6 +21,8 @@
  */
 
 #include "rsync.h"
+#include "default-cvsignore.h"
+#include "ifuncs.h"
 
 extern int am_server;
 extern int am_sender;
@@ -199,8 +201,7 @@ static void add_rule(filter_rule_list *listp, const char *pat, unsigned int pat_
 	} else
 		suf_len = 0;
 
-	if (!(rule->pattern = new_array(char, pre_len + pat_len + suf_len + 1)))
-		out_of_memory("add_rule");
+	rule->pattern = new_array(char, pre_len + pat_len + suf_len + 1);
 	if (pre_len) {
 		memcpy(rule->pattern, dirbuf + module_dirlen, pre_len);
 		for (cp = rule->pattern; cp < rule->pattern + pre_len; cp++) {
@@ -261,19 +262,14 @@ static void add_rule(filter_rule_list *listp, const char *pat, unsigned int pat_
 			}
 		}
 
-		if (!(lp = new_array0(filter_rule_list, 1)))
-			out_of_memory("add_rule");
+		lp = new_array0(filter_rule_list, 1);
 		if (asprintf(&lp->debug_type, " [per-dir %s]", cp) < 0)
 			out_of_memory("add_rule");
 		rule->u.mergelist = lp;
 
 		if (mergelist_cnt == mergelist_size) {
 			mergelist_size += 5;
-			mergelist_parents = realloc_array(mergelist_parents,
-						filter_rule *,
-						mergelist_size);
-			if (!mergelist_parents)
-				out_of_memory("add_rule");
+			mergelist_parents = realloc_array(mergelist_parents, filter_rule *, mergelist_size);
 		}
 		if (DEBUG_GTE(FILTER, 2)) {
 			rprintf(FINFO, "[%s] activating mergelist #%d%s\n",
@@ -497,8 +493,6 @@ void *push_local_filters(const char *dir, unsigned int dirlen)
 	push = (struct local_filter_state *)new_array(char,
 			  sizeof (struct local_filter_state)
 			+ (mergelist_cnt-1) * sizeof (filter_rule_list));
-	if (!push)
-		out_of_memory("push_local_filters");
 
 	push->mergelist_cnt = mergelist_cnt;
 	for (i = 0; i < mergelist_cnt; i++) {
@@ -821,8 +815,7 @@ static filter_rule *parse_rule_tok(const char **rulestr_ptr,
 	if (!*s)
 		return NULL;
 
-	if (!(rule = new0(filter_rule)))
-		out_of_memory("parse_rule_tok");
+	rule = new0(filter_rule);
 
 	/* Inherit from the template.  Don't inherit FILTRULES_SIDES; we check
 	 * that later. */
@@ -1051,16 +1044,6 @@ static filter_rule *parse_rule_tok(const char **rulestr_ptr,
 	return rule;
 }
 
-static char default_cvsignore[] =
-	/* These default ignored items come from the CVS manual. */
-	"RCS SCCS CVS CVS.adm RCSLOG cvslog.* tags TAGS"
-	" .make.state .nse_depinfo *~ #* .#* ,* _$* *$"
-	" *.old *.bak *.BAK *.orig *.rej .del-*"
-	" *.a *.olb *.o *.obj *.so *.exe"
-	" *.Z *.elc *.ln core"
-	/* The rest we added to suit ourself. */
-	" .svn/ .git/ .hg/ .bzr/";
-
 static void get_cvs_excludes(uint32 rflags)
 {
 	static int initialized = 0;
@@ -1070,7 +1053,7 @@ static void get_cvs_excludes(uint32 rflags)
 		return;
 	initialized = 1;
 
-	parse_filter_str(&cvs_filter_list, default_cvsignore,
+	parse_filter_str(&cvs_filter_list, DEFAULT_CVSIGNORE,
 			 rule_template(rflags | (protocol_version >= 30 ? FILTRULE_PERISHABLE : 0)),
 			 0);
 
@@ -1134,8 +1117,7 @@ void parse_filter_str(filter_rule_list *listp, const char *rulestr,
 				const char *name;
 				filter_rule *excl_self;
 
-				if (!(excl_self = new0(filter_rule)))
-					out_of_memory("parse_filter_str");
+				excl_self = new0(filter_rule);
 				/* Find the beginning of the basename and add an exclude for it. */
 				for (name = pat + pat_len; name > pat && name[-1] != '/'; name--) {}
 				add_rule(listp, name, (pat + pat_len) - name, excl_self, 0);

fileio.c

@@ -157,8 +157,6 @@ int write_file(int f, int use_seek, OFF_T offset, const char *buf, int len)
 				wf_writeBufSize = WRITE_SIZE * 8;
 				wf_writeBufCnt  = 0;
 				wf_writeBuf = new_array(char, wf_writeBufSize);
-				if (!wf_writeBuf)
-					out_of_memory("write_file");
 			}
 			r1 = (int)MIN((size_t)len, wf_writeBufSize - wf_writeBufCnt);
 			if (r1) {
@@ -217,8 +215,7 @@ struct map_struct *map_file(int fd, OFF_T len, int32 read_size, int32 blk_size)
 {
 	struct map_struct *map;
 
-	if (!(map = new0(struct map_struct)))
-		out_of_memory("map_file");
+	map = new0(struct map_struct);
 
 	if (blk_size && (read_size % blk_size))
 		read_size += blk_size - (read_size % blk_size);
@@ -261,8 +258,6 @@ char *map_ptr(struct map_struct *map, OFF_T offset, int32 len)
 	/* make sure we have allocated enough memory for the window */
 	if (window_size > map->p_size) {
 		map->p = realloc_array(map->p, char, window_size);
-		if (!map->p)
-			out_of_memory("map_ptr");
 		map->p_size = window_size;
 	}
 

flist.c

@@ -301,8 +301,7 @@ static void flist_expand(struct file_list *flist, int extra)
 	if (flist->malloced < flist->used + extra)
 		flist->malloced = flist->used + extra;
 
-	new_ptr = realloc_array(flist->files, struct file_struct *,
-				flist->malloced);
+	new_ptr = realloc_array(flist->files, struct file_struct *, flist->malloced);
 
 	if (DEBUG_GTE(FLIST, 1) && flist->malloced != FLIST_START) {
 		rprintf(FCLIENT, "[%s] expand file_list pointer array to %s bytes, did%s move\n",
@@ -312,9 +311,6 @@ static void flist_expand(struct file_list *flist, int extra)
 	}
 
 	flist->files = new_ptr;
-
-	if (!flist->files)
-		out_of_memory("flist_expand");
 }
 
 static void flist_done_allocating(struct file_list *flist)
@@ -1335,10 +1331,8 @@ struct file_struct *make_file(const char *fname, struct file_list *flist,
 		  + linkname_len;
 	if (pool)
 		bp = pool_alloc(pool, alloc_len, "make_file");
-	else {
-		if (!(bp = new_array(char, alloc_len)))
-			out_of_memory("make_file");
-	}
+	else
+		bp = new_array(char, alloc_len);
 
 	memset(bp, 0, extra_len + FILE_STRUCT_LEN);
 	bp += extra_len;
@@ -1661,8 +1655,7 @@ static void fsort(struct file_struct **fp, size_t num)
 	if (use_qsort)
 		qsort(fp, num, PTR_SIZE, file_compare);
 	else {
-		struct file_struct **tmp = new_array(struct file_struct *,
-						     (num+1) / 2);
+		struct file_struct **tmp = new_array(struct file_struct *, (num+1) / 2);
 		fsort_tmp(fp, num, tmp);
 		free(tmp);
 	}
@@ -1895,13 +1888,11 @@ static void send_implied_dirs(int f, struct file_list *flist, char *fname,
 	len = strlen(limit+1);
 	memcpy(&relname_list, F_DIR_RELNAMES_P(lastpath_struct), sizeof relname_list);
 	if (!relname_list) {
-		if (!(relname_list = new0(item_list)))
-			out_of_memory("send_implied_dirs");
+		relname_list = new0(item_list);
 		memcpy(F_DIR_RELNAMES_P(lastpath_struct), &relname_list, sizeof relname_list);
 	}
 	rnpp = EXPAND_ITEM_LIST(relname_list, relnamecache *, 32);
-	if (!(*rnpp = (relnamecache*)new_array(char, sizeof (relnamecache) + len)))
-		out_of_memory("send_implied_dirs");
+	*rnpp = (relnamecache*)new_array(char, sizeof (relnamecache) + len);
 	(*rnpp)->name_type = name_type;
 	strlcpy((*rnpp)->fname, limit+1, len + 1);
 
@@ -2059,8 +2050,7 @@ void send_extra_file_list(int f, int at_least)
 		}
 
 		if (need_unsorted_flist) {
-			if (!(flist->sorted = new_array(struct file_struct *, flist->used)))
-				out_of_memory("send_extra_file_list");
+			flist->sorted = new_array(struct file_struct *, flist->used);
 			memcpy(flist->sorted, flist->files,
 			       flist->used * sizeof (struct file_struct*));
 		} else
@@ -2414,8 +2404,7 @@ struct file_list *send_file_list(int f, int argc, char *argv[])
 	 * recursion mode, the sender marks duplicate dirs so that it can
 	 * send them together in a single file-list. */
 	if (need_unsorted_flist) {
-		if (!(flist->sorted = new_array(struct file_struct *, flist->used)))
-			out_of_memory("send_file_list");
+		flist->sorted = new_array(struct file_struct *, flist->used);
 		memcpy(flist->sorted, flist->files,
 		       flist->used * sizeof (struct file_struct*));
 	} else
@@ -2597,8 +2586,7 @@ struct file_list *recv_file_list(int f, int dir_ndx)
 		 * order and for calling flist_find()).  We keep the "files"
 		 * list unsorted for our exchange of index numbers with the
 		 * other side (since their names may not sort the same). */
-		if (!(flist->sorted = new_array(struct file_struct *, flist->used)))
-			out_of_memory("recv_file_list");
+		flist->sorted = new_array(struct file_struct *, flist->used);
 		memcpy(flist->sorted, flist->files,
 		       flist->used * sizeof (struct file_struct*));
 		if (inc_recurse && dir_flist->used > dstart) {
@@ -2808,8 +2796,7 @@ struct file_list *flist_new(int flags, char *msg)
 {
 	struct file_list *flist;
 
-	if (!(flist = new0(struct file_list)))
-		out_of_memory(msg);
+	flist = new0(struct file_list);
 
 	if (flags & FLIST_TEMP) {
 		if (!(flist->file_pool = pool_create(SMALL_EXTENT, 0,

generator.c

@@ -2227,8 +2227,6 @@ void generate_files(int f_out, const char *local_name)
 	if (delete_during == 2) {
 		deldelay_size = BIGPATHBUFLEN * 4;
 		deldelay_buf = new_array(char, deldelay_size);
-		if (!deldelay_buf)
-			out_of_memory("delete-delay");
 	}
 	info_levels[INFO_FLIST] = info_levels[INFO_PROGRESS] = 0;
 

getgroups.c

@@ -3,7 +3,7 @@
  * `id -G` on Linux, but it's too hard to find a portable equivalent.
  *
  * Copyright (C) 2002 Martin Pool
- * Copyright (C) 2003-2019 Wayne Davison
+ * Copyright (C) 2003-2020 Wayne Davison
  *
  * This program is free software; you can redistribute it and/or modify
  * it under the terms of the GNU General Public License version 3 as
@@ -20,8 +20,7 @@
 
 #include "rsync.h"
 
-int
-main(UNUSED(int argc), UNUSED(char *argv[]))
+ int main(UNUSED(int argc), UNUSED(char *argv[]))
 {
 	int n, i;
 	gid_t *list;

hashtable.c

@@ -35,9 +35,8 @@ struct hashtable *hashtable_create(int size, int key64)
 			size *= 2;
 	}
 
-	if (!(tbl = new(struct hashtable))
-	 || !(tbl->nodes = new_array0(char, size * node_size)))
-		out_of_memory("hashtable_create");
+	tbl = new(struct hashtable);
+	tbl->nodes = new_array0(char, size * node_size);
 	tbl->size = size;
 	tbl->entries = 0;
 	tbl->node_size = node_size;
@@ -94,8 +93,7 @@ void *hashtable_find(struct hashtable *tbl, int64 key, void *data_when_new)
 		int size = tbl->size * 2;
 		int i;
 
-		if (!(tbl->nodes = new_array0(char, size * tbl->node_size)))
-			out_of_memory("hashtable_node");
+		tbl->nodes = new_array0(char, size * tbl->node_size);
 		tbl->size = size;
 		tbl->entries = 0;
 

help-from-md.awk

@@ -0,0 +1,40 @@
+#!/usr/bin/awk -f
+
+# The caller must pass args: -v hfile=help-NAME.h NAME.NUM.md
+
+BEGIN {
+    heading = "/* DO NOT EDIT THIS FILE!  It is auto-generated from the option list in " ARGV[1] "! */"
+    findcomment = hfile
+    sub("\\.", "\\.", findcomment)
+    findcomment = "\\[comment\\].*" findcomment
+    backtick_cnt = 0
+    prints = ""
+}
+
+/^```/ {
+    backtick_cnt++
+    next
+}
+
+foundcomment {
+    if (backtick_cnt > 1) exit
+    if (backtick_cnt == 1) {
+	gsub(/"/, "\\\"")
+	prints = prints "\n  rprintf(F,\"" $0 "\\n\");"
+    }
+    next
+}
+
+$0 ~ findcomment {
+    foundcomment = 1
+    backtick_cnt = 0
+}
+
+END {
+    if (foundcomment && backtick_cnt > 1)
+	print heading "\n" prints > hfile
+    else {
+	print "Failed to find " hfile " section in " ARGV[1]
+	exit 1
+    }
+}

hlink.c

@@ -125,8 +125,7 @@ static void match_gnums(int32 *ndx_list, int ndx_count)
 		if (inc_recurse) {
 			node = hashtable_find(prior_hlinks, gnum, data_when_new);
 			if (node->data == data_when_new) {
-				if (!(node->data = new_array0(char, 5)))
-					out_of_memory("match_gnums");
+				node->data = new_array0(char, 5);
 				assert(gnum >= hlink_flist->ndx_start);
 				file->flags |= FLAG_HLINK_FIRST;
 				prev = -1;
@@ -190,8 +189,7 @@ void match_hard_links(struct file_list *flist)
 		int i, ndx_count = 0;
 		int32 *ndx_list;
 
-		if (!(ndx_list = new_array(int32, flist->used)))
-			out_of_memory("match_hard_links");
+		ndx_list = new_array(int32, flist->used);
 
 		for (i = 0; i < flist->used; i++) {
 			if (F_IS_HLINKED(flist->sorted[i]))
@@ -541,8 +539,7 @@ void finish_hard_link(struct file_struct *file, const char *fname, int fin_ndx,
 			exit_cleanup(RERR_MESSAGEIO);
 		}
 		free(node->data);
-		if (!(node->data = strdup(our_name)))
-			out_of_memory("finish_hard_link");
+		node->data = strdup(our_name);
 	}
 }
 

ifuncs.h

@@ -1,6 +1,6 @@
 /* Inline functions for rsync.
  *
- * Copyright (C) 2007-2019 Wayne Davison
+ * Copyright (C) 2007-2020 Wayne Davison
  *
  * This program is free software; you can redistribute it and/or modify
  * it under the terms of the GNU General Public License as published by
@@ -19,8 +19,7 @@
 static inline void
 alloc_xbuf(xbuf *xb, size_t sz)
 {
-	if (!(xb->buf = new_array(char, sz)))
-		out_of_memory("alloc_xbuf");
+	xb->buf = new_array(char, sz);
 	xb->size = sz;
 	xb->len = xb->pos = 0;
 }
@@ -29,8 +28,6 @@ static inline void
 realloc_xbuf(xbuf *xb, size_t sz)
 {
 	char *bf = realloc_array(xb->buf, char, sz);
-	if (!bf)
-		out_of_memory("realloc_xbuf");
 	xb->buf = bf;
 	xb->size = sz;
 }
@@ -104,3 +101,11 @@ free_stat_x(stat_x *sx_p)
     }
 #endif
 }
+
+static inline char *my_strdup(const char *str, const char *file, int line)
+{
+    int len = strlen(str)+1;
+    char *buf = my_alloc(do_malloc, len, 1, file, line);
+    memcpy(buf, str, len);
+    return buf;
+}

io.c

@@ -251,8 +251,7 @@ static size_t safe_read(int fd, char *buf, size_t len)
 		cnt = select(fd+1, &r_fds, NULL, &e_fds, &tv);
 		if (cnt <= 0) {
 			if (cnt < 0 && errno == EBADF) {
-				rsyserr(FERROR, errno, "safe_read select failed [%s]",
-					who_am_i());
+				rsyserr(FERROR, errno, "safe_read select failed");
 				exit_cleanup(RERR_FILEIO);
 			}
 			check_timeout(1, MSK_ALLOW_FLUSH);
@@ -271,8 +270,7 @@ static size_t safe_read(int fd, char *buf, size_t len)
 			if (n < 0) {
 				if (errno == EINTR)
 					continue;
-				rsyserr(FERROR, errno, "safe_read failed to read %ld bytes [%s]",
-					(long)len, who_am_i());
+				rsyserr(FERROR, errno, "safe_read failed to read %ld bytes", (long)len);
 				exit_cleanup(RERR_STREAMIO);
 			}
 			if ((got += (size_t)n) == len)
@@ -315,8 +313,8 @@ static void safe_write(int fd, const char *buf, size_t len)
 		if (errno != EINTR && errno != EWOULDBLOCK && errno != EAGAIN) {
 		  write_failed:
 			rsyserr(FERROR, errno,
-				"safe_write failed to write %ld bytes to %s [%s]",
-				(long)len, what_fd_is(fd), who_am_i());
+				"safe_write failed to write %ld bytes to %s",
+				(long)len, what_fd_is(fd));
 			exit_cleanup(RERR_STREAMIO);
 		}
 	} else {
@@ -337,8 +335,7 @@ static void safe_write(int fd, const char *buf, size_t len)
 		cnt = select(fd + 1, NULL, &w_fds, NULL, &tv);
 		if (cnt <= 0) {
 			if (cnt < 0 && errno == EBADF) {
-				rsyserr(FERROR, errno, "safe_write select failed on %s [%s]",
-					what_fd_is(fd), who_am_i());
+				rsyserr(FERROR, errno, "safe_write select failed on %s", what_fd_is(fd));
 				exit_cleanup(RERR_FILEIO);
 			}
 			if (io_timeout)
@@ -815,7 +812,7 @@ static char *perform_io(size_t needed, int flags)
 					msgs2stderr = 1;
 					iobuf.out_fd = -2;
 					iobuf.out.len = iobuf.msg.len = iobuf.raw_flushing_ends_before = 0;
-					rsyserr(FERROR_SOCKET, errno, "[%s] write error", who_am_i());
+					rsyserr(FERROR_SOCKET, errno, "write error");
 					drain_multiplex_messages();
 					exit_cleanup(RERR_SOCKETIO);
 				}
@@ -1242,8 +1239,7 @@ void read_args(int f_in, char *mod_name, char *buf, size_t bufsiz, int rl_nulls,
 	rl_flags |= (protect_args && ic_recv != (iconv_t)-1 ? RL_CONVERT : 0);
 #endif
 
-	if (!(argv = new_array(char *, maxargs)))
-		out_of_memory("read_args");
+	argv = new_array(char *, maxargs);
 	if (mod_name && !protect_args)
 		argv[argc++] = "rsyncd";
 
@@ -1256,8 +1252,7 @@ void read_args(int f_in, char *mod_name, char *buf, size_t bufsiz, int rl_nulls,
 
 		if (argc == maxargs-1) {
 			maxargs += MAX_ARGS;
-			if (!(argv = realloc_array(argv, char *, maxargs)))
-				out_of_memory("read_args");
+			argv = realloc_array(argv, char *, maxargs);
 		}
 
 		if (dot_pos) {
@@ -1265,8 +1260,7 @@ void read_args(int f_in, char *mod_name, char *buf, size_t bufsiz, int rl_nulls,
 				int len = strlen(buf);
 				if (request_len)
 					request_p[0][request_len++] = ' ';
-				if (!(*request_p = realloc_array(*request_p, char, request_len + len + 1)))
-					out_of_memory("read_args");
+				*request_p = realloc_array(*request_p, char, request_len + len + 1);
 				memcpy(*request_p + request_len, buf, len + 1);
 				request_len += len;
 			}
@@ -1275,8 +1269,7 @@ void read_args(int f_in, char *mod_name, char *buf, size_t bufsiz, int rl_nulls,
 			else
 				glob_expand(buf, &argv, &argc, &maxargs);
 		} else {
-			if (!(p = strdup(buf)))
-				out_of_memory("read_args");
+			p = strdup(buf);
 			argv[argc++] = p;
 			if (*p == '.' && p[1] == '\0')
 				dot_pos = argc;

lib/md-defines.h

@@ -0,0 +1,17 @@
+/* Keep this simple so both C and ASM can use it */
+
+#define MD4_DIGEST_LEN 16
+#define MD5_DIGEST_LEN 16
+#define MAX_DIGEST_LEN MD5_DIGEST_LEN
+
+#define CSUM_CHUNK 64
+
+#define CSUM_NONE 0
+#define CSUM_MD4_ARCHAIC 1
+#define CSUM_MD4_BUSTED 2
+#define CSUM_MD4_OLD 3
+#define CSUM_MD4 4
+#define CSUM_MD5 5
+#define CSUM_XXH64 6
+#define CSUM_XXH3_64 7
+#define CSUM_XXH3_128 8

lib/md5-asm-x86_64.S

@@ -24,11 +24,19 @@
  * show any significant difference in performance, though.
  */
 
+#include "config.h"
+#include "md-defines.h"
+
+#if !defined USE_OPENSSL && CSUM_CHUNK == 64
+
+#ifdef __APPLE__
+#define md5_process_asm _md5_process_asm
+#endif
+
 .text
 .align 16
 
 .globl md5_process_asm
-.type md5_process_asm,@function
 md5_process_asm:
 	push	%rbp
 	push	%rbx
@@ -689,5 +697,5 @@ md5_process_asm:
 	pop	%rbx
 	pop	%rbp
 	ret
-.L_md5_process_asm_end:
-.size md5_process_asm,.L_md5_process_asm_end-md5_process_asm
+
+#endif /* !USE_OPENSSL ... */

lib/md5.c

@@ -148,7 +148,7 @@ static void md5_process(md_context *ctx, const uchar data[CSUM_CHUNK])
 	ctx->D += D;
 }
 
-#if defined(HAVE_SIMD) && (CSUM_CHUNK == 64)
+#if defined HAVE_ASM && CSUM_CHUNK == 64
 extern void md5_process_asm(md_context *ctx, const void *data, size_t num);
 #endif
 
@@ -176,7 +176,7 @@ void md5_update(md_context *ctx, const uchar *input, uint32 length)
 		left = 0;
 	}
 
-#if defined(HAVE_SIMD) && (CSUM_CHUNK == 64)
+#if defined HAVE_ASM && CSUM_CHUNK == 64
 	if (length >= CSUM_CHUNK) {
 		uint32 chunks = length / CSUM_CHUNK;
 		md5_process_asm(ctx, input, chunks);

lib/mdigest.h

@@ -4,20 +4,7 @@
 #include "openssl/md4.h"
 #include "openssl/md5.h"
 #endif
-
-#define MD4_DIGEST_LEN 16
-#define MD5_DIGEST_LEN 16
-#define MAX_DIGEST_LEN MD5_DIGEST_LEN
-
-#define CSUM_CHUNK 64
-
-#define CSUM_NONE 0
-#define CSUM_MD4_ARCHAIC 1
-#define CSUM_MD4_BUSTED 2
-#define CSUM_MD4_OLD 3
-#define CSUM_MD4 4
-#define CSUM_MD5 5
-#define CSUM_XXH64 6
+#include "md-defines.h"
 
 typedef struct {
 	uint32 A, B, C, D;

lib/sysxattrs.h

@@ -1,9 +1,9 @@
 #ifdef SUPPORT_XATTRS
 
-#if defined HAVE_ATTR_XATTR_H
-#include <attr/xattr.h>
-#elif defined HAVE_SYS_XATTR_H
+#if defined HAVE_SYS_XATTR_H
 #include <sys/xattr.h>
+#elif defined HAVE_ATTR_XATTR_H
+#include <attr/xattr.h>
 #elif defined HAVE_SYS_EXTATTR_H
 #include <sys/extattr.h>
 #endif

loadparm.c

@@ -42,24 +42,20 @@
 
 #include "rsync.h"
 #include "itypes.h"
+#include "ifuncs.h"
+#include "default-dont-compress.h"
 
 extern item_list dparam_list;
 
 #define strequal(a, b) (strcasecmp(a, b)==0)
-#define BOOLSTR(b) ((b) ? "Yes" : "No")
 
 #ifndef LOG_DAEMON
 #define LOG_DAEMON 0
 #endif
 
-#define DEFAULT_DONT_COMPRESS "*.gz *.zip *.z *.rpm *.deb *.iso *.bz2" \
-	" *.t[gb]z *.7z *.mp[34] *.mov *.avi *.ogg *.jpg *.jpeg *.png" \
-	" *.lzo *.rzip *.lzma *.rar *.ace *.gpg *.xz *.txz *.lz *.tlz" \
-	" *.ogv *.web[mp] *.squashfs"
-
 /* the following are used by loadparm for option lists */
 typedef enum {
-	P_BOOL, P_BOOLREV, P_CHAR, P_INTEGER,
+	P_BOOL, P_BOOLREV, P_BOOL3, P_CHAR, P_INTEGER,
 	P_OCTAL, P_PATH, P_STRING, P_ENUM
 } parm_type;
 
@@ -90,234 +86,6 @@ struct parm_struct {
 #define LP_SNUM_OK(i) ((i) >= 0 && (i) < (int)section_list.count)
 #define SECTION_PTR(s, p) (((char*)(s)) + (ptrdiff_t)(((char*)(p))-(char*)&Vars.l))
 
-/* This structure describes global (ie., server-wide) parameters. */
-typedef struct {
-	char *bind_address;
-	char *daemon_chroot;
-	char *daemon_gid;
-	char *daemon_uid;
-	char *motd_file;
-	char *pid_file;
-	char *socket_options;
-
-	/* Each _EXP var tracks if the associated char* var has been expanded yet or not. */
-	BOOL bind_address_EXP;
-	BOOL daemon_chroot_EXP;
-	BOOL daemon_gid_EXP;
-	BOOL daemon_uid_EXP;
-	BOOL motd_file_EXP;
-	BOOL pid_file_EXP;
-	BOOL socket_options_EXP;
-
-	int listen_backlog;
-	int rsync_port;
-
-	BOOL haproxy_header;
-} global_vars;
-
-/* This structure describes a single section.  Their order must match the
- * initializers below, which you can accomplish by keeping each sub-section
- * sorted.  (e.g. in vim, just visually select each subsection and use !sort.)
- * NOTE: the char* variables MUST all remain at the start of the struct! */
-typedef struct {
-	char *auth_users;
-	char *charset;
-	char *comment;
-	char *dont_compress;
-	char *early_exec;
-	char *exclude;
-	char *exclude_from;
-	char *filter;
-	char *gid;
-	char *hosts_allow;
-	char *hosts_deny;
-	char *include;
-	char *include_from;
-	char *incoming_chmod;
-	char *lock_file;
-	char *log_file;
-	char *log_format;
-	char *name;
-	char *outgoing_chmod;
-	char *path;
-	char *postxfer_exec;
-	char *prexfer_exec;
-	char *refuse_options;
-	char *secrets_file;
-	char *syslog_tag;
-	char *temp_dir;
-	char *uid;
-
-	/* Each _EXP var tracks if the associated char* var has been expanded yet or not. */
-	BOOL auth_users_EXP;
-	BOOL charset_EXP;
-	BOOL comment_EXP;
-	BOOL dont_compress_EXP;
-	BOOL early_exec_EXP;
-	BOOL exclude_EXP;
-	BOOL exclude_from_EXP;
-	BOOL filter_EXP;
-	BOOL gid_EXP;
-	BOOL hosts_allow_EXP;
-	BOOL hosts_deny_EXP;
-	BOOL include_EXP;
-	BOOL include_from_EXP;
-	BOOL incoming_chmod_EXP;
-	BOOL lock_file_EXP;
-	BOOL log_file_EXP;
-	BOOL log_format_EXP;
-	BOOL name_EXP;
-	BOOL outgoing_chmod_EXP;
-	BOOL path_EXP;
-	BOOL postxfer_exec_EXP;
-	BOOL prexfer_exec_EXP;
-	BOOL refuse_options_EXP;
-	BOOL secrets_file_EXP;
-	BOOL syslog_tag_EXP;
-	BOOL temp_dir_EXP;
-	BOOL uid_EXP;
-
-	int max_connections;
-	int max_verbosity;
-	int syslog_facility;
-	int timeout;
-
-	BOOL fake_super;
-	BOOL forward_lookup;
-	BOOL ignore_errors;
-	BOOL ignore_nonreadable;
-	BOOL list;
-	BOOL munge_symlinks;
-	BOOL numeric_ids;
-	BOOL read_only;
-	BOOL reverse_lookup;
-	BOOL strict_modes;
-	BOOL transfer_logging;
-	BOOL use_chroot;
-	BOOL write_only;
-} local_vars;
-
-/* This structure describes the global variables (g) as well as the globally
- * specified values of the local variables (l), which are used when modules
- * don't specify their own values. */
-typedef struct {
-	global_vars g;
-	local_vars l;
-} all_vars;
-
-/* The application defaults for all the variables.  "Defaults" is
- * used to re-initialize "Vars" before each config-file read.
- *
- * In order to keep these sorted in the same way as the structure
- * above, use the variable name in the leading comment, including a
- * trailing ';' (to avoid a sorting problem with trailing digits). */
-static const all_vars Defaults = {
- /* ==== global_vars ==== */
- {
- /* bind_address; */		NULL,
- /* daemon_chroot; */		NULL,
- /* daemon_gid; */		NULL,
- /* daemon_uid; */		NULL,
- /* motd_file; */		NULL,
- /* pid_file; */		NULL,
- /* socket_options; */		NULL,
-
- /* bind_address_EXP; */	False,
- /* daemon_chroot_EXP; */	False,
- /* daemon_gid_EXP; */		False,
- /* daemon_uid_EXP; */		False,
- /* motd_file_EXP; */		False,
- /* pid_file_EXP; */		False,
- /* socket_options_EXP; */	False,
-
- /* listen_backlog; */		5,
- /* rsync_port; */		0,
-
- /* haproxy_header; */		False,
- },
-
- /* ==== local_vars ==== */
- {
- /* auth_users; */		NULL,
- /* charset; */ 		NULL,
- /* comment; */ 		NULL,
- /* dont_compress; */		DEFAULT_DONT_COMPRESS,
- /* early_exec; */		NULL,
- /* exclude; */ 		NULL,
- /* exclude_from; */		NULL,
- /* filter; */			NULL,
- /* gid; */			NULL,
- /* hosts_allow; */		NULL,
- /* hosts_deny; */		NULL,
- /* include; */			NULL,
- /* include_from; */		NULL,
- /* incoming_chmod; */		NULL,
- /* lock_file; */		DEFAULT_LOCK_FILE,
- /* log_file; */		NULL,
- /* log_format; */		"%o %h [%a] %m (%u) %f %l",
- /* name; */			NULL,
- /* outgoing_chmod; */		NULL,
- /* path; */			NULL,
- /* postxfer_exec; */		NULL,
- /* prexfer_exec; */		NULL,
- /* refuse_options; */		NULL,
- /* secrets_file; */		NULL,
- /* syslog_tag; */		"rsyncd",
- /* temp_dir; */ 		NULL,
- /* uid; */			NULL,
-
- /* auth_users_EXP; */		False,
- /* charset_EXP; */		False,
- /* comment_EXP; */		False,
- /* dont_compress_EXP; */	False,
- /* early_exec_EXP; */		False,
- /* exclude_EXP; */		False,
- /* exclude_from_EXP; */	False,
- /* filter_EXP; */		False,
- /* gid_EXP; */			False,
- /* hosts_allow_EXP; */		False,
- /* hosts_deny_EXP; */		False,
- /* include_EXP; */		False,
- /* include_from_EXP; */	False,
- /* incoming_chmod_EXP; */	False,
- /* lock_file_EXP; */		False,
- /* log_file_EXP; */		False,
- /* log_format_EXP; */		False,
- /* name_EXP; */		False,
- /* outgoing_chmod_EXP; */	False,
- /* path_EXP; */		False,
- /* postxfer_exec_EXP; */	False,
- /* prexfer_exec_EXP; */	False,
- /* refuse_options_EXP; */	False,
- /* secrets_file_EXP; */	False,
- /* syslog_tag_EXP; */		False,
- /* temp_dir_EXP; */		False,
- /* uid_EXP; */			False,
-
- /* max_connections; */		0,
- /* max_verbosity; */		1,
- /* syslog_facility; */		LOG_DAEMON,
- /* timeout; */			0,
-
- /* fake_super; */		False,
- /* forward_lookup; */		True,
- /* ignore_errors; */		False,
- /* ignore_nonreadable; */	False,
- /* list; */			True,
- /* munge_symlinks; */		(BOOL)-1,
- /* numeric_ids; */		(BOOL)-1,
- /* read_only; */		True,
- /* reverse_lookup; */		True,
- /* strict_modes; */		True,
- /* transfer_logging; */	False,
- /* use_chroot; */		True,
- /* write_only; */		False,
- }
-};
-
-/* The currently configured values for all the variables. */
-static all_vars Vars;
-
 /* Stack of "Vars" values used by the &include directive. */
 static item_list Vars_stack = EMPTY_ITEM_LIST;
 
@@ -327,9 +95,7 @@ static item_list section_list = EMPTY_ITEM_LIST;
 static int iSectionIndex = -1;
 static BOOL bInGlobalSection = True;
 
-#define NUMPARAMETERS (sizeof (parm_table) / sizeof (struct parm_struct))
-
-static struct enum_list enum_facilities[] = {
+static struct enum_list enum_syslog_facility[] = {
 #ifdef LOG_AUTH
 	{ LOG_AUTH, "auth" },
 #endif
@@ -396,96 +162,27 @@ static struct enum_list enum_facilities[] = {
 	{ -1, NULL }
 };
 
-static struct parm_struct parm_table[] =
-{
- {"address",           P_STRING, P_GLOBAL,&Vars.g.bind_address,        NULL,0},
- {"daemon chroot",     P_STRING, P_GLOBAL,&Vars.g.daemon_chroot,       NULL,0},
- {"daemon gid",        P_STRING, P_GLOBAL,&Vars.g.daemon_gid,          NULL,0},
- {"daemon uid",        P_STRING, P_GLOBAL,&Vars.g.daemon_uid,          NULL,0},
- {"listen backlog",    P_INTEGER,P_GLOBAL,&Vars.g.listen_backlog,      NULL,0},
- {"motd file",         P_STRING, P_GLOBAL,&Vars.g.motd_file,           NULL,0},
- {"pid file",          P_STRING, P_GLOBAL,&Vars.g.pid_file,            NULL,0},
- {"port",              P_INTEGER,P_GLOBAL,&Vars.g.rsync_port,          NULL,0},
- {"socket options",    P_STRING, P_GLOBAL,&Vars.g.socket_options,      NULL,0},
- {"haproxy header",    P_BOOL,   P_LOCAL, &Vars.g.haproxy_header,      NULL,0},
-
- {"auth users",        P_STRING, P_LOCAL, &Vars.l.auth_users,          NULL,0},
- {"charset",           P_STRING, P_LOCAL, &Vars.l.charset,             NULL,0},
- {"comment",           P_STRING, P_LOCAL, &Vars.l.comment,             NULL,0},
- {"dont compress",     P_STRING, P_LOCAL, &Vars.l.dont_compress,       NULL,0},
- {"early exec",        P_STRING, P_LOCAL, &Vars.l.early_exec,          NULL,0},
- {"exclude from",      P_STRING, P_LOCAL, &Vars.l.exclude_from,        NULL,0},
- {"exclude",           P_STRING, P_LOCAL, &Vars.l.exclude,             NULL,0},
- {"fake super",        P_BOOL,   P_LOCAL, &Vars.l.fake_super,          NULL,0},
- {"filter",            P_STRING, P_LOCAL, &Vars.l.filter,              NULL,0},
- {"forward lookup",    P_BOOL,   P_LOCAL, &Vars.l.forward_lookup,      NULL,0},
- {"gid",               P_STRING, P_LOCAL, &Vars.l.gid,                 NULL,0},
- {"hosts allow",       P_STRING, P_LOCAL, &Vars.l.hosts_allow,         NULL,0},
- {"hosts deny",        P_STRING, P_LOCAL, &Vars.l.hosts_deny,          NULL,0},
- {"ignore errors",     P_BOOL,   P_LOCAL, &Vars.l.ignore_errors,       NULL,0},
- {"ignore nonreadable",P_BOOL,   P_LOCAL, &Vars.l.ignore_nonreadable,  NULL,0},
- {"include from",      P_STRING, P_LOCAL, &Vars.l.include_from,        NULL,0},
- {"include",           P_STRING, P_LOCAL, &Vars.l.include,             NULL,0},
- {"incoming chmod",    P_STRING, P_LOCAL, &Vars.l.incoming_chmod,      NULL,0},
- {"list",              P_BOOL,   P_LOCAL, &Vars.l.list,                NULL,0},
- {"lock file",         P_STRING, P_LOCAL, &Vars.l.lock_file,           NULL,0},
- {"log file",          P_STRING, P_LOCAL, &Vars.l.log_file,            NULL,0},
- {"log format",        P_STRING, P_LOCAL, &Vars.l.log_format,          NULL,0},
- {"max connections",   P_INTEGER,P_LOCAL, &Vars.l.max_connections,     NULL,0},
- {"max verbosity",     P_INTEGER,P_LOCAL, &Vars.l.max_verbosity,       NULL,0},
- {"munge symlinks",    P_BOOL,   P_LOCAL, &Vars.l.munge_symlinks,      NULL,0},
- {"name",              P_STRING, P_LOCAL, &Vars.l.name,                NULL,0},
- {"numeric ids",       P_BOOL,   P_LOCAL, &Vars.l.numeric_ids,         NULL,0},
- {"outgoing chmod",    P_STRING, P_LOCAL, &Vars.l.outgoing_chmod,      NULL,0},
- {"path",              P_PATH,   P_LOCAL, &Vars.l.path,                NULL,0},
-#ifdef HAVE_PUTENV
- {"post-xfer exec",    P_STRING, P_LOCAL, &Vars.l.postxfer_exec,       NULL,0},
- {"pre-xfer exec",     P_STRING, P_LOCAL, &Vars.l.prexfer_exec,        NULL,0},
-#endif
- {"read only",         P_BOOL,   P_LOCAL, &Vars.l.read_only,           NULL,0},
- {"refuse options",    P_STRING, P_LOCAL, &Vars.l.refuse_options,      NULL,0},
- {"reverse lookup",    P_BOOL,   P_LOCAL, &Vars.l.reverse_lookup,      NULL,0},
- {"secrets file",      P_STRING, P_LOCAL, &Vars.l.secrets_file,        NULL,0},
- {"strict modes",      P_BOOL,   P_LOCAL, &Vars.l.strict_modes,        NULL,0},
- {"syslog facility",   P_ENUM,   P_LOCAL, &Vars.l.syslog_facility,     enum_facilities,0},
- {"syslog tag",        P_STRING, P_LOCAL, &Vars.l.syslog_tag,          NULL,0},
- {"temp dir",          P_PATH,   P_LOCAL, &Vars.l.temp_dir,            NULL,0},
- {"timeout",           P_INTEGER,P_LOCAL, &Vars.l.timeout,             NULL,0},
- {"transfer logging",  P_BOOL,   P_LOCAL, &Vars.l.transfer_logging,    NULL,0},
- {"uid",               P_STRING, P_LOCAL, &Vars.l.uid,                 NULL,0},
- {"use chroot",        P_BOOL,   P_LOCAL, &Vars.l.use_chroot,          NULL,0},
- {"write only",        P_BOOL,   P_LOCAL, &Vars.l.write_only,          NULL,0},
- {NULL,                P_BOOL,   P_NONE,  NULL,                        NULL,0}
-};
-
-/* Initialise the Default all_vars structure. */
-void reset_daemon_vars(void)
-{
-	memcpy(&Vars, &Defaults, sizeof Vars);
-}
-
 /* Expand %VAR% references.  Any unknown vars or unrecognized
  * syntax leaves the raw chars unchanged. */
-static char *expand_vars(char *str)
+static char *expand_vars(const char *str)
 {
-	char *buf, *t, *f;
+	char *buf, *t;
+	const char *f;
 	int bufsize;
 
 	if (!str || !strchr(str, '%'))
-		return str;
+		return (char *)str; /* TODO change return value to const char* at some point. */
 
 	bufsize = strlen(str) + 2048;
-	if ((buf = new_array(char, bufsize+1)) == NULL) /* +1 for trailing '\0' */
-		out_of_memory("expand_vars");
+	buf = new_array(char, bufsize+1); /* +1 for trailing '\0' */
 
 	for (t = buf, f = str; bufsize && *f; ) {
-		if (*f == '%' && *++f != '%') {
-			char *percent = strchr(f, '%');
-			if (percent) {
+		if (*f == '%' && isUpper(f+1)) {
+			char *percent = strchr(f+1, '%');
+			if (percent && percent - f < bufsize) {
 				char *val;
-				*percent = '\0';
-				val = getenv(f);
-				*percent = '%';
+				strlcpy(t, f+1, percent - f);
+				val = getenv(t);
 				if (val) {
 					int len = strlcpy(t, val, bufsize+1);
 					if (len > bufsize)
@@ -496,7 +193,6 @@ static char *expand_vars(char *str)
 					continue;
 				}
 			}
-			f--;
 		}
 		*t++ = *f++;
 		bufsize--;
@@ -514,6 +210,8 @@ static char *expand_vars(char *str)
 	return buf;
 }
 
+/* Each "char* foo" has an associated "BOOL foo_EXP" that tracks if the string has been expanded yet or not. */
+
 /* NOTE: use this function and all the FN_{GLOBAL,LOCAL} ones WITHOUT a trailing semicolon! */
 #define RETURN_EXPANDED(val) {if (!val ## _EXP) {val = expand_vars(val); val ## _EXP = True;} return val ? val : "";}
 
@@ -538,65 +236,24 @@ static char *expand_vars(char *str)
 #define FN_LOCAL_INTEGER(fn_name, val) \
  int fn_name(int i) {return LP_SNUM_OK(i)? iSECTION(i).val : Vars.l.val;}
 
-FN_GLOBAL_STRING(lp_bind_address, bind_address)
-FN_GLOBAL_STRING(lp_daemon_chroot, daemon_chroot)
-FN_GLOBAL_STRING(lp_daemon_gid, daemon_gid)
-FN_GLOBAL_STRING(lp_daemon_uid, daemon_uid)
-FN_GLOBAL_STRING(lp_motd_file, motd_file)
-FN_GLOBAL_STRING(lp_pid_file, pid_file)
-FN_GLOBAL_STRING(lp_socket_options, socket_options)
+/* The following include file contains:
+ *
+ * typedef global_vars - describes global (ie., server-wide) parameters.
+ * typedef local_vars - describes a single section.
+ * typedef all_vars - a combination of global_vars & local_vars.
+ * all_vars Defaults - the default values for all the variables.
+ * all_vars Vars - tThe currently configured values for all the variables.
+ * struct parm_struct parm_table - the strings & variables for the parser.
+ * FN_{LOCAL,GLOBAL}_{TYPE}() definition for all the lp_var_name() accessors.
+ */
 
-FN_GLOBAL_INTEGER(lp_listen_backlog, listen_backlog)
-FN_GLOBAL_INTEGER(lp_rsync_port, rsync_port)
+#include "daemon-parm.h"
 
-FN_GLOBAL_BOOL(lp_haproxy_header, haproxy_header)
-
-FN_LOCAL_STRING(lp_auth_users, auth_users)
-FN_LOCAL_STRING(lp_charset, charset)
-FN_LOCAL_STRING(lp_comment, comment)
-FN_LOCAL_STRING(lp_dont_compress, dont_compress)
-FN_LOCAL_STRING(lp_early_exec, early_exec)
-FN_LOCAL_STRING(lp_exclude, exclude)
-FN_LOCAL_STRING(lp_exclude_from, exclude_from)
-FN_LOCAL_STRING(lp_filter, filter)
-FN_LOCAL_STRING(lp_gid, gid)
-FN_LOCAL_STRING(lp_hosts_allow, hosts_allow)
-FN_LOCAL_STRING(lp_hosts_deny, hosts_deny)
-FN_LOCAL_STRING(lp_include, include)
-FN_LOCAL_STRING(lp_include_from, include_from)
-FN_LOCAL_STRING(lp_incoming_chmod, incoming_chmod)
-FN_LOCAL_STRING(lp_lock_file, lock_file)
-FN_LOCAL_STRING(lp_log_file, log_file)
-FN_LOCAL_STRING(lp_log_format, log_format)
-FN_LOCAL_STRING(lp_name, name)
-FN_LOCAL_STRING(lp_outgoing_chmod, outgoing_chmod)
-FN_LOCAL_STRING(lp_path, path)
-FN_LOCAL_STRING(lp_postxfer_exec, postxfer_exec)
-FN_LOCAL_STRING(lp_prexfer_exec, prexfer_exec)
-FN_LOCAL_STRING(lp_refuse_options, refuse_options)
-FN_LOCAL_STRING(lp_secrets_file, secrets_file)
-FN_LOCAL_STRING(lp_syslog_tag, syslog_tag)
-FN_LOCAL_STRING(lp_temp_dir, temp_dir)
-FN_LOCAL_STRING(lp_uid, uid)
-
-FN_LOCAL_INTEGER(lp_max_connections, max_connections)
-FN_LOCAL_INTEGER(lp_max_verbosity, max_verbosity)
-FN_LOCAL_INTEGER(lp_syslog_facility, syslog_facility)
-FN_LOCAL_INTEGER(lp_timeout, timeout)
-
-FN_LOCAL_BOOL(lp_fake_super, fake_super)
-FN_LOCAL_BOOL(lp_forward_lookup, forward_lookup)
-FN_LOCAL_BOOL(lp_ignore_errors, ignore_errors)
-FN_LOCAL_BOOL(lp_ignore_nonreadable, ignore_nonreadable)
-FN_LOCAL_BOOL(lp_list, list)
-FN_LOCAL_BOOL(lp_munge_symlinks, munge_symlinks)
-FN_LOCAL_BOOL(lp_numeric_ids, numeric_ids)
-FN_LOCAL_BOOL(lp_read_only, read_only)
-FN_LOCAL_BOOL(lp_reverse_lookup, reverse_lookup)
-FN_LOCAL_BOOL(lp_strict_modes, strict_modes)
-FN_LOCAL_BOOL(lp_transfer_logging, transfer_logging)
-FN_LOCAL_BOOL(lp_use_chroot, use_chroot)
-FN_LOCAL_BOOL(lp_write_only, write_only)
+/* Initialise the Default all_vars structure. */
+void reset_daemon_vars(void)
+{
+	memcpy(&Vars, &Defaults, sizeof Vars);
+}
 
 /* Assign a copy of v to *s.  Handles NULL strings.  We don't worry
  * about overwriting a malloc'd string because the long-running
@@ -605,10 +262,7 @@ FN_LOCAL_BOOL(lp_write_only, write_only)
  * the start, so any lost memory is inconsequential. */
 static inline void string_set(char **s, const char *v)
 {
-	if (!v)
-		*s = NULL;
-	else if (!(*s = strdup(v)))
-		out_of_memory("string_set");
+	*s = v ? strdup(v) : NULL;
 }
 
 /* Copy local_vars into a new section. No need to strdup since we don't free. */
@@ -624,19 +278,14 @@ static void init_section(local_vars *psection)
 	copy_section(psection, &Vars.l);
 }
 
-/* Do a case-insensitive, whitespace-ignoring string compare. */
-static int strwicmp(char *psz1, char *psz2)
+/* Do a case-insensitive, whitespace-ignoring string equality check. */
+static int strwiEQ(char *psz1, char *psz2)
 {
-	/* if BOTH strings are NULL, return TRUE, if ONE is NULL return */
-	/* appropriate value. */
+	/* If one or both strings are NULL, we return equality right away. */
 	if (psz1 == psz2)
-		return 0;
-
-	if (psz1 == NULL)
-		return -1;
-
-	if (psz2 == NULL)
 		return 1;
+	if (psz1 == NULL || psz2 == NULL)
+		return 0;
 
 	/* sync the strings on first non-whitespace */
 	while (1) {
@@ -644,12 +293,14 @@ static int strwicmp(char *psz1, char *psz2)
 			psz1++;
 		while (isSpace(psz2))
 			psz2++;
-		if (toUpper(psz1) != toUpper(psz2) || *psz1 == '\0' || *psz2 == '\0')
+		if (*psz1 == '\0' || *psz2 == '\0')
+			break;
+		if (toUpper(psz1) != toUpper(psz2))
 			break;
 		psz1++;
 		psz2++;
 	}
-	return *psz1 - *psz2;
+	return *psz1 == *psz2;
 }
 
 /* Find a section by name. Otherwise works like get_section. */
@@ -658,7 +309,7 @@ static int getsectionbyname(char *name)
 	int i;
 
 	for (i = section_list.count - 1; i >= 0; i--) {
-		if (strwicmp(iSECTION(i).name, name) == 0)
+		if (strwiEQ(iSECTION(i).name, name))
 			break;
 	}
 
@@ -698,7 +349,7 @@ static int map_parameter(char *parmname)
 		return -1;
 
 	for (iIndex = 0; parm_table[iIndex].label; iIndex++) {
-		if (strwicmp(parm_table[iIndex].label, parmname) == 0)
+		if (strwiEQ(parm_table[iIndex].label, parmname))
 			return iIndex;
 	}
 
@@ -709,16 +360,14 @@ static int map_parameter(char *parmname)
 /* Set a boolean variable from the text value stored in the passed string.
  * Returns True in success, False if the passed string does not correctly
  * represent a boolean. */
-static BOOL set_boolean(BOOL *pb, char *parmvalue)
+static BOOL set_boolean(BOOL *pb, char *parmvalue, int allow_unset)
 {
-	if (strwicmp(parmvalue, "yes") == 0
-	 || strwicmp(parmvalue, "true") == 0
-	 || strwicmp(parmvalue, "1") == 0)
+	if (strwiEQ(parmvalue, "yes") || strwiEQ(parmvalue, "true") || strwiEQ(parmvalue, "1"))
 		*pb = True;
-	else if (strwicmp(parmvalue, "no") == 0
-	      || strwicmp(parmvalue, "False") == 0
-	      || strwicmp(parmvalue, "0") == 0)
+	else if (strwiEQ(parmvalue, "no") || strwiEQ(parmvalue, "false") || strwiEQ(parmvalue, "0"))
 		*pb = False;
+	else if (allow_unset && (strwiEQ(parmvalue, "unset") || strwiEQ(parmvalue, "-1")))
+		*pb = Unset;
 	else {
 		rprintf(FLOG, "Badly formed boolean in configuration file: \"%s\".\n", parmvalue);
 		return False;
@@ -767,11 +416,15 @@ static BOOL do_parameter(char *parmname, char *parmvalue)
 
 	switch (parm_table[parmnum].type) {
 	case P_BOOL:
-		set_boolean(parm_ptr, parmvalue);
+		set_boolean(parm_ptr, parmvalue, False);
+		break;
+
+	case P_BOOL3:
+		set_boolean(parm_ptr, parmvalue, True);
 		break;
 
 	case P_BOOLREV:
-		set_boolean(parm_ptr, parmvalue);
+		set_boolean(parm_ptr, parmvalue, False);
 		*(BOOL *)parm_ptr = ! *(BOOL *)parm_ptr;
 		break;
 
@@ -841,7 +494,7 @@ static BOOL do_section(char *sectionname)
 		return True;
 	}
 
-	isglobal = strwicmp(sectionname, GLOBAL_NAME) == 0;
+	isglobal = strwiEQ(sectionname, GLOBAL_NAME);
 
 	/* At the end of the global section, add any --dparam items. */
 	if (bInGlobalSection && !isglobal) {

log.c

@@ -318,10 +318,10 @@ void rwrite(enum logcode code, const char *buf, int len, int is_utf8)
 		if (quiet)
 			return;
 		break;
-	//case FLOG:
-	//case FCLIENT:
-	//case FERROR_UTF8:
-	//case FERROR_SOCKET:
+	/*case FLOG:*/
+	/*case FCLIENT:*/
+	/*case FERROR_UTF8:*/
+	/*case FERROR_SOCKET:*/
 	default:
 		fprintf(stderr, "Bad logcode in rwrite(): %d [%s]\n", (int)code, who_am_i());
 		exit_cleanup(RERR_MESSAGEIO);
@@ -452,8 +452,7 @@ void rsyserr(enum logcode code, int errcode, const char *format, ...)
 	char buf[BIGPATHBUFLEN];
 	size_t len;
 
-	strlcpy(buf, RSYNC_NAME ": ", sizeof buf);
-	len = (sizeof RSYNC_NAME ": ") - 1;
+	len = snprintf(buf, sizeof buf, RSYNC_NAME ": [%s] ", who_am_i());
 
 	va_start(ap, format);
 	len += vsnprintf(buf + len, sizeof buf - len, format, ap);

main.c

@@ -22,6 +22,7 @@
 
 #include "rsync.h"
 #include "inums.h"
+#include "ifuncs.h"
 #include "io.h"
 #if defined CONFIG_LOCALE && defined HAVE_LOCALE_H
 #include <locale.h>
@@ -512,8 +513,6 @@ static pid_t do_cmd(char *cmd, char *machine, char *user, char **remote_argv, in
 		if (!cmd)
 			cmd = RSYNC_RSH;
 		cmd = need_to_free = strdup(cmd);
-		if (!cmd)
-			goto oom;
 
 		for (t = f = cmd; *f; f++) {
 			if (*f == ' ')
@@ -657,10 +656,6 @@ static pid_t do_cmd(char *cmd, char *machine, char *user, char **remote_argv, in
 		free(need_to_free);
 
 	return pid;
-
-  oom:
-	out_of_memory("do_cmd");
-	return 0; /* not reached */
 }
 
 /* The receiving side operates in one of two modes:
@@ -824,8 +819,6 @@ static void check_alt_basis_dirs(void)
 		if (dry_run > 1 && *bdir != '/') {
 			int len = curr_dir_len + 1 + bd_len + 1;
 			char *new = new_array(char, len);
-			if (!new)
-				out_of_memory("check_alt_basis_dirs");
 			if (slash && strncmp(bdir, "../", 3) == 0) {
 				/* We want to remove only one leading "../" prefix for
 				 * the directory we couldn't create in dry-run mode:
@@ -1339,19 +1332,12 @@ int client_run(int f_in, int f_out, pid_t pid, int argc, char *argv[])
 	return MAX(exit_code, exit_code2);
 }
 
-static int copy_argv(char *argv[])
+static void dup_argv(char *argv[])
 {
 	int i;
 
-	for (i = 0; argv[i]; i++) {
-		if (!(argv[i] = strdup(argv[i]))) {
-			rprintf (FERROR, "out of memory at %s(%d)\n",
-				 __FILE__, __LINE__);
-			return RERR_MALLOC;
-		}
-	}
-
-	return 0;
+	for (i = 0; argv[i]; i++)
+		argv[i] = strdup(argv[i]);
 }
 
 
@@ -1372,8 +1358,7 @@ static int start_client(int argc, char *argv[])
 
 	/* Don't clobber argv[] so that ps(1) can still show the right
 	 * command line. */
-	if ((ret = copy_argv(argv)) != 0)
-		return ret;
+	dup_argv(argv);
 
 	if (!read_batch) { /* for read_batch, NO source is specified */
 		char *path = check_for_hostspec(argv[0], &shell_machine, &rsync_port);

match.c

@@ -65,8 +65,6 @@ static void build_hash_table(struct sum_struct *s)
 		if (hash_table)
 			free(hash_table);
 		hash_table = new_array(int32, tablesize);
-		if (!hash_table)
-			out_of_memory("build_hash_table");
 		alloc_size = tablesize;
 	}
 

md2man

@@ -1,4 +1,4 @@
-#!/usr/bin/python3
+#!/usr/bin/env python3
 
 # This script takes a manpage written in markdown and turns it into an html web
 # page and a nroff man page.  The input file must have the name of the program
@@ -35,6 +35,7 @@ body, b, strong, u {
 code {
   font-family: 'Roboto Mono', monospace;
   font-weight: bold;
+  white-space: pre;
 }
 pre code {
   display: block;
@@ -64,7 +65,9 @@ MAN_END = """\
 
 NORM_FONT = ('\1', r"\fP")
 BOLD_FONT = ('\2', r"\fB")
-ULIN_FONT = ('\3', r"\fI")
+UNDR_FONT = ('\3', r"\fI")
+NBR_DASH = ('\4', r"\-")
+NBR_SPACE = ('\xa0', r"\ ")
 
 md_parser = None
 
@@ -78,46 +81,55 @@ def main():
         fi.srcdir = './'
 
     fi.title = fi.prog + '(' + fi.sect + ') man page'
-    fi.mtime = None
+    fi.mtime = 0
 
-    if os.path.lexists(fi.srcdir + '.git'):
-        fi.mtime = int(subprocess.check_output('git log -1 --format=%at'.split()))
-
-    chk_files = 'NEWS.md Makefile'.split()
-    for fn in chk_files:
-        try:
-            st = os.lstat(fi.srcdir + fn)
-        except:
-            die('Failed to find', fi.srcdir + fn)
-        if not fi.mtime:
-            fi.mtime = st.st_mtime
-
-    fi.date = time.strftime('%d %b %Y', time.localtime(fi.mtime))
+    git_dir = fi.srcdir + '.git'
+    if os.path.lexists(git_dir):
+        fi.mtime = int(subprocess.check_output(['git', '--git-dir', git_dir, 'log', '-1', '--format=%at']))
 
     env_subs = { 'prefix': os.environ.get('RSYNC_OVERRIDE_PREFIX', None) }
 
-    with open(fi.srcdir + 'Makefile', 'r', encoding='utf-8') as fh:
-        for line in fh:
-            m = re.match(r'^(\w+)=(.+)', line)
-            if not m:
-                continue
-            var, val = (m[1], m[2])
-            if var == 'prefix' and env_subs[var] is not None:
-                continue
-            while re.search(r'\$\{', val):
-                val = re.sub(r'\$\{(\w+)\}', lambda m: env_subs[m[1]], val)
-            env_subs[var] = val
-            if var == 'VERSION':
-                break
+    if args.test:
+        env_subs['VERSION'] = '1.0.0'
+        env_subs['libdir'] = '/usr'
+    else:
+        for fn in (fi.srcdir + 'version.h', 'Makefile'):
+            try:
+                st = os.lstat(fn)
+            except:
+                die('Failed to find', fi.srcdir + fn)
+            if not fi.mtime:
+                fi.mtime = st.st_mtime
+
+        with open(fi.srcdir + 'version.h', 'r', encoding='utf-8') as fh:
+            txt = fh.read()
+        m = re.search(r'"(.+?)"', txt)
+        env_subs['VERSION'] = m.group(1)
+
+        with open('Makefile', 'r', encoding='utf-8') as fh:
+            for line in fh:
+                m = re.match(r'^(\w+)=(.+)', line)
+                if not m:
+                    continue
+                var, val = (m.group(1), m.group(2))
+                if var == 'prefix' and env_subs[var] is not None:
+                    continue
+                while re.search(r'\$\{', val):
+                    val = re.sub(r'\$\{(\w+)\}', lambda m: env_subs[m.group(1)], val)
+                env_subs[var] = val
+                if var == 'srcdir':
+                    break
 
     with open(fi.fn, 'r', encoding='utf-8') as fh:
         txt = fh.read()
 
     txt = re.sub(r'@VERSION@', env_subs['VERSION'], txt)
     txt = re.sub(r'@LIBDIR@', env_subs['libdir'], txt)
+
     fi.html_in = md_parser(txt)
     txt = None
 
+    fi.date = time.strftime('%d %b %Y', time.localtime(fi.mtime))
     fi.man_headings = (fi.prog, fi.sect, fi.date, fi.prog + ' ' + env_subs['VERSION'])
 
     HtmlToManPage(fi)
@@ -151,6 +163,7 @@ class HtmlToManPage(HTMLParser):
                 at_first_tag_in_dd = False,
                 dt_from = None,
                 in_pre = False,
+                in_code = False,
                 html_out = [ HTML_START % fi.title ],
                 man_out = [ MAN_START % fi.man_headings ],
                 txt = '',
@@ -180,6 +193,8 @@ class HtmlToManPage(HTMLParser):
                     tag = 'dt'
                 else:
                     st.html_out.append('<dt>')
+            elif tag == 'p':
+                st.at_first_tag_in_dd = True # Kluge to suppress a .P at the start of an li.
             st.at_first_tag_in_li = False
         if tag == 'p':
             if not st.at_first_tag_in_dd:
@@ -200,12 +215,13 @@ class HtmlToManPage(HTMLParser):
             st.in_pre = True
             st.man_out.append(st.p_macro + ".nf\n")
         elif tag == 'code' and not st.in_pre:
+            st.in_code = True
             st.txt += BOLD_FONT[0]
         elif tag == 'strong' or tag == 'b':
             st.txt += BOLD_FONT[0]
         elif tag == 'em' or  tag == 'i':
             tag = 'u' # Change it into underline to be more like the man page
-            st.txt += ULIN_FONT[0]
+            st.txt += UNDR_FONT[0]
         elif tag == 'ol':
             start = 1
             for var, val in attrs_list:
@@ -266,7 +282,10 @@ class HtmlToManPage(HTMLParser):
         elif tag == 'pre':
             st.in_pre = False
             st.man_out.append(manify(txt) + "\n.fi\n")
-        elif (tag == 'code' and not st.in_pre) or tag == 'strong' or tag == 'b':
+        elif (tag == 'code' and not st.in_pre):
+            st.in_code = False
+            add_to_txt = NORM_FONT[0]
+        elif tag == 'strong' or tag == 'b':
             add_to_txt = NORM_FONT[0]
         elif tag == 'em' or  tag == 'i':
             tag = 'u' # Change it into underline to be more like the man page
@@ -295,12 +314,21 @@ class HtmlToManPage(HTMLParser):
             st.at_first_tag_in_dd = True
 
 
-    def handle_data(self, data):
+    def handle_data(self, txt):
         st = self.state
         if args.debug:
-            self.output_debug('DATA', (data,))
-        st.html_out.append(htmlify(data))
-        st.txt += data
+            self.output_debug('DATA', (txt,))
+        if st.in_pre:
+            html = htmlify(txt)
+        else:
+            txt = re.sub(r'\s--(\s)', NBR_SPACE[0] + r'--\1', txt).replace('--', NBR_DASH[0]*2)
+            txt = re.sub(r'(^|\W)-', r'\1' + NBR_DASH[0], txt)
+            html = htmlify(txt)
+            if st.in_code:
+                txt = re.sub(r'\s', NBR_SPACE[0], txt)
+                html = html.replace(NBR_DASH[0], '-').replace(NBR_SPACE[0], ' ') # <code> is non-breaking in CSS
+        st.html_out.append(html.replace(NBR_SPACE[0], '&nbsp;').replace(NBR_DASH[0], '-&#8288;'))
+        st.txt += txt
 
 
     def output_debug(self, event, extra):
@@ -318,9 +346,11 @@ class HtmlToManPage(HTMLParser):
 
 def manify(txt):
     return re.sub(r"^(['.])", r'\&\1', txt.replace('\\', '\\\\')
+            .replace(NBR_SPACE[0], NBR_SPACE[1])
+            .replace(NBR_DASH[0], NBR_DASH[1])
             .replace(NORM_FONT[0], NORM_FONT[1])
             .replace(BOLD_FONT[0], BOLD_FONT[1])
-            .replace(ULIN_FONT[0], ULIN_FONT[1]), flags=re.M)
+            .replace(UNDR_FONT[0], UNDR_FONT[1]), flags=re.M)
 
 
 def htmlify(txt):

options.c

@@ -21,6 +21,7 @@
 
 #include "rsync.h"
 #include "itypes.h"
+#include "ifuncs.h"
 #include "latest-year.h"
 #include <popt.h>
 
@@ -170,6 +171,7 @@ char *logfile_name = NULL;
 char *logfile_format = NULL;
 char *stdout_format = NULL;
 char *password_file = NULL;
+char *early_input_file = NULL;
 char *rsync_path = RSYNC_PATH;
 char *backup_dir = NULL;
 char backup_dir_buf[MAXPATHLEN];
@@ -180,12 +182,21 @@ int rsync_port = 0;
 int alt_dest_type = 0;
 int basis_dir_cnt = 0;
 
+#define DEFAULT_MAX_ALLOC (1024L * 1024 * 1024)
+size_t max_alloc = DEFAULT_MAX_ALLOC;
+char *max_alloc_arg;
+
+static int version_opt_cnt = 0;
 static int remote_option_alloc = 0;
 int remote_option_cnt = 0;
 const char **remote_options = NULL;
 const char *checksum_choice = NULL;
 const char *compress_choice = NULL;
 
+#ifndef __APPLE__ /* Do we need a configure check for this? */
+#define SUPPORT_ATIMES 1
+#endif
+
 int quiet = 0;
 int output_motd = 1;
 int log_before_transfer = 0;
@@ -200,8 +211,11 @@ int list_only = 0;
 char *batch_name = NULL;
 
 int need_unsorted_flist = 0;
+char *iconv_opt =
 #ifdef ICONV_OPTION
-char *iconv_opt = ICONV_OPTION;
+		ICONV_OPTION;
+#else
+		NULL;
 #endif
 
 struct chmod_mode_struct *chmod_modes = NULL;
@@ -308,9 +322,7 @@ static int refused_partial, refused_progress, refused_delete_before;
 static int refused_delete_during;
 static int refused_inplace, refused_no_iconv;
 static BOOL usermap_via_chown, groupmap_via_chown;
-#ifdef HAVE_SETVBUF
 static char *outbuf_mode;
-#endif
 static char *bwlimit_arg, *max_size_arg, *min_size_arg;
 static char tmp_partialdir[] = ".~tmp~";
 
@@ -375,8 +387,7 @@ static char *make_output_option(struct output_struct *words, short *levels, ucha
 		return NULL;
 
 	len++;
-	if (!(buf = new_array(char, len)))
-		out_of_memory("make_output_option");
+	buf = new_array(char, len);
 	pos = 0;
 
 	if (skipped || max < 5)
@@ -570,12 +581,15 @@ static char *istring(const char *fmt, int val)
 	return str;
 }
 
-static void print_capabilities(enum logcode f)
+static void print_info_flags(enum logcode f)
 {
 	STRUCT_STAT *dumstat;
 	char line_buf[75];
 	int line_len, j;
 	char *capabilities[] = {
+
+	"*Capabilities",
+
 		istring("%d-bit files", (int)(sizeof (OFF_T) * 8)),
 		istring("%d-bit inums", (int)(sizeof dumstat->st_ino * 8)), /* Don't check ino_t! */
 		istring("%d-bit timestamps", (int)(sizeof (time_t) * 8)),
@@ -601,6 +615,11 @@ static void print_capabilities(enum logcode f)
 #endif
 			"IPv6",
 
+#ifndef SUPPORT_ATIMES
+		"no "
+#endif
+			"atimes",
+
 		"batchfiles",
 
 #ifndef HAVE_FTRUNCATE
@@ -623,6 +642,13 @@ static void print_capabilities(enum logcode f)
 #endif
 			"xattrs",
 
+#ifdef RSYNC_USE_PROTECTED_ARGS
+		"default "
+#else
+		"optional "
+#endif
+			"protect-args",
+
 #ifndef ICONV_OPTION
 		"no "
 #endif
@@ -638,24 +664,40 @@ static void print_capabilities(enum logcode f)
 #endif
 			"prealloc",
 
+	"*Optimizations",
+
 #ifndef HAVE_SIMD
 		"no "
 #endif
 			"SIMD",
 
+#ifndef HAVE_ASM
+		"no "
+#endif
+			"asm",
+
+#ifndef USE_OPENSSL
+		"no "
+#endif
+			"openssl-crypto",
+
 		NULL
 	};
 
 	for (line_len = 0, j = 0; ; j++) {
-		char *cap = capabilities[j];
-		int cap_len = cap ? strlen(cap) : 1000;
-		int need_comma = cap && capabilities[j+1] != NULL ? 1 : 0;
-		if (line_len + 1 + cap_len + need_comma >= (int)sizeof line_buf) {
+		char *cap = capabilities[j], *next_cap = cap ? capabilities[j+1] : NULL;
+		int cap_len = cap && *cap != '*' ? strlen(cap) : 1000;
+		int need_comma = next_cap && *next_cap != '*' ? 1 : 0;
+		if (line_len && line_len + 1 + cap_len + need_comma >= (int)sizeof line_buf) {
 			rprintf(f, "   %s\n", line_buf);
 			line_len = 0;
 		}
 		if (!cap)
 			break;
+		if (*cap == '*') {
+			rprintf(f, "%s:\n", cap+1);
+			continue;
+		}
 		line_len += snprintf(line_buf+line_len, sizeof line_buf - line_len, " %s%s", cap, need_comma ? "," : "");
 	}
 }
@@ -671,10 +713,9 @@ static void print_rsync_version(enum logcode f)
 		RSYNC_NAME, RSYNC_VERSION, PROTOCOL_VERSION, subprotocol);
 
 	rprintf(f, "Copyright (C) 1996-" LATEST_YEAR " by Andrew Tridgell, Wayne Davison, and others.\n");
-	rprintf(f, "Web site: http://rsync.samba.org/\n");
+	rprintf(f, "Web site: https://rsync.samba.org/\n");
 
-	rprintf(f, "Capabilities:\n");
-	print_capabilities(f);
+	print_info_flags(f);
 
 	rprintf(f, "Checksum list:\n");
 	get_default_nno_list(&valid_checksums, tmpbuf, sizeof tmpbuf, '(');
@@ -728,7 +769,7 @@ void usage(enum logcode F)
   rprintf(F,"\n");
   rprintf(F,"Use \"rsync --daemon --help\" to see the daemon-mode command-line options.\n");
   rprintf(F,"Please see the rsync(1) and rsyncd.conf(5) man pages for full documentation.\n");
-  rprintf(F,"See http://rsync.samba.org/ for updates, bug reports, and answers\n");
+  rprintf(F,"See https://rsync.samba.org/ for updates, bug reports, and answers\n");
 }
 
 enum {OPT_SERVER = 1000, OPT_DAEMON, OPT_SENDER, OPT_EXCLUDE, OPT_EXCLUDE_FROM,
@@ -846,6 +887,7 @@ static struct poptOption long_options[] = {
   {"ignore-existing",  0,  POPT_ARG_NONE,   &ignore_existing, 0, 0, 0 },
   {"max-size",         0,  POPT_ARG_STRING, &max_size_arg, OPT_MAX_SIZE, 0, 0 },
   {"min-size",         0,  POPT_ARG_STRING, &min_size_arg, OPT_MIN_SIZE, 0, 0 },
+  {"max-alloc",        0,  POPT_ARG_STRING, &max_alloc_arg, 0, 0, 0 },
   {"sparse",          'S', POPT_ARG_VAL,    &sparse_files, 1, 0, 0 },
   {"no-sparse",        0,  POPT_ARG_VAL,    &sparse_files, 0, 0, 0 },
   {"no-S",             0,  POPT_ARG_VAL,    &sparse_files, 0, 0, 0 },
@@ -902,6 +944,7 @@ static struct poptOption long_options[] = {
   {"zc",               0,  POPT_ARG_STRING, &compress_choice, 0, 0, 0 },
   {"skip-compress",    0,  POPT_ARG_STRING, &skip_compress, 0, 0, 0 },
   {"compress-level",   0,  POPT_ARG_INT,    &do_compression_level, 0, 0, 0 },
+  {"zl",               0,  POPT_ARG_INT,    &do_compression_level, 0, 0, 0 },
   {0,                 'P', POPT_ARG_NONE,   0, 'P', 0, 0 },
   {"progress",         0,  POPT_ARG_VAL,    &do_progress, 1, 0, 0 },
   {"no-progress",      0,  POPT_ARG_VAL,    &do_progress, 0, 0, 0 },
@@ -961,6 +1004,7 @@ static struct poptOption long_options[] = {
   {"port",             0,  POPT_ARG_INT,    &rsync_port, 0, 0, 0 },
   {"sockopts",         0,  POPT_ARG_STRING, &sockopts, 0, 0, 0 },
   {"password-file",    0,  POPT_ARG_STRING, &password_file, 0, 0, 0 },
+  {"early-input",      0,  POPT_ARG_STRING, &early_input_file, 0, 0, 0 },
   {"blocking-io",      0,  POPT_ARG_VAL,    &blocking_io, 1, 0, 0 },
   {"no-blocking-io",   0,  POPT_ARG_VAL,    &blocking_io, 0, 0, 0 },
   {"outbuf",           0,  POPT_ARG_STRING, &outbuf_mode, 0, 0, 0 },
@@ -1142,6 +1186,9 @@ static void set_refuse_options(void)
 		parse_one_refuse_match(0, "log-file*", list_end);
 	}
 
+#ifndef SUPPORT_ATIMES
+	parse_one_refuse_match(0, "atimes", list_end);
+#endif
 #ifndef SUPPORT_HARD_LINKS
 	parse_one_refuse_match(0, "link-dest", list_end);
 #endif
@@ -1203,14 +1250,16 @@ static int count_args(const char **argv)
 	return i;
 }
 
-
-static OFF_T parse_size_arg(char **size_arg, char def_suf)
+/* If the size_arg is an invalid string or the value is < min_value, an error
+ * is put into err_buf & the return is -1.  Note that this parser does NOT
+ * support negative numbers, so a min_value < 0 doesn't make any sense. */
+static ssize_t parse_size_arg(char *size_arg, char def_suf, const char *opt_name, ssize_t min_value)
 {
-	int reps, mult, make_compatible = 0;
-	const char *arg;
-	OFF_T size = 1;
+	int reps, mult;
+	const char *arg, *err = "invalid";
+	ssize_t size = 1;
 
-	for (arg = *size_arg; isDigit(arg); arg++) {}
+	for (arg = size_arg; isDigit(arg); arg++) {}
 	if (*arg == '.')
 		for (arg++; isDigit(arg); arg++) {}
 	switch (*arg && *arg != '+' && *arg != '-' ? *arg++ : def_suf) {
@@ -1226,40 +1275,40 @@ static OFF_T parse_size_arg(char **size_arg, char def_suf)
 	case 'g': case 'G':
 		reps = 3;
 		break;
+	case 't': case 'T':
+		reps = 4;
+		break;
+	case 'p': case 'P':
+		reps = 5;
+		break;
 	default:
-		return -1;
+		goto failure;
 	}
 	if (*arg == 'b' || *arg == 'B')
-		mult = 1000, make_compatible = 1, arg++;
+		mult = 1000, arg++;
 	else if (!*arg || *arg == '+' || *arg == '-')
 		mult = 1024;
 	else if (strncasecmp(arg, "ib", 2) == 0)
 		mult = 1024, arg += 2;
 	else
-		return -1;
+		goto failure;
 	while (reps--)
 		size *= mult;
-	size *= atof(*size_arg);
-	if ((*arg == '+' || *arg == '-') && arg[1] == '1')
-		size += atoi(arg), make_compatible = 1, arg += 2;
+	size *= atof(size_arg);
+	if ((*arg == '+' || *arg == '-') && arg[1] == '1' && arg != size_arg)
+		size += atoi(arg), arg += 2;
 	if (*arg)
-		return -1;
-	if (size > 0 && make_compatible && def_suf == 'b') {
-		/* We convert this manually because we may need %lld precision,
-		 * and that's not a portable sprintf() escape. */
-		char buf[128], *s = buf + sizeof buf - 1;
-		OFF_T num = size;
-		*s = '\0';
-		while (num) {
-			*--s = (char)(num % 10) + '0';
-			num /= 10;
-		}
-		if (!(*size_arg = strdup(s)))
-			out_of_memory("parse_size_arg");
+		goto failure;
+	if (size < min_value) {
+		err = size < 0 ? "too big" : "too small";
+		goto failure;
 	}
 	return size;
-}
 
+failure:
+	snprintf(err_buf, sizeof err_buf, "--%s value is %s: %s\n", opt_name, err, size_arg);
+	return -1;
+}
 
 static void create_refuse_error(int which)
 {
@@ -1360,8 +1409,8 @@ int parse_arguments(int *argc_p, const char ***argv_p)
 
 		switch (opt) {
 		case 'V':
-			print_rsync_version(FINFO);
-			exit_cleanup(0);
+			version_opt_cnt++;
+			break;
 
 		case OPT_SERVER:
 			if (!am_server) {
@@ -1482,8 +1531,6 @@ int parse_arguments(int *argc_p, const char ***argv_p)
 			if (daemon_filter_list.head) {
 				int rej;
 				char *cp = strdup(arg);
-				if (!cp)
-					out_of_memory("parse_arguments");
 				if (!*cp)
 					rej = 1;
 				else {
@@ -1607,8 +1654,6 @@ int parse_arguments(int *argc_p, const char ***argv_p)
 				remote_option_alloc += 16;
 				remote_options = realloc_array(remote_options,
 							const char *, remote_option_alloc);
-				if (!remote_options)
-					out_of_memory("parse_arguments");
 				if (!remote_option_cnt)
 					remote_options[0] = "ARG0";
 			}
@@ -1638,39 +1683,25 @@ int parse_arguments(int *argc_p, const char ***argv_p)
 			break;
 
 		case OPT_MAX_SIZE:
-			if ((max_size = parse_size_arg(&max_size_arg, 'b')) < 0) {
-				snprintf(err_buf, sizeof err_buf,
-					"--max-size value is invalid: %s\n",
-					max_size_arg);
+			if ((max_size = parse_size_arg(max_size_arg, 'b', "max-size", 0)) < 0)
 				return 0;
-			}
+			max_size_arg = num_to_byte_string(max_size);
 			break;
 
 		case OPT_MIN_SIZE:
-			if ((min_size = parse_size_arg(&min_size_arg, 'b')) < 0) {
-				snprintf(err_buf, sizeof err_buf,
-					"--min-size value is invalid: %s\n",
-					min_size_arg);
+			if ((min_size = parse_size_arg(min_size_arg, 'b', "min-size", 0)) < 0)
 				return 0;
-			}
+			min_size_arg = num_to_byte_string(min_size);
 			break;
 
-		case OPT_BWLIMIT:
-			{
-				OFF_T limit = parse_size_arg(&bwlimit_arg, 'K');
-				if (limit < 0) {
-					snprintf(err_buf, sizeof err_buf,
-						"--bwlimit value is invalid: %s\n", bwlimit_arg);
-					return 0;
-				}
-				bwlimit = (limit + 512) / 1024;
-				if (limit && !bwlimit) {
-					snprintf(err_buf, sizeof err_buf,
-						"--bwlimit value is too small: %s\n", bwlimit_arg);
-					return 0;
-				}
-			}
+		case OPT_BWLIMIT: {
+			ssize_t size = parse_size_arg(bwlimit_arg, 'K', "bwlimit", 512);
+			if (size < 0)
+				return 0;
+			bwlimit_arg = num_to_byte_string(size);
+			bwlimit = (size + 512) / 1024;
 			break;
+		}
 
 		case OPT_APPEND:
 			if (am_server)
@@ -1847,6 +1878,23 @@ int parse_arguments(int *argc_p, const char ***argv_p)
 		}
 	}
 
+	if (version_opt_cnt) {
+		print_rsync_version(FINFO);
+		exit_cleanup(0);
+	}
+
+	if (!max_alloc_arg) {
+		max_alloc_arg = getenv("RSYNC_MAX_ALLOC");
+		if (max_alloc_arg && !*max_alloc_arg)
+			max_alloc_arg = NULL;
+	}
+	if (max_alloc_arg) {
+		ssize_t size = parse_size_arg(max_alloc_arg, 'B', "max-alloc", 1024*1024);
+		if (size < 0)
+			return 0;
+		max_alloc = size;
+	}
+
 	if (protect_args < 0) {
 		if (am_server)
 			protect_args = 0;
@@ -2170,6 +2218,7 @@ int parse_arguments(int *argc_p, const char ***argv_p)
 	}
 	if (backup_dir) {
 		size_t len;
+		make_backups = 1; /* --backup-dir implies --backup */
 		while (*backup_dir == '.' && backup_dir[1] == '/')
 			backup_dir += 2;
 		if (*backup_dir == '.' && backup_dir[1] == '\0')
@@ -2716,6 +2765,11 @@ void server_options(char **args, int *argc_p)
 		}
 	}
 
+	if (max_alloc_arg && max_alloc != DEFAULT_MAX_ALLOC) {
+		args[ac++] = "--max-alloc";
+		args[ac++] = max_alloc_arg;
+	}
+
 	/* --delete-missing-args needs the cooperation of both sides, but
 	 * the sender can handle --ignore-missing-args by itself. */
 	if (missing_args == 2)

packaging/auto-Makefile

@@ -0,0 +1,11 @@
+TARGETS := all install install-ssl-daemon install-all install-strip conf gen gensend reconfigure restatus \
+	proto man clean cleantests distclean test check check29 check30 installcheck splint doxygen doxygen-upload
+
+.PHONY: $(TARGETS) auto-prep
+
+$(TARGETS): auto-prep
+	make -C build $@
+
+auto-prep:
+	@if test x`packaging/prep-auto-dir` = x; then echo "auto-build-save is not setup"; exit 1; fi
+	@echo 'Build branch: '`readlink build/.branch | tr % /`

packaging/branch-from-patch

@@ -1,4 +1,4 @@
-#!/usr/bin/python3 -B
+#!/usr/bin/env -S python3 -B
 
 # This script turns one or more diff files in the patches dir (which is
 # expected to be a checkout of the rsync-patches git repo) into a branch
@@ -171,4 +171,4 @@ if __name__ == '__main__':
     args = parser.parse_args()
     main()
 
-# vim: sw=4 et
+# vim: sw=4 et ft=python

packaging/cull_options

@@ -1,4 +1,4 @@
-#!/usr/bin/perl
+#!/usr/bin/env perl
 # This script outputs some perl code that parses all possible options
 # that the code in options.c might send to the server.  This perl code
 # is included in the rrsync script.
@@ -23,6 +23,7 @@ our %long_opt = ( # These include some extra long-args that BackupPC uses:
     'perms' => 0,
     'recursive' => 0,
     'times' => 0,
+    'write-devices' => -1,
 );
 our $last_long_opt;
 
@@ -39,10 +40,10 @@ while (<IN>) {
 	$last_long_opt = $1;
 	$long_opt{$1} = 0 unless exists $long_opt{$1};
     } elsif (defined($last_long_opt)
-	&& /\Qargs[ac++]\E = ([^["\s]+);/ && $1 ne 'dest_option') {
+	&& /\Qargs[ac++]\E = ([^["\s]+);/) {
 	$long_opt{$last_long_opt} = 2;
 	undef $last_long_opt;
-    } elsif (/dest_option = "--([^"]+)"/) {
+    } elsif (/return "--([^"]+-dest)";/) {
 	$long_opt{$1} = 2;
 	undef $last_long_opt;
     } elsif (/\Qasprintf(\E[^,]+, "--([^"=]+)=/ || /\Qargs[ac++]\E = "--([^"=]+)=/ || /fmt = .*: "--([^"=]+)=/) {

packaging/lsb/rsync.spec

@@ -1,14 +1,14 @@
 Summary: A fast, versatile, remote (and local) file-copying tool
 Name: rsync
-Version: 3.2.0
-%define fullversion %{version}pre1
-Release: 0.1.pre1
-%define srcdir src-previews
+Version: 3.2.2
+%define fullversion %{version}
+Release: 1
+%define srcdir src
 Group: Applications/Internet
 License: GPL
-Source0: http://rsync.samba.org/ftp/rsync/%{srcdir}/rsync-%{fullversion}.tar.gz
-#Source1: http://rsync.samba.org/ftp/rsync/%{srcdir}/rsync-patches-%{fullversion}.tar.gz
-URL: http://rsync.samba.org/
+Source0: https://rsync.samba.org/ftp/rsync/%{srcdir}/rsync-%{fullversion}.tar.gz
+#Source1: https://rsync.samba.org/ftp/rsync/%{srcdir}/rsync-patches-%{fullversion}.tar.gz
+URL: https://rsync.samba.org/
 
 Prefix: %{_prefix}
 BuildRoot: /var/tmp/%{name}-root
@@ -66,7 +66,7 @@ rm -rf $RPM_BUILD_ROOT
 
 %files
 %defattr(-,root,root)
-%doc COPYING NEWS.md OLDNEWS.md README.md support/ tech_report.tex
+%doc COPYING NEWS.md README.md support/ tech_report.tex
 %config(noreplace) /etc/xinetd.d/rsync
 %{_prefix}/bin/rsync
 %{_prefix}/bin/rsync-ssl
@@ -79,9 +79,9 @@ rm -rf $RPM_BUILD_ROOT
 %dir /etc/rsync-ssl/certs
 
 %changelog
-* Sat Jun 13 2020 Wayne Davison <wayned@samba.org>
-Released 3.2.0pre1.
+* Sat Jul 04 2020 Wayne Davison <wayne@opencoder.net>
+Released 3.2.2.
 
-* Fri Mar 21 2008 Wayne Davison <wayned@samba.org>
+* Fri Mar 21 2008 Wayne Davison <wayne@opencoder.net>
 Added installation of /etc/xinetd.d/rsync file and some commented-out
 lines that demonstrate how to use the rsync-patches tar file.

packaging/md2html

@@ -0,0 +1,108 @@
+#!/usr/bin/env python3
+
+# Copyright (C) 2020 Wayne Davison
+#
+# This program is freely redistributable.
+
+import os, re, argparse
+
+HTML_START = """\
+<html><head>
+<title>%s</title>
+<link href="https://fonts.googleapis.com/css2?family=Roboto&family=Roboto+Mono&display=swap" rel="stylesheet">
+<style>
+body {
+  max-width: 50em;
+  margin: auto;
+}
+body, b, strong, u {
+  font-family: 'Roboto', sans-serif;
+}
+code {
+  font-family: 'Roboto Mono', monospace;
+  font-weight: bold;
+}
+pre code {
+  display: block;
+  font-weight: normal;
+}
+blockquote pre code {
+  background: #f1f1f1;
+}
+dd p:first-of-type {
+  margin-block-start: 0em;
+}
+table {
+  border-color: grey;
+  border-spacing: 0;
+}
+tr {
+  border-top: 1px solid grey;
+}
+tr:nth-child(2n) {
+  background-color: #f6f8fa;
+}
+th, td {
+  border: 1px solid #dfe2e5;
+  text-align: center;
+  padding-left: 1em;
+  padding-right: 1em;
+}
+</style>
+</head><body>
+"""
+
+HTML_END = """\
+</body></html>
+"""
+
+md_parser = None
+
+def main():
+    for mdfn in args.mdfiles:
+        if not mdfn.endswith('.md'):
+            print('Ignoring non-md input file:', mdfn)
+            continue
+        title = re.sub(r'.*/', '', mdfn).replace('.md', '')
+        htfn = mdfn.replace('.md', '.html')
+
+        print("Parsing", mdfn, '->', htfn)
+
+        with open(mdfn, 'r', encoding='utf-8') as fh:
+            txt = fh.read()
+
+        txt = re.sub(r'\s--\s', '\xa0-- ', txt)
+
+        html = md_parser(txt)
+
+        html = re.sub(r'(<code>)([\s\S]*?)(</code>)', lambda m: m[1] + re.sub(r'\s', '\xa0', m[2]) + m[3], html)
+        html = html.replace('--', '&#8209;&#8209;').replace("\xa0-", '&nbsp;&#8209;').replace("\xa0", '&nbsp;')
+        html = re.sub(r'(\W)-', r'\1&#8209;', html)
+
+        if os.path.lexists(htfn):
+            os.unlink(htfn)
+
+        with open(htfn, 'w', encoding='utf-8') as fh:
+            fh.write(HTML_START % title)
+            fh.write(html)
+            fh.write(HTML_END)
+
+
+def html_via_cmarkgfm(txt):
+    # Our NEWS.md file has a gfm table in it.
+    return cmarkgfm.github_flavored_markdown_to_html(txt)
+
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser(description='Output html for md pages.', add_help=False)
+    parser.add_argument("--help", "-h", action="help", help="Output this help message and exit.")
+    parser.add_argument("mdfiles", nargs='+', help="The .md files to turn into .html files.")
+    args = parser.parse_args()
+
+    try:
+        import cmarkgfm
+        md_parser = html_via_cmarkgfm
+    except:
+        die("Failed to find cmarkgfm for python3.")
+
+    main()

packaging/nightly-rsync

@@ -1,106 +0,0 @@
-#!/usr/bin/python3 -B
-
-# This script expects the directory ~/samba-rsync-ftp to exist and to be a
-# copy of the /home/ftp/pub/rsync dir on samba.org.  It also requires a
-# git checkout of rsync (feel free to use your normal rsync build dir as
-# long as it doesn't have any uncommitted changes).
-#
-# If this is run with -tu, it will make an updated "nightly" tar file in
-# the nightly dir.  It will also remove any old tar files, regenerate the
-# HTML man pages in the nightly dir, and then rsync the changes to the
-# samba.org server.
-
-import os, sys, re, argparse, glob
-from datetime import datetime, timezone
-from getpass import getpass
-
-sys.path = ['packaging'] + sys.path
-
-from pkglib import *
-
-# Where the local copy of /home/ftp/pub/rsync/dev/nightly should be updated.
-dest = os.environ['HOME'] + '/samba-rsync-ftp/dev/nightly'
-samba_host = os.environ['SAMBA_HOST']
-nightly_symlink = f"{dest}/rsync-HEAD.tar.gz"
-
-def main():
-    now = datetime.now(timezone.utc)
-    name = now.strftime('rsync-HEAD-%Y%m%d-%H%MGMT')
-    ztoday = now.strftime('%d %b %Y')
-    today = ztoday.lstrip('0')
-    gen_target = 'gensend' if args.upload else 'gen'
-
-    if not os.path.isdir(dest):
-        die("$dest does not exist")
-    if not os.path.isdir('.git'):
-        die("There is no .git dir in the current directory.")
-    if not os.path.exists('rsyncd.conf.5.md'):
-        die("There is no rsync checkout in the current directory.")
-
-    mandate_gensend_hook()
-
-    if args.make_tar:
-        check_git_state('master')
-        cmd_chk(['touch', 'NEWS.md'])
-        cmd_chk(['make', gen_target])
-        cmd_chk(['rsync', '-a', *glob.glob('*.[1-9].html'), dest])
-
-        gen_files = get_gen_files()
-
-        confversion = get_configure_version()
-
-        # All version values are strings!
-        last_version, last_protocol_version = get_OLDNEWS_version_info()
-        protocol_version, subprotocol_version = get_protocol_versions()
-
-        if 'dev' in confversion or 'pre' in confversion:
-            if last_protocol_version != protocol_version:
-                if subprotocol_version == '0':
-                    die("SUBPROTOCOL_VERSION must not be 0 for a non-final release with a changed PROTOCOL_VERSION.")
-            elif subprotocol_version != '0':
-                die("SUBPROTOCOL_VERSION must be 0 when the PROTOCOL_VERSION hasn't changed from the last release.")
-        elif subprotocol_version != '0':
-            die("SUBPROTOCOL_VERSION must be 0 for a final release.")
-
-        name_slash = name + '/'
-        tar_name = f"{name}.tar.gz"
-
-        print('Creating', tar_name)
-
-        cmd_chk(['rsync', '-a', *gen_files, name_slash])
-        cmd_chk(f"git archive --format=tar --prefix={name}/ HEAD | tar xf -")
-        cmd_chk(['support/git-set-file-times', '--quiet', '--prefix', name_slash])
-        cmd_chk(['fakeroot', 'tar', 'czf', os.path.join(dest, tar_name), name])
-        cmd_chk(['rm', '-rf', name])
-
-        if os.path.lexists(nightly_symlink):
-            os.unlink(nightly_symlink)
-        os.symlink(tar_name, nightly_symlink)
-
-    os.chdir(dest)
-
-    tar_files = list(reversed(sorted(glob.glob('rsync-HEAD-*'))))
-    if len(tar_files) > 10:
-        for fn in tar_files[10:]:
-            print('Removing', fn)
-            os.unlink(fn)
-
-    cmd_run('ls -ltr'.split())
-
-    if args.upload:
-        cmd = 'rsync -aivHP --delete-after'.split()
-        partial_dir = os.environ.get('RSYNC_PARTIAL_DIR', None)
-        if partial_dir:
-            cmd.append('-fR ' + partial_dir)
-        cmd_chk([*cmd, '.', f"{samba_host}:/home/ftp/pub/rsync/dev/nightly"])
-
-
-if __name__ == '__main__':
-    parser = argparse.ArgumentParser(description='A helper script for "nightly" tar files.', add_help=False)
-    parser.add_argument('--make-tar', '-t', action='store_true', help=f"Create a new tar file in {dest}.")
-    parser.add_argument('--upload', '-u', action='store_true', help="Upload the revised nightly dir to {samba_host}.")
-    parser.add_argument("--help", "-h", action="help", help="Output this help message and exit.")
-    args = parser.parse_args()
-    main()
-
-# vim: sw=4 et

packaging/patch-update

@@ -1,4 +1,4 @@
-#!/usr/bin/python3 -B
+#!/usr/bin/env -S python3 -B
 
 # This script is used to turn one or more of the "patch/BASE/*" branches
 # into one or more diffs in the "patches" directory.  Pass the option
@@ -13,9 +13,9 @@ sys.path = ['packaging'] + sys.path
 from pkglib import *
 
 MAKE_GEN_CMDS = [
-        'make -f prepare-source.mak conf'.split(),
-        './config.status'.split(),
-        'make gen'.split(),
+        './prepare-source'.split(),
+        'cd build && if test -f config.status ; then ./config.status ; else ../configure ; fi',
+        'make -C build gen'.split(),
         ]
 TMP_DIR = "patches.gen"
 
@@ -33,6 +33,9 @@ def main():
 
     master_commit = latest_git_hash(args.base_branch)
 
+    if cmd_txt_chk(['packaging/prep-auto-dir']) == '':
+        die('You must setup an auto-build-save dir to use this script.')
+
     if args.gen:
         if os.path.lexists(TMP_DIR):
             die(f'"{TMP_DIR}" must not exist in the current directory.')
@@ -42,7 +45,7 @@ def main():
             cmd_chk(cmd)
         cmd_chk(['rsync', '-a', *gen_files, f'{TMP_DIR}/master/'])
 
-    last_touch = time.time()
+    last_touch = int(time.time())
 
     # Start by finding all patches so that we can load all possible parents.
     patches = sorted(list(get_patch_branches(args.base_branch)))
@@ -91,9 +94,10 @@ def main():
     if args.gen:
         shutil.rmtree(TMP_DIR)
 
-    while last_touch >= time.time():
+    while last_touch >= int(time.time()):
         time.sleep(1)
     cmd_chk(['git', 'checkout', starting_branch])
+    cmd_chk(['packaging/prep-auto-dir'], discard='output')
 
 
 def update_patch(patch):
@@ -113,31 +117,30 @@ def update_patch(patch):
 
     print(f"======== {patch} ========")
 
-    while args.gen and last_touch >= time.time():
+    while args.gen and last_touch >= int(time.time()):
         time.sleep(1)
-    s = cmd_run(f"git checkout patch/{args.base_branch}/{patch}".split())
+
+    branch = f"patch/{args.base_branch}/{patch}"
+    s = cmd_run(['git', 'checkout', branch])
     if s.returncode != 0:
         return 0
 
     s = cmd_run(['git', 'merge', based_on])
     ok = s.returncode == 0
-    if not ok or args.shell:
-        m = re.search(r'([^/]+)$', parent)
-        parent_dir = m[1]
-        if not ok:
-            print(f'"git merge {based_on}" incomplete -- please fix.')
-        os.environ['PS1'] = f"[{parent_dir}] {patch}: "
-        while True:
-            s = cmd_run([os.environ.get('SHELL', '/bin/sh')])
-            if s.returncode != 0:
-                ans = input("Abort? [n/y] ")
-                if re.match(r'^y', ans, flags=re.I):
-                    return 0
-                continue
-            cur_branch, is_clean, status_txt = check_git_status(0)
-            if is_clean:
-                break
-            print(status_txt, end='')
+    if not ok or args.cmd or args.shell:
+        cmd_chk(['packaging/prep-auto-dir'], discard='output')
+    if not ok:
+        print(f'"git merge {based_on}" incomplete -- please fix.')
+        if not run_a_shell(parent, patch):
+            return 0
+    if args.cmd:
+        if cmd_run(args.cmd).returncode != 0:
+            if not run_a_shell(parent, patch):
+                return 0
+            ok = False
+    if ok and args.shell:
+        if not run_a_shell(parent, patch):
+            return 0
 
     with open(f"{args.patches_dir}/{patch}.diff", 'w', encoding='utf-8') as fh:
         fh.write(description[patch])
@@ -150,7 +153,7 @@ def update_patch(patch):
             cmd_chk(['rsync', '-a', *gen_files, f"{TMP_DIR}/{patch}/"])
         else:
             gen_files = [ ]
-        last_touch = time.time()
+        last_touch = int(time.time())
 
         proc = cmd_pipe(['git', 'diff', based_on])
         skipping = False
@@ -185,17 +188,38 @@ def update_patch(patch):
                 line =  plus_re.sub(r'+++ b/\1', line)
                 fh.write(line)
             proc.communicate()
-            for fn in gen_files:
-                os.unlink(fn)
 
     return 1
 
 
+def run_a_shell(parent, patch):
+    m = re.search(r'([^/]+)$', parent)
+    parent_dir = m[1]
+    os.environ['PS1'] = f"[{parent_dir}] {patch}: "
+
+    while True:
+        s = cmd_run([os.environ.get('SHELL', '/bin/sh')])
+        if s.returncode != 0:
+            ans = input("Abort? [n/y] ")
+            if re.match(r'^y', ans, flags=re.I):
+                return False
+            continue
+        cur_branch, is_clean, status_txt = check_git_status(0)
+        if is_clean:
+            break
+        print(status_txt, end='')
+
+    cmd_run('rm -f build/*.o build/*/*.o')
+
+    return True
+
+
 if __name__ == '__main__':
     parser = argparse.ArgumentParser(description="Turn a git branch back into a diff files in the patches dir.", add_help=False)
     parser.add_argument('--branch', '-b', dest='base_branch', metavar='BASE_BRANCH', default='master', help="The branch the patch is based on. Default: master.")
     parser.add_argument('--skip-check', action='store_true', help="Skip the check that ensures starting with a clean branch.")
     parser.add_argument('--shell', '-s', action='store_true', help="Launch a shell for every patch/BASE/* branch updated, not just when a conflict occurs.")
+    parser.add_argument('--cmd', '-c', help="Run a command in every patch branch.")
     parser.add_argument('--gen', metavar='DIR', nargs='?', const='', help='Include generated files. Optional DIR value overrides the default of using the "patches" dir.')
     parser.add_argument('--patches-dir', '-p', metavar='DIR', default='patches', help="Override the location of the rsync-patches dir. Default: patches.")
     parser.add_argument('patch_files', metavar='patches/DIFF_FILE', nargs='*', help="Specify what patch diff files to process. Default: all of them.")
@@ -207,4 +231,4 @@ if __name__ == '__main__':
         args.patches_dir = args.gen
     main()
 
-# vim: sw=4 et
+# vim: sw=4 et ft=python

packaging/pkglib.py

@@ -181,45 +181,50 @@ def mandate_gensend_hook():
 
 
 # Snag the GENFILES values out of the Makefile.in file and return them as a list.
-def get_gen_files():
+def get_gen_files(want_dir_plus_list=False):
     cont_re = re.compile(r'\\\n')
 
-    extras = [ ]
+    gen_files = [ ]
+
+    auto_dir = os.path.join('auto-build-save', cmd_txt('git rev-parse --abbrev-ref HEAD').strip().replace('/', '%'))
 
     with open('Makefile.in', 'r', encoding='utf-8') as fh:
         for line in fh:
-            if not extras:
+            if not gen_files:
                 chk = re.sub(r'^GENFILES=', '', line)
                 if line == chk:
                     continue
                 line = chk
             m = re.search(r'\\$', line)
             line = re.sub(r'^\s+|\s*\\\n?$|\s+$', '', line)
-            extras += line.split()
+            gen_files += line.split()
             if not m:
                 break
 
-    return extras
+    if want_dir_plus_list:
+        return (auto_dir, gen_files)
+
+    return [ os.path.join(auto_dir, fn) for fn in gen_files ]
 
 
-def get_configure_version():
-    with open('configure.ac', 'r', encoding='utf-8') as fh:
-        for line in fh:
-            m = re.match(r'^AC_INIT\(\[rsync\],\s*\[(\d.+?)\]', line)
-            if m:
-                return m[1]
-    die("Unable to find AC_INIT with version in configure.ac")
+def get_rsync_version():
+    with open('version.h', 'r', encoding='utf-8') as fh:
+        txt = fh.read()
+    m = re.match(r'^#define\s+RSYNC_VERSION\s+"(\d.+?)"', txt)
+    if m:
+        return m[1]
+    die("Unable to find RSYNC_VERSION define in version.h")
 
 
-def get_OLDNEWS_version_info():
-    rel_re = re.compile(r'^\| \d{2} \w{3} \d{4}\s+\|\s+(?P<ver>\d+\.\d+\.\d+)\s+\|\s+(?P<pdate>\d{2} \w{3} \d{4}\s+)?\|\s+(?P<pver>\d+)\s+\|')
+def get_NEWS_version_info():
+    rel_re = re.compile(r'^\| \S{2} \w{3} \d{4}\s+\|\s+(?P<ver>\d+\.\d+\.\d+)\s+\|\s+(?P<pdate>\d{2} \w{3} \d{4})?\s+\|\s+(?P<pver>\d+)\s+\|')
     last_version = last_protocol_version = None
     pdate = { }
 
-    with open('OLDNEWS.md', 'r', encoding='utf-8') as fh:
+    with open('NEWS.md', 'r', encoding='utf-8') as fh:
         for line in fh:
-            if not last_version:
-                m = re.search(r'(\d+\.\d+\.\d+)', line)
+            if not last_version: # Find the first non-dev|pre version with a release date.
+                m = re.search(r'rsync (\d+\.\d+\.\d+) .*\d\d\d\d', line)
                 if m:
                     last_version = m[1]
             m = rel_re.match(line)
@@ -228,12 +233,11 @@ def get_OLDNEWS_version_info():
                     pdate[m['ver']] = m['pdate']
                 if m['ver'] == last_version:
                     last_protocol_version = m['pver']
-                    break
 
     if not last_protocol_version:
         die(f"Unable to determine protocol_version for {last_version}.")
 
-    return last_version, last_protocol_version
+    return last_version, last_protocol_version, pdate
 
 
 def get_protocol_versions():

packaging/prep-auto-dir

@@ -0,0 +1,39 @@
+#!/bin/sh -e
+
+# This script will setup the build dir based on the current git branch and the
+# directory auto-build-save/$BRANCH.  We don't use a symlink for the build dir
+# because we want to maximize the ccache reuse, so all builds must happen in
+# the same real dir.  When a dir is moved out of auto-build-save/$BRANCH to the
+# build dir, it is replaced with a symlink so that it can still be found under
+# that dir.  The build dir also gets a .branch -> $BRANCH symlink so that we
+# can figure out the current build dir's branch.
+
+# To get started, just clone the rsync git repo and create the auto-build-save
+# dir.  If you have an existing git checkout and it is not in a pristine state,
+# run "make distclean" before creating the auto-build-save dir.
+
+auto_top='auto-build-save'
+if test -d $auto_top -a -d .git; then
+    desired_branch=`git rev-parse --abbrev-ref HEAD | tr / %`
+    auto_dir="$auto_top/$desired_branch"
+    if test -d build; then
+	cur_branch=`readlink build/.branch`
+    else
+	cur_branch='/'
+    fi
+    if test "$desired_branch" != "$cur_branch"; then
+	if test "$cur_branch" != /; then
+	    rm -f "$auto_top/$cur_branch"
+	    mv build "$auto_top/$cur_branch"
+	fi
+	test -d "$auto_dir" || mkdir "$auto_dir"
+	test -h "$auto_dir/.branch" || ln -s "$desired_branch" "$auto_dir/.branch"
+	mv "$auto_dir" build
+	ln -s ../build "$auto_dir"
+    fi
+    if test ! -h Makefile; then
+	rm -f Makefile
+	ln -s packaging/auto-Makefile Makefile
+    fi
+    echo $desired_branch
+fi

packaging/release-rsync

@@ -1,4 +1,4 @@
-#!/usr/bin/python3 -B
+#!/usr/bin/env -S python3 -B
 
 # This script expects the directory ~/samba-rsync-ftp to exist and to be a
 # copy of the /home/ftp/pub/rsync dir on samba.org.  When the script is done,
@@ -13,6 +13,7 @@ sys.path = ['packaging'] + sys.path
 
 from pkglib import *
 
+os.environ['LESS'] = 'mqeiXR'; # Make sure that -F is turned off and -R is turned on.
 dest = os.environ['HOME'] + '/samba-rsync-ftp'
 ORIGINAL_PATH = os.environ['PATH']
 
@@ -29,7 +30,11 @@ def main():
 
     signal.signal(signal.SIGINT, signal_handler)
 
-    gen_files = get_gen_files()
+    if cmd_txt_chk(['packaging/prep-auto-dir']) == '':
+        die('You must setup an auto-build-save dir to use this script.');
+
+    auto_dir, gen_files = get_gen_files(True)
+    gen_pathnames = [ os.path.join(auto_dir, fn) for fn in gen_files ]
 
     dash_line = '=' * 74
 
@@ -52,13 +57,13 @@ def main():
 
     check_git_state(args.master_branch, True, 'patches')
 
-    confversion = get_configure_version()
+    curversion = get_rsync_version()
 
     # All version values are strings!
-    lastversion, last_protocol_version = get_OLDNEWS_version_info()
+    lastversion, last_protocol_version, pdate = get_NEWS_version_info()
     protocol_version, subprotocol_version = get_protocol_versions()
 
-    version = confversion
+    version = curversion
     m = re.search(r'pre(\d+)', version)
     if m:
         version = re.sub(r'pre\d+', 'pre' + str(int(m[1]) + 1), version)
@@ -75,12 +80,9 @@ def main():
 
     v_ver = 'v' + version
     rsync_ver = 'rsync-' + version
-    rsync_lastver = 'rsync-' + lastversion
 
     if os.path.lexists(rsync_ver):
         die(f'"{rsync_ver}" must not exist in the current directory.')
-    if os.path.lexists(rsync_lastver):
-        die(f'"{rsync_lastver}" must not exist in the current directory.')
 
     out = cmd_txt_chk(['git', 'tag', '-l', v_ver])
     if out != '':
@@ -91,14 +93,18 @@ def main():
         cmd_chk(['git', 'tag', '-d', v_ver])
 
     version = re.sub(r'[-.]*pre[-.]*', 'pre', version)
-    if 'pre' in version and not confversion.endswith('dev'):
-        lastversion = confversion
+    if 'pre' in version and not curversion.endswith('dev'):
+        lastversion = curversion
 
     ans = input(f"Enter the previous version to produce a patch against: [{lastversion}] ")
     if ans != '':
         lastversion = ans
     lastversion = re.sub(r'[-.]*pre[-.]*', 'pre', lastversion)
 
+    rsync_lastver = 'rsync-' + lastversion
+    if os.path.lexists(rsync_lastver):
+        die(f'"{rsync_lastver}" must not exist in the current directory.')
+
     m = re.search(r'(pre\d+)', version)
     pre = m[1] if m else ''
 
@@ -110,11 +116,8 @@ def main():
         release += '.' + pre
 
     finalversion = re.sub(r'pre\d+', '', version)
-    if protocol_version == last_protocol_version:
-        proto_changed = 'unchanged'
-        proto_change_date = ' ' * 11
-    else:
-        proto_changed = 'changed'
+    proto_changed = protocol_version != last_protocol_version
+    if proto_changed:
         if finalversion in pdate:
             proto_change_date = pdate[finalversion]
         else:
@@ -123,6 +126,8 @@ def main():
                 if re.match(r'^\d\d \w\w\w \d\d\d\d$', ans):
                     break
             proto_change_date = ans
+    else:
+        proto_change_date = ' ' * 11
 
     if 'pre' in lastversion:
         if not pre:
@@ -151,8 +156,8 @@ release is "{release}"
 
 About to:
     - tweak SUBPROTOCOL_VERSION in rsync.h, if needed
-    - tweak the version in configure.ac and the spec files
-    - tweak NEWS.md and OLDNEWS.md to ensure header values are correct
+    - tweak the version in version.h and the spec files
+    - tweak NEWS.md to ensure header values are correct
     - generate configure.sh, config.h.in, and proto.h
     - page through the differences
 """)
@@ -166,16 +171,15 @@ About to:
         '%define srcdir': srcdir,
         }
 
-    tweak_files = 'configure.ac rsync.h NEWS.md OLDNEWS.md'.split()
+    tweak_files = 'version.h rsync.h NEWS.md'.split()
     tweak_files += glob.glob('packaging/*.spec')
     tweak_files += glob.glob('packaging/*/*.spec')
 
     for fn in tweak_files:
         with open(fn, 'r', encoding='utf-8') as fh:
             old_txt = txt = fh.read()
-        if 'configure' in fn:
-            x_re = re.compile(r'^(AC_INIT\(\[rsync\],\s*\[)\d.+?(\])', re.M)
-            txt = replace_or_die(x_re, r'\g<1>%s\2' % version, txt, f"Unable to update AC_INIT with version in {fn}")
+        if fn == 'version.h':
+            txt = f'#define RSYNC_VERSION "{version}"\n'
         elif '.spec' in fn:
             for var, val in specvars.items():
                 x_re = re.compile(r'^%s .*' % re.escape(var), re.M)
@@ -184,19 +188,20 @@ About to:
             txt = replace_or_die(x_re, r'%s \1' % cl_today, txt, f"Unable to update ChangeLog header in {fn}")
         elif fn == 'rsync.h':
             x_re = re.compile('(#define\s+SUBPROTOCOL_VERSION)\s+(\d+)')
-            repl = lambda m: m[1] + ' ' + '0' if not pre or proto_changed != 'changed' else 1 if m[2] == '0' else m[2]
+            repl = lambda m: m[1] + ' ' + ('0' if not pre or proto_changed else 1 if m[2] == '0' else m[2])
             txt = replace_or_die(x_re, repl, txt, f"Unable to find SUBPROTOCOL_VERSION define in {fn}")
         elif fn == 'NEWS.md':
-            x_re = re.compile(
-                    r'^(# NEWS for rsync %s )(\(UNRELEASED\))\s*(\n\nProtocol: )(\d+) (\([^)]+\))\n' % re.escape(finalversion),
-                    re.I)
-            repl = lambda m: m[1] + (m[2] if pre else f"({today})") + m[3] + f"{protocol_version} ({proto_changed})\n"
-            msg = (f"The first 3 lines of {fn} are not in the right format.  They must be:\n"
-                    + f"# NEWS for rsync {finalversion} (UNRELEASED)\n\n"
-                    + f"Protocol: {protocol_version} ({proto_changed})")
-            txt = replace_or_die(x_re, repl, txt, msg)
-        elif fn == 'OLDNEWS.md':
             efv = re.escape(finalversion)
+            x_re = re.compile(r'^<.+>\s+# NEWS for rsync %s \(UNRELEASED\)\s+## Changes in this version:\n' % efv
+                    + r'(\n### PROTOCOL NUMBER:\s+- The protocol number was changed to \d+\.\n)?')
+            rel_day = 'UNRELEASED' if pre else today
+            repl = (f'<a name="{finalversion}"></a>\n\n# NEWS for rsync {finalversion} ({rel_day})\n\n'
+                + '## Changes in this version:\n')
+            if proto_changed:
+                repl += f'\n### PROTOCOL NUMBER:\n\n - The protocol number was changed to {protocol_version}.\n'
+            good_top = re.sub(r'\(.*?\)', '(UNRELEASED)', repl, 1)
+            msg = f"The top lines of {fn} are not in the right format.  It should be:\n" + good_top
+            txt = replace_or_die(x_re, repl, txt, msg)
             x_re = re.compile(r'^(\| )(\S{2} \S{3} \d{4})(\s+\|\s+%s\s+\| ).{11}(\s+\| )\S{2}(\s+\|+)$' % efv, re.M)
             repl = lambda m: m[1] + (m[2] if pre else ztoday) + m[3] + proto_change_date + m[4] + protocol_version + m[5]
             txt = replace_or_die(x_re, repl, txt, f'Unable to find "| ?? ??? {year} | {finalversion} | ... |" line in {fn}')
@@ -216,11 +221,10 @@ About to:
     srctar_name = f"{rsync_ver}.tar.gz"
     pattar_name = f"rsync-patches-{version}.tar.gz"
     diff_name = f"{rsync_lastver}-{version}.diffs.gz"
-    srctar_file = f"{dest}/{srcdir}/{srctar_name}"
-    pattar_file = f"{dest}/{srcdir}/{pattar_name}"
-    diff_file = f"{dest}/{srcdiffdir}/{diff_name}"
-    news_file = f"{dest}/{srcdir}/{rsync_ver}-NEWS.md"
-    lasttar_file = f"{dest}/{lastsrcdir}/{rsync_lastver}.tar.gz"
+    srctar_file = os.path.join(dest, srcdir, srctar_name)
+    pattar_file = os.path.join(dest, srcdir, pattar_name)
+    diff_file = os.path.join(dest, srcdiffdir, diff_name)
+    lasttar_file = os.path.join(dest, lastsrcdir, rsync_lastver + '.tar.gz')
 
     print(f"""\
 {dash_line}
@@ -238,8 +242,7 @@ About to:
     if s.returncode:
         die('Aborting')
 
-    cmd_chk('make reconfigure ; make gen')
-    cmd_chk(['rsync', '-a', *gen_files, 'SaVeDiR/'])
+    cmd_chk('make gen')
 
     print(f'Creating any missing patch branches.')
     s = cmd_run(f'packaging/branch-from-patch --branch={args.master_branch} --add-missing')
@@ -254,10 +257,6 @@ About to:
     if re.match(r'^y', ans, re.I):
         print(f'\nVisiting all "patch/{args.master_branch}/*" branches ...')
         cmd_run(f"packaging/patch-update --branch={args.master_branch} --skip-check --shell")
-        cmd_run("rm -f *.[o15] *.html")
-        cmd_chk('rsync -a SaVeDiR/ .'.split())
-        shutil.rmtree('SaVeDiR')
-        cmd_chk('make gen'.split())
 
     if os.path.isdir('patches/.git'):
         s = cmd_run(f"cd patches && git commit -a -m 'The patches for {version}.'")
@@ -273,7 +272,7 @@ About to:
     - create release tar, "{srctar_name}"
     - generate {rsync_ver}/patches/* files
     - create patches tar, "{pattar_name}"
-    - update top-level README.md, *NEWS.md, TODO, and ChangeLog
+    - update top-level README.md, NEWS.md, TODO, and ChangeLog
     - update top-level rsync*.html manpages
     - gpg-sign the release files
     - update hard-linked top-level release files{skipping}
@@ -297,12 +296,12 @@ About to:
     os.environ['PATH'] = ORIGINAL_PATH
 
     # Extract the generated files from the old tar.
-    tweaked_gen_files = [ f"{rsync_lastver}/{x}" for x in gen_files ]
+    tweaked_gen_files = [ os.path.join(rsync_lastver, fn) for fn in gen_files ]
     cmd_run(['tar', 'xzf', lasttar_file, *tweaked_gen_files])
     os.rename(rsync_lastver, 'a')
 
     print(f"Creating {diff_file} ...")
-    cmd_chk(['rsync', '-a', *gen_files, 'b/'])
+    cmd_chk(['rsync', '-a', *gen_pathnames, 'b/'])
 
     sed_script = r's:^((---|\+\+\+) [ab]/[^\t]+)\t.*:\1:' # CAUTION: must not contain any single quotes!
     cmd_chk(f"(git diff v{lastversion} {v_ver} -- ':!.github'; diff -upN a b | sed -r '{sed_script}') | gzip -9 >{diff_file}")
@@ -325,15 +324,12 @@ About to:
     shutil.rmtree(rsync_ver)
 
     print(f"Updating the other files in {dest} ...")
-    cmd_chk('rsync -a README.md NEWS.md OLDNEWS.md TODO'.split() + [dest])
-    if os.path.lexists(news_file):
-        os.unlink(news_file)
-    os.link(f"{dest}/NEWS.md", news_file)
-    cmd_chk(f"git log --name-status | gzip -9 >{dest}/ChangeLog.gz")
+    md_files = 'README.md NEWS.md'.split()
+    html_files = [ fn for fn in gen_pathnames if fn.endswith('.html') ]
+    cmd_chk(['rsync', '-a', *md_files, *html_files, dest])
+    cmd_chk(["packaging/md2html"] + [ dest +'/'+ fn for fn in md_files ])
 
-    for md_fn in glob.glob('*.[1-9].md'):
-        html_fn = md_fn.replace('.md', '.html')
-        cmd_chk(['rsync', '-a', html_fn, os.path.join(dest, html_fn)])
+    cmd_chk(f"git log --name-status | gzip -9 >{dest}/ChangeLog.gz")
 
     for fn in (srctar_file, pattar_file, diff_file):
         asc_fn = fn + '.asc'
@@ -344,14 +340,13 @@ About to:
             die("gpg signing failed")
 
     if not pre:
-        for find in f'{dest}/rsync-*.gz {dest}/rsync-*.asc {dest}/rsync-*-NEWS.md {dest}/src-previews/rsync-*diffs.gz*'.split():
+        for find in f'{dest}/rsync-*.gz {dest}/rsync-*.asc {dest}/src-previews/rsync-*diffs.gz*'.split():
             for fn in glob.glob(find):
                 os.unlink(fn)
         top_link = [
                 srctar_file, f"{srctar_file}.asc",
                 pattar_file, f"{pattar_file}.asc",
                 diff_file, f"{diff_file}.asc",
-                news_file,
                 ]
         for fn in top_link:
             os.link(fn, re.sub(r'/src(-\w+)?/', '/', fn))
@@ -383,4 +378,4 @@ if __name__ == '__main__':
     args = parser.parse_args()
     main()
 
-# vim: sw=4 et
+# vim: sw=4 et ft=python

packaging/systemd/rsync.service

@@ -1,9 +1,31 @@
 [Unit]
 Description=fast remote file copy program daemon
 ConditionPathExists=/etc/rsyncd.conf
+After=network.target
+Documentation=man:rsync(1) man:rsyncd.conf(5)
 
 [Service]
 ExecStart=/usr/bin/rsync --daemon --no-detach
+RestartSec=1
+
+# Citing README.md:
+#
+#   [...] Using ssh is recommended for its security features.
+#
+#   Alternatively, rsync can run in `daemon' mode, listening on a socket.
+#   This is generally used for public file distribution, [...]
+#
+# So let's assume some extra security is more than welcome here. We do full
+# system protection (which makes /usr, /boot, & /etc read-only) and hide
+# devices. To override these defaults, it's best to do so in the drop-in
+# directory, often done via `systemctl edit rsync.service`. The file needs
+# just the bare minimum of the right [heading] and override values.
+# See systemd.unit(5) and search for "drop-in" for full details.
+
+ProtectSystem=full
+#ProtectHome=on|off|read-only
+PrivateDevices=on
+NoNewPrivileges=on
 
 [Install]
 WantedBy=multi-user.target

packaging/systemd/rsync.socket

@@ -0,0 +1,10 @@
+[Unit]
+Description=socket for fast remote file copy program daemon
+Conflicts=rsync.service
+
+[Socket]
+ListenStream=873
+Accept=true
+
+[Install]
+WantedBy=sockets.target

packaging/systemd/rsync@.service

@@ -0,0 +1,28 @@
+[Unit]
+Description=fast remote file copy program daemon
+After=network.target
+
+[Service]
+ExecStart=-/usr/bin/rsync --daemon
+StandardInput=socket
+StandardOutput=inherit
+StandardError=journal
+
+# Citing README.md:
+#
+#   [...] Using ssh is recommended for its security features.
+#
+#   Alternatively, rsync can run in `daemon' mode, listening on a socket.
+#   This is generally used for public file distribution, [...]
+#
+# So let's assume some extra security is more than welcome here. We do full
+# system protection (which makes /usr, /boot, & /etc read-only) and hide
+# devices. To override these defaults, it's best to do so in the drop-in
+# directory, often done via `systemctl edit rsync@.service`. The file needs
+# just the bare minimum of the right [heading] and override values.
+# See systemd.unit(5) and search for "drop-in" for full details.
+
+ProtectSystem=full
+#ProtectHome=on|off|read-only
+PrivateDevices=on
+NoNewPrivileges=on

packaging/var-checker

@@ -1,4 +1,4 @@
-#!/usr/bin/python3 -B
+#!/usr/bin/env -S python3 -B
 
 # This script checks the *.c files for extraneous "extern" variables,
 # for vars that are defined but not used, and for inconsistent array
@@ -84,4 +84,4 @@ if __name__ == '__main__':
     args = parser.parse_args()
     main()
 
-# vim: sw=4 et
+# vim: sw=4 et ft=python

packaging/year-tweak

@@ -1,4 +1,4 @@
-#!/usr/bin/python3
+#!/usr/bin/env python3
 
 # This uses the output from "support/git-set-file-times --list" to discern
 # the last-modified year of each *.c & *.h file and updates the copyright

params.c

@@ -212,11 +212,6 @@ static BOOL Section( FILE *InFile, BOOL (*sfunc)(char *) )
       {
       bSize += BUFR_INC;
       bufr   = realloc_array( bufr, char, bSize );
-      if( NULL == bufr )
-        {
-        rprintf(FLOG, "%s Memory re-allocation failure.", func);
-        return( False );
-        }
       }
 
     /* Handle a single character. */
@@ -306,11 +301,6 @@ static BOOL Parameter( FILE *InFile, BOOL (*pfunc)(char *, char *), int c )
       {
       bSize += BUFR_INC;
       bufr   = realloc_array( bufr, char, bSize );
-      if( NULL == bufr )
-        {
-        rprintf(FLOG, "%s Memory re-allocation failure.", func) ;
-        return( False );
-        }
       }
 
     switch( c )
@@ -382,11 +372,6 @@ static BOOL Parameter( FILE *InFile, BOOL (*pfunc)(char *, char *), int c )
       {
       bSize += BUFR_INC;
       bufr   = realloc_array( bufr, char, bSize );
-      if( NULL == bufr )
-        {
-        rprintf(FLOG, "%s Memory re-allocation failure.", func) ;
-        return( False );
-        }
       }
 
     switch( c )
@@ -639,12 +624,6 @@ int pm_process( char *FileName,
     {                                         /* allocate one, then parse,   */
     bSize = BUFR_INC;                         /* then free.                  */
     bufr = new_array( char, bSize );
-    if( NULL == bufr )
-      {
-      rprintf(FLOG, "%s memory allocation failure.\n", func);
-      fclose(InFile);
-      return( False );
-      }
     result = Parse( InFile, sfunc, pfunc );
     free( bufr );
     bufr  = NULL;

prepare-source

@@ -16,6 +16,26 @@ if test x"$dir" = x; then
     dir=.
 fi
 
+if test "$dir" = '.'; then
+    branch=`packaging/prep-auto-dir` || exit 1
+    if test x"$branch" != x; then
+	cd build || exit 1
+	dir=..
+    fi
+fi
+
+if test "$dir" != '.'; then
+    for lnk in configure.ac m4; do
+	if test ! -h $lnk; then
+	    rm -f $lnk # Just in case
+	    ln -s "$dir/$lnk" $lnk
+	fi
+    done
+    for fn in configure.sh config.h.in aclocal.m4; do
+	test ! -f $fn -a -f "$dir/$fn" && cp -p "$dir/$fn" $fn
+    done
+fi
+
 if test $# = 0; then
     set -- build
 fi
@@ -23,17 +43,18 @@ fi
 for action in "${@}"; do
     case "$action" in
     build|make)
-	(cd $dir && make -f prepare-source.mak)
+	make -f "$dir/prepare-source.mak"
 	;;
-    fetch)
-	$dir/rsync-ssl -iip --no-motd rsync://download.samba.org/rsyncftp/generated-files/'[ca]*' $dir
-	if ! perl --version >/dev/null 2>/dev/null; then
-	    $dir/rsync-ssl -iip --no-motd rsync://download.samba.org/rsyncftp/generated-files/'p*' .
+    fetch|fetchgen)
+	if test "$action" = fetchgen; then
+	    match='*'
+	else
+	    match='[ca]*'
 	fi
-	;;
-    fetchgen)
-	$dir/rsync-ssl -iip --no-motd rsync://download.samba.org/rsyncftp/generated-files/'[ca]*' $dir
-	$dir/rsync-ssl -iip --no-motd rsync://download.samba.org/rsyncftp/generated-files/'[^ca]*' .
+	$dir/rsync-ssl -iipc --no-motd "rsync://download.samba.org/rsyncftp/generated-files/$match" ./
+	test $? != 0 && continue
+	sleep 1 # The following files need to be newer than aclocal.m4
+	touch configure.sh config.h.in
 	;;
     fetchSRC)
 	./rsync-ssl -iipr --no-motd --exclude=/.git/ rsync://download.samba.org/ftp/pub/unpacked/rsync/ .
@@ -41,6 +62,7 @@ for action in "${@}"; do
     *)
 	echo "Unknown action: $action"
 	exit 1
+	;;
     esac
     if test $? = 0; then
 	exit

prepare-source.mak

@@ -1,3 +1,6 @@
+SHELL=/bin/sh
+
+.PHONY: conf
 conf: configure.sh config.h.in
 
 aclocal.m4: m4/*.m4

rounding.c

@@ -24,11 +24,11 @@
 
 struct test {
 	union file_extras extras[ARRAY_LEN];
-	struct file_struct file;
+	int64 test;
 };
 
 #define ACTUAL_SIZE	SIZEOF(struct test)
-#define EXPECTED_SIZE	(SIZEOF(union file_extras) * ARRAY_LEN + SIZEOF(struct file_struct))
+#define EXPECTED_SIZE	(SIZEOF(union file_extras) * ARRAY_LEN + SIZEOF(int64))
 
  int main(UNUSED(int argc), UNUSED(char *argv[]))
 {

rsync-ssl

@@ -1,6 +1,6 @@
-#!/bin/bash
+#!/usr/bin/env bash
 
-# This script supports using stunnel or openssl to secure an rsync daemon connection.
+# This script uses openssl, gnutls, or stunnel to secure an rsync daemon connection.
 
 # By default this script takes rsync args and hands them off to the actual
 # rsync command with an --rsh option that makes it open an SSL connection to an
@@ -31,10 +31,13 @@ function rsync_ssl_run {
 
 function rsync_ssl_helper {
     if [[ -z "$RSYNC_SSL_TYPE" ]]; then
-	found=`path_search stunnel4 stunnel openssl` || exit 1
+	found=`path_search openssl stunnel4 stunnel` || exit 1
 	if [[ "$found" == */openssl ]]; then
 	    RSYNC_SSL_TYPE=openssl
 	    RSYNC_SSL_OPENSSL="$found"
+	elif [[ "$found" == */gnutls-cli ]]; then
+	    RSYNC_SSL_TYPE=gnutls
+	    RSYNC_SSL_GNUTLS="$found"
 	else
 	    RSYNC_SSL_TYPE=stunnel
 	    RSYNC_SSL_STUNNEL="$found"
@@ -48,6 +51,12 @@ function rsync_ssl_helper {
 	    fi
 	    optsep=' '
 	    ;;
+	gnutls)
+	    if [[ -z "$RSYNC_SSL_GNUTLS" ]]; then
+		RSYNC_SSL_GNUTLS=`path_search gnutls-cli` || exit 1
+	    fi
+	    optsep=' '
+	    ;;
 	stunnel)
 	    if [[ -z "$RSYNC_SSL_STUNNEL" ]]; then
 		RSYNC_SSL_STUNNEL=`path_search stunnel4 stunnel` || exit 1
@@ -62,31 +71,42 @@ function rsync_ssl_helper {
 
     if [[ -z "$RSYNC_SSL_CERT" ]]; then
 	certopt=""
+	gnutls_cert_opt=""
     else
 	certopt="cert$optsep$RSYNC_SSL_CERT"
+	gnutls_cert_opt="--x509keyfile=$RSYNC_SSL_CERT"
     fi
 
     if [[ -z ${RSYNC_SSL_CA_CERT+x} ]]; then
 	# RSYNC_SSL_CA_CERT unset - default CA set AND verify:
 	# openssl:
 	caopt="-verify_return_error -verify 4"
+	# gnutls:
+	gnutls_opts=""
 	# stunnel:
+	# Since there is no way of using the default CA certificate collection,
+	# we cannot do any verification. Thus, stunnel should really only be
+	# used if nothing else is available.
 	cafile=""
-	verify=0
+	verify=""
     elif [[ "$RSYNC_SSL_CA_CERT" == "" ]]; then
 	# RSYNC_SSL_CA_CERT set but empty -do NO verifications:
 	# openssl:
 	caopt="-verify 1"
+	# gnutls:
+	gnutls_opts="--insecure"
 	# stunnel:
 	cafile=""
-	verify=0
+	verify="verifyChain = no"
     else
 	# RSYNC_SSL_CA_CERT set - use CA AND verify:
 	# openssl:
 	caopt="-CAfile $RSYNC_SSL_CA_CERT -verify_return_error -verify 4"
+	# gnutls:
+	gnutls_opts="--x509cafile=$RSYNC_SSL_CA_CERT"
 	# stunnel:
 	cafile="CAfile = $RSYNC_SSL_CA_CERT"
-	verify=3
+	verify="verifyChain = yes"
     fi
 
     port="${RSYNC_PORT:-0}"
@@ -110,6 +130,8 @@ function rsync_ssl_helper {
 
     if [[ $RSYNC_SSL_TYPE == openssl ]]; then
 	exec $RSYNC_SSL_OPENSSL s_client $caopt $certopt -quiet -verify_quiet -servername $hostname -connect $hostname:$port
+    elif [[ $RSYNC_SSL_TYPE == gnutls ]]; then
+	exec $RSYNC_SSL_GNUTLS --logfile=/dev/null $gnutls_cert_opt $gnutls_opts $hostname:$port
     else
 	# devzero@web.de came up with this no-tmpfile calling syntax:
 	exec $RSYNC_SSL_STUNNEL -fd 10 11<&0 <<EOF 10<&0 0<&11 11<&-
@@ -118,7 +140,7 @@ debug = crit
 connect = $hostname:$port
 client = yes
 TIMEOUTclose = 0
-verify = $verify
+$verify
 $certopt
 $cafile
 EOF
@@ -146,7 +168,8 @@ function path_search {
 }
 
 if [[ "$#" == 0 ]]; then
-    echo "Usage: rsync-ssl [--type=openssl|stunnel] RSYNC_ARG [...]" 1>&2
+    echo "Usage: rsync-ssl [--type=SSL_TYPE] RSYNC_ARG [...]" 1>&2
+    echo "The SSL_TYPE can be openssl or stunnel"
     exit 1
 fi
 

rsync-ssl.1.md

@@ -5,7 +5,7 @@ rsync-ssl - a helper script for connecting to an ssl rsync daemon
 # SYNOPSIS
 
 ```
-rsync-ssl [--type=openssl|stunnel] RSYNC_ARGS
+rsync-ssl [--type=SSL_TYPE] RSYNC_ARGS
 ```
 
 # DESCRIPTION
@@ -13,26 +13,32 @@ rsync-ssl [--type=openssl|stunnel] RSYNC_ARGS
 The rsync-ssl script helps you to run an rsync copy to/from an rsync daemon
 that requires ssl connections.
 
-If the **first** arg is a `--type=NAME` option, the script will only use that
-particular program to open an ssl connection instead of trying to find an
-stunnel or openssl executable via a simple heuristic (assuming that the
+The script requires that you specify an rsync-daemon arg in the style of either
+`hostname::` (with 2 colons) or `rsync://hostname/`.  The default port used for
+connecting is 874 (one higher than the normal 873) unless overridden in the
+environment.  You can specify an overriding port via `--port` or by including
+it in the normal spot in the URL format, though both of those require your
+rsync version to be at least 3.2.0.
+
+# OPTIONS
+
+If the **first** arg is a `--type=SSL_TYPE` option, the script will only use
+that particular program to open an ssl connection instead of trying to find an
+openssl or stunnel executable via a simple heuristic (assuming that the
 `RSYNC_SSL_TYPE` environment variable is not set as well -- see below).  This
-option must be one of `--type=openssl` or `--type=stunnel`.  The equal sign is
+option must specify one of `openssl` or `stunnel`.  The equal sign is
 required for this particular option.
 
 All the other options are passed through to the rsync command, so consult the
-**rsync** manpage for more information on how it works.
-
-Note that the stunnel connection type requires at least version 4 of stunnel,
-which should be the case on modern systems.
+**rsync**(1) manpage for more information on how it works.
 
 # ENVIRONMENT VARIABLES
 
 The ssl helper scripts are affected by the following environment variables:
 
 0.  `RSYNC_SSL_TYPE` Specifies the program type that should be used to open the
-    ssl connection.  It must be one of "openssl" or "stunnel".  The
-    `--type=NAME` option overrides this, if specified.
+    ssl connection.  It must be one of `openssl` or `stunnel`.  The
+    `--type=SSL_TYPE` option overrides this, when specified.
 0.  `RSYNC_SSL_PORT` If specified, the value is the port number that is used as
     the default when the user does not specify a port in their rsync command.
     When not specified, the default port number is 874.  (Note that older rsync
@@ -42,26 +48,46 @@ The ssl helper scripts are affected by the following environment variables:
     certificate to use for the connection.
 0.  `RSYNC_SSL_CA_CERT` If specified, the value is a filename that contains a
     certificate authority certificate that is used to validate the connection.
-0.  `RSYNC_SSL_STUNNEL` Specifies the stunnel executable to run when the
-    connection type is set to stunnel.  If unspecified, the $PATH is searched
-    first for "stunnel4" and then for "stunnel".
 0.  `RSYNC_SSL_OPENSSL` Specifies the openssl executable to run when the
     connection type is set to openssl.  If unspecified, the $PATH is searched
     for "openssl".
+0.  `RSYNC_SSL_GNUTLS` Specifies the gnutls-cli executable to run when the
+    connection type is set to gnutls.  If unspecified, the $PATH is searched
+    for "gnutls-cli".
+0.  `RSYNC_SSL_STUNNEL` Specifies the stunnel executable to run when the
+    connection type is set to stunnel.  If unspecified, the $PATH is searched
+    first for "stunnel4" and then for "stunnel".
 
 # EXAMPLES
 
->     rsync-ssl -aiv example.com::src/ dest
+>     rsync-ssl -aiv example.com::mod/ dest
 
->     rsync-ssl --type=openssl -aiv example.com::src/ dest
+>     rsync-ssl --type=openssl -aiv example.com::mod/ dest
+
+>     rsync-ssl -aiv --port 9874 example.com::mod/ dest
+
+>     rsync-ssl -aiv rsync://example.com:9874/mod/ dest
 
 # SEE ALSO
 
 **rsync**(1), **rsyncd.conf**(5)
 
+# CAVEATS
+
+Note that using an stunnel connection requires at least version 4 of stunnel,
+which should be the case on modern systems.  Also, it does not verify a
+connection against the CA certificate collection, so it only encrypts the
+connection without any cert validation unless you have specified the
+certificate environment options.
+
+This script also supports a `--type=gnutls` option, but at the time of this
+release the gnutls-cli command was dropping output, making it unusable.  If
+that bug has been fixed in your version, feel free to put gnutls into an
+exported RSYNC_SSL_TYPE environment variable to make its use the default.
+
 # BUGS
 
-Please report bugs! See the web site at <http://rsync.samba.org/>.
+Please report bugs! See the web site at <https://rsync.samba.org/>.
 
 # VERSION
 
@@ -72,7 +98,7 @@ This man page is current for version @VERSION@ of rsync.
 rsync is distributed under the GNU General Public License.  See the file
 COPYING for details.
 
-A web site is available at <http://rsync.samba.org/>.  The site includes an
+A web site is available at <https://rsync.samba.org/>.  The site includes an
 FAQ-O-Matic which may cover questions unanswered by this manual page.
 
 # AUTHOR
@@ -80,4 +106,4 @@ FAQ-O-Matic which may cover questions unanswered by this manual page.
 This manpage was written by Wayne Davison.
 
 Mailing lists for support and development are available at
-<http://lists.samba.org/>.
+<https://lists.samba.org/>.

rsync.1.md

@@ -70,14 +70,14 @@ RSYNC-DAEMON FEATURES VIA A REMOTE-SHELL CONNECTION" section for an exception
 to this latter rule).
 
 As a special case, if a single source arg is specified without a destination,
-the files are listed in an output format similar to "ls -l".
+the files are listed in an output format similar to "`ls -l`".
 
 As expected, if neither the source or destination path specify a remote host,
 the copy occurs locally (see also the `--list-only` option).
 
-Rsync refers to the local side as the "client" and the remote side as the
-"server".  Don't confuse "server" with an rsync daemon -- a daemon is always a
-server, but a server can be either a daemon or a remote-shell spawned process.
+Rsync refers to the local side as the client and the remote side as the server.
+Don't confuse server with an rsync daemon.  A daemon is always a server, but a
+server can be either a daemon or a remote-shell spawned process.
 
 # SETUP
 
@@ -115,7 +115,7 @@ same as all other posix-style programs).
 
 This would recursively transfer all files from the directory src/bar on the
 machine foo into the /data/tmp/bar directory on the local machine.  The files
-are transferred in "archive" mode, which ensures that symbolic links, devices,
+are transferred in archive mode, which ensures that symbolic links, devices,
 attributes, permissions, ownerships, etc. are preserved in the transfer.
 Additionally, compression will be used to reduce the size of data portions of
 the transfer.
@@ -381,7 +381,7 @@ detailed description below for a complete description.
 --write-devices          write to devices as files (implies --inplace)
 --dry-run, -n            perform a trial run with no changes made
 --whole-file, -W         copy files whole (w/o delta-xfer algorithm)
---checksum-choice=STR    choose the checksum algorithms
+--checksum-choice=STR    choose the checksum algorithm (aka --cc)
 --one-file-system, -x    don't cross filesystem boundaries
 --block-size=SIZE, -B    force a fixed checksum block-size
 --rsh=COMMAND, -e        specify the remote shell to use
@@ -403,6 +403,7 @@ detailed description below for a complete description.
 --max-delete=NUM         don't delete more than NUM files
 --max-size=SIZE          don't transfer any file larger than SIZE
 --min-size=SIZE          don't transfer any file smaller than SIZE
+--max-alloc=SIZE         change a limit relating to memory alloc
 --partial                keep partially transferred files
 --partial-dir=DIR        put a partially transferred file into DIR
 --delay-updates          put all updated files into place at end
@@ -422,7 +423,8 @@ detailed description below for a complete description.
 --copy-dest=DIR          ... and include copies of unchanged files
 --link-dest=DIR          hardlink to files in DIR when unchanged
 --compress, -z           compress file data during the transfer
---compress-level=NUM     explicitly set compression level
+--compress-choice=STR    choose the compression algorithm (aka --zc)
+--compress-level=NUM     explicitly set compression level (aka --zl)
 --skip-compress=LIST     skip compressing files with suffix in LIST
 --cvs-exclude, -C        auto-ignore files in the same way CVS does
 --filter=RULE, -f        add a file-filtering RULE
@@ -452,6 +454,7 @@ detailed description below for a complete description.
 --log-file=FILE          log what we're doing to the specified FILE
 --log-file-format=FMT    log updates using the specified FMT
 --password-file=FILE     read daemon-access password from FILE
+--early-input=FILE       use FILE for daemon's early exec input
 --list-only              list the files instead of copying them
 --bwlimit=RATE           limit socket I/O bandwidth
 --write-batch=FILE       write a batched update to FILE
@@ -498,7 +501,7 @@ parameter, the parameter is only listed after the long variant, even though it
 must also be specified for the short.  When specifying a parameter, you can
 either use the form `--option=param` or replace the '=' with whitespace.  The
 parameter may need to be quoted in some manner for it to survive the shell's
-command-line parsing.  Keep in mind that a leading tilde (\~) in a filename is
+command-line parsing.  Keep in mind that a leading tilde (`~`) in a filename is
 substituted by your shell, so `--option=~/foo` will not change the tilde into
 your home directory (remove the '=' for that).
 
@@ -507,8 +510,8 @@ your home directory (remove the '=' for that).
 0.  `--help`, `-h` `(*)`
 
     Print a short help page describing the options available in rsync and exit.
-    For backward-compatibility with older versions of rsync, the help will also
-    be output if you use the `-h` option without any other args.
+    (*) The `-h` short option will only invoke `--help` when used without other
+    options since it normally means `--human-readable`.
 
 0.  `--version`, `-V`
 
@@ -534,7 +537,7 @@ your home directory (remove the '=' for that).
     and `--debug` have a way to ask for help that tells you exactly what flags
     are set for each increase in verbosity.
 
-    However, do keep in mind that a daemon's "max verbosity" setting will limit
+    However, do keep in mind that a daemon's "`max verbosity`" setting will limit
     how high of a level the various individual flags can be set on the daemon
     side.  For instance, if the max is 2, then any info and/or debug flag that
     is set to a higher value than what would be set by `-vv` will be downgraded
@@ -560,7 +563,7 @@ your home directory (remove the '=' for that).
     This option was added to 3.1.0, so an older rsync on the server side might
     reject your attempts at fine-grained control (if one or more flags needed
     to be send to the server and the server was too old to understand them).
-    See also the "max verbosity" caveat above when dealing with a daemon.
+    See also the "`max verbosity`" caveat above when dealing with a daemon.
 
 0.  `--debug=FLAGS`
 
@@ -595,12 +598,12 @@ your home directory (remove the '=' for that).
     but it can delay messages behind a slew of data.
 
     One case where this is helpful is when sending really large files, since
-    errors that happen on a remote receiver tend to get delayed until afer the
-    file's data is fully sent.  It is also helpful for debugging, since it helps
-    to avoid overpopulating the protocol data with extra message data.
+    errors that happen on a remote receiver tend to get delayed until after the
+    file's data is fully sent.  It is also helpful for debugging, since it
+    helps to avoid overpopulating the protocol data with extra message data.
 
     The option does not affect the remote side of a transfer without using
-    `--remote-option` -- e.g. `-M--msgs2stderr` or `{-M,}--msgs2stderr`.
+    `--remote-option`, e.g. `-M--msgs2stderr` or `{-M,}--msgs2stderr`.
 
     Also keep in mind that connecting to a normal (non-remote-shell) daemon
     does not have a stderr channel to send messages back to the client side, so
@@ -640,7 +643,7 @@ your home directory (remove the '=' for that).
     after using another mirroring system which may not preserve timestamps
     exactly.
 
-0.  `--modify-window`, `-@`
+0.  `--modify-window=NUM`, `-@`
 
     When comparing two timestamps, rsync treats the timestamps as being equal
     if they differ by no more than the modify-window value.  The default is 0,
@@ -685,7 +688,7 @@ your home directory (remove the '=' for that).
     before-the-transfer "Does this file need to be updated?" check.
 
     The checksum used is auto-negotiated between the client and the server, but
-    can be overridden using either the `--checksum-choice` option or an
+    can be overridden using either the `--checksum-choice` (`--cc`) option or an
     environment variable that is discussed in that option's section.
 
 0.  `--archive`, `-a`
@@ -838,7 +841,7 @@ your home directory (remove the '=' for that).
 
 0.  `--backup-dir=DIR`
 
-    In combination with the `--backup` option, this tells rsync to store all
+    This implies the `--backup` option, and tells rsync to store all
     backups in the specified directory on the receiving side.  This can be used
     for incremental backups.  You can additionally specify a backup suffix
     using the `--suffix` option (otherwise the files backed up in the specified
@@ -1023,7 +1026,7 @@ your home directory (remove the '=' for that).
     local transfer, the client side is the sender.)
 
     This option has no affect on a daemon, since the daemon configures whether
-    it wants munged symlinks via its "munge symlinks" parameter.  See also the
+    it wants munged symlinks via its "`munge symlinks`" parameter.  See also the
     "munge-symlinks" perl script in the support directory of the source code.
 
 0.  `--copy-dirlinks`, `-k`
@@ -1221,7 +1224,7 @@ your home directory (remove the '=' for that).
     those used by `--fake-super`) unless you repeat the option (e.g. `-XX`).
     This "copy all xattrs" mode cannot be used with `--fake-super`.
 
-0.  `--chmod`
+0.  `--chmod=CHMOD`
 
     This option tells rsync to apply one or more comma-separated "chmod" modes
     to the permission of the files in the transfer.  The resulting value is
@@ -1400,7 +1403,7 @@ your home directory (remove the '=' for that).
 
     This option is overridden by both `--super` and `--no-super`.
 
-    See also the "fake super" setting in the daemon's rsyncd.conf file.
+    See also the "`fake super`" setting in the daemon's rsyncd.conf file.
 
 0.  `--sparse`, `-S`
 
@@ -1468,13 +1471,16 @@ your home directory (remove the '=' for that).
 
     The checksum options that you may be able to use are:
 
-    - `auto` (the default)
-    - `xxh64` (aka xxhash)
+    - `auto` (the default automatic choice)
+    - `xxh128`
+    - `xxh3`
+    - `xxh64` (aka `xxhash`)
     - `md5`
     - `md4`
     - `none`
 
-    Run `rsync -V` to see the default checksum list compiled into your version.
+    Run `rsync --version` to see the default checksum list compiled into your
+    version (which may differ from the list above).
 
     If "none" is specified for the first (or only) name, the `--whole-file`
     option is forced on and no checksum verification is performed on the
@@ -1482,24 +1488,24 @@ your home directory (remove the '=' for that).
     the `--checksum` option cannot be used.
 
     The "auto" option is the default, where rsync bases its algorithm choice on
-    a negotation between the client and the server as follows:
+    a negotiation between the client and the server as follows:
 
-    If both the client and the server are at least version 3.2.0, they will
-    exchange a list of checksum names and choose the first one in the list that
-    they have in common.  This typically means that they will choose xxh64 if
-    they both support it and fall back to MD5.  If one side of the transfer is
-    not new enough to support this checksum negotation, then a value is chosen
-    based on the protocol version (which chooses between MD5 and various
-    flavors of MD4 based on protocol age).
+    When both sides of the transfer are at least 3.2.0, rsync chooses the first
+    algorithm in the client's list of choices that is also in the server's list
+    of choices.  If no common checksum choice is found, rsync exits with
+    an error.  If the remote rsync is too old to support checksum negotiation,
+    a value is chosen based on the protocol version (which chooses between MD5
+    and various flavors of MD4 based on protocol age).
 
-    You can also override the checksum using the RSYNC_CHECKSUM_LIST
-    environment variable by setting it to a space-separated list of checksum
-    names that you consider acceptable.  If no common checksum is found, the
-    client exits with an error.  This method does not allow you to specify the
-    transfer checksum separately from the pre-transfer checksum, and it ignores
-    "auto" and all unknown checksum names.  If the remote rsync is not new
-    enough to handle a checksum negotiation list, the list is silently ignored
-    unless it contains the string "FAIL".
+    The default order can be customized by setting the environment variable
+    RSYNC_CHECKSUM_LIST to a space-separated list of acceptable checksum names.
+    If the string contains a "`&`" character, it is separated into the "client
+    string & server string", otherwise the same string
+    applies to both.  If the string (or string portion) contains no
+    non-whitespace characters, the default checksum list is used.  This method
+    does not allow you to specify the transfer checksum separately from the
+    pre-transfer checksum, and it discards "auto" and all unknown checksum
+    names.  A list with only invalid names results in a failed negotiation.
 
     The use of the `--checksum-choice` option overrides this environment list.
 
@@ -1726,12 +1732,16 @@ your home directory (remove the '=' for that).
     data that goes into the file-lists, and thus it doesn't affect deletions.
     It just limits the files that the receiver requests to be transferred.
 
-    The suffixes are as follows: "K" (or "KiB") is a kibibyte (1024), "M" (or
-    "MiB") is a mebibyte (1024\*1024), and "G" (or "GiB") is a gibibyte
-    (1024\*1024\*1024).  If you want the multiplier to be 1000 instead of 1024,
-    use "KB", "MB", or "GB". (Note: lower-case is also accepted for all
-    values.) Finally, if the suffix ends in either "+1" or "-1", the value will
-    be offset by one byte in the indicated direction.
+    The accepted suffix letters are: `B`, `K`, `G`, `T`, and `P` for bytes,
+    kilobytes/kibibytes, megabytes/mebibytes, gigabytes/gibibytes,
+    terabytes/tebibytes, and petabytes/pebibytes.  If you use a single-char
+    suffix or add-on "ib" to it (e.g. "G" or "GiB") then you get units that are
+    multiples of 1024.  If you use a two-letter suffix that ends with a "B"
+    (e.g. "kb") then you get units that are multiples of 1000.  The suffix
+    letters can be any mix of upper and lower-case that you want to use.
+
+    Finally, if the string ends with either "+1" or "-1", it is offset by one
+    byte in the indicated direction.  The largest possible value is `8192P-1`.
 
     Examples: `--max-size=1.5mb-1` is 1499999 bytes, and `--max-size=2g+1` is
     2147483649 bytes.
@@ -1746,7 +1756,29 @@ your home directory (remove the '=' for that).
 
     Note that rsync versions prior to 3.1.0 did not allow `--min-size=0`.
 
-0.  `--block-size=BLOCKSIZE`, `-B`
+0.  `--max-alloc=SIZE`
+
+    By default rsync limits an individual malloc/realloc to about 1GB in size.
+    For most people this limit works just fine and prevents a protocol error
+    causing rsync to request massive amounts of memory.  However, if you have
+    many millions of files in a transfer, a large amount of server memory, and
+    you don't want to split up your transfer into multiple parts, you can
+    increase the per-allocation limit to something larger and rsync will
+    consume more memory.
+
+    Keep in mind that this is not a limit on the total size of allocated
+    memory.  It is a sanity-check value for each individual allocation.
+
+    See the `--max-size` option for a description of how SIZE can be specified.
+    The default suffix if none is given is bytes.
+
+    You can set a default value using the environment variable RSYNC_MAX_ALLOC
+    using the same SIZE values as supported by this option.  If the remote
+    rsync doesn't understand the `--max-alloc` option, you can override an
+    environmental value by specifying `--max-alloc=1g`, which will make rsync
+    avoid sending the option to the remote side (because "1G" is the default).
+
+0.  `--block-size=SIZE`, `-B`
 
     This forces the block size used in rsync's delta-transfer algorithm to a
     fixed value.  It is normally selected based on the size of each file being
@@ -1852,6 +1884,8 @@ your home directory (remove the '=' for that).
     The exclude list is initialized to exclude the following items (these
     initial items are marked as perishable -- see the FILTER RULES section):
 
+    [comment]: # (This list gets used for the default-cvsignore.h file.)
+
     > `RCS`
     > `SCCS`
     > `CVS`
@@ -2066,6 +2100,10 @@ your home directory (remove the '=' for that).
 
     Rsync can also be configured (at build time) to have this option enabled by
     default (with is overridden by both the environment and the command-line).
+    Run `rsync --version` to check if this is the case, as it will display
+    "default protect-args" or "optional protect-args" depending on how it was
+    compiled.
+
     This option will eventually become a new default setting at some
     as-yet-undetermined point in the future.
 
@@ -2250,29 +2288,42 @@ your home directory (remove the '=' for that).
     destination machine, which reduces the amount of data being transmitted --
     something that is useful over a slow connection.
 
-    The "zlib" compression method typically achieves better compression ratios
-    than can be achieved by using a compressing remote shell or a compressing
-    transport because it takes advantage of the implicit information in the
-    matching data blocks that are not explicitly sent over the connection.
-    This matching-data compression comes at a cost of CPU, though, and can be
-    disabled by using the "zlibx" compresson method instead.  This can be
-    selected by repeating the `-z` option or specifying
-    `--compress-choice=zlibx`, but it only works if both sides of the transfer
-    are at least version 3.1.1.
+    Rsync supports multiple compression methods and will choose one for you
+    unless you force the choice using the `--compress-choice` (`--zc`) option.
 
-    Note that if you see an error about an option named `--old-compress` or
-    `--new-compress`, this is rsync trying to send the `--compress-choice=zlib`
-    or `--compress-choice=zlibx` option in a backward-compatible manner that
-    more rsync versions understand.  This error indicates that the older rsync
-    version will not allow you to force the compression type.
+    Run `rsync --version` to see the default compress list compiled into your
+    version.
 
-    See the `--skip-compress` option for the default list of file suffixes that
-    will not be compressed.
+    When both sides of the transfer are at least 3.2.0, rsync chooses the first
+    algorithm in the client's list of choices that is also in the server's list
+    of choices.  If no common compress choice is found, rsync exits with
+    an error.  If the remote rsync is too old to support checksum negotiation,
+    its list is assumed to be "zlib".
+
+    The default order can be customized by setting the environment variable
+    RSYNC_COMPRESS_LIST to a space-separated list of acceptable compression
+    names.  If the string contains a "`&`" character, it is separated into the
+    "client string & server string", otherwise the same string applies to both.
+    If the string (or string portion) contains no
+    non-whitespace characters, the default compress list is used.  Any unknown
+    compression names are discarded from the list, but a list with only invalid
+    names results in a failed negotiation.
+
+    There are some older rsync versions that were configured to reject a `-z`
+    option and require the use of `-zz` because their compression library was
+    not compatible with the default zlib compression method.  You can usually
+    ignore this weirdness unless the rsync server complains and tells you to
+    specify `-zz`.
+
+    See also the `--skip-compress` option for the default list of file suffixes
+    that will trasnferred with no (or minimal) compression.
 
 0.  `--compress-choice=STR`, `--zc=STR`
 
-    This option can be used to override the automatic selection of the
-    compression algorithm that is the default when `--compress` is used.
+    This option can be used to override the automatic negotiation of the
+    compression algorithm that occurs when `--compress` is used.  The option
+    implies `--compress` unless "none" was specified, which instead implies
+    `--no-compress`.
 
     The compression options that you may be able to use are:
 
@@ -2282,31 +2333,52 @@ your home directory (remove the '=' for that).
     - `zlib`
     - `none`
 
-    Run `rsync -V` to see the compress list compiled into your version.
+    Run `rsync --version` to see the default compress list compiled into your
+    version (which may differ from the list above).
 
-    The "zlibx" algorithm is given preference over "zlib" if both sides of the
-    transfer are at least version 3.2.0, otherwise it will choose "zlib" unless
-    you override it via something like `-zz`.  These 2 algorithms are the stame
-    except that "zlibx" does not try to include matched data that was not
-    transferred in the compression computations.
+    Note that if you see an error about an option named `--old-compress` or
+    `--new-compress`, this is rsync trying to send the `--compress-choice=zlib`
+    or `--compress-choice=zlibx` option in a backward-compatible manner that
+    more rsync versions understand.  This error indicates that the older rsync
+    version on the server will not allow you to force the compression type.
 
-    If "none" is specified, that is equivalent to using `--no-compress`.
+    Note that the "zlibx" compression algorithm is just the "zlib" algorithm
+    with matched data excluded from the compression stream (to try to make it
+    more compatible with an external zlib implementation).
 
-    This option implies `--compress` unless "none" was specified.
+0.  `--compress-level=NUM`, `--zl=NUM`
 
-    You can also override the compression negotation using the
-    RSYNC_COMPRESS_LIST environment variable by setting it to a space-separated
-    list of compression names that you consider acceptable.  If no common
-    compress choice is found, the client exits with an error.  It ignores
-    "auto" and all unknown compression names.  If the remote rsync is not new
-    enough to handle a compression negotiation list, the list is silently
-    ignored unless it contains the string "FAIL".
+    Explicitly set the compression level to use (see `--compress`, `-z`)
+    instead of letting it default.  The `--compress` option is implied as long
+    as the level chosen is not a "don't compress" level for the compression
+    algorithm that is in effect (e.g. zlib compression treats level 0 as
+    "off").
 
-0.  `--compress-level=NUM`
+    The level values vary depending on the checksum in effect.  Because rsync
+    will negotiate a checksum choice by default (when the remote rsync is new
+    enough), it can be good to combine this option with a `--compress-choice`
+    (`--zc`) option unless you're sure of the choice in effect.  For example:
 
-    Explicitly set the compression level to use (see `--compress`) instead of
-    letting it default.  If NUM is non-zero, the `--compress` option is
-    implied.
+    >     rsync -aiv --zc=zstd --zl=22 host:src/ dest/
+
+    For zlib & zlibx compression the valid values are from 1 to 9 with 6 being
+    the default.  Specifying 0 turns compression off, and specifying -1 chooses
+    the default of 6.
+
+    For zstd compression the valid values are from -131072 to 22 with 3 being
+    the default. Specifying 0 chooses the default of 3.
+
+    For lz4 compression there are no levels, so the value is always 0.
+
+    If you specify a too-large or too-small value, the number is silently
+    limited to a valid value.  This allows you to specify something like
+    `--zl=999999999` and be assured that you'll end up with the maximum
+    compression level no matter what algorithm was chosen.
+
+    If you want to know the compression level that is in effect, specify
+    `--debug=nstr` to see the "negotiated string" results.  This will report
+    something like "`Client compress: zstd (level 3)`" (along with the checksum
+    choice in effect).
 
 0.  `--skip-compress=LIST`
 
@@ -2314,18 +2386,20 @@ your home directory (remove the '=' for that).
     possible.  Rsync sets the compression level on a per-file basis based on
     the file's suffix.  If the compression algorithm has an "off" level (such
     as zlib/zlibx) then no compression occurs for those files.  Other
-    algorithms have the level minimized to reduces the CPU usage as much as
-    possible.
+    algorithms that support changing the streaming level on-the-fly will have
+    the level minimized to reduces the CPU usage as much as possible for a
+    matching file.  At this time, only zlib & zlibx compression support this
+    changing of levels on a per-file basis.
 
     The **LIST** should be one or more file suffixes (without the dot) separated
-    by slashes (/).  You may specify an empty string to indicate that no files
+    by slashes (`/`).  You may specify an empty string to indicate that no files
     should be skipped.
 
     Simple character-class matching is supported: each must consist of a list
     of letters inside the square brackets (e.g. no special classes, such as
     "[:alpha:]", are supported, and '-' has no special meaning).
 
-    The characters asterisk (\*) and question-mark (?) have no special meaning.
+    The characters asterisk (`*`) and question-mark (`?`) have no special meaning.
 
     Here's an example that specifies 6 suffixes to skip (since 1 of the 5 rules
     matches 2 suffixes):
@@ -2335,38 +2409,68 @@ your home directory (remove the '=' for that).
     The default file suffixes in the skip-compress list in this version of
     rsync are:
 
+    [comment]: # (This list gets used for the default-dont-compress.h file.)
+
     > 7z
     > ace
+    > apk
     > avi
     > bz2
     > deb
+    > flac
     > gpg
     > gz
     > iso
+    > jar
     > jpeg
     > jpg
     > lz
+    > lz4
     > lzma
     > lzo
+    > mkv
     > mov
     > mp3
     > mp4
+    > odb
+    > odf
+    > odg
+    > odi
+    > odm
+    > odp
+    > ods
+    > odt
     > ogg
     > ogv
+    > opus
+    > otg
+    > oth
+    > otp
+    > ots
+    > ott
+    > oxt
     > png
     > rar
     > rpm
+    > rz
     > rzip
     > squashfs
+    > sxc
+    > sxd
+    > sxg
+    > sxm
+    > sxw
     > tbz
     > tgz
     > tlz
     > txz
+    > tzo
     > webm
     > webp
     > xz
     > z
     > zip
+    > zst
 
     This list will be replaced by your `--skip-compress` list in all but one
     situation: a copy from a daemon rsync will add your skipped suffixes to its
@@ -2385,7 +2489,7 @@ your home directory (remove the '=' for that).
 
     If a user or group has no name on the source system or it has no match on
     the destination system, then the numeric ID from the source system is used
-    instead.  See also the comments on the "use chroot" setting in the
+    instead.  See also the comments on the "`use chroot`" setting in the
     rsyncd.conf manpage for information on how the chroot setting affects
     rsync's ability to look up the names of the users and groups and what you
     can do about it.
@@ -2399,7 +2503,7 @@ your home directory (remove the '=' for that).
     You may specify usernames or user IDs for the **FROM** and **TO** values,
     and the **FROM** value may also be a wild-card string, which will be
     matched against the sender's names (wild-cards do NOT match against ID
-    numbers, though see below for why a '\*' matches everything).  You may
+    numbers, though see below for why a '`*`' matches everything).  You may
     instead specify a range of ID numbers via an inclusive range: LOW-HIGH.
     For example:
 
@@ -2417,7 +2521,7 @@ your home directory (remove the '=' for that).
 
     Any IDs that do not have a name on the sending side are treated as having
     an empty name for the purpose of matching.  This allows them to be matched
-    via a "\*" or using an empty name.  For instance:
+    via a "`*`" or using an empty name.  For instance:
 
     >     --usermap=:nobody --groupmap=*:nobody
 
@@ -2441,22 +2545,22 @@ your home directory (remove the '=' for that).
     will occur.  If GROUP is empty, the trailing colon may be omitted, but if
     USER is empty, a leading colon must be supplied.
 
-    If you specify "--chown=foo:bar, this is exactly the same as specifying
+    If you specify "`--chown=foo:bar`", this is exactly the same as specifying
     "`--usermap=*:foo --groupmap=*:bar`", only easier.
 
-0.  `--timeout=TIMEOUT`
+0.  `--timeout=SECONDS`
 
     This option allows you to set a maximum I/O timeout in seconds.  If no data
     is transferred for the specified time then rsync will exit.  The default is
     0, which means no timeout.
 
-0.  `--contimeout`
+0.  `--contimeout=SECONDS`
 
     This option allows you to set the amount of time that rsync will wait for
     its connection to an rsync daemon to succeed.  If the timeout is reached,
     rsync exits with an error.
 
-0.  `--address`
+0.  `--address=ADDRESS`
 
     By default rsync will bind to the wildcard address when connecting to an
     rsync daemon.  The `--address` option allows you to specify a specific IP
@@ -2471,15 +2575,16 @@ your home directory (remove the '=' for that).
     the port as a part of the URL).  See also this option in the `--daemon`
     mode section.
 
-0.  `--sockopts`
+0.  `--sockopts=OPTIONS`
 
     This option can provide endless fun for people who like to tune their
     systems to the utmost degree.  You can set all sorts of socket options
     which may make transfers faster (or slower!).  Read the man page for the
     `setsockopt()` system call for details on some of the options you may be
     able to set.  By default no special socket options are set.  This only
-    affects direct socket connections to a remote rsync daemon.  This option
-    also exists in the `--daemon` mode section.
+    affects direct socket connections to a remote rsync daemon.
+
+    This option also exists in the `--daemon` mode section.
 
 0.  `--blocking-io`
 
@@ -2578,7 +2683,7 @@ your home directory (remove the '=' for that).
     character.  A default format of "%n%L" is assumed if either `--info=name`
     or `-v` is specified (this tells you just the name of the file and, if the
     item is a link, where it points).  For a full list of the possible escape
-    characters, see the "log format" setting in the rsyncd.conf manpage.
+    characters, see the "`log format`" setting in the rsyncd.conf manpage.
 
     Specifying the `--out-format` option implies the `--info=name` option,
     which will mention each file, dir, etc. that gets updated in a significant
@@ -2619,7 +2724,7 @@ your home directory (remove the '=' for that).
     file specified by the `--log-file` option (which must also be specified for
     this option to have any effect).  If you specify an empty string, updated
     files will not be mentioned in the log file.  For a list of the possible
-    escape characters, see the "log format" setting in the rsyncd.conf manpage.
+    escape characters, see the "`log format`" setting in the rsyncd.conf manpage.
 
     The default FORMAT used if `--log-file` is specified and this option is not
     is '%i %n%L'.
@@ -2685,8 +2790,8 @@ your home directory (remove the '=' for that).
     tabs) are always escaped, regardless of this option's setting.
 
     The escape idiom that started in 2.6.7 is to output a literal backslash
-    (\\) and a hash (#), followed by exactly 3 octal digits.  For example, a
-    newline would output as "\\#012".  A literal backslash that is in a
+    (`\`) and a hash (`#`), followed by exactly 3 octal digits.  For example, a
+    newline would output as "`\#012`".  A literal backslash that is in a
     filename is not escaped unless it is followed by a hash and 3 digits (0-9).
 
 0.  `--human-readable`, `-h`
@@ -2756,10 +2861,10 @@ your home directory (remove the '=' for that).
     rule may be ineffective at the end of your other rules, or (2) you may wish
     to override rsync's exclude choice.  For instance, if you want to make
     rsync clean-up any left-over partial-dirs that may be lying around, you
-    should specify `--delete-after` and add a "risk" filter rule, e.g. `-f 'R
-    .rsync-partial/'`. (Avoid using `--delete-before` or `--delete-during`
-    unless you don't need rsync to use any of the left-over partial-dir data
-    during the current run.)
+    should specify `--delete-after` and add a "risk" filter rule, e.g.
+    `-f 'R .rsync-partial/'`. (Avoid using `--delete-before` or
+    `--delete-during` unless you don't need rsync to use any of the left-over
+    partial-dir data during the current run.)
 
     IMPORTANT: the `--partial-dir` should not be writable by other users or it
     is a security risk.  E.g. AVOID "/tmp".
@@ -2781,7 +2886,7 @@ your home directory (remove the '=' for that).
     tmp).  This requires both ends of the transfer to be at least version
     3.2.0.
 
-    For the purposes of the daemon-config's "refuse options" setting,
+    For the purposes of the daemon-config's "`refuse options`" setting,
     `--partial-dir` does _not_ imply `--partial`.  This is so that a refusal of
     the `--partial` option can be used to disallow the overwriting of
     destination files with a partial transfer, while still allowing the safer
@@ -2923,7 +3028,7 @@ your home directory (remove the '=' for that).
 
     CAUTION: sending SIGVTALRM to an older rsync (pre-3.2.0) will kill it.
 
-0. `--password-file=FILE`
+0.  `--password-file=FILE`
 
     This option allows you to provide a password for accessing an rsync daemon
     via a file or via standard input if **FILE** is `-`.  The file should
@@ -2938,7 +3043,16 @@ your home directory (remove the '=' for that).
     authentication (i.e. if you have also specified a password in the daemon's
     config file).
 
-0. `--list-only`
+0.  `--early-input=FILE`
+
+    This option allows rsync to send up to 5K of data to the "early exec"
+    script on its stdin.  One possible use of this data is to give the script a
+    secret that can be used to mount an encrypted filesystem (which you should
+    unmount in the the "post-xfer exec" script).
+
+    The daemon must be at least version 3.2.1.
+
+0.  `--list-only`
 
     This option will cause the source files to be listed instead of
     transferred.  This option is inferred if there is a single source arg and
@@ -2967,7 +3081,7 @@ your home directory (remove the '=' for that).
     need to expand a directory's content), or turn on recursion and exclude the
     content of subdirectories: `-r --exclude='/*/*'`.
 
-0. `--bwlimit=RATE`
+0.  `--bwlimit=RATE`
 
     This option allows you to specify the maximum transfer rate for the data
     sent over the socket, specified in units per second.  The RATE value can be
@@ -2983,7 +3097,7 @@ your home directory (remove the '=' for that).
 
     Rsync writes data over the socket in blocks, and this option both limits
     the size of the blocks that rsync writes, and tries to keep the average
-    transfer rate at the requested limit.  Some "burstiness" may be seen where
+    transfer rate at the requested limit.  Some burstiness may be seen where
     rsync writes out a block of data and then sleeps to bring the average rate
     into compliance.
 
@@ -2993,13 +3107,18 @@ your home directory (remove the '=' for that).
     buffered, while other can show up as very slow when the flushing of the
     output buffer occurs.  This may be fixed in a future version.
 
-0. `--write-batch=FILE`
+0.  `--write-batch=FILE`
 
     Record a file that can later be applied to another identical destination
     with `--read-batch`.  See the "BATCH MODE" section for details, and also
     the `--only-write-batch` option.
 
-0. `--only-write-batch=FILE`
+    This option overrides the negotiated checksum & compress lists and always
+    negotiates a choice based on old-school md5/md4/zlib choices.  If you want
+    a more modern choice, use the `--checksum-choice` (`--cc`) and/or
+    `--compress-choice` (`--zc`) options.
+
+0.  `--only-write-batch=FILE`
 
     Works like `--write-batch`, except that no updates are made on the
     destination system when creating the batch.  This lets you transport the
@@ -3018,13 +3137,13 @@ your home directory (remove the '=' for that).
     into the batch file without having to flow over the wire to the receiver
     (when pulling, the sender is remote, and thus can't write the batch).
 
-0. `--read-batch=FILE`
+0.  `--read-batch=FILE`
 
     Apply all of the changes stored in FILE, a file previously generated by
     `--write-batch`.  If _FILE_ is `-`, the batch data will be read from
     standard input. See the "BATCH MODE" section for details.
 
-0. `--protocol=NUM`
+0.  `--protocol=NUM`
 
     Force an older protocol version to be used.  This is useful for creating a
     batch file that is compatible with an older version of rsync.  For
@@ -3034,7 +3153,7 @@ your home directory (remove the '=' for that).
     protocol version to be used in the batch file (assuming you can't upgrade
     the rsync on the reading system).
 
-0. `--iconv=CONVERT_SPEC`
+0.  `--iconv=CONVERT_SPEC`
 
     Rsync can convert filenames between character sets using this option.
     Using a CONVERT_SPEC of "." tells rsync to look up the default
@@ -3066,11 +3185,11 @@ your home directory (remove the '=' for that).
     free to specify just the local charset for a daemon transfer (e.g.
     `--iconv=utf8`).
 
-0. `--ipv4`, `-4` or `--ipv6`, `-6`
+0.  `--ipv4`, `-4` or `--ipv6`, `-6`
 
     Tells rsync to prefer IPv4/IPv6 when creating sockets or running ssh.  This
     affects sockets that rsync has direct control over, such as the outgoing
-    socket when directly contacting an rsync daemon, as well as the forwaring
+    socket when directly contacting an rsync daemon, as well as the forwarding
     of the `-4` or `-6` option to ssh when rsync can deduce that ssh is being
     used as the remote shell.  For other remote shells you'll need to specify
     the "`--rsh SHELL -4`" option directly (or whatever ipv4/ipv6 hint options
@@ -3079,12 +3198,10 @@ your home directory (remove the '=' for that).
     These options also exist in the `--daemon` mode section.
 
     If rsync was complied without support for IPv6, the `--ipv6` option will
-    have no effect.  The `rsync -V` output will contain "no IPv6" if is the
-    case.
+    have no effect.  The `rsync --version` output will contain "`no IPv6`" if
+    is the case.
 
-    See also these options in the `--daemon` mode section.
-
-0. `--checksum-seed=NUM`
+0.  `--checksum-seed=NUM`
 
     Set the checksum seed to the integer NUM.  This 4 byte checksum seed is
     included in each block and MD4 file checksum calculation (the more modern
@@ -3099,7 +3216,7 @@ your home directory (remove the '=' for that).
 
 The options allowed when starting an rsync daemon are as follows:
 
-0. `--daemon`
+0.  `--daemon`
 
     This tells rsync that it is to run as a daemon.  The daemon you start
     running may be accessed using an rsync client using the `host::module` or
@@ -3111,7 +3228,7 @@ The options allowed when starting an rsync daemon are as follows:
     each connect made by a client and respond to requests accordingly.  See the
     **rsyncd.conf**(5) man page for more details.
 
-0. `--address`
+0.  `--address=ADDRESS`
 
     By default rsync will bind to the wildcard address when run as a daemon
     with the `--daemon` option.  The `--address` option allows you to specify a
@@ -3119,14 +3236,14 @@ The options allowed when starting an rsync daemon are as follows:
     possible in conjunction with the `--config` option.  See also the "address"
     global option in the rsyncd.conf manpage.
 
-0. `--bwlimit=RATE`
+0.  `--bwlimit=RATE`
 
     This option allows you to specify the maximum transfer rate for the data
     the daemon sends over the socket.  The client can still specify a smaller
     `--bwlimit` value, but no larger value will be allowed.  See the client
     version of this option (above) for some extra details.
 
-0. `--config=FILE`
+0.  `--config=FILE`
 
     This specifies an alternate config file than the default.  This is only
     relevant when `--daemon` is specified.  The default is /etc/rsyncd.conf
@@ -3134,7 +3251,7 @@ The options allowed when starting an rsync daemon are as follows:
     user is not the super-user; in that case the default is rsyncd.conf in the
     current directory (typically $HOME).
 
-0. `--dparam=OVERRIDE`, `-M`
+0.  `--dparam=OVERRIDE`, `-M`
 
     This option can be used to set a daemon-config parameter when starting up
     rsync in daemon mode.  It is equivalent to adding the parameter at the end
@@ -3144,7 +3261,7 @@ The options allowed when starting an rsync daemon are as follows:
 
     >     rsync --daemon -M pidfile=/path/rsync.pid
 
-0. `--no-detach`
+0.  `--no-detach`
 
     When running as a daemon, this option instructs rsync to not detach itself
     and become a background process.  This option is required when running as a
@@ -3153,37 +3270,37 @@ The options allowed when starting an rsync daemon are as follows:
     `--no-detach` is also recommended when rsync is run under a debugger.  This
     option has no effect if rsync is run from inetd or sshd.
 
-0. `--port=PORT`
+0.  `--port=PORT`
 
     This specifies an alternate TCP port number for the daemon to listen on
     rather than the default of 873.  See also the "port" global option in the
     rsyncd.conf manpage.
 
-0. `--log-file=FILE`
+0.  `--log-file=FILE`
 
     This option tells the rsync daemon to use the given log-file name instead
-    of using the "log file" setting in the config file.
+    of using the "`log file`" setting in the config file.
 
-0. `--log-file-format=FORMAT`
+0.  `--log-file-format=FORMAT`
 
     This option tells the rsync daemon to use the given FORMAT string instead
-    of using the "log format" setting in the config file.  It also enables
-    "transfer logging" unless the string is empty, in which case transfer
+    of using the "`log format`" setting in the config file.  It also enables
+    "`transfer logging`" unless the string is empty, in which case transfer
     logging is turned off.
 
-0. `--sockopts`
+0.  `--sockopts`
 
     This overrides the `socket options` setting in the rsyncd.conf file and has
     the same syntax.
 
-0. `--verbose`, `-v`
+0.  `--verbose`, `-v`
 
     This option increases the amount of information the daemon logs during its
     startup phase.  After the client connects, the daemon's verbosity level
-    will be controlled by the options that the client used and the "max
-    verbosity" setting in the module's config section.
+    will be controlled by the options that the client used and the
+    "`max verbosity`" setting in the module's config section.
 
-0. `--ipv4`, `-4` or `--ipv6`, `-6`
+0.  `--ipv4`, `-4` or `--ipv6`, `-6`
 
     Tells rsync to prefer IPv4/IPv6 when creating the incoming sockets that the
     rsync daemon will use to listen for connections.  One of these options may
@@ -3195,10 +3312,10 @@ The options allowed when starting an rsync daemon are as follows:
     These options also exist in the regular rsync options section.
 
     If rsync was complied without support for IPv6, the `--ipv6` option will
-    have no effect.  The `rsync -V` output will contain "no IPv6" if is the
-    case.
+    have no effect.  The `rsync --version` output will contain "`no IPv6`" if
+    is the case.
 
-0. `--help`, `-h`
+0.  `--help`, `-h`
 
     When specified after `--daemon`, print a short help page describing the
     options available for starting an rsync daemon.
@@ -3241,19 +3358,23 @@ available rule prefixes:
 When rules are being read from a file, empty lines are ignored, as are comment
 lines that start with a "#".
 
-Note that the `--include`/`--exclude` command-line options do not allow the
+[comment]: # (Remember that markdown strips spaces from start/end of ` ... ` sequences!)
+[comment]: # (Thus, the `x ` sequences below use a literal non-breakable space!)
+
+Note that the `--include` & `--exclude` command-line options do not allow the
 full range of rule parsing as described above -- they only allow the
-specification of include/exclude patterns plus a "!" token to clear the list
-(and the normal comment parsing when rules are read from a file).  If a pattern
-does not begin with "- " (dash, space) or "+ " (plus, space), then the rule
-will be interpreted as if "+ " (for an include option) or "- " (for an exclude
-option) were prefixed to the string.  A `--filter` option, on the other hand,
-must always contain either a short or long rule name at the start of the rule.
+specification of include / exclude patterns plus a "`!`" token to clear the
+list (and the normal comment parsing when rules are read from a file).  If a
+pattern does not begin with "`- `" (dash, space) or "`+ `" (plus, space), then
+the rule will be interpreted as if "`+ `" (for an include option) or "`- `"
+(for an exclude option) were prefixed to the string.  A `--filter` option, on
+the other hand, must always contain either a short or long rule name at the
+start of the rule.
 
 Note also that the `--filter`, `--include`, and `--exclude` options take one
 rule/pattern each.  To add multiple ones, you can repeat the options on the
 command-line, use the merge-file syntax of the `--filter` option, or the
-`--include-from`/`--exclude-from` options.
+`--include-from` / `--exclude-from` options.
 
 # INCLUDE/EXCLUDE PATTERN RULES
 
@@ -3263,36 +3384,36 @@ include/exclude rules each specify a pattern that is matched against the names
 of the files that are going to be transferred.  These patterns can take several
 forms:
 
-- if the pattern starts with a / then it is anchored to a particular spot in
+- if the pattern starts with a `/` then it is anchored to a particular spot in
   the hierarchy of files, otherwise it is matched against the end of the
-  pathname.  This is similar to a leading ^ in regular expressions.  Thus
-  "/foo" would match a name of "foo" at either the "root of the transfer" (for
+  pathname.  This is similar to a leading `^` in regular expressions.  Thus
+  `/foo` would match a name of "foo" at either the "root of the transfer" (for
   a global rule) or in the merge-file's directory (for a per-directory rule).
-  An unqualified "foo" would match a name of "foo" anywhere in the tree because
+  An unqualified `foo` would match a name of "foo" anywhere in the tree because
   the algorithm is applied recursively from the top down; it behaves as if each
   path component gets a turn at being the end of the filename.  Even the
   unanchored "sub/foo" would match at any point in the hierarchy where a "foo"
   was found within a directory named "sub".  See the section on ANCHORING
   INCLUDE/EXCLUDE PATTERNS for a full discussion of how to specify a pattern
   that matches at the root of the transfer.
-- if the pattern ends with a / then it will only match a directory, not a
+- if the pattern ends with a `/` then it will only match a directory, not a
   regular file, symlink, or device.
 - rsync chooses between doing a simple string match and wildcard matching by
   checking if the pattern contains one of these three wildcard characters:
   '`*`', '`?`', and '`[`' .
 - a '`*`' matches any path component, but it stops at slashes.
 - use '`**`' to match anything, including slashes.
-- a '?' matches any character except a slash (/).
-- a '[' introduces a character class, such as [a-z] or [[:alpha:]].
+- a '`?`' matches any character except a slash (`/`).
+- a '`[`' introduces a character class, such as `[a-z]` or `[[:alpha:]]`.
 - in a wildcard pattern, a backslash can be used to escape a wildcard
   character, but it is matched literally when no wildcards are present.  This
   means that there is an extra level of backslash removal when a pattern
   contains wildcard characters compared to a pattern that has none.  e.g. if
   you add a wildcard to "`foo\bar`" (which matches the backslash) you would
   need to use "`foo\\bar*`" to avoid the "`\b`" becoming just "b".
-- if the pattern contains a / (not counting a trailing /) or a "`**`", then it
+- if the pattern contains a `/` (not counting a trailing /) or a "`**`", then it
   is matched against the full pathname, including any leading directories.  If
-  the pattern doesn't contain a / or a "`**`", then it is matched only against
+  the pattern doesn't contain a `/` or a "`**`", then it is matched only against
   the final component of the filename. (Remember that the algorithm is applied
   recursively so "full filename" can actually be any portion of a path from the
   starting directory on down.)
@@ -3307,20 +3428,20 @@ include/exclude patterns are applied recursively to the pathname of each node
 in the filesystem's tree (those inside the transfer).  The exclude patterns
 short-circuit the directory traversal stage as rsync finds the files to send.
 
-For instance, to include "/foo/bar/baz", the directories "/foo" and "/foo/bar"
+For instance, to include "`/foo/bar/baz`", the directories "`/foo`" and "`/foo/bar`"
 must not be excluded.  Excluding one of those parent directories prevents the
 examination of its content, cutting off rsync's recursion into those paths and
-rendering the include for "/foo/bar/baz" ineffectual (since rsync can't match
+rendering the include for "`/foo/bar/baz`" ineffectual (since rsync can't match
 something it never sees in the cut-off section of the directory hierarchy).
 
-The concept path exclusion is particularly important when using a trailing '\*'
+The concept path exclusion is particularly important when using a trailing '`*`'
 rule.  For instance, this won't work:
 
 >     + /some/path/this-file-will-not-be-found
 >     + /file-is-included
 >     - *
 
-This fails because the parent directory "some" is excluded by the '\*' rule, so
+This fails because the parent directory "some" is excluded by the '`*`' rule, so
 rsync never visits any of the files in the "some" or "some/path" directories.
 One solution is to ask for all directories in the hierarchy to be included by
 using a single rule: "`+ */`" (put it somewhere before the "`- *`" rule), and
@@ -3881,7 +4002,7 @@ values
 
 see also the comments on the `--delete` option
 
-Please report bugs! See the web site at <http://rsync.samba.org/>.
+Please report bugs! See the web site at <https://rsync.samba.org/>.
 
 # VERSION
 
@@ -3901,11 +4022,9 @@ that can be used with a restricted ssh login.
 rsync is distributed under the GNU General Public License.  See the file
 COPYING for details.
 
-A web site is available at <http://rsync.samba.org/>.  The site includes an
+A web site is available at <https://rsync.samba.org/>.  The site includes an
 FAQ-O-Matic which may cover questions unanswered by this manual page.
 
-The primary ftp site for rsync is <ftp://rsync.samba.org/pub/rsync>
-
 We would be delighted to hear from you if you like this program.  Please
 contact the mailing-list at <rsync@lists.samba.org>.
 
@@ -3928,4 +4047,4 @@ people have later contributed to it. It is currently maintained by Wayne
 Davison.
 
 Mailing lists for support and development are available at
-<http://lists.samba.org/>.
+<https://lists.samba.org/>.

rsync.c

@@ -57,12 +57,6 @@ extern struct chmod_mode_struct *daemon_chmod_modes;
 extern char *iconv_opt;
 #endif
 
-#ifdef ICONV_CONST
-iconv_t ic_chck = (iconv_t)-1;
-# ifdef ICONV_OPTION
-iconv_t ic_send = (iconv_t)-1, ic_recv = (iconv_t)-1;
-# endif
-
 #define UPDATED_OWNER (1<<0)
 #define UPDATED_GROUP (1<<1)
 #define UPDATED_MTIME (1<<2)
@@ -72,6 +66,12 @@ iconv_t ic_send = (iconv_t)-1, ic_recv = (iconv_t)-1;
 
 #define UPDATED_TIMES (UPDATED_MTIME|UPDATED_ATIME)
 
+#ifdef ICONV_CONST
+iconv_t ic_chck = (iconv_t)-1;
+# ifdef ICONV_OPTION
+iconv_t ic_send = (iconv_t)-1, ic_recv = (iconv_t)-1;
+# endif
+
 static const char *default_charset(void)
 {
 # if defined HAVE_LIBCHARSET_H && defined HAVE_LOCALE_CHARSET

rsync.h

@@ -20,6 +20,7 @@
 
 #define False 0
 #define True 1
+#define Unset (-1) /* Our BOOL values are always an int. */
 
 #define BLOCK_SIZE 700
 #define RSYNC_RSH_ENV "RSYNC_RSH"
@@ -297,6 +298,7 @@ enum delret {
 #include "errcode.h"
 
 #include "config.h"
+#include "version.h"
 
 /* The default RSYNC_RSH is always set in config.h. */
 
@@ -1078,6 +1080,9 @@ typedef struct {
 #define CPRES_LZ4 3
 #define CPRES_ZSTD 4
 
+#define NSTR_CHECKSUM 0
+#define NSTR_COMPRESS 1
+
 struct name_num_item {
 	int num;
 	const char *name, *main_name;
@@ -1262,12 +1267,19 @@ extern int errno;
 /* handler for null strings in printf format */
 #define NS(s) ((s)?(s):"<NULL>")
 
+extern char *do_malloc;
+
 /* Convenient wrappers for malloc and realloc.  Use them. */
-#define new(type) ((type*)malloc(sizeof (type)))
-#define new0(type) ((type*)calloc(1, sizeof (type)))
-#define new_array(type, num) ((type*)_new_array((num), sizeof (type), 0))
-#define new_array0(type, num) ((type*)_new_array((num), sizeof (type), 1))
-#define realloc_array(ptr, type, num) ((type*)_realloc_array((ptr), sizeof(type), (num)))
+#define new(type) ((type*)my_alloc(do_malloc, sizeof (type), 1, __FILE__, __LINE__))
+#define new0(type) ((type*)my_alloc(NULL, sizeof (type), 1, __FILE__, __LINE__))
+#define realloc_buf(ptr, num) my_alloc((ptr), (num), 1, __FILE__, __LINE__)
+
+#define new_array(type, num) ((type*)my_alloc(do_malloc, (num), sizeof (type), __FILE__, __LINE__))
+#define new_array0(type, num) ((type*)my_alloc(NULL, (num), sizeof (type), __FILE__, __LINE__))
+#define realloc_array(ptr, type, num) ((type*)my_alloc((ptr), (num), sizeof (type), __FILE__, __LINE__))
+
+#undef strdup
+#define strdup(s) my_strdup(s, __FILE__, __LINE__)
 
 /* use magic gcc attributes to catch format errors */
  void rprintf(enum logcode , const char *, ...)

rsync3.txt

@@ -396,7 +396,7 @@ Conflict resolution:
     would be useful.
 
 
-Moved files: <http://rsync.samba.org/cgi-bin/rsync.fom?file=44>
+Moved files:
 
   - There's no trivial way to detect renamed files, especially if they
     move between directories.
@@ -457,13 +457,11 @@ Streaming:
 
 Related work:
 
-  - mirror.pl http://freshmeat.net/project/mirror/
+  - mirror.pl
 
   - ProFTPd
 
   - Apache
 
-  - http://freshmeat.net/search/?site=Freshmeat&q=mirror&section=projects
-
   - BitTorrent -- p2p mirroring
     http://bitconjurer.org/BitTorrent/

rsyncd.conf.5.md

@@ -18,7 +18,7 @@ modules.
 
 The file consists of modules and parameters. A module begins with the name of
 the module in square brackets and continues until the next module begins.
-Modules contain parameters of the form "name = value".
+Modules contain parameters of the form `name = value`.
 
 The file is line-based -- that is, each newline-terminated line represents
 either a comment, a module name or a parameter.
@@ -234,11 +234,11 @@ the values of parameters.  See the GLOBAL PARAMETERS section for more details.
     allow the daemon to function.  By default the daemon runs without any
     chrooting.
 
-0.  `haproxy header`
+0.  `proxy protocol`
 
     When this parameter is enabled, all incoming connections must start with a
-    V1 or V2 haproxy header.  If the header is not found, the connection is
-    closed.
+    V1 or V2 proxy protocol header.  If the header is not found, the connection
+    is closed.
 
     Setting this to `true` requires a proxy server to forward source IP
     information to rsync, allowing you to log proper IP/host info and make use
@@ -282,9 +282,9 @@ the values of parameters.  See the GLOBAL PARAMETERS section for more details.
     This parameter tells rsync to modify all symlinks in the same way as the
     (non-daemon-affecting) `--munge-links` command-line option (using a method
     described below).  This should help protect your files from user trickery
-    when your daemon module is writable.  The default is disabled when "use
-    chroot" is on with an inside-chroot path of "/", OR if "daemon chroot" is
-    on, otherwise it is enabled.
+    when your daemon module is writable.  The default is disabled when
+    "use chroot" is on with an inside-chroot path of "/", OR if "daemon chroot"
+    is on, otherwise it is enabled.
 
     If you disable this parameter on a daemon that is not read-only, there are
     tricks that a user can play with uploaded symlinks to access
@@ -393,8 +393,8 @@ the values of parameters.  See the GLOBAL PARAMETERS section for more details.
     and `--debug` logging.  If the max value is 2, then no info and/or debug
     value that is higher than what would be set by `-vv` will be honored by the
     daemon in its logging.  To see how high of a verbosity level you need to
-    accept for a particular info/debug level, refer to "rsync --info=help" and
-    "rsync --debug=help".  For instance, it takes max-verbosity 4 to be able to
+    accept for a particular info/debug level, refer to `rsync --info=help` and
+    `rsync --debug=help`.  For instance, it takes max-verbosity 4 to be able to
     output debug TIME2 and FLIST3.
 
 0.  `lock file`
@@ -407,12 +407,11 @@ the values of parameters.  See the GLOBAL PARAMETERS section for more details.
 0.  `read only`
 
     This parameter determines whether clients will be able to upload files or
-    not. If "read only" is true then any attempted uploads will fail. If "read
-    only" is false then uploads will be possible if file permissions on the
-    daemon side allow them. The default is for all modules to be read only.
+    not. If "read only" is true then any attempted uploads will fail. If
+    "read only" is false then uploads will be possible if file permissions on
+    the daemon side allow them. The default is for all modules to be read only.
 
-    Note that "auth users" can override this setting on a
-    per-user basis.
+    Note that "auth users" can override this setting on a per-user basis.
 
 0.  `write only`
 
@@ -425,6 +424,22 @@ the values of parameters.  See the GLOBAL PARAMETERS section for more details.
     Helpful hint: you probably want to specify "refuse options = delete" for a
     write-only module.
 
+0.  `open noatime`
+
+    When set to True, this parameter tells the rsync daemon to open files with
+    the O_NOATIME flag
+    (on systems that support it) to avoid changing the access time of the files
+    that are being transferred.  If your OS does not support the O_NOATIME flag
+    then rsync will silently ignore this option.  Note also that some
+    filesystems are mounted to avoid updating the atime on read access even
+    without the O_NOATIME flag being set.
+
+    When set to False, this parameters ensures that files on the server are not
+    opened with O_NOATIME.
+
+    When set to Unset (the default) the user controls the setting via
+    `--open-noatime`.
+
 0.  `list`
 
     This parameter determines whether this module is listed when the client
@@ -511,7 +526,7 @@ the values of parameters.  See the GLOBAL PARAMETERS section for more details.
 
     The "filter" parameter takes a space-separated list of daemon filter rules,
     though it is smart enough to know not to split a token at an internal space
-    in a rule (e.g. "- /foo  - /bar" is parsed as two rules).  You may specify
+    in a rule (e.g. "`- /foo  - /bar`" is parsed as two rules).  You may specify
     one or more merge-file rules using the normal syntax.  Only one "filter"
     parameter can apply to a given module in the config file, so put all the
     rules you want in a single parameter.  Note that per-directory merge-file
@@ -523,8 +538,8 @@ the values of parameters.  See the GLOBAL PARAMETERS section for more details.
 0.  `exclude`
 
     This parameter takes a space-separated list of daemon exclude patterns.  As
-    with the client `--exclude` option, patterns can be qualified with "- " or
-    "+ " to explicitly indicate exclude/include.  Only one "exclude" parameter
+    with the client `--exclude` option, patterns can be qualified with "`- `" or
+    "`+ `" to explicitly indicate exclude/include.  Only one "exclude" parameter
     can apply to a given module.  See the "filter" parameter for a description
     of how excluded files affect the daemon.
 
@@ -580,9 +595,9 @@ the values of parameters.  See the GLOBAL PARAMETERS section for more details.
     authentication. If "auth users" is set then the client will be challenged
     to supply a username and password to connect to the module. A challenge
     response authentication protocol is used for this exchange. The plain text
-    usernames and passwords are stored in the file specified by the "secrets
-    file" parameter. The default is for all users to be able to connect without
-    a password (this is called "anonymous rsync").
+    usernames and passwords are stored in the file specified by the
+    "secrets file" parameter. The default is for all users to be able to
+    connect without a password (this is called "anonymous rsync").
 
     In addition to username matching, you can specify groupname matching via a
     '@' prefix.  When using groupname matching, the authenticating username
@@ -702,8 +717,8 @@ the values of parameters.  See the GLOBAL PARAMETERS section for more details.
     If both parameters are specified then the "hosts allow" parameter is
     checked first and a match results in the client being able to connect. The
     "hosts deny" parameter is then checked and a match means that the host is
-    rejected. If the host does not match either the "hosts allow" or the "hosts
-    deny" patterns then it is allowed to connect.
+    rejected. If the host does not match either the "hosts allow" or the
+    "hosts deny" patterns then it is allowed to connect.
 
     The default is no "hosts allow" parameter, which means all hosts can
     connect.
@@ -721,11 +736,11 @@ the values of parameters.  See the GLOBAL PARAMETERS section for more details.
 0.  `reverse lookup`
 
     Controls whether the daemon performs a reverse lookup on the client's IP
-    address to determine its hostname, which is used for "hosts allow"/"hosts
-    deny" checks and the "%h" log escape.  This is enabled by default, but you
-    may wish to disable it to save time if you know the lookup will not return
-    a useful result, in which case the daemon will use the name "UNDETERMINED"
-    instead.
+    address to determine its hostname, which is used for "hosts allow" &
+    "hosts deny" checks and the "%h" log escape.  This is enabled by default,
+    but you may wish to disable it to save time if you know the lookup will not
+    return a useful result, in which case the daemon will use the name
+    "UNDETERMINED" instead.
 
     If this parameter is enabled globally (even by default), rsync performs the
     lookup as soon as a client connects, so disabling it for a module will not
@@ -770,18 +785,18 @@ the values of parameters.  See the GLOBAL PARAMETERS section for more details.
     transfers when transfer logging is enabled.  The format is a text string
     containing embedded single-character escape sequences prefixed with a
     percent (%) character.  An optional numeric field width may also be
-    specified between the percent and the escape letter (e.g. "`%-50n %8l
-    %07p`").  In addition, one or more apostrophes may be specified prior to a
-    numerical escape to indicate that the numerical value should be made more
-    human-readable.  The 3 supported levels are the same as for the
+    specified between the percent and the escape letter (e.g.
+    "`%-50n %8l %07p`").  In addition, one or more apostrophes may be specified
+    prior to a numerical escape to indicate that the numerical value should be
+    made more human-readable.  The 3 supported levels are the same as for the
     `--human-readable` command-line option, though the default is for
     human-readability to be off.  Each added apostrophe increases the level
     (e.g. "`%''l %'b %f`").
 
-    The default log format is "%o %h [%a] %m (%u) %f %l", and a "%t [%p] " is
-    always prefixed when using the "log file" parameter.  (A perl script that
-    will summarize this default log format is included in the rsync source code
-    distribution in the "support" subdirectory: rsyncstats.)
+    The default log format is "`%o %h [%a] %m (%u) %f %l`", and a "`%t [%p] `"
+    is always prefixed when using the "log file" parameter.  (A perl script
+    that will summarize this default log format is included in the rsync source
+    code distribution in the "support" subdirectory: rsyncstats.)
 
     The single-character escapes that are understood are as follows:
 
@@ -801,8 +816,8 @@ the values of parameters.  See the GLOBAL PARAMETERS section for more details.
     - %h the remote host name (only available for a daemon)
     - %i an itemized list of what is being updated
     - %l the length of the file in bytes
-    - %L the string " -> SYMLINK", " => HARDLINK", or "" (where `SYMLINK` or
-      `HARDLINK` is a filename)
+    - %L the string "` -> SYMLINK`", "` => HARDLINK`", or "" (where `SYMLINK`
+      or `HARDLINK` is a filename)
     - %m the module name
     - %M the last-modified time of the file
     - %n the filename (short form; trailing "/" on dir)
@@ -867,7 +882,7 @@ the values of parameters.  See the GLOBAL PARAMETERS section for more details.
 
     >     refuse options = * no-iconv !a !v
 
-    As an additional aid (beginning in 3.2.0), refusing (or "!refusing") the
+    As an additional aid (beginning in 3.2.0), refusing (or "`!refusing`") the
     "a" or "archive"  option also affects all the options that the `--archive`
     option implies (`-rdlptgoD`), but only if the option  is matched explicitly
     (not using a wildcard). If you want to do something tricky, you can use
@@ -899,22 +914,22 @@ the values of parameters.  See the GLOBAL PARAMETERS section for more details.
     "`!compress*`" so that you also accept the `--compress-level` option.
 
     Note that the "write-devices" option is refused by default, but can be
-    explicitly accepted with "!write-devices".  The options "log-file" and
+    explicitly accepted with "`!write-devices`".  The options "log-file" and
     "log-file-format" are forcibly refused and cannot be accepted.
 
     Here are all the options that are not matched by wild-cards:
 
     - `--server`: Required for rsync to even work.
-    - `-e`: Required to convey compatibility flags to the server.
-    - `--log-format`: This is required to convey things like
-      `--itemize-changes` to a remote receiver. Is an older name for
-      `--out-format` that is still passed to the server for improved backward
-      compatibility and should not be confused with `--log-file-format`.
+    - `--rsh`, `-e`: Required to convey compatibility flags to the server.
+    - `--out-format`: This is required to convey output behavior to a remote
+      receiver.  While rsync passes the older alias `--log-format` for
+      compatibility reasons, this options should not be confused with
+      `--log-file-format`.
     - `--sender`: Use "write only" parameter instead of refusing this.
-    - `-n, --dry-run`: Who would want to disable this?
-    - `-s, --protect-args`: This actually makes transfers safer.
-    - `-0, --from0`: Make it easier to accept/refuse `--files-from` without
-      affecting this modifier.
+    - `--dry-run`, `-n`: Who would want to disable this?
+    - `--protect-args`, `-s`: This actually makes transfers safer.
+    - `--from0`, `-0`: Makes it easier to accept/refuse `--files-from` without
+      affecting this helpful modifier.
     - `--iconv`: This is auto-disabled based on "charset" parameter.
     - `--no-iconv`: Most transfers use this option.
     - `--checksum-seed`: Is a fairly rare, safe option.
@@ -936,10 +951,10 @@ the values of parameters.  See the GLOBAL PARAMETERS section for more details.
     compression occurs for those files.  Other algorithms have the level
     minimized to reduces the CPU usage as much as possible.
 
-    See the `--skip-compress` parameter in the `rsync`(1) manpage for the list
-    of file suffixes that are not compressed by default.  Specifying a value
-    for the "dont compress" parameter changes the default when the daemon is
-    the sender.
+    See the `--skip-compress` parameter in the **rsync**(1) manpage for the
+    list of file suffixes that are not compressed by default.  Specifying a
+    value for the "dont compress" parameter changes the default when the daemon
+    is the sender.
 
 0.  `early exec`, `pre-xfer exec`, `post-xfer exec`
 
@@ -958,7 +973,9 @@ the values of parameters.  See the GLOBAL PARAMETERS section for more details.
     request is known except for the module name.  This helper script can be
     used to setup a disk mount or decrypt some data into a module dir, but you
     may need to use `lock file` and `max connections` to avoid concurrency
-    issues.
+    issues.  If the client rsync specified the `--early-input=FILE` option, it
+    can send up to about 5K of data to the stdin of the early script.  The
+    stdin will otherwise be empty.
 
     Note that the `post-xfer exec` command is still run even if one of the
     other scripts returns an error code. The `pre-xfer exec` command will _not_
@@ -1066,8 +1083,53 @@ Also note that the rsync daemon protocol does not currently provide any
 encryption of the data that is transferred over the connection. Only
 authentication is provided. Use ssh as the transport if you want encryption.
 
-Future versions of rsync may support SSL for better authentication and
-encryption, but that is still being investigated.
+You can also make use of SSL/TLS encryption if you put rsync behind an
+SSL proxy.
+
+# SSL/TLS Daemon Setup
+
+When setting up an rsync daemon for access via SSL/TLS, you will need to
+configure a proxy (such as haproxy or nginx) as the front-end that handles the
+encryption.
+
+- You should limit the access to the backend-rsyncd port to only allow the
+  proxy to connect.  If it is on the same host as the proxy, then configuring
+  it to only listen on localhost is a good idea.
+- You should consider turning on the `proxy protocol` parameter if your proxy
+  supports sending that information.  The examples below assume that this is
+  enabled.
+
+An example haproxy setup is as follows:
+
+> ```
+> frontend fe_rsync-ssl
+>    bind :::874 ssl crt /etc/letsencrypt/example.com/combined.pem
+>    mode tcp
+>    use_backend be_rsync
+>
+> backend be_rsync
+>    mode tcp
+>    server local-rsync 127.0.0.1:873 check send-proxy
+> ```
+
+An example nginx proxy setup is as follows:
+
+> ```
+> stream {
+>    server {
+>        listen 874 ssl;
+>        listen [::]:874 ssl;
+>
+>        ssl_certificate /etc/letsencrypt/example.com/fullchain.pem;
+>        ssl_certificate_key /etc/letsencrypt/example.com/privkey.pem;
+>
+>        proxy_pass localhost:873;
+>        proxy_protocol on; # Requires "proxy protocol = true"
+>        proxy_timeout 1m;
+>        proxy_connect_timeout 5s;
+>    }
+> }
+> ```
 
 # EXAMPLES
 
@@ -1089,23 +1151,23 @@ A more sophisticated example would be:
 > max connections = 4
 > syslog facility = local5
 > pid file = /var/run/rsyncd.pid
-> 
+>
 > [ftp]
 >         path = /var/ftp/./pub
 >         comment = whole ftp area (approx 6.1 GB)
-> 
+>
 > [sambaftp]
 >         path = /var/ftp/./pub/samba
 >         comment = Samba ftp area (approx 300 MB)
-> 
+>
 > [rsyncftp]
 >         path = /var/ftp/./pub/rsync
 >         comment = rsync ftp area (approx 6 MB)
-> 
+>
 > [sambawww]
 >         path = /public_html/samba
 >         comment = Samba WWW pages (approx 240 MB)
-> 
+>
 > [cvs]
 >         path = /data/cvs
 >         comment = CVS repository (requires authentication)
@@ -1129,7 +1191,7 @@ The /etc/rsyncd.secrets file would look something like this:
 # BUGS
 
 Please report bugs! The rsync bug tracking system is online at
-<http://rsync.samba.org/>.
+<https://rsync.samba.org/>.
 
 # VERSION
 
@@ -1142,7 +1204,7 @@ COPYING for details.
 
 The primary ftp site for rsync is <ftp://rsync.samba.org/pub/rsync>
 
-A web site is available at <http://rsync.samba.org/>.
+A web site is available at <https://rsync.samba.org/>.
 
 We would be delighted to hear from you if you like this program.
 
@@ -1160,4 +1222,4 @@ rsync was written by Andrew Tridgell and Paul Mackerras.  Many people have
 later contributed to it.
 
 Mailing lists for support and development are available at
-<http://lists.samba.org/>.
+<https://lists.samba.org/>.

sender.c

@@ -65,13 +65,10 @@ BOOL extra_flist_sending_enabled;
  **/
 static struct sum_struct *receive_sums(int f)
 {
-	struct sum_struct *s;
-	int32 i;
+	struct sum_struct *s = new(struct sum_struct);
 	int lull_mod = protocol_version >= 31 ? 0 : allowed_lull * 5;
 	OFF_T offset = 0;
-
-	if (!(s = new(struct sum_struct)))
-		out_of_memory("receive_sums");
+	int32 i;
 
 	read_sum_head(f, s);
 
@@ -92,8 +89,7 @@ static struct sum_struct *receive_sums(int f)
 	if (s->count == 0)
 		return(s);
 
-	if (!(s->sums = new_array(struct sum_buf, s->count)))
-		out_of_memory("receive_sums");
+	s->sums = new_array(struct sum_buf, s->count);
 
 	for (i = 0; i < s->count; i++) {
 		s->sums[i].sum1 = read_int(f);

simd-checksum-x86_64.cpp

@@ -45,9 +45,10 @@
  * the available xmm registers, this optimized version may not be faster than
  * the pure C version anyway. Note that all x86-64 CPUs support at least SSE2.
  *
- * This file is compiled using GCC 4.8+'s C++ front end to allow the use of
- * the target attribute, selecting the fastest code path based on runtime
- * detection of CPU capabilities.
+ * This file is compiled using GCC 4.8+/clang 6+'s C++ front end to allow the
+ * use of the target attribute, selecting the fastest code path based on
+ * dispatch priority (GCC 5) or runtime detection of CPU capabilities (GCC 6+).
+ * GCC 4.x are not supported to ease configure.ac logic.
  */
 
 #ifdef __x86_64__
@@ -59,73 +60,34 @@
 
 #include <immintrin.h>
 
-/* Compatibility functions to let our SSSE3 algorithm run on SSE2 */
+/* Some clang versions don't like it when you use static with multi-versioned functions: linker errors */
+#ifdef __clang__
+#define MVSTATIC
+#else
+#define MVSTATIC static
+#endif
 
-__attribute__ ((target("sse2"))) static inline __m128i sse_interleave_odd_epi16(__m128i a, __m128i b)
-{
-    return _mm_packs_epi32(
-        _mm_srai_epi32(a, 16),
-        _mm_srai_epi32(b, 16)
-    );
-}
+// Missing from the headers on gcc 6 and older, clang 8 and older
+typedef long long __m128i_u __attribute__((__vector_size__(16), __may_alias__, __aligned__(1)));
+typedef long long __m256i_u __attribute__((__vector_size__(32), __may_alias__, __aligned__(1)));
 
-__attribute__ ((target("sse2"))) static inline __m128i sse_interleave_even_epi16(__m128i a, __m128i b)
-{
-    return sse_interleave_odd_epi16(
-        _mm_slli_si128(a, 2),
-        _mm_slli_si128(b, 2)
-    );
-}
+/* Compatibility macros to let our SSSE3 algorithm run with only SSE2.
+   These used to be neat individual functions with target attributes switching between SSE2 and SSSE3 implementations
+   as needed, but though this works perfectly with GCC, clang fails to inline those properly leading to a near 50%
+   performance drop - combined with static and inline modifiers gets you linker errors and even compiler crashes...
+*/
 
-__attribute__ ((target("sse2"))) static inline __m128i sse_mulu_odd_epi8(__m128i a, __m128i b)
-{
-    return _mm_mullo_epi16(
-        _mm_srli_epi16(a, 8),
-        _mm_srai_epi16(b, 8)
-    );
-}
+#define SSE2_INTERLEAVE_ODD_EPI16(a, b) _mm_packs_epi32(_mm_srai_epi32(a, 16), _mm_srai_epi32(b, 16))
+#define SSE2_INTERLEAVE_EVEN_EPI16(a, b) SSE2_INTERLEAVE_ODD_EPI16(_mm_slli_si128(a, 2), _mm_slli_si128(b, 2))
+#define SSE2_MULU_ODD_EPI8(a, b) _mm_mullo_epi16(_mm_srli_epi16(a, 8), _mm_srai_epi16(b, 8))
+#define SSE2_MULU_EVEN_EPI8(a, b) _mm_mullo_epi16(_mm_and_si128(a, _mm_set1_epi16(0xFF)), _mm_srai_epi16(_mm_slli_si128(b, 1), 8))
 
-__attribute__ ((target("sse2"))) static inline __m128i sse_mulu_even_epi8(__m128i a, __m128i b)
-{
-    return _mm_mullo_epi16(
-        _mm_and_si128(a, _mm_set1_epi16(0xFF)),
-        _mm_srai_epi16(_mm_slli_si128(b, 1), 8)
-    );
-}
+#define SSE2_HADDS_EPI16(a, b) _mm_adds_epi16(SSE2_INTERLEAVE_EVEN_EPI16(a, b), SSE2_INTERLEAVE_ODD_EPI16(a, b))
+#define SSE2_MADDUBS_EPI16(a, b) _mm_adds_epi16(SSE2_MULU_EVEN_EPI8(a, b), SSE2_MULU_ODD_EPI8(a, b))
 
-__attribute__ ((target("sse2"))) static inline __m128i sse_hadds_epi16(__m128i a, __m128i b)
-{
-    return _mm_adds_epi16(
-        sse_interleave_even_epi16(a, b),
-        sse_interleave_odd_epi16(a, b)
-    );
-}
-
-__attribute__ ((target("ssse3"))) static inline __m128i sse_hadds_epi16(__m128i a, __m128i b)
-{
-    return _mm_hadds_epi16(a, b);
-}
-
-__attribute__ ((target("sse2"))) static inline __m128i sse_maddubs_epi16(__m128i a, __m128i b)
-{
-    return _mm_adds_epi16(
-        sse_mulu_even_epi8(a, b),
-        sse_mulu_odd_epi8(a, b)
-    );
-}
-
-__attribute__ ((target("ssse3"))) static inline __m128i sse_maddubs_epi16(__m128i a, __m128i b)
-{
-    return _mm_maddubs_epi16(a, b);
-}
-
-/* These don't actually get called, but we need to define them. */
-__attribute__ ((target("default"))) static inline __m128i sse_interleave_odd_epi16(__m128i a, __m128i b) { return a; }
-__attribute__ ((target("default"))) static inline __m128i sse_interleave_even_epi16(__m128i a, __m128i b) { return a; }
-__attribute__ ((target("default"))) static inline __m128i sse_mulu_odd_epi8(__m128i a, __m128i b) { return a; }
-__attribute__ ((target("default"))) static inline __m128i sse_mulu_even_epi8(__m128i a, __m128i b) { return a; }
-__attribute__ ((target("default"))) static inline __m128i sse_hadds_epi16(__m128i a, __m128i b) { return a; }
-__attribute__ ((target("default"))) static inline __m128i sse_maddubs_epi16(__m128i a, __m128i b) { return a; }
+__attribute__ ((target("default"))) MVSTATIC int32 get_checksum1_avx2_64(schar* buf, int32 len, int32 i, uint32* ps1, uint32* ps2) { return i; }
+__attribute__ ((target("default"))) MVSTATIC int32 get_checksum1_ssse3_32(schar* buf, int32 len, int32 i, uint32* ps1, uint32* ps2) { return i; }
+__attribute__ ((target("default"))) MVSTATIC int32 get_checksum1_sse2_32(schar* buf, int32 len, int32 i, uint32* ps1, uint32* ps2) { return i; }
 
 /*
   Original loop per 4 bytes:
@@ -146,12 +108,7 @@ __attribute__ ((target("default"))) static inline __m128i sse_maddubs_epi16(__m1
     s1 += (uint32)(t1[0] + t1[1] + t1[2] + t1[3] + t1[4] + t1[5] + t1[6] + t1[7]) +
           32*CHAR_OFFSET;
  */
-/*
-  Both sse2 and ssse3 targets must be specified here or we lose (a lot) of
-  performance, possibly due to not unrolling+inlining the called targeted
-  functions.
- */
-__attribute__ ((target("sse2", "ssse3"))) static int32 get_checksum1_sse2_32(schar* buf, int32 len, int32 i, uint32* ps1, uint32* ps2)
+__attribute__ ((target("ssse3"))) MVSTATIC int32 get_checksum1_ssse3_32(schar* buf, int32 len, int32 i, uint32* ps1, uint32* ps2)
 {
     if (len > 32) {
         int aligned = ((uintptr_t)buf & 15) == 0;
@@ -167,16 +124,11 @@ __attribute__ ((target("sse2", "ssse3"))) static int32 get_checksum1_sse2_32(sch
 
         for (; i < (len-32); i+=32) {
             // Load ... 2*[int8*16]
-            // SSSE3 has _mm_lqqdu_si128, but this requires another
-            // target function for each SSE2 and SSSE3 loads. For reasons
-            // unknown (to me) we lose about 10% performance on some CPUs if
-            // we do that right here. We just use _mm_loadu_si128 as for all
-            // but a handful of specific old CPUs they are synonymous, and
-            // take the 1-5% hit on those specific CPUs where it isn't.
             __m128i in8_1, in8_2;
             if (!aligned) {
-                in8_1 = _mm_loadu_si128((__m128i_u*)&buf[i]);
-                in8_2 = _mm_loadu_si128((__m128i_u*)&buf[i + 16]);
+                // Synonymous with _mm_loadu_si128 on all but a handful of old CPUs
+                in8_1 = _mm_lddqu_si128((__m128i_u*)&buf[i]);
+                in8_2 = _mm_lddqu_si128((__m128i_u*)&buf[i + 16]);
             } else {
                 in8_1 = _mm_load_si128((__m128i_u*)&buf[i]);
                 in8_2 = _mm_load_si128((__m128i_u*)&buf[i + 16]);
@@ -185,13 +137,13 @@ __attribute__ ((target("sse2", "ssse3"))) static int32 get_checksum1_sse2_32(sch
             // (1*buf[i] + 1*buf[i+1]), (1*buf[i+2], 1*buf[i+3]), ... 2*[int16*8]
             // Fastest, even though multiply by 1
             __m128i mul_one = _mm_set1_epi8(1);
-            __m128i add16_1 = sse_maddubs_epi16(mul_one, in8_1);
-            __m128i add16_2 = sse_maddubs_epi16(mul_one, in8_2);
+            __m128i add16_1 = _mm_maddubs_epi16(mul_one, in8_1);
+            __m128i add16_2 = _mm_maddubs_epi16(mul_one, in8_2);
 
             // (4*buf[i] + 3*buf[i+1]), (2*buf[i+2], buf[i+3]), ... 2*[int16*8]
             __m128i mul_const = _mm_set1_epi32(4 + (3 << 8) + (2 << 16) + (1 << 24));
-            __m128i mul_add16_1 = sse_maddubs_epi16(mul_const, in8_1);
-            __m128i mul_add16_2 = sse_maddubs_epi16(mul_const, in8_2);
+            __m128i mul_add16_1 = _mm_maddubs_epi16(mul_const, in8_1);
+            __m128i mul_add16_2 = _mm_maddubs_epi16(mul_const, in8_2);
 
             // s2 += 32*s1
             ss2 = _mm_add_epi32(ss2, _mm_slli_epi32(ss1, 5));
@@ -224,7 +176,111 @@ __attribute__ ((target("sse2", "ssse3"))) static int32 get_checksum1_sse2_32(sch
             // [t1[0] + t1[1], t1[2] + t1[3] ...] [int16*8]
             // We could've combined this with generating sum_add32 above and
             // save an instruction but benchmarking shows that as being slower
-            __m128i add16 = sse_hadds_epi16(add16_1, add16_2);
+            __m128i add16 = _mm_hadds_epi16(add16_1, add16_2);
+
+            // [t1[0], t1[1], ...] -> [t1[0]*28 + t1[1]*24, ...] [int32*4]
+            __m128i mul32 = _mm_madd_epi16(add16, mul_t1);
+
+            // [sum(mul32), X, X, X] [int32*4]; faster than multiple _mm_hadd_epi32
+            mul32 = _mm_add_epi32(mul32, _mm_srli_si128(mul32, 4));
+            mul32 = _mm_add_epi32(mul32, _mm_srli_si128(mul32, 8));
+
+            // s2 += 28*t1[0] + 24*t1[1] + 20*t1[2] + 16*t1[3] + 12*t1[4] + 8*t1[5] + 4*t1[6]
+            ss2 = _mm_add_epi32(ss2, mul32);
+
+#if CHAR_OFFSET != 0
+            // s1 += 32*CHAR_OFFSET
+            __m128i char_offset_multiplier = _mm_set1_epi32(32 * CHAR_OFFSET);
+            ss1 = _mm_add_epi32(ss1, char_offset_multiplier);
+
+            // s2 += 528*CHAR_OFFSET
+            char_offset_multiplier = _mm_set1_epi32(528 * CHAR_OFFSET);
+            ss2 = _mm_add_epi32(ss2, char_offset_multiplier);
+#endif
+        }
+
+        _mm_store_si128((__m128i_u*)x, ss1);
+        *ps1 = x[0];
+        _mm_store_si128((__m128i_u*)x, ss2);
+        *ps2 = x[0];
+    }
+    return i;
+}
+
+/*
+  Same as SSSE3 version, but using macros defined above to emulate SSSE3 calls that are not available with SSE2.
+  For GCC-only the SSE2 and SSSE3 versions could be a single function calling other functions with the right
+  target attributes to emulate SSSE3 calls on SSE2 if needed, but clang doesn't inline those properly leading
+  to a near 50% performance drop.
+ */
+__attribute__ ((target("sse2"))) MVSTATIC int32 get_checksum1_sse2_32(schar* buf, int32 len, int32 i, uint32* ps1, uint32* ps2)
+{
+    if (len > 32) {
+        int aligned = ((uintptr_t)buf & 15) == 0;
+
+        uint32 x[4] = {0};
+        x[0] = *ps1;
+        __m128i ss1 = _mm_loadu_si128((__m128i_u*)x);
+        x[0] = *ps2;
+        __m128i ss2 = _mm_loadu_si128((__m128i_u*)x);
+
+        const int16 mul_t1_buf[8] = {28, 24, 20, 16, 12, 8, 4, 0};
+        __m128i mul_t1 = _mm_loadu_si128((__m128i_u*)mul_t1_buf);
+
+        for (; i < (len-32); i+=32) {
+            // Load ... 2*[int8*16]
+            __m128i in8_1, in8_2;
+            if (!aligned) {
+                in8_1 = _mm_loadu_si128((__m128i_u*)&buf[i]);
+                in8_2 = _mm_loadu_si128((__m128i_u*)&buf[i + 16]);
+            } else {
+                in8_1 = _mm_load_si128((__m128i_u*)&buf[i]);
+                in8_2 = _mm_load_si128((__m128i_u*)&buf[i + 16]);
+            }
+
+            // (1*buf[i] + 1*buf[i+1]), (1*buf[i+2], 1*buf[i+3]), ... 2*[int16*8]
+            // Fastest, even though multiply by 1
+            __m128i mul_one = _mm_set1_epi8(1);
+            __m128i add16_1 = SSE2_MADDUBS_EPI16(mul_one, in8_1);
+            __m128i add16_2 = SSE2_MADDUBS_EPI16(mul_one, in8_2);
+
+            // (4*buf[i] + 3*buf[i+1]), (2*buf[i+2], buf[i+3]), ... 2*[int16*8]
+            __m128i mul_const = _mm_set1_epi32(4 + (3 << 8) + (2 << 16) + (1 << 24));
+            __m128i mul_add16_1 = SSE2_MADDUBS_EPI16(mul_const, in8_1);
+            __m128i mul_add16_2 = SSE2_MADDUBS_EPI16(mul_const, in8_2);
+
+            // s2 += 32*s1
+            ss2 = _mm_add_epi32(ss2, _mm_slli_epi32(ss1, 5));
+
+            // [sum(t1[0]..t1[7]), X, X, X] [int32*4]; faster than multiple _mm_hadds_epi16
+            // Shifting left, then shifting right again and shuffling (rather than just
+            // shifting right as with mul32 below) to cheaply end up with the correct sign
+            // extension as we go from int16 to int32.
+            __m128i sum_add32 = _mm_add_epi16(add16_1, add16_2);
+            sum_add32 = _mm_add_epi16(sum_add32, _mm_slli_si128(sum_add32, 2));
+            sum_add32 = _mm_add_epi16(sum_add32, _mm_slli_si128(sum_add32, 4));
+            sum_add32 = _mm_add_epi16(sum_add32, _mm_slli_si128(sum_add32, 8));
+            sum_add32 = _mm_srai_epi32(sum_add32, 16);
+            sum_add32 = _mm_shuffle_epi32(sum_add32, 3);
+
+            // [sum(t2[0]..t2[7]), X, X, X] [int32*4]; faster than multiple _mm_hadds_epi16
+            __m128i sum_mul_add32 = _mm_add_epi16(mul_add16_1, mul_add16_2);
+            sum_mul_add32 = _mm_add_epi16(sum_mul_add32, _mm_slli_si128(sum_mul_add32, 2));
+            sum_mul_add32 = _mm_add_epi16(sum_mul_add32, _mm_slli_si128(sum_mul_add32, 4));
+            sum_mul_add32 = _mm_add_epi16(sum_mul_add32, _mm_slli_si128(sum_mul_add32, 8));
+            sum_mul_add32 = _mm_srai_epi32(sum_mul_add32, 16);
+            sum_mul_add32 = _mm_shuffle_epi32(sum_mul_add32, 3);
+
+            // s1 += t1[0] + t1[1] + t1[2] + t1[3] + t1[4] + t1[5] + t1[6] + t1[7]
+            ss1 = _mm_add_epi32(ss1, sum_add32);
+
+            // s2 += t2[0] + t2[1] + t2[2] + t2[3] + t2[4] + t2[5] + t2[6] + t2[7]
+            ss2 = _mm_add_epi32(ss2, sum_mul_add32);
+
+            // [t1[0] + t1[1], t1[2] + t1[3] ...] [int16*8]
+            // We could've combined this with generating sum_add32 above and
+            // save an instruction but benchmarking shows that as being slower
+            __m128i add16 = SSE2_HADDS_EPI16(add16_1, add16_2);
 
             // [t1[0], t1[1], ...] -> [t1[0]*28 + t1[1]*24, ...] [int32*4]
             __m128i mul32 = _mm_madd_epi16(add16, mul_t1);
@@ -270,7 +326,7 @@ __attribute__ ((target("sse2", "ssse3"))) static int32 get_checksum1_sse2_32(sch
     s1 += (uint32)(t1[0] + t1[1] + t1[2] + t1[3] + t1[4] + t1[5] + t1[6] + t1[7] + t1[8] + t1[9] + t1[10] + t1[11] + t1[12] + t1[13] + t1[14] + t1[15]) +
           64*CHAR_OFFSET;
  */
-__attribute__ ((target("avx2"))) static int32 get_checksum1_avx2_64(schar* buf, int32 len, int32 i, uint32* ps1, uint32* ps2)
+__attribute__ ((target("avx2"))) MVSTATIC int32 get_checksum1_avx2_64(schar* buf, int32 len, int32 i, uint32* ps1, uint32* ps2)
 {
     if (len > 64) {
         // Instructions reshuffled compared to SSE2 for slightly better performance
@@ -377,17 +433,7 @@ __attribute__ ((target("avx2"))) static int32 get_checksum1_avx2_64(schar* buf,
     return i;
 }
 
-__attribute__ ((target("default"))) static int32 get_checksum1_avx2_64(schar* buf, int32 len, int32 i, uint32* ps1, uint32* ps2)
-{
-    return i;
-}
-
-__attribute__ ((target("default"))) static int32 get_checksum1_sse2_32(schar* buf, int32 len, int32 i, uint32* ps1, uint32* ps2)
-{
-    return i;
-}
-
-static inline int32 get_checksum1_default_1(schar* buf, int32 len, int32 i, uint32* ps1, uint32* ps2)
+static int32 get_checksum1_default_1(schar* buf, int32 len, int32 i, uint32* ps1, uint32* ps2)
 {
     uint32 s1 = *ps1;
     uint32 s2 = *ps2;
@@ -403,9 +449,10 @@ static inline int32 get_checksum1_default_1(schar* buf, int32 len, int32 i, uint
     return i;
 }
 
-extern "C" {
-
-uint32 get_checksum1(char *buf1, int32 len)
+/* With GCC 10 putting this implementation inside 'extern "C"' causes an
+   assembler error. That worked fine on GCC 5-9 and clang 6-10...
+  */
+static inline uint32 get_checksum1_cpp(char *buf1, int32 len)
 {
     int32 i = 0;
     uint32 s1 = 0;
@@ -414,7 +461,10 @@ uint32 get_checksum1(char *buf1, int32 len)
     // multiples of 64 bytes using AVX2 (if available)
     i = get_checksum1_avx2_64((schar*)buf1, len, i, &s1, &s2);
 
-    // multiples of 32 bytes using SSE2/SSSE3 (if available)
+    // multiples of 32 bytes using SSSE3 (if available)
+    i = get_checksum1_ssse3_32((schar*)buf1, len, i, &s1, &s2);
+
+    // multiples of 32 bytes using SSE2 (if available)
     i = get_checksum1_sse2_32((schar*)buf1, len, i, &s1, &s2);
 
     // whatever is left
@@ -423,7 +473,70 @@ uint32 get_checksum1(char *buf1, int32 len)
     return (s1 & 0xffff) + (s2 << 16);
 }
 
-} // "C"
+extern "C" {
+
+uint32 get_checksum1(char *buf1, int32 len)
+{
+    return get_checksum1_cpp(buf1, len);
+}
+
+} // extern "C"
+
+#ifdef BENCHMARK_SIMD_CHECKSUM1
+#pragma clang optimize off
+#pragma GCC push_options
+#pragma GCC optimize ("O0")
+
+#define ROUNDS 1024
+#define BLOCK_LEN 1024*1024
+
+#ifndef CLOCK_MONOTONIC_RAW
+#define CLOCK_MONOTONIC_RAW CLOCK_MONOTONIC
+#endif
+
+static void benchmark(const char* desc, int32 (*func)(schar* buf, int32 len, int32 i, uint32* ps1, uint32* ps2), schar* buf, int32 len) {
+    struct timespec start, end;
+    uint64_t us;
+    uint32_t cs, s1, s2;
+    int i, next;
+
+    clock_gettime(CLOCK_MONOTONIC_RAW, &start);
+    for (i = 0; i < ROUNDS; i++) {
+        s1 = s2 = 0;
+        next = func((schar*)buf, len, 0, &s1, &s2);
+        get_checksum1_default_1((schar*)buf, len, next, &s1, &s2);
+    }
+    clock_gettime(CLOCK_MONOTONIC_RAW, &end);
+    us = next == 0 ? 0 : (end.tv_sec - start.tv_sec) * 1000000 + (end.tv_nsec - start.tv_nsec) / 1000;
+    cs = next == 0 ? 0 : (s1 & 0xffff) + (s2 << 16);
+    printf("%-5s :: %5.0f MB/s :: %08x\n", desc, us ? (float)(len / (1024 * 1024) * ROUNDS) / ((float)us / 1000000.0f) : 0, cs);
+}
+
+static int32 get_checksum1_auto(schar* buf, int32 len, int32 i, uint32* ps1, uint32* ps2) {
+    uint32 cs = get_checksum1((char*)buf, len);
+    *ps1 = cs & 0xffff;
+    *ps2 = cs >> 16;
+    return len;
+}
+
+int main() {
+    int i;
+    unsigned char* buf = (unsigned char*)malloc(BLOCK_LEN);
+    for (i = 0; i < BLOCK_LEN; i++) buf[i] = (i + (i % 3) + (i % 11)) % 256;
+
+    benchmark("Auto", get_checksum1_auto, (schar*)buf, BLOCK_LEN);
+    benchmark("Raw-C", get_checksum1_default_1, (schar*)buf, BLOCK_LEN);
+    benchmark("SSE2", get_checksum1_sse2_32, (schar*)buf, BLOCK_LEN);
+    benchmark("SSSE3", get_checksum1_ssse3_32, (schar*)buf, BLOCK_LEN);
+    benchmark("AVX2", get_checksum1_avx2_64, (schar*)buf, BLOCK_LEN);
+
+    free(buf);
+    return 0;
+}
+
+#pragma GCC pop_options
+#pragma clang optimize on
+#endif /* BENCHMARK_SIMD_CHECKSUM1 */
 
 #endif /* HAVE_SIMD */
 #endif /* __cplusplus */

socket.c

@@ -26,6 +26,7 @@
 
 #include "rsync.h"
 #include "itypes.h"
+#include "ifuncs.h"
 #ifdef HAVE_NETINET_IN_SYSTM_H
 #include <netinet/in_systm.h>
 #endif
@@ -248,8 +249,6 @@ int open_socket_out(char *host, int port, const char *bind_addr, int af_hint)
 
 	for (res = res0, addr_cnt = 0; res; res = res->ai_next, addr_cnt++) {}
 	errnos = new_array0(int, addr_cnt);
-	if (!errnos)
-		out_of_memory("open_socket_out");
 
 	s = -1;
 	/* Try to connect to all addresses for this machine until we get
@@ -354,8 +353,7 @@ int open_socket_out_wrapped(char *host, int port, const char *bind_addr, int af_
 				len += hlen;
 		}
 		f = prog;
-		if (!(prog = new_array(char, len)))
-			out_of_memory("open_socket_out_wrapped");
+		prog = new_array(char, len);
 		for (t = prog; *f; f++) {
 			if (*f == '%') {
 				switch (*++f) {
@@ -423,8 +421,6 @@ static int *open_socket_in(int type, int port, const char *bind_addr,
 
 	socks = new_array(int, maxs + 1);
 	errmsgs = new_array(char *, maxs);
-	if (!socks || !errmsgs)
-		out_of_memory("open_socket_in");
 
 	/* We may not be able to create the socket, if for example the
 	 * machine knows about IPv6 in the C library, but not in the
@@ -684,9 +680,6 @@ void set_socket_options(int fd, char *options)
 
 	options = strdup(options);
 
-	if (!options)
-		out_of_memory("set_socket_options");
-
 	for (tok = strtok(options, " \t,"); tok; tok = strtok(NULL," \t,")) {
 		int ret=0,i;
 		int value = 1;

support/atomic-rsync

@@ -1,4 +1,4 @@
-#!/usr/bin/perl
+#!/usr/bin/env perl
 #
 # This script lets you update a hierarchy of files in an atomic way by
 # first creating a new hierarchy using rsync's --link-dest option, and

support/cvs2includes

@@ -1,4 +1,4 @@
-#!/usr/bin/perl
+#!/usr/bin/env perl
 #
 # This script finds all CVS/Entries files in the current directory and below
 # and creates a local .cvsinclude file with non-inherited rules including each

support/deny-rsync

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 # Send an error message via the rsync-protocol to a non-daemon client rsync.
 #
 # Usage:  deny-rsync "message"

support/file-attr-restore

@@ -1,4 +1,4 @@
-#!/usr/bin/perl
+#!/usr/bin/env perl
 # This script will parse the output of "find ARG [ARG...] -ls" and
 # apply (at your discretion) the permissions, owner, and group info
 # it reads onto any existing files and dirs (it doesn't try to affect

support/files-to-excludes

@@ -1,4 +1,4 @@
-#!/usr/bin/perl
+#!/usr/bin/env perl
 # This script takes an input of filenames and outputs a set of
 # include/exclude directives that can be used by rsync to copy
 # just the indicated files using an --exclude-from=FILE option.

support/git-set-file-times

@@ -1,4 +1,4 @@
-#!/usr/bin/python3
+#!/usr/bin/env python3
 
 import os, re, argparse, subprocess
 from datetime import datetime

support/instant-rsyncd

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 
 # instant-rsyncd lets you quickly set up and start a simple, unprivileged rsync
 # daemon with a single module in the current directory.  I've found it

support/logfilter

@@ -1,4 +1,4 @@
-#!/usr/bin/perl
+#!/usr/bin/env perl
 # Filter the rsync daemon log messages by module name.  The log file can be
 # in either syslog format or rsync's own log-file format.  Note that the
 # MODULE_NAME parameter is used in a regular-expression match in order to

support/lsh

@@ -1,4 +1,4 @@
-#!/usr/bin/perl
+#!/usr/bin/env perl
 # This script can be used as a "remote shell" command that is only
 # capable of pretending to connect to "localhost".  This is useful
 # for testing or for running a local copy where the sender and the

support/mapfrom

@@ -1,4 +1,4 @@
-#!/usr/bin/perl
+#!/usr/bin/env perl
 # This helper script makes it easy to use a passwd or group file to map
 # values in a LOCAL transfer.  For instance, if you mount a backup that
 # does not have the same passwd setup as the local machine, you can do

support/mapto

@@ -1,4 +1,4 @@
-#!/usr/bin/perl
+#!/usr/bin/env perl
 # This helper script makes it easy to use a passwd or group file to map
 # values in a LOCAL transfer.  For instance, if you mount a backup that
 # does not have the same passwd setup as the local machine, you can do

support/mnt-excl

@@ -1,4 +1,4 @@
-#!/usr/bin/perl
+#!/usr/bin/env perl
 # This script takes a command-line arg of a source directory
 # that will be passed to rsync, and generates a set of excludes
 # that will exclude all mount points from the list.  This is

support/munge-symlinks

@@ -1,4 +1,4 @@
-#!/usr/bin/perl
+#!/usr/bin/env perl
 # This script will either prefix all symlink values with the string
 # "/rsyncd-munged/" or remove that prefix.
 

support/rrsync

@@ -1,8 +1,8 @@
-#!/usr/bin/perl
+#!/usr/bin/env perl
 # Name: /usr/local/bin/rrsync (should also have a symlink in /usr/bin)
 # Purpose: Restricts rsync to subdirectory declared in .ssh/authorized_keys
 # Author: Joe Smith <js-cgi@inwap.com> 30-Sep-2004
-# Modified by: Wayne Davison <wayned@samba.org>
+# Modified by: Wayne Davison <wayne@opencoder.net>
 use strict;
 
 use Socket;
@@ -62,7 +62,7 @@ die "$0 reading from write-only server not allowed\n" if $only eq 'w' && $am_sen
 # To disable a short-named option, add its letter to this string:
 our $short_disabled = 's';
 
-our $short_no_arg = 'ACDEHIJKLORSUWXbcdgklmnoprstuvxyz'; # DO NOT REMOVE ANY
+our $short_no_arg = 'ACDEHIJKLORSUWXbcdgklmnopqrstuvxyz'; # DO NOT REMOVE ANY
 our $short_with_num = '@B'; # DO NOT REMOVE ANY
 
 # To disable a long-named option, change its value to a -1.  The values mean:
@@ -76,6 +76,7 @@ our %long_opt = (
   'checksum-choice' => 1,
   'checksum-seed' => 1,
   'compare-dest' => 2,
+  'compress-choice' => 1,
   'compress-level' => 1,
   'copy-dest' => 2,
   'copy-unsafe-links' => 0,
@@ -110,6 +111,7 @@ our %long_opt = (
   'list-only' => 0,
   'log-file' => $only eq 'r' ? -1 : 3,
   'log-format' => 1,
+  'max-alloc' => 1,
   'max-delete' => 1,
   'max-size' => 1,
   'min-size' => 1,
@@ -120,6 +122,7 @@ our %long_opt = (
   'no-relative' => 0,
   'no-specials' => 0,
   'numeric-ids' => 0,
+  'old-compress' => 0,
   'one-file-system' => 0,
   'only-write-batch' => 1,
   'open-noatime' => 0,

support/rsync-no-vanished

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 
 IGNOREEXIT=24
 IGNOREOUT='^(file has vanished: |rsync warning: some files vanished before they could be transferred)'

support/rsync-slash-strip

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 # This script can be used as an rsync command-line filter that strips a single
 # trailing slash from each arg.  That treats "src/" the same as "src", thus
 # you need to use "src/." or "src//" for just the contents of the "src" dir.

support/rsyncstats

@@ -1,4 +1,4 @@
-#!/usr/bin/perl
+#!/usr/bin/env perl
 #
 # This script parses the default logfile format produced by rsync when running
 # as a daemon with transfer logging enabled.  It also parses a slightly tweaked

t_stub.c

@@ -33,6 +33,7 @@ int preserve_xattrs = 0;
 int preserve_perms = 0;
 int preserve_executability = 0;
 int open_noatime = 0;
+size_t max_alloc = 1024*1024*1024; /* max_alloc is needed when combined with util2.o */
 char *partial_dir;
 char *module_dir;
 filter_rule_list daemon_filter_list;

testhelp/maketree.py

@@ -1,4 +1,4 @@
-#! /usr/bin/python2.2
+#!/usr/bin/env python2
 
 # Copyright (C) 2002 by Martin Pool <mbp@samba.org>
 

testsuite/atimes.test

@@ -4,6 +4,8 @@
 
 . "$suitedir/rsync.fns"
 
+$RSYNC --version | grep "[, ] atimes" >/dev/null || test_skipped "Rsync is configured without atimes support"
+
 mkdir "$fromdir"
 
 touch "$fromdir/foo"

testsuite/daemon.test

@@ -91,12 +91,13 @@ drwxr-xr-x         DIR ####/##/## ##:##:## foo
 EOT
 diff $diffopt "$chkfile" "$outfile" || test_fail "test 3 failed"
 
-$RSYNC -rU localhost::test-from/f* \
-    | sed "$FILE_REPL" | sed "$DIR_REPL" | sed "$LS_REPL" \
-    | tee "$outfile"
-cat <<EOT >"$chkfile"
+if $RSYNC --version | grep "[, ] atimes" >/dev/null; then
+    $RSYNC -rU localhost::test-from/f* \
+	| sed "$FILE_REPL" | sed "$DIR_REPL" | sed "$LS_REPL" \
+	| tee "$outfile"
+    cat <<EOT >"$chkfile"
 drwxr-xr-x         DIR ####/##/## ##:##:##                     foo
 -rw-r--r--           4 ####/##/## ##:##:## ####/##/## ##:##:## foo/one
 EOT
-diff $diffopt "$chkfile" "$outfile" || test_fail "test 3 failed"
-
+    diff $diffopt "$chkfile" "$outfile" || test_fail "test 3 failed"
+fi

testsuite/itemize.test

@@ -18,7 +18,7 @@ outfile="$scratchdir/rsync.out"
 makepath "$fromdir/foo"
 makepath "$fromdir/bar/baz"
 cp -p "$srcdir/configure.ac" "$fromdir/foo/config1"
-cp -p "$srcdir/config.h.in" "$fromdir/foo/config2"
+cp -p "$srcdir/config.sub" "$fromdir/foo/config2"
 cp -p "$srcdir/rsync.h" "$fromdir/bar/baz/rsync"
 chmod 600 "$fromdir"/foo/config? "$fromdir/bar/baz/rsync"
 umask 0
@@ -84,7 +84,7 @@ rm "$todir/foo/sym"
 umask 0
 ln -s ../bar/baz "$todir/foo/sym"
 umask 022
-cp -p "$srcdir/config.h.in" "$fromdir/foo/config2"
+cp -p "$srcdir/config.sub" "$fromdir/foo/config2"
 chmod 600 "$fromdir/foo/config2"
 chmod 777 "$todir/bar/baz/rsync"
 

testsuite/rsync.fns

@@ -97,7 +97,8 @@ printmsg() {
 }
 
 rsync_ls_lR() {
-    find "$@" -print | sort | sed 's/ /\\ /g' | xargs "$TOOLDIR/tls" $TLS_ARGS
+    find "$@" -name .git -prune -o -name auto-build-save -prune -o -print | \
+	sort | sed 's/ /\\ /g' | xargs "$TOOLDIR/tls" $TLS_ARGS
 }
 
 get_testuid() {
@@ -296,8 +297,9 @@ use chroot = no
 munge symlinks = no
 hosts allow = localhost 127.0.0.0/24 192.168.0.0/16 10.0.0.0/8 $hostname
 log file = $logfile
-log format = %i %h [%a] %m (%u) %l %f%L
 transfer logging = yes
+# We don't define log format here so that the test-hidden module will default
+# to the internal static string (since we had a crash trying to tweak it).
 exclude = ? foobar.baz
 max verbosity = 4
 $uid_setting
@@ -305,16 +307,19 @@ $gid_setting
 
 [test-from]
 	path = $fromdir
+	log format = %i %h [%a] %m (%u) %l %f%L
 	read only = yes
 	comment = r/o
 
 [test-to]
 	path = $todir
+	log format = %i %h [%a] %m (%u) %l %f%L
 	read only = no
 	comment = r/w
 
 [test-scratch]
 	path = $scratchdir
+	log format = %i %h [%a] %m (%u) %l %f%L
 	read only = no
 
 [test-hidden]

token.c

@@ -129,8 +129,7 @@ static void add_suffix(struct suffix_tree **prior, char ltr, const char *str)
 		if (node->letter > ltr)
 			break;
 	}
-	if (!(newnode = new(struct suffix_tree)))
-		out_of_memory("add_suffix");
+	newnode = new(struct suffix_tree);
 	newnode->sibling = node;
 	newnode->child = NULL;
 	newnode->letter = ltr;
@@ -147,8 +146,7 @@ static void add_nocompress_suffixes(const char *str)
 	char *buf, *t;
 	const char *f = str;
 
-	if (!(buf = new_array(char, strlen(f) + 1)))
-		out_of_memory("add_nocompress_suffixes");
+	buf = new_array(char, strlen(f) + 1);
 
 	while (*f) {
 		if (*f == '/') {
@@ -186,8 +184,7 @@ static void init_set_compression(void)
 	else
 		f = lp_dont_compress(module_id);
 
-	if (!(match_list = t = new_array(char, strlen(f) + 2)))
-		out_of_memory("set_compression");
+	match_list = t = new_array(char, strlen(f) + 2);
 
 	per_file_default_level = do_compression_level;
 
@@ -282,11 +279,8 @@ static int32 simple_recv_token(int f, char **data)
 	static char *buf;
 	int32 n;
 
-	if (!buf) {
+	if (!buf)
 		buf = new_array(char, CHUNK_SIZE);
-		if (!buf)
-			out_of_memory("simple_recv_token");
-	}
 
 	if (residue == 0) {
 		int32 i = read_int(f);
@@ -373,8 +367,7 @@ send_deflated_token(int f, int32 token, struct map_struct *buf, OFF_T offset, in
 				rprintf(FERROR, "compression init failed\n");
 				exit_cleanup(RERR_PROTOCOL);
 			}
-			if ((obuf = new_array(char, OBUF_SIZE)) == NULL)
-				out_of_memory("send_deflated_token");
+			obuf = new_array(char, OBUF_SIZE);
 			init_done = 1;
 		} else
 			deflateReset(&tx_strm);
@@ -383,8 +376,7 @@ send_deflated_token(int f, int32 token, struct map_struct *buf, OFF_T offset, in
 		flush_pending = 0;
 	} else if (last_token == -2) {
 		run_start = token;
-	} else if (nb != 0 || token != last_token + 1
-		   || token >= run_start + 65536) {
+	} else if (nb != 0 || token != last_token + 1 || token >= run_start + 65536) {
 		/* output previous run */
 		r = run_start - last_run_end;
 		n = last_token - run_start;
@@ -518,9 +510,8 @@ static int32 recv_deflated_token(int f, char **data)
 					rprintf(FERROR, "inflate init failed\n");
 					exit_cleanup(RERR_PROTOCOL);
 				}
-				if (!(cbuf = new_array(char, MAX_DATA_COUNT))
-				 || !(dbuf = new_array(char, AVAIL_OUT_SIZE(CHUNK_SIZE))))
-					out_of_memory("recv_deflated_token");
+				cbuf = new_array(char, MAX_DATA_COUNT);
+				dbuf = new_array(char, AVAIL_OUT_SIZE(CHUNK_SIZE));
 				init_done = 1;
 			} else {
 				inflateReset(&rx_strm);
@@ -687,7 +678,6 @@ static void send_zstd_token(int f, int32 token, struct map_struct *buf, OFF_T of
 
 	/* initialization */
 	if (!comp_init_done) {
-
 		zstd_cctx = ZSTD_createCCtx();
 		if (!zstd_cctx) {
 			rprintf(FERROR, "compression init failed\n");
@@ -695,8 +685,6 @@ static void send_zstd_token(int f, int32 token, struct map_struct *buf, OFF_T of
 		}
 
 		obuf = new_array(char, OBUF_SIZE);
-		if (!obuf)
-			out_of_memory("send_deflated_token");
 
 		ZSTD_CCtx_setParameter(zstd_cctx, ZSTD_c_compressionLevel, do_compression_level);
 		zstd_out_buff.dst = obuf + 2;
@@ -710,10 +698,7 @@ static void send_zstd_token(int f, int32 token, struct map_struct *buf, OFF_T of
 		flush_pending = 0;
 	} else if (last_token == -2) {
 		run_start = token;
-
-	} else if (nb != 0 || token != last_token + 1
-		   || token >= run_start + 65536) {
-
+	} else if (nb != 0 || token != last_token + 1 || token >= run_start + 65536) {
 		/* output previous run */
 		r = run_start - last_run_end;
 		n = last_token - run_start;
@@ -795,7 +780,6 @@ static int32 recv_zstd_token(int f, char **data)
 	int r;
 
 	if (!decomp_init_done) {
-
 		zstd_dctx = ZSTD_createDCtx();
 		if (!zstd_dctx) {
 			rprintf(FERROR, "ZSTD_createDStream failed\n");
@@ -806,8 +790,6 @@ static int32 recv_zstd_token(int f, char **data)
 		out_buffer_size = ZSTD_DStreamOutSize() * 2;
 		cbuf = new_array(char, MAX_DATA_COUNT);
 		dbuf = new_array(char, out_buffer_size);
-		if (!cbuf || !dbuf)
-			out_of_memory("recv_zstd_token");
 
 		zstd_in_buff.src = cbuf;
 		zstd_out_buff.dst = dbuf;
@@ -815,30 +797,31 @@ static int32 recv_zstd_token(int f, char **data)
 		decomp_init_done = 1;
 	}
 
-	do {
-	switch (recv_state) {
-	case r_init:
-		recv_state = r_idle;
-		rx_token = 0;
-		break;
+	for (;;) {
+		switch (recv_state) {
+		case r_init:
+			recv_state = r_idle;
+			rx_token = 0;
+			break;
 
-	case r_idle:
-		flag = read_byte(f);
-		if ((flag & 0xC0) == DEFLATED_DATA) {
-			n = ((flag & 0x3f) << 8) + read_byte(f);
-			read_buf(f, cbuf, n);
+		case r_idle:
+			flag = read_byte(f);
+			if ((flag & 0xC0) == DEFLATED_DATA) {
+				n = ((flag & 0x3f) << 8) + read_byte(f);
+				read_buf(f, cbuf, n);
 
-			zstd_in_buff.size = n;
-			zstd_in_buff.pos = 0;
+				zstd_in_buff.size = n;
+				zstd_in_buff.pos = 0;
 
-			recv_state = r_inflating;
+				recv_state = r_inflating;
+				break;
+			}
 
-		} else if (flag == END_FLAG) {
-			/* that's all folks */
-			recv_state = r_init;
-			return 0;
-
-		} else {
+			if (flag == END_FLAG) {
+				/* that's all folks */
+				recv_state = r_init;
+				return 0;
+			}
 			/* here we have a token of some kind */
 			if (flag & TOKEN_REL) {
 				rx_token += flag & 0x3f;
@@ -851,45 +834,42 @@ static int32 recv_zstd_token(int f, char **data)
 				recv_state = r_running;
 			}
 			return -1 - rx_token;
+
+		case r_inflated: /* zstd doesn't get into this state */
+			break;
+
+		case r_inflating:
+			zstd_out_buff.size = out_buffer_size;
+			zstd_out_buff.pos = 0;
+
+			r = ZSTD_decompressStream(zstd_dctx, &zstd_out_buff, &zstd_in_buff);
+			n = zstd_out_buff.pos;
+			if (ZSTD_isError(r)) {
+				rprintf(FERROR, "ZSTD decomp returned %d (%d bytes)\n", r, n);
+				exit_cleanup(RERR_STREAMIO);
+			}
+
+			/*
+			 * If the input buffer is fully consumed and the output
+			 * buffer is not full then next step is to read more
+			 * data.
+			 */
+			if (zstd_in_buff.size == zstd_in_buff.pos && n < out_buffer_size)
+				recv_state = r_idle;
+
+			if (n != 0) {
+				*data = dbuf;
+				return n;
+			}
+			break;
+
+		case r_running:
+			++rx_token;
+			if (--rx_run == 0)
+				recv_state = r_idle;
+			return -1 - rx_token;
 		}
-		break;
-
-	case r_inflating:
-		zstd_out_buff.size = out_buffer_size;
-		zstd_out_buff.pos = 0;
-
-		r = ZSTD_decompressStream(zstd_dctx, &zstd_out_buff, &zstd_in_buff);
-		n = zstd_out_buff.pos;
-		if (ZSTD_isError(r)) {
-			rprintf(FERROR, "ZSTD decomp returned %d (%d bytes)\n", r, n);
-			exit_cleanup(RERR_STREAMIO);
-		}
-
-		/*
-		 * If the input buffer is fully consumed and the output
-		 * buffer is not full then next step is to read more
-		 * data.
-		 */
-		if (zstd_in_buff.size == zstd_in_buff.pos && n < out_buffer_size)
-			recv_state = r_idle;
-
-		if (n != 0) {
-			*data = dbuf;
-			return n;
-		}
-		break;
-
-	case r_running:
-		++rx_token;
-		if (--rx_run == 0)
-			recv_state = r_idle;
-		return -1 - rx_token;
-		break;
-
-	case r_inflated:
-		break;
 	}
-	} while (1);
 }
 #endif /* SUPPORT_ZSTD */
 
@@ -903,8 +883,7 @@ send_compressed_token(int f, int32 token, struct map_struct *buf, OFF_T offset,
 
 	if (last_token == -1) {
 		if (!init_done) {
-			if ((obuf = new_array(char, size)) == NULL)
-				out_of_memory("send_compressed_token");
+			obuf = new_array(char, size);
 			init_done = 1;
 		}
 		last_run_end = 0;
@@ -912,8 +891,7 @@ send_compressed_token(int f, int32 token, struct map_struct *buf, OFF_T offset,
 		flush_pending = 0;
 	} else if (last_token == -2) {
 		run_start = token;
-	} else if (nb != 0 || token != last_token + 1
-		   || token >= run_start + 65536) {
+	} else if (nb != 0 || token != last_token + 1 || token >= run_start + 65536) {
 		/* output previous run */
 		r = run_start - last_run_end;
 		n = last_token - run_start;
@@ -938,7 +916,6 @@ send_compressed_token(int f, int32 token, struct map_struct *buf, OFF_T offset,
 		const char *next_in;
 
 		do {
-			char *ptr = obuf;
 			char *next_out = obuf + 2;
 
 			if (available_out == 0) {
@@ -953,10 +930,10 @@ send_compressed_token(int f, int32 token, struct map_struct *buf, OFF_T offset,
 				exit_cleanup(RERR_STREAMIO);
 			}
 			if (available_out <= MAX_DATA_COUNT) {
-				ptr[0] = DEFLATED_DATA + (available_out >> 8);
-				ptr[1] = available_out;
+				obuf[0] = DEFLATED_DATA + (available_out >> 8);
+				obuf[1] = available_out;
 
-				write_buf(f, ptr, available_out + 2);
+				write_buf(f, obuf, available_out + 2);
 
 				available_out = 0;
 				nb -= available_in;
@@ -965,14 +942,14 @@ send_compressed_token(int f, int32 token, struct map_struct *buf, OFF_T offset,
 		} while (nb != 0);
 		flush_pending = token == -2;
 	}
-	if (token == -1)
+	if (token == -1) {
 		/* end of file - clean up */
 		write_byte(f, END_FLAG);
+	}
 }
 
 static int32 recv_compressed_token(int f, char **data)
 {
-	static int32 saved_flag;
 	static int init_done;
 	int32 n, flag;
 	int size = MAX(LZ4_compressBound(CHUNK_SIZE), MAX_DATA_COUNT+2);
@@ -984,21 +961,16 @@ static int32 recv_compressed_token(int f, char **data)
 		switch (recv_state) {
 		case r_init:
 			if (!init_done) {
-				if (!(cbuf = new_array(char, MAX_DATA_COUNT))
-				 || !(dbuf = new_array(char, size)))
-					out_of_memory("recv_compressed_token");
+				cbuf = new_array(char, MAX_DATA_COUNT);
+				dbuf = new_array(char, size);
 				init_done = 1;
 			}
 			recv_state = r_idle;
 			rx_token = 0;
 			break;
+
 		case r_idle:
-		case r_inflated:
-			if (saved_flag) {
-				flag = saved_flag & 0xff;
-				saved_flag = 0;
-			} else
-				flag = read_byte(f);
+			flag = read_byte(f);
 			if ((flag & 0xC0) == DEFLATED_DATA) {
 				n = ((flag & 0x3f) << 8) + read_byte(f);
 				read_buf(f, cbuf, n);
@@ -1008,9 +980,6 @@ static int32 recv_compressed_token(int f, char **data)
 				break;
 			}
 
-			if (recv_state == r_inflated)
-				recv_state = r_idle;
-
 			if (flag == END_FLAG) {
 				/* that's all folks */
 				recv_state = r_init;
@@ -1036,10 +1005,13 @@ static int32 recv_compressed_token(int f, char **data)
 				rprintf(FERROR, "uncompress failed: %d\n", avail_out);
 				exit_cleanup(RERR_STREAMIO);
 			}
-			recv_state = r_inflated;
+			recv_state = r_idle;
 			*data = dbuf;
 			return avail_out;
 
+		case r_inflated: /* lz4 doesn't get into this state */
+			break;
+
 		case r_running:
 			++rx_token;
 			if (--rx_run == 0)
@@ -1047,53 +1019,7 @@ static int32 recv_compressed_token(int f, char **data)
 			return -1 - rx_token;
 		}
 	}
-
 }
-
-# if 0
-static void see_uncompressed_token(char *buf, int32 len)
-{
-	static const char *next_in;
-	static int avail_in;
-	int avail_out;
-
-	int32 blklen;
-	char hdr[5];
-
-	avail_in = 0;
-	blklen = 0;
-	hdr[0] = 0;
-	do {
-		if (avail_in == 0 && len != 0) {
-			if (blklen == 0) {
-				/* Give it a fake stored-block header. */
-				next_in = hdr;
-				avail_in = 5;
-				blklen = len;
-				if (blklen > 0xffff)
-					blklen = 0xffff;
-				hdr[1] = blklen;
-				hdr[2] = blklen >> 8;
-				hdr[3] = ~hdr[1];
-				hdr[4] = ~hdr[2];
-			} else {
-				next_in = (char *)buf;
-				avail_in = blklen;
-				if (protocol_version >= 31) /* Newer protocols avoid a data-duplicating bug */
-					buf += blklen;
-				len -= blklen;
-				blklen = 0;
-			}
-		}
-		avail_out = LZ4_decompress_safe(next_in, dbuf, avail_in, LZ4_compressBound(CHUNK_SIZE));
-		if (avail_out < 0) {
-			rprintf(FERROR, "uncompress failed: %d\n", avail_out);
-			exit_cleanup(RERR_STREAMIO);
-		}
-
-	} while (len);
-}
-# endif /* 0 */
 #endif /* SUPPORT_LZ4 */
 
 /**
@@ -1135,30 +1061,23 @@ void send_token(int f, int32 token, struct map_struct *buf, OFF_T offset,
  */
 int32 recv_token(int f, char **data)
 {
-	int tok;
-
 	switch (do_compression) {
 	case CPRES_NONE:
-		tok = simple_recv_token(f,data);
-		break;
+		return simple_recv_token(f,data);
 	case CPRES_ZLIB:
 	case CPRES_ZLIBX:
-		tok = recv_deflated_token(f, data);
-		break;
+		return recv_deflated_token(f, data);
 #ifdef SUPPORT_ZSTD
 	case CPRES_ZSTD:
-		tok = recv_zstd_token(f, data);
-		break;
+		return recv_zstd_token(f, data);
 #endif
 #ifdef SUPPORT_LZ4
 	case CPRES_LZ4:
-		tok = recv_compressed_token(f, data);
-		break;
+		return recv_compressed_token(f, data);
 #endif
 	default:
 		assert(0);
 	}
-	return tok;
 }
 
 /*

uidlist.c

@@ -85,8 +85,6 @@ static struct idlist *add_to_list(struct idlist **root, id_t id, union name_or_i
 				  id_t id2, uint16 flags)
 {
 	struct idlist *node = new(struct idlist);
-	if (!node)
-		out_of_memory("add_to_list");
 	node->next = *root;
 	node->u = noiu;
 	node->id = id;
@@ -160,8 +158,6 @@ static int is_in_group(gid_t gid)
 		if ((ngroups = getgroups(0, NULL)) < 0)
 			ngroups = 0;
 		gidset = new_array(GETGROUPS_T, ngroups+1);
-		if (!gidset)
-			out_of_memory("is_in_group");
 		if (ngroups > 0)
 			ngroups = getgroups(ngroups, gidset);
 		/* The default gid might not be in the list on some systems. */
@@ -174,8 +170,6 @@ static int is_in_group(gid_t gid)
 		if (DEBUG_GTE(OWN, 2)) {
 			int pos;
 			char *gidbuf = new_array(char, ngroups*21+32);
-			if (!gidbuf)
-				out_of_memory("is_in_group");
 			pos = snprintf(gidbuf, 32, "process has %d gid%s: ", ngroups, ngroups == 1? "" : "s");
 			for (n = 0; n < ngroups; n++) {
 				pos += snprintf(gidbuf+pos, 21, " %d", (int)gidset[n]);
@@ -375,8 +369,6 @@ uid_t recv_user_name(int f, uid_t uid)
 	struct idlist *node;
 	int len = read_byte(f);
 	char *name = new_array(char, len+1);
-	if (!name)
-		out_of_memory("recv_user_name");
 	read_sbuf(f, name, len);
 	if (numeric_ids < 0) {
 		free(name);
@@ -391,8 +383,6 @@ gid_t recv_group_name(int f, gid_t gid, uint16 *flags_ptr)
 	struct idlist *node;
 	int len = read_byte(f);
 	char *name = new_array(char, len+1);
-	if (!name)
-		out_of_memory("recv_group_name");
 	read_sbuf(f, name, len);
 	if (numeric_ids < 0) {
 		free(name);
@@ -527,14 +517,14 @@ const char *getallgroups(uid_t uid, item_list *gid_list)
 		return "getpwuid failed";
 
 	gid_list->count = 0; /* We're overwriting any items in the list */
-	EXPAND_ITEM_LIST(gid_list, gid_t, 32);
+	(void)EXPAND_ITEM_LIST(gid_list, gid_t, 32);
 	size = gid_list->malloced;
 
 	/* Get all the process's groups, with the pw_gid group first. */
 	if (getgrouplist(pw->pw_name, pw->pw_gid, gid_list->items, &size) < 0) {
 		if (size > (int)gid_list->malloced) {
 			gid_list->count = gid_list->malloced;
-			EXPAND_ITEM_LIST(gid_list, gid_t, size);
+			(void)EXPAND_ITEM_LIST(gid_list, gid_t, size);
 			if (getgrouplist(pw->pw_name, pw->pw_gid, gid_list->items, &size) < 0)
 				size = -1;
 		} else
@@ -553,7 +543,7 @@ const char *getallgroups(uid_t uid, item_list *gid_list)
 				break;
 		}
 		if (j == size) { /* The default group wasn't found! */
-			EXPAND_ITEM_LIST(gid_list, gid_t, size+1);
+			(void)EXPAND_ITEM_LIST(gid_list, gid_t, size+1);
 			gid_array = gid_list->items;
 		}
 		gid_array[j] = gid_array[0];

util.c

@@ -592,8 +592,7 @@ int lock_range(int fd, int offset, int len)
 }
 
 #define ENSURE_MEMSPACE(buf, type, sz, req) \
-	if ((req) > sz && !(buf = realloc_array(buf, type, sz = MAX(sz * 2, req)))) \
-		out_of_memory("glob_expand")
+	do { if ((req) > sz) buf = realloc_array(buf, type, sz = MAX(sz * 2, req)); } while(0)
 
 static inline void call_glob_match(const char *name, int len, int from_glob,
 				   char *arg, int abpos, int fbpos);
@@ -695,8 +694,7 @@ static inline void call_glob_match(const char *name, int len, int from_glob,
 		glob_match(arg, abpos, fbpos);
 	} else {
 		ENSURE_MEMSPACE(glob.argv, char *, glob.maxargs, glob.argc + 1);
-		if (!(glob.argv[glob.argc++] = strdup(glob.arg_buf)))
-			out_of_memory("glob_match");
+		glob.argv[glob.argc++] = strdup(glob.arg_buf);
 	}
 }
 
@@ -720,8 +718,6 @@ int glob_expand(const char *arg, char ***argv_p, int *argc_p, int *maxargs_p)
 		s = sanitize_path(NULL, arg, "", 0, SP_KEEP_DOT_DIRS);
 	else {
 		s = strdup(arg);
-		if (!s)
-			out_of_memory("glob_expand");
 		clean_fname(s, CFN_KEEP_DOT_DIRS | CFN_KEEP_TRAILING_SLASH | CFN_COLLAPSE_DOT_DOT_DIRS);
 	}
 
@@ -771,8 +767,7 @@ void glob_expand_module(char *base1, char *arg, char ***argv_p, int *argc_p, int
 		return;
 	}
 
-	if (!(arg = strdup(arg)))
-		out_of_memory("glob_expand_module");
+	arg = strdup(arg);
 
 	if (asprintf(&base," %s/", base1) < 0)
 		out_of_memory("glob_expand_module");
@@ -1017,11 +1012,10 @@ char *sanitize_path(char *dest, const char *p, const char *rootdir, int depth, i
 			depth = 0;
 			p++;
 		}
-		if (dest) {
-			if (rlen + plen + 1 >= MAXPATHLEN)
-				return NULL;
-		} else if (!(dest = new_array(char, MAX(rlen + plen + 1, 2))))
-			out_of_memory("sanitize_path");
+		if (!dest)
+			dest = new_array(char, MAX(rlen + plen + 1, 2));
+		else if (rlen + plen + 1 >= MAXPATHLEN)
+			return NULL;
 		if (rlen) { /* only true if p previously started with a slash */
 			memcpy(dest, rootdir, rlen);
 			if (rlen > 1) /* a rootdir of len 1 is "/", so this avoids a 2nd slash */
@@ -1155,13 +1149,10 @@ char *normalize_path(char *path, BOOL force_newbuf, unsigned int *len_ptr)
 			return NULL;
 		curr_dir[curr_dir_len] = '/';
 		memcpy(curr_dir + curr_dir_len + 1, path, len + 1);
-		if (!(path = strdup(curr_dir)))
-			out_of_memory("normalize_path");
+		path = strdup(curr_dir);
 		curr_dir[curr_dir_len] = '\0';
-	} else if (force_newbuf) {
-		if (!(path = strdup(path)))
-			out_of_memory("normalize_path");
-	}
+	} else if (force_newbuf)
+		path = strdup(path);
 
 	len = clean_fname(path, CFN_COLLAPSE_DOT_DOT_DIRS | CFN_DROP_TRAILING_DOT_DIR);
 
@@ -1519,8 +1510,7 @@ struct bitbag *bitbag_create(int max_ndx)
 	struct bitbag *bb = new(struct bitbag);
 	bb->slot_cnt = (max_ndx + BB_PER_SLOT_BITS - 1) / BB_PER_SLOT_BITS;
 
-	if (!(bb->bits = (uint32**)calloc(bb->slot_cnt, sizeof (uint32*))))
-		out_of_memory("bitbag_create");
+	bb->bits = (uint32**)calloc(bb->slot_cnt, sizeof (uint32*));
 
 	return bb;
 }
@@ -1601,8 +1591,7 @@ void flist_ndx_push(flist_ndx_list *lp, int ndx)
 {
 	struct flist_ndx_item *item;
 
-	if (!(item = new(struct flist_ndx_item)))
-		out_of_memory("flist_ndx_push");
+	item = new(struct flist_ndx_item);
 	item->next = NULL;
 	item->ndx = ndx;
 	if (lp->tail)
@@ -1654,8 +1643,7 @@ void *expand_item_list(item_list *lp, size_t item_size, const char *desc, int in
 			new_size = 1;
 		if (new_size <= lp->malloced)
 			overflow_exit("expand_item_list");
-		/* Using _realloc_array() lets us pass the size, not a type. */
-		new_ptr = _realloc_array(lp->items, item_size, new_size);
+		new_ptr = realloc_buf(lp->items, new_size * item_size);
 		if (DEBUG_GTE(FLIST, 3)) {
 			rprintf(FINFO, "[%s] expand %s to %s bytes, did%s move\n",
 				who_am_i(), desc, big_num(new_size * item_size),