Preparing for release of 3.2.2pre1

Remove some extraneous vars, update some years, add an  rrsync opt, &
ensure some less options are set right when running release-rsync.

.github/workflows/ccpp.yml

@@ -1,4 +1,4 @@
-name: C CI
+name: build
 
 on:
   push:

.gitignore

@@ -49,4 +49,5 @@ aclocal.m4
 /testsuite/xattrs-hlink.test
 /patches
 /SaVeDiR
+/.gen-stash
 .deps

Makefile.in

@@ -77,7 +77,7 @@ all: Makefile rsync$(EXEEXT) stunnel-rsyncd.conf @MAKE_MAN@
 install: all
 	-${MKDIR_P} ${DESTDIR}${bindir}
 	${INSTALLCMD} ${INSTALL_STRIP} -m 755 rsync$(EXEEXT) ${DESTDIR}${bindir}
-	${INSTALLCMD} -m 755 rsync-ssl ${DESTDIR}${bindir}
+	${INSTALLCMD} -m 755 $(srcdir)/rsync-ssl ${DESTDIR}${bindir}
 	-${MKDIR_P} ${DESTDIR}${mandir}/man1
 	-${MKDIR_P} ${DESTDIR}${mandir}/man5
 	if test -f rsync.1; then ${INSTALLMAN} -m 644 rsync.1 ${DESTDIR}${mandir}/man1; fi

NEWS.md

@@ -1,7 +1,58 @@
-------------------------------------------------------------------------------
-<a name="3.2.1pre1"></a>
+<a name="3.2.2"></a>
 
-# NEWS for rsync 3.2.1pre1 (UNRELEASED)
+# NEWS for rsync 3.2.2 (UNRELEASED)
+
+Protocol: 31 (unchanged)
+
+## Changes since 3.2.1:
+
+### BUG FIXES:
+
+ - Fixed installing rsync-ssl script from an alternate build dir.
+
+ - Apple requires the asm function name to begin with an underscore.
+
+ - Avoid a test failure in the daemon test when --atimes is disabled.
+
+### ENHANCEMENTS:
+
+ - Allow the server side to restrict checksum & compression choices via the
+   same environment variables the client uses.  The env vars can be divided
+   into "client list & server list" by the "`&`" char or the same list can
+   apply to both.
+
+ - Simplify how the negotiation environment variables apply when interacting
+   with an older rsync version.
+
+ - Do not allow a negotiated checksum or compression choice of "none" unless
+   the user authorized it via an environment variable or command-line option.
+
+ - Added the `--max-alloc=SIZE` option to be able to override the memory
+   allocator's sanity-check limit.  It defaults to 1G (as before) but the error
+   message when exceeding it specifically mentions the new option so that you
+   can differentiate an out-of-memory error from a failure of this limit.  It
+   also allows you to specify the value via the RSYNC_MAX_ALLOC environment
+   variable.
+
+ - The memory allocation functions now automatically check for a failure and
+   die when out of memory.  This eliminated some caller-side check-and-die
+   code and added some missing sanity-checking of allocations.
+
+ - Improved the man page a bit more.
+
+### PACKAGING RELATED:
+
+ - Prepared the checksum code for an upcoming xxHash release that provides new
+   XXH3 & XXH128 checksum routines.  These will not be compiled into rsync
+   until the xxhash v0.8.0 include files are installed on the build host, and
+   that release is a few weeks away at the time this was written.  So, if it's
+   now the future and you have packaged and installed xxhash-0.8.0-devel, a
+   fresh rebuild of rsync 3.2.2 will give you the new checksum routines.
+
+------------------------------------------------------------------------------
+<a name="3.2.1"></a>
+
+# NEWS for rsync 3.2.1 (22 Jun 2020)
 
 Protocol: 31 (unchanged)
 
@@ -9,43 +60,53 @@ Protocol: 31 (unchanged)
 
 ### BUG FIXES:
 
- - Fixed a build issue with the MD5 assembly-language code by removing some
-   non-portable directives.
+ - Fixed a potential build issue with the MD5 assembly-language code by
+   removing some non-portable directives.
 
  - Use the preprocessor with the asm file to ensure that if the code is
    unneeded, it doesn't get built.
 
  - Avoid the stack getting set to executable when including the asm code.
 
- - Avoid some build issues with the SIMD code, including avoiding a clang++
-   core dump when `-g` is combined with `-O2`.
+ - Some improvements in the SIMD configure testing to try to avoid build
+   issues, such as avoiding a clang++ core dump when `-g` is combined with
+   `-O2`.  Note that clang++ is quite buggy in this area, and it does still
+   crash for some folks, so just use `--disable-simd` if you need to avoid
+   their buggy compiler (since the configure test is apparently not finding
+   all the compilers that will to crash and burn).
 
- - Fix an issue with the md2man code when building in an external dir.
+ - Fixed an issue in the md2man script when building from an alternate dir.
 
- - Disable --atimes on macOS (it apparently doesn't work).
+ - Disable `--atimes` on macOS (it apparently just ignores the atime change).
 
 ### ENHANCEMENTS:
 
- - Added `--early-input=FILE` option that allows the client to send some
-   data to the "early exec" daemon script on its stdin.
+ - The use of `--backup-dir=STR` now implies `--backup`.
 
- - Added "atimes" to the capabilities list that `--version` outputs.
+ - Added `--zl=NUM` as a short-hand for `--compress-level=NUM`.
+
+ - Added `--early-input=FILE` option that allows the client to send some
+   data to a daemon's (optional) "early exec" script on its stdin.
+
+ - Mention atimes in the capabilities list that `--version` outputs.
 
  - Mention either "default protect-args" or "optional protect-args" in the
    `--version` capabilities depending on how rsync was configured.
 
- - Some info on optimizations was elided from the `--version` capabilities
-   since they aren't really user-facing capabilities.  You can get the info
-   back (plus the status of a couple extra optimizations) by repeating the
+ - Some info on optimizations is now elided from the `--version` capabilities
+   since they aren't really user-facing capabilities.  You can still see the
+   info (plus the status of a couple extra optimizations) by repeating the
    `--version` option (e.g. `-VV`).
 
- - Updated various documented links to be https instead of http.
+ - Updated various URLs to be https instead of http.
+
+ - Some documentation improvements.
 
 ### PACKAGING RELATED:
 
- - If you had to use --disable-simd for 3.2.0, you should be able to remove
-   that and let it either succeed (e.g. if clang++ was core dumping) or
-   auto-disable.
+ - If you had to use `--disable-simd` for 3.2.0, you might want to try removing
+   that and see if it will succeed or auto-disable.  Some buggy clang++
+   compilers are still not auto disabled, though.
 
  - The MD5 asm code is now under its own configure flag (not shared with the
    SIMD setting), so if you have any issues compiling it, re-run configure with
@@ -76,7 +137,8 @@ Protocol: 31 (unchanged)
 
  - Make the atomic-rsync helper script have a more consistent error-exit.
 
- - Make sure that a signal handler calls `_exit()` instead of exit().
+ - Make sure that a signal handler's use of exit_cleanup() calls `_exit()`
+   instead of exit().
 
  - Various zlib fixes, including security fixes for CVE-2016-9843,
    CVE-2016-9842, CVE-2016-9841, and CVE-2016-9840.
@@ -98,7 +160,7 @@ Protocol: 31 (unchanged)
 
  - Fixed a couple bugs in the handling of files with the `--sparse` option.
 
- - Fixed a bug in the writing of the batch.sh file (w/--write-batch) when the
+ - Fixed a bug in the writing of the batch.sh file (w/`--write-batch`) when the
    source & destination args were not last on the command-line.
 
  - Avoid a hang when an overabundance of messages clogs up all the I/O buffers.
@@ -115,7 +177,7 @@ Protocol: 31 (unchanged)
  - Various checksum enhancements, including the optional use of openssl's MD4 &
    MD5 checksum algorithms, some x86-64 optimizations for the rolling checksum,
    some x86-64 optimizations for the (non-openssl) MD5 checksum, the addition
-   of xxhash checksum support, and a negotiation heuristic that ensures that it
+   of xxHash checksum support, and a negotiation heuristic that ensures that it
    is easier to add new checksum algorithms in the future.  The environment
    variable `RSYNC_CHECKSUM_LIST` can be used to customize the preference order
    of the negotiation, or use `--checksum-choice` (`--cc`) to force a choice.
@@ -126,19 +188,19 @@ Protocol: 31 (unchanged)
    `RSYNC_COMPRESS_LIST` can be used to customize the preference order of the
    negotiation, or use `--compress-choice` (`--zc`) to force a choice.
 
- - Added a --debug=NSTR option that outputs details of the new negotiation
+ - Added a `--debug=NSTR` option that outputs details of the new negotiation
    strings (for checksums and compression).  The first level just outputs the
    result of each negotiation on the client, level 2 outputs the values of the
    strings that were sent to and received from the server, and level 3 outputs
    all those values on the server side too (when the server was given the debug
    option).
 
- - The --debug=OPTS command-line option is no longer auto-forwarded to the
+ - The `--debug=OPTS` command-line option is no longer auto-forwarded to the
    remote rsync which allows for the client and server to have different levels
    of debug specified. This also allows for newer debug options to be
-   specified, such as using --debug=NSTR to see the negotiated hash result,
+   specified, such as using `--debug=NSTR` to see the negotiated hash result,
    without having the command fail if the server version is too old to handle
-   that debug item. Use -M--debug=OPTS to send the options to the remote side.
+   that debug item. Use `-M--debug=OPTS` to send the options to the remote side.
 
  - Added the `--atimes` option based on the long-standing patch (just with some
    fixes that the patch has been needing).
@@ -150,9 +212,9 @@ Protocol: 31 (unchanged)
  - Added openssl & preliminary gnutls support to the rsync-ssl script, which is
    now installed by default.  This was unified with the old stunnel-rsync
    helper script to simplify packaging.  Note that the script accepts the use
-   of --type=gnutls for gnutls testing, but does not look for gnutls-cli on the
-   path yet.  The use of type=gnutls will not work right until gnutls-cli no
-   longer drops data.
+   of `--type=gnutls` for gnutls testing, but does not look for gnutls-cli on
+   the path yet.  The use of `--type=gnutls` will not work right until
+   gnutls-cli no longer drops data.
 
  - Rsync was enhanced to set the `RSYNC_PORT` environment variable when running
    a daemon-over-rsh script. Its value is the user-specified port number (set
@@ -195,10 +257,10 @@ Protocol: 31 (unchanged)
  - Various man page improvements, including some html representations (that
    aren't installed by default).
 
- - Made -V the short option for --version and improved its information.
+ - Made `-V` the short option for `--version` and improved its information.
 
- - Pass the -4 or -6 option to the ssh command, making it easier to type than
-   `--rsh='ssh -4'` (or -6).
+ - Pass the `-4` or `-6` option to the ssh command, making it easier to type
+   than `--rsh='ssh -4'` (or the `-6` equivalent).
 
  - Added example config for rsyncd SSL proxy configs to rsyncd.conf.
 
@@ -235,8 +297,8 @@ Protocol: 31 (unchanged)
    required for a release-tar build, since it comes with pre-built man pages.
    Note that cmarkcfm is faster than commonmark, but they generate the same
    data.  The commonmark dependency is easiest to install since it's native
-   python, and can be installed via `pip3 install --user commonmark` if you
-   want to just install it for the build user (or omit `--user`).
+   python, and can even be installed via `pip3 install --user commonmark` if
+   you want to just install it for the build user.
 
  - Remove yodl _build_ dependency (if it was even listed before).
 
@@ -3267,8 +3329,8 @@ Protocol: 28 (unchanged)
  - If `make proto` doesn't find some changes in the prototypes, the proto.h
    file is left untouched (its time-stamp used to always be updated).
 
- - The variable $STRIP (that is optionally set by the install-strip target's
-   rule) was changed to `$INSTALL_STRIP` because some systems have $STRIP
+ - The variable `$STRIP` (that is optionally set by the install-strip target's
+   rule) was changed to `$INSTALL_STRIP` because some systems have `$STRIP`
    already set in the environment.
 
  - Fixed a build problem when `SUPPORT_HARD_LINKS` isn't defined.
@@ -4089,7 +4151,8 @@ Protocol: 25 (changed)
 
 | RELEASE DATE | VER.   | DATE OF COMMIT\* | PROTOCOL    |
 |--------------|--------|------------------|-------------|
-| ?? Jun 2020  | 3.2.1  |                  | 31          |
+| ?? Jun 2020  | 3.2.2  |                  | 31          |
+| 22 Jun 2020  | 3.2.1  |                  | 31          |
 | 19 Jun 2020  | 3.2.0  |                  | 31          |
 | 28 Jan 2018  | 3.1.3  |                  | 31          |
 | 21 Dec 2015  | 3.1.2  |                  | 31          |

TODO

@@ -49,7 +49,7 @@ Create test makefile target for some tests
 
 RELATED PROJECTS -----------------------------------------------------
 rsyncsh
-http://rsync.samba.org/rsync-and-debian/
+https://rsync.samba.org/rsync-and-debian/
 rsyncable gzip patch
 rsyncsplit as alternative to real integration with gzip?
 reverse rsync over HTTP Range
@@ -66,8 +66,8 @@ Use chroot only if supported
   If running as non-root, then don't fail, just give a warning.
   (There was a thread about this a while ago?)
 
-    http://lists.samba.org/pipermail/rsync/2001-August/thread.html
-    http://lists.samba.org/pipermail/rsync/2001-September/thread.html
+    https://lists.samba.org/pipermail/rsync/2001-August/thread.html
+    https://lists.samba.org/pipermail/rsync/2001-September/thread.html
 
                       --          --
 
@@ -204,7 +204,7 @@ Create more granular verbosity					2003/05/15
   fine grained selection of statistical reporting and what
   actions are logged.
 
-  http://lists.samba.org/archive/rsync/2003-May/006059.html
+  https://lists.samba.org/archive/rsync/2003-May/006059.html
 
                       --          --
 
@@ -287,7 +287,7 @@ Perhaps flush stdout like syslog
 
   Perhaps flush stdout after each filename, so that people trying to
   monitor progress in a log file can do so more easily.  See
-  http://bugs.debian.org/cgi-bin/bugreport.cgi?bug=48108
+  https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=48108
 
                       --          --
 
@@ -495,7 +495,7 @@ rsyncsh
                       --          --
 
 
-http://rsync.samba.org/rsync-and-debian/
+https://rsync.samba.org/rsync-and-debian/
 
 
                       --          --

access.c

@@ -19,6 +19,7 @@
  */
 
 #include "rsync.h"
+#include "ifuncs.h"
 
 static int allow_forward_dns;
 
@@ -52,10 +53,8 @@ static int match_hostname(const char **host_ptr, const char *addr, const char *t
 		if (strcmp(addr, inet_ntoa(*(struct in_addr*)(hp->h_addr_list[i]))) == 0) {
 			/* If reverse lookups are off, we'll use the conf-specified
 			 * hostname in preference to UNDETERMINED. */
-			if (host == undetermined_hostname) {
-				if (!(*host_ptr = strdup(tok)))
-					*host_ptr = undetermined_hostname;
-			}
+			if (host == undetermined_hostname)
+				*host_ptr = strdup(tok);
 			return 1;
 		}
 	}
@@ -241,9 +240,6 @@ static int access_match(const char *list, const char *addr, const char **host_pt
 	char *tok;
 	char *list2 = strdup(list);
 
-	if (!list2)
-		out_of_memory("access_match");
-
 	strlower(list2);
 
 	for (tok = strtok(list2, " ,\t"); tok; tok = strtok(NULL, " ,\t")) {

acls.c

@@ -168,8 +168,6 @@ static rsync_acl *create_racl(void)
 {
 	rsync_acl *racl = new(rsync_acl);
 
-	if (!racl)
-		out_of_memory("create_racl");
 	*racl = empty_rsync_acl;
 
 	return racl;
@@ -335,8 +333,7 @@ static BOOL unpack_smb_acl(SMB_ACL_T sacl, rsync_acl *racl)
 			qsort(temp_ida_list.items, temp_ida_list.count, sizeof (id_access), id_access_sorter);
 		}
 #endif
-		if (!(racl->names.idas = new_array(id_access, temp_ida_list.count)))
-			out_of_memory("unpack_smb_acl");
+		racl->names.idas = new_array(id_access, temp_ida_list.count);
 		memcpy(racl->names.idas, temp_ida_list.items, temp_ida_list.count * sizeof (id_access));
 	} else
 		racl->names.idas = NULL;
@@ -505,9 +502,7 @@ static int get_rsync_acl(const char *fname, rsync_acl *racl,
 
 		if (cnt) {
 			char *bp = buf + 4*4;
-			id_access *ida;
-			if (!(ida = racl->names.idas = new_array(id_access, cnt)))
-				out_of_memory("get_rsync_acl");
+			id_access *ida = racl->names.idas = new_array(id_access, cnt);
 			racl->names.count = cnt;
 			for ( ; cnt--; ida++, bp += 4+4) {
 				ida->id = IVAL(bp, 0);
@@ -703,12 +698,7 @@ static uchar recv_ida_entries(int f, ida_entries *ent)
 	uchar computed_mask_bits = 0;
 	int i, count = read_varint(f);
 
-	if (count) {
-		if (!(ent->idas = new_array(id_access, count)))
-			out_of_memory("recv_ida_entries");
-	} else
-		ent->idas = NULL;
-
+	ent->idas = count ? new_array(id_access, count) : NULL;
 	ent->count = count;
 
 	for (i = 0; i < count; i++) {

authenticate.c

@@ -20,6 +20,7 @@
 
 #include "rsync.h"
 #include "itypes.h"
+#include "ifuncs.h"
 
 extern int read_only;
 extern char *password_file;
@@ -250,8 +251,7 @@ char *auth_server(int f_in, int f_out, int module, const char *host,
 	}
 	*pass++ = '\0';
 
-	if (!(users = strdup(users)))
-		out_of_memory("auth_server");
+	users = strdup(users);
 
 	for (tok = strtok(users, " ,\t"); tok; tok = strtok(NULL, " ,\t")) {
 		char *opts;
@@ -287,8 +287,7 @@ char *auth_server(int f_in, int f_out, int module, const char *host,
 				else {
 					gid_t *gid_array = gid_list.items;
 					auth_uid_groups_cnt = gid_list.count;
-					if ((auth_uid_groups = new_array(char *, auth_uid_groups_cnt)) == NULL)
-						out_of_memory("auth_server");
+					auth_uid_groups = new_array(char *, auth_uid_groups_cnt);
 					for (j = 0; j < auth_uid_groups_cnt; j++)
 						auth_uid_groups[j] = gid_to_group(gid_array[j]);
 				}

batch.c

@@ -38,13 +38,10 @@ extern int do_compression;
 extern int inplace;
 extern int append_mode;
 extern int write_batch;
-extern int xfersum_type;
 extern int protocol_version;
 extern int raw_argc, cooked_argc;
 extern char **raw_argv, **cooked_argv;
 extern char *batch_name;
-extern const char *checksum_choice;
-extern const char *compress_choice;
 #ifdef ICONV_OPTION
 extern char *iconv_opt;
 #endif
@@ -269,14 +266,6 @@ void write_batch_shell_file(void)
 			err |= write_opt("--exclude-from", "-");
 	}
 
-	/* We need to make sure that any protocol-based or negotiated choices get accurately
-	 * reflected in the options we save AND that we avoid any need for --read-batch to
-	 * do a string-based negotiation (since we don't write them into the file). */
-	if (do_compression)
-		err |= write_opt("--compress-choice", compress_choice);
-	if (strchr(checksum_choice, ',') || xfersum_type != parse_csum_name(NULL, -1))
-		err |= write_opt("--checksum-choice", checksum_choice);
-
 	/* Elide the filename args from the option list, but scan for them in reverse. */
 	for (i = raw_argc-1, j = cooked_argc-1; i > 0 && j >= 0; i--) {
 		if (strcmp(raw_argv[i], cooked_argv[j]) == 0) {

checksum.c

@@ -27,8 +27,12 @@
  */
 
 #include "rsync.h"
+
 #ifdef SUPPORT_XXHASH
 #include "xxhash.h"
+# if XXH_VERSION_NUMBER >= 800
+#  define SUPPORT_XXH3 1
+# endif
 #endif
 
 extern int am_server;
@@ -40,6 +44,10 @@ extern const char *checksum_choice;
 
 struct name_num_obj valid_checksums = {
 	"checksum", NULL, NULL, 0, 0, {
+#ifdef SUPPORT_XXH3
+		{ CSUM_XXH3_128, "xxh128", NULL },
+		{ CSUM_XXH3_64, "xxh3", NULL },
+#endif
 #ifdef SUPPORT_XXHASH
 		{ CSUM_XXH64, "xxh64", NULL },
 		{ CSUM_XXH64, "xxhash", NULL },
@@ -85,7 +93,7 @@ static const char *checksum_name(int num)
 {
 	struct name_num_item *nni = get_nni_by_num(&valid_checksums, num);
 
-	return nni ? nni->name : num < CSUM_MD4 ? "MD4" : "UNKNOWN";
+	return nni ? nni->name : num < CSUM_MD4 ? "md4" : "UNKNOWN";
 }
 
 void parse_checksum_choice(int final_call)
@@ -99,6 +107,8 @@ void parse_checksum_choice(int final_call)
 			checksum_type = parse_csum_name(cp+1, -1);
 		} else
 			xfersum_type = checksum_type = parse_csum_name(checksum_choice, -1);
+		if (am_server && checksum_choice)
+			validate_choice_vs_env(NSTR_CHECKSUM, xfersum_type, checksum_type);
 	}
 
 	if (xfersum_type == CSUM_NONE)
@@ -133,10 +143,11 @@ int csum_len_for_type(int cst, BOOL flist_csum)
 		return MD4_DIGEST_LEN;
 	  case CSUM_MD5:
 		return MD5_DIGEST_LEN;
-#ifdef SUPPORT_XXHASH
 	  case CSUM_XXH64:
+	  case CSUM_XXH3_64:
 		return 64/8;
-#endif
+	  case CSUM_XXH3_128:
+		return 128/8;
 	  default: /* paranoia to prevent missing case values */
 		exit_cleanup(RERR_UNSUPPORTED);
 	}
@@ -158,10 +169,10 @@ int canonical_checksum(int csum_type)
 	  case CSUM_MD4:
 	  case CSUM_MD5:
 		return -1;
-#ifdef SUPPORT_XXHASH
 	  case CSUM_XXH64:
+	  case CSUM_XXH3_64:
+	  case CSUM_XXH3_128:
 		return 1;
-#endif
 	  default: /* paranoia to prevent missing case values */
 		exit_cleanup(RERR_UNSUPPORTED);
 	}
@@ -198,6 +209,17 @@ void get_checksum2(char *buf, int32 len, char *sum)
 	  case CSUM_XXH64:
 		SIVAL64(sum, 0, XXH64(buf, len, checksum_seed));
 		break;
+#endif
+#ifdef SUPPORT_XXH3
+	  case CSUM_XXH3_64:
+		SIVAL64(sum, 0, XXH3_64bits_withSeed(buf, len, checksum_seed));
+		break;
+	  case CSUM_XXH3_128: {
+		XXH128_hash_t digest = XXH3_128bits_withSeed(buf, len, checksum_seed);
+		SIVAL64(sum, 0, digest.low64);
+		SIVAL64(sum, 8, digest.high64);
+		break;
+	  }
 #endif
 	  case CSUM_MD5: {
 		MD5_CTX m5;
@@ -249,8 +271,6 @@ void get_checksum2(char *buf, int32 len, char *sum)
 				free(buf1);
 			buf1 = new_array(char, len+4);
 			len1 = len;
-			if (!buf1)
-				out_of_memory("get_checksum2");
 		}
 
 		memcpy(buf1, buf, len);
@@ -313,6 +333,45 @@ void file_checksum(const char *fname, const STRUCT_STAT *st_p, char *sum)
 		SIVAL64(sum, 0, XXH64_digest(state));
 		break;
 	  }
+#endif
+#ifdef SUPPORT_XXH3
+	  case CSUM_XXH3_64: {
+		static XXH3_state_t* state = NULL;
+		if (!state && !(state = XXH3_createState()))
+			out_of_memory("file_checksum");
+
+		XXH3_64bits_reset(state);
+
+		for (i = 0; i + CHUNK_SIZE <= len; i += CHUNK_SIZE)
+			XXH3_64bits_update(state, (uchar *)map_ptr(buf, i, CHUNK_SIZE), CHUNK_SIZE);
+
+		remainder = (int32)(len - i);
+		if (remainder > 0)
+			XXH3_64bits_update(state, (uchar *)map_ptr(buf, i, remainder), remainder);
+
+		SIVAL64(sum, 0, XXH3_64bits_digest(state));
+		break;
+	  }
+	  case CSUM_XXH3_128: {
+		XXH128_hash_t digest;
+		static XXH3_state_t* state = NULL;
+		if (!state && !(state = XXH3_createState()))
+			out_of_memory("file_checksum");
+
+		XXH3_128bits_reset(state);
+
+		for (i = 0; i + CHUNK_SIZE <= len; i += CHUNK_SIZE)
+			XXH3_128bits_update(state, (uchar *)map_ptr(buf, i, CHUNK_SIZE), CHUNK_SIZE);
+
+		remainder = (int32)(len - i);
+		if (remainder > 0)
+			XXH3_128bits_update(state, (uchar *)map_ptr(buf, i, remainder), remainder);
+
+		digest = XXH3_128bits_digest(state);
+		SIVAL64(sum, 0, digest.low64);
+		SIVAL64(sum, 8, digest.high64);
+		break;
+	  }
 #endif
 	  case CSUM_MD5: {
 		MD5_CTX m5;
@@ -389,6 +448,9 @@ static union {
 #ifdef SUPPORT_XXHASH
 static XXH64_state_t* xxh64_state;
 #endif
+#ifdef SUPPORT_XXH3
+static XXH3_state_t* xxh3_state;
+#endif
 static int cursum_type;
 
 void sum_init(int csum_type, int seed)
@@ -406,6 +468,18 @@ void sum_init(int csum_type, int seed)
 			out_of_memory("sum_init");
 		XXH64_reset(xxh64_state, 0);
 		break;
+#endif
+#ifdef SUPPORT_XXH3
+	  case CSUM_XXH3_64:
+		if (!xxh3_state && !(xxh3_state = XXH3_createState()))
+			out_of_memory("sum_init");
+		XXH3_64bits_reset(xxh3_state);
+		break;
+	  case CSUM_XXH3_128:
+		if (!xxh3_state && !(xxh3_state = XXH3_createState()))
+			out_of_memory("sum_init");
+		XXH3_128bits_reset(xxh3_state);
+		break;
 #endif
 	  case CSUM_MD5:
 		MD5_Init(&ctx.m5);
@@ -448,6 +522,14 @@ void sum_update(const char *p, int32 len)
 	  case CSUM_XXH64:
 		XXH64_update(xxh64_state, p, len);
 		break;
+#endif
+#ifdef SUPPORT_XXH3
+	  case CSUM_XXH3_64:
+		XXH3_64bits_update(xxh3_state, p, len);
+		break;
+	  case CSUM_XXH3_128:
+		XXH3_128bits_update(xxh3_state, p, len);
+		break;
 #endif
 	  case CSUM_MD5:
 		MD5_Update(&ctx.m5, (uchar *)p, len);
@@ -502,6 +584,17 @@ int sum_end(char *sum)
 	  case CSUM_XXH64:
 		SIVAL64(sum, 0, XXH64_digest(xxh64_state));
 		break;
+#endif
+#ifdef SUPPORT_XXH3
+	  case CSUM_XXH3_64:
+		SIVAL64(sum, 0, XXH3_64bits_digest(xxh3_state));
+		break;
+	  case CSUM_XXH3_128: {
+		XXH128_hash_t digest = XXH3_128bits_digest(xxh3_state);
+		SIVAL64(sum, 0, digest.low64);
+		SIVAL64(sum, 8, digest.high64);
+		break;
+	  }
 #endif
 	  case CSUM_MD5:
 		MD5_Final((uchar *)sum, &ctx.m5);

cleanup.c

@@ -221,8 +221,13 @@ NORETURN void _exit_cleanup(int code, const char *file, int line)
 		/* If line < 0, this exit is after a MSG_ERROR_EXIT event, so
 		 * we don't want to output a duplicate error. */
 		if ((exit_code && line > 0)
-		 || am_daemon || (logfile_name && (am_server || !INFO_GTE(STATS, 1))))
+		 || am_daemon || (logfile_name && (am_server || !INFO_GTE(STATS, 1)))) {
+#ifdef HAVE_USLEEP /* Try for a teeny delay if both sender & receiver are sending a msg at the same time. */
+			if (am_server && exit_code)
+				usleep(50);
+#endif
 			log_exit(exit_code, exit_file, exit_line);
+		}
 
 #include "case_N.h"
 		switch_step++;

clientname.c

@@ -481,7 +481,7 @@ static int valid_ipaddr(const char *s)
 
 		for (count = 0; count < 8; count++) {
 			if (!*s)
-				return saw_double_colon && count < 7;
+				return saw_double_colon;
 
 			if (strchr(s, ':') == NULL && strchr(s, '.') != NULL) {
 				if ((!saw_double_colon && count != 6) || (saw_double_colon && count > 6))

clientserver.c

@@ -235,8 +235,7 @@ int start_inband_exchange(int f_in, int f_out, const char *user, int argc, char
 	else
 		modlen = p - *argv;
 
-	if (!(modname = new_array(char, modlen+1+1))) /* room for '/' & '\0' */
-		out_of_memory("start_inband_exchange");
+	modname = new_array(char, modlen+1+1); /* room for '/' & '\0' */
 	strlcpy(modname, *argv, modlen + 1);
 	modname[modlen] = '/';
 	modname[modlen+1] = '\0';
@@ -1233,8 +1232,7 @@ int start_daemon(int f_in, int f_out)
 			io_printf(f_out, "@ERROR: invalid early_input length\n");
 			return -1;
 		}
-		if (!(early_input = new_array(char, early_input_len)))
-			out_of_memory("exchange_protocols");
+		early_input = new_array(char, early_input_len);
 		read_buf(f_in, early_input, early_input_len);
 
 		if (!read_line_old(f_in, line, sizeof line, 0))

compat.c

@@ -20,6 +20,7 @@
  */
 
 #include "rsync.h"
+#include "itypes.h"
 
 extern int am_server;
 extern int am_sender;
@@ -173,6 +174,8 @@ void parse_compress_choice(int final_call)
 			exit_cleanup(RERR_UNSUPPORTED);
 		}
 		do_compression = nni->num;
+		if (am_server)
+			validate_choice_vs_env(NSTR_COMPRESS, do_compression, -1);
 	} else if (do_compression)
 		do_compression = CPRES_ZLIB;
 	else
@@ -241,8 +244,7 @@ static void init_nno_saw(struct name_num_obj *nno, int val)
 	}
 
 	if (!nno->saw) {
-		if (!(nno->saw = new_array0(uchar, nno->saw_len)))
-			out_of_memory("init_nno_saw");
+		nno->saw = new_array0(uchar, nno->saw_len);
 
 		/* We'll take this opportunity to make sure that the main_name values are set right. */
 		for (cnt = 1, nni = nno->list; nni->name; nni++, cnt++) {
@@ -261,10 +263,14 @@ static void init_nno_saw(struct name_num_obj *nno, int val)
 static int parse_nni_str(struct name_num_obj *nno, const char *from, char *tobuf, int tobuf_len)
 {
 	char *to = tobuf, *tok = NULL;
-	int cnt = 0;
+	int saw_tok = 0, cnt = 0;
 
 	while (1) {
-		if (*from == ' ' || !*from) {
+		int at_space = isSpace(from);
+		char ch = *from++;
+		if (ch == '&')
+			ch = '\0';
+		if (!ch || at_space) {
 			if (tok) {
 				struct name_num_item *nni = get_nni_by_name(nno, tok, to - tok);
 				if (nni && !nno->saw[nni->num]) {
@@ -278,9 +284,10 @@ static int parse_nni_str(struct name_num_obj *nno, const char *from, char *tobuf
 					}
 				} else
 					to = tok - (tok != tobuf);
+				saw_tok = 1;
 				tok = NULL;
 			}
-			if (!*from++)
+			if (!ch)
 				break;
 			continue;
 		}
@@ -293,13 +300,19 @@ static int parse_nni_str(struct name_num_obj *nno, const char *from, char *tobuf
 			to = tok - (tok != tobuf);
 			break;
 		}
-		*to++ = *from++;
+		*to++ = ch;
 	}
 	*to = '\0';
 
+	if (saw_tok && to == tobuf)
+		return strlcpy(tobuf, "INVALID", MAX_NSTR_STRLEN);
+
 	return to - tobuf;
 }
 
+/* This routine is always called with a tmpbuf of MAX_NSTR_STRLEN length, but the
+ * buffer may be pre-populated with a "len" length string to use OR a len of -1
+ * to tell us to read a string from the fd. */
 static void recv_negotiate_str(int f_in, struct name_num_obj *nno, char *tmpbuf, int len)
 {
 	struct name_num_item *ret = NULL;
@@ -315,17 +328,26 @@ static void recv_negotiate_str(int f_in, struct name_num_obj *nno, char *tmpbuf,
 	}
 
 	if (len > 0) {
+		struct name_num_item *nni;
 		int best = nno->saw_len; /* We want best == 1 from the client list, so start with a big number. */
-		char *tok;
-		if (am_server)
-			init_nno_saw(nno, 1); /* Since we're parsing client names, anything we parse first is #1. */
-		for (tok = strtok(tmpbuf, " \t"); tok; tok = strtok(NULL, " \t")) {
-			struct name_num_item *nni = get_nni_by_name(nno, tok, -1);
+		char *space, *tok = tmpbuf;
+		while (tok) {
+			while (*tok == ' ') tok++; /* Should be unneeded... */
+			if (!*tok)
+				break;
+			if ((space = strchr(tok, ' ')) != NULL)
+				*space = '\0';
+			nni = get_nni_by_name(nno, tok, -1);
+			if (space) {
+				*space = ' ';
+				tok = space + 1;
+			} else
+				tok = NULL;
 			if (!nni || !nno->saw[nni->num] || best <= nno->saw[nni->num])
 				continue;
 			ret = nni;
 			best = nno->saw[nni->num];
-			if (best == 1)
+			if (best == 1 || am_server) /* The server side stops at the first acceptable client choice */
 				break;
 		}
 		if (ret) {
@@ -337,15 +359,82 @@ static void recv_negotiate_str(int f_in, struct name_num_obj *nno, char *tmpbuf,
 		}
 	}
 
-	if (!am_server)
-		rprintf(FERROR, "Failed to negotiate a common %s\n", nno->type);
+	if (!am_server || !do_negotiated_strings) {
+		char *cp = tmpbuf;
+		int j;
+		rprintf(FERROR, "Failed to negotiate a %s choice.\n", nno->type);
+		rprintf(FERROR, "%s list: %s\n", am_server ? "Client" : "Server", tmpbuf);
+		/* Recreate our original list from the saw values. This can't overflow our huge
+		 * buffer because we don't have enough valid entries to get anywhere close. */
+		for (j = 1; j <= nno->saw_len; j++) {
+			struct name_num_item *nni;
+			for (nni = nno->list; nni->name; nni++) {
+				if (nno->saw[nni->num] == j) {
+					*cp++ = ' ';
+					cp += strlcpy(cp, nni->name, MAX_NSTR_STRLEN - (cp - tmpbuf));
+					break;
+				}
+			}
+		}
+		rprintf(FERROR, "%s list:%s\n", am_server ? "Server" : "Client", tmpbuf);
+	}
+
 	exit_cleanup(RERR_UNSUPPORTED);
 }
 
+static const char *getenv_nstr(int ntype)
+{
+	const char *env_str = getenv(ntype == NSTR_COMPRESS ? "RSYNC_COMPRESS_LIST" : "RSYNC_CHECKSUM_LIST");
+
+	/* When writing a batch file, we always negotiate an old-style choice. */
+	if (write_batch) 
+		env_str = ntype == NSTR_COMPRESS ? "zlib" : protocol_version >= 30 ? "md5" : "md4";
+
+	if (am_server && env_str) {
+		char *cp = strchr(env_str, '&');
+		if (cp)
+			env_str = cp + 1;
+	}
+
+	return env_str;
+}
+
+void validate_choice_vs_env(int ntype, int num1, int num2)
+{
+	struct name_num_obj *nno = ntype == NSTR_COMPRESS ? &valid_compressions : &valid_checksums;
+	const char *list_str = getenv_nstr(ntype);
+	char tmpbuf[MAX_NSTR_STRLEN];
+
+	if (!list_str)
+		return;
+
+	while (isSpace(list_str)) list_str++;
+
+	if (!*list_str)
+		return;
+
+	init_nno_saw(nno, 0);
+	parse_nni_str(nno, list_str, tmpbuf, MAX_NSTR_STRLEN);
+
+	if (ntype == NSTR_CHECKSUM) /* If "md4" is in the env list, all the old MD4 choices are OK too. */
+		nno->saw[CSUM_MD4_ARCHAIC] = nno->saw[CSUM_MD4_BUSTED] = nno->saw[CSUM_MD4_OLD] = nno->saw[CSUM_MD4];
+
+	if (!nno->saw[num1] || (num2 >= 0 && !nno->saw[num2])) {
+		rprintf(FERROR, "Your --%s-choice value (%s) was refused by the server.\n", 
+			ntype == NSTR_COMPRESS ? "compress" : "checksum",
+			ntype == NSTR_COMPRESS ? compress_choice : checksum_choice);
+		exit_cleanup(RERR_UNSUPPORTED);
+	}
+
+	free(nno->saw);
+	nno->saw = NULL;
+}
+
 /* The saw buffer is initialized and used to store ordinal values from 1 to N
  * for the order of the args in the array.  If dup_markup == '\0', duplicates
  * are removed otherwise the char is prefixed to the duplicate term and, if it
- * is an opening paren/bracket/brace, the matching closing char is suffixed. */
+ * is an opening paren/bracket/brace, the matching closing char is suffixed.
+ * "none" is removed on the client side unless dup_markup != '\0'. */
 int get_default_nno_list(struct name_num_obj *nno, char *to_buf, int to_buf_len, char dup_markup)
 {
 	struct name_num_item *nni;
@@ -367,6 +456,8 @@ int get_default_nno_list(struct name_num_obj *nno, char *to_buf, int to_buf_len,
 				continue;
 			delim = dup_markup;
 		}
+		if (nni->num == 0 && !am_server && !dup_markup)
+			continue;
 		if (len)
 			to_buf[len++]= ' ';
 		if (delim) {
@@ -386,25 +477,15 @@ int get_default_nno_list(struct name_num_obj *nno, char *to_buf, int to_buf_len,
 	return len;
 }
 
-static void send_negotiate_str(int f_out, struct name_num_obj *nno, const char *env_name)
+static void send_negotiate_str(int f_out, struct name_num_obj *nno, int ntype)
 {
 	char tmpbuf[MAX_NSTR_STRLEN];
-	const char *list_str = getenv(env_name);
-	int len, fail_if_empty = list_str && strstr(list_str, "FAIL");
+	const char *list_str = getenv_nstr(ntype);
+	int len;
 
-	if (!do_negotiated_strings) {
-		if (!am_server && fail_if_empty) {
-			rprintf(FERROR, "Remote rsync is too old for %s negotiation\n", nno->type);
-			exit_cleanup(RERR_UNSUPPORTED);
-		}
-		return;
-	}
-
-	if (list_str && *list_str && (!am_server || local_server)) {
+	if (list_str && *list_str) {
 		init_nno_saw(nno, 0);
 		len = parse_nni_str(nno, list_str, tmpbuf, MAX_NSTR_STRLEN);
-		if (fail_if_empty && !len)
-			len = strlcpy(tmpbuf, "FAIL", MAX_NSTR_STRLEN);
 		list_str = tmpbuf;
 	} else
 		list_str = NULL;
@@ -419,15 +500,10 @@ static void send_negotiate_str(int f_out, struct name_num_obj *nno, const char *
 			rprintf(FINFO, "Client %s list (on client): %s\n", nno->type, tmpbuf);
 	}
 
-	if (local_server) {
-		/* A local server doesn't bother to send/recv the strings, it just constructs
-		 * and parses the same string on both sides. */
-		recv_negotiate_str(-1, nno, tmpbuf, len);
-	} else {
-		/* Each side sends their list of valid names to the other side and then both sides
-		 * pick the first name in the client's list that is also in the server's list. */
+	/* Each side sends their list of valid names to the other side and then both sides
+	 * pick the first name in the client's list that is also in the server's list. */
+	if (do_negotiated_strings)
 		write_vstring(f_out, tmpbuf, len);
-	}
 }
 
 static void negotiate_the_strings(int f_in, int f_out)
@@ -435,20 +511,35 @@ static void negotiate_the_strings(int f_in, int f_out)
 	/* We send all the negotiation strings before we start to read them to help avoid a slow startup. */
 
 	if (!checksum_choice)
-		send_negotiate_str(f_out, &valid_checksums, "RSYNC_CHECKSUM_LIST");
+		send_negotiate_str(f_out, &valid_checksums, NSTR_CHECKSUM);
 
 	if (do_compression && !compress_choice)
-		send_negotiate_str(f_out, &valid_compressions, "RSYNC_COMPRESS_LIST");
+		send_negotiate_str(f_out, &valid_compressions, NSTR_COMPRESS);
 
 	if (valid_checksums.saw) {
 		char tmpbuf[MAX_NSTR_STRLEN];
-		recv_negotiate_str(f_in, &valid_checksums, tmpbuf, -1);
+		int len;
+		if (do_negotiated_strings)
+			len = -1;
+		else
+			len = strlcpy(tmpbuf, protocol_version >= 30 ? "md5" : "md4", MAX_NSTR_STRLEN);
+		recv_negotiate_str(f_in, &valid_checksums, tmpbuf, len);
 	}
 
 	if (valid_compressions.saw) {
 		char tmpbuf[MAX_NSTR_STRLEN];
-		recv_negotiate_str(f_in, &valid_compressions, tmpbuf, -1);
+		int len;
+		if (do_negotiated_strings)
+			len = -1;
+		else
+			len = strlcpy(tmpbuf, "zlib", MAX_NSTR_STRLEN);
+		recv_negotiate_str(f_in, &valid_compressions, tmpbuf, len);
 	}
+
+	/* If the other side is too old to negotiate, the above steps just made sure that
+	 * the env didn't disallow the old algorithm. Mark things as non-negotiated. */
+	if (!do_negotiated_strings)
+		valid_checksums.negotiated_name = valid_compressions.negotiated_name = NULL;
 }
 
 void setup_protocol(int f_out,int f_in)
@@ -602,10 +693,8 @@ void setup_protocol(int f_out,int f_in)
 			if (local_server || strchr(client_info, 'I') != NULL)
 				compat_flags |= CF_INPLACE_PARTIAL_DIR;
 			if (local_server || strchr(client_info, 'v') != NULL) {
-				if (!write_batch || protocol_version >= 30) {
-					do_negotiated_strings = 1;
-					compat_flags |= CF_VARINT_FLIST_FLAGS;
-				}
+				do_negotiated_strings = 1;
+				compat_flags |= CF_VARINT_FLIST_FLAGS;
 			}
 			if (strchr(client_info, 'V') != NULL) { /* Support a pre-release 'V' that got superseded */
 				if (!write_batch)
@@ -654,6 +743,9 @@ void setup_protocol(int f_out,int f_in)
 #endif
 	}
 
+	if (read_batch)
+		do_negotiated_strings = 0;
+
 	if (need_unsorted_flist && (!am_sender || inc_recurse))
 		unsort_ndx = ++file_extra_cnt;
 

configure.ac

@@ -1,6 +1,6 @@
 dnl Process this file with autoconf to produce a configure script.
 
-AC_INIT([rsync],[3.2.1pre1],[https://rsync.samba.org/bugtracking.html])
+AC_INIT([rsync],[3.2.2pre1],[https://rsync.samba.org/bugtracking.html])
 
 AC_CONFIG_MACRO_DIR([m4])
 AC_CONFIG_SRCDIR([byteorder.h])
@@ -250,7 +250,9 @@ if test x"$SIMD" != x""; then
     SIMD='$(SIMD_'"$SIMD)"
     # We only use c++ for its target attribute dispatching, disable unneeded bulky features
     CXXFLAGS="$CXXFLAGS -fno-exceptions -fno-rtti"
-    case "$CXX" in
+    # Apple often has "g++" as a symlink for clang. Try to find out the truth.
+    CXX_VERSION=`$CXX --version 2>/dev/null | head -n 2`
+    case "$CXX_VERSION" in
     *clang*) CXXFLAGS="$CXXFLAGS -fno-slp-vectorize" ;; # avoid a performance hit
     esac
 else

exclude.c

@@ -22,6 +22,7 @@
 
 #include "rsync.h"
 #include "default-cvsignore.h"
+#include "ifuncs.h"
 
 extern int am_server;
 extern int am_sender;
@@ -200,8 +201,7 @@ static void add_rule(filter_rule_list *listp, const char *pat, unsigned int pat_
 	} else
 		suf_len = 0;
 
-	if (!(rule->pattern = new_array(char, pre_len + pat_len + suf_len + 1)))
-		out_of_memory("add_rule");
+	rule->pattern = new_array(char, pre_len + pat_len + suf_len + 1);
 	if (pre_len) {
 		memcpy(rule->pattern, dirbuf + module_dirlen, pre_len);
 		for (cp = rule->pattern; cp < rule->pattern + pre_len; cp++) {
@@ -262,19 +262,14 @@ static void add_rule(filter_rule_list *listp, const char *pat, unsigned int pat_
 			}
 		}
 
-		if (!(lp = new_array0(filter_rule_list, 1)))
-			out_of_memory("add_rule");
+		lp = new_array0(filter_rule_list, 1);
 		if (asprintf(&lp->debug_type, " [per-dir %s]", cp) < 0)
 			out_of_memory("add_rule");
 		rule->u.mergelist = lp;
 
 		if (mergelist_cnt == mergelist_size) {
 			mergelist_size += 5;
-			mergelist_parents = realloc_array(mergelist_parents,
-						filter_rule *,
-						mergelist_size);
-			if (!mergelist_parents)
-				out_of_memory("add_rule");
+			mergelist_parents = realloc_array(mergelist_parents, filter_rule *, mergelist_size);
 		}
 		if (DEBUG_GTE(FILTER, 2)) {
 			rprintf(FINFO, "[%s] activating mergelist #%d%s\n",
@@ -498,8 +493,6 @@ void *push_local_filters(const char *dir, unsigned int dirlen)
 	push = (struct local_filter_state *)new_array(char,
 			  sizeof (struct local_filter_state)
 			+ (mergelist_cnt-1) * sizeof (filter_rule_list));
-	if (!push)
-		out_of_memory("push_local_filters");
 
 	push->mergelist_cnt = mergelist_cnt;
 	for (i = 0; i < mergelist_cnt; i++) {
@@ -822,8 +815,7 @@ static filter_rule *parse_rule_tok(const char **rulestr_ptr,
 	if (!*s)
 		return NULL;
 
-	if (!(rule = new0(filter_rule)))
-		out_of_memory("parse_rule_tok");
+	rule = new0(filter_rule);
 
 	/* Inherit from the template.  Don't inherit FILTRULES_SIDES; we check
 	 * that later. */
@@ -1125,8 +1117,7 @@ void parse_filter_str(filter_rule_list *listp, const char *rulestr,
 				const char *name;
 				filter_rule *excl_self;
 
-				if (!(excl_self = new0(filter_rule)))
-					out_of_memory("parse_filter_str");
+				excl_self = new0(filter_rule);
 				/* Find the beginning of the basename and add an exclude for it. */
 				for (name = pat + pat_len; name > pat && name[-1] != '/'; name--) {}
 				add_rule(listp, name, (pat + pat_len) - name, excl_self, 0);

fileio.c

@@ -157,8 +157,6 @@ int write_file(int f, int use_seek, OFF_T offset, const char *buf, int len)
 				wf_writeBufSize = WRITE_SIZE * 8;
 				wf_writeBufCnt  = 0;
 				wf_writeBuf = new_array(char, wf_writeBufSize);
-				if (!wf_writeBuf)
-					out_of_memory("write_file");
 			}
 			r1 = (int)MIN((size_t)len, wf_writeBufSize - wf_writeBufCnt);
 			if (r1) {
@@ -217,8 +215,7 @@ struct map_struct *map_file(int fd, OFF_T len, int32 read_size, int32 blk_size)
 {
 	struct map_struct *map;
 
-	if (!(map = new0(struct map_struct)))
-		out_of_memory("map_file");
+	map = new0(struct map_struct);
 
 	if (blk_size && (read_size % blk_size))
 		read_size += blk_size - (read_size % blk_size);
@@ -261,8 +258,6 @@ char *map_ptr(struct map_struct *map, OFF_T offset, int32 len)
 	/* make sure we have allocated enough memory for the window */
 	if (window_size > map->p_size) {
 		map->p = realloc_array(map->p, char, window_size);
-		if (!map->p)
-			out_of_memory("map_ptr");
 		map->p_size = window_size;
 	}
 

flist.c

@@ -301,8 +301,7 @@ static void flist_expand(struct file_list *flist, int extra)
 	if (flist->malloced < flist->used + extra)
 		flist->malloced = flist->used + extra;
 
-	new_ptr = realloc_array(flist->files, struct file_struct *,
-				flist->malloced);
+	new_ptr = realloc_array(flist->files, struct file_struct *, flist->malloced);
 
 	if (DEBUG_GTE(FLIST, 1) && flist->malloced != FLIST_START) {
 		rprintf(FCLIENT, "[%s] expand file_list pointer array to %s bytes, did%s move\n",
@@ -312,9 +311,6 @@ static void flist_expand(struct file_list *flist, int extra)
 	}
 
 	flist->files = new_ptr;
-
-	if (!flist->files)
-		out_of_memory("flist_expand");
 }
 
 static void flist_done_allocating(struct file_list *flist)
@@ -1335,10 +1331,8 @@ struct file_struct *make_file(const char *fname, struct file_list *flist,
 		  + linkname_len;
 	if (pool)
 		bp = pool_alloc(pool, alloc_len, "make_file");
-	else {
-		if (!(bp = new_array(char, alloc_len)))
-			out_of_memory("make_file");
-	}
+	else
+		bp = new_array(char, alloc_len);
 
 	memset(bp, 0, extra_len + FILE_STRUCT_LEN);
 	bp += extra_len;
@@ -1661,8 +1655,7 @@ static void fsort(struct file_struct **fp, size_t num)
 	if (use_qsort)
 		qsort(fp, num, PTR_SIZE, file_compare);
 	else {
-		struct file_struct **tmp = new_array(struct file_struct *,
-						     (num+1) / 2);
+		struct file_struct **tmp = new_array(struct file_struct *, (num+1) / 2);
 		fsort_tmp(fp, num, tmp);
 		free(tmp);
 	}
@@ -1895,13 +1888,11 @@ static void send_implied_dirs(int f, struct file_list *flist, char *fname,
 	len = strlen(limit+1);
 	memcpy(&relname_list, F_DIR_RELNAMES_P(lastpath_struct), sizeof relname_list);
 	if (!relname_list) {
-		if (!(relname_list = new0(item_list)))
-			out_of_memory("send_implied_dirs");
+		relname_list = new0(item_list);
 		memcpy(F_DIR_RELNAMES_P(lastpath_struct), &relname_list, sizeof relname_list);
 	}
 	rnpp = EXPAND_ITEM_LIST(relname_list, relnamecache *, 32);
-	if (!(*rnpp = (relnamecache*)new_array(char, sizeof (relnamecache) + len)))
-		out_of_memory("send_implied_dirs");
+	*rnpp = (relnamecache*)new_array(char, sizeof (relnamecache) + len);
 	(*rnpp)->name_type = name_type;
 	strlcpy((*rnpp)->fname, limit+1, len + 1);
 
@@ -2059,8 +2050,7 @@ void send_extra_file_list(int f, int at_least)
 		}
 
 		if (need_unsorted_flist) {
-			if (!(flist->sorted = new_array(struct file_struct *, flist->used)))
-				out_of_memory("send_extra_file_list");
+			flist->sorted = new_array(struct file_struct *, flist->used);
 			memcpy(flist->sorted, flist->files,
 			       flist->used * sizeof (struct file_struct*));
 		} else
@@ -2414,8 +2404,7 @@ struct file_list *send_file_list(int f, int argc, char *argv[])
 	 * recursion mode, the sender marks duplicate dirs so that it can
 	 * send them together in a single file-list. */
 	if (need_unsorted_flist) {
-		if (!(flist->sorted = new_array(struct file_struct *, flist->used)))
-			out_of_memory("send_file_list");
+		flist->sorted = new_array(struct file_struct *, flist->used);
 		memcpy(flist->sorted, flist->files,
 		       flist->used * sizeof (struct file_struct*));
 	} else
@@ -2597,8 +2586,7 @@ struct file_list *recv_file_list(int f, int dir_ndx)
 		 * order and for calling flist_find()).  We keep the "files"
 		 * list unsorted for our exchange of index numbers with the
 		 * other side (since their names may not sort the same). */
-		if (!(flist->sorted = new_array(struct file_struct *, flist->used)))
-			out_of_memory("recv_file_list");
+		flist->sorted = new_array(struct file_struct *, flist->used);
 		memcpy(flist->sorted, flist->files,
 		       flist->used * sizeof (struct file_struct*));
 		if (inc_recurse && dir_flist->used > dstart) {
@@ -2808,8 +2796,7 @@ struct file_list *flist_new(int flags, char *msg)
 {
 	struct file_list *flist;
 
-	if (!(flist = new0(struct file_list)))
-		out_of_memory(msg);
+	flist = new0(struct file_list);
 
 	if (flags & FLIST_TEMP) {
 		if (!(flist->file_pool = pool_create(SMALL_EXTENT, 0,

generator.c

@@ -2227,8 +2227,6 @@ void generate_files(int f_out, const char *local_name)
 	if (delete_during == 2) {
 		deldelay_size = BIGPATHBUFLEN * 4;
 		deldelay_buf = new_array(char, deldelay_size);
-		if (!deldelay_buf)
-			out_of_memory("delete-delay");
 	}
 	info_levels[INFO_FLIST] = info_levels[INFO_PROGRESS] = 0;
 

getgroups.c

@@ -3,7 +3,7 @@
  * `id -G` on Linux, but it's too hard to find a portable equivalent.
  *
  * Copyright (C) 2002 Martin Pool
- * Copyright (C) 2003-2019 Wayne Davison
+ * Copyright (C) 2003-2020 Wayne Davison
  *
  * This program is free software; you can redistribute it and/or modify
  * it under the terms of the GNU General Public License version 3 as
@@ -20,8 +20,7 @@
 
 #include "rsync.h"
 
-int
-main(UNUSED(int argc), UNUSED(char *argv[]))
+ int main(UNUSED(int argc), UNUSED(char *argv[]))
 {
 	int n, i;
 	gid_t *list;

hashtable.c

@@ -35,9 +35,8 @@ struct hashtable *hashtable_create(int size, int key64)
 			size *= 2;
 	}
 
-	if (!(tbl = new(struct hashtable))
-	 || !(tbl->nodes = new_array0(char, size * node_size)))
-		out_of_memory("hashtable_create");
+	tbl = new(struct hashtable);
+	tbl->nodes = new_array0(char, size * node_size);
 	tbl->size = size;
 	tbl->entries = 0;
 	tbl->node_size = node_size;
@@ -94,8 +93,7 @@ void *hashtable_find(struct hashtable *tbl, int64 key, void *data_when_new)
 		int size = tbl->size * 2;
 		int i;
 
-		if (!(tbl->nodes = new_array0(char, size * tbl->node_size)))
-			out_of_memory("hashtable_node");
+		tbl->nodes = new_array0(char, size * tbl->node_size);
 		tbl->size = size;
 		tbl->entries = 0;
 

hlink.c

@@ -125,8 +125,7 @@ static void match_gnums(int32 *ndx_list, int ndx_count)
 		if (inc_recurse) {
 			node = hashtable_find(prior_hlinks, gnum, data_when_new);
 			if (node->data == data_when_new) {
-				if (!(node->data = new_array0(char, 5)))
-					out_of_memory("match_gnums");
+				node->data = new_array0(char, 5);
 				assert(gnum >= hlink_flist->ndx_start);
 				file->flags |= FLAG_HLINK_FIRST;
 				prev = -1;
@@ -190,8 +189,7 @@ void match_hard_links(struct file_list *flist)
 		int i, ndx_count = 0;
 		int32 *ndx_list;
 
-		if (!(ndx_list = new_array(int32, flist->used)))
-			out_of_memory("match_hard_links");
+		ndx_list = new_array(int32, flist->used);
 
 		for (i = 0; i < flist->used; i++) {
 			if (F_IS_HLINKED(flist->sorted[i]))
@@ -541,8 +539,7 @@ void finish_hard_link(struct file_struct *file, const char *fname, int fin_ndx,
 			exit_cleanup(RERR_MESSAGEIO);
 		}
 		free(node->data);
-		if (!(node->data = strdup(our_name)))
-			out_of_memory("finish_hard_link");
+		node->data = strdup(our_name);
 	}
 }
 

ifuncs.h

@@ -1,6 +1,6 @@
 /* Inline functions for rsync.
  *
- * Copyright (C) 2007-2019 Wayne Davison
+ * Copyright (C) 2007-2020 Wayne Davison
  *
  * This program is free software; you can redistribute it and/or modify
  * it under the terms of the GNU General Public License as published by
@@ -19,8 +19,7 @@
 static inline void
 alloc_xbuf(xbuf *xb, size_t sz)
 {
-	if (!(xb->buf = new_array(char, sz)))
-		out_of_memory("alloc_xbuf");
+	xb->buf = new_array(char, sz);
 	xb->size = sz;
 	xb->len = xb->pos = 0;
 }
@@ -29,8 +28,6 @@ static inline void
 realloc_xbuf(xbuf *xb, size_t sz)
 {
 	char *bf = realloc_array(xb->buf, char, sz);
-	if (!bf)
-		out_of_memory("realloc_xbuf");
 	xb->buf = bf;
 	xb->size = sz;
 }
@@ -104,3 +101,11 @@ free_stat_x(stat_x *sx_p)
     }
 #endif
 }
+
+static inline char *my_strdup(const char *str, const char *file, int line)
+{
+    int len = strlen(str)+1;
+    char *buf = my_alloc(do_malloc, len, 1, file, line);
+    memcpy(buf, str, len);
+    return buf;
+}

io.c

@@ -1239,8 +1239,7 @@ void read_args(int f_in, char *mod_name, char *buf, size_t bufsiz, int rl_nulls,
 	rl_flags |= (protect_args && ic_recv != (iconv_t)-1 ? RL_CONVERT : 0);
 #endif
 
-	if (!(argv = new_array(char *, maxargs)))
-		out_of_memory("read_args");
+	argv = new_array(char *, maxargs);
 	if (mod_name && !protect_args)
 		argv[argc++] = "rsyncd";
 
@@ -1253,8 +1252,7 @@ void read_args(int f_in, char *mod_name, char *buf, size_t bufsiz, int rl_nulls,
 
 		if (argc == maxargs-1) {
 			maxargs += MAX_ARGS;
-			if (!(argv = realloc_array(argv, char *, maxargs)))
-				out_of_memory("read_args");
+			argv = realloc_array(argv, char *, maxargs);
 		}
 
 		if (dot_pos) {
@@ -1262,8 +1260,7 @@ void read_args(int f_in, char *mod_name, char *buf, size_t bufsiz, int rl_nulls,
 				int len = strlen(buf);
 				if (request_len)
 					request_p[0][request_len++] = ' ';
-				if (!(*request_p = realloc_array(*request_p, char, request_len + len + 1)))
-					out_of_memory("read_args");
+				*request_p = realloc_array(*request_p, char, request_len + len + 1);
 				memcpy(*request_p + request_len, buf, len + 1);
 				request_len += len;
 			}
@@ -1272,8 +1269,7 @@ void read_args(int f_in, char *mod_name, char *buf, size_t bufsiz, int rl_nulls,
 			else
 				glob_expand(buf, &argv, &argc, &maxargs);
 		} else {
-			if (!(p = strdup(buf)))
-				out_of_memory("read_args");
+			p = strdup(buf);
 			argv[argc++] = p;
 			if (*p == '.' && p[1] == '\0')
 				dot_pos = argc;

lib/md-defines.h

@@ -13,3 +13,5 @@
 #define CSUM_MD4 4
 #define CSUM_MD5 5
 #define CSUM_XXH64 6
+#define CSUM_XXH3_64 7
+#define CSUM_XXH3_128 8

lib/md5-asm-x86_64.S

@@ -29,6 +29,10 @@
 
 #if !defined USE_OPENSSL && CSUM_CHUNK == 64
 
+#ifdef __APPLE__
+#define md5_process_asm _md5_process_asm
+#endif
+
 .text
 .align 16
 

loadparm.c

@@ -42,6 +42,7 @@
 
 #include "rsync.h"
 #include "itypes.h"
+#include "ifuncs.h"
 #include "default-dont-compress.h"
 
 extern item_list dparam_list;
@@ -471,8 +472,7 @@ static char *expand_vars(char *str)
 		return str;
 
 	bufsize = strlen(str) + 2048;
-	if ((buf = new_array(char, bufsize+1)) == NULL) /* +1 for trailing '\0' */
-		out_of_memory("expand_vars");
+	buf = new_array(char, bufsize+1); /* +1 for trailing '\0' */
 
 	for (t = buf, f = str; bufsize && *f; ) {
 		if (*f == '%' && *++f != '%') {
@@ -601,10 +601,7 @@ FN_LOCAL_BOOL(lp_write_only, write_only)
  * the start, so any lost memory is inconsequential. */
 static inline void string_set(char **s, const char *v)
 {
-	if (!v)
-		*s = NULL;
-	else if (!(*s = strdup(v)))
-		out_of_memory("string_set");
+	*s = v ? strdup(v) : NULL;
 }
 
 /* Copy local_vars into a new section. No need to strdup since we don't free. */

main.c

@@ -22,6 +22,7 @@
 
 #include "rsync.h"
 #include "inums.h"
+#include "ifuncs.h"
 #include "io.h"
 #if defined CONFIG_LOCALE && defined HAVE_LOCALE_H
 #include <locale.h>
@@ -512,8 +513,6 @@ static pid_t do_cmd(char *cmd, char *machine, char *user, char **remote_argv, in
 		if (!cmd)
 			cmd = RSYNC_RSH;
 		cmd = need_to_free = strdup(cmd);
-		if (!cmd)
-			goto oom;
 
 		for (t = f = cmd; *f; f++) {
 			if (*f == ' ')
@@ -657,10 +656,6 @@ static pid_t do_cmd(char *cmd, char *machine, char *user, char **remote_argv, in
 		free(need_to_free);
 
 	return pid;
-
-  oom:
-	out_of_memory("do_cmd");
-	return 0; /* not reached */
 }
 
 /* The receiving side operates in one of two modes:
@@ -824,8 +819,6 @@ static void check_alt_basis_dirs(void)
 		if (dry_run > 1 && *bdir != '/') {
 			int len = curr_dir_len + 1 + bd_len + 1;
 			char *new = new_array(char, len);
-			if (!new)
-				out_of_memory("check_alt_basis_dirs");
 			if (slash && strncmp(bdir, "../", 3) == 0) {
 				/* We want to remove only one leading "../" prefix for
 				 * the directory we couldn't create in dry-run mode:
@@ -1339,19 +1332,12 @@ int client_run(int f_in, int f_out, pid_t pid, int argc, char *argv[])
 	return MAX(exit_code, exit_code2);
 }
 
-static int copy_argv(char *argv[])
+static void dup_argv(char *argv[])
 {
 	int i;
 
-	for (i = 0; argv[i]; i++) {
-		if (!(argv[i] = strdup(argv[i]))) {
-			rprintf (FERROR, "out of memory at %s(%d)\n",
-				 __FILE__, __LINE__);
-			return RERR_MALLOC;
-		}
-	}
-
-	return 0;
+	for (i = 0; argv[i]; i++)
+		argv[i] = strdup(argv[i]);
 }
 
 
@@ -1372,8 +1358,7 @@ static int start_client(int argc, char *argv[])
 
 	/* Don't clobber argv[] so that ps(1) can still show the right
 	 * command line. */
-	if ((ret = copy_argv(argv)) != 0)
-		return ret;
+	dup_argv(argv);
 
 	if (!read_batch) { /* for read_batch, NO source is specified */
 		char *path = check_for_hostspec(argv[0], &shell_machine, &rsync_port);

match.c

@@ -65,8 +65,6 @@ static void build_hash_table(struct sum_struct *s)
 		if (hash_table)
 			free(hash_table);
 		hash_table = new_array(int32, tablesize);
-		if (!hash_table)
-			out_of_memory("build_hash_table");
 		alloc_size = tablesize;
 	}
 

options.c

@@ -21,6 +21,7 @@
 
 #include "rsync.h"
 #include "itypes.h"
+#include "ifuncs.h"
 #include "latest-year.h"
 #include <popt.h>
 
@@ -181,6 +182,10 @@ int rsync_port = 0;
 int alt_dest_type = 0;
 int basis_dir_cnt = 0;
 
+#define DEFAULT_MAX_ALLOC (1024L * 1024 * 1024)
+size_t max_alloc = DEFAULT_MAX_ALLOC;
+char *max_alloc_arg;
+
 static int version_opt_cnt = 0;
 static int remote_option_alloc = 0;
 int remote_option_cnt = 0;
@@ -382,8 +387,7 @@ static char *make_output_option(struct output_struct *words, short *levels, ucha
 		return NULL;
 
 	len++;
-	if (!(buf = new_array(char, len)))
-		out_of_memory("make_output_option");
+	buf = new_array(char, len);
 	pos = 0;
 
 	if (skipped || max < 5)
@@ -714,7 +718,7 @@ static void print_rsync_version(enum logcode f)
 		RSYNC_NAME, RSYNC_VERSION, PROTOCOL_VERSION, subprotocol);
 
 	rprintf(f, "Copyright (C) 1996-" LATEST_YEAR " by Andrew Tridgell, Wayne Davison, and others.\n");
-	rprintf(f, "Web site: http://rsync.samba.org/\n");
+	rprintf(f, "Web site: https://rsync.samba.org/\n");
 
 	rprintf(f, "Capabilities:\n");
 	print_capabilities(f);
@@ -771,7 +775,7 @@ void usage(enum logcode F)
   rprintf(F,"\n");
   rprintf(F,"Use \"rsync --daemon --help\" to see the daemon-mode command-line options.\n");
   rprintf(F,"Please see the rsync(1) and rsyncd.conf(5) man pages for full documentation.\n");
-  rprintf(F,"See http://rsync.samba.org/ for updates, bug reports, and answers\n");
+  rprintf(F,"See https://rsync.samba.org/ for updates, bug reports, and answers\n");
 }
 
 enum {OPT_SERVER = 1000, OPT_DAEMON, OPT_SENDER, OPT_EXCLUDE, OPT_EXCLUDE_FROM,
@@ -889,6 +893,7 @@ static struct poptOption long_options[] = {
   {"ignore-existing",  0,  POPT_ARG_NONE,   &ignore_existing, 0, 0, 0 },
   {"max-size",         0,  POPT_ARG_STRING, &max_size_arg, OPT_MAX_SIZE, 0, 0 },
   {"min-size",         0,  POPT_ARG_STRING, &min_size_arg, OPT_MIN_SIZE, 0, 0 },
+  {"max-alloc",        0,  POPT_ARG_STRING, &max_alloc_arg, 0, 0, 0 },
   {"sparse",          'S', POPT_ARG_VAL,    &sparse_files, 1, 0, 0 },
   {"no-sparse",        0,  POPT_ARG_VAL,    &sparse_files, 0, 0, 0 },
   {"no-S",             0,  POPT_ARG_VAL,    &sparse_files, 0, 0, 0 },
@@ -945,6 +950,7 @@ static struct poptOption long_options[] = {
   {"zc",               0,  POPT_ARG_STRING, &compress_choice, 0, 0, 0 },
   {"skip-compress",    0,  POPT_ARG_STRING, &skip_compress, 0, 0, 0 },
   {"compress-level",   0,  POPT_ARG_INT,    &do_compression_level, 0, 0, 0 },
+  {"zl",               0,  POPT_ARG_INT,    &do_compression_level, 0, 0, 0 },
   {0,                 'P', POPT_ARG_NONE,   0, 'P', 0, 0 },
   {"progress",         0,  POPT_ARG_VAL,    &do_progress, 1, 0, 0 },
   {"no-progress",      0,  POPT_ARG_VAL,    &do_progress, 0, 0, 0 },
@@ -1250,14 +1256,16 @@ static int count_args(const char **argv)
 	return i;
 }
 
-
-static OFF_T parse_size_arg(char **size_arg, char def_suf)
+/* If the size_arg is an invalid string or the value is < min_value, an error
+ * is put into err_buf & the return is -1.  Note that this parser does NOT
+ * support negative numbers, so a min_value < 0 doesn't make any sense. */
+static ssize_t parse_size_arg(char *size_arg, char def_suf, const char *opt_name, ssize_t min_value)
 {
-	int reps, mult, make_compatible = 0;
-	const char *arg;
-	OFF_T size = 1;
+	int reps, mult;
+	const char *arg, *err = "invalid";
+	ssize_t size = 1;
 
-	for (arg = *size_arg; isDigit(arg); arg++) {}
+	for (arg = size_arg; isDigit(arg); arg++) {}
 	if (*arg == '.')
 		for (arg++; isDigit(arg); arg++) {}
 	switch (*arg && *arg != '+' && *arg != '-' ? *arg++ : def_suf) {
@@ -1273,40 +1281,40 @@ static OFF_T parse_size_arg(char **size_arg, char def_suf)
 	case 'g': case 'G':
 		reps = 3;
 		break;
+	case 't': case 'T':
+		reps = 4;
+		break;
+	case 'p': case 'P':
+		reps = 5;
+		break;
 	default:
-		return -1;
+		goto failure;
 	}
 	if (*arg == 'b' || *arg == 'B')
-		mult = 1000, make_compatible = 1, arg++;
+		mult = 1000, arg++;
 	else if (!*arg || *arg == '+' || *arg == '-')
 		mult = 1024;
 	else if (strncasecmp(arg, "ib", 2) == 0)
 		mult = 1024, arg += 2;
 	else
-		return -1;
+		goto failure;
 	while (reps--)
 		size *= mult;
-	size *= atof(*size_arg);
-	if ((*arg == '+' || *arg == '-') && arg[1] == '1')
-		size += atoi(arg), make_compatible = 1, arg += 2;
+	size *= atof(size_arg);
+	if ((*arg == '+' || *arg == '-') && arg[1] == '1' && arg != size_arg)
+		size += atoi(arg), arg += 2;
 	if (*arg)
-		return -1;
-	if (size > 0 && make_compatible && def_suf == 'b') {
-		/* We convert this manually because we may need %lld precision,
-		 * and that's not a portable sprintf() escape. */
-		char buf[128], *s = buf + sizeof buf - 1;
-		OFF_T num = size;
-		*s = '\0';
-		while (num) {
-			*--s = (char)(num % 10) + '0';
-			num /= 10;
-		}
-		if (!(*size_arg = strdup(s)))
-			out_of_memory("parse_size_arg");
+		goto failure;
+	if (size < min_value) {
+		err = size < 0 ? "too big" : "too small";
+		goto failure;
 	}
 	return size;
-}
 
+failure:
+	snprintf(err_buf, sizeof err_buf, "--%s value is %s: %s\n", opt_name, err, size_arg);
+	return -1;
+}
 
 static void create_refuse_error(int which)
 {
@@ -1529,8 +1537,6 @@ int parse_arguments(int *argc_p, const char ***argv_p)
 			if (daemon_filter_list.head) {
 				int rej;
 				char *cp = strdup(arg);
-				if (!cp)
-					out_of_memory("parse_arguments");
 				if (!*cp)
 					rej = 1;
 				else {
@@ -1654,8 +1660,6 @@ int parse_arguments(int *argc_p, const char ***argv_p)
 				remote_option_alloc += 16;
 				remote_options = realloc_array(remote_options,
 							const char *, remote_option_alloc);
-				if (!remote_options)
-					out_of_memory("parse_arguments");
 				if (!remote_option_cnt)
 					remote_options[0] = "ARG0";
 			}
@@ -1685,39 +1689,25 @@ int parse_arguments(int *argc_p, const char ***argv_p)
 			break;
 
 		case OPT_MAX_SIZE:
-			if ((max_size = parse_size_arg(&max_size_arg, 'b')) < 0) {
-				snprintf(err_buf, sizeof err_buf,
-					"--max-size value is invalid: %s\n",
-					max_size_arg);
+			if ((max_size = parse_size_arg(max_size_arg, 'b', "max-size", 0)) < 0)
 				return 0;
-			}
+			max_size_arg = num_to_byte_string(max_size);
 			break;
 
 		case OPT_MIN_SIZE:
-			if ((min_size = parse_size_arg(&min_size_arg, 'b')) < 0) {
-				snprintf(err_buf, sizeof err_buf,
-					"--min-size value is invalid: %s\n",
-					min_size_arg);
+			if ((min_size = parse_size_arg(min_size_arg, 'b', "min-size", 0)) < 0)
 				return 0;
-			}
+			min_size_arg = num_to_byte_string(min_size);
 			break;
 
-		case OPT_BWLIMIT:
-			{
-				OFF_T limit = parse_size_arg(&bwlimit_arg, 'K');
-				if (limit < 0) {
-					snprintf(err_buf, sizeof err_buf,
-						"--bwlimit value is invalid: %s\n", bwlimit_arg);
-					return 0;
-				}
-				bwlimit = (limit + 512) / 1024;
-				if (limit && !bwlimit) {
-					snprintf(err_buf, sizeof err_buf,
-						"--bwlimit value is too small: %s\n", bwlimit_arg);
-					return 0;
-				}
-			}
+		case OPT_BWLIMIT: {
+			ssize_t size = parse_size_arg(bwlimit_arg, 'K', "bwlimit", 512);
+			if (size < 0)
+				return 0;
+			bwlimit_arg = num_to_byte_string(size);
+			bwlimit = (size + 512) / 1024;
 			break;
+		}
 
 		case OPT_APPEND:
 			if (am_server)
@@ -1899,6 +1889,18 @@ int parse_arguments(int *argc_p, const char ***argv_p)
 		exit_cleanup(0);
 	}
 
+	if (!max_alloc_arg) {
+		max_alloc_arg = getenv("RSYNC_MAX_ALLOC");
+		if (max_alloc_arg && !*max_alloc_arg)
+			max_alloc_arg = NULL;
+	}
+	if (max_alloc_arg) {
+		ssize_t size = parse_size_arg(max_alloc_arg, 'B', "max-alloc", 1024*1024);
+		if (size < 0)
+			return 0;
+		max_alloc = size;
+	}
+
 	if (protect_args < 0) {
 		if (am_server)
 			protect_args = 0;
@@ -2222,6 +2224,7 @@ int parse_arguments(int *argc_p, const char ***argv_p)
 	}
 	if (backup_dir) {
 		size_t len;
+		make_backups = 1; /* --backup-dir implies --backup */
 		while (*backup_dir == '.' && backup_dir[1] == '/')
 			backup_dir += 2;
 		if (*backup_dir == '.' && backup_dir[1] == '\0')
@@ -2768,6 +2771,11 @@ void server_options(char **args, int *argc_p)
 		}
 	}
 
+	if (max_alloc_arg && max_alloc != DEFAULT_MAX_ALLOC) {
+		args[ac++] = "--max-alloc";
+		args[ac++] = max_alloc_arg;
+	}
+
 	/* --delete-missing-args needs the cooperation of both sides, but
 	 * the sender can handle --ignore-missing-args by itself. */
 	if (missing_args == 2)

packaging/branch-from-patch

@@ -171,4 +171,4 @@ if __name__ == '__main__':
     args = parser.parse_args()
     main()
 
-# vim: sw=4 et
+# vim: sw=4 et ft=python

packaging/lsb/rsync.spec

@@ -1,14 +1,14 @@
 Summary: A fast, versatile, remote (and local) file-copying tool
 Name: rsync
-Version: 3.2.1
+Version: 3.2.2
 %define fullversion %{version}pre1
 Release: 0.1.pre1
 %define srcdir src-previews
 Group: Applications/Internet
 License: GPL
-Source0: http://rsync.samba.org/ftp/rsync/%{srcdir}/rsync-%{fullversion}.tar.gz
-#Source1: http://rsync.samba.org/ftp/rsync/%{srcdir}/rsync-patches-%{fullversion}.tar.gz
-URL: http://rsync.samba.org/
+Source0: https://rsync.samba.org/ftp/rsync/%{srcdir}/rsync-%{fullversion}.tar.gz
+#Source1: https://rsync.samba.org/ftp/rsync/%{srcdir}/rsync-patches-%{fullversion}.tar.gz
+URL: https://rsync.samba.org/
 
 Prefix: %{_prefix}
 BuildRoot: /var/tmp/%{name}-root
@@ -79,8 +79,8 @@ rm -rf $RPM_BUILD_ROOT
 %dir /etc/rsync-ssl/certs
 
 %changelog
-* Sun Jun 21 2020 Wayne Davison <wayne@opencoder.net>
-Released 3.2.1pre1.
+* Sat Jun 27 2020 Wayne Davison <wayne@opencoder.net>
+Released 3.2.2pre1.
 
 * Fri Mar 21 2008 Wayne Davison <wayne@opencoder.net>
 Added installation of /etc/xinetd.d/rsync file and some commented-out

packaging/patch-update

@@ -31,6 +31,10 @@ def main():
 
     starting_branch, args.base_branch = check_git_state(args.base_branch, not args.skip_check, args.patches_dir)
 
+    gen_stash(starting_branch)
+    if starting_branch == 'master':
+        cmd_run('md5sum configure.ac >.gen-stash/configure.ac.sum')
+
     master_commit = latest_git_hash(args.base_branch)
 
     if args.gen:
@@ -95,6 +99,8 @@ def main():
         time.sleep(1)
     cmd_chk(['git', 'checkout', starting_branch])
 
+    gen_unstash(starting_branch)
+
 
 def update_patch(patch):
     global last_touch
@@ -115,13 +121,16 @@ def update_patch(patch):
 
     while args.gen and last_touch >= time.time():
         time.sleep(1)
-    s = cmd_run(f"git checkout patch/{args.base_branch}/{patch}".split())
+
+    branch = f"patch/{args.base_branch}/{patch}"
+    s = cmd_run(['git', 'checkout', branch])
     if s.returncode != 0:
         return 0
 
     s = cmd_run(['git', 'merge', based_on])
     ok = s.returncode == 0
     if not ok or args.shell:
+        gen_unstash(branch)
         m = re.search(r'([^/]+)$', parent)
         parent_dir = m[1]
         if not ok:
@@ -138,6 +147,7 @@ def update_patch(patch):
             if is_clean:
                 break
             print(status_txt, end='')
+        gen_stash(branch)
 
     with open(f"{args.patches_dir}/{patch}.diff", 'w', encoding='utf-8') as fh:
         fh.write(description[patch])
@@ -191,6 +201,34 @@ def update_patch(patch):
     return 1
 
 
+# The autoconf cache dir can totally mess up if 2 different *.ac files have the same mtime!
+def gen_stash_prep(branch):
+    if os.path.isdir('autom4te.cache'):
+        shutil.rmtree('autom4te.cache')
+    return '.gen-stash/' + branch.replace('/', '%') + '.tar.gz'
+
+
+def gen_stash(branch):
+    files = 'configure.sh config.h.in Makefile'.split()
+    for fn in files:
+        if not os.path.exists(fn):
+            return
+    if not os.path.isdir('.gen-stash'):
+        os.mkdir('.gen-stash', 0o700)
+    tar_fn = gen_stash_prep(branch)
+    cmd_run(['./prepare-source'])
+    cmd_chk(['tar', 'czf', tar_fn, *files])
+
+
+def gen_unstash(branch):
+    tar_fn = gen_stash_prep(branch)
+    if os.path.exists(tar_fn):
+        cmd_chk(['tar', 'xf', tar_fn])
+    out, rc = cmd_txt_status('md5sum --status -c .gen-stash/configure.ac.sum'.split())
+    if rc:
+        print('*' * 10, "configure.ac differs from master version", '*' * 10)
+
+
 if __name__ == '__main__':
     parser = argparse.ArgumentParser(description="Turn a git branch back into a diff files in the patches dir.", add_help=False)
     parser.add_argument('--branch', '-b', dest='base_branch', metavar='BASE_BRANCH', default='master', help="The branch the patch is based on. Default: master.")
@@ -207,4 +245,4 @@ if __name__ == '__main__':
         args.patches_dir = args.gen
     main()
 
-# vim: sw=4 et
+# vim: sw=4 et ft=python

packaging/pkglib.py

@@ -218,8 +218,8 @@ def get_NEWS_version_info():
 
     with open('NEWS.md', 'r', encoding='utf-8') as fh:
         for line in fh:
-            if not last_version:
-                m = re.search(r'rsync (\d+\.\d+\.\d+).*\d\d\d\d', line)
+            if not last_version: # Find the first non-dev|pre version with a release date.
+                m = re.search(r'rsync (\d+\.\d+\.\d+) .*\d\d\d\d', line)
                 if m:
                     last_version = m[1]
             m = rel_re.match(line)

packaging/release-rsync

@@ -13,6 +13,7 @@ sys.path = ['packaging'] + sys.path
 
 from pkglib import *
 
+os.environ['LESS'] = 'mqeiXR'; # Make sure that -F is turned off and -R is turned on.
 dest = os.environ['HOME'] + '/samba-rsync-ftp'
 ORIGINAL_PATH = os.environ['PATH']
 
@@ -189,11 +190,12 @@ About to:
             txt = replace_or_die(x_re, repl, txt, f"Unable to find SUBPROTOCOL_VERSION define in {fn}")
         elif fn == 'NEWS.md':
             efv = re.escape(finalversion)
-            x_re = re.compile(r'^---+\n<.+>\s+# NEWS .+\s+Protocol: .+\n')
+            x_re = re.compile(r'^<.+>\s+# NEWS for rsync %s \(UNRELEASED\)\s+Protocol: .+\n' % efv)
             rel_day = 'UNRELEASED' if pre else today
-            repl = ('-'*78 + f'\n<a name="{version}"></a>\n\n# NEWS for rsync {version} ({rel_day})\n\n'
+            repl = (f'<a name="{finalversion}"></a>\n\n# NEWS for rsync {finalversion} ({rel_day})\n\n'
                 + f"Protocol: {protocol_version} ({proto_changed})\n")
-            msg = f"The top lines of {fn} are not in the right format.  They must be:\n" + repl
+            good_top = re.sub(r'\(.*?\)', '(UNRELEASED)', repl, 1)
+            msg = f"The top lines of {fn} are not in the right format.  It should be:\n" + good_top
             txt = replace_or_die(x_re, repl, txt, msg)
             x_re = re.compile(r'^(\| )(\S{2} \S{3} \d{4})(\s+\|\s+%s\s+\| ).{11}(\s+\| )\S{2}(\s+\|+)$' % efv, re.M)
             repl = lambda m: m[1] + (m[2] if pre else ztoday) + m[3] + proto_change_date + m[4] + protocol_version + m[5]
@@ -380,4 +382,4 @@ if __name__ == '__main__':
     args = parser.parse_args()
     main()
 
-# vim: sw=4 et
+# vim: sw=4 et ft=python

packaging/systemd/rsync.service

@@ -2,6 +2,7 @@
 Description=fast remote file copy program daemon
 ConditionPathExists=/etc/rsyncd.conf
 After=network.target
+Documentation=man:rsync(1) man:rsyncd.conf(5)
 
 [Service]
 ExecStart=/usr/bin/rsync --daemon --no-detach

packaging/var-checker

@@ -84,4 +84,4 @@ if __name__ == '__main__':
     args = parser.parse_args()
     main()
 
-# vim: sw=4 et
+# vim: sw=4 et ft=python

params.c

@@ -212,11 +212,6 @@ static BOOL Section( FILE *InFile, BOOL (*sfunc)(char *) )
       {
       bSize += BUFR_INC;
       bufr   = realloc_array( bufr, char, bSize );
-      if( NULL == bufr )
-        {
-        rprintf(FLOG, "%s Memory re-allocation failure.", func);
-        return( False );
-        }
       }
 
     /* Handle a single character. */
@@ -306,11 +301,6 @@ static BOOL Parameter( FILE *InFile, BOOL (*pfunc)(char *, char *), int c )
       {
       bSize += BUFR_INC;
       bufr   = realloc_array( bufr, char, bSize );
-      if( NULL == bufr )
-        {
-        rprintf(FLOG, "%s Memory re-allocation failure.", func) ;
-        return( False );
-        }
       }
 
     switch( c )
@@ -382,11 +372,6 @@ static BOOL Parameter( FILE *InFile, BOOL (*pfunc)(char *, char *), int c )
       {
       bSize += BUFR_INC;
       bufr   = realloc_array( bufr, char, bSize );
-      if( NULL == bufr )
-        {
-        rprintf(FLOG, "%s Memory re-allocation failure.", func) ;
-        return( False );
-        }
       }
 
     switch( c )
@@ -639,12 +624,6 @@ int pm_process( char *FileName,
     {                                         /* allocate one, then parse,   */
     bSize = BUFR_INC;                         /* then free.                  */
     bufr = new_array( char, bSize );
-    if( NULL == bufr )
-      {
-      rprintf(FLOG, "%s memory allocation failure.\n", func);
-      fclose(InFile);
-      return( False );
-      }
     result = Parse( InFile, sfunc, pfunc );
     free( bufr );
     bufr  = NULL;

rsync.1.md

@@ -381,7 +381,7 @@ detailed description below for a complete description.
 --write-devices          write to devices as files (implies --inplace)
 --dry-run, -n            perform a trial run with no changes made
 --whole-file, -W         copy files whole (w/o delta-xfer algorithm)
---checksum-choice=STR    choose the checksum algorithms
+--checksum-choice=STR    choose the checksum algorithm (aka --cc)
 --one-file-system, -x    don't cross filesystem boundaries
 --block-size=SIZE, -B    force a fixed checksum block-size
 --rsh=COMMAND, -e        specify the remote shell to use
@@ -403,6 +403,7 @@ detailed description below for a complete description.
 --max-delete=NUM         don't delete more than NUM files
 --max-size=SIZE          don't transfer any file larger than SIZE
 --min-size=SIZE          don't transfer any file smaller than SIZE
+--max-alloc=SIZE         change a limit relating to memory alloc
 --partial                keep partially transferred files
 --partial-dir=DIR        put a partially transferred file into DIR
 --delay-updates          put all updated files into place at end
@@ -422,7 +423,8 @@ detailed description below for a complete description.
 --copy-dest=DIR          ... and include copies of unchanged files
 --link-dest=DIR          hardlink to files in DIR when unchanged
 --compress, -z           compress file data during the transfer
---compress-level=NUM     explicitly set compression level
+--compress-choice=STR    choose the compression algorithm (aka --zc)
+--compress-level=NUM     explicitly set compression level (aka --zl)
 --skip-compress=LIST     skip compressing files with suffix in LIST
 --cvs-exclude, -C        auto-ignore files in the same way CVS does
 --filter=RULE, -f        add a file-filtering RULE
@@ -604,7 +606,7 @@ your home directory (remove the '=' for that).
     helps to avoid overpopulating the protocol data with extra message data.
 
     The option does not affect the remote side of a transfer without using
-    `--remote-option` -- e.g. `-M--msgs2stderr` or `{-M,}--msgs2stderr`.
+    `--remote-option`, e.g. `-M--msgs2stderr` or `{-M,}--msgs2stderr`.
 
     Also keep in mind that connecting to a normal (non-remote-shell) daemon
     does not have a stderr channel to send messages back to the client side, so
@@ -644,7 +646,7 @@ your home directory (remove the '=' for that).
     after using another mirroring system which may not preserve timestamps
     exactly.
 
-0.  `--modify-window`, `-@`
+0.  `--modify-window=NUM`, `-@`
 
     When comparing two timestamps, rsync treats the timestamps as being equal
     if they differ by no more than the modify-window value.  The default is 0,
@@ -842,7 +844,7 @@ your home directory (remove the '=' for that).
 
 0.  `--backup-dir=DIR`
 
-    In combination with the `--backup` option, this tells rsync to store all
+    This implies the `--backup` option, and tells rsync to store all
     backups in the specified directory on the receiving side.  This can be used
     for incremental backups.  You can additionally specify a backup suffix
     using the `--suffix` option (otherwise the files backed up in the specified
@@ -1225,7 +1227,7 @@ your home directory (remove the '=' for that).
     those used by `--fake-super`) unless you repeat the option (e.g. `-XX`).
     This "copy all xattrs" mode cannot be used with `--fake-super`.
 
-0.  `--chmod`
+0.  `--chmod=CHMOD`
 
     This option tells rsync to apply one or more comma-separated "chmod" modes
     to the permission of the files in the transfer.  The resulting value is
@@ -1472,13 +1474,16 @@ your home directory (remove the '=' for that).
 
     The checksum options that you may be able to use are:
 
-    - `auto` (the default)
-    - `xxh64` (aka xxhash)
+    - `auto` (the default automatic choice)
+    - `xxh128`
+    - `xxh3`
+    - `xxh64` (aka `xxhash`)
     - `md5`
     - `md4`
     - `none`
 
-    Run `rsync -V` to see the default checksum list compiled into your version.
+    Run `rsync -V` to see the default checksum list compiled into your version
+    (which may differ from the above list).
 
     If "none" is specified for the first (or only) name, the `--whole-file`
     option is forced on and no checksum verification is performed on the
@@ -1488,22 +1493,22 @@ your home directory (remove the '=' for that).
     The "auto" option is the default, where rsync bases its algorithm choice on
     a negotiation between the client and the server as follows:
 
-    If both the client and the server are at least version 3.2.0, they will
-    exchange a list of checksum names and choose the first one in the list that
-    they have in common.  This typically means that they will choose xxh64 if
-    they both support it and fall back to MD5.  If one side of the transfer is
-    not new enough to support this checksum negotiation, then a value is chosen
-    based on the protocol version (which chooses between MD5 and various
-    flavors of MD4 based on protocol age).
+    When both sides of the transfer are at least 3.2.0, rsync chooses the first
+    algorithm in the client's list of choices that is also in the server's list
+    of choices.  If no common checksum choice is found, the client exits with
+    an error.  If the remote rsync is too old to support checksum negotiation,
+    a value is chosen based on the protocol version (which chooses between MD5
+    and various flavors of MD4 based on protocol age).
 
-    You can also override the checksum using the RSYNC_CHECKSUM_LIST
-    environment variable by setting it to a space-separated list of checksum
-    names that you consider acceptable.  If no common checksum is found, the
-    client exits with an error.  This method does not allow you to specify the
-    transfer checksum separately from the pre-transfer checksum, and it ignores
-    "auto" and all unknown checksum names.  If the remote rsync is not new
-    enough to handle a checksum negotiation list, the list is silently ignored
-    unless it contains the string "FAIL".
+    The default order can be customized by setting the environment variable
+    RSYNC_CHECKSUM_LIST to a space-separated list of acceptable checksum names.
+    If the environment variable contains a "`&`" character, the string is
+    separated into the "client list & server list".  If the resulting string is
+    empty (or contains only whitespace) the default checksum list is used.
+    This method does not allow you to specify the transfer checksum separately
+    from the pre-transfer checksum, and it discards "auto" and all unknown
+    checksum names.  A list with only invalid names will result in a failed
+    negotiation.
 
     The use of the `--checksum-choice` option overrides this environment list.
 
@@ -1730,12 +1735,16 @@ your home directory (remove the '=' for that).
     data that goes into the file-lists, and thus it doesn't affect deletions.
     It just limits the files that the receiver requests to be transferred.
 
-    The suffixes are as follows: "K" (or "KiB") is a kibibyte (1024), "M" (or
-    "MiB") is a mebibyte (1024\*1024), and "G" (or "GiB") is a gibibyte
-    (1024\*1024\*1024).  If you want the multiplier to be 1000 instead of 1024,
-    use "KB", "MB", or "GB". (Note: lower-case is also accepted for all
-    values.) Finally, if the suffix ends in either "+1" or "-1", the value will
-    be offset by one byte in the indicated direction.
+    The accepted suffix letters are: `B`, `K`, `G`, `T`, and `P` for bytes,
+    kilobytes/kibibytes, megabytes/mebibytes, gigabytes/gibibytes,
+    terabytes/tebibytes, and petabytes/pebibytes.  If you use a single-char
+    suffix or add-on "ib" to it (e.g. "G" or "GiB") then you get units that are
+    multiples of 1024.  If you use a two-letter suffix that ends with a "B"
+    (e.g. "kb") then you get units that are multiples of 1000.  The suffix
+    letters can be any mix of upper and lower-case that you want to use.
+
+    Finally, if the string ends with either "+1" or "-1", it is offset by one
+    byte in the indicated direction.  The largest possible value is `8192P-1`.
 
     Examples: `--max-size=1.5mb-1` is 1499999 bytes, and `--max-size=2g+1` is
     2147483649 bytes.
@@ -1750,7 +1759,29 @@ your home directory (remove the '=' for that).
 
     Note that rsync versions prior to 3.1.0 did not allow `--min-size=0`.
 
-0.  `--block-size=BLOCKSIZE`, `-B`
+0.  `--max-alloc=SIZE`
+
+    By default rsync limits an individual malloc/realloc to about 1GB in size.
+    For most people this limit works just fine and prevents a protocol error
+    causing rsync to request massive amounts of memory.  However, if you have
+    many millions of files in a transfer, a large amount of server memory, and
+    you don't want to split up your transfer into multiple parts, you can
+    increase the per-allocation limit to something larger and rsync will
+    consume more memory.
+
+    Keep in mind that this is not a limit on the total size of allocated
+    memory.  It is a sanity-check value for each individual allocation.
+
+    See the `--max-size` option for a description of how SIZE can be specified.
+    The default suffix if none is given is bytes.
+
+    You can set a default value using the environment variable RSYNC_MAX_ALLOC
+    using the same SIZE values as supported by this option.  If the remote
+    rsync doesn't understand the `--max-alloc` option, you can override an
+    environmental value by specifying `--max-alloc=1g`, which will make rsync
+    avoid sending the option to the remote side (because "1G" is the default).
+
+0.  `--block-size=SIZE`, `-B`
 
     This forces the block size used in rsync's delta-transfer algorithm to a
     fixed value.  It is normally selected based on the size of each file being
@@ -2259,29 +2290,40 @@ your home directory (remove the '=' for that).
     destination machine, which reduces the amount of data being transmitted --
     something that is useful over a slow connection.
 
-    The "zlib" compression method typically achieves better compression ratios
-    than can be achieved by using a compressing remote shell or a compressing
-    transport because it takes advantage of the implicit information in the
-    matching data blocks that are not explicitly sent over the connection.
-    This matching-data compression comes at a cost of CPU, though, and can be
-    disabled by using the "zlibx" compresson method instead.  This can be
-    selected by repeating the `-z` option or specifying
-    `--compress-choice=zlibx`, but it only works if both sides of the transfer
-    are at least version 3.1.1.
+    Rsync supports multiple compression methods and will choose one for you
+    unless you force the choice using the `--compress-choice` option.
 
-    Note that if you see an error about an option named `--old-compress` or
-    `--new-compress`, this is rsync trying to send the `--compress-choice=zlib`
-    or `--compress-choice=zlibx` option in a backward-compatible manner that
-    more rsync versions understand.  This error indicates that the older rsync
-    version will not allow you to force the compression type.
+    Run `rsync -V` to see the default compress list compiled into your version.
 
-    See the `--skip-compress` option for the default list of file suffixes that
-    will not be compressed.
+    When both sides of the transfer are at least 3.2.0, rsync chooses the first
+    algorithm in the client's list of choices that is also in the server's list
+    of choices.  If no common compress choice is found, the client exits with
+    an error.  If the remote rsync is too old to support checksum negotiation,
+    its list is assumed to be "zlib".
+
+    The default order can be customized by setting the environment variable
+    RSYNC_COMPRESS_LIST to a space-separated list of acceptable compression
+    names.  If the environment variable contains a "`&`" character, the string
+    is separated into the "client list & server list".  If the resulting string
+    is empty (or contains only whitespace) the default compress list is used.
+    Any unknown compression names are discarded from the list, but a list with
+    only invalid names will result in a failed negotiation.
+
+    There are some older rsync versions that were configured to reject a `-z`
+    option and require the use of `-zz` because their compression library was
+    not compatible with the default zlib compression method.  You can usually
+    ignore this weirdness unless the rsync server complains and tells you to
+    specify `-zz`.
+
+    See also the `--skip-compress` option for the default list of file suffixes
+    that will trasnferred with no (or minimal) compression.
 
 0.  `--compress-choice=STR`, `--zc=STR`
 
-    This option can be used to override the automatic selection of the
-    compression algorithm that is the default when `--compress` is used.
+    This option can be used to override the automatic negotiation of the
+    compression algorithm that occurs when `--compress` is used.  The option
+    implies `--compress` unless "none" was specified, which instead implies
+    `--no-compress`.
 
     The compression options that you may be able to use are:
 
@@ -2291,31 +2333,52 @@ your home directory (remove the '=' for that).
     - `zlib`
     - `none`
 
-    Run `rsync -V` to see the compress list compiled into your version.
+    Run `rsync -V` to see the default compress list compiled into your version
+    (which may differ from the above list).
 
-    The "zlibx" algorithm is given preference over "zlib" if both sides of the
-    transfer are at least version 3.2.0, otherwise it will choose "zlib" unless
-    you override it via something like `-zz`.  These 2 algorithms are the stame
-    except that "zlibx" does not try to include matched data that was not
-    transferred in the compression computations.
+    Note that if you see an error about an option named `--old-compress` or
+    `--new-compress`, this is rsync trying to send the `--compress-choice=zlib`
+    or `--compress-choice=zlibx` option in a backward-compatible manner that
+    more rsync versions understand.  This error indicates that the older rsync
+    version on the server will not allow you to force the compression type.
 
-    If "none" is specified, that is equivalent to using `--no-compress`.
+    Note that the "zlibx" compression algorithm is just the "zlib" algorithm
+    with matched data excluded from the compression stream (to try to make it
+    more compatible with an external zlib implementation).
 
-    This option implies `--compress` unless "none" was specified.
+0.  `--compress-level=NUM`, `--zl=NUM`
 
-    You can also override the compression negotiation using the
-    RSYNC_COMPRESS_LIST environment variable by setting it to a space-separated
-    list of compression names that you consider acceptable.  If no common
-    compress choice is found, the client exits with an error.  It ignores
-    "auto" and all unknown compression names.  If the remote rsync is not new
-    enough to handle a compression negotiation list, the list is silently
-    ignored unless it contains the string "FAIL".
+    Explicitly set the compression level to use (see `--compress`, `-z`)
+    instead of letting it default.  The `--compress` option is implied as long
+    as the level chosen is not a "don't compress" level for the compression
+    algorithm that is in effect (e.g. zlib compression treats level 0 as
+    "off").
 
-0.  `--compress-level=NUM`
+    The level values vary depending on the checksum in effect.  Because rsync
+    will negotiate a checksum choice by default when the remote rsync is new
+    enough, it can be good to combine this option with a `--compress-choice`
+    (`--zc`) option unless you're sure of the choice in effect.  For example:
 
-    Explicitly set the compression level to use (see `--compress`) instead of
-    letting it default.  If NUM is non-zero, the `--compress` option is
-    implied.
+    >     rsync -aiv --zc=zstd --zl=22 host:src/ dest/
+
+    For zlib & zlibx compression the valid values are from 1 to 9 with 6 being
+    the default.  Specifying 0 turns compression off, and specifying -1 chooses
+    the default of 6.
+
+    For zstd compression the valid values are from -131072 to 22 with 3 being
+    the default. Specifying 0 chooses the default of 3.
+
+    For lz4 compression there are no levels, so the value is always 0.
+
+    If you specify a too-large or too-small value, the number is silently
+    limited to a valid value.  This allows you to specify something like
+    `--zl=999999999` and be assured that you'll end up with the maximum
+    compression level no matter what algorithm was chosen.
+
+    If you want to know the compression level that is in effect, specify
+    `--debug=nstr` to see the "negotiated string" results.  This will report
+    something like "`Client compress: zstd (level 3)`" (along with the checksum
+    choice in effect).
 
 0.  `--skip-compress=LIST`
 
@@ -2483,19 +2546,19 @@ your home directory (remove the '=' for that).
     If you specify "`--chown=foo:bar`", this is exactly the same as specifying
     "`--usermap=*:foo --groupmap=*:bar`", only easier.
 
-0.  `--timeout=TIMEOUT`
+0.  `--timeout=SECONDS`
 
     This option allows you to set a maximum I/O timeout in seconds.  If no data
     is transferred for the specified time then rsync will exit.  The default is
     0, which means no timeout.
 
-0.  `--contimeout`
+0.  `--contimeout=SECONDS`
 
     This option allows you to set the amount of time that rsync will wait for
     its connection to an rsync daemon to succeed.  If the timeout is reached,
     rsync exits with an error.
 
-0.  `--address`
+0.  `--address=ADDRESS`
 
     By default rsync will bind to the wildcard address when connecting to an
     rsync daemon.  The `--address` option allows you to specify a specific IP
@@ -2510,7 +2573,7 @@ your home directory (remove the '=' for that).
     the port as a part of the URL).  See also this option in the `--daemon`
     mode section.
 
-0.  `--sockopts`
+0.  `--sockopts=OPTIONS`
 
     This option can provide endless fun for people who like to tune their
     systems to the utmost degree.  You can set all sorts of socket options
@@ -2963,7 +3026,7 @@ your home directory (remove the '=' for that).
 
     CAUTION: sending SIGVTALRM to an older rsync (pre-3.2.0) will kill it.
 
-0. `--password-file=FILE`
+0.  `--password-file=FILE`
 
     This option allows you to provide a password for accessing an rsync daemon
     via a file or via standard input if **FILE** is `-`.  The file should
@@ -2978,7 +3041,7 @@ your home directory (remove the '=' for that).
     authentication (i.e. if you have also specified a password in the daemon's
     config file).
 
-0. `--early-input=FILE`
+0.  `--early-input=FILE`
 
     This option allows rsync to send up to 5K of data to the "early exec"
     script on its stdin.  One possible use of this data is to give the script a
@@ -2987,7 +3050,7 @@ your home directory (remove the '=' for that).
 
     The daemon must be at least version 3.2.1.
 
-0. `--list-only`
+0.  `--list-only`
 
     This option will cause the source files to be listed instead of
     transferred.  This option is inferred if there is a single source arg and
@@ -3016,7 +3079,7 @@ your home directory (remove the '=' for that).
     need to expand a directory's content), or turn on recursion and exclude the
     content of subdirectories: `-r --exclude='/*/*'`.
 
-0. `--bwlimit=RATE`
+0.  `--bwlimit=RATE`
 
     This option allows you to specify the maximum transfer rate for the data
     sent over the socket, specified in units per second.  The RATE value can be
@@ -3042,13 +3105,18 @@ your home directory (remove the '=' for that).
     buffered, while other can show up as very slow when the flushing of the
     output buffer occurs.  This may be fixed in a future version.
 
-0. `--write-batch=FILE`
+0.  `--write-batch=FILE`
 
     Record a file that can later be applied to another identical destination
     with `--read-batch`.  See the "BATCH MODE" section for details, and also
     the `--only-write-batch` option.
 
-0. `--only-write-batch=FILE`
+    This option overrides the negotiated checksum & compress lists and always
+    negotiates a choice based on old-school md5/md4/zlib choices.  If you want
+    a more modern choice, use the `--checksum-choice` (`--cc`) and/or
+    `--compress-choice` (`--zc`) options.
+
+0.  `--only-write-batch=FILE`
 
     Works like `--write-batch`, except that no updates are made on the
     destination system when creating the batch.  This lets you transport the
@@ -3067,13 +3135,13 @@ your home directory (remove the '=' for that).
     into the batch file without having to flow over the wire to the receiver
     (when pulling, the sender is remote, and thus can't write the batch).
 
-0. `--read-batch=FILE`
+0.  `--read-batch=FILE`
 
     Apply all of the changes stored in FILE, a file previously generated by
     `--write-batch`.  If _FILE_ is `-`, the batch data will be read from
     standard input. See the "BATCH MODE" section for details.
 
-0. `--protocol=NUM`
+0.  `--protocol=NUM`
 
     Force an older protocol version to be used.  This is useful for creating a
     batch file that is compatible with an older version of rsync.  For
@@ -3083,7 +3151,7 @@ your home directory (remove the '=' for that).
     protocol version to be used in the batch file (assuming you can't upgrade
     the rsync on the reading system).
 
-0. `--iconv=CONVERT_SPEC`
+0.  `--iconv=CONVERT_SPEC`
 
     Rsync can convert filenames between character sets using this option.
     Using a CONVERT_SPEC of "." tells rsync to look up the default
@@ -3115,7 +3183,7 @@ your home directory (remove the '=' for that).
     free to specify just the local charset for a daemon transfer (e.g.
     `--iconv=utf8`).
 
-0. `--ipv4`, `-4` or `--ipv6`, `-6`
+0.  `--ipv4`, `-4` or `--ipv6`, `-6`
 
     Tells rsync to prefer IPv4/IPv6 when creating sockets or running ssh.  This
     affects sockets that rsync has direct control over, such as the outgoing
@@ -3131,7 +3199,7 @@ your home directory (remove the '=' for that).
     have no effect.  The `rsync -V` output will contain "`no IPv6`" if is the
     case.
 
-0. `--checksum-seed=NUM`
+0.  `--checksum-seed=NUM`
 
     Set the checksum seed to the integer NUM.  This 4 byte checksum seed is
     included in each block and MD4 file checksum calculation (the more modern
@@ -3146,7 +3214,7 @@ your home directory (remove the '=' for that).
 
 The options allowed when starting an rsync daemon are as follows:
 
-0. `--daemon`
+0.  `--daemon`
 
     This tells rsync that it is to run as a daemon.  The daemon you start
     running may be accessed using an rsync client using the `host::module` or
@@ -3158,7 +3226,7 @@ The options allowed when starting an rsync daemon are as follows:
     each connect made by a client and respond to requests accordingly.  See the
     **rsyncd.conf**(5) man page for more details.
 
-0. `--address`
+0.  `--address=ADDRESS`
 
     By default rsync will bind to the wildcard address when run as a daemon
     with the `--daemon` option.  The `--address` option allows you to specify a
@@ -3166,14 +3234,14 @@ The options allowed when starting an rsync daemon are as follows:
     possible in conjunction with the `--config` option.  See also the "address"
     global option in the rsyncd.conf manpage.
 
-0. `--bwlimit=RATE`
+0.  `--bwlimit=RATE`
 
     This option allows you to specify the maximum transfer rate for the data
     the daemon sends over the socket.  The client can still specify a smaller
     `--bwlimit` value, but no larger value will be allowed.  See the client
     version of this option (above) for some extra details.
 
-0. `--config=FILE`
+0.  `--config=FILE`
 
     This specifies an alternate config file than the default.  This is only
     relevant when `--daemon` is specified.  The default is /etc/rsyncd.conf
@@ -3181,7 +3249,7 @@ The options allowed when starting an rsync daemon are as follows:
     user is not the super-user; in that case the default is rsyncd.conf in the
     current directory (typically $HOME).
 
-0. `--dparam=OVERRIDE`, `-M`
+0.  `--dparam=OVERRIDE`, `-M`
 
     This option can be used to set a daemon-config parameter when starting up
     rsync in daemon mode.  It is equivalent to adding the parameter at the end
@@ -3191,7 +3259,7 @@ The options allowed when starting an rsync daemon are as follows:
 
     >     rsync --daemon -M pidfile=/path/rsync.pid
 
-0. `--no-detach`
+0.  `--no-detach`
 
     When running as a daemon, this option instructs rsync to not detach itself
     and become a background process.  This option is required when running as a
@@ -3200,37 +3268,37 @@ The options allowed when starting an rsync daemon are as follows:
     `--no-detach` is also recommended when rsync is run under a debugger.  This
     option has no effect if rsync is run from inetd or sshd.
 
-0. `--port=PORT`
+0.  `--port=PORT`
 
     This specifies an alternate TCP port number for the daemon to listen on
     rather than the default of 873.  See also the "port" global option in the
     rsyncd.conf manpage.
 
-0. `--log-file=FILE`
+0.  `--log-file=FILE`
 
     This option tells the rsync daemon to use the given log-file name instead
     of using the "`log file`" setting in the config file.
 
-0. `--log-file-format=FORMAT`
+0.  `--log-file-format=FORMAT`
 
     This option tells the rsync daemon to use the given FORMAT string instead
     of using the "`log format`" setting in the config file.  It also enables
     "`transfer logging`" unless the string is empty, in which case transfer
     logging is turned off.
 
-0. `--sockopts`
+0.  `--sockopts`
 
     This overrides the `socket options` setting in the rsyncd.conf file and has
     the same syntax.
 
-0. `--verbose`, `-v`
+0.  `--verbose`, `-v`
 
     This option increases the amount of information the daemon logs during its
     startup phase.  After the client connects, the daemon's verbosity level
     will be controlled by the options that the client used and the
     "`max verbosity`" setting in the module's config section.
 
-0. `--ipv4`, `-4` or `--ipv6`, `-6`
+0.  `--ipv4`, `-4` or `--ipv6`, `-6`
 
     Tells rsync to prefer IPv4/IPv6 when creating the incoming sockets that the
     rsync daemon will use to listen for connections.  One of these options may
@@ -3245,7 +3313,7 @@ The options allowed when starting an rsync daemon are as follows:
     have no effect.  The `rsync -V` output will contain "`no IPv6`" if is the
     case.
 
-0. `--help`, `-h`
+0.  `--help`, `-h`
 
     When specified after `--daemon`, print a short help page describing the
     options available for starting an rsync daemon.

rsync.h

@@ -1078,6 +1078,9 @@ typedef struct {
 #define CPRES_LZ4 3
 #define CPRES_ZSTD 4
 
+#define NSTR_CHECKSUM 0
+#define NSTR_COMPRESS 1
+
 struct name_num_item {
 	int num;
 	const char *name, *main_name;
@@ -1262,12 +1265,18 @@ extern int errno;
 /* handler for null strings in printf format */
 #define NS(s) ((s)?(s):"<NULL>")
 
+extern char *do_malloc;
+
 /* Convenient wrappers for malloc and realloc.  Use them. */
-#define new(type) ((type*)malloc(sizeof (type)))
-#define new0(type) ((type*)calloc(1, sizeof (type)))
-#define new_array(type, num) ((type*)_new_array((num), sizeof (type), 0))
-#define new_array0(type, num) ((type*)_new_array((num), sizeof (type), 1))
-#define realloc_array(ptr, type, num) ((type*)_realloc_array((ptr), sizeof(type), (num)))
+#define new(type) ((type*)my_alloc(do_malloc, sizeof (type), 1, __FILE__, __LINE__))
+#define new0(type) ((type*)my_alloc(NULL, sizeof (type), 1, __FILE__, __LINE__))
+#define realloc_buf(ptr, num) my_alloc((ptr), (num), 1, __FILE__, __LINE__)
+
+#define new_array(type, num) ((type*)my_alloc(do_malloc, (num), sizeof (type), __FILE__, __LINE__))
+#define new_array0(type, num) ((type*)my_alloc(NULL, (num), sizeof (type), __FILE__, __LINE__))
+#define realloc_array(ptr, type, num) ((type*)my_alloc((ptr), (num), sizeof (type), __FILE__, __LINE__))
+
+#define strdup(s) my_strdup(s, __FILE__, __LINE__)
 
 /* use magic gcc attributes to catch format errors */
  void rprintf(enum logcode , const char *, ...)

rsync3.txt

@@ -396,7 +396,7 @@ Conflict resolution:
     would be useful.
 
 
-Moved files: <http://rsync.samba.org/cgi-bin/rsync.fom?file=44>
+Moved files:
 
   - There's no trivial way to detect renamed files, especially if they
     move between directories.
@@ -457,13 +457,11 @@ Streaming:
 
 Related work:
 
-  - mirror.pl http://freshmeat.net/project/mirror/
+  - mirror.pl
 
   - ProFTPd
 
   - Apache
 
-  - http://freshmeat.net/search/?site=Freshmeat&q=mirror&section=projects
-
   - BitTorrent -- p2p mirroring
     http://bitconjurer.org/BitTorrent/

rsyncd.conf.5.md

@@ -905,15 +905,15 @@ the values of parameters.  See the GLOBAL PARAMETERS section for more details.
 
     - `--server`: Required for rsync to even work.
     - `--rsh`, `-e`: Required to convey compatibility flags to the server.
-    - `--log-format`: This is required to convey things like
-      `--itemize-changes` to a remote receiver. Is an older name for
-      `--out-format` that is still passed to the server for improved backward
-      compatibility and should not be confused with `--log-file-format`.
+    - `--out-format`: This is required to convey output behavior to a remote
+      receiver.  While rsync passes the older alias `--log-format` for
+      compatibility reasons, this options should not be confused with
+      `--log-file-format`.
     - `--sender`: Use "write only" parameter instead of refusing this.
     - `--dry-run`, `-n`: Who would want to disable this?
-    - `--protect-args`, `-n`: This actually makes transfers safer.
-    - `--from0`, `-0`: Make it easier to accept/refuse `--files-from` without
-      affecting this modifier.
+    - `--protect-args`, `-s`: This actually makes transfers safer.
+    - `--from0`, `-0`: Makes it easier to accept/refuse `--files-from` without
+      affecting this helpful modifier.
     - `--iconv`: This is auto-disabled based on "charset" parameter.
     - `--no-iconv`: Most transfers use this option.
     - `--checksum-seed`: Is a fairly rare, safe option.
@@ -957,7 +957,9 @@ the values of parameters.  See the GLOBAL PARAMETERS section for more details.
     request is known except for the module name.  This helper script can be
     used to setup a disk mount or decrypt some data into a module dir, but you
     may need to use `lock file` and `max connections` to avoid concurrency
-    issues.
+    issues.  If the client rsync specified the `--early-input=FILE` option, it
+    can send up to about 5K of data to the stdin of the early script.  The
+    stdin will otherwise be empty.
 
     Note that the `post-xfer exec` command is still run even if one of the
     other scripts returns an error code. The `pre-xfer exec` command will _not_
@@ -1103,7 +1105,7 @@ An example nginx proxy setup is as follows:
 >        listen [::]:874 ssl;
 >
 >        ssl_certificate /etc/letsencrypt/example.com/fullchain.pem;
->        ssl_certificate_key /etc/letsencrypt/example.com/privkey.pem
+>        ssl_certificate_key /etc/letsencrypt/example.com/privkey.pem;
 >
 >        proxy_pass localhost:873;
 >        proxy_protocol on; # Requires "proxy protocol = true"

sender.c

@@ -65,13 +65,10 @@ BOOL extra_flist_sending_enabled;
  **/
 static struct sum_struct *receive_sums(int f)
 {
-	struct sum_struct *s;
-	int32 i;
+	struct sum_struct *s = new(struct sum_struct);
 	int lull_mod = protocol_version >= 31 ? 0 : allowed_lull * 5;
 	OFF_T offset = 0;
-
-	if (!(s = new(struct sum_struct)))
-		out_of_memory("receive_sums");
+	int32 i;
 
 	read_sum_head(f, s);
 
@@ -92,8 +89,7 @@ static struct sum_struct *receive_sums(int f)
 	if (s->count == 0)
 		return(s);
 
-	if (!(s->sums = new_array(struct sum_buf, s->count)))
-		out_of_memory("receive_sums");
+	s->sums = new_array(struct sum_buf, s->count);
 
 	for (i = 0; i < s->count; i++) {
 		s->sums[i].sum1 = read_int(f);

socket.c

@@ -26,6 +26,7 @@
 
 #include "rsync.h"
 #include "itypes.h"
+#include "ifuncs.h"
 #ifdef HAVE_NETINET_IN_SYSTM_H
 #include <netinet/in_systm.h>
 #endif
@@ -248,8 +249,6 @@ int open_socket_out(char *host, int port, const char *bind_addr, int af_hint)
 
 	for (res = res0, addr_cnt = 0; res; res = res->ai_next, addr_cnt++) {}
 	errnos = new_array0(int, addr_cnt);
-	if (!errnos)
-		out_of_memory("open_socket_out");
 
 	s = -1;
 	/* Try to connect to all addresses for this machine until we get
@@ -354,8 +353,7 @@ int open_socket_out_wrapped(char *host, int port, const char *bind_addr, int af_
 				len += hlen;
 		}
 		f = prog;
-		if (!(prog = new_array(char, len)))
-			out_of_memory("open_socket_out_wrapped");
+		prog = new_array(char, len);
 		for (t = prog; *f; f++) {
 			if (*f == '%') {
 				switch (*++f) {
@@ -423,8 +421,6 @@ static int *open_socket_in(int type, int port, const char *bind_addr,
 
 	socks = new_array(int, maxs + 1);
 	errmsgs = new_array(char *, maxs);
-	if (!socks || !errmsgs)
-		out_of_memory("open_socket_in");
 
 	/* We may not be able to create the socket, if for example the
 	 * machine knows about IPv6 in the C library, but not in the
@@ -684,9 +680,6 @@ void set_socket_options(int fd, char *options)
 
 	options = strdup(options);
 
-	if (!options)
-		out_of_memory("set_socket_options");
-
 	for (tok = strtok(options, " \t,"); tok; tok = strtok(NULL," \t,")) {
 		int ret=0,i;
 		int value = 1;

support/rrsync

@@ -111,6 +111,7 @@ our %long_opt = (
   'list-only' => 0,
   'log-file' => $only eq 'r' ? -1 : 3,
   'log-format' => 1,
+  'max-alloc' => 1,
   'max-delete' => 1,
   'max-size' => 1,
   'min-size' => 1,

t_stub.c

@@ -33,6 +33,7 @@ int preserve_xattrs = 0;
 int preserve_perms = 0;
 int preserve_executability = 0;
 int open_noatime = 0;
+size_t max_alloc = 1024*1024*1024; /* max_alloc is needed when combined with util2.o */
 char *partial_dir;
 char *module_dir;
 filter_rule_list daemon_filter_list;

testsuite/daemon.test

@@ -91,12 +91,13 @@ drwxr-xr-x         DIR ####/##/## ##:##:## foo
 EOT
 diff $diffopt "$chkfile" "$outfile" || test_fail "test 3 failed"
 
-$RSYNC -rU localhost::test-from/f* \
-    | sed "$FILE_REPL" | sed "$DIR_REPL" | sed "$LS_REPL" \
-    | tee "$outfile"
-cat <<EOT >"$chkfile"
+if $RSYNC --version | grep "[, ] atimes" >/dev/null; then
+    $RSYNC -rU localhost::test-from/f* \
+	| sed "$FILE_REPL" | sed "$DIR_REPL" | sed "$LS_REPL" \
+	| tee "$outfile"
+    cat <<EOT >"$chkfile"
 drwxr-xr-x         DIR ####/##/## ##:##:##                     foo
 -rw-r--r--           4 ####/##/## ##:##:## ####/##/## ##:##:## foo/one
 EOT
-diff $diffopt "$chkfile" "$outfile" || test_fail "test 3 failed"
-
+    diff $diffopt "$chkfile" "$outfile" || test_fail "test 3 failed"
+fi

token.c

@@ -129,8 +129,7 @@ static void add_suffix(struct suffix_tree **prior, char ltr, const char *str)
 		if (node->letter > ltr)
 			break;
 	}
-	if (!(newnode = new(struct suffix_tree)))
-		out_of_memory("add_suffix");
+	newnode = new(struct suffix_tree);
 	newnode->sibling = node;
 	newnode->child = NULL;
 	newnode->letter = ltr;
@@ -147,8 +146,7 @@ static void add_nocompress_suffixes(const char *str)
 	char *buf, *t;
 	const char *f = str;
 
-	if (!(buf = new_array(char, strlen(f) + 1)))
-		out_of_memory("add_nocompress_suffixes");
+	buf = new_array(char, strlen(f) + 1);
 
 	while (*f) {
 		if (*f == '/') {
@@ -186,8 +184,7 @@ static void init_set_compression(void)
 	else
 		f = lp_dont_compress(module_id);
 
-	if (!(match_list = t = new_array(char, strlen(f) + 2)))
-		out_of_memory("set_compression");
+	match_list = t = new_array(char, strlen(f) + 2);
 
 	per_file_default_level = do_compression_level;
 
@@ -282,11 +279,8 @@ static int32 simple_recv_token(int f, char **data)
 	static char *buf;
 	int32 n;
 
-	if (!buf) {
+	if (!buf)
 		buf = new_array(char, CHUNK_SIZE);
-		if (!buf)
-			out_of_memory("simple_recv_token");
-	}
 
 	if (residue == 0) {
 		int32 i = read_int(f);
@@ -373,8 +367,7 @@ send_deflated_token(int f, int32 token, struct map_struct *buf, OFF_T offset, in
 				rprintf(FERROR, "compression init failed\n");
 				exit_cleanup(RERR_PROTOCOL);
 			}
-			if ((obuf = new_array(char, OBUF_SIZE)) == NULL)
-				out_of_memory("send_deflated_token");
+			obuf = new_array(char, OBUF_SIZE);
 			init_done = 1;
 		} else
 			deflateReset(&tx_strm);
@@ -518,9 +511,8 @@ static int32 recv_deflated_token(int f, char **data)
 					rprintf(FERROR, "inflate init failed\n");
 					exit_cleanup(RERR_PROTOCOL);
 				}
-				if (!(cbuf = new_array(char, MAX_DATA_COUNT))
-				 || !(dbuf = new_array(char, AVAIL_OUT_SIZE(CHUNK_SIZE))))
-					out_of_memory("recv_deflated_token");
+				cbuf = new_array(char, MAX_DATA_COUNT);
+				dbuf = new_array(char, AVAIL_OUT_SIZE(CHUNK_SIZE));
 				init_done = 1;
 			} else {
 				inflateReset(&rx_strm);
@@ -695,8 +687,6 @@ static void send_zstd_token(int f, int32 token, struct map_struct *buf, OFF_T of
 		}
 
 		obuf = new_array(char, OBUF_SIZE);
-		if (!obuf)
-			out_of_memory("send_deflated_token");
 
 		ZSTD_CCtx_setParameter(zstd_cctx, ZSTD_c_compressionLevel, do_compression_level);
 		zstd_out_buff.dst = obuf + 2;
@@ -806,8 +796,6 @@ static int32 recv_zstd_token(int f, char **data)
 		out_buffer_size = ZSTD_DStreamOutSize() * 2;
 		cbuf = new_array(char, MAX_DATA_COUNT);
 		dbuf = new_array(char, out_buffer_size);
-		if (!cbuf || !dbuf)
-			out_of_memory("recv_zstd_token");
 
 		zstd_in_buff.src = cbuf;
 		zstd_out_buff.dst = dbuf;
@@ -903,8 +891,7 @@ send_compressed_token(int f, int32 token, struct map_struct *buf, OFF_T offset,
 
 	if (last_token == -1) {
 		if (!init_done) {
-			if ((obuf = new_array(char, size)) == NULL)
-				out_of_memory("send_compressed_token");
+			obuf = new_array(char, size);
 			init_done = 1;
 		}
 		last_run_end = 0;
@@ -984,9 +971,8 @@ static int32 recv_compressed_token(int f, char **data)
 		switch (recv_state) {
 		case r_init:
 			if (!init_done) {
-				if (!(cbuf = new_array(char, MAX_DATA_COUNT))
-				 || !(dbuf = new_array(char, size)))
-					out_of_memory("recv_compressed_token");
+				cbuf = new_array(char, MAX_DATA_COUNT);
+				dbuf = new_array(char, size);
 				init_done = 1;
 			}
 			recv_state = r_idle;

uidlist.c

@@ -85,8 +85,6 @@ static struct idlist *add_to_list(struct idlist **root, id_t id, union name_or_i
 				  id_t id2, uint16 flags)
 {
 	struct idlist *node = new(struct idlist);
-	if (!node)
-		out_of_memory("add_to_list");
 	node->next = *root;
 	node->u = noiu;
 	node->id = id;
@@ -160,8 +158,6 @@ static int is_in_group(gid_t gid)
 		if ((ngroups = getgroups(0, NULL)) < 0)
 			ngroups = 0;
 		gidset = new_array(GETGROUPS_T, ngroups+1);
-		if (!gidset)
-			out_of_memory("is_in_group");
 		if (ngroups > 0)
 			ngroups = getgroups(ngroups, gidset);
 		/* The default gid might not be in the list on some systems. */
@@ -174,8 +170,6 @@ static int is_in_group(gid_t gid)
 		if (DEBUG_GTE(OWN, 2)) {
 			int pos;
 			char *gidbuf = new_array(char, ngroups*21+32);
-			if (!gidbuf)
-				out_of_memory("is_in_group");
 			pos = snprintf(gidbuf, 32, "process has %d gid%s: ", ngroups, ngroups == 1? "" : "s");
 			for (n = 0; n < ngroups; n++) {
 				pos += snprintf(gidbuf+pos, 21, " %d", (int)gidset[n]);
@@ -375,8 +369,6 @@ uid_t recv_user_name(int f, uid_t uid)
 	struct idlist *node;
 	int len = read_byte(f);
 	char *name = new_array(char, len+1);
-	if (!name)
-		out_of_memory("recv_user_name");
 	read_sbuf(f, name, len);
 	if (numeric_ids < 0) {
 		free(name);
@@ -391,8 +383,6 @@ gid_t recv_group_name(int f, gid_t gid, uint16 *flags_ptr)
 	struct idlist *node;
 	int len = read_byte(f);
 	char *name = new_array(char, len+1);
-	if (!name)
-		out_of_memory("recv_group_name");
 	read_sbuf(f, name, len);
 	if (numeric_ids < 0) {
 		free(name);

util.c

@@ -592,8 +592,7 @@ int lock_range(int fd, int offset, int len)
 }
 
 #define ENSURE_MEMSPACE(buf, type, sz, req) \
-	if ((req) > sz && !(buf = realloc_array(buf, type, sz = MAX(sz * 2, req)))) \
-		out_of_memory("glob_expand")
+	do { if ((req) > sz) buf = realloc_array(buf, type, sz = MAX(sz * 2, req)); } while(0)
 
 static inline void call_glob_match(const char *name, int len, int from_glob,
 				   char *arg, int abpos, int fbpos);
@@ -695,8 +694,7 @@ static inline void call_glob_match(const char *name, int len, int from_glob,
 		glob_match(arg, abpos, fbpos);
 	} else {
 		ENSURE_MEMSPACE(glob.argv, char *, glob.maxargs, glob.argc + 1);
-		if (!(glob.argv[glob.argc++] = strdup(glob.arg_buf)))
-			out_of_memory("glob_match");
+		glob.argv[glob.argc++] = strdup(glob.arg_buf);
 	}
 }
 
@@ -720,8 +718,6 @@ int glob_expand(const char *arg, char ***argv_p, int *argc_p, int *maxargs_p)
 		s = sanitize_path(NULL, arg, "", 0, SP_KEEP_DOT_DIRS);
 	else {
 		s = strdup(arg);
-		if (!s)
-			out_of_memory("glob_expand");
 		clean_fname(s, CFN_KEEP_DOT_DIRS | CFN_KEEP_TRAILING_SLASH | CFN_COLLAPSE_DOT_DOT_DIRS);
 	}
 
@@ -771,8 +767,7 @@ void glob_expand_module(char *base1, char *arg, char ***argv_p, int *argc_p, int
 		return;
 	}
 
-	if (!(arg = strdup(arg)))
-		out_of_memory("glob_expand_module");
+	arg = strdup(arg);
 
 	if (asprintf(&base," %s/", base1) < 0)
 		out_of_memory("glob_expand_module");
@@ -1017,11 +1012,10 @@ char *sanitize_path(char *dest, const char *p, const char *rootdir, int depth, i
 			depth = 0;
 			p++;
 		}
-		if (dest) {
-			if (rlen + plen + 1 >= MAXPATHLEN)
-				return NULL;
-		} else if (!(dest = new_array(char, MAX(rlen + plen + 1, 2))))
-			out_of_memory("sanitize_path");
+		if (!dest)
+			dest = new_array(char, MAX(rlen + plen + 1, 2));
+		else if (rlen + plen + 1 >= MAXPATHLEN)
+			return NULL;
 		if (rlen) { /* only true if p previously started with a slash */
 			memcpy(dest, rootdir, rlen);
 			if (rlen > 1) /* a rootdir of len 1 is "/", so this avoids a 2nd slash */
@@ -1155,13 +1149,10 @@ char *normalize_path(char *path, BOOL force_newbuf, unsigned int *len_ptr)
 			return NULL;
 		curr_dir[curr_dir_len] = '/';
 		memcpy(curr_dir + curr_dir_len + 1, path, len + 1);
-		if (!(path = strdup(curr_dir)))
-			out_of_memory("normalize_path");
+		path = strdup(curr_dir);
 		curr_dir[curr_dir_len] = '\0';
-	} else if (force_newbuf) {
-		if (!(path = strdup(path)))
-			out_of_memory("normalize_path");
-	}
+	} else if (force_newbuf)
+		path = strdup(path);
 
 	len = clean_fname(path, CFN_COLLAPSE_DOT_DOT_DIRS | CFN_DROP_TRAILING_DOT_DIR);
 
@@ -1519,8 +1510,7 @@ struct bitbag *bitbag_create(int max_ndx)
 	struct bitbag *bb = new(struct bitbag);
 	bb->slot_cnt = (max_ndx + BB_PER_SLOT_BITS - 1) / BB_PER_SLOT_BITS;
 
-	if (!(bb->bits = (uint32**)calloc(bb->slot_cnt, sizeof (uint32*))))
-		out_of_memory("bitbag_create");
+	bb->bits = (uint32**)calloc(bb->slot_cnt, sizeof (uint32*));
 
 	return bb;
 }
@@ -1601,8 +1591,7 @@ void flist_ndx_push(flist_ndx_list *lp, int ndx)
 {
 	struct flist_ndx_item *item;
 
-	if (!(item = new(struct flist_ndx_item)))
-		out_of_memory("flist_ndx_push");
+	item = new(struct flist_ndx_item);
 	item->next = NULL;
 	item->ndx = ndx;
 	if (lp->tail)
@@ -1654,8 +1643,7 @@ void *expand_item_list(item_list *lp, size_t item_size, const char *desc, int in
 			new_size = 1;
 		if (new_size <= lp->malloced)
 			overflow_exit("expand_item_list");
-		/* Using _realloc_array() lets us pass the size, not a type. */
-		new_ptr = _realloc_array(lp->items, item_size, new_size);
+		new_ptr = realloc_buf(lp->items, new_size * item_size);
 		if (DEBUG_GTE(FLIST, 3)) {
 			rprintf(FINFO, "[%s] expand %s to %s bytes, did%s move\n",
 				who_am_i(), desc, big_num(new_size * item_size),

util2.c

@@ -25,6 +25,10 @@
 #include "itypes.h"
 #include "inums.h"
 
+extern size_t max_alloc;
+
+char *do_malloc = "42";
+
 /**
  * Sleep for a specified number of milliseconds.
  *
@@ -67,22 +71,39 @@ int msleep(int t)
 	return True;
 }
 
-#define MALLOC_MAX 0x40000000
-
-void *_new_array(unsigned long num, unsigned int size, int use_calloc)
+/* Convert a num manually because the needed %lld precision is not a portable sprintf() escape. */
+char *num_to_byte_string(ssize_t num)
 {
-	if (num >= MALLOC_MAX/size)
-		return NULL;
-	return use_calloc ? calloc(num, size) : malloc(num * size);
+	char buf[128], *s = buf + sizeof buf - 1;
+
+	*s = '\0';
+	while (num) {
+		*--s = (char)(num % 10) + '0';
+		num /= 10;
+	}
+	return strdup(s);
 }
 
-void *_realloc_array(void *ptr, unsigned int size, size_t num)
+void *my_alloc(void *ptr, size_t num, size_t size, const char *file, int line)
 {
-	if (num >= MALLOC_MAX/size)
-		return NULL;
+	if (num >= max_alloc/size) {
+		if (!file)
+			return NULL;
+		rprintf(FERROR, "[%s] exceeded --max-alloc=%s setting (file=%s, line=%d)\n",
+			who_am_i(), num_to_byte_string(max_alloc), file, line);
+		exit_cleanup(RERR_MALLOC);
+	}
 	if (!ptr)
-		return malloc(size * num);
-	return realloc(ptr, size * num);
+		ptr = calloc(num, size);
+	else if (ptr == do_malloc)
+		ptr = malloc(num * size);
+	else
+		ptr = realloc(ptr, num * size);
+	if (!ptr && file) {
+		rprintf(FERROR, "[%s] out of memory (file=%s, line=%d)\n", who_am_i(), file, line);
+		exit_cleanup(RERR_MALLOC);
+	}
+	return ptr;
 }
 
 const char *sum_as_hex(int csum_type, const char *sum, int flist_csum)

xattrs.c

@@ -145,8 +145,6 @@ static ssize_t get_xattr_names(const char *fname)
 	if (!namebuf) {
 		namebuf_len = 1024;
 		namebuf = new_array(char, namebuf_len);
-		if (!namebuf)
-			out_of_memory("get_xattr_names");
 	}
 
 	while (1) {
@@ -174,8 +172,6 @@ static ssize_t get_xattr_names(const char *fname)
 			free(namebuf);
 		namebuf_len = list_len + 1024;
 		namebuf = new_array(char, namebuf_len);
-		if (!namebuf)
-			out_of_memory("get_xattr_names");
 	}
 
 	return list_len;
@@ -205,8 +201,7 @@ static char *get_xattr_data(const char *fname, const char *name, size_t *len_ptr
 		extra_len = 1; /* request non-zero amount of memory */
 	if (datum_len + extra_len < datum_len)
 		overflow_exit("get_xattr_data");
-	if (!(ptr = new_array(char, datum_len + extra_len)))
-		out_of_memory("get_xattr_data");
+	ptr = new_array(char, datum_len + extra_len);
 
 	if (datum_len) {
 		size_t len = sys_lgetxattr(fname, name, ptr, datum_len);
@@ -279,8 +274,7 @@ static int rsync_xal_get(const char *fname, item_list *xalp)
 			sum_update(ptr, datum_len);
 			free(ptr);
 
-			if (!(ptr = new_array(char, name_offset + name_len)))
-				out_of_memory("rsync_xal_get");
+			ptr = new_array(char, name_offset + name_len);
 			*ptr = XSTATE_ABBREV;
 			sum_end(ptr + 1);
 		} else
@@ -481,8 +475,6 @@ static int rsync_xal_store(item_list *xalp)
 		out_of_memory("rsync_xal_h hashtable_create()");
 
 	new_ref = new0(rsync_xa_list_ref);
-	if (new_ref == NULL)
-		out_of_memory("new0(rsync_xa_list_ref)");
 	new_ref->ndx = ndx;
 
 	node = hashtable_find(rsync_xal_h, new_list->key, new_ref);
@@ -759,8 +751,6 @@ int recv_xattr_request(struct file_struct *file, int f_in)
 		if (rxa->name_len + rxa->datum_len < rxa->name_len)
 			overflow_exit("recv_xattr_request");
 		rxa->datum = new_array(char, rxa->datum_len + rxa->name_len);
-		if (!rxa->datum)
-			out_of_memory("recv_xattr_request");
 		name = rxa->datum + rxa->datum_len;
 		memcpy(name, rxa->name, rxa->name_len);
 		rxa->name = name;
@@ -813,8 +803,6 @@ void receive_xattr(int f, struct file_struct *file)
 		 || (dget_len + extra_len + name_len < dget_len + extra_len))
 			overflow_exit("receive_xattr");
 		ptr = new_array(char, dget_len + extra_len + name_len);
-		if (!ptr)
-			out_of_memory("receive_xattr");
 		name = ptr + dget_len + extra_len;
 		read_buf(f, name, name_len);
 		if (name_len < 1 || name[name_len-1] != '\0') {