Preparing for release of 3.0.5pre1

Gets rid of a DNS delay waiting for a lookup failure.

.gitignore

@@ -3,10 +3,13 @@
 dummy
 ID
 Makefile
+Makefile.old
 configure.sh
+configure.sh.old
 config.cache
 config.h
 config.h.in
+config.h.in.old
 config.log
 config.status
 /proto.h
@@ -32,6 +35,7 @@ config.status
 /rounding.h
 /doc/rsync.pdf
 /doc/rsync.ps
+/support/savetransfer
 /testsuite/chown-fake.test
 /testsuite/devices-fake.test
 /patches

Makefile.in

@@ -126,11 +126,23 @@ conf:
 conf_stop: configure.sh config.h.in
 
 configure.sh config.h.in: configure.in aclocal.m4
+	@if test -f configure.sh; then cp -p configure.sh configure.sh.old; else touch configure.sh.old; fi
+	@if test -f config.h.in; then cp -p config.h.in config.h.in.old; else touch config.h.in.old; fi
 	autoconf -o configure.sh
 	autoheader && touch config.h.in
-	@echo 'Configure files changed -- perhaps run:'
-	@echo '  make reconfigure'
-	@exit 1
+	@if diff configure.sh configure.sh.old >/dev/null 2>&1; then \
+	    echo "configure.sh is unchanged."; \
+	    rm configure.sh.old; \
+	fi
+	@if diff config.h.in config.h.in.old >/dev/null 2>&1; then \
+	    echo "config.h.in is unchanged."; \
+	    rm config.h.in.old; \
+	fi
+	@if test -f configure.sh.old -o -f config.h.in.old; then \
+	    echo 'Configure files changed -- perhaps run:'; \
+	    echo '  make reconfigure'; \
+	    exit 1; \
+	fi
 
 reconfigure: configure.sh
 	./config.status --recheck
@@ -139,9 +151,15 @@ reconfigure: configure.sh
 make_stop: Makefile
 
 Makefile: Makefile.in config.status
+	@if test -f Makefile; then cp -p Makefile Makefile.old; else touch Makefile.old; fi
 	@./config.status
-	@echo "Makefile updated -- rerun your make command."
-	@exit 1
+	@if diff Makefile Makefile.old >/dev/null 2>&1; then \
+	    echo "Makefile is unchanged."; \
+	    rm Makefile.old; \
+	else \
+	    echo "Makefile updated -- rerun your make command."; \
+	    exit 1; \
+	fi
 
 proto: proto.h-tstamp
 
@@ -165,7 +183,7 @@ rsyncd.conf.5: rsyncd.conf.yo
 
 clean: cleantests
 	rm -f *~ $(OBJS) $(CHECK_PROGS) $(CHECK_OBJS) $(CHECK_SYMLINKS) \
-		rounding rounding.h
+		rounding rounding.h *.old
 
 cleantests:
 	rm -rf ./testtmp*

NEWS

@@ -1,89 +1,35 @@
-NEWS for rsync 3.0.3 (29 Jun 2008)
+NEWS for rsync 3.0.5 (UNRELEASED)
 Protocol: 30 (unchanged)
-Changes since 3.0.2:
+Changes since 3.0.4:
 
   BUG FIXES:
 
-    - Fixed a wildcard matching problem in the daemon when a module has
-      "use chroot" enabled.
+    - Initialize xattr data in a couple spots in the hlink code, which avoids a
+      crash when the xattr pointer's memory happens to start out non-zero.
+      Also fixed the itemizing of an alt-dest file's xattrs when hard-linking.
 
-    - Fixed a crash bug in the hard-link code.
+    - Don't send a bogus "-" option to an older server if there were no short
+      options specified.
 
-    - Fixed the sending of xattr directory information when the code finds a
-      --link-dest or --copy-dest directory with unchanged xattrs -- the
-      destination directory now gets these unchanged xattrs properly applied.
+    - Fixed skipping of unneeded updates in a batch file when incremental
+      recursion is active.  Added a test for this.  Made batch-mode handle
+      "redo" files properly (and without hanging).
 
-    - Fixed an xattr-sending glitch that could cause an "Internal abbrev"
-      error.
+    - Fix the %P logfile escape when the daemon logs from inside a chroot.
 
-    - Fixed the combination of --xattrs and --backup.
+    - Fixed the use of -s (--protect-args) when used with a remote source or
+      destination that had an empty path (e.g. "host:").  Also fixed a problem
+      when -s was used when accessing a daemon via a remote-shell.
 
-    - The generator no longer allows a '.' dir to be excluded by a daemon-
-      exclude rule.
+    - Got rid of an annoying delay when accessing a daemon via a remote-shell.
 
-    - Fixed deletion handling when copying a single, empty directory (with no
-      files) to a differently named, non-existent directory.
+    - Properly ignore (superfluous) source args on a --read-batch command.
 
-    - Fixed the conversion of spaces into dashes in the %M log escape.
-
-    - Fixed several places in the code that were not returning the right
-      errno when a function failed.
-
-    - Fixed the backing up of a device or special file into a backup dir.
-
-    - Moved the setting of the socket options prior to the connect().
-
-    - If rsync exits in the middle of a --progress output, it now outputs a
-      newline to help prevent the progress line from being overwritten.
-
-    - Fixed a problem with how a destination path with a trailing slash or
-      a trailing dot-dir was compared against the daemon excludes.
-
-    - Fixed the sending of large (size > 16GB) files when talking to an older
-      rsync (protocols < 30):  we now use a compatible block size limit.
-
-    - If a file's length is so huge that we overflow a checksum buffer count
-      (i.e. several hundred TB), warn the user and avoid sending an invalid
-      checksum struct over the wire.
-
-    - If a source arg is excluded, --relative no longer adds the excluded
-      arg's implied dirs to the transfer.  This fix also made the exclude
-      check happen in the better place in the sending code.
-
-    - Use the overflow_exit() function for overflows, not out_of_memory().
-
-    - Improved the code to better handle a system that has only 32-bit file
-      offsets.
+    - Improved the manpage's description of the '*' wildcard to remove the
+      confusing "non-empty" qualifier.
 
   ENHANCEMENTS:
 
-    - The rsyncd.conf manpage now consistently refers to the parameters in
-      the daemon config file as "parameters".
-
-    - The description of the --inplace option was improved.
-
-  EXTRAS:
-
-    - Added a new script in the support directory, deny-rsync, which allows
-      an admin to (temporarily) replace the rsync command with a script that
-      sends an error message to the remote client via the rsync protocol.
-
-  DEVELOPER RELATED:
-
-    - Fixed a testcase failure if the tests are run as root and made some
-      compatibility improvements.
-
-    - Improved the daemon tests, including checking module comments, the
-      listing of files, and the ensuring that daemon excludes can't affect
-      a dot-dir arg.
-
-    - Improved some build rules for those that build in a separate directory
-      from the source, including better install rules for the man pages, and
-      the fixing of a proto.h-tstamp rule that could make the binaries get
-      rebuild without cause.
-
-    - Improved the testsuite to work around a problem with some utilities
-      (e.g. cp -p & touch -r) rounding sub-second timestamps.
-
-    - Ensure that the early patches don't cause any generated-file hunks to
-      bleed-over into patches that follow.
+    - Made the support/atomic-rsync script able to perform a fully atomic
+      update of the copied hierarchy when the destination is setup using a
+      particular symlink idiom.

OLDNEWS

@@ -1,3 +1,160 @@
+NEWS for rsync 3.0.4 (6 Sep 2008)
+Protocol: 30 (unchanged)
+Changes since 3.0.3:
+
+  BUG FIXES:
+
+    - Fixed a bug in the hard-linking code where it would sometimes try to
+      allocate 0 bytes of memory (which fails on some OSes, such as AIX).
+
+    - Fixed the hard-linking of files from a device that has a device number
+      of 0 (which seems to be a common device number on NetBSD).
+
+    - Fixed the handling of a --partial-dir that cannot be created.  This
+      particularly impacts the --delay-updates option (since the files cannot
+      be delayed without a partial-dir), and was potentially destructive if
+      the --remove-source-files was also specified.
+
+    - Fixed a couple issues in the --fake-super handling of xattrs when the
+      destination files have root-level attributes (e.g. selinux values) that
+      a non-root copy can't affect.
+
+    - Improved the keep-alive check in the generator to fire consistently in
+      incremental-recursion mode when --timeout is enabled.
+
+    - The --iconv option now converts the content of a symlink too, instead
+      of leaving it in the wrong character-set (requires 3.0.4 on both sides
+      of the transfer).
+
+    - When using --iconv, if a filename fails to convert on the receiving side,
+      this no longer makes deletions in the root-dir of the transfer fail
+      silently (the user now gets a warning about deletions being disabled
+      due to IO error as long as --ignore-errors was not specified).
+
+    - When using --iconv, if a server-side receiver can't convert a filename,
+      the error message sent back to the client no longer mangles the name
+      with the wrong charset conversion.
+
+    - Fixed a potential alignment issue in the IRIX ACL code when allocating
+      the initial "struct acl" object.  Also, cast mallocs to avoid warnings.
+
+    - Changed some errors that were going to stdout to go to stderr.
+
+    - Made human_num() and human_dnum() able to output a negative number
+      (rather than outputting a cryptic string of punctuation).
+
+  ENHANCEMENTS:
+
+    - Rsync will avoid sending an -e option to the server if an older protocol
+      is requested (and thus the option would not be useful).  This lets the
+      user specify the --protocol=29 option to access an overly-restrictive
+      server that is rejecting the protocol-30 use of -e to the server.
+
+    - Improved the message output for an RERR_PARTIAL exit.
+
+  DEVELOPER RELATED:
+
+    - The Makefile will not halt for just a timestamp change on the Makefile
+      or the configure files, only for actual changes in content.
+
+    - Changed some commands in the testsuite's xattrs.test that called "rsync"
+      instead of "$RSYNC".
+
+    - Enhanced the release scripts to be able to handle a branch release and
+      to do even more consistency checks on the files.
+
+
+NEWS for rsync 3.0.3 (29 Jun 2008)
+Protocol: 30 (unchanged)
+Changes since 3.0.2:
+
+  BUG FIXES:
+
+    - Fixed a wildcard matching problem in the daemon when a module has
+      "use chroot" enabled.
+
+    - Fixed a crash bug in the hard-link code.
+
+    - Fixed the sending of xattr directory information when the code finds a
+      --link-dest or --copy-dest directory with unchanged xattrs -- the
+      destination directory now gets these unchanged xattrs properly applied.
+
+    - Fixed an xattr-sending glitch that could cause an "Internal abbrev"
+      error.
+
+    - Fixed the combination of --xattrs and --backup.
+
+    - The generator no longer allows a '.' dir to be excluded by a daemon-
+      exclude rule.
+
+    - Fixed deletion handling when copying a single, empty directory (with no
+      files) to a differently named, non-existent directory.
+
+    - Fixed the conversion of spaces into dashes in the %M log escape.
+
+    - Fixed several places in the code that were not returning the right
+      errno when a function failed.
+
+    - Fixed the backing up of a device or special file into a backup dir.
+
+    - Moved the setting of the socket options prior to the connect().
+
+    - If rsync exits in the middle of a --progress output, it now outputs a
+      newline to help prevent the progress line from being overwritten.
+
+    - Fixed a problem with how a destination path with a trailing slash or
+      a trailing dot-dir was compared against the daemon excludes.
+
+    - Fixed the sending of large (size > 16GB) files when talking to an older
+      rsync (protocols < 30):  we now use a compatible block size limit.
+
+    - If a file's length is so huge that we overflow a checksum buffer count
+      (i.e. several hundred TB), warn the user and avoid sending an invalid
+      checksum struct over the wire.
+
+    - If a source arg is excluded, --relative no longer adds the excluded
+      arg's implied dirs to the transfer.  This fix also made the exclude
+      check happen in the better place in the sending code.
+
+    - Use the overflow_exit() function for overflows, not out_of_memory().
+
+    - Improved the code to better handle a system that has only 32-bit file
+      offsets.
+
+  ENHANCEMENTS:
+
+    - The rsyncd.conf manpage now consistently refers to the parameters in
+      the daemon config file as "parameters".
+
+    - The description of the --inplace option was improved.
+
+  EXTRAS:
+
+    - Added a new script in the support directory, deny-rsync, which allows
+      an admin to (temporarily) replace the rsync command with a script that
+      sends an error message to the remote client via the rsync protocol.
+
+  DEVELOPER RELATED:
+
+    - Fixed a testcase failure if the tests are run as root and made some
+      compatibility improvements.
+
+    - Improved the daemon tests, including checking module comments, the
+      listing of files, and the ensuring that daemon excludes can't affect
+      a dot-dir arg.
+
+    - Improved some build rules for those that build in a separate directory
+      from the source, including better install rules for the man pages, and
+      the fixing of a proto.h-tstamp rule that could make the binaries get
+      rebuild without cause.
+
+    - Improved the testsuite to work around a problem with some utilities
+      (e.g. cp -p & touch -r) rounding sub-second timestamps.
+
+    - Ensure that the early patches don't cause any generated-file hunks to
+      bleed-over into patches that follow.
+
+
 NEWS for rsync 3.0.2 (8 Apr 2008)
 Protocol: 30 (unchanged)
 Changes since 3.0.1:
@@ -290,6 +447,11 @@ Changes since 2.6.9:
     - Fixed rsync's ability to remove files that are not writable by the file's
       owner when rsync is running as the same user.
 
+    - When transferring large files, the sender's hashtable of checksums is
+      kept at a more reasonable state of fullness (no more than 80% full) so
+      that the scanning of the hashtable will not bog down as the number of
+      blocks increases.
+
   ENHANCEMENTS:
 
     - A new incremental-recursion algorithm is now used when rsync is talking
@@ -2745,6 +2907,8 @@ Changes since 2.4.6:
 
 Partial Protocol History
 	RELEASE DATE	VER.	DATE OF COMMIT*	PROTOCOL
+	?? Oct 2008	3.0.5			30
+	06 Sep 2008	3.0.4			30
 	29 Jun 2008	3.0.3			30
 	08 Apr 2008	3.0.2			30
 	03 Apr 2008	3.0.1			30

clientname.c

@@ -108,6 +108,9 @@ char *client_name(int fd)
 		struct addrinfo hint, *answer;
 		int err;
 
+		if (strcmp(addr, "0.0.0.0") == 0)
+			return name_buf;
+
 		memset(&hint, 0, sizeof hint);
 
 #ifdef AI_NUMERICHOST

clientserver.c

@@ -75,6 +75,8 @@ struct chmod_mode_struct *daemon_chmod_modes;
 char *module_dir = NULL;
 unsigned int module_dirlen = 0;
 
+char *full_module_path;
+
 static int rl_nulls = 0;
 
 #ifdef HAVE_SIGACTION
@@ -395,10 +397,20 @@ static int read_arg_from_pipe(int fd, char *buf, int limit)
 	return bp - buf;
 }
 
+static int path_failure(int f_out, const char *dir, BOOL was_chdir)
+{
+	if (was_chdir)
+		rsyserr(FLOG, errno, "chdir %s failed\n", dir);
+	else
+		rprintf(FLOG, "normalize_path(%s) failed\n", dir);
+	io_printf(f_out, "@ERROR: chdir failed\n");
+	return -1;
+}
+
 static int rsync_module(int f_in, int f_out, int i, char *addr, char *host)
 {
 	int argc;
-	char **argv, **orig_argv, **orig_early_argv, *chroot_path = NULL;
+	char **argv, **orig_argv, **orig_early_argv, *module_chdir;
 	char line[BIGPATHBUFLEN];
 	uid_t uid = (uid_t)-2;  /* canonically "nobody" */
 	gid_t gid = (gid_t)-2;
@@ -501,28 +513,28 @@ static int rsync_module(int f_in, int f_out, int i, char *addr, char *host)
 	module_dir = lp_path(i);
 	if (use_chroot) {
 		if ((p = strstr(module_dir, "/./")) != NULL) {
-			*p = '\0';
-			p += 2;
-		} else if ((p = strdup("/")) == NULL) /* MEMORY LEAK */
-			out_of_memory("rsync_module");
+			*p = '\0'; /* Temporary... */
+			if (!(module_chdir = normalize_path(module_dir, True, NULL)))
+				return path_failure(f_out, module_dir, False);
+			*p = '/';
+			if (!(p = normalize_path(p + 2, True, &module_dirlen)))
+				return path_failure(f_out, strstr(module_dir, "/./"), False);
+			if (!(full_module_path = normalize_path(module_dir, False, NULL)))
+				full_module_path = module_dir;
+			module_dir = p;
+		} else {
+			if (!(module_chdir = normalize_path(module_dir, False, NULL)))
+				return path_failure(f_out, module_dir, False);
+			full_module_path = module_chdir;
+			module_dir = "/";
+			module_dirlen = 1;
+		}
+	} else {
+		if (!(module_chdir = normalize_path(module_dir, False, &module_dirlen)))
+			return path_failure(f_out, module_dir, False);
+		full_module_path = module_dir = module_chdir;
 	}
 
-	/* We do a change_dir() that doesn't actually call chdir()
-	 * just to make a relative path absolute. */
-	strlcpy(line, curr_dir, sizeof line);
-	if (!change_dir(module_dir, CD_SKIP_CHDIR))
-		goto chdir_failed;
-	if (strcmp(curr_dir, module_dir) != 0
-	 && (module_dir = strdup(curr_dir)) == NULL)
-		out_of_memory("rsync_module");
-	change_dir(line, CD_SKIP_CHDIR); /* Restore curr_dir. */
-
-	if (use_chroot) {
-		chroot_path = module_dir;
-		module_dir = p; /* p is "/" or our inside-chroot path */
-	}
-	module_dirlen = clean_fname(module_dir, CFN_COLLAPSE_DOT_DOT_DIRS | CFN_DROP_TRAILING_DOT_DIR);
-
 	if (module_dirlen == 1) {
 		module_dirlen = 0;
 		set_filter_dir("/", 1);
@@ -557,16 +569,8 @@ static int rsync_module(int f_in, int f_out, int i, char *addr, char *host)
 		char *modname, *modpath, *hostaddr, *hostname, *username;
 		int status;
 
-		if (!use_chroot)
-			p = module_dir;
-		else if (module_dirlen) {
-			pathjoin(line, sizeof line, chroot_path, module_dir+1);
-			p = line;
-		} else
-			p = chroot_path;
-
 		if (asprintf(&modname, "RSYNC_MODULE_NAME=%s", name) < 0
-		 || asprintf(&modpath, "RSYNC_MODULE_PATH=%s", p) < 0
+		 || asprintf(&modpath, "RSYNC_MODULE_PATH=%s", full_module_path) < 0
 		 || asprintf(&hostaddr, "RSYNC_HOST_ADDR=%s", addr) < 0
 		 || asprintf(&hostname, "RSYNC_HOST_NAME=%s", host) < 0
 		 || asprintf(&username, "RSYNC_USER_NAME=%s", auth_user) < 0)
@@ -666,25 +670,19 @@ static int rsync_module(int f_in, int f_out, int i, char *addr, char *host)
 		 * a warning, unless a "require chroot" flag is set,
 		 * in which case we fail.
 		 */
-		if (chroot(chroot_path)) {
-			rsyserr(FLOG, errno, "chroot %s failed", chroot_path);
+		if (chroot(module_chdir)) {
+			rsyserr(FLOG, errno, "chroot %s failed", module_chdir);
 			io_printf(f_out, "@ERROR: chroot failed\n");
 			return -1;
 		}
-		if (!change_dir(module_dir, CD_NORMAL))
-			goto chdir_failed;
-		if (module_dirlen)
-			sanitize_paths = 1;
-	} else {
-		if (!change_dir(module_dir, CD_NORMAL)) {
-		  chdir_failed:
-			rsyserr(FLOG, errno, "chdir %s failed\n", module_dir);
-			io_printf(f_out, "@ERROR: chdir failed\n");
-			return -1;
-		}
-		sanitize_paths = 1;
+		module_chdir = module_dir;
 	}
 
+	if (!change_dir(module_chdir, CD_NORMAL))
+		return path_failure(f_out, module_chdir, True);
+	if (module_dirlen || !use_chroot)
+		sanitize_paths = 1;
+
 	if ((munge_symlinks = lp_munge_symlinks(i)) < 0)
 		munge_symlinks = !use_chroot || module_dirlen;
 	if (munge_symlinks) {

compat.c

@@ -57,12 +57,14 @@ extern struct filter_list_struct filter_list;
 extern int need_unsorted_flist;
 #ifdef ICONV_OPTION
 extern iconv_t ic_send, ic_recv;
+extern char *iconv_opt;
 #endif
 
 /* These index values are for the file-list's extra-attribute array. */
 int uid_ndx, gid_ndx, acls_ndx, xattrs_ndx, unsort_ndx;
 
 int receiver_symlink_times = 0; /* receiver can set the time on a symlink */
+int sender_symlink_iconv = 0;	/* sender should convert symlink content */
 
 #ifdef ICONV_OPTION
 int filesfrom_convert = 0;
@@ -70,6 +72,7 @@ int filesfrom_convert = 0;
 
 #define CF_INC_RECURSE	 (1<<0)
 #define CF_SYMLINK_TIMES (1<<1)
+#define CF_SYMLINK_ICONV (1<<2)
 
 static const char *client_info;
 
@@ -248,6 +251,9 @@ void setup_protocol(int f_out,int f_in)
 			compat_flags = allow_inc_recurse ? CF_INC_RECURSE : 0;
 #if defined HAVE_LUTIMES && defined HAVE_UTIMES
 			compat_flags |= CF_SYMLINK_TIMES;
+#endif
+#ifdef ICONV_OPTION
+			compat_flags |= CF_SYMLINK_ICONV;
 #endif
 			write_byte(f_out, compat_flags);
 		} else
@@ -262,6 +268,11 @@ void setup_protocol(int f_out,int f_in)
 #if defined HAVE_LUTIMES && defined HAVE_UTIMES
 		else
 			receiver_symlink_times = 1;
+#endif
+#ifdef ICONV_OPTION
+		sender_symlink_iconv = iconv_opt && (am_server
+		    ? strchr(client_info, 's') != NULL
+		    : !!(compat_flags & CF_SYMLINK_ICONV));
 #endif
 		if (inc_recurse && !allow_inc_recurse) {
 			/* This should only be able to happen in a batch. */

configure.in

@@ -5,7 +5,7 @@ AC_CONFIG_SRCDIR([byteorder.h])
 AC_CONFIG_HEADER(config.h)
 AC_PREREQ(2.59)
 
-RSYNC_VERSION=3.0.3
+RSYNC_VERSION=3.0.5pre1
 AC_SUBST(RSYNC_VERSION)
 AC_MSG_NOTICE([Configuring rsync $RSYNC_VERSION])
 

exclude.c

@@ -476,7 +476,7 @@ void *push_local_filters(const char *dir, unsigned int dirlen)
 					  XFLG_ANCHORED2ABS);
 		} else {
 			io_error |= IOERR_GENERAL;
-			rprintf(FINFO,
+			rprintf(FERROR,
 			    "cannot add local filter rules in long-named directory: %s\n",
 			    full_fname(dirbuf));
 		}

flist.c

@@ -67,6 +67,7 @@ extern int protocol_version;
 extern int sanitize_paths;
 extern int munge_symlinks;
 extern int need_unsorted_flist;
+extern int sender_symlink_iconv;
 extern int unsort_ndx;
 extern struct stats stats;
 extern char *filesfrom_host;
@@ -375,7 +376,7 @@ int change_pathname(struct file_struct *file, const char *dir, int dirlen)
 	if (!change_dir(dir, CD_NORMAL)) {
 	  chdir_error:
 		io_error |= IOERR_GENERAL;
-		rsyserr(FERROR, errno, "change_dir %s failed", full_fname(dir));
+		rsyserr(FERROR_XFER, errno, "change_dir %s failed", full_fname(dir));
 		if (dir != orig_dir)
 			change_dir(orig_dir, CD_NORMAL);
 		pathname = NULL;
@@ -386,7 +387,11 @@ int change_pathname(struct file_struct *file, const char *dir, int dirlen)
 	return 1;
 }
 
-static void send_file_entry(int f, const char *fname, struct file_struct *file, int ndx, int first_ndx)
+static void send_file_entry(int f, const char *fname, struct file_struct *file,
+#ifdef SUPPORT_LINKS
+			    const char *symlink_name, int symlink_len,
+#endif
+			    int ndx, int first_ndx)
 {
 	static time_t modtime;
 	static mode_t mode;
@@ -575,11 +580,9 @@ static void send_file_entry(int f, const char *fname, struct file_struct *file,
 	}
 
 #ifdef SUPPORT_LINKS
-	if (preserve_links && S_ISLNK(mode)) {
-		const char *sl = F_SYMLINK(file);
-		int len = strlen(sl);
-		write_varint30(f, len);
-		write_buf(f, sl, len);
+	if (symlink_len) {
+		write_varint30(f, symlink_len);
+		write_buf(f, symlink_name, symlink_len);
 	}
 #endif
 
@@ -675,12 +678,12 @@ static struct file_struct *recv_file_entry(struct file_list *flist,
 
 		if (iconvbufs(ic_recv, &inbuf, &outbuf, 0) < 0) {
 			io_error |= IOERR_GENERAL;
-			rprintf(FINFO,
+			rprintf(FERROR_UTF8,
 			    "[%s] cannot convert filename: %s (%s)\n",
 			    who_am_i(), lastname, strerror(errno));
 			outbuf.len = 0;
 		}
-		outbuf.buf[outbuf.len] = '\0';
+		thisname[outbuf.len] = '\0';
 	}
 #endif
 
@@ -811,6 +814,13 @@ static struct file_struct *recv_file_entry(struct file_list *flist,
 				linkname_len - 1);
 			overflow_exit("recv_file_entry");
 		}
+#ifdef ICONV_OPTION
+		/* We don't know how much extra room we need to convert
+		 * the as-yet-unread symlink data, so let's hope that a
+		 * double-size buffer is plenty. */
+		if (sender_symlink_iconv)
+			linkname_len *= 2;
+#endif
 		if (munge_symlinks)
 			linkname_len += SYMLINK_PREFIX_LEN;
 	}
@@ -945,14 +955,40 @@ static struct file_struct *recv_file_entry(struct file_list *flist,
 		if (first_hlink_ndx >= flist->ndx_start) {
 			struct file_struct *first = flist->files[first_hlink_ndx - flist->ndx_start];
 			memcpy(bp, F_SYMLINK(first), linkname_len);
-		} else if (munge_symlinks) {
-			strlcpy(bp, SYMLINK_PREFIX, linkname_len);
-			bp += SYMLINK_PREFIX_LEN;
-			linkname_len -= SYMLINK_PREFIX_LEN;
-			read_sbuf(f, bp, linkname_len - 1);
 		} else {
-			read_sbuf(f, bp, linkname_len - 1);
-			if (sanitize_paths)
+			if (munge_symlinks) {
+				strlcpy(bp, SYMLINK_PREFIX, linkname_len);
+				bp += SYMLINK_PREFIX_LEN;
+				linkname_len -= SYMLINK_PREFIX_LEN;
+			}
+#ifdef ICONV_OPTION
+			if (sender_symlink_iconv) {
+				xbuf outbuf, inbuf;
+
+				alloc_len = linkname_len;
+				linkname_len /= 2;
+
+				/* Read the symlink data into the end of our double-sized
+				 * buffer and then convert it into the right spot. */
+				INIT_XBUF(inbuf, bp + alloc_len - linkname_len,
+					  linkname_len - 1, (size_t)-1);
+				read_sbuf(f, inbuf.buf, inbuf.len);
+				INIT_XBUF(outbuf, bp, 0, alloc_len);
+
+				if (iconvbufs(ic_recv, &inbuf, &outbuf, 0) < 0) {
+					io_error |= IOERR_GENERAL;
+					rprintf(FERROR_XFER,
+					    "[%s] cannot convert symlink data for: %s (%s)\n",
+					    who_am_i(), full_fname(thisname), strerror(errno));
+					bp = (char*)file->basename;
+					*bp++ = '\0';
+					outbuf.len = 0;
+				}
+				bp[outbuf.len] = '\0';
+			} else
+#endif
+				read_sbuf(f, bp, linkname_len - 1);
+			if (sanitize_paths && !munge_symlinks && *bp)
 				sanitize_path(bp, bp, "", lastdir_depth, SP_DEFAULT);
 		}
 	}
@@ -1045,7 +1081,7 @@ struct file_struct *make_file(const char *fname, struct file_list *flist,
 
 	if (strlcpy(thisname, fname, sizeof thisname) >= sizeof thisname) {
 		io_error |= IOERR_GENERAL;
-		rprintf(FINFO, "skipping overly long name: %s\n", fname);
+		rprintf(FERROR_XFER, "skipping overly long name: %s\n", fname);
 		return NULL;
 	}
 	clean_fname(thisname, 0);
@@ -1090,7 +1126,7 @@ struct file_struct *make_file(const char *fname, struct file_list *flist,
 			}
 		} else {
 			io_error |= IOERR_GENERAL;
-			rsyserr(FERROR_XFER, save_errno, "readlink %s failed",
+			rsyserr(FERROR_XFER, save_errno, "readlink_stat(%s) failed",
 				full_fname(thisname));
 		}
 		return NULL;
@@ -1213,8 +1249,8 @@ struct file_struct *make_file(const char *fname, struct file_list *flist,
 		if (protocol_version >= 28
 		 ? (!S_ISDIR(st.st_mode) && st.st_nlink > 1)
 		 : S_ISREG(st.st_mode)) {
-			tmp_dev = st.st_dev;
-			tmp_ino = st.st_ino;
+			tmp_dev = (int64)st.st_dev + 1;
+			tmp_ino = (int64)st.st_ino;
 		} else
 			tmp_dev = 0;
 	}
@@ -1310,10 +1346,27 @@ static struct file_struct *send_file_name(int f, struct file_list *flist,
 
 	if (f >= 0) {
 		char fbuf[MAXPATHLEN];
+#ifdef SUPPORT_LINKS
+		const char *symlink_name;
+		int symlink_len;
+#ifdef ICONV_OPTION
+		char symlink_buf[MAXPATHLEN];
+#endif
+#endif
 #if defined SUPPORT_ACLS || defined SUPPORT_XATTRS
 		stat_x sx;
 #endif
 
+#ifdef SUPPORT_LINKS
+		if (preserve_links && S_ISLNK(file->mode)) {
+			symlink_name = F_SYMLINK(file);
+			symlink_len = strlen(symlink_name);
+		} else {
+			symlink_name = NULL;
+			symlink_len = 0;
+		}
+#endif
+
 #ifdef ICONV_OPTION
 		if (ic_send != (iconv_t)-1) {
 			xbuf outbuf, inbuf;
@@ -1326,19 +1379,38 @@ static struct file_struct *send_file_name(int f, struct file_list *flist,
 				if (iconvbufs(ic_send, &inbuf, &outbuf, 0) < 0)
 					goto convert_error;
 				outbuf.size += 2;
-				outbuf.buf[outbuf.len++] = '/';
+				fbuf[outbuf.len++] = '/';
 			}
 
 			INIT_XBUF_STRLEN(inbuf, (char*)file->basename);
 			if (iconvbufs(ic_send, &inbuf, &outbuf, 0) < 0) {
 			  convert_error:
 				io_error |= IOERR_GENERAL;
-				rprintf(FINFO,
+				rprintf(FERROR_XFER,
 				    "[%s] cannot convert filename: %s (%s)\n",
 				    who_am_i(), f_name(file, fbuf), strerror(errno));
 				return NULL;
 			}
-			outbuf.buf[outbuf.len] = '\0';
+			fbuf[outbuf.len] = '\0';
+
+#ifdef SUPPORT_LINKS
+			if (symlink_len && sender_symlink_iconv) {
+				INIT_XBUF(inbuf, (char*)symlink_name, symlink_len, (size_t)-1);
+				INIT_CONST_XBUF(outbuf, symlink_buf);
+				if (iconvbufs(ic_send, &inbuf, &outbuf, 0) < 0) {
+					io_error |= IOERR_GENERAL;
+					f_name(file, fbuf);
+					rprintf(FERROR_XFER,
+					    "[%s] cannot convert symlink data for: %s (%s)\n",
+					    who_am_i(), full_fname(fbuf), strerror(errno));
+					return NULL;
+				}
+				symlink_buf[outbuf.len] = '\0';
+
+				symlink_name = symlink_buf;
+				symlink_len = outbuf.len;
+			}
+#endif
 		} else
 #endif
 			f_name(file, fbuf);
@@ -1363,7 +1435,11 @@ static struct file_struct *send_file_name(int f, struct file_list *flist,
 		}
 #endif
 
-		send_file_entry(f, fbuf, file, flist->used, flist->ndx_start);
+		send_file_entry(f, fbuf, file,
+#ifdef SUPPORT_LINKS
+				symlink_name, symlink_len,
+#endif
+				flist->used, flist->ndx_start);
 
 #ifdef SUPPORT_ACLS
 		if (preserve_acls && !S_ISLNK(file->mode)) {
@@ -1558,14 +1634,14 @@ static void send_directory(int f, struct file_list *flist, char *fbuf, int len,
 			continue;
 		if (strlcpy(p, dname, remainder) >= remainder) {
 			io_error |= IOERR_GENERAL;
-			rprintf(FINFO,
+			rprintf(FERROR_XFER,
 				"cannot send long-named file %s\n",
 				full_fname(fbuf));
 			continue;
 		}
 		if (dname[0] == '\0') {
 			io_error |= IOERR_GENERAL;
-			rprintf(FINFO,
+			rprintf(FERROR_XFER,
 				"cannot send file with empty name in %s\n",
 				full_fname(fbuf));
 			continue;
@@ -2164,7 +2240,7 @@ struct file_list *send_file_list(int f, int argc, char *argv[])
 	if (inc_recurse) {
 		send_dir_depth = 1;
 		add_dirs_to_tree(-1, flist, dir_count);
-		if (!file_total || strcmp(flist->sorted[0]->basename, ".") != 0) 
+		if (!file_total || strcmp(flist->sorted[flist->low]->basename, ".") != 0)
 			flist->parent_ndx = -1;
 		flist_done_allocating(flist);
 		if (send_dir_ndx < 0) {
@@ -2232,8 +2308,8 @@ struct file_list *recv_file_list(int f)
 		maybe_emit_filelist_progress(flist->used);
 
 		if (verbose > 2) {
-			rprintf(FINFO, "recv_file_name(%s)\n",
-				f_name(file, NULL));
+			char *name = f_name(file, NULL);
+			rprintf(FINFO, "recv_file_name(%s)\n", NS(name));
 		}
 	}
 	file_total += flist->used;
@@ -2288,7 +2364,7 @@ struct file_list *recv_file_list(int f)
 		else
 			io_error |= read_int(f);
 	} else if (inc_recurse && flist->ndx_start == 1) {
-		if (!file_total || strcmp(flist->sorted[0]->basename, ".") != 0) 
+		if (!file_total || strcmp(flist->sorted[flist->low]->basename, ".") != 0)
 			flist->parent_ndx = -1;
 	}
 

generator.c

@@ -108,7 +108,7 @@ static int deletion_count = 0; /* used to implement --max-delete */
 static int deldelay_size = 0, deldelay_cnt = 0;
 static char *deldelay_buf = NULL;
 static int deldelay_fd = -1;
-static int lull_mod;
+static int loopchk_limit;
 static int dir_tweaking;
 static int symlink_timeset_failed_flags;
 static int need_retouch_dir_times;
@@ -2058,10 +2058,13 @@ static void touch_up_dirs(struct file_list *flist, int ndx)
 			 && cmp_time(st.st_mtime, file->modtime) != 0)
 				set_modtime(fname, file->modtime, file->mode);
 		}
-		if (allowed_lull && !(counter % lull_mod))
-			maybe_send_keepalive();
-		else if (!(counter & 0xFF))
-			maybe_flush_socket(0);
+		if (counter >= loopchk_limit) {
+			if (allowed_lull)
+				maybe_send_keepalive();
+			else
+				maybe_flush_socket(0);
+			counter = 0;
+		}
 	}
 }
 
@@ -2075,8 +2078,7 @@ void check_for_finished_files(int itemizing, enum logcode code, int check_redo)
 	while (1) {
 #ifdef SUPPORT_HARD_LINKS
 		if (preserve_hard_links && (ndx = get_hlink_num()) != -1) {
-			flist = flist_for_ndx(ndx);
-			assert(flist != NULL);
+			flist = flist_for_ndx(ndx, "check_for_finished_files.1");
 			file = flist->files[ndx - flist->ndx_start];
 			assert(file->flags & FLAG_HLINKED);
 			finish_hard_link(file, f_name(file, fbuf), ndx, NULL, itemizing, code, -1);
@@ -2099,7 +2101,7 @@ void check_for_finished_files(int itemizing, enum logcode code, int check_redo)
 			ignore_times++;
 
 			flist = cur_flist;
-			cur_flist = flist_for_ndx(ndx);
+			cur_flist = flist_for_ndx(ndx, "check_for_finished_files.2");
 
 			file = cur_flist->files[ndx - cur_flist->ndx_start];
 			if (solo_file)
@@ -2148,7 +2150,7 @@ void check_for_finished_files(int itemizing, enum logcode code, int check_redo)
 
 void generate_files(int f_out, const char *local_name)
 {
-	int i, ndx;
+	int i, ndx, next_loopchk = 0;
 	char fbuf[MAXPATHLEN];
 	int itemizing;
 	enum logcode code;
@@ -2174,7 +2176,7 @@ void generate_files(int f_out, const char *local_name)
 	solo_file = local_name;
 	dir_tweaking = !(list_only || solo_file || dry_run);
 	need_retouch_dir_times = preserve_times > 1;
-	lull_mod = allowed_lull * 5;
+	loopchk_limit = allowed_lull ? allowed_lull * 5 : 200;
 	symlink_timeset_failed_flags = ITEM_REPORT_TIME
 	    | (protocol_version >= 30 || !am_server ? ITEM_REPORT_TIMEFAIL : 0);
 	implied_dirs_are_missing = relative_paths && !implied_dirs && protocol_version < 30;
@@ -2258,10 +2260,13 @@ void generate_files(int f_out, const char *local_name)
 
 			check_for_finished_files(itemizing, code, 0);
 
-			if (allowed_lull && !(i % lull_mod))
-				maybe_send_keepalive();
-			else if (!(i & 0xFF))
-				maybe_flush_socket(0);
+			if (i + cur_flist->ndx_start >= next_loopchk) {
+				if (allowed_lull)
+					maybe_send_keepalive();
+				else
+					maybe_flush_socket(0);
+				next_loopchk += loopchk_limit;
+			}
 		}
 
 		if (!inc_recurse) {
@@ -2277,6 +2282,9 @@ void generate_files(int f_out, const char *local_name)
 		}
 	} while ((cur_flist = cur_flist->next) != NULL);
 
+	if (read_batch && inc_recurse)
+		write_ndx(f_out, NDX_DONE);
+
 	if (delete_during)
 		delete_in_dir(NULL, NULL, &dev_zero);
 	phase++;
@@ -2329,7 +2337,7 @@ void generate_files(int f_out, const char *local_name)
 		touch_up_dirs(dir_flist, -1);
 
 	if (max_delete >= 0 && deletion_count > max_delete) {
-		rprintf(FINFO,
+		rprintf(FWARNING,
 			"Deletions stopped due to --max-delete limit (%d skipped)\n",
 			deletion_count - max_delete);
 		io_error |= IOERR_DEL_LIMIT;

hlink.c

@@ -30,6 +30,7 @@ extern int inc_recurse;
 extern int do_xfers;
 extern int link_dest;
 extern int preserve_acls;
+extern int preserve_xattrs;
 extern int make_backups;
 extern int protocol_version;
 extern int remove_source_files;
@@ -37,7 +38,7 @@ extern int stdout_format_has_i;
 extern int maybe_ATTRS_REPORT;
 extern int unsort_ndx;
 extern char *basis_dir[];
-extern struct file_list *cur_flist, *first_flist;
+extern struct file_list *cur_flist;
 
 #ifdef SUPPORT_HARD_LINKS
 
@@ -119,14 +120,15 @@ static void match_gnums(int32 *ndx_list, int ndx_count)
 		if (inc_recurse) {
 			node = hashtable_find(prior_hlinks, gnum, 1);
 			if (!node->data) {
-				node->data = new_array0(char, 5);
+				if (!(node->data = new_array0(char, 5)))
+					out_of_memory("match_gnums");
 				assert(gnum >= hlink_flist->ndx_start);
 				file->flags |= FLAG_HLINK_FIRST;
 				prev = -1;
 			} else if (CVAL(node->data, 0) == 0) {
 				struct file_list *flist;
 				prev = IVAL(node->data, 1);
-				flist = flist_for_ndx(prev);
+				flist = flist_for_ndx(prev, NULL);
 				if (flist)
 					flist->files[prev - flist->ndx_start]->flags &= ~FLAG_HLINK_LAST;
 				else {
@@ -179,7 +181,7 @@ static void match_gnums(int32 *ndx_list, int ndx_count)
  * to first when we're done. */
 void match_hard_links(struct file_list *flist)
 {
-	if (!list_only) {
+	if (!list_only && flist->used) {
 		int i, ndx_count = 0;
 		int32 *ndx_list;
 
@@ -255,7 +257,7 @@ static char *check_prior(struct file_struct *file, int gnum,
 	while (1) {
 		struct file_list *flist;
 		if (prev_ndx < 0
-		 || (flist = flist_for_ndx(prev_ndx)) == NULL)
+		 || (flist = flist_for_ndx(prev_ndx, NULL)) == NULL)
 			break;
 		fp = flist->files[prev_ndx - flist->ndx_start];
 		if (!(fp->flags & FLAG_SKIP_HLINK)) {
@@ -366,6 +368,9 @@ int hard_link_check(struct file_struct *file, int ndx, const char *fname,
 		int j = 0;
 #ifdef SUPPORT_ACLS
 		alt_sx.acc_acl = alt_sx.def_acl = NULL;
+#endif
+#ifdef SUPPORT_XATTRS
+		alt_sx.xattr = NULL;
 #endif
 		do {
 			pathjoin(cmpbuf, MAXPATHLEN, basis_dir[j], fname);
@@ -395,19 +400,37 @@ int hard_link_check(struct file_struct *file, int ndx, const char *fname,
 			sxp->st = alt_sx.st;
 #ifdef SUPPORT_ACLS
 			if (preserve_acls && !S_ISLNK(file->mode)) {
-				if (!ACL_READY(*sxp))
+				free_acl(sxp);
+				if (!ACL_READY(alt_sx))
 					get_acl(cmpbuf, sxp);
 				else {
 					sxp->acc_acl = alt_sx.acc_acl;
 					sxp->def_acl = alt_sx.def_acl;
+					alt_sx.acc_acl = alt_sx.def_acl = NULL;
 				}
 			}
 #endif
-		}
-#ifdef SUPPORT_ACLS
-		else if (preserve_acls)
-			free_acl(&alt_sx);
+#ifdef SUPPORT_XATTRS
+			if (preserve_xattrs) {
+				free_xattr(sxp);
+				if (!XATTR_READY(alt_sx))
+					get_xattr(cmpbuf, sxp);
+				else {
+					sxp->xattr = alt_sx.xattr;
+					alt_sx.xattr = NULL;
+				}
+			}
 #endif
+		} else {
+#ifdef SUPPORT_ACLS
+			if (preserve_acls)
+				free_acl(&alt_sx);
+#endif
+#ifdef SUPPORT_XATTRS
+			if (preserve_xattrs)
+				free_xattr(&alt_sx);
+#endif
+		}
 	}
 
 	if (maybe_hard_link(file, ndx, fname, statret, sxp, prev_name, &prev_st,
@@ -474,19 +497,13 @@ void finish_hard_link(struct file_struct *file, const char *fname, int fin_ndx,
 #ifdef SUPPORT_ACLS
 	prev_sx.acc_acl = prev_sx.def_acl = NULL;
 #endif
+#ifdef SUPPORT_XATTRS
+	prev_sx.xattr = NULL;
+#endif
 
 	while ((ndx = prev_ndx) >= 0) {
 		int val;
-		flist = flist_for_ndx(ndx);
-		if (flist == NULL) {
-			int start1 = first_flist ? first_flist->ndx_start : 0;
-			int start2 = first_flist ? first_flist->prev->ndx_start : 0;
-			int used = first_flist ? first_flist->prev->used : 0;
-			rprintf(FERROR,
-				"File index not found: %d (%d - %d)\n",
-				ndx, start1 - 1, start2 + used - 1);
-			exit_cleanup(RERR_PROTOCOL);
-		}
+		flist = flist_for_ndx(ndx, "finish_hard_link");
 		file = flist->files[ndx - flist->ndx_start];
 		file->flags = (file->flags & ~FLAG_HLINK_FIRST) | FLAG_HLINK_DONE;
 		prev_ndx = F_HL_PREV(file);
@@ -498,6 +515,10 @@ void finish_hard_link(struct file_struct *file, const char *fname, int fin_ndx,
 #ifdef SUPPORT_ACLS
 		if (preserve_acls)
 			free_acl(&prev_sx);
+#endif
+#ifdef SUPPORT_XATTRS
+		if (preserve_xattrs)
+			free_xattr(&prev_sx);
 #endif
 		if (val < 0)
 			continue;

io.c

@@ -101,7 +101,7 @@ static char ff_lastchar;
 #ifdef ICONV_OPTION
 static xbuf iconv_buf = EMPTY_XBUF;
 #endif
-static int defer_forwarding_messages = 0, defer_forwarding_keep = 0;
+static int defer_forwarding_messages = 0, keep_defer_forwarding = 0;
 static int select_timeout = SELECT_TIMEOUT;
 static int active_filecnt = 0;
 static OFF_T active_bytecnt = 0;
@@ -124,16 +124,7 @@ static void writefd(int fd, const char *buf, size_t len);
 static void writefd_unbuffered(int fd, const char *buf, size_t len);
 static void mplex_write(int fd, enum msgcode code, const char *buf, size_t len, int convert);
 
-struct flist_ndx_item {
-	struct flist_ndx_item *next;
-	int ndx;
-};
-
-struct flist_ndx_list {
-	struct flist_ndx_item *head, *tail;
-};
-
-static struct flist_ndx_list redo_list, hlink_list;
+static flist_ndx_list redo_list, hlink_list;
 
 struct msg_list_item {
 	struct msg_list_item *next;
@@ -147,45 +138,10 @@ struct msg_list {
 
 static struct msg_list msg_queue;
 
-static void flist_ndx_push(struct flist_ndx_list *lp, int ndx)
-{
-	struct flist_ndx_item *item;
-
-	if (!(item = new(struct flist_ndx_item)))
-		out_of_memory("flist_ndx_push");
-	item->next = NULL;
-	item->ndx = ndx;
-	if (lp->tail)
-		lp->tail->next = item;
-	else
-		lp->head = item;
-	lp->tail = item;
-}
-
-static int flist_ndx_pop(struct flist_ndx_list *lp)
-{
-	struct flist_ndx_item *next;
-	int ndx;
-
-	if (!lp->head)
-		return -1;
-
-	ndx = lp->head->ndx;
-	next = lp->head->next;
-	free(lp->head);
-	lp->head = next;
-	if (!next)
-		lp->tail = NULL;
-
-	return ndx;
-}
-
 static void got_flist_entry_status(enum festatus status, const char *buf)
 {
 	int ndx = IVAL(buf, 0);
-	struct file_list *flist = flist_for_ndx(ndx);
-
-	assert(flist != NULL);
+	struct file_list *flist = flist_for_ndx(ndx, "got_flist_entry_status");
 
 	if (remove_source_files) {
 		active_filecnt--;
@@ -208,6 +164,11 @@ static void got_flist_entry_status(enum festatus status, const char *buf)
 		}
 		break;
 	case FES_REDO:
+		if (read_batch) {
+			if (inc_recurse)
+				flist->in_progress++;
+			break;
+		}
 		if (inc_recurse)
 			flist->to_redo++;
 		flist_ndx_push(&redo_list, ndx);
@@ -296,33 +257,32 @@ static void msg_list_add(struct msg_list *lst, int code, const char *buf, int le
 	lst->tail = m;
 }
 
+static inline int flush_a_msg(int fd)
+{
+	struct msg_list_item *m = msg_queue.head;
+	int len = IVAL(m->buf, 0) & 0xFFFFFF;
+	int tag = *((uchar*)m->buf+3) - MPLEX_BASE;
+
+	if (!(msg_queue.head = m->next))
+		msg_queue.tail = NULL;
+
+	defer_forwarding_messages++;
+	mplex_write(fd, tag, m->buf + 4, len, m->convert);
+	defer_forwarding_messages--;
+
+	free(m);
+
+	return len;
+}
+
 static void msg_flush(void)
 {
 	if (am_generator) {
-		while (msg_queue.head && io_multiplexing_out) {
-			struct msg_list_item *m = msg_queue.head;
-			int len = IVAL(m->buf, 0) & 0xFFFFFF;
-			int tag = *((uchar*)m->buf+3) - MPLEX_BASE;
-			if (!(msg_queue.head = m->next))
-				msg_queue.tail = NULL;
-			stats.total_written += len + 4;
-			defer_forwarding_messages++;
-			mplex_write(sock_f_out, tag, m->buf + 4, len, m->convert);
-			defer_forwarding_messages--;
-			free(m);
-		}
+		while (msg_queue.head && io_multiplexing_out)
+			stats.total_written += flush_a_msg(sock_f_out) + 4;
 	} else {
-		while (msg_queue.head) {
-			struct msg_list_item *m = msg_queue.head;
-			int len = IVAL(m->buf, 0) & 0xFFFFFF;
-			int tag = *((uchar*)m->buf+3) - MPLEX_BASE;
-			if (!(msg_queue.head = m->next))
-				msg_queue.tail = NULL;
-			defer_forwarding_messages++;
-			mplex_write(msg_fd_out, tag, m->buf + 4, len, m->convert);
-			defer_forwarding_messages--;
-			free(m);
-		}
+		while (msg_queue.head)
+			(void)flush_a_msg(msg_fd_out);
 	}
 }
 
@@ -449,6 +409,7 @@ static void read_msg_fd(void)
 		got_flist_entry_status(FES_NO_SEND, buf);
 		break;
 	case MSG_ERROR_SOCKET:
+	case MSG_ERROR_UTF8:
 	case MSG_CLIENT:
 		if (!am_generator)
 			goto invalid_msg;
@@ -531,9 +492,9 @@ static void mplex_write(int fd, enum msgcode code, const char *buf, size_t len,
 
 	SIVAL(buffer, 0, ((MPLEX_BASE + (int)code)<<24) + len);
 
-	defer_forwarding_keep = 1; /* defer_forwarding_messages++ on return */
+	keep_defer_forwarding++; /* defer_forwarding_messages++ on return */
 	writefd_unbuffered(fd, buffer, n+4);
-	defer_forwarding_keep = 0;
+	keep_defer_forwarding--;
 
 	if (len > n)
 		writefd_unbuffered(fd, buf+n, len-n);
@@ -568,10 +529,9 @@ void send_msg_int(enum msgcode code, int num)
 
 void wait_for_receiver(void)
 {
-	if (iobuf_out_cnt)
-		io_flush(NORMAL_FLUSH);
-	else
-		read_msg_fd();
+	if (io_flush(NORMAL_FLUSH))
+		return;
+	read_msg_fd();
 }
 
 int get_redo_num(void)
@@ -1555,24 +1515,34 @@ static void writefd_unbuffered(int fd, const char *buf, size_t len)
 	}
 
 	no_flush--;
-	defer_inc -= defer_forwarding_keep;
+	if (keep_defer_forwarding)
+		defer_inc--;
 	if (!(defer_forwarding_messages -= defer_inc) && !no_flush)
 		msg_flush();
 }
 
-void io_flush(int flush_it_all)
+int io_flush(int flush_it_all)
 {
-	if (!iobuf_out_cnt || no_flush)
-		return;
+	int flushed_something = 0;
 
-	if (io_multiplexing_out)
-		mplex_write(sock_f_out, MSG_DATA, iobuf_out, iobuf_out_cnt, 0);
-	else
-		writefd_unbuffered(iobuf_f_out, iobuf_out, iobuf_out_cnt);
-	iobuf_out_cnt = 0;
+	if (no_flush)
+		return 0;
 
-	if (flush_it_all && !defer_forwarding_messages)
+	if (iobuf_out_cnt) {
+		if (io_multiplexing_out)
+			mplex_write(sock_f_out, MSG_DATA, iobuf_out, iobuf_out_cnt, 0);
+		else
+			writefd_unbuffered(iobuf_f_out, iobuf_out, iobuf_out_cnt);
+		iobuf_out_cnt = 0;
+		flushed_something = 1;
+	}
+
+	if (flush_it_all && !defer_forwarding_messages && msg_queue.head) {
 		msg_flush();
+		flushed_something = 1;
+	}
+
+	return flushed_something;
 }
 
 static void writefd(int fd, const char *buf, size_t len)

lib/sysacls.c

@@ -551,7 +551,7 @@ SMB_ACL_T sys_acl_init(int count)
 	 * acl[] array, this actually allocates an ACL with room
 	 * for (count+1) entries
 	 */
-	if ((a = (SMB_ACL_T)SMB_MALLOC(sizeof(struct SMB_ACL_T) + count * sizeof(struct acl))) == NULL) {
+	if ((a = (SMB_ACL_T)SMB_MALLOC(sizeof a[0] + count * sizeof (struct acl))) == NULL) {
 		errno = ENOMEM;
 		return NULL;
 	}
@@ -1007,7 +1007,7 @@ SMB_ACL_T sys_acl_init(int count)
 	 * acl[] array, this actually allocates an ACL with room
 	 * for (count+1) entries
 	 */
-	if ((a = SMB_MALLOC(sizeof(struct SMB_ACL_T) + count * sizeof(struct acl))) == NULL) {
+	if ((a = (SMB_ACL_T)SMB_MALLOC(sizeof a[0] + count * sizeof(struct acl))) == NULL) {
 		errno = ENOMEM;
 		return NULL;
 	}
@@ -1638,14 +1638,14 @@ SMB_ACL_T sys_acl_init(int count)
 		return NULL;
 	}
 
-	if ((a = SMB_MALLOC(sizeof(struct SMB_ACL_T) + sizeof(struct acl))) == NULL) {
+	if ((a = (SMB_ACL_T)SMB_MALLOC(sizeof a[0] + sizeof (struct acl))) == NULL) {
 		errno = ENOMEM;
 		return NULL;
 	}
 
 	a->next = -1;
 	a->freeaclp = False;
-	a->aclp = (struct acl *)(&a->aclp + sizeof(struct acl *));
+	a->aclp = (struct acl *)((char *)a + sizeof a[0]);
 	a->aclp->acl_cnt = 0;
 
 	return a;

log.c

@@ -55,7 +55,7 @@ extern iconv_t ic_chck;
 extern iconv_t ic_send, ic_recv;
 #endif
 extern char curr_dir[];
-extern char *module_dir;
+extern char *full_module_path;
 extern unsigned int module_dirlen;
 
 static int log_initialised;
@@ -85,7 +85,7 @@ struct {
 	{ RERR_SIGNAL     , "received SIGINT, SIGTERM, or SIGHUP" },
 	{ RERR_WAITCHILD  , "waitpid() failed" },
 	{ RERR_MALLOC     , "error allocating core memory buffers" },
-	{ RERR_PARTIAL    , "some files could not be transferred" },
+	{ RERR_PARTIAL    , "some files/attrs were not transferred (see previous errors)" },
 	{ RERR_VANISHED   , "some files vanished before they could be transferred" },
 	{ RERR_TIMEOUT    , "timeout in data send/receive" },
 	{ RERR_CONTIMEOUT , "timeout waiting for daemon connection" },
@@ -257,13 +257,17 @@ void rwrite(enum logcode code, const char *buf, int len, int is_utf8)
 
 	if (am_server && msg_fd_out >= 0) {
 		assert(!is_utf8);
-		/* Pass the message to our sibling. */
+		/* Pass the message to our sibling in native charset. */
 		send_msg((enum msgcode)code, buf, len, 0);
 		return;
 	}
 
 	if (code == FERROR_SOCKET) /* This gets simplified for a non-sibling. */
 		code = FERROR;
+	else if (code == FERROR_UTF8) {
+		is_utf8 = 1;
+		code = FERROR;
+	}
 
 	if (code == FCLIENT)
 		code = FINFO;
@@ -599,7 +603,7 @@ static void log_formatted(enum logcode code, const char *format, const char *op,
 			n = timestring(time(NULL));
 			break;
 		case 'P':
-			n = module_dir;
+			n = full_module_path;
 			break;
 		case 'u':
 			n = auth_user;

main.c

@@ -469,7 +469,7 @@ static pid_t do_cmd(char *cmd, char *machine, char *user, char **remote_argv, in
 #ifdef ICONV_CONST
 		setup_iconv();
 #endif
-		if (protect_args)
+		if (protect_args && !daemon_over_rsh)
 			send_protected_args(*f_out_p, args);
 	}
 
@@ -1187,8 +1187,8 @@ static int start_client(int argc, char *argv[])
 			rprintf(FERROR, "remote destination is not allowed with --read-batch\n");
 			exit_cleanup(RERR_SYNTAX);
 		}
-		remote_argv = argv + argc - 1;
-		remote_argc = 1;
+		remote_argv = argv += argc - 1;
+		remote_argc = argc = 1;
 	}
 
 	if (am_sender) {

options.c

@@ -1808,25 +1808,40 @@ void server_options(char **args, int *argc_p)
 	if (do_compression)
 		argstr[x++] = 'z';
 
-	/* We make use of the -e option to let the server know about any
-	 * pre-release protocol version && some behavior flags. */
-	argstr[x++] = 'e';
-#if SUBPROTOCOL_VERSION != 0
-	if (protocol_version == PROTOCOL_VERSION) {
-		x += snprintf(argstr+x, sizeof argstr - x,
-			      "%d.%d", PROTOCOL_VERSION, SUBPROTOCOL_VERSION);
-	} else
-#endif
-		argstr[x++] = '.';
 	set_allow_inc_recurse();
-	if (allow_inc_recurse)
-		argstr[x++] = 'i';
-#if defined HAVE_LUTIMES && defined HAVE_UTIMES
-	argstr[x++] = 'L';
+
+	/* Checking the pre-negotiated value allows --protocol=29 override. */
+	if (protocol_version >= 30) {
+		/* We make use of the -e option to let the server know about
+		 * any pre-release protocol version && some behavior flags. */
+		argstr[x++] = 'e';
+#if SUBPROTOCOL_VERSION != 0
+		if (protocol_version == PROTOCOL_VERSION) {
+			x += snprintf(argstr+x, sizeof argstr - x,
+				      "%d.%d",
+				      PROTOCOL_VERSION, SUBPROTOCOL_VERSION);
+		} else
 #endif
+			argstr[x++] = '.';
+		if (allow_inc_recurse)
+			argstr[x++] = 'i';
+#if defined HAVE_LUTIMES && defined HAVE_UTIMES
+		argstr[x++] = 'L';
+#endif
+#ifdef ICONV_OPTION
+		argstr[x++] = 's';
+#endif
+	}
+
+	if (x >= (int)sizeof argstr) { /* Not possible... */
+		rprintf(FERROR, "argstr overflow in server_options().\n");
+		exit_cleanup(RERR_MALLOC);
+	}
+
 	argstr[x] = '\0';
 
-	args[ac++] = argstr;
+	if (x > 1)
+		args[ac++] = argstr;
 
 #ifdef ICONV_OPTION
 	if (iconv_opt) {
@@ -2008,7 +2023,6 @@ void server_options(char **args, int *argc_p)
 			 *   and it may be an older version that doesn't know this
 			 *   option, so don't send it if client is the sender.
 			 */
-			int i;
 			for (i = 0; i < basis_dir_cnt; i++) {
 				args[ac++] = dest_option;
 				args[ac++] = basis_dir[i];
@@ -2048,6 +2062,11 @@ void server_options(char **args, int *argc_p)
 	else if (remove_source_files)
 		args[ac++] = "--remove-sent-files";
 
+	if (ac > MAX_SERVER_ARGS) { /* Not possible... */
+		rprintf(FERROR, "argc overflow in server_options().\n");
+		exit_cleanup(RERR_MALLOC);
+	}
+
 	*argc_p = ac;
 	return;
 

packaging/lsb/rsync.spec

@@ -1,9 +1,9 @@
 Summary: A fast, versatile, remote (and local) file-copying tool
 Name: rsync
-Version: 3.0.3
-%define fullversion %{version}
-Release: 1
-%define srcdir src
+Version: 3.0.5
+%define fullversion %{version}pre1
+Release: 0.1.pre1
+%define srcdir src-previews
 Group: Applications/Internet
 Source0: http://rsync.samba.org/ftp/rsync/%{srcdir}/rsync-%{fullversion}.tar.gz
 #Source1: http://rsync.samba.org/ftp/rsync/%{srcdir}/rsync-patches-%{fullversion}.tar.gz
@@ -66,8 +66,8 @@ rm -rf $RPM_BUILD_ROOT
 %{_mandir}/man5/rsyncd.conf.5*
 
 %changelog
-* Sun Jun 29 2008 Wayne Davison <wayned@samba.org>
-Released 3.0.3.
+* Sat Oct 11 2008 Wayne Davison <wayned@samba.org>
+Released 3.0.5pre1.
 
 * Fri Mar 21 2008 Wayne Davison <wayned@samba.org>
 Added installation of /etc/xinetd.d/rsync file and some commented-out

packaging/patch-update

@@ -13,6 +13,7 @@ my $tmp_dir = "patches.$$";
 
 &Getopt::Long::Configure('bundling');
 &usage if !&GetOptions(
+    'branch|b=s' => \( my $master_branch = 'master' ),
     'skip-check' => \( my $skip_branch_check ),
     'shell|s' => \( my $launch_shell ),
     'gen:s' => \( my $incl_generated_files ),
@@ -53,15 +54,13 @@ if ($incl_generated_files) {
 }
 our $last_touch = time;
 
-my(%patches, %local_patch);
+my %patches;
 
 # Start by finding all patches so that we can load all possible parents.
-open(PIPE, '-|', 'git', 'branch', '-a') or die $!;
+open(PIPE, '-|', 'git', 'branch', '-l') or die $!;
 while (<PIPE>) {
-    if (m# origin/patch/(.*)#) {
+    if (m# patch/(.*)#) {
 	$patches{$1} = 1;
-    } elsif (m# patch/(.*)#) {
-	$patches{$1} = $local_patch{$1} = 1;
     }
 }
 close PIPE;
@@ -70,19 +69,23 @@ my @patches = sort keys %patches;
 
 my(%parent, %description);
 foreach my $patch (@patches) {
-    my $branch = ($local_patch{$patch} ? '' : 'origin/') . "patch/$patch";
+    my $branch = "patch/$patch";
     my $desc = '';
-    open(PIPE, '-|', 'git', 'diff', '-U1000', "master...$branch", '--', "PATCH.$patch") or die $!;
+    open(PIPE, '-|', 'git', 'diff', '-U1000', "$master_branch...$branch", '--', "PATCH.$patch") or die $!;
     while (<PIPE>) {
 	last if /^@@ /;
     }
     while (<PIPE>) {
 	next unless s/^[ +]//;
 	if (m#patch -p1 <patches/(\S+)\.diff# && $1 ne $patch) {
-	    $parent{$patch} = $1;
+	    my $parent = $parent{$patch} = $1;
+	    if (!$patches{$parent}) {
+		die "Parent of $patch is not a local branch: $parent\n";
+	    }
 	}
 	$desc .= $_;
     }
+    close PIPE;
     $description{$patch} = $desc;
 }
 
@@ -90,8 +93,11 @@ if (@ARGV) {
     # Limit the list of patches to actually process based on @ARGV.
     @patches = ( );
     foreach (@ARGV) {
-	s{^(patches|patch|origin/patch)/} {};
+	s{^patch(es)?/} {};
 	s{\.diff$} {};
+	if (!$patches{$_}) {
+	    die "Local branch not available for patch: $_\n";
+	}
 	push(@patches, $_);
     }
 }
@@ -123,17 +129,13 @@ sub update_patch
 	}
 	$parent = "patch/$parent";
     } else {
-	$parent = 'master';
+	$parent = $master_branch;
     }
 
     print "======== $patch ========\n";
 
     sleep 1 while $incl_generated_files && $last_touch >= time;
-    if ($local_patch{$patch}) {
-	system "git checkout patch/$patch" and return 0;
-    } else {
-	system "git checkout --track -b patch/$patch origin/patch/$patch" and return 0;
-    }
+    system "git checkout patch/$patch" and return 0;
 
     my $ok = system("git merge $parent") == 0;
     if (!$ok || $launch_shell) {
@@ -174,8 +176,13 @@ sub update_patch
     close PIPE;
 
     if ($incl_generated_files) {
-	$parent =~ s#.*/##;
-	open(PIPE, '-|', 'diff', '-up', "$tmp_dir/$parent", "$tmp_dir/$patch") or die $!;
+	my $parent_dir;
+	if ($parent eq $master_branch) {
+	    $parent_dir = 'master';
+	} else {
+	    ($parent_dir) = $parent =~ m{([^/]+)$};
+	}
+	open(PIPE, '-|', 'diff', '-up', "$tmp_dir/$parent_dir", "$tmp_dir/$patch") or die $!;
 	while (<PIPE>) {
 	    s#^(diff -up) $tmp_dir/[^/]+/(.*?) $tmp_dir/[^/]+/(.*)#$1 a/$2 b/$3#o;
 	    s#^\Q---\E $tmp_dir/[^/]+/([^\t]+)\t.*#--- a/$1#o;

packaging/release-rsync

@@ -7,6 +7,7 @@ use strict;
 # ~/samba-rsync-ftp dir will be ready to be rsynced to samba.org.
 
 use Cwd;
+use Getopt::Long;
 use Term::ReadKey;
 use Date::Format;
 
@@ -14,6 +15,13 @@ my $dest = $ENV{HOME} . '/samba-rsync-ftp';
 my $passfile = $ENV{HOME} . '/.rsyncpass';
 my $path = $ENV{PATH};
 
+&Getopt::Long::Configure('bundling');
+&usage if !&GetOptions(
+    'branch|b=s' => \( my $master_branch = 'master' ),
+    'help|h' => \( my $help_opt ),
+);
+&usage if $help_opt;
+
 my $now = time;
 my $cl_today = time2str('* %a %b %d %Y', $now);
 my $year = time2str('%Y', $now);
@@ -56,7 +64,7 @@ open(IN, '-|', 'git status') or die $!;
 my $status = join('', <IN>);
 close IN;
 die "The checkout is not clean:\n", $status unless $status =~ /\nnothing to commit \(working directory clean\)/;
-die "The checkout is not on the master branch.\n" unless $status =~ /^# On branch master\n/;
+die "The checkout is not on the $master_branch branch.\n" unless $status =~ /^# On branch $master_branch\n/;
 
 my $confversion;
 open(IN, '<', 'configure.in') or die $!;
@@ -221,7 +229,7 @@ print $break, <<EOT;
 
 About to:
     - commit all version changes
-    - merge the master branch into the patch/* branches
+    - merge the $master_branch branch into the patch/* branches
     - update the files in the "patches" dir and OPTIONALLY
       (if you type 'y') to launch a shell for each patch
 
@@ -232,11 +240,11 @@ my $ans = <STDIN>;
 system "git commit -a -m 'Preparing for release of $version'" and exit 1;
 
 print "Updating files in \"patches\" dir ...\n";
-system "packaging/patch-update";
+system "packaging/patch-update --branch=$master_branch";
 
 if ($ans =~ /^y/i) {
     print "\nVisiting all \"patch/*\" branches ...\n";
-    system "packaging/patch-update --shell";
+    system "packaging/patch-update --branch=$master_branch --shell";
 }
 
 print $break, <<EOT;
@@ -305,7 +313,7 @@ system "fakeroot tar czf $srctar_file rsync-$version; rm -rf rsync-$version";
 print "Updating files in \"rsync-$version/patches\" dir ...\n";
 mkdir("rsync-$version", 0755);
 mkdir("rsync-$version/patches", 0755);
-system "packaging/patch-update --skip-check --gen=rsync-$version/patches";
+system "packaging/patch-update --skip-check --branch=$master_branch --gen=rsync-$version/patches";
 
 print "Creating $pattar_file ...\n";
 system "fakeroot tar chzf $pattar_file rsync-$version/patches; rm -rf rsync-$version";
@@ -343,3 +351,15 @@ Local changes are done.  When you're satisfied, push the git repository
 and rsync the release files.  Remember to announce the release on *BOTH*
 rsync-announce@lists.samba.org and rsync@lists.samba.org (and the web)!
 EOT
+
+exit;
+
+sub usage
+{
+    die <<EOT;
+Usage: release-rsync [OPTIONS]
+
+-b, --branch=BRANCH   The branch to release (default: master)
+-h, --help            Display this help message
+EOT
+}

receiver.c

@@ -59,6 +59,7 @@ extern struct filter_list_struct daemon_filter_list;
 
 static struct bitbag *delayed_bits = NULL;
 static int phase = 0, redoing = 0;
+static flist_ndx_list batch_redo_list;
 /* We're either updating the basis file or an identical copy: */
 static int updating_basis_or_equiv;
 
@@ -348,25 +349,60 @@ static void handle_delayed_updates(char *local_name)
 	}
 }
 
-static int get_next_gen_ndx(int fd, int next_gen_ndx, int desired_ndx)
+static void no_batched_update(int ndx, BOOL is_redo)
 {
-	while (next_gen_ndx < desired_ndx) {
-		if (next_gen_ndx >= 0) {
-			struct file_struct *file = cur_flist->files[next_gen_ndx];
-			rprintf(FERROR_XFER,
-				"(No batched update for%s \"%s\")\n",
-				file->flags & FLAG_FILE_SENT ? " resend of" : "",
-				f_name(file, NULL));
-		}
-		next_gen_ndx = read_int(fd);
-		if (next_gen_ndx == -1) {
-			if (inc_recurse)
-				next_gen_ndx = first_flist->prev->used + first_flist->prev->ndx_start;
-			else
-				next_gen_ndx = cur_flist->used;
+	struct file_list *flist = flist_for_ndx(ndx, "no_batched_update");
+	struct file_struct *file = flist->files[ndx - flist->ndx_start];
+
+	rprintf(FERROR_XFER, "(No batched update for%s \"%s\")\n",
+		is_redo ? " resend of" : "", f_name(file, NULL));
+
+	if (inc_recurse)
+		send_msg_int(MSG_NO_SEND, ndx);
+}
+
+static int we_want_redo(int desired_ndx)
+{
+	static int redo_ndx = -1;
+
+	while (redo_ndx < desired_ndx) {
+		if (redo_ndx >= 0)
+			no_batched_update(redo_ndx, True);
+		if ((redo_ndx = flist_ndx_pop(&batch_redo_list)) < 0)
+			return 0;
+	}
+
+	if (redo_ndx == desired_ndx) {
+		redo_ndx = -1;
+		return 1;
+	}
+
+	return 0;
+}
+
+static int gen_wants_ndx(int desired_ndx)
+{
+	static int next_ndx = -1;
+	static BOOL got_eof = 0;
+
+	if (got_eof)
+		return 0;
+
+	while (next_ndx < desired_ndx) {
+		if (next_ndx >= 0)
+			no_batched_update(next_ndx, False);
+		if ((next_ndx = read_int(batch_gen_fd)) < 0) {
+			got_eof = True;
+			return 0;
 		}
 	}
-	return next_gen_ndx;
+
+	if (next_ndx == desired_ndx) {
+		next_ndx = -1;
+		return 1;
+	}
+
+	return 0;
 }
 
 /**
@@ -375,7 +411,6 @@ static int get_next_gen_ndx(int fd, int next_gen_ndx, int desired_ndx)
  * Receiver process runs on the same host as the generator process. */
 int recv_files(int f_in, char *local_name)
 {
-	int next_gen_ndx = -1;
 	int fd1,fd2;
 	STRUCT_STAT st;
 	int iflags, xlen;
@@ -410,17 +445,13 @@ int recv_files(int f_in, char *local_name)
 					 xname, &xlen);
 		if (ndx == NDX_DONE) {
 			if (inc_recurse && first_flist) {
+				if (read_batch)
+					gen_wants_ndx(first_flist->used + first_flist->ndx_start);
 				flist_free(first_flist);
 				if (first_flist)
 					continue;
-			}
-			if (read_batch && cur_flist) {
-				int high = inc_recurse
-				    ? first_flist->prev->used + first_flist->prev->ndx_start
-				    : cur_flist->used;
-				get_next_gen_ndx(batch_gen_fd, next_gen_ndx, high);
-				next_gen_ndx = -1;
-			}
+			} else if (read_batch && first_flist)
+				gen_wants_ndx(first_flist->used);
 			if (++phase > max_phase)
 				break;
 			if (verbose > 2)
@@ -509,17 +540,15 @@ int recv_files(int f_in, char *local_name)
 		}
 
 		if (read_batch) {
-			next_gen_ndx = get_next_gen_ndx(batch_gen_fd, next_gen_ndx, ndx);
-			if (ndx < next_gen_ndx) {
+			if (!(redoing ? we_want_redo(ndx) : gen_wants_ndx(ndx))) {
 				rprintf(FINFO,
-					"(Skipping batched update for \"%s\")\n",
+					"(Skipping batched update for%s \"%s\")\n",
+					redoing ? " resend of" : "",
 					fname);
 				discard_receive_data(f_in, F_LENGTH(file));
-				if (inc_recurse)
-					send_msg_int(MSG_NO_SEND, ndx);
+				file->flags |= FLAG_FILE_SENT;
 				continue;
 			}
-			next_gen_ndx = -1;
 		}
 
 		partialptr = partial_dir ? partial_dir_fname(fname) : fname;
@@ -698,23 +727,33 @@ int recv_files(int f_in, char *local_name)
 				do_unlink(partialptr);
 				handle_partial_dir(partialptr, PDIR_DELETE);
 			}
-		} else if (keep_partial && partialptr
-		    && handle_partial_dir(partialptr, PDIR_CREATE)) {
-			if (!finish_transfer(partialptr, fnametmp, fnamecmp, NULL,
-					     file, recv_ok, !partial_dir))
+		} else if (keep_partial && partialptr) {
+			if (!handle_partial_dir(partialptr, PDIR_CREATE)) {
+				rprintf(FERROR,
+				    "Unable to create partial-dir for %s -- discarding %s.\n",
+				    local_name ? local_name : f_name(file, NULL),
+				    recv_ok ? "completed file" : "partial file");
+				do_unlink(fnametmp);
+				recv_ok = -1;
+			} else if (!finish_transfer(partialptr, fnametmp, fnamecmp, NULL,
+						    file, recv_ok, !partial_dir))
 				recv_ok = -1;
 			else if (delay_updates && recv_ok) {
 				bitbag_set_bit(delayed_bits, ndx);
 				recv_ok = 2;
-			}
-		} else {
-			partialptr = NULL;
+			} else
+				partialptr = NULL;
+		} else
 			do_unlink(fnametmp);
-		}
 
 		cleanup_disable();
 
+		if (read_batch)
+			file->flags |= FLAG_FILE_SENT;
+
 		switch (recv_ok) {
+		case 2:
+			break;
 		case 1:
 			if (remove_source_files || inc_recurse
 			 || (preserve_hard_links && F_IS_HLINKED(file)))
@@ -744,6 +783,8 @@ int recv_files(int f_in, char *local_name)
 					keptstr, redostr);
 			}
 			if (!redoing) {
+				if (read_batch)
+					flist_ndx_push(&batch_redo_list, ndx);
 				send_msg_int(MSG_REDO, ndx);
 				file->flags |= FLAG_FILE_SENT;
 			} else if (inc_recurse)

rsync.c

@@ -221,16 +221,19 @@ void send_protected_args(int fd, char *args[])
 	if (verbose > 1)
 		print_child_argv("protected args:", args + i + 1);
 	do {
+		if (!args[i][0])
+			write_buf(fd, ".", 2);
 #ifdef ICONV_OPTION
-		if (convert) {
+		else if (convert) {
 			INIT_XBUF_STRLEN(inbuf, args[i]);
 			iconvbufs(ic_send, &inbuf, &outbuf,
 				  ICB_EXPAND_OUT | ICB_INCLUDE_BAD | ICB_INCLUDE_INCOMPLETE);
 			outbuf.buf[outbuf.len] = '\0';
 			write_buf(fd, outbuf.buf, outbuf.len + 1);
 			outbuf.len = 0;
-		} else
+		}
 #endif
+		else
 			write_buf(fd, args[i], strlen(args[i]) + 1);
 	} while (args[++i]);
 	write_byte(fd, 0);
@@ -257,8 +260,17 @@ int read_ndx_and_attrs(int f_in, int *iflag_ptr, uchar *type_ptr,
 			break;
 		if (ndx == NDX_DONE)
 			return ndx;
-		if (!inc_recurse || am_sender)
-			goto invalid_ndx;
+		if (!inc_recurse || am_sender) {
+			int last;
+			if (first_flist)
+				last = first_flist->prev->ndx_start + first_flist->prev->used - 1;
+			else
+				last = -1;
+			rprintf(FERROR,
+				"Invalid file index: %d (%d - %d) [%s]\n",
+				ndx, NDX_DONE, last, who_am_i());
+			exit_cleanup(RERR_PROTOCOL);
+		}
 		if (ndx == NDX_FLIST_EOF) {
 			flist_eof = 1;
 			send_msg(MSG_FLIST_EOF, "", 0, 0);
@@ -268,9 +280,10 @@ int read_ndx_and_attrs(int f_in, int *iflag_ptr, uchar *type_ptr,
 		if (ndx < 0 || ndx >= dir_flist->used) {
 			ndx = NDX_FLIST_OFFSET - ndx;
 			rprintf(FERROR,
-				"[%s] Invalid dir index: %d (%d - %d)\n",
-				who_am_i(), ndx, NDX_FLIST_OFFSET,
-				NDX_FLIST_OFFSET - dir_flist->used + 1);
+				"Invalid dir index: %d (%d - %d) [%s]\n",
+				ndx, NDX_FLIST_OFFSET,
+				NDX_FLIST_OFFSET - dir_flist->used + 1,
+				who_am_i());
 			exit_cleanup(RERR_PROTOCOL);
 		}
 
@@ -299,17 +312,7 @@ int read_ndx_and_attrs(int f_in, int *iflag_ptr, uchar *type_ptr,
 		goto read_loop;
 	}
 
-	if (!(flist = flist_for_ndx(ndx))) {
-		int start, used;
-	  invalid_ndx:
-		start = first_flist ? first_flist->ndx_start : 0;
-		used = first_flist ? first_flist->used : 0;
-		rprintf(FERROR,
-			"Invalid file index: %d (%d - %d) with iflags %x [%s]\n",
-			ndx, start - 1, start + used -1, iflags, who_am_i());
-		exit_cleanup(RERR_PROTOCOL);
-	}
-	cur_flist = flist;
+	cur_flist = flist_for_ndx(ndx, "read_ndx_and_attrs");
 
 	if (iflags & ITEM_BASIS_TYPE_FOLLOWS)
 		fnamecmp_type = read_byte(f_in);
@@ -610,23 +613,40 @@ int finish_transfer(const char *fname, const char *fnametmp,
 	return 1;
 }
 
-struct file_list *flist_for_ndx(int ndx)
+struct file_list *flist_for_ndx(int ndx, const char *fatal_error_loc)
 {
 	struct file_list *flist = cur_flist;
 
 	if (!flist && !(flist = first_flist))
-		return NULL;
+		goto not_found;
 
 	while (ndx < flist->ndx_start-1) {
 		if (flist == first_flist)
-			return NULL;
+			goto not_found;
 		flist = flist->prev;
 	}
 	while (ndx >= flist->ndx_start + flist->used) {
 		if (!(flist = flist->next))
-			return NULL;
+			goto not_found;
 	}
 	return flist;
+
+  not_found:
+	if (fatal_error_loc) {
+		int first, last;
+		if (first_flist) {
+			first = first_flist->ndx_start - 1;
+			last = first_flist->prev->ndx_start + first_flist->prev->used - 1;
+		} else {
+			first = 0;
+			last = -1;
+		}
+		rprintf(FERROR,
+			"File-list index %d not in %d - %d (%s) [%s]\n",
+			ndx, first, last, fatal_error_loc, who_am_i());
+		exit_cleanup(RERR_PROTOCOL);
+	}
+	return NULL;
 }
 
 const char *who_am_i(void)

rsync.h

@@ -211,6 +211,7 @@ enum logcode {
     FERROR_XFER=1, FINFO=2, /* sent over socket for any protocol */
     FERROR=3, FWARNING=4, /* sent over socket for protocols >= 30 */
     FERROR_SOCKET=5, FLOG=6, /* only sent via receiver -> generator pipe */
+    FERROR_UTF8=8, /* only sent via receiver -> generator pipe */
     FCLIENT=7 /* never transmitted (e.g. server converts to FINFO) */
 };
 
@@ -221,6 +222,7 @@ enum msgcode {
 	MSG_ERROR_XFER=FERROR_XFER, MSG_INFO=FINFO, /* remote logging */
 	MSG_ERROR=FERROR, MSG_WARNING=FWARNING, /* protocol-30 remote logging */
 	MSG_ERROR_SOCKET=FERROR_SOCKET, /* sibling logging */
+	MSG_ERROR_UTF8=FERROR_UTF8, /* sibling logging */
 	MSG_LOG=FLOG, MSG_CLIENT=FCLIENT, /* sibling logging */
 	MSG_REDO=9,	/* reprocess indicated flist index */
 	MSG_FLIST=20,	/* extra file list over sibling socket */
@@ -830,6 +832,15 @@ struct stats {
 
 struct chmod_mode_struct;
 
+struct flist_ndx_item {
+	struct flist_ndx_item *next;
+	int ndx;
+};
+
+typedef struct {
+	struct flist_ndx_item *head, *tail;
+} flist_ndx_list;
+
 #define EMPTY_ITEM_LIST {NULL, 0, 0}
 
 typedef struct {

rsync.yo

@@ -1,5 +1,5 @@
 mailto(rsync-bugs@samba.org)
-manpage(rsync)(1)(29 Jun 2008)()()
+manpage(rsync)(1)(11 Oct 2008)()()
 manpagename(rsync)(a fast, versatile, remote (and local) file-copying tool)
 manpagesynopsis()
 
@@ -464,9 +464,9 @@ dit(bf(--version)) print the rsync version number and exit.
 dit(bf(-v, --verbose)) This option increases the amount of information you
 are given during the transfer.  By default, rsync works silently. A
 single bf(-v) will give you information about what files are being
-transferred and a brief summary at the end. Two bf(-v) flags will give you
+transferred and a brief summary at the end. Two bf(-v) options will give you
 information on what files are being skipped and slightly more
-information at the end. More than two bf(-v) flags should only be used if
+information at the end. More than two bf(-v) options should only be used if
 you are debugging rsync.
 
 Note that the names of the transferred files that are output are done using
@@ -480,7 +480,7 @@ any way.  See the bf(--out-format) option for more details.
 
 dit(bf(-q, --quiet)) This option decreases the amount of information you
 are given during the transfer, notably suppressing information messages
-from the remote server. This flag is useful when invoking rsync from
+from the remote server. This option is useful when invoking rsync from
 cron.
 
 dit(bf(--no-motd)) This option affects the information that is output
@@ -1117,7 +1117,7 @@ directories that are being synchronized.  You must have asked rsync to
 send the whole directory (e.g. "dir" or "dir/") without using a wildcard
 for the directory's contents (e.g. "dir/*") since the wildcard is expanded
 by the shell and rsync thus gets a request to transfer individual files, not
-the files' parent directory.  Files that are excluded from transfer are
+the files' parent directory.  Files that are excluded from the transfer are
 also excluded from being deleted unless you use the bf(--delete-excluded)
 option or mark the rules as only matching on the sending side (see the
 include/exclude modifiers in the FILTER RULES section).
@@ -1139,7 +1139,7 @@ destination.  You can override this with the bf(--ignore-errors) option.
 The bf(--delete) option may be combined with one of the --delete-WHEN options
 without conflict, as well as bf(--delete-excluded).  However, if none of the
 --delete-WHEN options are specified, rsync will choose the
-bf(--delete-during) algorithm when talking to an rsync 3.0.0 or newer, and
+bf(--delete-during) algorithm when talking to rsync 3.0.0 or newer, and
 the bf(--delete-before) algorithm when talking to an older rsync.  See also
 bf(--delete-delay) and bf(--delete-after).
 
@@ -1156,19 +1156,26 @@ algorithm that requires rsync to scan all the files in the transfer into
 memory at once (see bf(--recursive)).
 
 dit(bf(--delete-during, --del)) Request that the file-deletions on the
-receiving side be done incrementally as the transfer happens.  This is
-a faster method than choosing the before- or after-transfer algorithm,
-but it is only supported beginning with rsync version 2.6.4.
+receiving side be done incrementally as the transfer happens.  The
+per-directory delete scan is done right before each directory is checked
+for updates, so it behaves like a more efficient bf(--delete-before),
+including doing the deletions prior to any per-directory filter files
+being updated.  This option was first added in rsync version 2.6.4.
 See bf(--delete) (which is implied) for more details on file-deletion.
 
 dit(bf(--delete-delay)) Request that the file-deletions on the receiving
-side be computed during the transfer, and then removed after the transfer
-completes.  If the number of removed files overflows an internal buffer, a
+side be computed during the transfer (like bf(--delete-during)), and then
+removed after the transfer completes.  This is useful when combined with
+bf(--delay-updates) and/or bf(--fuzzy), and is more efficient than using
+bf(--delete-after) (but can behave differently, since bf(--delete-after)
+computes the deletions in a separate pass after all updates are done).
+If the number of removed files overflows an internal buffer, a
 temporary file will be created on the receiving side to hold the names (it
 is removed while open, so you shouldn't see it during the transfer).  If
 the creation of the temporary file fails, rsync will try to fall back to
 using bf(--delete-after) (which it cannot do if bf(--recursive) is doing an
 incremental scan).
+See bf(--delete) (which is implied) for more details on file-deletion.
 
 dit(bf(--delete-after)) Request that the file-deletions on the receiving
 side be done after the transfer has completed.  This is useful if you
@@ -1522,6 +1529,11 @@ An example:
 
 quote(tt(  rsync -av --link-dest=$PWD/prior_dir host:src_dir/ new_dir/))
 
+If file's aren't linking, double-check their attributes.  Also check if some
+attributes are getting forced outside of rsync's control, such a mount option
+that squishes root to a single user, or mounts a removable drive with generic
+ownership (such as OS X's "Ignore ownership on this volume" option).
+
 Beginning in version 2.6.4, multiple bf(--link-dest) directories may be
 provided, which will cause rsync to search the list in the order specified
 for an exact match.
@@ -1718,22 +1730,22 @@ you are talking to a recent enough rsync that it logs deletions instead of
 outputting them as a verbose message).
 
 dit(bf(--out-format=FORMAT)) This allows you to specify exactly what the
-rsync client outputs to the user on a per-update basis.  The format is a text
-string containing embedded single-character escape sequences prefixed with
-a percent (%) character.  For a list of the possible escape characters, see
-the "log format" setting in the rsyncd.conf manpage.
+rsync client outputs to the user on a per-update basis.  The format is a
+text string containing embedded single-character escape sequences prefixed
+with a percent (%) character.   A default format of "%n%L" is assumed if
+bf(-v) is specified (which reports the name
+of the file and, if the item is a link, where it points).  For a full list
+of the possible escape characters, see the "log format" setting in the
+rsyncd.conf manpage.
 
-Specifying this option will mention each file, dir, etc. that gets updated
-in a significant way (a transferred file, a recreated symlink/device, or a
-touched directory).  In addition, if the itemize-changes escape (%i) is
-included in the string, the logging of names increases to mention any
-item that is changed in any way (as long as the receiving side is at least
-2.6.4).  See the bf(--itemize-changes) option for a description of the
-output of "%i".
-
-The bf(--verbose) option implies a format of "%n%L", but you can use
-bf(--out-format) without bf(--verbose) if you like, or you can override
-the format of its per-file output using this option.
+Specifying the bf(--out-format) option
+will mention each file, dir, etc. that gets updated in a significant
+way (a transferred file, a recreated symlink/device, or a touched
+directory).  In addition, if the itemize-changes escape (%i) is included in
+the string (e.g. if the bf(--itemize-changes) option was used), the logging
+of names increases to mention any item that is changed in any way (as long
+as the receiving side is at least 2.6.4).  See the bf(--itemize-changes)
+option for a description of the output of "%i".
 
 Rsync will output the out-format string prior to a file's transfer unless
 one of the transfer-statistic escapes is requested, in which case the
@@ -2093,7 +2105,7 @@ by the server and defaults to the current code(time()).  This option
 is used to set a specific checksum seed, which is useful for
 applications that want repeatable block and file checksums, or
 in the case where the user wants a more random checksum seed.
-Note that setting NUM to 0 causes rsync to use the default of code(time())
+Setting NUM to 0 causes rsync to use the default of code(time())
 for checksum seed.
 enddit()
 
@@ -2264,7 +2276,7 @@ itemization(
   it() rsync chooses between doing a simple string match and wildcard
   matching by checking if the pattern contains one of these three wildcard
   characters: '*', '?', and '[' .
-  it() a '*' matches any non-empty path component (it stops at slashes).
+  it() a '*' matches any path component, but it stops at slashes.
   it() use '**' to match anything, including slashes.
   it() a '?' matches any character except a slash (/).
   it() a '[' introduces a character class, such as [a-z] or [[:alpha:]].
@@ -2882,7 +2894,7 @@ url(http://rsync.samba.org/)(http://rsync.samba.org/)
 
 manpagesection(VERSION)
 
-This man page is current for version 3.0.3 of rsync.
+This man page is current for version 3.0.5pre1 of rsync.
 
 manpagesection(INTERNAL OPTIONS)
 

rsyncd.conf.yo

@@ -1,5 +1,5 @@
 mailto(rsync-bugs@samba.org)
-manpage(rsyncd.conf)(5)(29 Jun 2008)()()
+manpage(rsyncd.conf)(5)(11 Oct 2008)()()
 manpagename(rsyncd.conf)(configuration file for rsync in daemon mode)
 manpagesynopsis()
 
@@ -101,12 +101,11 @@ who like to tune their systems to the utmost degree. You can set all
 sorts of socket options which may make transfers faster (or
 slower!). Read the man page for the code(setsockopt()) system call for
 details on some of the options you may be able to set. By default no
-special socket options are set.  These settings are superseded by the
-bf(--sockopts) command-line option.
+special socket options are set.  These settings can also be specified
+via the bf(--sockopts) command-line option.
 
 enddit()
 
-
 manpagesection(MODULE PARAMETERS)
 
 After the global parameters you should define a number of modules, each
@@ -213,7 +212,8 @@ to the exclude setting for the module so that
 a user can't try to create it.
 
 Note:  rsync makes no attempt to verify that any pre-existing symlinks in
-the hierarchy are as safe as you want them to be.  If you setup an rsync
+the module's hierarchy are as safe as you want them to be (unless, of
+course, it just copied in the whole hierarchy).  If you setup an rsync
 daemon on a new area or locally add symlinks, you can manually protect your
 symlinks from being abused by prefixing "/rsyncd-munged/" to the start of
 every symlink's value.  There is a perl script in the support directory
@@ -256,7 +256,7 @@ the transfer.  If this value is set on a per-module basis instead of
 globally, the global log will still contain any authorization failures
 or config-file error messages.
 
-If the daemon fails to open to specified file, it will fall back to
+If the daemon fails to open the specified file, it will fall back to
 using syslog and output an error about the failure.  (Note that the
 failure to open the specified log file used to be a fatal error.)
 
@@ -509,7 +509,7 @@ quote(itemization(
   it() %a the remote IP address
   it() %b the number of bytes actually transferred
   it() %B the permission bits of the file (e.g. rwxrwxrwt)
-  it() %c the checksum bytes received for this file (only when sending)
+  it() %c the total size of the block checksums received for the basis file (only when sending)
   it() %f the filename (long form on sender; no trailing "/")
   it() %G the gid of the file (decimal) or "DEFAULT"
   it() %h the remote host name
@@ -700,7 +700,7 @@ url(http://rsync.samba.org/)(http://rsync.samba.org/)
 
 manpagesection(VERSION)
 
-This man page is current for version 3.0.3 of rsync.
+This man page is current for version 3.0.5pre1 of rsync.
 
 manpagesection(CREDITS)
 

sender.c

@@ -127,13 +127,7 @@ void successful_send(int ndx)
 	if (!remove_source_files)
 		return;
 
-	if (!(flist = flist_for_ndx(ndx))) {
-		rprintf(FERROR,
-			"INTERNAL ERROR: unable to find flist for item %d\n",
-			ndx);
-		return;
-	}
-
+	flist = flist_for_ndx(ndx, "successful_send");
 	file = flist->files[ndx - flist->ndx_start];
 	if (!change_pathname(file, NULL, 0))
 		return;
@@ -281,7 +275,7 @@ void send_files(int f_in, int f_out)
 
 		if (!(s = receive_sums(f_in))) {
 			io_error |= IOERR_GENERAL;
-			rprintf(FERROR, "receive_sums failed\n");
+			rprintf(FERROR_XFER, "receive_sums failed\n");
 			exit_cleanup(RERR_PROTOCOL);
 		}
 
@@ -309,7 +303,7 @@ void send_files(int f_in, int f_out)
 		/* map the local file */
 		if (do_fstat(fd, &st) != 0) {
 			io_error |= IOERR_GENERAL;
-			rsyserr(FERROR, errno, "fstat failed");
+			rsyserr(FERROR_XFER, errno, "fstat failed");
 			free_sums(s);
 			close(fd);
 			exit_cleanup(RERR_PROTOCOL);

support/atomic-rsync

@@ -6,35 +6,49 @@
 # more details and some important caveats!**
 
 use strict;
+use warnings;
 use Cwd 'abs_path';
 
 my $RSYNC_PROG = '/usr/bin/rsync';
 my $RM_PROG = '/bin/rm';
 
 my $dest_dir = $ARGV[-1];
-usage(1) if $dest_dir eq '' || $dest_dir =~ /^--/;
+&usage if !defined $dest_dir || $dest_dir =~ /(^-|^$)/ || grep(/^--help/, @ARGV);
+$dest_dir =~ s{(?<=.)/+$} {};
 
 if (!-d $dest_dir) {
-    print STDERR "$dest_dir is not a directory.\n\n";
-    usage(1);
+    die "$dest_dir is not a directory.\nUse --help for help.\n";
 }
 
-if (@_ = grep(/^--(link|compare)-dest/, @ARGV)) {
+if (@_ = grep(/^--[a-z]+-dest\b/, @ARGV)) {
     $_ = join(' or ', @_);
-    print STDERR "You may not use $_ as an rsync option.\n\n";
-    usage(1);
+    die "You cannot use the $_ option with atomic-rsync.\nUse --help for help.\n";
 }
 
+my $symlink_content = readlink $dest_dir; # undef when a real dir
+
+my $dest_arg = $dest_dir;
+# This gives us the real destination dir, with all symlinks dereferenced.
 $dest_dir = abs_path($dest_dir);
 if ($dest_dir eq '/') {
-    print STDERR 'You must not use "/" as the destination directory.', "\n\n";
-    usage(1);
+    die qq|You must not use "/" as the destination directory.\nUse --help for help.\n|;
 }
 
-my $old_dir = "$dest_dir~old~";
-my $new_dir = $ARGV[-1] = "$dest_dir~new~";
+my($old_dir, $new_dir);
+if (defined $symlink_content && $dest_dir =~ /-([12])$/) {
+    my $num = 3 - $1;
+    $old_dir = undef;
+    ($new_dir = $dest_dir) =~ s/-[12]$/-$num/;
+    $symlink_content =~ s/-[12]$/-$num/;
+} else {
+    $old_dir = "$dest_dir~old~";
+    $new_dir = "$dest_dir~new~";
+}
 
-system($RM_PROG, '-rf', $old_dir) if -d $old_dir;
+$ARGV[-1] = "$new_dir/";
+
+system($RM_PROG, '-rf', $old_dir) if defined $old_dir && -d $old_dir;
+system($RM_PROG, '-rf', $new_dir) if -d $new_dir;
 
 if (system($RSYNC_PROG, "--link-dest=$dest_dir", @ARGV)) {
     if ($? == -1) {
@@ -48,17 +62,30 @@ if (system($RSYNC_PROG, "--link-dest=$dest_dir", @ARGV)) {
     exit $?;
 }
 
-rename($dest_dir, $old_dir) or die "Unable to rename $new_dir to $old_dir: $!";
+if (!defined $old_dir) {
+    atomic_symlink($symlink_content, $dest_arg);
+    exit;
+}
+
+rename($dest_dir, $old_dir) or die "Unable to rename $dest_dir to $old_dir: $!";
 rename($new_dir, $dest_dir) or die "Unable to rename $new_dir to $dest_dir: $!";
 
 exit;
 
+sub atomic_symlink
+{
+    my($target, $link) = @_;
+    my $newlink = "$link~new~";
+
+    unlink($newlink); # Just in case
+    symlink($target, $newlink) or die "Unable to symlink $newlink -> $target: $!\n";
+    rename($newlink, $link) or die "Unable to rename $newlink to $link: $!\n";
+}
+
 
 sub usage
 {
-    my($ret) = @_;
-    my $fh = $ret ? *STDERR : *STDOUT;
-    print $fh <<EOT;
+    die <<EOT;
 Usage: atomic-rsync [RSYNC-OPTIONS] HOST:/SOURCE/DIR/ /DEST/DIR/
        atomic-rsync [RSYNC-OPTIONS] HOST::MOD/DIR/ /DEST/DIR/
 
@@ -67,24 +94,29 @@ creating a new hierarchy (using hard-links to leverage the existing files),
 and then swapping the new hierarchy into place.  You must be pulling files
 to a local directory, and that directory must already exist.  For example:
 
+    mkdir /local/files-1
+    ln -s files-1 /local/files
     atomic-rsync -av host:/remote/files/ /local/files/
 
-This would make the transfer to the directory /local/files~new~ and then
-swap out /local/files at the end of the transfer by renaming it to
-/local/files~old~ and putting the new directory into its place.  The
-/local/files~old~ directory will be preserved until the next update, at
-which point it will be deleted.
+If /local/files is a symlink to a directory that ends in -1 or -2, the
+copy will go to the alternate suffix and the symlink will be changed to
+point to the new dir.  This is a fully atomic update.  If the destination
+is not a symlink (or not a symlink to a *-1 or a *-2 directory), this
+will instead create a directory with "~new~" suffixed, move the current
+directory to a name with "~old~" suffixed, and then move the ~new~
+directory to the original destination name (this double rename is not
+fully atomic, but is rapid).  In both cases, the prior destintaion
+directory will be preserved until the next update, at which point it
+will be deleted.
 
-Do NOT specify this command:
+In all likelihood, you do NOT want to specify this command:
 
     atomic-rsync -av host:/remote/files /local/
 
 ... UNLESS you want the entire /local dir to be swapped out!
 
 See the "rsync" command for its list of options.  You may not use the
---link-dest or --compare-dest options (since this script uses --link-dest
-to make the transfer efficient).  Also, the destination directory cannot
-be "/".
+--link-dest, --compare-dest, or --copy-dest options (since this script
+uses --link-dest to make the transfer efficient).
 EOT
-    exit $ret;
 }

testsuite/batch-mode.test

@@ -41,6 +41,8 @@ runtest "--read-batch from daemon" 'checkit "$RSYNC -av --read-batch=BATCH \"$to
 rm -rf "$todir"
 runtest "BATCH.sh use of --read-batch" 'checkit "./BATCH.sh" "$chkdir" "$todir"'
 
+runtest "do-nothing re-run of batch" 'checkit "./BATCH.sh" "$chkdir" "$todir"'
+
 rm -rf "$todir"
 mkdir "$todir" || test_fail "failed to restore empty destination directory"
 runtest "daemon recv --write-batch" 'checkit "\"$ignore23\" $RSYNC -av --write-batch=BATCH \"$fromdir/\" rsync://localhost/test-to" "$chkdir" "$todir"'

testsuite/xattrs.test

@@ -99,8 +99,12 @@ rm -rf "$todir"
 
 xset user.nice 'this is nice, but different' file1
 
+checkit "$RSYNC -aiX --fake-super . ../chk" "$fromdir" "$chkdir"
+
+cd "$chkdir"
 xls $files >"$scratchdir/xattrs.txt"
 
+cd "$fromdir"
 checkit "$RSYNC -aiX --fake-super --link-dest=../chk . ../to" "$chkdir" "$todir"
 
 cd "$todir"
@@ -116,10 +120,10 @@ fi
 cd "$fromdir"
 rm -rf "$todir" "$chkdir"
 
-rsync -aX file1 file2
-rsync -aX file1 file2 ../chk/
-rsync -aX --del ../chk/ .
-rsync -aX file1 ../lnk/
+$RSYNC -aX file1 file2
+$RSYNC -aX file1 file2 ../chk/
+$RSYNC -aX --del ../chk/ .
+$RSYNC -aX file1 ../lnk/
 
 xls file1 file2 >"$scratchdir/xattrs.txt"
 
@@ -132,7 +136,7 @@ cd "$fromdir"
 rm "$todir/file2"
 
 echo extra >file1
-rsync -aX . ../chk/
+$RSYNC -aX . ../chk/
 
 checkit "$RSYNC -aiiX . ../to" "$chkdir" "$todir"
 

util.c

@@ -437,7 +437,7 @@ int robust_rename(const char *from, const char *to, const char *partialptr,
 		case EXDEV:
 			if (partialptr) {
 				if (!handle_partial_dir(partialptr,PDIR_CREATE))
-					return -1;
+					return -2;
 				to = partialptr;
 			}
 			if (copy_file(from, to, -1, mode, 0) != 0)
@@ -1025,6 +1025,34 @@ int change_dir(const char *dir, int set_path_only)
 	return 1;
 }
 
+/* This will make a relative path absolute and clean it up via clean_fname().
+ * Returns the string, which might be newly allocated, or NULL on error. */
+char *normalize_path(char *path, BOOL force_newbuf, unsigned int *len_ptr)
+{
+	unsigned int len;
+
+	if (*path != '/') { /* Make path absolute. */
+		int len = strlen(path);
+		if (curr_dir_len + 1 + len >= sizeof curr_dir)
+			return NULL;
+		curr_dir[curr_dir_len] = '/';
+		memcpy(curr_dir + curr_dir_len + 1, path, len + 1);
+		if (!(path = strdup(curr_dir)))
+			out_of_memory("normalize_path");
+		curr_dir[curr_dir_len] = '\0';
+	} else if (force_newbuf) {
+		if (!(path = strdup(path)))
+			out_of_memory("normalize_path");
+	}
+
+	len = clean_fname(path, CFN_COLLAPSE_DOT_DOT_DIRS | CFN_DROP_TRAILING_DOT_DIR);
+
+	if (len_ptr)
+		*len_ptr = len;
+
+	return path;
+}
+
 /**
  * Return a quoted string with the full pathname of the indicated filename.
  * The string " (in MODNAME)" may also be appended.  The returned pointer
@@ -1112,12 +1140,16 @@ int handle_partial_dir(const char *fname, int create)
 		STRUCT_STAT st;
 		int statret = do_lstat(dir, &st);
 		if (statret == 0 && !S_ISDIR(st.st_mode)) {
-			if (do_unlink(dir) < 0)
+			if (do_unlink(dir) < 0) {
+				*fn = '/';
 				return 0;
+			}
 			statret = -1;
 		}
-		if (statret < 0 && do_mkdir(dir, 0700) < 0)
+		if (statret < 0 && do_mkdir(dir, 0700) < 0) {
+			*fn = '/';
 			return 0;
+		}
 	} else
 		do_rmdir(dir);
 	*fn = '/';
@@ -1189,6 +1221,28 @@ int unsafe_symlink(const char *dest, const char *src)
 	return (depth < 0);
 }
 
+#define HUMANIFY(mult) \
+	do { \
+		if (num >= mult || num <= -mult) { \
+			double dnum = (double)num / mult; \
+			char units; \
+			if (num < 0) \
+				dnum = -dnum; \
+			if (dnum < mult) \
+				units = 'K'; \
+			else if ((dnum /= mult) < mult) \
+				units = 'M'; \
+			else { \
+				dnum /= mult; \
+				units = 'G'; \
+			} \
+			if (num < 0) \
+				dnum = -dnum; \
+			snprintf(bufs[n], sizeof bufs[0], "%.2f%c", dnum, units); \
+			return bufs[n]; \
+		} \
+	} while (0)
+
 /* Return the int64 number as a string.  If the --human-readable option was
  * specified, we may output the number in K, M, or G units.  We can return
  * up to 4 buffers at a time. */
@@ -1197,27 +1251,15 @@ char *human_num(int64 num)
 	static char bufs[4][128]; /* more than enough room */
 	static unsigned int n;
 	char *s;
+	int negated;
 
 	n = (n + 1) % (sizeof bufs / sizeof bufs[0]);
 
 	if (human_readable) {
-		char units = '\0';
-		int mult = human_readable == 1 ? 1000 : 1024;
-		double dnum = 0;
-		if (num > mult*mult*mult) {
-			dnum = (double)num / (mult*mult*mult);
-			units = 'G';
-		} else if (num > mult*mult) {
-			dnum = (double)num / (mult*mult);
-			units = 'M';
-		} else if (num > mult) {
-			dnum = (double)num / mult;
-			units = 'K';
-		}
-		if (units) {
-			snprintf(bufs[n], sizeof bufs[0], "%.2f%c", dnum, units);
-			return bufs[n];
-		}
+		if (human_readable == 1)
+			HUMANIFY(1000);
+		else
+			HUMANIFY(1024);
 	}
 
 	s = bufs[n] + sizeof bufs[0] - 1;
@@ -1225,10 +1267,23 @@ char *human_num(int64 num)
 
 	if (!num)
 		*--s = '0';
+	if (num < 0) {
+		/* A maximum-size negated number can't fit as a positive,
+		 * so do one digit in negated form to start us off. */
+		*--s = (char)(-(num % 10)) + '0';
+		num = -(num / 10);
+		negated = 1;
+	} else
+		negated = 0;
+
 	while (num) {
 		*--s = (char)(num % 10) + '0';
 		num /= 10;
 	}
+
+	if (negated)
+		*--s = '-';
+
 	return s;
 }
 
@@ -1241,8 +1296,8 @@ char *human_dnum(double dnum, int decimal_digits)
 	int len = strlen(buf);
 	if (isDigit(buf + len - 1)) {
 		/* There's extra room in buf prior to the start of the num. */
-		buf -= decimal_digits + 1;
-		snprintf(buf, len + decimal_digits + 2, "%.*f", decimal_digits, dnum);
+		buf -= decimal_digits + 2;
+		snprintf(buf, len + decimal_digits + 3, "%.*f", decimal_digits, dnum);
 	}
 	return buf;
 }
@@ -1566,6 +1621,39 @@ int bitbag_next_bit(struct bitbag *bb, int after)
 	return -1;
 }
 
+void flist_ndx_push(flist_ndx_list *lp, int ndx)
+{
+	struct flist_ndx_item *item;
+
+	if (!(item = new(struct flist_ndx_item)))
+		out_of_memory("flist_ndx_push");
+	item->next = NULL;
+	item->ndx = ndx;
+	if (lp->tail)
+		lp->tail->next = item;
+	else
+		lp->head = item;
+	lp->tail = item;
+}
+
+int flist_ndx_pop(flist_ndx_list *lp)
+{
+	struct flist_ndx_item *next;
+	int ndx;
+
+	if (!lp->head)
+		return -1;
+
+	ndx = lp->head->ndx;
+	next = lp->head->next;
+	free(lp->head);
+	lp->head = next;
+	if (!next)
+		lp->tail = NULL;
+
+	return ndx;
+}
+
 void *expand_item_list(item_list *lp, size_t item_size,
 		       const char *desc, int incr)
 {

xattrs.c

@@ -210,7 +210,7 @@ static int rsync_xal_get(const char *fname, item_list *xalp)
 	size_t datum_len, name_offset;
 	char *name, *ptr;
 #ifdef HAVE_LINUX_XATTRS
-	int user_only = am_sender ? 0 : !am_root;
+	int user_only = am_sender ? 0 : am_root <= 0;
 #endif
 	rsync_xa *rxa;
 	int count;
@@ -294,7 +294,7 @@ int copy_xattrs(const char *source, const char *dest)
 	size_t datum_len;
 	char *name, *ptr;
 #ifdef HAVE_LINUX_XATTRS
-	int user_only = am_sender ? 0 : !am_root;
+	int user_only = am_root <= 0;
 #endif
 
 	/* This puts the name list into the "namebuf" buffer. */
@@ -745,6 +745,9 @@ static int rsync_xal_set(const char *fname, item_list *xalp,
 	ssize_t list_len;
 	size_t i, len;
 	char *name, *ptr, sum[MAX_DIGEST_LEN];
+#ifdef HAVE_LINUX_XATTRS
+	int user_only = am_root <= 0;
+#endif
 	size_t name_len;
 	int ret = 0;
 
@@ -820,8 +823,8 @@ static int rsync_xal_set(const char *fname, item_list *xalp,
 #ifdef HAVE_LINUX_XATTRS
 		/* We always ignore the system namespace, and non-root
 		 * ignores everything but the user namespace. */
-		if (am_root ? HAS_PREFIX(name, SYSTEM_PREFIX)
-			    : !HAS_PREFIX(name, USER_PREFIX))
+		if (user_only ? !HAS_PREFIX(name, USER_PREFIX)
+			      : HAS_PREFIX(name, SYSTEM_PREFIX))
 			continue;
 #endif
 		if (am_root < 0 && name_len > RPRE_LEN