Use a varint to send the file-list flags

If both sides support the "V" compatibility flag, we send the file-list
flags as a varint instead of a 1-or-2 byte value.  This upgrades the
number of reserved flag bits from 1 to 17 with very few extra bytes in
typical file-list data.

compat.c

@@ -52,6 +52,7 @@ extern int preserve_gid;
 extern int preserve_atimes;
 extern int preserve_acls;
 extern int preserve_xattrs;
+extern int xfer_flags_as_varint;
 extern int need_messages_from_generator;
 extern int delete_mode, delete_before, delete_during, delete_after;
 extern char *shell_cmd;
@@ -83,6 +84,7 @@ int filesfrom_convert = 0;
 #define CF_AVOID_XATTR_OPTIM (1<<4)
 #define CF_CHKSUM_SEED_FIX (1<<5)
 #define CF_INPLACE_PARTIAL_DIR (1<<6)
+#define CF_VARINT_FLIST_FLAGS (1<<7)
 
 static const char *client_info;
 
@@ -287,6 +289,8 @@ void setup_protocol(int f_out,int f_in)
 				compat_flags |= CF_CHKSUM_SEED_FIX;
 			if (local_server || strchr(client_info, 'I') != NULL)
 				compat_flags |= CF_INPLACE_PARTIAL_DIR;
+			if (local_server || strchr(client_info, 'V') != NULL)
+				compat_flags |= CF_VARINT_FLIST_FLAGS;
 			write_byte(f_out, compat_flags);
 		} else
 			compat_flags = read_byte(f_in);
@@ -294,6 +298,7 @@ void setup_protocol(int f_out,int f_in)
 		inc_recurse = compat_flags & CF_INC_RECURSE ? 1 : 0;
 		want_xattr_optim = protocol_version >= 31 && !(compat_flags & CF_AVOID_XATTR_OPTIM);
 		proper_seed_order = compat_flags & CF_CHKSUM_SEED_FIX ? 1 : 0;
+		xfer_flags_as_varint = compat_flags & CF_VARINT_FLIST_FLAGS ? 1 : 0;
 		if (am_sender) {
 			receiver_symlink_times = am_server
 			    ? strchr(client_info, 'L') != NULL

flist.c

@@ -101,6 +101,7 @@ int flist_cnt = 0; /* how many (non-tmp) file list objects exist */
 int file_total = 0; /* total of all active items over all file-lists */
 int file_old_total = 0; /* total of active items that will soon be gone */
 int flist_eof = 0; /* all the file-lists are now known */
+int xfer_flags_as_varint = 0;
 
 #define NORMAL_NAME 0
 #define SLASH_ENDING_NAME 1
@@ -528,11 +529,14 @@ static void send_file_entry(int f, const char *fname, struct file_struct *file,
 	if (l2 > 255)
 		xflags |= XMIT_LONG_NAME;
 
-	/* We must make sure we don't send a zero flag byte or the
-	 * other end will terminate the flist transfer.  Note that
-	 * the use of XMIT_TOP_DIR on a non-dir has no meaning, so
-	 * it's harmless way to add a bit to the first flag byte. */
-	if (protocol_version >= 28) {
+	/* We must avoid sending a flag value of 0 (or an initial byte of
+	 * 0 for the older xflags protocol) or it will signal the end of
+	 * the list.  Note that the use of XMIT_TOP_DIR on a non-dir has
+	 * no meaning, so it's a harmless way to add a bit to the first
+	 * flag byte. */
+	if (xfer_flags_as_varint)
+		write_varint(f, xflags ? xflags : XMIT_EXTENDED_FLAGS);
+	else if (protocol_version >= 28) {
 		if (!xflags && !S_ISDIR(mode))
 			xflags |= XMIT_TOP_DIR;
 		if ((xflags & 0xFF00) || !xflags) {
@@ -1969,6 +1973,18 @@ static void send1extra(int f, struct file_struct *file, struct file_list *flist)
 	free(relname_list);
 }
 
+static void write_end_of_flist(int f, int send_io_error)
+{
+	if (xfer_flags_as_varint) {
+		write_varint(f, 0);
+		write_varint(f, send_io_error ? io_error : 0);
+	} else if (send_io_error) {
+		write_shortint(f, XMIT_EXTENDED_FLAGS|XMIT_IO_ERROR_ENDLIST);
+		write_varint(f, io_error);
+	} else
+		write_byte(f, 0);
+}
+
 void send_extra_file_list(int f, int at_least)
 {
 	struct file_list *flist;
@@ -2020,14 +2036,13 @@ void send_extra_file_list(int f, int at_least)
 		}
 
 		if (io_error == save_io_error || ignore_errors)
-			write_byte(f, 0);
+			write_end_of_flist(f, 0);
 		else if (use_safe_inc_flist) {
-			write_shortint(f, XMIT_EXTENDED_FLAGS|XMIT_IO_ERROR_ENDLIST);
-			write_varint(f, io_error);
+			write_end_of_flist(f, 1);
 		} else {
 			if (delete_during)
 				fatal_unsafe_io_error();
-			write_byte(f, 0);
+			write_end_of_flist(f, 0);
 		}
 
 		if (need_unsorted_flist) {
@@ -2356,14 +2371,13 @@ struct file_list *send_file_list(int f, int argc, char *argv[])
 
 	/* Indicate end of file list */
 	if (io_error == 0 || ignore_errors)
-		write_byte(f, 0);
-	else if (use_safe_inc_flist) {
-		write_shortint(f, XMIT_EXTENDED_FLAGS|XMIT_IO_ERROR_ENDLIST);
-		write_varint(f, io_error);
-	} else {
+		write_end_of_flist(f, 0);
+	else if (use_safe_inc_flist)
+		write_end_of_flist(f, 1);
+	else {
 		if (delete_during && inc_recurse)
 			fatal_unsafe_io_error();
-		write_byte(f, 0);
+		write_end_of_flist(f, 0);
 	}
 
 #ifdef SUPPORT_HARD_LINKS
@@ -2482,22 +2496,34 @@ struct file_list *recv_file_list(int f, int dir_ndx)
 		dstart = 0;
 	}
 
-	while ((flags = read_byte(f)) != 0) {
+	while (1) {
 		struct file_struct *file;
 
-		if (protocol_version >= 28 && (flags & XMIT_EXTENDED_FLAGS))
-			flags |= read_byte(f) << 8;
-
-		if (flags == (XMIT_EXTENDED_FLAGS|XMIT_IO_ERROR_ENDLIST)) {
-			int err;
-			if (!use_safe_inc_flist) {
-				rprintf(FERROR, "Invalid flist flag: %x\n", flags);
-				exit_cleanup(RERR_PROTOCOL);
+		if (xfer_flags_as_varint) {
+			if ((flags = read_varint(f)) == 0) {
+				int err = read_varint(f);
+				if (!ignore_errors)
+					io_error |= err;
+				break;
+			}
+		} else {
+			if ((flags = read_byte(f)) == 0)
+				break;
+
+			if (protocol_version >= 28 && (flags & XMIT_EXTENDED_FLAGS))
+				flags |= read_byte(f) << 8;
+
+			if (flags == (XMIT_EXTENDED_FLAGS|XMIT_IO_ERROR_ENDLIST)) {
+				int err;
+				if (!use_safe_inc_flist) {
+					rprintf(FERROR, "Invalid flist flag: %x\n", flags);
+					exit_cleanup(RERR_PROTOCOL);
+				}
+				err = read_varint(f);
+				if (!ignore_errors)
+					io_error |= err;
+				break;
 			}
-			err = read_varint(f);
-			if (!ignore_errors)
-				io_error |= err;
-			break;
 		}
 
 		flist_expand(flist, 1);

io.c

@@ -2026,7 +2026,7 @@ void write_varint(int f, int32 x)
 	} else if (cnt > 1)
 		*b = b[cnt] | ~(bit*2-1);
 	else
-		*b = b[cnt];
+		*b = b[1];
 
 	write_buf(f, b, cnt);
 }

options.c

@@ -2578,6 +2578,7 @@ void server_options(char **args, int *argc_p)
 		eFlags[x++] = 'x'; /* xattr hardlink optimization not desired */
 		eFlags[x++] = 'C'; /* support checksum seed order fix */
 		eFlags[x++] = 'I'; /* support inplace_partial behavior */
+		eFlags[x++] = 'V'; /* use varint for flist flags */
 #undef eFlags
 	}
 

rsync.h

@@ -52,6 +52,7 @@
 #define XMIT_SAME_NAME (1<<5)
 #define XMIT_LONG_NAME (1<<6)
 #define XMIT_SAME_TIME (1<<7)
+
 #define XMIT_SAME_RDEV_MAJOR (1<<8)	/* protocols 28 - now (devices only) */
 #define XMIT_NO_CONTENT_DIR (1<<8)	/* protocols 30 - now (dirs only) */
 #define XMIT_HLINKED (1<<9)		/* protocols 28 - now (non-dirs) */
@@ -62,7 +63,12 @@
 #define XMIT_HLINK_FIRST (1<<12)	/* protocols 30 - now (HLINKED files only) */
 #define XMIT_IO_ERROR_ENDLIST (1<<12)	/* protocols 31*- now (w/XMIT_EXTENDED_FLAGS) (also protocol 30 w/'f' compat flag) */
 #define XMIT_MOD_NSEC (1<<13)		/* protocols 31 - now */
-#define XMIT_SAME_ATIME (1<<14) 	/* protocols ?? - now */
+#define XMIT_SAME_ATIME (1<<14) 	/* any protocol - restricted by command-line option */
+#define XMIT_RESERVED_15 (1<<15) 	/* reserved for future use */
+
+/* The following XMIT flags require an rsync that uses a varint for the flag values */
+
+#define XMIT_RESERVED_16 (1<<16) 	/* reserved for future use */
 
 /* These flags are used in the live flist data. */