Improved the incremental sending of file lists in two ways: (1)

when the sender gets an index of a file to send, we make sure to
send enough future file-list data relative to the new cur_flist
value before we send the data for the file transfer (ensuring that
the generator gets the data in a more timely manner), and (2) the
generator flushes the output socket after each file-list object
has completed its scan so that the sender knows in a more timely
manner that more list data is needed.

generator.c

@@ -1661,7 +1661,7 @@ static void touch_up_dirs(struct file_list *flist, int ndx)
 		if (allowed_lull && !(++j % lull_mod))
 			maybe_send_keepalive();
 		else if (!(j % 200))
-			maybe_flush_socket();
+			maybe_flush_socket(0);
 	}
 }
 
@@ -1725,8 +1725,10 @@ void check_for_finished_files(int itemizing, enum logcode code, int check_redo)
 		if (first_flist->in_progress || first_flist->to_redo)
 			break;
 
-		if (!read_batch)
+		if (!read_batch) {
 			write_ndx(sock_f_out, NDX_DONE);
+			maybe_flush_socket(1);
+		}
 
 		if (delete_during == 2 || !dir_tweaking) {
 			/* Skip directory touch-up. */
@@ -1847,7 +1849,7 @@ void generate_files(int f_out, const char *local_name)
 			if (allowed_lull && !(i % lull_mod))
 				maybe_send_keepalive();
 			else if (!(i % 200))
-				maybe_flush_socket();
+				maybe_flush_socket(0);
 		}
 
 		if (!inc_recurse) {

io.c

@@ -804,9 +804,10 @@ void io_end_buffering_out(void)
 	iobuf_f_out = -1;
 }
 
-void maybe_flush_socket(void)
+void maybe_flush_socket(int important)
 {
-	if (iobuf_out && iobuf_out_cnt && time(NULL) - last_io_out >= 5)
+	if (iobuf_out && iobuf_out_cnt
+	 && (important || time(NULL) - last_io_out >= 5))
 		io_flush(NORMAL_FLUSH);
 }
 

sender.c

@@ -44,6 +44,8 @@ extern int write_batch;
 extern struct stats stats;
 extern struct file_list *cur_flist, *first_flist;
 
+#define FILECNT_LOOKAHEAD 1000
+
 /**
  * @file
  *
@@ -180,7 +182,7 @@ void send_files(int f_in, int f_out)
 
 	while (1) {
 		if (inc_recurse)
-			send_extra_file_list(f_out, 1000);
+			send_extra_file_list(f_out, FILECNT_LOOKAHEAD);
 
 		/* This call also sets cur_flist. */
 		ndx = read_ndx_and_attrs(f_in, f_out, &iflags,
@@ -201,6 +203,9 @@ void send_files(int f_in, int f_out)
 			continue;
 		}
 
+		if (inc_recurse)
+			send_extra_file_list(f_out, FILECNT_LOOKAHEAD);
+
 		file = cur_flist->files[ndx - cur_flist->ndx_start];
 		if (F_ROOTDIR(file)) {
 			path = F_ROOTDIR(file);