fileio: coalesce --sparse writes instead of 1-KiB dribbles (issue #773)

write_file()'s sparse path sliced each span into SPARSE_WRITE_SIZE (1024-byte)
pieces and write_sparse() issued one write() syscall per slice.  Copying a
large *non-sparse* file with --sparse therefore cost roughly one write() per
kilobyte -- about a million write() calls for a 1 GiB file -- which on real
storage ran far slower than the same copy without --sparse (the bug report
measured 1.36 MB/s vs 391 MB/s, ~280x).  The 1024-byte chunk is also smaller
than a filesystem block, so it cannot even create finer holes than a plain
copy could.

Rewrite write_sparse() to scan the whole span itself: it looks for interior
runs of zeros that are at least SPARSE_WRITE_SIZE long -- the same hole
granularity rsync has always used -- and emits each intervening non-zero
region (which may include shorter zero runs not worth a hole) with a single
write().  do_punch_hole() advances the file offset just like the lseek() path,
so flushing a deferred hole between segments keeps the position correct.

The hole granularity is unchanged, so sparseness is identical; only the
syscall pattern changes.  Measured on a 100 MiB random (hole-free) file:
write() syscalls drop from 100,730 to 6,125 (~16x), now tracking the data's
natural chunking rather than its size in kilobytes.  Verified byte-identical
and equally sparse output for hole-free, large-hole, small-interior-hole,
all-zero, --inplace, and --preallocate cases.

testsuite/sparse-write-count_test.py copies a 16 MiB hole-free file under
strace and asserts the write() count stays far below the old size/1024
behaviour (it skips where strace is unavailable).

(cherry picked from commit 5ecab683a8)
(cherry picked from commit 6a15079cab)

fileio.c

@@ -71,11 +71,47 @@ int sparse_end(int f, OFF_T size)
 /* Note that the offset is just the caller letting us know where
  * the current file position is in the file. The use_seek arg tells
  * us that we should seek over matching data instead of writing it. */
+/* Flush any deferred run of zero bytes as a hole, advancing the file
+ * position past it (both do_lseek() and do_punch_hole() move the offset). */
+static int flush_sparse_hole(int f)
+{
+	if (!sparse_seek)
+		return 0;
+	if (sparse_past_write >= preallocated_len) {
+		if (do_lseek(f, sparse_seek, SEEK_CUR) < 0) {
+			sparse_seek = 0;
+			return -1;
+		}
+	} else if (do_punch_hole(f, sparse_past_write, sparse_seek) < 0) {
+		sparse_seek = 0;
+		return -1;
+	}
+	sparse_seek = 0;
+	return 0;
+}
+
+static int full_sparse_write(int f, const char *buf, int len)
+{
+	while (len > 0) {
+		int ret = write(f, buf, len);
+		if (ret <= 0) {
+			if (ret < 0 && errno == EINTR)
+				continue;
+			sparse_seek = 0;
+			return -1;
+		}
+		buf += ret;
+		len -= ret;
+	}
+	return 0;
+}
+
 static int write_sparse(int f, int use_seek, OFF_T offset, const char *buf, int len)
 {
-	int l1 = 0, l2 = 0;
-	int ret;
+	int l1, l2, i, start, end;
 
+	/* Always treat a leading and trailing run of zeros as a (deferred)
+	 * hole, since they may merge with holes in the adjacent write calls. */
 	for (l1 = 0; l1 < len && buf[l1] == 0; l1++) {}
 	for (l2 = 0; l2 < len-l1 && buf[len-(l2+1)] == 0; l2++) {}
 
@@ -84,36 +120,57 @@ static int write_sparse(int f, int use_seek, OFF_T offset, const char *buf, int
 	if (l1 == len)
 		return len;
 
-	if (sparse_seek) {
-		if (sparse_past_write >= preallocated_len) {
-			if (do_lseek(f, sparse_seek, SEEK_CUR) < 0)
-				return -1;
-		} else if (do_punch_hole(f, sparse_past_write, sparse_seek) < 0) {
-			sparse_seek = 0;
-			return -1;
-		}
-	}
-	sparse_seek = l2;
-	sparse_past_write = offset + len - l2;
-
 	if (use_seek) {
-		/* The in-place data already matches. */
+		/* The in-place data already matches, so just flush any pending
+		 * hole and seek over the middle without rescanning it. */
+		if (flush_sparse_hole(f) < 0)
+			return -1;
+		sparse_seek = l2;
+		sparse_past_write = offset + len - l2;
 		if (do_lseek(f, len - (l1+l2), SEEK_CUR) < 0)
 			return -1;
 		return len;
 	}
 
-	while ((ret = write(f, buf + l1, len - (l1+l2))) <= 0) {
-		if (ret < 0 && errno == EINTR)
+	/* Scan the middle [l1, len-l2) for interior runs of zeros that are at
+	 * least SPARSE_WRITE_SIZE long (the same hole granularity rsync has
+	 * always used).  Each non-zero span -- which may include shorter zero
+	 * runs not worth a hole -- is emitted with a single write() rather than
+	 * being chopped into SPARSE_WRITE_SIZE-byte writes, which made a copy of
+	 * a large non-sparse file issue ~one write() syscall per KiB. */
+	start = l1;
+	end = len - l2;
+	for (i = l1; i < end; ) {
+		int z;
+		if (buf[i] != 0) {
+			i++;
 			continue;
-		sparse_seek = 0;
-		return ret;
+		}
+		for (z = 1; i + z < end && buf[i+z] == 0; z++) {}
+		if (z < SPARSE_WRITE_SIZE) {
+			i += z;
+			continue;
+		}
+		if (i > start) {
+			if (flush_sparse_hole(f) < 0)
+				return -1;
+			if (full_sparse_write(f, buf + start, i - start) < 0)
+				return -1;
+			sparse_past_write = offset + i;
+		}
+		sparse_seek += z;
+		i += z;
+		start = i;
+	}
+	if (end > start) {
+		if (flush_sparse_hole(f) < 0)
+			return -1;
+		if (full_sparse_write(f, buf + start, end - start) < 0)
+			return -1;
 	}
 
-	if (ret != (int)(len - (l1+l2))) {
-		sparse_seek = 0;
-		return l1+ret;
-	}
+	sparse_seek = l2;
+	sparse_past_write = offset + len - l2;
 
 	return len;
 }
@@ -149,8 +206,10 @@ int write_file(int f, int use_seek, OFF_T offset, const char *buf, int len)
 	while (len > 0) {
 		int r1;
 		if (sparse_files > 0) {
-			int len1 = MIN(len, SPARSE_WRITE_SIZE);
-			r1 = write_sparse(f, use_seek, offset, buf, len1);
+			/* write_sparse() handles the whole span itself, scanning
+			 * for holes and coalescing the non-zero data into large
+			 * write()s instead of SPARSE_WRITE_SIZE-byte dribbles. */
+			r1 = write_sparse(f, use_seek, offset, buf, len);
 			offset += r1;
 		} else {
 			if (!wf_writeBuf) {