Add 4 KiB logical block footprint tracking to stats (#1070)

* Add 4 KiB logical block footprint tracking to stats

This adds a 'Number of 4 KiB logical blocks touched' metric to the --stats output to decouple network delta payload from actual local file modifications. Previously, a small amount of literal data scattered across a file (especially with --inplace) could result in a massive number of local file write operations with no visibility, and large sparse files masked their true write opreations (ignoring punch holes).

Technical details:
- Implemented a stateful block tracker in the receiver that calculates touched 4K boundaries using file offsets and lengths, including strict lseek awareness to accurately skip sparse file holes.
- Enforced strict per-file lifetime state with a reset hook inside receive_data(), successfully mitigating POSIX file descriptor (FD) recycling state leaks.
- Created MSG_BLOCK_STATS multiplex message to tunnel the block footprint safely out of the isolated receiver process and relay it over the network.
- Bumped PROTOCOL_VERSION to 33 and SUBPROTOCOL_VERSION to 8392 for safe PR testing.
- Added test suite covering contiguous, scattered, zero-byte, sparse file (hole-skipping), multi-file (FD reuse), batch mode, and maximum-I/O boundary conditions.

* io: fix block stats integration

---------

Co-authored-by: Zen Dodd <mail@steadytao.com>

.github/workflows/ubuntu-22.04-build.yml

@@ -46,7 +46,7 @@ jobs:
     - name: check
       run: sudo RSYNC_EXPECT_SKIPPED=@testsuite/skiplist/common.txt,@testsuite/skiplist/linux.txt make check
     - name: check30
-      run: sudo RSYNC_EXPECT_SKIPPED=@testsuite/skiplist/common.txt,@testsuite/skiplist/linux.txt make check30
+      run: sudo RSYNC_EXPECT_SKIPPED=@testsuite/skiplist/common.txt,@testsuite/skiplist/linux.txt,@testsuite/skiplist/proto30.txt make check30
     - name: check29
       run: sudo RSYNC_EXPECT_SKIPPED=@testsuite/skiplist/common.txt,@testsuite/skiplist/linux.txt,@testsuite/skiplist/proto29.txt make check29
     - name: check (TCP daemon transport)

.github/workflows/ubuntu-build.yml

@@ -42,7 +42,7 @@ jobs:
     - name: check
       run: sudo RSYNC_EXPECT_SKIPPED=@testsuite/skiplist/common.txt,@testsuite/skiplist/linux.txt make check
     - name: check30
-      run: sudo RSYNC_EXPECT_SKIPPED=@testsuite/skiplist/common.txt,@testsuite/skiplist/linux.txt make check30
+      run: sudo RSYNC_EXPECT_SKIPPED=@testsuite/skiplist/common.txt,@testsuite/skiplist/linux.txt,@testsuite/skiplist/proto30.txt make check30
     - name: check29
       run: sudo RSYNC_EXPECT_SKIPPED=@testsuite/skiplist/common.txt,@testsuite/skiplist/linux.txt,@testsuite/skiplist/proto29.txt make check29
     - name: check (TCP daemon transport)

fileio.c

@@ -34,12 +34,16 @@
 #define ALIGNED_LENGTH(len) ((((len) - 1) | (ALIGN_BOUNDARY-1)) + 1)
 
 extern int sparse_files;
+extern struct stats stats;
 
 OFF_T preallocated_len = 0;
 
 static OFF_T sparse_seek = 0;
 static OFF_T sparse_past_write = 0;
 
+static int last_tracked_fd = -1;
+static int64 last_touched_blk = -1;
+
 int sparse_end(int f, OFF_T size, int updating_basis_or_equiv)
 {
 	int ret = 0;
@@ -161,6 +165,8 @@ static int write_sparse(int f, int use_seek, OFF_T offset, const char *buf, int
 			continue;
 		}
 		if (i > start) {
+			if (!use_seek)
+				track_block_touches(f, offset + start, i - start);
 			if (emit_sparse_span(f, use_seek, buf + start, i - start) < 0)
 				return -1;
 			sparse_past_write = offset + i;
@@ -170,6 +176,8 @@ static int write_sparse(int f, int use_seek, OFF_T offset, const char *buf, int
 		start = i;
 	}
 	if (end > start) {
+		if (!use_seek)
+			track_block_touches(f, offset + start, end - start);
 		if (emit_sparse_span(f, use_seek, buf + start, end - start) < 0)
 			return -1;
 	}
@@ -201,6 +209,38 @@ int flush_write_file(int f)
 	return ret;
 }
 
+void reset_block_tracker(void)
+{
+	last_tracked_fd = -1;
+	last_touched_blk = -1;
+}
+
+void track_block_touches(int f, OFF_T offset, int32 len)
+{
+	int64 start_blk, end_blk, blocks_to_add = 0;
+
+	if (len <= 0)
+		return;
+	if (f != last_tracked_fd) {
+		last_tracked_fd = f;
+		last_touched_blk = -1;
+	}
+	start_blk = offset / 4096;
+	end_blk = start_blk + (((offset % 4096) + len - 1) / 4096);
+	if (start_blk > last_touched_blk)
+		blocks_to_add = end_blk - start_blk + 1;
+	else if (end_blk > last_touched_blk)
+		blocks_to_add = end_blk - last_touched_blk;
+	if (blocks_to_add > 0) {
+		if (INT64_MAX - stats.touched_blocks_4k < blocks_to_add)
+			stats.touched_blocks_4k = INT64_MAX;
+		else
+			stats.touched_blocks_4k += blocks_to_add;
+	}
+	if (end_blk > last_touched_blk)
+		last_touched_blk = end_blk;
+}
+
 /* write_file does not allow incomplete writes.  It loops internally
  * until len bytes are written or errno is set.  Note that use_seek and
  * offset are only used in sparse processing (see write_sparse()). */
@@ -208,6 +248,8 @@ int write_file(int f, int use_seek, OFF_T offset, const char *buf, int len)
 {
 	int ret = 0;
 
+	if (!use_seek && sparse_files == 0)
+		track_block_touches(f, offset, len);
 	while (len > 0) {
 		int r1;
 		if (sparse_files > 0) {

io.c

@@ -1718,6 +1718,18 @@ static void read_a_msg(void)
 		raw_read_buf((char*)&stats.total_read, sizeof stats.total_read);
 		iobuf.in_multiplexed = 1;
 		break;
+	case MSG_BLOCK_STATS: {
+		char b[8];
+
+		if (msg_bytes != 8 || protocol_version < 33 || (!am_generator && !am_sender))
+			goto invalid_msg;
+		raw_read_buf(b, 8);
+		stats.touched_blocks_4k = IVAL64(b, 0);
+		iobuf.in_multiplexed = 1;
+		if (am_server && am_generator)
+			send_msg(MSG_BLOCK_STATS, b, sizeof b, 0);
+		break;
+	}
 	case MSG_REDO:
 		if (msg_bytes != 4 || !am_generator)
 			goto invalid_msg;

main.c

@@ -443,6 +443,9 @@ static void output_summary(void)
 			human_num(stats.total_transferred_size));
 		rprintf(FINFO,"Literal data: %s bytes\n",
 			human_num(stats.literal_data));
+		if (protocol_version >= 33)
+			rprintf(FINFO,"Number of 4 KiB logical blocks touched: %s\n",
+				comma_num(stats.touched_blocks_4k));
 		rprintf(FINFO,"Matched data: %s bytes\n",
 			human_num(stats.matched_data));
 		rprintf(FINFO,"File list size: %s\n",
@@ -1107,6 +1110,11 @@ static int do_recv(int f_in, int f_out, char *local_name)
 
 		write_int(f_out, NDX_DONE);
 		send_msg(MSG_STATS, (char*)&stats.total_read, sizeof stats.total_read, 0);
+		if (protocol_version >= 33) {
+			char b[8];
+			SIVAL64(b, 0, stats.touched_blocks_4k);
+			send_msg(MSG_BLOCK_STATS, b, sizeof b, 0);
+		}
 		io_flush(FULL_FLUSH);
 
 		/* Handle any keep-alive packets from the post-processing work

receiver.c

@@ -485,6 +485,9 @@ static int receive_data(int f_in, char *fname_r, int fd_r, OFF_T size_r,
 	int32 i;
 	char *map = NULL;
 
+	/* Reset the block tracker's per-file state in case the OS reuses an fd. */
+	reset_block_tracker();
+
 #ifdef SUPPORT_PREALLOCATION
 	if (preallocate_files && fd != -1 && total_size > 0 && (!inplace_sizing || total_size > size_r)) {
 		/* Try to preallocate enough space for file's eventual length.  Can

rsync.1.md

@@ -3551,6 +3551,12 @@ sign) if you want the local shell to expand it.
       Note that this line is only output if deletions are in effect, and only
       if the negotiated protocol is at least 31 (the default when both sides
       are 3.1.0, September 2013, or newer).
+    - `Number of 4 KiB logical blocks touched` is the number of unique 4 KiB
+      logical file regions covered by rsync's own write operations. This metric
+      decouples network payload from the scope of local file modifications.
+      For example, a small amount of *Literal data* can result in a massive
+      number of touched blocks if the modifications are highly scattered
+      across a file, especially when using `--inplace`.
     - `Number of regular files transferred` is the count of normal files that
       were updated via rsync's delta-transfer algorithm, which does not include
       directories, symlinks, etc.

rsync.h

@@ -111,7 +111,7 @@
 
 /* Update this if you make incompatible changes and ALSO update the
  * SUBPROTOCOL_VERSION if it is not a final (official) release. */
-#define PROTOCOL_VERSION 32
+#define PROTOCOL_VERSION 33
 
 /* This is used when working on a new protocol version or for any unofficial
  * protocol tweaks.  It should be a non-zero value for each pre-release repo
@@ -125,7 +125,7 @@
  * All older protocol versions MUST be compatible with the final, official
  * release of the protocol, so don't tweak the code to change the protocol
  * behavior for an older protocol version. */
-#define SUBPROTOCOL_VERSION 0
+#define SUBPROTOCOL_VERSION 8392 /* For testing */
 
 /* We refuse to interoperate with versions that are not in this range.
  * Note that we assume we'll work with later versions: the onus is on
@@ -299,6 +299,7 @@ enum msgcode {
 	MSG_LOG=FLOG, MSG_CLIENT=FCLIENT, /* sibling logging */
 	MSG_REDO=9,	/* reprocess indicated flist index */
 	MSG_STATS=10,	/* message has stats data for generator */
+	MSG_BLOCK_STATS=11,/* message has block-level stats for sender */
 	MSG_IO_ERROR=22,/* the sending side had an I/O error */
 	MSG_IO_TIMEOUT=33,/* tell client about a daemon's timeout value */
 	MSG_NOOP=42,	/* a do-nothing message (legacy protocol-30 only) */
@@ -1080,6 +1081,7 @@ struct stats {
 	int64 total_read;
 	int64 literal_data;
 	int64 matched_data;
+	int64 touched_blocks_4k;
 	int64 flist_buildtime;
 	int64 flist_xfertime;
 	int64 flist_size;

testsuite/skiplist/README.md

@@ -27,6 +27,7 @@ different tests merge cleanly.
 | `macos.txt` | macOS-only additions |
 | `cygwin.txt` | Cygwin-only additions |
 | `proto29.txt` | additions for a `--protocol=29` run, on any platform |
+| `proto30.txt` | additions for a `--protocol=30` run, on any platform |
 
 Compose them with commas; the result is the union, so listing a test twice is
 harmless. Plain test names may be mixed in with `@FILE` entries.

testsuite/skiplist/proto29.txt

@@ -14,3 +14,4 @@ daemon-copylinks-parent-target-regression  # the stdio_daemon client speaks prot
 partial-protected-regular-retry-linux  # one-inplace partial staging needs protocol >= 30 (forced 29)
 scanner-batch-flag-mismatch  # xattrs (-X) need protocol 30+
 symlink-exclude-xattr  # xattr (-X) transfer requires protocol 30+ (negotiated 29)
+write-touched-blocks  # logical-block statistics require protocol 33+ (forced 29)

testsuite/skiplist/proto30.txt

@@ -0,0 +1,9 @@
+# Tests expected to SKIP.  One name per line, '#' starts a comment; the file
+# must stay sorted and duplicate-free (runtests.py enforces both).  Referenced
+# from a workflow as RSYNC_EXPECT_SKIPPED=@testsuite/skiplist/<file>[,@...].
+# See testsuite/skiplist/README.md.
+#
+# Additions for a --protocol=30 run (make check30), on top of the platform
+# files.
+
+write-touched-blocks  # logical-block statistics require protocol 33+ (forced 30)

testsuite/write-touched-blocks_test.py

@@ -0,0 +1,171 @@
+import os
+import shlex
+import subprocess
+import sys
+from pathlib import Path
+
+from rsyncfns import (
+    FROMDIR, RSYNC, SCRATCHDIR,
+    makepath, test_fail, test_skipped
+)
+
+rsync_args = shlex.split(str(RSYNC))
+for arg in rsync_args:
+    if arg.startswith('--protocol='):
+        prot_version = int(arg.split('=')[1])
+        if prot_version < 33:
+            test_skipped(f"Skipping write-touched-blocks: feature requires protocol 33, but CI forced {prot_version}")
+
+src = FROMDIR
+makepath(src)
+
+base_file = src / 'base.bin'
+
+# Generate 4 MiB of random data in memory
+data = os.urandom(4 * 1024 * 1024)
+
+def setup_test(dest_name):
+    """Resets the base file and creates a clean destination file."""
+    dest_path = SCRATCHDIR / dest_name
+    base_file.write_bytes(data)
+    dest_path.write_bytes(data)
+    return dest_path
+
+def run_client(src_path, dest_path):
+    rsync_cmd = shlex.split(str(RSYNC))
+    argv = rsync_cmd + ['-a', '--stats', '--inplace', '-I', '--no-whole-file',
+            str(src_path), str(dest_path)]
+    return subprocess.run(argv, stdout=subprocess.PIPE, stderr=subprocess.STDOUT,
+                          text=True)
+
+# TEST 1: Contiguous Write (1 Block)
+dest_contig = setup_test('dest_contiguous.bin')
+with open(base_file, 'r+b') as f:
+    f.write(b'\x00' * 3000)
+
+proc = run_client(base_file, dest_contig)
+if proc.returncode != 0:
+    test_fail(f"rsync failed on contiguous test:\n{proc.stdout}")
+if "Number of 4 KiB logical blocks touched: 1\n" not in proc.stdout:
+    test_fail(f"Contiguous check failed! Expected 1 block. Output:\n{proc.stdout}")
+
+# TEST 2: Scattered Write (10 Blocks)
+dest_scatter = setup_test('dest_scattered.bin')
+with open(base_file, 'r+b') as f:
+    for i in range(1, 11):
+        f.seek(i * 4096)
+        old_byte = f.read(1)[0]
+        f.seek(i * 4096)
+        f.write(bytes([old_byte ^ 0xFF]))
+
+proc = run_client(base_file, dest_scatter)
+if proc.returncode != 0:
+    test_fail(f"rsync failed on scattered test:\n{proc.stdout}")
+if "Number of 4 KiB logical blocks touched: 10\n" not in proc.stdout:
+    test_fail(f"Scattered check failed! Expected 10 blocks. Output:\n{proc.stdout}")
+
+# TEST 3: Identical Files (0 Blocks Edge Case)
+dest_zero = setup_test('dest_zero.bin')
+
+proc = run_client(base_file, dest_zero)
+if proc.returncode != 0:
+    test_fail(f"rsync failed on zero-block test:\n{proc.stdout}")
+if "Number of 4 KiB logical blocks touched: 0\n" not in proc.stdout:
+    test_fail(f"Zero check failed! Expected 0 blocks. Output:\n{proc.stdout}")
+
+# TEST 4: Full File Write (1,024 Blocks)
+dest_full = setup_test('dest_full.bin')
+# Overwrite the entire 4 MiB base file with brand new random data
+# This forces the delta algorithm to find 0 matches and write all 1,024 blocks.
+base_file.write_bytes(os.urandom(4 * 1024 * 1024))
+
+proc = run_client(base_file, dest_full)
+if proc.returncode != 0:
+    test_fail(f"rsync failed on full write test:\n{proc.stdout}")
+if "Number of 4 KiB logical blocks touched: 1,024\n" not in proc.stdout:
+    test_fail(f"Full write check failed! Expected 1,024 blocks. Output:\n{proc.stdout}")
+
+# TEST 5: Sparse File Write (Hole skipping)
+sparse_src = src / 'sparse.bin'
+sparse_dest = SCRATCHDIR / 'sparse_dest.bin'
+
+# Create a file with written data on the ends, but a massive 4 MiB hole in the middle.
+# 1 block data + 1,024 blocks hole + 1 block data = 1,026 blocks total size.
+with open(sparse_src, 'wb') as f:
+    f.write(os.urandom(4096))             # Block 1 (Data)
+    f.seek(4 * 1024 * 1024, os.SEEK_CUR)  # The Hole (4 MiB of nothing)
+    f.write(os.urandom(4096))             # Block 1026 (Data)
+
+# We must run this WITH --sparse (-S) and WITHOUT --inplace to force
+# the receiver to create a brand new sparse file from scratch using write_sparse().
+rsync_cmd = shlex.split(str(RSYNC))
+argv_sparse = rsync_cmd + ['-a', '--stats', '--sparse', str(sparse_src), str(sparse_dest)]
+proc = subprocess.run(argv_sparse, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True)
+
+if proc.returncode != 0:
+    test_fail(f"rsync failed on sparse test:\n{proc.stdout}")
+
+# Even though the file is over 4 MiB in size, only 2 logical 4K blocks
+# should be written because the rest was skipped via lseek.
+if "Number of 4 KiB logical blocks touched: 2\n" not in proc.stdout:
+    test_fail(f"Sparse check failed! Expected 2 logical blocks written. Output:\n{proc.stdout}")
+
+# TEST 6: Multiple Files
+# Creates two separate 4KB files. If the tracker fails to reset between
+# files due to FD recycling, it will report 1 block instead of 2.
+fd_src_dir = src / 'fd_test'
+fd_dest_dir = SCRATCHDIR / 'fd_dest'
+makepath(fd_src_dir)
+makepath(fd_dest_dir)
+
+# Create two distinct 1-block files
+(fd_src_dir / 'fileA.bin').write_bytes(os.urandom(4096))
+(fd_src_dir / 'fileB.bin').write_bytes(os.urandom(4096))
+
+# Sync the whole directory so rsync processes both in one process lifespan
+rsync_cmd = shlex.split(str(RSYNC))
+argv_fd = rsync_cmd + ['-a', '--stats', '--inplace', str(fd_src_dir) + '/', str(fd_dest_dir) + '/']
+proc = subprocess.run(argv_fd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True)
+
+if proc.returncode != 0:
+    test_fail(f"rsync failed on multiple files test:\n{proc.stdout}")
+
+if "Number of 4 KiB logical blocks touched: 2\n" not in proc.stdout:
+    test_fail(f"FD Reuse bug confirmed! Expected 2 blocks (1 per file). Output:\n{proc.stdout}")
+
+# TEST 7: Batch Mode State Reset (--read-batch)
+batch_src_dir = src / 'batch_src'
+batch_dest_dir = SCRATCHDIR / 'batch_dest'
+batch_file = SCRATCHDIR / 'test_batch.rsync'
+makepath(batch_src_dir)
+makepath(batch_dest_dir)
+
+# Create two 4KB source files with random data
+(batch_src_dir / 'fileA.bin').write_bytes(os.urandom(4096))
+(batch_src_dir / 'fileB.bin').write_bytes(os.urandom(4096))
+
+# Create two 4KB zeroed destination files (forces the delta algorithm to write exactly 1 block per file)
+(batch_dest_dir / 'fileA.bin').write_bytes(b'\x00' * 4096)
+(batch_dest_dir / 'fileB.bin').write_bytes(b'\x00' * 4096)
+
+# Step 1: Generate the batch file (Sender side)
+# We MUST use '-I' because the files have identical sizes and timestamps.
+rsync_cmd = shlex.split(str(RSYNC))
+argv_write_batch = rsync_cmd + ['-a', '-I', '--only-write-batch=' + str(batch_file),
+                    str(batch_src_dir) + '/', str(batch_dest_dir) + '/']
+subprocess.run(argv_write_batch, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
+
+# Step 2: Apply the batch file and collect stats (Receiver side)
+argv_read_batch = rsync_cmd + ['-a', '-I', '--inplace', '--stats',
+                   '--read-batch=' + str(batch_file), str(batch_dest_dir) + '/']
+
+proc = subprocess.run(argv_read_batch, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True)
+
+if proc.returncode != 0:
+    test_fail(f"rsync failed on read-batch test:\n{proc.stdout}")
+
+# If the state leaked across the batch read, this would output 1 block.
+if "Number of 4 KiB logical blocks touched: 2\n" not in proc.stdout:
+    test_fail(f"Batch Mode tracker check failed! Expected 2 blocks (1 per file). Output:\n{proc.stdout}")
+
+print("write-touched-blocks: cleanly distinguishes contiguous, scattered, zero, full, sparse, multi-file, and batch writes")