testsuite: regression test for the hashtable integer overflow

Add t_hashtable_overflow (links the real hashtable.o, sets a realistic
--max-alloc, requests an absurd size) and a test asserting hashtable_create now
rejects it with RERR_MALLOC instead of under-allocating and crashing on the OOB
node access a regressed build would hit.  The helper defines its own
info_levels/debug_levels (as the other t_* helpers do) for the DEBUG_GTE macro
in the linked hashtable.o.

Makefile.in

@@ -58,13 +58,13 @@ TLS_OBJ = tls.o syscall.o util2.o t_stub.o lib/compat.o lib/snprintf.o lib/perms
 # Programs we must have to run the test cases
 CHECK_PROGS = rsync$(EXEEXT) tls$(EXEEXT) getgroups$(EXEEXT) getfsdev$(EXEEXT) \
 	testrun$(EXEEXT) trimslash$(EXEEXT) t_unsafe$(EXEEXT) t_chmod_secure$(EXEEXT) \
-	t_rename_secure$(EXEEXT) t_symlink_secure$(EXEEXT) t_secure_relpath$(EXEEXT) t_acl$(EXEEXT) wildtest$(EXEEXT) simdtest$(EXEEXT)
+	t_rename_secure$(EXEEXT) t_symlink_secure$(EXEEXT) t_secure_relpath$(EXEEXT) t_acl$(EXEEXT) t_hashtable_overflow$(EXEEXT) wildtest$(EXEEXT) simdtest$(EXEEXT)
 
 CHECK_SYMLINKS = testsuite/chown-fake_test.py testsuite/devices-fake_test.py \
 	testsuite/xattrs-hlink_test.py testsuite/exclude-lsh_test.py
 
 # Objects for CHECK_PROGS to clean
-CHECK_OBJS=tls.o testrun.o getgroups.o getfsdev.o t_stub.o t_unsafe.o t_chmod_secure.o t_rename_secure.o t_symlink_secure.o t_secure_relpath.o t_acl.o trimslash.o wildtest.o
+CHECK_OBJS=tls.o testrun.o getgroups.o getfsdev.o t_stub.o t_unsafe.o t_chmod_secure.o t_rename_secure.o t_symlink_secure.o t_secure_relpath.o t_acl.o t_hashtable_overflow.o trimslash.o wildtest.o
 # Compile-only feature-shape checks.
 CHECK_COMPILE_OBJS=syscall-no-at-fdcwd.o
 
@@ -209,6 +209,10 @@ T_UNSAFE_OBJ = t_unsafe.o syscall.o util1.o util2.o t_stub.o lib/compat.o lib/sn
 t_unsafe$(EXEEXT): $(T_UNSAFE_OBJ)
 	$(CC) $(CFLAGS) $(LDFLAGS) -o $@ $(T_UNSAFE_OBJ) $(LIBS)
 
+T_HASHTABLE_OVERFLOW_OBJ = t_hashtable_overflow.o hashtable.o util2.o t_stub.o lib/compat.o lib/snprintf.o lib/wildmatch.o
+t_hashtable_overflow$(EXEEXT): $(T_HASHTABLE_OVERFLOW_OBJ)
+	$(CC) $(CFLAGS) $(LDFLAGS) -o $@ $(T_HASHTABLE_OVERFLOW_OBJ) $(LIBS)
+
 T_CHMOD_SECURE_OBJ = t_chmod_secure.o syscall.o util1.o util2.o t_stub.o lib/compat.o lib/snprintf.o lib/wildmatch.o lib/permstring.o
 t_chmod_secure$(EXEEXT): $(T_CHMOD_SECURE_OBJ)
 	$(CC) $(CFLAGS) $(LDFLAGS) -o $@ $(T_CHMOD_SECURE_OBJ) $(LIBS)

t_hashtable_overflow.c

@@ -0,0 +1,50 @@
+/*
+ * Regression harness for the hashtable size*node_size integer overflow
+ * (Leonid Bugaev May-2026 re-audit, KI-11/12).
+ *
+ * hashtable_create() once computed the slot-array byte count as
+ * new_array0(char, size * node_size) in 32-bit int arithmetic; for a large
+ * peer/data-driven size the product wrapped to a tiny value, under-allocating
+ * the table while tbl->size recorded the huge size -- a later node access then
+ * ran out of bounds (heap overflow / SEGV).  The fix passes size and node_size
+ * as separate factors so my_alloc's --max-alloc guard rejects the oversized
+ * request, exiting RERR_MALLOC instead of under-allocating.
+ *
+ * This harness sets a realistic max_alloc (t_stub leaves it at SIZE_MAX) and
+ * asks for an absurd size: the fixed code exits RERR_MALLOC; a regressed build
+ * under-allocates and crashes on the node access below.  Not linked into rsync.
+ *
+ * This program is free software; you can redistribute it and/or modify it under
+ * the terms of the GNU General Public License version 3 as published by the
+ * Free Software Foundation.
+ */
+
+#include "rsync.h"
+
+extern size_t max_alloc;	/* defined in util2.o/t_stub.o */
+short info_levels[COUNT_INFO], debug_levels[COUNT_DEBUG];	/* for DEBUG_GTE in hashtable.o */
+
+int main(UNUSED(int argc), UNUSED(char *argv[]))
+{
+	struct hashtable *tbl;
+	int i;
+
+	/* A realistic --max-alloc cap (the default is 1 GiB) so my_alloc's guard
+	 * can engage; t_stub.o leaves max_alloc == SIZE_MAX. */
+	max_alloc = (size_t)1024 * 1024 * 1024;
+
+	/* 2^28 buckets * 16-byte node = 2^32 bytes: the product wraps int to ~0 in
+	 * the unfixed code.  The fix must reject this (exit RERR_MALLOC) rather than
+	 * under-allocate. */
+	tbl = hashtable_create(1 << 28, 0);
+
+	/* Unreachable with the fix (hashtable_create exits above).  If a regression
+	 * lets it return, touch a node near the claimed end -- an under-allocated
+	 * table faults here -- and report the unexpected survival as a failure. */
+	for (i = 0; i < tbl->size; i += tbl->size / 64 + 1) {
+		struct ht_int32_node *node = HT_NODE(tbl, tbl->nodes, i);
+		node->key = i;
+	}
+	fprintf(stderr, "FAIL: hashtable_create(1<<28) was not rejected\n");
+	return 1;
+}

testsuite/hashtable-overflow_test.py

@@ -0,0 +1,37 @@
+#!/usr/bin/env python3
+# Regression test for the hashtable size*node_size integer overflow (KI-11/12).
+#
+# hashtable_create() once computed the slot-array byte count in 32-bit int
+# (new_array0(char, size * node_size)); a large peer/data-driven size wrapped the
+# product to a tiny value, under-allocating the table while tbl->size kept the
+# huge size -- a later node access ran out of bounds (heap overflow / SEGV).
+#
+# The t_hashtable_overflow helper (built by make check) links the real
+# hashtable.o, sets a realistic --max-alloc, and asks for an absurd size.  The
+# fix passes the factors separately so my_alloc's guard rejects it, exiting
+# RERR_MALLOC; a regressed build under-allocates and crashes on the node access.
+
+import subprocess
+
+from rsyncfns import TOOLDIR, test_fail, test_skipped
+
+RERR_MALLOC = 22   # errcode.h
+
+helper = TOOLDIR / 't_hashtable_overflow'
+if not helper.is_file():
+    test_skipped("t_hashtable_overflow helper not built")
+
+proc = subprocess.run([str(helper)], stdout=subprocess.PIPE, stderr=subprocess.PIPE,
+                      timeout=30)
+rc = proc.returncode
+if rc == RERR_MALLOC:
+    print("hashtable-overflow: hashtable_create rejected the oversized size "
+          "(exited RERR_MALLOC) instead of under-allocating")
+elif rc < 0:
+    test_fail(f"t_hashtable_overflow crashed (signal {-rc}): the hashtable "
+              "size*node_size integer overflow under-allocated the table\n"
+              + (proc.stderr or b'').decode('utf-8', 'replace'))
+else:
+    test_fail(f"t_hashtable_overflow exited {rc}, expected RERR_MALLOC ({RERR_MALLOC}): "
+              "the oversized hashtable_create was not rejected\n"
+              + (proc.stderr or b'').decode('utf-8', 'replace'))