mirror/rsync
1
0
Fork 0
mirror of https://github.com/RsyncProject/rsync.git synced 2026-05-24 23:05:52 -04:00
Code Issues Packages Projects Releases Wiki Activity

testsuite: cross-directory/temp/backup/dest coverage at depth

Browse Source 
Fill the highest-restructure-risk gap: options that do two-directory / rename /
outside-tree work, asserted at >=3 levels deep with the aux tree kept outside
the main tree, and asserting the option's specific property rather than just
tree equality (which the ported tests already cover).

  alt-dest-deep  --link-dest hardlinks unchanged files (same inode), --copy-dest
                 copies (never links), --compare-dest omits unchanged files;
                 ref tree outside both src and dest.
  temp-dir       cross-dir temp->final rename at depth; temp dir left clean; a
                 missing --temp-dir fails (so the option is proven consulted).
  partial        --partial keeps the partial in the dest file; relative
                 --partial-dir stages per-directory at depth (pre-seed +
                 interrupt/resume); absolute --partial-dir writes the partial
                 outside the tree.
  inplace        --inplace keeps the destination inode across a delta update;
                 the default temp+rename path replaces it.
  append         --append completes truncated files tail-only; --append-verify
                 repairs a corrupted prefix (protocol >= 30).
  backup-deep    --suffix saves <name>S beside the new file; --backup-dir
                 relocates old files to a parallel deep tree outside the dest
                 and captures deletions under --delete.

All green on master and under --protocol=29/30.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
Andrew Tridgell
2026-05-24 07:47:10 +10:00
parent 1d828f35ca
commit d6124a82a4
7 changed files with 568 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

85
testsuite/alt-dest-deep_test.py Normal file
View File

@@ -0,0 +1,85 @@
#!/usr/bin/env python3
"""Property-level coverage of --link-dest / --copy-dest / --compare-dest at
depth and across directory boundaries.
The existing alt-dest_test.py is a faithful port that checks tree equality;
this companion asserts the *distinguishing* property of each option, at every
level of a >=3-deep tree, with the alternate-destination tree placed OUTSIDE
both the source and destination trees (a sibling, not a parent/child):
--link-dest unchanged files are HARD-LINKED to the reference (same inode);
a changed file is transferred fresh (not linked).
--copy-dest unchanged files are COPIED from the reference (never linked);
dest is complete and byte-identical to the source.
--compare-dest unchanged files are NEITHER transferred NOR created in dest;
only a changed/new file lands in dest.
These options drive the two-dirfd / outside-tree path handling the resolver
restructure rewrites, so they must be guarded at depth.
"""
import os
from rsyncfns import (
FROMDIR, SCRATCHDIR, TODIR,
assert_exists, assert_hardlinked, assert_not_exists, assert_not_hardlinked,
assert_same, make_tree, rmtree, run_rsync, walk_files,
)
src = FROMDIR
ref = SCRATCHDIR / 'altref' # sibling of from/ and to/ -- outside both trees
rmtree(src)
rmtree(ref)
rmtree(TODIR)
# A >=3-deep source: f0 at the root, then d1/f1, d1/d2/f2, d1/d2/d3/f3.
make_tree(src, depth=3, data=True)
# Reference tree == an exact copy of the source, so every file is "identical"
# for the alt-dest comparison.
run_rsync('-a', f'{src}/', f'{ref}/')
# Now make the deepest file differ so it must be transferred in every mode.
changed = os.path.join('d1', 'd2', 'd3', 'f3')
with open(src / changed, 'ab') as f:
f.write(b'a changed deep tail\n')
rels = [p.relative_to(src) for p in walk_files(src)]
assert os.path.join('d1', 'd2', 'd3', 'f3') in [str(r) for r in rels]
def run_to(opt):
rmtree(TODIR)
run_rsync('-a', f'--{opt}={ref}', f'{src}/', f'{TODIR}/')
# --- --link-dest: unchanged -> hardlink to ref; changed -> fresh copy -------
run_to('link-dest')
for rel in rels:
d, r = TODIR / rel, ref / rel
if str(rel) == changed:
assert_not_hardlinked(d, r, label=f'link-dest changed {rel}')
assert_same(d, src / rel, label=f'link-dest changed {rel}')
else:
assert_hardlinked(d, r, label=f'link-dest unchanged {rel}')
# --- --copy-dest: every file copied (never linked), dest complete -----------
run_to('copy-dest')
for rel in rels:
d, r = TODIR / rel, ref / rel
assert_exists(d, label=f'copy-dest {rel}')
assert_same(d, src / rel, label=f'copy-dest {rel}')
assert_not_hardlinked(d, r, label=f'copy-dest {rel}')
# --- --compare-dest: unchanged absent from dest; only the changed file lands -
run_to('compare-dest')
for rel in rels:
d = TODIR / rel
if str(rel) == changed:
assert_exists(d, label=f'compare-dest changed {rel}')
assert_same(d, src / rel, label=f'compare-dest changed {rel}')
else:
assert_not_exists(d, label=f'compare-dest unchanged {rel}')
print("alt-dest-deep: link-dest/copy-dest/compare-dest verified at depth")

73
testsuite/append_test.py Normal file
View File

@@ -0,0 +1,73 @@
#!/usr/bin/env python3
"""Coverage of --append and --append-verify at depth.
--append assumes each destination file is a prefix of the (longer) source and
transfers only the bytes past the existing size; it does NOT re-check the data
already present, and it never touches a file that is already the same size or
larger. --append-verify works the same way but folds the existing data into the
whole-file checksum, so a transfer whose result fails verification is re-sent
with a normal --inplace pass. Exercise both on files >=3 levels deep.
"""
import os
from rsyncfns import (
FROMDIR, TODIR,
assert_same, forced_protocol, make_tree, rmtree, run_rsync, test_fail,
walk_files,
)
src = FROMDIR
deep = os.path.join('d1', 'd2', 'd3', 'f3')
def seed_source():
rmtree(src)
make_tree(src, depth=3, data=True, data_size=8192)
return [p.relative_to(src) for p in walk_files(src)]
def dest_prefix(rels, *, corrupt=False, frac=0.5):
"""Build a destination holding the first `frac` of each source file (a
valid prefix), optionally corrupting the deep file's leading bytes."""
rmtree(TODIR)
for rel in rels:
dst = TODIR / rel
dst.parent.mkdir(parents=True, exist_ok=True)
full = (src / rel).read_bytes()
dst.write_bytes(full[: int(len(full) * frac)])
if corrupt:
p = TODIR / deep
bad = bytearray(p.read_bytes())
bad[0:64] = b'\x00' * 64
p.write_bytes(bytes(bad))
# --- --append completes truncated destinations at every level ---------------
rels = seed_source()
dest_prefix(rels)
run_rsync('-a', '--append', f'{src}/', f'{TODIR}/')
for rel in rels:
assert_same(TODIR / rel, src / rel, label=f'append {rel}')
# The split between non-verifying --append and verifying --append-verify only
# exists at protocol >= 30; at protocol 29 plain --append still verifies, so
# skip the distinguishing sub-cases there.
proto = forced_protocol()
if proto is not None and proto < 30:
print(f"append: protocol {proto} -- skipping the --append/--append-verify "
"split (verifying-append behaviour predates the protocol-30 split)")
else:
# plain --append trusts a corrupted prefix (leaves it wrong)
dest_prefix(rels, corrupt=True)
run_rsync('-a', '--append', f'{src}/', f'{TODIR}/')
if (TODIR / deep).read_bytes() == (src / deep).read_bytes():
test_fail("plain --append unexpectedly repaired a corrupted prefix "
"(it should append only and trust the existing data)")
# --append-verify detects the bad prefix and re-sends the whole file
dest_prefix(rels, corrupt=True)
run_rsync('-a', '--append-verify', f'{src}/', f'{TODIR}/')
assert_same(TODIR / deep, src / deep, label='append-verify deep')
print("append: tail-only completion at depth; append-verify repairs prefix")

86
testsuite/backup-deep_test.py Normal file
View File

@@ -0,0 +1,86 @@
#!/usr/bin/env python3
"""Property-level coverage of --backup / --suffix / --backup-dir at depth.
backup_test.py is the ported regression test (2 levels, no custom suffix); this
companion checks the concrete outcome of each backup mode at >=3 levels and,
for --backup-dir, with the backup tree placed OUTSIDE the destination (a
sibling) so the old file is renamed across directories into a parallel deep
path -- the cross-directory operation the resolver restructure rewrites.
Asserts, at every level of the tree:
* --backup --suffix=S saves the overwritten file as <name>S beside the new
one (old content in the backup, new content in place);
* --backup --backup-dir=DIR relocates the old file to DIR/<relpath>,
preserving the deep structure, while the destination gets the new content;
* --backup-dir together with --delete routes a deletion into the backup
tree instead of losing it.
"""
import os
from rsyncfns import (
FROMDIR, SCRATCHDIR, TODIR,
assert_not_exists, assert_same, make_tree, rmtree, run_rsync, test_fail,
walk_files,
)
src = FROMDIR
bak = SCRATCHDIR / 'backups' # sibling of from/ and to/ -- outside both trees
def seed():
"""Fresh v1 source, a matching destination, and the old (v1) bytes; then
mutate the source to v2 so the next sync overwrites every file."""
rmtree(src)
rmtree(TODIR)
rmtree(bak)
make_tree(src, depth=3, data=True, data_size=4096)
rels = [p.relative_to(src) for p in walk_files(src)]
run_rsync('-a', f'{src}/', f'{TODIR}/')
old = {rel: (src / rel).read_bytes() for rel in rels}
for rel in rels: # v1 -> v2
with open(src / rel, 'ab') as f:
f.write(b'\nversion-2 tail\n')
return rels, old
# --- --backup --suffix=.old (same directory) --------------------------------
rels, old = seed()
run_rsync('-a', '-b', '--suffix=.old', '--no-whole-file',
f'{src}/', f'{TODIR}/')
for rel in rels:
assert_same(TODIR / rel, src / rel, label=f'suffix new {rel}')
backup = (TODIR / rel)
backup = backup.with_name(backup.name + '.old')
if not backup.is_file():
test_fail(f"--suffix backup missing for {rel}: {backup}")
if backup.read_bytes() != old[rel]:
test_fail(f"--suffix backup of {rel} does not hold the old content")
# --- --backup-dir at depth, outside the dest tree (cross-dir) ---------------
rels, old = seed()
run_rsync('-a', '-b', f'--backup-dir={bak}', '--no-whole-file',
f'{src}/', f'{TODIR}/')
for rel in rels:
assert_same(TODIR / rel, src / rel, label=f'backup-dir new {rel}')
saved = bak / rel
if not saved.is_file():
test_fail(f"--backup-dir did not preserve deep path for {rel}: {saved}")
if saved.read_bytes() != old[rel]:
test_fail(f"--backup-dir copy of {rel} does not hold the old content")
# --- --backup-dir captures a deletion under --delete ------------------------
rels, old = seed()
# Add a deep file to the destination that is absent from the source.
extra = os.path.join('d1', 'd2', 'd3', 'goner')
(TODIR / extra).write_bytes(b'about to be deleted\n')
run_rsync('-a', '--delete', '-b', f'--backup-dir={bak}', '--no-whole-file',
f'{src}/', f'{TODIR}/')
assert_not_exists(TODIR / extra, label='deleted file removed from dest')
saved = bak / extra
if not saved.is_file():
test_fail(f"--backup-dir did not capture the deletion of {extra}")
if saved.read_bytes() != b'about to be deleted\n':
test_fail("captured deletion has the wrong content")
print("backup-deep: suffix / backup-dir / delete-capture verified at depth")

62
testsuite/delay-updates-deep_test.py Normal file
View File

@@ -0,0 +1,62 @@
#!/usr/bin/env python3
"""Property-level coverage of --delay-updates at depth.
--delay-updates writes each updated file into a per-directory staging dir
(.~tmp~) during the transfer and only renames them into place at the very end,
so an interrupted run never leaves a half-written file visible. The staging dir
sits inside each destination directory, so the staging write and the
end-of-run rename are parent-directory operations the resolver restructure
rewrites; the ported delay-updates_test.py only exercises the tree root, so
this companion drives a >=3-deep tree.
Asserts, at every level of the tree:
* a --delay-updates transfer reproduces the source and leaves no .~tmp~
staging directory behind;
* a stale file pre-planted in a deep .~tmp~ staging dir is overwritten
cleanly and the staging dir is removed.
"""
import os
from rsyncfns import (
FROMDIR, TODIR,
assert_same, make_tree, rmtree, run_rsync, test_fail, walk_dirs,
walk_files,
)
src = FROMDIR
deepdir = os.path.join('d1', 'd2', 'd3')
def no_staging_left():
leftover = [p for p in walk_dirs(TODIR) if p.name == '.~tmp~']
if leftover:
test_fail(f"--delay-updates left staging dirs behind: {leftover}")
# --- initial --delay-updates over a deep tree -------------------------------
rmtree(src)
rmtree(TODIR)
make_tree(src, depth=3, data=True, data_size=4096)
rels = [p.relative_to(src) for p in walk_files(src)]
run_rsync('-a', '--delay-updates', f'{src}/', f'{TODIR}/')
for rel in rels:
assert_same(TODIR / rel, src / rel, label=f'delay-updates initial {rel}')
no_staging_left()
# --- update every file, with a stale staging file planted deep --------------
for rel in rels:
with open(src / rel, 'ab') as f:
f.write(b'\nupdated content\n')
stage = TODIR / deepdir / '.~tmp~'
stage.mkdir(parents=True, exist_ok=True)
(stage / 'f3').write_bytes(b'stale staged junk\n') # must be overwritten
run_rsync('-a', '--delay-updates', f'{src}/', f'{TODIR}/')
for rel in rels:
assert_same(TODIR / rel, src / rel, label=f'delay-updates update {rel}')
no_staging_left()
print("delay-updates-deep: staging + clean overwrite verified at depth")

71
testsuite/inplace_test.py Normal file
View File

@@ -0,0 +1,71 @@
#!/usr/bin/env python3
"""Coverage of --inplace at depth.
--inplace updates the destination file directly instead of writing a temp copy
and renaming it over the original, so across a delta update the destination
keeps the SAME inode. Without --inplace the receiver creates a fresh temp file
and renames it, giving the destination a NEW inode. Both behaviours hinge on
how the receiver resolves the destination directory and (for the default mode)
performs the temp->final rename, which the path restructure rewrites; verify
them on a file >=3 levels deep.
"""
import os
from rsyncfns import (
FROMDIR, TODIR,
assert_same, make_tree, rmtree, run_rsync, test_fail,
)
src = FROMDIR
deep = os.path.join('d1', 'd2', 'd3', 'f3')
def seed():
rmtree(src)
rmtree(TODIR)
make_tree(src, depth=3, data=True, data_size=200000)
def inode(path):
return os.stat(path).st_ino
def modify_deep():
# Rewrite a chunk in the middle so it is a genuine delta, not just a tail
# append. Bump mtime by a clear margin (the whole test runs inside one
# second, so a "now" touch would collide with the destination's mtime and
# the size-unchanged file would be skipped by the quick check).
p = src / deep
data = bytearray(p.read_bytes())
data[1000:1100] = bytes((b ^ 0xFF) for b in data[1000:1100])
p.write_bytes(bytes(data))
st = os.stat(p)
os.utime(p, (st.st_atime, st.st_mtime + 100))
# --- --inplace keeps the destination inode across a delta update ------------
seed()
run_rsync('-a', f'{src}/', f'{TODIR}/')
ino_before = inode(TODIR / deep)
modify_deep()
run_rsync('-a', '--inplace', '--no-whole-file', f'{src}/', f'{TODIR}/')
assert_same(TODIR / deep, src / deep, label='inplace content')
if inode(TODIR / deep) != ino_before:
test_fail("--inplace changed the destination inode at depth "
f"({ino_before} -> {inode(TODIR / deep)})")
# --- control: the default (temp+rename) path replaces the inode -------------
seed()
run_rsync('-a', f'{src}/', f'{TODIR}/')
ino_before = inode(TODIR / deep)
modify_deep()
run_rsync('-a', '--no-whole-file', f'{src}/', f'{TODIR}/')
assert_same(TODIR / deep, src / deep, label='default content')
if inode(TODIR / deep) == ino_before:
test_fail("default (non-inplace) delta update unexpectedly kept the "
"destination inode at depth -- temp+rename did not run")
print("inplace: same-inode update at depth verified; default replaces inode")

130
testsuite/partial_test.py Normal file
View File

@@ -0,0 +1,130 @@
#!/usr/bin/env python3
"""Coverage of --partial / --partial-dir at depth and across directory
boundaries.
--partial keeps a partially transferred file so a later run can resume it.
--partial-dir=DIR keeps the partial in DIR instead of the destination file:
a RELATIVE dir is created in (and removed from) each destination file's own
directory; an ABSOLUTE dir is a reserved location that holds partials by
basename. All of this is parent- and cross-directory path resolution -- what
the resolver restructure rewrites -- so exercise it on a file several levels
deep, with the absolute partial-dir kept OUTSIDE the destination tree.
Note: a *delta* resume from an absolute partial-dir currently fails whole-file
verification on master (it re-puts the partial and never converges). This test
therefore only asserts the cross-directory WRITE of the partial for that case
and completes it with --whole-file, which is the clearly-correct baseline.
"""
import os
import signal
import subprocess
import time
from rsyncfns import (
FROMDIR, SCRATCHDIR, TODIR,
assert_same, make_data_file, makepath, rmtree, rsync_argv, run_rsync,
test_fail,
)
src = FROMDIR
deepdir = os.path.join('d1', 'd2', 'd3')
deep = os.path.join(deepdir, 'f3')
FULL = 12_000_000
def seed_big():
rmtree(src)
rmtree(TODIR)
makepath(src / deepdir)
make_data_file(src / deep, FULL)
def is_prefix(partial) -> bool:
pb = partial.read_bytes()
return 0 < len(pb) < FULL and (src / deep).read_bytes()[:len(pb)] == pb
def interrupt_transfer(extra_args, partial_path):
"""Start a deliberately slow transfer, SIGTERM it once the receiver's
in-progress temp (.f3.XXXXXX) has some data, and wait for `partial_path`
(where this mode keeps the partial) to materialise.
The bandwidth limit is low so the multi-second transfer cannot finish
before we interrupt it -- important under a loaded parallel run (-j16),
where the polling loop can lag by seconds. We then poll for the partial,
since rsync moves it into place from its exit_cleanup handler."""
proc = subprocess.Popen(
rsync_argv('-a', '--no-whole-file', '--bwlimit=400', *extra_args,
f'{src}/', f'{TODIR}/'),
stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
tdir = TODIR / deepdir
caught = False
deadline = time.monotonic() + 30
while time.monotonic() < deadline:
if proc.poll() is not None:
break # exited before we caught it
if tdir.is_dir():
temps = [p for p in tdir.glob('.f3.*')
if p.is_file() and p.stat().st_size > 0]
if temps:
caught = True
break
time.sleep(0.02)
proc.send_signal(signal.SIGTERM)
proc.wait()
if not caught:
test_fail("never caught an in-progress temp (transfer finished too "
"fast to interrupt)")
# rsync moves the partial into place from exit_cleanup; give it a moment.
pdeadline = time.monotonic() + 5
while time.monotonic() < pdeadline:
if partial_path.is_file() and partial_path.stat().st_size > 0:
return
time.sleep(0.05)
# --- 1. --partial (no dir): partial kept in the dest file itself, at depth --
seed_big()
interrupt_transfer(['--partial'], TODIR / deep)
if not (TODIR / deep).is_file() or not is_prefix(TODIR / deep):
test_fail("--partial did not leave a valid partial in the dest file")
run_rsync('-a', '--partial', '--no-whole-file', f'{src}/', f'{TODIR}/')
assert_same(TODIR / deep, src / deep, label='--partial resume')
# --- 2. relative --partial-dir at depth: deterministic clean pre-seed -------
rmtree(src)
rmtree(TODIR)
makepath(src / deepdir, TODIR / deepdir / '.rsync-partial')
make_data_file(src / deep, 1_000_000)
full = (src / deep).read_bytes()
(TODIR / deepdir / '.rsync-partial' / 'f3').write_bytes(full[:400_000])
run_rsync('-a', '--partial-dir=.rsync-partial', '--no-whole-file',
f'{src}/', f'{TODIR}/')
assert_same(TODIR / deep, src / deep, label='rel partial-dir preseed')
if (TODIR / deepdir / '.rsync-partial').exists():
test_fail("relative --partial-dir not removed after the partial was used")
# --- 3. relative --partial-dir at depth: interrupt then resume -------------
seed_big()
part = TODIR / deepdir / '.rsync-partial' / 'f3'
interrupt_transfer(['--partial-dir=.rsync-partial'], part)
if not part.is_file() or not is_prefix(part):
test_fail("relative --partial-dir did not keep a valid partial at depth")
run_rsync('-a', '--partial-dir=.rsync-partial', '--no-whole-file',
f'{src}/', f'{TODIR}/')
assert_same(TODIR / deep, src / deep, label='rel partial-dir resume')
# --- 4. absolute --partial-dir OUTSIDE the tree (cross-dir): interrupt -----
ext = SCRATCHDIR / 'partials' # sibling of from/ and to/ -- outside both
rmtree(ext)
ext.mkdir()
seed_big()
interrupt_transfer([f'--partial-dir={ext}'], ext / 'f3')
if not (ext / 'f3').is_file() or not is_prefix(ext / 'f3'):
test_fail("absolute --partial-dir did not write the partial to the "
"outside-tree dir")
run_rsync('-a', f'--partial-dir={ext}', '--whole-file', f'{src}/', f'{TODIR}/')
assert_same(TODIR / deep, src / deep, label='abs partial-dir resume')
print("partial: --partial + relative/absolute --partial-dir verified at depth")

61
testsuite/temp-dir_test.py Normal file
View File

@@ -0,0 +1,61 @@
#!/usr/bin/env python3
"""Coverage of --temp-dir (-T) at depth and across directory boundaries.
--temp-dir tells the receiver to create its scratch/temp copies in DIR rather
than in the destination directory, then rename the finished file into place.
That rename crosses from the temp directory to a destination directory several
levels deep -- exactly the two-directory operation the path-resolution
restructure rewrites -- so it must be guarded at depth with the temp dir kept
OUTSIDE the destination tree.
Asserts:
* a transfer with --temp-dir pointing outside the dest tree reproduces the
source byte-for-byte at every level;
* no temp/scratch files are left behind in the temp dir or the dest tree;
* a non-existent --temp-dir makes the receiver fail (so we know the option
is actually consulted, not silently ignored).
"""
import os
from rsyncfns import (
FROMDIR, SCRATCHDIR, TODIR,
assert_same, make_tree, rmtree, run_rsync, test_fail, walk_files,
)
src = FROMDIR
tmp = SCRATCHDIR / 'scratch-temp' # sibling of from/ and to/ -- outside both
rmtree(src)
rmtree(TODIR)
rmtree(tmp)
tmp.mkdir()
make_tree(src, depth=3, data=True)
rels = [p.relative_to(src) for p in walk_files(src)]
# Transfer with the temp dir outside the destination tree.
run_rsync('-a', f'--temp-dir={tmp}', f'{src}/', f'{TODIR}/')
for rel in rels:
assert_same(TODIR / rel, src / rel, label=f'temp-dir {rel}')
# The temp dir must be clean afterwards (every scratch file renamed away).
leftover = sorted(p for p in tmp.rglob('*'))
if leftover:
test_fail(f"--temp-dir left scratch files behind: {leftover}")
# No stray rsync temp files (.name.XXXXXX) anywhere in the dest tree.
strays = [p for p in TODIR.rglob('.*') if p.is_file()]
if strays:
test_fail(f"dest tree contains stray temp files: {strays}")
# Negative: a missing temp dir must cause a receiver failure, proving the
# option is honoured rather than ignored.
rmtree(TODIR)
proc = run_rsync('-a', f'--temp-dir={SCRATCHDIR}/does-not-exist',
f'{src}/', f'{TODIR}/', check=False)
if proc.returncode == 0:
test_fail("--temp-dir pointing at a missing directory unexpectedly "
"succeeded")
print("temp-dir: cross-dir rename at depth verified; missing temp dir fails")