mirror of
https://github.com/RsyncProject/rsync.git
synced 2026-09-12 21:28:25 -04:00
simd: stop the AVX2 rolling checksum reading past its buffer
The loop is software-pipelined: each iteration folds in the 64 bytes it preloaded last time and preloads the next 64. Nothing stopped the final iteration doing that preload, so it always read the 64 bytes after the region it was asked to checksum. Not an edge case. The assembly processes len&~63 and leaves the remainder to the caller, so the remainder is by construction under 64 bytes and the preload passed buf+len on every call, by 64 minus the remainder. It normally landed in slack inside the map_ptr() window and nothing noticed. Where the buffer ended near an unmapped page it was a SIGSEGV in the middle of a transfer -- reported on macOS x86-64 by Roland Kletzing, whose `partial` run died with "connection unexpectedly closed" because the generator had crashed. A guard page reproduces it on Linux too, so it was latent there, not absent. Run the pipelined loop one block short and finish the last block in .last, which does the same arithmetic without the preload. No per-iteration cost, and checksums are bit-identical -- simdtest compares every implementation against the C reference. The earlier fix for that report, "lib: use .balign in md5 x86-64 asm", was to the md5 assembly. It addressed the linker alignment warning that appeared alongside, not this.
This commit is contained in:
1 parent
bb6329bc8c
commit
2979d8eddc
1 file changed
+31
-1
+31
-1
@@ -46,6 +46,14 @@ get_checksum1_avx2_asm:
|
||||
vpxor xmm1, xmm1, xmm1 # reset both partial sums accumulators.
|
||||
vpxor xmm4, xmm4, xmm4
|
||||
mov eax, [r8]
|
||||
# The loop below is software-pipelined: it preloads the 64 bytes AFTER the
|
||||
# ones it is folding in. Run it one block short and finish that block in
|
||||
# .last, which does not preload -- otherwise the final iteration reads 64
|
||||
# bytes past the end of what the caller gave us. The remainder (len & 63)
|
||||
# is always under 64, so that read always crossed buf+len; it only faulted
|
||||
# when the buffer happened to end at a page boundary.
|
||||
sub esi, 1
|
||||
jz .last
|
||||
.p2align 4 # should fit into the LSD allocation queue.
|
||||
.loop:
|
||||
vpmaddubsw ymm0, ymm15, ymm2 # s1 partial sums
|
||||
@@ -70,11 +78,33 @@ get_checksum1_avx2_asm:
|
||||
vpaddd ymm6, ymm10, ymm6 # 32*CHAR_OFFSET
|
||||
vpaddd ymm1, ymm13, ymm1 # 528*CHAR_OFFSET
|
||||
#endif
|
||||
vmovdqa ymm2, ymm8 # move the next 64 bytes
|
||||
vmovdqa ymm2, ymm8 # move the next 64 bytes
|
||||
vmovdqa ymm3, ymm9 # into the right registers
|
||||
sub esi, 1
|
||||
jnz .loop
|
||||
|
||||
# The final 64 bytes, already in ymm2/ymm3. Same arithmetic as the loop
|
||||
# body with the preload, the prefetch and the loop control dropped.
|
||||
.last:
|
||||
vpmaddubsw ymm0, ymm15, ymm2 # s1 partial sums
|
||||
vpmaddubsw ymm5, ymm15, ymm3
|
||||
vpaddd ymm4, ymm4, ymm6
|
||||
vpaddw ymm5, ymm5, ymm0
|
||||
vpsrld ymm0, ymm5, 16
|
||||
vpaddw ymm5, ymm0, ymm5
|
||||
vpaddd ymm6, ymm5, ymm6
|
||||
vpmaddubsw ymm2, ymm7, ymm2 # s2 partial sums
|
||||
vpmaddubsw ymm3, ymm12, ymm3
|
||||
vpaddw ymm3, ymm2, ymm3
|
||||
vpsrldq ymm2, ymm3, 2
|
||||
vpaddd ymm3, ymm2, ymm3
|
||||
vpaddd ymm1, ymm1, ymm3
|
||||
|
||||
#if CHAR_OFFSET != 0
|
||||
vpaddd ymm6, ymm10, ymm6 # 32*CHAR_OFFSET
|
||||
vpaddd ymm1, ymm13, ymm1 # 528*CHAR_OFFSET
|
||||
#endif
|
||||
|
||||
# now we reduce the partial sums.
|
||||
vpslld ymm3, ymm4, 6
|
||||
vpsrldq ymm2, ymm6, 4
|
||||
|
||||
Reference in new issue
Block a user