Preparing for release of 3.2.4

man 5 systemd.service:
> Setting this to on-failure is the recommended choice for long-running services

Partial fix for https://bugzilla.samba.org/show_bug.cgi?id=13463

Signed-off-by: Simon Deziel <simon@sdeziel.info>

.cirrus.yml

@@ -1,7 +1,7 @@
 freebsd_task:
   name: FreeBSD
   freebsd_instance:
-    image_family: freebsd-12-2
+    image_family: freebsd-13-0
   env:
     PATH: /usr/local/bin:$PATH
   prep_script:

.github/workflows/build.yml

@@ -86,16 +86,15 @@ jobs:
           rrsync
 
   cygwin-build:
-    runs-on: windows-latest
+    runs-on: windows-2022
     if: (github.event_name == 'schedule' || contains(github.event.head_commit.message, '[buildall]'))
     steps:
     - uses: actions/checkout@v2
-    - uses: crazy-max/ghaction-chocolatey@v1.2.2
-      with:
-        args: install -y --no-progress cygwin cyg-get
+    - name: cygwin
+      run: choco install -y --no-progress cygwin cyg-get
     - name: prep
       run: |
-        cyg-get make autoconf automake gcc-core attr libattr-devel python38 python38-pip libzstd-devel liblz4-devel libssl-devel libxxhash0 libxxhash-devel
+        cyg-get make autoconf automake gcc-core attr libattr-devel python39 python39-pip libzstd-devel liblz4-devel libssl-devel libxxhash0 libxxhash-devel
         curl.exe -o git-version.h https://gist.githubusercontent.com/WayneD/c11243fa374fc64d4e42f2855c8e3827/raw/rsync-git-version.h
         echo "C:/tools/cygwin/bin" >>$Env:GITHUB_PATH
     - name: commonmark

.gitignore

@@ -16,6 +16,7 @@ aclocal.m4
 /proto.h
 /proto.h-tstamp
 /rsync*.[15]
+/rrsync
 /rrsync*.1
 /rsync*.html
 /rrsync*.html

INSTALL.md

@@ -13,11 +13,11 @@ You need to have a C compiler installed and optionally a C++ compiler in order
 to try to build some hardware-accelerated checksum routines.  Rsync also needs
 a modern awk, which might be provided via gawk or nawk on some OSes.
 
-## Autoconf & man pages
+## Autoconf & manpages
 
 If you're installing from the git repo (instead of a release tar file) you'll
 also need the GNU autotools (autoconf & automake) and your choice of 2 python3
-markdown libraries: cmarkgfm or commonmark (needed to generate the man pages).
+markdown libraries: cmarkgfm or commonmark (needed to generate the manpages).
 If your OS doesn't provide a python3-cmarkgfm or python3-commonmark package,
 you can run the following to install the commonmark python library for your
 build user (after installing python3's pip package):
@@ -28,7 +28,7 @@ You can test if you've got it fixed by running (from the rsync checkout):
 
 >     ./md2man --test rsync-ssl.1.md
 
-Alternately, you can avoid generating the man pages by fetching the very latest
+Alternately, you can avoid generating the manpages by fetching the very latest
 versions (that match the latest git source) from the [generated-files][6] dir.
 One way to do that is to run:
 

Makefile.in

@@ -30,11 +30,13 @@ SHELL=/bin/sh
 .SUFFIXES:
 .SUFFIXES: .c .o
 
-SIMD_x86_64=simd-checksum-x86_64.o simd-checksum-avx2.o
-ASM_x86_64=lib/md5-asm-x86_64.o
+ROLL_SIMD_x86_64=simd-checksum-x86_64.o
+ROLL_ASM_x86_64=simd-checksum-avx2.o
+MD5_ASM_x86_64=lib/md5-asm-x86_64.o
 
 GENFILES=configure.sh aclocal.m4 config.h.in rsync.1 rsync.1.html \
-	 rsync-ssl.1 rsync-ssl.1.html rsyncd.conf.5 rsyncd.conf.5.html
+	 rsync-ssl.1 rsync-ssl.1.html rsyncd.conf.5 rsyncd.conf.5.html \
+	 @GEN_RRSYNC@
 HEADERS=byteorder.h config.h errcode.h proto.h rsync.h ifuncs.h itypes.h inums.h \
 	lib/pool_alloc.h lib/mdigest.h lib/md-defines.h
 LIBOBJ=lib/wildmatch.o lib/compat.o lib/snprintf.o lib/mdfour.o lib/md5.o \
@@ -45,7 +47,7 @@ OBJS1=flist.o rsync.o generator.o receiver.o cleanup.o sender.o exclude.o \
 	util1.o util2.o main.o checksum.o match.o syscall.o log.o backup.o delete.o
 OBJS2=options.o io.o compat.o hlink.o token.o uidlist.o socket.o hashtable.o \
 	usage.o fileio.o batch.o clientname.o chmod.o acls.o xattrs.o
-OBJS3=progress.o pipe.o @ASM@ @SIMD@
+OBJS3=progress.o pipe.o @MD5_ASM@ @ROLL_SIMD@ @ROLL_ASM@
 DAEMON_OBJ = params.o loadparm.o clientserver.o access.o connection.o authenticate.o
 popt_OBJS=popt/findme.o  popt/popt.o  popt/poptconfig.o \
 	popt/popthelp.o popt/poptparse.o
@@ -146,13 +148,13 @@ git-version.h: ALWAYS_RUN
 ALWAYS_RUN:
 
 simd-checksum-x86_64.o: simd-checksum-x86_64.cpp
-	@$(srcdir)/cmd-or-msg disable-simd $(CXX) -I. $(CXXFLAGS) $(CPPFLAGS) -c -o $@ $(srcdir)/simd-checksum-x86_64.cpp
+	@$(srcdir)/cmd-or-msg disable-roll-simd $(CXX) -I. $(CXXFLAGS) $(CPPFLAGS) -c -o $@ $(srcdir)/simd-checksum-x86_64.cpp
 
 simd-checksum-avx2.o: simd-checksum-avx2.S
-	@$(srcdir)/cmd-or-msg disable-asm $(CC) $(CFLAGS) --include=$(srcdir)/rsync.h -DAVX2_ASM -I. @NOEXECSTACK@ -c -o $@ $(srcdir)/simd-checksum-avx2.S
+	@$(srcdir)/cmd-or-msg disable-roll-asm $(CC) $(CFLAGS) -I. @NOEXECSTACK@ -c -o $@ $(srcdir)/simd-checksum-avx2.S
 
-lib/md5-asm-x86_64.o: lib/md5-asm-x86_64.S config.h lib/md-defines.h
-	@$(srcdir)/cmd-or-msg disable-asm $(CC) -I. @NOEXECSTACK@ -c -o $@ $(srcdir)/lib/md5-asm-x86_64.S
+lib/md5-asm-x86_64.o: lib/md5-asm-x86_64.S lib/md-defines.h
+	@$(srcdir)/cmd-or-msg disable-md5-asm $(CC) -I. @NOEXECSTACK@ -c -o $@ $(srcdir)/lib/md5-asm-x86_64.S
 
 tls$(EXEEXT): $(TLS_OBJ)
 	$(CC) $(CFLAGS) $(LDFLAGS) -o $@ $(TLS_OBJ) $(LIBS)
@@ -271,9 +273,9 @@ rrsync.1: support/rrsync.1.md md-convert Makefile
 
 .PHONY: clean
 clean: cleantests
-	rm -f *~ $(OBJS) $(CHECK_PROGS) $(CHECK_OBJS) $(CHECK_SYMLINKS) \
-		git-version.h rounding rounding.h *.old rsync*.1 rsync*.5 rsync*.html \
-		daemon-parm.h help-*.h default-*.h proto.h proto.h-tstamp
+	rm -f *~ $(OBJS) $(CHECK_PROGS) $(CHECK_OBJS) $(CHECK_SYMLINKS) @MAKE_RRSYNC@ \
+		git-version.h rounding rounding.h *.old rsync*.1 rsync*.5 @MAKE_RRSYNC_1@ \
+		*.html daemon-parm.h help-*.h default-*.h proto.h proto.h-tstamp
 
 .PHONY: cleantests
 cleantests:

NEWS.md

@@ -1,32 +1,32 @@
-<a name="3.2.4"></a>
-
-# NEWS for rsync 3.2.4 (UNRELEASED)
+# NEWS for rsync 3.2.4 (15 Apr 2022)
 
 ## Changes in this version:
 
 ### BEHAVIOR CHANGES:
 
  - A new form of arg protection was added that works similarly to the older
-   `--protect-args` (`-s`) option but in a way that avoids breaking things like
-   rrsync (the restricted rsync script): rsync now uses backslash escaping for
-   sending "shell-active" characters to the remote shell. This includes spaces,
-   so fetching a remote file via a simple quoted filename value now works by
-   default without any extra quoting:
+   [`--protect-args`](rsync.1#opt) (`-s`) option but in a way that avoids
+   breaking things like rrsync (the restricted rsync script): rsync now uses
+   backslash escaping for sending "shell-active" characters to the remote
+   shell. This includes spaces, so fetching a remote file via a simple quoted
+   filename value now works by default without any extra quoting:
 
    ```shell
        rsync -aiv host:'a simple file.pdf' .
    ```
 
    Wildcards are not escaped in filename args, but they are escaped in options
-   like the `--suffix` and `--usermap` values.  If your rsync script depends on
-   the old arg-splitting behavior, either run it with the `--old-args` option
-   or `export RSYNC_OLD_ARGS=1` in the script's environment.
+   like the [`--suffix`](rsync.1#opt) and [`--usermap`](rsync.1#opt) values.
+   If your rsync script depends on the old arg-splitting behavior, either run
+   it with the [`--old-args`](rsync.1#opt) option or `export RSYNC_OLD_ARGS=1`
+   in the script's environment.  See also the [ADVANCED USAGE](rsync.1#)
+   section of rsync's manpage for how to use a more modern arg style.
 
  - A long-standing bug was preventing rsync from figuring out the current
    locale's decimal point character, which made rsync always output numbers
    using the "C" locale.  Since this is now fixed in 3.2.4, a script that
-   parses rsync's decimal numbers (e.g. from the verbose footer) should be sure
-   to setup the environment in a way that the output continues to be in the C
+   parses rsync's decimal numbers (e.g. from the verbose footer) may want to
+   setup the environment in a way that the output continues to be in the C
    locale.  For instance, one of the following should work fine:
 
    ```shell
@@ -44,64 +44,93 @@
        export LC_NUMERIC=C.UTF-8
    ```
 
+### SECURITY FIXES:
+
+ - A fix for CVE-2018-25032 in the bundled zlib (memory corruption issue).
+
 ### BUG FIXES:
 
- - Fixed a bug with `--inplace` + `--sparse` (and a lack of `--whole-file`)
-   where the destination file could get reconstructed with bogus data.  Since
-   the bug can also be avoided by using (the seemingly redundant) `--no-W` on
-   the receiving side, the latest rsync will now send `--no-W` to a remote
-   receiver when this option combination occurs.  If your client rsync is not
-   new enough to do this for you (or if you're just paranoid), you can manually
-   specify `--no-W -M--no-W` (when not using `--whole-file`) to make sure the
+ - Fixed a bug with [`--inplace`](rsync.1#opt) + [`--sparse`](rsync.1#opt) (and
+   a lack of [`--whole-file`](rsync.1#opt)) where the destination file could
+   get reconstructed with bogus data.  Since the bug can also be avoided by
+   using (the seemingly redundant) [`--no-W`](rsync.1#opt) on the receiving
+   side, the latest rsync will now send `--no-W` to a remote receiver when this
+   option combination occurs.  If your client rsync is not new enough to do
+   this for you (or if you're just paranoid), you can manually specify `--no-W
+   -M--no-W` (when not using [`--whole-file`](rsync.1#opt)) to make sure the
    bug is avoided.
 
- - Fixed a bug with `--mkpath` if a single-file copy specifies an existing
-   destination dir with a non-existing destination filename.
+ - Fixed a bug with [`--mkpath`](rsync.1#opt) if a single-file copy specifies
+   an existing destination dir with a non-existing destination filename.
 
  - Fixed `--update -vv` to output "is uptodate" instead of "is newer" messages
-   for files that are being skipped due to an identical modify time.  (This
-   was a new output quirk in 3.2.3.)
+   for files that are being skipped due to an identical modify time.  (This was
+   a new output quirk in 3.2.3.)
 
  - When doing an append transfer, the sending side's file must not get shorter
    or it is skipped. Fixes a crash that could occur when the size changes to 0
    in the middle of the send negotiations.
 
- - When dealing with special files (see `--specials`) in an alt-dest hierarchy,
-   rsync now checks the non-permission mode bits to ensure that the 2 special
-   files are really the same before hard-linking them together.
+ - When dealing with special files (see [`--specials`](rsync.1#opt)) in an
+   alt-dest hierarchy, rsync now checks the non-permission mode bits to ensure
+   that the 2 special files are really the same before hard-linking them
+   together.
 
- - Fixed a bug where `--delay-updates` with stale partial data could cause a
-   file to fail to update.
+ - Fixed a bug where [`--delay-updates`](rsync.1#opt) with stale partial data
+   could cause a file to fail to update.
 
- - Fixed a few places that would output an INFO message with `--info=NAME` that
-   should only have been output given `--verbose` or `--itemize-changes`.
+ - Fixed a few places that would output an INFO message with
+   [`--info=NAME`](rsync.1#opt) that should only have been output given
+   [`--verbose`](rsync.1#opt) or [`--itemize-changes`](rsync.1#opt).
 
- - Avoid a weird failure if you run a local copy with a (useless) `--rsh`
-   option that contains a `V` in the command.
+ - Avoid a weird failure if you run a local copy with a (useless)
+   [`--rsh`](rsync.1#opt) option that contains a `V` in the command.
 
  - Fixed a long-standing compression bug where the compression level of the
    first file transferred affected the level for all future files.  Also, the
    per-file compression skipping has apparently never worked, so it is now
    documented as being ineffective.
 
- - Improved how the `--stop-at`, `--stop-after`, and `--time-limit` options
-   check to see if the allowed time is over, which should make rsync exit more
-   consistently.
+ - Fixed a truncate error when a `--write-devices` copy wrote a file onto a
+   device that was shorter than the device.
 
- - Tweak --progress to display "??:??:??" when the time-remaining value is
-   so large as to be meaningless.
+ - Made `--write-devices` support both `--checksum` and `--no-whole-file` when
+   copying to a device.
+
+ - Improved how the [`--stop-at`](rsync.1#opt), [`--stop-after`](rsync.1#opt),
+   and (the deprecated) [`--time-limit`](rsync.1#opt) options check to see if
+   the allowed time is over, which should make rsync exit more consistently.
+
+ - Tweak --progress to display "`??:??:??`" when the time-remaining value is so
+   large as to be meaningless.
+
+ - Silence some chmod warnings about symlinks when it looks like we have a
+   function to set their permissions but they can't really be set.
+
+ - Fixed a potential issue in git-set-file-times when handling commits with
+   high-bit characters in the description & when handling a description that
+   might mimick the git raw-commit deliniators.  (See the support dir.)
+
+ - The bundled systemd/rsync.service file now includes `Restart=on-failure`.
 
 ### ENHANCEMENTS:
 
  - Use openssl's `-verify_hostname` option in the rsync-ssl script.
 
- - Added extra info to the "FILENAME exists" output of `--ignore-existing` when
-   `--info=skip2` is used.  The skip message becomes "FILENAME exists (INFO)"
-   where the INFO is one of "type change", "sum change" (requires `-c`), "file
-   change" (based on the quick check), "attr change", or "uptodate". Prior
-   versions only supported `--info=skip1`.
+ - Added extra info to the "FILENAME exists" output of
+   [`--ignore-existing`](rsync.1#opt) when [`--info=skip2`](rsync.1#opt) is
+   used.  The skip message becomes "FILENAME exists (INFO)" where the INFO is
+   one of "type change", "sum change" (requires [`--checksum`](rsync.1#opt)),
+   "file change" (based on the quick check), "attr change", or "uptodate".
+   Prior versions only supported `--info=skip1`.
 
- - Added the `--fsync` option (promoted from the patches repo).
+ - Added the [`--fsync`](rsync.1#opt) option (promoted from the patches repo).
+
+ - Added the [`--copy-devices`](rsync.1#opt) option.  Compared to the
+   historical version from the rsync-patches repo, this version: properly
+   handles `--checksum`; fixes a truncation bug when doing an `--inplace` copy
+   onto a longer file; fixes several bugs in the `--itemize` output; and only
+   the sending side needs the enhanced rsync for the copy to work.
 
  - Reduced memory usage for an incremental transfer that has a bunch of small
    directories.
@@ -109,40 +138,49 @@
  - The rsync daemon can now handle a client address with an implied "%scope"
    suffix.
 
- - Added support for `--atimes` on macOS and fixed a bug where it wouldn't work
-   without `--times`.
+ - Added support for [`--atimes`](rsync.1#opt) on macOS and fixed a bug where
+   it wouldn't work without [`--times`](rsync.1#opt).
 
  - Rsync can now update the xattrs on a read-only file when your user can
    temporarily add user-write permission to the file. (It always worked for a
    root transfer.)
 
- - Rsync can now work around an `--inplace` update of a file that is being
-   refused due to the Linux fs.protected_regular sysctl setting.
+ - Rsync can now work around an [`--inplace`](rsync.1#opt) update of a file
+   that is being refused due to the Linux fs.protected_regular sysctl setting.
 
- - When `--chown`, `--usermap`, or `--groupmap` is specified, rsync now makes
-   sure that the appropriate `--owner` and/or `--group` option is enabled.
+ - When [`--chown`](rsync.1#opt), [`--usermap`](rsync.1#opt), or
+   [`--groupmap`](rsync.1#opt) is specified, rsync now makes sure that the
+   appropriate [`--owner`](rsync.1#opt) and/or [`--group`](rsync.1#opt) options
+   are enabled.
 
- - Added the `--info=NONREG` setting to control if rsync should warn about
-   non-regular files in the transfer.  This is enabled by default (keeping the
-   behavior the same as before), so specifying `--info=nonreg0` can be used to
-   turn the warnings off.
+ - Added the [`--info=NONREG`](rsync.1#opt) setting to control if rsync should
+   warn about non-regular files in the transfer.  This is enabled by default
+   (keeping the behavior the same as before), so specifying `--info=nonreg0`
+   can be used to turn the warnings off.
 
- - More ASM optimizations from Shark64.
+ - An optional asm optimization for the rolling checksum from Shark64. Enable
+   it with `./configure --enable-roll-asm`.
+
+ - Using `--debug=FILTER` now outputs a caution message if a filter rule
+   has trailing whitespace.
 
  - Transformed rrsync into a python script with improvements:
    - Security has been beefed up.
    - The known rsync options were updated to include recent additions.
-   - Make rrsync reject `-L`, `-K`, & `-k` by default to make it harder to
+   - Make rrsync reject [`--copy-links`](rsync.1#opt) (`-L`),
+     [`--copy-dirlinks`](rsync.1#opt) (`-k`), &
+     [`--keep-dirlinks`](rsync.1#opt) (`-K`) by default to make it harder to
      exploit any out-of-subdir symlinks.
-   - A new rrsync option of `-munge` tells rrsync to always enable rsync's
-     `--munge-links` option on the server side.
-   - A new rrsync option of `-no-lock` disables a new single-use locking idiom
-     that is the default when `-ro` is not used (useful with `-munge`).
-   - A new rrsync option of `-no-del` disables all `--remove*` and `--delete*`
-     options on the server side.
+   - A new rrsync option of [`-munge`](rrsync.1#opt) tells rrsync to always
+     enable rsync's [`--munge-links`](rsync.1#opt) option on the server side.
+   - A new rrsync option of [`-no-lock`](rrsync.1#opt) disables a new
+     single-use locking idiom that is the default when [`-ro`](rrsync.1#opt) is
+     not used (useful with [`-munge`](rrsync.1#opt)).
+   - A new rrsync option of [`-no-del`](rrsync.1#opt) disables all `--remove*`
+     and `--delete*` rsync options on the server side.
    - The log format has been tweaked slightly to add seconds to the timestamp
      and to output the command executed as a tuple (making the args clearer).
-   - An rrsync.1 man page was added (in the support dir with rrsync).
+   - An rrsync.1 manpage was added (in the support dir with rrsync).
 
  - Added options to the lsh script to facilitate rrsync testing. (See the
    support dir.)
@@ -161,12 +199,12 @@
  - Try to support a client that sent a remote rsync a wacko stderr file handle
    (such as an older File::RsyncP perl library used by BackupPC).
 
- - Some man page improvements.
+ - Lots of manpage improvements, including better HTML versions.
 
 ### PACKAGING RELATED:
 
  - Give configure the `--with-rrsync` option if you want `make install` to
-   install the (now python3) rrsync script and its new man page.
+   install the (now python3) rrsync script and its new manpage.
 
  - If the rrsync script is installed, its package should be changed to depend
    on python3 and the (suggested but not mandatory) python3 braceexpand lib.
@@ -177,18 +215,32 @@
    using the output of `git describe` when building inside a non-shallow git
    checkout, though.)
 
- - Improved the IPv6 determination in configure.
+ - Renamed configure's `--enable-simd` option to `--enable-roll-simd` and added
+   the option `--enable-roll-asm` to use the new asm version of the code.  Both
+   are x86_64/amd64 only.
 
- - Made SIMD & ASM configure default to "no" on non-Linux hosts due to various
-   reports of problems on NetBSD & macOS hosts.  These tests were also tweaked
-   to allow enabling the feature on a host_cpu of amd64 (was only x86_64).
+ - Renamed configure's `--enable-asm` option to `--enable-md5-asm` to avoid
+   confusion with the asm option for the rolling checksum.  It is also honored
+   even when openssl crypto is in use.  This allows: normal MD4 & MD5, normal
+   MD4 + asm MD5, openssl MD4 & MD5, or openssl MD4 + asm MD5 depending on the
+   configure options selected.
+
+ - Made SIMD & asm configure checks default to "no" on non-Linux hosts due to
+   various reports of problems on NetBSD & macOS hosts.  These were also
+   tweaked to allow enabling the feature on a host_cpu of amd64 (was only
+   allowed on x86_64 before).
 
  - Fixed configure to not fail at the SIMD check when cross-compiling.
 
+ - Improved the IPv6 determination in configure.
+
  - Compile the C files with `-pedantic-errors` (when possible) so that we will
    get warned if a static initialization overflows in the future (among other
    things).
 
+ - When linking with an external zlib, rsync renames its `read_buf()` function
+   to `read_buf_()` to avoid a symbol clash on an unpatched zlib.
+
  - Added a SECURITY.md file.
 
 ### DEVELOPER RELATED:
@@ -212,9 +264,7 @@
 
  - Use mallinfo2() instead of mallinfo(), when available.
 
-
 ------------------------------------------------------------------------------
-<a name="3.2.3"></a>
 
 # NEWS for rsync 3.2.3 (6 Aug 2020)
 
@@ -228,65 +278,70 @@
  - Fixed a bug in the xattr code that was not leaving room for the "rsync."
    prefix in some instances where it needed to be added.
 
- - Restored the ability to use `--bwlimit=0` to specify no bandwidth limit.  (It
-   was accidentally broken in 3.2.2.)
+ - Restored the ability to use [`--bwlimit=0`](rsync.1#opt) to specify no
+   bandwidth limit.  (It was accidentally broken in 3.2.2.)
 
- - Fixed a bug when combining `--delete-missing-args` with `--no-implied-dirs` &
-   `-R` where rsync might create the destination path of a missing arg.  The
-   code also avoids some superfluous warnings for nested paths of removed args.
+ - Fixed a bug when combining [`--delete-missing-args`](rsync.1#opt) with
+   [`--no-implied-dirs`](rsync.1#opt) & [`-R`](rsync.1#opt) where rsync might
+   create the destination path of a missing arg.  The code also avoids some
+   superfluous warnings for nested paths of removed args.
 
  - Fixed an issue where hard-linked devices could cause the rdev_major value to
    get out of sync between the sender and the receiver, which could cause a
    device to get created with the wrong major value in its major,minor pair.
 
- - Rsync now complains about a missing `--temp-dir` before starting any file
-   transfers.
+ - Rsync now complains about a missing [`--temp-dir`](rsync.1#opt) before
+   starting any file transfers.
 
  - A completely empty source arg is now a fatal error.  This doesn't change
    the handling of implied dot-dir args such as "localhost:" and such.
 
 ### ENHANCEMENTS:
 
- - Allow `--max-alloc=0` to specify no limit to the alloc sanity check.
+ - Allow [`--max-alloc=0`](rsync.1#opt) to specify no limit to the alloc sanity
+   check.
 
- - Allow `--block-size=SIZE` to specify the size using units (e.g. "100K").
+ - Allow [`--block-size=SIZE`](rsync.1#opt) to specify the size using units
+   (e.g. "100K").
 
  - The name of the id-0 user & group are now sent to the receiver along with
    the other user/group names in the transfer (instead of assuming that both
    sides have the same id-0 names).
 
- - Added the `--stop-after=MINS` and `--stop-at=DATE_TIME` options (with the
-   `--time-limit=MINS` option accepted as an alias for `--stop-after`).  This
-   is an enhanced version of the time-limit patch from the patches repo.
+ - Added the [`--stop-after`](rsync.1#opt) and [`--stop-at`](rsync.1#opt)
+   options (with a [`--time-limit`](rsync.1#opt) alias for `--stop-after`).
+   This is an enhanced version of the time-limit patch from the patches repo.
 
- - Added the `name converter` daemon parameter to make it easier to convert
-   user & group names inside a chrooted daemon module.  This is based on the
-   nameconverter patch with some improvements, including a tweak to the request
-   protocol (so if you used this patch in the past, be sure to update your
-   converter script to use newlines instead of null chars).
+ - Added the [`name converter`](rsyncd.conf.5#opt) daemon parameter to make it
+   easier to convert user & group names inside a chrooted daemon module.  This
+   is based on the nameconverter patch with some improvements, including a
+   tweak to the request protocol (so if you used this patch in the past, be
+   sure to update your converter script to use newlines instead of null chars).
 
- - Added `--crtimes` (`-N`) option for preserving the file's create time (I
-   believe that this is macOS only at the moment).
+ - Added [`--crtimes`](rsync.1#opt) (`-N`) option for preserving the file's
+   create time (I believe that this is macOS only at the moment).
 
- - Added `--mkpath` option to tell rsync that it should create a non-existing
-   path component of the destination arg.
+ - Added [`--mkpath`](rsync.1#opt) option to tell rsync that it should create a
+   non-existing path component of the destination arg.
 
- - Added `--stderr=errors|all|client` to replace the `--msgs2stderr` and
-   `--no-msgs2stderr` options (which are still accepted).  The default use of
-   stderr was changed to be `--stderr=errors` where all the processes that have
-   stderr available output directly to stderr, which should help error messages
-   get to the user more quickly, especially when doing a push (which includes
-   local copying).  This also allows rsync to exit quickly when a receiver
-   failure occurs, since rsync doesn't need to try to keep the connection alive
-   long enough for the fatal error to go from the receiver to the generator to
-   the sender.  The old default can be requested via `--stderr=client`.  Also
-   changed is that a non-default stderr mode is conveyed to the remote rsync
-   (using the older option names) instead of requiring the user to use
-   `--remote-option` (`-M`) to tell the remote rsync what to do.
+ - Added [`--stderr=errors|all|client`](rsync.1#opt) to replace the
+   `--msgs2stderr` and `--no-msgs2stderr` options (which are still accepted).
+   The default use of stderr was changed to be `--stderr=errors` where all the
+   processes that have stderr available output directly to stderr, which should
+   help error messages get to the user more quickly, especially when doing a
+   push (which includes local copying).  This also allows rsync to exit quickly
+   when a receiver failure occurs, since rsync doesn't need to try to keep the
+   connection alive long enough for the fatal error to go from the receiver to
+   the generator to the sender.  The old default can be requested via
+   `--stderr=client`.  Also changed is that a non-default stderr mode is
+   conveyed to the remote rsync (using the older option names) instead of
+   requiring the user to use [`--remote-option`](rsync.1#opt) (`-M`) to tell
+   the remote rsync what to do.
 
- - Added the ability to specify "@netgroup" names to the `hosts allow` and
-   `hosts deny` daemon parameters.  This is a finalized version of the
-   netgroup-auth patch from the patches repo.
+ - Added the ability to specify "@netgroup" names to the [`hosts
+   allow`](rsyncd.conf.5#opt) and [`hosts deny`](rsyncd.conf.5#opt) daemon
+   parameters.  This is a finalized version of the netgroup-auth patch from the
+   patches repo.
 
  - Rsync can now hard-link symlinks on FreeBSD due to it making use of the
    linkat() function when it is available.
@@ -315,7 +370,6 @@
    (with a fallback to the old 1-char string kluge for older compilers).
 
 ------------------------------------------------------------------------------
-<a name="3.2.2"></a>
 
 # NEWS for rsync 3.2.2 (4 Jul 2020)
 
@@ -369,7 +423,7 @@
 
  - Put optimizations into their own list in the `--version` output.
 
- - Improved the man page a bit more.
+ - Improved the manpage a bit more.
 
 ### PACKAGING RELATED:
 
@@ -391,7 +445,6 @@
    can create the interrelated structs and accessors that loadparm.c needs.
 
 ------------------------------------------------------------------------------
-<a name="3.2.1"></a>
 
 # NEWS for rsync 3.2.1 (22 Jun 2020)
 
@@ -454,7 +507,6 @@
  - Merged the OLDNEWS.md file into NEWS.md.
 
 ------------------------------------------------------------------------------
-<a name="3.2.0"></a>
 
 # NEWS for rsync 3.2.0 (19 Jun 2020)
 
@@ -598,7 +650,7 @@
  - The daemon now locks its pid file (when configured to use one) so that it
    will not fail to start when the file exists but no daemon is running.
 
- - Various man page improvements, including some html representations (that
+ - Various manpage improvements, including some html representations (that
    aren't installed by default).
 
  - Made `-V` the short option for `--version` and improved its information.
@@ -615,7 +667,7 @@
 
  - Add installed bash script: /usr/bin/rsync-ssl
 
- - Add installed man page: /usr/man/man1/rsync-ssl.1
+ - Add installed manpage: /usr/man/man1/rsync-ssl.1
 
  - Tweak auxiliary doc file names, such as: README.md, INSTALL.md, & NEWS.md.
 
@@ -637,8 +689,8 @@
    SIMD checksum optimizations.
 
  - Add _build_ dependency for _either_ python3-cmarkcfm or python3-commonmark
-   to allow for patching of man pages or building a git release.  This is not
-   required for a release-tar build, since it comes with pre-built man pages.
+   to allow for patching of manpages or building a git release.  This is not
+   required for a release-tar build, since it comes with pre-built manpages.
    Note that cmarkcfm is faster than commonmark, but they generate the same
    data.  The commonmark dependency is easiest to install since it's native
    python, and can even be installed via `pip3 install --user commonmark` if
@@ -651,7 +703,7 @@
  - Silenced some annoying warnings about major() & minor() by improving an
    autoconf include-file check.
 
- - Converted the man pages from yodl to markdown. They are now processed via a
+ - Converted the manpages from yodl to markdown. They are now processed via a
    simple python3 script using the cmarkgfm **or** commonmark library.  This
    should make it easier to package rsync, since yodl is rather obscure.
 
@@ -666,7 +718,6 @@
  - Some code typos were fixed (as pointed out by a Fossies run).
 
 ------------------------------------------------------------------------------
-<a name="3.1.3"></a>
 
 # NEWS for rsync 3.1.3 (28 Jan 2018)
 
@@ -730,14 +781,13 @@
 
 ### DEVELOPER RELATED:
 
- - Tweak the `make` output when yodl isn't around to create the man pages.
+ - Tweak the `make` output when yodl isn't around to create the manpages.
 
  - Changed an obsolete autoconf compile macro.
 
- - Support newer yodl versions when converting man pages.
+ - Support newer yodl versions when converting manpages.
 
 ------------------------------------------------------------------------------
-<a name="3.1.2"></a>
 
 # NEWS for rsync 3.1.2 (21 Dec 2015)
 
@@ -803,7 +853,6 @@
  - Improved the m4 generation rules and some autoconf idioms.
 
 ------------------------------------------------------------------------------
-<a name="3.1.1"></a>
 
 # NEWS for rsync 3.1.1 (22 Jun 2014)
 
@@ -923,7 +972,6 @@
    and/or zlib code is put early in the CFLAGS.
 
 ------------------------------------------------------------------------------
-<a name="3.1.0"></a>
 
 # NEWS for rsync 3.1.0 (28 Sep 2013)
 
@@ -1174,7 +1222,6 @@
  - Fixed some build issues for Android and Minix.
 
 ------------------------------------------------------------------------------
-<a name="3.0.9"></a>
 
 # NEWS for rsync 3.0.9 (23 Sep 2011)
 
@@ -1227,14 +1274,13 @@
 
  - Avoid trying to reference `SO_BROADCAST` if the OS doesn't support it.
 
- - Fix some issues with the post-processing of the man pages.
+ - Fix some issues with the post-processing of the manpages.
 
  - Fixed the user home-dir handling in the lsh script. (See the support dir.)
 
  - Some minor manpage improvements.
 
 ------------------------------------------------------------------------------
-<a name="3.0.8"></a>
 
 # NEWS for rsync 3.0.8 (26 Mar 2011)
 
@@ -1373,7 +1419,6 @@
  - Fixed the testsuite/xattrs.test script on OS X.
 
 ------------------------------------------------------------------------------
-<a name="3.0.7"></a>
 
 # NEWS for rsync 3.0.7 (31 Dec 2009)
 
@@ -1441,7 +1486,6 @@
  - The testsuite no longer uses `id -u`, so it works better on Solaris.
 
 ------------------------------------------------------------------------------
-<a name="3.0.6"></a>
 
 # NEWS for rsync 3.0.6 (8 May 2009)
 
@@ -1500,7 +1544,6 @@
  - Fixed an failure transferring special files from Solaris to Linux.
 
 ------------------------------------------------------------------------------
-<a name="3.0.5"></a>
 
 # NEWS for rsync 3.0.5 (28 Dec 2008)
 
@@ -1565,7 +1608,6 @@
    idiom. (See the support dir.)
 
 ------------------------------------------------------------------------------
-<a name="3.0.4"></a>
 
 # NEWS for rsync 3.0.4 (6 Sep 2008)
 
@@ -1633,7 +1675,6 @@
    even more consistency checks on the files.
 
 ------------------------------------------------------------------------------
-<a name="3.0.3"></a>
 
 # NEWS for rsync 3.0.3 (29 Jun 2008)
 
@@ -1713,7 +1754,7 @@
    of files, and the ensuring that daemon excludes can't affect a dot-dir arg.
 
  - Improved some build rules for those that build in a separate directory from
-   the source, including better install rules for the man pages, and the fixing
+   the source, including better install rules for the manpages, and the fixing
    of a proto.h-tstamp rule that could make the binaries get rebuild without
    cause.
 
@@ -1724,7 +1765,6 @@
    bleed-over into patches that follow.
 
 ------------------------------------------------------------------------------
-<a name="3.0.2"></a>
 
 # NEWS for rsync 3.0.2 (8 Apr 2008)
 
@@ -1746,7 +1786,6 @@
    packaging dir.
 
 ------------------------------------------------------------------------------
-<a name="3.0.1"></a>
 
 # NEWS for rsync 3.0.1 (3 Apr 2008)
 
@@ -1885,7 +1924,6 @@
  - Updated the build scripts to work with a revised FTP directory structure.
 
 ------------------------------------------------------------------------------
-<a name="3.0.0"></a>
 
 # NEWS for rsync 3.0.0 (1 Mar 2008)
 
@@ -2236,7 +2274,6 @@
    the 3.0.0 release.
 
 ------------------------------------------------------------------------------
-<a name="2.6.9"></a>
 
 # NEWS for rsync 2.6.9 (6 Nov 2006)
 
@@ -2325,7 +2362,7 @@
 
  - Added the `--log-file=FILE` and `--log-file-format=FORMAT` options. These
    can be used to tell any rsync to output what it is doing to a log file.
-   They work with a client rsync, a non-daemon server rsync (see the man page
+   They work with a client rsync, a non-daemon server rsync (see the manpage
    for instructions), and also allows the overriding of rsyncd.conf settings
    when starting a daemon.
 
@@ -2396,7 +2433,6 @@
    consistent opening comments.
 
 ------------------------------------------------------------------------------
-<a name="2.6.8"></a>
 
 # NEWS for rsync 2.6.8 (22 Apr 2006)
 
@@ -2466,7 +2502,6 @@
    actions to a file (something that only a daemon supports at present).
 
 ------------------------------------------------------------------------------
-<a name="2.6.7"></a>
 
 # NEWS for rsync 2.6.7 (11 Mar 2006)
 
@@ -2621,7 +2656,7 @@
  - Added two config items to the rsyncd.conf parsing: `pre-xfer exec` and
    `post-xfer exec`. These allow a command to be specified on a per-module
    basis that will be run before and/or after a daemon-mode transfer. (See the
-   man page for a list of the environment variables that are set with
+   manpage for a list of the environment variables that are set with
    information about the transfer.)
 
  - When using the `--relative` option, you can now insert a dot dir in the
@@ -2790,7 +2825,6 @@
    ~/.popt.
 
 ------------------------------------------------------------------------------
-<a name="2.6.6"></a>
 
 # NEWS for rsync 2.6.6 (28 Jul 2005)
 
@@ -2856,7 +2890,6 @@
    (log-format w/%i) and some double-verbose messages.
 
 ------------------------------------------------------------------------------
-<a name="2.6.5"></a>
 
 # NEWS for rsync 2.6.5 (1 Jun 2005)
 
@@ -2991,7 +3024,7 @@
    usually run with the `--no-detach` option that was necessary to see the
    error on stderr).
 
- - The man pages now consistently refer to an rsync daemon as a `daemon`
+ - The manpages now consistently refer to an rsync daemon as a `daemon`
    instead of a `server` (to distinguish it from the server process in a
    non-daemon transfer).
 
@@ -3037,7 +3070,6 @@
    enables the optional copying of extended attributes.
 
 ------------------------------------------------------------------------------
-<a name="2.6.4"></a>
 
 # NEWS for rsync 2.6.4 (30 March 2005)
 
@@ -3419,7 +3451,6 @@
  - Improved configure to better handle cross-compiling.
 
 ------------------------------------------------------------------------------
-<a name="2.6.3"></a>
 
 # NEWS for rsync 2.6.3 (30 Sep 2004)
 
@@ -3560,12 +3591,12 @@
    without using a temporary file. The matching of existing data in the
    destination file can be severely limited by this, but there are also cases
    where this is more efficient (such as appending data).  Use only when needed
-   (see the man page for more details).
+   (see the manpage for more details).
 
  - Added the `write only` option for the daemon's config file.
 
  - Added long-option names for `-4` and `-6` (namely `--ipv4` and `--ipv6`) and
-   documented all these options in the man page.
+   documented all these options in the manpage.
 
  - Improved the handling of the `--bwlimit` option so that it's less bursty,
    more accurate, and works properly over a larger range of values.
@@ -3640,7 +3671,7 @@
 ### BUILD CHANGES:
 
  - Added a `gen` target to rebuild most of the generated files, including
-   configure, config.h.in, the man pages, and proto.h.
+   configure, config.h.in, the manpages, and proto.h.
 
  - If `make proto` doesn't find some changes in the prototypes, the proto.h
    file is left untouched (its time-stamp used to always be updated).
@@ -3663,7 +3694,6 @@
    removed.
 
 ------------------------------------------------------------------------------
-<a name="2.6.2"></a>
 
 # NEWS for rsync 2.6.2 (30 Apr 2004)
 
@@ -3705,7 +3735,6 @@
  - Two new diffs were added to the patches dir.
 
 ------------------------------------------------------------------------------
-<a name="2.6.1"></a>
 
 # NEWS for rsync 2.6.1 (26 Apr 2004)
 
@@ -3897,7 +3926,6 @@
    applied, and rebuilt the rest.
 
 ------------------------------------------------------------------------------
-<a name="2.6.0"></a>
 
 # NEWS for rsync 2.6.0 (1 Jan 2004)
 
@@ -4037,7 +4065,6 @@
    other side (primarily for testing purposes).  (Wayne Davison)
 
 ------------------------------------------------------------------------------
-<a name="2.5.7"></a>
 
 # NEWS for rsync 2.5.7 (4 Dec 2003)
 
@@ -4049,7 +4076,6 @@
    Andrea Barisani)
 
 ------------------------------------------------------------------------------
-<a name="2.5.6"></a>
 
 # NEWS for rsync 2.5.6, aka "the dwd-between-jobs release" (26 Jan 2003)
 
@@ -4144,7 +4170,6 @@
    should build on more platforms. (Paul Green)
 
 ------------------------------------------------------------------------------
-<a name="2.5.5"></a>
 
 # NEWS for rsync 2.5.5, aka Snowy River (2 Apr 2002)
 
@@ -4183,7 +4208,6 @@
  - Improved network error handling. (Greg A. Woods)
 
 ------------------------------------------------------------------------------
-<a name="2.5.4"></a>
 
 # NEWS for rsync 2.5.4, aka "Imitation lizard skin" (13 Mar 2002)
 
@@ -4203,7 +4227,6 @@
  - Additional test cases for `--compress`. (Martin Pool)
 
 ------------------------------------------------------------------------------
-<a name="2.5.3"></a>
 
 # NEWS for rsync 2.5.3, aka "Happy 26" (11 Mar 2002)
 
@@ -4246,13 +4269,12 @@
  - Added `--no-whole-file` and `--no-blocking-io` options (Dave Dykstra)
 
  - Made the `--write-batch` and `--read-batch` options actually work and added
-   documentation in the man page (Jos Backus)
+   documentation in the manpage (Jos Backus)
 
  - If the daemon is unable to fork a child to accept a connection, print an
    error message. (Colin Walters)
 
 ------------------------------------------------------------------------------
-<a name="2.5.2"></a>
 
 # NEWS for rsync 2.5.2 (26 Jan 2002)
 
@@ -4301,7 +4323,6 @@
    Razor. (Debian #124286)
 
 ------------------------------------------------------------------------------
-<a name="2.5.1"></a>
 
 # NEWS for rsync 2.5.1 (3 Jan 2002)
 
@@ -4336,7 +4357,6 @@
  - Clearer error messages for some conditions.
 
 ------------------------------------------------------------------------------
-<a name="2.5.0"></a>
 
 # NEWS for rsync 2.5.0 (30 Nov 2001)
 
@@ -4462,7 +4482,7 @@
 
 | RELEASE DATE | VER.   | DATE OF COMMIT\* | PROTOCOL    |
 |--------------|--------|------------------|-------------|
-| ?? Jan 2022  | 3.2.4  |                  | 31          |
+| 15 Apr 2022  | 3.2.4  |                  | 31          |
 | 06 Aug 2020  | 3.2.3  |                  | 31          |
 | 04 Jul 2020  | 3.2.2  |                  | 31          |
 | 22 Jun 2020  | 3.2.1  |                  | 31          |

README.md

@@ -65,8 +65,8 @@ RSYNC DAEMONS
 -------------
 
 Rsync can also talk to "rsync daemons" which can provide anonymous or
-authenticated rsync.  See the rsyncd.conf(5) man page for details on how
-to setup an rsync daemon.  See the rsync(1) man page for info on how to
+authenticated rsync.  See the rsyncd.conf(5) manpage for details on how
+to setup an rsync daemon.  See the rsync(1) manpage for info on how to
 connect to an rsync daemon.
 
 

checksum.c

@@ -3,7 +3,7 @@
  *
  * Copyright (C) 1996 Andrew Tridgell
  * Copyright (C) 1996 Paul Mackerras
- * Copyright (C) 2004-2020 Wayne Davison
+ * Copyright (C) 2004-2022 Wayne Davison
  *
  * This program is free software; you can redistribute it and/or modify
  * it under the terms of the GNU General Public License as published by
@@ -179,7 +179,7 @@ int canonical_checksum(int csum_type)
 	return 0;
 }
 
-#ifndef HAVE_SIMD /* See simd-checksum-*.cpp. */
+#ifndef USE_ROLL_SIMD /* See simd-checksum-*.cpp. */
 /*
   a simple 32 bit checksum that can be updated from either end
   (inspired by Mark Adler's Adler-32 checksum)
@@ -222,23 +222,23 @@ void get_checksum2(char *buf, int32 len, char *sum)
 	  }
 #endif
 	  case CSUM_MD5: {
-		MD5_CTX m5;
+		md5_context m5;
 		uchar seedbuf[4];
-		MD5_Init(&m5);
+		md5_begin(&m5);
 		if (proper_seed_order) {
 			if (checksum_seed) {
 				SIVALu(seedbuf, 0, checksum_seed);
-				MD5_Update(&m5, seedbuf, 4);
+				md5_update(&m5, seedbuf, 4);
 			}
-			MD5_Update(&m5, (uchar *)buf, len);
+			md5_update(&m5, (uchar *)buf, len);
 		} else {
-			MD5_Update(&m5, (uchar *)buf, len);
+			md5_update(&m5, (uchar *)buf, len);
 			if (checksum_seed) {
 				SIVALu(seedbuf, 0, checksum_seed);
-				MD5_Update(&m5, seedbuf, 4);
+				md5_update(&m5, seedbuf, 4);
 			}
 		}
-		MD5_Final((uchar *)sum, &m5);
+		md5_result(&m5, (uchar *)sum);
 		break;
 	  }
 	  case CSUM_MD4:
@@ -374,18 +374,18 @@ void file_checksum(const char *fname, const STRUCT_STAT *st_p, char *sum)
 	  }
 #endif
 	  case CSUM_MD5: {
-		MD5_CTX m5;
+		md5_context m5;
 
-		MD5_Init(&m5);
+		md5_begin(&m5);
 
 		for (i = 0; i + CHUNK_SIZE <= len; i += CHUNK_SIZE)
-			MD5_Update(&m5, (uchar *)map_ptr(buf, i, CHUNK_SIZE), CHUNK_SIZE);
+			md5_update(&m5, (uchar *)map_ptr(buf, i, CHUNK_SIZE), CHUNK_SIZE);
 
 		remainder = (int32)(len - i);
 		if (remainder > 0)
-			MD5_Update(&m5, (uchar *)map_ptr(buf, i, remainder), remainder);
+			md5_update(&m5, (uchar *)map_ptr(buf, i, remainder), remainder);
 
-		MD5_Final((uchar *)sum, &m5);
+		md5_result(&m5, (uchar *)sum);
 		break;
 	  }
 	  case CSUM_MD4:
@@ -443,7 +443,7 @@ static union {
 #ifdef USE_OPENSSL
 	MD4_CTX m4;
 #endif
-	MD5_CTX m5;
+	md5_context m5;
 } ctx;
 #ifdef SUPPORT_XXHASH
 static XXH64_state_t* xxh64_state;
@@ -482,7 +482,7 @@ void sum_init(int csum_type, int seed)
 		break;
 #endif
 	  case CSUM_MD5:
-		MD5_Init(&ctx.m5);
+		md5_begin(&ctx.m5);
 		break;
 	  case CSUM_MD4:
 #ifdef USE_OPENSSL
@@ -532,7 +532,7 @@ void sum_update(const char *p, int32 len)
 		break;
 #endif
 	  case CSUM_MD5:
-		MD5_Update(&ctx.m5, (uchar *)p, len);
+		md5_update(&ctx.m5, (uchar *)p, len);
 		break;
 	  case CSUM_MD4:
 #ifdef USE_OPENSSL
@@ -597,7 +597,7 @@ int sum_end(char *sum)
 	  }
 #endif
 	  case CSUM_MD5:
-		MD5_Final((uchar *)sum, &ctx.m5);
+		md5_result(&ctx.m5, (uchar *)sum);
 		break;
 	  case CSUM_MD4:
 #ifdef USE_OPENSSL

clientserver.c

@@ -47,6 +47,7 @@ extern int protocol_version;
 extern int io_timeout;
 extern int no_detach;
 extern int write_batch;
+extern int old_style_args;
 extern int default_af_hint;
 extern int logfile_format_has_i;
 extern int logfile_format_has_o_or_i;
@@ -288,20 +289,45 @@ int start_inband_exchange(int f_in, int f_out, const char *user, int argc, char
 
 	sargs[sargc++] = ".";
 
+	if (!old_style_args)
+		snprintf(line, sizeof line, " %.*s/", modlen, modname);
+
 	while (argc > 0) {
 		if (sargc >= MAX_ARGS - 1) {
 		  arg_overflow:
 			rprintf(FERROR, "internal: args[] overflowed in do_cmd()\n");
 			exit_cleanup(RERR_SYNTAX);
 		}
-		if (strncmp(*argv, modname, modlen) == 0
-		 && argv[0][modlen] == '\0')
+		if (strncmp(*argv, modname, modlen) == 0 && argv[0][modlen] == '\0')
 			sargs[sargc++] = modname; /* we send "modname/" */
-		else if (**argv == '-') {
-			if (asprintf(sargs + sargc++, "./%s", *argv) < 0)
-				out_of_memory("start_inband_exchange");
-		} else
-			sargs[sargc++] = *argv;
+		else {
+			char *arg = *argv;
+			int extra_chars = *arg == '-' ? 2 : 0; /* a leading dash needs a "./" prefix. */
+			/* If --old-args was not specified, make sure that the arg won't split at a mod name! */
+			if (!old_style_args && (p = strstr(arg, line)) != NULL) {
+				do {
+					extra_chars += 2;
+				} while ((p = strstr(p+1, line)) != NULL);
+			}
+			if (extra_chars) {
+				char *f = arg;
+				char *t = arg = new_array(char, strlen(arg) + extra_chars + 1);
+				if (*f == '-') {
+					*t++ = '.';
+					*t++ = '/';
+				}
+				while (*f) {
+					if (*f == ' ' && strncmp(f, line, modlen+2) == 0) {
+						*t++ = '[';
+						*t++ = *f++;
+						*t++ = ']';
+					} else
+						*t++ = *f++;
+				}
+				*t = '\0';
+			}
+			sargs[sargc++] = arg;
+		}
 		argv++;
 		argc--;
 	}

compat.c

@@ -604,7 +604,7 @@ void setup_protocol(int f_out,int f_in)
 	if (remote_protocol < MIN_PROTOCOL_VERSION
 	 || remote_protocol > MAX_PROTOCOL_VERSION) {
 		rprintf(FERROR,"protocol version mismatch -- is your shell clean?\n");
-		rprintf(FERROR,"(see the rsync man page for an explanation)\n");
+		rprintf(FERROR,"(see the rsync manpage for an explanation)\n");
 		exit_cleanup(RERR_PROTOCOL);
 	}
 	if (remote_protocol < OLD_PROTOCOL_VERSION) {

configure.ac

@@ -83,7 +83,7 @@ if test x"$enable_profile" = x"yes"; then
 	CFLAGS="$CFLAGS -pg"
 fi
 
-AC_MSG_CHECKING([if md2man can create man pages])
+AC_MSG_CHECKING([if md2man can create manpages])
 if test x"$ac_cv_path_PYTHON3" = x; then
     AC_MSG_RESULT(no - python3 not found)
     md2man_works=no
@@ -101,7 +101,7 @@ fi
 
 AC_MSG_CHECKING([if we require man-page building])
 AC_ARG_ENABLE([md2man],
-	AS_HELP_STRING([--disable-md2man],[disable to omit man page creation]))
+	AS_HELP_STRING([--disable-md2man],[disable to omit manpage creation]))
 if test x"$enable_md2man" != x"no"; then
     if test -f "$srcdir/rsync.1"; then
 	AC_MSG_RESULT(optional)
@@ -109,7 +109,7 @@ if test x"$enable_md2man" != x"no"; then
 	AC_MSG_RESULT(required)
 	if test x"$md2man_works" = x"no"; then
 	    err_msg="$err_msg$nl- You need python3 and either the cmarkgfm OR commonmark python3 lib in order"
-	    err_msg="$err_msg$nl  to build man pages based on the git source (man pages are included in the"
+	    err_msg="$err_msg$nl  to build manpages based on the git source (manpages are included in the"
 	    err_msg="$err_msg$nl  official release tar files)."
 	    no_lib="$no_lib md2man"
 	fi
@@ -136,12 +136,13 @@ if test x"$GCC" = x"yes"; then
 fi
 
 AC_ARG_WITH(rrsync,
-        AS_HELP_STRING([--with-rrsync],[also install the rrsync script and its man page]))
+        AS_HELP_STRING([--with-rrsync],[also install the rrsync script and its manpage]))
 if test x"$with_rrsync" != x"yes"; then
     with_rrsync=no
 else
     MAKE_RRSYNC='rrsync'
     MAKE_RRSYNC_1='rrsync.1'
+    GEN_RRSYNC='rrsync.1 rrsync.1.html'
 fi
 AC_SUBST(with_rrsync)
 
@@ -228,12 +229,12 @@ fi
 AC_DEFINE_UNQUOTED(NOBODY_USER, "$NOBODY_USER", [unprivileged user--e.g. nobody])
 AC_DEFINE_UNQUOTED(NOBODY_GROUP, "$NOBODY_GROUP", [unprivileged group for unprivileged user])
 
-# SIMD optimizations
-SIMD=
+# rolling-checksum SIMD optimizations
+ROLL_SIMD=
 
-AC_MSG_CHECKING([whether to enable SIMD optimizations])
-AC_ARG_ENABLE(simd,
-    AS_HELP_STRING([--enable-simd],[enable/disable to control SIMD optimizations (requires c++)]))
+AC_MSG_CHECKING([whether to enable rolling-checksum SIMD optimizations])
+AC_ARG_ENABLE(roll-simd,
+    AS_HELP_STRING([--enable-roll-simd],[enable/disable to control rolling-checksum SIMD optimizations (requires c++)]))
 
 # Clag is crashing with -g -O2, so we'll get rid of -g for now.
 CXXFLAGS=`echo "$CXXFLAGS" | sed 's/-g //'`
@@ -262,14 +263,14 @@ __attribute__ ((target("ssse3"))) void more_testing(char* buf, int len)
 }
 ]])
 
-if test x"$enable_simd" = x""; then
+if test x"$enable_roll_simd" = x""; then
     case "$host_os" in
 	*linux*) ;;
-	*) enable_simd=no ;;
+	*) enable_roll_simd=no ;;
     esac
 fi
 
-if test x"$enable_simd" != x"no"; then
+if test x"$enable_roll_simd" != x"no"; then
     # For x86-64 SIMD, g++ >=5 or clang++ >=7 is required
     if test x"$host_cpu" = x"x86_64" || test x"$host_cpu" = x"amd64"; then
 	AC_LANG(C++)
@@ -282,23 +283,23 @@ if test x"$enable_simd" != x"no"; then
 	AC_LANG(C)
 	if test x"$CXX_OK" = x"yes"; then
 	    # AC_MSG_RESULT() is called below.
-	    SIMD="$host_cpu"
-	elif test x"$enable_simd" = x"yes"; then
+	    ROLL_SIMD="$host_cpu"
+	elif test x"$enable_roll_simd" = x"yes"; then
 	    AC_MSG_RESULT(error)
-	    AC_MSG_ERROR(The SIMD compilation test failed.
-Omit --enable-simd to continue without it.)
+	    AC_MSG_ERROR(The rolling-checksum SIMD compilation test failed.
+Omit --enable-roll-simd to continue without it.)
 	fi
-    elif test x"$enable_simd" = x"yes"; then
+    elif test x"$enable_roll_simd" = x"yes"; then
         AC_MSG_RESULT(unavailable)
-        AC_MSG_ERROR(The SIMD optimizations are currently x86_64|amd64 only.
-Omit --enable-simd to continue without it.)
+        AC_MSG_ERROR(The rolling-checksum SIMD optimizations are currently x86_64|amd64 only.
+Omit --enable-roll-simd to continue without it.)
     fi
 fi
 
-if test x"$SIMD" != x""; then
-    AC_MSG_RESULT([yes ($SIMD)])
-    AC_DEFINE(HAVE_SIMD, 1, [Define to 1 to enable SIMD optimizations])
-    SIMD='$(SIMD_'"$SIMD)"
+if test x"$ROLL_SIMD" != x""; then
+    AC_MSG_RESULT([yes ($ROLL_SIMD)])
+    AC_DEFINE(USE_ROLL_SIMD, 1, [Define to 1 to enable rolling-checksum SIMD optimizations])
+    ROLL_SIMD='$(ROLL_SIMD_'"$ROLL_SIMD)"
     # We only use c++ for its target attribute dispatching, disable unneeded bulky features
     CXXFLAGS="$CXXFLAGS -fno-exceptions -fno-rtti"
     # Apple often has "g++" as a symlink for clang. Try to find out the truth.
@@ -310,7 +311,7 @@ else
     AC_MSG_RESULT(no)
 fi
 
-AC_SUBST(SIMD)
+AC_SUBST(ROLL_SIMD)
 
 AC_MSG_CHECKING([if assembler accepts noexecstack])
 OLD_CFLAGS="$CFLAGS"
@@ -321,52 +322,19 @@ AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[ ]], [[return 0;]])],
 CFLAGS="$OLD_CFLAGS"
 AC_SUBST(NOEXECSTACK)
 
-ASM=
-
-AC_MSG_CHECKING([whether to enable ASM optimizations])
-AC_ARG_ENABLE(asm,
-    AS_HELP_STRING([--enable-asm],[enable/disable to control ASM optimizations]))
-
-if test x"$enable_asm" = x""; then
-    case "$host_os" in
-	*linux*) ;;
-	*) enable_asm=no ;;
-    esac
-fi
-
-if test x"$enable_asm" != x"no"; then
-    if test x"$host_cpu" = x"x86_64" || test x"$host_cpu" = x"amd64"; then
-	ASM="$host_cpu"
-    elif test x"$enable_asm" = x"yes"; then
-        AC_MSG_RESULT(unavailable)
-        AC_MSG_ERROR(The ASM optimizations are currently x86_64|amd64 only.
-Omit --enable-asm to continue without it.)
-    fi
-fi
-
-if test x"$ASM" != x""; then
-    AC_MSG_RESULT([yes ($ASM)])
-    AC_DEFINE(HAVE_ASM, 1, [Define to 1 to enable ASM optimizations])
-    ASM='$(ASM_'"$ASM)"
-else
-    AC_MSG_RESULT(no)
-fi
-
-AC_SUBST(ASM)
-
 # arrgh. libc in some old debian version screwed up the largefile
 # stuff, getting byte range locking wrong
 AC_CACHE_CHECK([for broken largefile support],rsync_cv_HAVE_BROKEN_LARGEFILE,[
 AC_RUN_IFELSE([AC_LANG_SOURCE([[
 #define _FILE_OFFSET_BITS 64
-#include <stdio.h>
-#include <fcntl.h>
-#ifdef HAVE_SYS_TYPES_H
-#include <sys/types.h>
+$ac_includes_default
+#ifdef HAVE_FCNTL_H
+# include <fcntl.h>
+#elif defined HAVE_SYS_FCNTL_H
+# include <sys/fcntl.h>
 #endif
+#ifdef HAVE_SYS_WAIT_H
 #include <sys/wait.h>
-#if HAVE_UNISTD_H
-#include <unistd.h>
 #endif
 
 int main(void)
@@ -411,7 +379,7 @@ AS_HELP_STRING([--disable-ipv6],[disable to omit ipv6 support]),
 	;;
   esac ],
 
-  AC_TRY_RUN([ /* AF_INET6 avalable check */
+  AC_TRY_RUN([ /* AF_INET6 availability check */
 #include <stdlib.h>
 #include <sys/types.h>
 #include <sys/socket.h>
@@ -456,7 +424,8 @@ if test x"$enable_openssl" != x"no"; then
     if test x"$ac_cv_header_openssl_md4_h" = x"yes" && test x"$ac_cv_header_openssl_md5_h" = x"yes"; then
       AC_MSG_RESULT(yes)
       AC_SEARCH_LIBS(MD5_Init, crypto,
-          [AC_DEFINE(USE_OPENSSL)],
+          [AC_DEFINE(USE_OPENSSL)
+	   enable_openssl=yes],
           [err_msg="$err_msg$nl- Failed to find MD5_Init function in openssl crypto lib.";
 	   no_lib="$no_lib openssl"])
     else
@@ -464,10 +433,67 @@ if test x"$enable_openssl" != x"no"; then
 	err_msg="$err_msg$nl- Failed to find openssl/md4.h and openssl/md5.h for openssl crypto lib support."
 	no_lib="$no_lib openssl"
     fi
+    if test x"$enable_md5_asm" != x"yes"; then
+	enable_md5_asm=no
+    fi
 else
     AC_MSG_RESULT(no)
 fi
 
+MD5_ASM=
+
+AC_MSG_CHECKING([whether to enable MD5 ASM optimizations])
+AC_ARG_ENABLE(md5-asm,
+    AS_HELP_STRING([--enable-md5-asm],[enable/disable to control MD5 ASM optimizations]))
+
+if test x"$enable_md5_asm" = x""; then
+    case "$host_os" in
+	*linux*) ;;
+	*) enable_md5_asm=no ;;
+    esac
+fi
+
+if test x"$enable_md5_asm" != x"no"; then
+    if test x"$host_cpu" = x"x86_64" || test x"$host_cpu" = x"amd64"; then
+	MD5_ASM="$host_cpu"
+    elif test x"$enable_md5_asm" = x"yes"; then
+        AC_MSG_RESULT(unavailable)
+        AC_MSG_ERROR(The ASM optimizations are currently x86_64|amd64 only.
+Omit --enable-md5-asm to continue without it.)
+    fi
+fi
+
+if test x"$MD5_ASM" != x""; then
+    AC_MSG_RESULT([yes ($MD5_ASM)])
+    AC_DEFINE(USE_MD5_ASM, 1, [Define to 1 to enable MD5 ASM optimizations])
+    MD5_ASM='$(MD5_ASM_'"$MD5_ASM)"
+else
+    AC_MSG_RESULT(no)
+fi
+
+AC_SUBST(MD5_ASM)
+
+ROLL_ASM=
+
+AC_MSG_CHECKING([whether to enable rolling-checksum ASM optimizations])
+AC_ARG_ENABLE(roll-asm,
+    AS_HELP_STRING([--enable-roll-asm],[enable/disable to control rolling-checksum ASM optimizations (requires --enable-roll-simd)]))
+
+if test x"$ROLL_SIMD" = x""; then
+    enable_roll_asm=no
+fi
+
+if test x"$enable_roll_asm" = x"yes"; then
+    ROLL_ASM="$host_cpu"
+    AC_MSG_RESULT([yes ($ROLL_ASM)])
+    AC_DEFINE(USE_ROLL_ASM, 1, [Define to 1 to enable rolling-checksum ASM optimizations (requires --enable-roll-simd)])
+    ROLL_ASM='$(ROLL_ASM_'"$ROLL_ASM)"
+else
+    AC_MSG_RESULT(no)
+fi
+
+AC_SUBST(ROLL_ASM)
+
 AC_MSG_CHECKING([whether to enable xxhash checksum support])
 AC_ARG_ENABLE([xxhash],
 	AS_HELP_STRING([--disable-xxhash],[disable to omit xxhash checksums]))
@@ -537,7 +563,7 @@ if test x"$no_lib" != x; then
     echo "$err_msg"
     echo ""
     echo "See the INSTALL file for hints on how to install the missing libraries and/or"
-    echo "how to generate (or fetch) man pages:"
+    echo "how to generate (or fetch) manpages:"
     echo "    https://github.com/WayneD/rsync/blob/master/INSTALL.md"
     echo ""
     echo "To disable one or more features, the relevant configure options are:"
@@ -1257,6 +1283,7 @@ AC_SUBST(BUILD_POPT)
 AC_SUBST(BUILD_ZLIB)
 AC_SUBST(MAKE_RRSYNC)
 AC_SUBST(MAKE_RRSYNC_1)
+AC_SUBST(GEN_RRSYNC)
 AC_SUBST(MAKE_MAN)
 
 AC_CHECK_FUNCS(_acl __acl _facl __facl)

exclude.c

@@ -4,7 +4,7 @@
  * Copyright (C) 1996-2001 Andrew Tridgell <tridge@samba.org>
  * Copyright (C) 1996 Paul Mackerras
  * Copyright (C) 2002 Martin Pool
- * Copyright (C) 2003-2020 Wayne Davison
+ * Copyright (C) 2003-2022 Wayne Davison
  *
  * This program is free software; you can redistribute it and/or modify
  * it under the terms of the GNU General Public License as published by
@@ -152,13 +152,17 @@ static void add_rule(filter_rule_list *listp, const char *pat, unsigned int pat_
 {
 	const char *cp;
 	unsigned int pre_len, suf_len, slash_cnt = 0;
+	char *mention_rule_suffix;
 
-	if (DEBUG_GTE(FILTER, 2)) {
-		rprintf(FINFO, "[%s] add_rule(%s%.*s%s)%s\n",
+	if (DEBUG_GTE(FILTER, 1) && pat_len && (pat[pat_len-1] == ' ' || pat[pat_len-1] == '\t'))
+		mention_rule_suffix = " -- CAUTION: trailing whitespace!";
+	else
+		mention_rule_suffix = DEBUG_GTE(FILTER, 2) ? "" : NULL;
+	if (mention_rule_suffix) {
+		rprintf(FINFO, "[%s] add_rule(%s%.*s%s)%s%s\n",
 			who_am_i(), get_rule_prefix(rule, pat, 0, NULL),
-			(int)pat_len, pat,
-			(rule->rflags & FILTRULE_DIRECTORY) ? "/" : "",
-			listp->debug_type);
+			(int)pat_len, pat, (rule->rflags & FILTRULE_DIRECTORY) ? "/" : "",
+			listp->debug_type, mention_rule_suffix);
 	}
 
 	/* These flags also indicate that we're reading a list that

flist.c

@@ -43,6 +43,7 @@ extern int use_qsort;
 extern int xfer_dirs;
 extern int filesfrom_fd;
 extern int one_file_system;
+extern int copy_devices;
 extern int copy_dirlinks;
 extern int preserve_uid;
 extern int preserve_gid;
@@ -700,6 +701,7 @@ static struct file_struct *recv_file_entry(int f, struct file_list *flist, int x
 	int alloc_len, basename_len, linkname_len;
 	int extra_len = file_extra_cnt * EXTRA_LEN;
 	int first_hlink_ndx = -1;
+	char real_ISREG_entry;
 	int64 file_length;
 #ifdef CAN_SET_NSEC
 	uint32 modtime_nsec;
@@ -814,6 +816,7 @@ static struct file_struct *recv_file_entry(int f, struct file_list *flist, int x
 				linkname_len = strlen(F_SYMLINK(first)) + 1;
 			else
 				linkname_len = 0;
+			real_ISREG_entry = S_ISREG(mode) ? 1 : 0;
 			goto create_object;
 		}
 	}
@@ -941,10 +944,20 @@ static struct file_struct *recv_file_entry(int f, struct file_list *flist, int x
 #endif
 		linkname_len = 0;
 
+	if (copy_devices && IS_DEVICE(mode)) {
+		/* This is impossible in the official release, but some pre-release patches
+		 * didn't convert the device into a file before sending, so we'll do it here
+		 * (even though the length is typically 0 and any checksum data is zeros). */
+		mode = S_IFREG | (mode & ACCESSPERMS);
+		modtime = time(NULL); /* The mtime on the device is not up-to-date, so set it to "now". */
+		real_ISREG_entry = 0;
+	} else
+		real_ISREG_entry = S_ISREG(mode) ? 1 : 0;
+
 #ifdef SUPPORT_HARD_LINKS
   create_object:
 	if (preserve_hard_links) {
-		if (protocol_version < 28 && S_ISREG(mode))
+		if (protocol_version < 28 && real_ISREG_entry)
 			xflags |= XMIT_HLINKED;
 		if (xflags & XMIT_HLINKED)
 			extra_len += (inc_recurse+1) * EXTRA_LEN;
@@ -1160,8 +1173,8 @@ static struct file_struct *recv_file_entry(int f, struct file_list *flist, int x
 	}
 #endif
 
-	if (always_checksum && (S_ISREG(mode) || protocol_version < 28)) {
-		if (S_ISREG(mode))
+	if (always_checksum && (real_ISREG_entry || protocol_version < 28)) {
+		if (real_ISREG_entry)
 			bp = F_SUM(file);
 		else {
 			/* Prior to 28, we get a useless set of nulls. */
@@ -1360,6 +1373,18 @@ struct file_struct *make_file(const char *fname, struct file_list *flist,
 	linkname_len = 0;
 #endif
 
+	if (copy_devices && am_sender && IS_DEVICE(st.st_mode)) {
+		if (st.st_size == 0) {
+			int fd = do_open(fname, O_RDONLY, 0);
+			if (fd >= 0) {
+				st.st_size = get_device_size(fd, fname);
+				close(fd);
+			}
+		}
+		st.st_mode = S_IFREG | (st.st_mode & ACCESSPERMS);
+		st.st_mtime = time(NULL); /* The mtime on the device is not up-to-date, so set it to "now". */
+	}
+
 #ifdef ST_MTIME_NSEC
 	if (st.ST_MTIME_NSEC && protocol_version >= 31)
 		extra_len += EXTRA_LEN;

generator.c

@@ -35,11 +35,11 @@ extern int inc_recurse;
 extern int relative_paths;
 extern int implied_dirs;
 extern int keep_dirlinks;
+extern int write_devices;
 extern int preserve_acls;
 extern int preserve_xattrs;
 extern int preserve_links;
 extern int preserve_devices;
-extern int write_devices;
 extern int preserve_specials;
 extern int preserve_hard_links;
 extern int preserve_executability;
@@ -532,7 +532,7 @@ void itemize(const char *fnamecmp, struct file_struct *file, int ndx, int statre
 				iflags |= ITEM_REPORT_CRTIME;
 		}
 #endif
-#if !defined HAVE_LCHMOD && !defined HAVE_SETATTRLIST
+#ifndef CAN_CHMOD_SYMLINK
 		if (S_ISLNK(file->mode)) {
 			;
 		} else
@@ -1793,6 +1793,12 @@ static void recv_generator(char *fname, struct file_struct *file, int ndx,
 		goto cleanup;
 	}
 
+	if (write_devices && IS_DEVICE(sx.st.st_mode) && sx.st.st_size == 0) {
+		/* This early open into fd skips the regular open below. */
+		if ((fd = do_open(fnamecmp, O_RDONLY, 0)) >= 0)
+			real_sx.st.st_size = sx.st.st_size = get_device_size(fd, fnamecmp);
+	}
+
 	if (fnamecmp_type <= FNAMECMP_BASIS_DIR_HIGH)
 		;
 	else if (fnamecmp_type >= FNAMECMP_FUZZY)
@@ -1858,7 +1864,7 @@ static void recv_generator(char *fname, struct file_struct *file, int ndx,
 	}
 
 	/* open the file */
-	if ((fd = do_open(fnamecmp, O_RDONLY, 0)) < 0) {
+	if (fd < 0 && (fd = do_open(fnamecmp, O_RDONLY, 0)) < 0) {
 		rsyserr(FERROR, errno, "failed to open %s, continuing",
 			full_fname(fnamecmp));
 	  pretend_missing:
@@ -1875,11 +1881,9 @@ static void recv_generator(char *fname, struct file_struct *file, int ndx,
 
 	if (inplace && make_backups > 0 && fnamecmp_type == FNAMECMP_FNAME) {
 		if (!(backupptr = get_backup_name(fname))) {
-			close(fd);
 			goto cleanup;
 		}
 		if (!(back_file = make_file(fname, NULL, NULL, 0, NO_FILTERS))) {
-			close(fd);
 			goto pretend_missing;
 		}
 		if (robust_unlink(backupptr) && errno != ENOENT) {
@@ -1887,14 +1891,12 @@ static void recv_generator(char *fname, struct file_struct *file, int ndx,
 				full_fname(backupptr));
 			unmake_file(back_file);
 			back_file = NULL;
-			close(fd);
 			goto cleanup;
 		}
 		if ((f_copy = do_open(backupptr, O_WRONLY | O_CREAT | O_TRUNC | O_EXCL, 0600)) < 0) {
 			rsyserr(FERROR_XFER, errno, "open %s", full_fname(backupptr));
 			unmake_file(back_file);
 			back_file = NULL;
-			close(fd);
 			goto cleanup;
 		}
 		fnamecmp_type = FNAMECMP_BACKUP;
@@ -1945,7 +1947,6 @@ static void recv_generator(char *fname, struct file_struct *file, int ndx,
 		write_sum_head(f_out, NULL);
 	else if (sx.st.st_size <= 0) {
 		write_sum_head(f_out, NULL);
-		close(fd);
 	} else {
 		if (generate_and_send_sums(fd, sx.st.st_size, f_out, f_copy) < 0) {
 			rprintf(FWARNING,
@@ -1953,10 +1954,11 @@ static void recv_generator(char *fname, struct file_struct *file, int ndx,
 				fnamecmp);
 			write_sum_head(f_out, NULL);
 		}
-		close(fd);
 	}
 
   cleanup:
+	if (fd >= 0)
+		close(fd);
 	if (back_file) {
 		int save_preserve_xattrs = preserve_xattrs;
 		if (f_copy >= 0)

io.c

@@ -1843,6 +1843,7 @@ int64 read_longint(int f)
 #endif
 }
 
+/* Debugging note: this will be named read_buf_() when using an external zlib. */
 void read_buf(int f, char *buf, size_t len)
 {
 	if (f != iobuf.in_fd) {

lib/md5-asm-x86_64.S

@@ -27,7 +27,7 @@
 #include "config.h"
 #include "md-defines.h"
 
-#if !defined USE_OPENSSL && CSUM_CHUNK == 64
+#ifdef USE_MD5_ASM /* { */
 
 #ifdef __APPLE__
 #define md5_process_asm _md5_process_asm
@@ -698,4 +698,4 @@ md5_process_asm:
 	pop	%rbp
 	ret
 
-#endif /* !USE_OPENSSL ... */
+#endif /* } USE_MD5_ASM */

lib/md5.c

@@ -2,7 +2,7 @@
  * RFC 1321 compliant MD5 implementation
  *
  * Copyright (C) 2001-2003 Christophe Devine
- * Copyright (C) 2007-2020 Wayne Davison
+ * Copyright (C) 2007-2022 Wayne Davison
  *
  * This program is free software; you can redistribute it and/or modify
  * it under the terms of the GNU General Public License as published by
@@ -20,7 +20,7 @@
 
 #include "rsync.h"
 
-#ifndef USE_OPENSSL
+#if !defined USE_OPENSSL || USE_MD5_ASM /* { */
 void md5_begin(md_context *ctx)
 {
 	ctx->A = 0x67452301;
@@ -148,7 +148,10 @@ static void md5_process(md_context *ctx, const uchar data[CSUM_CHUNK])
 	ctx->D += D;
 }
 
-#if defined HAVE_ASM && CSUM_CHUNK == 64
+#ifdef USE_MD5_ASM
+#if CSUM_CHUNK != 64
+#error The MD5 ASM code does not support CSUM_CHUNK != 64
+#endif
 extern void md5_process_asm(md_context *ctx, const void *data, size_t num);
 #endif
 
@@ -176,20 +179,20 @@ void md5_update(md_context *ctx, const uchar *input, uint32 length)
 		left = 0;
 	}
 
-#if defined HAVE_ASM && CSUM_CHUNK == 64
+#ifdef USE_MD5_ASM /* { */
 	if (length >= CSUM_CHUNK) {
 		uint32 chunks = length / CSUM_CHUNK;
 		md5_process_asm(ctx, input, chunks);
 		length -= chunks * CSUM_CHUNK;
 		input += chunks * CSUM_CHUNK;
 	}
-#else
+#else /* } { */
 	while (length >= CSUM_CHUNK) {
 		md5_process(ctx, input);
 		length -= CSUM_CHUNK;
 		input  += CSUM_CHUNK;
 	}
-#endif
+#endif /* } */
 
 	if (length)
 		memcpy(ctx->buffer + left, input, length);
@@ -221,9 +224,9 @@ void md5_result(md_context *ctx, uchar digest[MD5_DIGEST_LEN])
 	SIVALu(digest, 8, ctx->C);
 	SIVALu(digest, 12, ctx->D);
 }
-#endif
+#endif /* } */
 
-#ifdef TEST_MD5
+#ifdef TEST_MD5 /* { */
 
 void get_md5(uchar *out, const uchar *input, int n)
 {
@@ -317,4 +320,4 @@ int main(int argc, char *argv[])
 	return 0;
 }
 
-#endif
+#endif /* } */

lib/mdigest.h

@@ -17,12 +17,13 @@ void mdfour_begin(md_context *md);
 void mdfour_update(md_context *md, const uchar *in, uint32 length);
 void mdfour_result(md_context *md, uchar digest[MD4_DIGEST_LEN]);
 
-#ifndef USE_OPENSSL
-#define MD5_CTX md_context
-#define MD5_Init md5_begin
-#define MD5_Update md5_update
-#define MD5_Final(digest, cptr) md5_result(cptr, digest)
-
+#if defined USE_OPENSSL && !defined USE_MD5_ASM
+#define md5_context MD5_CTX
+#define md5_begin MD5_Init
+#define md5_update MD5_Update
+#define md5_result(cptr, digest) MD5_Final(digest, cptr)
+#else
+#define md5_context md_context
 void md5_begin(md_context *ctx);
 void md5_update(md_context *ctx, const uchar *input, uint32 length);
 void md5_result(md_context *ctx, uchar digest[MD5_DIGEST_LEN]);

lib/snprintf.c

@@ -20,7 +20,7 @@
  * for string length.  This covers a nasty loophole.
  *
  * The other functions are there to prevent NULL pointers from
- * causing nast effects.
+ * causing nasty effects.
  *
  * More Recently:
  *  Brandon Long <blong@fiction.net> 9/15/96 for mutt 0.43

main.c

@@ -48,6 +48,7 @@ extern int called_from_signal_handler;
 extern int need_messages_from_generator;
 extern int kluge_around_eof;
 extern int got_xfer_error;
+extern int old_style_args;
 extern int msgs2stderr;
 extern int module_id;
 extern int read_only;
@@ -477,7 +478,7 @@ static void show_malloc_stats(void)
 
 #define PRINT_ALLOC_NUM(title, descr, num) \
 	rprintf(FINFO, "  %-11s%10" SIZE_T_FMT_MOD "d   (" descr ")\n", \
-	       title ":", (SIZE_T_FMT_CAST)(num));
+		title ":", (SIZE_T_FMT_CAST)(num));
 
 	PRINT_ALLOC_NUM("arena", "bytes from sbrk", mi.arena);
 	PRINT_ALLOC_NUM("ordblks", "chunks not in use", mi.ordblks);
@@ -1470,6 +1471,10 @@ static int start_client(int argc, char *argv[])
 		rsync_port = 0;
 	}
 
+	/* A local transfer doesn't unbackslash anything, so leave the args alone. */
+	if (local_server)
+		old_style_args = 2;
+
 	if (!rsync_port && remote_argc && !**remote_argv) /* Turn an empty arg into a dot dir. */
 		*remote_argv = ".";
 

md-convert

@@ -32,6 +32,8 @@
 import os, sys, re, argparse, subprocess, time
 from html.parser import HTMLParser
 
+VALID_PAGES = 'README INSTALL COPYING rsync.1 rrsync.1 rsync-ssl.1 rsyncd.conf.5'.split()
+
 CONSUMES_TXT = set('h1 h2 h3 p li pre'.split())
 
 HTML_START = """\
@@ -47,7 +49,7 @@ body {
 body, b, strong, u {
   font-family: 'Roboto', sans-serif;
 }
-a.tgt { font-face: symbol; font-weight: 400; font-size: 70%; visibility: hidden; text-decoration: none; color: #ddd; padding: 0 4px; border: 0; vertical-align: top; }
+a.tgt { font-face: symbol; font-weight: 400; font-size: 70%; visibility: hidden; text-decoration: none; color: #ddd; padding: 0 4px; border: 0; }
 a.tgt:after { content: '🔗'; }
 a.tgt:hover { color: #444; background-color: #eaeaea; }
 h1:hover > a.tgt, h2:hover > a.tgt, h3:hover > a.tgt, dt:hover > a.tgt { visibility: visible; }
@@ -111,6 +113,21 @@ UNDR_FONT = ('\3', r"\fI")
 NBR_DASH = ('\4', r"\-")
 NBR_SPACE = ('\xa0', r"\ ")
 
+FILENAME_RE = re.compile(r'^(?P<fn>(?P<srcdir>.+/)?(?P<name>(?P<prog>[^/]+?)(\.(?P<sect>\d+))?)\.md)$')
+ASSIGNMENT_RE = re.compile(r'^(\w+)=(.+)')
+QUOTED_RE = re.compile(r'"(.+?)"')
+VAR_REF_RE = re.compile(r'\$\{(\w+)\}')
+VERSION_RE = re.compile(r' (\d[.\d]+)[, ]')
+BIN_CHARS_RE = re.compile(r'[\1-\7]+')
+SPACE_DOUBLE_DASH_RE = re.compile(r'\s--(\s)')
+NON_SPACE_SINGLE_DASH_RE = re.compile(r'(^|\W)-')
+WHITESPACE_RE = re.compile(r'\s')
+CODE_BLOCK_RE = re.compile(r'[%s]([^=%s]+)[=%s]' % (BOLD_FONT[0], NORM_FONT[0], NORM_FONT[0]))
+NBR_DASH_RE = re.compile(r'[%s]' % NBR_DASH[0])
+INVALID_TARGET_CHARS_RE = re.compile(r'[^-A-Za-z0-9._]')
+INVALID_START_CHAR_RE = re.compile(r'^([^A-Za-z0-9])')
+MANIFY_LINESTART_RE = re.compile(r"^(['.])", flags=re.M)
+
 md_parser = None
 env_subs = { }
 
@@ -125,15 +142,15 @@ def main():
 
 
 def parse_md_file(mdfn):
-    fi = re.match(r'^(?P<fn>(?P<srcdir>.+/)?(?P<name>(?P<prog>[^/]+?)(\.(?P<sect>\d+))?)\.md)$', mdfn)
+    fi = FILENAME_RE.match(mdfn)
     if not fi:
         die('Failed to parse a md input file name:', mdfn)
     fi = argparse.Namespace(**fi.groupdict())
     fi.want_manpage = not not fi.sect
     if fi.want_manpage:
-        fi.title = fi.prog + '(' + fi.sect + ') man page'
+        fi.title = fi.prog + '(' + fi.sect + ') manpage'
     else:
-        fi.title = fi.prog
+        fi.title = fi.prog + ' for rsync'
 
     if fi.want_manpage:
         if not env_subs:
@@ -172,6 +189,8 @@ def parse_md_file(mdfn):
     if fi.want_manpage:
         output_list += [ (fi.name, fi.man_out) ]
     for fn, txt in output_list:
+        if args.dest and args.dest != '.':
+            fn = os.path.join(args.dest, fn)
         if os.path.lexists(fn):
             os.unlink(fn)
         print("Wrote:", fn)
@@ -205,19 +224,19 @@ def find_man_substitutions():
 
         with open(srcdir + 'version.h', 'r', encoding='utf-8') as fh:
             txt = fh.read()
-        m = re.search(r'"(.+?)"', txt)
+        m = QUOTED_RE.search(txt)
         env_subs['VERSION'] = m.group(1)
 
         with open('Makefile', 'r', encoding='utf-8') as fh:
             for line in fh:
-                m = re.match(r'^(\w+)=(.+)', line)
+                m = ASSIGNMENT_RE.match(line)
                 if not m:
                     continue
                 var, val = (m.group(1), m.group(2))
                 if var == 'prefix' and env_subs[var] is not None:
                     continue
-                while re.search(r'\$\{', val):
-                    val = re.sub(r'\$\{(\w+)\}', lambda m: env_subs[m.group(1)], val)
+                while VAR_REF_RE.search(val):
+                    val = VAR_REF_RE.sub(lambda m: env_subs[m.group(1)], val)
                 env_subs[var] = val
                 if var == 'srcdir':
                     break
@@ -251,9 +270,10 @@ class TransformHtml(HTMLParser):
                 derived_hashtags = set(),
                 referenced_hashtags = set(),
                 bad_hashtags = set(),
-                prior_target = None,
+                latest_targets = [ ],
                 opt_prefix = 'opt',
                 a_txt_start = None,
+                target_suf = '',
                 )
 
         if st.want_manpage:
@@ -276,11 +296,11 @@ class TransformHtml(HTMLParser):
         fi.man_out = ''.join(st.man_out)
         st.man_out = None
 
-        for href, txt in st.derived_hashtags:
-            derived = txt2target(txt, href[1:])
+        for tgt, txt in st.derived_hashtags:
+            derived = txt2target(txt, tgt)
             if derived not in st.created_hashtags:
-                txt = re.sub(r'[\1-\7]+', '', txt.replace(NBR_DASH[0], '-').replace(NBR_SPACE[0], ' '))
-                warn('Unknown derived hashtag link in', self.fn, 'based on:', (href, txt))
+                txt = BIN_CHARS_RE.sub('', txt.replace(NBR_DASH[0], '-').replace(NBR_SPACE[0], ' '))
+                warn('Unknown derived hashtag link in', self.fn, 'based on:', (tgt, txt))
 
         for bad in st.bad_hashtags:
             if bad in st.created_hashtags:
@@ -330,7 +350,7 @@ class TransformHtml(HTMLParser):
             st.txt += BOLD_FONT[0]
         elif tag == 'em' or  tag == 'i':
             if st.want_manpage:
-                tag = 'u' # Change it into underline to be more like the man page
+                tag = 'u' # Change it into underline to be more like the manpage
                 st.txt += UNDR_FONT[0]
         elif tag == 'ol':
             start = 1
@@ -362,13 +382,19 @@ class TransformHtml(HTMLParser):
             st.a_href = None
             for var, val in attrs_list:
                 if var == 'href':
-                    if val in ('#', '#opt', '#daemon-opt'):
-                        st.a_href = val
-                    elif val.startswith('#'):
-                        st.referenced_hashtags.add(val[1:])
-                        if val[1:] == st.prior_target:
-                            warn('Found link to the current section in', self.fn + ':', val)
-                    elif not val.startswith(('https://', 'http://', 'mailto:', 'ftp:', './')):
+                    if val.startswith(('https://', 'http://', 'mailto:', 'ftp:')):
+                        pass # nothing to check
+                    elif '#' in val:
+                        pg, tgt = val.split('#', 2)
+                        if pg and pg not in VALID_PAGES or '#' in tgt:
+                            st.bad_hashtags.add(val)
+                        elif tgt in ('', 'opt', 'dopt'):
+                            st.a_href = val
+                        elif pg == '':
+                            st.referenced_hashtags.add(tgt)
+                            if tgt in st.latest_targets:
+                                warn('Found link to the current section in', self.fn + ':', val)
+                    elif val not in VALID_PAGES:
                         st.bad_hashtags.add(val)
             st.a_txt_start = len(st.txt)
         st.html_out.append('<' + tag + ''.join(' ' + var + '="' + htmlify(val) + '"' for var, val in attrs_list) + '>')
@@ -385,19 +411,28 @@ class TransformHtml(HTMLParser):
         else:
             txt = None
         add_to_txt = None
-        if tag == 'h1' or tag == 'h2':
+        if tag == 'h1':
+            tgt = txt
+            target_suf = ''
+            if tgt.startswith('NEWS for '):
+                m = VERSION_RE.search(tgt)
+                if m:
+                    tgt = m.group(1)
+                    st.target_suf = '-' + tgt
+            self.add_targets(tag, tgt)
+        elif tag == 'h2':
             st.man_out.append(st.p_macro + '.SH "' + manify(txt) + '"\n')
-            self.add_target(txt)
-            st.opt_prefix = 'daemon-opt' if txt == 'DAEMON OPTIONS' else 'opt'
+            self.add_targets(tag, txt, st.target_suf)
+            st.opt_prefix = 'dopt' if txt == 'DAEMON OPTIONS' else 'opt'
         elif tag == 'h3':
             st.man_out.append(st.p_macro + '.SS "' + manify(txt) + '"\n')
-            self.add_target(txt)
+            self.add_targets(tag, txt, st.target_suf)
         elif tag == 'p':
             if st.dt_from == 'p':
                 tag = 'dt'
                 st.man_out.append('.IP "' + manify(txt) + '"\n')
                 if txt.startswith(BOLD_FONT[0]):
-                    self.add_target(txt)
+                    self.add_targets(tag, txt)
                 st.dt_from = None
             elif txt != '':
                 st.man_out.append(manify(txt) + "\n")
@@ -421,7 +456,7 @@ class TransformHtml(HTMLParser):
             add_to_txt = NORM_FONT[0]
         elif tag == 'em' or  tag == 'i':
             if st.want_manpage:
-                tag = 'u' # Change it into underline to be more like the man page
+                tag = 'u' # Change it into underline to be more like the manpage
                 add_to_txt = NORM_FONT[0]
         elif tag == 'ol' or tag == 'ul':
             if st.list_state.pop() == 'dl':
@@ -439,11 +474,13 @@ class TransformHtml(HTMLParser):
                 find = 'href="' + st.a_href + '"'
                 for j in range(len(st.html_out)-1, 0, -1):
                     if find in st.html_out[j]:
-                        derived = txt2target(atxt, st.a_href[1:])
-                        if derived == st.prior_target:
-                            warn('Found link to the current section in', self.fn + ':', derived)
-                        st.derived_hashtags.add((st.a_href, atxt))
-                        st.html_out[j] = st.html_out[j].replace(find, 'href="#' + derived + '"')
+                        pg, tgt = st.a_href.split('#', 2)
+                        derived = txt2target(atxt, tgt)
+                        if pg == '':
+                            if derived in st.latest_targets:
+                                warn('Found link to the current section in', self.fn + ':', st.a_href)
+                            st.derived_hashtags.add((tgt, atxt))
+                        st.html_out[j] = st.html_out[j].replace(find, 'href="' + pg + '#' + derived + '"')
                         break
                 else:
                     die('INTERNAL ERROR: failed to find href in html data:', find)
@@ -472,23 +509,47 @@ class TransformHtml(HTMLParser):
         if st.in_pre:
             html = htmlify(txt)
         else:
-            txt = re.sub(r'\s--(\s)', NBR_SPACE[0] + r'--\1', txt).replace('--', NBR_DASH[0]*2)
-            txt = re.sub(r'(^|\W)-', r'\1' + NBR_DASH[0], txt)
+            txt = SPACE_DOUBLE_DASH_RE.sub(NBR_SPACE[0] + r'--\1', txt).replace('--', NBR_DASH[0]*2)
+            txt = NON_SPACE_SINGLE_DASH_RE.sub(r'\1' + NBR_DASH[0], txt)
             html = htmlify(txt)
             if st.in_code:
-                txt = re.sub(r'\s', NBR_SPACE[0], txt)
+                txt = WHITESPACE_RE.sub(NBR_SPACE[0], txt)
                 html = html.replace(NBR_DASH[0], '-').replace(NBR_SPACE[0], ' ') # <code> is non-breaking in CSS
         st.html_out.append(html.replace(NBR_SPACE[0], '&nbsp;').replace(NBR_DASH[0], '-&#8288;'))
         st.txt += txt
 
 
-    def add_target(self, txt):
+    def add_targets(self, tag, txt, suf=None):
         st = self.state
-        txt = txt2target(txt, st.opt_prefix)
-        if txt:
-            st.html_out.append('<a id="' + txt + '" href="#' + txt + '" class="tgt"></a>')
+        tag = '<' + tag + '>'
+        targets = CODE_BLOCK_RE.findall(txt)
+        if not targets:
+            targets = [ txt ]
+        tag_pos = 0
+        for txt in targets:
+            txt = txt2target(txt, st.opt_prefix)
+            if not txt:
+                continue
+            if suf:
+                txt += suf
+            if txt in st.created_hashtags:
+                for j in range(2, 1000):
+                    chk = txt + '-' + str(j)
+                    if chk not in st.created_hashtags:
+                        print('Made link target unique:', chk)
+                        txt = chk
+                        break
+            if tag_pos == 0:
+                tag_pos -= 1
+                while st.html_out[tag_pos] != tag:
+                    tag_pos -= 1
+                st.html_out[tag_pos] = tag[:-1] + ' id="' + txt + '">'
+                st.html_out.append('<a href="#' + txt + '" class="tgt"></a>')
+                tag_pos -= 1 # take into account the append
+            else:
+                st.html_out[tag_pos] = '<span id="' + txt + '"></span>' + st.html_out[tag_pos]
             st.created_hashtags.add(txt)
-            st.prior_target = txt
+        st.latest_targets = targets
 
 
     def output_debug(self, event, extra):
@@ -505,24 +566,27 @@ class TransformHtml(HTMLParser):
 
 
 def txt2target(txt, opt_prefix):
-    txt = re.sub(r'[%s](.+?)[=%s].*' % (BOLD_FONT[0], NORM_FONT[0]), r'\1', txt.strip())
-    txt = re.sub(r'[%s]' % NBR_DASH[0], '-', txt)
-    txt = re.sub(r'[\1-\7]+', '', txt)
-    txt = re.sub(r'[^-A-Za-z0-9._]', '_', txt)
+    txt = txt.strip().rstrip(':')
+    m = CODE_BLOCK_RE.search(txt)
+    if m:
+        txt = m.group(1)
+    txt = NBR_DASH_RE.sub('-', txt)
+    txt = BIN_CHARS_RE.sub('', txt)
+    txt = INVALID_TARGET_CHARS_RE.sub('_', txt)
     if opt_prefix and txt.startswith('-'):
         txt = opt_prefix + txt
     else:
-        txt = re.sub(r'^([^A-Za-z])', r't\1', txt)
+        txt = INVALID_START_CHAR_RE.sub(r't\1', txt)
     return txt
 
 
 def manify(txt):
-    return re.sub(r"^(['.])", r'\&\1', txt.replace('\\', '\\\\')
+    return MANIFY_LINESTART_RE.sub(r'\&\1', txt.replace('\\', '\\\\')
             .replace(NBR_SPACE[0], NBR_SPACE[1])
             .replace(NBR_DASH[0], NBR_DASH[1])
             .replace(NORM_FONT[0], NORM_FONT[1])
             .replace(BOLD_FONT[0], BOLD_FONT[1])
-            .replace(UNDR_FONT[0], UNDR_FONT[1]), flags=re.M)
+            .replace(UNDR_FONT[0], UNDR_FONT[1]))
 
 
 def htmlify(txt):
@@ -543,6 +607,7 @@ def die(*msg):
 if __name__ == '__main__':
     parser = argparse.ArgumentParser(description="Output html and (optionally) nroff for markdown pages.", add_help=False)
     parser.add_argument('--test', action='store_true', help="Just test the parsing without outputting any files.")
+    parser.add_argument('--dest', metavar='DIR', help="Put files into DIR instead of the current directory.")
     parser.add_argument('--debug', '-D', action='count', default=0, help='Output copious info on the html parsing. Repeat for even more.')
     parser.add_argument("--help", "-h", action="help", help="Output this help message and exit.")
     parser.add_argument("mdfiles", nargs='+', help="The source .md files to convert.")

options.c

@@ -47,6 +47,7 @@ int append_mode = 0;
 int keep_dirlinks = 0;
 int copy_dirlinks = 0;
 int copy_links = 0;
+int copy_devices = 0;
 int write_devices = 0;
 int preserve_links = 0;
 int preserve_hard_links = 0;
@@ -656,6 +657,7 @@ static struct poptOption long_options[] = {
   {"no-D",             0,  POPT_ARG_NONE,   0, OPT_NO_D, 0, 0 },
   {"devices",          0,  POPT_ARG_VAL,    &preserve_devices, 1, 0, 0 },
   {"no-devices",       0,  POPT_ARG_VAL,    &preserve_devices, 0, 0, 0 },
+  {"copy-devices",     0,  POPT_ARG_NONE,   &copy_devices, 0, 0, 0 },
   {"write-devices",    0,  POPT_ARG_VAL,    &write_devices, 1, 0, 0 },
   {"no-write-devices", 0,  POPT_ARG_VAL,    &write_devices, 0, 0, 0 },
   {"specials",         0,  POPT_ARG_VAL,    &preserve_specials, 1, 0, 0 },
@@ -949,6 +951,7 @@ static void set_refuse_options(void)
 		 || strcmp("iconv", longName) == 0
 		 || strcmp("no-iconv", longName) == 0
 		 || strcmp("checksum-seed", longName) == 0
+		 || strcmp("copy-devices", longName) == 0 /* disable wild-match (it gets refused below) */
 		 || strcmp("write-devices", longName) == 0 /* disable wild-match (it gets refused below) */
 		 || strcmp("log-format", longName) == 0 /* aka out-format (NOT log-file-format) */
 		 || strcmp("sender", longName) == 0
@@ -960,6 +963,7 @@ static void set_refuse_options(void)
 	assert(list_end != NULL);
 
 	if (am_daemon) { /* Refused by default, but can be accepted via a negated exact match. */
+		parse_one_refuse_match(0, "copy-devices", list_end);
 		parse_one_refuse_match(0, "write-devices", list_end);
 	}
 
@@ -1933,10 +1937,18 @@ int parse_arguments(int *argc_p, const char ***argv_p)
 	}
 
 	if (old_style_args < 0) {
-		if (!am_server && (arg = getenv("RSYNC_OLD_ARGS")) != NULL && *arg)
+		if (!am_server && protect_args <= 0 && (arg = getenv("RSYNC_OLD_ARGS")) != NULL && *arg) {
+			protect_args = 0;
 			old_style_args = atoi(arg);
-		else
+		} else
 			old_style_args = 0;
+	} else if (old_style_args) {
+		if (protect_args > 0) {
+			snprintf(err_buf, sizeof err_buf,
+				 "--protect-args conflicts with --old-args.\n");
+			return 0;
+		}
+		protect_args = 0;
 	}
 
 	if (protect_args < 0) {
@@ -2909,6 +2921,9 @@ void server_options(char **args, int *argc_p)
 	else if (remove_source_files)
 		args[ac++] = "--remove-sent-files";
 
+	if (copy_devices && !am_sender)
+		args[ac++] = "--copy-devices";
+
 	if (preallocate_files && am_sender)
 		args[ac++] = "--preallocate";
 

packaging/lsb/rsync.spec

@@ -1,9 +1,9 @@
 Summary: A fast, versatile, remote (and local) file-copying tool
 Name: rsync
 Version: 3.2.4
-%define fullversion %{version}pre2
-Release: 0.1.pre2
-%define srcdir src-previews
+%define fullversion %{version}
+Release: 1
+%define srcdir src
 Group: Applications/Internet
 License: GPL
 Source0: https://rsync.samba.org/ftp/rsync/%{srcdir}/rsync-%{fullversion}.tar.gz
@@ -79,8 +79,8 @@ rm -rf $RPM_BUILD_ROOT
 %dir /etc/rsync-ssl/certs
 
 %changelog
-* Sat Jan 15 2022 Wayne Davison <wayne@opencoder.net>
-Released 3.2.4pre2.
+* Fri Apr 15 2022 Wayne Davison <wayne@opencoder.net>
+Released 3.2.4.
 
 * Fri Mar 21 2008 Wayne Davison <wayne@opencoder.net>
 Added installation of /etc/xinetd.d/rsync file and some commented-out

packaging/pkglib.py

@@ -181,7 +181,7 @@ def mandate_gensend_hook():
             die('Please add a "make gensend" into your', hook, 'script.')
 
 
-# Snag the GENFILES values out of the Makefile.in file and return them as a list.
+# Snag the GENFILES values out of the Makefile file and return them as a list.
 def get_gen_files(want_dir_plus_list=False):
     cont_re = re.compile(r'\\\n')
 
@@ -189,7 +189,7 @@ def get_gen_files(want_dir_plus_list=False):
 
     auto_dir = os.path.join('auto-build-save', cmd_txt('git rev-parse --abbrev-ref HEAD').out.strip().replace('/', '%'))
 
-    with open('Makefile.in', 'r', encoding='utf-8') as fh:
+    with open(auto_dir + '/Makefile', 'r', encoding='utf-8') as fh:
         for line in fh:
             if not gen_files:
                 chk = re.sub(r'^GENFILES=', '', line)

packaging/release-rsync

@@ -206,10 +206,10 @@ About to:
             txt = replace_or_die(x_re, repl, txt, f"Unable to find SUBPROTOCOL_VERSION define in {fn}")
         elif fn == 'NEWS.md':
             efv = re.escape(finalversion)
-            x_re = re.compile(r'^<.+>\s+# NEWS for rsync %s \(UNRELEASED\)\s+## Changes in this version:\n' % efv
+            x_re = re.compile(r'^# NEWS for rsync %s \(UNRELEASED\)\s+## Changes in this version:\n' % efv
                     + r'(\n### PROTOCOL NUMBER:\s+- The protocol number was changed to \d+\.\n)?')
             rel_day = 'UNRELEASED' if pre else today
-            repl = (f'<a name="{finalversion}"></a>\n\n# NEWS for rsync {finalversion} ({rel_day})\n\n'
+            repl = (f'# NEWS for rsync {finalversion} ({rel_day})\n\n'
                 + '## Changes in this version:\n')
             if proto_changed:
                 repl += f'\n### PROTOCOL NUMBER:\n\n - The protocol number was changed to {protocol_version}.\n'
@@ -341,7 +341,7 @@ About to:
     md_files = 'README.md NEWS.md INSTALL.md'.split()
     html_files = [ fn for fn in gen_pathnames if fn.endswith('.html') ]
     cmd_chk(['rsync', '-a', *md_files, *html_files, dest])
-    cmd_chk(["./md-convert"] + [ dest +'/'+ fn for fn in md_files ])
+    cmd_chk(["./md-convert", "--dest", dest, *md_files])
 
     cmd_chk(f"git log --name-status | gzip -9 >{dest}/ChangeLog.gz")
 

packaging/systemd/rsync.service

@@ -7,6 +7,7 @@ Documentation=man:rsync(1) man:rsyncd.conf(5)
 [Service]
 ExecStart=/usr/bin/rsync --daemon --no-detach
 RestartSec=1
+Restart=on-failure
 
 # Citing README.md:
 #

prepare-source

@@ -6,7 +6,7 @@
 #
 #   build     build the config files [the default w/no arg]
 #   fetch     fetch the latest dev autoconfig files
-#   fetchgen  fetch all the latest dev generated files (including man pages)
+#   fetchgen  fetch all the latest dev generated files (including manpages)
 #   fetchSRC  fetch the latest dev source files [NON-GENERATED FILES]
 #
 # The script stops after the first successful action.

receiver.c

@@ -808,14 +808,16 @@ int recv_files(int f_in, int f_out, char *local_name)
 			continue;
 		}
 
-		if (fd1 != -1 && !(S_ISREG(st.st_mode) || (write_devices && IS_DEVICE(st.st_mode)))) {
+		if (write_devices && IS_DEVICE(st.st_mode)) {
+			if (fd1 != -1 && st.st_size == 0)
+				st.st_size = get_device_size(fd1, fname);
+			/* Mark the file entry as a device so that we don't try to truncate it later on. */
+			file->mode = S_IFBLK | (file->mode & ACCESSPERMS);
+		} else if (fd1 != -1 && !(S_ISREG(st.st_mode))) {
 			close(fd1);
 			fd1 = -1;
 		}
 
-		if (fd1 != -1 && IS_DEVICE(st.st_mode) && st.st_size == 0)
-			st.st_size = get_device_size(fd1, fname);
-
 		/* If we're not preserving permissions, change the file-list's
 		 * mode based on the local permissions and some heuristics. */
 		if (!preserve_perms) {

rsync-ssl.1.md

@@ -8,8 +8,8 @@ rsync-ssl - a helper script for connecting to an ssl rsync daemon
 rsync-ssl [--type=SSL_TYPE] RSYNC_ARGS
 ```
 
-The online version of this man page (that includes cross-linking of topics)
-is available at <https://download.samba.org/pub/rsync/rsync.1>.
+The online version of this manpage (that includes cross-linking of topics)
+is available at <https://download.samba.org/pub/rsync/rsync-ssl.1>.
 
 ## DESCRIPTION
 
@@ -96,7 +96,7 @@ The ssl helper scripts are affected by the following environment variables:
 
 ## SEE ALSO
 
-[**rsync**(1)](./rsync.1), [**rsyncd.conf**(5)](./rsyncd.conf.5)
+[**rsync**(1)](rsync.1), [**rsyncd.conf**(5)](rsyncd.conf.5)
 
 ## CAVEATS
 
@@ -117,12 +117,12 @@ Please report bugs! See the web site at <https://rsync.samba.org/>.
 
 ## VERSION
 
-This man page is current for version @VERSION@ of rsync.
+This manpage is current for version @VERSION@ of rsync.
 
 ## CREDITS
 
-rsync is distributed under the GNU General Public License.  See the file
-[COPYING](./COPYING) for details.
+Rsync is distributed under the GNU General Public License.  See the file
+[COPYING](COPYING) for details.
 
 A web site is available at <https://rsync.samba.org/>.  The site includes an
 FAQ-O-Matic which may cover questions unanswered by this manual page.

rsync.h

@@ -18,11 +18,6 @@
  * with this program; if not, visit the http://fsf.org website.
  */
 
-/* a non-zero CHAR_OFFSET makes the rolling sum stronger, but is
-   incompatible with older versions :-( */
-#define CHAR_OFFSET 0
-
-#ifndef AVX2_ASM /* do not include the rest of file for assembly */
 #define False 0
 #define True 1
 #define Unset (-1) /* Our BOOL values are always an int. */
@@ -43,6 +38,9 @@
 
 #define BACKUP_SUFFIX "~"
 
+/* a non-zero CHAR_OFFSET makes the rolling sum stronger, but is
+   incompatible with older versions :-( */
+#define CHAR_OFFSET 0
 
 /* These flags are only used during the flist transfer. */
 
@@ -1174,6 +1172,10 @@ struct name_num_obj {
 	struct name_num_item list[10]; /* we'll get a compile error/warning if this is ever too small */
 };
 
+#ifdef EXTERNAL_ZLIB
+#define read_buf read_buf_
+#endif
+
 #ifndef __cplusplus
 #include "proto.h"
 #endif
@@ -1477,7 +1479,6 @@ const char *get_panic_action(void);
     fprintf(stderr, "%s in %s at line %d\n", msg, __FILE__, __LINE__); \
     exit_cleanup(RERR_UNSUPPORTED); \
 } while (0)
-#endif  /* AVX2_ASM */
 
 #ifdef HAVE_MALLINFO2
 #define MEM_ALLOC_INFO mallinfo2

rsyncd.conf.5.md

@@ -6,7 +6,7 @@ rsyncd.conf - configuration file for rsync in daemon mode
 
 rsyncd.conf
 
-The online version of this man page (that includes cross-linking of topics)
+The online version of this manpage (that includes cross-linking of topics)
 is available at <https://download.samba.org/pub/rsync/rsyncd.conf.5>.
 
 ## DESCRIPTION
@@ -45,8 +45,7 @@ Case is not significant in boolean values, but is preserved in string values.
 
 ## LAUNCHING THE RSYNC DAEMON
 
-The rsync daemon is launched by specifying the `--daemon` option to
-rsync.
+The rsync daemon is launched by specifying the `--daemon` option to rsync.
 
 The daemon must run with root privileges if you wish to use chroot, to bind to
 a port numbered under 1024 (as is the default 873), or to set file ownership.
@@ -99,9 +98,9 @@ a literal % into a value is to use %%.
 
 0.  `motd file`
 
-    This parameter allows you to specify a "message of the day" to display to
-    clients on each connect. This usually contains site information and any
-    legal notices. The default is no motd file.  This can be overridden by the
+    This parameter allows you to specify a "message of the day" (MOTD) to display
+    to clients on each connect. This usually contains site information and any
+    legal notices. The default is no MOTD file.  This can be overridden by the
     `--dparam=motdfile=FILE` command-line option when starting the daemon.
 
 0.  `pid file`
@@ -129,7 +128,7 @@ a literal % into a value is to use %%.
 
     This parameter can provide endless fun for people who like to tune their
     systems to the utmost degree. You can set all sorts of socket options which
-    may make transfers faster (or slower!). Read the man page for the
+    may make transfers faster (or slower!). Read the manpage for the
     **setsockopt()** system call for details on some of the options you may be
     able to set. By default no special socket options are set.  These settings
     can also be specified via the `--sockopts` command-line option.
@@ -717,7 +716,7 @@ the values of parameters.  See the GLOBAL PARAMETERS section for more details.
       addresses which match the masked IP address will be allowed in.
     - a hostname pattern using wildcards. If the hostname of the connecting IP
       (as determined by a reverse lookup) matches the wildcarded name (using
-      the same rules as normal unix filename matching), the client is allowed
+      the same rules as normal Unix filename matching), the client is allowed
       in.  This only works if "[reverse lookup](#)" is enabled (the default).
     - a hostname. A plain hostname is matched against the reverse DNS of the
       connecting IP (if "[reverse lookup](#)" is enabled), and/or the IP of the
@@ -934,9 +933,10 @@ the values of parameters.  See the GLOBAL PARAMETERS section for more details.
     If you are un-refusing the compress option, you may want to match
     "`!compress*`" if you also want to allow the `--compress-level` option.
 
-    Note that the "write-devices" option is refused by default, but can be
-    explicitly accepted with "`!write-devices`".  The options "log-file" and
-    "log-file-format" are forcibly refused and cannot be accepted.
+    Note that the "copy-devices" & "write-devices" options are refused by
+    default, but they can be explicitly accepted with "`!copy-devices`" and/or
+    "`!write-devices`".  The options "log-file" and "log-file-format" are
+    forcibly refused and cannot be accepted.
 
     Here are all the options that are not matched by wild-cards:
 
@@ -1211,7 +1211,7 @@ The /etc/rsyncd.secrets file would look something like this:
 
 ## SEE ALSO
 
-[**rsync**(1)](./rsync.1), [**rsync-ssl**(1)](./rsync-ssl.1)
+[**rsync**(1)](rsync.1), [**rsync-ssl**(1)](rsync-ssl.1)
 
 ## BUGS
 
@@ -1220,12 +1220,12 @@ Please report bugs! The rsync bug tracking system is online at
 
 ## VERSION
 
-This man page is current for version @VERSION@ of rsync.
+This manpage is current for version @VERSION@ of rsync.
 
 ## CREDITS
 
-rsync is distributed under the GNU General Public License.  See the file
-[COPYING](./COPYING) for details.
+Rsync is distributed under the GNU General Public License.  See the file
+[COPYING](COPYING) for details.
 
 An rsync web site is available at <https://rsync.samba.org/> and its github
 project is <https://github.com/WayneD/rsync>.
@@ -1237,7 +1237,7 @@ Thanks to Karsten Thygesen for his many suggestions and documentation!
 
 ## AUTHOR
 
-rsync was written by Andrew Tridgell and Paul Mackerras.  Many people have
+Rsync was written by Andrew Tridgell and Paul Mackerras.  Many people have
 later contributed to it.
 
 Mailing lists for support and development are available at

sender.c

@@ -37,6 +37,7 @@ extern int io_error;
 extern int flist_eof;
 extern int whole_file;
 extern int allowed_lull;
+extern int copy_devices;
 extern int preserve_xattrs;
 extern int protocol_version;
 extern int remove_source_files;
@@ -366,6 +367,15 @@ void send_files(int f_in, int f_out)
 			exit_cleanup(RERR_FILEIO);
 		}
 
+		if (IS_DEVICE(st.st_mode)) {
+			if (!copy_devices) {
+				rprintf(FERROR, "attempt to copy device contents without --copy-devices\n");
+				exit_cleanup(RERR_PROTOCOL);
+			}
+			if (st.st_size == 0)
+				st.st_size = get_device_size(fd, fname);
+		}
+
 		if (append_mode > 0 && st.st_size < F_LENGTH(file)) {
 			rprintf(FWARNING, "skipped diminished file: %s\n",
 				full_fname(fname));

simd-checksum-avx2.S

@@ -1,15 +1,21 @@
+#include "config.h"
+
+#ifdef USE_ROLL_ASM /* { */
+
+#define CHAR_OFFSET 0 /* Keep this the same as rsync.h, which isn't likely to change. */
+
 #ifdef __APPLE__
-#define get_checksum1_avx2  _get_checksum1_avx2
+#define get_checksum1_avx2_asm  _get_checksum1_avx2_asm
 #endif
 
 .intel_syntax noprefix
 .text
 
 	.p2align 5
-	.globl get_checksum1_avx2
+	.globl get_checksum1_avx2_asm
 
 # rdi=*buf, esi=len, edx=i, rcx= *ps1, r8= *ps2
-get_checksum1_avx2:
+get_checksum1_avx2_asm:
 	vmovd	xmm6,[rcx] # load *ps1
 	lea	eax, [rsi-128] # at least 128 bytes to process?
 	cmp	edx, eax
@@ -167,3 +173,5 @@ get_checksum1_avx2:
 	.byte 3
 	.byte 2
 	.byte 1
+
+#endif /* } USE_ROLL_ASM */

simd-checksum-x86_64.cpp

@@ -51,12 +51,12 @@
  * GCC 4.x are not supported to ease configure.ac logic.
  */
 
-#ifdef __x86_64__
-#ifdef __cplusplus
+#ifdef __x86_64__ /* { */
+#ifdef __cplusplus /* { */
 
 #include "rsync.h"
 
-#ifdef HAVE_SIMD
+#ifdef USE_ROLL_SIMD /* { */
 
 #include <immintrin.h>
 
@@ -85,6 +85,9 @@ typedef long long __m256i_u __attribute__((__vector_size__(32), __may_alias__, _
 #define SSE2_HADDS_EPI16(a, b) _mm_adds_epi16(SSE2_INTERLEAVE_EVEN_EPI16(a, b), SSE2_INTERLEAVE_ODD_EPI16(a, b))
 #define SSE2_MADDUBS_EPI16(a, b) _mm_adds_epi16(SSE2_MULU_EVEN_EPI8(a, b), SSE2_MULU_ODD_EPI8(a, b))
 
+#ifndef USE_ROLL_ASM
+__attribute__ ((target("default"))) MVSTATIC int32 get_checksum1_avx2_64(schar* buf, int32 len, int32 i, uint32* ps1, uint32* ps2) { return i; }
+#endif
 __attribute__ ((target("default"))) MVSTATIC int32 get_checksum1_ssse3_32(schar* buf, int32 len, int32 i, uint32* ps1, uint32* ps2) { return i; }
 __attribute__ ((target("default"))) MVSTATIC int32 get_checksum1_sse2_32(schar* buf, int32 len, int32 i, uint32* ps1, uint32* ps2) { return i; }
 
@@ -245,7 +248,7 @@ __attribute__ ((target("sse2"))) MVSTATIC int32 get_checksum1_sse2_32(schar* buf
 
             // (4*buf[i] + 3*buf[i+1]), (2*buf[i+2], buf[i+3]), ... 2*[int16*8]
             __m128i mul_const = _mm_set1_epi32(4 + (3 << 8) + (2 << 16) + (1 << 24));
-            __m128i mul_add16_1 = SSE2_MADDUBS_EPI16(mul_const, in8_1);
+            __m128i mul_add16_1 = SSE2_MADDUBS_EPI16(mul_const, in8_1);
             __m128i mul_add16_2 = SSE2_MADDUBS_EPI16(mul_const, in8_2);
 
             // s2 += 32*s1
@@ -310,7 +313,127 @@ __attribute__ ((target("sse2"))) MVSTATIC int32 get_checksum1_sse2_32(schar* buf
     return i;
 }
 
-extern "C" __attribute__ ((target("avx2"))) int32 get_checksum1_avx2(schar* buf, int32 len, int32 i, uint32* ps1, uint32* ps2);
+#ifdef USE_ROLL_ASM /* { */
+
+extern "C" __attribute__ ((target("avx2"))) int32 get_checksum1_avx2_asm(schar* buf, int32 len, int32 i, uint32* ps1, uint32* ps2);
+
+#else /* } { */
+
+/*
+  AVX2 loop per 64 bytes:
+    int16 t1[16];
+    int16 t2[16];
+    for (int j = 0; j < 16; j++) {
+      t1[j] = buf[j*4 + i] + buf[j*4 + i+1] + buf[j*4 + i+2] + buf[j*4 + i+3];
+      t2[j] = 4*buf[j*4 + i] + 3*buf[j*4 + i+1] + 2*buf[j*4 + i+2] + buf[j*4 + i+3];
+    }
+    s2 += 64*s1 + (uint32)(
+              60*t1[0] + 56*t1[1] + 52*t1[2] + 48*t1[3] + 44*t1[4] + 40*t1[5] + 36*t1[6] + 32*t1[7] + 28*t1[8] + 24*t1[9] + 20*t1[10] + 16*t1[11] + 12*t1[12] + 8*t1[13] + 4*t1[14] +
+              t2[0] + t2[1] + t2[2] + t2[3] + t2[4] + t2[5] + t2[6] + t2[7] + t2[8] + t2[9] + t2[10] + t2[11] + t2[12] + t2[13] + t2[14] + t2[15]
+          ) + 2080*CHAR_OFFSET;
+    s1 += (uint32)(t1[0] + t1[1] + t1[2] + t1[3] + t1[4] + t1[5] + t1[6] + t1[7] + t1[8] + t1[9] + t1[10] + t1[11] + t1[12] + t1[13] + t1[14] + t1[15]) +
+          64*CHAR_OFFSET;
+ */
+
+__attribute__ ((target("avx2"))) MVSTATIC int32 get_checksum1_avx2_64(schar* buf, int32 len, int32 i, uint32* ps1, uint32* ps2)
+{
+    if (len > 64) {
+
+        uint32 x[4] = {0};
+        __m128i ss1 = _mm_cvtsi32_si128(*ps1);
+        __m128i ss2 = _mm_cvtsi32_si128(*ps2);
+
+        const char mul_t1_buf[16] = {60, 56, 52, 48, 44, 40, 36, 32, 28, 24, 20, 16, 12, 8, 4, 0};
+	__m128i tmp = _mm_load_si128((__m128i*) mul_t1_buf);
+        __m256i mul_t1 = _mm256_cvtepu8_epi16(tmp);
+	__m256i mul_const = _mm256_broadcastd_epi32(_mm_cvtsi32_si128(4 | (3 << 8) | (2 << 16) | (1 << 24)));
+        __m256i mul_one;
+       	    mul_one = _mm256_abs_epi8(_mm256_cmpeq_epi16(mul_one,mul_one)); // set all vector elements to 1
+
+        for (; i < (len-64); i+=64) {
+            // Load ... 4*[int8*16]
+            __m256i in8_1, in8_2;
+	    __m128i in8_1_low, in8_2_low, in8_1_high, in8_2_high;
+	    in8_1_low = _mm_loadu_si128((__m128i_u*)&buf[i]);
+	    in8_2_low = _mm_loadu_si128((__m128i_u*)&buf[i+16]);
+	    in8_1_high = _mm_loadu_si128((__m128i_u*)&buf[i+32]);
+	    in8_2_high = _mm_loadu_si128((__m128i_u*)&buf[i+48]);
+	    in8_1 = _mm256_inserti128_si256(_mm256_castsi128_si256(in8_1_low), in8_1_high,1);
+	    in8_2 = _mm256_inserti128_si256(_mm256_castsi128_si256(in8_2_low), in8_2_high,1);
+
+            // (1*buf[i] + 1*buf[i+1]), (1*buf[i+2], 1*buf[i+3]), ... 2*[int16*8]
+            // Fastest, even though multiply by 1
+            __m256i add16_1 = _mm256_maddubs_epi16(mul_one, in8_1);
+            __m256i add16_2 = _mm256_maddubs_epi16(mul_one, in8_2);
+
+            // (4*buf[i] + 3*buf[i+1]), (2*buf[i+2], buf[i+3]), ... 2*[int16*8]
+            __m256i mul_add16_1 = _mm256_maddubs_epi16(mul_const, in8_1);
+            __m256i mul_add16_2 = _mm256_maddubs_epi16(mul_const, in8_2);
+
+            // s2 += 64*s1
+            ss2 = _mm_add_epi32(ss2, _mm_slli_epi32(ss1, 6));
+
+            // [sum(t1[0]..t1[7]), X, X, X] [int32*4]; faster than multiple _mm_hadds_epi16
+            __m256i sum_add32 = _mm256_add_epi16(add16_1, add16_2);
+            sum_add32 = _mm256_add_epi16(sum_add32, _mm256_srli_epi32(sum_add32, 16));
+            sum_add32 = _mm256_add_epi16(sum_add32, _mm256_srli_si256(sum_add32, 4));
+            sum_add32 = _mm256_add_epi16(sum_add32, _mm256_srli_si256(sum_add32, 8));
+
+            // [sum(t2[0]..t2[7]), X, X, X] [int32*4]; faster than multiple _mm_hadds_epi16
+            __m256i sum_mul_add32 = _mm256_add_epi16(mul_add16_1, mul_add16_2);
+            sum_mul_add32 = _mm256_add_epi16(sum_mul_add32, _mm256_srli_epi32(sum_mul_add32, 16));
+            sum_mul_add32 = _mm256_add_epi16(sum_mul_add32, _mm256_srli_si256(sum_mul_add32, 4));
+            sum_mul_add32 = _mm256_add_epi16(sum_mul_add32, _mm256_srli_si256(sum_mul_add32, 8));
+
+            // s1 += t1[0] + t1[1] + t1[2] + t1[3] + t1[4] + t1[5] + t1[6] + t1[7]
+	    __m128i sum_add32_hi = _mm256_extracti128_si256(sum_add32, 0x1);
+            ss1 = _mm_add_epi32(ss1, _mm256_castsi256_si128(sum_add32));
+            ss1 = _mm_add_epi32(ss1, sum_add32_hi);
+
+            // s2 += t2[0] + t2[1] + t2[2] + t2[3] + t2[4] + t2[5] + t2[6] + t2[7]
+	    __m128i sum_mul_add32_hi = _mm256_extracti128_si256(sum_mul_add32, 0x1);
+            ss2 = _mm_add_epi32(ss2, _mm256_castsi256_si128(sum_mul_add32));
+            ss2 = _mm_add_epi32(ss2, sum_mul_add32_hi);
+
+            // [t1[0] + t1[1], t1[2] + t1[3] ...] [int16*8]
+            // We could've combined this with generating sum_add32 above and
+            // save an instruction but benchmarking shows that as being slower
+            __m256i add16 = _mm256_hadds_epi16(add16_1, add16_2);
+
+            // [t1[0], t1[1], ...] -> [t1[0]*28 + t1[1]*24, ...] [int32*4]
+            __m256i mul32 = _mm256_madd_epi16(add16, mul_t1);
+
+            // [sum(mul32), X, X, X] [int32*4]; faster than multiple _mm_hadd_epi32
+            mul32 = _mm256_add_epi32(mul32, _mm256_srli_si256(mul32, 4));
+            mul32 = _mm256_add_epi32(mul32, _mm256_srli_si256(mul32, 8));
+	    // prefetch 2 cacheline ahead
+            _mm_prefetch(&buf[i + 160], _MM_HINT_T0);
+
+            // s2 += 28*t1[0] + 24*t1[1] + 20*t1[2] + 16*t1[3] + 12*t1[4] + 8*t1[5] + 4*t1[6]
+	    __m128i mul32_hi = _mm256_extracti128_si256(mul32, 0x1);
+            ss2 = _mm_add_epi32(ss2, _mm256_castsi256_si128(mul32));
+            ss2 = _mm_add_epi32(ss2, mul32_hi);
+
+#if CHAR_OFFSET != 0
+            // s1 += 32*CHAR_OFFSET
+            __m128i char_offset_multiplier = _mm_set1_epi32(32 * CHAR_OFFSET);
+            ss1 = _mm_add_epi32(ss1, char_offset_multiplier);
+
+            // s2 += 528*CHAR_OFFSET
+            char_offset_multiplier = _mm_set1_epi32(528 * CHAR_OFFSET);
+            ss2 = _mm_add_epi32(ss2, char_offset_multiplier);
+#endif
+        }
+
+        _mm_store_si128((__m128i_u*)x, ss1);
+        *ps1 = x[0];
+        _mm_store_si128((__m128i_u*)x, ss2);
+        *ps2 = x[0];
+    }
+    return i;
+}
+
+#endif /* } !USE_ROLL_ASM */
 
 static int32 get_checksum1_default_1(schar* buf, int32 len, int32 i, uint32* ps1, uint32* ps2)
 {
@@ -338,7 +461,11 @@ static inline uint32 get_checksum1_cpp(char *buf1, int32 len)
     uint32 s2 = 0;
 
     // multiples of 64 bytes using AVX2 (if available)
-    i = get_checksum1_avx2((schar*)buf1, len, i, &s1, &s2);
+#ifdef USE_ROLL_ASM
+    i = get_checksum1_avx2_asm((schar*)buf1, len, i, &s1, &s2);
+#else
+    i = get_checksum1_avx2_64((schar*)buf1, len, i, &s1, &s2);
+#endif
 
     // multiples of 32 bytes using SSSE3 (if available)
     i = get_checksum1_ssse3_32((schar*)buf1, len, i, &s1, &s2);
@@ -407,7 +534,11 @@ int main() {
     benchmark("Raw-C", get_checksum1_default_1, (schar*)buf, BLOCK_LEN);
     benchmark("SSE2", get_checksum1_sse2_32, (schar*)buf, BLOCK_LEN);
     benchmark("SSSE3", get_checksum1_ssse3_32, (schar*)buf, BLOCK_LEN);
-    benchmark("AVX2", get_checksum1_avx2, (schar*)buf, BLOCK_LEN);
+#ifdef USE_ROLL_ASM
+    benchmark("AVX2-ASM", get_checksum1_avx2_asm, (schar*)buf, BLOCK_LEN);
+#else
+    benchmark("AVX2", get_checksum1_avx2_64, (schar*)buf, BLOCK_LEN);
+#endif
 
     free(buf);
     return 0;
@@ -417,6 +548,6 @@ int main() {
 #pragma clang optimize on
 #endif /* BENCHMARK_SIMD_CHECKSUM1 */
 
-#endif /* HAVE_SIMD */
-#endif /* __cplusplus */
-#endif /* __x86_64__ */
+#endif /* } USE_ROLL_SIMD */
+#endif /* } __cplusplus */
+#endif /* } __x86_64__ */

support/git-set-file-times

@@ -38,7 +38,7 @@ def main():
                 print_line(fn, mtime, mtime)
             ls.discard(fn)
 
-    cmd = git + 'log -r --name-only --no-color --pretty=raw --no-renames -z'.split()
+    cmd = git + 'log -r --name-only --format=%x00commit%x20%H%n%x00commit_time%x20%ct%n --no-renames -z'.split()
     if args.tree:
         cmd.append(args.tree)
     cmd += ['--'] + args.files
@@ -46,7 +46,7 @@ def main():
     proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, encoding='utf-8')
     for line in proc.stdout:
         line = line.strip()
-        m = re.match(r'^committer .*? (\d+) [-+]\d+$', line)
+        m = re.match(r'^\0commit_time (\d+)$', line)
         if m:
             commit_time = int(m[1])
         elif NULL_COMMIT_RE.search(line):

support/rrsync.1.md

@@ -8,6 +8,9 @@ rrsync - a script to setup restricted rsync users via ssh logins
 rrsync [-ro|-rw] [-munge] [-no-del] [-no-lock] DIR
 ```
 
+The single non-option argument specifies the restricted _DIR_ to use. It can be
+relative to the user's home directory or an absolute path.
+
 The online version of this man page (that includes cross-linking of topics)
 is available at <https://download.samba.org/pub/rsync/rrsync.1>.
 
@@ -30,7 +33,7 @@ ssh-key line that should be restricted:
 Then, ensure that the rrsync script has your desired option restrictions. You
 may want to copy the script to a local bin dir with a unique name if you want
 to have multiple configurations. One or more rrsync options can be specified
-prior to the `DIR` if you want to further restrict the transfer.
+prior to the _DIR_ if you want to further restrict the transfer.
 
 To use an rsync daemon setup, edit the user's `~/.ssh/authorized_keys` file and
 add a prefix like one of the following (followed by a space) in front of each
@@ -43,9 +46,8 @@ ssh-key line that should be restricted:
 
 Then, ensure that the rsyncd.conf file is created with one or more module names
 with the appropriate path and option restrictions.  If rsync's
-[`--config`](./rsync.1#daemon-opt--config) option is omitted, it defaults to
-`~/rsyncd.conf`.  See the `rsyncd.conf` man page for details of how to
-configure an rsync daemon.
+[`--config`](rsync.1#dopt) option is omitted, it defaults to `~/rsyncd.conf`.
+See the `rsyncd.conf` man page for details of how to configure an rsync daemon.
 
 When using rrsync, there can be just one restricted dir per authorized key.  A
 daemon setup, on the other hand, allows multiple module names inside the config
@@ -53,33 +55,46 @@ file, each one with its own path setting.
 
 The remainder of this man page is dedicated to using the rrsync script.
 
-## OPTION SUMMARY
+## OPTIONS
 
-```
--ro        Allow only reading from the DIR. Implies -no-del and -no-lock.
--wo        Allow only writing to the DIR.
--munge     Enable rsync's --munge-links on the server side.
--no-del    Disable rsync's --delete* and --remove* options.
--no-lock   Avoid the single-run (per-user) lock check.
--help, -h  Output this help message and exit.
-```
+0.  `-ro`
 
-A single non-option argument specifies the restricted DIR to use. It can be
-relative to the user's home directory or an absolute path.
+    Allow only reading from the DIR. Implies [`-no-del`](#opt) and
+    [`-no-lock`](#opt).
+
+0.  `-wo`
+
+    Allow only writing to the DIR.
+
+0.  `-munge`
+
+    Enable rsync's [`--munge-links`](rsync.1#opt) on the server side.
+
+0.  `-no-del`
+
+    Disable rsync's `--delete*` and `--remove*` options.
+
+0.  `-no-lock`
+
+    Avoid the single-run (per-user) lock check.  Useful with [`-munge`](#opt).
+
+0.  `-help`, `-h`
+
+    Output this help message and exit.
 
 ## SECURITY RESTRICTIONS
 
 The rrsync script validates the path arguments it is sent to try to restrict
 them to staying within the specified DIR.
 
-The rrsync script rejects rsync's [`--copy-links`](./rsync.1#opt--copy-links)
-option (by default) so that a copy cannot dereference a symlink within the DIR
-to get to a file outside the DIR.
+The rrsync script rejects rsync's [`--copy-links`](rsync.1#opt) option (by
+default) so that a copy cannot dereference a symlink within the DIR to get to a
+file outside the DIR.
 
-The rrsync script rejects rsync's
-[`--protect-args`](./rsync.1#opt--protect-args) (`-s`) option because it would
-allow options to be sent to the server-side that the script cannot check.  If
-you want to support `--protect-args`, use a daemon-over-ssh setup.
+The rrsync script rejects rsync's [`--protect-args`](rsync.1#opt) (`-s`) option
+because it would allow options to be sent to the server-side that the script
+cannot check.  If you want to support `--protect-args`, use a daemon-over-ssh
+setup.
 
 The rrsync script accepts just a subset of rsync's options that the real rsync
 uses when running the server command.  A few extra convenience options are also
@@ -104,7 +119,7 @@ The `~/.ssh/authorized_keys` file might have lines in it like this:
 
 ## SEE ALSO
 
-[**rsync**(1)](./rsync.1)
+[**rsync**(1)](rsync.1)
 
 ## VERSION
 
@@ -113,7 +128,7 @@ This man page is current for version @VERSION@ of rsync.
 ## CREDITS
 
 rsync is distributed under the GNU General Public License.  See the file
-[COPYING](./COPYING) for details.
+[COPYING](COPYING) for details.
 
 An rsync web site is available at <https://rsync.samba.org/> and its github
 project is <https://github.com/WayneD/rsync>.

syscall.c

@@ -247,7 +247,7 @@ int do_chmod(const char *path, mode_t mode)
 		else if (errno != ENOTSUP)
 			break;
 #endif
-
+		/* FALLTHROUGH */
 	default:
 		if (S_ISLNK(mode)) {
 # if defined HAVE_SETATTRLIST
@@ -257,7 +257,10 @@ int do_chmod(const char *path, mode_t mode)
 			memset(&attrList, 0, sizeof attrList);
 			attrList.bitmapcount = ATTR_BIT_MAP_COUNT;
 			attrList.commonattr = ATTR_CMN_ACCESSMASK;
-			code = setattrlist(path, &attrList, &m, sizeof m, FSOPT_NOFOLLOW);
+			if ((code = setattrlist(path, &attrList, &m, sizeof m, FSOPT_NOFOLLOW)) == 0)
+				break;
+			if (errno == ENOTSUP)
+				code = 1;
 # else
 			code = 1;
 # endif

testsuite/00-hello.test

@@ -1,9 +1,11 @@
 #!/bin/sh
 
-test_fail() {
-    echo "$@" >&2
-    exit 1
-}
+# Test some foundational things.
+
+. "$suitedir/rsync.fns"
+
+RSYNC_RSH="$scratchdir/src/support/lsh.sh"
+export RSYNC_RSH
 
 echo $0 running
 
@@ -12,3 +14,48 @@ $RSYNC --version || test_fail '--version output failed'
 $RSYNC --info=help || test_fail '--info=help output failed'
 
 $RSYNC --debug=help || test_fail '--debug=help output failed'
+
+weird_name="A weird)name"
+
+mkdir "$fromdir"
+mkdir "$fromdir/$weird_name"
+
+append_line() {
+    echo "$1"
+    echo "$1" >>"$fromdir/$weird_name/file"
+}
+
+append_line test1
+checkit "$RSYNC -ai '$fromdir/' '$todir/'" "$fromdir" "$todir"
+
+copy_weird() {
+    checkit "$RSYNC $1 --rsync-path='$RSYNC' '$2$fromdir/$weird_name/' '$3$todir/$weird_name'" "$fromdir" "$todir"
+}
+
+append_line test2
+copy_weird '-ai' 'lh:' ''
+
+append_line test3
+copy_weird '-ai' '' 'lh:'
+
+append_line test4
+copy_weird '-ais' 'lh:' ''
+
+append_line test5
+copy_weird '-ais' '' 'lh:'
+
+echo test6
+
+touch "$fromdir/one" "$fromdir/two"
+(cd "$fromdir" && $RSYNC -ai --old-args --rsync-path="$RSYNC" lh:'one two' "$todir/")
+if [ ! -f "$todir/one" ] || [ ! -f "$todir/two" ]; then
+    test_fail "old-args copy of 'one two' failed"
+fi
+
+echo test7
+
+rm "$todir/one" "$todir/two"
+(cd "$fromdir" && RSYNC_OLD_ARGS=1 $RSYNC -ai --rsync-path="$RSYNC" lh:'one two' "$todir/")
+
+# The script would have aborted on error, so getting here means we've won.
+exit 0

testsuite/alt-dest.test

@@ -0,0 +1,68 @@
+#!/bin/sh
+
+# Copyright (C) 2004-2022 Wayne Davison
+
+# This program is distributable under the terms of the GNU GPL (see
+# COPYING).
+
+# Test rsync handling of --compare-dest and similar options.
+
+. "$suitedir/rsync.fns"
+
+alt1dir="$tmpdir/alt1"
+alt2dir="$tmpdir/alt2"
+alt3dir="$tmpdir/alt3"
+
+SSH="$scratchdir/src/support/lsh.sh"
+
+# Build some files/dirs/links to copy
+
+hands_setup
+
+# Setup the alt and chk dirs
+$RSYNC -av --include=text --include='*/' --exclude='*' "$fromdir/" "$alt1dir/"
+$RSYNC -av --include=etc-ltr-list --include='*/' --exclude='*' "$fromdir/" "$alt2dir/"
+
+# Create a side dir where there is a candidate destfile of the same name as a sourcefile
+echo "This is a test file" >"$fromdir/likely"
+
+mkdir "$alt3dir"
+echo "This is a test file" >"$alt3dir/likely"
+
+sleep 1
+touch "$fromdir/dir/text" "$fromdir/likely"
+
+$RSYNC -av --exclude=/text --exclude=etc-ltr-list "$fromdir/" "$chkdir/"
+
+# Let's do it!
+checkit "$RSYNC -avv --no-whole-file \
+    --compare-dest='$alt1dir' --compare-dest='$alt2dir' \
+    '$fromdir/' '$todir/'" "$chkdir" "$todir"
+
+rm -rf "$todir"
+checkit "$RSYNC -avv --no-whole-file \
+    --copy-dest='$alt1dir' --copy-dest='$alt2dir' \
+    '$fromdir/' '$todir/'" "$fromdir" "$todir"
+
+# Test that copy_file() works correctly with tmpfiles
+for maybe_inplace in '' --inplace; do
+    rm -rf "$todir"
+    checkit "$RSYNC -av $maybe_inplace --copy-dest='$alt3dir' \
+	'$fromdir/' '$todir/'" "$fromdir" "$todir"
+
+    for srchost in '' 'localhost:'; do
+	if [ -z "$srchost" ]; then
+	    desthost='localhost:'
+	else
+	    desthost=''
+	fi
+
+	rm -rf "$todir"
+	checkit "$RSYNC -ave '$SSH' --rsync-path='$RSYNC' $maybe_inplace \
+	    --copy-dest='$alt3dir' '$srchost$fromdir/' '$desthost$todir/'" \
+	    "$fromdir" "$todir"
+    done
+done
+
+# The script would have aborted on error, so getting here means we've won.
+exit 0

testsuite/compare-dest.test

@@ -1,37 +0,0 @@
-#!/bin/sh
-
-# Copyright (C) 2004-2022 Wayne Davison
-
-# This program is distributable under the terms of the GNU GPL (see
-# COPYING).
-
-# Test rsync handling of the --compare-dest option.
-
-. "$suitedir/rsync.fns"
-
-alt1dir="$tmpdir/alt1"
-alt2dir="$tmpdir/alt2"
-
-# Build some files/dirs/links to copy
-
-hands_setup
-
-# Setup the alt and chk dirs
-$RSYNC -av --include=text --include='*/' --exclude='*' "$fromdir/" "$alt1dir/"
-$RSYNC -av --include=etc-ltr-list --include='*/' --exclude='*' "$fromdir/" "$alt2dir/"
-
-sleep 1
-touch "$fromdir/dir/text"
-
-$RSYNC -av --exclude=/text --exclude=etc-ltr-list "$fromdir/" "$chkdir/"
-
-# Let's do it!
-checkit "$RSYNC -avv --no-whole-file \
-    --compare-dest='$alt1dir' --compare-dest='$alt2dir' \
-    '$fromdir/' '$todir/'" "$chkdir" "$todir"
-checkit "$RSYNC -avv --no-whole-file \
-    --copy-dest='$alt1dir' --copy-dest='$alt2dir' \
-    '$fromdir/' '$todir/'" "$fromdir" "$todir"
-
-# The script would have aborted on error, so getting here means we've won.
-exit 0

testsuite/crtimes.test

@@ -10,7 +10,7 @@ $RSYNC --version | grep "[, ] crtimes" >/dev/null || test_skipped "Rsync is conf
 # Setting it to a newer time affects just the mtime.
 
 mkdir "$fromdir"
-echo hiho "$fromdir/foo"
+echo hiho >"$fromdir/foo"
 
 touch -t 200101011111.11 "$fromdir"
 touch -t 200202022222.22 "$fromdir"

usage.c

@@ -1,7 +1,7 @@
 /*
  * Some usage & version related functions.
  *
- * Copyright (C) 2002-2020 Wayne Davison
+ * Copyright (C) 2002-2022 Wayne Davison
  *
  * This program is free software; you can redistribute it and/or modify
  * it under the terms of the GNU General Public License as published by
@@ -139,21 +139,26 @@ static void print_info_flags(enum logcode f)
 
 	"*Optimizations",
 
-#ifndef HAVE_SIMD
+#ifndef USE_ROLL_SIMD
 		"no "
 #endif
-			"SIMD",
+			"SIMD-roll",
 
-#ifndef HAVE_ASM
+#ifndef USE_ROLL_ASM
 		"no "
 #endif
-			"asm",
+			"asm-roll",
 
 #ifndef USE_OPENSSL
 		"no "
 #endif
 			"openssl-crypto",
 
+#ifndef USE_MD5_ASM
+		"no "
+#endif
+			"asm-MD5",
+
 		NULL
 	};
 
@@ -240,7 +245,7 @@ void usage(enum logcode F)
 #include "help-rsync.h"
   rprintf(F,"\n");
   rprintf(F,"Use \"rsync --daemon --help\" to see the daemon-mode command-line options.\n");
-  rprintf(F,"Please see the rsync(1) and rsyncd.conf(5) man pages for full documentation.\n");
+  rprintf(F,"Please see the rsync(1) and rsyncd.conf(5) manpages for full documentation.\n");
   rprintf(F,"See https://rsync.samba.org/ for updates, bug reports, and answers\n");
 }
 
@@ -253,7 +258,7 @@ void daemon_usage(enum logcode F)
 #include "help-rsyncd.h"
   rprintf(F,"\n");
   rprintf(F,"If you were not trying to invoke rsync as a daemon, avoid using any of the\n");
-  rprintf(F,"daemon-specific rsync options.  See also the rsyncd.conf(5) man page.\n");
+  rprintf(F,"daemon-specific rsync options.  See also the rsyncd.conf(5) manpage.\n");
 }
 
 const char *rsync_version(void)

util1.c

@@ -320,16 +320,48 @@ static int safe_read(int desc, char *ptr, size_t len)
 	return n_chars;
 }
 
-/* Copy a file.  If ofd < 0, copy_file unlinks and opens the "dest" file.
- * Otherwise, it just writes to and closes the provided file descriptor.
+/* Remove existing file @dest and reopen, creating a new file with @mode */
+static int unlink_and_reopen(const char *dest, mode_t mode)
+{
+	int ofd;
+
+	if (robust_unlink(dest) && errno != ENOENT) {
+		int save_errno = errno;
+		rsyserr(FERROR_XFER, errno, "unlink %s", full_fname(dest));
+		errno = save_errno;
+		return -1;
+	}
+
+#ifdef SUPPORT_XATTRS
+	if (preserve_xattrs)
+		mode |= S_IWUSR;
+#endif
+	mode &= INITACCESSPERMS;
+	if ((ofd = do_open(dest, O_WRONLY | O_CREAT | O_TRUNC | O_EXCL, mode)) < 0) {
+		int save_errno = errno;
+		rsyserr(FERROR_XFER, save_errno, "open %s", full_fname(dest));
+		errno = save_errno;
+		return -1;
+	}
+	return ofd;
+}
+
+/* Copy contents of file @source to file @dest with mode @mode.
+ *
+ * If @tmpfilefd is < 0, copy_file unlinks @dest and then opens a new
+ * file with name @dest.
+ *
+ * Otherwise, copy_file writes to and closes the provided file
+ * descriptor.
+ *
  * In either case, if --xattrs are being preserved, the dest file will
  * have its xattrs set from the source file.
  *
  * This is used in conjunction with the --temp-dir, --backup, and
  * --copy-dest options. */
-int copy_file(const char *source, const char *dest, int ofd, mode_t mode)
+int copy_file(const char *source, const char *dest, int tmpfilefd, mode_t mode)
 {
-	int ifd;
+	int ifd, ofd;
 	char buf[1024 * 8];
 	int len;   /* Number of bytes read into `buf'. */
 	OFF_T prealloc_len = 0, offset = 0;
@@ -341,23 +373,12 @@ int copy_file(const char *source, const char *dest, int ofd, mode_t mode)
 		return -1;
 	}
 
-	if (ofd < 0) {
-		if (robust_unlink(dest) && errno != ENOENT) {
+	if (tmpfilefd >= 0) {
+		ofd = tmpfilefd;
+	} else {
+		ofd = unlink_and_reopen(dest, mode);
+		if (ofd < 0) {
 			int save_errno = errno;
-			rsyserr(FERROR_XFER, errno, "unlink %s", full_fname(dest));
-			close(ifd);
-			errno = save_errno;
-			return -1;
-		}
-
-#ifdef SUPPORT_XATTRS
-		if (preserve_xattrs)
-			mode |= S_IWUSR;
-#endif
-		mode &= INITACCESSPERMS;
-		if ((ofd = do_open(dest, O_WRONLY | O_CREAT | O_TRUNC | O_EXCL, mode)) < 0) {
-			int save_errno = errno;
-			rsyserr(FERROR_XFER, save_errno, "open %s", full_fname(dest));
 			close(ifd);
 			errno = save_errno;
 			return -1;

version.h

@@ -1 +1 @@
-#define RSYNC_VERSION "3.2.4pre2"
+#define RSYNC_VERSION "3.2.4"

zlib/deflate.c

@@ -227,11 +227,6 @@ int ZEXPORT deflateInit2_(strm, level, method, windowBits, memLevel, strategy,
     int wrap = 1;
     static const char my_version[] = ZLIB_VERSION;
 
-    ushf *overlay;
-    /* We overlay pending_buf and d_buf+l_buf. This works since the average
-     * output size for (length,distance) codes is <= 24 bits.
-     */
-
     if (version == Z_NULL || version[0] != my_version[0] ||
         stream_size != sizeof(z_stream)) {
         return Z_VERSION_ERROR;
@@ -300,9 +295,47 @@ int ZEXPORT deflateInit2_(strm, level, method, windowBits, memLevel, strategy,
 
     s->lit_bufsize = 1 << (memLevel + 6); /* 16K elements by default */
 
-    overlay = (ushf *) ZALLOC(strm, s->lit_bufsize, sizeof(ush)+2);
-    s->pending_buf = (uchf *) overlay;
-    s->pending_buf_size = (ulg)s->lit_bufsize * (sizeof(ush)+2L);
+    /* We overlay pending_buf and sym_buf. This works since the average size
+     * for length/distance pairs over any compressed block is assured to be 31
+     * bits or less.
+     *
+     * Analysis: The longest fixed codes are a length code of 8 bits plus 5
+     * extra bits, for lengths 131 to 257. The longest fixed distance codes are
+     * 5 bits plus 13 extra bits, for distances 16385 to 32768. The longest
+     * possible fixed-codes length/distance pair is then 31 bits total.
+     *
+     * sym_buf starts one-fourth of the way into pending_buf. So there are
+     * three bytes in sym_buf for every four bytes in pending_buf. Each symbol
+     * in sym_buf is three bytes -- two for the distance and one for the
+     * literal/length. As each symbol is consumed, the pointer to the next
+     * sym_buf value to read moves forward three bytes. From that symbol, up to
+     * 31 bits are written to pending_buf. The closest the written pending_buf
+     * bits gets to the next sym_buf symbol to read is just before the last
+     * code is written. At that time, 31*(n-2) bits have been written, just
+     * after 24*(n-2) bits have been consumed from sym_buf. sym_buf starts at
+     * 8*n bits into pending_buf. (Note that the symbol buffer fills when n-1
+     * symbols are written.) The closest the writing gets to what is unread is
+     * then n+14 bits. Here n is lit_bufsize, which is 16384 by default, and
+     * can range from 128 to 32768.
+     *
+     * Therefore, at a minimum, there are 142 bits of space between what is
+     * written and what is read in the overlain buffers, so the symbols cannot
+     * be overwritten by the compressed data. That space is actually 139 bits,
+     * due to the three-bit fixed-code block header.
+     *
+     * That covers the case where either Z_FIXED is specified, forcing fixed
+     * codes, or when the use of fixed codes is chosen, because that choice
+     * results in a smaller compressed block than dynamic codes. That latter
+     * condition then assures that the above analysis also covers all dynamic
+     * blocks. A dynamic-code block will only be chosen to be emitted if it has
+     * fewer bits than a fixed-code block would for the same set of symbols.
+     * Therefore its average symbol length is assured to be less than 31. So
+     * the compressed data for a dynamic block also cannot overwrite the
+     * symbols from which it is being constructed.
+     */
+
+    s->pending_buf = (uchf *) ZALLOC(strm, s->lit_bufsize, 4);
+    s->pending_buf_size = (ulg)s->lit_bufsize * 4;
 
     if (s->window == Z_NULL || s->prev == Z_NULL || s->head == Z_NULL ||
         s->pending_buf == Z_NULL) {
@@ -311,8 +344,12 @@ int ZEXPORT deflateInit2_(strm, level, method, windowBits, memLevel, strategy,
         deflateEnd (strm);
         return Z_MEM_ERROR;
     }
-    s->d_buf = overlay + s->lit_bufsize/sizeof(ush);
-    s->l_buf = s->pending_buf + (1+sizeof(ush))*s->lit_bufsize;
+    s->sym_buf = s->pending_buf + s->lit_bufsize;
+    s->sym_end = (s->lit_bufsize - 1) * 3;
+    /* We avoid equality with lit_bufsize*3 because of wraparound at 64K
+     * on 16 bit machines and because stored blocks are restricted to
+     * 64K-1 bytes.
+     */
 
     s->level = level;
     s->strategy = strategy;
@@ -473,7 +510,7 @@ int ZEXPORT deflatePrime (strm, bits, value)
 
     if (strm == Z_NULL || strm->state == Z_NULL) return Z_STREAM_ERROR;
     s = strm->state;
-    if ((Bytef *)(s->d_buf) < s->pending_out + ((Buf_size + 7) >> 3))
+    if (s->sym_buf < s->pending_out + ((Buf_size + 7) >> 3))
         return Z_BUF_ERROR;
     do {
         put = Buf_size - s->bi_valid;
@@ -1022,7 +1059,6 @@ int ZEXPORT deflateCopy (dest, source)
 #else
     deflate_state *ds;
     deflate_state *ss;
-    ushf *overlay;
 
 
     if (source == Z_NULL || dest == Z_NULL || source->state == Z_NULL) {
@@ -1042,8 +1078,7 @@ int ZEXPORT deflateCopy (dest, source)
     ds->window = (Bytef *) ZALLOC(dest, ds->w_size, 2*sizeof(Byte));
     ds->prev   = (Posf *)  ZALLOC(dest, ds->w_size, sizeof(Pos));
     ds->head   = (Posf *)  ZALLOC(dest, ds->hash_size, sizeof(Pos));
-    overlay = (ushf *) ZALLOC(dest, ds->lit_bufsize, sizeof(ush)+2);
-    ds->pending_buf = (uchf *) overlay;
+    ds->pending_buf = (uchf *) ZALLOC(dest, ds->lit_bufsize, 4);
 
     if (ds->window == Z_NULL || ds->prev == Z_NULL || ds->head == Z_NULL ||
         ds->pending_buf == Z_NULL) {
@@ -1057,8 +1092,7 @@ int ZEXPORT deflateCopy (dest, source)
     zmemcpy(ds->pending_buf, ss->pending_buf, (uInt)ds->pending_buf_size);
 
     ds->pending_out = ds->pending_buf + (ss->pending_out - ss->pending_buf);
-    ds->d_buf = overlay + ds->lit_bufsize/sizeof(ush);
-    ds->l_buf = ds->pending_buf + (1+sizeof(ush))*ds->lit_bufsize;
+    ds->sym_buf = ds->pending_buf + ds->lit_bufsize;
 
     ds->l_desc.dyn_tree = ds->dyn_ltree;
     ds->d_desc.dyn_tree = ds->dyn_dtree;
@@ -1737,7 +1771,7 @@ local block_state deflate_fast(s, flush)
         FLUSH_BLOCK(s, 1);
         return finish_done;
     }
-    if (s->last_lit)
+    if (s->sym_next)
         FLUSH_BLOCK(s, 0);
     return block_done;
 }
@@ -1878,7 +1912,7 @@ local block_state deflate_slow(s, flush)
         FLUSH_BLOCK(s, 1);
         return finish_done;
     }
-    if (s->last_lit)
+    if (s->sym_next)
         FLUSH_BLOCK(s, 0);
     return block_done;
 }
@@ -1953,7 +1987,7 @@ local block_state deflate_rle(s, flush)
         FLUSH_BLOCK(s, 1);
         return finish_done;
     }
-    if (s->last_lit)
+    if (s->sym_next)
         FLUSH_BLOCK(s, 0);
     return block_done;
 }
@@ -1992,7 +2026,7 @@ local block_state deflate_huff(s, flush)
         FLUSH_BLOCK(s, 1);
         return finish_done;
     }
-    if (s->last_lit)
+    if (s->sym_next)
         FLUSH_BLOCK(s, 0);
     return block_done;
 }

zlib/deflate.h

@@ -214,7 +214,7 @@ typedef struct internal_state {
     /* Depth of each subtree used as tie breaker for trees of equal frequency
      */
 
-    uchf *l_buf;          /* buffer for literals or lengths */
+    uchf *sym_buf;        /* buffer for distances and literals/lengths */
 
     uInt  lit_bufsize;
     /* Size of match buffer for literals/lengths.  There are 4 reasons for
@@ -236,13 +236,8 @@ typedef struct internal_state {
      *   - I can't count above 4
      */
 
-    uInt last_lit;      /* running index in l_buf */
-
-    ushf *d_buf;
-    /* Buffer for distances. To simplify the code, d_buf and l_buf have
-     * the same number of elements. To use different lengths, an extra flag
-     * array would be necessary.
-     */
+    uInt sym_next;      /* running index in sym_buf */
+    uInt sym_end;       /* symbol table full when sym_next reaches this */
 
     ulg opt_len;        /* bit length of current block with optimal trees */
     ulg static_len;     /* bit length of current block with static trees */
@@ -322,20 +317,22 @@ void ZLIB_INTERNAL _tr_stored_block OF((deflate_state *s, charf *buf,
 
 # define _tr_tally_lit(s, c, flush) \
   { uch cc = (c); \
-    s->d_buf[s->last_lit] = 0; \
-    s->l_buf[s->last_lit++] = cc; \
+    s->sym_buf[s->sym_next++] = 0; \
+    s->sym_buf[s->sym_next++] = 0; \
+    s->sym_buf[s->sym_next++] = cc; \
     s->dyn_ltree[cc].Freq++; \
-    flush = (s->last_lit == s->lit_bufsize-1); \
+    flush = (s->sym_next == s->sym_end); \
    }
 # define _tr_tally_dist(s, distance, length, flush) \
   { uch len = (length); \
     ush dist = (distance); \
-    s->d_buf[s->last_lit] = dist; \
-    s->l_buf[s->last_lit++] = len; \
+    s->sym_buf[s->sym_next++] = dist; \
+    s->sym_buf[s->sym_next++] = dist >> 8; \
+    s->sym_buf[s->sym_next++] = len; \
     dist--; \
     s->dyn_ltree[_length_code[len]+LITERALS+1].Freq++; \
     s->dyn_dtree[d_code(dist)].Freq++; \
-    flush = (s->last_lit == s->lit_bufsize-1); \
+    flush = (s->sym_next == s->sym_end); \
   }
 #else
 # define _tr_tally_lit(s, c, flush) flush = _tr_tally(s, 0, c)

zlib/trees.c

@@ -418,7 +418,7 @@ local void init_block(s)
 
     s->dyn_ltree[END_BLOCK].Freq = 1;
     s->opt_len = s->static_len = 0L;
-    s->last_lit = s->matches = 0;
+    s->sym_next = s->matches = 0;
 }
 
 #define SMALLEST 1
@@ -943,7 +943,7 @@ void ZLIB_INTERNAL _tr_flush_block(s, buf, stored_len, last)
 
         Tracev((stderr, "\nopt %lu(%lu) stat %lu(%lu) stored %lu lit %u ",
                 opt_lenb, s->opt_len, static_lenb, s->static_len, stored_len,
-                s->last_lit));
+                s->sym_next / 3));
 
         if (static_lenb <= opt_lenb) opt_lenb = static_lenb;
 
@@ -1012,8 +1012,9 @@ int ZLIB_INTERNAL _tr_tally (s, dist, lc)
     unsigned dist;  /* distance of matched string */
     unsigned lc;    /* match length-MIN_MATCH or unmatched char (if dist==0) */
 {
-    s->d_buf[s->last_lit] = (ush)dist;
-    s->l_buf[s->last_lit++] = (uch)lc;
+    s->sym_buf[s->sym_next++] = dist;
+    s->sym_buf[s->sym_next++] = dist >> 8;
+    s->sym_buf[s->sym_next++] = lc;
     if (dist == 0) {
         /* lc is the unmatched char */
         s->dyn_ltree[lc].Freq++;
@@ -1028,30 +1029,7 @@ int ZLIB_INTERNAL _tr_tally (s, dist, lc)
         s->dyn_ltree[_length_code[lc]+LITERALS+1].Freq++;
         s->dyn_dtree[d_code(dist)].Freq++;
     }
-
-#ifdef TRUNCATE_BLOCK
-    /* Try to guess if it is profitable to stop the current block here */
-    if ((s->last_lit & 0x1fff) == 0 && s->level > 2) {
-        /* Compute an upper bound for the compressed length */
-        ulg out_length = (ulg)s->last_lit*8L;
-        ulg in_length = (ulg)((long)s->strstart - s->block_start);
-        int dcode;
-        for (dcode = 0; dcode < D_CODES; dcode++) {
-            out_length += (ulg)s->dyn_dtree[dcode].Freq *
-                (5L+extra_dbits[dcode]);
-        }
-        out_length >>= 3;
-        Tracev((stderr,"\nlast_lit %u, in %ld, out ~%ld(%ld%%) ",
-               s->last_lit, in_length, out_length,
-               100L - out_length*100L/in_length));
-        if (s->matches < s->last_lit/2 && out_length < in_length/2) return 1;
-    }
-#endif
-    return (s->last_lit == s->lit_bufsize-1);
-    /* We avoid equality with lit_bufsize because of wraparound at 64K
-     * on 16 bit machines and because stored blocks are restricted to
-     * 64K-1 bytes.
-     */
+    return (s->sym_next == s->sym_end);
 }
 
 /* ===========================================================================
@@ -1064,13 +1042,14 @@ local void compress_block(s, ltree, dtree)
 {
     unsigned dist;      /* distance of matched string */
     int lc;             /* match length or unmatched char (if dist == 0) */
-    unsigned lx = 0;    /* running index in l_buf */
+    unsigned sx = 0;    /* running index in sym_buf */
     unsigned code;      /* the code to send */
     int extra;          /* number of extra bits to send */
 
-    if (s->last_lit != 0) do {
-        dist = s->d_buf[lx];
-        lc = s->l_buf[lx++];
+    if (s->sym_next != 0) do {
+        dist = s->sym_buf[sx++] & 0xff;
+        dist += (unsigned)(s->sym_buf[sx++] & 0xff) << 8;
+        lc = s->sym_buf[sx++];
         if (dist == 0) {
             send_code(s, lc, ltree); /* send a literal byte */
             Tracecv(isgraph(lc), (stderr," '%c' ", lc));
@@ -1095,11 +1074,10 @@ local void compress_block(s, ltree, dtree)
             }
         } /* literal or match pair ? */
 
-        /* Check that the overlay between pending_buf and d_buf+l_buf is ok: */
-        Assert((uInt)(s->pending) < s->lit_bufsize + 2*lx,
-               "pendingBuf overflow");
+        /* Check that the overlay between pending_buf and sym_buf is ok: */
+        Assert(s->pending < s->lit_bufsize + sx, "pendingBuf overflow");
 
-    } while (lx < s->last_lit);
+    } while (sx < s->sym_next);
 
     send_code(s, END_BLOCK, ltree);
 }