Preparing for release of 3.2.4pre3

Ensure that a remote rsync daemon will not split a filename arg unless
the user asked for `--old-args`.

.cirrus.yml

@@ -1,13 +1,14 @@
 freebsd_task:
   name: FreeBSD
   freebsd_instance:
-    image: freebsd-12-1-release-amd64
+    image_family: freebsd-12-2
   env:
     PATH: /usr/local/bin:$PATH
   prep_script:
     - dd if=/dev/zero of=/tmp/zpool bs=1M count=1024
     - zpool create -m `pwd`/testtmp zpool /tmp/zpool
-    - pkg install -y autotools xxhash zstd liblz4
+    - pkg install -y bash autotools m4 xxhash zstd liblz4 wget
+    - wget -O git-version.h https://gist.githubusercontent.com/WayneD/c11243fa374fc64d4e42f2855c8e3827/raw/rsync-git-version.h
   configure_script:
     - CPPFLAGS=-I/usr/local/include/ LDFLAGS=-L/usr/local/lib/ ./configure --disable-md2man
   make_script:
@@ -17,6 +18,6 @@ freebsd_task:
   info_script:
     - rsync --version
   test_script:
-    - RSYNC_MAX_SKIPPED=3 make check
+    - RSYNC_EXPECT_SKIPPED=acls-default,acls,crtimes,protected-regular make check
   ssl_file_list_script:
     - rsync-ssl --no-motd download.samba.org::rsyncftp/ || true

.github/workflows/build.yml

@@ -16,14 +16,13 @@ jobs:
     runs-on: ubuntu-20.04
     steps:
     - uses: actions/checkout@v2
-      with:
-        fetch-depth: 0
     - name: prep
       run: |
-        sudo apt-get install acl libacl1-dev attr libattr1-dev liblz4-dev libzstd-dev libxxhash-dev python3-cmarkgfm openssl
-        echo "::add-path::/usr/local/bin"
+        sudo apt-get install acl libacl1-dev attr libattr1-dev liblz4-dev libzstd-dev libxxhash-dev python3-cmarkgfm openssl wget
+        wget -O git-version.h https://gist.githubusercontent.com/WayneD/c11243fa374fc64d4e42f2855c8e3827/raw/rsync-git-version.h
+        echo "/usr/local/bin" >>$GITHUB_PATH
     - name: configure
-      run: ./configure
+      run: ./configure --with-rrsync
     - name: make
       run: make
     - name: install
@@ -31,11 +30,11 @@ jobs:
     - name: info
       run: rsync --version
     - name: check
-      run: sudo RSYNC_MAX_SKIPPED=1 make check
+      run: sudo RSYNC_EXPECT_SKIPPED=crtimes make check
     - name: check30
-      run: sudo RSYNC_MAX_SKIPPED=1 make check30
+      run: sudo RSYNC_EXPECT_SKIPPED=crtimes make check30
     - name: check29
-      run: sudo RSYNC_MAX_SKIPPED=1 make check29
+      run: sudo RSYNC_EXPECT_SKIPPED=crtimes make check29
     - name: ssl file list
       run: rsync-ssl --no-motd download.samba.org::rsyncftp/ || true
     - name: save artifact
@@ -48,6 +47,8 @@ jobs:
           rsync.1
           rsync-ssl.1
           rsyncd.conf.5
+          rrsync.1
+          rrsync
 
   macos-build:
     runs-on: macos-latest
@@ -55,11 +56,12 @@ jobs:
     - uses: actions/checkout@v2
     - name: prep
       run: |
-        brew install automake openssl xxhash zstd lz4
+        brew install automake openssl xxhash zstd lz4 wget
         sudo pip3 install commonmark
-        echo "::add-path::/usr/local/bin"
+        wget -O git-version.h https://gist.githubusercontent.com/WayneD/c11243fa374fc64d4e42f2855c8e3827/raw/rsync-git-version.h
+        echo "/usr/local/bin" >>$GITHUB_PATH
     - name: configure
-      run: CPPFLAGS=-I/usr/local/opt/openssl/include/ LDFLAGS=-L/usr/local/opt/openssl/lib/ ./configure
+      run: CPPFLAGS=-I/usr/local/opt/openssl/include/ LDFLAGS=-L/usr/local/opt/openssl/lib/ ./configure --with-rrsync
     - name: make
       run: make
     - name: install
@@ -67,7 +69,7 @@ jobs:
     - name: info
       run: rsync --version
     - name: check
-      run: sudo make check
+      run: sudo RSYNC_EXPECT_SKIPPED=acls-default,chmod-temp-dir,chown-fake,devices-fake,dir-sgid,protected-regular,xattrs-hlink,xattrs make check
     - name: ssl file list
       run: rsync-ssl --no-motd download.samba.org::rsyncftp/ || true
     - name: save artifact
@@ -80,6 +82,8 @@ jobs:
           rsync.1
           rsync-ssl.1
           rsyncd.conf.5
+          rrsync.1
+          rrsync
 
   cygwin-build:
     runs-on: windows-latest
@@ -91,10 +95,13 @@ jobs:
         args: install -y --no-progress cygwin cyg-get
     - name: prep
       run: |
-        cyg-get make autoconf automake gcc-core attr libattr-devel python3 python36-commonmark libzstd-devel liblz4-devel libssl-devel
-        echo "::add-path::C:/tools/cygwin/bin"
+        cyg-get make autoconf automake gcc-core attr libattr-devel python38 python38-pip libzstd-devel liblz4-devel libssl-devel libxxhash0 libxxhash-devel
+        curl.exe -o git-version.h https://gist.githubusercontent.com/WayneD/c11243fa374fc64d4e42f2855c8e3827/raw/rsync-git-version.h
+        echo "C:/tools/cygwin/bin" >>$Env:GITHUB_PATH
+    - name: commonmark
+      run: bash -c 'python3 -mpip install --user commonmark'
     - name: configure
-      run: bash -c './configure --disable-xxhash'
+      run: bash -c './configure --with-rrsync'
     - name: make
       run: bash -c 'make'
     - name: install
@@ -102,7 +109,7 @@ jobs:
     - name: info
       run: bash -c '/usr/local/bin/rsync --version'
     - name: check
-      run: bash -c 'make check'
+      run: bash -c 'RSYNC_EXPECT_SKIPPED=acls-default,acls,chown,devices,dir-sgid,protected-regular make check'
     - name: ssl file list
       run: bash -c 'PATH="/usr/local/bin:$PATH" rsync-ssl --no-motd download.samba.org::rsyncftp/ || true'
     - name: save artifact
@@ -115,3 +122,5 @@ jobs:
           rsync.1
           rsync-ssl.1
           rsyncd.conf.5
+          rrsync.1
+          rrsync

.gitignore

@@ -15,9 +15,10 @@ config.status
 aclocal.m4
 /proto.h
 /proto.h-tstamp
-/rsync*.1
-/rsync*.5
+/rsync*.[15]
+/rrsync*.1
 /rsync*.html
+/rrsync*.html
 /help-rsync*.h
 /default-cvsignore.h
 /default-dont-compress.h
@@ -28,6 +29,7 @@ aclocal.m4
 /conftest*
 /dox
 /getgroups
+/gists
 /gmon.out
 /rsync
 /stunnel-rsyncd.conf

COPYING

@@ -1,7 +1,16 @@
+REGARDING OPENSSL AND XXHASH
+
+In addition, as a special exception, the copyright holders give
+permission to dynamically link rsync with the OpenSSL and xxhash
+libraries when those libraries are being distributed in compliance
+with their license terms, and to distribute a dynamically linked
+combination of rsync and these libraries.  This is also considered
+to be covered under the GPL's System Libraries exception.
+
                     GNU GENERAL PUBLIC LICENSE
                        Version 3, 29 June 2007
 
- Copyright (C) 2007 Free Software Foundation, Inc. <http://fsf.org/>
+ Copyright (C) 2007 Free Software Foundation, Inc. <https://fsf.org/>
  Everyone is permitted to copy and distribute verbatim copies
  of this license document, but changing it is not allowed.
 
@@ -645,7 +654,7 @@ the "copyright" line and a pointer to where the full notice is found.
     GNU General Public License for more details.
 
     You should have received a copy of the GNU General Public License
-    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+    along with this program.  If not, see <https://www.gnu.org/licenses/>.
 
 Also add information on how to contact you by electronic and paper mail.
 
@@ -664,20 +673,11 @@ might be different; for a GUI interface, you would use an "about box".
   You should also get your employer (if you work as a programmer) or school,
 if any, to sign a "copyright disclaimer" for the program, if necessary.
 For more information on this, and how to apply and follow the GNU GPL, see
-<http://www.gnu.org/licenses/>.
+<https://www.gnu.org/licenses/>.
 
   The GNU General Public License does not permit incorporating your program
 into proprietary programs.  If your program is a subroutine library, you
 may consider it more useful to permit linking proprietary applications with
 the library.  If this is what you want to do, use the GNU Lesser General
 Public License instead of this License.  But first, please read
-<http://www.gnu.org/philosophy/why-not-lgpl.html>.
-
-REGARDING OPENSSL AND XXHASH
-
-In addition, as a special exception, the copyright holders give
-permission to dynamically link rsync with the OpenSSL and xxhash
-libraries when those libraries are being distributed in compliance
-with their license terms, and to distribute a dynamically linked
-combination of rsync and these libraries.  This is also considered
-to be covered under the GPL's System Libraries exception.
+<https://www.gnu.org/licenses/why-not-lgpl.html>.

INSTALL.md

@@ -22,9 +22,9 @@ If your OS doesn't provide a python3-cmarkgfm or python3-commonmark package,
 you can run the following to install the commonmark python library for your
 build user (after installing python3's pip package):
 
->     pip3 install --user commonmark
+>     python3 -mpip install --user commonmark
 
-You can test if you've got it fixed by running (from the src dir):
+You can test if you've got it fixed by running (from the rsync checkout):
 
 >     ./md2man --test rsync-ssl.1.md
 
@@ -53,7 +53,8 @@ installed to manipulate xattrs and to run the rsync testsuite.
 The [xxHash library][1] provides extremely fast checksum functions that can
 make the "rsync algorithm" run much more quickly, especially when matching
 blocks in large files.  Installing this development library adds xxhash
-checksums as the default checksum algorithm.
+checksums as the default checksum algorithm.  You'll need at least v0.8.0
+if you want rsync to include the full range of its checksum algorithms.
 
 [1]: https://cyan4973.github.io/xxHash/
 
@@ -100,7 +101,7 @@ like.
     >     sudo apt install -y attr libattr1-dev
     >     sudo apt install -y libxxhash-dev
     >     sudo apt install -y libzstd-dev
-    >     sudo apt install -y libzlz4-dev
+    >     sudo apt install -y liblz4-dev
     >     sudo apt install -y libssl-dev
 
  -  For CentOS (use EPEL for python3-pip):
@@ -113,7 +114,7 @@ like.
     >     sudo yum -y install libzstd-devel
     >     sudo yum -y install lz4-devel
     >     sudo yum -y install openssl-devel
-    >     pip3 install --user commonmark
+    >     python3 -mpip install --user commonmark
 
  -  For Fedora 33:
 
@@ -141,12 +142,18 @@ like.
 
  -  For Cygwin (with all cygwin programs stopped, run the appropriate setup program from a cmd shell):
 
-    >     setup-x86_64 --quiet-mode -P make,gawk,autoconf,automake,gcc-core,python3,python36-commonmark
+    >     setup-x86_64 --quiet-mode -P make,gawk,autoconf,automake,gcc-core,python38,python38-pip
     >     setup-x86_64 --quiet-mode -P attr,libattr-devel
     >     setup-x86_64 --quiet-mode -P libzstd-devel
     >     setup-x86_64 --quiet-mode -P liblz4-devel
     >     setup-x86_64 --quiet-mode -P libssl-devel
 
+    Sometimes cygwin has commonmark packaged and sometimes it doesn't. Now that
+    its python38 has stabilized, you could install python38-commonmark. Or just
+    avoid the issue by running this from a bash shell as your build user:
+
+    >     python3 -mpip install --user commonmark
+
 ## Build and install
 
 After installing the various libraries, you need to configure, build, and
@@ -171,9 +178,9 @@ config.h, or just override them in your /etc/rsyncd.conf file.
 As of 2.4.7, rsync uses Eric Troan's popt option-parsing library.  A
 cut-down copy of a recent release is included in the rsync distribution,
 and will be used if there is no popt library on your build host, or if
-the --with-included-popt option is passed to ./configure.
+the `--with-included-popt` option is passed to ./configure.
 
-If you configure using --enable-maintainer-mode, then rsync will try
+If you configure using `--enable-maintainer-mode`, then rsync will try
 to pop up an xterm on DISPLAY=:0 if it crashes.  You might find this
 useful, but it should be turned off for production builds.
 
@@ -187,7 +194,7 @@ This is helpful when using the branch-from-patch and patch-update scripts
 to maintain the official rsync patches.  If you ever need to build from
 a "detached head" git position then you'll need to manually chdir into
 the build dir to run make.  I also like to create 2 more symlinks in the
-source dir:  ln -s build/rsync . ; ln -s build/testtmp .
+source dir:  `ln -s build/rsync . ; ln -s build/testtmp .`
 
 ## Make compatibility
 
@@ -221,7 +228,7 @@ Some versions of Mac OS X (Darwin) seem to have an IPv6 stack, but do
 not completely implement the "New Sockets" API.
 
 [This site][5] says that Apple started to support IPv6 in 10.2 (Jaguar).  If
-your build fails, try again after running configure with --disable-ipv6.
+your build fails, try again after running configure with `--disable-ipv6`.
 
 [5]: http://www.ipv6.org/impl/mac.html
 

Makefile.in

@@ -6,6 +6,7 @@ exec_prefix=@exec_prefix@
 bindir=@bindir@
 libdir=@libdir@/rsync
 mandir=@mandir@
+with_rrsync=@with_rrsync@
 
 LIBS=@LIBS@
 CC=@CC@
@@ -29,28 +30,29 @@ SHELL=/bin/sh
 .SUFFIXES:
 .SUFFIXES: .c .o
 
-SIMD_x86_64=simd-checksum-x86_64.o
+SIMD_x86_64=simd-checksum-x86_64.o simd-checksum-avx2.o
 ASM_x86_64=lib/md5-asm-x86_64.o
 
 GENFILES=configure.sh aclocal.m4 config.h.in rsync.1 rsync.1.html \
-	 rsync-ssl.1 rsync-ssl.1.html rsyncd.conf.5 rsyncd.conf.5.html
+	 rsync-ssl.1 rsync-ssl.1.html rsyncd.conf.5 rsyncd.conf.5.html \
+	 @GEN_RRSYNC@
 HEADERS=byteorder.h config.h errcode.h proto.h rsync.h ifuncs.h itypes.h inums.h \
-	lib/pool_alloc.h lib/mdigest.h lib/md-defines.h version.h
+	lib/pool_alloc.h lib/mdigest.h lib/md-defines.h
 LIBOBJ=lib/wildmatch.o lib/compat.o lib/snprintf.o lib/mdfour.o lib/md5.o \
 	lib/permstring.o lib/pool_alloc.o lib/sysacls.o lib/sysxattrs.o @LIBOBJS@
 zlib_OBJS=zlib/deflate.o zlib/inffast.o zlib/inflate.o zlib/inftrees.o \
 	zlib/trees.o zlib/zutil.o zlib/adler32.o zlib/compress.o zlib/crc32.o
 OBJS1=flist.o rsync.o generator.o receiver.o cleanup.o sender.o exclude.o \
-	util.o util2.o main.o checksum.o match.o syscall.o log.o backup.o delete.o
+	util1.o util2.o main.o checksum.o match.o syscall.o log.o backup.o delete.o
 OBJS2=options.o io.o compat.o hlink.o token.o uidlist.o socket.o hashtable.o \
 	usage.o fileio.o batch.o clientname.o chmod.o acls.o xattrs.o
-OBJS3=progress.o pipe.o @ASM@
+OBJS3=progress.o pipe.o @ASM@ @SIMD@
 DAEMON_OBJ = params.o loadparm.o clientserver.o access.o connection.o authenticate.o
 popt_OBJS=popt/findme.o  popt/popt.o  popt/poptconfig.o \
 	popt/popthelp.o popt/poptparse.o
-OBJS=$(OBJS1) $(OBJS2) $(OBJS3) @SIMD@ $(DAEMON_OBJ) $(LIBOBJ) @BUILD_ZLIB@ @BUILD_POPT@
+OBJS=$(OBJS1) $(OBJS2) $(OBJS3) $(DAEMON_OBJ) $(LIBOBJ) @BUILD_ZLIB@ @BUILD_POPT@
 
-TLS_OBJ = tls.o syscall.o t_stub.o lib/compat.o lib/snprintf.o lib/permstring.o lib/sysxattrs.o @BUILD_POPT@
+TLS_OBJ = tls.o syscall.o util2.o t_stub.o lib/compat.o lib/snprintf.o lib/permstring.o lib/sysxattrs.o @BUILD_POPT@
 
 # Programs we must have to run the test cases
 CHECK_PROGS = rsync$(EXEEXT) tls$(EXEEXT) getgroups$(EXEEXT) getfsdev$(EXEEXT) \
@@ -67,7 +69,7 @@ CHECK_OBJS=tls.o testrun.o getgroups.o getfsdev.o t_stub.o t_unsafe.o trimslash.
 	$(CC) -I. -I$(srcdir) $(CFLAGS) $(CPPFLAGS) -c $< @CC_SHOBJ_FLAG@
 @OBJ_RESTORE@
 
-all: Makefile rsync$(EXEEXT) stunnel-rsyncd.conf @MAKE_MAN@
+all: Makefile rsync$(EXEEXT) stunnel-rsyncd.conf @MAKE_RRSYNC@ @MAKE_MAN@
 .PHONY: all
 
 .PHONY: install
@@ -80,6 +82,10 @@ install: all
 	if test -f rsync.1; then $(INSTALLMAN) -m 644 rsync.1 $(DESTDIR)$(mandir)/man1; fi
 	if test -f rsync-ssl.1; then $(INSTALLMAN) -m 644 rsync-ssl.1 $(DESTDIR)$(mandir)/man1; fi
 	if test -f rsyncd.conf.5; then $(INSTALLMAN) -m 644 rsyncd.conf.5 $(DESTDIR)$(mandir)/man5; fi
+	if test "$(with_rrsync)" = yes; then \
+	    $(INSTALLCMD) -m 755 rrsync $(DESTDIR)$(bindir); \
+	    if test -f rrsync.1; then $(INSTALLMAN) -m 644 rrsync.1 $(DESTDIR)$(mandir)/man1; fi; \
+	fi
 
 install-ssl-daemon: stunnel-rsyncd.conf
 	-$(MKDIR_P) $(DESTDIR)/etc/stunnel
@@ -96,10 +102,13 @@ install-strip:
 rsync$(EXEEXT): $(OBJS)
 	$(CC) $(CFLAGS) $(LDFLAGS) -o $@ $(OBJS) $(LIBS)
 
+rrsync: support/rrsync
+	cp -p $(srcdir)/support/rrsync rrsync
+
 $(OBJS): $(HEADERS)
 $(CHECK_OBJS): $(HEADERS)
 tls.o xattrs.o: lib/sysxattrs.h
-usage.o: latest-year.h help-rsync.h help-rsyncd.h git-version.h default-cvsignore.h
+usage.o: version.h latest-year.h help-rsync.h help-rsyncd.h git-version.h default-cvsignore.h
 loadparm.o: default-dont-compress.h daemon-parm.h
 
 flist.o: rounding.h
@@ -131,17 +140,20 @@ rounding.h: rounding.c rsync.h proto.h
 	fi
 	@rm -f rounding.out
 
-# While $(wildcard ...) is a GNU make idiom, at least other makes should just turn it into an
-# empty string (we need something that will vanish if we're not building a git checkout).
-# If you want an updated git version w/o GNU make, remove git-version.h after a pull.
-git-version.h: mkgitver $(wildcard $(srcdir)/.git/logs/HEAD)
+git-version.h: ALWAYS_RUN
 	$(srcdir)/mkgitver
 
+.PHONY: ALWAYS_RUN
+ALWAYS_RUN:
+
 simd-checksum-x86_64.o: simd-checksum-x86_64.cpp
-	@$(srcdir)/cmdormsg disable-simd $(CXX) -I. $(CXXFLAGS) $(CPPFLAGS) -c -o $@ $(srcdir)/simd-checksum-x86_64.cpp
+	@$(srcdir)/cmd-or-msg disable-simd $(CXX) -I. $(CXXFLAGS) $(CPPFLAGS) -c -o $@ $(srcdir)/simd-checksum-x86_64.cpp
+
+simd-checksum-avx2.o: simd-checksum-avx2.S
+	@$(srcdir)/cmd-or-msg disable-asm $(CC) $(CFLAGS) --include=$(srcdir)/rsync.h -DAVX2_ASM -I. @NOEXECSTACK@ -c -o $@ $(srcdir)/simd-checksum-avx2.S
 
 lib/md5-asm-x86_64.o: lib/md5-asm-x86_64.S config.h lib/md-defines.h
-	@$(srcdir)/cmdormsg disable-asm $(CC) -I. @NOEXECSTACK@ -c -o $@ $(srcdir)/lib/md5-asm-x86_64.S
+	@$(srcdir)/cmd-or-msg disable-asm $(CC) -I. @NOEXECSTACK@ -c -o $@ $(srcdir)/lib/md5-asm-x86_64.S
 
 tls$(EXEEXT): $(TLS_OBJ)
 	$(CC) $(CFLAGS) $(LDFLAGS) -o $@ $(TLS_OBJ) $(LIBS)
@@ -155,11 +167,11 @@ getgroups$(EXEEXT): getgroups.o
 getfsdev$(EXEEXT): getfsdev.o
 	$(CC) $(CFLAGS) $(LDFLAGS) -o $@ getfsdev.o $(LIBS)
 
-TRIMSLASH_OBJ = trimslash.o syscall.o t_stub.o lib/compat.o lib/snprintf.o
+TRIMSLASH_OBJ = trimslash.o syscall.o util2.o t_stub.o lib/compat.o lib/snprintf.o
 trimslash$(EXEEXT): $(TRIMSLASH_OBJ)
 	$(CC) $(CFLAGS) $(LDFLAGS) -o $@ $(TRIMSLASH_OBJ) $(LIBS)
 
-T_UNSAFE_OBJ = t_unsafe.o syscall.o util.o util2.o t_stub.o lib/compat.o lib/snprintf.o lib/wildmatch.o
+T_UNSAFE_OBJ = t_unsafe.o syscall.o util1.o util2.o t_stub.o lib/compat.o lib/snprintf.o lib/wildmatch.o
 t_unsafe$(EXEEXT): $(T_UNSAFE_OBJ)
 	$(CC) $(CFLAGS) $(LDFLAGS) -o $@ $(T_UNSAFE_OBJ) $(LIBS)
 
@@ -167,11 +179,15 @@ t_unsafe$(EXEEXT): $(T_UNSAFE_OBJ)
 conf: configure.sh config.h.in
 
 .PHONY: gen
-gen: conf proto.h man
+gen: conf proto.h man git-version.h
 
 .PHONY: gensend
 gensend: gen
-	rsync -aic $(GENFILES) $${SAMBA_HOST-samba.org}:/home/ftp/pub/rsync/generated-files/
+	if ! diff git-version.h $(srcdir)/gists/rsync-git-version.h >/dev/null; then \
+	    ./rsync -ai git-version.h $(srcdir)/gists/rsync-git-version.h && \
+	    (cd $(srcdir)/gists && git commit --allow-empty-message -m '' rsync-git-version.h && git push) ; \
+	fi
+	rsync -aic $(GENFILES) git-version.h $${SAMBA_HOST-samba.org}:/home/ftp/pub/rsync/generated-files/ || true
 
 aclocal.m4: $(srcdir)/m4/*.m4
 	aclocal -I $(srcdir)/m4
@@ -193,7 +209,7 @@ configure.sh config.h.in: configure.ac aclocal.m4
 	else \
 	    echo "config.h.in has CHANGED."; \
 	fi
-	@if test -f configure.sh.old -o -f config.h.in.old; then \
+	@if test -f configure.sh.old || test -f config.h.in.old; then \
 	    if test "$(MAKECMDGOALS)" = reconfigure; then \
 		echo 'Continuing with "make reconfigure".'; \
 	    else \
@@ -240,22 +256,25 @@ proto.h-tstamp: $(srcdir)/*.c $(srcdir)/lib/compat.c daemon-parm.h
 	$(AWK) -f $(srcdir)/mkproto.awk $(srcdir)/*.c $(srcdir)/lib/compat.c daemon-parm.h
 
 .PHONY: man
-man: rsync.1 rsync-ssl.1 rsyncd.conf.5
+man: rsync.1 rsync-ssl.1 rsyncd.conf.5 @MAKE_RRSYNC_1@
 
-rsync.1: rsync.1.md md2man version.h Makefile
-	@$(srcdir)/maybe-make-man $(srcdir) rsync.1.md
+rsync.1: rsync.1.md md-convert version.h Makefile
+	@$(srcdir)/maybe-make-man rsync.1.md
 
-rsync-ssl.1: rsync-ssl.1.md md2man version.h Makefile
-	@$(srcdir)/maybe-make-man $(srcdir) rsync-ssl.1.md
+rsync-ssl.1: rsync-ssl.1.md md-convert version.h Makefile
+	@$(srcdir)/maybe-make-man rsync-ssl.1.md
 
-rsyncd.conf.5: rsyncd.conf.5.md md2man version.h Makefile
-	@$(srcdir)/maybe-make-man $(srcdir) rsyncd.conf.5.md
+rsyncd.conf.5: rsyncd.conf.5.md md-convert version.h Makefile
+	@$(srcdir)/maybe-make-man rsyncd.conf.5.md
+
+rrsync.1: support/rrsync.1.md md-convert Makefile
+	@$(srcdir)/maybe-make-man support/rrsync.1.md
 
 .PHONY: clean
 clean: cleantests
-	rm -f *~ $(OBJS) $(CHECK_PROGS) $(CHECK_OBJS) $(CHECK_SYMLINKS) \
-		rounding rounding.h *.old rsync*.1 rsync*.5 rsync*.html \
-		daemon-parm.h help-*.h default-*.h proto.h proto.h-tstamp
+	rm -f *~ $(OBJS) $(CHECK_PROGS) $(CHECK_OBJS) $(CHECK_SYMLINKS) @MAKE_RRSYNC@ \
+		git-version.h rounding rounding.h *.old rsync*.1 rsync*.5 @MAKE_RRSYNC_1@ \
+		*.html daemon-parm.h help-*.h default-*.h proto.h proto.h-tstamp
 
 .PHONY: cleantests
 cleantests:

NEWS.md

@@ -1,4 +1,230 @@
-<a name="3.2.3"></a>
+# NEWS for rsync 3.2.4 (UNRELEASED)
+
+## Changes in this version:
+
+### BEHAVIOR CHANGES:
+
+ - A new form of arg protection was added that works similarly to the older
+   [`--protect-args`](rsync.1#opt) (`-s`) option but in a way that avoids
+   breaking things like rrsync (the restricted rsync script): rsync now uses
+   backslash escaping for sending "shell-active" characters to the remote
+   shell. This includes spaces, so fetching a remote file via a simple quoted
+   filename value now works by default without any extra quoting:
+
+   ```shell
+       rsync -aiv host:'a simple file.pdf' .
+   ```
+
+   Wildcards are not escaped in filename args, but they are escaped in options
+   like the [`--suffix`](rsync.1#opt) and [`--usermap`](rsync.1#opt) values.
+   If your rsync script depends on the old arg-splitting behavior, either run
+   it with the [`--old-args`](rsync.1#opt) option or `export RSYNC_OLD_ARGS=1`
+   in the script's environment.  See also the [ADVANCED USAGE](rsync.1#)
+   section of rsync's man page.
+
+ - A long-standing bug was preventing rsync from figuring out the current
+   locale's decimal point character, which made rsync always output numbers
+   using the "C" locale.  Since this is now fixed in 3.2.4, a script that
+   parses rsync's decimal numbers (e.g. from the verbose footer) should be sure
+   to setup the environment in a way that the output continues to be in the C
+   locale.  For instance, one of the following should work fine:
+
+   ```shell
+       export LC_ALL=C.UTF-8
+   ```
+
+   or if iconv translations are needed:
+
+   ```shell
+       if [ "${LC_ALL:-}" ]; then
+           export LANG="$LC_ALL"
+           export LC_CTYPE="$LC_ALL"
+           unset LC_ALL
+       fi
+       export LC_NUMERIC=C.UTF-8
+   ```
+
+### BUG FIXES:
+
+ - Fixed a bug with [`--inplace`](rsync.1#opt) + [`--sparse`](rsync.1#opt) (and
+   a lack of [`--whole-file`](rsync.1#opt)) where the destination file could
+   get reconstructed with bogus data.  Since the bug can also be avoided by
+   using (the seemingly redundant) [`--no-W`](rsync.1#opt) on the receiving
+   side, the latest rsync will now send `--no-W` to a remote receiver when this
+   option combination occurs.  If your client rsync is not new enough to do
+   this for you (or if you're just paranoid), you can manually specify `--no-W
+   -M--no-W` (when not using [`--whole-file`](rsync.1#opt)) to make sure the
+   bug is avoided.
+
+ - Fixed a bug with [`--mkpath`](rsync.1#opt) if a single-file copy specifies
+   an existing destination dir with a non-existing destination filename.
+
+ - Fixed `--update -vv` to output "is uptodate" instead of "is newer" messages
+   for files that are being skipped due to an identical modify time.  (This was
+   a new output quirk in 3.2.3.)
+
+ - When doing an append transfer, the sending side's file must not get shorter
+   or it is skipped. Fixes a crash that could occur when the size changes to 0
+   in the middle of the send negotiations.
+
+ - When dealing with special files (see [`--specials`](rsync.1#opt)) in an
+   alt-dest hierarchy, rsync now checks the non-permission mode bits to ensure
+   that the 2 special files are really the same before hard-linking them
+   together.
+
+ - Fixed a bug where [`--delay-updates`](rsync.1#opt) with stale partial data
+   could cause a file to fail to update.
+
+ - Fixed a few places that would output an INFO message with
+   [`--info=NAME`](rsync.1#opt) that should only have been output given
+   [`--verbose`](rsync.1#opt) or [`--itemize-changes`](rsync.1#opt).
+
+ - Avoid a weird failure if you run a local copy with a (useless)
+   [`--rsh`](rsync.1#opt) option that contains a `V` in the command.
+
+ - Fixed a long-standing compression bug where the compression level of the
+   first file transferred affected the level for all future files.  Also, the
+   per-file compression skipping has apparently never worked, so it is now
+   documented as being ineffective.
+
+ - Improved how the [`--stop-at`](rsync.1#opt), [`--stop-after`](rsync.1#opt),
+   and (the deprecated) [`--time-limit`](rsync.1#opt) options check to see if
+   the allowed time is over, which should make rsync exit more consistently.
+
+ - Tweak --progress to display "`??:??:??`" when the time-remaining value is so
+   large as to be meaningless.
+
+ - Silence some chmod warnings about symlinks when it looks like we have a
+   function to set their permissions but they can't really be set.
+
+### ENHANCEMENTS:
+
+ - Use openssl's `-verify_hostname` option in the rsync-ssl script.
+
+ - Added extra info to the "FILENAME exists" output of
+   [`--ignore-existing`](rsync.1#opt) when [`--info=skip2`](rsync.1#opt) is
+   used.  The skip message becomes "FILENAME exists (INFO)" where the INFO is
+   one of "type change", "sum change" (requires [`--checksum`](rsync.1#opt)),
+   "file change" (based on the quick check), "attr change", or "uptodate".
+   Prior versions only supported `--info=skip1`.
+
+ - Added the [`--fsync`](rsync.1#opt) option (promoted from the patches repo).
+
+ - Reduced memory usage for an incremental transfer that has a bunch of small
+   directories.
+
+ - The rsync daemon can now handle a client address with an implied "%scope"
+   suffix.
+
+ - Added support for [`--atimes`](rsync.1#opt) on macOS and fixed a bug where
+   it wouldn't work without [`--times`](rsync.1#opt).
+
+ - Rsync can now update the xattrs on a read-only file when your user can
+   temporarily add user-write permission to the file. (It always worked for a
+   root transfer.)
+
+ - Rsync can now work around an [`--inplace`](rsync.1#opt) update of a file
+   that is being refused due to the Linux fs.protected_regular sysctl setting.
+
+ - When [`--chown`](rsync.1#opt), [`--usermap`](rsync.1#opt), or
+   [`--groupmap`](rsync.1#opt), is specified, rsync now makes sure that the
+   appropriate [`--owner`](rsync.1#opt) and/or [`--group`](rsync.1#opt) options
+   are enabled.
+
+ - Added the [`--info=NONREG`](rsync.1#opt) setting to control if rsync should
+   warn about non-regular files in the transfer.  This is enabled by default
+   (keeping the behavior the same as before), so specifying `--info=nonreg0`
+   can be used to turn the warnings off.
+
+ - More ASM optimizations from Shark64.
+
+ - Transformed rrsync into a python script with improvements:
+   - Security has been beefed up.
+   - The known rsync options were updated to include recent additions.
+   - Make rrsync reject [`--copy-links`](rsync.1#opt) (`-L`),
+     [`--copy-dirlinks`](rsync.1#opt) (`-k`), &
+     [`--keep-dirlinks`](rsync.1#opt) (`-K`) by default to make it harder to
+     exploit any out-of-subdir symlinks.
+   - A new rrsync option of [`-munge`](rrsync.1#opt) tells rrsync to always
+     enable rsync's [`--munge-links`](rsync.1#opt) option on the server side.
+   - A new rrsync option of [`-no-lock`](rrsync.1#opt) disables a new
+     single-use locking idiom that is the default when [`-ro`](rrsync.1#opt) is
+     not used (useful with [`-munge`](rrsync.1#opt)).
+   - A new rrsync option of [`-no-del`](rrsync.1#opt) disables all `--remove*`
+     and `--delete*` rsync options on the server side.
+   - The log format has been tweaked slightly to add seconds to the timestamp
+     and to output the command executed as a tuple (making the args clearer).
+   - An rrsync.1 man page was added (in the support dir with rrsync).
+
+ - Added options to the lsh script to facilitate rrsync testing. (See the
+   support dir.)
+
+ - Transformed the atomic-rsync script into a python script and added the
+   ability to ignore one or more non-zero exit codes.  By default, it now
+   ignores code 24, the file-vanished exit code. (See the support dir.)
+
+ - Transformed the munge-symlinks script into python. (See the support dir.)
+
+ - Improved the rsync-no-vanished script to not join stdout & stderr together.
+   (See the support dir.)
+
+ - Work around a glibc bug where lchmod() breaks in a chroot w/o /proc mounted.
+
+ - Try to support a client that sent a remote rsync a wacko stderr file handle
+   (such as an older File::RsyncP perl library used by BackupPC).
+
+ - Lots of man page improvements, including better html versions.
+
+### PACKAGING RELATED:
+
+ - Give configure the `--with-rrsync` option if you want `make install` to
+   install the (now python3) rrsync script and its new man page.
+
+ - If the rrsync script is installed, its package should be changed to depend
+   on python3 and the (suggested but not mandatory) python3 braceexpand lib.
+
+ - When creating a package from a non-release version (w/o a git checkout), the
+   packager can elect to create git-version.h and define RSYNC_GITVER to the
+   string they want `--version` to output.  (The file is still auto-generated
+   using the output of `git describe` when building inside a non-shallow git
+   checkout, though.)
+
+ - Improved the IPv6 determination in configure.
+
+ - Made SIMD & ASM configure default to "no" on non-Linux hosts due to various
+   reports of problems on NetBSD & macOS hosts.  These tests were also tweaked
+   to allow enabling the feature on a host_cpu of amd64 (was only x86_64).
+
+ - Fixed configure to not fail at the SIMD check when cross-compiling.
+
+ - Compile the C files with `-pedantic-errors` (when possible) so that we will
+   get warned if a static initialization overflows in the future (among other
+   things).
+
+ - Added a SECURITY.md file.
+
+### DEVELOPER RELATED:
+
+ - Made it easier to write rsync tests that diff the output while also checking
+   the status code, and used the idiom to improve the existing tests. (See the
+   `checkdiff` and `checkdiff2` idioms in the `testsuite/*.test` files.
+
+ - The packaging scripts & related python lib got some minor enhancements.
+
+### INTERNAL
+
+ - Use setenv() instead of putenv() when it is available.
+
+ - Improve the logic in compat.c so that we don't need to try to remember to
+   sprinkle `!local_server` exceptions throughout the protocol logic.
+
+ - One more C99 Flexible Array improvement (started in the last release) and
+   make use of the C99 `%zd` format string when printing size_t values (when
+   possible).
+
+ - Use mallinfo2() instead of mallinfo(), when available.
+
+------------------------------------------------------------------------------
 
 # NEWS for rsync 3.2.3 (6 Aug 2020)
 
@@ -12,67 +238,72 @@
  - Fixed a bug in the xattr code that was not leaving room for the "rsync."
    prefix in some instances where it needed to be added.
 
- - Restored the ability to use `--bwlimit=0` to specify no bandwidth limit.  (It
-   was accidentally broken in 3.2.2.)
+ - Restored the ability to use [`--bwlimit=0`](rsync.1#opt) to specify no
+   bandwidth limit.  (It was accidentally broken in 3.2.2.)
 
- - Fix a bug when combining `--delete-missing-args` with `--no-implied-dirs` &
-   `-R` where rsync might create the destination path of a missing arg.  The
-   code also avoids some superfluous warnings for nested paths of removed args.
+ - Fixed a bug when combining [`--delete-missing-args`](rsync.1#opt) with
+   [`--no-implied-dirs`](rsync.1#opt) & [`-R`](rsync.1#opt) where rsync might
+   create the destination path of a missing arg.  The code also avoids some
+   superfluous warnings for nested paths of removed args.
 
  - Fixed an issue where hard-linked devices could cause the rdev_major value to
    get out of sync between the sender and the receiver, which could cause a
    device to get created with the wrong major value in its major,minor pair.
 
- - Rsync now complains about a missing `--temp-dir` before starting any file
-   transfers.
+ - Rsync now complains about a missing [`--temp-dir`](rsync.1#opt) before
+   starting any file transfers.
 
  - A completely empty source arg is now a fatal error.  This doesn't change
    the handling of implied dot-dir args such as "localhost:" and such.
 
 ### ENHANCEMENTS:
 
- - Allow `--max-alloc=0` to specify no limit to the alloc sanity check.
+ - Allow [`--max-alloc=0`](rsync.1#opt) to specify no limit to the alloc sanity
+   check.
 
- - Allow `--block-size=SIZE` to specify the size using units (e.g. "100K").
+ - Allow [`--block-size=SIZE`](rsync.1#opt) to specify the size using units
+   (e.g. "100K").
 
  - The name of the id-0 user & group are now sent to the receiver along with
    the other user/group names in the transfer (instead of assuming that both
    sides have the same id-0 names).
 
- - Added the `--stop-after=MINS` and `--stop-at=DATE_TIME` options (with the
-   `--time-limit=MINS` option accepted as an alias for `--stop-after`).  This
-   is an enhanced version of the time-limit patch from the patches repo.
+ - Added the [`--stop-after`](rsync.1#opt) and [`--stop-at`](rsync.1#opt)
+   options (with a [`--time-limit`](rsync.1#opt) alias for `--stop-after`).
+   This is an enhanced version of the time-limit patch from the patches repo.
 
- - Added the `name converter` daemon parameter to make it easier to convert
-   user & group names inside a chrooted daemon module.  This is based on the
-   nameconverter patch with some improvements, including a tweak to the request
-   protocol (so if you used this patch in the past, be sure to update your
-   converter script to use newlines instead of null chars).
+ - Added the [`name converter`](rsyncd.conf.5#opt) daemon parameter to make it
+   easier to convert user & group names inside a chrooted daemon module.  This
+   is based on the nameconverter patch with some improvements, including a
+   tweak to the request protocol (so if you used this patch in the past, be
+   sure to update your converter script to use newlines instead of null chars).
 
- - Added `--crtimes` (`-N`) option for preserving the file's create time (I
-   believe that this is macOS only at the moment).
+ - Added [`--crtimes`](rsync.1#opt) (`-N`) option for preserving the file's
+   create time (I believe that this is macOS only at the moment).
 
- - Added `--mkpath` option to tell rsync that it should create a non-existing
-   path component of the destination arg.
+ - Added [`--mkpath`](rsync.1#opt) option to tell rsync that it should create a
+   non-existing path component of the destination arg.
 
- - Added `--stderr=errors|all|client` to replace the `--msgs2stderr` and
-   `--no-msgs2stderr` options (which are still accepted).  The default use of
-   stderr was changed to be `--stderr=errors` where all the processes that have
-   stderr available output directly to stderr, which should help error messages
-   get to the user more quickly, especially when doing a push (which includes
-   local copying).  This also allows rsync to exit quickly when a receiver
-   failure occurs, since rsync doesn't need to try to keep the connection alive
-   long enough for the fatal error to go from the receiver to the generator to
-   the sender.  The old default can be requested via `--stderr=client`.  Also
-   changed is that a non-default stderr mode is conveyed to the remote rsync
-   (using the older option names) instead of requiring the user to use
-   `--remote-option` (`-M`) to tell the remote rsync what to do.
+ - Added [`--stderr=errors|all|client`](rsync.1#opt) to replace the
+   `--msgs2stderr` and `--no-msgs2stderr` options (which are still accepted).
+   The default use of stderr was changed to be `--stderr=errors` where all the
+   processes that have stderr available output directly to stderr, which should
+   help error messages get to the user more quickly, especially when doing a
+   push (which includes local copying).  This also allows rsync to exit quickly
+   when a receiver failure occurs, since rsync doesn't need to try to keep the
+   connection alive long enough for the fatal error to go from the receiver to
+   the generator to the sender.  The old default can be requested via
+   `--stderr=client`.  Also changed is that a non-default stderr mode is
+   conveyed to the remote rsync (using the older option names) instead of
+   requiring the user to use [`--remote-option`](rsync.1#opt) (`-M`) to tell
+   the remote rsync what to do.
 
- - Added the ability to specify "@netgroup" names to the `hosts allow` and
-   `hosts deny` daemon parameters.  This is a finalized version of the
-   netgroup-auth patch from the patches repo.
+ - Added the ability to specify "@netgroup" names to the [`hosts
+   allow`](rsyncd.conf.5#opt) and [`hosts deny`](rsyncd.conf.5#opt) daemon
+   parameters.  This is a finalized version of the netgroup-auth patch from the
+   patches repo.
 
- - Rsync can now hard-link symlinks on FreeBSD due to it making ues of the
+ - Rsync can now hard-link symlinks on FreeBSD due to it making use of the
    linkat() function when it is available.
 
  - Output file+line info on out-of-memory & overflow errors while also avoiding
@@ -99,7 +330,6 @@
    (with a fallback to the old 1-char string kluge for older compilers).
 
 ------------------------------------------------------------------------------
-<a name="3.2.2"></a>
 
 # NEWS for rsync 3.2.2 (4 Jul 2020)
 
@@ -175,7 +405,6 @@
    can create the interrelated structs and accessors that loadparm.c needs.
 
 ------------------------------------------------------------------------------
-<a name="3.2.1"></a>
 
 # NEWS for rsync 3.2.1 (22 Jun 2020)
 
@@ -238,7 +467,6 @@
  - Merged the OLDNEWS.md file into NEWS.md.
 
 ------------------------------------------------------------------------------
-<a name="3.2.0"></a>
 
 # NEWS for rsync 3.2.0 (19 Jun 2020)
 
@@ -450,7 +678,6 @@
  - Some code typos were fixed (as pointed out by a Fossies run).
 
 ------------------------------------------------------------------------------
-<a name="3.1.3"></a>
 
 # NEWS for rsync 3.1.3 (28 Jan 2018)
 
@@ -468,7 +695,8 @@
 
  - Don't output about a new backup dir without appropriate info verbosity.
 
- - Fixed some issues with the sort functions in support/rsyncstats script.
+ - Fixed some issues with the sort functions in the rsyncstats script (in the
+   support dir).
 
  - Added a way to specify daemon config lists (e.g. users, groups, etc) that
    contain spaces (see `auth users` in the latest rsyncd.conf manpage).
@@ -520,7 +748,6 @@
  - Support newer yodl versions when converting man pages.
 
 ------------------------------------------------------------------------------
-<a name="3.1.2"></a>
 
 # NEWS for rsync 3.1.2 (21 Dec 2015)
 
@@ -586,7 +813,6 @@
  - Improved the m4 generation rules and some autoconf idioms.
 
 ------------------------------------------------------------------------------
-<a name="3.1.1"></a>
 
 # NEWS for rsync 3.1.1 (22 Jun 2014)
 
@@ -679,7 +905,7 @@
    non-bundled zlib. See the `--new-compress` and `--old-compress` options in
    the manpage.
 
- - Added the support/rsync-no-vanished wrapper script.
+ - Added the rsync-no-vanished shell script. (See the support dir.)
 
  - Made configure more prominently mention when we failed to find yodl (in case
    the user wants to be able to generate manpages from `*.yo` files).
@@ -706,7 +932,6 @@
    and/or zlib code is put early in the CFLAGS.
 
 ------------------------------------------------------------------------------
-<a name="3.1.0"></a>
 
 # NEWS for rsync 3.1.0 (28 Sep 2013)
 
@@ -752,7 +977,7 @@
  - Fixed a bug in the iconv code when EINVAL or EILSEQ is returned with a full
    output buffer.
 
- - Fixed some rare bugs in `--iconv` processing that might cause a multibyte
+ - Fixed some rare bugs in `--iconv` processing that might cause a multi-byte
    character to get translated incorrectly.
 
  - Fixed a bogus `vanished file` error if some files were specified with `./`
@@ -858,7 +1083,7 @@
    hard-link it into place so that the upcoming replacement of the destination
    file will be atomic (for the normal, non-inplace logic).
 
- - Added the ability to synchronize nano-second modified times.
+ - Added the ability to synchronize nanosecond modified times.
 
  - Added a few more default suffixes for the `dont compress` settings.
 
@@ -957,7 +1182,6 @@
  - Fixed some build issues for Android and Minix.
 
 ------------------------------------------------------------------------------
-<a name="3.0.9"></a>
 
 # NEWS for rsync 3.0.9 (23 Sep 2011)
 
@@ -1012,12 +1236,11 @@
 
  - Fix some issues with the post-processing of the man pages.
 
- - Fixed the user home-dir handling in the support/lsh script.
+ - Fixed the user home-dir handling in the lsh script. (See the support dir.)
 
  - Some minor manpage improvements.
 
 ------------------------------------------------------------------------------
-<a name="3.0.8"></a>
 
 # NEWS for rsync 3.0.8 (26 Mar 2011)
 
@@ -1130,10 +1353,10 @@
    reject an attempt to supply one (can configure `--with-included-popt` if
    your system's popt library doesn't yet have this fix).
 
- - A couple minor option tweaks to the support/rrsync script, and also some
-   regex changes that make vim highlighting happier.
+ - A couple minor option tweaks to the rrsync script, and also some regex
+   changes that make vim highlighting happier. (See the support dir.)
 
- - Fixed some issues in the support/mnt-excl script.
+ - Fixed some issues in the mnt-excl script. (See the support dir.)
 
  - Various manpage improvements.
 
@@ -1156,7 +1379,6 @@
  - Fixed the testsuite/xattrs.test script on OS X.
 
 ------------------------------------------------------------------------------
-<a name="3.0.7"></a>
 
 # NEWS for rsync 3.0.7 (31 Dec 2009)
 
@@ -1170,7 +1392,7 @@
    that hasn't really been created.
 
  - Fixed a problem with `--compress` (`-z`) where the receiving side could
-   return the error "inflate (token) returned -5".
+   return the error "`inflate (token) returned -5`".
 
  - Fixed a bug where `--delete-during` could delete in a directory before it
    noticed that the sending side sent an I/O error for that directory (both
@@ -1189,7 +1411,7 @@
  - An absolute-path filter rule (i.e. with a '/' modifier) no longer loses its
    modifier when sending the filter rules to the remote rsync.
 
- - Improved the "--delete does not work without -r or -d" message.
+ - Improved the "`--delete does not work without -r or -d`" message.
 
  - Improved rsync's handling of `--timeout` to avoid a weird timeout case where
    the sender could timeout even though it has recently written data to the
@@ -1224,7 +1446,6 @@
  - The testsuite no longer uses `id -u`, so it works better on Solaris.
 
 ------------------------------------------------------------------------------
-<a name="3.0.6"></a>
 
 # NEWS for rsync 3.0.6 (8 May 2009)
 
@@ -1283,7 +1504,6 @@
  - Fixed an failure transferring special files from Solaris to Linux.
 
 ------------------------------------------------------------------------------
-<a name="3.0.5"></a>
 
 # NEWS for rsync 3.0.5 (28 Dec 2008)
 
@@ -1343,12 +1563,11 @@
 
 ### ENHANCEMENTS:
 
- - Made the support/atomic-rsync script able to perform a fully atomic update
-   of the copied hierarchy when the destination is setup using a particular
-   symlink idiom.
+ - Made the atomic-rsync script able to perform a fully atomic update of the
+   copied hierarchy when the destination is setup using a particular symlink
+   idiom. (See the support dir.)
 
 ------------------------------------------------------------------------------
-<a name="3.0.4"></a>
 
 # NEWS for rsync 3.0.4 (6 Sep 2008)
 
@@ -1416,7 +1635,6 @@
    even more consistency checks on the files.
 
 ------------------------------------------------------------------------------
-<a name="3.0.3"></a>
 
 # NEWS for rsync 3.0.3 (29 Jun 2008)
 
@@ -1500,14 +1718,13 @@
    of a proto.h-tstamp rule that could make the binaries get rebuild without
    cause.
 
- - Improved the testsuite to work around a problem with some utilities (e.g. cp
-   -p & touch -r) rounding sub-second timestamps.
+ - Improved the testsuite to work around a problem with some utilities (e.g.
+   `cp -p` & `touch -r`) rounding sub-second timestamps.
 
  - Ensure that the early patches don't cause any generated-file hunks to
    bleed-over into patches that follow.
 
 ------------------------------------------------------------------------------
-<a name="3.0.2"></a>
 
 # NEWS for rsync 3.0.2 (8 Apr 2008)
 
@@ -1529,7 +1746,6 @@
    packaging dir.
 
 ------------------------------------------------------------------------------
-<a name="3.0.1"></a>
 
 # NEWS for rsync 3.0.1 (3 Apr 2008)
 
@@ -1613,8 +1829,9 @@
 
  - Fixed the inclusion of per-dir merge files from implied dirs.
 
- - Fixed the support/rrsync script to work with the latest options that rsync
-   sends (including its flag-specifying use of `-e` to the server).
+ - Fixed the rrsync script to work with the latest options that rsync sends,
+   including its flag-specifying use of `-e` to the server. (See the support
+   dir.)
 
 ### ENHANCEMENTS:
 
@@ -1667,7 +1884,6 @@
  - Updated the build scripts to work with a revised FTP directory structure.
 
 ------------------------------------------------------------------------------
-<a name="3.0.0"></a>
 
 # NEWS for rsync 3.0.0 (1 Mar 2008)
 
@@ -2018,7 +2234,6 @@
    the 3.0.0 release.
 
 ------------------------------------------------------------------------------
-<a name="2.6.9"></a>
 
 # NEWS for rsync 2.6.9 (6 Nov 2006)
 
@@ -2178,7 +2393,6 @@
    consistent opening comments.
 
 ------------------------------------------------------------------------------
-<a name="2.6.8"></a>
 
 # NEWS for rsync 2.6.8 (22 Apr 2006)
 
@@ -2248,7 +2462,6 @@
    actions to a file (something that only a daemon supports at present).
 
 ------------------------------------------------------------------------------
-<a name="2.6.7"></a>
 
 # NEWS for rsync 2.6.7 (11 Mar 2006)
 
@@ -2263,7 +2476,7 @@
    option, below.
 
  - The way rsync escapes unreadable characters has changed. First, rsync now
-   has support for recognizing valid multibyte character sequences in your
+   has support for recognizing valid multi-byte character sequences in your
    current locale, allowing it to escape fewer characters than before for a
    locale such as UTF-8. Second, it now uses an escape idiom of `\#123`, which
    is the literal string `\#` followed by exactly 3 octal digits. Rsync no
@@ -2572,7 +2785,6 @@
    ~/.popt.
 
 ------------------------------------------------------------------------------
-<a name="2.6.6"></a>
 
 # NEWS for rsync 2.6.6 (28 Jul 2005)
 
@@ -2618,10 +2830,10 @@
  - Made the `max verbosity` setting in the rsyncd.conf file settable on a
    per-module basis (which now matches the documentation).
 
- - The support/rrsync script has been upgraded to verify the args of options
-   that take args (instead of rejecting any such options). The script was also
-   changed to try to be more secure and to fix a problem in the parsing of a
-   pull operation that has multiple sources.
+ - The rrsync script has been upgraded to verify the args of options that take
+   args (instead of rejecting any such options). It was also changed to try to
+   be more secure and to fix a problem in the parsing of a pull operation that
+   has multiple source args. (See the support dir.)
 
  - Improved the documentation that explains the difference between a normal
    daemon transfer and a daemon-over remote-shell transfer.
@@ -2638,7 +2850,6 @@
    (log-format w/%i) and some double-verbose messages.
 
 ------------------------------------------------------------------------------
-<a name="2.6.5"></a>
 
 # NEWS for rsync 2.6.5 (1 Jun 2005)
 
@@ -2819,7 +3030,6 @@
    enables the optional copying of extended attributes.
 
 ------------------------------------------------------------------------------
-<a name="2.6.4"></a>
 
 # NEWS for rsync 2.6.4 (30 March 2005)
 
@@ -3201,7 +3411,6 @@
  - Improved configure to better handle cross-compiling.
 
 ------------------------------------------------------------------------------
-<a name="2.6.3"></a>
 
 # NEWS for rsync 2.6.3 (30 Sep 2004)
 
@@ -3445,7 +3654,6 @@
    removed.
 
 ------------------------------------------------------------------------------
-<a name="2.6.2"></a>
 
 # NEWS for rsync 2.6.2 (30 Apr 2004)
 
@@ -3487,7 +3695,6 @@
  - Two new diffs were added to the patches dir.
 
 ------------------------------------------------------------------------------
-<a name="2.6.1"></a>
 
 # NEWS for rsync 2.6.1 (26 Apr 2004)
 
@@ -3679,7 +3886,6 @@
    applied, and rebuilt the rest.
 
 ------------------------------------------------------------------------------
-<a name="2.6.0"></a>
 
 # NEWS for rsync 2.6.0 (1 Jan 2004)
 
@@ -3819,7 +4025,6 @@
    other side (primarily for testing purposes).  (Wayne Davison)
 
 ------------------------------------------------------------------------------
-<a name="2.5.7"></a>
 
 # NEWS for rsync 2.5.7 (4 Dec 2003)
 
@@ -3831,7 +4036,6 @@
    Andrea Barisani)
 
 ------------------------------------------------------------------------------
-<a name="2.5.6"></a>
 
 # NEWS for rsync 2.5.6, aka "the dwd-between-jobs release" (26 Jan 2003)
 
@@ -3926,7 +4130,6 @@
    should build on more platforms. (Paul Green)
 
 ------------------------------------------------------------------------------
-<a name="2.5.5"></a>
 
 # NEWS for rsync 2.5.5, aka Snowy River (2 Apr 2002)
 
@@ -3965,7 +4168,6 @@
  - Improved network error handling. (Greg A. Woods)
 
 ------------------------------------------------------------------------------
-<a name="2.5.4"></a>
 
 # NEWS for rsync 2.5.4, aka "Imitation lizard skin" (13 Mar 2002)
 
@@ -3985,7 +4187,6 @@
  - Additional test cases for `--compress`. (Martin Pool)
 
 ------------------------------------------------------------------------------
-<a name="2.5.3"></a>
 
 # NEWS for rsync 2.5.3, aka "Happy 26" (11 Mar 2002)
 
@@ -4034,7 +4235,6 @@
    error message. (Colin Walters)
 
 ------------------------------------------------------------------------------
-<a name="2.5.2"></a>
 
 # NEWS for rsync 2.5.2 (26 Jan 2002)
 
@@ -4083,7 +4283,6 @@
    Razor. (Debian #124286)
 
 ------------------------------------------------------------------------------
-<a name="2.5.1"></a>
 
 # NEWS for rsync 2.5.1 (3 Jan 2002)
 
@@ -4118,7 +4317,6 @@
  - Clearer error messages for some conditions.
 
 ------------------------------------------------------------------------------
-<a name="2.5.0"></a>
 
 # NEWS for rsync 2.5.0 (30 Nov 2001)
 
@@ -4210,7 +4408,7 @@
    - HP PA-RISC HP-UX 11.11 cc
    - IRIX 6.5 MIPS cc
    - IRIX 6.5 MIPS gcc
-   - Mac OS X PPC (--disable-ipv6) cc
+   - Mac OS X PPC (`--disable-ipv6`) cc
    - NetBSD 1.5 i386 gcc
    - NetBSD Current i386 cc
    - OpenBSD 2.5 Sparc gcc
@@ -4244,6 +4442,7 @@
 
 | RELEASE DATE | VER.   | DATE OF COMMIT\* | PROTOCOL    |
 |--------------|--------|------------------|-------------|
+| ?? Jan 2022  | 3.2.4  |                  | 31          |
 | 06 Aug 2020  | 3.2.3  |                  | 31          |
 | 04 Jul 2020  | 3.2.2  |                  | 31          |
 | 22 Jun 2020  | 3.2.1  |                  | 31          |
@@ -4314,3 +4513,5 @@
 
 \* DATE OF COMMIT is the date the protocol change was committed to version
 control.
+
+@USE_GFM_PARSER@

README.md

@@ -23,8 +23,9 @@ options.  To get a complete list of supported options type:
 
     rsync --help
 
-See the manpage for more detailed information.
+See the [manpage][0] for more detailed information.
 
+[0]: https://download.samba.org/pub/rsync/rsync.1
 
 BUILDING AND INSTALLING
 -----------------------

SECURITY.md

@@ -0,0 +1,12 @@
+# Security Policy
+
+## Supported Versions
+
+Only the current release of the software is actively supported.  If you need
+help backporting fixes into an older release, feel free to ask.
+
+## Reporting a Vulnerability
+
+Email your vulnerability information to rsync's maintainer:
+
+  Wayne Davison <wayne@opencoder.net>

access.c

@@ -2,7 +2,7 @@
  * Routines to authenticate access to a daemon (hosts allow/deny).
  *
  * Copyright (C) 1998 Andrew Tridgell
- * Copyright (C) 2004-2020 Wayne Davison
+ * Copyright (C) 2004-2022 Wayne Davison
  *
  * This program is free software; you can redistribute it and/or modify
  * it under the terms of the GNU General Public License as published by
@@ -20,6 +20,9 @@
 
 #include "rsync.h"
 #include "ifuncs.h"
+#ifdef HAVE_NETGROUP_H
+#include <netgroup.h>
+#endif
 
 static int allow_forward_dns;
 

acls.c

@@ -3,7 +3,7 @@
  *
  * Copyright (C) 1996 Andrew Tridgell
  * Copyright (C) 1996 Paul Mackerras
- * Copyright (C) 2006-2020 Wayne Davison
+ * Copyright (C) 2006-2022 Wayne Davison
  *
  * This program is free software; you can redistribute it and/or modify
  * it under the terms of the GNU General Public License as published by
@@ -763,6 +763,7 @@ static int recv_rsync_acl(int f, item_list *racl_list, SMB_ACL_TYPE_T type, mode
 #ifdef HAVE_OSX_ACLS
 	/* If we received a superfluous mask, throw it away. */
 	duo_item->racl.mask_obj = NO_ENTRY;
+	(void)mode;
 #else
 	if (duo_item->racl.names.count && duo_item->racl.mask_obj == NO_ENTRY) {
 		/* Mask must be non-empty with lists. */

backup.c

@@ -2,7 +2,7 @@
  * Backup handling code.
  *
  * Copyright (C) 1999 Andrew Tridgell
- * Copyright (C) 2003-2020 Wayne Davison
+ * Copyright (C) 2003-2022 Wayne Davison
  *
  * This program is free software; you can redistribute it and/or modify
  * it under the terms of the GNU General Public License as published by
@@ -304,7 +304,8 @@ int make_backup(const char *fname, BOOL prefer_rename)
 #endif
 
 	if (!ret && !S_ISREG(file->mode)) {
-		rprintf(FINFO, "make_bak: skipping non-regular file %s\n", fname);
+		if (INFO_GTE(NONREG, 1))
+			rprintf(FINFO, "make_bak: skipping non-regular file %s\n", fname);
 		unmake_file(file);
 #ifdef SUPPORT_ACLS
 		uncache_tmp_acls();

clientname.c

@@ -3,7 +3,7 @@
  *
  * Copyright (C) 1992-2001 Andrew Tridgell <tridge@samba.org>
  * Copyright (C) 2001, 2002 Martin Pool <mbp@samba.org>
- * Copyright (C) 2002-2020 Wayne Davison
+ * Copyright (C) 2002-2022 Wayne Davison
  *
  * This program is free software; you can redistribute it and/or modify
  * it under the terms of the GNU General Public License as published by
@@ -49,7 +49,7 @@ static char ipaddr_buf[100];
 
 static void client_sockaddr(int fd, struct sockaddr_storage *ss, socklen_t *ss_len);
 static int check_name(const char *ipaddr, const struct sockaddr_storage *ss, char *name_buf, size_t name_buf_size);
-static int valid_ipaddr(const char *s);
+static int valid_ipaddr(const char *s, int allow_scope);
 
 /* Return the IP addr of the client as a string. */
 char *client_addr(int fd)
@@ -73,7 +73,7 @@ char *client_addr(int fd)
 			if ((p = strchr(ipaddr_buf, ' ')) != NULL)
 				*p = '\0';
 		}
-		if (valid_ipaddr(ipaddr_buf))
+		if (valid_ipaddr(ipaddr_buf, True))
 			return ipaddr_buf;
 	}
 
@@ -213,12 +213,14 @@ int read_proxy_protocol_header(int fd)
 				if (size != sizeof hdr.v2.addr.ip4)
 					return 0;
 				inet_ntop(AF_INET, hdr.v2.addr.ip4.src_addr, ipaddr_buf, sizeof ipaddr_buf);
-				return valid_ipaddr(ipaddr_buf);
+				return valid_ipaddr(ipaddr_buf, False);
+#ifdef INET6
 			case PROXY_FAM_TCPv6:
 				if (size != sizeof hdr.v2.addr.ip6)
 					return 0;
 				inet_ntop(AF_INET6, hdr.v2.addr.ip6.src_addr, ipaddr_buf, sizeof ipaddr_buf);
-				return valid_ipaddr(ipaddr_buf);
+				return valid_ipaddr(ipaddr_buf, False);
+#endif
 			default:
 				break;
 			}
@@ -274,7 +276,7 @@ int read_proxy_protocol_header(int fd)
 		if ((sp = strchr(p, ' ')) == NULL)
 			return 0;
 		*sp = '\0';
-		if (!valid_ipaddr(p))
+		if (!valid_ipaddr(p, False))
 			return 0;
 		strlcpy(ipaddr_buf, p, sizeof ipaddr_buf); /* It will always fit when valid. */
 
@@ -282,7 +284,7 @@ int read_proxy_protocol_header(int fd)
 		if ((sp = strchr(p, ' ')) == NULL)
 			return 0;
 		*sp = '\0';
-		if (!valid_ipaddr(p))
+		if (!valid_ipaddr(p, False))
 			return 0;
 		/* Ignore destination address. */
 
@@ -464,7 +466,7 @@ static int check_name(const char *ipaddr, const struct sockaddr_storage *ss, cha
 }
 
 /* Returns 1 for a valid IPv4 or IPv6 addr, or 0 for a bad one. */
-static int valid_ipaddr(const char *s)
+static int valid_ipaddr(const char *s, int allow_scope)
 {
 	int i;
 
@@ -482,6 +484,11 @@ static int valid_ipaddr(const char *s)
 		for (count = 0; count < 8; count++) {
 			if (!*s)
 				return saw_double_colon;
+			if (allow_scope && *s == '%') {
+				if (saw_double_colon)
+					break;
+				return 0;
+			}
 
 			if (strchr(s, ':') == NULL && strchr(s, '.') != NULL) {
 				if ((!saw_double_colon && count != 6) || (saw_double_colon && count > 6))
@@ -507,8 +514,11 @@ static int valid_ipaddr(const char *s)
 			}
 		}
 
-		if (!ipv4_at_end)
-			return !*s;
+		if (!ipv4_at_end) {
+			if (allow_scope && *s == '%')
+				for (s++; isAlNum(s); s++) { }
+			return !*s && s[-1] != '%';
+		}
 	}
 
 	/* IPv4 */

clientserver.c

@@ -3,7 +3,7 @@
  *
  * Copyright (C) 1998-2001 Andrew Tridgell <tridge@samba.org>
  * Copyright (C) 2001-2002 Martin Pool <mbp@samba.org>
- * Copyright (C) 2002-2020 Wayne Davison
+ * Copyright (C) 2002-2022 Wayne Davison
  *
  * This program is free software; you can redistribute it and/or modify
  * it under the terms of the GNU General Public License as published by
@@ -47,6 +47,7 @@ extern int protocol_version;
 extern int io_timeout;
 extern int no_detach;
 extern int write_batch;
+extern int old_style_args;
 extern int default_af_hint;
 extern int logfile_format_has_i;
 extern int logfile_format_has_o_or_i;
@@ -288,20 +289,45 @@ int start_inband_exchange(int f_in, int f_out, const char *user, int argc, char
 
 	sargs[sargc++] = ".";
 
+	if (!old_style_args)
+		snprintf(line, sizeof line, " %.*s/", modlen, modname);
+
 	while (argc > 0) {
 		if (sargc >= MAX_ARGS - 1) {
 		  arg_overflow:
 			rprintf(FERROR, "internal: args[] overflowed in do_cmd()\n");
 			exit_cleanup(RERR_SYNTAX);
 		}
-		if (strncmp(*argv, modname, modlen) == 0
-		 && argv[0][modlen] == '\0')
+		if (strncmp(*argv, modname, modlen) == 0 && argv[0][modlen] == '\0')
 			sargs[sargc++] = modname; /* we send "modname/" */
-		else if (**argv == '-') {
-			if (asprintf(sargs + sargc++, "./%s", *argv) < 0)
-				out_of_memory("start_inband_exchange");
-		} else
-			sargs[sargc++] = *argv;
+		else {
+			char *arg = *argv;
+			int extra_chars = *arg == '-' ? 2 : 0; /* a leading dash needs a "./" prefix. */
+			/* If --old-args was not specified, make sure that the arg won't split at a mod name! */
+			if (!old_style_args && (p = strstr(arg, line)) != NULL) {
+				do {
+					extra_chars += 2;
+				} while ((p = strstr(p+1, line)) != NULL);
+			}
+			if (extra_chars) {
+				char *f = arg;
+				char *t = arg = new_array(char, strlen(arg) + extra_chars + 1);
+				if (*f == '-') {
+					*t++ = '.';
+					*t++ = '/';
+				}
+				while (*f) {
+					if (*f == ' ' && strncmp(f, line, modlen+2) == 0) {
+						*t++ = '[';
+						*t++ = *f++;
+						*t++ = ']';
+					} else
+						*t++ = *f++;
+				}
+				*t = '\0';
+			}
+			sargs[sargc++] = arg;
+		}
 		argv++;
 		argc--;
 	}
@@ -380,7 +406,7 @@ int start_inband_exchange(int f_in, int f_out, const char *user, int argc, char
 	return 0;
 }
 
-#ifdef HAVE_PUTENV
+#if defined HAVE_SETENV || defined HAVE_PUTENV
 static int read_arg_from_pipe(int fd, char *buf, int limit)
 {
 	char *bp = buf, *eob = buf + limit - 1;
@@ -405,23 +431,57 @@ static int read_arg_from_pipe(int fd, char *buf, int limit)
 
 static void set_env_str(const char *var, const char *str)
 {
+#ifdef HAVE_SETENV
+	if (setenv(var, str, 1) < 0)
+		out_of_memory("set_env_str");
+#else
 #ifdef HAVE_PUTENV
 	char *mem;
 	if (asprintf(&mem, "%s=%s", var, str) < 0)
 		out_of_memory("set_env_str");
 	putenv(mem);
+#else
+	(void)var;
+	(void)str;
+#endif
 #endif
 }
 
+#if defined HAVE_SETENV || defined HAVE_PUTENV
+
+static void set_envN_str(const char *var, int num, const char *str)
+{
+#ifdef HAVE_SETENV
+	char buf[128];
+	(void)snprintf(buf, sizeof buf, "%s%d", var, num);
+	if (setenv(buf, str, 1) < 0)
+		out_of_memory("set_env_str");
+#else
 #ifdef HAVE_PUTENV
+	char *mem;
+	if (asprintf(&mem, "%s%d=%s", var, num, str) < 0)
+		out_of_memory("set_envN_str");
+	putenv(mem);
+#endif
+#endif
+}
+
 void set_env_num(const char *var, long num)
 {
+#ifdef HAVE_SETENV
+	char val[64];
+	(void)snprintf(val, sizeof val, "%ld", num);
+	if (setenv(var, val, 1) < 0)
+		out_of_memory("set_env_str");
+#else
+#ifdef HAVE_PUTENV
 	char *mem;
 	if (asprintf(&mem, "%s=%ld", var, num) < 0)
 		out_of_memory("set_env_num");
 	putenv(mem);
-}
 #endif
+#endif
+}
 
 /* Used for "early exec", "pre-xfer exec", and the "name converter" script. */
 static pid_t start_pre_exec(const char *cmd, int *arg_fd_ptr, int *error_fd_ptr)
@@ -451,15 +511,13 @@ static pid_t start_pre_exec(const char *cmd, int *arg_fd_ptr, int *error_fd_ptr)
 		set_env_str("RSYNC_REQUEST", buf);
 
 		for (j = 0; ; j++) {
-			char *p;
 			len = read_arg_from_pipe(arg_fd, buf, BIGPATHBUFLEN);
 			if (len <= 0) {
 				if (!len)
 					break;
 				_exit(1);
 			}
-			if (asprintf(&p, "RSYNC_ARG%d=%s", j, buf) >= 0)
-				putenv(p);
+			set_envN_str("RSYNC_ARG", j, buf);
 		}
 
 		dup2(arg_fd, STDIN_FILENO);
@@ -490,6 +548,8 @@ static pid_t start_pre_exec(const char *cmd, int *arg_fd_ptr, int *error_fd_ptr)
 	return pid;
 }
 
+#endif
+
 static void write_pre_exec_args(int write_fd, char *request, char **early_argv, char **argv, int exec_type)
 {
 	int j = 0;
@@ -809,7 +869,7 @@ static int rsync_module(int f_in, int f_out, int i, const char *addr, const char
 
 	log_init(1);
 
-#ifdef HAVE_PUTENV
+#if defined HAVE_SETENV || defined HAVE_PUTENV
 	if ((*lp_early_exec(module_id) || *lp_prexfer_exec(module_id)
 	  || *lp_postxfer_exec(module_id) || *lp_name_converter(module_id))
 	 && !getenv("RSYNC_NO_XFER_EXEC")) {

compat.c

@@ -3,7 +3,7 @@
  *
  * Copyright (C) Andrew Tridgell 1996
  * Copyright (C) Paul Mackerras 1996
- * Copyright (C) 2004-2020 Wayne Davison
+ * Copyright (C) 2004-2022 Wayne Davison
  *
  * This program is free software; you can redistribute it and/or modify
  * it under the terms of the GNU General Public License as published by
@@ -21,6 +21,7 @@
 
 #include "rsync.h"
 #include "itypes.h"
+#include "ifuncs.h"
 
 extern int am_server;
 extern int am_sender;
@@ -51,6 +52,8 @@ extern int need_messages_from_generator;
 extern int delete_mode, delete_before, delete_during, delete_after;
 extern int do_compression;
 extern int do_compression_level;
+extern int saw_stderr_opt;
+extern int msgs2stderr;
 extern char *shell_cmd;
 extern char *partial_dir;
 extern char *files_from;
@@ -153,7 +156,13 @@ static void check_sub_protocol(void)
 
 void set_allow_inc_recurse(void)
 {
-	client_info = shell_cmd ? shell_cmd : "";
+	if (!local_server)
+		client_info = shell_cmd ? shell_cmd : "";
+	else if (am_server) {
+		char buf[64];
+		maybe_add_e_option(buf, sizeof buf);
+		client_info = *buf ? strdup(buf+1) : ""; /* The +1 skips the leading "e". */
+	}
 
 	if (!recurse || use_qsort)
 		allow_inc_recurse = 0;
@@ -161,8 +170,7 @@ void set_allow_inc_recurse(void)
 	 && (delete_before || delete_after
 	  || delay_updates || prune_empty_dirs))
 		allow_inc_recurse = 0;
-	else if (am_server && !local_server
-	 && (strchr(client_info, 'i') == NULL))
+	else if (am_server && strchr(client_info, 'i') == NULL)
 		allow_inc_recurse = 0;
 }
 
@@ -558,7 +566,7 @@ void setup_protocol(int f_out,int f_in)
 		atimes_ndx = (file_extra_cnt += EXTRA64_CNT);
 	if (preserve_crtimes)
 		crtimes_ndx = (file_extra_cnt += EXTRA64_CNT);
-	if (am_sender) /* This is most likely in the in64 union as well. */
+	if (am_sender) /* This is most likely in the file_extras64 union as well. */
 		pathname_ndx = (file_extra_cnt += PTR_EXTRA_CNT);
 	else
 		depth_ndx = ++file_extra_cnt;
@@ -616,6 +624,9 @@ void setup_protocol(int f_out,int f_in)
 	if (read_batch)
 		check_batch_flags();
 
+	if (!saw_stderr_opt && protocol_version <= 28 && am_server)
+		msgs2stderr = 0; /* The client side may not have stderr setup for us. */
+
 #ifndef SUPPORT_PREALLOCATION
 	if (preallocate_files && !am_sender) {
 		rprintf(FERROR, "preallocation is not supported on this %s\n",
@@ -691,17 +702,17 @@ void setup_protocol(int f_out,int f_in)
 #ifdef ICONV_OPTION
 			compat_flags |= CF_SYMLINK_ICONV;
 #endif
-			if (local_server || strchr(client_info, 'f') != NULL)
+			if (strchr(client_info, 'f') != NULL)
 				compat_flags |= CF_SAFE_FLIST;
-			if (local_server || strchr(client_info, 'x') != NULL)
+			if (strchr(client_info, 'x') != NULL)
 				compat_flags |= CF_AVOID_XATTR_OPTIM;
-			if (local_server || strchr(client_info, 'C') != NULL)
+			if (strchr(client_info, 'C') != NULL)
 				compat_flags |= CF_CHKSUM_SEED_FIX;
-			if (local_server || strchr(client_info, 'I') != NULL)
+			if (strchr(client_info, 'I') != NULL)
 				compat_flags |= CF_INPLACE_PARTIAL_DIR;
-			if (local_server || strchr(client_info, 'u') != NULL)
+			if (strchr(client_info, 'u') != NULL)
 				compat_flags |= CF_ID0_NAMES;
-			if (local_server || strchr(client_info, 'v') != NULL) {
+			if (strchr(client_info, 'v') != NULL) {
 				do_negotiated_strings = 1;
 				compat_flags |= CF_VARINT_FLIST_FLAGS;
 			}
@@ -737,7 +748,7 @@ void setup_protocol(int f_out,int f_in)
 #endif
 #ifdef ICONV_OPTION
 		sender_symlink_iconv = iconv_opt && (am_server
-		    ? local_server || strchr(client_info, 's') != NULL
+		    ? strchr(client_info, 's') != NULL
 		    : !!(compat_flags & CF_SYMLINK_ICONV));
 #endif
 		if (inc_recurse && !allow_inc_recurse) {

configure.ac

@@ -2,6 +2,21 @@ dnl Process this file with autoconf to produce a configure script.
 
 AC_INIT([rsync],[ ],[https://rsync.samba.org/bug-tracking.html])
 
+AC_C_BIGENDIAN
+AC_HEADER_DIRENT
+AC_HEADER_TIME
+AC_HEADER_SYS_WAIT
+AC_CHECK_HEADERS(sys/fcntl.h sys/select.h fcntl.h sys/time.h sys/unistd.h \
+    unistd.h utime.h compat.h sys/param.h ctype.h sys/wait.h sys/stat.h \
+    sys/ioctl.h sys/filio.h string.h stdlib.h sys/socket.h sys/mode.h grp.h \
+    sys/un.h sys/attr.h arpa/inet.h arpa/nameser.h locale.h sys/types.h \
+    netdb.h malloc.h float.h limits.h iconv.h libcharset.h langinfo.h mcheck.h \
+    sys/acl.h acl/libacl.h attr/xattr.h sys/xattr.h sys/extattr.h dl.h \
+    popt.h popt/popt.h linux/falloc.h netinet/in_systm.h netgroup.h \
+    zlib.h xxhash.h openssl/md4.h openssl/md5.h zstd.h lz4.h sys/file.h)
+AC_CHECK_HEADERS([netinet/ip.h], [], [], [[#include <netinet/in.h>]])
+AC_HEADER_MAJOR_FIXED
+
 AC_CONFIG_MACRO_DIR([m4])
 AC_CONFIG_SRCDIR([byteorder.h])
 AC_CONFIG_HEADER(config.h)
@@ -26,7 +41,7 @@ AC_CONFIG_LIBOBJ_DIR([lib])
 
 AC_MSG_CHECKING([whether to include debugging symbols])
 AC_ARG_ENABLE(debug,
-	AS_HELP_STRING([--disable-debug],[disable debugging symbols and features]))
+	AS_HELP_STRING([--disable-debug],[disable to omit debugging symbols and features]))
 
 if test x"$enable_debug" = x"no"; then
     AC_MSG_RESULT(no)
@@ -63,7 +78,7 @@ nl='
 '
 
 AC_ARG_ENABLE(profile,
-	AS_HELP_STRING([--enable-profile],[turn on CPU profiling]))
+	AS_HELP_STRING([--enable-profile],[enable to turn on CPU profiling]))
 if test x"$enable_profile" = x"yes"; then
 	CFLAGS="$CFLAGS -pg"
 fi
@@ -86,7 +101,7 @@ fi
 
 AC_MSG_CHECKING([if we require man-page building])
 AC_ARG_ENABLE([md2man],
-	AS_HELP_STRING([--disable-md2man],[disable md2man for man page creation]))
+	AS_HELP_STRING([--disable-md2man],[disable to omit man page creation]))
 if test x"$enable_md2man" != x"no"; then
     if test -f "$srcdir/rsync.1"; then
 	AC_MSG_RESULT(optional)
@@ -102,12 +117,11 @@ if test x"$enable_md2man" != x"no"; then
     MAKE_MAN=man
 else
     AC_MSG_RESULT(no)
-    MAKE_MAN=''
 fi
 
 # Specifically, this turns on panic_action handling.
 AC_ARG_ENABLE(maintainer-mode,
-	AS_HELP_STRING([--enable-maintainer-mode],[turn on extra debug features]))
+	AS_HELP_STRING([--enable-maintainer-mode],[enable to turn on extra debug features]))
 if test x"$enable_maintainer_mode" = x"yes"; then
 	CFLAGS="$CFLAGS -DMAINTAINER_MODE"
 fi
@@ -121,6 +135,17 @@ if test x"$GCC" = x"yes"; then
 	CFLAGS="$CFLAGS -Wall -W"
 fi
 
+AC_ARG_WITH(rrsync,
+        AS_HELP_STRING([--with-rrsync],[also install the rrsync script and its man page]))
+if test x"$with_rrsync" != x"yes"; then
+    with_rrsync=no
+else
+    MAKE_RRSYNC='rrsync'
+    MAKE_RRSYNC_1='rrsync.1'
+    GEN_RRSYNC='rrsync.1 rrsync.1.html'
+fi
+AC_SUBST(with_rrsync)
+
 AC_ARG_WITH(included-popt,
         AS_HELP_STRING([--with-included-popt],[use bundled popt library, not from system]))
 
@@ -180,6 +205,11 @@ AC_DEFINE_UNQUOTED(RSYNC_RSH, "$RSYNC_RSH", [default -e command])
 AC_PATH_PROG(SHELL_PATH, sh, /bin/sh, [/usr/xpg4/bin$PATH_SEPARATOR$PATH])
 AC_PATH_PROG(FAKEROOT_PATH, fakeroot, /usr/bin/fakeroot, [/usr/xpg4/bin$PATH_SEPARATOR$PATH])
 
+AC_ARG_WITH(nobody-user,
+    AS_HELP_STRING([--with-nobody-user=USER],[set the default unprivileged user (default nobody)]),
+    [ NOBODY_USER="$with_nobody_user" ],
+    [ NOBODY_USER="nobody" ])
+
 AC_ARG_WITH(nobody-group,
     AS_HELP_STRING([--with-nobody-group=GROUP],[set the default unprivileged group (default nobody or nogroup)]),
     [ NOBODY_GROUP="$with_nobody_group" ])
@@ -196,7 +226,7 @@ if test x"$with_nobody_group" = x; then
     AC_MSG_RESULT($NOBODY_GROUP)
 fi
 
-AC_DEFINE_UNQUOTED(NOBODY_USER, "nobody", [unprivileged user--e.g. nobody])
+AC_DEFINE_UNQUOTED(NOBODY_USER, "$NOBODY_USER", [unprivileged user--e.g. nobody])
 AC_DEFINE_UNQUOTED(NOBODY_GROUP, "$NOBODY_GROUP", [unprivileged group for unprivileged user])
 
 # SIMD optimizations
@@ -204,16 +234,14 @@ SIMD=
 
 AC_MSG_CHECKING([whether to enable SIMD optimizations])
 AC_ARG_ENABLE(simd,
-    AS_HELP_STRING([--disable-simd],[disable SIMD optimizations (requires c++)]))
+    AS_HELP_STRING([--enable-simd],[enable/disable to control SIMD optimizations (requires c++)]))
 
 # Clag is crashing with -g -O2, so we'll get rid of -g for now.
 CXXFLAGS=`echo "$CXXFLAGS" | sed 's/-g //'`
-
-if test x"$enable_simd" != x"no"; then
-    # For x86-64 SIMD, g++ >=5 or clang++ >=7 is required
-    if test x"$build_cpu" = x"x86_64"; then
-	AC_LANG(C++)
-	AC_RUN_IFELSE([AC_LANG_PROGRAM([[#include <stdio.h>
+m4_define(SIMD_X86_64_TEST, [[#include <stdio.h>
+#if HAVE_STDLIB_H
+#include <stdlib.h>
+#endif
 #include <immintrin.h>
 __attribute__ ((target("default"))) int test_ssse3(int x) { return x; }
 __attribute__ ((target("default"))) int test_sse2(int x) { return x; }
@@ -233,11 +261,29 @@ __attribute__ ((target("ssse3"))) void more_testing(char* buf, int len)
 	in8_2 = _mm_lddqu_si128((__m128i_u*)&buf[i + 16]);
     }
 }
-]], [[if (test_ssse3(42) != 42 || test_sse2(42) != 42 || test_avx2(42) != 42) exit(1);]])],[CXX_OK=yes],[CXX_OK=no])
+]])
+
+if test x"$enable_simd" = x""; then
+    case "$host_os" in
+	*linux*) ;;
+	*) enable_simd=no ;;
+    esac
+fi
+
+if test x"$enable_simd" != x"no"; then
+    # For x86-64 SIMD, g++ >=5 or clang++ >=7 is required
+    if test x"$host_cpu" = x"x86_64" || test x"$host_cpu" = x"amd64"; then
+	AC_LANG(C++)
+	if test x"$host" = x"$build"; then
+	    AC_RUN_IFELSE([AC_LANG_PROGRAM([SIMD_X86_64_TEST],[[if (test_ssse3(42) != 42 || test_sse2(42) != 42 || test_avx2(42) != 42) exit(1);]])],
+		[CXX_OK=yes],[CXX_OK=no])
+	else
+	    AC_COMPILE_IFELSE([AC_LANG_PROGRAM([SIMD_X86_64_TEST])],[CXX_OK=yes],[CXX_OK=no])
+	fi
 	AC_LANG(C)
 	if test x"$CXX_OK" = x"yes"; then
 	    # AC_MSG_RESULT() is called below.
-	    SIMD="x86_64"
+	    SIMD="$host_cpu"
 	elif test x"$enable_simd" = x"yes"; then
 	    AC_MSG_RESULT(error)
 	    AC_MSG_ERROR(The SIMD compilation test failed.
@@ -245,7 +291,7 @@ Omit --enable-simd to continue without it.)
 	fi
     elif test x"$enable_simd" = x"yes"; then
         AC_MSG_RESULT(unavailable)
-        AC_MSG_ERROR(The SIMD optimizations are currently x86_64 only.
+        AC_MSG_ERROR(The SIMD optimizations are currently x86_64|amd64 only.
 Omit --enable-simd to continue without it.)
     fi
 fi
@@ -280,14 +326,21 @@ ASM=
 
 AC_MSG_CHECKING([whether to enable ASM optimizations])
 AC_ARG_ENABLE(asm,
-    AS_HELP_STRING([--disable-asm],[disable ASM optimizations]))
+    AS_HELP_STRING([--enable-asm],[enable/disable to control ASM optimizations]))
+
+if test x"$enable_asm" = x""; then
+    case "$host_os" in
+	*linux*) ;;
+	*) enable_asm=no ;;
+    esac
+fi
 
 if test x"$enable_asm" != x"no"; then
-    if test x"$build_cpu" = x"x86_64"; then
-	ASM="$build_cpu"
+    if test x"$host_cpu" = x"x86_64" || test x"$host_cpu" = x"amd64"; then
+	ASM="$host_cpu"
     elif test x"$enable_asm" = x"yes"; then
         AC_MSG_RESULT(unavailable)
-        AC_MSG_ERROR(The ASM optimizations are currently x86_64 only.
+        AC_MSG_ERROR(The ASM optimizations are currently x86_64|amd64 only.
 Omit --enable-asm to continue without it.)
     fi
 fi
@@ -309,7 +362,9 @@ AC_RUN_IFELSE([AC_LANG_SOURCE([[
 #define _FILE_OFFSET_BITS 64
 #include <stdio.h>
 #include <fcntl.h>
+#ifdef HAVE_SYS_TYPES_H
 #include <sys/types.h>
+#endif
 #include <sys/wait.h>
 #if HAVE_UNISTD_H
 #include <unistd.h>
@@ -345,132 +400,39 @@ if test x"$rsync_cv_HAVE_BROKEN_LARGEFILE" != x"yes"; then
    AC_SYS_LARGEFILE
 fi
 
-ipv6type=unknown
-ipv6lib=none
-ipv6trylibc=yes
-
+AC_MSG_CHECKING([whether to enable ipv6])
 AC_ARG_ENABLE(ipv6,
-	AS_HELP_STRING([--disable-ipv6],[turn off IPv6 support]))
-if test x"$enable_ipv6" != x"no"; then
-	AC_MSG_CHECKING([ipv6 stack type])
-	for i in inria kame linux-glibc linux-inet6 solaris toshiba v6d zeta cygwin TANDEM; do
-		case $i in
-		inria)
-			# http://www.kame.net/
-			AC_EGREP_CPP(yes, [
-#include <netinet/in.h>
-#ifdef IPV6_INRIA_VERSION
-yes
-#endif],
-				[ipv6type=$i;
-				AC_DEFINE(INET6, 1, [true if you have IPv6])
-				])
-			;;
-		kame)
-			# http://www.kame.net/
-			AC_EGREP_CPP(yes, [
-#include <netinet/in.h>
-#ifdef __KAME__
-yes
-#endif],
-				[ipv6type=$i;
-				AC_DEFINE(INET6, 1, [true if you have IPv6])])
-			;;
-		linux-glibc)
-			# http://www.v6.linux.or.jp/
-			AC_EGREP_CPP(yes, [
-#include <features.h>
-#if defined(__GLIBC__) && __GLIBC__ >= 2 && __GLIBC_MINOR__ >= 1
-yes
-#endif],
-				[ipv6type=$i;
-AC_DEFINE(INET6, 1, [true if you have IPv6])])
-			;;
-		linux-inet6)
-			# http://www.v6.linux.or.jp/
-			if test -d /usr/inet6 -o -f /usr/inet6/lib/libinet6.a; then
-				ipv6type=$i
-				ipv6lib=inet6
-				ipv6libdir=/usr/inet6/lib
-				ipv6trylibc=yes;
-				AC_DEFINE(INET6, 1, [true if you have IPv6])
-				CFLAGS="-I/usr/inet6/include $CFLAGS"
-			fi
-			;;
-		solaris)
-			# http://www.sun.com
-			AC_EGREP_CPP(yes, [
-#include <netinet/ip6.h>
-#ifdef __sun
-yes
-#endif],
-				[ipv6type=$i;
-				AC_DEFINE(INET6, 1, [true if you have IPv6])])
-			;;
-		toshiba)
-			AC_EGREP_CPP(yes, [
-#include <sys/param.h>
-#ifdef _TOSHIBA_INET6
-yes
-#endif],
-				[ipv6type=$i;
-				ipv6lib=inet6;
-				ipv6libdir=/usr/local/v6/lib;
-				AC_DEFINE(INET6, 1, [true if you have IPv6])])
-			;;
-		v6d)
-			AC_EGREP_CPP(yes, [
-#include </usr/local/v6/include/sys/v6config.h>
-#ifdef __V6D__
-yes
-#endif],
-				[ipv6type=$i;
-				ipv6lib=v6;
-				ipv6libdir=/usr/local/v6/lib;
-				AC_DEFINE(INET6, 1, [true if you have IPv6])])
-			;;
-		zeta)
-			AC_EGREP_CPP(yes, [
-#include <sys/param.h>
-#ifdef _ZETA_MINAMI_INET6
-yes
-#endif],
-				[ipv6type=$i;
-				ipv6lib=inet6;
-				ipv6libdir=/usr/local/v6/lib;
-				AC_DEFINE(INET6, 1, [true if you have IPv6])])
-			;;
-		cygwin)
-			AC_EGREP_CPP(yes, [
-#include <netinet/in.h>
-#ifdef _CYGWIN_IN6_H
-yes
-#endif],
-				[ipv6type=$i;
-				AC_DEFINE(INET6, 1, [true if you have IPv6])])
-			;;
-		TANDEM)
-			AC_EGREP_CPP(yes, [
-#include <netinet/ip6.h>
-#ifdef __TANDEM
-yes
-#endif],
-				[ipv6type=$i;
-				AC_DEFINE(INET6, 1, [true if you have IPv6])])
-			;;
-		esac
-		if test "$ipv6type" != "unknown"; then
-			break
-		fi
-	done
-	AC_MSG_RESULT($ipv6type)
+AS_HELP_STRING([--disable-ipv6],[disable to omit ipv6 support]),
+[ case "$enableval" in
+  no)
+	AC_MSG_RESULT(no)
+	;;
+  *)	AC_MSG_RESULT(yes)
+	AC_DEFINE(INET6, 1, [true if you have IPv6])
+	;;
+  esac ],
 
-	AC_SEARCH_LIBS(getaddrinfo, inet6)
-fi
+  AC_TRY_RUN([ /* AF_INET6 availability check */
+#include <stdlib.h>
+#include <sys/types.h>
+#include <sys/socket.h>
+main()
+{
+   if (socket(AF_INET6, SOCK_STREAM, 0) < 0)
+     exit(1);
+   else
+     exit(0);
+}
+],
+  AC_MSG_RESULT(yes)
+  AC_DEFINE(INET6, 1, [true if you have IPv6]),
+  AC_MSG_RESULT(no),
+  AC_MSG_RESULT(no)
+))
 
 dnl Do you want to disable use of locale functions
 AC_ARG_ENABLE([locale],
-	AS_HELP_STRING([--disable-locale],[disable locale features]))
+	AS_HELP_STRING([--disable-locale],[disable to omit locale features]))
 AH_TEMPLATE([CONFIG_LOCALE],
 [Undefine if you do not want locale features.  By default this is defined.])
 if test x"$enable_locale" != x"no"; then
@@ -486,23 +448,9 @@ case $host_os in
 	       * ) AC_MSG_RESULT(no);;
 esac
 
-AC_C_BIGENDIAN
-AC_HEADER_DIRENT
-AC_HEADER_TIME
-AC_HEADER_SYS_WAIT
-AC_CHECK_HEADERS(sys/fcntl.h sys/select.h fcntl.h sys/time.h sys/unistd.h \
-    unistd.h utime.h grp.h compat.h sys/param.h ctype.h sys/wait.h \
-    sys/ioctl.h sys/filio.h string.h stdlib.h sys/socket.h sys/mode.h \
-    sys/un.h sys/attr.h mcheck.h arpa/inet.h arpa/nameser.h locale.h \
-    netdb.h malloc.h float.h limits.h iconv.h libcharset.h langinfo.h \
-    sys/acl.h acl/libacl.h attr/xattr.h sys/xattr.h sys/extattr.h dl.h \
-    popt.h popt/popt.h linux/falloc.h netinet/in_systm.h netinet/ip.h \
-    zlib.h xxhash.h openssl/md4.h openssl/md5.h zstd.h lz4.h sys/file.h)
-AC_HEADER_MAJOR_FIXED
-
 AC_MSG_CHECKING([whether to enable use of openssl crypto library])
 AC_ARG_ENABLE([openssl],
-	AS_HELP_STRING([--disable-openssl],[disable openssl crypto library]))
+	AS_HELP_STRING([--disable-openssl],[disable to omit openssl crypto library]))
 AH_TEMPLATE([USE_OPENSSL],
 [Undefine if you do not want to use openssl crypto library.  By default this is defined.])
 if test x"$enable_openssl" != x"no"; then
@@ -523,7 +471,7 @@ fi
 
 AC_MSG_CHECKING([whether to enable xxhash checksum support])
 AC_ARG_ENABLE([xxhash],
-	AS_HELP_STRING([--disable-xxhash],[disable xxhash checksums]))
+	AS_HELP_STRING([--disable-xxhash],[disable to omit xxhash checksums]))
 AH_TEMPLATE([SUPPORT_XXHASH],
 [Undefine if you do not want xxhash checksums.  By default this is defined.])
 if test x"$enable_xxhash" != x"no"; then
@@ -544,7 +492,7 @@ fi
 
 AC_MSG_CHECKING([whether to enable zstd compression])
 AC_ARG_ENABLE([zstd],
-        AC_HELP_STRING([--disable-zstd], [disable zstd compression]))
+        AC_HELP_STRING([--disable-zstd], [disable to omit zstd compression]))
 AH_TEMPLATE([SUPPORT_ZSTD],
 [Undefine if you do not want zstd compression.  By default this is defined.])
 if test x"$enable_zstd" != x"no"; then
@@ -565,7 +513,7 @@ fi
 
 AC_MSG_CHECKING([whether to enable LZ4 compression])
 AC_ARG_ENABLE([lz4],
-        AC_HELP_STRING([--disable-lz4], [disable LZ4 compression]))
+        AC_HELP_STRING([--disable-lz4], [disable to omit LZ4 compression]))
 AH_TEMPLATE([SUPPORT_LZ4],
 [Undefine if you do not want LZ4 compression.  By default this is defined.])
 if test x"$enable_lz4" != x"no"; then
@@ -603,7 +551,9 @@ fi
 
 AC_CACHE_CHECK([if makedev takes 3 args],rsync_cv_MAKEDEV_TAKES_3_ARGS,[
 AC_RUN_IFELSE([AC_LANG_SOURCE([[
+#ifdef HAVE_SYS_TYPES_H
 #include <sys/types.h>
+#endif
 #ifdef MAJOR_IN_MKDEV
 #include <sys/mkdev.h>
 # if !defined makedev && (defined mkdev || defined _WIN32 || defined __WIN32__)
@@ -719,7 +669,9 @@ AC_SEARCH_LIBS(libiconv_open, iconv)
 AC_MSG_CHECKING([for iconv declaration])
 AC_CACHE_VAL(am_cv_proto_iconv, [
     AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[
+#if HAVE_STDLIB_H
 #include <stdlib.h>
+#endif
 #include <iconv.h>
 extern
 #ifdef __cplusplus
@@ -743,31 +695,47 @@ dnl AC_MSG_NOTICE([Looking in libraries: $LIBS])
 AC_REPLACE_FUNCS([inet_ntop inet_pton])
 
 AC_HAVE_TYPE([struct addrinfo], [#include <netdb.h>])
-AC_HAVE_TYPE([struct sockaddr_storage], [#include <sys/types.h>
-#include <sys/socket.h>])
+AC_HAVE_TYPE([struct sockaddr_storage], [
+#ifdef HAVE_SYS_TYPES_H
+#include <sys/types.h>
+#endif
+#ifdef HAVE_SYS_SOCKET_H
+#include <sys/socket.h>
+#endif])
 
 # Irix 6.5 has getaddrinfo but not the corresponding defines, so use
 #   builtin getaddrinfo if one of the defines don't exist
 AC_CACHE_CHECK([whether defines needed by getaddrinfo exist],
                rsync_cv_HAVE_GETADDR_DEFINES,[
 			AC_EGREP_CPP(yes, [
-			#include <sys/types.h>
-			#include <sys/socket.h>
-			#include <netdb.h>
-			#ifdef AI_PASSIVE
-			yes
-			#endif],
+#ifdef HAVE_SYS_TYPES_H
+#include <sys/types.h>
+#endif
+#ifdef HAVE_SYS_SOCKET_H
+#include <sys/socket.h>
+#endif
+#ifdef HAVE_NETDB_H
+#include <netdb.h>
+#endif
+#ifdef AI_PASSIVE
+yes
+#endif],
 			rsync_cv_HAVE_GETADDR_DEFINES=yes,
 			rsync_cv_HAVE_GETADDR_DEFINES=no)])
-AS_IF([test x"$rsync_cv_HAVE_GETADDR_DEFINES" = x"yes" -a x"$ac_cv_type_struct_addrinfo" = x"yes"],[
+AS_IF([test x"$rsync_cv_HAVE_GETADDR_DEFINES" = x"yes" && test x"$ac_cv_type_struct_addrinfo" = x"yes"],[
 	# Tru64 UNIX has getaddrinfo() but has it renamed in libc as
 	# something else so we must include <netdb.h> to get the
 	# redefinition.
 	AC_CHECK_FUNCS(getaddrinfo, ,
 		[AC_MSG_CHECKING([for getaddrinfo by including <netdb.h>])
-		AC_LINK_IFELSE([AC_LANG_PROGRAM([[#include <sys/types.h>
-		#include <sys/socket.h>
-		#include <netdb.h>]], [[getaddrinfo(NULL, NULL, NULL, NULL);]])],[AC_MSG_RESULT([yes])
+		AC_LINK_IFELSE([AC_LANG_PROGRAM([[
+#ifdef HAVE_SYS_TYPES_H
+#include <sys/types.h>
+#endif
+#ifdef HAVE_SYS_SOCKET_H
+#include <sys/socket.h>
+#endif
+#include <netdb.h>]], [[getaddrinfo(NULL, NULL, NULL, NULL);]])],[AC_MSG_RESULT([yes])
 			AC_DEFINE(HAVE_GETADDRINFO, 1,
 				[Define to 1 if you have the "getaddrinfo" function and required types.])],[AC_MSG_RESULT([no])
 			AC_LIBOBJ([getaddrinfo])])])
@@ -777,16 +745,24 @@ AC_CHECK_MEMBER([struct sockaddr.sa_len],
 		[ AC_DEFINE(HAVE_SOCKADDR_LEN, 1, [Do we have sockaddr.sa_len?]) ],
 		[],
 		[
+#ifdef HAVE_SYS_TYPES_H
 #include <sys/types.h>
+#endif
+#ifdef HAVE_SYS_SOCKET_H
 #include <sys/socket.h>
+#endif
 ])
 
 AC_CHECK_MEMBER([struct sockaddr_in.sin_len],
 		[ AC_DEFINE(HAVE_SOCKADDR_IN_LEN, 1, [Do we have sockaddr_in.sin_len?]) ],
 		[],
 		[
+#ifdef HAVE_SYS_TYPES_H
 #include <sys/types.h>
+#endif
+#ifdef HAVE_SYS_SOCKET_H
 #include <sys/socket.h>
+#endif
 #include <netinet/in.h>
 ])
 
@@ -794,8 +770,12 @@ AC_CHECK_MEMBER([struct sockaddr_un.sun_len],
 		[ AC_DEFINE(HAVE_SOCKADDR_UN_LEN, 1, [Do we have sockaddr_un.sun_len?]) ],
 		[],
 		[
+#ifdef HAVE_SYS_TYPES_H
 #include <sys/types.h>
+#endif
+#ifdef HAVE_SYS_SOCKET_H
 #include <sys/socket.h>
+#endif
 #include <netinet/in.h>
 ])
 
@@ -803,8 +783,12 @@ AC_CHECK_MEMBER([struct sockaddr_in6.sin6_scope_id],
 		[ AC_DEFINE(HAVE_SOCKADDR_IN6_SCOPE_ID, 1, [Do we have sockaddr_in6.sin6_scope_id?]) ],
 		[],
 		[
+#ifdef HAVE_SYS_TYPES_H
 #include <sys/types.h>
+#endif
+#ifdef HAVE_SYS_SOCKET_H
 #include <sys/socket.h>
+#endif
 #include <netinet/in.h>
 ])
 
@@ -848,11 +832,12 @@ AC_CHECK_FUNCS(waitpid wait4 getcwd chown chmod lchmod mknod mkfifo \
     fchmod fstat ftruncate strchr readlink link utime utimes lutimes strftime \
     chflags getattrlist mktime innetgr linkat \
     memmove lchown vsnprintf snprintf vasprintf asprintf setsid strpbrk \
-    strlcat strlcpy strtol mallinfo getgroups setgroups geteuid getegid \
+    strlcat strlcpy strtol mallinfo mallinfo2 getgroups setgroups geteuid getegid \
     setlocale setmode open64 lseek64 mkstemp64 mtrace va_copy __va_copy \
     seteuid strerror putenv iconv_open locale_charset nl_langinfo getxattr \
     extattr_get_link sigaction sigprocmask setattrlist getgrouplist \
-    initgroups utimensat posix_fallocate attropen setvbuf nanosleep usleep)
+    initgroups utimensat posix_fallocate attropen setvbuf nanosleep usleep \
+    setenv unsetenv)
 
 dnl cygwin iconv.h defines iconv_open as libiconv_open
 if test x"$ac_cv_func_iconv_open" != x"yes"; then
@@ -863,7 +848,9 @@ dnl Preallocation stuff (also fallocate, posix_fallocate function tests above):
 
 AC_CACHE_CHECK([for useable fallocate],rsync_cv_have_fallocate,[
 AC_LINK_IFELSE([AC_LANG_PROGRAM([[#include <fcntl.h>
-#include <sys/types.h>]], [[fallocate(0, 0, 0, 0);]])],[rsync_cv_have_fallocate=yes],[rsync_cv_have_fallocate=no])])
+#ifdef HAVE_SYS_TYPES_H
+#include <sys/types.h>
+#endif]], [[fallocate(0, 0, 0, 0);]])],[rsync_cv_have_fallocate=yes],[rsync_cv_have_fallocate=no])])
 if test x"$rsync_cv_have_fallocate" = x"yes"; then
     AC_DEFINE(HAVE_FALLOCATE, 1, [Define to 1 if you have the fallocate function and it compiles and links without error])
 fi
@@ -900,8 +887,12 @@ AC_PREPROC_IFELSE([AC_LANG_SOURCE([[
 
 AC_CACHE_CHECK([for SYS_fallocate],rsync_cv_have_sys_fallocate,[
 AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[#include <sys/syscall.h>
+#ifdef HAVE_UNISTD_H
 #include <unistd.h>
-#include <sys/types.h>]], [[syscall(SYS_fallocate, 0, 0, (loff_t)0, (loff_t)0);]])],[rsync_cv_have_sys_fallocate=yes],[rsync_cv_have_sys_fallocate=no])])
+#endif
+#ifdef HAVE_SYS_TYPES_H
+#include <sys/types.h>
+#endif]], [[syscall(SYS_fallocate, 0, 0, (loff_t)0, (loff_t)0);]])],[rsync_cv_have_sys_fallocate=yes],[rsync_cv_have_sys_fallocate=no])])
 if test x"$rsync_cv_have_sys_fallocate" = x"yes"; then
     AC_DEFINE(HAVE_SYS_FALLOCATE, 1, [Define to 1 if you have the SYS_fallocate syscall number])
 fi
@@ -928,7 +919,7 @@ if test $ac_cv_func_getpgrp = yes; then
 fi
 
 AC_ARG_ENABLE(iconv-open,
-    AS_HELP_STRING([--disable-iconv-open],[disable all use of iconv_open() function]),
+    AS_HELP_STRING([--disable-iconv-open],[disable to avoid all use of iconv_open()]),
     [], [enable_iconv_open=$ac_cv_func_iconv_open])
 
 if test x"$enable_iconv_open" != x"no"; then
@@ -936,7 +927,7 @@ if test x"$enable_iconv_open" != x"no"; then
 fi
 
 AC_ARG_ENABLE(iconv,
-    AS_HELP_STRING([--disable-iconv],[disable rsync's --iconv option]),
+    AS_HELP_STRING([--disable-iconv],[disable to omit the --iconv option]),
     [], [enable_iconv=$enable_iconv_open])
 AH_TEMPLATE([ICONV_OPTION],
 [Define if you want the --iconv option.  Specifying a value will set the
@@ -1020,8 +1011,12 @@ fi
 
 AC_CACHE_CHECK([for working socketpair],rsync_cv_HAVE_SOCKETPAIR,[
 AC_RUN_IFELSE([AC_LANG_SOURCE([[
+#ifdef HAVE_SYS_TYPES_H
 #include <sys/types.h>
+#endif
+#ifdef HAVE_SYS_SOCKET_H
 #include <sys/socket.h>
+#endif
 
 int main(void) {
        int fd[2];
@@ -1046,6 +1041,21 @@ elif test x"$ac_cv_header_popt_h" != x"yes"; then
     with_included_popt=yes
 fi
 
+if test x"$GCC" = x"yes"; then
+    if test x"$with_included_popt" != x"yes"; then
+	# Turn pedantic warnings into errors to ensure an array-init overflow is an error.
+	CFLAGS="$CFLAGS -pedantic-errors"
+    else
+	# Our internal popt code cannot be compiled with pedantic warnings as errors, so try to
+	# turn off pedantic warnings (which will not lose the error for array-init overflow).
+	# Older gcc versions don't understand -Wno-pedantic, so check if --help=warnings lists
+	# -Wpedantic and use that as a flag.
+	case `$CC --help=warnings 2>/dev/null | grep Wpedantic` in
+	    *-Wpedantic*) CFLAGS="$CFLAGS -pedantic-errors -Wno-pedantic" ;;
+	esac
+    fi
+fi
+
 AC_MSG_CHECKING([whether to use included libpopt])
 if test x"$with_included_popt" = x"yes"; then
     AC_MSG_RESULT($srcdir/popt)
@@ -1088,7 +1098,10 @@ if test x"$rsync_cv_SIGNED_CHAR_OK" = x"yes"; then
 fi
 
 AC_CACHE_CHECK([for broken readdir],rsync_cv_HAVE_BROKEN_READDIR,[
-AC_RUN_IFELSE([AC_LANG_SOURCE([[#include <sys/types.h>
+AC_RUN_IFELSE([AC_LANG_SOURCE([[
+#ifdef HAVE_SYS_TYPES_H
+#include <sys/types.h>
+#endif
 #include <dirent.h>
 int main(void) { struct dirent *di; DIR *d = opendir("."); di = readdir(d);
 if (di && di->d_name[-2] == '.' && di->d_name[-1] == 0 &&
@@ -1098,7 +1111,10 @@ if test x"$rsync_cv_HAVE_BROKEN_READDIR" = x"yes"; then
 fi
 
 AC_CACHE_CHECK([for utimbuf],rsync_cv_HAVE_STRUCT_UTIMBUF,[
-AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[#include <sys/types.h>
+AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[
+#ifdef HAVE_SYS_TYPES_H
+#include <sys/types.h>
+#endif
 #include <utime.h>]], [[struct utimbuf tbuf;  tbuf.actime = 0; tbuf.modtime = 1; return utime("foo.c",&tbuf);]])],[rsync_cv_HAVE_STRUCT_UTIMBUF=yes],[rsync_cv_HAVE_STRUCT_UTIMBUF=no])])
 if test x"$rsync_cv_HAVE_STRUCT_UTIMBUF" = x"yes"; then
     AC_DEFINE(HAVE_STRUCT_UTIMBUF, 1, [Define to 1 if you have the "struct utimbuf" type])
@@ -1106,17 +1122,23 @@ fi
 
 AC_CACHE_CHECK([if gettimeofday takes tz argument],rsync_cv_HAVE_GETTIMEOFDAY_TZ,[
 AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[#include <sys/time.h>
-#include <unistd.h>]], [[struct timeval tv; return gettimeofday(&tv, NULL);]])],[rsync_cv_HAVE_GETTIMEOFDAY_TZ=yes],[rsync_cv_HAVE_GETTIMEOFDAY_TZ=no])])
+#ifdef HAVE_UNISTD_H
+#include <unistd.h>
+#endif]], [[struct timeval tv; return gettimeofday(&tv, NULL);]])],[rsync_cv_HAVE_GETTIMEOFDAY_TZ=yes],[rsync_cv_HAVE_GETTIMEOFDAY_TZ=no])])
 if test x"$rsync_cv_HAVE_GETTIMEOFDAY_TZ" != x"no"; then
     AC_DEFINE(HAVE_GETTIMEOFDAY_TZ, 1, [Define to 1 if gettimeofday() takes a time-zone arg])
 fi
 
 AC_CACHE_CHECK([for C99 vsnprintf],rsync_cv_HAVE_C99_VSNPRINTF,[
 AC_RUN_IFELSE([AC_LANG_SOURCE([[
+#ifdef HAVE_SYS_TYPES_H
 #include <sys/types.h>
+#endif
 #include <stdarg.h>
 #include <stdio.h>
+#if HAVE_STDLIB_H
 #include <stdlib.h>
+#endif
 #include <string.h>
 void foo(const char *format, ...) {
        va_list ap;
@@ -1139,9 +1161,13 @@ fi
 
 AC_CACHE_CHECK([for secure mkstemp],rsync_cv_HAVE_SECURE_MKSTEMP,[
 AC_RUN_IFELSE([AC_LANG_SOURCE([[#include <stdlib.h>
+#ifdef HAVE_SYS_TYPES_H
 #include <sys/types.h>
+#endif
 #include <sys/stat.h>
+#ifdef HAVE_UNISTD_H
 #include <unistd.h>
+#endif
 int main(void) {
   struct stat st;
   char tpl[20]="/tmp/test.XXXXXX";
@@ -1230,6 +1256,9 @@ AC_SUBST(OBJ_RESTORE)
 AC_SUBST(CC_SHOBJ_FLAG)
 AC_SUBST(BUILD_POPT)
 AC_SUBST(BUILD_ZLIB)
+AC_SUBST(MAKE_RRSYNC)
+AC_SUBST(MAKE_RRSYNC_1)
+AC_SUBST(GEN_RRSYNC)
 AC_SUBST(MAKE_MAN)
 
 AC_CHECK_FUNCS(_acl __acl _facl __facl)
@@ -1238,7 +1267,7 @@ AC_CHECK_FUNCS(_acl __acl _facl __facl)
 
 AC_MSG_CHECKING([whether to support ACLs])
 AC_ARG_ENABLE(acl-support,
-	AS_HELP_STRING([--disable-acl-support],[disable ACL support]))
+	AS_HELP_STRING([--disable-acl-support],[disable to omit ACL support]))
 
 if test x"$enable_acl_support" = x"no"; then
     AC_MSG_RESULT(no)
@@ -1284,16 +1313,26 @@ else
 	AC_MSG_RESULT(running tests:)
 	AC_CHECK_LIB(acl,acl_get_file)
 	AC_CACHE_CHECK([for ACL support],samba_cv_HAVE_POSIX_ACLS,[
-	    AC_LINK_IFELSE([AC_LANG_PROGRAM([[#include <sys/types.h>
-#include <sys/acl.h>]], [[ acl_t acl; int entry_id; acl_entry_t *entry_p; return acl_get_entry( acl, entry_id, entry_p);]])],[samba_cv_HAVE_POSIX_ACLS=yes],[samba_cv_HAVE_POSIX_ACLS=no])])
+	    AC_LINK_IFELSE([AC_LANG_PROGRAM([[
+#ifdef HAVE_SYS_TYPES_H
+#include <sys/types.h>
+#endif
+#ifdef HAVE_SYS_ACL_H
+#include <sys/acl.h>
+#endif]], [[ acl_t acl; int entry_id; acl_entry_t *entry_p; return acl_get_entry( acl, entry_id, entry_p);]])],[samba_cv_HAVE_POSIX_ACLS=yes],[samba_cv_HAVE_POSIX_ACLS=no])])
 	AC_MSG_CHECKING(ACL test results)
 	if test x"$samba_cv_HAVE_POSIX_ACLS" = x"yes"; then
 	    AC_MSG_RESULT(Using posix ACLs)
 	    AC_DEFINE(HAVE_POSIX_ACLS, 1, [true if you have posix ACLs])
 	    AC_DEFINE(SUPPORT_ACLS, 1)
 	    AC_CACHE_CHECK([for acl_get_perm_np],samba_cv_HAVE_ACL_GET_PERM_NP,[
-		AC_LINK_IFELSE([AC_LANG_PROGRAM([[#include <sys/types.h>
-#include <sys/acl.h>]], [[ acl_permset_t permset_d; acl_perm_t perm; return acl_get_perm_np( permset_d, perm);]])],[samba_cv_HAVE_ACL_GET_PERM_NP=yes],[samba_cv_HAVE_ACL_GET_PERM_NP=no])])
+		AC_LINK_IFELSE([AC_LANG_PROGRAM([[
+#ifdef HAVE_SYS_TYPES_H
+#include <sys/types.h>
+#endif
+#ifdef HAVE_SYS_ACL_H
+#include <sys/acl.h>
+#endif]], [[ acl_permset_t permset_d; acl_perm_t perm; return acl_get_perm_np( permset_d, perm);]])],[samba_cv_HAVE_ACL_GET_PERM_NP=yes],[samba_cv_HAVE_ACL_GET_PERM_NP=no])])
 	    if test x"$samba_cv_HAVE_ACL_GET_PERM_NP" = x"yes"; then
 		AC_DEFINE(HAVE_ACL_GET_PERM_NP, 1, [true if you have acl_get_perm_np])
 	    fi
@@ -1312,7 +1351,7 @@ fi
 # check for extended attribute support
 AC_MSG_CHECKING(whether to support extended attributes)
 AC_ARG_ENABLE(xattr-support,
-    AS_HELP_STRING([--disable-xattr-support],[disable extended attributes]),
+    AS_HELP_STRING([--disable-xattr-support],[disable to omit extended attributes]),
     [], [case "$ac_cv_func_getxattr$ac_cv_func_extattr_get_link$ac_cv_func_attropen" in
 	*yes*) enable_xattr_support=maybe ;;
 	*) enable_xattr_support=no ;;
@@ -1358,7 +1397,7 @@ else
     esac
 fi
 
-if test x"$enable_acl_support" = x"no" -o x"$enable_xattr_support" = x"no" -o x"$enable_iconv" = x"no"; then
+if test x"$enable_acl_support" = x"no" || test x"$enable_xattr_support" = x"no" || test x"$enable_iconv" = x"no"; then
     AC_MSG_CHECKING([whether $CC supports -Wno-unused-parameter])
     OLD_CFLAGS="$CFLAGS"
     CFLAGS="$CFLAGS -Wno-unused-parameter"

flist.c

@@ -4,7 +4,7 @@
  * Copyright (C) 1996 Andrew Tridgell
  * Copyright (C) 1996 Paul Mackerras
  * Copyright (C) 2001, 2002 Martin Pool <mbp@samba.org>
- * Copyright (C) 2002-2020 Wayne Davison
+ * Copyright (C) 2002-2022 Wayne Davison
  *
  * This program is free software; you can redistribute it and/or modify
  * it under the terms of the GNU General Public License as published by
@@ -295,6 +295,8 @@ static void flist_expand(struct file_list *flist, int extra)
 		flist->malloced = FLIST_START;
 	else if (flist->malloced >= FLIST_LINEAR)
 		flist->malloced += FLIST_LINEAR;
+	else if (flist->malloced < FLIST_START_LARGE/16)
+		flist->malloced *= 4;
 	else
 		flist->malloced *= 2;
 
@@ -305,7 +307,7 @@ static void flist_expand(struct file_list *flist, int extra)
 
 	new_ptr = realloc_array(flist->files, struct file_struct *, flist->malloced);
 
-	if (DEBUG_GTE(FLIST, 1) && flist->malloced != FLIST_START) {
+	if (DEBUG_GTE(FLIST, 1) && flist->files) {
 		rprintf(FCLIENT, "[%s] expand file_list pointer array to %s bytes, did%s move\n",
 		    who_am_i(),
 		    big_num(sizeof flist->files[0] * flist->malloced),
@@ -1438,7 +1440,7 @@ struct file_struct *make_file(const char *fname, struct file_list *flist,
 		F_ATIME(file) = st.st_atime;
 #ifdef SUPPORT_CRTIMES
 	if (crtimes_ndx)
-		F_CRTIME(file) = get_create_time(fname);
+		F_CRTIME(file) = get_create_time(fname, &st);
 #endif
 
 	if (basename != thisname)
@@ -2186,8 +2188,10 @@ struct file_list *send_file_list(int f, int argc, char *argv[])
 #endif
 
 	flist = cur_flist = flist_new(0, "send_file_list");
+	flist_expand(flist, FLIST_START_LARGE);
 	if (inc_recurse) {
 		dir_flist = flist_new(FLIST_TEMP, "send_file_list");
+		flist_expand(dir_flist, FLIST_START_LARGE);
 		flags |= FLAG_DIVERT_DIRS;
 	} else
 		dir_flist = cur_flist;
@@ -2541,10 +2545,13 @@ struct file_list *recv_file_list(int f, int dir_ndx)
 #endif
 
 	flist = flist_new(0, "recv_file_list");
+	flist_expand(flist, FLIST_START_LARGE);
 
 	if (inc_recurse) {
-		if (flist->ndx_start == 1)
+		if (flist->ndx_start == 1) {
 			dir_flist = flist_new(FLIST_TEMP, "recv_file_list");
+			flist_expand(dir_flist, FLIST_START_LARGE);
+		}
 		dstart = dir_flist->used;
 	} else {
 		dir_flist = flist;

generator.c

@@ -4,7 +4,7 @@
  * Copyright (C) 1996-2000 Andrew Tridgell
  * Copyright (C) 1996 Paul Mackerras
  * Copyright (C) 2002 Martin Pool <mbp@samba.org>
- * Copyright (C) 2003-2020 Wayne Davison
+ * Copyright (C) 2003-2022 Wayne Davison
  *
  * This program is free software; you can redistribute it and/or modify
  * it under the terms of the GNU General Public License as published by
@@ -44,7 +44,9 @@ extern int preserve_specials;
 extern int preserve_hard_links;
 extern int preserve_executability;
 extern int preserve_perms;
-extern int preserve_times;
+extern int preserve_mtimes;
+extern int omit_dir_times;
+extern int omit_link_times;
 extern int delete_mode;
 extern int delete_before;
 extern int delete_during;
@@ -112,10 +114,6 @@ static int need_retouch_dir_times;
 static int need_retouch_dir_perms;
 static const char *solo_file = NULL;
 
-enum nonregtype {
-	TYPE_DIR, TYPE_SPECIAL, TYPE_DEVICE, TYPE_SYMLINK
-};
-
 /* Forward declarations. */
 #ifdef SUPPORT_HARD_LINKS
 static void handle_skipped_hlink(struct file_struct *file, int itemizing,
@@ -186,7 +184,8 @@ static int remember_delete(struct file_struct *file, const char *fname, int flag
 static int read_delay_line(char *buf, int *flags_p)
 {
 	static int read_pos = 0;
-	int j, len, mode;
+	unsigned int mode;
+	int j, len;
 	char *bp, *past_space;
 
 	while (1) {
@@ -270,7 +269,7 @@ static void do_delayed_deletions(char *delbuf)
  * MAXPATHLEN buffer with the name of the directory in it (the functions we
  * call will append names onto the end, but the old dir value will be restored
  * on exit). */
-static void delete_in_dir(char *fbuf, struct file_struct *file, dev_t *fs_dev)
+static void delete_in_dir(char *fbuf, struct file_struct *file, dev_t fs_dev)
 {
 	static int already_warned = 0;
 	static struct hashtable *dev_tbl;
@@ -305,12 +304,12 @@ static void delete_in_dir(char *fbuf, struct file_struct *file, dev_t *fs_dev)
 		if (!dev_tbl)
 			dev_tbl = hashtable_create(16, HT_KEY64);
 		if (file->flags & FLAG_TOP_DIR) {
-			hashtable_find(dev_tbl, *fs_dev+1, "");
-			filesystem_dev = *fs_dev;
-		} else if (filesystem_dev != *fs_dev) {
-			if (!hashtable_find(dev_tbl, *fs_dev+1, NULL))
+			hashtable_find(dev_tbl, fs_dev+1, "");
+			filesystem_dev = fs_dev;
+		} else if (filesystem_dev != fs_dev) {
+			if (!hashtable_find(dev_tbl, fs_dev+1, NULL))
 				return;
-			filesystem_dev = *fs_dev; /* it's a prior top-dir dev */
+			filesystem_dev = fs_dev; /* it's a prior top-dir dev */
 		}
 	}
 
@@ -379,9 +378,9 @@ static void do_delete_pass(void)
 		 || !S_ISDIR(st.st_mode))
 			continue;
 
-		delete_in_dir(fbuf, file, &st.st_dev);
+		delete_in_dir(fbuf, file, st.st_dev);
 	}
-	delete_in_dir(NULL, NULL, &dev_zero);
+	delete_in_dir(NULL, NULL, dev_zero);
 
 	if (INFO_GTE(FLIST, 2) && !am_server)
 		rprintf(FINFO, "                    \r");
@@ -402,7 +401,7 @@ static inline int any_time_differs(stat_x *sxp, struct file_struct *file, UNUSED
 #ifdef SUPPORT_CRTIMES
 	if (!differs && crtimes_ndx) {
 		if (sxp->crtime == 0)
-			sxp->crtime = get_create_time(fname);
+			sxp->crtime = get_create_time(fname, &sxp->st);
 		differs = !same_time(sxp->crtime, 0, F_CRTIME(file), 0);
 	}
 #endif
@@ -463,7 +462,7 @@ int unchanged_attrs(const char *fname, struct file_struct *file, stat_x *sxp)
 {
 	if (S_ISLNK(file->mode)) {
 #ifdef CAN_SET_SYMLINK_TIMES
-		if (preserve_times & PRESERVE_LINK_TIMES && any_time_differs(sxp, file, fname))
+		if (preserve_mtimes && !omit_link_times && any_time_differs(sxp, file, fname))
 			return 0;
 #endif
 #ifdef CAN_CHMOD_SYMLINK
@@ -483,7 +482,7 @@ int unchanged_attrs(const char *fname, struct file_struct *file, stat_x *sxp)
 			return 0;
 #endif
 	} else {
-		if (preserve_times && any_time_differs(sxp, file, fname))
+		if (preserve_mtimes && any_time_differs(sxp, file, fname))
 			return 0;
 		if (perms_differ(file, sxp))
 			return 0;
@@ -507,9 +506,9 @@ void itemize(const char *fnamecmp, struct file_struct *file, int ndx, int statre
 	     const char *xname)
 {
 	if (statret >= 0) { /* A from-dest-dir statret can == 1! */
-		int keep_time = !preserve_times ? 0
-		    : S_ISDIR(file->mode) ? preserve_times & PRESERVE_DIR_TIMES
-		    : S_ISLNK(file->mode) ? preserve_times & PRESERVE_LINK_TIMES
+		int keep_time = !preserve_mtimes ? 0
+		    : S_ISDIR(file->mode) ? !omit_dir_times
+		    : S_ISLNK(file->mode) ? !omit_link_times
 		    : 1;
 
 		if (S_ISREG(file->mode) && F_LENGTH(file) != sxp->st.st_size)
@@ -528,12 +527,12 @@ void itemize(const char *fnamecmp, struct file_struct *file, int ndx, int statre
 #ifdef SUPPORT_CRTIMES
 		if (crtimes_ndx) {
 			if (sxp->crtime == 0)
-				sxp->crtime = get_create_time(fnamecmp);
+				sxp->crtime = get_create_time(fnamecmp, &sxp->st);
 			if (!same_time(sxp->crtime, 0, F_CRTIME(file), 0))
 				iflags |= ITEM_REPORT_CRTIME;
 		}
 #endif
-#if !defined HAVE_LCHMOD && !defined HAVE_SETATTRLIST
+#ifndef CAN_CHMOD_SYMLINK
 		if (S_ISLNK(file->mode)) {
 			;
 		} else
@@ -599,30 +598,77 @@ void itemize(const char *fnamecmp, struct file_struct *file, int ndx, int statre
 	}
 }
 
-
-/* Perform our quick-check heuristic for determining if a file is unchanged. */
-int unchanged_file(char *fn, struct file_struct *file, STRUCT_STAT *st)
+static enum filetype get_file_type(mode_t mode)
 {
-	if (st->st_size != F_LENGTH(file))
-		return 0;
-
-	/* if always checksum is set then we use the checksum instead
-	   of the file time to determine whether to sync */
-	if (always_checksum > 0 && S_ISREG(st->st_mode)) {
-		char sum[MAX_DIGEST_LEN];
-		file_checksum(fn, st, sum);
-		return memcmp(sum, F_SUM(file), flist_csum_len) == 0;
-	}
-
-	if (size_only > 0)
-		return 1;
-
-	if (ignore_times)
-		return 0;
-
-	return !mtime_differs(st, file);
+	if (S_ISREG(mode))
+		return FT_REG;
+	if (S_ISLNK(mode))
+		return FT_SYMLINK;
+	if (S_ISDIR(mode))
+		return FT_DIR;
+	if (IS_SPECIAL(mode))
+		return FT_SPECIAL;
+	if (IS_DEVICE(mode))
+		return FT_DEVICE;
+	return FT_UNSUPPORTED;
 }
 
+/* Perform our quick-check heuristic for determining if a file is unchanged. */
+int quick_check_ok(enum filetype ftype, const char *fn, struct file_struct *file, STRUCT_STAT *st)
+{
+	switch (ftype) {
+	  case FT_REG:
+		if (st->st_size != F_LENGTH(file))
+			return 0;
+
+		/* If always_checksum is set then we use the checksum instead
+		 * of the file mtime to determine whether to sync. */
+		if (always_checksum > 0) {
+			char sum[MAX_DIGEST_LEN];
+			file_checksum(fn, st, sum);
+			return memcmp(sum, F_SUM(file), flist_csum_len) == 0;
+		}
+
+		if (size_only > 0)
+			return 1;
+
+		if (ignore_times)
+			return 0;
+
+		if (mtime_differs(st, file))
+			return 0;
+		break;
+	  case FT_DIR:
+		break;
+	  case FT_SYMLINK: {
+#ifdef SUPPORT_LINKS
+		char lnk[MAXPATHLEN];
+		int len = do_readlink(fn, lnk, MAXPATHLEN-1);
+		if (len <= 0)
+			return 0;
+		lnk[len] = '\0';
+		if (strcmp(lnk, F_SYMLINK(file)) != 0)
+			return 0;
+		break;
+#else
+		return -1;
+#endif
+	  }
+	  case FT_SPECIAL:
+		if (!BITS_EQUAL(file->mode, st->st_mode, _S_IFMT))
+			return 0;
+		break;
+	  case FT_DEVICE: {
+		uint32 *devp = F_RDEV_P(file);
+		if (st->st_rdev != MAKEDEV(DEV_MAJOR(devp), DEV_MINOR(devp)))
+			return 0;
+		break;
+	  }
+	  case FT_UNSUPPORTED:
+		return -1;
+	}
+	return 1;
+}
 
 /*
  * set (initialize) the size entries in the per-file sum_struct
@@ -907,7 +953,7 @@ static int try_dests_reg(struct file_struct *file, char *fname, int ndx,
 			best_match = j;
 			match_level = 1;
 		}
-		if (!unchanged_file(cmpbuf, file, &sxp->st))
+		if (!quick_check_ok(FT_REG, cmpbuf, file, &sxp->st))
 			continue;
 		if (match_level == 1) {
 			best_match = j;
@@ -1006,29 +1052,14 @@ static int try_dests_non(struct file_struct *file, char *fname, int ndx,
 {
 	int best_match = -1;
 	int match_level = 0;
-	enum nonregtype type;
-	uint32 *devp;
-#ifdef SUPPORT_LINKS
-	char lnk[MAXPATHLEN];
-	int len;
-#endif
+	enum filetype ftype = get_file_type(file->mode);
 	int j = 0;
 
 #ifndef SUPPORT_LINKS
-	if (S_ISLNK(file->mode))
+	if (ftype == FT_SYMLINK)
 		return -1;
 #endif
-	if (S_ISDIR(file->mode)) {
-		type = TYPE_DIR;
-	} else if (IS_SPECIAL(file->mode))
-		type = TYPE_SPECIAL;
-	else if (IS_DEVICE(file->mode))
-		type = TYPE_DEVICE;
-#ifdef SUPPORT_LINKS
-	else if (S_ISLNK(file->mode))
-		type = TYPE_SYMLINK;
-#endif
-	else {
+	if (ftype == FT_REG || ftype == FT_UNSUPPORTED) {
 		rprintf(FERROR,
 			"internal: try_dests_non() called with invalid mode (%o)\n",
 			(int)file->mode);
@@ -1039,53 +1070,14 @@ static int try_dests_non(struct file_struct *file, char *fname, int ndx,
 		pathjoin(cmpbuf, MAXPATHLEN, basis_dir[j], fname);
 		if (link_stat(cmpbuf, &sxp->st, 0) < 0)
 			continue;
-		switch (type) {
-		case TYPE_DIR:
-			if (!S_ISDIR(sxp->st.st_mode))
-				continue;
-			break;
-		case TYPE_SPECIAL:
-			if (!IS_SPECIAL(sxp->st.st_mode))
-				continue;
-			break;
-		case TYPE_DEVICE:
-			if (!IS_DEVICE(sxp->st.st_mode))
-				continue;
-			break;
-		case TYPE_SYMLINK:
-#ifdef SUPPORT_LINKS
-			if (!S_ISLNK(sxp->st.st_mode))
-				continue;
-			break;
-#else
-			return -1;
-#endif
-		}
+		if (ftype != get_file_type(sxp->st.st_mode))
+			continue;
 		if (match_level < 1) {
 			match_level = 1;
 			best_match = j;
 		}
-		switch (type) {
-		case TYPE_DIR:
-		case TYPE_SPECIAL:
-			break;
-		case TYPE_DEVICE:
-			devp = F_RDEV_P(file);
-			if (sxp->st.st_rdev != MAKEDEV(DEV_MAJOR(devp), DEV_MINOR(devp)))
-				continue;
-			break;
-		case TYPE_SYMLINK:
-#ifdef SUPPORT_LINKS
-			if ((len = do_readlink(cmpbuf, lnk, MAXPATHLEN-1)) <= 0)
-				continue;
-			lnk[len] = '\0';
-			if (strcmp(lnk, F_SYMLINK(file)) != 0)
-				continue;
-			break;
-#else
-			return -1;
-#endif
-		}
+		if (!quick_check_ok(ftype, cmpbuf, file, &sxp->st))
+			continue;
 		if (match_level < 2) {
 			match_level = 2;
 			best_match = j;
@@ -1130,14 +1122,14 @@ static int try_dests_non(struct file_struct *file, char *fname, int ndx,
 			match_level = 2;
 		if (itemizing && stdout_format_has_i
 		 && (INFO_GTE(NAME, 2) || stdout_format_has_i > 1)) {
-			int chg = alt_dest_type == COMPARE_DEST && type != TYPE_DIR ? 0
+			int chg = alt_dest_type == COMPARE_DEST && ftype != FT_DIR ? 0
 			    : ITEM_LOCAL_CHANGE + (match_level == 3 ? ITEM_XNAME_FOLLOWS : 0);
 			char *lp = match_level == 3 ? "" : NULL;
 			itemize(cmpbuf, file, ndx, 0, sxp, chg + ITEM_MATCHED, 0, lp);
 		}
 		if (INFO_GTE(NAME, 2) && maybe_ATTRS_REPORT) {
 			rprintf(FCLIENT, "%s%s is uptodate\n",
-				fname, type == TYPE_DIR ? "/" : "");
+				fname, ftype == FT_DIR ? "/" : "");
 		}
 		return -2;
 	}
@@ -1231,7 +1223,8 @@ static void recv_generator(char *fname, struct file_struct *file, int ndx,
 	char fnamecmpbuf[MAXPATHLEN];
 	uchar fnamecmp_type;
 	int del_opts = delete_mode || force_delete ? DEL_RECURSE : 0;
-	int is_dir = !S_ISDIR(file->mode) ? 0
+	enum filetype stype, ftype = get_file_type(file->mode);
+	int is_dir = ftype != FT_DIR ? 0
 		   : inc_recurse && ndx != cur_flist->ndx_start - 1 ? -1
 		   : 1;
 
@@ -1278,7 +1271,6 @@ static void recv_generator(char *fname, struct file_struct *file, int ndx,
 			return;
 		}
 	}
-	sx.crtime = 0;
 
 	if (dry_run > 1 || (dry_missing_dir && is_below(file, dry_missing_dir))) {
 		int i;
@@ -1380,10 +1372,27 @@ static void recv_generator(char *fname, struct file_struct *file, int ndx,
 	 && !am_root && sx.st.st_uid == our_uid)
 		del_opts |= DEL_NO_UID_WRITE;
 
+	if (statret == 0)
+		stype = get_file_type(sx.st.st_mode);
+	else
+		stype = FT_UNSUPPORTED;
+
 	if (ignore_existing > 0 && statret == 0
-	 && (!is_dir || !S_ISDIR(sx.st.st_mode))) {
-		if (INFO_GTE(SKIP, 1) && is_dir >= 0)
-			rprintf(FINFO, "%s exists\n", fname);
+	 && (!is_dir || stype != FT_DIR)) {
+		if (INFO_GTE(SKIP, 1) && is_dir >= 0) {
+			const char *suf = "";
+			if (INFO_GTE(SKIP, 2)) {
+				if (ftype != stype)
+					suf = " (type change)";
+				else if (!quick_check_ok(ftype, fname, file, &sx.st))
+					suf = always_checksum ? " (sum change)" : " (file change)";
+				else if (!unchanged_attrs(fname, file, &sx))
+					suf = " (attr change)";
+				else
+					suf = " (uptodate)";
+			}
+			rprintf(FINFO, "%s exists%s\n", fname, suf);
+		}
 #ifdef SUPPORT_HARD_LINKS
 		if (F_IS_HLINKED(file))
 			handle_skipped_hlink(file, itemizing, code, f_out);
@@ -1404,7 +1413,7 @@ static void recv_generator(char *fname, struct file_struct *file, int ndx,
 		} else
 			added_perms = 0;
 		if (is_dir < 0) {
-			if (!(preserve_times & PRESERVE_DIR_TIMES))
+			if (!preserve_mtimes || omit_dir_times)
 				goto cleanup;
 			/* In inc_recurse mode we want to make sure any missing
 			 * directories get created while we're still processing
@@ -1412,7 +1421,7 @@ static void recv_generator(char *fname, struct file_struct *file, int ndx,
 			 * dir's mtime right away).  We will handle the dir in
 			 * full later (right before we handle its contents). */
 			if (statret == 0
-			 && (S_ISDIR(sx.st.st_mode)
+			 && (stype == FT_DIR
 			  || delete_item(fname, sx.st.st_mode, del_opts | DEL_FOR_DIR) != 0))
 				goto cleanup; /* Any errors get reported later. */
 			if (do_mkdir(fname, (file->mode|added_perms) & 0700) == 0)
@@ -1424,7 +1433,7 @@ static void recv_generator(char *fname, struct file_struct *file, int ndx,
 		 * file of that name and it is *not* a directory, then
 		 * we need to delete it.  If it doesn't exist, then
 		 * (perhaps recursively) create it. */
-		if (statret == 0 && !S_ISDIR(sx.st.st_mode)) {
+		if (statret == 0 && stype != FT_DIR) {
 			if (delete_item(fname, sx.st.st_mode, del_opts | DEL_FOR_DIR) != 0)
 				goto skipping_dir_contents;
 			statret = -1;
@@ -1508,7 +1517,7 @@ static void recv_generator(char *fname, struct file_struct *file, int ndx,
 		else if (delete_during && f_out != -1 && !phase
 		    && !(file->flags & FLAG_MISSING_DIR)) {
 			if (file->flags & FLAG_CONTENT_DIR)
-				delete_in_dir(fname, file, &real_sx.st.st_dev);
+				delete_in_dir(fname, file, real_sx.st.st_dev);
 			else
 				change_local_filter_dir(fname, strlen(fname), F_DEPTH(file));
 		}
@@ -1519,7 +1528,7 @@ static void recv_generator(char *fname, struct file_struct *file, int ndx,
 	/* If we're not preserving permissions, change the file-list's
 	 * mode based on the local permissions and some heuristics. */
 	if (!preserve_perms) {
-		int exists = statret == 0 && !S_ISDIR(sx.st.st_mode);
+		int exists = statret == 0 && stype != FT_DIR;
 		file->mode = dest_mode(file->mode, sx.st.st_mode, dflt_perms, exists);
 	}
 
@@ -1529,7 +1538,7 @@ static void recv_generator(char *fname, struct file_struct *file, int ndx,
 		goto cleanup;
 #endif
 
-	if (preserve_links && S_ISLNK(file->mode)) {
+	if (preserve_links && ftype == FT_SYMLINK) {
 #ifdef SUPPORT_LINKS
 		const char *sl = F_SYMLINK(file);
 		if (safe_symlinks && unsafe_symlink(sl, fname)) {
@@ -1546,12 +1555,7 @@ static void recv_generator(char *fname, struct file_struct *file, int ndx,
 			goto cleanup;
 		}
 		if (statret == 0) {
-			char lnk[MAXPATHLEN];
-			int len;
-
-			if (S_ISLNK(sx.st.st_mode)
-			 && (len = do_readlink(fname, lnk, MAXPATHLEN-1)) > 0
-			 && strncmp(lnk, sl, len) == 0 && sl[len] == '\0') {
+			if (stype == FT_SYMLINK && quick_check_ok(stype, fname, file, &sx.st)) {
 				/* The link is pointing to the right place. */
 				set_file_attrs(fname, file, &sx, NULL, maybe_ATTRS_REPORT);
 				if (itemizing)
@@ -1584,7 +1588,7 @@ static void recv_generator(char *fname, struct file_struct *file, int ndx,
 		if (atomic_create(file, fname, sl, NULL, MAKEDEV(0, 0), &sx, statret == 0 ? DEL_FOR_SYMLINK : 0)) {
 			set_file_attrs(fname, file, NULL, NULL, 0);
 			if (itemizing) {
-				if (statret == 0 && !S_ISLNK(sx.st.st_mode))
+				if (statret == 0 && stype != FT_SYMLINK)
 					statret = -1;
 				itemize(fnamecmp, file, ndx, statret, &sx,
 					ITEM_LOCAL_CHANGE|ITEM_REPORT_CHANGE, 0, NULL);
@@ -1605,28 +1609,22 @@ static void recv_generator(char *fname, struct file_struct *file, int ndx,
 		goto cleanup;
 	}
 
-	if ((am_root && preserve_devices && IS_DEVICE(file->mode))
-	 || (preserve_specials && IS_SPECIAL(file->mode))) {
+	if ((am_root && preserve_devices && ftype == FT_DEVICE)
+	 || (preserve_specials && ftype == FT_SPECIAL)) {
 		dev_t rdev;
-		int del_for_flag = 0;
-		if (IS_DEVICE(file->mode)) {
+		int del_for_flag;
+		if (ftype == FT_DEVICE) {
 			uint32 *devp = F_RDEV_P(file);
 			rdev = MAKEDEV(DEV_MAJOR(devp), DEV_MINOR(devp));
-		} else
+			del_for_flag = DEL_FOR_DEVICE;
+		} else {
 			rdev = 0;
+			del_for_flag = DEL_FOR_SPECIAL;
+		}
 		if (statret == 0) {
-			if (IS_DEVICE(file->mode)) {
-				if (!IS_DEVICE(sx.st.st_mode))
-					statret = -1;
-				del_for_flag = DEL_FOR_DEVICE;
-			} else {
-				if (!IS_SPECIAL(sx.st.st_mode))
-					statret = -1;
-				del_for_flag = DEL_FOR_SPECIAL;
-			}
-			if (statret == 0
-			 && BITS_EQUAL(sx.st.st_mode, file->mode, _S_IFMT)
-			 && (IS_SPECIAL(sx.st.st_mode) || sx.st.st_rdev == rdev)) {
+			if (ftype != stype)
+				statret = -1;
+			else if (quick_check_ok(ftype, fname, file, &sx.st)) {
 				/* The device or special file is identical. */
 				set_file_attrs(fname, file, &sx, NULL, maybe_ATTRS_REPORT);
 				if (itemizing)
@@ -1679,10 +1677,12 @@ static void recv_generator(char *fname, struct file_struct *file, int ndx,
 		goto cleanup;
 	}
 
-	if (!S_ISREG(file->mode)) {
-		if (solo_file)
-			fname = f_name(file, NULL);
-		rprintf(FINFO, "skipping non-regular file \"%s\"\n", fname);
+	if (ftype != FT_REG) {
+		if (INFO_GTE(NONREG, 1)) {
+			if (solo_file)
+				fname = f_name(file, NULL);
+			rprintf(FINFO, "skipping non-regular file \"%s\"\n", fname);
+		}
 		goto cleanup;
 	}
 
@@ -1703,7 +1703,7 @@ static void recv_generator(char *fname, struct file_struct *file, int ndx,
 		goto cleanup;
 	}
 
-	if (update_only > 0 && statret == 0 && file->modtime - sx.st.st_mtime <= modify_window) {
+	if (update_only > 0 && statret == 0 && file->modtime - sx.st.st_mtime < modify_window) {
 		if (INFO_GTE(SKIP, 1))
 			rprintf(FINFO, "%s is newer\n", fname);
 #ifdef SUPPORT_HARD_LINKS
@@ -1715,7 +1715,7 @@ static void recv_generator(char *fname, struct file_struct *file, int ndx,
 
 	fnamecmp_type = FNAMECMP_FNAME;
 
-	if (statret == 0 && !(S_ISREG(sx.st.st_mode) || (write_devices && IS_DEVICE(sx.st.st_mode)))) {
+	if (statret == 0 && !(stype == FT_REG || (write_devices && stype == FT_DEVICE))) {
 		if (delete_item(fname, sx.st.st_mode, del_opts | DEL_FOR_FILE) != 0)
 			goto cleanup;
 		statret = -1;
@@ -1749,7 +1749,7 @@ static void recv_generator(char *fname, struct file_struct *file, int ndx,
 		partialptr = NULL;
 
 	if (statret != 0 && fuzzy_basis) {
-		if (need_fuzzy_dirlist && S_ISREG(file->mode)) {
+		if (need_fuzzy_dirlist) {
 			const char *dn = file->dirname ? file->dirname : ".";
 			int i;
 			strlcpy(fnamecmpbuf, dn, sizeof fnamecmpbuf);
@@ -1797,7 +1797,7 @@ static void recv_generator(char *fname, struct file_struct *file, int ndx,
 		;
 	else if (fnamecmp_type >= FNAMECMP_FUZZY)
 		;
-	else if (unchanged_file(fnamecmp, file, &sx.st)) {
+	else if (quick_check_ok(FT_REG, fnamecmp, file, &sx.st)) {
 		if (partialptr) {
 			do_unlink(partialptr);
 			handle_partial_dir(partialptr, PDIR_DELETE);
@@ -2242,7 +2242,7 @@ void generate_files(int f_out, const char *local_name)
 	}
 	solo_file = local_name;
 	dir_tweaking = !(list_only || solo_file || dry_run);
-	need_retouch_dir_times = preserve_times & PRESERVE_DIR_TIMES;
+	need_retouch_dir_times = preserve_mtimes && !omit_dir_times;
 	loopchk_limit = allowed_lull ? allowed_lull * 5 : 200;
 	symlink_timeset_failed_flags = ITEM_REPORT_TIME
 	    | (protocol_version >= 30 || !am_server ? ITEM_REPORT_TIMEFAIL : 0);
@@ -2295,7 +2295,7 @@ void generate_files(int f_out, const char *local_name)
 						dirdev = MAKEDEV(DEV_MAJOR(devp), DEV_MINOR(devp));
 					} else
 						dirdev = MAKEDEV(0, 0);
-					delete_in_dir(fbuf, fp, &dirdev);
+					delete_in_dir(fbuf, fp, dirdev);
 				} else
 					change_local_filter_dir(fbuf, strlen(fbuf), F_DEPTH(fp));
 			}
@@ -2342,7 +2342,7 @@ void generate_files(int f_out, const char *local_name)
 	} while ((cur_flist = cur_flist->next) != NULL);
 
 	if (delete_during)
-		delete_in_dir(NULL, NULL, &dev_zero);
+		delete_in_dir(NULL, NULL, dev_zero);
 	phase++;
 	if (DEBUG_GTE(GENR, 1))
 		rprintf(FINFO, "generate_files phase=%d\n", phase);

hlink.c

@@ -406,7 +406,7 @@ int hard_link_check(struct file_struct *file, int ndx, char *fname,
 				}
 				break;
 			}
-			if (!unchanged_file(cmpbuf, file, &alt_sx.st))
+			if (!quick_check_ok(FT_REG, cmpbuf, file, &alt_sx.st))
 				continue;
 			statret = 1;
 			if (unchanged_attrs(cmpbuf, file, &alt_sx))

ifuncs.h

@@ -1,6 +1,6 @@
 /* Inline functions for rsync.
  *
- * Copyright (C) 2007-2020 Wayne Davison
+ * Copyright (C) 2007-2022 Wayne Davison
  *
  * This program is free software; you can redistribute it and/or modify
  * it under the terms of the GNU General Public License as published by
@@ -75,6 +75,7 @@ d_name(struct dirent *di)
 static inline void
 init_stat_x(stat_x *sx_p)
 {
+	sx_p->crtime = 0;
 #ifdef SUPPORT_ACLS
 	sx_p->acc_acl = sx_p->def_acl = NULL;
 #endif

install-sh

@@ -115,7 +115,7 @@ else
 # might cause directories to be created, which would be especially bad 
 # if $src (and thus $dsttmp) contains '*'.
 
-	if [ -f $src -o -d $src ]
+	if [ -f $src ] || [ -d $src ]
 	then
 		true
 	else

io.c

@@ -4,7 +4,7 @@
  * Copyright (C) 1996-2001 Andrew Tridgell
  * Copyright (C) 1996 Paul Mackerras
  * Copyright (C) 2001, 2002 Martin Pool <mbp@samba.org>
- * Copyright (C) 2003-2020 Wayne Davison
+ * Copyright (C) 2003-2022 Wayne Davison
  *
  * This program is free software; you can redistribute it and/or modify
  * it under the terms of the GNU General Public License as published by
@@ -264,15 +264,18 @@ static size_t safe_read(int fd, char *buf, size_t len)
 			rprintf(FINFO, "select exception on fd %d\n", fd); */
 
 		if (FD_ISSET(fd, &r_fds)) {
-			int n = read(fd, buf + got, len - got);
-			if (DEBUG_GTE(IO, 2))
-				rprintf(FINFO, "[%s] safe_read(%d)=%ld\n", who_am_i(), fd, (long)n);
+			ssize_t n = read(fd, buf + got, len - got);
+			if (DEBUG_GTE(IO, 2)) {
+				rprintf(FINFO, "[%s] safe_read(%d)=%" SIZE_T_FMT_MOD "d\n",
+					who_am_i(), fd, (SIZE_T_FMT_CAST)n);
+			}
 			if (n == 0)
 				break;
 			if (n < 0) {
 				if (errno == EINTR)
 					continue;
-				rsyserr(FERROR, errno, "safe_read failed to read %ld bytes", (long)len);
+				rsyserr(FERROR, errno, "safe_read failed to read %" SIZE_T_FMT_MOD "d bytes",
+					(SIZE_T_FMT_CAST)len);
 				exit_cleanup(RERR_STREAMIO);
 			}
 			if ((got += (size_t)n) == len)
@@ -304,7 +307,7 @@ static const char *what_fd_is(int fd)
  * is not used on the socket except very early in the transfer. */
 static void safe_write(int fd, const char *buf, size_t len)
 {
-	int n;
+	ssize_t n;
 
 	assert(fd != iobuf.out_fd);
 
@@ -315,8 +318,8 @@ static void safe_write(int fd, const char *buf, size_t len)
 		if (errno != EINTR && errno != EWOULDBLOCK && errno != EAGAIN) {
 		  write_failed:
 			rsyserr(FERROR, errno,
-				"safe_write failed to write %ld bytes to %s",
-				(long)len, what_fd_is(fd));
+				"safe_write failed to write %" SIZE_T_FMT_MOD "d bytes to %s",
+				(SIZE_T_FMT_CAST)len, what_fd_is(fd));
 			exit_cleanup(RERR_STREAMIO);
 		}
 	} else {
@@ -362,7 +365,7 @@ static void safe_write(int fd, const char *buf, size_t len)
  * a chunk of data and put it into the output buffer. */
 static void forward_filesfrom_data(void)
 {
-	int len;
+	ssize_t len;
 
 	len = read(ff_forward_fd, ff_xb.buf + ff_xb.len, ff_xb.size - ff_xb.len);
 	if (len <= 0) {
@@ -377,8 +380,10 @@ static void forward_filesfrom_data(void)
 		return;
 	}
 
-	if (DEBUG_GTE(IO, 2))
-		rprintf(FINFO, "[%s] files-from read=%ld\n", who_am_i(), (long)len);
+	if (DEBUG_GTE(IO, 2)) {
+		rprintf(FINFO, "[%s] files-from read=%" SIZE_T_FMT_MOD "d\n",
+			who_am_i(), (SIZE_T_FMT_CAST)len);
+	}
 
 #ifdef ICONV_OPTION
 	len += ff_xb.len;
@@ -562,52 +567,59 @@ static char *perform_io(size_t needed, int flags)
 	case PIO_NEED_INPUT:
 		/* We never resize the circular input buffer. */
 		if (iobuf.in.size < needed) {
-			rprintf(FERROR, "need to read %ld bytes, iobuf.in.buf is only %ld bytes.\n",
-				(long)needed, (long)iobuf.in.size);
+			rprintf(FERROR, "need to read %" SIZE_T_FMT_MOD "d bytes,"
+					" iobuf.in.buf is only %" SIZE_T_FMT_MOD "d bytes.\n",
+				(SIZE_T_FMT_CAST)needed, (SIZE_T_FMT_CAST)iobuf.in.size);
 			exit_cleanup(RERR_PROTOCOL);
 		}
 
 		if (msgs2stderr == 1 && DEBUG_GTE(IO, 3)) {
-			rprintf(FINFO, "[%s] perform_io(%ld, %sinput)\n",
-				who_am_i(), (long)needed, flags & PIO_CONSUME_INPUT ? "consume&" : "");
+			rprintf(FINFO, "[%s] perform_io(%" SIZE_T_FMT_MOD "d, %sinput)\n",
+				who_am_i(), (SIZE_T_FMT_CAST)needed, flags & PIO_CONSUME_INPUT ? "consume&" : "");
 		}
 		break;
 
 	case PIO_NEED_OUTROOM:
 		/* We never resize the circular output buffer. */
 		if (iobuf.out.size - iobuf.out_empty_len < needed) {
-			fprintf(stderr, "need to write %ld bytes, iobuf.out.buf is only %ld bytes.\n",
-				(long)needed, (long)(iobuf.out.size - iobuf.out_empty_len));
+			fprintf(stderr, "need to write %" SIZE_T_FMT_MOD "d bytes,"
+					" iobuf.out.buf is only %" SIZE_T_FMT_MOD "d bytes.\n",
+				(SIZE_T_FMT_CAST)needed, (SIZE_T_FMT_CAST)(iobuf.out.size - iobuf.out_empty_len));
 			exit_cleanup(RERR_PROTOCOL);
 		}
 
 		if (msgs2stderr == 1 && DEBUG_GTE(IO, 3)) {
-			rprintf(FINFO, "[%s] perform_io(%ld, outroom) needs to flush %ld\n",
-				who_am_i(), (long)needed,
+			rprintf(FINFO, "[%s] perform_io(%" SIZE_T_FMT_MOD "d,"
+				       " outroom) needs to flush %" SIZE_T_FMT_MOD "d\n",
+				who_am_i(), (SIZE_T_FMT_CAST)needed,
 				iobuf.out.len + needed > iobuf.out.size
-				? (long)(iobuf.out.len + needed - iobuf.out.size) : 0L);
+				? (SIZE_T_FMT_CAST)(iobuf.out.len + needed - iobuf.out.size) : (SIZE_T_FMT_CAST)0);
 		}
 		break;
 
 	case PIO_NEED_MSGROOM:
 		/* We never resize the circular message buffer. */
 		if (iobuf.msg.size < needed) {
-			fprintf(stderr, "need to write %ld bytes, iobuf.msg.buf is only %ld bytes.\n",
-				(long)needed, (long)iobuf.msg.size);
+			fprintf(stderr, "need to write %" SIZE_T_FMT_MOD "d bytes,"
+					" iobuf.msg.buf is only %" SIZE_T_FMT_MOD "d bytes.\n",
+				(SIZE_T_FMT_CAST)needed, (SIZE_T_FMT_CAST)iobuf.msg.size);
 			exit_cleanup(RERR_PROTOCOL);
 		}
 
 		if (msgs2stderr == 1 && DEBUG_GTE(IO, 3)) {
-			rprintf(FINFO, "[%s] perform_io(%ld, msgroom) needs to flush %ld\n",
-				who_am_i(), (long)needed,
+			rprintf(FINFO, "[%s] perform_io(%" SIZE_T_FMT_MOD "d,"
+				       " msgroom) needs to flush %" SIZE_T_FMT_MOD "d\n",
+				who_am_i(), (SIZE_T_FMT_CAST)needed,
 				iobuf.msg.len + needed > iobuf.msg.size
-				? (long)(iobuf.msg.len + needed - iobuf.msg.size) : 0L);
+				? (SIZE_T_FMT_CAST)(iobuf.msg.len + needed - iobuf.msg.size) : (SIZE_T_FMT_CAST)0);
 		}
 		break;
 
 	case 0:
-		if (msgs2stderr == 1 && DEBUG_GTE(IO, 3))
-			rprintf(FINFO, "[%s] perform_io(%ld, %d)\n", who_am_i(), (long)needed, flags);
+		if (msgs2stderr == 1 && DEBUG_GTE(IO, 3)) {
+			rprintf(FINFO, "[%s] perform_io(%" SIZE_T_FMT_MOD "d, %d)\n",
+				who_am_i(), (SIZE_T_FMT_CAST)needed, flags);
+		}
 		break;
 
 	default:
@@ -665,8 +677,8 @@ static char *perform_io(size_t needed, int flags)
 					      ((MPLEX_BASE + (int)MSG_DATA)<<24) + iobuf.out.len - 4);
 
 					if (msgs2stderr == 1 && DEBUG_GTE(IO, 1)) {
-						rprintf(FINFO, "[%s] send_msg(%d, %ld)\n",
-							who_am_i(), (int)MSG_DATA, (long)iobuf.out.len - 4);
+						rprintf(FINFO, "[%s] send_msg(%d, %" SIZE_T_FMT_MOD "d)\n",
+							who_am_i(), (int)MSG_DATA, (SIZE_T_FMT_CAST)iobuf.out.len - 4);
 					}
 
 					/* reserve room for the next MSG_DATA header */
@@ -757,7 +769,7 @@ static char *perform_io(size_t needed, int flags)
 
 		if (iobuf.in_fd >= 0 && FD_ISSET(iobuf.in_fd, &r_fds)) {
 			size_t len, pos = iobuf.in.pos + iobuf.in.len;
-			int n;
+			ssize_t n;
 			if (pos >= iobuf.in.size) {
 				pos -= iobuf.in.size;
 				len = iobuf.in.size - iobuf.in.len;
@@ -784,15 +796,13 @@ static char *perform_io(size_t needed, int flags)
 					exit_cleanup(RERR_SOCKETIO);
 				}
 			}
-			if (msgs2stderr == 1 && DEBUG_GTE(IO, 2))
-				rprintf(FINFO, "[%s] recv=%ld\n", who_am_i(), (long)n);
+			if (msgs2stderr == 1 && DEBUG_GTE(IO, 2)) {
+				rprintf(FINFO, "[%s] recv=%" SIZE_T_FMT_MOD "d\n",
+					who_am_i(), (SIZE_T_FMT_CAST)n);
+			}
 
-			if (io_timeout || stop_at_utime) {
+			if (io_timeout) {
 				last_io_in = time(NULL);
-				if (stop_at_utime && last_io_in >= stop_at_utime) {
-					rprintf(FERROR, "stopping at requested limit\n");
-					exit_cleanup(RERR_TIMEOUT);
-				}
 				if (io_timeout && flags & PIO_NEED_INPUT)
 					maybe_send_keepalive(last_io_in, 0);
 			}
@@ -801,9 +811,14 @@ static char *perform_io(size_t needed, int flags)
 			iobuf.in.len += n;
 		}
 
+		if (stop_at_utime && time(NULL) >= stop_at_utime) {
+			rprintf(FERROR, "stopping at requested limit\n");
+			exit_cleanup(RERR_TIMEOUT);
+		}
+
 		if (out && FD_ISSET(iobuf.out_fd, &w_fds)) {
 			size_t len = iobuf.raw_flushing_ends_before ? iobuf.raw_flushing_ends_before - out->pos : out->len;
-			int n;
+			ssize_t n;
 
 			if (bwlimit_writemax && len > bwlimit_writemax)
 				len = bwlimit_writemax;
@@ -824,8 +839,8 @@ static char *perform_io(size_t needed, int flags)
 				}
 			}
 			if (msgs2stderr == 1 && DEBUG_GTE(IO, 2)) {
-				rprintf(FINFO, "[%s] %s sent=%ld\n",
-					who_am_i(), out == &iobuf.out ? "out" : "msg", (long)n);
+				rprintf(FINFO, "[%s] %s sent=%" SIZE_T_FMT_MOD "d\n",
+					who_am_i(), out == &iobuf.out ? "out" : "msg", (SIZE_T_FMT_CAST)n);
 			}
 
 			if (io_timeout)
@@ -945,8 +960,10 @@ int send_msg(enum msgcode code, const char *buf, size_t len, int convert)
 	if (!OUT_MULTIPLEXED)
 		return 0;
 
-	if (want_debug)
-		rprintf(FINFO, "[%s] send_msg(%d, %ld)\n", who_am_i(), (int)code, (long)len);
+	if (want_debug) {
+		rprintf(FINFO, "[%s] send_msg(%d, %" SIZE_T_FMT_MOD "d)\n",
+			who_am_i(), (int)code, (SIZE_T_FMT_CAST)len);
+	}
 
 	/* When checking for enough free space for this message, we need to
 	 * make sure that there is space for the 4-byte header, plus we'll
@@ -962,9 +979,9 @@ int send_msg(enum msgcode code, const char *buf, size_t len, int convert)
 #endif
 		needed = len + 4 + 3;
 	if (iobuf.msg.len + needed > iobuf.msg.size) {
-		if (!am_receiver)
+		if (am_sender)
 			perform_io(needed, PIO_NEED_MSGROOM);
-		else { /* We allow the receiver to increase their iobuf.msg size to avoid a deadlock. */
+		else { /* We sometimes allow the iobuf.msg size to increase to avoid a deadlock. */
 			size_t old_size = iobuf.msg.size;
 			restore_iobuf_size(&iobuf.msg);
 			realloc_xbuf(&iobuf.msg, iobuf.msg.size * 2);
@@ -1021,8 +1038,10 @@ int send_msg(enum msgcode code, const char *buf, size_t len, int convert)
 
 	SIVAL(hdr, 0, ((MPLEX_BASE + (int)code)<<24) + len);
 
-	if (want_debug && convert > 0)
-		rprintf(FINFO, "[%s] converted msg len=%ld\n", who_am_i(), (long)len);
+	if (want_debug && convert > 0) {
+		rprintf(FINFO, "[%s] converted msg len=%" SIZE_T_FMT_MOD "d\n",
+			who_am_i(), (SIZE_T_FMT_CAST)len);
+	}
 
 	return 1;
 }
@@ -1436,8 +1455,10 @@ static void read_a_msg(void)
 	msg_bytes = tag & 0xFFFFFF;
 	tag = (tag >> 24) - MPLEX_BASE;
 
-	if (msgs2stderr == 1 && DEBUG_GTE(IO, 1))
-		rprintf(FINFO, "[%s] got msg=%d, len=%ld\n", who_am_i(), (int)tag, (long)msg_bytes);
+	if (msgs2stderr == 1 && DEBUG_GTE(IO, 1)) {
+		rprintf(FINFO, "[%s] got msg=%d, len=%" SIZE_T_FMT_MOD "d\n",
+			who_am_i(), (int)tag, (SIZE_T_FMT_CAST)msg_bytes);
+	}
 
 	switch (tag) {
 	case MSG_DATA:
@@ -1613,8 +1634,10 @@ static void read_a_msg(void)
 		else
 			goto invalid_msg;
 		iobuf.in_multiplexed = 1;
-		if (DEBUG_GTE(EXIT, 3))
-			rprintf(FINFO, "[%s] got MSG_ERROR_EXIT with %ld bytes\n", who_am_i(), (long)msg_bytes);
+		if (DEBUG_GTE(EXIT, 3)) {
+			rprintf(FINFO, "[%s] got MSG_ERROR_EXIT with %" SIZE_T_FMT_MOD "d bytes\n",
+					who_am_i(), (SIZE_T_FMT_CAST)msg_bytes);
+		}
 		if (msg_bytes == 0) {
 			if (!am_sender && !am_generator) {
 				if (DEBUG_GTE(EXIT, 3)) {

itypes.h

@@ -1,6 +1,6 @@
 /* Inline functions for rsync.
  *
- * Copyright (C) 2007-2020 Wayne Davison
+ * Copyright (C) 2007-2022 Wayne Davison
  *
  * This program is free software; you can redistribute it and/or modify
  * it under the terms of the GNU General Public License as published by
@@ -40,6 +40,12 @@ isSpace(const char *ptr)
 	return isspace(*(unsigned char *)ptr);
 }
 
+static inline int
+isAlNum(const char *ptr)
+{
+	return isalnum(*(unsigned char *)ptr);
+}
+
 static inline int
 isLower(const char *ptr)
 {

latest-year.h

@@ -1 +1 @@
-#define LATEST_YEAR "2020"
+#define LATEST_YEAR "2022"

lib/pool_alloc.c

@@ -9,8 +9,7 @@ struct alloc_pool
 	size_t			size;		/* extent size		*/
 	size_t			quantum;	/* allocation quantum	*/
 	struct pool_extent	*extents;	/* top extent is "live" */
-	void			(*bomb)();	/* function to call if
-						 * malloc fails		*/
+	void			(*bomb)();	/* called if malloc fails */
 	int			flags;
 
 	/* statistical data */
@@ -49,7 +48,7 @@ pool_create(size_t size, size_t quantum, void (*bomb)(const char*, const char*,
 {
 	struct alloc_pool *pool;
 
-	if ((MINALIGN & (MINALIGN - 1)) != 0) {
+	if ((MINALIGN & (MINALIGN - 1)) != (0)) {
 		if (bomb)
 			(*bomb)("Compiler error: MINALIGN is not a power of 2", __FILE__, __LINE__);
 		return NULL;

lib/snprintf.c

@@ -20,7 +20,7 @@
  * for string length.  This covers a nasty loophole.
  *
  * The other functions are there to prevent NULL pointers from
- * causing nast effects.
+ * causing nasty effects.
  *
  * More Recently:
  *  Brandon Long <blong@fiction.net> 9/15/96 for mutt 0.43

lib/sysacls.c

@@ -1739,7 +1739,6 @@ int sys_acl_free_acl(SMB_ACL_T acl_d)
 int sys_acl_get_entry(SMB_ACL_T theacl, int entry_id, SMB_ACL_ENTRY_T *entry_p)
 {
 	struct acl_entry_link *link;
-	struct new_acl_entry *entry;
 	int keep_going;
 
 	if (entry_id == SMB_ACL_FIRST_ENTRY)
@@ -1765,10 +1764,15 @@ int sys_acl_get_entry(SMB_ACL_T theacl, int entry_id, SMB_ACL_ENTRY_T *entry_p)
 	for (keep_going = 0; keep_going < theacl->count; keep_going++)
 		link = link->nextp;
 
-	entry = *entry_p =  link->entryp;
+	*entry_p =  link->entryp;
 
-	DEBUG(10, ("*entry_p is %d\n", entry_p));
-	DEBUG(10, ("*entry_p->ace_access is %d\n", entry->ace_access));
+#if 0
+	{
+		struct new_acl_entry *entry = *entry_p;
+		DEBUG(10, ("*entry_p is %lx\n", (long)entry));
+		DEBUG(10, ("*entry_p->ace_access is %d\n", entry->ace_access));
+	}
+#endif
 
 	/* Increment count */
 	theacl->count++;
@@ -2297,7 +2301,7 @@ int sys_acl_create_entry(SMB_ACL_T *pacl, SMB_ACL_ENTRY_T *pentry)
 {
 	struct acl_entry_link *theacl;
 	struct acl_entry_link *acl_entryp;
-	struct acl_entry_link *temp_entry;
+	struct acl_entry_link *temp_entry = NULL;
 	int counting;
 
 	DEBUG(10, ("Entering the sys_acl_create_entry\n"));

lib/sysxattrs.c

@@ -2,7 +2,7 @@
  * Extended attribute support for rsync.
  *
  * Copyright (C) 2004 Red Hat, Inc.
- * Copyright (C) 2003-2019 Wayne Davison
+ * Copyright (C) 2003-2022 Wayne Davison
  * Written by Jay Fenlason.
  *
  * This program is free software; you can redistribute it and/or modify
@@ -67,7 +67,7 @@ ssize_t sys_lgetxattr(const char *path, const char *name, void *value, size_t si
 		u_int32_t offset = len;
 		size_t data_retrieved = len;
 		while (data_retrieved < size) {
-			len = getxattr(path, name, value + offset, size - data_retrieved, offset, XATTR_NOFOLLOW);
+			len = getxattr(path, name, (char*)value + offset, size - data_retrieved, offset, XATTR_NOFOLLOW);
 			if (len <= 0)
 				break;
 			data_retrieved += len;
@@ -167,7 +167,7 @@ static ssize_t read_xattr(int attrfd, void *buf, size_t buflen)
 	} else {
 		size_t bufpos;
 		for (bufpos = 0; bufpos < sb.st_size; ) {
-			ssize_t cnt = read(attrfd, buf + bufpos, sb.st_size - bufpos);
+			ssize_t cnt = read(attrfd, (char*)buf + bufpos, sb.st_size - bufpos);
 			if (cnt <= 0) {
 				if (cnt < 0 && errno == EINTR)
 					continue;
@@ -218,7 +218,7 @@ int sys_lsetxattr(const char *path, const char *name, const void *value, size_t
 		return -1;
 
 	for (bufpos = 0; bufpos < size; ) {
-		ssize_t cnt = write(attrfd, value+bufpos, size);
+		ssize_t cnt = write(attrfd, (char*)value + bufpos, size);
 		if (cnt <= 0) {
 			if (cnt < 0 && errno == EINTR)
 				continue;
@@ -274,7 +274,8 @@ ssize_t sys_llistxattr(const char *path, char *list, size_t size)
 		 && (dp->d_name[10] == 'o' || dp->d_name[10] == 'w'))
 			continue;
 
-		if ((ret += len+1) > size) {
+		ret += len + 1;
+		if ((size_t)ret > size) {
 			if (size == 0)
 				continue;
 			ret = -1;

loadparm.c

@@ -437,7 +437,7 @@ static BOOL do_parameter(char *parmname, char *parmvalue)
 		break;
 
 	case P_OCTAL:
-		sscanf(parmvalue, "%o", (int *)parm_ptr);
+		sscanf(parmvalue, "%o", (unsigned int *)parm_ptr);
 		break;
 
 	case P_PATH:

log.c

@@ -3,7 +3,7 @@
  *
  * Copyright (C) 1998-2001 Andrew Tridgell <tridge@samba.org>
  * Copyright (C) 2000-2001 Martin Pool <mbp@samba.org>
- * Copyright (C) 2003-2020 Wayne Davison
+ * Copyright (C) 2003-2022 Wayne Davison
  *
  * This program is free software; you can redistribute it and/or modify
  * it under the terms of the GNU General Public License as published by
@@ -34,7 +34,7 @@ extern int module_id;
 extern int allow_8bit_chars;
 extern int protocol_version;
 extern int always_checksum;
-extern int preserve_times;
+extern int preserve_mtimes;
 extern int msgs2stderr;
 extern int xfersum_type;
 extern int checksum_type;
@@ -706,7 +706,7 @@ static void log_formatted(enum logcode code, const char *format, const char *op,
 				c[1] = 'L';
 				c[3] = '.';
 				c[4] = !(iflags & ITEM_REPORT_TIME) ? '.'
-				     : !preserve_times || !receiver_symlink_times
+				     : !preserve_mtimes || !receiver_symlink_times
 				    || (iflags & ITEM_REPORT_TIMEFAIL) ? 'T' : 't';
 			} else {
 				c[1] = S_ISDIR(file->mode) ? 'd'
@@ -714,7 +714,7 @@ static void log_formatted(enum logcode code, const char *format, const char *op,
 				     : IS_DEVICE(file->mode) ? 'D' : 'f';
 				c[3] = !(iflags & ITEM_REPORT_SIZE) ? '.' : 's';
 				c[4] = !(iflags & ITEM_REPORT_TIME) ? '.'
-				     : !preserve_times ? 'T' : 't';
+				     : !preserve_mtimes ? 'T' : 't';
 			}
 			c[2] = !(iflags & ITEM_REPORT_CHANGE) ? '.' : 'c';
 			c[5] = !(iflags & ITEM_REPORT_PERMS) ? '.' : 'p';
@@ -838,14 +838,24 @@ void maybe_log_item(struct file_struct *file, int iflags, int itemizing, const c
 
 void log_delete(const char *fname, int mode)
 {
-	static struct {
-		union file_extras ex[4]; /* just in case... */
-		struct file_struct file;
-	} x; /* Zero-initialized due to static declaration. */
+	static struct file_struct *file = NULL;
 	int len = strlen(fname);
 	const char *fmt;
 
-	x.file.mode = mode;
+	if (!file) {
+		int extra_len = (file_extra_cnt + 2) * EXTRA_LEN;
+		char *bp;
+#if EXTRA_ROUNDING > 0
+		if (extra_len & (EXTRA_ROUNDING * EXTRA_LEN))
+			extra_len = (extra_len | (EXTRA_ROUNDING * EXTRA_LEN)) + EXTRA_LEN;
+#endif
+
+		bp = new_array0(char, FILE_STRUCT_LEN + extra_len + 1);
+		bp += extra_len;
+		file = (struct file_struct *)bp;
+	}
+
+	file->mode = mode;
 
 	if (am_server && protocol_version >= 29 && len < MAXPATHLEN) {
 		if (S_ISDIR(mode))
@@ -855,14 +865,14 @@ void log_delete(const char *fname, int mode)
 		;
 	else {
 		fmt = stdout_format_has_o_or_i ? stdout_format : "deleting %n";
-		log_formatted(FCLIENT, fmt, "del.", &x.file, fname, ITEM_DELETED, NULL);
+		log_formatted(FCLIENT, fmt, "del.", file, fname, ITEM_DELETED, NULL);
 	}
 
 	if (!logfile_name || dry_run || !logfile_format)
 		return;
 
 	fmt = logfile_format_has_o_or_i ? logfile_format : "deleting %n";
-	log_formatted(FLOG, fmt, "del.", &x.file, fname, ITEM_DELETED, NULL);
+	log_formatted(FLOG, fmt, "del.", file, fname, ITEM_DELETED, NULL);
 }
 
 /*

main.c

@@ -4,7 +4,7 @@
  * Copyright (C) 1996-2001 Andrew Tridgell <tridge@samba.org>
  * Copyright (C) 1996 Paul Mackerras
  * Copyright (C) 2001, 2002 Martin Pool <mbp@samba.org>
- * Copyright (C) 2003-2020 Wayne Davison
+ * Copyright (C) 2003-2022 Wayne Davison
  *
  * This program is free software; you can redistribute it and/or modify
  * it under the terms of the GNU General Public License as published by
@@ -48,6 +48,7 @@ extern int called_from_signal_handler;
 extern int need_messages_from_generator;
 extern int kluge_around_eof;
 extern int got_xfer_error;
+extern int old_style_args;
 extern int msgs2stderr;
 extern int module_id;
 extern int read_only;
@@ -87,6 +88,7 @@ extern BOOL shutting_down;
 extern int backup_dir_len;
 extern int basis_dir_cnt;
 extern int default_af_hint;
+extern int stdout_format_has_i;
 extern struct stats stats;
 extern char *stdout_format;
 extern char *logfile_format;
@@ -466,38 +468,33 @@ static void output_summary(void)
  **/
 static void show_malloc_stats(void)
 {
-#ifdef HAVE_MALLINFO
-	struct mallinfo mi;
-
-	mi = mallinfo();
+#ifdef MEM_ALLOC_INFO
+	struct MEM_ALLOC_INFO mi = MEM_ALLOC_INFO(); /* mallinfo or mallinfo2 */
 
 	rprintf(FCLIENT, "\n");
 	rprintf(FINFO, RSYNC_NAME "[%d] (%s%s%s) heap statistics:\n",
 		(int)getpid(), am_server ? "server " : "",
 		am_daemon ? "daemon " : "", who_am_i());
-	rprintf(FINFO, "  arena:     %10ld   (bytes from sbrk)\n",
-		(long)mi.arena);
-	rprintf(FINFO, "  ordblks:   %10ld   (chunks not in use)\n",
-		(long)mi.ordblks);
-	rprintf(FINFO, "  smblks:    %10ld\n",
-		(long)mi.smblks);
-	rprintf(FINFO, "  hblks:     %10ld   (chunks from mmap)\n",
-		(long)mi.hblks);
-	rprintf(FINFO, "  hblkhd:    %10ld   (bytes from mmap)\n",
-		(long)mi.hblkhd);
-	rprintf(FINFO, "  allmem:    %10ld   (bytes from sbrk + mmap)\n",
-		(long)mi.arena + mi.hblkhd);
-	rprintf(FINFO, "  usmblks:   %10ld\n",
-		(long)mi.usmblks);
-	rprintf(FINFO, "  fsmblks:   %10ld\n",
-		(long)mi.fsmblks);
-	rprintf(FINFO, "  uordblks:  %10ld   (bytes used)\n",
-		(long)mi.uordblks);
-	rprintf(FINFO, "  fordblks:  %10ld   (bytes free)\n",
-		(long)mi.fordblks);
-	rprintf(FINFO, "  keepcost:  %10ld   (bytes in releasable chunk)\n",
-		(long)mi.keepcost);
-#endif /* HAVE_MALLINFO */
+
+#define PRINT_ALLOC_NUM(title, descr, num) \
+	rprintf(FINFO, "  %-11s%10" SIZE_T_FMT_MOD "d   (" descr ")\n", \
+		title ":", (SIZE_T_FMT_CAST)(num));
+
+	PRINT_ALLOC_NUM("arena", "bytes from sbrk", mi.arena);
+	PRINT_ALLOC_NUM("ordblks", "chunks not in use", mi.ordblks);
+	PRINT_ALLOC_NUM("smblks", "free fastbin blocks", mi.smblks);
+	PRINT_ALLOC_NUM("hblks", "chunks from mmap", mi.hblks);
+	PRINT_ALLOC_NUM("hblkhd", "bytes from mmap", mi.hblkhd);
+	PRINT_ALLOC_NUM("allmem", "bytes from sbrk + mmap", mi.arena + mi.hblkhd);
+	PRINT_ALLOC_NUM("usmblks", "always 0", mi.usmblks);
+	PRINT_ALLOC_NUM("fsmblks", "bytes in freed fastbin blocks", mi.fsmblks);
+	PRINT_ALLOC_NUM("uordblks", "bytes used", mi.uordblks);
+	PRINT_ALLOC_NUM("fordblks", "bytes free", mi.fordblks);
+	PRINT_ALLOC_NUM("keepcost", "bytes in releasable chunk", mi.keepcost);
+
+#undef PRINT_ALLOC_NUM
+
+#endif /* MEM_ALLOC_INFO */
 }
 
 
@@ -611,11 +608,7 @@ static pid_t do_cmd(char *cmd, char *machine, char *user, char **remote_argv, in
 				rprintf(FERROR, "internal: args[] overflowed in do_cmd()\n");
 				exit_cleanup(RERR_SYNTAX);
 			}
-			if (**remote_argv == '-') {
-				if (asprintf(args + argc++, "./%s", *remote_argv++) < 0)
-					out_of_memory("do_cmd");
-			} else
-				args[argc++] = *remote_argv++;
+			args[argc++] = safe_arg(NULL, *remote_argv++);
 			remote_argc--;
 		}
 	}
@@ -721,18 +714,21 @@ static char *get_local_name(struct file_list *flist, char *dest_path)
 	trailing_slash = cp && !cp[1];
 
 	if (mkpath_dest_arg && statret < 0 && (cp || file_total > 1)) {
+		int save_errno = errno;
 		int ret = make_path(dest_path, file_total > 1 && !trailing_slash ? 0 : MKP_DROP_NAME);
 		if (ret < 0)
 			goto mkdir_error;
-		if (INFO_GTE(NAME, 1)) {
+		if (ret && (INFO_GTE(NAME, 1) || stdout_format_has_i)) {
 			if (file_total == 1 || trailing_slash)
 				*cp = '\0';
 			rprintf(FINFO, "created %d director%s for %s\n", ret, ret == 1 ? "y" : "ies", dest_path);
 			if (file_total == 1 || trailing_slash)
 				*cp = '/';
 		}
-		if (file_total > 1 || trailing_slash)
+		if (ret)
 			statret = do_stat(dest_path, &st);
+		else
+			errno = save_errno;
 	}
 
 	if (statret == 0) {
@@ -788,7 +784,7 @@ static char *get_local_name(struct file_list *flist, char *dest_path)
 		 && strcmp(flist->files[flist->low]->basename, ".") == 0)
 			flist->files[0]->flags |= FLAG_DIR_CREATED;
 
-		if (INFO_GTE(NAME, 1))
+		if (INFO_GTE(NAME, 1) || stdout_format_has_i)
 			rprintf(FINFO, "created directory %s\n", dest_path);
 
 		if (dry_run) {
@@ -1475,6 +1471,10 @@ static int start_client(int argc, char *argv[])
 		rsync_port = 0;
 	}
 
+	/* A local transfer doesn't unbackslash anything, so leave the args alone. */
+	if (local_server)
+		old_style_args = 2;
+
 	if (!rsync_port && remote_argc && !**remote_argv) /* Turn an empty arg into a dot dir. */
 		*remote_argv = ".";
 
@@ -1564,6 +1564,8 @@ static int start_client(int argc, char *argv[])
 #ifdef HAVE_PUTENV
 	if (daemon_connection)
 		set_env_num("RSYNC_PORT", env_port);
+#else
+	(void)env_port;
 #endif
 
 	pid = do_cmd(shell_cmd, shell_machine, shell_user, remote_argv, remote_argc, &f_in, &f_out);
@@ -1636,7 +1638,6 @@ void remember_children(UNUSED(int val))
 #endif
 }
 
-
 /**
  * This routine catches signals and tries to send them to gdb.
  *
@@ -1660,7 +1661,6 @@ const char *get_panic_action(void)
 	return "xterm -display :0 -T Panic -n Panic -e gdb /proc/%d/exe %d";
 }
 
-
 /**
  * Handle a fatal signal by launching a debugger, controlled by $RSYNC_PANIC_ACTION.
  *
@@ -1684,6 +1684,22 @@ static void rsync_panic_handler(UNUSED(int whatsig))
 }
 #endif
 
+static void unset_env_var(const char *var)
+{
+#ifdef HAVE_UNSETENV
+	unsetenv(var);
+#else
+#ifdef HAVE_PUTENV
+	char *mem;
+	if (asprintf(&mem, "%s=", var) < 0)
+		out_of_memory("unset_env_var");
+	putenv(mem);
+#else
+	(void)var;
+#endif
+#endif
+}
+
 
 int main(int argc,char *argv[])
 {
@@ -1721,6 +1737,8 @@ int main(int argc,char *argv[])
 	our_gid = MY_GID();
 	am_root = our_uid == ROOT_UID;
 
+	unset_env_var("DISPLAY");
+
 	memset(&stats, 0, sizeof(stats));
 
 	/* Even a non-daemon runs needs the default config values to be set, e.g.
@@ -1739,6 +1757,7 @@ int main(int argc,char *argv[])
 
 #if defined CONFIG_LOCALE && defined HAVE_SETLOCALE
 	setlocale(LC_CTYPE, "");
+	setlocale(LC_NUMERIC, "");
 #endif
 
 	if (!parse_arguments(&argc, (const char ***) &argv)) {

maybe-make-man

@@ -1,22 +1,17 @@
 #!/bin/sh
 
-if [ x"$2" = x ]; then
-    echo "Usage: $0 SRC_DIR NAME.NUM.md" 1>&2
+if [ $# != 1 ]; then
+    echo "Usage: $0 NAME.NUM.md" 1>&2
     exit 1
 fi
 
-srcdir="$1"
-inname="$2"
+inname="$1"
+srcdir=`dirname "$0"`
 flagfile="$srcdir/.md2man-works"
 
-if [ ! -d "$srcdir" ]; then
-    echo "The specified SRC_DIR is not a directory: $srcdir" 1>&2
-    exit 1
-fi
-
 if [ ! -f "$flagfile" ]; then
     # We test our smallest manpage just to see if the python setup works.
-    if "$srcdir/md2man" --test "$srcdir/rsync-ssl.1.md" >/dev/null 2>&1; then
+    if "$srcdir/md-convert" --test "$srcdir/rsync-ssl.1.md" >/dev/null 2>&1; then
 	touch $flagfile
     else
 	outname=`echo "$inname" | sed 's/\.md$//'`
@@ -37,4 +32,4 @@ if [ ! -f "$flagfile" ]; then
     fi
 fi
 
-"$srcdir/md2man" "$srcdir/$inname"
+"$srcdir/md-convert" "$srcdir/$inname"

md-convert

@@ -0,0 +1,630 @@
+#!/usr/bin/env python3
+
+# This script transforms markdown files into html and (optionally) nroff. The
+# output files are written into the current directory named for the input file
+# without the .md suffix and either the .html suffix or no suffix.
+#
+# If the input .md file has a section number at the end of the name (e.g.,
+# rsync.1.md) a nroff file is also output (PROJ.NUM.md -> PROJ.NUM).
+#
+# The markdown input format has one extra extension: if a numbered list starts
+# at 0, it is turned into a description list. The dl's dt tag is taken from the
+# contents of the first tag inside the li, which is usually a p, code, or
+# strong tag.
+#
+# The cmarkgfm or commonmark lib is used to transforms the input file into
+# html.  Then, the html.parser is used as a state machine that lets us tweak
+# the html and (optionally) output nroff data based on the html tags.
+#
+# If the string @USE_GFM_PARSER@ exists in the file, the string is removed and
+# a github-flavored-markup parser is used to parse the file.
+#
+# The man-page .md files also get the vars @VERSION@, @BINDIR@, and @LIBDIR@
+# substituted.  Some of these values depend on the Makefile $(prefix) (see the
+# generated Makefile).  If the maintainer wants to build files for /usr/local
+# while creating release-ready man-page files for /usr, use the environment to
+# set RSYNC_OVERRIDE_PREFIX=/usr.
+
+# Copyright (C) 2020 - 2021 Wayne Davison
+#
+# This program is freely redistributable.
+
+import os, sys, re, argparse, subprocess, time
+from html.parser import HTMLParser
+
+VALID_PAGES = 'README INSTALL COPYING rsync.1 rrsync.1 rsync-ssl.1 rsyncd.conf.5'.split()
+
+CONSUMES_TXT = set('h1 h2 h3 p li pre'.split())
+
+HTML_START = """\
+<html><head>
+<title>%TITLE%</title>
+<meta charset="UTF-8"/>
+<link href="https://fonts.googleapis.com/css2?family=Roboto&family=Roboto+Mono&display=swap" rel="stylesheet">
+<style>
+body {
+  max-width: 50em;
+  margin: auto;
+}
+body, b, strong, u {
+  font-family: 'Roboto', sans-serif;
+}
+a.tgt { font-face: symbol; font-weight: 400; font-size: 70%; visibility: hidden; text-decoration: none; color: #ddd; padding: 0 4px; border: 0; }
+a.tgt:after { content: '🔗'; }
+a.tgt:hover { color: #444; background-color: #eaeaea; }
+h1:hover > a.tgt, h2:hover > a.tgt, h3:hover > a.tgt, dt:hover > a.tgt { visibility: visible; }
+code {
+  font-family: 'Roboto Mono', monospace;
+  font-weight: bold;
+  white-space: pre;
+}
+pre code {
+  display: block;
+  font-weight: normal;
+}
+blockquote pre code {
+  background: #f1f1f1;
+}
+dd p:first-of-type {
+  margin-block-start: 0em;
+}
+</style>
+</head><body>
+"""
+
+TABLE_STYLE = """\
+table {
+  border-color: grey;
+  border-spacing: 0;
+}
+tr {
+  border-top: 1px solid grey;
+}
+tr:nth-child(2n) {
+  background-color: #f6f8fa;
+}
+th, td {
+  border: 1px solid #dfe2e5;
+  text-align: center;
+  padding-left: 1em;
+  padding-right: 1em;
+}
+"""
+
+MAN_HTML_END = """\
+<div style="float: right"><p><i>%s</i></p></div>
+"""
+
+HTML_END = """\
+</body></html>
+"""
+
+MAN_START = r"""
+.TH "%s" "%s" "%s" "%s" "User Commands"
+.\" prefix=%s
+""".lstrip()
+
+MAN_END = """\
+"""
+
+NORM_FONT = ('\1', r"\fP")
+BOLD_FONT = ('\2', r"\fB")
+UNDR_FONT = ('\3', r"\fI")
+NBR_DASH = ('\4', r"\-")
+NBR_SPACE = ('\xa0', r"\ ")
+
+FILENAME_RE = re.compile(r'^(?P<fn>(?P<srcdir>.+/)?(?P<name>(?P<prog>[^/]+?)(\.(?P<sect>\d+))?)\.md)$')
+ASSIGNMENT_RE = re.compile(r'^(\w+)=(.+)')
+QUOTED_RE = re.compile(r'"(.+?)"')
+VAR_REF_RE = re.compile(r'\$\{(\w+)\}')
+VERSION_RE = re.compile(r' (\d[.\d]+)[, ]')
+BIN_CHARS_RE = re.compile(r'[\1-\7]+')
+SPACE_DOUBLE_DASH_RE = re.compile(r'\s--(\s)')
+NON_SPACE_SINGLE_DASH_RE = re.compile(r'(^|\W)-')
+WHITESPACE_RE = re.compile(r'\s')
+CODE_BLOCK_RE = re.compile(r'[%s]([^=%s]+)[=%s]' % (BOLD_FONT[0], NORM_FONT[0], NORM_FONT[0]))
+NBR_DASH_RE = re.compile(r'[%s]' % NBR_DASH[0])
+INVALID_TARGET_CHARS_RE = re.compile(r'[^-A-Za-z0-9._]')
+INVALID_START_CHAR_RE = re.compile(r'^([^A-Za-z0-9])')
+MANIFY_LINESTART_RE = re.compile(r"^(['.])", flags=re.M)
+
+md_parser = None
+env_subs = { }
+
+warning_count = 0
+
+def main():
+    for mdfn in args.mdfiles:
+        parse_md_file(mdfn)
+
+    if args.test:
+        print("The test was successful.")
+
+
+def parse_md_file(mdfn):
+    fi = FILENAME_RE.match(mdfn)
+    if not fi:
+        die('Failed to parse a md input file name:', mdfn)
+    fi = argparse.Namespace(**fi.groupdict())
+    fi.want_manpage = not not fi.sect
+    if fi.want_manpage:
+        fi.title = fi.prog + '(' + fi.sect + ') man page'
+    else:
+        fi.title = fi.prog + ' for rsync'
+
+    if fi.want_manpage:
+        if not env_subs:
+            find_man_substitutions()
+        prog_ver = 'rsync ' + env_subs['VERSION']
+        if fi.prog != 'rsync':
+            prog_ver = fi.prog + ' from ' + prog_ver
+        fi.man_headings = (fi.prog, fi.sect, env_subs['date'], prog_ver, env_subs['prefix'])
+
+    with open(mdfn, 'r', encoding='utf-8') as fh:
+        txt = fh.read()
+
+    use_gfm_parser = '@USE_GFM_PARSER@' in txt
+    if use_gfm_parser:
+        txt = txt.replace('@USE_GFM_PARSER@', '')
+
+    if fi.want_manpage:
+        txt = (txt.replace('@VERSION@', env_subs['VERSION'])
+                  .replace('@BINDIR@', env_subs['bindir'])
+                  .replace('@LIBDIR@', env_subs['libdir']))
+
+    if use_gfm_parser:
+        if not gfm_parser:
+            die('Input file requires cmarkgfm parser:', mdfn)
+        fi.html_in = gfm_parser(txt)
+    else:
+        fi.html_in = md_parser(txt)
+    txt = None
+
+    TransformHtml(fi)
+
+    if args.test:
+        return
+
+    output_list = [ (fi.name + '.html', fi.html_out) ]
+    if fi.want_manpage:
+        output_list += [ (fi.name, fi.man_out) ]
+    for fn, txt in output_list:
+        if args.dest and args.dest != '.':
+            fn = os.path.join(args.dest, fn)
+        if os.path.lexists(fn):
+            os.unlink(fn)
+        print("Wrote:", fn)
+        with open(fn, 'w', encoding='utf-8') as fh:
+            fh.write(txt)
+
+
+def find_man_substitutions():
+    srcdir = os.path.dirname(sys.argv[0]) + '/'
+    mtime = 0
+
+    git_dir = srcdir + '.git'
+    if os.path.lexists(git_dir):
+        mtime = int(subprocess.check_output(['git', '--git-dir', git_dir, 'log', '-1', '--format=%at']))
+
+    # Allow "prefix" to be overridden via the environment:
+    env_subs['prefix'] = os.environ.get('RSYNC_OVERRIDE_PREFIX', None)
+
+    if args.test:
+        env_subs['VERSION'] = '1.0.0'
+        env_subs['bindir'] = '/usr/bin'
+        env_subs['libdir'] = '/usr/lib/rsync'
+    else:
+        for fn in (srcdir + 'version.h', 'Makefile'):
+            try:
+                st = os.lstat(fn)
+            except OSError:
+                die('Failed to find', srcdir + fn)
+            if not mtime:
+                mtime = st.st_mtime
+
+        with open(srcdir + 'version.h', 'r', encoding='utf-8') as fh:
+            txt = fh.read()
+        m = QUOTED_RE.search(txt)
+        env_subs['VERSION'] = m.group(1)
+
+        with open('Makefile', 'r', encoding='utf-8') as fh:
+            for line in fh:
+                m = ASSIGNMENT_RE.match(line)
+                if not m:
+                    continue
+                var, val = (m.group(1), m.group(2))
+                if var == 'prefix' and env_subs[var] is not None:
+                    continue
+                while VAR_REF_RE.search(val):
+                    val = VAR_REF_RE.sub(lambda m: env_subs[m.group(1)], val)
+                env_subs[var] = val
+                if var == 'srcdir':
+                    break
+
+    env_subs['date'] = time.strftime('%d %b %Y', time.localtime(mtime))
+
+
+def html_via_commonmark(txt):
+    return commonmark.HtmlRenderer().render(commonmark.Parser().parse(txt))
+
+
+class TransformHtml(HTMLParser):
+    def __init__(self, fi):
+        HTMLParser.__init__(self, convert_charrefs=True)
+
+        self.fn = fi.fn
+
+        st = self.state = argparse.Namespace(
+                list_state = [ ],
+                p_macro = ".P\n",
+                at_first_tag_in_li = False,
+                at_first_tag_in_dd = False,
+                dt_from = None,
+                in_pre = False,
+                in_code = False,
+                html_out = [ HTML_START.replace('%TITLE%', fi.title) ],
+                man_out = [ ],
+                txt = '',
+                want_manpage = fi.want_manpage,
+                created_hashtags = set(),
+                derived_hashtags = set(),
+                referenced_hashtags = set(),
+                bad_hashtags = set(),
+                latest_targets = [ ],
+                opt_prefix = 'opt',
+                a_txt_start = None,
+                target_suf = '',
+                )
+
+        if st.want_manpage:
+            st.man_out.append(MAN_START % fi.man_headings)
+
+        if '</table>' in fi.html_in:
+            st.html_out[0] = st.html_out[0].replace('</style>', TABLE_STYLE + '</style>')
+
+        self.feed(fi.html_in)
+        fi.html_in = None
+
+        if st.want_manpage:
+            st.html_out.append(MAN_HTML_END % env_subs['date'])
+        st.html_out.append(HTML_END)
+        st.man_out.append(MAN_END)
+
+        fi.html_out = ''.join(st.html_out)
+        st.html_out = None
+
+        fi.man_out = ''.join(st.man_out)
+        st.man_out = None
+
+        for tgt, txt in st.derived_hashtags:
+            derived = txt2target(txt, tgt)
+            if derived not in st.created_hashtags:
+                txt = BIN_CHARS_RE.sub('', txt.replace(NBR_DASH[0], '-').replace(NBR_SPACE[0], ' '))
+                warn('Unknown derived hashtag link in', self.fn, 'based on:', (tgt, txt))
+
+        for bad in st.bad_hashtags:
+            if bad in st.created_hashtags:
+                warn('Missing "#" in hashtag link in', self.fn + ':', bad)
+            else:
+                warn('Unknown non-hashtag link in', self.fn + ':', bad)
+
+        for bad in st.referenced_hashtags - st.created_hashtags:
+            warn('Unknown hashtag link in', self.fn + ':', '#' + bad)
+
+    def handle_starttag(self, tag, attrs_list):
+        st = self.state
+        if args.debug:
+            self.output_debug('START', (tag, attrs_list))
+        if st.at_first_tag_in_li:
+            if st.list_state[-1] == 'dl':
+                st.dt_from = tag
+                if tag == 'p':
+                    tag = 'dt'
+                else:
+                    st.html_out.append('<dt>')
+            elif tag == 'p':
+                st.at_first_tag_in_dd = True # Kluge to suppress a .P at the start of an li.
+            st.at_first_tag_in_li = False
+        if tag == 'p':
+            if not st.at_first_tag_in_dd:
+                st.man_out.append(st.p_macro)
+        elif tag == 'li':
+            st.at_first_tag_in_li = True
+            lstate = st.list_state[-1]
+            if lstate == 'dl':
+                return
+            if lstate == 'o':
+                st.man_out.append(".IP o\n")
+            else:
+                st.man_out.append(".IP " + str(lstate) + ".\n")
+                st.list_state[-1] += 1
+        elif tag == 'blockquote':
+            st.man_out.append(".RS 4\n")
+        elif tag == 'pre':
+            st.in_pre = True
+            st.man_out.append(st.p_macro + ".nf\n")
+        elif tag == 'code' and not st.in_pre:
+            st.in_code = True
+            st.txt += BOLD_FONT[0]
+        elif tag == 'strong' or tag == 'b':
+            st.txt += BOLD_FONT[0]
+        elif tag == 'em' or  tag == 'i':
+            if st.want_manpage:
+                tag = 'u' # Change it into underline to be more like the man page
+                st.txt += UNDR_FONT[0]
+        elif tag == 'ol':
+            start = 1
+            for var, val in attrs_list:
+                if var == 'start':
+                    start = int(val) # We only support integers.
+                    break
+            if st.list_state:
+                st.man_out.append(".RS\n")
+            if start == 0:
+                tag = 'dl'
+                attrs_list = [ ]
+                st.list_state.append('dl')
+            else:
+                st.list_state.append(start)
+            st.man_out.append(st.p_macro)
+            st.p_macro = ".IP\n"
+        elif tag == 'ul':
+            st.man_out.append(st.p_macro)
+            if st.list_state:
+                st.man_out.append(".RS\n")
+                st.p_macro = ".IP\n"
+            st.list_state.append('o')
+        elif tag == 'hr':
+            st.man_out.append(".l\n")
+            st.html_out.append("<hr />")
+            return
+        elif tag == 'a':
+            st.a_href = None
+            for var, val in attrs_list:
+                if var == 'href':
+                    if val.startswith(('https://', 'http://', 'mailto:', 'ftp:')):
+                        pass # nothing to check
+                    elif '#' in val:
+                        pg, tgt = val.split('#', 2)
+                        if pg and pg not in VALID_PAGES or '#' in tgt:
+                            st.bad_hashtags.add(val)
+                        elif tgt in ('', 'opt', 'dopt'):
+                            st.a_href = val
+                        elif pg == '':
+                            st.referenced_hashtags.add(tgt)
+                            if tgt in st.latest_targets:
+                                warn('Found link to the current section in', self.fn + ':', val)
+                    elif val not in VALID_PAGES:
+                        st.bad_hashtags.add(val)
+            st.a_txt_start = len(st.txt)
+        st.html_out.append('<' + tag + ''.join(' ' + var + '="' + htmlify(val) + '"' for var, val in attrs_list) + '>')
+        st.at_first_tag_in_dd = False
+
+
+    def handle_endtag(self, tag):
+        st = self.state
+        if args.debug:
+            self.output_debug('END', (tag,))
+        if tag in CONSUMES_TXT or st.dt_from == tag:
+            txt = st.txt.strip()
+            st.txt = ''
+        else:
+            txt = None
+        add_to_txt = None
+        if tag == 'h1':
+            tgt = txt
+            target_suf = ''
+            if tgt.startswith('NEWS for '):
+                m = VERSION_RE.search(tgt)
+                if m:
+                    tgt = m.group(1)
+                    st.target_suf = '-' + tgt
+            self.add_targets(tag, tgt)
+        elif tag == 'h2':
+            st.man_out.append(st.p_macro + '.SH "' + manify(txt) + '"\n')
+            self.add_targets(tag, txt, st.target_suf)
+            st.opt_prefix = 'dopt' if txt == 'DAEMON OPTIONS' else 'opt'
+        elif tag == 'h3':
+            st.man_out.append(st.p_macro + '.SS "' + manify(txt) + '"\n')
+            self.add_targets(tag, txt, st.target_suf)
+        elif tag == 'p':
+            if st.dt_from == 'p':
+                tag = 'dt'
+                st.man_out.append('.IP "' + manify(txt) + '"\n')
+                if txt.startswith(BOLD_FONT[0]):
+                    self.add_targets(tag, txt)
+                st.dt_from = None
+            elif txt != '':
+                st.man_out.append(manify(txt) + "\n")
+        elif tag == 'li':
+            if st.list_state[-1] == 'dl':
+                if st.at_first_tag_in_li:
+                    die("Invalid 0. -> td translation")
+                tag = 'dd'
+            if txt != '':
+                st.man_out.append(manify(txt) + "\n")
+            st.at_first_tag_in_li = False
+        elif tag == 'blockquote':
+            st.man_out.append(".RE\n")
+        elif tag == 'pre':
+            st.in_pre = False
+            st.man_out.append(manify(txt) + "\n.fi\n")
+        elif (tag == 'code' and not st.in_pre):
+            st.in_code = False
+            add_to_txt = NORM_FONT[0]
+        elif tag == 'strong' or tag == 'b':
+            add_to_txt = NORM_FONT[0]
+        elif tag == 'em' or  tag == 'i':
+            if st.want_manpage:
+                tag = 'u' # Change it into underline to be more like the man page
+                add_to_txt = NORM_FONT[0]
+        elif tag == 'ol' or tag == 'ul':
+            if st.list_state.pop() == 'dl':
+                tag = 'dl'
+            if st.list_state:
+                st.man_out.append(".RE\n")
+            else:
+                st.p_macro = ".P\n"
+            st.at_first_tag_in_dd = False
+        elif tag == 'hr':
+            return
+        elif tag == 'a':
+            if st.a_href:
+                atxt = st.txt[st.a_txt_start:]
+                find = 'href="' + st.a_href + '"'
+                for j in range(len(st.html_out)-1, 0, -1):
+                    if find in st.html_out[j]:
+                        pg, tgt = st.a_href.split('#', 2)
+                        derived = txt2target(atxt, tgt)
+                        if pg == '':
+                            if derived in st.latest_targets:
+                                warn('Found link to the current section in', self.fn + ':', st.a_href)
+                            st.derived_hashtags.add((tgt, atxt))
+                        st.html_out[j] = st.html_out[j].replace(find, 'href="' + pg + '#' + derived + '"')
+                        break
+                else:
+                    die('INTERNAL ERROR: failed to find href in html data:', find)
+        st.html_out.append('</' + tag + '>')
+        if add_to_txt:
+            if txt is None:
+                st.txt += add_to_txt
+            else:
+                txt += add_to_txt
+        if st.dt_from == tag:
+            st.man_out.append('.IP "' + manify(txt) + '"\n')
+            st.html_out.append('</dt><dd>')
+            st.at_first_tag_in_dd = True
+            st.dt_from = None
+        elif tag == 'dt':
+            st.html_out.append('<dd>')
+            st.at_first_tag_in_dd = True
+
+
+    def handle_data(self, txt):
+        st = self.state
+        if '](' in txt:
+            warn('Malformed link in', self.fn + ':', txt)
+        if args.debug:
+            self.output_debug('DATA', (txt,))
+        if st.in_pre:
+            html = htmlify(txt)
+        else:
+            txt = SPACE_DOUBLE_DASH_RE.sub(NBR_SPACE[0] + r'--\1', txt).replace('--', NBR_DASH[0]*2)
+            txt = NON_SPACE_SINGLE_DASH_RE.sub(r'\1' + NBR_DASH[0], txt)
+            html = htmlify(txt)
+            if st.in_code:
+                txt = WHITESPACE_RE.sub(NBR_SPACE[0], txt)
+                html = html.replace(NBR_DASH[0], '-').replace(NBR_SPACE[0], ' ') # <code> is non-breaking in CSS
+        st.html_out.append(html.replace(NBR_SPACE[0], '&nbsp;').replace(NBR_DASH[0], '-&#8288;'))
+        st.txt += txt
+
+
+    def add_targets(self, tag, txt, suf=None):
+        st = self.state
+        tag = '<' + tag + '>'
+        targets = CODE_BLOCK_RE.findall(txt)
+        if not targets:
+            targets = [ txt ]
+        tag_pos = 0
+        for txt in targets:
+            txt = txt2target(txt, st.opt_prefix)
+            if not txt:
+                continue
+            if suf:
+                txt += suf
+            if txt in st.created_hashtags:
+                for j in range(2, 1000):
+                    chk = txt + '-' + str(j)
+                    if chk not in st.created_hashtags:
+                        print('Made link target unique:', chk)
+                        txt = chk
+                        break
+            if tag_pos == 0:
+                tag_pos -= 1
+                while st.html_out[tag_pos] != tag:
+                    tag_pos -= 1
+                st.html_out[tag_pos] = tag[:-1] + ' id="' + txt + '">'
+                st.html_out.append('<a href="#' + txt + '" class="tgt"></a>')
+                tag_pos -= 1 # take into account the append
+            else:
+                st.html_out[tag_pos] = '<span id="' + txt + '"></span>' + st.html_out[tag_pos]
+            st.created_hashtags.add(txt)
+        st.latest_targets = targets
+
+
+    def output_debug(self, event, extra):
+        import pprint
+        st = self.state
+        if args.debug < 2:
+            st = argparse.Namespace(**vars(st))
+            if len(st.html_out) > 2:
+                st.html_out = ['...'] + st.html_out[-2:]
+            if len(st.man_out) > 2:
+                st.man_out = ['...'] + st.man_out[-2:]
+        print(event, extra)
+        pprint.PrettyPrinter(indent=2).pprint(vars(st))
+
+
+def txt2target(txt, opt_prefix):
+    txt = txt.strip().rstrip(':')
+    m = CODE_BLOCK_RE.search(txt)
+    if m:
+        txt = m.group(1)
+    txt = NBR_DASH_RE.sub('-', txt)
+    txt = BIN_CHARS_RE.sub('', txt)
+    txt = INVALID_TARGET_CHARS_RE.sub('_', txt)
+    if opt_prefix and txt.startswith('-'):
+        txt = opt_prefix + txt
+    else:
+        txt = INVALID_START_CHAR_RE.sub(r't\1', txt)
+    return txt
+
+
+def manify(txt):
+    return MANIFY_LINESTART_RE.sub(r'\&\1', txt.replace('\\', '\\\\')
+            .replace(NBR_SPACE[0], NBR_SPACE[1])
+            .replace(NBR_DASH[0], NBR_DASH[1])
+            .replace(NORM_FONT[0], NORM_FONT[1])
+            .replace(BOLD_FONT[0], BOLD_FONT[1])
+            .replace(UNDR_FONT[0], UNDR_FONT[1]))
+
+
+def htmlify(txt):
+    return txt.replace('&', '&amp;').replace('<', '&lt;').replace('>', '&gt;').replace('"', '&quot;')
+
+
+def warn(*msg):
+    print(*msg, file=sys.stderr)
+    global warning_count
+    warning_count += 1
+
+
+def die(*msg):
+    warn(*msg)
+    sys.exit(1)
+
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser(description="Output html and (optionally) nroff for markdown pages.", add_help=False)
+    parser.add_argument('--test', action='store_true', help="Just test the parsing without outputting any files.")
+    parser.add_argument('--dest', metavar='DIR', help="Put files into DIR instead of the current directory.")
+    parser.add_argument('--debug', '-D', action='count', default=0, help='Output copious info on the html parsing. Repeat for even more.')
+    parser.add_argument("--help", "-h", action="help", help="Output this help message and exit.")
+    parser.add_argument("mdfiles", nargs='+', help="The source .md files to convert.")
+    args = parser.parse_args()
+
+    try:
+        import cmarkgfm
+        md_parser = cmarkgfm.markdown_to_html
+        gfm_parser = cmarkgfm.github_flavored_markdown_to_html
+    except:
+        try:
+            import commonmark
+            md_parser = html_via_commonmark
+        except:
+            die("Failed to find cmarkgfm or commonmark for python3.")
+        gfm_parser = None
+
+    main()
+    if warning_count:
+        sys.exit(1)

md2man

@@ -1,383 +0,0 @@
-#!/usr/bin/env python3
-
-# This script takes a manpage written in markdown and turns it into an html web
-# page and a nroff man page.  The input file must have the name of the program
-# and the section in this format: NAME.NUM.md.  The output files are written
-# into the current directory named NAME.NUM.html and NAME.NUM.  The input
-# format has one extra extension: if a numbered list starts at 0, it is turned
-# into a description list. The dl's dt tag is taken from the contents of the
-# first tag inside the li, which is usually a p, code, or strong tag.  The
-# cmarkgfm or commonmark lib is used to transforms the input file into html.
-# The html.parser is used as a state machine that both tweaks the html and
-# outputs the nroff data based on the html tags.
-#
-# Copyright (C) 2020 Wayne Davison
-#
-# This program is freely redistributable.
-
-import sys, os, re, argparse, subprocess, time
-from html.parser import HTMLParser
-
-CONSUMES_TXT = set('h1 h2 p li pre'.split())
-
-HTML_START = """\
-<html><head>
-<title>%s</title>
-<link href="https://fonts.googleapis.com/css2?family=Roboto&family=Roboto+Mono&display=swap" rel="stylesheet">
-<style>
-body {
-  max-width: 50em;
-  margin: auto;
-}
-body, b, strong, u {
-  font-family: 'Roboto', sans-serif;
-}
-code {
-  font-family: 'Roboto Mono', monospace;
-  font-weight: bold;
-  white-space: pre;
-}
-pre code {
-  display: block;
-  font-weight: normal;
-}
-blockquote pre code {
-  background: #f1f1f1;
-}
-dd p:first-of-type {
-  margin-block-start: 0em;
-}
-</style>
-</head><body>
-"""
-
-HTML_END = """\
-<div style="float: right"><p><i>%s</i></p></div>
-</body></html>
-"""
-
-MAN_START = r"""
-.TH "%s" "%s" "%s" "%s" "User Commands"
-""".lstrip()
-
-MAN_END = """\
-"""
-
-NORM_FONT = ('\1', r"\fP")
-BOLD_FONT = ('\2', r"\fB")
-UNDR_FONT = ('\3', r"\fI")
-NBR_DASH = ('\4', r"\-")
-NBR_SPACE = ('\xa0', r"\ ")
-
-md_parser = None
-
-def main():
-    fi = re.match(r'^(?P<fn>(?P<srcdir>.+/)?(?P<name>(?P<prog>[^/]+)\.(?P<sect>\d+))\.md)$', args.mdfile)
-    if not fi:
-        die('Failed to parse NAME.NUM.md out of input file:', args.mdfile)
-    fi = argparse.Namespace(**fi.groupdict())
-
-    if not fi.srcdir:
-        fi.srcdir = './'
-
-    fi.title = fi.prog + '(' + fi.sect + ') man page'
-    fi.mtime = 0
-
-    git_dir = fi.srcdir + '.git'
-    if os.path.lexists(git_dir):
-        fi.mtime = int(subprocess.check_output(['git', '--git-dir', git_dir, 'log', '-1', '--format=%at']))
-
-    env_subs = { 'prefix': os.environ.get('RSYNC_OVERRIDE_PREFIX', None) }
-
-    if args.test:
-        env_subs['VERSION'] = '1.0.0'
-        env_subs['libdir'] = '/usr'
-    else:
-        for fn in (fi.srcdir + 'version.h', 'Makefile'):
-            try:
-                st = os.lstat(fn)
-            except:
-                die('Failed to find', fi.srcdir + fn)
-            if not fi.mtime:
-                fi.mtime = st.st_mtime
-
-        with open(fi.srcdir + 'version.h', 'r', encoding='utf-8') as fh:
-            txt = fh.read()
-        m = re.search(r'"(.+?)"', txt)
-        env_subs['VERSION'] = m.group(1)
-
-        with open('Makefile', 'r', encoding='utf-8') as fh:
-            for line in fh:
-                m = re.match(r'^(\w+)=(.+)', line)
-                if not m:
-                    continue
-                var, val = (m.group(1), m.group(2))
-                if var == 'prefix' and env_subs[var] is not None:
-                    continue
-                while re.search(r'\$\{', val):
-                    val = re.sub(r'\$\{(\w+)\}', lambda m: env_subs[m.group(1)], val)
-                env_subs[var] = val
-                if var == 'srcdir':
-                    break
-
-    with open(fi.fn, 'r', encoding='utf-8') as fh:
-        txt = fh.read()
-
-    txt = re.sub(r'@VERSION@', env_subs['VERSION'], txt)
-    txt = re.sub(r'@LIBDIR@', env_subs['libdir'], txt)
-
-    fi.html_in = md_parser(txt)
-    txt = None
-
-    fi.date = time.strftime('%d %b %Y', time.localtime(fi.mtime))
-    fi.man_headings = (fi.prog, fi.sect, fi.date, fi.prog + ' ' + env_subs['VERSION'])
-
-    HtmlToManPage(fi)
-
-    if args.test:
-        print("The test was successful.")
-        return
-
-    for fn, txt in ((fi.name + '.html', fi.html_out), (fi.name, fi.man_out)):
-        print("Wrote:", fn)
-        with open(fn, 'w', encoding='utf-8') as fh:
-            fh.write(txt)
-
-
-def html_via_commonmark(txt):
-    return commonmark.HtmlRenderer().render(commonmark.Parser().parse(txt))
-
-
-class HtmlToManPage(HTMLParser):
-    def __init__(self, fi):
-        HTMLParser.__init__(self, convert_charrefs=True)
-
-        st = self.state = argparse.Namespace(
-                list_state = [ ],
-                p_macro = ".P\n",
-                at_first_tag_in_li = False,
-                at_first_tag_in_dd = False,
-                dt_from = None,
-                in_pre = False,
-                in_code = False,
-                html_out = [ HTML_START % fi.title ],
-                man_out = [ MAN_START % fi.man_headings ],
-                txt = '',
-                )
-
-        self.feed(fi.html_in)
-        fi.html_in = None
-
-        st.html_out.append(HTML_END % fi.date)
-        st.man_out.append(MAN_END)
-
-        fi.html_out = ''.join(st.html_out)
-        st.html_out = None
-
-        fi.man_out = ''.join(st.man_out)
-        st.man_out = None
-
-
-    def handle_starttag(self, tag, attrs_list):
-        st = self.state
-        if args.debug:
-            self.output_debug('START', (tag, attrs_list))
-        if st.at_first_tag_in_li:
-            if st.list_state[-1] == 'dl':
-                st.dt_from = tag
-                if tag == 'p':
-                    tag = 'dt'
-                else:
-                    st.html_out.append('<dt>')
-            elif tag == 'p':
-                st.at_first_tag_in_dd = True # Kluge to suppress a .P at the start of an li.
-            st.at_first_tag_in_li = False
-        if tag == 'p':
-            if not st.at_first_tag_in_dd:
-                st.man_out.append(st.p_macro)
-        elif tag == 'li':
-            st.at_first_tag_in_li = True
-            lstate = st.list_state[-1]
-            if lstate == 'dl':
-                return
-            if lstate == 'o':
-                st.man_out.append(".IP o\n")
-            else:
-                st.man_out.append(".IP " + str(lstate) + ".\n")
-                st.list_state[-1] += 1
-        elif tag == 'blockquote':
-            st.man_out.append(".RS 4\n")
-        elif tag == 'pre':
-            st.in_pre = True
-            st.man_out.append(st.p_macro + ".nf\n")
-        elif tag == 'code' and not st.in_pre:
-            st.in_code = True
-            st.txt += BOLD_FONT[0]
-        elif tag == 'strong' or tag == 'b':
-            st.txt += BOLD_FONT[0]
-        elif tag == 'em' or  tag == 'i':
-            tag = 'u' # Change it into underline to be more like the man page
-            st.txt += UNDR_FONT[0]
-        elif tag == 'ol':
-            start = 1
-            for var, val in attrs_list:
-                if var == 'start':
-                    start = int(val) # We only support integers.
-                    break
-            if st.list_state:
-                st.man_out.append(".RS\n")
-            if start == 0:
-                tag = 'dl'
-                attrs_list = [ ]
-                st.list_state.append('dl')
-            else:
-                st.list_state.append(start)
-            st.man_out.append(st.p_macro)
-            st.p_macro = ".IP\n"
-        elif tag == 'ul':
-            st.man_out.append(st.p_macro)
-            if st.list_state:
-                st.man_out.append(".RS\n")
-                st.p_macro = ".IP\n"
-            st.list_state.append('o')
-        st.html_out.append('<' + tag + ''.join(' ' + var + '="' + htmlify(val) + '"' for var, val in attrs_list) + '>')
-        st.at_first_tag_in_dd = False
-
-
-    def handle_endtag(self, tag):
-        st = self.state
-        if args.debug:
-            self.output_debug('END', (tag,))
-        if tag in CONSUMES_TXT or st.dt_from == tag:
-            txt = st.txt.strip()
-            st.txt = ''
-        else:
-            txt = None
-        add_to_txt = None
-        if tag == 'h1':
-            st.man_out.append(st.p_macro + '.SH "' + manify(txt) + '"\n')
-        elif tag == 'h2':
-            st.man_out.append(st.p_macro + '.SS "' + manify(txt) + '"\n')
-        elif tag == 'p':
-            if st.dt_from == 'p':
-                tag = 'dt'
-                st.man_out.append('.IP "' + manify(txt) + '"\n')
-                st.dt_from = None
-            elif txt != '':
-                st.man_out.append(manify(txt) + "\n")
-        elif tag == 'li':
-            if st.list_state[-1] == 'dl':
-                if st.at_first_tag_in_li:
-                    die("Invalid 0. -> td translation")
-                tag = 'dd'
-            if txt != '':
-                st.man_out.append(manify(txt) + "\n")
-            st.at_first_tag_in_li = False
-        elif tag == 'blockquote':
-            st.man_out.append(".RE\n")
-        elif tag == 'pre':
-            st.in_pre = False
-            st.man_out.append(manify(txt) + "\n.fi\n")
-        elif (tag == 'code' and not st.in_pre):
-            st.in_code = False
-            add_to_txt = NORM_FONT[0]
-        elif tag == 'strong' or tag == 'b':
-            add_to_txt = NORM_FONT[0]
-        elif tag == 'em' or  tag == 'i':
-            tag = 'u' # Change it into underline to be more like the man page
-            add_to_txt = NORM_FONT[0]
-        elif tag == 'ol' or tag == 'ul':
-            if st.list_state.pop() == 'dl':
-                tag = 'dl'
-            if st.list_state:
-                st.man_out.append(".RE\n")
-            else:
-                st.p_macro = ".P\n"
-            st.at_first_tag_in_dd = False
-        st.html_out.append('</' + tag + '>')
-        if add_to_txt:
-            if txt is None:
-                st.txt += add_to_txt
-            else:
-                txt += add_to_txt
-        if st.dt_from == tag:
-            st.man_out.append('.IP "' + manify(txt) + '"\n')
-            st.html_out.append('</dt><dd>')
-            st.at_first_tag_in_dd = True
-            st.dt_from = None
-        elif tag == 'dt':
-            st.html_out.append('<dd>')
-            st.at_first_tag_in_dd = True
-
-
-    def handle_data(self, txt):
-        st = self.state
-        if args.debug:
-            self.output_debug('DATA', (txt,))
-        if st.in_pre:
-            html = htmlify(txt)
-        else:
-            txt = re.sub(r'\s--(\s)', NBR_SPACE[0] + r'--\1', txt).replace('--', NBR_DASH[0]*2)
-            txt = re.sub(r'(^|\W)-', r'\1' + NBR_DASH[0], txt)
-            html = htmlify(txt)
-            if st.in_code:
-                txt = re.sub(r'\s', NBR_SPACE[0], txt)
-                html = html.replace(NBR_DASH[0], '-').replace(NBR_SPACE[0], ' ') # <code> is non-breaking in CSS
-        st.html_out.append(html.replace(NBR_SPACE[0], '&nbsp;').replace(NBR_DASH[0], '-&#8288;'))
-        st.txt += txt
-
-
-    def output_debug(self, event, extra):
-        import pprint
-        st = self.state
-        if args.debug < 2:
-            st = argparse.Namespace(**vars(st))
-            if len(st.html_out) > 2:
-                st.html_out = ['...'] + st.html_out[-2:]
-            if len(st.man_out) > 2:
-                st.man_out = ['...'] + st.man_out[-2:]
-        print(event, extra)
-        pprint.PrettyPrinter(indent=2).pprint(vars(st))
-
-
-def manify(txt):
-    return re.sub(r"^(['.])", r'\&\1', txt.replace('\\', '\\\\')
-            .replace(NBR_SPACE[0], NBR_SPACE[1])
-            .replace(NBR_DASH[0], NBR_DASH[1])
-            .replace(NORM_FONT[0], NORM_FONT[1])
-            .replace(BOLD_FONT[0], BOLD_FONT[1])
-            .replace(UNDR_FONT[0], UNDR_FONT[1]), flags=re.M)
-
-
-def htmlify(txt):
-    return txt.replace('&', '&amp;').replace('<', '&lt;').replace('>', '&gt;').replace('"', '&quot;')
-
-
-def warn(*msg):
-    print(*msg, file=sys.stderr)
-
-
-def die(*msg):
-    warn(*msg)
-    sys.exit(1)
-
-
-if __name__ == '__main__':
-    parser = argparse.ArgumentParser(description='Transform a NAME.NUM.md markdown file into a NAME.NUM.html web page & a NAME.NUM man page.', add_help=False)
-    parser.add_argument('--test', action='store_true', help='Test if we can parse the input w/o updating any files.')
-    parser.add_argument('--debug', '-D', action='count', default=0, help='Output copious info on the html parsing. Repeat for even more.')
-    parser.add_argument("--help", "-h", action="help", help="Output this help message and exit.")
-    parser.add_argument('mdfile', help="The NAME.NUM.md file to parse.")
-    args = parser.parse_args()
-
-    try:
-        import cmarkgfm
-        md_parser = cmarkgfm.markdown_to_html
-    except:
-        try:
-            import commonmark
-            md_parser = html_via_commonmark
-        except:
-            die("Failed to find cmarkgfm or commonmark for python3.")
-
-    main()

md2man

@@ -0,0 +1 @@
+md-convert

mkgitver

@@ -3,9 +3,18 @@
 srcdir=`dirname $0`
 gitver=`git describe --abbrev=8 2>/dev/null`
 
-if test x"$gitver" != x; then
-    gitver=\""$gitver"\"
-else
-    gitver=RSYNC_VERSION
+if [ ! -f git-version.h ]; then
+    touch git-version.h
 fi
-echo "#define RSYNC_GITVER $gitver" >git-version.h
+
+case "$gitver" in
+    *.*)
+	echo "#define RSYNC_GITVER \"$gitver\"" >git-version.h.new
+	if ! diff git-version.h.new git-version.h >/dev/null; then
+	    echo "Updating git-version.h"
+	    mv git-version.h.new git-version.h
+	else
+	    rm git-version.h.new
+	fi
+	;;
+esac

options.c

@@ -3,7 +3,7 @@
  *
  * Copyright (C) 1998-2001 Andrew Tridgell <tridge@samba.org>
  * Copyright (C) 2000, 2001, 2002 Martin Pool <mbp@samba.org>
- * Copyright (C) 2002-2020 Wayne Davison
+ * Copyright (C) 2002-2022 Wayne Davison
  *
  * This program is free software; you can redistribute it and/or modify
  * it under the terms of the GNU General Public License as published by
@@ -58,14 +58,17 @@ int preserve_devices = 0;
 int preserve_specials = 0;
 int preserve_uid = 0;
 int preserve_gid = 0;
-int preserve_times = 0;
+int preserve_mtimes = 0;
 int preserve_atimes = 0;
 int preserve_crtimes = 0;
+int omit_dir_times = 0;
+int omit_link_times = 0;
 int update_only = 0;
 int open_noatime = 0;
 int cvs_exclude = 0;
 int dry_run = 0;
 int do_xfers = 1;
+int do_fsync = 0;
 int ignore_times = 0;
 int delete_mode = 0;
 int delete_during = 0;
@@ -88,6 +91,7 @@ int implied_dirs = 1;
 int missing_args = 0; /* 0 = FERROR_XFER, 1 = ignore, 2 = delete */
 int numeric_ids = 0;
 int msgs2stderr = 2; /* Default: send errors to stderr for local & remote-shell transfers */
+int saw_stderr_opt = 0;
 int allow_8bit_chars = 0;
 int force_delete = 0;
 int io_timeout = 0;
@@ -98,6 +102,7 @@ int filesfrom_fd = -1;
 char *filesfrom_host = NULL;
 int eol_nulls = 0;
 int protect_args = -1;
+int old_style_args = -1;
 int human_readable = 1;
 int recurse = 0;
 int mkpath_dest_arg = 0;
@@ -227,7 +232,7 @@ static const char *debug_verbosity[] = {
 #define MAX_VERBOSITY ((int)(sizeof debug_verbosity / sizeof debug_verbosity[0]) - 1)
 
 static const char *info_verbosity[1+MAX_VERBOSITY] = {
-	/*0*/ NULL,
+	/*0*/ "NONREG",
 	/*1*/ "COPY,DEL,FLIST,MISC,NAME,STATS,SYMSAFE",
 	/*2*/ "BACKUP,MISC2,MOUNT,NAME2,REMOVE,SKIP",
 };
@@ -265,9 +270,10 @@ static struct output_struct info_words[COUNT_INFO+1] = {
 	INFO_WORD(MISC, W_SND|W_REC, "Mention miscellaneous information (levels 1-2)"),
 	INFO_WORD(MOUNT, W_SND|W_REC, "Mention mounts that were found or skipped"),
 	INFO_WORD(NAME, W_SND|W_REC, "Mention 1) updated file/dir names, 2) unchanged names"),
+	INFO_WORD(NONREG, W_REC, "Mention skipped non-regular files (default 1, 0 disables)"),
 	INFO_WORD(PROGRESS, W_CLI, "Mention 1) per-file progress or 2) total transfer progress"),
 	INFO_WORD(REMOVE, W_SND, "Mention files removed on the sending side"),
-	INFO_WORD(SKIP, W_REC, "Mention files that are skipped due to options used"),
+	INFO_WORD(SKIP, W_REC, "Mention files skipped due to transfer overrides (levels 1-2)"),
 	INFO_WORD(STATS, W_CLI|W_SRV, "Mention statistics at end of run (levels 1-3)"),
 	INFO_WORD(SYMSAFE, W_SND|W_REC, "Mention symlinks that are unsafe"),
 	{ NULL, "--info", 0, 0, 0, 0 }
@@ -307,8 +313,6 @@ static int verbose = 0;
 static int do_stats = 0;
 static int do_progress = 0;
 static int daemon_opt;   /* sets am_daemon after option error-reporting */
-static int omit_dir_times = 0;
-static int omit_link_times = 0;
 static int F_option_cnt = 0;
 static int modify_window_set;
 static int itemize_changes = 0;
@@ -487,9 +491,9 @@ static void output_item_help(struct output_struct *words)
 
 	rprintf(FINFO, fmt, "HELP", "Output this help message");
 	rprintf(FINFO, "\n");
-	rprintf(FINFO, "Options added for each increase in verbose level:\n");
+	rprintf(FINFO, "Options added at each level of verbosity:\n");
 
-	for (j = 1; j <= MAX_VERBOSITY; j++) {
+	for (j = 0; j <= MAX_VERBOSITY; j++) {
 		parse_output_words(words, levels, verbosity[j], HELP_PRIORITY);
 		opt = make_output_option(words, levels, W_CLI|W_SRV|W_SND|W_REC);
 		if (opt) {
@@ -508,7 +512,7 @@ static void set_output_verbosity(int level, uchar priority)
 	if (level > MAX_VERBOSITY)
 		level = MAX_VERBOSITY;
 
-	for (j = 1; j <= level; j++) {
+	for (j = 0; j <= level; j++) {
 		parse_output_words(info_words, info_levels, info_verbosity[j], priority);
 		parse_output_words(debug_words, debug_levels, debug_verbosity[j], priority);
 	}
@@ -527,7 +531,7 @@ void limit_output_verbosity(int level)
 	memset(debug_limits, 0, sizeof debug_limits);
 
 	/* Compute the level limits in the above arrays. */
-	for (j = 1; j <= level; j++) {
+	for (j = 0; j <= level; j++) {
 		parse_output_words(info_words, info_limits, info_verbosity[j], LIMIT_PRIORITY);
 		parse_output_words(debug_words, debug_limits, debug_verbosity[j], LIMIT_PRIORITY);
 	}
@@ -574,7 +578,7 @@ enum {OPT_SERVER = 1000, OPT_DAEMON, OPT_SENDER, OPT_EXCLUDE, OPT_EXCLUDE_FROM,
       OPT_READ_BATCH, OPT_WRITE_BATCH, OPT_ONLY_WRITE_BATCH, OPT_MAX_SIZE,
       OPT_NO_D, OPT_APPEND, OPT_NO_ICONV, OPT_INFO, OPT_DEBUG, OPT_BLOCK_SIZE,
       OPT_USERMAP, OPT_GROUPMAP, OPT_CHOWN, OPT_BWLIMIT, OPT_STDERR,
-      OPT_OLD_COMPRESS, OPT_NEW_COMPRESS, OPT_NO_COMPRESS,
+      OPT_OLD_COMPRESS, OPT_NEW_COMPRESS, OPT_NO_COMPRESS, OPT_OLD_ARGS,
       OPT_STOP_AFTER, OPT_STOP_AT,
       OPT_REFUSED_BASE = 9000};
 
@@ -621,9 +625,9 @@ static struct poptOption long_options[] = {
   {"xattrs",          'X', POPT_ARG_NONE,   0, 'X', 0, 0 },
   {"no-xattrs",        0,  POPT_ARG_VAL,    &preserve_xattrs, 0, 0, 0 },
   {"no-X",             0,  POPT_ARG_VAL,    &preserve_xattrs, 0, 0, 0 },
-  {"times",           't', POPT_ARG_VAL,    &preserve_times, 1, 0, 0 },
-  {"no-times",         0,  POPT_ARG_VAL,    &preserve_times, 0, 0, 0 },
-  {"no-t",             0,  POPT_ARG_VAL,    &preserve_times, 0, 0, 0 },
+  {"times",           't', POPT_ARG_VAL,    &preserve_mtimes, 1, 0, 0 },
+  {"no-times",         0,  POPT_ARG_VAL,    &preserve_mtimes, 0, 0, 0 },
+  {"no-t",             0,  POPT_ARG_VAL,    &preserve_mtimes, 0, 0, 0 },
   {"atimes",          'U', POPT_ARG_NONE,   0, 'U', 0, 0 },
   {"no-atimes",        0,  POPT_ARG_VAL,    &preserve_atimes, 0, 0, 0 },
   {"no-U",             0,  POPT_ARG_VAL,    &preserve_atimes, 0, 0, 0 },
@@ -777,6 +781,8 @@ static struct poptOption long_options[] = {
   {"files-from",       0,  POPT_ARG_STRING, &files_from, 0, 0, 0 },
   {"from0",           '0', POPT_ARG_VAL,    &eol_nulls, 1, 0, 0},
   {"no-from0",         0,  POPT_ARG_VAL,    &eol_nulls, 0, 0, 0},
+  {"old-args",         0,  POPT_ARG_NONE,   0, OPT_OLD_ARGS, 0, 0},
+  {"no-old-args",      0,  POPT_ARG_VAL,    &old_style_args, 0, 0, 0},
   {"protect-args",    's', POPT_ARG_VAL,    &protect_args, 1, 0, 0},
   {"no-protect-args",  0,  POPT_ARG_VAL,    &protect_args, 0, 0, 0},
   {"no-s",             0,  POPT_ARG_VAL,    &protect_args, 0, 0, 0},
@@ -789,6 +795,7 @@ static struct poptOption long_options[] = {
   {"no-timeout",       0,  POPT_ARG_VAL,    &io_timeout, 0, 0, 0 },
   {"contimeout",       0,  POPT_ARG_INT,    &connect_timeout, 0, 0, 0 },
   {"no-contimeout",    0,  POPT_ARG_VAL,    &connect_timeout, 0, 0, 0 },
+  {"fsync",            0,  POPT_ARG_NONE,   &do_fsync, 0, 0, 0 },
   {"stop-after",       0,  POPT_ARG_STRING, 0, OPT_STOP_AFTER, 0, 0 },
   {"time-limit",       0,  POPT_ARG_STRING, 0, OPT_STOP_AFTER, 0, 0 }, /* earlier stop-after name */
   {"stop-at",          0,  POPT_ARG_STRING, 0, OPT_STOP_AT, 0, 0 },
@@ -1516,7 +1523,7 @@ int parse_arguments(int *argc_p, const char ***argv_p)
 			preserve_links = 1;
 #endif
 			preserve_perms = 1;
-			preserve_times = 1;
+			preserve_mtimes = 1;
 			preserve_gid = 1;
 			preserve_uid = 1;
 			preserve_devices = 1;
@@ -1601,6 +1608,13 @@ int parse_arguments(int *argc_p, const char ***argv_p)
 			compress_choice = NULL;
 			break;
 
+		case OPT_OLD_ARGS:
+			if (old_style_args <= 0)
+				old_style_args = 1;
+			else
+				old_style_args++;
+			break;
+
 		case 'M':
 			arg = poptGetOptArg(pc);
 			if (*arg != '-') {
@@ -1745,6 +1759,7 @@ int parse_arguments(int *argc_p, const char ***argv_p)
 			}
 			usermap = (char *)poptGetOptArg(pc);
 			usermap_via_chown = False;
+			preserve_uid = 1;
 			break;
 
 		case OPT_GROUPMAP:
@@ -1760,6 +1775,7 @@ int parse_arguments(int *argc_p, const char ***argv_p)
 			}
 			groupmap = (char *)poptGetOptArg(pc);
 			groupmap_via_chown = False;
+			preserve_gid = 1;
 			break;
 
 		case OPT_CHOWN: {
@@ -1783,6 +1799,7 @@ int parse_arguments(int *argc_p, const char ***argv_p)
 				if (asprintf(&usermap, "*:%.*s", len, chown) < 0)
 					out_of_memory("parse_arguments");
 				usermap_via_chown = True;
+				preserve_uid = 1;
 			}
 			if (arg && *arg) {
 				if (groupmap) {
@@ -1798,6 +1815,7 @@ int parse_arguments(int *argc_p, const char ***argv_p)
 				if (asprintf(&groupmap, "*:%s", arg) < 0)
 					out_of_memory("parse_arguments");
 				groupmap_via_chown = True;
+				preserve_gid = 1;
 			}
 			break;
 		}
@@ -1875,6 +1893,7 @@ int parse_arguments(int *argc_p, const char ***argv_p)
 					"--stderr mode \"%s\" is not one of errors, all, or client\n", arg);
 				return 0;
 			}
+			saw_stderr_opt = 1;
 			break;
 		}
 
@@ -1893,6 +1912,9 @@ int parse_arguments(int *argc_p, const char ***argv_p)
 		}
 	}
 
+	if (msgs2stderr != 2)
+		saw_stderr_opt = 1;
+
 	if (version_opt_cnt) {
 		print_rsync_version(FINFO);
 		exit_cleanup(0);
@@ -1910,6 +1932,21 @@ int parse_arguments(int *argc_p, const char ***argv_p)
 		max_alloc = size;
 	}
 
+	if (old_style_args < 0) {
+		if (!am_server && protect_args <= 0 && (arg = getenv("RSYNC_OLD_ARGS")) != NULL && *arg) {
+			protect_args = 0;
+			old_style_args = atoi(arg);
+		} else
+			old_style_args = 0;
+	} else if (old_style_args) {
+		if (protect_args > 0) {
+			snprintf(err_buf, sizeof err_buf,
+				 "--protect-args conflicts with --old-args.\n");
+			return 0;
+		}
+		protect_args = 0;
+	}
+
 	if (protect_args < 0) {
 		if (am_server)
 			protect_args = 0;
@@ -2255,20 +2292,8 @@ int parse_arguments(int *argc_p, const char ***argv_p)
 		parse_filter_str(&filter_list, backup_dir_buf, rule_template(0), 0);
 	}
 
-	if (preserve_times) {
-		preserve_times = PRESERVE_FILE_TIMES;
-		if (!omit_dir_times)
-			preserve_times |= PRESERVE_DIR_TIMES;
-#ifdef CAN_SET_SYMLINK_TIMES
-		if (!omit_link_times)
-			preserve_times |= PRESERVE_LINK_TIMES;
-#endif
-	}
-
-	if (make_backups && !backup_dir) {
-		omit_dir_times = 0; /* Implied, so avoid -O to sender. */
-		preserve_times &= ~PRESERVE_DIR_TIMES;
-	}
+	if (make_backups && !backup_dir)
+		omit_dir_times = -1; /* Implied, so avoid -O to sender. */
 
 	if (stdout_format) {
 		if (am_server && log_format_has(stdout_format, 'I'))
@@ -2447,6 +2472,61 @@ int parse_arguments(int *argc_p, const char ***argv_p)
 }
 
 
+static char SPLIT_ARG_WHEN_OLD[1];
+
+/**
+ * Do backslash quoting of any weird chars in "arg", append the resulting
+ * string to the end of the "opt" (which gets a "=" appended if it is not
+ * an empty or NULL string), and return the (perhaps malloced) result.
+ * If opt is NULL, arg is considered a filename arg that allows wildcards.
+ * If it is "" or any other value, it is considered an option.
+ **/
+char *safe_arg(const char *opt, const char *arg)
+{
+#define SHELL_CHARS "!#$&;|<>(){}\"' \t\\"
+#define WILD_CHARS  "*?[]" /* We don't allow remote brace expansion */
+	BOOL is_filename_arg = !opt;
+	char *escapes = is_filename_arg ? SHELL_CHARS : WILD_CHARS SHELL_CHARS;
+	BOOL escape_leading_dash = is_filename_arg && *arg == '-';
+	int len1 = opt && *opt ? strlen(opt) + 1 : 0;
+	int len2 = strlen(arg);
+	int extras = escape_leading_dash ? 2 : 0;
+	char *ret;
+	if (!protect_args && old_style_args < 2 && (!old_style_args || (!is_filename_arg && opt != SPLIT_ARG_WHEN_OLD))) {
+		const char *f;
+		for (f = arg; *f; f++) {
+			if (strchr(escapes, *f))
+				extras++;
+		}
+	}
+	if (!len1 && !extras)
+		return (char*)arg;
+	ret = new_array(char, len1 + len2 + extras + 1);
+	if (len1) {
+		memcpy(ret, opt, len1-1);
+		ret[len1-1] = '=';
+	}
+	if (escape_leading_dash) {
+		ret[len1++] = '.';
+		ret[len1++] = '/';
+		extras -= 2;
+	}
+	if (!extras)
+		memcpy(ret + len1, arg, len2);
+	else {
+		const char *f = arg;
+		char *t = ret + len1;
+		while (*f) {
+			if (strchr(escapes, *f))
+				*t++ = '\\';
+			*t++ = *f++;
+		}
+	}
+	ret[len1+len2+extras] = '\0';
+	return ret;
+}
+
+
 /**
  * Construct a filtered list of options to pass through from the
  * client to the server.
@@ -2496,7 +2576,7 @@ void server_options(char **args, int *argc_p)
 			argstr[x++] = 'K';
 		if (prune_empty_dirs)
 			argstr[x++] = 'm';
-		if (omit_dir_times)
+		if (omit_dir_times > 0)
 			argstr[x++] = 'O';
 		if (omit_link_times)
 			argstr[x++] = 'J';
@@ -2529,7 +2609,7 @@ void server_options(char **args, int *argc_p)
 		argstr[x++] = 'g';
 	if (preserve_devices) /* ignore preserve_specials here */
 		argstr[x++] = 'D';
-	if (preserve_times)
+	if (preserve_mtimes)
 		argstr[x++] = 't';
 	if (preserve_atimes) {
 		argstr[x++] = 'U';
@@ -2577,47 +2657,8 @@ void server_options(char **args, int *argc_p)
 
 	set_allow_inc_recurse();
 
-	/* We don't really know the actual protocol_version at this point,
-	 * but checking the pre-negotiated value allows the user to use a
-	 * --protocol=29 override to avoid the use of this -eFLAGS opt. */
-	if (protocol_version >= 30) {
-		/* Use "eFlags" alias so that cull_options doesn't think that these are no-arg option letters. */
-#define eFlags argstr
-		/* We make use of the -e option to let the server know about
-		 * any pre-release protocol version && some behavior flags. */
-		eFlags[x++] = 'e';
-#if SUBPROTOCOL_VERSION != 0
-		if (protocol_version == PROTOCOL_VERSION) {
-			x += snprintf(argstr+x, sizeof argstr - x,
-				      "%d.%d",
-				      PROTOCOL_VERSION, SUBPROTOCOL_VERSION);
-		} else
-#endif
-			eFlags[x++] = '.';
-		if (allow_inc_recurse)
-			eFlags[x++] = 'i';
-#ifdef CAN_SET_SYMLINK_TIMES
-		eFlags[x++] = 'L'; /* symlink time-setting support */
-#endif
-#ifdef ICONV_OPTION
-		eFlags[x++] = 's'; /* symlink iconv translation support */
-#endif
-		eFlags[x++] = 'f'; /* flist I/O-error safety support */
-		eFlags[x++] = 'x'; /* xattr hardlink optimization not desired */
-		eFlags[x++] = 'C'; /* support checksum seed order fix */
-		eFlags[x++] = 'I'; /* support inplace_partial behavior */
-		eFlags[x++] = 'v'; /* use varint for flist & compat flags; negotiate checksum */
-		eFlags[x++] = 'u'; /* include name of uid 0 & gid 0 in the id map */
-		/* NOTE: Avoid using 'V' -- it was the high bit of a write_byte() that became write_varint(). */
-#undef eFlags
-	}
-
-	if (x >= (int)sizeof argstr) { /* Not possible... */
-		rprintf(FERROR, "argstr overflow in server_options().\n");
-		exit_cleanup(RERR_MALLOC);
-	}
-
-	argstr[x] = '\0';
+	/* This '\0'-terminates argstr and makes sure it didn't overflow. */
+	x += maybe_add_e_option(argstr + x, (int)sizeof argstr - x);
 
 	if (x > 1)
 		args[ac++] = argstr;
@@ -2629,9 +2670,7 @@ void server_options(char **args, int *argc_p)
 			set++;
 		else
 			set = iconv_opt;
-		if (asprintf(&arg, "--iconv=%s", set) < 0)
-			goto oom;
-		args[ac++] = arg;
+		args[ac++] = safe_arg("--iconv", set);
 	}
 #endif
 
@@ -2697,33 +2736,24 @@ void server_options(char **args, int *argc_p)
 	}
 
 	if (backup_dir) {
+		/* This split idiom allows for ~/path expansion via the shell. */
 		args[ac++] = "--backup-dir";
-		args[ac++] = backup_dir;
+		args[ac++] = safe_arg("", backup_dir);
 	}
 
 	/* Only send --suffix if it specifies a non-default value. */
-	if (strcmp(backup_suffix, backup_dir ? "" : BACKUP_SUFFIX) != 0) {
-		/* We use the following syntax to avoid weirdness with '~'. */
-		if (asprintf(&arg, "--suffix=%s", backup_suffix) < 0)
-			goto oom;
-		args[ac++] = arg;
-	}
+	if (strcmp(backup_suffix, backup_dir ? "" : BACKUP_SUFFIX) != 0)
+		args[ac++] = safe_arg("--suffix", backup_suffix);
 
-	if (checksum_choice) {
-		if (asprintf(&arg, "--checksum-choice=%s", checksum_choice) < 0)
-			goto oom;
-		args[ac++] = arg;
-	}
+	if (checksum_choice)
+		args[ac++] = safe_arg("--checksum-choice", checksum_choice);
 
 	if (do_compression == CPRES_ZLIBX)
 		args[ac++] = "--new-compress";
 	else if (compress_choice && do_compression == CPRES_ZLIB)
 		args[ac++] = "--old-compress";
-	else if (compress_choice) {
-		if (asprintf(&arg, "--compress-choice=%s", compress_choice) < 0)
-			goto oom;
-		args[ac++] = arg;
-	}
+	else if (compress_choice)
+		args[ac++] = safe_arg("--compress-choice", compress_choice);
 
 	if (am_sender) {
 		if (max_delete > 0) {
@@ -2732,14 +2762,10 @@ void server_options(char **args, int *argc_p)
 			args[ac++] = arg;
 		} else if (max_delete == 0)
 			args[ac++] = "--max-delete=-1";
-		if (min_size >= 0) {
-			args[ac++] = "--min-size";
-			args[ac++] = min_size_arg;
-		}
-		if (max_size >= 0) {
-			args[ac++] = "--max-size";
-			args[ac++] = max_size_arg;
-		}
+		if (min_size >= 0)
+			args[ac++] = safe_arg("--min-size", min_size_arg);
+		if (max_size >= 0)
+			args[ac++] = safe_arg("--max-size", max_size_arg);
 		if (delete_before)
 			args[ac++] = "--delete-before";
 		else if (delete_during == 2)
@@ -2763,17 +2789,12 @@ void server_options(char **args, int *argc_p)
 		if (do_stats)
 			args[ac++] = "--stats";
 	} else {
-		if (skip_compress) {
-			if (asprintf(&arg, "--skip-compress=%s", skip_compress) < 0)
-				goto oom;
-			args[ac++] = arg;
-		}
+		if (skip_compress)
+			args[ac++] = safe_arg("--skip-compress", skip_compress);
 	}
 
-	if (max_alloc_arg && max_alloc != DEFAULT_MAX_ALLOC) {
-		args[ac++] = "--max-alloc";
-		args[ac++] = max_alloc_arg;
-	}
+	if (max_alloc_arg && max_alloc != DEFAULT_MAX_ALLOC)
+		args[ac++] = safe_arg("--max-alloc", max_alloc_arg);
 
 	/* --delete-missing-args needs the cooperation of both sides, but
 	 * the sender can handle --ignore-missing-args by itself. */
@@ -2798,7 +2819,7 @@ void server_options(char **args, int *argc_p)
 	if (partial_dir && am_sender) {
 		if (partial_dir != tmp_partialdir) {
 			args[ac++] = "--partial-dir";
-			args[ac++] = partial_dir;
+			args[ac++] = safe_arg("", partial_dir);
 		}
 		if (delay_updates)
 			args[ac++] = "--delay-updates";
@@ -2821,17 +2842,11 @@ void server_options(char **args, int *argc_p)
 		args[ac++] = "--use-qsort";
 
 	if (am_sender) {
-		if (usermap) {
-			if (asprintf(&arg, "--usermap=%s", usermap) < 0)
-				goto oom;
-			args[ac++] = arg;
-		}
+		if (usermap)
+			args[ac++] = safe_arg("--usermap", usermap);
 
-		if (groupmap) {
-			if (asprintf(&arg, "--groupmap=%s", groupmap) < 0)
-				goto oom;
-			args[ac++] = arg;
-		}
+		if (groupmap)
+			args[ac++] = safe_arg("--groupmap", groupmap);
 
 		if (ignore_existing)
 			args[ac++] = "--ignore-existing";
@@ -2842,9 +2857,12 @@ void server_options(char **args, int *argc_p)
 
 		if (tmpdir) {
 			args[ac++] = "--temp-dir";
-			args[ac++] = tmpdir;
+			args[ac++] = safe_arg("", tmpdir);
 		}
 
+		if (do_fsync)
+			args[ac++] = "--fsync";
+
 		if (basis_dir[0]) {
 			/* the server only needs this option if it is not the sender,
 			 *   and it may be an older version that doesn't know this
@@ -2852,7 +2870,7 @@ void server_options(char **args, int *argc_p)
 			 */
 			for (i = 0; i < basis_dir_cnt; i++) {
 				args[ac++] = alt_dest_opt(0);
-				args[ac++] = basis_dir[i];
+				args[ac++] = safe_arg("", basis_dir[i]);
 			}
 		}
 	}
@@ -2867,13 +2885,17 @@ void server_options(char **args, int *argc_p)
 		if (append_mode > 1)
 			args[ac++] = "--append";
 		args[ac++] = "--append";
-	} else if (inplace)
+	} else if (inplace) {
 		args[ac++] = "--inplace";
+		/* Work around a bug in older rsync versions (on the remote side) for --inplace --sparse */
+		if (sparse_files && !whole_file && am_sender)
+			args[ac++] = "--no-W";
+	}
 
 	if (files_from && (!am_sender || filesfrom_host)) {
 		if (filesfrom_host) {
 			args[ac++] = "--files-from";
-			args[ac++] = files_from;
+			args[ac++] = safe_arg("", files_from);
 			if (eol_nulls)
 				args[ac++] = "--from0";
 		} else {
@@ -2916,7 +2938,7 @@ void server_options(char **args, int *argc_p)
 			exit_cleanup(RERR_SYNTAX);
 		}
 		for (j = 1; j <= remote_option_cnt; j++)
-			args[ac++] = (char*)remote_options[j];
+			args[ac++] = safe_arg(SPLIT_ARG_WHEN_OLD, remote_options[j]);
 	}
 
 	*argc_p = ac;
@@ -2926,6 +2948,52 @@ void server_options(char **args, int *argc_p)
 	out_of_memory("server_options");
 }
 
+int maybe_add_e_option(char *buf, int buf_len)
+{
+	int x = 0;
+
+	/* We don't really know the actual protocol_version at this point,
+	 * but checking the pre-negotiated value allows the user to use a
+	 * --protocol=29 override to avoid the use of this -eFLAGS opt. */
+	if (protocol_version >= 30 && buf_len > 0) {
+		/* We make use of the -e option to let the server know about
+		 * any pre-release protocol version && some behavior flags. */
+		buf[x++] = 'e';
+
+#if SUBPROTOCOL_VERSION != 0
+		if (protocol_version == PROTOCOL_VERSION)
+			x += snprintf(buf + x, buf_len - x, "%d.%d", PROTOCOL_VERSION, SUBPROTOCOL_VERSION);
+		else
+#endif
+			buf[x++] = '.';
+		if (allow_inc_recurse)
+			buf[x++] = 'i';
+#ifdef CAN_SET_SYMLINK_TIMES
+		buf[x++] = 'L'; /* symlink time-setting support */
+#endif
+#ifdef ICONV_OPTION
+		buf[x++] = 's'; /* symlink iconv translation support */
+#endif
+		buf[x++] = 'f'; /* flist I/O-error safety support */
+		buf[x++] = 'x'; /* xattr hardlink optimization not desired */
+		buf[x++] = 'C'; /* support checksum seed order fix */
+		buf[x++] = 'I'; /* support inplace_partial behavior */
+		buf[x++] = 'v'; /* use varint for flist & compat flags; negotiate checksum */
+		buf[x++] = 'u'; /* include name of uid 0 & gid 0 in the id map */
+
+		/* NOTE: Avoid using 'V' -- it was represented with the high bit of a write_byte() that became a write_varint(). */
+	}
+
+	if (x >= buf_len) { /* Not possible... */
+		rprintf(FERROR, "overflow in add_e_flags().\n");
+		exit_cleanup(RERR_MALLOC);
+	}
+
+	buf[x] = '\0';
+
+	return x;
+}
+
 /* If str points to a valid hostspec, return allocated memory containing the
  * [USER@]HOST part of the string, and set the path_start_ptr to the part of
  * the string after the host part.  Otherwise, return NULL.  If port_ptr is

packaging/auto-Makefile

@@ -1,6 +1,6 @@
 TARGETS := all install install-ssl-daemon install-all install-strip conf gen gensend reconfigure restatus \
 	proto man clean cleantests distclean test check check29 check30 installcheck splint \
-	doxygen doxygen-upload finddead
+	doxygen doxygen-upload finddead rrsync
 
 .PHONY: $(TARGETS) auto-prep
 

packaging/cull-options

@@ -0,0 +1,147 @@
+#!/usr/bin/env python3
+# This script outputs either perl or python code that parses all possible options
+# that the code in options.c might send to the server.  The resulting code is then
+# included in the rrsync script.
+
+import re, argparse
+
+short_no_arg = { }
+short_with_num = { '@': 1 }
+long_opts = { # These include some extra long-args that BackupPC uses:
+        'block-size': 1,
+        'daemon': -1,
+        'debug': 1,
+        'fake-super': 0,
+        'fuzzy': 0,
+        'group': 0,
+        'hard-links': 0,
+        'ignore-times': 0,
+        'info': 1,
+        'links': 0,
+        'log-file': 3,
+        'munge-links': 0,
+        'no-munge-links': -1,
+        'one-file-system': 0,
+        'owner': 0,
+        'perms': 0,
+        'recursive': 0,
+        'stderr': 1,
+        'times': 0,
+        'write-devices': -1,
+        }
+
+def main():
+    last_long_opt = None
+
+    with open('../options.c') as fh:
+        for line in fh:
+            m = re.search(r"argstr\[x\+\+\] = '([^.ie])'", line)
+            if m:
+                short_no_arg[m.group(1)] = 1
+                last_long_opt = None
+                continue
+
+            m = re.search(r'asprintf\([^,]+, "-([a-zA-Z0-9])\%l?[ud]"', line)
+            if m:
+                short_with_num[m.group(1)] = 1
+                last_long_opt = None
+                continue
+
+            m = re.search(r'args\[ac\+\+\] = "--([^"=]+)"', line)
+            if m:
+                last_long_opt = m.group(1)
+                if last_long_opt not in long_opts:
+                    long_opts[last_long_opt] = 0
+                else:
+                    last_long_opt = None
+                continue
+
+            if last_long_opt:
+                m = re.search(r'args\[ac\+\+\] = safe_arg\("", ([^[("\s]+)\);', line)
+                if m:
+                    long_opts[last_long_opt] = 2
+                    last_long_opt = None
+                    continue
+                if 'args[ac++] = ' in line:
+                    last_long_opt = None
+
+            m = re.search(r'return "--([^"]+-dest)";', line)
+            if m:
+                long_opts[m.group(1)] = 2
+                last_long_opt = None
+                continue
+
+            m = re.search(r'asprintf\([^,]+, "--([^"=]+)=', line)
+            if not m:
+                m = re.search(r'args\[ac\+\+\] = "--([^"=]+)=', line)
+                if not m:
+                    m = re.search(r'args\[ac\+\+\] = safe_arg\("--([^"=]+)"', line)
+                    if not m:
+                        m = re.search(r'fmt = .*: "--([^"=]+)=', line)
+            if m:
+                long_opts[m.group(1)] = 1
+                last_long_opt = None
+
+    long_opts['files-from'] = 3
+
+    txt = """\
+### START of options data produced by the cull-options script. ###
+
+# To disable a short-named option, add its letter to this string:
+"""
+
+    txt += str_assign('short_disabled', 's') + "\n"
+    txt += '# These are also disabled when the restricted dir is not "/":\n'
+    txt += str_assign('short_disabled_subdir', 'KLk') + "\n"
+    txt += '# These are all possible short options that we will accept (when not disabled above):\n'
+    txt += str_assign('short_no_arg', ''.join(sorted(short_no_arg)), 'DO NOT REMOVE ANY')
+    txt += str_assign('short_with_num', ''.join(sorted(short_with_num)), 'DO NOT REMOVE ANY')
+   
+    txt += """
+# To disable a long-named option, change its value to a -1.  The values mean:
+# 0 = the option has no arg; 1 = the arg doesn't need any checking; 2 = only
+# check the arg when receiving; and 3 = always check the arg.
+"""
+
+    print(txt, end='')
+
+    if args.python:
+        print("long_opts = {")
+        sep = ':'
+    else:
+        print("our %long_opt = (")
+        sep = ' =>'
+
+    for opt in sorted(long_opts):
+        if opt.startswith(('min-', 'max-')):
+            val = 1
+        else:
+            val = long_opts[opt]
+        print(' ', repr(opt) + sep, str(val) + ',')
+
+    if args.python:
+        print("}")
+    else:
+        print(");")
+    print("\n### END of options data produced by the cull-options script. ###")
+
+
+def str_assign(name, val, comment=None):
+    comment = ' # ' + comment if comment else ''
+    if args.python:
+        return name + ' = ' + repr(val) + comment + "\n"
+    return 'our $' + name + ' = ' + repr(val) + ';' + comment + "\n"
+
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser(description="Output culled rsync options for rrsync.", add_help=False)
+    out_group = parser.add_mutually_exclusive_group()
+    out_group.add_argument('--perl', action='store_true', help="Output perl code.")
+    out_group.add_argument('--python', action='store_true', help="Output python code (the default).")
+    parser.add_argument('--help', '-h', action='help', help="Output this help message and exit.")
+    args = parser.parse_args()
+    if not args.perl:
+        args.python = True
+    main()
+
+# vim: sw=4 et

packaging/cull_options

@@ -1,85 +0,0 @@
-#!/usr/bin/env perl
-# This script outputs some perl code that parses all possible options
-# that the code in options.c might send to the server.  This perl code
-# is included in the rrsync script.
-use strict;
-
-our %short_no_arg;
-our %short_with_num = ( '@' => 1 );
-our %long_opt = ( # These include some extra long-args that BackupPC uses:
-    'block-size' => 1,
-    'daemon' => -1,
-    'debug' => 1,
-    'fake-super' => 0,
-    'fuzzy' => 0,
-    'group' => 0,
-    'hard-links' => 0,
-    'ignore-times' => 0,
-    'info' => 1,
-    'links' => 0,
-    'log-file' => 3,
-    'one-file-system' => 0,
-    'owner' => 0,
-    'perms' => 0,
-    'recursive' => 0,
-    'times' => 0,
-    'write-devices' => -1,
-);
-our $last_long_opt;
-
-open(IN, '../options.c') or die "Unable to open ../options.c: $!\n";
-
-while (<IN>) {
-    if (/\Qargstr[x++]\E = '([^.ie])'/) {
-	$short_no_arg{$1} = 1;
-	undef $last_long_opt;
-    } elsif (/\Qasprintf(\E[^,]+, "-([a-zA-Z0-9])\%l?[ud]"/) {
-	$short_with_num{$1} = 1;
-	undef $last_long_opt;
-    } elsif (/\Qargs[ac++]\E = "--([^"=]+)"/) {
-	$last_long_opt = $1;
-	$long_opt{$1} = 0 unless exists $long_opt{$1};
-    } elsif (defined($last_long_opt)
-	&& /\Qargs[ac++]\E = ([^["\s]+);/) {
-	$long_opt{$last_long_opt} = 2;
-	undef $last_long_opt;
-    } elsif (/return "--([^"]+-dest)";/) {
-	$long_opt{$1} = 2;
-	undef $last_long_opt;
-    } elsif (/\Qasprintf(\E[^,]+, "--([^"=]+)=/ || /\Qargs[ac++]\E = "--([^"=]+)=/ || /fmt = .*: "--([^"=]+)=/) {
-	$long_opt{$1} = 1;
-	undef $last_long_opt;
-    }
-}
-close IN;
-
-my $short_no_arg = join('', sort keys %short_no_arg);
-my $short_with_num = join('', sort keys %short_with_num);
-
-print <<EOT;
-
-# These options are the only options that rsync might send to the server,
-# and only in the option format that the stock rsync produces.
-
-# To disable a short-named option, add its letter to this string:
-our \$short_disabled = 's';
-
-our \$short_no_arg = '$short_no_arg'; # DO NOT REMOVE ANY
-our \$short_with_num = '$short_with_num'; # DO NOT REMOVE ANY
-
-# To disable a long-named option, change its value to a -1.  The values mean:
-# 0 = the option has no arg; 1 = the arg doesn't need any checking; 2 = only
-# check the arg when receiving; and 3 = always check the arg.
-our \%long_opt = (
-EOT
-
-foreach my $opt (sort keys %long_opt) {
-    my $val = $long_opt{$opt};
-    $val = 1 if $opt =~ /^(max-|min-)/;
-    $val = 3 if $opt eq 'files-from';
-    $val = q"$only eq 'r' ? -1 : " . $val if $opt =~ /^(remove-|log-file)/;
-    $val = q"$only eq 'w' ? -1 : " . $val if $opt eq 'sender';
-    print "  '$opt' => $val,\n";
-}
-
-print ");\n\n";

packaging/lsb/rsync.spec

@@ -1,9 +1,9 @@
 Summary: A fast, versatile, remote (and local) file-copying tool
 Name: rsync
-Version: 3.2.3
-%define fullversion %{version}
-Release: 1
-%define srcdir src
+Version: 3.2.4
+%define fullversion %{version}pre3
+Release: 0.1.pre3
+%define srcdir src-previews
 Group: Applications/Internet
 License: GPL
 Source0: https://rsync.samba.org/ftp/rsync/%{srcdir}/rsync-%{fullversion}.tar.gz
@@ -79,8 +79,8 @@ rm -rf $RPM_BUILD_ROOT
 %dir /etc/rsync-ssl/certs
 
 %changelog
-* Thu Aug 06 2020 Wayne Davison <wayne@opencoder.net>
-Released 3.2.3.
+* Tue Jan 18 2022 Wayne Davison <wayne@opencoder.net>
+Released 3.2.4pre3.
 
 * Fri Mar 21 2008 Wayne Davison <wayne@opencoder.net>
 Added installation of /etc/xinetd.d/rsync file and some commented-out

packaging/md2html

@@ -1,104 +0,0 @@
-#!/usr/bin/env python3
-
-# Copyright (C) 2020 Wayne Davison
-#
-# This program is freely redistributable.
-
-import os, re, argparse
-
-HTML_START = """\
-<html><head>
-<title>%s</title>
-<link href="https://fonts.googleapis.com/css2?family=Roboto&family=Roboto+Mono&display=swap" rel="stylesheet">
-<style>
-body {
-  max-width: 50em;
-  margin: auto;
-}
-body, b, strong, u {
-  font-family: 'Roboto', sans-serif;
-}
-code {
-  font-family: 'Roboto Mono', monospace;
-  font-weight: bold;
-}
-pre code {
-  display: block;
-  font-weight: normal;
-}
-blockquote pre code {
-  background: #f1f1f1;
-}
-dd p:first-of-type {
-  margin-block-start: 0em;
-}
-table {
-  border-color: grey;
-  border-spacing: 0;
-}
-tr {
-  border-top: 1px solid grey;
-}
-tr:nth-child(2n) {
-  background-color: #f6f8fa;
-}
-th, td {
-  border: 1px solid #dfe2e5;
-  text-align: center;
-  padding-left: 1em;
-  padding-right: 1em;
-}
-</style>
-</head><body>
-"""
-
-HTML_END = """\
-</body></html>
-"""
-
-md_parser = None
-
-def main():
-    for mdfn in args.mdfiles:
-        if not mdfn.endswith('.md'):
-            print('Ignoring non-md input file:', mdfn)
-            continue
-        title = re.sub(r'.*/', '', mdfn).replace('.md', '')
-        htfn = mdfn.replace('.md', '.html')
-
-        print("Parsing", mdfn, '->', htfn)
-
-        with open(mdfn, 'r', encoding='utf-8') as fh:
-            txt = fh.read()
-
-        txt = re.sub(r'\s--\s', '\xa0-- ', txt)
-
-        html = md_parser(txt)
-
-        html = re.sub(r'(?<!<pre>)(<code>)([\s\S]*?)(</code>)', lambda m: m[1] + re.sub(r'\s', '\xa0', m[2]) + m[3], html)
-        html = html.replace('--', '&#8209;&#8209;').replace("\xa0-", '&nbsp;&#8209;').replace("\xa0", '&nbsp;')
-        html = re.sub(r'(\W)-', r'\1&#8209;', html)
-
-        if os.path.lexists(htfn):
-            os.unlink(htfn)
-
-        with open(htfn, 'w', encoding='utf-8') as fh:
-            fh.write(HTML_START % title)
-            fh.write(html)
-            fh.write(HTML_END)
-
-
-if __name__ == '__main__':
-    parser = argparse.ArgumentParser(description='Output html for md pages.', add_help=False)
-    parser.add_argument("--help", "-h", action="help", help="Output this help message and exit.")
-    parser.add_argument("mdfiles", nargs='+', help="The .md files to turn into .html files.")
-    args = parser.parse_args()
-
-    try:
-        import cmarkgfm
-        # Our NEWS.md file has a gfm table in it.
-        md_parser = cmarkgfm.github_flavored_markdown_to_html
-    except:
-        die("Failed to find cmarkgfm for python3.")
-
-    main()

packaging/patch-update

@@ -33,7 +33,7 @@ def main():
 
     master_commit = latest_git_hash(args.base_branch)
 
-    if cmd_txt_chk(['packaging/prep-auto-dir']) == '':
+    if cmd_txt_chk(['packaging/prep-auto-dir']).out == '':
         die('You must setup an auto-build-save dir to use this script.')
 
     if args.gen:

packaging/pkglib.py

@@ -1,4 +1,4 @@
-import os, sys, re, subprocess
+import os, sys, re, subprocess, argparse
 
 # This python3 library provides a few helpful routines that are
 # used by the latest packaging scripts.
@@ -23,27 +23,32 @@ def set_default_encoding(enc):
 
 
 # Set shell=True if the cmd is a string; sets a default encoding unless raw=True was specified.
-def _tweak_opts(cmd, opts, **maybe_set):
-    # This sets any maybe_set value that isn't already set AND creates a copy of opts for us.
-    opts = {**maybe_set, **opts}
+def _tweak_opts(cmd, opts, **maybe_set_args):
+    def _maybe_set(o, **msa): # Only set a value if the user didn't already set it.
+        for var, val in msa.items():
+            if var not in o:
+                o[var] = val
+
+    opts = opts.copy()
+    _maybe_set(opts, **maybe_set_args)
 
     if type(cmd) == str:
-        opts = {'shell': True, **opts}
+        _maybe_set(opts, shell=True)
 
     want_raw = opts.pop('raw', False)
     if default_encoding and not want_raw:
-        opts = {'encoding': default_encoding, **opts}
+        _maybe_set(opts, encoding=default_encoding)
 
     capture = opts.pop('capture', None)
     if capture:
         if capture == 'stdout':
-            opts = {'stdout': subprocess.PIPE, **opts}
+            _maybe_set(opts, stdout=subprocess.PIPE)
         elif capture == 'stderr':
-            opts = {'stderr': subprocess.PIPE, **opts}
+            _maybe_set(opts, stderr=subprocess.PIPE)
         elif capture == 'output':
-            opts = {'stdout': subprocess.PIPE, 'stderr': subprocess.PIPE, **opts}
+            _maybe_set(opts, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
         elif capture == 'combined':
-            opts = {'stdout': subprocess.PIPE, 'stderr': subprocess.STDOUT, **opts}
+            _maybe_set(opts, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
 
     discard = opts.pop('discard', None)
     if discard:
@@ -66,30 +71,26 @@ def cmd_chk(cmd, **opts):
     return subprocess.run(cmd, **_tweak_opts(cmd, opts, check=True))
 
 
-# Capture stdout in a string and return the (output, return_code) tuple.
-# Use capture='combined' opt to get both stdout and stderr together.
-def cmd_txt_status(cmd, **opts):
+# Capture stdout in a string and return an object with out, err, and rc (return code).
+# It defaults to capture='stdout' (so err is empty) but can be overridden using
+# capture='combined' or capture='output' (the latter populates the err value).
+def cmd_txt(cmd, **opts):
     input = opts.pop('input', None)
     if input is not None:
         opts['stdin'] = subprocess.PIPE
     proc = subprocess.Popen(cmd, **_tweak_opts(cmd, opts, capture='stdout'))
-    out = proc.communicate(input=input)[0]
-    return (out, proc.returncode)
+    out, err = proc.communicate(input=input)
+    return argparse.Namespace(out=out, err=err, rc=proc.returncode)
 
 
-# Like cmd_txt_status() but just return the output.
-def cmd_txt(cmd, **opts):
-    return cmd_txt_status(cmd, **opts)[0]
-
-
-# Capture stdout in a string and return the output if the command has a 0 return code.
-# Otherwise it throws an exception that indicates the return code and the output.
+# Just like calling cmd_txt() except that it raises an error if the command has a non-0 return code.
+# The raised error includes the cmd, the return code, and the captured output.
 def cmd_txt_chk(cmd, **opts):
-    out, rc = cmd_txt_status(cmd, **opts)
-    if rc != 0:
-        cmd_err = f'Command "{cmd}" returned non-zero exit status "{rc}" and output:\n{out}'
+    ct = cmd_txt(cmd, **opts)
+    if ct.rc != 0:
+        cmd_err = f'Command "{cmd}" returned non-0 exit status "{ct.rc}" and output:\n{ct.out}{ct.err}'
         raise Exception(cmd_err)
-    return out
+    return ct
 
 
 # Starts a piped-output command of stdout (by default) and leaves it up to you to read
@@ -102,7 +103,7 @@ def cmd_pipe(cmd, **opts):
 # arg fatal_unless_clean can be used to make that non-fatal.  Returns a
 # tuple of the current branch, the is_clean flag, and the status text.
 def check_git_status(fatal_unless_clean=True, subdir='.'):
-    status_txt = cmd_txt_chk(f"cd '{subdir}' && git status")
+    status_txt = cmd_txt_chk(f"cd '{subdir}' && git status").out
     is_clean = re.search(r'\nnothing to commit.+working (directory|tree) clean', status_txt) != None
 
     if not is_clean and fatal_unless_clean:
@@ -149,7 +150,7 @@ def check_git_state(master_branch, fatal_unless_clean=True, check_extra_dir=None
 
 # Return the git hash of the most recent commit.
 def latest_git_hash(branch):
-    out = cmd_txt_chk(['git', 'log', '-1', '--no-color', branch])
+    out = cmd_txt_chk(['git', 'log', '-1', '--no-color', branch]).out
     m = re.search(r'^commit (\S+)', out, flags=re.M)
     if not m:
         die(f"Unable to determine commit hash for master branch: {branch}")
@@ -175,20 +176,20 @@ def mandate_gensend_hook():
         print('Creating hook file:', hook)
         cmd_chk(['./rsync', '-a', 'packaging/pre-push', hook])
     else:
-        out, rc = cmd_txt_status(['fgrep', 'make gensend', hook], discard='output')
-        if rc:
+        ct = cmd_txt(['fgrep', 'make gensend', hook], discard='output')
+        if ct.rc:
             die('Please add a "make gensend" into your', hook, 'script.')
 
 
-# Snag the GENFILES values out of the Makefile.in file and return them as a list.
+# Snag the GENFILES values out of the Makefile file and return them as a list.
 def get_gen_files(want_dir_plus_list=False):
     cont_re = re.compile(r'\\\n')
 
     gen_files = [ ]
 
-    auto_dir = os.path.join('auto-build-save', cmd_txt('git rev-parse --abbrev-ref HEAD').strip().replace('/', '%'))
+    auto_dir = os.path.join('auto-build-save', cmd_txt('git rev-parse --abbrev-ref HEAD').out.strip().replace('/', '%'))
 
-    with open('Makefile.in', 'r', encoding='utf-8') as fh:
+    with open(auto_dir + '/Makefile', 'r', encoding='utf-8') as fh:
         for line in fh:
             if not gen_files:
                 chk = re.sub(r'^GENFILES=', '', line)

packaging/pre-push

@@ -1,3 +1,16 @@
-#!/bin/sh
+#!/bin/bash -e
+
 cat >/dev/null # Just discard stdin data
-make gensend
+
+if [[ -f /proc/$PPID/cmdline ]]; then
+    while read -d $'\0' arg ; do
+	if [[ "$arg" == '--tags' ]] ; then
+	    exit 0
+	fi
+    done </proc/$PPID/cmdline
+fi
+
+branch=`git rev-parse --abbrev-ref HEAD`
+if [[ "$branch" = master && "$*" == *github* ]]; then
+    make gensend
+fi

packaging/prep-auto-dir

@@ -13,7 +13,7 @@
 # run "make distclean" before creating the auto-build-save dir.
 
 auto_top='auto-build-save'
-if test -d $auto_top -a -d .git; then
+if test -d $auto_top && test -d .git; then
     desired_branch=`git rev-parse --abbrev-ref HEAD | tr / %`
     if test "$desired_branch" = HEAD; then
 	echo "ERROR: switch to the right build dir manually when in detached HEAD mode." 1>&2

packaging/release-rsync

@@ -18,6 +18,9 @@ dest = os.environ['HOME'] + '/samba-rsync-ftp'
 ORIGINAL_PATH = os.environ['PATH']
 
 def main():
+    if not os.path.isfile('packaging/release-rsync'):
+        die('You must run this script from the top of your rsync checkout.')
+
     now = datetime.now()
     cl_today = now.strftime('* %a %b %d %Y')
     year = now.strftime('%Y')
@@ -30,7 +33,7 @@ def main():
 
     signal.signal(signal.SIGINT, signal_handler)
 
-    if cmd_txt_chk(['packaging/prep-auto-dir']) == '':
+    if cmd_txt_chk(['packaging/prep-auto-dir']).out == '':
         die('You must setup an auto-build-save dir to use this script.');
 
     auto_dir, gen_files = get_gen_files(True)
@@ -44,6 +47,17 @@ def main():
 {dash_line}
 """)
 
+    with open('build/rsync.1') as fh:
+        for line in fh:
+            if line.startswith(r'.\" prefix='):
+                doc_prefix = line.split('=')[1].strip()
+                if doc_prefix != '/usr':
+                    warn(f"*** The documentation was built with prefix {doc_prefix} instead of /usr ***")
+                    die("*** Read the md2man script for a way to override this. ***")
+                break
+            if line.startswith('.P'):
+                die("Failed to find the prefix comment at the start of the rsync.1 manpage.")
+
     if not os.path.isdir(dest):
         die(dest, "dest does not exist")
     if not os.path.isdir('.git'):
@@ -84,7 +98,7 @@ def main():
     if os.path.lexists(rsync_ver):
         die(f'"{rsync_ver}" must not exist in the current directory.')
 
-    out = cmd_txt_chk(['git', 'tag', '-l', v_ver])
+    out = cmd_txt_chk(['git', 'tag', '-l', v_ver]).out
     if out != '':
         print(f"Tag {v_ver} already exists.")
         ans = input("\nDelete tag or quit? [Q/del] ")
@@ -192,10 +206,10 @@ About to:
             txt = replace_or_die(x_re, repl, txt, f"Unable to find SUBPROTOCOL_VERSION define in {fn}")
         elif fn == 'NEWS.md':
             efv = re.escape(finalversion)
-            x_re = re.compile(r'^<.+>\s+# NEWS for rsync %s \(UNRELEASED\)\s+## Changes in this version:\n' % efv
+            x_re = re.compile(r'^# NEWS for rsync %s \(UNRELEASED\)\s+## Changes in this version:\n' % efv
                     + r'(\n### PROTOCOL NUMBER:\s+- The protocol number was changed to \d+\.\n)?')
             rel_day = 'UNRELEASED' if pre else today
-            repl = (f'<a name="{finalversion}"></a>\n\n# NEWS for rsync {finalversion} ({rel_day})\n\n'
+            repl = (f'# NEWS for rsync {finalversion} ({rel_day})\n\n'
                 + '## Changes in this version:\n')
             if proto_changed:
                 repl += f'\n### PROTOCOL NUMBER:\n\n - The protocol number was changed to {protocol_version}.\n'
@@ -216,7 +230,7 @@ About to:
     cmd_chk(['packaging/year-tweak'])
 
     print(dash_line)
-    cmd_run("git diff --color | less -p '^diff .*'")
+    cmd_run("git diff")
 
     srctar_name = f"{rsync_ver}.tar.gz"
     pattar_name = f"rsync-patches-{version}.tar.gz"
@@ -282,13 +296,13 @@ About to:
     # TODO: is there a better way to ensure that our passphrase is in the agent?
     cmd_run("touch TeMp; gpg --sign TeMp; rm TeMp*")
 
-    out = cmd_txt(f"git tag -s -m 'Version {version}.' {v_ver}", capture='combined')
+    out = cmd_txt(f"git tag -s -m 'Version {version}.' {v_ver}", capture='combined').out
     print(out, end='')
     if 'bad passphrase' in out or 'failed' in out:
         die('Aborting')
 
     if os.path.isdir('patches/.git'):
-        out = cmd_txt(f"cd patches && git tag -s -m 'Version {version}.' {v_ver}", capture='combined')
+        out = cmd_txt(f"cd patches && git tag -s -m 'Version {version}.' {v_ver}", capture='combined').out
         print(out, end='')
         if 'bad passphrase' in out or 'failed' in out:
             die('Aborting')
@@ -327,7 +341,7 @@ About to:
     md_files = 'README.md NEWS.md INSTALL.md'.split()
     html_files = [ fn for fn in gen_pathnames if fn.endswith('.html') ]
     cmd_chk(['rsync', '-a', *md_files, *html_files, dest])
-    cmd_chk(["packaging/md2html"] + [ dest +'/'+ fn for fn in md_files ])
+    cmd_chk(["./md-convert", "--dest", dest, *md_files])
 
     cmd_chk(f"git log --name-status | gzip -9 >{dest}/ChangeLog.gz")
 

packaging/smart-make

@@ -4,8 +4,6 @@ set -e
 
 export LANG=C
 
-make=`which gmake 2>/dev/null` || make=`which make 2>/dev/null`
-
 branch=`packaging/prep-auto-dir`
 if test x"$branch" = x; then
     srcdir=.
@@ -40,8 +38,8 @@ fi
 
 ./config.status
 
-$make all
+make all
 
 if test x"$1" = x"check"; then
-    $make check
+    make check
 fi

packaging/var-checker

@@ -4,7 +4,7 @@
 # for vars that are defined but not used, and for inconsistent array
 # sizes.  Run it from inside the main rsync directory.
 
-import re, argparse, glob
+import os, sys, re, argparse, glob
 
 VARS_RE = re.compile(r'^(?!(?:extern|enum)\s)([a-zA-Z]\S*\s+.*);', re.M)
 EXTERNS_RE = re.compile(r'^extern\s+(.*);', re.M)
@@ -15,8 +15,15 @@ def main():
     add_syscall_c = set('t_stub.c t_unsafe.c tls.c trimslash.c'.split())
     add_util_c = set('t_stub.c t_unsafe.c'.split())
 
+    if not os.path.exists('syscall.c'):
+        if os.path.exists('var-checker'):
+            os.chdir('..')
+        else:
+            print("Couldn't find the source dir.")
+            sys.exit(1)
+
     syscall_c = slurp_file('syscall.c', True)
-    util_c = slurp_file('util.c', True)
+    util_c = slurp_file('util1.c', True)
 
     for fn in sorted(glob.glob('*.c')):
         txt = slurp_file(fn)

prepare-source

@@ -32,7 +32,7 @@ if test "$dir" != '.'; then
 	fi
     done
     for fn in configure.sh config.h.in aclocal.m4; do
-	test ! -f $fn -a -f "$dir/$fn" && cp -p "$dir/$fn" $fn
+	test ! -f $fn && test -f "$dir/$fn" && cp -p "$dir/$fn" $fn
     done
 fi
 

progress.c

@@ -4,7 +4,7 @@
  * Copyright (C) 1996-2000 Andrew Tridgell
  * Copyright (C) 1996 Paul Mackerras
  * Copyright (C) 2001, 2002 Martin Pool <mbp@samba.org>
- * Copyright (C) 2003-2020 Wayne Davison
+ * Copyright (C) 2003-2022 Wayne Davison
  *
  * This program is free software; you can redistribute it and/or modify
  * it under the terms of the GNU General Public License as published by
@@ -115,13 +115,13 @@ static void rprint_progress(OFF_T ofs, OFF_T size, struct timeval *now, int is_l
 		units = "kB/s";
 	}
 
-	if (remain < 0)
+	if (remain < 0 || remain > 9999.0 * 3600.0)
 		strlcpy(rembuf, "  ??:??:??", sizeof rembuf);
 	else {
-		snprintf(rembuf, sizeof rembuf, "%4d:%02d:%02d",
-			 (int) (remain / 3600.0),
-			 (int) (remain / 60.0) % 60,
-			 (int) remain % 60);
+		snprintf(rembuf, sizeof rembuf, "%4u:%02u:%02u",
+			 (unsigned int) (remain / 3600.0),
+			 (unsigned int) (remain / 60.0) % 60,
+			 (unsigned int) remain % 60);
 	}
 
 	output_needs_newline = 0;

receiver.c

@@ -3,7 +3,7 @@
  *
  * Copyright (C) 1996-2000 Andrew Tridgell
  * Copyright (C) 1996 Paul Mackerras
- * Copyright (C) 2003-2020 Wayne Davison
+ * Copyright (C) 2003-2022 Wayne Davison
  *
  * This program is free software; you can redistribute it and/or modify
  * it under the terms of the GNU General Public License as published by
@@ -41,6 +41,7 @@ extern int preserve_hard_links;
 extern int preserve_perms;
 extern int write_devices;
 extern int preserve_xattrs;
+extern int do_fsync;
 extern int basis_dir_cnt;
 extern int make_backups;
 extern int cleanup_got_literal;
@@ -394,6 +395,11 @@ static int receive_data(int f_in, char *fname_r, int fd_r, OFF_T size_r,
 
 	sum_len = sum_end(file_sum1);
 
+	if (do_fsync && fd != -1 && fsync(fd) != 0) {
+		rsyserr(FERROR, errno, "fsync failed on %s", full_fname(fname));
+		exit_cleanup(RERR_FILEIO);
+	}
+
 	if (mapbuf)
 		unmap_file(mapbuf);
 
@@ -539,6 +545,9 @@ int recv_files(int f_in, int f_out, char *local_name)
 	if (delay_updates)
 		delayed_bits = bitbag_create(cur_flist->used + 1);
 
+	if (whole_file < 0)
+		whole_file = 0;
+
 	progress_init();
 
 	while (1) {
@@ -826,6 +835,12 @@ int recv_files(int f_in, int f_out, char *local_name)
 		if (inplace || one_inplace)  {
 			fnametmp = one_inplace ? partialptr : fname;
 			fd2 = do_open(fnametmp, O_WRONLY|O_CREAT, 0600);
+#ifdef linux
+			if (fd2 == -1 && errno == EACCES) {
+				/* Maybe the error was due to protected_regular setting? */
+				fd2 = do_open(fname, O_WRONLY, 0600);
+			}
+#endif
 			if (fd2 == -1) {
 				rsyserr(FERROR_XFER, errno, "open %s failed",
 					full_fname(fnametmp));
@@ -878,7 +893,7 @@ int recv_files(int f_in, int f_out, char *local_name)
 					do_unlink(partialptr);
 				handle_partial_dir(partialptr, PDIR_DELETE);
 			}
-		} else if (keep_partial && partialptr && !one_inplace) {
+		} else if (keep_partial && partialptr && (!one_inplace || delay_updates)) {
 			if (!handle_partial_dir(partialptr, PDIR_CREATE)) {
 				rprintf(FERROR,
 					"Unable to create partial-dir for %s -- discarding %s.\n",
@@ -912,7 +927,7 @@ int recv_files(int f_in, int f_out, char *local_name)
 			break;
 		case 0: {
 			enum logcode msgtype = redoing ? FERROR_XFER : FWARNING;
-			if (msgtype == FERROR_XFER || INFO_GTE(NAME, 1)) {
+			if (msgtype == FERROR_XFER || INFO_GTE(NAME, 1) || stdout_format_has_i) {
 				char *errstr, *redostr, *keptstr;
 				if (!(keep_partial && partialptr) && !inplace)
 					keptstr = "discarded";

rsync-ssl

@@ -73,8 +73,16 @@ function rsync_ssl_helper {
 	certopt=""
 	gnutls_cert_opt=""
     else
-	certopt="cert$optsep$RSYNC_SSL_CERT"
-	gnutls_cert_opt="--x509keyfile=$RSYNC_SSL_CERT"
+	certopt="-cert$optsep$RSYNC_SSL_CERT"
+	gnutls_cert_opt="--x509certfile=$RSYNC_SSL_CERT"
+    fi
+
+    if [[ -z "$RSYNC_SSL_KEY" ]]; then
+	keyopt=""
+	gnutls_key_opt=""
+    else
+	keyopt="-key$optsep$RSYNC_SSL_KEY"
+	gnutls_key_opt="--x509keyfile=$RSYNC_SSL_KEY"
     fi
 
     if [[ -z ${RSYNC_SSL_CA_CERT+x} ]]; then
@@ -129,9 +137,9 @@ function rsync_ssl_helper {
     fi
 
     if [[ $RSYNC_SSL_TYPE == openssl ]]; then
-	exec $RSYNC_SSL_OPENSSL s_client $caopt $certopt -quiet -verify_quiet -servername $hostname -connect $hostname:$port
+	exec $RSYNC_SSL_OPENSSL s_client $caopt $certopt $keyopt -quiet -verify_quiet -servername $hostname -verify_hostname $hostname -connect $hostname:$port
     elif [[ $RSYNC_SSL_TYPE == gnutls ]]; then
-	exec $RSYNC_SSL_GNUTLS --logfile=/dev/null $gnutls_cert_opt $gnutls_opts $hostname:$port
+	exec $RSYNC_SSL_GNUTLS --logfile=/dev/null $gnutls_cert_opt $gnutls_key_opt $gnutls_opts $hostname:$port
     else
 	# devzero@web.de came up with this no-tmpfile calling syntax:
 	exec $RSYNC_SSL_STUNNEL -fd 10 11<&0 <<EOF 10<&0 0<&11 11<&-

rsync-ssl.1.md

@@ -1,14 +1,17 @@
-# NAME
+## NAME
 
 rsync-ssl - a helper script for connecting to an ssl rsync daemon
 
-# SYNOPSIS
+## SYNOPSIS
 
 ```
 rsync-ssl [--type=SSL_TYPE] RSYNC_ARGS
 ```
 
-# DESCRIPTION
+The online version of this man page (that includes cross-linking of topics)
+is available at <https://download.samba.org/pub/rsync/rsync-ssl.1>.
+
+## DESCRIPTION
 
 The rsync-ssl script helps you to run an rsync copy to/from an rsync daemon
 that requires ssl connections.
@@ -20,7 +23,7 @@ environment.  You can specify an overriding port via `--port` or by including
 it in the normal spot in the URL format, though both of those require your
 rsync version to be at least 3.2.0.
 
-# OPTIONS
+## OPTIONS
 
 If the **first** arg is a `--type=SSL_TYPE` option, the script will only use
 that particular program to open an ssl connection instead of trying to find an
@@ -32,33 +35,56 @@ required for this particular option.
 All the other options are passed through to the rsync command, so consult the
 **rsync**(1) manpage for more information on how it works.
 
-# ENVIRONMENT VARIABLES
+## ENVIRONMENT VARIABLES
 
 The ssl helper scripts are affected by the following environment variables:
 
-0.  `RSYNC_SSL_TYPE` Specifies the program type that should be used to open the
-    ssl connection.  It must be one of `openssl` or `stunnel`.  The
-    `--type=SSL_TYPE` option overrides this, when specified.
-0.  `RSYNC_SSL_PORT` If specified, the value is the port number that is used as
-    the default when the user does not specify a port in their rsync command.
-    When not specified, the default port number is 874.  (Note that older rsync
-    versions (prior to 3.2.0) did not communicate an overriding port number
-    value to the helper script.)
-0.  `RSYNC_SSL_CERT` If specified, the value is a filename that contains a
-    certificate to use for the connection.
-0.  `RSYNC_SSL_CA_CERT` If specified, the value is a filename that contains a
-    certificate authority certificate that is used to validate the connection.
-0.  `RSYNC_SSL_OPENSSL` Specifies the openssl executable to run when the
-    connection type is set to openssl.  If unspecified, the $PATH is searched
-    for "openssl".
-0.  `RSYNC_SSL_GNUTLS` Specifies the gnutls-cli executable to run when the
-    connection type is set to gnutls.  If unspecified, the $PATH is searched
-    for "gnutls-cli".
-0.  `RSYNC_SSL_STUNNEL` Specifies the stunnel executable to run when the
-    connection type is set to stunnel.  If unspecified, the $PATH is searched
-    first for "stunnel4" and then for "stunnel".
+0.  `RSYNC_SSL_TYPE`
 
-# EXAMPLES
+    Specifies the program type that should be used to open the ssl connection.
+    It must be one of `openssl` or `stunnel`.  The `--type=SSL_TYPE` option
+    overrides this, when specified.
+
+0.  `RSYNC_SSL_PORT`
+
+    If specified, the value is the port number that is used as the default when
+    the user does not specify a port in their rsync command.  When not
+    specified, the default port number is 874.  (Note that older rsync versions
+    (prior to 3.2.0) did not communicate an overriding port number value to the
+    helper script.)
+
+0.  `RSYNC_SSL_CERT`
+
+    If specified, the value is a filename that contains a certificate to use
+    for the connection.
+
+0.  `RSYNC_SSL_KEY`
+
+    If specified, the value is a filename that contains a key for the provided
+    certificate to use for the connection.
+
+0.  `RSYNC_SSL_CA_CERT`
+
+    If specified, the value is a filename that contains a certificate authority
+    certificate that is used to validate the connection.
+
+0.  `RSYNC_SSL_OPENSSL`
+
+    Specifies the openssl executable to run when the connection type is set to
+    openssl.  If unspecified, the $PATH is searched for "openssl".
+
+0.  `RSYNC_SSL_GNUTLS`
+
+    Specifies the gnutls-cli executable to run when the connection type is set
+    to gnutls.  If unspecified, the $PATH is searched for "gnutls-cli".
+
+0.  `RSYNC_SSL_STUNNEL`
+
+    Specifies the stunnel executable to run when the connection type is set to
+    stunnel.  If unspecified, the $PATH is searched first for "stunnel4" and
+    then for "stunnel".
+
+## EXAMPLES
 
 >     rsync-ssl -aiv example.com::mod/ dest
 
@@ -68,11 +94,11 @@ The ssl helper scripts are affected by the following environment variables:
 
 >     rsync-ssl -aiv rsync://example.com:9874/mod/ dest
 
-# SEE ALSO
+## SEE ALSO
 
-**rsync**(1), **rsyncd.conf**(5)
+[**rsync**(1)](rsync.1), [**rsyncd.conf**(5)](rsyncd.conf.5)
 
-# CAVEATS
+## CAVEATS
 
 Note that using an stunnel connection requires at least version 4 of stunnel,
 which should be the case on modern systems.  Also, it does not verify a
@@ -85,23 +111,23 @@ release the gnutls-cli command was dropping output, making it unusable.  If
 that bug has been fixed in your version, feel free to put gnutls into an
 exported RSYNC_SSL_TYPE environment variable to make its use the default.
 
-# BUGS
+## BUGS
 
 Please report bugs! See the web site at <https://rsync.samba.org/>.
 
-# VERSION
+## VERSION
 
 This man page is current for version @VERSION@ of rsync.
 
-# CREDITS
+## CREDITS
 
-rsync is distributed under the GNU General Public License.  See the file
-COPYING for details.
+Rsync is distributed under the GNU General Public License.  See the file
+[COPYING](COPYING) for details.
 
 A web site is available at <https://rsync.samba.org/>.  The site includes an
 FAQ-O-Matic which may cover questions unanswered by this manual page.
 
-# AUTHOR
+## AUTHOR
 
 This manpage was written by Wayne Davison.
 

rsync.c

@@ -3,7 +3,7 @@
  *
  * Copyright (C) 1996 Andrew Tridgell
  * Copyright (C) 1996 Paul Mackerras
- * Copyright (C) 2003-2020 Wayne Davison
+ * Copyright (C) 2003-2022 Wayne Davison
  *
  * This program is free software; you can redistribute it and/or modify
  * it under the terms of the GNU General Public License as published by
@@ -32,7 +32,9 @@ extern int preserve_acls;
 extern int preserve_xattrs;
 extern int preserve_perms;
 extern int preserve_executability;
-extern int preserve_times;
+extern int preserve_mtimes;
+extern int omit_dir_times;
+extern int omit_link_times;
 extern int am_root;
 extern int am_server;
 extern int am_daemon;
@@ -63,8 +65,7 @@ extern char *iconv_opt;
 #define UPDATED_ATIME (1<<3)
 #define UPDATED_ACLS  (1<<4)
 #define UPDATED_MODE  (1<<5)
-
-#define UPDATED_TIMES (UPDATED_MTIME|UPDATED_ATIME)
+#define UPDATED_CRTIME (1<<6)
 
 #ifdef ICONV_CONST
 iconv_t ic_chck = (iconv_t)-1;
@@ -576,17 +577,20 @@ int set_file_attrs(const char *fname, struct file_struct *file, stat_x *sxp,
 		set_xattr(fname, file, fnamecmp, sxp);
 #endif
 
-	if (!preserve_times
-	 || (!(preserve_times & PRESERVE_DIR_TIMES) && S_ISDIR(sxp->st.st_mode))
-	 || (!(preserve_times & PRESERVE_LINK_TIMES) && S_ISLNK(sxp->st.st_mode)))
-		flags |= ATTRS_SKIP_MTIME | ATTRS_SKIP_ATIME;
-	else if (sxp != &sx2)
-		memcpy(&sx2.st, &sxp->st, sizeof (sx2.st));
-	if (!atimes_ndx || S_ISDIR(sxp->st.st_mode))
-		flags |= ATTRS_SKIP_ATIME;
-	/* Don't set the creation date on the root folder of an HFS+ volume. */
-	if (sxp->st.st_ino == 2 && S_ISDIR(sxp->st.st_mode))
-		flags |= ATTRS_SKIP_CRTIME;
+	if ((omit_dir_times && S_ISDIR(sxp->st.st_mode))
+	 || (omit_link_times && S_ISLNK(sxp->st.st_mode)))
+		flags |= ATTRS_SKIP_MTIME | ATTRS_SKIP_ATIME | ATTRS_SKIP_CRTIME;
+	else {
+		if (!preserve_mtimes)
+			flags |= ATTRS_SKIP_MTIME;
+		if (!atimes_ndx || S_ISDIR(sxp->st.st_mode))
+			flags |= ATTRS_SKIP_ATIME;
+		/* Don't set the creation date on the root folder of an HFS+ volume. */
+		if (sxp->st.st_ino == 2 && S_ISDIR(sxp->st.st_mode))
+			flags |= ATTRS_SKIP_CRTIME;
+	}
+	if (sxp != &sx2)
+		memcpy(&sx2.st, &sxp->st, sizeof sx2.st);
 	if (!(flags & ATTRS_SKIP_MTIME) && !same_mtime(file, &sxp->st, flags & ATTRS_ACCURATE_TIME)) {
 		sx2.st.st_mtime = file->modtime;
 #ifdef ST_MTIME_NSEC
@@ -604,28 +608,36 @@ int set_file_attrs(const char *fname, struct file_struct *file, stat_x *sxp,
 			updated |= UPDATED_ATIME;
 		}
 	}
-	if (updated & UPDATED_TIMES) {
-		int ret = set_times(fname, &sx2.st);
-		if (ret < 0) {
-			rsyserr(FERROR_XFER, errno, "failed to set times on %s",
-				full_fname(fname));
-			goto cleanup;
-		}
-		if (ret > 0) { /* ret == 1 if symlink could not be set */
-			updated &= ~UPDATED_TIMES;
-			file->flags |= FLAG_TIME_FAILED;
-		}
-	}
 #ifdef SUPPORT_CRTIMES
 	if (crtimes_ndx && !(flags & ATTRS_SKIP_CRTIME)) {
 		time_t file_crtime = F_CRTIME(file);
 		if (sxp->crtime == 0)
-			sxp->crtime = get_create_time(fname);
-		if (!same_time(sxp->crtime, 0L, file_crtime, 0L)
-		 && set_create_time(fname, file_crtime) == 0)
-			updated = 1;
+			sxp->crtime = get_create_time(fname, &sxp->st);
+		if (!same_time(sxp->crtime, 0L, file_crtime, 0L)) {
+			if (
+#ifdef HAVE_GETATTRLIST
+			     do_setattrlist_crtime(fname, file_crtime) == 0
+#elif defined __CYGWIN__
+			     do_SetFileTime(fname, file_crtime) == 0
+#else
+#error Unknown crtimes implementation
+#endif
+			)
+				updated |= UPDATED_CRTIME;
+		}
 	}
 #endif
+	if (updated & (UPDATED_MTIME|UPDATED_ATIME)) {
+		int ret = set_times(fname, &sx2.st);
+		if (ret < 0) {
+			rsyserr(FERROR_XFER, errno, "failed to set times on %s", full_fname(fname));
+			goto cleanup;
+		}
+		if (ret > 0) { /* ret == 1 if symlink could not be set */
+			updated &= ~(UPDATED_MTIME|UPDATED_ATIME);
+			file->flags |= FLAG_TIME_FAILED;
+		}
+	}
 
 #ifdef SUPPORT_ACLS
 	/* It's OK to call set_acl() now, even for a dir, as the generator

rsync.h

@@ -2,7 +2,7 @@
  * Copyright (C) 1996, 2000 Andrew Tridgell
  * Copyright (C) 1996 Paul Mackerras
  * Copyright (C) 2001, 2002 Martin Pool <mbp@samba.org>
- * Copyright (C) 2003-2020 Wayne Davison
+ * Copyright (C) 2003-2022 Wayne Davison
  *
  * This program is free software; you can redistribute it and/or modify
  * it under the terms of the GNU General Public License as published by
@@ -18,6 +18,11 @@
  * with this program; if not, visit the http://fsf.org website.
  */
 
+/* a non-zero CHAR_OFFSET makes the rolling sum stronger, but is
+   incompatible with older versions :-( */
+#define CHAR_OFFSET 0
+
+#ifndef AVX2_ASM /* do not include the rest of file for assembly */
 #define False 0
 #define True 1
 #define Unset (-1) /* Our BOOL values are always an int. */
@@ -38,9 +43,6 @@
 
 #define BACKUP_SUFFIX "~"
 
-/* a non-zero CHAR_OFFSET makes the rolling sum stronger, but is
-   incompatible with older versions :-( */
-#define CHAR_OFFSET 0
 
 /* These flags are only used during the flist transfer. */
 
@@ -108,12 +110,22 @@
 #define BITS_EQUAL(b1,b2,mask) (((unsigned)(b1) & (unsigned)(mask)) \
 			     == ((unsigned)(b2) & (unsigned)(mask)))
 
-/* update this if you make incompatible changes */
+/* Update this if you make incompatible changes and ALSO update the
+ * SUBPROTOCOL_VERSION if it is not a final (official) release. */
 #define PROTOCOL_VERSION 31
 
-/* This is used when working on a new protocol version in CVS, and should
- * be a new non-zero value for each CVS change that affects the protocol.
- * It must ALWAYS be 0 when the protocol goes final (and NEVER before)! */
+/* This is used when working on a new protocol version or for any unofficial
+ * protocol tweaks.  It should be a non-zero value for each pre-release repo
+ * change that affects the protocol. The official pre-release versions should
+ * start with 1 (after incrementing the PROTOCOL_VERSION) and go up by 1 for
+ * each new protocol change.  For unofficial changes, pick a fairly large
+ * random number that will hopefully not collide with anyone else's unofficial
+ * protocol.  It must ALWAYS be 0 when the protocol goes final (and official)
+ * and NEVER before!  When rsync negotiates a protocol match, it will only
+ * allow the newest protocol to be used if the SUBPROTOCOL_VERSION matches.
+ * All older protocol versions MUST be compatible with the final, official
+ * release of the protocol, so don't tweak the code to change the protocol
+ * behavior for an older protocol version. */
 #define SUBPROTOCOL_VERSION 0
 
 /* We refuse to interoperate with versions that are not in this range.
@@ -267,6 +279,10 @@ enum msgcode {
 	MSG_NO_SEND=102,/* sender failed to open a file we wanted */
 };
 
+enum filetype {
+	FT_UNSUPPORTED, FT_REG, FT_DIR, FT_SYMLINK, FT_SPECIAL, FT_DEVICE
+};
+
 #define NDX_DONE -1
 #define NDX_FLIST_EOF -2
 #define NDX_DEL_STATS -3
@@ -300,7 +316,6 @@ enum delret {
 #include "errcode.h"
 
 #include "config.h"
-#include "version.h"
 
 /* The default RSYNC_RSH is always set in config.h. */
 
@@ -478,7 +493,6 @@ enum delret {
 #ifndef __TANDEM
 #define MAKEDEV(devmajor,devminor) makedev(devmajor,devminor)
 #else
-# include <sys/stat.h>
 # define major DEV_TO_MAJOR
 # define minor DEV_TO_MINOR
 # define MAKEDEV MAJORMINOR_TO_DEV
@@ -570,11 +584,11 @@ typedef unsigned int size_t;
 #endif
 #endif
 
-#ifndef __APPLE__ /* Do we need a configure check for this? */
+#if !defined __APPLE__ || defined HAVE_GETATTRLIST
 #define SUPPORT_ATIMES 1
 #endif
 
-#ifdef HAVE_GETATTRLIST
+#if defined HAVE_GETATTRLIST || defined __CYGWIN__
 #define SUPPORT_CRTIMES 1
 #endif
 
@@ -766,6 +780,11 @@ struct ht_int64_node {
 
 #if defined __STDC_VERSION__ && __STDC_VERSION__ >= 199901L
 #define USE_FLEXIBLE_ARRAY 1
+#define SIZE_T_FMT_MOD "z" /* printf supports %zd */
+#define SIZE_T_FMT_CAST size_t
+#else
+#define SIZE_T_FMT_MOD "l" /* printf supports %ld */
+#define SIZE_T_FMT_CAST long
 #endif
 
 union file_extras {
@@ -904,8 +923,9 @@ extern int xattrs_ndx;
  * Start the flist array at FLIST_START entries and grow it
  * by doubling until FLIST_LINEAR then grow by FLIST_LINEAR
  */
-#define FLIST_START	(32 * 1024)
-#define FLIST_LINEAR	(FLIST_START * 512)
+#define FLIST_START		(32)
+#define FLIST_START_LARGE	(32 * 1024)
+#define FLIST_LINEAR		(FLIST_START_LARGE * 512)
 
 /*
  * Extent size for allocation pools: A minimum size of 128KB
@@ -1151,7 +1171,7 @@ struct name_num_obj {
 	uchar *saw;
 	int saw_len;
 	int negotiated_num;
-	struct name_num_item list[8]; /* A big-enough len (we'll get a compile error if it is ever too small) */
+	struct name_num_item list[10]; /* we'll get a compile error/warning if this is ever too small */
 };
 
 #ifndef __cplusplus
@@ -1312,10 +1332,6 @@ extern int errno;
 #define IS_SPECIAL(mode) (S_ISSOCK(mode) || S_ISFIFO(mode))
 #define IS_DEVICE(mode) (S_ISCHR(mode) || S_ISBLK(mode))
 
-#define PRESERVE_FILE_TIMES	(1<<0)
-#define PRESERVE_DIR_TIMES	(1<<1)
-#define PRESERVE_LINK_TIMES	(1<<2)
-
 /* Initial mask on permissions given to temporary files.  Mask off setuid
      bits and group access because of potential race-condition security
      holes, and mask other access because mode 707 is bizarre */
@@ -1405,7 +1421,8 @@ extern short info_levels[], debug_levels[];
 #define INFO_MISC (INFO_FLIST+1)
 #define INFO_MOUNT (INFO_MISC+1)
 #define INFO_NAME (INFO_MOUNT+1)
-#define INFO_PROGRESS (INFO_NAME+1)
+#define INFO_NONREG (INFO_NAME+1)
+#define INFO_PROGRESS (INFO_NONREG+1)
 #define INFO_REMOVE (INFO_PROGRESS+1)
 #define INFO_SKIP (INFO_REMOVE+1)
 #define INFO_STATS (INFO_SKIP+1)
@@ -1460,3 +1477,10 @@ const char *get_panic_action(void);
     fprintf(stderr, "%s in %s at line %d\n", msg, __FILE__, __LINE__); \
     exit_cleanup(RERR_UNSUPPORTED); \
 } while (0)
+#endif  /* AVX2_ASM */
+
+#ifdef HAVE_MALLINFO2
+#define MEM_ALLOC_INFO mallinfo2
+#elif defined HAVE_MALLINFO
+#define MEM_ALLOC_INFO mallinfo
+#endif

rsyncd.conf.5.md

@@ -1,12 +1,15 @@
-# NAME
+## NAME
 
 rsyncd.conf - configuration file for rsync in daemon mode
 
-# SYNOPSIS
+## SYNOPSIS
 
 rsyncd.conf
 
-# DESCRIPTION
+The online version of this man page (that includes cross-linking of topics)
+is available at <https://download.samba.org/pub/rsync/rsyncd.conf.5>.
+
+## DESCRIPTION
 
 The rsyncd.conf file is the runtime configuration file for rsync when run as an
 rsync daemon.
@@ -14,7 +17,7 @@ rsync daemon.
 The rsyncd.conf file controls authentication, access, logging and available
 modules.
 
-# FILE FORMAT
+## FILE FORMAT
 
 The file consists of modules and parameters. A module begins with the name of
 the module in square brackets and continues until the next module begins.
@@ -40,10 +43,9 @@ The values following the equals sign in parameters are all either a string (no
 quotes needed) or a boolean, which may be given as yes/no, 0/1 or true/false.
 Case is not significant in boolean values, but is preserved in string values.
 
-# LAUNCHING THE RSYNC DAEMON
+## LAUNCHING THE RSYNC DAEMON
 
-The rsync daemon is launched by specifying the `--daemon` option to
-rsync.
+The rsync daemon is launched by specifying the `--daemon` option to rsync.
 
 The daemon must run with root privileges if you wish to use chroot, to bind to
 a port numbered under 1024 (as is the default 873), or to set file ownership.
@@ -60,16 +62,16 @@ When run via inetd you should add a line like this to /etc/services:
 
 and a single line something like this to /etc/inetd.conf:
 
->     rsync   stream  tcp     nowait  root   /usr/bin/rsync rsyncd --daemon
+>     rsync   stream  tcp     nowait  root   @BINDIR@/rsync rsyncd --daemon
 
-Replace "/usr/bin/rsync" with the path to where you have rsync installed on
+Replace "@BINDIR@/rsync" with the path to where you have rsync installed on
 your system.  You will then need to send inetd a HUP signal to tell it to
 reread its config file.
 
 Note that you should **not** send the rsync daemon a HUP signal to force it to
 reread the `rsyncd.conf` file. The file is re-read on each client connection.
 
-# GLOBAL PARAMETERS
+## GLOBAL PARAMETERS
 
 The first parameters in the file (before a [module] header) are the global
 parameters.  Rsync also allows for the use of a "[global]" module name to
@@ -136,7 +138,7 @@ a literal % into a value is to use %%.
     You can override the default backlog value when the daemon listens for
     connections.  It defaults to 5.
 
-# MODULE PARAMETERS
+## MODULE PARAMETERS
 
 After the global parameters you should define a number of modules, each module
 exports a directory tree as a symbolic name. Modules are exported by specifying
@@ -177,7 +179,7 @@ the values of parameters.  See the GLOBAL PARAMETERS section for more details.
 
 0.  `use chroot`
 
-    If "use chroot" is true, the rsync daemon will chroot to the "path" before
+    If "use chroot" is true, the rsync daemon will chroot to the "[path](#)" before
     starting the file transfer with the client.  This has the advantage of
     extra protection against possible implementation security holes, but it has
     the disadvantages of requiring super-user privileges, of not being able to
@@ -186,7 +188,7 @@ the values of parameters.  See the GLOBAL PARAMETERS section for more details.
     below).
 
     As an additional safety feature, you can specify a dot-dir in the module's
-    "path" to indicate the point where the chroot should occur.  This allows
+    "[path](#)" to indicate the point where the chroot should occur.  This allows
     rsync to run in a chroot with a non-"/" path for the top of the transfer
     hierarchy.  Doing this guards against unintended library loading (since
     those absolute paths will not be inside the transfer hierarchy unless you
@@ -197,18 +199,18 @@ the values of parameters.  See the GLOBAL PARAMETERS section for more details.
     chroot would have used the whole path, and the inside-chroot path would
     have been "/".
 
-    When both "use chroot" and "daemon chroot" are false, OR the inside-chroot
+    When both "use chroot" and "[daemon chroot](#)" are false, OR the inside-chroot
     path of "use chroot" is not "/", rsync will: (1) munge symlinks by default
-    for security reasons (see "munge symlinks" for a way to turn this off, but
+    for security reasons (see "[munge symlinks](#)" for a way to turn this off, but
     only if you trust your users), (2) substitute leading slashes in absolute
     paths with the module's path (so that options such as `--backup-dir`,
     `--compare-dest`, etc. interpret an absolute path as rooted in the module's
-    "path" dir), and (3) trim ".." path elements from args if rsync believes
+    "[path](#)" dir), and (3) trim ".." path elements from args if rsync believes
     they would escape the module hierarchy.  The default for "use chroot" is
     true, and is the safer choice (especially if the module is not read-only).
 
-    When this parameter is enabled *and* the "name converter" parameter is
-    *not* set, the "numeric ids" parameter will default to being enabled
+    When this parameter is enabled *and* the "[name converter](#)" parameter is
+    *not* set, the "[numeric ids](#)" parameter will default to being enabled
     (disabling name lookups).  This means that if you manually setup
     name-lookup libraries in your chroot (instead of using a name converter)
     that you need to explicitly set `numeric ids = false` for rsync to do name
@@ -217,16 +219,16 @@ the values of parameters.  See the GLOBAL PARAMETERS section for more details.
     If you copy library resources into the module's chroot area, you should
     protect them through your OS's normal user/group or ACL settings (to
     prevent the rsync module's user from being able to change them), and then
-    hide them from the user's view via "exclude" (see how in the discussion of
+    hide them from the user's view via "[exclude](#)" (see how in the discussion of
     that parameter).  However, it's easier and safer to setup a name converter.
 
 0.  `daemon chroot`
 
     This parameter specifies a path to which the daemon will chroot before
-    beginning communication with clients. Module paths (and any "use chroot"
+    beginning communication with clients. Module paths (and any "[use chroot](#)"
     settings) will then be related to this one. This lets you choose if you
     want the whole daemon to be chrooted (with this setting), just the
-    transfers to be chrooted (with "use chroot"), or both.  Keep in mind that
+    transfers to be chrooted (with "[use chroot](#)"), or both.  Keep in mind that
     the "daemon chroot" area may need various OS/lib/etc files installed to
     allow the daemon to function.  By default the daemon runs without any
     chrooting.
@@ -284,11 +286,11 @@ the values of parameters.  See the GLOBAL PARAMETERS section for more details.
     transfer behave as if the client had passed the `--numeric-ids`
     command-line option.  By default, this parameter is enabled for chroot
     modules and disabled for non-chroot modules.  Also keep in mind that
-    uid/gid preservation requires the module to be running as root (see "uid")
-    or for "fake super" to be configured.
+    uid/gid preservation requires the module to be running as root (see "[uid](#)")
+    or for "[fake super](#)" to be configured.
 
     A chroot-enabled module should not have this parameter set to false unless
-    you're using a "name converter" program *or* you've taken steps to ensure
+    you're using a "[name converter](#)" program *or* you've taken steps to ensure
     that the module has the necessary resources it needs to translate names and
     that it is not possible for a user to change those resources.
 
@@ -298,12 +300,12 @@ the values of parameters.  See the GLOBAL PARAMETERS section for more details.
     (non-daemon-affecting) `--munge-links` command-line option (using a method
     described below).  This should help protect your files from user trickery
     when your daemon module is writable.  The default is disabled when
-    "use chroot" is on with an inside-chroot path of "/", OR if "daemon chroot"
+    "[use chroot](#)" is on with an inside-chroot path of "/", OR if "[daemon chroot](#)"
     is on, otherwise it is enabled.
 
     If you disable this parameter on a daemon that is not read-only, there are
     tricks that a user can play with uploaded symlinks to access
-    daemon-excluded items (if your module has any), and, if "use chroot" is
+    daemon-excluded items (if your module has any), and, if "[use chroot](#)" is
     off, rsync can even be tricked into showing or changing data that is
     outside the module's path (as access-permissions allow).
 
@@ -324,7 +326,7 @@ the values of parameters.  See the GLOBAL PARAMETERS section for more details.
     the source code named "munge-symlinks" that can be used to add or remove
     this prefix from your symlinks.
 
-    When this parameter is disabled on a writable module and "use chroot" is
+    When this parameter is disabled on a writable module and "[use chroot](#)" is
     off (or the inside-chroot path is not "/"), incoming symlinks will be
     modified to drop a leading slash and to remove ".." path elements that
     rsync believes will allow a symlink to escape the module's hierarchy.
@@ -340,10 +342,10 @@ the values of parameters.  See the GLOBAL PARAMETERS section for more details.
     conversion in a chroot module without extra files in the chroot area, and
     also ensures that name-translation is done in a consistent manner.  If the
     "charset" parameter is not set, the `--iconv` option is refused, just as if
-    "iconv" had been specified via "refuse options".
+    "iconv" had been specified via "[refuse options](#)".
 
     If you wish to force users to always use `--iconv` for a particular module,
-    add "no-iconv" to the "refuse options" parameter.  Keep in mind that this
+    add "no-iconv" to the "[refuse options](#)" parameter.  Keep in mind that this
     will restrict access to your module to very new rsync clients.
 
 0.  `max connections`
@@ -352,7 +354,7 @@ the values of parameters.  See the GLOBAL PARAMETERS section for more details.
     connections you will allow.  Any clients connecting when the maximum has
     been reached will receive a message telling them to try later.  The default
     is 0, which means no limit.  A negative value disables the module.  See
-    also the "lock file" parameter.
+    also the "[lock file](#)" parameter.
 
 0.  `log file`
 
@@ -381,7 +383,7 @@ the values of parameters.  See the GLOBAL PARAMETERS section for more details.
     facility name which is defined on your system. Common names are auth,
     authpriv, cron, daemon, ftp, kern, lpr, mail, news, security, syslog, user,
     uucp, local0, local1, local2, local3, local4, local5, local6 and local7.
-    The default is daemon.  This setting has no effect if the "log file"
+    The default is daemon.  This setting has no effect if the "[log file](#)"
     setting is a non-empty string (either set in the per-modules settings, or
     inherited from the global settings).
 
@@ -389,7 +391,7 @@ the values of parameters.  See the GLOBAL PARAMETERS section for more details.
 
     This parameter allows you to specify the syslog tag to use when logging
     messages from the rsync daemon. The default is "rsyncd".  This setting has
-    no effect if the "log file" setting is a non-empty string (either set in
+    no effect if the "[log file](#)" setting is a non-empty string (either set in
     the per-modules settings, or inherited from the global settings).
 
     For example, if you wanted each authenticated user's name to be included in
@@ -414,7 +416,7 @@ the values of parameters.  See the GLOBAL PARAMETERS section for more details.
 
 0.  `lock file`
 
-    This parameter specifies the file to use to support the "max connections"
+    This parameter specifies the file to use to support the "[max connections](#)"
     parameter. The rsync daemon uses record locking on this file to ensure that
     the max connections limit is not exceeded for the modules sharing the lock
     file.  The default is `/var/run/rsyncd.lock`.
@@ -426,7 +428,7 @@ the values of parameters.  See the GLOBAL PARAMETERS section for more details.
     "read only" is false then uploads will be possible if file permissions on
     the daemon side allow them. The default is for all modules to be read only.
 
-    Note that "auth users" can override this setting on a per-user basis.
+    Note that "[auth users](#)" can override this setting on a per-user basis.
 
 0.  `write only`
 
@@ -460,8 +462,8 @@ the values of parameters.  See the GLOBAL PARAMETERS section for more details.
     This parameter determines whether this module is listed when the client
     asks for a listing of available modules.  In addition, if this is false,
     the daemon will pretend the module does not exist when a client denied by
-    "hosts allow" or "hosts deny" attempts to access it.  Realize that if
-    "reverse lookup" is disabled globally but enabled for the module, the
+    "[hosts allow](#)" or "[hosts deny](#)" attempts to access it.  Realize that if
+    "[reverse lookup](#)" is disabled globally but enabled for the module, the
     resulting reverse lookup to a potentially client-controlled DNS server may
     still reveal to the client that it hit an existing module.  The default is
     for modules to be listable.
@@ -470,10 +472,10 @@ the values of parameters.  See the GLOBAL PARAMETERS section for more details.
 
     This parameter specifies the user name or user ID that file transfers to
     and from that module should take place as when the daemon was run as root.
-    In combination with the "gid" parameter this determines what file
+    In combination with the "[gid](#)" parameter this determines what file
     permissions are available. The default when run by a super-user is to
     switch to the system's "nobody" user.  The default for a non-super-user is
-    to not try to change the user.  See also the "gid" parameter.
+    to not try to change the user.  See also the "[gid](#)" parameter.
 
     The RSYNC_USER_NAME environment variable may be used to request that rsync
     run as the authorizing user.  For example, if you want a rsync to run as
@@ -489,7 +491,7 @@ the values of parameters.  See the GLOBAL PARAMETERS section for more details.
     accessing the module.  The first one will be the default group, and any
     extra ones be set as supplemental groups.  You may also specify a "`*`" as
     the first gid in the list, which will be replaced by all the normal groups
-    for the transfer's user (see "uid").  The default when run by a super-user
+    for the transfer's user (see "[uid](#)").  The default when run by a super-user
     is to switch to your OS's "nobody" (or perhaps "nogroup") group with no
     other supplementary groups.  The default for a non-super-user is to not
     change any group attributes (and indeed, your OS may not allow a
@@ -505,13 +507,13 @@ the values of parameters.  See the GLOBAL PARAMETERS section for more details.
 
     This parameter specifies a uid under which the daemon will run. The daemon
     usually runs as user root, and when this is left unset the user is left
-    unchanged. See also the "uid" parameter.
+    unchanged. See also the "[uid](#)" parameter.
 
 0.  `daemon gid`
 
     This parameter specifies a gid under which the daemon will run. The daemon
     usually runs as group root, and when this is left unset, the group is left
-    unchanged. See also the "gid" parameter.
+    unchanged. See also the "[gid](#)" parameter.
 
 0.  `fake super`
 
@@ -532,8 +534,8 @@ the values of parameters.  See the GLOBAL PARAMETERS section for more details.
     or tampering with private administrative files, such as files you may add
     to support uid/gid name translations.
 
-    The daemon filter chain is built from the "filter", "include from",
-    "include", "exclude from", and "exclude" parameters, in that order of
+    The daemon filter chain is built from the "filter", "[include from](#)",
+    "[include](#)", "[exclude from](#)", and "[exclude](#)" parameters, in that order of
     priority.  Anchored patterns are anchored at the root of the module.  To
     prevent access to an entire subtree, for example, "`/secret`", you **must**
     exclude everything in the subtree; the easiest way to do this is with a
@@ -560,8 +562,8 @@ the values of parameters.  See the GLOBAL PARAMETERS section for more details.
 
 0.  `include`
 
-    Use an "include" to override the effects of the "exclude" parameter.  Only
-    one "include" parameter can apply to a given module.  See the "filter"
+    Use an "include" to override the effects of the "[exclude](#)" parameter.  Only
+    one "include" parameter can apply to a given module.  See the "[filter](#)"
     parameter for a description of how excluded files affect the daemon.
 
 0.  `exclude from`
@@ -569,14 +571,14 @@ the values of parameters.  See the GLOBAL PARAMETERS section for more details.
     This parameter specifies the name of a file on the daemon that contains
     daemon exclude patterns, one per line.  Only one "exclude from" parameter
     can apply to a given module; if you have multiple exclude-from files, you
-    can specify them as a merge file in the "filter" parameter.  See the
-    "filter" parameter for a description of how excluded files affect the
+    can specify them as a merge file in the "[filter](#)" parameter.  See the
+    "[filter](#)" parameter for a description of how excluded files affect the
     daemon.
 
 0.  `include from`
 
-    Analogue of "exclude from" for a file of daemon include patterns.  Only one
-    "include from" parameter can apply to a given module.  See the "filter"
+    Analogue of "[exclude from](#)" for a file of daemon include patterns.  Only one
+    "include from" parameter can apply to a given module.  See the "[filter](#)"
     parameter for a description of how excluded files affect the daemon.
 
 0.  `incoming chmod`
@@ -611,7 +613,7 @@ the values of parameters.  See the GLOBAL PARAMETERS section for more details.
     to supply a username and password to connect to the module. A challenge
     response authentication protocol is used for this exchange. The plain text
     usernames and passwords are stored in the file specified by the
-    "secrets file" parameter. The default is for all users to be able to
+    "[secrets file](#)" parameter. The default is for all users to be able to
     connect without a password (this is called "anonymous rsync").
 
     In addition to username matching, you can specify groupname matching via a
@@ -623,7 +625,7 @@ the values of parameters.  See the GLOBAL PARAMETERS section for more details.
     Finally, options may be specified after a colon (:).  The options allow you
     to "deny" a user or a group, set the access to "ro" (read-only), or set the
     access to "rw" (read/write).  Setting an auth-rule-specific ro/rw setting
-    overrides the module's "read only" setting.
+    overrides the module's "[read only](#)" setting.
 
     Be sure to put the rules in the order you want them to be matched, because
     the checking stops at the first matching user or group, and that is the
@@ -661,7 +663,7 @@ the values of parameters.  See the GLOBAL PARAMETERS section for more details.
 
     This parameter specifies the name of a file that contains the
     username:password and/or @groupname:password pairs used for authenticating
-    this module. This file is only consulted if the "auth users" parameter is
+    this module. This file is only consulted if the "[auth users](#)" parameter is
     specified.  The file is line-based and contains one name:password pair per
     line.  Any line has a hash (#) as the very first character on the line is
     considered a comment and is skipped.  The passwords can contain any
@@ -675,14 +677,14 @@ the values of parameters.  See the GLOBAL PARAMETERS section for more details.
     "@groupname:password" line for the group that triggered the authentication.
 
     It is up to you what kind of password entries you want to include, either
-    users, groups, or both.  The use of group rules in "auth users" does not
+    users, groups, or both.  The use of group rules in "[auth users](#)" does not
     require that you specify a group password if you do not want to use shared
     passwords.
 
     There is no default for the "secrets file" parameter, you must choose a
     name (such as `/etc/rsyncd.secrets`).  The file must normally not be
-    readable by "other"; see "strict modes".  If the file is not found or is
-    rejected, no logins for a "user auth" module will be possible.
+    readable by "other"; see "[strict modes](#)".  If the file is not found or is
+    rejected, no logins for an "[auth users](#)" module will be possible.
 
 0.  `strict modes`
 
@@ -715,10 +717,10 @@ the values of parameters.  See the GLOBAL PARAMETERS section for more details.
     - a hostname pattern using wildcards. If the hostname of the connecting IP
       (as determined by a reverse lookup) matches the wildcarded name (using
       the same rules as normal unix filename matching), the client is allowed
-      in.  This only works if "reverse lookup" is enabled (the default).
+      in.  This only works if "[reverse lookup](#)" is enabled (the default).
     - a hostname. A plain hostname is matched against the reverse DNS of the
-      connecting IP (if "reverse lookup" is enabled), and/or the IP of the
-      given hostname is matched against the connecting IP (if "forward lookup"
+      connecting IP (if "[reverse lookup](#)" is enabled), and/or the IP of the
+      given hostname is matched against the connecting IP (if "[forward lookup](#)"
       is enabled, as it is by default).  Any match will be allowed in.
     - an '@' followed by a netgroup name, which will match if the reverse DNS
       of the connecting IP is in the specified netgroup.
@@ -730,11 +732,11 @@ the values of parameters.  See the GLOBAL PARAMETERS section for more details.
     >     fe80::%link1/64
     >     fe80::%link1/ffff:ffff:ffff:ffff::
 
-    You can also combine "hosts allow" with "hosts deny" as a way to add
+    You can also combine "hosts allow" with "[hosts deny](#)" as a way to add
     exceptions to your deny list.  When both parameters are specified, the
     "hosts allow" parameter is checked first and a match results in the client
     being able to connect.  A non-allowed host is then matched against the
-    "hosts deny" list to see if it should be rejected.  A host that does not
+    "[hosts deny](#)" list to see if it should be rejected.  A host that does not
     match either list is allowed to connect.
 
     The default is no "hosts allow" parameter, which means all hosts can
@@ -745,7 +747,7 @@ the values of parameters.  See the GLOBAL PARAMETERS section for more details.
     This parameter allows you to specify a list of comma- and/or
     whitespace-separated patterns that are matched against a connecting clients
     hostname and IP address. If the pattern matches then the connection is
-    rejected. See the "hosts allow" parameter for more information.
+    rejected. See the "[hosts allow](#)" parameter for more information.
 
     The default is no "hosts deny" parameter, which means all hosts can
     connect.
@@ -753,8 +755,8 @@ the values of parameters.  See the GLOBAL PARAMETERS section for more details.
 0.  `reverse lookup`
 
     Controls whether the daemon performs a reverse lookup on the client's IP
-    address to determine its hostname, which is used for "hosts allow" &
-    "hosts deny" checks and the "%h" log escape.  This is enabled by default,
+    address to determine its hostname, which is used for "[hosts allow](#)" &
+    "[hosts deny](#)" checks and the "%h" log escape.  This is enabled by default,
     but you may wish to disable it to save time if you know the lookup will not
     return a useful result, in which case the daemon will use the name
     "UNDETERMINED" instead.
@@ -794,7 +796,7 @@ the values of parameters.  See the GLOBAL PARAMETERS section for more details.
     logs the transfer at the end, so if a transfer is aborted, no mention will
     be made in the log file.
 
-    If you want to customize the log lines, see the "log format" parameter.
+    If you want to customize the log lines, see the "[log format](#)" parameter.
 
 0.  `log format`
 
@@ -811,7 +813,7 @@ the values of parameters.  See the GLOBAL PARAMETERS section for more details.
     (e.g. "`%''l %'b %f`").
 
     The default log format is "`%o %h [%a] %m (%u) %f %l`", and a "`%t [%p] `"
-    is always prefixed when using the "log file" parameter.  (A perl script
+    is always prefixed when using the "[log file](#)" parameter.  (A perl script
     that will summarize this default log format is included in the rsync source
     code distribution in the "support" subdirectory: rsyncstats.)
 
@@ -922,13 +924,14 @@ the values of parameters.  See the GLOBAL PARAMETERS section for more details.
 
     >     refuse options = * !a !delete* delete-after
 
-    A note on refusing "compress" -- it is better to set the "dont compress"
-    daemon parameter to "`*`" because that disables compression silently
+    A note on refusing "compress": it may be better to set the "[dont compress](#)"
+    daemon parameter to "`*`" and ensure that `RSYNC_COMPRESS_LIST=zlib` is set
+    in the environment of the daemon in order to disable compression silently
     instead of returning an error that forces the client to remove the `-z`
     option.
 
-    If you are un-refusing the compress option, you probably want to match
-    "`!compress*`" so that you also accept the `--compress-level` option.
+    If you are un-refusing the compress option, you may want to match
+    "`!compress*`" if you also want to allow the `--compress-level` option.
 
     Note that the "write-devices" option is refused by default, but can be
     explicitly accepted with "`!write-devices`".  The options "log-file" and
@@ -942,18 +945,22 @@ the values of parameters.  See the GLOBAL PARAMETERS section for more details.
       receiver.  While rsync passes the older alias `--log-format` for
       compatibility reasons, this options should not be confused with
       `--log-file-format`.
-    - `--sender`: Use "write only" parameter instead of refusing this.
+    - `--sender`: Use "[write only](#)" parameter instead of refusing this.
     - `--dry-run`, `-n`: Who would want to disable this?
     - `--protect-args`, `-s`: This actually makes transfers safer.
     - `--from0`, `-0`: Makes it easier to accept/refuse `--files-from` without
       affecting this helpful modifier.
-    - `--iconv`: This is auto-disabled based on "charset" parameter.
+    - `--iconv`: This is auto-disabled based on "[charset](#)" parameter.
     - `--no-iconv`: Most transfers use this option.
     - `--checksum-seed`: Is a fairly rare, safe option.
     - `--write-devices`: Is non-wild but also auto-disabled.
 
 0.  `dont compress`
 
+    **NOTE:** This parameter currently has no effect except in one instance: if
+    it is set to "`*`" then it minimizes or disables compression for all files
+    (for those that don't want to refuse the `--compress` option completely).
+
     This parameter allows you to select filenames based on wildcard patterns
     that should not be compressed when pulling files from the daemon (no
     analogous parameter exists to govern the pushing of files to a daemon).
@@ -964,14 +971,14 @@ the values of parameters.  See the GLOBAL PARAMETERS section for more details.
     The "dont compress" parameter takes a space-separated list of
     case-insensitive wildcard patterns. Any source filename matching one of the
     patterns will be compressed as little as possible during the transfer.  If
-    the compression algorithm has an "off" level (such as zlib/zlibx) then no
-    compression occurs for those files.  Other algorithms have the level
-    minimized to reduces the CPU usage as much as possible.
+    the compression algorithm has an "off" level, then no compression occurs
+    for those files.  If an algorithms has the ability to change the level in
+    mid-stream, it will be minimized to reduce the CPU usage as much as
+    possible.
 
     See the `--skip-compress` parameter in the **rsync**(1) manpage for the
-    list of file suffixes that are not compressed by default.  Specifying a
-    value for the "dont compress" parameter changes the default when the daemon
-    is the sender.
+    list of file suffixes that are skipped by default if this parameter is not
+    set.
 
 0.  `early exec`, `pre-xfer exec`, `post-xfer exec`
 
@@ -1033,7 +1040,7 @@ the values of parameters.  See the GLOBAL PARAMETERS section for more details.
     **system()** call's default shell), and use RSYNC_NO_XFER_EXEC to disable
     both options completely.
 
-# CONFIG DIRECTIVES
+## CONFIG DIRECTIVES
 
 There are currently two config directives available that allow a config file to
 incorporate the contents of other files:  `&include` and `&merge`.  Both allow
@@ -1088,7 +1095,7 @@ This would merge any `/etc/rsyncd.d/*.inc` files (for global values that should
 stay in effect), and then include any `/etc/rsyncd.d/*.conf` files (defining
 modules without any global-value cross-talk).
 
-# AUTHENTICATION STRENGTH
+## AUTHENTICATION STRENGTH
 
 The authentication protocol used in rsync is a 128 bit MD4 based challenge
 response system. This is fairly weak protection, though (with at least one
@@ -1103,7 +1110,7 @@ authentication is provided. Use ssh as the transport if you want encryption.
 You can also make use of SSL/TLS encryption if you put rsync behind an
 SSL proxy.
 
-# SSL/TLS Daemon Setup
+## SSL/TLS Daemon Setup
 
 When setting up an rsync daemon for access via SSL/TLS, you will need to
 configure a proxy (such as haproxy or nginx) as the front-end that handles the
@@ -1148,7 +1155,7 @@ An example nginx proxy setup is as follows:
 > }
 > ```
 
-# EXAMPLES
+## EXAMPLES
 
 A simple rsyncd.conf file that allow anonymous rsync to a ftp area at
 `/home/ftp` would be:
@@ -1197,45 +1204,39 @@ The /etc/rsyncd.secrets file would look something like this:
 >     tridge:mypass
 >     susan:herpass
 
-# FILES
+## FILES
 
 /etc/rsyncd.conf or rsyncd.conf
 
-# SEE ALSO
+## SEE ALSO
 
-**rsync**(1), **rsync-ssl**(1)
+[**rsync**(1)](rsync.1), [**rsync-ssl**(1)](rsync-ssl.1)
 
-# BUGS
+## BUGS
 
 Please report bugs! The rsync bug tracking system is online at
 <https://rsync.samba.org/>.
 
-# VERSION
+## VERSION
 
 This man page is current for version @VERSION@ of rsync.
 
-# CREDITS
+## CREDITS
 
-rsync is distributed under the GNU General Public License.  See the file
-COPYING for details.
+Rsync is distributed under the GNU General Public License.  See the file
+[COPYING](COPYING) for details.
 
-The primary ftp site for rsync is <ftp://rsync.samba.org/pub/rsync>
+An rsync web site is available at <https://rsync.samba.org/> and its github
+project is <https://github.com/WayneD/rsync>.
 
-A web site is available at <https://rsync.samba.org/>.
-
-We would be delighted to hear from you if you like this program.
-
-This program uses the zlib compression library written by Jean-loup Gailly and
-Mark Adler.
-
-# THANKS
+## THANKS
 
 Thanks to Warren Stanley for his original idea and patch for the rsync daemon.
 Thanks to Karsten Thygesen for his many suggestions and documentation!
 
-# AUTHOR
+## AUTHOR
 
-rsync was written by Andrew Tridgell and Paul Mackerras.  Many people have
+Rsync was written by Andrew Tridgell and Paul Mackerras.  Many people have
 later contributed to it.
 
 Mailing lists for support and development are available at

runtests.sh

@@ -1,7 +1,7 @@
 #! /bin/sh
 
 # Copyright (C) 2001, 2002 by Martin Pool <mbp@samba.org>
-# Copyright (C) 2003-2020 Wayne Davison
+# Copyright (C) 2003-2022 Wayne Davison
 
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License version
@@ -155,7 +155,7 @@ if test x"$TOOLDIR" = x; then
     TOOLDIR=`pwd`
 fi
 srcdir=`dirname $0`
-if test x"$srcdir" = x -o x"$srcdir" = x.; then
+if test x"$srcdir" = x || test x"$srcdir" = x.; then
     srcdir="$TOOLDIR"
 fi
 if test x"$rsync_bin" = x; then
@@ -167,10 +167,10 @@ RSYNC="$rsync_bin $*"
 #RSYNC="valgrind $rsync_bin $*"
 
 TLS_ARGS=''
-if egrep '^#define HAVE_LUTIMES 1' config.h >/dev/null; then
+if grep -E '^#define HAVE_LUTIMES 1' config.h >/dev/null; then
     TLS_ARGS="$TLS_ARGS -l"
 fi
-if egrep '#undef CHOWN_MODIFIES_SYMLINK' config.h >/dev/null; then
+if grep -E '#undef CHOWN_MODIFIES_SYMLINK' config.h >/dev/null; then
     TLS_ARGS="$TLS_ARGS -L"
 fi
 
@@ -226,6 +226,8 @@ if [ ! -d "$srcdir" ]; then
     exit 2
 fi
 
+expect_skipped="${RSYNC_EXPECT_SKIPPED-IGNORE}"
+skipped_list=''
 skipped=0
 missing=0
 passed=0
@@ -236,7 +238,7 @@ failed=0
 # failure to aid investigation.  We don't remove the testtmp subdir at
 # the end so that it can be configured as a symlink to a filesystem that
 # has ACLs and xattr support enabled (if desired).
-scratchbase="$TOOLDIR"/testtmp
+scratchbase="${scratchbase:-$TOOLDIR}"/testtmp
 echo "    scratchbase=$scratchbase"
 [ -d "$scratchbase" ] || mkdir "$scratchbase"
 
@@ -265,10 +267,12 @@ maybe_discard_scratch() {
 
 if [ "x$whichtests" = x ]; then
     whichtests="*.test"
+    full_run=yes
+else
+    full_run=no
 fi
 
-for testscript in $suitedir/$whichtests
-do
+for testscript in $suitedir/$whichtests; do
     testbase=`echo $testscript | sed -e 's!.*/!!' -e 's/.test\$//'`
     scratchdir="$scratchbase/$testbase"
 
@@ -284,7 +288,7 @@ do
     result=$?
     set -e
 
-    if [ "x$always_log" = xyes -o \( $result != 0 -a $result != 77 -a $result != 78 \) ]
+    if [ "x$always_log" = xyes ] || ( [ $result != 0 ] && [ $result != 77 ] && [ $result != 78 ] )
     then
 	echo "----- $testbase log follows"
 	cat "$scratchdir/test.log"
@@ -306,6 +310,7 @@ do
 	# backticks will fill the whole file onto one line, which is a feature
 	whyskipped=`cat "$scratchdir/whyskipped"`
 	echo "SKIP    $testbase ($whyskipped)"
+	skipped_list="$skipped_list,$testbase"
 	skipped=`expr $skipped + 1`
 	maybe_discard_scratch
 	;;
@@ -331,6 +336,15 @@ echo "      $passed passed"
 [ "$failed" -gt 0 ]  && echo "      $failed failed"
 [ "$skipped" -gt 0 ] && echo "      $skipped skipped"
 [ "$missing" -gt 0 ] && echo "      $missing missing"
+if [ "$full_run" = yes ] && [ "$expect_skipped" != IGNORE ]; then
+    skipped_list=`echo "$skipped_list" | sed 's/^,//'`
+    echo "----- skipped results:"
+    echo "      expected: $expect_skipped"
+    echo "      got:      $skipped_list"
+else
+    skipped_list=''
+    expect_skipped=''
+fi
 echo '------------------------------------------------------------'
 
 # OK, so expr exits with 0 if the result is neither null nor zero; and
@@ -339,7 +353,7 @@ echo '------------------------------------------------------------'
 # because -e is set.
 
 result=`expr $failed + $missing || true`
-if [ "$result" = 0 -a "$skipped" -gt "${RSYNC_MAX_SKIPPED:-9999}" ]; then
+if [ "$result" = 0 ] && [ "$skipped_list" != "$expect_skipped" ]; then
     result=1
 fi
 echo "overall result is $result"

sender.c

@@ -3,7 +3,7 @@
  *
  * Copyright (C) 1996 Andrew Tridgell
  * Copyright (C) 1996 Paul Mackerras
- * Copyright (C) 2003-2020 Wayne Davison
+ * Copyright (C) 2003-2022 Wayne Davison
  *
  * This program is free software; you can redistribute it and/or modify
  * it under the terms of the GNU General Public License as published by
@@ -35,6 +35,7 @@ extern int append_mode;
 extern int copy_links;
 extern int io_error;
 extern int flist_eof;
+extern int whole_file;
 extern int allowed_lull;
 extern int preserve_xattrs;
 extern int protocol_version;
@@ -204,6 +205,9 @@ void send_files(int f_in, int f_out)
 	if (DEBUG_GTE(SEND, 1))
 		rprintf(FINFO, "send_files starting\n");
 
+	if (whole_file < 0)
+		whole_file = 0;
+
 	progress_init();
 
 	while (1) {
@@ -362,6 +366,16 @@ void send_files(int f_in, int f_out)
 			exit_cleanup(RERR_FILEIO);
 		}
 
+		if (append_mode > 0 && st.st_size < F_LENGTH(file)) {
+			rprintf(FWARNING, "skipped diminished file: %s\n",
+				full_fname(fname));
+			free_sums(s);
+			close(fd);
+			if (protocol_version >= 30)
+				send_msg_int(MSG_NO_SEND, ndx);
+			continue;
+		}
+
 		if (st.st_size) {
 			int32 read_size = MAX(s->blength * 3, MAX_MAP_SIZE);
 			mbuf = map_file(fd, st.st_size, read_size, s->blength);

simd-checksum-avx2.S

@@ -0,0 +1,169 @@
+#ifdef __APPLE__
+#define get_checksum1_avx2  _get_checksum1_avx2
+#endif
+
+.intel_syntax noprefix
+.text
+
+	.p2align 5
+	.globl get_checksum1_avx2
+
+# rdi=*buf, esi=len, edx=i, rcx= *ps1, r8= *ps2
+get_checksum1_avx2:
+	vmovd	xmm6,[rcx] # load *ps1
+	lea	eax, [rsi-128] # at least 128 bytes to process?
+	cmp	edx, eax
+	jg	.exit
+	lea	rax, .mul_T2[rip]
+	vmovntdqa ymm7, [rax]   # load T2 multiplication constants
+	vmovntdqa ymm12,[rax+32]# from memory.
+	vpcmpeqd  ymm15, ymm15, ymm15 # set all elements to -1.
+
+#if CHAR_OFFSET != 0
+	mov	eax, 32*CHAR_OFFSET
+	vmovd	xmm10, eax
+	vpbroadcastd	ymm10, xmm10
+	mov	eax, 528*CHAR_OFFSET
+	vmovd	xmm13, eax
+	vpbroadcastd ymm13, xmm13
+#endif
+	vpabsb	ymm15, ymm15  # set all byte size elements to 1.
+	add	rdi, rdx
+	vmovdqu ymm2, [rdi]   # preload the first 64 bytes.
+	vmovdqu ymm3, [rdi+32]
+	and	esi, ~63   # only needed during final reduction, 
+			   # done here to avoid a longer nop for 
+			   # alignment below.
+	add	edx, esi
+	shr	rsi, 6	   # longer opcode for alignment
+	add	rdi, 64
+	vpxor	xmm1, xmm1, xmm1 # reset both partial sums accumulators.
+	vpxor	xmm4, xmm4, xmm4
+	mov	eax, [r8]
+	.p2align 4 		# should fit into the LSD allocation queue.
+.loop:
+	vpmaddubsw	ymm0, ymm15, ymm2  # s1 partial sums
+	vpmaddubsw	ymm5, ymm15, ymm3 
+	vmovdqu	ymm8,  [rdi]	# preload the next 
+	vmovdqu	ymm9,  [rdi+32] # 64 bytes.
+	add	rdi, 64
+	vpaddd	ymm4, ymm4, ymm6
+	vpaddw	ymm5, ymm5, ymm0
+	vpsrld	ymm0, ymm5, 16
+	vpaddw	ymm5, ymm0, ymm5
+	vpaddd	ymm6, ymm5, ymm6
+	vpmaddubsw	ymm2, ymm7, ymm2  # s2 partial sums
+	vpmaddubsw	ymm3, ymm12, ymm3
+	prefetcht0	[rdi+384] # prefetch 6 cachelines ahead.
+	vpaddw	ymm3, ymm2, ymm3
+	vpsrldq	ymm2, ymm3, 2
+	vpaddd	ymm3, ymm2, ymm3
+	vpaddd	ymm1, ymm1, ymm3
+
+#if CHAR_OFFSET != 0
+	vpaddd ymm6, ymm10, ymm6  #  32*CHAR_OFFSET
+	vpaddd ymm1, ymm13, ymm1  # 528*CHAR_OFFSET
+#endif
+	vmovdqa ymm2, ymm8   # move the next 64 bytes 
+	vmovdqa ymm3, ymm9   # into the right registers
+	sub	esi, 1
+	jnz	.loop
+
+	# now we reduce the partial sums.
+	vpslld	ymm3, ymm4, 6
+	vpsrldq	ymm2, ymm6, 4
+	
+	vpaddd  ymm0, ymm3, ymm1
+	vpaddd	ymm6, ymm2, ymm6
+	vpsrlq	ymm3, ymm0, 32
+	
+	vpsrldq	ymm2, ymm6, 8
+	vpaddd	ymm0, ymm3, ymm0
+	vpsrldq	ymm3, ymm0, 8
+	vpaddd	ymm6, ymm2, ymm6
+	vpaddd	ymm0, ymm3, ymm0
+	vextracti128	xmm2, ymm6, 0x1
+	vextracti128	xmm1, ymm0, 0x1
+	vpaddd	xmm6, xmm2, xmm6
+	vmovd	[rcx], xmm6
+	vpaddd	xmm1, xmm1, xmm0
+	vmovd	ecx, xmm1
+	add	eax, ecx
+	mov	[r8], eax
+.exit:
+	vzeroupper
+	mov	eax, edx
+	ret
+
+#ifdef __APPLE__
+.data
+	.align 6
+#else
+.section	.rodata
+	.p2align 6 
+#endif
+.mul_T2: 
+	.byte 64
+	.byte 63
+	.byte 62
+	.byte 61
+	.byte 60
+	.byte 59
+	.byte 58
+	.byte 57
+	.byte 56
+	.byte 55
+	.byte 54
+	.byte 53
+	.byte 52
+	.byte 51
+	.byte 50
+	.byte 49
+	.byte 48
+	.byte 47
+	.byte 46
+	.byte 45
+	.byte 44
+	.byte 43
+	.byte 42
+	.byte 41
+	.byte 40
+	.byte 39
+	.byte 38
+	.byte 37
+	.byte 36
+	.byte 35
+	.byte 34
+	.byte 33
+	.byte 32
+	.byte 31
+	.byte 30
+	.byte 29
+	.byte 28
+	.byte 27
+	.byte 26
+	.byte 25
+	.byte 24
+	.byte 23
+	.byte 22
+	.byte 21
+	.byte 20
+	.byte 19
+	.byte 18
+	.byte 17
+	.byte 16
+	.byte 15
+	.byte 14
+	.byte 13
+	.byte 12
+	.byte 11
+	.byte 10
+	.byte 9
+	.byte 8
+	.byte 7
+	.byte 6
+	.byte 5
+	.byte 4
+	.byte 3
+	.byte 2
+	.byte 1

simd-checksum-x86_64.cpp

@@ -85,7 +85,6 @@ typedef long long __m256i_u __attribute__((__vector_size__(32), __may_alias__, _
 #define SSE2_HADDS_EPI16(a, b) _mm_adds_epi16(SSE2_INTERLEAVE_EVEN_EPI16(a, b), SSE2_INTERLEAVE_ODD_EPI16(a, b))
 #define SSE2_MADDUBS_EPI16(a, b) _mm_adds_epi16(SSE2_MULU_EVEN_EPI8(a, b), SSE2_MULU_ODD_EPI8(a, b))
 
-__attribute__ ((target("default"))) MVSTATIC int32 get_checksum1_avx2_64(schar* buf, int32 len, int32 i, uint32* ps1, uint32* ps2) { return i; }
 __attribute__ ((target("default"))) MVSTATIC int32 get_checksum1_ssse3_32(schar* buf, int32 len, int32 i, uint32* ps1, uint32* ps2) { return i; }
 __attribute__ ((target("default"))) MVSTATIC int32 get_checksum1_sse2_32(schar* buf, int32 len, int32 i, uint32* ps1, uint32* ps2) { return i; }
 
@@ -246,7 +245,7 @@ __attribute__ ((target("sse2"))) MVSTATIC int32 get_checksum1_sse2_32(schar* buf
 
             // (4*buf[i] + 3*buf[i+1]), (2*buf[i+2], buf[i+3]), ... 2*[int16*8]
             __m128i mul_const = _mm_set1_epi32(4 + (3 << 8) + (2 << 16) + (1 << 24));
-            __m128i mul_add16_1 = SSE2_MADDUBS_EPI16(mul_const, in8_1);
+            __m128i mul_add16_1 = SSE2_MADDUBS_EPI16(mul_const, in8_1);
             __m128i mul_add16_2 = SSE2_MADDUBS_EPI16(mul_const, in8_2);
 
             // s2 += 32*s1
@@ -311,127 +310,7 @@ __attribute__ ((target("sse2"))) MVSTATIC int32 get_checksum1_sse2_32(schar* buf
     return i;
 }
 
-/*
-  AVX2 loop per 64 bytes:
-    int16 t1[16];
-    int16 t2[16];
-    for (int j = 0; j < 16; j++) {
-      t1[j] = buf[j*4 + i] + buf[j*4 + i+1] + buf[j*4 + i+2] + buf[j*4 + i+3];
-      t2[j] = 4*buf[j*4 + i] + 3*buf[j*4 + i+1] + 2*buf[j*4 + i+2] + buf[j*4 + i+3];
-    }
-    s2 += 64*s1 + (uint32)(
-              60*t1[0] + 56*t1[1] + 52*t1[2] + 48*t1[3] + 44*t1[4] + 40*t1[5] + 36*t1[6] + 32*t1[7] + 28*t1[8] + 24*t1[9] + 20*t1[10] + 16*t1[11] + 12*t1[12] + 8*t1[13] + 4*t1[14] +
-              t2[0] + t2[1] + t2[2] + t2[3] + t2[4] + t2[5] + t2[6] + t2[7] + t2[8] + t2[9] + t2[10] + t2[11] + t2[12] + t2[13] + t2[14] + t2[15]
-          ) + 2080*CHAR_OFFSET;
-    s1 += (uint32)(t1[0] + t1[1] + t1[2] + t1[3] + t1[4] + t1[5] + t1[6] + t1[7] + t1[8] + t1[9] + t1[10] + t1[11] + t1[12] + t1[13] + t1[14] + t1[15]) +
-          64*CHAR_OFFSET;
- */
-__attribute__ ((target("avx2"))) MVSTATIC int32 get_checksum1_avx2_64(schar* buf, int32 len, int32 i, uint32* ps1, uint32* ps2)
-{
-    if (len > 64) {
-        // Instructions reshuffled compared to SSE2 for slightly better performance
-        int aligned = ((uintptr_t)buf & 31) == 0;
-
-        uint32 x[8] = {0};
-        x[0] = *ps1;
-        __m256i ss1 = _mm256_lddqu_si256((__m256i_u*)x);
-        x[0] = *ps2;
-        __m256i ss2 = _mm256_lddqu_si256((__m256i_u*)x);
-
-        // The order gets shuffled compared to SSE2
-        const int16 mul_t1_buf[16] = {60, 56, 52, 48, 28, 24, 20, 16, 44, 40, 36, 32, 12, 8, 4, 0};
-        __m256i mul_t1 = _mm256_lddqu_si256((__m256i_u*)mul_t1_buf);
-
-        for (; i < (len-64); i+=64) {
-            // Load ... 2*[int8*32]
-            __m256i in8_1, in8_2;
-            if (!aligned) {
-                in8_1 = _mm256_lddqu_si256((__m256i_u*)&buf[i]);
-                in8_2 = _mm256_lddqu_si256((__m256i_u*)&buf[i + 32]);
-            } else {
-                in8_1 = _mm256_load_si256((__m256i_u*)&buf[i]);
-                in8_2 = _mm256_load_si256((__m256i_u*)&buf[i + 32]);
-            }
-
-            // Prefetch for next loops. This has no observable effect on the
-            // tested AMD but makes as much as 20% difference on the Intel.
-            // Curiously that same Intel sees no benefit from this with SSE2
-            // or SSSE3.
-            _mm_prefetch(&buf[i + 64], _MM_HINT_T0);
-            _mm_prefetch(&buf[i + 96], _MM_HINT_T0);
-            _mm_prefetch(&buf[i + 128], _MM_HINT_T0);
-            _mm_prefetch(&buf[i + 160], _MM_HINT_T0);
-
-            // (1*buf[i] + 1*buf[i+1]), (1*buf[i+2], 1*buf[i+3]), ... 2*[int16*16]
-            // Fastest, even though multiply by 1
-            __m256i mul_one = _mm256_set1_epi8(1);
-            __m256i add16_1 = _mm256_maddubs_epi16(mul_one, in8_1);
-            __m256i add16_2 = _mm256_maddubs_epi16(mul_one, in8_2);
-
-            // (4*buf[i] + 3*buf[i+1]), (2*buf[i+2], buf[i+3]), ... 2*[int16*16]
-            __m256i mul_const = _mm256_set1_epi32(4 + (3 << 8) + (2 << 16) + (1 << 24));
-            __m256i mul_add16_1 = _mm256_maddubs_epi16(mul_const, in8_1);
-            __m256i mul_add16_2 = _mm256_maddubs_epi16(mul_const, in8_2);
-
-            // s2 += 64*s1
-            ss2 = _mm256_add_epi32(ss2, _mm256_slli_epi32(ss1, 6));
-
-            // [t1[0] + t1[1], t1[2] + t1[3] ...] [int16*16]
-            __m256i add16 = _mm256_hadds_epi16(add16_1, add16_2);
-
-            // [t1[0], t1[1], ...] -> [t1[0]*60 + t1[1]*56, ...] [int32*8]
-            __m256i mul32 = _mm256_madd_epi16(add16, mul_t1);
-
-            // [sum(t1[0]..t1[15]), X, X, X, X, X, X, X] [int32*8]
-            __m256i sum_add32 = _mm256_add_epi16(add16_1, add16_2);
-            sum_add32 = _mm256_add_epi16(sum_add32, _mm256_permute4x64_epi64(sum_add32, 2 + (3 << 2) + (0 << 4) + (1 << 6)));
-            sum_add32 = _mm256_add_epi16(sum_add32, _mm256_slli_si256(sum_add32, 2));
-            sum_add32 = _mm256_add_epi16(sum_add32, _mm256_slli_si256(sum_add32, 4));
-            sum_add32 = _mm256_add_epi16(sum_add32, _mm256_slli_si256(sum_add32, 8));
-            sum_add32 = _mm256_srai_epi32(sum_add32, 16);
-            sum_add32 = _mm256_shuffle_epi32(sum_add32, 3);
-
-            // s1 += t1[0] + t1[1] + t1[2] + t1[3] + t1[4] + t1[5] + t1[6] + t1[7] + t1[8] + t1[9] + t1[10] + t1[11] + t1[12] + t1[13] + t1[14] + t1[15]
-            ss1 = _mm256_add_epi32(ss1, sum_add32);
-
-            // [sum(t2[0]..t2[15]), X, X, X, X, X, X, X] [int32*8]
-            __m256i sum_mul_add32 = _mm256_add_epi16(mul_add16_1, mul_add16_2);
-            sum_mul_add32 = _mm256_add_epi16(sum_mul_add32, _mm256_permute4x64_epi64(sum_mul_add32, 2 + (3 << 2) + (0 << 4) + (1 << 6)));
-            sum_mul_add32 = _mm256_add_epi16(sum_mul_add32, _mm256_slli_si256(sum_mul_add32, 2));
-            sum_mul_add32 = _mm256_add_epi16(sum_mul_add32, _mm256_slli_si256(sum_mul_add32, 4));
-            sum_mul_add32 = _mm256_add_epi16(sum_mul_add32, _mm256_slli_si256(sum_mul_add32, 8));
-            sum_mul_add32 = _mm256_srai_epi32(sum_mul_add32, 16);
-            sum_mul_add32 = _mm256_shuffle_epi32(sum_mul_add32, 3);
-
-            // s2 += t2[0] + t2[1] + t2[2] + t2[3] + t2[4] + t2[5] + t2[6] + t2[7] + t2[8] + t2[9] + t2[10] + t2[11] + t2[12] + t2[13] + t2[14] + t2[15]
-            ss2 = _mm256_add_epi32(ss2, sum_mul_add32);
-
-            // [sum(mul32), X, X, X, X, X, X, X] [int32*8]
-            mul32 = _mm256_add_epi32(mul32, _mm256_permute2x128_si256(mul32, mul32, 1));
-            mul32 = _mm256_add_epi32(mul32, _mm256_srli_si256(mul32, 4));
-            mul32 = _mm256_add_epi32(mul32, _mm256_srli_si256(mul32, 8));
-
-            // s2 += 60*t1[0] + 56*t1[1] + 52*t1[2] + 48*t1[3] + 44*t1[4] + 40*t1[5] + 36*t1[6] + 32*t1[7] + 28*t1[8] + 24*t1[9] + 20*t1[10] + 16*t1[11] + 12*t1[12] + 8*t1[13] + 4*t1[14]
-            ss2 = _mm256_add_epi32(ss2, mul32);
-
-#if CHAR_OFFSET != 0
-            // s1 += 64*CHAR_OFFSET
-            __m256i char_offset_multiplier = _mm256_set1_epi32(64 * CHAR_OFFSET);
-            ss1 = _mm256_add_epi32(ss1, char_offset_multiplier);
-
-            // s2 += 2080*CHAR_OFFSET
-            char_offset_multiplier = _mm256_set1_epi32(2080 * CHAR_OFFSET);
-            ss2 = _mm256_add_epi32(ss2, char_offset_multiplier);
-#endif
-        }
-
-        _mm256_store_si256((__m256i_u*)x, ss1);
-        *ps1 = x[0];
-        _mm256_store_si256((__m256i_u*)x, ss2);
-        *ps2 = x[0];
-    }
-    return i;
-}
+extern "C" __attribute__ ((target("avx2"))) int32 get_checksum1_avx2(schar* buf, int32 len, int32 i, uint32* ps1, uint32* ps2);
 
 static int32 get_checksum1_default_1(schar* buf, int32 len, int32 i, uint32* ps1, uint32* ps2)
 {
@@ -459,7 +338,7 @@ static inline uint32 get_checksum1_cpp(char *buf1, int32 len)
     uint32 s2 = 0;
 
     // multiples of 64 bytes using AVX2 (if available)
-    i = get_checksum1_avx2_64((schar*)buf1, len, i, &s1, &s2);
+    i = get_checksum1_avx2((schar*)buf1, len, i, &s1, &s2);
 
     // multiples of 32 bytes using SSSE3 (if available)
     i = get_checksum1_ssse3_32((schar*)buf1, len, i, &s1, &s2);
@@ -521,14 +400,14 @@ static int32 get_checksum1_auto(schar* buf, int32 len, int32 i, uint32* ps1, uin
 
 int main() {
     int i;
-    unsigned char* buf = (unsigned char*)malloc(BLOCK_LEN);
+    unsigned char* buf = (unsigned char*)aligned_alloc(64,BLOCK_LEN);
     for (i = 0; i < BLOCK_LEN; i++) buf[i] = (i + (i % 3) + (i % 11)) % 256;
 
     benchmark("Auto", get_checksum1_auto, (schar*)buf, BLOCK_LEN);
     benchmark("Raw-C", get_checksum1_default_1, (schar*)buf, BLOCK_LEN);
     benchmark("SSE2", get_checksum1_sse2_32, (schar*)buf, BLOCK_LEN);
     benchmark("SSSE3", get_checksum1_ssse3_32, (schar*)buf, BLOCK_LEN);
-    benchmark("AVX2", get_checksum1_avx2_64, (schar*)buf, BLOCK_LEN);
+    benchmark("AVX2", get_checksum1_avx2, (schar*)buf, BLOCK_LEN);
 
     free(buf);
     return 0;

support/atomic-rsync

@@ -1,92 +1,94 @@
-#!/usr/bin/env perl
-#
+#!/usr/bin/env python3
 # This script lets you update a hierarchy of files in an atomic way by
 # first creating a new hierarchy using rsync's --link-dest option, and
 # then swapping the hierarchy into place.  **See the usage message for
 # more details and some important caveats!**
 
-use strict;
-use warnings;
-use Cwd 'abs_path';
+import os, sys, re, subprocess, shutil
 
-my $RSYNC_PROG = '/usr/bin/rsync';
-my $RM_PROG = '/bin/rm';
+ALT_DEST_ARG_RE = re.compile('^--[a-z][^ =]+-dest(=|$)')
 
-my $dest_dir = $ARGV[-1];
-&usage if !defined $dest_dir || $dest_dir =~ /(^-|^$)/ || grep(/^--help/, @ARGV);
-$dest_dir =~ s{(?<=.)/+$} {};
+RSYNC_PROG = '/usr/bin/rsync'
 
-if (!-d $dest_dir) {
-    die "$dest_dir is not a directory.\nUse --help for help.\n";
-}
+def main():
+    cmd_args = sys.argv[1:]
+    if '--help' in cmd_args:
+        usage_and_exit()
 
-if (@_ = grep(/^--[a-z]+-dest\b/, @ARGV)) {
-    $_ = join(' or ', @_);
-    die "You cannot use the $_ option with atomic-rsync.\nUse --help for help.\n";
-}
+    if len(cmd_args) < 2:
+        usage_and_exit(True)
 
-my $symlink_content = readlink $dest_dir; # undef when a real dir
+    dest_dir = cmd_args[-1].rstrip('/')
+    if dest_dir == '' or dest_dir.startswith('-'):
+        usage_and_exit(True)
 
-my $dest_arg = $dest_dir;
-# This gives us the real destination dir, with all symlinks dereferenced.
-$dest_dir = abs_path($dest_dir);
-if ($dest_dir eq '/') {
-    die qq|You must not use "/" as the destination directory.\nUse --help for help.\n|;
-}
+    if not os.path.isdir(dest_dir):
+        die(dest_dir, "is not a directory or a symlink to a dir.\nUse --help for help.")
 
-my($old_dir, $new_dir);
-if (defined $symlink_content && $dest_dir =~ /-([12])$/) {
-    my $num = 3 - $1;
-    $old_dir = undef;
-    ($new_dir = $dest_dir) =~ s/-[12]$/-$num/;
-    $symlink_content =~ s/-[12]$/-$num/;
-} else {
-    $old_dir = "$dest_dir~old~";
-    $new_dir = "$dest_dir~new~";
-}
+    bad_args = [ arg for arg in cmd_args if ALT_DEST_ARG_RE.match(arg) ]
+    if bad_args:
+        die("You cannot use the", ' or '.join(bad_args), "option with atomic-rsync.\nUse --help for help.")
 
-$ARGV[-1] = "$new_dir/";
+    # We ignore exit-code 24 (file vanished) by default.
+    allowed_exit_codes = '0 ' + os.environ.get('ATOMIC_RSYNC_OK_CODES', '24')
+    try:
+        allowed_exit_codes = set(int(num) for num in re.split(r'[, ]+', allowed_exit_codes) if num != '')
+    except ValueError:
+        die('Invalid integer in ATOMIC_RSYNC_OK_CODES:', allowed_exit_codes[2:])
 
-system($RM_PROG, '-rf', $old_dir) if defined $old_dir && -d $old_dir;
-system($RM_PROG, '-rf', $new_dir) if -d $new_dir;
+    symlink_content = os.readlink(dest_dir) if os.path.islink(dest_dir) else None
 
-if (system($RSYNC_PROG, "--link-dest=$dest_dir", @ARGV)) {
-    if ($? == -1) {
-	print "failed to execute $RSYNC_PROG: $!\n";
-    } elsif ($? & 127) {
-	printf "child died with signal %d, %s coredump\n",
-	    ($? & 127),  ($? & 128) ? 'with' : 'without';
-    } else {
-	printf "child exited with value %d\n", $? >> 8;
-    }
-    exit 1;
-}
+    dest_arg = dest_dir
+    dest_dir = os.path.realpath(dest_dir) # The real destination dir with all symlinks dereferenced
+    if dest_dir == '/':
+        die('You must not use "/" as the destination directory.\nUse --help for help.')
 
-if (!defined $old_dir) {
-    atomic_symlink($symlink_content, $dest_arg);
-    exit;
-}
+    old_dir = new_dir = None
+    if symlink_content is not None and dest_dir.endswith(('-1','-2')):
+        if not symlink_content.endswith(dest_dir[-2:]):
+            die("Symlink suffix out of sync with dest_dir name:", symlink_content, 'vs', dest_dir)
+        num = 3 - int(dest_dir[-1]);
+        old_dir = None
+        new_dir = dest_dir[:-1] + str(num)
+        symlink_content = symlink_content[:-1] + str(num)
+    else:
+        old_dir = dest_dir + '~old~'
+        new_dir = dest_dir + '~new~'
 
-rename($dest_dir, $old_dir) or die "Unable to rename $dest_dir to $old_dir: $!";
-rename($new_dir, $dest_dir) or die "Unable to rename $new_dir to $dest_dir: $!";
+    cmd_args[-1] = new_dir + '/'
 
-exit;
+    if old_dir is not None and os.path.isdir(old_dir):
+        shutil.rmtree(old_dir)
+    if os.path.isdir(new_dir):
+        shutil.rmtree(new_dir)
 
-sub atomic_symlink
-{
-    my($target, $link) = @_;
-    my $newlink = "$link~new~";
+    child = subprocess.run([RSYNC_PROG, '--link-dest=' + dest_dir, *cmd_args])
+    if child.returncode not in allowed_exit_codes:
+        die('The rsync copy failed with code', child.returncode, exitcode=child.returncode)
 
-    unlink($newlink); # Just in case
-    symlink($target, $newlink) or die "Unable to symlink $newlink -> $target: $!\n";
-    rename($newlink, $link) or die "Unable to rename $newlink to $link: $!\n";
-}
+    if not os.path.isdir(new_dir):
+        die('The rsync copy failed to create:', new_dir)
+
+    if old_dir is None:
+        atomic_symlink(symlink_content, dest_arg)
+    else:
+        os.rename(dest_dir, old_dir)
+        os.rename(new_dir, dest_dir)
 
 
-sub usage
-{
-    die <<EOT;
-Usage: atomic-rsync [RSYNC-OPTIONS] HOST:/SOURCE/DIR/ /DEST/DIR/
+def atomic_symlink(target, link):
+    newlink = link + "~new~"
+    try:
+        os.unlink(newlink); # Just in case
+    except OSError:
+        pass
+    os.symlink(target, newlink)
+    os.rename(newlink, link)
+
+
+def usage_and_exit(use_stderr=False):
+    usage_msg = """\
+Usage: atomic-rsync [RSYNC-OPTIONS] [HOST:]/SOURCE/DIR/ /DEST/DIR/
        atomic-rsync [RSYNC-OPTIONS] HOST::MOD/DIR/ /DEST/DIR/
 
 This script lets you update a hierarchy of files in an atomic way by first
@@ -96,27 +98,41 @@ to a local directory, and that directory must already exist.  For example:
 
     mkdir /local/files-1
     ln -s files-1 /local/files
-    atomic-rsync -av host:/remote/files/ /local/files/
+    atomic-rsync -aiv host:/remote/files/ /local/files/
 
-If /local/files is a symlink to a directory that ends in -1 or -2, the
-copy will go to the alternate suffix and the symlink will be changed to
-point to the new dir.  This is a fully atomic update.  If the destination
-is not a symlink (or not a symlink to a *-1 or a *-2 directory), this
-will instead create a directory with "~new~" suffixed, move the current
-directory to a name with "~old~" suffixed, and then move the ~new~
-directory to the original destination name (this double rename is not
-fully atomic, but is rapid).  In both cases, the prior destintaion
-directory will be preserved until the next update, at which point it
-will be deleted.
+If /local/files is a symlink to a directory that ends in -1 or -2, the copy
+will go to the alternate suffix and the symlink will be changed to point to
+the new dir.  This is a fully atomic update.  If the destination is not a
+symlink (or not a symlink to a *-1 or a *-2 directory), this will instead
+create a directory with "~new~" suffixed, move the current directory to a
+name with "~old~" suffixed, and then move the ~new~ directory to the original
+destination name (this double rename is not fully atomic, but is rapid).  In
+both cases, the prior destintaion directory will be preserved until the next
+update, at which point it will be deleted.
 
-In all likelihood, you do NOT want to specify this command:
+By default, rsync exit-code 24 (file vanished) is allowed without halting the
+atomic update.  If you want to change that, specify the environment variable
+ATOMIC_RSYNC_OK_CODES with numeric values separated by spaces and/or commas.
+Specify an empty string to only allow a successful copy.  An override example:
 
-    atomic-rsync -av host:/remote/files /local/
+    ATOMIC_RSYNC_OK_CODES='23 24' atomic-rsync -aiv host:src/ dest/
 
-... UNLESS you want the entire /local dir to be swapped out!
+See the errcode.h file for a list of all the exit codes.
 
 See the "rsync" command for its list of options.  You may not use the
 --link-dest, --compare-dest, or --copy-dest options (since this script
 uses --link-dest to make the transfer efficient).
-EOT
-}
+"""
+    print(usage_msg, file=sys.stderr if use_stderr else sys.stdout)
+    sys.exit(1 if use_stderr else 0)
+
+
+def die(*args, exitcode=1):
+    print(*args, file=sys.stderr)
+    sys.exit(exitcode)
+
+
+if __name__ == '__main__':
+    main()
+
+# vim: sw=4 et

support/git-set-file-times

@@ -27,11 +27,12 @@ def main():
 
     if not args.tree:
         # All modified files keep their current mtime.
-        proc = subprocess.Popen(git + 'ls-files -m -z'.split(), stdout=subprocess.PIPE, encoding='utf-8')
+        proc = subprocess.Popen(git + 'status -z --no-renames'.split(), stdout=subprocess.PIPE, encoding='utf-8')
         out = proc.communicate()[0]
         for fn in out.split('\0'):
-            if fn == '':
+            if fn == '' or (fn[0] != 'M' and fn[1] != 'M'):
                 continue
+            fn = fn[3:]
             if args.list:
                 mtime = os.lstat(fn).st_mtime
                 print_line(fn, mtime, mtime)

support/lsh

@@ -15,6 +15,8 @@ GetOptions(
     'b|c|D|e|F|i|L|m|O|o|p|R|S|w=s' => sub { }, # Ignore
     'no-cd' => \( my $no_chdir ),
     'sudo' => \( my $use_sudo ),
+    'rrsync=s' => \( my $rrsync_dir ),
+    'rropts=s' => \( my $rrsync_opts ),
 ) or &usage;
 &usage unless @ARGV > 1;
 
@@ -67,22 +69,40 @@ unless ($no_chdir) {
     chdir $home_dir or die "Unable to chdir to $home_dir: $!\n";
 }
 
-push @cmd, '/bin/sh', '-c', "@ARGV";
+if ($rrsync_dir) {
+    $ENV{SSH_ORIGINAL_COMMAND} = join(' ', @ARGV);
+    push @cmd, 'rrsync';
+    if ($rrsync_opts) {
+	foreach my $opt (split(/[ ,]+/, $rrsync_opts)) {
+	    $opt = "-$opt" unless $opt =~ /^-/;
+	    push @cmd, $opt;
+	}
+    }
+    push @cmd, $rrsync_dir;
+} else {
+    push @cmd, '/bin/sh', '-c', "@ARGV";
+}
 exec @cmd;
 die "Failed to exec: $!\n";
 
 sub usage
 {
     die <<EOT;
-Usage: lsh [-l USER] [--sudo] [--no-cd] localhost COMMAND [...]
+Usage: lsh [OPTIONS] localhost|lh COMMAND [...]
 
 This is a "local shell" command that works like a remote shell but only for the
 local host.  This is useful for rsync testing or for running a local copy where
 the sender and the receiver need to use different options (e.g. --fake-super).
-If the -l option is used, we try to become the USER, either directly (when
-root) or by using "sudo -H -u USER" (requires --sudo option).
 
-Note that if you pass hostname "lh" instead of "localhost" that the --no-cd
-option is implied.  The default is to "cd \$HOME" to simulate ssh behavior.
+Options:
+
+-l USER        Choose the USER that lsh tries to become.
+--no-cd        Skip the chdir \$HOME (the default with hostname "lh")
+--sudo         Use sudo -H -l USER to become root or the specified USER.
+--rrsync=DIR   Test rrsync restricted copying without using ssh.
+--rropts=STR   The string "munge,no-del,no-lock" would pass 3 options to
+               rrsync (must be combined with --rrsync=DIR).
+
+The script also ignores a bunch of single-letter ssh options.
 EOT
 }

support/lsh.sh

@@ -4,7 +4,8 @@
 # for testing or for running a local copy where the sender and the
 # receiver needs to use different options (e.g. --fake-super).  If
 # we get a -l USER option, we try to use "sudo -u USER" to run the
-# command.
+# command.  Supports only the hostnames "localhost" and "lh", with
+# the latter implying the --no-cd option.
 
 user=''
 do_cd=y # Default path is user's home dir (just like ssh) unless host is "lh".

support/munge-symlinks

@@ -1,60 +1,71 @@
-#!/usr/bin/env perl
+#!/usr/bin/env python3
 # This script will either prefix all symlink values with the string
 # "/rsyncd-munged/" or remove that prefix.
 
-use strict;
-use Getopt::Long;
+import os, sys, argparse
 
-my $SYMLINK_PREFIX = '/rsyncd-munged/';
+SYMLINK_PREFIX = '/rsyncd-munged/'
+PREFIX_LEN = len(SYMLINK_PREFIX)
 
-my $munge_opt;
+def main():
+    for arg in args.names:
+        if os.path.islink(arg):
+            process_one_arg(arg)
+        elif os.path.isdir(arg):
+            for fn in find_symlinks(arg):
+                process_one_arg(fn)
+        else:
+            print("Arg is not a symlink or a dir:", arg, file=sys.stderr)
 
-&GetOptions(
-    'munge' => sub { $munge_opt = 1 },
-    'unmunge' => sub { $munge_opt = 0 },
-    'all' => \( my $all_opt ),
-    'help|h' => \( my $help_opt ),
-) or &usage;
 
-&usage if $help_opt || !defined $munge_opt;
+def find_symlinks(path):
+    for entry in os.scandir(path):
+        if entry.is_symlink():
+            yield entry.path
+        elif entry.is_dir(follow_symlinks=False):
+            yield from find_symlinks(entry.path)
 
-my $munged_re = $all_opt ? qr/^($SYMLINK_PREFIX)+(?=.)/ : qr/^$SYMLINK_PREFIX(?=.)/;
 
-push(@ARGV, '.') unless @ARGV;
+def process_one_arg(fn):
+    lnk = os.readlink(fn)
+    if args.unmunge:
+        if not lnk.startswith(SYMLINK_PREFIX):
+            return
+        lnk = lnk[PREFIX_LEN:]
+        while args.all and lnk.startswith(SYMLINK_PREFIX):
+            lnk = lnk[PREFIX_LEN:]
+    else:
+        if not args.all and lnk.startswith(SYMLINK_PREFIX):
+            return
+        lnk = SYMLINK_PREFIX + lnk
 
-open(PIPE, '-|', 'find', @ARGV, '-type', 'l') or die $!;
+    try:
+        os.unlink(fn)
+    except OSError as e:
+        print("Unable to unlink symlink:", str(e), file=sys.stderr)
+        return
+    try:
+        os.symlink(lnk, fn)
+    except OSError as e:
+        print("Unable to recreate symlink", fn, '->', lnk + ':', str(e), file=sys.stderr)
+        return
+    print(fn, '->', lnk)
 
-while (<PIPE>) {
-    chomp;
-    my $lnk = readlink($_) or next;
-    if ($munge_opt) {
-	next if !$all_opt && $lnk =~ /$munged_re/;
-	$lnk =~ s/^/$SYMLINK_PREFIX/;
-    } else {
-	next unless $lnk =~ s/$munged_re//;
-    }
-    if (!unlink($_)) {
-	warn "Unable to unlink symlink: $_ ($!)\n";
-    } elsif (!symlink($lnk, $_)) {
-	warn "Unable to recreate symlink: $_ -> $lnk ($!)\n";
-    } else {
-	print "$_ -> $lnk\n";
-    }
-}
 
-close PIPE;
-exit;
+if __name__ == '__main__':
+    our_desc = """\
+Adds or removes the %s prefix to/from the start of each symlink's value.
+When given the name of a directory, affects all the symlinks in that directory hierarchy.
+""" % SYMLINK_PREFIX
+    epilog = 'See the "munge symlinks" option in the rsyncd.conf manpage for more details.'
+    parser = argparse.ArgumentParser(description=our_desc, epilog=epilog, add_help=False)
+    uniq_group = parser.add_mutually_exclusive_group()
+    uniq_group.add_argument('--munge', action='store_true', help="Add the prefix to symlinks (the default).")
+    uniq_group.add_argument('--unmunge', action='store_true', help="Remove the prefix from symlinks.")
+    parser.add_argument('--all', action='store_true', help="Always adds the prefix when munging (even if already munged) or removes multiple instances of the prefix when unmunging.")
+    parser.add_argument('--help', '-h', action='help', help="Output this help message and exit.")
+    parser.add_argument('names', metavar='NAME', nargs='+', help="One or more directories and/or symlinks to process.")
+    args = parser.parse_args()
+    main()
 
-sub usage
-{
-    die <<EOT;
-Usage: munge-symlinks --munge|--unmunge [--all] [DIR|SYMLINK...]
-
---munge      Add the $SYMLINK_PREFIX prefix to symlinks if not already
-             present, or always when combined with --all.
---unmunge    Remove one $SYMLINK_PREFIX prefix from symlinks or all
-             such prefixes with --all.
-
-See the "munge symlinks" option in the rsyncd.conf manpage for more details.
-EOT
-}
+# vim: sw=4 et

support/rrsync

@@ -1,252 +1,378 @@
-#!/usr/bin/env perl
-# Name: /usr/local/bin/rrsync (should also have a symlink in /usr/bin)
-# Purpose: Restricts rsync to subdirectory declared in .ssh/authorized_keys
-# Author: Joe Smith <js-cgi@inwap.com> 30-Sep-2004
-# Modified by: Wayne Davison <wayne@opencoder.net>
-use strict;
+#!/usr/bin/env python3
 
-use Socket;
-use Cwd 'abs_path';
-use File::Glob ':glob';
+# Restricts rsync to subdirectory declared in .ssh/authorized_keys.  See
+# the rrsync man page for details of how to make use of this script.
 
-# You may configure these values to your liking.  See also the section
-# of options if you want to disable any options that rsync accepts.
-use constant RSYNC => '/usr/bin/rsync';
-use constant LOGFILE => 'rrsync.log';
+# NOTE: install python3 braceexpand to support brace expansion in the args!
 
-my $Usage = <<EOM;
-Use 'command="$0 [-ro|-wo] SUBDIR"'
-	in front of lines in $ENV{HOME}/.ssh/authorized_keys
-EOM
+# Originally a perl script by: Joe Smith <js-cgi@inwap.com> 30-Sep-2004
+# Python version by: Wayne Davison <wayne@opencoder.net>
 
-# Handle the -ro and -wo options.
-our $only = '';
-while (@ARGV && $ARGV[0] =~ /^-([rw])o$/) {
-    my $r_or_w = $1;
-    if ($only && $only ne $r_or_w) {
-	die "$0: the -ro and -wo options conflict.\n";
-    }
-    $only = $r_or_w;
-    shift;
-}
+# You may configure these 2 values to your liking.  See also the section of
+# short & long options if you want to disable any options that rsync accepts.
+RSYNC = '/usr/bin/rsync'
+LOGFILE = 'rrsync.log' # NOTE: the file must exist for a line to be appended!
 
-our $subdir = shift;
-die "$0: No subdirectory specified\n$Usage" unless defined $subdir;
-$subdir = abs_path($subdir);
-die "$0: Restricted directory does not exist!\n" if $subdir ne '/' && !-d $subdir;
+# The following options are mainly the options that a client rsync can send
+# to the server, and usually just in the one option format that the stock
+# rsync produces. However, there are some additional convenience options
+# added as well, and thus a few options are present in both the short and
+# long lists (such as --group, --owner, and --perms).
 
-# The client uses "rsync -av -e ssh src/ server:dir/", and sshd on the server
-# executes this program when .ssh/authorized_keys has 'command="..."'.
-# For example:
-# command="rrsync logs/client" ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAIEAzGhEeNlPr...
-# command="rrsync -ro results" ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAIEAmkHG1WCjC...
-#
-# Format of the environment variables set by sshd:
-# SSH_ORIGINAL_COMMAND=rsync --server          -vlogDtpr --partial . ARG # push
-# SSH_ORIGINAL_COMMAND=rsync --server --sender -vlogDtpr --partial . ARGS # pull
-# SSH_CONNECTION=client_addr client_port server_port
+# NOTE when disabling: check for both a short & long version of the option!
 
-my $command = $ENV{SSH_ORIGINAL_COMMAND};
-die "$0: Not invoked via sshd\n$Usage"	unless defined $command;
-die "$0: SSH_ORIGINAL_COMMAND='$command' is not rsync\n" unless $command =~ s/^rsync\s+//;
-die "$0: --server option is not first\n" unless $command =~ /^--server\s/;
-our $am_sender = $command =~ /^--server\s+--sender\s/; # Restrictive on purpose!
-die "$0 sending to read-only server not allowed\n" if $only eq 'r' && !$am_sender;
-die "$0 reading from write-only server not allowed\n" if $only eq 'w' && $am_sender;
-
-### START of options data produced by the cull_options script. ###
-
-# These options are the only options that rsync might send to the server,
-# and only in the option format that the stock rsync produces.
+### START of options data produced by the cull-options script. ###
 
 # To disable a short-named option, add its letter to this string:
-our $short_disabled = 's';
+short_disabled = 's'
 
-our $short_no_arg = 'ACDEHIJKLORSUWXbcdgklmnopqrstuvxyz'; # DO NOT REMOVE ANY
-our $short_with_num = '@B'; # DO NOT REMOVE ANY
+# These are also disabled when the restricted dir is not "/":
+short_disabled_subdir = 'KLk'
+
+# These are all possible short options that we will accept (when not disabled above):
+short_no_arg = 'ACDEHIJKLNORSUWXbcdgklmnopqrstuvxyz' # DO NOT REMOVE ANY
+short_with_num = '@B' # DO NOT REMOVE ANY
 
 # To disable a long-named option, change its value to a -1.  The values mean:
 # 0 = the option has no arg; 1 = the arg doesn't need any checking; 2 = only
 # check the arg when receiving; and 3 = always check the arg.
-our %long_opt = (
-  'append' => 0,
-  'backup-dir' => 2,
-  'block-size' => 1,
-  'bwlimit' => 1,
-  'checksum-choice' => 1,
-  'checksum-seed' => 1,
-  'compare-dest' => 2,
-  'compress-choice' => 1,
-  'compress-level' => 1,
-  'copy-dest' => 2,
-  'copy-unsafe-links' => 0,
-  'daemon' => -1,
-  'debug' => 1,
-  'delay-updates' => 0,
-  'delete' => 0,
-  'delete-after' => 0,
-  'delete-before' => 0,
-  'delete-delay' => 0,
-  'delete-during' => 0,
-  'delete-excluded' => 0,
-  'delete-missing-args' => 0,
-  'existing' => 0,
-  'fake-super' => 0,
-  'files-from' => 3,
-  'force' => 0,
-  'from0' => 0,
-  'fuzzy' => 0,
-  'group' => 0,
-  'groupmap' => 1,
-  'hard-links' => 0,
-  'iconv' => 1,
-  'ignore-errors' => 0,
-  'ignore-existing' => 0,
-  'ignore-missing-args' => 0,
-  'ignore-times' => 0,
-  'info' => 1,
-  'inplace' => 0,
-  'link-dest' => 2,
-  'links' => 0,
-  'list-only' => 0,
-  'log-file' => $only eq 'r' ? -1 : 3,
-  'log-format' => 1,
-  'max-alloc' => 1,
-  'max-delete' => 1,
-  'max-size' => 1,
-  'min-size' => 1,
-  'modify-window' => 1,
-  'new-compress' => 0,
-  'no-implied-dirs' => 0,
-  'no-r' => 0,
-  'no-relative' => 0,
-  'no-specials' => 0,
-  'numeric-ids' => 0,
-  'old-compress' => 0,
-  'one-file-system' => 0,
-  'only-write-batch' => 1,
-  'open-noatime' => 0,
-  'owner' => 0,
-  'partial' => 0,
-  'partial-dir' => 2,
-  'perms' => 0,
-  'preallocate' => 0,
-  'recursive' => 0,
-  'remove-sent-files' => $only eq 'r' ? -1 : 0,
-  'remove-source-files' => $only eq 'r' ? -1 : 0,
-  'safe-links' => 0,
-  'sender' => $only eq 'w' ? -1 : 0,
-  'server' => 0,
-  'size-only' => 0,
-  'skip-compress' => 1,
-  'specials' => 0,
-  'stats' => 0,
-  'suffix' => 1,
-  'super' => 0,
-  'temp-dir' => 2,
-  'timeout' => 1,
-  'times' => 0,
-  'use-qsort' => 0,
-  'usermap' => 1,
-  'write-devices' => -1,
-);
-
-### END of options data produced by the cull_options script. ###
-
-if ($short_disabled ne '') {
-    $short_no_arg =~ s/[$short_disabled]//go;
-    $short_with_num =~ s/[$short_disabled]//go;
-}
-$short_no_arg = "[$short_no_arg]" if length($short_no_arg) > 1;
-$short_with_num = "[$short_with_num]" if length($short_with_num) > 1;
-
-my $write_log = -f LOGFILE && open(LOG, '>>', LOGFILE);
-
-chdir($subdir) or die "$0: Unable to chdir to restricted dir: $!\n";
-
-my(@opts, @args);
-my $in_options = 1;
-my $last_opt = '';
-my $check_type;
-while ($command =~ /((?:[^\s\\]+|\\.[^\s\\]*)+)/g) {
-  $_ = $1;
-  if ($check_type) {
-    push(@opts, check_arg($last_opt, $_, $check_type));
-    $check_type = 0;
-  } elsif ($in_options) {
-    push(@opts, $_);
-    if ($_ eq '.') {
-      $in_options = 0;
-    } else {
-      die "$0: invalid option: '-'\n" if $_ eq '-';
-      next if /^-$short_no_arg*(e\d*\.\w*)?$/o || /^-$short_with_num\d+$/o;
-
-      my($opt,$arg) = /^--([^=]+)(?:=(.*))?$/;
-      my $disabled;
-      if (defined $opt) {
-	my $ct = $long_opt{$opt};
-	last unless defined $ct;
-	next if $ct == 0;
-	if ($ct > 0) {
-	  if (!defined $arg) {
-	    $check_type = $ct;
-	    $last_opt = $opt;
-	    next;
-	  }
-	  $arg = check_arg($opt, $arg, $ct);
-	  $opts[-1] =~ s/=.*/=$arg/;
-	  next;
-	}
-	$disabled = 1;
-	$opt = "--$opt";
-      } elsif ($short_disabled ne '') {
-	$disabled = /^-$short_no_arg*([$short_disabled])/o;
-	$opt = "-$1";
-      }
-
-      last unless $disabled; # Generate generic failure
-      die "$0: option $opt has been disabled on this server.\n";
-    }
-  } else {
-    if ($subdir ne '/') {
-      # Validate args to ensure they don't try to leave our restricted dir.
-      s{//+}{/}g;
-      s{^/}{};
-      s{^$}{.};
-    }
-    push(@args, bsd_glob($_, GLOB_LIMIT|GLOB_NOCHECK|GLOB_BRACE|GLOB_QUOTE));
-  }
-}
-die "$0: invalid rsync-command syntax or options\n" if $in_options;
-
-if ($subdir ne '/') {
-    die "$0: do not use .. in any path!\n" if grep m{(^|/)\.\.(/|$)}, @args;
+long_opts = {
+  'append': 0,
+  'backup-dir': 2,
+  'block-size': 1,
+  'bwlimit': 1,
+  'checksum-choice': 1,
+  'checksum-seed': 1,
+  'compare-dest': 2,
+  'compress-choice': 1,
+  'compress-level': 1,
+  'copy-dest': 2,
+  'copy-unsafe-links': 0,
+  'daemon': -1,
+  'debug': 1,
+  'delay-updates': 0,
+  'delete': 0,
+  'delete-after': 0,
+  'delete-before': 0,
+  'delete-delay': 0,
+  'delete-during': 0,
+  'delete-excluded': 0,
+  'delete-missing-args': 0,
+  'existing': 0,
+  'fake-super': 0,
+  'files-from': 3,
+  'force': 0,
+  'from0': 0,
+  'fsync': 0,
+  'fuzzy': 0,
+  'group': 0,
+  'groupmap': 1,
+  'hard-links': 0,
+  'iconv': 1,
+  'ignore-errors': 0,
+  'ignore-existing': 0,
+  'ignore-missing-args': 0,
+  'ignore-times': 0,
+  'info': 1,
+  'inplace': 0,
+  'link-dest': 2,
+  'links': 0,
+  'list-only': 0,
+  'log-file': 3,
+  'log-format': 1,
+  'max-alloc': 1,
+  'max-delete': 1,
+  'max-size': 1,
+  'min-size': 1,
+  'mkpath': 0,
+  'modify-window': 1,
+  'msgs2stderr': 0,
+  'munge-links': 0,
+  'new-compress': 0,
+  'no-W': 0,
+  'no-implied-dirs': 0,
+  'no-msgs2stderr': 0,
+  'no-munge-links': -1,
+  'no-r': 0,
+  'no-relative': 0,
+  'no-specials': 0,
+  'numeric-ids': 0,
+  'old-compress': 0,
+  'one-file-system': 0,
+  'only-write-batch': 1,
+  'open-noatime': 0,
+  'owner': 0,
+  'partial': 0,
+  'partial-dir': 2,
+  'perms': 0,
+  'preallocate': 0,
+  'recursive': 0,
+  'remove-sent-files': 0,
+  'remove-source-files': 0,
+  'safe-links': 0,
+  'sender': 0,
+  'server': 0,
+  'size-only': 0,
+  'skip-compress': 1,
+  'specials': 0,
+  'stats': 0,
+  'stderr': 1,
+  'suffix': 1,
+  'super': 0,
+  'temp-dir': 2,
+  'timeout': 1,
+  'times': 0,
+  'use-qsort': 0,
+  'usermap': 1,
+  'write-devices': -1,
 }
 
-@args = ( '.' ) if !@args;
+### END of options data produced by the cull-options script. ###
 
-if ($write_log) {
-  my ($mm,$hh) = (localtime)[1,2];
-  my $host = $ENV{SSH_CONNECTION} || 'unknown';
-  $host =~ s/ .*//; # Keep only the client's IP addr
-  $host =~ s/^::ffff://;
-  $host = gethostbyaddr(inet_aton($host),AF_INET) || $host;
-  printf LOG "%02d:%02d %-13s [%s]\n", $hh, $mm, $host, "@opts @args";
-  close LOG;
-}
+import os, sys, re, argparse, glob, socket, time, subprocess
+from argparse import RawTextHelpFormatter
 
-# Note: This assumes that the rsync protocol will not be maliciously hijacked.
-exec(RSYNC, @opts, '--', @args) or die "exec(rsync @opts -- @args) failed: $? $!";
+try:
+    from braceexpand import braceexpand
+except:
+    braceexpand = lambda x: [ DE_BACKSLASH_RE.sub(r'\1', x) ]
 
-sub check_arg
-{
-  my($opt, $arg, $type) = @_;
-  $arg =~ s/\\(.)/$1/g;
-  if ($subdir ne '/' && ($type == 3 || ($type == 2 && !$am_sender))) {
-    $arg =~ s{//}{/}g;
-    die "Do not use .. in --$opt; anchor the path at the root of your restricted dir.\n"
-      if $arg =~ m{(^|/)\.\.(/|$)};
-    $arg =~ s{^/}{$subdir/};
-  }
-  $arg;
-}
+HAS_DOT_DOT_RE = re.compile(r'(^|/)\.\.(/|$)')
+LONG_OPT_RE = re.compile(r'^--([^=]+)(?:=(.*))?$')
+DE_BACKSLASH_RE = re.compile(r'\\(.)')
+
+def main():
+    if not os.path.isdir(args.dir):
+        die("Restricted directory does not exist!")
+
+    # The format of the environment variables set by sshd:
+    #   SSH_ORIGINAL_COMMAND:
+    #     rsync --server          -vlogDtpre.iLsfxCIvu --etc . ARG  # push
+    #     rsync --server --sender -vlogDtpre.iLsfxCIvu --etc . ARGS # pull
+    #   SSH_CONNECTION (client_ip client_port server_ip server_port):
+    #     192.168.1.100 64106 192.168.1.2 22
+
+    command = os.environ.get('SSH_ORIGINAL_COMMAND', None)
+    if not command:
+        die("Not invoked via sshd")
+    command = command.split(' ', 2)
+    if command[0:1] != ['rsync']:
+        die("SSH_ORIGINAL_COMMAND does not run rsync")
+    if command[1:2] != ['--server']:
+        die("--server option is not the first arg")
+    command = '' if len(command) < 3 else command[2]
+
+    global am_sender
+    am_sender = command.startswith("--sender ") # Restrictive on purpose!
+    if args.ro and not am_sender:
+        die("sending to read-only server is not allowed")
+    if args.wo and am_sender:
+        die("reading from write-only server is not allowed")
+
+    if args.wo or not am_sender:
+        long_opts['sender'] = -1
+    if args.no_del:
+        for opt in long_opts:
+            if opt.startswith(('remove', 'delete')):
+                long_opts[opt] = -1
+    if args.ro:
+        long_opts['log-file'] = -1
+
+    if args.dir != '/':
+        global short_disabled
+        short_disabled += short_disabled_subdir
+
+    short_no_arg_re = short_no_arg
+    short_with_num_re = short_with_num
+    if short_disabled:
+        for ltr in short_disabled:
+            short_no_arg_re = short_no_arg_re.replace(ltr, '')
+            short_with_num_re = short_with_num_re.replace(ltr, '')
+        short_disabled_re = re.compile(r'^-[%s]*([%s])' % (short_no_arg_re, short_disabled))
+    short_no_arg_re = re.compile(r'^-(?=.)[%s]*(e\d*\.\w*)?$' % short_no_arg_re)
+    short_with_num_re = re.compile(r'^-[%s]\d+$' % short_with_num_re)
+
+    log_fh = open(LOGFILE, 'a') if os.path.isfile(LOGFILE) else None
+
+    try:
+        os.chdir(args.dir)
+    except OSError as e:
+        die('unable to chdir to restricted dir:', str(e))
+
+    rsync_opts = [ '--server' ]
+    rsync_args = [ ]
+    saw_the_dot_arg = False
+    last_opt = check_type = None
+
+    for arg in re.findall(r'(?:[^\s\\]+|\\.[^\s\\]*)+', command):
+        if check_type:
+            rsync_opts.append(validated_arg(last_opt, arg, check_type))
+            check_type = None
+        elif saw_the_dot_arg:
+            # NOTE: an arg that starts with a '-' is safe due to our use of "--" in the cmd tuple.
+            try:
+                b_e = braceexpand(arg) # Also removes backslashes
+            except: # Handle errors such as unbalanced braces by just de-backslashing the arg:
+                b_e = [ DE_BACKSLASH_RE.sub(r'\1', arg) ]
+            for xarg in b_e:
+                rsync_args += validated_arg('arg', xarg, wild=True)
+        else: # parsing the option args
+            if arg == '.':
+                saw_the_dot_arg = True
+                continue
+            rsync_opts.append(arg)
+            if short_no_arg_re.match(arg) or short_with_num_re.match(arg):
+                continue
+            disabled = False
+            m = LONG_OPT_RE.match(arg)
+            if m:
+                opt = m.group(1)
+                opt_arg = m.group(2)
+                ct = long_opts.get(opt, None)
+                if ct is None:
+                    break # Generate generic failure due to unfinished arg parsing
+                if ct == 0:
+                    continue
+                opt = '--' + opt
+                if ct > 0:
+                    if opt_arg is not None:
+                        rsync_opts[-1] = opt + '=' + validated_arg(opt, opt_arg, ct)
+                    else:
+                        check_type = ct
+                        last_opt = opt
+                    continue
+                disabled = True
+            elif short_disabled:
+                m = short_disabled_re.match(arg)
+                if m:
+                    disabled = True
+                    opt = '-' + m.group(1)
+
+            if disabled:
+                die("option", opt, "has been disabled on this server.")
+            break # Generate a generic failure
+
+    if not saw_the_dot_arg:
+        die("invalid rsync-command syntax or options")
+
+    if args.munge:
+        rsync_opts.append('--munge-links')
+
+    if not rsync_args:
+        rsync_args = [ '.' ]
+
+    cmd = (RSYNC, *rsync_opts, '--', '.', *rsync_args)
+
+    if log_fh:
+        now = time.localtime()
+        host = os.environ.get('SSH_CONNECTION', 'unknown').split()[0] # Drop everything after the IP addr
+        if host.startswith('::ffff:'):
+            host = host[7:]
+        try:
+            host = socket.gethostbyaddr(socket.inet_aton(host))
+        except:
+            pass
+        log_fh.write("%02d:%02d:%02d %-16s %s\n" % (now.tm_hour, now.tm_min, now.tm_sec, host, str(cmd)))
+        log_fh.close()
+
+    # NOTE: This assumes that the rsync protocol will not be maliciously hijacked.
+    if args.no_lock:
+        os.execlp(RSYNC, *cmd)
+        die("execlp(", RSYNC, *cmd, ')  failed')
+    child = subprocess.run(cmd)
+    if child.returncode != 0:
+        sys.exit(child.returncode)
+
+
+def validated_arg(opt, arg, typ=3, wild=False):
+    if opt != 'arg': # arg values already have their backslashes removed.
+        arg = DE_BACKSLASH_RE.sub(r'\1', arg)
+
+    orig_arg = arg
+    if arg.startswith('./'):
+        arg = arg[1:]
+    arg = arg.replace('//', '/')
+    if args.dir != '/':
+        if HAS_DOT_DOT_RE.search(arg):
+            die("do not use .. in", opt, "(anchor the path at the root of your restricted dir)")
+        if arg.startswith('/'):
+            arg = args.dir + arg
+
+    if wild:
+        got = glob.glob(arg)
+        if not got:
+            got = [ arg ]
+    else:
+        got = [ arg ]
+
+    ret = [ ]
+    for arg in got:
+        if args.dir != '/' and arg != '.' and (typ == 3 or (typ == 2 and not am_sender)):
+            arg_has_trailing_slash = arg.endswith('/')
+            if arg_has_trailing_slash:
+                arg = arg[:-1]
+            else:
+                arg_has_trailing_slash_dot = arg.endswith('/.')
+                if arg_has_trailing_slash_dot:
+                    arg = arg[:-2]
+            real_arg = os.path.realpath(arg)
+            if arg != real_arg and not real_arg.startswith(args.dir_slash):
+                die('unsafe arg:', orig_arg, [arg, real_arg])
+            if arg_has_trailing_slash:
+                arg += '/'
+            elif arg_has_trailing_slash_dot:
+                arg += '/.'
+            if opt == 'arg' and arg.startswith(args.dir_slash):
+                arg = arg[args.dir_slash_len:]
+                if arg == '':
+                    arg = '.'
+        ret.append(arg)
+
+    return ret if wild else ret[0]
+
+
+def lock_or_die(dirname):
+    import fcntl
+    global lock_handle
+    lock_handle = os.open(dirname, os.O_RDONLY)
+    try:
+        fcntl.flock(lock_handle, fcntl.LOCK_EX | fcntl.LOCK_NB)
+    except:
+        die('Another instance of rrsync is already accessing this directory.')
+
+
+def die(*msg):
+    print(sys.argv[0], 'error:', *msg, file=sys.stderr)
+    if sys.stdin.isatty():
+        arg_parser.print_help(sys.stderr)
+    sys.exit(1)
+
+
+# This class displays the --help to the user on argparse error IFF they're running it interactively.
+class OurArgParser(argparse.ArgumentParser):
+    def error(self, msg):
+        die(msg)
+
+
+if __name__ == '__main__':
+    our_desc = """Use "man rrsync" to learn how to restrict ssh users to using a restricted rsync command."""
+    arg_parser = OurArgParser(description=our_desc, add_help=False)
+    only_group = arg_parser.add_mutually_exclusive_group()
+    only_group.add_argument('-ro', action='store_true', help="Allow only reading from the DIR. Implies -no-del and -no-lock.")
+    only_group.add_argument('-wo', action='store_true', help="Allow only writing to the DIR.")
+    arg_parser.add_argument('-munge', action='store_true', help="Enable rsync's --munge-links on the server side.")
+    arg_parser.add_argument('-no-del', action='store_true', help="Disable rsync's --delete* and --remove* options.")
+    arg_parser.add_argument('-no-lock', action='store_true', help="Avoid the single-run (per-user) lock check.")
+    arg_parser.add_argument('-help', '-h', action='help', help="Output this help message and exit.")
+    arg_parser.add_argument('dir', metavar='DIR', help="The restricted directory to use.")
+    args = arg_parser.parse_args()
+    args.dir = os.path.realpath(args.dir)
+    args.dir_slash = args.dir + '/'
+    args.dir_slash_len = len(args.dir_slash)
+    if args.ro:
+        args.no_del = True
+    elif not args.no_lock:
+        lock_or_die(args.dir)
+    main()
+
+# vim: sw=4 et

support/rrsync.1.md

@@ -0,0 +1,139 @@
+## NAME
+
+rrsync - a script to setup restricted rsync users via ssh logins
+
+## SYNOPSIS
+
+```
+rrsync [-ro|-rw] [-munge] [-no-del] [-no-lock] DIR
+```
+
+The single non-option argument specifies the restricted _DIR_ to use. It can be
+relative to the user's home directory or an absolute path.
+
+The online version of this man page (that includes cross-linking of topics)
+is available at <https://download.samba.org/pub/rsync/rrsync.1>.
+
+## DESCRIPTION
+
+A user's ssh login can be restricted to only allow the running of an rsync
+transfer in one of two easy ways: forcing the running of the rrsync script
+or forcing the running of an rsync daemon-over-ssh command.
+
+To use the rrsync script, edit the user's `~/.ssh/authorized_keys` file and add
+a prefix like one of the following (followed by a space) in front of each
+ssh-key line that should be restricted:
+
+> ```
+> command="rrsync DIR"
+> command="rrsync -ro DIR"
+> command="rrsync -munge -no-del DIR"
+> ```
+
+Then, ensure that the rrsync script has your desired option restrictions. You
+may want to copy the script to a local bin dir with a unique name if you want
+to have multiple configurations. One or more rrsync options can be specified
+prior to the _DIR_ if you want to further restrict the transfer.
+
+To use an rsync daemon setup, edit the user's `~/.ssh/authorized_keys` file and
+add a prefix like one of the following (followed by a space) in front of each
+ssh-key line that should be restricted:
+
+> ```
+> command="rsync --server --daemon ."
+> command="rsync --server --daemon --config=/PATH/TO/rsyncd.conf ."
+> ```
+
+Then, ensure that the rsyncd.conf file is created with one or more module names
+with the appropriate path and option restrictions.  If rsync's
+[`--config`](rsync.1#dopt) option is omitted, it defaults to `~/rsyncd.conf`.
+See the `rsyncd.conf` man page for details of how to configure an rsync daemon.
+
+When using rrsync, there can be just one restricted dir per authorized key.  A
+daemon setup, on the other hand, allows multiple module names inside the config
+file, each one with its own path setting.
+
+The remainder of this man page is dedicated to using the rrsync script.
+
+## OPTIONS
+
+0.  `-ro`
+
+    Allow only reading from the DIR. Implies [`-no-del`](#opt) and
+    [`-no-lock`](#opt).
+
+0.  `-wo`
+
+    Allow only writing to the DIR.
+
+0.  `-munge`
+
+    Enable rsync's [`--munge-links`](rsync.1#opt) on the server side.
+
+0.  `-no-del`
+
+    Disable rsync's `--delete*` and `--remove*` options.
+
+0.  `-no-lock`
+
+    Avoid the single-run (per-user) lock check.  Useful with [`-munge`](#opt).
+
+0.  `-help`, `-h`
+
+    Output this help message and exit.
+
+## SECURITY RESTRICTIONS
+
+The rrsync script validates the path arguments it is sent to try to restrict
+them to staying within the specified DIR.
+
+The rrsync script rejects rsync's [`--copy-links`](rsync.1#opt) option (by
+default) so that a copy cannot dereference a symlink within the DIR to get to a
+file outside the DIR.
+
+The rrsync script rejects rsync's [`--protect-args`](rsync.1#opt) (`-s`) option
+because it would allow options to be sent to the server-side that the script
+cannot check.  If you want to support `--protect-args`, use a daemon-over-ssh
+setup.
+
+The rrsync script accepts just a subset of rsync's options that the real rsync
+uses when running the server command.  A few extra convenience options are also
+included to help it to interact with BackupPC and accept some convenient user
+overrides.
+
+The script (or a copy of it) can be manually edited if you want it to customize
+the option handling.
+
+## EXAMPLES
+
+The `~/.ssh/authorized_keys` file might have lines in it like this:
+
+> ```
+> command="rrsync client/logs" ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAIEAzG...
+> command="rrsync -ro results" ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAIEAmk...
+> ```
+
+## FILES
+
+~/.ssh/authorized_keys
+
+## SEE ALSO
+
+[**rsync**(1)](rsync.1)
+
+## VERSION
+
+This man page is current for version @VERSION@ of rsync.
+
+## CREDITS
+
+rsync is distributed under the GNU General Public License.  See the file
+[COPYING](COPYING) for details.
+
+An rsync web site is available at <https://rsync.samba.org/> and its github
+project is <https://github.com/WayneD/rsync>.
+
+## AUTHOR
+
+The original rrsync perl script was written by Joe Smith.  Many people have
+later contributed to it.  The python version was created by Wayne Davison.

support/rsync-no-vanished

@@ -1,12 +1,21 @@
 #!/usr/bin/env bash
 
+REAL_RSYNC=/usr/bin/rsync
 IGNOREEXIT=24
 IGNOREOUT='^(file has vanished: |rsync warning: some files vanished before they could be transferred)'
 
+# If someone installs this as "rsync", make sure we don't affect a server run.
+for arg in "${@}"; do
+    if [[ "$arg" == --server ]]; then
+	exec $REAL_RSYNC "${@}"
+	exit $? # Not reached
+    fi
+done
+
 set -o pipefail
 
-rsync "${@}" 2>&1 | (egrep -v "$IGNOREOUT" || true)
-ret=$?
+# This filters stderr without merging it with stdout:
+{ $REAL_RSYNC "${@}" 2>&1 1>&3 3>&- | grep -E -v "$IGNOREOUT"; ret=${PIPESTATUS[0]}; } 3>&1 1>&2
 
 if [[ $ret == $IGNOREEXIT ]]; then
     ret=0

support/rsync-slash-strip

@@ -6,12 +6,19 @@
 #
 # To use this, name it something like "rs", put it somewhere in your path, and
 # then use "rs" in place of "rsync" when you are typing your copy commands.
+
+REAL_RSYNC=/usr/bin/rsync
+
 args=()
 for arg in "${@}"; do
+    if [[ "$arg" == --server ]]; then
+	exec $REAL_RSYNC "${@}"
+	exit $? # Not reached
+    fi
     if [[ "$arg" == / ]]; then
 	args=("${args[@]}" /)
     else
 	args=("${args[@]}" "${arg%/}")
     fi
 done
-exec /usr/bin/rsync "${args[@]}"
+exec $REAL_RSYNC "${args[@]}"

syscall.c

@@ -4,7 +4,7 @@
  *
  * Copyright (C) 1998 Andrew Tridgell
  * Copyright (C) 2002 Martin Pool
- * Copyright (C) 2003-2020 Wayne Davison
+ * Copyright (C) 2003-2022 Wayne Davison
  *
  * This program is free software; you can redistribute it and/or modify
  * it under the terms of the GNU General Public License as published by
@@ -55,12 +55,16 @@ extern int open_noatime;
 #endif
 
 #ifdef SUPPORT_CRTIMES
+#ifdef HAVE_GETATTRLIST
 #pragma pack(push, 4)
 struct create_time {
 	uint32 length;
 	struct timespec crtime;
 };
 #pragma pack(pop)
+#elif defined __CYGWIN__
+#include <windows.h>
+#endif
 #endif
 
 #define RETURN_ERROR_IF(x,e) \
@@ -73,15 +77,15 @@ struct create_time {
 
 #define RETURN_ERROR_IF_RO_OR_LO RETURN_ERROR_IF(read_only || list_only, EROFS)
 
-int do_unlink(const char *fname)
+int do_unlink(const char *path)
 {
 	if (dry_run) return 0;
 	RETURN_ERROR_IF_RO_OR_LO;
-	return unlink(fname);
+	return unlink(path);
 }
 
 #ifdef SUPPORT_LINKS
-int do_symlink(const char *lnk, const char *fname)
+int do_symlink(const char *lnk, const char *path)
 {
 	if (dry_run) return 0;
 	RETURN_ERROR_IF_RO_OR_LO;
@@ -91,7 +95,7 @@ int do_symlink(const char *lnk, const char *fname)
 	 * and write the lnk into it. */
 	if (am_root < 0) {
 		int ok, len = strlen(lnk);
-		int fd = open(fname, O_WRONLY|O_CREAT|O_TRUNC, S_IWUSR|S_IRUSR);
+		int fd = open(path, O_WRONLY|O_CREAT|O_TRUNC, S_IWUSR|S_IRUSR);
 		if (fd < 0)
 			return -1;
 		ok = write(fd, lnk, len) == len;
@@ -101,7 +105,7 @@ int do_symlink(const char *lnk, const char *fname)
 	}
 #endif
 
-	return symlink(lnk, fname);
+	return symlink(lnk, path);
 }
 
 #if defined NO_SYMLINK_XATTRS || defined NO_SYMLINK_USER_XATTRS
@@ -227,27 +231,43 @@ int do_open(const char *pathname, int flags, mode_t mode)
 #ifdef HAVE_CHMOD
 int do_chmod(const char *path, mode_t mode)
 {
+	static int switch_step = 0;
 	int code;
+
 	if (dry_run) return 0;
 	RETURN_ERROR_IF_RO_OR_LO;
-#ifdef HAVE_LCHMOD
-	code = lchmod(path, mode & CHMOD_BITS);
-#else
-	if (S_ISLNK(mode)) {
-# if defined HAVE_SETATTRLIST
-		struct attrlist attrList;
-		uint32_t m = mode & CHMOD_BITS; /* manpage is wrong: not mode_t! */
 
-		memset(&attrList, 0, sizeof attrList);
-		attrList.bitmapcount = ATTR_BIT_MAP_COUNT;
-		attrList.commonattr = ATTR_CMN_ACCESSMASK;
-		code = setattrlist(path, &attrList, &m, sizeof m, FSOPT_NOFOLLOW);
+	switch (switch_step) {
+#ifdef HAVE_LCHMOD
+	case 0:
+		if ((code = lchmod(path, mode & CHMOD_BITS)) == 0)
+			break;
+		if (errno == ENOSYS)
+			switch_step++;
+		else if (errno != ENOTSUP)
+			break;
+#endif
+
+	default:
+		if (S_ISLNK(mode)) {
+# if defined HAVE_SETATTRLIST
+			struct attrlist attrList;
+			uint32_t m = mode & CHMOD_BITS; /* manpage is wrong: not mode_t! */
+
+			memset(&attrList, 0, sizeof attrList);
+			attrList.bitmapcount = ATTR_BIT_MAP_COUNT;
+			attrList.commonattr = ATTR_CMN_ACCESSMASK;
+			if ((code = setattrlist(path, &attrList, &m, sizeof m, FSOPT_NOFOLLOW)) == 0)
+				break;
+			if (errno == ENOTSUP)
+				code = 1;
 # else
-		code = 1;
+			code = 1;
 # endif
-	} else
-		code = chmod(path, mode & CHMOD_BITS); /* DISCOURAGED FUNCTION */
-#endif /* !HAVE_LCHMOD */
+		} else
+			code = chmod(path, mode & CHMOD_BITS); /* DISCOURAGED FUNCTION */
+		break;
+	}
 	if (code != 0 && (preserve_perms || preserve_executability))
 		return code;
 	return 0;
@@ -295,12 +315,12 @@ void trim_trailing_slashes(char *name)
 	}
 }
 
-int do_mkdir(char *fname, mode_t mode)
+int do_mkdir(char *path, mode_t mode)
 {
 	if (dry_run) return 0;
 	RETURN_ERROR_IF_RO_OR_LO;
-	trim_trailing_slashes(fname);
-	return mkdir(fname, mode);
+	trim_trailing_slashes(path);
+	return mkdir(path, mode);
 }
 
 /* like mkstemp but forces permissions */
@@ -334,25 +354,25 @@ int do_mkstemp(char *template, mode_t perms)
 #endif
 }
 
-int do_stat(const char *fname, STRUCT_STAT *st)
+int do_stat(const char *path, STRUCT_STAT *st)
 {
 #ifdef USE_STAT64_FUNCS
-	return stat64(fname, st);
+	return stat64(path, st);
 #else
-	return stat(fname, st);
+	return stat(path, st);
 #endif
 }
 
-int do_lstat(const char *fname, STRUCT_STAT *st)
+int do_lstat(const char *path, STRUCT_STAT *st)
 {
 #ifdef SUPPORT_LINKS
 # ifdef USE_STAT64_FUNCS
-	return lstat64(fname, st);
+	return lstat64(path, st);
 # else
-	return lstat(fname, st);
+	return lstat(path, st);
 # endif
 #else
-	return do_stat(fname, st);
+	return do_stat(path, st);
 #endif
 }
 
@@ -380,41 +400,29 @@ OFF_T do_lseek(int fd, OFF_T offset, int whence)
 }
 
 #ifdef HAVE_SETATTRLIST
-int do_setattrlist_times(const char *fname, STRUCT_STAT *stp)
+int do_setattrlist_times(const char *path, STRUCT_STAT *stp)
 {
 	struct attrlist attrList;
-	struct timespec ts;
+	struct timespec ts[2];
 
 	if (dry_run) return 0;
 	RETURN_ERROR_IF_RO_OR_LO;
 
-	ts.tv_sec = stp->st_mtime;
-	ts.tv_nsec = stp->ST_MTIME_NSEC;
+	/* Yes, this is in the opposite order of utime and similar. */
+	ts[0].tv_sec = stp->st_mtime;
+	ts[0].tv_nsec = stp->ST_MTIME_NSEC;
+
+	ts[1].tv_sec = stp->st_atime;
+	ts[1].tv_nsec = stp->ST_ATIME_NSEC;
 
 	memset(&attrList, 0, sizeof attrList);
 	attrList.bitmapcount = ATTR_BIT_MAP_COUNT;
-	attrList.commonattr = ATTR_CMN_MODTIME;
-	return setattrlist(fname, &attrList, &ts, sizeof ts, FSOPT_NOFOLLOW);
+	attrList.commonattr = ATTR_CMN_MODTIME | ATTR_CMN_ACCTIME;
+	return setattrlist(path, &attrList, ts, sizeof ts, FSOPT_NOFOLLOW);
 }
-#endif
 
 #ifdef SUPPORT_CRTIMES
-time_t get_create_time(const char *path)
-{
-	static struct create_time attrBuf;
-	struct attrlist attrList;
-
-	memset(&attrList, 0, sizeof attrList);
-	attrList.bitmapcount = ATTR_BIT_MAP_COUNT;
-	attrList.commonattr = ATTR_CMN_CRTIME;
-	if (getattrlist(path, &attrList, &attrBuf, sizeof attrBuf, FSOPT_NOFOLLOW) < 0)
-		return 0;
-	return attrBuf.crtime.tv_sec;
-}
-#endif
-
-#ifdef SUPPORT_CRTIMES
-int set_create_time(const char *path, time_t crtime)
+int do_setattrlist_crtime(const char *path, time_t crtime)
 {
 	struct attrlist attrList;
 	struct timespec ts;
@@ -431,9 +439,61 @@ int set_create_time(const char *path, time_t crtime)
 	return setattrlist(path, &attrList, &ts, sizeof ts, FSOPT_NOFOLLOW);
 }
 #endif
+#endif /* HAVE_SETATTRLIST */
+
+#ifdef SUPPORT_CRTIMES
+time_t get_create_time(const char *path, STRUCT_STAT *stp)
+{
+#ifdef HAVE_GETATTRLIST
+	static struct create_time attrBuf;
+	struct attrlist attrList;
+
+	(void)stp;
+	memset(&attrList, 0, sizeof attrList);
+	attrList.bitmapcount = ATTR_BIT_MAP_COUNT;
+	attrList.commonattr = ATTR_CMN_CRTIME;
+	if (getattrlist(path, &attrList, &attrBuf, sizeof attrBuf, FSOPT_NOFOLLOW) < 0)
+		return 0;
+	return attrBuf.crtime.tv_sec;
+#elif defined __CYGWIN__
+	(void)path;
+	return stp->st_birthtime;
+#else
+#error Unknown crtimes implementation
+#endif
+}
+
+#if defined __CYGWIN__
+int do_SetFileTime(const char *path, time_t crtime)
+{
+	if (dry_run) return 0;
+	RETURN_ERROR_IF_RO_OR_LO;
+
+	int cnt = MultiByteToWideChar(CP_UTF8, 0, path, -1, NULL, 0);
+	if (cnt == 0)
+	    return -1;
+	WCHAR *pathw = new_array(WCHAR, cnt);
+	if (!pathw)
+	    return -1;
+	MultiByteToWideChar(CP_UTF8, 0, path, -1, pathw, cnt);
+	HANDLE handle = CreateFileW(pathw, FILE_WRITE_ATTRIBUTES, FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE,
+				    NULL, OPEN_EXISTING, FILE_FLAG_BACKUP_SEMANTICS, NULL);
+	free(pathw);
+	if (handle == INVALID_HANDLE_VALUE)
+	    return -1;
+	int64 temp_time = Int32x32To64(crtime, 10000000) + 116444736000000000LL;
+	FILETIME birth_time;
+	birth_time.dwLowDateTime = (DWORD)temp_time;
+	birth_time.dwHighDateTime = (DWORD)(temp_time >> 32);
+	int ok = SetFileTime(handle, &birth_time, NULL, NULL);
+	CloseHandle(handle);
+	return ok ? 0 : -1;
+}
+#endif
+#endif /* SUPPORT_CRTIMES */
 
 #ifdef HAVE_UTIMENSAT
-int do_utimensat(const char *fname, STRUCT_STAT *stp)
+int do_utimensat(const char *path, STRUCT_STAT *stp)
 {
 	struct timespec t[2];
 
@@ -452,12 +512,12 @@ int do_utimensat(const char *fname, STRUCT_STAT *stp)
 #else
 	t[1].tv_nsec = 0;
 #endif
-	return utimensat(AT_FDCWD, fname, t, AT_SYMLINK_NOFOLLOW);
+	return utimensat(AT_FDCWD, path, t, AT_SYMLINK_NOFOLLOW);
 }
 #endif
 
 #ifdef HAVE_LUTIMES
-int do_lutimes(const char *fname, STRUCT_STAT *stp)
+int do_lutimes(const char *path, STRUCT_STAT *stp)
 {
 	struct timeval t[2];
 
@@ -476,12 +536,12 @@ int do_lutimes(const char *fname, STRUCT_STAT *stp)
 #else
 	t[1].tv_usec = 0;
 #endif
-	return lutimes(fname, t);
+	return lutimes(path, t);
 }
 #endif
 
 #ifdef HAVE_UTIMES
-int do_utimes(const char *fname, STRUCT_STAT *stp)
+int do_utimes(const char *path, STRUCT_STAT *stp)
 {
 	struct timeval t[2];
 
@@ -500,11 +560,11 @@ int do_utimes(const char *fname, STRUCT_STAT *stp)
 #else
 	t[1].tv_usec = 0;
 #endif
-	return utimes(fname, t);
+	return utimes(path, t);
 }
 
 #elif defined HAVE_UTIME
-int do_utime(const char *fname, STRUCT_STAT *stp)
+int do_utime(const char *path, STRUCT_STAT *stp)
 {
 #ifdef HAVE_STRUCT_UTIMBUF
 	struct utimbuf tbuf;
@@ -518,11 +578,11 @@ int do_utime(const char *fname, STRUCT_STAT *stp)
 # ifdef HAVE_STRUCT_UTIMBUF
 	tbuf.actime = stp->st_atime;
 	tbuf.modtime = stp->st_mtime;
-	return utime(fname, &tbuf);
+	return utime(path, &tbuf);
 # else
 	t[0] = stp->st_atime;
 	t[1] = stp->st_mtime;
-	return utime(fname, t);
+	return utime(path, t);
 # endif
 }
 

t_stub.c

@@ -3,7 +3,7 @@
  * functions, so that module test harnesses can run standalone.
  *
  * Copyright (C) 2001, 2002 Martin Pool <mbp@samba.org>
- * Copyright (C) 2003-2020 Wayne Davison
+ * Copyright (C) 2003-2022 Wayne Davison
  *
  * This program is free software; you can redistribute it and/or modify
  * it under the terms of the GNU General Public License as published by
@@ -21,6 +21,7 @@
 
 #include "rsync.h"
 
+int do_fsync = 0;
 int inplace = 0;
 int modify_window = 0;
 int preallocate_files = 0;
@@ -28,10 +29,11 @@ int protect_args = 0;
 int module_id = -1;
 int relative_paths = 0;
 int module_dirlen = 0;
-int preserve_times = 0;
+int preserve_mtimes = 0;
 int preserve_xattrs = 0;
 int preserve_perms = 0;
 int preserve_executability = 0;
+int omit_link_times = 0;
 int open_noatime = 0;
 size_t max_alloc = 0; /* max_alloc is needed when combined with util2.o */
 char *partial_dir;

testsuite/00-hello.test

@@ -1,9 +1,11 @@
-#! /bin/sh
+#!/bin/sh
 
-test_fail() {
-    echo "$@" >&2
-    exit 1
-}
+# Test some foundational things.
+
+. "$suitedir/rsync.fns"
+
+RSYNC_RSH="$scratchdir/src/support/lsh.sh"
+export RSYNC_RSH
 
 echo $0 running
 
@@ -12,3 +14,48 @@ $RSYNC --version || test_fail '--version output failed'
 $RSYNC --info=help || test_fail '--info=help output failed'
 
 $RSYNC --debug=help || test_fail '--debug=help output failed'
+
+weird_name="A weird)name"
+
+mkdir "$fromdir"
+mkdir "$fromdir/$weird_name"
+
+append_line() {
+    echo "$1"
+    echo "$1" >>"$fromdir/$weird_name/file"
+}
+
+append_line test1
+checkit "$RSYNC -ai '$fromdir/' '$todir/'" "$fromdir" "$todir"
+
+copy_weird() {
+    checkit "$RSYNC $1 \"$2$fromdir/$weird_name/\" \"$3$todir/$weird_name\"" "$fromdir" "$todir"
+}
+
+append_line test2
+copy_weird '-ai' 'lh:' ''
+
+append_line test3
+copy_weird '-ai' '' 'lh:'
+
+append_line test4
+copy_weird '-ais' 'lh:' ''
+
+append_line test5
+copy_weird '-ais' '' 'lh:'
+
+echo test6
+
+touch "$fromdir/one" "$fromdir/two"
+(cd "$fromdir" && $RSYNC -ai --old-args lh:'one two' "$todir/")
+if [ ! -f "$todir/one" ] || [ ! -f "$todir/two" ]; then
+    test_fail "old-args copy of 'one two' failed"
+fi
+
+echo test7
+
+rm "$todir/one" "$todir/two"
+(cd "$fromdir" && RSYNC_OLD_ARGS=1 $RSYNC -ai lh:'one two' "$todir/")
+
+# The script would have aborted on error, so getting here means we've won.
+exit 0

testsuite/README.testsuite

@@ -26,5 +26,3 @@ you could include the log messages when reporting a failure.
 
 These tests also run automatically on the build farm, and you can see
 the results on http://build.samba.org/.
-
-

testsuite/acls-default.test

@@ -1,4 +1,4 @@
-#! /bin/sh
+#!/bin/sh
 
 # This program is distributable under the terms of the GNU GPL (see
 # COPYING).
@@ -21,7 +21,7 @@ testit() {
     todir="$scratchdir/$1"
     mkdir "$todir"
     $setfacl_nodef "$todir"
-    if [ "$2" ]; then
+    if [ -n "$2" ]; then
 	case "$setfacl_nodef" in
 	*-k*) opts="-dm $2" ;;
 	*) opts="-m `echo $2 | sed 's/\([ugom]:\)/d:\1/g'`"

testsuite/acls.test

@@ -1,4 +1,4 @@
-#! /bin/sh
+#!/bin/sh
 
 # This program is distributable under the terms of the GNU GPL (see
 # COPYING).
@@ -17,8 +17,8 @@ files='foo file1 file2'
 
 case "$setfacl_nodef" in
 true)
-    if ! chmod --help 2>&1 | fgrep +a >/dev/null; then
-	test_skipped "I don't know how to use setfacl or chmod for ACLs"
+    if ! chmod --help 2>&1 | grep -F +a >/dev/null; then
+        test_skipped "I don't know how to use setfacl or chmod for ACLs"
     fi
     chmod +a "root allow read,write,execute" "$fromdir/foo" || test_skipped "Your filesystem has ACLs disabled"
     chmod +a "root allow read,execute" "$fromdir/file1"
@@ -27,7 +27,7 @@ true)
     chmod +a "root allow read,execute" "$fromdir/file2"
 
     see_acls() {
-	ls -le "${@}"
+        ls -le "${@}"
     }
     ;;
 *)
@@ -45,7 +45,7 @@ true)
     setfacl -m u:0:5 "$fromdir/file2"
 
     see_acls() {
-	getfacl "${@}"
+        getfacl "${@}"
     }
     ;;
 esac

testsuite/atimes.test

@@ -1,4 +1,4 @@
-#! /bin/sh
+#!/bin/sh
 
 # Test rsync copying atimes
 

testsuite/backup.test

@@ -1,6 +1,6 @@
-#! /bin/sh
+#!/bin/sh
 
-# Copyright (C) 2004-2020 Wayne Davison
+# Copyright (C) 2004-2022 Wayne Davison
 
 # This program is distributable under the terms of the GNU GPL (see
 # COPYING).
@@ -15,8 +15,6 @@ makepath "$fromdir/deep" "$bakdir/dname"
 name1="$fromdir/deep/name1"
 name2="$fromdir/deep/name2"
 
-outfile="$scratchdir/rsync.out"
-
 cat "$srcdir"/[gr]*.[ch] > "$name1"
 cat "$srcdir"/[et]*.[ch] > "$name2"
 
@@ -26,8 +24,7 @@ checkit "$RSYNC -ai --info=backup '$fromdir/' '$chkdir/'" "$fromdir" "$chkdir"
 cat "$srcdir"/[fgpr]*.[ch] > "$name1"
 cat "$srcdir"/[etw]*.[ch] > "$name2"
 
-$RSYNC -ai --info=backup --no-whole-file --backup "$fromdir/" "$todir/" \
-    | tee "$outfile"
+checktee "$RSYNC -ai --info=backup --no-whole-file --backup '$fromdir/' '$todir/'"
 for fn in deep/name1 deep/name2; do
     grep "backed up $fn to $fn~" "$outfile" >/dev/null || test_fail "no backup message output for $fn"
     diff $diffopt "$fromdir/$fn" "$todir/$fn" || test_fail "copy of $fn failed"

testsuite/batch-mode.test

@@ -1,4 +1,4 @@
-#! /bin/sh
+#!/bin/sh
 
 # Copyright (C) 2004 by Chris Shoemaker <c.shoemaker@cox.net>
 

testsuite/chgrp.test

@@ -1,4 +1,4 @@
-#! /bin/sh
+#!/bin/sh
 
 # Copyright (C) 2002 by Martin Pool <mbp@samba.org>
 
@@ -16,8 +16,7 @@
 mygrps="`rsync_getgroups`" || test_fail "Can't get groups"
 mkdir "$fromdir"
 
-for g in $mygrps
-do
+for g in $mygrps; do
     name="$fromdir/foo-$g"
     date > "$name"
     chgrp "$g" "$name" || test_fail "Can't chgrp"

testsuite/chmod-option.test

@@ -1,4 +1,4 @@
-#! /bin/sh
+#!/bin/sh
 
 # Copyright (C) 2002 by Martin Pool <mbp@samba.org>
 
@@ -31,7 +31,7 @@ chmod 770 "$dir2"
 # Copy the files we've created over to another directory
 checkit "$RSYNC -avv '$fromdir/' '$checkdir/'" "$fromdir" "$checkdir"
 
-# And then manually make the changes which should occur 
+# And then manually make the changes which should occur
 umask 002
 chmod ug-s,a+rX "$checkdir"/*
 chmod +w "$checkdir" "$checkdir"/dir*

testsuite/chmod-temp-dir.test

@@ -1,6 +1,6 @@
-#! /bin/sh
+#!/bin/sh
 
-# Copyright (C) 2004-2020 Wayne Davison
+# Copyright (C) 2004-2022 Wayne Davison
 
 # This program is distributable under the terms of the GNU GPL (see
 # COPYING).
@@ -17,7 +17,7 @@ sdev=`$TOOLDIR/getfsdev $scratchdir`
 tdev=$sdev
 
 for tmpdir2 in "${RSYNC_TEST_TMP:-/override-tmp-not-specified}" /run/shm /var/tmp /tmp; do
-    [ -d "$tmpdir2" -a -w "$tmpdir2" ] || continue
+    [ -d "$tmpdir2" ] && [ -w "$tmpdir2" ] || continue
     tdev=`$TOOLDIR/getfsdev "$tmpdir2"`
     [ x$sdev != x$tdev ] && break
 done

testsuite/chmod.test

@@ -1,6 +1,6 @@
-#! /bin/sh
+#!/bin/sh
 
-# Copyright (C) 2004-2020 Wayne Davison
+# Copyright (C) 2004-2022 Wayne Davison
 
 # This program is distributable under the terms of the GNU GPL (see
 # COPYING).

testsuite/chown.test

@@ -1,4 +1,4 @@
-#! /bin/sh
+#!/bin/sh
 
 # Copyright (C) 2002 by Martin Pool <mbp@samba.org>
 

testsuite/compare-dest.test

@@ -1,6 +1,6 @@
-#! /bin/sh
+#!/bin/sh
 
-# Copyright (C) 2004-2020 Wayne Davison
+# Copyright (C) 2004-2022 Wayne Davison
 
 # This program is distributable under the terms of the GNU GPL (see
 # COPYING).

testsuite/crtimes.test

@@ -1,4 +1,4 @@
-#! /bin/sh
+#!/bin/sh
 
 # Test rsync copying create times
 
@@ -10,7 +10,7 @@ $RSYNC --version | grep "[, ] crtimes" >/dev/null || test_skipped "Rsync is conf
 # Setting it to a newer time affects just the mtime.
 
 mkdir "$fromdir"
-echo hiho "$fromdir/foo"
+echo hiho >"$fromdir/foo"
 
 touch -t 200101011111.11 "$fromdir"
 touch -t 200202022222.22 "$fromdir"

testsuite/daemon.test

@@ -21,9 +21,6 @@
 
 . "$suitedir/rsync.fns"
 
-chkfile="$scratchdir/rsync.chk"
-outfile="$scratchdir/rsync.out"
-
 SSH="src/support/lsh.sh --no-cd"
 FILE_REPL='s/^\([^d][^ ]*\) *\(..........[0-9]\) /\1 \2 /'
 DIR_REPL='s/^\(d[^ ]*\)  *[0-9][.,0-9]* /\1         DIR /'
@@ -57,21 +54,17 @@ test-to        	r/w
 test-scratch   	NOCOMMENT
 EOT
 
-$RSYNC -ve "$SSH" --rsync-path="$RSYNC$confopt" localhost:: | tee "$outfile"
+checkdiff2 "$RSYNC -ve '$SSH' --rsync-path='$RSYNC$confopt' localhost::"
 echo '===='
-diff $diffopt "$chkfile" "$outfile" || test_fail "test 0 failed"
 
 RSYNC_CONNECT_PROG="$RSYNC --config=$conf --daemon"
 export RSYNC_CONNECT_PROG
 
-$RSYNC -v localhost:: | tee "$outfile"
+checkdiff2 "$RSYNC -v localhost::"
 echo '===='
-diff $diffopt "$chkfile" "$outfile" || test_fail "test 1 failed"
 
-$RSYNC -r localhost::test-hidden \
-    | sed "$FILE_REPL" | sed "$DIR_REPL" | sed "$LS_REPL" \
-    | tee "$outfile"
-cat <<EOT >"$chkfile"
+checkdiff "$RSYNC -r localhost::test-hidden" \
+	"sed -e '$FILE_REPL' -e '$DIR_REPL' -e '$LS_REPL'" <<EOT
 drwxr-xr-x         DIR ####/##/## ##:##:## .
 drwxr-xr-x         DIR ####/##/## ##:##:## bar
 -rw-r--r--           4 ####/##/## ##:##:## bar/two
@@ -80,24 +73,18 @@ drwxr-xr-x         DIR ####/##/## ##:##:## bar/baz
 drwxr-xr-x         DIR ####/##/## ##:##:## foo
 -rw-r--r--           4 ####/##/## ##:##:## foo/one
 EOT
-diff $diffopt "$chkfile" "$outfile" || test_fail "test 2 failed"
 
-$RSYNC -r localhost::test-from/f* \
-    | sed "$FILE_REPL" | sed "$DIR_REPL" | sed "$LS_REPL" \
-    | tee "$outfile"
-cat <<EOT >"$chkfile"
+checkdiff "$RSYNC -r localhost::test-from/f*" \
+	"sed -e '$FILE_REPL' -e '$DIR_REPL' -e '$LS_REPL'" <<EOT
 drwxr-xr-x         DIR ####/##/## ##:##:## foo
 -rw-r--r--           4 ####/##/## ##:##:## foo/one
 EOT
 diff $diffopt "$chkfile" "$outfile" || test_fail "test 3 failed"
 
 if $RSYNC --version | grep "[, ] atimes" >/dev/null; then
-    $RSYNC -rU localhost::test-from/f* \
-	| sed "$FILE_REPL" | sed "$DIR_REPL" | sed "$LS_REPL" \
-	| tee "$outfile"
-    cat <<EOT >"$chkfile"
+    checkdiff "$RSYNC -rU localhost::test-from/f*" \
+	"sed -e '$FILE_REPL' -e '$DIR_REPL' -e '$LS_REPL'" <<EOT
 drwxr-xr-x         DIR ####/##/## ##:##:##                     foo
 -rw-r--r--           4 ####/##/## ##:##:## ####/##/## ##:##:## foo/one
 EOT
-    diff $diffopt "$chkfile" "$outfile" || test_fail "test 3 failed"
 fi

testsuite/delay-updates.test

@@ -0,0 +1,21 @@
+#!/bin/sh
+
+# Test rsync --delay-updates
+
+. "$suitedir/rsync.fns"
+
+mkdir "$fromdir"
+
+echo 1 > "$fromdir/foo"
+
+checkit "$RSYNC -aiv --delay-updates \"$fromdir/\" \"$todir/\"" "$fromdir" "$todir"
+
+mkdir "$todir/.~tmp~"
+echo 2 > "$todir/.~tmp~/foo"
+touch -r .. "$todir/.~tmp~/foo" "$todir/foo"
+echo 3 > "$fromdir/foo"
+
+checkit "$RSYNC -aiv --delay-updates \"$fromdir/\" \"$todir/\"" "$fromdir" "$todir"
+
+# The script would have aborted on error, so getting here means we've won.
+exit 0

testsuite/delete.test

@@ -1,11 +1,11 @@
-#! /bin/sh
+#!/bin/sh
 
-# Copyright (C) 2005-2020 Wayne Davison
+# Copyright (C) 2005-2022 Wayne Davison
 
 # This program is distributable under the terms of the GNU GPL (see
 # COPYING).
 
-# Test rsync handling of various delete directives.  
+# Test rsync handling of various delete directives.
 
 . "$suitedir/rsync.fns"
 
@@ -23,12 +23,12 @@ echo extra >"$todir"/emptydir/subdir/remove4
 # Also, make sure that --dry-run --del doesn't output anything extraneous.
 $RSYNC -av "$fromdir/" "$chkdir/copy/" >"$tmpdir/copy.out" 2>&1
 cat "$tmpdir/copy.out"
-egrep -v '^(created directory|sent|total size) ' "$tmpdir/copy.out" >"$tmpdir/copy.new"
+grep -E -v '^(created directory|sent|total size) ' "$tmpdir/copy.out" >"$tmpdir/copy.new"
 mv "$tmpdir/copy.new" "$tmpdir/copy.out"
 
 $RSYNC -avn --del "$fromdir/" "$chkdir/copy2/" >"$tmpdir/copy2.out" 2>&1 || true
 cat "$tmpdir/copy2.out"
-egrep -v '^(created directory|sent|total size) ' "$tmpdir/copy2.out" >"$tmpdir/copy2.new"
+grep -E -v '^(created directory|sent|total size) ' "$tmpdir/copy2.out" >"$tmpdir/copy2.new"
 mv "$tmpdir/copy2.new" "$tmpdir/copy2.out"
 
 diff $diffopt "$tmpdir/copy.out" "$tmpdir/copy2.out"

testsuite/devices.test

@@ -1,4 +1,4 @@
-#! /bin/sh
+#!/bin/sh
 
 # Copyright (C) 2002 by Martin Pool <mbp@samba.org>
 
@@ -9,9 +9,6 @@
 
 . "$suitedir/rsync.fns"
 
-chkfile="$scratchdir/rsync.chk"
-outfile="$scratchdir/rsync.out"
-
 # Build some hardlinks
 
 case $0 in
@@ -116,33 +113,21 @@ mkfifo "$fromdir/fifo" || mknod "$fromdir/fifo" p || test_skipped "Can't run mkf
 # Work around time rounding/truncating issue by touching both files.
 touch -r "$fromdir/block" "$fromdir/block" "$fromdir/block2"
 
-$RSYNC -ai "$fromdir/block" "$todir/block2" \
-    | tee "$outfile"
-cat <<EOT >"$chkfile"
+checkdiff "$RSYNC -ai '$fromdir/block' '$todir/block2'" <<EOT
 cD$all_plus block
 EOT
-diff $diffopt "$chkfile" "$outfile" || test_fail "test 1 failed"
 
-$RSYNC -ai "$fromdir/block2" "$todir/block" \
-    | tee "$outfile"
-cat <<EOT >"$chkfile"
+checkdiff "$RSYNC -ai '$fromdir/block2' '$todir/block'" <<EOT
 cD$all_plus block2
 EOT
-diff $diffopt "$chkfile" "$outfile" || test_fail "test 2 failed"
 
 sleep 1
 
-$RSYNC -Di "$fromdir/block3" "$todir/block" \
-    | tee "$outfile"
-cat <<EOT >"$chkfile"
+checkdiff "$RSYNC -Di '$fromdir/block3' '$todir/block'" <<EOT
 cDc.T.$dots block3
 EOT
-diff $diffopt "$chkfile" "$outfile" || test_fail "test 3 failed"
 
-$RSYNC -aiHvv "$fromdir/" "$todir/" \
-    | tee "$outfile"
-filter_outfile
-cat <<EOT >"$chkfile"
+cat >"$chkfile" <<EOT
 .d..t.$dots ./
 cDc.t.$dots block
 cDc...$dots block2
@@ -157,7 +142,8 @@ if test "$CAN_HLINK_SPECIAL" = no; then
     grep -v block3.5 <"$chkfile" >"$chkfile.new"
     mv "$chkfile.new" "$chkfile"
 fi
-diff $diffopt "$chkfile" "$outfile" || test_fail "test 4 failed"
+
+checkdiff2 "$RSYNC -aiHvv '$fromdir/' '$todir/'" v_filt
 
 echo "check how the directory listings compare with diff:"
 echo ""
@@ -167,9 +153,8 @@ diff $diffopt "$tmpdir/ls-from" "$tmpdir/ls-to"
 
 if test "$CAN_HLINK_SPECIAL" = yes; then
     set -x
-    $RSYNC -aii --link-dest="$todir" "$fromdir/" "$chkdir/" \
-	| tee "$outfile"
-    cat <<EOT >"$chkfile"
+    checkdiff "$RSYNC -aii --link-dest='$todir' '$fromdir/' '$chkdir/'" <<EOT
+created directory $chkdir
 cd$allspace ./
 hD$allspace block
 hD$allspace block2
@@ -180,7 +165,6 @@ hD$allspace char2
 hD$allspace char3
 hS$allspace fifo
 EOT
-    diff $diffopt "$chkfile" "$outfile" || test_fail "test 5 failed"
 fi
 
 # The script would have aborted on error, so getting here means we've won.

testsuite/dir-sgid.test

@@ -1,4 +1,4 @@
-#! /bin/sh
+#!/bin/sh
 
 # This program is distributable under the terms of the GNU GPL (see
 # COPYING).

testsuite/duplicates.test

@@ -1,11 +1,11 @@
-#! /bin/sh
+#!/bin/sh
 
 # Copyright (C) 2002 by Martin Pool <mbp@samba.org>
 
 # This program is distributable under the terms of the GNU GPL (see
 # COPYING).
 
-# Test rsync handling of duplicate filenames.  
+# Test rsync handling of duplicate filenames.
 
 # It's quite possible that the user might specify the same source file
 # more than once on the command line, perhaps through shell variables
@@ -29,18 +29,14 @@ name2="$fromdir/name2"
 echo "This is the file" > "$name1"
 ln -s "$name1" "$name2" || test_fail "can't create symlink"
 
-outfile="$scratchdir/rsync.out"
-
 checkit "$RSYNC -avv '$fromdir/' '$fromdir/' '$fromdir/' '$fromdir/' '$fromdir/' '$fromdir/' '$fromdir/' '$fromdir/' '$fromdir/' '$fromdir/' '$todir/'" "$fromdir" "$todir" \
     | tee "$outfile"
 
 # Make sure each file was only copied once...
-if [ `grep -c '^name1$' "$outfile"` != 1 ]
-then
+if [ `grep -c '^name1$' "$outfile"` != 1 ]; then
     test_fail "name1 was not copied exactly once"
 fi
-if [ `grep -c '^name2 -> ' "$outfile"` != 1 ]
-then
+if [ `grep -c '^name2 -> ' "$outfile"` != 1 ]; then
     test_fail "name2 was not copied exactly once"
 fi
 

testsuite/exclude.test

@@ -1,6 +1,6 @@
-#! /bin/sh
+#!/bin/sh
 
-# Copyright (C) 2003-2020 Wayne Davison
+# Copyright (C) 2003-2022 Wayne Davison
 
 # This program is distributable under the terms of the GNU GPL (see
 # COPYING).
@@ -12,9 +12,6 @@
 
 . "$suitedir/rsync.fns"
 
-chkfile="$scratchdir/rsync.chk"
-outfile="$scratchdir/rsync.out"
-
 CVSIGNORE='*.junk'
 export CVSIGNORE
 
@@ -118,7 +115,8 @@ ln -s too "$fromdir/bar/down/to/foo/sym"
 
 # Start to prep an --update test dir
 mkdir "$scratchdir/up1" "$scratchdir/up2"
-touch "$scratchdir/up1/older" "$scratchdir/up2/newer"
+touch "$scratchdir/up1/dst-newness" "$scratchdir/up2/src-newness"
+touch "$scratchdir/up1/same-newness" "$scratchdir/up2/same-newness"
 touch "$scratchdir/up1/extra-src" "$scratchdir/up2/extra-dest"
 
 # Create chkdir with what we expect to be excluded.
@@ -133,7 +131,7 @@ rm "$chkdir"/bar/down/to/foo/to "$chkdir"/bar/down/to/foo/file[235-9]
 rm "$chkdir"/mid/for/foo/extra
 
 # Finish prep for the --update test (run last)
-touch "$scratchdir/up1/newer" "$scratchdir/up2/older"
+touch "$scratchdir/up1/src-newness" "$scratchdir/up2/dst-newness"
 
 # Un-tweak the directory times in our first (weak) exclude test (though
 # it's a good test of the --existing option).
@@ -227,13 +225,13 @@ checkit "$RSYNC -avv $relative_opts --exclude='$fromdir/foo/down' \
     '$fromdir/foo' '$todir'" "$chkdir$fromdir/foo" "$todir$fromdir/foo"
 
 # Now we'll test the --update option.
-$RSYNC -aiO --update touch "$scratchdir/up1/" "$scratchdir/up2/" \
-    | tee "$outfile"
-cat <<EOT >"$chkfile"
+checkdiff "$RSYNC -aiiO --update --info=skip '$scratchdir/up1/' '$scratchdir/up2/'" \
+	"grep -v '^\.d$allspace'" <<EOT
+dst-newness is newer
 >f$all_plus extra-src
->f..t.$dots newer
+.f$allspace same-newness
+>f..t.$dots src-newness
 EOT
-diff $diffopt "$chkfile" "$outfile" || test_fail "--update test failed"
 
 # The script would have aborted on error, so getting here means we've won.
 exit 0

testsuite/executability.test

@@ -1,4 +1,4 @@
-#! /bin/sh
+#!/bin/sh
 
 # This program is distributable under the terms of the GNU GPL (see
 # COPYING).

testsuite/fuzzy.test

@@ -1,6 +1,6 @@
-#! /bin/sh
+#!/bin/sh
 
-# Copyright (C) 2005-2020 Wayne Davison
+# Copyright (C) 2005-2022 Wayne Davison
 
 # This program is distributable under the terms of the GNU GPL (see
 # COPYING).

testsuite/hardlinks.test

@@ -1,4 +1,4 @@
-#! /bin/sh
+#!/bin/sh
 
 # Copyright (C) 2002 by Martin Pool <mbp@samba.org>
 
@@ -13,8 +13,6 @@
 
 SSH="$scratchdir/src/support/lsh.sh"
 
-outfile="$scratchdir/rsync.out"
-
 # Build some hardlinks
 
 fromdir="$scratchdir/from"
@@ -70,7 +68,7 @@ echo "This is another file" >"$fromdir/solo"
 ln "$fromdir/solo" "$chkdir/solo" || test_fail "Can't create hardlink"
 
 # Make sure that the checksum data doesn't slide due to an HLINK_BUMP() change.
-$RSYNC -aHivc --debug=HLINK5 "$fromdir/" "$chkdir/" | tee "$outfile"
+checktee "$RSYNC -aHivc --debug=HLINK5 '$fromdir/' '$chkdir/'"
 grep solo "$outfile" && test_fail "Erroneous copy of solo file occurred!"
 
 # Make sure there's nothing wrong with sending a single file with -H

testsuite/itemize.test

@@ -1,6 +1,6 @@
-#! /bin/sh
+#!/bin/sh
 
-# Copyright (C) 2005-2020 Wayne Davison
+# Copyright (C) 2005-2022 Wayne Davison
 
 # This program is distributable under the terms of the GNU GPL (see
 # COPYING).
@@ -12,9 +12,6 @@
 
 to2dir="$tmpdir/to2"
 
-chkfile="$scratchdir/rsync.chk"
-outfile="$scratchdir/rsync.out"
-
 makepath "$fromdir/foo"
 makepath "$fromdir/bar/baz"
 cp_p "$srcdir/configure.ac" "$fromdir/foo/config1"
@@ -28,10 +25,18 @@ ln "$fromdir/foo/config1" "$fromdir/foo/extra"
 rm -f "$to2dir"
 
 # Check if rsync is set to hard-link symlinks.
-if egrep '^#define CAN_HARDLINK_SYMLINK 1' config.h >/dev/null; then
+if $RSYNC --version | grep "[, ] hardlink-symlinks" >/dev/null; then
     L=hL
+    sym_dots="$allspace"
+    L_sym_dots=".L$allspace"
+    is_uptodate='is uptodate'
+    touch "$chkfile.extra"
 else
     L=cL
+    sym_dots="c.t.$dots"
+    L_sym_dots="cL$sym_dots"
+    is_uptodate='-> ../bar/baz/rsync'
+    echo "cL$sym_dots foo/sym $is_uptodate" >"$chkfile.extra"
 fi
 
 # Check if rsync can preserve time on symlinks
@@ -48,9 +53,8 @@ case "$RSYNC" in
     ;;
 esac
 
-$RSYNC -iplr "$fromdir/" "$todir/" \
-    | tee "$outfile"
-cat <<EOT >"$chkfile"
+checkdiff "$RSYNC -iplr '$fromdir/' '$todir/'" <<EOT
+created directory $todir
 cd$all_plus ./
 cd$all_plus bar/
 cd$all_plus bar/baz/
@@ -61,22 +65,18 @@ cd$all_plus foo/
 >f$all_plus foo/extra
 cL$all_plus foo/sym -> ../bar/baz/rsync
 EOT
-diff $diffopt "$chkfile" "$outfile" || test_fail "test 1 failed"
 
 # Ensure there are no accidental directory-time problems.
 $RSYNC -a -f '-! */' "$fromdir/" "$todir"
 
 cp_p "$srcdir/configure.ac" "$fromdir/foo/config2"
 chmod 601 "$fromdir/foo/config2"
-$RSYNC -iplrH "$fromdir/" "$todir/" \
-    | tee "$outfile"
-cat <<EOT >"$chkfile"
+checkdiff "$RSYNC -iplrH '$fromdir/' '$todir/'" <<EOT
 >f..T.$dots bar/baz/rsync
 >f..T.$dots foo/config1
 >f.sTp$dots foo/config2
 hf..T.$dots foo/extra => foo/config1
 EOT
-diff $diffopt "$chkfile" "$outfile" || test_fail "test 2 failed"
 
 $RSYNC -a -f '-! */' "$fromdir/" "$todir"
 cp_p "$srcdir/config.sub" "$fromdir/foo/config2"
@@ -88,38 +88,30 @@ umask 022
 chmod 600 "$fromdir/foo/config2"
 chmod 777 "$todir/bar/baz/rsync"
 
-$RSYNC -iplrtc "$fromdir/" "$todir/" \
-    | tee "$outfile"
-cat <<EOT >"$chkfile"
+checkdiff "$RSYNC -iplrtc '$fromdir/' '$todir/'" <<EOT
 .f..tp$dots bar/baz/rsync
 .d..t.$dots foo/
 .f..t.$dots foo/config1
 >fcstp$dots foo/config2
 cLc$T.$dots foo/sym -> ../bar/baz/rsync
 EOT
-diff $diffopt "$chkfile" "$outfile" || test_fail "test 3 failed"
 
 cp_p "$srcdir/configure.ac" "$fromdir/foo/config2"
 chmod 600 "$fromdir/foo/config2"
 # Lack of -t is for unchanged hard-link stress-test!
-$RSYNC -vvplrH "$fromdir/" "$todir/" \
-    | tee "$outfile"
-filter_outfile
-cat <<EOT >"$chkfile"
+checkdiff "$RSYNC -vvplrH '$fromdir/' '$todir/'" \
+	v_filt <<EOT
 bar/baz/rsync is uptodate
 foo/config1 is uptodate
 foo/extra is uptodate
 foo/sym is uptodate
 foo/config2
 EOT
-diff $diffopt "$chkfile" "$outfile" || test_fail "test 4 failed"
 
 chmod 747 "$todir/bar/baz/rsync"
 $RSYNC -a -f '-! */' "$fromdir/" "$todir"
-$RSYNC -ivvplrtH "$fromdir/" "$todir/" \
-    | tee "$outfile"
-filter_outfile
-cat <<EOT >"$chkfile"
+checkdiff "$RSYNC -ivvplrtH '$fromdir/' '$todir/'" \
+	v_filt <<EOT
 .d$allspace ./
 .d$allspace bar/
 .d$allspace bar/baz/
@@ -130,47 +122,23 @@ cat <<EOT >"$chkfile"
 hf$allspace foo/extra
 .L$allspace foo/sym -> ../bar/baz/rsync
 EOT
-diff $diffopt "$chkfile" "$outfile" || test_fail "test 5 failed"
 
 chmod 757 "$todir/foo/config1"
 touch "$todir/foo/config2"
-$RSYNC -vplrtH "$fromdir/" "$todir/" \
-    | tee "$outfile"
-filter_outfile
-cat <<EOT >"$chkfile"
+checkdiff "$RSYNC -vplrtH '$fromdir/' '$todir/'" \
+	v_filt <<EOT
 foo/config2
 EOT
-diff $diffopt "$chkfile" "$outfile" || test_fail "test 6 failed"
 
 chmod 757 "$todir/foo/config1"
 touch "$todir/foo/config2"
-$RSYNC -iplrtH "$fromdir/" "$todir/" \
-    | tee "$outfile"
-cat <<EOT >"$chkfile"
+checkdiff "$RSYNC -iplrtH '$fromdir/' '$todir/'" <<EOT
 .f...p$dots foo/config1
 >f..t.$dots foo/config2
 EOT
-diff $diffopt "$chkfile" "$outfile" || test_fail "test 7 failed"
 
-$RSYNC -ivvplrtH --copy-dest=../to "$fromdir/" "$to2dir/" \
-    | tee "$outfile"
-filter_outfile
-case `tail -1 "$outfile"` in
-cLc.t*)
-    sym_dots="c.t.$dots"
-    L_sym_dots="cL$sym_dots"
-    is_uptodate='-> ../bar/baz/rsync'
-    echo "cL$sym_dots foo/sym $is_uptodate" >"$chkfile.extra"
-    L=cL
-    ;;
-*)
-    sym_dots="$allspace"
-    L_sym_dots=".L$allspace"
-    is_uptodate='is uptodate'
-    touch "$chkfile.extra"
-    ;;
-esac
-cat <<EOT >"$chkfile"
+checkdiff "$RSYNC -ivvplrtH --copy-dest=../to '$fromdir/' '$to2dir/'" \
+	v_filt <<EOT
 cd$allspace ./
 cd$allspace bar/
 cd$allspace bar/baz/
@@ -181,21 +149,17 @@ cf$allspace foo/config2
 hf$allspace foo/extra => foo/config1
 cL$sym_dots foo/sym -> ../bar/baz/rsync
 EOT
-diff $diffopt "$chkfile" "$outfile" || test_fail "test 8 failed"
 
 rm -rf "$to2dir"
-$RSYNC -iplrtH --copy-dest=../to "$fromdir/" "$to2dir/" \
-    | tee "$outfile"
 cat - "$chkfile.extra" <<EOT >"$chkfile"
+created directory $to2dir
 hf$allspace foo/extra => foo/config1
 EOT
-diff $diffopt "$chkfile" "$outfile" || test_fail "test 9 failed"
+checkdiff2 "$RSYNC -iplrtH --copy-dest=../to '$fromdir/' '$to2dir/'"
 
 rm -rf "$to2dir"
-$RSYNC -vvplrtH --copy-dest="$todir" "$fromdir/" "$to2dir/" \
-    | tee "$outfile"
-filter_outfile
-cat <<EOT >"$chkfile"
+checkdiff "$RSYNC -vvplrtH --copy-dest='$todir' '$fromdir/' '$to2dir/'" \
+	v_filt <<EOT
 ./ is uptodate
 bar/ is uptodate
 bar/baz/ is uptodate
@@ -206,13 +170,10 @@ foo/config2 is uptodate
 foo/sym $is_uptodate
 foo/extra => foo/config1
 EOT
-diff $diffopt "$chkfile" "$outfile" || test_fail "test 10 failed"
 
 rm -rf "$to2dir"
-$RSYNC -ivvplrtH --link-dest="$todir" "$fromdir/" "$to2dir/" \
-    | tee "$outfile"
-filter_outfile
-cat <<EOT >"$chkfile"
+checkdiff "$RSYNC -ivvplrtH --link-dest='$todir' '$fromdir/' '$to2dir/'" \
+	v_filt <<EOT
 cd$allspace ./
 cd$allspace bar/
 cd$allspace bar/baz/
@@ -223,27 +184,19 @@ hf$allspace foo/config2
 hf$allspace foo/extra => foo/config1
 $L$sym_dots foo/sym -> ../bar/baz/rsync
 EOT
-diff $diffopt "$chkfile" "$outfile" || test_fail "test 11 failed"
 
 rm -rf "$to2dir"
-$RSYNC -iplrtH --dry-run --link-dest=../to "$fromdir/" "$to2dir/" \
-    | tee "$outfile"
 cat - "$chkfile.extra" <<EOT >"$chkfile"
+created directory $to2dir
 EOT
-diff $diffopt "$chkfile" "$outfile" || test_fail "test 12 failed"
+checkdiff2 "$RSYNC -iplrtH --dry-run --link-dest=../to '$fromdir/' '$to2dir/'"
 
 rm -rf "$to2dir"
-$RSYNC -iplrtH --link-dest=../to "$fromdir/" "$to2dir/" \
-    | tee "$outfile"
-cat - "$chkfile.extra" <<EOT >"$chkfile"
-EOT
-diff $diffopt "$chkfile" "$outfile" || test_fail "test 13 failed"
+checkdiff2 "$RSYNC -iplrtH --link-dest=../to '$fromdir/' '$to2dir/'"
 
 rm -rf "$to2dir"
-$RSYNC -vvplrtH --link-dest="$todir" "$fromdir/" "$to2dir/" \
-    | tee "$outfile"
-filter_outfile
-cat <<EOT >"$chkfile"
+checkdiff "$RSYNC -vvplrtH --link-dest='$todir' '$fromdir/' '$to2dir/'" \
+	v_filt <<EOT
 ./ is uptodate
 bar/ is uptodate
 bar/baz/ is uptodate
@@ -254,13 +207,10 @@ foo/config2 is uptodate
 foo/extra is uptodate
 foo/sym $is_uptodate
 EOT
-diff $diffopt "$chkfile" "$outfile" || test_fail "test 14 failed"
 
 rm -rf "$to2dir"
-$RSYNC -ivvplrtH --compare-dest="$todir" "$fromdir/" "$to2dir/" \
-    | tee "$outfile"
-filter_outfile
-cat <<EOT >"$chkfile"
+checkdiff "$RSYNC -ivvplrtH --compare-dest='$todir' '$fromdir/' '$to2dir/'" \
+	v_filt <<EOT
 cd$allspace ./
 cd$allspace bar/
 cd$allspace bar/baz/
@@ -271,20 +221,16 @@ cd$allspace foo/
 .f$allspace foo/extra
 $L_sym_dots foo/sym -> ../bar/baz/rsync
 EOT
-diff $diffopt "$chkfile" "$outfile" || test_fail "test 15 failed"
 
 rm -rf "$to2dir"
-$RSYNC -iplrtH --compare-dest="$todir" "$fromdir/" "$to2dir/" \
-    | tee "$outfile"
 cat - "$chkfile.extra" <<EOT >"$chkfile"
+created directory $to2dir
 EOT
-diff $diffopt "$chkfile" "$outfile" || test_fail "test 16 failed"
+checkdiff2 "$RSYNC -iplrtH --compare-dest='$todir' '$fromdir/' '$to2dir/'"
 
 rm -rf "$to2dir"
-$RSYNC -vvplrtH --compare-dest="$todir" "$fromdir/" "$to2dir/" \
-    | tee "$outfile"
-filter_outfile
-cat <<EOT >"$chkfile"
+checkdiff "$RSYNC -vvplrtH --compare-dest='$todir' '$fromdir/' '$to2dir/'" \
+	v_filt <<EOT
 ./ is uptodate
 bar/ is uptodate
 bar/baz/ is uptodate
@@ -295,7 +241,6 @@ foo/config2 is uptodate
 foo/extra is uptodate
 foo/sym $is_uptodate
 EOT
-diff $diffopt "$chkfile" "$outfile" || test_fail "test 17 failed"
 
 # The script would have aborted on error, so getting here means we've won.
 exit 0

testsuite/merge.test

@@ -1,6 +1,6 @@
-#! /bin/sh
+#!/bin/sh
 
-# Copyright (C) 2004-2020 Wayne Davison
+# Copyright (C) 2004-2022 Wayne Davison
 
 # This program is distributable under the terms of the GNU GPL (see
 # COPYING).