Preparing for release of 3.2.5pre2

From zlib commit eff308af425b67093bab25f80f1ae950166bece1.
Fixes CVE-2022-37434.

.cirrus.yml

@@ -0,0 +1,23 @@
+freebsd_task:
+  name: FreeBSD
+  freebsd_instance:
+    image_family: freebsd-13-0
+  env:
+    PATH: /usr/local/bin:$PATH
+  prep_script:
+    - dd if=/dev/zero of=/tmp/zpool bs=1M count=1024
+    - zpool create -m `pwd`/testtmp zpool /tmp/zpool
+    - pkg install -y bash autotools m4 xxhash zstd liblz4 wget
+    - wget -O git-version.h https://gist.githubusercontent.com/WayneD/c11243fa374fc64d4e42f2855c8e3827/raw/rsync-git-version.h
+  configure_script:
+    - CPPFLAGS=-I/usr/local/include/ LDFLAGS=-L/usr/local/lib/ ./configure --disable-md2man
+  make_script:
+    - make
+  install_script:
+    - make install
+  info_script:
+    - rsync --version
+  test_script:
+    - RSYNC_EXPECT_SKIPPED=acls-default,acls,crtimes,protected-regular make check
+  ssl_file_list_script:
+    - rsync-ssl --no-motd download.samba.org::rsyncftp/ || true

.gitattributes

@@ -0,0 +1 @@
+* text=auto eol=lf

.github/workflows/build.yml

@@ -0,0 +1,125 @@
+name: build
+
+on:
+  push:
+    branches: [ master ]
+    paths-ignore: [ .cirrus.yml ]
+  pull_request:
+    branches: [ master ]
+    paths-ignore: [ .cirrus.yml ]
+  schedule:
+    - cron: '42 8 * * *'
+
+jobs:
+
+  ubuntu-build:
+    runs-on: ubuntu-20.04
+    steps:
+    - uses: actions/checkout@v2
+    - name: prep
+      run: |
+        sudo apt-get install acl libacl1-dev attr libattr1-dev liblz4-dev libzstd-dev libxxhash-dev python3-cmarkgfm openssl wget
+        wget -O git-version.h https://gist.githubusercontent.com/WayneD/c11243fa374fc64d4e42f2855c8e3827/raw/rsync-git-version.h
+        echo "/usr/local/bin" >>$GITHUB_PATH
+    - name: configure
+      run: ./configure --with-rrsync
+    - name: make
+      run: make
+    - name: install
+      run: sudo make install
+    - name: info
+      run: rsync --version
+    - name: check
+      run: sudo RSYNC_EXPECT_SKIPPED=crtimes make check
+    - name: check30
+      run: sudo RSYNC_EXPECT_SKIPPED=crtimes make check30
+    - name: check29
+      run: sudo RSYNC_EXPECT_SKIPPED=crtimes make check29
+    - name: ssl file list
+      run: rsync-ssl --no-motd download.samba.org::rsyncftp/ || true
+    - name: save artifact
+      uses: actions/upload-artifact@v2
+      with:
+        name: ubuntu-bin
+        path: |
+          rsync
+          rsync-ssl
+          rsync.1
+          rsync-ssl.1
+          rsyncd.conf.5
+          rrsync.1
+          rrsync
+
+  macos-build:
+    runs-on: macos-latest
+    steps:
+    - uses: actions/checkout@v2
+    - name: prep
+      run: |
+        brew install automake openssl xxhash zstd lz4 wget
+        sudo pip3 install commonmark
+        wget -O git-version.h https://gist.githubusercontent.com/WayneD/c11243fa374fc64d4e42f2855c8e3827/raw/rsync-git-version.h
+        echo "/usr/local/bin" >>$GITHUB_PATH
+    - name: configure
+      run: CPPFLAGS=-I/usr/local/opt/openssl/include/ LDFLAGS=-L/usr/local/opt/openssl/lib/ ./configure --with-rrsync
+    - name: make
+      run: make
+    - name: install
+      run: sudo make install
+    - name: info
+      run: rsync --version
+    - name: check
+      run: sudo RSYNC_EXPECT_SKIPPED=acls-default,chmod-temp-dir,chown-fake,devices-fake,dir-sgid,protected-regular,xattrs-hlink,xattrs make check
+    - name: ssl file list
+      run: rsync-ssl --no-motd download.samba.org::rsyncftp/ || true
+    - name: save artifact
+      uses: actions/upload-artifact@v2
+      with:
+        name: macos-bin
+        path: |
+          rsync
+          rsync-ssl
+          rsync.1
+          rsync-ssl.1
+          rsyncd.conf.5
+          rrsync.1
+          rrsync
+
+  cygwin-build:
+    runs-on: windows-2022
+    if: (github.event_name == 'schedule' || contains(github.event.head_commit.message, '[buildall]'))
+    steps:
+    - uses: actions/checkout@v2
+    - name: cygwin
+      run: choco install -y --no-progress cygwin cyg-get
+    - name: prep
+      run: |
+        cyg-get make autoconf automake gcc-core attr libattr-devel python39 python39-pip libzstd-devel liblz4-devel libssl-devel libxxhash0 libxxhash-devel
+        curl.exe -o git-version.h https://gist.githubusercontent.com/WayneD/c11243fa374fc64d4e42f2855c8e3827/raw/rsync-git-version.h
+        echo "C:/tools/cygwin/bin" >>$Env:GITHUB_PATH
+    - name: commonmark
+      run: bash -c 'python3 -mpip install --user commonmark'
+    - name: configure
+      run: bash -c './configure --with-rrsync'
+    - name: make
+      run: bash -c 'make'
+    - name: install
+      run: bash -c 'make install'
+    - name: info
+      run: bash -c '/usr/local/bin/rsync --version'
+    - name: check
+      run: bash -c 'RSYNC_EXPECT_SKIPPED=acls-default,acls,chown,devices,dir-sgid,protected-regular make check'
+    - name: ssl file list
+      run: bash -c 'PATH="/usr/local/bin:$PATH" rsync-ssl --no-motd download.samba.org::rsyncftp/ || true'
+    - name: save artifact
+      uses: actions/upload-artifact@v2
+      with:
+        name: cygwin-bin
+        path: |
+          rsync.exe
+          rsync-ssl
+          rsync.1
+          rsync-ssl.1
+          rsyncd.conf.5
+          rrsync.1
+          rrsync

.github/workflows/ccpp.yml

@@ -1,31 +0,0 @@
-name: build
-
-on:
-  push:
-    branches: [ master ]
-  pull_request:
-    branches: [ master ]
-
-jobs:
-  build:
-
-    runs-on: ubuntu-20.04
-
-    steps:
-    - uses: actions/checkout@v2
-    - name: prepare-packages
-      run: sudo apt-get install fakeroot acl libacl1-dev attr libattr1-dev liblz4-dev libzstd-dev libxxhash-dev python3-cmarkgfm
-    - name: prepare-source
-      run: ./prepare-source
-    - name: configure
-      run: ./configure --with-included-popt --with-included-zlib
-    - name: make
-      run: make
-    - name: version-summary
-      run: ./rsync --version
-    - name: make check
-      run: make check
-    - name: make check30
-      run: make check30
-    - name: make check29
-      run: make check29

.gitignore

@@ -15,22 +15,27 @@ config.status
 aclocal.m4
 /proto.h
 /proto.h-tstamp
-/rsync*.1
-/rsync*.5
+/rsync*.[15]
+/rrsync
+/rrsync*.1
 /rsync*.html
+/rrsync*.html
 /help-rsync*.h
 /default-cvsignore.h
 /default-dont-compress.h
+/daemon-parm.h
 /.md2man-works
 /autom4te*.cache
 /confdefs.h
 /conftest*
 /dox
 /getgroups
+/gists
 /gmon.out
 /rsync
 /stunnel-rsyncd.conf
 /shconfig
+/git-version.h
 /testdir
 /tests-dont-exist
 /testtmp
@@ -52,3 +57,4 @@ aclocal.m4
 /build
 /auto-build-save
 .deps
+/*.exe

COPYING

@@ -1,7 +1,16 @@
+REGARDING OPENSSL AND XXHASH
+
+In addition, as a special exception, the copyright holders give
+permission to dynamically link rsync with the OpenSSL and xxhash
+libraries when those libraries are being distributed in compliance
+with their license terms, and to distribute a dynamically linked
+combination of rsync and these libraries.  This is also considered
+to be covered under the GPL's System Libraries exception.
+
                     GNU GENERAL PUBLIC LICENSE
                        Version 3, 29 June 2007
 
- Copyright (C) 2007 Free Software Foundation, Inc. <http://fsf.org/>
+ Copyright (C) 2007 Free Software Foundation, Inc. <https://fsf.org/>
  Everyone is permitted to copy and distribute verbatim copies
  of this license document, but changing it is not allowed.
 
@@ -645,7 +654,7 @@ the "copyright" line and a pointer to where the full notice is found.
     GNU General Public License for more details.
 
     You should have received a copy of the GNU General Public License
-    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+    along with this program.  If not, see <https://www.gnu.org/licenses/>.
 
 Also add information on how to contact you by electronic and paper mail.
 
@@ -664,20 +673,11 @@ might be different; for a GUI interface, you would use an "about box".
   You should also get your employer (if you work as a programmer) or school,
 if any, to sign a "copyright disclaimer" for the program, if necessary.
 For more information on this, and how to apply and follow the GNU GPL, see
-<http://www.gnu.org/licenses/>.
+<https://www.gnu.org/licenses/>.
 
   The GNU General Public License does not permit incorporating your program
 into proprietary programs.  If your program is a subroutine library, you
 may consider it more useful to permit linking proprietary applications with
 the library.  If this is what you want to do, use the GNU Lesser General
 Public License instead of this License.  But first, please read
-<http://www.gnu.org/philosophy/why-not-lgpl.html>.
-
-REGARDING OPENSSL AND XXHASH
-
-In addition, as a special exception, the copyright holders give
-permission to dynamically link rsync with the OpenSSL and xxhash
-libraries when those libraries are being distributed in compliance
-with their license terms, and to distribute a dynamically linked
-combination of rsync and these libraries.  This is also considered
-to be covered under the GPL's System Libraries exception.
+<https://www.gnu.org/licenses/why-not-lgpl.html>.

INSTALL.md

@@ -1,13 +1,172 @@
-To build and install rsync:
+# How to build and install rsync
 
-    $ ./configure
-    $ make
-    # make install
+When building rsync, you'll want to install various libraries in order to get
+all the features enabled.  The configure script will alert you when the
+newest libraries are missing and tell you the appropriate `--disable-LIB`
+option to use if you want to just skip that feature.  What follows are various
+support libraries that you may want to install to build rsync with the maximum
+features (the impatient can skip down to the package summary):
 
-You may set the installation directory and other parameters by options
-to ./configure.  To see them, use:
+## The basic setup
 
-    $ ./configure --help
+You need to have a C compiler installed and optionally a C++ compiler in order
+to try to build some hardware-accelerated checksum routines.  Rsync also needs
+a modern awk, which might be provided via gawk or nawk on some OSes.
+
+## Autoconf & manpages
+
+If you're installing from the git repo (instead of a release tar file) you'll
+also need the GNU autotools (autoconf & automake) and your choice of 2 python3
+markdown libraries: cmarkgfm or commonmark (needed to generate the manpages).
+If your OS doesn't provide a python3-cmarkgfm or python3-commonmark package,
+you can run the following to install the commonmark python library for your
+build user (after installing python3's pip package):
+
+>     python3 -mpip install --user commonmark
+
+You can test if you've got it fixed by running (from the rsync checkout):
+
+>     ./md-convert --test rsync-ssl.1.md
+
+Alternately, you can avoid generating the manpages by fetching the very latest
+versions (that match the latest git source) from the [generated-files][6] dir.
+One way to do that is to run:
+
+>     ./prepare-source fetchgen
+
+[6]: https://download.samba.org/pub/rsync/generated-files/
+
+## ACL support
+
+To support copying ACL file information, make sure you have an acl
+development library installed. It also helps to have the helper programs
+installed to manipulate ACLs and to run the rsync testsuite.
+
+## Xattr support
+
+To support copying xattr file information, make sure you have an attr
+development library installed. It also helps to have the helper programs
+installed to manipulate xattrs and to run the rsync testsuite.
+
+## xxhash
+
+The [xxHash library][1] provides extremely fast checksum functions that can
+make the "rsync algorithm" run much more quickly, especially when matching
+blocks in large files.  Installing this development library adds xxhash
+checksums as the default checksum algorithm.  You'll need at least v0.8.0
+if you want rsync to include the full range of its checksum algorithms.
+
+[1]: https://cyan4973.github.io/xxHash/
+
+## zstd
+
+The [zstd library][2] compression algorithm that uses less CPU than
+the default zlib algorithm at the same compression level.  Note that you
+need at least version 1.4, so you might need to skip the zstd compression if
+you can only install a 1.3 release.  Installing this development library
+adds zstd compression as the default compression algorithm.
+
+[2]: http://facebook.github.io/zstd/
+
+## lz4
+
+The [lz4 library][3] compression algorithm that uses very little CPU, though
+it also has the smallest compression ratio of other algorithms.  Installing
+this development library adds lz4 compression as an available compression
+algorithm.
+
+[3]: https://lz4.github.io/lz4/
+
+## openssl crypto
+
+The [openssl crypto library][4] provides some hardware accelerated checksum
+algorithms for MD4 and MD5.  Installing this development library makes rsync
+use the (potentially) faster checksum routines when computing MD4 & MD5
+checksums.
+
+[4]: https://www.openssl.org/docs/man1.0.2/man3/crypto.html
+
+## Package summary
+
+To help you get the libraries installed, here are some package install commands
+for various OSes.  The commands are split up to correspond with the above
+items, but feel free to combine the package names into a single install, if you
+like.
+
+ -  For Debian and Ubuntu (Debian Buster users may want to briefly(?) enable
+    buster-backports to update zstd from 1.3 to 1.4):
+
+    >     sudo apt install -y gcc g++ gawk autoconf automake python3-cmarkgfm
+    >     sudo apt install -y acl libacl1-dev
+    >     sudo apt install -y attr libattr1-dev
+    >     sudo apt install -y libxxhash-dev
+    >     sudo apt install -y libzstd-dev
+    >     sudo apt install -y liblz4-dev
+    >     sudo apt install -y libssl-dev
+
+ -  For CentOS (use EPEL for python3-pip):
+
+    >     sudo yum -y install epel-release
+    >     sudo yum -y install gcc g++ gawk autoconf automake python3-pip
+    >     sudo yum -y install acl libacl-devel
+    >     sudo yum -y install attr libattr-devel
+    >     sudo yum -y install xxhash-devel
+    >     sudo yum -y install libzstd-devel
+    >     sudo yum -y install lz4-devel
+    >     sudo yum -y install openssl-devel
+    >     python3 -mpip install --user commonmark
+
+ -  For Fedora 33:
+
+    >     sudo dnf -y install acl libacl-devel
+    >     sudo dnf -y install attr libattr-devel
+    >     sudo dnf -y install xxhash-devel
+    >     sudo dnf -y install libzstd-devel
+    >     sudo dnf -y install lz4-devel
+    >     sudo dnf -y install openssl-devel
+
+ -  For FreeBSD (this assumes that the python3 version is 3.7):
+
+    >     sudo pkg install -y autotools python3 py37-CommonMark
+    >     sudo pkg install -y xxhash
+    >     sudo pkg install -y zstd
+    >     sudo pkg install -y liblz4
+
+ -  For macOS:
+
+    >     brew install automake
+    >     brew install xxhash
+    >     brew install zstd
+    >     brew install lz4
+    >     brew install openssl
+
+ -  For Cygwin (with all cygwin programs stopped, run the appropriate setup program from a cmd shell):
+
+    >     setup-x86_64 --quiet-mode -P make,gawk,autoconf,automake,gcc-core,python38,python38-pip
+    >     setup-x86_64 --quiet-mode -P attr,libattr-devel
+    >     setup-x86_64 --quiet-mode -P libzstd-devel
+    >     setup-x86_64 --quiet-mode -P liblz4-devel
+    >     setup-x86_64 --quiet-mode -P libssl-devel
+
+    Sometimes cygwin has commonmark packaged and sometimes it doesn't. Now that
+    its python38 has stabilized, you could install python38-commonmark. Or just
+    avoid the issue by running this from a bash shell as your build user:
+
+    >     python3 -mpip install --user commonmark
+
+## Build and install
+
+After installing the various libraries, you need to configure, build, and
+install the source:
+
+>      ./configure
+>      make
+>      sudo make install
+
+The default install path is /usr/local/bin, but you can set the installation
+directory and other parameters using options to ./configure.  To see them, use:
+
+>     ./configure --help
 
 Configure tries to figure out if the local system uses group "nobody" or
 "nogroup" by looking in the /etc/group file.  (This is only used for the
@@ -19,56 +178,65 @@ config.h, or just override them in your /etc/rsyncd.conf file.
 As of 2.4.7, rsync uses Eric Troan's popt option-parsing library.  A
 cut-down copy of a recent release is included in the rsync distribution,
 and will be used if there is no popt library on your build host, or if
-the --with-included-popt option is passed to ./configure.
+the `--with-included-popt` option is passed to ./configure.
 
-If you configure using --enable-maintainer-mode, then rsync will try
+If you configure using `--enable-maintainer-mode`, then rsync will try
 to pop up an xterm on DISPLAY=:0 if it crashes.  You might find this
 useful, but it should be turned off for production builds.
 
-MAKE COMPATIBILITY
-------------------
+If you want to automatically use a separate "build" directory based on
+the current git branch name, start with a pristine git checkout and run
+"mkdir auto-build-save" before you run the first ./configure command.
+That will cause a fresh build dir to spring into existence along with a
+special Makefile symlink that allows you to run "make" and "./configure"
+from the source dir (the "build" dir gets auto switched based on branch).
+This is helpful when using the branch-from-patch and patch-update scripts
+to maintain the official rsync patches.  If you ever need to build from
+a "detached head" git position then you'll need to manually chdir into
+the build dir to run make.  I also like to create 2 more symlinks in the
+source dir:  `ln -s build/rsync . ; ln -s build/testtmp .`
+
+## Make compatibility
 
 Note that Makefile.in has a rule that uses a wildcard in a prerequisite.  If
 your make has a problem with this rule, you will see an error like this:
 
     Don't know how to make ./*.c
 
-You can change the "proto.h-tstamp" target in Makefile.in to list all the *.c
+You can change the "proto.h-tstamp" target in Makefile.in to list all the \*.c
 filenames explicitly in order to avoid this issue.
 
-RPM NOTES
----------
+## RPM notes
 
 Under packaging you will find .spec files for several distributions.
 The .spec file in packaging/lsb can be used for Linux systems that
 adhere to the Linux Standards Base (e.g., RedHat and others).
 
-HP-UX NOTES
------------
+## HP-UX notes
 
 The HP-UX 10.10 "bundled" C compiler seems not to be able to cope with
 ANSI C.  You may see this error message in config.log if ./configure
 fails:
 
-  (Bundled) cc: "configure", line 2162: error 1705: Function prototypes are an ANSI feature.
+    (Bundled) cc: "configure", line 2162: error 1705: Function prototypes are an ANSI feature.
 
 Install gcc or HP's "ANSI/C Compiler".
 
-MAC OSX NOTES
--------------
+## Mac OS X notes
 
 Some versions of Mac OS X (Darwin) seem to have an IPv6 stack, but do
-not completely implement the "New Sockets" API.  
+not completely implement the "New Sockets" API.
 
-<http://www.ipv6.org/impl/mac.html> says that Apple started to support
-IPv6 in 10.2 (Jaguar).  If your build fails, try again after running
-configure with --disable-ipv6.
+[This site][5] says that Apple started to support IPv6 in 10.2 (Jaguar).  If
+your build fails, try again after running configure with `--disable-ipv6`.
 
-IBM AIX NOTES
--------------
+[5]: http://www.ipv6.org/impl/mac.html
+
+## IBM AIX notes
 
 IBM AIX has a largefile problem with mkstemp.  See IBM PR-51921.
-The workaround is to append the below to config.h
-	#ifdef _LARGE_FILES
-	#undef HAVE_SECURE_MKSTEMP
-	#endif
+The workaround is to append the following to config.h:
+
+>     #ifdef _LARGE_FILES
+>     #undef HAVE_SECURE_MKSTEMP
+>     #endif

Makefile.in

@@ -1,4 +1,4 @@
-# The input file that configure uses to create the Makefile for rsync.
+# The Makefile for rsync (configure creates it from Makefile.in).
 
 prefix=@prefix@
 datarootdir=@datarootdir@
@@ -6,6 +6,7 @@ exec_prefix=@exec_prefix@
 bindir=@bindir@
 libdir=@libdir@/rsync
 mandir=@mandir@
+with_rrsync=@with_rrsync@
 
 LIBS=@LIBS@
 CC=@CC@
@@ -29,28 +30,30 @@ SHELL=/bin/sh
 .SUFFIXES:
 .SUFFIXES: .c .o
 
-SIMD_x86_64=simd-checksum-x86_64.o
-ASM_x86_64=lib/md5-asm-x86_64.o
+ROLL_SIMD_x86_64=simd-checksum-x86_64.o
+ROLL_ASM_x86_64=simd-checksum-avx2.o
+MD5_ASM_x86_64=lib/md5-asm-x86_64.o
 
-GENFILES=configure.sh aclocal.m4 config.h.in proto.h proto.h-tstamp rsync.1 rsync.1.html \
-	 rsync-ssl.1 rsync-ssl.1.html rsyncd.conf.5 rsyncd.conf.5.html
+GENFILES=configure.sh aclocal.m4 config.h.in rsync.1 rsync.1.html \
+	 rsync-ssl.1 rsync-ssl.1.html rsyncd.conf.5 rsyncd.conf.5.html \
+	 @GEN_RRSYNC@
 HEADERS=byteorder.h config.h errcode.h proto.h rsync.h ifuncs.h itypes.h inums.h \
-	lib/pool_alloc.h lib/mdigest.h lib/md-defines.h version.h
+	lib/pool_alloc.h lib/mdigest.h lib/md-defines.h
 LIBOBJ=lib/wildmatch.o lib/compat.o lib/snprintf.o lib/mdfour.o lib/md5.o \
 	lib/permstring.o lib/pool_alloc.o lib/sysacls.o lib/sysxattrs.o @LIBOBJS@
 zlib_OBJS=zlib/deflate.o zlib/inffast.o zlib/inflate.o zlib/inftrees.o \
 	zlib/trees.o zlib/zutil.o zlib/adler32.o zlib/compress.o zlib/crc32.o
 OBJS1=flist.o rsync.o generator.o receiver.o cleanup.o sender.o exclude.o \
-	util.o util2.o main.o checksum.o match.o syscall.o log.o backup.o delete.o
+	util1.o util2.o main.o checksum.o match.o syscall.o log.o backup.o delete.o
 OBJS2=options.o io.o compat.o hlink.o token.o uidlist.o socket.o hashtable.o \
-	fileio.o batch.o clientname.o chmod.o acls.o xattrs.o
-OBJS3=progress.o pipe.o @ASM@
+	usage.o fileio.o batch.o clientname.o chmod.o acls.o xattrs.o
+OBJS3=progress.o pipe.o @MD5_ASM@ @ROLL_SIMD@ @ROLL_ASM@
 DAEMON_OBJ = params.o loadparm.o clientserver.o access.o connection.o authenticate.o
 popt_OBJS=popt/findme.o  popt/popt.o  popt/poptconfig.o \
 	popt/popthelp.o popt/poptparse.o
-OBJS=$(OBJS1) $(OBJS2) $(OBJS3) @SIMD@ $(DAEMON_OBJ) $(LIBOBJ) @BUILD_ZLIB@ @BUILD_POPT@
+OBJS=$(OBJS1) $(OBJS2) $(OBJS3) $(DAEMON_OBJ) $(LIBOBJ) @BUILD_ZLIB@ @BUILD_POPT@
 
-TLS_OBJ = tls.o syscall.o t_stub.o lib/compat.o lib/snprintf.o lib/permstring.o lib/sysxattrs.o @BUILD_POPT@
+TLS_OBJ = tls.o syscall.o util2.o t_stub.o lib/compat.o lib/snprintf.o lib/permstring.o lib/sysxattrs.o @BUILD_POPT@
 
 # Programs we must have to run the test cases
 CHECK_PROGS = rsync$(EXEEXT) tls$(EXEEXT) getgroups$(EXEEXT) getfsdev$(EXEEXT) \
@@ -67,28 +70,32 @@ CHECK_OBJS=tls.o testrun.o getgroups.o getfsdev.o t_stub.o t_unsafe.o trimslash.
 	$(CC) -I. -I$(srcdir) $(CFLAGS) $(CPPFLAGS) -c $< @CC_SHOBJ_FLAG@
 @OBJ_RESTORE@
 
+all: Makefile rsync$(EXEEXT) stunnel-rsyncd.conf @MAKE_RRSYNC@ @MAKE_MAN@
 .PHONY: all
-all: Makefile rsync$(EXEEXT) stunnel-rsyncd.conf @MAKE_MAN@
 
 .PHONY: install
 install: all
-	-${MKDIR_P} ${DESTDIR}${bindir}
-	${INSTALLCMD} ${INSTALL_STRIP} -m 755 rsync$(EXEEXT) ${DESTDIR}${bindir}
-	${INSTALLCMD} -m 755 $(srcdir)/rsync-ssl ${DESTDIR}${bindir}
-	-${MKDIR_P} ${DESTDIR}${mandir}/man1
-	-${MKDIR_P} ${DESTDIR}${mandir}/man5
-	if test -f rsync.1; then ${INSTALLMAN} -m 644 rsync.1 ${DESTDIR}${mandir}/man1; fi
-	if test -f rsync-ssl.1; then ${INSTALLMAN} -m 644 rsync-ssl.1 ${DESTDIR}${mandir}/man1; fi
-	if test -f rsyncd.conf.5; then ${INSTALLMAN} -m 644 rsyncd.conf.5 ${DESTDIR}${mandir}/man5; fi
+	-$(MKDIR_P) $(DESTDIR)$(bindir)
+	$(INSTALLCMD) $(INSTALL_STRIP) -m 755 rsync$(EXEEXT) $(DESTDIR)$(bindir)
+	$(INSTALLCMD) -m 755 $(srcdir)/rsync-ssl $(DESTDIR)$(bindir)
+	-$(MKDIR_P) $(DESTDIR)$(mandir)/man1
+	-$(MKDIR_P) $(DESTDIR)$(mandir)/man5
+	if test -f rsync.1; then $(INSTALLMAN) -m 644 rsync.1 $(DESTDIR)$(mandir)/man1; fi
+	if test -f rsync-ssl.1; then $(INSTALLMAN) -m 644 rsync-ssl.1 $(DESTDIR)$(mandir)/man1; fi
+	if test -f rsyncd.conf.5; then $(INSTALLMAN) -m 644 rsyncd.conf.5 $(DESTDIR)$(mandir)/man5; fi
+	if test "$(with_rrsync)" = yes; then \
+	    $(INSTALLCMD) -m 755 rrsync $(DESTDIR)$(bindir); \
+	    if test -f rrsync.1; then $(INSTALLMAN) -m 644 rrsync.1 $(DESTDIR)$(mandir)/man1; fi; \
+	fi
 
 install-ssl-daemon: stunnel-rsyncd.conf
-	-${MKDIR_P} ${DESTDIR}/etc/stunnel
-	${INSTALLCMD} -m 644 stunnel-rsyncd.conf ${DESTDIR}/etc/stunnel/rsyncd.conf
+	-$(MKDIR_P) $(DESTDIR)/etc/stunnel
+	$(INSTALLCMD) -m 644 stunnel-rsyncd.conf $(DESTDIR)/etc/stunnel/rsyncd.conf
 	@if ! ls /etc/rsync-ssl/certs/server.* >/dev/null 2>/dev/null; then \
 	    echo "Note that you'll need to install the certificate used by /etc/stunnel/rsyncd.conf"; \
 	fi
 
-install-all: install install-ssl-client install-ssl-daemon
+install-all: install install-ssl-daemon
 
 install-strip:
 	$(MAKE) INSTALL_STRIP='-s' install
@@ -96,12 +103,14 @@ install-strip:
 rsync$(EXEEXT): $(OBJS)
 	$(CC) $(CFLAGS) $(LDFLAGS) -o $@ $(OBJS) $(LIBS)
 
+rrsync: support/rrsync
+	cp -p $(srcdir)/support/rrsync rrsync
+
 $(OBJS): $(HEADERS)
 $(CHECK_OBJS): $(HEADERS)
 tls.o xattrs.o: lib/sysxattrs.h
-options.o: latest-year.h help-rsync.h help-rsyncd.h
-exclude.o: default-cvsignore.h
-loadparm.o: default-dont-compress.h
+usage.o: version.h latest-year.h help-rsync.h help-rsyncd.h git-version.h default-cvsignore.h
+loadparm.o: default-dont-compress.h daemon-parm.h
 
 flist.o: rounding.h
 
@@ -111,6 +120,9 @@ default-cvsignore.h default-dont-compress.h: rsync.1.md define-from-md.awk
 help-rsync.h help-rsyncd.h: rsync.1.md help-from-md.awk
 	$(AWK) -f $(srcdir)/help-from-md.awk -v hfile=$@ $(srcdir)/rsync.1.md
 
+daemon-parm.h: daemon-parm.txt daemon-parm.awk
+	$(AWK) -f $(srcdir)/daemon-parm.awk $(srcdir)/daemon-parm.txt
+
 rounding.h: rounding.c rsync.h proto.h
 	@for r in 0 1 3; do \
 	    if $(CC) $(CFLAGS) $(CPPFLAGS) $(LDFLAGS) -o rounding -DEXTRA_ROUNDING=$$r -I. $(srcdir)/rounding.c >rounding.out 2>&1; then \
@@ -129,11 +141,20 @@ rounding.h: rounding.c rsync.h proto.h
 	fi
 	@rm -f rounding.out
 
-simd-checksum-x86_64.o: simd-checksum-x86_64.cpp
-	@$(srcdir)/cmdormsg disable-simd $(CXX) -I. $(CXXFLAGS) $(CPPFLAGS) -c -o $@ $(srcdir)/simd-checksum-x86_64.cpp
+git-version.h: ALWAYS_RUN
+	$(srcdir)/mkgitver
 
-lib/md5-asm-x86_64.o: lib/md5-asm-x86_64.S config.h lib/md-defines.h
-	@$(srcdir)/cmdormsg disable-asm $(CC) -I. @NOEXECSTACK@ -c -o $@ $(srcdir)/lib/md5-asm-x86_64.S
+.PHONY: ALWAYS_RUN
+ALWAYS_RUN:
+
+simd-checksum-x86_64.o: simd-checksum-x86_64.cpp
+	@$(srcdir)/cmd-or-msg disable-roll-simd $(CXX) -I. $(CXXFLAGS) $(CPPFLAGS) -c -o $@ $(srcdir)/simd-checksum-x86_64.cpp
+
+simd-checksum-avx2.o: simd-checksum-avx2.S
+	@$(srcdir)/cmd-or-msg disable-roll-asm $(CC) $(CFLAGS) -I. @NOEXECSTACK@ -c -o $@ $(srcdir)/simd-checksum-avx2.S
+
+lib/md5-asm-x86_64.o: lib/md5-asm-x86_64.S lib/md-defines.h
+	@$(srcdir)/cmd-or-msg disable-md5-asm $(CC) -I. @NOEXECSTACK@ -c -o $@ $(srcdir)/lib/md5-asm-x86_64.S
 
 tls$(EXEEXT): $(TLS_OBJ)
 	$(CC) $(CFLAGS) $(LDFLAGS) -o $@ $(TLS_OBJ) $(LIBS)
@@ -147,11 +168,11 @@ getgroups$(EXEEXT): getgroups.o
 getfsdev$(EXEEXT): getfsdev.o
 	$(CC) $(CFLAGS) $(LDFLAGS) -o $@ getfsdev.o $(LIBS)
 
-TRIMSLASH_OBJ = trimslash.o syscall.o t_stub.o lib/compat.o lib/snprintf.o
+TRIMSLASH_OBJ = trimslash.o syscall.o util2.o t_stub.o lib/compat.o lib/snprintf.o
 trimslash$(EXEEXT): $(TRIMSLASH_OBJ)
 	$(CC) $(CFLAGS) $(LDFLAGS) -o $@ $(TRIMSLASH_OBJ) $(LIBS)
 
-T_UNSAFE_OBJ = t_unsafe.o syscall.o util.o util2.o t_stub.o lib/compat.o lib/snprintf.o lib/wildmatch.o
+T_UNSAFE_OBJ = t_unsafe.o syscall.o util1.o util2.o t_stub.o lib/compat.o lib/snprintf.o lib/wildmatch.o
 t_unsafe$(EXEEXT): $(T_UNSAFE_OBJ)
 	$(CC) $(CFLAGS) $(LDFLAGS) -o $@ $(T_UNSAFE_OBJ) $(LIBS)
 
@@ -159,11 +180,15 @@ t_unsafe$(EXEEXT): $(T_UNSAFE_OBJ)
 conf: configure.sh config.h.in
 
 .PHONY: gen
-gen: conf proto.h man
+gen: conf proto.h man git-version.h
 
 .PHONY: gensend
 gensend: gen
-	rsync -aic $(GENFILES) $${SAMBA_HOST-samba.org}:/home/ftp/pub/rsync/generated-files/
+	if ! diff git-version.h $(srcdir)/gists/rsync-git-version.h >/dev/null; then \
+	    ./rsync -ai git-version.h $(srcdir)/gists/rsync-git-version.h && \
+	    (cd $(srcdir)/gists && git commit --allow-empty-message -m '' rsync-git-version.h && git push) ; \
+	fi
+	rsync -aic $(GENFILES) git-version.h $${SAMBA_HOST-samba.org}:/home/ftp/pub/rsync/generated-files/ || true
 
 aclocal.m4: $(srcdir)/m4/*.m4
 	aclocal -I $(srcdir)/m4
@@ -185,7 +210,7 @@ configure.sh config.h.in: configure.ac aclocal.m4
 	else \
 	    echo "config.h.in has CHANGED."; \
 	fi
-	@if test -f configure.sh.old -o -f config.h.in.old; then \
+	@if test -f configure.sh.old || test -f config.h.in.old; then \
 	    if test "$(MAKECMDGOALS)" = reconfigure; then \
 		echo 'Continuing with "make reconfigure".'; \
 	    else \
@@ -228,25 +253,29 @@ proto: proto.h-tstamp
 proto.h: proto.h-tstamp
 	@if test -f proto.h; then :; else cp -p $(srcdir)/proto.h .; fi
 
-proto.h-tstamp: $(srcdir)/*.c $(srcdir)/lib/compat.c config.h
-	$(AWK) -f $(srcdir)/mkproto.awk $(srcdir)/*.c $(srcdir)/lib/compat.c
+proto.h-tstamp: $(srcdir)/*.c $(srcdir)/lib/compat.c daemon-parm.h
+	$(AWK) -f $(srcdir)/mkproto.awk $(srcdir)/*.c $(srcdir)/lib/compat.c daemon-parm.h
 
 .PHONY: man
-man: rsync.1 rsync-ssl.1 rsyncd.conf.5
+man: rsync.1 rsync-ssl.1 rsyncd.conf.5 @MAKE_RRSYNC_1@
 
-rsync.1: rsync.1.md md2man version.h Makefile
-	@$(srcdir)/maybe-make-man $(srcdir) rsync.1.md
+rsync.1: rsync.1.md md-convert version.h Makefile
+	@$(srcdir)/maybe-make-man rsync.1.md
 
-rsync-ssl.1: rsync-ssl.1.md md2man version.h Makefile
-	@$(srcdir)/maybe-make-man $(srcdir) rsync-ssl.1.md
+rsync-ssl.1: rsync-ssl.1.md md-convert version.h Makefile
+	@$(srcdir)/maybe-make-man rsync-ssl.1.md
 
-rsyncd.conf.5: rsyncd.conf.5.md md2man version.h Makefile
-	@$(srcdir)/maybe-make-man $(srcdir) rsyncd.conf.5.md
+rsyncd.conf.5: rsyncd.conf.5.md md-convert version.h Makefile
+	@$(srcdir)/maybe-make-man rsyncd.conf.5.md
+
+rrsync.1: support/rrsync.1.md md-convert Makefile
+	@$(srcdir)/maybe-make-man support/rrsync.1.md
 
 .PHONY: clean
 clean: cleantests
-	rm -f *~ $(OBJS) $(CHECK_PROGS) $(CHECK_OBJS) $(CHECK_SYMLINKS) \
-		rounding rounding.h *.old rsync*.1 rsync*.5 rsync*.html
+	rm -f *~ $(OBJS) $(CHECK_PROGS) $(CHECK_OBJS) $(CHECK_SYMLINKS) @MAKE_RRSYNC@ \
+		git-version.h rounding rounding.h *.old rsync*.1 rsync*.5 @MAKE_RRSYNC_1@ \
+		*.html daemon-parm.h help-*.h default-*.h proto.h proto.h-tstamp
 
 .PHONY: cleantests
 cleantests:
@@ -257,16 +286,11 @@ cleantests:
 # the source directory.
 .PHONY: distclean
 distclean: clean
-	rm -f Makefile config.h config.status
-	rm -f stunnel-rsyncd.conf
-	rm -f lib/dummy popt/dummy zlib/dummy
-	rm -f $(srcdir)/Makefile $(srcdir)/config.h $(srcdir)/config.status
-	rm -f $(srcdir)/lib/dummy $(srcdir)/popt/dummy $(srcdir)/zlib/dummy
-	rm -f config.cache config.log
-	rm -f $(srcdir)/config.cache $(srcdir)/config.log
-	rm -f shconfig $(srcdir)/shconfig
-	rm -f $(GENFILES)
-	rm -rf autom4te.cache
+	for dir in $(srcdir) . ; do \
+	    (cd "$$dir" && rm -rf Makefile config.h config.status stunnel-rsyncd.conf \
+	     lib/dummy popt/dummy zlib/dummy config.cache config.log shconfig \
+	     $(GENFILES) autom4te.cache) ; \
+	done
 
 # this target is really just for my use. It only works on a limited
 # range of machines and is used to produce a list of potentially
@@ -276,6 +300,7 @@ finddead:
 	nm *.o */*.o |grep 'U ' | awk '{print $$2}' | sort -u > nmused.txt
 	nm *.o */*.o |grep 'T ' | awk '{print $$3}' | sort -u > nmfns.txt
 	comm -13 nmused.txt nmfns.txt
+	@rm nmused.txt nmfns.txt
 
 # 'check' is the GNU name, 'test' is the name for everybody else :-)
 .PHONY: test

README.md

@@ -23,8 +23,18 @@ options.  To get a complete list of supported options type:
 
     rsync --help
 
-See the manpage for more detailed information.
+See the [manpage][0] for more detailed information.
 
+[0]: https://download.samba.org/pub/rsync/rsync.1
+
+BUILDING AND INSTALLING
+-----------------------
+
+If you need to build rsync yourself, check out the [INSTALL][1] page for
+information on what libraries and packages you can use to get the maximum
+features in your build.
+
+[1]: https://github.com/WayneD/rsync/blob/master/INSTALL.md
 
 SETUP
 -----
@@ -55,17 +65,17 @@ RSYNC DAEMONS
 -------------
 
 Rsync can also talk to "rsync daemons" which can provide anonymous or
-authenticated rsync.  See the rsyncd.conf(5) man page for details on how
-to setup an rsync daemon.  See the rsync(1) man page for info on how to
+authenticated rsync.  See the rsyncd.conf(5) manpage for details on how
+to setup an rsync daemon.  See the rsync(1) manpage for info on how to
 connect to an rsync daemon.
 
 
 WEB SITE
 --------
 
-The main rsync web site is here:
+For more information, visit the [main rsync web site][2].
 
->  https://rsync.samba.org/
+[2]: https://rsync.samba.org/
 
 You'll find a FAQ list, downloads, resources, HTML versions of the
 manpages, etc.
@@ -77,25 +87,25 @@ MAILING LISTS
 There is a mailing list for the discussion of rsync and its applications
 that is open to anyone to join.  New releases are announced on this
 list, and there is also an announcement-only mailing list for those that
-want official announcements.  See the mailing-list page for full
-details:
+want official announcements.  See the [mailing-list page][3] for full
+details.
 
->  https://rsync.samba.org/lists.html
+[3]: https://rsync.samba.org/lists.html
 
 
 BUG REPORTS
 -----------
 
-To visit this web page for full the details on bug reporting:
+The [bug-tracking web page][4] has full details on bug reporting.
 
->  https://rsync.samba.org/bugtracking.html
+[4]: https://rsync.samba.org/bug-tracking.html
 
-That page contains links to the current bug list, and information on how
-to report a bug well.  You might also like to try searching the Internet
-for the error message you've received, or looking in the mailing list
-archives at:
+That page contains links to the current bug list, and information on how to
+do a good job when reporting a bug.  You might also like to try searching
+the Internet for the error message you've received, or looking in the
+[mailing list archives][5].
 
->  https://mail-archive.com/rsync@lists.samba.org/
+[5]: https://mail-archive.com/rsync@lists.samba.org/
 
 To send a bug report, follow the instructions on the bug-tracking
 page of the web site.
@@ -108,18 +118,15 @@ GIT REPOSITORY
 
 If you want to get the very latest version of rsync direct from the
 source code repository, then you will need to use git.  The git repo
-is hosted on github and on samba's site.  Feel free to access it here:
+is hosted [on GitHub][6] and [on Samba's site][7].
 
->  https://github.com/WayneD/rsync
+[6]: https://github.com/WayneD/rsync
+[7]: https://git.samba.org/?p=rsync.git;a=summary
 
-A backup git repo is available on the samba site:
+See [the download page][8] for full details on all the ways to grab the
+source.
 
->     git clone git://git.samba.org/rsync.git
-
-See the download page for full details on all the ways to grab the
-source:
-
->  https://rsync.samba.org/download.html
+[8]: https://rsync.samba.org/download.html
 
 
 COPYRIGHT
@@ -130,13 +137,8 @@ maintained by Wayne Davison.  It has been improved by many developers
 from around the world.
 
 Rsync may be used, modified and redistributed only under the terms of
-the GNU General Public License, found in the file COPYING in this
-distribution, or at:
+the GNU General Public License, found in the file [COPYING][9] in this
+distribution, or at [the Free Software Foundation][10].
 
->  https://www.fsf.org/licenses/gpl.html
-
-
-AVAILABILITY
-------------
-
-The main web site for rsync is https://rsync.samba.org/
+[9]: https://github.com/WayneD/rsync/blob/master/COPYING
+[10]: https://www.fsf.org/licenses/gpl.html

SECURITY.md

@@ -0,0 +1,12 @@
+# Security Policy
+
+## Supported Versions
+
+Only the current release of the software is actively supported.  If you need
+help backporting fixes into an older release, feel free to ask.
+
+## Reporting a Vulnerability
+
+Email your vulnerability information to rsync's maintainer:
+
+  Wayne Davison <wayne@opencoder.net>

access.c

@@ -2,7 +2,7 @@
  * Routines to authenticate access to a daemon (hosts allow/deny).
  *
  * Copyright (C) 1998 Andrew Tridgell
- * Copyright (C) 2004-2020 Wayne Davison
+ * Copyright (C) 2004-2022 Wayne Davison
  *
  * This program is free software; you can redistribute it and/or modify
  * it under the terms of the GNU General Public License as published by
@@ -20,6 +20,9 @@
 
 #include "rsync.h"
 #include "ifuncs.h"
+#ifdef HAVE_NETGROUP_H
+#include <netgroup.h>
+#endif
 
 static int allow_forward_dns;
 
@@ -34,6 +37,11 @@ static int match_hostname(const char **host_ptr, const char *addr, const char *t
 	if (!host || !*host)
 		return 0;
 
+#ifdef HAVE_INNETGR
+	if (*tok == '@' && tok[1])
+		return innetgr(tok + 1, host, NULL, NULL);
+#endif
+
 	/* First check if the reverse-DNS-determined hostname matches. */
 	if (iwildmatch(tok, host))
 		return 1;

acls.c

@@ -3,7 +3,7 @@
  *
  * Copyright (C) 1996 Andrew Tridgell
  * Copyright (C) 1996 Paul Mackerras
- * Copyright (C) 2006-2020 Wayne Davison
+ * Copyright (C) 2006-2022 Wayne Davison
  *
  * This program is free software; you can redistribute it and/or modify
  * it under the terms of the GNU General Public License as published by
@@ -763,6 +763,7 @@ static int recv_rsync_acl(int f, item_list *racl_list, SMB_ACL_TYPE_T type, mode
 #ifdef HAVE_OSX_ACLS
 	/* If we received a superfluous mask, throw it away. */
 	duo_item->racl.mask_obj = NO_ENTRY;
+	(void)mode;
 #else
 	if (duo_item->racl.names.count && duo_item->racl.mask_obj == NO_ENTRY) {
 		/* Mask must be non-empty with lists. */

authenticate.c

@@ -119,7 +119,7 @@ static const char *check_secret(int module, const char *user, const char *group,
 		if ((st.st_mode & 06) != 0) {
 			rprintf(FLOG, "secrets file must not be other-accessible (see strict modes option)\n");
 			ok = 0;
-		} else if (MY_UID() == 0 && st.st_uid != 0) {
+		} else if (MY_UID() == ROOT_UID && st.st_uid != ROOT_UID) {
 			rprintf(FLOG, "secrets file must be owned by root when running as root (see strict modes)\n");
 			ok = 0;
 		}
@@ -196,7 +196,7 @@ static const char *getpassf(const char *filename)
 			rprintf(FERROR, "ERROR: password file must not be other-accessible\n");
 			exit_cleanup(RERR_SYNTAX);
 		}
-		if (MY_UID() == 0 && st.st_uid != 0) {
+		if (MY_UID() == ROOT_UID && st.st_uid != ROOT_UID) {
 			rprintf(FERROR, "ERROR: password file must be owned by root when running as root\n");
 			exit_cleanup(RERR_SYNTAX);
 		}
@@ -227,7 +227,7 @@ char *auth_server(int f_in, int f_out, int module, const char *host,
 	char *users = lp_auth_users(module);
 	char challenge[MAX_DIGEST_LEN*2];
 	char line[BIGPATHBUFLEN];
-	char **auth_uid_groups = NULL;
+	const char **auth_uid_groups = NULL;
 	int auth_uid_groups_cnt = -1;
 	const char *err = NULL;
 	int group_match = -1;
@@ -287,7 +287,7 @@ char *auth_server(int f_in, int f_out, int module, const char *host,
 				else {
 					gid_t *gid_array = gid_list.items;
 					auth_uid_groups_cnt = gid_list.count;
-					auth_uid_groups = new_array(char *, auth_uid_groups_cnt);
+					auth_uid_groups = new_array(const char *, auth_uid_groups_cnt);
 					for (j = 0; j < auth_uid_groups_cnt; j++)
 						auth_uid_groups[j] = gid_to_group(gid_array[j]);
 				}
@@ -313,7 +313,7 @@ char *auth_server(int f_in, int f_out, int module, const char *host,
 	else if (opt_ch == 'd')
 		err = "denied by rule";
 	else {
-		char *group = group_match >= 0 ? auth_uid_groups[group_match] : NULL;
+		const char *group = group_match >= 0 ? auth_uid_groups[group_match] : NULL;
 		err = check_secret(module, line, group, challenge, pass);
 	}
 
@@ -324,7 +324,7 @@ char *auth_server(int f_in, int f_out, int module, const char *host,
 		int j;
 		for (j = 0; j < auth_uid_groups_cnt; j++) {
 			if (auth_uid_groups[j])
-				free(auth_uid_groups[j]);
+				free((char*)auth_uid_groups[j]);
 		}
 		free(auth_uid_groups);
 	}

backup.c

@@ -2,7 +2,7 @@
  * Backup handling code.
  *
  * Copyright (C) 1999 Andrew Tridgell
- * Copyright (C) 2003-2020 Wayne Davison
+ * Copyright (C) 2003-2022 Wayne Davison
  *
  * This program is free software; you can redistribute it and/or modify
  * it under the terms of the GNU General Public License as published by
@@ -304,7 +304,8 @@ int make_backup(const char *fname, BOOL prefer_rename)
 #endif
 
 	if (!ret && !S_ISREG(file->mode)) {
-		rprintf(FINFO, "make_bak: skipping non-regular file %s\n", fname);
+		if (INFO_GTE(NONREG, 1))
+			rprintf(FINFO, "make_bak: skipping non-regular file %s\n", fname);
 		unmake_file(file);
 #ifdef SUPPORT_ACLS
 		uncache_tmp_acls();

checksum.c

@@ -3,7 +3,7 @@
  *
  * Copyright (C) 1996 Andrew Tridgell
  * Copyright (C) 1996 Paul Mackerras
- * Copyright (C) 2004-2020 Wayne Davison
+ * Copyright (C) 2004-2022 Wayne Davison
  *
  * This program is free software; you can redistribute it and/or modify
  * it under the terms of the GNU General Public License as published by
@@ -29,7 +29,7 @@
 #include "rsync.h"
 
 #ifdef SUPPORT_XXHASH
-#include "xxhash.h"
+#include <xxhash.h>
 # if XXH_VERSION_NUMBER >= 800
 #  define SUPPORT_XXH3 1
 # endif
@@ -179,7 +179,7 @@ int canonical_checksum(int csum_type)
 	return 0;
 }
 
-#ifndef HAVE_SIMD /* See simd-checksum-*.cpp. */
+#ifndef USE_ROLL_SIMD /* See simd-checksum-*.cpp. */
 /*
   a simple 32 bit checksum that can be updated from either end
   (inspired by Mark Adler's Adler-32 checksum)
@@ -222,23 +222,23 @@ void get_checksum2(char *buf, int32 len, char *sum)
 	  }
 #endif
 	  case CSUM_MD5: {
-		MD5_CTX m5;
+		md5_context m5;
 		uchar seedbuf[4];
-		MD5_Init(&m5);
+		md5_begin(&m5);
 		if (proper_seed_order) {
 			if (checksum_seed) {
 				SIVALu(seedbuf, 0, checksum_seed);
-				MD5_Update(&m5, seedbuf, 4);
+				md5_update(&m5, seedbuf, 4);
 			}
-			MD5_Update(&m5, (uchar *)buf, len);
+			md5_update(&m5, (uchar *)buf, len);
 		} else {
-			MD5_Update(&m5, (uchar *)buf, len);
+			md5_update(&m5, (uchar *)buf, len);
 			if (checksum_seed) {
 				SIVALu(seedbuf, 0, checksum_seed);
-				MD5_Update(&m5, seedbuf, 4);
+				md5_update(&m5, seedbuf, 4);
 			}
 		}
-		MD5_Final((uchar *)sum, &m5);
+		md5_result(&m5, (uchar *)sum);
 		break;
 	  }
 	  case CSUM_MD4:
@@ -374,18 +374,18 @@ void file_checksum(const char *fname, const STRUCT_STAT *st_p, char *sum)
 	  }
 #endif
 	  case CSUM_MD5: {
-		MD5_CTX m5;
+		md5_context m5;
 
-		MD5_Init(&m5);
+		md5_begin(&m5);
 
 		for (i = 0; i + CHUNK_SIZE <= len; i += CHUNK_SIZE)
-			MD5_Update(&m5, (uchar *)map_ptr(buf, i, CHUNK_SIZE), CHUNK_SIZE);
+			md5_update(&m5, (uchar *)map_ptr(buf, i, CHUNK_SIZE), CHUNK_SIZE);
 
 		remainder = (int32)(len - i);
 		if (remainder > 0)
-			MD5_Update(&m5, (uchar *)map_ptr(buf, i, remainder), remainder);
+			md5_update(&m5, (uchar *)map_ptr(buf, i, remainder), remainder);
 
-		MD5_Final((uchar *)sum, &m5);
+		md5_result(&m5, (uchar *)sum);
 		break;
 	  }
 	  case CSUM_MD4:
@@ -443,7 +443,7 @@ static union {
 #ifdef USE_OPENSSL
 	MD4_CTX m4;
 #endif
-	MD5_CTX m5;
+	md5_context m5;
 } ctx;
 #ifdef SUPPORT_XXHASH
 static XXH64_state_t* xxh64_state;
@@ -482,7 +482,7 @@ void sum_init(int csum_type, int seed)
 		break;
 #endif
 	  case CSUM_MD5:
-		MD5_Init(&ctx.m5);
+		md5_begin(&ctx.m5);
 		break;
 	  case CSUM_MD4:
 #ifdef USE_OPENSSL
@@ -532,7 +532,7 @@ void sum_update(const char *p, int32 len)
 		break;
 #endif
 	  case CSUM_MD5:
-		MD5_Update(&ctx.m5, (uchar *)p, len);
+		md5_update(&ctx.m5, (uchar *)p, len);
 		break;
 	  case CSUM_MD4:
 #ifdef USE_OPENSSL
@@ -574,7 +574,7 @@ void sum_update(const char *p, int32 len)
 }
 
 /* NOTE: all the callers of sum_end() pass in a pointer to a buffer that is
- * MAX_DIGEST_LEN in size, so even if the csum-len is shorter that that (i.e.
+ * MAX_DIGEST_LEN in size, so even if the csum-len is shorter than that (i.e.
  * CSUM_MD4_ARCHAIC), we don't have to worry about limiting the data we write
  * into the "sum" buffer. */
 int sum_end(char *sum)
@@ -597,7 +597,7 @@ int sum_end(char *sum)
 	  }
 #endif
 	  case CSUM_MD5:
-		MD5_Final((uchar *)sum, &ctx.m5);
+		md5_result(&ctx.m5, (uchar *)sum);
 		break;
 	  case CSUM_MD4:
 #ifdef USE_OPENSSL

cleanup.c

@@ -137,7 +137,7 @@ NORETURN void _exit_cleanup(int code, const char *file, int line)
 		if (DEBUG_GTE(EXIT, 2)) {
 			rprintf(FINFO,
 				"[%s] _exit_cleanup(code=%d, file=%s, line=%d): entered\n",
-				who_am_i(), code, file, line);
+				who_am_i(), code, src_file(file), line);
 		}
 
 #include "case_N.h"
@@ -222,10 +222,6 @@ NORETURN void _exit_cleanup(int code, const char *file, int line)
 		 * we don't want to output a duplicate error. */
 		if ((exit_code && line > 0)
 		 || am_daemon || (logfile_name && (am_server || !INFO_GTE(STATS, 1)))) {
-#ifdef HAVE_USLEEP /* A tiny delay just in case both sender & receiver are sending a msg at the same time. */
-			if (am_server && exit_code)
-				usleep(50);
-#endif
 			log_exit(exit_code, exit_file, exit_line);
 		}
 

clientname.c

@@ -3,7 +3,7 @@
  *
  * Copyright (C) 1992-2001 Andrew Tridgell <tridge@samba.org>
  * Copyright (C) 2001, 2002 Martin Pool <mbp@samba.org>
- * Copyright (C) 2002-2020 Wayne Davison
+ * Copyright (C) 2002-2022 Wayne Davison
  *
  * This program is free software; you can redistribute it and/or modify
  * it under the terms of the GNU General Public License as published by
@@ -49,7 +49,7 @@ static char ipaddr_buf[100];
 
 static void client_sockaddr(int fd, struct sockaddr_storage *ss, socklen_t *ss_len);
 static int check_name(const char *ipaddr, const struct sockaddr_storage *ss, char *name_buf, size_t name_buf_size);
-static int valid_ipaddr(const char *s);
+static int valid_ipaddr(const char *s, int allow_scope);
 
 /* Return the IP addr of the client as a string. */
 char *client_addr(int fd)
@@ -73,7 +73,7 @@ char *client_addr(int fd)
 			if ((p = strchr(ipaddr_buf, ' ')) != NULL)
 				*p = '\0';
 		}
-		if (valid_ipaddr(ipaddr_buf))
+		if (valid_ipaddr(ipaddr_buf, True))
 			return ipaddr_buf;
 	}
 
@@ -139,7 +139,7 @@ char *client_name(const char *ipaddr)
 		break;
 #endif
 	default:
-		assert(0);
+		NOISY_DEATH("Unknown ai_family value");
 	}
 	freeaddrinfo(answer);
 
@@ -156,7 +156,7 @@ char *client_name(const char *ipaddr)
 }
 
 
-/* Try to read an proxy protocol header (V1 or V2). Returns 1 on success or 0 on failure. */
+/* Try to read a proxy protocol header (V1 or V2). Returns 1 on success or 0 on failure. */
 int read_proxy_protocol_header(int fd)
 {
 	union {
@@ -213,12 +213,14 @@ int read_proxy_protocol_header(int fd)
 				if (size != sizeof hdr.v2.addr.ip4)
 					return 0;
 				inet_ntop(AF_INET, hdr.v2.addr.ip4.src_addr, ipaddr_buf, sizeof ipaddr_buf);
-				return valid_ipaddr(ipaddr_buf);
+				return valid_ipaddr(ipaddr_buf, False);
+#ifdef INET6
 			case PROXY_FAM_TCPv6:
 				if (size != sizeof hdr.v2.addr.ip6)
 					return 0;
 				inet_ntop(AF_INET6, hdr.v2.addr.ip6.src_addr, ipaddr_buf, sizeof ipaddr_buf);
-				return valid_ipaddr(ipaddr_buf);
+				return valid_ipaddr(ipaddr_buf, False);
+#endif
 			default:
 				break;
 			}
@@ -274,7 +276,7 @@ int read_proxy_protocol_header(int fd)
 		if ((sp = strchr(p, ' ')) == NULL)
 			return 0;
 		*sp = '\0';
-		if (!valid_ipaddr(p))
+		if (!valid_ipaddr(p, False))
 			return 0;
 		strlcpy(ipaddr_buf, p, sizeof ipaddr_buf); /* It will always fit when valid. */
 
@@ -282,7 +284,7 @@ int read_proxy_protocol_header(int fd)
 		if ((sp = strchr(p, ' ')) == NULL)
 			return 0;
 		*sp = '\0';
-		if (!valid_ipaddr(p))
+		if (!valid_ipaddr(p, False))
 			return 0;
 		/* Ignore destination address. */
 
@@ -464,7 +466,7 @@ static int check_name(const char *ipaddr, const struct sockaddr_storage *ss, cha
 }
 
 /* Returns 1 for a valid IPv4 or IPv6 addr, or 0 for a bad one. */
-static int valid_ipaddr(const char *s)
+static int valid_ipaddr(const char *s, int allow_scope)
 {
 	int i;
 
@@ -482,6 +484,11 @@ static int valid_ipaddr(const char *s)
 		for (count = 0; count < 8; count++) {
 			if (!*s)
 				return saw_double_colon;
+			if (allow_scope && *s == '%') {
+				if (saw_double_colon)
+					break;
+				return 0;
+			}
 
 			if (strchr(s, ':') == NULL && strchr(s, '.') != NULL) {
 				if ((!saw_double_colon && count != 6) || (saw_double_colon && count > 6))
@@ -507,8 +514,11 @@ static int valid_ipaddr(const char *s)
 			}
 		}
 
-		if (!ipv4_at_end)
-			return !*s;
+		if (!ipv4_at_end) {
+			if (allow_scope && *s == '%')
+				for (s++; isAlNum(s); s++) { }
+			return !*s && s[-1] != '%';
+		}
 	}
 
 	/* IPv4 */

clientserver.c

@@ -3,7 +3,7 @@
  *
  * Copyright (C) 1998-2001 Andrew Tridgell <tridge@samba.org>
  * Copyright (C) 2001-2002 Martin Pool <mbp@samba.org>
- * Copyright (C) 2002-2020 Wayne Davison
+ * Copyright (C) 2002-2022 Wayne Davison
  *
  * This program is free software; you can redistribute it and/or modify
  * it under the terms of the GNU General Public License as published by
@@ -21,6 +21,7 @@
 
 #include "rsync.h"
 #include "itypes.h"
+#include "ifuncs.h"
 
 extern int quiet;
 extern int dry_run;
@@ -36,8 +37,8 @@ extern int protect_args;
 extern int ignore_errors;
 extern int preserve_xattrs;
 extern int kluge_around_eof;
-extern int daemon_over_rsh;
 extern int munge_symlinks;
+extern int open_noatime;
 extern int sanitize_paths;
 extern int numeric_ids;
 extern int filesfrom_fd;
@@ -46,6 +47,7 @@ extern int protocol_version;
 extern int io_timeout;
 extern int no_detach;
 extern int write_batch;
+extern int old_style_args;
 extern int default_af_hint;
 extern int logfile_format_has_i;
 extern int logfile_format_has_o_or_i;
@@ -70,6 +72,7 @@ int module_id = -1;
 int pid_file_fd = -1;
 int early_input_len = 0;
 char *early_input = NULL;
+pid_t namecvt_pid = 0;
 struct chmod_mode_struct *daemon_chmod_modes;
 
 #define EARLY_INPUT_CMD "#early_input="
@@ -84,6 +87,7 @@ unsigned int module_dirlen = 0;
 char *full_module_path;
 
 static int rl_nulls = 0;
+static int namecvt_fd_req = -1, namecvt_fd_ans = -1;
 
 #ifdef HAVE_SIGACTION
 static struct sigaction sigact;
@@ -278,10 +282,6 @@ int start_inband_exchange(int f_in, int f_out, const char *user, int argc, char
 		fclose(f);
 	}
 
-	/* set daemon_over_rsh to false since we need to build the
-	 * true set of args passed through the rsh/ssh connection;
-	 * this is a no-op for direct-socket-connection mode */
-	daemon_over_rsh = 0;
 	server_options(sargs, &sargc);
 
 	if (sargc >= MAX_ARGS - 2)
@@ -289,20 +289,45 @@ int start_inband_exchange(int f_in, int f_out, const char *user, int argc, char
 
 	sargs[sargc++] = ".";
 
+	if (!old_style_args)
+		snprintf(line, sizeof line, " %.*s/", modlen, modname);
+
 	while (argc > 0) {
 		if (sargc >= MAX_ARGS - 1) {
 		  arg_overflow:
 			rprintf(FERROR, "internal: args[] overflowed in do_cmd()\n");
 			exit_cleanup(RERR_SYNTAX);
 		}
-		if (strncmp(*argv, modname, modlen) == 0
-		 && argv[0][modlen] == '\0')
+		if (strncmp(*argv, modname, modlen) == 0 && argv[0][modlen] == '\0')
 			sargs[sargc++] = modname; /* we send "modname/" */
-		else if (**argv == '-') {
-			if (asprintf(sargs + sargc++, "./%s", *argv) < 0)
-				out_of_memory("start_inband_exchange");
-		} else
-			sargs[sargc++] = *argv;
+		else {
+			char *arg = *argv;
+			int extra_chars = *arg == '-' ? 2 : 0; /* a leading dash needs a "./" prefix. */
+			/* If --old-args was not specified, make sure that the arg won't split at a mod name! */
+			if (!old_style_args && (p = strstr(arg, line)) != NULL) {
+				do {
+					extra_chars += 2;
+				} while ((p = strstr(p+1, line)) != NULL);
+			}
+			if (extra_chars) {
+				char *f = arg;
+				char *t = arg = new_array(char, strlen(arg) + extra_chars + 1);
+				if (*f == '-') {
+					*t++ = '.';
+					*t++ = '/';
+				}
+				while (*f) {
+					if (*f == ' ' && strncmp(f, line, modlen+2) == 0) {
+						*t++ = '[';
+						*t++ = *f++;
+						*t++ = ']';
+					} else
+						*t++ = *f++;
+				}
+				*t = '\0';
+			}
+			sargs[sargc++] = arg;
+		}
 		argv++;
 		argc--;
 	}
@@ -381,7 +406,7 @@ int start_inband_exchange(int f_in, int f_out, const char *user, int argc, char
 	return 0;
 }
 
-#ifdef HAVE_PUTENV
+#if defined HAVE_SETENV || defined HAVE_PUTENV
 static int read_arg_from_pipe(int fd, char *buf, int limit)
 {
 	char *bp = buf, *eob = buf + limit - 1;
@@ -406,25 +431,59 @@ static int read_arg_from_pipe(int fd, char *buf, int limit)
 
 static void set_env_str(const char *var, const char *str)
 {
+#ifdef HAVE_SETENV
+	if (setenv(var, str, 1) < 0)
+		out_of_memory("set_env_str");
+#else
 #ifdef HAVE_PUTENV
 	char *mem;
 	if (asprintf(&mem, "%s=%s", var, str) < 0)
 		out_of_memory("set_env_str");
 	putenv(mem);
+#else
+	(void)var;
+	(void)str;
+#endif
 #endif
 }
 
+#if defined HAVE_SETENV || defined HAVE_PUTENV
+
+static void set_envN_str(const char *var, int num, const char *str)
+{
+#ifdef HAVE_SETENV
+	char buf[128];
+	(void)snprintf(buf, sizeof buf, "%s%d", var, num);
+	if (setenv(buf, str, 1) < 0)
+		out_of_memory("set_env_str");
+#else
 #ifdef HAVE_PUTENV
+	char *mem;
+	if (asprintf(&mem, "%s%d=%s", var, num, str) < 0)
+		out_of_memory("set_envN_str");
+	putenv(mem);
+#endif
+#endif
+}
+
 void set_env_num(const char *var, long num)
 {
+#ifdef HAVE_SETENV
+	char val[64];
+	(void)snprintf(val, sizeof val, "%ld", num);
+	if (setenv(var, val, 1) < 0)
+		out_of_memory("set_env_str");
+#else
+#ifdef HAVE_PUTENV
 	char *mem;
 	if (asprintf(&mem, "%s=%ld", var, num) < 0)
 		out_of_memory("set_env_num");
 	putenv(mem);
-}
 #endif
+#endif
+}
 
-/* Used for both early exec & pre-xfer exec */
+/* Used for "early exec", "pre-xfer exec", and the "name converter" script. */
 static pid_t start_pre_exec(const char *cmd, int *arg_fd_ptr, int *error_fd_ptr)
 {
 	int arg_fds[2], error_fds[2], arg_fd;
@@ -452,15 +511,13 @@ static pid_t start_pre_exec(const char *cmd, int *arg_fd_ptr, int *error_fd_ptr)
 		set_env_str("RSYNC_REQUEST", buf);
 
 		for (j = 0; ; j++) {
-			char *p;
 			len = read_arg_from_pipe(arg_fd, buf, BIGPATHBUFLEN);
 			if (len <= 0) {
 				if (!len)
 					break;
 				_exit(1);
 			}
-			if (asprintf(&p, "RSYNC_ARG%d=%s", j, buf) >= 0)
-				putenv(p);
+			set_envN_str("RSYNC_ARG", j, buf);
 		}
 
 		dup2(arg_fd, STDIN_FILENO);
@@ -491,7 +548,9 @@ static pid_t start_pre_exec(const char *cmd, int *arg_fd_ptr, int *error_fd_ptr)
 	return pid;
 }
 
-static void write_pre_exec_args(int write_fd, char *request, char **early_argv, char **argv, int am_early)
+#endif
+
+static void write_pre_exec_args(int write_fd, char *request, char **early_argv, char **argv, int exec_type)
 {
 	int j = 0;
 
@@ -510,10 +569,11 @@ static void write_pre_exec_args(int write_fd, char *request, char **early_argv,
 	}
 	write_byte(write_fd, 0);
 
-	if (am_early && early_input_len)
+	if (exec_type == 1 && early_input_len)
 		write_buf(write_fd, early_input, early_input_len);
 
-	close(write_fd);
+	if (exec_type != 2) /* the name converter needs this left open */
+		close(write_fd);
 }
 
 static char *finish_pre_exec(const char *desc, pid_t pid, int read_fd)
@@ -695,17 +755,17 @@ static int rsync_module(int f_in, int f_out, int i, const char *addr, const char
 
 	module_id = i;
 
-	if (lp_transfer_logging(i) && !logfile_format)
-		logfile_format = lp_log_format(i);
+	if (lp_transfer_logging(module_id) && !logfile_format)
+		logfile_format = lp_log_format(module_id);
 	if (log_format_has(logfile_format, 'i'))
 		logfile_format_has_i = 1;
 	if (logfile_format_has_i || log_format_has(logfile_format, 'o'))
 		logfile_format_has_o_or_i = 1;
 
 	uid = MY_UID();
-	am_root = (uid == 0);
+	am_root = (uid == ROOT_UID);
 
-	p = *lp_uid(i) ? lp_uid(i) : am_root ? NOBODY_USER : NULL;
+	p = *lp_uid(module_id) ? lp_uid(module_id) : am_root ? NOBODY_USER : NULL;
 	if (p) {
 		if (!user_to_uid(p, &uid, True)) {
 			rprintf(FLOG, "Invalid uid %s\n", p);
@@ -716,7 +776,7 @@ static int rsync_module(int f_in, int f_out, int i, const char *addr, const char
 	} else
 		set_uid = 0;
 
-	p = *lp_gid(i) ? conf_strtok(lp_gid(i)) : NULL;
+	p = *lp_gid(module_id) ? conf_strtok(lp_gid(module_id)) : NULL;
 	if (p) {
 		/* The "*" gid must be the first item in the list. */
 		if (strcmp(p, "*") == 0) {
@@ -749,7 +809,7 @@ static int rsync_module(int f_in, int f_out, int i, const char *addr, const char
 			return -1;
 	}
 
-	module_dir = lp_path(i);
+	module_dir = lp_path(module_id);
 	if (*module_dir == '\0') {
 		rprintf(FLOG, "No path specified for module %s\n", name);
 		io_printf(f_out, "@ERROR: no path setting.\n");
@@ -786,38 +846,39 @@ static int rsync_module(int f_in, int f_out, int i, const char *addr, const char
 	} else
 		set_filter_dir(module_dir, module_dirlen);
 
-	p = lp_filter(i);
+	p = lp_filter(module_id);
 	parse_filter_str(&daemon_filter_list, p, rule_template(FILTRULE_WORD_SPLIT),
 		XFLG_ABS_IF_SLASH | XFLG_DIR2WILD3);
 
-	p = lp_include_from(i);
+	p = lp_include_from(module_id);
 	parse_filter_file(&daemon_filter_list, p, rule_template(FILTRULE_INCLUDE),
 		XFLG_ABS_IF_SLASH | XFLG_DIR2WILD3 | XFLG_OLD_PREFIXES | XFLG_FATAL_ERRORS);
 
-	p = lp_include(i);
+	p = lp_include(module_id);
 	parse_filter_str(&daemon_filter_list, p,
 		rule_template(FILTRULE_INCLUDE | FILTRULE_WORD_SPLIT),
 		XFLG_ABS_IF_SLASH | XFLG_DIR2WILD3 | XFLG_OLD_PREFIXES);
 
-	p = lp_exclude_from(i);
+	p = lp_exclude_from(module_id);
 	parse_filter_file(&daemon_filter_list, p, rule_template(0),
 		XFLG_ABS_IF_SLASH | XFLG_DIR2WILD3 | XFLG_OLD_PREFIXES | XFLG_FATAL_ERRORS);
 
-	p = lp_exclude(i);
+	p = lp_exclude(module_id);
 	parse_filter_str(&daemon_filter_list, p, rule_template(FILTRULE_WORD_SPLIT),
 		XFLG_ABS_IF_SLASH | XFLG_DIR2WILD3 | XFLG_OLD_PREFIXES);
 
 	log_init(1);
 
-#ifdef HAVE_PUTENV
-	if ((*lp_early_exec(i) || *lp_prexfer_exec(i) || *lp_postxfer_exec(i))
+#if defined HAVE_SETENV || defined HAVE_PUTENV
+	if ((*lp_early_exec(module_id) || *lp_prexfer_exec(module_id)
+	  || *lp_postxfer_exec(module_id) || *lp_name_converter(module_id))
 	 && !getenv("RSYNC_NO_XFER_EXEC")) {
 		set_env_num("RSYNC_PID", (long)getpid());
 
 		/* For post-xfer exec, fork a new process to run the rsync
 		 * daemon while this process waits for the exit status and
 		 * runs the indicated command at that point. */
-		if (*lp_postxfer_exec(i)) {
+		if (*lp_postxfer_exec(module_id)) {
 			pid_t pid = fork();
 			if (pid < 0) {
 				rsyserr(FLOG, errno, "fork failed");
@@ -837,7 +898,7 @@ static int rsync_module(int f_in, int f_out, int i, const char *addr, const char
 				else
 					status = -1;
 				set_env_num("RSYNC_EXIT_STATUS", status);
-				if (shell_exec(lp_postxfer_exec(i)) < 0)
+				if (shell_exec(lp_postxfer_exec(module_id)) < 0)
 					status = -1;
 				_exit(status);
 			}
@@ -845,9 +906,9 @@ static int rsync_module(int f_in, int f_out, int i, const char *addr, const char
 
 		/* For early exec, fork a child process to run the indicated
 		 * command and wait for it to exit. */
-		if (*lp_early_exec(i)) {
+		if (*lp_early_exec(module_id)) {
 			int arg_fd;
-			pid_t pid = start_pre_exec(lp_early_exec(i), &arg_fd, NULL);
+			pid_t pid = start_pre_exec(lp_early_exec(module_id), &arg_fd, NULL);
 			if (pid == (pid_t)-1) {
 				rsyserr(FLOG, errno, "early exec preparation failed");
 				io_printf(f_out, "@ERROR: early exec preparation failed\n");
@@ -864,14 +925,23 @@ static int rsync_module(int f_in, int f_out, int i, const char *addr, const char
 		/* For pre-xfer exec, fork a child process to run the indicated
 		 * command, though it first waits for the parent process to
 		 * send us the user's request via a pipe. */
-		if (*lp_prexfer_exec(i)) {
-			pre_exec_pid = start_pre_exec(lp_prexfer_exec(i), &pre_exec_arg_fd, &pre_exec_error_fd);
+		if (*lp_prexfer_exec(module_id)) {
+			pre_exec_pid = start_pre_exec(lp_prexfer_exec(module_id), &pre_exec_arg_fd, &pre_exec_error_fd);
 			if (pre_exec_pid == (pid_t)-1) {
 				rsyserr(FLOG, errno, "pre-xfer exec preparation failed");
 				io_printf(f_out, "@ERROR: pre-xfer exec preparation failed\n");
 				return -1;
 			}
 		}
+
+		if (*lp_name_converter(module_id)) {
+			namecvt_pid = start_pre_exec(lp_name_converter(module_id), &namecvt_fd_req, &namecvt_fd_ans);
+			if (namecvt_pid == (pid_t)-1) {
+				rsyserr(FLOG, errno, "name-converter exec preparation failed");
+				io_printf(f_out, "@ERROR: name-converter exec preparation failed\n");
+				return -1;
+			}
+		}
 	}
 #endif
 
@@ -906,7 +976,7 @@ static int rsync_module(int f_in, int f_out, int i, const char *addr, const char
 	if (module_dirlen || (!use_chroot && !*lp_daemon_chroot()))
 		sanitize_paths = 1;
 
-	if ((munge_symlinks = lp_munge_symlinks(i)) < 0)
+	if ((munge_symlinks = lp_munge_symlinks(module_id)) < 0)
 		munge_symlinks = !use_chroot || module_dirlen;
 	if (munge_symlinks) {
 		STRUCT_STAT st;
@@ -958,11 +1028,11 @@ static int rsync_module(int f_in, int f_out, int i, const char *addr, const char
 		}
 
 		our_uid = MY_UID();
-		am_root = (our_uid == 0);
+		am_root = (our_uid == ROOT_UID);
 	}
 
-	if (lp_temp_dir(i) && *lp_temp_dir(i)) {
-		tmpdir = lp_temp_dir(i);
+	if (lp_temp_dir(module_id) && *lp_temp_dir(module_id)) {
+		tmpdir = lp_temp_dir(module_id);
 		if (strlen(tmpdir) >= MAXPATHLEN - 10) {
 			rprintf(FLOG,
 				"the 'temp dir' value for %s is WAY too long -- ignoring.\n",
@@ -989,7 +1059,12 @@ static int rsync_module(int f_in, int f_out, int i, const char *addr, const char
 	} else
 		orig_early_argv = NULL;
 
+	/* The default is to use the user's setting unless the module sets True or False. */
+	if (lp_open_noatime(module_id) >= 0)
+		open_noatime = lp_open_noatime(module_id);
+
 	munge_symlinks = save_munge_symlinks; /* The client mustn't control this. */
+
 	if (am_daemon > 0)
 		msgs2stderr = 0; /* A non-rsh-run daemon doesn't have stderr for msgs. */
 
@@ -998,6 +1073,9 @@ static int rsync_module(int f_in, int f_out, int i, const char *addr, const char
 		err_msg = finish_pre_exec("pre-xfer exec", pre_exec_pid, pre_exec_error_fd);
 	}
 
+	if (namecvt_pid)
+		write_pre_exec_args(namecvt_fd_req, request, orig_early_argv, orig_argv, 2);
+
 	if (orig_early_argv)
 		free(orig_early_argv);
 
@@ -1008,7 +1086,7 @@ static int rsync_module(int f_in, int f_out, int i, const char *addr, const char
 	if (write_batch < 0)
 		dry_run = 1;
 
-	if (lp_fake_super(i)) {
+	if (lp_fake_super(module_id)) {
 		if (preserve_xattrs > 1)
 			preserve_xattrs = 1;
 		am_root = -1;
@@ -1033,7 +1111,7 @@ static int rsync_module(int f_in, int f_out, int i, const char *addr, const char
 
 #ifndef DEBUG
 	/* don't allow the logs to be flooded too fast */
-	limit_output_verbosity(lp_max_verbosity(i));
+	limit_output_verbosity(lp_max_verbosity(module_id));
 #endif
 
 	if (protocol_version < 23 && (protocol_version == 22 || am_sender))
@@ -1094,20 +1172,21 @@ static int rsync_module(int f_in, int f_out, int i, const char *addr, const char
 #endif
 
 	if (!numeric_ids
-	 && (use_chroot ? lp_numeric_ids(i) != False : lp_numeric_ids(i) == True))
+	 && (use_chroot ? lp_numeric_ids(module_id) != False && !*lp_name_converter(module_id)
+		        : lp_numeric_ids(module_id) == True))
 		numeric_ids = -1; /* Set --numeric-ids w/o breaking protocol. */
 
-	if (lp_timeout(i) && (!io_timeout || lp_timeout(i) < io_timeout))
-		set_io_timeout(lp_timeout(i));
+	if (lp_timeout(module_id) && (!io_timeout || lp_timeout(module_id) < io_timeout))
+		set_io_timeout(lp_timeout(module_id));
 
 	/* If we have some incoming/outgoing chmod changes, append them to
 	 * any user-specified changes (making our changes have priority).
 	 * We also get a pointer to just our changes so that a receiver
 	 * process can use them separately if --perms wasn't specified. */
 	if (am_sender)
-		p = lp_outgoing_chmod(i);
+		p = lp_outgoing_chmod(module_id);
 	else
-		p = lp_incoming_chmod(i);
+		p = lp_incoming_chmod(module_id);
 	if (*p && !(daemon_chmod_modes = parse_chmod(p, &chmod_modes))) {
 		rprintf(FLOG, "Invalid \"%sing chmod\" directive: %s\n",
 			am_sender ? "outgo" : "incom", p);
@@ -1118,6 +1197,38 @@ static int rsync_module(int f_in, int f_out, int i, const char *addr, const char
 	return 0;
 }
 
+BOOL namecvt_call(const char *cmd, const char **name_p, id_t *id_p)
+{
+	char buf[1024];
+	int got, len;
+
+	if (*name_p)
+		len = snprintf(buf, sizeof buf, "%s %s\n", cmd, *name_p);
+	else
+		len = snprintf(buf, sizeof buf, "%s %ld\n", cmd, (long)*id_p);
+	if (len >= (int)sizeof buf) {
+		rprintf(FERROR, "namecvt_call() request was too large.\n");
+		exit_cleanup(RERR_UNSUPPORTED);
+	}
+
+	while ((got = write(namecvt_fd_req, buf, len)) != len) {
+		if (got < 0 && errno == EINTR)
+			continue;
+		rprintf(FERROR, "Connection to name-converter failed.\n");
+		exit_cleanup(RERR_SOCKETIO);
+	}
+
+	if (!read_line_old(namecvt_fd_ans, buf, sizeof buf, 0))
+		return False;
+
+	if (*name_p)
+		*id_p = (id_t)atol(buf);
+	else
+		*name_p = strdup(buf);
+
+	return True;
+}
+
 /* send a list of available modules to the client. Don't list those
    with "list = False". */
 static void send_listing(int fd)
@@ -1207,7 +1318,7 @@ int start_daemon(int f_in, int f_out)
 			return -1;
 		}
 		our_uid = MY_UID();
-		am_root = (our_uid == 0);
+		am_root = (our_uid == ROOT_UID);
 	}
 
 	addr = client_addr(f_in);
@@ -1400,7 +1511,7 @@ int daemon_main(void)
 	log_init(0);
 
 	rprintf(FLOG, "rsyncd version %s starting, listening on port %d\n",
-		RSYNC_VERSION, rsync_port);
+		rsync_version(), rsync_port);
 	/* TODO: If listening on a particular address, then show that
 	 * address too.  In fact, why not just do getnameinfo on the
 	 * local address??? */

compat.c

@@ -3,7 +3,7 @@
  *
  * Copyright (C) Andrew Tridgell 1996
  * Copyright (C) Paul Mackerras 1996
- * Copyright (C) 2004-2020 Wayne Davison
+ * Copyright (C) 2004-2022 Wayne Davison
  *
  * This program is free software; you can redistribute it and/or modify
  * it under the terms of the GNU General Public License as published by
@@ -21,6 +21,7 @@
 
 #include "rsync.h"
 #include "itypes.h"
+#include "ifuncs.h"
 
 extern int am_server;
 extern int am_sender;
@@ -43,6 +44,7 @@ extern int protect_args;
 extern int preserve_uid;
 extern int preserve_gid;
 extern int preserve_atimes;
+extern int preserve_crtimes;
 extern int preserve_acls;
 extern int preserve_xattrs;
 extern int xfer_flags_as_varint;
@@ -50,6 +52,8 @@ extern int need_messages_from_generator;
 extern int delete_mode, delete_before, delete_during, delete_after;
 extern int do_compression;
 extern int do_compression_level;
+extern int saw_stderr_opt;
+extern int msgs2stderr;
 extern char *shell_cmd;
 extern char *partial_dir;
 extern char *files_from;
@@ -73,9 +77,10 @@ int want_xattr_optim = 0;
 int proper_seed_order = 0;
 int inplace_partial = 0;
 int do_negotiated_strings = 0;
+int xmit_id0_names = 0;
 
 /* These index values are for the file-list's extra-attribute array. */
-int pathname_ndx, depth_ndx, atimes_ndx, uid_ndx, gid_ndx, acls_ndx, xattrs_ndx, unsort_ndx;
+int pathname_ndx, depth_ndx, atimes_ndx, crtimes_ndx, uid_ndx, gid_ndx, acls_ndx, xattrs_ndx, unsort_ndx;
 
 int receiver_symlink_times = 0; /* receiver can set the time on a symlink */
 int sender_symlink_iconv = 0;	/* sender should convert symlink content */
@@ -109,6 +114,7 @@ struct name_num_obj valid_compressions = {
 #define CF_CHKSUM_SEED_FIX (1<<5)
 #define CF_INPLACE_PARTIAL_DIR (1<<6)
 #define CF_VARINT_FLIST_FLAGS (1<<7)
+#define CF_ID0_NAMES (1<<8)
 
 static const char *client_info;
 
@@ -150,7 +156,13 @@ static void check_sub_protocol(void)
 
 void set_allow_inc_recurse(void)
 {
-	client_info = shell_cmd ? shell_cmd : "";
+	if (!local_server)
+		client_info = shell_cmd ? shell_cmd : "";
+	else if (am_server) {
+		char buf[64];
+		maybe_add_e_option(buf, sizeof buf);
+		client_info = *buf ? strdup(buf+1) : ""; /* The +1 skips the leading "e". */
+	}
 
 	if (!recurse || use_qsort)
 		allow_inc_recurse = 0;
@@ -158,8 +170,7 @@ void set_allow_inc_recurse(void)
 	 && (delete_before || delete_after
 	  || delay_updates || prune_empty_dirs))
 		allow_inc_recurse = 0;
-	else if (am_server && !local_server
-	 && (strchr(client_info, 'i') == NULL))
+	else if (am_server && strchr(client_info, 'i') == NULL)
 		allow_inc_recurse = 0;
 }
 
@@ -553,7 +564,9 @@ void setup_protocol(int f_out,int f_in)
 	 * aligned for direct int64-pointer memory access. */
 	if (preserve_atimes)
 		atimes_ndx = (file_extra_cnt += EXTRA64_CNT);
-	if (am_sender) /* This is most likely in the in64 union as well. */
+	if (preserve_crtimes)
+		crtimes_ndx = (file_extra_cnt += EXTRA64_CNT);
+	if (am_sender) /* This is most likely in the file_extras64 union as well. */
 		pathname_ndx = (file_extra_cnt += PTR_EXTRA_CNT);
 	else
 		depth_ndx = ++file_extra_cnt;
@@ -591,7 +604,7 @@ void setup_protocol(int f_out,int f_in)
 	if (remote_protocol < MIN_PROTOCOL_VERSION
 	 || remote_protocol > MAX_PROTOCOL_VERSION) {
 		rprintf(FERROR,"protocol version mismatch -- is your shell clean?\n");
-		rprintf(FERROR,"(see the rsync man page for an explanation)\n");
+		rprintf(FERROR,"(see the rsync manpage for an explanation)\n");
 		exit_cleanup(RERR_PROTOCOL);
 	}
 	if (remote_protocol < OLD_PROTOCOL_VERSION) {
@@ -611,6 +624,9 @@ void setup_protocol(int f_out,int f_in)
 	if (read_batch)
 		check_batch_flags();
 
+	if (!saw_stderr_opt && protocol_version <= 28 && am_server)
+		msgs2stderr = 0; /* The client side may not have stderr setup for us. */
+
 #ifndef SUPPORT_PREALLOCATION
 	if (preallocate_files && !am_sender) {
 		rprintf(FERROR, "preallocation is not supported on this %s\n",
@@ -686,15 +702,17 @@ void setup_protocol(int f_out,int f_in)
 #ifdef ICONV_OPTION
 			compat_flags |= CF_SYMLINK_ICONV;
 #endif
-			if (local_server || strchr(client_info, 'f') != NULL)
+			if (strchr(client_info, 'f') != NULL)
 				compat_flags |= CF_SAFE_FLIST;
-			if (local_server || strchr(client_info, 'x') != NULL)
+			if (strchr(client_info, 'x') != NULL)
 				compat_flags |= CF_AVOID_XATTR_OPTIM;
-			if (local_server || strchr(client_info, 'C') != NULL)
+			if (strchr(client_info, 'C') != NULL)
 				compat_flags |= CF_CHKSUM_SEED_FIX;
-			if (local_server || strchr(client_info, 'I') != NULL)
+			if (strchr(client_info, 'I') != NULL)
 				compat_flags |= CF_INPLACE_PARTIAL_DIR;
-			if (local_server || strchr(client_info, 'v') != NULL) {
+			if (strchr(client_info, 'u') != NULL)
+				compat_flags |= CF_ID0_NAMES;
+			if (strchr(client_info, 'v') != NULL) {
 				do_negotiated_strings = 1;
 				compat_flags |= CF_VARINT_FLIST_FLAGS;
 			}
@@ -714,6 +732,11 @@ void setup_protocol(int f_out,int f_in)
 		want_xattr_optim = protocol_version >= 31 && !(compat_flags & CF_AVOID_XATTR_OPTIM);
 		proper_seed_order = compat_flags & CF_CHKSUM_SEED_FIX ? 1 : 0;
 		xfer_flags_as_varint = compat_flags & CF_VARINT_FLIST_FLAGS ? 1 : 0;
+		xmit_id0_names = compat_flags & CF_ID0_NAMES ? 1 : 0;
+		if (!xfer_flags_as_varint && preserve_crtimes) {
+			fprintf(stderr, "Both rsync versions must be at least 3.2.0 for --crtimes.\n");
+			exit_cleanup(RERR_PROTOCOL);
+		}
 		if (am_sender) {
 			receiver_symlink_times = am_server
 			    ? strchr(client_info, 'L') != NULL
@@ -725,7 +748,7 @@ void setup_protocol(int f_out,int f_in)
 #endif
 #ifdef ICONV_OPTION
 		sender_symlink_iconv = iconv_opt && (am_server
-		    ? local_server || strchr(client_info, 's') != NULL
+		    ? strchr(client_info, 's') != NULL
 		    : !!(compat_flags & CF_SYMLINK_ICONV));
 #endif
 		if (inc_recurse && !allow_inc_recurse) {

daemon-parm.awk

@@ -0,0 +1,114 @@
+#!/usr/bin/awk -f
+
+# The caller must pass arg: daemon-parm.txt
+# The resulting code is output into daemon-parm.h
+
+BEGIN {
+    heading = "/* DO NOT EDIT THIS FILE!  It is auto-generated from a list of values in " ARGV[1] "! */\n\n"
+    sect = psect = defines = accessors = prior_ptype = ""
+    parms = "\nstatic struct parm_struct parm_table[] = {"
+    comment_fmt = "\n/********** %s **********/\n"
+    tdstruct = "typedef struct {"
+}
+
+/^\s*$/ { next }
+/^#/ { next }
+
+/^Globals:/ {
+    if (defines != "") {
+	print "The Globals section must come first!"
+	defines = ""
+	exit
+    }
+    defines = tdstruct
+    values = "\nstatic const all_vars Defaults = {\n    { /* Globals: */\n"
+    exps = exp_values = sprintf(comment_fmt, "EXP")
+    sect = "GLOBAL"
+    psect = ", P_GLOBAL, &Vars.g."
+    next
+}
+
+/^Locals:/ {
+    if (sect == "") {
+	print "The Locals section must come after the Globals!"
+	exit
+    }
+    defines = defines exps "} global_vars;\n\n" tdstruct
+    values = values exp_values "\n    }, { /* Locals: */\n"
+    exps = exp_values = sprintf(comment_fmt, "EXP")
+    sect = "LOCAL"
+    psect = ", P_LOCAL, &Vars.l."
+    next
+}
+
+/^(STRING|CHAR|PATH|INTEGER|ENUM|OCTAL|BOOL|BOOLREV|BOOL3)[ \t]/ {
+    ptype = $1
+    name = $2
+    $1 = $2 = ""
+    sub(/^[ \t]+/, "")
+
+    if (ptype != prior_ptype) {
+	comment = sprintf(comment_fmt, ptype)
+	defines = defines comment
+	values = values comment
+	parms = parms "\n"
+	accessors = accessors "\n"
+	prior_ptype = ptype
+    }
+
+    if (ptype == "STRING" || ptype == "PATH") {
+	atype = "STRING"
+	vtype = "char*"
+    } else if (ptype ~ /BOOL/) {
+	atype = vtype = "BOOL"
+    } else if (ptype == "CHAR") {
+	atype = "CHAR"
+	vtype = "char"
+    } else {
+	atype = "INTEGER"
+	vtype = "int"
+    }
+
+    # The name might be var_name|public_name
+    pubname = name
+    sub(/\|.*/, "", name)
+    sub(/.*\|/, "", pubname)
+    gsub(/_/, " ", pubname)
+    gsub(/-/, "", name)
+
+    if (ptype == "ENUM")
+	enum = "enum_" name
+    else
+	enum = "NULL"
+
+    defines = defines "\t" vtype " " name ";\n"
+    values = values "\t" $0 ", /* " name " */\n"
+    parms = parms " {\"" pubname "\", P_" ptype psect name ", " enum ", 0},\n"
+    accessors = accessors "FN_" sect "_" atype "(lp_" name ", " name ")\n"
+
+    if (vtype == "char*") {
+	exps = exps "\tBOOL " name "_EXP;\n"
+	exp_values = exp_values "\tFalse, /* " name "_EXP */\n"
+    }
+
+    next
+}
+
+/./ {
+    print "Extraneous line:" $0
+    defines = ""
+    exit
+}
+
+END {
+    if (sect != "" && defines != "") {
+	defines = defines exps "} local_vars;\n\n"
+	defines = defines tdstruct "\n\tglobal_vars g;\n\tlocal_vars l;\n} all_vars;\n"
+	values = values exp_values "\n    }\n};\n\nstatic all_vars Vars;\n"
+	parms = parms "\n {NULL, P_BOOL, P_NONE, NULL, NULL, 0}\n};\n"
+	print heading defines values parms accessors > "daemon-parm.h"
+    } else {
+	print "Failed to parse the data in " ARGV[1]
+	exit 1
+    }
+}

daemon-parm.txt

@@ -0,0 +1,68 @@
+Globals: ================================================================
+
+STRING	bind_address|address	NULL
+STRING	daemon_chroot		NULL
+STRING	daemon_gid		NULL
+STRING	daemon_uid		NULL
+STRING	motd_file		NULL
+STRING	pid_file		NULL
+STRING	socket_options		NULL
+
+INTEGER	listen_backlog		5
+INTEGER	rsync_port|port		0
+
+BOOL	proxy_protocol		False
+
+Locals: =================================================================
+
+STRING	auth_users		NULL
+STRING	charset			NULL
+STRING	comment			NULL
+STRING	dont_compress		DEFAULT_DONT_COMPRESS
+STRING	early_exec		NULL
+STRING	exclude			NULL
+STRING	exclude_from		NULL
+STRING	filter			NULL
+STRING	gid			NULL
+STRING	hosts_allow		NULL
+STRING	hosts_deny		NULL
+STRING	include			NULL
+STRING	include_from		NULL
+STRING	incoming_chmod		NULL
+STRING	lock_file		DEFAULT_LOCK_FILE
+STRING	log_file		NULL
+STRING	log_format		"%o %h [%a] %m (%u) %f %l"
+STRING	name			NULL
+STRING	name_converter		NULL
+STRING	outgoing_chmod		NULL
+STRING	post-xfer_exec		NULL
+STRING	pre-xfer_exec		NULL
+STRING	refuse_options		NULL
+STRING	secrets_file		NULL
+STRING	syslog_tag		"rsyncd"
+STRING	uid			NULL
+
+PATH	path			NULL
+PATH	temp_dir		NULL
+
+INTEGER	max_connections		0
+INTEGER	max_verbosity		1
+INTEGER	timeout			0
+
+ENUM	syslog_facility		LOG_DAEMON
+
+BOOL	fake_super		False
+BOOL	forward_lookup		True
+BOOL	ignore_errors		False
+BOOL	ignore_nonreadable	False
+BOOL	list			True
+BOOL	read_only		True
+BOOL	reverse_lookup		True
+BOOL	strict_modes		True
+BOOL	transfer_logging	False
+BOOL	use_chroot		True
+BOOL	write_only		False
+
+BOOL3	munge_symlinks		Unset
+BOOL3	numeric_ids		Unset
+BOOL3	open_noatime		Unset

exclude.c

@@ -4,7 +4,7 @@
  * Copyright (C) 1996-2001 Andrew Tridgell <tridge@samba.org>
  * Copyright (C) 1996 Paul Mackerras
  * Copyright (C) 2002 Martin Pool
- * Copyright (C) 2003-2020 Wayne Davison
+ * Copyright (C) 2003-2022 Wayne Davison
  *
  * This program is free software; you can redistribute it and/or modify
  * it under the terms of the GNU General Public License as published by
@@ -21,23 +21,30 @@
  */
 
 #include "rsync.h"
-#include "default-cvsignore.h"
 #include "ifuncs.h"
 
 extern int am_server;
 extern int am_sender;
+extern int am_generator;
 extern int eol_nulls;
 extern int io_error;
+extern int xfer_dirs;
+extern int recurse;
 extern int local_server;
 extern int prune_empty_dirs;
 extern int ignore_perishable;
+extern int old_style_args;
+extern int relative_paths;
 extern int delete_mode;
 extern int delete_excluded;
 extern int cvs_exclude;
 extern int sanitize_paths;
 extern int protocol_version;
+extern int read_batch;
+extern int list_only;
 extern int module_id;
 
+extern char *filesfrom_host;
 extern char curr_dir[MAXPATHLEN];
 extern unsigned int curr_dir_len;
 extern unsigned int module_dirlen;
@@ -45,8 +52,10 @@ extern unsigned int module_dirlen;
 filter_rule_list filter_list = { .debug_type = "" };
 filter_rule_list cvs_filter_list = { .debug_type = " [global CVS]" };
 filter_rule_list daemon_filter_list = { .debug_type = " [daemon]" };
+filter_rule_list implied_filter_list = { .debug_type = " [implied]" };
 
 int saw_xattr_filter = 0;
+int trust_sender_filter = 0;
 
 /* Need room enough for ":MODS " prefix plus some room to grow. */
 #define MAX_RULE_PREFIX (16)
@@ -153,13 +162,17 @@ static void add_rule(filter_rule_list *listp, const char *pat, unsigned int pat_
 {
 	const char *cp;
 	unsigned int pre_len, suf_len, slash_cnt = 0;
+	char *mention_rule_suffix;
 
-	if (DEBUG_GTE(FILTER, 2)) {
-		rprintf(FINFO, "[%s] add_rule(%s%.*s%s)%s\n",
+	if (DEBUG_GTE(FILTER, 1) && pat_len && (pat[pat_len-1] == ' ' || pat[pat_len-1] == '\t'))
+		mention_rule_suffix = " -- CAUTION: trailing whitespace!";
+	else
+		mention_rule_suffix = DEBUG_GTE(FILTER, 2) ? "" : NULL;
+	if (mention_rule_suffix) {
+		rprintf(FINFO, "[%s] add_rule(%s%.*s%s)%s%s\n",
 			who_am_i(), get_rule_prefix(rule, pat, 0, NULL),
-			(int)pat_len, pat,
-			(rule->rflags & FILTRULE_DIRECTORY) ? "/" : "",
-			listp->debug_type);
+			(int)pat_len, pat, (rule->rflags & FILTRULE_DIRECTORY) ? "/" : "",
+			listp->debug_type, mention_rule_suffix);
 	}
 
 	/* These flags also indicate that we're reading a list that
@@ -289,6 +302,231 @@ static void add_rule(filter_rule_list *listp, const char *pat, unsigned int pat_
 	}
 }
 
+/* If the wildcards failed, the remote shell might give us a file matching the literal
+ * wildcards.  Since "*" & "?" already match themselves, this just needs to deal with
+ * failed "[foo]" idioms.
+ */
+static void maybe_add_literal_brackets_rule(filter_rule const *based_on, int arg_len)
+{
+	filter_rule *rule;
+	const char *arg = based_on->pattern, *cp;
+	char *p;
+	int cnt = 0;
+
+	if (arg_len < 0)
+		arg_len = strlen(arg);
+
+	cp = arg;
+	while (*cp) {
+		if (*cp == '\\' && cp[1]) {
+			cp++;
+		} else if (*cp == '[')
+			cnt++;
+		cp++;
+	}
+	if (!cnt)
+		return;
+
+	rule = new0(filter_rule);
+	rule->rflags = based_on->rflags;
+	rule->u.slash_cnt = based_on->u.slash_cnt;
+	p = rule->pattern = new_array(char, arg_len + cnt + 1);
+	cp = arg;
+	while (*cp) {
+		if (*cp == '\\' && cp[1]) {
+			*p++ = *cp++;
+		} else if (*cp == '[')
+			*p++ = '\\';
+		*p++ = *cp++;
+	}
+	*p++ = '\0';
+
+	rule->next = implied_filter_list.head;
+	implied_filter_list.head = rule;
+	if (DEBUG_GTE(FILTER, 3)) {
+		rprintf(FINFO, "[%s] add_implied_include(%s%s)\n", who_am_i(), rule->pattern,
+			rule->rflags & FILTRULE_DIRECTORY ? "/" : "");
+	}
+}
+
+static char *partial_string_buf = NULL;
+static int partial_string_len = 0;
+void implied_include_partial_string(const char *s_start, const char *s_end)
+{
+	partial_string_len = s_end - s_start;
+	if (partial_string_len <= 0 || partial_string_len >= MAXPATHLEN) { /* too-large should be impossible... */
+		partial_string_len = 0;
+		return;
+	}
+	if (!partial_string_buf)
+		partial_string_buf = new_array(char, MAXPATHLEN);
+	memcpy(partial_string_buf, s_start, partial_string_len);
+}
+
+void free_implied_include_partial_string()
+{
+	if (partial_string_buf) {
+		free(partial_string_buf);
+		partial_string_buf = NULL;
+	}
+}
+
+/* Each arg the client sends to the remote sender turns into an implied include
+ * that the receiver uses to validate the file list from the sender. */
+void add_implied_include(const char *arg)
+{
+	filter_rule *rule;
+	int arg_len, saw_wild = 0, saw_live_open_brkt = 0, backslash_cnt = 0;
+	int slash_cnt = 1; /* We know we're adding a leading slash. */
+	const char *cp;
+	char *p;
+	if (am_server || old_style_args || list_only || read_batch || filesfrom_host != NULL)
+		return;
+	if (partial_string_len) {
+		arg_len = strlen(arg);
+		if (partial_string_len + arg_len >= MAXPATHLEN)
+			return; /* Should be impossible... */
+		memcpy(partial_string_buf + partial_string_len, arg, arg_len + 1);
+		partial_string_len = 0;
+		arg = partial_string_buf;
+	}
+	if (relative_paths) {
+		if ((cp = strstr(arg, "/./")) != NULL)
+			arg = cp + 3;
+	} else if ((cp = strrchr(arg, '/')) != NULL) {
+		arg = cp + 1;
+		if (*arg == '.' && arg[1] == '\0')
+			arg++;
+	}
+	arg_len = strlen(arg);
+	if (arg_len) {
+		if (strpbrk(arg, "*[?")) {
+			/* We need to add room to escape backslashes if wildcard chars are present. */
+			cp = arg;
+			while ((cp = strchr(cp, '\\')) != NULL) {
+				arg_len++;
+				cp++;
+			}
+			saw_wild = 1;
+		}
+		arg_len++; /* Leave room for the prefixed slash */
+		rule = new0(filter_rule);
+		if (!implied_filter_list.head)
+			implied_filter_list.head = implied_filter_list.tail = rule;
+		else {
+			rule->next = implied_filter_list.head;
+			implied_filter_list.head = rule;
+		}
+		rule->rflags = FILTRULE_INCLUDE + (saw_wild ? FILTRULE_WILD : 0);
+		p = rule->pattern = new_array(char, arg_len + 1);
+		*p++ = '/';
+		cp = arg;
+		while (*cp) {
+			switch (*cp) {
+			  case '\\':
+				if (cp[1] == ']')
+					cp++; /* A \] in a filter might cause a problem w/o wildcards. */
+				else if (!strchr("*[?", cp[1])) {
+					backslash_cnt++;
+					if (saw_wild)
+						*p++ = '\\';
+				}
+				*p++ = *cp++;
+				break;
+			  case '/':
+				if (p[-1] == '/') { /* This is safe because of the initial slash. */
+					cp++;
+					break;
+				}
+				if (relative_paths) {
+					filter_rule const *ent;
+					int found = 0;
+					*p = '\0';
+					for (ent = implied_filter_list.head; ent; ent = ent->next) {
+						if (ent != rule && strcmp(ent->pattern, rule->pattern) == 0) {
+							found = 1;
+							break;
+						}
+					}
+					if (!found) {
+						filter_rule *R_rule = new0(filter_rule);
+						R_rule->rflags = FILTRULE_INCLUDE | FILTRULE_DIRECTORY;
+						/* Check if our sub-path has wildcards or escaped backslashes */
+						if (saw_wild && strpbrk(rule->pattern, "*[?\\"))
+							R_rule->rflags |= FILTRULE_WILD;
+						R_rule->pattern = strdup(rule->pattern);
+						R_rule->u.slash_cnt = slash_cnt;
+						R_rule->next = implied_filter_list.head;
+						implied_filter_list.head = R_rule;
+						if (DEBUG_GTE(FILTER, 3)) {
+							rprintf(FINFO, "[%s] add_implied_include(%s/)\n",
+								who_am_i(), R_rule->pattern);
+						}
+						if (saw_live_open_brkt)
+							maybe_add_literal_brackets_rule(R_rule, -1);
+					}
+				}
+				slash_cnt++;
+				*p++ = *cp++;
+				break;
+			  case '[':
+				saw_live_open_brkt = 1;
+				*p++ = *cp++;
+				break;
+			  default:
+				*p++ = *cp++;
+				break;
+			}
+		}
+		*p = '\0';
+		rule->u.slash_cnt = slash_cnt;
+		arg = rule->pattern;
+		arg_len = p - arg; /* We recompute it due to backslash weirdness. */
+		if (DEBUG_GTE(FILTER, 3))
+			rprintf(FINFO, "[%s] add_implied_include(%s)\n", who_am_i(), rule->pattern);
+		if (saw_live_open_brkt)
+			maybe_add_literal_brackets_rule(rule, arg_len);
+	}
+
+	if (recurse || xfer_dirs) {
+		/* Now create a rule with an added "/" & "**" or "*" at the end */
+		rule = new0(filter_rule);
+		rule->rflags = FILTRULE_INCLUDE | FILTRULE_WILD;
+		if (recurse)
+			rule->rflags |= FILTRULE_WILD2;
+		/* We must leave enough room for / * * \0. */
+		if (!saw_wild && backslash_cnt) {
+			/* We are appending a wildcard, so now the backslashes need to be escaped. */
+			p = rule->pattern = new_array(char, arg_len + backslash_cnt + 3 + 1);
+			cp = arg;
+			while (*cp) {
+				if (*cp == '\\')
+					*p++ = '\\';
+				*p++ = *cp++;
+			}
+		} else {
+			p = rule->pattern = new_array(char, arg_len + 3 + 1);
+			if (arg_len) {
+				memcpy(p, arg, arg_len);
+				p += arg_len;
+			}
+		}
+		if (p[-1] != '/')
+			*p++ = '/';
+		*p++ = '*';
+		if (recurse)
+			*p++ = '*';
+		*p = '\0';
+		rule->u.slash_cnt = slash_cnt + 1;
+		rule->next = implied_filter_list.head;
+		implied_filter_list.head = rule;
+		if (DEBUG_GTE(FILTER, 3))
+			rprintf(FINFO, "[%s] add_implied_include(%s)\n", who_am_i(), rule->pattern);
+		if (saw_live_open_brkt)
+			maybe_add_literal_brackets_rule(rule, p - rule->pattern);
+	}
+}
+
 /* This frees any non-inherited items, leaving just inherited items on the list. */
 static void pop_filter_list(filter_rule_list *listp)
 {
@@ -703,11 +941,12 @@ static void report_filter_result(enum logcode code, char const *name,
 				 filter_rule const *ent,
 				 int name_flags, const char *type)
 {
+	int log_level = am_sender || am_generator ? 1 : 3;
+
 	/* If a trailing slash is present to match only directories,
 	 * then it is stripped out by add_rule().  So as a special
-	 * case we add it back in here. */
-
-	if (DEBUG_GTE(FILTER, 1)) {
+	 * case we add it back in the log output. */
+	if (DEBUG_GTE(FILTER, log_level)) {
 		static char *actions[2][2]
 		    = { {"show", "hid"}, {"risk", "protect"} };
 		const char *w = who_am_i();
@@ -715,7 +954,7 @@ static void report_filter_result(enum logcode code, char const *name,
 			      : name_flags & NAME_IS_DIR ? "directory"
 			      : "file";
 		rprintf(code, "[%s] %sing %s %s because of pattern %s%s%s\n",
-		    w, actions[*w!='s'][!(ent->rflags & FILTRULE_INCLUDE)],
+		    w, actions[*w=='g'][!(ent->rflags & FILTRULE_INCLUDE)],
 		    t, name, ent->pattern,
 		    ent->rflags & FILTRULE_DIRECTORY ? "/" : "", type);
 	}
@@ -887,6 +1126,7 @@ static filter_rule *parse_rule_tok(const char **rulestr_ptr,
 		}
 		switch (ch) {
 		case ':':
+			trust_sender_filter = 1;
 			rule->rflags |= FILTRULE_PERDIR_MERGE
 				      | FILTRULE_FINISH_SETUP;
 			/* FALL THROUGH */
@@ -1053,7 +1293,7 @@ static void get_cvs_excludes(uint32 rflags)
 		return;
 	initialized = 1;
 
-	parse_filter_str(&cvs_filter_list, DEFAULT_CVSIGNORE,
+	parse_filter_str(&cvs_filter_list, default_cvsignore(),
 			 rule_template(rflags | (protocol_version >= 30 ? FILTRULE_PERISHABLE : 0)),
 			 0);
 

flist.c

@@ -4,7 +4,7 @@
  * Copyright (C) 1996 Andrew Tridgell
  * Copyright (C) 1996 Paul Mackerras
  * Copyright (C) 2001, 2002 Martin Pool <mbp@samba.org>
- * Copyright (C) 2002-2020 Wayne Davison
+ * Copyright (C) 2002-2022 Wayne Davison
  *
  * This program is free software; you can redistribute it and/or modify
  * it under the terms of the GNU General Public License as published by
@@ -43,6 +43,7 @@ extern int use_qsort;
 extern int xfer_dirs;
 extern int filesfrom_fd;
 extern int one_file_system;
+extern int copy_devices;
 extern int copy_dirlinks;
 extern int preserve_uid;
 extern int preserve_gid;
@@ -56,6 +57,7 @@ extern int delete_during;
 extern int missing_args;
 extern int eol_nulls;
 extern int atimes_ndx;
+extern int crtimes_ndx;
 extern int relative_paths;
 extern int implied_dirs;
 extern int ignore_perishable;
@@ -71,6 +73,7 @@ extern int need_unsorted_flist;
 extern int sender_symlink_iconv;
 extern int output_needs_newline;
 extern int sender_keeps_checksum;
+extern int trust_sender_filter;
 extern int unsort_ndx;
 extern uid_t our_uid;
 extern struct stats stats;
@@ -81,8 +84,7 @@ extern char curr_dir[MAXPATHLEN];
 
 extern struct chmod_mode_struct *chmod_modes;
 
-extern filter_rule_list filter_list;
-extern filter_rule_list daemon_filter_list;
+extern filter_rule_list filter_list, implied_filter_list, daemon_filter_list;
 
 #ifdef ICONV_OPTION
 extern int filesfrom_convert;
@@ -133,6 +135,7 @@ static char empty_sum[MAX_DIGEST_LEN];
 static int flist_count_offset; /* for --delete --progress */
 static int show_filelist_progress;
 
+static struct file_list *flist_new(int flags, const char *msg);
 static void flist_sort_and_clean(struct file_list *flist, int strip_root);
 static void output_flist(struct file_list *flist);
 
@@ -293,6 +296,8 @@ static void flist_expand(struct file_list *flist, int extra)
 		flist->malloced = FLIST_START;
 	else if (flist->malloced >= FLIST_LINEAR)
 		flist->malloced += FLIST_LINEAR;
+	else if (flist->malloced < FLIST_START_LARGE/16)
+		flist->malloced *= 4;
 	else
 		flist->malloced *= 2;
 
@@ -303,7 +308,7 @@ static void flist_expand(struct file_list *flist, int extra)
 
 	new_ptr = realloc_array(flist->files, struct file_struct *, flist->malloced);
 
-	if (DEBUG_GTE(FLIST, 1) && flist->malloced != FLIST_START) {
+	if (DEBUG_GTE(FLIST, 1) && flist->files) {
 		rprintf(FCLIENT, "[%s] expand file_list pointer array to %s bytes, did%s move\n",
 		    who_am_i(),
 		    big_num(sizeof flist->files[0] * flist->malloced),
@@ -377,6 +382,9 @@ static void send_file_entry(int f, const char *fname, struct file_struct *file,
 			    int ndx, int first_ndx)
 {
 	static time_t modtime, atime;
+#ifdef SUPPORT_CRTIMES
+	static time_t crtime;
+#endif
 	static mode_t mode;
 #ifdef SUPPORT_HARD_LINKS
 	static int64 dev;
@@ -443,7 +451,7 @@ static void send_file_entry(int f, const char *fname, struct file_struct *file,
 		if (protocol_version < 28)
 			xflags |= XMIT_SAME_RDEV_pre28;
 		else {
-			rdev = MAKEDEV(major(rdev), 0);
+			rdev = MAKEDEV(rdev_major, 0);
 			xflags |= XMIT_SAME_RDEV_MAJOR;
 			if (protocol_version < 30)
 				xflags |= XMIT_RDEV_MINOR_8_pre30;
@@ -482,6 +490,13 @@ static void send_file_entry(int f, const char *fname, struct file_struct *file,
 		else
 			atime = F_ATIME(file);
 	}
+#ifdef SUPPORT_CRTIMES
+	if (crtimes_ndx) {
+		crtime = F_CRTIME(file);
+		if (crtime == modtime)
+			xflags |= XMIT_CRTIME_EQ_MTIME;
+	}
+#endif
 
 #ifdef SUPPORT_HARD_LINKS
 	if (tmp_dev != -1) {
@@ -569,6 +584,10 @@ static void send_file_entry(int f, const char *fname, struct file_struct *file,
 	}
 	if (xflags & XMIT_MOD_NSEC)
 		write_varint(f, F_MOD_NSEC(file));
+#ifdef SUPPORT_CRTIMES
+	if (crtimes_ndx && !(xflags & XMIT_CRTIME_EQ_MTIME))
+		write_varlong(f, crtime, 4);
+#endif
 	if (!(xflags & XMIT_SAME_MODE))
 		write_int(f, to_wire_mode(mode));
 	if (atimes_ndx && !S_ISDIR(mode) && !(xflags & XMIT_SAME_ATIME))
@@ -661,6 +680,9 @@ static void send_file_entry(int f, const char *fname, struct file_struct *file,
 static struct file_struct *recv_file_entry(int f, struct file_list *flist, int xflags)
 {
 	static int64 modtime, atime;
+#ifdef SUPPORT_CRTIMES
+	static time_t crtime;
+#endif
 	static mode_t mode;
 #ifdef SUPPORT_HARD_LINKS
 	static int64 dev;
@@ -679,8 +701,11 @@ static struct file_struct *recv_file_entry(int f, struct file_list *flist, int x
 	int alloc_len, basename_len, linkname_len;
 	int extra_len = file_extra_cnt * EXTRA_LEN;
 	int first_hlink_ndx = -1;
+	char real_ISREG_entry;
 	int64 file_length;
+#ifdef CAN_SET_NSEC
 	uint32 modtime_nsec;
+#endif
 	const char *basename;
 	struct file_struct *file;
 	alloc_pool_t *pool;
@@ -767,23 +792,31 @@ static struct file_struct *recv_file_entry(int f, struct file_list *flist, int x
 			struct file_struct *first = flist->files[first_hlink_ndx - flist->ndx_start];
 			file_length = F_LENGTH(first);
 			modtime = first->modtime;
+#ifdef CAN_SET_NSEC
 			modtime_nsec = F_MOD_NSEC_or_0(first);
+#endif
 			mode = first->mode;
 			if (atimes_ndx && !S_ISDIR(mode))
 				atime = F_ATIME(first);
+#ifdef SUPPORT_CRTIMES
+			if (crtimes_ndx)
+				crtime = F_CRTIME(first);
+#endif
 			if (preserve_uid)
 				uid = F_OWNER(first);
 			if (preserve_gid)
 				gid = F_GROUP(first);
 			if (preserve_devices && IS_DEVICE(mode)) {
 				uint32 *devp = F_RDEV_P(first);
-				rdev = MAKEDEV(DEV_MAJOR(devp), DEV_MINOR(devp));
+				rdev_major = DEV_MAJOR(devp);
+				rdev = MAKEDEV(rdev_major, DEV_MINOR(devp));
 				extra_len += DEV_EXTRA_CNT * EXTRA_LEN;
 			}
 			if (preserve_links && S_ISLNK(mode))
 				linkname_len = strlen(F_SYMLINK(first)) + 1;
 			else
 				linkname_len = 0;
+			real_ISREG_entry = S_ISREG(mode) ? 1 : 0;
 			goto create_object;
 		}
 	}
@@ -804,9 +837,28 @@ static struct file_struct *recv_file_entry(int f, struct file_list *flist, int x
 			modtime = read_int(f);
 	}
 	if (xflags & XMIT_MOD_NSEC)
+#ifndef CAN_SET_NSEC
+		(void)read_varint(f);
+#else
 		modtime_nsec = read_varint(f);
 	else
 		modtime_nsec = 0;
+#endif
+#ifdef SUPPORT_CRTIMES
+	if (crtimes_ndx) {
+		if (xflags & XMIT_CRTIME_EQ_MTIME)
+			crtime = modtime;
+		else
+			crtime = read_varlong(f, 4);
+#if SIZEOF_TIME_T < SIZEOF_INT64
+		if (!am_generator && (int64)(time_t)crtime != crtime) {
+			rprintf(FERROR_XFER,
+				"Create time value of %s truncated on receiver.\n",
+				lastname);
+		}
+#endif
+	}
+#endif
 	if (!(xflags & XMIT_SAME_MODE))
 		mode = from_wire_mode(read_int(f));
 	if (atimes_ndx && !S_ISDIR(mode) && !(xflags & XMIT_SAME_ATIME)) {
@@ -892,10 +944,20 @@ static struct file_struct *recv_file_entry(int f, struct file_list *flist, int x
 #endif
 		linkname_len = 0;
 
+	if (copy_devices && IS_DEVICE(mode)) {
+		/* This is impossible in the official release, but some pre-release patches
+		 * didn't convert the device into a file before sending, so we'll do it here
+		 * (even though the length is typically 0 and any checksum data is zeros). */
+		mode = S_IFREG | (mode & ACCESSPERMS);
+		modtime = time(NULL); /* The mtime on the device is not up-to-date, so set it to "now". */
+		real_ISREG_entry = 0;
+	} else
+		real_ISREG_entry = S_ISREG(mode) ? 1 : 0;
+
 #ifdef SUPPORT_HARD_LINKS
   create_object:
 	if (preserve_hard_links) {
-		if (protocol_version < 28 && S_ISREG(mode))
+		if (protocol_version < 28 && real_ISREG_entry)
 			xflags |= XMIT_HLINKED;
 		if (xflags & XMIT_HLINKED)
 			extra_len += (inc_recurse+1) * EXTRA_LEN;
@@ -924,6 +986,19 @@ static struct file_struct *recv_file_entry(int f, struct file_list *flist, int x
 		exit_cleanup(RERR_UNSUPPORTED);
 	}
 
+	if (*thisname != '.' || thisname[1] != '\0') {
+		int filt_flags = S_ISDIR(mode) ? NAME_IS_DIR : NAME_IS_FILE;
+		if (!trust_sender_filter /* a per-dir filter rule means we must trust the sender's filtering */
+		 && filter_list.head && check_filter(&filter_list, FINFO, thisname, filt_flags) < 0) {
+			rprintf(FERROR, "ERROR: rejecting excluded file-list name: %s\n", thisname);
+			exit_cleanup(RERR_PROTOCOL);
+		}
+		if (implied_filter_list.head && check_filter(&implied_filter_list, FINFO, thisname, filt_flags) <= 0) {
+			rprintf(FERROR, "ERROR: rejecting unrequested file-list name: %s\n", thisname);
+			exit_cleanup(RERR_PROTOCOL);
+		}
+	}
+
 	if (inc_recurse && S_ISDIR(mode)) {
 		if (one_file_system) {
 			/* Room to save the dir's device for -x */
@@ -988,6 +1063,10 @@ static struct file_struct *recv_file_entry(int f, struct file_list *flist, int x
 	}
 	if (atimes_ndx && !S_ISDIR(mode))
 		F_ATIME(file) = atime;
+#ifdef SUPPORT_CRTIMES
+	if (crtimes_ndx)
+		F_CRTIME(file) = crtime;
+#endif
 	if (unsort_ndx)
 		F_NDX(file) = flist->used + flist->ndx_start;
 
@@ -1107,8 +1186,8 @@ static struct file_struct *recv_file_entry(int f, struct file_list *flist, int x
 	}
 #endif
 
-	if (always_checksum && (S_ISREG(mode) || protocol_version < 28)) {
-		if (S_ISREG(mode))
+	if (always_checksum && (real_ISREG_entry || protocol_version < 28)) {
+		if (real_ISREG_entry)
 			bp = F_SUM(file);
 		else {
 			/* Prior to 28, we get a useless set of nulls. */
@@ -1307,6 +1386,18 @@ struct file_struct *make_file(const char *fname, struct file_list *flist,
 	linkname_len = 0;
 #endif
 
+	if (copy_devices && am_sender && IS_DEVICE(st.st_mode)) {
+		if (st.st_size == 0) {
+			int fd = do_open(fname, O_RDONLY, 0);
+			if (fd >= 0) {
+				st.st_size = get_device_size(fd, fname);
+				close(fd);
+			}
+		}
+		st.st_mode = S_IFREG | (st.st_mode & ACCESSPERMS);
+		st.st_mtime = time(NULL); /* The mtime on the device is not up-to-date, so set it to "now". */
+	}
+
 #ifdef ST_MTIME_NSEC
 	if (st.ST_MTIME_NSEC && protocol_version >= 31)
 		extra_len += EXTRA_LEN;
@@ -1385,6 +1476,10 @@ struct file_struct *make_file(const char *fname, struct file_list *flist,
 		file->flags |= FLAG_OWNED_BY_US;
 	if (atimes_ndx && !S_ISDIR(file->mode))
 		F_ATIME(file) = st.st_atime;
+#ifdef SUPPORT_CRTIMES
+	if (crtimes_ndx)
+		F_CRTIME(file) = get_create_time(fname, &st);
+#endif
 
 	if (basename != thisname)
 		file->dirname = lastdir;
@@ -1892,7 +1987,7 @@ static void send_implied_dirs(int f, struct file_list *flist, char *fname,
 		memcpy(F_DIR_RELNAMES_P(lastpath_struct), &relname_list, sizeof relname_list);
 	}
 	rnpp = EXPAND_ITEM_LIST(relname_list, relnamecache *, 32);
-	*rnpp = (relnamecache*)new_array(char, sizeof (relnamecache) + len);
+	*rnpp = (relnamecache*)new_array(char, RELNAMECACHE_LEN + len + 1);
 	(*rnpp)->name_type = name_type;
 	strlcpy((*rnpp)->fname, limit+1, len + 1);
 
@@ -2051,8 +2146,7 @@ void send_extra_file_list(int f, int at_least)
 
 		if (need_unsorted_flist) {
 			flist->sorted = new_array(struct file_struct *, flist->used);
-			memcpy(flist->sorted, flist->files,
-			       flist->used * sizeof (struct file_struct*));
+			memcpy(flist->sorted, flist->files, flist->used * PTR_SIZE);
 		} else
 			flist->sorted = flist->files;
 
@@ -2132,8 +2226,10 @@ struct file_list *send_file_list(int f, int argc, char *argv[])
 #endif
 
 	flist = cur_flist = flist_new(0, "send_file_list");
+	flist_expand(flist, FLIST_START_LARGE);
 	if (inc_recurse) {
 		dir_flist = flist_new(FLIST_TEMP, "send_file_list");
+		flist_expand(dir_flist, FLIST_START_LARGE);
 		flags |= FLAG_DIVERT_DIRS;
 	} else
 		dir_flist = cur_flist;
@@ -2405,8 +2501,7 @@ struct file_list *send_file_list(int f, int argc, char *argv[])
 	 * send them together in a single file-list. */
 	if (need_unsorted_flist) {
 		flist->sorted = new_array(struct file_struct *, flist->used);
-		memcpy(flist->sorted, flist->files,
-		       flist->used * sizeof (struct file_struct*));
+		memcpy(flist->sorted, flist->files, flist->used * PTR_SIZE);
 	} else
 		flist->sorted = flist->files;
 	flist_sort_and_clean(flist, 0);
@@ -2414,7 +2509,7 @@ struct file_list *send_file_list(int f, int argc, char *argv[])
 	file_old_total += flist->used;
 
 	if (numeric_ids <= 0 && !inc_recurse)
-		send_id_list(f);
+		send_id_lists(f);
 
 	/* send the io_error flag */
 	if (protocol_version < 30)
@@ -2488,10 +2583,13 @@ struct file_list *recv_file_list(int f, int dir_ndx)
 #endif
 
 	flist = flist_new(0, "recv_file_list");
+	flist_expand(flist, FLIST_START_LARGE);
 
 	if (inc_recurse) {
-		if (flist->ndx_start == 1)
+		if (flist->ndx_start == 1) {
 			dir_flist = flist_new(FLIST_TEMP, "recv_file_list");
+			flist_expand(dir_flist, FLIST_START_LARGE);
+		}
 		dstart = dir_flist->used;
 	} else {
 		dir_flist = flist;
@@ -2587,8 +2685,7 @@ struct file_list *recv_file_list(int f, int dir_ndx)
 		 * list unsorted for our exchange of index numbers with the
 		 * other side (since their names may not sort the same). */
 		flist->sorted = new_array(struct file_struct *, flist->used);
-		memcpy(flist->sorted, flist->files,
-		       flist->used * sizeof (struct file_struct*));
+		memcpy(flist->sorted, flist->files, flist->used * PTR_SIZE);
 		if (inc_recurse && dir_flist->used > dstart) {
 			static int dir_flist_malloced = 0;
 			if (dir_flist_malloced < dir_flist->malloced) {
@@ -2598,7 +2695,7 @@ struct file_list *recv_file_list(int f, int dir_ndx)
 				dir_flist_malloced = dir_flist->malloced;
 			}
 			memcpy(dir_flist->sorted + dstart, dir_flist->files + dstart,
-			       (dir_flist->used - dstart) * sizeof (struct file_struct*));
+			       (dir_flist->used - dstart) * PTR_SIZE);
 			fsort(dir_flist->sorted + dstart, dir_flist->used - dstart);
 		}
 	} else {
@@ -2730,28 +2827,28 @@ int flist_find(struct file_list *flist, struct file_struct *f)
  * 1=match directories, 0=match non-directories, or -1=match either. */
 int flist_find_name(struct file_list *flist, const char *fname, int want_dir_match)
 {
-	struct { /* We have to create a temporary file_struct for the search. */
-		struct file_struct f;
-		char name_space[MAXPATHLEN];
-	} t;
+	static struct file_struct *f;
 	char fbuf[MAXPATHLEN];
 	const char *slash = strrchr(fname, '/');
 	const char *basename = slash ? slash+1 : fname;
 
-	memset(&t.f, 0, FILE_STRUCT_LEN);
-	memcpy((void *)t.f.basename, basename, strlen(basename)+1);
+	if (!f)
+		f = (struct file_struct*)new_array(char, FILE_STRUCT_LEN + MAXPATHLEN + 1);
+
+	memset(f, 0, FILE_STRUCT_LEN);
+	memcpy((void*)f->basename, basename, strlen(basename)+1);
 
 	if (slash) {
 		strlcpy(fbuf, fname, slash - fname + 1);
-		t.f.dirname = fbuf;
+		f->dirname = fbuf;
 	} else
-		t.f.dirname = NULL;
+		f->dirname = NULL;
 
-	t.f.mode = want_dir_match > 0 ? S_IFDIR : S_IFREG;
+	f->mode = want_dir_match > 0 ? S_IFDIR : S_IFREG;
 
 	if (want_dir_match < 0)
-		return flist_find_ignore_dirness(flist, &t.f);
-	return flist_find(flist, &t.f);
+		return flist_find_ignore_dirness(flist, f);
+	return flist_find(flist, f);
 }
 
 /* Search for an identically-named item in the file list.  Differs from
@@ -2792,25 +2889,20 @@ void clear_file(struct file_struct *file)
 }
 
 /* Allocate a new file list. */
-struct file_list *flist_new(int flags, char *msg)
+static struct file_list *flist_new(int flags, const char *msg)
 {
 	struct file_list *flist;
 
 	flist = new0(struct file_list);
 
 	if (flags & FLIST_TEMP) {
-		if (!(flist->file_pool = pool_create(SMALL_EXTENT, 0,
-						     out_of_memory,
-						     POOL_INTERN)))
+		if (!(flist->file_pool = pool_create(SMALL_EXTENT, 0, _out_of_memory, POOL_INTERN)))
 			out_of_memory(msg);
 	} else {
 		/* This is a doubly linked list with prev looping back to
 		 * the end of the list, but the last next pointer is NULL. */
 		if (!first_flist) {
-			flist->file_pool = pool_create(NORMAL_EXTENT, 0,
-						       out_of_memory,
-						       POOL_INTERN);
-			if (!flist->file_pool)
+			if (!(flist->file_pool = pool_create(NORMAL_EXTENT, 0, _out_of_memory, POOL_INTERN)))
 				out_of_memory(msg);
 
 			flist->ndx_start = flist->flist_num = inc_recurse ? 1 : 0;

generator.c

@@ -4,7 +4,7 @@
  * Copyright (C) 1996-2000 Andrew Tridgell
  * Copyright (C) 1996 Paul Mackerras
  * Copyright (C) 2002 Martin Pool <mbp@samba.org>
- * Copyright (C) 2003-2020 Wayne Davison
+ * Copyright (C) 2003-2022 Wayne Davison
  *
  * This program is free software; you can redistribute it and/or modify
  * it under the terms of the GNU General Public License as published by
@@ -35,16 +35,18 @@ extern int inc_recurse;
 extern int relative_paths;
 extern int implied_dirs;
 extern int keep_dirlinks;
+extern int write_devices;
 extern int preserve_acls;
 extern int preserve_xattrs;
 extern int preserve_links;
 extern int preserve_devices;
-extern int write_devices;
 extern int preserve_specials;
 extern int preserve_hard_links;
 extern int preserve_executability;
 extern int preserve_perms;
-extern int preserve_times;
+extern int preserve_mtimes;
+extern int omit_dir_times;
+extern int omit_link_times;
 extern int delete_mode;
 extern int delete_before;
 extern int delete_during;
@@ -84,7 +86,7 @@ extern int list_only;
 extern int read_batch;
 extern int write_batch;
 extern int safe_symlinks;
-extern long block_size; /* "long" because popt can't set an int32. */
+extern int32 block_size;
 extern int unsort_ndx;
 extern int max_delete;
 extern int force_delete;
@@ -112,10 +114,6 @@ static int need_retouch_dir_times;
 static int need_retouch_dir_perms;
 static const char *solo_file = NULL;
 
-enum nonregtype {
-	TYPE_DIR, TYPE_SPECIAL, TYPE_DEVICE, TYPE_SYMLINK
-};
-
 /* Forward declarations. */
 #ifdef SUPPORT_HARD_LINKS
 static void handle_skipped_hlink(struct file_struct *file, int itemizing,
@@ -186,7 +184,8 @@ static int remember_delete(struct file_struct *file, const char *fname, int flag
 static int read_delay_line(char *buf, int *flags_p)
 {
 	static int read_pos = 0;
-	int j, len, mode;
+	unsigned int mode;
+	int j, len;
 	char *bp, *past_space;
 
 	while (1) {
@@ -270,7 +269,7 @@ static void do_delayed_deletions(char *delbuf)
  * MAXPATHLEN buffer with the name of the directory in it (the functions we
  * call will append names onto the end, but the old dir value will be restored
  * on exit). */
-static void delete_in_dir(char *fbuf, struct file_struct *file, dev_t *fs_dev)
+static void delete_in_dir(char *fbuf, struct file_struct *file, dev_t fs_dev)
 {
 	static int already_warned = 0;
 	static struct hashtable *dev_tbl;
@@ -305,12 +304,12 @@ static void delete_in_dir(char *fbuf, struct file_struct *file, dev_t *fs_dev)
 		if (!dev_tbl)
 			dev_tbl = hashtable_create(16, HT_KEY64);
 		if (file->flags & FLAG_TOP_DIR) {
-			hashtable_find(dev_tbl, *fs_dev+1, "");
-			filesystem_dev = *fs_dev;
-		} else if (filesystem_dev != *fs_dev) {
-			if (!hashtable_find(dev_tbl, *fs_dev+1, NULL))
+			hashtable_find(dev_tbl, fs_dev+1, "");
+			filesystem_dev = fs_dev;
+		} else if (filesystem_dev != fs_dev) {
+			if (!hashtable_find(dev_tbl, fs_dev+1, NULL))
 				return;
-			filesystem_dev = *fs_dev; /* it's a prior top-dir dev */
+			filesystem_dev = fs_dev; /* it's a prior top-dir dev */
 		}
 	}
 
@@ -379,9 +378,9 @@ static void do_delete_pass(void)
 		 || !S_ISDIR(st.st_mode))
 			continue;
 
-		delete_in_dir(fbuf, file, &st.st_dev);
+		delete_in_dir(fbuf, file, st.st_dev);
 	}
-	delete_in_dir(NULL, NULL, &dev_zero);
+	delete_in_dir(NULL, NULL, dev_zero);
 
 	if (INFO_GTE(FLIST, 2) && !am_server)
 		rprintf(FINFO, "                    \r");
@@ -396,6 +395,19 @@ static inline int mtime_differs(STRUCT_STAT *stp, struct file_struct *file)
 #endif
 }
 
+static inline int any_time_differs(stat_x *sxp, struct file_struct *file, UNUSED(const char *fname))
+{
+	int differs = mtime_differs(&sxp->st, file);
+#ifdef SUPPORT_CRTIMES
+	if (!differs && crtimes_ndx) {
+		if (sxp->crtime == 0)
+			sxp->crtime = get_create_time(fname, &sxp->st);
+		differs = !same_time(sxp->crtime, 0, F_CRTIME(file), 0);
+	}
+#endif
+	return differs;
+}
+
 static inline int perms_differ(struct file_struct *file, stat_x *sxp)
 {
 	if (preserve_perms)
@@ -450,7 +462,7 @@ int unchanged_attrs(const char *fname, struct file_struct *file, stat_x *sxp)
 {
 	if (S_ISLNK(file->mode)) {
 #ifdef CAN_SET_SYMLINK_TIMES
-		if (preserve_times & PRESERVE_LINK_TIMES && mtime_differs(&sxp->st, file))
+		if (preserve_mtimes && !omit_link_times && any_time_differs(sxp, file, fname))
 			return 0;
 #endif
 #ifdef CAN_CHMOD_SYMLINK
@@ -470,7 +482,7 @@ int unchanged_attrs(const char *fname, struct file_struct *file, stat_x *sxp)
 			return 0;
 #endif
 	} else {
-		if (preserve_times && mtime_differs(&sxp->st, file))
+		if (preserve_mtimes && any_time_differs(sxp, file, fname))
 			return 0;
 		if (perms_differ(file, sxp))
 			return 0;
@@ -494,9 +506,9 @@ void itemize(const char *fnamecmp, struct file_struct *file, int ndx, int statre
 	     const char *xname)
 {
 	if (statret >= 0) { /* A from-dest-dir statret can == 1! */
-		int keep_time = !preserve_times ? 0
-		    : S_ISDIR(file->mode) ? preserve_times & PRESERVE_DIR_TIMES
-		    : S_ISLNK(file->mode) ? preserve_times & PRESERVE_LINK_TIMES
+		int keep_time = !preserve_mtimes ? 0
+		    : S_ISDIR(file->mode) ? !omit_dir_times
+		    : S_ISLNK(file->mode) ? !omit_link_times
 		    : 1;
 
 		if (S_ISREG(file->mode) && F_LENGTH(file) != sxp->st.st_size)
@@ -512,7 +524,15 @@ void itemize(const char *fnamecmp, struct file_struct *file, int ndx, int statre
 		if (atimes_ndx && !S_ISDIR(file->mode) && !S_ISLNK(file->mode)
 		 && !same_time(F_ATIME(file), 0, sxp->st.st_atime, 0))
 			iflags |= ITEM_REPORT_ATIME;
-#if !defined HAVE_LCHMOD && !defined HAVE_SETATTRLIST
+#ifdef SUPPORT_CRTIMES
+		if (crtimes_ndx) {
+			if (sxp->crtime == 0)
+				sxp->crtime = get_create_time(fnamecmp, &sxp->st);
+			if (!same_time(sxp->crtime, 0, F_CRTIME(file), 0))
+				iflags |= ITEM_REPORT_CRTIME;
+		}
+#endif
+#ifndef CAN_CHMOD_SYMLINK
 		if (S_ISLNK(file->mode)) {
 			;
 		} else
@@ -578,30 +598,77 @@ void itemize(const char *fnamecmp, struct file_struct *file, int ndx, int statre
 	}
 }
 
-
-/* Perform our quick-check heuristic for determining if a file is unchanged. */
-int unchanged_file(char *fn, struct file_struct *file, STRUCT_STAT *st)
+static enum filetype get_file_type(mode_t mode)
 {
-	if (st->st_size != F_LENGTH(file))
-		return 0;
-
-	/* if always checksum is set then we use the checksum instead
-	   of the file time to determine whether to sync */
-	if (always_checksum > 0 && S_ISREG(st->st_mode)) {
-		char sum[MAX_DIGEST_LEN];
-		file_checksum(fn, st, sum);
-		return memcmp(sum, F_SUM(file), flist_csum_len) == 0;
-	}
-
-	if (size_only > 0)
-		return 1;
-
-	if (ignore_times)
-		return 0;
-
-	return !mtime_differs(st, file);
+	if (S_ISREG(mode))
+		return FT_REG;
+	if (S_ISLNK(mode))
+		return FT_SYMLINK;
+	if (S_ISDIR(mode))
+		return FT_DIR;
+	if (IS_SPECIAL(mode))
+		return FT_SPECIAL;
+	if (IS_DEVICE(mode))
+		return FT_DEVICE;
+	return FT_UNSUPPORTED;
 }
 
+/* Perform our quick-check heuristic for determining if a file is unchanged. */
+int quick_check_ok(enum filetype ftype, const char *fn, struct file_struct *file, STRUCT_STAT *st)
+{
+	switch (ftype) {
+	  case FT_REG:
+		if (st->st_size != F_LENGTH(file))
+			return 0;
+
+		/* If always_checksum is set then we use the checksum instead
+		 * of the file mtime to determine whether to sync. */
+		if (always_checksum > 0) {
+			char sum[MAX_DIGEST_LEN];
+			file_checksum(fn, st, sum);
+			return memcmp(sum, F_SUM(file), flist_csum_len) == 0;
+		}
+
+		if (size_only > 0)
+			return 1;
+
+		if (ignore_times)
+			return 0;
+
+		if (mtime_differs(st, file))
+			return 0;
+		break;
+	  case FT_DIR:
+		break;
+	  case FT_SYMLINK: {
+#ifdef SUPPORT_LINKS
+		char lnk[MAXPATHLEN];
+		int len = do_readlink(fn, lnk, MAXPATHLEN-1);
+		if (len <= 0)
+			return 0;
+		lnk[len] = '\0';
+		if (strcmp(lnk, F_SYMLINK(file)) != 0)
+			return 0;
+		break;
+#else
+		return -1;
+#endif
+	  }
+	  case FT_SPECIAL:
+		if (!BITS_EQUAL(file->mode, st->st_mode, _S_IFMT))
+			return 0;
+		break;
+	  case FT_DEVICE: {
+		uint32 *devp = F_RDEV_P(file);
+		if (st->st_rdev != MAKEDEV(DEV_MAJOR(devp), DEV_MINOR(devp)))
+			return 0;
+		break;
+	  }
+	  case FT_UNSUPPORTED:
+		return -1;
+	}
+	return 1;
+}
 
 /*
  * set (initialize) the size entries in the per-file sum_struct
@@ -886,7 +953,7 @@ static int try_dests_reg(struct file_struct *file, char *fname, int ndx,
 			best_match = j;
 			match_level = 1;
 		}
-		if (!unchanged_file(cmpbuf, file, &sxp->st))
+		if (!quick_check_ok(FT_REG, cmpbuf, file, &sxp->st))
 			continue;
 		if (match_level == 1) {
 			best_match = j;
@@ -985,29 +1052,14 @@ static int try_dests_non(struct file_struct *file, char *fname, int ndx,
 {
 	int best_match = -1;
 	int match_level = 0;
-	enum nonregtype type;
-	uint32 *devp;
-#ifdef SUPPORT_LINKS
-	char lnk[MAXPATHLEN];
-	int len;
-#endif
+	enum filetype ftype = get_file_type(file->mode);
 	int j = 0;
 
 #ifndef SUPPORT_LINKS
-	if (S_ISLNK(file->mode))
+	if (ftype == FT_SYMLINK)
 		return -1;
 #endif
-	if (S_ISDIR(file->mode)) {
-		type = TYPE_DIR;
-	} else if (IS_SPECIAL(file->mode))
-		type = TYPE_SPECIAL;
-	else if (IS_DEVICE(file->mode))
-		type = TYPE_DEVICE;
-#ifdef SUPPORT_LINKS
-	else if (S_ISLNK(file->mode))
-		type = TYPE_SYMLINK;
-#endif
-	else {
+	if (ftype == FT_REG || ftype == FT_UNSUPPORTED) {
 		rprintf(FERROR,
 			"internal: try_dests_non() called with invalid mode (%o)\n",
 			(int)file->mode);
@@ -1018,53 +1070,14 @@ static int try_dests_non(struct file_struct *file, char *fname, int ndx,
 		pathjoin(cmpbuf, MAXPATHLEN, basis_dir[j], fname);
 		if (link_stat(cmpbuf, &sxp->st, 0) < 0)
 			continue;
-		switch (type) {
-		case TYPE_DIR:
-			if (!S_ISDIR(sxp->st.st_mode))
-				continue;
-			break;
-		case TYPE_SPECIAL:
-			if (!IS_SPECIAL(sxp->st.st_mode))
-				continue;
-			break;
-		case TYPE_DEVICE:
-			if (!IS_DEVICE(sxp->st.st_mode))
-				continue;
-			break;
-		case TYPE_SYMLINK:
-#ifdef SUPPORT_LINKS
-			if (!S_ISLNK(sxp->st.st_mode))
-				continue;
-			break;
-#else
-			return -1;
-#endif
-		}
+		if (ftype != get_file_type(sxp->st.st_mode))
+			continue;
 		if (match_level < 1) {
 			match_level = 1;
 			best_match = j;
 		}
-		switch (type) {
-		case TYPE_DIR:
-		case TYPE_SPECIAL:
-			break;
-		case TYPE_DEVICE:
-			devp = F_RDEV_P(file);
-			if (sxp->st.st_rdev != MAKEDEV(DEV_MAJOR(devp), DEV_MINOR(devp)))
-				continue;
-			break;
-		case TYPE_SYMLINK:
-#ifdef SUPPORT_LINKS
-			if ((len = do_readlink(cmpbuf, lnk, MAXPATHLEN-1)) <= 0)
-				continue;
-			lnk[len] = '\0';
-			if (strcmp(lnk, F_SYMLINK(file)) != 0)
-				continue;
-			break;
-#else
-			return -1;
-#endif
-		}
+		if (!quick_check_ok(ftype, cmpbuf, file, &sxp->st))
+			continue;
 		if (match_level < 2) {
 			match_level = 2;
 			best_match = j;
@@ -1109,14 +1122,14 @@ static int try_dests_non(struct file_struct *file, char *fname, int ndx,
 			match_level = 2;
 		if (itemizing && stdout_format_has_i
 		 && (INFO_GTE(NAME, 2) || stdout_format_has_i > 1)) {
-			int chg = alt_dest_type == COMPARE_DEST && type != TYPE_DIR ? 0
+			int chg = alt_dest_type == COMPARE_DEST && ftype != FT_DIR ? 0
 			    : ITEM_LOCAL_CHANGE + (match_level == 3 ? ITEM_XNAME_FOLLOWS : 0);
 			char *lp = match_level == 3 ? "" : NULL;
 			itemize(cmpbuf, file, ndx, 0, sxp, chg + ITEM_MATCHED, 0, lp);
 		}
 		if (INFO_GTE(NAME, 2) && maybe_ATTRS_REPORT) {
 			rprintf(FCLIENT, "%s%s is uptodate\n",
-				fname, type == TYPE_DIR ? "/" : "");
+				fname, ftype == FT_DIR ? "/" : "");
 		}
 		return -2;
 	}
@@ -1131,6 +1144,7 @@ static void list_file_entry(struct file_struct *f)
 	int size_width = human_readable ? 14 : 11;
 	int mtime_width = 1 + strlen(mtime_str);
 	int atime_width = atimes_ndx ? mtime_width : 0;
+	int crtime_width = crtimes_ndx ? mtime_width : 0;
 
 	if (!F_IS_ACTIVE(f)) {
 		/* this can happen if duplicate names were removed */
@@ -1141,10 +1155,11 @@ static void list_file_entry(struct file_struct *f)
 
 	if (missing_args == 2 && f->mode == 0) {
 		rprintf(FINFO, "%-*s %s\n",
-			10 + 1 + size_width + mtime_width + atime_width, "*missing",
+			10 + 1 + size_width + mtime_width + atime_width + crtime_width, "*missing",
 			f_name(f, NULL));
 	} else {
 		const char *atime_str = atimes_ndx && !S_ISDIR(f->mode) ? timestring(F_ATIME(f)) : "";
+		const char *crtime_str = crtimes_ndx ? timestring(F_CRTIME(f)) : "";
 		const char *arrow, *lnk;
 
 		permstring(permbuf, f->mode);
@@ -1157,9 +1172,9 @@ static void list_file_entry(struct file_struct *f)
 #endif
 			arrow = lnk = "";
 
-		rprintf(FINFO, "%s %*s %s%*s %s%s%s\n",
+		rprintf(FINFO, "%s %*s %s%*s%*s %s%s%s\n",
 			permbuf, size_width, human_num(F_LENGTH(f)),
-			timestring(f->modtime), atime_width, atime_str,
+			timestring(f->modtime), atime_width, atime_str, crtime_width, crtime_str,
 			f_name(f, NULL), arrow, lnk);
 	}
 }
@@ -1208,7 +1223,8 @@ static void recv_generator(char *fname, struct file_struct *file, int ndx,
 	char fnamecmpbuf[MAXPATHLEN];
 	uchar fnamecmp_type;
 	int del_opts = delete_mode || force_delete ? DEL_RECURSE : 0;
-	int is_dir = !S_ISDIR(file->mode) ? 0
+	enum filetype stype, ftype = get_file_type(file->mode);
+	int is_dir = ftype != FT_DIR ? 0
 		   : inc_recurse && ndx != cur_flist->ndx_start - 1 ? -1
 		   : 1;
 
@@ -1277,20 +1293,25 @@ static void recv_generator(char *fname, struct file_struct *file, int ndx,
 			 * this function was asked to process in the file list. */
 			if (!inc_recurse
 			 && (*dn != '.' || dn[1]) /* Avoid an issue with --relative and the "." dir. */
-			 && (!prior_dir_file || strcmp(dn, f_name(prior_dir_file, NULL)) != 0)
-			 && flist_find_name(cur_flist, dn, 1) < 0) {
+			 && (!prior_dir_file || strcmp(dn, f_name(prior_dir_file, NULL)) != 0)) {
+				int ok = 0, j = flist_find_name(cur_flist, dn, -1);
+				if (j >= 0) {
+					struct file_struct *f = cur_flist->sorted[j];
+					if (S_ISDIR(f->mode) || (missing_args == 2 && !file->mode && !f->mode))
+						ok = 1;
+				}
 				/* The --delete-missing-args option can actually put invalid entries into
 				 * the file list, so if that option was specified, we'll just complain about
 				 * it and allow it. */
-				if (missing_args == 2 && file->mode == 0)
+				if (!ok && missing_args == 2 && file->mode == 0 && j < 0)
 					rprintf(FERROR, "WARNING: parent dir is absent in the file list: %s\n", dn);
-				else {
+				else if (!ok) {
 					rprintf(FERROR, "ABORTING due to invalid path from sender: %s/%s\n",
 						dn, file->basename);
 					exit_cleanup(RERR_PROTOCOL);
 				}
 			}
-			if (relative_paths && !implied_dirs
+			if (relative_paths && !implied_dirs && file->mode != 0
 			 && do_stat(dn, &sx.st) < 0) {
 				if (dry_run)
 					goto parent_is_dry_missing;
@@ -1351,10 +1372,27 @@ static void recv_generator(char *fname, struct file_struct *file, int ndx,
 	 && !am_root && sx.st.st_uid == our_uid)
 		del_opts |= DEL_NO_UID_WRITE;
 
+	if (statret == 0)
+		stype = get_file_type(sx.st.st_mode);
+	else
+		stype = FT_UNSUPPORTED;
+
 	if (ignore_existing > 0 && statret == 0
-	 && (!is_dir || !S_ISDIR(sx.st.st_mode))) {
-		if (INFO_GTE(SKIP, 1) && is_dir >= 0)
-			rprintf(FINFO, "%s exists\n", fname);
+	 && (!is_dir || stype != FT_DIR)) {
+		if (INFO_GTE(SKIP, 1) && is_dir >= 0) {
+			const char *suf = "";
+			if (INFO_GTE(SKIP, 2)) {
+				if (ftype != stype)
+					suf = " (type change)";
+				else if (!quick_check_ok(ftype, fname, file, &sx.st))
+					suf = always_checksum ? " (sum change)" : " (file change)";
+				else if (!unchanged_attrs(fname, file, &sx))
+					suf = " (attr change)";
+				else
+					suf = " (uptodate)";
+			}
+			rprintf(FINFO, "%s exists%s\n", fname, suf);
+		}
 #ifdef SUPPORT_HARD_LINKS
 		if (F_IS_HLINKED(file))
 			handle_skipped_hlink(file, itemizing, code, f_out);
@@ -1375,7 +1413,7 @@ static void recv_generator(char *fname, struct file_struct *file, int ndx,
 		} else
 			added_perms = 0;
 		if (is_dir < 0) {
-			if (!(preserve_times & PRESERVE_DIR_TIMES))
+			if (!preserve_mtimes || omit_dir_times)
 				goto cleanup;
 			/* In inc_recurse mode we want to make sure any missing
 			 * directories get created while we're still processing
@@ -1383,7 +1421,7 @@ static void recv_generator(char *fname, struct file_struct *file, int ndx,
 			 * dir's mtime right away).  We will handle the dir in
 			 * full later (right before we handle its contents). */
 			if (statret == 0
-			 && (S_ISDIR(sx.st.st_mode)
+			 && (stype == FT_DIR
 			  || delete_item(fname, sx.st.st_mode, del_opts | DEL_FOR_DIR) != 0))
 				goto cleanup; /* Any errors get reported later. */
 			if (do_mkdir(fname, (file->mode|added_perms) & 0700) == 0)
@@ -1395,7 +1433,7 @@ static void recv_generator(char *fname, struct file_struct *file, int ndx,
 		 * file of that name and it is *not* a directory, then
 		 * we need to delete it.  If it doesn't exist, then
 		 * (perhaps recursively) create it. */
-		if (statret == 0 && !S_ISDIR(sx.st.st_mode)) {
+		if (statret == 0 && stype != FT_DIR) {
 			if (delete_item(fname, sx.st.st_mode, del_opts | DEL_FOR_DIR) != 0)
 				goto skipping_dir_contents;
 			statret = -1;
@@ -1479,7 +1517,7 @@ static void recv_generator(char *fname, struct file_struct *file, int ndx,
 		else if (delete_during && f_out != -1 && !phase
 		    && !(file->flags & FLAG_MISSING_DIR)) {
 			if (file->flags & FLAG_CONTENT_DIR)
-				delete_in_dir(fname, file, &real_sx.st.st_dev);
+				delete_in_dir(fname, file, real_sx.st.st_dev);
 			else
 				change_local_filter_dir(fname, strlen(fname), F_DEPTH(file));
 		}
@@ -1490,7 +1528,7 @@ static void recv_generator(char *fname, struct file_struct *file, int ndx,
 	/* If we're not preserving permissions, change the file-list's
 	 * mode based on the local permissions and some heuristics. */
 	if (!preserve_perms) {
-		int exists = statret == 0 && !S_ISDIR(sx.st.st_mode);
+		int exists = statret == 0 && stype != FT_DIR;
 		file->mode = dest_mode(file->mode, sx.st.st_mode, dflt_perms, exists);
 	}
 
@@ -1500,7 +1538,7 @@ static void recv_generator(char *fname, struct file_struct *file, int ndx,
 		goto cleanup;
 #endif
 
-	if (preserve_links && S_ISLNK(file->mode)) {
+	if (preserve_links && ftype == FT_SYMLINK) {
 #ifdef SUPPORT_LINKS
 		const char *sl = F_SYMLINK(file);
 		if (safe_symlinks && unsafe_symlink(sl, fname)) {
@@ -1517,12 +1555,7 @@ static void recv_generator(char *fname, struct file_struct *file, int ndx,
 			goto cleanup;
 		}
 		if (statret == 0) {
-			char lnk[MAXPATHLEN];
-			int len;
-
-			if (S_ISLNK(sx.st.st_mode)
-			 && (len = do_readlink(fname, lnk, MAXPATHLEN-1)) > 0
-			 && strncmp(lnk, sl, len) == 0 && sl[len] == '\0') {
+			if (stype == FT_SYMLINK && quick_check_ok(stype, fname, file, &sx.st)) {
 				/* The link is pointing to the right place. */
 				set_file_attrs(fname, file, &sx, NULL, maybe_ATTRS_REPORT);
 				if (itemizing)
@@ -1555,7 +1588,7 @@ static void recv_generator(char *fname, struct file_struct *file, int ndx,
 		if (atomic_create(file, fname, sl, NULL, MAKEDEV(0, 0), &sx, statret == 0 ? DEL_FOR_SYMLINK : 0)) {
 			set_file_attrs(fname, file, NULL, NULL, 0);
 			if (itemizing) {
-				if (statret == 0 && !S_ISLNK(sx.st.st_mode))
+				if (statret == 0 && stype != FT_SYMLINK)
 					statret = -1;
 				itemize(fnamecmp, file, ndx, statret, &sx,
 					ITEM_LOCAL_CHANGE|ITEM_REPORT_CHANGE, 0, NULL);
@@ -1576,28 +1609,22 @@ static void recv_generator(char *fname, struct file_struct *file, int ndx,
 		goto cleanup;
 	}
 
-	if ((am_root && preserve_devices && IS_DEVICE(file->mode))
-	 || (preserve_specials && IS_SPECIAL(file->mode))) {
+	if ((am_root && preserve_devices && ftype == FT_DEVICE)
+	 || (preserve_specials && ftype == FT_SPECIAL)) {
 		dev_t rdev;
-		int del_for_flag = 0;
-		if (IS_DEVICE(file->mode)) {
+		int del_for_flag;
+		if (ftype == FT_DEVICE) {
 			uint32 *devp = F_RDEV_P(file);
 			rdev = MAKEDEV(DEV_MAJOR(devp), DEV_MINOR(devp));
-		} else
+			del_for_flag = DEL_FOR_DEVICE;
+		} else {
 			rdev = 0;
+			del_for_flag = DEL_FOR_SPECIAL;
+		}
 		if (statret == 0) {
-			if (IS_DEVICE(file->mode)) {
-				if (!IS_DEVICE(sx.st.st_mode))
-					statret = -1;
-				del_for_flag = DEL_FOR_DEVICE;
-			} else {
-				if (!IS_SPECIAL(sx.st.st_mode))
-					statret = -1;
-				del_for_flag = DEL_FOR_SPECIAL;
-			}
-			if (statret == 0
-			 && BITS_EQUAL(sx.st.st_mode, file->mode, _S_IFMT)
-			 && (IS_SPECIAL(sx.st.st_mode) || sx.st.st_rdev == rdev)) {
+			if (ftype != stype)
+				statret = -1;
+			else if (quick_check_ok(ftype, fname, file, &sx.st)) {
 				/* The device or special file is identical. */
 				set_file_attrs(fname, file, &sx, NULL, maybe_ATTRS_REPORT);
 				if (itemizing)
@@ -1650,10 +1677,12 @@ static void recv_generator(char *fname, struct file_struct *file, int ndx,
 		goto cleanup;
 	}
 
-	if (!S_ISREG(file->mode)) {
-		if (solo_file)
-			fname = f_name(file, NULL);
-		rprintf(FINFO, "skipping non-regular file \"%s\"\n", fname);
+	if (ftype != FT_REG) {
+		if (INFO_GTE(NONREG, 1)) {
+			if (solo_file)
+				fname = f_name(file, NULL);
+			rprintf(FINFO, "skipping non-regular file \"%s\"\n", fname);
+		}
 		goto cleanup;
 	}
 
@@ -1674,7 +1703,7 @@ static void recv_generator(char *fname, struct file_struct *file, int ndx,
 		goto cleanup;
 	}
 
-	if (update_only > 0 && statret == 0 && file->modtime - sx.st.st_mtime <= modify_window) {
+	if (update_only > 0 && statret == 0 && file->modtime - sx.st.st_mtime < modify_window) {
 		if (INFO_GTE(SKIP, 1))
 			rprintf(FINFO, "%s is newer\n", fname);
 #ifdef SUPPORT_HARD_LINKS
@@ -1686,7 +1715,7 @@ static void recv_generator(char *fname, struct file_struct *file, int ndx,
 
 	fnamecmp_type = FNAMECMP_FNAME;
 
-	if (statret == 0 && !(S_ISREG(sx.st.st_mode) || (write_devices && IS_DEVICE(sx.st.st_mode)))) {
+	if (statret == 0 && !(stype == FT_REG || (write_devices && stype == FT_DEVICE))) {
 		if (delete_item(fname, sx.st.st_mode, del_opts | DEL_FOR_FILE) != 0)
 			goto cleanup;
 		statret = -1;
@@ -1720,7 +1749,7 @@ static void recv_generator(char *fname, struct file_struct *file, int ndx,
 		partialptr = NULL;
 
 	if (statret != 0 && fuzzy_basis) {
-		if (need_fuzzy_dirlist && S_ISREG(file->mode)) {
+		if (need_fuzzy_dirlist) {
 			const char *dn = file->dirname ? file->dirname : ".";
 			int i;
 			strlcpy(fnamecmpbuf, dn, sizeof fnamecmpbuf);
@@ -1764,11 +1793,17 @@ static void recv_generator(char *fname, struct file_struct *file, int ndx,
 		goto cleanup;
 	}
 
+	if (write_devices && IS_DEVICE(sx.st.st_mode) && sx.st.st_size == 0) {
+		/* This early open into fd skips the regular open below. */
+		if ((fd = do_open(fnamecmp, O_RDONLY, 0)) >= 0)
+			real_sx.st.st_size = sx.st.st_size = get_device_size(fd, fnamecmp);
+	}
+
 	if (fnamecmp_type <= FNAMECMP_BASIS_DIR_HIGH)
 		;
 	else if (fnamecmp_type >= FNAMECMP_FUZZY)
 		;
-	else if (unchanged_file(fnamecmp, file, &sx.st)) {
+	else if (quick_check_ok(FT_REG, fnamecmp, file, &sx.st)) {
 		if (partialptr) {
 			do_unlink(partialptr);
 			handle_partial_dir(partialptr, PDIR_DELETE);
@@ -1829,7 +1864,7 @@ static void recv_generator(char *fname, struct file_struct *file, int ndx,
 	}
 
 	/* open the file */
-	if ((fd = do_open(fnamecmp, O_RDONLY, 0)) < 0) {
+	if (fd < 0 && (fd = do_open(fnamecmp, O_RDONLY, 0)) < 0) {
 		rsyserr(FERROR, errno, "failed to open %s, continuing",
 			full_fname(fnamecmp));
 	  pretend_missing:
@@ -1846,11 +1881,9 @@ static void recv_generator(char *fname, struct file_struct *file, int ndx,
 
 	if (inplace && make_backups > 0 && fnamecmp_type == FNAMECMP_FNAME) {
 		if (!(backupptr = get_backup_name(fname))) {
-			close(fd);
 			goto cleanup;
 		}
 		if (!(back_file = make_file(fname, NULL, NULL, 0, NO_FILTERS))) {
-			close(fd);
 			goto pretend_missing;
 		}
 		if (robust_unlink(backupptr) && errno != ENOENT) {
@@ -1858,14 +1891,12 @@ static void recv_generator(char *fname, struct file_struct *file, int ndx,
 				full_fname(backupptr));
 			unmake_file(back_file);
 			back_file = NULL;
-			close(fd);
 			goto cleanup;
 		}
 		if ((f_copy = do_open(backupptr, O_WRONLY | O_CREAT | O_TRUNC | O_EXCL, 0600)) < 0) {
 			rsyserr(FERROR_XFER, errno, "open %s", full_fname(backupptr));
 			unmake_file(back_file);
 			back_file = NULL;
-			close(fd);
 			goto cleanup;
 		}
 		fnamecmp_type = FNAMECMP_BACKUP;
@@ -1916,7 +1947,6 @@ static void recv_generator(char *fname, struct file_struct *file, int ndx,
 		write_sum_head(f_out, NULL);
 	else if (sx.st.st_size <= 0) {
 		write_sum_head(f_out, NULL);
-		close(fd);
 	} else {
 		if (generate_and_send_sums(fd, sx.st.st_size, f_out, f_copy) < 0) {
 			rprintf(FWARNING,
@@ -1924,10 +1954,11 @@ static void recv_generator(char *fname, struct file_struct *file, int ndx,
 				fnamecmp);
 			write_sum_head(f_out, NULL);
 		}
-		close(fd);
 	}
 
   cleanup:
+	if (fd >= 0)
+		close(fd);
 	if (back_file) {
 		int save_preserve_xattrs = preserve_xattrs;
 		if (f_copy >= 0)
@@ -2213,7 +2244,7 @@ void generate_files(int f_out, const char *local_name)
 	}
 	solo_file = local_name;
 	dir_tweaking = !(list_only || solo_file || dry_run);
-	need_retouch_dir_times = preserve_times & PRESERVE_DIR_TIMES;
+	need_retouch_dir_times = preserve_mtimes && !omit_dir_times;
 	loopchk_limit = allowed_lull ? allowed_lull * 5 : 200;
 	symlink_timeset_failed_flags = ITEM_REPORT_TIME
 	    | (protocol_version >= 30 || !am_server ? ITEM_REPORT_TIMEFAIL : 0);
@@ -2266,7 +2297,7 @@ void generate_files(int f_out, const char *local_name)
 						dirdev = MAKEDEV(DEV_MAJOR(devp), DEV_MINOR(devp));
 					} else
 						dirdev = MAKEDEV(0, 0);
-					delete_in_dir(fbuf, fp, &dirdev);
+					delete_in_dir(fbuf, fp, dirdev);
 				} else
 					change_local_filter_dir(fbuf, strlen(fbuf), F_DEPTH(fp));
 			}
@@ -2313,7 +2344,7 @@ void generate_files(int f_out, const char *local_name)
 	} while ((cur_flist = cur_flist->next) != NULL);
 
 	if (delete_during)
-		delete_in_dir(NULL, NULL, &dev_zero);
+		delete_in_dir(NULL, NULL, dev_zero);
 	phase++;
 	if (DEBUG_GTE(GENR, 1))
 		rprintf(FINFO, "generate_files phase=%d\n", phase);

hlink.c

@@ -406,7 +406,7 @@ int hard_link_check(struct file_struct *file, int ndx, char *fname,
 				}
 				break;
 			}
-			if (!unchanged_file(cmpbuf, file, &alt_sx.st))
+			if (!quick_check_ok(FT_REG, cmpbuf, file, &alt_sx.st))
 				continue;
 			statret = 1;
 			if (unchanged_attrs(cmpbuf, file, &alt_sx))

ifuncs.h

@@ -1,6 +1,6 @@
 /* Inline functions for rsync.
  *
- * Copyright (C) 2007-2020 Wayne Davison
+ * Copyright (C) 2007-2022 Wayne Davison
  *
  * This program is free software; you can redistribute it and/or modify
  * it under the terms of the GNU General Public License as published by
@@ -75,6 +75,7 @@ d_name(struct dirent *di)
 static inline void
 init_stat_x(stat_x *sx_p)
 {
+	sx_p->crtime = 0;
 #ifdef SUPPORT_ACLS
 	sx_p->acc_acl = sx_p->def_acl = NULL;
 #endif
@@ -105,7 +106,7 @@ free_stat_x(stat_x *sx_p)
 static inline char *my_strdup(const char *str, const char *file, int line)
 {
     int len = strlen(str)+1;
-    char *buf = my_alloc(do_malloc, len, 1, file, line);
+    char *buf = my_alloc(NULL, len, 1, file, line);
     memcpy(buf, str, len);
     return buf;
 }

install-sh

@@ -115,7 +115,7 @@ else
 # might cause directories to be created, which would be especially bad 
 # if $src (and thus $dsttmp) contains '*'.
 
-	if [ -f $src -o -d $src ]
+	if [ -f $src ] || [ -d $src ]
 	then
 		true
 	else

io.c

@@ -4,7 +4,7 @@
  * Copyright (C) 1996-2001 Andrew Tridgell
  * Copyright (C) 1996 Paul Mackerras
  * Copyright (C) 2001, 2002 Martin Pool <mbp@samba.org>
- * Copyright (C) 2003-2020 Wayne Davison
+ * Copyright (C) 2003-2022 Wayne Davison
  *
  * This program is free software; you can redistribute it and/or modify
  * it under the terms of the GNU General Public License as published by
@@ -54,12 +54,14 @@ extern int read_batch;
 extern int compat_flags;
 extern int protect_args;
 extern int checksum_seed;
+extern int daemon_connection;
 extern int protocol_version;
 extern int remove_source_files;
 extern int preserve_hard_links;
 extern BOOL extra_flist_sending_enabled;
 extern BOOL flush_ok_after_signal;
 extern struct stats stats;
+extern time_t stop_at_utime;
 extern struct file_list *cur_flist;
 #ifdef ICONV_OPTION
 extern int filesfrom_convert;
@@ -262,15 +264,18 @@ static size_t safe_read(int fd, char *buf, size_t len)
 			rprintf(FINFO, "select exception on fd %d\n", fd); */
 
 		if (FD_ISSET(fd, &r_fds)) {
-			int n = read(fd, buf + got, len - got);
-			if (DEBUG_GTE(IO, 2))
-				rprintf(FINFO, "[%s] safe_read(%d)=%ld\n", who_am_i(), fd, (long)n);
+			ssize_t n = read(fd, buf + got, len - got);
+			if (DEBUG_GTE(IO, 2)) {
+				rprintf(FINFO, "[%s] safe_read(%d)=%" SIZE_T_FMT_MOD "d\n",
+					who_am_i(), fd, (SIZE_T_FMT_CAST)n);
+			}
 			if (n == 0)
 				break;
 			if (n < 0) {
 				if (errno == EINTR)
 					continue;
-				rsyserr(FERROR, errno, "safe_read failed to read %ld bytes", (long)len);
+				rsyserr(FERROR, errno, "safe_read failed to read %" SIZE_T_FMT_MOD "d bytes",
+					(SIZE_T_FMT_CAST)len);
 				exit_cleanup(RERR_STREAMIO);
 			}
 			if ((got += (size_t)n) == len)
@@ -302,7 +307,7 @@ static const char *what_fd_is(int fd)
  * is not used on the socket except very early in the transfer. */
 static void safe_write(int fd, const char *buf, size_t len)
 {
-	int n;
+	ssize_t n;
 
 	assert(fd != iobuf.out_fd);
 
@@ -313,8 +318,8 @@ static void safe_write(int fd, const char *buf, size_t len)
 		if (errno != EINTR && errno != EWOULDBLOCK && errno != EAGAIN) {
 		  write_failed:
 			rsyserr(FERROR, errno,
-				"safe_write failed to write %ld bytes to %s",
-				(long)len, what_fd_is(fd));
+				"safe_write failed to write %" SIZE_T_FMT_MOD "d bytes to %s",
+				(SIZE_T_FMT_CAST)len, what_fd_is(fd));
 			exit_cleanup(RERR_STREAMIO);
 		}
 	} else {
@@ -360,7 +365,7 @@ static void safe_write(int fd, const char *buf, size_t len)
  * a chunk of data and put it into the output buffer. */
 static void forward_filesfrom_data(void)
 {
-	int len;
+	ssize_t len;
 
 	len = read(ff_forward_fd, ff_xb.buf + ff_xb.len, ff_xb.size - ff_xb.len);
 	if (len <= 0) {
@@ -371,12 +376,15 @@ static void forward_filesfrom_data(void)
 			free_xbuf(&ff_xb);
 			if (ff_reenable_multiplex >= 0)
 				io_start_multiplex_out(ff_reenable_multiplex);
+			free_implied_include_partial_string();
 		}
 		return;
 	}
 
-	if (DEBUG_GTE(IO, 2))
-		rprintf(FINFO, "[%s] files-from read=%ld\n", who_am_i(), (long)len);
+	if (DEBUG_GTE(IO, 2)) {
+		rprintf(FINFO, "[%s] files-from read=%" SIZE_T_FMT_MOD "d\n",
+			who_am_i(), (SIZE_T_FMT_CAST)len);
+	}
 
 #ifdef ICONV_OPTION
 	len += ff_xb.len;
@@ -412,6 +420,7 @@ static void forward_filesfrom_data(void)
 		while (s != eob) {
 			if (*s++ == '\0') {
 				ff_xb.len = s - sob - 1;
+				add_implied_include(sob);
 				if (iconvbufs(ic_send, &ff_xb, &iobuf.out, flags) < 0)
 					exit_cleanup(RERR_PROTOCOL); /* impossible? */
 				write_buf(iobuf.out_fd, s-1, 1); /* Send the '\0'. */
@@ -427,6 +436,7 @@ static void forward_filesfrom_data(void)
 			ff_lastchar = '\0';
 		else {
 			/* Handle a partial string specially, saving any incomplete chars. */
+			implied_include_partial_string(sob, s);
 			flags &= ~ICB_INCLUDE_INCOMPLETE;
 			if (iconvbufs(ic_send, &ff_xb, &iobuf.out, flags) < 0) {
 				if (errno == E2BIG)
@@ -443,13 +453,17 @@ static void forward_filesfrom_data(void)
 		char *f = ff_xb.buf + ff_xb.pos;
 		char *t = ff_xb.buf;
 		char *eob = f + len;
+		char *cur = t;
 		/* Eliminate any multi-'\0' runs. */
 		while (f != eob) {
 			if (!(*t++ = *f++)) {
+				add_implied_include(cur);
+				cur = t;
 				while (f != eob && *f == '\0')
 					f++;
 			}
 		}
+		implied_include_partial_string(cur, t);
 		ff_lastchar = f[-1];
 		if ((len = t - ff_xb.buf) != 0) {
 			/* This will not circle back to perform_io() because we only get
@@ -463,7 +477,7 @@ void reduce_iobuf_size(xbuf *out, size_t new_size)
 {
 	if (new_size < out->size) {
 		/* Avoid weird buffer interactions by only outputting this to stderr. */
-		if (msgs2stderr && DEBUG_GTE(IO, 4)) {
+		if (msgs2stderr == 1 && DEBUG_GTE(IO, 4)) {
 			const char *name = out == &iobuf.out ? "iobuf.out"
 					 : out == &iobuf.msg ? "iobuf.msg"
 					 : NULL;
@@ -481,7 +495,7 @@ void restore_iobuf_size(xbuf *out)
 	if (IOBUF_WAS_REDUCED(out->size)) {
 		size_t new_size = IOBUF_RESTORE_SIZE(out->size);
 		/* Avoid weird buffer interactions by only outputting this to stderr. */
-		if (msgs2stderr && DEBUG_GTE(IO, 4)) {
+		if (msgs2stderr == 1 && DEBUG_GTE(IO, 4)) {
 			const char *name = out == &iobuf.out ? "iobuf.out"
 					 : out == &iobuf.msg ? "iobuf.msg"
 					 : NULL;
@@ -560,52 +574,59 @@ static char *perform_io(size_t needed, int flags)
 	case PIO_NEED_INPUT:
 		/* We never resize the circular input buffer. */
 		if (iobuf.in.size < needed) {
-			rprintf(FERROR, "need to read %ld bytes, iobuf.in.buf is only %ld bytes.\n",
-				(long)needed, (long)iobuf.in.size);
+			rprintf(FERROR, "need to read %" SIZE_T_FMT_MOD "d bytes,"
+					" iobuf.in.buf is only %" SIZE_T_FMT_MOD "d bytes.\n",
+				(SIZE_T_FMT_CAST)needed, (SIZE_T_FMT_CAST)iobuf.in.size);
 			exit_cleanup(RERR_PROTOCOL);
 		}
 
-		if (msgs2stderr && DEBUG_GTE(IO, 3)) {
-			rprintf(FINFO, "[%s] perform_io(%ld, %sinput)\n",
-				who_am_i(), (long)needed, flags & PIO_CONSUME_INPUT ? "consume&" : "");
+		if (msgs2stderr == 1 && DEBUG_GTE(IO, 3)) {
+			rprintf(FINFO, "[%s] perform_io(%" SIZE_T_FMT_MOD "d, %sinput)\n",
+				who_am_i(), (SIZE_T_FMT_CAST)needed, flags & PIO_CONSUME_INPUT ? "consume&" : "");
 		}
 		break;
 
 	case PIO_NEED_OUTROOM:
 		/* We never resize the circular output buffer. */
 		if (iobuf.out.size - iobuf.out_empty_len < needed) {
-			fprintf(stderr, "need to write %ld bytes, iobuf.out.buf is only %ld bytes.\n",
-				(long)needed, (long)(iobuf.out.size - iobuf.out_empty_len));
+			fprintf(stderr, "need to write %" SIZE_T_FMT_MOD "d bytes,"
+					" iobuf.out.buf is only %" SIZE_T_FMT_MOD "d bytes.\n",
+				(SIZE_T_FMT_CAST)needed, (SIZE_T_FMT_CAST)(iobuf.out.size - iobuf.out_empty_len));
 			exit_cleanup(RERR_PROTOCOL);
 		}
 
-		if (msgs2stderr && DEBUG_GTE(IO, 3)) {
-			rprintf(FINFO, "[%s] perform_io(%ld, outroom) needs to flush %ld\n",
-				who_am_i(), (long)needed,
+		if (msgs2stderr == 1 && DEBUG_GTE(IO, 3)) {
+			rprintf(FINFO, "[%s] perform_io(%" SIZE_T_FMT_MOD "d,"
+				       " outroom) needs to flush %" SIZE_T_FMT_MOD "d\n",
+				who_am_i(), (SIZE_T_FMT_CAST)needed,
 				iobuf.out.len + needed > iobuf.out.size
-				? (long)(iobuf.out.len + needed - iobuf.out.size) : 0L);
+				? (SIZE_T_FMT_CAST)(iobuf.out.len + needed - iobuf.out.size) : (SIZE_T_FMT_CAST)0);
 		}
 		break;
 
 	case PIO_NEED_MSGROOM:
 		/* We never resize the circular message buffer. */
 		if (iobuf.msg.size < needed) {
-			fprintf(stderr, "need to write %ld bytes, iobuf.msg.buf is only %ld bytes.\n",
-				(long)needed, (long)iobuf.msg.size);
+			fprintf(stderr, "need to write %" SIZE_T_FMT_MOD "d bytes,"
+					" iobuf.msg.buf is only %" SIZE_T_FMT_MOD "d bytes.\n",
+				(SIZE_T_FMT_CAST)needed, (SIZE_T_FMT_CAST)iobuf.msg.size);
 			exit_cleanup(RERR_PROTOCOL);
 		}
 
-		if (msgs2stderr && DEBUG_GTE(IO, 3)) {
-			rprintf(FINFO, "[%s] perform_io(%ld, msgroom) needs to flush %ld\n",
-				who_am_i(), (long)needed,
+		if (msgs2stderr == 1 && DEBUG_GTE(IO, 3)) {
+			rprintf(FINFO, "[%s] perform_io(%" SIZE_T_FMT_MOD "d,"
+				       " msgroom) needs to flush %" SIZE_T_FMT_MOD "d\n",
+				who_am_i(), (SIZE_T_FMT_CAST)needed,
 				iobuf.msg.len + needed > iobuf.msg.size
-				? (long)(iobuf.msg.len + needed - iobuf.msg.size) : 0L);
+				? (SIZE_T_FMT_CAST)(iobuf.msg.len + needed - iobuf.msg.size) : (SIZE_T_FMT_CAST)0);
 		}
 		break;
 
 	case 0:
-		if (msgs2stderr && DEBUG_GTE(IO, 3))
-			rprintf(FINFO, "[%s] perform_io(%ld, %d)\n", who_am_i(), (long)needed, flags);
+		if (msgs2stderr == 1 && DEBUG_GTE(IO, 3)) {
+			rprintf(FINFO, "[%s] perform_io(%" SIZE_T_FMT_MOD "d, %d)\n",
+				who_am_i(), (SIZE_T_FMT_CAST)needed, flags);
+		}
 		break;
 
 	default:
@@ -662,9 +683,9 @@ static char *perform_io(size_t needed, int flags)
 					SIVAL(iobuf.out.buf + iobuf.raw_data_header_pos, 0,
 					      ((MPLEX_BASE + (int)MSG_DATA)<<24) + iobuf.out.len - 4);
 
-					if (msgs2stderr && DEBUG_GTE(IO, 1)) {
-						rprintf(FINFO, "[%s] send_msg(%d, %ld)\n",
-							who_am_i(), (int)MSG_DATA, (long)iobuf.out.len - 4);
+					if (msgs2stderr == 1 && DEBUG_GTE(IO, 1)) {
+						rprintf(FINFO, "[%s] send_msg(%d, %" SIZE_T_FMT_MOD "d)\n",
+							who_am_i(), (int)MSG_DATA, (SIZE_T_FMT_CAST)iobuf.out.len - 4);
 					}
 
 					/* reserve room for the next MSG_DATA header */
@@ -755,7 +776,7 @@ static char *perform_io(size_t needed, int flags)
 
 		if (iobuf.in_fd >= 0 && FD_ISSET(iobuf.in_fd, &r_fds)) {
 			size_t len, pos = iobuf.in.pos + iobuf.in.len;
-			int n;
+			ssize_t n;
 			if (pos >= iobuf.in.size) {
 				pos -= iobuf.in.size;
 				len = iobuf.in.size - iobuf.in.len;
@@ -782,12 +803,14 @@ static char *perform_io(size_t needed, int flags)
 					exit_cleanup(RERR_SOCKETIO);
 				}
 			}
-			if (msgs2stderr && DEBUG_GTE(IO, 2))
-				rprintf(FINFO, "[%s] recv=%ld\n", who_am_i(), (long)n);
+			if (msgs2stderr == 1 && DEBUG_GTE(IO, 2)) {
+				rprintf(FINFO, "[%s] recv=%" SIZE_T_FMT_MOD "d\n",
+					who_am_i(), (SIZE_T_FMT_CAST)n);
+			}
 
 			if (io_timeout) {
 				last_io_in = time(NULL);
-				if (flags & PIO_NEED_INPUT)
+				if (io_timeout && flags & PIO_NEED_INPUT)
 					maybe_send_keepalive(last_io_in, 0);
 			}
 			stats.total_read += n;
@@ -795,9 +818,14 @@ static char *perform_io(size_t needed, int flags)
 			iobuf.in.len += n;
 		}
 
+		if (stop_at_utime && time(NULL) >= stop_at_utime) {
+			rprintf(FERROR, "stopping at requested limit\n");
+			exit_cleanup(RERR_TIMEOUT);
+		}
+
 		if (out && FD_ISSET(iobuf.out_fd, &w_fds)) {
 			size_t len = iobuf.raw_flushing_ends_before ? iobuf.raw_flushing_ends_before - out->pos : out->len;
-			int n;
+			ssize_t n;
 
 			if (bwlimit_writemax && len > bwlimit_writemax)
 				len = bwlimit_writemax;
@@ -817,9 +845,9 @@ static char *perform_io(size_t needed, int flags)
 					exit_cleanup(RERR_SOCKETIO);
 				}
 			}
-			if (msgs2stderr && DEBUG_GTE(IO, 2)) {
-				rprintf(FINFO, "[%s] %s sent=%ld\n",
-					who_am_i(), out == &iobuf.out ? "out" : "msg", (long)n);
+			if (msgs2stderr == 1 && DEBUG_GTE(IO, 2)) {
+				rprintf(FINFO, "[%s] %s sent=%" SIZE_T_FMT_MOD "d\n",
+					who_am_i(), out == &iobuf.out ? "out" : "msg", (SIZE_T_FMT_CAST)n);
 			}
 
 			if (io_timeout)
@@ -912,7 +940,11 @@ void noop_io_until_death(void)
 {
 	char buf[1024];
 
-	if (!iobuf.in.buf || !iobuf.out.buf || iobuf.in_fd < 0 || iobuf.out_fd < 0 || kluge_around_eof || msgs2stderr)
+	if (!iobuf.in.buf || !iobuf.out.buf || iobuf.in_fd < 0 || iobuf.out_fd < 0 || kluge_around_eof)
+		return;
+
+	/* If we're talking to a daemon over a socket, don't short-circuit this logic */
+	if (msgs2stderr && daemon_connection >= 0)
 		return;
 
 	kluge_around_eof = 2;
@@ -930,13 +962,15 @@ int send_msg(enum msgcode code, const char *buf, size_t len, int convert)
 {
 	char *hdr;
 	size_t needed, pos;
-	BOOL want_debug = DEBUG_GTE(IO, 1) && convert >= 0 && (msgs2stderr || code != MSG_INFO);
+	BOOL want_debug = DEBUG_GTE(IO, 1) && convert >= 0 && (msgs2stderr == 1 || code != MSG_INFO);
 
 	if (!OUT_MULTIPLEXED)
 		return 0;
 
-	if (want_debug)
-		rprintf(FINFO, "[%s] send_msg(%d, %ld)\n", who_am_i(), (int)code, (long)len);
+	if (want_debug) {
+		rprintf(FINFO, "[%s] send_msg(%d, %" SIZE_T_FMT_MOD "d)\n",
+			who_am_i(), (int)code, (SIZE_T_FMT_CAST)len);
+	}
 
 	/* When checking for enough free space for this message, we need to
 	 * make sure that there is space for the 4-byte header, plus we'll
@@ -952,9 +986,9 @@ int send_msg(enum msgcode code, const char *buf, size_t len, int convert)
 #endif
 		needed = len + 4 + 3;
 	if (iobuf.msg.len + needed > iobuf.msg.size) {
-		if (!am_receiver)
+		if (am_sender)
 			perform_io(needed, PIO_NEED_MSGROOM);
-		else { /* We allow the receiver to increase their iobuf.msg size to avoid a deadlock. */
+		else { /* We sometimes allow the iobuf.msg size to increase to avoid a deadlock. */
 			size_t old_size = iobuf.msg.size;
 			restore_iobuf_size(&iobuf.msg);
 			realloc_xbuf(&iobuf.msg, iobuf.msg.size * 2);
@@ -1011,8 +1045,10 @@ int send_msg(enum msgcode code, const char *buf, size_t len, int convert)
 
 	SIVAL(hdr, 0, ((MPLEX_BASE + (int)code)<<24) + len);
 
-	if (want_debug && convert > 0)
-		rprintf(FINFO, "[%s] converted msg len=%ld\n", who_am_i(), (long)len);
+	if (want_debug && convert > 0) {
+		rprintf(FINFO, "[%s] converted msg len=%" SIZE_T_FMT_MOD "d\n",
+			who_am_i(), (SIZE_T_FMT_CAST)len);
+	}
 
 	return 1;
 }
@@ -1285,7 +1321,7 @@ void read_args(int f_in, char *mod_name, char *buf, size_t bufsiz, int rl_nulls,
 
 BOOL io_start_buffering_out(int f_out)
 {
-	if (msgs2stderr && DEBUG_GTE(IO, 2))
+	if (msgs2stderr == 1 && DEBUG_GTE(IO, 2))
 		rprintf(FINFO, "[%s] io_start_buffering_out(%d)\n", who_am_i(), f_out);
 
 	if (iobuf.out.buf) {
@@ -1304,7 +1340,7 @@ BOOL io_start_buffering_out(int f_out)
 
 BOOL io_start_buffering_in(int f_in)
 {
-	if (msgs2stderr && DEBUG_GTE(IO, 2))
+	if (msgs2stderr == 1 && DEBUG_GTE(IO, 2))
 		rprintf(FINFO, "[%s] io_start_buffering_in(%d)\n", who_am_i(), f_in);
 
 	if (iobuf.in.buf) {
@@ -1323,7 +1359,7 @@ BOOL io_start_buffering_in(int f_in)
 
 void io_end_buffering_in(BOOL free_buffers)
 {
-	if (msgs2stderr && DEBUG_GTE(IO, 2)) {
+	if (msgs2stderr == 1 && DEBUG_GTE(IO, 2)) {
 		rprintf(FINFO, "[%s] io_end_buffering_in(IOBUF_%s_BUFS)\n",
 			who_am_i(), free_buffers ? "FREE" : "KEEP");
 	}
@@ -1338,7 +1374,7 @@ void io_end_buffering_in(BOOL free_buffers)
 
 void io_end_buffering_out(BOOL free_buffers)
 {
-	if (msgs2stderr && DEBUG_GTE(IO, 2)) {
+	if (msgs2stderr == 1 && DEBUG_GTE(IO, 2)) {
 		rprintf(FINFO, "[%s] io_end_buffering_out(IOBUF_%s_BUFS)\n",
 			who_am_i(), free_buffers ? "FREE" : "KEEP");
 	}
@@ -1426,8 +1462,10 @@ static void read_a_msg(void)
 	msg_bytes = tag & 0xFFFFFF;
 	tag = (tag >> 24) - MPLEX_BASE;
 
-	if (DEBUG_GTE(IO, 1) && msgs2stderr)
-		rprintf(FINFO, "[%s] got msg=%d, len=%ld\n", who_am_i(), (int)tag, (long)msg_bytes);
+	if (msgs2stderr == 1 && DEBUG_GTE(IO, 1)) {
+		rprintf(FINFO, "[%s] got msg=%d, len=%" SIZE_T_FMT_MOD "d\n",
+			who_am_i(), (int)tag, (SIZE_T_FMT_CAST)msg_bytes);
+	}
 
 	switch (tag) {
 	case MSG_DATA:
@@ -1603,8 +1641,10 @@ static void read_a_msg(void)
 		else
 			goto invalid_msg;
 		iobuf.in_multiplexed = 1;
-		if (DEBUG_GTE(EXIT, 3))
-			rprintf(FINFO, "[%s] got MSG_ERROR_EXIT with %ld bytes\n", who_am_i(), (long)msg_bytes);
+		if (DEBUG_GTE(EXIT, 3)) {
+			rprintf(FINFO, "[%s] got MSG_ERROR_EXIT with %" SIZE_T_FMT_MOD "d bytes\n",
+					who_am_i(), (SIZE_T_FMT_CAST)msg_bytes);
+		}
 		if (msg_bytes == 0) {
 			if (!am_sender && !am_generator) {
 				if (DEBUG_GTE(EXIT, 3)) {
@@ -1810,6 +1850,7 @@ int64 read_longint(int f)
 #endif
 }
 
+/* Debugging note: this will be named read_buf_() when using an external zlib. */
 void read_buf(int f, char *buf, size_t len)
 {
 	if (f != iobuf.in_fd) {
@@ -2298,7 +2339,7 @@ void io_start_multiplex_out(int fd)
 {
 	io_flush(FULL_FLUSH);
 
-	if (msgs2stderr && DEBUG_GTE(IO, 2))
+	if (msgs2stderr == 1 && DEBUG_GTE(IO, 2))
 		rprintf(FINFO, "[%s] io_start_multiplex_out(%d)\n", who_am_i(), fd);
 
 	if (!iobuf.msg.buf)
@@ -2315,7 +2356,7 @@ void io_start_multiplex_out(int fd)
 /* Setup for multiplexing a MSG_* stream with the data stream. */
 void io_start_multiplex_in(int fd)
 {
-	if (msgs2stderr && DEBUG_GTE(IO, 2))
+	if (msgs2stderr == 1 && DEBUG_GTE(IO, 2))
 		rprintf(FINFO, "[%s] io_start_multiplex_in(%d)\n", who_am_i(), fd);
 
 	iobuf.in_multiplexed = 1; /* See also IN_MULTIPLEXED */
@@ -2326,7 +2367,7 @@ int io_end_multiplex_in(int mode)
 {
 	int ret = iobuf.in_multiplexed ? iobuf.in_fd : -1;
 
-	if (msgs2stderr && DEBUG_GTE(IO, 2))
+	if (msgs2stderr == 1 && DEBUG_GTE(IO, 2))
 		rprintf(FINFO, "[%s] io_end_multiplex_in(mode=%d)\n", who_am_i(), mode);
 
 	iobuf.in_multiplexed = 0;
@@ -2344,7 +2385,7 @@ int io_end_multiplex_out(int mode)
 {
 	int ret = iobuf.out_empty_len ? iobuf.out_fd : -1;
 
-	if (msgs2stderr && DEBUG_GTE(IO, 2))
+	if (msgs2stderr == 1 && DEBUG_GTE(IO, 2))
 		rprintf(FINFO, "[%s] io_end_multiplex_out(mode=%d)\n", who_am_i(), mode);
 
 	if (mode != MPLX_TO_BUFFERED)

itypes.h

@@ -1,6 +1,6 @@
 /* Inline functions for rsync.
  *
- * Copyright (C) 2007-2020 Wayne Davison
+ * Copyright (C) 2007-2022 Wayne Davison
  *
  * This program is free software; you can redistribute it and/or modify
  * it under the terms of the GNU General Public License as published by
@@ -40,6 +40,12 @@ isSpace(const char *ptr)
 	return isspace(*(unsigned char *)ptr);
 }
 
+static inline int
+isAlNum(const char *ptr)
+{
+	return isalnum(*(unsigned char *)ptr);
+}
+
 static inline int
 isLower(const char *ptr)
 {

latest-year.h

@@ -1 +1 @@
-#define LATEST_YEAR "2020"
+#define LATEST_YEAR "2022"

lib/compat.c

@@ -24,16 +24,24 @@
 
 static char number_separator;
 
-#ifndef HAVE_STRDUP
- char *strdup(char *s)
+char get_number_separator(void)
 {
-	int len = strlen(s) + 1;
-	char *ret = (char *)malloc(len);
-	if (ret)
-		memcpy(ret, s, len);
-	return ret;
+	if (!number_separator) {
+		char buf[32];
+		snprintf(buf, sizeof buf, "%f", 3.14);
+		if (strchr(buf, '.') != NULL)
+			number_separator = ',';
+		else
+			number_separator = '.';
+	}
+
+	return number_separator;
+}
+
+char get_decimal_point(void)
+{
+	return get_number_separator() == ',' ? '.' : ',';
 }
-#endif
 
 #ifndef HAVE_GETCWD
  char *getcwd(char *buf, int size)
@@ -155,30 +163,6 @@ int sys_gettimeofday(struct timeval *tv)
 #endif
 }
 
-#define HUMANIFY(mult) \
-	do { \
-		if (num >= mult || num <= -mult) { \
-			double dnum = (double)num / mult; \
-			char units; \
-			if (num < 0) \
-				dnum = -dnum; \
-			if (dnum < mult) \
-				units = 'K'; \
-			else if ((dnum /= mult) < mult) \
-				units = 'M'; \
-			else if ((dnum /= mult) < mult) \
-				units = 'G'; \
-			else { \
-				dnum /= mult; \
-				units = 'T'; \
-			} \
-			if (num < 0) \
-				dnum = -dnum; \
-			snprintf(bufs[n], sizeof bufs[0], "%.2f%c", dnum, units); \
-			return bufs[n]; \
-		} \
-	} while (0)
-
 /* Return the int64 number as a string.  If the human_flag arg is non-zero,
  * we may output the number in K, M, G, or T units.  If we don't add a unit
  * suffix, we will append the fract string, if it is non-NULL.  We can
@@ -190,22 +174,35 @@ char *do_big_num(int64 num, int human_flag, const char *fract)
 	char *s;
 	int len, negated;
 
-	if (human_flag && !number_separator) {
-		char buf[32];
-		snprintf(buf, sizeof buf, "%f", 3.14);
-		if (strchr(buf, '.') != NULL)
-			number_separator = ',';
-		else
-			number_separator = '.';
-	}
+	if (human_flag && !number_separator)
+		(void)get_number_separator();
 
 	n = (n + 1) % (sizeof bufs / sizeof bufs[0]);
 
 	if (human_flag > 1) {
-		if (human_flag == 2)
-			HUMANIFY(1000);
-		else
-			HUMANIFY(1024);
+		int mult = human_flag == 2 ? 1000 : 1024;
+		if (num >= mult || num <= -mult) {
+			double dnum = (double)num / mult;
+			char units;
+			if (num < 0)
+				dnum = -dnum;
+			if (dnum < mult)
+				units = 'K';
+			else if ((dnum /= mult) < mult)
+				units = 'M';
+			else if ((dnum /= mult) < mult)
+				units = 'G';
+			else if ((dnum /= mult) < mult)
+				units = 'T';
+			else {
+				dnum /= mult;
+				units = 'P';
+			}
+			if (num < 0)
+				dnum = -dnum;
+			snprintf(bufs[n], sizeof bufs[0], "%.2f%c", dnum, units);
+			return bufs[n];
+		}
 	}
 
 	s = bufs[n] + sizeof bufs[0] - 1;

lib/md5-asm-x86_64.S

@@ -27,7 +27,7 @@
 #include "config.h"
 #include "md-defines.h"
 
-#if !defined USE_OPENSSL && CSUM_CHUNK == 64
+#ifdef USE_MD5_ASM /* { */
 
 #ifdef __APPLE__
 #define md5_process_asm _md5_process_asm
@@ -698,4 +698,4 @@ md5_process_asm:
 	pop	%rbp
 	ret
 
-#endif /* !USE_OPENSSL ... */
+#endif /* } USE_MD5_ASM */

lib/md5.c

@@ -2,7 +2,7 @@
  * RFC 1321 compliant MD5 implementation
  *
  * Copyright (C) 2001-2003 Christophe Devine
- * Copyright (C) 2007-2020 Wayne Davison
+ * Copyright (C) 2007-2022 Wayne Davison
  *
  * This program is free software; you can redistribute it and/or modify
  * it under the terms of the GNU General Public License as published by
@@ -20,7 +20,7 @@
 
 #include "rsync.h"
 
-#ifndef USE_OPENSSL
+#if !defined USE_OPENSSL || USE_MD5_ASM /* { */
 void md5_begin(md_context *ctx)
 {
 	ctx->A = 0x67452301;
@@ -148,7 +148,10 @@ static void md5_process(md_context *ctx, const uchar data[CSUM_CHUNK])
 	ctx->D += D;
 }
 
-#if defined HAVE_ASM && CSUM_CHUNK == 64
+#ifdef USE_MD5_ASM
+#if CSUM_CHUNK != 64
+#error The MD5 ASM code does not support CSUM_CHUNK != 64
+#endif
 extern void md5_process_asm(md_context *ctx, const void *data, size_t num);
 #endif
 
@@ -176,20 +179,20 @@ void md5_update(md_context *ctx, const uchar *input, uint32 length)
 		left = 0;
 	}
 
-#if defined HAVE_ASM && CSUM_CHUNK == 64
+#ifdef USE_MD5_ASM /* { */
 	if (length >= CSUM_CHUNK) {
 		uint32 chunks = length / CSUM_CHUNK;
 		md5_process_asm(ctx, input, chunks);
 		length -= chunks * CSUM_CHUNK;
 		input += chunks * CSUM_CHUNK;
 	}
-#else
+#else /* } { */
 	while (length >= CSUM_CHUNK) {
 		md5_process(ctx, input);
 		length -= CSUM_CHUNK;
 		input  += CSUM_CHUNK;
 	}
-#endif
+#endif /* } */
 
 	if (length)
 		memcpy(ctx->buffer + left, input, length);
@@ -221,9 +224,9 @@ void md5_result(md_context *ctx, uchar digest[MD5_DIGEST_LEN])
 	SIVALu(digest, 8, ctx->C);
 	SIVALu(digest, 12, ctx->D);
 }
-#endif
+#endif /* } */
 
-#ifdef TEST_MD5
+#ifdef TEST_MD5 /* { */
 
 void get_md5(uchar *out, const uchar *input, int n)
 {
@@ -317,4 +320,4 @@ int main(int argc, char *argv[])
 	return 0;
 }
 
-#endif
+#endif /* } */

lib/mdigest.h

@@ -17,12 +17,13 @@ void mdfour_begin(md_context *md);
 void mdfour_update(md_context *md, const uchar *in, uint32 length);
 void mdfour_result(md_context *md, uchar digest[MD4_DIGEST_LEN]);
 
-#ifndef USE_OPENSSL
-#define MD5_CTX md_context
-#define MD5_Init md5_begin
-#define MD5_Update md5_update
-#define MD5_Final(digest, cptr) md5_result(cptr, digest)
-
+#if defined USE_OPENSSL && !defined USE_MD5_ASM
+#define md5_context MD5_CTX
+#define md5_begin MD5_Init
+#define md5_update MD5_Update
+#define md5_result(cptr, digest) MD5_Final(digest, cptr)
+#else
+#define md5_context md_context
 void md5_begin(md_context *ctx);
 void md5_update(md_context *ctx, const uchar *input, uint32 length);
 void md5_result(md_context *ctx, uchar digest[MD5_DIGEST_LEN]);

lib/pool_alloc.3

@@ -33,7 +33,7 @@ pool_alloc, pool_free, pool_free_old, pool_talloc, pool_tfree, pool_create, pool
 .SH SYNOPSIS
 .B #include "pool_alloc.h"
 
-\fBstruct alloc_pool *pool_create(size_t \fIsize\fB, size_t \fIquantum\fB, void (*\fIbomb\fB)(char *), int \fIflags\fB);
+\fBstruct alloc_pool *pool_create(size_t \fIsize\fB, size_t \fIquantum\fB, void (*\fIbomb\fB)(char*,char*,int), int \fIflags\fB);
 
 \fBvoid pool_destroy(struct alloc_pool *\fIpool\fB);
 

lib/pool_alloc.c

@@ -9,8 +9,7 @@ struct alloc_pool
 	size_t			size;		/* extent size		*/
 	size_t			quantum;	/* allocation quantum	*/
 	struct pool_extent	*extents;	/* top extent is "live" */
-	void			(*bomb)();	/* function to call if
-						 * malloc fails		*/
+	void			(*bomb)();	/* called if malloc fails */
 	int			flags;
 
 	/* statistical data */
@@ -45,13 +44,13 @@ struct align_test {
 #define PTR_ADD(b,o)	( (void*) ((char*)(b) + (o)) )
 
 alloc_pool_t
-pool_create(size_t size, size_t quantum, void (*bomb)(const char *), int flags)
+pool_create(size_t size, size_t quantum, void (*bomb)(const char*, const char*, int), int flags)
 {
 	struct alloc_pool *pool;
 
-	if ((MINALIGN & (MINALIGN - 1)) != 0) {
+	if ((MINALIGN & (MINALIGN - 1)) != (0)) {
 		if (bomb)
-			(*bomb)("Compiler error: MINALIGN is not a power of 2\n");
+			(*bomb)("Compiler error: MINALIGN is not a power of 2", __FILE__, __LINE__);
 		return NULL;
 	}
 
@@ -169,7 +168,7 @@ pool_alloc(alloc_pool_t p, size_t len, const char *bomb_msg)
 
   bomb_out:
 	if (pool->bomb)
-		(*pool->bomb)(bomb_msg);
+		(*pool->bomb)(bomb_msg, __FILE__, __LINE__);
 	return NULL;
 }
 

lib/pool_alloc.h

@@ -7,7 +7,7 @@
 
 typedef void *alloc_pool_t;
 
-alloc_pool_t pool_create(size_t size, size_t quantum, void (*bomb)(const char *), int flags);
+alloc_pool_t pool_create(size_t size, size_t quantum, void (*bomb)(const char*, const char*, int), int flags);
 void pool_destroy(alloc_pool_t pool);
 void *pool_alloc(alloc_pool_t pool, size_t size, const char *bomb_msg);
 void pool_free(alloc_pool_t pool, size_t size, void *addr);

lib/snprintf.c

@@ -20,7 +20,7 @@
  * for string length.  This covers a nasty loophole.
  *
  * The other functions are there to prevent NULL pointers from
- * causing nast effects.
+ * causing nasty effects.
  *
  * More Recently:
  *  Brandon Long <blong@fiction.net> 9/15/96 for mutt 0.43

lib/sysacls.h

@@ -3,7 +3,7 @@
  * Version 2.2.x
  * Portable SMB ACL interface
  * Copyright (C) Jeremy Allison 2000
- * Copyright (C) 2007-2019 Wayne Davison
+ * Copyright (C) 2007-2020 Wayne Davison
  *
  * This program is free software; you can redistribute it and/or modify
  * it under the terms of the GNU General Public License as published by
@@ -139,7 +139,9 @@ typedef struct acl *SMB_ACL_ENTRY_T;
 
 /* Based on the Solaris & UnixWare code. */
 
+#ifndef __TANDEM
 #undef GROUP
+#endif
 #include <sys/aclv.h>
 
 /* SVR4.2 ES/MP ACLs */

lib/sysxattrs.c

@@ -2,7 +2,7 @@
  * Extended attribute support for rsync.
  *
  * Copyright (C) 2004 Red Hat, Inc.
- * Copyright (C) 2003-2019 Wayne Davison
+ * Copyright (C) 2003-2022 Wayne Davison
  * Written by Jay Fenlason.
  *
  * This program is free software; you can redistribute it and/or modify
@@ -67,7 +67,7 @@ ssize_t sys_lgetxattr(const char *path, const char *name, void *value, size_t si
 		u_int32_t offset = len;
 		size_t data_retrieved = len;
 		while (data_retrieved < size) {
-			len = getxattr(path, name, value + offset, size - data_retrieved, offset, XATTR_NOFOLLOW);
+			len = getxattr(path, name, (char*)value + offset, size - data_retrieved, offset, XATTR_NOFOLLOW);
 			if (len <= 0)
 				break;
 			data_retrieved += len;
@@ -167,7 +167,7 @@ static ssize_t read_xattr(int attrfd, void *buf, size_t buflen)
 	} else {
 		size_t bufpos;
 		for (bufpos = 0; bufpos < sb.st_size; ) {
-			ssize_t cnt = read(attrfd, buf + bufpos, sb.st_size - bufpos);
+			ssize_t cnt = read(attrfd, (char*)buf + bufpos, sb.st_size - bufpos);
 			if (cnt <= 0) {
 				if (cnt < 0 && errno == EINTR)
 					continue;
@@ -218,7 +218,7 @@ int sys_lsetxattr(const char *path, const char *name, const void *value, size_t
 		return -1;
 
 	for (bufpos = 0; bufpos < size; ) {
-		ssize_t cnt = write(attrfd, value+bufpos, size);
+		ssize_t cnt = write(attrfd, (char*)value + bufpos, size);
 		if (cnt <= 0) {
 			if (cnt < 0 && errno == EINTR)
 				continue;
@@ -274,7 +274,8 @@ ssize_t sys_llistxattr(const char *path, char *list, size_t size)
 		 && (dp->d_name[10] == 'o' || dp->d_name[10] == 'w'))
 			continue;
 
-		if ((ret += len+1) > size) {
+		ret += len + 1;
+		if ((size_t)ret > size) {
 			if (size == 0)
 				continue;
 			ret = -1;

loadparm.c

@@ -48,7 +48,6 @@
 extern item_list dparam_list;
 
 #define strequal(a, b) (strcasecmp(a, b)==0)
-#define BOOLSTR(b) ((b) ? "Yes" : "No")
 
 #ifndef LOG_DAEMON
 #define LOG_DAEMON 0
@@ -56,7 +55,7 @@ extern item_list dparam_list;
 
 /* the following are used by loadparm for option lists */
 typedef enum {
-	P_BOOL, P_BOOLREV, P_CHAR, P_INTEGER,
+	P_BOOL, P_BOOLREV, P_BOOL3, P_CHAR, P_INTEGER,
 	P_OCTAL, P_PATH, P_STRING, P_ENUM
 } parm_type;
 
@@ -87,234 +86,6 @@ struct parm_struct {
 #define LP_SNUM_OK(i) ((i) >= 0 && (i) < (int)section_list.count)
 #define SECTION_PTR(s, p) (((char*)(s)) + (ptrdiff_t)(((char*)(p))-(char*)&Vars.l))
 
-/* This structure describes global (ie., server-wide) parameters. */
-typedef struct {
-	char *bind_address;
-	char *daemon_chroot;
-	char *daemon_gid;
-	char *daemon_uid;
-	char *motd_file;
-	char *pid_file;
-	char *socket_options;
-
-	/* Each _EXP var tracks if the associated char* var has been expanded yet or not. */
-	BOOL bind_address_EXP;
-	BOOL daemon_chroot_EXP;
-	BOOL daemon_gid_EXP;
-	BOOL daemon_uid_EXP;
-	BOOL motd_file_EXP;
-	BOOL pid_file_EXP;
-	BOOL socket_options_EXP;
-
-	int listen_backlog;
-	int rsync_port;
-
-	BOOL proxy_protocol;
-} global_vars;
-
-/* This structure describes a single section.  Their order must match the
- * initializers below, which you can accomplish by keeping each sub-section
- * sorted.  (e.g. in vim, just visually select each subsection and use !sort.)
- * NOTE: the char* variables MUST all remain at the start of the struct! */
-typedef struct {
-	char *auth_users;
-	char *charset;
-	char *comment;
-	char *dont_compress;
-	char *early_exec;
-	char *exclude;
-	char *exclude_from;
-	char *filter;
-	char *gid;
-	char *hosts_allow;
-	char *hosts_deny;
-	char *include;
-	char *include_from;
-	char *incoming_chmod;
-	char *lock_file;
-	char *log_file;
-	char *log_format;
-	char *name;
-	char *outgoing_chmod;
-	char *path;
-	char *postxfer_exec;
-	char *prexfer_exec;
-	char *refuse_options;
-	char *secrets_file;
-	char *syslog_tag;
-	char *temp_dir;
-	char *uid;
-
-	/* Each _EXP var tracks if the associated char* var has been expanded yet or not. */
-	BOOL auth_users_EXP;
-	BOOL charset_EXP;
-	BOOL comment_EXP;
-	BOOL dont_compress_EXP;
-	BOOL early_exec_EXP;
-	BOOL exclude_EXP;
-	BOOL exclude_from_EXP;
-	BOOL filter_EXP;
-	BOOL gid_EXP;
-	BOOL hosts_allow_EXP;
-	BOOL hosts_deny_EXP;
-	BOOL include_EXP;
-	BOOL include_from_EXP;
-	BOOL incoming_chmod_EXP;
-	BOOL lock_file_EXP;
-	BOOL log_file_EXP;
-	BOOL log_format_EXP;
-	BOOL name_EXP;
-	BOOL outgoing_chmod_EXP;
-	BOOL path_EXP;
-	BOOL postxfer_exec_EXP;
-	BOOL prexfer_exec_EXP;
-	BOOL refuse_options_EXP;
-	BOOL secrets_file_EXP;
-	BOOL syslog_tag_EXP;
-	BOOL temp_dir_EXP;
-	BOOL uid_EXP;
-
-	int max_connections;
-	int max_verbosity;
-	int syslog_facility;
-	int timeout;
-
-	BOOL fake_super;
-	BOOL forward_lookup;
-	BOOL ignore_errors;
-	BOOL ignore_nonreadable;
-	BOOL list;
-	BOOL munge_symlinks;
-	BOOL numeric_ids;
-	BOOL read_only;
-	BOOL reverse_lookup;
-	BOOL strict_modes;
-	BOOL transfer_logging;
-	BOOL use_chroot;
-	BOOL write_only;
-} local_vars;
-
-/* This structure describes the global variables (g) as well as the globally
- * specified values of the local variables (l), which are used when modules
- * don't specify their own values. */
-typedef struct {
-	global_vars g;
-	local_vars l;
-} all_vars;
-
-/* The application defaults for all the variables.  "Defaults" is
- * used to re-initialize "Vars" before each config-file read.
- *
- * In order to keep these sorted in the same way as the structure
- * above, use the variable name in the leading comment, including a
- * trailing ';' (to avoid a sorting problem with trailing digits). */
-static const all_vars Defaults = {
- /* ==== global_vars ==== */
- {
- /* bind_address; */		NULL,
- /* daemon_chroot; */		NULL,
- /* daemon_gid; */		NULL,
- /* daemon_uid; */		NULL,
- /* motd_file; */		NULL,
- /* pid_file; */		NULL,
- /* socket_options; */		NULL,
-
- /* bind_address_EXP; */	False,
- /* daemon_chroot_EXP; */	False,
- /* daemon_gid_EXP; */		False,
- /* daemon_uid_EXP; */		False,
- /* motd_file_EXP; */		False,
- /* pid_file_EXP; */		False,
- /* socket_options_EXP; */	False,
-
- /* listen_backlog; */		5,
- /* rsync_port; */		0,
-
- /* proxy_protocol; */		False,
- },
-
- /* ==== local_vars ==== */
- {
- /* auth_users; */		NULL,
- /* charset; */ 		NULL,
- /* comment; */ 		NULL,
- /* dont_compress; */		DEFAULT_DONT_COMPRESS,
- /* early_exec; */		NULL,
- /* exclude; */ 		NULL,
- /* exclude_from; */		NULL,
- /* filter; */			NULL,
- /* gid; */			NULL,
- /* hosts_allow; */		NULL,
- /* hosts_deny; */		NULL,
- /* include; */			NULL,
- /* include_from; */		NULL,
- /* incoming_chmod; */		NULL,
- /* lock_file; */		DEFAULT_LOCK_FILE,
- /* log_file; */		NULL,
- /* log_format; */		"%o %h [%a] %m (%u) %f %l",
- /* name; */			NULL,
- /* outgoing_chmod; */		NULL,
- /* path; */			NULL,
- /* postxfer_exec; */		NULL,
- /* prexfer_exec; */		NULL,
- /* refuse_options; */		NULL,
- /* secrets_file; */		NULL,
- /* syslog_tag; */		"rsyncd",
- /* temp_dir; */ 		NULL,
- /* uid; */			NULL,
-
- /* auth_users_EXP; */		False,
- /* charset_EXP; */		False,
- /* comment_EXP; */		False,
- /* dont_compress_EXP; */	False,
- /* early_exec_EXP; */		False,
- /* exclude_EXP; */		False,
- /* exclude_from_EXP; */	False,
- /* filter_EXP; */		False,
- /* gid_EXP; */			False,
- /* hosts_allow_EXP; */		False,
- /* hosts_deny_EXP; */		False,
- /* include_EXP; */		False,
- /* include_from_EXP; */	False,
- /* incoming_chmod_EXP; */	False,
- /* lock_file_EXP; */		False,
- /* log_file_EXP; */		False,
- /* log_format_EXP; */		False,
- /* name_EXP; */		False,
- /* outgoing_chmod_EXP; */	False,
- /* path_EXP; */		False,
- /* postxfer_exec_EXP; */	False,
- /* prexfer_exec_EXP; */	False,
- /* refuse_options_EXP; */	False,
- /* secrets_file_EXP; */	False,
- /* syslog_tag_EXP; */		False,
- /* temp_dir_EXP; */		False,
- /* uid_EXP; */			False,
-
- /* max_connections; */		0,
- /* max_verbosity; */		1,
- /* syslog_facility; */		LOG_DAEMON,
- /* timeout; */			0,
-
- /* fake_super; */		False,
- /* forward_lookup; */		True,
- /* ignore_errors; */		False,
- /* ignore_nonreadable; */	False,
- /* list; */			True,
- /* munge_symlinks; */		(BOOL)-1,
- /* numeric_ids; */		(BOOL)-1,
- /* read_only; */		True,
- /* reverse_lookup; */		True,
- /* strict_modes; */		True,
- /* transfer_logging; */	False,
- /* use_chroot; */		True,
- /* write_only; */		False,
- }
-};
-
-/* The currently configured values for all the variables. */
-static all_vars Vars;
-
 /* Stack of "Vars" values used by the &include directive. */
 static item_list Vars_stack = EMPTY_ITEM_LIST;
 
@@ -324,9 +95,7 @@ static item_list section_list = EMPTY_ITEM_LIST;
 static int iSectionIndex = -1;
 static BOOL bInGlobalSection = True;
 
-#define NUMPARAMETERS (sizeof (parm_table) / sizeof (struct parm_struct))
-
-static struct enum_list enum_facilities[] = {
+static struct enum_list enum_syslog_facility[] = {
 #ifdef LOG_AUTH
 	{ LOG_AUTH, "auth" },
 #endif
@@ -393,95 +162,27 @@ static struct enum_list enum_facilities[] = {
 	{ -1, NULL }
 };
 
-static struct parm_struct parm_table[] =
-{
- {"address",           P_STRING, P_GLOBAL,&Vars.g.bind_address,        NULL,0},
- {"daemon chroot",     P_STRING, P_GLOBAL,&Vars.g.daemon_chroot,       NULL,0},
- {"daemon gid",        P_STRING, P_GLOBAL,&Vars.g.daemon_gid,          NULL,0},
- {"daemon uid",        P_STRING, P_GLOBAL,&Vars.g.daemon_uid,          NULL,0},
- {"listen backlog",    P_INTEGER,P_GLOBAL,&Vars.g.listen_backlog,      NULL,0},
- {"motd file",         P_STRING, P_GLOBAL,&Vars.g.motd_file,           NULL,0},
- {"pid file",          P_STRING, P_GLOBAL,&Vars.g.pid_file,            NULL,0},
- {"port",              P_INTEGER,P_GLOBAL,&Vars.g.rsync_port,          NULL,0},
- {"proxy protocol",    P_BOOL,   P_LOCAL, &Vars.g.proxy_protocol,      NULL,0},
- {"socket options",    P_STRING, P_GLOBAL,&Vars.g.socket_options,      NULL,0},
-
- {"auth users",        P_STRING, P_LOCAL, &Vars.l.auth_users,          NULL,0},
- {"charset",           P_STRING, P_LOCAL, &Vars.l.charset,             NULL,0},
- {"comment",           P_STRING, P_LOCAL, &Vars.l.comment,             NULL,0},
- {"dont compress",     P_STRING, P_LOCAL, &Vars.l.dont_compress,       NULL,0},
- {"early exec",        P_STRING, P_LOCAL, &Vars.l.early_exec,          NULL,0},
- {"exclude from",      P_STRING, P_LOCAL, &Vars.l.exclude_from,        NULL,0},
- {"exclude",           P_STRING, P_LOCAL, &Vars.l.exclude,             NULL,0},
- {"fake super",        P_BOOL,   P_LOCAL, &Vars.l.fake_super,          NULL,0},
- {"filter",            P_STRING, P_LOCAL, &Vars.l.filter,              NULL,0},
- {"forward lookup",    P_BOOL,   P_LOCAL, &Vars.l.forward_lookup,      NULL,0},
- {"gid",               P_STRING, P_LOCAL, &Vars.l.gid,                 NULL,0},
- {"hosts allow",       P_STRING, P_LOCAL, &Vars.l.hosts_allow,         NULL,0},
- {"hosts deny",        P_STRING, P_LOCAL, &Vars.l.hosts_deny,          NULL,0},
- {"ignore errors",     P_BOOL,   P_LOCAL, &Vars.l.ignore_errors,       NULL,0},
- {"ignore nonreadable",P_BOOL,   P_LOCAL, &Vars.l.ignore_nonreadable,  NULL,0},
- {"include from",      P_STRING, P_LOCAL, &Vars.l.include_from,        NULL,0},
- {"include",           P_STRING, P_LOCAL, &Vars.l.include,             NULL,0},
- {"incoming chmod",    P_STRING, P_LOCAL, &Vars.l.incoming_chmod,      NULL,0},
- {"list",              P_BOOL,   P_LOCAL, &Vars.l.list,                NULL,0},
- {"lock file",         P_STRING, P_LOCAL, &Vars.l.lock_file,           NULL,0},
- {"log file",          P_STRING, P_LOCAL, &Vars.l.log_file,            NULL,0},
- {"log format",        P_STRING, P_LOCAL, &Vars.l.log_format,          NULL,0},
- {"max connections",   P_INTEGER,P_LOCAL, &Vars.l.max_connections,     NULL,0},
- {"max verbosity",     P_INTEGER,P_LOCAL, &Vars.l.max_verbosity,       NULL,0},
- {"munge symlinks",    P_BOOL,   P_LOCAL, &Vars.l.munge_symlinks,      NULL,0},
- {"name",              P_STRING, P_LOCAL, &Vars.l.name,                NULL,0},
- {"numeric ids",       P_BOOL,   P_LOCAL, &Vars.l.numeric_ids,         NULL,0},
- {"outgoing chmod",    P_STRING, P_LOCAL, &Vars.l.outgoing_chmod,      NULL,0},
- {"path",              P_PATH,   P_LOCAL, &Vars.l.path,                NULL,0},
-#ifdef HAVE_PUTENV
- {"post-xfer exec",    P_STRING, P_LOCAL, &Vars.l.postxfer_exec,       NULL,0},
- {"pre-xfer exec",     P_STRING, P_LOCAL, &Vars.l.prexfer_exec,        NULL,0},
-#endif
- {"read only",         P_BOOL,   P_LOCAL, &Vars.l.read_only,           NULL,0},
- {"refuse options",    P_STRING, P_LOCAL, &Vars.l.refuse_options,      NULL,0},
- {"reverse lookup",    P_BOOL,   P_LOCAL, &Vars.l.reverse_lookup,      NULL,0},
- {"secrets file",      P_STRING, P_LOCAL, &Vars.l.secrets_file,        NULL,0},
- {"strict modes",      P_BOOL,   P_LOCAL, &Vars.l.strict_modes,        NULL,0},
- {"syslog facility",   P_ENUM,   P_LOCAL, &Vars.l.syslog_facility,     enum_facilities,0},
- {"syslog tag",        P_STRING, P_LOCAL, &Vars.l.syslog_tag,          NULL,0},
- {"temp dir",          P_PATH,   P_LOCAL, &Vars.l.temp_dir,            NULL,0},
- {"timeout",           P_INTEGER,P_LOCAL, &Vars.l.timeout,             NULL,0},
- {"transfer logging",  P_BOOL,   P_LOCAL, &Vars.l.transfer_logging,    NULL,0},
- {"uid",               P_STRING, P_LOCAL, &Vars.l.uid,                 NULL,0},
- {"use chroot",        P_BOOL,   P_LOCAL, &Vars.l.use_chroot,          NULL,0},
- {"write only",        P_BOOL,   P_LOCAL, &Vars.l.write_only,          NULL,0},
- {NULL,                P_BOOL,   P_NONE,  NULL,                        NULL,0}
-};
-
-/* Initialise the Default all_vars structure. */
-void reset_daemon_vars(void)
-{
-	memcpy(&Vars, &Defaults, sizeof Vars);
-}
-
 /* Expand %VAR% references.  Any unknown vars or unrecognized
  * syntax leaves the raw chars unchanged. */
-static char *expand_vars(char *str)
+static char *expand_vars(const char *str)
 {
-	char *buf, *t, *f;
+	char *buf, *t;
+	const char *f;
 	int bufsize;
 
 	if (!str || !strchr(str, '%'))
-		return str;
+		return (char *)str; /* TODO change return value to const char* at some point. */
 
 	bufsize = strlen(str) + 2048;
 	buf = new_array(char, bufsize+1); /* +1 for trailing '\0' */
 
 	for (t = buf, f = str; bufsize && *f; ) {
-		if (*f == '%' && *++f != '%') {
-			char *percent = strchr(f, '%');
-			if (percent) {
+		if (*f == '%' && isUpper(f+1)) {
+			char *percent = strchr(f+1, '%');
+			if (percent && percent - f < bufsize) {
 				char *val;
-				*percent = '\0';
-				val = getenv(f);
-				*percent = '%';
+				strlcpy(t, f+1, percent - f);
+				val = getenv(t);
 				if (val) {
 					int len = strlcpy(t, val, bufsize+1);
 					if (len > bufsize)
@@ -492,7 +193,6 @@ static char *expand_vars(char *str)
 					continue;
 				}
 			}
-			f--;
 		}
 		*t++ = *f++;
 		bufsize--;
@@ -510,6 +210,8 @@ static char *expand_vars(char *str)
 	return buf;
 }
 
+/* Each "char* foo" has an associated "BOOL foo_EXP" that tracks if the string has been expanded yet or not. */
+
 /* NOTE: use this function and all the FN_{GLOBAL,LOCAL} ones WITHOUT a trailing semicolon! */
 #define RETURN_EXPANDED(val) {if (!val ## _EXP) {val = expand_vars(val); val ## _EXP = True;} return val ? val : "";}
 
@@ -534,65 +236,24 @@ static char *expand_vars(char *str)
 #define FN_LOCAL_INTEGER(fn_name, val) \
  int fn_name(int i) {return LP_SNUM_OK(i)? iSECTION(i).val : Vars.l.val;}
 
-FN_GLOBAL_STRING(lp_bind_address, bind_address)
-FN_GLOBAL_STRING(lp_daemon_chroot, daemon_chroot)
-FN_GLOBAL_STRING(lp_daemon_gid, daemon_gid)
-FN_GLOBAL_STRING(lp_daemon_uid, daemon_uid)
-FN_GLOBAL_STRING(lp_motd_file, motd_file)
-FN_GLOBAL_STRING(lp_pid_file, pid_file)
-FN_GLOBAL_STRING(lp_socket_options, socket_options)
+/* The following include file contains:
+ *
+ * typedef global_vars - describes global (ie., server-wide) parameters.
+ * typedef local_vars - describes a single section.
+ * typedef all_vars - a combination of global_vars & local_vars.
+ * all_vars Defaults - the default values for all the variables.
+ * all_vars Vars - the currently configured values for all the variables.
+ * struct parm_struct parm_table - the strings & variables for the parser.
+ * FN_{LOCAL,GLOBAL}_{TYPE}() definition for all the lp_var_name() accessors.
+ */
 
-FN_GLOBAL_INTEGER(lp_listen_backlog, listen_backlog)
-FN_GLOBAL_INTEGER(lp_rsync_port, rsync_port)
+#include "daemon-parm.h"
 
-FN_GLOBAL_BOOL(lp_proxy_protocol, proxy_protocol)
-
-FN_LOCAL_STRING(lp_auth_users, auth_users)
-FN_LOCAL_STRING(lp_charset, charset)
-FN_LOCAL_STRING(lp_comment, comment)
-FN_LOCAL_STRING(lp_dont_compress, dont_compress)
-FN_LOCAL_STRING(lp_early_exec, early_exec)
-FN_LOCAL_STRING(lp_exclude, exclude)
-FN_LOCAL_STRING(lp_exclude_from, exclude_from)
-FN_LOCAL_STRING(lp_filter, filter)
-FN_LOCAL_STRING(lp_gid, gid)
-FN_LOCAL_STRING(lp_hosts_allow, hosts_allow)
-FN_LOCAL_STRING(lp_hosts_deny, hosts_deny)
-FN_LOCAL_STRING(lp_include, include)
-FN_LOCAL_STRING(lp_include_from, include_from)
-FN_LOCAL_STRING(lp_incoming_chmod, incoming_chmod)
-FN_LOCAL_STRING(lp_lock_file, lock_file)
-FN_LOCAL_STRING(lp_log_file, log_file)
-FN_LOCAL_STRING(lp_log_format, log_format)
-FN_LOCAL_STRING(lp_name, name)
-FN_LOCAL_STRING(lp_outgoing_chmod, outgoing_chmod)
-FN_LOCAL_STRING(lp_path, path)
-FN_LOCAL_STRING(lp_postxfer_exec, postxfer_exec)
-FN_LOCAL_STRING(lp_prexfer_exec, prexfer_exec)
-FN_LOCAL_STRING(lp_refuse_options, refuse_options)
-FN_LOCAL_STRING(lp_secrets_file, secrets_file)
-FN_LOCAL_STRING(lp_syslog_tag, syslog_tag)
-FN_LOCAL_STRING(lp_temp_dir, temp_dir)
-FN_LOCAL_STRING(lp_uid, uid)
-
-FN_LOCAL_INTEGER(lp_max_connections, max_connections)
-FN_LOCAL_INTEGER(lp_max_verbosity, max_verbosity)
-FN_LOCAL_INTEGER(lp_syslog_facility, syslog_facility)
-FN_LOCAL_INTEGER(lp_timeout, timeout)
-
-FN_LOCAL_BOOL(lp_fake_super, fake_super)
-FN_LOCAL_BOOL(lp_forward_lookup, forward_lookup)
-FN_LOCAL_BOOL(lp_ignore_errors, ignore_errors)
-FN_LOCAL_BOOL(lp_ignore_nonreadable, ignore_nonreadable)
-FN_LOCAL_BOOL(lp_list, list)
-FN_LOCAL_BOOL(lp_munge_symlinks, munge_symlinks)
-FN_LOCAL_BOOL(lp_numeric_ids, numeric_ids)
-FN_LOCAL_BOOL(lp_read_only, read_only)
-FN_LOCAL_BOOL(lp_reverse_lookup, reverse_lookup)
-FN_LOCAL_BOOL(lp_strict_modes, strict_modes)
-FN_LOCAL_BOOL(lp_transfer_logging, transfer_logging)
-FN_LOCAL_BOOL(lp_use_chroot, use_chroot)
-FN_LOCAL_BOOL(lp_write_only, write_only)
+/* Initialise the Default all_vars structure. */
+void reset_daemon_vars(void)
+{
+	memcpy(&Vars, &Defaults, sizeof Vars);
+}
 
 /* Assign a copy of v to *s.  Handles NULL strings.  We don't worry
  * about overwriting a malloc'd string because the long-running
@@ -617,19 +278,14 @@ static void init_section(local_vars *psection)
 	copy_section(psection, &Vars.l);
 }
 
-/* Do a case-insensitive, whitespace-ignoring string compare. */
-static int strwicmp(char *psz1, char *psz2)
+/* Do a case-insensitive, whitespace-ignoring string equality check. */
+static int strwiEQ(char *psz1, char *psz2)
 {
-	/* if BOTH strings are NULL, return TRUE, if ONE is NULL return */
-	/* appropriate value. */
+	/* If one or both strings are NULL, we return equality right away. */
 	if (psz1 == psz2)
-		return 0;
-
-	if (psz1 == NULL)
-		return -1;
-
-	if (psz2 == NULL)
 		return 1;
+	if (psz1 == NULL || psz2 == NULL)
+		return 0;
 
 	/* sync the strings on first non-whitespace */
 	while (1) {
@@ -637,12 +293,14 @@ static int strwicmp(char *psz1, char *psz2)
 			psz1++;
 		while (isSpace(psz2))
 			psz2++;
-		if (toUpper(psz1) != toUpper(psz2) || *psz1 == '\0' || *psz2 == '\0')
+		if (*psz1 == '\0' || *psz2 == '\0')
+			break;
+		if (toUpper(psz1) != toUpper(psz2))
 			break;
 		psz1++;
 		psz2++;
 	}
-	return *psz1 - *psz2;
+	return *psz1 == *psz2;
 }
 
 /* Find a section by name. Otherwise works like get_section. */
@@ -651,7 +309,7 @@ static int getsectionbyname(char *name)
 	int i;
 
 	for (i = section_list.count - 1; i >= 0; i--) {
-		if (strwicmp(iSECTION(i).name, name) == 0)
+		if (strwiEQ(iSECTION(i).name, name))
 			break;
 	}
 
@@ -691,7 +349,7 @@ static int map_parameter(char *parmname)
 		return -1;
 
 	for (iIndex = 0; parm_table[iIndex].label; iIndex++) {
-		if (strwicmp(parm_table[iIndex].label, parmname) == 0)
+		if (strwiEQ(parm_table[iIndex].label, parmname))
 			return iIndex;
 	}
 
@@ -702,16 +360,14 @@ static int map_parameter(char *parmname)
 /* Set a boolean variable from the text value stored in the passed string.
  * Returns True in success, False if the passed string does not correctly
  * represent a boolean. */
-static BOOL set_boolean(BOOL *pb, char *parmvalue)
+static BOOL set_boolean(BOOL *pb, char *parmvalue, int allow_unset)
 {
-	if (strwicmp(parmvalue, "yes") == 0
-	 || strwicmp(parmvalue, "true") == 0
-	 || strwicmp(parmvalue, "1") == 0)
+	if (strwiEQ(parmvalue, "yes") || strwiEQ(parmvalue, "true") || strwiEQ(parmvalue, "1"))
 		*pb = True;
-	else if (strwicmp(parmvalue, "no") == 0
-	      || strwicmp(parmvalue, "False") == 0
-	      || strwicmp(parmvalue, "0") == 0)
+	else if (strwiEQ(parmvalue, "no") || strwiEQ(parmvalue, "false") || strwiEQ(parmvalue, "0"))
 		*pb = False;
+	else if (allow_unset && (strwiEQ(parmvalue, "unset") || strwiEQ(parmvalue, "-1")))
+		*pb = Unset;
 	else {
 		rprintf(FLOG, "Badly formed boolean in configuration file: \"%s\".\n", parmvalue);
 		return False;
@@ -760,11 +416,15 @@ static BOOL do_parameter(char *parmname, char *parmvalue)
 
 	switch (parm_table[parmnum].type) {
 	case P_BOOL:
-		set_boolean(parm_ptr, parmvalue);
+		set_boolean(parm_ptr, parmvalue, False);
+		break;
+
+	case P_BOOL3:
+		set_boolean(parm_ptr, parmvalue, True);
 		break;
 
 	case P_BOOLREV:
-		set_boolean(parm_ptr, parmvalue);
+		set_boolean(parm_ptr, parmvalue, False);
 		*(BOOL *)parm_ptr = ! *(BOOL *)parm_ptr;
 		break;
 
@@ -777,7 +437,7 @@ static BOOL do_parameter(char *parmname, char *parmvalue)
 		break;
 
 	case P_OCTAL:
-		sscanf(parmvalue, "%o", (int *)parm_ptr);
+		sscanf(parmvalue, "%o", (unsigned int *)parm_ptr);
 		break;
 
 	case P_PATH:
@@ -834,7 +494,7 @@ static BOOL do_section(char *sectionname)
 		return True;
 	}
 
-	isglobal = strwicmp(sectionname, GLOBAL_NAME) == 0;
+	isglobal = strwiEQ(sectionname, GLOBAL_NAME);
 
 	/* At the end of the global section, add any --dparam items. */
 	if (bInGlobalSection && !isglobal) {

log.c

@@ -3,7 +3,7 @@
  *
  * Copyright (C) 1998-2001 Andrew Tridgell <tridge@samba.org>
  * Copyright (C) 2000-2001 Martin Pool <mbp@samba.org>
- * Copyright (C) 2003-2020 Wayne Davison
+ * Copyright (C) 2003-2022 Wayne Davison
  *
  * This program is free software; you can redistribute it and/or modify
  * it under the terms of the GNU General Public License as published by
@@ -34,7 +34,7 @@ extern int module_id;
 extern int allow_8bit_chars;
 extern int protocol_version;
 extern int always_checksum;
-extern int preserve_times;
+extern int preserve_mtimes;
 extern int msgs2stderr;
 extern int xfersum_type;
 extern int checksum_type;
@@ -251,7 +251,7 @@ static void filtered_fwrite(FILE *f, const char *in_buf, int in_len, int use_isp
 void rwrite(enum logcode code, const char *buf, int len, int is_utf8)
 {
 	char trailing_CR_or_NL;
-	FILE *f = msgs2stderr ? stderr : stdout;
+	FILE *f = msgs2stderr == 1 ? stderr : stdout;
 #ifdef ICONV_OPTION
 	iconv_t ic = is_utf8 && ic_recv != (iconv_t)-1 ? ic_recv : ic_chck;
 #else
@@ -263,7 +263,7 @@ void rwrite(enum logcode code, const char *buf, int len, int is_utf8)
 	if (len < 0)
 		exit_cleanup(RERR_MESSAGEIO);
 
-	if (msgs2stderr) {
+	if (msgs2stderr == 1) {
 		/* A normal daemon can get msgs2stderr set if the socket is busted, so we
 		 * change the message destination into an FLOG message in order to try to
 		 * get some info about an abnormal-exit into the log file. An rsh daemon
@@ -327,7 +327,7 @@ void rwrite(enum logcode code, const char *buf, int len, int is_utf8)
 		exit_cleanup(RERR_MESSAGEIO);
 	}
 
-	if (am_server && !msgs2stderr) {
+	if (am_server && msgs2stderr != 1 && (msgs2stderr != 2 || f != stderr)) {
 		enum msgcode msg = (enum msgcode)code;
 		if (protocol_version < 30) {
 			if (msg == MSG_ERROR)
@@ -350,8 +350,7 @@ void rwrite(enum logcode code, const char *buf, int len, int is_utf8)
 		output_needs_newline = 0;
 	}
 
-	trailing_CR_or_NL = len && (buf[len-1] == '\n' || buf[len-1] == '\r')
-			  ? buf[--len] : 0;
+	trailing_CR_or_NL = len && (buf[len-1] == '\n' || buf[len-1] == '\r') ? buf[--len] : '\0';
 
 	if (len && buf[0] == '\r') {
 		fputc('\r', f);
@@ -372,7 +371,12 @@ void rwrite(enum logcode code, const char *buf, int len, int is_utf8)
 			iconvbufs(ic, &inbuf, &outbuf, inbuf.pos ? 0 : ICB_INIT);
 			ierrno = errno;
 			if (outbuf.len) {
-				filtered_fwrite(f, convbuf, outbuf.len, 0, 0);
+				char trailing = inbuf.len ? '\0' : trailing_CR_or_NL;
+				filtered_fwrite(f, convbuf, outbuf.len, 0, trailing);
+				if (trailing) {
+					trailing_CR_or_NL = '\0';
+					fflush(f);
+				}
 				outbuf.len = 0;
 			}
 			/* Log one byte of illegal/incomplete sequence and continue with
@@ -702,7 +706,7 @@ static void log_formatted(enum logcode code, const char *format, const char *op,
 				c[1] = 'L';
 				c[3] = '.';
 				c[4] = !(iflags & ITEM_REPORT_TIME) ? '.'
-				     : !preserve_times || !receiver_symlink_times
+				     : !preserve_mtimes || !receiver_symlink_times
 				    || (iflags & ITEM_REPORT_TIMEFAIL) ? 'T' : 't';
 			} else {
 				c[1] = S_ISDIR(file->mode) ? 'd'
@@ -710,14 +714,15 @@ static void log_formatted(enum logcode code, const char *format, const char *op,
 				     : IS_DEVICE(file->mode) ? 'D' : 'f';
 				c[3] = !(iflags & ITEM_REPORT_SIZE) ? '.' : 's';
 				c[4] = !(iflags & ITEM_REPORT_TIME) ? '.'
-				     : !preserve_times ? 'T' : 't';
+				     : !preserve_mtimes ? 'T' : 't';
 			}
 			c[2] = !(iflags & ITEM_REPORT_CHANGE) ? '.' : 'c';
 			c[5] = !(iflags & ITEM_REPORT_PERMS) ? '.' : 'p';
 			c[6] = !(iflags & ITEM_REPORT_OWNER) ? '.' : 'o';
 			c[7] = !(iflags & ITEM_REPORT_GROUP) ? '.' : 'g';
-			c[8] = !(iflags & ITEM_REPORT_ATIME) ? '.'
-			     : S_ISLNK(file->mode) ? 'U' : 'u';
+			c[8] = !(iflags & (ITEM_REPORT_ATIME|ITEM_REPORT_CRTIME)) ? '.'
+			     : BITS_SET(iflags, ITEM_REPORT_ATIME|ITEM_REPORT_CRTIME) ? 'b'
+			     : iflags & ITEM_REPORT_ATIME ? 'u' : 'n';
 			c[9] = !(iflags & ITEM_REPORT_ACL) ? '.' : 'a';
 			c[10] = !(iflags & ITEM_REPORT_XATTR) ? '.' : 'x';
 			c[11] = '\0';
@@ -833,14 +838,24 @@ void maybe_log_item(struct file_struct *file, int iflags, int itemizing, const c
 
 void log_delete(const char *fname, int mode)
 {
-	static struct {
-		union file_extras ex[4]; /* just in case... */
-		struct file_struct file;
-	} x; /* Zero-initialized due to static declaration. */
+	static struct file_struct *file = NULL;
 	int len = strlen(fname);
 	const char *fmt;
 
-	x.file.mode = mode;
+	if (!file) {
+		int extra_len = (file_extra_cnt + 2) * EXTRA_LEN;
+		char *bp;
+#if EXTRA_ROUNDING > 0
+		if (extra_len & (EXTRA_ROUNDING * EXTRA_LEN))
+			extra_len = (extra_len | (EXTRA_ROUNDING * EXTRA_LEN)) + EXTRA_LEN;
+#endif
+
+		bp = new_array0(char, FILE_STRUCT_LEN + extra_len + 1);
+		bp += extra_len;
+		file = (struct file_struct *)bp;
+	}
+
+	file->mode = mode;
 
 	if (am_server && protocol_version >= 29 && len < MAXPATHLEN) {
 		if (S_ISDIR(mode))
@@ -850,14 +865,14 @@ void log_delete(const char *fname, int mode)
 		;
 	else {
 		fmt = stdout_format_has_o_or_i ? stdout_format : "deleting %n";
-		log_formatted(FCLIENT, fmt, "del.", &x.file, fname, ITEM_DELETED, NULL);
+		log_formatted(FCLIENT, fmt, "del.", file, fname, ITEM_DELETED, NULL);
 	}
 
 	if (!logfile_name || dry_run || !logfile_format)
 		return;
 
 	fmt = logfile_format_has_o_or_i ? logfile_format : "deleting %n";
-	log_formatted(FLOG, fmt, "del.", &x.file, fname, ITEM_DELETED, NULL);
+	log_formatted(FLOG, fmt, "del.", file, fname, ITEM_DELETED, NULL);
 }
 
 /*
@@ -886,10 +901,10 @@ void log_exit(int code, const char *file, int line)
 		/* VANISHED is not an error, only a warning */
 		if (code == RERR_VANISHED) {
 			rprintf(FWARNING, "rsync warning: %s (code %d) at %s(%d) [%s=%s]\n",
-				name, code, file, line, who_am_i(), RSYNC_VERSION);
+				name, code, src_file(file), line, who_am_i(), rsync_version());
 		} else {
 			rprintf(FERROR, "rsync error: %s (code %d) at %s(%d) [%s=%s]\n",
-				name, code, file, line, who_am_i(), RSYNC_VERSION);
+				name, code, src_file(file), line, who_am_i(), rsync_version());
 		}
 	}
 }

main.c

@@ -4,7 +4,7 @@
  * Copyright (C) 1996-2001 Andrew Tridgell <tridge@samba.org>
  * Copyright (C) 1996 Paul Mackerras
  * Copyright (C) 2001, 2002 Martin Pool <mbp@samba.org>
- * Copyright (C) 2003-2020 Wayne Davison
+ * Copyright (C) 2003-2022 Wayne Davison
  *
  * This program is free software; you can redistribute it and/or modify
  * it under the terms of the GNU General Public License as published by
@@ -28,6 +28,9 @@
 #include <locale.h>
 #endif
 #include <popt.h>
+#ifdef __TANDEM
+#include <floss.h(floss_execlp)>
+#endif
 
 extern int dry_run;
 extern int list_only;
@@ -45,6 +48,7 @@ extern int called_from_signal_handler;
 extern int need_messages_from_generator;
 extern int kluge_around_eof;
 extern int got_xfer_error;
+extern int old_style_args;
 extern int msgs2stderr;
 extern int module_id;
 extern int read_only;
@@ -54,6 +58,7 @@ extern int copy_unsafe_links;
 extern int keep_dirlinks;
 extern int preserve_hard_links;
 extern int protocol_version;
+extern int mkpath_dest_arg;
 extern int file_total;
 extern int recurse;
 extern int xfer_dirs;
@@ -83,6 +88,8 @@ extern BOOL shutting_down;
 extern int backup_dir_len;
 extern int basis_dir_cnt;
 extern int default_af_hint;
+extern int stdout_format_has_i;
+extern int trust_sender_filter;
 extern struct stats stats;
 extern char *stdout_format;
 extern char *logfile_format;
@@ -93,18 +100,19 @@ extern char *shell_cmd;
 extern char *password_file;
 extern char *backup_dir;
 extern char *copy_as;
+extern char *tmpdir;
 extern char curr_dir[MAXPATHLEN];
 extern char backup_dir_buf[MAXPATHLEN];
 extern char *basis_dir[MAX_BASIS_DIRS+1];
 extern struct file_list *first_flist;
-extern filter_rule_list daemon_filter_list;
+extern filter_rule_list daemon_filter_list, implied_filter_list;
 
 uid_t our_uid;
 gid_t our_gid;
 int am_receiver = 0;  /* Only set to 1 after the receiver/generator fork. */
 int am_generator = 0; /* Only set to 1 after the receiver/generator fork. */
 int local_server = 0;
-int daemon_over_rsh = 0;
+int daemon_connection = 0; /* 0 = no daemon, 1 = daemon via remote shell, -1 = daemon via socket */
 mode_t orig_umask = 0;
 int batch_gen_fd = -1;
 int sender_keeps_checksum = 0;
@@ -299,7 +307,7 @@ static void become_copy_as_user()
 
 	our_uid = MY_UID();
 	our_gid = MY_GID();
-	am_root = (our_uid == 0);
+	am_root = (our_uid == ROOT_UID);
 
 	if (gname)
 		gname[-1] = ':';
@@ -461,38 +469,33 @@ static void output_summary(void)
  **/
 static void show_malloc_stats(void)
 {
-#ifdef HAVE_MALLINFO
-	struct mallinfo mi;
-
-	mi = mallinfo();
+#ifdef MEM_ALLOC_INFO
+	struct MEM_ALLOC_INFO mi = MEM_ALLOC_INFO(); /* mallinfo or mallinfo2 */
 
 	rprintf(FCLIENT, "\n");
 	rprintf(FINFO, RSYNC_NAME "[%d] (%s%s%s) heap statistics:\n",
 		(int)getpid(), am_server ? "server " : "",
 		am_daemon ? "daemon " : "", who_am_i());
-	rprintf(FINFO, "  arena:     %10ld   (bytes from sbrk)\n",
-		(long)mi.arena);
-	rprintf(FINFO, "  ordblks:   %10ld   (chunks not in use)\n",
-		(long)mi.ordblks);
-	rprintf(FINFO, "  smblks:    %10ld\n",
-		(long)mi.smblks);
-	rprintf(FINFO, "  hblks:     %10ld   (chunks from mmap)\n",
-		(long)mi.hblks);
-	rprintf(FINFO, "  hblkhd:    %10ld   (bytes from mmap)\n",
-		(long)mi.hblkhd);
-	rprintf(FINFO, "  allmem:    %10ld   (bytes from sbrk + mmap)\n",
-		(long)mi.arena + mi.hblkhd);
-	rprintf(FINFO, "  usmblks:   %10ld\n",
-		(long)mi.usmblks);
-	rprintf(FINFO, "  fsmblks:   %10ld\n",
-		(long)mi.fsmblks);
-	rprintf(FINFO, "  uordblks:  %10ld   (bytes used)\n",
-		(long)mi.uordblks);
-	rprintf(FINFO, "  fordblks:  %10ld   (bytes free)\n",
-		(long)mi.fordblks);
-	rprintf(FINFO, "  keepcost:  %10ld   (bytes in releasable chunk)\n",
-		(long)mi.keepcost);
-#endif /* HAVE_MALLINFO */
+
+#define PRINT_ALLOC_NUM(title, descr, num) \
+	rprintf(FINFO, "  %-11s%10" SIZE_T_FMT_MOD "d   (" descr ")\n", \
+		title ":", (SIZE_T_FMT_CAST)(num));
+
+	PRINT_ALLOC_NUM("arena", "bytes from sbrk", mi.arena);
+	PRINT_ALLOC_NUM("ordblks", "chunks not in use", mi.ordblks);
+	PRINT_ALLOC_NUM("smblks", "free fastbin blocks", mi.smblks);
+	PRINT_ALLOC_NUM("hblks", "chunks from mmap", mi.hblks);
+	PRINT_ALLOC_NUM("hblkhd", "bytes from mmap", mi.hblkhd);
+	PRINT_ALLOC_NUM("allmem", "bytes from sbrk + mmap", mi.arena + mi.hblkhd);
+	PRINT_ALLOC_NUM("usmblks", "always 0", mi.usmblks);
+	PRINT_ALLOC_NUM("fsmblks", "bytes in freed fastbin blocks", mi.fsmblks);
+	PRINT_ALLOC_NUM("uordblks", "bytes used", mi.uordblks);
+	PRINT_ALLOC_NUM("fordblks", "bytes free", mi.fordblks);
+	PRINT_ALLOC_NUM("keepcost", "bytes in releasable chunk", mi.keepcost);
+
+#undef PRINT_ALLOC_NUM
+
+#endif /* MEM_ALLOC_INFO */
 }
 
 
@@ -562,12 +565,12 @@ static pid_t do_cmd(char *cmd, char *machine, char *user, char **remote_argv, in
 #ifdef HAVE_REMSH
 		/* remsh (on HPUX) takes the arguments the other way around */
 		args[argc++] = machine;
-		if (user && !(daemon_over_rsh && dash_l_set)) {
+		if (user && !(daemon_connection && dash_l_set)) {
 			args[argc++] = "-l";
 			args[argc++] = user;
 		}
 #else
-		if (user && !(daemon_over_rsh && dash_l_set)) {
+		if (user && !(daemon_connection && dash_l_set)) {
 			args[argc++] = "-l";
 			args[argc++] = user;
 		}
@@ -587,7 +590,11 @@ static pid_t do_cmd(char *cmd, char *machine, char *user, char **remote_argv, in
 		if (blocking_io < 0 && (strcmp(t, "rsh") == 0 || strcmp(t, "remsh") == 0))
 			blocking_io = 1;
 
-		server_options(args, &argc);
+		if (daemon_connection > 0) {
+			args[argc++] = "--server";
+			args[argc++] = "--daemon";
+		} else
+			server_options(args, &argc);
 
 		if (argc >= MAX_ARGS - 2)
 			goto arg_overflow;
@@ -595,18 +602,14 @@ static pid_t do_cmd(char *cmd, char *machine, char *user, char **remote_argv, in
 
 	args[argc++] = ".";
 
-	if (!daemon_over_rsh) {
+	if (!daemon_connection) {
 		while (remote_argc > 0) {
 			if (argc >= MAX_ARGS - 1) {
 			  arg_overflow:
 				rprintf(FERROR, "internal: args[] overflowed in do_cmd()\n");
 				exit_cleanup(RERR_SYNTAX);
 			}
-			if (**remote_argv == '-') {
-				if (asprintf(args + argc++, "./%s", *remote_argv++) < 0)
-					out_of_memory("do_cmd");
-			} else
-				args[argc++] = *remote_argv++;
+			args[argc++] = safe_arg(NULL, *remote_argv++);
 			remote_argc--;
 		}
 	}
@@ -633,6 +636,7 @@ static pid_t do_cmd(char *cmd, char *machine, char *user, char **remote_argv, in
 #ifdef ICONV_CONST
 		setup_iconv();
 #endif
+		trust_sender_filter = 1;
 	} else if (local_server) {
 		/* If the user didn't request --[no-]whole-file, force
 		 * it on, but only if we're not batch processing. */
@@ -648,7 +652,7 @@ static pid_t do_cmd(char *cmd, char *machine, char *user, char **remote_argv, in
 #ifdef ICONV_CONST
 		setup_iconv();
 #endif
-		if (protect_args && !daemon_over_rsh)
+		if (protect_args && !daemon_connection)
 			send_protected_args(*f_out_p, args);
 	}
 
@@ -674,7 +678,7 @@ static pid_t do_cmd(char *cmd, char *machine, char *user, char **remote_argv, in
 static char *get_local_name(struct file_list *flist, char *dest_path)
 {
 	STRUCT_STAT st;
-	int statret;
+	int statret, trailing_slash;
 	char *cp;
 
 	if (DEBUG_GTE(RECV, 1)) {
@@ -707,7 +711,29 @@ static char *get_local_name(struct file_list *flist, char *dest_path)
 	}
 
 	/* See what currently exists at the destination. */
-	if ((statret = do_stat(dest_path, &st)) == 0) {
+	statret = do_stat(dest_path, &st);
+	cp = strrchr(dest_path, '/');
+	trailing_slash = cp && !cp[1];
+
+	if (mkpath_dest_arg && statret < 0 && (cp || file_total > 1)) {
+		int save_errno = errno;
+		int ret = make_path(dest_path, file_total > 1 && !trailing_slash ? 0 : MKP_DROP_NAME);
+		if (ret < 0)
+			goto mkdir_error;
+		if (ret && (INFO_GTE(NAME, 1) || stdout_format_has_i)) {
+			if (file_total == 1 || trailing_slash)
+				*cp = '\0';
+			rprintf(FINFO, "created %d director%s for %s\n", ret, ret == 1 ? "y" : "ies", dest_path);
+			if (file_total == 1 || trailing_slash)
+				*cp = '/';
+		}
+		if (ret)
+			statret = do_stat(dest_path, &st);
+		else
+			errno = save_errno;
+	}
+
+	if (statret == 0) {
 		/* If the destination is a dir, enter it and use mode 1. */
 		if (S_ISDIR(st.st_mode)) {
 			if (!change_dir(dest_path, CD_NORMAL)) {
@@ -737,15 +763,12 @@ static char *get_local_name(struct file_list *flist, char *dest_path)
 		exit_cleanup(RERR_FILESELECT);
 	}
 
-	cp = strrchr(dest_path, '/');
-
 	/* If we need a destination directory because the transfer is not
 	 * of a single non-directory or the user has requested one via a
 	 * destination path ending in a slash, create one and use mode 1. */
-	if (file_total > 1 || (cp && !cp[1])) {
-		/* Lop off the final slash (if any). */
-		if (cp && !cp[1])
-			*cp = '\0';
+	if (file_total > 1 || trailing_slash) {
+		if (trailing_slash)
+			*cp = '\0'; /* Lop off the final slash (if any). */
 
 		if (statret == 0) {
 			rprintf(FERROR, "ERROR: destination path is not a directory\n");
@@ -753,6 +776,7 @@ static char *get_local_name(struct file_list *flist, char *dest_path)
 		}
 
 		if (do_mkdir(dest_path, ACCESSPERMS) != 0) {
+		    mkdir_error:
 			rsyserr(FERROR, errno, "mkdir %s failed",
 				full_fname(dest_path));
 			exit_cleanup(RERR_FILEIO);
@@ -762,7 +786,7 @@ static char *get_local_name(struct file_list *flist, char *dest_path)
 		 && strcmp(flist->files[flist->low]->basename, ".") == 0)
 			flist->files[0]->flags |= FLAG_DIR_CREATED;
 
-		if (INFO_GTE(NAME, 1))
+		if (INFO_GTE(NAME, 1) || stdout_format_has_i)
 			rprintf(FINFO, "created directory %s\n", dest_path);
 
 		if (dry_run) {
@@ -977,6 +1001,23 @@ static int do_recv(int f_in, int f_out, char *local_name)
 			backup_dir_buf[backup_dir_len-1] = '/';
 	}
 
+	if (tmpdir) {
+		STRUCT_STAT st;
+		int ret = do_stat(tmpdir, &st);
+		if (ret < 0 || !S_ISDIR(st.st_mode)) {
+			if (ret == 0) {
+				rprintf(FERROR, "The temp-dir is not a directory: %s\n", tmpdir);
+				exit_cleanup(RERR_SYNTAX);
+			}
+			if (errno == ENOENT) {
+				rprintf(FERROR, "The temp-dir does not exist: %s\n", tmpdir);
+				exit_cleanup(RERR_SYNTAX);
+			}
+			rprintf(FERROR, "Failed to stat temp-dir %s: %s\n", tmpdir, strerror(errno));
+			exit_cleanup(RERR_FILEIO);
+		}
+	}
+
 	io_flush(FULL_FLUSH);
 
 	if ((pid = do_fork()) == -1) {
@@ -1037,6 +1078,7 @@ static int do_recv(int f_in, int f_out, char *local_name)
 	}
 
 	am_generator = 1;
+	implied_filter_list.head = implied_filter_list.tail = NULL;
 	flist_receiving_enabled = True;
 
 	io_end_multiplex_in(MPLX_SWITCHING);
@@ -1083,7 +1125,7 @@ static void do_server_recv(int f_in, int f_out, int argc, char *argv[])
 	char *local_name = NULL;
 	int negated_levels;
 
-	if (filesfrom_fd >= 0 && !msgs2stderr && protocol_version < 31) {
+	if (filesfrom_fd >= 0 && msgs2stderr != 1 && protocol_version < 31) {
 		/* We can't mix messages with files-from data on the socket,
 		 * so temporarily turn off info/debug messages. */
 		negate_output_levels();
@@ -1386,7 +1428,7 @@ static int start_client(int argc, char *argv[])
 			}
 			am_sender = 0;
 			if (rsync_port)
-				daemon_over_rsh = shell_cmd ? 1 : -1;
+				daemon_connection = shell_cmd ? 1 : -1;
 		} else { /* source is local, check dest arg */
 			am_sender = 1;
 
@@ -1418,7 +1460,7 @@ static int start_client(int argc, char *argv[])
 			} else { /* hostspec was found, so dest is remote */
 				argv[argc] = path;
 				if (rsync_port)
-					daemon_over_rsh = shell_cmd ? 1 : -1;
+					daemon_connection = shell_cmd ? 1 : -1;
 			}
 		}
 	} else {  /* read_batch */
@@ -1432,6 +1474,10 @@ static int start_client(int argc, char *argv[])
 		rsync_port = 0;
 	}
 
+	/* A local transfer doesn't unbackslash anything, so leave the args alone. */
+	if (local_server)
+		old_style_args = 2;
+
 	if (!rsync_port && remote_argc && !**remote_argv) /* Turn an empty arg into a dot dir. */
 		*remote_argv = ".";
 
@@ -1439,8 +1485,15 @@ static int start_client(int argc, char *argv[])
 		char *dummy_host;
 		int dummy_port = rsync_port;
 		int i;
+		if (!argv[0][0])
+			goto invalid_empty;
 		/* For local source, extra source args must not have hostspec. */
 		for (i = 1; i < argc; i++) {
+			if (!argv[i][0]) {
+			    invalid_empty:
+				rprintf(FERROR, "Empty source arg specified.\n");
+				exit_cleanup(RERR_SYNTAX);
+			}
 			if (check_for_hostspec(argv[i], &dummy_host, &dummy_port)) {
 				rprintf(FERROR, "Unexpected remote arg: %s\n", argv[i]);
 				exit_cleanup(RERR_SYNTAX);
@@ -1450,6 +1503,8 @@ static int start_client(int argc, char *argv[])
 		char *dummy_host;
 		int dummy_port = rsync_port;
 		int i;
+		if (filesfrom_fd < 0)
+			add_implied_include(remote_argv[0]);
 		/* For remote source, any extra source args must have either
 		 * the same hostname or an empty hostname. */
 		for (i = 1; i < remote_argc; i++) {
@@ -1473,6 +1528,7 @@ static int start_client(int argc, char *argv[])
 			if (!rsync_port && !*arg) /* Turn an empty arg into a dot dir. */
 				arg = ".";
 			remote_argv[i] = arg;
+			add_implied_include(arg);
 		}
 	}
 
@@ -1481,10 +1537,10 @@ static int start_client(int argc, char *argv[])
 	else
 		env_port = rsync_port;
 
-	if (daemon_over_rsh < 0)
+	if (daemon_connection < 0)
 		return start_socket_client(shell_machine, remote_argc, remote_argv, argc, argv);
 
-	if (password_file && !daemon_over_rsh) {
+	if (password_file && !daemon_connection) {
 		rprintf(FERROR, "The --password-file option may only be "
 				"used when accessing an rsync daemon.\n");
 		exit_cleanup(RERR_SYNTAX);
@@ -1512,15 +1568,17 @@ static int start_client(int argc, char *argv[])
 	}
 
 #ifdef HAVE_PUTENV
-	if (daemon_over_rsh)
+	if (daemon_connection)
 		set_env_num("RSYNC_PORT", env_port);
+#else
+	(void)env_port;
 #endif
 
 	pid = do_cmd(shell_cmd, shell_machine, shell_user, remote_argv, remote_argc, &f_in, &f_out);
 
 	/* if we're running an rsync server on the remote host over a
 	 * remote shell command, we need to do the RSYNCD protocol first */
-	if (daemon_over_rsh) {
+	if (daemon_connection) {
 		int tmpret;
 		tmpret = start_inband_exchange(f_in, f_out, shell_user, remote_argc, remote_argv);
 		if (tmpret < 0)
@@ -1552,11 +1610,13 @@ static void sigusr2_handler(UNUSED(int val))
 	_exit(0);
 }
 
+#if defined SIGINFO || defined SIGVTALRM
 static void siginfo_handler(UNUSED(int val))
 {
 	if (!am_server && !INFO_GTE(PROGRESS, 1))
 		want_progress_now = True;
 }
+#endif
 
 void remember_children(UNUSED(int val))
 {
@@ -1584,7 +1644,6 @@ void remember_children(UNUSED(int val))
 #endif
 }
 
-
 /**
  * This routine catches signals and tries to send them to gdb.
  *
@@ -1608,7 +1667,6 @@ const char *get_panic_action(void)
 	return "xterm -display :0 -T Panic -n Panic -e gdb /proc/%d/exe %d";
 }
 
-
 /**
  * Handle a fatal signal by launching a debugger, controlled by $RSYNC_PANIC_ACTION.
  *
@@ -1632,6 +1690,22 @@ static void rsync_panic_handler(UNUSED(int whatsig))
 }
 #endif
 
+static void unset_env_var(const char *var)
+{
+#ifdef HAVE_UNSETENV
+	unsetenv(var);
+#else
+#ifdef HAVE_PUTENV
+	char *mem;
+	if (asprintf(&mem, "%s=", var) < 0)
+		out_of_memory("unset_env_var");
+	putenv(mem);
+#else
+	(void)var;
+#endif
+#endif
+}
+
 
 int main(int argc,char *argv[])
 {
@@ -1667,7 +1741,9 @@ int main(int argc,char *argv[])
 	starttime = time(NULL);
 	our_uid = MY_UID();
 	our_gid = MY_GID();
-	am_root = our_uid == 0;
+	am_root = our_uid == ROOT_UID;
+
+	unset_env_var("DISPLAY");
 
 	memset(&stats, 0, sizeof(stats));
 
@@ -1687,6 +1763,7 @@ int main(int argc,char *argv[])
 
 #if defined CONFIG_LOCALE && defined HAVE_SETLOCALE
 	setlocale(LC_CTYPE, "");
+	setlocale(LC_NUMERIC, "");
 #endif
 
 	if (!parse_arguments(&argc, (const char ***) &argv)) {

maybe-make-man

@@ -1,22 +1,17 @@
 #!/bin/sh
 
-if [ x"$2" = x ]; then
-    echo "Usage: $0 SRC_DIR NAME.NUM.md" 1>&2
+if [ $# != 1 ]; then
+    echo "Usage: $0 NAME.NUM.md" 1>&2
     exit 1
 fi
 
-srcdir="$1"
-inname="$2"
+inname="$1"
+srcdir=`dirname "$0"`
 flagfile="$srcdir/.md2man-works"
 
-if [ ! -d "$srcdir" ]; then
-    echo "The specified SRC_DIR is not a directory: $srcdir" 1>&2
-    exit 1
-fi
-
 if [ ! -f "$flagfile" ]; then
     # We test our smallest manpage just to see if the python setup works.
-    if "$srcdir/md2man" --test "$srcdir/rsync-ssl.1.md" >/dev/null 2>&1; then
+    if "$srcdir/md-convert" --test "$srcdir/rsync-ssl.1.md" >/dev/null 2>&1; then
 	touch $flagfile
     else
 	outname=`echo "$inname" | sed 's/\.md$//'`
@@ -37,4 +32,4 @@ if [ ! -f "$flagfile" ]; then
     fi
 fi
 
-"$srcdir/md2man" "$srcdir/$inname"
+"$srcdir/md-convert" "$srcdir/$inname"

md-convert

@@ -0,0 +1,634 @@
+#!/usr/bin/env python3
+
+# This script transforms markdown files into html and (optionally) nroff. The
+# output files are written into the current directory named for the input file
+# without the .md suffix and either the .html suffix or no suffix.
+#
+# If the input .md file has a section number at the end of the name (e.g.,
+# rsync.1.md) a nroff file is also output (PROJ.NUM.md -> PROJ.NUM).
+#
+# The markdown input format has one extra extension: if a numbered list starts
+# at 0, it is turned into a description list. The dl's dt tag is taken from the
+# contents of the first tag inside the li, which is usually a p, code, or
+# strong tag.
+#
+# The cmarkgfm or commonmark lib is used to transforms the input file into
+# html.  Then, the html.parser is used as a state machine that lets us tweak
+# the html and (optionally) output nroff data based on the html tags.
+#
+# If the string @USE_GFM_PARSER@ exists in the file, the string is removed and
+# a github-flavored-markup parser is used to parse the file.
+#
+# The man-page .md files also get the vars @VERSION@, @BINDIR@, and @LIBDIR@
+# substituted.  Some of these values depend on the Makefile $(prefix) (see the
+# generated Makefile).  If the maintainer wants to build files for /usr/local
+# while creating release-ready man-page files for /usr, use the environment to
+# set RSYNC_OVERRIDE_PREFIX=/usr.
+
+# Copyright (C) 2020 - 2021 Wayne Davison
+#
+# This program is freely redistributable.
+
+import os, sys, re, argparse, subprocess, time
+from html.parser import HTMLParser
+
+VALID_PAGES = 'README INSTALL COPYING rsync.1 rrsync.1 rsync-ssl.1 rsyncd.conf.5'.split()
+
+CONSUMES_TXT = set('h1 h2 h3 p li pre'.split())
+
+HTML_START = """\
+<html><head>
+<title>%TITLE%</title>
+<meta charset="UTF-8"/>
+<link href="https://fonts.googleapis.com/css2?family=Roboto&family=Roboto+Mono&display=swap" rel="stylesheet">
+<style>
+body {
+  max-width: 50em;
+  margin: auto;
+}
+body, b, strong, u {
+  font-family: 'Roboto', sans-serif;
+}
+a.tgt { font-face: symbol; font-weight: 400; font-size: 70%; visibility: hidden; text-decoration: none; color: #ddd; padding: 0 4px; border: 0; }
+a.tgt:after { content: '🔗'; }
+a.tgt:hover { color: #444; background-color: #eaeaea; }
+h1:hover > a.tgt, h2:hover > a.tgt, h3:hover > a.tgt, dt:hover > a.tgt { visibility: visible; }
+code {
+  font-family: 'Roboto Mono', monospace;
+  font-weight: bold;
+  white-space: pre;
+}
+pre code {
+  display: block;
+  font-weight: normal;
+}
+blockquote pre code {
+  background: #f1f1f1;
+}
+dd p:first-of-type {
+  margin-block-start: 0em;
+}
+</style>
+</head><body>
+"""
+
+TABLE_STYLE = """\
+table {
+  border-color: grey;
+  border-spacing: 0;
+}
+tr {
+  border-top: 1px solid grey;
+}
+tr:nth-child(2n) {
+  background-color: #f6f8fa;
+}
+th, td {
+  border: 1px solid #dfe2e5;
+  text-align: center;
+  padding-left: 1em;
+  padding-right: 1em;
+}
+"""
+
+MAN_HTML_END = """\
+<div style="float: right"><p><i>%s</i></p></div>
+"""
+
+HTML_END = """\
+</body></html>
+"""
+
+MAN_START = r"""
+.TH "%s" "%s" "%s" "%s" "User Commands"
+.\" prefix=%s
+""".lstrip()
+
+MAN_END = """\
+"""
+
+NORM_FONT = ('\1', r"\fP")
+BOLD_FONT = ('\2', r"\fB")
+UNDR_FONT = ('\3', r"\fI")
+NBR_DASH = ('\4', r"\-")
+NBR_SPACE = ('\xa0', r"\ ")
+
+FILENAME_RE = re.compile(r'^(?P<fn>(?P<srcdir>.+/)?(?P<name>(?P<prog>[^/]+?)(\.(?P<sect>\d+))?)\.md)$')
+ASSIGNMENT_RE = re.compile(r'^(\w+)=(.+)')
+VER_RE = re.compile(r'^#define\s+RSYNC_VERSION\s+"(\d.+?)"', re.M)
+TZ_RE = re.compile(r'^#define\s+MAINTAINER_TZ_OFFSET\s+(-?\d+(\.\d+)?)', re.M)
+VAR_REF_RE = re.compile(r'\$\{(\w+)\}')
+VERSION_RE = re.compile(r' (\d[.\d]+)[, ]')
+BIN_CHARS_RE = re.compile(r'[\1-\7]+')
+SPACE_DOUBLE_DASH_RE = re.compile(r'\s--(\s)')
+NON_SPACE_SINGLE_DASH_RE = re.compile(r'(^|\W)-')
+WHITESPACE_RE = re.compile(r'\s')
+CODE_BLOCK_RE = re.compile(r'[%s]([^=%s]+)[=%s]' % (BOLD_FONT[0], NORM_FONT[0], NORM_FONT[0]))
+NBR_DASH_RE = re.compile(r'[%s]' % NBR_DASH[0])
+INVALID_TARGET_CHARS_RE = re.compile(r'[^-A-Za-z0-9._]')
+INVALID_START_CHAR_RE = re.compile(r'^([^A-Za-z0-9])')
+MANIFY_LINESTART_RE = re.compile(r"^(['.])", flags=re.M)
+
+md_parser = None
+env_subs = { }
+
+warning_count = 0
+
+def main():
+    for mdfn in args.mdfiles:
+        parse_md_file(mdfn)
+
+    if args.test:
+        print("The test was successful.")
+
+
+def parse_md_file(mdfn):
+    fi = FILENAME_RE.match(mdfn)
+    if not fi:
+        die('Failed to parse a md input file name:', mdfn)
+    fi = argparse.Namespace(**fi.groupdict())
+    fi.want_manpage = not not fi.sect
+    if fi.want_manpage:
+        fi.title = fi.prog + '(' + fi.sect + ') manpage'
+    else:
+        fi.title = fi.prog + ' for rsync'
+
+    if fi.want_manpage:
+        if not env_subs:
+            find_man_substitutions()
+        prog_ver = 'rsync ' + env_subs['VERSION']
+        if fi.prog != 'rsync':
+            prog_ver = fi.prog + ' from ' + prog_ver
+        fi.man_headings = (fi.prog, fi.sect, env_subs['date'], prog_ver, env_subs['prefix'])
+
+    with open(mdfn, 'r', encoding='utf-8') as fh:
+        txt = fh.read()
+
+    use_gfm_parser = '@USE_GFM_PARSER@' in txt
+    if use_gfm_parser:
+        txt = txt.replace('@USE_GFM_PARSER@', '')
+
+    if fi.want_manpage:
+        txt = (txt.replace('@VERSION@', env_subs['VERSION'])
+                  .replace('@BINDIR@', env_subs['bindir'])
+                  .replace('@LIBDIR@', env_subs['libdir']))
+
+    if use_gfm_parser:
+        if not gfm_parser:
+            die('Input file requires cmarkgfm parser:', mdfn)
+        fi.html_in = gfm_parser(txt)
+    else:
+        fi.html_in = md_parser(txt)
+    txt = None
+
+    TransformHtml(fi)
+
+    if args.test:
+        return
+
+    output_list = [ (fi.name + '.html', fi.html_out) ]
+    if fi.want_manpage:
+        output_list += [ (fi.name, fi.man_out) ]
+    for fn, txt in output_list:
+        if args.dest and args.dest != '.':
+            fn = os.path.join(args.dest, fn)
+        if os.path.lexists(fn):
+            os.unlink(fn)
+        print("Wrote:", fn)
+        with open(fn, 'w', encoding='utf-8') as fh:
+            fh.write(txt)
+
+
+def find_man_substitutions():
+    srcdir = os.path.dirname(sys.argv[0]) + '/'
+    mtime = 0
+
+    git_dir = srcdir + '.git'
+    if os.path.lexists(git_dir):
+        mtime = int(subprocess.check_output(['git', '--git-dir', git_dir, 'log', '-1', '--format=%at']))
+
+    # Allow "prefix" to be overridden via the environment:
+    env_subs['prefix'] = os.environ.get('RSYNC_OVERRIDE_PREFIX', None)
+
+    if args.test:
+        env_subs['VERSION'] = '1.0.0'
+        env_subs['bindir'] = '/usr/bin'
+        env_subs['libdir'] = '/usr/lib/rsync'
+        tz_offset = 0
+    else:
+        for fn in (srcdir + 'version.h', 'Makefile'):
+            try:
+                st = os.lstat(fn)
+            except OSError:
+                die('Failed to find', srcdir + fn)
+            if not mtime:
+                mtime = st.st_mtime
+
+        with open(srcdir + 'version.h', 'r', encoding='utf-8') as fh:
+            txt = fh.read()
+        m = VER_RE.search(txt)
+        env_subs['VERSION'] = m.group(1)
+        m = TZ_RE.search(txt) # the tzdata lib may not be installed, so we use a simple hour offset
+        tz_offset = float(m.group(1)) * 60 * 60
+
+        with open('Makefile', 'r', encoding='utf-8') as fh:
+            for line in fh:
+                m = ASSIGNMENT_RE.match(line)
+                if not m:
+                    continue
+                var, val = (m.group(1), m.group(2))
+                if var == 'prefix' and env_subs[var] is not None:
+                    continue
+                while VAR_REF_RE.search(val):
+                    val = VAR_REF_RE.sub(lambda m: env_subs[m.group(1)], val)
+                env_subs[var] = val
+                if var == 'srcdir':
+                    break
+
+    env_subs['date'] = time.strftime('%d %b %Y', time.gmtime(mtime + tz_offset)).lstrip('0')
+
+
+def html_via_commonmark(txt):
+    return commonmark.HtmlRenderer().render(commonmark.Parser().parse(txt))
+
+
+class TransformHtml(HTMLParser):
+    def __init__(self, fi):
+        HTMLParser.__init__(self, convert_charrefs=True)
+
+        self.fn = fi.fn
+
+        st = self.state = argparse.Namespace(
+                list_state = [ ],
+                p_macro = ".P\n",
+                at_first_tag_in_li = False,
+                at_first_tag_in_dd = False,
+                dt_from = None,
+                in_pre = False,
+                in_code = False,
+                html_out = [ HTML_START.replace('%TITLE%', fi.title) ],
+                man_out = [ ],
+                txt = '',
+                want_manpage = fi.want_manpage,
+                created_hashtags = set(),
+                derived_hashtags = set(),
+                referenced_hashtags = set(),
+                bad_hashtags = set(),
+                latest_targets = [ ],
+                opt_prefix = 'opt',
+                a_txt_start = None,
+                target_suf = '',
+                )
+
+        if st.want_manpage:
+            st.man_out.append(MAN_START % fi.man_headings)
+
+        if '</table>' in fi.html_in:
+            st.html_out[0] = st.html_out[0].replace('</style>', TABLE_STYLE + '</style>')
+
+        self.feed(fi.html_in)
+        fi.html_in = None
+
+        if st.want_manpage:
+            st.html_out.append(MAN_HTML_END % env_subs['date'])
+        st.html_out.append(HTML_END)
+        st.man_out.append(MAN_END)
+
+        fi.html_out = ''.join(st.html_out)
+        st.html_out = None
+
+        fi.man_out = ''.join(st.man_out)
+        st.man_out = None
+
+        for tgt, txt in st.derived_hashtags:
+            derived = txt2target(txt, tgt)
+            if derived not in st.created_hashtags:
+                txt = BIN_CHARS_RE.sub('', txt.replace(NBR_DASH[0], '-').replace(NBR_SPACE[0], ' '))
+                warn('Unknown derived hashtag link in', self.fn, 'based on:', (tgt, txt))
+
+        for bad in st.bad_hashtags:
+            if bad in st.created_hashtags:
+                warn('Missing "#" in hashtag link in', self.fn + ':', bad)
+            else:
+                warn('Unknown non-hashtag link in', self.fn + ':', bad)
+
+        for bad in st.referenced_hashtags - st.created_hashtags:
+            warn('Unknown hashtag link in', self.fn + ':', '#' + bad)
+
+    def handle_starttag(self, tag, attrs_list):
+        st = self.state
+        if args.debug:
+            self.output_debug('START', (tag, attrs_list))
+        if st.at_first_tag_in_li:
+            if st.list_state[-1] == 'dl':
+                st.dt_from = tag
+                if tag == 'p':
+                    tag = 'dt'
+                else:
+                    st.html_out.append('<dt>')
+            elif tag == 'p':
+                st.at_first_tag_in_dd = True # Kluge to suppress a .P at the start of an li.
+            st.at_first_tag_in_li = False
+        if tag == 'p':
+            if not st.at_first_tag_in_dd:
+                st.man_out.append(st.p_macro)
+        elif tag == 'li':
+            st.at_first_tag_in_li = True
+            lstate = st.list_state[-1]
+            if lstate == 'dl':
+                return
+            if lstate == 'o':
+                st.man_out.append(".IP o\n")
+            else:
+                st.man_out.append(".IP " + str(lstate) + ".\n")
+                st.list_state[-1] += 1
+        elif tag == 'blockquote':
+            st.man_out.append(".RS 4\n")
+        elif tag == 'pre':
+            st.in_pre = True
+            st.man_out.append(st.p_macro + ".nf\n")
+        elif tag == 'code' and not st.in_pre:
+            st.in_code = True
+            st.txt += BOLD_FONT[0]
+        elif tag == 'strong' or tag == 'b':
+            st.txt += BOLD_FONT[0]
+        elif tag == 'em' or  tag == 'i':
+            if st.want_manpage:
+                tag = 'u' # Change it into underline to be more like the manpage
+                st.txt += UNDR_FONT[0]
+        elif tag == 'ol':
+            start = 1
+            for var, val in attrs_list:
+                if var == 'start':
+                    start = int(val) # We only support integers.
+                    break
+            if st.list_state:
+                st.man_out.append(".RS\n")
+            if start == 0:
+                tag = 'dl'
+                attrs_list = [ ]
+                st.list_state.append('dl')
+            else:
+                st.list_state.append(start)
+            st.man_out.append(st.p_macro)
+            st.p_macro = ".IP\n"
+        elif tag == 'ul':
+            st.man_out.append(st.p_macro)
+            if st.list_state:
+                st.man_out.append(".RS\n")
+                st.p_macro = ".IP\n"
+            st.list_state.append('o')
+        elif tag == 'hr':
+            st.man_out.append(".l\n")
+            st.html_out.append("<hr />")
+            return
+        elif tag == 'a':
+            st.a_href = None
+            for var, val in attrs_list:
+                if var == 'href':
+                    if val.startswith(('https://', 'http://', 'mailto:', 'ftp:')):
+                        pass # nothing to check
+                    elif '#' in val:
+                        pg, tgt = val.split('#', 2)
+                        if pg and pg not in VALID_PAGES or '#' in tgt:
+                            st.bad_hashtags.add(val)
+                        elif tgt in ('', 'opt', 'dopt'):
+                            st.a_href = val
+                        elif pg == '':
+                            st.referenced_hashtags.add(tgt)
+                            if tgt in st.latest_targets:
+                                warn('Found link to the current section in', self.fn + ':', val)
+                    elif val not in VALID_PAGES:
+                        st.bad_hashtags.add(val)
+            st.a_txt_start = len(st.txt)
+        st.html_out.append('<' + tag + ''.join(' ' + var + '="' + htmlify(val) + '"' for var, val in attrs_list) + '>')
+        st.at_first_tag_in_dd = False
+
+
+    def handle_endtag(self, tag):
+        st = self.state
+        if args.debug:
+            self.output_debug('END', (tag,))
+        if tag in CONSUMES_TXT or st.dt_from == tag:
+            txt = st.txt.strip()
+            st.txt = ''
+        else:
+            txt = None
+        add_to_txt = None
+        if tag == 'h1':
+            tgt = txt
+            target_suf = ''
+            if tgt.startswith('NEWS for '):
+                m = VERSION_RE.search(tgt)
+                if m:
+                    tgt = m.group(1)
+                    st.target_suf = '-' + tgt
+            self.add_targets(tag, tgt)
+        elif tag == 'h2':
+            st.man_out.append(st.p_macro + '.SH "' + manify(txt) + '"\n')
+            self.add_targets(tag, txt, st.target_suf)
+            st.opt_prefix = 'dopt' if txt == 'DAEMON OPTIONS' else 'opt'
+        elif tag == 'h3':
+            st.man_out.append(st.p_macro + '.SS "' + manify(txt) + '"\n')
+            self.add_targets(tag, txt, st.target_suf)
+        elif tag == 'p':
+            if st.dt_from == 'p':
+                tag = 'dt'
+                st.man_out.append('.IP "' + manify(txt) + '"\n')
+                if txt.startswith(BOLD_FONT[0]):
+                    self.add_targets(tag, txt)
+                st.dt_from = None
+            elif txt != '':
+                st.man_out.append(manify(txt) + "\n")
+        elif tag == 'li':
+            if st.list_state[-1] == 'dl':
+                if st.at_first_tag_in_li:
+                    die("Invalid 0. -> td translation")
+                tag = 'dd'
+            if txt != '':
+                st.man_out.append(manify(txt) + "\n")
+            st.at_first_tag_in_li = False
+        elif tag == 'blockquote':
+            st.man_out.append(".RE\n")
+        elif tag == 'pre':
+            st.in_pre = False
+            st.man_out.append(manify(txt) + "\n.fi\n")
+        elif (tag == 'code' and not st.in_pre):
+            st.in_code = False
+            add_to_txt = NORM_FONT[0]
+        elif tag == 'strong' or tag == 'b':
+            add_to_txt = NORM_FONT[0]
+        elif tag == 'em' or  tag == 'i':
+            if st.want_manpage:
+                tag = 'u' # Change it into underline to be more like the manpage
+                add_to_txt = NORM_FONT[0]
+        elif tag == 'ol' or tag == 'ul':
+            if st.list_state.pop() == 'dl':
+                tag = 'dl'
+            if st.list_state:
+                st.man_out.append(".RE\n")
+            else:
+                st.p_macro = ".P\n"
+            st.at_first_tag_in_dd = False
+        elif tag == 'hr':
+            return
+        elif tag == 'a':
+            if st.a_href:
+                atxt = st.txt[st.a_txt_start:]
+                find = 'href="' + st.a_href + '"'
+                for j in range(len(st.html_out)-1, 0, -1):
+                    if find in st.html_out[j]:
+                        pg, tgt = st.a_href.split('#', 2)
+                        derived = txt2target(atxt, tgt)
+                        if pg == '':
+                            if derived in st.latest_targets:
+                                warn('Found link to the current section in', self.fn + ':', st.a_href)
+                            st.derived_hashtags.add((tgt, atxt))
+                        st.html_out[j] = st.html_out[j].replace(find, 'href="' + pg + '#' + derived + '"')
+                        break
+                else:
+                    die('INTERNAL ERROR: failed to find href in html data:', find)
+        st.html_out.append('</' + tag + '>')
+        if add_to_txt:
+            if txt is None:
+                st.txt += add_to_txt
+            else:
+                txt += add_to_txt
+        if st.dt_from == tag:
+            st.man_out.append('.IP "' + manify(txt) + '"\n')
+            st.html_out.append('</dt><dd>')
+            st.at_first_tag_in_dd = True
+            st.dt_from = None
+        elif tag == 'dt':
+            st.html_out.append('<dd>')
+            st.at_first_tag_in_dd = True
+
+
+    def handle_data(self, txt):
+        st = self.state
+        if '](' in txt:
+            warn('Malformed link in', self.fn + ':', txt)
+        if args.debug:
+            self.output_debug('DATA', (txt,))
+        if st.in_pre:
+            html = htmlify(txt)
+        else:
+            txt = SPACE_DOUBLE_DASH_RE.sub(NBR_SPACE[0] + r'--\1', txt).replace('--', NBR_DASH[0]*2)
+            txt = NON_SPACE_SINGLE_DASH_RE.sub(r'\1' + NBR_DASH[0], txt)
+            html = htmlify(txt)
+            if st.in_code:
+                txt = WHITESPACE_RE.sub(NBR_SPACE[0], txt)
+                html = html.replace(NBR_DASH[0], '-').replace(NBR_SPACE[0], ' ') # <code> is non-breaking in CSS
+        st.html_out.append(html.replace(NBR_SPACE[0], '&nbsp;').replace(NBR_DASH[0], '-&#8288;'))
+        st.txt += txt
+
+
+    def add_targets(self, tag, txt, suf=None):
+        st = self.state
+        tag = '<' + tag + '>'
+        targets = CODE_BLOCK_RE.findall(txt)
+        if not targets:
+            targets = [ txt ]
+        tag_pos = 0
+        for txt in targets:
+            txt = txt2target(txt, st.opt_prefix)
+            if not txt:
+                continue
+            if suf:
+                txt += suf
+            if txt in st.created_hashtags:
+                for j in range(2, 1000):
+                    chk = txt + '-' + str(j)
+                    if chk not in st.created_hashtags:
+                        print('Made link target unique:', chk)
+                        txt = chk
+                        break
+            if tag_pos == 0:
+                tag_pos -= 1
+                while st.html_out[tag_pos] != tag:
+                    tag_pos -= 1
+                st.html_out[tag_pos] = tag[:-1] + ' id="' + txt + '">'
+                st.html_out.append('<a href="#' + txt + '" class="tgt"></a>')
+                tag_pos -= 1 # take into account the append
+            else:
+                st.html_out[tag_pos] = '<span id="' + txt + '"></span>' + st.html_out[tag_pos]
+            st.created_hashtags.add(txt)
+        st.latest_targets = targets
+
+
+    def output_debug(self, event, extra):
+        import pprint
+        st = self.state
+        if args.debug < 2:
+            st = argparse.Namespace(**vars(st))
+            if len(st.html_out) > 2:
+                st.html_out = ['...'] + st.html_out[-2:]
+            if len(st.man_out) > 2:
+                st.man_out = ['...'] + st.man_out[-2:]
+        print(event, extra)
+        pprint.PrettyPrinter(indent=2).pprint(vars(st))
+
+
+def txt2target(txt, opt_prefix):
+    txt = txt.strip().rstrip(':')
+    m = CODE_BLOCK_RE.search(txt)
+    if m:
+        txt = m.group(1)
+    txt = NBR_DASH_RE.sub('-', txt)
+    txt = BIN_CHARS_RE.sub('', txt)
+    txt = INVALID_TARGET_CHARS_RE.sub('_', txt)
+    if opt_prefix and txt.startswith('-'):
+        txt = opt_prefix + txt
+    else:
+        txt = INVALID_START_CHAR_RE.sub(r't\1', txt)
+    return txt
+
+
+def manify(txt):
+    return MANIFY_LINESTART_RE.sub(r'\&\1', txt.replace('\\', '\\\\')
+            .replace(NBR_SPACE[0], NBR_SPACE[1])
+            .replace(NBR_DASH[0], NBR_DASH[1])
+            .replace(NORM_FONT[0], NORM_FONT[1])
+            .replace(BOLD_FONT[0], BOLD_FONT[1])
+            .replace(UNDR_FONT[0], UNDR_FONT[1]))
+
+
+def htmlify(txt):
+    return txt.replace('&', '&amp;').replace('<', '&lt;').replace('>', '&gt;').replace('"', '&quot;')
+
+
+def warn(*msg):
+    print(*msg, file=sys.stderr)
+    global warning_count
+    warning_count += 1
+
+
+def die(*msg):
+    warn(*msg)
+    sys.exit(1)
+
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser(description="Convert markdown into html and (optionally) nroff. Each input filename must have a .md suffix, which is changed to .html for the output filename. If the input filename ends with .num.md (e.g. foo.1.md) then a nroff file is also output with the input filename's .md suffix removed (e.g. foo.1).", add_help=False)
+    parser.add_argument('--test', action='store_true', help="Just test the parsing without outputting any files.")
+    parser.add_argument('--dest', metavar='DIR', help="Create files in DIR instead of the current directory.")
+    parser.add_argument('--debug', '-D', action='count', default=0, help='Output copious info on the html parsing. Repeat for even more.')
+    parser.add_argument("--help", "-h", action="help", help="Output this help message and exit.")
+    parser.add_argument("mdfiles", metavar='FILE.md', nargs='+', help="One or more .md files to convert.")
+    args = parser.parse_args()
+
+    try:
+        import cmarkgfm
+        md_parser = cmarkgfm.markdown_to_html
+        gfm_parser = cmarkgfm.github_flavored_markdown_to_html
+    except:
+        try:
+            import commonmark
+            md_parser = html_via_commonmark
+        except:
+            die("Failed to find cmarkgfm or commonmark for python3.")
+        gfm_parser = None
+
+    main()
+    if warning_count:
+        sys.exit(1)

md2man

@@ -1,387 +0,0 @@
-#!/usr/bin/env python3
-
-# This script takes a manpage written in markdown and turns it into an html web
-# page and a nroff man page.  The input file must have the name of the program
-# and the section in this format: NAME.NUM.md.  The output files are written
-# into the current directory named NAME.NUM.html and NAME.NUM.  The input
-# format has one extra extension: if a numbered list starts at 0, it is turned
-# into a description list. The dl's dt tag is taken from the contents of the
-# first tag inside the li, which is usually a p, code, or strong tag.  The
-# cmarkgfm or commonmark lib is used to transforms the input file into html.
-# The html.parser is used as a state machine that both tweaks the html and
-# outputs the nroff data based on the html tags.
-#
-# Copyright (C) 2020 Wayne Davison
-#
-# This program is freely redistributable.
-
-import sys, os, re, argparse, subprocess, time
-from html.parser import HTMLParser
-
-CONSUMES_TXT = set('h1 h2 p li pre'.split())
-
-HTML_START = """\
-<html><head>
-<title>%s</title>
-<link href="https://fonts.googleapis.com/css2?family=Roboto&family=Roboto+Mono&display=swap" rel="stylesheet">
-<style>
-body {
-  max-width: 50em;
-  margin: auto;
-}
-body, b, strong, u {
-  font-family: 'Roboto', sans-serif;
-}
-code {
-  font-family: 'Roboto Mono', monospace;
-  font-weight: bold;
-  white-space: pre;
-}
-pre code {
-  display: block;
-  font-weight: normal;
-}
-blockquote pre code {
-  background: #f1f1f1;
-}
-dd p:first-of-type {
-  margin-block-start: 0em;
-}
-</style>
-</head><body>
-"""
-
-HTML_END = """\
-<div style="float: right"><p><i>%s</i></p></div>
-</body></html>
-"""
-
-MAN_START = r"""
-.TH "%s" "%s" "%s" "%s" "User Commands"
-""".lstrip()
-
-MAN_END = """\
-"""
-
-NORM_FONT = ('\1', r"\fP")
-BOLD_FONT = ('\2', r"\fB")
-UNDR_FONT = ('\3', r"\fI")
-NBR_DASH = ('\4', r"\-")
-NBR_SPACE = ('\xa0', r"\ ")
-
-md_parser = None
-
-def main():
-    fi = re.match(r'^(?P<fn>(?P<srcdir>.+/)?(?P<name>(?P<prog>[^/]+)\.(?P<sect>\d+))\.md)$', args.mdfile)
-    if not fi:
-        die('Failed to parse NAME.NUM.md out of input file:', args.mdfile)
-    fi = argparse.Namespace(**fi.groupdict())
-
-    if not fi.srcdir:
-        fi.srcdir = './'
-
-    fi.title = fi.prog + '(' + fi.sect + ') man page'
-    fi.mtime = 0
-
-    git_dir = fi.srcdir + '.git'
-    if os.path.lexists(git_dir):
-        fi.mtime = int(subprocess.check_output(['git', '--git-dir', git_dir, 'log', '-1', '--format=%at']))
-
-    env_subs = { 'prefix': os.environ.get('RSYNC_OVERRIDE_PREFIX', None) }
-
-    if args.test:
-        env_subs['VERSION'] = '1.0.0'
-        env_subs['libdir'] = '/usr'
-    else:
-        for fn in (fi.srcdir + 'version.h', 'Makefile'):
-            try:
-                st = os.lstat(fn)
-            except:
-                die('Failed to find', fi.srcdir + fn)
-            if not fi.mtime:
-                fi.mtime = st.st_mtime
-
-        with open(fi.srcdir + 'version.h', 'r', encoding='utf-8') as fh:
-            txt = fh.read()
-        m = re.search(r'"(.+?)"', txt)
-        env_subs['VERSION'] = m.group(1)
-
-        with open('Makefile', 'r', encoding='utf-8') as fh:
-            for line in fh:
-                m = re.match(r'^(\w+)=(.+)', line)
-                if not m:
-                    continue
-                var, val = (m.group(1), m.group(2))
-                if var == 'prefix' and env_subs[var] is not None:
-                    continue
-                while re.search(r'\$\{', val):
-                    val = re.sub(r'\$\{(\w+)\}', lambda m: env_subs[m.group(1)], val)
-                env_subs[var] = val
-                if var == 'srcdir':
-                    break
-
-    with open(fi.fn, 'r', encoding='utf-8') as fh:
-        txt = fh.read()
-
-    txt = re.sub(r'@VERSION@', env_subs['VERSION'], txt)
-    txt = re.sub(r'@LIBDIR@', env_subs['libdir'], txt)
-
-    fi.html_in = md_parser(txt)
-    txt = None
-
-    fi.date = time.strftime('%d %b %Y', time.localtime(fi.mtime))
-    fi.man_headings = (fi.prog, fi.sect, fi.date, fi.prog + ' ' + env_subs['VERSION'])
-
-    HtmlToManPage(fi)
-
-    if args.test:
-        print("The test was successful.")
-        return
-
-    for fn, txt in ((fi.name + '.html', fi.html_out), (fi.name, fi.man_out)):
-        print("Wrote:", fn)
-        with open(fn, 'w', encoding='utf-8') as fh:
-            fh.write(txt)
-
-
-def html_via_cmarkgfm(txt):
-    return cmarkgfm.markdown_to_html(txt)
-
-
-def html_via_commonmark(txt):
-    return commonmark.HtmlRenderer().render(commonmark.Parser().parse(txt))
-
-
-class HtmlToManPage(HTMLParser):
-    def __init__(self, fi):
-        HTMLParser.__init__(self, convert_charrefs=True)
-
-        st = self.state = argparse.Namespace(
-                list_state = [ ],
-                p_macro = ".P\n",
-                at_first_tag_in_li = False,
-                at_first_tag_in_dd = False,
-                dt_from = None,
-                in_pre = False,
-                in_code = False,
-                html_out = [ HTML_START % fi.title ],
-                man_out = [ MAN_START % fi.man_headings ],
-                txt = '',
-                )
-
-        self.feed(fi.html_in)
-        fi.html_in = None
-
-        st.html_out.append(HTML_END % fi.date)
-        st.man_out.append(MAN_END)
-
-        fi.html_out = ''.join(st.html_out)
-        st.html_out = None
-
-        fi.man_out = ''.join(st.man_out)
-        st.man_out = None
-
-
-    def handle_starttag(self, tag, attrs_list):
-        st = self.state
-        if args.debug:
-            self.output_debug('START', (tag, attrs_list))
-        if st.at_first_tag_in_li:
-            if st.list_state[-1] == 'dl':
-                st.dt_from = tag
-                if tag == 'p':
-                    tag = 'dt'
-                else:
-                    st.html_out.append('<dt>')
-            elif tag == 'p':
-                st.at_first_tag_in_dd = True # Kluge to suppress a .P at the start of an li.
-            st.at_first_tag_in_li = False
-        if tag == 'p':
-            if not st.at_first_tag_in_dd:
-                st.man_out.append(st.p_macro)
-        elif tag == 'li':
-            st.at_first_tag_in_li = True
-            lstate = st.list_state[-1]
-            if lstate == 'dl':
-                return
-            if lstate == 'o':
-                st.man_out.append(".IP o\n")
-            else:
-                st.man_out.append(".IP " + str(lstate) + ".\n")
-                st.list_state[-1] += 1
-        elif tag == 'blockquote':
-            st.man_out.append(".RS 4\n")
-        elif tag == 'pre':
-            st.in_pre = True
-            st.man_out.append(st.p_macro + ".nf\n")
-        elif tag == 'code' and not st.in_pre:
-            st.in_code = True
-            st.txt += BOLD_FONT[0]
-        elif tag == 'strong' or tag == 'b':
-            st.txt += BOLD_FONT[0]
-        elif tag == 'em' or  tag == 'i':
-            tag = 'u' # Change it into underline to be more like the man page
-            st.txt += UNDR_FONT[0]
-        elif tag == 'ol':
-            start = 1
-            for var, val in attrs_list:
-                if var == 'start':
-                    start = int(val) # We only support integers.
-                    break
-            if st.list_state:
-                st.man_out.append(".RS\n")
-            if start == 0:
-                tag = 'dl'
-                attrs_list = [ ]
-                st.list_state.append('dl')
-            else:
-                st.list_state.append(start)
-            st.man_out.append(st.p_macro)
-            st.p_macro = ".IP\n"
-        elif tag == 'ul':
-            st.man_out.append(st.p_macro)
-            if st.list_state:
-                st.man_out.append(".RS\n")
-                st.p_macro = ".IP\n"
-            st.list_state.append('o')
-        st.html_out.append('<' + tag + ''.join(' ' + var + '="' + htmlify(val) + '"' for var, val in attrs_list) + '>')
-        st.at_first_tag_in_dd = False
-
-
-    def handle_endtag(self, tag):
-        st = self.state
-        if args.debug:
-            self.output_debug('END', (tag,))
-        if tag in CONSUMES_TXT or st.dt_from == tag:
-            txt = st.txt.strip()
-            st.txt = ''
-        else:
-            txt = None
-        add_to_txt = None
-        if tag == 'h1':
-            st.man_out.append(st.p_macro + '.SH "' + manify(txt) + '"\n')
-        elif tag == 'h2':
-            st.man_out.append(st.p_macro + '.SS "' + manify(txt) + '"\n')
-        elif tag == 'p':
-            if st.dt_from == 'p':
-                tag = 'dt'
-                st.man_out.append('.IP "' + manify(txt) + '"\n')
-                st.dt_from = None
-            elif txt != '':
-                st.man_out.append(manify(txt) + "\n")
-        elif tag == 'li':
-            if st.list_state[-1] == 'dl':
-                if st.at_first_tag_in_li:
-                    die("Invalid 0. -> td translation")
-                tag = 'dd'
-            if txt != '':
-                st.man_out.append(manify(txt) + "\n")
-            st.at_first_tag_in_li = False
-        elif tag == 'blockquote':
-            st.man_out.append(".RE\n")
-        elif tag == 'pre':
-            st.in_pre = False
-            st.man_out.append(manify(txt) + "\n.fi\n")
-        elif (tag == 'code' and not st.in_pre):
-            st.in_code = False
-            add_to_txt = NORM_FONT[0]
-        elif tag == 'strong' or tag == 'b':
-            add_to_txt = NORM_FONT[0]
-        elif tag == 'em' or  tag == 'i':
-            tag = 'u' # Change it into underline to be more like the man page
-            add_to_txt = NORM_FONT[0]
-        elif tag == 'ol' or tag == 'ul':
-            if st.list_state.pop() == 'dl':
-                tag = 'dl'
-            if st.list_state:
-                st.man_out.append(".RE\n")
-            else:
-                st.p_macro = ".P\n"
-            st.at_first_tag_in_dd = False
-        st.html_out.append('</' + tag + '>')
-        if add_to_txt:
-            if txt is None:
-                st.txt += add_to_txt
-            else:
-                txt += add_to_txt
-        if st.dt_from == tag:
-            st.man_out.append('.IP "' + manify(txt) + '"\n')
-            st.html_out.append('</dt><dd>')
-            st.at_first_tag_in_dd = True
-            st.dt_from = None
-        elif tag == 'dt':
-            st.html_out.append('<dd>')
-            st.at_first_tag_in_dd = True
-
-
-    def handle_data(self, txt):
-        st = self.state
-        if args.debug:
-            self.output_debug('DATA', (txt,))
-        if st.in_pre:
-            html = htmlify(txt)
-        else:
-            txt = re.sub(r'\s--(\s)', NBR_SPACE[0] + r'--\1', txt).replace('--', NBR_DASH[0]*2)
-            txt = re.sub(r'(^|\W)-', r'\1' + NBR_DASH[0], txt)
-            html = htmlify(txt)
-            if st.in_code:
-                txt = re.sub(r'\s', NBR_SPACE[0], txt)
-                html = html.replace(NBR_DASH[0], '-').replace(NBR_SPACE[0], ' ') # <code> is non-breaking in CSS
-        st.html_out.append(html.replace(NBR_SPACE[0], '&nbsp;').replace(NBR_DASH[0], '-&#8288;'))
-        st.txt += txt
-
-
-    def output_debug(self, event, extra):
-        import pprint
-        st = self.state
-        if args.debug < 2:
-            st = argparse.Namespace(**vars(st))
-            if len(st.html_out) > 2:
-                st.html_out = ['...'] + st.html_out[-2:]
-            if len(st.man_out) > 2:
-                st.man_out = ['...'] + st.man_out[-2:]
-        print(event, extra)
-        pprint.PrettyPrinter(indent=2).pprint(vars(st))
-
-
-def manify(txt):
-    return re.sub(r"^(['.])", r'\&\1', txt.replace('\\', '\\\\')
-            .replace(NBR_SPACE[0], NBR_SPACE[1])
-            .replace(NBR_DASH[0], NBR_DASH[1])
-            .replace(NORM_FONT[0], NORM_FONT[1])
-            .replace(BOLD_FONT[0], BOLD_FONT[1])
-            .replace(UNDR_FONT[0], UNDR_FONT[1]), flags=re.M)
-
-
-def htmlify(txt):
-    return txt.replace('&', '&amp;').replace('<', '&lt;').replace('>', '&gt;').replace('"', '&quot;')
-
-
-def warn(*msg):
-    print(*msg, file=sys.stderr)
-
-
-def die(*msg):
-    warn(*msg)
-    sys.exit(1)
-
-
-if __name__ == '__main__':
-    parser = argparse.ArgumentParser(description='Transform a NAME.NUM.md markdown file into a NAME.NUM.html web page & a NAME.NUM man page.', add_help=False)
-    parser.add_argument('--test', action='store_true', help='Test if we can parse the input w/o updating any files.')
-    parser.add_argument('--debug', '-D', action='count', default=0, help='Output copious info on the html parsing. Repeat for even more.')
-    parser.add_argument("--help", "-h", action="help", help="Output this help message and exit.")
-    parser.add_argument('mdfile', help="The NAME.NUM.md file to parse.")
-    args = parser.parse_args()
-
-    try:
-        import cmarkgfm
-        md_parser = html_via_cmarkgfm
-    except:
-        try:
-            import commonmark
-            md_parser = html_via_commonmark
-        except:
-            die("Failed to find cmarkgfm or commonmark for python3.")
-
-    main()

md2man

@@ -0,0 +1 @@
+md-convert

mkgitver

@@ -0,0 +1,20 @@
+#!/bin/sh
+
+srcdir=`dirname $0`
+gitver=`git describe --abbrev=8 2>/dev/null`
+
+if [ ! -f git-version.h ]; then
+    touch git-version.h
+fi
+
+case "$gitver" in
+    *.*)
+	echo "#define RSYNC_GITVER \"$gitver\"" >git-version.h.new
+	if ! diff git-version.h.new git-version.h >/dev/null; then
+	    echo "Updating git-version.h"
+	    mv git-version.h.new git-version.h
+	else
+	    rm git-version.h.new
+	fi
+	;;
+esac

mkproto.awk

@@ -2,6 +2,7 @@
 
 BEGIN {
     while ((getline i < "proto.h") > 0) old_protos = old_protos ? old_protos "\n" i : i
+    close("proto.h")
     protos = "/* This file is automatically generated with \"make proto\". DO NOT EDIT */\n"
 }
 
@@ -35,5 +36,5 @@ inheader {
 
 END {
     if (old_protos != protos) print protos > "proto.h"
-    printf "" > "proto.h-tstamp"
+    system("touch proto.h-tstamp")
 }

packaging/auto-Makefile

@@ -1,8 +1,12 @@
 TARGETS := all install install-ssl-daemon install-all install-strip conf gen gensend reconfigure restatus \
-	proto man clean cleantests distclean test check check29 check30 installcheck splint doxygen doxygen-upload
+	proto man clean cleantests distclean test check check29 check30 installcheck splint \
+	doxygen doxygen-upload finddead rrsync
 
-.PHONY: $(TARGETS)
+.PHONY: $(TARGETS) auto-prep
 
-$(TARGETS):
-	@if test x`packaging/prep-auto-dir` = x; then echo "auto-build-save is not setup"; exit 1; fi
+$(TARGETS): auto-prep
 	make -C build $@
+
+auto-prep:
+	@if test x`packaging/prep-auto-dir` = x; then echo "auto-build-save is not setup"; exit 1; fi
+	@echo 'Build branch: '`readlink build/.branch | tr % /`

packaging/cull-options

@@ -0,0 +1,147 @@
+#!/usr/bin/env python3
+# This script outputs either perl or python code that parses all possible options
+# that the code in options.c might send to the server.  The resulting code is then
+# included in the rrsync script.
+
+import re, argparse
+
+short_no_arg = { }
+short_with_num = { '@': 1 }
+long_opts = { # These include some extra long-args that BackupPC uses:
+        'block-size': 1,
+        'daemon': -1,
+        'debug': 1,
+        'fake-super': 0,
+        'fuzzy': 0,
+        'group': 0,
+        'hard-links': 0,
+        'ignore-times': 0,
+        'info': 1,
+        'links': 0,
+        'log-file': 3,
+        'munge-links': 0,
+        'no-munge-links': -1,
+        'one-file-system': 0,
+        'owner': 0,
+        'perms': 0,
+        'recursive': 0,
+        'stderr': 1,
+        'times': 0,
+        'write-devices': -1,
+        }
+
+def main():
+    last_long_opt = None
+
+    with open('../options.c') as fh:
+        for line in fh:
+            m = re.search(r"argstr\[x\+\+\] = '([^.ie])'", line)
+            if m:
+                short_no_arg[m.group(1)] = 1
+                last_long_opt = None
+                continue
+
+            m = re.search(r'asprintf\([^,]+, "-([a-zA-Z0-9])\%l?[ud]"', line)
+            if m:
+                short_with_num[m.group(1)] = 1
+                last_long_opt = None
+                continue
+
+            m = re.search(r'args\[ac\+\+\] = "--([^"=]+)"', line)
+            if m:
+                last_long_opt = m.group(1)
+                if last_long_opt not in long_opts:
+                    long_opts[last_long_opt] = 0
+                else:
+                    last_long_opt = None
+                continue
+
+            if last_long_opt:
+                m = re.search(r'args\[ac\+\+\] = safe_arg\("", ([^[("\s]+)\);', line)
+                if m:
+                    long_opts[last_long_opt] = 2
+                    last_long_opt = None
+                    continue
+                if 'args[ac++] = ' in line:
+                    last_long_opt = None
+
+            m = re.search(r'return "--([^"]+-dest)";', line)
+            if m:
+                long_opts[m.group(1)] = 2
+                last_long_opt = None
+                continue
+
+            m = re.search(r'asprintf\([^,]+, "--([^"=]+)=', line)
+            if not m:
+                m = re.search(r'args\[ac\+\+\] = "--([^"=]+)=', line)
+                if not m:
+                    m = re.search(r'args\[ac\+\+\] = safe_arg\("--([^"=]+)"', line)
+                    if not m:
+                        m = re.search(r'fmt = .*: "--([^"=]+)=', line)
+            if m:
+                long_opts[m.group(1)] = 1
+                last_long_opt = None
+
+    long_opts['files-from'] = 3
+
+    txt = """\
+### START of options data produced by the cull-options script. ###
+
+# To disable a short-named option, add its letter to this string:
+"""
+
+    txt += str_assign('short_disabled', 's') + "\n"
+    txt += '# These are also disabled when the restricted dir is not "/":\n'
+    txt += str_assign('short_disabled_subdir', 'KLk') + "\n"
+    txt += '# These are all possible short options that we will accept (when not disabled above):\n'
+    txt += str_assign('short_no_arg', ''.join(sorted(short_no_arg)), 'DO NOT REMOVE ANY')
+    txt += str_assign('short_with_num', ''.join(sorted(short_with_num)), 'DO NOT REMOVE ANY')
+   
+    txt += """
+# To disable a long-named option, change its value to a -1.  The values mean:
+# 0 = the option has no arg; 1 = the arg doesn't need any checking; 2 = only
+# check the arg when receiving; and 3 = always check the arg.
+"""
+
+    print(txt, end='')
+
+    if args.python:
+        print("long_opts = {")
+        sep = ':'
+    else:
+        print("our %long_opt = (")
+        sep = ' =>'
+
+    for opt in sorted(long_opts):
+        if opt.startswith(('min-', 'max-')):
+            val = 1
+        else:
+            val = long_opts[opt]
+        print(' ', repr(opt) + sep, str(val) + ',')
+
+    if args.python:
+        print("}")
+    else:
+        print(");")
+    print("\n### END of options data produced by the cull-options script. ###")
+
+
+def str_assign(name, val, comment=None):
+    comment = ' # ' + comment if comment else ''
+    if args.python:
+        return name + ' = ' + repr(val) + comment + "\n"
+    return 'our $' + name + ' = ' + repr(val) + ';' + comment + "\n"
+
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser(description="Output culled rsync options for rrsync.", add_help=False)
+    out_group = parser.add_mutually_exclusive_group()
+    out_group.add_argument('--perl', action='store_true', help="Output perl code.")
+    out_group.add_argument('--python', action='store_true', help="Output python code (the default).")
+    parser.add_argument('--help', '-h', action='help', help="Output this help message and exit.")
+    args = parser.parse_args()
+    if not args.perl:
+        args.python = True
+    main()
+
+# vim: sw=4 et

packaging/cull_options

@@ -1,85 +0,0 @@
-#!/usr/bin/env perl
-# This script outputs some perl code that parses all possible options
-# that the code in options.c might send to the server.  This perl code
-# is included in the rrsync script.
-use strict;
-
-our %short_no_arg;
-our %short_with_num = ( '@' => 1 );
-our %long_opt = ( # These include some extra long-args that BackupPC uses:
-    'block-size' => 1,
-    'daemon' => -1,
-    'debug' => 1,
-    'fake-super' => 0,
-    'fuzzy' => 0,
-    'group' => 0,
-    'hard-links' => 0,
-    'ignore-times' => 0,
-    'info' => 1,
-    'links' => 0,
-    'log-file' => 3,
-    'one-file-system' => 0,
-    'owner' => 0,
-    'perms' => 0,
-    'recursive' => 0,
-    'times' => 0,
-    'write-devices' => -1,
-);
-our $last_long_opt;
-
-open(IN, '../options.c') or die "Unable to open ../options.c: $!\n";
-
-while (<IN>) {
-    if (/\Qargstr[x++]\E = '([^.ie])'/) {
-	$short_no_arg{$1} = 1;
-	undef $last_long_opt;
-    } elsif (/\Qasprintf(\E[^,]+, "-([a-zA-Z0-9])\%l?[ud]"/) {
-	$short_with_num{$1} = 1;
-	undef $last_long_opt;
-    } elsif (/\Qargs[ac++]\E = "--([^"=]+)"/) {
-	$last_long_opt = $1;
-	$long_opt{$1} = 0 unless exists $long_opt{$1};
-    } elsif (defined($last_long_opt)
-	&& /\Qargs[ac++]\E = ([^["\s]+);/) {
-	$long_opt{$last_long_opt} = 2;
-	undef $last_long_opt;
-    } elsif (/return "--([^"]+-dest)";/) {
-	$long_opt{$1} = 2;
-	undef $last_long_opt;
-    } elsif (/\Qasprintf(\E[^,]+, "--([^"=]+)=/ || /\Qargs[ac++]\E = "--([^"=]+)=/ || /fmt = .*: "--([^"=]+)=/) {
-	$long_opt{$1} = 1;
-	undef $last_long_opt;
-    }
-}
-close IN;
-
-my $short_no_arg = join('', sort keys %short_no_arg);
-my $short_with_num = join('', sort keys %short_with_num);
-
-print <<EOT;
-
-# These options are the only options that rsync might send to the server,
-# and only in the option format that the stock rsync produces.
-
-# To disable a short-named option, add its letter to this string:
-our \$short_disabled = 's';
-
-our \$short_no_arg = '$short_no_arg'; # DO NOT REMOVE ANY
-our \$short_with_num = '$short_with_num'; # DO NOT REMOVE ANY
-
-# To disable a long-named option, change its value to a -1.  The values mean:
-# 0 = the option has no arg; 1 = the arg doesn't need any checking; 2 = only
-# check the arg when receiving; and 3 = always check the arg.
-our \%long_opt = (
-EOT
-
-foreach my $opt (sort keys %long_opt) {
-    my $val = $long_opt{$opt};
-    $val = 1 if $opt =~ /^(max-|min-)/;
-    $val = 3 if $opt eq 'files-from';
-    $val = q"$only eq 'r' ? -1 : " . $val if $opt =~ /^(remove-|log-file)/;
-    $val = q"$only eq 'w' ? -1 : " . $val if $opt eq 'sender';
-    print "  '$opt' => $val,\n";
-}
-
-print ");\n\n";

packaging/lsb/rsync.spec

@@ -1,6 +1,6 @@
 Summary: A fast, versatile, remote (and local) file-copying tool
 Name: rsync
-Version: 3.2.2
+Version: 3.2.5
 %define fullversion %{version}pre2
 Release: 0.1.pre2
 %define srcdir src-previews
@@ -79,8 +79,8 @@ rm -rf $RPM_BUILD_ROOT
 %dir /etc/rsync-ssl/certs
 
 %changelog
-* Sun Jun 28 2020 Wayne Davison <wayne@opencoder.net>
-Released 3.2.2pre2.
+* Mon Aug 08 2022 Wayne Davison <wayne@opencoder.net>
+Released 3.2.5pre2.
 
 * Fri Mar 21 2008 Wayne Davison <wayne@opencoder.net>
 Added installation of /etc/xinetd.d/rsync file and some commented-out

packaging/md2html

@@ -1,99 +0,0 @@
-#!/usr/bin/env python3
-
-# Copyright (C) 2020 Wayne Davison
-#
-# This program is freely redistributable.
-
-import os, re, argparse
-
-HTML_START = """\
-<html><head>
-<title>%s</title>
-<link href="https://fonts.googleapis.com/css2?family=Roboto&family=Roboto+Mono&display=swap" rel="stylesheet">
-<style>
-body {
-  max-width: 50em;
-  margin: auto;
-}
-body, b, strong, u {
-  font-family: 'Roboto', sans-serif;
-}
-code {
-  font-family: 'Roboto Mono', monospace;
-  font-weight: bold;
-}
-pre code {
-  display: block;
-  font-weight: normal;
-}
-blockquote pre code {
-  background: #f1f1f1;
-}
-dd p:first-of-type {
-  margin-block-start: 0em;
-}
-</style>
-</head><body>
-"""
-
-HTML_END = """\
-</body></html>
-"""
-
-md_parser = None
-
-def main():
-    for mdfn in args.mdfiles:
-        if not mdfn.endswith('.md'):
-            print('Ignoring non-md input file:', mdfn)
-            continue
-        title = re.sub(r'.*/', '', mdfn).replace('.md', '')
-        htfn = mdfn.replace('.md', '.html')
-
-        print("Parsing", mdfn, '->', htfn)
-
-        with open(mdfn, 'r', encoding='utf-8') as fh:
-            txt = fh.read()
-
-        txt = re.sub(r'\s--\s', '\xa0-- ', txt)
-
-        html = md_parser(txt)
-
-        html = re.sub(r'(<code>)([\s\S]*?)(</code>)', lambda m: m[1] + re.sub(r'\s', '\xa0', m[2]) + m[3], html)
-        html = html.replace('--', '&#8209;&#8209;').replace("\xa0-", '&nbsp;&#8209;').replace("\xa0", '&nbsp;')
-        html = re.sub(r'(\W)-', r'\1&#8209;', html)
-
-        if os.path.lexists(htfn):
-            os.unlink(htfn)
-
-        with open(htfn, 'w', encoding='utf-8') as fh:
-            fh.write(HTML_START % title)
-            fh.write(html)
-            fh.write(HTML_END)
-
-
-def html_via_cmarkgfm(txt):
-    return cmarkgfm.markdown_to_html(txt)
-
-
-def html_via_commonmark(txt):
-    return commonmark.HtmlRenderer().render(commonmark.Parser().parse(txt))
-
-
-if __name__ == '__main__':
-    parser = argparse.ArgumentParser(description='Output html for md pages.', add_help=False)
-    parser.add_argument("--help", "-h", action="help", help="Output this help message and exit.")
-    parser.add_argument("mdfiles", nargs='+', help="The .md files to turn into .html files.")
-    args = parser.parse_args()
-
-    try:
-        import cmarkgfm
-        md_parser = html_via_cmarkgfm
-    except:
-        try:
-            import commonmark
-            md_parser = html_via_commonmark
-        except:
-            die("Failed to find cmarkgfm or commonmark for python3.")
-
-    main()

packaging/patch-update

@@ -33,7 +33,7 @@ def main():
 
     master_commit = latest_git_hash(args.base_branch)
 
-    if cmd_txt_chk(['packaging/prep-auto-dir']) == '':
+    if cmd_txt_chk(['packaging/prep-auto-dir']).out == '':
         die('You must setup an auto-build-save dir to use this script.')
 
     if args.gen:
@@ -127,25 +127,29 @@ def update_patch(patch):
 
     s = cmd_run(['git', 'merge', based_on])
     ok = s.returncode == 0
-    if not ok or args.shell:
+    skip_shell = False
+    if not ok or args.cmd or args.make or args.shell:
         cmd_chk(['packaging/prep-auto-dir'], discard='output')
-        m = re.search(r'([^/]+)$', parent)
-        parent_dir = m[1]
-        if not ok:
-            print(f'"git merge {based_on}" incomplete -- please fix.')
-        os.environ['PS1'] = f"[{parent_dir}] {patch}: "
-        while True:
-            s = cmd_run([os.environ.get('SHELL', '/bin/sh')])
-            if s.returncode != 0:
-                ans = input("Abort? [n/y] ")
-                if re.match(r'^y', ans, flags=re.I):
-                    return 0
-                continue
-            cur_branch, is_clean, status_txt = check_git_status(0)
-            if is_clean:
-                break
-            print(status_txt, end='')
-        cmd_run('rm -f build/*.o build/*/*.o')
+    if not ok:
+        print(f'"git merge {based_on}" incomplete -- please fix.')
+        if not run_a_shell(parent, patch):
+            return 0
+        if not args.make and not args.cmd:
+            skip_shell = True
+    if args.make:
+        if cmd_run(['packaging/smart-make']).returncode != 0:
+            if not run_a_shell(parent, patch):
+                return 0
+            if not args.cmd:
+                skip_shell = True
+    if args.cmd:
+        if cmd_run(args.cmd).returncode != 0:
+            if not run_a_shell(parent, patch):
+                return 0
+            skip_shell = True
+    if args.shell and not skip_shell:
+        if not run_a_shell(parent, patch):
+            return 0
 
     with open(f"{args.patches_dir}/{patch}.diff", 'w', encoding='utf-8') as fh:
         fh.write(description[patch])
@@ -193,16 +197,38 @@ def update_patch(patch):
                 line =  plus_re.sub(r'+++ b/\1', line)
                 fh.write(line)
             proc.communicate()
-            for fn in gen_files:
-                os.unlink(fn)
 
     return 1
 
 
+def run_a_shell(parent, patch):
+    m = re.search(r'([^/]+)$', parent)
+    parent_dir = m[1]
+    os.environ['PS1'] = f"[{parent_dir}] {patch}: "
+
+    while True:
+        s = cmd_run([os.environ.get('SHELL', '/bin/sh')])
+        if s.returncode != 0:
+            ans = input("Abort? [n/y] ")
+            if re.match(r'^y', ans, flags=re.I):
+                return False
+            continue
+        cur_branch, is_clean, status_txt = check_git_status(0)
+        if is_clean:
+            break
+        print(status_txt, end='')
+
+    cmd_run('rm -f build/*.o build/*/*.o')
+
+    return True
+
+
 if __name__ == '__main__':
     parser = argparse.ArgumentParser(description="Turn a git branch back into a diff files in the patches dir.", add_help=False)
     parser.add_argument('--branch', '-b', dest='base_branch', metavar='BASE_BRANCH', default='master', help="The branch the patch is based on. Default: master.")
     parser.add_argument('--skip-check', action='store_true', help="Skip the check that ensures starting with a clean branch.")
+    parser.add_argument('--make', '-m', action='store_true', help="Run the smart-make script in every patch branch.")
+    parser.add_argument('--cmd', '-c', help="Run a command in every patch branch.")
     parser.add_argument('--shell', '-s', action='store_true', help="Launch a shell for every patch/BASE/* branch updated, not just when a conflict occurs.")
     parser.add_argument('--gen', metavar='DIR', nargs='?', const='', help='Include generated files. Optional DIR value overrides the default of using the "patches" dir.')
     parser.add_argument('--patches-dir', '-p', metavar='DIR', default='patches', help="Override the location of the rsync-patches dir. Default: patches.")

packaging/pkglib.py

@@ -1,4 +1,4 @@
-import os, sys, re, subprocess
+import os, sys, re, subprocess, argparse
 
 # This python3 library provides a few helpful routines that are
 # used by the latest packaging scripts.
@@ -23,27 +23,32 @@ def set_default_encoding(enc):
 
 
 # Set shell=True if the cmd is a string; sets a default encoding unless raw=True was specified.
-def _tweak_opts(cmd, opts, **maybe_set):
-    # This sets any maybe_set value that isn't already set AND creates a copy of opts for us.
-    opts = {**maybe_set, **opts}
+def _tweak_opts(cmd, opts, **maybe_set_args):
+    def _maybe_set(o, **msa): # Only set a value if the user didn't already set it.
+        for var, val in msa.items():
+            if var not in o:
+                o[var] = val
+
+    opts = opts.copy()
+    _maybe_set(opts, **maybe_set_args)
 
     if type(cmd) == str:
-        opts = {'shell': True, **opts}
+        _maybe_set(opts, shell=True)
 
     want_raw = opts.pop('raw', False)
     if default_encoding and not want_raw:
-        opts = {'encoding': default_encoding, **opts}
+        _maybe_set(opts, encoding=default_encoding)
 
     capture = opts.pop('capture', None)
     if capture:
         if capture == 'stdout':
-            opts = {'stdout': subprocess.PIPE, **opts}
+            _maybe_set(opts, stdout=subprocess.PIPE)
         elif capture == 'stderr':
-            opts = {'stderr': subprocess.PIPE, **opts}
+            _maybe_set(opts, stderr=subprocess.PIPE)
         elif capture == 'output':
-            opts = {'stdout': subprocess.PIPE, 'stderr': subprocess.PIPE, **opts}
+            _maybe_set(opts, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
         elif capture == 'combined':
-            opts = {'stdout': subprocess.PIPE, 'stderr': subprocess.STDOUT, **opts}
+            _maybe_set(opts, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
 
     discard = opts.pop('discard', None)
     if discard:
@@ -66,30 +71,26 @@ def cmd_chk(cmd, **opts):
     return subprocess.run(cmd, **_tweak_opts(cmd, opts, check=True))
 
 
-# Capture stdout in a string and return the (output, return_code) tuple.
-# Use capture='combined' opt to get both stdout and stderr together.
-def cmd_txt_status(cmd, **opts):
+# Capture stdout in a string and return an object with out, err, and rc (return code).
+# It defaults to capture='stdout' (so err is empty) but can be overridden using
+# capture='combined' or capture='output' (the latter populates the err value).
+def cmd_txt(cmd, **opts):
     input = opts.pop('input', None)
     if input is not None:
         opts['stdin'] = subprocess.PIPE
     proc = subprocess.Popen(cmd, **_tweak_opts(cmd, opts, capture='stdout'))
-    out = proc.communicate(input=input)[0]
-    return (out, proc.returncode)
+    out, err = proc.communicate(input=input)
+    return argparse.Namespace(out=out, err=err, rc=proc.returncode)
 
 
-# Like cmd_txt_status() but just return the output.
-def cmd_txt(cmd, **opts):
-    return cmd_txt_status(cmd, **opts)[0]
-
-
-# Capture stdout in a string and return the output if the command has a 0 return code.
-# Otherwise it throws an exception that indicates the return code and the output.
+# Just like calling cmd_txt() except that it raises an error if the command has a non-0 return code.
+# The raised error includes the cmd, the return code, and the captured output.
 def cmd_txt_chk(cmd, **opts):
-    out, rc = cmd_txt_status(cmd, **opts)
-    if rc != 0:
-        cmd_err = f'Command "{cmd}" returned non-zero exit status "{rc}" and output:\n{out}'
+    ct = cmd_txt(cmd, **opts)
+    if ct.rc != 0:
+        cmd_err = f'Command "{cmd}" returned non-0 exit status "{ct.rc}" and output:\n{ct.out}{ct.err}'
         raise Exception(cmd_err)
-    return out
+    return ct
 
 
 # Starts a piped-output command of stdout (by default) and leaves it up to you to read
@@ -102,7 +103,7 @@ def cmd_pipe(cmd, **opts):
 # arg fatal_unless_clean can be used to make that non-fatal.  Returns a
 # tuple of the current branch, the is_clean flag, and the status text.
 def check_git_status(fatal_unless_clean=True, subdir='.'):
-    status_txt = cmd_txt_chk(f"cd '{subdir}' && git status")
+    status_txt = cmd_txt_chk(f"cd '{subdir}' && git status").out
     is_clean = re.search(r'\nnothing to commit.+working (directory|tree) clean', status_txt) != None
 
     if not is_clean and fatal_unless_clean:
@@ -149,7 +150,7 @@ def check_git_state(master_branch, fatal_unless_clean=True, check_extra_dir=None
 
 # Return the git hash of the most recent commit.
 def latest_git_hash(branch):
-    out = cmd_txt_chk(['git', 'log', '-1', '--no-color', branch])
+    out = cmd_txt_chk(['git', 'log', '-1', '--no-color', branch]).out
     m = re.search(r'^commit (\S+)', out, flags=re.M)
     if not m:
         die(f"Unable to determine commit hash for master branch: {branch}")
@@ -175,20 +176,20 @@ def mandate_gensend_hook():
         print('Creating hook file:', hook)
         cmd_chk(['./rsync', '-a', 'packaging/pre-push', hook])
     else:
-        out, rc = cmd_txt_status(['fgrep', 'make gensend', hook], discard='output')
-        if rc:
+        ct = cmd_txt(['fgrep', 'make gensend', hook], discard='output')
+        if ct.rc:
             die('Please add a "make gensend" into your', hook, 'script.')
 
 
-# Snag the GENFILES values out of the Makefile.in file and return them as a list.
+# Snag the GENFILES values out of the Makefile file and return them as a list.
 def get_gen_files(want_dir_plus_list=False):
     cont_re = re.compile(r'\\\n')
 
     gen_files = [ ]
 
-    auto_dir = os.path.join('auto-build-save', cmd_txt('git rev-parse --abbrev-ref HEAD').strip().replace('/', '%'))
+    auto_dir = os.path.join('auto-build-save', cmd_txt('git rev-parse --abbrev-ref HEAD').out.strip().replace('/', '%'))
 
-    with open('Makefile.in', 'r', encoding='utf-8') as fh:
+    with open(auto_dir + '/Makefile', 'r', encoding='utf-8') as fh:
         for line in fh:
             if not gen_files:
                 chk = re.sub(r'^GENFILES=', '', line)
@@ -217,7 +218,7 @@ def get_rsync_version():
 
 
 def get_NEWS_version_info():
-    rel_re = re.compile(r'^\| \d{2} \w{3} \d{4}\s+\|\s+(?P<ver>\d+\.\d+\.\d+)\s+\|\s+(?P<pdate>\d{2} \w{3} \d{4}\s+)?\|\s+(?P<pver>\d+)\s+\|')
+    rel_re = re.compile(r'^\| \S{2} \w{3} \d{4}\s+\|\s+(?P<ver>\d+\.\d+\.\d+)\s+\|\s+(?P<pdate>\d{2} \w{3} \d{4})?\s+\|\s+(?P<pver>\d+)\s+\|')
     last_version = last_protocol_version = None
     pdate = { }
 
@@ -233,12 +234,11 @@ def get_NEWS_version_info():
                     pdate[m['ver']] = m['pdate']
                 if m['ver'] == last_version:
                     last_protocol_version = m['pver']
-                    break
 
     if not last_protocol_version:
         die(f"Unable to determine protocol_version for {last_version}.")
 
-    return last_version, last_protocol_version
+    return last_version, last_protocol_version, pdate
 
 
 def get_protocol_versions():

packaging/pre-push

@@ -1,3 +1,16 @@
-#!/bin/sh
+#!/bin/bash -e
+
 cat >/dev/null # Just discard stdin data
-make gensend
+
+if [[ -f /proc/$PPID/cmdline ]]; then
+    while read -d $'\0' arg ; do
+	if [[ "$arg" == '--tags' ]] ; then
+	    exit 0
+	fi
+    done </proc/$PPID/cmdline
+fi
+
+branch=`git rev-parse --abbrev-ref HEAD`
+if [[ "$branch" = master && "$*" == *github* ]]; then
+    make gensend
+fi

packaging/prep-auto-dir

@@ -13,8 +13,12 @@
 # run "make distclean" before creating the auto-build-save dir.
 
 auto_top='auto-build-save'
-if test -d $auto_top -a -d .git; then
+if test -d $auto_top && test -d .git; then
     desired_branch=`git rev-parse --abbrev-ref HEAD | tr / %`
+    if test "$desired_branch" = HEAD; then
+	echo "ERROR: switch to the right build dir manually when in detached HEAD mode." 1>&2
+	exit 1
+    fi
     auto_dir="$auto_top/$desired_branch"
     if test -d build; then
 	cur_branch=`readlink build/.branch`

packaging/release-rsync

@@ -18,6 +18,9 @@ dest = os.environ['HOME'] + '/samba-rsync-ftp'
 ORIGINAL_PATH = os.environ['PATH']
 
 def main():
+    if not os.path.isfile('packaging/release-rsync'):
+        die('You must run this script from the top of your rsync checkout.')
+
     now = datetime.now()
     cl_today = now.strftime('* %a %b %d %Y')
     year = now.strftime('%Y')
@@ -30,7 +33,7 @@ def main():
 
     signal.signal(signal.SIGINT, signal_handler)
 
-    if cmd_txt_chk(['packaging/prep-auto-dir']) == '':
+    if cmd_txt_chk(['packaging/prep-auto-dir']).out == '':
         die('You must setup an auto-build-save dir to use this script.');
 
     auto_dir, gen_files = get_gen_files(True)
@@ -44,6 +47,17 @@ def main():
 {dash_line}
 """)
 
+    with open('build/rsync.1') as fh:
+        for line in fh:
+            if line.startswith(r'.\" prefix='):
+                doc_prefix = line.split('=')[1].strip()
+                if doc_prefix != '/usr':
+                    warn(f"*** The documentation was built with prefix {doc_prefix} instead of /usr ***")
+                    die("*** Read the md2man script for a way to override this. ***")
+                break
+            if line.startswith('.P'):
+                die("Failed to find the prefix comment at the start of the rsync.1 manpage.")
+
     if not os.path.isdir(dest):
         die(dest, "dest does not exist")
     if not os.path.isdir('.git'):
@@ -60,7 +74,7 @@ def main():
     curversion = get_rsync_version()
 
     # All version values are strings!
-    lastversion, last_protocol_version = get_NEWS_version_info()
+    lastversion, last_protocol_version, pdate = get_NEWS_version_info()
     protocol_version, subprotocol_version = get_protocol_versions()
 
     version = curversion
@@ -84,7 +98,7 @@ def main():
     if os.path.lexists(rsync_ver):
         die(f'"{rsync_ver}" must not exist in the current directory.')
 
-    out = cmd_txt_chk(['git', 'tag', '-l', v_ver])
+    out = cmd_txt_chk(['git', 'tag', '-l', v_ver]).out
     if out != '':
         print(f"Tag {v_ver} already exists.")
         ans = input("\nDelete tag or quit? [Q/del] ")
@@ -116,11 +130,8 @@ def main():
         release += '.' + pre
 
     finalversion = re.sub(r'pre\d+', '', version)
-    if protocol_version == last_protocol_version:
-        proto_changed = 'unchanged'
-        proto_change_date = ' ' * 11
-    else:
-        proto_changed = 'changed'
+    proto_changed = protocol_version != last_protocol_version
+    if proto_changed:
         if finalversion in pdate:
             proto_change_date = pdate[finalversion]
         else:
@@ -129,6 +140,8 @@ def main():
                 if re.match(r'^\d\d \w\w\w \d\d\d\d$', ans):
                     break
             proto_change_date = ans
+    else:
+        proto_change_date = ' ' * 11
 
     if 'pre' in lastversion:
         if not pre:
@@ -180,7 +193,9 @@ About to:
         with open(fn, 'r', encoding='utf-8') as fh:
             old_txt = txt = fh.read()
         if fn == 'version.h':
-            txt = f'#define RSYNC_VERSION "{version}"\n'
+            x_re = re.compile(r'^(#define RSYNC_VERSION).*', re.M)
+            msg = f"Unable to update RSYNC_VERSION in {fn}"
+            txt = replace_or_die(x_re, r'\1 "%s"' % version, txt, msg)
         elif '.spec' in fn:
             for var, val in specvars.items():
                 x_re = re.compile(r'^%s .*' % re.escape(var), re.M)
@@ -189,14 +204,17 @@ About to:
             txt = replace_or_die(x_re, r'%s \1' % cl_today, txt, f"Unable to update ChangeLog header in {fn}")
         elif fn == 'rsync.h':
             x_re = re.compile('(#define\s+SUBPROTOCOL_VERSION)\s+(\d+)')
-            repl = lambda m: m[1] + ' ' + '0' if not pre or proto_changed != 'changed' else 1 if m[2] == '0' else m[2]
+            repl = lambda m: m[1] + ' ' + ('0' if not pre or not proto_changed else '1' if m[2] == '0' else m[2])
             txt = replace_or_die(x_re, repl, txt, f"Unable to find SUBPROTOCOL_VERSION define in {fn}")
         elif fn == 'NEWS.md':
             efv = re.escape(finalversion)
-            x_re = re.compile(r'^<.+>\s+# NEWS for rsync %s \(UNRELEASED\)\s+Protocol: .+\n' % efv)
+            x_re = re.compile(r'^# NEWS for rsync %s \(UNRELEASED\)\s+## Changes in this version:\n' % efv
+                    + r'(\n### PROTOCOL NUMBER:\s+- The protocol number was changed to \d+\.\n)?')
             rel_day = 'UNRELEASED' if pre else today
-            repl = (f'<a name="{finalversion}"></a>\n\n# NEWS for rsync {finalversion} ({rel_day})\n\n'
-                + f"Protocol: {protocol_version} ({proto_changed})\n")
+            repl = (f'# NEWS for rsync {finalversion} ({rel_day})\n\n'
+                + '## Changes in this version:\n')
+            if proto_changed:
+                repl += f'\n### PROTOCOL NUMBER:\n\n - The protocol number was changed to {protocol_version}.\n'
             good_top = re.sub(r'\(.*?\)', '(UNRELEASED)', repl, 1)
             msg = f"The top lines of {fn} are not in the right format.  It should be:\n" + good_top
             txt = replace_or_die(x_re, repl, txt, msg)
@@ -214,7 +232,7 @@ About to:
     cmd_chk(['packaging/year-tweak'])
 
     print(dash_line)
-    cmd_run("git diff --color | less -p '^diff .*'")
+    cmd_run("git diff")
 
     srctar_name = f"{rsync_ver}.tar.gz"
     pattar_name = f"rsync-patches-{version}.tar.gz"
@@ -231,8 +249,8 @@ About to:
     - git commit all changes
     - generate the manpages
     - merge the {args.master_branch} branch into the patch/{args.master_branch}/* branches
-    - update the files in the "patches" dir and OPTIONALLY
-      (if you type 'y') to launch a shell for each patch
+    - update the files in the "patches" dir and OPTIONALLY (if you type 'y') to
+      run patch-update with the --make option (which opens a shell on error)
 """)
     ans = input("<Press Enter OR 'y' to continue> ")
 
@@ -253,8 +271,8 @@ About to:
         die('Aborting')
 
     if re.match(r'^y', ans, re.I):
-        print(f'\nVisiting all "patch/{args.master_branch}/*" branches ...')
-        cmd_run(f"packaging/patch-update --branch={args.master_branch} --skip-check --shell")
+        print(f'\nRunning smart-make on all "patch/{args.master_branch}/*" branches ...')
+        cmd_run(f"packaging/patch-update --branch={args.master_branch} --skip-check --make")
 
     if os.path.isdir('patches/.git'):
         s = cmd_run(f"cd patches && git commit -a -m 'The patches for {version}.'")
@@ -280,13 +298,13 @@ About to:
     # TODO: is there a better way to ensure that our passphrase is in the agent?
     cmd_run("touch TeMp; gpg --sign TeMp; rm TeMp*")
 
-    out = cmd_txt(f"git tag -s -m 'Version {version}.' {v_ver}", capture='combined')
+    out = cmd_txt(f"git tag -s -m 'Version {version}.' {v_ver}", capture='combined').out
     print(out, end='')
     if 'bad passphrase' in out or 'failed' in out:
         die('Aborting')
 
     if os.path.isdir('patches/.git'):
-        out = cmd_txt(f"cd patches && git tag -s -m 'Version {version}.' {v_ver}", capture='combined')
+        out = cmd_txt(f"cd patches && git tag -s -m 'Version {version}.' {v_ver}", capture='combined').out
         print(out, end='')
         if 'bad passphrase' in out or 'failed' in out:
             die('Aborting')
@@ -322,10 +340,10 @@ About to:
     shutil.rmtree(rsync_ver)
 
     print(f"Updating the other files in {dest} ...")
-    md_files = 'README.md NEWS.md'.split()
+    md_files = 'README.md NEWS.md INSTALL.md'.split()
     html_files = [ fn for fn in gen_pathnames if fn.endswith('.html') ]
     cmd_chk(['rsync', '-a', *md_files, *html_files, dest])
-    cmd_chk(["packaging/md2html"] + [ dest +'/'+ fn for fn in md_files ])
+    cmd_chk(["./md-convert", "--dest", dest, *md_files])
 
     cmd_chk(f"git log --name-status | gzip -9 >{dest}/ChangeLog.gz")
 

packaging/smart-make

@@ -0,0 +1,45 @@
+#!/bin/sh
+ 
+set -e
+
+export LANG=C
+
+branch=`packaging/prep-auto-dir`
+if test x"$branch" = x; then
+    srcdir=.
+else
+    cd build
+    srcdir=..
+fi
+
+if test -f configure.sh; then
+    cp -p configure.sh configure.sh.old
+else
+    touch configure.sh.old
+fi
+
+if test -f .fetch; then
+    $srcdir/prepare-source fetch
+else
+    $srcdir/prepare-source
+fi
+
+if diff configure.sh configure.sh.old >/dev/null 2>&1; then
+    echo "configure.sh is unchanged."
+    rm configure.sh.old
+else
+    echo "configure.sh has CHANGED."
+    if test -f config.status; then
+	./config.status --recheck
+    else
+	$srcdir/configure
+    fi
+fi
+
+./config.status
+
+make all
+
+if test x"$1" = x"check"; then
+    make check
+fi

packaging/systemd/rsync.service

@@ -7,6 +7,7 @@ Documentation=man:rsync(1) man:rsyncd.conf(5)
 [Service]
 ExecStart=/usr/bin/rsync --daemon --no-detach
 RestartSec=1
+Restart=on-failure
 
 # Citing README.md:
 #
@@ -16,14 +17,14 @@ RestartSec=1
 #   This is generally used for public file distribution, [...]
 #
 # So let's assume some extra security is more than welcome here. We do full
-# system protection (which makes it read-only) and hide users' homes and
+# system protection (which makes /usr, /boot, & /etc read-only) and hide
 # devices. To override these defaults, it's best to do so in the drop-in
 # directory, often done via `systemctl edit rsync.service`. The file needs
 # just the bare minimum of the right [heading] and override values.
 # See systemd.unit(5) and search for "drop-in" for full details.
 
 ProtectSystem=full
-ProtectHome=on
+#ProtectHome=on|off|read-only
 PrivateDevices=on
 NoNewPrivileges=on
 

packaging/systemd/rsync@.service

@@ -16,13 +16,13 @@ StandardError=journal
 #   This is generally used for public file distribution, [...]
 #
 # So let's assume some extra security is more than welcome here. We do full
-# system protection (which makes it read-only) and hide users' homes and
+# system protection (which makes /usr, /boot, & /etc read-only) and hide
 # devices. To override these defaults, it's best to do so in the drop-in
 # directory, often done via `systemctl edit rsync@.service`. The file needs
 # just the bare minimum of the right [heading] and override values.
 # See systemd.unit(5) and search for "drop-in" for full details.
 
 ProtectSystem=full
-ProtectHome=on
+#ProtectHome=on|off|read-only
 PrivateDevices=on
 NoNewPrivileges=on

packaging/var-checker

@@ -4,7 +4,7 @@
 # for vars that are defined but not used, and for inconsistent array
 # sizes.  Run it from inside the main rsync directory.
 
-import re, argparse, glob
+import os, sys, re, argparse, glob
 
 VARS_RE = re.compile(r'^(?!(?:extern|enum)\s)([a-zA-Z]\S*\s+.*);', re.M)
 EXTERNS_RE = re.compile(r'^extern\s+(.*);', re.M)
@@ -15,8 +15,15 @@ def main():
     add_syscall_c = set('t_stub.c t_unsafe.c tls.c trimslash.c'.split())
     add_util_c = set('t_stub.c t_unsafe.c'.split())
 
+    if not os.path.exists('syscall.c'):
+        if os.path.exists('var-checker'):
+            os.chdir('..')
+        else:
+            print("Couldn't find the source dir.")
+            sys.exit(1)
+
     syscall_c = slurp_file('syscall.c', True)
-    util_c = slurp_file('util.c', True)
+    util_c = slurp_file('util1.c', True)
 
     for fn in sorted(glob.glob('*.c')):
         txt = slurp_file(fn)

popt/system.h

@@ -11,6 +11,10 @@ extern __const __int32_t *__ctype_toupper;
 /*@=declundef@*/
 #endif
 
+#ifdef __TANDEM
+# include <floss.h(floss_execvp,floss_read)>
+#endif
+
 #include <ctype.h>
 
 #include <errno.h>

prepare-source

@@ -5,8 +5,8 @@
 # Specify one action or more than one to provide a fall-back:
 #
 #   build     build the config files [the default w/no arg]
-#   fetch     fetch the latest dev config files
-#   fetchgen  fetch all the latest dev generated files
+#   fetch     fetch the latest dev autoconfig files
+#   fetchgen  fetch all the latest dev generated files (including manpages)
 #   fetchSRC  fetch the latest dev source files [NON-GENERATED FILES]
 #
 # The script stops after the first successful action.
@@ -32,7 +32,7 @@ if test "$dir" != '.'; then
 	fi
     done
     for fn in configure.sh config.h.in aclocal.m4; do
-	test ! -f $fn -a -f "$dir/$fn" && cp -p "$dir/$fn" $fn
+	test ! -f $fn && test -f "$dir/$fn" && cp -p "$dir/$fn" $fn
     done
 fi
 

prepare-source.mak

@@ -1,7 +1,7 @@
 SHELL=/bin/sh
 
-.PHONY: conf
 conf: configure.sh config.h.in
+.PHONY: conf
 
 aclocal.m4: m4/*.m4
 	aclocal -I m4

progress.c

@@ -4,7 +4,7 @@
  * Copyright (C) 1996-2000 Andrew Tridgell
  * Copyright (C) 1996 Paul Mackerras
  * Copyright (C) 2001, 2002 Martin Pool <mbp@samba.org>
- * Copyright (C) 2003-2020 Wayne Davison
+ * Copyright (C) 2003-2022 Wayne Davison
  *
  * This program is free software; you can redistribute it and/or modify
  * it under the terms of the GNU General Public License as published by
@@ -115,13 +115,13 @@ static void rprint_progress(OFF_T ofs, OFF_T size, struct timeval *now, int is_l
 		units = "kB/s";
 	}
 
-	if (remain < 0)
+	if (remain < 0 || remain > 9999.0 * 3600.0)
 		strlcpy(rembuf, "  ??:??:??", sizeof rembuf);
 	else {
-		snprintf(rembuf, sizeof rembuf, "%4d:%02d:%02d",
-			 (int) (remain / 3600.0),
-			 (int) (remain / 60.0) % 60,
-			 (int) remain % 60);
+		snprintf(rembuf, sizeof rembuf, "%4u:%02u:%02u",
+			 (unsigned int) (remain / 3600.0),
+			 (unsigned int) (remain / 60.0) % 60,
+			 (unsigned int) remain % 60);
 	}
 
 	output_needs_newline = 0;

receiver.c

@@ -3,7 +3,7 @@
  *
  * Copyright (C) 1996-2000 Andrew Tridgell
  * Copyright (C) 1996 Paul Mackerras
- * Copyright (C) 2003-2020 Wayne Davison
+ * Copyright (C) 2003-2022 Wayne Davison
  *
  * This program is free software; you can redistribute it and/or modify
  * it under the terms of the GNU General Public License as published by
@@ -41,6 +41,7 @@ extern int preserve_hard_links;
 extern int preserve_perms;
 extern int write_devices;
 extern int preserve_xattrs;
+extern int do_fsync;
 extern int basis_dir_cnt;
 extern int make_backups;
 extern int cleanup_got_literal;
@@ -394,6 +395,11 @@ static int receive_data(int f_in, char *fname_r, int fd_r, OFF_T size_r,
 
 	sum_len = sum_end(file_sum1);
 
+	if (do_fsync && fd != -1 && fsync(fd) != 0) {
+		rsyserr(FERROR, errno, "fsync failed on %s", full_fname(fname));
+		exit_cleanup(RERR_FILEIO);
+	}
+
 	if (mapbuf)
 		unmap_file(mapbuf);
 
@@ -539,6 +545,9 @@ int recv_files(int f_in, int f_out, char *local_name)
 	if (delay_updates)
 		delayed_bits = bitbag_create(cur_flist->used + 1);
 
+	if (whole_file < 0)
+		whole_file = 0;
+
 	progress_init();
 
 	while (1) {
@@ -584,10 +593,13 @@ int recv_files(int f_in, int f_out, char *local_name)
 		if (DEBUG_GTE(RECV, 1))
 			rprintf(FINFO, "recv_files(%s)\n", fname);
 
-		if (daemon_filter_list.head && (*fname != '.' || fname[1] != '\0')
-		 && check_filter(&daemon_filter_list, FLOG, fname, 0) < 0) {
-			rprintf(FERROR, "attempt to hack rsync failed.\n");
-			exit_cleanup(RERR_PROTOCOL);
+		if (daemon_filter_list.head && (*fname != '.' || fname[1] != '\0')) {
+			int filt_flags = S_ISDIR(file->mode) ? NAME_IS_DIR : NAME_IS_FILE;
+			if (check_filter(&daemon_filter_list, FLOG, fname, filt_flags) < 0) {
+				rprintf(FERROR, "ERROR: rejecting file transfer request for daemon excluded file: %s\n",
+					fname);
+				exit_cleanup(RERR_PROTOCOL);
+			}
 		}
 
 #ifdef SUPPORT_XATTRS
@@ -799,14 +811,16 @@ int recv_files(int f_in, int f_out, char *local_name)
 			continue;
 		}
 
-		if (fd1 != -1 && !(S_ISREG(st.st_mode) || (write_devices && IS_DEVICE(st.st_mode)))) {
+		if (write_devices && IS_DEVICE(st.st_mode)) {
+			if (fd1 != -1 && st.st_size == 0)
+				st.st_size = get_device_size(fd1, fname);
+			/* Mark the file entry as a device so that we don't try to truncate it later on. */
+			file->mode = S_IFBLK | (file->mode & ACCESSPERMS);
+		} else if (fd1 != -1 && !(S_ISREG(st.st_mode))) {
 			close(fd1);
 			fd1 = -1;
 		}
 
-		if (fd1 != -1 && IS_DEVICE(st.st_mode) && st.st_size == 0)
-			st.st_size = get_device_size(fd1, fname);
-
 		/* If we're not preserving permissions, change the file-list's
 		 * mode based on the local permissions and some heuristics. */
 		if (!preserve_perms) {
@@ -826,6 +840,12 @@ int recv_files(int f_in, int f_out, char *local_name)
 		if (inplace || one_inplace)  {
 			fnametmp = one_inplace ? partialptr : fname;
 			fd2 = do_open(fnametmp, O_WRONLY|O_CREAT, 0600);
+#ifdef linux
+			if (fd2 == -1 && errno == EACCES) {
+				/* Maybe the error was due to protected_regular setting? */
+				fd2 = do_open(fname, O_WRONLY, 0600);
+			}
+#endif
 			if (fd2 == -1) {
 				rsyserr(FERROR_XFER, errno, "open %s failed",
 					full_fname(fnametmp));
@@ -878,7 +898,7 @@ int recv_files(int f_in, int f_out, char *local_name)
 					do_unlink(partialptr);
 				handle_partial_dir(partialptr, PDIR_DELETE);
 			}
-		} else if (keep_partial && partialptr && !one_inplace) {
+		} else if (keep_partial && partialptr && (!one_inplace || delay_updates)) {
 			if (!handle_partial_dir(partialptr, PDIR_CREATE)) {
 				rprintf(FERROR,
 					"Unable to create partial-dir for %s -- discarding %s.\n",
@@ -912,7 +932,7 @@ int recv_files(int f_in, int f_out, char *local_name)
 			break;
 		case 0: {
 			enum logcode msgtype = redoing ? FERROR_XFER : FWARNING;
-			if (msgtype == FERROR_XFER || INFO_GTE(NAME, 1)) {
+			if (msgtype == FERROR_XFER || INFO_GTE(NAME, 1) || stdout_format_has_i) {
 				char *errstr, *redostr, *keptstr;
 				if (!(keep_partial && partialptr) && !inplace)
 					keptstr = "discarded";

rsync-ssl

@@ -73,8 +73,16 @@ function rsync_ssl_helper {
 	certopt=""
 	gnutls_cert_opt=""
     else
-	certopt="cert$optsep$RSYNC_SSL_CERT"
-	gnutls_cert_opt="--x509keyfile=$RSYNC_SSL_CERT"
+	certopt="-cert$optsep$RSYNC_SSL_CERT"
+	gnutls_cert_opt="--x509certfile=$RSYNC_SSL_CERT"
+    fi
+
+    if [[ -z "$RSYNC_SSL_KEY" ]]; then
+	keyopt=""
+	gnutls_key_opt=""
+    else
+	keyopt="-key$optsep$RSYNC_SSL_KEY"
+	gnutls_key_opt="--x509keyfile=$RSYNC_SSL_KEY"
     fi
 
     if [[ -z ${RSYNC_SSL_CA_CERT+x} ]]; then
@@ -129,9 +137,9 @@ function rsync_ssl_helper {
     fi
 
     if [[ $RSYNC_SSL_TYPE == openssl ]]; then
-	exec $RSYNC_SSL_OPENSSL s_client $caopt $certopt -quiet -verify_quiet -servername $hostname -connect $hostname:$port
+	exec $RSYNC_SSL_OPENSSL s_client $caopt $certopt $keyopt -quiet -verify_quiet -servername $hostname -verify_hostname $hostname -connect $hostname:$port
     elif [[ $RSYNC_SSL_TYPE == gnutls ]]; then
-	exec $RSYNC_SSL_GNUTLS --logfile=/dev/null $gnutls_cert_opt $gnutls_opts $hostname:$port
+	exec $RSYNC_SSL_GNUTLS --logfile=/dev/null $gnutls_cert_opt $gnutls_key_opt $gnutls_opts $hostname:$port
     else
 	# devzero@web.de came up with this no-tmpfile calling syntax:
 	exec $RSYNC_SSL_STUNNEL -fd 10 11<&0 <<EOF 10<&0 0<&11 11<&-

rsync-ssl.1.md

@@ -1,14 +1,17 @@
-# NAME
+## NAME
 
 rsync-ssl - a helper script for connecting to an ssl rsync daemon
 
-# SYNOPSIS
+## SYNOPSIS
 
 ```
 rsync-ssl [--type=SSL_TYPE] RSYNC_ARGS
 ```
 
-# DESCRIPTION
+The online version of this manpage (that includes cross-linking of topics)
+is available at <https://download.samba.org/pub/rsync/rsync-ssl.1>.
+
+## DESCRIPTION
 
 The rsync-ssl script helps you to run an rsync copy to/from an rsync daemon
 that requires ssl connections.
@@ -20,7 +23,7 @@ environment.  You can specify an overriding port via `--port` or by including
 it in the normal spot in the URL format, though both of those require your
 rsync version to be at least 3.2.0.
 
-# OPTIONS
+## OPTIONS
 
 If the **first** arg is a `--type=SSL_TYPE` option, the script will only use
 that particular program to open an ssl connection instead of trying to find an
@@ -32,33 +35,56 @@ required for this particular option.
 All the other options are passed through to the rsync command, so consult the
 **rsync**(1) manpage for more information on how it works.
 
-# ENVIRONMENT VARIABLES
+## ENVIRONMENT VARIABLES
 
 The ssl helper scripts are affected by the following environment variables:
 
-0.  `RSYNC_SSL_TYPE` Specifies the program type that should be used to open the
-    ssl connection.  It must be one of `openssl` or `stunnel`.  The
-    `--type=SSL_TYPE` option overrides this, when specified.
-0.  `RSYNC_SSL_PORT` If specified, the value is the port number that is used as
-    the default when the user does not specify a port in their rsync command.
-    When not specified, the default port number is 874.  (Note that older rsync
-    versions (prior to 3.2.0) did not communicate an overriding port number
-    value to the helper script.)
-0.  `RSYNC_SSL_CERT` If specified, the value is a filename that contains a
-    certificate to use for the connection.
-0.  `RSYNC_SSL_CA_CERT` If specified, the value is a filename that contains a
-    certificate authority certificate that is used to validate the connection.
-0.  `RSYNC_SSL_OPENSSL` Specifies the openssl executable to run when the
-    connection type is set to openssl.  If unspecified, the $PATH is searched
-    for "openssl".
-0.  `RSYNC_SSL_GNUTLS` Specifies the gnutls-cli executable to run when the
-    connection type is set to gnutls.  If unspecified, the $PATH is searched
-    for "gnutls-cli".
-0.  `RSYNC_SSL_STUNNEL` Specifies the stunnel executable to run when the
-    connection type is set to stunnel.  If unspecified, the $PATH is searched
-    first for "stunnel4" and then for "stunnel".
+0.  `RSYNC_SSL_TYPE`
 
-# EXAMPLES
+    Specifies the program type that should be used to open the ssl connection.
+    It must be one of `openssl` or `stunnel`.  The `--type=SSL_TYPE` option
+    overrides this, when specified.
+
+0.  `RSYNC_SSL_PORT`
+
+    If specified, the value is the port number that is used as the default when
+    the user does not specify a port in their rsync command.  When not
+    specified, the default port number is 874.  (Note that older rsync versions
+    (prior to 3.2.0) did not communicate an overriding port number value to the
+    helper script.)
+
+0.  `RSYNC_SSL_CERT`
+
+    If specified, the value is a filename that contains a certificate to use
+    for the connection.
+
+0.  `RSYNC_SSL_KEY`
+
+    If specified, the value is a filename that contains a key for the provided
+    certificate to use for the connection.
+
+0.  `RSYNC_SSL_CA_CERT`
+
+    If specified, the value is a filename that contains a certificate authority
+    certificate that is used to validate the connection.
+
+0.  `RSYNC_SSL_OPENSSL`
+
+    Specifies the openssl executable to run when the connection type is set to
+    openssl.  If unspecified, the $PATH is searched for "openssl".
+
+0.  `RSYNC_SSL_GNUTLS`
+
+    Specifies the gnutls-cli executable to run when the connection type is set
+    to gnutls.  If unspecified, the $PATH is searched for "gnutls-cli".
+
+0.  `RSYNC_SSL_STUNNEL`
+
+    Specifies the stunnel executable to run when the connection type is set to
+    stunnel.  If unspecified, the $PATH is searched first for "stunnel4" and
+    then for "stunnel".
+
+## EXAMPLES
 
 >     rsync-ssl -aiv example.com::mod/ dest
 
@@ -68,11 +94,16 @@ The ssl helper scripts are affected by the following environment variables:
 
 >     rsync-ssl -aiv rsync://example.com:9874/mod/ dest
 
-# SEE ALSO
+## THE SERVER SIDE
 
-**rsync**(1), **rsyncd.conf**(5)
+For help setting up an SSL/TLS supporting rsync, see the [instructions in
+rsyncd.conf](rsyncd.conf.5#SSL_TLS_Daemon_Setup).
 
-# CAVEATS
+## SEE ALSO
+
+[**rsync**(1)](rsync.1), [**rsyncd.conf**(5)](rsyncd.conf.5)
+
+## CAVEATS
 
 Note that using an stunnel connection requires at least version 4 of stunnel,
 which should be the case on modern systems.  Also, it does not verify a
@@ -85,23 +116,23 @@ release the gnutls-cli command was dropping output, making it unusable.  If
 that bug has been fixed in your version, feel free to put gnutls into an
 exported RSYNC_SSL_TYPE environment variable to make its use the default.
 
-# BUGS
+## BUGS
 
 Please report bugs! See the web site at <https://rsync.samba.org/>.
 
-# VERSION
+## VERSION
 
-This man page is current for version @VERSION@ of rsync.
+This manpage is current for version @VERSION@ of rsync.
 
-# CREDITS
+## CREDITS
 
-rsync is distributed under the GNU General Public License.  See the file
-COPYING for details.
+Rsync is distributed under the GNU General Public License.  See the file
+[COPYING](COPYING) for details.
 
 A web site is available at <https://rsync.samba.org/>.  The site includes an
 FAQ-O-Matic which may cover questions unanswered by this manual page.
 
-# AUTHOR
+## AUTHOR
 
 This manpage was written by Wayne Davison.
 

rsync.c

@@ -3,7 +3,7 @@
  *
  * Copyright (C) 1996 Andrew Tridgell
  * Copyright (C) 1996 Paul Mackerras
- * Copyright (C) 2003-2020 Wayne Davison
+ * Copyright (C) 2003-2022 Wayne Davison
  *
  * This program is free software; you can redistribute it and/or modify
  * it under the terms of the GNU General Public License as published by
@@ -32,7 +32,9 @@ extern int preserve_acls;
 extern int preserve_xattrs;
 extern int preserve_perms;
 extern int preserve_executability;
-extern int preserve_times;
+extern int preserve_mtimes;
+extern int omit_dir_times;
+extern int omit_link_times;
 extern int am_root;
 extern int am_server;
 extern int am_daemon;
@@ -63,8 +65,7 @@ extern char *iconv_opt;
 #define UPDATED_ATIME (1<<3)
 #define UPDATED_ACLS  (1<<4)
 #define UPDATED_MODE  (1<<5)
-
-#define UPDATED_TIMES (UPDATED_MTIME|UPDATED_ATIME)
+#define UPDATED_CRTIME (1<<6)
 
 #ifdef ICONV_CONST
 iconv_t ic_chck = (iconv_t)-1;
@@ -576,14 +577,20 @@ int set_file_attrs(const char *fname, struct file_struct *file, stat_x *sxp,
 		set_xattr(fname, file, fnamecmp, sxp);
 #endif
 
-	if (!preserve_times
-	 || (!(preserve_times & PRESERVE_DIR_TIMES) && S_ISDIR(sxp->st.st_mode))
-	 || (!(preserve_times & PRESERVE_LINK_TIMES) && S_ISLNK(sxp->st.st_mode)))
-		flags |= ATTRS_SKIP_MTIME | ATTRS_SKIP_ATIME;
-	else if (sxp != &sx2)
-		memcpy(&sx2.st, &sxp->st, sizeof (sx2.st));
-	if (!atimes_ndx || S_ISDIR(sxp->st.st_mode))
-		flags |= ATTRS_SKIP_ATIME;
+	if ((omit_dir_times && S_ISDIR(sxp->st.st_mode))
+	 || (omit_link_times && S_ISLNK(sxp->st.st_mode)))
+		flags |= ATTRS_SKIP_MTIME | ATTRS_SKIP_ATIME | ATTRS_SKIP_CRTIME;
+	else {
+		if (!preserve_mtimes)
+			flags |= ATTRS_SKIP_MTIME;
+		if (!atimes_ndx || S_ISDIR(sxp->st.st_mode))
+			flags |= ATTRS_SKIP_ATIME;
+		/* Don't set the creation date on the root folder of an HFS+ volume. */
+		if (sxp->st.st_ino == 2 && S_ISDIR(sxp->st.st_mode))
+			flags |= ATTRS_SKIP_CRTIME;
+	}
+	if (sxp != &sx2)
+		memcpy(&sx2.st, &sxp->st, sizeof sx2.st);
 	if (!(flags & ATTRS_SKIP_MTIME) && !same_mtime(file, &sxp->st, flags & ATTRS_ACCURATE_TIME)) {
 		sx2.st.st_mtime = file->modtime;
 #ifdef ST_MTIME_NSEC
@@ -601,15 +608,33 @@ int set_file_attrs(const char *fname, struct file_struct *file, stat_x *sxp,
 			updated |= UPDATED_ATIME;
 		}
 	}
-	if (updated & UPDATED_TIMES) {
+#ifdef SUPPORT_CRTIMES
+	if (crtimes_ndx && !(flags & ATTRS_SKIP_CRTIME)) {
+		time_t file_crtime = F_CRTIME(file);
+		if (sxp->crtime == 0)
+			sxp->crtime = get_create_time(fname, &sxp->st);
+		if (!same_time(sxp->crtime, 0L, file_crtime, 0L)) {
+			if (
+#ifdef HAVE_GETATTRLIST
+			     do_setattrlist_crtime(fname, file_crtime) == 0
+#elif defined __CYGWIN__
+			     do_SetFileTime(fname, file_crtime) == 0
+#else
+#error Unknown crtimes implementation
+#endif
+			)
+				updated |= UPDATED_CRTIME;
+		}
+	}
+#endif
+	if (updated & (UPDATED_MTIME|UPDATED_ATIME)) {
 		int ret = set_times(fname, &sx2.st);
 		if (ret < 0) {
-			rsyserr(FERROR_XFER, errno, "failed to set times on %s",
-				full_fname(fname));
+			rsyserr(FERROR_XFER, errno, "failed to set times on %s", full_fname(fname));
 			goto cleanup;
 		}
 		if (ret > 0) { /* ret == 1 if symlink could not be set */
-			updated &= ~UPDATED_TIMES;
+			updated &= ~(UPDATED_MTIME|UPDATED_ATIME);
 			file->flags |= FLAG_TIME_FAILED;
 		}
 	}
@@ -718,7 +743,7 @@ int finish_transfer(const char *fname, const char *fnametmp,
 
 	/* Change permissions before putting the file into place. */
 	set_file_attrs(fnametmp, file, NULL, fnamecmp,
-		       ok_to_set_time ? ATTRS_ACCURATE_TIME : ATTRS_SKIP_MTIME | ATTRS_SKIP_ATIME);
+		       ok_to_set_time ? ATTRS_ACCURATE_TIME : ATTRS_SKIP_MTIME | ATTRS_SKIP_ATIME | ATTRS_SKIP_CRTIME);
 
 	/* move tmp file over real file */
 	if (DEBUG_GTE(RECV, 1))
@@ -743,7 +768,7 @@ int finish_transfer(const char *fname, const char *fnametmp,
 
   do_set_file_attrs:
 	set_file_attrs(fnametmp, file, NULL, fnamecmp,
-		       ok_to_set_time ? ATTRS_ACCURATE_TIME : ATTRS_SKIP_MTIME | ATTRS_SKIP_ATIME);
+		       ok_to_set_time ? ATTRS_ACCURATE_TIME : ATTRS_SKIP_MTIME | ATTRS_SKIP_ATIME | ATTRS_SKIP_CRTIME);
 
 	if (temp_copy_name) {
 		if (do_rename(fnametmp, fname) < 0) {

rsync.h

@@ -2,7 +2,7 @@
  * Copyright (C) 1996, 2000 Andrew Tridgell
  * Copyright (C) 1996 Paul Mackerras
  * Copyright (C) 2001, 2002 Martin Pool <mbp@samba.org>
- * Copyright (C) 2003-2020 Wayne Davison
+ * Copyright (C) 2003-2022 Wayne Davison
  *
  * This program is free software; you can redistribute it and/or modify
  * it under the terms of the GNU General Public License as published by
@@ -20,6 +20,7 @@
 
 #define False 0
 #define True 1
+#define Unset (-1) /* Our BOOL values are always an int. */
 
 #define BLOCK_SIZE 700
 #define RSYNC_RSH_ENV "RSYNC_RSH"
@@ -69,7 +70,7 @@
 /* The following XMIT flags require an rsync that uses a varint for the flag values */
 
 #define XMIT_RESERVED_16 (1<<16) 	/* reserved for future fileflags use */
-#define XMIT_RESERVED_17 (1<<17) 	/* reserved for future crtimes use */
+#define XMIT_CRTIME_EQ_MTIME (1<<17)	/* any protocol - restricted by command-line option */
 
 /* These flags are used in the live flist data. */
 
@@ -107,12 +108,22 @@
 #define BITS_EQUAL(b1,b2,mask) (((unsigned)(b1) & (unsigned)(mask)) \
 			     == ((unsigned)(b2) & (unsigned)(mask)))
 
-/* update this if you make incompatible changes */
+/* Update this if you make incompatible changes and ALSO update the
+ * SUBPROTOCOL_VERSION if it is not a final (official) release. */
 #define PROTOCOL_VERSION 31
 
-/* This is used when working on a new protocol version in CVS, and should
- * be a new non-zero value for each CVS change that affects the protocol.
- * It must ALWAYS be 0 when the protocol goes final (and NEVER before)! */
+/* This is used when working on a new protocol version or for any unofficial
+ * protocol tweaks.  It should be a non-zero value for each pre-release repo
+ * change that affects the protocol. The official pre-release versions should
+ * start with 1 (after incrementing the PROTOCOL_VERSION) and go up by 1 for
+ * each new protocol change.  For unofficial changes, pick a fairly large
+ * random number that will hopefully not collide with anyone else's unofficial
+ * protocol.  It must ALWAYS be 0 when the protocol goes final (and official)
+ * and NEVER before!  When rsync negotiates a protocol match, it will only
+ * allow the newest protocol to be used if the SUBPROTOCOL_VERSION matches.
+ * All older protocol versions MUST be compatible with the final, official
+ * release of the protocol, so don't tweak the code to change the protocol
+ * behavior for an older protocol version. */
 #define SUBPROTOCOL_VERSION 0
 
 /* We refuse to interoperate with versions that are not in this range.
@@ -181,6 +192,7 @@
 #define ATTRS_SKIP_MTIME	(1<<1)
 #define ATTRS_ACCURATE_TIME	(1<<2)
 #define ATTRS_SKIP_ATIME	(1<<3)
+#define ATTRS_SKIP_CRTIME	(1<<5)
 
 #define MSG_FLUSH	2
 #define FULL_FLUSH	1
@@ -208,6 +220,7 @@
 #define ITEM_REPORT_GROUP (1<<6)
 #define ITEM_REPORT_ACL (1<<7)
 #define ITEM_REPORT_XATTR (1<<8)
+#define ITEM_REPORT_CRTIME (1<<10)
 #define ITEM_BASIS_TYPE_FOLLOWS (1<<11)
 #define ITEM_XNAME_FOLLOWS (1<<12)
 #define ITEM_IS_NEW (1<<13)
@@ -264,6 +277,10 @@ enum msgcode {
 	MSG_NO_SEND=102,/* sender failed to open a file we wanted */
 };
 
+enum filetype {
+	FT_UNSUPPORTED, FT_REG, FT_DIR, FT_SYMLINK, FT_SPECIAL, FT_DEVICE
+};
+
 #define NDX_DONE -1
 #define NDX_FLIST_EOF -2
 #define NDX_DEL_STATS -3
@@ -297,7 +314,6 @@ enum delret {
 #include "errcode.h"
 
 #include "config.h"
-#include "version.h"
 
 /* The default RSYNC_RSH is always set in config.h. */
 
@@ -322,6 +338,9 @@ enum delret {
 # endif
 # include <string.h>
 #endif
+#ifdef HAVE_BSD_STRING_H
+# include <bsd/string.h>
+#endif
 #ifdef HAVE_STRINGS_H
 # include <strings.h>
 #endif
@@ -441,7 +460,9 @@ enum delret {
 #include <netdb.h>
 #endif
 #include <syslog.h>
+#ifdef HAVE_SYS_FILE_H
 #include <sys/file.h>
+#endif
 
 #ifdef HAVE_DIRENT_H
 # include <dirent.h>
@@ -470,7 +491,22 @@ enum delret {
 #ifdef MAKEDEV_TAKES_3_ARGS
 #define MAKEDEV(devmajor,devminor) makedev(0,devmajor,devminor)
 #else
+#ifndef __TANDEM
 #define MAKEDEV(devmajor,devminor) makedev(devmajor,devminor)
+#else
+# define major DEV_TO_MAJOR
+# define minor DEV_TO_MINOR
+# define MAKEDEV MAJORMINOR_TO_DEV
+#endif
+#endif
+
+#ifdef __TANDEM
+# include <floss.h(floss_read,floss_write,floss_fork,floss_execvp)>
+# include <floss.h(floss_getpwuid,floss_select,floss_seteuid)>
+# define S_IEXEC S_IXUSR
+# define ROOT_UID 65535
+#else
+# define ROOT_UID 0
 #endif
 
 #ifdef HAVE_COMPAT_H
@@ -549,6 +585,14 @@ typedef unsigned int size_t;
 #endif
 #endif
 
+#if !defined __APPLE__ || defined HAVE_GETATTRLIST
+#define SUPPORT_ATIMES 1
+#endif
+
+#if defined HAVE_GETATTRLIST || defined __CYGWIN__
+#define SUPPORT_CRTIMES 1
+#endif
+
 /* Find a variable that is either exactly 32-bits or longer.
  * If some code depends on 32-bit truncation, it will need to
  * take special action in a "#if SIZEOF_INT32 > 4" section. */
@@ -692,6 +736,10 @@ struct ht_int64_node {
 #define NAME_MAX 255
 #endif
 
+#ifndef SIZE_MAX
+#define SIZE_MAX ((size_t)-1)
+#endif
+
 #ifndef INADDR_NONE
 #define INADDR_NONE 0xffffffff
 #endif
@@ -731,6 +779,15 @@ struct ht_int64_node {
 # error Character pointers are not 4 or 8 bytes.
 #endif
 
+#if defined __STDC_VERSION__ && __STDC_VERSION__ >= 199901L
+#define USE_FLEXIBLE_ARRAY 1
+#define SIZE_T_FMT_MOD "z" /* printf supports %zd */
+#define SIZE_T_FMT_CAST size_t
+#else
+#define SIZE_T_FMT_MOD "l" /* printf supports %ld */
+#define SIZE_T_FMT_CAST long
+#endif
+
 union file_extras {
 	int32 num;
 	uint32 unum;
@@ -752,12 +809,17 @@ struct file_struct {
 	uint32 len32;		/* Lowest 32 bits of the file's length */
 	uint16 mode;		/* The item's type and permissions */
 	uint16 flags;		/* The FLAG_* bits for this item */
-	const char basename[1];	/* The basename (AKA filename) follows */
+#ifdef USE_FLEXIBLE_ARRAY
+	const char basename[];	/* The basename (AKA filename) follows */
+#else
+	const char basename[1];	/* A kluge that should work like a flexible array */
+#endif
 };
 
 extern int file_extra_cnt;
 extern int inc_recurse;
 extern int atimes_ndx;
+extern int crtimes_ndx;
 extern int pathname_ndx;
 extern int depth_ndx;
 extern int uid_ndx;
@@ -765,7 +827,11 @@ extern int gid_ndx;
 extern int acls_ndx;
 extern int xattrs_ndx;
 
+#ifdef USE_FLEXIBLE_ARRAY
+#define FILE_STRUCT_LEN (sizeof (struct file_struct))
+#else
 #define FILE_STRUCT_LEN (offsetof(struct file_struct, basename))
+#endif
 #define EXTRA_LEN (sizeof (union file_extras))
 #define DEV_EXTRA_CNT 2
 #define DIRNODE_EXTRA_CNT 3
@@ -816,6 +882,7 @@ extern int xattrs_ndx;
 #define F_XATTR(f) REQ_EXTRA(f, xattrs_ndx)->num
 #define F_NDX(f) REQ_EXTRA(f, unsort_ndx)->num
 #define F_ATIME(f) REQ_EXTRA64(f, atimes_ndx)->num
+#define F_CRTIME(f) REQ_EXTRA64(f, crtimes_ndx)->num
 
 /* These items are per-entry optional: */
 #define F_HL_GNUM(f) OPT_EXTRA(f, START_BUMP(f))->num /* non-dirs */
@@ -857,8 +924,9 @@ extern int xattrs_ndx;
  * Start the flist array at FLIST_START entries and grow it
  * by doubling until FLIST_LINEAR then grow by FLIST_LINEAR
  */
-#define FLIST_START	(32 * 1024)
-#define FLIST_LINEAR	(FLIST_START * 512)
+#define FLIST_START		(32)
+#define FLIST_START_LARGE	(32 * 1024)
+#define FLIST_LINEAR		(FLIST_START_LARGE * 512)
 
 /*
  * Extent size for allocation pools: A minimum size of 128KB
@@ -1034,9 +1102,19 @@ typedef struct {
 
 typedef struct {
 	char name_type;
-	char fname[1]; /* has variable size */
+#ifdef USE_FLEXIBLE_ARRAY
+	char fname[]; /* has variable size */
+#else
+	char fname[1]; /* A kluge that should work like a flexible array */
+#endif
 } relnamecache;
 
+#ifdef USE_FLEXIBLE_ARRAY
+#define RELNAMECACHE_LEN (sizeof (relnamecache))
+#else
+#define RELNAMECACHE_LEN (offsetof(relnamecache, fname))
+#endif
+
 #include "byteorder.h"
 #include "lib/mdigest.h"
 #include "lib/wildmatch.h"
@@ -1058,6 +1136,7 @@ typedef struct {
 
 typedef struct {
     STRUCT_STAT st;
+    time_t crtime;
 #ifdef SUPPORT_ACLS
     struct rsync_acl *acc_acl; /* access ACL */
     struct rsync_acl *def_acl; /* default ACL */
@@ -1093,9 +1172,13 @@ struct name_num_obj {
 	uchar *saw;
 	int saw_len;
 	int negotiated_num;
-	struct name_num_item list[];
+	struct name_num_item list[10]; /* we'll get a compile error/warning if this is ever too small */
 };
 
+#ifdef EXTERNAL_ZLIB
+#define read_buf read_buf_
+#endif
+
 #ifndef __cplusplus
 #include "proto.h"
 #endif
@@ -1254,10 +1337,6 @@ extern int errno;
 #define IS_SPECIAL(mode) (S_ISSOCK(mode) || S_ISFIFO(mode))
 #define IS_DEVICE(mode) (S_ISCHR(mode) || S_ISBLK(mode))
 
-#define PRESERVE_FILE_TIMES	(1<<0)
-#define PRESERVE_DIR_TIMES	(1<<1)
-#define PRESERVE_LINK_TIMES	(1<<2)
-
 /* Initial mask on permissions given to temporary files.  Mask off setuid
      bits and group access because of potential race-condition security
      holes, and mask other access because mode 707 is bizarre */
@@ -1266,19 +1345,23 @@ extern int errno;
 /* handler for null strings in printf format */
 #define NS(s) ((s)?(s):"<NULL>")
 
-extern char *do_malloc;
+extern char *do_calloc;
 
 /* Convenient wrappers for malloc and realloc.  Use them. */
-#define new(type) ((type*)my_alloc(do_malloc, sizeof (type), 1, __FILE__, __LINE__))
-#define new0(type) ((type*)my_alloc(NULL, sizeof (type), 1, __FILE__, __LINE__))
+#define new(type) ((type*)my_alloc(NULL, sizeof (type), 1, __FILE__, __LINE__))
+#define new0(type) ((type*)my_alloc(do_calloc, sizeof (type), 1, __FILE__, __LINE__))
 #define realloc_buf(ptr, num) my_alloc((ptr), (num), 1, __FILE__, __LINE__)
 
-#define new_array(type, num) ((type*)my_alloc(do_malloc, (num), sizeof (type), __FILE__, __LINE__))
-#define new_array0(type, num) ((type*)my_alloc(NULL, (num), sizeof (type), __FILE__, __LINE__))
+#define new_array(type, num) ((type*)my_alloc(NULL, (num), sizeof (type), __FILE__, __LINE__))
+#define new_array0(type, num) ((type*)my_alloc(do_calloc, (num), sizeof (type), __FILE__, __LINE__))
 #define realloc_array(ptr, type, num) ((type*)my_alloc((ptr), (num), sizeof (type), __FILE__, __LINE__))
 
+#undef strdup
 #define strdup(s) my_strdup(s, __FILE__, __LINE__)
 
+#define out_of_memory(msg) _out_of_memory(msg, __FILE__, __LINE__)
+#define overflow_exit(msg) _overflow_exit(msg, __FILE__, __LINE__)
+
 /* use magic gcc attributes to catch format errors */
  void rprintf(enum logcode , const char *, ...)
      __attribute__((format (printf, 2, 3)))
@@ -1343,7 +1426,8 @@ extern short info_levels[], debug_levels[];
 #define INFO_MISC (INFO_FLIST+1)
 #define INFO_MOUNT (INFO_MISC+1)
 #define INFO_NAME (INFO_MOUNT+1)
-#define INFO_PROGRESS (INFO_NAME+1)
+#define INFO_NONREG (INFO_NAME+1)
+#define INFO_PROGRESS (INFO_NONREG+1)
 #define INFO_REMOVE (INFO_PROGRESS+1)
 #define INFO_SKIP (INFO_REMOVE+1)
 #define INFO_STATS (INFO_SKIP+1)
@@ -1393,3 +1477,14 @@ char *getpass(const char *prompt);
 #ifdef MAINTAINER_MODE
 const char *get_panic_action(void);
 #endif
+
+#define NOISY_DEATH(msg) do { \
+    fprintf(stderr, "%s in %s at line %d\n", msg, __FILE__, __LINE__); \
+    exit_cleanup(RERR_UNSUPPORTED); \
+} while (0)
+
+#ifdef HAVE_MALLINFO2
+#define MEM_ALLOC_INFO mallinfo2
+#elif defined HAVE_MALLINFO
+#define MEM_ALLOC_INFO mallinfo
+#endif

rsyncd.conf.5.md

@@ -1,12 +1,15 @@
-# NAME
+## NAME
 
 rsyncd.conf - configuration file for rsync in daemon mode
 
-# SYNOPSIS
+## SYNOPSIS
 
 rsyncd.conf
 
-# DESCRIPTION
+The online version of this manpage (that includes cross-linking of topics)
+is available at <https://download.samba.org/pub/rsync/rsyncd.conf.5>.
+
+## DESCRIPTION
 
 The rsyncd.conf file is the runtime configuration file for rsync when run as an
 rsync daemon.
@@ -14,7 +17,7 @@ rsync daemon.
 The rsyncd.conf file controls authentication, access, logging and available
 modules.
 
-# FILE FORMAT
+## FILE FORMAT
 
 The file consists of modules and parameters. A module begins with the name of
 the module in square brackets and continues until the next module begins.
@@ -40,10 +43,9 @@ The values following the equals sign in parameters are all either a string (no
 quotes needed) or a boolean, which may be given as yes/no, 0/1 or true/false.
 Case is not significant in boolean values, but is preserved in string values.
 
-# LAUNCHING THE RSYNC DAEMON
+## LAUNCHING THE RSYNC DAEMON
 
-The rsync daemon is launched by specifying the `--daemon` option to
-rsync.
+The rsync daemon is launched by specifying the `--daemon` option to rsync.
 
 The daemon must run with root privileges if you wish to use chroot, to bind to
 a port numbered under 1024 (as is the default 873), or to set file ownership.
@@ -60,16 +62,16 @@ When run via inetd you should add a line like this to /etc/services:
 
 and a single line something like this to /etc/inetd.conf:
 
->     rsync   stream  tcp     nowait  root   /usr/bin/rsync rsyncd --daemon
+>     rsync   stream  tcp     nowait  root   @BINDIR@/rsync rsyncd --daemon
 
-Replace "/usr/bin/rsync" with the path to where you have rsync installed on
+Replace "@BINDIR@/rsync" with the path to where you have rsync installed on
 your system.  You will then need to send inetd a HUP signal to tell it to
 reread its config file.
 
 Note that you should **not** send the rsync daemon a HUP signal to force it to
 reread the `rsyncd.conf` file. The file is re-read on each client connection.
 
-# GLOBAL PARAMETERS
+## GLOBAL PARAMETERS
 
 The first parameters in the file (before a [module] header) are the global
 parameters.  Rsync also allows for the use of a "[global]" module name to
@@ -96,9 +98,9 @@ a literal % into a value is to use %%.
 
 0.  `motd file`
 
-    This parameter allows you to specify a "message of the day" to display to
-    clients on each connect. This usually contains site information and any
-    legal notices. The default is no motd file.  This can be overridden by the
+    This parameter allows you to specify a "message of the day" (MOTD) to display
+    to clients on each connect. This usually contains site information and any
+    legal notices. The default is no MOTD file.  This can be overridden by the
     `--dparam=motdfile=FILE` command-line option when starting the daemon.
 
 0.  `pid file`
@@ -126,7 +128,7 @@ a literal % into a value is to use %%.
 
     This parameter can provide endless fun for people who like to tune their
     systems to the utmost degree. You can set all sorts of socket options which
-    may make transfers faster (or slower!). Read the man page for the
+    may make transfers faster (or slower!). Read the manpage for the
     **setsockopt()** system call for details on some of the options you may be
     able to set. By default no special socket options are set.  These settings
     can also be specified via the `--sockopts` command-line option.
@@ -136,7 +138,7 @@ a literal % into a value is to use %%.
     You can override the default backlog value when the daemon listens for
     connections.  It defaults to 5.
 
-# MODULE PARAMETERS
+## MODULE PARAMETERS
 
 After the global parameters you should define a number of modules, each module
 exports a directory tree as a symbolic name. Modules are exported by specifying
@@ -177,7 +179,7 @@ the values of parameters.  See the GLOBAL PARAMETERS section for more details.
 
 0.  `use chroot`
 
-    If "use chroot" is true, the rsync daemon will chroot to the "path" before
+    If "use chroot" is true, the rsync daemon will chroot to the "[path](#)" before
     starting the file transfer with the client.  This has the advantage of
     extra protection against possible implementation security holes, but it has
     the disadvantages of requiring super-user privileges, of not being able to
@@ -186,7 +188,7 @@ the values of parameters.  See the GLOBAL PARAMETERS section for more details.
     below).
 
     As an additional safety feature, you can specify a dot-dir in the module's
-    "path" to indicate the point where the chroot should occur.  This allows
+    "[path](#)" to indicate the point where the chroot should occur.  This allows
     rsync to run in a chroot with a non-"/" path for the top of the transfer
     hierarchy.  Doing this guards against unintended library loading (since
     those absolute paths will not be inside the transfer hierarchy unless you
@@ -197,39 +199,36 @@ the values of parameters.  See the GLOBAL PARAMETERS section for more details.
     chroot would have used the whole path, and the inside-chroot path would
     have been "/".
 
-    When both "use chroot" and "daemon chroot" are false, OR the inside-chroot
+    When both "use chroot" and "[daemon chroot](#)" are false, OR the inside-chroot
     path of "use chroot" is not "/", rsync will: (1) munge symlinks by default
-    for security reasons (see "munge symlinks" for a way to turn this off, but
+    for security reasons (see "[munge symlinks](#)" for a way to turn this off, but
     only if you trust your users), (2) substitute leading slashes in absolute
     paths with the module's path (so that options such as `--backup-dir`,
     `--compare-dest`, etc. interpret an absolute path as rooted in the module's
-    "path" dir), and (3) trim ".." path elements from args if rsync believes
+    "[path](#)" dir), and (3) trim ".." path elements from args if rsync believes
     they would escape the module hierarchy.  The default for "use chroot" is
     true, and is the safer choice (especially if the module is not read-only).
 
-    When this parameter is enabled, the "numeric-ids" option will also default
-    to being enabled (disabling name lookups).  See below for what a chroot
-    needs in order for name lookups to succeed.
+    When this parameter is enabled *and* the "[name converter](#)" parameter is
+    *not* set, the "[numeric ids](#)" parameter will default to being enabled
+    (disabling name lookups).  This means that if you manually setup
+    name-lookup libraries in your chroot (instead of using a name converter)
+    that you need to explicitly set `numeric ids = false` for rsync to do name
+    lookups.
 
     If you copy library resources into the module's chroot area, you should
     protect them through your OS's normal user/group or ACL settings (to
     prevent the rsync module's user from being able to change them), and then
-    hide them from the user's view via "exclude" (see how in the discussion of
-    that parameter).  At that point it will be safe to enable the mapping of
-    users and groups by name using the "numeric ids" daemon parameter (see
-    below).
-
-    Note also that you are free to setup custom user/group information in the
-    chroot area that is different from your normal system.  For example, you
-    could abbreviate the list of users and groups.
+    hide them from the user's view via "[exclude](#)" (see how in the discussion of
+    that parameter).  However, it's easier and safer to setup a name converter.
 
 0.  `daemon chroot`
 
     This parameter specifies a path to which the daemon will chroot before
-    beginning communication with clients. Module paths (and any "use chroot"
+    beginning communication with clients. Module paths (and any "[use chroot](#)"
     settings) will then be related to this one. This lets you choose if you
     want the whole daemon to be chrooted (with this setting), just the
-    transfers to be chrooted (with "use chroot"), or both.  Keep in mind that
+    transfers to be chrooted (with "[use chroot](#)"), or both.  Keep in mind that
     the "daemon chroot" area may need various OS/lib/etc files installed to
     allow the daemon to function.  By default the daemon runs without any
     chrooting.
@@ -258,6 +257,27 @@ the values of parameters.  See the GLOBAL PARAMETERS section for more details.
     others, then you will need to setup multiple rsync daemon processes on
     different ports.
 
+0.  `name converter`
+
+    This parameter lets you specify a program that will be run by the rsync
+    daemon to do user & group conversions between names & ids.  This script
+    is started prior to any chroot being setup, and runs as the daemon user
+    (not the transfer user).  You can specify a fully qualified pathname or
+    a program name that is on the $PATH.
+
+    The program can be used to do normal user & group lookups without having to
+    put any extra files into the chroot area of the module *or* you can do
+    customized conversions.
+
+    The nameconvert program has access to all of the environment variables that
+    are described in the section on `pre-xfer exec`.  This is useful if you
+    want to customize the conversion using information about the module and/or
+    the copy request.
+
+    There is a sample python script in the support dir named "nameconvert" that
+    implements the normal user & group lookups.  Feel free to customize it or
+    just use it as documentation to implement your own.
+
 0.  `numeric ids`
 
     Enabling this parameter disables the mapping of users and groups by name
@@ -266,16 +286,13 @@ the values of parameters.  See the GLOBAL PARAMETERS section for more details.
     transfer behave as if the client had passed the `--numeric-ids`
     command-line option.  By default, this parameter is enabled for chroot
     modules and disabled for non-chroot modules.  Also keep in mind that
-    uid/gid preservation requires the module to be running as root (see "uid")
-    or for "fake super" to be configured.
+    uid/gid preservation requires the module to be running as root (see "[uid](#)")
+    or for "[fake super](#)" to be configured.
 
-    A chroot-enabled module should not have this parameter enabled unless
-    you've taken steps to ensure that the module has the necessary resources it
-    needs to translate names, and that it is not possible for a user to change
-    those resources.  That includes being the code being able to call functions
-    like **getpwuid()**, **getgrgid()**, **getpwname()**, and **getgrnam()**.
-    You should test what libraries and config files are required for your OS
-    and get those setup before starting to test name mapping in rsync.
+    A chroot-enabled module should not have this parameter set to false unless
+    you're using a "[name converter](#)" program *or* you've taken steps to ensure
+    that the module has the necessary resources it needs to translate names and
+    that it is not possible for a user to change those resources.
 
 0.  `munge symlinks`
 
@@ -283,12 +300,12 @@ the values of parameters.  See the GLOBAL PARAMETERS section for more details.
     (non-daemon-affecting) `--munge-links` command-line option (using a method
     described below).  This should help protect your files from user trickery
     when your daemon module is writable.  The default is disabled when
-    "use chroot" is on with an inside-chroot path of "/", OR if "daemon chroot"
+    "[use chroot](#)" is on with an inside-chroot path of "/", OR if "[daemon chroot](#)"
     is on, otherwise it is enabled.
 
     If you disable this parameter on a daemon that is not read-only, there are
     tricks that a user can play with uploaded symlinks to access
-    daemon-excluded items (if your module has any), and, if "use chroot" is
+    daemon-excluded items (if your module has any), and, if "[use chroot](#)" is
     off, rsync can even be tricked into showing or changing data that is
     outside the module's path (as access-permissions allow).
 
@@ -309,7 +326,7 @@ the values of parameters.  See the GLOBAL PARAMETERS section for more details.
     the source code named "munge-symlinks" that can be used to add or remove
     this prefix from your symlinks.
 
-    When this parameter is disabled on a writable module and "use chroot" is
+    When this parameter is disabled on a writable module and "[use chroot](#)" is
     off (or the inside-chroot path is not "/"), incoming symlinks will be
     modified to drop a leading slash and to remove ".." path elements that
     rsync believes will allow a symlink to escape the module's hierarchy.
@@ -325,10 +342,10 @@ the values of parameters.  See the GLOBAL PARAMETERS section for more details.
     conversion in a chroot module without extra files in the chroot area, and
     also ensures that name-translation is done in a consistent manner.  If the
     "charset" parameter is not set, the `--iconv` option is refused, just as if
-    "iconv" had been specified via "refuse options".
+    "iconv" had been specified via "[refuse options](#)".
 
     If you wish to force users to always use `--iconv` for a particular module,
-    add "no-iconv" to the "refuse options" parameter.  Keep in mind that this
+    add "no-iconv" to the "[refuse options](#)" parameter.  Keep in mind that this
     will restrict access to your module to very new rsync clients.
 
 0.  `max connections`
@@ -337,7 +354,7 @@ the values of parameters.  See the GLOBAL PARAMETERS section for more details.
     connections you will allow.  Any clients connecting when the maximum has
     been reached will receive a message telling them to try later.  The default
     is 0, which means no limit.  A negative value disables the module.  See
-    also the "lock file" parameter.
+    also the "[lock file](#)" parameter.
 
 0.  `log file`
 
@@ -366,7 +383,7 @@ the values of parameters.  See the GLOBAL PARAMETERS section for more details.
     facility name which is defined on your system. Common names are auth,
     authpriv, cron, daemon, ftp, kern, lpr, mail, news, security, syslog, user,
     uucp, local0, local1, local2, local3, local4, local5, local6 and local7.
-    The default is daemon.  This setting has no effect if the "log file"
+    The default is daemon.  This setting has no effect if the "[log file](#)"
     setting is a non-empty string (either set in the per-modules settings, or
     inherited from the global settings).
 
@@ -374,7 +391,7 @@ the values of parameters.  See the GLOBAL PARAMETERS section for more details.
 
     This parameter allows you to specify the syslog tag to use when logging
     messages from the rsync daemon. The default is "rsyncd".  This setting has
-    no effect if the "log file" setting is a non-empty string (either set in
+    no effect if the "[log file](#)" setting is a non-empty string (either set in
     the per-modules settings, or inherited from the global settings).
 
     For example, if you wanted each authenticated user's name to be included in
@@ -399,7 +416,7 @@ the values of parameters.  See the GLOBAL PARAMETERS section for more details.
 
 0.  `lock file`
 
-    This parameter specifies the file to use to support the "max connections"
+    This parameter specifies the file to use to support the "[max connections](#)"
     parameter. The rsync daemon uses record locking on this file to ensure that
     the max connections limit is not exceeded for the modules sharing the lock
     file.  The default is `/var/run/rsyncd.lock`.
@@ -411,7 +428,7 @@ the values of parameters.  See the GLOBAL PARAMETERS section for more details.
     "read only" is false then uploads will be possible if file permissions on
     the daemon side allow them. The default is for all modules to be read only.
 
-    Note that "auth users" can override this setting on a per-user basis.
+    Note that "[auth users](#)" can override this setting on a per-user basis.
 
 0.  `write only`
 
@@ -424,13 +441,29 @@ the values of parameters.  See the GLOBAL PARAMETERS section for more details.
     Helpful hint: you probably want to specify "refuse options = delete" for a
     write-only module.
 
+0.  `open noatime`
+
+    When set to True, this parameter tells the rsync daemon to open files with
+    the O_NOATIME flag
+    (on systems that support it) to avoid changing the access time of the files
+    that are being transferred.  If your OS does not support the O_NOATIME flag
+    then rsync will silently ignore this option.  Note also that some
+    filesystems are mounted to avoid updating the atime on read access even
+    without the O_NOATIME flag being set.
+
+    When set to False, this parameters ensures that files on the server are not
+    opened with O_NOATIME.
+
+    When set to Unset (the default) the user controls the setting via
+    `--open-noatime`.
+
 0.  `list`
 
     This parameter determines whether this module is listed when the client
     asks for a listing of available modules.  In addition, if this is false,
     the daemon will pretend the module does not exist when a client denied by
-    "hosts allow" or "hosts deny" attempts to access it.  Realize that if
-    "reverse lookup" is disabled globally but enabled for the module, the
+    "[hosts allow](#)" or "[hosts deny](#)" attempts to access it.  Realize that if
+    "[reverse lookup](#)" is disabled globally but enabled for the module, the
     resulting reverse lookup to a potentially client-controlled DNS server may
     still reveal to the client that it hit an existing module.  The default is
     for modules to be listable.
@@ -439,10 +472,10 @@ the values of parameters.  See the GLOBAL PARAMETERS section for more details.
 
     This parameter specifies the user name or user ID that file transfers to
     and from that module should take place as when the daemon was run as root.
-    In combination with the "gid" parameter this determines what file
+    In combination with the "[gid](#)" parameter this determines what file
     permissions are available. The default when run by a super-user is to
     switch to the system's "nobody" user.  The default for a non-super-user is
-    to not try to change the user.  See also the "gid" parameter.
+    to not try to change the user.  See also the "[gid](#)" parameter.
 
     The RSYNC_USER_NAME environment variable may be used to request that rsync
     run as the authorizing user.  For example, if you want a rsync to run as
@@ -458,7 +491,7 @@ the values of parameters.  See the GLOBAL PARAMETERS section for more details.
     accessing the module.  The first one will be the default group, and any
     extra ones be set as supplemental groups.  You may also specify a "`*`" as
     the first gid in the list, which will be replaced by all the normal groups
-    for the transfer's user (see "uid").  The default when run by a super-user
+    for the transfer's user (see "[uid](#)").  The default when run by a super-user
     is to switch to your OS's "nobody" (or perhaps "nogroup") group with no
     other supplementary groups.  The default for a non-super-user is to not
     change any group attributes (and indeed, your OS may not allow a
@@ -474,13 +507,13 @@ the values of parameters.  See the GLOBAL PARAMETERS section for more details.
 
     This parameter specifies a uid under which the daemon will run. The daemon
     usually runs as user root, and when this is left unset the user is left
-    unchanged. See also the "uid" parameter.
+    unchanged. See also the "[uid](#)" parameter.
 
 0.  `daemon gid`
 
     This parameter specifies a gid under which the daemon will run. The daemon
     usually runs as group root, and when this is left unset, the group is left
-    unchanged. See also the "gid" parameter.
+    unchanged. See also the "[gid](#)" parameter.
 
 0.  `fake super`
 
@@ -501,8 +534,8 @@ the values of parameters.  See the GLOBAL PARAMETERS section for more details.
     or tampering with private administrative files, such as files you may add
     to support uid/gid name translations.
 
-    The daemon filter chain is built from the "filter", "include from",
-    "include", "exclude from", and "exclude" parameters, in that order of
+    The daemon filter chain is built from the "filter", "[include from](#)",
+    "[include](#)", "[exclude from](#)", and "[exclude](#)" parameters, in that order of
     priority.  Anchored patterns are anchored at the root of the module.  To
     prevent access to an entire subtree, for example, "`/secret`", you **must**
     exclude everything in the subtree; the easiest way to do this is with a
@@ -529,8 +562,8 @@ the values of parameters.  See the GLOBAL PARAMETERS section for more details.
 
 0.  `include`
 
-    Use an "include" to override the effects of the "exclude" parameter.  Only
-    one "include" parameter can apply to a given module.  See the "filter"
+    Use an "include" to override the effects of the "[exclude](#)" parameter.  Only
+    one "include" parameter can apply to a given module.  See the "[filter](#)"
     parameter for a description of how excluded files affect the daemon.
 
 0.  `exclude from`
@@ -538,14 +571,14 @@ the values of parameters.  See the GLOBAL PARAMETERS section for more details.
     This parameter specifies the name of a file on the daemon that contains
     daemon exclude patterns, one per line.  Only one "exclude from" parameter
     can apply to a given module; if you have multiple exclude-from files, you
-    can specify them as a merge file in the "filter" parameter.  See the
-    "filter" parameter for a description of how excluded files affect the
+    can specify them as a merge file in the "[filter](#)" parameter.  See the
+    "[filter](#)" parameter for a description of how excluded files affect the
     daemon.
 
 0.  `include from`
 
-    Analogue of "exclude from" for a file of daemon include patterns.  Only one
-    "include from" parameter can apply to a given module.  See the "filter"
+    Analogue of "[exclude from](#)" for a file of daemon include patterns.  Only one
+    "include from" parameter can apply to a given module.  See the "[filter](#)"
     parameter for a description of how excluded files affect the daemon.
 
 0.  `incoming chmod`
@@ -580,7 +613,7 @@ the values of parameters.  See the GLOBAL PARAMETERS section for more details.
     to supply a username and password to connect to the module. A challenge
     response authentication protocol is used for this exchange. The plain text
     usernames and passwords are stored in the file specified by the
-    "secrets file" parameter. The default is for all users to be able to
+    "[secrets file](#)" parameter. The default is for all users to be able to
     connect without a password (this is called "anonymous rsync").
 
     In addition to username matching, you can specify groupname matching via a
@@ -592,7 +625,7 @@ the values of parameters.  See the GLOBAL PARAMETERS section for more details.
     Finally, options may be specified after a colon (:).  The options allow you
     to "deny" a user or a group, set the access to "ro" (read-only), or set the
     access to "rw" (read/write).  Setting an auth-rule-specific ro/rw setting
-    overrides the module's "read only" setting.
+    overrides the module's "[read only](#)" setting.
 
     Be sure to put the rules in the order you want them to be matched, because
     the checking stops at the first matching user or group, and that is the
@@ -630,7 +663,7 @@ the values of parameters.  See the GLOBAL PARAMETERS section for more details.
 
     This parameter specifies the name of a file that contains the
     username:password and/or @groupname:password pairs used for authenticating
-    this module. This file is only consulted if the "auth users" parameter is
+    this module. This file is only consulted if the "[auth users](#)" parameter is
     specified.  The file is line-based and contains one name:password pair per
     line.  Any line has a hash (#) as the very first character on the line is
     considered a comment and is skipped.  The passwords can contain any
@@ -644,14 +677,14 @@ the values of parameters.  See the GLOBAL PARAMETERS section for more details.
     "@groupname:password" line for the group that triggered the authentication.
 
     It is up to you what kind of password entries you want to include, either
-    users, groups, or both.  The use of group rules in "auth users" does not
+    users, groups, or both.  The use of group rules in "[auth users](#)" does not
     require that you specify a group password if you do not want to use shared
     passwords.
 
     There is no default for the "secrets file" parameter, you must choose a
     name (such as `/etc/rsyncd.secrets`).  The file must normally not be
-    readable by "other"; see "strict modes".  If the file is not found or is
-    rejected, no logins for a "user auth" module will be possible.
+    readable by "other"; see "[strict modes](#)".  If the file is not found or is
+    rejected, no logins for an "[auth users](#)" module will be possible.
 
 0.  `strict modes`
 
@@ -669,7 +702,7 @@ the values of parameters.  See the GLOBAL PARAMETERS section for more details.
     client's hostname and IP address.  If none of the patterns match, then the
     connection is rejected.
 
-    Each pattern can be in one of five forms:
+    Each pattern can be in one of six forms:
 
     - a dotted decimal IPv4 address of the form a.b.c.d, or an IPv6 address of
       the form a:b:c::d:e:f. In this case the incoming machine's IP address
@@ -683,12 +716,14 @@ the values of parameters.  See the GLOBAL PARAMETERS section for more details.
       addresses which match the masked IP address will be allowed in.
     - a hostname pattern using wildcards. If the hostname of the connecting IP
       (as determined by a reverse lookup) matches the wildcarded name (using
-      the same rules as normal unix filename matching), the client is allowed
-      in.  This only works if "reverse lookup" is enabled (the default).
+      the same rules as normal Unix filename matching), the client is allowed
+      in.  This only works if "[reverse lookup](#)" is enabled (the default).
     - a hostname. A plain hostname is matched against the reverse DNS of the
-      connecting IP (if "reverse lookup" is enabled), and/or the IP of the
-      given hostname is matched against the connecting IP (if "forward lookup"
+      connecting IP (if "[reverse lookup](#)" is enabled), and/or the IP of the
+      given hostname is matched against the connecting IP (if "[forward lookup](#)"
       is enabled, as it is by default).  Any match will be allowed in.
+    - an '@' followed by a netgroup name, which will match if the reverse DNS
+      of the connecting IP is in the specified netgroup.
 
     Note IPv6 link-local addresses can have a scope in the address
     specification:
@@ -697,12 +732,12 @@ the values of parameters.  See the GLOBAL PARAMETERS section for more details.
     >     fe80::%link1/64
     >     fe80::%link1/ffff:ffff:ffff:ffff::
 
-    You can also combine "hosts allow" with a separate "hosts deny" parameter.
-    If both parameters are specified then the "hosts allow" parameter is
-    checked first and a match results in the client being able to connect. The
-    "hosts deny" parameter is then checked and a match means that the host is
-    rejected. If the host does not match either the "hosts allow" or the
-    "hosts deny" patterns then it is allowed to connect.
+    You can also combine "hosts allow" with "[hosts deny](#)" as a way to add
+    exceptions to your deny list.  When both parameters are specified, the
+    "hosts allow" parameter is checked first and a match results in the client
+    being able to connect.  A non-allowed host is then matched against the
+    "[hosts deny](#)" list to see if it should be rejected.  A host that does not
+    match either list is allowed to connect.
 
     The default is no "hosts allow" parameter, which means all hosts can
     connect.
@@ -712,7 +747,7 @@ the values of parameters.  See the GLOBAL PARAMETERS section for more details.
     This parameter allows you to specify a list of comma- and/or
     whitespace-separated patterns that are matched against a connecting clients
     hostname and IP address. If the pattern matches then the connection is
-    rejected. See the "hosts allow" parameter for more information.
+    rejected. See the "[hosts allow](#)" parameter for more information.
 
     The default is no "hosts deny" parameter, which means all hosts can
     connect.
@@ -720,8 +755,8 @@ the values of parameters.  See the GLOBAL PARAMETERS section for more details.
 0.  `reverse lookup`
 
     Controls whether the daemon performs a reverse lookup on the client's IP
-    address to determine its hostname, which is used for "hosts allow" &
-    "hosts deny" checks and the "%h" log escape.  This is enabled by default,
+    address to determine its hostname, which is used for "[hosts allow](#)" &
+    "[hosts deny](#)" checks and the "%h" log escape.  This is enabled by default,
     but you may wish to disable it to save time if you know the lookup will not
     return a useful result, in which case the daemon will use the name
     "UNDETERMINED" instead.
@@ -761,7 +796,7 @@ the values of parameters.  See the GLOBAL PARAMETERS section for more details.
     logs the transfer at the end, so if a transfer is aborted, no mention will
     be made in the log file.
 
-    If you want to customize the log lines, see the "log format" parameter.
+    If you want to customize the log lines, see the "[log format](#)" parameter.
 
 0.  `log format`
 
@@ -778,7 +813,7 @@ the values of parameters.  See the GLOBAL PARAMETERS section for more details.
     (e.g. "`%''l %'b %f`").
 
     The default log format is "`%o %h [%a] %m (%u) %f %l`", and a "`%t [%p] `"
-    is always prefixed when using the "log file" parameter.  (A perl script
+    is always prefixed when using the "[log file](#)" parameter.  (A perl script
     that will summarize this default log format is included in the rsync source
     code distribution in the "support" subdirectory: rsyncstats.)
 
@@ -889,17 +924,19 @@ the values of parameters.  See the GLOBAL PARAMETERS section for more details.
 
     >     refuse options = * !a !delete* delete-after
 
-    A note on refusing "compress" -- it is better to set the "dont compress"
-    daemon parameter to "`*`" because that disables compression silently
+    A note on refusing "compress": it may be better to set the "[dont compress](#)"
+    daemon parameter to "`*`" and ensure that `RSYNC_COMPRESS_LIST=zlib` is set
+    in the environment of the daemon in order to disable compression silently
     instead of returning an error that forces the client to remove the `-z`
     option.
 
-    If you are un-refusing the compress option, you probably want to match
-    "`!compress*`" so that you also accept the `--compress-level` option.
+    If you are un-refusing the compress option, you may want to match
+    "`!compress*`" if you also want to allow the `--compress-level` option.
 
-    Note that the "write-devices" option is refused by default, but can be
-    explicitly accepted with "`!write-devices`".  The options "log-file" and
-    "log-file-format" are forcibly refused and cannot be accepted.
+    Note that the "copy-devices" & "write-devices" options are refused by
+    default, but they can be explicitly accepted with "`!copy-devices`" and/or
+    "`!write-devices`".  The options "log-file" and "log-file-format" are
+    forcibly refused and cannot be accepted.
 
     Here are all the options that are not matched by wild-cards:
 
@@ -909,18 +946,22 @@ the values of parameters.  See the GLOBAL PARAMETERS section for more details.
       receiver.  While rsync passes the older alias `--log-format` for
       compatibility reasons, this options should not be confused with
       `--log-file-format`.
-    - `--sender`: Use "write only" parameter instead of refusing this.
+    - `--sender`: Use "[write only](#)" parameter instead of refusing this.
     - `--dry-run`, `-n`: Who would want to disable this?
     - `--protect-args`, `-s`: This actually makes transfers safer.
     - `--from0`, `-0`: Makes it easier to accept/refuse `--files-from` without
       affecting this helpful modifier.
-    - `--iconv`: This is auto-disabled based on "charset" parameter.
+    - `--iconv`: This is auto-disabled based on "[charset](#)" parameter.
     - `--no-iconv`: Most transfers use this option.
     - `--checksum-seed`: Is a fairly rare, safe option.
     - `--write-devices`: Is non-wild but also auto-disabled.
 
 0.  `dont compress`
 
+    **NOTE:** This parameter currently has no effect except in one instance: if
+    it is set to "`*`" then it minimizes or disables compression for all files
+    (for those that don't want to refuse the `--compress` option completely).
+
     This parameter allows you to select filenames based on wildcard patterns
     that should not be compressed when pulling files from the daemon (no
     analogous parameter exists to govern the pushing of files to a daemon).
@@ -931,14 +972,14 @@ the values of parameters.  See the GLOBAL PARAMETERS section for more details.
     The "dont compress" parameter takes a space-separated list of
     case-insensitive wildcard patterns. Any source filename matching one of the
     patterns will be compressed as little as possible during the transfer.  If
-    the compression algorithm has an "off" level (such as zlib/zlibx) then no
-    compression occurs for those files.  Other algorithms have the level
-    minimized to reduces the CPU usage as much as possible.
+    the compression algorithm has an "off" level, then no compression occurs
+    for those files.  If an algorithms has the ability to change the level in
+    mid-stream, it will be minimized to reduce the CPU usage as much as
+    possible.
 
     See the `--skip-compress` parameter in the **rsync**(1) manpage for the
-    list of file suffixes that are not compressed by default.  Specifying a
-    value for the "dont compress" parameter changes the default when the daemon
-    is the sender.
+    list of file suffixes that are skipped by default if this parameter is not
+    set.
 
 0.  `early exec`, `pre-xfer exec`, `post-xfer exec`
 
@@ -1000,7 +1041,7 @@ the values of parameters.  See the GLOBAL PARAMETERS section for more details.
     **system()** call's default shell), and use RSYNC_NO_XFER_EXEC to disable
     both options completely.
 
-# CONFIG DIRECTIVES
+## CONFIG DIRECTIVES
 
 There are currently two config directives available that allow a config file to
 incorporate the contents of other files:  `&include` and `&merge`.  Both allow
@@ -1055,7 +1096,7 @@ This would merge any `/etc/rsyncd.d/*.inc` files (for global values that should
 stay in effect), and then include any `/etc/rsyncd.d/*.conf` files (defining
 modules without any global-value cross-talk).
 
-# AUTHENTICATION STRENGTH
+## AUTHENTICATION STRENGTH
 
 The authentication protocol used in rsync is a 128 bit MD4 based challenge
 response system. This is fairly weak protection, though (with at least one
@@ -1070,18 +1111,18 @@ authentication is provided. Use ssh as the transport if you want encryption.
 You can also make use of SSL/TLS encryption if you put rsync behind an
 SSL proxy.
 
-# SSL/TLS Daemon Setup
+## SSL/TLS Daemon Setup
 
 When setting up an rsync daemon for access via SSL/TLS, you will need to
-configure a proxy (such as haproxy or nginx) as the front-end that handles the
-encryption.
+configure a TCP proxy (such as haproxy or nginx) as the front-end that handles
+the encryption.
 
 - You should limit the access to the backend-rsyncd port to only allow the
   proxy to connect.  If it is on the same host as the proxy, then configuring
   it to only listen on localhost is a good idea.
-- You should consider turning on the `proxy protocol` parameter if your proxy
-  supports sending that information.  The examples below assume that this is
-  enabled.
+- You should consider turning on the `proxy protocol` rsync-daemon parameter if
+  your proxy supports sending that information.  The examples below assume that
+  this is enabled.
 
 An example haproxy setup is as follows:
 
@@ -1108,14 +1149,14 @@ An example nginx proxy setup is as follows:
 >        ssl_certificate_key /etc/letsencrypt/example.com/privkey.pem;
 >
 >        proxy_pass localhost:873;
->        proxy_protocol on; # Requires "proxy protocol = true"
+>        proxy_protocol on; # Requires rsyncd.conf "proxy protocol = true"
 >        proxy_timeout 1m;
 >        proxy_connect_timeout 5s;
 >    }
 > }
 > ```
 
-# EXAMPLES
+## DAEMON CONFIG EXAMPLES
 
 A simple rsyncd.conf file that allow anonymous rsync to a ftp area at
 `/home/ftp` would be:
@@ -1164,46 +1205,41 @@ The /etc/rsyncd.secrets file would look something like this:
 >     tridge:mypass
 >     susan:herpass
 
-# FILES
+## FILES
 
 /etc/rsyncd.conf or rsyncd.conf
 
-# SEE ALSO
+## SEE ALSO
 
-**rsync**(1), **rsync-ssl**(1)
+[**rsync**(1)](rsync.1), [**rsync-ssl**(1)](rsync-ssl.1)
 
-# BUGS
+## BUGS
 
 Please report bugs! The rsync bug tracking system is online at
 <https://rsync.samba.org/>.
 
-# VERSION
+## VERSION
 
-This man page is current for version @VERSION@ of rsync.
+This manpage is current for version @VERSION@ of rsync.
 
-# CREDITS
+## CREDITS
 
-rsync is distributed under the GNU General Public License.  See the file
-COPYING for details.
+Rsync is distributed under the GNU General Public License.  See the file
+[COPYING](COPYING) for details.
 
-The primary ftp site for rsync is <ftp://rsync.samba.org/pub/rsync>
+An rsync web site is available at <https://rsync.samba.org/> and its github
+project is <https://github.com/WayneD/rsync>.
 
-A web site is available at <https://rsync.samba.org/>.
-
-We would be delighted to hear from you if you like this program.
-
-This program uses the zlib compression library written by Jean-loup Gailly and
-Mark Adler.
-
-# THANKS
+## THANKS
 
 Thanks to Warren Stanley for his original idea and patch for the rsync daemon.
 Thanks to Karsten Thygesen for his many suggestions and documentation!
 
-# AUTHOR
+## AUTHOR
 
-rsync was written by Andrew Tridgell and Paul Mackerras.  Many people have
-later contributed to it.
+Rsync was originally written by Andrew Tridgell and Paul Mackerras.  Many
+people have later contributed to it. It is currently maintained by Wayne
+Davison.
 
 Mailing lists for support and development are available at
 <https://lists.samba.org/>.

runtests.sh

@@ -1,7 +1,7 @@
 #! /bin/sh
 
 # Copyright (C) 2001, 2002 by Martin Pool <mbp@samba.org>
-# Copyright (C) 2003-2020 Wayne Davison
+# Copyright (C) 2003-2022 Wayne Davison
 
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License version
@@ -155,7 +155,7 @@ if test x"$TOOLDIR" = x; then
     TOOLDIR=`pwd`
 fi
 srcdir=`dirname $0`
-if test x"$srcdir" = x -o x"$srcdir" = x.; then
+if test x"$srcdir" = x || test x"$srcdir" = x.; then
     srcdir="$TOOLDIR"
 fi
 if test x"$rsync_bin" = x; then
@@ -167,10 +167,10 @@ RSYNC="$rsync_bin $*"
 #RSYNC="valgrind $rsync_bin $*"
 
 TLS_ARGS=''
-if egrep '^#define HAVE_LUTIMES 1' config.h >/dev/null; then
+if grep -E '^#define HAVE_LUTIMES 1' config.h >/dev/null; then
     TLS_ARGS="$TLS_ARGS -l"
 fi
-if egrep '#undef CHOWN_MODIFIES_SYMLINK' config.h >/dev/null; then
+if grep -E '#undef CHOWN_MODIFIES_SYMLINK' config.h >/dev/null; then
     TLS_ARGS="$TLS_ARGS -L"
 fi
 
@@ -226,6 +226,8 @@ if [ ! -d "$srcdir" ]; then
     exit 2
 fi
 
+expect_skipped="${RSYNC_EXPECT_SKIPPED-IGNORE}"
+skipped_list=''
 skipped=0
 missing=0
 passed=0
@@ -236,7 +238,7 @@ failed=0
 # failure to aid investigation.  We don't remove the testtmp subdir at
 # the end so that it can be configured as a symlink to a filesystem that
 # has ACLs and xattr support enabled (if desired).
-scratchbase="$TOOLDIR"/testtmp
+scratchbase="${scratchbase:-$TOOLDIR}"/testtmp
 echo "    scratchbase=$scratchbase"
 [ -d "$scratchbase" ] || mkdir "$scratchbase"
 
@@ -249,7 +251,7 @@ prep_scratch() {
     [ -d "$scratchdir" ] && chmod -R u+rwX "$scratchdir" && rm -rf "$scratchdir"
     mkdir "$scratchdir"
     # Get rid of default ACLs and dir-setgid to avoid confusing some tests.
-    $setfacl_nodef "$scratchdir" || true
+    $setfacl_nodef "$scratchdir" 2>/dev/null || true
     chmod g-s "$scratchdir"
     case "$srcdir" in
     /*) ln -s "$srcdir" "$scratchdir/src" ;;
@@ -265,10 +267,12 @@ maybe_discard_scratch() {
 
 if [ "x$whichtests" = x ]; then
     whichtests="*.test"
+    full_run=yes
+else
+    full_run=no
 fi
 
-for testscript in $suitedir/$whichtests
-do
+for testscript in $suitedir/$whichtests; do
     testbase=`echo $testscript | sed -e 's!.*/!!' -e 's/.test\$//'`
     scratchdir="$scratchbase/$testbase"
 
@@ -284,7 +288,7 @@ do
     result=$?
     set -e
 
-    if [ "x$always_log" = xyes -o \( $result != 0 -a $result != 77 -a $result != 78 \) ]
+    if [ "x$always_log" = xyes ] || ( [ $result != 0 ] && [ $result != 77 ] && [ $result != 78 ] )
     then
 	echo "----- $testbase log follows"
 	cat "$scratchdir/test.log"
@@ -306,6 +310,7 @@ do
 	# backticks will fill the whole file onto one line, which is a feature
 	whyskipped=`cat "$scratchdir/whyskipped"`
 	echo "SKIP    $testbase ($whyskipped)"
+	skipped_list="$skipped_list,$testbase"
 	skipped=`expr $skipped + 1`
 	maybe_discard_scratch
 	;;
@@ -331,6 +336,15 @@ echo "      $passed passed"
 [ "$failed" -gt 0 ]  && echo "      $failed failed"
 [ "$skipped" -gt 0 ] && echo "      $skipped skipped"
 [ "$missing" -gt 0 ] && echo "      $missing missing"
+if [ "$full_run" = yes ] && [ "$expect_skipped" != IGNORE ]; then
+    skipped_list=`echo "$skipped_list" | sed 's/^,//'`
+    echo "----- skipped results:"
+    echo "      expected: $expect_skipped"
+    echo "      got:      $skipped_list"
+else
+    skipped_list=''
+    expect_skipped=''
+fi
 echo '------------------------------------------------------------'
 
 # OK, so expr exits with 0 if the result is neither null nor zero; and
@@ -339,5 +353,8 @@ echo '------------------------------------------------------------'
 # because -e is set.
 
 result=`expr $failed + $missing || true`
+if [ "$result" = 0 ] && [ "$skipped_list" != "$expect_skipped" ]; then
+    result=1
+fi
 echo "overall result is $result"
 exit $result

sender.c

@@ -3,7 +3,7 @@
  *
  * Copyright (C) 1996 Andrew Tridgell
  * Copyright (C) 1996 Paul Mackerras
- * Copyright (C) 2003-2020 Wayne Davison
+ * Copyright (C) 2003-2022 Wayne Davison
  *
  * This program is free software; you can redistribute it and/or modify
  * it under the terms of the GNU General Public License as published by
@@ -35,7 +35,9 @@ extern int append_mode;
 extern int copy_links;
 extern int io_error;
 extern int flist_eof;
+extern int whole_file;
 extern int allowed_lull;
+extern int copy_devices;
 extern int preserve_xattrs;
 extern int protocol_version;
 extern int remove_source_files;
@@ -204,6 +206,9 @@ void send_files(int f_in, int f_out)
 	if (DEBUG_GTE(SEND, 1))
 		rprintf(FINFO, "send_files starting\n");
 
+	if (whole_file < 0)
+		whole_file = 0;
+
 	progress_init();
 
 	while (1) {
@@ -362,6 +367,25 @@ void send_files(int f_in, int f_out)
 			exit_cleanup(RERR_FILEIO);
 		}
 
+		if (IS_DEVICE(st.st_mode)) {
+			if (!copy_devices) {
+				rprintf(FERROR, "attempt to copy device contents without --copy-devices\n");
+				exit_cleanup(RERR_PROTOCOL);
+			}
+			if (st.st_size == 0)
+				st.st_size = get_device_size(fd, fname);
+		}
+
+		if (append_mode > 0 && st.st_size < F_LENGTH(file)) {
+			rprintf(FWARNING, "skipped diminished file: %s\n",
+				full_fname(fname));
+			free_sums(s);
+			close(fd);
+			if (protocol_version >= 30)
+				send_msg_int(MSG_NO_SEND, ndx);
+			continue;
+		}
+
 		if (st.st_size) {
 			int32 read_size = MAX(s->blength * 3, MAX_MAP_SIZE);
 			mbuf = map_file(fd, st.st_size, read_size, s->blength);

simd-checksum-avx2.S

@@ -0,0 +1,177 @@
+#include "config.h"
+
+#ifdef USE_ROLL_ASM /* { */
+
+#define CHAR_OFFSET 0 /* Keep this the same as rsync.h, which isn't likely to change. */
+
+#ifdef __APPLE__
+#define get_checksum1_avx2_asm  _get_checksum1_avx2_asm
+#endif
+
+.intel_syntax noprefix
+.text
+
+	.p2align 5
+	.globl get_checksum1_avx2_asm
+
+# rdi=*buf, esi=len, edx=i, rcx= *ps1, r8= *ps2
+get_checksum1_avx2_asm:
+	vmovd	xmm6,[rcx] # load *ps1
+	lea	eax, [rsi-128] # at least 128 bytes to process?
+	cmp	edx, eax
+	jg	.exit
+	lea	rax, .mul_T2[rip]
+	vmovntdqa ymm7, [rax]   # load T2 multiplication constants
+	vmovntdqa ymm12,[rax+32]# from memory.
+	vpcmpeqd  ymm15, ymm15, ymm15 # set all elements to -1.
+
+#if CHAR_OFFSET != 0
+	mov	eax, 32*CHAR_OFFSET
+	vmovd	xmm10, eax
+	vpbroadcastd	ymm10, xmm10
+	mov	eax, 528*CHAR_OFFSET
+	vmovd	xmm13, eax
+	vpbroadcastd ymm13, xmm13
+#endif
+	vpabsb	ymm15, ymm15  # set all byte size elements to 1.
+	add	rdi, rdx
+	vmovdqu ymm2, [rdi]   # preload the first 64 bytes.
+	vmovdqu ymm3, [rdi+32]
+	and	esi, ~63   # only needed during final reduction, 
+			   # done here to avoid a longer nop for 
+			   # alignment below.
+	add	edx, esi
+	shr	rsi, 6	   # longer opcode for alignment
+	add	rdi, 64
+	vpxor	xmm1, xmm1, xmm1 # reset both partial sums accumulators.
+	vpxor	xmm4, xmm4, xmm4
+	mov	eax, [r8]
+	.p2align 4 		# should fit into the LSD allocation queue.
+.loop:
+	vpmaddubsw	ymm0, ymm15, ymm2  # s1 partial sums
+	vpmaddubsw	ymm5, ymm15, ymm3 
+	vmovdqu	ymm8,  [rdi]	# preload the next 
+	vmovdqu	ymm9,  [rdi+32] # 64 bytes.
+	add	rdi, 64
+	vpaddd	ymm4, ymm4, ymm6
+	vpaddw	ymm5, ymm5, ymm0
+	vpsrld	ymm0, ymm5, 16
+	vpaddw	ymm5, ymm0, ymm5
+	vpaddd	ymm6, ymm5, ymm6
+	vpmaddubsw	ymm2, ymm7, ymm2  # s2 partial sums
+	vpmaddubsw	ymm3, ymm12, ymm3
+	prefetcht0	[rdi+384] # prefetch 6 cachelines ahead.
+	vpaddw	ymm3, ymm2, ymm3
+	vpsrldq	ymm2, ymm3, 2
+	vpaddd	ymm3, ymm2, ymm3
+	vpaddd	ymm1, ymm1, ymm3
+
+#if CHAR_OFFSET != 0
+	vpaddd ymm6, ymm10, ymm6  #  32*CHAR_OFFSET
+	vpaddd ymm1, ymm13, ymm1  # 528*CHAR_OFFSET
+#endif
+	vmovdqa ymm2, ymm8   # move the next 64 bytes 
+	vmovdqa ymm3, ymm9   # into the right registers
+	sub	esi, 1
+	jnz	.loop
+
+	# now we reduce the partial sums.
+	vpslld	ymm3, ymm4, 6
+	vpsrldq	ymm2, ymm6, 4
+	
+	vpaddd  ymm0, ymm3, ymm1
+	vpaddd	ymm6, ymm2, ymm6
+	vpsrlq	ymm3, ymm0, 32
+	
+	vpsrldq	ymm2, ymm6, 8
+	vpaddd	ymm0, ymm3, ymm0
+	vpsrldq	ymm3, ymm0, 8
+	vpaddd	ymm6, ymm2, ymm6
+	vpaddd	ymm0, ymm3, ymm0
+	vextracti128	xmm2, ymm6, 0x1
+	vextracti128	xmm1, ymm0, 0x1
+	vpaddd	xmm6, xmm2, xmm6
+	vmovd	[rcx], xmm6
+	vpaddd	xmm1, xmm1, xmm0
+	vmovd	ecx, xmm1
+	add	eax, ecx
+	mov	[r8], eax
+.exit:
+	vzeroupper
+	mov	eax, edx
+	ret
+
+#ifdef __APPLE__
+.data
+	.align 6
+#else
+.section	.rodata
+	.p2align 6 
+#endif
+.mul_T2: 
+	.byte 64
+	.byte 63
+	.byte 62
+	.byte 61
+	.byte 60
+	.byte 59
+	.byte 58
+	.byte 57
+	.byte 56
+	.byte 55
+	.byte 54
+	.byte 53
+	.byte 52
+	.byte 51
+	.byte 50
+	.byte 49
+	.byte 48
+	.byte 47
+	.byte 46
+	.byte 45
+	.byte 44
+	.byte 43
+	.byte 42
+	.byte 41
+	.byte 40
+	.byte 39
+	.byte 38
+	.byte 37
+	.byte 36
+	.byte 35
+	.byte 34
+	.byte 33
+	.byte 32
+	.byte 31
+	.byte 30
+	.byte 29
+	.byte 28
+	.byte 27
+	.byte 26
+	.byte 25
+	.byte 24
+	.byte 23
+	.byte 22
+	.byte 21
+	.byte 20
+	.byte 19
+	.byte 18
+	.byte 17
+	.byte 16
+	.byte 15
+	.byte 14
+	.byte 13
+	.byte 12
+	.byte 11
+	.byte 10
+	.byte 9
+	.byte 8
+	.byte 7
+	.byte 6
+	.byte 5
+	.byte 4
+	.byte 3
+	.byte 2
+	.byte 1
+
+#endif /* } USE_ROLL_ASM */

simd-checksum-x86_64.cpp

@@ -51,12 +51,12 @@
  * GCC 4.x are not supported to ease configure.ac logic.
  */
 
-#ifdef __x86_64__
-#ifdef __cplusplus
+#ifdef __x86_64__ /* { */
+#ifdef __cplusplus /* { */
 
 #include "rsync.h"
 
-#ifdef HAVE_SIMD
+#ifdef USE_ROLL_SIMD /* { */
 
 #include <immintrin.h>
 
@@ -85,7 +85,9 @@ typedef long long __m256i_u __attribute__((__vector_size__(32), __may_alias__, _
 #define SSE2_HADDS_EPI16(a, b) _mm_adds_epi16(SSE2_INTERLEAVE_EVEN_EPI16(a, b), SSE2_INTERLEAVE_ODD_EPI16(a, b))
 #define SSE2_MADDUBS_EPI16(a, b) _mm_adds_epi16(SSE2_MULU_EVEN_EPI8(a, b), SSE2_MULU_ODD_EPI8(a, b))
 
+#ifndef USE_ROLL_ASM
 __attribute__ ((target("default"))) MVSTATIC int32 get_checksum1_avx2_64(schar* buf, int32 len, int32 i, uint32* ps1, uint32* ps2) { return i; }
+#endif
 __attribute__ ((target("default"))) MVSTATIC int32 get_checksum1_ssse3_32(schar* buf, int32 len, int32 i, uint32* ps1, uint32* ps2) { return i; }
 __attribute__ ((target("default"))) MVSTATIC int32 get_checksum1_sse2_32(schar* buf, int32 len, int32 i, uint32* ps1, uint32* ps2) { return i; }
 
@@ -311,6 +313,12 @@ __attribute__ ((target("sse2"))) MVSTATIC int32 get_checksum1_sse2_32(schar* buf
     return i;
 }
 
+#ifdef USE_ROLL_ASM /* { */
+
+extern "C" __attribute__ ((target("avx2"))) int32 get_checksum1_avx2_asm(schar* buf, int32 len, int32 i, uint32* ps1, uint32* ps2);
+
+#else /* } { */
+
 /*
   AVX2 loop per 64 bytes:
     int16 t1[16];
@@ -326,113 +334,107 @@ __attribute__ ((target("sse2"))) MVSTATIC int32 get_checksum1_sse2_32(schar* buf
     s1 += (uint32)(t1[0] + t1[1] + t1[2] + t1[3] + t1[4] + t1[5] + t1[6] + t1[7] + t1[8] + t1[9] + t1[10] + t1[11] + t1[12] + t1[13] + t1[14] + t1[15]) +
           64*CHAR_OFFSET;
  */
+
 __attribute__ ((target("avx2"))) MVSTATIC int32 get_checksum1_avx2_64(schar* buf, int32 len, int32 i, uint32* ps1, uint32* ps2)
 {
     if (len > 64) {
-        // Instructions reshuffled compared to SSE2 for slightly better performance
-        int aligned = ((uintptr_t)buf & 31) == 0;
 
-        uint32 x[8] = {0};
-        x[0] = *ps1;
-        __m256i ss1 = _mm256_lddqu_si256((__m256i_u*)x);
-        x[0] = *ps2;
-        __m256i ss2 = _mm256_lddqu_si256((__m256i_u*)x);
+        uint32 x[4] = {0};
+        __m128i ss1 = _mm_cvtsi32_si128(*ps1);
+        __m128i ss2 = _mm_cvtsi32_si128(*ps2);
 
-        // The order gets shuffled compared to SSE2
-        const int16 mul_t1_buf[16] = {60, 56, 52, 48, 28, 24, 20, 16, 44, 40, 36, 32, 12, 8, 4, 0};
-        __m256i mul_t1 = _mm256_lddqu_si256((__m256i_u*)mul_t1_buf);
+        const char mul_t1_buf[16] = {60, 56, 52, 48, 44, 40, 36, 32, 28, 24, 20, 16, 12, 8, 4, 0};
+	__m128i tmp = _mm_load_si128((__m128i*) mul_t1_buf);
+        __m256i mul_t1 = _mm256_cvtepu8_epi16(tmp);
+	__m256i mul_const = _mm256_broadcastd_epi32(_mm_cvtsi32_si128(4 | (3 << 8) | (2 << 16) | (1 << 24)));
+        __m256i mul_one;
+       	    mul_one = _mm256_abs_epi8(_mm256_cmpeq_epi16(mul_one,mul_one)); // set all vector elements to 1
 
         for (; i < (len-64); i+=64) {
-            // Load ... 2*[int8*32]
+            // Load ... 4*[int8*16]
             __m256i in8_1, in8_2;
-            if (!aligned) {
-                in8_1 = _mm256_lddqu_si256((__m256i_u*)&buf[i]);
-                in8_2 = _mm256_lddqu_si256((__m256i_u*)&buf[i + 32]);
-            } else {
-                in8_1 = _mm256_load_si256((__m256i_u*)&buf[i]);
-                in8_2 = _mm256_load_si256((__m256i_u*)&buf[i + 32]);
-            }
+	    __m128i in8_1_low, in8_2_low, in8_1_high, in8_2_high;
+	    in8_1_low = _mm_loadu_si128((__m128i_u*)&buf[i]);
+	    in8_2_low = _mm_loadu_si128((__m128i_u*)&buf[i+16]);
+	    in8_1_high = _mm_loadu_si128((__m128i_u*)&buf[i+32]);
+	    in8_2_high = _mm_loadu_si128((__m128i_u*)&buf[i+48]);
+	    in8_1 = _mm256_inserti128_si256(_mm256_castsi128_si256(in8_1_low), in8_1_high,1);
+	    in8_2 = _mm256_inserti128_si256(_mm256_castsi128_si256(in8_2_low), in8_2_high,1);
 
-            // Prefetch for next loops. This has no observable effect on the
-            // tested AMD but makes as much as 20% difference on the Intel.
-            // Curiously that same Intel sees no benefit from this with SSE2
-            // or SSSE3.
-            _mm_prefetch(&buf[i + 64], _MM_HINT_T0);
-            _mm_prefetch(&buf[i + 96], _MM_HINT_T0);
-            _mm_prefetch(&buf[i + 128], _MM_HINT_T0);
-            _mm_prefetch(&buf[i + 160], _MM_HINT_T0);
-
-            // (1*buf[i] + 1*buf[i+1]), (1*buf[i+2], 1*buf[i+3]), ... 2*[int16*16]
+            // (1*buf[i] + 1*buf[i+1]), (1*buf[i+2], 1*buf[i+3]), ... 2*[int16*8]
             // Fastest, even though multiply by 1
-            __m256i mul_one = _mm256_set1_epi8(1);
             __m256i add16_1 = _mm256_maddubs_epi16(mul_one, in8_1);
             __m256i add16_2 = _mm256_maddubs_epi16(mul_one, in8_2);
 
-            // (4*buf[i] + 3*buf[i+1]), (2*buf[i+2], buf[i+3]), ... 2*[int16*16]
-            __m256i mul_const = _mm256_set1_epi32(4 + (3 << 8) + (2 << 16) + (1 << 24));
+            // (4*buf[i] + 3*buf[i+1]), (2*buf[i+2], buf[i+3]), ... 2*[int16*8]
             __m256i mul_add16_1 = _mm256_maddubs_epi16(mul_const, in8_1);
             __m256i mul_add16_2 = _mm256_maddubs_epi16(mul_const, in8_2);
 
             // s2 += 64*s1
-            ss2 = _mm256_add_epi32(ss2, _mm256_slli_epi32(ss1, 6));
+            ss2 = _mm_add_epi32(ss2, _mm_slli_epi32(ss1, 6));
 
-            // [t1[0] + t1[1], t1[2] + t1[3] ...] [int16*16]
+            // [sum(t1[0]..t1[7]), X, X, X] [int32*4]; faster than multiple _mm_hadds_epi16
+            __m256i sum_add32 = _mm256_add_epi16(add16_1, add16_2);
+            sum_add32 = _mm256_add_epi16(sum_add32, _mm256_srli_epi32(sum_add32, 16));
+            sum_add32 = _mm256_add_epi16(sum_add32, _mm256_srli_si256(sum_add32, 4));
+            sum_add32 = _mm256_add_epi16(sum_add32, _mm256_srli_si256(sum_add32, 8));
+
+            // [sum(t2[0]..t2[7]), X, X, X] [int32*4]; faster than multiple _mm_hadds_epi16
+            __m256i sum_mul_add32 = _mm256_add_epi16(mul_add16_1, mul_add16_2);
+            sum_mul_add32 = _mm256_add_epi16(sum_mul_add32, _mm256_srli_epi32(sum_mul_add32, 16));
+            sum_mul_add32 = _mm256_add_epi16(sum_mul_add32, _mm256_srli_si256(sum_mul_add32, 4));
+            sum_mul_add32 = _mm256_add_epi16(sum_mul_add32, _mm256_srli_si256(sum_mul_add32, 8));
+
+            // s1 += t1[0] + t1[1] + t1[2] + t1[3] + t1[4] + t1[5] + t1[6] + t1[7]
+	    __m128i sum_add32_hi = _mm256_extracti128_si256(sum_add32, 0x1);
+            ss1 = _mm_add_epi32(ss1, _mm256_castsi256_si128(sum_add32));
+            ss1 = _mm_add_epi32(ss1, sum_add32_hi);
+
+            // s2 += t2[0] + t2[1] + t2[2] + t2[3] + t2[4] + t2[5] + t2[6] + t2[7]
+	    __m128i sum_mul_add32_hi = _mm256_extracti128_si256(sum_mul_add32, 0x1);
+            ss2 = _mm_add_epi32(ss2, _mm256_castsi256_si128(sum_mul_add32));
+            ss2 = _mm_add_epi32(ss2, sum_mul_add32_hi);
+
+            // [t1[0] + t1[1], t1[2] + t1[3] ...] [int16*8]
+            // We could've combined this with generating sum_add32 above and
+            // save an instruction but benchmarking shows that as being slower
             __m256i add16 = _mm256_hadds_epi16(add16_1, add16_2);
 
-            // [t1[0], t1[1], ...] -> [t1[0]*60 + t1[1]*56, ...] [int32*8]
+            // [t1[0], t1[1], ...] -> [t1[0]*28 + t1[1]*24, ...] [int32*4]
             __m256i mul32 = _mm256_madd_epi16(add16, mul_t1);
 
-            // [sum(t1[0]..t1[15]), X, X, X, X, X, X, X] [int32*8]
-            __m256i sum_add32 = _mm256_add_epi16(add16_1, add16_2);
-            sum_add32 = _mm256_add_epi16(sum_add32, _mm256_permute4x64_epi64(sum_add32, 2 + (3 << 2) + (0 << 4) + (1 << 6)));
-            sum_add32 = _mm256_add_epi16(sum_add32, _mm256_slli_si256(sum_add32, 2));
-            sum_add32 = _mm256_add_epi16(sum_add32, _mm256_slli_si256(sum_add32, 4));
-            sum_add32 = _mm256_add_epi16(sum_add32, _mm256_slli_si256(sum_add32, 8));
-            sum_add32 = _mm256_srai_epi32(sum_add32, 16);
-            sum_add32 = _mm256_shuffle_epi32(sum_add32, 3);
-
-            // s1 += t1[0] + t1[1] + t1[2] + t1[3] + t1[4] + t1[5] + t1[6] + t1[7] + t1[8] + t1[9] + t1[10] + t1[11] + t1[12] + t1[13] + t1[14] + t1[15]
-            ss1 = _mm256_add_epi32(ss1, sum_add32);
-
-            // [sum(t2[0]..t2[15]), X, X, X, X, X, X, X] [int32*8]
-            __m256i sum_mul_add32 = _mm256_add_epi16(mul_add16_1, mul_add16_2);
-            sum_mul_add32 = _mm256_add_epi16(sum_mul_add32, _mm256_permute4x64_epi64(sum_mul_add32, 2 + (3 << 2) + (0 << 4) + (1 << 6)));
-            sum_mul_add32 = _mm256_add_epi16(sum_mul_add32, _mm256_slli_si256(sum_mul_add32, 2));
-            sum_mul_add32 = _mm256_add_epi16(sum_mul_add32, _mm256_slli_si256(sum_mul_add32, 4));
-            sum_mul_add32 = _mm256_add_epi16(sum_mul_add32, _mm256_slli_si256(sum_mul_add32, 8));
-            sum_mul_add32 = _mm256_srai_epi32(sum_mul_add32, 16);
-            sum_mul_add32 = _mm256_shuffle_epi32(sum_mul_add32, 3);
-
-            // s2 += t2[0] + t2[1] + t2[2] + t2[3] + t2[4] + t2[5] + t2[6] + t2[7] + t2[8] + t2[9] + t2[10] + t2[11] + t2[12] + t2[13] + t2[14] + t2[15]
-            ss2 = _mm256_add_epi32(ss2, sum_mul_add32);
-
-            // [sum(mul32), X, X, X, X, X, X, X] [int32*8]
-            mul32 = _mm256_add_epi32(mul32, _mm256_permute2x128_si256(mul32, mul32, 1));
+            // [sum(mul32), X, X, X] [int32*4]; faster than multiple _mm_hadd_epi32
             mul32 = _mm256_add_epi32(mul32, _mm256_srli_si256(mul32, 4));
             mul32 = _mm256_add_epi32(mul32, _mm256_srli_si256(mul32, 8));
+	    // prefetch 2 cacheline ahead
+            _mm_prefetch(&buf[i + 160], _MM_HINT_T0);
 
-            // s2 += 60*t1[0] + 56*t1[1] + 52*t1[2] + 48*t1[3] + 44*t1[4] + 40*t1[5] + 36*t1[6] + 32*t1[7] + 28*t1[8] + 24*t1[9] + 20*t1[10] + 16*t1[11] + 12*t1[12] + 8*t1[13] + 4*t1[14]
-            ss2 = _mm256_add_epi32(ss2, mul32);
+            // s2 += 28*t1[0] + 24*t1[1] + 20*t1[2] + 16*t1[3] + 12*t1[4] + 8*t1[5] + 4*t1[6]
+	    __m128i mul32_hi = _mm256_extracti128_si256(mul32, 0x1);
+            ss2 = _mm_add_epi32(ss2, _mm256_castsi256_si128(mul32));
+            ss2 = _mm_add_epi32(ss2, mul32_hi);
 
 #if CHAR_OFFSET != 0
-            // s1 += 64*CHAR_OFFSET
-            __m256i char_offset_multiplier = _mm256_set1_epi32(64 * CHAR_OFFSET);
-            ss1 = _mm256_add_epi32(ss1, char_offset_multiplier);
+            // s1 += 32*CHAR_OFFSET
+            __m128i char_offset_multiplier = _mm_set1_epi32(32 * CHAR_OFFSET);
+            ss1 = _mm_add_epi32(ss1, char_offset_multiplier);
 
-            // s2 += 2080*CHAR_OFFSET
-            char_offset_multiplier = _mm256_set1_epi32(2080 * CHAR_OFFSET);
-            ss2 = _mm256_add_epi32(ss2, char_offset_multiplier);
+            // s2 += 528*CHAR_OFFSET
+            char_offset_multiplier = _mm_set1_epi32(528 * CHAR_OFFSET);
+            ss2 = _mm_add_epi32(ss2, char_offset_multiplier);
 #endif
         }
 
-        _mm256_store_si256((__m256i_u*)x, ss1);
+        _mm_store_si128((__m128i_u*)x, ss1);
         *ps1 = x[0];
-        _mm256_store_si256((__m256i_u*)x, ss2);
+        _mm_store_si128((__m128i_u*)x, ss2);
         *ps2 = x[0];
     }
     return i;
 }
 
+#endif /* } !USE_ROLL_ASM */
+
 static int32 get_checksum1_default_1(schar* buf, int32 len, int32 i, uint32* ps1, uint32* ps2)
 {
     uint32 s1 = *ps1;
@@ -459,7 +461,11 @@ static inline uint32 get_checksum1_cpp(char *buf1, int32 len)
     uint32 s2 = 0;
 
     // multiples of 64 bytes using AVX2 (if available)
+#ifdef USE_ROLL_ASM
+    i = get_checksum1_avx2_asm((schar*)buf1, len, i, &s1, &s2);
+#else
     i = get_checksum1_avx2_64((schar*)buf1, len, i, &s1, &s2);
+#endif
 
     // multiples of 32 bytes using SSSE3 (if available)
     i = get_checksum1_ssse3_32((schar*)buf1, len, i, &s1, &s2);
@@ -521,14 +527,18 @@ static int32 get_checksum1_auto(schar* buf, int32 len, int32 i, uint32* ps1, uin
 
 int main() {
     int i;
-    unsigned char* buf = (unsigned char*)malloc(BLOCK_LEN);
+    unsigned char* buf = (unsigned char*)aligned_alloc(64,BLOCK_LEN);
     for (i = 0; i < BLOCK_LEN; i++) buf[i] = (i + (i % 3) + (i % 11)) % 256;
 
     benchmark("Auto", get_checksum1_auto, (schar*)buf, BLOCK_LEN);
     benchmark("Raw-C", get_checksum1_default_1, (schar*)buf, BLOCK_LEN);
     benchmark("SSE2", get_checksum1_sse2_32, (schar*)buf, BLOCK_LEN);
     benchmark("SSSE3", get_checksum1_ssse3_32, (schar*)buf, BLOCK_LEN);
+#ifdef USE_ROLL_ASM
+    benchmark("AVX2-ASM", get_checksum1_avx2_asm, (schar*)buf, BLOCK_LEN);
+#else
     benchmark("AVX2", get_checksum1_avx2_64, (schar*)buf, BLOCK_LEN);
+#endif
 
     free(buf);
     return 0;
@@ -538,6 +548,6 @@ int main() {
 #pragma clang optimize on
 #endif /* BENCHMARK_SIMD_CHECKSUM1 */
 
-#endif /* HAVE_SIMD */
-#endif /* __cplusplus */
-#endif /* __x86_64__ */
+#endif /* } USE_ROLL_SIMD */
+#endif /* } __cplusplus */
+#endif /* } __x86_64__ */

support/atomic-rsync

@@ -1,92 +1,94 @@
-#!/usr/bin/env perl
-#
+#!/usr/bin/env python3
 # This script lets you update a hierarchy of files in an atomic way by
 # first creating a new hierarchy using rsync's --link-dest option, and
 # then swapping the hierarchy into place.  **See the usage message for
 # more details and some important caveats!**
 
-use strict;
-use warnings;
-use Cwd 'abs_path';
+import os, sys, re, subprocess, shutil
 
-my $RSYNC_PROG = '/usr/bin/rsync';
-my $RM_PROG = '/bin/rm';
+ALT_DEST_ARG_RE = re.compile('^--[a-z][^ =]+-dest(=|$)')
 
-my $dest_dir = $ARGV[-1];
-&usage if !defined $dest_dir || $dest_dir =~ /(^-|^$)/ || grep(/^--help/, @ARGV);
-$dest_dir =~ s{(?<=.)/+$} {};
+RSYNC_PROG = '/usr/bin/rsync'
 
-if (!-d $dest_dir) {
-    die "$dest_dir is not a directory.\nUse --help for help.\n";
-}
+def main():
+    cmd_args = sys.argv[1:]
+    if '--help' in cmd_args:
+        usage_and_exit()
 
-if (@_ = grep(/^--[a-z]+-dest\b/, @ARGV)) {
-    $_ = join(' or ', @_);
-    die "You cannot use the $_ option with atomic-rsync.\nUse --help for help.\n";
-}
+    if len(cmd_args) < 2:
+        usage_and_exit(True)
 
-my $symlink_content = readlink $dest_dir; # undef when a real dir
+    dest_dir = cmd_args[-1].rstrip('/')
+    if dest_dir == '' or dest_dir.startswith('-'):
+        usage_and_exit(True)
 
-my $dest_arg = $dest_dir;
-# This gives us the real destination dir, with all symlinks dereferenced.
-$dest_dir = abs_path($dest_dir);
-if ($dest_dir eq '/') {
-    die qq|You must not use "/" as the destination directory.\nUse --help for help.\n|;
-}
+    if not os.path.isdir(dest_dir):
+        die(dest_dir, "is not a directory or a symlink to a dir.\nUse --help for help.")
 
-my($old_dir, $new_dir);
-if (defined $symlink_content && $dest_dir =~ /-([12])$/) {
-    my $num = 3 - $1;
-    $old_dir = undef;
-    ($new_dir = $dest_dir) =~ s/-[12]$/-$num/;
-    $symlink_content =~ s/-[12]$/-$num/;
-} else {
-    $old_dir = "$dest_dir~old~";
-    $new_dir = "$dest_dir~new~";
-}
+    bad_args = [ arg for arg in cmd_args if ALT_DEST_ARG_RE.match(arg) ]
+    if bad_args:
+        die("You cannot use the", ' or '.join(bad_args), "option with atomic-rsync.\nUse --help for help.")
 
-$ARGV[-1] = "$new_dir/";
+    # We ignore exit-code 24 (file vanished) by default.
+    allowed_exit_codes = '0 ' + os.environ.get('ATOMIC_RSYNC_OK_CODES', '24')
+    try:
+        allowed_exit_codes = set(int(num) for num in re.split(r'[, ]+', allowed_exit_codes) if num != '')
+    except ValueError:
+        die('Invalid integer in ATOMIC_RSYNC_OK_CODES:', allowed_exit_codes[2:])
 
-system($RM_PROG, '-rf', $old_dir) if defined $old_dir && -d $old_dir;
-system($RM_PROG, '-rf', $new_dir) if -d $new_dir;
+    symlink_content = os.readlink(dest_dir) if os.path.islink(dest_dir) else None
 
-if (system($RSYNC_PROG, "--link-dest=$dest_dir", @ARGV)) {
-    if ($? == -1) {
-	print "failed to execute $RSYNC_PROG: $!\n";
-    } elsif ($? & 127) {
-	printf "child died with signal %d, %s coredump\n",
-	    ($? & 127),  ($? & 128) ? 'with' : 'without';
-    } else {
-	printf "child exited with value %d\n", $? >> 8;
-    }
-    exit 1;
-}
+    dest_arg = dest_dir
+    dest_dir = os.path.realpath(dest_dir) # The real destination dir with all symlinks dereferenced
+    if dest_dir == '/':
+        die('You must not use "/" as the destination directory.\nUse --help for help.')
 
-if (!defined $old_dir) {
-    atomic_symlink($symlink_content, $dest_arg);
-    exit;
-}
+    old_dir = new_dir = None
+    if symlink_content is not None and dest_dir.endswith(('-1','-2')):
+        if not symlink_content.endswith(dest_dir[-2:]):
+            die("Symlink suffix out of sync with dest_dir name:", symlink_content, 'vs', dest_dir)
+        num = 3 - int(dest_dir[-1]);
+        old_dir = None
+        new_dir = dest_dir[:-1] + str(num)
+        symlink_content = symlink_content[:-1] + str(num)
+    else:
+        old_dir = dest_dir + '~old~'
+        new_dir = dest_dir + '~new~'
 
-rename($dest_dir, $old_dir) or die "Unable to rename $dest_dir to $old_dir: $!";
-rename($new_dir, $dest_dir) or die "Unable to rename $new_dir to $dest_dir: $!";
+    cmd_args[-1] = new_dir + '/'
 
-exit;
+    if old_dir is not None and os.path.isdir(old_dir):
+        shutil.rmtree(old_dir)
+    if os.path.isdir(new_dir):
+        shutil.rmtree(new_dir)
 
-sub atomic_symlink
-{
-    my($target, $link) = @_;
-    my $newlink = "$link~new~";
+    child = subprocess.run([RSYNC_PROG, '--link-dest=' + dest_dir, *cmd_args])
+    if child.returncode not in allowed_exit_codes:
+        die('The rsync copy failed with code', child.returncode, exitcode=child.returncode)
 
-    unlink($newlink); # Just in case
-    symlink($target, $newlink) or die "Unable to symlink $newlink -> $target: $!\n";
-    rename($newlink, $link) or die "Unable to rename $newlink to $link: $!\n";
-}
+    if not os.path.isdir(new_dir):
+        die('The rsync copy failed to create:', new_dir)
+
+    if old_dir is None:
+        atomic_symlink(symlink_content, dest_arg)
+    else:
+        os.rename(dest_dir, old_dir)
+        os.rename(new_dir, dest_dir)
 
 
-sub usage
-{
-    die <<EOT;
-Usage: atomic-rsync [RSYNC-OPTIONS] HOST:/SOURCE/DIR/ /DEST/DIR/
+def atomic_symlink(target, link):
+    newlink = link + "~new~"
+    try:
+        os.unlink(newlink); # Just in case
+    except OSError:
+        pass
+    os.symlink(target, newlink)
+    os.rename(newlink, link)
+
+
+def usage_and_exit(use_stderr=False):
+    usage_msg = """\
+Usage: atomic-rsync [RSYNC-OPTIONS] [HOST:]/SOURCE/DIR/ /DEST/DIR/
        atomic-rsync [RSYNC-OPTIONS] HOST::MOD/DIR/ /DEST/DIR/
 
 This script lets you update a hierarchy of files in an atomic way by first
@@ -96,27 +98,41 @@ to a local directory, and that directory must already exist.  For example:
 
     mkdir /local/files-1
     ln -s files-1 /local/files
-    atomic-rsync -av host:/remote/files/ /local/files/
+    atomic-rsync -aiv host:/remote/files/ /local/files/
 
-If /local/files is a symlink to a directory that ends in -1 or -2, the
-copy will go to the alternate suffix and the symlink will be changed to
-point to the new dir.  This is a fully atomic update.  If the destination
-is not a symlink (or not a symlink to a *-1 or a *-2 directory), this
-will instead create a directory with "~new~" suffixed, move the current
-directory to a name with "~old~" suffixed, and then move the ~new~
-directory to the original destination name (this double rename is not
-fully atomic, but is rapid).  In both cases, the prior destintaion
-directory will be preserved until the next update, at which point it
-will be deleted.
+If /local/files is a symlink to a directory that ends in -1 or -2, the copy
+will go to the alternate suffix and the symlink will be changed to point to
+the new dir.  This is a fully atomic update.  If the destination is not a
+symlink (or not a symlink to a *-1 or a *-2 directory), this will instead
+create a directory with "~new~" suffixed, move the current directory to a
+name with "~old~" suffixed, and then move the ~new~ directory to the original
+destination name (this double rename is not fully atomic, but is rapid).  In
+both cases, the prior destintaion directory will be preserved until the next
+update, at which point it will be deleted.
 
-In all likelihood, you do NOT want to specify this command:
+By default, rsync exit-code 24 (file vanished) is allowed without halting the
+atomic update.  If you want to change that, specify the environment variable
+ATOMIC_RSYNC_OK_CODES with numeric values separated by spaces and/or commas.
+Specify an empty string to only allow a successful copy.  An override example:
 
-    atomic-rsync -av host:/remote/files /local/
+    ATOMIC_RSYNC_OK_CODES='23 24' atomic-rsync -aiv host:src/ dest/
 
-... UNLESS you want the entire /local dir to be swapped out!
+See the errcode.h file for a list of all the exit codes.
 
 See the "rsync" command for its list of options.  You may not use the
 --link-dest, --compare-dest, or --copy-dest options (since this script
 uses --link-dest to make the transfer efficient).
-EOT
-}
+"""
+    print(usage_msg, file=sys.stderr if use_stderr else sys.stdout)
+    sys.exit(1 if use_stderr else 0)
+
+
+def die(*args, exitcode=1):
+    print(*args, file=sys.stderr)
+    sys.exit(exitcode)
+
+
+if __name__ == '__main__':
+    main()
+
+# vim: sw=4 et

support/git-set-file-times

@@ -27,17 +27,18 @@ def main():
 
     if not args.tree:
         # All modified files keep their current mtime.
-        proc = subprocess.Popen(git + 'ls-files -m -z'.split(), stdout=subprocess.PIPE, encoding='utf-8')
+        proc = subprocess.Popen(git + 'status -z --no-renames'.split(), stdout=subprocess.PIPE, encoding='utf-8')
         out = proc.communicate()[0]
         for fn in out.split('\0'):
-            if fn == '':
+            if fn == '' or (fn[0] != 'M' and fn[1] != 'M'):
                 continue
+            fn = fn[3:]
             if args.list:
                 mtime = os.lstat(fn).st_mtime
                 print_line(fn, mtime, mtime)
             ls.discard(fn)
 
-    cmd = git + 'log -r --name-only --no-color --pretty=raw --no-renames -z'.split()
+    cmd = git + 'log -r --name-only --format=%x00commit%x20%H%n%x00commit_time%x20%ct%n --no-renames -z'.split()
     if args.tree:
         cmd.append(args.tree)
     cmd += ['--'] + args.files
@@ -45,7 +46,7 @@ def main():
     proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, encoding='utf-8')
     for line in proc.stdout:
         line = line.strip()
-        m = re.match(r'^committer .*? (\d+) [-+]\d+$', line)
+        m = re.match(r'^\0commit_time (\d+)$', line)
         if m:
             commit_time = int(m[1])
         elif NULL_COMMIT_RE.search(line):
@@ -79,10 +80,10 @@ def print_line(fn, mtime, commit_time):
 
 
 if __name__ == '__main__':
-    parser = argparse.ArgumentParser(description="Set the times of the current git checkout to their last-changed time.", add_help=False)
+    parser = argparse.ArgumentParser(description="Set the times of the files in the current git checkout to their last-changed time.", add_help=False)
     parser.add_argument('--git-dir', metavar='GIT_DIR', help="The git dir to query (defaults to affecting the current git checkout).")
     parser.add_argument('--tree', metavar='TREE-ISH', help="The tree-ish to query (defaults to the current branch).")
-    parser.add_argument('--prefix', metavar='PREFIX_STR', help="Prepend the PREFIX_STR to each filename we tweak.")
+    parser.add_argument('--prefix', metavar='PREFIX_STR', help="Prepend the PREFIX_STR to each filename we tweak (defaults to the top of current checkout).")
     parser.add_argument('--quiet', '-q', action='store_true', help="Don't output the changed-file information.")
     parser.add_argument('--list', '-l', action='count', help="List files & times instead of changing them. Repeat for Unix timestamp instead of human readable.")
     parser.add_argument('files', metavar='FILE', nargs='*', help="Specify a subset of checked-out files to tweak.")

support/lsh

@@ -1,10 +1,6 @@
 #!/usr/bin/env perl
-# This script can be used as a "remote shell" command that is only
-# capable of pretending to connect to "localhost".  This is useful
-# for testing or for running a local copy where the sender and the
-# receiver needs to use different options (e.g. --fake-super).  If
-# we get -l USER, we try to become the USER, either directly (must
-# be root) or by using "sudo -H -u USER" (requires --sudo option).
+# This is a "local shell" command that works like a remote shell but only for
+# the local host.  See the usage message for more details.
 
 use strict;
 use warnings;
@@ -19,6 +15,8 @@ GetOptions(
     'b|c|D|e|F|i|L|m|O|o|p|R|S|w=s' => sub { }, # Ignore
     'no-cd' => \( my $no_chdir ),
     'sudo' => \( my $use_sudo ),
+    'rrsync=s' => \( my $rrsync_dir ),
+    'rropts=s' => \( my $rrsync_opts ),
 ) or &usage;
 &usage unless @ARGV > 1;
 
@@ -71,16 +69,40 @@ unless ($no_chdir) {
     chdir $home_dir or die "Unable to chdir to $home_dir: $!\n";
 }
 
-push @cmd, '/bin/sh', '-c', "@ARGV";
+if ($rrsync_dir) {
+    $ENV{SSH_ORIGINAL_COMMAND} = join(' ', @ARGV);
+    push @cmd, 'rrsync';
+    if ($rrsync_opts) {
+	foreach my $opt (split(/[ ,]+/, $rrsync_opts)) {
+	    $opt = "-$opt" unless $opt =~ /^-/;
+	    push @cmd, $opt;
+	}
+    }
+    push @cmd, $rrsync_dir;
+} else {
+    push @cmd, '/bin/sh', '-c', "@ARGV";
+}
 exec @cmd;
 die "Failed to exec: $!\n";
 
 sub usage
 {
     die <<EOT;
-Usage: lsh [-l user] [--sudo] [--no-cd] localhost COMMAND [...]
+Usage: lsh [OPTIONS] localhost|lh COMMAND [...]
 
-Note that if you pass hostname "lh" instead of "localhost" that
-the --no-cd option is implied.
+This is a "local shell" command that works like a remote shell but only for the
+local host.  This is useful for rsync testing or for running a local copy where
+the sender and the receiver need to use different options (e.g. --fake-super).
+
+Options:
+
+-l USER        Choose the USER that lsh tries to become.
+--no-cd        Skip the chdir \$HOME (the default with hostname "lh")
+--sudo         Use sudo -H -l USER to become root or the specified USER.
+--rrsync=DIR   Test rrsync restricted copying without using ssh.
+--rropts=STR   The string "munge,no-del,no-lock" would pass 3 options to
+               rrsync (must be combined with --rrsync=DIR).
+
+The script also ignores a bunch of single-letter ssh options.
 EOT
 }

support/lsh.sh

@@ -4,7 +4,8 @@
 # for testing or for running a local copy where the sender and the
 # receiver needs to use different options (e.g. --fake-super).  If
 # we get a -l USER option, we try to use "sudo -u USER" to run the
-# command.
+# command.  Supports only the hostnames "localhost" and "lh", with
+# the latter implying the --no-cd option.
 
 user=''
 do_cd=y # Default path is user's home dir (just like ssh) unless host is "lh".

support/munge-symlinks

@@ -1,60 +1,71 @@
-#!/usr/bin/env perl
+#!/usr/bin/env python3
 # This script will either prefix all symlink values with the string
 # "/rsyncd-munged/" or remove that prefix.
 
-use strict;
-use Getopt::Long;
+import os, sys, argparse
 
-my $SYMLINK_PREFIX = '/rsyncd-munged/';
+SYMLINK_PREFIX = '/rsyncd-munged/'
+PREFIX_LEN = len(SYMLINK_PREFIX)
 
-my $munge_opt;
+def main():
+    for arg in args.names:
+        if os.path.islink(arg):
+            process_one_arg(arg)
+        elif os.path.isdir(arg):
+            for fn in find_symlinks(arg):
+                process_one_arg(fn)
+        else:
+            print("Arg is not a symlink or a dir:", arg, file=sys.stderr)
 
-&GetOptions(
-    'munge' => sub { $munge_opt = 1 },
-    'unmunge' => sub { $munge_opt = 0 },
-    'all' => \( my $all_opt ),
-    'help|h' => \( my $help_opt ),
-) or &usage;
 
-&usage if $help_opt || !defined $munge_opt;
+def find_symlinks(path):
+    for entry in os.scandir(path):
+        if entry.is_symlink():
+            yield entry.path
+        elif entry.is_dir(follow_symlinks=False):
+            yield from find_symlinks(entry.path)
 
-my $munged_re = $all_opt ? qr/^($SYMLINK_PREFIX)+(?=.)/ : qr/^$SYMLINK_PREFIX(?=.)/;
 
-push(@ARGV, '.') unless @ARGV;
+def process_one_arg(fn):
+    lnk = os.readlink(fn)
+    if args.unmunge:
+        if not lnk.startswith(SYMLINK_PREFIX):
+            return
+        lnk = lnk[PREFIX_LEN:]
+        while args.all and lnk.startswith(SYMLINK_PREFIX):
+            lnk = lnk[PREFIX_LEN:]
+    else:
+        if not args.all and lnk.startswith(SYMLINK_PREFIX):
+            return
+        lnk = SYMLINK_PREFIX + lnk
 
-open(PIPE, '-|', 'find', @ARGV, '-type', 'l') or die $!;
+    try:
+        os.unlink(fn)
+    except OSError as e:
+        print("Unable to unlink symlink:", str(e), file=sys.stderr)
+        return
+    try:
+        os.symlink(lnk, fn)
+    except OSError as e:
+        print("Unable to recreate symlink", fn, '->', lnk + ':', str(e), file=sys.stderr)
+        return
+    print(fn, '->', lnk)
 
-while (<PIPE>) {
-    chomp;
-    my $lnk = readlink($_) or next;
-    if ($munge_opt) {
-	next if !$all_opt && $lnk =~ /$munged_re/;
-	$lnk =~ s/^/$SYMLINK_PREFIX/;
-    } else {
-	next unless $lnk =~ s/$munged_re//;
-    }
-    if (!unlink($_)) {
-	warn "Unable to unlink symlink: $_ ($!)\n";
-    } elsif (!symlink($lnk, $_)) {
-	warn "Unable to recreate symlink: $_ -> $lnk ($!)\n";
-    } else {
-	print "$_ -> $lnk\n";
-    }
-}
 
-close PIPE;
-exit;
+if __name__ == '__main__':
+    our_desc = """\
+Adds or removes the %s prefix to/from the start of each symlink's value.
+When given the name of a directory, affects all the symlinks in that directory hierarchy.
+""" % SYMLINK_PREFIX
+    epilog = 'See the "munge symlinks" option in the rsyncd.conf manpage for more details.'
+    parser = argparse.ArgumentParser(description=our_desc, epilog=epilog, add_help=False)
+    uniq_group = parser.add_mutually_exclusive_group()
+    uniq_group.add_argument('--munge', action='store_true', help="Add the prefix to symlinks (the default).")
+    uniq_group.add_argument('--unmunge', action='store_true', help="Remove the prefix from symlinks.")
+    parser.add_argument('--all', action='store_true', help="Always adds the prefix when munging (even if already munged) or removes multiple instances of the prefix when unmunging.")
+    parser.add_argument('--help', '-h', action='help', help="Output this help message and exit.")
+    parser.add_argument('names', metavar='NAME', nargs='+', help="One or more directories and/or symlinks to process.")
+    args = parser.parse_args()
+    main()
 
-sub usage
-{
-    die <<EOT;
-Usage: munge-symlinks --munge|--unmunge [--all] [DIR|SYMLINK...]
-
---munge      Add the $SYMLINK_PREFIX prefix to symlinks if not already
-             present, or always when combined with --all.
---unmunge    Remove one $SYMLINK_PREFIX prefix from symlinks or all
-             such prefixes with --all.
-
-See the "munge symlinks" option in the rsyncd.conf manpage for more details.
-EOT
-}
+# vim: sw=4 et

support/nameconvert

@@ -0,0 +1,50 @@
+#!/usr/bin/env python3
+
+# This implements a simple protocol to do user & group conversions between
+# names & ids.  All input and output consists of simple strings with a
+# terminating newline.
+#
+# The requests can be:
+#
+#   uid ID_NUM\n  ->  NAME\n
+#   gid ID_NUM\n  ->  NAME\n
+#   usr NAME\n    ->  ID_NUM\n
+#   grp NAME\n    ->  ID_NUM\n
+#
+# An unknown ID_NUM or NAME results in an empty return value.
+#
+# This is used by an rsync daemon when configured with the "name converter" and
+# (often) "use chroot = true".  While this converter uses real user & group
+# lookups you could change it to use any mapping idiom you'd like.
+
+import sys, argparse, pwd, grp
+
+def main():
+    for line in sys.stdin:
+        try:
+            req, arg = line.rstrip().split(' ', 1)
+        except:
+            req = None
+        try:
+            if req == 'uid':
+                ans = pwd.getpwuid(int(arg)).pw_name
+            elif req == 'gid':
+                ans = grp.getgrgid(int(arg)).gr_name
+            elif req == 'usr':
+                ans = pwd.getpwnam(arg).pw_uid
+            elif req == 'grp':
+                ans = grp.getgrnam(arg).gr_gid
+            else:
+                print("Invalid request", file=sys.stderr)
+                sys.exit(1)
+        except KeyError:
+            ans = ''
+        print(ans, flush=True)
+
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser(description="Convert users & groups between names & numbers for an rsync daemon.")
+    args = parser.parse_args()
+    main()
+
+# vim: sw=4 et

support/rrsync

@@ -1,252 +1,378 @@
-#!/usr/bin/env perl
-# Name: /usr/local/bin/rrsync (should also have a symlink in /usr/bin)
-# Purpose: Restricts rsync to subdirectory declared in .ssh/authorized_keys
-# Author: Joe Smith <js-cgi@inwap.com> 30-Sep-2004
-# Modified by: Wayne Davison <wayne@opencoder.net>
-use strict;
+#!/usr/bin/env python3
 
-use Socket;
-use Cwd 'abs_path';
-use File::Glob ':glob';
+# Restricts rsync to subdirectory declared in .ssh/authorized_keys.  See
+# the rrsync man page for details of how to make use of this script.
 
-# You may configure these values to your liking.  See also the section
-# of options if you want to disable any options that rsync accepts.
-use constant RSYNC => '/usr/bin/rsync';
-use constant LOGFILE => 'rrsync.log';
+# NOTE: install python3 braceexpand to support brace expansion in the args!
 
-my $Usage = <<EOM;
-Use 'command="$0 [-ro|-wo] SUBDIR"'
-	in front of lines in $ENV{HOME}/.ssh/authorized_keys
-EOM
+# Originally a perl script by: Joe Smith <js-cgi@inwap.com> 30-Sep-2004
+# Python version by: Wayne Davison <wayne@opencoder.net>
 
-# Handle the -ro and -wo options.
-our $only = '';
-while (@ARGV && $ARGV[0] =~ /^-([rw])o$/) {
-    my $r_or_w = $1;
-    if ($only && $only ne $r_or_w) {
-	die "$0: the -ro and -wo options conflict.\n";
-    }
-    $only = $r_or_w;
-    shift;
-}
+# You may configure these 2 values to your liking.  See also the section of
+# short & long options if you want to disable any options that rsync accepts.
+RSYNC = '/usr/bin/rsync'
+LOGFILE = 'rrsync.log' # NOTE: the file must exist for a line to be appended!
 
-our $subdir = shift;
-die "$0: No subdirectory specified\n$Usage" unless defined $subdir;
-$subdir = abs_path($subdir);
-die "$0: Restricted directory does not exist!\n" if $subdir ne '/' && !-d $subdir;
+# The following options are mainly the options that a client rsync can send
+# to the server, and usually just in the one option format that the stock
+# rsync produces. However, there are some additional convenience options
+# added as well, and thus a few options are present in both the short and
+# long lists (such as --group, --owner, and --perms).
 
-# The client uses "rsync -av -e ssh src/ server:dir/", and sshd on the server
-# executes this program when .ssh/authorized_keys has 'command="..."'.
-# For example:
-# command="rrsync logs/client" ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAIEAzGhEeNlPr...
-# command="rrsync -ro results" ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAIEAmkHG1WCjC...
-#
-# Format of the environment variables set by sshd:
-# SSH_ORIGINAL_COMMAND=rsync --server          -vlogDtpr --partial . ARG # push
-# SSH_ORIGINAL_COMMAND=rsync --server --sender -vlogDtpr --partial . ARGS # pull
-# SSH_CONNECTION=client_addr client_port server_port
+# NOTE when disabling: check for both a short & long version of the option!
 
-my $command = $ENV{SSH_ORIGINAL_COMMAND};
-die "$0: Not invoked via sshd\n$Usage"	unless defined $command;
-die "$0: SSH_ORIGINAL_COMMAND='$command' is not rsync\n" unless $command =~ s/^rsync\s+//;
-die "$0: --server option is not first\n" unless $command =~ /^--server\s/;
-our $am_sender = $command =~ /^--server\s+--sender\s/; # Restrictive on purpose!
-die "$0 sending to read-only server not allowed\n" if $only eq 'r' && !$am_sender;
-die "$0 reading from write-only server not allowed\n" if $only eq 'w' && $am_sender;
-
-### START of options data produced by the cull_options script. ###
-
-# These options are the only options that rsync might send to the server,
-# and only in the option format that the stock rsync produces.
+### START of options data produced by the cull-options script. ###
 
 # To disable a short-named option, add its letter to this string:
-our $short_disabled = 's';
+short_disabled = 's'
 
-our $short_no_arg = 'ACDEHIJKLORSUWXbcdgklmnopqrstuvxyz'; # DO NOT REMOVE ANY
-our $short_with_num = '@B'; # DO NOT REMOVE ANY
+# These are also disabled when the restricted dir is not "/":
+short_disabled_subdir = 'KLk'
+
+# These are all possible short options that we will accept (when not disabled above):
+short_no_arg = 'ACDEHIJKLNORSUWXbcdgklmnopqrstuvxyz' # DO NOT REMOVE ANY
+short_with_num = '@B' # DO NOT REMOVE ANY
 
 # To disable a long-named option, change its value to a -1.  The values mean:
 # 0 = the option has no arg; 1 = the arg doesn't need any checking; 2 = only
 # check the arg when receiving; and 3 = always check the arg.
-our %long_opt = (
-  'append' => 0,
-  'backup-dir' => 2,
-  'block-size' => 1,
-  'bwlimit' => 1,
-  'checksum-choice' => 1,
-  'checksum-seed' => 1,
-  'compare-dest' => 2,
-  'compress-choice' => 1,
-  'compress-level' => 1,
-  'copy-dest' => 2,
-  'copy-unsafe-links' => 0,
-  'daemon' => -1,
-  'debug' => 1,
-  'delay-updates' => 0,
-  'delete' => 0,
-  'delete-after' => 0,
-  'delete-before' => 0,
-  'delete-delay' => 0,
-  'delete-during' => 0,
-  'delete-excluded' => 0,
-  'delete-missing-args' => 0,
-  'existing' => 0,
-  'fake-super' => 0,
-  'files-from' => 3,
-  'force' => 0,
-  'from0' => 0,
-  'fuzzy' => 0,
-  'group' => 0,
-  'groupmap' => 1,
-  'hard-links' => 0,
-  'iconv' => 1,
-  'ignore-errors' => 0,
-  'ignore-existing' => 0,
-  'ignore-missing-args' => 0,
-  'ignore-times' => 0,
-  'info' => 1,
-  'inplace' => 0,
-  'link-dest' => 2,
-  'links' => 0,
-  'list-only' => 0,
-  'log-file' => $only eq 'r' ? -1 : 3,
-  'log-format' => 1,
-  'max-alloc' => 1,
-  'max-delete' => 1,
-  'max-size' => 1,
-  'min-size' => 1,
-  'modify-window' => 1,
-  'new-compress' => 0,
-  'no-implied-dirs' => 0,
-  'no-r' => 0,
-  'no-relative' => 0,
-  'no-specials' => 0,
-  'numeric-ids' => 0,
-  'old-compress' => 0,
-  'one-file-system' => 0,
-  'only-write-batch' => 1,
-  'open-noatime' => 0,
-  'owner' => 0,
-  'partial' => 0,
-  'partial-dir' => 2,
-  'perms' => 0,
-  'preallocate' => 0,
-  'recursive' => 0,
-  'remove-sent-files' => $only eq 'r' ? -1 : 0,
-  'remove-source-files' => $only eq 'r' ? -1 : 0,
-  'safe-links' => 0,
-  'sender' => $only eq 'w' ? -1 : 0,
-  'server' => 0,
-  'size-only' => 0,
-  'skip-compress' => 1,
-  'specials' => 0,
-  'stats' => 0,
-  'suffix' => 1,
-  'super' => 0,
-  'temp-dir' => 2,
-  'timeout' => 1,
-  'times' => 0,
-  'use-qsort' => 0,
-  'usermap' => 1,
-  'write-devices' => -1,
-);
-
-### END of options data produced by the cull_options script. ###
-
-if ($short_disabled ne '') {
-    $short_no_arg =~ s/[$short_disabled]//go;
-    $short_with_num =~ s/[$short_disabled]//go;
-}
-$short_no_arg = "[$short_no_arg]" if length($short_no_arg) > 1;
-$short_with_num = "[$short_with_num]" if length($short_with_num) > 1;
-
-my $write_log = -f LOGFILE && open(LOG, '>>', LOGFILE);
-
-chdir($subdir) or die "$0: Unable to chdir to restricted dir: $!\n";
-
-my(@opts, @args);
-my $in_options = 1;
-my $last_opt = '';
-my $check_type;
-while ($command =~ /((?:[^\s\\]+|\\.[^\s\\]*)+)/g) {
-  $_ = $1;
-  if ($check_type) {
-    push(@opts, check_arg($last_opt, $_, $check_type));
-    $check_type = 0;
-  } elsif ($in_options) {
-    push(@opts, $_);
-    if ($_ eq '.') {
-      $in_options = 0;
-    } else {
-      die "$0: invalid option: '-'\n" if $_ eq '-';
-      next if /^-$short_no_arg*(e\d*\.\w*)?$/o || /^-$short_with_num\d+$/o;
-
-      my($opt,$arg) = /^--([^=]+)(?:=(.*))?$/;
-      my $disabled;
-      if (defined $opt) {
-	my $ct = $long_opt{$opt};
-	last unless defined $ct;
-	next if $ct == 0;
-	if ($ct > 0) {
-	  if (!defined $arg) {
-	    $check_type = $ct;
-	    $last_opt = $opt;
-	    next;
-	  }
-	  $arg = check_arg($opt, $arg, $ct);
-	  $opts[-1] =~ s/=.*/=$arg/;
-	  next;
-	}
-	$disabled = 1;
-	$opt = "--$opt";
-      } elsif ($short_disabled ne '') {
-	$disabled = /^-$short_no_arg*([$short_disabled])/o;
-	$opt = "-$1";
-      }
-
-      last unless $disabled; # Generate generic failure
-      die "$0: option $opt has been disabled on this server.\n";
-    }
-  } else {
-    if ($subdir ne '/') {
-      # Validate args to ensure they don't try to leave our restricted dir.
-      s{//+}{/}g;
-      s{^/}{};
-      s{^$}{.};
-    }
-    push(@args, bsd_glob($_, GLOB_LIMIT|GLOB_NOCHECK|GLOB_BRACE|GLOB_QUOTE));
-  }
-}
-die "$0: invalid rsync-command syntax or options\n" if $in_options;
-
-if ($subdir ne '/') {
-    die "$0: do not use .. in any path!\n" if grep m{(^|/)\.\.(/|$)}, @args;
+long_opts = {
+  'append': 0,
+  'backup-dir': 2,
+  'block-size': 1,
+  'bwlimit': 1,
+  'checksum-choice': 1,
+  'checksum-seed': 1,
+  'compare-dest': 2,
+  'compress-choice': 1,
+  'compress-level': 1,
+  'copy-dest': 2,
+  'copy-unsafe-links': 0,
+  'daemon': -1,
+  'debug': 1,
+  'delay-updates': 0,
+  'delete': 0,
+  'delete-after': 0,
+  'delete-before': 0,
+  'delete-delay': 0,
+  'delete-during': 0,
+  'delete-excluded': 0,
+  'delete-missing-args': 0,
+  'existing': 0,
+  'fake-super': 0,
+  'files-from': 3,
+  'force': 0,
+  'from0': 0,
+  'fsync': 0,
+  'fuzzy': 0,
+  'group': 0,
+  'groupmap': 1,
+  'hard-links': 0,
+  'iconv': 1,
+  'ignore-errors': 0,
+  'ignore-existing': 0,
+  'ignore-missing-args': 0,
+  'ignore-times': 0,
+  'info': 1,
+  'inplace': 0,
+  'link-dest': 2,
+  'links': 0,
+  'list-only': 0,
+  'log-file': 3,
+  'log-format': 1,
+  'max-alloc': 1,
+  'max-delete': 1,
+  'max-size': 1,
+  'min-size': 1,
+  'mkpath': 0,
+  'modify-window': 1,
+  'msgs2stderr': 0,
+  'munge-links': 0,
+  'new-compress': 0,
+  'no-W': 0,
+  'no-implied-dirs': 0,
+  'no-msgs2stderr': 0,
+  'no-munge-links': -1,
+  'no-r': 0,
+  'no-relative': 0,
+  'no-specials': 0,
+  'numeric-ids': 0,
+  'old-compress': 0,
+  'one-file-system': 0,
+  'only-write-batch': 1,
+  'open-noatime': 0,
+  'owner': 0,
+  'partial': 0,
+  'partial-dir': 2,
+  'perms': 0,
+  'preallocate': 0,
+  'recursive': 0,
+  'remove-sent-files': 0,
+  'remove-source-files': 0,
+  'safe-links': 0,
+  'sender': 0,
+  'server': 0,
+  'size-only': 0,
+  'skip-compress': 1,
+  'specials': 0,
+  'stats': 0,
+  'stderr': 1,
+  'suffix': 1,
+  'super': 0,
+  'temp-dir': 2,
+  'timeout': 1,
+  'times': 0,
+  'use-qsort': 0,
+  'usermap': 1,
+  'write-devices': -1,
 }
 
-@args = ( '.' ) if !@args;
+### END of options data produced by the cull-options script. ###
 
-if ($write_log) {
-  my ($mm,$hh) = (localtime)[1,2];
-  my $host = $ENV{SSH_CONNECTION} || 'unknown';
-  $host =~ s/ .*//; # Keep only the client's IP addr
-  $host =~ s/^::ffff://;
-  $host = gethostbyaddr(inet_aton($host),AF_INET) || $host;
-  printf LOG "%02d:%02d %-13s [%s]\n", $hh, $mm, $host, "@opts @args";
-  close LOG;
-}
+import os, sys, re, argparse, glob, socket, time, subprocess
+from argparse import RawTextHelpFormatter
 
-# Note: This assumes that the rsync protocol will not be maliciously hijacked.
-exec(RSYNC, @opts, '--', @args) or die "exec(rsync @opts -- @args) failed: $? $!";
+try:
+    from braceexpand import braceexpand
+except:
+    braceexpand = lambda x: [ DE_BACKSLASH_RE.sub(r'\1', x) ]
 
-sub check_arg
-{
-  my($opt, $arg, $type) = @_;
-  $arg =~ s/\\(.)/$1/g;
-  if ($subdir ne '/' && ($type == 3 || ($type == 2 && !$am_sender))) {
-    $arg =~ s{//}{/}g;
-    die "Do not use .. in --$opt; anchor the path at the root of your restricted dir.\n"
-      if $arg =~ m{(^|/)\.\.(/|$)};
-    $arg =~ s{^/}{$subdir/};
-  }
-  $arg;
-}
+HAS_DOT_DOT_RE = re.compile(r'(^|/)\.\.(/|$)')
+LONG_OPT_RE = re.compile(r'^--([^=]+)(?:=(.*))?$')
+DE_BACKSLASH_RE = re.compile(r'\\(.)')
+
+def main():
+    if not os.path.isdir(args.dir):
+        die("Restricted directory does not exist!")
+
+    # The format of the environment variables set by sshd:
+    #   SSH_ORIGINAL_COMMAND:
+    #     rsync --server          -vlogDtpre.iLsfxCIvu --etc . ARG  # push
+    #     rsync --server --sender -vlogDtpre.iLsfxCIvu --etc . ARGS # pull
+    #   SSH_CONNECTION (client_ip client_port server_ip server_port):
+    #     192.168.1.100 64106 192.168.1.2 22
+
+    command = os.environ.get('SSH_ORIGINAL_COMMAND', None)
+    if not command:
+        die("Not invoked via sshd")
+    command = command.split(' ', 2)
+    if command[0:1] != ['rsync']:
+        die("SSH_ORIGINAL_COMMAND does not run rsync")
+    if command[1:2] != ['--server']:
+        die("--server option is not the first arg")
+    command = '' if len(command) < 3 else command[2]
+
+    global am_sender
+    am_sender = command.startswith("--sender ") # Restrictive on purpose!
+    if args.ro and not am_sender:
+        die("sending to read-only server is not allowed")
+    if args.wo and am_sender:
+        die("reading from write-only server is not allowed")
+
+    if args.wo or not am_sender:
+        long_opts['sender'] = -1
+    if args.no_del:
+        for opt in long_opts:
+            if opt.startswith(('remove', 'delete')):
+                long_opts[opt] = -1
+    if args.ro:
+        long_opts['log-file'] = -1
+
+    if args.dir != '/':
+        global short_disabled
+        short_disabled += short_disabled_subdir
+
+    short_no_arg_re = short_no_arg
+    short_with_num_re = short_with_num
+    if short_disabled:
+        for ltr in short_disabled:
+            short_no_arg_re = short_no_arg_re.replace(ltr, '')
+            short_with_num_re = short_with_num_re.replace(ltr, '')
+        short_disabled_re = re.compile(r'^-[%s]*([%s])' % (short_no_arg_re, short_disabled))
+    short_no_arg_re = re.compile(r'^-(?=.)[%s]*(e\d*\.\w*)?$' % short_no_arg_re)
+    short_with_num_re = re.compile(r'^-[%s]\d+$' % short_with_num_re)
+
+    log_fh = open(LOGFILE, 'a') if os.path.isfile(LOGFILE) else None
+
+    try:
+        os.chdir(args.dir)
+    except OSError as e:
+        die('unable to chdir to restricted dir:', str(e))
+
+    rsync_opts = [ '--server' ]
+    rsync_args = [ ]
+    saw_the_dot_arg = False
+    last_opt = check_type = None
+
+    for arg in re.findall(r'(?:[^\s\\]+|\\.[^\s\\]*)+', command):
+        if check_type:
+            rsync_opts.append(validated_arg(last_opt, arg, check_type))
+            check_type = None
+        elif saw_the_dot_arg:
+            # NOTE: an arg that starts with a '-' is safe due to our use of "--" in the cmd tuple.
+            try:
+                b_e = braceexpand(arg) # Also removes backslashes
+            except: # Handle errors such as unbalanced braces by just de-backslashing the arg:
+                b_e = [ DE_BACKSLASH_RE.sub(r'\1', arg) ]
+            for xarg in b_e:
+                rsync_args += validated_arg('arg', xarg, wild=True)
+        else: # parsing the option args
+            if arg == '.':
+                saw_the_dot_arg = True
+                continue
+            rsync_opts.append(arg)
+            if short_no_arg_re.match(arg) or short_with_num_re.match(arg):
+                continue
+            disabled = False
+            m = LONG_OPT_RE.match(arg)
+            if m:
+                opt = m.group(1)
+                opt_arg = m.group(2)
+                ct = long_opts.get(opt, None)
+                if ct is None:
+                    break # Generate generic failure due to unfinished arg parsing
+                if ct == 0:
+                    continue
+                opt = '--' + opt
+                if ct > 0:
+                    if opt_arg is not None:
+                        rsync_opts[-1] = opt + '=' + validated_arg(opt, opt_arg, ct)
+                    else:
+                        check_type = ct
+                        last_opt = opt
+                    continue
+                disabled = True
+            elif short_disabled:
+                m = short_disabled_re.match(arg)
+                if m:
+                    disabled = True
+                    opt = '-' + m.group(1)
+
+            if disabled:
+                die("option", opt, "has been disabled on this server.")
+            break # Generate a generic failure
+
+    if not saw_the_dot_arg:
+        die("invalid rsync-command syntax or options")
+
+    if args.munge:
+        rsync_opts.append('--munge-links')
+
+    if not rsync_args:
+        rsync_args = [ '.' ]
+
+    cmd = (RSYNC, *rsync_opts, '--', '.', *rsync_args)
+
+    if log_fh:
+        now = time.localtime()
+        host = os.environ.get('SSH_CONNECTION', 'unknown').split()[0] # Drop everything after the IP addr
+        if host.startswith('::ffff:'):
+            host = host[7:]
+        try:
+            host = socket.gethostbyaddr(socket.inet_aton(host))
+        except:
+            pass
+        log_fh.write("%02d:%02d:%02d %-16s %s\n" % (now.tm_hour, now.tm_min, now.tm_sec, host, str(cmd)))
+        log_fh.close()
+
+    # NOTE: This assumes that the rsync protocol will not be maliciously hijacked.
+    if args.no_lock:
+        os.execlp(RSYNC, *cmd)
+        die("execlp(", RSYNC, *cmd, ')  failed')
+    child = subprocess.run(cmd)
+    if child.returncode != 0:
+        sys.exit(child.returncode)
+
+
+def validated_arg(opt, arg, typ=3, wild=False):
+    if opt != 'arg': # arg values already have their backslashes removed.
+        arg = DE_BACKSLASH_RE.sub(r'\1', arg)
+
+    orig_arg = arg
+    if arg.startswith('./'):
+        arg = arg[1:]
+    arg = arg.replace('//', '/')
+    if args.dir != '/':
+        if HAS_DOT_DOT_RE.search(arg):
+            die("do not use .. in", opt, "(anchor the path at the root of your restricted dir)")
+        if arg.startswith('/'):
+            arg = args.dir + arg
+
+    if wild:
+        got = glob.glob(arg)
+        if not got:
+            got = [ arg ]
+    else:
+        got = [ arg ]
+
+    ret = [ ]
+    for arg in got:
+        if args.dir != '/' and arg != '.' and (typ == 3 or (typ == 2 and not am_sender)):
+            arg_has_trailing_slash = arg.endswith('/')
+            if arg_has_trailing_slash:
+                arg = arg[:-1]
+            else:
+                arg_has_trailing_slash_dot = arg.endswith('/.')
+                if arg_has_trailing_slash_dot:
+                    arg = arg[:-2]
+            real_arg = os.path.realpath(arg)
+            if arg != real_arg and not real_arg.startswith(args.dir_slash):
+                die('unsafe arg:', orig_arg, [arg, real_arg])
+            if arg_has_trailing_slash:
+                arg += '/'
+            elif arg_has_trailing_slash_dot:
+                arg += '/.'
+            if opt == 'arg' and arg.startswith(args.dir_slash):
+                arg = arg[args.dir_slash_len:]
+                if arg == '':
+                    arg = '.'
+        ret.append(arg)
+
+    return ret if wild else ret[0]
+
+
+def lock_or_die(dirname):
+    import fcntl
+    global lock_handle
+    lock_handle = os.open(dirname, os.O_RDONLY)
+    try:
+        fcntl.flock(lock_handle, fcntl.LOCK_EX | fcntl.LOCK_NB)
+    except:
+        die('Another instance of rrsync is already accessing this directory.')
+
+
+def die(*msg):
+    print(sys.argv[0], 'error:', *msg, file=sys.stderr)
+    if sys.stdin.isatty():
+        arg_parser.print_help(sys.stderr)
+    sys.exit(1)
+
+
+# This class displays the --help to the user on argparse error IFF they're running it interactively.
+class OurArgParser(argparse.ArgumentParser):
+    def error(self, msg):
+        die(msg)
+
+
+if __name__ == '__main__':
+    our_desc = """Use "man rrsync" to learn how to restrict ssh users to using a restricted rsync command."""
+    arg_parser = OurArgParser(description=our_desc, add_help=False)
+    only_group = arg_parser.add_mutually_exclusive_group()
+    only_group.add_argument('-ro', action='store_true', help="Allow only reading from the DIR. Implies -no-del and -no-lock.")
+    only_group.add_argument('-wo', action='store_true', help="Allow only writing to the DIR.")
+    arg_parser.add_argument('-munge', action='store_true', help="Enable rsync's --munge-links on the server side.")
+    arg_parser.add_argument('-no-del', action='store_true', help="Disable rsync's --delete* and --remove* options.")
+    arg_parser.add_argument('-no-lock', action='store_true', help="Avoid the single-run (per-user) lock check.")
+    arg_parser.add_argument('-help', '-h', action='help', help="Output this help message and exit.")
+    arg_parser.add_argument('dir', metavar='DIR', help="The restricted directory to use.")
+    args = arg_parser.parse_args()
+    args.dir = os.path.realpath(args.dir)
+    args.dir_slash = args.dir + '/'
+    args.dir_slash_len = len(args.dir_slash)
+    if args.ro:
+        args.no_del = True
+    elif not args.no_lock:
+        lock_or_die(args.dir)
+    main()
+
+# vim: sw=4 et

support/rrsync.1.md

@@ -0,0 +1,142 @@
+## NAME
+
+rrsync - a script to setup restricted rsync users via ssh logins
+
+## SYNOPSIS
+
+```
+rrsync [-ro|-rw] [-munge] [-no-del] [-no-lock] DIR
+```
+
+The single non-option argument specifies the restricted _DIR_ to use. It can be
+relative to the user's home directory or an absolute path.
+
+The online version of this manpage (that includes cross-linking of topics)
+is available at <https://download.samba.org/pub/rsync/rrsync.1>.
+
+## DESCRIPTION
+
+A user's ssh login can be restricted to only allow the running of an rsync
+transfer in one of two easy ways:
+
+* forcing the running of the rrsync script
+* forcing the running of an rsync daemon-over-ssh command.
+
+To use the rrsync script, edit the user's `~/.ssh/authorized_keys` file and add
+a prefix like one of the following (followed by a space) in front of each
+ssh-key line that should be restricted:
+
+> ```
+> command="rrsync DIR"
+> command="rrsync -ro DIR"
+> command="rrsync -munge -no-del DIR"
+> ```
+
+Then, ensure that the rrsync script has your desired option restrictions. You
+may want to copy the script to a local bin dir with a unique name if you want
+to have multiple configurations. One or more rrsync options can be specified
+prior to the _DIR_ if you want to further restrict the transfer.
+
+To use an rsync daemon setup, edit the user's `~/.ssh/authorized_keys` file and
+add a prefix like one of the following (followed by a space) in front of each
+ssh-key line that should be restricted:
+
+> ```
+> command="rsync --server --daemon ."
+> command="rsync --server --daemon --config=/PATH/TO/rsyncd.conf ."
+> ```
+
+Then, ensure that the rsyncd.conf file is created with one or more module names
+with the appropriate path and option restrictions.  If rsync's
+[`--config`](rsync.1#dopt) option is omitted, it defaults to `~/rsyncd.conf`.
+See the [**rsyncd.conf**(5)](rsyncd.conf.5) manpage for details of how to
+configure an rsync daemon.
+
+When using rrsync, there can be just one restricted dir per authorized key.  A
+daemon setup, on the other hand, allows multiple module names inside the config
+file, each one with its own path setting.
+
+The remainder of this manpage is dedicated to using the rrsync script.
+
+## OPTIONS
+
+0.  `-ro`
+
+    Allow only reading from the DIR. Implies [`-no-del`](#opt) and
+    [`-no-lock`](#opt).
+
+0.  `-wo`
+
+    Allow only writing to the DIR.
+
+0.  `-munge`
+
+    Enable rsync's [`--munge-links`](rsync.1#opt) on the server side.
+
+0.  `-no-del`
+
+    Disable rsync's `--delete*` and `--remove*` options.
+
+0.  `-no-lock`
+
+    Avoid the single-run (per-user) lock check.  Useful with [`-munge`](#opt).
+
+0.  `-help`, `-h`
+
+    Output this help message and exit.
+
+## SECURITY RESTRICTIONS
+
+The rrsync script validates the path arguments it is sent to try to restrict
+them to staying within the specified DIR.
+
+The rrsync script rejects rsync's [`--copy-links`](rsync.1#opt) option (by
+default) so that a copy cannot dereference a symlink within the DIR to get to a
+file outside the DIR.
+
+The rrsync script rejects rsync's [`--protect-args`](rsync.1#opt) (`-s`) option
+because it would allow options to be sent to the server-side that the script
+cannot check.  If you want to support `--protect-args`, use a daemon-over-ssh
+setup.
+
+The rrsync script accepts just a subset of rsync's options that the real rsync
+uses when running the server command.  A few extra convenience options are also
+included to help it to interact with BackupPC and accept some convenient user
+overrides.
+
+The script (or a copy of it) can be manually edited if you want it to customize
+the option handling.
+
+## EXAMPLES
+
+The `~/.ssh/authorized_keys` file might have lines in it like this:
+
+> ```
+> command="rrsync client/logs" ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAIEAzG...
+> command="rrsync -ro results" ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAIEAmk...
+> ```
+
+## FILES
+
+~/.ssh/authorized_keys
+
+## SEE ALSO
+
+[**rsync**(1)](rsync.1), [**rsyncd.conf**(5)](rsyncd.conf.5)
+
+## VERSION
+
+This manpage is current for version @VERSION@ of rsync.
+
+## CREDITS
+
+rsync is distributed under the GNU General Public License.  See the file
+[COPYING](COPYING) for details.
+
+An rsync web site is available at <https://rsync.samba.org/> and its github
+project is <https://github.com/WayneD/rsync>.
+
+## AUTHOR
+
+The original rrsync perl script was written by Joe Smith.  Many people have
+later contributed to it.  The python version was created by Wayne Davison.

support/rsync-no-vanished

@@ -1,12 +1,21 @@
 #!/usr/bin/env bash
 
+REAL_RSYNC=/usr/bin/rsync
 IGNOREEXIT=24
 IGNOREOUT='^(file has vanished: |rsync warning: some files vanished before they could be transferred)'
 
+# If someone installs this as "rsync", make sure we don't affect a server run.
+for arg in "${@}"; do
+    if [[ "$arg" == --server ]]; then
+	exec $REAL_RSYNC "${@}"
+	exit $? # Not reached
+    fi
+done
+
 set -o pipefail
 
-rsync "${@}" 2>&1 | (egrep -v "$IGNOREOUT" || true)
-ret=$?
+# This filters stderr without merging it with stdout:
+{ $REAL_RSYNC "${@}" 2>&1 1>&3 3>&- | grep -E -v "$IGNOREOUT"; ret=${PIPESTATUS[0]}; } 3>&1 1>&2
 
 if [[ $ret == $IGNOREEXIT ]]; then
     ret=0

support/rsync-slash-strip

@@ -6,12 +6,19 @@
 #
 # To use this, name it something like "rs", put it somewhere in your path, and
 # then use "rs" in place of "rsync" when you are typing your copy commands.
+
+REAL_RSYNC=/usr/bin/rsync
+
 args=()
 for arg in "${@}"; do
+    if [[ "$arg" == --server ]]; then
+	exec $REAL_RSYNC "${@}"
+	exit $? # Not reached
+    fi
     if [[ "$arg" == / ]]; then
 	args=("${args[@]}" /)
     else
 	args=("${args[@]}" "${arg%/}")
     fi
 done
-exec /usr/bin/rsync "${args[@]}"
+exec $REAL_RSYNC "${args[@]}"

syscall.c

@@ -4,7 +4,7 @@
  *
  * Copyright (C) 1998 Andrew Tridgell
  * Copyright (C) 2002 Martin Pool
- * Copyright (C) 2003-2020 Wayne Davison
+ * Copyright (C) 2003-2022 Wayne Davison
  *
  * This program is free software; you can redistribute it and/or modify
  * it under the terms of the GNU General Public License as published by
@@ -54,6 +54,19 @@ extern int open_noatime;
 # endif
 #endif
 
+#ifdef SUPPORT_CRTIMES
+#ifdef HAVE_GETATTRLIST
+#pragma pack(push, 4)
+struct create_time {
+	uint32 length;
+	struct timespec crtime;
+};
+#pragma pack(pop)
+#elif defined __CYGWIN__
+#include <windows.h>
+#endif
+#endif
+
 #define RETURN_ERROR_IF(x,e) \
 	do { \
 		if (x) { \
@@ -64,15 +77,15 @@ extern int open_noatime;
 
 #define RETURN_ERROR_IF_RO_OR_LO RETURN_ERROR_IF(read_only || list_only, EROFS)
 
-int do_unlink(const char *fname)
+int do_unlink(const char *path)
 {
 	if (dry_run) return 0;
 	RETURN_ERROR_IF_RO_OR_LO;
-	return unlink(fname);
+	return unlink(path);
 }
 
 #ifdef SUPPORT_LINKS
-int do_symlink(const char *lnk, const char *fname)
+int do_symlink(const char *lnk, const char *path)
 {
 	if (dry_run) return 0;
 	RETURN_ERROR_IF_RO_OR_LO;
@@ -82,7 +95,7 @@ int do_symlink(const char *lnk, const char *fname)
 	 * and write the lnk into it. */
 	if (am_root < 0) {
 		int ok, len = strlen(lnk);
-		int fd = open(fname, O_WRONLY|O_CREAT|O_TRUNC, S_IWUSR|S_IRUSR);
+		int fd = open(path, O_WRONLY|O_CREAT|O_TRUNC, S_IWUSR|S_IRUSR);
 		if (fd < 0)
 			return -1;
 		ok = write(fd, lnk, len) == len;
@@ -92,7 +105,7 @@ int do_symlink(const char *lnk, const char *fname)
 	}
 #endif
 
-	return symlink(lnk, fname);
+	return symlink(lnk, path);
 }
 
 #if defined NO_SYMLINK_XATTRS || defined NO_SYMLINK_USER_XATTRS
@@ -120,12 +133,16 @@ ssize_t do_readlink(const char *path, char *buf, size_t bufsiz)
 #endif
 #endif
 
-#ifdef HAVE_LINK
+#if defined HAVE_LINK || defined HAVE_LINKAT
 int do_link(const char *old_path, const char *new_path)
 {
 	if (dry_run) return 0;
 	RETURN_ERROR_IF_RO_OR_LO;
+#ifdef HAVE_LINKAT
+	return linkat(AT_FDCWD, old_path, AT_FDCWD, new_path, 0);
+#else
 	return link(old_path, new_path);
+#endif
 }
 #endif
 
@@ -214,27 +231,43 @@ int do_open(const char *pathname, int flags, mode_t mode)
 #ifdef HAVE_CHMOD
 int do_chmod(const char *path, mode_t mode)
 {
+	static int switch_step = 0;
 	int code;
+
 	if (dry_run) return 0;
 	RETURN_ERROR_IF_RO_OR_LO;
-#ifdef HAVE_LCHMOD
-	code = lchmod(path, mode & CHMOD_BITS);
-#else
-	if (S_ISLNK(mode)) {
-# if defined HAVE_SETATTRLIST
-		struct attrlist attrList;
-		uint32_t m = mode & CHMOD_BITS; /* manpage is wrong: not mode_t! */
 
-		memset(&attrList, 0, sizeof attrList);
-		attrList.bitmapcount = ATTR_BIT_MAP_COUNT;
-		attrList.commonattr = ATTR_CMN_ACCESSMASK;
-		code = setattrlist(path, &attrList, &m, sizeof m, FSOPT_NOFOLLOW);
+	switch (switch_step) {
+#ifdef HAVE_LCHMOD
+	case 0:
+		if ((code = lchmod(path, mode & CHMOD_BITS)) == 0)
+			break;
+		if (errno == ENOSYS)
+			switch_step++;
+		else if (errno != ENOTSUP)
+			break;
+#endif
+		/* FALLTHROUGH */
+	default:
+		if (S_ISLNK(mode)) {
+# if defined HAVE_SETATTRLIST
+			struct attrlist attrList;
+			uint32_t m = mode & CHMOD_BITS; /* manpage is wrong: not mode_t! */
+
+			memset(&attrList, 0, sizeof attrList);
+			attrList.bitmapcount = ATTR_BIT_MAP_COUNT;
+			attrList.commonattr = ATTR_CMN_ACCESSMASK;
+			if ((code = setattrlist(path, &attrList, &m, sizeof m, FSOPT_NOFOLLOW)) == 0)
+				break;
+			if (errno == ENOTSUP)
+				code = 1;
 # else
-		code = 1;
+			code = 1;
 # endif
-	} else
-		code = chmod(path, mode & CHMOD_BITS); /* DISCOURAGED FUNCTION */
-#endif /* !HAVE_LCHMOD */
+		} else
+			code = chmod(path, mode & CHMOD_BITS); /* DISCOURAGED FUNCTION */
+		break;
+	}
 	if (code != 0 && (preserve_perms || preserve_executability))
 		return code;
 	return 0;
@@ -282,12 +315,12 @@ void trim_trailing_slashes(char *name)
 	}
 }
 
-int do_mkdir(char *fname, mode_t mode)
+int do_mkdir(char *path, mode_t mode)
 {
 	if (dry_run) return 0;
 	RETURN_ERROR_IF_RO_OR_LO;
-	trim_trailing_slashes(fname);
-	return mkdir(fname, mode);
+	trim_trailing_slashes(path);
+	return mkdir(path, mode);
 }
 
 /* like mkstemp but forces permissions */
@@ -321,25 +354,25 @@ int do_mkstemp(char *template, mode_t perms)
 #endif
 }
 
-int do_stat(const char *fname, STRUCT_STAT *st)
+int do_stat(const char *path, STRUCT_STAT *st)
 {
 #ifdef USE_STAT64_FUNCS
-	return stat64(fname, st);
+	return stat64(path, st);
 #else
-	return stat(fname, st);
+	return stat(path, st);
 #endif
 }
 
-int do_lstat(const char *fname, STRUCT_STAT *st)
+int do_lstat(const char *path, STRUCT_STAT *st)
 {
 #ifdef SUPPORT_LINKS
 # ifdef USE_STAT64_FUNCS
-	return lstat64(fname, st);
+	return lstat64(path, st);
 # else
-	return lstat(fname, st);
+	return lstat(path, st);
 # endif
 #else
-	return do_stat(fname, st);
+	return do_stat(path, st);
 #endif
 }
 
@@ -367,7 +400,29 @@ OFF_T do_lseek(int fd, OFF_T offset, int whence)
 }
 
 #ifdef HAVE_SETATTRLIST
-int do_setattrlist_times(const char *fname, STRUCT_STAT *stp)
+int do_setattrlist_times(const char *path, STRUCT_STAT *stp)
+{
+	struct attrlist attrList;
+	struct timespec ts[2];
+
+	if (dry_run) return 0;
+	RETURN_ERROR_IF_RO_OR_LO;
+
+	/* Yes, this is in the opposite order of utime and similar. */
+	ts[0].tv_sec = stp->st_mtime;
+	ts[0].tv_nsec = stp->ST_MTIME_NSEC;
+
+	ts[1].tv_sec = stp->st_atime;
+	ts[1].tv_nsec = stp->ST_ATIME_NSEC;
+
+	memset(&attrList, 0, sizeof attrList);
+	attrList.bitmapcount = ATTR_BIT_MAP_COUNT;
+	attrList.commonattr = ATTR_CMN_MODTIME | ATTR_CMN_ACCTIME;
+	return setattrlist(path, &attrList, ts, sizeof ts, FSOPT_NOFOLLOW);
+}
+
+#ifdef SUPPORT_CRTIMES
+int do_setattrlist_crtime(const char *path, time_t crtime)
 {
 	struct attrlist attrList;
 	struct timespec ts;
@@ -375,18 +430,70 @@ int do_setattrlist_times(const char *fname, STRUCT_STAT *stp)
 	if (dry_run) return 0;
 	RETURN_ERROR_IF_RO_OR_LO;
 
-	ts.tv_sec = stp->st_mtime;
-	ts.tv_nsec = stp->ST_MTIME_NSEC;
+	ts.tv_sec = crtime;
+	ts.tv_nsec = 0;
 
 	memset(&attrList, 0, sizeof attrList);
 	attrList.bitmapcount = ATTR_BIT_MAP_COUNT;
-	attrList.commonattr = ATTR_CMN_MODTIME;
-	return setattrlist(fname, &attrList, &ts, sizeof ts, FSOPT_NOFOLLOW);
+	attrList.commonattr = ATTR_CMN_CRTIME;
+	return setattrlist(path, &attrList, &ts, sizeof ts, FSOPT_NOFOLLOW);
 }
 #endif
+#endif /* HAVE_SETATTRLIST */
+
+#ifdef SUPPORT_CRTIMES
+time_t get_create_time(const char *path, STRUCT_STAT *stp)
+{
+#ifdef HAVE_GETATTRLIST
+	static struct create_time attrBuf;
+	struct attrlist attrList;
+
+	(void)stp;
+	memset(&attrList, 0, sizeof attrList);
+	attrList.bitmapcount = ATTR_BIT_MAP_COUNT;
+	attrList.commonattr = ATTR_CMN_CRTIME;
+	if (getattrlist(path, &attrList, &attrBuf, sizeof attrBuf, FSOPT_NOFOLLOW) < 0)
+		return 0;
+	return attrBuf.crtime.tv_sec;
+#elif defined __CYGWIN__
+	(void)path;
+	return stp->st_birthtime;
+#else
+#error Unknown crtimes implementation
+#endif
+}
+
+#if defined __CYGWIN__
+int do_SetFileTime(const char *path, time_t crtime)
+{
+	if (dry_run) return 0;
+	RETURN_ERROR_IF_RO_OR_LO;
+
+	int cnt = MultiByteToWideChar(CP_UTF8, 0, path, -1, NULL, 0);
+	if (cnt == 0)
+	    return -1;
+	WCHAR *pathw = new_array(WCHAR, cnt);
+	if (!pathw)
+	    return -1;
+	MultiByteToWideChar(CP_UTF8, 0, path, -1, pathw, cnt);
+	HANDLE handle = CreateFileW(pathw, FILE_WRITE_ATTRIBUTES, FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE,
+				    NULL, OPEN_EXISTING, FILE_FLAG_BACKUP_SEMANTICS, NULL);
+	free(pathw);
+	if (handle == INVALID_HANDLE_VALUE)
+	    return -1;
+	int64 temp_time = Int32x32To64(crtime, 10000000) + 116444736000000000LL;
+	FILETIME birth_time;
+	birth_time.dwLowDateTime = (DWORD)temp_time;
+	birth_time.dwHighDateTime = (DWORD)(temp_time >> 32);
+	int ok = SetFileTime(handle, &birth_time, NULL, NULL);
+	CloseHandle(handle);
+	return ok ? 0 : -1;
+}
+#endif
+#endif /* SUPPORT_CRTIMES */
 
 #ifdef HAVE_UTIMENSAT
-int do_utimensat(const char *fname, STRUCT_STAT *stp)
+int do_utimensat(const char *path, STRUCT_STAT *stp)
 {
 	struct timespec t[2];
 
@@ -405,12 +512,12 @@ int do_utimensat(const char *fname, STRUCT_STAT *stp)
 #else
 	t[1].tv_nsec = 0;
 #endif
-	return utimensat(AT_FDCWD, fname, t, AT_SYMLINK_NOFOLLOW);
+	return utimensat(AT_FDCWD, path, t, AT_SYMLINK_NOFOLLOW);
 }
 #endif
 
 #ifdef HAVE_LUTIMES
-int do_lutimes(const char *fname, STRUCT_STAT *stp)
+int do_lutimes(const char *path, STRUCT_STAT *stp)
 {
 	struct timeval t[2];
 
@@ -429,12 +536,12 @@ int do_lutimes(const char *fname, STRUCT_STAT *stp)
 #else
 	t[1].tv_usec = 0;
 #endif
-	return lutimes(fname, t);
+	return lutimes(path, t);
 }
 #endif
 
 #ifdef HAVE_UTIMES
-int do_utimes(const char *fname, STRUCT_STAT *stp)
+int do_utimes(const char *path, STRUCT_STAT *stp)
 {
 	struct timeval t[2];
 
@@ -453,11 +560,11 @@ int do_utimes(const char *fname, STRUCT_STAT *stp)
 #else
 	t[1].tv_usec = 0;
 #endif
-	return utimes(fname, t);
+	return utimes(path, t);
 }
 
 #elif defined HAVE_UTIME
-int do_utime(const char *fname, STRUCT_STAT *stp)
+int do_utime(const char *path, STRUCT_STAT *stp)
 {
 #ifdef HAVE_STRUCT_UTIMBUF
 	struct utimbuf tbuf;
@@ -471,11 +578,11 @@ int do_utime(const char *fname, STRUCT_STAT *stp)
 # ifdef HAVE_STRUCT_UTIMBUF
 	tbuf.actime = stp->st_atime;
 	tbuf.modtime = stp->st_mtime;
-	return utime(fname, &tbuf);
+	return utime(path, &tbuf);
 # else
 	t[0] = stp->st_atime;
 	t[1] = stp->st_mtime;
-	return utime(fname, t);
+	return utime(path, t);
 # endif
 }
 
@@ -523,7 +630,7 @@ OFF_T do_fallocate(int fd, OFF_T offset, OFF_T length)
 
 /* Punch a hole at pos for len bytes. The current file position must be at pos and will be
  * changed to be at pos + len. */
-int do_punch_hole(int fd, UNUSED(OFF_T pos), OFF_T len)
+int do_punch_hole(int fd, OFF_T pos, OFF_T len)
 {
 #ifdef HAVE_FALLOCATE
 # ifdef HAVE_FALLOC_FL_PUNCH_HOLE
@@ -540,6 +647,8 @@ int do_punch_hole(int fd, UNUSED(OFF_T pos), OFF_T len)
 		return 0;
 	}
 # endif
+#else
+	(void)pos;
 #endif
 	{
 		char zeros[4096];

t_stub.c

@@ -3,7 +3,7 @@
  * functions, so that module test harnesses can run standalone.
  *
  * Copyright (C) 2001, 2002 Martin Pool <mbp@samba.org>
- * Copyright (C) 2003-2020 Wayne Davison
+ * Copyright (C) 2003-2022 Wayne Davison
  *
  * This program is free software; you can redistribute it and/or modify
  * it under the terms of the GNU General Public License as published by
@@ -21,6 +21,7 @@
 
 #include "rsync.h"
 
+int do_fsync = 0;
 int inplace = 0;
 int modify_window = 0;
 int preallocate_files = 0;
@@ -28,12 +29,13 @@ int protect_args = 0;
 int module_id = -1;
 int relative_paths = 0;
 int module_dirlen = 0;
-int preserve_times = 0;
+int preserve_mtimes = 0;
 int preserve_xattrs = 0;
 int preserve_perms = 0;
 int preserve_executability = 0;
+int omit_link_times = 0;
 int open_noatime = 0;
-size_t max_alloc = 1024*1024*1024; /* max_alloc is needed when combined with util2.o */
+size_t max_alloc = 0; /* max_alloc is needed when combined with util2.o */
 char *partial_dir;
 char *module_dir;
 filter_rule_list daemon_filter_list;

testsuite/00-hello.test

@@ -1,9 +1,11 @@
-#! /bin/sh
+#!/bin/sh
 
-test_fail() {
-    echo "$@" >&2
-    exit 1
-}
+# Test some foundational things.
+
+. "$suitedir/rsync.fns"
+
+RSYNC_RSH="$scratchdir/src/support/lsh.sh"
+export RSYNC_RSH
 
 echo $0 running
 
@@ -12,3 +14,48 @@ $RSYNC --version || test_fail '--version output failed'
 $RSYNC --info=help || test_fail '--info=help output failed'
 
 $RSYNC --debug=help || test_fail '--debug=help output failed'
+
+weird_name="A weird)name"
+
+mkdir "$fromdir"
+mkdir "$fromdir/$weird_name"
+
+append_line() {
+    echo "$1"
+    echo "$1" >>"$fromdir/$weird_name/file"
+}
+
+append_line test1
+checkit "$RSYNC -ai '$fromdir/' '$todir/'" "$fromdir" "$todir"
+
+copy_weird() {
+    checkit "$RSYNC $1 --rsync-path='$RSYNC' '$2$fromdir/$weird_name/' '$3$todir/$weird_name'" "$fromdir" "$todir"
+}
+
+append_line test2
+copy_weird '-ai' 'lh:' ''
+
+append_line test3
+copy_weird '-ai' '' 'lh:'
+
+append_line test4
+copy_weird '-ais' 'lh:' ''
+
+append_line test5
+copy_weird '-ais' '' 'lh:'
+
+echo test6
+
+touch "$fromdir/one" "$fromdir/two"
+(cd "$fromdir" && $RSYNC -ai --old-args --rsync-path="$RSYNC" lh:'one two' "$todir/")
+if [ ! -f "$todir/one" ] || [ ! -f "$todir/two" ]; then
+    test_fail "old-args copy of 'one two' failed"
+fi
+
+echo test7
+
+rm "$todir/one" "$todir/two"
+(cd "$fromdir" && RSYNC_OLD_ARGS=1 $RSYNC -ai --rsync-path="$RSYNC" lh:'one two' "$todir/")
+
+# The script would have aborted on error, so getting here means we've won.
+exit 0

testsuite/README.testsuite

@@ -26,5 +26,3 @@ you could include the log messages when reporting a failure.
 
 These tests also run automatically on the build farm, and you can see
 the results on http://build.samba.org/.
-
-

testsuite/acls-default.test

@@ -1,4 +1,4 @@
-#! /bin/sh
+#!/bin/sh
 
 # This program is distributable under the terms of the GNU GPL (see
 # COPYING).
@@ -21,7 +21,7 @@ testit() {
     todir="$scratchdir/$1"
     mkdir "$todir"
     $setfacl_nodef "$todir"
-    if [ "$2" ]; then
+    if [ -n "$2" ]; then
 	case "$setfacl_nodef" in
 	*-k*) opts="-dm $2" ;;
 	*) opts="-m `echo $2 | sed 's/\([ugom]:\)/d:\1/g'`"

testsuite/acls.test

@@ -1,4 +1,4 @@
-#! /bin/sh
+#!/bin/sh
 
 # This program is distributable under the terms of the GNU GPL (see
 # COPYING).
@@ -17,8 +17,8 @@ files='foo file1 file2'
 
 case "$setfacl_nodef" in
 true)
-    if ! chmod --help 2>&1 | fgrep +a >/dev/null; then
-	test_skipped "I don't know how to use setfacl or chmod for ACLs"
+    if ! chmod --help 2>&1 | grep -F +a >/dev/null; then
+        test_skipped "I don't know how to use setfacl or chmod for ACLs"
     fi
     chmod +a "root allow read,write,execute" "$fromdir/foo" || test_skipped "Your filesystem has ACLs disabled"
     chmod +a "root allow read,execute" "$fromdir/file1"
@@ -27,7 +27,7 @@ true)
     chmod +a "root allow read,execute" "$fromdir/file2"
 
     see_acls() {
-	ls -le "${@}"
+        ls -le "${@}"
     }
     ;;
 *)
@@ -45,7 +45,7 @@ true)
     setfacl -m u:0:5 "$fromdir/file2"
 
     see_acls() {
-	getfacl "${@}"
+        getfacl "${@}"
     }
     ;;
 esac

testsuite/alt-dest.test

@@ -0,0 +1,68 @@
+#!/bin/sh
+
+# Copyright (C) 2004-2022 Wayne Davison
+
+# This program is distributable under the terms of the GNU GPL (see
+# COPYING).
+
+# Test rsync handling of --compare-dest and similar options.
+
+. "$suitedir/rsync.fns"
+
+alt1dir="$tmpdir/alt1"
+alt2dir="$tmpdir/alt2"
+alt3dir="$tmpdir/alt3"
+
+SSH="$scratchdir/src/support/lsh.sh"
+
+# Build some files/dirs/links to copy
+
+hands_setup
+
+# Setup the alt and chk dirs
+$RSYNC -av --include=text --include='*/' --exclude='*' "$fromdir/" "$alt1dir/"
+$RSYNC -av --include=etc-ltr-list --include='*/' --exclude='*' "$fromdir/" "$alt2dir/"
+
+# Create a side dir where there is a candidate destfile of the same name as a sourcefile
+echo "This is a test file" >"$fromdir/likely"
+
+mkdir "$alt3dir"
+echo "This is a test file" >"$alt3dir/likely"
+
+sleep 1
+touch "$fromdir/dir/text" "$fromdir/likely"
+
+$RSYNC -av --exclude=/text --exclude=etc-ltr-list "$fromdir/" "$chkdir/"
+
+# Let's do it!
+checkit "$RSYNC -avv --no-whole-file \
+    --compare-dest='$alt1dir' --compare-dest='$alt2dir' \
+    '$fromdir/' '$todir/'" "$chkdir" "$todir"
+
+rm -rf "$todir"
+checkit "$RSYNC -avv --no-whole-file \
+    --copy-dest='$alt1dir' --copy-dest='$alt2dir' \
+    '$fromdir/' '$todir/'" "$fromdir" "$todir"
+
+# Test that copy_file() works correctly with tmpfiles
+for maybe_inplace in '' --inplace; do
+    rm -rf "$todir"
+    checkit "$RSYNC -av $maybe_inplace --copy-dest='$alt3dir' \
+	'$fromdir/' '$todir/'" "$fromdir" "$todir"
+
+    for srchost in '' 'localhost:'; do
+	if [ -z "$srchost" ]; then
+	    desthost='localhost:'
+	else
+	    desthost=''
+	fi
+
+	rm -rf "$todir"
+	checkit "$RSYNC -ave '$SSH' --rsync-path='$RSYNC' $maybe_inplace \
+	    --copy-dest='$alt3dir' '$srchost$fromdir/' '$desthost$todir/'" \
+	    "$fromdir" "$todir"
+    done
+done
+
+# The script would have aborted on error, so getting here means we've won.
+exit 0