Add early exec daemon parameter.

Inspired by Ciprian Dorin Craciun's `bootstrap exec` patch.

NEWS.md

@@ -47,6 +47,9 @@ Protocol: 31 (unchanged)
 
  - Avoid a hang when an overabundance of messages clogs up all the I/O buffers.
 
+ - Fixed a mismatch in the RSYNC_PID values when running both a `pre-xfer exec`
+   and a `post-xfer exec`.
+
 ### ENHANCEMENTS:
 
  - Various checksum enhancements, including the optional use of openssl's MD4 &
@@ -95,6 +98,10 @@ Protocol: 31 (unchanged)
  - Added negated matching to the daemon's `refuse options` setting by using
    match strings that start with a `!` (such as `!compress*`).
 
+ - Added an `early exec` daemon parameter that runs a script before the
+   transfer parameters are known, allowing some early setup based on module
+   name.
+
  - Added status output in response to a signal (via both SIGINFO & SIGVTALRM).
 
  - Added a `--copy-as=USER` option to give some extra security to root-run

clientserver.c

@@ -349,61 +349,6 @@ int start_inband_exchange(int f_in, int f_out, const char *user, int argc, char
 	return 0;
 }
 
-static char *finish_pre_exec(pid_t pid, int write_fd, int read_fd, char *request,
-			     char **early_argv, char **argv)
-{
-	char buf[BIGPATHBUFLEN], *bp;
-	int j = 0, status = -1, msglen = sizeof buf - 1;
-
-	if (!request)
-		request = "(NONE)";
-
-	write_buf(write_fd, request, strlen(request)+1);
-	if (early_argv) {
-		for ( ; *early_argv; early_argv++)
-			write_buf(write_fd, *early_argv, strlen(*early_argv)+1);
-		j = 1; /* Skip arg0 name in argv. */
-	}
-	for ( ; argv[j]; j++)
-		write_buf(write_fd, argv[j], strlen(argv[j])+1);
-	write_byte(write_fd, 0);
-
-	close(write_fd);
-
-	/* Read the stdout from the pre-xfer exec program.  This it is only
-	 * displayed to the user if the script also returns an error status. */
-	for (bp = buf; msglen > 0; msglen -= j) {
-		if ((j = read(read_fd, bp, msglen)) <= 0) {
-			if (j == 0)
-				break;
-			if (errno == EINTR)
-				continue;
-			break; /* Just ignore the read error for now... */
-		}
-		bp += j;
-		if (j > 1 && bp[-1] == '\n' && bp[-2] == '\r') {
-			bp--;
-			j--;
-			bp[-1] = '\n';
-		}
-	}
-	*bp = '\0';
-
-	close(read_fd);
-
-	if (wait_process(pid, &status, 0) < 0
-	 || !WIFEXITED(status) || WEXITSTATUS(status) != 0) {
-		char *e;
-		if (asprintf(&e, "pre-xfer exec returned failure (%d)%s%s%s\n%s",
-			     status, status < 0 ? ": " : "",
-			     status < 0 ? strerror(errno) : "",
-			     *buf ? ":" : "", buf) < 0)
-			return "out_of_memory in finish_pre_exec\n";
-		return e;
-	}
-	return NULL;
-}
-
 #ifdef HAVE_PUTENV
 static int read_arg_from_pipe(int fd, char *buf, int limit)
 {
@@ -427,6 +372,168 @@ static int read_arg_from_pipe(int fd, char *buf, int limit)
 }
 #endif
 
+static void set_env_str(const char *var, const char *str)
+{
+#ifdef HAVE_PUTENV
+	char *mem;
+	if (asprintf(&mem, "%s=%s", var, str) < 0)
+		out_of_memory("set_env_str");
+	putenv(mem);
+#endif
+}
+
+#ifdef HAVE_PUTENV
+void set_env_num(const char *var, long num)
+{
+	char *mem;
+	if (asprintf(&mem, "%s=%ld", var, num) < 0)
+		out_of_memory("set_env_num");
+	putenv(mem);
+}
+#endif
+
+/* Used for both early exec & pre-xfer exec */
+static pid_t start_pre_exec(const char *cmd, int *arg_fd_ptr, int *error_fd_ptr)
+{
+	int arg_fds[2], error_fds[2], arg_fd, error_fd;
+	pid_t pid;
+
+	if ((error_fd_ptr && pipe(error_fds) < 0) || (arg_fd_ptr && pipe(arg_fds) < 0) || (pid = fork()) < 0)
+		return (pid_t)-1;
+
+	if (pid == 0) {
+		char buf[BIGPATHBUFLEN];
+		int j, len, status;
+
+		if (error_fd_ptr) {
+			close(error_fds[0]);
+			error_fd = error_fds[1];
+			set_blocking(error_fd);
+		}
+
+		if (arg_fd_ptr) {
+			close(arg_fds[1]);
+			arg_fd = arg_fds[0];
+			set_blocking(arg_fd);
+
+			len = read_arg_from_pipe(arg_fd, buf, BIGPATHBUFLEN);
+			if (len <= 0)
+				_exit(1);
+			set_env_str("RSYNC_REQUEST", buf);
+
+			for (j = 0; ; j++) {
+				char *p;
+				len = read_arg_from_pipe(arg_fd, buf, BIGPATHBUFLEN);
+				if (len <= 0) {
+					if (!len)
+						break;
+					_exit(1);
+				}
+				if (asprintf(&p, "RSYNC_ARG%d=%s", j, buf) >= 0)
+					putenv(p);
+			}
+			close(arg_fd);
+		}
+
+		if (error_fd_ptr) {
+			close(STDIN_FILENO);
+			dup2(error_fd, STDOUT_FILENO);
+			close(error_fd);
+		}
+
+		status = shell_exec(cmd);
+
+		if (!WIFEXITED(status))
+			_exit(1);
+		_exit(WEXITSTATUS(status));
+	}
+
+	if (error_fd_ptr) {
+		close(error_fds[1]);
+		error_fd = *error_fd_ptr = error_fds[0];
+		set_blocking(error_fd);
+	}
+
+	if (arg_fd_ptr) {
+		close(arg_fds[0]);
+		arg_fd = *arg_fd_ptr = arg_fds[1];
+		set_blocking(arg_fd);
+	}
+
+	return pid;
+}
+
+static void write_pre_exec_args(int write_fd, char *request, char **early_argv, char **argv)
+{
+	int j = 0;
+
+	if (!request)
+		request = "(NONE)";
+
+	write_buf(write_fd, request, strlen(request)+1);
+	if (early_argv) {
+		for ( ; *early_argv; early_argv++)
+			write_buf(write_fd, *early_argv, strlen(*early_argv)+1);
+		j = 1; /* Skip arg0 name in argv. */
+	}
+	for ( ; argv[j]; j++)
+		write_buf(write_fd, argv[j], strlen(argv[j])+1);
+	write_byte(write_fd, 0);
+
+	close(write_fd);
+}
+
+static char *finish_pre_exec(const char *desc, pid_t pid, int read_fd)
+{
+	char buf[BIGPATHBUFLEN], *bp, *cr;
+	int j, status = -1, msglen = sizeof buf - 1;
+
+	if (read_fd >= 0) {
+		/* Read the stdout from the program.  This it is only displayed
+		 * to the user if the script also returns an error status. */
+		for (bp = buf, cr = buf; msglen > 0; msglen -= j) {
+			if ((j = read(read_fd, bp, msglen)) <= 0) {
+				if (j == 0)
+					break;
+				if (errno == EINTR)
+					continue;
+				break; /* Just ignore the read error for now... */
+			}
+			bp[j] = '\0';
+			while (1) {
+				if ((cr = strchr(cr, '\r')) == NULL) {
+					cr = bp + j;
+					break;
+				}
+				if (!cr[1])
+					break; /* wait for more data before we decide what to do */
+				if (cr[1] == '\n') {
+					memmove(cr, cr+1, j - (cr - bp));
+					j--;
+				} else
+					cr++;
+			}
+			bp += j;
+		}
+		*bp = '\0';
+
+		close(read_fd);
+	} else
+		*buf = '\0';
+
+	if (wait_process(pid, &status, 0) < 0
+	 || !WIFEXITED(status) || WEXITSTATUS(status) != 0) {
+		char *e;
+		if (asprintf(&e, "%s returned failure (%d)%s%s%s\n%s",
+			     desc, status, status < 0 ? ": " : "",
+			     status < 0 ? strerror(errno) : "",
+			     *buf ? ":" : "", buf) < 0)
+			return "out_of_memory in finish_pre_exec\n";
+		return e;
+	}
+	return NULL;
+}
+
 static int path_failure(int f_out, const char *dir, BOOL was_chdir)
 {
 	if (was_chdir)
@@ -478,26 +585,6 @@ static struct passwd *want_all_groups(int f_out, uid_t uid)
 }
 #endif
 
-static void set_env_str(const char *var, const char *str)
-{
-#ifdef HAVE_PUTENV
-	char *mem;
-	if (asprintf(&mem, "%s=%s", var, str) < 0)
-		out_of_memory("set_env_str");
-	putenv(mem);
-#endif
-}
-
-#ifdef HAVE_PUTENV
-void set_env_num(const char *var, long num)
-{
-	char *mem;
-	if (asprintf(&mem, "%s=%ld", var, num) < 0)
-		out_of_memory("set_env_num");
-	putenv(mem);
-}
-#endif
-
 static int rsync_module(int f_in, int f_out, int i, const char *addr, const char *host)
 {
 	int argc;
@@ -690,8 +777,9 @@ static int rsync_module(int f_in, int f_out, int i, const char *addr, const char
 	log_init(1);
 
 #ifdef HAVE_PUTENV
-	if ((*lp_prexfer_exec(i) || *lp_postxfer_exec(i)) && !getenv("RSYNC_NO_XFER_EXEC")) {
-		int status;
+	if ((*lp_early_exec(i) || *lp_prexfer_exec(i) || *lp_postxfer_exec(i))
+	 && !getenv("RSYNC_NO_XFER_EXEC")) {
+		set_env_num("RSYNC_PID", (long)getpid());
 
 		/* For post-xfer exec, fork a new process to run the rsync
 		 * daemon while this process waits for the exit status and
@@ -704,10 +792,10 @@ static int rsync_module(int f_in, int f_out, int i, const char *addr, const char
 				return -1;
 			}
 			if (pid) {
+				int status;
 				close(f_in);
 				if (f_out != f_in)
 					close(f_out);
-				set_env_num("RSYNC_PID", (long)pid);
 				if (wait_process(pid, &status, 0) < 0)
 					status = -1;
 				set_env_num("RSYNC_RAW_STATUS", status);
@@ -721,56 +809,33 @@ static int rsync_module(int f_in, int f_out, int i, const char *addr, const char
 				_exit(status);
 			}
 		}
+
+		/* For early exec, fork a child process to run the indicated
+		 * command and wait for it to exit. */
+		if (*lp_early_exec(i)) {
+			pid_t pid = start_pre_exec(lp_early_exec(i), NULL, NULL);
+			if (pid == (pid_t)-1) {
+				rsyserr(FLOG, errno, "early exec preparation failed");
+				io_printf(f_out, "@ERROR: early exec preparation failed\n");
+				return -1;
+			}
+			if (finish_pre_exec("early exec", pid, -1) != NULL) {
+				rsyserr(FLOG, errno, "early exec failed");
+				io_printf(f_out, "@ERROR: early exec failed\n");
+				return -1;
+			}
+		}
+
 		/* For pre-xfer exec, fork a child process to run the indicated
 		 * command, though it first waits for the parent process to
 		 * send us the user's request via a pipe. */
 		if (*lp_prexfer_exec(i)) {
-			int arg_fds[2], error_fds[2];
-			set_env_num("RSYNC_PID", (long)getpid());
-			if (pipe(arg_fds) < 0 || pipe(error_fds) < 0 || (pre_exec_pid = fork()) < 0) {
+			pre_exec_pid = start_pre_exec(lp_prexfer_exec(i), &pre_exec_arg_fd, &pre_exec_error_fd);
+			if (pre_exec_pid == (pid_t)-1) {
 				rsyserr(FLOG, errno, "pre-xfer exec preparation failed");
 				io_printf(f_out, "@ERROR: pre-xfer exec preparation failed\n");
 				return -1;
 			}
-			if (pre_exec_pid == 0) {
-				char buf[BIGPATHBUFLEN];
-				int j, len;
-				close(arg_fds[1]);
-				close(error_fds[0]);
-				pre_exec_arg_fd = arg_fds[0];
-				pre_exec_error_fd = error_fds[1];
-				set_blocking(pre_exec_arg_fd);
-				set_blocking(pre_exec_error_fd);
-				len = read_arg_from_pipe(pre_exec_arg_fd, buf, BIGPATHBUFLEN);
-				if (len <= 0)
-					_exit(1);
-				set_env_str("RSYNC_REQUEST", buf);
-				for (j = 0; ; j++) {
-					len = read_arg_from_pipe(pre_exec_arg_fd, buf,
-								 BIGPATHBUFLEN);
-					if (len <= 0) {
-						if (!len)
-							break;
-						_exit(1);
-					}
-					if (asprintf(&p, "RSYNC_ARG%d=%s", j, buf) >= 0)
-						putenv(p);
-				}
-				close(pre_exec_arg_fd);
-				close(STDIN_FILENO);
-				dup2(pre_exec_error_fd, STDOUT_FILENO);
-				close(pre_exec_error_fd);
-				status = shell_exec(lp_prexfer_exec(i));
-				if (!WIFEXITED(status))
-					_exit(1);
-				_exit(WEXITSTATUS(status));
-			}
-			close(arg_fds[0]);
-			close(error_fds[1]);
-			pre_exec_arg_fd = arg_fds[1];
-			pre_exec_error_fd = error_fds[0];
-			set_blocking(pre_exec_arg_fd);
-			set_blocking(pre_exec_error_fd);
 		}
 	}
 #endif
@@ -889,8 +954,8 @@ static int rsync_module(int f_in, int f_out, int i, const char *addr, const char
 		msgs2stderr = 0; /* A non-rsh-run daemon doesn't have stderr for msgs. */
 
 	if (pre_exec_pid) {
-		err_msg = finish_pre_exec(pre_exec_pid, pre_exec_arg_fd, pre_exec_error_fd,
-					  request, orig_early_argv, orig_argv);
+		write_pre_exec_args(pre_exec_arg_fd, request, orig_early_argv, orig_argv);
+		err_msg = finish_pre_exec("pre-xfer exec", pre_exec_pid, pre_exec_error_fd);
 	}
 
 	if (orig_early_argv)

loadparm.c

@@ -122,6 +122,7 @@ typedef struct {
 	char *charset;
 	char *comment;
 	char *dont_compress;
+	char *early_exec;
 	char *exclude;
 	char *exclude_from;
 	char *filter;
@@ -150,6 +151,7 @@ typedef struct {
 	BOOL charset_EXP;
 	BOOL comment_EXP;
 	BOOL dont_compress_EXP;
+	BOOL early_exec_EXP;
 	BOOL exclude_EXP;
 	BOOL exclude_from_EXP;
 	BOOL filter_EXP;
@@ -236,7 +238,8 @@ static const all_vars Defaults = {
  /* charset; */ 		NULL,
  /* comment; */ 		NULL,
  /* dont_compress; */		DEFAULT_DONT_COMPRESS,
- /* exclude; */			NULL,
+ /* early_exec; */		NULL,
+ /* exclude; */ 		NULL,
  /* exclude_from; */		NULL,
  /* filter; */			NULL,
  /* gid; */			NULL,
@@ -263,6 +266,7 @@ static const all_vars Defaults = {
  /* charset_EXP; */		False,
  /* comment_EXP; */		False,
  /* dont_compress_EXP; */	False,
+ /* early_exec_EXP; */		False,
  /* exclude_EXP; */		False,
  /* exclude_from_EXP; */	False,
  /* filter_EXP; */		False,
@@ -404,6 +408,7 @@ static struct parm_struct parm_table[] =
  {"charset",           P_STRING, P_LOCAL, &Vars.l.charset,             NULL,0},
  {"comment",           P_STRING, P_LOCAL, &Vars.l.comment,             NULL,0},
  {"dont compress",     P_STRING, P_LOCAL, &Vars.l.dont_compress,       NULL,0},
+ {"early exec",        P_STRING, P_LOCAL, &Vars.l.early_exec,          NULL,0},
  {"exclude from",      P_STRING, P_LOCAL, &Vars.l.exclude_from,        NULL,0},
  {"exclude",           P_STRING, P_LOCAL, &Vars.l.exclude,             NULL,0},
  {"fake super",        P_BOOL,   P_LOCAL, &Vars.l.fake_super,          NULL,0},
@@ -543,6 +548,7 @@ FN_LOCAL_STRING(lp_auth_users, auth_users)
 FN_LOCAL_STRING(lp_charset, charset)
 FN_LOCAL_STRING(lp_comment, comment)
 FN_LOCAL_STRING(lp_dont_compress, dont_compress)
+FN_LOCAL_STRING(lp_early_exec, early_exec)
 FN_LOCAL_STRING(lp_exclude, exclude)
 FN_LOCAL_STRING(lp_exclude_from, exclude_from)
 FN_LOCAL_STRING(lp_filter, filter)

rsyncd.conf.5.md

@@ -917,15 +917,28 @@ the values of parameters.  See the GLOBAL PARAMETERS section for more details.
     for the "dont compress" parameter changes the default when the daemon is
     the sender.
 
-0.  `pre-xfer exec`, `post-xfer exec`
+0.  `early exec`, `pre-xfer exec`, `post-xfer exec`
 
-    You may specify a command to be run before and/or after the transfer.  If
-    the `pre-xfer exec` command fails, the transfer is aborted before it
-    begins.  Any output from the script on stdout (up to several KB) will be
-    displayed to the user when aborting, but is NOT displayed if the script
-    returns success.  Any output from the script on stderr goes to the daemon's
-    stderr, which is typically discarded (though see --no-detatch option for a
-    way to see the stderr output, which can assist with debugging).
+    You may specify a command to be run in the early stages of the connection,
+    or right before and/or after the transfer.  If the `early exec` or
+    `pre-xfer exec` command returns an error code, the transfer is aborted
+    before it begins.  Any output from the `pre-xfer exec` command on stdout
+    (up to several KB) will be displayed to the user when aborting, but is
+    _not_ displayed if the script returns success.  The other programs cannot
+    send any text to the user.  All output except for the `pre-xfer exec`
+    stdout goes to the corresponding daemon's stdout/stderr, which is typically
+    discarded.  See the `--no-detatch` option for a way to see the daemon's
+    output, which can assist with debugging.
+
+    Note that the `early exec` command runs before any part of the transfer
+    request is known except for the module name.  This helper script can be
+    used to setup a disk mount or decrypt some data into a module dir, but you
+    may need to use `lock file` and `max connections` to avoid concurrency
+    issues.
+
+    Note that the `post-xfer exec` command is still run even if one of the
+    other scripts returns an error code. The `pre-xfer exec` command will _not_
+    be run, however, if the `early exec` command fails.
 
     The following environment variables will be set, though some are specific
     to the pre-xfer or the post-xfer environment: