mirror/rsync
1
0
Fork 0
mirror of https://github.com/RsyncProject/rsync.git synced 2026-05-19 04:15:56 -04:00
Code Issues Packages Projects Releases Wiki Activity

More compress changes

Browse Source 
- Add the zlibx (external-code compatible) compression name.
- Re-enable zlib support with the external library so it can be
  tried as a fallback if zlibx isn't available.
- Add --compress-choice=STR (aka -zz=STR) option.
- Make --cc=STR an alias for --checksum-choice=STR.
- Hook up the new compression negotiation logic.
This commit is contained in:
Wayne Davison
2020-05-24 15:45:59 -07:00
parent 4af8403aa2
commit 64d5ea39c0
5 changed files with 126 additions and 46 deletions
Download Patch File Download Diff File Expand all files Collapse all files

38
compat.c
View File

@@ -89,10 +89,17 @@ int filesfrom_convert = 0;
#define CPRES_NONE 0
#define CPRES_ZLIB 1
#define CPRES_ZLIBX 2
struct name_num_obj valid_compressions = {
"compress", NULL, NULL, 0, 0, {
#ifndef EXTERNAL_ZLIB
{ CPRES_ZLIB, "zlib", NULL },
#endif
{ CPRES_ZLIBX, "zlibx", NULL },
#ifdef EXTERNAL_ZLIB
{ CPRES_ZLIB, "zlib", NULL },
#endif
{ CPRES_NONE, "none", NULL },
{ 0, NULL, NULL }
}
@@ -160,6 +167,37 @@ void set_allow_inc_recurse(void)
allow_inc_recurse = 0;
}
void parse_compress_choice(int final_call)
{
int num;
if (valid_compressions.negotiated_name)
num = valid_compressions.negotiated_num;
else if (compress_choice) {
struct name_num_item *nni = get_nni_by_name(&valid_compressions, compress_choice, -1);
if (!nni) {
rprintf(FERROR, "unknown compress name: %s\n", compress_choice);
exit_cleanup(RERR_UNSUPPORTED);
}
num = nni->num;
} else
num = CPRES_NONE;
if (num == CPRES_NONE) {
do_compression = 0;
compress_choice = NULL;
} else if (num > 0)
do_compression = num != CPRES_ZLIB ? 2 : 1;
if (final_call && DEBUG_GTE(NSTR, am_server ? 2 : 1)) {
const char *c_s = am_server ? "Server" : "Client";
if (valid_compressions.negotiated_name)
rprintf(FINFO, "%s negotiated compress: %s\n", c_s, valid_compressions.negotiated_name);
else
rprintf(FINFO, "%s compress: %s\n", c_s, do_compression ? compress_choice : "none");
}
}
struct name_num_item *get_nni_by_name(struct name_num_obj *nno, const char *name, int len)
{
struct name_num_item *nni;

1
flist.c
View File

@@ -143,6 +143,7 @@ void init_flist(void)
(int)FILE_STRUCT_LEN, (int)EXTRA_LEN);
}
parse_checksum_choice(1); /* Sets checksum_type && xfersum_type */
parse_compress_choice(1); /* Sets do_compression */
flist_csum_len = csum_len_for_type(checksum_type, 1);
show_filelist_progress = INFO_GTE(FLIST, 1) && xfer_dirs && !am_server && !inc_recurse;

62
options.c
View File

@@ -840,6 +840,7 @@ enum {OPT_VERSION = 1000, OPT_DAEMON, OPT_SENDER, OPT_EXCLUDE, OPT_EXCLUDE_FROM,
OPT_READ_BATCH, OPT_WRITE_BATCH, OPT_ONLY_WRITE_BATCH, OPT_MAX_SIZE,
OPT_NO_D, OPT_APPEND, OPT_NO_ICONV, OPT_INFO, OPT_DEBUG,
OPT_USERMAP, OPT_GROUPMAP, OPT_CHOWN, OPT_BWLIMIT,
OPT_OLD_COMPRESS, OPT_NEW_COMPRESS, OPT_NO_COMPRESS,
OPT_SERVER, OPT_REFUSED_BASE = 9000};
static struct poptOption long_options[] = {
@@ -981,11 +982,12 @@ static struct poptOption long_options[] = {
{"cvs-exclude", 'C', POPT_ARG_NONE, &cvs_exclude, 0, 0, 0 },
{"whole-file", 'W', POPT_ARG_VAL, &whole_file, 1, 0, 0 },
{"no-whole-file", 0, POPT_ARG_VAL, &whole_file, 0, 0, 0 },
{"checksum-choice", 0, POPT_ARG_STRING, &checksum_choice, 0, 0, 0 },
{"no-W", 0, POPT_ARG_VAL, &whole_file, 0, 0, 0 },
{"checksum", 'c', POPT_ARG_VAL, &always_checksum, 1, 0, 0 },
{"no-checksum", 0, POPT_ARG_VAL, &always_checksum, 0, 0, 0 },
{"no-c", 0, POPT_ARG_VAL, &always_checksum, 0, 0, 0 },
{"checksum-choice", 0, POPT_ARG_STRING, &checksum_choice, 0, 0, 0 },
{"cc", 0, POPT_ARG_STRING, &checksum_choice, 0, 0, 0 },
{"block-size", 'B', POPT_ARG_LONG, &block_size, 0, 0, 0 },
{"compare-dest", 0, POPT_ARG_STRING, 0, OPT_COMPARE_DEST, 0, 0 },
{"copy-dest", 0, POPT_ARG_STRING, 0, OPT_COPY_DEST, 0, 0 },
@@ -994,10 +996,12 @@ static struct poptOption long_options[] = {
{"no-fuzzy", 0, POPT_ARG_VAL, &fuzzy_basis, 0, 0, 0 },
{"no-y", 0, POPT_ARG_VAL, &fuzzy_basis, 0, 0, 0 },
{"compress", 'z', POPT_ARG_NONE, 0, 'z', 0, 0 },
{"old-compress", 0, POPT_ARG_VAL, &do_compression, 1, 0, 0 },
{"new-compress", 0, POPT_ARG_VAL, &do_compression, 2, 0, 0 },
{"no-compress", 0, POPT_ARG_VAL, &do_compression, 0, 0, 0 },
{"no-z", 0, POPT_ARG_VAL, &do_compression, 0, 0, 0 },
{"old-compress", 0, POPT_ARG_NONE, 0, OPT_OLD_COMPRESS, 0, 0 },
{"new-compress", 0, POPT_ARG_NONE, 0, OPT_NEW_COMPRESS, 0, 0 },
{"no-compress", 0, POPT_ARG_NONE, 0, OPT_NO_COMPRESS, 0, 0 },
{"no-z", 0, POPT_ARG_NONE, 0, OPT_NO_COMPRESS, 0, 0 },
{"compress-choice", 0, POPT_ARG_STRING, &compress_choice, 0, 0, 0 },
{"zz", 0, POPT_ARG_STRING, &compress_choice, 0, 0, 0 },
{"skip-compress", 0, POPT_ARG_STRING, &skip_compress, 0, 0, 0 },
{"compress-level", 0, POPT_ARG_INT, &def_compress_level, 0, 0, 0 },
{0, 'P', POPT_ARG_NONE, 0, 'P', 0, 0 },
@@ -1668,6 +1672,19 @@ int parse_arguments(int *argc_p, const char ***argv_p)
do_compression++;
break;
case OPT_OLD_COMPRESS:
compress_choice = "zlib";
break;
case OPT_NEW_COMPRESS:
compress_choice = "zlibx";
break;
case OPT_NO_COMPRESS:
do_compression = 0;
compress_choice = NULL;
break;
case 'M':
arg = poptGetOptArg(pc);
if (*arg != '-') {
@@ -1948,6 +1965,13 @@ int parse_arguments(int *argc_p, const char ***argv_p)
exit_cleanup(0);
}
if (!compress_choice && do_compression > 1)
compress_choice = "zlibx";
if (compress_choice && strcasecmp(compress_choice, "auto") != 0)
parse_compress_choice(0); /* Can twiddle do_compression and possibly NULL-out compress_choice */
else
compress_choice = NULL;
if (do_compression || def_compress_level != NOT_SPECIFIED) {
if (def_compress_level == NOT_SPECIFIED)
def_compress_level = Z_DEFAULT_COMPRESSION;
@@ -1955,24 +1979,15 @@ int parse_arguments(int *argc_p, const char ***argv_p)
snprintf(err_buf, sizeof err_buf, "--compress-level value is invalid: %d\n",
def_compress_level);
return 0;
} else if (def_compress_level == Z_NO_COMPRESSION)
} else if (def_compress_level == Z_NO_COMPRESSION) {
do_compression = 0;
else if (!do_compression)
compress_choice = NULL;
} else if (!do_compression)
do_compression = 1;
if (do_compression && refused_compress) {
create_refuse_error(refused_compress);
return 0;
}
#ifdef EXTERNAL_ZLIB
if (do_compression == 1) {
snprintf(err_buf, sizeof err_buf,
"This rsync lacks old-style --compress due to its external zlib. Try -zz.\n");
if (am_server)
return 0;
fprintf(stderr, "%s" "Continuing without compression.\n\n", err_buf);
do_compression = 0;
}
#endif
}
#ifdef HAVE_SETVBUF
@@ -2750,6 +2765,16 @@ void server_options(char **args, int *argc_p)
args[ac++] = arg;
}
if ((!compress_choice && do_compression > 1) || (compress_choice && strcasecmp(compress_choice, "zlibx") == 0))
args[ac++] = "--new-compress";
else if (compress_choice && strcasecmp(compress_choice, "zlib") == 0)
args[ac++] = "--old-compress";
else if (compress_choice) {
if (asprintf(&arg, "--compress-choice=%s", compress_choice) < 0)
goto oom;
args[ac++] = arg;
}
if (am_sender) {
if (max_delete > 0) {
if (asprintf(&arg, "--max-delete=%d", max_delete) < 0)
@@ -2930,9 +2955,6 @@ void server_options(char **args, int *argc_p)
exit_cleanup(RERR_MALLOC);
}
if (do_compression > 1)
args[ac++] = "--new-compress";
if (remote_option_cnt) {
int j;
if (ac + remote_option_cnt > MAX_SERVER_ARGS) {

55
rsync.yo
View File

@@ -1370,7 +1370,7 @@ destination machines is higher than the bandwidth to disk (especially when the
the source and destination are specified as local paths, but only if no
batch-writing option is in effect.
dit(bf(--checksum-choice=STR)) This option overrides the checksum algorithms.
dit(bf(--checksum-choice=STR, --cc=STR)) This option overrides the checksum algorithms.
If one algorithm name is specified, it is used for both the transfer checksums
and (assuming bf(--checksum) is specified) the pre-transfer checksums. If two
comma-separated names are supplied, the first name affects the transfer
@@ -1402,7 +1402,7 @@ error. This method does not allow you to specify the transfer checksum
separately from the pre-transfer checksum, and it ignores "auto" and all
unknown checksum names. If the remote rsync is not new enough to handle a
checksum negotiation list, the list is silently ignored unless it contains the
string "FAIL" in it.
string "FAIL".
The use of the bf(--checksum-choice) option overrides this environment list.
@@ -2056,27 +2056,54 @@ dit(bf(-z, --compress)) With this option, rsync compresses the file data
as it is sent to the destination machine, which reduces the amount of data
being transmitted -- something that is useful over a slow connection.
Note that this option typically achieves better compression ratios than can
be achieved by using a compressing remote shell or a compressing transport
The "zlib" compression method typically achieves better compression ratios than
can be achieved by using a compressing remote shell or a compressing transport
because it takes advantage of the implicit information in the matching data
blocks that are not explicitly sent over the connection. This matching-data
compression comes at a cost of CPU, though, and can be disabled by repeating
the bf(-z) option, but only if both sides are at least version 3.1.1.
compression comes at a cost of CPU, though, and can be disabled by using the
"zlibx" compresson method instead. This can be selected by repeating the
bf(-z) option or specifying bf(--compress-choice=zlibx), but it only works if
both sides of the transfer are at least version 3.1.1.
Note that if your version of rsync was compiled with an external zlib (instead
of the zlib that comes packaged with rsync) then it will not support the
old-style compression, only the new-style (repeated-option) compression. In
the future this new-style compression will likely become the default.
of the zlib that comes packaged with rsync) then it will give preference to
using the "zlibx" algorithm over the "zlib" algorithm since the external zlib
code doesn't seem to handle the extra compression properly. You can try
forcing the regular algorithm via bf(--zz=zlib) and be on the lookout for
transfer failures. If all else fails, disable compression altogether.
The client rsync requests new-style compression on the server via the
bf(--new-compress) option, so if you see that option rejected it means that
the server is not new enough to support bf(-zz). Rsync also accepts the
bf(--old-compress) option for a future time when new-style compression
becomes the default.
Note that if you see an error about an option named bf(--old-compress) or
bf(--new-compress), this is rsync trying to send the bf(--compress-choice=zlib)
or bf(--compress-choice=zlibx) option in a backward-compatible manner that more
rsync versions understand. This error indicates that the older rsync version
will not allow you to force the compression type.
See the bf(--skip-compress) option for the default list of file suffixes
that will not be compressed.
dit(bf(--compress-choice=STR, --zz=STR)) This option can be used to override the
automatic selection of the compression algorithm that is the default when
bf(--compress) is used.
Currently the STR can be "zlib", "zlibx", or "none".
The "zlib" algorithm is given preference over "zlibx" if your rsync was
compiled with the internal zlib code, otherwise that preference is reversed.
These 2 algorithms are the stame except that "zlibx" does not try to include
matched data that was not transferred in the compression computations.
If "none" is specified, that is equivalent to using bf(--no-compress).
This option implies bf(--compress) unless "none" was specified.
You can also override the compression negotation using the RSYNC_COMPRESS_LIST
environment variable by setting it to a space-separated list of compression
names that you consider acceptable. If no common compress choice is found, the
client exits with an error. It ignores "auto" and all unknown compression
names. If the remote rsync is not new enough to handle a compression
negotiation list, the list is silently ignored unless it contains the string
"FAIL".
dit(bf(--compress-level=NUM)) Explicitly set the compression level to use
(see bf(--compress)) instead of letting it default. If NUM is non-zero,
the bf(--compress) option is implied.

16
token.c
View File

@@ -29,6 +29,10 @@ extern int module_id;
extern int def_compress_level;
extern char *skip_compress;
#ifndef Z_INSERT_ONLY
#define Z_INSERT_ONLY Z_SYNC_FLUSH
#endif
static int compression_level, per_file_default_level;
struct suffix_tree {
@@ -405,7 +409,6 @@ send_deflated_token(int f, int32 token, struct map_struct *buf, OFF_T offset,
} else if (token != -2 && do_compression == 1) {
/* Add the data in the current block to the compressor's
* history and hash table. */
#ifndef EXTERNAL_ZLIB
do {
/* Break up long sections in the same way that
* see_deflate_token() does. */
@@ -424,11 +427,6 @@ send_deflated_token(int f, int32 token, struct map_struct *buf, OFF_T offset,
exit_cleanup(RERR_STREAMIO);
}
} while (toklen > 0);
#else
toklen++;
rprintf(FERROR, "Impossible error in external-zlib code (1).\n");
exit_cleanup(RERR_STREAMIO);
#endif
}
}
@@ -579,7 +577,6 @@ static int32 recv_deflated_token(int f, char **data)
*/
static void see_deflate_token(char *buf, int32 len)
{
#ifndef EXTERNAL_ZLIB
int r;
int32 blklen;
unsigned char hdr[5];
@@ -617,11 +614,6 @@ static void see_deflate_token(char *buf, int32 len)
exit_cleanup(RERR_STREAMIO);
}
} while (len || rx_strm.avail_out == 0);
#else
buf++; len++;
rprintf(FERROR, "Impossible error in external-zlib code (2).\n");
exit_cleanup(RERR_STREAMIO);
#endif
}
/**