From 22a3ac0b5538ec6c1ff222570413cbdb74fef67b Mon Sep 17 00:00:00 2001 From: Wayne Davison Date: Sat, 19 Apr 2014 12:11:11 -0700 Subject: Add new-style compression that skips matching data. Adding new-style compression that only compresses the literal data that is sent over the wire and not also matching file data that was not sent. This new-style compression is compatible with external zlib instances, and will eventually become the default (once enough time has passed that all servers support the --new-compress and --old-compress options). NOTE: if you build rsync with an external zlib (i.e. if you specified configure --with-included-zlib=no) you will ONLY get support for the --new-compress option! A client will treat -z as uncompressed (with a warning) and a server will exit with an error (unless -zz was used). --- configure.ac | 1 + options.c | 53 ++++++++++++++++++++++++++----------- rsync.yo | 15 ++++++++++- testsuite/daemon-gzip-download.test | 2 +- testsuite/daemon-gzip-upload.test | 2 +- token.c | 38 +++++++++++++++----------- 6 files changed, 77 insertions(+), 34 deletions(-) diff --git a/configure.ac b/configure.ac index dccb2aaf..c7b28c52 100644 --- a/configure.ac +++ b/configure.ac @@ -792,6 +792,7 @@ if test x"$with_included_zlib" = x"yes"; then BUILD_ZLIB='$(zlib_OBJS)' CFLAGS="$CFLAGS -I$srcdir/zlib" else + AC_DEFINE(EXTERNAL_ZLIB, 1, [Define to 1 if using external zlib]) AC_MSG_RESULT(no) fi diff --git a/options.c b/options.c index dc9e62a4..0aa64bf0 100644 --- a/options.c +++ b/options.c @@ -32,6 +32,8 @@ extern unsigned int module_dirlen; extern filter_rule_list filter_list; extern filter_rule_list daemon_filter_list; +#define NOT_SPECIFIED (-42) + int make_backups = 0; /** @@ -75,7 +77,7 @@ int protocol_version = PROTOCOL_VERSION; int sparse_files = 0; int preallocate_files = 0; int do_compression = 0; -int def_compress_level = Z_DEFAULT_COMPRESSION; +int def_compress_level = NOT_SPECIFIED; int am_root = 0; /* 0 = normal, 1 = root, 2 = --super, -1 = --fake-super */ int am_server = 0; int am_sender = 0; @@ -965,10 +967,12 @@ static struct poptOption long_options[] = { {"no-fuzzy", 0, POPT_ARG_VAL, &fuzzy_basis, 0, 0, 0 }, {"no-y", 0, POPT_ARG_VAL, &fuzzy_basis, 0, 0, 0 }, {"compress", 'z', POPT_ARG_NONE, 0, 'z', 0, 0 }, + {"old-compress", 0, POPT_ARG_VAL, &do_compression, 1, 0, 0 }, + {"new-compress", 0, POPT_ARG_VAL, &do_compression, 2, 0, 0 }, {"no-compress", 0, POPT_ARG_VAL, &do_compression, 0, 0, 0 }, {"no-z", 0, POPT_ARG_VAL, &do_compression, 0, 0, 0 }, {"skip-compress", 0, POPT_ARG_STRING, &skip_compress, 0, 0, 0 }, - {"compress-level", 0, POPT_ARG_INT, &def_compress_level, 'z', 0, 0 }, + {"compress-level", 0, POPT_ARG_INT, &def_compress_level, 0, 0, 0 }, {0, 'P', POPT_ARG_NONE, 0, 'P', 0, 0 }, {"progress", 0, POPT_ARG_VAL, &do_progress, 1, 0, 0 }, {"no-progress", 0, POPT_ARG_VAL, &do_progress, 0, 0, 0 }, @@ -1543,18 +1547,7 @@ int parse_arguments(int *argc_p, const char ***argv_p) break; case 'z': - if (def_compress_level < Z_DEFAULT_COMPRESSION - || def_compress_level > Z_BEST_COMPRESSION) { - snprintf(err_buf, sizeof err_buf, - "--compress-level value is invalid: %d\n", - def_compress_level); - return 0; - } - do_compression = def_compress_level != Z_NO_COMPRESSION; - if (do_compression && refused_compress) { - create_refuse_error(refused_compress); - return 0; - } + do_compression++; break; case 'M': @@ -1829,6 +1822,33 @@ int parse_arguments(int *argc_p, const char ***argv_p) exit_cleanup(0); } + if (do_compression || def_compress_level != NOT_SPECIFIED) { + if (def_compress_level == NOT_SPECIFIED) + def_compress_level = Z_DEFAULT_COMPRESSION; + else if (def_compress_level < Z_DEFAULT_COMPRESSION || def_compress_level > Z_BEST_COMPRESSION) { + snprintf(err_buf, sizeof err_buf, "--compress-level value is invalid: %d\n", + def_compress_level); + return 0; + } else if (def_compress_level == Z_NO_COMPRESSION) + do_compression = 0; + else if (!do_compression) + do_compression = 1; + if (do_compression && refused_compress) { + create_refuse_error(refused_compress); + return 0; + } +#ifdef EXTERNAL_ZLIB + if (do_compression == 1) { + snprintf(err_buf, sizeof err_buf, + "This rsync lacks old-style --compress due to its external zlib. Try -zz.\n"); + if (am_server) + return 0; + fprintf(stderr, "%s" "Continuing without compression.\n\n", err_buf); + do_compression = 0; + } +#endif + } + #ifdef HAVE_SETVBUF if (outbuf_mode && !am_server) { int mode = *(uchar *)outbuf_mode; @@ -2451,7 +2471,7 @@ void server_options(char **args, int *argc_p) } if (sparse_files) argstr[x++] = 'S'; - if (do_compression) + if (do_compression == 1) argstr[x++] = 'z'; set_allow_inc_recurse(); @@ -2747,6 +2767,9 @@ void server_options(char **args, int *argc_p) exit_cleanup(RERR_MALLOC); } + if (do_compression > 1) + args[ac++] = "--new-compress"; + if (remote_option_cnt) { int j; if (ac + remote_option_cnt > MAX_SERVER_ARGS) { diff --git a/rsync.yo b/rsync.yo index eafa3854..c99d150c 100644 --- a/rsync.yo +++ b/rsync.yo @@ -1867,7 +1867,20 @@ being transmitted -- something that is useful over a slow connection. Note that this option typically achieves better compression ratios than can be achieved by using a compressing remote shell or a compressing transport because it takes advantage of the implicit information in the matching data -blocks that are not explicitly sent over the connection. +blocks that are not explicitly sent over the connection. This matching-data +compression comes at a cost of CPU, though, and can be disabled by repeating +the bf(-z) option, but only if both sides are at least version 3.1.1. + +Note that if your version of rsync was compiled with an external zlib (instead +of the zlib that comes packaged with rsync) then it will not support the +old-style compression, only the new-style (repeated-option) compression. In +the future this new-style compression will likely become the default. + +The client rsync requests new-style compression on the server via the +bf(--new-compress) option, so if you see that option rejected it means that +the server is not new enough to support bf(-zz). Rsync also accepts the +bf(--old-compress) option for a future time when new-style compression +becomes the default. See the bf(--skip-compress) option for the default list of file suffixes that will not be compressed. diff --git a/testsuite/daemon-gzip-download.test b/testsuite/daemon-gzip-download.test index 89a112ff..57dd820b 100644 --- a/testsuite/daemon-gzip-download.test +++ b/testsuite/daemon-gzip-download.test @@ -31,7 +31,7 @@ hands_setup # Build chkdir with a normal rsync and an --exclude. $RSYNC -av --exclude=foobar.baz "$fromdir/" "$chkdir/" -checkit "$RSYNC -avvvvz localhost::test-from/ '$todir/'" "$chkdir" "$todir" +checkit "$RSYNC -avvvvzz localhost::test-from/ '$todir/'" "$chkdir" "$todir" # The script would have aborted on error, so getting here means we've won. exit 0 diff --git a/testsuite/daemon-gzip-upload.test b/testsuite/daemon-gzip-upload.test index 11c52bad..b2110ea6 100644 --- a/testsuite/daemon-gzip-upload.test +++ b/testsuite/daemon-gzip-upload.test @@ -25,7 +25,7 @@ hands_setup # Build chkdir with a normal rsync and an --exclude. $RSYNC -av --exclude=foobar.baz "$fromdir/" "$chkdir/" -checkit "'$ignore23' $RSYNC -avvvvz '$fromdir/' localhost::test-to/" "$chkdir" "$todir" +checkit "'$ignore23' $RSYNC -avvvvzz '$fromdir/' localhost::test-to/" "$chkdir" "$todir" # The script would have aborted on error, so getting here means we've won. exit 0 diff --git a/token.c b/token.c index bd4d52fe..8cc55329 100644 --- a/token.c +++ b/token.c @@ -23,10 +23,6 @@ #include "itypes.h" #include -#ifndef Z_INSERT_ONLY -#define Z_INSERT_ONLY Z_SYNC_FLUSH -#endif - extern int do_compression; extern int protocol_version; extern int module_id; @@ -406,9 +402,10 @@ send_deflated_token(int f, int32 token, struct map_struct *buf, OFF_T offset, if (token == -1) { /* end of file - clean up */ write_byte(f, END_FLAG); - } else if (token != -2) { + } else if (token != -2 && do_compression == 1) { /* Add the data in the current block to the compressor's * history and hash table. */ +#ifndef EXTERNAL_ZLIB do { /* Break up long sections in the same way that * see_deflate_token() does. */ @@ -418,17 +415,20 @@ send_deflated_token(int f, int32 token, struct map_struct *buf, OFF_T offset, tx_strm.avail_in = n1; if (protocol_version >= 31) /* Newer protocols avoid a data-duplicating bug */ offset += n1; - do { - tx_strm.next_out = (Bytef *) obuf; - tx_strm.avail_out = AVAIL_OUT_SIZE(CHUNK_SIZE); - r = deflate(&tx_strm, Z_INSERT_ONLY); - if (r != Z_OK) { - rprintf(FERROR, "deflate on token returned %d (%d bytes left)\n", - r, tx_strm.avail_in); - exit_cleanup(RERR_STREAMIO); - } - } while (tx_strm.avail_in != 0); + tx_strm.next_out = (Bytef *) obuf; + tx_strm.avail_out = AVAIL_OUT_SIZE(CHUNK_SIZE); + r = deflate(&tx_strm, Z_INSERT_ONLY); + if (r != Z_OK || tx_strm.avail_in != 0) { + rprintf(FERROR, "deflate on token returned %d (%d bytes left)\n", + r, tx_strm.avail_in); + exit_cleanup(RERR_STREAMIO); + } } while (toklen > 0); +#else + toklen++; + rprintf(FERROR, "Impossible error in external-zlib code (1).\n"); + exit_cleanup(RERR_STREAMIO); +#endif } } @@ -579,6 +579,7 @@ static int32 recv_deflated_token(int f, char **data) */ static void see_deflate_token(char *buf, int32 len) { +#ifndef EXTERNAL_ZLIB int r; int32 blklen; unsigned char hdr[5]; @@ -616,6 +617,11 @@ static void see_deflate_token(char *buf, int32 len) exit_cleanup(RERR_STREAMIO); } } while (len || rx_strm.avail_out == 0); +#else + buf++; len++; + rprintf(FERROR, "Impossible error in external-zlib code (2).\n"); + exit_cleanup(RERR_STREAMIO); +#endif } /** @@ -655,6 +661,6 @@ int32 recv_token(int f, char **data) */ void see_token(char *data, int32 toklen) { - if (do_compression) + if (do_compression == 1) see_deflate_token(data, toklen); } -- cgit v1.2.3