diff options
| -rw-r--r-- | Makefile.in | 7 | ||||
| -rw-r--r-- | configure.in | 1 | ||||
| -rw-r--r-- | match.c | 24 | ||||
| -rw-r--r-- | token.c | 515 | ||||
| -rw-r--r-- | zlib/deflate.c | 30 | ||||
| -rw-r--r-- | zlib/zlib.h | 1 | ||||
| -rw-r--r-- | zlib/zutil.h | 3 |
7 files changed, 330 insertions, 251 deletions
diff --git a/Makefile.in b/Makefile.in index 81cc812b..1264c483 100644 --- a/Makefile.in +++ b/Makefile.in @@ -20,11 +20,14 @@ SHELL=/bin/sh .SUFFIXES: .SUFFIXES: .c .o -LIBOBJ=lib/getopt.o lib/fnmatch.o lib/zlib.o lib/compat.o +LIBOBJ=lib/getopt.o lib/fnmatch.o lib/compat.o +ZLIBOBJ=zlib/deflate.o zlib/infblock.o zlib/infcodes.o zlib/inffast.o \ + zlib/inflate.o zlib/inftrees.o zlib/infutil.o zlib/trees.o \ + zlib/zutil.o zlib/adler32.o OBJS1=rsync.o exclude.o util.o md4.o main.o checksum.o match.o syscall.o log.o OBJS2=options.o flist.o io.o compat.o hlink.o token.o uidlist.o socket.o DAEMON_OBJ = params.o loadparm.o clientserver.o access.o connection.o authenticate.o -OBJS=$(OBJS1) $(OBJS2) $(DAEMON_OBJ) $(LIBOBJ) +OBJS=$(OBJS1) $(OBJS2) $(DAEMON_OBJ) $(LIBOBJ) $(ZLIBOBJ) # note that the -I. is needed to handle config.h when using VPATH .c.o: diff --git a/configure.in b/configure.in index fdf417e1..97f1d49a 100644 --- a/configure.in +++ b/configure.in @@ -118,3 +118,4 @@ fi]) AC_OUTPUT(Makefile lib/dummy) +AC_OUTPUT(Makefile zlib/dummy) @@ -214,21 +214,15 @@ static void hash_search(int f,struct sum_struct *s, --k; } - if (!do_compression) { - /* By matching early we avoid re-reading the - data 3 times in the case where a token - match comes a long way after last - match. The 3 reads are caused by the - running match, the checksum update and the - literal send. - - we don't enable this for the compressed - case yet as the deflated token code can't - handle it. Paul is working on it */ - if (offset-last_match >= CHUNK_SIZE+s->n && - (end-offset > CHUNK_SIZE)) { - matched(f,s,buf,offset - s->n, -2); - } + /* By matching early we avoid re-reading the + data 3 times in the case where a token + match comes a long way after last + match. The 3 reads are caused by the + running match, the checksum update and the + literal send. */ + if (offset-last_match >= CHUNK_SIZE+s->n && + (end-offset > CHUNK_SIZE)) { + matched(f,s,buf,offset - s->n, -2); } } while (++offset < end); @@ -18,7 +18,7 @@ */ #include "rsync.h" -#include "lib/zlib.h" +#include "zlib/zlib.h" extern int do_compression; @@ -69,19 +69,6 @@ static void simple_send_token(int f,int token, } -/* Memory allocation/freeing routines, called by zlib stuff. */ -static void * -z_alloc(void *opaque, uInt items, uInt size) -{ - return malloc(items * size); -} - -static void -z_free(void *opaque, void *adrs, uInt nbytes) -{ - free(adrs); -} - /* Flag bytes in compressed stream are encoded as follows: */ #define END_FLAG 0 /* that's all folks */ #define TOKEN_LONG 0x20 /* followed by 32-bit token number */ @@ -108,100 +95,129 @@ static void send_deflated_token(int f, int token, struct map_struct *buf, OFF_T offset, int nb, int toklen) { - int n, r; - static int init_done; - - if (last_token == -1) { - /* initialization */ - if (!init_done) { - tx_strm.next_in = NULL; - tx_strm.zalloc = z_alloc; - tx_strm.zfree = z_free; - if (deflateInit2(&tx_strm, Z_DEFAULT_COMPRESSION, 8, - -15, 8, Z_DEFAULT_STRATEGY) != Z_OK) { - rprintf(FERROR, "compression init failed\n"); - exit_cleanup(1); - } - if ((obuf = malloc(MAX_DATA_COUNT+2)) == NULL) - out_of_memory("send_deflated_token"); - init_done = 1; - } else - deflateReset(&tx_strm); - run_start = token; - last_run_end = 0; - - } else if (nb != 0 || token != last_token + 1 - || token >= run_start + 65536) { - /* output previous run */ - r = run_start - last_run_end; - n = last_token - run_start; - if (r >= 0 && r <= 63) { - write_byte(f, (n==0? TOKEN_REL: TOKENRUN_REL) + r); - } else { - write_byte(f, (n==0? TOKEN_LONG: TOKENRUN_LONG)); - write_int(f, run_start); + int n, r; + static int init_done, flush_pending; + + if (last_token == -1) { + /* initialization */ + if (!init_done) { + tx_strm.next_in = NULL; + tx_strm.zalloc = NULL; + tx_strm.zfree = NULL; + if (deflateInit2(&tx_strm, Z_DEFAULT_COMPRESSION, + Z_DEFLATED, -15, 8, + Z_DEFAULT_STRATEGY) != Z_OK) { + rprintf(FERROR, "compression init failed\n"); + exit_cleanup(1); + } + if ((obuf = malloc(MAX_DATA_COUNT+2)) == NULL) + out_of_memory("send_deflated_token"); + init_done = 1; + } else + deflateReset(&tx_strm); + last_run_end = 0; + run_start = token; + flush_pending = 0; + + } else if (last_token == -2) { + run_start = token; + + } else if (nb != 0 || token != last_token + 1 + || token >= run_start + 65536) { + /* output previous run */ + r = run_start - last_run_end; + n = last_token - run_start; + if (r >= 0 && r <= 63) { + write_byte(f, (n==0? TOKEN_REL: TOKENRUN_REL) + r); + } else { + write_byte(f, (n==0? TOKEN_LONG: TOKENRUN_LONG)); + write_int(f, run_start); + } + if (n != 0) { + write_byte(f, n); + write_byte(f, n >> 8); + } + last_run_end = last_token; + run_start = token; } - if (n != 0) { - write_byte(f, n); - write_byte(f, n >> 8); + + last_token = token; + + if (nb != 0 || flush_pending) { + /* deflate the data starting at offset */ + int flush = Z_NO_FLUSH; + tx_strm.avail_in = 0; + tx_strm.avail_out = 0; + do { + if (tx_strm.avail_in == 0 && nb != 0) { + /* give it some more input */ + n = MIN(nb, CHUNK_SIZE); + tx_strm.next_in = (Bytef *) + map_ptr(buf, offset, n); + tx_strm.avail_in = n; + nb -= n; + offset += n; + } + if (tx_strm.avail_out == 0) { + tx_strm.next_out = (Bytef *)(obuf + 2); + tx_strm.avail_out = MAX_DATA_COUNT; + if (flush != Z_NO_FLUSH) { + /* + * We left the last 4 bytes in the + * buffer, in case they are the + * last 4. Move them to the front. + */ + memcpy(tx_strm.next_out, + obuf+MAX_DATA_COUNT-2, 4); + tx_strm.next_out += 4; + tx_strm.avail_out -= 4; + } + } + if (nb == 0 && token != -2) + flush = Z_SYNC_FLUSH; + r = deflate(&tx_strm, flush); + if (r != Z_OK) { + rprintf(FERROR, "deflate returned %d\n", r); + exit_cleanup(1); + } + if (nb == 0 || tx_strm.avail_out == 0) { + n = MAX_DATA_COUNT - tx_strm.avail_out; + if (flush != Z_NO_FLUSH) { + /* + * We have to trim off the last 4 + * bytes of output when flushing + * (they are just 0, 0, ff, ff). + */ + n -= 4; + } + if (n > 0) { + obuf[0] = DEFLATED_DATA + (n >> 8); + obuf[1] = n; + write_buf(f, obuf, n+2); + } + } + } while (nb != 0 || tx_strm.avail_out == 0); + flush_pending = token == -2; } - last_run_end = last_token; - run_start = token; - } - last_token = token; + if (token == -1) { + /* end of file - clean up */ + write_byte(f, END_FLAG); - if (nb != 0) { - /* deflate the data starting at offset */ - tx_strm.avail_in = 0; - tx_strm.avail_out = 0; - do { - if (tx_strm.avail_in == 0 && nb != 0) { - /* give it some more input */ - n = MIN(nb, CHUNK_SIZE); - tx_strm.next_in = (Bytef *)map_ptr(buf, offset, n); - tx_strm.avail_in = n; - nb -= n; - offset += n; - } - if (tx_strm.avail_out == 0) { - tx_strm.next_out = (Bytef *)(obuf + 2); + } else if (token != -2) { + /* add the data in the current block to the compressor's + history and hash table */ + tx_strm.next_in = (Bytef *) map_ptr(buf, offset, toklen); + tx_strm.avail_in = toklen; + tx_strm.next_out = (Bytef *) obuf; tx_strm.avail_out = MAX_DATA_COUNT; - } - r = deflate(&tx_strm, nb? Z_NO_FLUSH: Z_PACKET_FLUSH); - if (r != Z_OK) { - rprintf(FERROR, "deflate returned %d\n", r); - exit_cleanup(1); - } - if (nb == 0 || tx_strm.avail_out == 0) { - n = MAX_DATA_COUNT - tx_strm.avail_out; - if (n > 0) { - obuf[0] = DEFLATED_DATA + (n >> 8); - obuf[1] = n; - write_buf(f, obuf, n+2); + r = deflate(&tx_strm, Z_INSERT_ONLY); + if (r != Z_OK || tx_strm.avail_in != 0) { + rprintf(FERROR, "deflate on token returned %d (%d bytes left)\n", + r, tx_strm.avail_in); + exit_cleanup(1); } - } - } while (nb != 0 || tx_strm.avail_out == 0); - } - - if (token != -1) { - /* add the data in the current block to the compressor's - history and hash table */ - tx_strm.next_in = (Bytef *)map_ptr(buf, offset, toklen); - tx_strm.avail_in = toklen; - tx_strm.next_out = NULL; - tx_strm.avail_out = 2 * toklen; - r = deflate(&tx_strm, Z_INSERT_ONLY); - if (r != Z_OK || tx_strm.avail_in != 0) { - rprintf(FERROR, "deflate on token returned %d (%d bytes left)\n", - r, tx_strm.avail_in); - exit_cleanup(1); } - - } else { - /* end of file - clean up */ - write_byte(f, END_FLAG); - } } @@ -221,113 +237,129 @@ static int rx_run; static int recv_deflated_token(int f, char **data) { - int n, r, flag; - static int init_done; - static int saved_flag; - - for (;;) { - switch (recv_state) { - case r_init: - if (!init_done) { - rx_strm.next_out = NULL; - rx_strm.zalloc = z_alloc; - rx_strm.zfree = z_free; - if (inflateInit2(&rx_strm, -15) != Z_OK) { - rprintf(FERROR, "inflate init failed\n"); - exit_cleanup(1); - } - if ((cbuf = malloc(MAX_DATA_COUNT)) == NULL - || (dbuf = malloc(CHUNK_SIZE)) == NULL) - out_of_memory("recv_deflated_token"); - init_done = 1; - } else { - inflateReset(&rx_strm); - } - recv_state = r_idle; - rx_token = 0; - break; - - case r_idle: - case r_inflated: - if (saved_flag) { - flag = saved_flag & 0xff; - saved_flag = 0; - } else - flag = read_byte(f); - if ((flag & 0xC0) == DEFLATED_DATA) { - n = ((flag & 0x3f) << 8) + read_byte(f); - read_buf(f, cbuf, n); - rx_strm.next_in = (Bytef *)cbuf; - rx_strm.avail_in = n; - recv_state = r_inflating; - break; - } - if (recv_state == r_inflated) { - /* check previous inflated stuff ended correctly */ - rx_strm.avail_in = 0; - rx_strm.next_out = (Bytef *)dbuf; - rx_strm.avail_out = CHUNK_SIZE; - r = inflate(&rx_strm, Z_PACKET_FLUSH); - n = CHUNK_SIZE - rx_strm.avail_out; - if (r != Z_OK) { - rprintf(FERROR, "inflate flush returned %d (%d bytes)\n", - r, n); - exit_cleanup(1); - } - if (n != 0) { - /* have to return some more data and - save the flag for later. */ - saved_flag = flag + 0x10000; - if (rx_strm.avail_out != 0) + int n, r, flag; + static int init_done; + static int saved_flag; + + for (;;) { + switch (recv_state) { + case r_init: + if (!init_done) { + rx_strm.next_out = NULL; + rx_strm.zalloc = NULL; + rx_strm.zfree = NULL; + if (inflateInit2(&rx_strm, -15) != Z_OK) { + rprintf(FERROR, "inflate init failed\n"); + exit_cleanup(1); + } + if ((cbuf = malloc(MAX_DATA_COUNT)) == NULL + || (dbuf = malloc(CHUNK_SIZE)) == NULL) + out_of_memory("recv_deflated_token"); + init_done = 1; + } else { + inflateReset(&rx_strm); + } recv_state = r_idle; - *data = dbuf; - return n; + rx_token = 0; + break; + + case r_idle: + case r_inflated: + if (saved_flag) { + flag = saved_flag & 0xff; + saved_flag = 0; + } else + flag = read_byte(f); + if ((flag & 0xC0) == DEFLATED_DATA) { + n = ((flag & 0x3f) << 8) + read_byte(f); + read_buf(f, cbuf, n); + rx_strm.next_in = (Bytef *)cbuf; + rx_strm.avail_in = n; + recv_state = r_inflating; + break; + } + if (recv_state == r_inflated) { + /* check previous inflated stuff ended correctly */ + rx_strm.avail_in = 0; + rx_strm.next_out = (Bytef *)dbuf; + rx_strm.avail_out = CHUNK_SIZE; + r = inflate(&rx_strm, Z_SYNC_FLUSH); + n = CHUNK_SIZE - rx_strm.avail_out; + /* + * Z_BUF_ERROR just means no progress was + * made, i.e. the decompressor didn't have + * any pending output for us. + */ + if (r != Z_OK && r != Z_BUF_ERROR) { + rprintf(FERROR, "inflate flush returned %d (%d bytes)\n", + r, n); + exit_cleanup(1); + } + if (n != 0 && r != Z_BUF_ERROR) { + /* have to return some more data and + save the flag for later. */ + saved_flag = flag + 0x10000; + *data = dbuf; + return n; + } + /* + * At this point the decompressor should + * be expecting to see the 0, 0, ff, ff bytes. + */ + if (!inflateSyncPoint(&rx_strm)) { + rprintf(FERROR, "decompressor lost sync!\n"); + exit_cleanup(1); + } + rx_strm.avail_in = 4; + rx_strm.next_in = (Bytef *)cbuf; + cbuf[0] = cbuf[1] = 0; + cbuf[2] = cbuf[3] = 0xff; + inflate(&rx_strm, Z_SYNC_FLUSH); + recv_state = r_idle; + } + if (flag == END_FLAG) { + /* that's all folks */ + recv_state = r_init; + return 0; + } + + /* here we have a token of some kind */ + if (flag & TOKEN_REL) { + rx_token += flag & 0x3f; + flag >>= 6; + } else + rx_token = read_int(f); + if (flag & 1) { + rx_run = read_byte(f); + rx_run += read_byte(f) << 8; + recv_state = r_running; + } + return -1 - rx_token; + + case r_inflating: + rx_strm.next_out = (Bytef *)dbuf; + rx_strm.avail_out = CHUNK_SIZE; + r = inflate(&rx_strm, Z_NO_FLUSH); + n = CHUNK_SIZE - rx_strm.avail_out; + if (r != Z_OK) { + rprintf(FERROR, "inflate returned %d (%d bytes)\n", r, n); + exit_cleanup(1); + } + if (rx_strm.avail_in == 0) + recv_state = r_inflated; + if (n != 0) { + *data = dbuf; + return n; + } + break; + + case r_running: + ++rx_token; + if (--rx_run == 0) + recv_state = r_idle; + return -1 - rx_token; } - recv_state = r_idle; - } - if (flag == END_FLAG) { - /* that's all folks */ - recv_state = r_init; - return 0; - } - - /* here we have a token of some kind */ - if (flag & TOKEN_REL) { - rx_token += flag & 0x3f; - flag >>= 6; - } else - rx_token = read_int(f); - if (flag & 1) { - rx_run = read_byte(f); - rx_run += read_byte(f) << 8; - recv_state = r_running; - } - return -1 - rx_token; - - case r_inflating: - rx_strm.next_out = (Bytef *)dbuf; - rx_strm.avail_out = CHUNK_SIZE; - r = inflate(&rx_strm, Z_NO_FLUSH); - n = CHUNK_SIZE - rx_strm.avail_out; - if (r != Z_OK) { - rprintf(FERROR, "inflate returned %d (%d bytes)\n", r, n); - exit_cleanup(1); - } - if (rx_strm.avail_in == 0) - recv_state = r_inflated; - if (n != 0) { - *data = dbuf; - return n; - } - break; - - case r_running: - ++rx_token; - if (--rx_run == 0) - recv_state = r_idle; - return -1 - rx_token; } - } } /* @@ -337,15 +369,40 @@ recv_deflated_token(int f, char **data) void see_deflate_token(char *buf, int len) { - int r; - - rx_strm.next_in = (Bytef *)buf; - rx_strm.avail_in = len; - r = inflateIncomp(&rx_strm); - if (r != Z_OK) { - rprintf(FERROR, "inflateIncomp returned %d\n", r); - exit_cleanup(1); - } + int r, blklen; + unsigned char hdr[5]; + + rx_strm.avail_in = 0; + blklen = 0; + hdr[0] = 0; + do { + if (rx_strm.avail_in == 0 && len != 0) { + if (blklen == 0) { + /* Give it a fake stored-block header. */ + rx_strm.next_in = (Bytef *)hdr; + rx_strm.avail_in = 5; + blklen = len; + if (blklen > 0xffff) + blklen = 0xffff; + hdr[1] = blklen; + hdr[2] = blklen >> 8; + hdr[3] = ~hdr[1]; + hdr[4] = ~hdr[2]; + } else { + rx_strm.next_in = (Bytef *)buf; + rx_strm.avail_in = blklen; + len -= blklen; + blklen = 0; + } + } + rx_strm.next_out = (Bytef *)dbuf; + rx_strm.avail_out = CHUNK_SIZE; + r = inflate(&rx_strm, Z_SYNC_FLUSH); + if (r != Z_OK) { + rprintf(FERROR, "inflate (token) returned %d\n", r); + exit_cleanup(1); + } + } while (len || rx_strm.avail_out == 0); } /* @@ -356,11 +413,11 @@ see_deflate_token(char *buf, int len) void send_token(int f,int token,struct map_struct *buf,OFF_T offset, int n,int toklen) { - if (!do_compression) { - simple_send_token(f,token,buf,offset,n); - } else { - send_deflated_token(f, token, buf, offset, n, toklen); - } + if (!do_compression) { + simple_send_token(f,token,buf,offset,n); + } else { + send_deflated_token(f, token, buf, offset, n, toklen); + } } @@ -372,14 +429,14 @@ void send_token(int f,int token,struct map_struct *buf,OFF_T offset, */ int recv_token(int f,char **data) { - int tok; - - if (!do_compression) { - tok = simple_recv_token(f,data); - } else { - tok = recv_deflated_token(f, data); - } - return tok; + int tok; + + if (!do_compression) { + tok = simple_recv_token(f,data); + } else { + tok = recv_deflated_token(f, data); + } + return tok; } /* @@ -387,6 +444,6 @@ int recv_token(int f,char **data) */ void see_token(char *data, int toklen) { - if (do_compression) - see_deflate_token(data, toklen); + if (do_compression) + see_deflate_token(data, toklen); } diff --git a/zlib/deflate.c b/zlib/deflate.c index 490813fe..95b84d0d 100644 --- a/zlib/deflate.c +++ b/zlib/deflate.c @@ -80,7 +80,7 @@ local block_state deflate_slow OF((deflate_state *s, int flush)); local void lm_init OF((deflate_state *s)); local void putShortMSB OF((deflate_state *s, uInt b)); local void flush_pending OF((z_streamp strm)); -local int read_buf OF((z_streamp strm, Bytef *buf, unsigned size)); +local int dread_buf OF((z_streamp strm, Bytef *buf, unsigned size)); #ifdef ASMV void match_init OF((void)); /* asm code initialization */ uInt longest_match OF((deflate_state *s, IPos cur_match)); @@ -411,7 +411,7 @@ local void putShortMSB (s, b) * Flush as much pending output as possible. All deflate() output goes * through this function so some applications may wish to modify it * to avoid allocating a large strm->next_out buffer and copying into it. - * (See also read_buf()). + * (See also dread_buf()). */ local void flush_pending(strm) z_streamp strm; @@ -441,7 +441,7 @@ int ZEXPORT deflate (strm, flush) deflate_state *s; if (strm == Z_NULL || strm->state == Z_NULL || - flush > Z_FINISH || flush < 0) { + flush > Z_INSERT_ONLY || flush < 0) { return Z_STREAM_ERROR; } s = strm->state; @@ -657,7 +657,7 @@ int ZEXPORT deflateCopy (dest, source) * allocating a large strm->next_in buffer and copying from it. * (See also flush_pending()). */ -local int read_buf(strm, buf, size) +local int dread_buf(strm, buf, size) z_streamp strm; Bytef *buf; unsigned size; @@ -1028,7 +1028,7 @@ local void fill_window(s) */ Assert(more >= 2, "more < 2"); - n = read_buf(s->strm, s->window + s->strstart + s->lookahead, more); + n = dread_buf(s->strm, s->window + s->strstart + s->lookahead, more); s->lookahead += n; /* Initialize the hash value now that we have some input: */ @@ -1162,6 +1162,12 @@ local block_state deflate_fast(s, flush) INSERT_STRING(s, s->strstart, hash_head); } + if (flush == Z_INSERT_ONLY) { + s->strstart++; + s->lookahead--; + continue; + } + /* Find the longest match, discarding those <= prev_length. * At this point we have always match_length < MIN_MATCH */ @@ -1221,6 +1227,10 @@ local block_state deflate_fast(s, flush) } if (bflush) FLUSH_BLOCK(s, 0); } + if (flush == Z_INSERT_ONLY) { + s->block_start = s->strstart; + return need_more; + } FLUSH_BLOCK(s, flush == Z_FINISH); return flush == Z_FINISH ? finish_done : block_done; } @@ -1259,6 +1269,12 @@ local block_state deflate_slow(s, flush) INSERT_STRING(s, s->strstart, hash_head); } + if (flush == Z_INSERT_ONLY) { + s->strstart++; + s->lookahead--; + continue; + } + /* Find the longest match, discarding those <= prev_length. */ s->prev_length = s->match_length, s->prev_match = s->match_start; @@ -1337,6 +1353,10 @@ local block_state deflate_slow(s, flush) s->lookahead--; } } + if (flush == Z_INSERT_ONLY) { + s->block_start = s->strstart; + return need_more; + } Assert (flush != Z_NO_FLUSH, "no flush?"); if (s->match_available) { Tracevv((stderr,"%c", s->window[s->strstart-1])); diff --git a/zlib/zlib.h b/zlib/zlib.h index 787b5fd9..d0edd792 100644 --- a/zlib/zlib.h +++ b/zlib/zlib.h @@ -127,6 +127,7 @@ typedef z_stream FAR *z_streamp; #define Z_SYNC_FLUSH 2 #define Z_FULL_FLUSH 3 #define Z_FINISH 4 +#define Z_INSERT_ONLY 5 /* Allowed flush values; see deflate() below for details */ #define Z_OK 0 diff --git a/zlib/zutil.h b/zlib/zutil.h index 5e403d1d..b11655be 100644 --- a/zlib/zutil.h +++ b/zlib/zutil.h @@ -13,8 +13,10 @@ #ifndef _Z_UTIL_H #define _Z_UTIL_H +#include "../rsync.h" #include "zlib.h" +#if 0 #ifdef STDC # include <stddef.h> # include <string.h> @@ -25,6 +27,7 @@ #else # include <errno.h> #endif +#endif #ifndef local # define local static |