Update to use the new zlib-1.1.2 code.

The compressed token code now handles the null (-2)
token from the match logic.
This commit is contained in:
Paul Mackerras committed 1998-05-22 06:58:52 +00:00
1 parent 45f133b976
commit 5914bf15d2
7 files changed
+328 -249

No files matched your search

+5 -2
View File
@@ -20,11 +20,14 @@ SHELL=/bin/sh
.SUFFIXES:
.SUFFIXES: .c .o
LIBOBJ=lib/getopt.o lib/fnmatch.o lib/zlib.o lib/compat.o
LIBOBJ=lib/getopt.o lib/fnmatch.o lib/compat.o
ZLIBOBJ=zlib/deflate.o zlib/infblock.o zlib/infcodes.o zlib/inffast.o \
zlib/inflate.o zlib/inftrees.o zlib/infutil.o zlib/trees.o \
zlib/zutil.o zlib/adler32.o
OBJS1=rsync.o exclude.o util.o md4.o main.o checksum.o match.o syscall.o log.o
OBJS2=options.o flist.o io.o compat.o hlink.o token.o uidlist.o socket.o
DAEMON_OBJ = params.o loadparm.o clientserver.o access.o connection.o authenticate.o
OBJS=$(OBJS1) $(OBJS2) $(DAEMON_OBJ) $(LIBOBJ)
OBJS=$(OBJS1) $(OBJS2) $(DAEMON_OBJ) $(LIBOBJ) $(ZLIBOBJ)
# note that the -I. is needed to handle config.h when using VPATH
.c.o:
+1
View File
@@ -118,3 +118,4 @@ fi])
AC_OUTPUT(Makefile lib/dummy)
AC_OUTPUT(Makefile zlib/dummy)
+9 -15
View File
@@ -214,21 +214,15 @@ static void hash_search(int f,struct sum_struct *s,
--k;
}
if (!do_compression) {
/* By matching early we avoid re-reading the
data 3 times in the case where a token
match comes a long way after last
match. The 3 reads are caused by the
running match, the checksum update and the
literal send.
we don't enable this for the compressed
case yet as the deflated token code can't
handle it. Paul is working on it */
if (offset-last_match >= CHUNK_SIZE+s->n &&
(end-offset > CHUNK_SIZE)) {
matched(f,s,buf,offset - s->n, -2);
}
/* By matching early we avoid re-reading the
data 3 times in the case where a token
match comes a long way after last
match. The 3 reads are caused by the
running match, the checksum update and the
literal send. */
if (offset-last_match >= CHUNK_SIZE+s->n &&
(end-offset > CHUNK_SIZE)) {
matched(f,s,buf,offset - s->n, -2);
}
} while (++offset < end);
+284 -227
View File
@@ -18,7 +18,7 @@
*/
#include "rsync.h"
#include "lib/zlib.h"
#include "zlib/zlib.h"
extern int do_compression;
@@ -69,19 +69,6 @@ static void simple_send_token(int f,int token,
}
/* Memory allocation/freeing routines, called by zlib stuff. */
static void *
z_alloc(void *opaque, uInt items, uInt size)
{
return malloc(items * size);
}
static void
z_free(void *opaque, void *adrs, uInt nbytes)
{
free(adrs);
}
/* Flag bytes in compressed stream are encoded as follows: */
#define END_FLAG 0 /* that's all folks */
#define TOKEN_LONG 0x20 /* followed by 32-bit token number */
@@ -108,100 +95,129 @@ static void
send_deflated_token(int f, int token,
struct map_struct *buf, OFF_T offset, int nb, int toklen)
{
int n, r;
static int init_done;
int n, r;
static int init_done, flush_pending;
if (last_token == -1) {
/* initialization */
if (!init_done) {
tx_strm.next_in = NULL;
tx_strm.zalloc = z_alloc;
tx_strm.zfree = z_free;
if (deflateInit2(&tx_strm, Z_DEFAULT_COMPRESSION, 8,
-15, 8, Z_DEFAULT_STRATEGY) != Z_OK) {
rprintf(FERROR, "compression init failed\n");
exit_cleanup(1);
}
if ((obuf = malloc(MAX_DATA_COUNT+2)) == NULL)
out_of_memory("send_deflated_token");
init_done = 1;
} else
deflateReset(&tx_strm);
run_start = token;
last_run_end = 0;
if (last_token == -1) {
/* initialization */
if (!init_done) {
tx_strm.next_in = NULL;
tx_strm.zalloc = NULL;
tx_strm.zfree = NULL;
if (deflateInit2(&tx_strm, Z_DEFAULT_COMPRESSION,
Z_DEFLATED, -15, 8,
Z_DEFAULT_STRATEGY) != Z_OK) {
rprintf(FERROR, "compression init failed\n");
exit_cleanup(1);
}
if ((obuf = malloc(MAX_DATA_COUNT+2)) == NULL)
out_of_memory("send_deflated_token");
init_done = 1;
} else
deflateReset(&tx_strm);
last_run_end = 0;
run_start = token;
flush_pending = 0;
} else if (nb != 0 || token != last_token + 1
|| token >= run_start + 65536) {
/* output previous run */
r = run_start - last_run_end;
n = last_token - run_start;
if (r >= 0 && r <= 63) {
write_byte(f, (n==0? TOKEN_REL: TOKENRUN_REL) + r);
} else {
write_byte(f, (n==0? TOKEN_LONG: TOKENRUN_LONG));
write_int(f, run_start);
}
if (n != 0) {
write_byte(f, n);
write_byte(f, n >> 8);
}
last_run_end = last_token;
run_start = token;
}
} else if (last_token == -2) {
run_start = token;
last_token = token;
if (nb != 0) {
/* deflate the data starting at offset */
tx_strm.avail_in = 0;
tx_strm.avail_out = 0;
do {
if (tx_strm.avail_in == 0 && nb != 0) {
/* give it some more input */
n = MIN(nb, CHUNK_SIZE);
tx_strm.next_in = (Bytef *)map_ptr(buf, offset, n);
tx_strm.avail_in = n;
nb -= n;
offset += n;
}
if (tx_strm.avail_out == 0) {
tx_strm.next_out = (Bytef *)(obuf + 2);
tx_strm.avail_out = MAX_DATA_COUNT;
}
r = deflate(&tx_strm, nb? Z_NO_FLUSH: Z_PACKET_FLUSH);
if (r != Z_OK) {
rprintf(FERROR, "deflate returned %d\n", r);
exit_cleanup(1);
}
if (nb == 0 || tx_strm.avail_out == 0) {
n = MAX_DATA_COUNT - tx_strm.avail_out;
if (n > 0) {
obuf[0] = DEFLATED_DATA + (n >> 8);
obuf[1] = n;
write_buf(f, obuf, n+2);
} else if (nb != 0 || token != last_token + 1
|| token >= run_start + 65536) {
/* output previous run */
r = run_start - last_run_end;
n = last_token - run_start;
if (r >= 0 && r <= 63) {
write_byte(f, (n==0? TOKEN_REL: TOKENRUN_REL) + r);
} else {
write_byte(f, (n==0? TOKEN_LONG: TOKENRUN_LONG));
write_int(f, run_start);
}
}
} while (nb != 0 || tx_strm.avail_out == 0);
}
if (token != -1) {
/* add the data in the current block to the compressor's
history and hash table */
tx_strm.next_in = (Bytef *)map_ptr(buf, offset, toklen);
tx_strm.avail_in = toklen;
tx_strm.next_out = NULL;
tx_strm.avail_out = 2 * toklen;
r = deflate(&tx_strm, Z_INSERT_ONLY);
if (r != Z_OK || tx_strm.avail_in != 0) {
rprintf(FERROR, "deflate on token returned %d (%d bytes left)\n",
r, tx_strm.avail_in);
exit_cleanup(1);
if (n != 0) {
write_byte(f, n);
write_byte(f, n >> 8);
}
last_run_end = last_token;
run_start = token;
}
} else {
/* end of file - clean up */
write_byte(f, END_FLAG);
}
last_token = token;
if (nb != 0 || flush_pending) {
/* deflate the data starting at offset */
int flush = Z_NO_FLUSH;
tx_strm.avail_in = 0;
tx_strm.avail_out = 0;
do {
if (tx_strm.avail_in == 0 && nb != 0) {
/* give it some more input */
n = MIN(nb, CHUNK_SIZE);
tx_strm.next_in = (Bytef *)
map_ptr(buf, offset, n);
tx_strm.avail_in = n;
nb -= n;
offset += n;
}
if (tx_strm.avail_out == 0) {
tx_strm.next_out = (Bytef *)(obuf + 2);
tx_strm.avail_out = MAX_DATA_COUNT;
if (flush != Z_NO_FLUSH) {
/*
* We left the last 4 bytes in the
* buffer, in case they are the
* last 4. Move them to the front.
*/
memcpy(tx_strm.next_out,
obuf+MAX_DATA_COUNT-2, 4);
tx_strm.next_out += 4;
tx_strm.avail_out -= 4;
}
}
if (nb == 0 && token != -2)
flush = Z_SYNC_FLUSH;
r = deflate(&tx_strm, flush);
if (r != Z_OK) {
rprintf(FERROR, "deflate returned %d\n", r);
exit_cleanup(1);
}
if (nb == 0 || tx_strm.avail_out == 0) {
n = MAX_DATA_COUNT - tx_strm.avail_out;
if (flush != Z_NO_FLUSH) {
/*
* We have to trim off the last 4
* bytes of output when flushing
* (they are just 0, 0, ff, ff).
*/
n -= 4;
}
if (n > 0) {
obuf[0] = DEFLATED_DATA + (n >> 8);
obuf[1] = n;
write_buf(f, obuf, n+2);
}
}
} while (nb != 0 || tx_strm.avail_out == 0);
flush_pending = token == -2;
}
if (token == -1) {
/* end of file - clean up */
write_byte(f, END_FLAG);
} else if (token != -2) {
/* add the data in the current block to the compressor's
history and hash table */
tx_strm.next_in = (Bytef *) map_ptr(buf, offset, toklen);
tx_strm.avail_in = toklen;
tx_strm.next_out = (Bytef *) obuf;
tx_strm.avail_out = MAX_DATA_COUNT;
r = deflate(&tx_strm, Z_INSERT_ONLY);
if (r != Z_OK || tx_strm.avail_in != 0) {
rprintf(FERROR, "deflate on token returned %d (%d bytes left)\n",
r, tx_strm.avail_in);
exit_cleanup(1);
}
}
}
@@ -221,113 +237,129 @@ static int rx_run;
static int
recv_deflated_token(int f, char **data)
{
int n, r, flag;
static int init_done;
static int saved_flag;
int n, r, flag;
static int init_done;
static int saved_flag;
for (;;) {
switch (recv_state) {
case r_init:
if (!init_done) {
rx_strm.next_out = NULL;
rx_strm.zalloc = z_alloc;
rx_strm.zfree = z_free;
if (inflateInit2(&rx_strm, -15) != Z_OK) {
rprintf(FERROR, "inflate init failed\n");
exit_cleanup(1);
}
if ((cbuf = malloc(MAX_DATA_COUNT)) == NULL
|| (dbuf = malloc(CHUNK_SIZE)) == NULL)
out_of_memory("recv_deflated_token");
init_done = 1;
} else {
inflateReset(&rx_strm);
}
recv_state = r_idle;
rx_token = 0;
break;
case r_idle:
case r_inflated:
if (saved_flag) {
flag = saved_flag & 0xff;
saved_flag = 0;
} else
flag = read_byte(f);
if ((flag & 0xC0) == DEFLATED_DATA) {
n = ((flag & 0x3f) << 8) + read_byte(f);
read_buf(f, cbuf, n);
rx_strm.next_in = (Bytef *)cbuf;
rx_strm.avail_in = n;
recv_state = r_inflating;
break;
}
if (recv_state == r_inflated) {
/* check previous inflated stuff ended correctly */
rx_strm.avail_in = 0;
rx_strm.next_out = (Bytef *)dbuf;
rx_strm.avail_out = CHUNK_SIZE;
r = inflate(&rx_strm, Z_PACKET_FLUSH);
n = CHUNK_SIZE - rx_strm.avail_out;
if (r != Z_OK) {
rprintf(FERROR, "inflate flush returned %d (%d bytes)\n",
r, n);
exit_cleanup(1);
}
if (n != 0) {
/* have to return some more data and
save the flag for later. */
saved_flag = flag + 0x10000;
if (rx_strm.avail_out != 0)
for (;;) {
switch (recv_state) {
case r_init:
if (!init_done) {
rx_strm.next_out = NULL;
rx_strm.zalloc = NULL;
rx_strm.zfree = NULL;
if (inflateInit2(&rx_strm, -15) != Z_OK) {
rprintf(FERROR, "inflate init failed\n");
exit_cleanup(1);
}
if ((cbuf = malloc(MAX_DATA_COUNT)) == NULL
|| (dbuf = malloc(CHUNK_SIZE)) == NULL)
out_of_memory("recv_deflated_token");
init_done = 1;
} else {
inflateReset(&rx_strm);
}
recv_state = r_idle;
*data = dbuf;
return n;
rx_token = 0;
break;
case r_idle:
case r_inflated:
if (saved_flag) {
flag = saved_flag & 0xff;
saved_flag = 0;
} else
flag = read_byte(f);
if ((flag & 0xC0) == DEFLATED_DATA) {
n = ((flag & 0x3f) << 8) + read_byte(f);
read_buf(f, cbuf, n);
rx_strm.next_in = (Bytef *)cbuf;
rx_strm.avail_in = n;
recv_state = r_inflating;
break;
}
if (recv_state == r_inflated) {
/* check previous inflated stuff ended correctly */
rx_strm.avail_in = 0;
rx_strm.next_out = (Bytef *)dbuf;
rx_strm.avail_out = CHUNK_SIZE;
r = inflate(&rx_strm, Z_SYNC_FLUSH);
n = CHUNK_SIZE - rx_strm.avail_out;
/*
* Z_BUF_ERROR just means no progress was
* made, i.e. the decompressor didn't have
* any pending output for us.
*/
if (r != Z_OK && r != Z_BUF_ERROR) {
rprintf(FERROR, "inflate flush returned %d (%d bytes)\n",
r, n);
exit_cleanup(1);
}
if (n != 0 && r != Z_BUF_ERROR) {
/* have to return some more data and
save the flag for later. */
saved_flag = flag + 0x10000;
*data = dbuf;
return n;
}
/*
* At this point the decompressor should
* be expecting to see the 0, 0, ff, ff bytes.
*/
if (!inflateSyncPoint(&rx_strm)) {
rprintf(FERROR, "decompressor lost sync!\n");
exit_cleanup(1);
}
rx_strm.avail_in = 4;
rx_strm.next_in = (Bytef *)cbuf;
cbuf[0] = cbuf[1] = 0;
cbuf[2] = cbuf[3] = 0xff;
inflate(&rx_strm, Z_SYNC_FLUSH);
recv_state = r_idle;
}
if (flag == END_FLAG) {
/* that's all folks */
recv_state = r_init;
return 0;
}
/* here we have a token of some kind */
if (flag & TOKEN_REL) {
rx_token += flag & 0x3f;
flag >>= 6;
} else
rx_token = read_int(f);
if (flag & 1) {
rx_run = read_byte(f);
rx_run += read_byte(f) << 8;
recv_state = r_running;
}
return -1 - rx_token;
case r_inflating:
rx_strm.next_out = (Bytef *)dbuf;
rx_strm.avail_out = CHUNK_SIZE;
r = inflate(&rx_strm, Z_NO_FLUSH);
n = CHUNK_SIZE - rx_strm.avail_out;
if (r != Z_OK) {
rprintf(FERROR, "inflate returned %d (%d bytes)\n", r, n);
exit_cleanup(1);
}
if (rx_strm.avail_in == 0)
recv_state = r_inflated;
if (n != 0) {
*data = dbuf;
return n;
}
break;
case r_running:
++rx_token;
if (--rx_run == 0)
recv_state = r_idle;
return -1 - rx_token;
}
recv_state = r_idle;
}
if (flag == END_FLAG) {
/* that's all folks */
recv_state = r_init;
return 0;
}
/* here we have a token of some kind */
if (flag & TOKEN_REL) {
rx_token += flag & 0x3f;
flag >>= 6;
} else
rx_token = read_int(f);
if (flag & 1) {
rx_run = read_byte(f);
rx_run += read_byte(f) << 8;
recv_state = r_running;
}
return -1 - rx_token;
case r_inflating:
rx_strm.next_out = (Bytef *)dbuf;
rx_strm.avail_out = CHUNK_SIZE;
r = inflate(&rx_strm, Z_NO_FLUSH);
n = CHUNK_SIZE - rx_strm.avail_out;
if (r != Z_OK) {
rprintf(FERROR, "inflate returned %d (%d bytes)\n", r, n);
exit_cleanup(1);
}
if (rx_strm.avail_in == 0)
recv_state = r_inflated;
if (n != 0) {
*data = dbuf;
return n;
}
break;
case r_running:
++rx_token;
if (--rx_run == 0)
recv_state = r_idle;
return -1 - rx_token;
}
}
}
/*
@@ -337,15 +369,40 @@ recv_deflated_token(int f, char **data)
void
see_deflate_token(char *buf, int len)
{
int r;
int r, blklen;
unsigned char hdr[5];
rx_strm.next_in = (Bytef *)buf;
rx_strm.avail_in = len;
r = inflateIncomp(&rx_strm);
if (r != Z_OK) {
rprintf(FERROR, "inflateIncomp returned %d\n", r);
exit_cleanup(1);
}
rx_strm.avail_in = 0;
blklen = 0;
hdr[0] = 0;
do {
if (rx_strm.avail_in == 0 && len != 0) {
if (blklen == 0) {
/* Give it a fake stored-block header. */
rx_strm.next_in = (Bytef *)hdr;
rx_strm.avail_in = 5;
blklen = len;
if (blklen > 0xffff)
blklen = 0xffff;
hdr[1] = blklen;
hdr[2] = blklen >> 8;
hdr[3] = ~hdr[1];
hdr[4] = ~hdr[2];
} else {
rx_strm.next_in = (Bytef *)buf;
rx_strm.avail_in = blklen;
len -= blklen;
blklen = 0;
}
}
rx_strm.next_out = (Bytef *)dbuf;
rx_strm.avail_out = CHUNK_SIZE;
r = inflate(&rx_strm, Z_SYNC_FLUSH);
if (r != Z_OK) {
rprintf(FERROR, "inflate (token) returned %d\n", r);
exit_cleanup(1);
}
} while (len || rx_strm.avail_out == 0);
}
/*
@@ -356,11 +413,11 @@ see_deflate_token(char *buf, int len)
void send_token(int f,int token,struct map_struct *buf,OFF_T offset,
int n,int toklen)
{
if (!do_compression) {
simple_send_token(f,token,buf,offset,n);
} else {
send_deflated_token(f, token, buf, offset, n, toklen);
}
if (!do_compression) {
simple_send_token(f,token,buf,offset,n);
} else {
send_deflated_token(f, token, buf, offset, n, toklen);
}
}
@@ -372,14 +429,14 @@ void send_token(int f,int token,struct map_struct *buf,OFF_T offset,
*/
int recv_token(int f,char **data)
{
int tok;
int tok;
if (!do_compression) {
tok = simple_recv_token(f,data);
} else {
tok = recv_deflated_token(f, data);
}
return tok;
if (!do_compression) {
tok = simple_recv_token(f,data);
} else {
tok = recv_deflated_token(f, data);
}
return tok;
}
/*
@@ -387,6 +444,6 @@ int recv_token(int f,char **data)
*/
void see_token(char *data, int toklen)
{
if (do_compression)
see_deflate_token(data, toklen);
if (do_compression)
see_deflate_token(data, toklen);
}
+25 -5
View File
@@ -80,7 +80,7 @@ local block_state deflate_slow OF((deflate_state *s, int flush));
local void lm_init OF((deflate_state *s));
local void putShortMSB OF((deflate_state *s, uInt b));
local void flush_pending OF((z_streamp strm));
local int read_buf OF((z_streamp strm, Bytef *buf, unsigned size));
local int dread_buf OF((z_streamp strm, Bytef *buf, unsigned size));
#ifdef ASMV
void match_init OF((void)); /* asm code initialization */
uInt longest_match OF((deflate_state *s, IPos cur_match));
@@ -411,7 +411,7 @@ local void putShortMSB (s, b)
* Flush as much pending output as possible. All deflate() output goes
* through this function so some applications may wish to modify it
* to avoid allocating a large strm->next_out buffer and copying into it.
* (See also read_buf()).
* (See also dread_buf()).
*/
local void flush_pending(strm)
z_streamp strm;
@@ -441,7 +441,7 @@ int ZEXPORT deflate (strm, flush)
deflate_state *s;
if (strm == Z_NULL || strm->state == Z_NULL ||
flush > Z_FINISH || flush < 0) {
flush > Z_INSERT_ONLY || flush < 0) {
return Z_STREAM_ERROR;
}
s = strm->state;
@@ -657,7 +657,7 @@ int ZEXPORT deflateCopy (dest, source)
* allocating a large strm->next_in buffer and copying from it.
* (See also flush_pending()).
*/
local int read_buf(strm, buf, size)
local int dread_buf(strm, buf, size)
z_streamp strm;
Bytef *buf;
unsigned size;
@@ -1028,7 +1028,7 @@ local void fill_window(s)
*/
Assert(more >= 2, "more < 2");
n = read_buf(s->strm, s->window + s->strstart + s->lookahead, more);
n = dread_buf(s->strm, s->window + s->strstart + s->lookahead, more);
s->lookahead += n;
/* Initialize the hash value now that we have some input: */
@@ -1162,6 +1162,12 @@ local block_state deflate_fast(s, flush)
INSERT_STRING(s, s->strstart, hash_head);
}
if (flush == Z_INSERT_ONLY) {
s->strstart++;
s->lookahead--;
continue;
}
/* Find the longest match, discarding those <= prev_length.
* At this point we have always match_length < MIN_MATCH
*/
@@ -1221,6 +1227,10 @@ local block_state deflate_fast(s, flush)
}
if (bflush) FLUSH_BLOCK(s, 0);
}
if (flush == Z_INSERT_ONLY) {
s->block_start = s->strstart;
return need_more;
}
FLUSH_BLOCK(s, flush == Z_FINISH);
return flush == Z_FINISH ? finish_done : block_done;
}
@@ -1259,6 +1269,12 @@ local block_state deflate_slow(s, flush)
INSERT_STRING(s, s->strstart, hash_head);
}
if (flush == Z_INSERT_ONLY) {
s->strstart++;
s->lookahead--;
continue;
}
/* Find the longest match, discarding those <= prev_length.
*/
s->prev_length = s->match_length, s->prev_match = s->match_start;
@@ -1337,6 +1353,10 @@ local block_state deflate_slow(s, flush)
s->lookahead--;
}
}
if (flush == Z_INSERT_ONLY) {
s->block_start = s->strstart;
return need_more;
}
Assert (flush != Z_NO_FLUSH, "no flush?");
if (s->match_available) {
Tracevv((stderr,"%c", s->window[s->strstart-1]));
+1
View File
@@ -127,6 +127,7 @@ typedef z_stream FAR *z_streamp;
#define Z_SYNC_FLUSH 2
#define Z_FULL_FLUSH 3
#define Z_FINISH 4
#define Z_INSERT_ONLY 5
/* Allowed flush values; see deflate() below for details */
#define Z_OK 0
+3
View File
@@ -13,8 +13,10 @@
#ifndef _Z_UTIL_H
#define _Z_UTIL_H
#include "../rsync.h"
#include "zlib.h"
#if 0
#ifdef STDC
# include <stddef.h>
# include <string.h>
@@ -25,6 +27,7 @@
#else
# include <errno.h>
#endif
#endif
#ifndef local
# define local static