added resend logic

added checksum on all files

checksum.c

@@ -83,7 +83,8 @@ void get_checksum2(char *buf,int len,char *sum)
   for(i = 0; i + CSUM_CHUNK <= len; i += CSUM_CHUNK) {
     MDupdate(&MD, buf1+i, CSUM_CHUNK*8);
   }
-  MDupdate(&MD, buf1+i, (len-i)*8);
+  if (len - i > 0)
+    MDupdate(&MD, buf1+i, (len-i)*8);
 
   sum_put(&MD,sum);
 }
@@ -112,8 +113,10 @@ void file_checksum(char *fname,char *sum,off_t size)
     MDupdate(&MD, tmpchunk, CSUM_CHUNK*8);
   }
 
-  bcopy(map_ptr(buf,i,len-i),tmpchunk,len-i);
-  MDupdate(&MD, tmpchunk, (len-i)*8);
+  if (len - i > 0) {
+    bcopy(map_ptr(buf,i,len-i),tmpchunk,len-i);
+    MDupdate(&MD, tmpchunk, (len-i)*8);
+  }
 
   sum_put(&MD,sum);
 
@@ -127,6 +130,53 @@ void checksum_init(void)
 }
 
 
+
+static MDstruct sumMD;
+static int sumresidue;
+static char sumrbuf[CSUM_CHUNK];
+
+void sum_init(void)
+{
+  MDbegin(&sumMD);
+  sumresidue=0;
+}
+
+void sum_update(char *p,int len)
+{
+  int i;
+  if (sumresidue) {
+    i = MIN(CSUM_CHUNK-sumresidue,len);
+    bcopy(p,sumrbuf+sumresidue,i);
+    MDupdate(&sumMD, sumrbuf, (i+sumresidue)*8);
+    len -= i;
+    p += i;
+  }
+
+  for(i = 0; i + CSUM_CHUNK <= len; i += CSUM_CHUNK) {
+    bcopy(p+i,sumrbuf,CSUM_CHUNK);
+    MDupdate(&sumMD, sumrbuf, CSUM_CHUNK*8);
+  }
+
+  if (len - i > 0) {
+    sumresidue = len-i;
+    bcopy(p+i,sumrbuf,sumresidue);
+  } else {
+    sumresidue = 0;    
+  }
+}
+
+void sum_end(char *sum)
+{
+  if (sumresidue)
+    MDupdate(&sumMD, sumrbuf, sumresidue*8);
+
+  SIVAL(sum,0,sumMD.buffer[0]);
+  SIVAL(sum,4,sumMD.buffer[1]);
+  SIVAL(sum,8,sumMD.buffer[2]);
+  SIVAL(sum,12,sumMD.buffer[3]);  
+}
+
+
 #ifdef CHECKSUM_MAIN
  int main(int argc,char *argv[])
 {

io.c

@@ -129,8 +129,8 @@ int read_int(int f)
   char b[4];
   if ((ret=readfd(f,b,4)) != 4) {
     if (verbose > 1) 
-      fprintf(FERROR,"Error reading %d bytes : %s\n",
-	      4,ret==-1?strerror(errno):"EOF");
+      fprintf(FERROR,"(%d) Error reading %d bytes : %s\n",
+	      getpid(),4,ret==-1?strerror(errno):"EOF");
     exit_cleanup(1);
   }
   total_read += 4;
@@ -142,8 +142,8 @@ void read_buf(int f,char *buf,int len)
   int ret;
   if ((ret=readfd(f,buf,len)) != len) {
     if (verbose > 1) 
-      fprintf(FERROR,"Error reading %d bytes : %s\n",
-	      len,ret==-1?strerror(errno):"EOF");
+      fprintf(FERROR,"(%d) Error reading %d bytes : %s\n",
+	      getpid(),len,ret==-1?strerror(errno):"EOF");
     exit_cleanup(1);
   }
   total_read += len;

match.c

@@ -24,6 +24,8 @@ extern int csum_length;
 extern int verbose;
 extern int am_server;
 
+extern int remote_version;
+
 typedef unsigned short tag;
 
 #define TABLESIZE (1<<16)
@@ -93,23 +95,53 @@ static void matched(int f,struct sum_struct *s,struct map_struct *buf,
 		    int offset,int i)
 {
   int n = offset - last_match;
-  
+  int j;
+
   if (verbose > 2)
     if (i != -1)
       fprintf(FERROR,"match at %d last_match=%d j=%d len=%d n=%d\n",
 	      (int)offset,(int)last_match,i,(int)s->sums[i].len,n);
 
-  send_token(f,i,buf,last_match,n);
-
+  send_token(f,i,buf,last_match,n,i==-1?0:s->sums[i].len);
   data_transfer += n;
 
-  if (i != -1)
-    last_match = offset + s->sums[i].len;
-  
-  if (i != -1)
-    last_match = offset + s->sums[i].len;
   if (n > 0)
     write_flush(f);
+
+  if (i != -1)
+    n += s->sums[i].len;
+  
+  for (j=0;j<n;j+=CHUNK_SIZE) {
+    int n1 = MIN(CHUNK_SIZE,n-j);
+    sum_update(map_ptr(buf,last_match+j,n1),n1);
+  }
+
+
+  if (i != -1)
+    last_match = offset + s->sums[i].len;
+
+}
+
+
+static inline char *window_ptr(struct map_struct *buf,int off,int len)
+{
+  static char *p=NULL;
+  static int p_len = 0;
+  static int p_off = 0;  
+
+  if (off == 0) {
+    p_off = 0;
+    p_len = CHUNK_SIZE;
+    p = map_ptr(buf,p_off,p_len);    
+  }
+
+  while (off+len > p_off+p_len) {
+    p_off += CHUNK_SIZE;
+    p_len = CHUNK_SIZE;
+    p = map_ptr(buf,p_off,p_len);  
+  }
+
+  return(p + (off-p_off));
 }
 
 
@@ -127,7 +159,7 @@ static void hash_search(int f,struct sum_struct *s,
 
   k = MIN(len, s->n);
 
-  map = map_ptr(buf,0,k);
+  map = window_ptr(buf,0,k);
 
   sum = get_checksum1(map, k);
   s1 = sum & 0xFFFF;
@@ -165,7 +197,7 @@ static void hash_search(int f,struct sum_struct *s,
 
 	  if (!done_csum2) {
 	    int l = MIN(s->n,len-offset);
-	    map = map_ptr(buf,offset,l);
+	    map = window_ptr(buf,offset,l);
 	    get_checksum2(map,l,sum2);
 	    done_csum2 = 1;
 	  }
@@ -173,7 +205,7 @@ static void hash_search(int f,struct sum_struct *s,
 	    matched(f,s,buf,len,offset,i);
 	    offset += s->sums[i].len - 1;
 	    k = MIN((len-offset), s->n);
-	    map = map_ptr(buf,offset,k);
+	    map = window_ptr(buf,offset,k);
 	    sum = get_checksum1(map, k);
 	    s1 = sum & 0xFFFF;
 	    s2 = sum >> 16;
@@ -188,7 +220,7 @@ static void hash_search(int f,struct sum_struct *s,
     }
 
     /* Trim off the first byte from the checksum */
-    map = map_ptr(buf,offset,k+1);
+    map = window_ptr(buf,offset,k+1);
     s1 -= map[0];
     s2 -= k * map[0];
 
@@ -203,17 +235,22 @@ static void hash_search(int f,struct sum_struct *s,
   } while (++offset < end);
 
   matched(f,s,buf,len,len,-1);
+  window_ptr(buf,len-1,1);
 }
 
 
 void match_sums(int f,struct sum_struct *s,struct map_struct *buf,off_t len)
 {
+  char file_sum[SUM_LENGTH];
+
   last_match = 0;
   false_alarms = 0;
   tag_hits = 0;
   matches=0;
   data_transfer=0;
 
+  sum_init();
+
   if (len > 0 && s->count>0) {
     build_hash_table(s);
 
@@ -228,6 +265,14 @@ void match_sums(int f,struct sum_struct *s,struct map_struct *buf,off_t len)
     matched(f,s,buf,len,len,-1);
   }
 
+  sum_end(file_sum);
+
+  if (remote_version >= 14) {
+    if (verbose > 2)
+      fprintf(FERROR,"sending file_sum\n");
+    write_buf(f,file_sum,SUM_LENGTH);
+  }
+
   if (targets) {
     free(targets);
     targets=NULL;

rsync.c

@@ -262,7 +262,7 @@ void recv_generator(char *fname,struct file_list *flist,int i,int f_out)
   struct file_struct *file = &flist->files[i];
 
   if (verbose > 2)
-    fprintf(FERROR,"recv_generator(%s)\n",fname);
+    fprintf(FERROR,"recv_generator(%s,%d)\n",fname,i);
 
   statret = lstat(fname,&st);
 
@@ -383,10 +383,13 @@ void recv_generator(char *fname,struct file_list *flist,int i,int f_out)
   }
 
   if (verbose > 3)
-    fprintf(FERROR,"mapped %s of size %d\n",fname,(int)st.st_size);
+    fprintf(FERROR,"gen mapped %s of size %d\n",fname,(int)st.st_size);
 
   s = generate_sums(buf,st.st_size,block_size);
 
+  if (verbose > 2)
+    fprintf(FERROR,"sending sums for %d\n",i);
+
   write_int(f_out,i);
   send_sums(s,f_out);
   write_flush(f_out);
@@ -399,22 +402,29 @@ void recv_generator(char *fname,struct file_list *flist,int i,int f_out)
 
 
 
-static void receive_data(int f_in,struct map_struct *buf,int fd,char *fname)
+static int receive_data(int f_in,struct map_struct *buf,int fd,char *fname)
 {
   int i,n,remainder,len,count;
   off_t offset = 0;
   off_t offset2;
   char *data;
+  static char file_sum1[SUM_LENGTH];
+  static char file_sum2[SUM_LENGTH];
+  char *map=NULL;
 
   count = read_int(f_in);
   n = read_int(f_in);
   remainder = read_int(f_in);
 
+  sum_init();
+
   for (i=recv_token(f_in,&data); i != 0; i=recv_token(f_in,&data)) {
     if (i > 0) {
       if (verbose > 3)
 	fprintf(FERROR,"data recv %d at %d\n",i,(int)offset);
 
+      sum_update(data,i);
+
       if (write_sparse(fd,data,i) != i) {
 	fprintf(FERROR,"write failed on %s : %s\n",fname,strerror(errno));
 	exit_cleanup(1);
@@ -431,7 +441,11 @@ static void receive_data(int f_in,struct map_struct *buf,int fd,char *fname)
 	fprintf(FERROR,"chunk[%d] of size %d at %d offset=%d\n",
 		i,len,(int)offset2,(int)offset);
 
-      if (write_sparse(fd,map_ptr(buf,offset2,len),len) != len) {
+      map = map_ptr(buf,offset2,len);
+
+      sum_update(map,len);
+
+      if (write_sparse(fd,map,len) != len) {
 	fprintf(FERROR,"write failed on %s : %s\n",fname,strerror(errno));
 	exit_cleanup(1);
       }
@@ -443,6 +457,17 @@ static void receive_data(int f_in,struct map_struct *buf,int fd,char *fname)
     fprintf(FERROR,"write failed on %s : %s\n",fname,strerror(errno));
     exit_cleanup(1);
   }
+
+  sum_end(file_sum1);
+
+  if (remote_version >= 14) {
+    read_buf(f_in,file_sum2,SUM_LENGTH);
+    if (verbose > 2)
+      fprintf(FERROR,"got file_sum\n");
+    if (memcmp(file_sum1,file_sum2,SUM_LENGTH) != 0)
+      return 0;
+  }
+  return 1;
 }
 
 
@@ -510,6 +535,7 @@ int recv_files(int f_in,struct file_list *flist,char *local_name,int f_gen)
   int i;
   struct file_struct *file;
   int phase=0;
+  int recv_ok;
 
   if (verbose > 2) {
     fprintf(FERROR,"recv_files(%d) starting\n",flist->count);
@@ -520,11 +546,14 @@ int recv_files(int f_in,struct file_list *flist,char *local_name,int f_gen)
   }
 
   while (1) 
-    {
+    {      
       i = read_int(f_in);
       if (i == -1) {
 	if (phase==0 && remote_version >= 13) {
 	  phase++;
+	  csum_length = SUM_LENGTH;
+	  if (verbose > 2)
+	    fprintf(FERROR,"recv_files phase=%d\n",phase);
 	  write_int(f_gen,-1);
 	  write_flush(f_gen);
 	  continue;
@@ -564,13 +593,12 @@ int recv_files(int f_in,struct file_list *flist,char *local_name,int f_gen)
 
       if (fd1 != -1 && st.st_size > 0) {
 	buf = map_file(fd1,st.st_size);
+	if (verbose > 2)
+	  fprintf(FERROR,"recv mapped %s of size %d\n",fname,(int)st.st_size);
       } else {
 	buf = NULL;
       }
 
-      if (verbose > 2)
-	fprintf(FERROR,"mapped %s of size %d\n",fname,(int)st.st_size);
-
       /* open tmp file */
       sprintf(fnametmp,"%s.XXXXXX",fname);
       if (NULL == mktemp(fnametmp)) {
@@ -589,7 +617,7 @@ int recv_files(int f_in,struct file_list *flist,char *local_name,int f_gen)
 	printf("%s\n",fname);
 
       /* recv file data */
-      receive_data(f_in,buf,fd2,fname);
+      recv_ok = receive_data(f_in,buf,fd2,fname);
 
       if (fd1 != -1) {
 	unmap_file(buf);
@@ -618,6 +646,12 @@ int recv_files(int f_in,struct file_list *flist,char *local_name,int f_gen)
       cleanup_fname = NULL;
 
       set_perms(fname,file,NULL,0);
+
+      if (!recv_ok) {
+	if (verbose > 1)
+	  fprintf(FERROR,"redoing %s(%d)\n",fname,i);
+	write_int(f_gen,i);
+      }
     }
 
   if (verbose > 2)
@@ -651,8 +685,11 @@ off_t send_files(struct file_list *flist,int f_out,int f_in)
       if (i == -1) {
 	if (phase==0 && remote_version >= 13) {
 	  phase++;
+	  csum_length = SUM_LENGTH;
 	  write_int(f_out,-1);
 	  write_flush(f_out);
+	  if (verbose > 2)
+	    fprintf(FERROR,"send_files phase=%d\n",phase);
 	  continue;
 	}
 	break;
@@ -748,6 +785,7 @@ off_t send_files(struct file_list *flist,int f_out,int f_in)
 void generate_files(int f,struct file_list *flist,char *local_name,int f_recv)
 {
   int i;
+  int phase=0;
 
   if (verbose > 2)
     fprintf(FERROR,"generator starting pid=%d count=%d\n",
@@ -769,19 +807,27 @@ void generate_files(int f,struct file_list *flist,char *local_name,int f_recv)
 		   flist,i,f);
   }
 
+  phase++;
+  csum_length = SUM_LENGTH;
+  ignore_times=1;
+
+  if (verbose > 2)
+    fprintf(FERROR,"generate_files phase=%d\n",phase);
+
   write_int(f,-1);
   write_flush(f);
 
   if (remote_version >= 13) {
-    /* go to the full checksum if anything has failed so far */
-    csum_length = SUM_LENGTH;
-
     for (i=read_int(f_recv); i != -1; i=read_int(f_recv)) {
       struct file_struct *file = &flist->files[i];
       recv_generator(local_name?local_name:file->name,
 		     flist,i,f);    
     }
 
+    phase++;
+    if (verbose > 2)
+      fprintf(FERROR,"generate_files phase=%d\n",phase);
+
     write_int(f,-1);
     write_flush(f);
   }

rsync.h

@@ -34,7 +34,7 @@
 #define SAME_TIME (1<<7)
 
 /* update this if you make incompatible changes */
-#define PROTOCOL_VERSION 13
+#define PROTOCOL_VERSION 14
 #define MIN_PROTOCOL_VERSION 10
 #define MAX_PROTOCOL_VERSION 20
 

token.c

@@ -73,7 +73,8 @@ static void simple_send_token(int f,int token,
  * If token == -1 then we have reached EOF 
  * If n == 0 then don't send a buffer
  */
-void send_token(int f,int token,struct map_struct *buf,int offset,int n)
+void send_token(int f,int token,struct map_struct *buf,int offset,
+		int n,int toklen)
 {
   if (!do_compression) {
     simple_send_token(f,token,buf,offset,n);