mirror/rsync
1
0
Fork 0
mirror of https://github.com/RsyncProject/rsync.git synced 2026-05-14 09:55:51 -04:00
Code Issues Packages Projects Releases Wiki Activity

Reduce memory usage (#235)

Browse Source 
In 2004, an allocation optimization has been added to the file
list handling code, that preallocates 32k of file_struct pointers
in a file_list. This optimization predates the incremental
recursion feature, for which it is not appropriate anymore. When
copying a tree containing a large number of small directories,
using the incremental recursion, rsync allocates many short
file_lists. Suddenly, the unused file_struct pointers can easily
take 90-95% of the memory allocated by rsync.
This commit is contained in:
Jindřich Makovička
2021-10-01 21:04:59 +02:00
committed by GitHub
parent 3911c23866
commit ae1f002999
2 changed files with 12 additions and 4 deletions
Download Patch File Download Diff File Expand all files Collapse all files

11
flist.c
View File

@@ -295,6 +295,8 @@ static void flist_expand(struct file_list *flist, int extra)
flist->malloced = FLIST_START;
else if (flist->malloced >= FLIST_LINEAR)
flist->malloced += FLIST_LINEAR;
else if (flist->malloced < FLIST_START_LARGE/16)
flist->malloced *= 4;
else
flist->malloced *= 2;
@@ -305,7 +307,7 @@ static void flist_expand(struct file_list *flist, int extra)
new_ptr = realloc_array(flist->files, struct file_struct *, flist->malloced);
if (DEBUG_GTE(FLIST, 1) && flist->malloced != FLIST_START) {
if (DEBUG_GTE(FLIST, 1) && flist->files) {
rprintf(FCLIENT, "[%s] expand file_list pointer array to %s bytes, did%s move\n",
who_am_i(),
big_num(sizeof flist->files[0] * flist->malloced),
@@ -2186,8 +2188,10 @@ struct file_list *send_file_list(int f, int argc, char *argv[])
#endif
flist = cur_flist = flist_new(0, "send_file_list");
flist_expand(flist, FLIST_START_LARGE);
if (inc_recurse) {
dir_flist = flist_new(FLIST_TEMP, "send_file_list");
flist_expand(dir_flist, FLIST_START_LARGE);
flags |= FLAG_DIVERT_DIRS;
} else
dir_flist = cur_flist;
@@ -2541,10 +2545,13 @@ struct file_list *recv_file_list(int f, int dir_ndx)
#endif
flist = flist_new(0, "recv_file_list");
flist_expand(flist, FLIST_START_LARGE);
if (inc_recurse) {
if (flist->ndx_start == 1)
if (flist->ndx_start == 1) {
dir_flist = flist_new(FLIST_TEMP, "recv_file_list");
flist_expand(dir_flist, FLIST_START_LARGE);
}
dstart = dir_flist->used;
} else {
dir_flist = flist;

5
rsync.h
View File

@@ -918,8 +918,9 @@ extern int xattrs_ndx;
* Start the flist array at FLIST_START entries and grow it
* by doubling until FLIST_LINEAR then grow by FLIST_LINEAR
*/
#define FLIST_START (32 * 1024)
#define FLIST_LINEAR (FLIST_START * 512)
#define FLIST_START (32)
#define FLIST_START_LARGE (32 * 1024)
#define FLIST_LINEAR (FLIST_START_LARGE * 512)
/*
* Extent size for allocation pools: A minimum size of 128KB