mirror of
https://github.com/kopia/kopia.git
synced 2026-01-23 22:07:54 -05:00
68b8afd43f
* feat(snapshots): improved performance when uploading huge files This is controlled by an upload policy which specifies the size threshold above which indvidual files are uploaded in parts and concatenated. This allows multiple threads to run splitting, hashing, compression and encryption in parallel, which was previously only possible across multiple files, but not when a single file was being uploaded. The default is 2GiB for now, so this feature only kicks in for very larger files. In the future we may lower this. Benchmark involved uploading a single 42.1 GB file which was a VM disk snapshot of fresh Ubuntu installation (fresh EXT4 partition with lots of zero bytes) to a brand-new filesystem repository on local SSD of M1 Pro Macbook Pro 2021. * before: 59-63s (~700 MB/s) * after: 15-17s (~2.6 GB/s) * additional test to ensure files are really e2e readable
51 lines
2.2 KiB
Go
51 lines
2.2 KiB
Go
package cli_test
|
|
|
|
import (
|
|
"testing"
|
|
|
|
"github.com/stretchr/testify/require"
|
|
|
|
"github.com/kopia/kopia/internal/testutil"
|
|
"github.com/kopia/kopia/tests/testenv"
|
|
)
|
|
|
|
func TestSetUploadPolicy(t *testing.T) {
|
|
e := testenv.NewCLITest(t, testenv.RepoFormatNotImportant, testenv.NewInProcRunner(t))
|
|
defer e.RunAndExpectSuccess(t, "repo", "disconnect")
|
|
|
|
e.RunAndExpectSuccess(t, "repo", "create", "filesystem", "--path", e.RepoDir)
|
|
|
|
lines := e.RunAndExpectSuccess(t, "policy", "show", "--global")
|
|
lines = compressSpaces(lines)
|
|
require.Contains(t, lines, " Max parallel snapshots (server/UI): 1 (defined for this target)")
|
|
require.Contains(t, lines, " Max parallel file reads: - (defined for this target)")
|
|
require.Contains(t, lines, " Parallel upload above size: 2 GiB (defined for this target)")
|
|
|
|
// make some directory we'll be setting policy on
|
|
td := testutil.TempDirectory(t)
|
|
|
|
lines = e.RunAndExpectSuccess(t, "policy", "show", td)
|
|
lines = compressSpaces(lines)
|
|
require.Contains(t, lines, " Max parallel snapshots (server/UI): 1 inherited from (global)")
|
|
require.Contains(t, lines, " Max parallel file reads: - inherited from (global)")
|
|
require.Contains(t, lines, " Parallel upload above size: 2 GiB inherited from (global)")
|
|
|
|
e.RunAndExpectSuccess(t, "policy", "set", "--global", "--max-parallel-snapshots=7", "--max-parallel-file-reads=33", "--parallel-upload-above-size-mib=4096")
|
|
|
|
lines = e.RunAndExpectSuccess(t, "policy", "show", td)
|
|
lines = compressSpaces(lines)
|
|
|
|
require.Contains(t, lines, " Max parallel snapshots (server/UI): 7 inherited from (global)")
|
|
require.Contains(t, lines, " Max parallel file reads: 33 inherited from (global)")
|
|
require.Contains(t, lines, " Parallel upload above size: 4 GiB inherited from (global)")
|
|
|
|
e.RunAndExpectSuccess(t, "policy", "set", "--global", "--max-parallel-snapshots=default", "--max-parallel-file-reads=default", "--parallel-upload-above-size-mib=default")
|
|
|
|
lines = e.RunAndExpectSuccess(t, "policy", "show", td)
|
|
lines = compressSpaces(lines)
|
|
|
|
require.Contains(t, lines, " Max parallel snapshots (server/UI): 1 inherited from (global)")
|
|
require.Contains(t, lines, " Max parallel file reads: - inherited from (global)")
|
|
require.Contains(t, lines, " Parallel upload above size: 2 GiB inherited from (global)")
|
|
}
|