mirror of
https://github.com/kopia/kopia.git
synced 2026-01-24 06:18:02 -05:00
68b8afd43f
* feat(snapshots): improved performance when uploading huge files This is controlled by an upload policy which specifies the size threshold above which indvidual files are uploaded in parts and concatenated. This allows multiple threads to run splitting, hashing, compression and encryption in parallel, which was previously only possible across multiple files, but not when a single file was being uploaded. The default is 2GiB for now, so this feature only kicks in for very larger files. In the future we may lower this. Benchmark involved uploading a single 42.1 GB file which was a VM disk snapshot of fresh Ubuntu installation (fresh EXT4 partition with lots of zero bytes) to a brand-new filesystem repository on local SSD of M1 Pro Macbook Pro 2021. * before: 59-63s (~700 MB/s) * after: 15-17s (~2.6 GB/s) * additional test to ensure files are really e2e readable
38 lines
1.3 KiB
Go
38 lines
1.3 KiB
Go
package cli
|
|
|
|
import (
|
|
"context"
|
|
|
|
"github.com/alecthomas/kingpin"
|
|
|
|
"github.com/kopia/kopia/snapshot/policy"
|
|
)
|
|
|
|
type policyUploadFlags struct {
|
|
maxParallelUploads string
|
|
maxParallelFileReads string
|
|
parallelizeUploadAboveSizeMiB string
|
|
}
|
|
|
|
func (c *policyUploadFlags) setup(cmd *kingpin.CmdClause) {
|
|
cmd.Flag("max-parallel-file-reads", "Maximum number of parallel file reads").StringVar(&c.maxParallelFileReads)
|
|
cmd.Flag("max-parallel-snapshots", "Maximum number of parallel snapshots (server, KopiaUI only)").StringVar(&c.maxParallelUploads)
|
|
cmd.Flag("parallel-upload-above-size-mib", "Use parallel uploads above size").StringVar(&c.parallelizeUploadAboveSizeMiB)
|
|
}
|
|
|
|
func (c *policyUploadFlags) setUploadPolicyFromFlags(ctx context.Context, up *policy.UploadPolicy, changeCount *int) error {
|
|
if err := applyOptionalInt(ctx, "max parallel file reads", &up.MaxParallelFileReads, c.maxParallelFileReads, changeCount); err != nil {
|
|
return err
|
|
}
|
|
|
|
if err := applyOptionalInt(ctx, "max parallel snapshots", &up.MaxParallelSnapshots, c.maxParallelUploads, changeCount); err != nil {
|
|
return err
|
|
}
|
|
|
|
if err := applyOptionalInt64MiB(ctx, "parallel upload above size", &up.ParallelUploadAboveSize, c.parallelizeUploadAboveSizeMiB, changeCount); err != nil {
|
|
return err
|
|
}
|
|
|
|
return nil
|
|
}
|