mirror of
https://github.com/kopia/kopia.git
synced 2026-01-23 22:07:54 -05:00
* feat(snapshots): improved performance when uploading huge files This is controlled by an upload policy which specifies the size threshold above which indvidual files are uploaded in parts and concatenated. This allows multiple threads to run splitting, hashing, compression and encryption in parallel, which was previously only possible across multiple files, but not when a single file was being uploaded. The default is 2GiB for now, so this feature only kicks in for very larger files. In the future we may lower this. Benchmark involved uploading a single 42.1 GB file which was a VM disk snapshot of fresh Ubuntu installation (fresh EXT4 partition with lots of zero bytes) to a brand-new filesystem repository on local SSD of M1 Pro Macbook Pro 2021. * before: 59-63s (~700 MB/s) * after: 15-17s (~2.6 GB/s) * additional test to ensure files are really e2e readable
311 lines
6.8 KiB
Go
311 lines
6.8 KiB
Go
package cli
|
|
|
|
import (
|
|
"context"
|
|
"sort"
|
|
"strconv"
|
|
|
|
"github.com/pkg/errors"
|
|
|
|
"github.com/kopia/kopia/internal/units"
|
|
"github.com/kopia/kopia/repo"
|
|
"github.com/kopia/kopia/repo/compression"
|
|
"github.com/kopia/kopia/snapshot/policy"
|
|
)
|
|
|
|
type commandPolicySet struct {
|
|
policyTargetFlags
|
|
inherit []bool // not really a list, just an optional boolean
|
|
|
|
policyActionFlags
|
|
policyCompressionFlags
|
|
policyErrorFlags
|
|
policyFilesFlags
|
|
policyLoggingFlags
|
|
policyRetentionFlags
|
|
policySchedulingFlags
|
|
policyUploadFlags
|
|
}
|
|
|
|
func (c *commandPolicySet) setup(svc appServices, parent commandParent) {
|
|
cmd := parent.Command("set", "Set snapshot policy for a single directory, user@host or a global policy.")
|
|
c.policyTargetFlags.setup(cmd)
|
|
cmd.Flag(inheritPolicyString, "Enable or disable inheriting policies from the parent").BoolListVar(&c.inherit)
|
|
|
|
c.policyActionFlags.setup(cmd)
|
|
c.policyCompressionFlags.setup(cmd)
|
|
c.policyErrorFlags.setup(cmd)
|
|
c.policyFilesFlags.setup(cmd)
|
|
c.policyLoggingFlags.setup(cmd)
|
|
c.policyRetentionFlags.setup(cmd)
|
|
c.policySchedulingFlags.setup(cmd)
|
|
c.policyUploadFlags.setup(cmd)
|
|
|
|
cmd.Action(svc.repositoryWriterAction(c.run))
|
|
}
|
|
|
|
// nolint:gochecknoglobals
|
|
var booleanEnumValues = []string{"true", "false", "inherit"}
|
|
|
|
const (
|
|
inheritPolicyString = "inherit"
|
|
defaultPolicyString = "default"
|
|
)
|
|
|
|
func (c *commandPolicySet) run(ctx context.Context, rep repo.RepositoryWriter) error {
|
|
targets, err := c.policyTargets(ctx, rep)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
for _, target := range targets {
|
|
p, err := policy.GetDefinedPolicy(ctx, rep, target)
|
|
|
|
switch {
|
|
case errors.Is(err, policy.ErrPolicyNotFound):
|
|
p = &policy.Policy{}
|
|
case err != nil:
|
|
return errors.Wrap(err, "could not get defined policy")
|
|
}
|
|
|
|
log(ctx).Infof("Setting policy for %v", target)
|
|
|
|
changeCount := 0
|
|
if err := c.setPolicyFromFlags(ctx, p, &changeCount); err != nil {
|
|
return err
|
|
}
|
|
|
|
if changeCount == 0 {
|
|
return errors.New("no changes specified")
|
|
}
|
|
|
|
if err := policy.SetPolicy(ctx, rep, target, p); err != nil {
|
|
return errors.Wrapf(err, "can't save policy for %v", target)
|
|
}
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
func (c *commandPolicySet) setPolicyFromFlags(ctx context.Context, p *policy.Policy, changeCount *int) error {
|
|
if err := c.setRetentionPolicyFromFlags(ctx, &p.RetentionPolicy, changeCount); err != nil {
|
|
return errors.Wrap(err, "retention policy")
|
|
}
|
|
|
|
if err := c.setFilesPolicyFromFlags(ctx, &p.FilesPolicy, changeCount); err != nil {
|
|
return errors.Wrap(err, "files policy")
|
|
}
|
|
|
|
if err := c.setErrorHandlingPolicyFromFlags(ctx, &p.ErrorHandlingPolicy, changeCount); err != nil {
|
|
return errors.Wrap(err, "error handling policy")
|
|
}
|
|
|
|
if err := c.setCompressionPolicyFromFlags(ctx, &p.CompressionPolicy, changeCount); err != nil {
|
|
return errors.Wrap(err, "compression policy")
|
|
}
|
|
|
|
if err := c.setSchedulingPolicyFromFlags(ctx, &p.SchedulingPolicy, changeCount); err != nil {
|
|
return errors.Wrap(err, "scheduling policy")
|
|
}
|
|
|
|
if err := c.setActionsFromFlags(ctx, &p.Actions, changeCount); err != nil {
|
|
return errors.Wrap(err, "actions policy")
|
|
}
|
|
|
|
if err := c.setLoggingPolicyFromFlags(ctx, &p.LoggingPolicy, changeCount); err != nil {
|
|
return errors.Wrap(err, "actions policy")
|
|
}
|
|
|
|
if err := c.setUploadPolicyFromFlags(ctx, &p.UploadPolicy, changeCount); err != nil {
|
|
return errors.Wrap(err, "upload policy")
|
|
}
|
|
|
|
// It's not really a list, just optional boolean, last one wins.
|
|
for _, inherit := range c.inherit {
|
|
*changeCount++
|
|
|
|
p.NoParent = !inherit
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
func applyPolicyStringList(ctx context.Context, desc string, val *[]string, add, remove []string, clear bool, changeCount *int) {
|
|
if clear {
|
|
log(ctx).Infof(" - removing all from %q", desc)
|
|
|
|
*changeCount++
|
|
|
|
*val = nil
|
|
|
|
return
|
|
}
|
|
|
|
entries := map[string]bool{}
|
|
for _, b := range *val {
|
|
entries[b] = true
|
|
}
|
|
|
|
for _, b := range add {
|
|
*changeCount++
|
|
|
|
log(ctx).Infof(" - adding %q to %q", b, desc)
|
|
|
|
entries[b] = true
|
|
}
|
|
|
|
for _, b := range remove {
|
|
*changeCount++
|
|
|
|
log(ctx).Infof(" - removing %q from %q", b, desc)
|
|
delete(entries, b)
|
|
}
|
|
|
|
var s []string
|
|
for k := range entries {
|
|
s = append(s, k)
|
|
}
|
|
|
|
sort.Strings(s)
|
|
|
|
*val = s
|
|
}
|
|
|
|
func applyOptionalInt(ctx context.Context, desc string, val **policy.OptionalInt, str string, changeCount *int) error {
|
|
if str == "" {
|
|
// not changed
|
|
return nil
|
|
}
|
|
|
|
if str == inheritPolicyString || str == defaultPolicyString {
|
|
*changeCount++
|
|
|
|
log(ctx).Infof(" - resetting %q to a default value inherited from parent.", desc)
|
|
|
|
*val = nil
|
|
|
|
return nil
|
|
}
|
|
|
|
// nolint:gomnd
|
|
v, err := strconv.ParseInt(str, 10, 32)
|
|
if err != nil {
|
|
return errors.Wrapf(err, "can't parse the %v %q", desc, str)
|
|
}
|
|
|
|
i := policy.OptionalInt(v)
|
|
*changeCount++
|
|
|
|
log(ctx).Infof(" - setting %q to %v.", desc, i)
|
|
*val = &i
|
|
|
|
return nil
|
|
}
|
|
|
|
func applyOptionalInt64MiB(ctx context.Context, desc string, val **policy.OptionalInt64, str string, changeCount *int) error {
|
|
if str == "" {
|
|
// not changed
|
|
return nil
|
|
}
|
|
|
|
if str == inheritPolicyString || str == defaultPolicyString {
|
|
*changeCount++
|
|
|
|
log(ctx).Infof(" - resetting %q to a default value inherited from parent.", desc)
|
|
|
|
*val = nil
|
|
|
|
return nil
|
|
}
|
|
|
|
// nolint:gomnd
|
|
v, err := strconv.ParseInt(str, 10, 32)
|
|
if err != nil {
|
|
return errors.Wrapf(err, "can't parse the %v %q", desc, str)
|
|
}
|
|
|
|
// convert MiB to bytes
|
|
v *= 1 << 20 // nolint:gomnd
|
|
|
|
i := policy.OptionalInt64(v)
|
|
*changeCount++
|
|
|
|
log(ctx).Infof(" - setting %q to %v.", desc, units.BytesStringBase2(v))
|
|
|
|
*val = &i
|
|
|
|
return nil
|
|
}
|
|
|
|
func applyPolicyNumber64(ctx context.Context, desc string, val *int64, str string, changeCount *int) error {
|
|
if str == "" {
|
|
// not changed
|
|
return nil
|
|
}
|
|
|
|
if str == inheritPolicyString || str == defaultPolicyString {
|
|
*changeCount++
|
|
|
|
log(ctx).Infof(" - resetting %q to a default value inherited from parent.", desc)
|
|
|
|
*val = 0
|
|
|
|
return nil
|
|
}
|
|
|
|
// nolint:gomnd
|
|
v, err := strconv.ParseInt(str, 10, 64)
|
|
if err != nil {
|
|
return errors.Wrapf(err, "can't parse the %q %q", desc, str)
|
|
}
|
|
|
|
*changeCount++
|
|
|
|
log(ctx).Infof(" - setting %q to %v.", desc, v)
|
|
*val = v
|
|
|
|
return nil
|
|
}
|
|
|
|
func applyPolicyBoolPtr(ctx context.Context, desc string, val **policy.OptionalBool, str string, changeCount *int) error {
|
|
if str == "" {
|
|
// not changed
|
|
return nil
|
|
}
|
|
|
|
if str == inheritPolicyString || str == defaultPolicyString {
|
|
*changeCount++
|
|
|
|
log(ctx).Infof(" - resetting %q to a default value inherited from parent.", desc)
|
|
|
|
*val = nil
|
|
|
|
return nil
|
|
}
|
|
|
|
v, err := strconv.ParseBool(str)
|
|
if err != nil {
|
|
return errors.Wrapf(err, "can't parse the %q %q", desc, str)
|
|
}
|
|
|
|
*changeCount++
|
|
|
|
log(ctx).Infof(" - setting %q to %v.", desc, v)
|
|
|
|
ov := policy.OptionalBool(v)
|
|
*val = &ov
|
|
|
|
return nil
|
|
}
|
|
|
|
func supportedCompressionAlgorithms() []string {
|
|
var res []string
|
|
for name := range compression.ByName {
|
|
res = append(res, string(name))
|
|
}
|
|
|
|
sort.Strings(res)
|
|
|
|
return append([]string{inheritPolicyString, "none"}, res...)
|
|
}
|