From a97b89662dbd5898b59d65bfbc3a9b742757dc01 Mon Sep 17 00:00:00 2001 From: Jarek Kowalski Date: Sat, 7 Dec 2019 00:01:08 -0800 Subject: [PATCH] policy: added compression policy --- cli/command_policy_set.go | 64 +++++++++++++++++++++++++ cli/command_policy_show.go | 37 +++++++++++++++ snapshot/policy/compression_policy.go | 67 +++++++++++++++++++++++++++ snapshot/policy/policy.go | 13 ++++-- 4 files changed, 176 insertions(+), 5 deletions(-) create mode 100644 snapshot/policy/compression_policy.go diff --git a/cli/command_policy_set.go b/cli/command_policy_set.go index ffa4b82cd..b441f628b 100644 --- a/cli/command_policy_set.go +++ b/cli/command_policy_set.go @@ -9,6 +9,7 @@ import ( "github.com/pkg/errors" "github.com/kopia/kopia/repo" + "github.com/kopia/kopia/repo/object" "github.com/kopia/kopia/snapshot/policy" ) @@ -34,6 +35,19 @@ var ( policySetRemoveIgnore = policySetCommand.Flag("remove-ignore", "List of paths to remove from the ignore list").PlaceHolder("PATTERN").Strings() policySetClearIgnore = policySetCommand.Flag("clear-ignore", "Clear list of paths in the ignore list").Bool() + // Name of compression algorithm. + policySetCompressionAlgorithm = policySetCommand.Flag("compression", "Compression algorithm").Enum(supportedCompressionAlgorithms()...) + + // Files to only compress. + policySetAddOnlyCompress = policySetCommand.Flag("add-only-compress", "List of extensions to add to the only-compress list").PlaceHolder("PATTERN").Strings() + policySetRemoveOnlyCompress = policySetCommand.Flag("remove-only-compress", "List of extensions to remove from the only-compress list").PlaceHolder("PATTERN").Strings() + policySetClearOnlyCompress = policySetCommand.Flag("clear-only-compress", "Clear list of extensions in the only-compress list").Bool() + + // Files to never compress. + policySetAddNeverCompress = policySetCommand.Flag("add-never-compress", "List of extensions to add to the never compress list").PlaceHolder("PATTERN").Strings() + policySetRemoveNeverCompress = policySetCommand.Flag("remove-never-compress", "List of extensions to remove from the never compress list").PlaceHolder("PATTERN").Strings() + policySetClearNeverCompress = policySetCommand.Flag("clear-never-compress", "Clear list of extensions in the never compress list").Bool() + // Dot-ignore iles to look at. policySetAddDotIgnore = policySetCommand.Flag("add-dot-ignore", "List of paths to add to the dot-ignore list").PlaceHolder("FILENAME").Strings() policySetRemoveDotIgnore = policySetCommand.Flag("remove-dot-ignore", "List of paths to remove from the dot-ignore list").PlaceHolder("FILENAME").Strings() @@ -90,6 +104,7 @@ func setPolicyFromFlags(p *policy.Policy, changeCount *int) error { } setFilesPolicyFromFlags(&p.FilesPolicy, changeCount) + setCompressionPolicyFromFlags(&p.CompressionPolicy, changeCount) if err := setSchedulingPolicyFromFlags(&p.SchedulingPolicy, changeCount); err != nil { return errors.Wrap(err, "scheduling policy") @@ -195,6 +210,44 @@ func setSchedulingPolicyFromFlags(sp *policy.SchedulingPolicy, changeCount *int) return nil } +func setCompressionPolicyFromFlags(p *policy.CompressionPolicy, changeCount *int) { + if v := *policySetCompressionAlgorithm; v != "" { + *changeCount++ + + if v == inheritPolicyString { + printStderr(" - resetting compression algorithm to default value inherited from parent\n") + + p.CompressorName = "" + } else { + printStderr(" - setting compression algorithm to %v\n", v) + + p.CompressorName = object.CompressorName(v) + } + } + + if *policySetClearOnlyCompress { + *changeCount++ + + p.OnlyCompress = nil + + printStderr(" - removing all only-compress extensions\n") + } else { + p.OnlyCompress = addRemoveDedupeAndSort("only-compress extensions", + p.OnlyCompress, *policySetAddOnlyCompress, *policySetRemoveOnlyCompress, changeCount) + } + + if *policySetClearNeverCompress { + *changeCount++ + + p.NeverCompress = nil + + printStderr(" - removing all never-compress extensions\n") + } else { + p.NeverCompress = addRemoveDedupeAndSort("never-compress extensions", + p.NeverCompress, *policySetAddNeverCompress, *policySetRemoveNeverCompress, changeCount) + } +} + func addRemoveDedupeAndSort(desc string, base, add, remove []string, changeCount *int) []string { entries := map[string]bool{} for _, b := range base { @@ -278,3 +331,14 @@ func applyPolicyNumber64(desc string, val *int64, str string, changeCount *int) return nil } + +func supportedCompressionAlgorithms() []string { + var res []string + for name := range object.CompressorsByName { + res = append(res, string(name)) + } + + sort.Strings(res) + + return append([]string{inheritPolicyString}, res...) +} diff --git a/cli/command_policy_show.go b/cli/command_policy_show.go index 02873521a..d0da03113 100644 --- a/cli/command_policy_show.go +++ b/cli/command_policy_show.go @@ -81,6 +81,8 @@ func printPolicy(p *policy.Policy, parents []*policy.Policy) { printFilesPolicy(p, parents) printStdout("\n") printSchedulingPolicy(p, parents) + printStdout("\n") + printCompressionPolicy(p, parents) } func printRetentionPolicy(p *policy.Policy, parents []*policy.Policy) { @@ -178,6 +180,41 @@ func printSchedulingPolicy(p *policy.Policy, parents []*policy.Policy) { } } +func printCompressionPolicy(p *policy.Policy, parents []*policy.Policy) { + printStdout("Compression policy:\n") + + if p.CompressionPolicy.CompressorName != "" { + printStdout(" Compressor: %10v %v\n", p.CompressionPolicy.CompressorName, getDefinitionPoint(parents, func(pol *policy.Policy) bool { + return pol.CompressionPolicy.CompressorName != "" + })) + } + + switch { + case len(p.CompressionPolicy.OnlyCompress) > 0: + printStdout(" Only compress files with the following extensions:\n") + + for _, rule := range p.CompressionPolicy.OnlyCompress { + rule := rule + printStdout(" %-30v %v\n", rule, getDefinitionPoint(parents, func(pol *policy.Policy) bool { + return containsString(pol.CompressionPolicy.OnlyCompress, rule) + })) + } + + case len(p.CompressionPolicy.NeverCompress) > 0: + printStdout(" Try to compress all files except the following extensions:\n") + + for _, rule := range p.CompressionPolicy.NeverCompress { + rule := rule + printStdout(" %-30v %v\n", rule, getDefinitionPoint(parents, func(pol *policy.Policy) bool { + return containsString(pol.CompressionPolicy.NeverCompress, rule) + })) + } + + default: + printStdout(" Try to compress all files regardless of extensions.\n") + } +} + func valueOrNotSet(p *int) string { if p == nil { return "-" diff --git a/snapshot/policy/compression_policy.go b/snapshot/policy/compression_policy.go new file mode 100644 index 000000000..5810c0421 --- /dev/null +++ b/snapshot/policy/compression_policy.go @@ -0,0 +1,67 @@ +package policy + +import ( + "path/filepath" + "sort" + + "github.com/kopia/kopia/repo/object" +) + +// CompressionPolicy specifies compression policy. +type CompressionPolicy struct { + CompressorName object.CompressorName `json:"compressorName,omitempty"` + OnlyCompress []string `json:"onlyCompress"` + NeverCompress []string `json:"neverCompress"` +} + +func (p *CompressionPolicy) CompressorForFile(fname string) object.CompressorName { + ext := filepath.Ext(fname) + + if len(p.OnlyCompress) > 0 && isInSortedSlice(ext, p.OnlyCompress) { + return p.CompressorName + } + + if isInSortedSlice(ext, p.NeverCompress) { + return "" + } + + return p.CompressorName +} + +// Merge applies default values from the provided policy. +func (p *CompressionPolicy) Merge(src CompressionPolicy) { + if p.CompressorName == "" { + p.CompressorName = src.CompressorName + } + + p.OnlyCompress = mergeStrings(p.OnlyCompress, src.OnlyCompress) + p.NeverCompress = mergeStrings(p.NeverCompress, src.NeverCompress) +} + +var defaultCompressionPolicy = CompressionPolicy{} + +func mergeStrings(s1, s2 []string) []string { + merged := map[string]bool{} + + for _, v := range s1 { + merged[v] = true + } + + for _, v := range s2 { + merged[v] = true + } + + var result []string + for v := range merged { + result = append(result, v) + } + + sort.Strings(result) + + return result +} + +func isInSortedSlice(s string, slice []string) bool { + x := sort.SearchStrings(slice, s) + return x < len(slice) && slice[x] == s +} diff --git a/snapshot/policy/policy.go b/snapshot/policy/policy.go index a76738379..e7d4f0afb 100644 --- a/snapshot/policy/policy.go +++ b/snapshot/policy/policy.go @@ -13,11 +13,12 @@ var ErrPolicyNotFound = errors.New("policy not found") // Policy describes snapshot policy for a single source. type Policy struct { - Labels map[string]string `json:"-"` - RetentionPolicy RetentionPolicy `json:"retention,omitempty"` - FilesPolicy FilesPolicy `json:"files,omitempty"` - SchedulingPolicy SchedulingPolicy `json:"scheduling,omitempty"` - NoParent bool `json:"noParent,omitempty"` + Labels map[string]string `json:"-"` + RetentionPolicy RetentionPolicy `json:"retention,omitempty"` + FilesPolicy FilesPolicy `json:"files,omitempty"` + SchedulingPolicy SchedulingPolicy `json:"scheduling,omitempty"` + CompressionPolicy CompressionPolicy `json:"compression,omitempty"` + NoParent bool `json:"noParent,omitempty"` } func (p *Policy) String() string { @@ -59,12 +60,14 @@ func MergePolicies(policies []*Policy) *Policy { merged.RetentionPolicy.Merge(p.RetentionPolicy) merged.FilesPolicy.Merge(p.FilesPolicy) merged.SchedulingPolicy.Merge(p.SchedulingPolicy) + merged.CompressionPolicy.Merge(p.CompressionPolicy) } // Merge default expiration policy. merged.RetentionPolicy.Merge(defaultRetentionPolicy) merged.FilesPolicy.Merge(defaultFilesPolicy) merged.SchedulingPolicy.Merge(defaultSchedulingPolicy) + merged.CompressionPolicy.Merge(defaultCompressionPolicy) return &merged }