policy: added compression policy

cli/command_policy_set.go

@@ -9,6 +9,7 @@ import (
 	"github.com/pkg/errors"
 
 	"github.com/kopia/kopia/repo"
+	"github.com/kopia/kopia/repo/object"
 	"github.com/kopia/kopia/snapshot/policy"
 )
 
@@ -34,6 +35,19 @@ var (
 	policySetRemoveIgnore = policySetCommand.Flag("remove-ignore", "List of paths to remove from the ignore list").PlaceHolder("PATTERN").Strings()
 	policySetClearIgnore  = policySetCommand.Flag("clear-ignore", "Clear list of paths in the ignore list").Bool()
 
+	// Name of compression algorithm.
+	policySetCompressionAlgorithm = policySetCommand.Flag("compression", "Compression algorithm").Enum(supportedCompressionAlgorithms()...)
+
+	// Files to only compress.
+	policySetAddOnlyCompress    = policySetCommand.Flag("add-only-compress", "List of extensions to add to the only-compress list").PlaceHolder("PATTERN").Strings()
+	policySetRemoveOnlyCompress = policySetCommand.Flag("remove-only-compress", "List of extensions to remove from the only-compress list").PlaceHolder("PATTERN").Strings()
+	policySetClearOnlyCompress  = policySetCommand.Flag("clear-only-compress", "Clear list of extensions in the only-compress list").Bool()
+
+	// Files to never compress.
+	policySetAddNeverCompress    = policySetCommand.Flag("add-never-compress", "List of extensions to add to the never compress list").PlaceHolder("PATTERN").Strings()
+	policySetRemoveNeverCompress = policySetCommand.Flag("remove-never-compress", "List of extensions to remove from the never compress list").PlaceHolder("PATTERN").Strings()
+	policySetClearNeverCompress  = policySetCommand.Flag("clear-never-compress", "Clear list of extensions in the never compress list").Bool()
+
 	// Dot-ignore iles to look at.
 	policySetAddDotIgnore    = policySetCommand.Flag("add-dot-ignore", "List of paths to add to the dot-ignore list").PlaceHolder("FILENAME").Strings()
 	policySetRemoveDotIgnore = policySetCommand.Flag("remove-dot-ignore", "List of paths to remove from the dot-ignore list").PlaceHolder("FILENAME").Strings()
@@ -90,6 +104,7 @@ func setPolicyFromFlags(p *policy.Policy, changeCount *int) error {
 	}
 
 	setFilesPolicyFromFlags(&p.FilesPolicy, changeCount)
+	setCompressionPolicyFromFlags(&p.CompressionPolicy, changeCount)
 
 	if err := setSchedulingPolicyFromFlags(&p.SchedulingPolicy, changeCount); err != nil {
 		return errors.Wrap(err, "scheduling policy")
@@ -195,6 +210,44 @@ func setSchedulingPolicyFromFlags(sp *policy.SchedulingPolicy, changeCount *int)
 	return nil
 }
 
+func setCompressionPolicyFromFlags(p *policy.CompressionPolicy, changeCount *int) {
+	if v := *policySetCompressionAlgorithm; v != "" {
+		*changeCount++
+
+		if v == inheritPolicyString {
+			printStderr(" - resetting compression algorithm to default value inherited from parent\n")
+
+			p.CompressorName = ""
+		} else {
+			printStderr(" - setting compression algorithm to %v\n", v)
+
+			p.CompressorName = object.CompressorName(v)
+		}
+	}
+
+	if *policySetClearOnlyCompress {
+		*changeCount++
+
+		p.OnlyCompress = nil
+
+		printStderr(" - removing all only-compress extensions\n")
+	} else {
+		p.OnlyCompress = addRemoveDedupeAndSort("only-compress extensions",
+			p.OnlyCompress, *policySetAddOnlyCompress, *policySetRemoveOnlyCompress, changeCount)
+	}
+
+	if *policySetClearNeverCompress {
+		*changeCount++
+
+		p.NeverCompress = nil
+
+		printStderr(" - removing all never-compress extensions\n")
+	} else {
+		p.NeverCompress = addRemoveDedupeAndSort("never-compress extensions",
+			p.NeverCompress, *policySetAddNeverCompress, *policySetRemoveNeverCompress, changeCount)
+	}
+}
+
 func addRemoveDedupeAndSort(desc string, base, add, remove []string, changeCount *int) []string {
 	entries := map[string]bool{}
 	for _, b := range base {
@@ -278,3 +331,14 @@ func applyPolicyNumber64(desc string, val *int64, str string, changeCount *int)
 
 	return nil
 }
+
+func supportedCompressionAlgorithms() []string {
+	var res []string
+	for name := range object.CompressorsByName {
+		res = append(res, string(name))
+	}
+
+	sort.Strings(res)
+
+	return append([]string{inheritPolicyString}, res...)
+}

cli/command_policy_show.go

@@ -81,6 +81,8 @@ func printPolicy(p *policy.Policy, parents []*policy.Policy) {
 	printFilesPolicy(p, parents)
 	printStdout("\n")
 	printSchedulingPolicy(p, parents)
+	printStdout("\n")
+	printCompressionPolicy(p, parents)
 }
 
 func printRetentionPolicy(p *policy.Policy, parents []*policy.Policy) {
@@ -178,6 +180,41 @@ func printSchedulingPolicy(p *policy.Policy, parents []*policy.Policy) {
 	}
 }
 
+func printCompressionPolicy(p *policy.Policy, parents []*policy.Policy) {
+	printStdout("Compression policy:\n")
+
+	if p.CompressionPolicy.CompressorName != "" {
+		printStdout("  Compressor:     %10v  %v\n", p.CompressionPolicy.CompressorName, getDefinitionPoint(parents, func(pol *policy.Policy) bool {
+			return pol.CompressionPolicy.CompressorName != ""
+		}))
+	}
+
+	switch {
+	case len(p.CompressionPolicy.OnlyCompress) > 0:
+		printStdout("  Only compress files with the following extensions:\n")
+
+		for _, rule := range p.CompressionPolicy.OnlyCompress {
+			rule := rule
+			printStdout("    %-30v %v\n", rule, getDefinitionPoint(parents, func(pol *policy.Policy) bool {
+				return containsString(pol.CompressionPolicy.OnlyCompress, rule)
+			}))
+		}
+
+	case len(p.CompressionPolicy.NeverCompress) > 0:
+		printStdout("  Try to compress all files except the following extensions:\n")
+
+		for _, rule := range p.CompressionPolicy.NeverCompress {
+			rule := rule
+			printStdout("    %-30v %v\n", rule, getDefinitionPoint(parents, func(pol *policy.Policy) bool {
+				return containsString(pol.CompressionPolicy.NeverCompress, rule)
+			}))
+		}
+
+	default:
+		printStdout("  Try to compress all files regardless of extensions.\n")
+	}
+}
+
 func valueOrNotSet(p *int) string {
 	if p == nil {
 		return "-"

snapshot/policy/compression_policy.go

@@ -0,0 +1,67 @@
+package policy
+
+import (
+	"path/filepath"
+	"sort"
+
+	"github.com/kopia/kopia/repo/object"
+)
+
+// CompressionPolicy specifies compression policy.
+type CompressionPolicy struct {
+	CompressorName object.CompressorName `json:"compressorName,omitempty"`
+	OnlyCompress   []string              `json:"onlyCompress"`
+	NeverCompress  []string              `json:"neverCompress"`
+}
+
+func (p *CompressionPolicy) CompressorForFile(fname string) object.CompressorName {
+	ext := filepath.Ext(fname)
+
+	if len(p.OnlyCompress) > 0 && isInSortedSlice(ext, p.OnlyCompress) {
+		return p.CompressorName
+	}
+
+	if isInSortedSlice(ext, p.NeverCompress) {
+		return ""
+	}
+
+	return p.CompressorName
+}
+
+// Merge applies default values from the provided policy.
+func (p *CompressionPolicy) Merge(src CompressionPolicy) {
+	if p.CompressorName == "" {
+		p.CompressorName = src.CompressorName
+	}
+
+	p.OnlyCompress = mergeStrings(p.OnlyCompress, src.OnlyCompress)
+	p.NeverCompress = mergeStrings(p.NeverCompress, src.NeverCompress)
+}
+
+var defaultCompressionPolicy = CompressionPolicy{}
+
+func mergeStrings(s1, s2 []string) []string {
+	merged := map[string]bool{}
+
+	for _, v := range s1 {
+		merged[v] = true
+	}
+
+	for _, v := range s2 {
+		merged[v] = true
+	}
+
+	var result []string
+	for v := range merged {
+		result = append(result, v)
+	}
+
+	sort.Strings(result)
+
+	return result
+}
+
+func isInSortedSlice(s string, slice []string) bool {
+	x := sort.SearchStrings(slice, s)
+	return x < len(slice) && slice[x] == s
+}

snapshot/policy/policy.go

@@ -13,11 +13,12 @@ var ErrPolicyNotFound = errors.New("policy not found")
 
 // Policy describes snapshot policy for a single source.
 type Policy struct {
-	Labels           map[string]string `json:"-"`
-	RetentionPolicy  RetentionPolicy   `json:"retention,omitempty"`
-	FilesPolicy      FilesPolicy       `json:"files,omitempty"`
-	SchedulingPolicy SchedulingPolicy  `json:"scheduling,omitempty"`
-	NoParent         bool              `json:"noParent,omitempty"`
+	Labels            map[string]string `json:"-"`
+	RetentionPolicy   RetentionPolicy   `json:"retention,omitempty"`
+	FilesPolicy       FilesPolicy       `json:"files,omitempty"`
+	SchedulingPolicy  SchedulingPolicy  `json:"scheduling,omitempty"`
+	CompressionPolicy CompressionPolicy `json:"compression,omitempty"`
+	NoParent          bool              `json:"noParent,omitempty"`
 }
 
 func (p *Policy) String() string {
@@ -59,12 +60,14 @@ func MergePolicies(policies []*Policy) *Policy {
 		merged.RetentionPolicy.Merge(p.RetentionPolicy)
 		merged.FilesPolicy.Merge(p.FilesPolicy)
 		merged.SchedulingPolicy.Merge(p.SchedulingPolicy)
+		merged.CompressionPolicy.Merge(p.CompressionPolicy)
 	}
 
 	// Merge default expiration policy.
 	merged.RetentionPolicy.Merge(defaultRetentionPolicy)
 	merged.FilesPolicy.Merge(defaultFilesPolicy)
 	merged.SchedulingPolicy.Merge(defaultSchedulingPolicy)
+	merged.CompressionPolicy.Merge(defaultCompressionPolicy)
 
 	return &merged
 }