From d2105aea401a2e0696ecfa69dd7262947e2b6544 Mon Sep 17 00:00:00 2001 From: Jarek Kowalski Date: Sat, 19 Aug 2017 23:36:49 -0700 Subject: [PATCH] added files policy include/exclude configuration and checking during Upload --- cli/command_backup.go | 2 +- cli/command_expire.go | 44 +++++++++++------------ cli/command_policy_set.go | 76 ++++++++++++++++++++++++++++++--------- snapshot/policy.go | 61 ++++++++++++++++++++++++++----- snapshot/stats.go | 3 ++ snapshot/upload.go | 10 +++++- 6 files changed, 148 insertions(+), 48 deletions(-) diff --git a/cli/command_backup.go b/cli/command_backup.go index f785a5638..a9f5e1e7b 100644 --- a/cli/command_backup.go +++ b/cli/command_backup.go @@ -99,7 +99,7 @@ func runBackupCommand(c *kingpin.ParseContext) error { return err } - u.Files = policy.Files + u.FilesPolicy = policy.FilesPolicy manifest, err := u.Upload(localEntry, sourceInfo, oldManifest) if err != nil { diff --git a/cli/command_expire.go b/cli/command_expire.go index 6d5a66e67..6d0d8137a 100644 --- a/cli/command_expire.go +++ b/cli/command_expire.go @@ -35,24 +35,24 @@ func expireSnapshotsForSingleSource(snapshots []*snapshot.Manifest, pol *snapsho var hourlyCutoffTime time.Time var weeklyCutoffTime time.Time - if pol.Expiration.KeepAnnual != nil { - annualCutoffTime = time.Now().AddDate(-*pol.Expiration.KeepAnnual, 0, 0) + if pol.ExpirationPolicy.KeepAnnual != nil { + annualCutoffTime = time.Now().AddDate(-*pol.ExpirationPolicy.KeepAnnual, 0, 0) } - if pol.Expiration.KeepMonthly != nil { - monthlyCutoffTime = time.Now().AddDate(0, -*pol.Expiration.KeepMonthly, 0) + if pol.ExpirationPolicy.KeepMonthly != nil { + monthlyCutoffTime = time.Now().AddDate(0, -*pol.ExpirationPolicy.KeepMonthly, 0) } - if pol.Expiration.KeepDaily != nil { - dailyCutoffTime = time.Now().AddDate(0, 0, -*pol.Expiration.KeepDaily) + if pol.ExpirationPolicy.KeepDaily != nil { + dailyCutoffTime = time.Now().AddDate(0, 0, -*pol.ExpirationPolicy.KeepDaily) } - if pol.Expiration.KeepHourly != nil { - hourlyCutoffTime = time.Now().Add(time.Duration(-*pol.Expiration.KeepHourly) * time.Hour) + if pol.ExpirationPolicy.KeepHourly != nil { + hourlyCutoffTime = time.Now().Add(time.Duration(-*pol.ExpirationPolicy.KeepHourly) * time.Hour) } - if pol.Expiration.KeepWeekly != nil { - weeklyCutoffTime = time.Now().AddDate(0, 0, -7**pol.Expiration.KeepWeekly) + if pol.ExpirationPolicy.KeepWeekly != nil { + weeklyCutoffTime = time.Now().AddDate(0, 0, -7**pol.ExpirationPolicy.KeepWeekly) } fmt.Printf("\n%v\n", pol.Source) @@ -72,24 +72,24 @@ func expireSnapshotsForSingleSource(snapshots []*snapshot.Manifest, pol *snapsho continue } - if pol.Expiration.KeepLatest != nil { - registerSnapshot(fmt.Sprintf("%v", i), "latest", *pol.Expiration.KeepLatest) + if pol.ExpirationPolicy.KeepLatest != nil { + registerSnapshot(fmt.Sprintf("%v", i), "latest", *pol.ExpirationPolicy.KeepLatest) } - if s.StartTime.After(annualCutoffTime) && pol.Expiration.KeepAnnual != nil { - registerSnapshot(s.StartTime.Format("2006"), "annual", *pol.Expiration.KeepAnnual) + if s.StartTime.After(annualCutoffTime) && pol.ExpirationPolicy.KeepAnnual != nil { + registerSnapshot(s.StartTime.Format("2006"), "annual", *pol.ExpirationPolicy.KeepAnnual) } - if s.StartTime.After(monthlyCutoffTime) && pol.Expiration.KeepMonthly != nil { - registerSnapshot(s.StartTime.Format("2006-01"), "monthly", *pol.Expiration.KeepMonthly) + if s.StartTime.After(monthlyCutoffTime) && pol.ExpirationPolicy.KeepMonthly != nil { + registerSnapshot(s.StartTime.Format("2006-01"), "monthly", *pol.ExpirationPolicy.KeepMonthly) } - if s.StartTime.After(weeklyCutoffTime) && pol.Expiration.KeepWeekly != nil { + if s.StartTime.After(weeklyCutoffTime) && pol.ExpirationPolicy.KeepWeekly != nil { yyyy, wk := s.StartTime.ISOWeek() - registerSnapshot(fmt.Sprintf("%04v-%02v", yyyy, wk), "weekly", *pol.Expiration.KeepWeekly) + registerSnapshot(fmt.Sprintf("%04v-%02v", yyyy, wk), "weekly", *pol.ExpirationPolicy.KeepWeekly) } - if s.StartTime.After(dailyCutoffTime) && pol.Expiration.KeepDaily != nil { - registerSnapshot(s.StartTime.Format("2006-01-02"), "daily", *pol.Expiration.KeepDaily) + if s.StartTime.After(dailyCutoffTime) && pol.ExpirationPolicy.KeepDaily != nil { + registerSnapshot(s.StartTime.Format("2006-01-02"), "daily", *pol.ExpirationPolicy.KeepDaily) } - if s.StartTime.After(hourlyCutoffTime) && pol.Expiration.KeepHourly != nil { - registerSnapshot(s.StartTime.Format("2006-01-02 15"), "hourly", *pol.Expiration.KeepHourly) + if s.StartTime.After(hourlyCutoffTime) && pol.ExpirationPolicy.KeepHourly != nil { + registerSnapshot(s.StartTime.Format("2006-01-02 15"), "hourly", *pol.ExpirationPolicy.KeepHourly) } tm := s.StartTime.Local().Format("2006-01-02 15:04:05 MST") diff --git a/cli/command_policy_set.go b/cli/command_policy_set.go index 4db295821..5f7e0cfad 100644 --- a/cli/command_policy_set.go +++ b/cli/command_policy_set.go @@ -3,6 +3,7 @@ import ( "fmt" "log" + "sort" "strconv" "github.com/kopia/kopia/snapshot" @@ -18,17 +19,25 @@ policySetFrequency = policySetCommand.Flag("min-duration-between-backups", "Minimum duration between snapshots").Duration() // Expiration policies. - policySetKeepLatest = policySetCommand.Flag("keep-latest", "Number of most recent backups to keep per source (or 'inherit')").String() - policySetKeepHourly = policySetCommand.Flag("keep-hourly", "Number of most-recent hourly backups to keep per source (or 'inherit')").String() - policySetKeepDaily = policySetCommand.Flag("keep-daily", "Number of most-recent daily backups to keep per source (or 'inherit')").String() - policySetKeepWeekly = policySetCommand.Flag("keep-weekly", "Number of most-recent weekly backups to keep per source (or 'inherit')").String() - policySetKeepMonthly = policySetCommand.Flag("keep-monthly", "Number of most-recent monthly backups to keep per source (or 'inherit')").String() - policySetKeepAnnual = policySetCommand.Flag("keep-annual", "Number of most-recent annual backups to keep per source (or 'inherit')").String() + policySetKeepLatest = policySetCommand.Flag("keep-latest", "Number of most recent backups to keep per source (or 'inherit')").PlaceHolder("N").String() + policySetKeepHourly = policySetCommand.Flag("keep-hourly", "Number of most-recent hourly backups to keep per source (or 'inherit')").PlaceHolder("N").String() + policySetKeepDaily = policySetCommand.Flag("keep-daily", "Number of most-recent daily backups to keep per source (or 'inherit')").PlaceHolder("N").String() + policySetKeepWeekly = policySetCommand.Flag("keep-weekly", "Number of most-recent weekly backups to keep per source (or 'inherit')").PlaceHolder("N").String() + policySetKeepMonthly = policySetCommand.Flag("keep-monthly", "Number of most-recent monthly backups to keep per source (or 'inherit')").PlaceHolder("N").String() + policySetKeepAnnual = policySetCommand.Flag("keep-annual", "Number of most-recent annual backups to keep per source (or 'inherit')").PlaceHolder("N").String() - // Files to ignore. - policySetAddIgnore = policySetCommand.Flag("add-ignore", "List of paths to add to ignore list").Strings() - policySetRemoveIgnore = policySetCommand.Flag("remove-ignore", "List of paths to remove from ignore list").Strings() - policySetReplaceIgnore = policySetCommand.Flag("set-ignore", "List of paths to replace ignore list with").Strings() + // Files to include (by default everything). + policySetAddInclude = policySetCommand.Flag("add-include", "List of paths to add to the include list").PlaceHolder("PATTERN").Strings() + policySetRemoveInclude = policySetCommand.Flag("remove-include", "List of paths to remove from the include list").PlaceHolder("PATTERN").Strings() + policySetClearInclude = policySetCommand.Flag("clear-include", "Clear list of paths in the include list").Bool() + + // Files to exclude. + policySetAddExclude = policySetCommand.Flag("add-exclude", "List of paths to add to the exclude list").PlaceHolder("PATTERN").Strings() + policySetRemoveExclude = policySetCommand.Flag("remove-exclude", "List of paths to remove from the exclude list").PlaceHolder("PATTERN").Strings() + policySetClearExclude = policySetCommand.Flag("clear-exclude", "Clear list of paths in the exclude list").Bool() + + // General policy. + policySetInherit = policySetCommand.Flag("inherit", "Enable or disable inheriting policies from the parent").BoolList() ) func init() { @@ -53,30 +62,47 @@ func setPolicy(context *kingpin.ParseContext) error { } } - if err := applyPolicyNumber(target, "number of annual backups to keep", &p.Expiration.KeepAnnual, *policySetKeepAnnual); err != nil { + if err := applyPolicyNumber(target, "number of annual backups to keep", &p.ExpirationPolicy.KeepAnnual, *policySetKeepAnnual); err != nil { return err } - if err := applyPolicyNumber(target, "number of monthly backups to keep", &p.Expiration.KeepMonthly, *policySetKeepMonthly); err != nil { + if err := applyPolicyNumber(target, "number of monthly backups to keep", &p.ExpirationPolicy.KeepMonthly, *policySetKeepMonthly); err != nil { return err } - if err := applyPolicyNumber(target, "number of weekly backups to keep", &p.Expiration.KeepWeekly, *policySetKeepWeekly); err != nil { + if err := applyPolicyNumber(target, "number of weekly backups to keep", &p.ExpirationPolicy.KeepWeekly, *policySetKeepWeekly); err != nil { return err } - if err := applyPolicyNumber(target, "number of daily backups to keep", &p.Expiration.KeepDaily, *policySetKeepDaily); err != nil { + if err := applyPolicyNumber(target, "number of daily backups to keep", &p.ExpirationPolicy.KeepDaily, *policySetKeepDaily); err != nil { return err } - if err := applyPolicyNumber(target, "number of hourly backups to keep", &p.Expiration.KeepHourly, *policySetKeepHourly); err != nil { + if err := applyPolicyNumber(target, "number of hourly backups to keep", &p.ExpirationPolicy.KeepHourly, *policySetKeepHourly); err != nil { return err } - if err := applyPolicyNumber(target, "number of latest backups to keep", &p.Expiration.KeepLatest, *policySetKeepLatest); err != nil { + if err := applyPolicyNumber(target, "number of latest backups to keep", &p.ExpirationPolicy.KeepLatest, *policySetKeepLatest); err != nil { return err } + // It's not really a list, just optional boolean. + for _, inherit := range *policySetInherit { + p.NoParent = !inherit + } + + for _, path := range *policySetAddExclude { + p.FilesPolicy.Exclude = addString(p.FilesPolicy.Exclude, path) + } + + for _, path := range *policySetRemoveExclude { + p.FilesPolicy.Exclude = removeString(p.FilesPolicy.Exclude, path) + } + + if *policySetClearExclude { + p.FilesPolicy.Exclude = nil + } + if err := mgr.SavePolicy(p); err != nil { return fmt.Errorf("can't save policy for %v: %v", target, err) } @@ -85,6 +111,24 @@ func setPolicy(context *kingpin.ParseContext) error { return nil } +func addString(p []string, s string) []string { + p = append(removeString(p, s), s) + sort.Strings(p) + return p +} + +func removeString(p []string, s string) []string { + var result []string + + for _, item := range p { + if item == s { + continue + } + result = append(result, item) + } + return result +} + func applyPolicyNumber(src *snapshot.SourceInfo, desc string, val **int, str string) error { if str == "" { // not changed diff --git a/snapshot/policy.go b/snapshot/policy.go index 262392b52..303a1732f 100644 --- a/snapshot/policy.go +++ b/snapshot/policy.go @@ -3,6 +3,10 @@ import ( "bytes" "encoding/json" + "log" + "path/filepath" + + "github.com/kopia/kopia/fs" ) // ExpirationPolicy describes snapshot expiration policy. @@ -31,14 +35,45 @@ type FilesPolicy struct { MaxSize *int `json:"maxSize,omitempty"` } +// ShouldInclude determines whether given filesystem entry should be included based on the policy. +func (p *FilesPolicy) ShouldInclude(e *fs.EntryMetadata) bool { + if len(p.Include) > 0 { + include := false + for _, i := range p.Include { + if fileNameMatches(e.Name, i) { + include = true + break + } + } + if !include { + // have include rules, but none of them matched + return false + } + } + + if len(p.Exclude) > 0 { + for _, ex := range p.Exclude { + if fileNameMatches(e.Name, ex) { + return false + } + } + } + + if p.MaxSize != nil && e.Type == fs.EntryTypeFile && e.FileSize > int64(*p.MaxSize) { + return false + } + + return true +} + var defaultFilesPolicy = &FilesPolicy{} // Policy describes snapshot policy for a single source. type Policy struct { - Source SourceInfo `json:"source"` - Expiration ExpirationPolicy `json:"expiration"` - Files FilesPolicy `json:"files"` - NoParent bool `json:"noParent,omitempty"` + Source SourceInfo `json:"source"` + ExpirationPolicy ExpirationPolicy `json:"expiration"` + FilesPolicy FilesPolicy `json:"files"` + NoParent bool `json:"noParent,omitempty"` } func (p *Policy) String() string { @@ -50,6 +85,16 @@ func (p *Policy) String() string { return buf.String() } +func fileNameMatches(fname string, pattern string) bool { + ok, err := filepath.Match(pattern, fname) + if err != nil { + log.Printf("warning: %v, assuming %q does not match the pattern", err, fname) + return false + } + + return ok +} + func mergePolicies(policies []*Policy) *Policy { var merged Policy @@ -58,13 +103,13 @@ func mergePolicies(policies []*Policy) *Policy { break } - mergeExpirationPolicy(&merged.Expiration, &p.Expiration) - mergeFilesPolicy(&merged.Files, &p.Files) + mergeExpirationPolicy(&merged.ExpirationPolicy, &p.ExpirationPolicy) + mergeFilesPolicy(&merged.FilesPolicy, &p.FilesPolicy) } // Merge default expiration policy. - mergeExpirationPolicy(&merged.Expiration, defaultExpirationPolicy) - mergeFilesPolicy(&merged.Files, defaultFilesPolicy) + mergeExpirationPolicy(&merged.ExpirationPolicy, defaultExpirationPolicy) + mergeFilesPolicy(&merged.FilesPolicy, defaultFilesPolicy) return &merged } diff --git a/snapshot/stats.go b/snapshot/stats.go index da39094c9..bfa2ccfae 100644 --- a/snapshot/stats.go +++ b/snapshot/stats.go @@ -10,6 +10,9 @@ type Stats struct { TotalFileCount int `json:"fileCount"` TotalFileSize int64 `json:"totalSize"` + ExcludedFileCount int `json:"excludedFileCount"` + ExcludedTotalFileSize int64 `json:"excludedTotalSize"` + CachedFiles int `json:"cachedFiles"` NonCachedFiles int `json:"nonCachedFiles"` diff --git a/snapshot/upload.go b/snapshot/upload.go index 8638cbb3d..9cc4897ab 100644 --- a/snapshot/upload.go +++ b/snapshot/upload.go @@ -39,7 +39,8 @@ func metadataHash(e *fs.EntryMetadata) uint64 { type Uploader struct { Progress UploadProgress - Files FilesPolicy + // specifies criteria for including and excluding files. + FilesPolicy FilesPolicy // automatically cancel the Upload after certain number of bytes MaxUploadBytes int64 @@ -279,6 +280,13 @@ func uploadDirInternal( e := entry.Metadata() entryRelativePath := relativePath + "/" + e.Name + if !u.FilesPolicy.ShouldInclude(e) { + log.Printf("ignoring %q", entryRelativePath) + u.stats.ExcludedFileCount++ + u.stats.ExcludedTotalFileSize += e.FileSize + continue + } + var de *dir.Entry var hash uint64