mirror/kopia
1
0
Fork 0
mirror of https://github.com/kopia/kopia.git synced 2026-05-24 14:44:47 -04:00
Code Issues Packages Projects Releases Wiki Activity

added files policy include/exclude configuration and checking during Upload

Browse Source
This commit is contained in:
Jarek Kowalski
2017-08-19 23:36:49 -07:00
parent 61cc2b46be
commit d2105aea40
6 changed files with 148 additions and 48 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
cli/command_backup.go
View File

@@ -99,7 +99,7 @@ func runBackupCommand(c *kingpin.ParseContext) error {
return err
}
u.Files = policy.Files
u.FilesPolicy = policy.FilesPolicy
manifest, err := u.Upload(localEntry, sourceInfo, oldManifest)
if err != nil {

44
cli/command_expire.go
View File

@@ -35,24 +35,24 @@ func expireSnapshotsForSingleSource(snapshots []*snapshot.Manifest, pol *snapsho
var hourlyCutoffTime time.Time
var weeklyCutoffTime time.Time
if pol.Expiration.KeepAnnual != nil {
annualCutoffTime = time.Now().AddDate(-*pol.Expiration.KeepAnnual, 0, 0)
if pol.ExpirationPolicy.KeepAnnual != nil {
annualCutoffTime = time.Now().AddDate(-*pol.ExpirationPolicy.KeepAnnual, 0, 0)
}
if pol.Expiration.KeepMonthly != nil {
monthlyCutoffTime = time.Now().AddDate(0, -*pol.Expiration.KeepMonthly, 0)
if pol.ExpirationPolicy.KeepMonthly != nil {
monthlyCutoffTime = time.Now().AddDate(0, -*pol.ExpirationPolicy.KeepMonthly, 0)
}
if pol.Expiration.KeepDaily != nil {
dailyCutoffTime = time.Now().AddDate(0, 0, -*pol.Expiration.KeepDaily)
if pol.ExpirationPolicy.KeepDaily != nil {
dailyCutoffTime = time.Now().AddDate(0, 0, -*pol.ExpirationPolicy.KeepDaily)
}
if pol.Expiration.KeepHourly != nil {
hourlyCutoffTime = time.Now().Add(time.Duration(-*pol.Expiration.KeepHourly) * time.Hour)
if pol.ExpirationPolicy.KeepHourly != nil {
hourlyCutoffTime = time.Now().Add(time.Duration(-*pol.ExpirationPolicy.KeepHourly) * time.Hour)
}
if pol.Expiration.KeepWeekly != nil {
weeklyCutoffTime = time.Now().AddDate(0, 0, -7**pol.Expiration.KeepWeekly)
if pol.ExpirationPolicy.KeepWeekly != nil {
weeklyCutoffTime = time.Now().AddDate(0, 0, -7**pol.ExpirationPolicy.KeepWeekly)
}
fmt.Printf("\n%v\n", pol.Source)
@@ -72,24 +72,24 @@ func expireSnapshotsForSingleSource(snapshots []*snapshot.Manifest, pol *snapsho
continue
}
if pol.Expiration.KeepLatest != nil {
registerSnapshot(fmt.Sprintf("%v", i), "latest", *pol.Expiration.KeepLatest)
if pol.ExpirationPolicy.KeepLatest != nil {
registerSnapshot(fmt.Sprintf("%v", i), "latest", *pol.ExpirationPolicy.KeepLatest)
}
if s.StartTime.After(annualCutoffTime) && pol.Expiration.KeepAnnual != nil {
registerSnapshot(s.StartTime.Format("2006"), "annual", *pol.Expiration.KeepAnnual)
if s.StartTime.After(annualCutoffTime) && pol.ExpirationPolicy.KeepAnnual != nil {
registerSnapshot(s.StartTime.Format("2006"), "annual", *pol.ExpirationPolicy.KeepAnnual)
}
if s.StartTime.After(monthlyCutoffTime) && pol.Expiration.KeepMonthly != nil {
registerSnapshot(s.StartTime.Format("2006-01"), "monthly", *pol.Expiration.KeepMonthly)
if s.StartTime.After(monthlyCutoffTime) && pol.ExpirationPolicy.KeepMonthly != nil {
registerSnapshot(s.StartTime.Format("2006-01"), "monthly", *pol.ExpirationPolicy.KeepMonthly)
}
if s.StartTime.After(weeklyCutoffTime) && pol.Expiration.KeepWeekly != nil {
if s.StartTime.After(weeklyCutoffTime) && pol.ExpirationPolicy.KeepWeekly != nil {
yyyy, wk := s.StartTime.ISOWeek()
registerSnapshot(fmt.Sprintf("%04v-%02v", yyyy, wk), "weekly", *pol.Expiration.KeepWeekly)
registerSnapshot(fmt.Sprintf("%04v-%02v", yyyy, wk), "weekly", *pol.ExpirationPolicy.KeepWeekly)
}
if s.StartTime.After(dailyCutoffTime) && pol.Expiration.KeepDaily != nil {
registerSnapshot(s.StartTime.Format("2006-01-02"), "daily", *pol.Expiration.KeepDaily)
if s.StartTime.After(dailyCutoffTime) && pol.ExpirationPolicy.KeepDaily != nil {
registerSnapshot(s.StartTime.Format("2006-01-02"), "daily", *pol.ExpirationPolicy.KeepDaily)
}
if s.StartTime.After(hourlyCutoffTime) && pol.Expiration.KeepHourly != nil {
registerSnapshot(s.StartTime.Format("2006-01-02 15"), "hourly", *pol.Expiration.KeepHourly)
if s.StartTime.After(hourlyCutoffTime) && pol.ExpirationPolicy.KeepHourly != nil {
registerSnapshot(s.StartTime.Format("2006-01-02 15"), "hourly", *pol.ExpirationPolicy.KeepHourly)
}
tm := s.StartTime.Local().Format("2006-01-02 15:04:05 MST")

76
cli/command_policy_set.go
View File

@@ -3,6 +3,7 @@
import (
"fmt"
"log"
"sort"
"strconv"
"github.com/kopia/kopia/snapshot"
@@ -18,17 +19,25 @@
policySetFrequency = policySetCommand.Flag("min-duration-between-backups", "Minimum duration between snapshots").Duration()
// Expiration policies.
policySetKeepLatest = policySetCommand.Flag("keep-latest", "Number of most recent backups to keep per source (or 'inherit')").String()
policySetKeepHourly = policySetCommand.Flag("keep-hourly", "Number of most-recent hourly backups to keep per source (or 'inherit')").String()
policySetKeepDaily = policySetCommand.Flag("keep-daily", "Number of most-recent daily backups to keep per source (or 'inherit')").String()
policySetKeepWeekly = policySetCommand.Flag("keep-weekly", "Number of most-recent weekly backups to keep per source (or 'inherit')").String()
policySetKeepMonthly = policySetCommand.Flag("keep-monthly", "Number of most-recent monthly backups to keep per source (or 'inherit')").String()
policySetKeepAnnual = policySetCommand.Flag("keep-annual", "Number of most-recent annual backups to keep per source (or 'inherit')").String()
policySetKeepLatest = policySetCommand.Flag("keep-latest", "Number of most recent backups to keep per source (or 'inherit')").PlaceHolder("N").String()
policySetKeepHourly = policySetCommand.Flag("keep-hourly", "Number of most-recent hourly backups to keep per source (or 'inherit')").PlaceHolder("N").String()
policySetKeepDaily = policySetCommand.Flag("keep-daily", "Number of most-recent daily backups to keep per source (or 'inherit')").PlaceHolder("N").String()
policySetKeepWeekly = policySetCommand.Flag("keep-weekly", "Number of most-recent weekly backups to keep per source (or 'inherit')").PlaceHolder("N").String()
policySetKeepMonthly = policySetCommand.Flag("keep-monthly", "Number of most-recent monthly backups to keep per source (or 'inherit')").PlaceHolder("N").String()
policySetKeepAnnual = policySetCommand.Flag("keep-annual", "Number of most-recent annual backups to keep per source (or 'inherit')").PlaceHolder("N").String()
// Files to ignore.
policySetAddIgnore = policySetCommand.Flag("add-ignore", "List of paths to add to ignore list").Strings()
policySetRemoveIgnore = policySetCommand.Flag("remove-ignore", "List of paths to remove from ignore list").Strings()
policySetReplaceIgnore = policySetCommand.Flag("set-ignore", "List of paths to replace ignore list with").Strings()
// Files to include (by default everything).
policySetAddInclude = policySetCommand.Flag("add-include", "List of paths to add to the include list").PlaceHolder("PATTERN").Strings()
policySetRemoveInclude = policySetCommand.Flag("remove-include", "List of paths to remove from the include list").PlaceHolder("PATTERN").Strings()
policySetClearInclude = policySetCommand.Flag("clear-include", "Clear list of paths in the include list").Bool()
// Files to exclude.
policySetAddExclude = policySetCommand.Flag("add-exclude", "List of paths to add to the exclude list").PlaceHolder("PATTERN").Strings()
policySetRemoveExclude = policySetCommand.Flag("remove-exclude", "List of paths to remove from the exclude list").PlaceHolder("PATTERN").Strings()
policySetClearExclude = policySetCommand.Flag("clear-exclude", "Clear list of paths in the exclude list").Bool()
// General policy.
policySetInherit = policySetCommand.Flag("inherit", "Enable or disable inheriting policies from the parent").BoolList()
)
func init() {
@@ -53,30 +62,47 @@ func setPolicy(context *kingpin.ParseContext) error {
}
}
if err := applyPolicyNumber(target, "number of annual backups to keep", &p.Expiration.KeepAnnual, *policySetKeepAnnual); err != nil {
if err := applyPolicyNumber(target, "number of annual backups to keep", &p.ExpirationPolicy.KeepAnnual, *policySetKeepAnnual); err != nil {
return err
}
if err := applyPolicyNumber(target, "number of monthly backups to keep", &p.Expiration.KeepMonthly, *policySetKeepMonthly); err != nil {
if err := applyPolicyNumber(target, "number of monthly backups to keep", &p.ExpirationPolicy.KeepMonthly, *policySetKeepMonthly); err != nil {
return err
}
if err := applyPolicyNumber(target, "number of weekly backups to keep", &p.Expiration.KeepWeekly, *policySetKeepWeekly); err != nil {
if err := applyPolicyNumber(target, "number of weekly backups to keep", &p.ExpirationPolicy.KeepWeekly, *policySetKeepWeekly); err != nil {
return err
}
if err := applyPolicyNumber(target, "number of daily backups to keep", &p.Expiration.KeepDaily, *policySetKeepDaily); err != nil {
if err := applyPolicyNumber(target, "number of daily backups to keep", &p.ExpirationPolicy.KeepDaily, *policySetKeepDaily); err != nil {
return err
}
if err := applyPolicyNumber(target, "number of hourly backups to keep", &p.Expiration.KeepHourly, *policySetKeepHourly); err != nil {
if err := applyPolicyNumber(target, "number of hourly backups to keep", &p.ExpirationPolicy.KeepHourly, *policySetKeepHourly); err != nil {
return err
}
if err := applyPolicyNumber(target, "number of latest backups to keep", &p.Expiration.KeepLatest, *policySetKeepLatest); err != nil {
if err := applyPolicyNumber(target, "number of latest backups to keep", &p.ExpirationPolicy.KeepLatest, *policySetKeepLatest); err != nil {
return err
}
// It's not really a list, just optional boolean.
for _, inherit := range *policySetInherit {
p.NoParent = !inherit
}
for _, path := range *policySetAddExclude {
p.FilesPolicy.Exclude = addString(p.FilesPolicy.Exclude, path)
}
for _, path := range *policySetRemoveExclude {
p.FilesPolicy.Exclude = removeString(p.FilesPolicy.Exclude, path)
}
if *policySetClearExclude {
p.FilesPolicy.Exclude = nil
}
if err := mgr.SavePolicy(p); err != nil {
return fmt.Errorf("can't save policy for %v: %v", target, err)
}
@@ -85,6 +111,24 @@ func setPolicy(context *kingpin.ParseContext) error {
return nil
}
func addString(p []string, s string) []string {
p = append(removeString(p, s), s)
sort.Strings(p)
return p
}
func removeString(p []string, s string) []string {
var result []string
for _, item := range p {
if item == s {
continue
}
result = append(result, item)
}
return result
}
func applyPolicyNumber(src *snapshot.SourceInfo, desc string, val **int, str string) error {
if str == "" {
// not changed

61
snapshot/policy.go
View File

@@ -3,6 +3,10 @@
import (
"bytes"
"encoding/json"
"log"
"path/filepath"
"github.com/kopia/kopia/fs"
)
// ExpirationPolicy describes snapshot expiration policy.
@@ -31,14 +35,45 @@ type FilesPolicy struct {
MaxSize *int `json:"maxSize,omitempty"`
}
// ShouldInclude determines whether given filesystem entry should be included based on the policy.
func (p *FilesPolicy) ShouldInclude(e *fs.EntryMetadata) bool {
if len(p.Include) > 0 {
include := false
for _, i := range p.Include {
if fileNameMatches(e.Name, i) {
include = true
break
}
}
if !include {
// have include rules, but none of them matched
return false
}
}
if len(p.Exclude) > 0 {
for _, ex := range p.Exclude {
if fileNameMatches(e.Name, ex) {
return false
}
}
}
if p.MaxSize != nil && e.Type == fs.EntryTypeFile && e.FileSize > int64(*p.MaxSize) {
return false
}
return true
}
var defaultFilesPolicy = &FilesPolicy{}
// Policy describes snapshot policy for a single source.
type Policy struct {
Source SourceInfo `json:"source"`
Expiration ExpirationPolicy `json:"expiration"`
Files FilesPolicy `json:"files"`
NoParent bool `json:"noParent,omitempty"`
Source SourceInfo `json:"source"`
ExpirationPolicy ExpirationPolicy `json:"expiration"`
FilesPolicy FilesPolicy `json:"files"`
NoParent bool `json:"noParent,omitempty"`
}
func (p *Policy) String() string {
@@ -50,6 +85,16 @@ func (p *Policy) String() string {
return buf.String()
}
func fileNameMatches(fname string, pattern string) bool {
ok, err := filepath.Match(pattern, fname)
if err != nil {
log.Printf("warning: %v, assuming %q does not match the pattern", err, fname)
return false
}
return ok
}
func mergePolicies(policies []*Policy) *Policy {
var merged Policy
@@ -58,13 +103,13 @@ func mergePolicies(policies []*Policy) *Policy {
break
}
mergeExpirationPolicy(&merged.Expiration, &p.Expiration)
mergeFilesPolicy(&merged.Files, &p.Files)
mergeExpirationPolicy(&merged.ExpirationPolicy, &p.ExpirationPolicy)
mergeFilesPolicy(&merged.FilesPolicy, &p.FilesPolicy)
}
// Merge default expiration policy.
mergeExpirationPolicy(&merged.Expiration, defaultExpirationPolicy)
mergeFilesPolicy(&merged.Files, defaultFilesPolicy)
mergeExpirationPolicy(&merged.ExpirationPolicy, defaultExpirationPolicy)
mergeFilesPolicy(&merged.FilesPolicy, defaultFilesPolicy)
return &merged
}

3
snapshot/stats.go
View File

@@ -10,6 +10,9 @@ type Stats struct {
TotalFileCount int `json:"fileCount"`
TotalFileSize int64 `json:"totalSize"`
ExcludedFileCount int `json:"excludedFileCount"`
ExcludedTotalFileSize int64 `json:"excludedTotalSize"`
CachedFiles int `json:"cachedFiles"`
NonCachedFiles int `json:"nonCachedFiles"`

10
snapshot/upload.go
View File

@@ -39,7 +39,8 @@ func metadataHash(e *fs.EntryMetadata) uint64 {
type Uploader struct {
Progress UploadProgress
Files FilesPolicy
// specifies criteria for including and excluding files.
FilesPolicy FilesPolicy
// automatically cancel the Upload after certain number of bytes
MaxUploadBytes int64
@@ -279,6 +280,13 @@ func uploadDirInternal(
e := entry.Metadata()
entryRelativePath := relativePath + "/" + e.Name
if !u.FilesPolicy.ShouldInclude(e) {
log.Printf("ignoring %q", entryRelativePath)
u.stats.ExcludedFileCount++
u.stats.ExcludedTotalFileSize += e.FileSize
continue
}
var de *dir.Entry
var hash uint64