From 3ef4593b03641109e437c79278caec398a4defd5 Mon Sep 17 00:00:00 2001 From: Jarek Kowalski Date: Wed, 29 Aug 2018 23:08:40 -0700 Subject: [PATCH] refactored policy management into separate policy/ package --- cli/command_policy.go | 5 +-- cli/command_policy_edit.go | 12 +++---- cli/command_policy_ls.go | 4 +-- cli/command_policy_remove.go | 4 +-- cli/command_policy_set.go | 19 ++++++----- cli/command_policy_show.go | 36 ++++++++++---------- cli/command_snapshot_create.go | 5 +-- cli/command_snapshot_estimate.go | 3 +- cli/command_snapshot_expire.go | 5 +-- cli/command_snapshot_list.go | 5 +-- internal/server/server.go | 5 +-- internal/server/source_manager.go | 3 +- internal/serverapi/serverapi.go | 5 +-- {snapshot => policy}/expire.go | 14 ++++---- {snapshot => policy}/policy.go | 9 ++--- {snapshot => policy}/policy_manager.go | 41 +++++++++++++---------- {snapshot => policy}/retention_policy.go | 10 +++--- {snapshot => policy}/scheduling_policy.go | 2 +- snapshot/manager.go | 3 -- snapshot/source.go | 2 +- 20 files changed, 105 insertions(+), 87 deletions(-) rename {snapshot => policy}/expire.go (65%) rename {snapshot => policy}/policy.go (88%) rename {snapshot => policy}/policy_manager.go (78%) rename {snapshot => policy}/retention_policy.go (91%) rename {snapshot => policy}/scheduling_policy.go (99%) diff --git a/cli/command_policy.go b/cli/command_policy.go index deee19543..3442b9246 100644 --- a/cli/command_policy.go +++ b/cli/command_policy.go @@ -3,17 +3,18 @@ package cli import ( "fmt" + "github.com/kopia/kopia/policy" "github.com/kopia/kopia/snapshot" ) -func policyTargets(pmgr *snapshot.PolicyManager, globalFlag *bool, targetsFlag *[]string) ([]snapshot.SourceInfo, error) { +func policyTargets(pmgr *policy.Manager, globalFlag *bool, targetsFlag *[]string) ([]snapshot.SourceInfo, error) { if *globalFlag == (len(*targetsFlag) > 0) { return nil, fmt.Errorf("must pass either '--global' or a list of path targets") } if *globalFlag { return []snapshot.SourceInfo{ - snapshot.GlobalPolicySourceInfo, + policy.GlobalPolicySourceInfo, }, nil } diff --git a/cli/command_policy_edit.go b/cli/command_policy_edit.go index e494c9f6f..e2d2d4fd7 100644 --- a/cli/command_policy_edit.go +++ b/cli/command_policy_edit.go @@ -8,8 +8,8 @@ import ( "strings" "github.com/kopia/kopia/internal/editor" + "github.com/kopia/kopia/policy" "github.com/kopia/kopia/repo" - "github.com/kopia/kopia/snapshot" ) const policyEditHelpText = ` @@ -57,7 +57,7 @@ func init() { } func editPolicy(ctx context.Context, rep *repo.Repository) error { - mgr := snapshot.NewPolicyManager(rep) + mgr := policy.NewPolicyManager(rep) targets, err := policyTargets(mgr, policyEditGlobal, policyEditTargets) if err != nil { @@ -66,8 +66,8 @@ func editPolicy(ctx context.Context, rep *repo.Repository) error { for _, target := range targets { original, err := mgr.GetDefinedPolicy(target) - if err == snapshot.ErrPolicyNotFound { - original = &snapshot.Policy{} + if err == policy.ErrPolicyNotFound { + original = &policy.Policy{} } printStderr("Editing policy for %v using external editor...\n", target) @@ -77,10 +77,10 @@ func editPolicy(ctx context.Context, rep *repo.Repository) error { s = insertHelpText(s, ` "files": {`, policyEditFilesHelpText) s = insertHelpText(s, ` "scheduling": {`, policyEditSchedulingHelpText) - var updated *snapshot.Policy + var updated *policy.Policy if err := editor.EditLoop("policy.conf", s, func(edited string) error { - updated = &snapshot.Policy{} + updated = &policy.Policy{} d := json.NewDecoder(bytes.NewBufferString(edited)) d.DisallowUnknownFields() return d.Decode(updated) diff --git a/cli/command_policy_ls.go b/cli/command_policy_ls.go index 36ae2218c..b038d9883 100644 --- a/cli/command_policy_ls.go +++ b/cli/command_policy_ls.go @@ -5,8 +5,8 @@ import ( "fmt" "sort" + "github.com/kopia/kopia/policy" "github.com/kopia/kopia/repo" - "github.com/kopia/kopia/snapshot" ) var ( @@ -18,7 +18,7 @@ func init() { } func listPolicies(ctx context.Context, rep *repo.Repository) error { - mgr := snapshot.NewPolicyManager(rep) + mgr := policy.NewPolicyManager(rep) policies, err := mgr.ListPolicies() if err != nil { diff --git a/cli/command_policy_remove.go b/cli/command_policy_remove.go index 4a1313f93..4f37efb18 100644 --- a/cli/command_policy_remove.go +++ b/cli/command_policy_remove.go @@ -3,8 +3,8 @@ package cli import ( "context" + "github.com/kopia/kopia/policy" "github.com/kopia/kopia/repo" - "github.com/kopia/kopia/snapshot" ) var ( @@ -18,7 +18,7 @@ func init() { } func removePolicy(ctx context.Context, rep *repo.Repository) error { - mgr := snapshot.NewPolicyManager(rep) + mgr := policy.NewPolicyManager(rep) targets, err := policyTargets(mgr, policyRemoveGlobal, policyRemoveTargets) if err != nil { diff --git a/cli/command_policy_set.go b/cli/command_policy_set.go index 6659508d5..69f14d50d 100644 --- a/cli/command_policy_set.go +++ b/cli/command_policy_set.go @@ -8,6 +8,7 @@ import ( "strings" "github.com/kopia/kopia/fs/ignorefs" + "github.com/kopia/kopia/policy" "github.com/kopia/kopia/repo" "github.com/kopia/kopia/snapshot" ) @@ -53,7 +54,7 @@ func init() { } func setPolicy(ctx context.Context, rep *repo.Repository) error { - mgr := snapshot.NewPolicyManager(rep) + mgr := policy.NewPolicyManager(rep) targets, err := policyTargets(mgr, policySetGlobal, policySetTargets) if err != nil { @@ -62,8 +63,8 @@ func setPolicy(ctx context.Context, rep *repo.Repository) error { for _, target := range targets { p, err := mgr.GetDefinedPolicy(target) - if err == snapshot.ErrPolicyNotFound { - p = &snapshot.Policy{} + if err == policy.ErrPolicyNotFound { + p = &policy.Policy{} } printStderr("Setting policy for %v\n", target) @@ -85,7 +86,7 @@ func setPolicy(ctx context.Context, rep *repo.Repository) error { return nil } -func setPolicyFromFlags(target snapshot.SourceInfo, p *snapshot.Policy, changeCount *int) error { +func setPolicyFromFlags(target snapshot.SourceInfo, p *policy.Policy, changeCount *int) error { if err := setRetentionPolicyFromFlags(&p.RetentionPolicy, changeCount); err != nil { return fmt.Errorf("retention policy: %v", err) } @@ -129,7 +130,7 @@ func setFilesPolicyFromFlags(fp *ignorefs.FilesPolicy, changeCount *int) error { return nil } -func setRetentionPolicyFromFlags(rp *snapshot.RetentionPolicy, changeCount *int) error { +func setRetentionPolicyFromFlags(rp *policy.RetentionPolicy, changeCount *int) error { cases := []struct { desc string max **int @@ -151,7 +152,7 @@ func setRetentionPolicyFromFlags(rp *snapshot.RetentionPolicy, changeCount *int) return nil } -func setSchedulingPolicyFromFlags(sp *snapshot.SchedulingPolicy, changeCount *int) error { +func setSchedulingPolicyFromFlags(sp *policy.SchedulingPolicy, changeCount *int) error { // It's not really a list, just optional value. for _, interval := range *policySetInterval { *changeCount++ @@ -161,7 +162,7 @@ func setSchedulingPolicyFromFlags(sp *snapshot.SchedulingPolicy, changeCount *in } if len(*policySetTimesOfDay) > 0 { - var timesOfDay []snapshot.TimeOfDay + var timesOfDay []policy.TimeOfDay for _, tods := range *policySetTimesOfDay { for _, tod := range strings.Split(tods, ",") { @@ -170,7 +171,7 @@ func setSchedulingPolicyFromFlags(sp *snapshot.SchedulingPolicy, changeCount *in break } - var timeOfDay snapshot.TimeOfDay + var timeOfDay policy.TimeOfDay if err := timeOfDay.Parse(tod); err != nil { return fmt.Errorf("unable to parse time of day: %v", err) } @@ -179,7 +180,7 @@ func setSchedulingPolicyFromFlags(sp *snapshot.SchedulingPolicy, changeCount *in } *changeCount++ - sp.TimesOfDay = snapshot.SortAndDedupeTimesOfDay(timesOfDay) + sp.TimesOfDay = policy.SortAndDedupeTimesOfDay(timesOfDay) if timesOfDay == nil { printStderr(" - resetting snapshot times of day to default\n") diff --git a/cli/command_policy_show.go b/cli/command_policy_show.go index beda33cce..24b256a36 100644 --- a/cli/command_policy_show.go +++ b/cli/command_policy_show.go @@ -5,8 +5,8 @@ import ( "fmt" "github.com/kopia/kopia/internal/units" + "github.com/kopia/kopia/policy" "github.com/kopia/kopia/repo" - "github.com/kopia/kopia/snapshot" ) var ( @@ -21,7 +21,7 @@ func init() { } func showPolicy(ctx context.Context, rep *repo.Repository) error { - pmgr := snapshot.NewPolicyManager(rep) + pmgr := policy.NewPolicyManager(rep) targets, err := policyTargets(pmgr, policyShowGlobal, policyShowTargets) if err != nil { @@ -44,7 +44,7 @@ func showPolicy(ctx context.Context, rep *repo.Repository) error { return nil } -func getDefinitionPoint(parents []*snapshot.Policy, match func(p *snapshot.Policy) bool) string { +func getDefinitionPoint(parents []*policy.Policy, match func(p *policy.Policy) bool) string { for i, p := range parents { if match(p) { if i == 0 { @@ -71,7 +71,7 @@ func containsString(s []string, v string) bool { return false } -func printPolicy(p *snapshot.Policy, parents []*snapshot.Policy) { +func printPolicy(p *policy.Policy, parents []*policy.Policy) { printStdout("Policy for %v:\n", p.Target()) printRetentionPolicy(p, parents) @@ -81,41 +81,41 @@ func printPolicy(p *snapshot.Policy, parents []*snapshot.Policy) { printSchedulingPolicy(p, parents) } -func printRetentionPolicy(p *snapshot.Policy, parents []*snapshot.Policy) { +func printRetentionPolicy(p *policy.Policy, parents []*policy.Policy) { printStdout("Keep:\n") printStdout(" Annual snapshots: %3v %v\n", valueOrNotSet(p.RetentionPolicy.KeepAnnual), - getDefinitionPoint(parents, func(pol *snapshot.Policy) bool { + getDefinitionPoint(parents, func(pol *policy.Policy) bool { return pol.RetentionPolicy.KeepAnnual != nil })) printStdout(" Monthly snapshots: %3v %v\n", valueOrNotSet(p.RetentionPolicy.KeepMonthly), - getDefinitionPoint(parents, func(pol *snapshot.Policy) bool { + getDefinitionPoint(parents, func(pol *policy.Policy) bool { return pol.RetentionPolicy.KeepMonthly != nil })) printStdout(" Weekly snapshots: %3v %v\n", valueOrNotSet(p.RetentionPolicy.KeepWeekly), - getDefinitionPoint(parents, func(pol *snapshot.Policy) bool { + getDefinitionPoint(parents, func(pol *policy.Policy) bool { return pol.RetentionPolicy.KeepWeekly != nil })) printStdout(" Daily snapshots: %3v %v\n", valueOrNotSet(p.RetentionPolicy.KeepDaily), - getDefinitionPoint(parents, func(pol *snapshot.Policy) bool { + getDefinitionPoint(parents, func(pol *policy.Policy) bool { return pol.RetentionPolicy.KeepDaily != nil })) printStdout(" Hourly snapshots: %3v %v\n", valueOrNotSet(p.RetentionPolicy.KeepHourly), - getDefinitionPoint(parents, func(pol *snapshot.Policy) bool { + getDefinitionPoint(parents, func(pol *policy.Policy) bool { return pol.RetentionPolicy.KeepHourly != nil })) printStdout(" Latest snapshots: %3v %v\n", valueOrNotSet(p.RetentionPolicy.KeepLatest), - getDefinitionPoint(parents, func(pol *snapshot.Policy) bool { + getDefinitionPoint(parents, func(pol *policy.Policy) bool { return pol.RetentionPolicy.KeepLatest != nil })) } -func printFilesPolicy(p *snapshot.Policy, parents []*snapshot.Policy) { +func printFilesPolicy(p *policy.Policy, parents []*policy.Policy) { printStdout("Files policy:\n") if len(p.FilesPolicy.IgnoreRules) > 0 { @@ -124,7 +124,7 @@ func printFilesPolicy(p *snapshot.Policy, parents []*snapshot.Policy) { printStdout(" No ignore rules.\n") } for _, rule := range p.FilesPolicy.IgnoreRules { - printStdout(" %-30v %v\n", rule, getDefinitionPoint(parents, func(pol *snapshot.Policy) bool { + printStdout(" %-30v %v\n", rule, getDefinitionPoint(parents, func(pol *policy.Policy) bool { return containsString(pol.FilesPolicy.IgnoreRules, rule) })) } @@ -132,22 +132,22 @@ func printFilesPolicy(p *snapshot.Policy, parents []*snapshot.Policy) { printStdout(" Read ignore rules from files:\n") } for _, dotFile := range p.FilesPolicy.DotIgnoreFiles { - printStdout(" %-30v %v\n", dotFile, getDefinitionPoint(parents, func(pol *snapshot.Policy) bool { + printStdout(" %-30v %v\n", dotFile, getDefinitionPoint(parents, func(pol *policy.Policy) bool { return containsString(pol.FilesPolicy.DotIgnoreFiles, dotFile) })) } if maxSize := p.FilesPolicy.MaxFileSize; maxSize > 0 { printStdout(" Ignore files above: %10v %v\n", units.BytesStringBase2(maxSize), - getDefinitionPoint(parents, func(pol *snapshot.Policy) bool { + getDefinitionPoint(parents, func(pol *policy.Policy) bool { return pol.FilesPolicy.MaxFileSize != 0 })) } } -func printSchedulingPolicy(p *snapshot.Policy, parents []*snapshot.Policy) { +func printSchedulingPolicy(p *policy.Policy, parents []*policy.Policy) { if p.SchedulingPolicy.Interval() != 0 { - printStdout("Snapshot interval: %10v %v\n", p.SchedulingPolicy.Interval(), getDefinitionPoint(parents, func(pol *snapshot.Policy) bool { + printStdout("Snapshot interval: %10v %v\n", p.SchedulingPolicy.Interval(), getDefinitionPoint(parents, func(pol *policy.Policy) bool { return pol.SchedulingPolicy.Interval() != 0 })) } @@ -155,7 +155,7 @@ func printSchedulingPolicy(p *snapshot.Policy, parents []*snapshot.Policy) { if len(p.SchedulingPolicy.TimesOfDay) > 0 { printStdout("Snapshot times:\n") for _, tod := range p.SchedulingPolicy.TimesOfDay { - printStdout(" %9v %v\n", tod, getDefinitionPoint(parents, func(pol *snapshot.Policy) bool { + printStdout(" %9v %v\n", tod, getDefinitionPoint(parents, func(pol *policy.Policy) bool { for _, t := range pol.SchedulingPolicy.TimesOfDay { if t == tod { return true diff --git a/cli/command_snapshot_create.go b/cli/command_snapshot_create.go index 0fc4bd6db..9472b093e 100644 --- a/cli/command_snapshot_create.go +++ b/cli/command_snapshot_create.go @@ -12,6 +12,7 @@ import ( "strings" "time" + "github.com/kopia/kopia/policy" "github.com/kopia/kopia/repo" "github.com/kopia/kopia/snapshot" ) @@ -34,7 +35,7 @@ var ( func runBackupCommand(ctx context.Context, rep *repo.Repository) error { mgr := snapshot.NewManager(rep) - pmgr := snapshot.NewPolicyManager(rep) + pmgr := policy.NewPolicyManager(rep) sources := *snapshotCreateSources if *snapshotCreateAll { @@ -85,7 +86,7 @@ func runBackupCommand(ctx context.Context, rep *repo.Repository) error { return fmt.Errorf("encountered %v errors:\n%v", len(finalErrors), strings.Join(finalErrors, "\n")) } -func snapshotSingleSource(ctx context.Context, rep *repo.Repository, mgr *snapshot.Manager, pmgr *snapshot.PolicyManager, u *snapshot.Uploader, sourceInfo snapshot.SourceInfo) error { +func snapshotSingleSource(ctx context.Context, rep *repo.Repository, mgr *snapshot.Manager, pmgr *policy.Manager, u *snapshot.Uploader, sourceInfo snapshot.SourceInfo) error { t0 := time.Now() rep.Blocks.ResetStats() diff --git a/cli/command_snapshot_estimate.go b/cli/command_snapshot_estimate.go index b1ee03b50..63ab01bb2 100644 --- a/cli/command_snapshot_estimate.go +++ b/cli/command_snapshot_estimate.go @@ -7,6 +7,7 @@ import ( "time" "github.com/kopia/kopia/fs/ignorefs" + "github.com/kopia/kopia/policy" "github.com/kopia/kopia/repo" @@ -61,7 +62,7 @@ func makeBuckets() buckets { } func runSnapshotEstimateCommand(ctx context.Context, rep *repo.Repository) error { - pmgr := snapshot.NewPolicyManager(rep) + pmgr := policy.NewPolicyManager(rep) path, err := filepath.Abs(*snapshotEstimateSource) if err != nil { diff --git a/cli/command_snapshot_expire.go b/cli/command_snapshot_expire.go index 1634e5038..b6e16922f 100644 --- a/cli/command_snapshot_expire.go +++ b/cli/command_snapshot_expire.go @@ -4,6 +4,7 @@ import ( "context" "fmt" + "github.com/kopia/kopia/policy" "github.com/kopia/kopia/repo" "github.com/kopia/kopia/snapshot" ) @@ -53,7 +54,7 @@ func getSnapshotNamesToExpire(mgr *snapshot.Manager) ([]string, error) { func runExpireCommand(ctx context.Context, rep *repo.Repository) error { mgr := snapshot.NewManager(rep) - pmgr := snapshot.NewPolicyManager(rep) + pmgr := policy.NewPolicyManager(rep) snapshotNames, err := getSnapshotNamesToExpire(mgr) if err != nil { return err @@ -64,7 +65,7 @@ func runExpireCommand(ctx context.Context, rep *repo.Repository) error { return err } snapshots = filterHostAndUser(snapshots) - toDelete, err := snapshot.GetExpiredSnapshots(pmgr, snapshots) + toDelete, err := policy.GetExpiredSnapshots(pmgr, snapshots) if err != nil { return err } diff --git a/cli/command_snapshot_list.go b/cli/command_snapshot_list.go index 0e0bdb5ee..dc13db2d0 100644 --- a/cli/command_snapshot_list.go +++ b/cli/command_snapshot_list.go @@ -10,6 +10,7 @@ import ( "github.com/kopia/kopia/fs/repofs" "github.com/kopia/kopia/internal/units" "github.com/kopia/kopia/object" + "github.com/kopia/kopia/policy" "github.com/kopia/kopia/repo" "github.com/kopia/kopia/snapshot" ) @@ -85,12 +86,12 @@ func runSnapshotsCommand(ctx context.Context, rep *repo.Repository) error { return err } - polMgr := snapshot.NewPolicyManager(rep) + polMgr := policy.NewPolicyManager(rep) return outputManifestGroups(ctx, manifests, strings.Split(relPath, "/"), mgr, polMgr) } -func outputManifestGroups(ctx context.Context, manifests []*snapshot.Manifest, relPathParts []string, mgr *snapshot.Manager, polMgr *snapshot.PolicyManager) error { +func outputManifestGroups(ctx context.Context, manifests []*snapshot.Manifest, relPathParts []string, mgr *snapshot.Manager, polMgr *policy.Manager) error { separator := "" for _, snapshotGroup := range snapshot.GroupBySource(manifests) { src := snapshotGroup[0].Source diff --git a/internal/server/server.go b/internal/server/server.go index 305b1d820..196ffcbe2 100644 --- a/internal/server/server.go +++ b/internal/server/server.go @@ -10,6 +10,7 @@ import ( "github.com/bmizerany/pat" "github.com/kopia/kopia/internal/kopialogging" "github.com/kopia/kopia/internal/serverapi" + "github.com/kopia/kopia/policy" "github.com/kopia/kopia/repo" "github.com/kopia/kopia/snapshot" ) @@ -22,7 +23,7 @@ type Server struct { username string rep *repo.Repository snapshotManager *snapshot.Manager - policyManager *snapshot.PolicyManager + policyManager *policy.Manager mu sync.RWMutex sourceManagers map[snapshot.SourceInfo]*sourceManager @@ -128,7 +129,7 @@ func New(ctx context.Context, rep *repo.Repository, hostname string, username st username: username, rep: rep, snapshotManager: snapshot.NewManager(rep), - policyManager: snapshot.NewPolicyManager(rep), + policyManager: policy.NewPolicyManager(rep), sourceManagers: map[snapshot.SourceInfo]*sourceManager{}, uploadSemaphore: make(chan struct{}, 1), } diff --git a/internal/server/source_manager.go b/internal/server/source_manager.go index d04433363..ab0d90f09 100644 --- a/internal/server/source_manager.go +++ b/internal/server/source_manager.go @@ -7,6 +7,7 @@ import ( "github.com/kopia/kopia/fs/localfs" "github.com/kopia/kopia/internal/serverapi" + "github.com/kopia/kopia/policy" "github.com/kopia/kopia/snapshot" ) @@ -24,7 +25,7 @@ type sourceManager struct { closed chan struct{} mu sync.RWMutex - pol *snapshot.Policy + pol *policy.Policy state string nextSnapshotTime time.Time lastCompleteSnapshot *snapshot.Manifest diff --git a/internal/serverapi/serverapi.go b/internal/serverapi/serverapi.go index d3f2f4c3d..ed16675a8 100644 --- a/internal/serverapi/serverapi.go +++ b/internal/serverapi/serverapi.go @@ -4,6 +4,7 @@ import ( "time" "github.com/kopia/kopia/block" + "github.com/kopia/kopia/policy" "github.com/kopia/kopia/snapshot" ) @@ -24,7 +25,7 @@ type SourcesResponse struct { type SourceStatus struct { Source snapshot.SourceInfo `json:"source"` Status string `json:"status"` - Policy *snapshot.Policy `json:"policy"` + Policy *policy.Policy `json:"policy"` LastSnapshotSize int64 `json:"lastSnapshotSize,omitempty"` LastSnapshotTime time.Time `json:"lastSnapshotTime,omitempty"` NextSnapshotTime time.Time `json:"nextSnapshotTime,omitempty"` @@ -40,7 +41,7 @@ type SourceStatus struct { type PolicyListEntry struct { ID string `json:"id"` Target snapshot.SourceInfo `json:"target"` - Policy *snapshot.Policy `json:"policy"` + Policy *policy.Policy `json:"policy"` } // PoliciesResponse is the response of 'policies' HTTP API command. diff --git a/snapshot/expire.go b/policy/expire.go similarity index 65% rename from snapshot/expire.go rename to policy/expire.go index 3699a3794..bbc54c170 100644 --- a/snapshot/expire.go +++ b/policy/expire.go @@ -1,13 +1,15 @@ -package snapshot +package policy import ( "strings" + + "github.com/kopia/kopia/snapshot" ) // GetExpiredSnapshots computes the set of snapshot manifests that are not retained according to the policy. -func GetExpiredSnapshots(pmgr *PolicyManager, snapshots []*Manifest) ([]*Manifest, error) { - var toDelete []*Manifest - for _, snapshotGroup := range GroupBySource(snapshots) { +func GetExpiredSnapshots(pmgr *Manager, snapshots []*snapshot.Manifest) ([]*snapshot.Manifest, error) { + var toDelete []*snapshot.Manifest + for _, snapshotGroup := range snapshot.GroupBySource(snapshots) { td, err := getExpiredSnapshotsForSource(pmgr, snapshotGroup) if err != nil { return nil, err @@ -17,7 +19,7 @@ func GetExpiredSnapshots(pmgr *PolicyManager, snapshots []*Manifest) ([]*Manifes return toDelete, nil } -func getExpiredSnapshotsForSource(pmgr *PolicyManager, snapshots []*Manifest) ([]*Manifest, error) { +func getExpiredSnapshotsForSource(pmgr *Manager, snapshots []*snapshot.Manifest) ([]*snapshot.Manifest, error) { src := snapshots[0].Source pol, _, err := pmgr.GetEffectivePolicy(src) if err != nil { @@ -26,7 +28,7 @@ func getExpiredSnapshotsForSource(pmgr *PolicyManager, snapshots []*Manifest) ([ pol.RetentionPolicy.ComputeRetentionReasons(snapshots) - var toDelete []*Manifest + var toDelete []*snapshot.Manifest for _, s := range snapshots { if len(s.RetentionReasons) == 0 { log.Debugf(" deleting %v", s.StartTime) diff --git a/snapshot/policy.go b/policy/policy.go similarity index 88% rename from snapshot/policy.go rename to policy/policy.go index 6ea08fb84..e5054b383 100644 --- a/snapshot/policy.go +++ b/policy/policy.go @@ -1,4 +1,4 @@ -package snapshot +package policy import ( "bytes" @@ -6,6 +6,7 @@ import ( "errors" "github.com/kopia/kopia/fs/ignorefs" + "github.com/kopia/kopia/snapshot" ) // ErrPolicyNotFound is returned when the policy is not found. @@ -36,9 +37,9 @@ func (p *Policy) ID() string { return p.Labels["id"] } -// Target returns the SourceInfo describing username, host and path targeted by the policy. -func (p *Policy) Target() SourceInfo { - return SourceInfo{ +// Target returns the snapshot.SourceInfo describing username, host and path targeted by the policy. +func (p *Policy) Target() snapshot.SourceInfo { + return snapshot.SourceInfo{ Host: p.Labels["hostname"], UserName: p.Labels["username"], Path: p.Labels["path"], diff --git a/snapshot/policy_manager.go b/policy/policy_manager.go similarity index 78% rename from snapshot/policy_manager.go rename to policy/policy_manager.go index 9c1e86de7..e4a1e01b8 100644 --- a/snapshot/policy_manager.go +++ b/policy/policy_manager.go @@ -1,4 +1,5 @@ -package snapshot +// Package policy implements management of snapshot policies. +package policy import ( "fmt" @@ -6,20 +7,26 @@ import ( "strings" "github.com/kopia/kopia/fs/ignorefs" - + "github.com/kopia/kopia/internal/kopialogging" "github.com/kopia/kopia/manifest" "github.com/kopia/kopia/repo" + "github.com/kopia/kopia/snapshot" ) -// PolicyManager manages snapshotting policies. -type PolicyManager struct { +// GlobalPolicySourceInfo is a source where global policy is attached. +var GlobalPolicySourceInfo = snapshot.SourceInfo{} + +var log = kopialogging.Logger("kopia/policy") + +// Manager manages snapshotting policies. +type Manager struct { repository *repo.Repository } // GetEffectivePolicy calculates effective snapshot policy for a given source by combining the source-specifc policy (if any) // with parent policies. The source must contain a path. // Returns the effective policies and all source policies that contributed to that (most specific first). -func (m *PolicyManager) GetEffectivePolicy(si SourceInfo) (*Policy, []*Policy, error) { +func (m *Manager) GetEffectivePolicy(si snapshot.SourceInfo) (*Policy, []*Policy, error) { var md []*manifest.EntryMetadata // Find policies applying to paths all the way up to the root. @@ -36,10 +43,10 @@ func (m *PolicyManager) GetEffectivePolicy(si SourceInfo) (*Policy, []*Policy, e } // Try user@host policy - md = append(md, m.repository.Manifests.Find(labelsForSource(SourceInfo{Host: si.Host, UserName: si.UserName}))...) + md = append(md, m.repository.Manifests.Find(labelsForSource(snapshot.SourceInfo{Host: si.Host, UserName: si.UserName}))...) // Try host-level policy. - md = append(md, m.repository.Manifests.Find(labelsForSource(SourceInfo{Host: si.Host}))...) + md = append(md, m.repository.Manifests.Find(labelsForSource(snapshot.SourceInfo{Host: si.Host}))...) // Global policy. globalManifests := m.repository.Manifests.Find(labelsForSource(GlobalPolicySourceInfo)) @@ -62,8 +69,8 @@ func (m *PolicyManager) GetEffectivePolicy(si SourceInfo) (*Policy, []*Policy, e return merged, policies, nil } -// GetDefinedPolicy returns the policy defined on the provided SourceInfo or ErrPolicyNotFound if not present. -func (m *PolicyManager) GetDefinedPolicy(si SourceInfo) (*Policy, error) { +// GetDefinedPolicy returns the policy defined on the provided snapshot.SourceInfo or ErrPolicyNotFound if not present. +func (m *Manager) GetDefinedPolicy(si snapshot.SourceInfo) (*Policy, error) { md := m.repository.Manifests.Find(labelsForSource(si)) if len(md) == 0 { @@ -95,7 +102,7 @@ func (m *PolicyManager) GetDefinedPolicy(si SourceInfo) (*Policy, error) { } // SetPolicy sets the policy on a given source. -func (m *PolicyManager) SetPolicy(si SourceInfo, pol *Policy) error { +func (m *Manager) SetPolicy(si snapshot.SourceInfo, pol *Policy) error { md := m.repository.Manifests.Find(labelsForSource(si)) if _, err := m.repository.Manifests.Put(labelsForSource(si), pol); err != nil { @@ -110,7 +117,7 @@ func (m *PolicyManager) SetPolicy(si SourceInfo, pol *Policy) error { } // RemovePolicy removes the policy for a given source. -func (m *PolicyManager) RemovePolicy(si SourceInfo) error { +func (m *Manager) RemovePolicy(si snapshot.SourceInfo) error { md := m.repository.Manifests.Find(labelsForSource(si)) for _, em := range md { m.repository.Manifests.Delete(em.ID) @@ -120,7 +127,7 @@ func (m *PolicyManager) RemovePolicy(si SourceInfo) error { } // GetPolicyByID gets the policy for a given unique ID or ErrPolicyNotFound if not found. -func (m *PolicyManager) GetPolicyByID(id string) (*Policy, error) { +func (m *Manager) GetPolicyByID(id string) (*Policy, error) { p := &Policy{} if err := m.repository.Manifests.Get(id, &p); err != nil { if err == manifest.ErrNotFound { @@ -132,7 +139,7 @@ func (m *PolicyManager) GetPolicyByID(id string) (*Policy, error) { } // ListPolicies returns a list of all policies. -func (m *PolicyManager) ListPolicies() ([]*Policy, error) { +func (m *Manager) ListPolicies() ([]*Policy, error) { ids := m.repository.Manifests.Find(map[string]string{ "type": "policy", }) @@ -160,7 +167,7 @@ func (m *PolicyManager) ListPolicies() ([]*Policy, error) { } // FilesPolicyGetter returns ignorefs.FilesPolicyGetter for a given source. -func (m *PolicyManager) FilesPolicyGetter(si SourceInfo) (ignorefs.FilesPolicyGetter, error) { +func (m *Manager) FilesPolicyGetter(si snapshot.SourceInfo) (ignorefs.FilesPolicyGetter, error) { result := ignorefs.FilesPolicyMap{} pol, _, err := m.GetEffectivePolicy(si) @@ -208,7 +215,7 @@ func (m *PolicyManager) FilesPolicyGetter(si SourceInfo) (ignorefs.FilesPolicyGe return result, nil } -func labelsForSource(si SourceInfo) map[string]string { +func labelsForSource(si snapshot.SourceInfo) map[string]string { switch { case si.Path != "": return map[string]string{ @@ -241,6 +248,6 @@ func labelsForSource(si SourceInfo) map[string]string { } // NewPolicyManager creates new policy manager for a given repository. -func NewPolicyManager(r *repo.Repository) *PolicyManager { - return &PolicyManager{r} +func NewPolicyManager(r *repo.Repository) *Manager { + return &Manager{r} } diff --git a/snapshot/retention_policy.go b/policy/retention_policy.go similarity index 91% rename from snapshot/retention_policy.go rename to policy/retention_policy.go index b664221ae..f78f4e490 100644 --- a/snapshot/retention_policy.go +++ b/policy/retention_policy.go @@ -1,8 +1,10 @@ -package snapshot +package policy import ( "fmt" "time" + + "github.com/kopia/kopia/snapshot" ) // RetentionPolicy describes snapshot retention policy. @@ -17,7 +19,7 @@ type RetentionPolicy struct { // ComputeRetentionReasons computes the reasons why each snapshot is retained, based on // the settings in retention policy and stores them in RetentionReason field. -func (r *RetentionPolicy) ComputeRetentionReasons(manifests []*Manifest) { +func (r *RetentionPolicy) ComputeRetentionReasons(manifests []*snapshot.Manifest) { now := time.Now() maxTime := now.Add(365 * 24 * time.Hour) @@ -40,12 +42,12 @@ func (r *RetentionPolicy) ComputeRetentionReasons(manifests []*Manifest) { ids := make(map[string]bool) idCounters := make(map[string]int) - for i, s := range SortByTime(manifests, true) { + for i, s := range snapshot.SortByTime(manifests, true) { s.RetentionReasons = r.getRetentionReasons(i, s, cutoff, ids, idCounters) } } -func (r *RetentionPolicy) getRetentionReasons(i int, s *Manifest, cutoff cutoffTimes, ids map[string]bool, idCounters map[string]int) []string { +func (r *RetentionPolicy) getRetentionReasons(i int, s *snapshot.Manifest, cutoff cutoffTimes, ids map[string]bool, idCounters map[string]int) []string { if s.IncompleteReason != "" { return nil } diff --git a/snapshot/scheduling_policy.go b/policy/scheduling_policy.go similarity index 99% rename from snapshot/scheduling_policy.go rename to policy/scheduling_policy.go index 28d98f7c6..bdc312939 100644 --- a/snapshot/scheduling_policy.go +++ b/policy/scheduling_policy.go @@ -1,4 +1,4 @@ -package snapshot +package policy import ( "fmt" diff --git a/snapshot/manager.go b/snapshot/manager.go index 4846ef977..84d91f24a 100644 --- a/snapshot/manager.go +++ b/snapshot/manager.go @@ -8,9 +8,6 @@ import ( var log = kopialogging.Logger("kopia/snapshot") -// GlobalPolicySourceInfo is a source where global policy is attached. -var GlobalPolicySourceInfo = SourceInfo{} - // Manager manages filesystem snapshots. type Manager struct { Repository *repo.Repository diff --git a/snapshot/source.go b/snapshot/source.go index fc4c8b851..6dcb7f500 100644 --- a/snapshot/source.go +++ b/snapshot/source.go @@ -30,7 +30,7 @@ func (ssi SourceInfo) String() string { // or may be 'username@host:path' where path, username and host are not processed. func ParseSourceInfo(path string, hostname string, username string) (SourceInfo, error) { if path == "(global)" { - return GlobalPolicySourceInfo, nil + return SourceInfo{}, nil } p1 := strings.Index(path, "@")