From 4faf3cd9d02493d1223bc6c3094da239ec9690bc Mon Sep 17 00:00:00 2001 From: Jarek Kowalski Date: Sat, 16 Jun 2018 12:17:58 -0700 Subject: [PATCH] finalized CLI to manipulate policies --- cli/command_policy.go | 6 +- cli/command_policy_ls.go | 7 +- cli/command_policy_remove.go | 2 +- cli/command_policy_set.go | 163 +++++++++++++++++++-------- cli/command_policy_show.go | 162 +++++++++++++++++++------- cli/command_snapshot_create.go | 2 +- cli/command_snapshot_estimate.go | 2 +- cli/command_snapshot_expire.go | 2 +- cli/command_snapshot_list.go | 2 +- internal/server/api_policy_list.go | 8 +- internal/server/api_snapshot_list.go | 2 +- internal/server/source_manager.go | 2 +- snapshot/policy.go | 4 +- snapshot/policy_manager.go | 31 ++++- snapshot/scheduling_policy.go | 53 ++++++++- 15 files changed, 338 insertions(+), 110 deletions(-) diff --git a/cli/command_policy.go b/cli/command_policy.go index c43194992..deee19543 100644 --- a/cli/command_policy.go +++ b/cli/command_policy.go @@ -6,7 +6,7 @@ "github.com/kopia/kopia/snapshot" ) -func policyTargets(globalFlag *bool, targetsFlag *[]string) ([]snapshot.SourceInfo, error) { +func policyTargets(pmgr *snapshot.PolicyManager, globalFlag *bool, targetsFlag *[]string) ([]snapshot.SourceInfo, error) { if *globalFlag == (len(*targetsFlag) > 0) { return nil, fmt.Errorf("must pass either '--global' or a list of path targets") } @@ -19,6 +19,10 @@ func policyTargets(globalFlag *bool, targetsFlag *[]string) ([]snapshot.SourceIn var res []snapshot.SourceInfo for _, ts := range *targetsFlag { + if t, err := pmgr.GetPolicyByID(ts); err == nil { + res = append(res, t.Target()) + continue + } target, err := snapshot.ParseSourceInfo(ts, getHostName(), getUserName()) if err != nil { return nil, err diff --git a/cli/command_policy_ls.go b/cli/command_policy_ls.go index 3d5afddf7..36ae2218c 100644 --- a/cli/command_policy_ls.go +++ b/cli/command_policy_ls.go @@ -3,6 +3,7 @@ import ( "context" "fmt" + "sort" "github.com/kopia/kopia/repo" "github.com/kopia/kopia/snapshot" @@ -24,8 +25,12 @@ func listPolicies(ctx context.Context, rep *repo.Repository) error { return err } + sort.Slice(policies, func(i, j int) bool { + return policies[i].Target().String() < policies[j].Target().String() + }) + for _, pol := range policies { - fmt.Println(pol.Labels) + fmt.Println(pol.ID(), pol.Target()) } return nil diff --git a/cli/command_policy_remove.go b/cli/command_policy_remove.go index d6964e135..22547045f 100644 --- a/cli/command_policy_remove.go +++ b/cli/command_policy_remove.go @@ -22,7 +22,7 @@ func init() { func removePolicy(ctx context.Context, rep *repo.Repository) error { mgr := snapshot.NewPolicyManager(rep) - targets, err := policyTargets(policyRemoveGlobal, policyRemoveTargets) + targets, err := policyTargets(mgr, policyRemoveGlobal, policyRemoveTargets) if err != nil { return err } diff --git a/cli/command_policy_set.go b/cli/command_policy_set.go index c275a87b9..e3cb5eb42 100644 --- a/cli/command_policy_set.go +++ b/cli/command_policy_set.go @@ -3,10 +3,10 @@ import ( "context" "fmt" + "os" "sort" "strconv" - - "github.com/rs/zerolog/log" + "strings" "github.com/kopia/kopia/repo" "github.com/kopia/kopia/snapshot" @@ -18,7 +18,8 @@ policySetGlobal = policySetCommand.Flag("global", "Set global policy").Bool() // Frequency - policySetFrequency = policySetCommand.Flag("min-duration-between-backups", "Minimum duration between snapshots").DurationList() + policySetInterval = policySetCommand.Flag("snapshot-interval", "Interval between snapshots").DurationList() + policySetTimesOfDay = policySetCommand.Flag("snapshot-time", "Times of day when to take snapshot (HH:mm)").Strings() // Expiration policies. policySetKeepLatest = policySetCommand.Flag("keep-latest", "Number of most recent backups to keep per source (or 'inherit')").PlaceHolder("N").String() @@ -37,6 +38,7 @@ policySetAddExclude = policySetCommand.Flag("add-exclude", "List of paths to add to the exclude list").PlaceHolder("PATTERN").Strings() policySetRemoveExclude = policySetCommand.Flag("remove-exclude", "List of paths to remove from the exclude list").PlaceHolder("PATTERN").Strings() policySetClearExclude = policySetCommand.Flag("clear-exclude", "Clear list of paths in the exclude list").Bool() + policySetMaxFileSize = policySetCommand.Flag("max-file-size", "Exclude files above given size").PlaceHolder("N").String() // General policy. policySetInherit = policySetCommand.Flag("inherit", "Enable or disable inheriting policies from the parent").BoolList() @@ -49,7 +51,7 @@ func init() { func setPolicy(ctx context.Context, rep *repo.Repository) error { mgr := snapshot.NewPolicyManager(rep) - targets, err := policyTargets(policySetGlobal, policySetTargets) + targets, err := policyTargets(mgr, policySetGlobal, policySetTargets) if err != nil { return err } @@ -60,10 +62,17 @@ func setPolicy(ctx context.Context, rep *repo.Repository) error { p = &snapshot.Policy{} } - if err := setPolicyFromFlags(target, p); err != nil { + fmt.Fprintf(os.Stderr, "Setting policy for %v\n", target) + changeCount := 0 + + if err := setPolicyFromFlags(target, p, &changeCount); err != nil { return err } + if changeCount == 0 { + return fmt.Errorf("no changes specified") + } + if err := mgr.SetPolicy(target, p); err != nil { return fmt.Errorf("can't save policy for %v: %v", target, err) } @@ -72,61 +81,125 @@ func setPolicy(ctx context.Context, rep *repo.Repository) error { return nil } -func setPolicyFromFlags(target snapshot.SourceInfo, p *snapshot.Policy) error { - cases := []struct { - desc string - max **int - flagValue *string - }{ - {"number of annual backups to keep", &p.RetentionPolicy.KeepAnnual, policySetKeepAnnual}, - {"number of monthly backups to keep", &p.RetentionPolicy.KeepMonthly, policySetKeepMonthly}, - {"number of weekly backups to keep", &p.RetentionPolicy.KeepWeekly, policySetKeepWeekly}, - {"number of daily backups to keep", &p.RetentionPolicy.KeepDaily, policySetKeepDaily}, - {"number of hourly backups to keep", &p.RetentionPolicy.KeepHourly, policySetKeepHourly}, - {"number of latest backups to keep", &p.RetentionPolicy.KeepLatest, policySetKeepLatest}, +func setPolicyFromFlags(target snapshot.SourceInfo, p *snapshot.Policy, changeCount *int) error { + if err := setRetentionPolicyFromFlags(&p.RetentionPolicy, changeCount); err != nil { + return fmt.Errorf("retention policy: %v", err) } - for _, c := range cases { - if err := applyPolicyNumber(target, c.desc, c.max, *c.flagValue); err != nil { - return err - } + if err := setFilesPolicyFromFlags(&p.FilesPolicy, changeCount); err != nil { + return fmt.Errorf("files policy: %v", err) } - // It's not really a list, just optional boolean. + if err := setSchedulingPolicyFromFlags(&p.SchedulingPolicy, changeCount); err != nil { + return fmt.Errorf("scheduling policy: %v", err) + } + + if err := applyPolicyNumber("maximum file size", &p.FilesPolicy.MaxSize, *policySetMaxFileSize, changeCount); err != nil { + return fmt.Errorf("maximum file size: %v", err) + } + + // It's not really a list, just optional boolean, last one wins. for _, inherit := range *policySetInherit { + *changeCount++ p.NoParent = !inherit - break - } - - if *policySetClearExclude { - p.FilesPolicy.Exclude = nil - } else { - p.FilesPolicy.Exclude = addRemoveDedupeAndSort(p.FilesPolicy.Exclude, *policySetAddExclude, *policySetRemoveExclude) - } - if *policySetClearInclude { - p.FilesPolicy.Include = nil - } else { - p.FilesPolicy.Include = addRemoveDedupeAndSort(p.FilesPolicy.Include, *policySetAddInclude, *policySetRemoveInclude) - } - - // It's not really a list, just optional value. - for _, freq := range *policySetFrequency { - p.SchedulingPolicy.MaxFrequency = &freq - break } return nil } -func addRemoveDedupeAndSort(base, add, remove []string) []string { +func setFilesPolicyFromFlags(fp *snapshot.FilesPolicy, changeCount *int) error { + if *policySetClearExclude { + *changeCount++ + fmt.Fprintf(os.Stderr, " - removing all rules for exclude files\n") + fp.Exclude = nil + } else { + fp.Exclude = addRemoveDedupeAndSort("excluded files", fp.Exclude, *policySetAddExclude, *policySetRemoveExclude, changeCount) + } + if *policySetClearInclude { + *changeCount++ + fp.Include = nil + fmt.Fprintf(os.Stderr, " - removing all rules for include files\n") + } else { + fp.Include = addRemoveDedupeAndSort("included files", fp.Include, *policySetAddInclude, *policySetRemoveInclude, changeCount) + } + return nil +} + +func setRetentionPolicyFromFlags(rp *snapshot.RetentionPolicy, changeCount *int) error { + cases := []struct { + desc string + max **int + flagValue *string + }{ + {"number of annual backups to keep", &rp.KeepAnnual, policySetKeepAnnual}, + {"number of monthly backups to keep", &rp.KeepMonthly, policySetKeepMonthly}, + {"number of weekly backups to keep", &rp.KeepWeekly, policySetKeepWeekly}, + {"number of daily backups to keep", &rp.KeepDaily, policySetKeepDaily}, + {"number of hourly backups to keep", &rp.KeepHourly, policySetKeepHourly}, + {"number of latest backups to keep", &rp.KeepLatest, policySetKeepLatest}, + } + + for _, c := range cases { + if err := applyPolicyNumber(c.desc, c.max, *c.flagValue, changeCount); err != nil { + return err + } + } + return nil +} + +func setSchedulingPolicyFromFlags(sp *snapshot.SchedulingPolicy, changeCount *int) error { + // It's not really a list, just optional value. + for _, interval := range *policySetInterval { + *changeCount++ + sp.Interval = &interval + fmt.Fprintf(os.Stderr, " - setting snapshot interval to %v\n", sp.Interval) + break + } + + if len(*policySetTimesOfDay) > 0 { + var timesOfDay []snapshot.TimeOfDay + + for _, tods := range *policySetTimesOfDay { + for _, tod := range strings.Split(tods, ",") { + if tod == "inherit" { + timesOfDay = nil + break + } + + var timeOfDay snapshot.TimeOfDay + if err := timeOfDay.Parse(tod); err != nil { + return fmt.Errorf("unable to parse time of day: %v", err) + } + timesOfDay = append(timesOfDay, timeOfDay) + } + } + *changeCount++ + + sp.TimesOfDay = snapshot.SortAndDedupeTimesOfDay(timesOfDay) + + if timesOfDay == nil { + fmt.Fprintf(os.Stderr, " - resetting snapshot times of day to default\n") + } else { + fmt.Fprintf(os.Stderr, " - setting snapshot times to %v\n", timesOfDay) + } + } + + return nil +} + +func addRemoveDedupeAndSort(desc string, base, add, remove []string, changeCount *int) []string { entries := map[string]bool{} for _, b := range base { entries[b] = true } for _, b := range add { + *changeCount++ + fmt.Fprintf(os.Stderr, " - adding %v to %v\n", b, desc) entries[b] = true } for _, b := range remove { + *changeCount++ + fmt.Fprintf(os.Stderr, " - removing %v from %v\n", b, desc) delete(entries, b) } @@ -138,14 +211,15 @@ func addRemoveDedupeAndSort(base, add, remove []string) []string { return s } -func applyPolicyNumber(src snapshot.SourceInfo, desc string, val **int, str string) error { +func applyPolicyNumber(desc string, val **int, str string, changeCount *int) error { if str == "" { // not changed return nil } if str == "inherit" || str == "default" { - log.Printf("Resetting %v for %q to a default value inherited from parent.", desc, src) + *changeCount++ + fmt.Fprintf(os.Stderr, " - resetting %v to a default value inherited from parent.\n", desc) *val = nil return nil } @@ -156,7 +230,8 @@ func applyPolicyNumber(src snapshot.SourceInfo, desc string, val **int, str stri } i := int(v) - log.Printf("Setting %v on %q to %v.", desc, src, i) + *changeCount++ + fmt.Fprintf(os.Stderr, " - setting %v to %v.\n", desc, i) *val = &i return nil } diff --git a/cli/command_policy_show.go b/cli/command_policy_show.go index 0dfd760f6..000d6ab3c 100644 --- a/cli/command_policy_show.go +++ b/cli/command_policy_show.go @@ -1,9 +1,9 @@ package cli import ( - "bytes" "context" "fmt" + "io" "os" "github.com/kopia/kopia/internal/units" @@ -12,10 +12,10 @@ ) var ( - policyShowCommand = policyCommands.Command("show", "Show snapshot policy.").Alias("get") - policyShowEffective = policyShowCommand.Flag("effective", "Show effective policy").Bool() - policyShowGlobal = policyShowCommand.Flag("global", "Get global policy").Bool() - policyShowTargets = policyShowCommand.Arg("target", "Target to show the policy for").Strings() + policyShowCommand = policyCommands.Command("show", "Show snapshot policy.").Alias("get") + policyShowGlobal = policyShowCommand.Flag("global", "Get global policy").Bool() + policyShowTargets = policyShowCommand.Arg("target", "Target to show the policy for").Strings() + policyShowJSON = policyShowCommand.Flag("json", "Show JSON").Short('j').Bool() ) func init() { @@ -25,79 +25,155 @@ func init() { func showPolicy(ctx context.Context, rep *repo.Repository) error { pmgr := snapshot.NewPolicyManager(rep) - targets, err := policyTargets(policyShowGlobal, policyShowTargets) + targets, err := policyTargets(pmgr, policyShowGlobal, policyShowTargets) if err != nil { return err } for _, target := range targets { - var p *snapshot.Policy - var policyKind string - var err error + effective, policies, err := pmgr.GetEffectivePolicy(target) + if err != nil { + return fmt.Errorf("can't get effective policy for %q: %v", target, err) + } - if *policyShowEffective { - p, err = pmgr.GetEffectivePolicy(target) - policyKind = "effective" + if *policyShowJSON { + fmt.Println(effective) } else { - p, err = pmgr.GetDefinedPolicy(target) - policyKind = "defined" + printPolicy(os.Stdout, effective, policies) } - - if err == nil { - fmt.Printf("The %v policy for %q:\n", policyKind, target) - fmt.Println(policyToString(p)) - continue - } - - if err == snapshot.ErrPolicyNotFound { - fmt.Fprintf(os.Stderr, "No %v policy for %q, pass --effective to compute effective policy used for backups.\n", policyKind, target) - continue - } - - return fmt.Errorf("can't get %v policy for %q: %v", policyKind, target, err) } return nil } -func policyToString(p *snapshot.Policy) string { - var buf bytes.Buffer +func getDefinitionPoint(parents []*snapshot.Policy, match func(p *snapshot.Policy) bool) string { + for i, p := range parents { + if match(p) { + if i == 0 { + return "(defined for this target)" + } - fmt.Fprintf(&buf, "Retention policy:\n") - fmt.Fprintf(&buf, " keep annual:%v monthly:%v weekly:%v daily:%v hourly:%v latest:%v\n", + return "inherited from " + p.Target().String() + } + if p.NoParent { + break + } + } + + return "(default)" + +} + +func containsString(s []string, v string) bool { + for _, item := range s { + if item == v { + return true + } + } + return false +} + +func printPolicy(w io.Writer, p *snapshot.Policy, parents []*snapshot.Policy) { + fmt.Fprintf(w, "Policy for %v:\n", p.Target()) + + printRetentionPolicy(w, p, parents) + fmt.Fprintf(w, "\n") + printFilesPolicy(w, p, parents) + fmt.Fprintf(w, "\n") + printSchedulingPolicy(w, p, parents) +} + +func printRetentionPolicy(w io.Writer, p *snapshot.Policy, parents []*snapshot.Policy) { + fmt.Fprintf(w, "Keep:\n") + fmt.Fprintf(w, " Annual snapshots: %3v %v\n", valueOrNotSet(p.RetentionPolicy.KeepAnnual), + getDefinitionPoint(parents, func(pol *snapshot.Policy) bool { + return pol.RetentionPolicy.KeepAnnual != nil + })) + fmt.Fprintf(w, " Monthly snapshots: %3v %v\n", valueOrNotSet(p.RetentionPolicy.KeepMonthly), + getDefinitionPoint(parents, func(pol *snapshot.Policy) bool { + return pol.RetentionPolicy.KeepMonthly != nil + })) + fmt.Fprintf(w, " Weekly snapshots: %3v %v\n", valueOrNotSet(p.RetentionPolicy.KeepWeekly), + getDefinitionPoint(parents, func(pol *snapshot.Policy) bool { + return pol.RetentionPolicy.KeepWeekly != nil + })) + fmt.Fprintf(w, " Daily snapshots: %3v %v\n", valueOrNotSet(p.RetentionPolicy.KeepDaily), + getDefinitionPoint(parents, func(pol *snapshot.Policy) bool { + return pol.RetentionPolicy.KeepDaily != nil + })) + fmt.Fprintf(w, " Hourly snapshots: %3v %v\n", valueOrNotSet(p.RetentionPolicy.KeepHourly), + getDefinitionPoint(parents, func(pol *snapshot.Policy) bool { + return pol.RetentionPolicy.KeepHourly != nil + })) + fmt.Fprintf(w, " Latest snapshots: %3v %v\n", valueOrNotSet(p.RetentionPolicy.KeepLatest), - ) + getDefinitionPoint(parents, func(pol *snapshot.Policy) bool { + return pol.RetentionPolicy.KeepLatest != nil + })) +} - fmt.Fprintf(&buf, "Files policy:\n") +func printFilesPolicy(w io.Writer, p *snapshot.Policy, parents []*snapshot.Policy) { + fmt.Fprintf(w, "Files policy:\n") if len(p.FilesPolicy.Include) == 0 { - fmt.Fprintf(&buf, " Include all files\n") + fmt.Fprintf(w, " Include all files.\n") } else { - fmt.Fprintf(&buf, " Include only:\n") + fmt.Fprintf(w, " Include only:\n") } for _, inc := range p.FilesPolicy.Include { - fmt.Fprintf(&buf, " %v\n", inc) + fmt.Fprintf(w, " %-30v %v\n", inc, getDefinitionPoint(parents, func(pol *snapshot.Policy) bool { + return containsString(pol.FilesPolicy.Include, inc) + })) } if len(p.FilesPolicy.Exclude) > 0 { - fmt.Fprintf(&buf, " Exclude:\n") - } - for _, exc := range p.FilesPolicy.Exclude { - fmt.Fprintf(&buf, " %v\n", exc) + fmt.Fprintf(w, " Exclude:\n") + for _, exc := range p.FilesPolicy.Exclude { + fmt.Fprintf(w, " %-30v %v\n", exc, getDefinitionPoint(parents, func(pol *snapshot.Policy) bool { + return containsString(pol.FilesPolicy.Exclude, exc) + })) + } + } else { + fmt.Fprintf(w, " No excluded files.\n") } if s := p.FilesPolicy.MaxSize; s != nil { - fmt.Fprintf(&buf, " Exclude files above size: %v\n", units.BytesStringBase2(int64(*s))) + fmt.Fprintf(w, " Exclude files above: %10v %v\n", + units.BytesStringBase2(int64(*s)), + getDefinitionPoint(parents, func(pol *snapshot.Policy) bool { + return pol.FilesPolicy.MaxSize != nil + })) + } +} + +func printSchedulingPolicy(w io.Writer, p *snapshot.Policy, parents []*snapshot.Policy) { + if p.SchedulingPolicy.Interval != nil { + fmt.Fprintf(w, "Snapshot interval: %10v %v\n", p.SchedulingPolicy.Interval, getDefinitionPoint(parents, func(pol *snapshot.Policy) bool { + return pol.SchedulingPolicy.Interval != nil + })) + } + if len(p.SchedulingPolicy.TimesOfDay) > 0 { + fmt.Fprintf(w, "Snapshot times:\n") + for _, tod := range p.SchedulingPolicy.TimesOfDay { + fmt.Fprintf(w, " %9v %v\n", tod, getDefinitionPoint(parents, func(pol *snapshot.Policy) bool { + for _, t := range pol.SchedulingPolicy.TimesOfDay { + if t == tod { + return true + } + } + + return false + })) + } } - return buf.String() } func valueOrNotSet(p *int) string { if p == nil { - return "(none)" + return "-" } return fmt.Sprintf("%v", *p) diff --git a/cli/command_snapshot_create.go b/cli/command_snapshot_create.go index 22cb5953a..19032bcbd 100644 --- a/cli/command_snapshot_create.go +++ b/cli/command_snapshot_create.go @@ -90,7 +90,7 @@ func runBackupCommand(ctx context.Context, rep *repo.Repository) error { func snapshotSingleSource(ctx context.Context, rep *repo.Repository, mgr *snapshot.Manager, pmgr *snapshot.PolicyManager, u *snapshot.Uploader, sourceInfo snapshot.SourceInfo) error { t0 := time.Now() rep.Blocks.ResetStats() - policy, err := pmgr.GetEffectivePolicy(sourceInfo) + policy, _, err := pmgr.GetEffectivePolicy(sourceInfo) if err != nil { return fmt.Errorf("unable to get backup policy for source %v: %v", sourceInfo, err) } diff --git a/cli/command_snapshot_estimate.go b/cli/command_snapshot_estimate.go index ff6493568..84334d12b 100644 --- a/cli/command_snapshot_estimate.go +++ b/cli/command_snapshot_estimate.go @@ -68,7 +68,7 @@ func runSnapshotEstimateCommand(ctx context.Context, rep *repo.Repository) error } sourceInfo := snapshot.SourceInfo{Path: filepath.Clean(path), Host: getHostName(), UserName: getUserName()} - policy, err := pmgr.GetEffectivePolicy(sourceInfo) + policy, _, err := pmgr.GetEffectivePolicy(sourceInfo) if err != nil { return fmt.Errorf("unable to get backup policy for source %v: %v", sourceInfo, err) } diff --git a/cli/command_snapshot_expire.go b/cli/command_snapshot_expire.go index eb1e963e7..5db3b01d0 100644 --- a/cli/command_snapshot_expire.go +++ b/cli/command_snapshot_expire.go @@ -69,7 +69,7 @@ func expireSnapshots(pmgr *snapshot.PolicyManager, snapshots []*snapshot.Manifes func expireSnapshotsForSingleSource(pmgr *snapshot.PolicyManager, snapshots []*snapshot.Manifest) ([]string, error) { src := snapshots[0].Source - pol, err := pmgr.GetEffectivePolicy(src) + pol, _, err := pmgr.GetEffectivePolicy(src) if err != nil { return nil, err } diff --git a/cli/command_snapshot_list.go b/cli/command_snapshot_list.go index 1eeb1203d..1c5d714d7 100644 --- a/cli/command_snapshot_list.go +++ b/cli/command_snapshot_list.go @@ -98,7 +98,7 @@ func outputManifestGroups(ctx context.Context, manifests []*snapshot.Manifest, r fmt.Printf("%v%v\n", separator, src) separator = "\n" - pol, err := polMgr.GetEffectivePolicy(src) + pol, _, err := polMgr.GetEffectivePolicy(src) if err != nil { log.Warn().Msgf("unable to determine effective policy for %v", src) } else { diff --git a/internal/server/api_policy_list.go b/internal/server/api_policy_list.go index f7d24e1c5..e785628a3 100644 --- a/internal/server/api_policy_list.go +++ b/internal/server/api_policy_list.go @@ -8,7 +8,7 @@ type policyListEntry struct { ID string `json:"id"` - Source snapshot.SourceInfo `json:"source"` + Target snapshot.SourceInfo `json:"target"` Policy *snapshot.Policy `json:"policy"` } @@ -27,13 +27,13 @@ func (s *Server) handlePolicyList(r *http.Request) (interface{}, *apiError) { } for _, pol := range policies { - src := pol.Source() - if !sourceMatchesURLFilter(src, r.URL.Query()) { + target := pol.Target() + if !sourceMatchesURLFilter(target, r.URL.Query()) { continue } resp.Policies = append(resp.Policies, &policyListEntry{ ID: pol.ID(), - Source: src, + Target: target, Policy: pol, }) } diff --git a/internal/server/api_snapshot_list.go b/internal/server/api_snapshot_list.go index ca2ffcd8c..295eb72b0 100644 --- a/internal/server/api_snapshot_list.go +++ b/internal/server/api_snapshot_list.go @@ -43,7 +43,7 @@ func (s *Server) handleSourceSnapshotList(r *http.Request) (interface{}, *apiErr continue } - pol, err := s.policyManager.GetEffectivePolicy(first.Source) + pol, _, err := s.policyManager.GetEffectivePolicy(first.Source) if err == nil { pol.RetentionPolicy.ComputeRetentionReasons(grp) } diff --git a/internal/server/source_manager.go b/internal/server/source_manager.go index 7b9830cd5..41cf8e3e1 100644 --- a/internal/server/source_manager.go +++ b/internal/server/source_manager.go @@ -69,7 +69,7 @@ func (s *sourceManager) run() { func (s *sourceManager) refreshStatus() { log.Info().Msgf("refreshing status for %v", s.src) - pol, err := s.server.policyManager.GetEffectivePolicy(s.src) + pol, _, err := s.server.policyManager.GetEffectivePolicy(s.src) if err != nil { s.setStatus("FAILED") return diff --git a/snapshot/policy.go b/snapshot/policy.go index 222052388..ee6257d84 100644 --- a/snapshot/policy.go +++ b/snapshot/policy.go @@ -31,11 +31,13 @@ func (p *Policy) String() string { return buf.String() } +// ID returns globally unique identifier of the policy. func (p *Policy) ID() string { return p.Labels["id"] } -func (p *Policy) Source() SourceInfo { +// Target returns the SourceInfo describing username, host and path targeted by the policy. +func (p *Policy) Target() SourceInfo { return SourceInfo{ Host: p.Labels["hostname"], UserName: p.Labels["username"], diff --git a/snapshot/policy_manager.go b/snapshot/policy_manager.go index 7680c1132..f0bbe3b92 100644 --- a/snapshot/policy_manager.go +++ b/snapshot/policy_manager.go @@ -6,6 +6,7 @@ "github.com/kopia/kopia/manifest" "github.com/kopia/kopia/repo" + "github.com/rs/zerolog/log" ) // PolicyManager manages snapshotting policies. @@ -15,12 +16,14 @@ type PolicyManager struct { // GetEffectivePolicy calculates effective snapshot policy for a given source by combining the source-specifc policy (if any) // with parent policies. The source must contain a path. -func (m *PolicyManager) GetEffectivePolicy(si SourceInfo) (*Policy, error) { +// Returns the effective policies and all source policies that contributed to that (most specific first). +func (m *PolicyManager) GetEffectivePolicy(si SourceInfo) (*Policy, []*Policy, error) { var md []*manifest.EntryMetadata // Find policies applying to paths all the way up to the root. for tmp := si; len(si.Path) > 0; { - md = append(md, m.repository.Manifests.Find(labelsForSource(si))...) + manifests := m.repository.Manifests.Find(labelsForSource(tmp)) + md = append(md, manifests...) parentPath := filepath.Dir(tmp.Path) if parentPath == tmp.Path { @@ -37,18 +40,24 @@ func (m *PolicyManager) GetEffectivePolicy(si SourceInfo) (*Policy, error) { md = append(md, m.repository.Manifests.Find(labelsForSource(SourceInfo{Host: si.Host}))...) // Global policy. - md = append(md, m.repository.Manifests.Find(labelsForSource(GlobalPolicySourceInfo))...) + globalManifests := m.repository.Manifests.Find(labelsForSource(GlobalPolicySourceInfo)) + md = append(md, globalManifests...) var policies []*Policy for _, em := range md { p := &Policy{} if err := m.repository.Manifests.Get(em.ID, &p); err != nil { - return nil, fmt.Errorf("got unexpected error when loading policy item %v: %v", em.ID, err) + return nil, nil, fmt.Errorf("got unexpected error when loading policy item %v: %v", em.ID, err) } + p.Labels = em.Labels policies = append(policies, p) + log.Printf("loaded parent policy for %v: %v", si, p.Target()) } - return MergePolicies(policies), nil + merged := MergePolicies(policies) + merged.Labels = labelsForSource(si) + + return merged, policies, nil } // GetDefinedPolicy returns the policy defined on the provided SourceInfo or ErrPolicyNotFound if not present. @@ -108,6 +117,18 @@ func (m *PolicyManager) RemovePolicy(si SourceInfo) error { return nil } +// GetPolicyByID gets the policy for a given unique ID or ErrPolicyNotFound if not found. +func (m *PolicyManager) GetPolicyByID(id string) (*Policy, error) { + p := &Policy{} + if err := m.repository.Manifests.Get(id, &p); err != nil { + if err == manifest.ErrNotFound { + return nil, ErrPolicyNotFound + } + } + + return p, nil +} + // ListPolicies returns a list of all policies. func (m *PolicyManager) ListPolicies() ([]*Policy, error) { ids := m.repository.Manifests.Find(map[string]string{ diff --git a/snapshot/scheduling_policy.go b/snapshot/scheduling_policy.go index 7542790f4..4a7519b3c 100644 --- a/snapshot/scheduling_policy.go +++ b/snapshot/scheduling_policy.go @@ -1,16 +1,61 @@ package snapshot -import "time" +import ( + "fmt" + "sort" + "time" +) + +// TimeOfDay represents the time of day (hh:mm) using 24-hour time format. +type TimeOfDay struct { + Hour int `json:"hour"` + Minute int `json:"min"` +} + +// Parse parses the time of day. +func (t *TimeOfDay) Parse(s string) error { + if _, err := fmt.Sscanf(s, "%v:%02v", &t.Hour, &t.Minute); err != nil { + return fmt.Errorf("invalid time of day, must be HH:MM") + } + if t.Hour < 0 || t.Hour > 23 { + return fmt.Errorf("invalid hour %q, must be between 0 and 23", s) + } + if t.Minute < 0 || t.Minute > 59 { + return fmt.Errorf("invalid minute %q, must be between 0 and 59", s) + } + + return nil +} + +// TimeOfDay returns string representation of time of day. +func (t TimeOfDay) String() string { + return fmt.Sprintf("%v:%02v", t.Hour, t.Minute) +} + +// SortAndDedupeTimesOfDay sorts the slice of times of day and removes duplicates. +func SortAndDedupeTimesOfDay(tod []TimeOfDay) []TimeOfDay { + sort.Slice(tod, func(i, j int) bool { + if a, b := tod[i].Hour, tod[j].Hour; a != b { + return a < b + } + return tod[i].Minute < tod[j].Minute + }) + + return tod +} // SchedulingPolicy describes policy for scheduling snapshots. type SchedulingPolicy struct { - MaxFrequency *time.Duration `json:"frequency"` + Interval *time.Duration `json:"interval"` + TimesOfDay []TimeOfDay `json:"timeOfDay"` } func mergeSchedulingPolicy(dst, src *SchedulingPolicy) { - if dst.MaxFrequency == nil { - dst.MaxFrequency = src.MaxFrequency + if dst.Interval == nil { + dst.Interval = src.Interval } + dst.TimesOfDay = SortAndDedupeTimesOfDay( + append(append([]TimeOfDay(nil), src.TimesOfDay...), dst.TimesOfDay...)) } var defaultSchedulingPolicy = &SchedulingPolicy{}