From 8bab3eb3f1acbeae1a36c36991ecff35efbf5264 Mon Sep 17 00:00:00 2001 From: Jarek Kowalski Date: Mon, 20 Feb 2017 17:50:20 -0800 Subject: [PATCH] policies work in progress --- cmd/kopia/command_ls.go | 2 +- cmd/kopia/command_policy.go | 5 ++ cmd/kopia/command_policy_ls.go | 32 +++++++++++ cmd/kopia/command_policy_set.go | 48 ++++++++++++++++ cmd/kopia/command_policy_show.go | 32 +++++++++++ cmd/kopia/command_vault_ls.go | 2 +- snapshot/manager.go | 96 ++++++++++++++++++++++++++++++++ snapshot/policy.go | 10 ++++ snapshot/source.go | 16 ++++++ 9 files changed, 241 insertions(+), 2 deletions(-) create mode 100644 cmd/kopia/command_policy.go create mode 100644 cmd/kopia/command_policy_ls.go create mode 100644 cmd/kopia/command_policy_set.go create mode 100644 cmd/kopia/command_policy_show.go create mode 100644 snapshot/policy.go diff --git a/cmd/kopia/command_ls.go b/cmd/kopia/command_ls.go index c87e304a6..f3939da4b 100644 --- a/cmd/kopia/command_ls.go +++ b/cmd/kopia/command_ls.go @@ -13,7 +13,7 @@ ) var ( - lsCommand = app.Command("ls", "List a directory stored in repository object.").Alias("list") + lsCommand = app.Command("list", "List a directory stored in repository object.").Alias("ls") lsCommandLong = lsCommand.Flag("long", "Long output").Short('l').Bool() lsCommandPath = lsCommand.Arg("path", "Path").Required().String() diff --git a/cmd/kopia/command_policy.go b/cmd/kopia/command_policy.go new file mode 100644 index 000000000..a17cc8724 --- /dev/null +++ b/cmd/kopia/command_policy.go @@ -0,0 +1,5 @@ +package main + +var ( + policyCommands = app.Command("policy", "Commands to manipulate snapshotting policies.") +) diff --git a/cmd/kopia/command_policy_ls.go b/cmd/kopia/command_policy_ls.go new file mode 100644 index 000000000..2abf7f45a --- /dev/null +++ b/cmd/kopia/command_policy_ls.go @@ -0,0 +1,32 @@ +package main + +import ( + "fmt" + + "github.com/kopia/kopia/snapshot" + kingpin "gopkg.in/alecthomas/kingpin.v2" +) + +var ( + policyListCommand = policyCommands.Command("list", "List policies.").Alias("ls") +) + +func init() { + policyListCommand.Action(listPolicies) +} + +func listPolicies(context *kingpin.ParseContext) error { + conn := mustOpenConnection() + mgr := snapshot.NewManager(conn) + + entries, err := mgr.ListPolicies() + if err != nil { + return err + } + + for _, e := range entries { + fmt.Println(e) + } + + return nil +} diff --git a/cmd/kopia/command_policy_set.go b/cmd/kopia/command_policy_set.go new file mode 100644 index 000000000..b3fa45a24 --- /dev/null +++ b/cmd/kopia/command_policy_set.go @@ -0,0 +1,48 @@ +package main + +import ( + "log" + + "github.com/kopia/kopia/snapshot" + kingpin "gopkg.in/alecthomas/kingpin.v2" +) + +var ( + policySetCommand = policyCommands.Command("set", "Set snapshot policy for a single directory, user@host or a global policy.") + policySetTarget = policySetCommand.Flag("target", "Target of a policy ('global','user@host','@host') or a path").Required().String() + + // Frequency + policySetFrequency = policySetCommand.Flag("min-duration-between-backups", "Minimum duration between snapshots").Duration() + + // Expiration policies. + policySetKeepLatest = policySetCommand.Flag("keep-latest", "Number of most recent backups to keep per source").Int() + policySetKeepHourly = policySetCommand.Flag("keep-hourly", "Number of most-recent hourly backups to keep per source").Int() + policySetKeepDaily = policySetCommand.Flag("keep-daily", "Number of most-recent daily backups to keep per source").Int() + policySetKeepWeekly = policySetCommand.Flag("keep-weekly", "Number of most-recent weekly backups to keep per source").Int() + policySetKeepMonthly = policySetCommand.Flag("keep-monthly", "Number of most-recent monthly backups to keep per source").Int() + policySetKeepAnnual = policySetCommand.Flag("keep-annual", "Number of most-recent annual backups to keep per source").Int() + + // Files to ignore. + policySetAddIgnore = policySetCommand.Flag("add-ignore", "List of paths to add to ignore list").Strings() + policySetRemoveIgnore = policySetCommand.Flag("remove-ignore", "List of paths to remove from ignore list").Strings() + policySetReplaceIgnore = policySetCommand.Flag("set-ignore", "List of paths to replace ignore list with").Strings() +) + +func init() { + policySetCommand.Action(setPolicy) +} + +func setPolicy(context *kingpin.ParseContext) error { + conn := mustOpenConnection() + mgr := snapshot.NewManager(conn) + _ = mgr + + target, err := snapshot.ParseSourceInfo(*policySetTarget, getHostName(), getUserName()) + if err != nil { + return err + } + + log.Printf("target: %v", target) + + return nil +} diff --git a/cmd/kopia/command_policy_show.go b/cmd/kopia/command_policy_show.go new file mode 100644 index 000000000..a1c3c6ca5 --- /dev/null +++ b/cmd/kopia/command_policy_show.go @@ -0,0 +1,32 @@ +package main + +import ( + "fmt" + + "github.com/kopia/kopia/snapshot" + kingpin "gopkg.in/alecthomas/kingpin.v2" +) + +var ( + policyShowCommand = policyCommands.Command("show", "Show snapshot policy.") +) + +func init() { + policyShowCommand.Action(showPolicy) +} + +func showPolicy(context *kingpin.ParseContext) error { + conn := mustOpenConnection() + mgr := snapshot.NewManager(conn) + + entries, err := mgr.ListPolicies() + if err != nil { + return err + } + + for _, e := range entries { + fmt.Println(e) + } + + return nil +} diff --git a/cmd/kopia/command_vault_ls.go b/cmd/kopia/command_vault_ls.go index baa7a7646..3c8a2150a 100644 --- a/cmd/kopia/command_vault_ls.go +++ b/cmd/kopia/command_vault_ls.go @@ -7,7 +7,7 @@ ) var ( - vaultListCommand = vaultCommands.Command("ls", "List contents of a vault").Alias("list") + vaultListCommand = vaultCommands.Command("list", "List contents of a vault").Alias("ls") vaultListPrefix = vaultListCommand.Flag("prefix", "Prefix").String() ) diff --git a/snapshot/manager.go b/snapshot/manager.go index 795f5c048..245bd1e16 100644 --- a/snapshot/manager.go +++ b/snapshot/manager.go @@ -8,12 +8,18 @@ "math" "strings" + "errors" + "github.com/kopia/kopia" "github.com/kopia/kopia/vault" ) const sourcePrefix = "S" const backupPrefix = "B" +const policyPrefix = "P" + +// ErrPolicyNotFound is returned when the policy is not found. +var ErrPolicyNotFound = errors.New("policy not found") // Manager manages filesystem snapshots. type Manager struct { @@ -134,6 +140,96 @@ func (m *Manager) ListSnapshotManifests(src *SourceInfo, limit int) ([]string, e return m.vault.List(backupPrefix+prefix, limit) } +// GetPolicy loads snapshot policy for a given source, optionally fall back to default. +func (m *Manager) GetPolicy(src *SourceInfo, fallback bool) (*Policy, error) { + if p, err := m.getRawPolicy(src); err != ErrPolicyNotFound { + return p, err + } + + if !fallback { + return nil, ErrPolicyNotFound + } + + if src.Path != "" { + userHostDefault := *src + userHostDefault.Path = "" + + if p, err := m.getRawPolicy(&userHostDefault); err != ErrPolicyNotFound { + return p, nil + } + } + + return m.getRawPolicy(&SourceInfo{"", "", ""}) +} + +// SavePolicy persists the given snapshot policy. +func (m *Manager) SavePolicy(p *Policy) error { + itemID := fmt.Sprintf("%v%v", policyPrefix, p.Source.HashString()) + + b, err := json.Marshal(p) + if err != nil { + return fmt.Errorf("cannot marshal policy to JSON: %v", err) + } + + return m.vault.Put(itemID, b) +} + +func (m *Manager) getRawPolicy(src *SourceInfo) (*Policy, error) { + itemID := fmt.Sprintf("%v%v", policyPrefix, src.HashString()) + + return m.getPolicyItem(itemID) +} + +func (m *Manager) getPolicyItem(itemID string) (*Policy, error) { + b, err := m.vault.Get(itemID) + if err == vault.ErrItemNotFound { + return nil, ErrPolicyNotFound + } + + if err != nil { + return nil, err + } + + var s Policy + if err := json.Unmarshal(b, &s); err != nil { + return nil, fmt.Errorf("invalid policy: %v", err) + } + + return &s, nil +} + +// ListPolicies returns a list of all policies stored in a vault. +func (m *Manager) ListPolicies() ([]*Policy, error) { + names, err := m.vault.List(policyPrefix, -1) + if err != nil { + return nil, err + } + + result := make([]*Policy, len(names)) + sem := make(chan bool, 50) + + for i, n := range names { + sem <- true + go func(i int, n string) { + defer func() { <-sem }() + + p, err := m.getPolicyItem(n) + if err != nil { + log.Printf("WARNING: Unable to parse policy %v: %v", n, err) + return + } + result[i] = p + }(i, n) + } + + for i := 0; i < cap(sem); i++ { + sem <- true + } + close(sem) + + return result, nil +} + // NewManager creates new snapshot manager for a given connection. func NewManager(conn *kopia.Connection) *Manager { return &Manager{conn.Vault} diff --git a/snapshot/policy.go b/snapshot/policy.go new file mode 100644 index 000000000..700e923ce --- /dev/null +++ b/snapshot/policy.go @@ -0,0 +1,10 @@ +package snapshot + +// Expiration describes snapshot expiration policy. +type Expiration struct { +} + +// Policy describes snapshot policy for a single source. +type Policy struct { + Source *SourceInfo +} diff --git a/snapshot/source.go b/snapshot/source.go index 3fb0989bb..7a4837ebd 100644 --- a/snapshot/source.go +++ b/snapshot/source.go @@ -26,6 +26,10 @@ func (ssi SourceInfo) String() string { // SourceInfo. The path may be bare (in which case it's interpreted as local path and canonicalized) // or may be 'username@host:path' where path, username and host are not processed. func ParseSourceInfo(path string, hostname string, username string) (SourceInfo, error) { + if path == "(global)" { + return SourceInfo{}, nil + } + p1 := strings.Index(path, "@") p2 := strings.Index(path, ":") @@ -37,6 +41,18 @@ func ParseSourceInfo(path string, hostname string, username string) (SourceInfo, }, nil } + if p1 >= 0 && p2 < 0 { + if p1+1 < len(path) { + // support @host and user@host without path + return SourceInfo{ + UserName: path[0:p1], + Host: path[p1+1:], + }, nil + } + + return SourceInfo{}, fmt.Errorf("invalid hostname in %q", path) + } + absPath, err := filepath.Abs(path) if err != nil { return SourceInfo{}, fmt.Errorf("invalid directory: '%s': %s", path, err)