From fbffab8bc9cccd357a98b2cf9a6069f94877bb29 Mon Sep 17 00:00:00 2001 From: Jarek Kowalski Date: Sat, 29 Sep 2018 09:51:15 +0200 Subject: [PATCH] cli: create global policy when repository is created --- cli/command_repository_create.go | 78 +++++++++++++++++++++++- tests/end_to_end_test/end_to_end_test.go | 34 ++++++++++- 2 files changed, 106 insertions(+), 6 deletions(-) diff --git a/cli/command_repository_create.go b/cli/command_repository_create.go index 57cc479ac..47b3b99bb 100644 --- a/cli/command_repository_create.go +++ b/cli/command_repository_create.go @@ -4,8 +4,11 @@ "context" "errors" "fmt" + "strings" + "github.com/kopia/kopia/fs/ignorefs" "github.com/kopia/kopia/internal/units" + "github.com/kopia/kopia/policy" "github.com/kopia/kopia/repo" "github.com/kopia/kopia/repo/block" "github.com/kopia/kopia/repo/object" @@ -25,6 +28,18 @@ createOverwrite = createCommand.Flag("overwrite", "Overwrite existing data (DANGEROUS).").Bool() createOnly = createCommand.Flag("create-only", "Create repository, but don't connect to it.").Short('c').Bool() + + createGlobalPolicyKeepLatest = createCommand.Flag("keep-latest", "Number of most recent backups to keep per source").PlaceHolder("N").Default("10").Int() + createGlobalPolicyKeepHourly = createCommand.Flag("keep-hourly", "Number of most-recent hourly backups to keep per source").PlaceHolder("N").Default("48").Int() + createGlobalPolicyKeepDaily = createCommand.Flag("keep-daily", "Number of most-recent daily backups to keep per source").PlaceHolder("N").Default("14").Int() + createGlobalPolicyKeepWeekly = createCommand.Flag("keep-weekly", "Number of most-recent weekly backups to keep per source").PlaceHolder("N").Default("25").Int() + createGlobalPolicyKeepMonthly = createCommand.Flag("keep-monthly", "Number of most-recent monthly backups to keep per source").PlaceHolder("N").Default("24").Int() + createGlobalPolicyKeepAnnual = createCommand.Flag("keep-annual", "Number of most-recent annual backups to keep per source").PlaceHolder("N").Default("3").Int() + + createGlobalPolicyDotIgnoreFiles = createCommand.Flag("dot-ignore", "List of dotfiles to look for ignore rules").Default(".kopiaignore").Strings() + + createGlobalPolicyInterval = createCommand.Flag("snapshot-interval", "Interval between snapshots").Duration() + createGlobalPolicyTimesOfDay = createCommand.Flag("snapshot-time", "Times of day when to take snapshot (HH:mm)").Strings() ) func init() { @@ -65,7 +80,7 @@ func runCreateCommandWithStorage(ctx context.Context, st storage.Storage) error options := newRepositoryOptionsFromFlags() - creds := mustGetPasswordFromFlags(true, false) + password := mustGetPasswordFromFlags(true, false) printStderr("Initializing repository with:\n") printStderr(" metadata encryption: %v\n", options.MetadataEncryptionAlgorithm) @@ -84,7 +99,7 @@ func runCreateCommandWithStorage(ctx context.Context, st storage.Storage) error printStderr(" object splitter: NEVER\n") } - if err := repo.Initialize(ctx, st, options, creds); err != nil { + if err := repo.Initialize(ctx, st, options, password); err != nil { return fmt.Errorf("cannot initialize repository: %v", err) } @@ -92,5 +107,62 @@ func runCreateCommandWithStorage(ctx context.Context, st storage.Storage) error return nil } - return runConnectCommandWithStorageAndPassword(ctx, st, creds) + if err := runConnectCommandWithStorageAndPassword(ctx, st, password); err != nil { + return fmt.Errorf("unable to connect to repository: %v", err) + } + + return populateRepository(ctx, password) +} + +func populateRepository(ctx context.Context, password string) error { + rep, err := repo.Open(ctx, repositoryConfigFileName(), password, applyOptionsFromFlags(nil)) + if err != nil { + return fmt.Errorf("unable to open repository: %v", err) + } + defer rep.Close(ctx) //nolint:errcheck + + globalPolicy, err := getInitialGlobalPolicy() + if err != nil { + return fmt.Errorf("unable to initialize global policy: %v", err) + } + + if err := policy.SetPolicy(ctx, rep, policy.GlobalPolicySourceInfo, globalPolicy); err != nil { + return fmt.Errorf("unable to set global policy: %v", err) + } + + printPolicy(globalPolicy, nil) + return nil +} + +func getInitialGlobalPolicy() (*policy.Policy, error) { + var sp policy.SchedulingPolicy + + sp.SetInterval(*createGlobalPolicyInterval) + var timesOfDay []policy.TimeOfDay + + for _, tods := range *createGlobalPolicyTimesOfDay { + for _, tod := range strings.Split(tods, ",") { + var timeOfDay policy.TimeOfDay + if err := timeOfDay.Parse(tod); err != nil { + return nil, fmt.Errorf("unable to parse time of day: %v", err) + } + timesOfDay = append(timesOfDay, timeOfDay) + } + } + sp.TimesOfDay = policy.SortAndDedupeTimesOfDay(timesOfDay) + + return &policy.Policy{ + FilesPolicy: ignorefs.FilesPolicy{ + DotIgnoreFiles: *createGlobalPolicyDotIgnoreFiles, + }, + RetentionPolicy: policy.RetentionPolicy{ + KeepLatest: createGlobalPolicyKeepLatest, + KeepHourly: createGlobalPolicyKeepHourly, + KeepDaily: createGlobalPolicyKeepDaily, + KeepWeekly: createGlobalPolicyKeepWeekly, + KeepMonthly: createGlobalPolicyKeepMonthly, + KeepAnnual: createGlobalPolicyKeepAnnual, + }, + SchedulingPolicy: sp, + }, nil } diff --git a/tests/end_to_end_test/end_to_end_test.go b/tests/end_to_end_test/end_to_end_test.go index 9d02efb71..58baf0bbc 100644 --- a/tests/end_to_end_test/end_to_end_test.go +++ b/tests/end_to_end_test/end_to_end_test.go @@ -2,6 +2,7 @@ import ( "encoding/hex" + "fmt" "io" "io/ioutil" "math/rand" @@ -95,6 +96,20 @@ func TestEndToEnd(t *testing.T) { defer e.runAndExpectSuccess(t, "repo", "disconnect") e.runAndExpectSuccess(t, "repo", "create", "filesystem", "--path", e.repoDir) + + // make sure we can read policy + e.runAndExpectSuccess(t, "policy", "show", "--global") + + // verify we created global policy entry + globalPolicyBlockID := e.runAndVerifyOutputLineCount(t, 1, "block", "ls")[0] + e.runAndExpectSuccess(t, "block", "show", "-jz", globalPolicyBlockID) + + // make sure the policy is visible in the manifest list + e.runAndVerifyOutputLineCount(t, 1, "manifest", "list", "--filter=type:policy", "--filter=policyType:global") + + // make sure the policy is visible in the policy list + e.runAndVerifyOutputLineCount(t, 1, "policy", "list") + e.runAndExpectSuccess(t, "repo", "disconnect") e.runAndExpectSuccess(t, "repo", "connect", "filesystem", "--path", e.repoDir) e.runAndExpectSuccess(t, "repo", "status") @@ -116,8 +131,8 @@ func TestEndToEnd(t *testing.T) { t.Errorf("unexpected number of sources: %v, want %v in %#v", got, want, sources) } - // expect 5 blocks, each snapshot creation adds one index block. - e.runAndVerifyOutputLineCount(t, 5, "blockindex", "ls") + // expect 5 blocks, each snapshot creation adds one index block + e.runAndVerifyOutputLineCount(t, 6, "blockindex", "ls") e.runAndExpectSuccess(t, "blockindex", "optimize") e.runAndVerifyOutputLineCount(t, 1, "blockindex", "ls") @@ -171,10 +186,23 @@ func (e *testenv) run(t *testing.T, args ...string) ([]string, error) { c := exec.Command(e.exe, cmdArgs...) c.Env = append(os.Environ(), e.environment...) o, err := c.CombinedOutput() - t.Logf("finished 'kopia %v' with err=%v and output:\n%v", strings.Join(args, " "), err, string(o)) + t.Logf("finished 'kopia %v' with err=%v and output:\n%v", strings.Join(args, " "), err, trimOutput(string(o))) return splitLines(string(o)), err } +func trimOutput(s string) string { + lines := splitLines(s) + if len(lines) <= 100 { + return s + } + + lines2 := append([]string(nil), lines[0:50]...) + lines2 = append(lines2, fmt.Sprintf("/* %v lines removed */", len(lines)-100)) + lines2 = append(lines2, lines[len(lines)-50:]...) + + return strings.Join(lines2, "\n") + +} func listSnapshotsAndExpectSuccess(t *testing.T, e *testenv, targets ...string) []sourceInfo { lines := e.runAndExpectSuccess(t, append([]string{"snapshot", "list"}, targets...)...) return mustParseSnapshots(t, lines)