diff --git a/cli/command_policy.go b/cli/command_policy.go index 886b3b816..3e31b564f 100644 --- a/cli/command_policy.go +++ b/cli/command_policy.go @@ -13,11 +13,13 @@ ) type commandPolicy struct { - edit commandPolicyEdit - list commandPolicyList - delete commandPolicyDelete - set commandPolicySet - show commandPolicyShow + edit commandPolicyEdit + list commandPolicyList + delete commandPolicyDelete + set commandPolicySet + show commandPolicyShow + export commandPolicyExport + pImport commandPolicyImport } func (c *commandPolicy) setup(svc appServices, parent commandParent) { @@ -28,6 +30,8 @@ func (c *commandPolicy) setup(svc appServices, parent commandParent) { c.delete.setup(svc, cmd) c.set.setup(svc, cmd) c.show.setup(svc, cmd) + c.export.setup(svc, cmd) + c.pImport.setup(svc, cmd) } type policyTargetFlags struct { diff --git a/cli/command_policy_export.go b/cli/command_policy_export.go new file mode 100644 index 000000000..4c8f6c1cc --- /dev/null +++ b/cli/command_policy_export.go @@ -0,0 +1,123 @@ +package cli + +import ( + "context" + "encoding/json" + "fmt" + "io" + "os" + + "github.com/pkg/errors" + + "github.com/kopia/kopia/internal/impossible" + "github.com/kopia/kopia/repo" + "github.com/kopia/kopia/snapshot/policy" +) + +type commandPolicyExport struct { + policyTargetFlags + filePath string + overwrite bool + + jsonIndent bool + + svc appServices +} + +const exportFilePerms = 0o600 + +func (c *commandPolicyExport) setup(svc appServices, parent commandParent) { + cmd := parent.Command("export", "Exports the policy to the specified file, or to stdout if none is specified.") + cmd.Flag("to-file", "File path to export to").StringVar(&c.filePath) + cmd.Flag("overwrite", "Overwrite the file if it exists").BoolVar(&c.overwrite) + + cmd.Flag("json-indent", "Output result in indented JSON format").Hidden().BoolVar(&c.jsonIndent) + + c.policyTargetFlags.setup(cmd) + + c.svc = svc + + cmd.Action(svc.repositoryReaderAction(c.run)) +} + +func (c *commandPolicyExport) run(ctx context.Context, rep repo.Repository) error { + output, err := getOutput(c) + if err != nil { + return err + } + + file, ok := output.(*os.File) + if ok { + defer file.Close() //nolint:errcheck + } + + policies := make(map[string]*policy.Policy) + + if c.policyTargetFlags.global || len(c.policyTargetFlags.targets) > 0 { + targets, err := c.policyTargets(ctx, rep) + if err != nil { + return err + } + + for _, target := range targets { + definedPolicy, err := policy.GetDefinedPolicy(ctx, rep, target) + if err != nil { + return errors.Wrapf(err, "can't get defined policy for %q", target) + } + + policies[target.String()] = definedPolicy + } + } else { + ps, err := policy.ListPolicies(ctx, rep) + if err != nil { + return errors.Wrap(err, "failed to list policies") + } + + for _, policy := range ps { + policies[policy.Target().String()] = policy + } + } + + var toWrite []byte + + if c.jsonIndent { + toWrite, err = json.MarshalIndent(policies, "", " ") + } else { + toWrite, err = json.Marshal(policies) + } + + impossible.PanicOnError(err) + + _, err = fmt.Fprintf(output, "%s", toWrite) + + return errors.Wrap(err, "unable to write policy to output") +} + +func getOutput(c *commandPolicyExport) (io.Writer, error) { + var err error + + if c.filePath == "" { + if c.overwrite { + return nil, errors.New("overwrite was passed but no file path was given") + } + + return c.svc.stdout(), nil + } + + var file *os.File + + if c.overwrite { + file, err = os.Create(c.filePath) + } else { + file, err = os.OpenFile(c.filePath, os.O_WRONLY|os.O_CREATE|os.O_EXCL, exportFilePerms) + if os.IsExist(err) { + return nil, errors.Wrap(err, "file already exists and overwrite flag is not set") + } + } + + if err != nil { + return nil, errors.Wrap(err, "error opening file to write to") + } + + return file, nil +} diff --git a/cli/command_policy_export_test.go b/cli/command_policy_export_test.go new file mode 100644 index 000000000..e119a66ab --- /dev/null +++ b/cli/command_policy_export_test.go @@ -0,0 +1,129 @@ +package cli_test + +import ( + "os" + "path" + "testing" + + "github.com/stretchr/testify/assert" + + "github.com/kopia/kopia/internal/testutil" + "github.com/kopia/kopia/snapshot" + "github.com/kopia/kopia/snapshot/policy" + "github.com/kopia/kopia/tests/testenv" +) + +func TestExportPolicy(t *testing.T) { + e := testenv.NewCLITest(t, testenv.RepoFormatNotImportant, testenv.NewInProcRunner(t)) + defer e.RunAndExpectSuccess(t, "repo", "disconnect") + + e.RunAndExpectSuccess(t, "repo", "create", "filesystem", "--path", e.RepoDir, "--override-username=user", "--override-hostname=host") + + // check if we get the default global policy + var policies1 map[string]*policy.Policy + + testutil.MustParseJSONLines(t, e.RunAndExpectSuccess(t, "policy", "export"), &policies1) + + assert.Len(t, policies1, 1, "unexpected number of policies") + assert.Equal(t, policy.DefaultPolicy, policies1["(global)"], "unexpected policy") + + var policies2 map[string]*policy.Policy + + // we only have one policy, so exporting all policies should be the same as exporting the global policy explicitly + testutil.MustParseJSONLines(t, e.RunAndExpectSuccess(t, "policy", "export", "(global)"), &policies2) + + assert.Len(t, policies2, 1, "unexpected number of policies") + assert.Equal(t, policies1, policies2, "unexpected policy") + + // create a new policy + td := testutil.TempDirectory(t) + id := snapshot.SourceInfo{ + Host: "host", + UserName: "user", + Path: td, + }.String() + + e.RunAndExpectSuccess(t, "policy", "set", td, "--splitter=FIXED-4M") + + expectedPolicy := &policy.Policy{ + SplitterPolicy: policy.SplitterPolicy{ + Algorithm: "FIXED-4M", + }, + } + expectedPolicies := map[string]*policy.Policy{ + "(global)": policy.DefaultPolicy, + id: expectedPolicy, + } + + // check if we get the new policy + var policies3 map[string]*policy.Policy + testutil.MustParseJSONLines(t, e.RunAndExpectSuccess(t, "policy", "export", id), &policies3) + + assert.Len(t, policies3, 1, "unexpected number of policies") + assert.Equal(t, expectedPolicy, policies3[id], "unexpected policy") + + // specifying a local id should return the same policy + var policies4 map[string]*policy.Policy + testutil.MustParseJSONLines(t, e.RunAndExpectSuccess(t, "policy", "export", td), &policies4) // note: td, not id + + assert.Len(t, policies4, 1, "unexpected number of policies") + assert.Equal(t, expectedPolicy, policies4[id], "unexpected policy") // thee key is always the full id however + + // exporting without specifying a policy should return all policies + var policies5 map[string]*policy.Policy + testutil.MustParseJSONLines(t, e.RunAndExpectSuccess(t, "policy", "export"), &policies5) + + assert.Len(t, policies5, 2, "unexpected number of policies") + assert.Equal(t, expectedPolicies, policies5, "unexpected policy") + + // sanity check if --to-file works + exportPath := path.Join(td, "exported.json") + + e.RunAndExpectSuccess(t, "policy", "export", "--to-file", exportPath) + exportedContent, err := os.ReadFile(exportPath) + if err != nil { + t.Fatalf("unable to read exported file: %v", err) + } + + var policies6 map[string]*policy.Policy + testutil.MustParseJSONLines(t, []string{string(exportedContent)}, &policies6) + + assert.Equal(t, expectedPolicies, policies6, "unexpected policy") + + // should not overwrite existing file + e.RunAndExpectFailure(t, "policy", "export", "--to-file", exportPath, id) + + // unless --overwrite is passed + e.RunAndExpectSuccess(t, "policy", "export", "--overwrite", "--to-file", exportPath, id) + + exportedContent, err = os.ReadFile(exportPath) + if err != nil { + t.Fatalf("unable to read exported file: %v", err) + } + + var policies7 map[string]*policy.Policy + testutil.MustParseJSONLines(t, []string{string(exportedContent)}, &policies7) + + // we specified id, so only that policy should be exported + assert.Len(t, policies7, 1, "unexpected number of policies") + assert.Equal(t, expectedPolicy, policies5[id], "unexpected policy") + + // pretty-printed JSON should be different but also correct + policies8prettyJSON := e.RunAndExpectSuccess(t, "policy", "export", "--json-indent") + + var policies8pretty map[string]*policy.Policy + testutil.MustParseJSONLines(t, policies8prettyJSON, &policies8pretty) + + policies8JSON := e.RunAndExpectSuccess(t, "policy", "export") + var policies8 map[string]*policy.Policy + testutil.MustParseJSONLines(t, policies8JSON, &policies8) + + assert.Equal(t, policies8, policies8pretty, "pretty-printing should not change the content") + assert.NotEqual(t, policies8JSON, policies8prettyJSON, "pretty-printed JSON should be different") + + // --overwrite and no --to-file should fail + e.RunAndExpectFailure(t, "policy", "export", "--overwrite") + + // writing to inaccessible file should fail + e.RunAndExpectFailure(t, "policy", "export", "--to-file", "/not/a/real/file/path") +} diff --git a/cli/command_policy_import.go b/cli/command_policy_import.go new file mode 100644 index 000000000..f9012d39f --- /dev/null +++ b/cli/command_policy_import.go @@ -0,0 +1,129 @@ +package cli + +import ( + "context" + "encoding/json" + "io" + "os" + "slices" + + "github.com/pkg/errors" + + "github.com/kopia/kopia/repo" + "github.com/kopia/kopia/snapshot" + "github.com/kopia/kopia/snapshot/policy" +) + +type commandPolicyImport struct { + policyTargetFlags + filePath string + allowUnknownFields bool + deleteOtherPolicies bool + + svc appServices +} + +func (c *commandPolicyImport) setup(svc appServices, parent commandParent) { + cmd := parent.Command("import", "Imports policies from a specified file, or stdin if no file is specified.") + cmd.Flag("from-file", "File path to import from").StringVar(&c.filePath) + cmd.Flag("allow-unknown-fields", "Allow unknown fields in the policy file").BoolVar(&c.allowUnknownFields) + cmd.Flag("delete-other-policies", "Delete all other policies, keeping only those that got imported").BoolVar(&c.deleteOtherPolicies) + + c.policyTargetFlags.setup(cmd) + c.svc = svc + + cmd.Action(svc.repositoryWriterAction(c.run)) +} + +func (c *commandPolicyImport) run(ctx context.Context, rep repo.RepositoryWriter) error { + var input io.Reader + + var err error + + if c.filePath != "" { + file, err := os.Open(c.filePath) + if err != nil { + return errors.Wrap(err, "unable to read policy file") + } + + defer file.Close() //nolint:errcheck + + input = file + } else { + input = c.svc.stdin() + } + + policies := make(map[string]*policy.Policy) + d := json.NewDecoder(input) + + if !c.allowUnknownFields { + d.DisallowUnknownFields() + } + + err = d.Decode(&policies) + if err != nil { + return errors.Wrap(err, "unable to decode policy file as valid json") + } + + var targetLimit []snapshot.SourceInfo + + if c.policyTargetFlags.global || len(c.policyTargetFlags.targets) > 0 { + targetLimit, err = c.policyTargets(ctx, rep) + if err != nil { + return err + } + } + + shouldImportSource := func(target snapshot.SourceInfo) bool { + if targetLimit == nil { + return true + } + + return slices.Contains(targetLimit, target) + } + + importedSources := make([]string, 0, len(policies)) + + for ts, newPolicy := range policies { + target, err := snapshot.ParseSourceInfo(ts, rep.ClientOptions().Hostname, rep.ClientOptions().Username) + if err != nil { + return errors.Wrapf(err, "unable to parse source info: %q", ts) + } + + if !shouldImportSource(target) { + continue + } + // used for deleteOtherPolicies + importedSources = append(importedSources, ts) + + if err := policy.SetPolicy(ctx, rep, target, newPolicy); err != nil { + return errors.Wrapf(err, "can't save policy for %v", target) + } + } + + if c.deleteOtherPolicies { + err := deleteOthers(ctx, rep, importedSources) + if err != nil { + return err + } + } + + return nil +} + +func deleteOthers(ctx context.Context, rep repo.RepositoryWriter, importedSources []string) error { + ps, err := policy.ListPolicies(ctx, rep) + if err != nil { + return errors.Wrap(err, "failed to list policies") + } + + for _, p := range ps { + if !slices.Contains(importedSources, p.Target().String()) { + if err := policy.RemovePolicy(ctx, rep, p.Target()); err != nil { + return errors.Wrapf(err, "can't delete policy for %v", p.Target()) + } + } + } + + return nil +} diff --git a/cli/command_policy_import_test.go b/cli/command_policy_import_test.go new file mode 100644 index 000000000..e1700065f --- /dev/null +++ b/cli/command_policy_import_test.go @@ -0,0 +1,176 @@ +package cli_test + +import ( + "encoding/json" + "os" + "path" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + + "github.com/kopia/kopia/internal/testutil" + "github.com/kopia/kopia/snapshot" + "github.com/kopia/kopia/snapshot/policy" + "github.com/kopia/kopia/tests/testenv" +) + +// note: dependent on policy export working. +func TestImportPolicy(t *testing.T) { + e := testenv.NewCLITest(t, testenv.RepoFormatNotImportant, testenv.NewInProcRunner(t)) + defer e.RunAndExpectSuccess(t, "repo", "disconnect") + + e.RunAndExpectSuccess(t, "repo", "create", "filesystem", "--path", e.RepoDir, "--override-username=user", "--override-hostname=host") + + td := testutil.TempDirectory(t) + policyFilePath := path.Join(td, "policy.json") + + // poor man's deep copy + defaultPolicyJSON, err := json.Marshal(policy.DefaultPolicy) + if err != nil { + t.Fatalf("unable to marshal policy: %v", err) + } + var defaultPolicy *policy.Policy + testutil.MustParseJSONLines(t, []string{string(defaultPolicyJSON)}, &defaultPolicy) + + specifiedPolicies := map[string]*policy.Policy{ + "(global)": defaultPolicy, + } + makePolicyFile := func() { + data, err := json.Marshal(specifiedPolicies) + if err != nil { + t.Fatalf("unable to marshal policy: %v", err) + } + + err = os.WriteFile(policyFilePath, data, 0o600) + if err != nil { + t.Fatalf("unable to write policy file: %v", err) + } + } + + // sanity check that we have the default global policy + assertPoliciesEqual(t, e, specifiedPolicies) + + // change the global policy + specifiedPolicies["(global)"].SplitterPolicy.Algorithm = "FIXED-4M" + makePolicyFile() + e.RunAndExpectSuccess(t, "policy", "import", "--from-file", policyFilePath) + assertPoliciesEqual(t, e, specifiedPolicies) + + // create a new policy + id := snapshot.SourceInfo{ + Host: "host", + UserName: "user", + Path: filepath.ToSlash(td), + }.String() + + specifiedPolicies[id] = &policy.Policy{ + SplitterPolicy: policy.SplitterPolicy{ + Algorithm: "FIXED-8M", + }, + } + makePolicyFile() + e.RunAndExpectSuccess(t, "policy", "import", "--from-file", policyFilePath) + assertPoliciesEqual(t, e, specifiedPolicies) + + // import from a file specifying changes in both policies but limiting import to only one + specifiedPolicies["(global)"].CompressionPolicy.CompressorName = "zstd" + specifiedPolicies[id].CompressionPolicy.CompressorName = "gzip" + makePolicyFile() + e.RunAndExpectSuccess(t, "policy", "import", "--from-file", policyFilePath, "(global)") + + // local policy should not have changed + specifiedPolicies[id].CompressionPolicy.CompressorName = "" + assertPoliciesEqual(t, e, specifiedPolicies) + + specifiedPolicies[id].CompressionPolicy.CompressorName = "gzip" + e.RunAndExpectSuccess(t, "policy", "import", "--from-file", policyFilePath, id) + assertPoliciesEqual(t, e, specifiedPolicies) + + // deleting values should work + specifiedPolicies[id].CompressionPolicy.CompressorName = "" + makePolicyFile() + e.RunAndExpectSuccess(t, "policy", "import", "--from-file", policyFilePath, id) + assertPoliciesEqual(t, e, specifiedPolicies) + + // create a new policy + td2 := testutil.TempDirectory(t) + id2 := snapshot.SourceInfo{ + Host: "host", + UserName: "user", + Path: filepath.ToSlash(td2), + }.String() + policy2 := &policy.Policy{ + MetadataCompressionPolicy: policy.MetadataCompressionPolicy{ + CompressorName: "zstd", + }, + } + specifiedPolicies[id2] = policy2 + makePolicyFile() + e.RunAndExpectSuccess(t, "policy", "import", "--from-file", policyFilePath, id2) + assertPoliciesEqual(t, e, specifiedPolicies) + + // unknown fields should be disallowed by default + err = os.WriteFile(policyFilePath, []byte(`{ "`+id2+`": { "not-a-real-field": 50, "metadataCompression": { "compressorName": "zstd" } } }`), 0o600) + if err != nil { + t.Fatalf("unable to write policy file: %v", err) + } + + e.RunAndExpectFailure(t, "policy", "import", "--from-file", policyFilePath, id2) + + // unless explicitly allowed + e.RunAndExpectSuccess(t, "policy", "import", "--from-file", policyFilePath, "--allow-unknown-fields", id2) + assertPoliciesEqual(t, e, specifiedPolicies) // no change + + // deleteOtherPolicies should work + delete(specifiedPolicies, id2) + makePolicyFile() + e.RunAndExpectSuccess(t, "policy", "import", "--from-file", policyFilePath, "--delete-other-policies") + assertPoliciesEqual(t, e, specifiedPolicies) + + // add it back in + specifiedPolicies[id2] = policy2 + makePolicyFile() + e.RunAndExpectSuccess(t, "policy", "import", "--from-file", policyFilePath) + assertPoliciesEqual(t, e, specifiedPolicies) + + // deleteOtherPolicies should work with specified targets as well + // don't change policy file + e.RunAndExpectSuccess(t, "policy", "import", "--from-file", policyFilePath, "--delete-other-policies", "(global)", id) + delete(specifiedPolicies, id2) + assertPoliciesEqual(t, e, specifiedPolicies) + + // --global should be equivalent to (global) + specifiedPolicies[id2] = policy2 + makePolicyFile() + e.RunAndExpectSuccess(t, "policy", "import", "--from-file", policyFilePath, "--global") + delete(specifiedPolicies, id2) // should NOT have been imported + assertPoliciesEqual(t, e, specifiedPolicies) + + // sanity check against (global) + e.RunAndExpectSuccess(t, "policy", "import", "--from-file", policyFilePath, "(global)") + assertPoliciesEqual(t, e, specifiedPolicies) + + // another sanity check + e.RunAndExpectSuccess(t, "policy", "import", "--from-file", policyFilePath) + specifiedPolicies[id2] = policy2 + assertPoliciesEqual(t, e, specifiedPolicies) + + // reading an invalid file should fail + e.RunAndExpectFailure(t, "policy", "import", "--from-file", "/not/a/real/file") + + // invalid targets should fail + err = os.WriteFile(policyFilePath, []byte(`{ "userwithouthost@": { "metadataCompression": { "compressorName": "zstd" } } }`), 0o600) + if err != nil { + t.Fatalf("unable to write policy file: %v", err) + } + e.RunAndExpectFailure(t, "policy", "import", "--from-file", policyFilePath) +} + +func assertPoliciesEqual(t *testing.T, e *testenv.CLITest, expected map[string]*policy.Policy) { + t.Helper() + var policies map[string]*policy.Policy + testutil.MustParseJSONLines(t, e.RunAndExpectSuccess(t, "policy", "export"), &policies) + + assert.Equal(t, expected, policies, "unexpected policies") +} diff --git a/site/content/docs/Getting started/_index.md b/site/content/docs/Getting started/_index.md index 0065a83c1..294425e35 100755 --- a/site/content/docs/Getting started/_index.md +++ b/site/content/docs/Getting started/_index.md @@ -325,7 +325,7 @@ Files policy: .kopiaignore inherited from (global) ``` -Finally, to list all policies for a `repository`, we can use [`kopia policy list`](../reference/command-line/common/policy-list/): +To list all policies for a `repository`, we can use [`kopia policy list`](../reference/command-line/common/policy-list/): ``` $ kopia policy list @@ -334,6 +334,56 @@ $ kopia policy list 2339ab4739bb29688bf26a3a841cf68f jarek@jareks-mbp:/Users/jarek/Projects/Kopia/site/node_modules ``` +Finally, you can also import and export policies using the [`kopia policy import`](../reference/command-line/common/policy-import/) and [`kopia policy export`](../reference/command-line/common/policy-export/) commands: + +``` +$ kopia policy import --from-file import.json +$ kopia policy export --to-file export.json +``` + +In the above example, `import.json` and `export.json` share the same format, which is a JSON map of policy identifiers to defined policies, for example: + +``` +{ + "(global)": { + "retention": { + "keepLatest": 10, + "keepHourly": 48, + ... + }, + ... + }, + "foo@bar:/home/foobar": { + "retention": { + "keepLatest": 5, + "keepHourly": 24, + ... + }, + ... + } +} +``` + +You can optionally limit which policies are imported or exported by specifying the policy identifiers as arguments to the `kopia policy import` and `kopia policy export` commands: + +``` +$ kopia policy import --from-file import.json "(global)" "foo@bar:/home/foobar" +$ kopia policy export --to-file export.json "(global)" "foo@bar:/home/foobar" +``` + +Both commands support using stdin/stdout: + +``` +$ cat file.json | kopia policy import +$ kopia policy export > file.json +``` + +You can use the `--delete-other-policies` flag to delete all policies that are not imported. This command would delete any policy besides `(global)` and `foo@bar:/home/foobar`: + +``` +$ kopia policy import --from-file import.json --delete-other-policies "(global)" "foo@bar:/home/foobar" +``` + #### Examining Repository Structure Kopia CLI provides low-level commands to examine the contents of repository, perform maintenance actions, and get deeper insight into how the data is laid out.