mirror/kopia
1
0
Fork 0
mirror of https://github.com/kopia/kopia.git synced 2026-04-22 15:08:09 -04:00
Code Issues Packages Projects Releases Wiki Activity

feat(cli): add policy import/export commands to im-/export policies from/to json (#4020)

Browse Source 
* feat(cli): add policy export command

* feat(cli): add policy import command

* chore(cli): remove unused policyTargetFlags in commandPolicyImport

* feat(cli): allow limiting which policies to import

* docs(cli): document policy import and export commands

* feat(cli): add policy import/export commands

* fix(cli): apply suggestions

* fix(general): unintentional commit

* feat(cli: add policy export tests

* feat(cli): add policy import tests

* feat(cli): add deleteOtherPolicies option for policy import

* docs(cli): stdin/stdout in policy import/export and --delete-other-policies flag

* chore(cli): fix linter issues

* fix(cli): fix newly introduced errors

* fix(cli): fix failing windows tests

* chore(cli): more test coverage for policy import/export

* fixed windows test

---------

Co-authored-by: Jarek Kowalski <jaak@jkowalski.net>
This commit is contained in:
blenderfreaky
2024-10-28 02:36:12 +01:00
committed by GitHub
parent ccc84fb00d
commit a9e178edff
6 changed files with 617 additions and 6 deletions
Download Patch File Download Diff File Expand all files Collapse all files

14
cli/command_policy.go
View File

@@ -13,11 +13,13 @@
)
type commandPolicy struct {
edit commandPolicyEdit
list commandPolicyList
delete commandPolicyDelete
set commandPolicySet
show commandPolicyShow
edit commandPolicyEdit
list commandPolicyList
delete commandPolicyDelete
set commandPolicySet
show commandPolicyShow
export commandPolicyExport
pImport commandPolicyImport
}
func (c *commandPolicy) setup(svc appServices, parent commandParent) {
@@ -28,6 +30,8 @@ func (c *commandPolicy) setup(svc appServices, parent commandParent) {
c.delete.setup(svc, cmd)
c.set.setup(svc, cmd)
c.show.setup(svc, cmd)
c.export.setup(svc, cmd)
c.pImport.setup(svc, cmd)
}
type policyTargetFlags struct {

123
cli/command_policy_export.go Normal file
View File

@@ -0,0 +1,123 @@
package cli
import (
"context"
"encoding/json"
"fmt"
"io"
"os"
"github.com/pkg/errors"
"github.com/kopia/kopia/internal/impossible"
"github.com/kopia/kopia/repo"
"github.com/kopia/kopia/snapshot/policy"
)
type commandPolicyExport struct {
policyTargetFlags
filePath string
overwrite bool
jsonIndent bool
svc appServices
}
const exportFilePerms = 0o600
func (c *commandPolicyExport) setup(svc appServices, parent commandParent) {
cmd := parent.Command("export", "Exports the policy to the specified file, or to stdout if none is specified.")
cmd.Flag("to-file", "File path to export to").StringVar(&c.filePath)
cmd.Flag("overwrite", "Overwrite the file if it exists").BoolVar(&c.overwrite)
cmd.Flag("json-indent", "Output result in indented JSON format").Hidden().BoolVar(&c.jsonIndent)
c.policyTargetFlags.setup(cmd)
c.svc = svc
cmd.Action(svc.repositoryReaderAction(c.run))
}
func (c *commandPolicyExport) run(ctx context.Context, rep repo.Repository) error {
output, err := getOutput(c)
if err != nil {
return err
}
file, ok := output.(*os.File)
if ok {
defer file.Close() //nolint:errcheck
}
policies := make(map[string]*policy.Policy)
if c.policyTargetFlags.global || len(c.policyTargetFlags.targets) > 0 {
targets, err := c.policyTargets(ctx, rep)
if err != nil {
return err
}
for _, target := range targets {
definedPolicy, err := policy.GetDefinedPolicy(ctx, rep, target)
if err != nil {
return errors.Wrapf(err, "can't get defined policy for %q", target)
}
policies[target.String()] = definedPolicy
}
} else {
ps, err := policy.ListPolicies(ctx, rep)
if err != nil {
return errors.Wrap(err, "failed to list policies")
}
for _, policy := range ps {
policies[policy.Target().String()] = policy
}
}
var toWrite []byte
if c.jsonIndent {
toWrite, err = json.MarshalIndent(policies, "", " ")
} else {
toWrite, err = json.Marshal(policies)
}
impossible.PanicOnError(err)
_, err = fmt.Fprintf(output, "%s", toWrite)
return errors.Wrap(err, "unable to write policy to output")
}
func getOutput(c *commandPolicyExport) (io.Writer, error) {
var err error
if c.filePath == "" {
if c.overwrite {
return nil, errors.New("overwrite was passed but no file path was given")
}
return c.svc.stdout(), nil
}
var file *os.File
if c.overwrite {
file, err = os.Create(c.filePath)
} else {
file, err = os.OpenFile(c.filePath, os.O_WRONLY|os.O_CREATE|os.O_EXCL, exportFilePerms)
if os.IsExist(err) {
return nil, errors.Wrap(err, "file already exists and overwrite flag is not set")
}
}
if err != nil {
return nil, errors.Wrap(err, "error opening file to write to")
}
return file, nil
}

129
cli/command_policy_export_test.go Normal file
View File

@@ -0,0 +1,129 @@
package cli_test
import (
"os"
"path"
"testing"
"github.com/stretchr/testify/assert"
"github.com/kopia/kopia/internal/testutil"
"github.com/kopia/kopia/snapshot"
"github.com/kopia/kopia/snapshot/policy"
"github.com/kopia/kopia/tests/testenv"
)
func TestExportPolicy(t *testing.T) {
e := testenv.NewCLITest(t, testenv.RepoFormatNotImportant, testenv.NewInProcRunner(t))
defer e.RunAndExpectSuccess(t, "repo", "disconnect")
e.RunAndExpectSuccess(t, "repo", "create", "filesystem", "--path", e.RepoDir, "--override-username=user", "--override-hostname=host")
// check if we get the default global policy
var policies1 map[string]*policy.Policy
testutil.MustParseJSONLines(t, e.RunAndExpectSuccess(t, "policy", "export"), &policies1)
assert.Len(t, policies1, 1, "unexpected number of policies")
assert.Equal(t, policy.DefaultPolicy, policies1["(global)"], "unexpected policy")
var policies2 map[string]*policy.Policy
// we only have one policy, so exporting all policies should be the same as exporting the global policy explicitly
testutil.MustParseJSONLines(t, e.RunAndExpectSuccess(t, "policy", "export", "(global)"), &policies2)
assert.Len(t, policies2, 1, "unexpected number of policies")
assert.Equal(t, policies1, policies2, "unexpected policy")
// create a new policy
td := testutil.TempDirectory(t)
id := snapshot.SourceInfo{
Host: "host",
UserName: "user",
Path: td,
}.String()
e.RunAndExpectSuccess(t, "policy", "set", td, "--splitter=FIXED-4M")
expectedPolicy := &policy.Policy{
SplitterPolicy: policy.SplitterPolicy{
Algorithm: "FIXED-4M",
},
}
expectedPolicies := map[string]*policy.Policy{
"(global)": policy.DefaultPolicy,
id: expectedPolicy,
}
// check if we get the new policy
var policies3 map[string]*policy.Policy
testutil.MustParseJSONLines(t, e.RunAndExpectSuccess(t, "policy", "export", id), &policies3)
assert.Len(t, policies3, 1, "unexpected number of policies")
assert.Equal(t, expectedPolicy, policies3[id], "unexpected policy")
// specifying a local id should return the same policy
var policies4 map[string]*policy.Policy
testutil.MustParseJSONLines(t, e.RunAndExpectSuccess(t, "policy", "export", td), &policies4) // note: td, not id
assert.Len(t, policies4, 1, "unexpected number of policies")
assert.Equal(t, expectedPolicy, policies4[id], "unexpected policy") // thee key is always the full id however
// exporting without specifying a policy should return all policies
var policies5 map[string]*policy.Policy
testutil.MustParseJSONLines(t, e.RunAndExpectSuccess(t, "policy", "export"), &policies5)
assert.Len(t, policies5, 2, "unexpected number of policies")
assert.Equal(t, expectedPolicies, policies5, "unexpected policy")
// sanity check if --to-file works
exportPath := path.Join(td, "exported.json")
e.RunAndExpectSuccess(t, "policy", "export", "--to-file", exportPath)
exportedContent, err := os.ReadFile(exportPath)
if err != nil {
t.Fatalf("unable to read exported file: %v", err)
}
var policies6 map[string]*policy.Policy
testutil.MustParseJSONLines(t, []string{string(exportedContent)}, &policies6)
assert.Equal(t, expectedPolicies, policies6, "unexpected policy")
// should not overwrite existing file
e.RunAndExpectFailure(t, "policy", "export", "--to-file", exportPath, id)
// unless --overwrite is passed
e.RunAndExpectSuccess(t, "policy", "export", "--overwrite", "--to-file", exportPath, id)
exportedContent, err = os.ReadFile(exportPath)
if err != nil {
t.Fatalf("unable to read exported file: %v", err)
}
var policies7 map[string]*policy.Policy
testutil.MustParseJSONLines(t, []string{string(exportedContent)}, &policies7)
// we specified id, so only that policy should be exported
assert.Len(t, policies7, 1, "unexpected number of policies")
assert.Equal(t, expectedPolicy, policies5[id], "unexpected policy")
// pretty-printed JSON should be different but also correct
policies8prettyJSON := e.RunAndExpectSuccess(t, "policy", "export", "--json-indent")
var policies8pretty map[string]*policy.Policy
testutil.MustParseJSONLines(t, policies8prettyJSON, &policies8pretty)
policies8JSON := e.RunAndExpectSuccess(t, "policy", "export")
var policies8 map[string]*policy.Policy
testutil.MustParseJSONLines(t, policies8JSON, &policies8)
assert.Equal(t, policies8, policies8pretty, "pretty-printing should not change the content")
assert.NotEqual(t, policies8JSON, policies8prettyJSON, "pretty-printed JSON should be different")
// --overwrite and no --to-file should fail
e.RunAndExpectFailure(t, "policy", "export", "--overwrite")
// writing to inaccessible file should fail
e.RunAndExpectFailure(t, "policy", "export", "--to-file", "/not/a/real/file/path")
}

129
cli/command_policy_import.go Normal file
View File

@@ -0,0 +1,129 @@
package cli
import (
"context"
"encoding/json"
"io"
"os"
"slices"
"github.com/pkg/errors"
"github.com/kopia/kopia/repo"
"github.com/kopia/kopia/snapshot"
"github.com/kopia/kopia/snapshot/policy"
)
type commandPolicyImport struct {
policyTargetFlags
filePath string
allowUnknownFields bool
deleteOtherPolicies bool
svc appServices
}
func (c *commandPolicyImport) setup(svc appServices, parent commandParent) {
cmd := parent.Command("import", "Imports policies from a specified file, or stdin if no file is specified.")
cmd.Flag("from-file", "File path to import from").StringVar(&c.filePath)
cmd.Flag("allow-unknown-fields", "Allow unknown fields in the policy file").BoolVar(&c.allowUnknownFields)
cmd.Flag("delete-other-policies", "Delete all other policies, keeping only those that got imported").BoolVar(&c.deleteOtherPolicies)
c.policyTargetFlags.setup(cmd)
c.svc = svc
cmd.Action(svc.repositoryWriterAction(c.run))
}
func (c *commandPolicyImport) run(ctx context.Context, rep repo.RepositoryWriter) error {
var input io.Reader
var err error
if c.filePath != "" {
file, err := os.Open(c.filePath)
if err != nil {
return errors.Wrap(err, "unable to read policy file")
}
defer file.Close() //nolint:errcheck
input = file
} else {
input = c.svc.stdin()
}
policies := make(map[string]*policy.Policy)
d := json.NewDecoder(input)
if !c.allowUnknownFields {
d.DisallowUnknownFields()
}
err = d.Decode(&policies)
if err != nil {
return errors.Wrap(err, "unable to decode policy file as valid json")
}
var targetLimit []snapshot.SourceInfo
if c.policyTargetFlags.global || len(c.policyTargetFlags.targets) > 0 {
targetLimit, err = c.policyTargets(ctx, rep)
if err != nil {
return err
}
}
shouldImportSource := func(target snapshot.SourceInfo) bool {
if targetLimit == nil {
return true
}
return slices.Contains(targetLimit, target)
}
importedSources := make([]string, 0, len(policies))
for ts, newPolicy := range policies {
target, err := snapshot.ParseSourceInfo(ts, rep.ClientOptions().Hostname, rep.ClientOptions().Username)
if err != nil {
return errors.Wrapf(err, "unable to parse source info: %q", ts)
}
if !shouldImportSource(target) {
continue
}
// used for deleteOtherPolicies
importedSources = append(importedSources, ts)
if err := policy.SetPolicy(ctx, rep, target, newPolicy); err != nil {
return errors.Wrapf(err, "can't save policy for %v", target)
}
}
if c.deleteOtherPolicies {
err := deleteOthers(ctx, rep, importedSources)
if err != nil {
return err
}
}
return nil
}
func deleteOthers(ctx context.Context, rep repo.RepositoryWriter, importedSources []string) error {
ps, err := policy.ListPolicies(ctx, rep)
if err != nil {
return errors.Wrap(err, "failed to list policies")
}
for _, p := range ps {
if !slices.Contains(importedSources, p.Target().String()) {
if err := policy.RemovePolicy(ctx, rep, p.Target()); err != nil {
return errors.Wrapf(err, "can't delete policy for %v", p.Target())
}
}
}
return nil
}

176
cli/command_policy_import_test.go Normal file
View File

@@ -0,0 +1,176 @@
package cli_test
import (
"encoding/json"
"os"
"path"
"path/filepath"
"testing"
"github.com/stretchr/testify/assert"
"github.com/kopia/kopia/internal/testutil"
"github.com/kopia/kopia/snapshot"
"github.com/kopia/kopia/snapshot/policy"
"github.com/kopia/kopia/tests/testenv"
)
// note: dependent on policy export working.
func TestImportPolicy(t *testing.T) {
e := testenv.NewCLITest(t, testenv.RepoFormatNotImportant, testenv.NewInProcRunner(t))
defer e.RunAndExpectSuccess(t, "repo", "disconnect")
e.RunAndExpectSuccess(t, "repo", "create", "filesystem", "--path", e.RepoDir, "--override-username=user", "--override-hostname=host")
td := testutil.TempDirectory(t)
policyFilePath := path.Join(td, "policy.json")
// poor man's deep copy
defaultPolicyJSON, err := json.Marshal(policy.DefaultPolicy)
if err != nil {
t.Fatalf("unable to marshal policy: %v", err)
}
var defaultPolicy *policy.Policy
testutil.MustParseJSONLines(t, []string{string(defaultPolicyJSON)}, &defaultPolicy)
specifiedPolicies := map[string]*policy.Policy{
"(global)": defaultPolicy,
}
makePolicyFile := func() {
data, err := json.Marshal(specifiedPolicies)
if err != nil {
t.Fatalf("unable to marshal policy: %v", err)
}
err = os.WriteFile(policyFilePath, data, 0o600)
if err != nil {
t.Fatalf("unable to write policy file: %v", err)
}
}
// sanity check that we have the default global policy
assertPoliciesEqual(t, e, specifiedPolicies)
// change the global policy
specifiedPolicies["(global)"].SplitterPolicy.Algorithm = "FIXED-4M"
makePolicyFile()
e.RunAndExpectSuccess(t, "policy", "import", "--from-file", policyFilePath)
assertPoliciesEqual(t, e, specifiedPolicies)
// create a new policy
id := snapshot.SourceInfo{
Host: "host",
UserName: "user",
Path: filepath.ToSlash(td),
}.String()
specifiedPolicies[id] = &policy.Policy{
SplitterPolicy: policy.SplitterPolicy{
Algorithm: "FIXED-8M",
},
}
makePolicyFile()
e.RunAndExpectSuccess(t, "policy", "import", "--from-file", policyFilePath)
assertPoliciesEqual(t, e, specifiedPolicies)
// import from a file specifying changes in both policies but limiting import to only one
specifiedPolicies["(global)"].CompressionPolicy.CompressorName = "zstd"
specifiedPolicies[id].CompressionPolicy.CompressorName = "gzip"
makePolicyFile()
e.RunAndExpectSuccess(t, "policy", "import", "--from-file", policyFilePath, "(global)")
// local policy should not have changed
specifiedPolicies[id].CompressionPolicy.CompressorName = ""
assertPoliciesEqual(t, e, specifiedPolicies)
specifiedPolicies[id].CompressionPolicy.CompressorName = "gzip"
e.RunAndExpectSuccess(t, "policy", "import", "--from-file", policyFilePath, id)
assertPoliciesEqual(t, e, specifiedPolicies)
// deleting values should work
specifiedPolicies[id].CompressionPolicy.CompressorName = ""
makePolicyFile()
e.RunAndExpectSuccess(t, "policy", "import", "--from-file", policyFilePath, id)
assertPoliciesEqual(t, e, specifiedPolicies)
// create a new policy
td2 := testutil.TempDirectory(t)
id2 := snapshot.SourceInfo{
Host: "host",
UserName: "user",
Path: filepath.ToSlash(td2),
}.String()
policy2 := &policy.Policy{
MetadataCompressionPolicy: policy.MetadataCompressionPolicy{
CompressorName: "zstd",
},
}
specifiedPolicies[id2] = policy2
makePolicyFile()
e.RunAndExpectSuccess(t, "policy", "import", "--from-file", policyFilePath, id2)
assertPoliciesEqual(t, e, specifiedPolicies)
// unknown fields should be disallowed by default
err = os.WriteFile(policyFilePath, []byte(`{ "`+id2+`": { "not-a-real-field": 50, "metadataCompression": { "compressorName": "zstd" } } }`), 0o600)
if err != nil {
t.Fatalf("unable to write policy file: %v", err)
}
e.RunAndExpectFailure(t, "policy", "import", "--from-file", policyFilePath, id2)
// unless explicitly allowed
e.RunAndExpectSuccess(t, "policy", "import", "--from-file", policyFilePath, "--allow-unknown-fields", id2)
assertPoliciesEqual(t, e, specifiedPolicies) // no change
// deleteOtherPolicies should work
delete(specifiedPolicies, id2)
makePolicyFile()
e.RunAndExpectSuccess(t, "policy", "import", "--from-file", policyFilePath, "--delete-other-policies")
assertPoliciesEqual(t, e, specifiedPolicies)
// add it back in
specifiedPolicies[id2] = policy2
makePolicyFile()
e.RunAndExpectSuccess(t, "policy", "import", "--from-file", policyFilePath)
assertPoliciesEqual(t, e, specifiedPolicies)
// deleteOtherPolicies should work with specified targets as well
// don't change policy file
e.RunAndExpectSuccess(t, "policy", "import", "--from-file", policyFilePath, "--delete-other-policies", "(global)", id)
delete(specifiedPolicies, id2)
assertPoliciesEqual(t, e, specifiedPolicies)
// --global should be equivalent to (global)
specifiedPolicies[id2] = policy2
makePolicyFile()
e.RunAndExpectSuccess(t, "policy", "import", "--from-file", policyFilePath, "--global")
delete(specifiedPolicies, id2) // should NOT have been imported
assertPoliciesEqual(t, e, specifiedPolicies)
// sanity check against (global)
e.RunAndExpectSuccess(t, "policy", "import", "--from-file", policyFilePath, "(global)")
assertPoliciesEqual(t, e, specifiedPolicies)
// another sanity check
e.RunAndExpectSuccess(t, "policy", "import", "--from-file", policyFilePath)
specifiedPolicies[id2] = policy2
assertPoliciesEqual(t, e, specifiedPolicies)
// reading an invalid file should fail
e.RunAndExpectFailure(t, "policy", "import", "--from-file", "/not/a/real/file")
// invalid targets should fail
err = os.WriteFile(policyFilePath, []byte(`{ "userwithouthost@": { "metadataCompression": { "compressorName": "zstd" } } }`), 0o600)
if err != nil {
t.Fatalf("unable to write policy file: %v", err)
}
e.RunAndExpectFailure(t, "policy", "import", "--from-file", policyFilePath)
}
func assertPoliciesEqual(t *testing.T, e *testenv.CLITest, expected map[string]*policy.Policy) {
t.Helper()
var policies map[string]*policy.Policy
testutil.MustParseJSONLines(t, e.RunAndExpectSuccess(t, "policy", "export"), &policies)
assert.Equal(t, expected, policies, "unexpected policies")
}

52
site/content/docs/Getting started/_index.md
View File

@@ -325,7 +325,7 @@ Files policy:
.kopiaignore inherited from (global)
```
Finally, to list all policies for a `repository`, we can use [`kopia policy list`](../reference/command-line/common/policy-list/):
To list all policies for a `repository`, we can use [`kopia policy list`](../reference/command-line/common/policy-list/):
```
$ kopia policy list
@@ -334,6 +334,56 @@ $ kopia policy list
2339ab4739bb29688bf26a3a841cf68f jarek@jareks-mbp:/Users/jarek/Projects/Kopia/site/node_modules
```
Finally, you can also import and export policies using the [`kopia policy import`](../reference/command-line/common/policy-import/) and [`kopia policy export`](../reference/command-line/common/policy-export/) commands:
```
$ kopia policy import --from-file import.json
$ kopia policy export --to-file export.json
```
In the above example, `import.json` and `export.json` share the same format, which is a JSON map of policy identifiers to defined policies, for example:
```
{
"(global)": {
"retention": {
"keepLatest": 10,
"keepHourly": 48,
...
},
...
},
"foo@bar:/home/foobar": {
"retention": {
"keepLatest": 5,
"keepHourly": 24,
...
},
...
}
}
```
You can optionally limit which policies are imported or exported by specifying the policy identifiers as arguments to the `kopia policy import` and `kopia policy export` commands:
```
$ kopia policy import --from-file import.json "(global)" "foo@bar:/home/foobar"
$ kopia policy export --to-file export.json "(global)" "foo@bar:/home/foobar"
```
Both commands support using stdin/stdout:
```
$ cat file.json | kopia policy import
$ kopia policy export > file.json
```
You can use the `--delete-other-policies` flag to delete all policies that are not imported. This command would delete any policy besides `(global)` and `foo@bar:/home/foobar`:
```
$ kopia policy import --from-file import.json --delete-other-policies "(global)" "foo@bar:/home/foobar"
```
#### Examining Repository Structure
Kopia CLI provides low-level commands to examine the contents of repository, perform maintenance actions, and get deeper insight into how the data is laid out.