From 525720db9504bb48f2d65afcfa473d35acba77a9 Mon Sep 17 00:00:00 2001 From: Jarek Kowalski Date: Sat, 20 Nov 2021 20:53:25 -0800 Subject: [PATCH] cli: added 'snapshot pin' command (#1528) --- cli/command_snapshot.go | 2 + cli/command_snapshot_create.go | 5 ++ cli/command_snapshot_list.go | 4 ++ cli/command_snapshot_pin.go | 78 ++++++++++++++++++++++++++++ cli/command_snapshot_pin_test.go | 89 ++++++++++++++++++++++++++++++++ snapshot/manager.go | 18 +++++++ snapshot/manifest.go | 37 +++++++++++++ snapshot/policy/expire.go | 4 +- snapshot/snapshot_test.go | 23 +++++++++ 9 files changed, 258 insertions(+), 2 deletions(-) create mode 100644 cli/command_snapshot_pin.go create mode 100644 cli/command_snapshot_pin_test.go diff --git a/cli/command_snapshot.go b/cli/command_snapshot.go index 8151179f8..73f5c4679 100644 --- a/cli/command_snapshot.go +++ b/cli/command_snapshot.go @@ -10,6 +10,7 @@ type commandSnapshot struct { gc commandSnapshotGC list commandSnapshotList migrate commandSnapshotMigrate + pin commandSnapshotPin restore commandSnapshotRestore verify commandSnapshotVerify } @@ -25,6 +26,7 @@ func (c *commandSnapshot) setup(svc advancedAppServices, parent commandParent) { c.gc.setup(svc, cmd) c.list.setup(svc, cmd) c.migrate.setup(svc, cmd) + c.pin.setup(svc, cmd) c.restore.setup(svc, cmd) c.verify.setup(svc, cmd) } diff --git a/cli/command_snapshot_create.go b/cli/command_snapshot_create.go index 49d32aaed..528abd644 100644 --- a/cli/command_snapshot_create.go +++ b/cli/command_snapshot_create.go @@ -38,6 +38,8 @@ type commandSnapshotCreate struct { snapshotCreateCheckpointUploadLimitMB int64 snapshotCreateTags []string + pins []string + logDirDetail int logEntryDetail int @@ -63,6 +65,7 @@ func (c *commandSnapshotCreate) setup(svc appServices, parent commandParent) { cmd.Flag("force-disable-actions", "Disable snapshot actions even if globally enabled on this client").Hidden().BoolVar(&c.snapshotCreateForceDisableActions) cmd.Flag("stdin-file", "File path to be used for stdin data snapshot.").StringVar(&c.snapshotCreateStdinFileName) cmd.Flag("tags", "Tags applied on the snapshot. Must be provided in the : format.").StringsVar(&c.snapshotCreateTags) + cmd.Flag("pin", "Create a pinned snapshot that's will not expire automatically").StringsVar(&c.pins) c.logDirDetail = -1 c.logEntryDetail = -1 @@ -287,6 +290,8 @@ func (c *commandSnapshotCreate) snapshotSingleSource(ctx context.Context, rep re manifest.Description = c.snapshotCreateDescription manifest.Tags = tags + manifest.UpdatePins(c.pins, nil) + startTimeOverride, _ := parseTimestamp(c.snapshotCreateStartTime) endTimeOverride, _ := parseTimestamp(c.snapshotCreateEndTime) diff --git a/cli/command_snapshot_list.go b/cli/command_snapshot_list.go index 6e8c350dc..10b81a3c9 100644 --- a/cli/command_snapshot_list.go +++ b/cli/command_snapshot_list.go @@ -312,6 +312,10 @@ func (c *commandSnapshotList) entryBits(ctx context.Context, m *snapshot.Manifes } } + if len(m.Pins) > 0 { + bits = append(bits, "pins:"+strings.Join(m.Pins, ",")) + } + return bits, col } diff --git a/cli/command_snapshot_pin.go b/cli/command_snapshot_pin.go new file mode 100644 index 000000000..6af3d0a97 --- /dev/null +++ b/cli/command_snapshot_pin.go @@ -0,0 +1,78 @@ +package cli + +import ( + "context" + + "github.com/pkg/errors" + + "github.com/kopia/kopia/repo" + "github.com/kopia/kopia/repo/manifest" + "github.com/kopia/kopia/repo/object" + "github.com/kopia/kopia/snapshot" +) + +type commandSnapshotPin struct { + addPins []string + removePins []string + snapshotIDs []string +} + +func (c *commandSnapshotPin) setup(svc appServices, parent commandParent) { + cmd := parent.Command("pin", "Add or remove pins preventing snapshot deletion") + cmd.Flag("add", "Add pins").StringsVar(&c.addPins) + cmd.Flag("remove", "Remove pins").StringsVar(&c.removePins) + cmd.Arg("id", "Snapshot ID or root object ID").Required().StringsVar(&c.snapshotIDs) + cmd.Action(svc.repositoryWriterAction(c.run)) +} + +func (c *commandSnapshotPin) run(ctx context.Context, rep repo.RepositoryWriter) error { + if len(c.addPins)+len(c.removePins) == 0 { + return errors.Errorf("must specify --add and/or --remove") + } + + for _, id := range c.snapshotIDs { + m, err := snapshot.LoadSnapshot(ctx, rep, manifest.ID(id)) + if err == nil { + if err = c.pinSnapshot(ctx, rep, m); err != nil { + return errors.Wrapf(err, "error pinning %v", id) + } + } else if !errors.Is(err, snapshot.ErrSnapshotNotFound) { + return errors.Wrapf(err, "error loading snapshot %v", id) + } else if err := c.pinSnapshotsByRootObjectID(ctx, rep, object.ID(id)); err != nil { + return errors.Wrapf(err, "error pinning snapshots by root ID %v", id) + } + } + + return nil +} + +func (c *commandSnapshotPin) pinSnapshotsByRootObjectID(ctx context.Context, rep repo.RepositoryWriter, rootID object.ID) error { + manifests, err := snapshot.FindSnapshotsByRootObjectID(ctx, rep, rootID) + if err != nil { + return errors.Wrapf(err, "unable to find snapshots by root %v", rootID) + } + + if len(manifests) == 0 { + return errors.Errorf("no snapshots matched %v", rootID) + } + + for _, m := range manifests { + if err := c.pinSnapshot(ctx, rep, m); err != nil { + return errors.Wrap(err, "error pinning") + } + } + + return nil +} + +func (c *commandSnapshotPin) pinSnapshot(ctx context.Context, rep repo.RepositoryWriter, m *snapshot.Manifest) error { + if !m.UpdatePins(c.addPins, c.removePins) { + log(ctx).Infof("No change for snapshot at %v of %v", formatTimestamp(m.StartTime), m.Source) + + return nil + } + + log(ctx).Infof("Updating snapshot at %v of %v", formatTimestamp(m.StartTime), m.Source) + + return errors.Wrap(snapshot.UpdateSnapshot(ctx, rep, m), "error updating snapshot") +} diff --git a/cli/command_snapshot_pin_test.go b/cli/command_snapshot_pin_test.go new file mode 100644 index 000000000..9ef946a40 --- /dev/null +++ b/cli/command_snapshot_pin_test.go @@ -0,0 +1,89 @@ +package cli_test + +import ( + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/require" + + "github.com/kopia/kopia/internal/testutil" + "github.com/kopia/kopia/snapshot" + "github.com/kopia/kopia/tests/testenv" +) + +func TestSnapshotPin(t *testing.T) { + t.Parallel() + + runner := testenv.NewInProcRunner(t) + e := testenv.NewCLITest(t, testenv.RepoFormatNotImportant, runner) + + defer e.RunAndExpectSuccess(t, "repo", "disconnect") + e.RunAndExpectSuccess(t, "repo", "create", "filesystem", "--path", e.RepoDir) + + srcdir := testutil.TempDirectory(t) + require.NoError(t, os.WriteFile(filepath.Join(srcdir, "some-file2"), []byte{1, 2, 3}, 0o755)) + + var man snapshot.Manifest + + e.RunAndExpectSuccess(t, "policy", "set", srcdir, "--keep-latest=3", "--keep-hourly=0", "--keep-daily=0", "--keep-monthly=0", "--keep-weekly=0", "--keep-annual=0") + + testutil.MustParseJSONLines(t, e.RunAndExpectSuccess(t, "snapshot", "create", srcdir, "--pin=a", "--pin=b", "--json"), &man) + require.Equal(t, []string{"a", "b"}, man.Pins) + + e.RunAndExpectSuccess(t, "snapshot", "list") + + // create more unpinned snapshots + e.RunAndExpectSuccess(t, "snapshot", "create", srcdir) + e.RunAndExpectSuccess(t, "snapshot", "create", srcdir) + e.RunAndExpectSuccess(t, "snapshot", "create", srcdir) + e.RunAndExpectSuccess(t, "snapshot", "create", srcdir) + e.RunAndExpectSuccess(t, "snapshot", "create", srcdir) + + var snapshots []*snapshot.Manifest + + testutil.MustParseJSONLines(t, e.RunAndExpectSuccess(t, "snapshot", "list", "--json"), &snapshots) + snapshots = snapshot.SortByTime(snapshots, false) + + // make sure the pinned one is on top. + require.Len(t, snapshots, 4) + require.Equal(t, []string{"a", "b"}, snapshots[0].Pins) + require.Empty(t, snapshots[1].Pins) + require.Empty(t, snapshots[2].Pins) + require.Empty(t, snapshots[3].Pins) + + // neither --add nor --remove were provided + e.RunAndExpectFailure(t, "snapshot", "pin", string(snapshots[3].ID)) + e.RunAndExpectSuccess(t, "snapshot", "pin", string(snapshots[0].ID), "--add=c", "--remove=b") + e.RunAndExpectSuccess(t, "snapshot", "pin", string(snapshots[3].ID), "--add=d") + + var snapshots2 []*snapshot.Manifest + + testutil.MustParseJSONLines(t, e.RunAndExpectSuccess(t, "snapshot", "list", "--json"), &snapshots2) + snapshots2 = snapshot.SortByTime(snapshots2, false) + require.Len(t, snapshots2, 4) + + require.Equal(t, []string{"a", "c"}, snapshots2[0].Pins) + require.Empty(t, snapshots2[1].Pins) + require.Empty(t, snapshots2[2].Pins) + require.Equal(t, []string{"d"}, snapshots2[3].Pins) + + // create more unpinned snapshots + e.RunAndExpectSuccess(t, "snapshot", "create", srcdir) + e.RunAndExpectSuccess(t, "snapshot", "create", srcdir) + e.RunAndExpectSuccess(t, "snapshot", "create", srcdir) + e.RunAndExpectSuccess(t, "snapshot", "create", srcdir) + e.RunAndExpectSuccess(t, "snapshot", "create", srcdir) + + var snapshots3 []*snapshot.Manifest + + testutil.MustParseJSONLines(t, e.RunAndExpectSuccess(t, "snapshot", "list", "--json"), &snapshots3) + snapshots3 = snapshot.SortByTime(snapshots3, false) + require.Len(t, snapshots3, 5) + + require.Equal(t, []string{"a", "c"}, snapshots3[0].Pins) + require.Equal(t, []string{"d"}, snapshots3[1].Pins) + require.Empty(t, snapshots3[2].Pins) + require.Empty(t, snapshots3[3].Pins) + require.Empty(t, snapshots3[4].Pins) +} diff --git a/snapshot/manager.go b/snapshot/manager.go index ded2f4183..626298928 100644 --- a/snapshot/manager.go +++ b/snapshot/manager.go @@ -228,6 +228,24 @@ func FindSnapshotsByRootObjectID(ctx context.Context, rep repo.Repository, rootI return result, nil } +// UpdateSnapshot updates the snapshot by saving the provided data and deleting old manifest ID. +func UpdateSnapshot(ctx context.Context, rep repo.RepositoryWriter, m *Manifest) error { + oldID := m.ID + + newID, err := SaveSnapshot(ctx, rep, m) + if err != nil { + return errors.Wrap(err, "error saving snapshot") + } + + if oldID != newID { + if err := rep.DeleteManifest(ctx, oldID); err != nil { + return errors.Wrap(err, "error deleting old manifest") + } + } + + return nil +} + func entryIDs(entries []*manifest.EntryMetadata) []manifest.ID { var ids []manifest.ID for _, e := range entries { diff --git a/snapshot/manifest.go b/snapshot/manifest.go index f564fb179..1a1ecf679 100644 --- a/snapshot/manifest.go +++ b/snapshot/manifest.go @@ -31,6 +31,43 @@ type Manifest struct { RetentionReasons []string `json:"-"` Tags map[string]string `json:"tags,omitempty"` + + // list of manually-defined pins which prevent the snapshot from being deleted. + Pins []string `json:"pins,omitempty"` +} + +// UpdatePins updates pins in the provided manifest. +func (m *Manifest) UpdatePins(add, remove []string) bool { + newPins := map[string]bool{} + changed := false + + for _, r := range m.Pins { + newPins[r] = true + } + + for _, r := range add { + if !newPins[r] { + newPins[r] = true + changed = true + } + } + + for _, r := range remove { + if newPins[r] { + delete(newPins, r) + + changed = true + } + } + + m.Pins = nil + for r := range newPins { + m.Pins = append(m.Pins, r) + } + + sort.Strings(m.Pins) + + return changed } // EntryType is a type of a filesystem entry. diff --git a/snapshot/policy/expire.go b/snapshot/policy/expire.go index 87e9b026a..14c8809d9 100644 --- a/snapshot/policy/expire.go +++ b/snapshot/policy/expire.go @@ -61,11 +61,11 @@ func getExpiredSnapshotsForSource(ctx context.Context, rep repo.Repository, snap var toDelete []*snapshot.Manifest for _, s := range snapshots { - if len(s.RetentionReasons) == 0 { + if len(s.RetentionReasons) == 0 && len(s.Pins) == 0 { log(ctx).Debugf(" deleting %v", s.StartTime) toDelete = append(toDelete, s) } else { - log(ctx).Debugf(" keeping %v reasons: [%v]", s.StartTime, strings.Join(s.RetentionReasons, ",")) + log(ctx).Debugf(" keeping %v retention: [%v] pins: [%v]", s.StartTime, strings.Join(s.RetentionReasons, ","), strings.Join(s.Pins, ",")) } } diff --git a/snapshot/snapshot_test.go b/snapshot/snapshot_test.go index 581e063ae..633abfd0d 100644 --- a/snapshot/snapshot_test.go +++ b/snapshot/snapshot_test.go @@ -8,6 +8,8 @@ "sort" "testing" + "github.com/stretchr/testify/require" + "github.com/kopia/kopia/internal/repotesting" "github.com/kopia/kopia/internal/testlogging" "github.com/kopia/kopia/repo" @@ -76,6 +78,15 @@ func TestSnapshotsAPI(t *testing.T) { verifySnapshotManifestIDs(t, env.RepositoryWriter, &src2, []manifest.ID{id3}) verifySources(t, env.RepositoryWriter, src1, src2) verifyLoadSnapshots(t, env.RepositoryWriter, []manifest.ID{id1, id2, id3}, []*snapshot.Manifest{manifest1, manifest2, manifest3}) + + require.True(t, manifest3.UpdatePins([]string{"new-pin"}, nil)) + require.NoError(t, snapshot.UpdateSnapshot(ctx, env.RepositoryWriter, manifest3)) + + require.NotEqual(t, manifest3.ID, id3) + + updated3, err := snapshot.LoadSnapshot(ctx, env.RepositoryWriter, manifest3.ID) + require.NoError(t, err) + require.Equal(t, updated3, manifest3) } func verifySnapshotManifestIDs(t *testing.T, rep repo.Repository, src *snapshot.SourceInfo, expected []manifest.ID) { @@ -249,3 +260,15 @@ func TestParseInvalidSourceInfo(t *testing.T) { } } } + +func TestUpdatePins(t *testing.T) { + m := snapshot.Manifest{} + + require.True(t, m.UpdatePins([]string{"d", "c", "b"}, nil)) + require.False(t, m.UpdatePins([]string{"d", "c", "b"}, nil)) + require.Equal(t, []string{"b", "c", "d"}, m.Pins) // pins are sorted + + require.True(t, m.UpdatePins([]string{"e", "a"}, []string{"c"})) + require.False(t, m.UpdatePins([]string{"e", "a"}, []string{"c"})) + require.Equal(t, []string{"a", "b", "d", "e"}, m.Pins) +}