mirror/kopia
1
0
Fork 0
mirror of https://github.com/kopia/kopia.git synced 2026-05-24 06:34:46 -04:00
Code Issues Packages Projects Releases Wiki Activity

cli: added 'snapshot pin' command (#1528)

Browse Source
This commit is contained in:
Jarek Kowalski
2021-11-20 20:53:25 -08:00
committed by GitHub
parent 4bebaa7198
commit 525720db95
9 changed files with 258 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
cli/command_snapshot.go
View File

@@ -10,6 +10,7 @@ type commandSnapshot struct {
gc commandSnapshotGC
list commandSnapshotList
migrate commandSnapshotMigrate
pin commandSnapshotPin
restore commandSnapshotRestore
verify commandSnapshotVerify
}
@@ -25,6 +26,7 @@ func (c *commandSnapshot) setup(svc advancedAppServices, parent commandParent) {
c.gc.setup(svc, cmd)
c.list.setup(svc, cmd)
c.migrate.setup(svc, cmd)
c.pin.setup(svc, cmd)
c.restore.setup(svc, cmd)
c.verify.setup(svc, cmd)
}

5
cli/command_snapshot_create.go
View File

@@ -38,6 +38,8 @@ type commandSnapshotCreate struct {
snapshotCreateCheckpointUploadLimitMB int64
snapshotCreateTags []string
pins []string
logDirDetail int
logEntryDetail int
@@ -63,6 +65,7 @@ func (c *commandSnapshotCreate) setup(svc appServices, parent commandParent) {
cmd.Flag("force-disable-actions", "Disable snapshot actions even if globally enabled on this client").Hidden().BoolVar(&c.snapshotCreateForceDisableActions)
cmd.Flag("stdin-file", "File path to be used for stdin data snapshot.").StringVar(&c.snapshotCreateStdinFileName)
cmd.Flag("tags", "Tags applied on the snapshot. Must be provided in the <key>:<value> format.").StringsVar(&c.snapshotCreateTags)
cmd.Flag("pin", "Create a pinned snapshot that's will not expire automatically").StringsVar(&c.pins)
c.logDirDetail = -1
c.logEntryDetail = -1
@@ -287,6 +290,8 @@ func (c *commandSnapshotCreate) snapshotSingleSource(ctx context.Context, rep re
manifest.Description = c.snapshotCreateDescription
manifest.Tags = tags
manifest.UpdatePins(c.pins, nil)
startTimeOverride, _ := parseTimestamp(c.snapshotCreateStartTime)
endTimeOverride, _ := parseTimestamp(c.snapshotCreateEndTime)

4
cli/command_snapshot_list.go
View File

@@ -312,6 +312,10 @@ func (c *commandSnapshotList) entryBits(ctx context.Context, m *snapshot.Manifes
}
}
if len(m.Pins) > 0 {
bits = append(bits, "pins:"+strings.Join(m.Pins, ","))
}
return bits, col
}

78
cli/command_snapshot_pin.go Normal file
View File

@@ -0,0 +1,78 @@
package cli
import (
"context"
"github.com/pkg/errors"
"github.com/kopia/kopia/repo"
"github.com/kopia/kopia/repo/manifest"
"github.com/kopia/kopia/repo/object"
"github.com/kopia/kopia/snapshot"
)
type commandSnapshotPin struct {
addPins []string
removePins []string
snapshotIDs []string
}
func (c *commandSnapshotPin) setup(svc appServices, parent commandParent) {
cmd := parent.Command("pin", "Add or remove pins preventing snapshot deletion")
cmd.Flag("add", "Add pins").StringsVar(&c.addPins)
cmd.Flag("remove", "Remove pins").StringsVar(&c.removePins)
cmd.Arg("id", "Snapshot ID or root object ID").Required().StringsVar(&c.snapshotIDs)
cmd.Action(svc.repositoryWriterAction(c.run))
}
func (c *commandSnapshotPin) run(ctx context.Context, rep repo.RepositoryWriter) error {
if len(c.addPins)+len(c.removePins) == 0 {
return errors.Errorf("must specify --add and/or --remove")
}
for _, id := range c.snapshotIDs {
m, err := snapshot.LoadSnapshot(ctx, rep, manifest.ID(id))
if err == nil {
if err = c.pinSnapshot(ctx, rep, m); err != nil {
return errors.Wrapf(err, "error pinning %v", id)
}
} else if !errors.Is(err, snapshot.ErrSnapshotNotFound) {
return errors.Wrapf(err, "error loading snapshot %v", id)
} else if err := c.pinSnapshotsByRootObjectID(ctx, rep, object.ID(id)); err != nil {
return errors.Wrapf(err, "error pinning snapshots by root ID %v", id)
}
}
return nil
}
func (c *commandSnapshotPin) pinSnapshotsByRootObjectID(ctx context.Context, rep repo.RepositoryWriter, rootID object.ID) error {
manifests, err := snapshot.FindSnapshotsByRootObjectID(ctx, rep, rootID)
if err != nil {
return errors.Wrapf(err, "unable to find snapshots by root %v", rootID)
}
if len(manifests) == 0 {
return errors.Errorf("no snapshots matched %v", rootID)
}
for _, m := range manifests {
if err := c.pinSnapshot(ctx, rep, m); err != nil {
return errors.Wrap(err, "error pinning")
}
}
return nil
}
func (c *commandSnapshotPin) pinSnapshot(ctx context.Context, rep repo.RepositoryWriter, m *snapshot.Manifest) error {
if !m.UpdatePins(c.addPins, c.removePins) {
log(ctx).Infof("No change for snapshot at %v of %v", formatTimestamp(m.StartTime), m.Source)
return nil
}
log(ctx).Infof("Updating snapshot at %v of %v", formatTimestamp(m.StartTime), m.Source)
return errors.Wrap(snapshot.UpdateSnapshot(ctx, rep, m), "error updating snapshot")
}

89
cli/command_snapshot_pin_test.go Normal file
View File

@@ -0,0 +1,89 @@
package cli_test
import (
"os"
"path/filepath"
"testing"
"github.com/stretchr/testify/require"
"github.com/kopia/kopia/internal/testutil"
"github.com/kopia/kopia/snapshot"
"github.com/kopia/kopia/tests/testenv"
)
func TestSnapshotPin(t *testing.T) {
t.Parallel()
runner := testenv.NewInProcRunner(t)
e := testenv.NewCLITest(t, testenv.RepoFormatNotImportant, runner)
defer e.RunAndExpectSuccess(t, "repo", "disconnect")
e.RunAndExpectSuccess(t, "repo", "create", "filesystem", "--path", e.RepoDir)
srcdir := testutil.TempDirectory(t)
require.NoError(t, os.WriteFile(filepath.Join(srcdir, "some-file2"), []byte{1, 2, 3}, 0o755))
var man snapshot.Manifest
e.RunAndExpectSuccess(t, "policy", "set", srcdir, "--keep-latest=3", "--keep-hourly=0", "--keep-daily=0", "--keep-monthly=0", "--keep-weekly=0", "--keep-annual=0")
testutil.MustParseJSONLines(t, e.RunAndExpectSuccess(t, "snapshot", "create", srcdir, "--pin=a", "--pin=b", "--json"), &man)
require.Equal(t, []string{"a", "b"}, man.Pins)
e.RunAndExpectSuccess(t, "snapshot", "list")
// create more unpinned snapshots
e.RunAndExpectSuccess(t, "snapshot", "create", srcdir)
e.RunAndExpectSuccess(t, "snapshot", "create", srcdir)
e.RunAndExpectSuccess(t, "snapshot", "create", srcdir)
e.RunAndExpectSuccess(t, "snapshot", "create", srcdir)
e.RunAndExpectSuccess(t, "snapshot", "create", srcdir)
var snapshots []*snapshot.Manifest
testutil.MustParseJSONLines(t, e.RunAndExpectSuccess(t, "snapshot", "list", "--json"), &snapshots)
snapshots = snapshot.SortByTime(snapshots, false)
// make sure the pinned one is on top.
require.Len(t, snapshots, 4)
require.Equal(t, []string{"a", "b"}, snapshots[0].Pins)
require.Empty(t, snapshots[1].Pins)
require.Empty(t, snapshots[2].Pins)
require.Empty(t, snapshots[3].Pins)
// neither --add nor --remove were provided
e.RunAndExpectFailure(t, "snapshot", "pin", string(snapshots[3].ID))
e.RunAndExpectSuccess(t, "snapshot", "pin", string(snapshots[0].ID), "--add=c", "--remove=b")
e.RunAndExpectSuccess(t, "snapshot", "pin", string(snapshots[3].ID), "--add=d")
var snapshots2 []*snapshot.Manifest
testutil.MustParseJSONLines(t, e.RunAndExpectSuccess(t, "snapshot", "list", "--json"), &snapshots2)
snapshots2 = snapshot.SortByTime(snapshots2, false)
require.Len(t, snapshots2, 4)
require.Equal(t, []string{"a", "c"}, snapshots2[0].Pins)
require.Empty(t, snapshots2[1].Pins)
require.Empty(t, snapshots2[2].Pins)
require.Equal(t, []string{"d"}, snapshots2[3].Pins)
// create more unpinned snapshots
e.RunAndExpectSuccess(t, "snapshot", "create", srcdir)
e.RunAndExpectSuccess(t, "snapshot", "create", srcdir)
e.RunAndExpectSuccess(t, "snapshot", "create", srcdir)
e.RunAndExpectSuccess(t, "snapshot", "create", srcdir)
e.RunAndExpectSuccess(t, "snapshot", "create", srcdir)
var snapshots3 []*snapshot.Manifest
testutil.MustParseJSONLines(t, e.RunAndExpectSuccess(t, "snapshot", "list", "--json"), &snapshots3)
snapshots3 = snapshot.SortByTime(snapshots3, false)
require.Len(t, snapshots3, 5)
require.Equal(t, []string{"a", "c"}, snapshots3[0].Pins)
require.Equal(t, []string{"d"}, snapshots3[1].Pins)
require.Empty(t, snapshots3[2].Pins)
require.Empty(t, snapshots3[3].Pins)
require.Empty(t, snapshots3[4].Pins)
}

18
snapshot/manager.go
View File

@@ -228,6 +228,24 @@ func FindSnapshotsByRootObjectID(ctx context.Context, rep repo.Repository, rootI
return result, nil
}
// UpdateSnapshot updates the snapshot by saving the provided data and deleting old manifest ID.
func UpdateSnapshot(ctx context.Context, rep repo.RepositoryWriter, m *Manifest) error {
oldID := m.ID
newID, err := SaveSnapshot(ctx, rep, m)
if err != nil {
return errors.Wrap(err, "error saving snapshot")
}
if oldID != newID {
if err := rep.DeleteManifest(ctx, oldID); err != nil {
return errors.Wrap(err, "error deleting old manifest")
}
}
return nil
}
func entryIDs(entries []*manifest.EntryMetadata) []manifest.ID {
var ids []manifest.ID
for _, e := range entries {

37
snapshot/manifest.go
View File

@@ -31,6 +31,43 @@ type Manifest struct {
RetentionReasons []string `json:"-"`
Tags map[string]string `json:"tags,omitempty"`
// list of manually-defined pins which prevent the snapshot from being deleted.
Pins []string `json:"pins,omitempty"`
}
// UpdatePins updates pins in the provided manifest.
func (m *Manifest) UpdatePins(add, remove []string) bool {
newPins := map[string]bool{}
changed := false
for _, r := range m.Pins {
newPins[r] = true
}
for _, r := range add {
if !newPins[r] {
newPins[r] = true
changed = true
}
}
for _, r := range remove {
if newPins[r] {
delete(newPins, r)
changed = true
}
}
m.Pins = nil
for r := range newPins {
m.Pins = append(m.Pins, r)
}
sort.Strings(m.Pins)
return changed
}
// EntryType is a type of a filesystem entry.

4
snapshot/policy/expire.go
View File

@@ -61,11 +61,11 @@ func getExpiredSnapshotsForSource(ctx context.Context, rep repo.Repository, snap
var toDelete []*snapshot.Manifest
for _, s := range snapshots {
if len(s.RetentionReasons) == 0 {
if len(s.RetentionReasons) == 0 && len(s.Pins) == 0 {
log(ctx).Debugf(" deleting %v", s.StartTime)
toDelete = append(toDelete, s)
} else {
log(ctx).Debugf(" keeping %v reasons: [%v]", s.StartTime, strings.Join(s.RetentionReasons, ","))
log(ctx).Debugf(" keeping %v retention: [%v] pins: [%v]", s.StartTime, strings.Join(s.RetentionReasons, ","), strings.Join(s.Pins, ","))
}
}

23
snapshot/snapshot_test.go
View File

@@ -8,6 +8,8 @@
"sort"
"testing"
"github.com/stretchr/testify/require"
"github.com/kopia/kopia/internal/repotesting"
"github.com/kopia/kopia/internal/testlogging"
"github.com/kopia/kopia/repo"
@@ -76,6 +78,15 @@ func TestSnapshotsAPI(t *testing.T) {
verifySnapshotManifestIDs(t, env.RepositoryWriter, &src2, []manifest.ID{id3})
verifySources(t, env.RepositoryWriter, src1, src2)
verifyLoadSnapshots(t, env.RepositoryWriter, []manifest.ID{id1, id2, id3}, []*snapshot.Manifest{manifest1, manifest2, manifest3})
require.True(t, manifest3.UpdatePins([]string{"new-pin"}, nil))
require.NoError(t, snapshot.UpdateSnapshot(ctx, env.RepositoryWriter, manifest3))
require.NotEqual(t, manifest3.ID, id3)
updated3, err := snapshot.LoadSnapshot(ctx, env.RepositoryWriter, manifest3.ID)
require.NoError(t, err)
require.Equal(t, updated3, manifest3)
}
func verifySnapshotManifestIDs(t *testing.T, rep repo.Repository, src *snapshot.SourceInfo, expected []manifest.ID) {
@@ -249,3 +260,15 @@ func TestParseInvalidSourceInfo(t *testing.T) {
}
}
}
func TestUpdatePins(t *testing.T) {
m := snapshot.Manifest{}
require.True(t, m.UpdatePins([]string{"d", "c", "b"}, nil))
require.False(t, m.UpdatePins([]string{"d", "c", "b"}, nil))
require.Equal(t, []string{"b", "c", "d"}, m.Pins) // pins are sorted
require.True(t, m.UpdatePins([]string{"e", "a"}, []string{"c"}))
require.False(t, m.UpdatePins([]string{"e", "a"}, []string{"c"}))
require.Equal(t, []string{"a", "b", "d", "e"}, m.Pins)
}