From 7673753050625e80ffeeb359869d3eb56d1a94e2 Mon Sep 17 00:00:00 2001 From: Jarek Kowalski Date: Sun, 5 Dec 2021 20:49:41 -0800 Subject: [PATCH] Merge retention tags in snapshot lists (#1567) * cli: refactored snapshot list * cli: show range tags in snapshot list For example if N snapshots are coalesced together because they have identical roots we may emit now: ``` 2021-03-31 23:09:27 PDT ked3400debc7dd61baffab070bafd59cd (monthly-10) 2021-04-30 06:12:53 PDT kd0576d212e55a831b7ff1636f90a7233 (monthly-4..9) + 5 identical snapshots until 2021-09-30 23:00:19 PDT 2021-10-31 23:22:25 PDT k846bf22aa2863d27f05e820f840b14f8 (monthly-3) 2021-11-08 21:29:31 PST k5793ddcd61ef27b93c75ab74a5828176 (latest-1..3,hourly-1..13,daily-1..7,weekly-1..4,monthly-1..2,annual-1) + 18 identical snapshots until 2021-12-04 10:09:54 PST ``` * server: server-side coalescing of snapshot * ui: added coalescing of retention tags --- cli/command_snapshot_list.go | 127 ++++++++++++++------- htmlui/src/SnapshotsTable.js | 55 ++++----- internal/server/api_policies.go | 10 +- internal/server/api_snapshots.go | 64 ++++++++--- internal/serverapi/client_wrappers.go | 12 +- internal/serverapi/serverapi.go | 6 +- snapshot/policy/retention_policy.go | 120 +++++++++++++++++++ snapshot/policy/retention_policy_test.go | 29 ++++- tests/end_to_end_test/server_start_test.go | 24 ++-- 9 files changed, 334 insertions(+), 113 deletions(-) diff --git a/cli/command_snapshot_list.go b/cli/command_snapshot_list.go index 0809d62eb..f371a5f08 100644 --- a/cli/command_snapshot_list.go +++ b/cli/command_snapshot_list.go @@ -196,29 +196,26 @@ func (c *commandSnapshotList) outputManifestGroups(ctx context.Context, rep repo return nil } +type snapshotListRow struct { + firstStartTime time.Time + lastStartTime time.Time + count int + oid object.ID + bits []string + retentionReasons []string + pins []string + color *color.Color +} + func (c *commandSnapshotList) outputManifestFromSingleSource(ctx context.Context, rep repo.Repository, manifests []*snapshot.Manifest, parts []string) error { - var ( - count int - lastTotalFileSize int64 - previousOID object.ID - elidedCount int - maxElidedTime time.Time - ) + var lastTotalFileSize int64 manifests = snapshot.SortByTime(manifests, false) if c.maxResultsPerPath > 0 && len(manifests) > c.maxResultsPerPath { manifests = manifests[len(manifests)-c.maxResultsPerPath:] } - outputElided := func() { - if elidedCount > 0 { - c.out.printStdout( - " + %v identical snapshots until %v\n", - elidedCount, - formatTimestamp(maxElidedTime), - ) - } - } + var rows []*snapshotListRow for _, m := range manifests { root, err := snapshotfs.SnapshotRoot(rep, m) @@ -244,34 +241,86 @@ func (c *commandSnapshotList) outputManifestFromSingleSource(ctx context.Context bits, col := c.entryBits(ctx, m, ent, lastTotalFileSize) - oid := ent.(object.HasObjectID).ObjectID() - if !c.snapshotListShowIdentical && oid == previousOID { - elidedCount++ - - maxElidedTime = m.StartTime - - continue - } - - outputElided() - - elidedCount = 0 - previousOID = oid - - col.Fprint(c.out.stdout(), fmt.Sprintf(" %v %v %v\n", formatTimestamp(m.StartTime), oid, strings.Join(bits, " "))) //nolint:errcheck - - count++ + rows = append(rows, &snapshotListRow{ + firstStartTime: m.StartTime, + lastStartTime: m.StartTime, + count: 1, + oid: ent.(object.HasObjectID).ObjectID(), + bits: bits, + retentionReasons: m.RetentionReasons, + pins: m.Pins, + color: col, + }) if m.IncompleteReason == "" { lastTotalFileSize = m.Stats.TotalFileSize } } - outputElided() + if !c.snapshotListShowIdentical { + rows = c.mergeIdenticalRows(rows) + } + + c.outputSnapshotRows(rows) return nil } +func (c *commandSnapshotList) mergeIdenticalRows(rows []*snapshotListRow) []*snapshotListRow { + var result []*snapshotListRow + + for _, r := range rows { + if len(result) == 0 { + result = append(result, r) + continue + } + + last := result[len(result)-1] + + if r.oid == last.oid { + last.count++ + last.lastStartTime = r.lastStartTime + last.retentionReasons = append(last.retentionReasons, r.retentionReasons...) + last.pins = append(last.pins, r.pins...) + } else { + result = append(result, r) + } + } + + for _, r := range result { + r.retentionReasons = policy.CompactRetentionReasons(r.retentionReasons) + r.pins = policy.CompactPins(r.pins) + } + + return result +} + +func (c *commandSnapshotList) outputSnapshotRows(rows []*snapshotListRow) { + for _, row := range rows { + bits := append([]string(nil), row.bits...) + + if c.snapshotListShowRetentionReasons { + if len(row.retentionReasons) > 0 { + bits = append(bits, "("+strings.Join(row.retentionReasons, ",")+")") + } + } + + if len(row.pins) > 0 { + bits = append(bits, "pins:"+strings.Join(row.pins, ",")) + } + + row.color.Fprint(c.out.stdout(), fmt.Sprintf(" %v %v %v\n", formatTimestamp(row.firstStartTime), row.oid, strings.Join(bits, " "))) //nolint:errcheck + + if row.count > 1 { + c.out.printStdout( + " + %v identical snapshots until %v\n", + row.count-1, + formatTimestamp(row.lastStartTime), + ) + } + } +} + func (c *commandSnapshotList) entryBits(ctx context.Context, m *snapshot.Manifest, ent fs.Entry, lastTotalFileSize int64) (bits []string, col *color.Color) { col = color.New() // default color @@ -312,16 +361,6 @@ func (c *commandSnapshotList) entryBits(ctx context.Context, m *snapshot.Manifes } } - if c.snapshotListShowRetentionReasons { - if len(m.RetentionReasons) > 0 { - bits = append(bits, "("+strings.Join(m.RetentionReasons, ",")+")") - } - } - - if len(m.Pins) > 0 { - bits = append(bits, "pins:"+strings.Join(m.Pins, ",")) - } - return bits, col } diff --git a/htmlui/src/SnapshotsTable.js b/htmlui/src/SnapshotsTable.js index f061bc3bc..887ae2275 100644 --- a/htmlui/src/SnapshotsTable.js +++ b/htmlui/src/SnapshotsTable.js @@ -39,7 +39,18 @@ export class SnapshotsTable extends Component { }; this.onChange = this.onChange.bind(this); } + + componentDidUpdate(oldProps, oldState) { + if (this.state.showHidden !== oldState.showHidden) { + this.fetchSnapshots(); + } + } + componentDidMount() { + this.fetchSnapshots(); + } + + fetchSnapshots() { let q = parseQuery(this.props.location.search); this.setState({ @@ -50,11 +61,19 @@ export class SnapshotsTable extends Component { hiddenCount: 0, selectedSnapshot: null, }); - const u = '/api/v1/snapshots?' + sourceQueryStringParams(q); + + let u = '/api/v1/snapshots?' + sourceQueryStringParams(q); + + if (this.state.showHidden) { + u += "&all=1"; + } + axios.get(u).then(result => { console.log('got snapshots', result.data); this.setState({ snapshots: result.data.snapshots, + unfilteredCount: result.data.unfilteredCount, + uniqueCount: result.data.uniqueCount, isLoading: false, }); }).catch(error => this.setState({ @@ -63,28 +82,6 @@ export class SnapshotsTable extends Component { })); } - coalesceSnapshots(s) { - let filteredSnapshots = []; - - let lastRootID = ""; - let hiddenCount = 0; - - for (let i = 0; i < s.length; i++) { - if (s[i].rootID !== lastRootID) { - filteredSnapshots.push(s[i]); - } else { - hiddenCount++; - } - lastRootID = s[i].rootID; - } - - if (this.state.showHidden) { - return { filteredSnapshots: s, hiddenCount: hiddenCount }; - } - - return { filteredSnapshots, hiddenCount }; - } - selectSnapshot(x) { this.setState({ selectedSnapshot: x, @@ -98,7 +95,7 @@ export class SnapshotsTable extends Component { } render() { - let { snapshots, isLoading, error } = this.state; + let { snapshots, unfilteredCount, uniqueCount, isLoading, error } = this.state; if (error) { return

{error.message}

; } @@ -109,8 +106,6 @@ export class SnapshotsTable extends Component { snapshots.sort((a, b) => -compare(a.startTime, b.startTime)); - let { filteredSnapshots, hiddenCount } = this.coalesceSnapshots(snapshots); - const columns = [{ id: 'startTime', Header: 'Start time', @@ -148,20 +143,20 @@ export class SnapshotsTable extends Component {   - Displaying {filteredSnapshots.length !== snapshots.length ? filteredSnapshots.length + ' out of ' + snapshots.length : snapshots.length} snapshots of {this.state.userName}@{this.state.host}:{this.state.path} - {hiddenCount > 0 && + Displaying {snapshots.length !== unfilteredCount ? snapshots.length + ' out of ' + unfilteredCount : snapshots.length} snapshots of {this.state.userName}@{this.state.host}:{this.state.path} + {unfilteredCount !== uniqueCount && <>  }
- + ; } diff --git a/internal/server/api_policies.go b/internal/server/api_policies.go index 6be45c797..8be1c29ef 100644 --- a/internal/server/api_policies.go +++ b/internal/server/api_policies.go @@ -42,7 +42,7 @@ func (s *Server) handlePolicyList(ctx context.Context, r *http.Request, body []b return resp, nil } -func getPolicyTargetFromURL(u *url.URL) snapshot.SourceInfo { +func getSnapshotSourceFromURL(u *url.URL) snapshot.SourceInfo { host := u.Query().Get("host") path := u.Query().Get("path") username := u.Query().Get("userName") @@ -55,7 +55,7 @@ func getPolicyTargetFromURL(u *url.URL) snapshot.SourceInfo { } func (s *Server) handlePolicyGet(ctx context.Context, r *http.Request, body []byte) (interface{}, *apiError) { - pol, err := policy.GetDefinedPolicy(ctx, s.rep, getPolicyTargetFromURL(r.URL)) + pol, err := policy.GetDefinedPolicy(ctx, s.rep, getSnapshotSourceFromURL(r.URL)) if errors.Is(err, policy.ErrPolicyNotFound) { return nil, requestError(serverapi.ErrorNotFound, "policy not found") } @@ -70,7 +70,7 @@ func (s *Server) handlePolicyResolve(ctx context.Context, r *http.Request, body return nil, requestError(serverapi.ErrorMalformedRequest, "unable to decode request: "+err.Error()) } - target := getPolicyTargetFromURL(r.URL) + target := getSnapshotSourceFromURL(r.URL) // build a list of parents policies, err := policy.GetPolicyHierarchy(ctx, s.rep, target, nil) @@ -110,7 +110,7 @@ func (s *Server) handlePolicyDelete(ctx context.Context, r *http.Request, body [ return nil, repositoryNotWritableError() } - sourceInfo := getPolicyTargetFromURL(r.URL) + sourceInfo := getSnapshotSourceFromURL(r.URL) if err := repo.WriteSession(ctx, s.rep, repo.WriteSessionOptions{ Purpose: "PolicyDelete", @@ -135,7 +135,7 @@ func (s *Server) handlePolicyPut(ctx context.Context, r *http.Request, body []by return nil, repositoryNotWritableError() } - sourceInfo := getPolicyTargetFromURL(r.URL) + sourceInfo := getSnapshotSourceFromURL(r.URL) if err := repo.WriteSession(ctx, s.rep, repo.WriteSessionOptions{ Purpose: "PolicyPut", diff --git a/internal/server/api_snapshots.go b/internal/server/api_snapshots.go index dcccb7fcb..fa3c3b39e 100644 --- a/internal/server/api_snapshots.go +++ b/internal/server/api_snapshots.go @@ -11,7 +11,9 @@ ) func (s *Server) handleSnapshotList(ctx context.Context, r *http.Request, body []byte) (interface{}, *apiError) { - manifestIDs, err := snapshot.ListSnapshotManifests(ctx, s.rep, nil, nil) + si := getSnapshotSourceFromURL(r.URL) + + manifestIDs, err := snapshot.ListSnapshotManifests(ctx, s.rep, &si, nil) if err != nil { return nil, internalServerError(err) } @@ -21,30 +23,60 @@ func (s *Server) handleSnapshotList(ctx context.Context, r *http.Request, body [ return nil, internalServerError(err) } + manifests = snapshot.SortByTime(manifests, false) + resp := &serverapi.SnapshotsResponse{ Snapshots: []*serverapi.Snapshot{}, } - groups := snapshot.GroupBySource(manifests) - for _, grp := range groups { - first := grp[0] - if !sourceMatchesURLFilter(first.Source, r.URL.Query()) { - continue - } + pol, _, _, err := policy.GetEffectivePolicy(ctx, s.rep, si) + if err == nil { + pol.RetentionPolicy.ComputeRetentionReasons(manifests) + } - pol, _, _, err := policy.GetEffectivePolicy(ctx, s.rep, first.Source) - if err == nil { - pol.RetentionPolicy.ComputeRetentionReasons(grp) - } + for _, m := range manifests { + resp.Snapshots = append(resp.Snapshots, convertSnapshotManifest(m)) + } - for _, m := range grp { - resp.Snapshots = append(resp.Snapshots, convertSnapshotManifest(m)) - } + resp.UnfilteredCount = len(resp.Snapshots) + + if r.URL.Query().Get("all") == "" { + resp.Snapshots = uniqueSnapshots(resp.Snapshots) + resp.UniqueCount = len(resp.Snapshots) + } else { + resp.UniqueCount = len(uniqueSnapshots(resp.Snapshots)) } return resp, nil } +func uniqueSnapshots(rows []*serverapi.Snapshot) []*serverapi.Snapshot { + var result []*serverapi.Snapshot + + for _, r := range rows { + if len(result) == 0 { + result = append(result, r) + continue + } + + last := result[len(result)-1] + + if r.RootEntry == last.RootEntry { + last.RetentionReasons = append(last.RetentionReasons, r.RetentionReasons...) + last.Pins = append(last.Pins, r.Pins...) + } else { + result = append(result, r) + } + } + + for _, r := range result { + r.RetentionReasons = policy.CompactRetentionReasons(r.RetentionReasons) + r.Pins = policy.CompactPins(r.Pins) + } + + return result +} + func sourceMatchesURLFilter(src snapshot.SourceInfo, query url.Values) bool { if v := query.Get("host"); v != "" && src.Host != v { return false @@ -64,13 +96,13 @@ func sourceMatchesURLFilter(src snapshot.SourceInfo, query url.Values) bool { func convertSnapshotManifest(m *snapshot.Manifest) *serverapi.Snapshot { e := &serverapi.Snapshot{ ID: m.ID, - Source: m.Source, Description: m.Description, StartTime: m.StartTime, EndTime: m.EndTime, IncompleteReason: m.IncompleteReason, RootEntry: m.RootObjectID().String(), - RetentionReasons: m.RetentionReasons, + RetentionReasons: append([]string{}, m.RetentionReasons...), + Pins: append([]string{}, m.Pins...), } if re := m.RootEntry; re != nil { diff --git a/internal/serverapi/client_wrappers.go b/internal/serverapi/client_wrappers.go index bf9990030..c36205b80 100644 --- a/internal/serverapi/client_wrappers.go +++ b/internal/serverapi/client_wrappers.go @@ -128,10 +128,16 @@ func ListSources(ctx context.Context, c *apiclient.KopiaAPIClient, match *snapsh return resp, nil } -// ListSnapshots lists the snapshots managed by the server for a given source filter. -func ListSnapshots(ctx context.Context, c *apiclient.KopiaAPIClient, match *snapshot.SourceInfo) (*SnapshotsResponse, error) { +// ListSnapshots lists the snapshots managed by the server for a given source source. +func ListSnapshots(ctx context.Context, c *apiclient.KopiaAPIClient, src snapshot.SourceInfo, all bool) (*SnapshotsResponse, error) { resp := &SnapshotsResponse{} - if err := c.Get(ctx, "snapshots"+matchSourceParameters(match), nil, resp); err != nil { + + u := "snapshots" + matchSourceParameters(&src) + if all { + u += "&all=1" + } + + if err := c.Get(ctx, u, nil, resp); err != nil { return nil, errors.Wrap(err, "ListSnapshots") } diff --git a/internal/serverapi/serverapi.go b/internal/serverapi/serverapi.go index 3e65faff4..696c3a78f 100644 --- a/internal/serverapi/serverapi.go +++ b/internal/serverapi/serverapi.go @@ -149,7 +149,6 @@ type CreateSnapshotSourceResponse struct { // Snapshot describes single snapshot entry. type Snapshot struct { ID manifest.ID `json:"id"` - Source snapshot.SourceInfo `json:"source"` Description string `json:"description"` StartTime time.Time `json:"startTime"` EndTime time.Time `json:"endTime"` @@ -157,11 +156,14 @@ type Snapshot struct { Summary *fs.DirectorySummary `json:"summary"` RootEntry string `json:"rootID"` RetentionReasons []string `json:"retention"` + Pins []string `json:"pins"` } // SnapshotsResponse contains a list of snapshots. type SnapshotsResponse struct { - Snapshots []*Snapshot `json:"snapshots"` + Snapshots []*Snapshot `json:"snapshots"` + UnfilteredCount int `json:"unfilteredCount"` + UniqueCount int `json:"uniqueCount"` } // MountSnapshotRequest contains request to mount a snapshot. diff --git a/snapshot/policy/retention_policy.go b/snapshot/policy/retention_policy.go index 65e9f5010..aafc07a73 100644 --- a/snapshot/policy/retention_policy.go +++ b/snapshot/policy/retention_policy.go @@ -2,6 +2,9 @@ import ( "fmt" + "sort" + "strconv" + "strings" "time" "github.com/kopia/kopia/snapshot" @@ -153,6 +156,8 @@ func (r *RetentionPolicy) getRetentionReasons(i int, s *snapshot.Manifest, cutof } } + SortRetentionTags(keepReasons) + return keepReasons } @@ -211,3 +216,118 @@ func (r *RetentionPolicy) Merge(src RetentionPolicy, def *RetentionPolicyDefinit mergeOptionalInt(&r.KeepMonthly, src.KeepMonthly, &def.KeepMonthly, si) mergeOptionalInt(&r.KeepAnnual, src.KeepAnnual, &def.KeepAnnual, si) } + +// CompactRetentionReasons returns compressed retention reasons given a list of retention reasons. +func CompactRetentionReasons(reasons []string) []string { + reasonsByPrefix := map[string][]int{} + + result := []string{} + + for _, r := range reasons { + prefix, suffix := prefixSuffix(r) + + n, err := strconv.Atoi(suffix) + if err != nil { + result = append(result, r) + continue + } + + reasonsByPrefix[prefix] = append(reasonsByPrefix[prefix], n) + } + + for prefix, v := range reasonsByPrefix { + result = appendRLE(result, prefix, v) + } + + SortRetentionTags(result) + + return result +} + +func prefixSuffix(s string) (prefix, suffix string) { + if p := strings.LastIndex(s, "-"); p < 0 { + prefix = s + suffix = "" + } else { + prefix = s[0:p] + suffix = s[p+1:] + } + + return +} + +func appendRLE(out []string, prefix string, numbers []int) []string { + sort.Ints(numbers) + + runStart := numbers[0] + runEnd := numbers[0] + + appendRun := func() { + if runStart == runEnd { + out = append(out, fmt.Sprintf("%v-%v", prefix, runStart)) + } else { + out = append(out, fmt.Sprintf("%v-%v..%v", prefix, runStart, runEnd)) + } + } + + for _, num := range numbers[1:] { + if num == runEnd+1 { + runEnd = num + } else { + appendRun() + + runStart = num + runEnd = runStart + } + } + + appendRun() + + return out +} + +// CompactPins returns compressed pins reasons given a list of pins. +func CompactPins(pins []string) []string { + cnt := map[string]int{} + + for _, p := range pins { + cnt[p]++ + } + + result := []string{} + + for k := range cnt { + result = append(result, k) + } + + sort.Strings(result) + + return result +} + +var retentionPrefixSortValue = map[string]int{ + "latest": 1, + "hourly": 2, // nolint:gomnd + "daily": 3, // nolint:gomnd + "weekly": 4, // nolint:gomnd + "monthly": 5, // nolint:gomnd + "annual": 6, // nolint:gomnd +} + +// SortRetentionTags sorts the provided retention tags in canonical order. +func SortRetentionTags(tags []string) { + sort.Slice(tags, func(i, j int) bool { + p1, s1 := prefixSuffix(tags[i]) + p2, s2 := prefixSuffix(tags[j]) + + if l, r := retentionPrefixSortValue[p1], retentionPrefixSortValue[p2]; l != r { + return l < r + } + + if l, r := p1, p2; l != r { + return p1 < p2 + } + + return s1 < s2 + }) +} diff --git a/snapshot/policy/retention_policy_test.go b/snapshot/policy/retention_policy_test.go index a4b891ad1..43b10c1fb 100644 --- a/snapshot/policy/retention_policy_test.go +++ b/snapshot/policy/retention_policy_test.go @@ -7,6 +7,7 @@ "time" "github.com/google/go-cmp/cmp" + "github.com/stretchr/testify/require" "github.com/kopia/kopia/snapshot" ) @@ -108,7 +109,7 @@ func TestRetentionPolicyTest(t *testing.T) { "2020-04-01T15:00:00Z": {"daily-2"}, "2020-04-02T12:00:00Z": {"latest-3", "hourly-3"}, "2020-04-02T13:00:00Z": {"latest-2", "hourly-2"}, - "2020-04-02T15:00:00Z": {"latest-1", "monthly-1", "daily-1", "hourly-1"}, + "2020-04-02T15:00:00Z": {"latest-1", "hourly-1", "daily-1", "monthly-1"}, "incomplete-2020-04-02T15:01:00Z": {}, // incomplete, too old "incomplete-2020-04-02T16:01:00Z": {}, // incomplete, too old "incomplete-2020-04-02T17:01:00Z": {}, // incomplete, too old @@ -206,3 +207,29 @@ func TestRetentionPolicyTest(t *testing.T) { }) } } + +func TestCompactPins(t *testing.T) { + require.Equal(t, + []string{"a", "b", "d", "x", "z"}, + CompactPins([]string{ + "z", "x", "a", "b", "d", "b", "z", + })) +} + +func TestCompactRetentionrRasons(t *testing.T) { + cases := []struct { + input []string + want []string + }{ + {input: nil, want: []string{}}, + {[]string{"latest-1", "latest-2"}, []string{"latest-1..2"}}, + {[]string{"latest-1", "daily-3", "latest-2", "daily-2"}, []string{"latest-1..2", "daily-2..3"}}, + {[]string{"latest-1", "weekly-7", "latest-2"}, []string{"latest-1..2", "weekly-7"}}, + {[]string{"latest-1", "latest-2", "latest-5", "latest-6", "latest-7"}, []string{"latest-1..2", "latest-5..7"}}, + {[]string{"latest-1", "zrogue", "arogue", "latest-2"}, []string{"arogue", "zrogue", "latest-1..2"}}, + } + + for _, tc := range cases { + require.Equal(t, tc.want, CompactRetentionReasons(tc.input)) + } +} diff --git a/tests/end_to_end_test/server_start_test.go b/tests/end_to_end_test/server_start_test.go index 779c5a65e..fcde08439 100644 --- a/tests/end_to_end_test/server_start_test.go +++ b/tests/end_to_end_test/server_start_test.go @@ -138,18 +138,18 @@ func TestServerStart(t *testing.T) { verifySourceCount(t, cli, &snapshot.SourceInfo{Host: "no-such-host"}, 0) verifySourceCount(t, cli, &snapshot.SourceInfo{Host: "fake-hostname", UserName: "fake-username", Path: sharedTestDataDir2}, 1) - verifySnapshotCount(t, cli, nil, 2) - verifySnapshotCount(t, cli, &snapshot.SourceInfo{Host: "fake-hostname", UserName: "fake-username", Path: sharedTestDataDir1}, 2) - verifySnapshotCount(t, cli, &snapshot.SourceInfo{Host: "fake-hostname", UserName: "fake-username", Path: sharedTestDataDir2}, 0) - verifySnapshotCount(t, cli, &snapshot.SourceInfo{Host: "no-such-host"}, 0) + verifySnapshotCount(t, cli, snapshot.SourceInfo{Host: "fake-hostname", UserName: "fake-username", Path: sharedTestDataDir1}, true, 2) + verifySnapshotCount(t, cli, snapshot.SourceInfo{Host: "fake-hostname", UserName: "fake-username", Path: sharedTestDataDir1}, false, 1) + verifySnapshotCount(t, cli, snapshot.SourceInfo{Host: "fake-hostname", UserName: "fake-username", Path: sharedTestDataDir2}, true, 0) + verifySnapshotCount(t, cli, snapshot.SourceInfo{Host: "no-such-host"}, true, 0) uploadMatchingSnapshots(t, cli, &snapshot.SourceInfo{Host: "fake-hostname", UserName: "fake-username", Path: sharedTestDataDir2}) - waitForSnapshotCount(ctx, t, cli, &snapshot.SourceInfo{Host: "fake-hostname", UserName: "fake-username", Path: sharedTestDataDir2}, 1) + waitForSnapshotCount(ctx, t, cli, snapshot.SourceInfo{Host: "fake-hostname", UserName: "fake-username", Path: sharedTestDataDir2}, 1) _, err = serverapi.CancelUpload(ctx, cli, nil) require.NoError(t, err) - snaps := verifySnapshotCount(t, cli, &snapshot.SourceInfo{Host: "fake-hostname", UserName: "fake-username", Path: sharedTestDataDir2}, 1) + snaps := verifySnapshotCount(t, cli, snapshot.SourceInfo{Host: "fake-hostname", UserName: "fake-username", Path: sharedTestDataDir2}, true, 1) rootPayload, err := serverapi.GetObject(ctx, cli, snaps[0].RootEntry) require.NoError(t, err) @@ -180,7 +180,7 @@ func TestServerStart(t *testing.T) { require.Len(t, policies.Policies, 1) require.Equal(t, keepDaily, *policies.Policies[0].Policy.RetentionPolicy.KeepDaily) - waitForSnapshotCount(ctx, t, cli, &snapshot.SourceInfo{Host: "fake-hostname", UserName: "fake-username", Path: sharedTestDataDir3}, 1) + waitForSnapshotCount(ctx, t, cli, snapshot.SourceInfo{Host: "fake-hostname", UserName: "fake-username", Path: sharedTestDataDir3}, 1) } func TestServerCreateAndConnectViaAPI(t *testing.T) { @@ -305,7 +305,7 @@ func TestConnectToExistingRepositoryViaAPI(t *testing.T) { uploadMatchingSnapshots(t, cli, &si) - snaps := waitForSnapshotCount(ctx, t, cli, &si, 3) + snaps := waitForSnapshotCount(ctx, t, cli, si, 3) // we're reproducing the bug described in, after connecting to repo via API, next snapshot size becomes zero. // https://kopia.discourse.group/t/kopia-0-7-0-not-backing-up-any-files-repro-needed/136/6?u=jkowalski @@ -383,13 +383,13 @@ func verifyServerConnected(t *testing.T, cli *apiclient.KopiaAPIClient, want boo return st } -func waitForSnapshotCount(ctx context.Context, t *testing.T, cli *apiclient.KopiaAPIClient, match *snapshot.SourceInfo, want int) []*serverapi.Snapshot { +func waitForSnapshotCount(ctx context.Context, t *testing.T, cli *apiclient.KopiaAPIClient, src snapshot.SourceInfo, want int) []*serverapi.Snapshot { t.Helper() var result []*serverapi.Snapshot err := retry.PeriodicallyNoValue(ctx, 1*time.Second, 180, "wait for snapshots", func() error { - snapshots, err := serverapi.ListSnapshots(testlogging.Context(t), cli, match) + snapshots, err := serverapi.ListSnapshots(testlogging.Context(t), cli, src, true) if err != nil { return errors.Wrap(err, "error listing sources") } @@ -437,10 +437,10 @@ func uploadMatchingSnapshots(t *testing.T, cli *apiclient.KopiaAPIClient, match } } -func verifySnapshotCount(t *testing.T, cli *apiclient.KopiaAPIClient, match *snapshot.SourceInfo, want int) []*serverapi.Snapshot { +func verifySnapshotCount(t *testing.T, cli *apiclient.KopiaAPIClient, src snapshot.SourceInfo, all bool, want int) []*serverapi.Snapshot { t.Helper() - snapshots, err := serverapi.ListSnapshots(testlogging.Context(t), cli, match) + snapshots, err := serverapi.ListSnapshots(testlogging.Context(t), cli, src, all) require.NoError(t, err) if got := len(snapshots.Snapshots); got != want {