mirror/kopia
1
0
Fork 0
mirror of https://github.com/kopia/kopia.git synced 2026-05-18 11:44:36 -04:00
Code Issues Packages Projects Releases Wiki Activity

Merge retention tags in snapshot lists (#1567)

Browse Source 
* cli: refactored snapshot list

* cli: show range tags in snapshot list

For example if N snapshots are coalesced together because they
have identical roots we may emit now:

```
  2021-03-31 23:09:27 PDT ked3400debc7dd61baffab070bafd59cd (monthly-10)
  2021-04-30 06:12:53 PDT kd0576d212e55a831b7ff1636f90a7233 (monthly-4..9)
  + 5 identical snapshots until 2021-09-30 23:00:19 PDT
  2021-10-31 23:22:25 PDT k846bf22aa2863d27f05e820f840b14f8 (monthly-3)
  2021-11-08 21:29:31 PST k5793ddcd61ef27b93c75ab74a5828176 (latest-1..3,hourly-1..13,daily-1..7,weekly-1..4,monthly-1..2,annual-1)
  + 18 identical snapshots until 2021-12-04 10:09:54 PST
```

* server: server-side coalescing of snapshot

* ui: added coalescing of retention tags
This commit is contained in:
Jarek Kowalski
2021-12-05 20:49:41 -08:00
committed by GitHub
parent 7a7de5c3f4
commit 7673753050
9 changed files with 334 additions and 113 deletions
Download Patch File Download Diff File Expand all files Collapse all files

127
cli/command_snapshot_list.go
View File

@@ -196,29 +196,26 @@ func (c *commandSnapshotList) outputManifestGroups(ctx context.Context, rep repo
return nil
}
type snapshotListRow struct {
firstStartTime time.Time
lastStartTime time.Time
count int
oid object.ID
bits []string
retentionReasons []string
pins []string
color *color.Color
}
func (c *commandSnapshotList) outputManifestFromSingleSource(ctx context.Context, rep repo.Repository, manifests []*snapshot.Manifest, parts []string) error {
var (
count int
lastTotalFileSize int64
previousOID object.ID
elidedCount int
maxElidedTime time.Time
)
var lastTotalFileSize int64
manifests = snapshot.SortByTime(manifests, false)
if c.maxResultsPerPath > 0 && len(manifests) > c.maxResultsPerPath {
manifests = manifests[len(manifests)-c.maxResultsPerPath:]
}
outputElided := func() {
if elidedCount > 0 {
c.out.printStdout(
" + %v identical snapshots until %v\n",
elidedCount,
formatTimestamp(maxElidedTime),
)
}
}
var rows []*snapshotListRow
for _, m := range manifests {
root, err := snapshotfs.SnapshotRoot(rep, m)
@@ -244,34 +241,86 @@ func (c *commandSnapshotList) outputManifestFromSingleSource(ctx context.Context
bits, col := c.entryBits(ctx, m, ent, lastTotalFileSize)
oid := ent.(object.HasObjectID).ObjectID()
if !c.snapshotListShowIdentical && oid == previousOID {
elidedCount++
maxElidedTime = m.StartTime
continue
}
outputElided()
elidedCount = 0
previousOID = oid
col.Fprint(c.out.stdout(), fmt.Sprintf(" %v %v %v\n", formatTimestamp(m.StartTime), oid, strings.Join(bits, " "))) //nolint:errcheck
count++
rows = append(rows, &snapshotListRow{
firstStartTime: m.StartTime,
lastStartTime: m.StartTime,
count: 1,
oid: ent.(object.HasObjectID).ObjectID(),
bits: bits,
retentionReasons: m.RetentionReasons,
pins: m.Pins,
color: col,
})
if m.IncompleteReason == "" {
lastTotalFileSize = m.Stats.TotalFileSize
}
}
outputElided()
if !c.snapshotListShowIdentical {
rows = c.mergeIdenticalRows(rows)
}
c.outputSnapshotRows(rows)
return nil
}
func (c *commandSnapshotList) mergeIdenticalRows(rows []*snapshotListRow) []*snapshotListRow {
var result []*snapshotListRow
for _, r := range rows {
if len(result) == 0 {
result = append(result, r)
continue
}
last := result[len(result)-1]
if r.oid == last.oid {
last.count++
last.lastStartTime = r.lastStartTime
last.retentionReasons = append(last.retentionReasons, r.retentionReasons...)
last.pins = append(last.pins, r.pins...)
} else {
result = append(result, r)
}
}
for _, r := range result {
r.retentionReasons = policy.CompactRetentionReasons(r.retentionReasons)
r.pins = policy.CompactPins(r.pins)
}
return result
}
func (c *commandSnapshotList) outputSnapshotRows(rows []*snapshotListRow) {
for _, row := range rows {
bits := append([]string(nil), row.bits...)
if c.snapshotListShowRetentionReasons {
if len(row.retentionReasons) > 0 {
bits = append(bits, "("+strings.Join(row.retentionReasons, ",")+")")
}
}
if len(row.pins) > 0 {
bits = append(bits, "pins:"+strings.Join(row.pins, ","))
}
row.color.Fprint(c.out.stdout(), fmt.Sprintf(" %v %v %v\n", formatTimestamp(row.firstStartTime), row.oid, strings.Join(bits, " "))) //nolint:errcheck
if row.count > 1 {
c.out.printStdout(
" + %v identical snapshots until %v\n",
row.count-1,
formatTimestamp(row.lastStartTime),
)
}
}
}
func (c *commandSnapshotList) entryBits(ctx context.Context, m *snapshot.Manifest, ent fs.Entry, lastTotalFileSize int64) (bits []string, col *color.Color) {
col = color.New() // default color
@@ -312,16 +361,6 @@ func (c *commandSnapshotList) entryBits(ctx context.Context, m *snapshot.Manifes
}
}
if c.snapshotListShowRetentionReasons {
if len(m.RetentionReasons) > 0 {
bits = append(bits, "("+strings.Join(m.RetentionReasons, ",")+")")
}
}
if len(m.Pins) > 0 {
bits = append(bits, "pins:"+strings.Join(m.Pins, ","))
}
return bits, col
}

55
htmlui/src/SnapshotsTable.js
View File

@@ -39,7 +39,18 @@ export class SnapshotsTable extends Component {
};
this.onChange = this.onChange.bind(this);
}
componentDidUpdate(oldProps, oldState) {
if (this.state.showHidden !== oldState.showHidden) {
this.fetchSnapshots();
}
}
componentDidMount() {
this.fetchSnapshots();
}
fetchSnapshots() {
let q = parseQuery(this.props.location.search);
this.setState({
@@ -50,11 +61,19 @@ export class SnapshotsTable extends Component {
hiddenCount: 0,
selectedSnapshot: null,
});
const u = '/api/v1/snapshots?' + sourceQueryStringParams(q);
let u = '/api/v1/snapshots?' + sourceQueryStringParams(q);
if (this.state.showHidden) {
u += "&all=1";
}
axios.get(u).then(result => {
console.log('got snapshots', result.data);
this.setState({
snapshots: result.data.snapshots,
unfilteredCount: result.data.unfilteredCount,
uniqueCount: result.data.uniqueCount,
isLoading: false,
});
}).catch(error => this.setState({
@@ -63,28 +82,6 @@ export class SnapshotsTable extends Component {
}));
}
coalesceSnapshots(s) {
let filteredSnapshots = [];
let lastRootID = "";
let hiddenCount = 0;
for (let i = 0; i < s.length; i++) {
if (s[i].rootID !== lastRootID) {
filteredSnapshots.push(s[i]);
} else {
hiddenCount++;
}
lastRootID = s[i].rootID;
}
if (this.state.showHidden) {
return { filteredSnapshots: s, hiddenCount: hiddenCount };
}
return { filteredSnapshots, hiddenCount };
}
selectSnapshot(x) {
this.setState({
selectedSnapshot: x,
@@ -98,7 +95,7 @@ export class SnapshotsTable extends Component {
}
render() {
let { snapshots, isLoading, error } = this.state;
let { snapshots, unfilteredCount, uniqueCount, isLoading, error } = this.state;
if (error) {
return <p>{error.message}</p>;
}
@@ -109,8 +106,6 @@ export class SnapshotsTable extends Component {
snapshots.sort((a, b) => -compare(a.startTime, b.startTime));
let { filteredSnapshots, hiddenCount } = this.coalesceSnapshots(snapshots);
const columns = [{
id: 'startTime',
Header: 'Start time',
@@ -148,20 +143,20 @@ export class SnapshotsTable extends Component {
<Col>
<GoBackButton onClick={this.props.history.goBack} />
&nbsp;
Displaying {filteredSnapshots.length !== snapshots.length ? filteredSnapshots.length + ' out of ' + snapshots.length : snapshots.length} snapshots of&nbsp;<b>{this.state.userName}@{this.state.host}:{this.state.path}</b>
{hiddenCount > 0 &&
Displaying {snapshots.length !== unfilteredCount ? snapshots.length + ' out of ' + unfilteredCount : snapshots.length} snapshots of&nbsp;<b>{this.state.userName}@{this.state.host}:{this.state.path}</b>
{unfilteredCount !== uniqueCount &&
<>&nbsp;<Form.Group controlId="formBasicCheckbox">
<Form.Check
type="checkbox"
checked={this.state.showHidden}
label={'Show ' + hiddenCount + ' identical snapshots'}
label={'Show ' + unfilteredCount + ' individual snapshots'}
onChange={this.onChange} />
</Form.Group></>}
</Col>
</Row>
<hr />
<Row>
<MyTable data={filteredSnapshots} columns={columns} />
<MyTable data={snapshots} columns={columns} />
</Row>
</div>;
}

10
internal/server/api_policies.go
View File

@@ -42,7 +42,7 @@ func (s *Server) handlePolicyList(ctx context.Context, r *http.Request, body []b
return resp, nil
}
func getPolicyTargetFromURL(u *url.URL) snapshot.SourceInfo {
func getSnapshotSourceFromURL(u *url.URL) snapshot.SourceInfo {
host := u.Query().Get("host")
path := u.Query().Get("path")
username := u.Query().Get("userName")
@@ -55,7 +55,7 @@ func getPolicyTargetFromURL(u *url.URL) snapshot.SourceInfo {
}
func (s *Server) handlePolicyGet(ctx context.Context, r *http.Request, body []byte) (interface{}, *apiError) {
pol, err := policy.GetDefinedPolicy(ctx, s.rep, getPolicyTargetFromURL(r.URL))
pol, err := policy.GetDefinedPolicy(ctx, s.rep, getSnapshotSourceFromURL(r.URL))
if errors.Is(err, policy.ErrPolicyNotFound) {
return nil, requestError(serverapi.ErrorNotFound, "policy not found")
}
@@ -70,7 +70,7 @@ func (s *Server) handlePolicyResolve(ctx context.Context, r *http.Request, body
return nil, requestError(serverapi.ErrorMalformedRequest, "unable to decode request: "+err.Error())
}
target := getPolicyTargetFromURL(r.URL)
target := getSnapshotSourceFromURL(r.URL)
// build a list of parents
policies, err := policy.GetPolicyHierarchy(ctx, s.rep, target, nil)
@@ -110,7 +110,7 @@ func (s *Server) handlePolicyDelete(ctx context.Context, r *http.Request, body [
return nil, repositoryNotWritableError()
}
sourceInfo := getPolicyTargetFromURL(r.URL)
sourceInfo := getSnapshotSourceFromURL(r.URL)
if err := repo.WriteSession(ctx, s.rep, repo.WriteSessionOptions{
Purpose: "PolicyDelete",
@@ -135,7 +135,7 @@ func (s *Server) handlePolicyPut(ctx context.Context, r *http.Request, body []by
return nil, repositoryNotWritableError()
}
sourceInfo := getPolicyTargetFromURL(r.URL)
sourceInfo := getSnapshotSourceFromURL(r.URL)
if err := repo.WriteSession(ctx, s.rep, repo.WriteSessionOptions{
Purpose: "PolicyPut",

64
internal/server/api_snapshots.go
View File

@@ -11,7 +11,9 @@
)
func (s *Server) handleSnapshotList(ctx context.Context, r *http.Request, body []byte) (interface{}, *apiError) {
manifestIDs, err := snapshot.ListSnapshotManifests(ctx, s.rep, nil, nil)
si := getSnapshotSourceFromURL(r.URL)
manifestIDs, err := snapshot.ListSnapshotManifests(ctx, s.rep, &si, nil)
if err != nil {
return nil, internalServerError(err)
}
@@ -21,30 +23,60 @@ func (s *Server) handleSnapshotList(ctx context.Context, r *http.Request, body [
return nil, internalServerError(err)
}
manifests = snapshot.SortByTime(manifests, false)
resp := &serverapi.SnapshotsResponse{
Snapshots: []*serverapi.Snapshot{},
}
groups := snapshot.GroupBySource(manifests)
for _, grp := range groups {
first := grp[0]
if !sourceMatchesURLFilter(first.Source, r.URL.Query()) {
continue
}
pol, _, _, err := policy.GetEffectivePolicy(ctx, s.rep, si)
if err == nil {
pol.RetentionPolicy.ComputeRetentionReasons(manifests)
}
pol, _, _, err := policy.GetEffectivePolicy(ctx, s.rep, first.Source)
if err == nil {
pol.RetentionPolicy.ComputeRetentionReasons(grp)
}
for _, m := range manifests {
resp.Snapshots = append(resp.Snapshots, convertSnapshotManifest(m))
}
for _, m := range grp {
resp.Snapshots = append(resp.Snapshots, convertSnapshotManifest(m))
}
resp.UnfilteredCount = len(resp.Snapshots)
if r.URL.Query().Get("all") == "" {
resp.Snapshots = uniqueSnapshots(resp.Snapshots)
resp.UniqueCount = len(resp.Snapshots)
} else {
resp.UniqueCount = len(uniqueSnapshots(resp.Snapshots))
}
return resp, nil
}
func uniqueSnapshots(rows []*serverapi.Snapshot) []*serverapi.Snapshot {
var result []*serverapi.Snapshot
for _, r := range rows {
if len(result) == 0 {
result = append(result, r)
continue
}
last := result[len(result)-1]
if r.RootEntry == last.RootEntry {
last.RetentionReasons = append(last.RetentionReasons, r.RetentionReasons...)
last.Pins = append(last.Pins, r.Pins...)
} else {
result = append(result, r)
}
}
for _, r := range result {
r.RetentionReasons = policy.CompactRetentionReasons(r.RetentionReasons)
r.Pins = policy.CompactPins(r.Pins)
}
return result
}
func sourceMatchesURLFilter(src snapshot.SourceInfo, query url.Values) bool {
if v := query.Get("host"); v != "" && src.Host != v {
return false
@@ -64,13 +96,13 @@ func sourceMatchesURLFilter(src snapshot.SourceInfo, query url.Values) bool {
func convertSnapshotManifest(m *snapshot.Manifest) *serverapi.Snapshot {
e := &serverapi.Snapshot{
ID: m.ID,
Source: m.Source,
Description: m.Description,
StartTime: m.StartTime,
EndTime: m.EndTime,
IncompleteReason: m.IncompleteReason,
RootEntry: m.RootObjectID().String(),
RetentionReasons: m.RetentionReasons,
RetentionReasons: append([]string{}, m.RetentionReasons...),
Pins: append([]string{}, m.Pins...),
}
if re := m.RootEntry; re != nil {

12
internal/serverapi/client_wrappers.go
View File

@@ -128,10 +128,16 @@ func ListSources(ctx context.Context, c *apiclient.KopiaAPIClient, match *snapsh
return resp, nil
}
// ListSnapshots lists the snapshots managed by the server for a given source filter.
func ListSnapshots(ctx context.Context, c *apiclient.KopiaAPIClient, match *snapshot.SourceInfo) (*SnapshotsResponse, error) {
// ListSnapshots lists the snapshots managed by the server for a given source source.
func ListSnapshots(ctx context.Context, c *apiclient.KopiaAPIClient, src snapshot.SourceInfo, all bool) (*SnapshotsResponse, error) {
resp := &SnapshotsResponse{}
if err := c.Get(ctx, "snapshots"+matchSourceParameters(match), nil, resp); err != nil {
u := "snapshots" + matchSourceParameters(&src)
if all {
u += "&all=1"
}
if err := c.Get(ctx, u, nil, resp); err != nil {
return nil, errors.Wrap(err, "ListSnapshots")
}

6
internal/serverapi/serverapi.go
View File

@@ -149,7 +149,6 @@ type CreateSnapshotSourceResponse struct {
// Snapshot describes single snapshot entry.
type Snapshot struct {
ID manifest.ID `json:"id"`
Source snapshot.SourceInfo `json:"source"`
Description string `json:"description"`
StartTime time.Time `json:"startTime"`
EndTime time.Time `json:"endTime"`
@@ -157,11 +156,14 @@ type Snapshot struct {
Summary *fs.DirectorySummary `json:"summary"`
RootEntry string `json:"rootID"`
RetentionReasons []string `json:"retention"`
Pins []string `json:"pins"`
}
// SnapshotsResponse contains a list of snapshots.
type SnapshotsResponse struct {
Snapshots []*Snapshot `json:"snapshots"`
Snapshots []*Snapshot `json:"snapshots"`
UnfilteredCount int `json:"unfilteredCount"`
UniqueCount int `json:"uniqueCount"`
}
// MountSnapshotRequest contains request to mount a snapshot.

120
snapshot/policy/retention_policy.go
View File

@@ -2,6 +2,9 @@
import (
"fmt"
"sort"
"strconv"
"strings"
"time"
"github.com/kopia/kopia/snapshot"
@@ -153,6 +156,8 @@ func (r *RetentionPolicy) getRetentionReasons(i int, s *snapshot.Manifest, cutof
}
}
SortRetentionTags(keepReasons)
return keepReasons
}
@@ -211,3 +216,118 @@ func (r *RetentionPolicy) Merge(src RetentionPolicy, def *RetentionPolicyDefinit
mergeOptionalInt(&r.KeepMonthly, src.KeepMonthly, &def.KeepMonthly, si)
mergeOptionalInt(&r.KeepAnnual, src.KeepAnnual, &def.KeepAnnual, si)
}
// CompactRetentionReasons returns compressed retention reasons given a list of retention reasons.
func CompactRetentionReasons(reasons []string) []string {
reasonsByPrefix := map[string][]int{}
result := []string{}
for _, r := range reasons {
prefix, suffix := prefixSuffix(r)
n, err := strconv.Atoi(suffix)
if err != nil {
result = append(result, r)
continue
}
reasonsByPrefix[prefix] = append(reasonsByPrefix[prefix], n)
}
for prefix, v := range reasonsByPrefix {
result = appendRLE(result, prefix, v)
}
SortRetentionTags(result)
return result
}
func prefixSuffix(s string) (prefix, suffix string) {
if p := strings.LastIndex(s, "-"); p < 0 {
prefix = s
suffix = ""
} else {
prefix = s[0:p]
suffix = s[p+1:]
}
return
}
func appendRLE(out []string, prefix string, numbers []int) []string {
sort.Ints(numbers)
runStart := numbers[0]
runEnd := numbers[0]
appendRun := func() {
if runStart == runEnd {
out = append(out, fmt.Sprintf("%v-%v", prefix, runStart))
} else {
out = append(out, fmt.Sprintf("%v-%v..%v", prefix, runStart, runEnd))
}
}
for _, num := range numbers[1:] {
if num == runEnd+1 {
runEnd = num
} else {
appendRun()
runStart = num
runEnd = runStart
}
}
appendRun()
return out
}
// CompactPins returns compressed pins reasons given a list of pins.
func CompactPins(pins []string) []string {
cnt := map[string]int{}
for _, p := range pins {
cnt[p]++
}
result := []string{}
for k := range cnt {
result = append(result, k)
}
sort.Strings(result)
return result
}
var retentionPrefixSortValue = map[string]int{
"latest": 1,
"hourly": 2, // nolint:gomnd
"daily": 3, // nolint:gomnd
"weekly": 4, // nolint:gomnd
"monthly": 5, // nolint:gomnd
"annual": 6, // nolint:gomnd
}
// SortRetentionTags sorts the provided retention tags in canonical order.
func SortRetentionTags(tags []string) {
sort.Slice(tags, func(i, j int) bool {
p1, s1 := prefixSuffix(tags[i])
p2, s2 := prefixSuffix(tags[j])
if l, r := retentionPrefixSortValue[p1], retentionPrefixSortValue[p2]; l != r {
return l < r
}
if l, r := p1, p2; l != r {
return p1 < p2
}
return s1 < s2
})
}

29
snapshot/policy/retention_policy_test.go
View File

@@ -7,6 +7,7 @@
"time"
"github.com/google/go-cmp/cmp"
"github.com/stretchr/testify/require"
"github.com/kopia/kopia/snapshot"
)
@@ -108,7 +109,7 @@ func TestRetentionPolicyTest(t *testing.T) {
"2020-04-01T15:00:00Z": {"daily-2"},
"2020-04-02T12:00:00Z": {"latest-3", "hourly-3"},
"2020-04-02T13:00:00Z": {"latest-2", "hourly-2"},
"2020-04-02T15:00:00Z": {"latest-1", "monthly-1", "daily-1", "hourly-1"},
"2020-04-02T15:00:00Z": {"latest-1", "hourly-1", "daily-1", "monthly-1"},
"incomplete-2020-04-02T15:01:00Z": {}, // incomplete, too old
"incomplete-2020-04-02T16:01:00Z": {}, // incomplete, too old
"incomplete-2020-04-02T17:01:00Z": {}, // incomplete, too old
@@ -206,3 +207,29 @@ func TestRetentionPolicyTest(t *testing.T) {
})
}
}
func TestCompactPins(t *testing.T) {
require.Equal(t,
[]string{"a", "b", "d", "x", "z"},
CompactPins([]string{
"z", "x", "a", "b", "d", "b", "z",
}))
}
func TestCompactRetentionrRasons(t *testing.T) {
cases := []struct {
input []string
want []string
}{
{input: nil, want: []string{}},
{[]string{"latest-1", "latest-2"}, []string{"latest-1..2"}},
{[]string{"latest-1", "daily-3", "latest-2", "daily-2"}, []string{"latest-1..2", "daily-2..3"}},
{[]string{"latest-1", "weekly-7", "latest-2"}, []string{"latest-1..2", "weekly-7"}},
{[]string{"latest-1", "latest-2", "latest-5", "latest-6", "latest-7"}, []string{"latest-1..2", "latest-5..7"}},
{[]string{"latest-1", "zrogue", "arogue", "latest-2"}, []string{"arogue", "zrogue", "latest-1..2"}},
}
for _, tc := range cases {
require.Equal(t, tc.want, CompactRetentionReasons(tc.input))
}
}

24
tests/end_to_end_test/server_start_test.go
View File

@@ -138,18 +138,18 @@ func TestServerStart(t *testing.T) {
verifySourceCount(t, cli, &snapshot.SourceInfo{Host: "no-such-host"}, 0)
verifySourceCount(t, cli, &snapshot.SourceInfo{Host: "fake-hostname", UserName: "fake-username", Path: sharedTestDataDir2}, 1)
verifySnapshotCount(t, cli, nil, 2)
verifySnapshotCount(t, cli, &snapshot.SourceInfo{Host: "fake-hostname", UserName: "fake-username", Path: sharedTestDataDir1}, 2)
verifySnapshotCount(t, cli, &snapshot.SourceInfo{Host: "fake-hostname", UserName: "fake-username", Path: sharedTestDataDir2}, 0)
verifySnapshotCount(t, cli, &snapshot.SourceInfo{Host: "no-such-host"}, 0)
verifySnapshotCount(t, cli, snapshot.SourceInfo{Host: "fake-hostname", UserName: "fake-username", Path: sharedTestDataDir1}, true, 2)
verifySnapshotCount(t, cli, snapshot.SourceInfo{Host: "fake-hostname", UserName: "fake-username", Path: sharedTestDataDir1}, false, 1)
verifySnapshotCount(t, cli, snapshot.SourceInfo{Host: "fake-hostname", UserName: "fake-username", Path: sharedTestDataDir2}, true, 0)
verifySnapshotCount(t, cli, snapshot.SourceInfo{Host: "no-such-host"}, true, 0)
uploadMatchingSnapshots(t, cli, &snapshot.SourceInfo{Host: "fake-hostname", UserName: "fake-username", Path: sharedTestDataDir2})
waitForSnapshotCount(ctx, t, cli, &snapshot.SourceInfo{Host: "fake-hostname", UserName: "fake-username", Path: sharedTestDataDir2}, 1)
waitForSnapshotCount(ctx, t, cli, snapshot.SourceInfo{Host: "fake-hostname", UserName: "fake-username", Path: sharedTestDataDir2}, 1)
_, err = serverapi.CancelUpload(ctx, cli, nil)
require.NoError(t, err)
snaps := verifySnapshotCount(t, cli, &snapshot.SourceInfo{Host: "fake-hostname", UserName: "fake-username", Path: sharedTestDataDir2}, 1)
snaps := verifySnapshotCount(t, cli, snapshot.SourceInfo{Host: "fake-hostname", UserName: "fake-username", Path: sharedTestDataDir2}, true, 1)
rootPayload, err := serverapi.GetObject(ctx, cli, snaps[0].RootEntry)
require.NoError(t, err)
@@ -180,7 +180,7 @@ func TestServerStart(t *testing.T) {
require.Len(t, policies.Policies, 1)
require.Equal(t, keepDaily, *policies.Policies[0].Policy.RetentionPolicy.KeepDaily)
waitForSnapshotCount(ctx, t, cli, &snapshot.SourceInfo{Host: "fake-hostname", UserName: "fake-username", Path: sharedTestDataDir3}, 1)
waitForSnapshotCount(ctx, t, cli, snapshot.SourceInfo{Host: "fake-hostname", UserName: "fake-username", Path: sharedTestDataDir3}, 1)
}
func TestServerCreateAndConnectViaAPI(t *testing.T) {
@@ -305,7 +305,7 @@ func TestConnectToExistingRepositoryViaAPI(t *testing.T) {
uploadMatchingSnapshots(t, cli, &si)
snaps := waitForSnapshotCount(ctx, t, cli, &si, 3)
snaps := waitForSnapshotCount(ctx, t, cli, si, 3)
// we're reproducing the bug described in, after connecting to repo via API, next snapshot size becomes zero.
// https://kopia.discourse.group/t/kopia-0-7-0-not-backing-up-any-files-repro-needed/136/6?u=jkowalski
@@ -383,13 +383,13 @@ func verifyServerConnected(t *testing.T, cli *apiclient.KopiaAPIClient, want boo
return st
}
func waitForSnapshotCount(ctx context.Context, t *testing.T, cli *apiclient.KopiaAPIClient, match *snapshot.SourceInfo, want int) []*serverapi.Snapshot {
func waitForSnapshotCount(ctx context.Context, t *testing.T, cli *apiclient.KopiaAPIClient, src snapshot.SourceInfo, want int) []*serverapi.Snapshot {
t.Helper()
var result []*serverapi.Snapshot
err := retry.PeriodicallyNoValue(ctx, 1*time.Second, 180, "wait for snapshots", func() error {
snapshots, err := serverapi.ListSnapshots(testlogging.Context(t), cli, match)
snapshots, err := serverapi.ListSnapshots(testlogging.Context(t), cli, src, true)
if err != nil {
return errors.Wrap(err, "error listing sources")
}
@@ -437,10 +437,10 @@ func uploadMatchingSnapshots(t *testing.T, cli *apiclient.KopiaAPIClient, match
}
}
func verifySnapshotCount(t *testing.T, cli *apiclient.KopiaAPIClient, match *snapshot.SourceInfo, want int) []*serverapi.Snapshot {
func verifySnapshotCount(t *testing.T, cli *apiclient.KopiaAPIClient, src snapshot.SourceInfo, all bool, want int) []*serverapi.Snapshot {
t.Helper()
snapshots, err := serverapi.ListSnapshots(testlogging.Context(t), cli, match)
snapshots, err := serverapi.ListSnapshots(testlogging.Context(t), cli, src, all)
require.NoError(t, err)
if got := len(snapshots.Snapshots); got != want {