From e7e8e6bbb04f5b2dc599669cbd773a7bd0b769aa Mon Sep 17 00:00:00 2001 From: ashmrtn Date: Thu, 20 Apr 2023 07:40:15 -0700 Subject: [PATCH] feat(repository): Reduce memory usage when parsing manifests (#2956) * Custom array deserializer functions Create custom functions to deserialize manifest JSON arrays. This can avoid the pathological case in the stdlib JSON decoder because it won't have to read the whole input to determine if the array is a valid JSON object. * Wire up custom decoder * Minor code cleanup * fix some lint errors * make json parsing of manifest a bit more robust * cleanup state in calling function * Add some tests * Add case insensitive test * Linter fixups --- repo/manifest/committed_manifest_manager.go | 10 +- repo/manifest/serialized.go | 123 +++ repo/manifest/serialized_test.go | 67 ++ repo/manifest/testdata/manifests.go | 879 ++++++++++++++++++++ 4 files changed, 1075 insertions(+), 4 deletions(-) create mode 100644 repo/manifest/serialized_test.go create mode 100644 repo/manifest/testdata/manifests.go diff --git a/repo/manifest/committed_manifest_manager.go b/repo/manifest/committed_manifest_manager.go index 048d2862f..32638a3a6 100644 --- a/repo/manifest/committed_manifest_manager.go +++ b/repo/manifest/committed_manifest_manager.go @@ -346,11 +346,13 @@ func loadManifestContent(ctx context.Context, b contentManager, contentID conten return man, errors.Wrapf(err, "unable to unpack manifest data %q", contentID) } - if err := json.NewDecoder(gz).Decode(&man); err != nil { - return man, errors.Wrapf(err, "unable to parse manifest %q", contentID) - } + // Will be GC-ed even if we don't close it? + //nolint:errcheck + defer gz.Close() - return man, nil + man, err = decodeManifestArray(gz) + + return man, errors.Wrapf(err, "unable to parse manifest %q", contentID) } func newCommittedManager(b contentManager) *committedManifestManager { diff --git a/repo/manifest/serialized.go b/repo/manifest/serialized.go index 05afb1772..a99f2ff9e 100644 --- a/repo/manifest/serialized.go +++ b/repo/manifest/serialized.go @@ -2,7 +2,11 @@ import ( "encoding/json" + "io" + "strings" "time" + + "github.com/pkg/errors" ) type manifest struct { @@ -16,3 +20,122 @@ type manifestEntry struct { Deleted bool `json:"deleted,omitempty"` Content json.RawMessage `json:"data"` } + +const ( + objectOpen = "{" + objectClose = "}" + arrayOpen = "[" + arrayClose = "]" +) + +var errEOF = errors.New("unexpected end of input") + +func expectDelimToken(dec *json.Decoder, expectedToken string) error { + t, err := dec.Token() + if errors.Is(err, io.EOF) { + return errors.WithStack(errEOF) + } else if err != nil { + return errors.Wrap(err, "reading JSON token") + } + + d, ok := t.(json.Delim) + if !ok { + return errors.Errorf("unexpected token: (%T) %v", t, t) + } else if d.String() != expectedToken { + return errors.Errorf("unexpected token; wanted %s, got %s", expectedToken, d) + } + + return nil +} + +func stringToken(dec *json.Decoder) (string, error) { + t, err := dec.Token() + if errors.Is(err, io.EOF) { + return "", errors.WithStack(errEOF) + } else if err != nil { + return "", errors.Wrap(err, "reading JSON token") + } + + l, ok := t.(string) + if !ok { + return "", errors.Errorf("unexpected token (%T) %v; wanted field name", t, t) + } + + return l, nil +} + +func decodeManifestArray(r io.Reader) (manifest, error) { + var ( + dec = json.NewDecoder(r) + res = manifest{} + ) + + if err := expectDelimToken(dec, objectOpen); err != nil { + return res, err + } + + // Need to manually decode fields here since we can't reuse the stdlib + // decoder due to memory issues. + if err := parseFields(dec, &res); err != nil { + return res, err + } + + // Consumes closing object curly brace after we're done. Don't need to check + // for EOF because json.Decode only guarantees decoding the next JSON item in + // the stream so this follows that. + return res, expectDelimToken(dec, objectClose) +} + +func parseFields(dec *json.Decoder, res *manifest) error { + var seen bool + + for dec.More() { + l, err := stringToken(dec) + if err != nil { + return err + } + + // Only have `entries` field right now. Skip other fields. + if !strings.EqualFold("entries", l) { + continue + } + + if seen { + return errors.New("repeated Entries field") + } + + seen = true + + if err := decodeArray(dec, &res.Entries); err != nil { + return err + } + } + + return nil +} + +// decodeArray decodes an array of *manifestEntry and returns them in output. If +// an error occurs output may contain intermediate state. +// +// This can be made into a generic function pretty easily if it's needed in +// other places. +func decodeArray(dec *json.Decoder, output *[]*manifestEntry) error { + // Consume starting bracket. + if err := expectDelimToken(dec, arrayOpen); err != nil { + return err + } + + // Read elements. + for dec.More() { + var tmp *manifestEntry + + if err := dec.Decode(&tmp); err != nil { + return errors.Wrap(err, "decoding array element") + } + + *output = append(*output, tmp) + } + + // Consume ending bracket. + return expectDelimToken(dec, arrayClose) +} diff --git a/repo/manifest/serialized_test.go b/repo/manifest/serialized_test.go new file mode 100644 index 000000000..1529adcc0 --- /dev/null +++ b/repo/manifest/serialized_test.go @@ -0,0 +1,67 @@ +package manifest + +import ( + "bytes" + "encoding/json" + "reflect" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/kopia/kopia/repo/manifest/testdata" +) + +func TestManifestDecode_GoodInput(t *testing.T) { + table := []struct { + name string + input []byte + }{ + { + name: "MultipleManifests", + input: []byte(testdata.GoodManifests), + }, + { + name: "IgnoredField", + input: []byte(testdata.IgnoredField), + }, + { + name: "StopsAtStructEnd", + input: []byte(testdata.ExtraInputAtEnd), + }, + { + name: "CaseInsensitive", + input: []byte(testdata.CaseInsensitive), + }, + } + + for _, test := range table { + t.Run(test.name, func(t *testing.T) { + stdlibDec := manifest{} + + stdReader := bytes.NewReader(test.input) + require.NoError(t, json.NewDecoder(stdReader).Decode(&stdlibDec)) + + arrReader := bytes.NewReader(test.input) + arrDec, err := decodeManifestArray(arrReader) + require.NoError(t, err) + + assert.Equal(t, stdlibDec, arrDec) + + assert.True(t, reflect.DeepEqual(stdlibDec, arrDec)) + }) + } +} + +func TestManifestDecode_BadInput(t *testing.T) { + for _, test := range testdata.BadInputs { + t.Run(test.Name, func(t *testing.T) { + r := bytes.NewReader([]byte(test.Input)) + _, err := decodeManifestArray(r) + + t.Logf("%v", err) + + assert.Error(t, err) + }) + } +} diff --git a/repo/manifest/testdata/manifests.go b/repo/manifest/testdata/manifests.go new file mode 100644 index 000000000..07274c24f --- /dev/null +++ b/repo/manifest/testdata/manifests.go @@ -0,0 +1,879 @@ +package testdata + +type testInput struct { + Name string + Input string +} + +var ( + BadInputs = []testInput{ + { + Name: "RepeatedEntriesField", + Input: ` +{ + "entries": [ + { + "id": "25905b6f222a153561543baea0a67043", + "labels": { + "hostname": "host-name", + "path": "/root/tmp/test", + "type": "snapshot", + "username": "user-name" + }, + "modified": "2023-03-16T20:46:59.70714Z", + "data": { + "id": "", + "source": { + "host": "host-name", + "userName": "user-name", + "path": "/root/tmp/test" + }, + "description": "", + "startTime": "2023-03-16T20:46:55.76843Z", + "endTime": "2023-03-16T20:46:59.707064Z", + "stats": { + "totalSize": 536927459, + "excludedTotalSize": 0, + "fileCount": 18, + "cachedFiles": 0, + "nonCachedFiles": 18, + "dirCount": 14, + "excludedFileCount": 0, + "excludedDirCount": 0, + "ignoredErrorCount": 0, + "errorCount": 0 + }, + "rootEntry": { + "name": "test", + "type": "d", + "mode": "0777", + "mtime": "1754-08-30T22:43:41.128654848Z", + "obj": "k74647859396c88127696f426b4c79088", + "summ": { + "size": 536927459, + "files": 18, + "symlinks": 0, + "dirs": 14, + "maxTime": "2023-03-16T20:46:56.187394Z", + "numFailed": 0 + } + }, + } + }, + ], + entries: [] +}`, + }, + { + Name: "MissingObjectStart", + Input: ` + "entries": [ + { + "id": "25905b6f222a153561543baea0a67043", + "labels": { + "hostname": "host-name", + "path": "/root/tmp/test", + "type": "snapshot", + "username": "user-name" + }, + "modified": "2023-03-16T20:46:59.70714Z", + "data": { + "id": "", + "source": { + "host": "host-name", + "userName": "user-name", + "path": "/root/tmp/test" + }, + "description": "", + "startTime": "2023-03-16T20:46:55.76843Z", + "endTime": "2023-03-16T20:46:59.707064Z", + "stats": { + "totalSize": 536927459, + "excludedTotalSize": 0, + "fileCount": 18, + "cachedFiles": 0, + "nonCachedFiles": 18, + "dirCount": 14, + "excludedFileCount": 0, + "excludedDirCount": 0, + "ignoredErrorCount": 0, + "errorCount": 0 + }, + "rootEntry": { + "name": "test", + "type": "d", + "mode": "0777", + "mtime": "1754-08-30T22:43:41.128654848Z", + "obj": "k74647859396c88127696f426b4c79088", + "summ": { + "size": 536927459, + "files": 18, + "symlinks": 0, + "dirs": 14, + "maxTime": "2023-03-16T20:46:56.187394Z", + "numFailed": 0 + } + } + } + } + ] +}`, + }, + { + Name: "MissingObjectEnd", + Input: ` +{ + "entries": [ + { + "id": "25905b6f222a153561543baea0a67043", + "labels": { + "hostname": "host-name", + "path": "/root/tmp/test", + "type": "snapshot", + "username": "user-name" + }, + "modified": "2023-03-16T20:46:59.70714Z", + "data": { + "id": "", + "source": { + "host": "host-name", + "userName": "user-name", + "path": "/root/tmp/test" + }, + "description": "", + "startTime": "2023-03-16T20:46:55.76843Z", + "endTime": "2023-03-16T20:46:59.707064Z", + "stats": { + "totalSize": 536927459, + "excludedTotalSize": 0, + "fileCount": 18, + "cachedFiles": 0, + "nonCachedFiles": 18, + "dirCount": 14, + "excludedFileCount": 0, + "excludedDirCount": 0, + "ignoredErrorCount": 0, + "errorCount": 0 + }, + "rootEntry": { + "name": "test", + "type": "d", + "mode": "0777", + "mtime": "1754-08-30T22:43:41.128654848Z", + "obj": "k74647859396c88127696f426b4c79088", + "summ": { + "size": 536927459, + "files": 18, + "symlinks": 0, + "dirs": 14, + "maxTime": "2023-03-16T20:46:56.187394Z", + "numFailed": 0 + } + } + } + } + ]`, + }, + { + Name: "MissingArrayStart", + Input: ` +{ + "entries": + { + "id": "25905b6f222a153561543baea0a67043", + "labels": { + "hostname": "host-name", + "path": "/root/tmp/test", + "type": "snapshot", + "username": "user-name" + }, + "modified": "2023-03-16T20:46:59.70714Z", + "data": { + "id": "", + "source": { + "host": "host-name", + "userName": "user-name", + "path": "/root/tmp/test" + }, + "description": "", + "startTime": "2023-03-16T20:46:55.76843Z", + "endTime": "2023-03-16T20:46:59.707064Z", + "stats": { + "totalSize": 536927459, + "excludedTotalSize": 0, + "fileCount": 18, + "cachedFiles": 0, + "nonCachedFiles": 18, + "dirCount": 14, + "excludedFileCount": 0, + "excludedDirCount": 0, + "ignoredErrorCount": 0, + "errorCount": 0 + }, + "rootEntry": { + "name": "test", + "type": "d", + "mode": "0777", + "mtime": "1754-08-30T22:43:41.128654848Z", + "obj": "k74647859396c88127696f426b4c79088", + "summ": { + "size": 536927459, + "files": 18, + "symlinks": 0, + "dirs": 14, + "maxTime": "2023-03-16T20:46:56.187394Z", + "numFailed": 0 + } + } + } + } + ] +}`, + }, + { + Name: "MissingArrayEnd", + Input: ` +{ + "entries": [ + { + "id": "25905b6f222a153561543baea0a67043", + "labels": { + "hostname": "host-name", + "path": "/root/tmp/test", + "type": "snapshot", + "username": "user-name" + }, + "modified": "2023-03-16T20:46:59.70714Z", + "data": { + "id": "", + "source": { + "host": "host-name", + "userName": "user-name", + "path": "/root/tmp/test" + }, + "description": "", + "startTime": "2023-03-16T20:46:55.76843Z", + "endTime": "2023-03-16T20:46:59.707064Z", + "stats": { + "totalSize": 536927459, + "excludedTotalSize": 0, + "fileCount": 18, + "cachedFiles": 0, + "nonCachedFiles": 18, + "dirCount": 14, + "excludedFileCount": 0, + "excludedDirCount": 0, + "ignoredErrorCount": 0, + "errorCount": 0 + }, + "rootEntry": { + "name": "test", + "type": "d", + "mode": "0777", + "mtime": "1754-08-30T22:43:41.128654848Z", + "obj": "k74647859396c88127696f426b4c79088", + "summ": { + "size": 536927459, + "files": 18, + "symlinks": 0, + "dirs": 14, + "maxTime": "2023-03-16T20:46:56.187394Z", + "numFailed": 0 + } + } + } + } +}`, + }, + { + Name: "MissingInnerObjectStart", + Input: ` +{ + "entries": [ + "id": "25905b6f222a153561543baea0a67043", + "labels": { + "hostname": "host-name", + "path": "/root/tmp/test", + "type": "snapshot", + "username": "user-name" + }, + "modified": "2023-03-16T20:46:59.70714Z", + "data": { + "id": "", + "source": { + "host": "host-name", + "userName": "user-name", + "path": "/root/tmp/test" + }, + "description": "", + "startTime": "2023-03-16T20:46:55.76843Z", + "endTime": "2023-03-16T20:46:59.707064Z", + "stats": { + "totalSize": 536927459, + "excludedTotalSize": 0, + "fileCount": 18, + "cachedFiles": 0, + "nonCachedFiles": 18, + "dirCount": 14, + "excludedFileCount": 0, + "excludedDirCount": 0, + "ignoredErrorCount": 0, + "errorCount": 0 + }, + "rootEntry": { + "name": "test", + "type": "d", + "mode": "0777", + "mtime": "1754-08-30T22:43:41.128654848Z", + "obj": "k74647859396c88127696f426b4c79088", + "summ": { + "size": 536927459, + "files": 18, + "symlinks": 0, + "dirs": 14, + "maxTime": "2023-03-16T20:46:56.187394Z", + "numFailed": 0 + } + } + } + }, + ] +}`, + }, + { + Name: "BadInnerObject", + Input: ` +{ + "entries": [ + { + "id": "25905b6f222a153561543baea0a67043", + "labels": { + "hostname": "host-name", + "path": "/root/tmp/test", + "type": "snapshot", + "username": "user-name" + }, + "modified": "2023-03-16T20:46:59.70714Z", + "data": { + "id": "", + "source": { + "host": "host-name", + "userName": "user-name", + "path": "/root/tmp/test" + }, + "description": "", + "startTime": "2023-03-16T20:46:55.76843Z", + "endTime": "2023-03-16T20:46:59.707064Z", + "stats": { + "totalSize": 536927459, + "excludedTotalSize": 0, + "fileCount": 18, + "cachedFiles": 0, + "nonCachedFiles": 18, + "dirCount": 14, + "excludedFileCount": 0, + "excludedDirCount": 0, + "ignoredErrorCount": 0, + "errorCount": 0 + }, + "rootEntry": { + "name": "test", + "type": "d", + "mode": "0777", + "mtime": "1754-08-30T22:43:41.128654848Z", + "obj": "k74647859396c88127696f426b4c79088", + "summ": { + "size": 536927459, + "files": 18, + "symlinks": 0, + "dirs": 14, + "maxTime": "2023-03-16T20:46:56.187394Z", + "numFailed": 0 + } + } + } + ] +}`, + }, + { + Name: "MissingInnerObjectEnd", + Input: ` +{ + "entries": [ + { + "id": "25905b6f222a153561543baea0a67043", + "labels": { + "hostname": "host-name", + "path": "/root/tmp/test", + "type": "snapshot", + "username": "user-name" + }, + "modified": "2023-03-16T20:46:59.70714Z", + "data": { + "id": "", + "source": { + "host": "host-name", + "userName": "user-name", + "path": "/root/tmp/test" + }, + "description": "", + "startTime": "2023-03-16T20:46:55.76843Z", + "endTime": "2023-03-16T20:46:59.707064Z", + "stats": { + "totalSize": 536927459, + "excludedTotalSize": 0, + "fileCount": 18, + "cachedFiles": 0, + "nonCachedFiles": 18, + "dirCount": 14, + "excludedFileCount": 0, + "excludedDirCount": 0, + "ignoredErrorCount": 0, + "errorCount": 0 + }, + "rootEntry": { + "name": "test", + "type": "d", + "mode": "0777", + "mtime": "1754-08-30T22:43:41.128654848Z", + "obj": "k74647859396c88127696f426b4c79088", + "summ": { + "size": 536927459, + "files": 18, + "symlinks": 0, + "dirs": 14, + "maxTime": "2023-03-16T20:46:56.187394Z", + "numFailed": 0 + } + } + } + ] +}`, + }, + } +) + +const ( + GoodManifests = ` +{ + "entries": [ + { + "id": "2e14cba9427c57223dd768bd1ddf694c", + "labels": { + "hostname": "host-name", + "path": "/root/tmp/test", + "tag": "value", + "type": "snapshot", + "username": "user-name" + }, + "modified": "2023-03-17T01:08:32.962808Z", + "data": { + "id": "", + "source": { + "host": "host-name", + "userName": "user-name", + "path": "/root/tmp/test" + }, + "description": "", + "startTime": "2023-03-17T01:08:29.674573Z", + "endTime": "2023-03-17T01:08:32.962614Z", + "stats": { + "totalSize": 427221, + "excludedTotalSize": 0, + "fileCount": 143, + "cachedFiles": 0, + "nonCachedFiles": 143, + "dirCount": 10, + "excludedFileCount": 0, + "excludedDirCount": 0, + "ignoredErrorCount": 0, + "errorCount": 0 + }, + "rootEntry": { + "name": "test", + "type": "d", + "mode": "0777", + "mtime": "1754-08-30T22:43:41.128654848Z", + "obj": "kfe00a91781912fc352edca26571a5f83", + "summ": { + "size": 427221, + "files": 143, + "symlinks": 0, + "dirs": 10, + "maxTime": "2023-03-17T01:08:29.677079Z", + "numFailed": 0 + } + }, + "tags": { + "tag": "value" + } + } + }, + { + "id": "2c54893efd80bcda7102f622da5c63ee", + "labels": { + "hostname": "host-name", + "path": "/root/tmp/test", + "type": "snapshot", + "username": "user-name" + }, + "modified": "2023-03-17T01:11:34.506121Z", + "data": { + "id": "", + "source": { + "host": "host-name", + "userName": "user-name", + "path": "/root/tmp/test" + }, + "description": "", + "startTime": "2023-03-17T01:11:22.34148Z", + "endTime": "2023-03-17T01:11:34.505952Z", + "stats": { + "totalSize": 427221, + "excludedTotalSize": 0, + "fileCount": 2, + "cachedFiles": 141, + "nonCachedFiles": 2, + "dirCount": 10, + "excludedFileCount": 0, + "excludedDirCount": 0, + "ignoredErrorCount": 0, + "errorCount": 0 + }, + "rootEntry": { + "name": "test", + "type": "d", + "mode": "0777", + "mtime": "1754-08-30T22:43:41.128654848Z", + "obj": "k4f1a9e8049091615cbe4ad93507680f3", + "summ": { + "size": 427221, + "files": 143, + "symlinks": 0, + "dirs": 10, + "maxTime": "2023-03-17T01:11:22.725375Z", + "numFailed": 0 + } + } + } + } + ] +} +` + IgnoredField = ` +{ + "entries": [ + { + "id": "2e14cba9427c57223dd768bd1ddf694c", + "labels": { + "hostname": "host-name", + "path": "/root/tmp/test", + "tag": "value", + "type": "snapshot", + "username": "user-name" + }, + "modified": "2023-03-17T01:08:32.962808Z", + "data": { + "id": "", + "source": { + "host": "host-name", + "userName": "user-name", + "path": "/root/tmp/test" + }, + "description": "", + "startTime": "2023-03-17T01:08:29.674573Z", + "endTime": "2023-03-17T01:08:32.962614Z", + "stats": { + "totalSize": 427221, + "excludedTotalSize": 0, + "fileCount": 143, + "cachedFiles": 0, + "nonCachedFiles": 143, + "dirCount": 10, + "excludedFileCount": 0, + "excludedDirCount": 0, + "ignoredErrorCount": 0, + "errorCount": 0 + }, + "rootEntry": { + "name": "test", + "type": "d", + "mode": "0777", + "mtime": "1754-08-30T22:43:41.128654848Z", + "obj": "kfe00a91781912fc352edca26571a5f83", + "summ": { + "size": 427221, + "files": 143, + "symlinks": 0, + "dirs": 10, + "maxTime": "2023-03-17T01:08:29.677079Z", + "numFailed": 0 + } + }, + "tags": { + "tag": "value" + } + } + }, + { + "id": "2c54893efd80bcda7102f622da5c63ee", + "labels": { + "hostname": "host-name", + "path": "/root/tmp/test", + "type": "snapshot", + "username": "user-name" + }, + "modified": "2023-03-17T01:11:34.506121Z", + "data": { + "id": "", + "source": { + "host": "host-name", + "userName": "user-name", + "path": "/root/tmp/test" + }, + "description": "", + "startTime": "2023-03-17T01:11:22.34148Z", + "endTime": "2023-03-17T01:11:34.505952Z", + "stats": { + "totalSize": 427221, + "excludedTotalSize": 0, + "fileCount": 2, + "cachedFiles": 141, + "nonCachedFiles": 2, + "dirCount": 10, + "excludedFileCount": 0, + "excludedDirCount": 0, + "ignoredErrorCount": 0, + "errorCount": 0 + }, + "rootEntry": { + "name": "test", + "type": "d", + "mode": "0777", + "mtime": "1754-08-30T22:43:41.128654848Z", + "obj": "k4f1a9e8049091615cbe4ad93507680f3", + "summ": { + "size": 427221, + "files": 143, + "symlinks": 0, + "dirs": 10, + "maxTime": "2023-03-17T01:11:22.725375Z", + "numFailed": 0 + } + } + } + } + ], + "ignored": "hello world" +}` + ExtraInputAtEnd = ` +{ + "entries": [ + { + "id": "2e14cba9427c57223dd768bd1ddf694c", + "labels": { + "hostname": "host-name", + "path": "/root/tmp/test", + "tag": "value", + "type": "snapshot", + "username": "user-name" + }, + "modified": "2023-03-17T01:08:32.962808Z", + "data": { + "id": "", + "source": { + "host": "host-name", + "userName": "user-name", + "path": "/root/tmp/test" + }, + "description": "", + "startTime": "2023-03-17T01:08:29.674573Z", + "endTime": "2023-03-17T01:08:32.962614Z", + "stats": { + "totalSize": 427221, + "excludedTotalSize": 0, + "fileCount": 143, + "cachedFiles": 0, + "nonCachedFiles": 143, + "dirCount": 10, + "excludedFileCount": 0, + "excludedDirCount": 0, + "ignoredErrorCount": 0, + "errorCount": 0 + }, + "rootEntry": { + "name": "test", + "type": "d", + "mode": "0777", + "mtime": "1754-08-30T22:43:41.128654848Z", + "obj": "kfe00a91781912fc352edca26571a5f83", + "summ": { + "size": 427221, + "files": 143, + "symlinks": 0, + "dirs": 10, + "maxTime": "2023-03-17T01:08:29.677079Z", + "numFailed": 0 + } + }, + "tags": { + "tag": "value" + } + } + }, + { + "id": "2c54893efd80bcda7102f622da5c63ee", + "labels": { + "hostname": "host-name", + "path": "/root/tmp/test", + "type": "snapshot", + "username": "user-name" + }, + "modified": "2023-03-17T01:11:34.506121Z", + "data": { + "id": "", + "source": { + "host": "host-name", + "userName": "user-name", + "path": "/root/tmp/test" + }, + "description": "", + "startTime": "2023-03-17T01:11:22.34148Z", + "endTime": "2023-03-17T01:11:34.505952Z", + "stats": { + "totalSize": 427221, + "excludedTotalSize": 0, + "fileCount": 2, + "cachedFiles": 141, + "nonCachedFiles": 2, + "dirCount": 10, + "excludedFileCount": 0, + "excludedDirCount": 0, + "ignoredErrorCount": 0, + "errorCount": 0 + }, + "rootEntry": { + "name": "test", + "type": "d", + "mode": "0777", + "mtime": "1754-08-30T22:43:41.128654848Z", + "obj": "k4f1a9e8049091615cbe4ad93507680f3", + "summ": { + "size": 427221, + "files": 143, + "symlinks": 0, + "dirs": 10, + "maxTime": "2023-03-17T01:11:22.725375Z", + "numFailed": 0 + } + } + } + } + ] +}abcdefg` + CaseInsensitive = ` +{ + "Entries": [ + { + "id": "2e14cba9427c57223dd768bd1ddf694c", + "labels": { + "hostname": "host-name", + "path": "/root/tmp/test", + "tag": "value", + "type": "snapshot", + "username": "user-name" + }, + "modified": "2023-03-17T01:08:32.962808Z", + "data": { + "id": "", + "source": { + "host": "host-name", + "userName": "user-name", + "path": "/root/tmp/test" + }, + "description": "", + "startTime": "2023-03-17T01:08:29.674573Z", + "endTime": "2023-03-17T01:08:32.962614Z", + "stats": { + "totalSize": 427221, + "excludedTotalSize": 0, + "fileCount": 143, + "cachedFiles": 0, + "nonCachedFiles": 143, + "dirCount": 10, + "excludedFileCount": 0, + "excludedDirCount": 0, + "ignoredErrorCount": 0, + "errorCount": 0 + }, + "rootEntry": { + "name": "test", + "type": "d", + "mode": "0777", + "mtime": "1754-08-30T22:43:41.128654848Z", + "obj": "kfe00a91781912fc352edca26571a5f83", + "summ": { + "size": 427221, + "files": 143, + "symlinks": 0, + "dirs": 10, + "maxTime": "2023-03-17T01:08:29.677079Z", + "numFailed": 0 + } + }, + "tags": { + "tag": "value" + } + } + }, + { + "id": "2c54893efd80bcda7102f622da5c63ee", + "labels": { + "hostname": "host-name", + "path": "/root/tmp/test", + "type": "snapshot", + "username": "user-name" + }, + "modified": "2023-03-17T01:11:34.506121Z", + "data": { + "id": "", + "source": { + "host": "host-name", + "userName": "user-name", + "path": "/root/tmp/test" + }, + "description": "", + "startTime": "2023-03-17T01:11:22.34148Z", + "endTime": "2023-03-17T01:11:34.505952Z", + "stats": { + "totalSize": 427221, + "excludedTotalSize": 0, + "fileCount": 2, + "cachedFiles": 141, + "nonCachedFiles": 2, + "dirCount": 10, + "excludedFileCount": 0, + "excludedDirCount": 0, + "ignoredErrorCount": 0, + "errorCount": 0 + }, + "rootEntry": { + "name": "test", + "type": "d", + "mode": "0777", + "mtime": "1754-08-30T22:43:41.128654848Z", + "obj": "k4f1a9e8049091615cbe4ad93507680f3", + "summ": { + "size": 427221, + "files": 143, + "symlinks": 0, + "dirs": 10, + "maxTime": "2023-03-17T01:11:22.725375Z", + "numFailed": 0 + } + } + } + } + ] +}` +)