From 6217df1a87d0a740cd70ffe392988ab25d23ebb5 Mon Sep 17 00:00:00 2001 From: Jarek Kowalski Date: Mon, 25 Nov 2019 23:24:59 -0800 Subject: [PATCH] lint: switched to 1.21 and fixed a ton of whitespace issues discovered by new wsl linter --- .golangci.yml | 9 +- cli/app.go | 2 + cli/cli_progress.go | 3 + cli/command_benchmark_crypto.go | 8 +- cli/command_benchmark_splitters.go | 11 ++- cli/command_blob_gc.go | 3 + cli/command_blob_show.go | 1 + cli/command_cache_clear.go | 2 + cli/command_cache_info.go | 3 + cli/command_content_list.go | 2 + cli/command_content_rewrite.go | 3 + cli/command_content_stats.go | 12 ++- cli/command_content_verify.go | 5 + cli/command_diff.go | 2 + cli/command_ls.go | 6 +- cli/command_manifest_ls.go | 2 + cli/command_manifest_show.go | 3 + cli/command_mount_browse.go | 4 + cli/command_mount_fuse.go | 1 + cli/command_mount_webdav.go | 4 + cli/command_policy.go | 2 + cli/command_policy_edit.go | 3 + cli/command_policy_remove.go | 1 + cli/command_policy_set.go | 21 +++- cli/command_policy_show.go | 8 +- cli/command_repository_connect.go | 2 + cli/command_repository_connect_from_config.go | 1 + cli/command_repository_create.go | 5 + cli/command_repository_repair.go | 7 +- cli/command_repository_status.go | 6 ++ cli/command_server_start.go | 4 + cli/command_show.go | 4 + cli/command_snapshot_create.go | 13 +++ cli/command_snapshot_estimate.go | 16 ++- cli/command_snapshot_expire.go | 4 + cli/command_snapshot_list.go | 19 +++- cli/command_snapshot_migrate.go | 12 ++- cli/command_snapshot_verify.go | 17 +++- cli/config.go | 3 + cli/memory_tracking.go | 5 + cli/objref.go | 5 + cli/password.go | 6 ++ cli/show_utils.go | 1 + cli/storage_filesystem.go | 3 + cli/storage_gcs.go | 1 - cli/storage_providers.go | 4 +- fs/cachefs/cache.go | 7 ++ fs/cachefs/cache_test.go | 17 +++- fs/cachefs/cachefs.go | 1 + fs/ignorefs/ignorefs.go | 9 ++ fs/ignorefs/ignorefs_test.go | 10 ++ fs/localfs/local_fs.go | 10 +- fs/localfs/local_fs_nonwindows.go | 1 + fs/localfs/local_fs_test.go | 10 +- fs/loggingfs/loggingfs.go | 4 + internal/blobtesting/asserts.go | 2 + internal/blobtesting/faulty.go | 13 +++ internal/blobtesting/map.go | 12 +++ internal/blobtesting/map_test.go | 2 + internal/blobtesting/verify.go | 3 + internal/diff/diff.go | 13 ++- internal/editor/editor.go | 7 ++ internal/fusemount/fusefs.go | 1 + internal/ignore/ignore.go | 5 +- internal/logfile/logfile.go | 10 +- internal/mockfs/mockfs.go | 5 + internal/ospath/ospath_xdg.go | 1 + internal/parallelwork/parallel_work_queue.go | 4 + internal/repotesting/repotesting.go | 4 + internal/retry/retry.go | 4 + internal/scrubber/scrub_sensitive.go | 2 + internal/server/api_policy_list.go | 1 + internal/server/api_snapshot_list.go | 3 + internal/server/api_sources_list.go | 1 + internal/server/server.go | 4 + internal/server/source_manager.go | 17 +++- internal/serverapi/client.go | 3 + internal/throttle/round_tripper.go | 3 + internal/throttle/round_tripper_test.go | 16 ++- internal/units/units.go | 1 + internal/webdavmount/webdavmount.go | 9 ++ repo/blob/config.go | 2 + repo/blob/filesystem/filesystem_storage.go | 12 +++ .../filesystem/filesystem_storage_test.go | 4 + repo/blob/gcs/gcs_storage.go | 6 ++ repo/blob/gcs/gcs_storage_test.go | 2 + repo/blob/logging/logging_storage.go | 7 ++ repo/blob/logging/logging_storage_test.go | 5 + repo/blob/registry.go | 5 +- repo/blob/s3/s3_storage.go | 7 ++ repo/blob/s3/s3_storage_test.go | 3 + repo/blob/sftp/sftp_storage.go | 9 ++ repo/blob/sftp/sftp_storage_test.go | 2 + repo/blob/sharded/sharded.go | 5 +- repo/blob/storage.go | 10 ++ repo/blob/webdav/webdav_storage.go | 3 + repo/blob/webdav/webdav_storage_test.go | 2 + repo/connect.go | 3 + repo/content/block_manager_compaction.go | 14 ++- repo/content/builder.go | 12 +++ repo/content/cache_hmac.go | 3 + repo/content/committed_content_index.go | 9 ++ .../committed_content_index_disk_cache.go | 5 + .../committed_content_index_mem_cache.go | 1 + repo/content/content_cache.go | 7 ++ repo/content/content_cache_test.go | 10 ++ repo/content/content_formatter.go | 8 ++ repo/content/content_formatter_test.go | 4 + repo/content/content_formatting_options.go | 1 + repo/content/content_id_to_bytes.go | 5 + repo/content/content_index_recovery.go | 12 +++ repo/content/content_index_recovery_test.go | 2 + repo/content/content_manager.go | 25 +++++ repo/content/content_manager_iterate.go | 11 +++ repo/content/content_manager_lock_free.go | 25 +++++ repo/content/content_manager_test.go | 97 ++++++++++++++++++- repo/content/format.go | 1 + repo/content/index.go | 12 +++ repo/content/list_cache.go | 5 + repo/content/merged.go | 9 ++ repo/content/merged_test.go | 8 ++ repo/content/packindex_internal_test.go | 4 +- repo/content/packindex_test.go | 21 +++- repo/crypto_key_derivation.go | 1 + repo/format_block.go | 10 ++ repo/format_block_test.go | 3 + repo/initialize.go | 3 + repo/local_config.go | 2 + repo/manifest/manifest_manager.go | 13 +++ repo/manifest/manifest_manager_test.go | 16 +++ repo/object/object_manager.go | 7 +- repo/object/object_manager_test.go | 15 +++ repo/object/object_reader.go | 11 ++- repo/object/object_splitter.go | 1 + repo/object/object_splitter_test.go | 8 ++ repo/object/object_writer.go | 7 ++ repo/object/objectid.go | 2 + repo/object/splitter_buzhash32.go | 1 + repo/object/splitter_fixed.go | 1 + repo/object/splitter_rabinkarp64.go | 2 + repo/open.go | 1 + repo/repository.go | 3 + repo/repository_test.go | 22 ++++- repo/upgrade.go | 5 +- site/cli2md/cli2md.go | 17 ++++ snapshot/gc/gc.go | 9 ++ snapshot/manager.go | 10 ++ snapshot/manifest.go | 2 + snapshot/policy/expire.go | 6 ++ snapshot/policy/policy.go | 2 + snapshot/policy/policy_manager.go | 15 ++- snapshot/policy/retention_policy.go | 8 ++ snapshot/policy/scheduling_policy.go | 3 + snapshot/snapshot_test.go | 23 ++++- snapshot/snapshotfs/all_sources.go | 1 + snapshot/snapshotfs/repofs.go | 1 + snapshot/snapshotfs/snapshot_tree_walker.go | 3 +- snapshot/snapshotfs/source_directories.go | 1 + snapshot/snapshotfs/source_snapshots.go | 1 + snapshot/snapshotfs/upload.go | 39 ++++++++ snapshot/snapshotfs/upload_test.go | 13 ++- snapshot/stats_test.go | 1 + tests/end_to_end_test/end_to_end_test.go | 29 +++++- .../repository_stress_test.go | 46 ++++++--- tests/stress_test/stress_test.go | 7 ++ tools/tools.mk | 2 +- 166 files changed, 1166 insertions(+), 79 deletions(-) diff --git a/.golangci.yml b/.golangci.yml index 17b71b20b..8d6c453e0 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -35,9 +35,6 @@ linters-settings: - experimental disabled-checks: - wrapperFunc - whitespace: - multi-func: true - linters: enable-all: true disable: @@ -45,6 +42,7 @@ linters: - prealloc - gochecknoglobals - gochecknoinits + - whitespace run: skip-dirs: @@ -55,9 +53,12 @@ issues: - text: "weak cryptographic primitive" linters: - gosec + - text: "Line contains TODO" + linters: + - godox # golangci.com configuration # https://github.com/golangci/golangci/wiki/Configuration service: - golangci-lint-version: 1.18.x # use the fixed version to not introduce new linters unexpectedly + golangci-lint-version: 1.21.x # use the fixed version to not introduce new linters unexpectedly diff --git a/cli/app.go b/cli/app.go index 443d46411..b50ad5bac 100644 --- a/cli/app.go +++ b/cli/app.go @@ -38,7 +38,9 @@ func helpFullAction(ctx *kingpin.ParseContext) error { _ = app.UsageForContextWithTemplate(ctx, 0, kingpin.DefaultUsageTemplate) + os.Exit(0) + return nil } diff --git a/cli/cli_progress.go b/cli/cli_progress.go index 64d383b0e..8df63aab6 100644 --- a/cli/cli_progress.go +++ b/cli/cli_progress.go @@ -29,6 +29,7 @@ func (p *singleProgress) toString(details bool) string { dur := time.Since(p.startTime) extraInfo := "" + if dur > 1*time.Second && details { extraInfo = " " + units.BitsPerSecondsString(8*float64(p.progress)/time.Since(p.startTime).Seconds()) } @@ -90,8 +91,10 @@ func (mp *multiProgress) Report(desc string, progress, total int64) { for i, p := range mp.items { segments = append(segments, p.toString(i > 0)) } + if found.progress >= found.total && foundPos == len(segments)-1 { mp.items = append(mp.items[0:foundPos], mp.items[foundPos+1:]...) + if len(segments) > 0 { log.Notice(segments[len(segments)-1]) } diff --git a/cli/command_benchmark_crypto.go b/cli/command_benchmark_crypto.go index 549daefc1..9033588aa 100644 --- a/cli/command_benchmark_crypto.go +++ b/cli/command_benchmark_crypto.go @@ -23,9 +23,11 @@ type benchResult struct { encryption string throughput float64 } + var results []benchResult data := make([]byte, *benchmarkCryptoBlockSize) + for _, ha := range content.SupportedHashAlgorithms() { for _, ea := range content.SupportedEncryptionAlgorithms() { isEncrypted := ea != "NONE" @@ -44,7 +46,9 @@ type benchResult struct { } log.Infof("Benchmarking hash '%v' and encryption '%v'... (%v x %v bytes)", ha, ea, *benchmarkCryptoRepeat, len(data)) + t0 := time.Now() + hashCount := *benchmarkCryptoRepeat for i := 0; i < hashCount; i++ { contentID := h(data) @@ -53,6 +57,7 @@ type benchResult struct { break } } + hashTime := time.Since(t0) bytesPerSecond := float64(len(data)) * float64(hashCount) / hashTime.Seconds() @@ -65,10 +70,11 @@ type benchResult struct { }) printStdout(" %-20v %-20v %v\n", "Hash", "Encryption", "Throughput") printStdout("-----------------------------------------------------------------\n") + for ndx, r := range results { printStdout("%3d. %-20v %-20v %v / second\n", ndx, r.hash, r.encryption, units.BytesStringBase2(int64(r.throughput))) - } + return nil } diff --git a/cli/command_benchmark_splitters.go b/cli/command_benchmark_splitters.go index 5fe66407a..5290af191 100644 --- a/cli/command_benchmark_splitters.go +++ b/cli/command_benchmark_splitters.go @@ -35,6 +35,7 @@ type benchResult struct { // generate data blocks var dataBlocks [][]byte + rnd := rand.New(rand.NewSource(*benchmarkSplitterRandSeed)) for i := 0; i < *benchmarkSplitterBlockCount; i++ { @@ -47,24 +48,31 @@ type benchResult struct { for _, sp := range object.SupportedSplitters { fact := object.GetSplitterFactory(sp) + var segmentLengths []int t0 := time.Now() + for _, data := range dataBlocks { s := fact() l := 0 + for _, d := range data { l++ + if s.ShouldSplit(d) { segmentLengths = append(segmentLengths, l) l = 0 } } + if l > 0 { segmentLengths = append(segmentLengths, l) } } + dur := time.Since(t0) + sort.Ints(segmentLengths) r := benchResult{ @@ -93,6 +101,7 @@ type benchResult struct { return results[i].duration < results[j].duration }) printStdout("-----------------------------------------------------------------\n") + for ndx, r := range results { printStdout("%3v. %-25v %6v ms count:%v min:%v 10th:%v 25th:%v 50th:%v 75th:%v 90th:%v max:%v\n", ndx, @@ -100,8 +109,8 @@ type benchResult struct { r.duration.Nanoseconds()/1e6, r.segmentCount, r.min, r.p10, r.p25, r.p50, r.p75, r.p90, r.max) - } + return nil } diff --git a/cli/command_blob_gc.go b/cli/command_blob_gc.go index 22aa416cd..f5f5778c2 100644 --- a/cli/command_blob_gc.go +++ b/cli/command_blob_gc.go @@ -18,6 +18,7 @@ func runBlobGarbageCollectCommand(ctx context.Context, rep *repo.Repository) error { var mu sync.Mutex + var unused []blob.Metadata if err := rep.Content.IterateUnreferencedBlobs(ctx, *blobGarbageCollectParallel, func(bm blob.Metadata) error { @@ -39,6 +40,7 @@ func runBlobGarbageCollectCommand(ctx context.Context, rep *repo.Repository) err for _, u := range unused { totalBytes += u.Length } + printStderr("Would delete %v unused blobs (%v bytes), pass '--delete=yes' to actually delete.\n", len(unused), totalBytes) return nil @@ -46,6 +48,7 @@ func runBlobGarbageCollectCommand(ctx context.Context, rep *repo.Repository) err for _, u := range unused { printStderr("Deleting unused blob %q (%v bytes)...\n", u.BlobID, u.Length) + if err := rep.Blobs.DeleteBlob(ctx, u.BlobID); err != nil { return errors.Wrapf(err, "unable to delete blob %q", u.BlobID) } diff --git a/cli/command_blob_show.go b/cli/command_blob_show.go index 3673c2328..3db9e4f50 100644 --- a/cli/command_blob_show.go +++ b/cli/command_blob_show.go @@ -23,6 +23,7 @@ func runBlobShow(ctx context.Context, rep *repo.Repository) error { if err != nil { return errors.Wrapf(err, "error getting %v", blobID) } + if _, err := io.Copy(os.Stdout, bytes.NewReader(d)); err != nil { return err } diff --git a/cli/command_cache_clear.go b/cli/command_cache_clear.go index 887ea9608..4dc08966f 100644 --- a/cli/command_cache_clear.go +++ b/cli/command_cache_clear.go @@ -16,6 +16,7 @@ func runCacheClearCommand(ctx context.Context, rep *repo.Repository) error { if d := rep.Content.CachingOptions.CacheDirectory; d != "" { printStderr("Clearing cache directory: %v.\n", d) + err := os.RemoveAll(d) if err != nil { return err @@ -26,6 +27,7 @@ func runCacheClearCommand(ctx context.Context, rep *repo.Repository) error { } printStderr("Cache cleared.\n") + return nil } diff --git a/cli/command_cache_info.go b/cli/command_cache_info.go index df8203471..96787b26d 100644 --- a/cli/command_cache_info.go +++ b/cli/command_cache_info.go @@ -37,7 +37,9 @@ func runCacheInfoCommand(ctx context.Context, rep *repo.Repository) error { if !ent.IsDir() { continue } + subdir := filepath.Join(rep.Content.CachingOptions.CacheDirectory, ent.Name()) + fileCount, totalFileSize, err := scanCacheDir(subdir) if err != nil { return err @@ -47,6 +49,7 @@ func runCacheInfoCommand(ctx context.Context, rep *repo.Repository) error { if l, ok := path2Limit[ent.Name()]; ok { maybeLimit = fmt.Sprintf(" (limit %v)", units.BytesStringBase10(l)) } + fmt.Printf("%v: %v files %v%v\n", subdir, fileCount, units.BytesStringBase10(totalFileSize), maybeLimit) } diff --git a/cli/command_content_list.go b/cli/command_content_list.go index a2630973f..5bcd7ef66 100644 --- a/cli/command_content_list.go +++ b/cli/command_content_list.go @@ -23,7 +23,9 @@ func runContentListCommand(ctx context.Context, rep *repo.Repository) error { var count int32 + var totalSize int64 + err := rep.Content.IterateContents( content.IterateOptions{ Prefix: content.ID(*contentListPrefix), diff --git a/cli/command_content_rewrite.go b/cli/command_content_rewrite.go index 617ac7710..67886a142 100644 --- a/cli/command_content_rewrite.go +++ b/cli/command_content_rewrite.go @@ -41,6 +41,7 @@ func runContentRewriteCommand(ctx context.Context, rep *repo.Repository) error { for i := 0; i < *contentRewriteParallelism; i++ { wg.Add(1) + go func() { defer wg.Done() @@ -88,6 +89,7 @@ func runContentRewriteCommand(ctx context.Context, rep *repo.Repository) error { func getContentToRewrite(ctx context.Context, rep *repo.Repository) <-chan contentInfoOrError { ch := make(chan contentInfoOrError) + go func() { defer close(ch) @@ -114,6 +116,7 @@ func toContentIDs(s []string) []content.ID { for _, cid := range s { result = append(result, content.ID(cid)) } + return result } diff --git a/cli/command_content_stats.go b/cli/command_content_stats.go index f8660e040..d136e18fc 100644 --- a/cli/command_content_stats.go +++ b/cli/command_content_stats.go @@ -17,17 +17,20 @@ func runContentStatsCommand(ctx context.Context, rep *repo.Repository) error { var sizeThreshold uint32 = 10 + countMap := map[uint32]int{} totalSizeOfContentsUnder := map[uint32]int64{} + var sizeThresholds []uint32 + for i := 0; i < 8; i++ { sizeThresholds = append(sizeThresholds, sizeThreshold) countMap[sizeThreshold] = 0 sizeThreshold *= 10 } - var totalSize int64 - var count int64 + var totalSize, count int64 + if err := rep.Content.IterateContents( content.IterateOptions{}, func(b content.Info) error { @@ -51,13 +54,17 @@ func(b content.Info) error { fmt.Println("Count:", count) fmt.Println("Total:", sizeToString(totalSize)) + if count == 0 { return nil } + fmt.Println("Average:", sizeToString(totalSize/count)) fmt.Printf("Histogram:\n\n") + var lastSize uint32 + for _, size := range sizeThresholds { fmt.Printf("%9v between %v and %v (total %v)\n", countMap[size]-countMap[lastSize], @@ -65,6 +72,7 @@ func(b content.Info) error { sizeToString(int64(size)), sizeToString(totalSizeOfContentsUnder[size]-totalSizeOfContentsUnder[lastSize]), ) + lastSize = size } diff --git a/cli/command_content_verify.go b/cli/command_content_verify.go index 1796bac8d..a61039708 100644 --- a/cli/command_content_verify.go +++ b/cli/command_content_verify.go @@ -22,6 +22,7 @@ func runContentVerifyCommand(ctx context.Context, rep *repo.Repository) error { if contentID == "all" { return verifyAllContents(ctx, rep) } + if err := contentVerify(ctx, rep, contentID); err != nil { return err } @@ -32,6 +33,7 @@ func runContentVerifyCommand(ctx context.Context, rep *repo.Repository) error { func verifyAllContents(ctx context.Context, rep *repo.Repository) error { var errorCount int32 + err := rep.Content.IterateContents(content.IterateOptions{ Parallel: *contentVerifyParallel, }, func(ci content.Info) error { @@ -40,6 +42,7 @@ func verifyAllContents(ctx context.Context, rep *repo.Repository) error { } return nil }) + if err != nil { return errors.Wrap(err, "iterate contents") } @@ -56,7 +59,9 @@ func contentVerify(ctx context.Context, r *repo.Repository, contentID content.ID log.Warningf("content %v is invalid: %v", contentID, err) return err } + log.Infof("content %v is ok", contentID) + return nil } diff --git a/cli/command_diff.go b/cli/command_diff.go index 0bb80f26e..06bd3169c 100644 --- a/cli/command_diff.go +++ b/cli/command_diff.go @@ -25,6 +25,7 @@ func runDiffCommand(ctx context.Context, rep *repo.Repository) error { if err != nil { return err } + oid2, err := parseObjectID(ctx, rep, *diffSecondObjectPath) if err != nil { return err @@ -32,6 +33,7 @@ func runDiffCommand(ctx context.Context, rep *repo.Repository) error { isDir1 := strings.HasPrefix(string(oid1), "k") isDir2 := strings.HasPrefix(string(oid2), "k") + if isDir1 != isDir2 { return errors.New("arguments do diff must both be directories or both non-directories") } diff --git a/cli/command_ls.go b/cli/command_ls.go index 15226d7cd..6bf053eaf 100644 --- a/cli/command_ls.go +++ b/cli/command_ls.go @@ -51,8 +51,10 @@ func listDirectory(ctx context.Context, rep *repo.Repository, prefix string, oid for _, e := range entries { var info string + objectID := e.(object.HasObjectID).ObjectID() oid := objectID.String() + switch { case *lsCommandLong: info = fmt.Sprintf( @@ -71,7 +73,9 @@ func listDirectory(ctx context.Context, rep *repo.Repository, prefix string, oid default: info = nameToDisplay(prefix, e) } + fmt.Println(info) + if *lsCommandRecursive && e.Mode().IsDir() { if listerr := listDirectory(ctx, rep, prefix+e.Name()+"/", objectID, indent+" "); listerr != nil { return listerr @@ -86,8 +90,8 @@ func nameToDisplay(prefix string, e fs.Entry) string { suffix := "" if e.IsDir() { suffix = "/" - } + if *lsCommandLong || *lsCommandRecursive { return prefix + e.Name() + suffix } diff --git a/cli/command_manifest_ls.go b/cli/command_manifest_ls.go index de81ae610..1ab0121ec 100644 --- a/cli/command_manifest_ls.go +++ b/cli/command_manifest_ls.go @@ -63,9 +63,11 @@ func sortedMapValues(m map[string]string) string { if k == "type" { continue } + result = append(result, fmt.Sprintf("%v:%v", k, v)) } sort.Strings(result) + return strings.Join(result, " ") } diff --git a/cli/command_manifest_show.go b/cli/command_manifest_show.go index b6a4f5292..ca1e0fe1e 100644 --- a/cli/command_manifest_show.go +++ b/cli/command_manifest_show.go @@ -25,6 +25,7 @@ func toManifestIDs(s []string) []manifest.ID { for _, it := range s { result = append(result, manifest.ID(it)) } + return result } @@ -43,9 +44,11 @@ func showManifestItems(ctx context.Context, rep *repo.Repository) error { printStderr("// id: %v\n", it) printStderr("// length: %v\n", md.Length) printStderr("// modified: %v\n", formatTimestamp(md.ModTime)) + for k, v := range md.Labels { printStderr("// label %v:%v\n", k, v) } + if showerr := showContentWithFlags(bytes.NewReader(b), false, true); showerr != nil { return showerr } diff --git a/cli/command_mount_browse.go b/cli/command_mount_browse.go index 3d6496954..0c9efe538 100644 --- a/cli/command_mount_browse.go +++ b/cli/command_mount_browse.go @@ -32,6 +32,7 @@ func browseMount(mountPoint, addr string) error { func openInWebBrowser(mountPoint, addr string) error { startWebBrowser(addr) waitForCtrlC() + return nil } @@ -42,6 +43,7 @@ func openInOSBrowser(mountPoint, addr string) error { startWebBrowser(addr) waitForCtrlC() + return nil } @@ -50,6 +52,7 @@ func netUSE(mountPoint, addr string) error { c.Stdout = os.Stdout c.Stderr = os.Stderr c.Stdin = os.Stdin + if err := c.Run(); err != nil { return errors.Wrap(err, "unable to mount") } @@ -61,6 +64,7 @@ func netUSE(mountPoint, addr string) error { c.Stdout = os.Stdout c.Stderr = os.Stderr c.Stdin = os.Stdin + if err := c.Run(); err != nil { return errors.Wrap(err, "unable to unmount") } diff --git a/cli/command_mount_fuse.go b/cli/command_mount_fuse.go index 2a1746391..10f4d5445 100644 --- a/cli/command_mount_fuse.go +++ b/cli/command_mount_fuse.go @@ -50,5 +50,6 @@ func mountDirectoryFUSE(entry fs.Directory, mountPoint string) error { } // wait for mount to stop. <-fuseConnection.Ready + return fuseConnection.MountError } diff --git a/cli/command_mount_webdav.go b/cli/command_mount_webdav.go index 7590eefe6..e36caa616 100644 --- a/cli/command_mount_webdav.go +++ b/cli/command_mount_webdav.go @@ -21,6 +21,7 @@ func webdavServerLogger(r *http.Request, err error) { if r := r.Header.Get("Range"); r != "" { maybeRange = " " + r } + if err != nil { log.Debugf("%v %v%v err: %v", r.Method, r.URL.RequestURI(), maybeRange, err) } else { @@ -51,6 +52,7 @@ func mountDirectoryWebDAV(entry fs.Directory, mountPoint string) error { var wg sync.WaitGroup wg.Add(1) + go func() { defer wg.Done() printStderr("Server listening at http://%v/ Press Ctrl-C to shut down.\n", s.Addr) @@ -67,6 +69,8 @@ func mountDirectoryWebDAV(entry fs.Directory, mountPoint string) error { if err := s.Shutdown(context.Background()); err != nil { log.Warningf("shutdown failed: %v", err) } + wg.Wait() + return nil } diff --git a/cli/command_policy.go b/cli/command_policy.go index ba1af94bc..1a7951600 100644 --- a/cli/command_policy.go +++ b/cli/command_policy.go @@ -23,12 +23,14 @@ func policyTargets(ctx context.Context, rep *repo.Repository, globalFlag *bool, } var res []snapshot.SourceInfo + for _, ts := range *targetsFlag { // try loading policy by its manifest ID if t, err := policy.GetPolicyByID(ctx, rep, manifest.ID(ts)); err == nil { res = append(res, t.Target()) continue } + target, err := snapshot.ParseSourceInfo(ts, getHostName(), getUserName()) if err != nil { return nil, err diff --git a/cli/command_policy_edit.go b/cli/command_policy_edit.go index 8786bb698..48e8961eb 100644 --- a/cli/command_policy_edit.go +++ b/cli/command_policy_edit.go @@ -97,7 +97,9 @@ func editPolicy(ctx context.Context, rep *repo.Repository) error { printStderr("Updated policy for %v\n%v\n", target, prettyJSON(updated)) fmt.Print("Save updated policy? (y/N) ") + var shouldSave string + fmt.Scanf("%v", &shouldSave) //nolint:errcheck if strings.HasPrefix(strings.ToLower(shouldSave), "y") { @@ -115,6 +117,7 @@ func prettyJSON(v interface{}) string { e := json.NewEncoder(&b) e.SetIndent("", " ") e.Encode(v) //nolint:errcheck + return b.String() } diff --git a/cli/command_policy_remove.go b/cli/command_policy_remove.go index b5a09de4c..f3e6b93c1 100644 --- a/cli/command_policy_remove.go +++ b/cli/command_policy_remove.go @@ -26,6 +26,7 @@ func removePolicy(ctx context.Context, rep *repo.Repository) error { for _, target := range targets { log.Infof("Removing policy on %q...", target) + if err := policy.RemovePolicy(ctx, rep, target); err != nil { return err } diff --git a/cli/command_policy_set.go b/cli/command_policy_set.go index 1393af5a5..c9ca92a4b 100644 --- a/cli/command_policy_set.go +++ b/cli/command_policy_set.go @@ -67,8 +67,8 @@ func setPolicy(ctx context.Context, rep *repo.Repository) error { } printStderr("Setting policy for %v\n", target) - changeCount := 0 + changeCount := 0 if err := setPolicyFromFlags(p, &changeCount); err != nil { return err } @@ -103,6 +103,7 @@ func setPolicyFromFlags(p *policy.Policy, changeCount *int) error { // It's not really a list, just optional boolean, last one wins. for _, inherit := range *policySetInherit { *changeCount++ + p.NoParent = !inherit } @@ -113,13 +114,17 @@ func setFilesPolicyFromFlags(fp *ignorefs.FilesPolicy, changeCount *int) { if *policySetClearDotIgnore { *changeCount++ printStderr(" - removing all rules for dot-ignore files\n") + fp.DotIgnoreFiles = nil } else { fp.DotIgnoreFiles = addRemoveDedupeAndSort("dot-ignore files", fp.DotIgnoreFiles, *policySetAddDotIgnore, *policySetRemoveDotIgnore, changeCount) } + if *policySetClearIgnore { *changeCount++ + fp.IgnoreRules = nil + printStderr(" - removing all ignore rules\n") } else { fp.IgnoreRules = addRemoveDedupeAndSort("ignored files", fp.IgnoreRules, *policySetAddIgnore, *policySetRemoveIgnore, changeCount) @@ -145,6 +150,7 @@ func setRetentionPolicyFromFlags(rp *policy.RetentionPolicy, changeCount *int) e return err } } + return nil } @@ -154,6 +160,7 @@ func setSchedulingPolicyFromFlags(sp *policy.SchedulingPolicy, changeCount *int) *changeCount++ sp.SetInterval(interval) printStderr(" - setting snapshot interval to %v\n", sp.Interval()) + break } @@ -171,6 +178,7 @@ func setSchedulingPolicyFromFlags(sp *policy.SchedulingPolicy, changeCount *int) if err := timeOfDay.Parse(tod); err != nil { return errors.Wrap(err, "unable to parse time of day") } + timesOfDay = append(timesOfDay, timeOfDay) } } @@ -193,11 +201,14 @@ func addRemoveDedupeAndSort(desc string, base, add, remove []string, changeCount for _, b := range base { entries[b] = true } + for _, b := range add { *changeCount++ printStderr(" - adding %v to %v\n", b, desc) + entries[b] = true } + for _, b := range remove { *changeCount++ printStderr(" - removing %v from %v\n", b, desc) @@ -208,7 +219,9 @@ func addRemoveDedupeAndSort(desc string, base, add, remove []string, changeCount for k := range entries { s = append(s, k) } + sort.Strings(s) + return s } @@ -221,7 +234,9 @@ func applyPolicyNumber(desc string, val **int, str string, changeCount *int) err if str == inheritPolicyString || str == "default" { *changeCount++ printStderr(" - resetting %v to a default value inherited from parent.\n", desc) + *val = nil + return nil } @@ -234,6 +249,7 @@ func applyPolicyNumber(desc string, val **int, str string, changeCount *int) err *changeCount++ printStderr(" - setting %v to %v.\n", desc, i) *val = &i + return nil } @@ -246,7 +262,9 @@ func applyPolicyNumber64(desc string, val *int64, str string, changeCount *int) if str == inheritPolicyString || str == "default" { *changeCount++ printStderr(" - resetting %v to a default value inherited from parent.\n", desc) + *val = 0 + return nil } @@ -258,5 +276,6 @@ func applyPolicyNumber64(desc string, val *int64, str string, changeCount *int) *changeCount++ printStderr(" - setting %v to %v.\n", desc, v) *val = v + return nil } diff --git a/cli/command_policy_show.go b/cli/command_policy_show.go index a3109fa4e..02873521a 100644 --- a/cli/command_policy_show.go +++ b/cli/command_policy_show.go @@ -54,13 +54,13 @@ func getDefinitionPoint(parents []*policy.Policy, match func(p *policy.Policy) b return "inherited from " + p.Target().String() } + if p.NoParent { break } } return "(default)" - } func containsString(s []string, v string) bool { @@ -69,6 +69,7 @@ func containsString(s []string, v string) bool { return true } } + return false } @@ -124,21 +125,25 @@ func printFilesPolicy(p *policy.Policy, parents []*policy.Policy) { } else { printStdout(" No ignore rules.\n") } + for _, rule := range p.FilesPolicy.IgnoreRules { rule := rule printStdout(" %-30v %v\n", rule, getDefinitionPoint(parents, func(pol *policy.Policy) bool { return containsString(pol.FilesPolicy.IgnoreRules, rule) })) } + if len(p.FilesPolicy.DotIgnoreFiles) > 0 { printStdout(" Read ignore rules from files:\n") } + for _, dotFile := range p.FilesPolicy.DotIgnoreFiles { dotFile := dotFile printStdout(" %-30v %v\n", dotFile, getDefinitionPoint(parents, func(pol *policy.Policy) bool { return containsString(pol.FilesPolicy.DotIgnoreFiles, dotFile) })) } + if maxSize := p.FilesPolicy.MaxFileSize; maxSize > 0 { printStdout(" Ignore files above: %10v %v\n", units.BytesStringBase2(maxSize), @@ -157,6 +162,7 @@ func printSchedulingPolicy(p *policy.Policy, parents []*policy.Policy) { if len(p.SchedulingPolicy.TimesOfDay) > 0 { printStdout("Snapshot times:\n") + for _, tod := range p.SchedulingPolicy.TimesOfDay { tod := tod printStdout(" %9v %v\n", tod, getDefinitionPoint(parents, func(pol *policy.Policy) bool { diff --git a/cli/command_repository_connect.go b/cli/command_repository_connect.go index b28da7931..fc29d9296 100644 --- a/cli/command_repository_connect.go +++ b/cli/command_repository_connect.go @@ -51,6 +51,7 @@ func runConnectCommandWithStorage(ctx context.Context, st blob.Storage) error { if err != nil { return errors.Wrap(err, "getting password") } + return runConnectCommandWithStorageAndPassword(ctx, st, password) } @@ -69,5 +70,6 @@ func runConnectCommandWithStorageAndPassword(ctx context.Context, st blob.Storag } printStderr("Connected to repository.\n") + return nil } diff --git a/cli/command_repository_connect_from_config.go b/cli/command_repository_connect_from_config.go index 73d044ba9..476b1abad 100644 --- a/cli/command_repository_connect_from_config.go +++ b/cli/command_repository_connect_from_config.go @@ -53,6 +53,7 @@ func connectToStorageFromConfigToken(ctx context.Context) (blob.Storage, error) } passwordFromToken = pass + return blob.NewStorage(ctx, ci) } diff --git a/cli/command_repository_create.go b/cli/command_repository_create.go index 88c64ca26..1e4c2e2bb 100644 --- a/cli/command_repository_create.go +++ b/cli/command_repository_create.go @@ -55,6 +55,7 @@ func newRepositoryOptionsFromFlags() *repo.NewRepositoryOptions { func ensureEmpty(ctx context.Context, s blob.Storage) error { hasDataError := errors.New("has data") + err := s.ListBlobs(ctx, "", func(cb blob.Metadata) error { return hasDataError }) @@ -115,6 +116,7 @@ func populateRepository(ctx context.Context, password string) error { } printPolicy(globalPolicy, nil) + return nil } @@ -122,6 +124,7 @@ func getInitialGlobalPolicy() (*policy.Policy, error) { var sp policy.SchedulingPolicy sp.SetInterval(*createGlobalPolicyInterval) + var timesOfDay []policy.TimeOfDay for _, tods := range *createGlobalPolicyTimesOfDay { @@ -130,9 +133,11 @@ func getInitialGlobalPolicy() (*policy.Policy, error) { if err := timeOfDay.Parse(tod); err != nil { return nil, errors.Wrap(err, "unable to parse time of day") } + timesOfDay = append(timesOfDay, timeOfDay) } } + sp.TimesOfDay = policy.SortAndDedupeTimesOfDay(timesOfDay) return &policy.Policy{ diff --git a/cli/command_repository_repair.go b/cli/command_repository_repair.go index d3f9ec680..295c03d5f 100644 --- a/cli/command_repository_repair.go +++ b/cli/command_repository_repair.go @@ -20,16 +20,19 @@ func packBlockPrefixes() []string { var str []string + for _, p := range content.PackBlobIDPrefixes { str = append(str, string(p)) } - return str + return str } + func runRepairCommandWithStorage(ctx context.Context, st blob.Storage) error { if err := maybeRecoverFormatBlob(ctx, st); err != nil { return err } + return nil } @@ -37,6 +40,7 @@ func maybeRecoverFormatBlob(ctx context.Context, st blob.Storage) error { switch *repairCommandRecoverFormatBlob { case "auto": log.Infof("looking for format blob...") + if _, err := st.GetBlob(ctx, repo.FormatBlobID, 0, -1); err == nil { log.Infof("format blob already exists, not recovering, pass --recover-format=yes") return nil @@ -82,7 +86,6 @@ func recoverFormatBlob(ctx context.Context, st blob.Storage, prefixes []string) default: return err } - } return errors.New("could not find a replica of a format blob") diff --git a/cli/command_repository_status.go b/cli/command_repository_status.go index 52ef6d570..9e877fcb8 100644 --- a/cli/command_repository_status.go +++ b/cli/command_repository_status.go @@ -47,6 +47,7 @@ func runStatusCommand(ctx context.Context, rep *repo.Repository) error { if *statusReconnectTokenIncludePassword { var err error + pass, err = getPasswordFromFlags(false, true) if err != nil { return errors.Wrap(err, "getting password") @@ -59,6 +60,7 @@ func runStatusCommand(ctx context.Context, rep *repo.Repository) error { } fmt.Printf("\nTo reconnect to the repository use:\n\n$ kopia repository connect from-config --token %v\n\n", tok) + if pass != "" { fmt.Printf("NOTICE: The token printed above can be trivially decoded to reveal the repository password. Do not store it in an unsecured place.\n") } @@ -76,16 +78,20 @@ func scanCacheDir(dirname string) (fileCount int, totalFileLength int64, err err for _, e := range entries { if e.IsDir() { subdir := filepath.Join(dirname, e.Name()) + c, l, err2 := scanCacheDir(subdir) if err2 != nil { return 0, 0, err2 } + fileCount += c totalFileLength += l + continue } fileCount++ + totalFileLength += e.Size() } diff --git a/cli/command_server_start.go b/cli/command_server_start.go index f4c9f78f4..6ade9b3b1 100644 --- a/cli/command_server_start.go +++ b/cli/command_server_start.go @@ -38,12 +38,14 @@ func runServer(ctx context.Context, rep *repo.Repository) error { url := "http://" + *serverAddress log.Infof("starting server on %v", url) http.Handle("/api/", maybeRequireAuth(srv.APIHandlers())) + if *serverStartHTMLPath != "" { fileServer := http.FileServer(http.Dir(*serverStartHTMLPath)) http.Handle("/", maybeRequireAuth(fileServer)) } else if *serverStartUI { http.Handle("/", maybeRequireAuth(http.FileServer(server.AssetFile()))) } + return http.ListenAndServe(*serverAddress, nil) } @@ -66,6 +68,7 @@ func (a requireAuth) ServeHTTP(w http.ResponseWriter, r *http.Request) { if !ok { w.Header().Set("WWW-Authenticate", `Basic realm="Kopia"`) http.Error(w, "Missing credentials.\n", http.StatusUnauthorized) + return } @@ -75,6 +78,7 @@ func (a requireAuth) ServeHTTP(w http.ResponseWriter, r *http.Request) { if valid != 1 { w.Header().Set("WWW-Authenticate", `Basic realm="Kopia"`) http.Error(w, "Access denied.\n", http.StatusUnauthorized) + return } diff --git a/cli/command_show.go b/cli/command_show.go index 231f486d9..409456fc7 100644 --- a/cli/command_show.go +++ b/cli/command_show.go @@ -18,12 +18,16 @@ func runCatCommand(ctx context.Context, rep *repo.Repository) error { if err != nil { return err } + r, err := rep.Objects.Open(ctx, oid) if err != nil { return err } + defer r.Close() //nolint:errcheck + _, err = io.Copy(os.Stdout, r) + return err } diff --git a/cli/command_snapshot_create.go b/cli/command_snapshot_create.go index 1d7f2e6b3..ccd5fe836 100644 --- a/cli/command_snapshot_create.go +++ b/cli/command_snapshot_create.go @@ -34,11 +34,13 @@ func runBackupCommand(ctx context.Context, rep *repo.Repository) error { sources := *snapshotCreateSources + if *snapshotCreateAll { local, err := getLocalBackupPaths(ctx, rep) if err != nil { return err } + sources = append(sources, local...) } @@ -62,13 +64,16 @@ func runBackupCommand(ctx context.Context, rep *repo.Repository) error { for _, snapshotDir := range sources { log.Debugf("Backing up %v", snapshotDir) + dir, err := filepath.Abs(snapshotDir) if err != nil { return errors.Errorf("invalid source: '%s': %s", snapshotDir, err) } sourceInfo := snapshot.SourceInfo{Path: filepath.Clean(dir), Host: getHostName(), UserName: getUserName()} + log.Infof("snapshotting %v", sourceInfo) + if err := snapshotSingleSource(ctx, rep, u, sourceInfo); err != nil { finalErrors = append(finalErrors, err.Error()) } @@ -83,6 +88,7 @@ func runBackupCommand(ctx context.Context, rep *repo.Repository) error { func snapshotSingleSource(ctx context.Context, rep *repo.Repository, u *snapshotfs.Uploader, sourceInfo snapshot.SourceInfo) error { t0 := time.Now() + rep.Content.ResetStats() localEntry, err := getLocalFSEntry(sourceInfo.Path) @@ -101,6 +107,7 @@ func snapshotSingleSource(ctx context.Context, rep *repo.Repository, u *snapshot } log.Infof("uploading %v using %v previous manifests", sourceInfo, len(previous)) + manifest, err := u.Upload(ctx, localEntry, sourceInfo, previous...) if err != nil { return err @@ -116,6 +123,7 @@ func snapshotSingleSource(ctx context.Context, rep *repo.Repository, u *snapshot printStderr("uploaded snapshot %v (root %v) in %v\n", snapID, manifest.RootObjectID(), time.Since(t0)) _, err = policy.ApplyRetentionPolicy(ctx, rep, sourceInfo, true) + return err } @@ -129,13 +137,16 @@ func findPreviousSnapshotManifest(ctx context.Context, rep *repo.Repository, sou // phase 1 - find latest complete snapshot. var previousComplete *snapshot.Manifest + var previousCompleteStartTime time.Time + var result []*snapshot.Manifest for _, p := range man { if noLaterThan != nil && p.StartTime.After(*noLaterThan) { continue } + if p.IncompleteReason == "" && (previousComplete == nil || p.StartTime.After(previousComplete.StartTime)) { previousComplete = p previousCompleteStartTime = p.StartTime @@ -151,6 +162,7 @@ func findPreviousSnapshotManifest(ctx context.Context, rep *repo.Repository, sou if noLaterThan != nil && p.StartTime.After(*noLaterThan) { continue } + if p.IncompleteReason != "" && p.StartTime.After(previousCompleteStartTime) { result = append(result, p) } @@ -192,6 +204,7 @@ func getDefaultUserName() string { } u := currentUser.Username + if runtime.GOOS == "windows" { if p := strings.Index(u, "\\"); p >= 0 { // On Windows ignore domain name. diff --git a/cli/command_snapshot_estimate.go b/cli/command_snapshot_estimate.go index a4d3dd29b..2cdb7205a 100644 --- a/cli/command_snapshot_estimate.go +++ b/cli/command_snapshot_estimate.go @@ -36,6 +36,7 @@ type bucket struct { func (b *bucket) add(fname string, size int64) { b.Count++ b.TotalSize += size + if len(b.Examples) < 10 { b.Examples = append(b.Examples, fmt.Sprintf("%v - %v", fname, units.BytesStringBase10(size))) } @@ -72,12 +73,14 @@ func runSnapshotEstimateCommand(ctx context.Context, rep *repo.Repository) error sourceInfo := snapshot.SourceInfo{Path: filepath.Clean(path), Host: getHostName(), UserName: getUserName()} var stats snapshot.Stats + ib := makeBuckets() eb := makeBuckets() onIgnoredFile := func(relativePath string, e fs.Entry) { log.Noticef("ignoring %v", relativePath) eb.add(relativePath, e.Size()) + if e.IsDir() { stats.ExcludedDirCount++ } else { @@ -90,14 +93,17 @@ func runSnapshotEstimateCommand(ctx context.Context, rep *repo.Repository) error if err != nil { return err } + if dir, ok := entry.(fs.Directory); ok { ignorePolicy, err := policy.FilesPolicyGetter(ctx, rep, sourceInfo) if err != nil { return err } + entry = ignorefs.New(dir, ignorePolicy, ignorefs.ReportIgnoredFiles(onIgnoredFile)) } - if err := estimate(ctx, ".", entry, &stats, ib, eb); err != nil { + + if err := estimate(ctx, ".", entry, &stats, ib); err != nil { return err } @@ -122,7 +128,9 @@ func showBuckets(b buckets) { if bucket.Count == 0 { continue } + fmt.Printf(" with size over %-5v: %7v files, total size %v\n", units.BytesStringBase10(bucket.MinSize), bucket.Count, units.BytesStringBase10(bucket.TotalSize)) + if *snapshotEstimateShowFiles { for _, sample := range bucket.Examples { fmt.Printf(" %v\n", sample) @@ -131,19 +139,20 @@ func showBuckets(b buckets) { } } -func estimate(ctx context.Context, relativePath string, entry fs.Entry, stats *snapshot.Stats, ib, eb buckets) error { +func estimate(ctx context.Context, relativePath string, entry fs.Entry, stats *snapshot.Stats, ib buckets) error { switch entry := entry.(type) { case fs.Directory: if !*snapshotEstimateQuiet { printStderr("Scanning %v\n", relativePath) } + children, err := entry.Readdir(ctx) if err != nil { return err } for _, child := range children { - if err := estimate(ctx, filepath.Join(relativePath, child.Name()), child, stats, ib, eb); err != nil { + if err := estimate(ctx, filepath.Join(relativePath, child.Name()), child, stats, ib); err != nil { return err } } @@ -153,6 +162,7 @@ func estimate(ctx context.Context, relativePath string, entry fs.Entry, stats *s stats.TotalFileCount++ stats.TotalFileSize += entry.Size() } + return nil } diff --git a/cli/command_snapshot_expire.go b/cli/command_snapshot_expire.go index 756480848..21752cf72 100644 --- a/cli/command_snapshot_expire.go +++ b/cli/command_snapshot_expire.go @@ -23,11 +23,13 @@ func getSnapshotSourcesToExpire(ctx context.Context, rep *repo.Repository) ([]sn } var result []snapshot.SourceInfo + for _, p := range *snapshotExpirePaths { src, err := snapshot.ParseSourceInfo(p, getHostName(), getUserName()) if err != nil { return nil, err } + result = append(result, src) } @@ -54,10 +56,12 @@ func runExpireCommand(ctx context.Context, rep *repo.Repository) error { printStderr("Nothing to delete for %v.\n", src) continue } + if *snapshotExpireDelete { printStderr("Deleted %v snapshots of %v...\n", len(deleted), src) } else { printStderr("%v snapshot(s) of %v would be deleted. Pass --delete to do it.\n", len(deleted), src) + for _, it := range deleted { printStderr(" %v\n", formatTimestamp(it.StartTime)) } diff --git a/cli/command_snapshot_list.go b/cli/command_snapshot_list.go index c85216eda..9b8221b41 100644 --- a/cli/command_snapshot_list.go +++ b/cli/command_snapshot_list.go @@ -57,6 +57,7 @@ func findSnapshotsForSource(ctx context.Context, rep *repo.Repository, sourceInf if parentPath == sourceInfo.Path { break } + sourceInfo.Path = parentPath } @@ -103,7 +104,6 @@ func shouldOutputSnapshotSource(src snapshot.SourceInfo) bool { if src.Host != getHostName() { return false - } return src.UserName == getUserName() @@ -111,14 +111,18 @@ func shouldOutputSnapshotSource(src snapshot.SourceInfo) bool { func outputManifestGroups(ctx context.Context, rep *repo.Repository, manifests []*snapshot.Manifest, relPathParts []string) error { separator := "" + var anyOutput bool + for _, snapshotGroup := range snapshot.GroupBySource(manifests) { src := snapshotGroup[0].Source if !shouldOutputSnapshotSource(src) { log.Debugf("skipping %v", src) continue } + fmt.Printf("%v%v\n", separator, src) + separator = "\n" anyOutput = true @@ -128,6 +132,7 @@ func outputManifestGroups(ctx context.Context, rep *repo.Repository, manifests [ } else { pol.RetentionPolicy.ComputeRetentionReasons(snapshotGroup) } + if err := outputManifestFromSingleSource(ctx, rep, snapshotGroup, relPathParts); err != nil { return err } @@ -143,6 +148,7 @@ func outputManifestGroups(ctx context.Context, rep *repo.Repository, manifests [ //nolint:gocyclo,funlen func outputManifestFromSingleSource(ctx context.Context, rep *repo.Repository, manifests []*snapshot.Manifest, parts []string) error { var count int + var lastTotalFileSize int64 manifests = snapshot.SortByTime(manifests, false) @@ -151,7 +157,9 @@ func outputManifestFromSingleSource(ctx context.Context, rep *repo.Repository, m } var previousOID object.ID + var elidedCount int + var maxElidedTime time.Time outputElided := func() { @@ -170,6 +178,7 @@ func outputManifestFromSingleSource(ctx context.Context, rep *repo.Repository, m fmt.Printf(" %v %v\n", formatTimestamp(m.StartTime), err) continue } + ent, err := getNestedEntry(ctx, root, parts) if err != nil { fmt.Printf(" %v %v\n", formatTimestamp(m.StartTime), err) @@ -182,10 +191,12 @@ func outputManifestFromSingleSource(ctx context.Context, rep *repo.Repository, m } var bits []string + if m.IncompleteReason != "" { if !*snapshotListIncludeIncomplete { continue } + bits = append(bits, "incomplete:"+m.IncompleteReason) } @@ -197,6 +208,7 @@ func outputManifestFromSingleSource(ctx context.Context, rep *repo.Repository, m fmt.Sprintf("uid:%v", ent.Owner().UserID), fmt.Sprintf("gid:%v", ent.Owner().GroupID)) } + if *snapshotListShowModTime { bits = append(bits, fmt.Sprintf("modified:%v", formatTimestamp(ent.ModTime()))) } @@ -227,13 +239,16 @@ func outputManifestFromSingleSource(ctx context.Context, rep *repo.Repository, m oid := ent.(object.HasObjectID).ObjectID() if !*snapshotListShowIdentical && oid == previousOID { elidedCount++ + maxElidedTime = m.StartTime + continue } previousOID = oid outputElided() + elidedCount = 0 fmt.Printf( @@ -244,10 +259,12 @@ func outputManifestFromSingleSource(ctx context.Context, rep *repo.Repository, m ) count++ + if m.IncompleteReason == "" { lastTotalFileSize = m.Stats.TotalFileSize } } + outputElided() return nil diff --git a/cli/command_snapshot_migrate.go b/cli/command_snapshot_migrate.go index fa7be2329..f5998c69f 100644 --- a/cli/command_snapshot_migrate.go +++ b/cli/command_snapshot_migrate.go @@ -40,9 +40,9 @@ func runMigrateCommand(ctx context.Context, destRepo *repo.Repository) error { } semaphore := make(chan struct{}, *migrateParallelism) - var wg sync.WaitGroup var ( + wg sync.WaitGroup mu sync.Mutex canceled bool activeUploaders = map[snapshot.SourceInfo]*snapshotfs.Uploader{} @@ -77,6 +77,7 @@ func runMigrateCommand(ctx context.Context, destRepo *repo.Repository) error { wg.Add(1) semaphore <- struct{}{} + go func(s snapshot.SourceInfo) { defer func() { mu.Lock() @@ -92,6 +93,7 @@ func runMigrateCommand(ctx context.Context, destRepo *repo.Repository) error { } }(s) } + wg.Wait() return nil @@ -117,6 +119,7 @@ func migrateSingleSource(ctx context.Context, uploader *snapshotfs.Uploader, sou if err != nil { return err } + snapshots, err := snapshot.LoadSnapshots(ctx, sourceRepo, manifests) if err != nil { return errors.Wrapf(err, "unable to load snapshot manifests for %v", s) @@ -134,7 +137,6 @@ func migrateSingleSource(ctx context.Context, uploader *snapshotfs.Uploader, sou if err := migrateSingleSourceSnapshot(ctx, uploader, sourceRepo, destRepo, s, m); err != nil { return err } - } return nil @@ -145,18 +147,21 @@ func migrateSingleSourceSnapshot(ctx context.Context, uploader *snapshotfs.Uploa log.Infof("ignoring incomplete %v at %v", s, formatTimestamp(m.StartTime)) return nil } + sourceEntry := snapshotfs.DirectoryEntry(sourceRepo, m.RootObjectID(), nil) existing, err := findPreviousSnapshotManifestWithStartTime(ctx, destRepo, m.Source, m.StartTime) if err != nil { return err } + if existing != nil { log.Infof("already migrated %v at %v", s, formatTimestamp(m.StartTime)) return nil } log.Infof("migrating snapshot of %v at %v", s, formatTimestamp(m.StartTime)) + previous, err := findPreviousSnapshotManifest(ctx, destRepo, m.Source, &m.StartTime) if err != nil { return err @@ -170,11 +175,13 @@ func migrateSingleSourceSnapshot(ctx context.Context, uploader *snapshotfs.Uploa newm.StartTime = m.StartTime newm.EndTime = m.EndTime newm.Description = m.Description + if newm.IncompleteReason == "" { if _, err := snapshot.SaveSnapshot(ctx, destRepo, newm); err != nil { return errors.Wrap(err, "cannot save manifest") } } + return nil } @@ -182,6 +189,7 @@ func filterSnapshotsToMigrate(s []*snapshot.Manifest) []*snapshot.Manifest { if *migrateLatestOnly && len(s) > 0 { s = s[0:1] } + return s } diff --git a/cli/command_snapshot_verify.go b/cli/command_snapshot_verify.go index 7cf1c7684..49f890f31 100644 --- a/cli/command_snapshot_verify.go +++ b/cli/command_snapshot_verify.go @@ -46,6 +46,7 @@ type verifier struct { func (v *verifier) progressCallback(enqueued, active, completed int64) { elapsed := time.Since(v.startTime) maybeTimeRemaining := "" + if elapsed > 1*time.Second && enqueued > 0 && completed > 0 { completedRatio := float64(completed) / float64(enqueued) predictedSeconds := elapsed.Seconds() / completedRatio @@ -56,6 +57,7 @@ func (v *verifier) progressCallback(enqueued, active, completed int64) { maybeTimeRemaining = fmt.Sprintf(" remaining %v (ETA %v)", dt.Truncate(1*time.Second), formatTimestamp(predictedEndTime.Truncate(1*time.Second))) } } + printStderr("Found %v objects, verifying %v, completed %v objects%v.\n", enqueued, active, completed, maybeTimeRemaining) } @@ -70,13 +72,12 @@ func (v *verifier) tooManyErrors() bool { return len(v.errors) >= *verifyCommandErrorThreshold } -func (v *verifier) reportError(path string, err error) bool { +func (v *verifier) reportError(path string, err error) { v.mu.Lock() defer v.mu.Unlock() log.Warningf("failed on %v: %v", path, err) v.errors = append(v.errors, err) - return len(v.errors) >= *verifyCommandErrorThreshold } func (v *verifier) shouldEnqueue(oid object.ID) bool { @@ -88,6 +89,7 @@ func (v *verifier) shouldEnqueue(oid object.ID) bool { } v.seen[oid] = true + return true } @@ -96,6 +98,7 @@ func (v *verifier) enqueueVerifyDirectory(ctx context.Context, oid object.ID, pa if !v.shouldEnqueue(oid) { return } + v.workQueue.EnqueueFront(func() error { return v.doVerifyDirectory(ctx, oid, path) }) @@ -106,6 +109,7 @@ func (v *verifier) enqueueVerifyObject(ctx context.Context, oid object.ID, path if !v.shouldEnqueue(oid) { return } + v.workQueue.EnqueueBack(func() error { return v.doVerifyObject(ctx, oid, path, expectedLength) }) @@ -115,6 +119,7 @@ func (v *verifier) doVerifyDirectory(ctx context.Context, oid object.ID, path st log.Debugf("verifying directory %q (%v)", path, oid) d := snapshotfs.DirectoryEntry(v.rep, oid, nil) + entries, err := d.Readdir(ctx) if err != nil { v.reportError(path, errors.Wrapf(err, "error reading %v", oid)) @@ -128,6 +133,7 @@ func (v *verifier) doVerifyDirectory(ctx context.Context, oid object.ID, path st objectID := e.(object.HasObjectID).ObjectID() childPath := path + "/" + e.Name() + if e.IsDir() { v.enqueueVerifyDirectory(ctx, objectID, childPath) } else { @@ -146,6 +152,7 @@ func (v *verifier) doVerifyObject(ctx context.Context, oid object.ID, path strin } var length int64 + var err error length, _, err = v.om.VerifyObject(ctx, oid) @@ -168,6 +175,7 @@ func (v *verifier) doVerifyObject(ctx context.Context, oid object.ID, path strin func (v *verifier) readEntireObject(ctx context.Context, oid object.ID, path string) error { log.Debugf("reading object %v %v", oid, path) + ctx = content.UsingContentCache(ctx, false) // also read the entire file @@ -178,6 +186,7 @@ func (v *verifier) readEntireObject(ctx context.Context, oid object.ID, path str defer r.Close() //nolint:errcheck _, err = io.Copy(ioutil.Discard, r) + return err } @@ -214,6 +223,7 @@ func enqueueRootsToVerify(ctx context.Context, v *verifier, rep *repo.Repository for _, man := range manifests { path := fmt.Sprintf("%v@%v", man.Source, formatTimestamp(man.StartTime)) + if man.RootEntry == nil { continue } @@ -248,11 +258,13 @@ func enqueueRootsToVerify(ctx context.Context, v *verifier, rep *repo.Repository func loadSourceManifests(ctx context.Context, rep *repo.Repository, sources []string) ([]*snapshot.Manifest, error) { var manifestIDs []manifest.ID + if *verifyCommandAllSources { man, err := snapshot.ListSnapshotManifests(ctx, rep, nil) if err != nil { return nil, err } + manifestIDs = append(manifestIDs, man...) } else { for _, srcStr := range sources { @@ -267,6 +279,7 @@ func loadSourceManifests(ctx context.Context, rep *repo.Repository, sources []st manifestIDs = append(manifestIDs, man...) } } + return snapshot.LoadSnapshots(ctx, rep, manifestIDs) } diff --git a/cli/config.go b/cli/config.go index fdeb20991..b7b97de34 100644 --- a/cli/config.go +++ b/cli/config.go @@ -40,6 +40,7 @@ func printStdout(msg string, args ...interface{}) { func onCtrlC(f func()) { c := make(chan os.Signal, 1) signal.Notify(c, os.Interrupt) + go func() { <-c f() @@ -49,6 +50,7 @@ func onCtrlC(f func()) { func waitForCtrlC() { // Wait until ctrl-c pressed done := make(chan bool) + onCtrlC(func() { if done != nil { close(done) @@ -63,6 +65,7 @@ func openRepository(ctx context.Context, opts *repo.Options) (*repo.Repository, if err != nil { return nil, errors.Wrap(err, "get password") } + r, err := repo.Open(ctx, repositoryConfigFileName(), pass, applyOptionsFromFlags(opts)) if os.IsNotExist(err) { return nil, errors.New("not connected to a repository, use 'kopia connect'") diff --git a/cli/memory_tracking.go b/cli/memory_tracking.go index 3389eec55..bac0324ca 100644 --- a/cli/memory_tracking.go +++ b/cli/memory_tracking.go @@ -20,18 +20,23 @@ func dumpMemoryUsage() { runtime.GC() + var ms runtime.MemStats + runtime.ReadMemStats(&ms) memoryTrackerMutex.Lock() defer memoryTrackerMutex.Unlock() memlog.Debugf("in use heap %v (delta %v max %v) stack %v (delta %v max %v)", ms.HeapInuse, int64(ms.HeapInuse-lastHeapUsage), maxHeapUsage, ms.StackInuse, int64(ms.StackInuse-lastStackInUse), maxStackInUse) + if ms.HeapInuse > maxHeapUsage { maxHeapUsage = ms.HeapInuse } + if ms.StackInuse > maxStackInUse { maxStackInUse = ms.StackInuse } + lastHeapUsage = ms.HeapInuse lastStackInUse = ms.StackInuse } diff --git a/cli/objref.go b/cli/objref.go index 133c48b98..b9b826db7 100644 --- a/cli/objref.go +++ b/cli/objref.go @@ -15,6 +15,7 @@ // ParseObjectID interprets the given ID string and returns corresponding object.ID. func parseObjectID(ctx context.Context, rep *repo.Repository, id string) (object.ID, error) { parts := strings.Split(id, "/") + oid, err := object.ParseID(parts[0]) if err != nil { return "", errors.Wrapf(err, "can't parse object ID %v", id) @@ -25,15 +26,18 @@ func parseObjectID(ctx context.Context, rep *repo.Repository, id string) (object } dir := snapshotfs.DirectoryEntry(rep, oid, nil) + return parseNestedObjectID(ctx, dir, parts[1:]) } func getNestedEntry(ctx context.Context, startingDir fs.Entry, parts []string) (fs.Entry, error) { current := startingDir + for _, part := range parts { if part == "" { continue } + dir, ok := current.(fs.Directory) if !ok { return nil, errors.Errorf("entry not found %q: parent is not a directory", part) @@ -60,5 +64,6 @@ func parseNestedObjectID(ctx context.Context, startingDir fs.Entry, parts []stri if err != nil { return "", err } + return e.(object.HasObjectID).ObjectID(), nil } diff --git a/cli/password.go b/cli/password.go index 3b6c7594d..038c2dbe9 100644 --- a/cli/password.go +++ b/cli/password.go @@ -25,6 +25,7 @@ func askForNewRepositoryPassword() (string, error) { if err != nil { return "", errors.Wrap(err, "password entry") } + p2, err := askPass("Re-enter password for verification: ") if err != nil { return "", errors.Wrap(err, "password verification") @@ -43,7 +44,9 @@ func askForExistingRepositoryPassword() (string, error) { if err != nil { return "", err } + fmt.Println() + return p1, nil } @@ -109,12 +112,14 @@ func getPersistedPassword(configFile, username string) (string, bool) { } log.Debugf("could not find persisted password") + return "", false } func persistPassword(configFile, username, password string) error { if *keyringEnabled { log.Debugf("saving password to OS keyring...") + err := keyring.Set(getKeyringItemID(configFile), username, password) if err == nil { log.Infof("Saved password") @@ -147,6 +152,7 @@ func deletePassword(configFile, username string) { func getKeyringItemID(configFile string) string { h := sha256.New() io.WriteString(h, configFile) //nolint:errcheck + return fmt.Sprintf("%v-%x", filepath.Base(configFile), h.Sum(nil)[0:8]) } diff --git a/cli/show_utils.go b/cli/show_utils.go index 230c7c8ba..924ebfcbb 100644 --- a/cli/show_utils.go +++ b/cli/show_utils.go @@ -43,6 +43,7 @@ func showContentWithFlags(rd io.Reader, unzip, indentJSON bool) error { } var buf1, buf2 bytes.Buffer + if indentJSON { if _, err := io.Copy(&buf1, rd); err != nil { return err diff --git a/cli/storage_filesystem.go b/cli/storage_filesystem.go index e37bbd0cd..834f90005 100644 --- a/cli/storage_filesystem.go +++ b/cli/storage_filesystem.go @@ -31,6 +31,7 @@ func connect(ctx context.Context, isNew bool) (blob.Storage, error) { if v := connectOwnerUID; v != "" { fso.FileUID = getIntPtrValue(v, 10) } + if v := connectOwnerGID; v != "" { fso.FileGID = getIntPtrValue(v, 10) } @@ -44,10 +45,12 @@ func connect(ctx context.Context, isNew bool) (blob.Storage, error) { if isNew { log.Infof("creating directory for repository: %v dir mode: %v", fso.Path, fso.DirectoryMode) + if err := os.MkdirAll(fso.Path, fso.DirectoryMode); err != nil { log.Warningf("unable to create directory: %v", fso.Path) } } + return filesystem.New(ctx, &fso) } diff --git a/cli/storage_gcs.go b/cli/storage_gcs.go index dca030847..775f97f3a 100644 --- a/cli/storage_gcs.go +++ b/cli/storage_gcs.go @@ -22,7 +22,6 @@ func(cmd *kingpin.CmdClause) { cmd.Flag("credentials-file", "Use the provided JSON file with credentials").ExistingFileVar(&options.ServiceAccountCredentials) cmd.Flag("max-download-speed", "Limit the download speed.").PlaceHolder("BYTES_PER_SEC").IntVar(&options.MaxDownloadSpeedBytesPerSecond) cmd.Flag("max-upload-speed", "Limit the upload speed.").PlaceHolder("BYTES_PER_SEC").IntVar(&options.MaxUploadSpeedBytesPerSecond) - }, func(ctx context.Context, isNew bool) (blob.Storage, error) { return gcs.New(ctx, &options) diff --git a/cli/storage_providers.go b/cli/storage_providers.go index b7c17b962..b396b4038 100644 --- a/cli/storage_providers.go +++ b/cli/storage_providers.go @@ -14,8 +14,8 @@ func RegisterStorageConnectFlags( name, description string, flags func(*kingpin.CmdClause), - connect func(ctx context.Context, isNew bool) (blob.Storage, error)) { - + connect func(ctx context.Context, isNew bool) (blob.Storage, error), +) { if name != "from-config" { // Set up 'create' subcommand cc := createCommand.Command(name, "Create repository in "+description) diff --git a/fs/cachefs/cache.go b/fs/cachefs/cache.go index dead9cbc9..98753da86 100644 --- a/fs/cachefs/cache.go +++ b/fs/cachefs/cache.go @@ -83,6 +83,7 @@ func (c *Cache) Readdir(ctx context.Context, d fs.Directory) (fs.Entries, error) if h, ok := d.(object.HasObjectID); ok { cacheID := string(h.ObjectID()) cacheExpiration := 24 * time.Hour + return c.getEntries(ctx, cacheID, cacheExpiration, d.Readdir) } @@ -93,9 +94,11 @@ func (c *Cache) getEntriesFromCacheLocked(id string) fs.Entries { if v, ok := c.data[id]; id != "" && ok { if time.Now().Before(v.expireAfter) { c.moveToHead(v) + if c.debug { log.Debugf("cache hit for %q (valid until %v)", id, v.expireAfter) } + return v.entries } @@ -103,6 +106,7 @@ func (c *Cache) getEntriesFromCacheLocked(id string) fs.Entries { if c.debug { log.Debugf("removing expired cache entry %q after %v", id, v.expireAfter) } + c.removeEntryLocked(v) } @@ -118,6 +122,7 @@ func (c *Cache) getEntries(ctx context.Context, id string, expirationTime time.D c.mu.Lock() defer c.mu.Unlock() + if entries := c.getEntriesFromCacheLocked(id); entries != nil { return entries, nil } @@ -125,6 +130,7 @@ func (c *Cache) getEntries(ctx context.Context, id string, expirationTime time.D if c.debug { log.Debugf("cache miss for %q", id) } + raw, err := cb(ctx) if err != nil { return nil, err @@ -140,6 +146,7 @@ func (c *Cache) getEntries(ctx context.Context, id string, expirationTime time.D entries: raw, expireAfter: time.Now().Add(expirationTime), } + c.addToHead(entry) c.data[id] = entry diff --git a/fs/cachefs/cache_test.go b/fs/cachefs/cache_test.go index 1d58131f9..5fae6869e 100644 --- a/fs/cachefs/cache_test.go +++ b/fs/cachefs/cache_test.go @@ -36,7 +36,9 @@ func (cs *cacheSource) get(id string) func(ctx context.Context) (fs.Entries, err func (cs *cacheSource) setEntryCount(id string, cnt int) { var fakeEntries fs.Entries + var fakeEntry fs.Entry + for i := 0; i < cnt; i++ { fakeEntries = append(fakeEntries, fakeEntry) } @@ -61,9 +63,11 @@ type cacheVerifier struct { func (cv *cacheVerifier) verifyCacheMiss(t *testing.T, id string) { actual := cv.cacheSource.callCounter[id] expected := cv.lastCallCounter[id] + 1 + if actual != expected { t.Errorf(errorPrefix()+"invalid call counter for %v, got %v, expected %v", id, actual, expected) } + cv.reset() } @@ -71,13 +75,15 @@ func (cv *cacheVerifier) verifyCacheHit(t *testing.T, id string) { if !reflect.DeepEqual(cv.lastCallCounter, cv.cacheSource.callCounter) { t.Errorf(errorPrefix()+" unexpected call counters for %v, got %v, expected %v", id, cv.cacheSource.callCounter, cv.lastCallCounter) } + cv.reset() } func (cv *cacheVerifier) verifyCacheOrdering(t *testing.T, expectedOrdering ...string) { var actualOrdering []string - var totalDirectoryEntries int - var totalDirectories int + + var totalDirectoryEntries, totalDirectories int + for e := cv.cache.head; e != nil; e = e.next { actualOrdering = append(actualOrdering, e.id) totalDirectoryEntries += len(e.entries) @@ -103,7 +109,6 @@ func (cv *cacheVerifier) verifyCacheOrdering(t *testing.T, expectedOrdering ...s if totalDirectoryEntries > cv.cache.maxDirectoryEntries { t.Errorf(errorPrefix()+"total directory entries exceeds limit: %v, expected %v", totalDirectoryEntries, cv.cache.maxDirectoryEntries) } - } func errorPrefix() string { @@ -146,6 +151,7 @@ func TestCache(t *testing.T) { MaxCachedDirectories: 4, MaxCachedEntries: 100, }) + if len(c.data) != 0 || c.totalDirectoryEntries != 0 || c.head != nil || c.tail != nil { t.Errorf("invalid initial state: %v %v %v %v", c.data, c.totalDirectoryEntries, c.head, c.tail) } @@ -159,6 +165,7 @@ func TestCache(t *testing.T) { id5 := "5" id6 := "6" id7 := "7" + cs.setEntryCount(id1, 3) cs.setEntryCount(id2, 3) cs.setEntryCount(id3, 3) @@ -241,7 +248,9 @@ func TestCacheGetEntriesLocking(t *testing.T) { cs := newCacheSource() cv := cacheVerifier{cacheSource: cs, cache: c} + const id1 = "1" + cs.setEntryCount(id1, 1) // fetch non-existing entry, the loader will return an error @@ -249,7 +258,9 @@ func TestCacheGetEntriesLocking(t *testing.T) { if err == nil { t.Fatal("Expected non-nil error when retrieving non-existing cache entry") } + const expectedEsLength = 0 + actualEsLength := len(actualEs) if actualEsLength != expectedEsLength { t.Fatal("Expected empty entries, got: ", actualEsLength) diff --git a/fs/cachefs/cachefs.go b/fs/cachefs/cachefs.go index cf699de04..3c2b1e03a 100644 --- a/fs/cachefs/cachefs.go +++ b/fs/cachefs/cachefs.go @@ -31,6 +31,7 @@ func (d *directory) Readdir(ctx context.Context) (fs.Entries, error) { for i, entry := range entries { wrapped[i] = wrapWithContext(entry, d.ctx) } + return wrapped, err } diff --git a/fs/ignorefs/ignorefs.go b/fs/ignorefs/ignorefs.go index ada0735de..128a78f41 100644 --- a/fs/ignorefs/ignorefs.go +++ b/fs/ignorefs/ignorefs.go @@ -46,6 +46,7 @@ func (c *ignoreContext) shouldIncludeByName(path string, e fs.Entry) bool { for _, oi := range c.onIgnore { oi(path, e) } + return false } } @@ -76,6 +77,7 @@ func (d *ignoreDirectory) Readdir(ctx context.Context) (fs.Entries, error) { } result := make(fs.Entries, 0, len(entries)) + for _, e := range entries { if !thisContext.shouldIncludeByName(d.relativePath+"/"+e.Name(), e) { continue @@ -145,6 +147,7 @@ func (c *ignoreContext) overrideFromPolicy(policy *FilesPolicy, dirPath string) if policy.NoParentDotIgnoreFiles { c.dotIgnoreFiles = nil } + if policy.NoParentIgnoreRules { c.matchers = nil } @@ -163,6 +166,7 @@ func (c *ignoreContext) overrideFromPolicy(policy *FilesPolicy, dirPath string) c.matchers = append(c.matchers, m) } + return nil } @@ -193,16 +197,21 @@ func (c *ignoreContext) loadDotIgnoreFiles(ctx context.Context, dirPath string, func combineAndDedupe(slices ...[]string) []string { var result []string + existing := map[string]bool{} + for _, slice := range slices { for _, it := range slice { if existing[it] { continue } + existing[it] = true + result = append(result, it) } } + return result } diff --git a/fs/ignorefs/ignorefs_test.go b/fs/ignorefs/ignorefs_test.go index 641d49567..d9a3fd989 100644 --- a/fs/ignorefs/ignorefs_test.go +++ b/fs/ignorefs/ignorefs_test.go @@ -33,6 +33,7 @@ func setupFilesystem() *mockfs.Directory { d1.AddFile("some-bin", dummyFileContents, 0) d2.AddFile("some-pkg", dummyFileContents, 0) + d4 := d3.AddDir("some-src", 0) d4.AddFile("f1", dummyFileContents, 0) @@ -231,19 +232,25 @@ func addAndSubtractFiles(original, added, removed []string) []string { for _, ri := range removed { m[ri] = true } + var result []string + for _, ai := range added { if !m[ai] { m[ai] = true + result = append(result, ai) } } + for _, oi := range original { if !m[oi] { result = append(result, oi) } } + sort.Strings(result) + return result } @@ -254,10 +261,12 @@ func walkTree(t *testing.T, dir fs.Directory) []string { walk = func(path string, d fs.Directory) error { output = append(output, path+"/") + entries, err := d.Readdir(context.Background()) if err != nil { return err } + for _, e := range entries { relPath := path + "/" + e.Name() @@ -269,6 +278,7 @@ func walkTree(t *testing.T, dir fs.Directory) []string { output = append(output, relPath) } } + return nil } diff --git a/fs/localfs/local_fs.go b/fs/localfs/local_fs.go index 3864b6966..b931b3c76 100644 --- a/fs/localfs/local_fs.go +++ b/fs/localfs/local_fs.go @@ -103,6 +103,7 @@ func (fsd *filesystemDirectory) Summary() *fs.DirectorySummary { func (fsd *filesystemDirectory) Readdir(ctx context.Context) (fs.Entries, error) { fullPath := fsd.fullPath() + f, direrr := os.Open(fullPath) if direrr != nil { return nil, direrr @@ -111,9 +112,13 @@ func (fsd *filesystemDirectory) Readdir(ctx context.Context) (fs.Entries, error) // start feeding directory entry names to namesCh namesCh := make(chan string, 200) + var namesErr error + var namesWG sync.WaitGroup + namesWG.Add(1) + go func() { defer namesWG.Done() defer close(namesCh) @@ -135,11 +140,13 @@ func (fsd *filesystemDirectory) Readdir(ctx context.Context) (fs.Entries, error) entriesCh := make(chan fs.Entry, 200) + var workersWG sync.WaitGroup + // launch N workers to os.Lstat() each name in parallel and push to entriesCh workers := 16 - var workersWG sync.WaitGroup for i := 0; i < workers; i++ { workersWG.Add(1) + go func() { defer workersWG.Done() @@ -186,6 +193,7 @@ func (f *fileWithMetadata) Entry() (fs.Entry, error) { if err != nil { return nil, err } + return &filesystemFile{newEntry(fi, filepath.Dir(f.Name()))}, nil } diff --git a/fs/localfs/local_fs_nonwindows.go b/fs/localfs/local_fs_nonwindows.go index 35084563a..b8ad56b31 100644 --- a/fs/localfs/local_fs_nonwindows.go +++ b/fs/localfs/local_fs_nonwindows.go @@ -15,5 +15,6 @@ func platformSpecificOwnerInfo(fi os.FileInfo) fs.OwnerInfo { oi.UserID = stat.Uid oi.GroupID = stat.Gid } + return oi } diff --git a/fs/localfs/local_fs_test.go b/fs/localfs/local_fs_test.go index 634ecc641..80e8913c0 100644 --- a/fs/localfs/local_fs_test.go +++ b/fs/localfs/local_fs_test.go @@ -13,9 +13,10 @@ "testing" ) -//nolint:gocyclo +//nolint:gocyclo,gocognit func TestFiles(t *testing.T) { ctx := context.Background() + var err error tmp, err := ioutil.TempDir("", "kopia") @@ -72,20 +73,26 @@ func TestFiles(t *testing.T) { if entries[0].Name() == "f1" && entries[0].Size() == 5 && entries[0].Mode().IsRegular() { goodCount++ } + if entries[1].Name() == "f2" && entries[1].Size() == 4 && entries[1].Mode().IsRegular() { goodCount++ } + if entries[2].Name() == "f3" && entries[2].Size() == 3 && entries[2].Mode().IsRegular() { goodCount++ } + if entries[3].Name() == "y" && entries[3].Size() == 0 && entries[3].Mode().IsDir() { goodCount++ } + if entries[4].Name() == "z" && entries[4].Size() == 0 && entries[4].Mode().IsDir() { goodCount++ } + if goodCount != 5 { t.Errorf("invalid dir data: %v good entries", goodCount) + for i, e := range entries { t.Logf("e[%v] = %v %v %v", i, e.Name(), e.Size(), e.Mode()) } @@ -94,6 +101,7 @@ func TestFiles(t *testing.T) { func assertNoError(t *testing.T, err error) { t.Helper() + if err != nil { t.Errorf("err: %v", err) } diff --git a/fs/loggingfs/loggingfs.go b/fs/loggingfs/loggingfs.go index f9296dd2b..fd3e5dac0 100644 --- a/fs/loggingfs/loggingfs.go +++ b/fs/loggingfs/loggingfs.go @@ -27,10 +27,12 @@ func (ld *loggingDirectory) Readdir(ctx context.Context) (fs.Entries, error) { entries, err := ld.Directory.Readdir(ctx) dt := time.Since(t0) ld.options.printf(ld.options.prefix+"Readdir(%v) took %v and returned %v items", ld.relativePath, dt, len(entries)) + loggingEntries := make(fs.Entries, len(entries)) for i, entry := range entries { loggingEntries[i] = wrapWithOptions(entry, ld.options, ld.relativePath+"/"+entry.Name()) } + return loggingEntries, err } @@ -72,9 +74,11 @@ func applyOptions(opts []Option) *loggingOptions { o := &loggingOptions{ printf: log.Debugf, } + for _, f := range opts { f(o) } + return o } diff --git a/internal/blobtesting/asserts.go b/internal/blobtesting/asserts.go index 435db6755..5646ad5af 100644 --- a/internal/blobtesting/asserts.go +++ b/internal/blobtesting/asserts.go @@ -86,6 +86,7 @@ func AssertGetBlobNotFound(ctx context.Context, t *testing.T, s blob.Storage, bl // AssertListResults asserts that the list results with given prefix return the specified list of names in order. func AssertListResults(ctx context.Context, t *testing.T, s blob.Storage, prefix blob.ID, want ...blob.ID) { t.Helper() + var names []blob.ID if err := s.ListBlobs(ctx, prefix, func(e blob.Metadata) error { @@ -108,5 +109,6 @@ func sorted(s []blob.ID) []blob.ID { sort.Slice(x, func(i, j int) bool { return x[i] < x[j] }) + return x } diff --git a/internal/blobtesting/faulty.go b/internal/blobtesting/faulty.go index e19cc0a0b..4778cfff0 100644 --- a/internal/blobtesting/faulty.go +++ b/internal/blobtesting/faulty.go @@ -33,6 +33,7 @@ func (s *FaultyStorage) GetBlob(ctx context.Context, id blob.ID, offset, length if err := s.getNextFault("GetBlob", id, offset, length); err != nil { return nil, err } + return s.Base.GetBlob(ctx, id, offset, length) } @@ -41,6 +42,7 @@ func (s *FaultyStorage) PutBlob(ctx context.Context, id blob.ID, data []byte) er if err := s.getNextFault("PutBlob", id, len(data)); err != nil { return err } + return s.Base.PutBlob(ctx, id, data) } @@ -49,6 +51,7 @@ func (s *FaultyStorage) DeleteBlob(ctx context.Context, id blob.ID) error { if err := s.getNextFault("DeleteBlob", id); err != nil { return err } + return s.Base.DeleteBlob(ctx, id) } @@ -71,6 +74,7 @@ func (s *FaultyStorage) Close(ctx context.Context) error { if err := s.getNextFault("Close"); err != nil { return err } + return s.Base.Close(ctx) } @@ -81,10 +85,12 @@ func (s *FaultyStorage) ConnectionInfo() blob.ConnectionInfo { func (s *FaultyStorage) getNextFault(method string, args ...interface{}) error { s.mu.Lock() + faults := s.Faults[method] if len(faults) == 0 { s.mu.Unlock() log.Debugf("no faults for %v %v", method, args) + return nil } @@ -95,21 +101,28 @@ func (s *FaultyStorage) getNextFault(method string, args ...interface{}) error { } else { s.Faults[method] = faults[1:] } + s.mu.Unlock() + if f.WaitFor != nil { log.Debugf("waiting for channel to be closed in %v %v", method, args) <-f.WaitFor } + if f.Sleep > 0 { log.Debugf("sleeping for %v in %v %v", f.Sleep, method, args) time.Sleep(f.Sleep) } + if f.ErrCallback != nil { err := f.ErrCallback() log.Debugf("returning %v for %v %v", err, method, args) + return err } + log.Debugf("returning %v for %v %v", f.Err, method, args) + return f.Err } diff --git a/internal/blobtesting/map.go b/internal/blobtesting/map.go index 664de9bdf..0d8813762 100644 --- a/internal/blobtesting/map.go +++ b/internal/blobtesting/map.go @@ -27,6 +27,7 @@ func (s *mapStorage) GetBlob(ctx context.Context, id blob.ID, offset, length int data, ok := s.data[id] if ok { data = append([]byte(nil), data...) + if length < 0 { return data, nil } @@ -39,6 +40,7 @@ func (s *mapStorage) GetBlob(ctx context.Context, id blob.ID, offset, length int if int(length) > len(data) { return nil, errors.New("invalid length") } + return data[0:length], nil } @@ -54,7 +56,9 @@ func (s *mapStorage) PutBlob(ctx context.Context, id blob.ID, data []byte) error } s.keyTime[id] = s.timeNow() + s.data[id] = append([]byte{}, data...) + return nil } @@ -64,6 +68,7 @@ func (s *mapStorage) DeleteBlob(ctx context.Context, id blob.ID) error { delete(s.data, id) delete(s.keyTime, id) + return nil } @@ -71,11 +76,13 @@ func (s *mapStorage) ListBlobs(ctx context.Context, prefix blob.ID, callback fun s.mutex.RLock() keys := []blob.ID{} + for k := range s.data { if strings.HasPrefix(string(k), string(prefix)) { keys = append(keys, k) } } + s.mutex.RUnlock() sort.Slice(keys, func(i, j int) bool { @@ -87,9 +94,11 @@ func (s *mapStorage) ListBlobs(ctx context.Context, prefix blob.ID, callback fun v, ok := s.data[k] ts := s.keyTime[k] s.mutex.RUnlock() + if !ok { continue } + if err := callback(blob.Metadata{ BlobID: k, Length: int64(len(v)), @@ -98,6 +107,7 @@ func (s *mapStorage) ListBlobs(ctx context.Context, prefix blob.ID, callback fun return err } } + return nil } @@ -130,8 +140,10 @@ func NewMapStorage(data DataMap, keyTime map[blob.ID]time.Time, timeNow func() t if keyTime == nil { keyTime = make(map[blob.ID]time.Time) } + if timeNow == nil { timeNow = time.Now } + return &mapStorage{data: data, keyTime: keyTime, timeNow: timeNow} } diff --git a/internal/blobtesting/map_test.go b/internal/blobtesting/map_test.go index 2e0cd88ca..b1258a8b4 100644 --- a/internal/blobtesting/map_test.go +++ b/internal/blobtesting/map_test.go @@ -7,9 +7,11 @@ func TestMapStorage(t *testing.T) { data := DataMap{} + r := NewMapStorage(data, nil, nil) if r == nil { t.Errorf("unexpected result: %v", r) } + VerifyStorage(context.Background(), t, r) } diff --git a/internal/blobtesting/verify.go b/internal/blobtesting/verify.go index 7a427ab33..0db31a0c3 100644 --- a/internal/blobtesting/verify.go +++ b/internal/blobtesting/verify.go @@ -55,9 +55,11 @@ func VerifyStorage(ctx context.Context, t *testing.T, r blob.Storage) { if err := r.DeleteBlob(ctx, blocks[0].blk); err != nil { t.Errorf("unable to delete block: %v", err) } + if err := r.DeleteBlob(ctx, blocks[0].blk); err != nil { t.Errorf("invalid error when deleting deleted block: %v", err) } + AssertListResults(ctx, t, r, "ab", blocks[2].blk, blocks[3].blk) AssertListResults(ctx, t, r, "", blocks[1].blk, blocks[2].blk, blocks[3].blk, blocks[4].blk) } @@ -68,6 +70,7 @@ func AssertConnectionInfoRoundTrips(ctx context.Context, t *testing.T, s blob.St t.Helper() ci := s.ConnectionInfo() + s2, err := blob.NewStorage(ctx, ci) if err != nil { t.Fatalf("err: %v", err) diff --git a/internal/diff/diff.go b/internal/diff/diff.go index e271f1237..a7078b441 100644 --- a/internal/diff/diff.go +++ b/internal/diff/diff.go @@ -41,7 +41,9 @@ func (c *Comparer) Close() error { func (c *Comparer) compareDirectories(ctx context.Context, dir1, dir2 fs.Directory, parent string) error { log.Debugf("comparing directories %v", parent) + var entries1, entries2 fs.Entries + var err error if dir1 != nil { @@ -61,7 +63,7 @@ func (c *Comparer) compareDirectories(ctx context.Context, dir1, dir2 fs.Directo return c.compareDirectoryEntries(ctx, entries1, entries2, parent) } -// nolint:gocyclo +// nolint:gocyclo,gocognit func (c *Comparer) compareEntry(ctx context.Context, e1, e2 fs.Entry, path string) error { // see if we have the same object IDs, which implies identical objects, thanks to content-addressable-storage if h1, ok := e1.(object.HasObjectID); ok { @@ -80,11 +82,13 @@ func (c *Comparer) compareEntry(ctx context.Context, e1, e2 fs.Entry, path strin } c.output("added file %v (%v bytes)\n", path, e2.Size()) + if f, ok := e2.(fs.File); ok { if err := c.compareFiles(ctx, nil, f, path); err != nil { return err } } + return nil } @@ -95,16 +99,19 @@ func (c *Comparer) compareEntry(ctx context.Context, e1, e2 fs.Entry, path strin } c.output("removed file %v (%v bytes)\n", path, e1.Size()) + if f, ok := e1.(fs.File); ok { if err := c.compareFiles(ctx, f, nil, path); err != nil { return err } } + return nil } dir1, isDir1 := e1.(fs.Directory) dir2, isDir2 := e2.(fs.Directory) + if isDir1 { if !isDir2 { // right is a non-directory, left is a directory @@ -122,6 +129,7 @@ func (c *Comparer) compareEntry(ctx context.Context, e1, e2 fs.Entry, path strin } c.output("changed %v at %v (size %v -> %v)\n", path, e2.ModTime().String(), e1.Size(), e2.Size()) + if f1, ok := e1.(fs.File); ok { if f2, ok := e2.(fs.File); ok { if err := c.compareFiles(ctx, f1, f2, path); err != nil { @@ -144,6 +152,7 @@ func (c *Comparer) compareDirectoryEntries(ctx context.Context, entries1, entrie if err := c.compareEntry(ctx, e1byname[entryName], e2, dirPath+"/"+entryName); err != nil { return errors.Wrapf(err, "error comparing %v", entryName) } + delete(e1byname, entryName) } @@ -198,6 +207,7 @@ func (c *Comparer) compareFiles(ctx context.Context, f1, f2 fs.File, fname strin cmd.Stdout = c.out cmd.Stderr = c.out cmd.Run() //nolint:errcheck + return nil } @@ -219,6 +229,7 @@ func (c *Comparer) downloadFile(ctx context.Context, f fs.File, fname string) er defer dst.Close() //nolint:errcheck _, err = io.Copy(dst, src) + return err } diff --git a/internal/editor/editor.go b/internal/editor/editor.go index 8b4b29e42..12d131770 100644 --- a/internal/editor/editor.go +++ b/internal/editor/editor.go @@ -25,6 +25,7 @@ func EditLoop(fname, initial string, parse func(updated string) error) error { if err != nil { return err } + tmpFile := filepath.Join(tmpDir, fname) defer os.RemoveAll(tmpDir) //nolint:errcheck @@ -51,7 +52,9 @@ func EditLoop(fname, initial string, parse func(updated string) error) error { fmt.Print("Reopen editor to fix? (Y/n) ") var shouldReopen string + fmt.Scanf("%s", &shouldReopen) //nolint:errcheck + if strings.HasPrefix(strings.ToLower(shouldReopen), "n") { return errors.New("aborted") } @@ -71,10 +74,12 @@ func readAndStripComments(fname string) (string, error) { for s.Scan() { l := s.Text() l = strings.TrimSpace(strings.Split(l, "#")[0]) + if l != "" { result = append(result, l) } } + return strings.Join(result, "\n"), nil } @@ -91,6 +96,7 @@ func editFile(file string) error { cmd.Stdout = os.Stdout log.Debugf("launching editor %q on file %q", editor, file) + err := cmd.Run() if err != nil { log.Errorf("unable to launch editor: %v", err) @@ -129,5 +135,6 @@ func parseEditor(s string) (cmd string, args []string) { } parts := strings.Split(s, " ") + return parts[0], parts[1:] } diff --git a/internal/fusemount/fusefs.go b/internal/fusemount/fusefs.go index 8a901e757..c7674c552 100644 --- a/internal/fusemount/fusefs.go +++ b/internal/fusemount/fusefs.go @@ -30,6 +30,7 @@ func (n *fuseNode) Attr(ctx context.Context, a *fuse.Attr) error { a.Mtime = m.ModTime() a.Uid = m.Owner().UserID a.Gid = m.Owner().GroupID + return nil } diff --git a/internal/ignore/ignore.go b/internal/ignore/ignore.go index 668e8ec26..a609f655d 100644 --- a/internal/ignore/ignore.go +++ b/internal/ignore/ignore.go @@ -19,8 +19,7 @@ func ParseGitIgnore(baseDir, pattern string) (Matcher, error) { baseDir += "/" } - var dirOnly bool - var negate bool + var dirOnly, negate bool // Trailing spaces are ignored unless they are quoted with backslash ("\"). if !strings.HasSuffix(pattern, "\\ ") { @@ -119,6 +118,7 @@ func parseNonGlobPattern(pattern string) (nameMatcher, error) { if strings.HasPrefix(pattern, "**/") { suffix := strings.TrimPrefix(pattern, "**/") suffixWithSlash := strings.TrimPrefix(pattern, "**") + return func(path string) bool { return path == suffix || strings.HasSuffix(path, suffixWithSlash) }, nil @@ -129,6 +129,7 @@ func parseNonGlobPattern(pattern string) (nameMatcher, error) { if strings.HasSuffix(pattern, "/**") { prefix := strings.TrimSuffix(pattern, "/**") prefixWithSlash := strings.TrimSuffix(pattern, "**") + return func(path string) bool { return path == prefix || strings.HasPrefix(path, prefixWithSlash) }, nil diff --git a/internal/logfile/logfile.go b/internal/logfile/logfile.go index e81aa4123..cf71e684d 100644 --- a/internal/logfile/logfile.go +++ b/internal/logfile/logfile.go @@ -45,11 +45,12 @@ // Initialize is invoked as part of command execution to create log file just before it's needed. func Initialize(ctx *kingpin.ParseContext) error { var logBackends []logging.Backend - var logFileName string - var symlinkName string + + var logFileName, symlinkName string if lfn := *logFile; lfn != "" { var err error + logFileName, err = filepath.Abs(lfn) if err != nil { return err @@ -57,10 +58,12 @@ func Initialize(ctx *kingpin.ParseContext) error { } var shouldSweepLogs bool + if logFileName == "" && *logDir != "" { logBaseName := fmt.Sprintf("%v%v-%v%v", logFileNamePrefix, time.Now().Format("20060102-150405"), os.Getpid(), logFileNameSuffix) logFileName = filepath.Join(*logDir, logBaseName) symlinkName = "kopia.latest.log" + if *logDirMaxAge > 0 || *logDirMaxFiles > 0 { shouldSweepLogs = true } @@ -69,6 +72,7 @@ func Initialize(ctx *kingpin.ParseContext) error { if logFileName != "" { logFileDir := filepath.Dir(logFileName) logFileBaseName := filepath.Base(logFileName) + if err := os.MkdirAll(logFileDir, 0700); err != nil { fmt.Fprintln(os.Stderr, "Unable to create logs directory:", err) // nolint:errcheck } @@ -106,6 +110,7 @@ func sweepLogDir(dirname string, maxCount int, maxAge time.Duration) { if maxAge > 0 { timeCutoff = time.Now().Add(-maxAge) } + if maxCount == 0 { maxCount = math.MaxInt32 } @@ -132,6 +137,7 @@ func sweepLogDir(dirname string, maxCount int, maxAge time.Duration) { } cnt++ + if cnt > maxCount || e.ModTime().Before(timeCutoff) { if err = os.Remove(filepath.Join(dirname, e.Name())); err != nil { log.Warningf("unable to remove log file: %v", err) diff --git a/internal/mockfs/mockfs.go b/internal/mockfs/mockfs.go index 4e94ca70a..504424c5a 100644 --- a/internal/mockfs/mockfs.go +++ b/internal/mockfs/mockfs.go @@ -121,6 +121,7 @@ func (imd *Directory) addChild(e fs.Entry) { if strings.Contains(e.Name(), "/") { panic("child name cannot contain '/'") } + imd.children = append(imd.children, e) imd.children.Sort() } @@ -130,23 +131,27 @@ func (imd *Directory) resolveSubdir(name string) (parent *Directory, leaf string for _, n := range parts[0 : len(parts)-1] { imd = imd.Subdir(n) } + return imd, parts[len(parts)-1] } // Subdir finds a subdirectory with a given name. func (imd *Directory) Subdir(name ...string) *Directory { i := imd + for _, n := range name { i2 := i.children.FindByName(n) if i2 == nil { panic(fmt.Sprintf("'%s' not found in '%s'", n, i.Name())) } + if !i2.IsDir() { panic(fmt.Sprintf("'%s' is not a directory in '%s'", n, i.Name())) } i = i2.(*Directory) } + return i } diff --git a/internal/ospath/ospath_xdg.go b/internal/ospath/ospath_xdg.go index 727edf520..237765339 100644 --- a/internal/ospath/ospath_xdg.go +++ b/internal/ospath/ospath_xdg.go @@ -13,6 +13,7 @@ func init() { } else { userSettingsDir = filepath.Join(os.Getenv("HOME"), ".config") } + if os.Getenv("XDG_CACHE_HOME") != "" { userLogsDir = os.Getenv("XDG_CACHE_HOME") } else { diff --git a/internal/parallelwork/parallel_work_queue.go b/internal/parallelwork/parallel_work_queue.go index cd6fc8873..09b2d2ebe 100644 --- a/internal/parallelwork/parallel_work_queue.go +++ b/internal/parallelwork/parallel_work_queue.go @@ -45,6 +45,7 @@ func (v *Queue) enqueue(front bool, callback CallbackFunc) { } else { v.queueItems.PushBack(callback) } + v.maybeReportProgress() v.monitor.Signal() } @@ -53,6 +54,7 @@ func (v *Queue) enqueue(front bool, callback CallbackFunc) { // is empty and all workers are idle. func (v *Queue) Process(workers int) error { var wg sync.WaitGroup + errors := make(chan error, workers) for i := 0; i < workers; i++ { @@ -102,6 +104,7 @@ func (v *Queue) dequeue() CallbackFunc { front := v.queueItems.Front() v.queueItems.Remove(front) + return front.Value.(CallbackFunc) } @@ -125,6 +128,7 @@ func (v *Queue) maybeReportProgress() { if time.Now().Before(v.nextReportTime) { return } + v.nextReportTime = time.Now().Add(1 * time.Second) cb(v.enqueuedWork, v.activeWorkerCount, v.completedWork) diff --git a/internal/repotesting/repotesting.go b/internal/repotesting/repotesting.go index 64c9a38aa..465aaf463 100644 --- a/internal/repotesting/repotesting.go +++ b/internal/repotesting/repotesting.go @@ -36,6 +36,7 @@ func (e *Environment) Setup(t *testing.T, opts ...func(*repo.NewRepositoryOption if err != nil { t.Fatalf("err: %v", err) } + e.storageDir, err = ioutil.TempDir("", "") if err != nil { t.Fatalf("err: %v", err) @@ -90,15 +91,18 @@ func (e *Environment) Close(t *testing.T) { if err := e.Repository.Close(context.Background()); err != nil { t.Fatalf("unable to close: %v", err) } + if e.connected { if err := repo.Disconnect(e.configFile()); err != nil { t.Errorf("error disconnecting: %v", err) } } + if err := os.Remove(e.configDir); err != nil { // should be empty, assuming Disconnect was successful t.Errorf("error removing config directory: %v", err) } + if err := os.RemoveAll(e.storageDir); err != nil { t.Errorf("error removing storage directory: %v", err) } diff --git a/internal/retry/retry.go b/internal/retry/retry.go index 30018581f..fb3fb6921 100644 --- a/internal/retry/retry.go +++ b/internal/retry/retry.go @@ -28,14 +28,17 @@ type AttemptFunc func() (interface{ // a certain limit. func WithExponentialBackoff(desc string, attempt AttemptFunc, isRetriableError IsRetriableFunc) (interface{}, error) { sleepAmount := retryInitialSleepAmount + for i := 0; i < maxAttempts; i++ { v, err := attempt() if !isRetriableError(err) { return v, err } + log.Debugf("got error %v when %v (#%v), sleeping for %v before retrying", err, desc, i, sleepAmount) time.Sleep(sleepAmount) sleepAmount *= 2 + if sleepAmount > retryMaxSleepAmount { sleepAmount = retryMaxSleepAmount } @@ -50,5 +53,6 @@ func WithExponentialBackoffNoValue(desc string, attempt func() error, isRetriabl _, err := WithExponentialBackoff(desc, func() (interface{}, error) { return nil, attempt() }, isRetriableError) + return err } diff --git a/internal/scrubber/scrub_sensitive.go b/internal/scrubber/scrub_sensitive.go index f1f6795b4..ba0f9f33d 100644 --- a/internal/scrubber/scrub_sensitive.go +++ b/internal/scrubber/scrub_sensitive.go @@ -14,6 +14,7 @@ func ScrubSensitiveData(v reflect.Value) reflect.Value { case reflect.Struct: res := reflect.New(v.Type()).Elem() + for i := 0; i < v.NumField(); i++ { fv := v.Field(i) @@ -27,6 +28,7 @@ func ScrubSensitiveData(v reflect.Value) reflect.Value { res.Field(i).Set(fv) } } + return res default: diff --git a/internal/server/api_policy_list.go b/internal/server/api_policy_list.go index 9cf70c75c..a7944a2af 100644 --- a/internal/server/api_policy_list.go +++ b/internal/server/api_policy_list.go @@ -23,6 +23,7 @@ func (s *Server) handlePolicyList(ctx context.Context, r *http.Request) (interfa if !sourceMatchesURLFilter(target, r.URL.Query()) { continue } + resp.Policies = append(resp.Policies, &serverapi.PolicyListEntry{ ID: pol.ID(), Target: target, diff --git a/internal/server/api_snapshot_list.go b/internal/server/api_snapshot_list.go index edfd33ed9..96859d493 100644 --- a/internal/server/api_snapshot_list.go +++ b/internal/server/api_snapshot_list.go @@ -42,6 +42,7 @@ func (s *Server) handleSourceSnapshotList(ctx context.Context, r *http.Request) resp := &snapshotListResponse{ Snapshots: []*snapshotListEntry{}, } + groups := snapshot.GroupBySource(manifests) for _, grp := range groups { first := grp[0] @@ -66,9 +67,11 @@ func sourceMatchesURLFilter(src snapshot.SourceInfo, query url.Values) bool { if v := query.Get("host"); v != "" && src.Host != v { return false } + if v := query.Get("userName"); v != "" && src.UserName != v { return false } + if v := query.Get("path"); v != "" && src.Path != v { return false } diff --git a/internal/server/api_sources_list.go b/internal/server/api_sources_list.go index a0325b365..c115460f6 100644 --- a/internal/server/api_sources_list.go +++ b/internal/server/api_sources_list.go @@ -17,6 +17,7 @@ func (s *Server) handleSourcesList(ctx context.Context, r *http.Request) (interf if !sourceMatchesURLFilter(v.src, r.URL.Query()) { continue } + resp.Sources = append(resp.Sources, v.Status()) } diff --git a/internal/server/server.go b/internal/server/server.go index 4e48e8913..4165369a5 100644 --- a/internal/server/server.go +++ b/internal/server/server.go @@ -30,6 +30,7 @@ type Server struct { // APIHandlers handles API requests. func (s *Server) APIHandlers() http.Handler { mux := http.NewServeMux() + mux.HandleFunc("/api/v1/status", s.handleAPI(s.handleStatus, "GET")) mux.HandleFunc("/api/v1/sources", s.handleAPI(s.handleSourcesList, "GET")) mux.HandleFunc("/api/v1/snapshots", s.handleAPI(s.handleSourceSnapshotList, "GET")) @@ -40,6 +41,7 @@ func (s *Server) APIHandlers() http.Handler { mux.HandleFunc("/api/v1/sources/resume", s.handleAPI(s.handleResume, "POST")) mux.HandleFunc("/api/v1/sources/upload", s.handleAPI(s.handleUpload, "POST")) mux.HandleFunc("/api/v1/sources/cancel", s.handleAPI(s.handleCancel, "POST")) + return mux } @@ -89,6 +91,7 @@ func (s *Server) forAllSourceManagersMatchingURLFilter(c func(s *sourceManager) if !sourceMatchesURLFilter(src, values) { continue } + resp.Sources[src.String()] = c(mgr) } @@ -114,6 +117,7 @@ func (s *Server) handleCancel(ctx context.Context, r *http.Request) (interface{} func (s *Server) beginUpload(src snapshot.SourceInfo) { log.Infof("waiting on semaphore to upload %v", src) s.uploadSemaphore <- struct{}{} + log.Infof("entered semaphore to upload %v", src) } diff --git a/internal/server/source_manager.go b/internal/server/source_manager.go index 3b20b9469..ebd1a03f3 100644 --- a/internal/server/source_manager.go +++ b/internal/server/source_manager.go @@ -54,8 +54,8 @@ func (s *sourceManager) Status() *serverapi.SourceStatus { st.UploadStatus.UploadingPath = s.uploadPath st.UploadStatus.UploadingPathCompleted = s.uploadPathCompleted st.UploadStatus.UploadingPathTotal = s.uploadPathTotal - return st + return st } func (s *sourceManager) setStatus(stat string) { @@ -77,8 +77,10 @@ func (s *sourceManager) run(ctx context.Context) { func (s *sourceManager) runLocal(ctx context.Context) { s.refreshStatus(ctx) + for { var timeBeforeNextSnapshot time.Duration + if !s.nextSnapshotTime.IsZero() { timeBeforeNextSnapshot = time.Until(s.nextSnapshotTime) log.Infof("time to next snapshot %v is %v", s.src, timeBeforeNextSnapshot) @@ -106,6 +108,7 @@ func (s *sourceManager) runLocal(ctx context.Context) { func (s *sourceManager) runRemote(ctx context.Context) { s.refreshStatus(ctx) s.setStatus("REMOTE") + for { select { case <-s.closed: @@ -164,15 +167,19 @@ func (s *sourceManager) snapshot(ctx context.Context) { log.Errorf("unable to create local filesystem: %v", err) return } + u := snapshotfs.NewUploader(s.server.rep) + polGetter, err := policy.FilesPolicyGetter(ctx, s.server.rep, s.src) if err != nil { log.Errorf("unable to create policy getter: %v", err) } + u.FilesPolicy = polGetter u.Progress = s log.Infof("starting upload of %v", s.src) + manifest, err := u.Upload(ctx, localEntry, s.src, s.lastCompleteSnapshot, s.lastSnapshot) if err != nil { log.Errorf("upload error: %v", err) @@ -191,6 +198,7 @@ func (s *sourceManager) snapshot(ctx context.Context) { } log.Infof("created snapshot %v", snapshotID) + if err := s.server.rep.Flush(ctx); err != nil { log.Errorf("unable to flush: %v", err) return @@ -199,11 +207,13 @@ func (s *sourceManager) snapshot(ctx context.Context) { func (s *sourceManager) findClosestNextSnapshotTime() time.Time { nextSnapshotTime := time.Now().Add(24 * time.Hour) + if s.pol != nil { // compute next snapshot time based on interval if interval := s.pol.SchedulingPolicy.IntervalSeconds; interval != 0 { interval := time.Duration(interval) * time.Second nt := s.lastSnapshot.StartTime.Add(interval).Truncate(interval) + if nt.Before(nextSnapshotTime) { nextSnapshotTime = nt } @@ -212,9 +222,11 @@ func (s *sourceManager) findClosestNextSnapshotTime() time.Time { for _, tod := range s.pol.SchedulingPolicy.TimesOfDay { nowLocalTime := time.Now().Local() localSnapshotTime := time.Date(nowLocalTime.Year(), nowLocalTime.Month(), nowLocalTime.Day(), tod.Hour, tod.Minute, 0, 0, time.Local) + if tod.Hour < nowLocalTime.Hour() || (tod.Hour == nowLocalTime.Hour() && tod.Minute < nowLocalTime.Minute()) { localSnapshotTime = localSnapshotTime.Add(24 * time.Hour) } + if localSnapshotTime.Before(nextSnapshotTime) { nextSnapshotTime = localSnapshotTime } @@ -226,6 +238,7 @@ func (s *sourceManager) findClosestNextSnapshotTime() time.Time { func (s *sourceManager) refreshStatus(ctx context.Context) { log.Debugf("refreshing state for %v", s.src) + pol, _, err := policy.GetEffectivePolicy(ctx, s.server.rep, s.src) if err != nil { s.setStatus("FAILED") @@ -233,6 +246,7 @@ func (s *sourceManager) refreshStatus(ctx context.Context) { } s.pol = pol + snapshots, err := snapshot.ListSnapshots(ctx, s.server.rep, s.src) if err != nil { s.setStatus("FAILED") @@ -240,6 +254,7 @@ func (s *sourceManager) refreshStatus(ctx context.Context) { } s.lastCompleteSnapshot = nil + snaps := snapshot.SortByTime(snapshots, true) if len(snaps) > 0 { s.lastSnapshot = snaps[0] diff --git a/internal/serverapi/client.go b/internal/serverapi/client.go index b0da222dd..4b3392a7d 100644 --- a/internal/serverapi/client.go +++ b/internal/serverapi/client.go @@ -25,6 +25,7 @@ func (c *Client) Get(path string, respPayload interface{}) error { if resp.StatusCode != 200 { return errors.Errorf("invalid server response: %v", resp.Status) } + if err := json.NewDecoder(resp.Body).Decode(respPayload); err != nil { return errors.Wrap(err, "malformed server response") } @@ -49,6 +50,7 @@ func (c *Client) Post(path string, reqPayload, respPayload interface{}) error { if resp.StatusCode != 200 { return errors.Errorf("invalid server response: %v", resp.Status) } + if err := json.NewDecoder(resp.Body).Decode(respPayload); err != nil { return errors.Wrap(err, "malformed server response") } @@ -61,5 +63,6 @@ func NewClient(serverAddress string, cli *http.Client) *Client { if cli == nil { cli = http.DefaultClient } + return &Client{"http://" + serverAddress + "/api/v1/", cli} } diff --git a/internal/throttle/round_tripper.go b/internal/throttle/round_tripper.go index 53e07d8c6..80a5c508b 100644 --- a/internal/throttle/round_tripper.go +++ b/internal/throttle/round_tripper.go @@ -18,15 +18,18 @@ type throttlingRoundTripper struct { func (rt *throttlingRoundTripper) RoundTrip(req *http.Request) (*http.Response, error) { if req.Body != nil && rt.uploadPool != nil { var err error + req.Body, err = rt.uploadPool.AddReader(req.Body) if err != nil { return nil, err } } + resp, err := rt.base.RoundTrip(req) if resp != nil && resp.Body != nil && rt.downloadPool != nil { resp.Body, err = rt.downloadPool.AddReader(resp.Body) } + return resp, err } diff --git a/internal/throttle/round_tripper_test.go b/internal/throttle/round_tripper_test.go index c082791e4..b521747b4 100644 --- a/internal/throttle/round_tripper_test.go +++ b/internal/throttle/round_tripper_test.go @@ -41,7 +41,7 @@ func (fp *fakePool) AddReader(r io.ReadCloser) (io.ReadCloser, error) { return r, nil } -//nolint:gocyclo +//nolint:gocyclo,gocognit func TestRoundTripper(t *testing.T) { downloadBody := ioutil.NopCloser(bytes.NewReader([]byte("data1"))) uploadBody := ioutil.NopCloser(bytes.NewReader([]byte("data1"))) @@ -56,11 +56,14 @@ func TestRoundTripper(t *testing.T) { // Empty request (no request, no response) uploadPool.reset() downloadPool.reset() + req1, resp1 := base.add(&http.Request{}, &http.Response{}) //nolint:bodyclose resp, err := rt.RoundTrip(req1) //nolint:bodyclose + if resp != resp1 || err != nil { t.Errorf("invalid response or error: %v", err) } + if len(downloadPool.readers) != 0 || len(uploadPool.readers) != 0 { t.Errorf("invalid pool contents: %v %v", downloadPool.readers, uploadPool.readers) } @@ -68,13 +71,16 @@ func TestRoundTripper(t *testing.T) { // Upload request uploadPool.reset() downloadPool.reset() + req2, resp2 := base.add(&http.Request{ //nolint:bodyclose Body: uploadBody, }, &http.Response{}) resp, err = rt.RoundTrip(req2) //nolint:bodyclose + if resp != resp2 || err != nil { t.Errorf("invalid response or error: %v", err) } + if len(downloadPool.readers) != 0 || len(uploadPool.readers) != 1 { t.Errorf("invalid pool contents: %v %v", downloadPool.readers, uploadPool.readers) } @@ -82,11 +88,14 @@ func TestRoundTripper(t *testing.T) { // Download request uploadPool.reset() downloadPool.reset() + req3, resp3 := base.add(&http.Request{}, &http.Response{Body: downloadBody}) //nolint:bodyclose resp, err = rt.RoundTrip(req3) //nolint:bodyclose + if resp != resp3 || err != nil { t.Errorf("invalid response or error: %v", err) } + if len(downloadPool.readers) != 1 || len(uploadPool.readers) != 0 { t.Errorf("invalid pool contents: %v %v", downloadPool.readers, uploadPool.readers) } @@ -94,11 +103,14 @@ func TestRoundTripper(t *testing.T) { // Upload/Download request uploadPool.reset() downloadPool.reset() + req4, resp4 := base.add(&http.Request{Body: uploadBody}, &http.Response{Body: downloadBody}) //nolint:bodyclose - resp, err = rt.RoundTrip(req4) //nolint:bodyclose + + resp, err = rt.RoundTrip(req4) //nolint:bodyclose if resp != resp4 || err != nil { t.Errorf("invalid response or error: %v", err) } + if len(downloadPool.readers) != 1 || len(uploadPool.readers) != 1 { t.Errorf("invalid pool contents: %v %v", downloadPool.readers, uploadPool.readers) } diff --git a/internal/units/units.go b/internal/units/units.go index 2f6590018..40472d26c 100644 --- a/internal/units/units.go +++ b/internal/units/units.go @@ -17,6 +17,7 @@ func toDecimalUnitString(f, thousand float64, prefixes []string, suffix string) if f < 0.9*thousand { return fmt.Sprintf("%v %v%v", niceNumber(f), prefixes[i], suffix) } + f /= thousand } diff --git a/internal/webdavmount/webdavmount.go b/internal/webdavmount/webdavmount.go index 750af1315..04d650ace 100644 --- a/internal/webdavmount/webdavmount.go +++ b/internal/webdavmount/webdavmount.go @@ -38,11 +38,13 @@ func (f *webdavFile) Stat() (os.FileInfo, error) { func (f *webdavFile) getReader() (fs.Reader, error) { f.mu.Lock() defer f.mu.Unlock() + if f.r == nil { r, err := f.entry.Open(f.ctx) if err != nil { return nil, err } + f.r = r } @@ -54,6 +56,7 @@ func (f *webdavFile) Read(b []byte) (int, error) { if err != nil { return 0, err } + return r.Read(b) } @@ -94,6 +97,7 @@ func (d *webdavDir) Readdir(n int) ([]os.FileInfo, error) { if err != nil { return nil, err } + if n > 0 && n < len(entries) { entries = entries[0:n] } @@ -102,6 +106,7 @@ func (d *webdavDir) Readdir(n int) ([]os.FileInfo, error) { for _, e := range entries { fis = append(fis, &webdavFileInfo{e}) } + return fis, nil } @@ -173,7 +178,9 @@ func (w *webdavFS) Stat(ctx context.Context, path string) (os.FileInfo, error) { func (w *webdavFS) findEntry(ctx context.Context, path string) (fs.Entry, error) { parts := removeEmpty(strings.Split(path, "/")) + var e fs.Entry = w.dir + for i, p := range parts { d, ok := e.(fs.Directory) if !ok { @@ -196,10 +203,12 @@ func (w *webdavFS) findEntry(ctx context.Context, path string) (fs.Entry, error) func removeEmpty(s []string) []string { result := s[:0] + for _, e := range s { if e == "" { continue } + result = append(result, e) } diff --git a/repo/blob/config.go b/repo/blob/config.go index 5361e0676..ed1b09bbc 100644 --- a/repo/blob/config.go +++ b/repo/blob/config.go @@ -24,10 +24,12 @@ func (c *ConnectionInfo) UnmarshalJSON(b []byte) error { } c.Type = raw.Type + f := factories[raw.Type] if f == nil { return errors.Errorf("storage type '%v' not registered", raw.Type) } + c.Config = f.defaultConfigFunc() if err := json.Unmarshal(raw.Data, c.Config); err != nil { return errors.Wrap(err, "unable to unmarshal config") diff --git a/repo/blob/filesystem/filesystem_storage.go b/repo/blob/filesystem/filesystem_storage.go index e883b0b77..51cfc57ce 100644 --- a/repo/blob/filesystem/filesystem_storage.go +++ b/repo/blob/filesystem/filesystem_storage.go @@ -56,13 +56,16 @@ func (fs *fsImpl) GetBlobFromPath(ctx context.Context, dirPath, path string, off if _, err = f.Seek(offset, io.SeekStart); err != nil { return nil, err } + b, err := ioutil.ReadAll(io.LimitReader(f, length)) if err != nil { return nil, err } + if int64(len(b)) != length { return nil, errors.Errorf("invalid length") } + return b, nil } @@ -71,7 +74,9 @@ func (fs *fsImpl) PutBlobInPath(ctx context.Context, dirPath, path string, data if _, err := rand.Read(randSuffix); err != nil { return errors.Wrap(err, "can't get random bytes") } + tempFile := fmt.Sprintf("%s.tmp.%x", path, randSuffix) + f, err := fs.createTempFileAndDir(tempFile) if err != nil { return errors.Wrap(err, "cannot create temporary file") @@ -80,6 +85,7 @@ func (fs *fsImpl) PutBlobInPath(ctx context.Context, dirPath, path string, data if _, err = f.Write(data); err != nil { return errors.Wrap(err, "can't write temporary file") } + if err = f.Close(); err != nil { return errors.Wrap(err, "can't close temporary file") } @@ -89,6 +95,7 @@ func (fs *fsImpl) PutBlobInPath(ctx context.Context, dirPath, path string, data if removeErr := os.Remove(tempFile); removeErr != nil { log.Warningf("can't remove temp file: %v", removeErr) } + return err } @@ -103,11 +110,13 @@ func (fs *fsImpl) PutBlobInPath(ctx context.Context, dirPath, path string, data func (fs *fsImpl) createTempFileAndDir(tempFile string) (*os.File, error) { flags := os.O_CREATE | os.O_WRONLY | os.O_EXCL + f, err := os.OpenFile(tempFile, flags, fs.fileMode()) if os.IsNotExist(err) { if err = os.MkdirAll(filepath.Dir(tempFile), fs.dirMode()); err != nil { return nil, errors.Wrap(err, "cannot create directory") } + return os.OpenFile(tempFile, flags, fs.fileMode()) } @@ -130,18 +139,21 @@ func (fs *fsImpl) ReadDir(ctx context.Context, dirname string) ([]os.FileInfo, e // TouchBlob updates file modification time to current time if it's sufficiently old. func (fs *fsStorage) TouchBlob(ctx context.Context, blobID blob.ID, threshold time.Duration) error { _, path := fs.Storage.GetShardedPathAndFilePath(blobID) + st, err := os.Stat(path) if err != nil { return err } n := time.Now() + age := n.Sub(st.ModTime()) if age < threshold { return nil } log.Debugf("updating timestamp on %v to %v", path, n) + return os.Chtimes(path, n, n) } diff --git a/repo/blob/filesystem/filesystem_storage_test.go b/repo/blob/filesystem/filesystem_storage_test.go index 627f24255..a344484cb 100644 --- a/repo/blob/filesystem/filesystem_storage_test.go +++ b/repo/blob/filesystem/filesystem_storage_test.go @@ -16,6 +16,7 @@ func TestFileStorage(t *testing.T) { t.Parallel() + ctx := context.Background() // Test varioush shard configurations. @@ -42,6 +43,7 @@ func TestFileStorage(t *testing.T) { blobtesting.VerifyStorage(ctx, t, r) blobtesting.AssertConnectionInfoRoundTrips(ctx, t, r) + if err := r.Close(ctx); err != nil { t.Fatalf("err: %v", err) } @@ -56,6 +58,7 @@ func TestFileStorage(t *testing.T) { func TestFileStorageTouch(t *testing.T) { t.Parallel() + ctx := context.Background() path, _ := ioutil.TempDir("", "r-fs") @@ -116,6 +119,7 @@ func verifyBlobTimestampOrder(t *testing.T, st blob.Storage, want ...blob.ID) { func assertNoError(t *testing.T, err error) { t.Helper() + if err != nil { t.Errorf("err: %v", err) } diff --git a/repo/blob/gcs/gcs_storage.go b/repo/blob/gcs/gcs_storage.go index 7223b1c24..8bbeaef83 100644 --- a/repo/blob/gcs/gcs_storage.go +++ b/repo/blob/gcs/gcs_storage.go @@ -125,8 +125,10 @@ func (gcs *gcsStorage) PutBlob(ctx context.Context, b blob.ID, data []byte) erro // cancel context before closing the writer causes it to abandon the upload. cancel() writer.Close() //nolint:errcheck + return translateError(err) } + defer cancel() // calling close before cancel() causes it to commit the upload. @@ -140,6 +142,7 @@ func (gcs *gcsStorage) DeleteBlob(ctx context.Context, b blob.ID) error { _, err := exponentialBackoff(fmt.Sprintf("DeleteBlob(%q)", b), attempt) err = translateError(err) + if err == blob.ErrBlobNotFound { return nil } @@ -165,6 +168,7 @@ func (gcs *gcsStorage) ListBlobs(ctx context.Context, prefix blob.ID, callback f }); cberr != nil { return cberr } + oa, err = lst.Next() } @@ -205,6 +209,7 @@ func tokenSourceFromCredentialsFile(ctx context.Context, fn string, scopes ...st if err != nil { return nil, errors.Wrap(err, "google.JWTConfigFromJSON") } + return cfg.TokenSource(ctx), nil } @@ -216,6 +221,7 @@ func tokenSourceFromCredentialsFile(ctx context.Context, fn string, scopes ...st // but this can be disabled by setting IgnoreDefaultCredentials to true. func New(ctx context.Context, opt *Options) (blob.Storage, error) { var ts oauth2.TokenSource + var err error scope := gcsclient.ScopeReadWrite diff --git a/repo/blob/gcs/gcs_storage_test.go b/repo/blob/gcs/gcs_storage_test.go index df8584f95..7a005907d 100644 --- a/repo/blob/gcs/gcs_storage_test.go +++ b/repo/blob/gcs/gcs_storage_test.go @@ -47,6 +47,7 @@ func TestGCSStorage(t *testing.T) { }); err != nil { t.Fatalf("unable to clear GCS bucket: %v", err) } + if err := st.Close(ctx); err != nil { t.Fatalf("err: %v", err) } @@ -69,6 +70,7 @@ func TestGCSStorageInvalid(t *testing.T) { } defer st.Close(ctx) + if err := st.PutBlob(ctx, "xxx", []byte{1, 2, 3}); err == nil { t.Errorf("unexpecte success when adding to non-existent bucket") } diff --git a/repo/blob/logging/logging_storage.go b/repo/blob/logging/logging_storage.go index 0771a77ca..608e0eb00 100644 --- a/repo/blob/logging/logging_storage.go +++ b/repo/blob/logging/logging_storage.go @@ -21,11 +21,13 @@ func (s *loggingStorage) GetBlob(ctx context.Context, id blob.ID, offset, length t0 := time.Now() result, err := s.base.GetBlob(ctx, id, offset, length) dt := time.Since(t0) + if len(result) < 20 { s.printf(s.prefix+"GetBlob(%q,%v,%v)=(%#v, %#v) took %v", id, offset, length, result, err, dt) } else { s.printf(s.prefix+"GetBlob(%q,%v,%v)=({%#v bytes}, %#v) took %v", id, offset, length, len(result), err, dt) } + return result, err } @@ -34,6 +36,7 @@ func (s *loggingStorage) PutBlob(ctx context.Context, id blob.ID, data []byte) e err := s.base.PutBlob(ctx, id, data) dt := time.Since(t0) s.printf(s.prefix+"PutBlob(%q,len=%v)=%#v took %v", id, len(data), err, dt) + return err } @@ -42,6 +45,7 @@ func (s *loggingStorage) DeleteBlob(ctx context.Context, id blob.ID) error { err := s.base.DeleteBlob(ctx, id) dt := time.Since(t0) s.printf(s.prefix+"DeleteBlob(%q)=%#v took %v", id, err, dt) + return err } @@ -53,6 +57,7 @@ func (s *loggingStorage) ListBlobs(ctx context.Context, prefix blob.ID, callback return callback(bi) }) s.printf(s.prefix+"ListBlobs(%q)=%v returned %v items and took %v", prefix, err, cnt, time.Since(t0)) + return err } @@ -61,6 +66,7 @@ func (s *loggingStorage) Close(ctx context.Context) error { err := s.base.Close(ctx) dt := time.Since(t0) s.printf(s.prefix+"Close()=%#v took %v", err, dt) + return err } @@ -74,6 +80,7 @@ func (s *loggingStorage) ConnectionInfo() blob.ConnectionInfo { // NewWrapper returns a Storage wrapper that logs all storage commands. func NewWrapper(wrapped blob.Storage, options ...Option) blob.Storage { s := &loggingStorage{base: wrapped, printf: log.Debugf} + for _, o := range options { o(s) } diff --git a/repo/blob/logging/logging_storage_test.go b/repo/blob/logging/logging_storage_test.go index 83bc3044f..3d924c2bf 100644 --- a/repo/blob/logging/logging_storage_test.go +++ b/repo/blob/logging/logging_storage_test.go @@ -10,6 +10,7 @@ func TestLoggingStorage(t *testing.T) { var outputCount int + myPrefix := "myprefix" myOutput := func(msg string, args ...interface{}) { if !strings.HasPrefix(msg, myPrefix) { @@ -20,6 +21,7 @@ func TestLoggingStorage(t *testing.T) { data := blobtesting.DataMap{} underlying := blobtesting.NewMapStorage(data, nil, nil) + st := NewWrapper(underlying, Output(myOutput), Prefix(myPrefix)) if st == nil { t.Fatalf("unexpected result: %v", st) @@ -27,12 +29,15 @@ func TestLoggingStorage(t *testing.T) { ctx := context.Background() blobtesting.VerifyStorage(ctx, t, st) + if err := st.Close(ctx); err != nil { t.Fatalf("err: %v", err) } + if outputCount == 0 { t.Errorf("did not write any output!") } + if got, want := st.ConnectionInfo().Type, underlying.ConnectionInfo().Type; got != want { t.Errorf("unexpected connection infor %v, want %v", got, want) } diff --git a/repo/blob/registry.go b/repo/blob/registry.go index 686c3bc9f..d9d8247c2 100644 --- a/repo/blob/registry.go +++ b/repo/blob/registry.go @@ -20,12 +20,13 @@ type storageFactory struct { func AddSupportedStorage( urlScheme string, defaultConfigFunc func() interface{}, - createStorageFunc func(context.Context, interface{}) (Storage, error)) { - + createStorageFunc func(context.Context, interface{}) (Storage, error), +) { f := &storageFactory{ defaultConfigFunc: defaultConfigFunc, createStorageFunc: createStorageFunc, } + factories[urlScheme] = f } diff --git a/repo/blob/s3/s3_storage.go b/repo/blob/s3/s3_storage.go index 071ed1012..98e8de98b 100644 --- a/repo/blob/s3/s3_storage.go +++ b/repo/blob/s3/s3_storage.go @@ -34,6 +34,7 @@ type s3Storage struct { func (s *s3Storage) GetBlob(ctx context.Context, b blob.ID, offset, length int64) ([]byte, error) { attempt := func() (interface{}, error) { var opt minio.GetObjectOptions + if length > 0 { if err := opt.SetRange(offset, offset+length-1); err != nil { return nil, errors.Wrap(err, "unable to set range") @@ -46,6 +47,7 @@ func (s *s3Storage) GetBlob(ctx context.Context, b blob.ID, offset, length int64 } defer o.Close() //nolint:errcheck + throttled, err := s.downloadThrottler.AddReader(o) if err != nil { return nil, err @@ -93,6 +95,7 @@ func translateError(err error) error { if me.StatusCode == 200 { return nil } + if me.StatusCode == 404 { return blob.ErrBlobNotFound } @@ -112,10 +115,12 @@ func (s *s3Storage) PutBlob(ctx context.Context, b blob.ID, data []byte) error { progressCallback(string(b), 0, int64(len(data))) defer progressCallback(string(b), int64(len(data)), int64(len(data))) } + n, err := s.cli.PutObject(s.BucketName, s.getObjectNameString(b), throttled, -1, minio.PutObjectOptions{ ContentType: "application/x-kopia", Progress: newProgressReader(progressCallback, string(b), int64(len(data))), }) + if err == io.EOF && n == 0 { // special case empty stream _, err = s.cli.PutObject(s.BucketName, s.getObjectNameString(b), bytes.NewBuffer(nil), 0, minio.PutObjectOptions{ @@ -132,6 +137,7 @@ func (s *s3Storage) DeleteBlob(ctx context.Context, b blob.ID) error { } _, err := exponentialBackoff(fmt.Sprintf("DeleteBlob(%q)", b), attempt) + return translateError(err) } @@ -189,6 +195,7 @@ func (r *progressReader) Read(b []byte) (int, error) { r.cb(r.blobID, r.completed, r.totalLength) r.lastReported = r.completed } + return len(b), nil } diff --git a/repo/blob/s3/s3_storage_test.go b/repo/blob/s3/s3_storage_test.go index 322acb636..eb3334196 100644 --- a/repo/blob/s3/s3_storage_test.go +++ b/repo/blob/s3/s3_storage_test.go @@ -35,8 +35,10 @@ func getBucketName() string { if err != nil { return "kopia-test-1" } + h := sha1.New() fmt.Fprintf(h, "%v", hn) + return fmt.Sprintf("kopia-test-%x", h.Sum(nil)[0:8]) } @@ -77,6 +79,7 @@ func TestS3Storage(t *testing.T) { blobtesting.VerifyStorage(ctx, t, st) blobtesting.AssertConnectionInfoRoundTrips(ctx, t, st) + if err := st.Close(ctx); err != nil { t.Fatalf("err: %v", err) } diff --git a/repo/blob/sftp/sftp_storage.go b/repo/blob/sftp/sftp_storage.go index 200f8f8f5..0a272202b 100644 --- a/repo/blob/sftp/sftp_storage.go +++ b/repo/blob/sftp/sftp_storage.go @@ -58,6 +58,7 @@ func (s *sftpImpl) GetBlobFromPath(ctx context.Context, dirPath, path string, of // and either return it all or return the offset/length bytes buf := new(bytes.Buffer) n, err := r.WriteTo(buf) + if err != nil { return nil, err } @@ -83,7 +84,9 @@ func (s *sftpImpl) PutBlobInPath(ctx context.Context, dirPath, path string, data if _, err := rand.Read(randSuffix); err != nil { return errors.Wrap(err, "can't get random bytes") } + tempFile := fmt.Sprintf("%s.tmp.%x", path, randSuffix) + f, err := s.createTempFileAndDir(tempFile) if err != nil { return errors.Wrap(err, "cannot create temporary file") @@ -102,6 +105,7 @@ func (s *sftpImpl) PutBlobInPath(ctx context.Context, dirPath, path string, data if removeErr := s.cli.Remove(tempFile); removeErr != nil { fmt.Printf("warning: can't remove temp file: %v", removeErr) } + return err } @@ -110,11 +114,13 @@ func (s *sftpImpl) PutBlobInPath(ctx context.Context, dirPath, path string, data func (s *sftpImpl) createTempFileAndDir(tempFile string) (*psftp.File, error) { flags := os.O_CREATE | os.O_WRONLY | os.O_EXCL + f, err := s.cli.OpenFile(tempFile, flags) if os.IsNotExist(err) { if err = s.cli.MkdirAll(filepath.Dir(tempFile)); err != nil { return nil, errors.Wrap(err, "cannot create directory") } + return s.cli.OpenFile(tempFile, flags) } @@ -181,6 +187,7 @@ func getHostKey(host, knownHosts string) (ssh.PublicKey, error) { defer file.Close() var hostKey ssh.PublicKey + var hosts []string scanner := bufio.NewScanner(file) @@ -209,6 +216,7 @@ func getSigner(path string) (ssh.Signer, error) { if err != nil { return nil, err } + return key, nil } @@ -246,6 +254,7 @@ func New(ctx context.Context, opts *Options) (blob.Storage, error) { } addr := fmt.Sprintf("%s:%d", opts.Host, opts.Port) + conn, err := ssh.Dial("tcp", addr, config) if err != nil { return nil, errors.Wrapf(err, "unable to dial [%s]: %+v", addr, config) diff --git a/repo/blob/sftp/sftp_storage_test.go b/repo/blob/sftp/sftp_storage_test.go index ef92f29e4..d8636f11a 100644 --- a/repo/blob/sftp/sftp_storage_test.go +++ b/repo/blob/sftp/sftp_storage_test.go @@ -47,6 +47,7 @@ func TestSFTPStorageValid(t *testing.T) { func assertNoError(t *testing.T, err error) { t.Helper() + if err != nil { t.Errorf("err: %v", err) } @@ -70,6 +71,7 @@ func createSFTPStorage(ctx context.Context, t *testing.T) (blob.Storage, error) if envPort == "" { t.Skip("KOPIA_SFTP_TEST_PORT not provided") } + port, err := strconv.ParseInt(envPort, 10, 64) if err != nil { t.Skip("skipping test because port is not numeric") diff --git a/repo/blob/sharded/sharded.go b/repo/blob/sharded/sharded.go index cbe1c524b..413c31c85 100644 --- a/repo/blob/sharded/sharded.go +++ b/repo/blob/sharded/sharded.go @@ -52,9 +52,9 @@ func (s Storage) ListBlobs(ctx context.Context, prefix blob.ID, callback func(bl for _, e := range entries { if e.IsDir() { - newPrefix := currentPrefix + e.Name() var match bool + newPrefix := currentPrefix + e.Name() if len(prefix) > len(newPrefix) { match = strings.HasPrefix(string(prefix), newPrefix) } else { @@ -98,9 +98,11 @@ func (s Storage) DeleteBlob(ctx context.Context, blobID blob.ID) error { func (s Storage) getShardDirectory(blobID blob.ID) (string, blob.ID) { shardPath := s.RootPath + if len(blobID) < 20 { return shardPath, blobID } + for _, size := range s.Shards { shardPath = filepath.Join(shardPath, string(blobID[0:size])) blobID = blobID[size:] @@ -112,5 +114,6 @@ func (s Storage) getShardDirectory(blobID blob.ID) (string, blob.ID) { func (s Storage) GetShardedPathAndFilePath(blobID blob.ID) (shardPath, filePath string) { shardPath, blobID = s.getShardDirectory(blobID) filePath = filepath.Join(shardPath, s.makeFileName(blobID)) + return } diff --git a/repo/blob/storage.go b/repo/blob/storage.go index fd3c99d09..cce0d6513 100644 --- a/repo/blob/storage.go +++ b/repo/blob/storage.go @@ -84,14 +84,18 @@ func IterateAllPrefixesInParallel(ctx context.Context, parallelism int, st Stora } var wg sync.WaitGroup + semaphore := make(chan struct{}, parallelism) errch := make(chan error, len(prefixes)) + for _, prefix := range prefixes { wg.Add(1) + prefix := prefix // acquire semaphore semaphore <- struct{}{} + go func() { defer wg.Done() defer func() { @@ -123,6 +127,7 @@ func ListAllBlobsConsistent(ctx context.Context, st Storage, prefix ID, maxAttem if err != nil { return nil, err } + if i > 0 && sameBlobs(result, previous) { return result, nil } @@ -137,17 +142,22 @@ func ListAllBlobsConsistent(ctx context.Context, st Storage, prefix ID, maxAttem func sameBlobs(b1, b2 []Metadata) bool { if len(b1) != len(b2) { log.Printf("a") + return false } + m := map[ID]Metadata{} + for _, b := range b1 { m[b.BlobID] = normalizeMetadata(b) } + for _, b := range b2 { if r := m[b.BlobID]; r != normalizeMetadata(b) { return false } } + return true } diff --git a/repo/blob/webdav/webdav_storage.go b/repo/blob/webdav/webdav_storage.go index 3a99fb8d0..7e3333276 100644 --- a/repo/blob/webdav/webdav_storage.go +++ b/repo/blob/webdav/webdav_storage.go @@ -51,7 +51,9 @@ func (d *davStorageImpl) GetBlobFromPath(ctx context.Context, dirPath, path stri if err != nil { return nil, d.translateError(err) } + data := v.([]byte) + if length < 0 { return data, nil } @@ -106,6 +108,7 @@ func (d *davStorageImpl) ReadDir(ctx context.Context, dir string) ([]os.FileInfo func (d *davStorageImpl) PutBlobInPath(ctx context.Context, dirPath, filePath string, data []byte) error { tmpPath := fmt.Sprintf("%v-%v", filePath, rand.Int63()) + if err := d.translateError(retry.WithExponentialBackoffNoValue("Write", func() error { return d.cli.Write(tmpPath, data, defaultFilePerm) }, isRetriable)); err != nil { diff --git a/repo/blob/webdav/webdav_storage_test.go b/repo/blob/webdav/webdav_storage_test.go index 64725cb76..aba7fdcda 100644 --- a/repo/blob/webdav/webdav_storage_test.go +++ b/repo/blob/webdav/webdav_storage_test.go @@ -35,6 +35,7 @@ func basicAuth(h http.Handler) http.HandlerFunc { func TestWebDAVStorageExternalServer(t *testing.T) { t.Parallel() + testURL := os.Getenv("KOPIA_WEBDAV_TEST_URL") if testURL == "" { t.Skip("KOPIA_WEBDAV_TEST_URL not provided") @@ -111,6 +112,7 @@ func verifyWebDAVStorage(t *testing.T, url, username, password string, shardSpec blobtesting.VerifyStorage(ctx, t, st) blobtesting.AssertConnectionInfoRoundTrips(ctx, t, st) + if err := st.Close(ctx); err != nil { t.Fatalf("err: %v", err) } diff --git a/repo/connect.go b/repo/connect.go index 015797a50..2db1ce3c4 100644 --- a/repo/connect.go +++ b/repo/connect.go @@ -85,14 +85,17 @@ func setupCaching(configPath string, lc *LocalConfig, opt content.CachingOptions lc.Caching.CacheDirectory = absCacheDir } + lc.Caching.MaxCacheSizeBytes = opt.MaxCacheSizeBytes lc.Caching.MaxMetadataCacheSizeBytes = opt.MaxMetadataCacheSizeBytes lc.Caching.MaxListCacheDurationSec = opt.MaxListCacheDurationSec log.Debugf("Creating cache directory '%v' with max size %v", lc.Caching.CacheDirectory, lc.Caching.MaxCacheSizeBytes) + if err := os.MkdirAll(lc.Caching.CacheDirectory, 0700); err != nil { log.Warningf("unablet to create cache directory: %v", err) } + return nil } diff --git a/repo/content/block_manager_compaction.go b/repo/content/block_manager_compaction.go index 96442e3aa..7af2b7699 100644 --- a/repo/content/block_manager_compaction.go +++ b/repo/content/block_manager_compaction.go @@ -27,6 +27,7 @@ func (bm *Manager) CompactIndexes(ctx context.Context, opt CompactOptions) error defer bm.unlock() log.Debugf("CompactIndexes(%+v)", opt) + if opt.MaxSmallBlobs < opt.MinSmallBlobs { return errors.Errorf("invalid content counts") } @@ -47,12 +48,15 @@ func (bm *Manager) CompactIndexes(ctx context.Context, opt CompactOptions) error func (bm *Manager) getContentsToCompact(indexBlobs []IndexBlobInfo, opt CompactOptions) []IndexBlobInfo { var nonCompactedContents []IndexBlobInfo + var totalSizeNonCompactedContents int64 var verySmallContents []IndexBlobInfo + var totalSizeVerySmallContents int64 var mediumSizedContents []IndexBlobInfo + var totalSizeMediumSizedContents int64 for _, b := range indexBlobs { @@ -61,6 +65,7 @@ func (bm *Manager) getContentsToCompact(indexBlobs []IndexBlobInfo, opt CompactO } nonCompactedContents = append(nonCompactedContents, b) + if b.Length < int64(bm.maxPackSize/20) { verySmallContents = append(verySmallContents, b) totalSizeVerySmallContents += b.Length @@ -68,6 +73,7 @@ func (bm *Manager) getContentsToCompact(indexBlobs []IndexBlobInfo, opt CompactO mediumSizedContents = append(mediumSizedContents, b) totalSizeMediumSizedContents += b.Length } + totalSizeNonCompactedContents += b.Length } @@ -83,6 +89,7 @@ func (bm *Manager) getContentsToCompact(indexBlobs []IndexBlobInfo, opt CompactO } formatLog.Debugf("compacting all %v non-compacted contents", len(nonCompactedContents)) + return nonCompactedContents } @@ -90,10 +97,12 @@ func (bm *Manager) compactAndDeleteIndexBlobs(ctx context.Context, indexBlobs [] if len(indexBlobs) <= 1 { return nil } - formatLog.Debugf("compacting %v contents", len(indexBlobs)) - t0 := time.Now() + formatLog.Debugf("compacting %v contents", len(indexBlobs)) + + t0 := time.Now() bld := make(packIndexBuilder) + for _, indexBlob := range indexBlobs { if err := bm.addIndexBlobsToBuilder(ctx, bld, indexBlob, opt); err != nil { return err @@ -118,6 +127,7 @@ func (bm *Manager) compactAndDeleteIndexBlobs(ctx context.Context, indexBlobs [] } bm.listCache.deleteListCache() + if err := bm.st.DeleteBlob(ctx, indexBlob.BlobID); err != nil { log.Warningf("unable to delete compacted blob %q: %v", indexBlob.BlobID, err) } diff --git a/repo/content/builder.go b/repo/content/builder.go index b05558831..15765c598 100644 --- a/repo/content/builder.go +++ b/repo/content/builder.go @@ -21,10 +21,12 @@ func (b packIndexBuilder) clone() packIndexBuilder { } r := packIndexBuilder{} + for k, v := range b { i2 := *v r[k] = &i2 } + return r } @@ -80,12 +82,14 @@ func (b packIndexBuilder) Build(output io.Writer) error { header[1] = byte(layout.keyLength) binary.BigEndian.PutUint16(header[2:4], uint16(layout.entryLength)) binary.BigEndian.PutUint32(header[4:8], uint32(layout.entryCount)) + if _, err := w.Write(header); err != nil { return errors.Wrap(err, "unable to write header") } // write all sorted contents. entry := make([]byte, layout.entryLength) + for _, it := range allContents { if err := writeEntry(w, it, layout, entry); err != nil { return errors.Wrap(err, "unable to write entry") @@ -106,17 +110,21 @@ func prepareExtraData(allContents []*Info, layout *indexLayout) []byte { if i == 0 { layout.keyLength = len(contentIDToBytes(it.ID)) } + if it.PackBlobID != "" { if _, ok := layout.packBlobIDOffsets[it.PackBlobID]; !ok { layout.packBlobIDOffsets[it.PackBlobID] = uint32(len(extraData)) extraData = append(extraData, []byte(it.PackBlobID)...) } } + if len(it.Payload) > 0 { panic("storing payloads in indexes is not supported") } } + layout.extraDataOffset = uint32(8 + layout.entryCount*(layout.keyLength+layout.entryLength)) + return extraData } @@ -133,6 +141,7 @@ func writeEntry(w io.Writer, it *Info, layout *indexLayout, entry []byte) error if _, err := w.Write(k); err != nil { return errors.Wrap(err, "error writing entry key") } + if _, err := w.Write(entry); err != nil { return errors.Wrap(err, "error writing entry") } @@ -152,14 +161,17 @@ func formatEntry(entry []byte, it *Info, layout *indexLayout) error { } binary.BigEndian.PutUint32(entryPackFileOffset, layout.extraDataOffset+layout.packBlobIDOffsets[it.PackBlobID]) + if it.Deleted { binary.BigEndian.PutUint32(entryPackedOffset, it.PackOffset|0x80000000) } else { binary.BigEndian.PutUint32(entryPackedOffset, it.PackOffset) } + binary.BigEndian.PutUint32(entryPackedLength, it.Length) timestampAndFlags |= uint64(it.FormatVersion) << 8 timestampAndFlags |= uint64(len(it.PackBlobID)) binary.BigEndian.PutUint64(entryTimestampAndFlags, timestampAndFlags) + return nil } diff --git a/repo/content/cache_hmac.go b/repo/content/cache_hmac.go index b40670c81..0c5fda5d5 100644 --- a/repo/content/cache_hmac.go +++ b/repo/content/cache_hmac.go @@ -9,6 +9,7 @@ func appendHMAC(data, secret []byte) []byte { h := hmac.New(sha256.New, secret) h.Write(data) // nolint:errcheck + return h.Sum(data) } @@ -24,9 +25,11 @@ func verifyAndStripHMAC(b, secret []byte) ([]byte, error) { h := hmac.New(sha256.New, secret) h.Write(data) // nolint:errcheck validSignature := h.Sum(nil) + if len(signature) != len(validSignature) { return nil, errors.New("invalid signature length") } + if hmac.Equal(validSignature, signature) { return data, nil } diff --git a/repo/content/committed_content_index.go b/repo/content/committed_content_index.go index 5b6eab09a..44a0adec0 100644 --- a/repo/content/committed_content_index.go +++ b/repo/content/committed_content_index.go @@ -32,9 +32,11 @@ func (b *committedContentIndex) getContent(contentID ID) (Info, error) { if info != nil { return *info, nil } + if err == nil { return Info{}, ErrContentNotFound } + return Info{}, err } @@ -58,8 +60,10 @@ func (b *committedContentIndex) addContent(indexBlobID blob.ID, data []byte, use if err != nil { return errors.Wrapf(err, "unable to open pack index %q", indexBlobID) } + b.inUse[indexBlobID] = ndx b.merged = append(b.merged, ndx) + return nil } @@ -92,10 +96,13 @@ func (b *committedContentIndex) use(packFiles []blob.ID) (bool, error) { if !b.packFilesChanged(packFiles) { return false, nil } + log.Debugf("set of index files has changed (had %v, now %v)", len(b.inUse), len(packFiles)) var newMerged mergedIndex + newInUse := map[blob.ID]packIndex{} + defer func() { newMerged.Close() //nolint:errcheck }() @@ -109,12 +116,14 @@ func (b *committedContentIndex) use(packFiles []blob.ID) (bool, error) { newMerged = append(newMerged, ndx) newInUse[e] = ndx } + b.merged = newMerged b.inUse = newInUse if err := b.cache.expireUnused(packFiles); err != nil { log.Warningf("unable to expire unused content index files: %v", err) } + newMerged = nil return true, nil diff --git a/repo/content/committed_content_index_disk_cache.go b/repo/content/committed_content_index_disk_cache.go index fdbaffd32..815673441 100644 --- a/repo/content/committed_content_index_disk_cache.go +++ b/repo/content/committed_content_index_disk_cache.go @@ -42,6 +42,7 @@ func (c *diskCommittedContentIndexCache) hasIndexBlobID(indexBlobID blob.ID) (bo if err == nil { return true, nil } + if os.IsNotExist(err) { return false, nil } @@ -71,6 +72,7 @@ func (c *diskCommittedContentIndexCache) addContentToCache(indexBlobID blob.ID, if err != nil { return err } + if !exists { return errors.Errorf("unsuccessful index write of content %q", indexBlobID) } @@ -88,6 +90,7 @@ func writeTempFileAtomic(dirname string, data []byte) (string, error) { tf, err = ioutil.TempFile(dirname, "tmp") } } + if err != nil { return "", errors.Wrap(err, "can't create tmp file") } @@ -95,6 +98,7 @@ func writeTempFileAtomic(dirname string, data []byte) (string, error) { if _, err := tf.Write(data); err != nil { return "", errors.Wrap(err, "can't write to temp file") } + if err := tf.Close(); err != nil { return "", errors.Errorf("can't close tmp file") } @@ -124,6 +128,7 @@ func (c *diskCommittedContentIndexCache) expireUnused(used []blob.ID) error { for _, rem := range remaining { if time.Since(rem.ModTime()) > unusedCommittedContentIndexCleanupTime { log.Debugf("removing unused %v %v", rem.Name(), rem.ModTime()) + if err := os.Remove(filepath.Join(c.dirname, rem.Name())); err != nil { log.Warningf("unable to remove unused index file: %v", err) } diff --git a/repo/content/committed_content_index_mem_cache.go b/repo/content/committed_content_index_mem_cache.go index c83fda34d..67467118d 100644 --- a/repo/content/committed_content_index_mem_cache.go +++ b/repo/content/committed_content_index_mem_cache.go @@ -31,6 +31,7 @@ func (m *memoryCommittedContentIndexCache) addContentToCache(indexBlobID blob.ID } m.contents[indexBlobID] = ndx + return nil } diff --git a/repo/content/content_cache.go b/repo/content/content_cache.go index 432370105..ddbd0d088 100644 --- a/repo/content/content_cache.go +++ b/repo/content/content_cache.go @@ -100,12 +100,14 @@ func (c *contentCache) readAndVerifyCacheContent(ctx context.Context, cacheKey c // ignore malformed contents log.Warningf("malformed content %v: %v", cacheKey, err) + return nil } if err != blob.ErrBlobNotFound { log.Warningf("unable to read cache %v: %v", cacheKey, err) } + return nil } @@ -150,6 +152,7 @@ func (h *contentMetadataHeap) Pop() interface{} { n := len(old) item := old[n-1] *h = old[0 : n-1] + return item } @@ -164,6 +167,7 @@ func (c *contentCache) sweepDirectory(ctx context.Context) (err error) { t0 := time.Now() var h contentMetadataHeap + var totalRetainedSize int64 err = c.cacheStorage.ListBlobs(ctx, "", func(it blob.Metadata) error { @@ -186,11 +190,13 @@ func (c *contentCache) sweepDirectory(ctx context.Context) (err error) { log.Debugf("finished sweeping directory in %v and retained %v/%v bytes (%v %%)", time.Since(t0), totalRetainedSize, c.maxSizeBytes, 100*totalRetainedSize/c.maxSizeBytes) c.lastTotalSizeBytes = totalRetainedSize + return nil } func newContentCache(ctx context.Context, st blob.Storage, caching CachingOptions, maxBytes int64, subdir string) (*contentCache, error) { var cacheStorage blob.Storage + var err error if maxBytes > 0 && caching.CacheDirectory != "" { @@ -228,6 +234,7 @@ func newContentCacheWithCacheStorage(ctx context.Context, st, cacheStorage blob. if err := c.sweepDirectory(ctx); err != nil { return nil, err } + go c.sweepDirectoryPeriodically(ctx) return c, nil diff --git a/repo/content/content_cache_test.go b/repo/content/content_cache_test.go index b3f2aaeec..df2ae0297 100644 --- a/repo/content/content_cache_test.go +++ b/repo/content/content_cache_test.go @@ -23,6 +23,7 @@ func newUnderlyingStorageForContentCacheTesting(t *testing.T) blob.Storage { st := blobtesting.NewMapStorage(data, nil, nil) assertNoError(t, st.PutBlob(ctx, "content-1", []byte{1, 2, 3, 4, 5, 6, 7, 8, 9, 10})) assertNoError(t, st.PutBlob(ctx, "content-4k", bytes.Repeat([]byte{1, 2, 3, 4}, 1000))) // 4000 bytes + return st } @@ -36,6 +37,7 @@ func TestCacheExpiration(t *testing.T) { if err != nil { t.Fatalf("err: %v", err) } + defer cache.close() ctx := context.Background() @@ -83,6 +85,7 @@ func TestDiskContentCache(t *testing.T) { if err != nil { t.Fatalf("error getting temp dir: %v", err) } + defer os.RemoveAll(tmpDir) cache, err := newContentCache(ctx, newUnderlyingStorageForContentCacheTesting(t), CachingOptions{ @@ -92,7 +95,9 @@ func TestDiskContentCache(t *testing.T) { if err != nil { t.Fatalf("err: %v", err) } + defer cache.close() + verifyContentCache(t, cache) } @@ -226,6 +231,7 @@ func TestCacheFailureToWrite(t *testing.T) { if err != nil { t.Errorf("error listing cache: %v", err) } + if len(all) != 0 { t.Errorf("invalid test - cache was written") } @@ -269,7 +275,9 @@ func TestCacheFailureToRead(t *testing.T) { func verifyStorageContentList(t *testing.T, st blob.Storage, expectedContents ...blob.ID) { t.Helper() + var foundContents []blob.ID + assertNoError(t, st.ListBlobs(context.Background(), "", func(bm blob.Metadata) error { foundContents = append(foundContents, bm.BlobID) return nil @@ -278,6 +286,7 @@ func verifyStorageContentList(t *testing.T, st blob.Storage, expectedContents .. sort.Slice(foundContents, func(i, j int) bool { return foundContents[i] < foundContents[j] }) + if !reflect.DeepEqual(foundContents, expectedContents) { t.Errorf("unexpected content list: %v, wanted %v", foundContents, expectedContents) } @@ -285,6 +294,7 @@ func verifyStorageContentList(t *testing.T, st blob.Storage, expectedContents .. func assertNoError(t *testing.T, err error) { t.Helper() + if err != nil { t.Errorf("err: %v", err) } diff --git a/repo/content/content_formatter.go b/repo/content/content_formatter.go index d96cd4161..8d8d6afd8 100644 --- a/repo/content/content_formatter.go +++ b/repo/content/content_formatter.go @@ -92,6 +92,7 @@ func symmetricEncrypt(createCipher func() (cipher.Block, error), iv, b []byte) ( ctr := cipher.NewCTR(blockCipher, iv[0:blockCipher.BlockSize()]) result := make([]byte, len(b)) ctr.XORKeyStream(result, b) + return result, nil } @@ -104,6 +105,7 @@ type salsaEncryptor struct { func (s salsaEncryptor) Decrypt(input, contentID []byte) ([]byte, error) { if s.hmacSecret != nil { var err error + input, err = verifyAndStripHMAC(input, s.hmacSecret) if err != nil { return nil, errors.Wrap(err, "verifyAndStripHMAC") @@ -134,9 +136,11 @@ func (s salsaEncryptor) encryptDecrypt(input, contentID []byte) ([]byte, error) if len(contentID) < s.nonceSize { return nil, errors.Errorf("hash too short, expected >=%v bytes, got %v", s.nonceSize, len(contentID)) } + result := make([]byte, len(input)) nonce := contentID[0:s.nonceSize] salsa20.XORKeyStream(result, input, nonce, s.key) + return result, nil } @@ -194,7 +198,9 @@ func SupportedHashAlgorithms() []string { for k := range hashFunctions { result = append(result, k) } + sort.Strings(result) + return result } @@ -203,7 +209,9 @@ func SupportedEncryptionAlgorithms() []string { for k := range encryptors { result = append(result, k) } + sort.Strings(result) + return result } diff --git a/repo/content/content_formatter_test.go b/repo/content/content_formatter_test.go index bb486eb1e..a60b5d1f4 100644 --- a/repo/content/content_formatter_test.go +++ b/repo/content/content_formatter_test.go @@ -37,19 +37,23 @@ func TestFormatters(t *testing.T) { if err != nil { key := hashAlgo + "/" + encryptionAlgo + errmsg := incompatibleAlgorithms[key] if errmsg == "" { t.Errorf("Algorithm %v not marked as incompatible and failed with %v", key, err) continue } + if err.Error() == errmsg { t.Errorf("unexpected error message %v, wanted %v", err.Error(), errmsg) continue } + continue } contentID := h(data) + cipherText, err := e.Encrypt(data, contentID) if err != nil || cipherText == nil { t.Errorf("invalid response from Encrypt: %v %v", cipherText, err) diff --git a/repo/content/content_formatting_options.go b/repo/content/content_formatting_options.go index 350d3d4ef..bb99f4ff9 100644 --- a/repo/content/content_formatting_options.go +++ b/repo/content/content_formatting_options.go @@ -22,5 +22,6 @@ func (o *FormattingOptions) DeriveKey(purpose []byte, length int) []byte { key := make([]byte, length) k := hkdf.New(sha256.New, o.MasterKey, purpose, nil) io.ReadFull(k, key) //nolint:errcheck + return key } diff --git a/repo/content/content_id_to_bytes.go b/repo/content/content_id_to_bytes.go index ec60fdd4e..196945c86 100644 --- a/repo/content/content_id_to_bytes.go +++ b/repo/content/content_id_to_bytes.go @@ -8,10 +8,13 @@ func bytesToContentID(b []byte) ID { if len(b) == 0 { return "" } + if b[0] == 0xff { return ID(b[1:]) } + prefix := "" + if b[0] != 0 { prefix = string(b[0:1]) } @@ -21,7 +24,9 @@ func bytesToContentID(b []byte) ID { func contentIDToBytes(c ID) []byte { var prefix []byte + var skip int + if len(c)%2 == 1 { prefix = []byte(c[0:1]) skip = 1 diff --git a/repo/content/content_index_recovery.go b/repo/content/content_index_recovery.go index 8a96f4c97..ba59172ae 100644 --- a/repo/content/content_index_recovery.go +++ b/repo/content/content_index_recovery.go @@ -60,11 +60,13 @@ func (p *packContentPostamble) toBytes() ([]byte, error) { checksum := crc32.ChecksumIEEE(buf[0:n]) binary.BigEndian.PutUint32(buf[n:], checksum) n += 4 + if n > 255 { return nil, errors.Errorf("postamble too long: %v", n) } buf[n] = byte(n) + return buf[0 : n+1], nil } @@ -83,8 +85,10 @@ func findPostamble(b []byte) *packContentPostamble { // too short, must be at least 5 bytes (checksum + own length) return nil } + postambleStart := len(b) - 1 - postambleLength postambleEnd := len(b) - 1 + if postambleStart < 0 { // invalid last byte return nil @@ -109,10 +113,12 @@ func decodePostamble(payload []byte) *packContentPostamble { // invalid flags return nil } + if flags != 1 { // unsupported flag return nil } + payload = payload[n:] ivLength, n := binary.Uvarint(payload) @@ -120,6 +126,7 @@ func decodePostamble(payload []byte) *packContentPostamble { // invalid flags return nil } + payload = payload[n:] if ivLength > uint64(len(payload)) { // invalid IV length @@ -134,6 +141,7 @@ func decodePostamble(payload []byte) *packContentPostamble { // invalid offset return nil } + payload = payload[n:] length, n := binary.Uvarint(payload) @@ -162,12 +170,14 @@ func (bm *lockFreeManager) buildLocalIndex(pending packIndexBuilder) ([]byte, er func (bm *lockFreeManager) appendPackFileIndexRecoveryData(contentData []byte, pending packIndexBuilder) ([]byte, error) { // build, encrypt and append local index localIndexOffset := len(contentData) + localIndex, err := bm.buildLocalIndex(pending) if err != nil { return nil, err } localIndexIV := bm.hashData(localIndex) + encryptedLocalIndex, err := bm.encryptor.Encrypt(localIndex, localIndexIV) if err != nil { return nil, err @@ -180,6 +190,7 @@ func (bm *lockFreeManager) appendPackFileIndexRecoveryData(contentData []byte, p } contentData = append(contentData, encryptedLocalIndex...) + postambleBytes, err := postamble.toBytes() if err != nil { return nil, err @@ -202,6 +213,7 @@ func (bm *lockFreeManager) appendPackFileIndexRecoveryData(contentData []byte, p func (bm *lockFreeManager) readPackFileLocalIndex(ctx context.Context, packFile blob.ID, packFileLength int64) ([]byte, error) { // TODO(jkowalski): optimize read when packFileLength is provided _ = packFileLength + payload, err := bm.st.GetBlob(ctx, packFile, 0, -1) if err != nil { return nil, err diff --git a/repo/content/content_index_recovery_test.go b/repo/content/content_index_recovery_test.go index 23b23e936..0b142ba0f 100644 --- a/repo/content/content_index_recovery_test.go +++ b/repo/content/content_index_recovery_test.go @@ -86,9 +86,11 @@ func TestContentIndexRecovery(t *testing.T) { verifyContent(ctx, t, bm, content1, seededRandomData(10, 100)) verifyContent(ctx, t, bm, content2, seededRandomData(11, 100)) verifyContent(ctx, t, bm, content3, seededRandomData(12, 100)) + if err := bm.Flush(ctx); err != nil { t.Errorf("flush error: %v", err) } + verifyContent(ctx, t, bm, content1, seededRandomData(10, 100)) verifyContent(ctx, t, bm, content2, seededRandomData(11, 100)) verifyContent(ctx, t, bm, content3, seededRandomData(12, 100)) diff --git a/repo/content/content_manager.go b/repo/content/content_manager.go index 74a30ae17..5fb564b7a 100644 --- a/repo/content/content_manager.go +++ b/repo/content/content_manager.go @@ -126,6 +126,7 @@ func (bm *Manager) DeleteContent(contentID ID) error { } bm.deletePreexistingContent(bi) + return nil } @@ -135,6 +136,7 @@ func (bm *Manager) deletePreexistingContent(ci Info) { if ci.Deleted { return } + pp := bm.getOrCreatePendingPackInfoLocked(packPrefixForContentID(ci.ID)) ci.Deleted = true ci.TimestampSeconds = bm.timeNow().Unix() @@ -145,6 +147,7 @@ func (bm *Manager) addToPackUnlocked(ctx context.Context, contentID ID, data []b prefix := packPrefixForContentID(contentID) data = cloneBytes(data) + bm.lock() // do not start new uploads while flushing @@ -182,6 +185,7 @@ func (bm *Manager) addToPackUnlocked(ctx context.Context, contentID ID, data []b Length: uint32(len(data)), TimestampSeconds: bm.timeNow().Unix(), } + shouldWrite := pp.currentPackDataLength >= bm.maxPackSize if shouldWrite { // we're about to write to storage without holding a lock @@ -189,6 +193,7 @@ func (bm *Manager) addToPackUnlocked(ctx context.Context, contentID ID, data []b delete(bm.pendingPacks, pp.prefix) bm.writingPacks = append(bm.writingPacks, pp) } + bm.unlock() // at this point we're unlocked so different goroutines can encrypt and @@ -247,12 +252,14 @@ func (bm *Manager) verifyCurrentPackItemsLocked() { func (bm *Manager) verifyPackIndexBuilderLocked() { for k, cpi := range bm.packIndexBuilder { bm.assertInvariant(cpi.ID == k, "content ID entry has invalid key: %v %v", cpi.ID, k) + if cpi.Deleted { bm.assertInvariant(cpi.PackBlobID == "", "content can't be both deleted and have a pack content: %v", cpi.ID) } else { bm.assertInvariant(cpi.PackBlobID != "", "content that's not deleted must have a pack content: %+v", cpi) bm.assertInvariant(cpi.FormatVersion == byte(bm.writeFormatVersion), "content that's not deleted must have a valid format version: %+v", cpi) } + bm.assertInvariant(cpi.TimestampSeconds != 0, "content has no timestamp: %v", cpi.ID) } } @@ -293,10 +300,12 @@ func (bm *Manager) flushPackIndexesLocked(ctx context.Context) error { if err := bm.committedContents.addContent(indexBlobID, dataCopy, true); err != nil { return errors.Wrap(err, "unable to add committed content") } + bm.packIndexBuilder = make(packIndexBuilder) } bm.flushPackIndexesAfter = bm.timeNow().Add(flushPackIndexTimeout) + return nil } @@ -318,6 +327,7 @@ func (bm *Manager) writePackAndAddToIndex(ctx context.Context, pp *pendingPackIn if !holdingLock { bm.lock() + defer func() { bm.cond.Broadcast() bm.unlock() @@ -333,11 +343,13 @@ func (bm *Manager) writePackAndAddToIndex(ctx context.Context, pp *pendingPackIn for _, info := range packFileIndex { bm.packIndexBuilder.Add(*info) } + return nil } // failure - add to failedPacks slice again bm.failedPacks = append(bm.failedPacks, pp) + return errors.Wrap(err, "error writing pack") } @@ -348,6 +360,7 @@ func (bm *Manager) prepareAndWritePackInternal(ctx context.Context, pp *pendingP } packFile := blob.ID(fmt.Sprintf("%v%x", pp.prefix, contentID)) + contentData, packFileIndex, err := bm.preparePackDataContent(ctx, pp, packFile) if err != nil { return nil, errors.Wrap(err, "error preparing data content") @@ -357,6 +370,7 @@ func (bm *Manager) prepareAndWritePackInternal(ctx context.Context, pp *pendingP if err := bm.writePackFileNotLocked(ctx, packFile, contentData); err != nil { return nil, errors.Wrap(err, "can't save pack data content") } + formatLog.Debugf("wrote pack file: %v (%v bytes)", packFile, len(contentData)) } @@ -365,11 +379,13 @@ func (bm *Manager) prepareAndWritePackInternal(ctx context.Context, pp *pendingP func removePendingPack(slice []*pendingPackInfo, pp *pendingPackInfo) []*pendingPackInfo { result := slice[:0] + for _, p := range slice { if p != pp { result = append(result, p) } } + return result } @@ -378,9 +394,11 @@ func (bm *Manager) Close(ctx context.Context) error { if err := bm.Flush(ctx); err != nil { return errors.Wrap(err, "error flushing") } + bm.contentCache.close() bm.metadataCache.close() close(bm.closed) + return nil } @@ -392,6 +410,7 @@ func (bm *Manager) Flush(ctx context.Context) error { defer bm.unlock() bm.flushing = true + defer func() { bm.flushing = false }() @@ -434,6 +453,7 @@ func packPrefixForContentID(contentID ID) blob.ID { if contentID.HasPrefix() { return PackBlobIDPrefixSpecial } + return PackBlobIDPrefixRegular } @@ -454,6 +474,7 @@ func (bm *Manager) WriteContent(ctx context.Context, data []byte, prefix ID) (ID if err := validatePrefix(prefix); err != nil { return "", err } + contentID := prefix + ID(hex.EncodeToString(bm.hashData(data))) // content already tracked @@ -464,6 +485,7 @@ func (bm *Manager) WriteContent(ctx context.Context, data []byte, prefix ID) (ID } err := bm.addToPackUnlocked(ctx, contentID, data, false) + return contentID, err } @@ -544,9 +566,12 @@ func (bm *Manager) Refresh(ctx context.Context) (bool, error) { defer bm.unlock() log.Debugf("Refresh started") + t0 := time.Now() + _, updated, err := bm.loadPackIndexesUnlocked(ctx) log.Debugf("Refresh completed in %v and updated=%v", time.Since(t0), updated) + return updated, err } diff --git a/repo/content/content_manager_iterate.go b/repo/content/content_manager_iterate.go index 9746a949b..ac0b26bcf 100644 --- a/repo/content/content_manager_iterate.go +++ b/repo/content/content_manager_iterate.go @@ -27,7 +27,9 @@ func maybeParallelExecutor(parallel int, originalCallback IterateCallback) (Iter workch := make(chan Info, parallel) workererrch := make(chan error, 1) + var wg sync.WaitGroup + var once sync.Once lastWorkerError := func() error { @@ -44,6 +46,7 @@ func maybeParallelExecutor(parallel int, originalCallback IterateCallback) (Iter close(workch) wg.Wait() }) + return lastWorkerError() } @@ -78,11 +81,13 @@ func (bm *Manager) snapshotUncommittedItems() packIndexBuilder { defer bm.unlock() overlay := bm.packIndexBuilder.clone() + for _, pp := range bm.pendingPacks { for _, pi := range pp.currentPackItems { overlay.Add(pi) } } + for _, pp := range bm.writingPacks { for _, pi := range pp.currentPackItems { overlay.Add(pi) @@ -114,6 +119,7 @@ func (bm *Manager) IterateContents(opts IterateOptions, callback IterateCallback if !strings.HasPrefix(string(i.ID), string(opts.Prefix)) { return nil } + return callback(i) } @@ -226,9 +232,11 @@ func(pi PackInfo) error { }); err != nil { return errors.Wrap(err, "error iterating packs") } + log.Infof("found %v pack blobs in use", len(usedPacks)) unusedCount := 0 + var prefixes []blob.ID if parallellism <= len(PackBlobIDPrefixes) { @@ -241,6 +249,7 @@ func(pi PackInfo) error { } } } + if err := blob.IterateAllPrefixesInParallel(ctx, parallellism, bm.st, prefixes, func(bm blob.Metadata) error { if usedPacks[bm.BlobID] { @@ -248,10 +257,12 @@ func(bm blob.Metadata) error { } unusedCount++ + return callback(bm) }); err != nil { return errors.Wrap(err, "error iterating blobs") } + log.Infof("found %v pack blobs not in use", unusedCount) return nil diff --git a/repo/content/content_manager_lock_free.go b/repo/content/content_manager_lock_free.go index 72e5d43fd..5bf632c89 100644 --- a/repo/content/content_manager_lock_free.go +++ b/repo/content/content_manager_lock_free.go @@ -91,13 +91,17 @@ func (bm *lockFreeManager) loadPackIndexesUnlocked(ctx context.Context) ([]Index for _, b := range contents { contentIDs = append(contentIDs, b.BlobID) } + var updated bool + updated, err = bm.committedContents.use(contentIDs) if err != nil { return nil, false, err } + return contents, updated, nil } + if err != blob.ErrBlobNotFound { return nil, false, err } @@ -111,17 +115,20 @@ func (bm *lockFreeManager) tryLoadPackIndexBlobsUnlocked(ctx context.Context, co if err != nil { return err } + if len(ch) == 0 { return nil } log.Infof("downloading %v new index blobs (%v bytes)...", len(ch), unprocessedIndexesSize) + var wg sync.WaitGroup errch := make(chan error, parallelFetches) for i := 0; i < parallelFetches; i++ { wg.Add(1) + go func() { defer wg.Done() @@ -147,6 +154,7 @@ func (bm *lockFreeManager) tryLoadPackIndexBlobsUnlocked(ctx context.Context, co for err := range errch { return err } + log.Infof("Index contents downloaded.") return nil @@ -155,19 +163,24 @@ func (bm *lockFreeManager) tryLoadPackIndexBlobsUnlocked(ctx context.Context, co // unprocessedIndexBlobsUnlocked returns a closed channel filled with content IDs that are not in committedContents cache. func (bm *lockFreeManager) unprocessedIndexBlobsUnlocked(contents []IndexBlobInfo) (resultCh <-chan blob.ID, totalSize int64, err error) { ch := make(chan blob.ID, len(contents)) + for _, c := range contents { has, err := bm.committedContents.cache.hasIndexBlobID(c.BlobID) if err != nil { return nil, 0, err } + if has { log.Debugf("index blob %q already in cache, skipping", c.BlobID) continue } + ch <- c.BlobID totalSize += c.Length } + close(ch) + return ch, totalSize, nil } @@ -250,6 +263,7 @@ func (bm *lockFreeManager) preparePackDataContent(ctx context.Context, pp *pendi packFileIndex := packIndexBuilder{} haveContent := false + for contentID, info := range pp.currentPackItems { if info.Payload == nil { // no payload, it's a deletion of a previously-committed content. @@ -260,6 +274,7 @@ func (bm *lockFreeManager) preparePackDataContent(ctx context.Context, pp *pendi haveContent = true var encrypted []byte + encrypted, err = bm.maybeEncryptContentDataForPacking(info.Payload, info.ID) if err != nil { return nil, nil, errors.Wrapf(err, "unable to encrypt %q", contentID) @@ -305,6 +320,7 @@ func (bm *lockFreeManager) preparePackDataContent(ctx context.Context, pp *pendi contentData, err = bm.appendPackFileIndexRecoveryData(contentData, packFileIndex) formatLog.Debugf("finished content %v bytes (%v bytes index)", len(contentData), len(contentData)-origContentLength) + return contentData, packFileIndex, err } @@ -329,6 +345,7 @@ func (bm *lockFreeManager) getIndexBlobInternal(ctx context.Context, blobID blob payload, err = bm.encryptor.Decrypt(payload, iv) atomic.AddInt64(&bm.stats.DecryptedBytes, int64(len(payload))) + if err != nil { return nil, err } @@ -350,6 +367,7 @@ func getIndexBlobIV(s blob.ID) ([]byte, error) { if p := strings.Index(string(s), "-"); p >= 0 { // nolint:gocritic s = s[0:p] } + return hex.DecodeString(string(s[len(s)-(aes.BlockSize*2):])) } @@ -357,6 +375,7 @@ func (bm *lockFreeManager) writePackFileNotLocked(ctx context.Context, packFile atomic.AddInt32(&bm.stats.WrittenContents, 1) atomic.AddInt64(&bm.stats.WrittenBytes, int64(len(data))) bm.listCache.deleteListCache() + return bm.st.PutBlob(ctx, packFile, data) } @@ -366,6 +385,7 @@ func (bm *lockFreeManager) encryptAndWriteContentNotLocked(ctx context.Context, // Encrypt the content in-place. atomic.AddInt64(&bm.stats.EncryptedBytes, int64(len(data))) + data2, err := bm.encryptor.Encrypt(data, hash) if err != nil { return "", err @@ -374,6 +394,7 @@ func (bm *lockFreeManager) encryptAndWriteContentNotLocked(ctx context.Context, atomic.AddInt32(&bm.stats.WrittenContents, 1) atomic.AddInt64(&bm.stats.WrittenBytes, int64(len(data2))) bm.listCache.deleteListCache() + if err := bm.st.PutBlob(ctx, blobID, data2); err != nil { return "", err } @@ -386,6 +407,7 @@ func (bm *lockFreeManager) hashData(data []byte) []byte { contentID := bm.hasher(data) atomic.AddInt32(&bm.stats.HashedContents, 1) atomic.AddInt64(&bm.stats.HashedBytes, int64(len(data))) + return contentID } @@ -396,12 +418,14 @@ func (bm *lockFreeManager) writePackIndexesNew(ctx context.Context, data []byte) func (bm *lockFreeManager) verifyChecksum(data, contentID []byte) error { expected := bm.hasher(data) expected = expected[len(expected)-aes.BlockSize:] + if !bytes.HasSuffix(contentID, expected) { atomic.AddInt32(&bm.stats.InvalidContents, 1) return errors.Errorf("invalid checksum for blob %x, expected %x", contentID, expected) } atomic.AddInt32(&bm.stats.ValidContents, 1) + return nil } @@ -417,6 +441,7 @@ func CreateHashAndEncryptor(f *FormattingOptions) (HashFunc, Encryptor, error) { } contentID := h(nil) + _, err = e.Encrypt(nil, contentID) if err != nil { return nil, nil, errors.Wrap(err, "invalid encryptor") diff --git a/repo/content/content_manager_test.go b/repo/content/content_manager_test.go index c9817b286..349f50363 100644 --- a/repo/content/content_manager_test.go +++ b/repo/content/content_manager_test.go @@ -41,6 +41,7 @@ func TestContentManagerEmptyFlush(t *testing.T) { keyTime := map[blob.ID]time.Time{} bm := newTestContentManager(data, keyTime, nil) bm.Flush(ctx) + if got, want := len(data), 0; got != want { t.Errorf("unexpected number of contents: %v, wanted %v", got, want) } @@ -53,9 +54,11 @@ func TestContentZeroBytes1(t *testing.T) { bm := newTestContentManager(data, keyTime, nil) contentID := writeContentAndVerify(ctx, t, bm, []byte{}) bm.Flush(ctx) + if got, want := len(data), 2; got != want { t.Errorf("unexpected number of contents: %v, wanted %v", got, want) } + dumpContentManagerData(t, data) bm = newTestContentManager(data, keyTime, nil) verifyContent(ctx, t, bm, contentID, []byte{}) @@ -69,6 +72,7 @@ func TestContentZeroBytes2(t *testing.T) { writeContentAndVerify(ctx, t, bm, seededRandomData(10, 10)) writeContentAndVerify(ctx, t, bm, []byte{}) bm.Flush(ctx) + if got, want := len(data), 2; got != want { t.Errorf("unexpected number of contents: %v, wanted %v", got, want) dumpContentManagerData(t, data) @@ -84,10 +88,13 @@ func TestContentManagerSmallContentWrites(t *testing.T) { for i := 0; i < 100; i++ { writeContentAndVerify(ctx, t, bm, seededRandomData(i, 10)) } + if got, want := len(data), 0; got != want { t.Errorf("unexpected number of contents: %v, wanted %v", got, want) } + bm.Flush(ctx) + if got, want := len(data), 2; got != want { t.Errorf("unexpected number of contents: %v, wanted %v", got, want) } @@ -102,10 +109,13 @@ func TestContentManagerDedupesPendingContents(t *testing.T) { for i := 0; i < 100; i++ { writeContentAndVerify(ctx, t, bm, seededRandomData(0, 999)) } + if got, want := len(data), 0; got != want { t.Errorf("unexpected number of contents: %v, wanted %v", got, want) } + bm.Flush(ctx) + if got, want := len(data), 2; got != want { t.Errorf("unexpected number of contents: %v, wanted %v", got, want) } @@ -121,6 +131,7 @@ func TestContentManagerDedupesPendingAndUncommittedContents(t *testing.T) { writeContentAndVerify(ctx, t, bm, seededRandomData(0, 950)) writeContentAndVerify(ctx, t, bm, seededRandomData(1, 950)) writeContentAndVerify(ctx, t, bm, seededRandomData(2, 10)) + if got, want := len(data), 0; got != want { t.Errorf("unexpected number of contents: %v, wanted %v", got, want) } @@ -129,9 +140,11 @@ func TestContentManagerDedupesPendingAndUncommittedContents(t *testing.T) { writeContentAndVerify(ctx, t, bm, seededRandomData(0, 950)) writeContentAndVerify(ctx, t, bm, seededRandomData(1, 950)) writeContentAndVerify(ctx, t, bm, seededRandomData(2, 10)) + if got, want := len(data), 0; got != want { t.Errorf("unexpected number of contents: %v, wanted %v", got, want) } + bm.Flush(ctx) // this flushes the pack content + index blob @@ -226,6 +239,7 @@ func TestContentManagerWriteMultiple(t *testing.T) { for i := 0; i < 5000; i++ { b := seededRandomData(i, i%113) + blkID, err := bm.WriteContent(ctx, b, "") if err != nil { t.Errorf("err: %v", err) @@ -243,6 +257,7 @@ func TestContentManagerWriteMultiple(t *testing.T) { if err := bm.Flush(ctx); err != nil { t.Fatalf("error flushing: %v", err) } + bm = newTestContentManager(data, keyTime, timeFunc) } @@ -250,6 +265,7 @@ func TestContentManagerWriteMultiple(t *testing.T) { if _, err := bm.GetContent(ctx, contentIDs[pos]); err != nil { dumpContentManagerData(t, data) t.Fatalf("can't read content %q: %v", contentIDs[pos], err) + continue } } @@ -278,6 +294,7 @@ func TestContentManagerFailedToWritePack(t *testing.T) { if err != nil { t.Fatalf("can't create bm: %v", err) } + logging.SetLevel(logging.DEBUG, "faulty-storage") faulty.Faults = map[string][]*blobtesting.Fault{ @@ -363,6 +380,7 @@ func TestContentManagerConcurrency(t *testing.T) { }); err != nil { t.Errorf("compaction error: %v", err) } + if got, want := getIndexCount(data), 1; got != want { t.Errorf("unexpected index count after compaction: %v, wanted %v", got, want) } @@ -374,6 +392,7 @@ func TestContentManagerConcurrency(t *testing.T) { verifyContent(ctx, t, bm5, bm1content, seededRandomData(31, 100)) verifyContent(ctx, t, bm5, bm2content, seededRandomData(32, 100)) verifyContent(ctx, t, bm5, bm3content, seededRandomData(33, 100)) + if err := bm5.CompactIndexes(ctx, CompactOptions{ MinSmallBlobs: 1, MaxSmallBlobs: 1, @@ -387,26 +406,36 @@ func TestDeleteContent(t *testing.T) { data := blobtesting.DataMap{} keyTime := map[blob.ID]time.Time{} bm := newTestContentManager(data, keyTime, nil) + content1 := writeContentAndVerify(ctx, t, bm, seededRandomData(10, 100)) + if err := bm.Flush(ctx); err != nil { t.Fatalf("error flushing: %v", err) } + dumpContents(t, bm, "after first flush") + content2 := writeContentAndVerify(ctx, t, bm, seededRandomData(11, 100)) + log.Infof("xxx deleting.") + if err := bm.DeleteContent(content1); err != nil { t.Fatalf("unable to delete content %v: %v", content1, err) } + log.Infof("yyy deleting.") + if err := bm.DeleteContent(content2); err != nil { t.Fatalf("unable to delete content %v: %v", content2, err) } + verifyContentNotFound(ctx, t, bm, content1) verifyContentNotFound(ctx, t, bm, content2) log.Infof("flushing") bm.Flush(ctx) log.Infof("flushed") log.Debugf("-----------") + bm = newTestContentManager(data, keyTime, nil) verifyContentNotFound(ctx, t, bm, content1) verifyContentNotFound(ctx, t, bm, content2) @@ -414,6 +443,7 @@ func TestDeleteContent(t *testing.T) { func TestParallelWrites(t *testing.T) { t.Parallel() + ctx := context.Background() data := blobtesting.DataMap{} @@ -432,22 +462,25 @@ func TestParallelWrites(t *testing.T) { }, }, } - bm := newTestContentManagerWithStorage(fs, nil) - - numWorkers := 8 var workersWG sync.WaitGroup + + var workerLock sync.RWMutex + + bm := newTestContentManagerWithStorage(fs, nil) + numWorkers := 8 closeWorkers := make(chan bool) // workerLock allows workers to append to their own list of IDs (when R-locked) in parallel. // W-lock allows flusher to capture the state without any worker being able to modify it. - var workerLock sync.RWMutex workerWritten := make([][]ID, numWorkers) // start numWorkers, each writing random block and recording it for workerID := 0; workerID < numWorkers; workerID++ { workerID := workerID + workersWG.Add(1) + go func() { defer workersWG.Done() for { @@ -465,8 +498,11 @@ func TestParallelWrites(t *testing.T) { } closeFlusher := make(chan bool) + var flusherWG sync.WaitGroup + flusherWG.Add(1) + go func() { defer flusherWG.Done() for { @@ -510,6 +546,7 @@ func TestParallelWrites(t *testing.T) { func TestFlushResumesWriters(t *testing.T) { t.Parallel() + ctx := context.Background() data := blobtesting.DataMap{} @@ -531,10 +568,13 @@ func TestFlushResumesWriters(t *testing.T) { bm := newTestContentManagerWithStorage(fs, nil) first := writeContentAndVerify(ctx, t, bm, []byte{1, 2, 3}) + var second ID var writeWG sync.WaitGroup + writeWG.Add(1) + go func() { defer writeWG.Done() // start a write while flush is ongoing, the write will block on the condition variable @@ -569,6 +609,7 @@ func verifyAllDataPresent(t *testing.T, data map[blob.ID][]byte, contentIDs map[ delete(contentIDs, ci.ID) return nil }) + if len(contentIDs) != 0 { t.Errorf("some blocks not written: %v", contentIDs) } @@ -594,6 +635,7 @@ func TestHandleWriteErrors(t *testing.T) { }) } } + return result } @@ -617,6 +659,7 @@ func TestHandleWriteErrors(t *testing.T) { for n, tc := range cases { tc := tc + t.Run(fmt.Sprintf("case-%v", n), func(t *testing.T) { data := blobtesting.DataMap{} keyTime := map[blob.ID]time.Time{} @@ -701,6 +744,7 @@ func TestDisableFlush(t *testing.T) { bm := newTestContentManager(data, keyTime, nil) bm.DisableIndexFlush() bm.DisableIndexFlush() + for i := 0; i < 500; i++ { writeContentAndVerify(ctx, t, bm, seededRandomData(i, 100)) } @@ -845,6 +889,7 @@ func TestIterateContents(t *testing.T) { if err := bm.DeleteContent(contentID4); err != nil { t.Fatalf("error deleting content 4 %v", err) } + t.Logf("contentID1: %v", contentID1) t.Logf("contentID2: %v", contentID2) t.Logf("contentID3: %v", contentID3) @@ -961,16 +1006,21 @@ func TestFindUnreferencedBlobs(t *testing.T) { bm := newTestContentManager(data, keyTime, nil) verifyUnreferencedBlobsCount(ctx, t, bm, 0) contentID := writeContentAndVerify(ctx, t, bm, seededRandomData(10, 100)) + log.Infof("flushing") + if err := bm.Flush(ctx); err != nil { t.Errorf("flush error: %v", err) } + dumpContents(t, bm, "after flush #1") dumpContentManagerData(t, data) verifyUnreferencedBlobsCount(ctx, t, bm, 0) + if err := bm.DeleteContent(contentID); err != nil { t.Errorf("error deleting content: %v", contentID) } + if err := bm.Flush(ctx); err != nil { t.Errorf("flush error: %v", err) } @@ -981,14 +1031,18 @@ func TestFindUnreferencedBlobs(t *testing.T) { verifyUnreferencedBlobsCount(ctx, t, bm, 0) assertNoError(t, bm.RewriteContent(ctx, contentID)) + if err := bm.Flush(ctx); err != nil { t.Errorf("flush error: %v", err) } + verifyUnreferencedBlobsCount(ctx, t, bm, 1) assertNoError(t, bm.RewriteContent(ctx, contentID)) + if err := bm.Flush(ctx); err != nil { t.Errorf("flush error: %v", err) } + verifyUnreferencedBlobsCount(ctx, t, bm, 2) } @@ -1001,18 +1055,24 @@ func TestFindUnreferencedBlobs2(t *testing.T) { contentID := writeContentAndVerify(ctx, t, bm, seededRandomData(10, 100)) writeContentAndVerify(ctx, t, bm, seededRandomData(11, 100)) dumpContents(t, bm, "after writing") + if err := bm.Flush(ctx); err != nil { t.Errorf("flush error: %v", err) } + dumpContents(t, bm, "after flush") verifyUnreferencedBlobsCount(ctx, t, bm, 0) + if err := bm.DeleteContent(contentID); err != nil { t.Errorf("error deleting content: %v", contentID) } + dumpContents(t, bm, "after delete") + if err := bm.Flush(ctx); err != nil { t.Errorf("flush error: %v", err) } + dumpContents(t, bm, "after flush") // content present in first pack, original pack is still referenced verifyUnreferencedBlobsCount(ctx, t, bm, 0) @@ -1020,8 +1080,11 @@ func TestFindUnreferencedBlobs2(t *testing.T) { func dumpContents(t *testing.T, bm *Manager, caption string) { t.Helper() + count := 0 + log.Infof("dumping %v contents", caption) + if err := bm.IterateContents(IterateOptions{IncludeDeleted: true}, func(ci Info) error { log.Debugf(" ci[%v]=%#v", count, ci) @@ -1031,12 +1094,15 @@ func(ci Info) error { t.Errorf("error listing contents: %v", err) return } + log.Infof("finished dumping %v %v contents", count, caption) } func verifyUnreferencedBlobsCount(ctx context.Context, t *testing.T, bm *Manager, want int) { t.Helper() + var unrefCount int32 + err := bm.IterateUnreferencedBlobs(ctx, 1, func(_ blob.Metadata) error { atomic.AddInt32(&unrefCount, 1) return nil @@ -1046,6 +1112,7 @@ func verifyUnreferencedBlobsCount(ctx context.Context, t *testing.T, bm *Manager } log.Infof("got %v expecting %v", unrefCount, want) + if got := int(unrefCount); got != want { t.Errorf("invalid number of unreferenced contents: %v, wanted %v", got, want) } @@ -1062,10 +1129,15 @@ func TestContentWriteAliasing(t *testing.T) { contentData[0] = 101 id2 := writeContentAndVerify(ctx, t, bm, contentData) bm.Flush(ctx) + contentData[0] = 102 + id3 := writeContentAndVerify(ctx, t, bm, contentData) + contentData[0] = 103 + id4 := writeContentAndVerify(ctx, t, bm, contentData) + verifyContent(ctx, t, bm, id1, []byte{100, 0, 0}) verifyContent(ctx, t, bm, id2, []byte{101, 0, 0}) verifyContent(ctx, t, bm, id3, []byte{102, 0, 0}) @@ -1080,6 +1152,7 @@ func TestContentReadAliasing(t *testing.T) { contentData := []byte{100, 0, 0} id1 := writeContentAndVerify(ctx, t, bm, contentData) + contentData2, err := bm.GetContent(ctx, id1) if err != nil { t.Fatalf("can't get content data: %v", err) @@ -1119,21 +1192,25 @@ func verifyVersionCompat(t *testing.T, writeVersion int) { if err != nil { t.Fatalf("unable to write %v bytes: %v", len(data), err) } + dataSet[cid] = data } verifyContentManagerDataSet(ctx, t, mgr, dataSet) // delete random 3 items (map iteration order is random) cnt := 0 + for blobID := range dataSet { t.Logf("deleting %v", blobID) assertNoError(t, mgr.DeleteContent(blobID)) delete(dataSet, blobID) cnt++ + if cnt >= 3 { break } } + if err := mgr.Flush(ctx); err != nil { t.Fatalf("failed to flush: %v", err) } @@ -1150,9 +1227,11 @@ func verifyVersionCompat(t *testing.T, writeVersion int) { }); err != nil { t.Fatalf("unable to compact indexes: %v", err) } + if err := mgr.Flush(ctx); err != nil { t.Fatalf("failed to flush: %v", err) } + verifyContentManagerDataSet(ctx, t, mgr, dataSet) // now open one more manager @@ -1183,6 +1262,7 @@ func newTestContentManagerWithStorage(st blob.Storage, timeFunc func() time.Time if timeFunc == nil { timeFunc = fakeTimeNowWithAutoAdvance(fakeTime, 1*time.Second) } + bm, err := newManagerWithOptions(context.Background(), st, &FormattingOptions{ Hash: "HMAC-SHA256", Encryption: "NONE", @@ -1193,7 +1273,9 @@ func newTestContentManagerWithStorage(st blob.Storage, timeFunc func() time.Time if err != nil { panic("can't create content manager: " + err.Error()) } + bm.checkInvariantsOnUnlock = true + return bm } @@ -1215,6 +1297,7 @@ func fakeTimeNowFrozen(t time.Time) func() time.Time { func fakeTimeNowWithAutoAdvance(t time.Time, dt time.Duration) func() time.Time { var mu sync.Mutex + return func() time.Time { mu.Lock() defer mu.Unlock() @@ -1288,6 +1371,7 @@ func flushWithRetries(ctx context.Context, t *testing.T, bm *Manager) int { if err != nil { t.Errorf("err: %v", err) } + return retryCount } @@ -1300,6 +1384,7 @@ func writeContentWithRetriesAndVerify(ctx context.Context, t *testing.T, bm *Man log.Warningf("WriteContent failed %v, retrying", err) contentID, err = bm.WriteContent(ctx, b, "") } + if err != nil { t.Errorf("err: %v", err) } @@ -1317,18 +1402,21 @@ func seededRandomData(seed, length int) []byte { b := make([]byte, length) rnd := rand.New(rand.NewSource(int64(seed))) rnd.Read(b) + return b } func hashValue(b []byte) string { h := hmac.New(sha256.New, hmacSecret) h.Write(b) //nolint:errcheck + return hex.EncodeToString(h.Sum(nil)) } func dumpContentManagerData(t *testing.T, data blobtesting.DataMap) { t.Helper() log.Infof("***data - %v items", len(data)) + for k, v := range data { if k[0] == 'n' { ndx, err := openPackIndex(bytes.NewReader(v)) @@ -1343,5 +1431,6 @@ func dumpContentManagerData(t *testing.T, data blobtesting.DataMap) { log.Infof("non-index %v (%v bytes)\n", k, len(v)) } } + log.Infof("*** end of data") } diff --git a/repo/content/format.go b/repo/content/format.go index 1b7edcaf9..5f5ae0763 100644 --- a/repo/content/format.go +++ b/repo/content/format.go @@ -43,6 +43,7 @@ func (e *entry) parse(b []byte) error { e.packFileOffset = binary.BigEndian.Uint32(b[8:12]) e.packedOffset = binary.BigEndian.Uint32(b[12:16]) e.packedLength = binary.BigEndian.Uint32(b[16:20]) + return nil } diff --git a/repo/content/index.go b/repo/content/index.go index b2e7261c3..ede2fb90e 100644 --- a/repo/content/index.go +++ b/repo/content/index.go @@ -63,8 +63,10 @@ func (b *index) Iterate(prefix ID, cb func(Info) error) error { if err != nil { return errors.Wrap(err, "could not find starting position") } + stride := b.hdr.keySize + b.hdr.valueSize entry := make([]byte, stride) + for i := startPos; i < b.hdr.entryCount; i++ { n, err := b.readerAt.ReadAt(entry, int64(8+stride*i)) if err != nil || n != len(entry) { @@ -78,20 +80,25 @@ func (b *index) Iterate(prefix ID, cb func(Info) error) error { if err != nil { return errors.Wrap(err, "invalid index data") } + if !strings.HasPrefix(string(i.ID), string(prefix)) { break } + if err := cb(i); err != nil { return err } } + return nil } func (b *index) findEntryPosition(contentID ID) (int, error) { stride := b.hdr.keySize + b.hdr.valueSize entryBuf := make([]byte, stride) + var readErr error + pos := sort.Search(b.hdr.entryCount, func(p int) bool { if readErr != nil { return false @@ -113,12 +120,14 @@ func (b *index) findEntry(contentID ID) ([]byte, error) { if len(key) != b.hdr.keySize { return nil, errors.Errorf("invalid content ID: %q", contentID) } + stride := b.hdr.keySize + b.hdr.valueSize position, err := b.findEntryPosition(contentID) if err != nil { return nil, err } + if position >= b.hdr.entryCount { return nil, nil } @@ -150,6 +159,7 @@ func (b *index) GetInfo(contentID ID) (*Info, error) { if err != nil { return nil, err } + return &i, err } @@ -164,6 +174,7 @@ func (b *index) entryToInfo(contentID ID, entryData []byte) (Info, error) { } packFile := make([]byte, e.PackFileLength()) + n, err := b.readerAt.ReadAt(packFile, int64(e.PackFileOffset())) if err != nil || n != int(e.PackFileLength()) { return Info{}, errors.Wrap(err, "can't read pack content ID") @@ -195,5 +206,6 @@ func openPackIndex(readerAt io.ReaderAt) (packIndex, error) { if err != nil { return nil, errors.Wrap(err, "invalid header") } + return &index{hdr: h, readerAt: readerAt}, nil } diff --git a/repo/content/list_cache.go b/repo/content/list_cache.go index a0e32ed83..7d1ef9b13 100644 --- a/repo/content/list_cache.go +++ b/repo/content/list_cache.go @@ -43,6 +43,7 @@ func (c *listCache) listIndexBlobs(ctx context.Context) ([]IndexBlobInfo, error) Timestamp: time.Now(), }) } + log.Debugf("found %v index blobs from source", len(contents)) return contents, err @@ -52,12 +53,15 @@ func (c *listCache) saveListToCache(ci *cachedList) { if c.cacheFile == "" { return } + log.Debugf("saving index blobs to cache: %v", len(ci.Contents)) + if data, err := json.Marshal(ci); err == nil { mySuffix := fmt.Sprintf(".tmp-%v-%v", os.Getpid(), time.Now().UnixNano()) if err := ioutil.WriteFile(c.cacheFile+mySuffix, appendHMAC(data, c.hmacSecret), 0600); err != nil { log.Warningf("unable to write list cache: %v", err) } + os.Rename(c.cacheFile+mySuffix, c.cacheFile) //nolint:errcheck os.Remove(c.cacheFile + mySuffix) //nolint:errcheck } @@ -111,6 +115,7 @@ func listIndexBlobsFromStorage(ctx context.Context, st blob.Storage) ([]IndexBlo } var results []IndexBlobInfo + for _, it := range snapshot { ii := IndexBlobInfo{ BlobID: it.BlobID, diff --git a/repo/content/merged.go b/repo/content/merged.go index 33e00a3b5..2ae02d62f 100644 --- a/repo/content/merged.go +++ b/repo/content/merged.go @@ -22,17 +22,20 @@ func (m mergedIndex) Close() error { // GetInfo returns information about a single content. If a content is not found, returns (nil,nil) func (m mergedIndex) GetInfo(id ID) (*Info, error) { var best *Info + for _, ndx := range m { i, err := ndx.GetInfo(id) if err != nil { return nil, err } + if i != nil { if best == nil || i.TimestampSeconds > best.TimestampSeconds || (i.TimestampSeconds == best.TimestampSeconds && !i.Deleted) { best = i } } } + return best, nil } @@ -65,11 +68,13 @@ func (h *nextInfoHeap) Pop() interface{} { n := len(old) x := old[n-1] *h = old[0 : n-1] + return x } func iterateChan(prefix ID, ndx packIndex, done chan bool) <-chan Info { ch := make(chan Info, 16) + go func() { defer close(ch) @@ -82,6 +87,7 @@ func iterateChan(prefix ID, ndx packIndex, done chan bool) <-chan Info { } }) }() + return ch } @@ -89,11 +95,14 @@ func iterateChan(prefix ID, ndx packIndex, done chan bool) <-chan Info { // all contents have been visited or until an error is returned by the callback. func (m mergedIndex) Iterate(prefix ID, cb func(i Info) error) error { var minHeap nextInfoHeap + done := make(chan bool) + defer close(done) for _, ndx := range m { ch := iterateChan(prefix, ndx, done) + it, ok := <-ch if ok { heap.Push(&minHeap, &nextInfo{it, ch}) diff --git a/repo/content/merged_test.go b/repo/content/merged_test.go index 8d420fad7..b718b4697 100644 --- a/repo/content/merged_test.go +++ b/repo/content/merged_test.go @@ -18,6 +18,7 @@ func TestMerged(t *testing.T) { if err != nil { t.Fatalf("can't create index: %v", err) } + i2, err := indexWithItems( Info{ID: "aabbcc", TimestampSeconds: 3, PackBlobID: "yy", PackOffset: 33}, Info{ID: "xaabbcc", TimestampSeconds: 1, PackBlobID: "xx", PackOffset: 111}, @@ -26,6 +27,7 @@ func TestMerged(t *testing.T) { if err != nil { t.Fatalf("can't create index: %v", err) } + i3, err := indexWithItems( Info{ID: "aabbcc", TimestampSeconds: 2, PackBlobID: "zz", PackOffset: 22}, Info{ID: "ddeeff", TimestampSeconds: 1, PackBlobID: "zz", PackOffset: 222}, @@ -37,15 +39,18 @@ func TestMerged(t *testing.T) { } m := mergedIndex{i1, i2, i3} + i, err := m.GetInfo("aabbcc") if err != nil || i == nil { t.Fatalf("unable to get info: %v", err) } + if got, want := i.PackOffset, uint32(33); got != want { t.Errorf("invalid pack offset %v, wanted %v", got, want) } var inOrder []ID + assertNoError(t, m.Iterate("", func(i Info) error { inOrder = append(inOrder, i.ID) if i.ID == "de1e1e" { @@ -82,12 +87,15 @@ func TestMerged(t *testing.T) { func indexWithItems(items ...Info) (packIndex, error) { b := make(packIndexBuilder) + for _, it := range items { b.Add(it) } + var buf bytes.Buffer if err := b.Build(&buf); err != nil { return nil, errors.Wrap(err, "build error") } + return openPackIndex(bytes.NewReader(buf.Bytes())) } diff --git a/repo/content/packindex_internal_test.go b/repo/content/packindex_internal_test.go index 01deb779b..65f5ac116 100644 --- a/repo/content/packindex_internal_test.go +++ b/repo/content/packindex_internal_test.go @@ -14,8 +14,8 @@ func TestRoundTrip(t *testing.T) { for _, tc := range cases { b := contentIDToBytes(tc) - got := bytesToContentID(b) - if got != tc { + + if got := bytesToContentID(b); got != tc { t.Errorf("%q did not round trip, got %q, wanted %q", tc, got, tc) } } diff --git a/repo/content/packindex_test.go b/repo/content/packindex_test.go index 8a1d86deb..9a1ed848c 100644 --- a/repo/content/packindex_test.go +++ b/repo/content/packindex_test.go @@ -21,28 +21,34 @@ func deterministicContentID(prefix string, id int) ID { if id%2 == 0 { prefix2 = "x" } + if id%7 == 0 { prefix2 = "y" } + if id%5 == 0 { prefix2 = "m" } + return ID(fmt.Sprintf("%v%x", prefix2, h.Sum(nil))) } func deterministicPackBlobID(id int) blob.ID { h := sha1.New() fmt.Fprintf(h, "%v", id) + return blob.ID(fmt.Sprintf("%x", h.Sum(nil))) } func deterministicPackedOffset(id int) uint32 { s := rand.NewSource(int64(id + 1)) rnd := rand.New(s) + return uint32(rnd.Int31()) } func deterministicPackedLength(id int) uint32 { s := rand.NewSource(int64(id + 2)) rnd := rand.New(s) + return uint32(rnd.Int31()) } func deterministicFormatVersion(id int) byte { @@ -93,18 +99,20 @@ func TestPackIndex(t *testing.T) { b3.Add(info) } - var buf1 bytes.Buffer - var buf2 bytes.Buffer - var buf3 bytes.Buffer + var buf1, buf2, buf3 bytes.Buffer + if err := b1.Build(&buf1); err != nil { t.Errorf("unable to build: %v", err) } + if err := b1.Build(&buf2); err != nil { t.Errorf("unable to build: %v", err) } + if err := b1.Build(&buf3); err != nil { t.Errorf("unable to build: %v", err) } + data1 := buf1.Bytes() data2 := buf2.Bytes() data3 := buf3.Bytes() @@ -112,6 +120,7 @@ func TestPackIndex(t *testing.T) { if !reflect.DeepEqual(data1, data2) { t.Errorf("builder output not stable: %x vs %x", hex.Dump(data1), hex.Dump(data2)) } + if !reflect.DeepEqual(data2, data3) { t.Errorf("builder output not stable: %x vs %x", hex.Dump(data2), hex.Dump(data3)) } @@ -132,12 +141,14 @@ func TestPackIndex(t *testing.T) { t.Errorf("unable to find %v", info.ID) continue } + if !reflect.DeepEqual(info, *info2) { t.Errorf("invalid value retrieved: %+v, wanted %+v", info2, info) } } cnt := 0 + assertNoError(t, ndx.Iterate("", func(info2 Info) error { info := infoMap[info2.ID] if !reflect.DeepEqual(info, info2) { @@ -146,6 +157,7 @@ func TestPackIndex(t *testing.T) { cnt++ return nil })) + if cnt != len(infoMap) { t.Errorf("invalid number of iterations: %v, wanted %v", cnt, len(infoMap)) } @@ -154,10 +166,12 @@ func TestPackIndex(t *testing.T) { for i := 0; i < 100; i++ { contentID := deterministicContentID("no-such-content", i) + v, err := ndx.GetInfo(contentID) if err != nil { t.Errorf("unable to get content %v: %v", contentID, err) } + if v != nil { t.Errorf("unexpected result when getting content %v: %v", contentID, v) } @@ -222,6 +236,7 @@ func fuzzTest(rnd *rand.Rand, originalData []byte, rounds int, callback func(d [ sectionsToDelete := rnd.Intn(3) for i := 0; i < sectionsToDelete; i++ { pos := rnd.Intn(len(data)) + deletedLength := rnd.Intn(10) if pos+deletedLength > len(data) { continue diff --git a/repo/crypto_key_derivation.go b/repo/crypto_key_derivation.go index ed97e0bab..572dcaaaf 100644 --- a/repo/crypto_key_derivation.go +++ b/repo/crypto_key_derivation.go @@ -29,5 +29,6 @@ func deriveKeyFromMasterKey(masterKey, uniqueID, purpose []byte, length int) []b key := make([]byte, length) k := hkdf.New(sha256.New, masterKey, uniqueID, purpose) io.ReadFull(k, key) //nolint:errcheck + return key } diff --git a/repo/format_block.go b/repo/format_block.go index bc7c476db..f5b3a00d0 100644 --- a/repo/format_block.go +++ b/repo/format_block.go @@ -106,6 +106,7 @@ func recoverFormatBlobWithLength(ctx context.Context, st blob.Storage, blobID bl if err != nil { return nil, err } + if l := int(prefixChunk[0]) + int(prefixChunk[1])<<8; l <= maxChecksummedFormatBytesLength && l+2 < len(prefixChunk) { if b, ok := verifyFormatBlobChecksum(prefixChunk[2 : 2+l]); ok { return b, nil @@ -117,6 +118,7 @@ func recoverFormatBlobWithLength(ctx context.Context, st blob.Storage, blobID bl if err != nil { return nil, err } + if l := int(suffixChunk[len(suffixChunk)-2]) + int(suffixChunk[len(suffixChunk)-1])<<8; l <= maxChecksummedFormatBytesLength && l+2 < len(suffixChunk) { if b, ok := verifyFormatBlobChecksum(suffixChunk[len(suffixChunk)-2-l : len(suffixChunk)-2]); ok { return b, nil @@ -136,6 +138,7 @@ func verifyFormatBlobChecksum(b []byte) ([]byte, bool) { h := hmac.New(sha256.New, formatBlobChecksumSecret) h.Write(data) //nolint:errcheck actualChecksum := h.Sum(nil) + if !hmac.Equal(actualChecksum, checksum) { return nil, false } @@ -147,6 +150,7 @@ func writeFormatBlob(ctx context.Context, st blob.Storage, f *formatBlob) error var buf bytes.Buffer e := json.NewEncoder(&buf) e.SetIndent("", " ") + if err := e.Encode(f); err != nil { return errors.Wrap(err, "unable to marshal format blob") } @@ -173,6 +177,7 @@ func (f *formatBlob) decryptFormatBytes(masterKey []byte) (*repositoryObjectForm if len(content) < aead.NonceSize() { return nil, errors.Errorf("invalid encrypted payload, too short") } + nonce := content[0:aead.NonceSize()] payload := content[aead.NonceSize():] @@ -201,6 +206,7 @@ func initCrypto(masterKey, repositoryID []byte) (cipher.AEAD, []byte, error) { if err != nil { return nil, nil, errors.Wrap(err, "cannot create cipher") } + aead, err := cipher.NewGCM(blk) if err != nil { return nil, nil, errors.Wrap(err, "cannot create cipher") @@ -220,10 +226,12 @@ func encryptFormatBytes(f *formatBlob, format *repositoryObjectFormat, masterKey if err != nil { return errors.Wrap(err, "can't marshal format to JSON") } + aead, authData, err := initCrypto(masterKey, repositoryID) if err != nil { return errors.Wrap(err, "unable to initialize crypto") } + nonceLength := aead.NonceSize() noncePlusContentLength := nonceLength + len(content) cipherText := make([]byte, noncePlusContentLength+aead.Overhead()) @@ -237,6 +245,7 @@ func encryptFormatBytes(f *formatBlob, format *repositoryObjectFormat, masterKey b := aead.Seal(cipherText[nonceLength:nonceLength], nonce, content, authData) content = nonce[0 : nonceLength+len(b)] f.EncryptedFormatBytes = content + return nil default: @@ -258,5 +267,6 @@ func addFormatBlobChecksumAndLength(fb []byte) ([]byte, error) { result := append([]byte(nil), byte(l), byte(l>>8)) result = append(result, checksummedFormatBytes...) result = append(result, byte(l), byte(l>>8)) + return result, nil } diff --git a/repo/format_block_test.go b/repo/format_block_test.go index 58bc05153..c4541e3d4 100644 --- a/repo/format_block_test.go +++ b/repo/format_block_test.go @@ -16,10 +16,12 @@ func TestFormatBlobRecovery(t *testing.T) { ctx := context.Background() someDataBlock := []byte("aadsdasdas") + checksummed, err := addFormatBlobChecksumAndLength(someDataBlock) if err != nil { t.Errorf("error appending checksum: %v", err) } + if got, want := len(checksummed), 2+2+sha256.Size+len(someDataBlock); got != want { t.Errorf("unexpected checksummed length: %v, want %v", got, want) } @@ -74,6 +76,7 @@ func TestFormatBlobRecovery(t *testing.T) { func assertNoError(t *testing.T, err error) { t.Helper() + if err != nil { t.Errorf("err: %v", err) } diff --git a/repo/initialize.go b/repo/initialize.go index 9ba5019a8..4a9765728 100644 --- a/repo/initialize.go +++ b/repo/initialize.go @@ -38,11 +38,13 @@ func Initialize(ctx context.Context, st blob.Storage, opt *NewRepositoryOptions, if err == nil { return errors.Errorf("repository already initialized") } + if err != blob.ErrBlobNotFound { return err } format := formatBlobFromOptions(opt) + masterKey, err := format.deriveMasterKeyFromPassword(password) if err != nil { return errors.Wrap(err, "unable to derive master key") @@ -101,6 +103,7 @@ func repositoryObjectFormatFromOptions(opt *NewRepositoryOptions) *repositoryObj func randomBytes(n int) []byte { b := make([]byte, n) io.ReadFull(rand.Reader, b) //nolint:errcheck + return b } diff --git a/repo/local_config.go b/repo/local_config.go index 7cad53346..0b764d041 100644 --- a/repo/local_config.go +++ b/repo/local_config.go @@ -34,7 +34,9 @@ func (lc *LocalConfig) Save(w io.Writer) error { if err != nil { return nil } + _, err = w.Write(b) + return err } diff --git a/repo/manifest/manifest_manager.go b/repo/manifest/manifest_manager.go index a30df80a4..11fe55d6f 100644 --- a/repo/manifest/manifest_manager.go +++ b/repo/manifest/manifest_manager.go @@ -60,6 +60,7 @@ func (m *Manager) Put(ctx context.Context, labels map[string]string, payload int if err := m.ensureInitialized(ctx); err != nil { return "", err } + m.mu.Lock() defer m.mu.Unlock() @@ -143,6 +144,7 @@ func (m *Manager) GetRaw(ctx context.Context, id ID) ([]byte, error) { if e == nil { e = m.committedEntries[id] } + if e == nil || e.Deleted { return nil, ErrNotFound } @@ -160,11 +162,13 @@ func (m *Manager) Find(ctx context.Context, labels map[string]string) ([]*EntryM defer m.mu.Unlock() var matches []*EntryMetadata + for _, e := range m.pendingEntries { if matchesLabels(e.Labels, labels) { matches = append(matches, cloneEntryMetadata(e)) } } + for _, e := range m.committedEntries { if m.pendingEntries[e.ID] != nil { // ignore committed that are also in pending @@ -179,6 +183,7 @@ func (m *Manager) Find(ctx context.Context, labels map[string]string) ([]*EntryM sort.Slice(matches, func(i, j int) bool { return matches[i].ModTime.Before(matches[j].ModTime) }) + return matches, nil } @@ -208,6 +213,7 @@ func (m *Manager) Flush(ctx context.Context) error { defer m.mu.Unlock() _, err := m.flushPendingEntriesLocked(ctx) + return err } @@ -264,6 +270,7 @@ func (m *Manager) Delete(ctx context.Context, id ID) error { ModTime: time.Now().UTC(), Deleted: true, } + return nil } @@ -303,10 +310,12 @@ func (m *Manager) loadCommittedContentsLocked(ctx context.Context) error { // success break } + if err == content.ErrContentNotFound { // try again, lost a race with another manifest manager which just did compaction continue } + return errors.Wrap(err, "unable to load manifest contents") } @@ -343,6 +352,7 @@ func (m *Manager) loadManifestContentsLocked(manifests map[content.ID]manifest) func (m *Manager) loadManifestContent(ctx context.Context, contentID content.ID) (manifest, error) { man := manifest{} + blk, err := m.b.GetContent(ctx, contentID) if err != nil { // do not wrap the error here, we want to propagate original ErrNotFound @@ -376,6 +386,7 @@ func (m *Manager) maybeCompactLocked(ctx context.Context) error { } log.Debugf("performing automatic compaction of %v contents", len(m.committedContentIDs)) + if err := m.compactLocked(ctx); err != nil { return errors.Wrap(err, "unable to compact manifest contents") } @@ -450,6 +461,7 @@ func (m *Manager) ensureInitialized(ctx context.Context) error { } m.initialized = true + return nil } @@ -458,6 +470,7 @@ func copyLabels(m map[string]string) map[string]string { for k, v := range m { r[k] = v } + return r } diff --git a/repo/manifest/manifest_manager_test.go b/repo/manifest/manifest_manager_test.go index 0e4915bb2..22b1242bf 100644 --- a/repo/manifest/manifest_manager_test.go +++ b/repo/manifest/manifest_manager_test.go @@ -46,6 +46,7 @@ func TestManifest(t *testing.T) { for _, tc := range cases { verifyMatches(ctx, t, mgr, tc.criteria, tc.expected) } + verifyItem(ctx, t, mgr, id1, labels1, item1) verifyItem(ctx, t, mgr, id2, labels2, item2) verifyItem(ctx, t, mgr, id3, labels3, item3) @@ -53,6 +54,7 @@ func TestManifest(t *testing.T) { if err := mgr.Flush(ctx); err != nil { t.Errorf("flush error: %v", err) } + if err := mgr.Flush(ctx); err != nil { t.Errorf("flush error: %v", err) } @@ -61,6 +63,7 @@ func TestManifest(t *testing.T) { for _, tc := range cases { verifyMatches(ctx, t, mgr, tc.criteria, tc.expected) } + verifyItem(ctx, t, mgr, id1, labels1, item1) verifyItem(ctx, t, mgr, id2, labels2, item2) verifyItem(ctx, t, mgr, id3, labels3, item3) @@ -68,27 +71,33 @@ func TestManifest(t *testing.T) { // flush underlying content manager and verify in new manifest manager. mgr.b.Flush(ctx) mgr2 := newManagerForTesting(ctx, t, data) + for _, tc := range cases { verifyMatches(ctx, t, mgr2, tc.criteria, tc.expected) } + verifyItem(ctx, t, mgr2, id1, labels1, item1) verifyItem(ctx, t, mgr2, id2, labels2, item2) verifyItem(ctx, t, mgr2, id3, labels3, item3) + if err := mgr2.Flush(ctx); err != nil { t.Errorf("flush error: %v", err) } // delete from one time.Sleep(1 * time.Second) + if err := mgr.Delete(ctx, id3); err != nil { t.Errorf("delete error: %v", err) } + verifyItemNotFound(ctx, t, mgr, id3) mgr.Flush(ctx) verifyItemNotFound(ctx, t, mgr, id3) // still found in another verifyItem(ctx, t, mgr2, id3, labels3, item3) + if err := mgr2.loadCommittedContentsLocked(ctx); err != nil { t.Errorf("unable to load: %v", err) } @@ -98,6 +107,7 @@ func TestManifest(t *testing.T) { } foundContents := 0 + if err := mgr.b.IterateContents( content.IterateOptions{Prefix: ContentPrefix}, func(ci content.Info) error { @@ -106,6 +116,7 @@ func(ci content.Info) error { }); err != nil { t.Errorf("unable to list manifest content: %v", err) } + if got, want := foundContents, 1; got != want { t.Errorf("unexpected number of blocks: %v, want %v", got, want) } @@ -200,6 +211,7 @@ func TestManifestInitCorruptedBlock(t *testing.T) { func addAndVerify(ctx context.Context, t *testing.T, mgr *Manager, labels map[string]string, data map[string]int) ID { t.Helper() + id, err := mgr.Put(ctx, labels, data) if err != nil { t.Errorf("unable to add %v (%v): %v", labels, data, err) @@ -207,6 +219,7 @@ func addAndVerify(ctx context.Context, t *testing.T, mgr *Manager, labels map[st } verifyItem(ctx, t, mgr, id, labels, data) + return id } @@ -247,14 +260,17 @@ func verifyMatches(ctx context.Context, t *testing.T, mgr *Manager, labels map[s t.Helper() var matches []ID + items, err := mgr.Find(ctx, labels) if err != nil { t.Errorf("error in Find(): %v", err) return } + for _, m := range items { matches = append(matches, m.ID) } + sortIDs(matches) sortIDs(expected) diff --git a/repo/object/object_manager.go b/repo/object/object_manager.go index 78b671435..3b63dc847 100644 --- a/repo/object/object_manager.go +++ b/repo/object/object_manager.go @@ -86,6 +86,7 @@ func (om *Manager) Open(ctx context.Context, objectID ID) (Reader, error) { // and returns the total length of the object and content IDs of which it is composed. func (om *Manager) VerifyObject(ctx context.Context, oid ID) (int64, []content.ID, error) { tracker := &contentIDTracker{} + l, err := om.verifyObjectInternal(ctx, oid, tracker) if err != nil { return 0, nil, err @@ -98,6 +99,7 @@ func (om *Manager) verifyIndirectObjectInternal(ctx context.Context, indexObject if _, err := om.verifyObjectInternal(ctx, indexObjectID, tracker); err != nil { return 0, errors.Wrap(err, "unable to read index") } + rd, err := om.Open(ctx, indexObjectID) if err != nil { return 0, err @@ -121,6 +123,7 @@ func (om *Manager) verifyIndirectObjectInternal(ctx context.Context, indexObject } totalLength := seekTable[len(seekTable)-1].endOffset() + return totalLength, nil } @@ -134,12 +137,13 @@ func (om *Manager) verifyObjectInternal(ctx context.Context, oid ID, tracker *co if err != nil { return 0, err } + tracker.addContentID(contentID) + return int64(p.Length), nil } return 0, errors.Errorf("unrecognized object type: %v", oid) - } func nullTrace(message string, args ...interface{}) { @@ -212,6 +216,7 @@ func (om *Manager) newRawReader(ctx context.Context, objectID ID) (Reader, error if err == content.ErrContentNotFound { return nil, ErrObjectNotFound } + if err != nil { return nil, errors.Wrap(err, "unexpected content error") } diff --git a/repo/object/object_manager_test.go b/repo/object/object_manager_test.go index 994b64a8a..d627c4353 100644 --- a/repo/object/object_manager_test.go +++ b/repo/object/object_manager_test.go @@ -44,6 +44,7 @@ func (f *fakeContentManager) WriteContent(ctx context.Context, data []byte, pref defer f.mu.Unlock() f.data[contentID] = append([]byte(nil), data...) + return contentID, nil } @@ -135,6 +136,7 @@ func TestWriterCompleteChunkInTwoWrites(t *testing.T) { writer.Write(b[0:50]) //nolint:errcheck writer.Write(b[0:50]) //nolint:errcheck result, err := writer.Result() + if !objectIDsEqual(result, "cd00e292c5970d3c5e2f0ffa5171e555bc46bfc4faddfb4a418b6840b86e79a3") { t.Errorf("unexpected result: %v err: %v", result, err) } @@ -183,9 +185,11 @@ func TestIndirection(t *testing.T) { writer := om.NewWriter(ctx, WriterOptions{}) writer.(*objectWriter).splitter = splitterFactory() + if _, err := writer.Write(contentBytes); err != nil { t.Errorf("write error: %v", err) } + result, err := writer.Result() if err != nil { t.Errorf("error getting writer results: %v", err) @@ -236,6 +240,7 @@ func TestHMAC(t *testing.T) { w := om.NewWriter(ctx, WriterOptions{}) w.Write(c) //nolint:errcheck result, err := w.Result() + if result.String() != "cad29ff89951a3c085c86cb7ed22b82b51f7bdfda24f932c7f9601f51d5975ba" { t.Errorf("unexpected result: %v err: %v", result.String(), err) } @@ -273,6 +278,7 @@ func TestReader(t *testing.T) { t.Errorf("cannot read all data for %v: %v", objectID, err) continue } + if !bytes.Equal(d, c.payload) { t.Errorf("incorrect payload for %v: expected: %v got: %v", objectID, c.payload, d) continue @@ -288,6 +294,7 @@ func TestReaderStoredBlockNotFound(t *testing.T) { if err != nil { t.Errorf("cannot parse object ID: %v", err) } + reader, err := om.Open(ctx, objectID) if err != ErrObjectNotFound || reader != nil { t.Errorf("unexpected result: reader: %v err: %v", reader, err) @@ -307,8 +314,11 @@ func TestEndToEndReadAndSeek(t *testing.T) { if _, err := writer.Write(randomData); err != nil { t.Errorf("write error: %v", err) } + objectID, err := writer.Result() + writer.Close() + if err != nil { t.Errorf("cannot get writer result for %v: %v", size, err) continue @@ -320,6 +330,7 @@ func TestEndToEndReadAndSeek(t *testing.T) { func verify(ctx context.Context, t *testing.T, om *Manager, objectID ID, expectedData []byte, testCaseID string) { t.Helper() + reader, err := om.Open(ctx, objectID) if err != nil { t.Errorf("cannot get reader for %v (%v): %v %v", testCaseID, objectID, err, string(debug.Stack())) @@ -330,14 +341,18 @@ func verify(ctx context.Context, t *testing.T, om *Manager, objectID ID, expecte for i := 0; i < 20; i++ { sampleSize := int(rand.Int31n(300)) seekOffset := int(rand.Int31n(int32(len(expectedData)))) + if seekOffset+sampleSize > len(expectedData) { sampleSize = len(expectedData) - seekOffset } + if sampleSize > 0 { got := make([]byte, sampleSize) + if offset, err := reader.Seek(int64(seekOffset), 0); err != nil || offset != int64(seekOffset) { t.Errorf("seek error: %v offset=%v expected:%v", err, offset, seekOffset) } + if n, err := reader.Read(got); err != nil || n != sampleSize { t.Errorf("invalid data: n=%v, expected=%v, err:%v", n, sampleSize, err) } diff --git a/repo/object/object_reader.go b/repo/object/object_reader.go index 99d1eda57..b1f8935da 100644 --- a/repo/object/object_reader.go +++ b/repo/object/object_reader.go @@ -37,9 +37,10 @@ func (r *objectReader) Read(buffer []byte) (int, error) { if r.currentChunkData != nil { toCopy := len(r.currentChunkData) - r.currentChunkPosition if toCopy == 0 { - // EOF on curren chunk + // EOF on current chunk r.closeCurrentChunk() r.currentChunkIndex++ + continue } @@ -49,10 +50,12 @@ func (r *objectReader) Read(buffer []byte) (int, error) { copy(buffer[readBytes:], r.currentChunkData[r.currentChunkPosition:r.currentChunkPosition+toCopy]) + r.currentChunkPosition += toCopy r.currentPosition += int64(toCopy) readBytes += toCopy remaining -= toCopy + continue } @@ -75,10 +78,12 @@ func (r *objectReader) Read(buffer []byte) (int, error) { func (r *objectReader) openCurrentChunk() error { st := r.seekTable[r.currentChunkIndex] + rd, err := r.repo.Open(r.ctx, st.Object) if err != nil { return err } + defer rd.Close() //nolint:errcheck b := make([]byte, st.Length) @@ -88,6 +93,7 @@ func (r *objectReader) openCurrentChunk() error { r.currentChunkData = b r.currentChunkPosition = 0 + return nil } @@ -98,6 +104,7 @@ func (r *objectReader) closeCurrentChunk() { func (r *objectReader) findChunkIndexForOffset(offset int64) (int, error) { left := 0 right := len(r.seekTable) - 1 + for left <= right { middle := (left + right) / 2 @@ -130,6 +137,7 @@ func (r *objectReader) Seek(offset int64, whence int) (int64, error) { r.currentChunkIndex = len(r.seekTable) r.currentChunkData = nil r.currentPosition = offset + return offset, nil } @@ -139,6 +147,7 @@ func (r *objectReader) Seek(offset int64, whence int) (int64, error) { } chunkStartOffset := r.seekTable[index].Start + if index != r.currentChunkIndex { r.closeCurrentChunk() r.currentChunkIndex = index diff --git a/repo/object/object_splitter.go b/repo/object/object_splitter.go index 931f3dd2d..39e0207f8 100644 --- a/repo/object/object_splitter.go +++ b/repo/object/object_splitter.go @@ -58,6 +58,7 @@ func init() { for k := range splitterFactories { SupportedSplitters = append(SupportedSplitters, k) } + sort.Strings(SupportedSplitters) } diff --git a/repo/object/object_splitter_test.go b/repo/object/object_splitter_test.go index 45bcfafe2..a3399bb63 100644 --- a/repo/object/object_splitter_test.go +++ b/repo/object/object_splitter_test.go @@ -34,6 +34,7 @@ func TestSplitters(t *testing.T) { func TestSplitterStability(t *testing.T) { r := rand.New(rand.NewSource(5)) rnd := make([]byte, 5000000) + if n, err := r.Read(rnd); n != len(rnd) || err != nil { t.Fatalf("can't initialize random data: %v", err) } @@ -67,18 +68,23 @@ func TestSplitterStability(t *testing.T) { maxSplit := 0 minSplit := int(math.MaxInt32) count := 0 + for i, p := range rnd { if !s.ShouldSplit(p) { continue } + l := i - lastSplit if l >= maxSplit { maxSplit = l } + if l < minSplit { minSplit = l } + count++ + lastSplit = i } @@ -94,9 +100,11 @@ func TestSplitterStability(t *testing.T) { if got, want := count, tc.count; got != want { t.Errorf("invalid split count %v, wanted %v", got, want) } + if got, want := minSplit, tc.minSplit; got != want { t.Errorf("min split %v, wanted %v", got, want) } + if got, want := maxSplit, tc.maxSplit; got != want { t.Errorf("max split %v, wanted %v", got, want) } diff --git a/repo/object/object_writer.go b/repo/object/object_writer.go index 1a97654fb..dbeb83f69 100644 --- a/repo/object/object_writer.go +++ b/repo/object/object_writer.go @@ -32,6 +32,7 @@ func (t *contentIDTracker) addContentID(contentID content.ID) { if t.contents == nil { t.contents = make(map[content.ID]bool) } + t.contents[contentID] = true } @@ -43,6 +44,7 @@ func (t *contentIDTracker) contentIDs() []content.ID { for k := range t.contents { result = append(result, k) } + return result } @@ -92,16 +94,19 @@ func (w *objectWriter) flushBuffer() error { w.currentPosition += int64(length) var b2 bytes.Buffer + w.buffer.WriteTo(&b2) //nolint:errcheck w.buffer.Reset() contentID, err := w.repo.contentMgr.WriteContent(w.ctx, b2.Bytes(), w.prefix) w.repo.trace("OBJECT_WRITER(%q) stored %v (%v bytes)", w.description, contentID, length) + if err != nil { return errors.Wrapf(err, "error when flushing chunk %d of %s", chunkID, w.description) } w.indirectIndex[chunkID].Object = DirectObjectID(contentID) + return nil } @@ -132,10 +137,12 @@ func (w *objectWriter) Result() (ID, error) { if err := json.NewEncoder(iw).Encode(ind); err != nil { return "", errors.Wrap(err, "unable to write indirect object index") } + oid, err := iw.Result() if err != nil { return "", err } + return IndirectObjectID(oid), nil } diff --git a/repo/object/objectid.go b/repo/object/objectid.go index a26a6cb66..4d487c7d2 100644 --- a/repo/object/objectid.go +++ b/repo/object/objectid.go @@ -40,6 +40,7 @@ func (i ID) ContentID() (content.ID, bool) { if strings.HasPrefix(string(i), "D") { return content.ID(i[1:]), true } + if strings.HasPrefix(string(i), "I") { return "", false } @@ -67,6 +68,7 @@ func (i ID) Validate() error { if contentID[0] < 'g' || contentID[0] > 'z' { return errors.Errorf("invalid content ID prefix: %v", contentID) } + contentID = contentID[1:] } diff --git a/repo/object/splitter_buzhash32.go b/repo/object/splitter_buzhash32.go index 5d22555fa..c84ea5ac7 100644 --- a/repo/object/splitter_buzhash32.go +++ b/repo/object/splitter_buzhash32.go @@ -18,6 +18,7 @@ type buzhash32Splitter struct { func (rs *buzhash32Splitter) ShouldSplit(b byte) bool { rs.rh.Roll(b) rs.count++ + if rs.rh.Sum32()&rs.mask == 0 && rs.count >= rs.minSize { rs.count = 0 return true diff --git a/repo/object/splitter_fixed.go b/repo/object/splitter_fixed.go index cce6dcacb..9735ded4e 100644 --- a/repo/object/splitter_fixed.go +++ b/repo/object/splitter_fixed.go @@ -7,6 +7,7 @@ type fixedSplitter struct { func (s *fixedSplitter) ShouldSplit(b byte) bool { s.cur++ + if s.cur >= s.chunkLength { s.cur = 0 return true diff --git a/repo/object/splitter_rabinkarp64.go b/repo/object/splitter_rabinkarp64.go index c14dea83b..e84e73f5d 100644 --- a/repo/object/splitter_rabinkarp64.go +++ b/repo/object/splitter_rabinkarp64.go @@ -16,10 +16,12 @@ type rabinKarp64Splitter struct { func (rs *rabinKarp64Splitter) ShouldSplit(b byte) bool { rs.rh.Roll(b) rs.count++ + if rs.rh.Sum64()&rs.mask == 0 && rs.count >= rs.minSize { rs.count = 0 return true } + if rs.count >= rs.maxSize { rs.count = 0 return true diff --git a/repo/open.go b/repo/open.go index 2dca008fb..e95b4f817 100644 --- a/repo/open.go +++ b/repo/open.go @@ -155,6 +155,7 @@ func (r *Repository) SetCachingConfig(opt content.CachingOptions) error { func readAndCacheFormatBlobBytes(ctx context.Context, st blob.Storage, cacheDirectory string) ([]byte, error) { cachedFile := filepath.Join(cacheDirectory, "kopia.repository") + if cacheDirectory != "" { b, err := ioutil.ReadFile(cachedFile) if err == nil { diff --git a/repo/repository.go b/repo/repository.go index 8c7c06117..fbbdcb21b 100644 --- a/repo/repository.go +++ b/repo/repository.go @@ -31,12 +31,15 @@ func (r *Repository) Close(ctx context.Context) error { if err := r.Manifests.Flush(ctx); err != nil { return errors.Wrap(err, "error flushing manifests") } + if err := r.Content.Close(ctx); err != nil { return errors.Wrap(err, "error closing content-addressable storage manager") } + if err := r.Blobs.Close(ctx); err != nil { return errors.Wrap(err, "error closing blob storage") } + return nil } diff --git a/repo/repository_test.go b/repo/repository_test.go index 941b69a22..2102f3941 100644 --- a/repo/repository_test.go +++ b/repo/repository_test.go @@ -58,6 +58,7 @@ func objectIDsEqual(o1, o2 object.ID) bool { func TestWriterCompleteChunkInTwoWrites(t *testing.T) { var env repotesting.Environment defer env.Setup(t).Close(t) + ctx := context.Background() b := make([]byte, 100) @@ -65,6 +66,7 @@ func TestWriterCompleteChunkInTwoWrites(t *testing.T) { writer.Write(b[0:50]) //nolint:errcheck writer.Write(b[0:50]) //nolint:errcheck result, err := writer.Result() + if result != "1d804f1f69df08f3f59070bf962de69433e3d61ac18522a805a84d8c92741340" { t.Errorf("unexpected result: %v err: %v", result, err) } @@ -97,15 +99,19 @@ func TestPackingSimple(t *testing.T) { if got, want := oid1a.String(), oid1b.String(); got != want { t.Errorf("oid1a(%q) != oid1b(%q)", got, want) } + if got, want := oid1a.String(), oid1c.String(); got != want { t.Errorf("oid1a(%q) != oid1c(%q)", got, want) } + if got, want := oid2a.String(), oid2b.String(); got != want { t.Errorf("oid2(%q)a != oidb(%q)", got, want) } + if got, want := oid2a.String(), oid2c.String(); got != want { t.Errorf("oid2(%q)a != oidc(%q)", got, want) } + if got, want := oid3a.String(), oid3b.String(); got != want { t.Errorf("oid3a(%q) != oid3b(%q)", got, want) } @@ -142,6 +148,7 @@ func TestPackingSimple(t *testing.T) { func TestHMAC(t *testing.T) { var env repotesting.Environment defer env.Setup(t).Close(t) + ctx := context.Background() c := bytes.Repeat([]byte{0xcd}, 50) @@ -149,6 +156,7 @@ func TestHMAC(t *testing.T) { w := env.Repository.Objects.NewWriter(ctx, object.WriterOptions{}) w.Write(c) //nolint:errcheck result, err := w.Result() + if result.String() != "367352007ee6ca9fa755ce8352347d092c17a24077fd33c62f655574a8cf906d" { t.Errorf("unexpected result: %v err: %v", result.String(), err) } @@ -157,6 +165,7 @@ func TestHMAC(t *testing.T) { func TestUpgrade(t *testing.T) { var env repotesting.Environment defer env.Setup(t).Close(t) + ctx := context.Background() if err := env.Repository.Upgrade(ctx); err != nil { @@ -171,12 +180,14 @@ func TestUpgrade(t *testing.T) { func TestReaderStoredBlockNotFound(t *testing.T) { var env repotesting.Environment defer env.Setup(t).Close(t) + ctx := context.Background() objectID, err := object.ParseID("Ddeadbeef") if err != nil { t.Errorf("cannot parse object ID: %v", err) } + reader, err := env.Repository.Objects.Open(ctx, objectID) if err != object.ErrObjectNotFound || reader != nil { t.Errorf("unexpected result: reader: %v err: %v", reader, err) @@ -187,8 +198,8 @@ func writeObject(ctx context.Context, t *testing.T, rep *repo.Repository, data [ w := rep.Objects.NewWriter(ctx, object.WriterOptions{}) if _, err := w.Write(data); err != nil { t.Fatalf("can't write object %q - write failed: %v", testCaseID, err) - } + oid, err := w.Result() if err != nil { t.Fatalf("can't write object %q - result failed: %v", testCaseID, err) @@ -199,6 +210,7 @@ func writeObject(ctx context.Context, t *testing.T, rep *repo.Repository, data [ func verify(ctx context.Context, t *testing.T, rep *repo.Repository, objectID object.ID, expectedData []byte, testCaseID string) { t.Helper() + reader, err := rep.Objects.Open(ctx, objectID) if err != nil { t.Errorf("cannot get reader for %v (%v): %v %v", testCaseID, objectID, err, string(debug.Stack())) @@ -209,14 +221,18 @@ func verify(ctx context.Context, t *testing.T, rep *repo.Repository, objectID ob for i := 0; i < 20; i++ { sampleSize := int(rand.Int31n(300)) seekOffset := int(rand.Int31n(int32(len(expectedData)))) + if seekOffset+sampleSize > len(expectedData) { sampleSize = len(expectedData) - seekOffset } + if sampleSize > 0 { got := make([]byte, sampleSize) + if offset, err := reader.Seek(int64(seekOffset), 0); err != nil || offset != int64(seekOffset) { t.Errorf("seek error: %v offset=%v expected:%v", err, offset, seekOffset) } + if n, err := reader.Read(got); err != nil || n != sampleSize { t.Errorf("invalid data: n=%v, expected=%v, err:%v", n, sampleSize, err) } @@ -275,10 +291,12 @@ func TestFormats(t *testing.T) { bytesToWrite := []byte(k) w := env.Repository.Objects.NewWriter(ctx, object.WriterOptions{}) w.Write(bytesToWrite) //nolint:errcheck + oid, err := w.Result() if err != nil { t.Errorf("error: %v", err) } + if !objectIDsEqual(oid, v) { t.Errorf("invalid oid for #%v\ngot:\n%#v\nexpected:\n%#v", caseIndex, oid.String(), v.String()) } @@ -288,10 +306,12 @@ func TestFormats(t *testing.T) { t.Errorf("open failed: %v", err) continue } + bytesRead, err := ioutil.ReadAll(rc) if err != nil { t.Errorf("error reading: %v", err) } + if !bytes.Equal(bytesRead, bytesToWrite) { t.Errorf("data mismatch, read:%x vs written:%v", bytesRead, bytesToWrite) } diff --git a/repo/upgrade.go b/repo/upgrade.go index 4a9bbfc59..f887ef56f 100644 --- a/repo/upgrade.go +++ b/repo/upgrade.go @@ -11,6 +11,7 @@ func (r *Repository) Upgrade(ctx context.Context) error { f := r.formatBlob log.Debug("decrypting format...") + repoConfig, err := f.decryptFormatBytes(r.masterKey) if err != nil { return errors.Wrap(err, "unable to decrypt repository config") @@ -18,17 +19,19 @@ func (r *Repository) Upgrade(ctx context.Context) error { var migrated bool - // TODO(jkowalski): add migration code here + // add migration code here if !migrated { log.Infof("nothing to do") return nil } log.Debug("encrypting format...") + if err := encryptFormatBytes(f, repoConfig, r.masterKey, f.UniqueID); err != nil { return errors.Errorf("unable to encrypt format bytes") } log.Infof("writing updated format content...") + return writeFormatBlob(ctx, r.Blobs, f) } diff --git a/site/cli2md/cli2md.go b/site/cli2md/cli2md.go index c93205793..0a2ec6128 100644 --- a/site/cli2md/cli2md.go +++ b/site/cli2md/cli2md.go @@ -31,6 +31,7 @@ func emitFlags(w io.Writer, flags []*kingpin.FlagModel) { if len(flags) == 0 { return } + fmt.Fprintf(w, "| Flag | Short | Defaut | Help |\n") fmt.Fprintf(w, "| ---- | ----- | --- | --- |\n") @@ -44,6 +45,7 @@ func emitFlags(w io.Writer, flags []*kingpin.FlagModel) { if f.Short != 0 { shortFlag = "`-" + string([]byte{byte(f.Short)}) + "`" } + defaultValue := "" if len(f.Default) > 0 { defaultValue = f.Default[0] @@ -61,11 +63,13 @@ func emitFlags(w io.Writer, flags []*kingpin.FlagModel) { if defaultValue == "" { defaultValue = "`false`" } + fmt.Fprintf(w, "| `--[no-]%v` | %v | %v | %v%v |\n", f.Name, shortFlag, defaultValue, maybeAdvanced, f.Help) } else { fmt.Fprintf(w, "| `--%v` | %v | %v | %v%v |\n", f.Name, shortFlag, defaultValue, maybeAdvanced, f.Help) } } + fmt.Fprintf(w, "\n") } @@ -89,6 +93,7 @@ func sortFlags(f []*kingpin.FlagModel) []*kingpin.FlagModel { return a.Name < b.Name }) + return f } @@ -96,8 +101,10 @@ func emitArgs(w io.Writer, args []*kingpin.ArgModel) { if len(args) == 0 { return } + fmt.Fprintf(w, "| Argument | Help |\n") fmt.Fprintf(w, "| -------- | --- |\n") + args2 := append([]*kingpin.ArgModel(nil), args...) sort.Slice(args2, func(i, j int) bool { return args2[i].Name < args2[j].Name @@ -106,6 +113,7 @@ func emitArgs(w io.Writer, args []*kingpin.ArgModel) { for _, f := range args2 { fmt.Fprintf(w, "| `%v` | %v |\n", f.Name, f.Help) } + fmt.Fprintf(w, "\n") } @@ -124,6 +132,7 @@ func generateAppFlags(app *kingpin.ApplicationModel) error { --- `, title, title) emitFlags(f, app.Flags) + return nil } @@ -132,6 +141,7 @@ func generateCommands(app *kingpin.ApplicationModel, section string, weight int, if err := os.MkdirAll(dir, 0755); err != nil { return err } + f, err := os.Create(filepath.Join(dir, "_index.md")) if err != nil { return errors.Wrap(err, "unable to create common flags file") @@ -169,6 +179,7 @@ func flattenCommands(cmds []*kingpin.CmdModel) []*kingpin.CmdModel { commonRoot.Commands = append(commonRoot.Commands, c) continue } + root := &kingpin.CmdModel{ Name: c.Name, FullCommand: c.FullCommand, @@ -193,6 +204,7 @@ func flattenChildren(cmd *kingpin.CmdModel, parentFlags []*kingpin.FlagModel, fo if forceHidden { cmdClone.Hidden = true } + cmdClone.Flags = cmdFlags result = append(result, &cmdClone) @@ -207,15 +219,20 @@ func flattenChildren(cmd *kingpin.CmdModel, parentFlags []*kingpin.FlagModel, fo func generateSubcommands(w io.Writer, dir, sectionTitle string, cmds []*kingpin.CmdModel, advanced bool) { cmds = append([]*kingpin.CmdModel(nil), cmds...) + first := true + for _, c := range cmds { if c.Hidden != advanced { continue } + if first { fmt.Fprintf(w, "\n### %v\n\n", sectionTitle) + first = false } + subcommandSlug := strings.Replace(c.FullCommand, " ", "-", -1) fmt.Fprintf(w, "* [`%v`](%v) - %v\n", c.FullCommand, subcommandSlug+"/", c.Help) generateSubcommandPage(filepath.Join(dir, subcommandSlug+".md"), c) diff --git a/snapshot/gc/gc.go b/snapshot/gc/gc.go index d825ff80b..4e4c9ea42 100644 --- a/snapshot/gc/gc.go +++ b/snapshot/gc/gc.go @@ -38,27 +38,33 @@ func findInUseContentIDs(ctx context.Context, rep *repo.Repository, used *sync.M w := snapshotfs.NewTreeWalker() w.EntryID = func(e fs.Entry) interface{} { return oidOf(e) } + for _, m := range manifests { root, err := snapshotfs.SnapshotRoot(rep, m) if err != nil { return errors.Wrap(err, "unable to get snapshot root") } + w.RootEntries = append(w.RootEntries, root) } w.ObjectCallback = func(entry fs.Entry) error { oid := oidOf(entry) _, contentIDs, err := rep.Objects.VerifyObject(ctx, oid) + if err != nil { return errors.Wrapf(err, "error verifying %v", oid) } + for _, cid := range contentIDs { used.Store(cid, nil) } + return nil } log.Info("looking for active contents") + if err := w.Run(ctx); err != nil { return errors.Wrap(err, "error walking snapshot tree") } @@ -67,6 +73,7 @@ func findInUseContentIDs(ctx context.Context, rep *repo.Repository, used *sync.M } // Run performs garbage collection on all the snapshots in the repository. +// nolint:gocognit func Run(ctx context.Context, rep *repo.Repository, minContentAge time.Duration, gcDelete bool) error { var used sync.Map if err := findInUseContentIDs(ctx, rep, &used); err != nil { @@ -74,9 +81,11 @@ func Run(ctx context.Context, rep *repo.Repository, minContentAge time.Duration, } var unusedCount, inUseCount, systemCount, tooRecentCount int32 + var totalUnusedBytes, totalInUseBytes, totalSystemBytes, totalTooRecentBytes int64 log.Info("looking for unreferenced contents") + if err := rep.Content.IterateContents(content.IterateOptions{}, func(ci content.Info) error { if manifest.ContentPrefix == ci.ID.Prefix() { atomic.AddInt32(&systemCount, 1) diff --git a/snapshot/manager.go b/snapshot/manager.go index 8499c0762..63a5c1ad0 100644 --- a/snapshot/manager.go +++ b/snapshot/manager.go @@ -54,6 +54,7 @@ func ListSnapshots(ctx context.Context, rep *repo.Repository, si SourceInfo) ([] if err != nil { return nil, errors.Wrap(err, "unable to find manifest entries") } + return LoadSnapshots(ctx, rep, entryIDs(entries)) } @@ -65,6 +66,7 @@ func loadSnapshot(ctx context.Context, rep *repo.Repository, manifestID manifest } sm.ID = manifestID + return sm, nil } @@ -73,9 +75,11 @@ func SaveSnapshot(ctx context.Context, rep *repo.Repository, man *Manifest) (man if man.Source.Host == "" { return "", errors.New("missing host") } + if man.Source.UserName == "" { return "", errors.New("missing username") } + if man.Source.Path == "" { return "", errors.New("missing path") } @@ -84,7 +88,9 @@ func SaveSnapshot(ctx context.Context, rep *repo.Repository, man *Manifest) (man if err != nil { return "", err } + man.ID = id + return id, nil } @@ -95,6 +101,7 @@ func LoadSnapshots(ctx context.Context, rep *repo.Repository, manifestIDs []mani for i, n := range manifestIDs { sem <- true + go func(i int, n manifest.ID) { defer func() { <-sem }() @@ -113,6 +120,7 @@ func LoadSnapshots(ctx context.Context, rep *repo.Repository, manifestIDs []mani close(sem) successful := result[:0] + for _, m := range result { if m != nil { successful = append(successful, m) @@ -136,6 +144,7 @@ func ListSnapshotManifests(ctx context.Context, rep *repo.Repository, src *Sourc if err != nil { return nil, errors.Wrap(err, "unable to find manifest entries") } + return entryIDs(entries), nil } @@ -144,5 +153,6 @@ func entryIDs(entries []*manifest.EntryMetadata) []manifest.ID { for _, e := range entries { ids = append(ids, e.ID) } + return ids } diff --git a/snapshot/manifest.go b/snapshot/manifest.go index 58962e0a8..4a5ee0b4c 100644 --- a/snapshot/manifest.go +++ b/snapshot/manifest.go @@ -49,6 +49,7 @@ func (p Permissions) MarshalJSON() ([]byte, error) { } s := "0" + strconv.FormatInt(int64(p), 8) + return json.Marshal(&s) } @@ -66,6 +67,7 @@ func (p *Permissions) UnmarshalJSON(b []byte) error { } *p = Permissions(v) + return nil } diff --git a/snapshot/policy/expire.go b/snapshot/policy/expire.go index df6e465c8..f70351620 100644 --- a/snapshot/policy/expire.go +++ b/snapshot/policy/expire.go @@ -33,18 +33,22 @@ func ApplyRetentionPolicy(ctx context.Context, rep *repo.Repository, sourceInfo func getExpiredSnapshots(ctx context.Context, rep *repo.Repository, snapshots []*snapshot.Manifest) ([]*snapshot.Manifest, error) { var toDelete []*snapshot.Manifest + for _, snapshotGroup := range snapshot.GroupBySource(snapshots) { td, err := getExpiredSnapshotsForSource(ctx, rep, snapshotGroup) if err != nil { return nil, err } + toDelete = append(toDelete, td...) } + return toDelete, nil } func getExpiredSnapshotsForSource(ctx context.Context, rep *repo.Repository, snapshots []*snapshot.Manifest) ([]*snapshot.Manifest, error) { src := snapshots[0].Source + pol, _, err := GetEffectivePolicy(ctx, rep, src) if err != nil { return nil, err @@ -53,6 +57,7 @@ func getExpiredSnapshotsForSource(ctx context.Context, rep *repo.Repository, sna pol.RetentionPolicy.ComputeRetentionReasons(snapshots) var toDelete []*snapshot.Manifest + for _, s := range snapshots { if len(s.RetentionReasons) == 0 { log.Debugf(" deleting %v", s.StartTime) @@ -61,5 +66,6 @@ func getExpiredSnapshotsForSource(ctx context.Context, rep *repo.Repository, sna log.Debugf(" keeping %v reasons: [%v]", s.StartTime, strings.Join(s.RetentionReasons, ",")) } } + return toDelete, nil } diff --git a/snapshot/policy/policy.go b/snapshot/policy/policy.go index e5054b383..5c133e69d 100644 --- a/snapshot/policy/policy.go +++ b/snapshot/policy/policy.go @@ -26,9 +26,11 @@ func (p *Policy) String() string { e := json.NewEncoder(&buf) e.SetIndent("", " ") + if err := e.Encode(p); err != nil { log.Warningf("unable to policy as JSON: %v", err) } + return buf.String() } diff --git a/snapshot/policy/policy_manager.go b/snapshot/policy/policy_manager.go index 26426c556..32e89de02 100644 --- a/snapshot/policy/policy_manager.go +++ b/snapshot/policy/policy_manager.go @@ -32,6 +32,7 @@ func GetEffectivePolicy(ctx context.Context, rep *repo.Repository, si snapshot.S if err != nil { return nil, nil, err } + md = append(md, manifests...) parentPath := filepath.Dir(tmp.Path) @@ -47,6 +48,7 @@ func GetEffectivePolicy(ctx context.Context, rep *repo.Repository, si snapshot.S if err != nil { return nil, nil, err } + md = append(md, userHostManifests...) // Try host-level policy. @@ -54,6 +56,7 @@ func GetEffectivePolicy(ctx context.Context, rep *repo.Repository, si snapshot.S if err != nil { return nil, nil, err } + md = append(md, hostManifests...) // Global policy. @@ -61,14 +64,17 @@ func GetEffectivePolicy(ctx context.Context, rep *repo.Repository, si snapshot.S if err != nil { return nil, nil, err } + md = append(md, globalManifests...) var policies []*Policy + for _, em := range md { p := &Policy{} if err := rep.Manifests.Get(ctx, em.ID, &p); err != nil { return nil, nil, errors.Wrapf(err, "got unexpected error when loading policy item %v", em.ID) } + p.Labels = em.Labels policies = append(policies, p) log.Debugf("loaded parent policy for %v: %v", si, p.Target()) @@ -109,6 +115,7 @@ func GetDefinedPolicy(ctx context.Context, rep *repo.Repository, si snapshot.Sou } p.Labels = em.Labels + return p, nil } @@ -176,8 +183,8 @@ func ListPolicies(ctx context.Context, rep *repo.Repository) ([]*Policy, error) for _, id := range ids { pol := &Policy{} - err := rep.Manifests.Get(ctx, id.ID, pol) - if err != nil { + + if err := rep.Manifests.Get(ctx, id.ID, pol); err != nil { return nil, err } @@ -234,12 +241,15 @@ func FilesPolicyGetter(ctx context.Context, rep *repo.Repository, si snapshot.So if err != nil { return nil, errors.Wrap(err, "unable to determine relative path") } + rel = "./" + rel log.Debugf("loading policy for %v (%v)", policyPath, rel) + pol := &Policy{} if err := rep.Manifests.Get(ctx, id.ID, pol); err != nil { return nil, errors.Wrapf(err, "unable to load policy %v", id.ID) } + result[rel] = &pol.FilesPolicy } @@ -275,5 +285,4 @@ func labelsForSource(si snapshot.SourceInfo) map[string]string { "policyType": "global", } } - } diff --git a/snapshot/policy/retention_policy.go b/snapshot/policy/retention_policy.go index 995172767..d77a1792c 100644 --- a/snapshot/policy/retention_policy.go +++ b/snapshot/policy/retention_policy.go @@ -46,6 +46,7 @@ func (r *RetentionPolicy) ComputeRetentionReasons(manifests []*snapshot.Manifest for i, s := range sorted { s.RetentionReasons = r.getRetentionReasons(i, s, cutoff, ids, idCounters) } + for _, s := range sorted { if s.IncompleteReason != "" { s.RetentionReasons = append(s.RetentionReasons, "incomplete") @@ -61,6 +62,7 @@ func (r *RetentionPolicy) getRetentionReasons(i int, s *snapshot.Manifest, cutof } keepReasons := []string{} + var zeroTime time.Time yyyy, wk := s.StartTime.ISOWeek() @@ -83,6 +85,7 @@ func (r *RetentionPolicy) getRetentionReasons(i int, s *snapshot.Manifest, cutof if c.max == nil { continue } + if s.StartTime.Before(c.cutoffTime) { continue } @@ -143,18 +146,23 @@ func (r *RetentionPolicy) Merge(src RetentionPolicy) { if r.KeepLatest == nil { r.KeepLatest = src.KeepLatest } + if r.KeepHourly == nil { r.KeepHourly = src.KeepHourly } + if r.KeepDaily == nil { r.KeepDaily = src.KeepDaily } + if r.KeepWeekly == nil { r.KeepWeekly = src.KeepWeekly } + if r.KeepMonthly == nil { r.KeepMonthly = src.KeepMonthly } + if r.KeepAnnual == nil { r.KeepAnnual = src.KeepAnnual } diff --git a/snapshot/policy/scheduling_policy.go b/snapshot/policy/scheduling_policy.go index 05fad0ab8..45a5153c7 100644 --- a/snapshot/policy/scheduling_policy.go +++ b/snapshot/policy/scheduling_policy.go @@ -19,9 +19,11 @@ func (t *TimeOfDay) Parse(s string) error { if _, err := fmt.Sscanf(s, "%v:%02v", &t.Hour, &t.Minute); err != nil { return errors.New("invalid time of day, must be HH:MM") } + if t.Hour < 0 || t.Hour > 23 { return errors.Errorf("invalid hour %q, must be between 0 and 23", s) } + if t.Minute < 0 || t.Minute > 59 { return errors.Errorf("invalid minute %q, must be between 0 and 59", s) } @@ -67,6 +69,7 @@ func (p *SchedulingPolicy) Merge(src SchedulingPolicy) { if p.IntervalSeconds == 0 { p.IntervalSeconds = src.IntervalSeconds } + p.TimesOfDay = SortAndDedupeTimesOfDay( append(append([]TimeOfDay(nil), src.TimesOfDay...), p.TimesOfDay...)) } diff --git a/snapshot/snapshot_test.go b/snapshot/snapshot_test.go index 94d0d1b8c..ca3816f3f 100644 --- a/snapshot/snapshot_test.go +++ b/snapshot/snapshot_test.go @@ -35,10 +35,12 @@ func TestSnapshotsAPI(t *testing.T) { verifySnapshotManifestIDs(t, env.Repository, &src2, nil) verifyListSnapshots(t, env.Repository, src1, []*snapshot.Manifest{}) verifyListSnapshots(t, env.Repository, src2, []*snapshot.Manifest{}) + manifest1 := &snapshot.Manifest{ Source: src1, Description: "some-description", } + id1 := mustSaveSnapshot(t, env.Repository, manifest1) verifySnapshotManifestIDs(t, env.Repository, nil, []manifest.ID{id1}) verifySnapshotManifestIDs(t, env.Repository, &src1, []manifest.ID{id1}) @@ -49,10 +51,12 @@ func TestSnapshotsAPI(t *testing.T) { Source: src1, Description: "some-other-description", } + id2 := mustSaveSnapshot(t, env.Repository, manifest2) if id1 == id2 { t.Errorf("expected different manifest IDs, got same: %v", id1) } + verifySnapshotManifestIDs(t, env.Repository, nil, []manifest.ID{id1, id2}) verifySnapshotManifestIDs(t, env.Repository, &src1, []manifest.ID{id1, id2}) verifySnapshotManifestIDs(t, env.Repository, &src2, nil) @@ -70,19 +74,20 @@ func TestSnapshotsAPI(t *testing.T) { verifyLoadSnapshots(t, env.Repository, []manifest.ID{id1, id2, id3}, []*snapshot.Manifest{manifest1, manifest2, manifest3}) } -func verifySnapshotManifestIDs(t *testing.T, rep *repo.Repository, src *snapshot.SourceInfo, expected []manifest.ID) []manifest.ID { +func verifySnapshotManifestIDs(t *testing.T, rep *repo.Repository, src *snapshot.SourceInfo, expected []manifest.ID) { t.Helper() + res, err := snapshot.ListSnapshotManifests(context.Background(), rep, src) if err != nil { t.Errorf("error listing snapshot manifests: %v", err) } + sortManifestIDs(res) sortManifestIDs(expected) + if !reflect.DeepEqual(res, expected) { t.Errorf("unexpected manifests: %v, wanted %v", res, expected) - return expected } - return res } func sortManifestIDs(s []manifest.ID) { @@ -93,10 +98,12 @@ func sortManifestIDs(s []manifest.ID) { func mustSaveSnapshot(t *testing.T, rep *repo.Repository, man *snapshot.Manifest) manifest.ID { t.Helper() + id, err := snapshot.SaveSnapshot(context.Background(), rep, man) if err != nil { t.Fatalf("error saving snapshot: %v", err) } + return id } @@ -113,6 +120,7 @@ func verifySources(t *testing.T, rep *repo.Repository, sources ...snapshot.Sourc func verifyListSnapshots(t *testing.T, rep *repo.Repository, src snapshot.SourceInfo, expected []*snapshot.Manifest) { t.Helper() + got, err := snapshot.ListSnapshots(context.Background(), rep, src) if err != nil { t.Errorf("error loading manifests: %v", err) @@ -121,9 +129,11 @@ func verifyListSnapshots(t *testing.T, rep *repo.Repository, src snapshot.Source if !reflect.DeepEqual(got, expected) { t.Errorf("unexpected manifests: %v, wanted %v", got, expected) + for i, m := range got { t.Logf("got[%v]=%#v", i, m) } + for i, m := range expected { t.Logf("want[%v]=%#v", i, m) } @@ -139,9 +149,11 @@ func verifyLoadSnapshots(t *testing.T, rep *repo.Repository, ids []manifest.ID, if !reflect.DeepEqual(got, expected) { t.Errorf("unexpected manifests: %v, wanted %v", got, expected) + for i, m := range got { t.Logf("got[%v]=%#v", i, m) } + for i, m := range expected { t.Logf("want[%v]=%#v", i, m) } @@ -151,6 +163,7 @@ func verifyLoadSnapshots(t *testing.T, rep *repo.Repository, ids []manifest.ID, func sorted(s []string) []string { res := append([]string(nil), s...) sort.Strings(res) + return res } @@ -188,19 +201,21 @@ func TestParseSourceInfo(t *testing.T) { t.Errorf("error parsing %q: %v", tc.path, err) continue } + if got != tc.want { t.Errorf("unexpected parsed value of %q: %v, wanted %v", tc.path, got, tc.want) } + got2, err := snapshot.ParseSourceInfo(got.String(), "default-host", "default-user") if err != nil { t.Errorf("error parsing %q: %v", tc.path, err) continue } + if got != got2 { t.Errorf("unexpected parsed value of %q: %v, wanted %v", got.String(), got2, got) } } - } func TestParseInvalidSourceInfo(t *testing.T) { diff --git a/snapshot/snapshotfs/all_sources.go b/snapshot/snapshotfs/all_sources.go index 59720d160..a1891874e 100644 --- a/snapshot/snapshotfs/all_sources.go +++ b/snapshot/snapshotfs/all_sources.go @@ -67,6 +67,7 @@ func (s *repositoryAllSources) Readdir(ctx context.Context) (fs.Entries, error) } result.Sort() + return result, nil } diff --git a/snapshot/snapshotfs/repofs.go b/snapshot/snapshotfs/repofs.go index de0af2015..2ac4ce8e7 100644 --- a/snapshot/snapshotfs/repofs.go +++ b/snapshot/snapshotfs/repofs.go @@ -120,6 +120,7 @@ func (rsl *repositorySymlink) Readlink(ctx context.Context) (string, error) { } defer r.Close() //nolint:errcheck + b, err := ioutil.ReadAll(r) if err != nil { return "", err diff --git a/snapshot/snapshotfs/snapshot_tree_walker.go b/snapshot/snapshotfs/snapshot_tree_walker.go index 007375fd4..63db8e32d 100644 --- a/snapshot/snapshotfs/snapshot_tree_walker.go +++ b/snapshot/snapshotfs/snapshot_tree_walker.go @@ -59,10 +59,11 @@ func (w *TreeWalker) Run(ctx context.Context) error { for _, root := range w.RootEntries { w.enqueueEntry(ctx, root) } + w.queue.ProgressCallback = func(enqueued, active, completed int64) { log.Infof("processed(%v/%v) active %v", completed, enqueued, active) - } + return w.queue.Process(w.Parallelism) } diff --git a/snapshot/snapshotfs/source_directories.go b/snapshot/snapshotfs/source_directories.go index 6f06f2425..2e8acbc6b 100644 --- a/snapshot/snapshotfs/source_directories.go +++ b/snapshot/snapshotfs/source_directories.go @@ -52,6 +52,7 @@ func (s *sourceDirectories) Readdir(ctx context.Context) (fs.Entries, error) { if err != nil { return nil, err } + var result fs.Entries for _, src := range sources { diff --git a/snapshot/snapshotfs/source_snapshots.go b/snapshot/snapshotfs/source_snapshots.go index c137cd948..8609a877e 100644 --- a/snapshot/snapshotfs/source_snapshots.go +++ b/snapshot/snapshotfs/source_snapshots.go @@ -89,6 +89,7 @@ func (s *sourceSnapshots) Readdir(ctx context.Context) (fs.Entries, error) { result = append(result, e) } + result.Sort() return result, nil diff --git a/snapshot/snapshotfs/upload.go b/snapshot/snapshotfs/upload.go index 097fa52f8..bc5b165c5 100644 --- a/snapshot/snapshotfs/upload.go +++ b/snapshot/snapshotfs/upload.go @@ -107,6 +107,7 @@ func (u *Uploader) uploadFileInternal(ctx context.Context, f fs.File) entryResul if err != nil { return entryResult{err: errors.Wrap(err, "unable to create dir entry")} } + de.FileSize = written return entryResult{de: de} @@ -137,7 +138,9 @@ func (u *Uploader) uploadSymlinkInternal(ctx context.Context, f fs.Symlink) entr if err != nil { return entryResult{err: errors.Wrap(err, "unable to create dir entry")} } + de.FileSize = written + return entryResult{de: de} } @@ -146,13 +149,16 @@ func (u *Uploader) addDirProgress(length int64) { u.currentDirCompleted += length c := u.currentDirCompleted shouldReport := false + if time.Now().After(u.nextProgressReportTime) { shouldReport = true u.nextProgressReportTime = time.Now().Add(100 * time.Millisecond) } + if c == u.currentDirTotalSize { shouldReport = true } + u.progressMutex.Unlock() if shouldReport { @@ -177,13 +183,16 @@ func (u *Uploader) copyWithProgress(dst io.Writer, src io.Reader, completed, len written += int64(wroteBytes) completed += int64(wroteBytes) u.addDirProgress(int64(wroteBytes)) + if length < completed { length = completed } } + if writeErr != nil { return written, writeErr } + if readBytes != wroteBytes { return written, io.ErrShortWrite } @@ -238,6 +247,7 @@ func (u *Uploader) uploadFile(ctx context.Context, file fs.File) (*snapshot.DirE if err != nil { return nil, errors.Wrap(err, "unable to create dir entry") } + de.DirSummary = &fs.DirectorySummary{ TotalFileCount: 1, TotalFileSize: res.de.FileSize, @@ -262,6 +272,7 @@ func (u *Uploader) uploadDir(ctx context.Context, rootDir fs.Directory, previous } de.DirSummary = &summ + return de, err } @@ -360,15 +371,19 @@ func metadataEquals(e1, e2 fs.Entry) bool { if l, r := e1.ModTime(), e2.ModTime(); !l.Equal(r) { return false } + if l, r := e1.Mode(), e2.Mode(); l != r { return false } + if l, r := e1.Size(), e2.Size(); l != r { return false } + if l, r := e1.Owner(), e2.Owner(); l != r { return false } + return true } @@ -382,7 +397,9 @@ func findCachedEntry(entry fs.Entry, prevEntries []fs.Entries) fs.Entry { log.Debugf("found non-matching entry for %v: %v %v %v", entry.Name(), ent.Mode(), ent.Size(), ent.ModTime()) } } + log.Debugf("could not find cache entry for %v", entry.Name()) + return nil } @@ -390,6 +407,7 @@ func findCachedEntry(entry fs.Entry, prevEntries []fs.Entries) fs.Entry { func objectIDPercent(obj object.ID) int { h := fnv.New32a() io.WriteString(h, obj.String()) //nolint:errcheck + return int(h.Sum32() % 100) } @@ -399,6 +417,7 @@ func (u *Uploader) maybeIgnoreCachedEntry(ent fs.Entry) fs.Entry { log.Debugf("ignoring valid cached object: %v", h.ObjectID()) return nil } + return ent } @@ -477,6 +496,7 @@ func (u *Uploader) prepareWorkItems(ctx context.Context, dirRelativePath string, func toChannel(items []*uploadWorkItem) <-chan *uploadWorkItem { ch := make(chan *uploadWorkItem) + go func() { defer close(ch) @@ -500,8 +520,10 @@ func (u *Uploader) launchWorkItems(workItems []*uploadWorkItem, wg *sync.WaitGro } ch := toChannel(workItems) + for i := 0; i < workerCount; i++ { wg.Add(1) + go func() { defer wg.Done() @@ -514,6 +536,7 @@ func (u *Uploader) launchWorkItems(workItems []*uploadWorkItem, wg *sync.WaitGro func (u *Uploader) processUploadWorkItems(workItems []*uploadWorkItem, dirManifest *snapshot.DirManifest) error { var wg sync.WaitGroup + u.launchWorkItems(workItems, &wg) // Read result channels in order. @@ -528,8 +551,10 @@ func (u *Uploader) processUploadWorkItems(workItems []*uploadWorkItem, dirManife if u.IgnoreFileErrors { u.stats.ReadErrors++ log.Warningf("unable to hash file %q: %s, ignoring", it.entryRelativePath, result.err) + continue } + return errors.Errorf("unable to process %q: %s", it.entryRelativePath, result.err) } @@ -574,6 +599,7 @@ func uniqueDirectories(dirs []fs.Directory) []fs.Directory { for _, d := range unique { result = append(result, d) } + return result } @@ -594,18 +620,23 @@ func uploadDirInternal( }() log.Debugf("reading directory %v", dirRelativePath) + entries, direrr := directory.Readdir(ctx) + log.Debugf("finished reading directory %v", dirRelativePath) + if direrr != nil { return "", fs.DirectorySummary{}, direrr } var prevEntries []fs.Entries + for _, d := range uniqueDirectories(previousDirs) { if ent := maybeReadDirectoryEntries(ctx, d); ent != nil { prevEntries = append(prevEntries, ent) } } + if len(entries) == 0 { summ.MaxModTime = directory.ModTime() } @@ -617,18 +648,23 @@ func uploadDirInternal( if err := u.processSubdirectories(ctx, dirRelativePath, entries, prevEntries, dirManifest, &summ); err != nil && err != errCancelled { return "", fs.DirectorySummary{}, err } + u.prepareProgress(dirRelativePath, entries) log.Debugf("preparing work items %v", dirRelativePath) workItems, workItemErr := u.prepareWorkItems(ctx, dirRelativePath, entries, prevEntries, &summ) log.Debugf("finished preparing work items %v", dirRelativePath) + if workItemErr != nil && workItemErr != errCancelled { return "", fs.DirectorySummary{}, workItemErr } + if err := u.processUploadWorkItems(workItems, dirManifest); err != nil && err != errCancelled { return "", fs.DirectorySummary{}, err } + log.Debugf("finished processing uploads %v", dirRelativePath) + dirManifest.Summary = &summ writer := u.repo.Objects.NewWriter(ctx, object.WriterOptions{ @@ -641,6 +677,7 @@ func uploadDirInternal( } oid, err := writer.Result() + return oid, summ, err } @@ -688,6 +725,7 @@ func (u *Uploader) Upload( previousManifests ...*snapshot.Manifest, ) (*snapshot.Manifest, error) { log.Debugf("Uploading %v", sourceInfo) + s := &snapshot.Manifest{ Source: sourceInfo, } @@ -703,6 +741,7 @@ func (u *Uploader) Upload( switch entry := source.(type) { case fs.Directory: var previousDirs []fs.Directory + for _, m := range previousManifests { if d := u.maybeOpenDirectoryFromManifest(m); d != nil { previousDirs = append(previousDirs, d) diff --git a/snapshot/snapshotfs/upload_test.go b/snapshot/snapshotfs/upload_test.go index 40b008f2e..a0bbff810 100644 --- a/snapshot/snapshotfs/upload_test.go +++ b/snapshot/snapshotfs/upload_test.go @@ -36,6 +36,7 @@ func (th *uploadTestHarness) cleanup() { func newUploadTestHarness() *uploadTestHarness { ctx := context.Background() + repoDir, err := ioutil.TempDir("", "kopia-repo") if err != nil { panic("cannot create temp directory: " + err.Error()) @@ -98,17 +99,22 @@ func newUploadTestHarness() *uploadTestHarness { func TestUpload(t *testing.T) { ctx := context.Background() th := newUploadTestHarness() + defer th.cleanup() - u := NewUploader(th.repo) log.Infof("Uploading s1") + + u := NewUploader(th.repo) + s1, err := u.Upload(ctx, th.sourceDir, snapshot.SourceInfo{}) if err != nil { t.Errorf("Upload error: %v", err) } + log.Infof("s1: %v", s1.RootEntry) log.Infof("Uploading s2") + s2, err := u.Upload(ctx, th.sourceDir, snapshot.SourceInfo{}, s1) if err != nil { t.Errorf("Upload error: %v", err) @@ -133,6 +139,7 @@ func TestUpload(t *testing.T) { // Add one more file, the s1.RootObjectID should change. th.sourceDir.AddFile("d2/d1/f3", []byte{1, 2, 3, 4, 5}, defaultPermissions) + s3, err := u.Upload(ctx, th.sourceDir, snapshot.SourceInfo{}, s1) if err != nil { t.Errorf("upload failed: %v", err) @@ -185,11 +192,13 @@ func TestUpload_Cancel(t *testing.T) { func TestUpload_TopLevelDirectoryReadFailure(t *testing.T) { ctx := context.Background() th := newUploadTestHarness() + defer th.cleanup() th.sourceDir.FailReaddir(errTest) u := NewUploader(th.repo) + s, err := u.Upload(ctx, th.sourceDir, snapshot.SourceInfo{}) if err != errTest { t.Errorf("expected error: %v", err) @@ -203,12 +212,14 @@ func TestUpload_TopLevelDirectoryReadFailure(t *testing.T) { func TestUpload_SubDirectoryReadFailure(t *testing.T) { ctx := context.Background() th := newUploadTestHarness() + defer th.cleanup() th.sourceDir.Subdir("d1").FailReaddir(errTest) u := NewUploader(th.repo) u.IgnoreFileErrors = false + _, err := u.Upload(ctx, th.sourceDir, snapshot.SourceInfo{}) if err == nil { t.Errorf("expected error") diff --git a/snapshot/stats_test.go b/snapshot/stats_test.go index d46545d95..2bd77a93a 100644 --- a/snapshot/stats_test.go +++ b/snapshot/stats_test.go @@ -33,6 +33,7 @@ func TestStats(t *testing.T) { for _, tc := range tcs { got := snapshot.Stats{} got.AddExcluded(tc.entry) + if got != tc.want { t.Errorf("Stats do not match, got: %#v, want %#v", got, tc.want) } diff --git a/tests/end_to_end_test/end_to_end_test.go b/tests/end_to_end_test/end_to_end_test.go index dbc8020e5..b629476b0 100644 --- a/tests/end_to_end_test/end_to_end_test.go +++ b/tests/end_to_end_test/end_to_end_test.go @@ -81,12 +81,15 @@ func (e *testenv) cleanup(t *testing.T) { t.Logf("skipped cleanup for failed test, examine repository: %v", e.repoDir) return } + if e.repoDir != "" { os.RemoveAll(e.repoDir) } + if e.configDir != "" { os.RemoveAll(e.configDir) } + if e.dataDir != "" { os.RemoveAll(e.dataDir) } @@ -347,35 +350,44 @@ func TestDiff(t *testing.T) { func (e *testenv) runAndExpectSuccess(t *testing.T, args ...string) []string { t.Helper() + stdout, err := e.run(t, args...) if err != nil { t.Fatalf("'kopia %v' failed with %v", strings.Join(args, " "), err) } + return stdout } func (e *testenv) runAndExpectFailure(t *testing.T, args ...string) []string { t.Helper() + stdout, err := e.run(t, args...) if err == nil { t.Fatalf("'kopia %v' succeeded, but expected failure", strings.Join(args, " ")) } + return stdout } func (e *testenv) runAndVerifyOutputLineCount(t *testing.T, wantLines int, args ...string) []string { t.Helper() + lines := e.runAndExpectSuccess(t, args...) if len(lines) != wantLines { t.Errorf("unexpected list of results of 'kopia %v': %v (%v lines), wanted %v", strings.Join(args, " "), lines, len(lines), wantLines) } + return lines } func (e *testenv) run(t *testing.T, args ...string) ([]string, error) { t.Helper() t.Logf("running 'kopia %v'", strings.Join(args, " ")) + // nolint:gosec cmdArgs := append(append([]string(nil), e.fixedArgs...), args...) + + // nolint:gosec c := exec.Command(e.exe, cmdArgs...) c.Env = append(os.Environ(), e.environment...) @@ -385,9 +397,11 @@ func (e *testenv) run(t *testing.T, args ...string) ([]string, error) { } var stderr []byte + var wg sync.WaitGroup wg.Add(1) + go func() { defer wg.Done() @@ -395,9 +409,10 @@ func (e *testenv) run(t *testing.T, args ...string) ([]string, error) { }() o, err := c.Output() - wg.Wait() + wg.Wait() t.Logf("finished 'kopia %v' with err=%v and output:\n%v\nstderr:\n%v\n", strings.Join(args, " "), err, trimOutput(string(o)), trimOutput(string(stderr))) + return splitLines(string(o)), err } @@ -412,8 +427,8 @@ func trimOutput(s string) string { lines2 = append(lines2, lines[len(lines)-50:]...) return strings.Join(lines2, "\n") - } + func listSnapshotsAndExpectSuccess(t *testing.T, e *testenv, targets ...string) []sourceInfo { lines := e.runAndExpectSuccess(t, append([]string{"snapshot", "list", "-l"}, targets...)...) return mustParseSnapshots(t, lines) @@ -468,7 +483,9 @@ func mustParseSnapshots(t *testing.T, lines []string) []sourceInfo { t.Errorf("snapshot without a source: %q", l) return nil } + currentSource.snapshots = append(currentSource.snapshots, mustParseSnaphotInfo(t, l[2:])) + continue } @@ -483,15 +500,18 @@ func mustParseSnapshots(t *testing.T, lines []string) []sourceInfo { func randomName() string { b := make([]byte, rand.Intn(10)+3) cryptorand.Read(b) // nolint:errcheck + return hex.EncodeToString(b) } func mustParseSnaphotInfo(t *testing.T, l string) snapshotInfo { parts := strings.Split(l, " ") + ts, err := time.Parse("2006-01-02 15:04:05 MST", strings.Join(parts[0:3], " ")) if err != nil { t.Fatalf("err: %v", err) } + return snapshotInfo{ time: ts, objectID: parts[3], @@ -500,12 +520,15 @@ func mustParseSnaphotInfo(t *testing.T, l string) snapshotInfo { func mustParseSourceInfo(t *testing.T, l string) sourceInfo { p1 := strings.Index(l, "@") + p2 := strings.Index(l, ":") + if p1 >= 0 && p2 > p1 { return sourceInfo{user: l[0:p1], host: l[p1+1 : p2], path: l[p2+1:]} } t.Fatalf("can't parse source info: %q", l) + return sourceInfo{} } @@ -519,11 +542,13 @@ func splitLines(s string) []string { for _, l := range strings.Split(s, "\n") { result = append(result, strings.TrimRight(l, "\r")) } + return result } func assertNoError(t *testing.T, err error) { t.Helper() + if err != nil { t.Errorf("err: %v", err) } diff --git a/tests/repository_stress_test/repository_stress_test.go b/tests/repository_stress_test/repository_stress_test.go index f3afe8ce9..db8538087 100644 --- a/tests/repository_stress_test/repository_stress_test.go +++ b/tests/repository_stress_test/repository_stress_test.go @@ -33,6 +33,7 @@ func TestStressRepository(t *testing.T) { if testing.Short() { t.Skip("skipping stress test during short tests") } + ctx := content.UsingListCache(context.Background(), false) tmpPath, err := ioutil.TempDir("", "kopia") @@ -53,9 +54,11 @@ func TestStressRepository(t *testing.T) { configFile2 := filepath.Join(tmpPath, "kopia2.config") assertNoError(t, os.MkdirAll(storagePath, 0700)) + st, err := filesystem.New(ctx, &filesystem.Options{ Path: storagePath, }) + if err != nil { t.Fatalf("unable to initialize storage: %v", err) } @@ -87,21 +90,37 @@ func TestStressRepository(t *testing.T) { cancel := make(chan struct{}) var wg sync.WaitGroup + wg.Add(1) + go longLivedRepositoryTest(ctx, t, cancel, configFile1, &wg) + wg.Add(1) + go longLivedRepositoryTest(ctx, t, cancel, configFile1, &wg) + wg.Add(1) + go longLivedRepositoryTest(ctx, t, cancel, configFile1, &wg) + wg.Add(1) + go longLivedRepositoryTest(ctx, t, cancel, configFile1, &wg) + wg.Add(1) + go longLivedRepositoryTest(ctx, t, cancel, configFile2, &wg) + wg.Add(1) + go longLivedRepositoryTest(ctx, t, cancel, configFile2, &wg) + wg.Add(1) + go longLivedRepositoryTest(ctx, t, cancel, configFile2, &wg) + wg.Add(1) + go longLivedRepositoryTest(ctx, t, cancel, configFile2, &wg) time.Sleep(5 * time.Second) @@ -124,6 +143,7 @@ func longLivedRepositoryTest(ctx context.Context, t *testing.T, cancel chan stru for i := 0; i < 4; i++ { wg2.Add(1) + go func() { defer wg2.Done() @@ -135,17 +155,6 @@ func longLivedRepositoryTest(ctx context.Context, t *testing.T, cancel chan stru } func repositoryTest(ctx context.Context, t *testing.T, cancel chan struct{}, rep *repo.Repository) { - // reopen := func(t *testing.T, r *repo.Repository) error { - // if err := rep.Close(ctx); err != nil { - // return errors.Wrap(err, "error closing") - // } - - // t0 := time.Now() - // rep, err = repo.Open(ctx, configFile, &repo.Options{}) - // log.Printf("reopened in %v", time.Since(t0)) - // return err - // } - workTypes := []*struct { name string fun func(ctx context.Context, t *testing.T, r *repo.Repository) error @@ -170,6 +179,7 @@ func repositoryTest(ctx context.Context, t *testing.T, cancel chan struct{}, rep } iter := 0 + for { select { case <-cancel: @@ -182,6 +192,7 @@ func repositoryTest(ctx context.Context, t *testing.T, cancel chan struct{}, rep for _, w := range workTypes { bits = append(bits, fmt.Sprintf("%v:%v", w.name, w.hitCount)) } + log.Printf("#%v %v %v goroutines", iter, strings.Join(bits, " "), runtime.NumGoroutine()) } iter++ @@ -190,23 +201,26 @@ func repositoryTest(ctx context.Context, t *testing.T, cancel chan struct{}, rep for _, w := range workTypes { if roulette < w.weight { w.hitCount++ + if err := w.fun(ctx, t, rep); err != nil { w.hitCount++ t.Errorf("error: %v", errors.Wrapf(err, "error running %v", w.name)) + return } + break } roulette -= w.weight } } - } func writeRandomBlock(ctx context.Context, t *testing.T, r *repo.Repository) error { data := make([]byte, 1000) cryptorand.Read(data) //nolint:errcheck + contentID, err := r.Content.WriteContent(ctx, data, "") if err == nil { knownBlocksMutex.Lock() @@ -218,6 +232,7 @@ func writeRandomBlock(ctx context.Context, t *testing.T, r *repo.Repository) err } knownBlocksMutex.Unlock() } + return err } @@ -227,6 +242,7 @@ func readKnownBlock(ctx context.Context, t *testing.T, r *repo.Repository) error knownBlocksMutex.Unlock() return nil } + contentID := knownBlocks[rand.Intn(len(knownBlocks))] knownBlocksMutex.Unlock() @@ -283,11 +299,15 @@ func readRandomManifest(ctx context.Context, t *testing.T, r *repo.Repository) e if err != nil { return err } + if len(manifests) == 0 { return nil } + n := rand.Intn(len(manifests)) + _, err = r.Manifests.GetRaw(ctx, manifests[n].ID) + return err } @@ -308,11 +328,13 @@ func writeRandomManifest(ctx context.Context, t *testing.T, r *repo.Repository) content1: content1val, content2: content2val, }) + return err } func assertNoError(t *testing.T, err error) { t.Helper() + if err != nil { t.Errorf("err: %v", err) } diff --git a/tests/stress_test/stress_test.go b/tests/stress_test/stress_test.go index 8a6059dbe..1823ecac0 100644 --- a/tests/stress_test/stress_test.go +++ b/tests/stress_test/stress_test.go @@ -82,11 +82,14 @@ type writtenBlock struct { for time.Now().Before(deadline) { l := rnd.Intn(30000) data := make([]byte, l) + if _, err := rnd.Read(data); err != nil { t.Errorf("err: %v", err) return } + dataCopy := append([]byte{}, data...) + contentID, err := bm.WriteContent(ctx, data, "") if err != nil { t.Errorf("err: %v", err) @@ -104,6 +107,7 @@ type writtenBlock struct { t.Errorf("flush error: %v", ferr) return } + bm, err = openMgr() if err != nil { t.Errorf("error opening: %v", err) @@ -115,15 +119,18 @@ type writtenBlock struct { if len(workerBlocks) > 5 { pos := rnd.Intn(len(workerBlocks)) previous := workerBlocks[pos] + d2, err := bm.GetContent(ctx, previous.contentID) if err != nil { t.Errorf("error verifying content %q: %v", previous.contentID, err) return } + if !reflect.DeepEqual(previous.data, d2) { t.Errorf("invalid previous data for %q %x %x", previous.contentID, d2, previous.data) return } + workerBlocks = append(workerBlocks[0:pos], workerBlocks[pos+1:]...) } } diff --git a/tools/tools.mk b/tools/tools.mk index b05b0f973..8c48ef7c9 100644 --- a/tools/tools.mk +++ b/tools/tools.mk @@ -3,7 +3,7 @@ TOOLS_DIR:=$(SELF_DIR)/.tools uname := $(shell uname -s) # tool versions -GOLANGCI_LINT_VERSION=v1.18.0 +GOLANGCI_LINT_VERSION=v1.21.0 NODE_VERSION=12.13.0 HUGO_VERSION=0.59.1