From 54edb97b3ade6897bf39c02edae4dc91ae42ee3f Mon Sep 17 00:00:00 2001 From: Jarek Kowalski Date: Sat, 1 Jun 2019 20:11:40 -0700 Subject: [PATCH] refactoring: renamed repo/block to repo/content Also introduced strongly typed content.ID and manifest.ID (instead of string) This aligns identifiers across all layers of repository: blob.ID content.ID object.ID manifest.ID --- cli/app.go | 14 +- cli/command_benchmark_crypto.go | 12 +- cli/command_blob_delete.go | 4 +- cli/command_block_gc.go | 50 - cli/command_block_index_optimize.go | 29 - cli/command_block_list.go | 101 -- cli/command_block_rewrite.go | 185 ---- cli/command_block_rm.go | 32 - cli/command_block_show.go | 38 - cli/command_block_verify.go | 62 -- cli/command_cache_info.go | 2 +- cli/command_cache_set.go | 4 +- cli/command_content_gc.go | 50 + cli/command_content_list.go | 101 ++ cli/command_content_rewrite.go | 193 ++++ cli/command_content_rm.go | 28 + cli/command_content_show.go | 39 + ...lock_stats.go => command_content_stats.go} | 52 +- cli/command_content_verify.go | 62 ++ ...ck_index_list.go => command_index_list.go} | 8 +- cli/command_index_optimize.go | 29 + ...ex_recover.go => command_index_recover.go} | 12 +- cli/command_manifest_rm.go | 12 +- cli/command_manifest_show.go | 12 +- cli/command_object_verify.go | 7 +- cli/command_policy.go | 4 +- cli/command_repository_connect.go | 4 +- cli/command_repository_create.go | 8 +- cli/command_repository_repair.go | 24 +- cli/command_repository_status.go | 8 +- cli/command_snapshot_create.go | 2 +- cli/command_snapshot_list.go | 7 +- examples/upload_download/main.go | 8 +- examples/upload_download/setup_repository.go | 4 +- internal/blobtesting/asserts.go | 40 +- internal/blobtesting/verify.go | 10 +- internal/repotesting/repotesting.go | 10 +- internal/server/api_snapshot_list.go | 3 +- internal/server/api_status.go | 2 +- internal/serverapi/serverapi.go | 10 +- repo/block/block_index_recovery_test.go | 91 -- repo/block/block_manager_test.go | 911 ------------------ repo/block/committed_block_index_mem_cache.go | 51 - repo/block/context.go | 34 - repo/connect.go | 12 +- .../block_manager_compaction.go | 68 +- repo/{block => content}/builder.go | 42 +- repo/{block => content}/cache_hmac.go | 2 +- repo/{block => content}/caching_options.go | 2 +- .../committed_content_index.go} | 40 +- .../committed_content_index_disk_cache.go} | 24 +- .../committed_content_index_mem_cache.go | 51 + .../content_cache.go} | 64 +- .../content_cache_test.go} | 100 +- .../content_formatter.go} | 44 +- .../content_formatter_test.go} | 8 +- .../content_formatting_options.go} | 4 +- .../{block => content}/content_id_to_bytes.go | 12 +- .../content_index_recovery.go} | 28 +- repo/content/content_index_recovery_test.go | 91 ++ .../content_manager.go} | 384 ++++---- repo/content/content_manager_test.go | 911 ++++++++++++++++++ repo/content/context.go | 34 + repo/{block => content}/format.go | 6 +- repo/{block => content}/index.go | 42 +- repo/{block => content}/info.go | 11 +- repo/{block => content}/list_cache.go | 18 +- repo/{block => content}/merged.go | 24 +- repo/{block => content}/merged_test.go | 38 +- .../packindex_internal_test.go | 6 +- repo/{block => content}/packindex_test.go | 48 +- repo/{block => content}/stats.go | 18 +- repo/crypto_key_derivation.go | 2 +- repo/format_block.go | 58 +- repo/format_block_test.go | 34 +- repo/initialize.go | 20 +- repo/local_config.go | 8 +- repo/manifest/manifest_entry.go | 2 +- repo/manifest/manifest_manager.go | 157 +-- repo/manifest/manifest_manager_test.go | 88 +- repo/manifest/serialized.go | 2 +- repo/object/object_manager.go | 60 +- repo/object/object_manager_test.go | 68 +- repo/object/object_reader.go | 6 +- repo/object/object_writer.go | 52 +- repo/object/objectid.go | 32 +- repo/open.go | 46 +- repo/repository.go | 20 +- repo/repository_test.go | 12 +- repo/upgrade.go | 6 +- site/content/docs/Architecture/_index.md | 4 +- snapshot/manager.go | 18 +- snapshot/manifest.go | 5 +- snapshot/policy/policy_manager.go | 4 +- snapshot/snapshot_test.go | 33 +- snapshot/snapshotfs/upload.go | 4 +- snapshot/stats.go | 4 +- tests/end_to_end_test/end_to_end_test.go | 36 +- .../repository_stress_test.go | 50 +- tests/stress_test/stress_test.go | 18 +- 100 files changed, 2692 insertions(+), 2658 deletions(-) delete mode 100644 cli/command_block_gc.go delete mode 100644 cli/command_block_index_optimize.go delete mode 100644 cli/command_block_list.go delete mode 100644 cli/command_block_rewrite.go delete mode 100644 cli/command_block_rm.go delete mode 100644 cli/command_block_show.go delete mode 100644 cli/command_block_verify.go create mode 100644 cli/command_content_gc.go create mode 100644 cli/command_content_list.go create mode 100644 cli/command_content_rewrite.go create mode 100644 cli/command_content_rm.go create mode 100644 cli/command_content_show.go rename cli/{command_block_stats.go => command_content_stats.go} (54%) create mode 100644 cli/command_content_verify.go rename cli/{command_block_index_list.go => command_index_list.go} (80%) create mode 100644 cli/command_index_optimize.go rename cli/{command_block_index_recover.go => command_index_recover.go} (76%) delete mode 100644 repo/block/block_index_recovery_test.go delete mode 100644 repo/block/block_manager_test.go delete mode 100644 repo/block/committed_block_index_mem_cache.go delete mode 100644 repo/block/context.go rename repo/{block => content}/block_manager_compaction.go (58%) rename repo/{block => content}/builder.go (79%) rename repo/{block => content}/cache_hmac.go (97%) rename repo/{block => content}/caching_options.go (95%) rename repo/{block/committed_block_index.go => content/committed_content_index.go} (63%) rename repo/{block/committed_block_index_disk_cache.go => content/committed_content_index_disk_cache.go} (73%) create mode 100644 repo/content/committed_content_index_mem_cache.go rename repo/{block/block_cache.go => content/content_cache.go} (63%) rename repo/{block/block_cache_test.go => content/content_cache_test.go} (59%) rename repo/{block/block_formatter.go => content/content_formatter.go} (84%) rename repo/{block/block_formatter_test.go => content/content_formatter_test.go} (93%) rename repo/{block/block_formatting_options.go => content/content_formatting_options.go} (90%) rename repo/{block => content}/content_id_to_bytes.go (64%) rename repo/{block/block_index_recovery.go => content/content_index_recovery.go} (88%) create mode 100644 repo/content/content_index_recovery_test.go rename repo/{block/block_manager.go => content/content_manager.go} (61%) create mode 100644 repo/content/content_manager_test.go create mode 100644 repo/content/context.go rename repo/{block => content}/format.go (94%) rename repo/{block => content}/index.go (76%) rename repo/{block => content}/info.go (64%) rename repo/{block => content}/list_cache.go (86%) rename repo/{block => content}/merged.go (74%) rename repo/{block => content}/merged_test.go (53%) rename repo/{block => content}/packindex_internal_test.go (78%) rename repo/{block => content}/packindex_test.go (81%) rename repo/{block => content}/stats.go (52%) diff --git a/cli/app.go b/cli/app.go index 2b85852ee..57d63d899 100644 --- a/cli/app.go +++ b/cli/app.go @@ -13,7 +13,7 @@ "github.com/kopia/kopia/internal/serverapi" "github.com/kopia/kopia/repo" "github.com/kopia/kopia/repo/blob" - "github.com/kopia/kopia/repo/block" + "github.com/kopia/kopia/repo/content" kingpin "gopkg.in/alecthomas/kingpin.v2" ) @@ -31,9 +31,9 @@ policyCommands = app.Command("policy", "Commands to manipulate snapshotting policies.").Alias("policies") serverCommands = app.Command("server", "Commands to control HTTP API server.") manifestCommands = app.Command("manifest", "Low-level commands to manipulate manifest items.").Hidden() - blockCommands = app.Command("block", "Commands to manipulate virtual blocks in repository.").Alias("blk").Hidden() + contentCommands = app.Command("content", "Commands to manipulate content in repository.").Alias("contents").Hidden() blobCommands = app.Command("blob", "Commands to manipulate BLOBs.").Hidden() - blockIndexCommands = app.Command("blockindex", "Commands to manipulate block index.").Hidden() + indexCommands = app.Command("index", "Commands to manipulate content index.").Hidden() benchmarkCommands = app.Command("benchmark", "Commands to test performance of algorithms.").Hidden() ) @@ -58,8 +58,8 @@ func serverAction(act func(ctx context.Context, cli *serverapi.Client) error) fu func repositoryAction(act func(ctx context.Context, rep *repo.Repository) error) func(ctx *kingpin.ParseContext) error { return func(kpc *kingpin.ParseContext) error { ctx := context.Background() - ctx = block.UsingBlockCache(ctx, *enableCaching) - ctx = block.UsingListCache(ctx, *enableListCaching) + ctx = content.UsingContentCache(ctx, *enableCaching) + ctx = content.UsingListCache(ctx, *enableListCaching) ctx = blob.WithUploadProgressCallback(ctx, func(desc string, progress, total int64) { cliProgress.Report("upload '"+desc+"'", progress, total) }) @@ -70,13 +70,13 @@ func repositoryAction(act func(ctx context.Context, rep *repo.Repository) error) storageType := rep.Blobs.ConnectionInfo().Type - reportStartupTime(storageType, rep.Blocks.Format.Version, repositoryOpenTime) + reportStartupTime(storageType, rep.Content.Format.Version, repositoryOpenTime) t1 := time.Now() err := act(ctx, rep) commandDuration := time.Since(t1) - reportSubcommandFinished(kpc.SelectedCommand.FullCommand(), err == nil, storageType, rep.Blocks.Format.Version, commandDuration) + reportSubcommandFinished(kpc.SelectedCommand.FullCommand(), err == nil, storageType, rep.Content.Format.Version, commandDuration) if cerr := rep.Close(ctx); cerr != nil { return errors.Wrap(cerr, "unable to close repository") } diff --git a/cli/command_benchmark_crypto.go b/cli/command_benchmark_crypto.go index e7089b6a0..d5bd95dfb 100644 --- a/cli/command_benchmark_crypto.go +++ b/cli/command_benchmark_crypto.go @@ -5,7 +5,7 @@ "time" "github.com/kopia/kopia/internal/units" - "github.com/kopia/kopia/repo/block" + "github.com/kopia/kopia/repo/content" kingpin "gopkg.in/alecthomas/kingpin.v2" ) @@ -27,14 +27,14 @@ type benchResult struct { var results []benchResult data := make([]byte, *benchmarkCryptoBlockSize) - for _, ha := range block.SupportedHashAlgorithms() { - for _, ea := range block.SupportedEncryptionAlgorithms() { + for _, ha := range content.SupportedHashAlgorithms() { + for _, ea := range content.SupportedEncryptionAlgorithms() { isEncrypted := ea != "NONE" if *benchmarkCryptoEncryption != isEncrypted { continue } - h, e, err := block.CreateHashAndEncryptor(block.FormattingOptions{ + h, e, err := content.CreateHashAndEncryptor(content.FormattingOptions{ Encryption: ea, Hash: ha, MasterKey: make([]byte, 32), @@ -48,8 +48,8 @@ type benchResult struct { t0 := time.Now() hashCount := *benchmarkCryptoRepeat for i := 0; i < hashCount; i++ { - blockID := h(data) - if _, encerr := e.Encrypt(data, blockID); encerr != nil { + contentID := h(data) + if _, encerr := e.Encrypt(data, contentID); encerr != nil { log.Warningf("encryption failed: %v", encerr) break } diff --git a/cli/command_blob_delete.go b/cli/command_blob_delete.go index a64bd5638..643ac48dc 100644 --- a/cli/command_blob_delete.go +++ b/cli/command_blob_delete.go @@ -14,7 +14,7 @@ blobDeleteBlobIDs = blobDeleteCommand.Arg("blobIDs", "Blob IDs").Required().Strings() ) -func runDeleteStorageBlocks(ctx context.Context, rep *repo.Repository) error { +func runDeleteBlobs(ctx context.Context, rep *repo.Repository) error { for _, b := range *blobDeleteBlobIDs { err := rep.Blobs.DeleteBlob(ctx, blob.ID(b)) if err != nil { @@ -26,5 +26,5 @@ func runDeleteStorageBlocks(ctx context.Context, rep *repo.Repository) error { } func init() { - blobDeleteCommand.Action(repositoryAction(runDeleteStorageBlocks)) + blobDeleteCommand.Action(repositoryAction(runDeleteBlobs)) } diff --git a/cli/command_block_gc.go b/cli/command_block_gc.go deleted file mode 100644 index 8dd59b34b..000000000 --- a/cli/command_block_gc.go +++ /dev/null @@ -1,50 +0,0 @@ -package cli - -import ( - "context" - - "github.com/pkg/errors" - - "github.com/kopia/kopia/repo" -) - -var ( - blockGarbageCollectCommand = blockCommands.Command("gc", "Garbage-collect unused storage blocks") - blockGarbageCollectCommandDelete = blockGarbageCollectCommand.Flag("delete", "Whether to delete unused block").String() -) - -func runBlockGarbageCollectAction(ctx context.Context, rep *repo.Repository) error { - unused, err := rep.Blocks.FindUnreferencedBlobs(ctx) - if err != nil { - return errors.Wrap(err, "error looking for unreferenced blobs") - } - - if len(unused) == 0 { - printStderr("No unused blocks found.\n") - return nil - } - - if *blockGarbageCollectCommandDelete != "yes" { - var totalBytes int64 - for _, u := range unused { - printStderr("unused %v (%v bytes)\n", u.BlobID, u.Length) - totalBytes += u.Length - } - printStderr("Would delete %v unused blocks (%v bytes), pass '--delete=yes' to actually delete.\n", len(unused), totalBytes) - - return nil - } - - for _, u := range unused { - printStderr("Deleting unused block %q (%v bytes)...\n", u.BlobID, u.Length) - if err := rep.Blobs.DeleteBlob(ctx, u.BlobID); err != nil { - return errors.Wrapf(err, "unable to delete block %q", u.BlobID) - } - } - - return nil -} - -func init() { - blockGarbageCollectCommand.Action(repositoryAction(runBlockGarbageCollectAction)) -} diff --git a/cli/command_block_index_optimize.go b/cli/command_block_index_optimize.go deleted file mode 100644 index 480ee2671..000000000 --- a/cli/command_block_index_optimize.go +++ /dev/null @@ -1,29 +0,0 @@ -package cli - -import ( - "context" - - "github.com/kopia/kopia/repo" - "github.com/kopia/kopia/repo/block" -) - -var ( - optimizeCommand = blockIndexCommands.Command("optimize", "Optimize block indexes.") - optimizeMinSmallBlocks = optimizeCommand.Flag("min-small-blocks", "Minimum number of small index blobs that can be left after compaction.").Default("1").Int() - optimizeMaxSmallBlocks = optimizeCommand.Flag("max-small-blocks", "Maximum number of small index blobs that can be left after compaction.").Default("1").Int() - optimizeSkipDeletedOlderThan = optimizeCommand.Flag("skip-deleted-older-than", "Skip deleted blocks above given age").Duration() - optimizeAllBlocks = optimizeCommand.Flag("all", "Optimize all indexes, even those above maximum size.").Bool() -) - -func runOptimizeCommand(ctx context.Context, rep *repo.Repository) error { - return rep.Blocks.CompactIndexes(ctx, block.CompactOptions{ - MinSmallBlocks: *optimizeMinSmallBlocks, - MaxSmallBlocks: *optimizeMaxSmallBlocks, - AllBlocks: *optimizeAllBlocks, - SkipDeletedOlderThan: *optimizeSkipDeletedOlderThan, - }) -} - -func init() { - optimizeCommand.Action(repositoryAction(runOptimizeCommand)) -} diff --git a/cli/command_block_list.go b/cli/command_block_list.go deleted file mode 100644 index a7428fefa..000000000 --- a/cli/command_block_list.go +++ /dev/null @@ -1,101 +0,0 @@ -package cli - -import ( - "context" - "fmt" - "sort" - - "github.com/kopia/kopia/repo" - "github.com/kopia/kopia/repo/blob" - "github.com/kopia/kopia/repo/block" -) - -var ( - blockListCommand = blockCommands.Command("list", "List blocks").Alias("ls") - blockListLong = blockListCommand.Flag("long", "Long output").Short('l').Bool() - blockListPrefix = blockListCommand.Flag("prefix", "Prefix").String() - blockListIncludeDeleted = blockListCommand.Flag("deleted", "Include deleted blocks").Bool() - blockListDeletedOnly = blockListCommand.Flag("deleted-only", "Only show deleted blocks").Bool() - blockListSort = blockListCommand.Flag("sort", "Sort order").Default("name").Enum("name", "size", "time", "none", "pack") - blockListReverse = blockListCommand.Flag("reverse", "Reverse sort").Short('r').Bool() - blockListSummary = blockListCommand.Flag("summary", "Summarize the list").Short('s').Bool() - blockListHuman = blockListCommand.Flag("human", "Human-readable output").Short('h').Bool() -) - -func runListBlocksAction(ctx context.Context, rep *repo.Repository) error { - blocks, err := rep.Blocks.ListBlockInfos(*blockListPrefix, *blockListIncludeDeleted || *blockListDeletedOnly) - if err != nil { - return err - } - - sortBlocks(blocks) - - var count int - var totalSize int64 - uniquePacks := map[blob.ID]bool{} - for _, b := range blocks { - if *blockListDeletedOnly && !b.Deleted { - continue - } - totalSize += int64(b.Length) - count++ - if b.PackBlobID != "" { - uniquePacks[b.PackBlobID] = true - } - if *blockListLong { - optionalDeleted := "" - if b.Deleted { - optionalDeleted = " (deleted)" - } - fmt.Printf("%v %v %v %v+%v%v\n", - b.BlockID, - formatTimestamp(b.Timestamp()), - b.PackBlobID, - b.PackOffset, - maybeHumanReadableBytes(*blockListHuman, int64(b.Length)), - optionalDeleted) - } else { - fmt.Printf("%v\n", b.BlockID) - } - } - - if *blockListSummary { - fmt.Printf("Total: %v blocks, %v packs, %v total size\n", - maybeHumanReadableCount(*blockListHuman, int64(count)), - maybeHumanReadableCount(*blockListHuman, int64(len(uniquePacks))), - maybeHumanReadableBytes(*blockListHuman, totalSize)) - } - - return nil -} - -func sortBlocks(blocks []block.Info) { - maybeReverse := func(b bool) bool { return b } - - if *blockListReverse { - maybeReverse = func(b bool) bool { return !b } - } - - switch *blockListSort { - case "name": - sort.Slice(blocks, func(i, j int) bool { return maybeReverse(blocks[i].BlockID < blocks[j].BlockID) }) - case "size": - sort.Slice(blocks, func(i, j int) bool { return maybeReverse(blocks[i].Length < blocks[j].Length) }) - case "time": - sort.Slice(blocks, func(i, j int) bool { return maybeReverse(blocks[i].TimestampSeconds < blocks[j].TimestampSeconds) }) - case "pack": - sort.Slice(blocks, func(i, j int) bool { return maybeReverse(comparePacks(blocks[i], blocks[j])) }) - } -} - -func comparePacks(a, b block.Info) bool { - if a, b := a.PackBlobID, b.PackBlobID; a != b { - return a < b - } - - return a.PackOffset < b.PackOffset -} - -func init() { - blockListCommand.Action(repositoryAction(runListBlocksAction)) -} diff --git a/cli/command_block_rewrite.go b/cli/command_block_rewrite.go deleted file mode 100644 index 7a2cb1e1f..000000000 --- a/cli/command_block_rewrite.go +++ /dev/null @@ -1,185 +0,0 @@ -package cli - -import ( - "context" - "fmt" - "strings" - "sync" - - "github.com/pkg/errors" - - "github.com/kopia/kopia/repo" - "github.com/kopia/kopia/repo/blob" - "github.com/kopia/kopia/repo/block" -) - -var ( - blockRewriteCommand = blockCommands.Command("rewrite", "Rewrite blocks using most recent format") - blockRewriteIDs = blockRewriteCommand.Arg("blockID", "Identifiers of blocks to rewrite").Strings() - blockRewriteParallelism = blockRewriteCommand.Flag("parallelism", "Number of parallel workers").Default("16").Int() - - blockRewriteShortPacks = blockRewriteCommand.Flag("short", "Rewrite blocks from short packs").Bool() - blockRewriteFormatVersion = blockRewriteCommand.Flag("format-version", "Rewrite blocks using the provided format version").Default("-1").Int() - blockRewritePackPrefix = blockRewriteCommand.Flag("pack-prefix", "Only rewrite pack blocks with a given prefix").String() - blockRewriteDryRun = blockRewriteCommand.Flag("dry-run", "Do not actually rewrite, only print what would happen").Short('n').Bool() -) - -type blockInfoOrError struct { - block.Info - err error -} - -func runRewriteBlocksAction(ctx context.Context, rep *repo.Repository) error { - blocks := getBlocksToRewrite(ctx, rep) - - var ( - mu sync.Mutex - totalBytes int64 - failedCount int - ) - - var wg sync.WaitGroup - - for i := 0; i < *blockRewriteParallelism; i++ { - wg.Add(1) - go func() { - defer wg.Done() - - for b := range blocks { - if b.err != nil { - log.Errorf("got error: %v", b.err) - mu.Lock() - failedCount++ - mu.Unlock() - return - } - - var optDeleted string - if b.Deleted { - optDeleted = " (deleted)" - } - - printStderr("Rewriting block %v (%v bytes) from pack %v%v\n", b.BlockID, b.Length, b.PackBlobID, optDeleted) - mu.Lock() - totalBytes += int64(b.Length) - mu.Unlock() - if *blockRewriteDryRun { - continue - } - if err := rep.Blocks.RewriteBlock(ctx, b.BlockID); err != nil { - log.Warningf("unable to rewrite block %q: %v", b.BlockID, err) - mu.Lock() - failedCount++ - mu.Unlock() - } - } - }() - } - - wg.Wait() - - printStderr("Total bytes rewritten %v\n", totalBytes) - - if failedCount == 0 { - return nil - } - - return errors.Errorf("failed to rewrite %v blocks", failedCount) -} - -func getBlocksToRewrite(ctx context.Context, rep *repo.Repository) <-chan blockInfoOrError { - ch := make(chan blockInfoOrError) - go func() { - defer close(ch) - - // get blocks listed on command line - findBlockInfos(ctx, rep, ch, *blockRewriteIDs) - - // add all blocks from short packs - if *blockRewriteShortPacks { - threshold := uint32(rep.Blocks.Format.MaxPackSize * 6 / 10) - findBlocksInShortPacks(ctx, rep, ch, threshold) - } - - // add all blocks with given format version - if *blockRewriteFormatVersion != -1 { - findBlocksWithFormatVersion(ctx, rep, ch, *blockRewriteFormatVersion) - } - }() - - return ch -} - -func findBlockInfos(ctx context.Context, rep *repo.Repository, ch chan blockInfoOrError, blockIDs []string) { - for _, blockID := range blockIDs { - i, err := rep.Blocks.BlockInfo(ctx, blockID) - if err != nil { - ch <- blockInfoOrError{err: errors.Wrapf(err, "unable to get info for block %q", blockID)} - } else { - ch <- blockInfoOrError{Info: i} - } - } -} - -func findBlocksWithFormatVersion(ctx context.Context, rep *repo.Repository, ch chan blockInfoOrError, version int) { - infos, err := rep.Blocks.ListBlockInfos("", true) - if err != nil { - ch <- blockInfoOrError{err: errors.Wrap(err, "unable to list index blobs")} - return - } - - for _, b := range infos { - if int(b.FormatVersion) == *blockRewriteFormatVersion && strings.HasPrefix(string(b.PackBlobID), *blockRewritePackPrefix) { - ch <- blockInfoOrError{Info: b} - } - } -} - -func findBlocksInShortPacks(ctx context.Context, rep *repo.Repository, ch chan blockInfoOrError, threshold uint32) { - log.Debugf("listing blocks...") - infos, err := rep.Blocks.ListBlockInfos("", true) - if err != nil { - ch <- blockInfoOrError{err: errors.Wrap(err, "unable to list index blobs")} - return - } - - log.Debugf("finding blocks in short packs...") - shortPackBlocks, err := findShortPackBlocks(infos, threshold) - if err != nil { - ch <- blockInfoOrError{err: errors.Wrap(err, "unable to find short pack blocks")} - return - } - log.Debugf("found %v short pack blocks", len(shortPackBlocks)) - - if len(shortPackBlocks) <= 1 { - fmt.Printf("Nothing to do, found %v short pack blocks\n", len(shortPackBlocks)) - } else { - for _, b := range infos { - if shortPackBlocks[b.PackBlobID] && strings.HasPrefix(string(b.PackBlobID), *blockRewritePackPrefix) { - ch <- blockInfoOrError{Info: b} - } - } - } -} - -func findShortPackBlocks(infos []block.Info, threshold uint32) (map[blob.ID]bool, error) { - packUsage := map[blob.ID]uint32{} - - for _, bi := range infos { - packUsage[bi.PackBlobID] += bi.Length - } - - shortPackBlocks := map[blob.ID]bool{} - - for blobID, usage := range packUsage { - if usage < threshold { - shortPackBlocks[blobID] = true - } - } - - return shortPackBlocks, nil -} - -func init() { - blockRewriteCommand.Action(repositoryAction(runRewriteBlocksAction)) -} diff --git a/cli/command_block_rm.go b/cli/command_block_rm.go deleted file mode 100644 index 5e4d6835f..000000000 --- a/cli/command_block_rm.go +++ /dev/null @@ -1,32 +0,0 @@ -package cli - -import ( - "context" - - "github.com/kopia/kopia/repo" -) - -var ( - removeBlockCommand = blockCommands.Command("remove", "Remove block(s)").Alias("rm") - - removeBlockIDs = removeBlockCommand.Arg("id", "IDs of blocks to remove").Required().Strings() -) - -func runRemoveBlockCommand(ctx context.Context, rep *repo.Repository) error { - for _, blockID := range *removeBlockIDs { - if err := removeBlock(rep, blockID); err != nil { - return err - } - } - - return nil -} - -func removeBlock(r *repo.Repository, blockID string) error { - return r.Blocks.DeleteBlock(blockID) -} - -func init() { - setupShowCommand(removeBlockCommand) - removeBlockCommand.Action(repositoryAction(runRemoveBlockCommand)) -} diff --git a/cli/command_block_show.go b/cli/command_block_show.go deleted file mode 100644 index 019dc4def..000000000 --- a/cli/command_block_show.go +++ /dev/null @@ -1,38 +0,0 @@ -package cli - -import ( - "bytes" - "context" - - "github.com/kopia/kopia/repo" -) - -var ( - showBlockCommand = blockCommands.Command("show", "Show contents of a block.").Alias("cat") - - showBlockIDs = showBlockCommand.Arg("id", "IDs of blocks to show").Required().Strings() -) - -func runShowBlockCommand(ctx context.Context, rep *repo.Repository) error { - for _, blockID := range *showBlockIDs { - if err := showBlock(ctx, rep, blockID); err != nil { - return err - } - } - - return nil -} - -func showBlock(ctx context.Context, r *repo.Repository, blockID string) error { - data, err := r.Blocks.GetBlock(ctx, blockID) - if err != nil { - return err - } - - return showContent(bytes.NewReader(data)) -} - -func init() { - setupShowCommand(showBlockCommand) - showBlockCommand.Action(repositoryAction(runShowBlockCommand)) -} diff --git a/cli/command_block_verify.go b/cli/command_block_verify.go deleted file mode 100644 index 3aa1629ea..000000000 --- a/cli/command_block_verify.go +++ /dev/null @@ -1,62 +0,0 @@ -package cli - -import ( - "context" - - "github.com/pkg/errors" - - "github.com/kopia/kopia/repo" -) - -var ( - verifyBlockCommand = blockCommands.Command("verify", "Verify contents of a block.") - - verifyBlockIDs = verifyBlockCommand.Arg("id", "IDs of blocks to show (or 'all')").Required().Strings() -) - -func runVerifyBlockCommand(ctx context.Context, rep *repo.Repository) error { - for _, blockID := range *verifyBlockIDs { - if blockID == "all" { - return verifyAllBlocks(ctx, rep) - } - if err := verifyBlock(ctx, rep, blockID); err != nil { - return err - } - } - - return nil -} - -func verifyAllBlocks(ctx context.Context, rep *repo.Repository) error { - blockIDs, err := rep.Blocks.ListBlocks("") - if err != nil { - return errors.Wrap(err, "unable to list blocks") - } - - var errorCount int - for _, blockID := range blockIDs { - if err := verifyBlock(ctx, rep, blockID); err != nil { - errorCount++ - } - } - if errorCount == 0 { - return nil - } - - return errors.Errorf("encountered %v errors", errorCount) -} - -func verifyBlock(ctx context.Context, r *repo.Repository, blockID string) error { - if _, err := r.Blocks.GetBlock(ctx, blockID); err != nil { - log.Warningf("block %v is invalid: %v", blockID, err) - return err - } - - log.Infof("block %v is ok", blockID) - - return nil -} - -func init() { - verifyBlockCommand.Action(repositoryAction(runVerifyBlockCommand)) -} diff --git a/cli/command_cache_info.go b/cli/command_cache_info.go index 76808c9de..bb3184e33 100644 --- a/cli/command_cache_info.go +++ b/cli/command_cache_info.go @@ -21,7 +21,7 @@ func runCacheInfoCommand(ctx context.Context, rep *repo.Repository) error { } log.Debugf("scanning cache...") - fileCount, totalFileSize, err := scanCacheDir(filepath.Join(rep.CacheDirectory, "blocks")) + fileCount, totalFileSize, err := scanCacheDir(filepath.Join(rep.CacheDirectory, "contents")) if err != nil { return err } diff --git a/cli/command_cache_set.go b/cli/command_cache_set.go index a17446c32..3e55a3d8f 100644 --- a/cli/command_cache_set.go +++ b/cli/command_cache_set.go @@ -4,7 +4,7 @@ "context" "github.com/kopia/kopia/repo" - "github.com/kopia/kopia/repo/block" + "github.com/kopia/kopia/repo/content" ) var ( @@ -16,7 +16,7 @@ ) func runCacheSetCommand(ctx context.Context, rep *repo.Repository) error { - opts := block.CachingOptions{ + opts := content.CachingOptions{ CacheDirectory: *cacheSetDirectory, MaxCacheSizeBytes: *cacheSetMaxCacheSizeMB << 20, MaxListCacheDurationSec: int(cacheSetMaxListCacheDuration.Seconds()), diff --git a/cli/command_content_gc.go b/cli/command_content_gc.go new file mode 100644 index 000000000..acb35f981 --- /dev/null +++ b/cli/command_content_gc.go @@ -0,0 +1,50 @@ +package cli + +import ( + "context" + + "github.com/pkg/errors" + + "github.com/kopia/kopia/repo" +) + +var ( + contentGarbageCollectCommand = contentCommands.Command("gc", "Garbage-collect unused blobs") + contentGarbageCollectCommandDelete = contentGarbageCollectCommand.Flag("delete", "Whether to delete unused blobs").String() +) + +func runContentGarbageCollectCommand(ctx context.Context, rep *repo.Repository) error { + unused, err := rep.Content.FindUnreferencedBlobs(ctx) + if err != nil { + return errors.Wrap(err, "error looking for unreferenced blobs") + } + + if len(unused) == 0 { + printStderr("No unused blobs found.\n") + return nil + } + + if *contentGarbageCollectCommandDelete != "yes" { + var totalBytes int64 + for _, u := range unused { + printStderr("unused %v (%v bytes)\n", u.BlobID, u.Length) + totalBytes += u.Length + } + printStderr("Would delete %v unused blobs (%v bytes), pass '--delete=yes' to actually delete.\n", len(unused), totalBytes) + + return nil + } + + for _, u := range unused { + printStderr("Deleting unused blob %q (%v bytes)...\n", u.BlobID, u.Length) + if err := rep.Blobs.DeleteBlob(ctx, u.BlobID); err != nil { + return errors.Wrapf(err, "unable to delete blob %q", u.BlobID) + } + } + + return nil +} + +func init() { + contentGarbageCollectCommand.Action(repositoryAction(runContentGarbageCollectCommand)) +} diff --git a/cli/command_content_list.go b/cli/command_content_list.go new file mode 100644 index 000000000..4b62144e0 --- /dev/null +++ b/cli/command_content_list.go @@ -0,0 +1,101 @@ +package cli + +import ( + "context" + "fmt" + "sort" + + "github.com/kopia/kopia/repo" + "github.com/kopia/kopia/repo/blob" + "github.com/kopia/kopia/repo/content" +) + +var ( + contentListCommand = contentCommands.Command("list", "List contents").Alias("ls") + contentListLong = contentListCommand.Flag("long", "Long output").Short('l').Bool() + contentListPrefix = contentListCommand.Flag("prefix", "Prefix").String() + contentListIncludeDeleted = contentListCommand.Flag("deleted", "Include deleted content").Bool() + contentListDeletedOnly = contentListCommand.Flag("deleted-only", "Only show deleted content").Bool() + contentListSort = contentListCommand.Flag("sort", "Sort order").Default("name").Enum("name", "size", "time", "none", "pack") + contentListReverse = contentListCommand.Flag("reverse", "Reverse sort").Short('r').Bool() + contentListSummary = contentListCommand.Flag("summary", "Summarize the list").Short('s').Bool() + contentListHuman = contentListCommand.Flag("human", "Human-readable output").Short('h').Bool() +) + +func runContentListCommand(ctx context.Context, rep *repo.Repository) error { + contents, err := rep.Content.ListContentInfos(content.ID(*contentListPrefix), *contentListIncludeDeleted || *contentListDeletedOnly) + if err != nil { + return err + } + + sortContents(contents) + + var count int + var totalSize int64 + uniquePacks := map[blob.ID]bool{} + for _, b := range contents { + if *contentListDeletedOnly && !b.Deleted { + continue + } + totalSize += int64(b.Length) + count++ + if b.PackBlobID != "" { + uniquePacks[b.PackBlobID] = true + } + if *contentListLong { + optionalDeleted := "" + if b.Deleted { + optionalDeleted = " (deleted)" + } + fmt.Printf("%v %v %v %v+%v%v\n", + b.ID, + formatTimestamp(b.Timestamp()), + b.PackBlobID, + b.PackOffset, + maybeHumanReadableBytes(*contentListHuman, int64(b.Length)), + optionalDeleted) + } else { + fmt.Printf("%v\n", b.ID) + } + } + + if *contentListSummary { + fmt.Printf("Total: %v contents, %v packs, %v total size\n", + maybeHumanReadableCount(*contentListHuman, int64(count)), + maybeHumanReadableCount(*contentListHuman, int64(len(uniquePacks))), + maybeHumanReadableBytes(*contentListHuman, totalSize)) + } + + return nil +} + +func sortContents(contents []content.Info) { + maybeReverse := func(b bool) bool { return b } + + if *contentListReverse { + maybeReverse = func(b bool) bool { return !b } + } + + switch *contentListSort { + case "name": + sort.Slice(contents, func(i, j int) bool { return maybeReverse(contents[i].ID < contents[j].ID) }) + case "size": + sort.Slice(contents, func(i, j int) bool { return maybeReverse(contents[i].Length < contents[j].Length) }) + case "time": + sort.Slice(contents, func(i, j int) bool { return maybeReverse(contents[i].TimestampSeconds < contents[j].TimestampSeconds) }) + case "pack": + sort.Slice(contents, func(i, j int) bool { return maybeReverse(comparePacks(contents[i], contents[j])) }) + } +} + +func comparePacks(a, b content.Info) bool { + if a, b := a.PackBlobID, b.PackBlobID; a != b { + return a < b + } + + return a.PackOffset < b.PackOffset +} + +func init() { + contentListCommand.Action(repositoryAction(runContentListCommand)) +} diff --git a/cli/command_content_rewrite.go b/cli/command_content_rewrite.go new file mode 100644 index 000000000..3903d9c06 --- /dev/null +++ b/cli/command_content_rewrite.go @@ -0,0 +1,193 @@ +package cli + +import ( + "context" + "fmt" + "strings" + "sync" + + "github.com/pkg/errors" + + "github.com/kopia/kopia/repo" + "github.com/kopia/kopia/repo/blob" + "github.com/kopia/kopia/repo/content" +) + +var ( + contentRewriteCommand = contentCommands.Command("rewrite", "Rewrite content using most recent format") + contentRewriteIDs = contentRewriteCommand.Arg("contentID", "Identifiers of contents to rewrite").Strings() + contentRewriteParallelism = contentRewriteCommand.Flag("parallelism", "Number of parallel workers").Default("16").Int() + + contentRewriteShortPacks = contentRewriteCommand.Flag("short", "Rewrite contents from short packs").Bool() + contentRewriteFormatVersion = contentRewriteCommand.Flag("format-version", "Rewrite contents using the provided format version").Default("-1").Int() + contentRewritePackPrefix = contentRewriteCommand.Flag("pack-prefix", "Only rewrite contents from pack blobs with a given prefix").String() + contentRewriteDryRun = contentRewriteCommand.Flag("dry-run", "Do not actually rewrite, only print what would happen").Short('n').Bool() +) + +type contentInfoOrError struct { + content.Info + err error +} + +func runContentRewriteCommand(ctx context.Context, rep *repo.Repository) error { + cnt := getContentToRewrite(ctx, rep) + + var ( + mu sync.Mutex + totalBytes int64 + failedCount int + ) + + var wg sync.WaitGroup + + for i := 0; i < *contentRewriteParallelism; i++ { + wg.Add(1) + go func() { + defer wg.Done() + + for c := range cnt { + if c.err != nil { + log.Errorf("got error: %v", c.err) + mu.Lock() + failedCount++ + mu.Unlock() + return + } + + var optDeleted string + if c.Deleted { + optDeleted = " (deleted)" + } + + printStderr("Rewriting content %v (%v bytes) from pack %v%v\n", c.ID, c.Length, c.PackBlobID, optDeleted) + mu.Lock() + totalBytes += int64(c.Length) + mu.Unlock() + if *contentRewriteDryRun { + continue + } + if err := rep.Content.RewriteContent(ctx, c.ID); err != nil { + log.Warningf("unable to rewrite content %q: %v", c.ID, err) + mu.Lock() + failedCount++ + mu.Unlock() + } + } + }() + } + + wg.Wait() + + printStderr("Total bytes rewritten %v\n", totalBytes) + + if failedCount == 0 { + return nil + } + + return errors.Errorf("failed to rewrite %v contents", failedCount) +} + +func getContentToRewrite(ctx context.Context, rep *repo.Repository) <-chan contentInfoOrError { + ch := make(chan contentInfoOrError) + go func() { + defer close(ch) + + // get content IDs listed on command line + findContentInfos(ctx, rep, ch, toContentIDs(*contentRewriteIDs)) + + // add all content IDs from short packs + if *contentRewriteShortPacks { + threshold := uint32(rep.Content.Format.MaxPackSize * 6 / 10) + findContentInShortPacks(ctx, rep, ch, threshold) + } + + // add all blocks with given format version + if *contentRewriteFormatVersion != -1 { + findContentWithFormatVersion(ctx, rep, ch, *contentRewriteFormatVersion) + } + }() + + return ch +} + +func toContentIDs(s []string) []content.ID { + var result []content.ID + for _, cid := range s { + result = append(result, content.ID(cid)) + } + return result +} + +func findContentInfos(ctx context.Context, rep *repo.Repository, ch chan contentInfoOrError, contentIDs []content.ID) { + for _, contentID := range contentIDs { + i, err := rep.Content.ContentInfo(ctx, contentID) + if err != nil { + ch <- contentInfoOrError{err: errors.Wrapf(err, "unable to get info for content %q", contentID)} + } else { + ch <- contentInfoOrError{Info: i} + } + } +} + +func findContentWithFormatVersion(ctx context.Context, rep *repo.Repository, ch chan contentInfoOrError, version int) { + infos, err := rep.Content.ListContentInfos("", true) + if err != nil { + ch <- contentInfoOrError{err: errors.Wrap(err, "unable to list index blobs")} + return + } + + for _, b := range infos { + if int(b.FormatVersion) == *contentRewriteFormatVersion && strings.HasPrefix(string(b.PackBlobID), *contentRewritePackPrefix) { + ch <- contentInfoOrError{Info: b} + } + } +} + +func findContentInShortPacks(ctx context.Context, rep *repo.Repository, ch chan contentInfoOrError, threshold uint32) { + log.Debugf("listing contents...") + infos, err := rep.Content.ListContentInfos("", true) + if err != nil { + ch <- contentInfoOrError{err: errors.Wrap(err, "unable to list index blobs")} + return + } + + log.Debugf("finding content in short pack blobs...") + shortPackBlocks, err := findShortPackBlobs(infos, threshold) + if err != nil { + ch <- contentInfoOrError{err: errors.Wrap(err, "unable to find short pack blobs")} + return + } + log.Debugf("found %v short pack blobs", len(shortPackBlocks)) + + if len(shortPackBlocks) <= 1 { + fmt.Printf("Nothing to do, found %v short pack blobs\n", len(shortPackBlocks)) + } else { + for _, b := range infos { + if shortPackBlocks[b.PackBlobID] && strings.HasPrefix(string(b.PackBlobID), *contentRewritePackPrefix) { + ch <- contentInfoOrError{Info: b} + } + } + } +} + +func findShortPackBlobs(infos []content.Info, threshold uint32) (map[blob.ID]bool, error) { + packUsage := map[blob.ID]uint32{} + + for _, bi := range infos { + packUsage[bi.PackBlobID] += bi.Length + } + + shortPackBlocks := map[blob.ID]bool{} + + for blobID, usage := range packUsage { + if usage < threshold { + shortPackBlocks[blobID] = true + } + } + + return shortPackBlocks, nil +} + +func init() { + contentRewriteCommand.Action(repositoryAction(runContentRewriteCommand)) +} diff --git a/cli/command_content_rm.go b/cli/command_content_rm.go new file mode 100644 index 000000000..caee608db --- /dev/null +++ b/cli/command_content_rm.go @@ -0,0 +1,28 @@ +package cli + +import ( + "context" + + "github.com/kopia/kopia/repo" +) + +var ( + contentRemoveCommand = contentCommands.Command("remove", "Remove content").Alias("rm") + + contentRemoveIDs = contentRemoveCommand.Arg("id", "IDs of content to remove").Required().Strings() +) + +func runContentRemoveCommand(ctx context.Context, rep *repo.Repository) error { + for _, contentID := range toContentIDs(*contentRemoveIDs) { + if err := rep.Content.DeleteContent(contentID); err != nil { + return err + } + } + + return nil +} + +func init() { + setupShowCommand(contentRemoveCommand) + contentRemoveCommand.Action(repositoryAction(runContentRemoveCommand)) +} diff --git a/cli/command_content_show.go b/cli/command_content_show.go new file mode 100644 index 000000000..7f7086896 --- /dev/null +++ b/cli/command_content_show.go @@ -0,0 +1,39 @@ +package cli + +import ( + "bytes" + "context" + + "github.com/kopia/kopia/repo" + "github.com/kopia/kopia/repo/content" +) + +var ( + contentShowCommand = contentCommands.Command("show", "Show contents by ID.").Alias("cat") + + contentShowIDs = contentShowCommand.Arg("id", "IDs of contents to show").Required().Strings() +) + +func runContentShowCommand(ctx context.Context, rep *repo.Repository) error { + for _, contentID := range toContentIDs(*contentShowIDs) { + if err := contentShow(ctx, rep, contentID); err != nil { + return err + } + } + + return nil +} + +func contentShow(ctx context.Context, r *repo.Repository, contentID content.ID) error { + data, err := r.Content.GetContent(ctx, contentID) + if err != nil { + return err + } + + return showContent(bytes.NewReader(data)) +} + +func init() { + setupShowCommand(contentShowCommand) + contentShowCommand.Action(repositoryAction(runContentShowCommand)) +} diff --git a/cli/command_block_stats.go b/cli/command_content_stats.go similarity index 54% rename from cli/command_block_stats.go rename to cli/command_content_stats.go index e7f8ea547..d7595949a 100644 --- a/cli/command_block_stats.go +++ b/cli/command_content_stats.go @@ -8,24 +8,26 @@ "github.com/kopia/kopia/internal/units" "github.com/kopia/kopia/repo" - "github.com/kopia/kopia/repo/block" + "github.com/kopia/kopia/repo/content" ) var ( - blockStatsCommand = blockCommands.Command("stats", "Block statistics") - blockStatsRaw = blockStatsCommand.Flag("raw", "Raw numbers").Short('r').Bool() + contentStatsCommand = contentCommands.Command("stats", "Content statistics") + contentStatsRaw = contentStatsCommand.Flag("raw", "Raw numbers").Short('r').Bool() ) -func runBlockStatsAction(ctx context.Context, rep *repo.Repository) error { - blocks, err := rep.Blocks.ListBlockInfos("", true) +func runContentStatsCommand(ctx context.Context, rep *repo.Repository) error { + contents, err := rep.Content.ListContentInfos("", true) if err != nil { return err } - sort.Slice(blocks, func(i, j int) bool { return blocks[i].Length < blocks[j].Length }) + sort.Slice(contents, func(i, j int) bool { + return contents[i].Length < contents[j].Length + }) var sizeThreshold uint32 = 10 countMap := map[uint32]int{} - totalSizeOfBlocksUnder := map[uint32]int64{} + totalSizeOfContentsUnder := map[uint32]int64{} var sizeThresholds []uint32 for i := 0; i < 8; i++ { sizeThresholds = append(sizeThresholds, sizeThreshold) @@ -34,51 +36,51 @@ func runBlockStatsAction(ctx context.Context, rep *repo.Repository) error { } var totalSize int64 - for _, b := range blocks { + for _, b := range contents { totalSize += int64(b.Length) for s := range countMap { if b.Length < s { countMap[s]++ - totalSizeOfBlocksUnder[s] += int64(b.Length) + totalSizeOfContentsUnder[s] += int64(b.Length) } } } - fmt.Printf("Block statistics\n") - if len(blocks) == 0 { + fmt.Printf("Content statistics\n") + if len(contents) == 0 { return nil } sizeToString := units.BytesStringBase10 - if *blockStatsRaw { + if *contentStatsRaw { sizeToString = func(l int64) string { return strconv.FormatInt(l, 10) } } fmt.Println("Size: ") fmt.Println(" Total ", sizeToString(totalSize)) - fmt.Println(" Average ", sizeToString(totalSize/int64(len(blocks)))) - fmt.Println(" 1st percentile ", sizeToString(percentileSize(1, blocks))) - fmt.Println(" 5th percentile ", sizeToString(percentileSize(5, blocks))) - fmt.Println(" 10th percentile ", sizeToString(percentileSize(10, blocks))) - fmt.Println(" 50th percentile ", sizeToString(percentileSize(50, blocks))) - fmt.Println(" 90th percentile ", sizeToString(percentileSize(90, blocks))) - fmt.Println(" 95th percentile ", sizeToString(percentileSize(95, blocks))) - fmt.Println(" 99th percentile ", sizeToString(percentileSize(99, blocks))) + fmt.Println(" Average ", sizeToString(totalSize/int64(len(contents)))) + fmt.Println(" 1st percentile ", sizeToString(percentileSize(1, contents))) + fmt.Println(" 5th percentile ", sizeToString(percentileSize(5, contents))) + fmt.Println(" 10th percentile ", sizeToString(percentileSize(10, contents))) + fmt.Println(" 50th percentile ", sizeToString(percentileSize(50, contents))) + fmt.Println(" 90th percentile ", sizeToString(percentileSize(90, contents))) + fmt.Println(" 95th percentile ", sizeToString(percentileSize(95, contents))) + fmt.Println(" 99th percentile ", sizeToString(percentileSize(99, contents))) fmt.Println("Counts:") for _, size := range sizeThresholds { - fmt.Printf(" %v blocks with size <%v (total %v)\n", countMap[size], sizeToString(int64(size)), sizeToString(totalSizeOfBlocksUnder[size])) + fmt.Printf(" %v contents with size <%v (total %v)\n", countMap[size], sizeToString(int64(size)), sizeToString(totalSizeOfContentsUnder[size])) } return nil } -func percentileSize(p int, blocks []block.Info) int64 { - pos := p * len(blocks) / 100 +func percentileSize(p int, contents []content.Info) int64 { + pos := p * len(contents) / 100 - return int64(blocks[pos].Length) + return int64(contents[pos].Length) } func init() { - blockStatsCommand.Action(repositoryAction(runBlockStatsAction)) + contentStatsCommand.Action(repositoryAction(runContentStatsCommand)) } diff --git a/cli/command_content_verify.go b/cli/command_content_verify.go new file mode 100644 index 000000000..28a0c306c --- /dev/null +++ b/cli/command_content_verify.go @@ -0,0 +1,62 @@ +package cli + +import ( + "context" + + "github.com/pkg/errors" + + "github.com/kopia/kopia/repo" + "github.com/kopia/kopia/repo/content" +) + +var ( + contentVerifyCommand = contentCommands.Command("verify", "Verify contents") + + contentVerifyIDs = contentVerifyCommand.Arg("id", "IDs of blocks to show (or 'all')").Required().Strings() +) + +func runContentVerifyCommand(ctx context.Context, rep *repo.Repository) error { + for _, contentID := range toContentIDs(*contentVerifyIDs) { + if contentID == "all" { + return verifyAllBlocks(ctx, rep) + } + if err := contentVerify(ctx, rep, contentID); err != nil { + return err + } + } + + return nil +} + +func verifyAllBlocks(ctx context.Context, rep *repo.Repository) error { + contentIDs, err := rep.Content.ListContents("") + if err != nil { + return errors.Wrap(err, "unable to list contents") + } + + var errorCount int + for _, contentID := range contentIDs { + if err := contentVerify(ctx, rep, contentID); err != nil { + errorCount++ + } + } + if errorCount == 0 { + return nil + } + + return errors.Errorf("encountered %v errors", errorCount) +} + +func contentVerify(ctx context.Context, r *repo.Repository, contentID content.ID) error { + if _, err := r.Content.GetContent(ctx, contentID); err != nil { + log.Warningf("content %v is invalid: %v", contentID, err) + return err + } + + log.Infof("content %v is ok", contentID) + return nil +} + +func init() { + contentVerifyCommand.Action(repositoryAction(runContentVerifyCommand)) +} diff --git a/cli/command_block_index_list.go b/cli/command_index_list.go similarity index 80% rename from cli/command_block_index_list.go rename to cli/command_index_list.go index f3fac44bc..7d603f452 100644 --- a/cli/command_block_index_list.go +++ b/cli/command_index_list.go @@ -9,13 +9,13 @@ ) var ( - blockIndexListCommand = blockIndexCommands.Command("list", "List block indexes").Alias("ls").Default() - blockIndexListSummary = blockIndexListCommand.Flag("summary", "Display block summary").Bool() - blockIndexListSort = blockIndexListCommand.Flag("sort", "Index block sort order").Default("time").Enum("time", "size", "name") + blockIndexListCommand = indexCommands.Command("list", "List content indexes").Alias("ls").Default() + blockIndexListSummary = blockIndexListCommand.Flag("summary", "Display index blob summary").Bool() + blockIndexListSort = blockIndexListCommand.Flag("sort", "Index blob sort order").Default("time").Enum("time", "size", "name") ) func runListBlockIndexesAction(ctx context.Context, rep *repo.Repository) error { - blks, err := rep.Blocks.IndexBlobs(ctx) + blks, err := rep.Content.IndexBlobs(ctx) if err != nil { return err } diff --git a/cli/command_index_optimize.go b/cli/command_index_optimize.go new file mode 100644 index 000000000..23e1af08c --- /dev/null +++ b/cli/command_index_optimize.go @@ -0,0 +1,29 @@ +package cli + +import ( + "context" + + "github.com/kopia/kopia/repo" + "github.com/kopia/kopia/repo/content" +) + +var ( + optimizeCommand = indexCommands.Command("optimize", "Optimize indexes blobs.") + optimizeMinSmallBlobs = optimizeCommand.Flag("min-small-blobs", "Minimum number of small index blobs that can be left after compaction.").Default("1").Int() + optimizeMaxSmallBlobs = optimizeCommand.Flag("max-small-blobs", "Maximum number of small index blobs that can be left after compaction.").Default("1").Int() + optimizeSkipDeletedOlderThan = optimizeCommand.Flag("skip-deleted-older-than", "Skip deleted blobs above given age").Duration() + optimizeAllIndexes = optimizeCommand.Flag("all", "Optimize all indexes, even those above maximum size.").Bool() +) + +func runOptimizeCommand(ctx context.Context, rep *repo.Repository) error { + return rep.Content.CompactIndexes(ctx, content.CompactOptions{ + MinSmallBlobs: *optimizeMinSmallBlobs, + MaxSmallBlobs: *optimizeMaxSmallBlobs, + AllIndexes: *optimizeAllIndexes, + SkipDeletedOlderThan: *optimizeSkipDeletedOlderThan, + }) +} + +func init() { + optimizeCommand.Action(repositoryAction(runOptimizeCommand)) +} diff --git a/cli/command_block_index_recover.go b/cli/command_index_recover.go similarity index 76% rename from cli/command_block_index_recover.go rename to cli/command_index_recover.go index e5116b5ce..8cf324c19 100644 --- a/cli/command_block_index_recover.go +++ b/cli/command_index_recover.go @@ -5,13 +5,13 @@ "github.com/kopia/kopia/repo" "github.com/kopia/kopia/repo/blob" - "github.com/kopia/kopia/repo/block" + "github.com/kopia/kopia/repo/content" ) var ( - blockIndexRecoverCommand = blockIndexCommands.Command("recover", "Recover block indexes from pack blocks") - blockIndexRecoverBlobIDs = blockIndexRecoverCommand.Flag("blobs", "Names of pack blobs to recover (default=all packs)").Strings() - blockIndexRecoverCommit = blockIndexRecoverCommand.Flag("commit", "Commit recovered blocks").Bool() + blockIndexRecoverCommand = indexCommands.Command("recover", "Recover indexes from pack blobs") + blockIndexRecoverBlobIDs = blockIndexRecoverCommand.Flag("blobs", "Names of pack blobs to recover from (default=all packs)").Strings() + blockIndexRecoverCommit = blockIndexRecoverCommand.Flag("commit", "Commit recovered content").Bool() ) func runRecoverBlockIndexesAction(ctx context.Context, rep *repo.Repository) error { @@ -31,7 +31,7 @@ func runRecoverBlockIndexesAction(ctx context.Context, rep *repo.Repository) err }() if len(*blockIndexRecoverBlobIDs) == 0 { - return rep.Blobs.ListBlobs(ctx, block.PackBlobIDPrefix, func(bm blob.Metadata) error { + return rep.Blobs.ListBlobs(ctx, content.PackBlobIDPrefix, func(bm blob.Metadata) error { recoverIndexFromSinglePackFile(ctx, rep, bm.BlobID, bm.Length, &totalCount) return nil }) @@ -45,7 +45,7 @@ func runRecoverBlockIndexesAction(ctx context.Context, rep *repo.Repository) err } func recoverIndexFromSinglePackFile(ctx context.Context, rep *repo.Repository, blobID blob.ID, length int64, totalCount *int) { - recovered, err := rep.Blocks.RecoverIndexFromPackBlob(ctx, blobID, length, *blockIndexRecoverCommit) + recovered, err := rep.Content.RecoverIndexFromPackBlob(ctx, blobID, length, *blockIndexRecoverCommit) if err != nil { log.Warningf("unable to recover index from %v: %v", blobID, err) return diff --git a/cli/command_manifest_rm.go b/cli/command_manifest_rm.go index 80acdad6b..6e476f64c 100644 --- a/cli/command_manifest_rm.go +++ b/cli/command_manifest_rm.go @@ -11,12 +11,8 @@ manifestRemoveItems = manifestRemoveCommand.Arg("item", "Items to remove").Required().Strings() ) -func init() { - manifestRemoveCommand.Action(repositoryAction(removeMetadataItem)) -} - -func removeMetadataItem(ctx context.Context, rep *repo.Repository) error { - for _, it := range *manifestRemoveItems { +func runManifestRemoveCommand(ctx context.Context, rep *repo.Repository) error { + for _, it := range toManifestIDs(*manifestRemoveItems) { if err := rep.Manifests.Delete(ctx, it); err != nil { return err } @@ -24,3 +20,7 @@ func removeMetadataItem(ctx context.Context, rep *repo.Repository) error { return nil } + +func init() { + manifestRemoveCommand.Action(repositoryAction(runManifestRemoveCommand)) +} diff --git a/cli/command_manifest_show.go b/cli/command_manifest_show.go index aeb320f96..b6a4f5292 100644 --- a/cli/command_manifest_show.go +++ b/cli/command_manifest_show.go @@ -7,6 +7,7 @@ "github.com/pkg/errors" "github.com/kopia/kopia/repo" + "github.com/kopia/kopia/repo/manifest" ) var ( @@ -18,8 +19,17 @@ func init() { manifestShowCommand.Action(repositoryAction(showManifestItems)) } +func toManifestIDs(s []string) []manifest.ID { + var result []manifest.ID + + for _, it := range s { + result = append(result, manifest.ID(it)) + } + return result +} + func showManifestItems(ctx context.Context, rep *repo.Repository) error { - for _, it := range *manifestShowItems { + for _, it := range toManifestIDs(*manifestShowItems) { md, err := rep.Manifests.GetMetadata(ctx, it) if err != nil { return errors.Wrapf(err, "error getting metadata for %q", it) diff --git a/cli/command_object_verify.go b/cli/command_object_verify.go index f285f705f..306055007 100644 --- a/cli/command_object_verify.go +++ b/cli/command_object_verify.go @@ -13,7 +13,8 @@ "github.com/kopia/kopia/internal/parallelwork" "github.com/kopia/kopia/repo" - "github.com/kopia/kopia/repo/block" + "github.com/kopia/kopia/repo/content" + "github.com/kopia/kopia/repo/manifest" "github.com/kopia/kopia/repo/object" "github.com/kopia/kopia/snapshot" "github.com/kopia/kopia/snapshot/snapshotfs" @@ -163,7 +164,7 @@ func (v *verifier) doVerifyObject(ctx context.Context, oid object.ID, path strin func (v *verifier) readEntireObject(ctx context.Context, oid object.ID, path string) error { log.Debugf("reading object %v %v", oid, path) - ctx = block.UsingBlockCache(ctx, false) + ctx = content.UsingContentCache(ctx, false) // also read the entire file r, err := v.om.Open(ctx, oid) @@ -240,7 +241,7 @@ func enqueueRootsToVerify(ctx context.Context, v *verifier, rep *repo.Repository } func loadSourceManifests(ctx context.Context, rep *repo.Repository, all bool, sources []string) ([]*snapshot.Manifest, error) { - var manifestIDs []string + var manifestIDs []manifest.ID if *verifyCommandAllSources { man, err := snapshot.ListSnapshotManifests(ctx, rep, nil) if err != nil { diff --git a/cli/command_policy.go b/cli/command_policy.go index 80e5fd34d..ba1af94bc 100644 --- a/cli/command_policy.go +++ b/cli/command_policy.go @@ -6,6 +6,7 @@ "github.com/pkg/errors" "github.com/kopia/kopia/repo" + "github.com/kopia/kopia/repo/manifest" "github.com/kopia/kopia/snapshot" "github.com/kopia/kopia/snapshot/policy" ) @@ -23,7 +24,8 @@ func policyTargets(ctx context.Context, rep *repo.Repository, globalFlag *bool, var res []snapshot.SourceInfo for _, ts := range *targetsFlag { - if t, err := policy.GetPolicyByID(ctx, rep, ts); err == nil { + // try loading policy by its manifest ID + if t, err := policy.GetPolicyByID(ctx, rep, manifest.ID(ts)); err == nil { res = append(res, t.Target()) continue } diff --git a/cli/command_repository_connect.go b/cli/command_repository_connect.go index 81ed51ecc..b5493e7b2 100644 --- a/cli/command_repository_connect.go +++ b/cli/command_repository_connect.go @@ -8,7 +8,7 @@ "github.com/kopia/kopia/repo" "github.com/kopia/kopia/repo/blob" - "github.com/kopia/kopia/repo/block" + "github.com/kopia/kopia/repo/content" "gopkg.in/alecthomas/kingpin.v2" ) @@ -32,7 +32,7 @@ func setupConnectOptions(cmd *kingpin.CmdClause) { func connectOptions() repo.ConnectOptions { return repo.ConnectOptions{ - CachingOptions: block.CachingOptions{ + CachingOptions: content.CachingOptions{ CacheDirectory: connectCacheDirectory, MaxCacheSizeBytes: connectMaxCacheSizeMB << 20, MaxListCacheDurationSec: int(connectMaxListCacheDuration.Seconds()), diff --git a/cli/command_repository_create.go b/cli/command_repository_create.go index c8abd8811..69fa3fd85 100644 --- a/cli/command_repository_create.go +++ b/cli/command_repository_create.go @@ -9,7 +9,7 @@ "github.com/kopia/kopia/fs/ignorefs" "github.com/kopia/kopia/repo" "github.com/kopia/kopia/repo/blob" - "github.com/kopia/kopia/repo/block" + "github.com/kopia/kopia/repo/content" "github.com/kopia/kopia/repo/object" "github.com/kopia/kopia/snapshot/policy" ) @@ -17,8 +17,8 @@ var ( createCommand = repositoryCommands.Command("create", "Create new repository in a specified location.") - createBlockHashFormat = createCommand.Flag("block-hash", "Block hash algorithm.").PlaceHolder("ALGO").Default(block.DefaultHash).Enum(block.SupportedHashAlgorithms()...) - createBlockEncryptionFormat = createCommand.Flag("encryption", "Block encryption algorithm.").PlaceHolder("ALGO").Default(block.DefaultEncryption).Enum(block.SupportedEncryptionAlgorithms()...) + createBlockHashFormat = createCommand.Flag("block-hash", "Block hash algorithm.").PlaceHolder("ALGO").Default(content.DefaultHash).Enum(content.SupportedHashAlgorithms()...) + createBlockEncryptionFormat = createCommand.Flag("encryption", "Block encryption algorithm.").PlaceHolder("ALGO").Default(content.DefaultEncryption).Enum(content.SupportedEncryptionAlgorithms()...) createSplitter = createCommand.Flag("object-splitter", "The splitter to use for new objects in the repository").Default(object.DefaultSplitter).Enum(object.SupportedSplitters...) createOnly = createCommand.Flag("create-only", "Create repository, but don't connect to it.").Short('c').Bool() @@ -42,7 +42,7 @@ func init() { func newRepositoryOptionsFromFlags() *repo.NewRepositoryOptions { return &repo.NewRepositoryOptions{ - BlockFormat: block.FormattingOptions{ + BlockFormat: content.FormattingOptions{ Hash: *createBlockHashFormat, Encryption: *createBlockEncryptionFormat, }, diff --git a/cli/command_repository_repair.go b/cli/command_repository_repair.go index 6e1334921..b88f7d783 100644 --- a/cli/command_repository_repair.go +++ b/cli/command_repository_repair.go @@ -12,20 +12,20 @@ var ( repairCommand = repositoryCommands.Command("repair", "Repairs respository.") - repairCommandRecoverFormatBlock = repairCommand.Flag("recover-format", "Recover format block from a copy").Default("auto").Enum("auto", "yes", "no") - repairCommandRecoverFormatBlockPrefix = repairCommand.Flag("recover-format-block-prefix", "Prefix of file names").Default("p").String() - repairDryDrun = repairCommand.Flag("dry-run", "Do not modify repository").Short('n').Bool() + repairCommandRecoverFormatBlob = repairCommand.Flag("recover-format", "Recover format blob from a copy").Default("auto").Enum("auto", "yes", "no") + repairCommandRecoverFormatBlobPrefix = repairCommand.Flag("recover-format-block-prefix", "Prefix of file names").Default("p").String() + repairDryDrun = repairCommand.Flag("dry-run", "Do not modify repository").Short('n').Bool() ) func runRepairCommandWithStorage(ctx context.Context, st blob.Storage) error { - if err := maybeRecoverFormatBlock(ctx, st, *repairCommandRecoverFormatBlockPrefix); err != nil { + if err := maybeRecoverFormatBlob(ctx, st, *repairCommandRecoverFormatBlobPrefix); err != nil { return err } return nil } -func maybeRecoverFormatBlock(ctx context.Context, st blob.Storage, prefix string) error { - switch *repairCommandRecoverFormatBlock { +func maybeRecoverFormatBlob(ctx context.Context, st blob.Storage, prefix string) error { + switch *repairCommandRecoverFormatBlob { case "auto": log.Infof("looking for format blob...") if _, err := st.GetBlob(ctx, repo.FormatBlobID, 0, -1); err == nil { @@ -37,15 +37,15 @@ func maybeRecoverFormatBlock(ctx context.Context, st blob.Storage, prefix string return nil } - return recoverFormatBlock(ctx, st, *repairCommandRecoverFormatBlockPrefix) + return recoverFormatBlob(ctx, st, *repairCommandRecoverFormatBlobPrefix) } -func recoverFormatBlock(ctx context.Context, st blob.Storage, prefix string) error { +func recoverFormatBlob(ctx context.Context, st blob.Storage, prefix string) error { errSuccess := errors.New("success") - err := st.ListBlobs(ctx, blob.ID(*repairCommandRecoverFormatBlockPrefix), func(bi blob.Metadata) error { - log.Infof("looking for replica of format block in %v...", bi.BlobID) - if b, err := repo.RecoverFormatBlock(ctx, st, bi.BlobID, bi.Length); err == nil { + err := st.ListBlobs(ctx, blob.ID(*repairCommandRecoverFormatBlobPrefix), func(bi blob.Metadata) error { + log.Infof("looking for replica of format blob in %v...", bi.BlobID) + if b, err := repo.RecoverFormatBlob(ctx, st, bi.BlobID, bi.Length); err == nil { if !*repairDryDrun { if puterr := st.PutBlob(ctx, repo.FormatBlobID, b); puterr != nil { return puterr @@ -63,7 +63,7 @@ func recoverFormatBlock(ctx context.Context, st blob.Storage, prefix string) err case errSuccess: return nil case nil: - return errors.New("could not find a replica of a format block") + return errors.New("could not find a replica of a format blob") default: return err } diff --git a/cli/command_repository_status.go b/cli/command_repository_status.go index c22e00eb8..421bacdea 100644 --- a/cli/command_repository_status.go +++ b/cli/command_repository_status.go @@ -38,10 +38,10 @@ func runStatusCommand(ctx context.Context, rep *repo.Repository) error { fmt.Println() fmt.Printf("Unique ID: %x\n", rep.UniqueID) fmt.Println() - fmt.Printf("Block hash: %v\n", rep.Blocks.Format.Hash) - fmt.Printf("Block encryption: %v\n", rep.Blocks.Format.Encryption) - fmt.Printf("Block fmt version: %v\n", rep.Blocks.Format.Version) - fmt.Printf("Max pack length: %v\n", units.BytesStringBase2(int64(rep.Blocks.Format.MaxPackSize))) + fmt.Printf("Block hash: %v\n", rep.Content.Format.Hash) + fmt.Printf("Block encryption: %v\n", rep.Content.Format.Encryption) + fmt.Printf("Block fmt version: %v\n", rep.Content.Format.Version) + fmt.Printf("Max pack length: %v\n", units.BytesStringBase2(int64(rep.Content.Format.MaxPackSize))) fmt.Printf("Splitter: %v\n", rep.Objects.Format.Splitter) return nil diff --git a/cli/command_snapshot_create.go b/cli/command_snapshot_create.go index 75a1e22ec..6a0cb2e81 100644 --- a/cli/command_snapshot_create.go +++ b/cli/command_snapshot_create.go @@ -85,7 +85,7 @@ func runBackupCommand(ctx context.Context, rep *repo.Repository) error { func snapshotSingleSource(ctx context.Context, rep *repo.Repository, u *snapshotfs.Uploader, sourceInfo snapshot.SourceInfo) error { t0 := time.Now() - rep.Blocks.ResetStats() + rep.Content.ResetStats() localEntry := mustGetLocalFSEntry(sourceInfo.Path) diff --git a/cli/command_snapshot_list.go b/cli/command_snapshot_list.go index 3375fa2ee..f27ca7e7b 100644 --- a/cli/command_snapshot_list.go +++ b/cli/command_snapshot_list.go @@ -12,6 +12,7 @@ "github.com/kopia/kopia/fs" "github.com/kopia/kopia/internal/units" "github.com/kopia/kopia/repo" + "github.com/kopia/kopia/repo/manifest" "github.com/kopia/kopia/repo/object" "github.com/kopia/kopia/snapshot" "github.com/kopia/kopia/snapshot/policy" @@ -34,7 +35,7 @@ maxResultsPerPath = snapshotListCommand.Flag("max-results", "Maximum number of entries per source.").Default("100").Short('n').Int() ) -func findSnapshotsForSource(ctx context.Context, rep *repo.Repository, sourceInfo snapshot.SourceInfo) (manifestIDs []string, relPath string, err error) { +func findSnapshotsForSource(ctx context.Context, rep *repo.Repository, sourceInfo snapshot.SourceInfo) (manifestIDs []manifest.ID, relPath string, err error) { for len(sourceInfo.Path) > 0 { list, err := snapshot.ListSnapshotManifests(ctx, rep, &sourceInfo) if err != nil { @@ -63,7 +64,7 @@ func findSnapshotsForSource(ctx context.Context, rep *repo.Repository, sourceInf return nil, "", nil } -func findManifestIDs(ctx context.Context, rep *repo.Repository, source string) ([]string, string, error) { +func findManifestIDs(ctx context.Context, rep *repo.Repository, source string) ([]manifest.ID, string, error) { if source == "" { man, err := snapshot.ListSnapshotManifests(ctx, rep, nil) return man, "", err @@ -200,7 +201,7 @@ func outputManifestFromSingleSource(ctx context.Context, rep *repo.Repository, m } if *snapshotListShowItemID { - bits = append(bits, "manifest:"+m.ID) + bits = append(bits, "manifest:"+string(m.ID)) } if *snapshotListShowHashCache { bits = append(bits, "hashcache:"+m.HashCacheID.String()) diff --git a/examples/upload_download/main.go b/examples/upload_download/main.go index 08a5f98bc..d49da0e65 100644 --- a/examples/upload_download/main.go +++ b/examples/upload_download/main.go @@ -28,13 +28,13 @@ func main() { uploadAndDownloadObjects(ctx, r) - // Now list blocks found in the repository. - blks, err := r.Blocks.ListBlocks("") + // Now list contents found in the repository. + cnts, err := r.Content.ListContents("") if err != nil { log.Printf("err: %v", err) } - for _, b := range blks { - log.Printf("found block %v", b) + for _, c := range cnts { + log.Printf("found content %v", c) } } diff --git a/examples/upload_download/setup_repository.go b/examples/upload_download/setup_repository.go index ab7c4d7c9..7e5be02a2 100644 --- a/examples/upload_download/setup_repository.go +++ b/examples/upload_download/setup_repository.go @@ -11,7 +11,7 @@ "github.com/kopia/kopia/repo" "github.com/kopia/kopia/repo/blob/filesystem" "github.com/kopia/kopia/repo/blob/logging" - "github.com/kopia/kopia/repo/block" + "github.com/kopia/kopia/repo/content" ) const ( @@ -44,7 +44,7 @@ func setupRepositoryAndConnect(ctx context.Context, password string) error { // now establish connection to repository and create configuration file. if err := repo.Connect(ctx, configFile, st, password, repo.ConnectOptions{ - CachingOptions: block.CachingOptions{ + CachingOptions: content.CachingOptions{ CacheDirectory: cacheDirectory, MaxCacheSizeBytes: 100000000, }, diff --git a/internal/blobtesting/asserts.go b/internal/blobtesting/asserts.go index 29e40d4e0..435db6755 100644 --- a/internal/blobtesting/asserts.go +++ b/internal/blobtesting/asserts.go @@ -10,18 +10,18 @@ "github.com/kopia/kopia/repo/blob" ) -// AssertGetBlock asserts that the specified storage block has correct content. -func AssertGetBlock(ctx context.Context, t *testing.T, s blob.Storage, block blob.ID, expected []byte) { +// AssertGetBlob asserts that the specified BLOB has correct content. +func AssertGetBlob(ctx context.Context, t *testing.T, s blob.Storage, blobID blob.ID, expected []byte) { t.Helper() - b, err := s.GetBlob(ctx, block, 0, -1) + b, err := s.GetBlob(ctx, blobID, 0, -1) if err != nil { - t.Errorf("GetBlob(%v) returned error %v, expected data: %v", block, err, expected) + t.Errorf("GetBlob(%v) returned error %v, expected data: %v", blobID, err, expected) return } if !bytes.Equal(b, expected) { - t.Errorf("GetBlob(%v) returned %x, but expected %x", block, b, expected) + t.Errorf("GetBlob(%v) returned %x, but expected %x", blobID, b, expected) } half := int64(len(expected) / 2) @@ -29,41 +29,41 @@ func AssertGetBlock(ctx context.Context, t *testing.T, s blob.Storage, block blo return } - b, err = s.GetBlob(ctx, block, 0, 0) + b, err = s.GetBlob(ctx, blobID, 0, 0) if err != nil { - t.Errorf("GetBlob(%v) returned error %v, expected data: %v", block, err, expected) + t.Errorf("GetBlob(%v) returned error %v, expected data: %v", blobID, err, expected) return } if len(b) != 0 { - t.Errorf("GetBlob(%v) returned non-zero length: %v", block, len(b)) + t.Errorf("GetBlob(%v) returned non-zero length: %v", blobID, len(b)) return } - b, err = s.GetBlob(ctx, block, 0, half) + b, err = s.GetBlob(ctx, blobID, 0, half) if err != nil { - t.Errorf("GetBlob(%v) returned error %v, expected data: %v", block, err, expected) + t.Errorf("GetBlob(%v) returned error %v, expected data: %v", blobID, err, expected) return } if !bytes.Equal(b, expected[0:half]) { - t.Errorf("GetBlob(%v) returned %x, but expected %x", block, b, expected[0:half]) + t.Errorf("GetBlob(%v) returned %x, but expected %x", blobID, b, expected[0:half]) } - b, err = s.GetBlob(ctx, block, half, int64(len(expected))-half) + b, err = s.GetBlob(ctx, blobID, half, int64(len(expected))-half) if err != nil { - t.Errorf("GetBlob(%v) returned error %v, expected data: %v", block, err, expected) + t.Errorf("GetBlob(%v) returned error %v, expected data: %v", blobID, err, expected) return } if !bytes.Equal(b, expected[len(expected)-int(half):]) { - t.Errorf("GetBlob(%v) returned %x, but expected %x", block, b, expected[len(expected)-int(half):]) + t.Errorf("GetBlob(%v) returned %x, but expected %x", blobID, b, expected[len(expected)-int(half):]) } - AssertInvalidOffsetLength(ctx, t, s, block, -3, 1) - AssertInvalidOffsetLength(ctx, t, s, block, int64(len(expected)), 3) - AssertInvalidOffsetLength(ctx, t, s, block, int64(len(expected)-1), 3) - AssertInvalidOffsetLength(ctx, t, s, block, int64(len(expected)+1), 3) + AssertInvalidOffsetLength(ctx, t, s, blobID, -3, 1) + AssertInvalidOffsetLength(ctx, t, s, blobID, int64(len(expected)), 3) + AssertInvalidOffsetLength(ctx, t, s, blobID, int64(len(expected)-1), 3) + AssertInvalidOffsetLength(ctx, t, s, blobID, int64(len(expected)+1), 3) } // AssertInvalidOffsetLength verifies that the given combination of (offset,length) fails on GetBlob() @@ -73,8 +73,8 @@ func AssertInvalidOffsetLength(ctx context.Context, t *testing.T, s blob.Storage } } -// AssertGetBlockNotFound asserts that GetBlob() for specified storage block returns ErrNotFound. -func AssertGetBlockNotFound(ctx context.Context, t *testing.T, s blob.Storage, blobID blob.ID) { +// AssertGetBlobNotFound asserts that GetBlob() for specified blobID returns ErrNotFound. +func AssertGetBlobNotFound(ctx context.Context, t *testing.T, s blob.Storage, blobID blob.ID) { t.Helper() b, err := s.GetBlob(ctx, blobID, 0, -1) diff --git a/internal/blobtesting/verify.go b/internal/blobtesting/verify.go index afa518e45..7a427ab33 100644 --- a/internal/blobtesting/verify.go +++ b/internal/blobtesting/verify.go @@ -24,7 +24,7 @@ func VerifyStorage(ctx context.Context, t *testing.T, r blob.Storage) { // First verify that blocks don't exist. for _, b := range blocks { - AssertGetBlockNotFound(ctx, t, r, b.blk) + AssertGetBlobNotFound(ctx, t, r, b.blk) } ctx2 := blob.WithUploadProgressCallback(ctx, func(desc string, completed, total int64) { @@ -34,10 +34,10 @@ func VerifyStorage(ctx context.Context, t *testing.T, r blob.Storage) { // Now add blocks. for _, b := range blocks { if err := r.PutBlob(ctx2, b.blk, b.contents); err != nil { - t.Errorf("can't put block: %v", err) + t.Errorf("can't put blob: %v", err) } - AssertGetBlock(ctx, t, r, b.blk, b.contents) + AssertGetBlob(ctx, t, r, b.blk, b.contents) } AssertListResults(ctx, t, r, "", blocks[0].blk, blocks[1].blk, blocks[2].blk, blocks[3].blk, blocks[4].blk) @@ -46,10 +46,10 @@ func VerifyStorage(ctx context.Context, t *testing.T, r blob.Storage) { // Overwrite blocks. for _, b := range blocks { if err := r.PutBlob(ctx, b.blk, b.contents); err != nil { - t.Errorf("can't put block: %v", err) + t.Errorf("can't put blob: %v", err) } - AssertGetBlock(ctx, t, r, b.blk, b.contents) + AssertGetBlob(ctx, t, r, b.blk, b.contents) } if err := r.DeleteBlob(ctx, blocks[0].blk); err != nil { diff --git a/internal/repotesting/repotesting.go b/internal/repotesting/repotesting.go index 688d2a26d..64c9a38aa 100644 --- a/internal/repotesting/repotesting.go +++ b/internal/repotesting/repotesting.go @@ -11,7 +11,7 @@ "github.com/kopia/kopia/repo" "github.com/kopia/kopia/repo/blob" "github.com/kopia/kopia/repo/blob/filesystem" - "github.com/kopia/kopia/repo/block" + "github.com/kopia/kopia/repo/content" "github.com/kopia/kopia/repo/object" ) @@ -42,7 +42,7 @@ func (e *Environment) Setup(t *testing.T, opts ...func(*repo.NewRepositoryOption } opt := &repo.NewRepositoryOptions{ - BlockFormat: block.FormattingOptions{ + BlockFormat: content.FormattingOptions{ HMACSecret: []byte{}, Hash: "HMAC-SHA256", Encryption: "NONE", @@ -121,8 +121,8 @@ func (e *Environment) MustReopen(t *testing.T) { } } -// VerifyStorageBlockCount verifies that the underlying storage contains the specified number of blocks. -func (e *Environment) VerifyStorageBlockCount(t *testing.T, want int) { +// VerifyBlobCount verifies that the underlying storage contains the specified number of blobs. +func (e *Environment) VerifyBlobCount(t *testing.T, want int) { var got int _ = e.Repository.Blobs.ListBlobs(context.Background(), "", func(_ blob.Metadata) error { @@ -131,6 +131,6 @@ func (e *Environment) VerifyStorageBlockCount(t *testing.T, want int) { }) if got != want { - t.Errorf("got unexpected number of storage blocks: %v, wanted %v", got, want) + t.Errorf("got unexpected number of BLOBs: %v, wanted %v", got, want) } } diff --git a/internal/server/api_snapshot_list.go b/internal/server/api_snapshot_list.go index 994adefa5..61da35652 100644 --- a/internal/server/api_snapshot_list.go +++ b/internal/server/api_snapshot_list.go @@ -8,12 +8,13 @@ "time" "github.com/kopia/kopia/fs" + "github.com/kopia/kopia/repo/manifest" "github.com/kopia/kopia/snapshot" "github.com/kopia/kopia/snapshot/policy" ) type snapshotListEntry struct { - ID string `json:"id"` + ID manifest.ID `json:"id"` Source snapshot.SourceInfo `json:"source"` Description string `json:"description"` StartTime time.Time `json:"startTime"` diff --git a/internal/server/api_status.go b/internal/server/api_status.go index 24c6f040e..fa670dd3c 100644 --- a/internal/server/api_status.go +++ b/internal/server/api_status.go @@ -8,7 +8,7 @@ ) func (s *Server) handleStatus(ctx context.Context, r *http.Request) (interface{}, *apiError) { - bf := s.rep.Blocks.Format + bf := s.rep.Content.Format bf.HMACSecret = nil bf.MasterKey = nil diff --git a/internal/serverapi/serverapi.go b/internal/serverapi/serverapi.go index 37b515031..3029c1144 100644 --- a/internal/serverapi/serverapi.go +++ b/internal/serverapi/serverapi.go @@ -3,17 +3,17 @@ import ( "time" - "github.com/kopia/kopia/repo/block" + "github.com/kopia/kopia/repo/content" "github.com/kopia/kopia/snapshot" "github.com/kopia/kopia/snapshot/policy" ) // StatusResponse is the response of 'status' HTTP API command. type StatusResponse struct { - ConfigFile string `json:"configFile"` - CacheDir string `json:"cacheDir"` - BlockFormatting block.FormattingOptions `json:"blockFormatting"` - Storage string `json:"storage"` + ConfigFile string `json:"configFile"` + CacheDir string `json:"cacheDir"` + BlockFormatting content.FormattingOptions `json:"blockFormatting"` + Storage string `json:"storage"` } // SourcesResponse is the response of 'sources' HTTP API command. diff --git a/repo/block/block_index_recovery_test.go b/repo/block/block_index_recovery_test.go deleted file mode 100644 index f9d9d2287..000000000 --- a/repo/block/block_index_recovery_test.go +++ /dev/null @@ -1,91 +0,0 @@ -package block - -import ( - "context" - "testing" - "time" - - "github.com/kopia/kopia/internal/blobtesting" - "github.com/kopia/kopia/repo/blob" -) - -func TestBlockIndexRecovery(t *testing.T) { - ctx := context.Background() - data := blobtesting.DataMap{} - keyTime := map[blob.ID]time.Time{} - bm := newTestBlockManager(data, keyTime, nil) - block1 := writeBlockAndVerify(ctx, t, bm, seededRandomData(10, 100)) - block2 := writeBlockAndVerify(ctx, t, bm, seededRandomData(11, 100)) - block3 := writeBlockAndVerify(ctx, t, bm, seededRandomData(12, 100)) - - if err := bm.Flush(ctx); err != nil { - t.Errorf("flush error: %v", err) - } - - // delete all index blobs - assertNoError(t, bm.st.ListBlobs(ctx, newIndexBlobPrefix, func(bi blob.Metadata) error { - log.Debugf("deleting %v", bi.BlobID) - return bm.st.DeleteBlob(ctx, bi.BlobID) - })) - - // now with index blobs gone, all blocks appear to not be found - bm = newTestBlockManager(data, keyTime, nil) - verifyBlockNotFound(ctx, t, bm, block1) - verifyBlockNotFound(ctx, t, bm, block2) - verifyBlockNotFound(ctx, t, bm, block3) - - totalRecovered := 0 - - // pass 1 - just list blocks to recover, but don't commit - err := bm.st.ListBlobs(ctx, PackBlobIDPrefix, func(bi blob.Metadata) error { - infos, err := bm.RecoverIndexFromPackBlob(ctx, bi.BlobID, bi.Length, false) - if err != nil { - return err - } - totalRecovered += len(infos) - log.Debugf("recovered %v blocks", len(infos)) - return nil - }) - if err != nil { - t.Errorf("error recovering: %v", err) - } - - if got, want := totalRecovered, 3; got != want { - t.Errorf("invalid # of blocks recovered: %v, want %v", got, want) - } - - // blocks are stil not found - verifyBlockNotFound(ctx, t, bm, block1) - verifyBlockNotFound(ctx, t, bm, block2) - verifyBlockNotFound(ctx, t, bm, block3) - - // pass 2 now pass commit=true to add recovered blocks to index - totalRecovered = 0 - - err = bm.st.ListBlobs(ctx, PackBlobIDPrefix, func(bi blob.Metadata) error { - infos, err := bm.RecoverIndexFromPackBlob(ctx, bi.BlobID, bi.Length, true) - if err != nil { - return err - } - totalRecovered += len(infos) - log.Debugf("recovered %v blocks", len(infos)) - return nil - }) - if err != nil { - t.Errorf("error recovering: %v", err) - } - - if got, want := totalRecovered, 3; got != want { - t.Errorf("invalid # of blocks recovered: %v, want %v", got, want) - } - - verifyBlock(ctx, t, bm, block1, seededRandomData(10, 100)) - verifyBlock(ctx, t, bm, block2, seededRandomData(11, 100)) - verifyBlock(ctx, t, bm, block3, seededRandomData(12, 100)) - if err := bm.Flush(ctx); err != nil { - t.Errorf("flush error: %v", err) - } - verifyBlock(ctx, t, bm, block1, seededRandomData(10, 100)) - verifyBlock(ctx, t, bm, block2, seededRandomData(11, 100)) - verifyBlock(ctx, t, bm, block3, seededRandomData(12, 100)) -} diff --git a/repo/block/block_manager_test.go b/repo/block/block_manager_test.go deleted file mode 100644 index d921a95d8..000000000 --- a/repo/block/block_manager_test.go +++ /dev/null @@ -1,911 +0,0 @@ -package block - -import ( - "bytes" - "context" - "crypto/hmac" - "crypto/sha256" - "encoding/hex" - "errors" - "fmt" - "math/rand" - "reflect" - "strings" - "sync" - "testing" - "time" - - logging "github.com/op/go-logging" - - "github.com/kopia/kopia/internal/blobtesting" - "github.com/kopia/kopia/repo/blob" -) - -const ( - maxPackSize = 2000 -) - -var fakeTime = time.Date(2017, 1, 1, 0, 0, 0, 0, time.UTC) -var hmacSecret = []byte{1, 2, 3} - -func init() { - logging.SetLevel(logging.DEBUG, "") -} - -func TestBlockManagerEmptyFlush(t *testing.T) { - ctx := context.Background() - data := blobtesting.DataMap{} - keyTime := map[blob.ID]time.Time{} - bm := newTestBlockManager(data, keyTime, nil) - bm.Flush(ctx) - if got, want := len(data), 0; got != want { - t.Errorf("unexpected number of blocks: %v, wanted %v", got, want) - } -} - -func TestBlockZeroBytes1(t *testing.T) { - ctx := context.Background() - data := blobtesting.DataMap{} - keyTime := map[blob.ID]time.Time{} - bm := newTestBlockManager(data, keyTime, nil) - blockID := writeBlockAndVerify(ctx, t, bm, []byte{}) - bm.Flush(ctx) - if got, want := len(data), 2; got != want { - t.Errorf("unexpected number of blocks: %v, wanted %v", got, want) - } - dumpBlockManagerData(t, data) - bm = newTestBlockManager(data, keyTime, nil) - verifyBlock(ctx, t, bm, blockID, []byte{}) -} - -func TestBlockZeroBytes2(t *testing.T) { - ctx := context.Background() - data := blobtesting.DataMap{} - keyTime := map[blob.ID]time.Time{} - bm := newTestBlockManager(data, keyTime, nil) - writeBlockAndVerify(ctx, t, bm, seededRandomData(10, 10)) - writeBlockAndVerify(ctx, t, bm, []byte{}) - bm.Flush(ctx) - if got, want := len(data), 2; got != want { - t.Errorf("unexpected number of blocks: %v, wanted %v", got, want) - dumpBlockManagerData(t, data) - } -} - -func TestBlockManagerSmallBlockWrites(t *testing.T) { - ctx := context.Background() - data := blobtesting.DataMap{} - keyTime := map[blob.ID]time.Time{} - bm := newTestBlockManager(data, keyTime, nil) - - for i := 0; i < 100; i++ { - writeBlockAndVerify(ctx, t, bm, seededRandomData(i, 10)) - } - if got, want := len(data), 0; got != want { - t.Errorf("unexpected number of blocks: %v, wanted %v", got, want) - } - bm.Flush(ctx) - if got, want := len(data), 2; got != want { - t.Errorf("unexpected number of blocks: %v, wanted %v", got, want) - } -} - -func TestBlockManagerDedupesPendingBlocks(t *testing.T) { - ctx := context.Background() - data := blobtesting.DataMap{} - keyTime := map[blob.ID]time.Time{} - bm := newTestBlockManager(data, keyTime, nil) - - for i := 0; i < 100; i++ { - writeBlockAndVerify(ctx, t, bm, seededRandomData(0, 999)) - } - if got, want := len(data), 0; got != want { - t.Errorf("unexpected number of blocks: %v, wanted %v", got, want) - } - bm.Flush(ctx) - if got, want := len(data), 2; got != want { - t.Errorf("unexpected number of blocks: %v, wanted %v", got, want) - } -} - -func TestBlockManagerDedupesPendingAndUncommittedBlocks(t *testing.T) { - ctx := context.Background() - data := blobtesting.DataMap{} - keyTime := map[blob.ID]time.Time{} - bm := newTestBlockManager(data, keyTime, nil) - - // no writes here, all data fits in a single pack. - writeBlockAndVerify(ctx, t, bm, seededRandomData(0, 950)) - writeBlockAndVerify(ctx, t, bm, seededRandomData(1, 950)) - writeBlockAndVerify(ctx, t, bm, seededRandomData(2, 10)) - if got, want := len(data), 0; got != want { - t.Errorf("unexpected number of blocks: %v, wanted %v", got, want) - } - - // no writes here - writeBlockAndVerify(ctx, t, bm, seededRandomData(0, 950)) - writeBlockAndVerify(ctx, t, bm, seededRandomData(1, 950)) - writeBlockAndVerify(ctx, t, bm, seededRandomData(2, 10)) - if got, want := len(data), 0; got != want { - t.Errorf("unexpected number of blocks: %v, wanted %v", got, want) - } - bm.Flush(ctx) - - // this flushes the pack block + index blob - if got, want := len(data), 2; got != want { - dumpBlockManagerData(t, data) - t.Errorf("unexpected number of blocks: %v, wanted %v", got, want) - } -} - -func TestBlockManagerEmpty(t *testing.T) { - ctx := context.Background() - data := blobtesting.DataMap{} - keyTime := map[blob.ID]time.Time{} - bm := newTestBlockManager(data, keyTime, nil) - - noSuchBlockID := string(hashValue([]byte("foo"))) - - b, err := bm.GetBlock(ctx, noSuchBlockID) - if err != ErrBlockNotFound { - t.Errorf("unexpected error when getting non-existent block: %v, %v", b, err) - } - - bi, err := bm.BlockInfo(ctx, noSuchBlockID) - if err != ErrBlockNotFound { - t.Errorf("unexpected error when getting non-existent block info: %v, %v", bi, err) - } - - if got, want := len(data), 0; got != want { - t.Errorf("unexpected number of blocks: %v, wanted %v", got, want) - } -} - -func verifyActiveIndexBlobCount(ctx context.Context, t *testing.T, bm *Manager, expected int) { - t.Helper() - - blks, err := bm.IndexBlobs(ctx) - if err != nil { - t.Errorf("error listing active index blobs: %v", err) - return - } - - if got, want := len(blks), expected; got != want { - t.Errorf("unexpected number of active index blobs %v, expected %v (%v)", got, want, blks) - } -} -func TestBlockManagerInternalFlush(t *testing.T) { - ctx := context.Background() - data := blobtesting.DataMap{} - keyTime := map[blob.ID]time.Time{} - bm := newTestBlockManager(data, keyTime, nil) - - for i := 0; i < 100; i++ { - b := make([]byte, 25) - rand.Read(b) - writeBlockAndVerify(ctx, t, bm, b) - } - - // 1 data block written, but no index yet. - if got, want := len(data), 1; got != want { - t.Errorf("unexpected number of blocks: %v, wanted %v", got, want) - } - - // do it again - should be 2 blocks + 1000 bytes pending. - for i := 0; i < 100; i++ { - b := make([]byte, 25) - rand.Read(b) - writeBlockAndVerify(ctx, t, bm, b) - } - - // 2 data blocks written, but no index yet. - if got, want := len(data), 2; got != want { - t.Errorf("unexpected number of blocks: %v, wanted %v", got, want) - } - - bm.Flush(ctx) - - // third block gets written, followed by index. - if got, want := len(data), 4; got != want { - dumpBlockManagerData(t, data) - t.Errorf("unexpected number of blocks: %v, wanted %v", got, want) - } -} - -func TestBlockManagerWriteMultiple(t *testing.T) { - ctx := context.Background() - data := blobtesting.DataMap{} - keyTime := map[blob.ID]time.Time{} - timeFunc := fakeTimeNowWithAutoAdvance(fakeTime, 1*time.Second) - bm := newTestBlockManager(data, keyTime, timeFunc) - - var blockIDs []string - - for i := 0; i < 5000; i++ { - //t.Logf("i=%v", i) - b := seededRandomData(i, i%113) - blkID, err := bm.WriteBlock(ctx, b, "") - if err != nil { - t.Errorf("err: %v", err) - } - - blockIDs = append(blockIDs, blkID) - - if i%17 == 0 { - //t.Logf("flushing %v", i) - if err := bm.Flush(ctx); err != nil { - t.Fatalf("error flushing: %v", err) - } - //dumpBlockManagerData(t, data) - } - - if i%41 == 0 { - //t.Logf("opening new manager: %v", i) - if err := bm.Flush(ctx); err != nil { - t.Fatalf("error flushing: %v", err) - } - //t.Logf("data block count: %v", len(data)) - //dumpBlockManagerData(t, data) - bm = newTestBlockManager(data, keyTime, timeFunc) - } - - pos := rand.Intn(len(blockIDs)) - if _, err := bm.GetBlock(ctx, blockIDs[pos]); err != nil { - dumpBlockManagerData(t, data) - t.Fatalf("can't read block %q: %v", blockIDs[pos], err) - continue - } - } -} - -// This is regression test for a bug where we would corrupt data when encryption -// was done in place and clobbered pending data in memory. -func TestBlockManagerFailedToWritePack(t *testing.T) { - ctx := context.Background() - data := blobtesting.DataMap{} - keyTime := map[blob.ID]time.Time{} - st := blobtesting.NewMapStorage(data, keyTime, nil) - faulty := &blobtesting.FaultyStorage{ - Base: st, - } - st = faulty - - bm, err := newManagerWithOptions(context.Background(), st, FormattingOptions{ - Version: 1, - Hash: "HMAC-SHA256-128", - Encryption: "AES-256-CTR", - MaxPackSize: maxPackSize, - HMACSecret: []byte("foo"), - MasterKey: []byte("0123456789abcdef0123456789abcdef"), - }, CachingOptions{}, fakeTimeNowFrozen(fakeTime), nil) - if err != nil { - t.Fatalf("can't create bm: %v", err) - } - logging.SetLevel(logging.DEBUG, "faulty-storage") - - faulty.Faults = map[string][]*blobtesting.Fault{ - "PutBlock": { - {Err: errors.New("booboo")}, - }, - } - - b1, err := bm.WriteBlock(ctx, seededRandomData(1, 10), "") - if err != nil { - t.Fatalf("can't create block: %v", err) - } - - if err := bm.Flush(ctx); err != nil { - t.Logf("expected flush error: %v", err) - } - - verifyBlock(ctx, t, bm, b1, seededRandomData(1, 10)) -} - -func TestBlockManagerConcurrency(t *testing.T) { - ctx := context.Background() - data := blobtesting.DataMap{} - keyTime := map[blob.ID]time.Time{} - bm := newTestBlockManager(data, keyTime, nil) - preexistingBlock := writeBlockAndVerify(ctx, t, bm, seededRandomData(10, 100)) - bm.Flush(ctx) - - dumpBlockManagerData(t, data) - bm1 := newTestBlockManager(data, keyTime, nil) - bm2 := newTestBlockManager(data, keyTime, nil) - bm3 := newTestBlockManager(data, keyTime, fakeTimeNowWithAutoAdvance(fakeTime.Add(1), 1*time.Second)) - - // all bm* can see pre-existing block - verifyBlock(ctx, t, bm1, preexistingBlock, seededRandomData(10, 100)) - verifyBlock(ctx, t, bm2, preexistingBlock, seededRandomData(10, 100)) - verifyBlock(ctx, t, bm3, preexistingBlock, seededRandomData(10, 100)) - - // write the same block in all managers. - sharedBlock := writeBlockAndVerify(ctx, t, bm1, seededRandomData(20, 100)) - writeBlockAndVerify(ctx, t, bm2, seededRandomData(20, 100)) - writeBlockAndVerify(ctx, t, bm3, seededRandomData(20, 100)) - - // write unique block per manager. - bm1block := writeBlockAndVerify(ctx, t, bm1, seededRandomData(31, 100)) - bm2block := writeBlockAndVerify(ctx, t, bm2, seededRandomData(32, 100)) - bm3block := writeBlockAndVerify(ctx, t, bm3, seededRandomData(33, 100)) - - // make sure they can't see each other's unflushed blocks. - verifyBlockNotFound(ctx, t, bm1, bm2block) - verifyBlockNotFound(ctx, t, bm1, bm3block) - verifyBlockNotFound(ctx, t, bm2, bm1block) - verifyBlockNotFound(ctx, t, bm2, bm3block) - verifyBlockNotFound(ctx, t, bm3, bm1block) - verifyBlockNotFound(ctx, t, bm3, bm2block) - - // now flush all writers, they still can't see each others' data. - bm1.Flush(ctx) - bm2.Flush(ctx) - bm3.Flush(ctx) - verifyBlockNotFound(ctx, t, bm1, bm2block) - verifyBlockNotFound(ctx, t, bm1, bm3block) - verifyBlockNotFound(ctx, t, bm2, bm1block) - verifyBlockNotFound(ctx, t, bm2, bm3block) - verifyBlockNotFound(ctx, t, bm3, bm1block) - verifyBlockNotFound(ctx, t, bm3, bm2block) - - // new block manager at this point can see all data. - bm4 := newTestBlockManager(data, keyTime, nil) - verifyBlock(ctx, t, bm4, preexistingBlock, seededRandomData(10, 100)) - verifyBlock(ctx, t, bm4, sharedBlock, seededRandomData(20, 100)) - verifyBlock(ctx, t, bm4, bm1block, seededRandomData(31, 100)) - verifyBlock(ctx, t, bm4, bm2block, seededRandomData(32, 100)) - verifyBlock(ctx, t, bm4, bm3block, seededRandomData(33, 100)) - - if got, want := getIndexCount(data), 4; got != want { - t.Errorf("unexpected index count before compaction: %v, wanted %v", got, want) - } - - if err := bm4.CompactIndexes(ctx, CompactOptions{ - MinSmallBlocks: 1, - MaxSmallBlocks: 1, - }); err != nil { - t.Errorf("compaction error: %v", err) - } - if got, want := getIndexCount(data), 1; got != want { - t.Errorf("unexpected index count after compaction: %v, wanted %v", got, want) - } - - // new block manager at this point can see all data. - bm5 := newTestBlockManager(data, keyTime, nil) - verifyBlock(ctx, t, bm5, preexistingBlock, seededRandomData(10, 100)) - verifyBlock(ctx, t, bm5, sharedBlock, seededRandomData(20, 100)) - verifyBlock(ctx, t, bm5, bm1block, seededRandomData(31, 100)) - verifyBlock(ctx, t, bm5, bm2block, seededRandomData(32, 100)) - verifyBlock(ctx, t, bm5, bm3block, seededRandomData(33, 100)) - if err := bm5.CompactIndexes(ctx, CompactOptions{ - MinSmallBlocks: 1, - MaxSmallBlocks: 1, - }); err != nil { - t.Errorf("compaction error: %v", err) - } -} - -func TestDeleteBlock(t *testing.T) { - ctx := context.Background() - data := blobtesting.DataMap{} - keyTime := map[blob.ID]time.Time{} - bm := newTestBlockManager(data, keyTime, nil) - block1 := writeBlockAndVerify(ctx, t, bm, seededRandomData(10, 100)) - bm.Flush(ctx) - block2 := writeBlockAndVerify(ctx, t, bm, seededRandomData(11, 100)) - if err := bm.DeleteBlock(block1); err != nil { - t.Errorf("unable to delete block: %v", block1) - } - if err := bm.DeleteBlock(block2); err != nil { - t.Errorf("unable to delete block: %v", block1) - } - verifyBlockNotFound(ctx, t, bm, block1) - verifyBlockNotFound(ctx, t, bm, block2) - bm.Flush(ctx) - log.Debugf("-----------") - bm = newTestBlockManager(data, keyTime, nil) - //dumpBlockManagerData(t, data) - verifyBlockNotFound(ctx, t, bm, block1) - verifyBlockNotFound(ctx, t, bm, block2) -} - -func TestRewriteNonDeleted(t *testing.T) { - const stepBehaviors = 3 - - // perform a sequence WriteBlock() RewriteBlock() GetBlock() - // where actionX can be (0=flush and reopen, 1=flush, 2=nothing) - for action1 := 0; action1 < stepBehaviors; action1++ { - for action2 := 0; action2 < stepBehaviors; action2++ { - t.Run(fmt.Sprintf("case-%v-%v", action1, action2), func(t *testing.T) { - ctx := context.Background() - data := blobtesting.DataMap{} - keyTime := map[blob.ID]time.Time{} - fakeNow := fakeTimeNowWithAutoAdvance(fakeTime, 1*time.Second) - bm := newTestBlockManager(data, keyTime, fakeNow) - - applyStep := func(action int) { - switch action { - case 0: - t.Logf("flushing and reopening") - bm.Flush(ctx) - bm = newTestBlockManager(data, keyTime, fakeNow) - case 1: - t.Logf("flushing") - bm.Flush(ctx) - case 2: - t.Logf("doing nothing") - } - } - - block1 := writeBlockAndVerify(ctx, t, bm, seededRandomData(10, 100)) - applyStep(action1) - assertNoError(t, bm.RewriteBlock(ctx, block1)) - applyStep(action2) - verifyBlock(ctx, t, bm, block1, seededRandomData(10, 100)) - dumpBlockManagerData(t, data) - }) - } - } -} - -func TestDisableFlush(t *testing.T) { - ctx := context.Background() - data := blobtesting.DataMap{} - keyTime := map[blob.ID]time.Time{} - bm := newTestBlockManager(data, keyTime, nil) - bm.DisableIndexFlush() - bm.DisableIndexFlush() - for i := 0; i < 500; i++ { - writeBlockAndVerify(ctx, t, bm, seededRandomData(i, 100)) - } - bm.Flush(ctx) // flush will not have effect - bm.EnableIndexFlush() - bm.Flush(ctx) // flush will not have effect - bm.EnableIndexFlush() - - verifyActiveIndexBlobCount(ctx, t, bm, 0) - bm.EnableIndexFlush() - verifyActiveIndexBlobCount(ctx, t, bm, 0) - bm.Flush(ctx) // flush will happen now - verifyActiveIndexBlobCount(ctx, t, bm, 1) -} - -func TestRewriteDeleted(t *testing.T) { - const stepBehaviors = 3 - - // perform a sequence WriteBlock() Delete() RewriteBlock() GetBlock() - // where actionX can be (0=flush and reopen, 1=flush, 2=nothing) - for action1 := 0; action1 < stepBehaviors; action1++ { - for action2 := 0; action2 < stepBehaviors; action2++ { - for action3 := 0; action3 < stepBehaviors; action3++ { - t.Run(fmt.Sprintf("case-%v-%v-%v", action1, action2, action3), func(t *testing.T) { - ctx := context.Background() - data := blobtesting.DataMap{} - keyTime := map[blob.ID]time.Time{} - fakeNow := fakeTimeNowWithAutoAdvance(fakeTime, 1*time.Second) - bm := newTestBlockManager(data, keyTime, fakeNow) - - applyStep := func(action int) { - switch action { - case 0: - t.Logf("flushing and reopening") - bm.Flush(ctx) - bm = newTestBlockManager(data, keyTime, fakeNow) - case 1: - t.Logf("flushing") - bm.Flush(ctx) - case 2: - t.Logf("doing nothing") - } - } - - block1 := writeBlockAndVerify(ctx, t, bm, seededRandomData(10, 100)) - applyStep(action1) - assertNoError(t, bm.DeleteBlock(block1)) - applyStep(action2) - if got, want := bm.RewriteBlock(ctx, block1), ErrBlockNotFound; got != want && got != nil { - t.Errorf("unexpected error %v, wanted %v", got, want) - } - applyStep(action3) - verifyBlockNotFound(ctx, t, bm, block1) - dumpBlockManagerData(t, data) - }) - } - } - } -} - -func TestDeleteAndRecreate(t *testing.T) { - ctx := context.Background() - // simulate race between delete/recreate and delete - // delete happens at t0+10, recreate at t0+20 and second delete time is parameterized. - // depending on it, the second delete results will be visible. - cases := []struct { - desc string - deletionTime time.Time - isVisible bool - }{ - {"deleted before delete and-recreate", fakeTime.Add(5 * time.Second), true}, - //{"deleted after delete and recreate", fakeTime.Add(25 * time.Second), false}, - } - - for _, tc := range cases { - t.Run(tc.desc, func(t *testing.T) { - // write a block - data := blobtesting.DataMap{} - keyTime := map[blob.ID]time.Time{} - bm := newTestBlockManager(data, keyTime, fakeTimeNowFrozen(fakeTime)) - block1 := writeBlockAndVerify(ctx, t, bm, seededRandomData(10, 100)) - bm.Flush(ctx) - - // delete but at given timestamp but don't commit yet. - bm0 := newTestBlockManager(data, keyTime, fakeTimeNowWithAutoAdvance(tc.deletionTime, 1*time.Second)) - assertNoError(t, bm0.DeleteBlock(block1)) - - // delete it at t0+10 - bm1 := newTestBlockManager(data, keyTime, fakeTimeNowWithAutoAdvance(fakeTime.Add(10*time.Second), 1*time.Second)) - verifyBlock(ctx, t, bm1, block1, seededRandomData(10, 100)) - assertNoError(t, bm1.DeleteBlock(block1)) - bm1.Flush(ctx) - - // recreate at t0+20 - bm2 := newTestBlockManager(data, keyTime, fakeTimeNowWithAutoAdvance(fakeTime.Add(20*time.Second), 1*time.Second)) - block2 := writeBlockAndVerify(ctx, t, bm2, seededRandomData(10, 100)) - bm2.Flush(ctx) - - // commit deletion from bm0 (t0+5) - bm0.Flush(ctx) - - //dumpBlockManagerData(t, data) - - if block1 != block2 { - t.Errorf("got invalid block %v, expected %v", block2, block1) - } - - bm3 := newTestBlockManager(data, keyTime, nil) - dumpBlockManagerData(t, data) - if tc.isVisible { - verifyBlock(ctx, t, bm3, block1, seededRandomData(10, 100)) - } else { - verifyBlockNotFound(ctx, t, bm3, block1) - } - }) - } -} - -func TestFindUnreferencedBlobs(t *testing.T) { - ctx := context.Background() - data := blobtesting.DataMap{} - keyTime := map[blob.ID]time.Time{} - bm := newTestBlockManager(data, keyTime, nil) - verifyUnreferencedStorageFilesCount(ctx, t, bm, 0) - blockID := writeBlockAndVerify(ctx, t, bm, seededRandomData(10, 100)) - if err := bm.Flush(ctx); err != nil { - t.Errorf("flush error: %v", err) - } - verifyUnreferencedStorageFilesCount(ctx, t, bm, 0) - if err := bm.DeleteBlock(blockID); err != nil { - t.Errorf("error deleting block: %v", blockID) - } - if err := bm.Flush(ctx); err != nil { - t.Errorf("flush error: %v", err) - } - - // block still present in first pack - verifyUnreferencedStorageFilesCount(ctx, t, bm, 0) - - assertNoError(t, bm.RewriteBlock(ctx, blockID)) - if err := bm.Flush(ctx); err != nil { - t.Errorf("flush error: %v", err) - } - verifyUnreferencedStorageFilesCount(ctx, t, bm, 1) - assertNoError(t, bm.RewriteBlock(ctx, blockID)) - if err := bm.Flush(ctx); err != nil { - t.Errorf("flush error: %v", err) - } - verifyUnreferencedStorageFilesCount(ctx, t, bm, 2) -} - -func TestFindUnreferencedBlobs2(t *testing.T) { - ctx := context.Background() - data := blobtesting.DataMap{} - keyTime := map[blob.ID]time.Time{} - bm := newTestBlockManager(data, keyTime, nil) - verifyUnreferencedStorageFilesCount(ctx, t, bm, 0) - blockID := writeBlockAndVerify(ctx, t, bm, seededRandomData(10, 100)) - writeBlockAndVerify(ctx, t, bm, seededRandomData(11, 100)) - dumpBlocks(t, bm, "after writing") - if err := bm.Flush(ctx); err != nil { - t.Errorf("flush error: %v", err) - } - dumpBlocks(t, bm, "after flush") - verifyUnreferencedStorageFilesCount(ctx, t, bm, 0) - if err := bm.DeleteBlock(blockID); err != nil { - t.Errorf("error deleting block: %v", blockID) - } - dumpBlocks(t, bm, "after delete") - if err := bm.Flush(ctx); err != nil { - t.Errorf("flush error: %v", err) - } - dumpBlocks(t, bm, "after flush") - // block present in first pack, original pack is still referenced - verifyUnreferencedStorageFilesCount(ctx, t, bm, 0) -} - -func dumpBlocks(t *testing.T, bm *Manager, caption string) { - t.Helper() - infos, err := bm.ListBlockInfos("", true) - if err != nil { - t.Errorf("error listing blocks: %v", err) - return - } - - log.Infof("**** dumping %v blocks %v", len(infos), caption) - for i, bi := range infos { - log.Debugf(" bi[%v]=%#v", i, bi) - } - log.Infof("finished dumping %v blocks", len(infos)) -} - -func verifyUnreferencedStorageFilesCount(ctx context.Context, t *testing.T, bm *Manager, want int) { - t.Helper() - unref, err := bm.FindUnreferencedBlobs(ctx) - if err != nil { - t.Errorf("error in FindUnreferencedBlobs: %v", err) - } - - log.Infof("got %v expecting %v", unref, want) - if got := len(unref); got != want { - t.Errorf("invalid number of unreferenced blocks: %v, wanted %v", got, want) - } -} - -func TestBlockWriteAliasing(t *testing.T) { - ctx := context.Background() - data := blobtesting.DataMap{} - keyTime := map[blob.ID]time.Time{} - bm := newTestBlockManager(data, keyTime, fakeTimeNowFrozen(fakeTime)) - - blockData := []byte{100, 0, 0} - id1 := writeBlockAndVerify(ctx, t, bm, blockData) - blockData[0] = 101 - id2 := writeBlockAndVerify(ctx, t, bm, blockData) - bm.Flush(ctx) - blockData[0] = 102 - id3 := writeBlockAndVerify(ctx, t, bm, blockData) - blockData[0] = 103 - id4 := writeBlockAndVerify(ctx, t, bm, blockData) - verifyBlock(ctx, t, bm, id1, []byte{100, 0, 0}) - verifyBlock(ctx, t, bm, id2, []byte{101, 0, 0}) - verifyBlock(ctx, t, bm, id3, []byte{102, 0, 0}) - verifyBlock(ctx, t, bm, id4, []byte{103, 0, 0}) -} - -func TestBlockReadAliasing(t *testing.T) { - ctx := context.Background() - data := blobtesting.DataMap{} - keyTime := map[blob.ID]time.Time{} - bm := newTestBlockManager(data, keyTime, fakeTimeNowFrozen(fakeTime)) - - blockData := []byte{100, 0, 0} - id1 := writeBlockAndVerify(ctx, t, bm, blockData) - blockData2, err := bm.GetBlock(ctx, id1) - if err != nil { - t.Fatalf("can't get block data: %v", err) - } - - blockData2[0]++ - verifyBlock(ctx, t, bm, id1, blockData) - bm.Flush(ctx) - verifyBlock(ctx, t, bm, id1, blockData) -} - -func TestVersionCompatibility(t *testing.T) { - for writeVer := minSupportedReadVersion; writeVer <= currentWriteVersion; writeVer++ { - t.Run(fmt.Sprintf("version-%v", writeVer), func(t *testing.T) { - verifyVersionCompat(t, writeVer) - }) - } -} - -func verifyVersionCompat(t *testing.T, writeVersion int) { - ctx := context.Background() - - // create block manager that writes 'writeVersion' and reads all versions >= minSupportedReadVersion - data := blobtesting.DataMap{} - keyTime := map[blob.ID]time.Time{} - mgr := newTestBlockManager(data, keyTime, nil) - mgr.writeFormatVersion = int32(writeVersion) - - dataSet := map[string][]byte{} - - for i := 0; i < 3000000; i = (i + 1) * 2 { - data := make([]byte, i) - rand.Read(data) - - cid, err := mgr.WriteBlock(ctx, data, "") - if err != nil { - t.Fatalf("unable to write %v bytes: %v", len(data), err) - } - dataSet[cid] = data - } - verifyBlockManagerDataSet(ctx, t, mgr, dataSet) - - // delete random 3 items (map iteration order is random) - cnt := 0 - for blobID := range dataSet { - t.Logf("deleting %v", blobID) - assertNoError(t, mgr.DeleteBlock(blobID)) - delete(dataSet, blobID) - cnt++ - if cnt >= 3 { - break - } - } - if err := mgr.Flush(ctx); err != nil { - t.Fatalf("failed to flush: %v", err) - } - - // create new manager that reads and writes using new version. - mgr = newTestBlockManager(data, keyTime, nil) - - // make sure we can read everything - verifyBlockManagerDataSet(ctx, t, mgr, dataSet) - - if err := mgr.CompactIndexes(ctx, CompactOptions{ - MinSmallBlocks: 1, - MaxSmallBlocks: 1, - }); err != nil { - t.Fatalf("unable to compact indexes: %v", err) - } - if err := mgr.Flush(ctx); err != nil { - t.Fatalf("failed to flush: %v", err) - } - verifyBlockManagerDataSet(ctx, t, mgr, dataSet) - - // now open one more manager - mgr = newTestBlockManager(data, keyTime, nil) - verifyBlockManagerDataSet(ctx, t, mgr, dataSet) -} - -func verifyBlockManagerDataSet(ctx context.Context, t *testing.T, mgr *Manager, dataSet map[string][]byte) { - for blockID, originalPayload := range dataSet { - v, err := mgr.GetBlock(ctx, blockID) - if err != nil { - t.Errorf("unable to read block %q: %v", blockID, err) - continue - } - - if !reflect.DeepEqual(v, originalPayload) { - t.Errorf("payload for %q does not match original: %v", v, originalPayload) - } - } -} - -func newTestBlockManager(data blobtesting.DataMap, keyTime map[blob.ID]time.Time, timeFunc func() time.Time) *Manager { - //st = logging.NewWrapper(st) - if timeFunc == nil { - timeFunc = fakeTimeNowWithAutoAdvance(fakeTime, 1*time.Second) - } - st := blobtesting.NewMapStorage(data, keyTime, timeFunc) - bm, err := newManagerWithOptions(context.Background(), st, FormattingOptions{ - Hash: "HMAC-SHA256", - Encryption: "NONE", - HMACSecret: hmacSecret, - MaxPackSize: maxPackSize, - Version: 1, - }, CachingOptions{}, timeFunc, nil) - if err != nil { - panic("can't create block manager: " + err.Error()) - } - bm.checkInvariantsOnUnlock = true - return bm -} - -func getIndexCount(d blobtesting.DataMap) int { - var cnt int - - for blobID := range d { - if strings.HasPrefix(string(blobID), newIndexBlobPrefix) { - cnt++ - } - } - - return cnt -} - -func fakeTimeNowFrozen(t time.Time) func() time.Time { - return fakeTimeNowWithAutoAdvance(t, 0) -} - -func fakeTimeNowWithAutoAdvance(t time.Time, dt time.Duration) func() time.Time { - var mu sync.Mutex - return func() time.Time { - mu.Lock() - defer mu.Unlock() - ret := t - t = t.Add(dt) - return ret - } -} - -func verifyBlockNotFound(ctx context.Context, t *testing.T, bm *Manager, blockID string) { - t.Helper() - - b, err := bm.GetBlock(ctx, blockID) - if err != ErrBlockNotFound { - t.Errorf("unexpected response from GetBlock(%q), got %v,%v, expected %v", blockID, b, err, ErrBlockNotFound) - } -} - -func verifyBlock(ctx context.Context, t *testing.T, bm *Manager, blockID string, b []byte) { - t.Helper() - - b2, err := bm.GetBlock(ctx, blockID) - if err != nil { - t.Errorf("unable to read block %q: %v", blockID, err) - return - } - - if got, want := b2, b; !reflect.DeepEqual(got, want) { - t.Errorf("block %q data mismatch: got %x (nil:%v), wanted %x (nil:%v)", blockID, got, got == nil, want, want == nil) - } - - bi, err := bm.BlockInfo(ctx, blockID) - if err != nil { - t.Errorf("error getting block info %q: %v", blockID, err) - } - - if got, want := bi.Length, uint32(len(b)); got != want { - t.Errorf("invalid block size for %q: %v, wanted %v", blockID, got, want) - } - -} -func writeBlockAndVerify(ctx context.Context, t *testing.T, bm *Manager, b []byte) string { - t.Helper() - - blockID, err := bm.WriteBlock(ctx, b, "") - if err != nil { - t.Errorf("err: %v", err) - } - - if got, want := blockID, string(hashValue(b)); got != want { - t.Errorf("invalid block ID for %x, got %v, want %v", b, got, want) - } - - verifyBlock(ctx, t, bm, blockID, b) - - return blockID -} - -func seededRandomData(seed int, length int) []byte { - b := make([]byte, length) - rnd := rand.New(rand.NewSource(int64(seed))) - rnd.Read(b) - return b -} - -func hashValue(b []byte) string { - h := hmac.New(sha256.New, hmacSecret) - h.Write(b) //nolint:errcheck - return hex.EncodeToString(h.Sum(nil)) -} - -func dumpBlockManagerData(t *testing.T, data blobtesting.DataMap) { - t.Helper() - for k, v := range data { - if k[0] == 'n' { - ndx, err := openPackIndex(bytes.NewReader(v)) - if err == nil { - t.Logf("index %v (%v bytes)", k, len(v)) - assertNoError(t, ndx.Iterate("", func(i Info) error { - t.Logf(" %+v\n", i) - return nil - })) - - } - } else { - t.Logf("data %v (%v bytes)\n", k, len(v)) - } - } -} diff --git a/repo/block/committed_block_index_mem_cache.go b/repo/block/committed_block_index_mem_cache.go deleted file mode 100644 index 6a8c1a92a..000000000 --- a/repo/block/committed_block_index_mem_cache.go +++ /dev/null @@ -1,51 +0,0 @@ -package block - -import ( - "bytes" - "sync" - - "github.com/pkg/errors" - - "github.com/kopia/kopia/repo/blob" -) - -type memoryCommittedBlockIndexCache struct { - mu sync.Mutex - blocks map[blob.ID]packIndex -} - -func (m *memoryCommittedBlockIndexCache) hasIndexBlobID(indexBlobID blob.ID) (bool, error) { - m.mu.Lock() - defer m.mu.Unlock() - - return m.blocks[indexBlobID] != nil, nil -} - -func (m *memoryCommittedBlockIndexCache) addBlockToCache(indexBlobID blob.ID, data []byte) error { - m.mu.Lock() - defer m.mu.Unlock() - - ndx, err := openPackIndex(bytes.NewReader(data)) - if err != nil { - return err - } - - m.blocks[indexBlobID] = ndx - return nil -} - -func (m *memoryCommittedBlockIndexCache) openIndex(indexBlobID blob.ID) (packIndex, error) { - m.mu.Lock() - defer m.mu.Unlock() - - v := m.blocks[indexBlobID] - if v == nil { - return nil, errors.Errorf("block not found in cache: %v", indexBlobID) - } - - return v, nil -} - -func (m *memoryCommittedBlockIndexCache) expireUnused(used []blob.ID) error { - return nil -} diff --git a/repo/block/context.go b/repo/block/context.go deleted file mode 100644 index b7f22abd2..000000000 --- a/repo/block/context.go +++ /dev/null @@ -1,34 +0,0 @@ -package block - -import "context" - -type contextKey string - -var useBlockCacheContextKey contextKey = "use-block-cache" -var useListCacheContextKey contextKey = "use-list-cache" - -// UsingBlockCache returns a derived context that causes block manager to use cache. -func UsingBlockCache(ctx context.Context, enabled bool) context.Context { - return context.WithValue(ctx, useBlockCacheContextKey, enabled) -} - -// UsingListCache returns a derived context that causes block manager to use cache. -func UsingListCache(ctx context.Context, enabled bool) context.Context { - return context.WithValue(ctx, useListCacheContextKey, enabled) -} - -func shouldUseBlockCache(ctx context.Context) bool { - if enabled, ok := ctx.Value(useBlockCacheContextKey).(bool); ok { - return enabled - } - - return true -} - -func shouldUseListCache(ctx context.Context) bool { - if enabled, ok := ctx.Value(useListCacheContextKey).(bool); ok { - return enabled - } - - return true -} diff --git a/repo/connect.go b/repo/connect.go index 94660fd5d..5ebb7c999 100644 --- a/repo/connect.go +++ b/repo/connect.go @@ -12,22 +12,22 @@ "github.com/pkg/errors" "github.com/kopia/kopia/repo/blob" - "github.com/kopia/kopia/repo/block" + "github.com/kopia/kopia/repo/content" ) // ConnectOptions specifies options when persisting configuration to connect to a repository. type ConnectOptions struct { - block.CachingOptions + content.CachingOptions } // Connect connects to the repository in the specified storage and persists the configuration and credentials in the file provided. func Connect(ctx context.Context, configFile string, st blob.Storage, password string, opt ConnectOptions) error { formatBytes, err := st.GetBlob(ctx, FormatBlobID, 0, -1) if err != nil { - return errors.Wrap(err, "unable to read format block") + return errors.Wrap(err, "unable to read format blob") } - f, err := parseFormatBlock(formatBytes) + f, err := parseFormatBlob(formatBytes) if err != nil { return err } @@ -61,9 +61,9 @@ func Connect(ctx context.Context, configFile string, st blob.Storage, password s return r.Close(ctx) } -func setupCaching(configPath string, lc *LocalConfig, opt block.CachingOptions, uniqueID []byte) error { +func setupCaching(configPath string, lc *LocalConfig, opt content.CachingOptions, uniqueID []byte) error { if opt.MaxCacheSizeBytes == 0 { - lc.Caching = block.CachingOptions{} + lc.Caching = content.CachingOptions{} return nil } diff --git a/repo/block/block_manager_compaction.go b/repo/content/block_manager_compaction.go similarity index 58% rename from repo/block/block_manager_compaction.go rename to repo/content/block_manager_compaction.go index dd3f921e5..b67680b4b 100644 --- a/repo/block/block_manager_compaction.go +++ b/repo/content/block_manager_compaction.go @@ -1,4 +1,4 @@ -package block +package content import ( "bytes" @@ -9,23 +9,23 @@ ) var autoCompactionOptions = CompactOptions{ - MinSmallBlocks: 4 * parallelFetches, - MaxSmallBlocks: 64, + MinSmallBlobs: 4 * parallelFetches, + MaxSmallBlobs: 64, } // CompactOptions provides options for compaction type CompactOptions struct { - MinSmallBlocks int - MaxSmallBlocks int - AllBlocks bool + MinSmallBlobs int + MaxSmallBlobs int + AllIndexes bool SkipDeletedOlderThan time.Duration } -// CompactIndexes performs compaction of index blobs ensuring that # of small blocks is between minSmallBlockCount and maxSmallBlockCount +// CompactIndexes performs compaction of index blobs ensuring that # of small contents is between minSmallContentCount and maxSmallContentCount func (bm *Manager) CompactIndexes(ctx context.Context, opt CompactOptions) error { log.Debugf("CompactIndexes(%+v)", opt) - if opt.MaxSmallBlocks < opt.MinSmallBlocks { - return errors.Errorf("invalid block counts") + if opt.MaxSmallBlobs < opt.MinSmallBlobs { + return errors.Errorf("invalid content counts") } indexBlobs, _, err := bm.loadPackIndexesUnlocked(ctx) @@ -33,61 +33,61 @@ func (bm *Manager) CompactIndexes(ctx context.Context, opt CompactOptions) error return errors.Wrap(err, "error loading indexes") } - blocksToCompact := bm.getBlocksToCompact(indexBlobs, opt) + contentsToCompact := bm.getContentsToCompact(indexBlobs, opt) - if err := bm.compactAndDeleteIndexBlobs(ctx, blocksToCompact, opt); err != nil { + if err := bm.compactAndDeleteIndexBlobs(ctx, contentsToCompact, opt); err != nil { log.Warningf("error performing quick compaction: %v", err) } return nil } -func (bm *Manager) getBlocksToCompact(indexBlobs []IndexBlobInfo, opt CompactOptions) []IndexBlobInfo { - var nonCompactedBlocks []IndexBlobInfo - var totalSizeNonCompactedBlocks int64 +func (bm *Manager) getContentsToCompact(indexBlobs []IndexBlobInfo, opt CompactOptions) []IndexBlobInfo { + var nonCompactedContents []IndexBlobInfo + var totalSizeNonCompactedContents int64 - var verySmallBlocks []IndexBlobInfo - var totalSizeVerySmallBlocks int64 + var verySmallContents []IndexBlobInfo + var totalSizeVerySmallContents int64 - var mediumSizedBlocks []IndexBlobInfo - var totalSizeMediumSizedBlocks int64 + var mediumSizedContents []IndexBlobInfo + var totalSizeMediumSizedContents int64 for _, b := range indexBlobs { - if b.Length > int64(bm.maxPackSize) && !opt.AllBlocks { + if b.Length > int64(bm.maxPackSize) && !opt.AllIndexes { continue } - nonCompactedBlocks = append(nonCompactedBlocks, b) + nonCompactedContents = append(nonCompactedContents, b) if b.Length < int64(bm.maxPackSize/20) { - verySmallBlocks = append(verySmallBlocks, b) - totalSizeVerySmallBlocks += b.Length + verySmallContents = append(verySmallContents, b) + totalSizeVerySmallContents += b.Length } else { - mediumSizedBlocks = append(mediumSizedBlocks, b) - totalSizeMediumSizedBlocks += b.Length + mediumSizedContents = append(mediumSizedContents, b) + totalSizeMediumSizedContents += b.Length } - totalSizeNonCompactedBlocks += b.Length + totalSizeNonCompactedContents += b.Length } - if len(nonCompactedBlocks) < opt.MinSmallBlocks { + if len(nonCompactedContents) < opt.MinSmallBlobs { // current count is below min allowed - nothing to do - formatLog.Debugf("no small blocks to compact") + formatLog.Debugf("no small contents to compact") return nil } - if len(verySmallBlocks) > len(nonCompactedBlocks)/2 && len(mediumSizedBlocks)+1 < opt.MinSmallBlocks { - formatLog.Debugf("compacting %v very small blocks", len(verySmallBlocks)) - return verySmallBlocks + if len(verySmallContents) > len(nonCompactedContents)/2 && len(mediumSizedContents)+1 < opt.MinSmallBlobs { + formatLog.Debugf("compacting %v very small contents", len(verySmallContents)) + return verySmallContents } - formatLog.Debugf("compacting all %v non-compacted blocks", len(nonCompactedBlocks)) - return nonCompactedBlocks + formatLog.Debugf("compacting all %v non-compacted contents", len(nonCompactedContents)) + return nonCompactedContents } func (bm *Manager) compactAndDeleteIndexBlobs(ctx context.Context, indexBlobs []IndexBlobInfo, opt CompactOptions) error { if len(indexBlobs) <= 1 { return nil } - formatLog.Debugf("compacting %v blocks", len(indexBlobs)) + formatLog.Debugf("compacting %v contents", len(indexBlobs)) t0 := time.Now() bld := make(packIndexBuilder) @@ -136,7 +136,7 @@ func (bm *Manager) addIndexBlobsToBuilder(ctx context.Context, bld packIndexBuil _ = index.Iterate("", func(i Info) error { if i.Deleted && opt.SkipDeletedOlderThan > 0 && time.Since(i.Timestamp()) > opt.SkipDeletedOlderThan { - log.Debugf("skipping block %v deleted at %v", i.BlockID, i.Timestamp()) + log.Debugf("skipping content %v deleted at %v", i.ID, i.Timestamp()) return nil } bld.Add(i) diff --git a/repo/block/builder.go b/repo/content/builder.go similarity index 79% rename from repo/block/builder.go rename to repo/content/builder.go index a83cea77c..50d1efbc8 100644 --- a/repo/block/builder.go +++ b/repo/content/builder.go @@ -1,4 +1,4 @@ -package block +package content import ( "bufio" @@ -11,29 +11,29 @@ "github.com/kopia/kopia/repo/blob" ) -// packIndexBuilder prepares and writes block index for writing. -type packIndexBuilder map[string]*Info +// packIndexBuilder prepares and writes content index. +type packIndexBuilder map[ID]*Info // Add adds a new entry to the builder or conditionally replaces it if the timestamp is greater. func (b packIndexBuilder) Add(i Info) { - old, ok := b[i.BlockID] + old, ok := b[i.ID] if !ok || i.TimestampSeconds >= old.TimestampSeconds { - b[i.BlockID] = &i + b[i.ID] = &i } } -func (b packIndexBuilder) sortedBlocks() []*Info { - var allBlocks []*Info +func (b packIndexBuilder) sortedContents() []*Info { + var allContents []*Info for _, v := range b { - allBlocks = append(allBlocks, v) + allContents = append(allContents, v) } - sort.Slice(allBlocks, func(i, j int) bool { - return allBlocks[i].BlockID < allBlocks[j].BlockID + sort.Slice(allContents, func(i, j int) bool { + return allContents[i].ID < allContents[j].ID }) - return allBlocks + return allContents } type indexLayout struct { @@ -46,18 +46,18 @@ type indexLayout struct { // Build writes the pack index to the provided output. func (b packIndexBuilder) Build(output io.Writer) error { - allBlocks := b.sortedBlocks() + allContents := b.sortedContents() layout := &indexLayout{ packBlobIDOffsets: map[blob.ID]uint32{}, keyLength: -1, entryLength: 20, - entryCount: len(allBlocks), + entryCount: len(allContents), } w := bufio.NewWriter(output) // prepare extra data to be appended at the end of an index. - extraData := prepareExtraData(allBlocks, layout) + extraData := prepareExtraData(allContents, layout) // write header header := make([]byte, 8) @@ -69,9 +69,9 @@ func (b packIndexBuilder) Build(output io.Writer) error { return errors.Wrap(err, "unable to write header") } - // write all sorted blocks. + // write all sorted contents. entry := make([]byte, layout.entryLength) - for _, it := range allBlocks { + for _, it := range allContents { if err := writeEntry(w, it, layout, entry); err != nil { return errors.Wrap(err, "unable to write entry") } @@ -84,12 +84,12 @@ func (b packIndexBuilder) Build(output io.Writer) error { return w.Flush() } -func prepareExtraData(allBlocks []*Info, layout *indexLayout) []byte { +func prepareExtraData(allContents []*Info, layout *indexLayout) []byte { var extraData []byte - for i, it := range allBlocks { + for i, it := range allContents { if i == 0 { - layout.keyLength = len(contentIDToBytes(it.BlockID)) + layout.keyLength = len(contentIDToBytes(it.ID)) } if it.PackBlobID != "" { if _, ok := layout.packBlobIDOffsets[it.PackBlobID]; !ok { @@ -106,7 +106,7 @@ func prepareExtraData(allBlocks []*Info, layout *indexLayout) []byte { } func writeEntry(w io.Writer, it *Info, layout *indexLayout, entry []byte) error { - k := contentIDToBytes(it.BlockID) + k := contentIDToBytes(it.ID) if len(k) != layout.keyLength { return errors.Errorf("inconsistent key length: %v vs %v", len(k), layout.keyLength) } @@ -133,7 +133,7 @@ func formatEntry(entry []byte, it *Info, layout *indexLayout) error { timestampAndFlags := uint64(it.TimestampSeconds) << 16 if len(it.PackBlobID) == 0 { - return errors.Errorf("empty pack block ID for %v", it.BlockID) + return errors.Errorf("empty pack content ID for %v", it.ID) } binary.BigEndian.PutUint32(entryPackFileOffset, layout.extraDataOffset+layout.packBlobIDOffsets[it.PackBlobID]) diff --git a/repo/block/cache_hmac.go b/repo/content/cache_hmac.go similarity index 97% rename from repo/block/cache_hmac.go rename to repo/content/cache_hmac.go index 73fb09908..dbc311f76 100644 --- a/repo/block/cache_hmac.go +++ b/repo/content/cache_hmac.go @@ -1,4 +1,4 @@ -package block +package content import "crypto/hmac" import "crypto/sha256" diff --git a/repo/block/caching_options.go b/repo/content/caching_options.go similarity index 95% rename from repo/block/caching_options.go rename to repo/content/caching_options.go index bd4b92bf1..383c22724 100644 --- a/repo/block/caching_options.go +++ b/repo/content/caching_options.go @@ -1,4 +1,4 @@ -package block +package content // CachingOptions specifies configuration of local cache. type CachingOptions struct { diff --git a/repo/block/committed_block_index.go b/repo/content/committed_content_index.go similarity index 63% rename from repo/block/committed_block_index.go rename to repo/content/committed_content_index.go index f90ff0a22..91c018db5 100644 --- a/repo/block/committed_block_index.go +++ b/repo/content/committed_content_index.go @@ -1,4 +1,4 @@ -package block +package content import ( "path/filepath" @@ -9,37 +9,37 @@ "github.com/kopia/kopia/repo/blob" ) -type committedBlockIndex struct { - cache committedBlockIndexCache +type committedContentIndex struct { + cache committedContentIndexCache mu sync.Mutex inUse map[blob.ID]packIndex merged mergedIndex } -type committedBlockIndexCache interface { +type committedContentIndexCache interface { hasIndexBlobID(indexBlob blob.ID) (bool, error) - addBlockToCache(indexBlob blob.ID, data []byte) error + addContentToCache(indexBlob blob.ID, data []byte) error openIndex(indexBlob blob.ID) (packIndex, error) expireUnused(used []blob.ID) error } -func (b *committedBlockIndex) getBlock(blockID string) (Info, error) { +func (b *committedContentIndex) getContent(contentID ID) (Info, error) { b.mu.Lock() defer b.mu.Unlock() - info, err := b.merged.GetInfo(blockID) + info, err := b.merged.GetInfo(contentID) if info != nil { return *info, nil } if err == nil { - return Info{}, ErrBlockNotFound + return Info{}, ErrContentNotFound } return Info{}, err } -func (b *committedBlockIndex) addBlock(indexBlobID blob.ID, data []byte, use bool) error { - if err := b.cache.addBlockToCache(indexBlobID, data); err != nil { +func (b *committedContentIndex) addContent(indexBlobID blob.ID, data []byte, use bool) error { + if err := b.cache.addContentToCache(indexBlobID, data); err != nil { return err } @@ -63,7 +63,7 @@ func (b *committedBlockIndex) addBlock(indexBlobID blob.ID, data []byte, use boo return nil } -func (b *committedBlockIndex) listBlocks(prefix string, cb func(i Info) error) error { +func (b *committedContentIndex) listContents(prefix ID, cb func(i Info) error) error { b.mu.Lock() m := append(mergedIndex(nil), b.merged...) b.mu.Unlock() @@ -71,7 +71,7 @@ func (b *committedBlockIndex) listBlocks(prefix string, cb func(i Info) error) e return m.Iterate(prefix, cb) } -func (b *committedBlockIndex) packFilesChanged(packFiles []blob.ID) bool { +func (b *committedContentIndex) packFilesChanged(packFiles []blob.ID) bool { if len(packFiles) != len(b.inUse) { return true } @@ -85,7 +85,7 @@ func (b *committedBlockIndex) packFilesChanged(packFiles []blob.ID) bool { return false } -func (b *committedBlockIndex) use(packFiles []blob.ID) (bool, error) { +func (b *committedContentIndex) use(packFiles []blob.ID) (bool, error) { b.mu.Lock() defer b.mu.Unlock() @@ -113,26 +113,26 @@ func (b *committedBlockIndex) use(packFiles []blob.ID) (bool, error) { b.inUse = newInUse if err := b.cache.expireUnused(packFiles); err != nil { - log.Warningf("unable to expire unused block index files: %v", err) + log.Warningf("unable to expire unused content index files: %v", err) } newMerged = nil return true, nil } -func newCommittedBlockIndex(caching CachingOptions) (*committedBlockIndex, error) { - var cache committedBlockIndexCache +func newCommittedContentIndex(caching CachingOptions) (*committedContentIndex, error) { + var cache committedContentIndexCache if caching.CacheDirectory != "" { dirname := filepath.Join(caching.CacheDirectory, "indexes") - cache = &diskCommittedBlockIndexCache{dirname} + cache = &diskCommittedContentIndexCache{dirname} } else { - cache = &memoryCommittedBlockIndexCache{ - blocks: map[blob.ID]packIndex{}, + cache = &memoryCommittedContentIndexCache{ + contents: map[blob.ID]packIndex{}, } } - return &committedBlockIndex{ + return &committedContentIndex{ cache: cache, inUse: map[blob.ID]packIndex{}, }, nil diff --git a/repo/block/committed_block_index_disk_cache.go b/repo/content/committed_content_index_disk_cache.go similarity index 73% rename from repo/block/committed_block_index_disk_cache.go rename to repo/content/committed_content_index_disk_cache.go index 36583554c..fdbaffd32 100644 --- a/repo/block/committed_block_index_disk_cache.go +++ b/repo/content/committed_content_index_disk_cache.go @@ -1,4 +1,4 @@ -package block +package content import ( "io/ioutil" @@ -14,19 +14,19 @@ ) const ( - simpleIndexSuffix = ".sndx" - unusedCommittedBlockIndexCleanupTime = 1 * time.Hour // delete unused committed index blobs after 1 hour + simpleIndexSuffix = ".sndx" + unusedCommittedContentIndexCleanupTime = 1 * time.Hour // delete unused committed index blobs after 1 hour ) -type diskCommittedBlockIndexCache struct { +type diskCommittedContentIndexCache struct { dirname string } -func (c *diskCommittedBlockIndexCache) indexBlobPath(indexBlobID blob.ID) string { +func (c *diskCommittedContentIndexCache) indexBlobPath(indexBlobID blob.ID) string { return filepath.Join(c.dirname, string(indexBlobID)+simpleIndexSuffix) } -func (c *diskCommittedBlockIndexCache) openIndex(indexBlobID blob.ID) (packIndex, error) { +func (c *diskCommittedContentIndexCache) openIndex(indexBlobID blob.ID) (packIndex, error) { fullpath := c.indexBlobPath(indexBlobID) f, err := mmap.Open(fullpath) @@ -37,7 +37,7 @@ func (c *diskCommittedBlockIndexCache) openIndex(indexBlobID blob.ID) (packIndex return openPackIndex(f) } -func (c *diskCommittedBlockIndexCache) hasIndexBlobID(indexBlobID blob.ID) (bool, error) { +func (c *diskCommittedContentIndexCache) hasIndexBlobID(indexBlobID blob.ID) (bool, error) { _, err := os.Stat(c.indexBlobPath(indexBlobID)) if err == nil { return true, nil @@ -49,7 +49,7 @@ func (c *diskCommittedBlockIndexCache) hasIndexBlobID(indexBlobID blob.ID) (bool return false, err } -func (c *diskCommittedBlockIndexCache) addBlockToCache(indexBlobID blob.ID, data []byte) error { +func (c *diskCommittedContentIndexCache) addContentToCache(indexBlobID blob.ID, data []byte) error { exists, err := c.hasIndexBlobID(indexBlobID) if err != nil { return err @@ -66,13 +66,13 @@ func (c *diskCommittedBlockIndexCache) addBlockToCache(indexBlobID blob.ID, data // rename() is atomic, so one process will succeed, but the other will fail if err := os.Rename(tmpFile, c.indexBlobPath(indexBlobID)); err != nil { - // verify that the block exists + // verify that the content exists exists, err := c.hasIndexBlobID(indexBlobID) if err != nil { return err } if !exists { - return errors.Errorf("unsuccessful index write of block %q", indexBlobID) + return errors.Errorf("unsuccessful index write of content %q", indexBlobID) } } @@ -102,7 +102,7 @@ func writeTempFileAtomic(dirname string, data []byte) (string, error) { return tf.Name(), nil } -func (c *diskCommittedBlockIndexCache) expireUnused(used []blob.ID) error { +func (c *diskCommittedContentIndexCache) expireUnused(used []blob.ID) error { entries, err := ioutil.ReadDir(c.dirname) if err != nil { return errors.Wrap(err, "can't list cache") @@ -122,7 +122,7 @@ func (c *diskCommittedBlockIndexCache) expireUnused(used []blob.ID) error { } for _, rem := range remaining { - if time.Since(rem.ModTime()) > unusedCommittedBlockIndexCleanupTime { + if time.Since(rem.ModTime()) > unusedCommittedContentIndexCleanupTime { log.Debugf("removing unused %v %v", rem.Name(), rem.ModTime()) if err := os.Remove(filepath.Join(c.dirname, rem.Name())); err != nil { log.Warningf("unable to remove unused index file: %v", err) diff --git a/repo/content/committed_content_index_mem_cache.go b/repo/content/committed_content_index_mem_cache.go new file mode 100644 index 000000000..c83fda34d --- /dev/null +++ b/repo/content/committed_content_index_mem_cache.go @@ -0,0 +1,51 @@ +package content + +import ( + "bytes" + "sync" + + "github.com/pkg/errors" + + "github.com/kopia/kopia/repo/blob" +) + +type memoryCommittedContentIndexCache struct { + mu sync.Mutex + contents map[blob.ID]packIndex +} + +func (m *memoryCommittedContentIndexCache) hasIndexBlobID(indexBlobID blob.ID) (bool, error) { + m.mu.Lock() + defer m.mu.Unlock() + + return m.contents[indexBlobID] != nil, nil +} + +func (m *memoryCommittedContentIndexCache) addContentToCache(indexBlobID blob.ID, data []byte) error { + m.mu.Lock() + defer m.mu.Unlock() + + ndx, err := openPackIndex(bytes.NewReader(data)) + if err != nil { + return err + } + + m.contents[indexBlobID] = ndx + return nil +} + +func (m *memoryCommittedContentIndexCache) openIndex(indexBlobID blob.ID) (packIndex, error) { + m.mu.Lock() + defer m.mu.Unlock() + + v := m.contents[indexBlobID] + if v == nil { + return nil, errors.Errorf("content not found in cache: %v", indexBlobID) + } + + return v, nil +} + +func (m *memoryCommittedContentIndexCache) expireUnused(used []blob.ID) error { + return nil +} diff --git a/repo/block/block_cache.go b/repo/content/content_cache.go similarity index 63% rename from repo/block/block_cache.go rename to repo/content/content_cache.go index d699d892b..6e76d9f2e 100644 --- a/repo/block/block_cache.go +++ b/repo/content/content_cache.go @@ -1,4 +1,4 @@ -package block +package content import ( "container/heap" @@ -19,7 +19,7 @@ defaultTouchThreshold = 10 * time.Minute ) -type blockCache struct { +type contentCache struct { st blob.Storage cacheStorage blob.Storage maxSizeBytes int64 @@ -33,12 +33,12 @@ type blockCache struct { closed chan struct{} } -type blockToucher interface { - TouchBlob(ctx context.Context, blockID blob.ID, threshold time.Duration) error +type contentToucher interface { + TouchBlob(ctx context.Context, contentID blob.ID, threshold time.Duration) error } func adjustCacheKey(cacheKey blob.ID) blob.ID { - // block IDs with odd length have a single-byte prefix. + // content IDs with odd length have a single-byte prefix. // move the prefix to the end of cache key to make sure the top level shard is spread 256 ways. if len(cacheKey)%2 == 1 { return cacheKey[1:] + cacheKey[0:1] @@ -47,12 +47,12 @@ func adjustCacheKey(cacheKey blob.ID) blob.ID { return cacheKey } -func (c *blockCache) getContentBlock(ctx context.Context, cacheKey blob.ID, blobID blob.ID, offset, length int64) ([]byte, error) { +func (c *contentCache) getContentContent(ctx context.Context, cacheKey blob.ID, blobID blob.ID, offset, length int64) ([]byte, error) { cacheKey = adjustCacheKey(cacheKey) - useCache := shouldUseBlockCache(ctx) && c.cacheStorage != nil + useCache := shouldUseContentCache(ctx) && c.cacheStorage != nil if useCache { - if b := c.readAndVerifyCacheBlock(ctx, cacheKey); b != nil { + if b := c.readAndVerifyCacheContent(ctx, cacheKey); b != nil { return b, nil } } @@ -72,12 +72,12 @@ func (c *blockCache) getContentBlock(ctx context.Context, cacheKey blob.ID, blob return b, err } -func (c *blockCache) readAndVerifyCacheBlock(ctx context.Context, cacheKey blob.ID) []byte { +func (c *contentCache) readAndVerifyCacheContent(ctx context.Context, cacheKey blob.ID) []byte { b, err := c.cacheStorage.GetBlob(ctx, cacheKey, 0, -1) if err == nil { b, err = verifyAndStripHMAC(b, c.hmacSecret) if err == nil { - if t, ok := c.cacheStorage.(blockToucher); ok { + if t, ok := c.cacheStorage.(contentToucher); ok { t.TouchBlob(ctx, cacheKey, c.touchThreshold) //nolint:errcheck } @@ -85,8 +85,8 @@ func (c *blockCache) readAndVerifyCacheBlock(ctx context.Context, cacheKey blob. return b } - // ignore malformed blocks - log.Warningf("malformed block %v: %v", cacheKey, err) + // ignore malformed contents + log.Warningf("malformed content %v: %v", cacheKey, err) return nil } @@ -96,11 +96,11 @@ func (c *blockCache) readAndVerifyCacheBlock(ctx context.Context, cacheKey blob. return nil } -func (c *blockCache) close() { +func (c *contentCache) close() { close(c.closed) } -func (c *blockCache) sweepDirectoryPeriodically(ctx context.Context) { +func (c *contentCache) sweepDirectoryPeriodically(ctx context.Context) { for { select { case <-c.closed: @@ -109,30 +109,30 @@ func (c *blockCache) sweepDirectoryPeriodically(ctx context.Context) { case <-time.After(c.sweepFrequency): err := c.sweepDirectory(ctx) if err != nil { - log.Warningf("blockCache sweep failed: %v", err) + log.Warningf("contentCache sweep failed: %v", err) } } } } -// A blockMetadataHeap implements heap.Interface and holds blob.Metadata. -type blockMetadataHeap []blob.Metadata +// A contentMetadataHeap implements heap.Interface and holds blob.Metadata. +type contentMetadataHeap []blob.Metadata -func (h blockMetadataHeap) Len() int { return len(h) } +func (h contentMetadataHeap) Len() int { return len(h) } -func (h blockMetadataHeap) Less(i, j int) bool { +func (h contentMetadataHeap) Less(i, j int) bool { return h[i].Timestamp.Before(h[j].Timestamp) } -func (h blockMetadataHeap) Swap(i, j int) { +func (h contentMetadataHeap) Swap(i, j int) { h[i], h[j] = h[j], h[i] } -func (h *blockMetadataHeap) Push(x interface{}) { +func (h *contentMetadataHeap) Push(x interface{}) { *h = append(*h, x.(blob.Metadata)) } -func (h *blockMetadataHeap) Pop() interface{} { +func (h *contentMetadataHeap) Pop() interface{} { old := *h n := len(old) item := old[n-1] @@ -140,7 +140,7 @@ func (h *blockMetadataHeap) Pop() interface{} { return item } -func (c *blockCache) sweepDirectory(ctx context.Context) (err error) { +func (c *contentCache) sweepDirectory(ctx context.Context) (err error) { c.mu.Lock() defer c.mu.Unlock() @@ -150,7 +150,7 @@ func (c *blockCache) sweepDirectory(ctx context.Context) (err error) { t0 := time.Now() - var h blockMetadataHeap + var h contentMetadataHeap var totalRetainedSize int64 err = c.cacheStorage.ListBlobs(ctx, "", func(it blob.Metadata) error { @@ -176,21 +176,21 @@ func (c *blockCache) sweepDirectory(ctx context.Context) (err error) { return nil } -func newBlockCache(ctx context.Context, st blob.Storage, caching CachingOptions) (*blockCache, error) { +func newContentCache(ctx context.Context, st blob.Storage, caching CachingOptions) (*contentCache, error) { var cacheStorage blob.Storage var err error if caching.MaxCacheSizeBytes > 0 && caching.CacheDirectory != "" { - blockCacheDir := filepath.Join(caching.CacheDirectory, "blocks") + contentCacheDir := filepath.Join(caching.CacheDirectory, "contents") - if _, err = os.Stat(blockCacheDir); os.IsNotExist(err) { - if err = os.MkdirAll(blockCacheDir, 0700); err != nil { + if _, err = os.Stat(contentCacheDir); os.IsNotExist(err) { + if err = os.MkdirAll(contentCacheDir, 0700); err != nil { return nil, err } } cacheStorage, err = filesystem.New(context.Background(), &filesystem.Options{ - Path: blockCacheDir, + Path: contentCacheDir, DirectoryShards: []int{2}, }) if err != nil { @@ -198,11 +198,11 @@ func newBlockCache(ctx context.Context, st blob.Storage, caching CachingOptions) } } - return newBlockCacheWithCacheStorage(ctx, st, cacheStorage, caching, defaultTouchThreshold, defaultSweepFrequency) + return newContentCacheWithCacheStorage(ctx, st, cacheStorage, caching, defaultTouchThreshold, defaultSweepFrequency) } -func newBlockCacheWithCacheStorage(ctx context.Context, st, cacheStorage blob.Storage, caching CachingOptions, touchThreshold time.Duration, sweepFrequency time.Duration) (*blockCache, error) { - c := &blockCache{ +func newContentCacheWithCacheStorage(ctx context.Context, st, cacheStorage blob.Storage, caching CachingOptions, touchThreshold time.Duration, sweepFrequency time.Duration) (*contentCache, error) { + c := &contentCache{ st: st, cacheStorage: cacheStorage, maxSizeBytes: caching.MaxCacheSizeBytes, diff --git a/repo/block/block_cache_test.go b/repo/content/content_cache_test.go similarity index 59% rename from repo/block/block_cache_test.go rename to repo/content/content_cache_test.go index cfbb8eed5..45d9c467d 100644 --- a/repo/block/block_cache_test.go +++ b/repo/content/content_cache_test.go @@ -1,4 +1,4 @@ -package block +package content import ( "bytes" @@ -17,12 +17,12 @@ "github.com/kopia/kopia/repo/blob" ) -func newUnderlyingStorageForBlockCacheTesting(t *testing.T) blob.Storage { +func newUnderlyingStorageForContentCacheTesting(t *testing.T) blob.Storage { ctx := context.Background() data := blobtesting.DataMap{} st := blobtesting.NewMapStorage(data, nil, nil) - assertNoError(t, st.PutBlob(ctx, "block-1", []byte{1, 2, 3, 4, 5, 6, 7, 8, 9, 10})) - assertNoError(t, st.PutBlob(ctx, "block-4k", bytes.Repeat([]byte{1, 2, 3, 4}, 1000))) // 4000 bytes + assertNoError(t, st.PutBlob(ctx, "content-1", []byte{1, 2, 3, 4, 5, 6, 7, 8, 9, 10})) + assertNoError(t, st.PutBlob(ctx, "content-4k", bytes.Repeat([]byte{1, 2, 3, 4}, 1000))) // 4000 bytes return st } @@ -30,9 +30,9 @@ func TestCacheExpiration(t *testing.T) { cacheData := blobtesting.DataMap{} cacheStorage := blobtesting.NewMapStorage(cacheData, nil, nil) - underlyingStorage := newUnderlyingStorageForBlockCacheTesting(t) + underlyingStorage := newUnderlyingStorageForContentCacheTesting(t) - cache, err := newBlockCacheWithCacheStorage(context.Background(), underlyingStorage, cacheStorage, CachingOptions{ + cache, err := newContentCacheWithCacheStorage(context.Background(), underlyingStorage, cacheStorage, CachingOptions{ MaxCacheSizeBytes: 10000, }, 0, 500*time.Millisecond) if err != nil { @@ -41,22 +41,22 @@ func TestCacheExpiration(t *testing.T) { defer cache.close() ctx := context.Background() - _, err = cache.getContentBlock(ctx, "00000a", "block-4k", 0, -1) // 4k + _, err = cache.getContentContent(ctx, "00000a", "content-4k", 0, -1) // 4k assertNoError(t, err) - _, err = cache.getContentBlock(ctx, "00000b", "block-4k", 0, -1) // 4k + _, err = cache.getContentContent(ctx, "00000b", "content-4k", 0, -1) // 4k assertNoError(t, err) - _, err = cache.getContentBlock(ctx, "00000c", "block-4k", 0, -1) // 4k + _, err = cache.getContentContent(ctx, "00000c", "content-4k", 0, -1) // 4k assertNoError(t, err) - _, err = cache.getContentBlock(ctx, "00000d", "block-4k", 0, -1) // 4k + _, err = cache.getContentContent(ctx, "00000d", "content-4k", 0, -1) // 4k assertNoError(t, err) // wait for a sweep time.Sleep(2 * time.Second) // 00000a and 00000b will be removed from cache because it's the oldest. - // to verify, let's remove block-4k from the underlying storage and make sure we can still read + // to verify, let's remove content-4k from the underlying storage and make sure we can still read // 00000c and 00000d from the cache but not 00000a nor 00000b - assertNoError(t, underlyingStorage.DeleteBlob(ctx, "block-4k")) + assertNoError(t, underlyingStorage.DeleteBlob(ctx, "content-4k")) cases := []struct { blobID blob.ID @@ -69,16 +69,16 @@ func TestCacheExpiration(t *testing.T) { } for _, tc := range cases { - _, got := cache.getContentBlock(ctx, tc.blobID, "block-4k", 0, -1) + _, got := cache.getContentContent(ctx, tc.blobID, "content-4k", 0, -1) if want := tc.expectedError; got != want { - t.Errorf("unexpected error when getting block %v: %v wanted %v", tc.blobID, got, want) + t.Errorf("unexpected error when getting content %v: %v wanted %v", tc.blobID, got, want) } else { - t.Logf("got correct error %v when reading block %v", tc.expectedError, tc.blobID) + t.Logf("got correct error %v when reading content %v", tc.expectedError, tc.blobID) } } } -func TestDiskBlockCache(t *testing.T) { +func TestDiskContentCache(t *testing.T) { ctx := context.Background() tmpDir, err := ioutil.TempDir("", "kopia") @@ -87,7 +87,7 @@ func TestDiskBlockCache(t *testing.T) { } defer os.RemoveAll(tmpDir) - cache, err := newBlockCache(ctx, newUnderlyingStorageForBlockCacheTesting(t), CachingOptions{ + cache, err := newContentCache(ctx, newUnderlyingStorageForContentCacheTesting(t), CachingOptions{ MaxCacheSizeBytes: 10000, CacheDirectory: tmpDir, }) @@ -96,13 +96,13 @@ func TestDiskBlockCache(t *testing.T) { t.Fatalf("err: %v", err) } defer cache.close() - verifyBlockCache(t, cache) + verifyContentCache(t, cache) } -func verifyBlockCache(t *testing.T, cache *blockCache) { +func verifyContentCache(t *testing.T, cache *contentCache) { ctx := context.Background() - t.Run("GetContentBlock", func(t *testing.T) { + t.Run("GetContentContent", func(t *testing.T) { cases := []struct { cacheKey blob.ID blobID blob.ID @@ -112,19 +112,19 @@ func verifyBlockCache(t *testing.T, cache *blockCache) { expected []byte err error }{ - {"xf0f0f1", "block-1", 1, 5, []byte{2, 3, 4, 5, 6}, nil}, - {"xf0f0f2", "block-1", 0, -1, []byte{1, 2, 3, 4, 5, 6, 7, 8, 9, 10}, nil}, - {"xf0f0f1", "block-1", 1, 5, []byte{2, 3, 4, 5, 6}, nil}, - {"xf0f0f2", "block-1", 0, -1, []byte{1, 2, 3, 4, 5, 6, 7, 8, 9, 10}, nil}, - {"xf0f0f3", "no-such-block", 0, -1, nil, blob.ErrBlobNotFound}, - {"xf0f0f4", "no-such-block", 10, 5, nil, blob.ErrBlobNotFound}, - {"f0f0f5", "block-1", 7, 3, []byte{8, 9, 10}, nil}, - {"xf0f0f6", "block-1", 11, 10, nil, errors.Errorf("invalid offset")}, - {"xf0f0f6", "block-1", -1, 5, nil, errors.Errorf("invalid offset")}, + {"xf0f0f1", "content-1", 1, 5, []byte{2, 3, 4, 5, 6}, nil}, + {"xf0f0f2", "content-1", 0, -1, []byte{1, 2, 3, 4, 5, 6, 7, 8, 9, 10}, nil}, + {"xf0f0f1", "content-1", 1, 5, []byte{2, 3, 4, 5, 6}, nil}, + {"xf0f0f2", "content-1", 0, -1, []byte{1, 2, 3, 4, 5, 6, 7, 8, 9, 10}, nil}, + {"xf0f0f3", "no-such-content", 0, -1, nil, blob.ErrBlobNotFound}, + {"xf0f0f4", "no-such-content", 10, 5, nil, blob.ErrBlobNotFound}, + {"f0f0f5", "content-1", 7, 3, []byte{8, 9, 10}, nil}, + {"xf0f0f6", "content-1", 11, 10, nil, errors.Errorf("invalid offset")}, + {"xf0f0f6", "content-1", -1, 5, nil, errors.Errorf("invalid offset")}, } for _, tc := range cases { - v, err := cache.getContentBlock(ctx, tc.cacheKey, tc.blobID, tc.offset, tc.length) + v, err := cache.getContentContent(ctx, tc.cacheKey, tc.blobID, tc.offset, tc.length) if (err != nil) != (tc.err != nil) { t.Errorf("unexpected error for %v: %+v, wanted %+v", tc.cacheKey, err, tc.err) } else if err != nil && err.Error() != tc.err.Error() { @@ -135,7 +135,7 @@ func verifyBlockCache(t *testing.T, cache *blockCache) { } } - verifyStorageBlockList(t, cache.cacheStorage, "f0f0f1x", "f0f0f2x", "f0f0f5") + verifyStorageContentList(t, cache.cacheStorage, "f0f0f1x", "f0f0f2x", "f0f0f5") }) t.Run("DataCorruption", func(t *testing.T) { @@ -149,12 +149,12 @@ func verifyBlockCache(t *testing.T, cache *blockCache) { d[0] ^= 1 if err := cache.cacheStorage.PutBlob(ctx, cacheKey, d); err != nil { - t.Fatalf("unable to write corrupted block: %v", err) + t.Fatalf("unable to write corrupted content: %v", err) } - v, err := cache.getContentBlock(ctx, "xf0f0f1", "block-1", 1, 5) + v, err := cache.getContentContent(ctx, "xf0f0f1", "content-1", 1, 5) if err != nil { - t.Fatalf("error in getContentBlock: %v", err) + t.Fatalf("error in getContentContent: %v", err) } if got, want := v, []byte{2, 3, 4, 5, 6}; !reflect.DeepEqual(v, want) { t.Errorf("invalid result when reading corrupted data: %v, wanted %v", got, want) @@ -167,7 +167,7 @@ func TestCacheFailureToOpen(t *testing.T) { cacheData := blobtesting.DataMap{} cacheStorage := blobtesting.NewMapStorage(cacheData, nil, nil) - underlyingStorage := newUnderlyingStorageForBlockCacheTesting(t) + underlyingStorage := newUnderlyingStorageForContentCacheTesting(t) faultyCache := &blobtesting.FaultyStorage{ Base: cacheStorage, Faults: map[string][]*blobtesting.Fault{ @@ -178,7 +178,7 @@ func TestCacheFailureToOpen(t *testing.T) { } // Will fail because of ListBlobs failure. - _, err := newBlockCacheWithCacheStorage(context.Background(), underlyingStorage, faultyCache, CachingOptions{ + _, err := newContentCacheWithCacheStorage(context.Background(), underlyingStorage, faultyCache, CachingOptions{ MaxCacheSizeBytes: 10000, }, 0, 5*time.Hour) if err == nil || !strings.Contains(err.Error(), someError.Error()) { @@ -186,7 +186,7 @@ func TestCacheFailureToOpen(t *testing.T) { } // ListBlobs fails only once, next time it succeeds. - cache, err := newBlockCacheWithCacheStorage(context.Background(), underlyingStorage, faultyCache, CachingOptions{ + cache, err := newContentCacheWithCacheStorage(context.Background(), underlyingStorage, faultyCache, CachingOptions{ MaxCacheSizeBytes: 10000, }, 0, 100*time.Millisecond) if err != nil { @@ -201,12 +201,12 @@ func TestCacheFailureToWrite(t *testing.T) { cacheData := blobtesting.DataMap{} cacheStorage := blobtesting.NewMapStorage(cacheData, nil, nil) - underlyingStorage := newUnderlyingStorageForBlockCacheTesting(t) + underlyingStorage := newUnderlyingStorageForContentCacheTesting(t) faultyCache := &blobtesting.FaultyStorage{ Base: cacheStorage, } - cache, err := newBlockCacheWithCacheStorage(context.Background(), underlyingStorage, faultyCache, CachingOptions{ + cache, err := newContentCacheWithCacheStorage(context.Background(), underlyingStorage, faultyCache, CachingOptions{ MaxCacheSizeBytes: 10000, }, 0, 5*time.Hour) if err != nil { @@ -222,7 +222,7 @@ func TestCacheFailureToWrite(t *testing.T) { }, } - v, err := cache.getContentBlock(ctx, "aa", "block-1", 0, 3) + v, err := cache.getContentContent(ctx, "aa", "content-1", 0, 3) if err != nil { t.Errorf("write failure wasn't ignored: %v", err) } @@ -245,12 +245,12 @@ func TestCacheFailureToRead(t *testing.T) { cacheData := blobtesting.DataMap{} cacheStorage := blobtesting.NewMapStorage(cacheData, nil, nil) - underlyingStorage := newUnderlyingStorageForBlockCacheTesting(t) + underlyingStorage := newUnderlyingStorageForContentCacheTesting(t) faultyCache := &blobtesting.FaultyStorage{ Base: cacheStorage, } - cache, err := newBlockCacheWithCacheStorage(context.Background(), underlyingStorage, faultyCache, CachingOptions{ + cache, err := newContentCacheWithCacheStorage(context.Background(), underlyingStorage, faultyCache, CachingOptions{ MaxCacheSizeBytes: 10000, }, 0, 5*time.Hour) if err != nil { @@ -267,7 +267,7 @@ func TestCacheFailureToRead(t *testing.T) { } for i := 0; i < 2; i++ { - v, err := cache.getContentBlock(ctx, "aa", "block-1", 0, 3) + v, err := cache.getContentContent(ctx, "aa", "content-1", 0, 3) if err != nil { t.Errorf("read failure wasn't ignored: %v", err) } @@ -278,19 +278,19 @@ func TestCacheFailureToRead(t *testing.T) { } } -func verifyStorageBlockList(t *testing.T, st blob.Storage, expectedBlocks ...blob.ID) { +func verifyStorageContentList(t *testing.T, st blob.Storage, expectedContents ...blob.ID) { t.Helper() - var foundBlocks []blob.ID + var foundContents []blob.ID assertNoError(t, st.ListBlobs(context.Background(), "", func(bm blob.Metadata) error { - foundBlocks = append(foundBlocks, bm.BlobID) + foundContents = append(foundContents, bm.BlobID) return nil })) - sort.Slice(foundBlocks, func(i, j int) bool { - return foundBlocks[i] < foundBlocks[j] + sort.Slice(foundContents, func(i, j int) bool { + return foundContents[i] < foundContents[j] }) - if !reflect.DeepEqual(foundBlocks, expectedBlocks) { - t.Errorf("unexpected block list: %v, wanted %v", foundBlocks, expectedBlocks) + if !reflect.DeepEqual(foundContents, expectedContents) { + t.Errorf("unexpected content list: %v, wanted %v", foundContents, expectedContents) } } diff --git a/repo/block/block_formatter.go b/repo/content/content_formatter.go similarity index 84% rename from repo/block/block_formatter.go rename to repo/content/content_formatter.go index 37758f91c..609f17530 100644 --- a/repo/block/block_formatter.go +++ b/repo/content/content_formatter.go @@ -1,4 +1,4 @@ -package block +package content import ( "crypto/aes" @@ -22,22 +22,22 @@ salsaKeyLength = 32 ) -// HashFunc computes hash of block of data using a cryptographic hash function, possibly with HMAC and/or truncation. +// HashFunc computes hash of content of data using a cryptographic hash function, possibly with HMAC and/or truncation. type HashFunc func(data []byte) []byte // HashFuncFactory returns a hash function for given formatting options. type HashFuncFactory func(o FormattingOptions) (HashFunc, error) -// Encryptor performs encryption and decryption of blocks of data. +// Encryptor performs encryption and decryption of contents of data. type Encryptor interface { // Encrypt returns encrypted bytes corresponding to the given plaintext. // Must not clobber the input slice and return ciphertext with additional padding and checksum. - Encrypt(plainText []byte, blockID []byte) ([]byte, error) + Encrypt(plainText []byte, contentID []byte) ([]byte, error) // Decrypt returns unencrypted bytes corresponding to the given ciphertext. // Must not clobber the input slice. If IsAuthenticated() == true, Decrypt will perform // authenticity check before decrypting. - Decrypt(cipherText []byte, blockID []byte) ([]byte, error) + Decrypt(cipherText []byte, contentID []byte) ([]byte, error) // IsAuthenticated returns true if encryption is authenticated. // In this case Decrypt() is expected to perform authenticity check. @@ -54,11 +54,11 @@ type Encryptor interface { type nullEncryptor struct { } -func (fi nullEncryptor) Encrypt(plainText []byte, blockID []byte) ([]byte, error) { +func (fi nullEncryptor) Encrypt(plainText []byte, contentID []byte) ([]byte, error) { return cloneBytes(plainText), nil } -func (fi nullEncryptor) Decrypt(cipherText []byte, blockID []byte) ([]byte, error) { +func (fi nullEncryptor) Decrypt(cipherText []byte, contentID []byte) ([]byte, error) { return cloneBytes(cipherText), nil } @@ -66,17 +66,17 @@ func (fi nullEncryptor) IsAuthenticated() bool { return false } -// ctrEncryptor implements encrypted format which uses CTR mode of a block cipher with nonce==IV. +// ctrEncryptor implements encrypted format which uses CTR mode of a content cipher with nonce==IV. type ctrEncryptor struct { createCipher func() (cipher.Block, error) } -func (fi ctrEncryptor) Encrypt(plainText []byte, blockID []byte) ([]byte, error) { - return symmetricEncrypt(fi.createCipher, blockID, plainText) +func (fi ctrEncryptor) Encrypt(plainText []byte, contentID []byte) ([]byte, error) { + return symmetricEncrypt(fi.createCipher, contentID, plainText) } -func (fi ctrEncryptor) Decrypt(cipherText []byte, blockID []byte) ([]byte, error) { - return symmetricEncrypt(fi.createCipher, blockID, cipherText) +func (fi ctrEncryptor) Decrypt(cipherText []byte, contentID []byte) ([]byte, error) { + return symmetricEncrypt(fi.createCipher, contentID, cipherText) } func (fi ctrEncryptor) IsAuthenticated() bool { @@ -101,7 +101,7 @@ type salsaEncryptor struct { hmacSecret []byte } -func (s salsaEncryptor) Decrypt(input []byte, blockID []byte) ([]byte, error) { +func (s salsaEncryptor) Decrypt(input []byte, contentID []byte) ([]byte, error) { if s.hmacSecret != nil { var err error input, err = verifyAndStripHMAC(input, s.hmacSecret) @@ -110,11 +110,11 @@ func (s salsaEncryptor) Decrypt(input []byte, blockID []byte) ([]byte, error) { } } - return s.encryptDecrypt(input, blockID) + return s.encryptDecrypt(input, contentID) } -func (s salsaEncryptor) Encrypt(input []byte, blockID []byte) ([]byte, error) { - v, err := s.encryptDecrypt(input, blockID) +func (s salsaEncryptor) Encrypt(input []byte, contentID []byte) ([]byte, error) { + v, err := s.encryptDecrypt(input, contentID) if err != nil { return nil, errors.Wrap(err, "decrypt") } @@ -130,17 +130,17 @@ func (s salsaEncryptor) IsAuthenticated() bool { return s.hmacSecret != nil } -func (s salsaEncryptor) encryptDecrypt(input []byte, blockID []byte) ([]byte, error) { - if len(blockID) < s.nonceSize { - return nil, errors.Errorf("hash too short, expected >=%v bytes, got %v", s.nonceSize, len(blockID)) +func (s salsaEncryptor) encryptDecrypt(input []byte, contentID []byte) ([]byte, error) { + if len(contentID) < s.nonceSize { + return nil, errors.Errorf("hash too short, expected >=%v bytes, got %v", s.nonceSize, len(contentID)) } result := make([]byte, len(input)) - nonce := blockID[0:s.nonceSize] + nonce := contentID[0:s.nonceSize] salsa20.XORKeyStream(result, input, nonce, s.key) return result, nil } -// truncatedHMACHashFuncFactory returns a HashFuncFactory that computes HMAC(hash, secret) of a given block of bytes +// truncatedHMACHashFuncFactory returns a HashFuncFactory that computes HMAC(hash, secret) of a given content of bytes // and truncates results to the given size. func truncatedHMACHashFuncFactory(hf func() hash.Hash, truncate int) HashFuncFactory { return func(o FormattingOptions) (HashFunc, error) { @@ -152,7 +152,7 @@ func truncatedHMACHashFuncFactory(hf func() hash.Hash, truncate int) HashFuncFac } } -// truncatedKeyedHashFuncFactory returns a HashFuncFactory that computes keyed hash of a given block of bytes +// truncatedKeyedHashFuncFactory returns a HashFuncFactory that computes keyed hash of a given content of bytes // and truncates results to the given size. func truncatedKeyedHashFuncFactory(hf func(key []byte) (hash.Hash, error), truncate int) HashFuncFactory { return func(o FormattingOptions) (HashFunc, error) { diff --git a/repo/block/block_formatter_test.go b/repo/content/content_formatter_test.go similarity index 93% rename from repo/block/block_formatter_test.go rename to repo/content/content_formatter_test.go index 0e26744d4..c740ecee7 100644 --- a/repo/block/block_formatter_test.go +++ b/repo/content/content_formatter_test.go @@ -1,4 +1,4 @@ -package block +package content import ( "bytes" @@ -49,13 +49,13 @@ func TestFormatters(t *testing.T) { continue } - blockID := h(data) - cipherText, err := e.Encrypt(data, blockID) + contentID := h(data) + cipherText, err := e.Encrypt(data, contentID) if err != nil || cipherText == nil { t.Errorf("invalid response from Encrypt: %v %v", cipherText, err) } - plainText, err := e.Decrypt(cipherText, blockID) + plainText, err := e.Decrypt(cipherText, contentID) if err != nil || plainText == nil { t.Errorf("invalid response from Decrypt: %v %v", plainText, err) } diff --git a/repo/block/block_formatting_options.go b/repo/content/content_formatting_options.go similarity index 90% rename from repo/block/block_formatting_options.go rename to repo/content/content_formatting_options.go index fcdcbddab..b5c46c251 100644 --- a/repo/block/block_formatting_options.go +++ b/repo/content/content_formatting_options.go @@ -1,4 +1,4 @@ -package block +package content import ( "crypto/sha256" @@ -7,7 +7,7 @@ "golang.org/x/crypto/hkdf" ) -// FormattingOptions describes the rules for formatting blocks in repository. +// FormattingOptions describes the rules for formatting contents in repository. type FormattingOptions struct { Version int `json:"version,omitempty"` // version number, must be "1" Hash string `json:"hash,omitempty"` // identifier of the hash algorithm used diff --git a/repo/block/content_id_to_bytes.go b/repo/content/content_id_to_bytes.go similarity index 64% rename from repo/block/content_id_to_bytes.go rename to repo/content/content_id_to_bytes.go index 136219d06..ec60fdd4e 100644 --- a/repo/block/content_id_to_bytes.go +++ b/repo/content/content_id_to_bytes.go @@ -1,25 +1,25 @@ -package block +package content import ( "encoding/hex" ) -func bytesToContentID(b []byte) string { +func bytesToContentID(b []byte) ID { if len(b) == 0 { return "" } if b[0] == 0xff { - return string(b[1:]) + return ID(b[1:]) } prefix := "" if b[0] != 0 { prefix = string(b[0:1]) } - return prefix + hex.EncodeToString(b[1:]) + return ID(prefix + hex.EncodeToString(b[1:])) } -func contentIDToBytes(c string) []byte { +func contentIDToBytes(c ID) []byte { var prefix []byte var skip int if len(c)%2 == 1 { @@ -29,7 +29,7 @@ func contentIDToBytes(c string) []byte { prefix = []byte{0} } - b, err := hex.DecodeString(c[skip:]) + b, err := hex.DecodeString(string(c[skip:])) if err != nil { return append([]byte{0xff}, []byte(c)...) } diff --git a/repo/block/block_index_recovery.go b/repo/content/content_index_recovery.go similarity index 88% rename from repo/block/block_index_recovery.go rename to repo/content/content_index_recovery.go index a9d4b7726..061e4402b 100644 --- a/repo/block/block_index_recovery.go +++ b/repo/content/content_index_recovery.go @@ -1,4 +1,4 @@ -package block +package content import ( "bytes" @@ -39,13 +39,13 @@ func (bm *Manager) RecoverIndexFromPackBlob(ctx context.Context, packFile blob.I return recovered, err } -type packBlockPostamble struct { +type packContentPostamble struct { localIndexIV []byte localIndexOffset uint32 localIndexLength uint32 } -func (p *packBlockPostamble) toBytes() ([]byte, error) { +func (p *packContentPostamble) toBytes() ([]byte, error) { // 4 varints + IV + 4 bytes of checksum + 1 byte of postamble length n := 0 buf := make([]byte, 4*binary.MaxVarintLen64+len(p.localIndexIV)+4+1) @@ -68,10 +68,10 @@ func (p *packBlockPostamble) toBytes() ([]byte, error) { return buf[0 : n+1], nil } -// findPostamble detects if a given block of bytes contains a possibly valid postamble, and returns it if so +// findPostamble detects if a given content of bytes contains a possibly valid postamble, and returns it if so // NOTE, even if this function returns a postamble, it should not be trusted to be correct, since it's not // cryptographically signed. this is to facilitate data recovery. -func findPostamble(b []byte) *packBlockPostamble { +func findPostamble(b []byte) *packContentPostamble { if len(b) == 0 { // no postamble return nil @@ -103,7 +103,7 @@ func findPostamble(b []byte) *packBlockPostamble { return decodePostamble(payload) } -func decodePostamble(payload []byte) *packBlockPostamble { +func decodePostamble(payload []byte) *packContentPostamble { flags, n := binary.Uvarint(payload) if n <= 0 { // invalid flags @@ -142,7 +142,7 @@ func decodePostamble(payload []byte) *packBlockPostamble { return nil } - return &packBlockPostamble{ + return &packContentPostamble{ localIndexIV: iv, localIndexLength: uint32(length), localIndexOffset: uint32(off), @@ -159,9 +159,9 @@ func (bm *Manager) buildLocalIndex(pending packIndexBuilder) ([]byte, error) { } // appendPackFileIndexRecoveryData appends data designed to help with recovery of pack index in case it gets damaged or lost. -func (bm *Manager) appendPackFileIndexRecoveryData(blockData []byte, pending packIndexBuilder) ([]byte, error) { +func (bm *Manager) appendPackFileIndexRecoveryData(contentData []byte, pending packIndexBuilder) ([]byte, error) { // build, encrypt and append local index - localIndexOffset := len(blockData) + localIndexOffset := len(contentData) localIndex, err := bm.buildLocalIndex(pending) if err != nil { return nil, err @@ -173,21 +173,21 @@ func (bm *Manager) appendPackFileIndexRecoveryData(blockData []byte, pending pac return nil, err } - postamble := packBlockPostamble{ + postamble := packContentPostamble{ localIndexIV: localIndexIV, localIndexOffset: uint32(localIndexOffset), localIndexLength: uint32(len(encryptedLocalIndex)), } - blockData = append(blockData, encryptedLocalIndex...) + contentData = append(contentData, encryptedLocalIndex...) postambleBytes, err := postamble.toBytes() if err != nil { return nil, err } - blockData = append(blockData, postambleBytes...) + contentData = append(contentData, postambleBytes...) - pa2 := findPostamble(blockData) + pa2 := findPostamble(contentData) if pa2 == nil { log.Fatalf("invalid postamble written, that could not be immediately decoded, it's a bug") } @@ -196,7 +196,7 @@ func (bm *Manager) appendPackFileIndexRecoveryData(blockData []byte, pending pac log.Fatalf("postamble did not round-trip: %v %v", postamble, *pa2) } - return blockData, nil + return contentData, nil } func (bm *Manager) readPackFileLocalIndex(ctx context.Context, packFile blob.ID, packFileLength int64) ([]byte, error) { diff --git a/repo/content/content_index_recovery_test.go b/repo/content/content_index_recovery_test.go new file mode 100644 index 000000000..2d8e50d60 --- /dev/null +++ b/repo/content/content_index_recovery_test.go @@ -0,0 +1,91 @@ +package content + +import ( + "context" + "testing" + "time" + + "github.com/kopia/kopia/internal/blobtesting" + "github.com/kopia/kopia/repo/blob" +) + +func TestContentIndexRecovery(t *testing.T) { + ctx := context.Background() + data := blobtesting.DataMap{} + keyTime := map[blob.ID]time.Time{} + bm := newTestContentManager(data, keyTime, nil) + content1 := writeContentAndVerify(ctx, t, bm, seededRandomData(10, 100)) + content2 := writeContentAndVerify(ctx, t, bm, seededRandomData(11, 100)) + content3 := writeContentAndVerify(ctx, t, bm, seededRandomData(12, 100)) + + if err := bm.Flush(ctx); err != nil { + t.Errorf("flush error: %v", err) + } + + // delete all index blobs + assertNoError(t, bm.st.ListBlobs(ctx, newIndexBlobPrefix, func(bi blob.Metadata) error { + log.Debugf("deleting %v", bi.BlobID) + return bm.st.DeleteBlob(ctx, bi.BlobID) + })) + + // now with index blobs gone, all contents appear to not be found + bm = newTestContentManager(data, keyTime, nil) + verifyContentNotFound(ctx, t, bm, content1) + verifyContentNotFound(ctx, t, bm, content2) + verifyContentNotFound(ctx, t, bm, content3) + + totalRecovered := 0 + + // pass 1 - just list contents to recover, but don't commit + err := bm.st.ListBlobs(ctx, PackBlobIDPrefix, func(bi blob.Metadata) error { + infos, err := bm.RecoverIndexFromPackBlob(ctx, bi.BlobID, bi.Length, false) + if err != nil { + return err + } + totalRecovered += len(infos) + log.Debugf("recovered %v contents", len(infos)) + return nil + }) + if err != nil { + t.Errorf("error recovering: %v", err) + } + + if got, want := totalRecovered, 3; got != want { + t.Errorf("invalid # of contents recovered: %v, want %v", got, want) + } + + // contents are stil not found + verifyContentNotFound(ctx, t, bm, content1) + verifyContentNotFound(ctx, t, bm, content2) + verifyContentNotFound(ctx, t, bm, content3) + + // pass 2 now pass commit=true to add recovered contents to index + totalRecovered = 0 + + err = bm.st.ListBlobs(ctx, PackBlobIDPrefix, func(bi blob.Metadata) error { + infos, err := bm.RecoverIndexFromPackBlob(ctx, bi.BlobID, bi.Length, true) + if err != nil { + return err + } + totalRecovered += len(infos) + log.Debugf("recovered %v contents", len(infos)) + return nil + }) + if err != nil { + t.Errorf("error recovering: %v", err) + } + + if got, want := totalRecovered, 3; got != want { + t.Errorf("invalid # of contents recovered: %v, want %v", got, want) + } + + verifyContent(ctx, t, bm, content1, seededRandomData(10, 100)) + verifyContent(ctx, t, bm, content2, seededRandomData(11, 100)) + verifyContent(ctx, t, bm, content3, seededRandomData(12, 100)) + if err := bm.Flush(ctx); err != nil { + t.Errorf("flush error: %v", err) + } + verifyContent(ctx, t, bm, content1, seededRandomData(10, 100)) + verifyContent(ctx, t, bm, content2, seededRandomData(11, 100)) + verifyContent(ctx, t, bm, content3, seededRandomData(12, 100)) +} diff --git a/repo/block/block_manager.go b/repo/content/content_manager.go similarity index 61% rename from repo/block/block_manager.go rename to repo/content/content_manager.go index 0886693a7..47f40dc3d 100644 --- a/repo/block/block_manager.go +++ b/repo/content/content_manager.go @@ -1,5 +1,5 @@ -// Package block implements repository support content-addressable storage blocks. -package block +// Package content implements repository support content-addressable storage contents. +package content import ( "bytes" @@ -25,8 +25,8 @@ ) var ( - log = repologging.Logger("kopia/block") - formatLog = repologging.Logger("kopia/block/format") + log = repologging.Logger("kopia/content") + formatLog = repologging.Logger("kopia/content/format") ) // PackBlobIDPrefix is the prefix for all pack blobs. @@ -51,8 +51,8 @@ indexLoadAttempts = 10 ) -// ErrBlockNotFound is returned when block is not found. -var ErrBlockNotFound = errors.New("block not found") +// ErrContentNotFound is returned when content is not found. +var ErrContentNotFound = errors.New("content not found") // IndexBlobInfo is an information about a single index blob managed by Manager. type IndexBlobInfo struct { @@ -61,23 +61,23 @@ type IndexBlobInfo struct { Timestamp time.Time } -// Manager manages storage blocks at a low level with encryption, deduplication and packaging. +// Manager builds content-addressable storage with encryption, deduplication and packaging on top of BLOB store. type Manager struct { Format FormattingOptions - stats Stats - blockCache *blockCache - listCache *listCache - st blob.Storage + stats Stats + contentCache *contentCache + listCache *listCache + st blob.Storage mu sync.Mutex locked bool checkInvariantsOnUnlock bool - currentPackItems map[string]Info // blocks that are in the pack block currently being built (all inline) - currentPackDataLength int // total length of all items in the current pack block - packIndexBuilder packIndexBuilder // blocks that are in index currently being built (current pack and all packs saved but not committed) - committedBlocks *committedBlockIndex + currentPackItems map[ID]Info // contents that are in the pack content currently being built (all inline) + currentPackDataLength int // total length of all items in the current pack content + packIndexBuilder packIndexBuilder // contents that are in index currently being built (current pack and all packs saved but not committed) + committedContents *committedContentIndex disableIndexFlushCount int flushPackIndexesAfter time.Time // time when those indexes should be flushed @@ -98,24 +98,24 @@ type Manager struct { repositoryFormatBytes []byte } -// DeleteBlock marks the given blockID as deleted. +// DeleteContent marks the given contentID as deleted. // -// NOTE: To avoid race conditions only blocks that cannot be possibly re-created -// should ever be deleted. That means that contents of such blocks should include some element +// NOTE: To avoid race conditions only contents that cannot be possibly re-created +// should ever be deleted. That means that contents of such contents should include some element // of randomness or a contemporaneous timestamp that will never reappear. -func (bm *Manager) DeleteBlock(blockID string) error { +func (bm *Manager) DeleteContent(contentID ID) error { bm.lock() defer bm.unlock() - log.Debugf("DeleteBlock(%q)", blockID) + log.Debugf("DeleteContent(%q)", contentID) - // We have this block in current pack index and it's already deleted there. - if bi, ok := bm.packIndexBuilder[blockID]; ok { + // We have this content in current pack index and it's already deleted there. + if bi, ok := bm.packIndexBuilder[contentID]; ok { if !bi.Deleted { if bi.PackBlobID == "" { // added and never committed, just forget about it. - delete(bm.packIndexBuilder, blockID) - delete(bm.currentPackItems, blockID) + delete(bm.packIndexBuilder, contentID) + delete(bm.currentPackItems, contentID) return nil } @@ -123,13 +123,13 @@ func (bm *Manager) DeleteBlock(blockID string) error { bi2 := *bi bi2.Deleted = true bi2.TimestampSeconds = bm.timeNow().Unix() - bm.setPendingBlock(bi2) + bm.setPendingContent(bi2) } return nil } - // We have this block in current pack index and it's already deleted there. - bi, err := bm.committedBlocks.getBlock(blockID) + // We have this content in current pack index and it's already deleted there. + bi, err := bm.committedContents.getContent(contentID) if err != nil { return err } @@ -143,23 +143,23 @@ func (bm *Manager) DeleteBlock(blockID string) error { bi2 := bi bi2.Deleted = true bi2.TimestampSeconds = bm.timeNow().Unix() - bm.setPendingBlock(bi2) + bm.setPendingContent(bi2) return nil } -func (bm *Manager) setPendingBlock(i Info) { +func (bm *Manager) setPendingContent(i Info) { bm.packIndexBuilder.Add(i) - bm.currentPackItems[i.BlockID] = i + bm.currentPackItems[i.ID] = i } -func (bm *Manager) addToPackLocked(ctx context.Context, blockID string, data []byte, isDeleted bool) error { +func (bm *Manager) addToPackLocked(ctx context.Context, contentID ID, data []byte, isDeleted bool) error { bm.assertLocked() data = cloneBytes(data) bm.currentPackDataLength += len(data) - bm.setPendingBlock(Info{ + bm.setPendingContent(Info{ Deleted: isDeleted, - BlockID: blockID, + ID: contentID, Payload: data, Length: uint32(len(data)), TimestampSeconds: bm.timeNow().Unix(), @@ -189,7 +189,7 @@ func (bm *Manager) finishPackAndMaybeFlushIndexesLocked(ctx context.Context) err return nil } -// Stats returns statistics about block manager operations. +// Stats returns statistics about content manager operations. func (bm *Manager) Stats() Stats { return bm.stats } @@ -225,29 +225,29 @@ func (bm *Manager) verifyInvariantsLocked() { func (bm *Manager) verifyCurrentPackItemsLocked() { for k, cpi := range bm.currentPackItems { - bm.assertInvariant(cpi.BlockID == k, "block ID entry has invalid key: %v %v", cpi.BlockID, k) - bm.assertInvariant(cpi.Deleted || cpi.PackBlobID == "", "block ID entry has unexpected pack block ID %v: %v", cpi.BlockID, cpi.PackBlobID) - bm.assertInvariant(cpi.TimestampSeconds != 0, "block has no timestamp: %v", cpi.BlockID) + bm.assertInvariant(cpi.ID == k, "content ID entry has invalid key: %v %v", cpi.ID, k) + bm.assertInvariant(cpi.Deleted || cpi.PackBlobID == "", "content ID entry has unexpected pack content ID %v: %v", cpi.ID, cpi.PackBlobID) + bm.assertInvariant(cpi.TimestampSeconds != 0, "content has no timestamp: %v", cpi.ID) bi, ok := bm.packIndexBuilder[k] - bm.assertInvariant(ok, "block ID entry not present in pack index builder: %v", cpi.BlockID) + bm.assertInvariant(ok, "content ID entry not present in pack index builder: %v", cpi.ID) bm.assertInvariant(reflect.DeepEqual(*bi, cpi), "current pack index does not match pack index builder: %v", cpi, *bi) } } func (bm *Manager) verifyPackIndexBuilderLocked() { for k, cpi := range bm.packIndexBuilder { - bm.assertInvariant(cpi.BlockID == k, "block ID entry has invalid key: %v %v", cpi.BlockID, k) - if _, ok := bm.currentPackItems[cpi.BlockID]; ok { - // ignore blocks also in currentPackItems + bm.assertInvariant(cpi.ID == k, "content ID entry has invalid key: %v %v", cpi.ID, k) + if _, ok := bm.currentPackItems[cpi.ID]; ok { + // ignore contents also in currentPackItems continue } if cpi.Deleted { - bm.assertInvariant(cpi.PackBlobID == "", "block can't be both deleted and have a pack block: %v", cpi.BlockID) + bm.assertInvariant(cpi.PackBlobID == "", "content can't be both deleted and have a pack content: %v", cpi.ID) } else { - bm.assertInvariant(cpi.PackBlobID != "", "block that's not deleted must have a pack block: %+v", cpi) - bm.assertInvariant(cpi.FormatVersion == byte(bm.writeFormatVersion), "block that's not deleted must have a valid format version: %+v", cpi) + bm.assertInvariant(cpi.PackBlobID != "", "content that's not deleted must have a pack content: %+v", cpi) + bm.assertInvariant(cpi.FormatVersion == byte(bm.writeFormatVersion), "content that's not deleted must have a valid format version: %+v", cpi) } - bm.assertInvariant(cpi.TimestampSeconds != 0, "block has no timestamp: %v", cpi.BlockID) + bm.assertInvariant(cpi.TimestampSeconds != 0, "content has no timestamp: %v", cpi.ID) } } @@ -264,7 +264,7 @@ func (bm *Manager) assertInvariant(ok bool, errorMsg string, arg ...interface{}) } func (bm *Manager) startPackIndexLocked() { - bm.currentPackItems = make(map[string]Info) + bm.currentPackItems = make(map[ID]Info) bm.currentPackDataLength = 0 } @@ -291,8 +291,8 @@ func (bm *Manager) flushPackIndexesLocked(ctx context.Context) error { return err } - if err := bm.committedBlocks.addBlock(indexBlobID, dataCopy, true); err != nil { - return errors.Wrap(err, "unable to add committed block") + if err := bm.committedContents.addContent(indexBlobID, dataCopy, true); err != nil { + return errors.Wrap(err, "unable to add committed content") } bm.packIndexBuilder = make(packIndexBuilder) } @@ -302,7 +302,7 @@ func (bm *Manager) flushPackIndexesLocked(ctx context.Context) error { } func (bm *Manager) writePackIndexesNew(ctx context.Context, data []byte) (blob.ID, error) { - return bm.encryptAndWriteBlockNotLocked(ctx, data, newIndexBlobPrefix) + return bm.encryptAndWriteContentNotLocked(ctx, data, newIndexBlobPrefix) } func (bm *Manager) finishPackLocked(ctx context.Context) error { @@ -311,36 +311,36 @@ func (bm *Manager) finishPackLocked(ctx context.Context) error { return nil } - if err := bm.writePackBlockLocked(ctx); err != nil { - return errors.Wrap(err, "error writing pack block") + if err := bm.writePackContentLocked(ctx); err != nil { + return errors.Wrap(err, "error writing pack content") } bm.startPackIndexLocked() return nil } -func (bm *Manager) writePackBlockLocked(ctx context.Context) error { +func (bm *Manager) writePackContentLocked(ctx context.Context) error { bm.assertLocked() - blockID := make([]byte, 16) - if _, err := cryptorand.Read(blockID); err != nil { + contentID := make([]byte, 16) + if _, err := cryptorand.Read(contentID); err != nil { return errors.Wrap(err, "unable to read crypto bytes") } - packFile := blob.ID(fmt.Sprintf("%v%x", PackBlobIDPrefix, blockID)) + packFile := blob.ID(fmt.Sprintf("%v%x", PackBlobIDPrefix, contentID)) - blockData, packFileIndex, err := bm.preparePackDataBlock(packFile) + contentData, packFileIndex, err := bm.preparePackDataContent(packFile) if err != nil { - return errors.Wrap(err, "error preparing data block") + return errors.Wrap(err, "error preparing data content") } - if len(blockData) > 0 { - if err := bm.writePackFileNotLocked(ctx, packFile, blockData); err != nil { - return errors.Wrap(err, "can't save pack data block") + if len(contentData) > 0 { + if err := bm.writePackFileNotLocked(ctx, packFile, contentData); err != nil { + return errors.Wrap(err, "can't save pack data content") } } - formatLog.Debugf("wrote pack file: %v (%v bytes)", packFile, len(blockData)) + formatLog.Debugf("wrote pack file: %v (%v bytes)", packFile, len(contentData)) for _, info := range packFileIndex { bm.packIndexBuilder.Add(*info) } @@ -348,39 +348,39 @@ func (bm *Manager) writePackBlockLocked(ctx context.Context) error { return nil } -func (bm *Manager) preparePackDataBlock(packFile blob.ID) ([]byte, packIndexBuilder, error) { - formatLog.Debugf("preparing block data with %v items", len(bm.currentPackItems)) +func (bm *Manager) preparePackDataContent(packFile blob.ID) ([]byte, packIndexBuilder, error) { + formatLog.Debugf("preparing content data with %v items", len(bm.currentPackItems)) - blockData, err := appendRandomBytes(append([]byte(nil), bm.repositoryFormatBytes...), rand.Intn(bm.maxPreambleLength-bm.minPreambleLength+1)+bm.minPreambleLength) + contentData, err := appendRandomBytes(append([]byte(nil), bm.repositoryFormatBytes...), rand.Intn(bm.maxPreambleLength-bm.minPreambleLength+1)+bm.minPreambleLength) if err != nil { - return nil, nil, errors.Wrap(err, "unable to prepare block preamble") + return nil, nil, errors.Wrap(err, "unable to prepare content preamble") } packFileIndex := packIndexBuilder{} - for blockID, info := range bm.currentPackItems { + for contentID, info := range bm.currentPackItems { if info.Payload == nil { continue } var encrypted []byte - encrypted, err = bm.maybeEncryptBlockDataForPacking(info.Payload, info.BlockID) + encrypted, err = bm.maybeEncryptContentDataForPacking(info.Payload, info.ID) if err != nil { - return nil, nil, errors.Wrapf(err, "unable to encrypt %q", blockID) + return nil, nil, errors.Wrapf(err, "unable to encrypt %q", contentID) } - formatLog.Debugf("adding %v length=%v deleted=%v", blockID, len(info.Payload), info.Deleted) + formatLog.Debugf("adding %v length=%v deleted=%v", contentID, len(info.Payload), info.Deleted) packFileIndex.Add(Info{ - BlockID: blockID, + ID: contentID, Deleted: info.Deleted, FormatVersion: byte(bm.writeFormatVersion), PackBlobID: packFile, - PackOffset: uint32(len(blockData)), + PackOffset: uint32(len(contentData)), Length: uint32(len(encrypted)), TimestampSeconds: info.TimestampSeconds, }) - blockData = append(blockData, encrypted...) + contentData = append(contentData, encrypted...) } if len(packFileIndex) == 0 { @@ -388,25 +388,25 @@ func (bm *Manager) preparePackDataBlock(packFile blob.ID) ([]byte, packIndexBuil } if bm.paddingUnit > 0 { - if missing := bm.paddingUnit - (len(blockData) % bm.paddingUnit); missing > 0 { - blockData, err = appendRandomBytes(blockData, missing) + if missing := bm.paddingUnit - (len(contentData) % bm.paddingUnit); missing > 0 { + contentData, err = appendRandomBytes(contentData, missing) if err != nil { - return nil, nil, errors.Wrap(err, "unable to prepare block postamble") + return nil, nil, errors.Wrap(err, "unable to prepare content postamble") } } } - origBlockLength := len(blockData) - blockData, err = bm.appendPackFileIndexRecoveryData(blockData, packFileIndex) + origContentLength := len(contentData) + contentData, err = bm.appendPackFileIndexRecoveryData(contentData, packFileIndex) - formatLog.Debugf("finished block %v bytes (%v bytes index)", len(blockData), len(blockData)-origBlockLength) - return blockData, packFileIndex, err + formatLog.Debugf("finished content %v bytes (%v bytes index)", len(contentData), len(contentData)-origContentLength) + return contentData, packFileIndex, err } -func (bm *Manager) maybeEncryptBlockDataForPacking(data []byte, blockID string) ([]byte, error) { - iv, err := getPackedBlockIV(blockID) +func (bm *Manager) maybeEncryptContentDataForPacking(data []byte, contentID ID) ([]byte, error) { + iv, err := getPackedContentIV(contentID) if err != nil { - return nil, errors.Wrapf(err, "unable to get packed block IV for %q", blockID) + return nil, errors.Wrapf(err, "unable to get packed content IV for %q", contentID) } return bm.encryptor.Encrypt(data, iv) @@ -441,23 +441,23 @@ func (bm *Manager) loadPackIndexesUnlocked(ctx context.Context) ([]IndexBlobInfo nextSleepTime *= 2 } - blocks, err := bm.listCache.listIndexBlobs(ctx) + contents, err := bm.listCache.listIndexBlobs(ctx) if err != nil { return nil, false, err } - err = bm.tryLoadPackIndexBlobsUnlocked(ctx, blocks) + err = bm.tryLoadPackIndexBlobsUnlocked(ctx, contents) if err == nil { - var blockIDs []blob.ID - for _, b := range blocks { - blockIDs = append(blockIDs, b.BlobID) + var contentIDs []blob.ID + for _, b := range contents { + contentIDs = append(contentIDs, b.BlobID) } var updated bool - updated, err = bm.committedBlocks.use(blockIDs) + updated, err = bm.committedContents.use(contentIDs) if err != nil { return nil, false, err } - return blocks, updated, nil + return contents, updated, nil } if err != blob.ErrBlobNotFound { return nil, false, err @@ -467,8 +467,8 @@ func (bm *Manager) loadPackIndexesUnlocked(ctx context.Context) ([]IndexBlobInfo return nil, false, errors.Errorf("unable to load pack indexes despite %v retries", indexLoadAttempts) } -func (bm *Manager) tryLoadPackIndexBlobsUnlocked(ctx context.Context, blocks []IndexBlobInfo) error { - ch, unprocessedIndexesSize, err := bm.unprocessedIndexBlobsUnlocked(blocks) +func (bm *Manager) tryLoadPackIndexBlobsUnlocked(ctx context.Context, contents []IndexBlobInfo) error { + ch, unprocessedIndexesSize, err := bm.unprocessedIndexBlobsUnlocked(contents) if err != nil { return err } @@ -493,8 +493,8 @@ func (bm *Manager) tryLoadPackIndexBlobsUnlocked(ctx context.Context, blocks []I return } - if err := bm.committedBlocks.addBlock(indexBlobID, data, false); err != nil { - errch <- errors.Wrap(err, "unable to add to committed block cache") + if err := bm.committedContents.addContent(indexBlobID, data, false); err != nil { + errch <- errors.Wrap(err, "unable to add to committed content cache") return } } @@ -508,52 +508,52 @@ func (bm *Manager) tryLoadPackIndexBlobsUnlocked(ctx context.Context, blocks []I for err := range errch { return err } - log.Infof("Index blocks downloaded.") + log.Infof("Index contents downloaded.") return nil } -// unprocessedIndexBlobsUnlocked returns a closed channel filled with block IDs that are not in committedBlocks cache. -func (bm *Manager) unprocessedIndexBlobsUnlocked(blocks []IndexBlobInfo) (<-chan blob.ID, int64, error) { +// unprocessedIndexBlobsUnlocked returns a closed channel filled with content IDs that are not in committedContents cache. +func (bm *Manager) unprocessedIndexBlobsUnlocked(contents []IndexBlobInfo) (<-chan blob.ID, int64, error) { var totalSize int64 - ch := make(chan blob.ID, len(blocks)) - for _, block := range blocks { - has, err := bm.committedBlocks.cache.hasIndexBlobID(block.BlobID) + ch := make(chan blob.ID, len(contents)) + for _, c := range contents { + has, err := bm.committedContents.cache.hasIndexBlobID(c.BlobID) if err != nil { return nil, 0, err } if has { - log.Debugf("index blob %q already in cache, skipping", block.BlobID) + log.Debugf("index blob %q already in cache, skipping", c.BlobID) continue } - ch <- block.BlobID - totalSize += block.Length + ch <- c.BlobID + totalSize += c.Length } close(ch) return ch, totalSize, nil } -// Close closes the block manager. +// Close closes the content manager. func (bm *Manager) Close() { - bm.blockCache.close() + bm.contentCache.close() close(bm.closed) } -// ListBlocks returns IDs of blocks matching given prefix. -func (bm *Manager) ListBlocks(prefix string) ([]string, error) { +// ListContents returns IDs of contents matching given prefix. +func (bm *Manager) ListContents(prefix ID) ([]ID, error) { bm.lock() defer bm.unlock() - var result []string + var result []ID appendToResult := func(i Info) error { - if i.Deleted || !strings.HasPrefix(i.BlockID, prefix) { + if i.Deleted || !strings.HasPrefix(string(i.ID), string(prefix)) { return nil } - if bi, ok := bm.packIndexBuilder[i.BlockID]; ok && bi.Deleted { + if bi, ok := bm.packIndexBuilder[i.ID]; ok && bi.Deleted { return nil } - result = append(result, i.BlockID) + result = append(result, i.ID) return nil } @@ -561,22 +561,22 @@ func (bm *Manager) ListBlocks(prefix string) ([]string, error) { _ = appendToResult(*bi) } - _ = bm.committedBlocks.listBlocks(prefix, appendToResult) + _ = bm.committedContents.listContents(prefix, appendToResult) return result, nil } -// ListBlockInfos returns the metadata about blocks with a given prefix and kind. -func (bm *Manager) ListBlockInfos(prefix string, includeDeleted bool) ([]Info, error) { +// ListContentInfos returns the metadata about contents with a given prefix and kind. +func (bm *Manager) ListContentInfos(prefix ID, includeDeleted bool) ([]Info, error) { bm.lock() defer bm.unlock() var result []Info appendToResult := func(i Info) error { - if (i.Deleted && !includeDeleted) || !strings.HasPrefix(i.BlockID, prefix) { + if (i.Deleted && !includeDeleted) || !strings.HasPrefix(string(i.ID), string(prefix)) { return nil } - if bi, ok := bm.packIndexBuilder[i.BlockID]; ok && bi.Deleted { + if bi, ok := bm.packIndexBuilder[i.ID]; ok && bi.Deleted { return nil } result = append(result, i) @@ -587,7 +587,7 @@ func (bm *Manager) ListBlockInfos(prefix string, includeDeleted bool) ([]Info, e _ = appendToResult(*bi) } - _ = bm.committedBlocks.listBlocks(prefix, appendToResult) + _ = bm.committedContents.listContents(prefix, appendToResult) return result, nil } @@ -598,7 +598,7 @@ func (bm *Manager) Flush(ctx context.Context) error { defer bm.unlock() if err := bm.finishPackLocked(ctx); err != nil { - return errors.Wrap(err, "error writing pending block") + return errors.Wrap(err, "error writing pending content") } if err := bm.flushPackIndexesLocked(ctx); err != nil { @@ -608,46 +608,46 @@ func (bm *Manager) Flush(ctx context.Context) error { return nil } -// RewriteBlock causes reads and re-writes a given block using the most recent format. -func (bm *Manager) RewriteBlock(ctx context.Context, blockID string) error { - bi, err := bm.getBlockInfo(blockID) +// RewriteContent causes reads and re-writes a given content using the most recent format. +func (bm *Manager) RewriteContent(ctx context.Context, contentID ID) error { + bi, err := bm.getContentInfo(contentID) if err != nil { return err } - data, err := bm.getBlockContentsUnlocked(ctx, bi) + data, err := bm.getContentContentsUnlocked(ctx, bi) if err != nil { return err } bm.lock() defer bm.unlock() - return bm.addToPackLocked(ctx, blockID, data, bi.Deleted) + return bm.addToPackLocked(ctx, contentID, data, bi.Deleted) } -// WriteBlock saves a given block of data to a pack group with a provided name and returns a blockID +// WriteContent saves a given content of data to a pack group with a provided name and returns a contentID // that's based on the contents of data written. -func (bm *Manager) WriteBlock(ctx context.Context, data []byte, prefix string) (string, error) { +func (bm *Manager) WriteContent(ctx context.Context, data []byte, prefix ID) (ID, error) { if err := validatePrefix(prefix); err != nil { return "", err } - blockID := prefix + hex.EncodeToString(bm.hashData(data)) + contentID := prefix + ID(hex.EncodeToString(bm.hashData(data))) - // block already tracked - if bi, err := bm.getBlockInfo(blockID); err == nil { + // content already tracked + if bi, err := bm.getContentInfo(contentID); err == nil { if !bi.Deleted { - return blockID, nil + return contentID, nil } } - log.Debugf("WriteBlock(%q) - new", blockID) + log.Debugf("WriteContent(%q) - new", contentID) bm.lock() defer bm.unlock() - err := bm.addToPackLocked(ctx, blockID, data, false) - return blockID, err + err := bm.addToPackLocked(ctx, contentID, data, false) + return contentID, err } -func validatePrefix(prefix string) error { +func validatePrefix(prefix ID) error { switch len(prefix) { case 0: return nil @@ -661,24 +661,24 @@ func validatePrefix(prefix string) error { } func (bm *Manager) writePackFileNotLocked(ctx context.Context, packFile blob.ID, data []byte) error { - atomic.AddInt32(&bm.stats.WrittenBlocks, 1) + atomic.AddInt32(&bm.stats.WrittenContents, 1) atomic.AddInt64(&bm.stats.WrittenBytes, int64(len(data))) bm.listCache.deleteListCache(ctx) return bm.st.PutBlob(ctx, packFile, data) } -func (bm *Manager) encryptAndWriteBlockNotLocked(ctx context.Context, data []byte, prefix blob.ID) (blob.ID, error) { +func (bm *Manager) encryptAndWriteContentNotLocked(ctx context.Context, data []byte, prefix blob.ID) (blob.ID, error) { hash := bm.hashData(data) blobID := prefix + blob.ID(hex.EncodeToString(hash)) - // Encrypt the block in-place. + // Encrypt the content in-place. atomic.AddInt64(&bm.stats.EncryptedBytes, int64(len(data))) data2, err := bm.encryptor.Encrypt(data, hash) if err != nil { return "", err } - atomic.AddInt32(&bm.stats.WrittenBlocks, 1) + atomic.AddInt32(&bm.stats.WrittenContents, 1) atomic.AddInt64(&bm.stats.WrittenBytes, int64(len(data2))) bm.listCache.deleteListCache(ctx) if err := bm.st.PutBlob(ctx, blobID, data2); err != nil { @@ -689,80 +689,80 @@ func (bm *Manager) encryptAndWriteBlockNotLocked(ctx context.Context, data []byt } func (bm *Manager) hashData(data []byte) []byte { - // Hash the block and compute encryption key. - blockID := bm.hasher(data) - atomic.AddInt32(&bm.stats.HashedBlocks, 1) + // Hash the content and compute encryption key. + contentID := bm.hasher(data) + atomic.AddInt32(&bm.stats.HashedContents, 1) atomic.AddInt64(&bm.stats.HashedBytes, int64(len(data))) - return blockID + return contentID } func cloneBytes(b []byte) []byte { return append([]byte{}, b...) } -// GetBlock gets the contents of a given block. If the block is not found returns blob.ErrBlobNotFound. -func (bm *Manager) GetBlock(ctx context.Context, blockID string) ([]byte, error) { - bi, err := bm.getBlockInfo(blockID) +// GetContent gets the contents of a given content. If the content is not found returns ErrContentNotFound. +func (bm *Manager) GetContent(ctx context.Context, contentID ID) ([]byte, error) { + bi, err := bm.getContentInfo(contentID) if err != nil { return nil, err } if bi.Deleted { - return nil, ErrBlockNotFound + return nil, ErrContentNotFound } - return bm.getBlockContentsUnlocked(ctx, bi) + return bm.getContentContentsUnlocked(ctx, bi) } -func (bm *Manager) getBlockInfo(blockID string) (Info, error) { +func (bm *Manager) getContentInfo(contentID ID) (Info, error) { bm.lock() defer bm.unlock() - // check added blocks, not written to any packs. - if bi, ok := bm.currentPackItems[blockID]; ok { + // check added contents, not written to any packs. + if bi, ok := bm.currentPackItems[contentID]; ok { return bi, nil } - // added blocks, written to packs but not yet added to indexes - if bi, ok := bm.packIndexBuilder[blockID]; ok { + // added contents, written to packs but not yet added to indexes + if bi, ok := bm.packIndexBuilder[contentID]; ok { return *bi, nil } - // read from committed block index - return bm.committedBlocks.getBlock(blockID) + // read from committed content index + return bm.committedContents.getContent(contentID) } -// BlockInfo returns information about a single block. -func (bm *Manager) BlockInfo(ctx context.Context, blockID string) (Info, error) { - bi, err := bm.getBlockInfo(blockID) +// ContentInfo returns information about a single content. +func (bm *Manager) ContentInfo(ctx context.Context, contentID ID) (Info, error) { + bi, err := bm.getContentInfo(contentID) if err != nil { - log.Debugf("BlockInfo(%q) - error %v", err) + log.Debugf("ContentInfo(%q) - error %v", err) return Info{}, err } if bi.Deleted { - log.Debugf("BlockInfo(%q) - deleted", blockID) + log.Debugf("ContentInfo(%q) - deleted", contentID) } else { - log.Debugf("BlockInfo(%q) - exists in %v", blockID, bi.PackBlobID) + log.Debugf("ContentInfo(%q) - exists in %v", contentID, bi.PackBlobID) } return bi, err } -// FindUnreferencedBlobs returns the list of unreferenced storage blocks. +// FindUnreferencedBlobs returns the list of unreferenced storage contents. func (bm *Manager) FindUnreferencedBlobs(ctx context.Context) ([]blob.Metadata, error) { - infos, err := bm.ListBlockInfos("", true) + infos, err := bm.ListContentInfos("", true) if err != nil { return nil, errors.Wrap(err, "unable to list index blobs") } - usedPackBlocks := findPackBlocksInUse(infos) + usedPackContents := findPackContentsInUse(infos) var unused []blob.Metadata err = bm.st.ListBlobs(ctx, PackBlobIDPrefix, func(bi blob.Metadata) error { - u := usedPackBlocks[bi.BlobID] + u := usedPackContents[bi.BlobID] if u > 0 { - log.Debugf("pack %v, in use by %v blocks", bi.BlobID, u) + log.Debugf("pack %v, in use by %v contents", bi.BlobID, u) return nil } @@ -770,13 +770,13 @@ func (bm *Manager) FindUnreferencedBlobs(ctx context.Context) ([]blob.Metadata, return nil }) if err != nil { - return nil, errors.Wrap(err, "error listing storage blocks") + return nil, errors.Wrap(err, "error listing storage contents") } return unused, nil } -func findPackBlocksInUse(infos []Info) map[blob.ID]int { +func findPackContentsInUse(infos []Info) map[blob.ID]int { packUsage := map[blob.ID]int{} for _, bi := range infos { @@ -786,20 +786,20 @@ func findPackBlocksInUse(infos []Info) map[blob.ID]int { return packUsage } -func (bm *Manager) getBlockContentsUnlocked(ctx context.Context, bi Info) ([]byte, error) { +func (bm *Manager) getContentContentsUnlocked(ctx context.Context, bi Info) ([]byte, error) { if bi.Payload != nil { return cloneBytes(bi.Payload), nil } - payload, err := bm.blockCache.getContentBlock(ctx, blob.ID(bi.BlockID), bi.PackBlobID, int64(bi.PackOffset), int64(bi.Length)) + payload, err := bm.contentCache.getContentContent(ctx, blob.ID(bi.ID), bi.PackBlobID, int64(bi.PackOffset), int64(bi.Length)) if err != nil { return nil, err } - atomic.AddInt32(&bm.stats.ReadBlocks, 1) + atomic.AddInt32(&bm.stats.ReadContents, 1) atomic.AddInt64(&bm.stats.ReadBytes, int64(len(payload))) - iv, err := getPackedBlockIV(bi.BlockID) + iv, err := getPackedContentIV(bi.ID) if err != nil { return nil, err } @@ -831,7 +831,7 @@ func (bm *Manager) decryptAndVerify(encrypted []byte, iv []byte) ([]byte, error) } func (bm *Manager) getIndexBlobInternal(ctx context.Context, blobID blob.ID) ([]byte, error) { - payload, err := bm.blockCache.getContentBlock(ctx, blobID, blobID, 0, -1) + payload, err := bm.contentCache.getContentContent(ctx, blobID, blobID, 0, -1) if err != nil { return nil, err } @@ -841,7 +841,7 @@ func (bm *Manager) getIndexBlobInternal(ctx context.Context, blobID blob.ID) ([] return nil, err } - atomic.AddInt32(&bm.stats.ReadBlocks, 1) + atomic.AddInt32(&bm.stats.ReadContents, 1) atomic.AddInt64(&bm.stats.ReadBytes, int64(len(payload))) payload, err = bm.encryptor.Decrypt(payload, iv) @@ -859,8 +859,8 @@ func (bm *Manager) getIndexBlobInternal(ctx context.Context, blobID blob.ID) ([] return payload, nil } -func getPackedBlockIV(blockID string) ([]byte, error) { - return hex.DecodeString(blockID[len(blockID)-(aes.BlockSize*2):]) +func getPackedContentIV(contentID ID) ([]byte, error) { + return hex.DecodeString(string(contentID[len(contentID)-(aes.BlockSize*2):])) } func getIndexBlobIV(s blob.ID) ([]byte, error) { @@ -870,15 +870,15 @@ func getIndexBlobIV(s blob.ID) ([]byte, error) { return hex.DecodeString(string(s[len(s)-(aes.BlockSize*2):])) } -func (bm *Manager) verifyChecksum(data []byte, blockID []byte) error { +func (bm *Manager) verifyChecksum(data []byte, contentID []byte) error { expected := bm.hasher(data) expected = expected[len(expected)-aes.BlockSize:] - if !bytes.HasSuffix(blockID, expected) { - atomic.AddInt32(&bm.stats.InvalidBlocks, 1) - return errors.Errorf("invalid checksum for blob %x, expected %x", blockID, expected) + if !bytes.HasSuffix(contentID, expected) { + atomic.AddInt32(&bm.stats.InvalidContents, 1) + return errors.Errorf("invalid checksum for blob %x, expected %x", contentID, expected) } - atomic.AddInt32(&bm.stats.ValidBlocks, 1) + atomic.AddInt32(&bm.stats.ValidContents, 1) return nil } @@ -902,7 +902,7 @@ func (bm *Manager) assertLocked() { } } -// Refresh reloads the committed block indexes. +// Refresh reloads the committed content indexes. func (bm *Manager) Refresh(ctx context.Context) (bool, error) { bm.mu.Lock() defer bm.mu.Unlock() @@ -916,11 +916,11 @@ func (bm *Manager) Refresh(ctx context.Context) (bool, error) { type cachedList struct { Timestamp time.Time `json:"timestamp"` - Blocks []IndexBlobInfo `json:"blocks"` + Contents []IndexBlobInfo `json:"contents"` } // listIndexBlobsFromStorage returns the list of index blobs in the given storage. -// The list of blocks is not guaranteed to be sorted. +// The list of contents is not guaranteed to be sorted. func listIndexBlobsFromStorage(ctx context.Context, st blob.Storage) ([]IndexBlobInfo, error) { snapshot, err := blob.ListAllBlobsConsistent(ctx, st, newIndexBlobPrefix, math.MaxInt32) if err != nil { @@ -940,7 +940,7 @@ func listIndexBlobsFromStorage(ctx context.Context, st blob.Storage) ([]IndexBlo return results, err } -// NewManager creates new block manager with given packing options and a formatter. +// NewManager creates new content manager with given packing options and a formatter. func NewManager(ctx context.Context, st blob.Storage, f FormattingOptions, caching CachingOptions, repositoryFormatBytes []byte) (*Manager, error) { return newManagerWithOptions(ctx, st, f, caching, time.Now, repositoryFormatBytes) } @@ -959,9 +959,9 @@ func newManagerWithOptions(ctx context.Context, st blob.Storage, f FormattingOpt return nil, err } - blockCache, err := newBlockCache(ctx, st, caching) + contentCache, err := newContentCache(ctx, st, caching) if err != nil { - return nil, errors.Wrap(err, "unable to initialize block cache") + return nil, errors.Wrap(err, "unable to initialize content cache") } listCache, err := newListCache(ctx, st, caching) @@ -969,9 +969,9 @@ func newManagerWithOptions(ctx context.Context, st blob.Storage, f FormattingOpt return nil, errors.Wrap(err, "unable to initialize list cache") } - blockIndex, err := newCommittedBlockIndex(caching) + contentIndex, err := newCommittedContentIndex(caching) if err != nil { - return nil, errors.Wrap(err, "unable to initialize committed block index") + return nil, errors.Wrap(err, "unable to initialize committed content index") } m := &Manager{ @@ -981,13 +981,13 @@ func newManagerWithOptions(ctx context.Context, st blob.Storage, f FormattingOpt maxPackSize: f.MaxPackSize, encryptor: encryptor, hasher: hasher, - currentPackItems: make(map[string]Info), + currentPackItems: make(map[ID]Info), packIndexBuilder: make(packIndexBuilder), - committedBlocks: blockIndex, + committedContents: contentIndex, minPreambleLength: defaultMinPreambleLength, maxPreambleLength: defaultMaxPreambleLength, paddingUnit: defaultPaddingUnit, - blockCache: blockCache, + contentCache: contentCache, listCache: listCache, st: st, repositoryFormatBytes: repositoryFormatBytes, @@ -1000,7 +1000,7 @@ func newManagerWithOptions(ctx context.Context, st blob.Storage, f FormattingOpt m.startPackIndexLocked() if err := m.CompactIndexes(ctx, autoCompactionOptions); err != nil { - return nil, errors.Wrap(err, "error initializing block manager") + return nil, errors.Wrap(err, "error initializing content manager") } return m, nil @@ -1017,8 +1017,8 @@ func CreateHashAndEncryptor(f FormattingOptions) (HashFunc, Encryptor, error) { return nil, nil, errors.Wrap(err, "unable to create encryptor") } - blockID := h(nil) - _, err = e.Encrypt(nil, blockID) + contentID := h(nil) + _, err = e.Encrypt(nil, contentID) if err != nil { return nil, nil, errors.Wrap(err, "invalid encryptor") } diff --git a/repo/content/content_manager_test.go b/repo/content/content_manager_test.go new file mode 100644 index 000000000..4e3f1358a --- /dev/null +++ b/repo/content/content_manager_test.go @@ -0,0 +1,911 @@ +package content + +import ( + "bytes" + "context" + "crypto/hmac" + "crypto/sha256" + "encoding/hex" + "errors" + "fmt" + "math/rand" + "reflect" + "strings" + "sync" + "testing" + "time" + + logging "github.com/op/go-logging" + + "github.com/kopia/kopia/internal/blobtesting" + "github.com/kopia/kopia/repo/blob" +) + +const ( + maxPackSize = 2000 +) + +var fakeTime = time.Date(2017, 1, 1, 0, 0, 0, 0, time.UTC) +var hmacSecret = []byte{1, 2, 3} + +func init() { + logging.SetLevel(logging.DEBUG, "") +} + +func TestContentManagerEmptyFlush(t *testing.T) { + ctx := context.Background() + data := blobtesting.DataMap{} + keyTime := map[blob.ID]time.Time{} + bm := newTestContentManager(data, keyTime, nil) + bm.Flush(ctx) + if got, want := len(data), 0; got != want { + t.Errorf("unexpected number of contents: %v, wanted %v", got, want) + } +} + +func TestContentZeroBytes1(t *testing.T) { + ctx := context.Background() + data := blobtesting.DataMap{} + keyTime := map[blob.ID]time.Time{} + bm := newTestContentManager(data, keyTime, nil) + contentID := writeContentAndVerify(ctx, t, bm, []byte{}) + bm.Flush(ctx) + if got, want := len(data), 2; got != want { + t.Errorf("unexpected number of contents: %v, wanted %v", got, want) + } + dumpContentManagerData(t, data) + bm = newTestContentManager(data, keyTime, nil) + verifyContent(ctx, t, bm, contentID, []byte{}) +} + +func TestContentZeroBytes2(t *testing.T) { + ctx := context.Background() + data := blobtesting.DataMap{} + keyTime := map[blob.ID]time.Time{} + bm := newTestContentManager(data, keyTime, nil) + writeContentAndVerify(ctx, t, bm, seededRandomData(10, 10)) + writeContentAndVerify(ctx, t, bm, []byte{}) + bm.Flush(ctx) + if got, want := len(data), 2; got != want { + t.Errorf("unexpected number of contents: %v, wanted %v", got, want) + dumpContentManagerData(t, data) + } +} + +func TestContentManagerSmallContentWrites(t *testing.T) { + ctx := context.Background() + data := blobtesting.DataMap{} + keyTime := map[blob.ID]time.Time{} + bm := newTestContentManager(data, keyTime, nil) + + for i := 0; i < 100; i++ { + writeContentAndVerify(ctx, t, bm, seededRandomData(i, 10)) + } + if got, want := len(data), 0; got != want { + t.Errorf("unexpected number of contents: %v, wanted %v", got, want) + } + bm.Flush(ctx) + if got, want := len(data), 2; got != want { + t.Errorf("unexpected number of contents: %v, wanted %v", got, want) + } +} + +func TestContentManagerDedupesPendingContents(t *testing.T) { + ctx := context.Background() + data := blobtesting.DataMap{} + keyTime := map[blob.ID]time.Time{} + bm := newTestContentManager(data, keyTime, nil) + + for i := 0; i < 100; i++ { + writeContentAndVerify(ctx, t, bm, seededRandomData(0, 999)) + } + if got, want := len(data), 0; got != want { + t.Errorf("unexpected number of contents: %v, wanted %v", got, want) + } + bm.Flush(ctx) + if got, want := len(data), 2; got != want { + t.Errorf("unexpected number of contents: %v, wanted %v", got, want) + } +} + +func TestContentManagerDedupesPendingAndUncommittedContents(t *testing.T) { + ctx := context.Background() + data := blobtesting.DataMap{} + keyTime := map[blob.ID]time.Time{} + bm := newTestContentManager(data, keyTime, nil) + + // no writes here, all data fits in a single pack. + writeContentAndVerify(ctx, t, bm, seededRandomData(0, 950)) + writeContentAndVerify(ctx, t, bm, seededRandomData(1, 950)) + writeContentAndVerify(ctx, t, bm, seededRandomData(2, 10)) + if got, want := len(data), 0; got != want { + t.Errorf("unexpected number of contents: %v, wanted %v", got, want) + } + + // no writes here + writeContentAndVerify(ctx, t, bm, seededRandomData(0, 950)) + writeContentAndVerify(ctx, t, bm, seededRandomData(1, 950)) + writeContentAndVerify(ctx, t, bm, seededRandomData(2, 10)) + if got, want := len(data), 0; got != want { + t.Errorf("unexpected number of contents: %v, wanted %v", got, want) + } + bm.Flush(ctx) + + // this flushes the pack content + index blob + if got, want := len(data), 2; got != want { + dumpContentManagerData(t, data) + t.Errorf("unexpected number of contents: %v, wanted %v", got, want) + } +} + +func TestContentManagerEmpty(t *testing.T) { + ctx := context.Background() + data := blobtesting.DataMap{} + keyTime := map[blob.ID]time.Time{} + bm := newTestContentManager(data, keyTime, nil) + + noSuchContentID := ID(hashValue([]byte("foo"))) + + b, err := bm.GetContent(ctx, noSuchContentID) + if err != ErrContentNotFound { + t.Errorf("unexpected error when getting non-existent content: %v, %v", b, err) + } + + bi, err := bm.ContentInfo(ctx, noSuchContentID) + if err != ErrContentNotFound { + t.Errorf("unexpected error when getting non-existent content info: %v, %v", bi, err) + } + + if got, want := len(data), 0; got != want { + t.Errorf("unexpected number of contents: %v, wanted %v", got, want) + } +} + +func verifyActiveIndexBlobCount(ctx context.Context, t *testing.T, bm *Manager, expected int) { + t.Helper() + + blks, err := bm.IndexBlobs(ctx) + if err != nil { + t.Errorf("error listing active index blobs: %v", err) + return + } + + if got, want := len(blks), expected; got != want { + t.Errorf("unexpected number of active index blobs %v, expected %v (%v)", got, want, blks) + } +} +func TestContentManagerInternalFlush(t *testing.T) { + ctx := context.Background() + data := blobtesting.DataMap{} + keyTime := map[blob.ID]time.Time{} + bm := newTestContentManager(data, keyTime, nil) + + for i := 0; i < 100; i++ { + b := make([]byte, 25) + rand.Read(b) + writeContentAndVerify(ctx, t, bm, b) + } + + // 1 data content written, but no index yet. + if got, want := len(data), 1; got != want { + t.Errorf("unexpected number of contents: %v, wanted %v", got, want) + } + + // do it again - should be 2 contents + 1000 bytes pending. + for i := 0; i < 100; i++ { + b := make([]byte, 25) + rand.Read(b) + writeContentAndVerify(ctx, t, bm, b) + } + + // 2 data contents written, but no index yet. + if got, want := len(data), 2; got != want { + t.Errorf("unexpected number of contents: %v, wanted %v", got, want) + } + + bm.Flush(ctx) + + // third content gets written, followed by index. + if got, want := len(data), 4; got != want { + dumpContentManagerData(t, data) + t.Errorf("unexpected number of contents: %v, wanted %v", got, want) + } +} + +func TestContentManagerWriteMultiple(t *testing.T) { + ctx := context.Background() + data := blobtesting.DataMap{} + keyTime := map[blob.ID]time.Time{} + timeFunc := fakeTimeNowWithAutoAdvance(fakeTime, 1*time.Second) + bm := newTestContentManager(data, keyTime, timeFunc) + + var contentIDs []ID + + for i := 0; i < 5000; i++ { + //t.Logf("i=%v", i) + b := seededRandomData(i, i%113) + blkID, err := bm.WriteContent(ctx, b, "") + if err != nil { + t.Errorf("err: %v", err) + } + + contentIDs = append(contentIDs, blkID) + + if i%17 == 0 { + //t.Logf("flushing %v", i) + if err := bm.Flush(ctx); err != nil { + t.Fatalf("error flushing: %v", err) + } + //dumpContentManagerData(t, data) + } + + if i%41 == 0 { + //t.Logf("opening new manager: %v", i) + if err := bm.Flush(ctx); err != nil { + t.Fatalf("error flushing: %v", err) + } + //t.Logf("data content count: %v", len(data)) + //dumpContentManagerData(t, data) + bm = newTestContentManager(data, keyTime, timeFunc) + } + + pos := rand.Intn(len(contentIDs)) + if _, err := bm.GetContent(ctx, contentIDs[pos]); err != nil { + dumpContentManagerData(t, data) + t.Fatalf("can't read content %q: %v", contentIDs[pos], err) + continue + } + } +} + +// This is regression test for a bug where we would corrupt data when encryption +// was done in place and clobbered pending data in memory. +func TestContentManagerFailedToWritePack(t *testing.T) { + ctx := context.Background() + data := blobtesting.DataMap{} + keyTime := map[blob.ID]time.Time{} + st := blobtesting.NewMapStorage(data, keyTime, nil) + faulty := &blobtesting.FaultyStorage{ + Base: st, + } + st = faulty + + bm, err := newManagerWithOptions(context.Background(), st, FormattingOptions{ + Version: 1, + Hash: "HMAC-SHA256-128", + Encryption: "AES-256-CTR", + MaxPackSize: maxPackSize, + HMACSecret: []byte("foo"), + MasterKey: []byte("0123456789abcdef0123456789abcdef"), + }, CachingOptions{}, fakeTimeNowFrozen(fakeTime), nil) + if err != nil { + t.Fatalf("can't create bm: %v", err) + } + logging.SetLevel(logging.DEBUG, "faulty-storage") + + faulty.Faults = map[string][]*blobtesting.Fault{ + "PutContent": { + {Err: errors.New("booboo")}, + }, + } + + b1, err := bm.WriteContent(ctx, seededRandomData(1, 10), "") + if err != nil { + t.Fatalf("can't create content: %v", err) + } + + if err := bm.Flush(ctx); err != nil { + t.Logf("expected flush error: %v", err) + } + + verifyContent(ctx, t, bm, b1, seededRandomData(1, 10)) +} + +func TestContentManagerConcurrency(t *testing.T) { + ctx := context.Background() + data := blobtesting.DataMap{} + keyTime := map[blob.ID]time.Time{} + bm := newTestContentManager(data, keyTime, nil) + preexistingContent := writeContentAndVerify(ctx, t, bm, seededRandomData(10, 100)) + bm.Flush(ctx) + + dumpContentManagerData(t, data) + bm1 := newTestContentManager(data, keyTime, nil) + bm2 := newTestContentManager(data, keyTime, nil) + bm3 := newTestContentManager(data, keyTime, fakeTimeNowWithAutoAdvance(fakeTime.Add(1), 1*time.Second)) + + // all bm* can see pre-existing content + verifyContent(ctx, t, bm1, preexistingContent, seededRandomData(10, 100)) + verifyContent(ctx, t, bm2, preexistingContent, seededRandomData(10, 100)) + verifyContent(ctx, t, bm3, preexistingContent, seededRandomData(10, 100)) + + // write the same content in all managers. + sharedContent := writeContentAndVerify(ctx, t, bm1, seededRandomData(20, 100)) + writeContentAndVerify(ctx, t, bm2, seededRandomData(20, 100)) + writeContentAndVerify(ctx, t, bm3, seededRandomData(20, 100)) + + // write unique content per manager. + bm1content := writeContentAndVerify(ctx, t, bm1, seededRandomData(31, 100)) + bm2content := writeContentAndVerify(ctx, t, bm2, seededRandomData(32, 100)) + bm3content := writeContentAndVerify(ctx, t, bm3, seededRandomData(33, 100)) + + // make sure they can't see each other's unflushed contents. + verifyContentNotFound(ctx, t, bm1, bm2content) + verifyContentNotFound(ctx, t, bm1, bm3content) + verifyContentNotFound(ctx, t, bm2, bm1content) + verifyContentNotFound(ctx, t, bm2, bm3content) + verifyContentNotFound(ctx, t, bm3, bm1content) + verifyContentNotFound(ctx, t, bm3, bm2content) + + // now flush all writers, they still can't see each others' data. + bm1.Flush(ctx) + bm2.Flush(ctx) + bm3.Flush(ctx) + verifyContentNotFound(ctx, t, bm1, bm2content) + verifyContentNotFound(ctx, t, bm1, bm3content) + verifyContentNotFound(ctx, t, bm2, bm1content) + verifyContentNotFound(ctx, t, bm2, bm3content) + verifyContentNotFound(ctx, t, bm3, bm1content) + verifyContentNotFound(ctx, t, bm3, bm2content) + + // new content manager at this point can see all data. + bm4 := newTestContentManager(data, keyTime, nil) + verifyContent(ctx, t, bm4, preexistingContent, seededRandomData(10, 100)) + verifyContent(ctx, t, bm4, sharedContent, seededRandomData(20, 100)) + verifyContent(ctx, t, bm4, bm1content, seededRandomData(31, 100)) + verifyContent(ctx, t, bm4, bm2content, seededRandomData(32, 100)) + verifyContent(ctx, t, bm4, bm3content, seededRandomData(33, 100)) + + if got, want := getIndexCount(data), 4; got != want { + t.Errorf("unexpected index count before compaction: %v, wanted %v", got, want) + } + + if err := bm4.CompactIndexes(ctx, CompactOptions{ + MinSmallBlobs: 1, + MaxSmallBlobs: 1, + }); err != nil { + t.Errorf("compaction error: %v", err) + } + if got, want := getIndexCount(data), 1; got != want { + t.Errorf("unexpected index count after compaction: %v, wanted %v", got, want) + } + + // new content manager at this point can see all data. + bm5 := newTestContentManager(data, keyTime, nil) + verifyContent(ctx, t, bm5, preexistingContent, seededRandomData(10, 100)) + verifyContent(ctx, t, bm5, sharedContent, seededRandomData(20, 100)) + verifyContent(ctx, t, bm5, bm1content, seededRandomData(31, 100)) + verifyContent(ctx, t, bm5, bm2content, seededRandomData(32, 100)) + verifyContent(ctx, t, bm5, bm3content, seededRandomData(33, 100)) + if err := bm5.CompactIndexes(ctx, CompactOptions{ + MinSmallBlobs: 1, + MaxSmallBlobs: 1, + }); err != nil { + t.Errorf("compaction error: %v", err) + } +} + +func TestDeleteContent(t *testing.T) { + ctx := context.Background() + data := blobtesting.DataMap{} + keyTime := map[blob.ID]time.Time{} + bm := newTestContentManager(data, keyTime, nil) + content1 := writeContentAndVerify(ctx, t, bm, seededRandomData(10, 100)) + bm.Flush(ctx) + content2 := writeContentAndVerify(ctx, t, bm, seededRandomData(11, 100)) + if err := bm.DeleteContent(content1); err != nil { + t.Errorf("unable to delete content: %v", content1) + } + if err := bm.DeleteContent(content2); err != nil { + t.Errorf("unable to delete content: %v", content1) + } + verifyContentNotFound(ctx, t, bm, content1) + verifyContentNotFound(ctx, t, bm, content2) + bm.Flush(ctx) + log.Debugf("-----------") + bm = newTestContentManager(data, keyTime, nil) + //dumpContentManagerData(t, data) + verifyContentNotFound(ctx, t, bm, content1) + verifyContentNotFound(ctx, t, bm, content2) +} + +func TestRewriteNonDeleted(t *testing.T) { + const stepBehaviors = 3 + + // perform a sequence WriteContent() RewriteContent() GetContent() + // where actionX can be (0=flush and reopen, 1=flush, 2=nothing) + for action1 := 0; action1 < stepBehaviors; action1++ { + for action2 := 0; action2 < stepBehaviors; action2++ { + t.Run(fmt.Sprintf("case-%v-%v", action1, action2), func(t *testing.T) { + ctx := context.Background() + data := blobtesting.DataMap{} + keyTime := map[blob.ID]time.Time{} + fakeNow := fakeTimeNowWithAutoAdvance(fakeTime, 1*time.Second) + bm := newTestContentManager(data, keyTime, fakeNow) + + applyStep := func(action int) { + switch action { + case 0: + t.Logf("flushing and reopening") + bm.Flush(ctx) + bm = newTestContentManager(data, keyTime, fakeNow) + case 1: + t.Logf("flushing") + bm.Flush(ctx) + case 2: + t.Logf("doing nothing") + } + } + + content1 := writeContentAndVerify(ctx, t, bm, seededRandomData(10, 100)) + applyStep(action1) + assertNoError(t, bm.RewriteContent(ctx, content1)) + applyStep(action2) + verifyContent(ctx, t, bm, content1, seededRandomData(10, 100)) + dumpContentManagerData(t, data) + }) + } + } +} + +func TestDisableFlush(t *testing.T) { + ctx := context.Background() + data := blobtesting.DataMap{} + keyTime := map[blob.ID]time.Time{} + bm := newTestContentManager(data, keyTime, nil) + bm.DisableIndexFlush() + bm.DisableIndexFlush() + for i := 0; i < 500; i++ { + writeContentAndVerify(ctx, t, bm, seededRandomData(i, 100)) + } + bm.Flush(ctx) // flush will not have effect + bm.EnableIndexFlush() + bm.Flush(ctx) // flush will not have effect + bm.EnableIndexFlush() + + verifyActiveIndexBlobCount(ctx, t, bm, 0) + bm.EnableIndexFlush() + verifyActiveIndexBlobCount(ctx, t, bm, 0) + bm.Flush(ctx) // flush will happen now + verifyActiveIndexBlobCount(ctx, t, bm, 1) +} + +func TestRewriteDeleted(t *testing.T) { + const stepBehaviors = 3 + + // perform a sequence WriteContent() Delete() RewriteContent() GetContent() + // where actionX can be (0=flush and reopen, 1=flush, 2=nothing) + for action1 := 0; action1 < stepBehaviors; action1++ { + for action2 := 0; action2 < stepBehaviors; action2++ { + for action3 := 0; action3 < stepBehaviors; action3++ { + t.Run(fmt.Sprintf("case-%v-%v-%v", action1, action2, action3), func(t *testing.T) { + ctx := context.Background() + data := blobtesting.DataMap{} + keyTime := map[blob.ID]time.Time{} + fakeNow := fakeTimeNowWithAutoAdvance(fakeTime, 1*time.Second) + bm := newTestContentManager(data, keyTime, fakeNow) + + applyStep := func(action int) { + switch action { + case 0: + t.Logf("flushing and reopening") + bm.Flush(ctx) + bm = newTestContentManager(data, keyTime, fakeNow) + case 1: + t.Logf("flushing") + bm.Flush(ctx) + case 2: + t.Logf("doing nothing") + } + } + + content1 := writeContentAndVerify(ctx, t, bm, seededRandomData(10, 100)) + applyStep(action1) + assertNoError(t, bm.DeleteContent(content1)) + applyStep(action2) + if got, want := bm.RewriteContent(ctx, content1), ErrContentNotFound; got != want && got != nil { + t.Errorf("unexpected error %v, wanted %v", got, want) + } + applyStep(action3) + verifyContentNotFound(ctx, t, bm, content1) + dumpContentManagerData(t, data) + }) + } + } + } +} + +func TestDeleteAndRecreate(t *testing.T) { + ctx := context.Background() + // simulate race between delete/recreate and delete + // delete happens at t0+10, recreate at t0+20 and second delete time is parameterized. + // depending on it, the second delete results will be visible. + cases := []struct { + desc string + deletionTime time.Time + isVisible bool + }{ + {"deleted before delete and-recreate", fakeTime.Add(5 * time.Second), true}, + //{"deleted after delete and recreate", fakeTime.Add(25 * time.Second), false}, + } + + for _, tc := range cases { + t.Run(tc.desc, func(t *testing.T) { + // write a content + data := blobtesting.DataMap{} + keyTime := map[blob.ID]time.Time{} + bm := newTestContentManager(data, keyTime, fakeTimeNowFrozen(fakeTime)) + content1 := writeContentAndVerify(ctx, t, bm, seededRandomData(10, 100)) + bm.Flush(ctx) + + // delete but at given timestamp but don't commit yet. + bm0 := newTestContentManager(data, keyTime, fakeTimeNowWithAutoAdvance(tc.deletionTime, 1*time.Second)) + assertNoError(t, bm0.DeleteContent(content1)) + + // delete it at t0+10 + bm1 := newTestContentManager(data, keyTime, fakeTimeNowWithAutoAdvance(fakeTime.Add(10*time.Second), 1*time.Second)) + verifyContent(ctx, t, bm1, content1, seededRandomData(10, 100)) + assertNoError(t, bm1.DeleteContent(content1)) + bm1.Flush(ctx) + + // recreate at t0+20 + bm2 := newTestContentManager(data, keyTime, fakeTimeNowWithAutoAdvance(fakeTime.Add(20*time.Second), 1*time.Second)) + content2 := writeContentAndVerify(ctx, t, bm2, seededRandomData(10, 100)) + bm2.Flush(ctx) + + // commit deletion from bm0 (t0+5) + bm0.Flush(ctx) + + //dumpContentManagerData(t, data) + + if content1 != content2 { + t.Errorf("got invalid content %v, expected %v", content2, content1) + } + + bm3 := newTestContentManager(data, keyTime, nil) + dumpContentManagerData(t, data) + if tc.isVisible { + verifyContent(ctx, t, bm3, content1, seededRandomData(10, 100)) + } else { + verifyContentNotFound(ctx, t, bm3, content1) + } + }) + } +} + +func TestFindUnreferencedBlobs(t *testing.T) { + ctx := context.Background() + data := blobtesting.DataMap{} + keyTime := map[blob.ID]time.Time{} + bm := newTestContentManager(data, keyTime, nil) + verifyUnreferencedStorageFilesCount(ctx, t, bm, 0) + contentID := writeContentAndVerify(ctx, t, bm, seededRandomData(10, 100)) + if err := bm.Flush(ctx); err != nil { + t.Errorf("flush error: %v", err) + } + verifyUnreferencedStorageFilesCount(ctx, t, bm, 0) + if err := bm.DeleteContent(contentID); err != nil { + t.Errorf("error deleting content: %v", contentID) + } + if err := bm.Flush(ctx); err != nil { + t.Errorf("flush error: %v", err) + } + + // content still present in first pack + verifyUnreferencedStorageFilesCount(ctx, t, bm, 0) + + assertNoError(t, bm.RewriteContent(ctx, contentID)) + if err := bm.Flush(ctx); err != nil { + t.Errorf("flush error: %v", err) + } + verifyUnreferencedStorageFilesCount(ctx, t, bm, 1) + assertNoError(t, bm.RewriteContent(ctx, contentID)) + if err := bm.Flush(ctx); err != nil { + t.Errorf("flush error: %v", err) + } + verifyUnreferencedStorageFilesCount(ctx, t, bm, 2) +} + +func TestFindUnreferencedBlobs2(t *testing.T) { + ctx := context.Background() + data := blobtesting.DataMap{} + keyTime := map[blob.ID]time.Time{} + bm := newTestContentManager(data, keyTime, nil) + verifyUnreferencedStorageFilesCount(ctx, t, bm, 0) + contentID := writeContentAndVerify(ctx, t, bm, seededRandomData(10, 100)) + writeContentAndVerify(ctx, t, bm, seededRandomData(11, 100)) + dumpContents(t, bm, "after writing") + if err := bm.Flush(ctx); err != nil { + t.Errorf("flush error: %v", err) + } + dumpContents(t, bm, "after flush") + verifyUnreferencedStorageFilesCount(ctx, t, bm, 0) + if err := bm.DeleteContent(contentID); err != nil { + t.Errorf("error deleting content: %v", contentID) + } + dumpContents(t, bm, "after delete") + if err := bm.Flush(ctx); err != nil { + t.Errorf("flush error: %v", err) + } + dumpContents(t, bm, "after flush") + // content present in first pack, original pack is still referenced + verifyUnreferencedStorageFilesCount(ctx, t, bm, 0) +} + +func dumpContents(t *testing.T, bm *Manager, caption string) { + t.Helper() + infos, err := bm.ListContentInfos("", true) + if err != nil { + t.Errorf("error listing contents: %v", err) + return + } + + log.Infof("**** dumping %v contents %v", len(infos), caption) + for i, bi := range infos { + log.Debugf(" bi[%v]=%#v", i, bi) + } + log.Infof("finished dumping %v contents", len(infos)) +} + +func verifyUnreferencedStorageFilesCount(ctx context.Context, t *testing.T, bm *Manager, want int) { + t.Helper() + unref, err := bm.FindUnreferencedBlobs(ctx) + if err != nil { + t.Errorf("error in FindUnreferencedBlobs: %v", err) + } + + log.Infof("got %v expecting %v", unref, want) + if got := len(unref); got != want { + t.Errorf("invalid number of unreferenced contents: %v, wanted %v", got, want) + } +} + +func TestContentWriteAliasing(t *testing.T) { + ctx := context.Background() + data := blobtesting.DataMap{} + keyTime := map[blob.ID]time.Time{} + bm := newTestContentManager(data, keyTime, fakeTimeNowFrozen(fakeTime)) + + contentData := []byte{100, 0, 0} + id1 := writeContentAndVerify(ctx, t, bm, contentData) + contentData[0] = 101 + id2 := writeContentAndVerify(ctx, t, bm, contentData) + bm.Flush(ctx) + contentData[0] = 102 + id3 := writeContentAndVerify(ctx, t, bm, contentData) + contentData[0] = 103 + id4 := writeContentAndVerify(ctx, t, bm, contentData) + verifyContent(ctx, t, bm, id1, []byte{100, 0, 0}) + verifyContent(ctx, t, bm, id2, []byte{101, 0, 0}) + verifyContent(ctx, t, bm, id3, []byte{102, 0, 0}) + verifyContent(ctx, t, bm, id4, []byte{103, 0, 0}) +} + +func TestContentReadAliasing(t *testing.T) { + ctx := context.Background() + data := blobtesting.DataMap{} + keyTime := map[blob.ID]time.Time{} + bm := newTestContentManager(data, keyTime, fakeTimeNowFrozen(fakeTime)) + + contentData := []byte{100, 0, 0} + id1 := writeContentAndVerify(ctx, t, bm, contentData) + contentData2, err := bm.GetContent(ctx, id1) + if err != nil { + t.Fatalf("can't get content data: %v", err) + } + + contentData2[0]++ + verifyContent(ctx, t, bm, id1, contentData) + bm.Flush(ctx) + verifyContent(ctx, t, bm, id1, contentData) +} + +func TestVersionCompatibility(t *testing.T) { + for writeVer := minSupportedReadVersion; writeVer <= currentWriteVersion; writeVer++ { + t.Run(fmt.Sprintf("version-%v", writeVer), func(t *testing.T) { + verifyVersionCompat(t, writeVer) + }) + } +} + +func verifyVersionCompat(t *testing.T, writeVersion int) { + ctx := context.Background() + + // create content manager that writes 'writeVersion' and reads all versions >= minSupportedReadVersion + data := blobtesting.DataMap{} + keyTime := map[blob.ID]time.Time{} + mgr := newTestContentManager(data, keyTime, nil) + mgr.writeFormatVersion = int32(writeVersion) + + dataSet := map[ID][]byte{} + + for i := 0; i < 3000000; i = (i + 1) * 2 { + data := make([]byte, i) + rand.Read(data) + + cid, err := mgr.WriteContent(ctx, data, "") + if err != nil { + t.Fatalf("unable to write %v bytes: %v", len(data), err) + } + dataSet[cid] = data + } + verifyContentManagerDataSet(ctx, t, mgr, dataSet) + + // delete random 3 items (map iteration order is random) + cnt := 0 + for blobID := range dataSet { + t.Logf("deleting %v", blobID) + assertNoError(t, mgr.DeleteContent(blobID)) + delete(dataSet, blobID) + cnt++ + if cnt >= 3 { + break + } + } + if err := mgr.Flush(ctx); err != nil { + t.Fatalf("failed to flush: %v", err) + } + + // create new manager that reads and writes using new version. + mgr = newTestContentManager(data, keyTime, nil) + + // make sure we can read everything + verifyContentManagerDataSet(ctx, t, mgr, dataSet) + + if err := mgr.CompactIndexes(ctx, CompactOptions{ + MinSmallBlobs: 1, + MaxSmallBlobs: 1, + }); err != nil { + t.Fatalf("unable to compact indexes: %v", err) + } + if err := mgr.Flush(ctx); err != nil { + t.Fatalf("failed to flush: %v", err) + } + verifyContentManagerDataSet(ctx, t, mgr, dataSet) + + // now open one more manager + mgr = newTestContentManager(data, keyTime, nil) + verifyContentManagerDataSet(ctx, t, mgr, dataSet) +} + +func verifyContentManagerDataSet(ctx context.Context, t *testing.T, mgr *Manager, dataSet map[ID][]byte) { + for contentID, originalPayload := range dataSet { + v, err := mgr.GetContent(ctx, contentID) + if err != nil { + t.Errorf("unable to read content %q: %v", contentID, err) + continue + } + + if !reflect.DeepEqual(v, originalPayload) { + t.Errorf("payload for %q does not match original: %v", v, originalPayload) + } + } +} + +func newTestContentManager(data blobtesting.DataMap, keyTime map[blob.ID]time.Time, timeFunc func() time.Time) *Manager { + //st = logging.NewWrapper(st) + if timeFunc == nil { + timeFunc = fakeTimeNowWithAutoAdvance(fakeTime, 1*time.Second) + } + st := blobtesting.NewMapStorage(data, keyTime, timeFunc) + bm, err := newManagerWithOptions(context.Background(), st, FormattingOptions{ + Hash: "HMAC-SHA256", + Encryption: "NONE", + HMACSecret: hmacSecret, + MaxPackSize: maxPackSize, + Version: 1, + }, CachingOptions{}, timeFunc, nil) + if err != nil { + panic("can't create content manager: " + err.Error()) + } + bm.checkInvariantsOnUnlock = true + return bm +} + +func getIndexCount(d blobtesting.DataMap) int { + var cnt int + + for blobID := range d { + if strings.HasPrefix(string(blobID), newIndexBlobPrefix) { + cnt++ + } + } + + return cnt +} + +func fakeTimeNowFrozen(t time.Time) func() time.Time { + return fakeTimeNowWithAutoAdvance(t, 0) +} + +func fakeTimeNowWithAutoAdvance(t time.Time, dt time.Duration) func() time.Time { + var mu sync.Mutex + return func() time.Time { + mu.Lock() + defer mu.Unlock() + ret := t + t = t.Add(dt) + return ret + } +} + +func verifyContentNotFound(ctx context.Context, t *testing.T, bm *Manager, contentID ID) { + t.Helper() + + b, err := bm.GetContent(ctx, contentID) + if err != ErrContentNotFound { + t.Errorf("unexpected response from GetContent(%q), got %v,%v, expected %v", contentID, b, err, ErrContentNotFound) + } +} + +func verifyContent(ctx context.Context, t *testing.T, bm *Manager, contentID ID, b []byte) { + t.Helper() + + b2, err := bm.GetContent(ctx, contentID) + if err != nil { + t.Errorf("unable to read content %q: %v", contentID, err) + return + } + + if got, want := b2, b; !reflect.DeepEqual(got, want) { + t.Errorf("content %q data mismatch: got %x (nil:%v), wanted %x (nil:%v)", contentID, got, got == nil, want, want == nil) + } + + bi, err := bm.ContentInfo(ctx, contentID) + if err != nil { + t.Errorf("error getting content info %q: %v", contentID, err) + } + + if got, want := bi.Length, uint32(len(b)); got != want { + t.Errorf("invalid content size for %q: %v, wanted %v", contentID, got, want) + } + +} +func writeContentAndVerify(ctx context.Context, t *testing.T, bm *Manager, b []byte) ID { + t.Helper() + + contentID, err := bm.WriteContent(ctx, b, "") + if err != nil { + t.Errorf("err: %v", err) + } + + if got, want := contentID, ID(hashValue(b)); got != want { + t.Errorf("invalid content ID for %x, got %v, want %v", b, got, want) + } + + verifyContent(ctx, t, bm, contentID, b) + + return contentID +} + +func seededRandomData(seed int, length int) []byte { + b := make([]byte, length) + rnd := rand.New(rand.NewSource(int64(seed))) + rnd.Read(b) + return b +} + +func hashValue(b []byte) string { + h := hmac.New(sha256.New, hmacSecret) + h.Write(b) //nolint:errcheck + return hex.EncodeToString(h.Sum(nil)) +} + +func dumpContentManagerData(t *testing.T, data blobtesting.DataMap) { + t.Helper() + for k, v := range data { + if k[0] == 'n' { + ndx, err := openPackIndex(bytes.NewReader(v)) + if err == nil { + t.Logf("index %v (%v bytes)", k, len(v)) + assertNoError(t, ndx.Iterate("", func(i Info) error { + t.Logf(" %+v\n", i) + return nil + })) + + } + } else { + t.Logf("data %v (%v bytes)\n", k, len(v)) + } + } +} diff --git a/repo/content/context.go b/repo/content/context.go new file mode 100644 index 000000000..d928a3fc4 --- /dev/null +++ b/repo/content/context.go @@ -0,0 +1,34 @@ +package content + +import "context" + +type contextKey string + +var useContentCacheContextKey contextKey = "use-content-cache" +var useListCacheContextKey contextKey = "use-list-cache" + +// UsingContentCache returns a derived context that causes content manager to use cache. +func UsingContentCache(ctx context.Context, enabled bool) context.Context { + return context.WithValue(ctx, useContentCacheContextKey, enabled) +} + +// UsingListCache returns a derived context that causes content manager to use cache. +func UsingListCache(ctx context.Context, enabled bool) context.Context { + return context.WithValue(ctx, useListCacheContextKey, enabled) +} + +func shouldUseContentCache(ctx context.Context) bool { + if enabled, ok := ctx.Value(useContentCacheContextKey).(bool); ok { + return enabled + } + + return true +} + +func shouldUseListCache(ctx context.Context) bool { + if enabled, ok := ctx.Value(useListCacheContextKey).(bool); ok { + return enabled + } + + return true +} diff --git a/repo/block/format.go b/repo/content/format.go similarity index 94% rename from repo/block/format.go rename to repo/content/format.go index d3fc85a26..ef2c74b92 100644 --- a/repo/block/format.go +++ b/repo/content/format.go @@ -1,4 +1,4 @@ -package block +package content import ( "encoding/binary" @@ -27,9 +27,9 @@ type entry struct { // big endian: // 48 most significant bits - 48-bit timestamp in seconds since 1970/01/01 UTC // 8 bits - format version (currently == 1) - // 8 least significant bits - length of pack block ID + // 8 least significant bits - length of pack content ID timestampAndFlags uint64 // - packFileOffset uint32 // 4 bytes, big endian, offset within index file where pack block ID begins + packFileOffset uint32 // 4 bytes, big endian, offset within index file where pack content ID begins packedOffset uint32 // 4 bytes, big endian, offset within pack file where the contents begin packedLength uint32 // 4 bytes, big endian, content length } diff --git a/repo/block/index.go b/repo/content/index.go similarity index 76% rename from repo/block/index.go rename to repo/content/index.go index 6209e16c4..b2e7261c3 100644 --- a/repo/block/index.go +++ b/repo/content/index.go @@ -1,4 +1,4 @@ -package block +package content import ( "bytes" @@ -12,12 +12,12 @@ "github.com/kopia/kopia/repo/blob" ) -// packIndex is a read-only index of packed blocks. +// packIndex is a read-only index of packed contents. type packIndex interface { io.Closer - GetInfo(blockID string) (*Info, error) - Iterate(prefix string, cb func(Info) error) error + GetInfo(contentID ID) (*Info, error) + Iterate(prefix ID, cb func(Info) error) error } type index struct { @@ -55,10 +55,10 @@ func readHeader(readerAt io.ReaderAt) (headerInfo, error) { return hi, nil } -// Iterate invokes the provided callback function for all blocks in the index, sorted alphabetically. +// Iterate invokes the provided callback function for all contents in the index, sorted alphabetically. // The iteration ends when the callback returns an error, which is propagated to the caller or when -// all blocks have been visited. -func (b *index) Iterate(prefix string, cb func(Info) error) error { +// all contents have been visited. +func (b *index) Iterate(prefix ID, cb func(Info) error) error { startPos, err := b.findEntryPosition(prefix) if err != nil { return errors.Wrap(err, "could not find starting position") @@ -78,7 +78,7 @@ func (b *index) Iterate(prefix string, cb func(Info) error) error { if err != nil { return errors.Wrap(err, "invalid index data") } - if !strings.HasPrefix(i.BlockID, prefix) { + if !strings.HasPrefix(string(i.ID), string(prefix)) { break } if err := cb(i); err != nil { @@ -88,7 +88,7 @@ func (b *index) Iterate(prefix string, cb func(Info) error) error { return nil } -func (b *index) findEntryPosition(blockID string) (int, error) { +func (b *index) findEntryPosition(contentID ID) (int, error) { stride := b.hdr.keySize + b.hdr.valueSize entryBuf := make([]byte, stride) var readErr error @@ -102,20 +102,20 @@ func (b *index) findEntryPosition(blockID string) (int, error) { return false } - return bytesToContentID(entryBuf[0:b.hdr.keySize]) >= blockID + return bytesToContentID(entryBuf[0:b.hdr.keySize]) >= contentID }) return pos, readErr } -func (b *index) findEntry(blockID string) ([]byte, error) { - key := contentIDToBytes(blockID) +func (b *index) findEntry(contentID ID) ([]byte, error) { + key := contentIDToBytes(contentID) if len(key) != b.hdr.keySize { - return nil, errors.Errorf("invalid block ID: %q", blockID) + return nil, errors.Errorf("invalid content ID: %q", contentID) } stride := b.hdr.keySize + b.hdr.valueSize - position, err := b.findEntryPosition(blockID) + position, err := b.findEntryPosition(contentID) if err != nil { return nil, err } @@ -135,9 +135,9 @@ func (b *index) findEntry(blockID string) ([]byte, error) { return nil, nil } -// GetInfo returns information about a given block. If a block is not found, nil is returned. -func (b *index) GetInfo(blockID string) (*Info, error) { - e, err := b.findEntry(blockID) +// GetInfo returns information about a given content. If a content is not found, nil is returned. +func (b *index) GetInfo(contentID ID) (*Info, error) { + e, err := b.findEntry(contentID) if err != nil { return nil, err } @@ -146,14 +146,14 @@ func (b *index) GetInfo(blockID string) (*Info, error) { return nil, nil } - i, err := b.entryToInfo(blockID, e) + i, err := b.entryToInfo(contentID, e) if err != nil { return nil, err } return &i, err } -func (b *index) entryToInfo(blockID string, entryData []byte) (Info, error) { +func (b *index) entryToInfo(contentID ID, entryData []byte) (Info, error) { if len(entryData) < 20 { return Info{}, errors.Errorf("invalid entry length: %v", len(entryData)) } @@ -166,11 +166,11 @@ func (b *index) entryToInfo(blockID string, entryData []byte) (Info, error) { packFile := make([]byte, e.PackFileLength()) n, err := b.readerAt.ReadAt(packFile, int64(e.PackFileOffset())) if err != nil || n != int(e.PackFileLength()) { - return Info{}, errors.Wrap(err, "can't read pack block ID") + return Info{}, errors.Wrap(err, "can't read pack content ID") } return Info{ - BlockID: blockID, + ID: contentID, Deleted: e.IsDeleted(), TimestampSeconds: e.TimestampSeconds(), FormatVersion: e.PackedFormatVersion(), diff --git a/repo/block/info.go b/repo/content/info.go similarity index 64% rename from repo/block/info.go rename to repo/content/info.go index 76f5c5750..c3d7bf08a 100644 --- a/repo/block/info.go +++ b/repo/content/info.go @@ -1,4 +1,4 @@ -package block +package content import ( "time" @@ -6,9 +6,12 @@ "github.com/kopia/kopia/repo/blob" ) -// Info is an information about a single block managed by Manager. +// ID is an identifier of content in content-addressable storage. +type ID string + +// Info is an information about a single piece of content managed by Manager. type Info struct { - BlockID string `json:"blockID"` + ID ID `json:"contentID"` Length uint32 `json:"length"` TimestampSeconds int64 `json:"time"` PackBlobID blob.ID `json:"packFile,omitempty"` @@ -18,7 +21,7 @@ type Info struct { FormatVersion byte `json:"formatVersion"` } -// Timestamp returns the time when a block was created or deleted. +// Timestamp returns the time when a content was created or deleted. func (i Info) Timestamp() time.Time { return time.Unix(i.TimestampSeconds, 0) } diff --git a/repo/block/list_cache.go b/repo/content/list_cache.go similarity index 86% rename from repo/block/list_cache.go rename to repo/content/list_cache.go index a1af8f382..c2556418a 100644 --- a/repo/block/list_cache.go +++ b/repo/content/list_cache.go @@ -1,4 +1,4 @@ -package block +package content import ( "context" @@ -23,35 +23,35 @@ type listCache struct { func (c *listCache) listIndexBlobs(ctx context.Context) ([]IndexBlobInfo, error) { if c.cacheFile != "" { - ci, err := c.readBlocksFromCache(ctx) + ci, err := c.readContentsFromCache(ctx) if err == nil { expirationTime := ci.Timestamp.Add(c.listCacheDuration) if time.Now().Before(expirationTime) { log.Debugf("retrieved list of index blobs from cache") - return ci.Blocks, nil + return ci.Contents, nil } } else if err != blob.ErrBlobNotFound { log.Warningf("unable to open cache file: %v", err) } } - blocks, err := listIndexBlobsFromStorage(ctx, c.st) + contents, err := listIndexBlobsFromStorage(ctx, c.st) if err == nil { c.saveListToCache(ctx, &cachedList{ - Blocks: blocks, + Contents: contents, Timestamp: time.Now(), }) } - log.Debugf("found %v index blobs from source", len(blocks)) + log.Debugf("found %v index blobs from source", len(contents)) - return blocks, err + return contents, err } func (c *listCache) saveListToCache(ctx context.Context, ci *cachedList) { if c.cacheFile == "" { return } - log.Debugf("saving index blobs to cache: %v", len(ci.Blocks)) + log.Debugf("saving index blobs to cache: %v", len(ci.Contents)) if data, err := json.Marshal(ci); err == nil { mySuffix := fmt.Sprintf(".tmp-%v-%v", os.Getpid(), time.Now().UnixNano()) if err := ioutil.WriteFile(c.cacheFile+mySuffix, appendHMAC(data, c.hmacSecret), 0600); err != nil { @@ -68,7 +68,7 @@ func (c *listCache) deleteListCache(ctx context.Context) { } } -func (c *listCache) readBlocksFromCache(ctx context.Context) (*cachedList, error) { +func (c *listCache) readContentsFromCache(ctx context.Context) (*cachedList, error) { if !shouldUseListCache(ctx) { return nil, blob.ErrBlobNotFound } diff --git a/repo/block/merged.go b/repo/content/merged.go similarity index 74% rename from repo/block/merged.go rename to repo/content/merged.go index 20140604c..20fd0ef37 100644 --- a/repo/block/merged.go +++ b/repo/content/merged.go @@ -1,4 +1,4 @@ -package block +package content import ( "container/heap" @@ -19,11 +19,11 @@ func (m mergedIndex) Close() error { return nil } -// GetInfo returns information about a single block. If a block is not found, returns (nil,nil) -func (m mergedIndex) GetInfo(contentID string) (*Info, error) { +// GetInfo returns information about a single content. If a content is not found, returns (nil,nil) +func (m mergedIndex) GetInfo(id ID) (*Info, error) { var best *Info for _, ndx := range m { - i, err := ndx.GetInfo(contentID) + i, err := ndx.GetInfo(id) if err != nil { return nil, err } @@ -45,7 +45,7 @@ type nextInfo struct { func (h nextInfoHeap) Len() int { return len(h) } func (h nextInfoHeap) Less(i, j int) bool { - if a, b := h[i].it.BlockID, h[j].it.BlockID; a != b { + if a, b := h[i].it.ID, h[j].it.ID; a != b { return a < b } @@ -68,7 +68,7 @@ func (h *nextInfoHeap) Pop() interface{} { return x } -func iterateChan(prefix string, ndx packIndex, done chan bool) <-chan Info { +func iterateChan(prefix ID, ndx packIndex, done chan bool) <-chan Info { ch := make(chan Info) go func() { defer close(ch) @@ -85,9 +85,9 @@ func iterateChan(prefix string, ndx packIndex, done chan bool) <-chan Info { return ch } -// Iterate invokes the provided callback for all unique block IDs in the underlying sources until either -// all blocks have been visited or until an error is returned by the callback. -func (m mergedIndex) Iterate(prefix string, cb func(i Info) error) error { +// Iterate invokes the provided callback for all unique content IDs in the underlying sources until either +// all contents have been visited or until an error is returned by the callback. +func (m mergedIndex) Iterate(prefix ID, cb func(i Info) error) error { var minHeap nextInfoHeap done := make(chan bool) defer close(done) @@ -104,8 +104,8 @@ func (m mergedIndex) Iterate(prefix string, cb func(i Info) error) error { for len(minHeap) > 0 { min := heap.Pop(&minHeap).(*nextInfo) - if pendingItem.BlockID != min.it.BlockID { - if pendingItem.BlockID != "" { + if pendingItem.ID != min.it.ID { + if pendingItem.ID != "" { if err := cb(pendingItem); err != nil { return err } @@ -122,7 +122,7 @@ func (m mergedIndex) Iterate(prefix string, cb func(i Info) error) error { } } - if pendingItem.BlockID != "" { + if pendingItem.ID != "" { return cb(pendingItem) } diff --git a/repo/block/merged_test.go b/repo/content/merged_test.go similarity index 53% rename from repo/block/merged_test.go rename to repo/content/merged_test.go index 9a984150f..8d420fad7 100644 --- a/repo/block/merged_test.go +++ b/repo/content/merged_test.go @@ -1,4 +1,4 @@ -package block +package content import ( "bytes" @@ -10,27 +10,27 @@ func TestMerged(t *testing.T) { i1, err := indexWithItems( - Info{BlockID: "aabbcc", TimestampSeconds: 1, PackBlobID: "xx", PackOffset: 11}, - Info{BlockID: "ddeeff", TimestampSeconds: 1, PackBlobID: "xx", PackOffset: 111}, - Info{BlockID: "z010203", TimestampSeconds: 1, PackBlobID: "xx", PackOffset: 111}, - Info{BlockID: "de1e1e", TimestampSeconds: 4, PackBlobID: "xx", PackOffset: 111}, + Info{ID: "aabbcc", TimestampSeconds: 1, PackBlobID: "xx", PackOffset: 11}, + Info{ID: "ddeeff", TimestampSeconds: 1, PackBlobID: "xx", PackOffset: 111}, + Info{ID: "z010203", TimestampSeconds: 1, PackBlobID: "xx", PackOffset: 111}, + Info{ID: "de1e1e", TimestampSeconds: 4, PackBlobID: "xx", PackOffset: 111}, ) if err != nil { t.Fatalf("can't create index: %v", err) } i2, err := indexWithItems( - Info{BlockID: "aabbcc", TimestampSeconds: 3, PackBlobID: "yy", PackOffset: 33}, - Info{BlockID: "xaabbcc", TimestampSeconds: 1, PackBlobID: "xx", PackOffset: 111}, - Info{BlockID: "de1e1e", TimestampSeconds: 4, PackBlobID: "xx", PackOffset: 222, Deleted: true}, + Info{ID: "aabbcc", TimestampSeconds: 3, PackBlobID: "yy", PackOffset: 33}, + Info{ID: "xaabbcc", TimestampSeconds: 1, PackBlobID: "xx", PackOffset: 111}, + Info{ID: "de1e1e", TimestampSeconds: 4, PackBlobID: "xx", PackOffset: 222, Deleted: true}, ) if err != nil { t.Fatalf("can't create index: %v", err) } i3, err := indexWithItems( - Info{BlockID: "aabbcc", TimestampSeconds: 2, PackBlobID: "zz", PackOffset: 22}, - Info{BlockID: "ddeeff", TimestampSeconds: 1, PackBlobID: "zz", PackOffset: 222}, - Info{BlockID: "k010203", TimestampSeconds: 1, PackBlobID: "xx", PackOffset: 111}, - Info{BlockID: "k020304", TimestampSeconds: 1, PackBlobID: "xx", PackOffset: 111}, + Info{ID: "aabbcc", TimestampSeconds: 2, PackBlobID: "zz", PackOffset: 22}, + Info{ID: "ddeeff", TimestampSeconds: 1, PackBlobID: "zz", PackOffset: 222}, + Info{ID: "k010203", TimestampSeconds: 1, PackBlobID: "xx", PackOffset: 111}, + Info{ID: "k020304", TimestampSeconds: 1, PackBlobID: "xx", PackOffset: 111}, ) if err != nil { t.Fatalf("can't create index: %v", err) @@ -45,24 +45,24 @@ func TestMerged(t *testing.T) { t.Errorf("invalid pack offset %v, wanted %v", got, want) } - var inOrder []string + var inOrder []ID assertNoError(t, m.Iterate("", func(i Info) error { - inOrder = append(inOrder, i.BlockID) - if i.BlockID == "de1e1e" { + inOrder = append(inOrder, i.ID) + if i.ID == "de1e1e" { if i.Deleted { - t.Errorf("iteration preferred deleted block over non-deleted") + t.Errorf("iteration preferred deleted content over non-deleted") } } return nil })) if i, err := m.GetInfo("de1e1e"); err != nil { - t.Errorf("error getting deleted block info: %v", err) + t.Errorf("error getting deleted content info: %v", err) } else if i.Deleted { - t.Errorf("GetInfo preferred deleted block over non-deleted") + t.Errorf("GetInfo preferred deleted content over non-deleted") } - expectedInOrder := []string{ + expectedInOrder := []ID{ "aabbcc", "ddeeff", "de1e1e", diff --git a/repo/block/packindex_internal_test.go b/repo/content/packindex_internal_test.go similarity index 78% rename from repo/block/packindex_internal_test.go rename to repo/content/packindex_internal_test.go index 305619855..01deb779b 100644 --- a/repo/block/packindex_internal_test.go +++ b/repo/content/packindex_internal_test.go @@ -1,9 +1,9 @@ -package block +package content import "testing" func TestRoundTrip(t *testing.T) { - cases := []string{ + cases := []ID{ "", "x", "aa", @@ -20,7 +20,7 @@ func TestRoundTrip(t *testing.T) { } } - if got, want := bytesToContentID(nil), ""; got != want { + if got, want := bytesToContentID(nil), ID(""); got != want { t.Errorf("unexpected content id %v, want %v", got, want) } } diff --git a/repo/block/packindex_test.go b/repo/content/packindex_test.go similarity index 81% rename from repo/block/packindex_test.go rename to repo/content/packindex_test.go index fed6555bb..ea6044e37 100644 --- a/repo/block/packindex_test.go +++ b/repo/content/packindex_test.go @@ -1,4 +1,4 @@ -package block +package content import ( "bytes" @@ -14,12 +14,12 @@ ) func TestPackIndex(t *testing.T) { - blockNumber := 0 + contentNumber := 0 - deterministicBlockID := func(prefix string, id int) string { + deterministicContentID := func(prefix string, id int) ID { h := sha1.New() fmt.Fprintf(h, "%v%v", prefix, id) - blockNumber++ + contentNumber++ prefix2 := "" if id%2 == 0 { @@ -31,12 +31,12 @@ func TestPackIndex(t *testing.T) { if id%5 == 0 { prefix2 = "m" } - return string(fmt.Sprintf("%v%x", prefix2, h.Sum(nil))) + return ID(fmt.Sprintf("%v%x", prefix2, h.Sum(nil))) } deterministicPackBlobID := func(id int) blob.ID { h := sha1.New() fmt.Fprintf(h, "%v", id) - blockNumber++ + contentNumber++ return blob.ID(fmt.Sprintf("%x", h.Sum(nil))) } @@ -60,23 +60,23 @@ func TestPackIndex(t *testing.T) { var infos []Info - // deleted blocks with all information + // deleted contents with all information for i := 0; i < 100; i++ { infos = append(infos, Info{ TimestampSeconds: randomUnixTime(), Deleted: true, - BlockID: deterministicBlockID("deleted-packed", i), + ID: deterministicContentID("deleted-packed", i), PackBlobID: deterministicPackBlobID(i), PackOffset: deterministicPackedOffset(i), Length: deterministicPackedLength(i), FormatVersion: deterministicFormatVersion(i), }) } - // non-deleted block + // non-deleted content for i := 0; i < 100; i++ { infos = append(infos, Info{ TimestampSeconds: randomUnixTime(), - BlockID: deterministicBlockID("packed", i), + ID: deterministicContentID("packed", i), PackBlobID: deterministicPackBlobID(i), PackOffset: deterministicPackedOffset(i), Length: deterministicPackedLength(i), @@ -84,13 +84,13 @@ func TestPackIndex(t *testing.T) { }) } - infoMap := map[string]Info{} + infoMap := map[ID]Info{} b1 := make(packIndexBuilder) b2 := make(packIndexBuilder) b3 := make(packIndexBuilder) for _, info := range infos { - infoMap[info.BlockID] = info + infoMap[info.ID] = info b1.Add(info) b2.Add(info) b3.Add(info) @@ -130,9 +130,9 @@ func TestPackIndex(t *testing.T) { defer ndx.Close() for _, info := range infos { - info2, err := ndx.GetInfo(info.BlockID) + info2, err := ndx.GetInfo(info.ID) if err != nil { - t.Errorf("unable to find %v", info.BlockID) + t.Errorf("unable to find %v", info.ID) continue } if !reflect.DeepEqual(info, *info2) { @@ -142,7 +142,7 @@ func TestPackIndex(t *testing.T) { cnt := 0 assertNoError(t, ndx.Iterate("", func(info2 Info) error { - info := infoMap[info2.BlockID] + info := infoMap[info2.ID] if !reflect.DeepEqual(info, info2) { t.Errorf("invalid value retrieved: %+v, wanted %+v", info2, info) } @@ -153,25 +153,25 @@ func TestPackIndex(t *testing.T) { t.Errorf("invalid number of iterations: %v, wanted %v", cnt, len(infoMap)) } - prefixes := []string{"a", "b", "f", "0", "3", "aa", "aaa", "aab", "fff", "m", "x", "y", "m0", "ma"} + prefixes := []ID{"a", "b", "f", "0", "3", "aa", "aaa", "aab", "fff", "m", "x", "y", "m0", "ma"} for i := 0; i < 100; i++ { - blockID := deterministicBlockID("no-such-block", i) - v, err := ndx.GetInfo(blockID) + contentID := deterministicContentID("no-such-content", i) + v, err := ndx.GetInfo(contentID) if err != nil { - t.Errorf("unable to get block %v: %v", blockID, err) + t.Errorf("unable to get content %v: %v", contentID, err) } if v != nil { - t.Errorf("unexpected result when getting block %v: %v", blockID, v) + t.Errorf("unexpected result when getting content %v: %v", contentID, v) } } for _, prefix := range prefixes { cnt2 := 0 - assertNoError(t, ndx.Iterate(string(prefix), func(info2 Info) error { + assertNoError(t, ndx.Iterate(prefix, func(info2 Info) error { cnt2++ - if !strings.HasPrefix(string(info2.BlockID), string(prefix)) { - t.Errorf("unexpected item %v when iterating prefix %v", info2.BlockID, prefix) + if !strings.HasPrefix(string(info2.ID), string(prefix)) { + t.Errorf("unexpected item %v when iterating prefix %v", info2.ID, prefix) } return nil })) @@ -192,7 +192,7 @@ func fuzzTestIndexOpen(t *testing.T, originalData []byte) { cnt := 0 _ = ndx.Iterate("", func(cb Info) error { if cnt < 10 { - _, _ = ndx.GetInfo(cb.BlockID) + _, _ = ndx.GetInfo(cb.ID) } cnt++ return nil diff --git a/repo/block/stats.go b/repo/content/stats.go similarity index 52% rename from repo/block/stats.go rename to repo/content/stats.go index b1483506f..14001dd3d 100644 --- a/repo/block/stats.go +++ b/repo/content/stats.go @@ -1,6 +1,6 @@ -package block +package content -// Stats exposes statistics about block operation. +// Stats exposes statistics about content operation. type Stats struct { // Keep int64 fields first to ensure they get aligned to at least 64-bit boundaries // which is required for atomic access on ARM and x86-32. @@ -10,13 +10,13 @@ type Stats struct { EncryptedBytes int64 `json:"encryptedBytes,omitempty"` HashedBytes int64 `json:"hashedBytes,omitempty"` - ReadBlocks int32 `json:"readBlocks,omitempty"` - WrittenBlocks int32 `json:"writtenBlocks,omitempty"` - CheckedBlocks int32 `json:"checkedBlocks,omitempty"` - HashedBlocks int32 `json:"hashedBlocks,omitempty"` - InvalidBlocks int32 `json:"invalidBlocks,omitempty"` - PresentBlocks int32 `json:"presentBlocks,omitempty"` - ValidBlocks int32 `json:"validBlocks,omitempty"` + ReadContents int32 `json:"readContents,omitempty"` + WrittenContents int32 `json:"writtenContents,omitempty"` + CheckedContents int32 `json:"checkedContents,omitempty"` + HashedContents int32 `json:"hashedContents,omitempty"` + InvalidContents int32 `json:"invalidContents,omitempty"` + PresentContents int32 `json:"presentContents,omitempty"` + ValidContents int32 `json:"validContents,omitempty"` } // Reset clears all repository statistics. diff --git a/repo/crypto_key_derivation.go b/repo/crypto_key_derivation.go index 83f6f58e2..bcc132a75 100644 --- a/repo/crypto_key_derivation.go +++ b/repo/crypto_key_derivation.go @@ -12,7 +12,7 @@ // defaultKeyDerivationAlgorithm is the key derivation algorithm for new configurations. const defaultKeyDerivationAlgorithm = "scrypt-65536-8-1" -func (f formatBlock) deriveMasterKeyFromPassword(password string) ([]byte, error) { +func (f formatBlob) deriveMasterKeyFromPassword(password string) ([]byte, error) { const masterKeySize = 32 switch f.KeyDerivationAlgorithm { diff --git a/repo/format_block.go b/repo/format_block.go index 3ad7c3bd3..bc7c476db 100644 --- a/repo/format_block.go +++ b/repo/format_block.go @@ -20,13 +20,13 @@ const ( maxChecksummedFormatBytesLength = 65000 - formatBlockChecksumSize = sha256.Size + formatBlobChecksumSize = sha256.Size ) -// formatBlockChecksumSecret is a HMAC secret used for checksumming the format block. +// formatBlobChecksumSecret is a HMAC secret used for checksumming the format content. // It's not really a secret, but will provide positive identification of blocks that // are repository format blocks. -var formatBlockChecksumSecret = []byte("kopia-repository") +var formatBlobChecksumSecret = []byte("kopia-repository") // FormatBlobID is the identifier of a BLOB that describes repository format. const FormatBlobID = "kopia.repository" @@ -35,10 +35,10 @@ purposeAESKey = []byte("AES") purposeAuthData = []byte("CHECKSUM") - errFormatBlockNotFound = errors.New("format block not found") + errFormatBlobNotFound = errors.New("format blob not found") ) -type formatBlock struct { +type formatBlob struct { Tool string `json:"tool"` BuildVersion string `json:"buildVersion"` BuildInfo string `json:"buildInfo"` @@ -57,22 +57,22 @@ type encryptedRepositoryConfig struct { Format repositoryObjectFormat `json:"format"` } -func parseFormatBlock(b []byte) (*formatBlock, error) { - f := &formatBlock{} +func parseFormatBlob(b []byte) (*formatBlob, error) { + f := &formatBlob{} if err := json.Unmarshal(b, &f); err != nil { - return nil, errors.Wrap(err, "invalid format block") + return nil, errors.Wrap(err, "invalid format blob") } return f, nil } -// RecoverFormatBlock attempts to recover format block replica from the specified file. -// The format block can be either the prefix or a suffix of the given file. +// RecoverFormatBlob attempts to recover format blob replica from the specified file. +// The format blob can be either the prefix or a suffix of the given file. // optionally the length can be provided (if known) to speed up recovery. -func RecoverFormatBlock(ctx context.Context, st blob.Storage, blobID blob.ID, optionalLength int64) ([]byte, error) { +func RecoverFormatBlob(ctx context.Context, st blob.Storage, blobID blob.ID, optionalLength int64) ([]byte, error) { if optionalLength > 0 { - return recoverFormatBlockWithLength(ctx, st, blobID, optionalLength) + return recoverFormatBlobWithLength(ctx, st, blobID, optionalLength) } var foundMetadata blob.Metadata @@ -91,10 +91,10 @@ func RecoverFormatBlock(ctx context.Context, st blob.Storage, blobID blob.ID, op return nil, blob.ErrBlobNotFound } - return recoverFormatBlockWithLength(ctx, st, foundMetadata.BlobID, foundMetadata.Length) + return recoverFormatBlobWithLength(ctx, st, foundMetadata.BlobID, foundMetadata.Length) } -func recoverFormatBlockWithLength(ctx context.Context, st blob.Storage, blobID blob.ID, length int64) ([]byte, error) { +func recoverFormatBlobWithLength(ctx context.Context, st blob.Storage, blobID blob.ID, length int64) ([]byte, error) { chunkLength := int64(65536) if chunkLength > length { chunkLength = length @@ -107,7 +107,7 @@ func recoverFormatBlockWithLength(ctx context.Context, st blob.Storage, blobID b return nil, err } if l := int(prefixChunk[0]) + int(prefixChunk[1])<<8; l <= maxChecksummedFormatBytesLength && l+2 < len(prefixChunk) { - if b, ok := verifyFormatBlockChecksum(prefixChunk[2 : 2+l]); ok { + if b, ok := verifyFormatBlobChecksum(prefixChunk[2 : 2+l]); ok { return b, nil } } @@ -118,22 +118,22 @@ func recoverFormatBlockWithLength(ctx context.Context, st blob.Storage, blobID b return nil, err } if l := int(suffixChunk[len(suffixChunk)-2]) + int(suffixChunk[len(suffixChunk)-1])<<8; l <= maxChecksummedFormatBytesLength && l+2 < len(suffixChunk) { - if b, ok := verifyFormatBlockChecksum(suffixChunk[len(suffixChunk)-2-l : len(suffixChunk)-2]); ok { + if b, ok := verifyFormatBlobChecksum(suffixChunk[len(suffixChunk)-2-l : len(suffixChunk)-2]); ok { return b, nil } } } - return nil, errFormatBlockNotFound + return nil, errFormatBlobNotFound } -func verifyFormatBlockChecksum(b []byte) ([]byte, bool) { - if len(b) < formatBlockChecksumSize { +func verifyFormatBlobChecksum(b []byte) ([]byte, bool) { + if len(b) < formatBlobChecksumSize { return nil, false } - data, checksum := b[0:len(b)-formatBlockChecksumSize], b[len(b)-formatBlockChecksumSize:] - h := hmac.New(sha256.New, formatBlockChecksumSecret) + data, checksum := b[0:len(b)-formatBlobChecksumSize], b[len(b)-formatBlobChecksumSize:] + h := hmac.New(sha256.New, formatBlobChecksumSecret) h.Write(data) //nolint:errcheck actualChecksum := h.Sum(nil) if !hmac.Equal(actualChecksum, checksum) { @@ -143,22 +143,22 @@ func verifyFormatBlockChecksum(b []byte) ([]byte, bool) { return data, true } -func writeFormatBlock(ctx context.Context, st blob.Storage, f *formatBlock) error { +func writeFormatBlob(ctx context.Context, st blob.Storage, f *formatBlob) error { var buf bytes.Buffer e := json.NewEncoder(&buf) e.SetIndent("", " ") if err := e.Encode(f); err != nil { - return errors.Wrap(err, "unable to marshal format block") + return errors.Wrap(err, "unable to marshal format blob") } if err := st.PutBlob(ctx, FormatBlobID, buf.Bytes()); err != nil { - return errors.Wrap(err, "unable to write format block") + return errors.Wrap(err, "unable to write format blob") } return nil } -func (f *formatBlock) decryptFormatBytes(masterKey []byte) (*repositoryObjectFormat, error) { +func (f *formatBlob) decryptFormatBytes(masterKey []byte) (*repositoryObjectFormat, error) { switch f.EncryptionAlgorithm { case "NONE": // do nothing return f.UnencryptedFormat, nil @@ -209,7 +209,7 @@ func initCrypto(masterKey, repositoryID []byte) (cipher.AEAD, []byte, error) { return aead, authData, nil } -func encryptFormatBytes(f *formatBlock, format *repositoryObjectFormat, masterKey, repositoryID []byte) error { +func encryptFormatBytes(f *formatBlob, format *repositoryObjectFormat, masterKey, repositoryID []byte) error { switch f.EncryptionAlgorithm { case "NONE": f.UnencryptedFormat = format @@ -244,14 +244,14 @@ func encryptFormatBytes(f *formatBlock, format *repositoryObjectFormat, masterKe } } -func addFormatBlockChecksumAndLength(fb []byte) ([]byte, error) { - h := hmac.New(sha256.New, formatBlockChecksumSecret) +func addFormatBlobChecksumAndLength(fb []byte) ([]byte, error) { + h := hmac.New(sha256.New, formatBlobChecksumSecret) h.Write(fb) //nolint:errcheck checksummedFormatBytes := h.Sum(fb) l := len(checksummedFormatBytes) if l > maxChecksummedFormatBytesLength { - return nil, errors.Errorf("format block too big: %v", l) + return nil, errors.Errorf("format blob too big: %v", l) } // return diff --git a/repo/format_block_test.go b/repo/format_block_test.go index c143681d8..87b6e8a87 100644 --- a/repo/format_block_test.go +++ b/repo/format_block_test.go @@ -10,13 +10,13 @@ "github.com/kopia/kopia/repo/blob" ) -func TestFormatBlockRecovery(t *testing.T) { +func TestFormatBlobRecovery(t *testing.T) { data := blobtesting.DataMap{} st := blobtesting.NewMapStorage(data, nil, nil) ctx := context.Background() someDataBlock := []byte("aadsdasdas") - checksummed, err := addFormatBlockChecksumAndLength(someDataBlock) + checksummed, err := addFormatBlobChecksumAndLength(someDataBlock) if err != nil { t.Errorf("error appending checksum: %v", err) } @@ -24,9 +24,9 @@ func TestFormatBlockRecovery(t *testing.T) { t.Errorf("unexpected checksummed length: %v, want %v", got, want) } - assertNoError(t, st.PutBlob(ctx, "some-block-by-itself", checksummed)) - assertNoError(t, st.PutBlob(ctx, "some-block-suffix", append(append([]byte(nil), 1, 2, 3), checksummed...))) - assertNoError(t, st.PutBlob(ctx, "some-block-prefix", append(append([]byte(nil), checksummed...), 1, 2, 3))) + assertNoError(t, st.PutBlob(ctx, "some-blob-by-itself", checksummed)) + assertNoError(t, st.PutBlob(ctx, "some-blob-suffix", append(append([]byte(nil), 1, 2, 3), checksummed...))) + assertNoError(t, st.PutBlob(ctx, "some-blob-prefix", append(append([]byte(nil), checksummed...), 1, 2, 3))) // mess up checksum checksummed[len(checksummed)-3] ^= 1 @@ -42,22 +42,22 @@ func TestFormatBlockRecovery(t *testing.T) { blobID blob.ID err error }{ - {"some-block-by-itself", nil}, - {"some-block-suffix", nil}, - {"some-block-prefix", nil}, - {"bad-checksum", errFormatBlockNotFound}, - {"no-such-block", blob.ErrBlobNotFound}, - {"zero-len", errFormatBlockNotFound}, - {"one-len", errFormatBlockNotFound}, - {"two-len", errFormatBlockNotFound}, - {"three-len", errFormatBlockNotFound}, - {"four-len", errFormatBlockNotFound}, - {"five-len", errFormatBlockNotFound}, + {"some-blob-by-itself", nil}, + {"some-blob-suffix", nil}, + {"some-blob-prefix", nil}, + {"bad-checksum", errFormatBlobNotFound}, + {"no-such-blob", blob.ErrBlobNotFound}, + {"zero-len", errFormatBlobNotFound}, + {"one-len", errFormatBlobNotFound}, + {"two-len", errFormatBlobNotFound}, + {"three-len", errFormatBlobNotFound}, + {"four-len", errFormatBlobNotFound}, + {"five-len", errFormatBlobNotFound}, } for _, tc := range cases { t.Run(string(tc.blobID), func(t *testing.T) { - v, err := RecoverFormatBlock(ctx, st, tc.blobID, -1) + v, err := RecoverFormatBlob(ctx, st, tc.blobID, -1) if tc.err == nil { if !reflect.DeepEqual(v, someDataBlock) || err != nil { t.Errorf("unexpected result or error: v=%v err=%v, expected success", v, err) diff --git a/repo/initialize.go b/repo/initialize.go index 57629d5ab..9ba5019a8 100644 --- a/repo/initialize.go +++ b/repo/initialize.go @@ -8,7 +8,7 @@ "github.com/pkg/errors" "github.com/kopia/kopia/repo/blob" - "github.com/kopia/kopia/repo/block" + "github.com/kopia/kopia/repo/content" "github.com/kopia/kopia/repo/object" ) @@ -22,7 +22,7 @@ // All fields are optional, when not provided, reasonable defaults will be used. type NewRepositoryOptions struct { UniqueID []byte // force the use of particular unique ID - BlockFormat block.FormattingOptions + BlockFormat content.FormattingOptions DisableHMAC bool ObjectFormat object.Format // object format } @@ -42,7 +42,7 @@ func Initialize(ctx context.Context, st blob.Storage, opt *NewRepositoryOptions, return err } - format := formatBlockFromOptions(opt) + format := formatBlobFromOptions(opt) masterKey, err := format.deriveMasterKeyFromPassword(password) if err != nil { return errors.Wrap(err, "unable to derive master key") @@ -52,15 +52,15 @@ func Initialize(ctx context.Context, st blob.Storage, opt *NewRepositoryOptions, return errors.Wrap(err, "unable to encrypt format bytes") } - if err := writeFormatBlock(ctx, st, format); err != nil { - return errors.Wrap(err, "unable to write format block") + if err := writeFormatBlob(ctx, st, format); err != nil { + return errors.Wrap(err, "unable to write format blob") } return nil } -func formatBlockFromOptions(opt *NewRepositoryOptions) *formatBlock { - f := &formatBlock{ +func formatBlobFromOptions(opt *NewRepositoryOptions) *formatBlob { + f := &formatBlob{ Tool: "https://github.com/kopia/kopia", BuildInfo: BuildInfo, KeyDerivationAlgorithm: defaultKeyDerivationAlgorithm, @@ -78,10 +78,10 @@ func formatBlockFromOptions(opt *NewRepositoryOptions) *formatBlock { func repositoryObjectFormatFromOptions(opt *NewRepositoryOptions) *repositoryObjectFormat { f := &repositoryObjectFormat{ - FormattingOptions: block.FormattingOptions{ + FormattingOptions: content.FormattingOptions{ Version: 1, - Hash: applyDefaultString(opt.BlockFormat.Hash, block.DefaultHash), - Encryption: applyDefaultString(opt.BlockFormat.Encryption, block.DefaultEncryption), + Hash: applyDefaultString(opt.BlockFormat.Hash, content.DefaultHash), + Encryption: applyDefaultString(opt.BlockFormat.Encryption, content.DefaultEncryption), HMACSecret: applyDefaultRandomBytes(opt.BlockFormat.HMACSecret, 32), MasterKey: applyDefaultRandomBytes(opt.BlockFormat.MasterKey, 32), MaxPackSize: applyDefaultInt(opt.BlockFormat.MaxPackSize, 20<<20), // 20 MB diff --git a/repo/local_config.go b/repo/local_config.go index e85303a3b..7cad53346 100644 --- a/repo/local_config.go +++ b/repo/local_config.go @@ -6,19 +6,19 @@ "os" "github.com/kopia/kopia/repo/blob" - "github.com/kopia/kopia/repo/block" + "github.com/kopia/kopia/repo/content" "github.com/kopia/kopia/repo/object" ) // LocalConfig is a configuration of Kopia stored in a configuration file. type LocalConfig struct { - Storage blob.ConnectionInfo `json:"storage"` - Caching block.CachingOptions `json:"caching"` + Storage blob.ConnectionInfo `json:"storage"` + Caching content.CachingOptions `json:"caching"` } // repositoryObjectFormat describes the format of objects in a repository. type repositoryObjectFormat struct { - block.FormattingOptions + content.FormattingOptions object.Format } diff --git a/repo/manifest/manifest_entry.go b/repo/manifest/manifest_entry.go index cc2ead40f..f76e327a9 100644 --- a/repo/manifest/manifest_entry.go +++ b/repo/manifest/manifest_entry.go @@ -5,7 +5,7 @@ // EntryMetadata contains metadata about manifest item. Each manifest item has one or more labels // Including required "type" label. type EntryMetadata struct { - ID string + ID ID Length int Labels map[string]string ModTime time.Time diff --git a/repo/manifest/manifest_manager.go b/repo/manifest/manifest_manager.go index 4d2f616f6..bdb7534a1 100644 --- a/repo/manifest/manifest_manager.go +++ b/repo/manifest/manifest_manager.go @@ -15,7 +15,7 @@ "github.com/pkg/errors" "github.com/kopia/kopia/internal/repologging" - "github.com/kopia/kopia/repo/blob" + "github.com/kopia/kopia/repo/content" ) var log = repologging.Logger("kopia/manifest") @@ -23,33 +23,36 @@ // ErrNotFound is returned when the metadata item is not found. var ErrNotFound = errors.New("not found") -const manifestBlockPrefix = "m" -const autoCompactionBlockCount = 16 +const manifestContentPrefix = "m" +const autoCompactionContentCount = 16 -type blockManager interface { - GetBlock(ctx context.Context, blockID string) ([]byte, error) - WriteBlock(ctx context.Context, data []byte, prefix string) (string, error) - DeleteBlock(blockID string) error - ListBlocks(prefix string) ([]string, error) +type contentManager interface { + GetContent(ctx context.Context, contentID content.ID) ([]byte, error) + WriteContent(ctx context.Context, data []byte, prefix content.ID) (content.ID, error) + DeleteContent(contentID content.ID) error + ListContents(prefix content.ID) ([]content.ID, error) DisableIndexFlush() EnableIndexFlush() Flush(ctx context.Context) error } +// ID is a unique identifier of a single manifest. +type ID string + // Manager organizes JSON manifests of various kinds, including snapshot manifests type Manager struct { mu sync.Mutex - b blockManager + b contentManager initialized bool - pendingEntries map[string]*manifestEntry + pendingEntries map[ID]*manifestEntry - committedEntries map[string]*manifestEntry - committedBlockIDs map[string]bool + committedEntries map[ID]*manifestEntry + committedContentIDs map[content.ID]bool } -// Put serializes the provided payload to JSON and persists it. Returns unique handle that represents the object. -func (m *Manager) Put(ctx context.Context, labels map[string]string, payload interface{}) (string, error) { +// Put serializes the provided payload to JSON and persists it. Returns unique identifier that represents the manifest. +func (m *Manager) Put(ctx context.Context, labels map[string]string, payload interface{}) (ID, error) { if labels["type"] == "" { return "", errors.Errorf("'type' label is required") } @@ -71,7 +74,7 @@ func (m *Manager) Put(ctx context.Context, labels map[string]string, payload int } e := &manifestEntry{ - ID: hex.EncodeToString(random), + ID: ID(hex.EncodeToString(random)), ModTime: time.Now().UTC(), Labels: copyLabels(labels), Content: b, @@ -83,7 +86,7 @@ func (m *Manager) Put(ctx context.Context, labels map[string]string, payload int } // GetMetadata returns metadata about provided manifest item or ErrNotFound if the item can't be found. -func (m *Manager) GetMetadata(ctx context.Context, id string) (*EntryMetadata, error) { +func (m *Manager) GetMetadata(ctx context.Context, id ID) (*EntryMetadata, error) { if err := m.ensureInitialized(ctx); err != nil { return nil, err } @@ -110,7 +113,7 @@ func (m *Manager) GetMetadata(ctx context.Context, id string) (*EntryMetadata, e // Get retrieves the contents of the provided manifest item by deserializing it as JSON to provided object. // If the manifest is not found, returns ErrNotFound. -func (m *Manager) Get(ctx context.Context, id string, data interface{}) error { +func (m *Manager) Get(ctx context.Context, id ID, data interface{}) error { if err := m.ensureInitialized(ctx); err != nil { return err } @@ -128,7 +131,7 @@ func (m *Manager) Get(ctx context.Context, id string, data interface{}) error { } // GetRaw returns raw contents of the provided manifest (JSON bytes) or ErrNotFound if not found. -func (m *Manager) GetRaw(ctx context.Context, id string) ([]byte, error) { +func (m *Manager) GetRaw(ctx context.Context, id ID) ([]byte, error) { if err := m.ensureInitialized(ctx); err != nil { return nil, err } @@ -208,7 +211,7 @@ func (m *Manager) Flush(ctx context.Context) error { return err } -func (m *Manager) flushPendingEntriesLocked(ctx context.Context) (string, error) { +func (m *Manager) flushPendingEntriesLocked(ctx context.Context) (content.ID, error) { if len(m.pendingEntries) == 0 { return "", nil } @@ -225,7 +228,7 @@ func (m *Manager) flushPendingEntriesLocked(ctx context.Context) (string, error) mustSucceed(gz.Flush()) mustSucceed(gz.Close()) - blockID, err := m.b.WriteBlock(ctx, buf.Bytes(), manifestBlockPrefix) + contentID, err := m.b.WriteContent(ctx, buf.Bytes(), manifestContentPrefix) if err != nil { return "", err } @@ -235,9 +238,9 @@ func (m *Manager) flushPendingEntriesLocked(ctx context.Context) (string, error) delete(m.pendingEntries, e.ID) } - m.committedBlockIDs[blockID] = true + m.committedContentIDs[contentID] = true - return blockID, nil + return contentID, nil } func mustSucceed(e error) { @@ -247,7 +250,7 @@ func mustSucceed(e error) { } // Delete marks the specified manifest ID for deletion. -func (m *Manager) Delete(ctx context.Context, id string) error { +func (m *Manager) Delete(ctx context.Context, id ID) error { if err := m.ensureInitialized(ctx); err != nil { return err } @@ -264,53 +267,53 @@ func (m *Manager) Delete(ctx context.Context, id string) error { return nil } -// Refresh updates the committed blocks from the underlying storage. +// Refresh updates the committed contents from the underlying storage. func (m *Manager) Refresh(ctx context.Context) error { m.mu.Lock() defer m.mu.Unlock() - return m.loadCommittedBlocksLocked(ctx) + return m.loadCommittedContentsLocked(ctx) } -func (m *Manager) loadCommittedBlocksLocked(ctx context.Context) error { - log.Debugf("listing manifest blocks") +func (m *Manager) loadCommittedContentsLocked(ctx context.Context) error { + log.Debugf("listing manifest contents") for { - blocks, err := m.b.ListBlocks(manifestBlockPrefix) + contents, err := m.b.ListContents(manifestContentPrefix) if err != nil { - return errors.Wrap(err, "unable to list manifest blocks") + return errors.Wrap(err, "unable to list manifest contents") } - m.committedEntries = map[string]*manifestEntry{} - m.committedBlockIDs = map[string]bool{} + m.committedEntries = map[ID]*manifestEntry{} + m.committedContentIDs = map[content.ID]bool{} - log.Debugf("found %v manifest blocks", len(blocks)) - err = m.loadManifestBlocks(ctx, blocks) + log.Debugf("found %v manifest contents", len(contents)) + err = m.loadManifestContents(ctx, contents) if err == nil { // success break } - if err == blob.ErrBlobNotFound { + if err == content.ErrContentNotFound { // try again, lost a race with another manifest manager which just did compaction continue } - return errors.Wrap(err, "unable to load manifest blocks") + return errors.Wrap(err, "unable to load manifest contents") } if err := m.maybeCompactLocked(ctx); err != nil { - return errors.Errorf("error auto-compacting blocks") + return errors.Errorf("error auto-compacting contents") } return nil } -func (m *Manager) loadManifestBlocks(ctx context.Context, blockIDs []string) error { +func (m *Manager) loadManifestContents(ctx context.Context, contentIDs []content.ID) error { t0 := time.Now() - for _, b := range blockIDs { - m.committedBlockIDs[b] = true + for _, b := range contentIDs { + m.committedContentIDs[b] = true } - manifests, err := m.loadBlocksInParallel(ctx, blockIDs) + manifests, err := m.loadContentsInParallel(ctx, contentIDs) if err != nil { return err } @@ -321,22 +324,22 @@ func (m *Manager) loadManifestBlocks(ctx context.Context, blockIDs []string) err } } - // after merging, remove blocks marked as deleted. + // after merging, remove contents marked as deleted. for k, e := range m.committedEntries { if e.Deleted { delete(m.committedEntries, k) } } - log.Debugf("finished loading manifest blocks in %v.", time.Since(t0)) + log.Debugf("finished loading manifest contents in %v.", time.Since(t0)) return nil } -func (m *Manager) loadBlocksInParallel(ctx context.Context, blockIDs []string) ([]manifest, error) { - errors := make(chan error, len(blockIDs)) - manifests := make(chan manifest, len(blockIDs)) - ch := make(chan string, len(blockIDs)) +func (m *Manager) loadContentsInParallel(ctx context.Context, contentIDs []content.ID) ([]manifest, error) { + errors := make(chan error, len(contentIDs)) + manifests := make(chan manifest, len(contentIDs)) + ch := make(chan content.ID, len(contentIDs)) var wg sync.WaitGroup for i := 0; i < 8; i++ { @@ -346,21 +349,21 @@ func (m *Manager) loadBlocksInParallel(ctx context.Context, blockIDs []string) ( for blk := range ch { t1 := time.Now() - man, err := m.loadManifestBlock(ctx, blk) + man, err := m.loadManifestContent(ctx, blk) if err != nil { errors <- err - log.Debugf("block %v failed to be loaded by worker %v in %v: %v.", blk, workerID, time.Since(t1), err) + log.Debugf("manifest content %v failed to be loaded by worker %v in %v: %v.", blk, workerID, time.Since(t1), err) } else { - log.Debugf("block %v loaded by worker %v in %v.", blk, workerID, time.Since(t1)) + log.Debugf("manifest content %v loaded by worker %v in %v.", blk, workerID, time.Since(t1)) manifests <- man } } }(i) } - // feed block IDs for goroutines - for _, b := range blockIDs { + // feed manifest content IDs for goroutines + for _, b := range contentIDs { ch <- b } close(ch) @@ -383,9 +386,9 @@ func (m *Manager) loadBlocksInParallel(ctx context.Context, blockIDs []string) ( return man, nil } -func (m *Manager) loadManifestBlock(ctx context.Context, blockID string) (manifest, error) { +func (m *Manager) loadManifestContent(ctx context.Context, contentID content.ID) (manifest, error) { man := manifest{} - blk, err := m.b.GetBlock(ctx, blockID) + blk, err := m.b.GetContent(ctx, contentID) if err != nil { // do not wrap the error here, we want to propagate original ErrNotFound // which causes a retry if we lose list/delete race. @@ -394,17 +397,17 @@ func (m *Manager) loadManifestBlock(ctx context.Context, blockID string) (manife gz, err := gzip.NewReader(bytes.NewReader(blk)) if err != nil { - return man, errors.Wrapf(err, "unable to unpack block %q", blockID) + return man, errors.Wrapf(err, "unable to unpack manifest data %q", contentID) } if err := json.NewDecoder(gz).Decode(&man); err != nil { - return man, errors.Wrapf(err, "unable to parse block %q", blockID) + return man, errors.Wrapf(err, "unable to parse manifest %q", contentID) } return man, nil } -// Compact performs compaction of manifest blocks. +// Compact performs compaction of manifest contents. func (m *Manager) Compact(ctx context.Context) error { m.mu.Lock() defer m.mu.Unlock() @@ -413,26 +416,26 @@ func (m *Manager) Compact(ctx context.Context) error { } func (m *Manager) maybeCompactLocked(ctx context.Context) error { - if len(m.committedBlockIDs) < autoCompactionBlockCount { + if len(m.committedContentIDs) < autoCompactionContentCount { return nil } - log.Debugf("performing automatic compaction of %v blocks", len(m.committedBlockIDs)) + log.Debugf("performing automatic compaction of %v contents", len(m.committedContentIDs)) if err := m.compactLocked(ctx); err != nil { - return errors.Wrap(err, "unable to compact manifest blocks") + return errors.Wrap(err, "unable to compact manifest contents") } if err := m.b.Flush(ctx); err != nil { - return errors.Wrap(err, "unable to flush blocks after auto-compaction") + return errors.Wrap(err, "unable to flush contents after auto-compaction") } return nil } func (m *Manager) compactLocked(ctx context.Context) error { - log.Debugf("compactLocked: pendingEntries=%v blockIDs=%v", len(m.pendingEntries), len(m.committedBlockIDs)) + log.Debugf("compactLocked: pendingEntries=%v contentIDs=%v", len(m.pendingEntries), len(m.committedContentIDs)) - if len(m.committedBlockIDs) == 1 && len(m.pendingEntries) == 0 { + if len(m.committedContentIDs) == 1 && len(m.pendingEntries) == 0 { return nil } @@ -445,23 +448,23 @@ func (m *Manager) compactLocked(ctx context.Context) error { m.pendingEntries[e.ID] = e } - blockID, err := m.flushPendingEntriesLocked(ctx) + contentID, err := m.flushPendingEntriesLocked(ctx) if err != nil { return err } - // add the newly-created block to the list, could be duplicate - for b := range m.committedBlockIDs { - if b == blockID { - // do not delete block that was just written. + // add the newly-created content to the list, could be duplicate + for b := range m.committedContentIDs { + if b == contentID { + // do not delete content that was just written. continue } - if err := m.b.DeleteBlock(b); err != nil { - return errors.Wrapf(err, "unable to delete block %q", b) + if err := m.b.DeleteContent(b); err != nil { + return errors.Wrapf(err, "unable to delete content %q", b) } - delete(m.committedBlockIDs, b) + delete(m.committedContentIDs, b) } return nil @@ -487,7 +490,7 @@ func (m *Manager) ensureInitialized(ctx context.Context) error { return nil } - if err := m.loadCommittedBlocksLocked(ctx); err != nil { + if err := m.loadCommittedContentsLocked(ctx); err != nil { return err } @@ -503,13 +506,13 @@ func copyLabels(m map[string]string) map[string]string { return r } -// NewManager returns new manifest manager for the provided block manager. -func NewManager(ctx context.Context, b blockManager) (*Manager, error) { +// NewManager returns new manifest manager for the provided content manager. +func NewManager(ctx context.Context, b contentManager) (*Manager, error) { m := &Manager{ - b: b, - pendingEntries: map[string]*manifestEntry{}, - committedEntries: map[string]*manifestEntry{}, - committedBlockIDs: map[string]bool{}, + b: b, + pendingEntries: map[ID]*manifestEntry{}, + committedEntries: map[ID]*manifestEntry{}, + committedContentIDs: map[content.ID]bool{}, } return m, nil diff --git a/repo/manifest/manifest_manager_test.go b/repo/manifest/manifest_manager_test.go index e84e9162d..c9b97cb17 100644 --- a/repo/manifest/manifest_manager_test.go +++ b/repo/manifest/manifest_manager_test.go @@ -8,19 +8,14 @@ "testing" "time" - "github.com/pkg/errors" - "github.com/kopia/kopia/internal/blobtesting" - "github.com/kopia/kopia/repo/block" + "github.com/kopia/kopia/repo/content" ) func TestManifest(t *testing.T) { ctx := context.Background() data := blobtesting.DataMap{} - mgr, setupErr := newManagerForTesting(ctx, t, data) - if setupErr != nil { - t.Fatalf("unable to open block manager: %v", setupErr) - } + mgr := newManagerForTesting(ctx, t, data) item1 := map[string]int{"foo": 1, "bar": 2} item2 := map[string]int{"foo": 2, "bar": 3} @@ -36,13 +31,13 @@ func TestManifest(t *testing.T) { cases := []struct { criteria map[string]string - expected []string + expected []ID }{ - {map[string]string{"color": "red"}, []string{id1, id3}}, - {map[string]string{"color": "blue"}, []string{id2}}, + {map[string]string{"color": "red"}, []ID{id1, id3}}, + {map[string]string{"color": "blue"}, []ID{id2}}, {map[string]string{"color": "green"}, nil}, - {map[string]string{"color": "red", "shape": "square"}, []string{id3}}, - {map[string]string{"color": "blue", "shape": "square"}, []string{id2}}, + {map[string]string{"color": "red", "shape": "square"}, []ID{id3}}, + {map[string]string{"color": "blue", "shape": "square"}, []ID{id2}}, {map[string]string{"color": "red", "shape": "circle"}, nil}, } @@ -69,12 +64,9 @@ func TestManifest(t *testing.T) { verifyItem(ctx, t, mgr, id2, labels2, item2) verifyItem(ctx, t, mgr, id3, labels3, item3) - // flush underlying block manager and verify in new manifest manager. + // flush underlying content manager and verify in new manifest manager. mgr.b.Flush(ctx) - mgr2, setupErr := newManagerForTesting(ctx, t, data) - if setupErr != nil { - t.Fatalf("can't open block manager: %v", setupErr) - } + mgr2 := newManagerForTesting(ctx, t, data) for _, tc := range cases { verifyMatches(ctx, t, mgr2, tc.criteria, tc.expected) } @@ -96,7 +88,7 @@ func TestManifest(t *testing.T) { // still found in another verifyItem(ctx, t, mgr2, id3, labels3, item3) - if err := mgr2.loadCommittedBlocksLocked(ctx); err != nil { + if err := mgr2.loadCommittedContentsLocked(ctx); err != nil { t.Errorf("unable to load: %v", err) } @@ -104,7 +96,7 @@ func TestManifest(t *testing.T) { t.Errorf("can't compact: %v", err) } - blks, err := mgr.b.ListBlocks(manifestBlockPrefix) + blks, err := mgr.b.ListContents(manifestContentPrefix) if err != nil { t.Errorf("unable to list manifest blocks: %v", err) } @@ -114,10 +106,7 @@ func TestManifest(t *testing.T) { mgr.b.Flush(ctx) - mgr3, err := newManagerForTesting(ctx, t, data) - if err != nil { - t.Fatalf("can't open manager: %v", err) - } + mgr3 := newManagerForTesting(ctx, t, data) verifyItem(ctx, t, mgr3, id1, labels1, item1) verifyItem(ctx, t, mgr3, id2, labels2, item2) @@ -129,7 +118,7 @@ func TestManifestInitCorruptedBlock(t *testing.T) { data := blobtesting.DataMap{} st := blobtesting.NewMapStorage(data, nil, nil) - f := block.FormattingOptions{ + f := content.FormattingOptions{ Hash: "HMAC-SHA256-128", Encryption: "NONE", MaxPackSize: 100000, @@ -137,7 +126,7 @@ func TestManifestInitCorruptedBlock(t *testing.T) { } // write some data to storage - bm, err := block.NewManager(ctx, st, f, block.CachingOptions{}, nil) + bm, err := content.NewManager(ctx, st, f, content.CachingOptions{}, nil) if err != nil { t.Fatalf("err: %v", err) } @@ -160,8 +149,8 @@ func TestManifestInitCorruptedBlock(t *testing.T) { } } - // make a new block manager based on corrupted data. - bm, err = block.NewManager(ctx, st, f, block.CachingOptions{}, nil) + // make a new content manager based on corrupted data. + bm, err = content.NewManager(ctx, st, f, content.CachingOptions{}, nil) if err != nil { t.Fatalf("err: %v", err) } @@ -200,7 +189,7 @@ func TestManifestInitCorruptedBlock(t *testing.T) { } } -func addAndVerify(ctx context.Context, t *testing.T, mgr *Manager, labels map[string]string, data map[string]int) string { +func addAndVerify(ctx context.Context, t *testing.T, mgr *Manager, labels map[string]string, data map[string]int) ID { t.Helper() id, err := mgr.Put(ctx, labels, data) if err != nil { @@ -212,7 +201,7 @@ func addAndVerify(ctx context.Context, t *testing.T, mgr *Manager, labels map[st return id } -func verifyItem(ctx context.Context, t *testing.T, mgr *Manager, id string, labels map[string]string, data map[string]int) { +func verifyItem(ctx context.Context, t *testing.T, mgr *Manager, id ID, labels map[string]string, data map[string]int) { t.Helper() l, err := mgr.GetMetadata(ctx, id) @@ -235,7 +224,7 @@ func verifyItem(ctx context.Context, t *testing.T, mgr *Manager, id string, labe } } -func verifyItemNotFound(ctx context.Context, t *testing.T, mgr *Manager, id string) { +func verifyItemNotFound(ctx context.Context, t *testing.T, mgr *Manager, id ID) { t.Helper() _, err := mgr.GetMetadata(ctx, id) @@ -245,10 +234,10 @@ func verifyItemNotFound(ctx context.Context, t *testing.T, mgr *Manager, id stri } } -func verifyMatches(ctx context.Context, t *testing.T, mgr *Manager, labels map[string]string, expected []string) { +func verifyMatches(ctx context.Context, t *testing.T, mgr *Manager, labels map[string]string, expected []ID) { t.Helper() - var matches []string + var matches []ID items, err := mgr.Find(ctx, labels) if err != nil { t.Errorf("error in Find(): %v", err) @@ -257,37 +246,45 @@ func verifyMatches(ctx context.Context, t *testing.T, mgr *Manager, labels map[s for _, m := range items { matches = append(matches, m.ID) } - sort.Strings(matches) - sort.Strings(expected) + sortIDs(matches) + sortIDs(expected) if !reflect.DeepEqual(matches, expected) { t.Errorf("invalid matches for %v: %v, expected %v", labels, matches, expected) } } -func newManagerForTesting(ctx context.Context, t *testing.T, data blobtesting.DataMap) (*Manager, error) { +func sortIDs(s []ID) { + sort.Slice(s, func(i, j int) bool { + return s[i] < s[j] + }) +} + +func newManagerForTesting(ctx context.Context, t *testing.T, data blobtesting.DataMap) *Manager { st := blobtesting.NewMapStorage(data, nil, nil) - bm, err := block.NewManager(ctx, st, block.FormattingOptions{ + bm, err := content.NewManager(ctx, st, content.FormattingOptions{ Hash: "HMAC-SHA256-128", Encryption: "NONE", MaxPackSize: 100000, Version: 1, - }, block.CachingOptions{}, nil) + }, content.CachingOptions{}, nil) if err != nil { - return nil, errors.Wrap(err, "can't create block manager") + t.Fatalf("can't create content manager: %v", err) } - return NewManager(ctx, bm) + mm, err := NewManager(ctx, bm) + if err != nil { + t.Fatalf("can't create manifest manager: %v", err) + } + + return mm } func TestManifestInvalidPut(t *testing.T) { ctx := context.Background() data := blobtesting.DataMap{} - mgr, setupErr := newManagerForTesting(ctx, t, data) - if setupErr != nil { - t.Fatalf("unable to open block manager: %v", setupErr) - } + mgr := newManagerForTesting(ctx, t, data) cases := []struct { labels map[string]string @@ -311,10 +308,7 @@ func TestManifestAutoCompaction(t *testing.T) { data := blobtesting.DataMap{} for i := 0; i < 100; i++ { - mgr, setupErr := newManagerForTesting(ctx, t, data) - if setupErr != nil { - t.Fatalf("unable to open block manager: %v", setupErr) - } + mgr := newManagerForTesting(ctx, t, data) item1 := map[string]int{"foo": 1, "bar": 2} labels1 := map[string]string{"type": "item", "color": "red"} diff --git a/repo/manifest/serialized.go b/repo/manifest/serialized.go index 34be024c9..05afb1772 100644 --- a/repo/manifest/serialized.go +++ b/repo/manifest/serialized.go @@ -10,7 +10,7 @@ type manifest struct { } type manifestEntry struct { - ID string `json:"id"` + ID ID `json:"id"` Labels map[string]string `json:"labels"` ModTime time.Time `json:"modified"` Deleted bool `json:"deleted,omitempty"` diff --git a/repo/object/object_manager.go b/repo/object/object_manager.go index 7070a2a94..7eddffb7d 100644 --- a/repo/object/object_manager.go +++ b/repo/object/object_manager.go @@ -9,7 +9,7 @@ "github.com/pkg/errors" - "github.com/kopia/kopia/repo/block" + "github.com/kopia/kopia/repo/content" ) // ErrObjectNotFound is returned when an object cannot be found. @@ -23,23 +23,23 @@ type Reader interface { Length() int64 } -type blockManager interface { - BlockInfo(ctx context.Context, blockID string) (block.Info, error) - GetBlock(ctx context.Context, blockID string) ([]byte, error) - WriteBlock(ctx context.Context, data []byte, prefix string) (string, error) +type contentManager interface { + ContentInfo(ctx context.Context, contentID content.ID) (content.Info, error) + GetContent(ctx context.Context, contentID content.ID) ([]byte, error) + WriteContent(ctx context.Context, data []byte, prefix content.ID) (content.ID, error) } // Format describes the format of objects in a repository. type Format struct { - Splitter string `json:"splitter,omitempty"` // splitter used to break objects into storage blocks + Splitter string `json:"splitter,omitempty"` // splitter used to break objects into pieces of content } // Manager implements a content-addressable storage on top of blob storage. type Manager struct { Format Format - blockMgr blockManager - trace func(message string, args ...interface{}) + contentMgr contentManager + trace func(message string, args ...interface{}) newSplitter func() Splitter } @@ -86,19 +86,19 @@ func (om *Manager) Open(ctx context.Context, objectID ID) (Reader, error) { } // VerifyObject ensures that all objects backing ObjectID are present in the repository -// and returns the total length of the object and storage blocks of which it is composed. -func (om *Manager) VerifyObject(ctx context.Context, oid ID) (int64, []string, error) { - blocks := &blockTracker{} - l, err := om.verifyObjectInternal(ctx, oid, blocks) +// and returns the total length of the object and content IDs of which it is composed. +func (om *Manager) VerifyObject(ctx context.Context, oid ID) (int64, []content.ID, error) { + tracker := &contentIDTracker{} + l, err := om.verifyObjectInternal(ctx, oid, tracker) if err != nil { return 0, nil, err } - return l, blocks.blockIDs(), nil + return l, tracker.contentIDs(), nil } -func (om *Manager) verifyIndirectObjectInternal(ctx context.Context, indexObjectID ID, blocks *blockTracker) (int64, error) { - if _, err := om.verifyObjectInternal(ctx, indexObjectID, blocks); err != nil { +func (om *Manager) verifyIndirectObjectInternal(ctx context.Context, indexObjectID ID, tracker *contentIDTracker) (int64, error) { + if _, err := om.verifyObjectInternal(ctx, indexObjectID, tracker); err != nil { return 0, errors.Wrap(err, "unable to read index") } rd, err := om.Open(ctx, indexObjectID) @@ -113,7 +113,7 @@ func (om *Manager) verifyIndirectObjectInternal(ctx context.Context, indexObject } for i, m := range seekTable { - l, err := om.verifyObjectInternal(ctx, m.Object, blocks) + l, err := om.verifyObjectInternal(ctx, m.Object, tracker) if err != nil { return 0, err } @@ -127,17 +127,17 @@ func (om *Manager) verifyIndirectObjectInternal(ctx context.Context, indexObject return totalLength, nil } -func (om *Manager) verifyObjectInternal(ctx context.Context, oid ID, blocks *blockTracker) (int64, error) { +func (om *Manager) verifyObjectInternal(ctx context.Context, oid ID, tracker *contentIDTracker) (int64, error) { if indexObjectID, ok := oid.IndexObjectID(); ok { - return om.verifyIndirectObjectInternal(ctx, indexObjectID, blocks) + return om.verifyIndirectObjectInternal(ctx, indexObjectID, tracker) } - if blockID, ok := oid.BlockID(); ok { - p, err := om.blockMgr.BlockInfo(ctx, blockID) + if contentID, ok := oid.ContentID(); ok { + p, err := om.contentMgr.ContentInfo(ctx, contentID) if err != nil { return 0, err } - blocks.addBlock(blockID) + tracker.addContentID(contentID) return int64(p.Length), nil } @@ -153,12 +153,12 @@ type ManagerOptions struct { Trace func(message string, args ...interface{}) } -// NewObjectManager creates an ObjectManager with the specified block manager and format. -func NewObjectManager(ctx context.Context, bm blockManager, f Format, opts ManagerOptions) (*Manager, error) { +// NewObjectManager creates an ObjectManager with the specified content manager and format. +func NewObjectManager(ctx context.Context, bm contentManager, f Format, opts ManagerOptions) (*Manager, error) { om := &Manager{ - blockMgr: bm, - Format: f, - trace: nullTrace, + contentMgr: bm, + Format: f, + trace: nullTrace, } splitterID := f.Splitter @@ -210,13 +210,13 @@ func (om *Manager) flattenListChunk(rawReader io.Reader) ([]indirectObjectEntry, } func (om *Manager) newRawReader(ctx context.Context, objectID ID) (Reader, error) { - if blockID, ok := objectID.BlockID(); ok { - payload, err := om.blockMgr.GetBlock(ctx, blockID) - if err == block.ErrBlockNotFound { + if contentID, ok := objectID.ContentID(); ok { + payload, err := om.contentMgr.GetContent(ctx, contentID) + if err == content.ErrContentNotFound { return nil, ErrObjectNotFound } if err != nil { - return nil, errors.Wrap(err, "unexpected block error") + return nil, errors.Wrap(err, "unexpected content error") } return newObjectReaderWithData(payload), nil diff --git a/repo/object/object_manager_test.go b/repo/object/object_manager_test.go index 7b7203323..4b143685c 100644 --- a/repo/object/object_manager_test.go +++ b/repo/object/object_manager_test.go @@ -16,58 +16,58 @@ "testing" "github.com/kopia/kopia/repo/blob" - "github.com/kopia/kopia/repo/block" + "github.com/kopia/kopia/repo/content" ) -type fakeBlockManager struct { +type fakeContentManager struct { mu sync.Mutex - data map[string][]byte + data map[content.ID][]byte } -func (f *fakeBlockManager) GetBlock(ctx context.Context, blockID string) ([]byte, error) { +func (f *fakeContentManager) GetContent(ctx context.Context, contentID content.ID) ([]byte, error) { f.mu.Lock() defer f.mu.Unlock() - if d, ok := f.data[blockID]; ok { + if d, ok := f.data[contentID]; ok { return append([]byte(nil), d...), nil } - return nil, block.ErrBlockNotFound + return nil, content.ErrContentNotFound } -func (f *fakeBlockManager) WriteBlock(ctx context.Context, data []byte, prefix string) (string, error) { +func (f *fakeContentManager) WriteContent(ctx context.Context, data []byte, prefix content.ID) (content.ID, error) { h := sha256.New() h.Write(data) //nolint:errcheck - blockID := prefix + string(hex.EncodeToString(h.Sum(nil))) + contentID := prefix + content.ID(hex.EncodeToString(h.Sum(nil))) f.mu.Lock() defer f.mu.Unlock() - f.data[blockID] = append([]byte(nil), data...) - return blockID, nil + f.data[contentID] = append([]byte(nil), data...) + return contentID, nil } -func (f *fakeBlockManager) BlockInfo(ctx context.Context, blockID string) (block.Info, error) { +func (f *fakeContentManager) ContentInfo(ctx context.Context, contentID content.ID) (content.Info, error) { f.mu.Lock() defer f.mu.Unlock() - if d, ok := f.data[blockID]; ok { - return block.Info{BlockID: blockID, Length: uint32(len(d))}, nil + if d, ok := f.data[contentID]; ok { + return content.Info{ID: contentID, Length: uint32(len(d))}, nil } - return block.Info{}, blob.ErrBlobNotFound + return content.Info{}, blob.ErrBlobNotFound } -func (f *fakeBlockManager) Flush(ctx context.Context) error { +func (f *fakeContentManager) Flush(ctx context.Context) error { return nil } -func setupTest(t *testing.T) (map[string][]byte, *Manager) { - return setupTestWithData(t, map[string][]byte{}, ManagerOptions{}) +func setupTest(t *testing.T) (map[content.ID][]byte, *Manager) { + return setupTestWithData(t, map[content.ID][]byte{}, ManagerOptions{}) } -func setupTestWithData(t *testing.T, data map[string][]byte, opts ManagerOptions) (map[string][]byte, *Manager) { - r, err := NewObjectManager(context.Background(), &fakeBlockManager{data: data}, Format{ +func setupTestWithData(t *testing.T, data map[content.ID][]byte, opts ManagerOptions) (map[content.ID][]byte, *Manager) { + r, err := NewObjectManager(context.Background(), &fakeContentManager{data: data}, Format{ Splitter: "FIXED-1M", }, opts) if err != nil { @@ -109,7 +109,7 @@ func TestWriters(t *testing.T) { t.Errorf("incorrect result for %v, expected: %v got: %v", c.data, c.objectID.String(), result.String()) } - if _, ok := c.objectID.BlockID(); !ok { + if _, ok := c.objectID.ContentID(); !ok { if len(data) != 0 { t.Errorf("unexpected data written to the storage: %v", data) } @@ -162,18 +162,18 @@ func TestIndirection(t *testing.T) { splitterFactory := newFixedSplitterFactory(1000) cases := []struct { dataLength int - expectedBlockCount int + expectedBlobCount int expectedIndirection int }{ - {dataLength: 200, expectedBlockCount: 1, expectedIndirection: 0}, - {dataLength: 1000, expectedBlockCount: 1, expectedIndirection: 0}, - {dataLength: 1001, expectedBlockCount: 3, expectedIndirection: 1}, - // 1 block of 1000 zeros, 1 block of 5 zeros + 1 index blob - {dataLength: 3005, expectedBlockCount: 3, expectedIndirection: 1}, - // 1 block of 1000 zeros + 1 index blob - {dataLength: 4000, expectedBlockCount: 2, expectedIndirection: 1}, - // 1 block of 1000 zeros + 1 index blob - {dataLength: 10000, expectedBlockCount: 2, expectedIndirection: 1}, + {dataLength: 200, expectedBlobCount: 1, expectedIndirection: 0}, + {dataLength: 1000, expectedBlobCount: 1, expectedIndirection: 0}, + {dataLength: 1001, expectedBlobCount: 3, expectedIndirection: 1}, + // 1 blob of 1000 zeros, 1 blob of 5 zeros + 1 index blob + {dataLength: 3005, expectedBlobCount: 3, expectedIndirection: 1}, + // 1 blob of 1000 zeros + 1 index blob + {dataLength: 4000, expectedBlobCount: 2, expectedIndirection: 1}, + // 1 blob of 1000 zeros + 1 index blob + {dataLength: 10000, expectedBlobCount: 2, expectedIndirection: 1}, } for _, c := range cases { @@ -197,8 +197,8 @@ func TestIndirection(t *testing.T) { t.Errorf("incorrect indirection level for size: %v: %v, expected %v", c.dataLength, indirectionLevel(result), c.expectedIndirection) } - if got, want := len(data), c.expectedBlockCount; got != want { - t.Errorf("unexpected block count for %v: %v, expected %v", c.dataLength, got, want) + if got, want := len(data), c.expectedBlobCount; got != want { + t.Errorf("unexpected blob count for %v: %v, expected %v", c.dataLength, got, want) } l, b, err := om.VerifyObject(ctx, result) @@ -210,8 +210,8 @@ func TestIndirection(t *testing.T) { t.Errorf("got invalid byte count for %q: %v, wanted %v", result, got, want) } - if got, want := len(b), c.expectedBlockCount; got != want { - t.Errorf("invalid block count for %v, got %v, wanted %v", result, got, want) + if got, want := len(b), c.expectedBlobCount; got != want { + t.Errorf("invalid blob count for %v, got %v, wanted %v", result, got, want) } verifyIndirectBlock(ctx, t, om, result) diff --git a/repo/object/object_reader.go b/repo/object/object_reader.go index a010803b3..6bc8fd4f3 100644 --- a/repo/object/object_reader.go +++ b/repo/object/object_reader.go @@ -68,14 +68,14 @@ func (r *objectReader) Read(buffer []byte) (int, error) { func (r *objectReader) openCurrentChunk() error { st := r.seekTable[r.currentChunkIndex] - blockData, err := r.repo.Open(r.ctx, st.Object) + rd, err := r.repo.Open(r.ctx, st.Object) if err != nil { return err } - defer blockData.Close() //nolint:errcheck + defer rd.Close() //nolint:errcheck b := make([]byte, st.Length) - if _, err := io.ReadFull(blockData, b); err != nil { + if _, err := io.ReadFull(rd, b); err != nil { return err } diff --git a/repo/object/object_writer.go b/repo/object/object_writer.go index eaf77f779..1a97654fb 100644 --- a/repo/object/object_writer.go +++ b/repo/object/object_writer.go @@ -8,6 +8,8 @@ "sync" "github.com/pkg/errors" + + "github.com/kopia/kopia/repo/content" ) // Writer allows writing content to the storage and supports automatic deduplication and encryption @@ -18,27 +20,27 @@ type Writer interface { Result() (ID, error) } -type blockTracker struct { - mu sync.Mutex - blocks map[string]bool +type contentIDTracker struct { + mu sync.Mutex + contents map[content.ID]bool } -func (t *blockTracker) addBlock(blockID string) { +func (t *contentIDTracker) addContentID(contentID content.ID) { t.mu.Lock() defer t.mu.Unlock() - if t.blocks == nil { - t.blocks = make(map[string]bool) + if t.contents == nil { + t.contents = make(map[content.ID]bool) } - t.blocks[blockID] = true + t.contents[contentID] = true } -func (t *blockTracker) blockIDs() []string { +func (t *contentIDTracker) contentIDs() []content.ID { t.mu.Lock() defer t.mu.Unlock() - result := make([]string, 0, len(t.blocks)) - for k := range t.blocks { + result := make([]content.ID, 0, len(t.contents)) + for k := range t.contents { result = append(result, k) } return result @@ -48,12 +50,12 @@ type objectWriter struct { ctx context.Context repo *Manager - prefix string + prefix content.ID buffer bytes.Buffer totalLength int64 currentPosition int64 - blockIndex []indirectObjectEntry + indirectIndex []indirectObjectEntry description string @@ -83,35 +85,35 @@ func (w *objectWriter) Write(data []byte) (n int, err error) { func (w *objectWriter) flushBuffer() error { length := w.buffer.Len() - chunkID := len(w.blockIndex) - w.blockIndex = append(w.blockIndex, indirectObjectEntry{}) - w.blockIndex[chunkID].Start = w.currentPosition - w.blockIndex[chunkID].Length = int64(length) + chunkID := len(w.indirectIndex) + w.indirectIndex = append(w.indirectIndex, indirectObjectEntry{}) + w.indirectIndex[chunkID].Start = w.currentPosition + w.indirectIndex[chunkID].Length = int64(length) w.currentPosition += int64(length) var b2 bytes.Buffer w.buffer.WriteTo(&b2) //nolint:errcheck w.buffer.Reset() - blockID, err := w.repo.blockMgr.WriteBlock(w.ctx, b2.Bytes(), w.prefix) - w.repo.trace("OBJECT_WRITER(%q) stored %v (%v bytes)", w.description, blockID, length) + contentID, err := w.repo.contentMgr.WriteContent(w.ctx, b2.Bytes(), w.prefix) + w.repo.trace("OBJECT_WRITER(%q) stored %v (%v bytes)", w.description, contentID, length) if err != nil { return errors.Wrapf(err, "error when flushing chunk %d of %s", chunkID, w.description) } - w.blockIndex[chunkID].Object = DirectObjectID(blockID) + w.indirectIndex[chunkID].Object = DirectObjectID(contentID) return nil } func (w *objectWriter) Result() (ID, error) { - if w.buffer.Len() > 0 || len(w.blockIndex) == 0 { + if w.buffer.Len() > 0 || len(w.indirectIndex) == 0 { if err := w.flushBuffer(); err != nil { return "", err } } - if len(w.blockIndex) == 1 { - return w.blockIndex[0].Object, nil + if len(w.indirectIndex) == 1 { + return w.indirectIndex[0].Object, nil } iw := &objectWriter{ @@ -124,11 +126,11 @@ func (w *objectWriter) Result() (ID, error) { ind := indirectObject{ StreamID: "kopia:indirect", - Entries: w.blockIndex, + Entries: w.indirectIndex, } if err := json.NewEncoder(iw).Encode(ind); err != nil { - return "", errors.Wrap(err, "unable to write indirect block index") + return "", errors.Wrap(err, "unable to write indirect object index") } oid, err := iw.Result() if err != nil { @@ -140,5 +142,5 @@ func (w *objectWriter) Result() (ID, error) { // WriterOptions can be passed to Repository.NewWriter() type WriterOptions struct { Description string - Prefix string // empty string or a single-character ('g'..'z') + Prefix content.ID // empty string or a single-character ('g'..'z') } diff --git a/repo/object/objectid.go b/repo/object/objectid.go index 6b4d8f37d..a26a6cb66 100644 --- a/repo/object/objectid.go +++ b/repo/object/objectid.go @@ -5,6 +5,8 @@ "strings" "github.com/pkg/errors" + + "github.com/kopia/kopia/repo/content" ) // ID is an identifier of a repository object. Repository objects can be stored. @@ -33,16 +35,16 @@ func (i ID) IndexObjectID() (ID, bool) { return "", false } -// BlockID returns the block ID of the underlying content storage block. -func (i ID) BlockID() (string, bool) { +// ContentID returns the ID of the underlying content. +func (i ID) ContentID() (content.ID, bool) { if strings.HasPrefix(string(i), "D") { - return string(i[1:]), true + return content.ID(i[1:]), true } if strings.HasPrefix(string(i), "I") { return "", false } - return string(i), true + return content.ID(i), true } // Validate checks the ID format for validity and reports any errors. @@ -55,21 +57,21 @@ func (i ID) Validate() error { return nil } - if blockID, ok := i.BlockID(); ok { - if len(blockID) < 2 { - return errors.Errorf("missing block ID") + if contentID, ok := i.ContentID(); ok { + if len(contentID) < 2 { + return errors.Errorf("missing content ID") } // odd length - firstcharacter must be a single character between 'g' and 'z' - if len(blockID)%2 == 1 { - if blockID[0] < 'g' || blockID[0] > 'z' { - return errors.Errorf("invalid block ID prefix: %v", blockID) + if len(contentID)%2 == 1 { + if contentID[0] < 'g' || contentID[0] > 'z' { + return errors.Errorf("invalid content ID prefix: %v", contentID) } - blockID = blockID[1:] + contentID = contentID[1:] } - if _, err := hex.DecodeString(blockID); err != nil { - return errors.Errorf("invalid blockID suffix, must be base-16 encoded: %v", blockID) + if _, err := hex.DecodeString(string(contentID)); err != nil { + return errors.Errorf("invalid contentID suffix, must be base-16 encoded: %v", contentID) } return nil @@ -79,8 +81,8 @@ func (i ID) Validate() error { } // DirectObjectID returns direct object ID based on the provided block ID. -func DirectObjectID(blockID string) ID { - return ID(blockID) +func DirectObjectID(contentID content.ID) ID { + return ID(contentID) } // IndirectObjectID returns indirect object ID based on the underlying index object ID. diff --git a/repo/open.go b/repo/open.go index c54e3cc4c..c0086a5a1 100644 --- a/repo/open.go +++ b/repo/open.go @@ -11,7 +11,7 @@ "github.com/kopia/kopia/internal/repologging" "github.com/kopia/kopia/repo/blob" "github.com/kopia/kopia/repo/blob/logging" - "github.com/kopia/kopia/repo/block" + "github.com/kopia/kopia/repo/content" "github.com/kopia/kopia/repo/manifest" "github.com/kopia/kopia/repo/object" ) @@ -75,20 +75,20 @@ func Open(ctx context.Context, configFile string, password string, options *Opti } // OpenWithConfig opens the repository with a given configuration, avoiding the need for a config file. -func OpenWithConfig(ctx context.Context, st blob.Storage, lc *LocalConfig, password string, options *Options, caching block.CachingOptions) (*Repository, error) { - log.Debugf("reading encrypted format block") - // Read cache block, potentially from cache. - fb, err := readAndCacheFormatBlockBytes(ctx, st, caching.CacheDirectory) +func OpenWithConfig(ctx context.Context, st blob.Storage, lc *LocalConfig, password string, options *Options, caching content.CachingOptions) (*Repository, error) { + log.Debugf("reading encrypted format blob") + // Read format blob, potentially from cache. + fb, err := readAndCacheFormatBlobBytes(ctx, st, caching.CacheDirectory) if err != nil { - return nil, errors.Wrap(err, "unable to read format block") + return nil, errors.Wrap(err, "unable to read format blob") } - f, err := parseFormatBlock(fb) + f, err := parseFormatBlob(fb) if err != nil { - return nil, errors.Wrap(err, "can't parse format block") + return nil, errors.Wrap(err, "can't parse format blob") } - fb, err = addFormatBlockChecksumAndLength(fb) + fb, err = addFormatBlobChecksumAndLength(fb) if err != nil { return nil, errors.Errorf("unable to add checksum") } @@ -110,39 +110,39 @@ func OpenWithConfig(ctx context.Context, st blob.Storage, lc *LocalConfig, passw fo.MaxPackSize = 20 << 20 // 20 MB } - log.Debugf("initializing block manager") - bm, err := block.NewManager(ctx, st, fo, caching, fb) + log.Debugf("initializing content-addressable storage manager") + cm, err := content.NewManager(ctx, st, fo, caching, fb) if err != nil { - return nil, errors.Wrap(err, "unable to open block manager") + return nil, errors.Wrap(err, "unable to open content manager") } log.Debugf("initializing object manager") - om, err := object.NewObjectManager(ctx, bm, repoConfig.Format, options.ObjectManagerOptions) + om, err := object.NewObjectManager(ctx, cm, repoConfig.Format, options.ObjectManagerOptions) if err != nil { return nil, errors.Wrap(err, "unable to open object manager") } log.Debugf("initializing manifest manager") - manifests, err := manifest.NewManager(ctx, bm) + manifests, err := manifest.NewManager(ctx, cm) if err != nil { return nil, errors.Wrap(err, "unable to open manifests") } return &Repository{ - Blocks: bm, + Content: cm, Objects: om, Blobs: st, Manifests: manifests, CacheDirectory: caching.CacheDirectory, UniqueID: f.UniqueID, - formatBlock: f, - masterKey: masterKey, + formatBlob: f, + masterKey: masterKey, }, nil } // SetCachingConfig changes caching configuration for a given repository config file. -func SetCachingConfig(ctx context.Context, configFile string, opt block.CachingOptions) error { +func SetCachingConfig(ctx context.Context, configFile string, opt content.CachingOptions) error { configFile, err := filepath.Abs(configFile) if err != nil { return err @@ -158,14 +158,14 @@ func SetCachingConfig(ctx context.Context, configFile string, opt block.CachingO return errors.Wrap(err, "cannot open storage") } - fb, err := readAndCacheFormatBlockBytes(ctx, st, "") + fb, err := readAndCacheFormatBlobBytes(ctx, st, "") if err != nil { - return errors.Wrap(err, "can't read format block") + return errors.Wrap(err, "can't read format blob") } - f, err := parseFormatBlock(fb) + f, err := parseFormatBlob(fb) if err != nil { - return errors.Wrap(err, "can't parse format block") + return errors.Wrap(err, "can't parse format blob") } if err = setupCaching(configFile, lc, opt, f.UniqueID); err != nil { @@ -184,7 +184,7 @@ func SetCachingConfig(ctx context.Context, configFile string, opt block.CachingO return nil } -func readAndCacheFormatBlockBytes(ctx context.Context, st blob.Storage, cacheDirectory string) ([]byte, error) { +func readAndCacheFormatBlobBytes(ctx context.Context, st blob.Storage, cacheDirectory string) ([]byte, error) { cachedFile := filepath.Join(cacheDirectory, "kopia.repository") if cacheDirectory != "" { b, err := ioutil.ReadFile(cachedFile) diff --git a/repo/repository.go b/repo/repository.go index b564f7f52..4692827e4 100644 --- a/repo/repository.go +++ b/repo/repository.go @@ -7,7 +7,7 @@ "github.com/pkg/errors" "github.com/kopia/kopia/repo/blob" - "github.com/kopia/kopia/repo/block" + "github.com/kopia/kopia/repo/content" "github.com/kopia/kopia/repo/manifest" "github.com/kopia/kopia/repo/object" ) @@ -15,7 +15,7 @@ // Repository represents storage where both content-addressable and user-addressable data is kept. type Repository struct { Blobs blob.Storage - Blocks *block.Manager + Content *content.Manager Objects *object.Manager Manifests *manifest.Manager UniqueID []byte @@ -23,8 +23,8 @@ type Repository struct { ConfigFile string CacheDirectory string - formatBlock *formatBlock - masterKey []byte + formatBlob *formatBlob + masterKey []byte } // Close closes the repository and releases all resources. @@ -32,8 +32,8 @@ func (r *Repository) Close(ctx context.Context) error { if err := r.Manifests.Flush(ctx); err != nil { return errors.Wrap(err, "error flushing manifests") } - if err := r.Blocks.Flush(ctx); err != nil { - return errors.Wrap(err, "error closing blocks") + if err := r.Content.Flush(ctx); err != nil { + return errors.Wrap(err, "error closing content-addressable storage manager") } if err := r.Blobs.Close(ctx); err != nil { return errors.Wrap(err, "error closing blob storage") @@ -47,21 +47,21 @@ func (r *Repository) Flush(ctx context.Context) error { return err } - return r.Blocks.Flush(ctx) + return r.Content.Flush(ctx) } // Refresh periodically makes external changes visible to repository. func (r *Repository) Refresh(ctx context.Context) error { - updated, err := r.Blocks.Refresh(ctx) + updated, err := r.Content.Refresh(ctx) if err != nil { - return errors.Wrap(err, "error refreshing block index") + return errors.Wrap(err, "error refreshing content index") } if !updated { return nil } - log.Debugf("block index refreshed") + log.Debugf("content index refreshed") if err := r.Manifests.Refresh(ctx); err != nil { return errors.Wrap(err, "error reloading manifests") diff --git a/repo/repository_test.go b/repo/repository_test.go index d71e5349a..a737819d9 100644 --- a/repo/repository_test.go +++ b/repo/repository_test.go @@ -13,7 +13,7 @@ "github.com/kopia/kopia/internal/repotesting" "github.com/kopia/kopia/repo" - "github.com/kopia/kopia/repo/block" + "github.com/kopia/kopia/repo/content" "github.com/kopia/kopia/repo/object" ) @@ -50,7 +50,7 @@ func TestWriters(t *testing.T) { t.Errorf("incorrect result for %v, expected: %v got: %v", c.data, c.objectID.String(), result.String()) } - env.Repository.Blocks.Flush(ctx) + env.Repository.Content.Flush(ctx) } } @@ -95,7 +95,7 @@ func TestPackingSimple(t *testing.T) { oid2c := writeObject(ctx, t, env.Repository, []byte(content2), "packed-object-2c") oid1c := writeObject(ctx, t, env.Repository, []byte(content1), "packed-object-1c") - env.Repository.Blocks.Flush(ctx) + env.Repository.Content.Flush(ctx) if got, want := oid1a.String(), oid1b.String(); got != want { t.Errorf("oid1a(%q) != oid1b(%q)", got, want) @@ -113,7 +113,7 @@ func TestPackingSimple(t *testing.T) { t.Errorf("oid3a(%q) != oid3b(%q)", got, want) } - env.VerifyStorageBlockCount(t, 3) + env.VerifyBlobCount(t, 3) env.MustReopen(t) @@ -121,7 +121,7 @@ func TestPackingSimple(t *testing.T) { verify(ctx, t, env.Repository, oid2a, []byte(content2), "packed-object-2") verify(ctx, t, env.Repository, oid3a, []byte(content3), "packed-object-3") - if err := env.Repository.Blocks.CompactIndexes(ctx, block.CompactOptions{MinSmallBlocks: 1, MaxSmallBlocks: 1}); err != nil { + if err := env.Repository.Content.CompactIndexes(ctx, content.CompactOptions{MinSmallBlobs: 1, MaxSmallBlobs: 1}); err != nil { t.Errorf("optimize error: %v", err) } @@ -131,7 +131,7 @@ func TestPackingSimple(t *testing.T) { verify(ctx, t, env.Repository, oid2a, []byte(content2), "packed-object-2") verify(ctx, t, env.Repository, oid3a, []byte(content3), "packed-object-3") - if err := env.Repository.Blocks.CompactIndexes(ctx, block.CompactOptions{MinSmallBlocks: 1, MaxSmallBlocks: 1}); err != nil { + if err := env.Repository.Content.CompactIndexes(ctx, content.CompactOptions{MinSmallBlobs: 1, MaxSmallBlobs: 1}); err != nil { t.Errorf("optimize error: %v", err) } diff --git a/repo/upgrade.go b/repo/upgrade.go index ef934bc9b..4a9bbfc59 100644 --- a/repo/upgrade.go +++ b/repo/upgrade.go @@ -8,7 +8,7 @@ // Upgrade upgrades repository data structures to the latest version. func (r *Repository) Upgrade(ctx context.Context) error { - f := r.formatBlock + f := r.formatBlob log.Debug("decrypting format...") repoConfig, err := f.decryptFormatBytes(r.masterKey) @@ -29,6 +29,6 @@ func (r *Repository) Upgrade(ctx context.Context) error { return errors.Errorf("unable to encrypt format bytes") } - log.Infof("writing updated format block...") - return writeFormatBlock(ctx, r.Blobs, f) + log.Infof("writing updated format content...") + return writeFormatBlob(ctx, r.Blobs, f) } diff --git a/site/content/docs/Architecture/_index.md b/site/content/docs/Architecture/_index.md index 3395b6470..9a240890f 100644 --- a/site/content/docs/Architecture/_index.md +++ b/site/content/docs/Architecture/_index.md @@ -49,7 +49,7 @@ Pack files in blob storage have random names and don't reveal anything about the CABS is not meant to be used directly, instead it's a building block for object storage (CAOS) and manifest storage layers (LAMS) described below. -The API for CABS can be found in https://godoc.org/github.com/kopia/kopia/repo/block +The API for CABS can be found in https://godoc.org/github.com/kopia/kopia/repo/content ### Content-Addressable Object Storage (CAOS) @@ -63,7 +63,7 @@ Object IDs can also have an optional single-letter prefix `g..z` that helps quic * `m` represents manifest block (e.g. `m0bf4da00801bd8c6ecfb66cffa67f32c`) * `h` represents hash-cache (e.g. `h2e88080490a83c4b1cb344d861a3f537`) -To represent objects larger than the size of a single CABS block, Kopia links together multiple blocks via special indirect JSON block. Such blocks are distinguished from regular blocks by the `I` prefix. For example very large hash-cache object might have an identifier such as `Ih746f0a60f744d0a69e397a6128356331` and JSON content: +To represent objects larger than the size of a single CABS block, Kopia links together multiple blocks via special indirect JSON content. Such blocks are distinguished from regular blocks by the `I` prefix. For example very large hash-cache object might have an identifier such as `Ih746f0a60f744d0a69e397a6128356331` and JSON content: ```json {"stream":"kopia:indirect","entries":[ diff --git a/snapshot/manager.go b/snapshot/manager.go index a28772b75..31bb52e81 100644 --- a/snapshot/manager.go +++ b/snapshot/manager.go @@ -58,7 +58,7 @@ func ListSnapshots(ctx context.Context, rep *repo.Repository, si SourceInfo) ([] } // loadSnapshot loads and parses a snapshot with a given ID. -func loadSnapshot(ctx context.Context, rep *repo.Repository, manifestID string) (*Manifest, error) { +func loadSnapshot(ctx context.Context, rep *repo.Repository, manifestID manifest.ID) (*Manifest, error) { sm := &Manifest{} if err := rep.Manifests.Get(ctx, manifestID, sm); err != nil { return nil, errors.Wrap(err, "unable to find manifest entries") @@ -69,7 +69,7 @@ func loadSnapshot(ctx context.Context, rep *repo.Repository, manifestID string) } // SaveSnapshot persists given snapshot manifest and returns manifest ID. -func SaveSnapshot(ctx context.Context, rep *repo.Repository, manifest *Manifest) (string, error) { +func SaveSnapshot(ctx context.Context, rep *repo.Repository, manifest *Manifest) (manifest.ID, error) { if manifest.Source.Host == "" { return "", errors.New("missing host") } @@ -89,13 +89,13 @@ func SaveSnapshot(ctx context.Context, rep *repo.Repository, manifest *Manifest) } // LoadSnapshots efficiently loads and parses a given list of snapshot IDs. -func LoadSnapshots(ctx context.Context, rep *repo.Repository, names []string) ([]*Manifest, error) { - result := make([]*Manifest, len(names)) +func LoadSnapshots(ctx context.Context, rep *repo.Repository, manifestIDs []manifest.ID) ([]*Manifest, error) { + result := make([]*Manifest, len(manifestIDs)) sem := make(chan bool, 50) - for i, n := range names { + for i, n := range manifestIDs { sem <- true - go func(i int, n string) { + go func(i int, n manifest.ID) { defer func() { <-sem }() m, err := loadSnapshot(ctx, rep, n) @@ -123,7 +123,7 @@ func LoadSnapshots(ctx context.Context, rep *repo.Repository, names []string) ([ } // ListSnapshotManifests returns the list of snapshot manifests for a given source or all sources if nil. -func ListSnapshotManifests(ctx context.Context, rep *repo.Repository, src *SourceInfo) ([]string, error) { +func ListSnapshotManifests(ctx context.Context, rep *repo.Repository, src *SourceInfo) ([]manifest.ID, error) { labels := map[string]string{ "type": "snapshot", } @@ -139,8 +139,8 @@ func ListSnapshotManifests(ctx context.Context, rep *repo.Repository, src *Sourc return entryIDs(entries), nil } -func entryIDs(entries []*manifest.EntryMetadata) []string { - var ids []string +func entryIDs(entries []*manifest.EntryMetadata) []manifest.ID { + var ids []manifest.ID for _, e := range entries { ids = append(ids, e.ID) } diff --git a/snapshot/manifest.go b/snapshot/manifest.go index ea70838c7..ff866a85d 100644 --- a/snapshot/manifest.go +++ b/snapshot/manifest.go @@ -7,13 +7,14 @@ "time" "github.com/kopia/kopia/fs" + "github.com/kopia/kopia/repo/manifest" "github.com/kopia/kopia/repo/object" ) // Manifest represents information about a single point-in-time filesystem snapshot. type Manifest struct { - ID string `json:"-"` - Source SourceInfo `json:"source"` + ID manifest.ID `json:"-"` + Source SourceInfo `json:"source"` Description string `json:"description"` StartTime time.Time `json:"startTime"` diff --git a/snapshot/policy/policy_manager.go b/snapshot/policy/policy_manager.go index f3e57f030..a8723aea7 100644 --- a/snapshot/policy/policy_manager.go +++ b/snapshot/policy/policy_manager.go @@ -152,7 +152,7 @@ func RemovePolicy(ctx context.Context, rep *repo.Repository, si snapshot.SourceI } // GetPolicyByID gets the policy for a given unique ID or ErrPolicyNotFound if not found. -func GetPolicyByID(ctx context.Context, rep *repo.Repository, id string) (*Policy, error) { +func GetPolicyByID(ctx context.Context, rep *repo.Repository, id manifest.ID) (*Policy, error) { p := &Policy{} if err := rep.Manifests.Get(ctx, id, &p); err != nil { if err == manifest.ErrNotFound { @@ -187,7 +187,7 @@ func ListPolicies(ctx context.Context, rep *repo.Repository) ([]*Policy, error) } pol.Labels = md.Labels - pol.Labels["id"] = id.ID + pol.Labels["id"] = string(id.ID) policies = append(policies, pol) } diff --git a/snapshot/snapshot_test.go b/snapshot/snapshot_test.go index 9b27f0199..94d0d1b8c 100644 --- a/snapshot/snapshot_test.go +++ b/snapshot/snapshot_test.go @@ -10,6 +10,7 @@ "github.com/kopia/kopia/internal/repotesting" "github.com/kopia/kopia/repo" + "github.com/kopia/kopia/repo/manifest" "github.com/kopia/kopia/snapshot" ) @@ -39,8 +40,8 @@ func TestSnapshotsAPI(t *testing.T) { Description: "some-description", } id1 := mustSaveSnapshot(t, env.Repository, manifest1) - verifySnapshotManifestIDs(t, env.Repository, nil, []string{id1}) - verifySnapshotManifestIDs(t, env.Repository, &src1, []string{id1}) + verifySnapshotManifestIDs(t, env.Repository, nil, []manifest.ID{id1}) + verifySnapshotManifestIDs(t, env.Repository, &src1, []manifest.ID{id1}) verifySnapshotManifestIDs(t, env.Repository, &src2, nil) verifyListSnapshots(t, env.Repository, src1, []*snapshot.Manifest{manifest1}) @@ -52,8 +53,8 @@ func TestSnapshotsAPI(t *testing.T) { if id1 == id2 { t.Errorf("expected different manifest IDs, got same: %v", id1) } - verifySnapshotManifestIDs(t, env.Repository, nil, []string{id1, id2}) - verifySnapshotManifestIDs(t, env.Repository, &src1, []string{id1, id2}) + verifySnapshotManifestIDs(t, env.Repository, nil, []manifest.ID{id1, id2}) + verifySnapshotManifestIDs(t, env.Repository, &src1, []manifest.ID{id1, id2}) verifySnapshotManifestIDs(t, env.Repository, &src2, nil) manifest3 := &snapshot.Manifest{ @@ -62,21 +63,21 @@ func TestSnapshotsAPI(t *testing.T) { } id3 := mustSaveSnapshot(t, env.Repository, manifest3) - verifySnapshotManifestIDs(t, env.Repository, nil, []string{id1, id2, id3}) - verifySnapshotManifestIDs(t, env.Repository, &src1, []string{id1, id2}) - verifySnapshotManifestIDs(t, env.Repository, &src2, []string{id3}) + verifySnapshotManifestIDs(t, env.Repository, nil, []manifest.ID{id1, id2, id3}) + verifySnapshotManifestIDs(t, env.Repository, &src1, []manifest.ID{id1, id2}) + verifySnapshotManifestIDs(t, env.Repository, &src2, []manifest.ID{id3}) verifySources(t, env.Repository, src1, src2) - verifyLoadSnapshots(t, env.Repository, []string{id1, id2, id3}, []*snapshot.Manifest{manifest1, manifest2, manifest3}) + verifyLoadSnapshots(t, env.Repository, []manifest.ID{id1, id2, id3}, []*snapshot.Manifest{manifest1, manifest2, manifest3}) } -func verifySnapshotManifestIDs(t *testing.T, rep *repo.Repository, src *snapshot.SourceInfo, expected []string) []string { +func verifySnapshotManifestIDs(t *testing.T, rep *repo.Repository, src *snapshot.SourceInfo, expected []manifest.ID) []manifest.ID { t.Helper() res, err := snapshot.ListSnapshotManifests(context.Background(), rep, src) if err != nil { t.Errorf("error listing snapshot manifests: %v", err) } - sort.Strings(res) - sort.Strings(expected) + sortManifestIDs(res) + sortManifestIDs(expected) if !reflect.DeepEqual(res, expected) { t.Errorf("unexpected manifests: %v, wanted %v", res, expected) return expected @@ -84,7 +85,13 @@ func verifySnapshotManifestIDs(t *testing.T, rep *repo.Repository, src *snapshot return res } -func mustSaveSnapshot(t *testing.T, rep *repo.Repository, man *snapshot.Manifest) string { +func sortManifestIDs(s []manifest.ID) { + sort.Slice(s, func(i, j int) bool { + return s[i] < s[j] + }) +} + +func mustSaveSnapshot(t *testing.T, rep *repo.Repository, man *snapshot.Manifest) manifest.ID { t.Helper() id, err := snapshot.SaveSnapshot(context.Background(), rep, man) if err != nil { @@ -123,7 +130,7 @@ func verifyListSnapshots(t *testing.T, rep *repo.Repository, src snapshot.Source } } -func verifyLoadSnapshots(t *testing.T, rep *repo.Repository, ids []string, expected []*snapshot.Manifest) { +func verifyLoadSnapshots(t *testing.T, rep *repo.Repository, ids []manifest.ID, expected []*snapshot.Manifest) { got, err := snapshot.LoadSnapshots(context.Background(), rep, ids) if err != nil { t.Errorf("error loading manifests: %v", err) diff --git a/snapshot/snapshotfs/upload.go b/snapshot/snapshotfs/upload.go index 5710ae4ea..fb6614b93 100644 --- a/snapshot/snapshotfs/upload.go +++ b/snapshot/snapshotfs/upload.go @@ -93,7 +93,7 @@ func (u *Uploader) cancelReason() string { return "cancelled" } - if mub := u.MaxUploadBytes; mub > 0 && u.repo.Blocks.Stats().WrittenBytes > mub { + if mub := u.MaxUploadBytes; mub > 0 && u.repo.Content.Stats().WrittenBytes > mub { return "limit reached" } @@ -663,7 +663,7 @@ func (u *Uploader) Upload( s.IncompleteReason = u.cancelReason() s.EndTime = time.Now() s.Stats = u.stats - s.Stats.Block = u.repo.Blocks.Stats() + s.Stats.Content = u.repo.Content.Stats() return s, nil } diff --git a/snapshot/stats.go b/snapshot/stats.go index f88bec06b..95026c22f 100644 --- a/snapshot/stats.go +++ b/snapshot/stats.go @@ -2,12 +2,12 @@ import ( "github.com/kopia/kopia/fs" - "github.com/kopia/kopia/repo/block" + "github.com/kopia/kopia/repo/content" ) // Stats keeps track of snapshot generation statistics. type Stats struct { - Block block.Stats `json:"repo,omitempty"` + Content content.Stats `json:"content,omitempty"` TotalDirectoryCount int `json:"dirCount"` TotalFileCount int `json:"fileCount"` diff --git a/tests/end_to_end_test/end_to_end_test.go b/tests/end_to_end_test/end_to_end_test.go index 4c745b89d..665555c67 100644 --- a/tests/end_to_end_test/end_to_end_test.go +++ b/tests/end_to_end_test/end_to_end_test.go @@ -104,8 +104,8 @@ func TestEndToEnd(t *testing.T) { t.Run("VerifyGlobalPolicy", func(t *testing.T) { // verify we created global policy entry - globalPolicyBlockID := e.runAndVerifyOutputLineCount(t, 1, "block", "ls")[0] - e.runAndExpectSuccess(t, "block", "show", "-jz", globalPolicyBlockID) + globalPolicyBlockID := e.runAndVerifyOutputLineCount(t, 1, "content", "ls")[0] + e.runAndExpectSuccess(t, "content", "show", "-jz", globalPolicyBlockID) // make sure the policy is visible in the manifest list e.runAndVerifyOutputLineCount(t, 1, "manifest", "list", "--filter=type:policy", "--filter=policyType:global") @@ -137,13 +137,13 @@ func TestEndToEnd(t *testing.T) { t.Errorf("unexpected number of sources: %v, want %v in %#v", got, want, sources) } - // expect 5 blocks, each snapshot creation adds one index blob - e.runAndVerifyOutputLineCount(t, 6, "blockindex", "ls") - e.runAndExpectSuccess(t, "blockindex", "optimize") - e.runAndVerifyOutputLineCount(t, 1, "blockindex", "ls") + // expect 5 blobs, each snapshot creation adds one index blob + e.runAndVerifyOutputLineCount(t, 6, "index", "ls") + e.runAndExpectSuccess(t, "index", "optimize") + e.runAndVerifyOutputLineCount(t, 1, "index", "ls") e.runAndExpectSuccess(t, "snapshot", "create", ".", dir1, dir2) - e.runAndVerifyOutputLineCount(t, 2, "blockindex", "ls") + e.runAndVerifyOutputLineCount(t, 2, "index", "ls") t.Run("Migrate", func(t *testing.T) { dstenv := newTestEnv(t) @@ -160,39 +160,39 @@ func TestEndToEnd(t *testing.T) { }) t.Run("RepairIndexBlobs", func(t *testing.T) { - blocksBefore := e.runAndExpectSuccess(t, "block", "ls") + contentsBefore := e.runAndExpectSuccess(t, "content", "ls") - lines := e.runAndVerifyOutputLineCount(t, 2, "blockindex", "ls") + lines := e.runAndVerifyOutputLineCount(t, 2, "index", "ls") for _, l := range lines { indexFile := strings.Split(l, " ")[0] e.runAndExpectSuccess(t, "blob", "delete", indexFile) } // there should be no index files at this point - e.runAndVerifyOutputLineCount(t, 0, "blockindex", "ls", "--no-list-caching") + e.runAndVerifyOutputLineCount(t, 0, "index", "ls", "--no-list-caching") // there should be no blocks, since there are no indexesto find them - e.runAndVerifyOutputLineCount(t, 0, "block", "ls") + e.runAndVerifyOutputLineCount(t, 0, "content", "ls") // now recover index from all blocks - e.runAndExpectSuccess(t, "blockindex", "recover", "--commit") + e.runAndExpectSuccess(t, "index", "recover", "--commit") // all recovered index entries are added as index file - e.runAndVerifyOutputLineCount(t, 1, "blockindex", "ls") - blocksAfter := e.runAndExpectSuccess(t, "block", "ls") - if diff := pretty.Compare(blocksBefore, blocksAfter); diff != "" { + e.runAndVerifyOutputLineCount(t, 1, "index", "ls") + contentsAfter := e.runAndExpectSuccess(t, "content", "ls") + if diff := pretty.Compare(contentsBefore, contentsAfter); diff != "" { t.Errorf("unexpected block diff after recovery: %v", diff) } }) - t.Run("RepairFormatBlock", func(t *testing.T) { + t.Run("RepairFormatBlob", func(t *testing.T) { // remove kopia.repository e.runAndExpectSuccess(t, "blob", "rm", "kopia.repository") e.runAndExpectSuccess(t, "repo", "disconnect") - // this will fail because the format block in the repository is not found + // this will fail because the format blob in the repository is not found e.runAndExpectFailure(t, "repo", "connect", "filesystem", "--path", e.repoDir) - // now run repair, which will recover the format block from one of the pack blocks. + // now run repair, which will recover the format blob from one of the pack blobs. e.runAndExpectSuccess(t, "repo", "repair", "filesystem", "--path", e.repoDir) // now connect can succeed diff --git a/tests/repository_stress_test/repository_stress_test.go b/tests/repository_stress_test/repository_stress_test.go index fcb9d1205..2453a6e04 100644 --- a/tests/repository_stress_test/repository_stress_test.go +++ b/tests/repository_stress_test/repository_stress_test.go @@ -18,13 +18,13 @@ "github.com/kopia/kopia/repo" "github.com/kopia/kopia/repo/blob/filesystem" - "github.com/kopia/kopia/repo/block" + "github.com/kopia/kopia/repo/content" ) const masterPassword = "foo-bar-baz-1234" var ( - knownBlocks []string + knownBlocks []content.ID knownBlocksMutex sync.Mutex ) @@ -32,7 +32,7 @@ func TestStressRepository(t *testing.T) { if testing.Short() { t.Skip("skipping stress test during short tests") } - ctx := block.UsingListCache(context.Background(), false) + ctx := content.UsingListCache(context.Background(), false) tmpPath, err := ioutil.TempDir("", "kopia") if err != nil { @@ -66,7 +66,7 @@ func TestStressRepository(t *testing.T) { // set up two parallel kopia connections, each with its own config file and cache. if err := repo.Connect(ctx, configFile1, st, masterPassword, repo.ConnectOptions{ - CachingOptions: block.CachingOptions{ + CachingOptions: content.CachingOptions{ CacheDirectory: filepath.Join(tmpPath, "cache1"), MaxCacheSizeBytes: 2000000000, }, @@ -75,7 +75,7 @@ func TestStressRepository(t *testing.T) { } if err := repo.Connect(ctx, configFile2, st, masterPassword, repo.ConnectOptions{ - CachingOptions: block.CachingOptions{ + CachingOptions: content.CachingOptions{ CacheDirectory: filepath.Join(tmpPath, "cache2"), MaxCacheSizeBytes: 2000000000, }, @@ -155,8 +155,8 @@ func repositoryTest(ctx context.Context, t *testing.T, cancel chan struct{}, rep {"writeRandomBlock", writeRandomBlock, 100, 0}, {"writeRandomManifest", writeRandomManifest, 100, 0}, {"readKnownBlock", readKnownBlock, 500, 0}, - {"listBlocks", listBlocks, 50, 0}, - {"listAndReadAllBlocks", listAndReadAllBlocks, 5, 0}, + {"listContents", listContents, 50, 0}, + {"listAndReadAllContents", listAndReadAllContents, 5, 0}, {"readRandomManifest", readRandomManifest, 50, 0}, {"compact", compact, 1, 0}, {"refresh", refresh, 3, 0}, @@ -208,14 +208,14 @@ func repositoryTest(ctx context.Context, t *testing.T, cancel chan struct{}, rep func writeRandomBlock(ctx context.Context, t *testing.T, r *repo.Repository) error { data := make([]byte, 1000) rand.Read(data) - blockID, err := r.Blocks.WriteBlock(ctx, data, "") + contentID, err := r.Content.WriteContent(ctx, data, "") if err == nil { knownBlocksMutex.Lock() if len(knownBlocks) >= 1000 { n := rand.Intn(len(knownBlocks)) - knownBlocks[n] = blockID + knownBlocks[n] = contentID } else { - knownBlocks = append(knownBlocks, blockID) + knownBlocks = append(knownBlocks, contentID) } knownBlocksMutex.Unlock() } @@ -228,36 +228,36 @@ func readKnownBlock(ctx context.Context, t *testing.T, r *repo.Repository) error knownBlocksMutex.Unlock() return nil } - blockID := knownBlocks[rand.Intn(len(knownBlocks))] + contentID := knownBlocks[rand.Intn(len(knownBlocks))] knownBlocksMutex.Unlock() - _, err := r.Blocks.GetBlock(ctx, blockID) - if err == nil || err == block.ErrBlockNotFound { + _, err := r.Content.GetContent(ctx, contentID) + if err == nil || err == content.ErrContentNotFound { return nil } return err } -func listBlocks(ctx context.Context, t *testing.T, r *repo.Repository) error { - _, err := r.Blocks.ListBlocks("") +func listContents(ctx context.Context, t *testing.T, r *repo.Repository) error { + _, err := r.Content.ListContents("") return err } -func listAndReadAllBlocks(ctx context.Context, t *testing.T, r *repo.Repository) error { - blocks, err := r.Blocks.ListBlocks("") +func listAndReadAllContents(ctx context.Context, t *testing.T, r *repo.Repository) error { + contentIDs, err := r.Content.ListContents("") if err != nil { return err } - for _, bi := range blocks { - _, err := r.Blocks.GetBlock(ctx, bi) + for _, cid := range contentIDs { + _, err := r.Content.GetContent(ctx, cid) if err != nil { - if err == block.ErrBlockNotFound && strings.HasPrefix(bi, "m") { - // this is ok, sometimes manifest manager will perform compaction and 'm' blocks will be marked as deleted + if err == content.ErrContentNotFound && strings.HasPrefix(string(cid), "m") { + // this is ok, sometimes manifest manager will perform compaction and 'm' contents will be marked as deleted continue } - return errors.Wrapf(err, "error reading block %v", bi) + return errors.Wrapf(err, "error reading content %v", cid) } } @@ -265,9 +265,9 @@ func listAndReadAllBlocks(ctx context.Context, t *testing.T, r *repo.Repository) } func compact(ctx context.Context, t *testing.T, r *repo.Repository) error { - return r.Blocks.CompactIndexes(ctx, block.CompactOptions{ - MinSmallBlocks: 1, - MaxSmallBlocks: 1, + return r.Content.CompactIndexes(ctx, content.CompactOptions{ + MinSmallBlobs: 1, + MaxSmallBlobs: 1, }) } diff --git a/tests/stress_test/stress_test.go b/tests/stress_test/stress_test.go index 6318fff43..beb5c0ea1 100644 --- a/tests/stress_test/stress_test.go +++ b/tests/stress_test/stress_test.go @@ -11,7 +11,7 @@ "github.com/kopia/kopia/internal/blobtesting" "github.com/kopia/kopia/repo/blob" - "github.com/kopia/kopia/repo/block" + "github.com/kopia/kopia/repo/content" ) const goroutineCount = 16 @@ -36,14 +36,14 @@ func TestStressBlockManager(t *testing.T) { func stressTestWithStorage(t *testing.T, st blob.Storage, duration time.Duration) { ctx := context.Background() - openMgr := func() (*block.Manager, error) { - return block.NewManager(ctx, st, block.FormattingOptions{ + openMgr := func() (*content.Manager, error) { + return content.NewManager(ctx, st, content.FormattingOptions{ Version: 1, Hash: "HMAC-SHA256-128", Encryption: "AES-256-CTR", MaxPackSize: 20000000, MasterKey: []byte{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15}, - }, block.CachingOptions{}, nil) + }, content.CachingOptions{}, nil) } seed0 := time.Now().Nanosecond() @@ -63,7 +63,7 @@ func stressTestWithStorage(t *testing.T, st blob.Storage, duration time.Duration }) } -func stressWorker(ctx context.Context, t *testing.T, deadline time.Time, workerID int, openMgr func() (*block.Manager, error), seed int64) { +func stressWorker(ctx context.Context, t *testing.T, deadline time.Time, workerID int, openMgr func() (*content.Manager, error), seed int64) { src := rand.NewSource(seed) rand := rand.New(src) @@ -73,7 +73,7 @@ func stressWorker(ctx context.Context, t *testing.T, deadline time.Time, workerI } type writtenBlock struct { - contentID string + contentID content.ID data []byte } @@ -87,7 +87,7 @@ type writtenBlock struct { return } dataCopy := append([]byte{}, data...) - contentID, err := bm.WriteBlock(ctx, data, "") + contentID, err := bm.WriteContent(ctx, data, "") if err != nil { t.Errorf("err: %v", err) return @@ -117,9 +117,9 @@ type writtenBlock struct { pos := rand.Intn(len(workerBlocks)) previous := workerBlocks[pos] //log.Printf("reading %v", previous.contentID) - d2, err := bm.GetBlock(ctx, previous.contentID) + d2, err := bm.GetContent(ctx, previous.contentID) if err != nil { - t.Errorf("error verifying block %q: %v", previous.contentID, err) + t.Errorf("error verifying content %q: %v", previous.contentID, err) return } if !reflect.DeepEqual(previous.data, d2) {