refactoring: renamed repo/block to repo/content

Also introduced strongly typed content.ID and manifest.ID (instead of string)

This aligns identifiers across all layers of repository:

blob.ID
content.ID
object.ID
manifest.ID

cli/app.go

@@ -13,7 +13,7 @@
 	"github.com/kopia/kopia/internal/serverapi"
 	"github.com/kopia/kopia/repo"
 	"github.com/kopia/kopia/repo/blob"
-	"github.com/kopia/kopia/repo/block"
+	"github.com/kopia/kopia/repo/content"
 
 	kingpin "gopkg.in/alecthomas/kingpin.v2"
 )
@@ -31,9 +31,9 @@
 	policyCommands     = app.Command("policy", "Commands to manipulate snapshotting policies.").Alias("policies")
 	serverCommands     = app.Command("server", "Commands to control HTTP API server.")
 	manifestCommands   = app.Command("manifest", "Low-level commands to manipulate manifest items.").Hidden()
-	blockCommands      = app.Command("block", "Commands to manipulate virtual blocks in repository.").Alias("blk").Hidden()
+	contentCommands    = app.Command("content", "Commands to manipulate content in repository.").Alias("contents").Hidden()
 	blobCommands       = app.Command("blob", "Commands to manipulate BLOBs.").Hidden()
-	blockIndexCommands = app.Command("blockindex", "Commands to manipulate block index.").Hidden()
+	indexCommands      = app.Command("index", "Commands to manipulate content index.").Hidden()
 	benchmarkCommands  = app.Command("benchmark", "Commands to test performance of algorithms.").Hidden()
 )
 
@@ -58,8 +58,8 @@ func serverAction(act func(ctx context.Context, cli *serverapi.Client) error) fu
 func repositoryAction(act func(ctx context.Context, rep *repo.Repository) error) func(ctx *kingpin.ParseContext) error {
 	return func(kpc *kingpin.ParseContext) error {
 		ctx := context.Background()
-		ctx = block.UsingBlockCache(ctx, *enableCaching)
-		ctx = block.UsingListCache(ctx, *enableListCaching)
+		ctx = content.UsingContentCache(ctx, *enableCaching)
+		ctx = content.UsingListCache(ctx, *enableListCaching)
 		ctx = blob.WithUploadProgressCallback(ctx, func(desc string, progress, total int64) {
 			cliProgress.Report("upload '"+desc+"'", progress, total)
 		})
@@ -70,13 +70,13 @@ func repositoryAction(act func(ctx context.Context, rep *repo.Repository) error)
 
 		storageType := rep.Blobs.ConnectionInfo().Type
 
-		reportStartupTime(storageType, rep.Blocks.Format.Version, repositoryOpenTime)
+		reportStartupTime(storageType, rep.Content.Format.Version, repositoryOpenTime)
 
 		t1 := time.Now()
 		err := act(ctx, rep)
 		commandDuration := time.Since(t1)
 
-		reportSubcommandFinished(kpc.SelectedCommand.FullCommand(), err == nil, storageType, rep.Blocks.Format.Version, commandDuration)
+		reportSubcommandFinished(kpc.SelectedCommand.FullCommand(), err == nil, storageType, rep.Content.Format.Version, commandDuration)
 		if cerr := rep.Close(ctx); cerr != nil {
 			return errors.Wrap(cerr, "unable to close repository")
 		}

cli/command_benchmark_crypto.go

@@ -5,7 +5,7 @@
 	"time"
 
 	"github.com/kopia/kopia/internal/units"
-	"github.com/kopia/kopia/repo/block"
+	"github.com/kopia/kopia/repo/content"
 
 	kingpin "gopkg.in/alecthomas/kingpin.v2"
 )
@@ -27,14 +27,14 @@ type benchResult struct {
 	var results []benchResult
 
 	data := make([]byte, *benchmarkCryptoBlockSize)
-	for _, ha := range block.SupportedHashAlgorithms() {
-		for _, ea := range block.SupportedEncryptionAlgorithms() {
+	for _, ha := range content.SupportedHashAlgorithms() {
+		for _, ea := range content.SupportedEncryptionAlgorithms() {
 			isEncrypted := ea != "NONE"
 			if *benchmarkCryptoEncryption != isEncrypted {
 				continue
 			}
 
-			h, e, err := block.CreateHashAndEncryptor(block.FormattingOptions{
+			h, e, err := content.CreateHashAndEncryptor(content.FormattingOptions{
 				Encryption: ea,
 				Hash:       ha,
 				MasterKey:  make([]byte, 32),
@@ -48,8 +48,8 @@ type benchResult struct {
 			t0 := time.Now()
 			hashCount := *benchmarkCryptoRepeat
 			for i := 0; i < hashCount; i++ {
-				blockID := h(data)
-				if _, encerr := e.Encrypt(data, blockID); encerr != nil {
+				contentID := h(data)
+				if _, encerr := e.Encrypt(data, contentID); encerr != nil {
 					log.Warningf("encryption failed: %v", encerr)
 					break
 				}

cli/command_blob_delete.go

@@ -14,7 +14,7 @@
 	blobDeleteBlobIDs = blobDeleteCommand.Arg("blobIDs", "Blob IDs").Required().Strings()
 )
 
-func runDeleteStorageBlocks(ctx context.Context, rep *repo.Repository) error {
+func runDeleteBlobs(ctx context.Context, rep *repo.Repository) error {
 	for _, b := range *blobDeleteBlobIDs {
 		err := rep.Blobs.DeleteBlob(ctx, blob.ID(b))
 		if err != nil {
@@ -26,5 +26,5 @@ func runDeleteStorageBlocks(ctx context.Context, rep *repo.Repository) error {
 }
 
 func init() {
-	blobDeleteCommand.Action(repositoryAction(runDeleteStorageBlocks))
+	blobDeleteCommand.Action(repositoryAction(runDeleteBlobs))
 }

cli/command_block_gc.go

@@ -1,50 +0,0 @@
-package cli
-
-import (
-	"context"
-
-	"github.com/pkg/errors"
-
-	"github.com/kopia/kopia/repo"
-)
-
-var (
-	blockGarbageCollectCommand       = blockCommands.Command("gc", "Garbage-collect unused storage blocks")
-	blockGarbageCollectCommandDelete = blockGarbageCollectCommand.Flag("delete", "Whether to delete unused block").String()
-)
-
-func runBlockGarbageCollectAction(ctx context.Context, rep *repo.Repository) error {
-	unused, err := rep.Blocks.FindUnreferencedBlobs(ctx)
-	if err != nil {
-		return errors.Wrap(err, "error looking for unreferenced blobs")
-	}
-
-	if len(unused) == 0 {
-		printStderr("No unused blocks found.\n")
-		return nil
-	}
-
-	if *blockGarbageCollectCommandDelete != "yes" {
-		var totalBytes int64
-		for _, u := range unused {
-			printStderr("unused %v (%v bytes)\n", u.BlobID, u.Length)
-			totalBytes += u.Length
-		}
-		printStderr("Would delete %v unused blocks (%v bytes), pass '--delete=yes' to actually delete.\n", len(unused), totalBytes)
-
-		return nil
-	}
-
-	for _, u := range unused {
-		printStderr("Deleting unused block %q (%v bytes)...\n", u.BlobID, u.Length)
-		if err := rep.Blobs.DeleteBlob(ctx, u.BlobID); err != nil {
-			return errors.Wrapf(err, "unable to delete block %q", u.BlobID)
-		}
-	}
-
-	return nil
-}
-
-func init() {
-	blockGarbageCollectCommand.Action(repositoryAction(runBlockGarbageCollectAction))
-}

cli/command_block_index_optimize.go

@@ -1,29 +0,0 @@
-package cli
-
-import (
-	"context"
-
-	"github.com/kopia/kopia/repo"
-	"github.com/kopia/kopia/repo/block"
-)
-
-var (
-	optimizeCommand              = blockIndexCommands.Command("optimize", "Optimize block indexes.")
-	optimizeMinSmallBlocks       = optimizeCommand.Flag("min-small-blocks", "Minimum number of small index blobs that can be left after compaction.").Default("1").Int()
-	optimizeMaxSmallBlocks       = optimizeCommand.Flag("max-small-blocks", "Maximum number of small index blobs that can be left after compaction.").Default("1").Int()
-	optimizeSkipDeletedOlderThan = optimizeCommand.Flag("skip-deleted-older-than", "Skip deleted blocks above given age").Duration()
-	optimizeAllBlocks            = optimizeCommand.Flag("all", "Optimize all indexes, even those above maximum size.").Bool()
-)
-
-func runOptimizeCommand(ctx context.Context, rep *repo.Repository) error {
-	return rep.Blocks.CompactIndexes(ctx, block.CompactOptions{
-		MinSmallBlocks:       *optimizeMinSmallBlocks,
-		MaxSmallBlocks:       *optimizeMaxSmallBlocks,
-		AllBlocks:            *optimizeAllBlocks,
-		SkipDeletedOlderThan: *optimizeSkipDeletedOlderThan,
-	})
-}
-
-func init() {
-	optimizeCommand.Action(repositoryAction(runOptimizeCommand))
-}

cli/command_block_list.go

@@ -1,101 +0,0 @@
-package cli
-
-import (
-	"context"
-	"fmt"
-	"sort"
-
-	"github.com/kopia/kopia/repo"
-	"github.com/kopia/kopia/repo/blob"
-	"github.com/kopia/kopia/repo/block"
-)
-
-var (
-	blockListCommand        = blockCommands.Command("list", "List blocks").Alias("ls")
-	blockListLong           = blockListCommand.Flag("long", "Long output").Short('l').Bool()
-	blockListPrefix         = blockListCommand.Flag("prefix", "Prefix").String()
-	blockListIncludeDeleted = blockListCommand.Flag("deleted", "Include deleted blocks").Bool()
-	blockListDeletedOnly    = blockListCommand.Flag("deleted-only", "Only show deleted blocks").Bool()
-	blockListSort           = blockListCommand.Flag("sort", "Sort order").Default("name").Enum("name", "size", "time", "none", "pack")
-	blockListReverse        = blockListCommand.Flag("reverse", "Reverse sort").Short('r').Bool()
-	blockListSummary        = blockListCommand.Flag("summary", "Summarize the list").Short('s').Bool()
-	blockListHuman          = blockListCommand.Flag("human", "Human-readable output").Short('h').Bool()
-)
-
-func runListBlocksAction(ctx context.Context, rep *repo.Repository) error {
-	blocks, err := rep.Blocks.ListBlockInfos(*blockListPrefix, *blockListIncludeDeleted || *blockListDeletedOnly)
-	if err != nil {
-		return err
-	}
-
-	sortBlocks(blocks)
-
-	var count int
-	var totalSize int64
-	uniquePacks := map[blob.ID]bool{}
-	for _, b := range blocks {
-		if *blockListDeletedOnly && !b.Deleted {
-			continue
-		}
-		totalSize += int64(b.Length)
-		count++
-		if b.PackBlobID != "" {
-			uniquePacks[b.PackBlobID] = true
-		}
-		if *blockListLong {
-			optionalDeleted := ""
-			if b.Deleted {
-				optionalDeleted = " (deleted)"
-			}
-			fmt.Printf("%v %v %v %v+%v%v\n",
-				b.BlockID,
-				formatTimestamp(b.Timestamp()),
-				b.PackBlobID,
-				b.PackOffset,
-				maybeHumanReadableBytes(*blockListHuman, int64(b.Length)),
-				optionalDeleted)
-		} else {
-			fmt.Printf("%v\n", b.BlockID)
-		}
-	}
-
-	if *blockListSummary {
-		fmt.Printf("Total: %v blocks, %v packs, %v total size\n",
-			maybeHumanReadableCount(*blockListHuman, int64(count)),
-			maybeHumanReadableCount(*blockListHuman, int64(len(uniquePacks))),
-			maybeHumanReadableBytes(*blockListHuman, totalSize))
-	}
-
-	return nil
-}
-
-func sortBlocks(blocks []block.Info) {
-	maybeReverse := func(b bool) bool { return b }
-
-	if *blockListReverse {
-		maybeReverse = func(b bool) bool { return !b }
-	}
-
-	switch *blockListSort {
-	case "name":
-		sort.Slice(blocks, func(i, j int) bool { return maybeReverse(blocks[i].BlockID < blocks[j].BlockID) })
-	case "size":
-		sort.Slice(blocks, func(i, j int) bool { return maybeReverse(blocks[i].Length < blocks[j].Length) })
-	case "time":
-		sort.Slice(blocks, func(i, j int) bool { return maybeReverse(blocks[i].TimestampSeconds < blocks[j].TimestampSeconds) })
-	case "pack":
-		sort.Slice(blocks, func(i, j int) bool { return maybeReverse(comparePacks(blocks[i], blocks[j])) })
-	}
-}
-
-func comparePacks(a, b block.Info) bool {
-	if a, b := a.PackBlobID, b.PackBlobID; a != b {
-		return a < b
-	}
-
-	return a.PackOffset < b.PackOffset
-}
-
-func init() {
-	blockListCommand.Action(repositoryAction(runListBlocksAction))
-}

cli/command_block_rewrite.go

@@ -1,185 +0,0 @@
-package cli
-
-import (
-	"context"
-	"fmt"
-	"strings"
-	"sync"
-
-	"github.com/pkg/errors"
-
-	"github.com/kopia/kopia/repo"
-	"github.com/kopia/kopia/repo/blob"
-	"github.com/kopia/kopia/repo/block"
-)
-
-var (
-	blockRewriteCommand     = blockCommands.Command("rewrite", "Rewrite blocks using most recent format")
-	blockRewriteIDs         = blockRewriteCommand.Arg("blockID", "Identifiers of blocks to rewrite").Strings()
-	blockRewriteParallelism = blockRewriteCommand.Flag("parallelism", "Number of parallel workers").Default("16").Int()
-
-	blockRewriteShortPacks    = blockRewriteCommand.Flag("short", "Rewrite blocks from short packs").Bool()
-	blockRewriteFormatVersion = blockRewriteCommand.Flag("format-version", "Rewrite blocks using the provided format version").Default("-1").Int()
-	blockRewritePackPrefix    = blockRewriteCommand.Flag("pack-prefix", "Only rewrite pack blocks with a given prefix").String()
-	blockRewriteDryRun        = blockRewriteCommand.Flag("dry-run", "Do not actually rewrite, only print what would happen").Short('n').Bool()
-)
-
-type blockInfoOrError struct {
-	block.Info
-	err error
-}
-
-func runRewriteBlocksAction(ctx context.Context, rep *repo.Repository) error {
-	blocks := getBlocksToRewrite(ctx, rep)
-
-	var (
-		mu          sync.Mutex
-		totalBytes  int64
-		failedCount int
-	)
-
-	var wg sync.WaitGroup
-
-	for i := 0; i < *blockRewriteParallelism; i++ {
-		wg.Add(1)
-		go func() {
-			defer wg.Done()
-
-			for b := range blocks {
-				if b.err != nil {
-					log.Errorf("got error: %v", b.err)
-					mu.Lock()
-					failedCount++
-					mu.Unlock()
-					return
-				}
-
-				var optDeleted string
-				if b.Deleted {
-					optDeleted = " (deleted)"
-				}
-
-				printStderr("Rewriting block %v (%v bytes) from pack %v%v\n", b.BlockID, b.Length, b.PackBlobID, optDeleted)
-				mu.Lock()
-				totalBytes += int64(b.Length)
-				mu.Unlock()
-				if *blockRewriteDryRun {
-					continue
-				}
-				if err := rep.Blocks.RewriteBlock(ctx, b.BlockID); err != nil {
-					log.Warningf("unable to rewrite block %q: %v", b.BlockID, err)
-					mu.Lock()
-					failedCount++
-					mu.Unlock()
-				}
-			}
-		}()
-	}
-
-	wg.Wait()
-
-	printStderr("Total bytes rewritten %v\n", totalBytes)
-
-	if failedCount == 0 {
-		return nil
-	}
-
-	return errors.Errorf("failed to rewrite %v blocks", failedCount)
-}
-
-func getBlocksToRewrite(ctx context.Context, rep *repo.Repository) <-chan blockInfoOrError {
-	ch := make(chan blockInfoOrError)
-	go func() {
-		defer close(ch)
-
-		// get blocks listed on command line
-		findBlockInfos(ctx, rep, ch, *blockRewriteIDs)
-
-		// add all blocks from short packs
-		if *blockRewriteShortPacks {
-			threshold := uint32(rep.Blocks.Format.MaxPackSize * 6 / 10)
-			findBlocksInShortPacks(ctx, rep, ch, threshold)
-		}
-
-		// add all blocks with given format version
-		if *blockRewriteFormatVersion != -1 {
-			findBlocksWithFormatVersion(ctx, rep, ch, *blockRewriteFormatVersion)
-		}
-	}()
-
-	return ch
-}
-
-func findBlockInfos(ctx context.Context, rep *repo.Repository, ch chan blockInfoOrError, blockIDs []string) {
-	for _, blockID := range blockIDs {
-		i, err := rep.Blocks.BlockInfo(ctx, blockID)
-		if err != nil {
-			ch <- blockInfoOrError{err: errors.Wrapf(err, "unable to get info for block %q", blockID)}
-		} else {
-			ch <- blockInfoOrError{Info: i}
-		}
-	}
-}
-
-func findBlocksWithFormatVersion(ctx context.Context, rep *repo.Repository, ch chan blockInfoOrError, version int) {
-	infos, err := rep.Blocks.ListBlockInfos("", true)
-	if err != nil {
-		ch <- blockInfoOrError{err: errors.Wrap(err, "unable to list index blobs")}
-		return
-	}
-
-	for _, b := range infos {
-		if int(b.FormatVersion) == *blockRewriteFormatVersion && strings.HasPrefix(string(b.PackBlobID), *blockRewritePackPrefix) {
-			ch <- blockInfoOrError{Info: b}
-		}
-	}
-}
-
-func findBlocksInShortPacks(ctx context.Context, rep *repo.Repository, ch chan blockInfoOrError, threshold uint32) {
-	log.Debugf("listing blocks...")
-	infos, err := rep.Blocks.ListBlockInfos("", true)
-	if err != nil {
-		ch <- blockInfoOrError{err: errors.Wrap(err, "unable to list index blobs")}
-		return
-	}
-
-	log.Debugf("finding blocks in short packs...")
-	shortPackBlocks, err := findShortPackBlocks(infos, threshold)
-	if err != nil {
-		ch <- blockInfoOrError{err: errors.Wrap(err, "unable to find short pack blocks")}
-		return
-	}
-	log.Debugf("found %v short pack blocks", len(shortPackBlocks))
-
-	if len(shortPackBlocks) <= 1 {
-		fmt.Printf("Nothing to do, found %v short pack blocks\n", len(shortPackBlocks))
-	} else {
-		for _, b := range infos {
-			if shortPackBlocks[b.PackBlobID] && strings.HasPrefix(string(b.PackBlobID), *blockRewritePackPrefix) {
-				ch <- blockInfoOrError{Info: b}
-			}
-		}
-	}
-}
-
-func findShortPackBlocks(infos []block.Info, threshold uint32) (map[blob.ID]bool, error) {
-	packUsage := map[blob.ID]uint32{}
-
-	for _, bi := range infos {
-		packUsage[bi.PackBlobID] += bi.Length
-	}
-
-	shortPackBlocks := map[blob.ID]bool{}
-
-	for blobID, usage := range packUsage {
-		if usage < threshold {
-			shortPackBlocks[blobID] = true
-		}
-	}
-
-	return shortPackBlocks, nil
-}
-
-func init() {
-	blockRewriteCommand.Action(repositoryAction(runRewriteBlocksAction))
-}

cli/command_block_rm.go

@@ -1,32 +0,0 @@
-package cli
-
-import (
-	"context"
-
-	"github.com/kopia/kopia/repo"
-)
-
-var (
-	removeBlockCommand = blockCommands.Command("remove", "Remove block(s)").Alias("rm")
-
-	removeBlockIDs = removeBlockCommand.Arg("id", "IDs of blocks to remove").Required().Strings()
-)
-
-func runRemoveBlockCommand(ctx context.Context, rep *repo.Repository) error {
-	for _, blockID := range *removeBlockIDs {
-		if err := removeBlock(rep, blockID); err != nil {
-			return err
-		}
-	}
-
-	return nil
-}
-
-func removeBlock(r *repo.Repository, blockID string) error {
-	return r.Blocks.DeleteBlock(blockID)
-}
-
-func init() {
-	setupShowCommand(removeBlockCommand)
-	removeBlockCommand.Action(repositoryAction(runRemoveBlockCommand))
-}

cli/command_block_show.go

@@ -1,38 +0,0 @@
-package cli
-
-import (
-	"bytes"
-	"context"
-
-	"github.com/kopia/kopia/repo"
-)
-
-var (
-	showBlockCommand = blockCommands.Command("show", "Show contents of a block.").Alias("cat")
-
-	showBlockIDs = showBlockCommand.Arg("id", "IDs of blocks to show").Required().Strings()
-)
-
-func runShowBlockCommand(ctx context.Context, rep *repo.Repository) error {
-	for _, blockID := range *showBlockIDs {
-		if err := showBlock(ctx, rep, blockID); err != nil {
-			return err
-		}
-	}
-
-	return nil
-}
-
-func showBlock(ctx context.Context, r *repo.Repository, blockID string) error {
-	data, err := r.Blocks.GetBlock(ctx, blockID)
-	if err != nil {
-		return err
-	}
-
-	return showContent(bytes.NewReader(data))
-}
-
-func init() {
-	setupShowCommand(showBlockCommand)
-	showBlockCommand.Action(repositoryAction(runShowBlockCommand))
-}

cli/command_block_verify.go

@@ -1,62 +0,0 @@
-package cli
-
-import (
-	"context"
-
-	"github.com/pkg/errors"
-
-	"github.com/kopia/kopia/repo"
-)
-
-var (
-	verifyBlockCommand = blockCommands.Command("verify", "Verify contents of a block.")
-
-	verifyBlockIDs = verifyBlockCommand.Arg("id", "IDs of blocks to show (or 'all')").Required().Strings()
-)
-
-func runVerifyBlockCommand(ctx context.Context, rep *repo.Repository) error {
-	for _, blockID := range *verifyBlockIDs {
-		if blockID == "all" {
-			return verifyAllBlocks(ctx, rep)
-		}
-		if err := verifyBlock(ctx, rep, blockID); err != nil {
-			return err
-		}
-	}
-
-	return nil
-}
-
-func verifyAllBlocks(ctx context.Context, rep *repo.Repository) error {
-	blockIDs, err := rep.Blocks.ListBlocks("")
-	if err != nil {
-		return errors.Wrap(err, "unable to list blocks")
-	}
-
-	var errorCount int
-	for _, blockID := range blockIDs {
-		if err := verifyBlock(ctx, rep, blockID); err != nil {
-			errorCount++
-		}
-	}
-	if errorCount == 0 {
-		return nil
-	}
-
-	return errors.Errorf("encountered %v errors", errorCount)
-}
-
-func verifyBlock(ctx context.Context, r *repo.Repository, blockID string) error {
-	if _, err := r.Blocks.GetBlock(ctx, blockID); err != nil {
-		log.Warningf("block %v is invalid: %v", blockID, err)
-		return err
-	}
-
-	log.Infof("block %v is ok", blockID)
-
-	return nil
-}
-
-func init() {
-	verifyBlockCommand.Action(repositoryAction(runVerifyBlockCommand))
-}

cli/command_cache_info.go

@@ -21,7 +21,7 @@ func runCacheInfoCommand(ctx context.Context, rep *repo.Repository) error {
 	}
 
 	log.Debugf("scanning cache...")
-	fileCount, totalFileSize, err := scanCacheDir(filepath.Join(rep.CacheDirectory, "blocks"))
+	fileCount, totalFileSize, err := scanCacheDir(filepath.Join(rep.CacheDirectory, "contents"))
 	if err != nil {
 		return err
 	}

cli/command_cache_set.go

@@ -4,7 +4,7 @@
 	"context"
 
 	"github.com/kopia/kopia/repo"
-	"github.com/kopia/kopia/repo/block"
+	"github.com/kopia/kopia/repo/content"
 )
 
 var (
@@ -16,7 +16,7 @@
 )
 
 func runCacheSetCommand(ctx context.Context, rep *repo.Repository) error {
-	opts := block.CachingOptions{
+	opts := content.CachingOptions{
 		CacheDirectory:          *cacheSetDirectory,
 		MaxCacheSizeBytes:       *cacheSetMaxCacheSizeMB << 20,
 		MaxListCacheDurationSec: int(cacheSetMaxListCacheDuration.Seconds()),

cli/command_content_gc.go

@@ -0,0 +1,50 @@
+package cli
+
+import (
+	"context"
+
+	"github.com/pkg/errors"
+
+	"github.com/kopia/kopia/repo"
+)
+
+var (
+	contentGarbageCollectCommand       = contentCommands.Command("gc", "Garbage-collect unused blobs")
+	contentGarbageCollectCommandDelete = contentGarbageCollectCommand.Flag("delete", "Whether to delete unused blobs").String()
+)
+
+func runContentGarbageCollectCommand(ctx context.Context, rep *repo.Repository) error {
+	unused, err := rep.Content.FindUnreferencedBlobs(ctx)
+	if err != nil {
+		return errors.Wrap(err, "error looking for unreferenced blobs")
+	}
+
+	if len(unused) == 0 {
+		printStderr("No unused blobs found.\n")
+		return nil
+	}
+
+	if *contentGarbageCollectCommandDelete != "yes" {
+		var totalBytes int64
+		for _, u := range unused {
+			printStderr("unused %v (%v bytes)\n", u.BlobID, u.Length)
+			totalBytes += u.Length
+		}
+		printStderr("Would delete %v unused blobs (%v bytes), pass '--delete=yes' to actually delete.\n", len(unused), totalBytes)
+
+		return nil
+	}
+
+	for _, u := range unused {
+		printStderr("Deleting unused blob %q (%v bytes)...\n", u.BlobID, u.Length)
+		if err := rep.Blobs.DeleteBlob(ctx, u.BlobID); err != nil {
+			return errors.Wrapf(err, "unable to delete blob %q", u.BlobID)
+		}
+	}
+
+	return nil
+}
+
+func init() {
+	contentGarbageCollectCommand.Action(repositoryAction(runContentGarbageCollectCommand))
+}

cli/command_content_list.go

@@ -0,0 +1,101 @@
+package cli
+
+import (
+	"context"
+	"fmt"
+	"sort"
+
+	"github.com/kopia/kopia/repo"
+	"github.com/kopia/kopia/repo/blob"
+	"github.com/kopia/kopia/repo/content"
+)
+
+var (
+	contentListCommand        = contentCommands.Command("list", "List contents").Alias("ls")
+	contentListLong           = contentListCommand.Flag("long", "Long output").Short('l').Bool()
+	contentListPrefix         = contentListCommand.Flag("prefix", "Prefix").String()
+	contentListIncludeDeleted = contentListCommand.Flag("deleted", "Include deleted content").Bool()
+	contentListDeletedOnly    = contentListCommand.Flag("deleted-only", "Only show deleted content").Bool()
+	contentListSort           = contentListCommand.Flag("sort", "Sort order").Default("name").Enum("name", "size", "time", "none", "pack")
+	contentListReverse        = contentListCommand.Flag("reverse", "Reverse sort").Short('r').Bool()
+	contentListSummary        = contentListCommand.Flag("summary", "Summarize the list").Short('s').Bool()
+	contentListHuman          = contentListCommand.Flag("human", "Human-readable output").Short('h').Bool()
+)
+
+func runContentListCommand(ctx context.Context, rep *repo.Repository) error {
+	contents, err := rep.Content.ListContentInfos(content.ID(*contentListPrefix), *contentListIncludeDeleted || *contentListDeletedOnly)
+	if err != nil {
+		return err
+	}
+
+	sortContents(contents)
+
+	var count int
+	var totalSize int64
+	uniquePacks := map[blob.ID]bool{}
+	for _, b := range contents {
+		if *contentListDeletedOnly && !b.Deleted {
+			continue
+		}
+		totalSize += int64(b.Length)
+		count++
+		if b.PackBlobID != "" {
+			uniquePacks[b.PackBlobID] = true
+		}
+		if *contentListLong {
+			optionalDeleted := ""
+			if b.Deleted {
+				optionalDeleted = " (deleted)"
+			}
+			fmt.Printf("%v %v %v %v+%v%v\n",
+				b.ID,
+				formatTimestamp(b.Timestamp()),
+				b.PackBlobID,
+				b.PackOffset,
+				maybeHumanReadableBytes(*contentListHuman, int64(b.Length)),
+				optionalDeleted)
+		} else {
+			fmt.Printf("%v\n", b.ID)
+		}
+	}
+
+	if *contentListSummary {
+		fmt.Printf("Total: %v contents, %v packs, %v total size\n",
+			maybeHumanReadableCount(*contentListHuman, int64(count)),
+			maybeHumanReadableCount(*contentListHuman, int64(len(uniquePacks))),
+			maybeHumanReadableBytes(*contentListHuman, totalSize))
+	}
+
+	return nil
+}
+
+func sortContents(contents []content.Info) {
+	maybeReverse := func(b bool) bool { return b }
+
+	if *contentListReverse {
+		maybeReverse = func(b bool) bool { return !b }
+	}
+
+	switch *contentListSort {
+	case "name":
+		sort.Slice(contents, func(i, j int) bool { return maybeReverse(contents[i].ID < contents[j].ID) })
+	case "size":
+		sort.Slice(contents, func(i, j int) bool { return maybeReverse(contents[i].Length < contents[j].Length) })
+	case "time":
+		sort.Slice(contents, func(i, j int) bool { return maybeReverse(contents[i].TimestampSeconds < contents[j].TimestampSeconds) })
+	case "pack":
+		sort.Slice(contents, func(i, j int) bool { return maybeReverse(comparePacks(contents[i], contents[j])) })
+	}
+}
+
+func comparePacks(a, b content.Info) bool {
+	if a, b := a.PackBlobID, b.PackBlobID; a != b {
+		return a < b
+	}
+
+	return a.PackOffset < b.PackOffset
+}
+
+func init() {
+	contentListCommand.Action(repositoryAction(runContentListCommand))
+}

cli/command_content_rewrite.go

@@ -0,0 +1,193 @@
+package cli
+
+import (
+	"context"
+	"fmt"
+	"strings"
+	"sync"
+
+	"github.com/pkg/errors"
+
+	"github.com/kopia/kopia/repo"
+	"github.com/kopia/kopia/repo/blob"
+	"github.com/kopia/kopia/repo/content"
+)
+
+var (
+	contentRewriteCommand     = contentCommands.Command("rewrite", "Rewrite content using most recent format")
+	contentRewriteIDs         = contentRewriteCommand.Arg("contentID", "Identifiers of contents to rewrite").Strings()
+	contentRewriteParallelism = contentRewriteCommand.Flag("parallelism", "Number of parallel workers").Default("16").Int()
+
+	contentRewriteShortPacks    = contentRewriteCommand.Flag("short", "Rewrite contents from short packs").Bool()
+	contentRewriteFormatVersion = contentRewriteCommand.Flag("format-version", "Rewrite contents using the provided format version").Default("-1").Int()
+	contentRewritePackPrefix    = contentRewriteCommand.Flag("pack-prefix", "Only rewrite contents from pack blobs with a given prefix").String()
+	contentRewriteDryRun        = contentRewriteCommand.Flag("dry-run", "Do not actually rewrite, only print what would happen").Short('n').Bool()
+)
+
+type contentInfoOrError struct {
+	content.Info
+	err error
+}
+
+func runContentRewriteCommand(ctx context.Context, rep *repo.Repository) error {
+	cnt := getContentToRewrite(ctx, rep)
+
+	var (
+		mu          sync.Mutex
+		totalBytes  int64
+		failedCount int
+	)
+
+	var wg sync.WaitGroup
+
+	for i := 0; i < *contentRewriteParallelism; i++ {
+		wg.Add(1)
+		go func() {
+			defer wg.Done()
+
+			for c := range cnt {
+				if c.err != nil {
+					log.Errorf("got error: %v", c.err)
+					mu.Lock()
+					failedCount++
+					mu.Unlock()
+					return
+				}
+
+				var optDeleted string
+				if c.Deleted {
+					optDeleted = " (deleted)"
+				}
+
+				printStderr("Rewriting content %v (%v bytes) from pack %v%v\n", c.ID, c.Length, c.PackBlobID, optDeleted)
+				mu.Lock()
+				totalBytes += int64(c.Length)
+				mu.Unlock()
+				if *contentRewriteDryRun {
+					continue
+				}
+				if err := rep.Content.RewriteContent(ctx, c.ID); err != nil {
+					log.Warningf("unable to rewrite content %q: %v", c.ID, err)
+					mu.Lock()
+					failedCount++
+					mu.Unlock()
+				}
+			}
+		}()
+	}
+
+	wg.Wait()
+
+	printStderr("Total bytes rewritten %v\n", totalBytes)
+
+	if failedCount == 0 {
+		return nil
+	}
+
+	return errors.Errorf("failed to rewrite %v contents", failedCount)
+}
+
+func getContentToRewrite(ctx context.Context, rep *repo.Repository) <-chan contentInfoOrError {
+	ch := make(chan contentInfoOrError)
+	go func() {
+		defer close(ch)
+
+		// get content IDs listed on command line
+		findContentInfos(ctx, rep, ch, toContentIDs(*contentRewriteIDs))
+
+		// add all content IDs from short packs
+		if *contentRewriteShortPacks {
+			threshold := uint32(rep.Content.Format.MaxPackSize * 6 / 10)
+			findContentInShortPacks(ctx, rep, ch, threshold)
+		}
+
+		// add all blocks with given format version
+		if *contentRewriteFormatVersion != -1 {
+			findContentWithFormatVersion(ctx, rep, ch, *contentRewriteFormatVersion)
+		}
+	}()
+
+	return ch
+}
+
+func toContentIDs(s []string) []content.ID {
+	var result []content.ID
+	for _, cid := range s {
+		result = append(result, content.ID(cid))
+	}
+	return result
+}
+
+func findContentInfos(ctx context.Context, rep *repo.Repository, ch chan contentInfoOrError, contentIDs []content.ID) {
+	for _, contentID := range contentIDs {
+		i, err := rep.Content.ContentInfo(ctx, contentID)
+		if err != nil {
+			ch <- contentInfoOrError{err: errors.Wrapf(err, "unable to get info for content %q", contentID)}
+		} else {
+			ch <- contentInfoOrError{Info: i}
+		}
+	}
+}
+
+func findContentWithFormatVersion(ctx context.Context, rep *repo.Repository, ch chan contentInfoOrError, version int) {
+	infos, err := rep.Content.ListContentInfos("", true)
+	if err != nil {
+		ch <- contentInfoOrError{err: errors.Wrap(err, "unable to list index blobs")}
+		return
+	}
+
+	for _, b := range infos {
+		if int(b.FormatVersion) == *contentRewriteFormatVersion && strings.HasPrefix(string(b.PackBlobID), *contentRewritePackPrefix) {
+			ch <- contentInfoOrError{Info: b}
+		}
+	}
+}
+
+func findContentInShortPacks(ctx context.Context, rep *repo.Repository, ch chan contentInfoOrError, threshold uint32) {
+	log.Debugf("listing contents...")
+	infos, err := rep.Content.ListContentInfos("", true)
+	if err != nil {
+		ch <- contentInfoOrError{err: errors.Wrap(err, "unable to list index blobs")}
+		return
+	}
+
+	log.Debugf("finding content in short pack blobs...")
+	shortPackBlocks, err := findShortPackBlobs(infos, threshold)
+	if err != nil {
+		ch <- contentInfoOrError{err: errors.Wrap(err, "unable to find short pack blobs")}
+		return
+	}
+	log.Debugf("found %v short pack blobs", len(shortPackBlocks))
+
+	if len(shortPackBlocks) <= 1 {
+		fmt.Printf("Nothing to do, found %v short pack blobs\n", len(shortPackBlocks))
+	} else {
+		for _, b := range infos {
+			if shortPackBlocks[b.PackBlobID] && strings.HasPrefix(string(b.PackBlobID), *contentRewritePackPrefix) {
+				ch <- contentInfoOrError{Info: b}
+			}
+		}
+	}
+}
+
+func findShortPackBlobs(infos []content.Info, threshold uint32) (map[blob.ID]bool, error) {
+	packUsage := map[blob.ID]uint32{}
+
+	for _, bi := range infos {
+		packUsage[bi.PackBlobID] += bi.Length
+	}
+
+	shortPackBlocks := map[blob.ID]bool{}
+
+	for blobID, usage := range packUsage {
+		if usage < threshold {
+			shortPackBlocks[blobID] = true
+		}
+	}
+
+	return shortPackBlocks, nil
+}
+
+func init() {
+	contentRewriteCommand.Action(repositoryAction(runContentRewriteCommand))
+}

cli/command_content_rm.go

@@ -0,0 +1,28 @@
+package cli
+
+import (
+	"context"
+
+	"github.com/kopia/kopia/repo"
+)
+
+var (
+	contentRemoveCommand = contentCommands.Command("remove", "Remove content").Alias("rm")
+
+	contentRemoveIDs = contentRemoveCommand.Arg("id", "IDs of content to remove").Required().Strings()
+)
+
+func runContentRemoveCommand(ctx context.Context, rep *repo.Repository) error {
+	for _, contentID := range toContentIDs(*contentRemoveIDs) {
+		if err := rep.Content.DeleteContent(contentID); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+func init() {
+	setupShowCommand(contentRemoveCommand)
+	contentRemoveCommand.Action(repositoryAction(runContentRemoveCommand))
+}

cli/command_content_show.go

@@ -0,0 +1,39 @@
+package cli
+
+import (
+	"bytes"
+	"context"
+
+	"github.com/kopia/kopia/repo"
+	"github.com/kopia/kopia/repo/content"
+)
+
+var (
+	contentShowCommand = contentCommands.Command("show", "Show contents by ID.").Alias("cat")
+
+	contentShowIDs = contentShowCommand.Arg("id", "IDs of contents to show").Required().Strings()
+)
+
+func runContentShowCommand(ctx context.Context, rep *repo.Repository) error {
+	for _, contentID := range toContentIDs(*contentShowIDs) {
+		if err := contentShow(ctx, rep, contentID); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+func contentShow(ctx context.Context, r *repo.Repository, contentID content.ID) error {
+	data, err := r.Content.GetContent(ctx, contentID)
+	if err != nil {
+		return err
+	}
+
+	return showContent(bytes.NewReader(data))
+}
+
+func init() {
+	setupShowCommand(contentShowCommand)
+	contentShowCommand.Action(repositoryAction(runContentShowCommand))
+}

cli/command_content_stats.go

@@ -8,24 +8,26 @@
 
 	"github.com/kopia/kopia/internal/units"
 	"github.com/kopia/kopia/repo"
-	"github.com/kopia/kopia/repo/block"
+	"github.com/kopia/kopia/repo/content"
 )
 
 var (
-	blockStatsCommand = blockCommands.Command("stats", "Block statistics")
-	blockStatsRaw     = blockStatsCommand.Flag("raw", "Raw numbers").Short('r').Bool()
+	contentStatsCommand = contentCommands.Command("stats", "Content statistics")
+	contentStatsRaw     = contentStatsCommand.Flag("raw", "Raw numbers").Short('r').Bool()
 )
 
-func runBlockStatsAction(ctx context.Context, rep *repo.Repository) error {
-	blocks, err := rep.Blocks.ListBlockInfos("", true)
+func runContentStatsCommand(ctx context.Context, rep *repo.Repository) error {
+	contents, err := rep.Content.ListContentInfos("", true)
 	if err != nil {
 		return err
 	}
-	sort.Slice(blocks, func(i, j int) bool { return blocks[i].Length < blocks[j].Length })
+	sort.Slice(contents, func(i, j int) bool {
+		return contents[i].Length < contents[j].Length
+	})
 
 	var sizeThreshold uint32 = 10
 	countMap := map[uint32]int{}
-	totalSizeOfBlocksUnder := map[uint32]int64{}
+	totalSizeOfContentsUnder := map[uint32]int64{}
 	var sizeThresholds []uint32
 	for i := 0; i < 8; i++ {
 		sizeThresholds = append(sizeThresholds, sizeThreshold)
@@ -34,51 +36,51 @@ func runBlockStatsAction(ctx context.Context, rep *repo.Repository) error {
 	}
 
 	var totalSize int64
-	for _, b := range blocks {
+	for _, b := range contents {
 		totalSize += int64(b.Length)
 		for s := range countMap {
 			if b.Length < s {
 				countMap[s]++
-				totalSizeOfBlocksUnder[s] += int64(b.Length)
+				totalSizeOfContentsUnder[s] += int64(b.Length)
 			}
 		}
 	}
 
-	fmt.Printf("Block statistics\n")
-	if len(blocks) == 0 {
+	fmt.Printf("Content statistics\n")
+	if len(contents) == 0 {
 		return nil
 	}
 
 	sizeToString := units.BytesStringBase10
-	if *blockStatsRaw {
+	if *contentStatsRaw {
 		sizeToString = func(l int64) string { return strconv.FormatInt(l, 10) }
 	}
 
 	fmt.Println("Size:          ")
 	fmt.Println("  Total              ", sizeToString(totalSize))
-	fmt.Println("  Average            ", sizeToString(totalSize/int64(len(blocks))))
-	fmt.Println("  1st percentile     ", sizeToString(percentileSize(1, blocks)))
-	fmt.Println("  5th percentile     ", sizeToString(percentileSize(5, blocks)))
-	fmt.Println("  10th percentile    ", sizeToString(percentileSize(10, blocks)))
-	fmt.Println("  50th percentile    ", sizeToString(percentileSize(50, blocks)))
-	fmt.Println("  90th percentile    ", sizeToString(percentileSize(90, blocks)))
-	fmt.Println("  95th percentile    ", sizeToString(percentileSize(95, blocks)))
-	fmt.Println("  99th percentile    ", sizeToString(percentileSize(99, blocks)))
+	fmt.Println("  Average            ", sizeToString(totalSize/int64(len(contents))))
+	fmt.Println("  1st percentile     ", sizeToString(percentileSize(1, contents)))
+	fmt.Println("  5th percentile     ", sizeToString(percentileSize(5, contents)))
+	fmt.Println("  10th percentile    ", sizeToString(percentileSize(10, contents)))
+	fmt.Println("  50th percentile    ", sizeToString(percentileSize(50, contents)))
+	fmt.Println("  90th percentile    ", sizeToString(percentileSize(90, contents)))
+	fmt.Println("  95th percentile    ", sizeToString(percentileSize(95, contents)))
+	fmt.Println("  99th percentile    ", sizeToString(percentileSize(99, contents)))
 
 	fmt.Println("Counts:")
 	for _, size := range sizeThresholds {
-		fmt.Printf("  %v blocks with size <%v (total %v)\n", countMap[size], sizeToString(int64(size)), sizeToString(totalSizeOfBlocksUnder[size]))
+		fmt.Printf("  %v contents with size <%v (total %v)\n", countMap[size], sizeToString(int64(size)), sizeToString(totalSizeOfContentsUnder[size]))
 	}
 
 	return nil
 }
 
-func percentileSize(p int, blocks []block.Info) int64 {
-	pos := p * len(blocks) / 100
+func percentileSize(p int, contents []content.Info) int64 {
+	pos := p * len(contents) / 100
 
-	return int64(blocks[pos].Length)
+	return int64(contents[pos].Length)
 }
 
 func init() {
-	blockStatsCommand.Action(repositoryAction(runBlockStatsAction))
+	contentStatsCommand.Action(repositoryAction(runContentStatsCommand))
 }

cli/command_content_verify.go

@@ -0,0 +1,62 @@
+package cli
+
+import (
+	"context"
+
+	"github.com/pkg/errors"
+
+	"github.com/kopia/kopia/repo"
+	"github.com/kopia/kopia/repo/content"
+)
+
+var (
+	contentVerifyCommand = contentCommands.Command("verify", "Verify contents")
+
+	contentVerifyIDs = contentVerifyCommand.Arg("id", "IDs of blocks to show (or 'all')").Required().Strings()
+)
+
+func runContentVerifyCommand(ctx context.Context, rep *repo.Repository) error {
+	for _, contentID := range toContentIDs(*contentVerifyIDs) {
+		if contentID == "all" {
+			return verifyAllBlocks(ctx, rep)
+		}
+		if err := contentVerify(ctx, rep, contentID); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+func verifyAllBlocks(ctx context.Context, rep *repo.Repository) error {
+	contentIDs, err := rep.Content.ListContents("")
+	if err != nil {
+		return errors.Wrap(err, "unable to list contents")
+	}
+
+	var errorCount int
+	for _, contentID := range contentIDs {
+		if err := contentVerify(ctx, rep, contentID); err != nil {
+			errorCount++
+		}
+	}
+	if errorCount == 0 {
+		return nil
+	}
+
+	return errors.Errorf("encountered %v errors", errorCount)
+}
+
+func contentVerify(ctx context.Context, r *repo.Repository, contentID content.ID) error {
+	if _, err := r.Content.GetContent(ctx, contentID); err != nil {
+		log.Warningf("content %v is invalid: %v", contentID, err)
+		return err
+	}
+
+	log.Infof("content %v is ok", contentID)
+	return nil
+}
+
+func init() {
+	contentVerifyCommand.Action(repositoryAction(runContentVerifyCommand))
+}

cli/command_index_list.go

@@ -9,13 +9,13 @@
 )
 
 var (
-	blockIndexListCommand = blockIndexCommands.Command("list", "List block indexes").Alias("ls").Default()
-	blockIndexListSummary = blockIndexListCommand.Flag("summary", "Display block summary").Bool()
-	blockIndexListSort    = blockIndexListCommand.Flag("sort", "Index block sort order").Default("time").Enum("time", "size", "name")
+	blockIndexListCommand = indexCommands.Command("list", "List content indexes").Alias("ls").Default()
+	blockIndexListSummary = blockIndexListCommand.Flag("summary", "Display index blob summary").Bool()
+	blockIndexListSort    = blockIndexListCommand.Flag("sort", "Index blob sort order").Default("time").Enum("time", "size", "name")
 )
 
 func runListBlockIndexesAction(ctx context.Context, rep *repo.Repository) error {
-	blks, err := rep.Blocks.IndexBlobs(ctx)
+	blks, err := rep.Content.IndexBlobs(ctx)
 	if err != nil {
 		return err
 	}

cli/command_index_optimize.go

@@ -0,0 +1,29 @@
+package cli
+
+import (
+	"context"
+
+	"github.com/kopia/kopia/repo"
+	"github.com/kopia/kopia/repo/content"
+)
+
+var (
+	optimizeCommand              = indexCommands.Command("optimize", "Optimize indexes blobs.")
+	optimizeMinSmallBlobs        = optimizeCommand.Flag("min-small-blobs", "Minimum number of small index blobs that can be left after compaction.").Default("1").Int()
+	optimizeMaxSmallBlobs        = optimizeCommand.Flag("max-small-blobs", "Maximum number of small index blobs that can be left after compaction.").Default("1").Int()
+	optimizeSkipDeletedOlderThan = optimizeCommand.Flag("skip-deleted-older-than", "Skip deleted blobs above given age").Duration()
+	optimizeAllIndexes           = optimizeCommand.Flag("all", "Optimize all indexes, even those above maximum size.").Bool()
+)
+
+func runOptimizeCommand(ctx context.Context, rep *repo.Repository) error {
+	return rep.Content.CompactIndexes(ctx, content.CompactOptions{
+		MinSmallBlobs:        *optimizeMinSmallBlobs,
+		MaxSmallBlobs:        *optimizeMaxSmallBlobs,
+		AllIndexes:           *optimizeAllIndexes,
+		SkipDeletedOlderThan: *optimizeSkipDeletedOlderThan,
+	})
+}
+
+func init() {
+	optimizeCommand.Action(repositoryAction(runOptimizeCommand))
+}

cli/command_index_recover.go

@@ -5,13 +5,13 @@
 
 	"github.com/kopia/kopia/repo"
 	"github.com/kopia/kopia/repo/blob"
-	"github.com/kopia/kopia/repo/block"
+	"github.com/kopia/kopia/repo/content"
 )
 
 var (
-	blockIndexRecoverCommand = blockIndexCommands.Command("recover", "Recover block indexes from pack blocks")
-	blockIndexRecoverBlobIDs = blockIndexRecoverCommand.Flag("blobs", "Names of pack blobs to recover (default=all packs)").Strings()
-	blockIndexRecoverCommit  = blockIndexRecoverCommand.Flag("commit", "Commit recovered blocks").Bool()
+	blockIndexRecoverCommand = indexCommands.Command("recover", "Recover indexes from pack blobs")
+	blockIndexRecoverBlobIDs = blockIndexRecoverCommand.Flag("blobs", "Names of pack blobs to recover from (default=all packs)").Strings()
+	blockIndexRecoverCommit  = blockIndexRecoverCommand.Flag("commit", "Commit recovered content").Bool()
 )
 
 func runRecoverBlockIndexesAction(ctx context.Context, rep *repo.Repository) error {
@@ -31,7 +31,7 @@ func runRecoverBlockIndexesAction(ctx context.Context, rep *repo.Repository) err
 	}()
 
 	if len(*blockIndexRecoverBlobIDs) == 0 {
-		return rep.Blobs.ListBlobs(ctx, block.PackBlobIDPrefix, func(bm blob.Metadata) error {
+		return rep.Blobs.ListBlobs(ctx, content.PackBlobIDPrefix, func(bm blob.Metadata) error {
 			recoverIndexFromSinglePackFile(ctx, rep, bm.BlobID, bm.Length, &totalCount)
 			return nil
 		})
@@ -45,7 +45,7 @@ func runRecoverBlockIndexesAction(ctx context.Context, rep *repo.Repository) err
 }
 
 func recoverIndexFromSinglePackFile(ctx context.Context, rep *repo.Repository, blobID blob.ID, length int64, totalCount *int) {
-	recovered, err := rep.Blocks.RecoverIndexFromPackBlob(ctx, blobID, length, *blockIndexRecoverCommit)
+	recovered, err := rep.Content.RecoverIndexFromPackBlob(ctx, blobID, length, *blockIndexRecoverCommit)
 	if err != nil {
 		log.Warningf("unable to recover index from %v: %v", blobID, err)
 		return

cli/command_manifest_rm.go

@@ -11,12 +11,8 @@
 	manifestRemoveItems   = manifestRemoveCommand.Arg("item", "Items to remove").Required().Strings()
 )
 
-func init() {
-	manifestRemoveCommand.Action(repositoryAction(removeMetadataItem))
-}
-
-func removeMetadataItem(ctx context.Context, rep *repo.Repository) error {
-	for _, it := range *manifestRemoveItems {
+func runManifestRemoveCommand(ctx context.Context, rep *repo.Repository) error {
+	for _, it := range toManifestIDs(*manifestRemoveItems) {
 		if err := rep.Manifests.Delete(ctx, it); err != nil {
 			return err
 		}
@@ -24,3 +20,7 @@ func removeMetadataItem(ctx context.Context, rep *repo.Repository) error {
 
 	return nil
 }
+
+func init() {
+	manifestRemoveCommand.Action(repositoryAction(runManifestRemoveCommand))
+}

cli/command_manifest_show.go

@@ -7,6 +7,7 @@
 	"github.com/pkg/errors"
 
 	"github.com/kopia/kopia/repo"
+	"github.com/kopia/kopia/repo/manifest"
 )
 
 var (
@@ -18,8 +19,17 @@ func init() {
 	manifestShowCommand.Action(repositoryAction(showManifestItems))
 }
 
+func toManifestIDs(s []string) []manifest.ID {
+	var result []manifest.ID
+
+	for _, it := range s {
+		result = append(result, manifest.ID(it))
+	}
+	return result
+}
+
 func showManifestItems(ctx context.Context, rep *repo.Repository) error {
-	for _, it := range *manifestShowItems {
+	for _, it := range toManifestIDs(*manifestShowItems) {
 		md, err := rep.Manifests.GetMetadata(ctx, it)
 		if err != nil {
 			return errors.Wrapf(err, "error getting metadata for %q", it)

cli/command_object_verify.go

@@ -13,7 +13,8 @@
 
 	"github.com/kopia/kopia/internal/parallelwork"
 	"github.com/kopia/kopia/repo"
-	"github.com/kopia/kopia/repo/block"
+	"github.com/kopia/kopia/repo/content"
+	"github.com/kopia/kopia/repo/manifest"
 	"github.com/kopia/kopia/repo/object"
 	"github.com/kopia/kopia/snapshot"
 	"github.com/kopia/kopia/snapshot/snapshotfs"
@@ -163,7 +164,7 @@ func (v *verifier) doVerifyObject(ctx context.Context, oid object.ID, path strin
 
 func (v *verifier) readEntireObject(ctx context.Context, oid object.ID, path string) error {
 	log.Debugf("reading object %v %v", oid, path)
-	ctx = block.UsingBlockCache(ctx, false)
+	ctx = content.UsingContentCache(ctx, false)
 
 	// also read the entire file
 	r, err := v.om.Open(ctx, oid)
@@ -240,7 +241,7 @@ func enqueueRootsToVerify(ctx context.Context, v *verifier, rep *repo.Repository
 }
 
 func loadSourceManifests(ctx context.Context, rep *repo.Repository, all bool, sources []string) ([]*snapshot.Manifest, error) {
-	var manifestIDs []string
+	var manifestIDs []manifest.ID
 	if *verifyCommandAllSources {
 		man, err := snapshot.ListSnapshotManifests(ctx, rep, nil)
 		if err != nil {

cli/command_policy.go

@@ -6,6 +6,7 @@
 	"github.com/pkg/errors"
 
 	"github.com/kopia/kopia/repo"
+	"github.com/kopia/kopia/repo/manifest"
 	"github.com/kopia/kopia/snapshot"
 	"github.com/kopia/kopia/snapshot/policy"
 )
@@ -23,7 +24,8 @@ func policyTargets(ctx context.Context, rep *repo.Repository, globalFlag *bool,
 
 	var res []snapshot.SourceInfo
 	for _, ts := range *targetsFlag {
-		if t, err := policy.GetPolicyByID(ctx, rep, ts); err == nil {
+		// try loading policy by its manifest ID
+		if t, err := policy.GetPolicyByID(ctx, rep, manifest.ID(ts)); err == nil {
 			res = append(res, t.Target())
 			continue
 		}

cli/command_repository_connect.go

@@ -8,7 +8,7 @@
 
 	"github.com/kopia/kopia/repo"
 	"github.com/kopia/kopia/repo/blob"
-	"github.com/kopia/kopia/repo/block"
+	"github.com/kopia/kopia/repo/content"
 
 	"gopkg.in/alecthomas/kingpin.v2"
 )
@@ -32,7 +32,7 @@ func setupConnectOptions(cmd *kingpin.CmdClause) {
 
 func connectOptions() repo.ConnectOptions {
 	return repo.ConnectOptions{
-		CachingOptions: block.CachingOptions{
+		CachingOptions: content.CachingOptions{
 			CacheDirectory:          connectCacheDirectory,
 			MaxCacheSizeBytes:       connectMaxCacheSizeMB << 20,
 			MaxListCacheDurationSec: int(connectMaxListCacheDuration.Seconds()),

cli/command_repository_create.go

@@ -9,7 +9,7 @@
 	"github.com/kopia/kopia/fs/ignorefs"
 	"github.com/kopia/kopia/repo"
 	"github.com/kopia/kopia/repo/blob"
-	"github.com/kopia/kopia/repo/block"
+	"github.com/kopia/kopia/repo/content"
 	"github.com/kopia/kopia/repo/object"
 	"github.com/kopia/kopia/snapshot/policy"
 )
@@ -17,8 +17,8 @@
 var (
 	createCommand = repositoryCommands.Command("create", "Create new repository in a specified location.")
 
-	createBlockHashFormat       = createCommand.Flag("block-hash", "Block hash algorithm.").PlaceHolder("ALGO").Default(block.DefaultHash).Enum(block.SupportedHashAlgorithms()...)
-	createBlockEncryptionFormat = createCommand.Flag("encryption", "Block encryption algorithm.").PlaceHolder("ALGO").Default(block.DefaultEncryption).Enum(block.SupportedEncryptionAlgorithms()...)
+	createBlockHashFormat       = createCommand.Flag("block-hash", "Block hash algorithm.").PlaceHolder("ALGO").Default(content.DefaultHash).Enum(content.SupportedHashAlgorithms()...)
+	createBlockEncryptionFormat = createCommand.Flag("encryption", "Block encryption algorithm.").PlaceHolder("ALGO").Default(content.DefaultEncryption).Enum(content.SupportedEncryptionAlgorithms()...)
 	createSplitter              = createCommand.Flag("object-splitter", "The splitter to use for new objects in the repository").Default(object.DefaultSplitter).Enum(object.SupportedSplitters...)
 
 	createOnly = createCommand.Flag("create-only", "Create repository, but don't connect to it.").Short('c').Bool()
@@ -42,7 +42,7 @@ func init() {
 
 func newRepositoryOptionsFromFlags() *repo.NewRepositoryOptions {
 	return &repo.NewRepositoryOptions{
-		BlockFormat: block.FormattingOptions{
+		BlockFormat: content.FormattingOptions{
 			Hash:       *createBlockHashFormat,
 			Encryption: *createBlockEncryptionFormat,
 		},

cli/command_repository_repair.go

@@ -12,20 +12,20 @@
 var (
 	repairCommand = repositoryCommands.Command("repair", "Repairs respository.")
 
-	repairCommandRecoverFormatBlock       = repairCommand.Flag("recover-format", "Recover format block from a copy").Default("auto").Enum("auto", "yes", "no")
-	repairCommandRecoverFormatBlockPrefix = repairCommand.Flag("recover-format-block-prefix", "Prefix of file names").Default("p").String()
-	repairDryDrun                         = repairCommand.Flag("dry-run", "Do not modify repository").Short('n').Bool()
+	repairCommandRecoverFormatBlob       = repairCommand.Flag("recover-format", "Recover format blob from a copy").Default("auto").Enum("auto", "yes", "no")
+	repairCommandRecoverFormatBlobPrefix = repairCommand.Flag("recover-format-block-prefix", "Prefix of file names").Default("p").String()
+	repairDryDrun                        = repairCommand.Flag("dry-run", "Do not modify repository").Short('n').Bool()
 )
 
 func runRepairCommandWithStorage(ctx context.Context, st blob.Storage) error {
-	if err := maybeRecoverFormatBlock(ctx, st, *repairCommandRecoverFormatBlockPrefix); err != nil {
+	if err := maybeRecoverFormatBlob(ctx, st, *repairCommandRecoverFormatBlobPrefix); err != nil {
 		return err
 	}
 	return nil
 }
 
-func maybeRecoverFormatBlock(ctx context.Context, st blob.Storage, prefix string) error {
-	switch *repairCommandRecoverFormatBlock {
+func maybeRecoverFormatBlob(ctx context.Context, st blob.Storage, prefix string) error {
+	switch *repairCommandRecoverFormatBlob {
 	case "auto":
 		log.Infof("looking for format blob...")
 		if _, err := st.GetBlob(ctx, repo.FormatBlobID, 0, -1); err == nil {
@@ -37,15 +37,15 @@ func maybeRecoverFormatBlock(ctx context.Context, st blob.Storage, prefix string
 		return nil
 	}
 
-	return recoverFormatBlock(ctx, st, *repairCommandRecoverFormatBlockPrefix)
+	return recoverFormatBlob(ctx, st, *repairCommandRecoverFormatBlobPrefix)
 }
 
-func recoverFormatBlock(ctx context.Context, st blob.Storage, prefix string) error {
+func recoverFormatBlob(ctx context.Context, st blob.Storage, prefix string) error {
 	errSuccess := errors.New("success")
 
-	err := st.ListBlobs(ctx, blob.ID(*repairCommandRecoverFormatBlockPrefix), func(bi blob.Metadata) error {
-		log.Infof("looking for replica of format block in %v...", bi.BlobID)
-		if b, err := repo.RecoverFormatBlock(ctx, st, bi.BlobID, bi.Length); err == nil {
+	err := st.ListBlobs(ctx, blob.ID(*repairCommandRecoverFormatBlobPrefix), func(bi blob.Metadata) error {
+		log.Infof("looking for replica of format blob in %v...", bi.BlobID)
+		if b, err := repo.RecoverFormatBlob(ctx, st, bi.BlobID, bi.Length); err == nil {
 			if !*repairDryDrun {
 				if puterr := st.PutBlob(ctx, repo.FormatBlobID, b); puterr != nil {
 					return puterr
@@ -63,7 +63,7 @@ func recoverFormatBlock(ctx context.Context, st blob.Storage, prefix string) err
 	case errSuccess:
 		return nil
 	case nil:
-		return errors.New("could not find a replica of a format block")
+		return errors.New("could not find a replica of a format blob")
 	default:
 		return err
 	}

cli/command_repository_status.go

@@ -38,10 +38,10 @@ func runStatusCommand(ctx context.Context, rep *repo.Repository) error {
 	fmt.Println()
 	fmt.Printf("Unique ID:           %x\n", rep.UniqueID)
 	fmt.Println()
-	fmt.Printf("Block hash:          %v\n", rep.Blocks.Format.Hash)
-	fmt.Printf("Block encryption:    %v\n", rep.Blocks.Format.Encryption)
-	fmt.Printf("Block fmt version:   %v\n", rep.Blocks.Format.Version)
-	fmt.Printf("Max pack length:     %v\n", units.BytesStringBase2(int64(rep.Blocks.Format.MaxPackSize)))
+	fmt.Printf("Block hash:          %v\n", rep.Content.Format.Hash)
+	fmt.Printf("Block encryption:    %v\n", rep.Content.Format.Encryption)
+	fmt.Printf("Block fmt version:   %v\n", rep.Content.Format.Version)
+	fmt.Printf("Max pack length:     %v\n", units.BytesStringBase2(int64(rep.Content.Format.MaxPackSize)))
 	fmt.Printf("Splitter:            %v\n", rep.Objects.Format.Splitter)
 
 	return nil

cli/command_snapshot_create.go

@@ -85,7 +85,7 @@ func runBackupCommand(ctx context.Context, rep *repo.Repository) error {
 
 func snapshotSingleSource(ctx context.Context, rep *repo.Repository, u *snapshotfs.Uploader, sourceInfo snapshot.SourceInfo) error {
 	t0 := time.Now()
-	rep.Blocks.ResetStats()
+	rep.Content.ResetStats()
 
 	localEntry := mustGetLocalFSEntry(sourceInfo.Path)
 

cli/command_snapshot_list.go

@@ -12,6 +12,7 @@
 	"github.com/kopia/kopia/fs"
 	"github.com/kopia/kopia/internal/units"
 	"github.com/kopia/kopia/repo"
+	"github.com/kopia/kopia/repo/manifest"
 	"github.com/kopia/kopia/repo/object"
 	"github.com/kopia/kopia/snapshot"
 	"github.com/kopia/kopia/snapshot/policy"
@@ -34,7 +35,7 @@
 	maxResultsPerPath                = snapshotListCommand.Flag("max-results", "Maximum number of entries per source.").Default("100").Short('n').Int()
 )
 
-func findSnapshotsForSource(ctx context.Context, rep *repo.Repository, sourceInfo snapshot.SourceInfo) (manifestIDs []string, relPath string, err error) {
+func findSnapshotsForSource(ctx context.Context, rep *repo.Repository, sourceInfo snapshot.SourceInfo) (manifestIDs []manifest.ID, relPath string, err error) {
 	for len(sourceInfo.Path) > 0 {
 		list, err := snapshot.ListSnapshotManifests(ctx, rep, &sourceInfo)
 		if err != nil {
@@ -63,7 +64,7 @@ func findSnapshotsForSource(ctx context.Context, rep *repo.Repository, sourceInf
 	return nil, "", nil
 }
 
-func findManifestIDs(ctx context.Context, rep *repo.Repository, source string) ([]string, string, error) {
+func findManifestIDs(ctx context.Context, rep *repo.Repository, source string) ([]manifest.ID, string, error) {
 	if source == "" {
 		man, err := snapshot.ListSnapshotManifests(ctx, rep, nil)
 		return man, "", err
@@ -200,7 +201,7 @@ func outputManifestFromSingleSource(ctx context.Context, rep *repo.Repository, m
 		}
 
 		if *snapshotListShowItemID {
-			bits = append(bits, "manifest:"+m.ID)
+			bits = append(bits, "manifest:"+string(m.ID))
 		}
 		if *snapshotListShowHashCache {
 			bits = append(bits, "hashcache:"+m.HashCacheID.String())

examples/upload_download/main.go

@@ -28,13 +28,13 @@ func main() {
 
 	uploadAndDownloadObjects(ctx, r)
 
-	// Now list blocks found in the repository.
-	blks, err := r.Blocks.ListBlocks("")
+	// Now list contents found in the repository.
+	cnts, err := r.Content.ListContents("")
 	if err != nil {
 		log.Printf("err: %v", err)
 	}
 
-	for _, b := range blks {
-		log.Printf("found block %v", b)
+	for _, c := range cnts {
+		log.Printf("found content %v", c)
 	}
 }

examples/upload_download/setup_repository.go

@@ -11,7 +11,7 @@
 	"github.com/kopia/kopia/repo"
 	"github.com/kopia/kopia/repo/blob/filesystem"
 	"github.com/kopia/kopia/repo/blob/logging"
-	"github.com/kopia/kopia/repo/block"
+	"github.com/kopia/kopia/repo/content"
 )
 
 const (
@@ -44,7 +44,7 @@ func setupRepositoryAndConnect(ctx context.Context, password string) error {
 
 		// now establish connection to repository and create configuration file.
 		if err := repo.Connect(ctx, configFile, st, password, repo.ConnectOptions{
-			CachingOptions: block.CachingOptions{
+			CachingOptions: content.CachingOptions{
 				CacheDirectory:    cacheDirectory,
 				MaxCacheSizeBytes: 100000000,
 			},

internal/blobtesting/asserts.go

@@ -10,18 +10,18 @@
 	"github.com/kopia/kopia/repo/blob"
 )
 
-// AssertGetBlock asserts that the specified storage block has correct content.
-func AssertGetBlock(ctx context.Context, t *testing.T, s blob.Storage, block blob.ID, expected []byte) {
+// AssertGetBlob asserts that the specified BLOB has correct content.
+func AssertGetBlob(ctx context.Context, t *testing.T, s blob.Storage, blobID blob.ID, expected []byte) {
 	t.Helper()
 
-	b, err := s.GetBlob(ctx, block, 0, -1)
+	b, err := s.GetBlob(ctx, blobID, 0, -1)
 	if err != nil {
-		t.Errorf("GetBlob(%v) returned error %v, expected data: %v", block, err, expected)
+		t.Errorf("GetBlob(%v) returned error %v, expected data: %v", blobID, err, expected)
 		return
 	}
 
 	if !bytes.Equal(b, expected) {
-		t.Errorf("GetBlob(%v) returned %x, but expected %x", block, b, expected)
+		t.Errorf("GetBlob(%v) returned %x, but expected %x", blobID, b, expected)
 	}
 
 	half := int64(len(expected) / 2)
@@ -29,41 +29,41 @@ func AssertGetBlock(ctx context.Context, t *testing.T, s blob.Storage, block blo
 		return
 	}
 
-	b, err = s.GetBlob(ctx, block, 0, 0)
+	b, err = s.GetBlob(ctx, blobID, 0, 0)
 	if err != nil {
-		t.Errorf("GetBlob(%v) returned error %v, expected data: %v", block, err, expected)
+		t.Errorf("GetBlob(%v) returned error %v, expected data: %v", blobID, err, expected)
 		return
 	}
 
 	if len(b) != 0 {
-		t.Errorf("GetBlob(%v) returned non-zero length: %v", block, len(b))
+		t.Errorf("GetBlob(%v) returned non-zero length: %v", blobID, len(b))
 		return
 	}
 
-	b, err = s.GetBlob(ctx, block, 0, half)
+	b, err = s.GetBlob(ctx, blobID, 0, half)
 	if err != nil {
-		t.Errorf("GetBlob(%v) returned error %v, expected data: %v", block, err, expected)
+		t.Errorf("GetBlob(%v) returned error %v, expected data: %v", blobID, err, expected)
 		return
 	}
 
 	if !bytes.Equal(b, expected[0:half]) {
-		t.Errorf("GetBlob(%v) returned %x, but expected %x", block, b, expected[0:half])
+		t.Errorf("GetBlob(%v) returned %x, but expected %x", blobID, b, expected[0:half])
 	}
 
-	b, err = s.GetBlob(ctx, block, half, int64(len(expected))-half)
+	b, err = s.GetBlob(ctx, blobID, half, int64(len(expected))-half)
 	if err != nil {
-		t.Errorf("GetBlob(%v) returned error %v, expected data: %v", block, err, expected)
+		t.Errorf("GetBlob(%v) returned error %v, expected data: %v", blobID, err, expected)
 		return
 	}
 
 	if !bytes.Equal(b, expected[len(expected)-int(half):]) {
-		t.Errorf("GetBlob(%v) returned %x, but expected %x", block, b, expected[len(expected)-int(half):])
+		t.Errorf("GetBlob(%v) returned %x, but expected %x", blobID, b, expected[len(expected)-int(half):])
 	}
 
-	AssertInvalidOffsetLength(ctx, t, s, block, -3, 1)
-	AssertInvalidOffsetLength(ctx, t, s, block, int64(len(expected)), 3)
-	AssertInvalidOffsetLength(ctx, t, s, block, int64(len(expected)-1), 3)
-	AssertInvalidOffsetLength(ctx, t, s, block, int64(len(expected)+1), 3)
+	AssertInvalidOffsetLength(ctx, t, s, blobID, -3, 1)
+	AssertInvalidOffsetLength(ctx, t, s, blobID, int64(len(expected)), 3)
+	AssertInvalidOffsetLength(ctx, t, s, blobID, int64(len(expected)-1), 3)
+	AssertInvalidOffsetLength(ctx, t, s, blobID, int64(len(expected)+1), 3)
 }
 
 // AssertInvalidOffsetLength verifies that the given combination of (offset,length) fails on GetBlob()
@@ -73,8 +73,8 @@ func AssertInvalidOffsetLength(ctx context.Context, t *testing.T, s blob.Storage
 	}
 }
 
-// AssertGetBlockNotFound asserts that GetBlob() for specified storage block returns ErrNotFound.
-func AssertGetBlockNotFound(ctx context.Context, t *testing.T, s blob.Storage, blobID blob.ID) {
+// AssertGetBlobNotFound asserts that GetBlob() for specified blobID returns ErrNotFound.
+func AssertGetBlobNotFound(ctx context.Context, t *testing.T, s blob.Storage, blobID blob.ID) {
 	t.Helper()
 
 	b, err := s.GetBlob(ctx, blobID, 0, -1)

internal/blobtesting/verify.go

@@ -24,7 +24,7 @@ func VerifyStorage(ctx context.Context, t *testing.T, r blob.Storage) {
 
 	// First verify that blocks don't exist.
 	for _, b := range blocks {
-		AssertGetBlockNotFound(ctx, t, r, b.blk)
+		AssertGetBlobNotFound(ctx, t, r, b.blk)
 	}
 
 	ctx2 := blob.WithUploadProgressCallback(ctx, func(desc string, completed, total int64) {
@@ -34,10 +34,10 @@ func VerifyStorage(ctx context.Context, t *testing.T, r blob.Storage) {
 	// Now add blocks.
 	for _, b := range blocks {
 		if err := r.PutBlob(ctx2, b.blk, b.contents); err != nil {
-			t.Errorf("can't put block: %v", err)
+			t.Errorf("can't put blob: %v", err)
 		}
 
-		AssertGetBlock(ctx, t, r, b.blk, b.contents)
+		AssertGetBlob(ctx, t, r, b.blk, b.contents)
 	}
 
 	AssertListResults(ctx, t, r, "", blocks[0].blk, blocks[1].blk, blocks[2].blk, blocks[3].blk, blocks[4].blk)
@@ -46,10 +46,10 @@ func VerifyStorage(ctx context.Context, t *testing.T, r blob.Storage) {
 	// Overwrite blocks.
 	for _, b := range blocks {
 		if err := r.PutBlob(ctx, b.blk, b.contents); err != nil {
-			t.Errorf("can't put block: %v", err)
+			t.Errorf("can't put blob: %v", err)
 		}
 
-		AssertGetBlock(ctx, t, r, b.blk, b.contents)
+		AssertGetBlob(ctx, t, r, b.blk, b.contents)
 	}
 
 	if err := r.DeleteBlob(ctx, blocks[0].blk); err != nil {

internal/repotesting/repotesting.go

@@ -11,7 +11,7 @@
 	"github.com/kopia/kopia/repo"
 	"github.com/kopia/kopia/repo/blob"
 	"github.com/kopia/kopia/repo/blob/filesystem"
-	"github.com/kopia/kopia/repo/block"
+	"github.com/kopia/kopia/repo/content"
 	"github.com/kopia/kopia/repo/object"
 )
 
@@ -42,7 +42,7 @@ func (e *Environment) Setup(t *testing.T, opts ...func(*repo.NewRepositoryOption
 	}
 
 	opt := &repo.NewRepositoryOptions{
-		BlockFormat: block.FormattingOptions{
+		BlockFormat: content.FormattingOptions{
 			HMACSecret: []byte{},
 			Hash:       "HMAC-SHA256",
 			Encryption: "NONE",
@@ -121,8 +121,8 @@ func (e *Environment) MustReopen(t *testing.T) {
 	}
 }
 
-// VerifyStorageBlockCount verifies that the underlying storage contains the specified number of blocks.
-func (e *Environment) VerifyStorageBlockCount(t *testing.T, want int) {
+// VerifyBlobCount verifies that the underlying storage contains the specified number of blobs.
+func (e *Environment) VerifyBlobCount(t *testing.T, want int) {
 	var got int
 
 	_ = e.Repository.Blobs.ListBlobs(context.Background(), "", func(_ blob.Metadata) error {
@@ -131,6 +131,6 @@ func (e *Environment) VerifyStorageBlockCount(t *testing.T, want int) {
 	})
 
 	if got != want {
-		t.Errorf("got unexpected number of storage blocks: %v, wanted %v", got, want)
+		t.Errorf("got unexpected number of BLOBs: %v, wanted %v", got, want)
 	}
 }

internal/server/api_snapshot_list.go

@@ -8,12 +8,13 @@
 	"time"
 
 	"github.com/kopia/kopia/fs"
+	"github.com/kopia/kopia/repo/manifest"
 	"github.com/kopia/kopia/snapshot"
 	"github.com/kopia/kopia/snapshot/policy"
 )
 
 type snapshotListEntry struct {
-	ID               string               `json:"id"`
+	ID               manifest.ID          `json:"id"`
 	Source           snapshot.SourceInfo  `json:"source"`
 	Description      string               `json:"description"`
 	StartTime        time.Time            `json:"startTime"`

internal/server/api_status.go

@@ -8,7 +8,7 @@
 )
 
 func (s *Server) handleStatus(ctx context.Context, r *http.Request) (interface{}, *apiError) {
-	bf := s.rep.Blocks.Format
+	bf := s.rep.Content.Format
 	bf.HMACSecret = nil
 	bf.MasterKey = nil
 

internal/serverapi/serverapi.go

@@ -3,17 +3,17 @@
 import (
 	"time"
 
-	"github.com/kopia/kopia/repo/block"
+	"github.com/kopia/kopia/repo/content"
 	"github.com/kopia/kopia/snapshot"
 	"github.com/kopia/kopia/snapshot/policy"
 )
 
 // StatusResponse is the response of 'status' HTTP API command.
 type StatusResponse struct {
-	ConfigFile      string                  `json:"configFile"`
-	CacheDir        string                  `json:"cacheDir"`
-	BlockFormatting block.FormattingOptions `json:"blockFormatting"`
-	Storage         string                  `json:"storage"`
+	ConfigFile      string                    `json:"configFile"`
+	CacheDir        string                    `json:"cacheDir"`
+	BlockFormatting content.FormattingOptions `json:"blockFormatting"`
+	Storage         string                    `json:"storage"`
 }
 
 // SourcesResponse is the response of 'sources' HTTP API command.

repo/block/block_index_recovery_test.go

@@ -1,91 +0,0 @@
-package block
-
-import (
-	"context"
-	"testing"
-	"time"
-
-	"github.com/kopia/kopia/internal/blobtesting"
-	"github.com/kopia/kopia/repo/blob"
-)
-
-func TestBlockIndexRecovery(t *testing.T) {
-	ctx := context.Background()
-	data := blobtesting.DataMap{}
-	keyTime := map[blob.ID]time.Time{}
-	bm := newTestBlockManager(data, keyTime, nil)
-	block1 := writeBlockAndVerify(ctx, t, bm, seededRandomData(10, 100))
-	block2 := writeBlockAndVerify(ctx, t, bm, seededRandomData(11, 100))
-	block3 := writeBlockAndVerify(ctx, t, bm, seededRandomData(12, 100))
-
-	if err := bm.Flush(ctx); err != nil {
-		t.Errorf("flush error: %v", err)
-	}
-
-	// delete all index blobs
-	assertNoError(t, bm.st.ListBlobs(ctx, newIndexBlobPrefix, func(bi blob.Metadata) error {
-		log.Debugf("deleting %v", bi.BlobID)
-		return bm.st.DeleteBlob(ctx, bi.BlobID)
-	}))
-
-	// now with index blobs gone, all blocks appear to not be found
-	bm = newTestBlockManager(data, keyTime, nil)
-	verifyBlockNotFound(ctx, t, bm, block1)
-	verifyBlockNotFound(ctx, t, bm, block2)
-	verifyBlockNotFound(ctx, t, bm, block3)
-
-	totalRecovered := 0
-
-	// pass 1 - just list blocks to recover, but don't commit
-	err := bm.st.ListBlobs(ctx, PackBlobIDPrefix, func(bi blob.Metadata) error {
-		infos, err := bm.RecoverIndexFromPackBlob(ctx, bi.BlobID, bi.Length, false)
-		if err != nil {
-			return err
-		}
-		totalRecovered += len(infos)
-		log.Debugf("recovered %v blocks", len(infos))
-		return nil
-	})
-	if err != nil {
-		t.Errorf("error recovering: %v", err)
-	}
-
-	if got, want := totalRecovered, 3; got != want {
-		t.Errorf("invalid # of blocks recovered: %v, want %v", got, want)
-	}
-
-	// blocks are stil not found
-	verifyBlockNotFound(ctx, t, bm, block1)
-	verifyBlockNotFound(ctx, t, bm, block2)
-	verifyBlockNotFound(ctx, t, bm, block3)
-
-	// pass 2 now pass commit=true to add recovered blocks to index
-	totalRecovered = 0
-
-	err = bm.st.ListBlobs(ctx, PackBlobIDPrefix, func(bi blob.Metadata) error {
-		infos, err := bm.RecoverIndexFromPackBlob(ctx, bi.BlobID, bi.Length, true)
-		if err != nil {
-			return err
-		}
-		totalRecovered += len(infos)
-		log.Debugf("recovered %v blocks", len(infos))
-		return nil
-	})
-	if err != nil {
-		t.Errorf("error recovering: %v", err)
-	}
-
-	if got, want := totalRecovered, 3; got != want {
-		t.Errorf("invalid # of blocks recovered: %v, want %v", got, want)
-	}
-
-	verifyBlock(ctx, t, bm, block1, seededRandomData(10, 100))
-	verifyBlock(ctx, t, bm, block2, seededRandomData(11, 100))
-	verifyBlock(ctx, t, bm, block3, seededRandomData(12, 100))
-	if err := bm.Flush(ctx); err != nil {
-		t.Errorf("flush error: %v", err)
-	}
-	verifyBlock(ctx, t, bm, block1, seededRandomData(10, 100))
-	verifyBlock(ctx, t, bm, block2, seededRandomData(11, 100))
-	verifyBlock(ctx, t, bm, block3, seededRandomData(12, 100))
-}

repo/block/block_manager_test.go

@@ -1,911 +0,0 @@
-package block
-
-import (
-	"bytes"
-	"context"
-	"crypto/hmac"
-	"crypto/sha256"
-	"encoding/hex"
-	"errors"
-	"fmt"
-	"math/rand"
-	"reflect"
-	"strings"
-	"sync"
-	"testing"
-	"time"
-
-	logging "github.com/op/go-logging"
-
-	"github.com/kopia/kopia/internal/blobtesting"
-	"github.com/kopia/kopia/repo/blob"
-)
-
-const (
-	maxPackSize = 2000
-)
-
-var fakeTime = time.Date(2017, 1, 1, 0, 0, 0, 0, time.UTC)
-var hmacSecret = []byte{1, 2, 3}
-
-func init() {
-	logging.SetLevel(logging.DEBUG, "")
-}
-
-func TestBlockManagerEmptyFlush(t *testing.T) {
-	ctx := context.Background()
-	data := blobtesting.DataMap{}
-	keyTime := map[blob.ID]time.Time{}
-	bm := newTestBlockManager(data, keyTime, nil)
-	bm.Flush(ctx)
-	if got, want := len(data), 0; got != want {
-		t.Errorf("unexpected number of blocks: %v, wanted %v", got, want)
-	}
-}
-
-func TestBlockZeroBytes1(t *testing.T) {
-	ctx := context.Background()
-	data := blobtesting.DataMap{}
-	keyTime := map[blob.ID]time.Time{}
-	bm := newTestBlockManager(data, keyTime, nil)
-	blockID := writeBlockAndVerify(ctx, t, bm, []byte{})
-	bm.Flush(ctx)
-	if got, want := len(data), 2; got != want {
-		t.Errorf("unexpected number of blocks: %v, wanted %v", got, want)
-	}
-	dumpBlockManagerData(t, data)
-	bm = newTestBlockManager(data, keyTime, nil)
-	verifyBlock(ctx, t, bm, blockID, []byte{})
-}
-
-func TestBlockZeroBytes2(t *testing.T) {
-	ctx := context.Background()
-	data := blobtesting.DataMap{}
-	keyTime := map[blob.ID]time.Time{}
-	bm := newTestBlockManager(data, keyTime, nil)
-	writeBlockAndVerify(ctx, t, bm, seededRandomData(10, 10))
-	writeBlockAndVerify(ctx, t, bm, []byte{})
-	bm.Flush(ctx)
-	if got, want := len(data), 2; got != want {
-		t.Errorf("unexpected number of blocks: %v, wanted %v", got, want)
-		dumpBlockManagerData(t, data)
-	}
-}
-
-func TestBlockManagerSmallBlockWrites(t *testing.T) {
-	ctx := context.Background()
-	data := blobtesting.DataMap{}
-	keyTime := map[blob.ID]time.Time{}
-	bm := newTestBlockManager(data, keyTime, nil)
-
-	for i := 0; i < 100; i++ {
-		writeBlockAndVerify(ctx, t, bm, seededRandomData(i, 10))
-	}
-	if got, want := len(data), 0; got != want {
-		t.Errorf("unexpected number of blocks: %v, wanted %v", got, want)
-	}
-	bm.Flush(ctx)
-	if got, want := len(data), 2; got != want {
-		t.Errorf("unexpected number of blocks: %v, wanted %v", got, want)
-	}
-}
-
-func TestBlockManagerDedupesPendingBlocks(t *testing.T) {
-	ctx := context.Background()
-	data := blobtesting.DataMap{}
-	keyTime := map[blob.ID]time.Time{}
-	bm := newTestBlockManager(data, keyTime, nil)
-
-	for i := 0; i < 100; i++ {
-		writeBlockAndVerify(ctx, t, bm, seededRandomData(0, 999))
-	}
-	if got, want := len(data), 0; got != want {
-		t.Errorf("unexpected number of blocks: %v, wanted %v", got, want)
-	}
-	bm.Flush(ctx)
-	if got, want := len(data), 2; got != want {
-		t.Errorf("unexpected number of blocks: %v, wanted %v", got, want)
-	}
-}
-
-func TestBlockManagerDedupesPendingAndUncommittedBlocks(t *testing.T) {
-	ctx := context.Background()
-	data := blobtesting.DataMap{}
-	keyTime := map[blob.ID]time.Time{}
-	bm := newTestBlockManager(data, keyTime, nil)
-
-	// no writes here, all data fits in a single pack.
-	writeBlockAndVerify(ctx, t, bm, seededRandomData(0, 950))
-	writeBlockAndVerify(ctx, t, bm, seededRandomData(1, 950))
-	writeBlockAndVerify(ctx, t, bm, seededRandomData(2, 10))
-	if got, want := len(data), 0; got != want {
-		t.Errorf("unexpected number of blocks: %v, wanted %v", got, want)
-	}
-
-	// no writes here
-	writeBlockAndVerify(ctx, t, bm, seededRandomData(0, 950))
-	writeBlockAndVerify(ctx, t, bm, seededRandomData(1, 950))
-	writeBlockAndVerify(ctx, t, bm, seededRandomData(2, 10))
-	if got, want := len(data), 0; got != want {
-		t.Errorf("unexpected number of blocks: %v, wanted %v", got, want)
-	}
-	bm.Flush(ctx)
-
-	// this flushes the pack block + index blob
-	if got, want := len(data), 2; got != want {
-		dumpBlockManagerData(t, data)
-		t.Errorf("unexpected number of blocks: %v, wanted %v", got, want)
-	}
-}
-
-func TestBlockManagerEmpty(t *testing.T) {
-	ctx := context.Background()
-	data := blobtesting.DataMap{}
-	keyTime := map[blob.ID]time.Time{}
-	bm := newTestBlockManager(data, keyTime, nil)
-
-	noSuchBlockID := string(hashValue([]byte("foo")))
-
-	b, err := bm.GetBlock(ctx, noSuchBlockID)
-	if err != ErrBlockNotFound {
-		t.Errorf("unexpected error when getting non-existent block: %v, %v", b, err)
-	}
-
-	bi, err := bm.BlockInfo(ctx, noSuchBlockID)
-	if err != ErrBlockNotFound {
-		t.Errorf("unexpected error when getting non-existent block info: %v, %v", bi, err)
-	}
-
-	if got, want := len(data), 0; got != want {
-		t.Errorf("unexpected number of blocks: %v, wanted %v", got, want)
-	}
-}
-
-func verifyActiveIndexBlobCount(ctx context.Context, t *testing.T, bm *Manager, expected int) {
-	t.Helper()
-
-	blks, err := bm.IndexBlobs(ctx)
-	if err != nil {
-		t.Errorf("error listing active index blobs: %v", err)
-		return
-	}
-
-	if got, want := len(blks), expected; got != want {
-		t.Errorf("unexpected number of active index blobs %v, expected %v (%v)", got, want, blks)
-	}
-}
-func TestBlockManagerInternalFlush(t *testing.T) {
-	ctx := context.Background()
-	data := blobtesting.DataMap{}
-	keyTime := map[blob.ID]time.Time{}
-	bm := newTestBlockManager(data, keyTime, nil)
-
-	for i := 0; i < 100; i++ {
-		b := make([]byte, 25)
-		rand.Read(b)
-		writeBlockAndVerify(ctx, t, bm, b)
-	}
-
-	// 1 data block written, but no index yet.
-	if got, want := len(data), 1; got != want {
-		t.Errorf("unexpected number of blocks: %v, wanted %v", got, want)
-	}
-
-	// do it again - should be 2 blocks + 1000 bytes pending.
-	for i := 0; i < 100; i++ {
-		b := make([]byte, 25)
-		rand.Read(b)
-		writeBlockAndVerify(ctx, t, bm, b)
-	}
-
-	// 2 data blocks written, but no index yet.
-	if got, want := len(data), 2; got != want {
-		t.Errorf("unexpected number of blocks: %v, wanted %v", got, want)
-	}
-
-	bm.Flush(ctx)
-
-	// third block gets written, followed by index.
-	if got, want := len(data), 4; got != want {
-		dumpBlockManagerData(t, data)
-		t.Errorf("unexpected number of blocks: %v, wanted %v", got, want)
-	}
-}
-
-func TestBlockManagerWriteMultiple(t *testing.T) {
-	ctx := context.Background()
-	data := blobtesting.DataMap{}
-	keyTime := map[blob.ID]time.Time{}
-	timeFunc := fakeTimeNowWithAutoAdvance(fakeTime, 1*time.Second)
-	bm := newTestBlockManager(data, keyTime, timeFunc)
-
-	var blockIDs []string
-
-	for i := 0; i < 5000; i++ {
-		//t.Logf("i=%v", i)
-		b := seededRandomData(i, i%113)
-		blkID, err := bm.WriteBlock(ctx, b, "")
-		if err != nil {
-			t.Errorf("err: %v", err)
-		}
-
-		blockIDs = append(blockIDs, blkID)
-
-		if i%17 == 0 {
-			//t.Logf("flushing %v", i)
-			if err := bm.Flush(ctx); err != nil {
-				t.Fatalf("error flushing: %v", err)
-			}
-			//dumpBlockManagerData(t, data)
-		}
-
-		if i%41 == 0 {
-			//t.Logf("opening new manager: %v", i)
-			if err := bm.Flush(ctx); err != nil {
-				t.Fatalf("error flushing: %v", err)
-			}
-			//t.Logf("data block count: %v", len(data))
-			//dumpBlockManagerData(t, data)
-			bm = newTestBlockManager(data, keyTime, timeFunc)
-		}
-
-		pos := rand.Intn(len(blockIDs))
-		if _, err := bm.GetBlock(ctx, blockIDs[pos]); err != nil {
-			dumpBlockManagerData(t, data)
-			t.Fatalf("can't read block %q: %v", blockIDs[pos], err)
-			continue
-		}
-	}
-}
-
-// This is regression test for a bug where we would corrupt data when encryption
-// was done in place and clobbered pending data in memory.
-func TestBlockManagerFailedToWritePack(t *testing.T) {
-	ctx := context.Background()
-	data := blobtesting.DataMap{}
-	keyTime := map[blob.ID]time.Time{}
-	st := blobtesting.NewMapStorage(data, keyTime, nil)
-	faulty := &blobtesting.FaultyStorage{
-		Base: st,
-	}
-	st = faulty
-
-	bm, err := newManagerWithOptions(context.Background(), st, FormattingOptions{
-		Version:     1,
-		Hash:        "HMAC-SHA256-128",
-		Encryption:  "AES-256-CTR",
-		MaxPackSize: maxPackSize,
-		HMACSecret:  []byte("foo"),
-		MasterKey:   []byte("0123456789abcdef0123456789abcdef"),
-	}, CachingOptions{}, fakeTimeNowFrozen(fakeTime), nil)
-	if err != nil {
-		t.Fatalf("can't create bm: %v", err)
-	}
-	logging.SetLevel(logging.DEBUG, "faulty-storage")
-
-	faulty.Faults = map[string][]*blobtesting.Fault{
-		"PutBlock": {
-			{Err: errors.New("booboo")},
-		},
-	}
-
-	b1, err := bm.WriteBlock(ctx, seededRandomData(1, 10), "")
-	if err != nil {
-		t.Fatalf("can't create block: %v", err)
-	}
-
-	if err := bm.Flush(ctx); err != nil {
-		t.Logf("expected flush error: %v", err)
-	}
-
-	verifyBlock(ctx, t, bm, b1, seededRandomData(1, 10))
-}
-
-func TestBlockManagerConcurrency(t *testing.T) {
-	ctx := context.Background()
-	data := blobtesting.DataMap{}
-	keyTime := map[blob.ID]time.Time{}
-	bm := newTestBlockManager(data, keyTime, nil)
-	preexistingBlock := writeBlockAndVerify(ctx, t, bm, seededRandomData(10, 100))
-	bm.Flush(ctx)
-
-	dumpBlockManagerData(t, data)
-	bm1 := newTestBlockManager(data, keyTime, nil)
-	bm2 := newTestBlockManager(data, keyTime, nil)
-	bm3 := newTestBlockManager(data, keyTime, fakeTimeNowWithAutoAdvance(fakeTime.Add(1), 1*time.Second))
-
-	// all bm* can see pre-existing block
-	verifyBlock(ctx, t, bm1, preexistingBlock, seededRandomData(10, 100))
-	verifyBlock(ctx, t, bm2, preexistingBlock, seededRandomData(10, 100))
-	verifyBlock(ctx, t, bm3, preexistingBlock, seededRandomData(10, 100))
-
-	// write the same block in all managers.
-	sharedBlock := writeBlockAndVerify(ctx, t, bm1, seededRandomData(20, 100))
-	writeBlockAndVerify(ctx, t, bm2, seededRandomData(20, 100))
-	writeBlockAndVerify(ctx, t, bm3, seededRandomData(20, 100))
-
-	// write unique block per manager.
-	bm1block := writeBlockAndVerify(ctx, t, bm1, seededRandomData(31, 100))
-	bm2block := writeBlockAndVerify(ctx, t, bm2, seededRandomData(32, 100))
-	bm3block := writeBlockAndVerify(ctx, t, bm3, seededRandomData(33, 100))
-
-	// make sure they can't see each other's unflushed blocks.
-	verifyBlockNotFound(ctx, t, bm1, bm2block)
-	verifyBlockNotFound(ctx, t, bm1, bm3block)
-	verifyBlockNotFound(ctx, t, bm2, bm1block)
-	verifyBlockNotFound(ctx, t, bm2, bm3block)
-	verifyBlockNotFound(ctx, t, bm3, bm1block)
-	verifyBlockNotFound(ctx, t, bm3, bm2block)
-
-	// now flush all writers, they still can't see each others' data.
-	bm1.Flush(ctx)
-	bm2.Flush(ctx)
-	bm3.Flush(ctx)
-	verifyBlockNotFound(ctx, t, bm1, bm2block)
-	verifyBlockNotFound(ctx, t, bm1, bm3block)
-	verifyBlockNotFound(ctx, t, bm2, bm1block)
-	verifyBlockNotFound(ctx, t, bm2, bm3block)
-	verifyBlockNotFound(ctx, t, bm3, bm1block)
-	verifyBlockNotFound(ctx, t, bm3, bm2block)
-
-	// new block manager at this point can see all data.
-	bm4 := newTestBlockManager(data, keyTime, nil)
-	verifyBlock(ctx, t, bm4, preexistingBlock, seededRandomData(10, 100))
-	verifyBlock(ctx, t, bm4, sharedBlock, seededRandomData(20, 100))
-	verifyBlock(ctx, t, bm4, bm1block, seededRandomData(31, 100))
-	verifyBlock(ctx, t, bm4, bm2block, seededRandomData(32, 100))
-	verifyBlock(ctx, t, bm4, bm3block, seededRandomData(33, 100))
-
-	if got, want := getIndexCount(data), 4; got != want {
-		t.Errorf("unexpected index count before compaction: %v, wanted %v", got, want)
-	}
-
-	if err := bm4.CompactIndexes(ctx, CompactOptions{
-		MinSmallBlocks: 1,
-		MaxSmallBlocks: 1,
-	}); err != nil {
-		t.Errorf("compaction error: %v", err)
-	}
-	if got, want := getIndexCount(data), 1; got != want {
-		t.Errorf("unexpected index count after compaction: %v, wanted %v", got, want)
-	}
-
-	// new block manager at this point can see all data.
-	bm5 := newTestBlockManager(data, keyTime, nil)
-	verifyBlock(ctx, t, bm5, preexistingBlock, seededRandomData(10, 100))
-	verifyBlock(ctx, t, bm5, sharedBlock, seededRandomData(20, 100))
-	verifyBlock(ctx, t, bm5, bm1block, seededRandomData(31, 100))
-	verifyBlock(ctx, t, bm5, bm2block, seededRandomData(32, 100))
-	verifyBlock(ctx, t, bm5, bm3block, seededRandomData(33, 100))
-	if err := bm5.CompactIndexes(ctx, CompactOptions{
-		MinSmallBlocks: 1,
-		MaxSmallBlocks: 1,
-	}); err != nil {
-		t.Errorf("compaction error: %v", err)
-	}
-}
-
-func TestDeleteBlock(t *testing.T) {
-	ctx := context.Background()
-	data := blobtesting.DataMap{}
-	keyTime := map[blob.ID]time.Time{}
-	bm := newTestBlockManager(data, keyTime, nil)
-	block1 := writeBlockAndVerify(ctx, t, bm, seededRandomData(10, 100))
-	bm.Flush(ctx)
-	block2 := writeBlockAndVerify(ctx, t, bm, seededRandomData(11, 100))
-	if err := bm.DeleteBlock(block1); err != nil {
-		t.Errorf("unable to delete block: %v", block1)
-	}
-	if err := bm.DeleteBlock(block2); err != nil {
-		t.Errorf("unable to delete block: %v", block1)
-	}
-	verifyBlockNotFound(ctx, t, bm, block1)
-	verifyBlockNotFound(ctx, t, bm, block2)
-	bm.Flush(ctx)
-	log.Debugf("-----------")
-	bm = newTestBlockManager(data, keyTime, nil)
-	//dumpBlockManagerData(t, data)
-	verifyBlockNotFound(ctx, t, bm, block1)
-	verifyBlockNotFound(ctx, t, bm, block2)
-}
-
-func TestRewriteNonDeleted(t *testing.T) {
-	const stepBehaviors = 3
-
-	// perform a sequence WriteBlock() <action1> RewriteBlock() <action2> GetBlock()
-	// where actionX can be (0=flush and reopen, 1=flush, 2=nothing)
-	for action1 := 0; action1 < stepBehaviors; action1++ {
-		for action2 := 0; action2 < stepBehaviors; action2++ {
-			t.Run(fmt.Sprintf("case-%v-%v", action1, action2), func(t *testing.T) {
-				ctx := context.Background()
-				data := blobtesting.DataMap{}
-				keyTime := map[blob.ID]time.Time{}
-				fakeNow := fakeTimeNowWithAutoAdvance(fakeTime, 1*time.Second)
-				bm := newTestBlockManager(data, keyTime, fakeNow)
-
-				applyStep := func(action int) {
-					switch action {
-					case 0:
-						t.Logf("flushing and reopening")
-						bm.Flush(ctx)
-						bm = newTestBlockManager(data, keyTime, fakeNow)
-					case 1:
-						t.Logf("flushing")
-						bm.Flush(ctx)
-					case 2:
-						t.Logf("doing nothing")
-					}
-				}
-
-				block1 := writeBlockAndVerify(ctx, t, bm, seededRandomData(10, 100))
-				applyStep(action1)
-				assertNoError(t, bm.RewriteBlock(ctx, block1))
-				applyStep(action2)
-				verifyBlock(ctx, t, bm, block1, seededRandomData(10, 100))
-				dumpBlockManagerData(t, data)
-			})
-		}
-	}
-}
-
-func TestDisableFlush(t *testing.T) {
-	ctx := context.Background()
-	data := blobtesting.DataMap{}
-	keyTime := map[blob.ID]time.Time{}
-	bm := newTestBlockManager(data, keyTime, nil)
-	bm.DisableIndexFlush()
-	bm.DisableIndexFlush()
-	for i := 0; i < 500; i++ {
-		writeBlockAndVerify(ctx, t, bm, seededRandomData(i, 100))
-	}
-	bm.Flush(ctx) // flush will not have effect
-	bm.EnableIndexFlush()
-	bm.Flush(ctx) // flush will not have effect
-	bm.EnableIndexFlush()
-
-	verifyActiveIndexBlobCount(ctx, t, bm, 0)
-	bm.EnableIndexFlush()
-	verifyActiveIndexBlobCount(ctx, t, bm, 0)
-	bm.Flush(ctx) // flush will happen now
-	verifyActiveIndexBlobCount(ctx, t, bm, 1)
-}
-
-func TestRewriteDeleted(t *testing.T) {
-	const stepBehaviors = 3
-
-	// perform a sequence WriteBlock() <action1> Delete() <action2> RewriteBlock() <action3> GetBlock()
-	// where actionX can be (0=flush and reopen, 1=flush, 2=nothing)
-	for action1 := 0; action1 < stepBehaviors; action1++ {
-		for action2 := 0; action2 < stepBehaviors; action2++ {
-			for action3 := 0; action3 < stepBehaviors; action3++ {
-				t.Run(fmt.Sprintf("case-%v-%v-%v", action1, action2, action3), func(t *testing.T) {
-					ctx := context.Background()
-					data := blobtesting.DataMap{}
-					keyTime := map[blob.ID]time.Time{}
-					fakeNow := fakeTimeNowWithAutoAdvance(fakeTime, 1*time.Second)
-					bm := newTestBlockManager(data, keyTime, fakeNow)
-
-					applyStep := func(action int) {
-						switch action {
-						case 0:
-							t.Logf("flushing and reopening")
-							bm.Flush(ctx)
-							bm = newTestBlockManager(data, keyTime, fakeNow)
-						case 1:
-							t.Logf("flushing")
-							bm.Flush(ctx)
-						case 2:
-							t.Logf("doing nothing")
-						}
-					}
-
-					block1 := writeBlockAndVerify(ctx, t, bm, seededRandomData(10, 100))
-					applyStep(action1)
-					assertNoError(t, bm.DeleteBlock(block1))
-					applyStep(action2)
-					if got, want := bm.RewriteBlock(ctx, block1), ErrBlockNotFound; got != want && got != nil {
-						t.Errorf("unexpected error %v, wanted %v", got, want)
-					}
-					applyStep(action3)
-					verifyBlockNotFound(ctx, t, bm, block1)
-					dumpBlockManagerData(t, data)
-				})
-			}
-		}
-	}
-}
-
-func TestDeleteAndRecreate(t *testing.T) {
-	ctx := context.Background()
-	// simulate race between delete/recreate and delete
-	// delete happens at t0+10, recreate at t0+20 and second delete time is parameterized.
-	// depending on it, the second delete results will be visible.
-	cases := []struct {
-		desc         string
-		deletionTime time.Time
-		isVisible    bool
-	}{
-		{"deleted before delete and-recreate", fakeTime.Add(5 * time.Second), true},
-		//{"deleted after delete and recreate", fakeTime.Add(25 * time.Second), false},
-	}
-
-	for _, tc := range cases {
-		t.Run(tc.desc, func(t *testing.T) {
-			// write a block
-			data := blobtesting.DataMap{}
-			keyTime := map[blob.ID]time.Time{}
-			bm := newTestBlockManager(data, keyTime, fakeTimeNowFrozen(fakeTime))
-			block1 := writeBlockAndVerify(ctx, t, bm, seededRandomData(10, 100))
-			bm.Flush(ctx)
-
-			// delete but at given timestamp but don't commit yet.
-			bm0 := newTestBlockManager(data, keyTime, fakeTimeNowWithAutoAdvance(tc.deletionTime, 1*time.Second))
-			assertNoError(t, bm0.DeleteBlock(block1))
-
-			// delete it at t0+10
-			bm1 := newTestBlockManager(data, keyTime, fakeTimeNowWithAutoAdvance(fakeTime.Add(10*time.Second), 1*time.Second))
-			verifyBlock(ctx, t, bm1, block1, seededRandomData(10, 100))
-			assertNoError(t, bm1.DeleteBlock(block1))
-			bm1.Flush(ctx)
-
-			// recreate at t0+20
-			bm2 := newTestBlockManager(data, keyTime, fakeTimeNowWithAutoAdvance(fakeTime.Add(20*time.Second), 1*time.Second))
-			block2 := writeBlockAndVerify(ctx, t, bm2, seededRandomData(10, 100))
-			bm2.Flush(ctx)
-
-			// commit deletion from bm0 (t0+5)
-			bm0.Flush(ctx)
-
-			//dumpBlockManagerData(t, data)
-
-			if block1 != block2 {
-				t.Errorf("got invalid block %v, expected %v", block2, block1)
-			}
-
-			bm3 := newTestBlockManager(data, keyTime, nil)
-			dumpBlockManagerData(t, data)
-			if tc.isVisible {
-				verifyBlock(ctx, t, bm3, block1, seededRandomData(10, 100))
-			} else {
-				verifyBlockNotFound(ctx, t, bm3, block1)
-			}
-		})
-	}
-}
-
-func TestFindUnreferencedBlobs(t *testing.T) {
-	ctx := context.Background()
-	data := blobtesting.DataMap{}
-	keyTime := map[blob.ID]time.Time{}
-	bm := newTestBlockManager(data, keyTime, nil)
-	verifyUnreferencedStorageFilesCount(ctx, t, bm, 0)
-	blockID := writeBlockAndVerify(ctx, t, bm, seededRandomData(10, 100))
-	if err := bm.Flush(ctx); err != nil {
-		t.Errorf("flush error: %v", err)
-	}
-	verifyUnreferencedStorageFilesCount(ctx, t, bm, 0)
-	if err := bm.DeleteBlock(blockID); err != nil {
-		t.Errorf("error deleting block: %v", blockID)
-	}
-	if err := bm.Flush(ctx); err != nil {
-		t.Errorf("flush error: %v", err)
-	}
-
-	// block still present in first pack
-	verifyUnreferencedStorageFilesCount(ctx, t, bm, 0)
-
-	assertNoError(t, bm.RewriteBlock(ctx, blockID))
-	if err := bm.Flush(ctx); err != nil {
-		t.Errorf("flush error: %v", err)
-	}
-	verifyUnreferencedStorageFilesCount(ctx, t, bm, 1)
-	assertNoError(t, bm.RewriteBlock(ctx, blockID))
-	if err := bm.Flush(ctx); err != nil {
-		t.Errorf("flush error: %v", err)
-	}
-	verifyUnreferencedStorageFilesCount(ctx, t, bm, 2)
-}
-
-func TestFindUnreferencedBlobs2(t *testing.T) {
-	ctx := context.Background()
-	data := blobtesting.DataMap{}
-	keyTime := map[blob.ID]time.Time{}
-	bm := newTestBlockManager(data, keyTime, nil)
-	verifyUnreferencedStorageFilesCount(ctx, t, bm, 0)
-	blockID := writeBlockAndVerify(ctx, t, bm, seededRandomData(10, 100))
-	writeBlockAndVerify(ctx, t, bm, seededRandomData(11, 100))
-	dumpBlocks(t, bm, "after writing")
-	if err := bm.Flush(ctx); err != nil {
-		t.Errorf("flush error: %v", err)
-	}
-	dumpBlocks(t, bm, "after flush")
-	verifyUnreferencedStorageFilesCount(ctx, t, bm, 0)
-	if err := bm.DeleteBlock(blockID); err != nil {
-		t.Errorf("error deleting block: %v", blockID)
-	}
-	dumpBlocks(t, bm, "after delete")
-	if err := bm.Flush(ctx); err != nil {
-		t.Errorf("flush error: %v", err)
-	}
-	dumpBlocks(t, bm, "after flush")
-	// block present in first pack, original pack is still referenced
-	verifyUnreferencedStorageFilesCount(ctx, t, bm, 0)
-}
-
-func dumpBlocks(t *testing.T, bm *Manager, caption string) {
-	t.Helper()
-	infos, err := bm.ListBlockInfos("", true)
-	if err != nil {
-		t.Errorf("error listing blocks: %v", err)
-		return
-	}
-
-	log.Infof("**** dumping %v blocks %v", len(infos), caption)
-	for i, bi := range infos {
-		log.Debugf(" bi[%v]=%#v", i, bi)
-	}
-	log.Infof("finished dumping %v blocks", len(infos))
-}
-
-func verifyUnreferencedStorageFilesCount(ctx context.Context, t *testing.T, bm *Manager, want int) {
-	t.Helper()
-	unref, err := bm.FindUnreferencedBlobs(ctx)
-	if err != nil {
-		t.Errorf("error in FindUnreferencedBlobs: %v", err)
-	}
-
-	log.Infof("got %v expecting %v", unref, want)
-	if got := len(unref); got != want {
-		t.Errorf("invalid number of unreferenced blocks: %v, wanted %v", got, want)
-	}
-}
-
-func TestBlockWriteAliasing(t *testing.T) {
-	ctx := context.Background()
-	data := blobtesting.DataMap{}
-	keyTime := map[blob.ID]time.Time{}
-	bm := newTestBlockManager(data, keyTime, fakeTimeNowFrozen(fakeTime))
-
-	blockData := []byte{100, 0, 0}
-	id1 := writeBlockAndVerify(ctx, t, bm, blockData)
-	blockData[0] = 101
-	id2 := writeBlockAndVerify(ctx, t, bm, blockData)
-	bm.Flush(ctx)
-	blockData[0] = 102
-	id3 := writeBlockAndVerify(ctx, t, bm, blockData)
-	blockData[0] = 103
-	id4 := writeBlockAndVerify(ctx, t, bm, blockData)
-	verifyBlock(ctx, t, bm, id1, []byte{100, 0, 0})
-	verifyBlock(ctx, t, bm, id2, []byte{101, 0, 0})
-	verifyBlock(ctx, t, bm, id3, []byte{102, 0, 0})
-	verifyBlock(ctx, t, bm, id4, []byte{103, 0, 0})
-}
-
-func TestBlockReadAliasing(t *testing.T) {
-	ctx := context.Background()
-	data := blobtesting.DataMap{}
-	keyTime := map[blob.ID]time.Time{}
-	bm := newTestBlockManager(data, keyTime, fakeTimeNowFrozen(fakeTime))
-
-	blockData := []byte{100, 0, 0}
-	id1 := writeBlockAndVerify(ctx, t, bm, blockData)
-	blockData2, err := bm.GetBlock(ctx, id1)
-	if err != nil {
-		t.Fatalf("can't get block data: %v", err)
-	}
-
-	blockData2[0]++
-	verifyBlock(ctx, t, bm, id1, blockData)
-	bm.Flush(ctx)
-	verifyBlock(ctx, t, bm, id1, blockData)
-}
-
-func TestVersionCompatibility(t *testing.T) {
-	for writeVer := minSupportedReadVersion; writeVer <= currentWriteVersion; writeVer++ {
-		t.Run(fmt.Sprintf("version-%v", writeVer), func(t *testing.T) {
-			verifyVersionCompat(t, writeVer)
-		})
-	}
-}
-
-func verifyVersionCompat(t *testing.T, writeVersion int) {
-	ctx := context.Background()
-
-	// create block manager that writes 'writeVersion' and reads all versions >= minSupportedReadVersion
-	data := blobtesting.DataMap{}
-	keyTime := map[blob.ID]time.Time{}
-	mgr := newTestBlockManager(data, keyTime, nil)
-	mgr.writeFormatVersion = int32(writeVersion)
-
-	dataSet := map[string][]byte{}
-
-	for i := 0; i < 3000000; i = (i + 1) * 2 {
-		data := make([]byte, i)
-		rand.Read(data)
-
-		cid, err := mgr.WriteBlock(ctx, data, "")
-		if err != nil {
-			t.Fatalf("unable to write %v bytes: %v", len(data), err)
-		}
-		dataSet[cid] = data
-	}
-	verifyBlockManagerDataSet(ctx, t, mgr, dataSet)
-
-	// delete random 3 items (map iteration order is random)
-	cnt := 0
-	for blobID := range dataSet {
-		t.Logf("deleting %v", blobID)
-		assertNoError(t, mgr.DeleteBlock(blobID))
-		delete(dataSet, blobID)
-		cnt++
-		if cnt >= 3 {
-			break
-		}
-	}
-	if err := mgr.Flush(ctx); err != nil {
-		t.Fatalf("failed to flush: %v", err)
-	}
-
-	// create new manager that reads and writes using new version.
-	mgr = newTestBlockManager(data, keyTime, nil)
-
-	// make sure we can read everything
-	verifyBlockManagerDataSet(ctx, t, mgr, dataSet)
-
-	if err := mgr.CompactIndexes(ctx, CompactOptions{
-		MinSmallBlocks: 1,
-		MaxSmallBlocks: 1,
-	}); err != nil {
-		t.Fatalf("unable to compact indexes: %v", err)
-	}
-	if err := mgr.Flush(ctx); err != nil {
-		t.Fatalf("failed to flush: %v", err)
-	}
-	verifyBlockManagerDataSet(ctx, t, mgr, dataSet)
-
-	// now open one more manager
-	mgr = newTestBlockManager(data, keyTime, nil)
-	verifyBlockManagerDataSet(ctx, t, mgr, dataSet)
-}
-
-func verifyBlockManagerDataSet(ctx context.Context, t *testing.T, mgr *Manager, dataSet map[string][]byte) {
-	for blockID, originalPayload := range dataSet {
-		v, err := mgr.GetBlock(ctx, blockID)
-		if err != nil {
-			t.Errorf("unable to read block %q: %v", blockID, err)
-			continue
-		}
-
-		if !reflect.DeepEqual(v, originalPayload) {
-			t.Errorf("payload for %q does not match original: %v", v, originalPayload)
-		}
-	}
-}
-
-func newTestBlockManager(data blobtesting.DataMap, keyTime map[blob.ID]time.Time, timeFunc func() time.Time) *Manager {
-	//st = logging.NewWrapper(st)
-	if timeFunc == nil {
-		timeFunc = fakeTimeNowWithAutoAdvance(fakeTime, 1*time.Second)
-	}
-	st := blobtesting.NewMapStorage(data, keyTime, timeFunc)
-	bm, err := newManagerWithOptions(context.Background(), st, FormattingOptions{
-		Hash:        "HMAC-SHA256",
-		Encryption:  "NONE",
-		HMACSecret:  hmacSecret,
-		MaxPackSize: maxPackSize,
-		Version:     1,
-	}, CachingOptions{}, timeFunc, nil)
-	if err != nil {
-		panic("can't create block manager: " + err.Error())
-	}
-	bm.checkInvariantsOnUnlock = true
-	return bm
-}
-
-func getIndexCount(d blobtesting.DataMap) int {
-	var cnt int
-
-	for blobID := range d {
-		if strings.HasPrefix(string(blobID), newIndexBlobPrefix) {
-			cnt++
-		}
-	}
-
-	return cnt
-}
-
-func fakeTimeNowFrozen(t time.Time) func() time.Time {
-	return fakeTimeNowWithAutoAdvance(t, 0)
-}
-
-func fakeTimeNowWithAutoAdvance(t time.Time, dt time.Duration) func() time.Time {
-	var mu sync.Mutex
-	return func() time.Time {
-		mu.Lock()
-		defer mu.Unlock()
-		ret := t
-		t = t.Add(dt)
-		return ret
-	}
-}
-
-func verifyBlockNotFound(ctx context.Context, t *testing.T, bm *Manager, blockID string) {
-	t.Helper()
-
-	b, err := bm.GetBlock(ctx, blockID)
-	if err != ErrBlockNotFound {
-		t.Errorf("unexpected response from GetBlock(%q), got %v,%v, expected %v", blockID, b, err, ErrBlockNotFound)
-	}
-}
-
-func verifyBlock(ctx context.Context, t *testing.T, bm *Manager, blockID string, b []byte) {
-	t.Helper()
-
-	b2, err := bm.GetBlock(ctx, blockID)
-	if err != nil {
-		t.Errorf("unable to read block %q: %v", blockID, err)
-		return
-	}
-
-	if got, want := b2, b; !reflect.DeepEqual(got, want) {
-		t.Errorf("block %q data mismatch: got %x (nil:%v), wanted %x (nil:%v)", blockID, got, got == nil, want, want == nil)
-	}
-
-	bi, err := bm.BlockInfo(ctx, blockID)
-	if err != nil {
-		t.Errorf("error getting block info %q: %v", blockID, err)
-	}
-
-	if got, want := bi.Length, uint32(len(b)); got != want {
-		t.Errorf("invalid block size for %q: %v, wanted %v", blockID, got, want)
-	}
-
-}
-func writeBlockAndVerify(ctx context.Context, t *testing.T, bm *Manager, b []byte) string {
-	t.Helper()
-
-	blockID, err := bm.WriteBlock(ctx, b, "")
-	if err != nil {
-		t.Errorf("err: %v", err)
-	}
-
-	if got, want := blockID, string(hashValue(b)); got != want {
-		t.Errorf("invalid block ID for %x, got %v, want %v", b, got, want)
-	}
-
-	verifyBlock(ctx, t, bm, blockID, b)
-
-	return blockID
-}
-
-func seededRandomData(seed int, length int) []byte {
-	b := make([]byte, length)
-	rnd := rand.New(rand.NewSource(int64(seed)))
-	rnd.Read(b)
-	return b
-}
-
-func hashValue(b []byte) string {
-	h := hmac.New(sha256.New, hmacSecret)
-	h.Write(b) //nolint:errcheck
-	return hex.EncodeToString(h.Sum(nil))
-}
-
-func dumpBlockManagerData(t *testing.T, data blobtesting.DataMap) {
-	t.Helper()
-	for k, v := range data {
-		if k[0] == 'n' {
-			ndx, err := openPackIndex(bytes.NewReader(v))
-			if err == nil {
-				t.Logf("index %v (%v bytes)", k, len(v))
-				assertNoError(t, ndx.Iterate("", func(i Info) error {
-					t.Logf("  %+v\n", i)
-					return nil
-				}))
-
-			}
-		} else {
-			t.Logf("data %v (%v bytes)\n", k, len(v))
-		}
-	}
-}

repo/block/committed_block_index_mem_cache.go

@@ -1,51 +0,0 @@
-package block
-
-import (
-	"bytes"
-	"sync"
-
-	"github.com/pkg/errors"
-
-	"github.com/kopia/kopia/repo/blob"
-)
-
-type memoryCommittedBlockIndexCache struct {
-	mu     sync.Mutex
-	blocks map[blob.ID]packIndex
-}
-
-func (m *memoryCommittedBlockIndexCache) hasIndexBlobID(indexBlobID blob.ID) (bool, error) {
-	m.mu.Lock()
-	defer m.mu.Unlock()
-
-	return m.blocks[indexBlobID] != nil, nil
-}
-
-func (m *memoryCommittedBlockIndexCache) addBlockToCache(indexBlobID blob.ID, data []byte) error {
-	m.mu.Lock()
-	defer m.mu.Unlock()
-
-	ndx, err := openPackIndex(bytes.NewReader(data))
-	if err != nil {
-		return err
-	}
-
-	m.blocks[indexBlobID] = ndx
-	return nil
-}
-
-func (m *memoryCommittedBlockIndexCache) openIndex(indexBlobID blob.ID) (packIndex, error) {
-	m.mu.Lock()
-	defer m.mu.Unlock()
-
-	v := m.blocks[indexBlobID]
-	if v == nil {
-		return nil, errors.Errorf("block not found in cache: %v", indexBlobID)
-	}
-
-	return v, nil
-}
-
-func (m *memoryCommittedBlockIndexCache) expireUnused(used []blob.ID) error {
-	return nil
-}

repo/block/context.go

@@ -1,34 +0,0 @@
-package block
-
-import "context"
-
-type contextKey string
-
-var useBlockCacheContextKey contextKey = "use-block-cache"
-var useListCacheContextKey contextKey = "use-list-cache"
-
-// UsingBlockCache returns a derived context that causes block manager to use cache.
-func UsingBlockCache(ctx context.Context, enabled bool) context.Context {
-	return context.WithValue(ctx, useBlockCacheContextKey, enabled)
-}
-
-// UsingListCache returns a derived context that causes block manager to use cache.
-func UsingListCache(ctx context.Context, enabled bool) context.Context {
-	return context.WithValue(ctx, useListCacheContextKey, enabled)
-}
-
-func shouldUseBlockCache(ctx context.Context) bool {
-	if enabled, ok := ctx.Value(useBlockCacheContextKey).(bool); ok {
-		return enabled
-	}
-
-	return true
-}
-
-func shouldUseListCache(ctx context.Context) bool {
-	if enabled, ok := ctx.Value(useListCacheContextKey).(bool); ok {
-		return enabled
-	}
-
-	return true
-}

repo/connect.go

@@ -12,22 +12,22 @@
 	"github.com/pkg/errors"
 
 	"github.com/kopia/kopia/repo/blob"
-	"github.com/kopia/kopia/repo/block"
+	"github.com/kopia/kopia/repo/content"
 )
 
 // ConnectOptions specifies options when persisting configuration to connect to a repository.
 type ConnectOptions struct {
-	block.CachingOptions
+	content.CachingOptions
 }
 
 // Connect connects to the repository in the specified storage and persists the configuration and credentials in the file provided.
 func Connect(ctx context.Context, configFile string, st blob.Storage, password string, opt ConnectOptions) error {
 	formatBytes, err := st.GetBlob(ctx, FormatBlobID, 0, -1)
 	if err != nil {
-		return errors.Wrap(err, "unable to read format block")
+		return errors.Wrap(err, "unable to read format blob")
 	}
 
-	f, err := parseFormatBlock(formatBytes)
+	f, err := parseFormatBlob(formatBytes)
 	if err != nil {
 		return err
 	}
@@ -61,9 +61,9 @@ func Connect(ctx context.Context, configFile string, st blob.Storage, password s
 	return r.Close(ctx)
 }
 
-func setupCaching(configPath string, lc *LocalConfig, opt block.CachingOptions, uniqueID []byte) error {
+func setupCaching(configPath string, lc *LocalConfig, opt content.CachingOptions, uniqueID []byte) error {
 	if opt.MaxCacheSizeBytes == 0 {
-		lc.Caching = block.CachingOptions{}
+		lc.Caching = content.CachingOptions{}
 		return nil
 	}
 

repo/content/block_manager_compaction.go

@@ -1,4 +1,4 @@
-package block
+package content
 
 import (
 	"bytes"
@@ -9,23 +9,23 @@
 )
 
 var autoCompactionOptions = CompactOptions{
-	MinSmallBlocks: 4 * parallelFetches,
-	MaxSmallBlocks: 64,
+	MinSmallBlobs: 4 * parallelFetches,
+	MaxSmallBlobs: 64,
 }
 
 // CompactOptions provides options for compaction
 type CompactOptions struct {
-	MinSmallBlocks       int
-	MaxSmallBlocks       int
-	AllBlocks            bool
+	MinSmallBlobs        int
+	MaxSmallBlobs        int
+	AllIndexes           bool
 	SkipDeletedOlderThan time.Duration
 }
 
-// CompactIndexes performs compaction of index blobs ensuring that # of small blocks is between minSmallBlockCount and maxSmallBlockCount
+// CompactIndexes performs compaction of index blobs ensuring that # of small contents is between minSmallContentCount and maxSmallContentCount
 func (bm *Manager) CompactIndexes(ctx context.Context, opt CompactOptions) error {
 	log.Debugf("CompactIndexes(%+v)", opt)
-	if opt.MaxSmallBlocks < opt.MinSmallBlocks {
-		return errors.Errorf("invalid block counts")
+	if opt.MaxSmallBlobs < opt.MinSmallBlobs {
+		return errors.Errorf("invalid content counts")
 	}
 
 	indexBlobs, _, err := bm.loadPackIndexesUnlocked(ctx)
@@ -33,61 +33,61 @@ func (bm *Manager) CompactIndexes(ctx context.Context, opt CompactOptions) error
 		return errors.Wrap(err, "error loading indexes")
 	}
 
-	blocksToCompact := bm.getBlocksToCompact(indexBlobs, opt)
+	contentsToCompact := bm.getContentsToCompact(indexBlobs, opt)
 
-	if err := bm.compactAndDeleteIndexBlobs(ctx, blocksToCompact, opt); err != nil {
+	if err := bm.compactAndDeleteIndexBlobs(ctx, contentsToCompact, opt); err != nil {
 		log.Warningf("error performing quick compaction: %v", err)
 	}
 
 	return nil
 }
 
-func (bm *Manager) getBlocksToCompact(indexBlobs []IndexBlobInfo, opt CompactOptions) []IndexBlobInfo {
-	var nonCompactedBlocks []IndexBlobInfo
-	var totalSizeNonCompactedBlocks int64
+func (bm *Manager) getContentsToCompact(indexBlobs []IndexBlobInfo, opt CompactOptions) []IndexBlobInfo {
+	var nonCompactedContents []IndexBlobInfo
+	var totalSizeNonCompactedContents int64
 
-	var verySmallBlocks []IndexBlobInfo
-	var totalSizeVerySmallBlocks int64
+	var verySmallContents []IndexBlobInfo
+	var totalSizeVerySmallContents int64
 
-	var mediumSizedBlocks []IndexBlobInfo
-	var totalSizeMediumSizedBlocks int64
+	var mediumSizedContents []IndexBlobInfo
+	var totalSizeMediumSizedContents int64
 
 	for _, b := range indexBlobs {
-		if b.Length > int64(bm.maxPackSize) && !opt.AllBlocks {
+		if b.Length > int64(bm.maxPackSize) && !opt.AllIndexes {
 			continue
 		}
 
-		nonCompactedBlocks = append(nonCompactedBlocks, b)
+		nonCompactedContents = append(nonCompactedContents, b)
 		if b.Length < int64(bm.maxPackSize/20) {
-			verySmallBlocks = append(verySmallBlocks, b)
-			totalSizeVerySmallBlocks += b.Length
+			verySmallContents = append(verySmallContents, b)
+			totalSizeVerySmallContents += b.Length
 		} else {
-			mediumSizedBlocks = append(mediumSizedBlocks, b)
-			totalSizeMediumSizedBlocks += b.Length
+			mediumSizedContents = append(mediumSizedContents, b)
+			totalSizeMediumSizedContents += b.Length
 		}
-		totalSizeNonCompactedBlocks += b.Length
+		totalSizeNonCompactedContents += b.Length
 	}
 
-	if len(nonCompactedBlocks) < opt.MinSmallBlocks {
+	if len(nonCompactedContents) < opt.MinSmallBlobs {
 		// current count is below min allowed - nothing to do
-		formatLog.Debugf("no small blocks to compact")
+		formatLog.Debugf("no small contents to compact")
 		return nil
 	}
 
-	if len(verySmallBlocks) > len(nonCompactedBlocks)/2 && len(mediumSizedBlocks)+1 < opt.MinSmallBlocks {
-		formatLog.Debugf("compacting %v very small blocks", len(verySmallBlocks))
-		return verySmallBlocks
+	if len(verySmallContents) > len(nonCompactedContents)/2 && len(mediumSizedContents)+1 < opt.MinSmallBlobs {
+		formatLog.Debugf("compacting %v very small contents", len(verySmallContents))
+		return verySmallContents
 	}
 
-	formatLog.Debugf("compacting all %v non-compacted blocks", len(nonCompactedBlocks))
-	return nonCompactedBlocks
+	formatLog.Debugf("compacting all %v non-compacted contents", len(nonCompactedContents))
+	return nonCompactedContents
 }
 
 func (bm *Manager) compactAndDeleteIndexBlobs(ctx context.Context, indexBlobs []IndexBlobInfo, opt CompactOptions) error {
 	if len(indexBlobs) <= 1 {
 		return nil
 	}
-	formatLog.Debugf("compacting %v blocks", len(indexBlobs))
+	formatLog.Debugf("compacting %v contents", len(indexBlobs))
 	t0 := time.Now()
 
 	bld := make(packIndexBuilder)
@@ -136,7 +136,7 @@ func (bm *Manager) addIndexBlobsToBuilder(ctx context.Context, bld packIndexBuil
 
 	_ = index.Iterate("", func(i Info) error {
 		if i.Deleted && opt.SkipDeletedOlderThan > 0 && time.Since(i.Timestamp()) > opt.SkipDeletedOlderThan {
-			log.Debugf("skipping block %v deleted at %v", i.BlockID, i.Timestamp())
+			log.Debugf("skipping content %v deleted at %v", i.ID, i.Timestamp())
 			return nil
 		}
 		bld.Add(i)

repo/content/builder.go

@@ -1,4 +1,4 @@
-package block
+package content
 
 import (
 	"bufio"
@@ -11,29 +11,29 @@
 	"github.com/kopia/kopia/repo/blob"
 )
 
-// packIndexBuilder prepares and writes block index for writing.
-type packIndexBuilder map[string]*Info
+// packIndexBuilder prepares and writes content index.
+type packIndexBuilder map[ID]*Info
 
 // Add adds a new entry to the builder or conditionally replaces it if the timestamp is greater.
 func (b packIndexBuilder) Add(i Info) {
-	old, ok := b[i.BlockID]
+	old, ok := b[i.ID]
 	if !ok || i.TimestampSeconds >= old.TimestampSeconds {
-		b[i.BlockID] = &i
+		b[i.ID] = &i
 	}
 }
 
-func (b packIndexBuilder) sortedBlocks() []*Info {
-	var allBlocks []*Info
+func (b packIndexBuilder) sortedContents() []*Info {
+	var allContents []*Info
 
 	for _, v := range b {
-		allBlocks = append(allBlocks, v)
+		allContents = append(allContents, v)
 	}
 
-	sort.Slice(allBlocks, func(i, j int) bool {
-		return allBlocks[i].BlockID < allBlocks[j].BlockID
+	sort.Slice(allContents, func(i, j int) bool {
+		return allContents[i].ID < allContents[j].ID
 	})
 
-	return allBlocks
+	return allContents
 }
 
 type indexLayout struct {
@@ -46,18 +46,18 @@ type indexLayout struct {
 
 // Build writes the pack index to the provided output.
 func (b packIndexBuilder) Build(output io.Writer) error {
-	allBlocks := b.sortedBlocks()
+	allContents := b.sortedContents()
 	layout := &indexLayout{
 		packBlobIDOffsets: map[blob.ID]uint32{},
 		keyLength:         -1,
 		entryLength:       20,
-		entryCount:        len(allBlocks),
+		entryCount:        len(allContents),
 	}
 
 	w := bufio.NewWriter(output)
 
 	// prepare extra data to be appended at the end of an index.
-	extraData := prepareExtraData(allBlocks, layout)
+	extraData := prepareExtraData(allContents, layout)
 
 	// write header
 	header := make([]byte, 8)
@@ -69,9 +69,9 @@ func (b packIndexBuilder) Build(output io.Writer) error {
 		return errors.Wrap(err, "unable to write header")
 	}
 
-	// write all sorted blocks.
+	// write all sorted contents.
 	entry := make([]byte, layout.entryLength)
-	for _, it := range allBlocks {
+	for _, it := range allContents {
 		if err := writeEntry(w, it, layout, entry); err != nil {
 			return errors.Wrap(err, "unable to write entry")
 		}
@@ -84,12 +84,12 @@ func (b packIndexBuilder) Build(output io.Writer) error {
 	return w.Flush()
 }
 
-func prepareExtraData(allBlocks []*Info, layout *indexLayout) []byte {
+func prepareExtraData(allContents []*Info, layout *indexLayout) []byte {
 	var extraData []byte
 
-	for i, it := range allBlocks {
+	for i, it := range allContents {
 		if i == 0 {
-			layout.keyLength = len(contentIDToBytes(it.BlockID))
+			layout.keyLength = len(contentIDToBytes(it.ID))
 		}
 		if it.PackBlobID != "" {
 			if _, ok := layout.packBlobIDOffsets[it.PackBlobID]; !ok {
@@ -106,7 +106,7 @@ func prepareExtraData(allBlocks []*Info, layout *indexLayout) []byte {
 }
 
 func writeEntry(w io.Writer, it *Info, layout *indexLayout, entry []byte) error {
-	k := contentIDToBytes(it.BlockID)
+	k := contentIDToBytes(it.ID)
 	if len(k) != layout.keyLength {
 		return errors.Errorf("inconsistent key length: %v vs %v", len(k), layout.keyLength)
 	}
@@ -133,7 +133,7 @@ func formatEntry(entry []byte, it *Info, layout *indexLayout) error {
 	timestampAndFlags := uint64(it.TimestampSeconds) << 16
 
 	if len(it.PackBlobID) == 0 {
-		return errors.Errorf("empty pack block ID for %v", it.BlockID)
+		return errors.Errorf("empty pack content ID for %v", it.ID)
 	}
 
 	binary.BigEndian.PutUint32(entryPackFileOffset, layout.extraDataOffset+layout.packBlobIDOffsets[it.PackBlobID])

repo/content/cache_hmac.go

@@ -1,4 +1,4 @@
-package block
+package content
 
 import "crypto/hmac"
 import "crypto/sha256"

repo/content/caching_options.go

@@ -1,4 +1,4 @@
-package block
+package content
 
 // CachingOptions specifies configuration of local cache.
 type CachingOptions struct {

repo/content/committed_content_index.go

@@ -1,4 +1,4 @@
-package block
+package content
 
 import (
 	"path/filepath"
@@ -9,37 +9,37 @@
 	"github.com/kopia/kopia/repo/blob"
 )
 
-type committedBlockIndex struct {
-	cache committedBlockIndexCache
+type committedContentIndex struct {
+	cache committedContentIndexCache
 
 	mu     sync.Mutex
 	inUse  map[blob.ID]packIndex
 	merged mergedIndex
 }
 
-type committedBlockIndexCache interface {
+type committedContentIndexCache interface {
 	hasIndexBlobID(indexBlob blob.ID) (bool, error)
-	addBlockToCache(indexBlob blob.ID, data []byte) error
+	addContentToCache(indexBlob blob.ID, data []byte) error
 	openIndex(indexBlob blob.ID) (packIndex, error)
 	expireUnused(used []blob.ID) error
 }
 
-func (b *committedBlockIndex) getBlock(blockID string) (Info, error) {
+func (b *committedContentIndex) getContent(contentID ID) (Info, error) {
 	b.mu.Lock()
 	defer b.mu.Unlock()
 
-	info, err := b.merged.GetInfo(blockID)
+	info, err := b.merged.GetInfo(contentID)
 	if info != nil {
 		return *info, nil
 	}
 	if err == nil {
-		return Info{}, ErrBlockNotFound
+		return Info{}, ErrContentNotFound
 	}
 	return Info{}, err
 }
 
-func (b *committedBlockIndex) addBlock(indexBlobID blob.ID, data []byte, use bool) error {
-	if err := b.cache.addBlockToCache(indexBlobID, data); err != nil {
+func (b *committedContentIndex) addContent(indexBlobID blob.ID, data []byte, use bool) error {
+	if err := b.cache.addContentToCache(indexBlobID, data); err != nil {
 		return err
 	}
 
@@ -63,7 +63,7 @@ func (b *committedBlockIndex) addBlock(indexBlobID blob.ID, data []byte, use boo
 	return nil
 }
 
-func (b *committedBlockIndex) listBlocks(prefix string, cb func(i Info) error) error {
+func (b *committedContentIndex) listContents(prefix ID, cb func(i Info) error) error {
 	b.mu.Lock()
 	m := append(mergedIndex(nil), b.merged...)
 	b.mu.Unlock()
@@ -71,7 +71,7 @@ func (b *committedBlockIndex) listBlocks(prefix string, cb func(i Info) error) e
 	return m.Iterate(prefix, cb)
 }
 
-func (b *committedBlockIndex) packFilesChanged(packFiles []blob.ID) bool {
+func (b *committedContentIndex) packFilesChanged(packFiles []blob.ID) bool {
 	if len(packFiles) != len(b.inUse) {
 		return true
 	}
@@ -85,7 +85,7 @@ func (b *committedBlockIndex) packFilesChanged(packFiles []blob.ID) bool {
 	return false
 }
 
-func (b *committedBlockIndex) use(packFiles []blob.ID) (bool, error) {
+func (b *committedContentIndex) use(packFiles []blob.ID) (bool, error) {
 	b.mu.Lock()
 	defer b.mu.Unlock()
 
@@ -113,26 +113,26 @@ func (b *committedBlockIndex) use(packFiles []blob.ID) (bool, error) {
 	b.inUse = newInUse
 
 	if err := b.cache.expireUnused(packFiles); err != nil {
-		log.Warningf("unable to expire unused block index files: %v", err)
+		log.Warningf("unable to expire unused content index files: %v", err)
 	}
 	newMerged = nil
 
 	return true, nil
 }
 
-func newCommittedBlockIndex(caching CachingOptions) (*committedBlockIndex, error) {
-	var cache committedBlockIndexCache
+func newCommittedContentIndex(caching CachingOptions) (*committedContentIndex, error) {
+	var cache committedContentIndexCache
 
 	if caching.CacheDirectory != "" {
 		dirname := filepath.Join(caching.CacheDirectory, "indexes")
-		cache = &diskCommittedBlockIndexCache{dirname}
+		cache = &diskCommittedContentIndexCache{dirname}
 	} else {
-		cache = &memoryCommittedBlockIndexCache{
-			blocks: map[blob.ID]packIndex{},
+		cache = &memoryCommittedContentIndexCache{
+			contents: map[blob.ID]packIndex{},
 		}
 	}
 
-	return &committedBlockIndex{
+	return &committedContentIndex{
 		cache: cache,
 		inUse: map[blob.ID]packIndex{},
 	}, nil

repo/content/committed_content_index_disk_cache.go

@@ -1,4 +1,4 @@
-package block
+package content
 
 import (
 	"io/ioutil"
@@ -14,19 +14,19 @@
 )
 
 const (
-	simpleIndexSuffix                    = ".sndx"
-	unusedCommittedBlockIndexCleanupTime = 1 * time.Hour // delete unused committed index blobs after 1 hour
+	simpleIndexSuffix                      = ".sndx"
+	unusedCommittedContentIndexCleanupTime = 1 * time.Hour // delete unused committed index blobs after 1 hour
 )
 
-type diskCommittedBlockIndexCache struct {
+type diskCommittedContentIndexCache struct {
 	dirname string
 }
 
-func (c *diskCommittedBlockIndexCache) indexBlobPath(indexBlobID blob.ID) string {
+func (c *diskCommittedContentIndexCache) indexBlobPath(indexBlobID blob.ID) string {
 	return filepath.Join(c.dirname, string(indexBlobID)+simpleIndexSuffix)
 }
 
-func (c *diskCommittedBlockIndexCache) openIndex(indexBlobID blob.ID) (packIndex, error) {
+func (c *diskCommittedContentIndexCache) openIndex(indexBlobID blob.ID) (packIndex, error) {
 	fullpath := c.indexBlobPath(indexBlobID)
 
 	f, err := mmap.Open(fullpath)
@@ -37,7 +37,7 @@ func (c *diskCommittedBlockIndexCache) openIndex(indexBlobID blob.ID) (packIndex
 	return openPackIndex(f)
 }
 
-func (c *diskCommittedBlockIndexCache) hasIndexBlobID(indexBlobID blob.ID) (bool, error) {
+func (c *diskCommittedContentIndexCache) hasIndexBlobID(indexBlobID blob.ID) (bool, error) {
 	_, err := os.Stat(c.indexBlobPath(indexBlobID))
 	if err == nil {
 		return true, nil
@@ -49,7 +49,7 @@ func (c *diskCommittedBlockIndexCache) hasIndexBlobID(indexBlobID blob.ID) (bool
 	return false, err
 }
 
-func (c *diskCommittedBlockIndexCache) addBlockToCache(indexBlobID blob.ID, data []byte) error {
+func (c *diskCommittedContentIndexCache) addContentToCache(indexBlobID blob.ID, data []byte) error {
 	exists, err := c.hasIndexBlobID(indexBlobID)
 	if err != nil {
 		return err
@@ -66,13 +66,13 @@ func (c *diskCommittedBlockIndexCache) addBlockToCache(indexBlobID blob.ID, data
 
 	// rename() is atomic, so one process will succeed, but the other will fail
 	if err := os.Rename(tmpFile, c.indexBlobPath(indexBlobID)); err != nil {
-		// verify that the block exists
+		// verify that the content exists
 		exists, err := c.hasIndexBlobID(indexBlobID)
 		if err != nil {
 			return err
 		}
 		if !exists {
-			return errors.Errorf("unsuccessful index write of block %q", indexBlobID)
+			return errors.Errorf("unsuccessful index write of content %q", indexBlobID)
 		}
 	}
 
@@ -102,7 +102,7 @@ func writeTempFileAtomic(dirname string, data []byte) (string, error) {
 	return tf.Name(), nil
 }
 
-func (c *diskCommittedBlockIndexCache) expireUnused(used []blob.ID) error {
+func (c *diskCommittedContentIndexCache) expireUnused(used []blob.ID) error {
 	entries, err := ioutil.ReadDir(c.dirname)
 	if err != nil {
 		return errors.Wrap(err, "can't list cache")
@@ -122,7 +122,7 @@ func (c *diskCommittedBlockIndexCache) expireUnused(used []blob.ID) error {
 	}
 
 	for _, rem := range remaining {
-		if time.Since(rem.ModTime()) > unusedCommittedBlockIndexCleanupTime {
+		if time.Since(rem.ModTime()) > unusedCommittedContentIndexCleanupTime {
 			log.Debugf("removing unused %v %v", rem.Name(), rem.ModTime())
 			if err := os.Remove(filepath.Join(c.dirname, rem.Name())); err != nil {
 				log.Warningf("unable to remove unused index file: %v", err)

repo/content/committed_content_index_mem_cache.go

@@ -0,0 +1,51 @@
+package content
+
+import (
+	"bytes"
+	"sync"
+
+	"github.com/pkg/errors"
+
+	"github.com/kopia/kopia/repo/blob"
+)
+
+type memoryCommittedContentIndexCache struct {
+	mu       sync.Mutex
+	contents map[blob.ID]packIndex
+}
+
+func (m *memoryCommittedContentIndexCache) hasIndexBlobID(indexBlobID blob.ID) (bool, error) {
+	m.mu.Lock()
+	defer m.mu.Unlock()
+
+	return m.contents[indexBlobID] != nil, nil
+}
+
+func (m *memoryCommittedContentIndexCache) addContentToCache(indexBlobID blob.ID, data []byte) error {
+	m.mu.Lock()
+	defer m.mu.Unlock()
+
+	ndx, err := openPackIndex(bytes.NewReader(data))
+	if err != nil {
+		return err
+	}
+
+	m.contents[indexBlobID] = ndx
+	return nil
+}
+
+func (m *memoryCommittedContentIndexCache) openIndex(indexBlobID blob.ID) (packIndex, error) {
+	m.mu.Lock()
+	defer m.mu.Unlock()
+
+	v := m.contents[indexBlobID]
+	if v == nil {
+		return nil, errors.Errorf("content not found in cache: %v", indexBlobID)
+	}
+
+	return v, nil
+}
+
+func (m *memoryCommittedContentIndexCache) expireUnused(used []blob.ID) error {
+	return nil
+}

repo/content/content_cache.go

@@ -1,4 +1,4 @@
-package block
+package content
 
 import (
 	"container/heap"
@@ -19,7 +19,7 @@
 	defaultTouchThreshold = 10 * time.Minute
 )
 
-type blockCache struct {
+type contentCache struct {
 	st             blob.Storage
 	cacheStorage   blob.Storage
 	maxSizeBytes   int64
@@ -33,12 +33,12 @@ type blockCache struct {
 	closed chan struct{}
 }
 
-type blockToucher interface {
-	TouchBlob(ctx context.Context, blockID blob.ID, threshold time.Duration) error
+type contentToucher interface {
+	TouchBlob(ctx context.Context, contentID blob.ID, threshold time.Duration) error
 }
 
 func adjustCacheKey(cacheKey blob.ID) blob.ID {
-	// block IDs with odd length have a single-byte prefix.
+	// content IDs with odd length have a single-byte prefix.
 	// move the prefix to the end of cache key to make sure the top level shard is spread 256 ways.
 	if len(cacheKey)%2 == 1 {
 		return cacheKey[1:] + cacheKey[0:1]
@@ -47,12 +47,12 @@ func adjustCacheKey(cacheKey blob.ID) blob.ID {
 	return cacheKey
 }
 
-func (c *blockCache) getContentBlock(ctx context.Context, cacheKey blob.ID, blobID blob.ID, offset, length int64) ([]byte, error) {
+func (c *contentCache) getContentContent(ctx context.Context, cacheKey blob.ID, blobID blob.ID, offset, length int64) ([]byte, error) {
 	cacheKey = adjustCacheKey(cacheKey)
 
-	useCache := shouldUseBlockCache(ctx) && c.cacheStorage != nil
+	useCache := shouldUseContentCache(ctx) && c.cacheStorage != nil
 	if useCache {
-		if b := c.readAndVerifyCacheBlock(ctx, cacheKey); b != nil {
+		if b := c.readAndVerifyCacheContent(ctx, cacheKey); b != nil {
 			return b, nil
 		}
 	}
@@ -72,12 +72,12 @@ func (c *blockCache) getContentBlock(ctx context.Context, cacheKey blob.ID, blob
 	return b, err
 }
 
-func (c *blockCache) readAndVerifyCacheBlock(ctx context.Context, cacheKey blob.ID) []byte {
+func (c *contentCache) readAndVerifyCacheContent(ctx context.Context, cacheKey blob.ID) []byte {
 	b, err := c.cacheStorage.GetBlob(ctx, cacheKey, 0, -1)
 	if err == nil {
 		b, err = verifyAndStripHMAC(b, c.hmacSecret)
 		if err == nil {
-			if t, ok := c.cacheStorage.(blockToucher); ok {
+			if t, ok := c.cacheStorage.(contentToucher); ok {
 				t.TouchBlob(ctx, cacheKey, c.touchThreshold) //nolint:errcheck
 			}
 
@@ -85,8 +85,8 @@ func (c *blockCache) readAndVerifyCacheBlock(ctx context.Context, cacheKey blob.
 			return b
 		}
 
-		// ignore malformed blocks
-		log.Warningf("malformed block %v: %v", cacheKey, err)
+		// ignore malformed contents
+		log.Warningf("malformed content %v: %v", cacheKey, err)
 		return nil
 	}
 
@@ -96,11 +96,11 @@ func (c *blockCache) readAndVerifyCacheBlock(ctx context.Context, cacheKey blob.
 	return nil
 }
 
-func (c *blockCache) close() {
+func (c *contentCache) close() {
 	close(c.closed)
 }
 
-func (c *blockCache) sweepDirectoryPeriodically(ctx context.Context) {
+func (c *contentCache) sweepDirectoryPeriodically(ctx context.Context) {
 	for {
 		select {
 		case <-c.closed:
@@ -109,30 +109,30 @@ func (c *blockCache) sweepDirectoryPeriodically(ctx context.Context) {
 		case <-time.After(c.sweepFrequency):
 			err := c.sweepDirectory(ctx)
 			if err != nil {
-				log.Warningf("blockCache sweep failed: %v", err)
+				log.Warningf("contentCache sweep failed: %v", err)
 			}
 		}
 	}
 }
 
-// A blockMetadataHeap implements heap.Interface and holds blob.Metadata.
-type blockMetadataHeap []blob.Metadata
+// A contentMetadataHeap implements heap.Interface and holds blob.Metadata.
+type contentMetadataHeap []blob.Metadata
 
-func (h blockMetadataHeap) Len() int { return len(h) }
+func (h contentMetadataHeap) Len() int { return len(h) }
 
-func (h blockMetadataHeap) Less(i, j int) bool {
+func (h contentMetadataHeap) Less(i, j int) bool {
 	return h[i].Timestamp.Before(h[j].Timestamp)
 }
 
-func (h blockMetadataHeap) Swap(i, j int) {
+func (h contentMetadataHeap) Swap(i, j int) {
 	h[i], h[j] = h[j], h[i]
 }
 
-func (h *blockMetadataHeap) Push(x interface{}) {
+func (h *contentMetadataHeap) Push(x interface{}) {
 	*h = append(*h, x.(blob.Metadata))
 }
 
-func (h *blockMetadataHeap) Pop() interface{} {
+func (h *contentMetadataHeap) Pop() interface{} {
 	old := *h
 	n := len(old)
 	item := old[n-1]
@@ -140,7 +140,7 @@ func (h *blockMetadataHeap) Pop() interface{} {
 	return item
 }
 
-func (c *blockCache) sweepDirectory(ctx context.Context) (err error) {
+func (c *contentCache) sweepDirectory(ctx context.Context) (err error) {
 	c.mu.Lock()
 	defer c.mu.Unlock()
 
@@ -150,7 +150,7 @@ func (c *blockCache) sweepDirectory(ctx context.Context) (err error) {
 
 	t0 := time.Now()
 
-	var h blockMetadataHeap
+	var h contentMetadataHeap
 	var totalRetainedSize int64
 
 	err = c.cacheStorage.ListBlobs(ctx, "", func(it blob.Metadata) error {
@@ -176,21 +176,21 @@ func (c *blockCache) sweepDirectory(ctx context.Context) (err error) {
 	return nil
 }
 
-func newBlockCache(ctx context.Context, st blob.Storage, caching CachingOptions) (*blockCache, error) {
+func newContentCache(ctx context.Context, st blob.Storage, caching CachingOptions) (*contentCache, error) {
 	var cacheStorage blob.Storage
 	var err error
 
 	if caching.MaxCacheSizeBytes > 0 && caching.CacheDirectory != "" {
-		blockCacheDir := filepath.Join(caching.CacheDirectory, "blocks")
+		contentCacheDir := filepath.Join(caching.CacheDirectory, "contents")
 
-		if _, err = os.Stat(blockCacheDir); os.IsNotExist(err) {
-			if err = os.MkdirAll(blockCacheDir, 0700); err != nil {
+		if _, err = os.Stat(contentCacheDir); os.IsNotExist(err) {
+			if err = os.MkdirAll(contentCacheDir, 0700); err != nil {
 				return nil, err
 			}
 		}
 
 		cacheStorage, err = filesystem.New(context.Background(), &filesystem.Options{
-			Path:            blockCacheDir,
+			Path:            contentCacheDir,
 			DirectoryShards: []int{2},
 		})
 		if err != nil {
@@ -198,11 +198,11 @@ func newBlockCache(ctx context.Context, st blob.Storage, caching CachingOptions)
 		}
 	}
 
-	return newBlockCacheWithCacheStorage(ctx, st, cacheStorage, caching, defaultTouchThreshold, defaultSweepFrequency)
+	return newContentCacheWithCacheStorage(ctx, st, cacheStorage, caching, defaultTouchThreshold, defaultSweepFrequency)
 }
 
-func newBlockCacheWithCacheStorage(ctx context.Context, st, cacheStorage blob.Storage, caching CachingOptions, touchThreshold time.Duration, sweepFrequency time.Duration) (*blockCache, error) {
-	c := &blockCache{
+func newContentCacheWithCacheStorage(ctx context.Context, st, cacheStorage blob.Storage, caching CachingOptions, touchThreshold time.Duration, sweepFrequency time.Duration) (*contentCache, error) {
+	c := &contentCache{
 		st:             st,
 		cacheStorage:   cacheStorage,
 		maxSizeBytes:   caching.MaxCacheSizeBytes,

repo/content/content_cache_test.go

@@ -1,4 +1,4 @@
-package block
+package content
 
 import (
 	"bytes"
@@ -17,12 +17,12 @@
 	"github.com/kopia/kopia/repo/blob"
 )
 
-func newUnderlyingStorageForBlockCacheTesting(t *testing.T) blob.Storage {
+func newUnderlyingStorageForContentCacheTesting(t *testing.T) blob.Storage {
 	ctx := context.Background()
 	data := blobtesting.DataMap{}
 	st := blobtesting.NewMapStorage(data, nil, nil)
-	assertNoError(t, st.PutBlob(ctx, "block-1", []byte{1, 2, 3, 4, 5, 6, 7, 8, 9, 10}))
-	assertNoError(t, st.PutBlob(ctx, "block-4k", bytes.Repeat([]byte{1, 2, 3, 4}, 1000))) // 4000 bytes
+	assertNoError(t, st.PutBlob(ctx, "content-1", []byte{1, 2, 3, 4, 5, 6, 7, 8, 9, 10}))
+	assertNoError(t, st.PutBlob(ctx, "content-4k", bytes.Repeat([]byte{1, 2, 3, 4}, 1000))) // 4000 bytes
 	return st
 }
 
@@ -30,9 +30,9 @@ func TestCacheExpiration(t *testing.T) {
 	cacheData := blobtesting.DataMap{}
 	cacheStorage := blobtesting.NewMapStorage(cacheData, nil, nil)
 
-	underlyingStorage := newUnderlyingStorageForBlockCacheTesting(t)
+	underlyingStorage := newUnderlyingStorageForContentCacheTesting(t)
 
-	cache, err := newBlockCacheWithCacheStorage(context.Background(), underlyingStorage, cacheStorage, CachingOptions{
+	cache, err := newContentCacheWithCacheStorage(context.Background(), underlyingStorage, cacheStorage, CachingOptions{
 		MaxCacheSizeBytes: 10000,
 	}, 0, 500*time.Millisecond)
 	if err != nil {
@@ -41,22 +41,22 @@ func TestCacheExpiration(t *testing.T) {
 	defer cache.close()
 
 	ctx := context.Background()
-	_, err = cache.getContentBlock(ctx, "00000a", "block-4k", 0, -1) // 4k
+	_, err = cache.getContentContent(ctx, "00000a", "content-4k", 0, -1) // 4k
 	assertNoError(t, err)
-	_, err = cache.getContentBlock(ctx, "00000b", "block-4k", 0, -1) // 4k
+	_, err = cache.getContentContent(ctx, "00000b", "content-4k", 0, -1) // 4k
 	assertNoError(t, err)
-	_, err = cache.getContentBlock(ctx, "00000c", "block-4k", 0, -1) // 4k
+	_, err = cache.getContentContent(ctx, "00000c", "content-4k", 0, -1) // 4k
 	assertNoError(t, err)
-	_, err = cache.getContentBlock(ctx, "00000d", "block-4k", 0, -1) // 4k
+	_, err = cache.getContentContent(ctx, "00000d", "content-4k", 0, -1) // 4k
 	assertNoError(t, err)
 
 	// wait for a sweep
 	time.Sleep(2 * time.Second)
 
 	// 00000a and 00000b will be removed from cache because it's the oldest.
-	// to verify, let's remove block-4k from the underlying storage and make sure we can still read
+	// to verify, let's remove content-4k from the underlying storage and make sure we can still read
 	// 00000c and 00000d from the cache but not 00000a nor 00000b
-	assertNoError(t, underlyingStorage.DeleteBlob(ctx, "block-4k"))
+	assertNoError(t, underlyingStorage.DeleteBlob(ctx, "content-4k"))
 
 	cases := []struct {
 		blobID        blob.ID
@@ -69,16 +69,16 @@ func TestCacheExpiration(t *testing.T) {
 	}
 
 	for _, tc := range cases {
-		_, got := cache.getContentBlock(ctx, tc.blobID, "block-4k", 0, -1)
+		_, got := cache.getContentContent(ctx, tc.blobID, "content-4k", 0, -1)
 		if want := tc.expectedError; got != want {
-			t.Errorf("unexpected error when getting block %v: %v wanted %v", tc.blobID, got, want)
+			t.Errorf("unexpected error when getting content %v: %v wanted %v", tc.blobID, got, want)
 		} else {
-			t.Logf("got correct error %v when reading block %v", tc.expectedError, tc.blobID)
+			t.Logf("got correct error %v when reading content %v", tc.expectedError, tc.blobID)
 		}
 	}
 }
 
-func TestDiskBlockCache(t *testing.T) {
+func TestDiskContentCache(t *testing.T) {
 	ctx := context.Background()
 
 	tmpDir, err := ioutil.TempDir("", "kopia")
@@ -87,7 +87,7 @@ func TestDiskBlockCache(t *testing.T) {
 	}
 	defer os.RemoveAll(tmpDir)
 
-	cache, err := newBlockCache(ctx, newUnderlyingStorageForBlockCacheTesting(t), CachingOptions{
+	cache, err := newContentCache(ctx, newUnderlyingStorageForContentCacheTesting(t), CachingOptions{
 		MaxCacheSizeBytes: 10000,
 		CacheDirectory:    tmpDir,
 	})
@@ -96,13 +96,13 @@ func TestDiskBlockCache(t *testing.T) {
 		t.Fatalf("err: %v", err)
 	}
 	defer cache.close()
-	verifyBlockCache(t, cache)
+	verifyContentCache(t, cache)
 }
 
-func verifyBlockCache(t *testing.T, cache *blockCache) {
+func verifyContentCache(t *testing.T, cache *contentCache) {
 	ctx := context.Background()
 
-	t.Run("GetContentBlock", func(t *testing.T) {
+	t.Run("GetContentContent", func(t *testing.T) {
 		cases := []struct {
 			cacheKey blob.ID
 			blobID   blob.ID
@@ -112,19 +112,19 @@ func verifyBlockCache(t *testing.T, cache *blockCache) {
 			expected []byte
 			err      error
 		}{
-			{"xf0f0f1", "block-1", 1, 5, []byte{2, 3, 4, 5, 6}, nil},
-			{"xf0f0f2", "block-1", 0, -1, []byte{1, 2, 3, 4, 5, 6, 7, 8, 9, 10}, nil},
-			{"xf0f0f1", "block-1", 1, 5, []byte{2, 3, 4, 5, 6}, nil},
-			{"xf0f0f2", "block-1", 0, -1, []byte{1, 2, 3, 4, 5, 6, 7, 8, 9, 10}, nil},
-			{"xf0f0f3", "no-such-block", 0, -1, nil, blob.ErrBlobNotFound},
-			{"xf0f0f4", "no-such-block", 10, 5, nil, blob.ErrBlobNotFound},
-			{"f0f0f5", "block-1", 7, 3, []byte{8, 9, 10}, nil},
-			{"xf0f0f6", "block-1", 11, 10, nil, errors.Errorf("invalid offset")},
-			{"xf0f0f6", "block-1", -1, 5, nil, errors.Errorf("invalid offset")},
+			{"xf0f0f1", "content-1", 1, 5, []byte{2, 3, 4, 5, 6}, nil},
+			{"xf0f0f2", "content-1", 0, -1, []byte{1, 2, 3, 4, 5, 6, 7, 8, 9, 10}, nil},
+			{"xf0f0f1", "content-1", 1, 5, []byte{2, 3, 4, 5, 6}, nil},
+			{"xf0f0f2", "content-1", 0, -1, []byte{1, 2, 3, 4, 5, 6, 7, 8, 9, 10}, nil},
+			{"xf0f0f3", "no-such-content", 0, -1, nil, blob.ErrBlobNotFound},
+			{"xf0f0f4", "no-such-content", 10, 5, nil, blob.ErrBlobNotFound},
+			{"f0f0f5", "content-1", 7, 3, []byte{8, 9, 10}, nil},
+			{"xf0f0f6", "content-1", 11, 10, nil, errors.Errorf("invalid offset")},
+			{"xf0f0f6", "content-1", -1, 5, nil, errors.Errorf("invalid offset")},
 		}
 
 		for _, tc := range cases {
-			v, err := cache.getContentBlock(ctx, tc.cacheKey, tc.blobID, tc.offset, tc.length)
+			v, err := cache.getContentContent(ctx, tc.cacheKey, tc.blobID, tc.offset, tc.length)
 			if (err != nil) != (tc.err != nil) {
 				t.Errorf("unexpected error for %v: %+v, wanted %+v", tc.cacheKey, err, tc.err)
 			} else if err != nil && err.Error() != tc.err.Error() {
@@ -135,7 +135,7 @@ func verifyBlockCache(t *testing.T, cache *blockCache) {
 			}
 		}
 
-		verifyStorageBlockList(t, cache.cacheStorage, "f0f0f1x", "f0f0f2x", "f0f0f5")
+		verifyStorageContentList(t, cache.cacheStorage, "f0f0f1x", "f0f0f2x", "f0f0f5")
 	})
 
 	t.Run("DataCorruption", func(t *testing.T) {
@@ -149,12 +149,12 @@ func verifyBlockCache(t *testing.T, cache *blockCache) {
 		d[0] ^= 1
 
 		if err := cache.cacheStorage.PutBlob(ctx, cacheKey, d); err != nil {
-			t.Fatalf("unable to write corrupted block: %v", err)
+			t.Fatalf("unable to write corrupted content: %v", err)
 		}
 
-		v, err := cache.getContentBlock(ctx, "xf0f0f1", "block-1", 1, 5)
+		v, err := cache.getContentContent(ctx, "xf0f0f1", "content-1", 1, 5)
 		if err != nil {
-			t.Fatalf("error in getContentBlock: %v", err)
+			t.Fatalf("error in getContentContent: %v", err)
 		}
 		if got, want := v, []byte{2, 3, 4, 5, 6}; !reflect.DeepEqual(v, want) {
 			t.Errorf("invalid result when reading corrupted data: %v, wanted %v", got, want)
@@ -167,7 +167,7 @@ func TestCacheFailureToOpen(t *testing.T) {
 
 	cacheData := blobtesting.DataMap{}
 	cacheStorage := blobtesting.NewMapStorage(cacheData, nil, nil)
-	underlyingStorage := newUnderlyingStorageForBlockCacheTesting(t)
+	underlyingStorage := newUnderlyingStorageForContentCacheTesting(t)
 	faultyCache := &blobtesting.FaultyStorage{
 		Base: cacheStorage,
 		Faults: map[string][]*blobtesting.Fault{
@@ -178,7 +178,7 @@ func TestCacheFailureToOpen(t *testing.T) {
 	}
 
 	// Will fail because of ListBlobs failure.
-	_, err := newBlockCacheWithCacheStorage(context.Background(), underlyingStorage, faultyCache, CachingOptions{
+	_, err := newContentCacheWithCacheStorage(context.Background(), underlyingStorage, faultyCache, CachingOptions{
 		MaxCacheSizeBytes: 10000,
 	}, 0, 5*time.Hour)
 	if err == nil || !strings.Contains(err.Error(), someError.Error()) {
@@ -186,7 +186,7 @@ func TestCacheFailureToOpen(t *testing.T) {
 	}
 
 	// ListBlobs fails only once, next time it succeeds.
-	cache, err := newBlockCacheWithCacheStorage(context.Background(), underlyingStorage, faultyCache, CachingOptions{
+	cache, err := newContentCacheWithCacheStorage(context.Background(), underlyingStorage, faultyCache, CachingOptions{
 		MaxCacheSizeBytes: 10000,
 	}, 0, 100*time.Millisecond)
 	if err != nil {
@@ -201,12 +201,12 @@ func TestCacheFailureToWrite(t *testing.T) {
 
 	cacheData := blobtesting.DataMap{}
 	cacheStorage := blobtesting.NewMapStorage(cacheData, nil, nil)
-	underlyingStorage := newUnderlyingStorageForBlockCacheTesting(t)
+	underlyingStorage := newUnderlyingStorageForContentCacheTesting(t)
 	faultyCache := &blobtesting.FaultyStorage{
 		Base: cacheStorage,
 	}
 
-	cache, err := newBlockCacheWithCacheStorage(context.Background(), underlyingStorage, faultyCache, CachingOptions{
+	cache, err := newContentCacheWithCacheStorage(context.Background(), underlyingStorage, faultyCache, CachingOptions{
 		MaxCacheSizeBytes: 10000,
 	}, 0, 5*time.Hour)
 	if err != nil {
@@ -222,7 +222,7 @@ func TestCacheFailureToWrite(t *testing.T) {
 		},
 	}
 
-	v, err := cache.getContentBlock(ctx, "aa", "block-1", 0, 3)
+	v, err := cache.getContentContent(ctx, "aa", "content-1", 0, 3)
 	if err != nil {
 		t.Errorf("write failure wasn't ignored: %v", err)
 	}
@@ -245,12 +245,12 @@ func TestCacheFailureToRead(t *testing.T) {
 
 	cacheData := blobtesting.DataMap{}
 	cacheStorage := blobtesting.NewMapStorage(cacheData, nil, nil)
-	underlyingStorage := newUnderlyingStorageForBlockCacheTesting(t)
+	underlyingStorage := newUnderlyingStorageForContentCacheTesting(t)
 	faultyCache := &blobtesting.FaultyStorage{
 		Base: cacheStorage,
 	}
 
-	cache, err := newBlockCacheWithCacheStorage(context.Background(), underlyingStorage, faultyCache, CachingOptions{
+	cache, err := newContentCacheWithCacheStorage(context.Background(), underlyingStorage, faultyCache, CachingOptions{
 		MaxCacheSizeBytes: 10000,
 	}, 0, 5*time.Hour)
 	if err != nil {
@@ -267,7 +267,7 @@ func TestCacheFailureToRead(t *testing.T) {
 	}
 
 	for i := 0; i < 2; i++ {
-		v, err := cache.getContentBlock(ctx, "aa", "block-1", 0, 3)
+		v, err := cache.getContentContent(ctx, "aa", "content-1", 0, 3)
 		if err != nil {
 			t.Errorf("read failure wasn't ignored: %v", err)
 		}
@@ -278,19 +278,19 @@ func TestCacheFailureToRead(t *testing.T) {
 	}
 }
 
-func verifyStorageBlockList(t *testing.T, st blob.Storage, expectedBlocks ...blob.ID) {
+func verifyStorageContentList(t *testing.T, st blob.Storage, expectedContents ...blob.ID) {
 	t.Helper()
-	var foundBlocks []blob.ID
+	var foundContents []blob.ID
 	assertNoError(t, st.ListBlobs(context.Background(), "", func(bm blob.Metadata) error {
-		foundBlocks = append(foundBlocks, bm.BlobID)
+		foundContents = append(foundContents, bm.BlobID)
 		return nil
 	}))
 
-	sort.Slice(foundBlocks, func(i, j int) bool {
-		return foundBlocks[i] < foundBlocks[j]
+	sort.Slice(foundContents, func(i, j int) bool {
+		return foundContents[i] < foundContents[j]
 	})
-	if !reflect.DeepEqual(foundBlocks, expectedBlocks) {
-		t.Errorf("unexpected block list: %v, wanted %v", foundBlocks, expectedBlocks)
+	if !reflect.DeepEqual(foundContents, expectedContents) {
+		t.Errorf("unexpected content list: %v, wanted %v", foundContents, expectedContents)
 	}
 }
 

repo/content/content_formatter.go

@@ -1,4 +1,4 @@
-package block
+package content
 
 import (
 	"crypto/aes"
@@ -22,22 +22,22 @@
 	salsaKeyLength       = 32
 )
 
-// HashFunc computes hash of block of data using a cryptographic hash function, possibly with HMAC and/or truncation.
+// HashFunc computes hash of content of data using a cryptographic hash function, possibly with HMAC and/or truncation.
 type HashFunc func(data []byte) []byte
 
 // HashFuncFactory returns a hash function for given formatting options.
 type HashFuncFactory func(o FormattingOptions) (HashFunc, error)
 
-// Encryptor performs encryption and decryption of blocks of data.
+// Encryptor performs encryption and decryption of contents of data.
 type Encryptor interface {
 	// Encrypt returns encrypted bytes corresponding to the given plaintext.
 	// Must not clobber the input slice and return ciphertext with additional padding and checksum.
-	Encrypt(plainText []byte, blockID []byte) ([]byte, error)
+	Encrypt(plainText []byte, contentID []byte) ([]byte, error)
 
 	// Decrypt returns unencrypted bytes corresponding to the given ciphertext.
 	// Must not clobber the input slice. If IsAuthenticated() == true, Decrypt will perform
 	// authenticity check before decrypting.
-	Decrypt(cipherText []byte, blockID []byte) ([]byte, error)
+	Decrypt(cipherText []byte, contentID []byte) ([]byte, error)
 
 	// IsAuthenticated returns true if encryption is authenticated.
 	// In this case Decrypt() is expected to perform authenticity check.
@@ -54,11 +54,11 @@ type Encryptor interface {
 type nullEncryptor struct {
 }
 
-func (fi nullEncryptor) Encrypt(plainText []byte, blockID []byte) ([]byte, error) {
+func (fi nullEncryptor) Encrypt(plainText []byte, contentID []byte) ([]byte, error) {
 	return cloneBytes(plainText), nil
 }
 
-func (fi nullEncryptor) Decrypt(cipherText []byte, blockID []byte) ([]byte, error) {
+func (fi nullEncryptor) Decrypt(cipherText []byte, contentID []byte) ([]byte, error) {
 	return cloneBytes(cipherText), nil
 }
 
@@ -66,17 +66,17 @@ func (fi nullEncryptor) IsAuthenticated() bool {
 	return false
 }
 
-// ctrEncryptor implements encrypted format which uses CTR mode of a block cipher with nonce==IV.
+// ctrEncryptor implements encrypted format which uses CTR mode of a content cipher with nonce==IV.
 type ctrEncryptor struct {
 	createCipher func() (cipher.Block, error)
 }
 
-func (fi ctrEncryptor) Encrypt(plainText []byte, blockID []byte) ([]byte, error) {
-	return symmetricEncrypt(fi.createCipher, blockID, plainText)
+func (fi ctrEncryptor) Encrypt(plainText []byte, contentID []byte) ([]byte, error) {
+	return symmetricEncrypt(fi.createCipher, contentID, plainText)
 }
 
-func (fi ctrEncryptor) Decrypt(cipherText []byte, blockID []byte) ([]byte, error) {
-	return symmetricEncrypt(fi.createCipher, blockID, cipherText)
+func (fi ctrEncryptor) Decrypt(cipherText []byte, contentID []byte) ([]byte, error) {
+	return symmetricEncrypt(fi.createCipher, contentID, cipherText)
 }
 
 func (fi ctrEncryptor) IsAuthenticated() bool {
@@ -101,7 +101,7 @@ type salsaEncryptor struct {
 	hmacSecret []byte
 }
 
-func (s salsaEncryptor) Decrypt(input []byte, blockID []byte) ([]byte, error) {
+func (s salsaEncryptor) Decrypt(input []byte, contentID []byte) ([]byte, error) {
 	if s.hmacSecret != nil {
 		var err error
 		input, err = verifyAndStripHMAC(input, s.hmacSecret)
@@ -110,11 +110,11 @@ func (s salsaEncryptor) Decrypt(input []byte, blockID []byte) ([]byte, error) {
 		}
 	}
 
-	return s.encryptDecrypt(input, blockID)
+	return s.encryptDecrypt(input, contentID)
 }
 
-func (s salsaEncryptor) Encrypt(input []byte, blockID []byte) ([]byte, error) {
-	v, err := s.encryptDecrypt(input, blockID)
+func (s salsaEncryptor) Encrypt(input []byte, contentID []byte) ([]byte, error) {
+	v, err := s.encryptDecrypt(input, contentID)
 	if err != nil {
 		return nil, errors.Wrap(err, "decrypt")
 	}
@@ -130,17 +130,17 @@ func (s salsaEncryptor) IsAuthenticated() bool {
 	return s.hmacSecret != nil
 }
 
-func (s salsaEncryptor) encryptDecrypt(input []byte, blockID []byte) ([]byte, error) {
-	if len(blockID) < s.nonceSize {
-		return nil, errors.Errorf("hash too short, expected >=%v bytes, got %v", s.nonceSize, len(blockID))
+func (s salsaEncryptor) encryptDecrypt(input []byte, contentID []byte) ([]byte, error) {
+	if len(contentID) < s.nonceSize {
+		return nil, errors.Errorf("hash too short, expected >=%v bytes, got %v", s.nonceSize, len(contentID))
 	}
 	result := make([]byte, len(input))
-	nonce := blockID[0:s.nonceSize]
+	nonce := contentID[0:s.nonceSize]
 	salsa20.XORKeyStream(result, input, nonce, s.key)
 	return result, nil
 }
 
-// truncatedHMACHashFuncFactory returns a HashFuncFactory that computes HMAC(hash, secret) of a given block of bytes
+// truncatedHMACHashFuncFactory returns a HashFuncFactory that computes HMAC(hash, secret) of a given content of bytes
 // and truncates results to the given size.
 func truncatedHMACHashFuncFactory(hf func() hash.Hash, truncate int) HashFuncFactory {
 	return func(o FormattingOptions) (HashFunc, error) {
@@ -152,7 +152,7 @@ func truncatedHMACHashFuncFactory(hf func() hash.Hash, truncate int) HashFuncFac
 	}
 }
 
-// truncatedKeyedHashFuncFactory returns a HashFuncFactory that computes keyed hash of a given block of bytes
+// truncatedKeyedHashFuncFactory returns a HashFuncFactory that computes keyed hash of a given content of bytes
 // and truncates results to the given size.
 func truncatedKeyedHashFuncFactory(hf func(key []byte) (hash.Hash, error), truncate int) HashFuncFactory {
 	return func(o FormattingOptions) (HashFunc, error) {

repo/content/content_formatter_test.go

@@ -1,4 +1,4 @@
-package block
+package content
 
 import (
 	"bytes"
@@ -49,13 +49,13 @@ func TestFormatters(t *testing.T) {
 				continue
 			}
 
-			blockID := h(data)
-			cipherText, err := e.Encrypt(data, blockID)
+			contentID := h(data)
+			cipherText, err := e.Encrypt(data, contentID)
 			if err != nil || cipherText == nil {
 				t.Errorf("invalid response from Encrypt: %v %v", cipherText, err)
 			}
 
-			plainText, err := e.Decrypt(cipherText, blockID)
+			plainText, err := e.Decrypt(cipherText, contentID)
 			if err != nil || plainText == nil {
 				t.Errorf("invalid response from Decrypt: %v %v", plainText, err)
 			}

repo/content/content_formatting_options.go

@@ -1,4 +1,4 @@
-package block
+package content
 
 import (
 	"crypto/sha256"
@@ -7,7 +7,7 @@
 	"golang.org/x/crypto/hkdf"
 )
 
-// FormattingOptions describes the rules for formatting blocks in repository.
+// FormattingOptions describes the rules for formatting contents in repository.
 type FormattingOptions struct {
 	Version     int    `json:"version,omitempty"`     // version number, must be "1"
 	Hash        string `json:"hash,omitempty"`        // identifier of the hash algorithm used

repo/content/content_id_to_bytes.go

@@ -1,25 +1,25 @@
-package block
+package content
 
 import (
 	"encoding/hex"
 )
 
-func bytesToContentID(b []byte) string {
+func bytesToContentID(b []byte) ID {
 	if len(b) == 0 {
 		return ""
 	}
 	if b[0] == 0xff {
-		return string(b[1:])
+		return ID(b[1:])
 	}
 	prefix := ""
 	if b[0] != 0 {
 		prefix = string(b[0:1])
 	}
 
-	return prefix + hex.EncodeToString(b[1:])
+	return ID(prefix + hex.EncodeToString(b[1:]))
 }
 
-func contentIDToBytes(c string) []byte {
+func contentIDToBytes(c ID) []byte {
 	var prefix []byte
 	var skip int
 	if len(c)%2 == 1 {
@@ -29,7 +29,7 @@ func contentIDToBytes(c string) []byte {
 		prefix = []byte{0}
 	}
 
-	b, err := hex.DecodeString(c[skip:])
+	b, err := hex.DecodeString(string(c[skip:]))
 	if err != nil {
 		return append([]byte{0xff}, []byte(c)...)
 	}

repo/content/content_index_recovery.go

@@ -1,4 +1,4 @@
-package block
+package content
 
 import (
 	"bytes"
@@ -39,13 +39,13 @@ func (bm *Manager) RecoverIndexFromPackBlob(ctx context.Context, packFile blob.I
 	return recovered, err
 }
 
-type packBlockPostamble struct {
+type packContentPostamble struct {
 	localIndexIV     []byte
 	localIndexOffset uint32
 	localIndexLength uint32
 }
 
-func (p *packBlockPostamble) toBytes() ([]byte, error) {
+func (p *packContentPostamble) toBytes() ([]byte, error) {
 	// 4 varints + IV + 4 bytes of checksum + 1 byte of postamble length
 	n := 0
 	buf := make([]byte, 4*binary.MaxVarintLen64+len(p.localIndexIV)+4+1)
@@ -68,10 +68,10 @@ func (p *packBlockPostamble) toBytes() ([]byte, error) {
 	return buf[0 : n+1], nil
 }
 
-// findPostamble detects if a given block of bytes contains a possibly valid postamble, and returns it if so
+// findPostamble detects if a given content of bytes contains a possibly valid postamble, and returns it if so
 // NOTE, even if this function returns a postamble, it should not be trusted to be correct, since it's not
 // cryptographically signed. this is to facilitate data recovery.
-func findPostamble(b []byte) *packBlockPostamble {
+func findPostamble(b []byte) *packContentPostamble {
 	if len(b) == 0 {
 		// no postamble
 		return nil
@@ -103,7 +103,7 @@ func findPostamble(b []byte) *packBlockPostamble {
 	return decodePostamble(payload)
 }
 
-func decodePostamble(payload []byte) *packBlockPostamble {
+func decodePostamble(payload []byte) *packContentPostamble {
 	flags, n := binary.Uvarint(payload)
 	if n <= 0 {
 		// invalid flags
@@ -142,7 +142,7 @@ func decodePostamble(payload []byte) *packBlockPostamble {
 		return nil
 	}
 
-	return &packBlockPostamble{
+	return &packContentPostamble{
 		localIndexIV:     iv,
 		localIndexLength: uint32(length),
 		localIndexOffset: uint32(off),
@@ -159,9 +159,9 @@ func (bm *Manager) buildLocalIndex(pending packIndexBuilder) ([]byte, error) {
 }
 
 // appendPackFileIndexRecoveryData appends data designed to help with recovery of pack index in case it gets damaged or lost.
-func (bm *Manager) appendPackFileIndexRecoveryData(blockData []byte, pending packIndexBuilder) ([]byte, error) {
+func (bm *Manager) appendPackFileIndexRecoveryData(contentData []byte, pending packIndexBuilder) ([]byte, error) {
 	// build, encrypt and append local index
-	localIndexOffset := len(blockData)
+	localIndexOffset := len(contentData)
 	localIndex, err := bm.buildLocalIndex(pending)
 	if err != nil {
 		return nil, err
@@ -173,21 +173,21 @@ func (bm *Manager) appendPackFileIndexRecoveryData(blockData []byte, pending pac
 		return nil, err
 	}
 
-	postamble := packBlockPostamble{
+	postamble := packContentPostamble{
 		localIndexIV:     localIndexIV,
 		localIndexOffset: uint32(localIndexOffset),
 		localIndexLength: uint32(len(encryptedLocalIndex)),
 	}
 
-	blockData = append(blockData, encryptedLocalIndex...)
+	contentData = append(contentData, encryptedLocalIndex...)
 	postambleBytes, err := postamble.toBytes()
 	if err != nil {
 		return nil, err
 	}
 
-	blockData = append(blockData, postambleBytes...)
+	contentData = append(contentData, postambleBytes...)
 
-	pa2 := findPostamble(blockData)
+	pa2 := findPostamble(contentData)
 	if pa2 == nil {
 		log.Fatalf("invalid postamble written, that could not be immediately decoded, it's a bug")
 	}
@@ -196,7 +196,7 @@ func (bm *Manager) appendPackFileIndexRecoveryData(blockData []byte, pending pac
 		log.Fatalf("postamble did not round-trip: %v %v", postamble, *pa2)
 	}
 
-	return blockData, nil
+	return contentData, nil
 }
 
 func (bm *Manager) readPackFileLocalIndex(ctx context.Context, packFile blob.ID, packFileLength int64) ([]byte, error) {

repo/content/content_index_recovery_test.go

@@ -0,0 +1,91 @@
+package content
+
+import (
+	"context"
+	"testing"
+	"time"
+
+	"github.com/kopia/kopia/internal/blobtesting"
+	"github.com/kopia/kopia/repo/blob"
+)
+
+func TestContentIndexRecovery(t *testing.T) {
+	ctx := context.Background()
+	data := blobtesting.DataMap{}
+	keyTime := map[blob.ID]time.Time{}
+	bm := newTestContentManager(data, keyTime, nil)
+	content1 := writeContentAndVerify(ctx, t, bm, seededRandomData(10, 100))
+	content2 := writeContentAndVerify(ctx, t, bm, seededRandomData(11, 100))
+	content3 := writeContentAndVerify(ctx, t, bm, seededRandomData(12, 100))
+
+	if err := bm.Flush(ctx); err != nil {
+		t.Errorf("flush error: %v", err)
+	}
+
+	// delete all index blobs
+	assertNoError(t, bm.st.ListBlobs(ctx, newIndexBlobPrefix, func(bi blob.Metadata) error {
+		log.Debugf("deleting %v", bi.BlobID)
+		return bm.st.DeleteBlob(ctx, bi.BlobID)
+	}))
+
+	// now with index blobs gone, all contents appear to not be found
+	bm = newTestContentManager(data, keyTime, nil)
+	verifyContentNotFound(ctx, t, bm, content1)
+	verifyContentNotFound(ctx, t, bm, content2)
+	verifyContentNotFound(ctx, t, bm, content3)
+
+	totalRecovered := 0
+
+	// pass 1 - just list contents to recover, but don't commit
+	err := bm.st.ListBlobs(ctx, PackBlobIDPrefix, func(bi blob.Metadata) error {
+		infos, err := bm.RecoverIndexFromPackBlob(ctx, bi.BlobID, bi.Length, false)
+		if err != nil {
+			return err
+		}
+		totalRecovered += len(infos)
+		log.Debugf("recovered %v contents", len(infos))
+		return nil
+	})
+	if err != nil {
+		t.Errorf("error recovering: %v", err)
+	}
+
+	if got, want := totalRecovered, 3; got != want {
+		t.Errorf("invalid # of contents recovered: %v, want %v", got, want)
+	}
+
+	// contents are stil not found
+	verifyContentNotFound(ctx, t, bm, content1)
+	verifyContentNotFound(ctx, t, bm, content2)
+	verifyContentNotFound(ctx, t, bm, content3)
+
+	// pass 2 now pass commit=true to add recovered contents to index
+	totalRecovered = 0
+
+	err = bm.st.ListBlobs(ctx, PackBlobIDPrefix, func(bi blob.Metadata) error {
+		infos, err := bm.RecoverIndexFromPackBlob(ctx, bi.BlobID, bi.Length, true)
+		if err != nil {
+			return err
+		}
+		totalRecovered += len(infos)
+		log.Debugf("recovered %v contents", len(infos))
+		return nil
+	})
+	if err != nil {
+		t.Errorf("error recovering: %v", err)
+	}
+
+	if got, want := totalRecovered, 3; got != want {
+		t.Errorf("invalid # of contents recovered: %v, want %v", got, want)
+	}
+
+	verifyContent(ctx, t, bm, content1, seededRandomData(10, 100))
+	verifyContent(ctx, t, bm, content2, seededRandomData(11, 100))
+	verifyContent(ctx, t, bm, content3, seededRandomData(12, 100))
+	if err := bm.Flush(ctx); err != nil {
+		t.Errorf("flush error: %v", err)
+	}
+	verifyContent(ctx, t, bm, content1, seededRandomData(10, 100))
+	verifyContent(ctx, t, bm, content2, seededRandomData(11, 100))
+	verifyContent(ctx, t, bm, content3, seededRandomData(12, 100))
+}

repo/content/content_manager.go

@@ -1,5 +1,5 @@
-// Package block implements repository support content-addressable storage blocks.
-package block
+// Package content implements repository support content-addressable storage contents.
+package content
 
 import (
 	"bytes"
@@ -25,8 +25,8 @@
 )
 
 var (
-	log       = repologging.Logger("kopia/block")
-	formatLog = repologging.Logger("kopia/block/format")
+	log       = repologging.Logger("kopia/content")
+	formatLog = repologging.Logger("kopia/content/format")
 )
 
 // PackBlobIDPrefix is the prefix for all pack blobs.
@@ -51,8 +51,8 @@
 	indexLoadAttempts = 10
 )
 
-// ErrBlockNotFound is returned when block is not found.
-var ErrBlockNotFound = errors.New("block not found")
+// ErrContentNotFound is returned when content is not found.
+var ErrContentNotFound = errors.New("content not found")
 
 // IndexBlobInfo is an information about a single index blob managed by Manager.
 type IndexBlobInfo struct {
@@ -61,23 +61,23 @@ type IndexBlobInfo struct {
 	Timestamp time.Time
 }
 
-// Manager manages storage blocks at a low level with encryption, deduplication and packaging.
+// Manager builds content-addressable storage with encryption, deduplication and packaging on top of BLOB store.
 type Manager struct {
 	Format FormattingOptions
 
-	stats      Stats
-	blockCache *blockCache
-	listCache  *listCache
-	st         blob.Storage
+	stats        Stats
+	contentCache *contentCache
+	listCache    *listCache
+	st           blob.Storage
 
 	mu                      sync.Mutex
 	locked                  bool
 	checkInvariantsOnUnlock bool
 
-	currentPackItems      map[string]Info  // blocks that are in the pack block currently being built (all inline)
-	currentPackDataLength int              // total length of all items in the current pack block
-	packIndexBuilder      packIndexBuilder // blocks that are in index currently being built (current pack and all packs saved but not committed)
-	committedBlocks       *committedBlockIndex
+	currentPackItems      map[ID]Info      // contents that are in the pack content currently being built (all inline)
+	currentPackDataLength int              // total length of all items in the current pack content
+	packIndexBuilder      packIndexBuilder // contents that are in index currently being built (current pack and all packs saved but not committed)
+	committedContents     *committedContentIndex
 
 	disableIndexFlushCount int
 	flushPackIndexesAfter  time.Time // time when those indexes should be flushed
@@ -98,24 +98,24 @@ type Manager struct {
 	repositoryFormatBytes []byte
 }
 
-// DeleteBlock marks the given blockID as deleted.
+// DeleteContent marks the given contentID as deleted.
 //
-// NOTE: To avoid race conditions only blocks that cannot be possibly re-created
-// should ever be deleted. That means that contents of such blocks should include some element
+// NOTE: To avoid race conditions only contents that cannot be possibly re-created
+// should ever be deleted. That means that contents of such contents should include some element
 // of randomness or a contemporaneous timestamp that will never reappear.
-func (bm *Manager) DeleteBlock(blockID string) error {
+func (bm *Manager) DeleteContent(contentID ID) error {
 	bm.lock()
 	defer bm.unlock()
 
-	log.Debugf("DeleteBlock(%q)", blockID)
+	log.Debugf("DeleteContent(%q)", contentID)
 
-	// We have this block in current pack index and it's already deleted there.
-	if bi, ok := bm.packIndexBuilder[blockID]; ok {
+	// We have this content in current pack index and it's already deleted there.
+	if bi, ok := bm.packIndexBuilder[contentID]; ok {
 		if !bi.Deleted {
 			if bi.PackBlobID == "" {
 				// added and never committed, just forget about it.
-				delete(bm.packIndexBuilder, blockID)
-				delete(bm.currentPackItems, blockID)
+				delete(bm.packIndexBuilder, contentID)
+				delete(bm.currentPackItems, contentID)
 				return nil
 			}
 
@@ -123,13 +123,13 @@ func (bm *Manager) DeleteBlock(blockID string) error {
 			bi2 := *bi
 			bi2.Deleted = true
 			bi2.TimestampSeconds = bm.timeNow().Unix()
-			bm.setPendingBlock(bi2)
+			bm.setPendingContent(bi2)
 		}
 		return nil
 	}
 
-	// We have this block in current pack index and it's already deleted there.
-	bi, err := bm.committedBlocks.getBlock(blockID)
+	// We have this content in current pack index and it's already deleted there.
+	bi, err := bm.committedContents.getContent(contentID)
 	if err != nil {
 		return err
 	}
@@ -143,23 +143,23 @@ func (bm *Manager) DeleteBlock(blockID string) error {
 	bi2 := bi
 	bi2.Deleted = true
 	bi2.TimestampSeconds = bm.timeNow().Unix()
-	bm.setPendingBlock(bi2)
+	bm.setPendingContent(bi2)
 	return nil
 }
 
-func (bm *Manager) setPendingBlock(i Info) {
+func (bm *Manager) setPendingContent(i Info) {
 	bm.packIndexBuilder.Add(i)
-	bm.currentPackItems[i.BlockID] = i
+	bm.currentPackItems[i.ID] = i
 }
 
-func (bm *Manager) addToPackLocked(ctx context.Context, blockID string, data []byte, isDeleted bool) error {
+func (bm *Manager) addToPackLocked(ctx context.Context, contentID ID, data []byte, isDeleted bool) error {
 	bm.assertLocked()
 
 	data = cloneBytes(data)
 	bm.currentPackDataLength += len(data)
-	bm.setPendingBlock(Info{
+	bm.setPendingContent(Info{
 		Deleted:          isDeleted,
-		BlockID:          blockID,
+		ID:               contentID,
 		Payload:          data,
 		Length:           uint32(len(data)),
 		TimestampSeconds: bm.timeNow().Unix(),
@@ -189,7 +189,7 @@ func (bm *Manager) finishPackAndMaybeFlushIndexesLocked(ctx context.Context) err
 	return nil
 }
 
-// Stats returns statistics about block manager operations.
+// Stats returns statistics about content manager operations.
 func (bm *Manager) Stats() Stats {
 	return bm.stats
 }
@@ -225,29 +225,29 @@ func (bm *Manager) verifyInvariantsLocked() {
 
 func (bm *Manager) verifyCurrentPackItemsLocked() {
 	for k, cpi := range bm.currentPackItems {
-		bm.assertInvariant(cpi.BlockID == k, "block ID entry has invalid key: %v %v", cpi.BlockID, k)
-		bm.assertInvariant(cpi.Deleted || cpi.PackBlobID == "", "block ID entry has unexpected pack block ID %v: %v", cpi.BlockID, cpi.PackBlobID)
-		bm.assertInvariant(cpi.TimestampSeconds != 0, "block has no timestamp: %v", cpi.BlockID)
+		bm.assertInvariant(cpi.ID == k, "content ID entry has invalid key: %v %v", cpi.ID, k)
+		bm.assertInvariant(cpi.Deleted || cpi.PackBlobID == "", "content ID entry has unexpected pack content ID %v: %v", cpi.ID, cpi.PackBlobID)
+		bm.assertInvariant(cpi.TimestampSeconds != 0, "content has no timestamp: %v", cpi.ID)
 		bi, ok := bm.packIndexBuilder[k]
-		bm.assertInvariant(ok, "block ID entry not present in pack index builder: %v", cpi.BlockID)
+		bm.assertInvariant(ok, "content ID entry not present in pack index builder: %v", cpi.ID)
 		bm.assertInvariant(reflect.DeepEqual(*bi, cpi), "current pack index does not match pack index builder: %v", cpi, *bi)
 	}
 }
 
 func (bm *Manager) verifyPackIndexBuilderLocked() {
 	for k, cpi := range bm.packIndexBuilder {
-		bm.assertInvariant(cpi.BlockID == k, "block ID entry has invalid key: %v %v", cpi.BlockID, k)
-		if _, ok := bm.currentPackItems[cpi.BlockID]; ok {
-			// ignore blocks also in currentPackItems
+		bm.assertInvariant(cpi.ID == k, "content ID entry has invalid key: %v %v", cpi.ID, k)
+		if _, ok := bm.currentPackItems[cpi.ID]; ok {
+			// ignore contents also in currentPackItems
 			continue
 		}
 		if cpi.Deleted {
-			bm.assertInvariant(cpi.PackBlobID == "", "block can't be both deleted and have a pack block: %v", cpi.BlockID)
+			bm.assertInvariant(cpi.PackBlobID == "", "content can't be both deleted and have a pack content: %v", cpi.ID)
 		} else {
-			bm.assertInvariant(cpi.PackBlobID != "", "block that's not deleted must have a pack block: %+v", cpi)
-			bm.assertInvariant(cpi.FormatVersion == byte(bm.writeFormatVersion), "block that's not deleted must have a valid format version: %+v", cpi)
+			bm.assertInvariant(cpi.PackBlobID != "", "content that's not deleted must have a pack content: %+v", cpi)
+			bm.assertInvariant(cpi.FormatVersion == byte(bm.writeFormatVersion), "content that's not deleted must have a valid format version: %+v", cpi)
 		}
-		bm.assertInvariant(cpi.TimestampSeconds != 0, "block has no timestamp: %v", cpi.BlockID)
+		bm.assertInvariant(cpi.TimestampSeconds != 0, "content has no timestamp: %v", cpi.ID)
 	}
 }
 
@@ -264,7 +264,7 @@ func (bm *Manager) assertInvariant(ok bool, errorMsg string, arg ...interface{})
 }
 
 func (bm *Manager) startPackIndexLocked() {
-	bm.currentPackItems = make(map[string]Info)
+	bm.currentPackItems = make(map[ID]Info)
 	bm.currentPackDataLength = 0
 }
 
@@ -291,8 +291,8 @@ func (bm *Manager) flushPackIndexesLocked(ctx context.Context) error {
 			return err
 		}
 
-		if err := bm.committedBlocks.addBlock(indexBlobID, dataCopy, true); err != nil {
-			return errors.Wrap(err, "unable to add committed block")
+		if err := bm.committedContents.addContent(indexBlobID, dataCopy, true); err != nil {
+			return errors.Wrap(err, "unable to add committed content")
 		}
 		bm.packIndexBuilder = make(packIndexBuilder)
 	}
@@ -302,7 +302,7 @@ func (bm *Manager) flushPackIndexesLocked(ctx context.Context) error {
 }
 
 func (bm *Manager) writePackIndexesNew(ctx context.Context, data []byte) (blob.ID, error) {
-	return bm.encryptAndWriteBlockNotLocked(ctx, data, newIndexBlobPrefix)
+	return bm.encryptAndWriteContentNotLocked(ctx, data, newIndexBlobPrefix)
 }
 
 func (bm *Manager) finishPackLocked(ctx context.Context) error {
@@ -311,36 +311,36 @@ func (bm *Manager) finishPackLocked(ctx context.Context) error {
 		return nil
 	}
 
-	if err := bm.writePackBlockLocked(ctx); err != nil {
-		return errors.Wrap(err, "error writing pack block")
+	if err := bm.writePackContentLocked(ctx); err != nil {
+		return errors.Wrap(err, "error writing pack content")
 	}
 
 	bm.startPackIndexLocked()
 	return nil
 }
 
-func (bm *Manager) writePackBlockLocked(ctx context.Context) error {
+func (bm *Manager) writePackContentLocked(ctx context.Context) error {
 	bm.assertLocked()
 
-	blockID := make([]byte, 16)
-	if _, err := cryptorand.Read(blockID); err != nil {
+	contentID := make([]byte, 16)
+	if _, err := cryptorand.Read(contentID); err != nil {
 		return errors.Wrap(err, "unable to read crypto bytes")
 	}
 
-	packFile := blob.ID(fmt.Sprintf("%v%x", PackBlobIDPrefix, blockID))
+	packFile := blob.ID(fmt.Sprintf("%v%x", PackBlobIDPrefix, contentID))
 
-	blockData, packFileIndex, err := bm.preparePackDataBlock(packFile)
+	contentData, packFileIndex, err := bm.preparePackDataContent(packFile)
 	if err != nil {
-		return errors.Wrap(err, "error preparing data block")
+		return errors.Wrap(err, "error preparing data content")
 	}
 
-	if len(blockData) > 0 {
-		if err := bm.writePackFileNotLocked(ctx, packFile, blockData); err != nil {
-			return errors.Wrap(err, "can't save pack data block")
+	if len(contentData) > 0 {
+		if err := bm.writePackFileNotLocked(ctx, packFile, contentData); err != nil {
+			return errors.Wrap(err, "can't save pack data content")
 		}
 	}
 
-	formatLog.Debugf("wrote pack file: %v (%v bytes)", packFile, len(blockData))
+	formatLog.Debugf("wrote pack file: %v (%v bytes)", packFile, len(contentData))
 	for _, info := range packFileIndex {
 		bm.packIndexBuilder.Add(*info)
 	}
@@ -348,39 +348,39 @@ func (bm *Manager) writePackBlockLocked(ctx context.Context) error {
 	return nil
 }
 
-func (bm *Manager) preparePackDataBlock(packFile blob.ID) ([]byte, packIndexBuilder, error) {
-	formatLog.Debugf("preparing block data with %v items", len(bm.currentPackItems))
+func (bm *Manager) preparePackDataContent(packFile blob.ID) ([]byte, packIndexBuilder, error) {
+	formatLog.Debugf("preparing content data with %v items", len(bm.currentPackItems))
 
-	blockData, err := appendRandomBytes(append([]byte(nil), bm.repositoryFormatBytes...), rand.Intn(bm.maxPreambleLength-bm.minPreambleLength+1)+bm.minPreambleLength)
+	contentData, err := appendRandomBytes(append([]byte(nil), bm.repositoryFormatBytes...), rand.Intn(bm.maxPreambleLength-bm.minPreambleLength+1)+bm.minPreambleLength)
 	if err != nil {
-		return nil, nil, errors.Wrap(err, "unable to prepare block preamble")
+		return nil, nil, errors.Wrap(err, "unable to prepare content preamble")
 	}
 
 	packFileIndex := packIndexBuilder{}
-	for blockID, info := range bm.currentPackItems {
+	for contentID, info := range bm.currentPackItems {
 		if info.Payload == nil {
 			continue
 		}
 
 		var encrypted []byte
-		encrypted, err = bm.maybeEncryptBlockDataForPacking(info.Payload, info.BlockID)
+		encrypted, err = bm.maybeEncryptContentDataForPacking(info.Payload, info.ID)
 		if err != nil {
-			return nil, nil, errors.Wrapf(err, "unable to encrypt %q", blockID)
+			return nil, nil, errors.Wrapf(err, "unable to encrypt %q", contentID)
 		}
 
-		formatLog.Debugf("adding %v length=%v deleted=%v", blockID, len(info.Payload), info.Deleted)
+		formatLog.Debugf("adding %v length=%v deleted=%v", contentID, len(info.Payload), info.Deleted)
 
 		packFileIndex.Add(Info{
-			BlockID:          blockID,
+			ID:               contentID,
 			Deleted:          info.Deleted,
 			FormatVersion:    byte(bm.writeFormatVersion),
 			PackBlobID:       packFile,
-			PackOffset:       uint32(len(blockData)),
+			PackOffset:       uint32(len(contentData)),
 			Length:           uint32(len(encrypted)),
 			TimestampSeconds: info.TimestampSeconds,
 		})
 
-		blockData = append(blockData, encrypted...)
+		contentData = append(contentData, encrypted...)
 	}
 
 	if len(packFileIndex) == 0 {
@@ -388,25 +388,25 @@ func (bm *Manager) preparePackDataBlock(packFile blob.ID) ([]byte, packIndexBuil
 	}
 
 	if bm.paddingUnit > 0 {
-		if missing := bm.paddingUnit - (len(blockData) % bm.paddingUnit); missing > 0 {
-			blockData, err = appendRandomBytes(blockData, missing)
+		if missing := bm.paddingUnit - (len(contentData) % bm.paddingUnit); missing > 0 {
+			contentData, err = appendRandomBytes(contentData, missing)
 			if err != nil {
-				return nil, nil, errors.Wrap(err, "unable to prepare block postamble")
+				return nil, nil, errors.Wrap(err, "unable to prepare content postamble")
 			}
 		}
 	}
 
-	origBlockLength := len(blockData)
-	blockData, err = bm.appendPackFileIndexRecoveryData(blockData, packFileIndex)
+	origContentLength := len(contentData)
+	contentData, err = bm.appendPackFileIndexRecoveryData(contentData, packFileIndex)
 
-	formatLog.Debugf("finished block %v bytes (%v bytes index)", len(blockData), len(blockData)-origBlockLength)
-	return blockData, packFileIndex, err
+	formatLog.Debugf("finished content %v bytes (%v bytes index)", len(contentData), len(contentData)-origContentLength)
+	return contentData, packFileIndex, err
 }
 
-func (bm *Manager) maybeEncryptBlockDataForPacking(data []byte, blockID string) ([]byte, error) {
-	iv, err := getPackedBlockIV(blockID)
+func (bm *Manager) maybeEncryptContentDataForPacking(data []byte, contentID ID) ([]byte, error) {
+	iv, err := getPackedContentIV(contentID)
 	if err != nil {
-		return nil, errors.Wrapf(err, "unable to get packed block IV for %q", blockID)
+		return nil, errors.Wrapf(err, "unable to get packed content IV for %q", contentID)
 	}
 
 	return bm.encryptor.Encrypt(data, iv)
@@ -441,23 +441,23 @@ func (bm *Manager) loadPackIndexesUnlocked(ctx context.Context) ([]IndexBlobInfo
 			nextSleepTime *= 2
 		}
 
-		blocks, err := bm.listCache.listIndexBlobs(ctx)
+		contents, err := bm.listCache.listIndexBlobs(ctx)
 		if err != nil {
 			return nil, false, err
 		}
 
-		err = bm.tryLoadPackIndexBlobsUnlocked(ctx, blocks)
+		err = bm.tryLoadPackIndexBlobsUnlocked(ctx, contents)
 		if err == nil {
-			var blockIDs []blob.ID
-			for _, b := range blocks {
-				blockIDs = append(blockIDs, b.BlobID)
+			var contentIDs []blob.ID
+			for _, b := range contents {
+				contentIDs = append(contentIDs, b.BlobID)
 			}
 			var updated bool
-			updated, err = bm.committedBlocks.use(blockIDs)
+			updated, err = bm.committedContents.use(contentIDs)
 			if err != nil {
 				return nil, false, err
 			}
-			return blocks, updated, nil
+			return contents, updated, nil
 		}
 		if err != blob.ErrBlobNotFound {
 			return nil, false, err
@@ -467,8 +467,8 @@ func (bm *Manager) loadPackIndexesUnlocked(ctx context.Context) ([]IndexBlobInfo
 	return nil, false, errors.Errorf("unable to load pack indexes despite %v retries", indexLoadAttempts)
 }
 
-func (bm *Manager) tryLoadPackIndexBlobsUnlocked(ctx context.Context, blocks []IndexBlobInfo) error {
-	ch, unprocessedIndexesSize, err := bm.unprocessedIndexBlobsUnlocked(blocks)
+func (bm *Manager) tryLoadPackIndexBlobsUnlocked(ctx context.Context, contents []IndexBlobInfo) error {
+	ch, unprocessedIndexesSize, err := bm.unprocessedIndexBlobsUnlocked(contents)
 	if err != nil {
 		return err
 	}
@@ -493,8 +493,8 @@ func (bm *Manager) tryLoadPackIndexBlobsUnlocked(ctx context.Context, blocks []I
 					return
 				}
 
-				if err := bm.committedBlocks.addBlock(indexBlobID, data, false); err != nil {
-					errch <- errors.Wrap(err, "unable to add to committed block cache")
+				if err := bm.committedContents.addContent(indexBlobID, data, false); err != nil {
+					errch <- errors.Wrap(err, "unable to add to committed content cache")
 					return
 				}
 			}
@@ -508,52 +508,52 @@ func (bm *Manager) tryLoadPackIndexBlobsUnlocked(ctx context.Context, blocks []I
 	for err := range errch {
 		return err
 	}
-	log.Infof("Index blocks downloaded.")
+	log.Infof("Index contents downloaded.")
 
 	return nil
 }
 
-// unprocessedIndexBlobsUnlocked returns a closed channel filled with block IDs that are not in committedBlocks cache.
-func (bm *Manager) unprocessedIndexBlobsUnlocked(blocks []IndexBlobInfo) (<-chan blob.ID, int64, error) {
+// unprocessedIndexBlobsUnlocked returns a closed channel filled with content IDs that are not in committedContents cache.
+func (bm *Manager) unprocessedIndexBlobsUnlocked(contents []IndexBlobInfo) (<-chan blob.ID, int64, error) {
 	var totalSize int64
-	ch := make(chan blob.ID, len(blocks))
-	for _, block := range blocks {
-		has, err := bm.committedBlocks.cache.hasIndexBlobID(block.BlobID)
+	ch := make(chan blob.ID, len(contents))
+	for _, c := range contents {
+		has, err := bm.committedContents.cache.hasIndexBlobID(c.BlobID)
 		if err != nil {
 			return nil, 0, err
 		}
 		if has {
-			log.Debugf("index blob %q already in cache, skipping", block.BlobID)
+			log.Debugf("index blob %q already in cache, skipping", c.BlobID)
 			continue
 		}
-		ch <- block.BlobID
-		totalSize += block.Length
+		ch <- c.BlobID
+		totalSize += c.Length
 	}
 	close(ch)
 	return ch, totalSize, nil
 }
 
-// Close closes the block manager.
+// Close closes the content manager.
 func (bm *Manager) Close() {
-	bm.blockCache.close()
+	bm.contentCache.close()
 	close(bm.closed)
 }
 
-// ListBlocks returns IDs of blocks matching given prefix.
-func (bm *Manager) ListBlocks(prefix string) ([]string, error) {
+// ListContents returns IDs of contents matching given prefix.
+func (bm *Manager) ListContents(prefix ID) ([]ID, error) {
 	bm.lock()
 	defer bm.unlock()
 
-	var result []string
+	var result []ID
 
 	appendToResult := func(i Info) error {
-		if i.Deleted || !strings.HasPrefix(i.BlockID, prefix) {
+		if i.Deleted || !strings.HasPrefix(string(i.ID), string(prefix)) {
 			return nil
 		}
-		if bi, ok := bm.packIndexBuilder[i.BlockID]; ok && bi.Deleted {
+		if bi, ok := bm.packIndexBuilder[i.ID]; ok && bi.Deleted {
 			return nil
 		}
-		result = append(result, i.BlockID)
+		result = append(result, i.ID)
 		return nil
 	}
 
@@ -561,22 +561,22 @@ func (bm *Manager) ListBlocks(prefix string) ([]string, error) {
 		_ = appendToResult(*bi)
 	}
 
-	_ = bm.committedBlocks.listBlocks(prefix, appendToResult)
+	_ = bm.committedContents.listContents(prefix, appendToResult)
 	return result, nil
 }
 
-// ListBlockInfos returns the metadata about blocks with a given prefix and kind.
-func (bm *Manager) ListBlockInfos(prefix string, includeDeleted bool) ([]Info, error) {
+// ListContentInfos returns the metadata about contents with a given prefix and kind.
+func (bm *Manager) ListContentInfos(prefix ID, includeDeleted bool) ([]Info, error) {
 	bm.lock()
 	defer bm.unlock()
 
 	var result []Info
 
 	appendToResult := func(i Info) error {
-		if (i.Deleted && !includeDeleted) || !strings.HasPrefix(i.BlockID, prefix) {
+		if (i.Deleted && !includeDeleted) || !strings.HasPrefix(string(i.ID), string(prefix)) {
 			return nil
 		}
-		if bi, ok := bm.packIndexBuilder[i.BlockID]; ok && bi.Deleted {
+		if bi, ok := bm.packIndexBuilder[i.ID]; ok && bi.Deleted {
 			return nil
 		}
 		result = append(result, i)
@@ -587,7 +587,7 @@ func (bm *Manager) ListBlockInfos(prefix string, includeDeleted bool) ([]Info, e
 		_ = appendToResult(*bi)
 	}
 
-	_ = bm.committedBlocks.listBlocks(prefix, appendToResult)
+	_ = bm.committedContents.listContents(prefix, appendToResult)
 
 	return result, nil
 }
@@ -598,7 +598,7 @@ func (bm *Manager) Flush(ctx context.Context) error {
 	defer bm.unlock()
 
 	if err := bm.finishPackLocked(ctx); err != nil {
-		return errors.Wrap(err, "error writing pending block")
+		return errors.Wrap(err, "error writing pending content")
 	}
 
 	if err := bm.flushPackIndexesLocked(ctx); err != nil {
@@ -608,46 +608,46 @@ func (bm *Manager) Flush(ctx context.Context) error {
 	return nil
 }
 
-// RewriteBlock causes reads and re-writes a given block using the most recent format.
-func (bm *Manager) RewriteBlock(ctx context.Context, blockID string) error {
-	bi, err := bm.getBlockInfo(blockID)
+// RewriteContent causes reads and re-writes a given content using the most recent format.
+func (bm *Manager) RewriteContent(ctx context.Context, contentID ID) error {
+	bi, err := bm.getContentInfo(contentID)
 	if err != nil {
 		return err
 	}
 
-	data, err := bm.getBlockContentsUnlocked(ctx, bi)
+	data, err := bm.getContentContentsUnlocked(ctx, bi)
 	if err != nil {
 		return err
 	}
 
 	bm.lock()
 	defer bm.unlock()
-	return bm.addToPackLocked(ctx, blockID, data, bi.Deleted)
+	return bm.addToPackLocked(ctx, contentID, data, bi.Deleted)
 }
 
-// WriteBlock saves a given block of data to a pack group with a provided name and returns a blockID
+// WriteContent saves a given content of data to a pack group with a provided name and returns a contentID
 // that's based on the contents of data written.
-func (bm *Manager) WriteBlock(ctx context.Context, data []byte, prefix string) (string, error) {
+func (bm *Manager) WriteContent(ctx context.Context, data []byte, prefix ID) (ID, error) {
 	if err := validatePrefix(prefix); err != nil {
 		return "", err
 	}
-	blockID := prefix + hex.EncodeToString(bm.hashData(data))
+	contentID := prefix + ID(hex.EncodeToString(bm.hashData(data)))
 
-	// block already tracked
-	if bi, err := bm.getBlockInfo(blockID); err == nil {
+	// content already tracked
+	if bi, err := bm.getContentInfo(contentID); err == nil {
 		if !bi.Deleted {
-			return blockID, nil
+			return contentID, nil
 		}
 	}
 
-	log.Debugf("WriteBlock(%q) - new", blockID)
+	log.Debugf("WriteContent(%q) - new", contentID)
 	bm.lock()
 	defer bm.unlock()
-	err := bm.addToPackLocked(ctx, blockID, data, false)
-	return blockID, err
+	err := bm.addToPackLocked(ctx, contentID, data, false)
+	return contentID, err
 }
 
-func validatePrefix(prefix string) error {
+func validatePrefix(prefix ID) error {
 	switch len(prefix) {
 	case 0:
 		return nil
@@ -661,24 +661,24 @@ func validatePrefix(prefix string) error {
 }
 
 func (bm *Manager) writePackFileNotLocked(ctx context.Context, packFile blob.ID, data []byte) error {
-	atomic.AddInt32(&bm.stats.WrittenBlocks, 1)
+	atomic.AddInt32(&bm.stats.WrittenContents, 1)
 	atomic.AddInt64(&bm.stats.WrittenBytes, int64(len(data)))
 	bm.listCache.deleteListCache(ctx)
 	return bm.st.PutBlob(ctx, packFile, data)
 }
 
-func (bm *Manager) encryptAndWriteBlockNotLocked(ctx context.Context, data []byte, prefix blob.ID) (blob.ID, error) {
+func (bm *Manager) encryptAndWriteContentNotLocked(ctx context.Context, data []byte, prefix blob.ID) (blob.ID, error) {
 	hash := bm.hashData(data)
 	blobID := prefix + blob.ID(hex.EncodeToString(hash))
 
-	// Encrypt the block in-place.
+	// Encrypt the content in-place.
 	atomic.AddInt64(&bm.stats.EncryptedBytes, int64(len(data)))
 	data2, err := bm.encryptor.Encrypt(data, hash)
 	if err != nil {
 		return "", err
 	}
 
-	atomic.AddInt32(&bm.stats.WrittenBlocks, 1)
+	atomic.AddInt32(&bm.stats.WrittenContents, 1)
 	atomic.AddInt64(&bm.stats.WrittenBytes, int64(len(data2)))
 	bm.listCache.deleteListCache(ctx)
 	if err := bm.st.PutBlob(ctx, blobID, data2); err != nil {
@@ -689,80 +689,80 @@ func (bm *Manager) encryptAndWriteBlockNotLocked(ctx context.Context, data []byt
 }
 
 func (bm *Manager) hashData(data []byte) []byte {
-	// Hash the block and compute encryption key.
-	blockID := bm.hasher(data)
-	atomic.AddInt32(&bm.stats.HashedBlocks, 1)
+	// Hash the content and compute encryption key.
+	contentID := bm.hasher(data)
+	atomic.AddInt32(&bm.stats.HashedContents, 1)
 	atomic.AddInt64(&bm.stats.HashedBytes, int64(len(data)))
-	return blockID
+	return contentID
 }
 
 func cloneBytes(b []byte) []byte {
 	return append([]byte{}, b...)
 }
 
-// GetBlock gets the contents of a given block. If the block is not found returns blob.ErrBlobNotFound.
-func (bm *Manager) GetBlock(ctx context.Context, blockID string) ([]byte, error) {
-	bi, err := bm.getBlockInfo(blockID)
+// GetContent gets the contents of a given content. If the content is not found returns ErrContentNotFound.
+func (bm *Manager) GetContent(ctx context.Context, contentID ID) ([]byte, error) {
+	bi, err := bm.getContentInfo(contentID)
 	if err != nil {
 		return nil, err
 	}
 
 	if bi.Deleted {
-		return nil, ErrBlockNotFound
+		return nil, ErrContentNotFound
 	}
 
-	return bm.getBlockContentsUnlocked(ctx, bi)
+	return bm.getContentContentsUnlocked(ctx, bi)
 }
 
-func (bm *Manager) getBlockInfo(blockID string) (Info, error) {
+func (bm *Manager) getContentInfo(contentID ID) (Info, error) {
 	bm.lock()
 	defer bm.unlock()
 
-	// check added blocks, not written to any packs.
-	if bi, ok := bm.currentPackItems[blockID]; ok {
+	// check added contents, not written to any packs.
+	if bi, ok := bm.currentPackItems[contentID]; ok {
 		return bi, nil
 	}
 
-	// added blocks, written to packs but not yet added to indexes
-	if bi, ok := bm.packIndexBuilder[blockID]; ok {
+	// added contents, written to packs but not yet added to indexes
+	if bi, ok := bm.packIndexBuilder[contentID]; ok {
 		return *bi, nil
 	}
 
-	// read from committed block index
-	return bm.committedBlocks.getBlock(blockID)
+	// read from committed content index
+	return bm.committedContents.getContent(contentID)
 }
 
-// BlockInfo returns information about a single block.
-func (bm *Manager) BlockInfo(ctx context.Context, blockID string) (Info, error) {
-	bi, err := bm.getBlockInfo(blockID)
+// ContentInfo returns information about a single content.
+func (bm *Manager) ContentInfo(ctx context.Context, contentID ID) (Info, error) {
+	bi, err := bm.getContentInfo(contentID)
 	if err != nil {
-		log.Debugf("BlockInfo(%q) - error %v", err)
+		log.Debugf("ContentInfo(%q) - error %v", err)
 		return Info{}, err
 	}
 
 	if bi.Deleted {
-		log.Debugf("BlockInfo(%q) - deleted", blockID)
+		log.Debugf("ContentInfo(%q) - deleted", contentID)
 	} else {
-		log.Debugf("BlockInfo(%q) - exists in %v", blockID, bi.PackBlobID)
+		log.Debugf("ContentInfo(%q) - exists in %v", contentID, bi.PackBlobID)
 	}
 
 	return bi, err
 }
 
-// FindUnreferencedBlobs returns the list of unreferenced storage blocks.
+// FindUnreferencedBlobs returns the list of unreferenced storage contents.
 func (bm *Manager) FindUnreferencedBlobs(ctx context.Context) ([]blob.Metadata, error) {
-	infos, err := bm.ListBlockInfos("", true)
+	infos, err := bm.ListContentInfos("", true)
 	if err != nil {
 		return nil, errors.Wrap(err, "unable to list index blobs")
 	}
 
-	usedPackBlocks := findPackBlocksInUse(infos)
+	usedPackContents := findPackContentsInUse(infos)
 
 	var unused []blob.Metadata
 	err = bm.st.ListBlobs(ctx, PackBlobIDPrefix, func(bi blob.Metadata) error {
-		u := usedPackBlocks[bi.BlobID]
+		u := usedPackContents[bi.BlobID]
 		if u > 0 {
-			log.Debugf("pack %v, in use by %v blocks", bi.BlobID, u)
+			log.Debugf("pack %v, in use by %v contents", bi.BlobID, u)
 			return nil
 		}
 
@@ -770,13 +770,13 @@ func (bm *Manager) FindUnreferencedBlobs(ctx context.Context) ([]blob.Metadata,
 		return nil
 	})
 	if err != nil {
-		return nil, errors.Wrap(err, "error listing storage blocks")
+		return nil, errors.Wrap(err, "error listing storage contents")
 	}
 
 	return unused, nil
 }
 
-func findPackBlocksInUse(infos []Info) map[blob.ID]int {
+func findPackContentsInUse(infos []Info) map[blob.ID]int {
 	packUsage := map[blob.ID]int{}
 
 	for _, bi := range infos {
@@ -786,20 +786,20 @@ func findPackBlocksInUse(infos []Info) map[blob.ID]int {
 	return packUsage
 }
 
-func (bm *Manager) getBlockContentsUnlocked(ctx context.Context, bi Info) ([]byte, error) {
+func (bm *Manager) getContentContentsUnlocked(ctx context.Context, bi Info) ([]byte, error) {
 	if bi.Payload != nil {
 		return cloneBytes(bi.Payload), nil
 	}
 
-	payload, err := bm.blockCache.getContentBlock(ctx, blob.ID(bi.BlockID), bi.PackBlobID, int64(bi.PackOffset), int64(bi.Length))
+	payload, err := bm.contentCache.getContentContent(ctx, blob.ID(bi.ID), bi.PackBlobID, int64(bi.PackOffset), int64(bi.Length))
 	if err != nil {
 		return nil, err
 	}
 
-	atomic.AddInt32(&bm.stats.ReadBlocks, 1)
+	atomic.AddInt32(&bm.stats.ReadContents, 1)
 	atomic.AddInt64(&bm.stats.ReadBytes, int64(len(payload)))
 
-	iv, err := getPackedBlockIV(bi.BlockID)
+	iv, err := getPackedContentIV(bi.ID)
 	if err != nil {
 		return nil, err
 	}
@@ -831,7 +831,7 @@ func (bm *Manager) decryptAndVerify(encrypted []byte, iv []byte) ([]byte, error)
 }
 
 func (bm *Manager) getIndexBlobInternal(ctx context.Context, blobID blob.ID) ([]byte, error) {
-	payload, err := bm.blockCache.getContentBlock(ctx, blobID, blobID, 0, -1)
+	payload, err := bm.contentCache.getContentContent(ctx, blobID, blobID, 0, -1)
 	if err != nil {
 		return nil, err
 	}
@@ -841,7 +841,7 @@ func (bm *Manager) getIndexBlobInternal(ctx context.Context, blobID blob.ID) ([]
 		return nil, err
 	}
 
-	atomic.AddInt32(&bm.stats.ReadBlocks, 1)
+	atomic.AddInt32(&bm.stats.ReadContents, 1)
 	atomic.AddInt64(&bm.stats.ReadBytes, int64(len(payload)))
 
 	payload, err = bm.encryptor.Decrypt(payload, iv)
@@ -859,8 +859,8 @@ func (bm *Manager) getIndexBlobInternal(ctx context.Context, blobID blob.ID) ([]
 	return payload, nil
 }
 
-func getPackedBlockIV(blockID string) ([]byte, error) {
-	return hex.DecodeString(blockID[len(blockID)-(aes.BlockSize*2):])
+func getPackedContentIV(contentID ID) ([]byte, error) {
+	return hex.DecodeString(string(contentID[len(contentID)-(aes.BlockSize*2):]))
 }
 
 func getIndexBlobIV(s blob.ID) ([]byte, error) {
@@ -870,15 +870,15 @@ func getIndexBlobIV(s blob.ID) ([]byte, error) {
 	return hex.DecodeString(string(s[len(s)-(aes.BlockSize*2):]))
 }
 
-func (bm *Manager) verifyChecksum(data []byte, blockID []byte) error {
+func (bm *Manager) verifyChecksum(data []byte, contentID []byte) error {
 	expected := bm.hasher(data)
 	expected = expected[len(expected)-aes.BlockSize:]
-	if !bytes.HasSuffix(blockID, expected) {
-		atomic.AddInt32(&bm.stats.InvalidBlocks, 1)
-		return errors.Errorf("invalid checksum for blob %x, expected %x", blockID, expected)
+	if !bytes.HasSuffix(contentID, expected) {
+		atomic.AddInt32(&bm.stats.InvalidContents, 1)
+		return errors.Errorf("invalid checksum for blob %x, expected %x", contentID, expected)
 	}
 
-	atomic.AddInt32(&bm.stats.ValidBlocks, 1)
+	atomic.AddInt32(&bm.stats.ValidContents, 1)
 	return nil
 }
 
@@ -902,7 +902,7 @@ func (bm *Manager) assertLocked() {
 	}
 }
 
-// Refresh reloads the committed block indexes.
+// Refresh reloads the committed content indexes.
 func (bm *Manager) Refresh(ctx context.Context) (bool, error) {
 	bm.mu.Lock()
 	defer bm.mu.Unlock()
@@ -916,11 +916,11 @@ func (bm *Manager) Refresh(ctx context.Context) (bool, error) {
 
 type cachedList struct {
 	Timestamp time.Time       `json:"timestamp"`
-	Blocks    []IndexBlobInfo `json:"blocks"`
+	Contents  []IndexBlobInfo `json:"contents"`
 }
 
 // listIndexBlobsFromStorage returns the list of index blobs in the given storage.
-// The list of blocks is not guaranteed to be sorted.
+// The list of contents is not guaranteed to be sorted.
 func listIndexBlobsFromStorage(ctx context.Context, st blob.Storage) ([]IndexBlobInfo, error) {
 	snapshot, err := blob.ListAllBlobsConsistent(ctx, st, newIndexBlobPrefix, math.MaxInt32)
 	if err != nil {
@@ -940,7 +940,7 @@ func listIndexBlobsFromStorage(ctx context.Context, st blob.Storage) ([]IndexBlo
 	return results, err
 }
 
-// NewManager creates new block manager with given packing options and a formatter.
+// NewManager creates new content manager with given packing options and a formatter.
 func NewManager(ctx context.Context, st blob.Storage, f FormattingOptions, caching CachingOptions, repositoryFormatBytes []byte) (*Manager, error) {
 	return newManagerWithOptions(ctx, st, f, caching, time.Now, repositoryFormatBytes)
 }
@@ -959,9 +959,9 @@ func newManagerWithOptions(ctx context.Context, st blob.Storage, f FormattingOpt
 		return nil, err
 	}
 
-	blockCache, err := newBlockCache(ctx, st, caching)
+	contentCache, err := newContentCache(ctx, st, caching)
 	if err != nil {
-		return nil, errors.Wrap(err, "unable to initialize block cache")
+		return nil, errors.Wrap(err, "unable to initialize content cache")
 	}
 
 	listCache, err := newListCache(ctx, st, caching)
@@ -969,9 +969,9 @@ func newManagerWithOptions(ctx context.Context, st blob.Storage, f FormattingOpt
 		return nil, errors.Wrap(err, "unable to initialize list cache")
 	}
 
-	blockIndex, err := newCommittedBlockIndex(caching)
+	contentIndex, err := newCommittedContentIndex(caching)
 	if err != nil {
-		return nil, errors.Wrap(err, "unable to initialize committed block index")
+		return nil, errors.Wrap(err, "unable to initialize committed content index")
 	}
 
 	m := &Manager{
@@ -981,13 +981,13 @@ func newManagerWithOptions(ctx context.Context, st blob.Storage, f FormattingOpt
 		maxPackSize:           f.MaxPackSize,
 		encryptor:             encryptor,
 		hasher:                hasher,
-		currentPackItems:      make(map[string]Info),
+		currentPackItems:      make(map[ID]Info),
 		packIndexBuilder:      make(packIndexBuilder),
-		committedBlocks:       blockIndex,
+		committedContents:     contentIndex,
 		minPreambleLength:     defaultMinPreambleLength,
 		maxPreambleLength:     defaultMaxPreambleLength,
 		paddingUnit:           defaultPaddingUnit,
-		blockCache:            blockCache,
+		contentCache:          contentCache,
 		listCache:             listCache,
 		st:                    st,
 		repositoryFormatBytes: repositoryFormatBytes,
@@ -1000,7 +1000,7 @@ func newManagerWithOptions(ctx context.Context, st blob.Storage, f FormattingOpt
 	m.startPackIndexLocked()
 
 	if err := m.CompactIndexes(ctx, autoCompactionOptions); err != nil {
-		return nil, errors.Wrap(err, "error initializing block manager")
+		return nil, errors.Wrap(err, "error initializing content manager")
 	}
 
 	return m, nil
@@ -1017,8 +1017,8 @@ func CreateHashAndEncryptor(f FormattingOptions) (HashFunc, Encryptor, error) {
 		return nil, nil, errors.Wrap(err, "unable to create encryptor")
 	}
 
-	blockID := h(nil)
-	_, err = e.Encrypt(nil, blockID)
+	contentID := h(nil)
+	_, err = e.Encrypt(nil, contentID)
 	if err != nil {
 		return nil, nil, errors.Wrap(err, "invalid encryptor")
 	}

repo/content/content_manager_test.go

@@ -0,0 +1,911 @@
+package content
+
+import (
+	"bytes"
+	"context"
+	"crypto/hmac"
+	"crypto/sha256"
+	"encoding/hex"
+	"errors"
+	"fmt"
+	"math/rand"
+	"reflect"
+	"strings"
+	"sync"
+	"testing"
+	"time"
+
+	logging "github.com/op/go-logging"
+
+	"github.com/kopia/kopia/internal/blobtesting"
+	"github.com/kopia/kopia/repo/blob"
+)
+
+const (
+	maxPackSize = 2000
+)
+
+var fakeTime = time.Date(2017, 1, 1, 0, 0, 0, 0, time.UTC)
+var hmacSecret = []byte{1, 2, 3}
+
+func init() {
+	logging.SetLevel(logging.DEBUG, "")
+}
+
+func TestContentManagerEmptyFlush(t *testing.T) {
+	ctx := context.Background()
+	data := blobtesting.DataMap{}
+	keyTime := map[blob.ID]time.Time{}
+	bm := newTestContentManager(data, keyTime, nil)
+	bm.Flush(ctx)
+	if got, want := len(data), 0; got != want {
+		t.Errorf("unexpected number of contents: %v, wanted %v", got, want)
+	}
+}
+
+func TestContentZeroBytes1(t *testing.T) {
+	ctx := context.Background()
+	data := blobtesting.DataMap{}
+	keyTime := map[blob.ID]time.Time{}
+	bm := newTestContentManager(data, keyTime, nil)
+	contentID := writeContentAndVerify(ctx, t, bm, []byte{})
+	bm.Flush(ctx)
+	if got, want := len(data), 2; got != want {
+		t.Errorf("unexpected number of contents: %v, wanted %v", got, want)
+	}
+	dumpContentManagerData(t, data)
+	bm = newTestContentManager(data, keyTime, nil)
+	verifyContent(ctx, t, bm, contentID, []byte{})
+}
+
+func TestContentZeroBytes2(t *testing.T) {
+	ctx := context.Background()
+	data := blobtesting.DataMap{}
+	keyTime := map[blob.ID]time.Time{}
+	bm := newTestContentManager(data, keyTime, nil)
+	writeContentAndVerify(ctx, t, bm, seededRandomData(10, 10))
+	writeContentAndVerify(ctx, t, bm, []byte{})
+	bm.Flush(ctx)
+	if got, want := len(data), 2; got != want {
+		t.Errorf("unexpected number of contents: %v, wanted %v", got, want)
+		dumpContentManagerData(t, data)
+	}
+}
+
+func TestContentManagerSmallContentWrites(t *testing.T) {
+	ctx := context.Background()
+	data := blobtesting.DataMap{}
+	keyTime := map[blob.ID]time.Time{}
+	bm := newTestContentManager(data, keyTime, nil)
+
+	for i := 0; i < 100; i++ {
+		writeContentAndVerify(ctx, t, bm, seededRandomData(i, 10))
+	}
+	if got, want := len(data), 0; got != want {
+		t.Errorf("unexpected number of contents: %v, wanted %v", got, want)
+	}
+	bm.Flush(ctx)
+	if got, want := len(data), 2; got != want {
+		t.Errorf("unexpected number of contents: %v, wanted %v", got, want)
+	}
+}
+
+func TestContentManagerDedupesPendingContents(t *testing.T) {
+	ctx := context.Background()
+	data := blobtesting.DataMap{}
+	keyTime := map[blob.ID]time.Time{}
+	bm := newTestContentManager(data, keyTime, nil)
+
+	for i := 0; i < 100; i++ {
+		writeContentAndVerify(ctx, t, bm, seededRandomData(0, 999))
+	}
+	if got, want := len(data), 0; got != want {
+		t.Errorf("unexpected number of contents: %v, wanted %v", got, want)
+	}
+	bm.Flush(ctx)
+	if got, want := len(data), 2; got != want {
+		t.Errorf("unexpected number of contents: %v, wanted %v", got, want)
+	}
+}
+
+func TestContentManagerDedupesPendingAndUncommittedContents(t *testing.T) {
+	ctx := context.Background()
+	data := blobtesting.DataMap{}
+	keyTime := map[blob.ID]time.Time{}
+	bm := newTestContentManager(data, keyTime, nil)
+
+	// no writes here, all data fits in a single pack.
+	writeContentAndVerify(ctx, t, bm, seededRandomData(0, 950))
+	writeContentAndVerify(ctx, t, bm, seededRandomData(1, 950))
+	writeContentAndVerify(ctx, t, bm, seededRandomData(2, 10))
+	if got, want := len(data), 0; got != want {
+		t.Errorf("unexpected number of contents: %v, wanted %v", got, want)
+	}
+
+	// no writes here
+	writeContentAndVerify(ctx, t, bm, seededRandomData(0, 950))
+	writeContentAndVerify(ctx, t, bm, seededRandomData(1, 950))
+	writeContentAndVerify(ctx, t, bm, seededRandomData(2, 10))
+	if got, want := len(data), 0; got != want {
+		t.Errorf("unexpected number of contents: %v, wanted %v", got, want)
+	}
+	bm.Flush(ctx)
+
+	// this flushes the pack content + index blob
+	if got, want := len(data), 2; got != want {
+		dumpContentManagerData(t, data)
+		t.Errorf("unexpected number of contents: %v, wanted %v", got, want)
+	}
+}
+
+func TestContentManagerEmpty(t *testing.T) {
+	ctx := context.Background()
+	data := blobtesting.DataMap{}
+	keyTime := map[blob.ID]time.Time{}
+	bm := newTestContentManager(data, keyTime, nil)
+
+	noSuchContentID := ID(hashValue([]byte("foo")))
+
+	b, err := bm.GetContent(ctx, noSuchContentID)
+	if err != ErrContentNotFound {
+		t.Errorf("unexpected error when getting non-existent content: %v, %v", b, err)
+	}
+
+	bi, err := bm.ContentInfo(ctx, noSuchContentID)
+	if err != ErrContentNotFound {
+		t.Errorf("unexpected error when getting non-existent content info: %v, %v", bi, err)
+	}
+
+	if got, want := len(data), 0; got != want {
+		t.Errorf("unexpected number of contents: %v, wanted %v", got, want)
+	}
+}
+
+func verifyActiveIndexBlobCount(ctx context.Context, t *testing.T, bm *Manager, expected int) {
+	t.Helper()
+
+	blks, err := bm.IndexBlobs(ctx)
+	if err != nil {
+		t.Errorf("error listing active index blobs: %v", err)
+		return
+	}
+
+	if got, want := len(blks), expected; got != want {
+		t.Errorf("unexpected number of active index blobs %v, expected %v (%v)", got, want, blks)
+	}
+}
+func TestContentManagerInternalFlush(t *testing.T) {
+	ctx := context.Background()
+	data := blobtesting.DataMap{}
+	keyTime := map[blob.ID]time.Time{}
+	bm := newTestContentManager(data, keyTime, nil)
+
+	for i := 0; i < 100; i++ {
+		b := make([]byte, 25)
+		rand.Read(b)
+		writeContentAndVerify(ctx, t, bm, b)
+	}
+
+	// 1 data content written, but no index yet.
+	if got, want := len(data), 1; got != want {
+		t.Errorf("unexpected number of contents: %v, wanted %v", got, want)
+	}
+
+	// do it again - should be 2 contents + 1000 bytes pending.
+	for i := 0; i < 100; i++ {
+		b := make([]byte, 25)
+		rand.Read(b)
+		writeContentAndVerify(ctx, t, bm, b)
+	}
+
+	// 2 data contents written, but no index yet.
+	if got, want := len(data), 2; got != want {
+		t.Errorf("unexpected number of contents: %v, wanted %v", got, want)
+	}
+
+	bm.Flush(ctx)
+
+	// third content gets written, followed by index.
+	if got, want := len(data), 4; got != want {
+		dumpContentManagerData(t, data)
+		t.Errorf("unexpected number of contents: %v, wanted %v", got, want)
+	}
+}
+
+func TestContentManagerWriteMultiple(t *testing.T) {
+	ctx := context.Background()
+	data := blobtesting.DataMap{}
+	keyTime := map[blob.ID]time.Time{}
+	timeFunc := fakeTimeNowWithAutoAdvance(fakeTime, 1*time.Second)
+	bm := newTestContentManager(data, keyTime, timeFunc)
+
+	var contentIDs []ID
+
+	for i := 0; i < 5000; i++ {
+		//t.Logf("i=%v", i)
+		b := seededRandomData(i, i%113)
+		blkID, err := bm.WriteContent(ctx, b, "")
+		if err != nil {
+			t.Errorf("err: %v", err)
+		}
+
+		contentIDs = append(contentIDs, blkID)
+
+		if i%17 == 0 {
+			//t.Logf("flushing %v", i)
+			if err := bm.Flush(ctx); err != nil {
+				t.Fatalf("error flushing: %v", err)
+			}
+			//dumpContentManagerData(t, data)
+		}
+
+		if i%41 == 0 {
+			//t.Logf("opening new manager: %v", i)
+			if err := bm.Flush(ctx); err != nil {
+				t.Fatalf("error flushing: %v", err)
+			}
+			//t.Logf("data content count: %v", len(data))
+			//dumpContentManagerData(t, data)
+			bm = newTestContentManager(data, keyTime, timeFunc)
+		}
+
+		pos := rand.Intn(len(contentIDs))
+		if _, err := bm.GetContent(ctx, contentIDs[pos]); err != nil {
+			dumpContentManagerData(t, data)
+			t.Fatalf("can't read content %q: %v", contentIDs[pos], err)
+			continue
+		}
+	}
+}
+
+// This is regression test for a bug where we would corrupt data when encryption
+// was done in place and clobbered pending data in memory.
+func TestContentManagerFailedToWritePack(t *testing.T) {
+	ctx := context.Background()
+	data := blobtesting.DataMap{}
+	keyTime := map[blob.ID]time.Time{}
+	st := blobtesting.NewMapStorage(data, keyTime, nil)
+	faulty := &blobtesting.FaultyStorage{
+		Base: st,
+	}
+	st = faulty
+
+	bm, err := newManagerWithOptions(context.Background(), st, FormattingOptions{
+		Version:     1,
+		Hash:        "HMAC-SHA256-128",
+		Encryption:  "AES-256-CTR",
+		MaxPackSize: maxPackSize,
+		HMACSecret:  []byte("foo"),
+		MasterKey:   []byte("0123456789abcdef0123456789abcdef"),
+	}, CachingOptions{}, fakeTimeNowFrozen(fakeTime), nil)
+	if err != nil {
+		t.Fatalf("can't create bm: %v", err)
+	}
+	logging.SetLevel(logging.DEBUG, "faulty-storage")
+
+	faulty.Faults = map[string][]*blobtesting.Fault{
+		"PutContent": {
+			{Err: errors.New("booboo")},
+		},
+	}
+
+	b1, err := bm.WriteContent(ctx, seededRandomData(1, 10), "")
+	if err != nil {
+		t.Fatalf("can't create content: %v", err)
+	}
+
+	if err := bm.Flush(ctx); err != nil {
+		t.Logf("expected flush error: %v", err)
+	}
+
+	verifyContent(ctx, t, bm, b1, seededRandomData(1, 10))
+}
+
+func TestContentManagerConcurrency(t *testing.T) {
+	ctx := context.Background()
+	data := blobtesting.DataMap{}
+	keyTime := map[blob.ID]time.Time{}
+	bm := newTestContentManager(data, keyTime, nil)
+	preexistingContent := writeContentAndVerify(ctx, t, bm, seededRandomData(10, 100))
+	bm.Flush(ctx)
+
+	dumpContentManagerData(t, data)
+	bm1 := newTestContentManager(data, keyTime, nil)
+	bm2 := newTestContentManager(data, keyTime, nil)
+	bm3 := newTestContentManager(data, keyTime, fakeTimeNowWithAutoAdvance(fakeTime.Add(1), 1*time.Second))
+
+	// all bm* can see pre-existing content
+	verifyContent(ctx, t, bm1, preexistingContent, seededRandomData(10, 100))
+	verifyContent(ctx, t, bm2, preexistingContent, seededRandomData(10, 100))
+	verifyContent(ctx, t, bm3, preexistingContent, seededRandomData(10, 100))
+
+	// write the same content in all managers.
+	sharedContent := writeContentAndVerify(ctx, t, bm1, seededRandomData(20, 100))
+	writeContentAndVerify(ctx, t, bm2, seededRandomData(20, 100))
+	writeContentAndVerify(ctx, t, bm3, seededRandomData(20, 100))
+
+	// write unique content per manager.
+	bm1content := writeContentAndVerify(ctx, t, bm1, seededRandomData(31, 100))
+	bm2content := writeContentAndVerify(ctx, t, bm2, seededRandomData(32, 100))
+	bm3content := writeContentAndVerify(ctx, t, bm3, seededRandomData(33, 100))
+
+	// make sure they can't see each other's unflushed contents.
+	verifyContentNotFound(ctx, t, bm1, bm2content)
+	verifyContentNotFound(ctx, t, bm1, bm3content)
+	verifyContentNotFound(ctx, t, bm2, bm1content)
+	verifyContentNotFound(ctx, t, bm2, bm3content)
+	verifyContentNotFound(ctx, t, bm3, bm1content)
+	verifyContentNotFound(ctx, t, bm3, bm2content)
+
+	// now flush all writers, they still can't see each others' data.
+	bm1.Flush(ctx)
+	bm2.Flush(ctx)
+	bm3.Flush(ctx)
+	verifyContentNotFound(ctx, t, bm1, bm2content)
+	verifyContentNotFound(ctx, t, bm1, bm3content)
+	verifyContentNotFound(ctx, t, bm2, bm1content)
+	verifyContentNotFound(ctx, t, bm2, bm3content)
+	verifyContentNotFound(ctx, t, bm3, bm1content)
+	verifyContentNotFound(ctx, t, bm3, bm2content)
+
+	// new content manager at this point can see all data.
+	bm4 := newTestContentManager(data, keyTime, nil)
+	verifyContent(ctx, t, bm4, preexistingContent, seededRandomData(10, 100))
+	verifyContent(ctx, t, bm4, sharedContent, seededRandomData(20, 100))
+	verifyContent(ctx, t, bm4, bm1content, seededRandomData(31, 100))
+	verifyContent(ctx, t, bm4, bm2content, seededRandomData(32, 100))
+	verifyContent(ctx, t, bm4, bm3content, seededRandomData(33, 100))
+
+	if got, want := getIndexCount(data), 4; got != want {
+		t.Errorf("unexpected index count before compaction: %v, wanted %v", got, want)
+	}
+
+	if err := bm4.CompactIndexes(ctx, CompactOptions{
+		MinSmallBlobs: 1,
+		MaxSmallBlobs: 1,
+	}); err != nil {
+		t.Errorf("compaction error: %v", err)
+	}
+	if got, want := getIndexCount(data), 1; got != want {
+		t.Errorf("unexpected index count after compaction: %v, wanted %v", got, want)
+	}
+
+	// new content manager at this point can see all data.
+	bm5 := newTestContentManager(data, keyTime, nil)
+	verifyContent(ctx, t, bm5, preexistingContent, seededRandomData(10, 100))
+	verifyContent(ctx, t, bm5, sharedContent, seededRandomData(20, 100))
+	verifyContent(ctx, t, bm5, bm1content, seededRandomData(31, 100))
+	verifyContent(ctx, t, bm5, bm2content, seededRandomData(32, 100))
+	verifyContent(ctx, t, bm5, bm3content, seededRandomData(33, 100))
+	if err := bm5.CompactIndexes(ctx, CompactOptions{
+		MinSmallBlobs: 1,
+		MaxSmallBlobs: 1,
+	}); err != nil {
+		t.Errorf("compaction error: %v", err)
+	}
+}
+
+func TestDeleteContent(t *testing.T) {
+	ctx := context.Background()
+	data := blobtesting.DataMap{}
+	keyTime := map[blob.ID]time.Time{}
+	bm := newTestContentManager(data, keyTime, nil)
+	content1 := writeContentAndVerify(ctx, t, bm, seededRandomData(10, 100))
+	bm.Flush(ctx)
+	content2 := writeContentAndVerify(ctx, t, bm, seededRandomData(11, 100))
+	if err := bm.DeleteContent(content1); err != nil {
+		t.Errorf("unable to delete content: %v", content1)
+	}
+	if err := bm.DeleteContent(content2); err != nil {
+		t.Errorf("unable to delete content: %v", content1)
+	}
+	verifyContentNotFound(ctx, t, bm, content1)
+	verifyContentNotFound(ctx, t, bm, content2)
+	bm.Flush(ctx)
+	log.Debugf("-----------")
+	bm = newTestContentManager(data, keyTime, nil)
+	//dumpContentManagerData(t, data)
+	verifyContentNotFound(ctx, t, bm, content1)
+	verifyContentNotFound(ctx, t, bm, content2)
+}
+
+func TestRewriteNonDeleted(t *testing.T) {
+	const stepBehaviors = 3
+
+	// perform a sequence WriteContent() <action1> RewriteContent() <action2> GetContent()
+	// where actionX can be (0=flush and reopen, 1=flush, 2=nothing)
+	for action1 := 0; action1 < stepBehaviors; action1++ {
+		for action2 := 0; action2 < stepBehaviors; action2++ {
+			t.Run(fmt.Sprintf("case-%v-%v", action1, action2), func(t *testing.T) {
+				ctx := context.Background()
+				data := blobtesting.DataMap{}
+				keyTime := map[blob.ID]time.Time{}
+				fakeNow := fakeTimeNowWithAutoAdvance(fakeTime, 1*time.Second)
+				bm := newTestContentManager(data, keyTime, fakeNow)
+
+				applyStep := func(action int) {
+					switch action {
+					case 0:
+						t.Logf("flushing and reopening")
+						bm.Flush(ctx)
+						bm = newTestContentManager(data, keyTime, fakeNow)
+					case 1:
+						t.Logf("flushing")
+						bm.Flush(ctx)
+					case 2:
+						t.Logf("doing nothing")
+					}
+				}
+
+				content1 := writeContentAndVerify(ctx, t, bm, seededRandomData(10, 100))
+				applyStep(action1)
+				assertNoError(t, bm.RewriteContent(ctx, content1))
+				applyStep(action2)
+				verifyContent(ctx, t, bm, content1, seededRandomData(10, 100))
+				dumpContentManagerData(t, data)
+			})
+		}
+	}
+}
+
+func TestDisableFlush(t *testing.T) {
+	ctx := context.Background()
+	data := blobtesting.DataMap{}
+	keyTime := map[blob.ID]time.Time{}
+	bm := newTestContentManager(data, keyTime, nil)
+	bm.DisableIndexFlush()
+	bm.DisableIndexFlush()
+	for i := 0; i < 500; i++ {
+		writeContentAndVerify(ctx, t, bm, seededRandomData(i, 100))
+	}
+	bm.Flush(ctx) // flush will not have effect
+	bm.EnableIndexFlush()
+	bm.Flush(ctx) // flush will not have effect
+	bm.EnableIndexFlush()
+
+	verifyActiveIndexBlobCount(ctx, t, bm, 0)
+	bm.EnableIndexFlush()
+	verifyActiveIndexBlobCount(ctx, t, bm, 0)
+	bm.Flush(ctx) // flush will happen now
+	verifyActiveIndexBlobCount(ctx, t, bm, 1)
+}
+
+func TestRewriteDeleted(t *testing.T) {
+	const stepBehaviors = 3
+
+	// perform a sequence WriteContent() <action1> Delete() <action2> RewriteContent() <action3> GetContent()
+	// where actionX can be (0=flush and reopen, 1=flush, 2=nothing)
+	for action1 := 0; action1 < stepBehaviors; action1++ {
+		for action2 := 0; action2 < stepBehaviors; action2++ {
+			for action3 := 0; action3 < stepBehaviors; action3++ {
+				t.Run(fmt.Sprintf("case-%v-%v-%v", action1, action2, action3), func(t *testing.T) {
+					ctx := context.Background()
+					data := blobtesting.DataMap{}
+					keyTime := map[blob.ID]time.Time{}
+					fakeNow := fakeTimeNowWithAutoAdvance(fakeTime, 1*time.Second)
+					bm := newTestContentManager(data, keyTime, fakeNow)
+
+					applyStep := func(action int) {
+						switch action {
+						case 0:
+							t.Logf("flushing and reopening")
+							bm.Flush(ctx)
+							bm = newTestContentManager(data, keyTime, fakeNow)
+						case 1:
+							t.Logf("flushing")
+							bm.Flush(ctx)
+						case 2:
+							t.Logf("doing nothing")
+						}
+					}
+
+					content1 := writeContentAndVerify(ctx, t, bm, seededRandomData(10, 100))
+					applyStep(action1)
+					assertNoError(t, bm.DeleteContent(content1))
+					applyStep(action2)
+					if got, want := bm.RewriteContent(ctx, content1), ErrContentNotFound; got != want && got != nil {
+						t.Errorf("unexpected error %v, wanted %v", got, want)
+					}
+					applyStep(action3)
+					verifyContentNotFound(ctx, t, bm, content1)
+					dumpContentManagerData(t, data)
+				})
+			}
+		}
+	}
+}
+
+func TestDeleteAndRecreate(t *testing.T) {
+	ctx := context.Background()
+	// simulate race between delete/recreate and delete
+	// delete happens at t0+10, recreate at t0+20 and second delete time is parameterized.
+	// depending on it, the second delete results will be visible.
+	cases := []struct {
+		desc         string
+		deletionTime time.Time
+		isVisible    bool
+	}{
+		{"deleted before delete and-recreate", fakeTime.Add(5 * time.Second), true},
+		//{"deleted after delete and recreate", fakeTime.Add(25 * time.Second), false},
+	}
+
+	for _, tc := range cases {
+		t.Run(tc.desc, func(t *testing.T) {
+			// write a content
+			data := blobtesting.DataMap{}
+			keyTime := map[blob.ID]time.Time{}
+			bm := newTestContentManager(data, keyTime, fakeTimeNowFrozen(fakeTime))
+			content1 := writeContentAndVerify(ctx, t, bm, seededRandomData(10, 100))
+			bm.Flush(ctx)
+
+			// delete but at given timestamp but don't commit yet.
+			bm0 := newTestContentManager(data, keyTime, fakeTimeNowWithAutoAdvance(tc.deletionTime, 1*time.Second))
+			assertNoError(t, bm0.DeleteContent(content1))
+
+			// delete it at t0+10
+			bm1 := newTestContentManager(data, keyTime, fakeTimeNowWithAutoAdvance(fakeTime.Add(10*time.Second), 1*time.Second))
+			verifyContent(ctx, t, bm1, content1, seededRandomData(10, 100))
+			assertNoError(t, bm1.DeleteContent(content1))
+			bm1.Flush(ctx)
+
+			// recreate at t0+20
+			bm2 := newTestContentManager(data, keyTime, fakeTimeNowWithAutoAdvance(fakeTime.Add(20*time.Second), 1*time.Second))
+			content2 := writeContentAndVerify(ctx, t, bm2, seededRandomData(10, 100))
+			bm2.Flush(ctx)
+
+			// commit deletion from bm0 (t0+5)
+			bm0.Flush(ctx)
+
+			//dumpContentManagerData(t, data)
+
+			if content1 != content2 {
+				t.Errorf("got invalid content %v, expected %v", content2, content1)
+			}
+
+			bm3 := newTestContentManager(data, keyTime, nil)
+			dumpContentManagerData(t, data)
+			if tc.isVisible {
+				verifyContent(ctx, t, bm3, content1, seededRandomData(10, 100))
+			} else {
+				verifyContentNotFound(ctx, t, bm3, content1)
+			}
+		})
+	}
+}
+
+func TestFindUnreferencedBlobs(t *testing.T) {
+	ctx := context.Background()
+	data := blobtesting.DataMap{}
+	keyTime := map[blob.ID]time.Time{}
+	bm := newTestContentManager(data, keyTime, nil)
+	verifyUnreferencedStorageFilesCount(ctx, t, bm, 0)
+	contentID := writeContentAndVerify(ctx, t, bm, seededRandomData(10, 100))
+	if err := bm.Flush(ctx); err != nil {
+		t.Errorf("flush error: %v", err)
+	}
+	verifyUnreferencedStorageFilesCount(ctx, t, bm, 0)
+	if err := bm.DeleteContent(contentID); err != nil {
+		t.Errorf("error deleting content: %v", contentID)
+	}
+	if err := bm.Flush(ctx); err != nil {
+		t.Errorf("flush error: %v", err)
+	}
+
+	// content still present in first pack
+	verifyUnreferencedStorageFilesCount(ctx, t, bm, 0)
+
+	assertNoError(t, bm.RewriteContent(ctx, contentID))
+	if err := bm.Flush(ctx); err != nil {
+		t.Errorf("flush error: %v", err)
+	}
+	verifyUnreferencedStorageFilesCount(ctx, t, bm, 1)
+	assertNoError(t, bm.RewriteContent(ctx, contentID))
+	if err := bm.Flush(ctx); err != nil {
+		t.Errorf("flush error: %v", err)
+	}
+	verifyUnreferencedStorageFilesCount(ctx, t, bm, 2)
+}
+
+func TestFindUnreferencedBlobs2(t *testing.T) {
+	ctx := context.Background()
+	data := blobtesting.DataMap{}
+	keyTime := map[blob.ID]time.Time{}
+	bm := newTestContentManager(data, keyTime, nil)
+	verifyUnreferencedStorageFilesCount(ctx, t, bm, 0)
+	contentID := writeContentAndVerify(ctx, t, bm, seededRandomData(10, 100))
+	writeContentAndVerify(ctx, t, bm, seededRandomData(11, 100))
+	dumpContents(t, bm, "after writing")
+	if err := bm.Flush(ctx); err != nil {
+		t.Errorf("flush error: %v", err)
+	}
+	dumpContents(t, bm, "after flush")
+	verifyUnreferencedStorageFilesCount(ctx, t, bm, 0)
+	if err := bm.DeleteContent(contentID); err != nil {
+		t.Errorf("error deleting content: %v", contentID)
+	}
+	dumpContents(t, bm, "after delete")
+	if err := bm.Flush(ctx); err != nil {
+		t.Errorf("flush error: %v", err)
+	}
+	dumpContents(t, bm, "after flush")
+	// content present in first pack, original pack is still referenced
+	verifyUnreferencedStorageFilesCount(ctx, t, bm, 0)
+}
+
+func dumpContents(t *testing.T, bm *Manager, caption string) {
+	t.Helper()
+	infos, err := bm.ListContentInfos("", true)
+	if err != nil {
+		t.Errorf("error listing contents: %v", err)
+		return
+	}
+
+	log.Infof("**** dumping %v contents %v", len(infos), caption)
+	for i, bi := range infos {
+		log.Debugf(" bi[%v]=%#v", i, bi)
+	}
+	log.Infof("finished dumping %v contents", len(infos))
+}
+
+func verifyUnreferencedStorageFilesCount(ctx context.Context, t *testing.T, bm *Manager, want int) {
+	t.Helper()
+	unref, err := bm.FindUnreferencedBlobs(ctx)
+	if err != nil {
+		t.Errorf("error in FindUnreferencedBlobs: %v", err)
+	}
+
+	log.Infof("got %v expecting %v", unref, want)
+	if got := len(unref); got != want {
+		t.Errorf("invalid number of unreferenced contents: %v, wanted %v", got, want)
+	}
+}
+
+func TestContentWriteAliasing(t *testing.T) {
+	ctx := context.Background()
+	data := blobtesting.DataMap{}
+	keyTime := map[blob.ID]time.Time{}
+	bm := newTestContentManager(data, keyTime, fakeTimeNowFrozen(fakeTime))
+
+	contentData := []byte{100, 0, 0}
+	id1 := writeContentAndVerify(ctx, t, bm, contentData)
+	contentData[0] = 101
+	id2 := writeContentAndVerify(ctx, t, bm, contentData)
+	bm.Flush(ctx)
+	contentData[0] = 102
+	id3 := writeContentAndVerify(ctx, t, bm, contentData)
+	contentData[0] = 103
+	id4 := writeContentAndVerify(ctx, t, bm, contentData)
+	verifyContent(ctx, t, bm, id1, []byte{100, 0, 0})
+	verifyContent(ctx, t, bm, id2, []byte{101, 0, 0})
+	verifyContent(ctx, t, bm, id3, []byte{102, 0, 0})
+	verifyContent(ctx, t, bm, id4, []byte{103, 0, 0})
+}
+
+func TestContentReadAliasing(t *testing.T) {
+	ctx := context.Background()
+	data := blobtesting.DataMap{}
+	keyTime := map[blob.ID]time.Time{}
+	bm := newTestContentManager(data, keyTime, fakeTimeNowFrozen(fakeTime))
+
+	contentData := []byte{100, 0, 0}
+	id1 := writeContentAndVerify(ctx, t, bm, contentData)
+	contentData2, err := bm.GetContent(ctx, id1)
+	if err != nil {
+		t.Fatalf("can't get content data: %v", err)
+	}
+
+	contentData2[0]++
+	verifyContent(ctx, t, bm, id1, contentData)
+	bm.Flush(ctx)
+	verifyContent(ctx, t, bm, id1, contentData)
+}
+
+func TestVersionCompatibility(t *testing.T) {
+	for writeVer := minSupportedReadVersion; writeVer <= currentWriteVersion; writeVer++ {
+		t.Run(fmt.Sprintf("version-%v", writeVer), func(t *testing.T) {
+			verifyVersionCompat(t, writeVer)
+		})
+	}
+}
+
+func verifyVersionCompat(t *testing.T, writeVersion int) {
+	ctx := context.Background()
+
+	// create content manager that writes 'writeVersion' and reads all versions >= minSupportedReadVersion
+	data := blobtesting.DataMap{}
+	keyTime := map[blob.ID]time.Time{}
+	mgr := newTestContentManager(data, keyTime, nil)
+	mgr.writeFormatVersion = int32(writeVersion)
+
+	dataSet := map[ID][]byte{}
+
+	for i := 0; i < 3000000; i = (i + 1) * 2 {
+		data := make([]byte, i)
+		rand.Read(data)
+
+		cid, err := mgr.WriteContent(ctx, data, "")
+		if err != nil {
+			t.Fatalf("unable to write %v bytes: %v", len(data), err)
+		}
+		dataSet[cid] = data
+	}
+	verifyContentManagerDataSet(ctx, t, mgr, dataSet)
+
+	// delete random 3 items (map iteration order is random)
+	cnt := 0
+	for blobID := range dataSet {
+		t.Logf("deleting %v", blobID)
+		assertNoError(t, mgr.DeleteContent(blobID))
+		delete(dataSet, blobID)
+		cnt++
+		if cnt >= 3 {
+			break
+		}
+	}
+	if err := mgr.Flush(ctx); err != nil {
+		t.Fatalf("failed to flush: %v", err)
+	}
+
+	// create new manager that reads and writes using new version.
+	mgr = newTestContentManager(data, keyTime, nil)
+
+	// make sure we can read everything
+	verifyContentManagerDataSet(ctx, t, mgr, dataSet)
+
+	if err := mgr.CompactIndexes(ctx, CompactOptions{
+		MinSmallBlobs: 1,
+		MaxSmallBlobs: 1,
+	}); err != nil {
+		t.Fatalf("unable to compact indexes: %v", err)
+	}
+	if err := mgr.Flush(ctx); err != nil {
+		t.Fatalf("failed to flush: %v", err)
+	}
+	verifyContentManagerDataSet(ctx, t, mgr, dataSet)
+
+	// now open one more manager
+	mgr = newTestContentManager(data, keyTime, nil)
+	verifyContentManagerDataSet(ctx, t, mgr, dataSet)
+}
+
+func verifyContentManagerDataSet(ctx context.Context, t *testing.T, mgr *Manager, dataSet map[ID][]byte) {
+	for contentID, originalPayload := range dataSet {
+		v, err := mgr.GetContent(ctx, contentID)
+		if err != nil {
+			t.Errorf("unable to read content %q: %v", contentID, err)
+			continue
+		}
+
+		if !reflect.DeepEqual(v, originalPayload) {
+			t.Errorf("payload for %q does not match original: %v", v, originalPayload)
+		}
+	}
+}
+
+func newTestContentManager(data blobtesting.DataMap, keyTime map[blob.ID]time.Time, timeFunc func() time.Time) *Manager {
+	//st = logging.NewWrapper(st)
+	if timeFunc == nil {
+		timeFunc = fakeTimeNowWithAutoAdvance(fakeTime, 1*time.Second)
+	}
+	st := blobtesting.NewMapStorage(data, keyTime, timeFunc)
+	bm, err := newManagerWithOptions(context.Background(), st, FormattingOptions{
+		Hash:        "HMAC-SHA256",
+		Encryption:  "NONE",
+		HMACSecret:  hmacSecret,
+		MaxPackSize: maxPackSize,
+		Version:     1,
+	}, CachingOptions{}, timeFunc, nil)
+	if err != nil {
+		panic("can't create content manager: " + err.Error())
+	}
+	bm.checkInvariantsOnUnlock = true
+	return bm
+}
+
+func getIndexCount(d blobtesting.DataMap) int {
+	var cnt int
+
+	for blobID := range d {
+		if strings.HasPrefix(string(blobID), newIndexBlobPrefix) {
+			cnt++
+		}
+	}
+
+	return cnt
+}
+
+func fakeTimeNowFrozen(t time.Time) func() time.Time {
+	return fakeTimeNowWithAutoAdvance(t, 0)
+}
+
+func fakeTimeNowWithAutoAdvance(t time.Time, dt time.Duration) func() time.Time {
+	var mu sync.Mutex
+	return func() time.Time {
+		mu.Lock()
+		defer mu.Unlock()
+		ret := t
+		t = t.Add(dt)
+		return ret
+	}
+}
+
+func verifyContentNotFound(ctx context.Context, t *testing.T, bm *Manager, contentID ID) {
+	t.Helper()
+
+	b, err := bm.GetContent(ctx, contentID)
+	if err != ErrContentNotFound {
+		t.Errorf("unexpected response from GetContent(%q), got %v,%v, expected %v", contentID, b, err, ErrContentNotFound)
+	}
+}
+
+func verifyContent(ctx context.Context, t *testing.T, bm *Manager, contentID ID, b []byte) {
+	t.Helper()
+
+	b2, err := bm.GetContent(ctx, contentID)
+	if err != nil {
+		t.Errorf("unable to read content %q: %v", contentID, err)
+		return
+	}
+
+	if got, want := b2, b; !reflect.DeepEqual(got, want) {
+		t.Errorf("content %q data mismatch: got %x (nil:%v), wanted %x (nil:%v)", contentID, got, got == nil, want, want == nil)
+	}
+
+	bi, err := bm.ContentInfo(ctx, contentID)
+	if err != nil {
+		t.Errorf("error getting content info %q: %v", contentID, err)
+	}
+
+	if got, want := bi.Length, uint32(len(b)); got != want {
+		t.Errorf("invalid content size for %q: %v, wanted %v", contentID, got, want)
+	}
+
+}
+func writeContentAndVerify(ctx context.Context, t *testing.T, bm *Manager, b []byte) ID {
+	t.Helper()
+
+	contentID, err := bm.WriteContent(ctx, b, "")
+	if err != nil {
+		t.Errorf("err: %v", err)
+	}
+
+	if got, want := contentID, ID(hashValue(b)); got != want {
+		t.Errorf("invalid content ID for %x, got %v, want %v", b, got, want)
+	}
+
+	verifyContent(ctx, t, bm, contentID, b)
+
+	return contentID
+}
+
+func seededRandomData(seed int, length int) []byte {
+	b := make([]byte, length)
+	rnd := rand.New(rand.NewSource(int64(seed)))
+	rnd.Read(b)
+	return b
+}
+
+func hashValue(b []byte) string {
+	h := hmac.New(sha256.New, hmacSecret)
+	h.Write(b) //nolint:errcheck
+	return hex.EncodeToString(h.Sum(nil))
+}
+
+func dumpContentManagerData(t *testing.T, data blobtesting.DataMap) {
+	t.Helper()
+	for k, v := range data {
+		if k[0] == 'n' {
+			ndx, err := openPackIndex(bytes.NewReader(v))
+			if err == nil {
+				t.Logf("index %v (%v bytes)", k, len(v))
+				assertNoError(t, ndx.Iterate("", func(i Info) error {
+					t.Logf("  %+v\n", i)
+					return nil
+				}))
+
+			}
+		} else {
+			t.Logf("data %v (%v bytes)\n", k, len(v))
+		}
+	}
+}

repo/content/context.go

@@ -0,0 +1,34 @@
+package content
+
+import "context"
+
+type contextKey string
+
+var useContentCacheContextKey contextKey = "use-content-cache"
+var useListCacheContextKey contextKey = "use-list-cache"
+
+// UsingContentCache returns a derived context that causes content manager to use cache.
+func UsingContentCache(ctx context.Context, enabled bool) context.Context {
+	return context.WithValue(ctx, useContentCacheContextKey, enabled)
+}
+
+// UsingListCache returns a derived context that causes content manager to use cache.
+func UsingListCache(ctx context.Context, enabled bool) context.Context {
+	return context.WithValue(ctx, useListCacheContextKey, enabled)
+}
+
+func shouldUseContentCache(ctx context.Context) bool {
+	if enabled, ok := ctx.Value(useContentCacheContextKey).(bool); ok {
+		return enabled
+	}
+
+	return true
+}
+
+func shouldUseListCache(ctx context.Context) bool {
+	if enabled, ok := ctx.Value(useListCacheContextKey).(bool); ok {
+		return enabled
+	}
+
+	return true
+}

repo/content/format.go

@@ -1,4 +1,4 @@
-package block
+package content
 
 import (
 	"encoding/binary"
@@ -27,9 +27,9 @@ type entry struct {
 	// big endian:
 	// 48 most significant bits - 48-bit timestamp in seconds since 1970/01/01 UTC
 	// 8 bits - format version (currently == 1)
-	// 8 least significant bits - length of pack block ID
+	// 8 least significant bits - length of pack content ID
 	timestampAndFlags uint64 //
-	packFileOffset    uint32 // 4 bytes, big endian, offset within index file where pack block ID begins
+	packFileOffset    uint32 // 4 bytes, big endian, offset within index file where pack content ID begins
 	packedOffset      uint32 // 4 bytes, big endian, offset within pack file where the contents begin
 	packedLength      uint32 // 4 bytes, big endian, content length
 }

repo/content/index.go

@@ -1,4 +1,4 @@
-package block
+package content
 
 import (
 	"bytes"
@@ -12,12 +12,12 @@
 	"github.com/kopia/kopia/repo/blob"
 )
 
-// packIndex is a read-only index of packed blocks.
+// packIndex is a read-only index of packed contents.
 type packIndex interface {
 	io.Closer
 
-	GetInfo(blockID string) (*Info, error)
-	Iterate(prefix string, cb func(Info) error) error
+	GetInfo(contentID ID) (*Info, error)
+	Iterate(prefix ID, cb func(Info) error) error
 }
 
 type index struct {
@@ -55,10 +55,10 @@ func readHeader(readerAt io.ReaderAt) (headerInfo, error) {
 	return hi, nil
 }
 
-// Iterate invokes the provided callback function for all blocks in the index, sorted alphabetically.
+// Iterate invokes the provided callback function for all contents in the index, sorted alphabetically.
 // The iteration ends when the callback returns an error, which is propagated to the caller or when
-// all blocks have been visited.
-func (b *index) Iterate(prefix string, cb func(Info) error) error {
+// all contents have been visited.
+func (b *index) Iterate(prefix ID, cb func(Info) error) error {
 	startPos, err := b.findEntryPosition(prefix)
 	if err != nil {
 		return errors.Wrap(err, "could not find starting position")
@@ -78,7 +78,7 @@ func (b *index) Iterate(prefix string, cb func(Info) error) error {
 		if err != nil {
 			return errors.Wrap(err, "invalid index data")
 		}
-		if !strings.HasPrefix(i.BlockID, prefix) {
+		if !strings.HasPrefix(string(i.ID), string(prefix)) {
 			break
 		}
 		if err := cb(i); err != nil {
@@ -88,7 +88,7 @@ func (b *index) Iterate(prefix string, cb func(Info) error) error {
 	return nil
 }
 
-func (b *index) findEntryPosition(blockID string) (int, error) {
+func (b *index) findEntryPosition(contentID ID) (int, error) {
 	stride := b.hdr.keySize + b.hdr.valueSize
 	entryBuf := make([]byte, stride)
 	var readErr error
@@ -102,20 +102,20 @@ func (b *index) findEntryPosition(blockID string) (int, error) {
 			return false
 		}
 
-		return bytesToContentID(entryBuf[0:b.hdr.keySize]) >= blockID
+		return bytesToContentID(entryBuf[0:b.hdr.keySize]) >= contentID
 	})
 
 	return pos, readErr
 }
 
-func (b *index) findEntry(blockID string) ([]byte, error) {
-	key := contentIDToBytes(blockID)
+func (b *index) findEntry(contentID ID) ([]byte, error) {
+	key := contentIDToBytes(contentID)
 	if len(key) != b.hdr.keySize {
-		return nil, errors.Errorf("invalid block ID: %q", blockID)
+		return nil, errors.Errorf("invalid content ID: %q", contentID)
 	}
 	stride := b.hdr.keySize + b.hdr.valueSize
 
-	position, err := b.findEntryPosition(blockID)
+	position, err := b.findEntryPosition(contentID)
 	if err != nil {
 		return nil, err
 	}
@@ -135,9 +135,9 @@ func (b *index) findEntry(blockID string) ([]byte, error) {
 	return nil, nil
 }
 
-// GetInfo returns information about a given block. If a block is not found, nil is returned.
-func (b *index) GetInfo(blockID string) (*Info, error) {
-	e, err := b.findEntry(blockID)
+// GetInfo returns information about a given content. If a content is not found, nil is returned.
+func (b *index) GetInfo(contentID ID) (*Info, error) {
+	e, err := b.findEntry(contentID)
 	if err != nil {
 		return nil, err
 	}
@@ -146,14 +146,14 @@ func (b *index) GetInfo(blockID string) (*Info, error) {
 		return nil, nil
 	}
 
-	i, err := b.entryToInfo(blockID, e)
+	i, err := b.entryToInfo(contentID, e)
 	if err != nil {
 		return nil, err
 	}
 	return &i, err
 }
 
-func (b *index) entryToInfo(blockID string, entryData []byte) (Info, error) {
+func (b *index) entryToInfo(contentID ID, entryData []byte) (Info, error) {
 	if len(entryData) < 20 {
 		return Info{}, errors.Errorf("invalid entry length: %v", len(entryData))
 	}
@@ -166,11 +166,11 @@ func (b *index) entryToInfo(blockID string, entryData []byte) (Info, error) {
 	packFile := make([]byte, e.PackFileLength())
 	n, err := b.readerAt.ReadAt(packFile, int64(e.PackFileOffset()))
 	if err != nil || n != int(e.PackFileLength()) {
-		return Info{}, errors.Wrap(err, "can't read pack block ID")
+		return Info{}, errors.Wrap(err, "can't read pack content ID")
 	}
 
 	return Info{
-		BlockID:          blockID,
+		ID:               contentID,
 		Deleted:          e.IsDeleted(),
 		TimestampSeconds: e.TimestampSeconds(),
 		FormatVersion:    e.PackedFormatVersion(),

repo/content/info.go

@@ -1,4 +1,4 @@
-package block
+package content
 
 import (
 	"time"
@@ -6,9 +6,12 @@
 	"github.com/kopia/kopia/repo/blob"
 )
 
-// Info is an information about a single block managed by Manager.
+// ID is an identifier of content in content-addressable storage.
+type ID string
+
+// Info is an information about a single piece of content managed by Manager.
 type Info struct {
-	BlockID          string  `json:"blockID"`
+	ID               ID      `json:"contentID"`
 	Length           uint32  `json:"length"`
 	TimestampSeconds int64   `json:"time"`
 	PackBlobID       blob.ID `json:"packFile,omitempty"`
@@ -18,7 +21,7 @@ type Info struct {
 	FormatVersion    byte    `json:"formatVersion"`
 }
 
-// Timestamp returns the time when a block was created or deleted.
+// Timestamp returns the time when a content was created or deleted.
 func (i Info) Timestamp() time.Time {
 	return time.Unix(i.TimestampSeconds, 0)
 }

repo/content/list_cache.go

@@ -1,4 +1,4 @@
-package block
+package content
 
 import (
 	"context"
@@ -23,35 +23,35 @@ type listCache struct {
 
 func (c *listCache) listIndexBlobs(ctx context.Context) ([]IndexBlobInfo, error) {
 	if c.cacheFile != "" {
-		ci, err := c.readBlocksFromCache(ctx)
+		ci, err := c.readContentsFromCache(ctx)
 		if err == nil {
 			expirationTime := ci.Timestamp.Add(c.listCacheDuration)
 			if time.Now().Before(expirationTime) {
 				log.Debugf("retrieved list of index blobs from cache")
-				return ci.Blocks, nil
+				return ci.Contents, nil
 			}
 		} else if err != blob.ErrBlobNotFound {
 			log.Warningf("unable to open cache file: %v", err)
 		}
 	}
 
-	blocks, err := listIndexBlobsFromStorage(ctx, c.st)
+	contents, err := listIndexBlobsFromStorage(ctx, c.st)
 	if err == nil {
 		c.saveListToCache(ctx, &cachedList{
-			Blocks:    blocks,
+			Contents:  contents,
 			Timestamp: time.Now(),
 		})
 	}
-	log.Debugf("found %v index blobs from source", len(blocks))
+	log.Debugf("found %v index blobs from source", len(contents))
 
-	return blocks, err
+	return contents, err
 }
 
 func (c *listCache) saveListToCache(ctx context.Context, ci *cachedList) {
 	if c.cacheFile == "" {
 		return
 	}
-	log.Debugf("saving index blobs to cache: %v", len(ci.Blocks))
+	log.Debugf("saving index blobs to cache: %v", len(ci.Contents))
 	if data, err := json.Marshal(ci); err == nil {
 		mySuffix := fmt.Sprintf(".tmp-%v-%v", os.Getpid(), time.Now().UnixNano())
 		if err := ioutil.WriteFile(c.cacheFile+mySuffix, appendHMAC(data, c.hmacSecret), 0600); err != nil {
@@ -68,7 +68,7 @@ func (c *listCache) deleteListCache(ctx context.Context) {
 	}
 }
 
-func (c *listCache) readBlocksFromCache(ctx context.Context) (*cachedList, error) {
+func (c *listCache) readContentsFromCache(ctx context.Context) (*cachedList, error) {
 	if !shouldUseListCache(ctx) {
 		return nil, blob.ErrBlobNotFound
 	}

repo/content/merged.go

@@ -1,4 +1,4 @@
-package block
+package content
 
 import (
 	"container/heap"
@@ -19,11 +19,11 @@ func (m mergedIndex) Close() error {
 	return nil
 }
 
-// GetInfo returns information about a single block. If a block is not found, returns (nil,nil)
-func (m mergedIndex) GetInfo(contentID string) (*Info, error) {
+// GetInfo returns information about a single content. If a content is not found, returns (nil,nil)
+func (m mergedIndex) GetInfo(id ID) (*Info, error) {
 	var best *Info
 	for _, ndx := range m {
-		i, err := ndx.GetInfo(contentID)
+		i, err := ndx.GetInfo(id)
 		if err != nil {
 			return nil, err
 		}
@@ -45,7 +45,7 @@ type nextInfo struct {
 
 func (h nextInfoHeap) Len() int { return len(h) }
 func (h nextInfoHeap) Less(i, j int) bool {
-	if a, b := h[i].it.BlockID, h[j].it.BlockID; a != b {
+	if a, b := h[i].it.ID, h[j].it.ID; a != b {
 		return a < b
 	}
 
@@ -68,7 +68,7 @@ func (h *nextInfoHeap) Pop() interface{} {
 	return x
 }
 
-func iterateChan(prefix string, ndx packIndex, done chan bool) <-chan Info {
+func iterateChan(prefix ID, ndx packIndex, done chan bool) <-chan Info {
 	ch := make(chan Info)
 	go func() {
 		defer close(ch)
@@ -85,9 +85,9 @@ func iterateChan(prefix string, ndx packIndex, done chan bool) <-chan Info {
 	return ch
 }
 
-// Iterate invokes the provided callback for all unique block IDs in the underlying sources until either
-// all blocks have been visited or until an error is returned by the callback.
-func (m mergedIndex) Iterate(prefix string, cb func(i Info) error) error {
+// Iterate invokes the provided callback for all unique content IDs in the underlying sources until either
+// all contents have been visited or until an error is returned by the callback.
+func (m mergedIndex) Iterate(prefix ID, cb func(i Info) error) error {
 	var minHeap nextInfoHeap
 	done := make(chan bool)
 	defer close(done)
@@ -104,8 +104,8 @@ func (m mergedIndex) Iterate(prefix string, cb func(i Info) error) error {
 
 	for len(minHeap) > 0 {
 		min := heap.Pop(&minHeap).(*nextInfo)
-		if pendingItem.BlockID != min.it.BlockID {
-			if pendingItem.BlockID != "" {
+		if pendingItem.ID != min.it.ID {
+			if pendingItem.ID != "" {
 				if err := cb(pendingItem); err != nil {
 					return err
 				}
@@ -122,7 +122,7 @@ func (m mergedIndex) Iterate(prefix string, cb func(i Info) error) error {
 		}
 	}
 
-	if pendingItem.BlockID != "" {
+	if pendingItem.ID != "" {
 		return cb(pendingItem)
 	}
 

repo/content/merged_test.go

@@ -1,4 +1,4 @@
-package block
+package content
 
 import (
 	"bytes"
@@ -10,27 +10,27 @@
 
 func TestMerged(t *testing.T) {
 	i1, err := indexWithItems(
-		Info{BlockID: "aabbcc", TimestampSeconds: 1, PackBlobID: "xx", PackOffset: 11},
-		Info{BlockID: "ddeeff", TimestampSeconds: 1, PackBlobID: "xx", PackOffset: 111},
-		Info{BlockID: "z010203", TimestampSeconds: 1, PackBlobID: "xx", PackOffset: 111},
-		Info{BlockID: "de1e1e", TimestampSeconds: 4, PackBlobID: "xx", PackOffset: 111},
+		Info{ID: "aabbcc", TimestampSeconds: 1, PackBlobID: "xx", PackOffset: 11},
+		Info{ID: "ddeeff", TimestampSeconds: 1, PackBlobID: "xx", PackOffset: 111},
+		Info{ID: "z010203", TimestampSeconds: 1, PackBlobID: "xx", PackOffset: 111},
+		Info{ID: "de1e1e", TimestampSeconds: 4, PackBlobID: "xx", PackOffset: 111},
 	)
 	if err != nil {
 		t.Fatalf("can't create index: %v", err)
 	}
 	i2, err := indexWithItems(
-		Info{BlockID: "aabbcc", TimestampSeconds: 3, PackBlobID: "yy", PackOffset: 33},
-		Info{BlockID: "xaabbcc", TimestampSeconds: 1, PackBlobID: "xx", PackOffset: 111},
-		Info{BlockID: "de1e1e", TimestampSeconds: 4, PackBlobID: "xx", PackOffset: 222, Deleted: true},
+		Info{ID: "aabbcc", TimestampSeconds: 3, PackBlobID: "yy", PackOffset: 33},
+		Info{ID: "xaabbcc", TimestampSeconds: 1, PackBlobID: "xx", PackOffset: 111},
+		Info{ID: "de1e1e", TimestampSeconds: 4, PackBlobID: "xx", PackOffset: 222, Deleted: true},
 	)
 	if err != nil {
 		t.Fatalf("can't create index: %v", err)
 	}
 	i3, err := indexWithItems(
-		Info{BlockID: "aabbcc", TimestampSeconds: 2, PackBlobID: "zz", PackOffset: 22},
-		Info{BlockID: "ddeeff", TimestampSeconds: 1, PackBlobID: "zz", PackOffset: 222},
-		Info{BlockID: "k010203", TimestampSeconds: 1, PackBlobID: "xx", PackOffset: 111},
-		Info{BlockID: "k020304", TimestampSeconds: 1, PackBlobID: "xx", PackOffset: 111},
+		Info{ID: "aabbcc", TimestampSeconds: 2, PackBlobID: "zz", PackOffset: 22},
+		Info{ID: "ddeeff", TimestampSeconds: 1, PackBlobID: "zz", PackOffset: 222},
+		Info{ID: "k010203", TimestampSeconds: 1, PackBlobID: "xx", PackOffset: 111},
+		Info{ID: "k020304", TimestampSeconds: 1, PackBlobID: "xx", PackOffset: 111},
 	)
 	if err != nil {
 		t.Fatalf("can't create index: %v", err)
@@ -45,24 +45,24 @@ func TestMerged(t *testing.T) {
 		t.Errorf("invalid pack offset %v, wanted %v", got, want)
 	}
 
-	var inOrder []string
+	var inOrder []ID
 	assertNoError(t, m.Iterate("", func(i Info) error {
-		inOrder = append(inOrder, i.BlockID)
-		if i.BlockID == "de1e1e" {
+		inOrder = append(inOrder, i.ID)
+		if i.ID == "de1e1e" {
 			if i.Deleted {
-				t.Errorf("iteration preferred deleted block over non-deleted")
+				t.Errorf("iteration preferred deleted content over non-deleted")
 			}
 		}
 		return nil
 	}))
 
 	if i, err := m.GetInfo("de1e1e"); err != nil {
-		t.Errorf("error getting deleted block info: %v", err)
+		t.Errorf("error getting deleted content info: %v", err)
 	} else if i.Deleted {
-		t.Errorf("GetInfo preferred deleted block over non-deleted")
+		t.Errorf("GetInfo preferred deleted content over non-deleted")
 	}
 
-	expectedInOrder := []string{
+	expectedInOrder := []ID{
 		"aabbcc",
 		"ddeeff",
 		"de1e1e",

repo/content/packindex_internal_test.go

@@ -1,9 +1,9 @@
-package block
+package content
 
 import "testing"
 
 func TestRoundTrip(t *testing.T) {
-	cases := []string{
+	cases := []ID{
 		"",
 		"x",
 		"aa",
@@ -20,7 +20,7 @@ func TestRoundTrip(t *testing.T) {
 		}
 	}
 
-	if got, want := bytesToContentID(nil), ""; got != want {
+	if got, want := bytesToContentID(nil), ID(""); got != want {
 		t.Errorf("unexpected content id %v, want %v", got, want)
 	}
 }

repo/content/packindex_test.go

@@ -1,4 +1,4 @@
-package block
+package content
 
 import (
 	"bytes"
@@ -14,12 +14,12 @@
 )
 
 func TestPackIndex(t *testing.T) {
-	blockNumber := 0
+	contentNumber := 0
 
-	deterministicBlockID := func(prefix string, id int) string {
+	deterministicContentID := func(prefix string, id int) ID {
 		h := sha1.New()
 		fmt.Fprintf(h, "%v%v", prefix, id)
-		blockNumber++
+		contentNumber++
 
 		prefix2 := ""
 		if id%2 == 0 {
@@ -31,12 +31,12 @@ func TestPackIndex(t *testing.T) {
 		if id%5 == 0 {
 			prefix2 = "m"
 		}
-		return string(fmt.Sprintf("%v%x", prefix2, h.Sum(nil)))
+		return ID(fmt.Sprintf("%v%x", prefix2, h.Sum(nil)))
 	}
 	deterministicPackBlobID := func(id int) blob.ID {
 		h := sha1.New()
 		fmt.Fprintf(h, "%v", id)
-		blockNumber++
+		contentNumber++
 		return blob.ID(fmt.Sprintf("%x", h.Sum(nil)))
 	}
 
@@ -60,23 +60,23 @@ func TestPackIndex(t *testing.T) {
 
 	var infos []Info
 
-	// deleted blocks with all information
+	// deleted contents with all information
 	for i := 0; i < 100; i++ {
 		infos = append(infos, Info{
 			TimestampSeconds: randomUnixTime(),
 			Deleted:          true,
-			BlockID:          deterministicBlockID("deleted-packed", i),
+			ID:               deterministicContentID("deleted-packed", i),
 			PackBlobID:       deterministicPackBlobID(i),
 			PackOffset:       deterministicPackedOffset(i),
 			Length:           deterministicPackedLength(i),
 			FormatVersion:    deterministicFormatVersion(i),
 		})
 	}
-	// non-deleted block
+	// non-deleted content
 	for i := 0; i < 100; i++ {
 		infos = append(infos, Info{
 			TimestampSeconds: randomUnixTime(),
-			BlockID:          deterministicBlockID("packed", i),
+			ID:               deterministicContentID("packed", i),
 			PackBlobID:       deterministicPackBlobID(i),
 			PackOffset:       deterministicPackedOffset(i),
 			Length:           deterministicPackedLength(i),
@@ -84,13 +84,13 @@ func TestPackIndex(t *testing.T) {
 		})
 	}
 
-	infoMap := map[string]Info{}
+	infoMap := map[ID]Info{}
 	b1 := make(packIndexBuilder)
 	b2 := make(packIndexBuilder)
 	b3 := make(packIndexBuilder)
 
 	for _, info := range infos {
-		infoMap[info.BlockID] = info
+		infoMap[info.ID] = info
 		b1.Add(info)
 		b2.Add(info)
 		b3.Add(info)
@@ -130,9 +130,9 @@ func TestPackIndex(t *testing.T) {
 	defer ndx.Close()
 
 	for _, info := range infos {
-		info2, err := ndx.GetInfo(info.BlockID)
+		info2, err := ndx.GetInfo(info.ID)
 		if err != nil {
-			t.Errorf("unable to find %v", info.BlockID)
+			t.Errorf("unable to find %v", info.ID)
 			continue
 		}
 		if !reflect.DeepEqual(info, *info2) {
@@ -142,7 +142,7 @@ func TestPackIndex(t *testing.T) {
 
 	cnt := 0
 	assertNoError(t, ndx.Iterate("", func(info2 Info) error {
-		info := infoMap[info2.BlockID]
+		info := infoMap[info2.ID]
 		if !reflect.DeepEqual(info, info2) {
 			t.Errorf("invalid value retrieved: %+v, wanted %+v", info2, info)
 		}
@@ -153,25 +153,25 @@ func TestPackIndex(t *testing.T) {
 		t.Errorf("invalid number of iterations: %v, wanted %v", cnt, len(infoMap))
 	}
 
-	prefixes := []string{"a", "b", "f", "0", "3", "aa", "aaa", "aab", "fff", "m", "x", "y", "m0", "ma"}
+	prefixes := []ID{"a", "b", "f", "0", "3", "aa", "aaa", "aab", "fff", "m", "x", "y", "m0", "ma"}
 
 	for i := 0; i < 100; i++ {
-		blockID := deterministicBlockID("no-such-block", i)
-		v, err := ndx.GetInfo(blockID)
+		contentID := deterministicContentID("no-such-content", i)
+		v, err := ndx.GetInfo(contentID)
 		if err != nil {
-			t.Errorf("unable to get block %v: %v", blockID, err)
+			t.Errorf("unable to get content %v: %v", contentID, err)
 		}
 		if v != nil {
-			t.Errorf("unexpected result when getting block %v: %v", blockID, v)
+			t.Errorf("unexpected result when getting content %v: %v", contentID, v)
 		}
 	}
 
 	for _, prefix := range prefixes {
 		cnt2 := 0
-		assertNoError(t, ndx.Iterate(string(prefix), func(info2 Info) error {
+		assertNoError(t, ndx.Iterate(prefix, func(info2 Info) error {
 			cnt2++
-			if !strings.HasPrefix(string(info2.BlockID), string(prefix)) {
-				t.Errorf("unexpected item %v when iterating prefix %v", info2.BlockID, prefix)
+			if !strings.HasPrefix(string(info2.ID), string(prefix)) {
+				t.Errorf("unexpected item %v when iterating prefix %v", info2.ID, prefix)
 			}
 			return nil
 		}))
@@ -192,7 +192,7 @@ func fuzzTestIndexOpen(t *testing.T, originalData []byte) {
 		cnt := 0
 		_ = ndx.Iterate("", func(cb Info) error {
 			if cnt < 10 {
-				_, _ = ndx.GetInfo(cb.BlockID)
+				_, _ = ndx.GetInfo(cb.ID)
 			}
 			cnt++
 			return nil

repo/content/stats.go

@@ -1,6 +1,6 @@
-package block
+package content
 
-// Stats exposes statistics about block operation.
+// Stats exposes statistics about content operation.
 type Stats struct {
 	// Keep int64 fields first to ensure they get aligned to at least 64-bit boundaries
 	// which is required for atomic access on ARM and x86-32.
@@ -10,13 +10,13 @@ type Stats struct {
 	EncryptedBytes int64 `json:"encryptedBytes,omitempty"`
 	HashedBytes    int64 `json:"hashedBytes,omitempty"`
 
-	ReadBlocks    int32 `json:"readBlocks,omitempty"`
-	WrittenBlocks int32 `json:"writtenBlocks,omitempty"`
-	CheckedBlocks int32 `json:"checkedBlocks,omitempty"`
-	HashedBlocks  int32 `json:"hashedBlocks,omitempty"`
-	InvalidBlocks int32 `json:"invalidBlocks,omitempty"`
-	PresentBlocks int32 `json:"presentBlocks,omitempty"`
-	ValidBlocks   int32 `json:"validBlocks,omitempty"`
+	ReadContents    int32 `json:"readContents,omitempty"`
+	WrittenContents int32 `json:"writtenContents,omitempty"`
+	CheckedContents int32 `json:"checkedContents,omitempty"`
+	HashedContents  int32 `json:"hashedContents,omitempty"`
+	InvalidContents int32 `json:"invalidContents,omitempty"`
+	PresentContents int32 `json:"presentContents,omitempty"`
+	ValidContents   int32 `json:"validContents,omitempty"`
 }
 
 // Reset clears all repository statistics.

repo/crypto_key_derivation.go

@@ -12,7 +12,7 @@
 // defaultKeyDerivationAlgorithm is the key derivation algorithm for new configurations.
 const defaultKeyDerivationAlgorithm = "scrypt-65536-8-1"
 
-func (f formatBlock) deriveMasterKeyFromPassword(password string) ([]byte, error) {
+func (f formatBlob) deriveMasterKeyFromPassword(password string) ([]byte, error) {
 	const masterKeySize = 32
 
 	switch f.KeyDerivationAlgorithm {

repo/format_block.go

@@ -20,13 +20,13 @@
 
 const (
 	maxChecksummedFormatBytesLength = 65000
-	formatBlockChecksumSize         = sha256.Size
+	formatBlobChecksumSize          = sha256.Size
 )
 
-// formatBlockChecksumSecret is a HMAC secret used for checksumming the format block.
+// formatBlobChecksumSecret is a HMAC secret used for checksumming the format content.
 // It's not really a secret, but will provide positive identification of blocks that
 // are repository format blocks.
-var formatBlockChecksumSecret = []byte("kopia-repository")
+var formatBlobChecksumSecret = []byte("kopia-repository")
 
 // FormatBlobID is the identifier of a BLOB that describes repository format.
 const FormatBlobID = "kopia.repository"
@@ -35,10 +35,10 @@
 	purposeAESKey   = []byte("AES")
 	purposeAuthData = []byte("CHECKSUM")
 
-	errFormatBlockNotFound = errors.New("format block not found")
+	errFormatBlobNotFound = errors.New("format blob not found")
 )
 
-type formatBlock struct {
+type formatBlob struct {
 	Tool         string `json:"tool"`
 	BuildVersion string `json:"buildVersion"`
 	BuildInfo    string `json:"buildInfo"`
@@ -57,22 +57,22 @@ type encryptedRepositoryConfig struct {
 	Format repositoryObjectFormat `json:"format"`
 }
 
-func parseFormatBlock(b []byte) (*formatBlock, error) {
-	f := &formatBlock{}
+func parseFormatBlob(b []byte) (*formatBlob, error) {
+	f := &formatBlob{}
 
 	if err := json.Unmarshal(b, &f); err != nil {
-		return nil, errors.Wrap(err, "invalid format block")
+		return nil, errors.Wrap(err, "invalid format blob")
 	}
 
 	return f, nil
 }
 
-// RecoverFormatBlock attempts to recover format block replica from the specified file.
-// The format block can be either the prefix or a suffix of the given file.
+// RecoverFormatBlob attempts to recover format blob replica from the specified file.
+// The format blob can be either the prefix or a suffix of the given file.
 // optionally the length can be provided (if known) to speed up recovery.
-func RecoverFormatBlock(ctx context.Context, st blob.Storage, blobID blob.ID, optionalLength int64) ([]byte, error) {
+func RecoverFormatBlob(ctx context.Context, st blob.Storage, blobID blob.ID, optionalLength int64) ([]byte, error) {
 	if optionalLength > 0 {
-		return recoverFormatBlockWithLength(ctx, st, blobID, optionalLength)
+		return recoverFormatBlobWithLength(ctx, st, blobID, optionalLength)
 	}
 
 	var foundMetadata blob.Metadata
@@ -91,10 +91,10 @@ func RecoverFormatBlock(ctx context.Context, st blob.Storage, blobID blob.ID, op
 		return nil, blob.ErrBlobNotFound
 	}
 
-	return recoverFormatBlockWithLength(ctx, st, foundMetadata.BlobID, foundMetadata.Length)
+	return recoverFormatBlobWithLength(ctx, st, foundMetadata.BlobID, foundMetadata.Length)
 }
 
-func recoverFormatBlockWithLength(ctx context.Context, st blob.Storage, blobID blob.ID, length int64) ([]byte, error) {
+func recoverFormatBlobWithLength(ctx context.Context, st blob.Storage, blobID blob.ID, length int64) ([]byte, error) {
 	chunkLength := int64(65536)
 	if chunkLength > length {
 		chunkLength = length
@@ -107,7 +107,7 @@ func recoverFormatBlockWithLength(ctx context.Context, st blob.Storage, blobID b
 			return nil, err
 		}
 		if l := int(prefixChunk[0]) + int(prefixChunk[1])<<8; l <= maxChecksummedFormatBytesLength && l+2 < len(prefixChunk) {
-			if b, ok := verifyFormatBlockChecksum(prefixChunk[2 : 2+l]); ok {
+			if b, ok := verifyFormatBlobChecksum(prefixChunk[2 : 2+l]); ok {
 				return b, nil
 			}
 		}
@@ -118,22 +118,22 @@ func recoverFormatBlockWithLength(ctx context.Context, st blob.Storage, blobID b
 			return nil, err
 		}
 		if l := int(suffixChunk[len(suffixChunk)-2]) + int(suffixChunk[len(suffixChunk)-1])<<8; l <= maxChecksummedFormatBytesLength && l+2 < len(suffixChunk) {
-			if b, ok := verifyFormatBlockChecksum(suffixChunk[len(suffixChunk)-2-l : len(suffixChunk)-2]); ok {
+			if b, ok := verifyFormatBlobChecksum(suffixChunk[len(suffixChunk)-2-l : len(suffixChunk)-2]); ok {
 				return b, nil
 			}
 		}
 	}
 
-	return nil, errFormatBlockNotFound
+	return nil, errFormatBlobNotFound
 }
 
-func verifyFormatBlockChecksum(b []byte) ([]byte, bool) {
-	if len(b) < formatBlockChecksumSize {
+func verifyFormatBlobChecksum(b []byte) ([]byte, bool) {
+	if len(b) < formatBlobChecksumSize {
 		return nil, false
 	}
 
-	data, checksum := b[0:len(b)-formatBlockChecksumSize], b[len(b)-formatBlockChecksumSize:]
-	h := hmac.New(sha256.New, formatBlockChecksumSecret)
+	data, checksum := b[0:len(b)-formatBlobChecksumSize], b[len(b)-formatBlobChecksumSize:]
+	h := hmac.New(sha256.New, formatBlobChecksumSecret)
 	h.Write(data) //nolint:errcheck
 	actualChecksum := h.Sum(nil)
 	if !hmac.Equal(actualChecksum, checksum) {
@@ -143,22 +143,22 @@ func verifyFormatBlockChecksum(b []byte) ([]byte, bool) {
 	return data, true
 }
 
-func writeFormatBlock(ctx context.Context, st blob.Storage, f *formatBlock) error {
+func writeFormatBlob(ctx context.Context, st blob.Storage, f *formatBlob) error {
 	var buf bytes.Buffer
 	e := json.NewEncoder(&buf)
 	e.SetIndent("", "  ")
 	if err := e.Encode(f); err != nil {
-		return errors.Wrap(err, "unable to marshal format block")
+		return errors.Wrap(err, "unable to marshal format blob")
 	}
 
 	if err := st.PutBlob(ctx, FormatBlobID, buf.Bytes()); err != nil {
-		return errors.Wrap(err, "unable to write format block")
+		return errors.Wrap(err, "unable to write format blob")
 	}
 
 	return nil
 }
 
-func (f *formatBlock) decryptFormatBytes(masterKey []byte) (*repositoryObjectFormat, error) {
+func (f *formatBlob) decryptFormatBytes(masterKey []byte) (*repositoryObjectFormat, error) {
 	switch f.EncryptionAlgorithm {
 	case "NONE": // do nothing
 		return f.UnencryptedFormat, nil
@@ -209,7 +209,7 @@ func initCrypto(masterKey, repositoryID []byte) (cipher.AEAD, []byte, error) {
 	return aead, authData, nil
 }
 
-func encryptFormatBytes(f *formatBlock, format *repositoryObjectFormat, masterKey, repositoryID []byte) error {
+func encryptFormatBytes(f *formatBlob, format *repositoryObjectFormat, masterKey, repositoryID []byte) error {
 	switch f.EncryptionAlgorithm {
 	case "NONE":
 		f.UnencryptedFormat = format
@@ -244,14 +244,14 @@ func encryptFormatBytes(f *formatBlock, format *repositoryObjectFormat, masterKe
 	}
 }
 
-func addFormatBlockChecksumAndLength(fb []byte) ([]byte, error) {
-	h := hmac.New(sha256.New, formatBlockChecksumSecret)
+func addFormatBlobChecksumAndLength(fb []byte) ([]byte, error) {
+	h := hmac.New(sha256.New, formatBlobChecksumSecret)
 	h.Write(fb) //nolint:errcheck
 	checksummedFormatBytes := h.Sum(fb)
 
 	l := len(checksummedFormatBytes)
 	if l > maxChecksummedFormatBytesLength {
-		return nil, errors.Errorf("format block too big: %v", l)
+		return nil, errors.Errorf("format blob too big: %v", l)
 	}
 
 	// return <length><checksummed-bytes><length>

repo/format_block_test.go

@@ -10,13 +10,13 @@
 	"github.com/kopia/kopia/repo/blob"
 )
 
-func TestFormatBlockRecovery(t *testing.T) {
+func TestFormatBlobRecovery(t *testing.T) {
 	data := blobtesting.DataMap{}
 	st := blobtesting.NewMapStorage(data, nil, nil)
 	ctx := context.Background()
 
 	someDataBlock := []byte("aadsdasdas")
-	checksummed, err := addFormatBlockChecksumAndLength(someDataBlock)
+	checksummed, err := addFormatBlobChecksumAndLength(someDataBlock)
 	if err != nil {
 		t.Errorf("error appending checksum: %v", err)
 	}
@@ -24,9 +24,9 @@ func TestFormatBlockRecovery(t *testing.T) {
 		t.Errorf("unexpected checksummed length: %v, want %v", got, want)
 	}
 
-	assertNoError(t, st.PutBlob(ctx, "some-block-by-itself", checksummed))
-	assertNoError(t, st.PutBlob(ctx, "some-block-suffix", append(append([]byte(nil), 1, 2, 3), checksummed...)))
-	assertNoError(t, st.PutBlob(ctx, "some-block-prefix", append(append([]byte(nil), checksummed...), 1, 2, 3)))
+	assertNoError(t, st.PutBlob(ctx, "some-blob-by-itself", checksummed))
+	assertNoError(t, st.PutBlob(ctx, "some-blob-suffix", append(append([]byte(nil), 1, 2, 3), checksummed...)))
+	assertNoError(t, st.PutBlob(ctx, "some-blob-prefix", append(append([]byte(nil), checksummed...), 1, 2, 3)))
 
 	// mess up checksum
 	checksummed[len(checksummed)-3] ^= 1
@@ -42,22 +42,22 @@ func TestFormatBlockRecovery(t *testing.T) {
 		blobID blob.ID
 		err    error
 	}{
-		{"some-block-by-itself", nil},
-		{"some-block-suffix", nil},
-		{"some-block-prefix", nil},
-		{"bad-checksum", errFormatBlockNotFound},
-		{"no-such-block", blob.ErrBlobNotFound},
-		{"zero-len", errFormatBlockNotFound},
-		{"one-len", errFormatBlockNotFound},
-		{"two-len", errFormatBlockNotFound},
-		{"three-len", errFormatBlockNotFound},
-		{"four-len", errFormatBlockNotFound},
-		{"five-len", errFormatBlockNotFound},
+		{"some-blob-by-itself", nil},
+		{"some-blob-suffix", nil},
+		{"some-blob-prefix", nil},
+		{"bad-checksum", errFormatBlobNotFound},
+		{"no-such-blob", blob.ErrBlobNotFound},
+		{"zero-len", errFormatBlobNotFound},
+		{"one-len", errFormatBlobNotFound},
+		{"two-len", errFormatBlobNotFound},
+		{"three-len", errFormatBlobNotFound},
+		{"four-len", errFormatBlobNotFound},
+		{"five-len", errFormatBlobNotFound},
 	}
 
 	for _, tc := range cases {
 		t.Run(string(tc.blobID), func(t *testing.T) {
-			v, err := RecoverFormatBlock(ctx, st, tc.blobID, -1)
+			v, err := RecoverFormatBlob(ctx, st, tc.blobID, -1)
 			if tc.err == nil {
 				if !reflect.DeepEqual(v, someDataBlock) || err != nil {
 					t.Errorf("unexpected result or error: v=%v err=%v, expected success", v, err)

repo/initialize.go

@@ -8,7 +8,7 @@
 	"github.com/pkg/errors"
 
 	"github.com/kopia/kopia/repo/blob"
-	"github.com/kopia/kopia/repo/block"
+	"github.com/kopia/kopia/repo/content"
 	"github.com/kopia/kopia/repo/object"
 )
 
@@ -22,7 +22,7 @@
 // All fields are optional, when not provided, reasonable defaults will be used.
 type NewRepositoryOptions struct {
 	UniqueID     []byte // force the use of particular unique ID
-	BlockFormat  block.FormattingOptions
+	BlockFormat  content.FormattingOptions
 	DisableHMAC  bool
 	ObjectFormat object.Format // object format
 }
@@ -42,7 +42,7 @@ func Initialize(ctx context.Context, st blob.Storage, opt *NewRepositoryOptions,
 		return err
 	}
 
-	format := formatBlockFromOptions(opt)
+	format := formatBlobFromOptions(opt)
 	masterKey, err := format.deriveMasterKeyFromPassword(password)
 	if err != nil {
 		return errors.Wrap(err, "unable to derive master key")
@@ -52,15 +52,15 @@ func Initialize(ctx context.Context, st blob.Storage, opt *NewRepositoryOptions,
 		return errors.Wrap(err, "unable to encrypt format bytes")
 	}
 
-	if err := writeFormatBlock(ctx, st, format); err != nil {
-		return errors.Wrap(err, "unable to write format block")
+	if err := writeFormatBlob(ctx, st, format); err != nil {
+		return errors.Wrap(err, "unable to write format blob")
 	}
 
 	return nil
 }
 
-func formatBlockFromOptions(opt *NewRepositoryOptions) *formatBlock {
-	f := &formatBlock{
+func formatBlobFromOptions(opt *NewRepositoryOptions) *formatBlob {
+	f := &formatBlob{
 		Tool:                   "https://github.com/kopia/kopia",
 		BuildInfo:              BuildInfo,
 		KeyDerivationAlgorithm: defaultKeyDerivationAlgorithm,
@@ -78,10 +78,10 @@ func formatBlockFromOptions(opt *NewRepositoryOptions) *formatBlock {
 
 func repositoryObjectFormatFromOptions(opt *NewRepositoryOptions) *repositoryObjectFormat {
 	f := &repositoryObjectFormat{
-		FormattingOptions: block.FormattingOptions{
+		FormattingOptions: content.FormattingOptions{
 			Version:     1,
-			Hash:        applyDefaultString(opt.BlockFormat.Hash, block.DefaultHash),
-			Encryption:  applyDefaultString(opt.BlockFormat.Encryption, block.DefaultEncryption),
+			Hash:        applyDefaultString(opt.BlockFormat.Hash, content.DefaultHash),
+			Encryption:  applyDefaultString(opt.BlockFormat.Encryption, content.DefaultEncryption),
 			HMACSecret:  applyDefaultRandomBytes(opt.BlockFormat.HMACSecret, 32),
 			MasterKey:   applyDefaultRandomBytes(opt.BlockFormat.MasterKey, 32),
 			MaxPackSize: applyDefaultInt(opt.BlockFormat.MaxPackSize, 20<<20), // 20 MB

repo/local_config.go

@@ -6,19 +6,19 @@
 	"os"
 
 	"github.com/kopia/kopia/repo/blob"
-	"github.com/kopia/kopia/repo/block"
+	"github.com/kopia/kopia/repo/content"
 	"github.com/kopia/kopia/repo/object"
 )
 
 // LocalConfig is a configuration of Kopia stored in a configuration file.
 type LocalConfig struct {
-	Storage blob.ConnectionInfo  `json:"storage"`
-	Caching block.CachingOptions `json:"caching"`
+	Storage blob.ConnectionInfo    `json:"storage"`
+	Caching content.CachingOptions `json:"caching"`
 }
 
 // repositoryObjectFormat describes the format of objects in a repository.
 type repositoryObjectFormat struct {
-	block.FormattingOptions
+	content.FormattingOptions
 	object.Format
 }
 

repo/manifest/manifest_entry.go

@@ -5,7 +5,7 @@
 // EntryMetadata contains metadata about manifest item. Each manifest item has one or more labels
 // Including required "type" label.
 type EntryMetadata struct {
-	ID      string
+	ID      ID
 	Length  int
 	Labels  map[string]string
 	ModTime time.Time

repo/manifest/manifest_manager.go

@@ -15,7 +15,7 @@
 	"github.com/pkg/errors"
 
 	"github.com/kopia/kopia/internal/repologging"
-	"github.com/kopia/kopia/repo/blob"
+	"github.com/kopia/kopia/repo/content"
 )
 
 var log = repologging.Logger("kopia/manifest")
@@ -23,33 +23,36 @@
 // ErrNotFound is returned when the metadata item is not found.
 var ErrNotFound = errors.New("not found")
 
-const manifestBlockPrefix = "m"
-const autoCompactionBlockCount = 16
+const manifestContentPrefix = "m"
+const autoCompactionContentCount = 16
 
-type blockManager interface {
-	GetBlock(ctx context.Context, blockID string) ([]byte, error)
-	WriteBlock(ctx context.Context, data []byte, prefix string) (string, error)
-	DeleteBlock(blockID string) error
-	ListBlocks(prefix string) ([]string, error)
+type contentManager interface {
+	GetContent(ctx context.Context, contentID content.ID) ([]byte, error)
+	WriteContent(ctx context.Context, data []byte, prefix content.ID) (content.ID, error)
+	DeleteContent(contentID content.ID) error
+	ListContents(prefix content.ID) ([]content.ID, error)
 	DisableIndexFlush()
 	EnableIndexFlush()
 	Flush(ctx context.Context) error
 }
 
+// ID is a unique identifier of a single manifest.
+type ID string
+
 // Manager organizes JSON manifests of various kinds, including snapshot manifests
 type Manager struct {
 	mu sync.Mutex
-	b  blockManager
+	b  contentManager
 
 	initialized    bool
-	pendingEntries map[string]*manifestEntry
+	pendingEntries map[ID]*manifestEntry
 
-	committedEntries  map[string]*manifestEntry
-	committedBlockIDs map[string]bool
+	committedEntries    map[ID]*manifestEntry
+	committedContentIDs map[content.ID]bool
 }
 
-// Put serializes the provided payload to JSON and persists it. Returns unique handle that represents the object.
-func (m *Manager) Put(ctx context.Context, labels map[string]string, payload interface{}) (string, error) {
+// Put serializes the provided payload to JSON and persists it. Returns unique identifier that represents the manifest.
+func (m *Manager) Put(ctx context.Context, labels map[string]string, payload interface{}) (ID, error) {
 	if labels["type"] == "" {
 		return "", errors.Errorf("'type' label is required")
 	}
@@ -71,7 +74,7 @@ func (m *Manager) Put(ctx context.Context, labels map[string]string, payload int
 	}
 
 	e := &manifestEntry{
-		ID:      hex.EncodeToString(random),
+		ID:      ID(hex.EncodeToString(random)),
 		ModTime: time.Now().UTC(),
 		Labels:  copyLabels(labels),
 		Content: b,
@@ -83,7 +86,7 @@ func (m *Manager) Put(ctx context.Context, labels map[string]string, payload int
 }
 
 // GetMetadata returns metadata about provided manifest item or ErrNotFound if the item can't be found.
-func (m *Manager) GetMetadata(ctx context.Context, id string) (*EntryMetadata, error) {
+func (m *Manager) GetMetadata(ctx context.Context, id ID) (*EntryMetadata, error) {
 	if err := m.ensureInitialized(ctx); err != nil {
 		return nil, err
 	}
@@ -110,7 +113,7 @@ func (m *Manager) GetMetadata(ctx context.Context, id string) (*EntryMetadata, e
 
 // Get retrieves the contents of the provided manifest item by deserializing it as JSON to provided object.
 // If the manifest is not found, returns ErrNotFound.
-func (m *Manager) Get(ctx context.Context, id string, data interface{}) error {
+func (m *Manager) Get(ctx context.Context, id ID, data interface{}) error {
 	if err := m.ensureInitialized(ctx); err != nil {
 		return err
 	}
@@ -128,7 +131,7 @@ func (m *Manager) Get(ctx context.Context, id string, data interface{}) error {
 }
 
 // GetRaw returns raw contents of the provided manifest (JSON bytes) or ErrNotFound if not found.
-func (m *Manager) GetRaw(ctx context.Context, id string) ([]byte, error) {
+func (m *Manager) GetRaw(ctx context.Context, id ID) ([]byte, error) {
 	if err := m.ensureInitialized(ctx); err != nil {
 		return nil, err
 	}
@@ -208,7 +211,7 @@ func (m *Manager) Flush(ctx context.Context) error {
 	return err
 }
 
-func (m *Manager) flushPendingEntriesLocked(ctx context.Context) (string, error) {
+func (m *Manager) flushPendingEntriesLocked(ctx context.Context) (content.ID, error) {
 	if len(m.pendingEntries) == 0 {
 		return "", nil
 	}
@@ -225,7 +228,7 @@ func (m *Manager) flushPendingEntriesLocked(ctx context.Context) (string, error)
 	mustSucceed(gz.Flush())
 	mustSucceed(gz.Close())
 
-	blockID, err := m.b.WriteBlock(ctx, buf.Bytes(), manifestBlockPrefix)
+	contentID, err := m.b.WriteContent(ctx, buf.Bytes(), manifestContentPrefix)
 	if err != nil {
 		return "", err
 	}
@@ -235,9 +238,9 @@ func (m *Manager) flushPendingEntriesLocked(ctx context.Context) (string, error)
 		delete(m.pendingEntries, e.ID)
 	}
 
-	m.committedBlockIDs[blockID] = true
+	m.committedContentIDs[contentID] = true
 
-	return blockID, nil
+	return contentID, nil
 }
 
 func mustSucceed(e error) {
@@ -247,7 +250,7 @@ func mustSucceed(e error) {
 }
 
 // Delete marks the specified manifest ID for deletion.
-func (m *Manager) Delete(ctx context.Context, id string) error {
+func (m *Manager) Delete(ctx context.Context, id ID) error {
 	if err := m.ensureInitialized(ctx); err != nil {
 		return err
 	}
@@ -264,53 +267,53 @@ func (m *Manager) Delete(ctx context.Context, id string) error {
 	return nil
 }
 
-// Refresh updates the committed blocks from the underlying storage.
+// Refresh updates the committed contents from the underlying storage.
 func (m *Manager) Refresh(ctx context.Context) error {
 	m.mu.Lock()
 	defer m.mu.Unlock()
 
-	return m.loadCommittedBlocksLocked(ctx)
+	return m.loadCommittedContentsLocked(ctx)
 }
 
-func (m *Manager) loadCommittedBlocksLocked(ctx context.Context) error {
-	log.Debugf("listing manifest blocks")
+func (m *Manager) loadCommittedContentsLocked(ctx context.Context) error {
+	log.Debugf("listing manifest contents")
 	for {
-		blocks, err := m.b.ListBlocks(manifestBlockPrefix)
+		contents, err := m.b.ListContents(manifestContentPrefix)
 		if err != nil {
-			return errors.Wrap(err, "unable to list manifest blocks")
+			return errors.Wrap(err, "unable to list manifest contents")
 		}
 
-		m.committedEntries = map[string]*manifestEntry{}
-		m.committedBlockIDs = map[string]bool{}
+		m.committedEntries = map[ID]*manifestEntry{}
+		m.committedContentIDs = map[content.ID]bool{}
 
-		log.Debugf("found %v manifest blocks", len(blocks))
-		err = m.loadManifestBlocks(ctx, blocks)
+		log.Debugf("found %v manifest contents", len(contents))
+		err = m.loadManifestContents(ctx, contents)
 		if err == nil {
 			// success
 			break
 		}
-		if err == blob.ErrBlobNotFound {
+		if err == content.ErrContentNotFound {
 			// try again, lost a race with another manifest manager which just did compaction
 			continue
 		}
-		return errors.Wrap(err, "unable to load manifest blocks")
+		return errors.Wrap(err, "unable to load manifest contents")
 	}
 
 	if err := m.maybeCompactLocked(ctx); err != nil {
-		return errors.Errorf("error auto-compacting blocks")
+		return errors.Errorf("error auto-compacting contents")
 	}
 
 	return nil
 }
 
-func (m *Manager) loadManifestBlocks(ctx context.Context, blockIDs []string) error {
+func (m *Manager) loadManifestContents(ctx context.Context, contentIDs []content.ID) error {
 	t0 := time.Now()
 
-	for _, b := range blockIDs {
-		m.committedBlockIDs[b] = true
+	for _, b := range contentIDs {
+		m.committedContentIDs[b] = true
 	}
 
-	manifests, err := m.loadBlocksInParallel(ctx, blockIDs)
+	manifests, err := m.loadContentsInParallel(ctx, contentIDs)
 	if err != nil {
 		return err
 	}
@@ -321,22 +324,22 @@ func (m *Manager) loadManifestBlocks(ctx context.Context, blockIDs []string) err
 		}
 	}
 
-	// after merging, remove blocks marked as deleted.
+	// after merging, remove contents marked as deleted.
 	for k, e := range m.committedEntries {
 		if e.Deleted {
 			delete(m.committedEntries, k)
 		}
 	}
 
-	log.Debugf("finished loading manifest blocks in %v.", time.Since(t0))
+	log.Debugf("finished loading manifest contents in %v.", time.Since(t0))
 
 	return nil
 }
 
-func (m *Manager) loadBlocksInParallel(ctx context.Context, blockIDs []string) ([]manifest, error) {
-	errors := make(chan error, len(blockIDs))
-	manifests := make(chan manifest, len(blockIDs))
-	ch := make(chan string, len(blockIDs))
+func (m *Manager) loadContentsInParallel(ctx context.Context, contentIDs []content.ID) ([]manifest, error) {
+	errors := make(chan error, len(contentIDs))
+	manifests := make(chan manifest, len(contentIDs))
+	ch := make(chan content.ID, len(contentIDs))
 	var wg sync.WaitGroup
 
 	for i := 0; i < 8; i++ {
@@ -346,21 +349,21 @@ func (m *Manager) loadBlocksInParallel(ctx context.Context, blockIDs []string) (
 
 			for blk := range ch {
 				t1 := time.Now()
-				man, err := m.loadManifestBlock(ctx, blk)
+				man, err := m.loadManifestContent(ctx, blk)
 
 				if err != nil {
 					errors <- err
-					log.Debugf("block %v failed to be loaded by worker %v in %v: %v.", blk, workerID, time.Since(t1), err)
+					log.Debugf("manifest content %v failed to be loaded by worker %v in %v: %v.", blk, workerID, time.Since(t1), err)
 				} else {
-					log.Debugf("block %v loaded by worker %v in %v.", blk, workerID, time.Since(t1))
+					log.Debugf("manifest content %v loaded by worker %v in %v.", blk, workerID, time.Since(t1))
 					manifests <- man
 				}
 			}
 		}(i)
 	}
 
-	// feed block IDs for goroutines
-	for _, b := range blockIDs {
+	// feed manifest content IDs for goroutines
+	for _, b := range contentIDs {
 		ch <- b
 	}
 	close(ch)
@@ -383,9 +386,9 @@ func (m *Manager) loadBlocksInParallel(ctx context.Context, blockIDs []string) (
 	return man, nil
 }
 
-func (m *Manager) loadManifestBlock(ctx context.Context, blockID string) (manifest, error) {
+func (m *Manager) loadManifestContent(ctx context.Context, contentID content.ID) (manifest, error) {
 	man := manifest{}
-	blk, err := m.b.GetBlock(ctx, blockID)
+	blk, err := m.b.GetContent(ctx, contentID)
 	if err != nil {
 		// do not wrap the error here, we want to propagate original ErrNotFound
 		// which causes a retry if we lose list/delete race.
@@ -394,17 +397,17 @@ func (m *Manager) loadManifestBlock(ctx context.Context, blockID string) (manife
 
 	gz, err := gzip.NewReader(bytes.NewReader(blk))
 	if err != nil {
-		return man, errors.Wrapf(err, "unable to unpack block %q", blockID)
+		return man, errors.Wrapf(err, "unable to unpack manifest data %q", contentID)
 	}
 
 	if err := json.NewDecoder(gz).Decode(&man); err != nil {
-		return man, errors.Wrapf(err, "unable to parse block %q", blockID)
+		return man, errors.Wrapf(err, "unable to parse manifest %q", contentID)
 	}
 
 	return man, nil
 }
 
-// Compact performs compaction of manifest blocks.
+// Compact performs compaction of manifest contents.
 func (m *Manager) Compact(ctx context.Context) error {
 	m.mu.Lock()
 	defer m.mu.Unlock()
@@ -413,26 +416,26 @@ func (m *Manager) Compact(ctx context.Context) error {
 }
 
 func (m *Manager) maybeCompactLocked(ctx context.Context) error {
-	if len(m.committedBlockIDs) < autoCompactionBlockCount {
+	if len(m.committedContentIDs) < autoCompactionContentCount {
 		return nil
 	}
 
-	log.Debugf("performing automatic compaction of %v blocks", len(m.committedBlockIDs))
+	log.Debugf("performing automatic compaction of %v contents", len(m.committedContentIDs))
 	if err := m.compactLocked(ctx); err != nil {
-		return errors.Wrap(err, "unable to compact manifest blocks")
+		return errors.Wrap(err, "unable to compact manifest contents")
 	}
 
 	if err := m.b.Flush(ctx); err != nil {
-		return errors.Wrap(err, "unable to flush blocks after auto-compaction")
+		return errors.Wrap(err, "unable to flush contents after auto-compaction")
 	}
 
 	return nil
 }
 
 func (m *Manager) compactLocked(ctx context.Context) error {
-	log.Debugf("compactLocked: pendingEntries=%v blockIDs=%v", len(m.pendingEntries), len(m.committedBlockIDs))
+	log.Debugf("compactLocked: pendingEntries=%v contentIDs=%v", len(m.pendingEntries), len(m.committedContentIDs))
 
-	if len(m.committedBlockIDs) == 1 && len(m.pendingEntries) == 0 {
+	if len(m.committedContentIDs) == 1 && len(m.pendingEntries) == 0 {
 		return nil
 	}
 
@@ -445,23 +448,23 @@ func (m *Manager) compactLocked(ctx context.Context) error {
 		m.pendingEntries[e.ID] = e
 	}
 
-	blockID, err := m.flushPendingEntriesLocked(ctx)
+	contentID, err := m.flushPendingEntriesLocked(ctx)
 	if err != nil {
 		return err
 	}
 
-	// add the newly-created block to the list, could be duplicate
-	for b := range m.committedBlockIDs {
-		if b == blockID {
-			// do not delete block that was just written.
+	// add the newly-created content to the list, could be duplicate
+	for b := range m.committedContentIDs {
+		if b == contentID {
+			// do not delete content that was just written.
 			continue
 		}
 
-		if err := m.b.DeleteBlock(b); err != nil {
-			return errors.Wrapf(err, "unable to delete block %q", b)
+		if err := m.b.DeleteContent(b); err != nil {
+			return errors.Wrapf(err, "unable to delete content %q", b)
 		}
 
-		delete(m.committedBlockIDs, b)
+		delete(m.committedContentIDs, b)
 	}
 
 	return nil
@@ -487,7 +490,7 @@ func (m *Manager) ensureInitialized(ctx context.Context) error {
 		return nil
 	}
 
-	if err := m.loadCommittedBlocksLocked(ctx); err != nil {
+	if err := m.loadCommittedContentsLocked(ctx); err != nil {
 		return err
 	}
 
@@ -503,13 +506,13 @@ func copyLabels(m map[string]string) map[string]string {
 	return r
 }
 
-// NewManager returns new manifest manager for the provided block manager.
-func NewManager(ctx context.Context, b blockManager) (*Manager, error) {
+// NewManager returns new manifest manager for the provided content manager.
+func NewManager(ctx context.Context, b contentManager) (*Manager, error) {
 	m := &Manager{
-		b:                 b,
-		pendingEntries:    map[string]*manifestEntry{},
-		committedEntries:  map[string]*manifestEntry{},
-		committedBlockIDs: map[string]bool{},
+		b:                   b,
+		pendingEntries:      map[ID]*manifestEntry{},
+		committedEntries:    map[ID]*manifestEntry{},
+		committedContentIDs: map[content.ID]bool{},
 	}
 
 	return m, nil

repo/manifest/manifest_manager_test.go

@@ -8,19 +8,14 @@
 	"testing"
 	"time"
 
-	"github.com/pkg/errors"
-
 	"github.com/kopia/kopia/internal/blobtesting"
-	"github.com/kopia/kopia/repo/block"
+	"github.com/kopia/kopia/repo/content"
 )
 
 func TestManifest(t *testing.T) {
 	ctx := context.Background()
 	data := blobtesting.DataMap{}
-	mgr, setupErr := newManagerForTesting(ctx, t, data)
-	if setupErr != nil {
-		t.Fatalf("unable to open block manager: %v", setupErr)
-	}
+	mgr := newManagerForTesting(ctx, t, data)
 
 	item1 := map[string]int{"foo": 1, "bar": 2}
 	item2 := map[string]int{"foo": 2, "bar": 3}
@@ -36,13 +31,13 @@ func TestManifest(t *testing.T) {
 
 	cases := []struct {
 		criteria map[string]string
-		expected []string
+		expected []ID
 	}{
-		{map[string]string{"color": "red"}, []string{id1, id3}},
-		{map[string]string{"color": "blue"}, []string{id2}},
+		{map[string]string{"color": "red"}, []ID{id1, id3}},
+		{map[string]string{"color": "blue"}, []ID{id2}},
 		{map[string]string{"color": "green"}, nil},
-		{map[string]string{"color": "red", "shape": "square"}, []string{id3}},
-		{map[string]string{"color": "blue", "shape": "square"}, []string{id2}},
+		{map[string]string{"color": "red", "shape": "square"}, []ID{id3}},
+		{map[string]string{"color": "blue", "shape": "square"}, []ID{id2}},
 		{map[string]string{"color": "red", "shape": "circle"}, nil},
 	}
 
@@ -69,12 +64,9 @@ func TestManifest(t *testing.T) {
 	verifyItem(ctx, t, mgr, id2, labels2, item2)
 	verifyItem(ctx, t, mgr, id3, labels3, item3)
 
-	// flush underlying block manager and verify in new manifest manager.
+	// flush underlying content manager and verify in new manifest manager.
 	mgr.b.Flush(ctx)
-	mgr2, setupErr := newManagerForTesting(ctx, t, data)
-	if setupErr != nil {
-		t.Fatalf("can't open block manager: %v", setupErr)
-	}
+	mgr2 := newManagerForTesting(ctx, t, data)
 	for _, tc := range cases {
 		verifyMatches(ctx, t, mgr2, tc.criteria, tc.expected)
 	}
@@ -96,7 +88,7 @@ func TestManifest(t *testing.T) {
 
 	// still found in another
 	verifyItem(ctx, t, mgr2, id3, labels3, item3)
-	if err := mgr2.loadCommittedBlocksLocked(ctx); err != nil {
+	if err := mgr2.loadCommittedContentsLocked(ctx); err != nil {
 		t.Errorf("unable to load: %v", err)
 	}
 
@@ -104,7 +96,7 @@ func TestManifest(t *testing.T) {
 		t.Errorf("can't compact: %v", err)
 	}
 
-	blks, err := mgr.b.ListBlocks(manifestBlockPrefix)
+	blks, err := mgr.b.ListContents(manifestContentPrefix)
 	if err != nil {
 		t.Errorf("unable to list manifest blocks: %v", err)
 	}
@@ -114,10 +106,7 @@ func TestManifest(t *testing.T) {
 
 	mgr.b.Flush(ctx)
 
-	mgr3, err := newManagerForTesting(ctx, t, data)
-	if err != nil {
-		t.Fatalf("can't open manager: %v", err)
-	}
+	mgr3 := newManagerForTesting(ctx, t, data)
 
 	verifyItem(ctx, t, mgr3, id1, labels1, item1)
 	verifyItem(ctx, t, mgr3, id2, labels2, item2)
@@ -129,7 +118,7 @@ func TestManifestInitCorruptedBlock(t *testing.T) {
 	data := blobtesting.DataMap{}
 	st := blobtesting.NewMapStorage(data, nil, nil)
 
-	f := block.FormattingOptions{
+	f := content.FormattingOptions{
 		Hash:        "HMAC-SHA256-128",
 		Encryption:  "NONE",
 		MaxPackSize: 100000,
@@ -137,7 +126,7 @@ func TestManifestInitCorruptedBlock(t *testing.T) {
 	}
 
 	// write some data to storage
-	bm, err := block.NewManager(ctx, st, f, block.CachingOptions{}, nil)
+	bm, err := content.NewManager(ctx, st, f, content.CachingOptions{}, nil)
 	if err != nil {
 		t.Fatalf("err: %v", err)
 	}
@@ -160,8 +149,8 @@ func TestManifestInitCorruptedBlock(t *testing.T) {
 		}
 	}
 
-	// make a new block manager based on corrupted data.
-	bm, err = block.NewManager(ctx, st, f, block.CachingOptions{}, nil)
+	// make a new content manager based on corrupted data.
+	bm, err = content.NewManager(ctx, st, f, content.CachingOptions{}, nil)
 	if err != nil {
 		t.Fatalf("err: %v", err)
 	}
@@ -200,7 +189,7 @@ func TestManifestInitCorruptedBlock(t *testing.T) {
 	}
 }
 
-func addAndVerify(ctx context.Context, t *testing.T, mgr *Manager, labels map[string]string, data map[string]int) string {
+func addAndVerify(ctx context.Context, t *testing.T, mgr *Manager, labels map[string]string, data map[string]int) ID {
 	t.Helper()
 	id, err := mgr.Put(ctx, labels, data)
 	if err != nil {
@@ -212,7 +201,7 @@ func addAndVerify(ctx context.Context, t *testing.T, mgr *Manager, labels map[st
 	return id
 }
 
-func verifyItem(ctx context.Context, t *testing.T, mgr *Manager, id string, labels map[string]string, data map[string]int) {
+func verifyItem(ctx context.Context, t *testing.T, mgr *Manager, id ID, labels map[string]string, data map[string]int) {
 	t.Helper()
 
 	l, err := mgr.GetMetadata(ctx, id)
@@ -235,7 +224,7 @@ func verifyItem(ctx context.Context, t *testing.T, mgr *Manager, id string, labe
 	}
 }
 
-func verifyItemNotFound(ctx context.Context, t *testing.T, mgr *Manager, id string) {
+func verifyItemNotFound(ctx context.Context, t *testing.T, mgr *Manager, id ID) {
 	t.Helper()
 
 	_, err := mgr.GetMetadata(ctx, id)
@@ -245,10 +234,10 @@ func verifyItemNotFound(ctx context.Context, t *testing.T, mgr *Manager, id stri
 	}
 }
 
-func verifyMatches(ctx context.Context, t *testing.T, mgr *Manager, labels map[string]string, expected []string) {
+func verifyMatches(ctx context.Context, t *testing.T, mgr *Manager, labels map[string]string, expected []ID) {
 	t.Helper()
 
-	var matches []string
+	var matches []ID
 	items, err := mgr.Find(ctx, labels)
 	if err != nil {
 		t.Errorf("error in Find(): %v", err)
@@ -257,37 +246,45 @@ func verifyMatches(ctx context.Context, t *testing.T, mgr *Manager, labels map[s
 	for _, m := range items {
 		matches = append(matches, m.ID)
 	}
-	sort.Strings(matches)
-	sort.Strings(expected)
+	sortIDs(matches)
+	sortIDs(expected)
 
 	if !reflect.DeepEqual(matches, expected) {
 		t.Errorf("invalid matches for %v: %v, expected %v", labels, matches, expected)
 	}
 }
 
-func newManagerForTesting(ctx context.Context, t *testing.T, data blobtesting.DataMap) (*Manager, error) {
+func sortIDs(s []ID) {
+	sort.Slice(s, func(i, j int) bool {
+		return s[i] < s[j]
+	})
+}
+
+func newManagerForTesting(ctx context.Context, t *testing.T, data blobtesting.DataMap) *Manager {
 	st := blobtesting.NewMapStorage(data, nil, nil)
 
-	bm, err := block.NewManager(ctx, st, block.FormattingOptions{
+	bm, err := content.NewManager(ctx, st, content.FormattingOptions{
 		Hash:        "HMAC-SHA256-128",
 		Encryption:  "NONE",
 		MaxPackSize: 100000,
 		Version:     1,
-	}, block.CachingOptions{}, nil)
+	}, content.CachingOptions{}, nil)
 	if err != nil {
-		return nil, errors.Wrap(err, "can't create block manager")
+		t.Fatalf("can't create content manager: %v", err)
 	}
 
-	return NewManager(ctx, bm)
+	mm, err := NewManager(ctx, bm)
+	if err != nil {
+		t.Fatalf("can't create manifest manager: %v", err)
+	}
+
+	return mm
 }
 
 func TestManifestInvalidPut(t *testing.T) {
 	ctx := context.Background()
 	data := blobtesting.DataMap{}
-	mgr, setupErr := newManagerForTesting(ctx, t, data)
-	if setupErr != nil {
-		t.Fatalf("unable to open block manager: %v", setupErr)
-	}
+	mgr := newManagerForTesting(ctx, t, data)
 
 	cases := []struct {
 		labels        map[string]string
@@ -311,10 +308,7 @@ func TestManifestAutoCompaction(t *testing.T) {
 	data := blobtesting.DataMap{}
 
 	for i := 0; i < 100; i++ {
-		mgr, setupErr := newManagerForTesting(ctx, t, data)
-		if setupErr != nil {
-			t.Fatalf("unable to open block manager: %v", setupErr)
-		}
+		mgr := newManagerForTesting(ctx, t, data)
 
 		item1 := map[string]int{"foo": 1, "bar": 2}
 		labels1 := map[string]string{"type": "item", "color": "red"}

repo/manifest/serialized.go

@@ -10,7 +10,7 @@ type manifest struct {
 }
 
 type manifestEntry struct {
-	ID      string            `json:"id"`
+	ID      ID                `json:"id"`
 	Labels  map[string]string `json:"labels"`
 	ModTime time.Time         `json:"modified"`
 	Deleted bool              `json:"deleted,omitempty"`

repo/object/object_manager.go

@@ -9,7 +9,7 @@
 
 	"github.com/pkg/errors"
 
-	"github.com/kopia/kopia/repo/block"
+	"github.com/kopia/kopia/repo/content"
 )
 
 // ErrObjectNotFound is returned when an object cannot be found.
@@ -23,23 +23,23 @@ type Reader interface {
 	Length() int64
 }
 
-type blockManager interface {
-	BlockInfo(ctx context.Context, blockID string) (block.Info, error)
-	GetBlock(ctx context.Context, blockID string) ([]byte, error)
-	WriteBlock(ctx context.Context, data []byte, prefix string) (string, error)
+type contentManager interface {
+	ContentInfo(ctx context.Context, contentID content.ID) (content.Info, error)
+	GetContent(ctx context.Context, contentID content.ID) ([]byte, error)
+	WriteContent(ctx context.Context, data []byte, prefix content.ID) (content.ID, error)
 }
 
 // Format describes the format of objects in a repository.
 type Format struct {
-	Splitter string `json:"splitter,omitempty"` // splitter used to break objects into storage blocks
+	Splitter string `json:"splitter,omitempty"` // splitter used to break objects into pieces of content
 }
 
 // Manager implements a content-addressable storage on top of blob storage.
 type Manager struct {
 	Format Format
 
-	blockMgr blockManager
-	trace    func(message string, args ...interface{})
+	contentMgr contentManager
+	trace      func(message string, args ...interface{})
 
 	newSplitter func() Splitter
 }
@@ -86,19 +86,19 @@ func (om *Manager) Open(ctx context.Context, objectID ID) (Reader, error) {
 }
 
 // VerifyObject ensures that all objects backing ObjectID are present in the repository
-// and returns the total length of the object and storage blocks of which it is composed.
-func (om *Manager) VerifyObject(ctx context.Context, oid ID) (int64, []string, error) {
-	blocks := &blockTracker{}
-	l, err := om.verifyObjectInternal(ctx, oid, blocks)
+// and returns the total length of the object and content IDs of which it is composed.
+func (om *Manager) VerifyObject(ctx context.Context, oid ID) (int64, []content.ID, error) {
+	tracker := &contentIDTracker{}
+	l, err := om.verifyObjectInternal(ctx, oid, tracker)
 	if err != nil {
 		return 0, nil, err
 	}
 
-	return l, blocks.blockIDs(), nil
+	return l, tracker.contentIDs(), nil
 }
 
-func (om *Manager) verifyIndirectObjectInternal(ctx context.Context, indexObjectID ID, blocks *blockTracker) (int64, error) {
-	if _, err := om.verifyObjectInternal(ctx, indexObjectID, blocks); err != nil {
+func (om *Manager) verifyIndirectObjectInternal(ctx context.Context, indexObjectID ID, tracker *contentIDTracker) (int64, error) {
+	if _, err := om.verifyObjectInternal(ctx, indexObjectID, tracker); err != nil {
 		return 0, errors.Wrap(err, "unable to read index")
 	}
 	rd, err := om.Open(ctx, indexObjectID)
@@ -113,7 +113,7 @@ func (om *Manager) verifyIndirectObjectInternal(ctx context.Context, indexObject
 	}
 
 	for i, m := range seekTable {
-		l, err := om.verifyObjectInternal(ctx, m.Object, blocks)
+		l, err := om.verifyObjectInternal(ctx, m.Object, tracker)
 		if err != nil {
 			return 0, err
 		}
@@ -127,17 +127,17 @@ func (om *Manager) verifyIndirectObjectInternal(ctx context.Context, indexObject
 	return totalLength, nil
 }
 
-func (om *Manager) verifyObjectInternal(ctx context.Context, oid ID, blocks *blockTracker) (int64, error) {
+func (om *Manager) verifyObjectInternal(ctx context.Context, oid ID, tracker *contentIDTracker) (int64, error) {
 	if indexObjectID, ok := oid.IndexObjectID(); ok {
-		return om.verifyIndirectObjectInternal(ctx, indexObjectID, blocks)
+		return om.verifyIndirectObjectInternal(ctx, indexObjectID, tracker)
 	}
 
-	if blockID, ok := oid.BlockID(); ok {
-		p, err := om.blockMgr.BlockInfo(ctx, blockID)
+	if contentID, ok := oid.ContentID(); ok {
+		p, err := om.contentMgr.ContentInfo(ctx, contentID)
 		if err != nil {
 			return 0, err
 		}
-		blocks.addBlock(blockID)
+		tracker.addContentID(contentID)
 		return int64(p.Length), nil
 	}
 
@@ -153,12 +153,12 @@ type ManagerOptions struct {
 	Trace func(message string, args ...interface{})
 }
 
-// NewObjectManager creates an ObjectManager with the specified block manager and format.
-func NewObjectManager(ctx context.Context, bm blockManager, f Format, opts ManagerOptions) (*Manager, error) {
+// NewObjectManager creates an ObjectManager with the specified content manager and format.
+func NewObjectManager(ctx context.Context, bm contentManager, f Format, opts ManagerOptions) (*Manager, error) {
 	om := &Manager{
-		blockMgr: bm,
-		Format:   f,
-		trace:    nullTrace,
+		contentMgr: bm,
+		Format:     f,
+		trace:      nullTrace,
 	}
 
 	splitterID := f.Splitter
@@ -210,13 +210,13 @@ func (om *Manager) flattenListChunk(rawReader io.Reader) ([]indirectObjectEntry,
 }
 
 func (om *Manager) newRawReader(ctx context.Context, objectID ID) (Reader, error) {
-	if blockID, ok := objectID.BlockID(); ok {
-		payload, err := om.blockMgr.GetBlock(ctx, blockID)
-		if err == block.ErrBlockNotFound {
+	if contentID, ok := objectID.ContentID(); ok {
+		payload, err := om.contentMgr.GetContent(ctx, contentID)
+		if err == content.ErrContentNotFound {
 			return nil, ErrObjectNotFound
 		}
 		if err != nil {
-			return nil, errors.Wrap(err, "unexpected block error")
+			return nil, errors.Wrap(err, "unexpected content error")
 		}
 
 		return newObjectReaderWithData(payload), nil

repo/object/object_manager_test.go

@@ -16,58 +16,58 @@
 	"testing"
 
 	"github.com/kopia/kopia/repo/blob"
-	"github.com/kopia/kopia/repo/block"
+	"github.com/kopia/kopia/repo/content"
 )
 
-type fakeBlockManager struct {
+type fakeContentManager struct {
 	mu   sync.Mutex
-	data map[string][]byte
+	data map[content.ID][]byte
 }
 
-func (f *fakeBlockManager) GetBlock(ctx context.Context, blockID string) ([]byte, error) {
+func (f *fakeContentManager) GetContent(ctx context.Context, contentID content.ID) ([]byte, error) {
 	f.mu.Lock()
 	defer f.mu.Unlock()
 
-	if d, ok := f.data[blockID]; ok {
+	if d, ok := f.data[contentID]; ok {
 		return append([]byte(nil), d...), nil
 	}
 
-	return nil, block.ErrBlockNotFound
+	return nil, content.ErrContentNotFound
 }
 
-func (f *fakeBlockManager) WriteBlock(ctx context.Context, data []byte, prefix string) (string, error) {
+func (f *fakeContentManager) WriteContent(ctx context.Context, data []byte, prefix content.ID) (content.ID, error) {
 	h := sha256.New()
 	h.Write(data) //nolint:errcheck
-	blockID := prefix + string(hex.EncodeToString(h.Sum(nil)))
+	contentID := prefix + content.ID(hex.EncodeToString(h.Sum(nil)))
 
 	f.mu.Lock()
 	defer f.mu.Unlock()
 
-	f.data[blockID] = append([]byte(nil), data...)
-	return blockID, nil
+	f.data[contentID] = append([]byte(nil), data...)
+	return contentID, nil
 }
 
-func (f *fakeBlockManager) BlockInfo(ctx context.Context, blockID string) (block.Info, error) {
+func (f *fakeContentManager) ContentInfo(ctx context.Context, contentID content.ID) (content.Info, error) {
 	f.mu.Lock()
 	defer f.mu.Unlock()
 
-	if d, ok := f.data[blockID]; ok {
-		return block.Info{BlockID: blockID, Length: uint32(len(d))}, nil
+	if d, ok := f.data[contentID]; ok {
+		return content.Info{ID: contentID, Length: uint32(len(d))}, nil
 	}
 
-	return block.Info{}, blob.ErrBlobNotFound
+	return content.Info{}, blob.ErrBlobNotFound
 }
 
-func (f *fakeBlockManager) Flush(ctx context.Context) error {
+func (f *fakeContentManager) Flush(ctx context.Context) error {
 	return nil
 }
 
-func setupTest(t *testing.T) (map[string][]byte, *Manager) {
-	return setupTestWithData(t, map[string][]byte{}, ManagerOptions{})
+func setupTest(t *testing.T) (map[content.ID][]byte, *Manager) {
+	return setupTestWithData(t, map[content.ID][]byte{}, ManagerOptions{})
 }
 
-func setupTestWithData(t *testing.T, data map[string][]byte, opts ManagerOptions) (map[string][]byte, *Manager) {
-	r, err := NewObjectManager(context.Background(), &fakeBlockManager{data: data}, Format{
+func setupTestWithData(t *testing.T, data map[content.ID][]byte, opts ManagerOptions) (map[content.ID][]byte, *Manager) {
+	r, err := NewObjectManager(context.Background(), &fakeContentManager{data: data}, Format{
 		Splitter: "FIXED-1M",
 	}, opts)
 	if err != nil {
@@ -109,7 +109,7 @@ func TestWriters(t *testing.T) {
 			t.Errorf("incorrect result for %v, expected: %v got: %v", c.data, c.objectID.String(), result.String())
 		}
 
-		if _, ok := c.objectID.BlockID(); !ok {
+		if _, ok := c.objectID.ContentID(); !ok {
 			if len(data) != 0 {
 				t.Errorf("unexpected data written to the storage: %v", data)
 			}
@@ -162,18 +162,18 @@ func TestIndirection(t *testing.T) {
 	splitterFactory := newFixedSplitterFactory(1000)
 	cases := []struct {
 		dataLength          int
-		expectedBlockCount  int
+		expectedBlobCount   int
 		expectedIndirection int
 	}{
-		{dataLength: 200, expectedBlockCount: 1, expectedIndirection: 0},
-		{dataLength: 1000, expectedBlockCount: 1, expectedIndirection: 0},
-		{dataLength: 1001, expectedBlockCount: 3, expectedIndirection: 1},
-		// 1 block of 1000 zeros, 1 block of 5 zeros + 1 index blob
-		{dataLength: 3005, expectedBlockCount: 3, expectedIndirection: 1},
-		// 1 block of 1000 zeros + 1 index blob
-		{dataLength: 4000, expectedBlockCount: 2, expectedIndirection: 1},
-		// 1 block of 1000 zeros + 1 index blob
-		{dataLength: 10000, expectedBlockCount: 2, expectedIndirection: 1},
+		{dataLength: 200, expectedBlobCount: 1, expectedIndirection: 0},
+		{dataLength: 1000, expectedBlobCount: 1, expectedIndirection: 0},
+		{dataLength: 1001, expectedBlobCount: 3, expectedIndirection: 1},
+		// 1 blob of 1000 zeros, 1 blob of 5 zeros + 1 index blob
+		{dataLength: 3005, expectedBlobCount: 3, expectedIndirection: 1},
+		// 1 blob of 1000 zeros + 1 index blob
+		{dataLength: 4000, expectedBlobCount: 2, expectedIndirection: 1},
+		// 1 blob of 1000 zeros + 1 index blob
+		{dataLength: 10000, expectedBlobCount: 2, expectedIndirection: 1},
 	}
 
 	for _, c := range cases {
@@ -197,8 +197,8 @@ func TestIndirection(t *testing.T) {
 			t.Errorf("incorrect indirection level for size: %v: %v, expected %v", c.dataLength, indirectionLevel(result), c.expectedIndirection)
 		}
 
-		if got, want := len(data), c.expectedBlockCount; got != want {
-			t.Errorf("unexpected block count for %v: %v, expected %v", c.dataLength, got, want)
+		if got, want := len(data), c.expectedBlobCount; got != want {
+			t.Errorf("unexpected blob count for %v: %v, expected %v", c.dataLength, got, want)
 		}
 
 		l, b, err := om.VerifyObject(ctx, result)
@@ -210,8 +210,8 @@ func TestIndirection(t *testing.T) {
 			t.Errorf("got invalid byte count for %q: %v, wanted %v", result, got, want)
 		}
 
-		if got, want := len(b), c.expectedBlockCount; got != want {
-			t.Errorf("invalid block count for %v, got %v, wanted %v", result, got, want)
+		if got, want := len(b), c.expectedBlobCount; got != want {
+			t.Errorf("invalid blob count for %v, got %v, wanted %v", result, got, want)
 		}
 
 		verifyIndirectBlock(ctx, t, om, result)

repo/object/object_reader.go

@@ -68,14 +68,14 @@ func (r *objectReader) Read(buffer []byte) (int, error) {
 
 func (r *objectReader) openCurrentChunk() error {
 	st := r.seekTable[r.currentChunkIndex]
-	blockData, err := r.repo.Open(r.ctx, st.Object)
+	rd, err := r.repo.Open(r.ctx, st.Object)
 	if err != nil {
 		return err
 	}
-	defer blockData.Close() //nolint:errcheck
+	defer rd.Close() //nolint:errcheck
 
 	b := make([]byte, st.Length)
-	if _, err := io.ReadFull(blockData, b); err != nil {
+	if _, err := io.ReadFull(rd, b); err != nil {
 		return err
 	}
 

repo/object/object_writer.go

@@ -8,6 +8,8 @@
 	"sync"
 
 	"github.com/pkg/errors"
+
+	"github.com/kopia/kopia/repo/content"
 )
 
 // Writer allows writing content to the storage and supports automatic deduplication and encryption
@@ -18,27 +20,27 @@ type Writer interface {
 	Result() (ID, error)
 }
 
-type blockTracker struct {
-	mu     sync.Mutex
-	blocks map[string]bool
+type contentIDTracker struct {
+	mu       sync.Mutex
+	contents map[content.ID]bool
 }
 
-func (t *blockTracker) addBlock(blockID string) {
+func (t *contentIDTracker) addContentID(contentID content.ID) {
 	t.mu.Lock()
 	defer t.mu.Unlock()
 
-	if t.blocks == nil {
-		t.blocks = make(map[string]bool)
+	if t.contents == nil {
+		t.contents = make(map[content.ID]bool)
 	}
-	t.blocks[blockID] = true
+	t.contents[contentID] = true
 }
 
-func (t *blockTracker) blockIDs() []string {
+func (t *contentIDTracker) contentIDs() []content.ID {
 	t.mu.Lock()
 	defer t.mu.Unlock()
 
-	result := make([]string, 0, len(t.blocks))
-	for k := range t.blocks {
+	result := make([]content.ID, 0, len(t.contents))
+	for k := range t.contents {
 		result = append(result, k)
 	}
 	return result
@@ -48,12 +50,12 @@ type objectWriter struct {
 	ctx  context.Context
 	repo *Manager
 
-	prefix      string
+	prefix      content.ID
 	buffer      bytes.Buffer
 	totalLength int64
 
 	currentPosition int64
-	blockIndex      []indirectObjectEntry
+	indirectIndex   []indirectObjectEntry
 
 	description string
 
@@ -83,35 +85,35 @@ func (w *objectWriter) Write(data []byte) (n int, err error) {
 
 func (w *objectWriter) flushBuffer() error {
 	length := w.buffer.Len()
-	chunkID := len(w.blockIndex)
-	w.blockIndex = append(w.blockIndex, indirectObjectEntry{})
-	w.blockIndex[chunkID].Start = w.currentPosition
-	w.blockIndex[chunkID].Length = int64(length)
+	chunkID := len(w.indirectIndex)
+	w.indirectIndex = append(w.indirectIndex, indirectObjectEntry{})
+	w.indirectIndex[chunkID].Start = w.currentPosition
+	w.indirectIndex[chunkID].Length = int64(length)
 	w.currentPosition += int64(length)
 
 	var b2 bytes.Buffer
 	w.buffer.WriteTo(&b2) //nolint:errcheck
 	w.buffer.Reset()
 
-	blockID, err := w.repo.blockMgr.WriteBlock(w.ctx, b2.Bytes(), w.prefix)
-	w.repo.trace("OBJECT_WRITER(%q) stored %v (%v bytes)", w.description, blockID, length)
+	contentID, err := w.repo.contentMgr.WriteContent(w.ctx, b2.Bytes(), w.prefix)
+	w.repo.trace("OBJECT_WRITER(%q) stored %v (%v bytes)", w.description, contentID, length)
 	if err != nil {
 		return errors.Wrapf(err, "error when flushing chunk %d of %s", chunkID, w.description)
 	}
 
-	w.blockIndex[chunkID].Object = DirectObjectID(blockID)
+	w.indirectIndex[chunkID].Object = DirectObjectID(contentID)
 	return nil
 }
 
 func (w *objectWriter) Result() (ID, error) {
-	if w.buffer.Len() > 0 || len(w.blockIndex) == 0 {
+	if w.buffer.Len() > 0 || len(w.indirectIndex) == 0 {
 		if err := w.flushBuffer(); err != nil {
 			return "", err
 		}
 	}
 
-	if len(w.blockIndex) == 1 {
-		return w.blockIndex[0].Object, nil
+	if len(w.indirectIndex) == 1 {
+		return w.indirectIndex[0].Object, nil
 	}
 
 	iw := &objectWriter{
@@ -124,11 +126,11 @@ func (w *objectWriter) Result() (ID, error) {
 
 	ind := indirectObject{
 		StreamID: "kopia:indirect",
-		Entries:  w.blockIndex,
+		Entries:  w.indirectIndex,
 	}
 
 	if err := json.NewEncoder(iw).Encode(ind); err != nil {
-		return "", errors.Wrap(err, "unable to write indirect block index")
+		return "", errors.Wrap(err, "unable to write indirect object index")
 	}
 	oid, err := iw.Result()
 	if err != nil {
@@ -140,5 +142,5 @@ func (w *objectWriter) Result() (ID, error) {
 // WriterOptions can be passed to Repository.NewWriter()
 type WriterOptions struct {
 	Description string
-	Prefix      string // empty string or a single-character ('g'..'z')
+	Prefix      content.ID // empty string or a single-character ('g'..'z')
 }

repo/object/objectid.go

@@ -5,6 +5,8 @@
 	"strings"
 
 	"github.com/pkg/errors"
+
+	"github.com/kopia/kopia/repo/content"
 )
 
 // ID is an identifier of a repository object. Repository objects can be stored.
@@ -33,16 +35,16 @@ func (i ID) IndexObjectID() (ID, bool) {
 	return "", false
 }
 
-// BlockID returns the block ID of the underlying content storage block.
-func (i ID) BlockID() (string, bool) {
+// ContentID returns the ID of the underlying content.
+func (i ID) ContentID() (content.ID, bool) {
 	if strings.HasPrefix(string(i), "D") {
-		return string(i[1:]), true
+		return content.ID(i[1:]), true
 	}
 	if strings.HasPrefix(string(i), "I") {
 		return "", false
 	}
 
-	return string(i), true
+	return content.ID(i), true
 }
 
 // Validate checks the ID format for validity and reports any errors.
@@ -55,21 +57,21 @@ func (i ID) Validate() error {
 		return nil
 	}
 
-	if blockID, ok := i.BlockID(); ok {
-		if len(blockID) < 2 {
-			return errors.Errorf("missing block ID")
+	if contentID, ok := i.ContentID(); ok {
+		if len(contentID) < 2 {
+			return errors.Errorf("missing content ID")
 		}
 
 		// odd length - firstcharacter must be a single character between 'g' and 'z'
-		if len(blockID)%2 == 1 {
-			if blockID[0] < 'g' || blockID[0] > 'z' {
-				return errors.Errorf("invalid block ID prefix: %v", blockID)
+		if len(contentID)%2 == 1 {
+			if contentID[0] < 'g' || contentID[0] > 'z' {
+				return errors.Errorf("invalid content ID prefix: %v", contentID)
 			}
-			blockID = blockID[1:]
+			contentID = contentID[1:]
 		}
 
-		if _, err := hex.DecodeString(blockID); err != nil {
-			return errors.Errorf("invalid blockID suffix, must be base-16 encoded: %v", blockID)
+		if _, err := hex.DecodeString(string(contentID)); err != nil {
+			return errors.Errorf("invalid contentID suffix, must be base-16 encoded: %v", contentID)
 		}
 
 		return nil
@@ -79,8 +81,8 @@ func (i ID) Validate() error {
 }
 
 // DirectObjectID returns direct object ID based on the provided block ID.
-func DirectObjectID(blockID string) ID {
-	return ID(blockID)
+func DirectObjectID(contentID content.ID) ID {
+	return ID(contentID)
 }
 
 // IndirectObjectID returns indirect object ID based on the underlying index object ID.

repo/open.go

@@ -11,7 +11,7 @@
 	"github.com/kopia/kopia/internal/repologging"
 	"github.com/kopia/kopia/repo/blob"
 	"github.com/kopia/kopia/repo/blob/logging"
-	"github.com/kopia/kopia/repo/block"
+	"github.com/kopia/kopia/repo/content"
 	"github.com/kopia/kopia/repo/manifest"
 	"github.com/kopia/kopia/repo/object"
 )
@@ -75,20 +75,20 @@ func Open(ctx context.Context, configFile string, password string, options *Opti
 }
 
 // OpenWithConfig opens the repository with a given configuration, avoiding the need for a config file.
-func OpenWithConfig(ctx context.Context, st blob.Storage, lc *LocalConfig, password string, options *Options, caching block.CachingOptions) (*Repository, error) {
-	log.Debugf("reading encrypted format block")
-	// Read cache block, potentially from cache.
-	fb, err := readAndCacheFormatBlockBytes(ctx, st, caching.CacheDirectory)
+func OpenWithConfig(ctx context.Context, st blob.Storage, lc *LocalConfig, password string, options *Options, caching content.CachingOptions) (*Repository, error) {
+	log.Debugf("reading encrypted format blob")
+	// Read format blob, potentially from cache.
+	fb, err := readAndCacheFormatBlobBytes(ctx, st, caching.CacheDirectory)
 	if err != nil {
-		return nil, errors.Wrap(err, "unable to read format block")
+		return nil, errors.Wrap(err, "unable to read format blob")
 	}
 
-	f, err := parseFormatBlock(fb)
+	f, err := parseFormatBlob(fb)
 	if err != nil {
-		return nil, errors.Wrap(err, "can't parse format block")
+		return nil, errors.Wrap(err, "can't parse format blob")
 	}
 
-	fb, err = addFormatBlockChecksumAndLength(fb)
+	fb, err = addFormatBlobChecksumAndLength(fb)
 	if err != nil {
 		return nil, errors.Errorf("unable to add checksum")
 	}
@@ -110,39 +110,39 @@ func OpenWithConfig(ctx context.Context, st blob.Storage, lc *LocalConfig, passw
 		fo.MaxPackSize = 20 << 20 // 20 MB
 	}
 
-	log.Debugf("initializing block manager")
-	bm, err := block.NewManager(ctx, st, fo, caching, fb)
+	log.Debugf("initializing content-addressable storage manager")
+	cm, err := content.NewManager(ctx, st, fo, caching, fb)
 	if err != nil {
-		return nil, errors.Wrap(err, "unable to open block manager")
+		return nil, errors.Wrap(err, "unable to open content manager")
 	}
 
 	log.Debugf("initializing object manager")
-	om, err := object.NewObjectManager(ctx, bm, repoConfig.Format, options.ObjectManagerOptions)
+	om, err := object.NewObjectManager(ctx, cm, repoConfig.Format, options.ObjectManagerOptions)
 	if err != nil {
 		return nil, errors.Wrap(err, "unable to open object manager")
 	}
 
 	log.Debugf("initializing manifest manager")
-	manifests, err := manifest.NewManager(ctx, bm)
+	manifests, err := manifest.NewManager(ctx, cm)
 	if err != nil {
 		return nil, errors.Wrap(err, "unable to open manifests")
 	}
 
 	return &Repository{
-		Blocks:         bm,
+		Content:        cm,
 		Objects:        om,
 		Blobs:          st,
 		Manifests:      manifests,
 		CacheDirectory: caching.CacheDirectory,
 		UniqueID:       f.UniqueID,
 
-		formatBlock: f,
-		masterKey:   masterKey,
+		formatBlob: f,
+		masterKey:  masterKey,
 	}, nil
 }
 
 // SetCachingConfig changes caching configuration for a given repository config file.
-func SetCachingConfig(ctx context.Context, configFile string, opt block.CachingOptions) error {
+func SetCachingConfig(ctx context.Context, configFile string, opt content.CachingOptions) error {
 	configFile, err := filepath.Abs(configFile)
 	if err != nil {
 		return err
@@ -158,14 +158,14 @@ func SetCachingConfig(ctx context.Context, configFile string, opt block.CachingO
 		return errors.Wrap(err, "cannot open storage")
 	}
 
-	fb, err := readAndCacheFormatBlockBytes(ctx, st, "")
+	fb, err := readAndCacheFormatBlobBytes(ctx, st, "")
 	if err != nil {
-		return errors.Wrap(err, "can't read format block")
+		return errors.Wrap(err, "can't read format blob")
 	}
 
-	f, err := parseFormatBlock(fb)
+	f, err := parseFormatBlob(fb)
 	if err != nil {
-		return errors.Wrap(err, "can't parse format block")
+		return errors.Wrap(err, "can't parse format blob")
 	}
 
 	if err = setupCaching(configFile, lc, opt, f.UniqueID); err != nil {
@@ -184,7 +184,7 @@ func SetCachingConfig(ctx context.Context, configFile string, opt block.CachingO
 	return nil
 }
 
-func readAndCacheFormatBlockBytes(ctx context.Context, st blob.Storage, cacheDirectory string) ([]byte, error) {
+func readAndCacheFormatBlobBytes(ctx context.Context, st blob.Storage, cacheDirectory string) ([]byte, error) {
 	cachedFile := filepath.Join(cacheDirectory, "kopia.repository")
 	if cacheDirectory != "" {
 		b, err := ioutil.ReadFile(cachedFile)

repo/repository.go

@@ -7,7 +7,7 @@
 	"github.com/pkg/errors"
 
 	"github.com/kopia/kopia/repo/blob"
-	"github.com/kopia/kopia/repo/block"
+	"github.com/kopia/kopia/repo/content"
 	"github.com/kopia/kopia/repo/manifest"
 	"github.com/kopia/kopia/repo/object"
 )
@@ -15,7 +15,7 @@
 // Repository represents storage where both content-addressable and user-addressable data is kept.
 type Repository struct {
 	Blobs     blob.Storage
-	Blocks    *block.Manager
+	Content   *content.Manager
 	Objects   *object.Manager
 	Manifests *manifest.Manager
 	UniqueID  []byte
@@ -23,8 +23,8 @@ type Repository struct {
 	ConfigFile     string
 	CacheDirectory string
 
-	formatBlock *formatBlock
-	masterKey   []byte
+	formatBlob *formatBlob
+	masterKey  []byte
 }
 
 // Close closes the repository and releases all resources.
@@ -32,8 +32,8 @@ func (r *Repository) Close(ctx context.Context) error {
 	if err := r.Manifests.Flush(ctx); err != nil {
 		return errors.Wrap(err, "error flushing manifests")
 	}
-	if err := r.Blocks.Flush(ctx); err != nil {
-		return errors.Wrap(err, "error closing blocks")
+	if err := r.Content.Flush(ctx); err != nil {
+		return errors.Wrap(err, "error closing content-addressable storage manager")
 	}
 	if err := r.Blobs.Close(ctx); err != nil {
 		return errors.Wrap(err, "error closing blob storage")
@@ -47,21 +47,21 @@ func (r *Repository) Flush(ctx context.Context) error {
 		return err
 	}
 
-	return r.Blocks.Flush(ctx)
+	return r.Content.Flush(ctx)
 }
 
 // Refresh periodically makes external changes visible to repository.
 func (r *Repository) Refresh(ctx context.Context) error {
-	updated, err := r.Blocks.Refresh(ctx)
+	updated, err := r.Content.Refresh(ctx)
 	if err != nil {
-		return errors.Wrap(err, "error refreshing block index")
+		return errors.Wrap(err, "error refreshing content index")
 	}
 
 	if !updated {
 		return nil
 	}
 
-	log.Debugf("block index refreshed")
+	log.Debugf("content index refreshed")
 
 	if err := r.Manifests.Refresh(ctx); err != nil {
 		return errors.Wrap(err, "error reloading manifests")

repo/repository_test.go

@@ -13,7 +13,7 @@
 
 	"github.com/kopia/kopia/internal/repotesting"
 	"github.com/kopia/kopia/repo"
-	"github.com/kopia/kopia/repo/block"
+	"github.com/kopia/kopia/repo/content"
 	"github.com/kopia/kopia/repo/object"
 )
 
@@ -50,7 +50,7 @@ func TestWriters(t *testing.T) {
 			t.Errorf("incorrect result for %v, expected: %v got: %v", c.data, c.objectID.String(), result.String())
 		}
 
-		env.Repository.Blocks.Flush(ctx)
+		env.Repository.Content.Flush(ctx)
 	}
 }
 
@@ -95,7 +95,7 @@ func TestPackingSimple(t *testing.T) {
 	oid2c := writeObject(ctx, t, env.Repository, []byte(content2), "packed-object-2c")
 	oid1c := writeObject(ctx, t, env.Repository, []byte(content1), "packed-object-1c")
 
-	env.Repository.Blocks.Flush(ctx)
+	env.Repository.Content.Flush(ctx)
 
 	if got, want := oid1a.String(), oid1b.String(); got != want {
 		t.Errorf("oid1a(%q) != oid1b(%q)", got, want)
@@ -113,7 +113,7 @@ func TestPackingSimple(t *testing.T) {
 		t.Errorf("oid3a(%q) != oid3b(%q)", got, want)
 	}
 
-	env.VerifyStorageBlockCount(t, 3)
+	env.VerifyBlobCount(t, 3)
 
 	env.MustReopen(t)
 
@@ -121,7 +121,7 @@ func TestPackingSimple(t *testing.T) {
 	verify(ctx, t, env.Repository, oid2a, []byte(content2), "packed-object-2")
 	verify(ctx, t, env.Repository, oid3a, []byte(content3), "packed-object-3")
 
-	if err := env.Repository.Blocks.CompactIndexes(ctx, block.CompactOptions{MinSmallBlocks: 1, MaxSmallBlocks: 1}); err != nil {
+	if err := env.Repository.Content.CompactIndexes(ctx, content.CompactOptions{MinSmallBlobs: 1, MaxSmallBlobs: 1}); err != nil {
 		t.Errorf("optimize error: %v", err)
 	}
 
@@ -131,7 +131,7 @@ func TestPackingSimple(t *testing.T) {
 	verify(ctx, t, env.Repository, oid2a, []byte(content2), "packed-object-2")
 	verify(ctx, t, env.Repository, oid3a, []byte(content3), "packed-object-3")
 
-	if err := env.Repository.Blocks.CompactIndexes(ctx, block.CompactOptions{MinSmallBlocks: 1, MaxSmallBlocks: 1}); err != nil {
+	if err := env.Repository.Content.CompactIndexes(ctx, content.CompactOptions{MinSmallBlobs: 1, MaxSmallBlobs: 1}); err != nil {
 		t.Errorf("optimize error: %v", err)
 	}
 

repo/upgrade.go

@@ -8,7 +8,7 @@
 
 // Upgrade upgrades repository data structures to the latest version.
 func (r *Repository) Upgrade(ctx context.Context) error {
-	f := r.formatBlock
+	f := r.formatBlob
 
 	log.Debug("decrypting format...")
 	repoConfig, err := f.decryptFormatBytes(r.masterKey)
@@ -29,6 +29,6 @@ func (r *Repository) Upgrade(ctx context.Context) error {
 		return errors.Errorf("unable to encrypt format bytes")
 	}
 
-	log.Infof("writing updated format block...")
-	return writeFormatBlock(ctx, r.Blobs, f)
+	log.Infof("writing updated format content...")
+	return writeFormatBlob(ctx, r.Blobs, f)
 }

site/content/docs/Architecture/_index.md

@@ -49,7 +49,7 @@ Pack files in blob storage have random names and don't reveal anything about the
 
 CABS is not meant to be used directly, instead it's a building block for object storage (CAOS) and manifest storage layers (LAMS) described below.
 
-The API for CABS can be found in https://godoc.org/github.com/kopia/kopia/repo/block
+The API for CABS can be found in https://godoc.org/github.com/kopia/kopia/repo/content
 
 ### Content-Addressable Object Storage (CAOS)
 
@@ -63,7 +63,7 @@ Object IDs can also have an optional single-letter prefix `g..z` that helps quic
 * `m` represents manifest block (e.g. `m0bf4da00801bd8c6ecfb66cffa67f32c`)
 * `h` represents hash-cache (e.g. `h2e88080490a83c4b1cb344d861a3f537`)
 
-To represent objects larger than the size of a single CABS block, Kopia links together multiple blocks via special indirect JSON block. Such blocks are distinguished from regular blocks by the `I` prefix. For example very large hash-cache object might have an identifier such as `Ih746f0a60f744d0a69e397a6128356331` and JSON content:
+To represent objects larger than the size of a single CABS block, Kopia links together multiple blocks via special indirect JSON content. Such blocks are distinguished from regular blocks by the `I` prefix. For example very large hash-cache object might have an identifier such as `Ih746f0a60f744d0a69e397a6128356331` and JSON content:
 
 ```json
 {"stream":"kopia:indirect","entries":[

snapshot/manager.go

@@ -58,7 +58,7 @@ func ListSnapshots(ctx context.Context, rep *repo.Repository, si SourceInfo) ([]
 }
 
 // loadSnapshot loads and parses a snapshot with a given ID.
-func loadSnapshot(ctx context.Context, rep *repo.Repository, manifestID string) (*Manifest, error) {
+func loadSnapshot(ctx context.Context, rep *repo.Repository, manifestID manifest.ID) (*Manifest, error) {
 	sm := &Manifest{}
 	if err := rep.Manifests.Get(ctx, manifestID, sm); err != nil {
 		return nil, errors.Wrap(err, "unable to find manifest entries")
@@ -69,7 +69,7 @@ func loadSnapshot(ctx context.Context, rep *repo.Repository, manifestID string)
 }
 
 // SaveSnapshot persists given snapshot manifest and returns manifest ID.
-func SaveSnapshot(ctx context.Context, rep *repo.Repository, manifest *Manifest) (string, error) {
+func SaveSnapshot(ctx context.Context, rep *repo.Repository, manifest *Manifest) (manifest.ID, error) {
 	if manifest.Source.Host == "" {
 		return "", errors.New("missing host")
 	}
@@ -89,13 +89,13 @@ func SaveSnapshot(ctx context.Context, rep *repo.Repository, manifest *Manifest)
 }
 
 // LoadSnapshots efficiently loads and parses a given list of snapshot IDs.
-func LoadSnapshots(ctx context.Context, rep *repo.Repository, names []string) ([]*Manifest, error) {
-	result := make([]*Manifest, len(names))
+func LoadSnapshots(ctx context.Context, rep *repo.Repository, manifestIDs []manifest.ID) ([]*Manifest, error) {
+	result := make([]*Manifest, len(manifestIDs))
 	sem := make(chan bool, 50)
 
-	for i, n := range names {
+	for i, n := range manifestIDs {
 		sem <- true
-		go func(i int, n string) {
+		go func(i int, n manifest.ID) {
 			defer func() { <-sem }()
 
 			m, err := loadSnapshot(ctx, rep, n)
@@ -123,7 +123,7 @@ func LoadSnapshots(ctx context.Context, rep *repo.Repository, names []string) ([
 }
 
 // ListSnapshotManifests returns the list of snapshot manifests for a given source or all sources if nil.
-func ListSnapshotManifests(ctx context.Context, rep *repo.Repository, src *SourceInfo) ([]string, error) {
+func ListSnapshotManifests(ctx context.Context, rep *repo.Repository, src *SourceInfo) ([]manifest.ID, error) {
 	labels := map[string]string{
 		"type": "snapshot",
 	}
@@ -139,8 +139,8 @@ func ListSnapshotManifests(ctx context.Context, rep *repo.Repository, src *Sourc
 	return entryIDs(entries), nil
 }
 
-func entryIDs(entries []*manifest.EntryMetadata) []string {
-	var ids []string
+func entryIDs(entries []*manifest.EntryMetadata) []manifest.ID {
+	var ids []manifest.ID
 	for _, e := range entries {
 		ids = append(ids, e.ID)
 	}

snapshot/manifest.go

@@ -7,13 +7,14 @@
 	"time"
 
 	"github.com/kopia/kopia/fs"
+	"github.com/kopia/kopia/repo/manifest"
 	"github.com/kopia/kopia/repo/object"
 )
 
 // Manifest represents information about a single point-in-time filesystem snapshot.
 type Manifest struct {
-	ID     string     `json:"-"`
-	Source SourceInfo `json:"source"`
+	ID     manifest.ID `json:"-"`
+	Source SourceInfo  `json:"source"`
 
 	Description string    `json:"description"`
 	StartTime   time.Time `json:"startTime"`

snapshot/policy/policy_manager.go

@@ -152,7 +152,7 @@ func RemovePolicy(ctx context.Context, rep *repo.Repository, si snapshot.SourceI
 }
 
 // GetPolicyByID gets the policy for a given unique ID or ErrPolicyNotFound if not found.
-func GetPolicyByID(ctx context.Context, rep *repo.Repository, id string) (*Policy, error) {
+func GetPolicyByID(ctx context.Context, rep *repo.Repository, id manifest.ID) (*Policy, error) {
 	p := &Policy{}
 	if err := rep.Manifests.Get(ctx, id, &p); err != nil {
 		if err == manifest.ErrNotFound {
@@ -187,7 +187,7 @@ func ListPolicies(ctx context.Context, rep *repo.Repository) ([]*Policy, error)
 		}
 
 		pol.Labels = md.Labels
-		pol.Labels["id"] = id.ID
+		pol.Labels["id"] = string(id.ID)
 		policies = append(policies, pol)
 	}
 

snapshot/snapshot_test.go

@@ -10,6 +10,7 @@
 
 	"github.com/kopia/kopia/internal/repotesting"
 	"github.com/kopia/kopia/repo"
+	"github.com/kopia/kopia/repo/manifest"
 	"github.com/kopia/kopia/snapshot"
 )
 
@@ -39,8 +40,8 @@ func TestSnapshotsAPI(t *testing.T) {
 		Description: "some-description",
 	}
 	id1 := mustSaveSnapshot(t, env.Repository, manifest1)
-	verifySnapshotManifestIDs(t, env.Repository, nil, []string{id1})
-	verifySnapshotManifestIDs(t, env.Repository, &src1, []string{id1})
+	verifySnapshotManifestIDs(t, env.Repository, nil, []manifest.ID{id1})
+	verifySnapshotManifestIDs(t, env.Repository, &src1, []manifest.ID{id1})
 	verifySnapshotManifestIDs(t, env.Repository, &src2, nil)
 	verifyListSnapshots(t, env.Repository, src1, []*snapshot.Manifest{manifest1})
 
@@ -52,8 +53,8 @@ func TestSnapshotsAPI(t *testing.T) {
 	if id1 == id2 {
 		t.Errorf("expected different manifest IDs, got same: %v", id1)
 	}
-	verifySnapshotManifestIDs(t, env.Repository, nil, []string{id1, id2})
-	verifySnapshotManifestIDs(t, env.Repository, &src1, []string{id1, id2})
+	verifySnapshotManifestIDs(t, env.Repository, nil, []manifest.ID{id1, id2})
+	verifySnapshotManifestIDs(t, env.Repository, &src1, []manifest.ID{id1, id2})
 	verifySnapshotManifestIDs(t, env.Repository, &src2, nil)
 
 	manifest3 := &snapshot.Manifest{
@@ -62,21 +63,21 @@ func TestSnapshotsAPI(t *testing.T) {
 	}
 
 	id3 := mustSaveSnapshot(t, env.Repository, manifest3)
-	verifySnapshotManifestIDs(t, env.Repository, nil, []string{id1, id2, id3})
-	verifySnapshotManifestIDs(t, env.Repository, &src1, []string{id1, id2})
-	verifySnapshotManifestIDs(t, env.Repository, &src2, []string{id3})
+	verifySnapshotManifestIDs(t, env.Repository, nil, []manifest.ID{id1, id2, id3})
+	verifySnapshotManifestIDs(t, env.Repository, &src1, []manifest.ID{id1, id2})
+	verifySnapshotManifestIDs(t, env.Repository, &src2, []manifest.ID{id3})
 	verifySources(t, env.Repository, src1, src2)
-	verifyLoadSnapshots(t, env.Repository, []string{id1, id2, id3}, []*snapshot.Manifest{manifest1, manifest2, manifest3})
+	verifyLoadSnapshots(t, env.Repository, []manifest.ID{id1, id2, id3}, []*snapshot.Manifest{manifest1, manifest2, manifest3})
 }
 
-func verifySnapshotManifestIDs(t *testing.T, rep *repo.Repository, src *snapshot.SourceInfo, expected []string) []string {
+func verifySnapshotManifestIDs(t *testing.T, rep *repo.Repository, src *snapshot.SourceInfo, expected []manifest.ID) []manifest.ID {
 	t.Helper()
 	res, err := snapshot.ListSnapshotManifests(context.Background(), rep, src)
 	if err != nil {
 		t.Errorf("error listing snapshot manifests: %v", err)
 	}
-	sort.Strings(res)
-	sort.Strings(expected)
+	sortManifestIDs(res)
+	sortManifestIDs(expected)
 	if !reflect.DeepEqual(res, expected) {
 		t.Errorf("unexpected manifests: %v, wanted %v", res, expected)
 		return expected
@@ -84,7 +85,13 @@ func verifySnapshotManifestIDs(t *testing.T, rep *repo.Repository, src *snapshot
 	return res
 }
 
-func mustSaveSnapshot(t *testing.T, rep *repo.Repository, man *snapshot.Manifest) string {
+func sortManifestIDs(s []manifest.ID) {
+	sort.Slice(s, func(i, j int) bool {
+		return s[i] < s[j]
+	})
+}
+
+func mustSaveSnapshot(t *testing.T, rep *repo.Repository, man *snapshot.Manifest) manifest.ID {
 	t.Helper()
 	id, err := snapshot.SaveSnapshot(context.Background(), rep, man)
 	if err != nil {
@@ -123,7 +130,7 @@ func verifyListSnapshots(t *testing.T, rep *repo.Repository, src snapshot.Source
 	}
 }
 
-func verifyLoadSnapshots(t *testing.T, rep *repo.Repository, ids []string, expected []*snapshot.Manifest) {
+func verifyLoadSnapshots(t *testing.T, rep *repo.Repository, ids []manifest.ID, expected []*snapshot.Manifest) {
 	got, err := snapshot.LoadSnapshots(context.Background(), rep, ids)
 	if err != nil {
 		t.Errorf("error loading manifests: %v", err)

snapshot/snapshotfs/upload.go

@@ -93,7 +93,7 @@ func (u *Uploader) cancelReason() string {
 		return "cancelled"
 	}
 
-	if mub := u.MaxUploadBytes; mub > 0 && u.repo.Blocks.Stats().WrittenBytes > mub {
+	if mub := u.MaxUploadBytes; mub > 0 && u.repo.Content.Stats().WrittenBytes > mub {
 		return "limit reached"
 	}
 
@@ -663,7 +663,7 @@ func (u *Uploader) Upload(
 	s.IncompleteReason = u.cancelReason()
 	s.EndTime = time.Now()
 	s.Stats = u.stats
-	s.Stats.Block = u.repo.Blocks.Stats()
+	s.Stats.Content = u.repo.Content.Stats()
 
 	return s, nil
 }

snapshot/stats.go

@@ -2,12 +2,12 @@
 
 import (
 	"github.com/kopia/kopia/fs"
-	"github.com/kopia/kopia/repo/block"
+	"github.com/kopia/kopia/repo/content"
 )
 
 // Stats keeps track of snapshot generation statistics.
 type Stats struct {
-	Block block.Stats `json:"repo,omitempty"`
+	Content content.Stats `json:"content,omitempty"`
 
 	TotalDirectoryCount int   `json:"dirCount"`
 	TotalFileCount      int   `json:"fileCount"`

tests/end_to_end_test/end_to_end_test.go

@@ -104,8 +104,8 @@ func TestEndToEnd(t *testing.T) {
 
 	t.Run("VerifyGlobalPolicy", func(t *testing.T) {
 		// verify we created global policy entry
-		globalPolicyBlockID := e.runAndVerifyOutputLineCount(t, 1, "block", "ls")[0]
-		e.runAndExpectSuccess(t, "block", "show", "-jz", globalPolicyBlockID)
+		globalPolicyBlockID := e.runAndVerifyOutputLineCount(t, 1, "content", "ls")[0]
+		e.runAndExpectSuccess(t, "content", "show", "-jz", globalPolicyBlockID)
 
 		// make sure the policy is visible in the manifest list
 		e.runAndVerifyOutputLineCount(t, 1, "manifest", "list", "--filter=type:policy", "--filter=policyType:global")
@@ -137,13 +137,13 @@ func TestEndToEnd(t *testing.T) {
 		t.Errorf("unexpected number of sources: %v, want %v in %#v", got, want, sources)
 	}
 
-	// expect 5 blocks, each snapshot creation adds one index blob
-	e.runAndVerifyOutputLineCount(t, 6, "blockindex", "ls")
-	e.runAndExpectSuccess(t, "blockindex", "optimize")
-	e.runAndVerifyOutputLineCount(t, 1, "blockindex", "ls")
+	// expect 5 blobs, each snapshot creation adds one index blob
+	e.runAndVerifyOutputLineCount(t, 6, "index", "ls")
+	e.runAndExpectSuccess(t, "index", "optimize")
+	e.runAndVerifyOutputLineCount(t, 1, "index", "ls")
 
 	e.runAndExpectSuccess(t, "snapshot", "create", ".", dir1, dir2)
-	e.runAndVerifyOutputLineCount(t, 2, "blockindex", "ls")
+	e.runAndVerifyOutputLineCount(t, 2, "index", "ls")
 
 	t.Run("Migrate", func(t *testing.T) {
 		dstenv := newTestEnv(t)
@@ -160,39 +160,39 @@ func TestEndToEnd(t *testing.T) {
 	})
 
 	t.Run("RepairIndexBlobs", func(t *testing.T) {
-		blocksBefore := e.runAndExpectSuccess(t, "block", "ls")
+		contentsBefore := e.runAndExpectSuccess(t, "content", "ls")
 
-		lines := e.runAndVerifyOutputLineCount(t, 2, "blockindex", "ls")
+		lines := e.runAndVerifyOutputLineCount(t, 2, "index", "ls")
 		for _, l := range lines {
 			indexFile := strings.Split(l, " ")[0]
 			e.runAndExpectSuccess(t, "blob", "delete", indexFile)
 		}
 
 		// there should be no index files at this point
-		e.runAndVerifyOutputLineCount(t, 0, "blockindex", "ls", "--no-list-caching")
+		e.runAndVerifyOutputLineCount(t, 0, "index", "ls", "--no-list-caching")
 		// there should be no blocks, since there are no indexesto find them
-		e.runAndVerifyOutputLineCount(t, 0, "block", "ls")
+		e.runAndVerifyOutputLineCount(t, 0, "content", "ls")
 
 		// now recover index from all blocks
-		e.runAndExpectSuccess(t, "blockindex", "recover", "--commit")
+		e.runAndExpectSuccess(t, "index", "recover", "--commit")
 
 		// all recovered index entries are added as index file
-		e.runAndVerifyOutputLineCount(t, 1, "blockindex", "ls")
-		blocksAfter := e.runAndExpectSuccess(t, "block", "ls")
-		if diff := pretty.Compare(blocksBefore, blocksAfter); diff != "" {
+		e.runAndVerifyOutputLineCount(t, 1, "index", "ls")
+		contentsAfter := e.runAndExpectSuccess(t, "content", "ls")
+		if diff := pretty.Compare(contentsBefore, contentsAfter); diff != "" {
 			t.Errorf("unexpected block diff after recovery: %v", diff)
 		}
 	})
 
-	t.Run("RepairFormatBlock", func(t *testing.T) {
+	t.Run("RepairFormatBlob", func(t *testing.T) {
 		// remove kopia.repository
 		e.runAndExpectSuccess(t, "blob", "rm", "kopia.repository")
 		e.runAndExpectSuccess(t, "repo", "disconnect")
 
-		// this will fail because the format block in the repository is not found
+		// this will fail because the format blob in the repository is not found
 		e.runAndExpectFailure(t, "repo", "connect", "filesystem", "--path", e.repoDir)
 
-		// now run repair, which will recover the format block from one of the pack blocks.
+		// now run repair, which will recover the format blob from one of the pack blobs.
 		e.runAndExpectSuccess(t, "repo", "repair", "filesystem", "--path", e.repoDir)
 
 		// now connect can succeed

tests/repository_stress_test/repository_stress_test.go

@@ -18,13 +18,13 @@
 
 	"github.com/kopia/kopia/repo"
 	"github.com/kopia/kopia/repo/blob/filesystem"
-	"github.com/kopia/kopia/repo/block"
+	"github.com/kopia/kopia/repo/content"
 )
 
 const masterPassword = "foo-bar-baz-1234"
 
 var (
-	knownBlocks      []string
+	knownBlocks      []content.ID
 	knownBlocksMutex sync.Mutex
 )
 
@@ -32,7 +32,7 @@ func TestStressRepository(t *testing.T) {
 	if testing.Short() {
 		t.Skip("skipping stress test during short tests")
 	}
-	ctx := block.UsingListCache(context.Background(), false)
+	ctx := content.UsingListCache(context.Background(), false)
 
 	tmpPath, err := ioutil.TempDir("", "kopia")
 	if err != nil {
@@ -66,7 +66,7 @@ func TestStressRepository(t *testing.T) {
 
 	// set up two parallel kopia connections, each with its own config file and cache.
 	if err := repo.Connect(ctx, configFile1, st, masterPassword, repo.ConnectOptions{
-		CachingOptions: block.CachingOptions{
+		CachingOptions: content.CachingOptions{
 			CacheDirectory:    filepath.Join(tmpPath, "cache1"),
 			MaxCacheSizeBytes: 2000000000,
 		},
@@ -75,7 +75,7 @@ func TestStressRepository(t *testing.T) {
 	}
 
 	if err := repo.Connect(ctx, configFile2, st, masterPassword, repo.ConnectOptions{
-		CachingOptions: block.CachingOptions{
+		CachingOptions: content.CachingOptions{
 			CacheDirectory:    filepath.Join(tmpPath, "cache2"),
 			MaxCacheSizeBytes: 2000000000,
 		},
@@ -155,8 +155,8 @@ func repositoryTest(ctx context.Context, t *testing.T, cancel chan struct{}, rep
 		{"writeRandomBlock", writeRandomBlock, 100, 0},
 		{"writeRandomManifest", writeRandomManifest, 100, 0},
 		{"readKnownBlock", readKnownBlock, 500, 0},
-		{"listBlocks", listBlocks, 50, 0},
-		{"listAndReadAllBlocks", listAndReadAllBlocks, 5, 0},
+		{"listContents", listContents, 50, 0},
+		{"listAndReadAllContents", listAndReadAllContents, 5, 0},
 		{"readRandomManifest", readRandomManifest, 50, 0},
 		{"compact", compact, 1, 0},
 		{"refresh", refresh, 3, 0},
@@ -208,14 +208,14 @@ func repositoryTest(ctx context.Context, t *testing.T, cancel chan struct{}, rep
 func writeRandomBlock(ctx context.Context, t *testing.T, r *repo.Repository) error {
 	data := make([]byte, 1000)
 	rand.Read(data)
-	blockID, err := r.Blocks.WriteBlock(ctx, data, "")
+	contentID, err := r.Content.WriteContent(ctx, data, "")
 	if err == nil {
 		knownBlocksMutex.Lock()
 		if len(knownBlocks) >= 1000 {
 			n := rand.Intn(len(knownBlocks))
-			knownBlocks[n] = blockID
+			knownBlocks[n] = contentID
 		} else {
-			knownBlocks = append(knownBlocks, blockID)
+			knownBlocks = append(knownBlocks, contentID)
 		}
 		knownBlocksMutex.Unlock()
 	}
@@ -228,36 +228,36 @@ func readKnownBlock(ctx context.Context, t *testing.T, r *repo.Repository) error
 		knownBlocksMutex.Unlock()
 		return nil
 	}
-	blockID := knownBlocks[rand.Intn(len(knownBlocks))]
+	contentID := knownBlocks[rand.Intn(len(knownBlocks))]
 	knownBlocksMutex.Unlock()
 
-	_, err := r.Blocks.GetBlock(ctx, blockID)
-	if err == nil || err == block.ErrBlockNotFound {
+	_, err := r.Content.GetContent(ctx, contentID)
+	if err == nil || err == content.ErrContentNotFound {
 		return nil
 	}
 
 	return err
 }
 
-func listBlocks(ctx context.Context, t *testing.T, r *repo.Repository) error {
-	_, err := r.Blocks.ListBlocks("")
+func listContents(ctx context.Context, t *testing.T, r *repo.Repository) error {
+	_, err := r.Content.ListContents("")
 	return err
 }
 
-func listAndReadAllBlocks(ctx context.Context, t *testing.T, r *repo.Repository) error {
-	blocks, err := r.Blocks.ListBlocks("")
+func listAndReadAllContents(ctx context.Context, t *testing.T, r *repo.Repository) error {
+	contentIDs, err := r.Content.ListContents("")
 	if err != nil {
 		return err
 	}
 
-	for _, bi := range blocks {
-		_, err := r.Blocks.GetBlock(ctx, bi)
+	for _, cid := range contentIDs {
+		_, err := r.Content.GetContent(ctx, cid)
 		if err != nil {
-			if err == block.ErrBlockNotFound && strings.HasPrefix(bi, "m") {
-				// this is ok, sometimes manifest manager will perform compaction and 'm' blocks will be marked as deleted
+			if err == content.ErrContentNotFound && strings.HasPrefix(string(cid), "m") {
+				// this is ok, sometimes manifest manager will perform compaction and 'm' contents will be marked as deleted
 				continue
 			}
-			return errors.Wrapf(err, "error reading block %v", bi)
+			return errors.Wrapf(err, "error reading content %v", cid)
 		}
 	}
 
@@ -265,9 +265,9 @@ func listAndReadAllBlocks(ctx context.Context, t *testing.T, r *repo.Repository)
 }
 
 func compact(ctx context.Context, t *testing.T, r *repo.Repository) error {
-	return r.Blocks.CompactIndexes(ctx, block.CompactOptions{
-		MinSmallBlocks: 1,
-		MaxSmallBlocks: 1,
+	return r.Content.CompactIndexes(ctx, content.CompactOptions{
+		MinSmallBlobs: 1,
+		MaxSmallBlobs: 1,
 	})
 }
 

tests/stress_test/stress_test.go

@@ -11,7 +11,7 @@
 
 	"github.com/kopia/kopia/internal/blobtesting"
 	"github.com/kopia/kopia/repo/blob"
-	"github.com/kopia/kopia/repo/block"
+	"github.com/kopia/kopia/repo/content"
 )
 
 const goroutineCount = 16
@@ -36,14 +36,14 @@ func TestStressBlockManager(t *testing.T) {
 func stressTestWithStorage(t *testing.T, st blob.Storage, duration time.Duration) {
 	ctx := context.Background()
 
-	openMgr := func() (*block.Manager, error) {
-		return block.NewManager(ctx, st, block.FormattingOptions{
+	openMgr := func() (*content.Manager, error) {
+		return content.NewManager(ctx, st, content.FormattingOptions{
 			Version:     1,
 			Hash:        "HMAC-SHA256-128",
 			Encryption:  "AES-256-CTR",
 			MaxPackSize: 20000000,
 			MasterKey:   []byte{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15},
-		}, block.CachingOptions{}, nil)
+		}, content.CachingOptions{}, nil)
 	}
 
 	seed0 := time.Now().Nanosecond()
@@ -63,7 +63,7 @@ func stressTestWithStorage(t *testing.T, st blob.Storage, duration time.Duration
 	})
 }
 
-func stressWorker(ctx context.Context, t *testing.T, deadline time.Time, workerID int, openMgr func() (*block.Manager, error), seed int64) {
+func stressWorker(ctx context.Context, t *testing.T, deadline time.Time, workerID int, openMgr func() (*content.Manager, error), seed int64) {
 	src := rand.NewSource(seed)
 	rand := rand.New(src)
 
@@ -73,7 +73,7 @@ func stressWorker(ctx context.Context, t *testing.T, deadline time.Time, workerI
 	}
 
 	type writtenBlock struct {
-		contentID string
+		contentID content.ID
 		data      []byte
 	}
 
@@ -87,7 +87,7 @@ type writtenBlock struct {
 			return
 		}
 		dataCopy := append([]byte{}, data...)
-		contentID, err := bm.WriteBlock(ctx, data, "")
+		contentID, err := bm.WriteContent(ctx, data, "")
 		if err != nil {
 			t.Errorf("err: %v", err)
 			return
@@ -117,9 +117,9 @@ type writtenBlock struct {
 			pos := rand.Intn(len(workerBlocks))
 			previous := workerBlocks[pos]
 			//log.Printf("reading %v", previous.contentID)
-			d2, err := bm.GetBlock(ctx, previous.contentID)
+			d2, err := bm.GetContent(ctx, previous.contentID)
 			if err != nil {
-				t.Errorf("error verifying block %q: %v", previous.contentID, err)
+				t.Errorf("error verifying content %q: %v", previous.contentID, err)
 				return
 			}
 			if !reflect.DeepEqual(previous.data, d2) {