content: switched defaults to use v2 index format and epoch-based indexes (#1275)

Makefile

@@ -13,7 +13,7 @@ rwildcard=$(foreach d,$(wildcard $1*),$(call rwildcard,$d/,$2) $(filter $(subst
 go_source_dirs=cli fs internal repo snapshot
 all_go_sources=$(foreach d,$(go_source_dirs),$(call rwildcard,$d/,*.go)) $(wildcard *.go)
 
-all: test lint vet integration-tests integration-tests-index-v2
+all: test lint vet integration-tests integration-tests-index-v1
 
 include tools/tools.mk
 
@@ -155,7 +155,7 @@ ci-tests: lint vet test
 
 ci-integration-tests:
 	$(MAKE) integration-tests
-	$(MAKE) integration-tests-index-v2
+	$(MAKE) integration-tests-index-v1
 	$(MAKE) robustness-tool-tests
 
 ci-publish-coverage:
@@ -233,8 +233,8 @@ integration-tests: build-integration-test-binary $(gotestsum) $(TESTING_ACTION_E
 	 $(GO_TEST) $(TEST_FLAGS) -count=$(REPEAT_TEST) -parallel $(PARALLEL) -timeout 3600s github.com/kopia/kopia/tests/end_to_end_test
 	 -$(gotestsum) tool slowest --jsonfile .tmp.integration-tests.json  --threshold 1000ms
 
-integration-tests-index-v2:
-	KOPIA_CREATE_INDEX_VERSION=2 KOPIA_ENABLE_INDEX_EPOCHS=true KOPIA_RUN_ALL_INTEGRATION_TESTS=true $(MAKE) integration-tests
+integration-tests-index-v1:
+	KOPIA_CREATE_INDEX_VERSION=1 KOPIA_ENABLE_INDEX_EPOCHS=false KOPIA_RUN_ALL_INTEGRATION_TESTS=true $(MAKE) integration-tests
 
 endurance-tests: export KOPIA_EXE ?= $(KOPIA_INTEGRATION_EXE)
 endurance-tests: export KOPIA_LOGS_DIR=$(CURDIR)/.logs

cli/command_blob_show_test.go

@@ -12,11 +12,27 @@ func TestBlobShow(t *testing.T) {
 
 	env.RunAndExpectSuccess(t, "repo", "create", "filesystem", "--path", env.RepoDir)
 
-	someNBlob := strings.Split(env.RunAndExpectSuccess(t, "blob", "list", "--prefix=n")[0], " ")[0]
+	var hasEpochManager bool
+
+	for _, line := range env.RunAndExpectSuccess(t, "repository", "status") {
+		if strings.HasPrefix(line, "Epoch Manager:") && strings.Contains(line, "enabled") {
+			hasEpochManager = true
+		}
+	}
+
 	someQBlob := strings.Split(env.RunAndExpectSuccess(t, "blob", "list", "--prefix=q")[0], " ")[0]
-	env.RunAndExpectSuccess(t, "blob", "show", someNBlob)
+
+	if hasEpochManager {
+		someXNBlob := strings.Split(env.RunAndExpectSuccess(t, "blob", "list", "--prefix=xn")[0], " ")[0]
+		env.RunAndExpectSuccess(t, "blob", "show", someXNBlob)
+		env.RunAndExpectSuccess(t, "blob", "show", "--decrypt", someXNBlob)
+	} else {
+		someNBlob := strings.Split(env.RunAndExpectSuccess(t, "blob", "list", "--prefix=n")[0], " ")[0]
+		env.RunAndExpectSuccess(t, "blob", "show", someNBlob)
+		env.RunAndExpectSuccess(t, "blob", "show", "--decrypt", someNBlob)
+	}
+
 	env.RunAndExpectSuccess(t, "blob", "show", someQBlob)
-	env.RunAndExpectSuccess(t, "blob", "show", "--decrypt", someNBlob)
 	// --decrypt will be ignored
 	env.RunAndExpectSuccess(t, "blob", "show", "--decrypt", someQBlob)
 }

cli/command_repository_create.go

@@ -46,7 +46,7 @@ func (c *commandRepositoryCreate) setup(svc advancedAppServices, parent commandP
 	cmd.Flag("create-only", "Create repository, but don't connect to it.").Short('c').BoolVar(&c.createOnly)
 	cmd.Flag("enable-password-change", "Enable password change").Hidden().Default("true").BoolVar(&c.enablePasswordChange)
 	cmd.Flag("index-version", "Force particular index version").Hidden().Envar("KOPIA_CREATE_INDEX_VERSION").IntVar(&c.createIndexVersion)
-	cmd.Flag("enable-index-epochs", "Enable index epochs").Hidden().Envar("KOPIA_ENABLE_INDEX_EPOCHS").BoolVar(&c.createIndexEpochs)
+	cmd.Flag("enable-index-epochs", "Enable index epochs").Hidden().Default("true").Envar("KOPIA_ENABLE_INDEX_EPOCHS").BoolVar(&c.createIndexEpochs)
 
 	c.co.setup(cmd)
 	c.svc = svc

cli/command_repository_set_parameters_test.go

@@ -11,7 +11,7 @@ import (
 func TestRepositorySetParameters(t *testing.T) {
 	env := testenv.NewCLITest(t, testenv.NewInProcRunner(t))
 
-	env.RunAndExpectSuccess(t, "repo", "create", "filesystem", "--path", env.RepoDir)
+	env.RunAndExpectSuccess(t, "repo", "create", "filesystem", "--path", env.RepoDir, "--index-version=1")
 	out := env.RunAndExpectSuccess(t, "repository", "status")
 
 	// default values
@@ -37,7 +37,7 @@ func TestRepositorySetParameters(t *testing.T) {
 func TestRepositorySetParametersUpgrade(t *testing.T) {
 	env := testenv.NewCLITest(t, testenv.NewInProcRunner(t))
 
-	env.RunAndExpectSuccess(t, "repo", "create", "filesystem", "--path", env.RepoDir)
+	env.RunAndExpectSuccess(t, "repo", "create", "filesystem", "--path", env.RepoDir, "--index-version=1", "--no-enable-index-epochs")
 	out := env.RunAndExpectSuccess(t, "repository", "status")
 
 	// default values

repo/content/builder.go

@@ -147,6 +147,10 @@ func (b packIndexBuilder) BuildStable(output io.Writer, version int) error {
 func (b packIndexBuilder) shard(maxShardSize int) []packIndexBuilder {
 	numShards := (len(b) + maxShardSize - 1) / maxShardSize
 	if numShards <= 1 {
+		if len(b) == 0 {
+			return []packIndexBuilder{}
+		}
+
 		return []packIndexBuilder{b}
 	}
 
@@ -164,7 +168,15 @@ func (b packIndexBuilder) shard(maxShardSize int) []packIndexBuilder {
 		result[shard][k] = v
 	}
 
-	return result
+	var nonEmpty []packIndexBuilder
+
+	for _, r := range result {
+		if len(r) > 0 {
+			nonEmpty = append(nonEmpty, r)
+		}
+	}
+
+	return nonEmpty
 }
 
 func (b packIndexBuilder) buildShards(indexVersion int, stable bool, shardSize int) ([]gather.Bytes, func(), error) {

repo/content/committed_read_manager.go

@@ -516,7 +516,7 @@ func NewSharedManager(ctx context.Context, st blob.Storage, f *FormattingOptions
 
 	actualIndexVersion := f.IndexVersion
 	if actualIndexVersion == 0 {
-		actualIndexVersion = DefaultIndexVersion
+		actualIndexVersion = legacyIndexVersion
 	}
 
 	if actualIndexVersion < v1IndexVersion || actualIndexVersion > v2IndexVersion {

repo/content/content_manager.go

@@ -35,7 +35,9 @@ const (
 
 	defaultIndexShardSize = 16e6 // slightly less than 2^24, which lets index use 24-bit/3-byte indexes
 
-	DefaultIndexVersion = 1
+	DefaultIndexVersion = 2
+
+	legacyIndexVersion = 1
 )
 
 // PackBlobIDPrefixes contains all possible prefixes for pack blobs.

repo/content/content_manager_test.go

@@ -49,7 +49,8 @@ func TestMain(m *testing.M) { testutil.MyTestMain(m) }
 func TestLegacyFormat(t *testing.T) {
 	testutil.RunAllTestsWithParam(t, &contentManagerSuite{
 		mutableParameters: MutableParameters{
-			MaxPackSize: maxPackSize,
+			IndexVersion: 1,
+			MaxPackSize:  maxPackSize,
 		},
 	})
 }

repo/content/index_v2.go

@@ -572,7 +572,7 @@ func (b packIndexBuilder) buildV2(output io.Writer) error {
 	extraData := b2.prepareExtraData(sortedInfos)
 
 	if b2.keyLength <= 1 {
-		return errors.Errorf("invalid key length: %v", b2.keyLength)
+		return errors.Errorf("invalid key length: %v for %v", b2.keyLength, len(b))
 	}
 
 	// write header