diff --git a/Makefile b/Makefile index ab8baf2d4..c15f09aca 100644 --- a/Makefile +++ b/Makefile @@ -13,7 +13,7 @@ rwildcard=$(foreach d,$(wildcard $1*),$(call rwildcard,$d/,$2) $(filter $(subst go_source_dirs=cli fs internal repo snapshot all_go_sources=$(foreach d,$(go_source_dirs),$(call rwildcard,$d/,*.go)) $(wildcard *.go) -all: test lint vet integration-tests integration-tests-index-v2 +all: test lint vet integration-tests integration-tests-index-v1 include tools/tools.mk @@ -155,7 +155,7 @@ ci-tests: lint vet test ci-integration-tests: $(MAKE) integration-tests - $(MAKE) integration-tests-index-v2 + $(MAKE) integration-tests-index-v1 $(MAKE) robustness-tool-tests ci-publish-coverage: @@ -233,8 +233,8 @@ integration-tests: build-integration-test-binary $(gotestsum) $(TESTING_ACTION_E $(GO_TEST) $(TEST_FLAGS) -count=$(REPEAT_TEST) -parallel $(PARALLEL) -timeout 3600s github.com/kopia/kopia/tests/end_to_end_test -$(gotestsum) tool slowest --jsonfile .tmp.integration-tests.json --threshold 1000ms -integration-tests-index-v2: - KOPIA_CREATE_INDEX_VERSION=2 KOPIA_ENABLE_INDEX_EPOCHS=true KOPIA_RUN_ALL_INTEGRATION_TESTS=true $(MAKE) integration-tests +integration-tests-index-v1: + KOPIA_CREATE_INDEX_VERSION=1 KOPIA_ENABLE_INDEX_EPOCHS=false KOPIA_RUN_ALL_INTEGRATION_TESTS=true $(MAKE) integration-tests endurance-tests: export KOPIA_EXE ?= $(KOPIA_INTEGRATION_EXE) endurance-tests: export KOPIA_LOGS_DIR=$(CURDIR)/.logs diff --git a/cli/command_blob_show_test.go b/cli/command_blob_show_test.go index ebf0880e8..b426f85a9 100644 --- a/cli/command_blob_show_test.go +++ b/cli/command_blob_show_test.go @@ -12,11 +12,27 @@ func TestBlobShow(t *testing.T) { env.RunAndExpectSuccess(t, "repo", "create", "filesystem", "--path", env.RepoDir) - someNBlob := strings.Split(env.RunAndExpectSuccess(t, "blob", "list", "--prefix=n")[0], " ")[0] + var hasEpochManager bool + + for _, line := range env.RunAndExpectSuccess(t, "repository", "status") { + if strings.HasPrefix(line, "Epoch Manager:") && strings.Contains(line, "enabled") { + hasEpochManager = true + } + } + someQBlob := strings.Split(env.RunAndExpectSuccess(t, "blob", "list", "--prefix=q")[0], " ")[0] - env.RunAndExpectSuccess(t, "blob", "show", someNBlob) + + if hasEpochManager { + someXNBlob := strings.Split(env.RunAndExpectSuccess(t, "blob", "list", "--prefix=xn")[0], " ")[0] + env.RunAndExpectSuccess(t, "blob", "show", someXNBlob) + env.RunAndExpectSuccess(t, "blob", "show", "--decrypt", someXNBlob) + } else { + someNBlob := strings.Split(env.RunAndExpectSuccess(t, "blob", "list", "--prefix=n")[0], " ")[0] + env.RunAndExpectSuccess(t, "blob", "show", someNBlob) + env.RunAndExpectSuccess(t, "blob", "show", "--decrypt", someNBlob) + } + env.RunAndExpectSuccess(t, "blob", "show", someQBlob) - env.RunAndExpectSuccess(t, "blob", "show", "--decrypt", someNBlob) // --decrypt will be ignored env.RunAndExpectSuccess(t, "blob", "show", "--decrypt", someQBlob) } diff --git a/cli/command_repository_create.go b/cli/command_repository_create.go index 62fed38a8..fa6d4e2d6 100644 --- a/cli/command_repository_create.go +++ b/cli/command_repository_create.go @@ -46,7 +46,7 @@ func (c *commandRepositoryCreate) setup(svc advancedAppServices, parent commandP cmd.Flag("create-only", "Create repository, but don't connect to it.").Short('c').BoolVar(&c.createOnly) cmd.Flag("enable-password-change", "Enable password change").Hidden().Default("true").BoolVar(&c.enablePasswordChange) cmd.Flag("index-version", "Force particular index version").Hidden().Envar("KOPIA_CREATE_INDEX_VERSION").IntVar(&c.createIndexVersion) - cmd.Flag("enable-index-epochs", "Enable index epochs").Hidden().Envar("KOPIA_ENABLE_INDEX_EPOCHS").BoolVar(&c.createIndexEpochs) + cmd.Flag("enable-index-epochs", "Enable index epochs").Hidden().Default("true").Envar("KOPIA_ENABLE_INDEX_EPOCHS").BoolVar(&c.createIndexEpochs) c.co.setup(cmd) c.svc = svc diff --git a/cli/command_repository_set_parameters_test.go b/cli/command_repository_set_parameters_test.go index 68e015146..ced925c88 100644 --- a/cli/command_repository_set_parameters_test.go +++ b/cli/command_repository_set_parameters_test.go @@ -11,7 +11,7 @@ import ( func TestRepositorySetParameters(t *testing.T) { env := testenv.NewCLITest(t, testenv.NewInProcRunner(t)) - env.RunAndExpectSuccess(t, "repo", "create", "filesystem", "--path", env.RepoDir) + env.RunAndExpectSuccess(t, "repo", "create", "filesystem", "--path", env.RepoDir, "--index-version=1") out := env.RunAndExpectSuccess(t, "repository", "status") // default values @@ -37,7 +37,7 @@ func TestRepositorySetParameters(t *testing.T) { func TestRepositorySetParametersUpgrade(t *testing.T) { env := testenv.NewCLITest(t, testenv.NewInProcRunner(t)) - env.RunAndExpectSuccess(t, "repo", "create", "filesystem", "--path", env.RepoDir) + env.RunAndExpectSuccess(t, "repo", "create", "filesystem", "--path", env.RepoDir, "--index-version=1", "--no-enable-index-epochs") out := env.RunAndExpectSuccess(t, "repository", "status") // default values diff --git a/repo/content/builder.go b/repo/content/builder.go index 8b2473fcc..77e018251 100644 --- a/repo/content/builder.go +++ b/repo/content/builder.go @@ -147,6 +147,10 @@ func (b packIndexBuilder) BuildStable(output io.Writer, version int) error { func (b packIndexBuilder) shard(maxShardSize int) []packIndexBuilder { numShards := (len(b) + maxShardSize - 1) / maxShardSize if numShards <= 1 { + if len(b) == 0 { + return []packIndexBuilder{} + } + return []packIndexBuilder{b} } @@ -164,7 +168,15 @@ func (b packIndexBuilder) shard(maxShardSize int) []packIndexBuilder { result[shard][k] = v } - return result + var nonEmpty []packIndexBuilder + + for _, r := range result { + if len(r) > 0 { + nonEmpty = append(nonEmpty, r) + } + } + + return nonEmpty } func (b packIndexBuilder) buildShards(indexVersion int, stable bool, shardSize int) ([]gather.Bytes, func(), error) { diff --git a/repo/content/committed_read_manager.go b/repo/content/committed_read_manager.go index 3fbcc5c2a..ac2daba83 100644 --- a/repo/content/committed_read_manager.go +++ b/repo/content/committed_read_manager.go @@ -516,7 +516,7 @@ func NewSharedManager(ctx context.Context, st blob.Storage, f *FormattingOptions actualIndexVersion := f.IndexVersion if actualIndexVersion == 0 { - actualIndexVersion = DefaultIndexVersion + actualIndexVersion = legacyIndexVersion } if actualIndexVersion < v1IndexVersion || actualIndexVersion > v2IndexVersion { diff --git a/repo/content/content_manager.go b/repo/content/content_manager.go index 70b6b2de5..71d58f066 100644 --- a/repo/content/content_manager.go +++ b/repo/content/content_manager.go @@ -35,7 +35,9 @@ const ( defaultIndexShardSize = 16e6 // slightly less than 2^24, which lets index use 24-bit/3-byte indexes - DefaultIndexVersion = 1 + DefaultIndexVersion = 2 + + legacyIndexVersion = 1 ) // PackBlobIDPrefixes contains all possible prefixes for pack blobs. diff --git a/repo/content/content_manager_test.go b/repo/content/content_manager_test.go index 131159443..5d2c3335d 100644 --- a/repo/content/content_manager_test.go +++ b/repo/content/content_manager_test.go @@ -49,7 +49,8 @@ func TestMain(m *testing.M) { testutil.MyTestMain(m) } func TestLegacyFormat(t *testing.T) { testutil.RunAllTestsWithParam(t, &contentManagerSuite{ mutableParameters: MutableParameters{ - MaxPackSize: maxPackSize, + IndexVersion: 1, + MaxPackSize: maxPackSize, }, }) } diff --git a/repo/content/index_v2.go b/repo/content/index_v2.go index acbfd6fb3..edceaf118 100644 --- a/repo/content/index_v2.go +++ b/repo/content/index_v2.go @@ -572,7 +572,7 @@ func (b packIndexBuilder) buildV2(output io.Writer) error { extraData := b2.prepareExtraData(sortedInfos) if b2.keyLength <= 1 { - return errors.Errorf("invalid key length: %v", b2.keyLength) + return errors.Errorf("invalid key length: %v for %v", b2.keyLength, len(b)) } // write header