fix(cli): fixed v0.18.0 failures using v0.17.0-generated config (#4255)

This was caused by the client using key derivation algorithm from a config file (which did not have it when it was generated using old version of Kopia). Fixes #4254
2025-12-23 22:57:50 -05:00 · 2024-11-17 10:15:00 -08:00
parent 68c5308e4e
commit b60cac4860
10 changed files with 99 additions and 13 deletions
--- a/3
+++ b/3
@@ -311,8 +311,9 @@ $(TESTING_ACTION_EXE): tests/testingaction/main.go

 compat-tests: export KOPIA_CURRENT_EXE=$(CURDIR)/$(kopia_ui_embedded_exe)
 compat-tests: export KOPIA_08_EXE=$(kopia08)
+compat-tests: export KOPIA_017_EXE=$(kopia017)
 compat-tests: GOTESTSUM_FLAGS=--format=testname --no-summary=skipped --jsonfile=.tmp.compat-tests.json
-compat-tests: $(kopia_ui_embedded_exe) $(kopia08) $(gotestsum)
+compat-tests: $(kopia_ui_embedded_exe) $(kopia08) $(kopia017) $(gotestsum)
 	$(GO_TEST) $(TEST_FLAGS) -count=$(REPEAT_TEST) -parallel $(PARALLEL) -timeout 3600s github.com/kopia/kopia/tests/compat_test
 	#  -$(gotestsum) tool slowest --jsonfile .tmp.compat-tests.json  --threshold 1000ms

--- a/cli/command_repository_connect_server.go
+++ b/cli/command_repository_connect_server.go
@@ -36,9 +36,6 @@ func (c *commandRepositoryConnectServer) setup(svc advancedAppServices, parent c

 func (c *commandRepositoryConnectServer) run(ctx context.Context) error {
 	localCacheKeyDerivationAlgorithm := c.connectAPIServerLocalCacheKeyDerivationAlgorithm
-	if localCacheKeyDerivationAlgorithm == "" {
-		localCacheKeyDerivationAlgorithm = repo.DefaultServerRepoCacheKeyDerivationAlgorithm
-	}

 	as := &repo.APIServerInfo{
 		BaseURL:                             strings.TrimSuffix(c.connectAPIServerURL, "/"),
--- a/internal/servertesting/servertesting.go
+++ b/internal/servertesting/servertesting.go
@@ -79,8 +79,6 @@ func StartServer(t *testing.T, env *repotesting.Environment, tls bool) *repo.API
 		asi.BaseURL = hs.URL
 	}

-	asi.LocalCacheKeyDerivationAlgorithm = repo.DefaultServerRepoCacheKeyDerivationAlgorithm
-
 	t.Cleanup(hs.Close)

 	return asi
--- a/repo/api_server_repository.go
+++ b/repo/api_server_repository.go
@@ -7,6 +7,11 @@
 )

 // APIServerInfo is remote repository configuration stored in local configuration.
+//
+// NOTE: this structure is persistent on disk may be read/written using
+// different versions of Kopia, so it must be backwards-compatible.
+//
+// Apply appropriate defaults when reading.
 type APIServerInfo struct {
 	BaseURL                             string `json:"url"`
 	TrustedServerCertificateFingerprint string `json:"serverCertFingerprint"`
--- a/repo/open.go
+++ b/repo/open.go
@@ -145,7 +145,12 @@ func getContentCacheOrNil(ctx context.Context, si *APIServerInfo, opt *content.C

 	const cacheEncryptionKeySize = 32

-	cacheEncryptionKey, err := crypto.DeriveKeyFromPassword(password, saltWithPurpose, cacheEncryptionKeySize, si.LocalCacheKeyDerivationAlgorithm)
+	keyAlgo := si.LocalCacheKeyDerivationAlgorithm
+	if keyAlgo == "" {
+		keyAlgo = DefaultServerRepoCacheKeyDerivationAlgorithm
+	}
+
+	cacheEncryptionKey, err := crypto.DeriveKeyFromPassword(password, saltWithPurpose, cacheEncryptionKeySize, keyAlgo)
 	if err != nil {
 		return nil, errors.Wrap(err, "unable to derive cache encryption key from password")
 	}
--- a/repo/repository_test.go
+++ b/repo/repository_test.go
@@ -668,10 +668,10 @@ func(ctx context.Context, w repo.RepositoryWriter) error {
 		},
 	})

-	defer rep.Close(ctx) //nolint:errcheck,staticcheck
-
 	require.NoError(t, err)

+	defer rep.Close(ctx) //nolint:errcheck,staticcheck
+
 	var oid object.ID

 	require.NoError(t, repo.WriteSession(ctx, rep, repo.WriteSessionOptions{}, func(ctx context.Context, w repo.RepositoryWriter) error {
--- a/tests/compat_test/compat_test.go
+++ b/tests/compat_test/compat_test.go
@@ -8,12 +8,14 @@

 	"github.com/stretchr/testify/require"

+	"github.com/kopia/kopia/internal/testutil"
 	"github.com/kopia/kopia/tests/testenv"
 )

 var (
 	kopiaCurrentExe = os.Getenv("KOPIA_CURRENT_EXE")
 	kopia08exe      = os.Getenv("KOPIA_08_EXE")
+	kopia017exe     = os.Getenv("KOPIA_017_EXE")
 )

 func TestRepoCreatedWith08CanBeOpenedWithCurrent(t *testing.T) {
@@ -131,3 +133,71 @@ func TestRepoCreatedWithCurrentCannotBeOpenedWith08(t *testing.T) {
 	e2 := testenv.NewCLITest(t, testenv.RepoFormatNotImportant, runner08)
 	e2.RunAndExpectFailure(t, "repo", "connect", "filesystem", "--path", e1.RepoDir)
 }
+
+func TestClientConnectedUsingV017CanConnectUsingCurrent(t *testing.T) {
+	t.Parallel()
+
+	if kopiaCurrentExe == "" {
+		t.Skip()
+	}
+
+	if kopia017exe == "" {
+		t.Skip()
+	}
+
+	runnerCurrent := testenv.NewExeRunnerWithBinary(t, kopiaCurrentExe)
+	runner017 := testenv.NewExeRunnerWithBinary(t, kopia017exe)
+
+	// create repository using v0.17 and start a server
+	e1 := testenv.NewCLITest(t, testenv.RepoFormatNotImportant, runner017)
+	e1.RunAndExpectSuccess(t, "repo", "create", "filesystem", "--path", e1.RepoDir)
+	e1.RunAndExpectSuccess(t, "server", "users", "add", "foo@bar", "--user-password", "baz")
+
+	var sp testutil.ServerParameters
+
+	tlsCert := filepath.Join(e1.ConfigDir, "tls.cert")
+	tlsKey := filepath.Join(e1.ConfigDir, "tls.key")
+
+	wait, kill := e1.RunAndProcessStderr(t, sp.ProcessOutput,
+		"server", "start",
+		"--address=localhost:0",
+		"--server-control-username=admin-user",
+		"--server-control-password=admin-pwd",
+		"--tls-generate-cert",
+		"--tls-key-file", tlsKey,
+		"--tls-cert-file", tlsCert,
+		"--tls-generate-rsa-key-size=2048", // use shorter key size to speed up generation
+	)
+
+	t.Logf("detected server parameters %#v", sp)
+
+	defer wait()
+	defer kill()
+
+	time.Sleep(3 * time.Second)
+
+	// connect to the server using 0.17
+	e2 := testenv.NewCLITest(t, testenv.RepoFormatNotImportant, runner017)
+	defer e2.RunAndExpectSuccess(t, "repo", "disconnect")
+
+	e2.RunAndExpectSuccess(t,
+		"repo", "connect", "server",
+		"--url", sp.BaseURL+"/",
+		"--server-cert-fingerprint", sp.SHA256Fingerprint,
+		"--override-username", "foo",
+		"--override-hostname", "bar",
+		"--password", "baz",
+	)
+
+	// we are providing custom password to connect, make sure we won't be providing
+	// (different) default password via environment variable, as command-line password
+	// takes precedence over persisted password.
+	delete(e2.Environment, "KOPIA_PASSWORD")
+
+	e2.RunAndExpectSuccess(t, "snapshot", "ls")
+
+	// now switch to using latest executable and old config file,
+	// everything should still work
+	e2.Runner = runnerCurrent
+	e2.RunAndExpectSuccess(t, "snapshot", "ls")
+}
--- a/tests/end_to_end_test/api_server_repository_test.go
+++ b/tests/end_to_end_test/api_server_repository_test.go
@@ -136,7 +136,6 @@ func testAPIServerRepository(t *testing.T, allowRepositoryUsers bool) {
 	rep, err := servertesting.ConnectAndOpenAPIServer(t, ctx2, &repo.APIServerInfo{
 		BaseURL:                             sp.BaseURL,
 		TrustedServerCertificateFingerprint: sp.SHA256Fingerprint,
-		LocalCacheKeyDerivationAlgorithm:    repo.DefaultServerRepoCacheKeyDerivationAlgorithm,
 	}, repo.ClientOptions{
 		Username: "foo",
 		Hostname: "bar",
@@ -259,7 +258,6 @@ func testAPIServerRepository(t *testing.T, allowRepositoryUsers bool) {
 	servertesting.ConnectAndOpenAPIServer(t, ctx, &repo.APIServerInfo{
 		BaseURL:                             sp.BaseURL,
 		TrustedServerCertificateFingerprint: sp.SHA256Fingerprint,
-		LocalCacheKeyDerivationAlgorithm:    repo.DefaultServerRepoCacheKeyDerivationAlgorithm,
 	}, repo.ClientOptions{
 		Username: "foo",
 		Hostname: "bar",
@@ -330,7 +328,6 @@ func TestFindManifestsPaginationOverGRPC(t *testing.T) {
 	rep, err := servertesting.ConnectAndOpenAPIServer(t, ctx, &repo.APIServerInfo{
 		BaseURL:                             sp.BaseURL,
 		TrustedServerCertificateFingerprint: sp.SHA256Fingerprint,
-		LocalCacheKeyDerivationAlgorithm:    repo.DefaultServerRepoCacheKeyDerivationAlgorithm,
 	}, repo.ClientOptions{
 		Username: "foo",
 		Hostname: "bar",
--- a/tools/gettool/checksums.txt
+++ b/tools/gettool/checksums.txt
@@ -25,6 +25,12 @@ https://github.com/gotestyourself/gotestsum/releases/download/v1.11.0/gotestsum_
 https://github.com/gotestyourself/gotestsum/releases/download/v1.11.0/gotestsum_1.11.0_linux_arm64.tar.gz: 51c7fe29216678edaaa96bb67e38d58437fd54a83468f58a32513995f575dcc3
 https://github.com/gotestyourself/gotestsum/releases/download/v1.11.0/gotestsum_1.11.0_linux_armv6.tar.gz: 79a6a904d73a7b6b010f82205803e0c0a8a202a63f51e93e555e2f9be8aa3ba3
 https://github.com/gotestyourself/gotestsum/releases/download/v1.11.0/gotestsum_1.11.0_windows_amd64.tar.gz: 1518b3dd6a44b5684e9732121933f52b9c3ccab3a6e9efdeac41e7b03f97d019
+https://github.com/kopia/kopia/releases/download/v0.17.0/kopia-0.17.0-linux-arm.tar.gz: 25804d7271a0dfe6d0821270c5640caa01da5e05a03a7c4783fd1edafb234d51
+https://github.com/kopia/kopia/releases/download/v0.17.0/kopia-0.17.0-linux-arm64.tar.gz: 9679415cd2717a90cb6a793aa2d4accde4059084245b27fa4807d7e13fbe40a0
+https://github.com/kopia/kopia/releases/download/v0.17.0/kopia-0.17.0-linux-x64.tar.gz: 6851bba9f49c2ca2cabc5bec85a813149a180472d1e338fad42a8285dad047ee
+https://github.com/kopia/kopia/releases/download/v0.17.0/kopia-0.17.0-macOS-arm64.tar.gz: e81b2b7a91501b7d4f834daaec78c11f910ed1a8f50ba46ba4c9725c87787c9e
+https://github.com/kopia/kopia/releases/download/v0.17.0/kopia-0.17.0-macOS-x64.tar.gz: 8f0c41835b70a1c529cb6e946193785b10df1da09c5bfe3cc96a3007226b04d6
+https://github.com/kopia/kopia/releases/download/v0.17.0/kopia-0.17.0-windows-x64.zip: 0bcfc616e52fe3847e7d0484fb6612512e926990277a37c8ced446dcce2dc6cb
 https://github.com/kopia/kopia/releases/download/v0.8.4/kopia-0.8.4-linux-arm.tar.gz: 31e9ecd9600dc60f98d4777fb64043b3431ad758dc7ba57d9a7661a103946d6f
 https://github.com/kopia/kopia/releases/download/v0.8.4/kopia-0.8.4-linux-arm64.tar.gz: 3ad81fd7e856ec177b737130710823ef0e64a344be1233d9a7ef456c78e535f2
 https://github.com/kopia/kopia/releases/download/v0.8.4/kopia-0.8.4-linux-x64.tar.gz: 118e3eece462d6e5bd8e357f6cbb48eabaecc3a22b99c804b54eaba6f6f1b7d5
--- a/tools/tools.mk
+++ b/tools/tools.mk
@@ -206,6 +206,13 @@ kopia08=$(kopia08_dir)$(slash)kopia$(exe_suffix)
 $(kopia08):
 	go run github.com/kopia/kopia/tools/gettool --tool kopia:$(kopia08_version) --output-dir $(kopia08_dir)

+kopia017_version=0.17.0
+kopia017_dir=$(TOOLS_DIR)$(slash)kopia-$(kopia017_version)
+kopia017=$(kopia017_dir)$(slash)kopia$(exe_suffix)
+
+$(kopia017):
+	go run github.com/kopia/kopia/tools/gettool --tool kopia:$(kopia017_version) --output-dir $(kopia017_dir)
+
 MINIO_MC_PATH=$(TOOLS_DIR)/bin/mc$(exe_suffix)

 $(MINIO_MC_PATH):
@@ -300,7 +307,7 @@ else
 maybehugo=
 endif

-ALL_TOOL_VERSIONS=node:$(NODE_VERSION),linter:$(GOLANGCI_LINT_VERSION),hugo:$(HUGO_VERSION),rclone:$(RCLONE_VERSION),gotestsum:$(GOTESTSUM_VERSION),goreleaser:$(GORELEASER_VERSION),kopia:0.8.4,gitchglog:$(GITCHGLOG_VERSION)
+ALL_TOOL_VERSIONS=node:$(NODE_VERSION),linter:$(GOLANGCI_LINT_VERSION),hugo:$(HUGO_VERSION),rclone:$(RCLONE_VERSION),gotestsum:$(GOTESTSUM_VERSION),goreleaser:$(GORELEASER_VERSION),kopia:0.8.4,kopia:0.17.0,gitchglog:$(GITCHGLOG_VERSION)

 verify-all-tool-checksums:
 	go run github.com/kopia/kopia/tools/gettool --test-all \