From b60cac48608bb8cd8fad69e535b5d351e49916f2 Mon Sep 17 00:00:00 2001 From: Jarek Kowalski Date: Sun, 17 Nov 2024 10:15:00 -0800 Subject: [PATCH] fix(cli): fixed v0.18.0 failures using v0.17.0-generated config (#4255) This was caused by the client using key derivation algorithm from a config file (which did not have it when it was generated using old version of Kopia). Fixes #4254 --- Makefile | 3 +- cli/command_repository_connect_server.go | 3 - internal/servertesting/servertesting.go | 2 - repo/api_server_repository.go | 5 ++ repo/open.go | 7 +- repo/repository_test.go | 4 +- tests/compat_test/compat_test.go | 70 +++++++++++++++++++ .../api_server_repository_test.go | 3 - tools/gettool/checksums.txt | 6 ++ tools/tools.mk | 9 ++- 10 files changed, 99 insertions(+), 13 deletions(-) diff --git a/Makefile b/Makefile index 41f638cc2..c5a225065 100644 --- a/Makefile +++ b/Makefile @@ -311,8 +311,9 @@ $(TESTING_ACTION_EXE): tests/testingaction/main.go compat-tests: export KOPIA_CURRENT_EXE=$(CURDIR)/$(kopia_ui_embedded_exe) compat-tests: export KOPIA_08_EXE=$(kopia08) +compat-tests: export KOPIA_017_EXE=$(kopia017) compat-tests: GOTESTSUM_FLAGS=--format=testname --no-summary=skipped --jsonfile=.tmp.compat-tests.json -compat-tests: $(kopia_ui_embedded_exe) $(kopia08) $(gotestsum) +compat-tests: $(kopia_ui_embedded_exe) $(kopia08) $(kopia017) $(gotestsum) $(GO_TEST) $(TEST_FLAGS) -count=$(REPEAT_TEST) -parallel $(PARALLEL) -timeout 3600s github.com/kopia/kopia/tests/compat_test # -$(gotestsum) tool slowest --jsonfile .tmp.compat-tests.json --threshold 1000ms diff --git a/cli/command_repository_connect_server.go b/cli/command_repository_connect_server.go index 35de211a2..2c436dea9 100644 --- a/cli/command_repository_connect_server.go +++ b/cli/command_repository_connect_server.go @@ -36,9 +36,6 @@ func (c *commandRepositoryConnectServer) setup(svc advancedAppServices, parent c func (c *commandRepositoryConnectServer) run(ctx context.Context) error { localCacheKeyDerivationAlgorithm := c.connectAPIServerLocalCacheKeyDerivationAlgorithm - if localCacheKeyDerivationAlgorithm == "" { - localCacheKeyDerivationAlgorithm = repo.DefaultServerRepoCacheKeyDerivationAlgorithm - } as := &repo.APIServerInfo{ BaseURL: strings.TrimSuffix(c.connectAPIServerURL, "/"), diff --git a/internal/servertesting/servertesting.go b/internal/servertesting/servertesting.go index 898e2a1fe..875118e0f 100644 --- a/internal/servertesting/servertesting.go +++ b/internal/servertesting/servertesting.go @@ -79,8 +79,6 @@ func StartServer(t *testing.T, env *repotesting.Environment, tls bool) *repo.API asi.BaseURL = hs.URL } - asi.LocalCacheKeyDerivationAlgorithm = repo.DefaultServerRepoCacheKeyDerivationAlgorithm - t.Cleanup(hs.Close) return asi diff --git a/repo/api_server_repository.go b/repo/api_server_repository.go index 13207e613..11cc40483 100644 --- a/repo/api_server_repository.go +++ b/repo/api_server_repository.go @@ -7,6 +7,11 @@ ) // APIServerInfo is remote repository configuration stored in local configuration. +// +// NOTE: this structure is persistent on disk may be read/written using +// different versions of Kopia, so it must be backwards-compatible. +// +// Apply appropriate defaults when reading. type APIServerInfo struct { BaseURL string `json:"url"` TrustedServerCertificateFingerprint string `json:"serverCertFingerprint"` diff --git a/repo/open.go b/repo/open.go index 52dbac0cb..2745b2522 100644 --- a/repo/open.go +++ b/repo/open.go @@ -145,7 +145,12 @@ func getContentCacheOrNil(ctx context.Context, si *APIServerInfo, opt *content.C const cacheEncryptionKeySize = 32 - cacheEncryptionKey, err := crypto.DeriveKeyFromPassword(password, saltWithPurpose, cacheEncryptionKeySize, si.LocalCacheKeyDerivationAlgorithm) + keyAlgo := si.LocalCacheKeyDerivationAlgorithm + if keyAlgo == "" { + keyAlgo = DefaultServerRepoCacheKeyDerivationAlgorithm + } + + cacheEncryptionKey, err := crypto.DeriveKeyFromPassword(password, saltWithPurpose, cacheEncryptionKeySize, keyAlgo) if err != nil { return nil, errors.Wrap(err, "unable to derive cache encryption key from password") } diff --git a/repo/repository_test.go b/repo/repository_test.go index f95108f1c..5c410c332 100644 --- a/repo/repository_test.go +++ b/repo/repository_test.go @@ -668,10 +668,10 @@ func(ctx context.Context, w repo.RepositoryWriter) error { }, }) - defer rep.Close(ctx) //nolint:errcheck,staticcheck - require.NoError(t, err) + defer rep.Close(ctx) //nolint:errcheck,staticcheck + var oid object.ID require.NoError(t, repo.WriteSession(ctx, rep, repo.WriteSessionOptions{}, func(ctx context.Context, w repo.RepositoryWriter) error { diff --git a/tests/compat_test/compat_test.go b/tests/compat_test/compat_test.go index 383472cb5..78987b7f4 100644 --- a/tests/compat_test/compat_test.go +++ b/tests/compat_test/compat_test.go @@ -8,12 +8,14 @@ "github.com/stretchr/testify/require" + "github.com/kopia/kopia/internal/testutil" "github.com/kopia/kopia/tests/testenv" ) var ( kopiaCurrentExe = os.Getenv("KOPIA_CURRENT_EXE") kopia08exe = os.Getenv("KOPIA_08_EXE") + kopia017exe = os.Getenv("KOPIA_017_EXE") ) func TestRepoCreatedWith08CanBeOpenedWithCurrent(t *testing.T) { @@ -131,3 +133,71 @@ func TestRepoCreatedWithCurrentCannotBeOpenedWith08(t *testing.T) { e2 := testenv.NewCLITest(t, testenv.RepoFormatNotImportant, runner08) e2.RunAndExpectFailure(t, "repo", "connect", "filesystem", "--path", e1.RepoDir) } + +func TestClientConnectedUsingV017CanConnectUsingCurrent(t *testing.T) { + t.Parallel() + + if kopiaCurrentExe == "" { + t.Skip() + } + + if kopia017exe == "" { + t.Skip() + } + + runnerCurrent := testenv.NewExeRunnerWithBinary(t, kopiaCurrentExe) + runner017 := testenv.NewExeRunnerWithBinary(t, kopia017exe) + + // create repository using v0.17 and start a server + e1 := testenv.NewCLITest(t, testenv.RepoFormatNotImportant, runner017) + e1.RunAndExpectSuccess(t, "repo", "create", "filesystem", "--path", e1.RepoDir) + e1.RunAndExpectSuccess(t, "server", "users", "add", "foo@bar", "--user-password", "baz") + + var sp testutil.ServerParameters + + tlsCert := filepath.Join(e1.ConfigDir, "tls.cert") + tlsKey := filepath.Join(e1.ConfigDir, "tls.key") + + wait, kill := e1.RunAndProcessStderr(t, sp.ProcessOutput, + "server", "start", + "--address=localhost:0", + "--server-control-username=admin-user", + "--server-control-password=admin-pwd", + "--tls-generate-cert", + "--tls-key-file", tlsKey, + "--tls-cert-file", tlsCert, + "--tls-generate-rsa-key-size=2048", // use shorter key size to speed up generation + ) + + t.Logf("detected server parameters %#v", sp) + + defer wait() + defer kill() + + time.Sleep(3 * time.Second) + + // connect to the server using 0.17 + e2 := testenv.NewCLITest(t, testenv.RepoFormatNotImportant, runner017) + defer e2.RunAndExpectSuccess(t, "repo", "disconnect") + + e2.RunAndExpectSuccess(t, + "repo", "connect", "server", + "--url", sp.BaseURL+"/", + "--server-cert-fingerprint", sp.SHA256Fingerprint, + "--override-username", "foo", + "--override-hostname", "bar", + "--password", "baz", + ) + + // we are providing custom password to connect, make sure we won't be providing + // (different) default password via environment variable, as command-line password + // takes precedence over persisted password. + delete(e2.Environment, "KOPIA_PASSWORD") + + e2.RunAndExpectSuccess(t, "snapshot", "ls") + + // now switch to using latest executable and old config file, + // everything should still work + e2.Runner = runnerCurrent + e2.RunAndExpectSuccess(t, "snapshot", "ls") +} diff --git a/tests/end_to_end_test/api_server_repository_test.go b/tests/end_to_end_test/api_server_repository_test.go index f76592629..1b20a5539 100644 --- a/tests/end_to_end_test/api_server_repository_test.go +++ b/tests/end_to_end_test/api_server_repository_test.go @@ -136,7 +136,6 @@ func testAPIServerRepository(t *testing.T, allowRepositoryUsers bool) { rep, err := servertesting.ConnectAndOpenAPIServer(t, ctx2, &repo.APIServerInfo{ BaseURL: sp.BaseURL, TrustedServerCertificateFingerprint: sp.SHA256Fingerprint, - LocalCacheKeyDerivationAlgorithm: repo.DefaultServerRepoCacheKeyDerivationAlgorithm, }, repo.ClientOptions{ Username: "foo", Hostname: "bar", @@ -259,7 +258,6 @@ func testAPIServerRepository(t *testing.T, allowRepositoryUsers bool) { servertesting.ConnectAndOpenAPIServer(t, ctx, &repo.APIServerInfo{ BaseURL: sp.BaseURL, TrustedServerCertificateFingerprint: sp.SHA256Fingerprint, - LocalCacheKeyDerivationAlgorithm: repo.DefaultServerRepoCacheKeyDerivationAlgorithm, }, repo.ClientOptions{ Username: "foo", Hostname: "bar", @@ -330,7 +328,6 @@ func TestFindManifestsPaginationOverGRPC(t *testing.T) { rep, err := servertesting.ConnectAndOpenAPIServer(t, ctx, &repo.APIServerInfo{ BaseURL: sp.BaseURL, TrustedServerCertificateFingerprint: sp.SHA256Fingerprint, - LocalCacheKeyDerivationAlgorithm: repo.DefaultServerRepoCacheKeyDerivationAlgorithm, }, repo.ClientOptions{ Username: "foo", Hostname: "bar", diff --git a/tools/gettool/checksums.txt b/tools/gettool/checksums.txt index d8ee53d16..7b7983bc6 100644 --- a/tools/gettool/checksums.txt +++ b/tools/gettool/checksums.txt @@ -25,6 +25,12 @@ https://github.com/gotestyourself/gotestsum/releases/download/v1.11.0/gotestsum_ https://github.com/gotestyourself/gotestsum/releases/download/v1.11.0/gotestsum_1.11.0_linux_arm64.tar.gz: 51c7fe29216678edaaa96bb67e38d58437fd54a83468f58a32513995f575dcc3 https://github.com/gotestyourself/gotestsum/releases/download/v1.11.0/gotestsum_1.11.0_linux_armv6.tar.gz: 79a6a904d73a7b6b010f82205803e0c0a8a202a63f51e93e555e2f9be8aa3ba3 https://github.com/gotestyourself/gotestsum/releases/download/v1.11.0/gotestsum_1.11.0_windows_amd64.tar.gz: 1518b3dd6a44b5684e9732121933f52b9c3ccab3a6e9efdeac41e7b03f97d019 +https://github.com/kopia/kopia/releases/download/v0.17.0/kopia-0.17.0-linux-arm.tar.gz: 25804d7271a0dfe6d0821270c5640caa01da5e05a03a7c4783fd1edafb234d51 +https://github.com/kopia/kopia/releases/download/v0.17.0/kopia-0.17.0-linux-arm64.tar.gz: 9679415cd2717a90cb6a793aa2d4accde4059084245b27fa4807d7e13fbe40a0 +https://github.com/kopia/kopia/releases/download/v0.17.0/kopia-0.17.0-linux-x64.tar.gz: 6851bba9f49c2ca2cabc5bec85a813149a180472d1e338fad42a8285dad047ee +https://github.com/kopia/kopia/releases/download/v0.17.0/kopia-0.17.0-macOS-arm64.tar.gz: e81b2b7a91501b7d4f834daaec78c11f910ed1a8f50ba46ba4c9725c87787c9e +https://github.com/kopia/kopia/releases/download/v0.17.0/kopia-0.17.0-macOS-x64.tar.gz: 8f0c41835b70a1c529cb6e946193785b10df1da09c5bfe3cc96a3007226b04d6 +https://github.com/kopia/kopia/releases/download/v0.17.0/kopia-0.17.0-windows-x64.zip: 0bcfc616e52fe3847e7d0484fb6612512e926990277a37c8ced446dcce2dc6cb https://github.com/kopia/kopia/releases/download/v0.8.4/kopia-0.8.4-linux-arm.tar.gz: 31e9ecd9600dc60f98d4777fb64043b3431ad758dc7ba57d9a7661a103946d6f https://github.com/kopia/kopia/releases/download/v0.8.4/kopia-0.8.4-linux-arm64.tar.gz: 3ad81fd7e856ec177b737130710823ef0e64a344be1233d9a7ef456c78e535f2 https://github.com/kopia/kopia/releases/download/v0.8.4/kopia-0.8.4-linux-x64.tar.gz: 118e3eece462d6e5bd8e357f6cbb48eabaecc3a22b99c804b54eaba6f6f1b7d5 diff --git a/tools/tools.mk b/tools/tools.mk index 79407b6c6..9685d11b4 100644 --- a/tools/tools.mk +++ b/tools/tools.mk @@ -206,6 +206,13 @@ kopia08=$(kopia08_dir)$(slash)kopia$(exe_suffix) $(kopia08): go run github.com/kopia/kopia/tools/gettool --tool kopia:$(kopia08_version) --output-dir $(kopia08_dir) +kopia017_version=0.17.0 +kopia017_dir=$(TOOLS_DIR)$(slash)kopia-$(kopia017_version) +kopia017=$(kopia017_dir)$(slash)kopia$(exe_suffix) + +$(kopia017): + go run github.com/kopia/kopia/tools/gettool --tool kopia:$(kopia017_version) --output-dir $(kopia017_dir) + MINIO_MC_PATH=$(TOOLS_DIR)/bin/mc$(exe_suffix) $(MINIO_MC_PATH): @@ -300,7 +307,7 @@ else maybehugo= endif -ALL_TOOL_VERSIONS=node:$(NODE_VERSION),linter:$(GOLANGCI_LINT_VERSION),hugo:$(HUGO_VERSION),rclone:$(RCLONE_VERSION),gotestsum:$(GOTESTSUM_VERSION),goreleaser:$(GORELEASER_VERSION),kopia:0.8.4,gitchglog:$(GITCHGLOG_VERSION) +ALL_TOOL_VERSIONS=node:$(NODE_VERSION),linter:$(GOLANGCI_LINT_VERSION),hugo:$(HUGO_VERSION),rclone:$(RCLONE_VERSION),gotestsum:$(GOTESTSUM_VERSION),goreleaser:$(GORELEASER_VERSION),kopia:0.8.4,kopia:0.17.0,gitchglog:$(GITCHGLOG_VERSION) verify-all-tool-checksums: go run github.com/kopia/kopia/tools/gettool --test-all \