feat(general): allow setting key derivation algorithm for format blob (#3779)

Adds support to set the algorithm to derive the key used to encrypt
the repository format blob.

cli/command_repository_create.go

@@ -7,6 +7,7 @@ import (
 	"github.com/alecthomas/kingpin/v2"
 	"github.com/pkg/errors"
 
+	"github.com/kopia/kopia/internal/crypto"
 	"github.com/kopia/kopia/repo"
 	"github.com/kopia/kopia/repo/blob"
 	"github.com/kopia/kopia/repo/ecc"
@@ -24,15 +25,16 @@ $ kopia repository validate-provider
 `
 
 type commandRepositoryCreate struct {
-	createBlockHashFormat         string
-	createBlockEncryptionFormat   string
-	createBlockECCFormat          string
-	createBlockECCOverheadPercent int
-	createSplitter                string
-	createOnly                    bool
-	createFormatVersion           int
-	retentionMode                 string
-	retentionPeriod               time.Duration
+	createBlockHashFormat             string
+	createBlockEncryptionFormat       string
+	createBlockECCFormat              string
+	createBlockECCOverheadPercent     int
+	createBlockKeyDerivationAlgorithm string
+	createSplitter                    string
+	createOnly                        bool
+	createFormatVersion               int
+	retentionMode                     string
+	retentionPeriod                   time.Duration
 
 	co  connectOptions
 	svc advancedAppServices
@@ -51,6 +53,8 @@ func (c *commandRepositoryCreate) setup(svc advancedAppServices, parent commandP
 	cmd.Flag("format-version", "Force a particular repository format version (1, 2 or 3, 0==default)").IntVar(&c.createFormatVersion)
 	cmd.Flag("retention-mode", "Set the blob retention-mode for supported storage backends.").EnumVar(&c.retentionMode, blob.Governance.String(), blob.Compliance.String())
 	cmd.Flag("retention-period", "Set the blob retention-period for supported storage backends.").DurationVar(&c.retentionPeriod)
+	//nolint:lll
+	cmd.Flag("format-block-key-derivation-algorithm", "Algorithm to derive the encryption key for the format block from the repository password").Default(crypto.DefaultKeyDerivationAlgorithm).EnumVar(&c.createBlockKeyDerivationAlgorithm, crypto.AllowedKeyDerivationAlgorithms()...)
 
 	c.co.setup(svc, cmd)
 	c.svc = svc
@@ -90,8 +94,9 @@ func (c *commandRepositoryCreate) newRepositoryOptionsFromFlags() *repo.NewRepos
 			Splitter: c.createSplitter,
 		},
 
-		RetentionMode:   blob.RetentionMode(c.retentionMode),
-		RetentionPeriod: c.retentionPeriod,
+		RetentionMode:                     blob.RetentionMode(c.retentionMode),
+		RetentionPeriod:                   c.retentionPeriod,
+		FormatBlockKeyDerivationAlgorithm: c.createBlockKeyDerivationAlgorithm,
 	}
 }
 
@@ -130,6 +135,7 @@ func (c *commandRepositoryCreate) runCreateCommandWithStorage(ctx context.Contex
 
 	log(ctx).Infof("  block hash:          %v", options.BlockFormat.Hash)
 	log(ctx).Infof("  encryption:          %v", options.BlockFormat.Encryption)
+	log(ctx).Infof("  key derivation:      %v", options.FormatBlockKeyDerivationAlgorithm)
 
 	if options.BlockFormat.ECC != "" && options.BlockFormat.ECCOverheadPercent > 0 {
 		log(ctx).Infof("  ecc:                 %v with %v%% overhead", options.BlockFormat.ECC, options.BlockFormat.ECCOverheadPercent)

repo/format/format_manager.go

@@ -455,6 +455,8 @@ func Initialize(ctx context.Context, st blob.Storage, formatBlob *KopiaRepositor
 		formatBlob.EncryptionAlgorithm = DefaultFormatEncryption
 	}
 
+	// In legacy versions, the KeyDerivationAlgorithm may not be present in the
+	// KopiaRepositoryJson. In those cases default to using Scrypt.
 	if formatBlob.KeyDerivationAlgorithm == "" {
 		formatBlob.KeyDerivationAlgorithm = DefaultKeyDerivationAlgorithm
 	}

repo/initialize.go

@@ -35,12 +35,13 @@ const (
 // NewRepositoryOptions specifies options that apply to newly created repositories.
 // All fields are optional, when not provided, reasonable defaults will be used.
 type NewRepositoryOptions struct {
-	UniqueID        []byte               `json:"uniqueID"` // force the use of particular unique ID
-	BlockFormat     format.ContentFormat `json:"blockFormat"`
-	DisableHMAC     bool                 `json:"disableHMAC"`
-	ObjectFormat    format.ObjectFormat  `json:"objectFormat"` // object format
-	RetentionMode   blob.RetentionMode   `json:"retentionMode,omitempty"`
-	RetentionPeriod time.Duration        `json:"retentionPeriod,omitempty"`
+	UniqueID                          []byte               `json:"uniqueID"` // force the use of particular unique ID
+	BlockFormat                       format.ContentFormat `json:"blockFormat"`
+	DisableHMAC                       bool                 `json:"disableHMAC"`
+	ObjectFormat                      format.ObjectFormat  `json:"objectFormat"` // object format
+	RetentionMode                     blob.RetentionMode   `json:"retentionMode,omitempty"`
+	RetentionPeriod                   time.Duration        `json:"retentionPeriod,omitempty"`
+	FormatBlockKeyDerivationAlgorithm string               `json:"formatBlockKeyDerivationAlgorithm,omitempty"`
 }
 
 // Initialize creates initial repository data structures in the specified storage with given credentials.
@@ -66,7 +67,7 @@ func formatBlobFromOptions(opt *NewRepositoryOptions) *format.KopiaRepositoryJSO
 		Tool:                   "https://github.com/kopia/kopia",
 		BuildInfo:              BuildInfo,
 		BuildVersion:           BuildVersion,
-		KeyDerivationAlgorithm: format.DefaultKeyDerivationAlgorithm,
+		KeyDerivationAlgorithm: opt.FormatBlockKeyDerivationAlgorithm,
 		UniqueID:               applyDefaultRandomBytes(opt.UniqueID, format.UniqueIDLengthBytes),
 		EncryptionAlgorithm:    format.DefaultFormatEncryption,
 	}

repo/repository_test.go

@@ -876,6 +876,7 @@ func TestDeriveKey(t *testing.T) {
 			NewRepositoryOptions: func(nro *repo.NewRepositoryOptions) {
 				// do not set nro.BlockFormat.MasterKey
 				nro.UniqueID = uniqueID
+				nro.FormatBlockKeyDerivationAlgorithm = format.DefaultKeyDerivationAlgorithm
 			},
 		})
 

tests/end_to_end_test/repository_connect_test.go

@@ -1,6 +1,7 @@
 package endtoend_test
 
 import (
+	"encoding/json"
 	"os"
 	"path/filepath"
 	"strings"
@@ -9,7 +10,9 @@ import (
 	"github.com/google/uuid"
 	"github.com/stretchr/testify/require"
 
+	"github.com/kopia/kopia/internal/crypto"
 	"github.com/kopia/kopia/internal/testutil"
+	"github.com/kopia/kopia/repo/format"
 	"github.com/kopia/kopia/tests/testenv"
 )
 
@@ -104,3 +107,45 @@ func TestReconnectUsingToken(t *testing.T) {
 	e.RunAndExpectSuccess(t, reconnectArgs...)
 	e.RunAndExpectSuccess(t, "repo", "status")
 }
+
+func TestRepoConnectKeyDerivationAlgorithm(t *testing.T) {
+	t.Parallel()
+	for _, algorithm := range crypto.AllowedKeyDerivationAlgorithms() {
+		runner := testenv.NewInProcRunner(t)
+		e := testenv.NewCLITest(t, testenv.RepoFormatNotImportant, runner)
+
+		e.RunAndExpectSuccess(t, "repo", "create", "filesystem", "--path", e.RepoDir, "--format-block-key-derivation-algorithm", algorithm)
+
+		e.RunAndExpectSuccess(t, "repo", "disconnect")
+		e.RunAndExpectSuccess(t, "repo", "connect", "filesystem", "--path", e.RepoDir)
+
+		kopiaRepoPath := filepath.Join(e.RepoDir, "kopia.repository.f")
+		dat, err := os.ReadFile(kopiaRepoPath)
+		require.NoError(t, err)
+		var repoJSON format.KopiaRepositoryJSON
+		json.Unmarshal(dat, &repoJSON)
+		require.Equal(t, repoJSON.KeyDerivationAlgorithm, algorithm)
+	}
+}
+
+func TestRepoConnectBadKeyDerivationAlgorithm(t *testing.T) {
+	t.Parallel()
+	runner := testenv.NewInProcRunner(t)
+	e := testenv.NewCLITest(t, testenv.RepoFormatNotImportant, runner)
+
+	e.RunAndExpectSuccess(t, "repo", "create", "filesystem", "--path", e.RepoDir, "--format-block-key-derivation-algorithm", crypto.DefaultKeyDerivationAlgorithm)
+	e.RunAndExpectSuccess(t, "repo", "disconnect")
+
+	kopiaRepoPath := filepath.Join(e.RepoDir, "kopia.repository.f")
+	dat, err := os.ReadFile(kopiaRepoPath)
+	require.NoError(t, err)
+	var repoJSON format.KopiaRepositoryJSON
+	json.Unmarshal(dat, &repoJSON)
+
+	repoJSON.KeyDerivationAlgorithm = "badalgorithm"
+
+	jsonString, _ := json.Marshal(repoJSON)
+	os.WriteFile(kopiaRepoPath, jsonString, os.ModePerm)
+
+	e.RunAndExpectFailure(t, "repo", "connect", "filesystem", "--path", e.RepoDir)
+}