refactor(repository): Move encryption into a common package in preparation to handle secrets (#3351)

internal/crypto/aesgcm.go

@@ -0,0 +1,80 @@
+// Package crypto implements common symmetric-encryption and key-derivation functions.
+package crypto
+
+import (
+	"crypto/aes"
+	"crypto/cipher"
+	"crypto/rand"
+	"io"
+
+	"github.com/pkg/errors"
+)
+
+//nolint:gochecknoglobals
+var (
+	purposeAESKey   = []byte("AES")
+	purposeAuthData = []byte("CHECKSUM")
+)
+
+func initCrypto(masterKey, salt []byte) (cipher.AEAD, []byte, error) {
+	aesKey := DeriveKeyFromMasterKey(masterKey, salt, purposeAESKey, 32)     //nolint:gomnd
+	authData := DeriveKeyFromMasterKey(masterKey, salt, purposeAuthData, 32) //nolint:gomnd
+
+	blk, err := aes.NewCipher(aesKey)
+	if err != nil {
+		return nil, nil, errors.Wrap(err, "cannot create cipher")
+	}
+
+	aead, err := cipher.NewGCM(blk)
+	if err != nil {
+		return nil, nil, errors.Wrap(err, "cannot create cipher")
+	}
+
+	return aead, authData, nil
+}
+
+// EncryptAes256Gcm encrypts data with AES 256 GCM.
+func EncryptAes256Gcm(data, masterKey, salt []byte) ([]byte, error) {
+	aead, authData, err := initCrypto(masterKey, salt)
+	if err != nil {
+		return nil, errors.Wrap(err, "unable to initialize crypto")
+	}
+
+	nonceLength := aead.NonceSize()
+	noncePlusContentLength := nonceLength + len(data)
+	cipherText := make([]byte, noncePlusContentLength+aead.Overhead())
+
+	// Store nonce at the beginning of ciphertext.
+	nonce := cipherText[0:nonceLength]
+	if _, err := io.ReadFull(rand.Reader, nonce); err != nil {
+		return nil, errors.Wrap(err, "error reading random bytes for nonce")
+	}
+
+	b := aead.Seal(cipherText[nonceLength:nonceLength], nonce, data, authData)
+	data = nonce[0 : nonceLength+len(b)]
+
+	return data, nil
+}
+
+// DecryptAes256Gcm encrypts data with AES 256 GCM.
+func DecryptAes256Gcm(data, masterKey, salt []byte) ([]byte, error) {
+	aead, authData, err := initCrypto(masterKey, salt)
+	if err != nil {
+		return nil, errors.Wrap(err, "cannot initialize cipher")
+	}
+
+	data = append([]byte(nil), data...)
+	if len(data) < aead.NonceSize() {
+		return nil, errors.Errorf("invalid encrypted payload, too short")
+	}
+
+	nonce := data[0:aead.NonceSize()]
+	payload := data[aead.NonceSize():]
+
+	plainText, err := aead.Open(payload[:0], nonce, payload, authData)
+	if err != nil {
+		return nil, errors.Errorf("unable to decrypt repository blob, invalid credentials?")
+	}
+
+	return plainText, nil
+}

internal/crypto/key_derivation.go

@@ -1,4 +1,4 @@
-package format
+package crypto
 
 import (
 	"crypto/sha256"
@@ -8,9 +8,9 @@ import (
 )
 
 // DeriveKeyFromMasterKey computes a key for a specific purpose and length using HKDF based on the master key.
-func DeriveKeyFromMasterKey(masterKey, uniqueID, purpose []byte, length int) []byte {
+func DeriveKeyFromMasterKey(masterKey, salt, purpose []byte, length int) []byte {
 	key := make([]byte, length)
-	k := hkdf.New(sha256.New, masterKey, uniqueID, purpose)
+	k := hkdf.New(sha256.New, masterKey, salt, purpose)
 
 	if _, err := io.ReadFull(k, key); err != nil {
 		panic("unable to derive key from master key, this should never happen")

internal/crypto/key_derivation_nontest.go

@@ -0,0 +1,26 @@
+//go:build !testing
+// +build !testing
+
+package crypto
+
+import (
+	"github.com/pkg/errors"
+	"golang.org/x/crypto/scrypt"
+)
+
+// DefaultKeyDerivationAlgorithm is the key derivation algorithm for new configurations.
+const DefaultKeyDerivationAlgorithm = "scrypt-65536-8-1"
+
+// DeriveKeyFromPassword derives encryption key using the provided password and per-repository unique ID.
+func DeriveKeyFromPassword(password string, salt []byte, algorithm string) ([]byte, error) {
+	const masterKeySize = 32
+
+	switch algorithm {
+	case "scrypt-65536-8-1":
+		//nolint:wrapcheck,gomnd
+		return scrypt.Key([]byte(password), salt, 65536, 8, 1, masterKeySize)
+
+	default:
+		return nil, errors.Errorf("unsupported key algorithm: %v", algorithm)
+	}
+}

internal/crypto/key_derivation_testing.go

@@ -1,7 +1,7 @@
 //go:build testing
 // +build testing
 
-package format
+package crypto
 
 import (
 	"crypto/sha256"
@@ -12,11 +12,11 @@ import (
 // DefaultKeyDerivationAlgorithm is the key derivation algorithm for new configurations.
 const DefaultKeyDerivationAlgorithm = "testing-only-insecure"
 
-// DeriveFormatEncryptionKeyFromPassword derives encryption key using the provided password and per-repository unique ID.
-func (f *KopiaRepositoryJSON) DeriveFormatEncryptionKeyFromPassword(password string) ([]byte, error) {
+// DeriveKeyFromPassword derives encryption key using the provided password and per-repository unique ID.
+func DeriveKeyFromPassword(password string, salt []byte, algorithm string) ([]byte, error) {
 	const masterKeySize = 32
 
-	switch f.KeyDerivationAlgorithm {
+	switch algorithm {
 	case DefaultKeyDerivationAlgorithm:
 		h := sha256.New()
 		if _, err := h.Write([]byte(password)); err != nil {
@@ -26,6 +26,6 @@ func (f *KopiaRepositoryJSON) DeriveFormatEncryptionKeyFromPassword(password str
 		return h.Sum(nil), nil
 
 	default:
-		return nil, errors.Errorf("unsupported key algorithm: %v", f.KeyDerivationAlgorithm)
+		return nil, errors.Errorf("unsupported key algorithm: %v", algorithm)
 	}
 }

repo/format/crypto_key_derivation_nontest.go

@@ -1,26 +0,0 @@
-//go:build !testing
-// +build !testing
-
-package format
-
-import (
-	"github.com/pkg/errors"
-	"golang.org/x/crypto/scrypt"
-)
-
-// DefaultKeyDerivationAlgorithm is the key derivation algorithm for new configurations.
-const DefaultKeyDerivationAlgorithm = "scrypt-65536-8-1"
-
-// DeriveFormatEncryptionKeyFromPassword derives encryption key using the provided password and per-repository unique ID.
-func (f *KopiaRepositoryJSON) DeriveFormatEncryptionKeyFromPassword(password string) ([]byte, error) {
-	const masterKeySize = 32
-
-	switch f.KeyDerivationAlgorithm {
-	case "scrypt-65536-8-1":
-		//nolint:wrapcheck,gomnd
-		return scrypt.Key([]byte(password), f.UniqueID, 65536, 8, 1, masterKeySize)
-
-	default:
-		return nil, errors.Errorf("unsupported key algorithm: %v", f.KeyDerivationAlgorithm)
-	}
-}

repo/format/format_blob.go

@@ -3,16 +3,13 @@ package format
 
 import (
 	"context"
-	"crypto/aes"
-	"crypto/cipher"
 	"crypto/hmac"
-	"crypto/rand"
 	"crypto/sha256"
 	"encoding/json"
-	"io"
 
 	"github.com/pkg/errors"
 
+	"github.com/kopia/kopia/internal/crypto"
 	"github.com/kopia/kopia/internal/gather"
 	"github.com/kopia/kopia/repo/blob"
 )
@@ -20,6 +17,9 @@ import (
 // DefaultFormatEncryption is the identifier of the default format blob encryption algorithm.
 const DefaultFormatEncryption = "AES256_GCM"
 
+// DefaultKeyDerivationAlgorithm is the key derivation algorithm for new configurations.
+const DefaultKeyDerivationAlgorithm = crypto.DefaultKeyDerivationAlgorithm
+
 const (
 	aes256GcmEncryption             = "AES256_GCM"
 	lengthOfRecoverBlockLength      = 2 // number of bytes used to store recover block length
@@ -37,9 +37,6 @@ var ErrInvalidPassword = errors.Errorf("invalid repository password") // +checkl
 
 //nolint:gochecknoglobals
 var (
-	purposeAESKey   = []byte("AES")
-	purposeAuthData = []byte("CHECKSUM")
-
 	// formatBlobChecksumSecret is a HMAC secret used for checksumming the format content.
 	// It's not really a secret, but will provide positive identification of blocks that
 	// are repository format blocks.
@@ -73,6 +70,16 @@ func ParseKopiaRepositoryJSON(b []byte) (*KopiaRepositoryJSON, error) {
 	return f, nil
 }
 
+// DeriveFormatEncryptionKeyFromPassword derives encryption key using the provided password and per-repository unique ID.
+func (f *KopiaRepositoryJSON) DeriveFormatEncryptionKeyFromPassword(password string) ([]byte, error) {
+	res, err := crypto.DeriveKeyFromPassword(password, f.UniqueID, f.KeyDerivationAlgorithm)
+	if err != nil {
+		return nil, errors.Wrap(err, "Failed to derive format encryption key")
+	}
+
+	return res, nil
+}
+
 // RecoverFormatBlob attempts to recover format blob replica from the specified file.
 // The format blob can be either the prefix or a suffix of the given file.
 // optionally the length can be provided (if known) to speed up recovery.
@@ -190,65 +197,22 @@ func (f *KopiaRepositoryJSON) WriteKopiaRepositoryBlobWithID(ctx context.Context
 	return nil
 }
 
-func initCrypto(masterKey, repositoryID []byte) (cipher.AEAD, []byte, error) {
-	aesKey := DeriveKeyFromMasterKey(masterKey, repositoryID, purposeAESKey, 32)     //nolint:gomnd
-	authData := DeriveKeyFromMasterKey(masterKey, repositoryID, purposeAuthData, 32) //nolint:gomnd
-
-	blk, err := aes.NewCipher(aesKey)
-	if err != nil {
-		return nil, nil, errors.Wrap(err, "cannot create cipher")
-	}
-
-	aead, err := cipher.NewGCM(blk)
-	if err != nil {
-		return nil, nil, errors.Wrap(err, "cannot create cipher")
-	}
-
-	return aead, authData, nil
-}
-
 func encryptRepositoryBlobBytesAes256Gcm(data, masterKey, repositoryID []byte) ([]byte, error) {
-	aead, authData, err := initCrypto(masterKey, repositoryID)
+	res, err := crypto.EncryptAes256Gcm(data, masterKey, repositoryID)
 	if err != nil {
-		return nil, errors.Wrap(err, "unable to initialize crypto")
+		return nil, errors.Wrap(err, "Failed to encrypt blob")
 	}
 
-	nonceLength := aead.NonceSize()
-	noncePlusContentLength := nonceLength + len(data)
-	cipherText := make([]byte, noncePlusContentLength+aead.Overhead())
-
-	// Store nonce at the beginning of ciphertext.
-	nonce := cipherText[0:nonceLength]
-	if _, err := io.ReadFull(rand.Reader, nonce); err != nil {
-		return nil, errors.Wrap(err, "error reading random bytes for nonce")
-	}
-
-	b := aead.Seal(cipherText[nonceLength:nonceLength], nonce, data, authData)
-	data = nonce[0 : nonceLength+len(b)]
-
-	return data, nil
+	return res, nil
 }
 
 func decryptRepositoryBlobBytesAes256Gcm(data, masterKey, repositoryID []byte) ([]byte, error) {
-	aead, authData, err := initCrypto(masterKey, repositoryID)
+	res, err := crypto.DecryptAes256Gcm(data, masterKey, repositoryID)
 	if err != nil {
-		return nil, errors.Wrap(err, "cannot initialize cipher")
+		return nil, errors.Wrap(err, "Failed to decrypt blob")
 	}
 
-	data = append([]byte(nil), data...)
-	if len(data) < aead.NonceSize() {
-		return nil, errors.Errorf("invalid encrypted payload, too short")
-	}
-
-	nonce := data[0:aead.NonceSize()]
-	payload := data[aead.NonceSize():]
-
-	plainText, err := aead.Open(payload[:0], nonce, payload, authData)
-	if err != nil {
-		return nil, errors.Errorf("unable to decrypt repository blob, invalid credentials?")
-	}
-
-	return plainText, nil
+	return res, nil
 }
 
 func addFormatBlobChecksumAndLength(fb []byte) ([]byte, error) {

repo/open.go

@@ -14,6 +14,7 @@ import (
 
 	"github.com/kopia/kopia/internal/cache"
 	"github.com/kopia/kopia/internal/cacheprot"
+	"github.com/kopia/kopia/internal/crypto"
 	"github.com/kopia/kopia/internal/feature"
 	"github.com/kopia/kopia/internal/metrics"
 	"github.com/kopia/kopia/internal/retry"
@@ -257,10 +258,10 @@ func openWithConfig(ctx context.Context, st blob.Storage, cliOpts ClientOptions,
 	}
 
 	if fmgr.SupportsPasswordChange() {
-		cacheOpts.HMACSecret = format.DeriveKeyFromMasterKey(fmgr.GetHmacSecret(), fmgr.UniqueID(), localCacheIntegrityPurpose, localCacheIntegrityHMACSecretLength)
+		cacheOpts.HMACSecret = crypto.DeriveKeyFromMasterKey(fmgr.GetHmacSecret(), fmgr.UniqueID(), localCacheIntegrityPurpose, localCacheIntegrityHMACSecretLength)
 	} else {
 		// deriving from ufb.FormatEncryptionKey was actually a bug, that only matters will change when we change the password
-		cacheOpts.HMACSecret = format.DeriveKeyFromMasterKey(fmgr.FormatEncryptionKey(), fmgr.UniqueID(), localCacheIntegrityPurpose, localCacheIntegrityHMACSecretLength)
+		cacheOpts.HMACSecret = crypto.DeriveKeyFromMasterKey(fmgr.FormatEncryptionKey(), fmgr.UniqueID(), localCacheIntegrityPurpose, localCacheIntegrityHMACSecretLength)
 	}
 
 	limits := throttlingLimitsFromConnectionInfo(ctx, st.ConnectionInfo())

repo/repository.go

@@ -10,6 +10,7 @@ import (
 	"go.opentelemetry.io/otel"
 
 	"github.com/kopia/kopia/internal/clock"
+	"github.com/kopia/kopia/internal/crypto"
 	"github.com/kopia/kopia/internal/metrics"
 	"github.com/kopia/kopia/repo/blob"
 	"github.com/kopia/kopia/repo/blob/throttling"
@@ -139,13 +140,13 @@ type directRepository struct {
 // DeriveKey derives encryption key of the provided length from the master key.
 func (r *directRepository) DeriveKey(purpose []byte, keyLength int) []byte {
 	if r.cmgr.ContentFormat().SupportsPasswordChange() {
-		return format.DeriveKeyFromMasterKey(r.cmgr.ContentFormat().GetMasterKey(), r.UniqueID(), purpose, keyLength)
+		return crypto.DeriveKeyFromMasterKey(r.cmgr.ContentFormat().GetMasterKey(), r.UniqueID(), purpose, keyLength)
 	}
 
 	// version of kopia <v0.9 had a bug where certain keys were derived directly from
 	// the password and not from the random master key. This made it impossible to change
 	// password.
-	return format.DeriveKeyFromMasterKey(r.fmgr.FormatEncryptionKey(), r.UniqueID(), purpose, keyLength)
+	return crypto.DeriveKeyFromMasterKey(r.fmgr.FormatEncryptionKey(), r.UniqueID(), purpose, keyLength)
 }
 
 // ClientOptions returns client options.

repo/repository_test.go

@@ -15,6 +15,7 @@ import (
 	"github.com/stretchr/testify/require"
 
 	"github.com/kopia/kopia/internal/cache"
+	"github.com/kopia/kopia/internal/crypto"
 	"github.com/kopia/kopia/internal/epoch"
 	"github.com/kopia/kopia/internal/gather"
 	"github.com/kopia/kopia/internal/metricid"
@@ -866,8 +867,8 @@ func TestDeriveKey(t *testing.T) {
 	formatEncryptionKeyFromPassword, err := j.DeriveFormatEncryptionKeyFromPassword(repotesting.DefaultPasswordForTesting)
 	require.NoError(t, err)
 
-	validV1KeyDerivedFromPassword := format.DeriveKeyFromMasterKey(formatEncryptionKeyFromPassword, uniqueID, testPurpose, testKeyLength)
-	validV2KeyDerivedFromMasterKey := format.DeriveKeyFromMasterKey(masterKey, uniqueID, testPurpose, testKeyLength)
+	validV1KeyDerivedFromPassword := crypto.DeriveKeyFromMasterKey(formatEncryptionKeyFromPassword, uniqueID, testPurpose, testKeyLength)
+	validV2KeyDerivedFromMasterKey := crypto.DeriveKeyFromMasterKey(masterKey, uniqueID, testPurpose, testKeyLength)
 
 	setup := func(v format.Version) repo.DirectRepositoryWriter {
 		_, env := repotesting.NewEnvironment(t, v, repotesting.Options{