From e653ec0a020ce7e15420e760b88705d694c7790d Mon Sep 17 00:00:00 2001 From: PhracturedBlue Date: Sun, 24 Sep 2023 17:44:50 -0700 Subject: [PATCH] refactor(repository): Move encryption into a common package in preparation to handle secrets (#3351) --- internal/crypto/aesgcm.go | 80 +++++++++++++++++++ .../crypto/key_derivation.go | 6 +- internal/crypto/key_derivation_nontest.go | 26 ++++++ .../crypto/key_derivation_testing.go | 10 +-- repo/format/crypto_key_derivation_nontest.go | 26 ------ repo/format/format_blob.go | 76 +++++------------- repo/open.go | 5 +- repo/repository.go | 5 +- repo/repository_test.go | 5 +- 9 files changed, 143 insertions(+), 96 deletions(-) create mode 100644 internal/crypto/aesgcm.go rename repo/format/crypto_key_derivation.go => internal/crypto/key_derivation.go (68%) create mode 100644 internal/crypto/key_derivation_nontest.go rename repo/format/crypto_key_derivation_testing.go => internal/crypto/key_derivation_testing.go (55%) delete mode 100644 repo/format/crypto_key_derivation_nontest.go diff --git a/internal/crypto/aesgcm.go b/internal/crypto/aesgcm.go new file mode 100644 index 000000000..2ca291437 --- /dev/null +++ b/internal/crypto/aesgcm.go @@ -0,0 +1,80 @@ +// Package crypto implements common symmetric-encryption and key-derivation functions. +package crypto + +import ( + "crypto/aes" + "crypto/cipher" + "crypto/rand" + "io" + + "github.com/pkg/errors" +) + +//nolint:gochecknoglobals +var ( + purposeAESKey = []byte("AES") + purposeAuthData = []byte("CHECKSUM") +) + +func initCrypto(masterKey, salt []byte) (cipher.AEAD, []byte, error) { + aesKey := DeriveKeyFromMasterKey(masterKey, salt, purposeAESKey, 32) //nolint:gomnd + authData := DeriveKeyFromMasterKey(masterKey, salt, purposeAuthData, 32) //nolint:gomnd + + blk, err := aes.NewCipher(aesKey) + if err != nil { + return nil, nil, errors.Wrap(err, "cannot create cipher") + } + + aead, err := cipher.NewGCM(blk) + if err != nil { + return nil, nil, errors.Wrap(err, "cannot create cipher") + } + + return aead, authData, nil +} + +// EncryptAes256Gcm encrypts data with AES 256 GCM. +func EncryptAes256Gcm(data, masterKey, salt []byte) ([]byte, error) { + aead, authData, err := initCrypto(masterKey, salt) + if err != nil { + return nil, errors.Wrap(err, "unable to initialize crypto") + } + + nonceLength := aead.NonceSize() + noncePlusContentLength := nonceLength + len(data) + cipherText := make([]byte, noncePlusContentLength+aead.Overhead()) + + // Store nonce at the beginning of ciphertext. + nonce := cipherText[0:nonceLength] + if _, err := io.ReadFull(rand.Reader, nonce); err != nil { + return nil, errors.Wrap(err, "error reading random bytes for nonce") + } + + b := aead.Seal(cipherText[nonceLength:nonceLength], nonce, data, authData) + data = nonce[0 : nonceLength+len(b)] + + return data, nil +} + +// DecryptAes256Gcm encrypts data with AES 256 GCM. +func DecryptAes256Gcm(data, masterKey, salt []byte) ([]byte, error) { + aead, authData, err := initCrypto(masterKey, salt) + if err != nil { + return nil, errors.Wrap(err, "cannot initialize cipher") + } + + data = append([]byte(nil), data...) + if len(data) < aead.NonceSize() { + return nil, errors.Errorf("invalid encrypted payload, too short") + } + + nonce := data[0:aead.NonceSize()] + payload := data[aead.NonceSize():] + + plainText, err := aead.Open(payload[:0], nonce, payload, authData) + if err != nil { + return nil, errors.Errorf("unable to decrypt repository blob, invalid credentials?") + } + + return plainText, nil +} diff --git a/repo/format/crypto_key_derivation.go b/internal/crypto/key_derivation.go similarity index 68% rename from repo/format/crypto_key_derivation.go rename to internal/crypto/key_derivation.go index 115adb24a..4dc02a706 100644 --- a/repo/format/crypto_key_derivation.go +++ b/internal/crypto/key_derivation.go @@ -1,4 +1,4 @@ -package format +package crypto import ( "crypto/sha256" @@ -8,9 +8,9 @@ import ( ) // DeriveKeyFromMasterKey computes a key for a specific purpose and length using HKDF based on the master key. -func DeriveKeyFromMasterKey(masterKey, uniqueID, purpose []byte, length int) []byte { +func DeriveKeyFromMasterKey(masterKey, salt, purpose []byte, length int) []byte { key := make([]byte, length) - k := hkdf.New(sha256.New, masterKey, uniqueID, purpose) + k := hkdf.New(sha256.New, masterKey, salt, purpose) if _, err := io.ReadFull(k, key); err != nil { panic("unable to derive key from master key, this should never happen") diff --git a/internal/crypto/key_derivation_nontest.go b/internal/crypto/key_derivation_nontest.go new file mode 100644 index 000000000..7ce781e19 --- /dev/null +++ b/internal/crypto/key_derivation_nontest.go @@ -0,0 +1,26 @@ +//go:build !testing +// +build !testing + +package crypto + +import ( + "github.com/pkg/errors" + "golang.org/x/crypto/scrypt" +) + +// DefaultKeyDerivationAlgorithm is the key derivation algorithm for new configurations. +const DefaultKeyDerivationAlgorithm = "scrypt-65536-8-1" + +// DeriveKeyFromPassword derives encryption key using the provided password and per-repository unique ID. +func DeriveKeyFromPassword(password string, salt []byte, algorithm string) ([]byte, error) { + const masterKeySize = 32 + + switch algorithm { + case "scrypt-65536-8-1": + //nolint:wrapcheck,gomnd + return scrypt.Key([]byte(password), salt, 65536, 8, 1, masterKeySize) + + default: + return nil, errors.Errorf("unsupported key algorithm: %v", algorithm) + } +} diff --git a/repo/format/crypto_key_derivation_testing.go b/internal/crypto/key_derivation_testing.go similarity index 55% rename from repo/format/crypto_key_derivation_testing.go rename to internal/crypto/key_derivation_testing.go index 47a8b100a..04ea1d930 100644 --- a/repo/format/crypto_key_derivation_testing.go +++ b/internal/crypto/key_derivation_testing.go @@ -1,7 +1,7 @@ //go:build testing // +build testing -package format +package crypto import ( "crypto/sha256" @@ -12,11 +12,11 @@ import ( // DefaultKeyDerivationAlgorithm is the key derivation algorithm for new configurations. const DefaultKeyDerivationAlgorithm = "testing-only-insecure" -// DeriveFormatEncryptionKeyFromPassword derives encryption key using the provided password and per-repository unique ID. -func (f *KopiaRepositoryJSON) DeriveFormatEncryptionKeyFromPassword(password string) ([]byte, error) { +// DeriveKeyFromPassword derives encryption key using the provided password and per-repository unique ID. +func DeriveKeyFromPassword(password string, salt []byte, algorithm string) ([]byte, error) { const masterKeySize = 32 - switch f.KeyDerivationAlgorithm { + switch algorithm { case DefaultKeyDerivationAlgorithm: h := sha256.New() if _, err := h.Write([]byte(password)); err != nil { @@ -26,6 +26,6 @@ func (f *KopiaRepositoryJSON) DeriveFormatEncryptionKeyFromPassword(password str return h.Sum(nil), nil default: - return nil, errors.Errorf("unsupported key algorithm: %v", f.KeyDerivationAlgorithm) + return nil, errors.Errorf("unsupported key algorithm: %v", algorithm) } } diff --git a/repo/format/crypto_key_derivation_nontest.go b/repo/format/crypto_key_derivation_nontest.go deleted file mode 100644 index 4ddf6bdc2..000000000 --- a/repo/format/crypto_key_derivation_nontest.go +++ /dev/null @@ -1,26 +0,0 @@ -//go:build !testing -// +build !testing - -package format - -import ( - "github.com/pkg/errors" - "golang.org/x/crypto/scrypt" -) - -// DefaultKeyDerivationAlgorithm is the key derivation algorithm for new configurations. -const DefaultKeyDerivationAlgorithm = "scrypt-65536-8-1" - -// DeriveFormatEncryptionKeyFromPassword derives encryption key using the provided password and per-repository unique ID. -func (f *KopiaRepositoryJSON) DeriveFormatEncryptionKeyFromPassword(password string) ([]byte, error) { - const masterKeySize = 32 - - switch f.KeyDerivationAlgorithm { - case "scrypt-65536-8-1": - //nolint:wrapcheck,gomnd - return scrypt.Key([]byte(password), f.UniqueID, 65536, 8, 1, masterKeySize) - - default: - return nil, errors.Errorf("unsupported key algorithm: %v", f.KeyDerivationAlgorithm) - } -} diff --git a/repo/format/format_blob.go b/repo/format/format_blob.go index 9ce6ba43a..0dcb94351 100644 --- a/repo/format/format_blob.go +++ b/repo/format/format_blob.go @@ -3,16 +3,13 @@ package format import ( "context" - "crypto/aes" - "crypto/cipher" "crypto/hmac" - "crypto/rand" "crypto/sha256" "encoding/json" - "io" "github.com/pkg/errors" + "github.com/kopia/kopia/internal/crypto" "github.com/kopia/kopia/internal/gather" "github.com/kopia/kopia/repo/blob" ) @@ -20,6 +17,9 @@ import ( // DefaultFormatEncryption is the identifier of the default format blob encryption algorithm. const DefaultFormatEncryption = "AES256_GCM" +// DefaultKeyDerivationAlgorithm is the key derivation algorithm for new configurations. +const DefaultKeyDerivationAlgorithm = crypto.DefaultKeyDerivationAlgorithm + const ( aes256GcmEncryption = "AES256_GCM" lengthOfRecoverBlockLength = 2 // number of bytes used to store recover block length @@ -37,9 +37,6 @@ var ErrInvalidPassword = errors.Errorf("invalid repository password") // +checkl //nolint:gochecknoglobals var ( - purposeAESKey = []byte("AES") - purposeAuthData = []byte("CHECKSUM") - // formatBlobChecksumSecret is a HMAC secret used for checksumming the format content. // It's not really a secret, but will provide positive identification of blocks that // are repository format blocks. @@ -73,6 +70,16 @@ func ParseKopiaRepositoryJSON(b []byte) (*KopiaRepositoryJSON, error) { return f, nil } +// DeriveFormatEncryptionKeyFromPassword derives encryption key using the provided password and per-repository unique ID. +func (f *KopiaRepositoryJSON) DeriveFormatEncryptionKeyFromPassword(password string) ([]byte, error) { + res, err := crypto.DeriveKeyFromPassword(password, f.UniqueID, f.KeyDerivationAlgorithm) + if err != nil { + return nil, errors.Wrap(err, "Failed to derive format encryption key") + } + + return res, nil +} + // RecoverFormatBlob attempts to recover format blob replica from the specified file. // The format blob can be either the prefix or a suffix of the given file. // optionally the length can be provided (if known) to speed up recovery. @@ -190,65 +197,22 @@ func (f *KopiaRepositoryJSON) WriteKopiaRepositoryBlobWithID(ctx context.Context return nil } -func initCrypto(masterKey, repositoryID []byte) (cipher.AEAD, []byte, error) { - aesKey := DeriveKeyFromMasterKey(masterKey, repositoryID, purposeAESKey, 32) //nolint:gomnd - authData := DeriveKeyFromMasterKey(masterKey, repositoryID, purposeAuthData, 32) //nolint:gomnd - - blk, err := aes.NewCipher(aesKey) - if err != nil { - return nil, nil, errors.Wrap(err, "cannot create cipher") - } - - aead, err := cipher.NewGCM(blk) - if err != nil { - return nil, nil, errors.Wrap(err, "cannot create cipher") - } - - return aead, authData, nil -} - func encryptRepositoryBlobBytesAes256Gcm(data, masterKey, repositoryID []byte) ([]byte, error) { - aead, authData, err := initCrypto(masterKey, repositoryID) + res, err := crypto.EncryptAes256Gcm(data, masterKey, repositoryID) if err != nil { - return nil, errors.Wrap(err, "unable to initialize crypto") + return nil, errors.Wrap(err, "Failed to encrypt blob") } - nonceLength := aead.NonceSize() - noncePlusContentLength := nonceLength + len(data) - cipherText := make([]byte, noncePlusContentLength+aead.Overhead()) - - // Store nonce at the beginning of ciphertext. - nonce := cipherText[0:nonceLength] - if _, err := io.ReadFull(rand.Reader, nonce); err != nil { - return nil, errors.Wrap(err, "error reading random bytes for nonce") - } - - b := aead.Seal(cipherText[nonceLength:nonceLength], nonce, data, authData) - data = nonce[0 : nonceLength+len(b)] - - return data, nil + return res, nil } func decryptRepositoryBlobBytesAes256Gcm(data, masterKey, repositoryID []byte) ([]byte, error) { - aead, authData, err := initCrypto(masterKey, repositoryID) + res, err := crypto.DecryptAes256Gcm(data, masterKey, repositoryID) if err != nil { - return nil, errors.Wrap(err, "cannot initialize cipher") + return nil, errors.Wrap(err, "Failed to decrypt blob") } - data = append([]byte(nil), data...) - if len(data) < aead.NonceSize() { - return nil, errors.Errorf("invalid encrypted payload, too short") - } - - nonce := data[0:aead.NonceSize()] - payload := data[aead.NonceSize():] - - plainText, err := aead.Open(payload[:0], nonce, payload, authData) - if err != nil { - return nil, errors.Errorf("unable to decrypt repository blob, invalid credentials?") - } - - return plainText, nil + return res, nil } func addFormatBlobChecksumAndLength(fb []byte) ([]byte, error) { diff --git a/repo/open.go b/repo/open.go index 06b39ad1b..4ed528531 100644 --- a/repo/open.go +++ b/repo/open.go @@ -14,6 +14,7 @@ import ( "github.com/kopia/kopia/internal/cache" "github.com/kopia/kopia/internal/cacheprot" + "github.com/kopia/kopia/internal/crypto" "github.com/kopia/kopia/internal/feature" "github.com/kopia/kopia/internal/metrics" "github.com/kopia/kopia/internal/retry" @@ -257,10 +258,10 @@ func openWithConfig(ctx context.Context, st blob.Storage, cliOpts ClientOptions, } if fmgr.SupportsPasswordChange() { - cacheOpts.HMACSecret = format.DeriveKeyFromMasterKey(fmgr.GetHmacSecret(), fmgr.UniqueID(), localCacheIntegrityPurpose, localCacheIntegrityHMACSecretLength) + cacheOpts.HMACSecret = crypto.DeriveKeyFromMasterKey(fmgr.GetHmacSecret(), fmgr.UniqueID(), localCacheIntegrityPurpose, localCacheIntegrityHMACSecretLength) } else { // deriving from ufb.FormatEncryptionKey was actually a bug, that only matters will change when we change the password - cacheOpts.HMACSecret = format.DeriveKeyFromMasterKey(fmgr.FormatEncryptionKey(), fmgr.UniqueID(), localCacheIntegrityPurpose, localCacheIntegrityHMACSecretLength) + cacheOpts.HMACSecret = crypto.DeriveKeyFromMasterKey(fmgr.FormatEncryptionKey(), fmgr.UniqueID(), localCacheIntegrityPurpose, localCacheIntegrityHMACSecretLength) } limits := throttlingLimitsFromConnectionInfo(ctx, st.ConnectionInfo()) diff --git a/repo/repository.go b/repo/repository.go index 1110625bf..e9451eda1 100644 --- a/repo/repository.go +++ b/repo/repository.go @@ -10,6 +10,7 @@ import ( "go.opentelemetry.io/otel" "github.com/kopia/kopia/internal/clock" + "github.com/kopia/kopia/internal/crypto" "github.com/kopia/kopia/internal/metrics" "github.com/kopia/kopia/repo/blob" "github.com/kopia/kopia/repo/blob/throttling" @@ -139,13 +140,13 @@ type directRepository struct { // DeriveKey derives encryption key of the provided length from the master key. func (r *directRepository) DeriveKey(purpose []byte, keyLength int) []byte { if r.cmgr.ContentFormat().SupportsPasswordChange() { - return format.DeriveKeyFromMasterKey(r.cmgr.ContentFormat().GetMasterKey(), r.UniqueID(), purpose, keyLength) + return crypto.DeriveKeyFromMasterKey(r.cmgr.ContentFormat().GetMasterKey(), r.UniqueID(), purpose, keyLength) } // version of kopia