mirror/kopia
1
0
Fork 0
mirror of https://github.com/kopia/kopia.git synced 2026-03-06 07:17:56 -05:00
Code Issues Packages Projects Releases Wiki Activity

encryption: removed old, non-authenticated encryption methods (#979)

Browse Source
This commit is contained in:
Jarek Kowalski
2021-04-12 21:24:19 -07:00
committed by GitHub
parent 85aa5e67f2
commit 2c3dfb7846
13 changed files with 56 additions and 363 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
internal/cache/storage_protection.go vendored
View File

@@ -102,9 +102,5 @@ func AuthenticatedEncryptionProtection(key []byte) (StorageProtection, error) {
return nil, errors.Wrap(err, "unable to create encryptor")
}
if !e.IsAuthenticated() {
return nil, errors.Wrap(err, "encryption is not authenticated!")
}
return authenticatedEncryptionProtection{e}, nil
}

32
repo/content/committed_read_manager.go
View File

@@ -1,9 +1,7 @@
package content
import (
"bytes"
"context"
"crypto/aes"
"os"
"sync"
"sync/atomic"
@@ -230,19 +228,15 @@ func (sm *SharedManager) decryptContentAndVerify(payload []byte, bi *Info) ([]by
func (sm *SharedManager) decryptAndVerify(encrypted, iv []byte) ([]byte, error) {
decrypted, err := sm.encryptor.Decrypt(nil, encrypted, iv)
if err != nil {
sm.Stats.foundInvalidContent()
return nil, errors.Wrap(err, "decrypt")
}
sm.Stats.foundValidContent()
sm.Stats.decrypted(len(decrypted))
if sm.encryptor.IsAuthenticated() {
// already verified
return decrypted, nil
}
// Since the encryption key is a function of data, we must be able to generate exactly the same key
// after decrypting the content. This serves as a checksum.
return decrypted, sm.verifyChecksum(decrypted, iv)
// already verified
return decrypted, nil
}
// IndexBlobs returns the list of active index blobs.
@@ -250,22 +244,6 @@ func (sm *SharedManager) IndexBlobs(ctx context.Context, includeInactive bool) (
return sm.indexBlobManager.listIndexBlobs(ctx, includeInactive)
}
func (sm *SharedManager) verifyChecksum(data, contentID []byte) error {
var hashOutput [maxHashSize]byte
expected := sm.hasher(hashOutput[:0], data)
expected = expected[len(expected)-aes.BlockSize:]
if !bytes.HasSuffix(contentID, expected) {
sm.Stats.foundInvalidContent()
return errors.Errorf("invalid checksum for blob %x, expected %x", contentID, expected)
}
sm.Stats.foundValidContent()
return nil
}
func (sm *SharedManager) setupReadManagerCaches(ctx context.Context, caching *CachingOptions) error {
dataCacheStorage, err := cache.NewStorageOrNil(ctx, caching.CacheDirectory, caching.MaxCacheSizeBytes, "contents")
if err != nil {
@@ -389,7 +367,7 @@ func NewSharedManager(ctx context.Context, st blob.Storage, f *FormattingOptions
repositoryFormatBytes: opts.RepositoryFormatBytes,
checkInvariantsOnUnlock: os.Getenv("KOPIA_VERIFY_INVARIANTS") != "",
writeFormatVersion: int32(f.Version),
encryptionBufferPool: buf.NewPool(ctx, defaultEncryptionBufferPoolSegmentSize+encryptor.MaxOverhead(), "content-manager-encryption"),
encryptionBufferPool: buf.NewPool(ctx, defaultEncryptionBufferPoolSegmentSize+encryptor.Overhead(), "content-manager-encryption"),
}
caching = caching.CloneOrDefault()

2
repo/content/content_manager_lock_free.go
View File

@@ -25,7 +25,7 @@ func (sm *SharedManager) maybeEncryptContentDataForPacking(output *gather.WriteB
return errors.Wrapf(err, "unable to get packed content IV for %q", contentID)
}
b := sm.encryptionBufferPool.Allocate(len(data) + sm.encryptor.MaxOverhead())
b := sm.encryptionBufferPool.Allocate(len(data) + sm.encryptor.Overhead())
defer b.Release()
cipherText, err := sm.encryptor.Encrypt(b.Data[:0], data, iv)

2
repo/content/index_blob_manager_test.go
View File

@@ -748,7 +748,7 @@ func newIndexBlobManagerForTesting(t *testing.T, st blob.Storage, localTimeNow f
t.Helper()
p := &FormattingOptions{
Encryption: encryption.DeprecatedNoneAlgorithm,
Encryption: encryption.DefaultAlgorithm,
Hash: hashing.DefaultAlgorithm,
}

10
repo/encryption/aes256_gcm_hmac_sha256_encryptor.go
View File

@@ -56,15 +56,7 @@ func (e aes256GCMHmacSha256) Encrypt(output, input, contentID []byte) ([]byte, e
return aeadSealWithRandomNonce(output, a, input, contentID)
}
func (e aes256GCMHmacSha256) IsAuthenticated() bool {
return true
}
func (e aes256GCMHmacSha256) IsDeprecated() bool {
return false
}
func (e aes256GCMHmacSha256) MaxOverhead() int {
func (e aes256GCMHmacSha256) Overhead() int {
return aes256GCMHmacSha256Overhead
}

10
repo/encryption/chacha20_poly1305_hmac_sha256_encryptor.go
View File

@@ -52,15 +52,7 @@ func (e chacha20poly1305hmacSha256Encryptor) Encrypt(output, input, contentID []
return aeadSealWithRandomNonce(output, a, input, contentID)
}
func (e chacha20poly1305hmacSha256Encryptor) IsAuthenticated() bool {
return true
}
func (e chacha20poly1305hmacSha256Encryptor) IsDeprecated() bool {
return false
}
func (e chacha20poly1305hmacSha256Encryptor) MaxOverhead() int {
func (e chacha20poly1305hmacSha256Encryptor) Overhead() int {
return chacha20poly1305hmacSha256EncryptorOverhead
}

91
repo/encryption/deprecated_ctr_encryptor.go
View File

@@ -1,91 +0,0 @@
package encryption
import (
"crypto/aes"
"crypto/cipher"
"github.com/pkg/errors"
)
const (
aes128KeyLength = 16
aes192KeyLength = 24
aes256KeyLength = 32
)
// ctrEncryptor implements encrypted format which uses CTR mode of a content cipher with nonce==IV.
type ctrEncryptor struct {
createCipher func() (cipher.Block, error)
}
func (fi ctrEncryptor) Encrypt(output, plainText, contentID []byte) ([]byte, error) {
return symmetricEncrypt(output, fi.createCipher, contentID, plainText)
}
func (fi ctrEncryptor) Decrypt(output, cipherText, contentID []byte) ([]byte, error) {
return symmetricEncrypt(output, fi.createCipher, contentID, cipherText)
}
func (fi ctrEncryptor) IsAuthenticated() bool {
return false
}
func (fi ctrEncryptor) IsDeprecated() bool {
return true
}
func (fi ctrEncryptor) MaxOverhead() int {
return 0
}
func symmetricEncrypt(output []byte, createCipher func() (cipher.Block, error), iv, b []byte) ([]byte, error) {
blockCipher, err := createCipher()
if err != nil {
return nil, err
}
if len(iv) < blockCipher.BlockSize() {
return nil, errors.Errorf("IV too short: %v expected >= %v", len(iv), blockCipher.BlockSize())
}
ctr := cipher.NewCTR(blockCipher, iv[0:blockCipher.BlockSize()])
result, out := sliceForAppend(output, len(b))
ctr.XORKeyStream(out, b)
return result, nil
}
func adjustKey(masterKey []byte, desiredKeySize int) ([]byte, error) {
if len(masterKey) == desiredKeySize {
return masterKey, nil
}
if desiredKeySize < len(masterKey) {
return masterKey[0:desiredKeySize], nil
}
return nil, errors.Errorf("required key too long %v, but only have %v", desiredKeySize, len(masterKey))
}
// newCTREncryptorFactory returns new EncryptorFactory that uses CTR with symmetric encryption (such as AES) and a given key size.
func newCTREncryptorFactory(keySize int, createCipherWithKey func(key []byte) (cipher.Block, error)) EncryptorFactory {
return func(o Parameters) (Encryptor, error) {
key, err := adjustKey(o.GetMasterKey(), keySize)
if err != nil {
return nil, errors.Wrap(err, "unable to get encryption key")
}
return ctrEncryptor{
createCipher: func() (cipher.Block, error) {
return createCipherWithKey(key)
},
}, nil
}
}
func init() {
Register("AES-128-CTR", "DEPRECATED: AES-128 in CTR mode", true, newCTREncryptorFactory(aes128KeyLength, aes.NewCipher))
Register("AES-192-CTR", "DEPRECATED: AES-192 in CTR mode", true, newCTREncryptorFactory(aes192KeyLength, aes.NewCipher))
Register("AES-256-CTR", "DEPRECATED: AES-256 in CTR mode", true, newCTREncryptorFactory(aes256KeyLength, aes.NewCipher))
}

100
repo/encryption/deprecated_salsa_encryptor.go
View File

@@ -1,100 +0,0 @@
package encryption
import (
"crypto/sha256"
"github.com/pkg/errors"
"golang.org/x/crypto/salsa20"
"github.com/kopia/kopia/internal/hmac"
)
const (
purposeEncryptionKey = "encryption"
purposeHMACSecret = "hmac"
hmacLength = 32
salsaKeyLength = 32
)
type salsaEncryptor struct {
nonceSize int
key *[32]byte
hmacSecret []byte
}
func (s salsaEncryptor) Decrypt(output, input, contentID []byte) ([]byte, error) {
if s.hmacSecret != nil {
var err error
input, err = hmac.VerifyAndStrip(input, s.hmacSecret)
if err != nil {
return nil, errors.Wrap(err, "hmac.VerifyAndStrip")
}
}
return s.encryptDecrypt(output, input, contentID)
}
func (s salsaEncryptor) Encrypt(output, input, contentID []byte) ([]byte, error) {
v, err := s.encryptDecrypt(output, input, contentID)
if err != nil {
return nil, errors.Wrap(err, "decrypt")
}
if s.hmacSecret == nil {
return v, nil
}
return hmac.Append(v, s.hmacSecret), nil
}
func (s salsaEncryptor) IsAuthenticated() bool {
return s.hmacSecret != nil
}
func (s salsaEncryptor) MaxOverhead() int {
if s.hmacSecret == nil {
return 0
}
return sha256.Size
}
func (s salsaEncryptor) encryptDecrypt(output, input, contentID []byte) ([]byte, error) {
if len(contentID) < s.nonceSize {
return nil, errors.Errorf("hash too short, expected >=%v bytes, got %v", s.nonceSize, len(contentID))
}
result, out := sliceForAppend(output, len(input))
nonce := contentID[0:s.nonceSize]
salsa20.XORKeyStream(out, input, nonce, s.key)
return result, nil
}
func (s salsaEncryptor) IsDeprecated() bool {
return true
}
func init() {
Register("SALSA20", "DEPRECATED: SALSA20 using shared key and 64-bit nonce", true, func(p Parameters) (Encryptor, error) {
var k [salsaKeyLength]byte
copy(k[:], p.GetMasterKey()[0:salsaKeyLength])
return salsaEncryptor{8, &k, nil}, nil
})
Register("SALSA20-HMAC", "DEPRECATED: SALSA20 with HMAC-SHA256 using shared key and 64-bit nonce", true, func(p Parameters) (Encryptor, error) {
encryptionKey, err := deriveKey(p, []byte(purposeEncryptionKey), salsaKeyLength)
if err != nil {
return nil, err
}
hmacSecret, err := deriveKey(p, []byte(purposeHMACSecret), hmacLength)
if err != nil {
return nil, err
}
var k [salsaKeyLength]byte
copy(k[:], encryptionKey)
return salsaEncryptor{8, &k, hmacSecret}, nil
})
}

41
repo/encryption/encryption.go
View File

@@ -10,7 +10,11 @@
"golang.org/x/crypto/hkdf"
)
const minDerivedKeyLength = 32
const (
minDerivedKeyLength = 32
purposeEncryptionKey = "encryption"
)
// Encryptor performs encryption and decryption of contents of data.
type Encryptor interface {
@@ -23,15 +27,8 @@ type Encryptor interface {
// authenticity check before decrypting.
Decrypt(output, cipherText, contentID []byte) ([]byte, error)
// IsAuthenticated returns true if encryption is authenticated.
// In this case Decrypt() is expected to perform authenticity check.
IsAuthenticated() bool
// IsDeprecated returns true if encryption is not recommended for new repositories.
IsDeprecated() bool
// MaxOverhead is the maximum number of bytes of overhead added by Encrypt()
MaxOverhead() int
// Overhead is the number of bytes of overhead added by Encrypt()
Overhead() int
}
// Parameters encapsulates all encryption parameters.
@@ -56,9 +53,6 @@ func CreateEncryptor(p Parameters) (Encryptor, error) {
// DefaultAlgorithm is the name of the default encryption algorithm.
const DefaultAlgorithm = "AES256-GCM-HMAC-SHA256"
// DeprecatedNoneAlgorithm is the name of the algorithm that does not encrypt.
const DeprecatedNoneAlgorithm = "NONE"
// SupportedAlgorithms returns the names of the supported encryption
// methods.
func SupportedAlgorithms(includeDeprecated bool) []string {
@@ -95,7 +89,6 @@ type encryptorInfo struct {
var encryptors = map[string]*encryptorInfo{}
// deriveKey uses HKDF to derive a key of a given length and a given purpose from parameters.
// nolint:unparam
func deriveKey(p Parameters, purpose []byte, length int) ([]byte, error) {
if length < minDerivedKeyLength {
return nil, errors.Errorf("derived key must be at least 32 bytes, was %v", length)
@@ -107,23 +100,3 @@ func deriveKey(p Parameters, purpose []byte, length int) ([]byte, error) {
return key, nil
}
// sliceForAppend takes a slice and a requested number of bytes. It returns a
// slice with the contents of the given slice followed by that many bytes and a
// second slice that aliases into it and contains only the extra bytes. If the
// original slice has sufficient capacity then no allocation is performed.
//
// From: https://golang.org/src/crypto/cipher/gcm.go
// Copyright 2013 The Go Authors. All rights reserved.
func sliceForAppend(in []byte, n int) (head, tail []byte) {
if total := len(in) + n; cap(in) >= total {
head = in[:total]
} else {
head = make([]byte, total)
copy(head, in)
}
tail = head[len(in):]
return
}

68
repo/encryption/encryption_test.go
View File

@@ -45,15 +45,13 @@ func TestRoundTrip(t *testing.T) {
t.Errorf("invalid response from Encrypt: %v %v", cipherText1, err)
}
if !e.IsDeprecated() && encryptionAlgo != encryption.DeprecatedNoneAlgorithm {
cipherText1b, err2 := e.Encrypt(nil, data, contentID1)
if err2 != nil || cipherText1b == nil {
t.Errorf("invalid response from Encrypt: %v %v", cipherText1, err2)
}
cipherText1b, err2 := e.Encrypt(nil, data, contentID1)
if err2 != nil || cipherText1b == nil {
t.Errorf("invalid response from Encrypt: %v %v", cipherText1, err2)
}
if bytes.Equal(cipherText1, cipherText1b) {
t.Errorf("multiple Encrypt returned the same ciphertext: %x", cipherText1)
}
if bytes.Equal(cipherText1, cipherText1b) {
t.Errorf("multiple Encrypt returned the same ciphertext: %x", cipherText1)
}
plainText1, err := e.Decrypt(nil, cipherText1, contentID1)
@@ -90,33 +88,19 @@ func TestRoundTrip(t *testing.T) {
t.Errorf("Encrypt()/Decrypt() does not round-trip: %x %x", plainText2, data)
}
if encryptionAlgo != encryption.DeprecatedNoneAlgorithm {
if bytes.Equal(cipherText1, cipherText2) {
t.Errorf("ciphertexts should be different, were %x", cipherText1)
}
if bytes.Equal(cipherText1, cipherText2) {
t.Errorf("ciphertexts should be different, were %x", cipherText1)
}
// decrypt using wrong content ID
badPlainText2, err := e.Decrypt(nil, cipherText2, contentID1)
if e.IsAuthenticated() {
if err == nil && encryptionAlgo != "SALSA20-HMAC" {
// "SALSA20-HMAC" is deprecated & wrong, and only validates that checksum is
// valid for some content, but does not validate that we decrypted the
// intended content.
t.Errorf("expected decrypt to fail for authenticated encryption")
}
} else {
if bytes.Equal(badPlainText2, plainText2) {
t.Errorf("decrypted plaintext matches, but it should not: %x", plainText2)
}
}
// decrypt using wrong content ID
if _, err := e.Decrypt(nil, cipherText2, contentID1); err == nil {
t.Fatalf("expected decrypt to fail for authenticated encryption")
}
// flip some bits in the cipherText
if e.IsAuthenticated() {
cipherText2[mathrand.Intn(len(cipherText2))] ^= byte(1 + mathrand.Intn(254))
if _, err := e.Decrypt(nil, cipherText2, contentID1); err == nil {
t.Errorf("expected decrypt failure on invalid ciphertext, got success")
}
}
// flip some bits in the cipherText
cipherText2[mathrand.Intn(len(cipherText2))] ^= byte(1 + mathrand.Intn(254))
if _, err := e.Decrypt(nil, cipherText2, contentID1); err == nil {
t.Errorf("expected decrypt failure on invalid ciphertext, got success")
}
})
}
@@ -136,17 +120,8 @@ func TestCiphertextSamples(t *testing.T) {
// samples of base16-encoded ciphertexts of payload encrypted with masterKey & contentID
samples: map[string]string{
"NONE": hex.EncodeToString([]byte("foo")),
"AES256-GCM-HMAC-SHA256": "e43ba07f85a6d70c5f1102ca06cf19c597e5f91e527b21f00fb76e8bec3fd1",
"CHACHA20-POLY1305-HMAC-SHA256": "118359f3d4d589d939efbbc3168ae4c77c51bcebce6845fe6ef5d11342faa6",
// deprecated
"AES-128-CTR": "54cd8d",
"AES-192-CTR": "2d084b",
"AES-256-CTR": "8a580a",
"SALSA20": "bf5ec3",
"SALSA20-HMAC": "8bf37fd9ec69843c3c2ac2a2cfdd59f36077206a15289efde640d0e677d03e6ac8f8ec",
},
},
{
@@ -156,17 +131,8 @@ func TestCiphertextSamples(t *testing.T) {
// samples of base16-encoded ciphertexts of payload encrypted with masterKey & contentID
samples: map[string]string{
"NONE": hex.EncodeToString([]byte("quick brown fox jumps over the lazy dog")),
"AES256-GCM-HMAC-SHA256": "eaad755a238f1daa4052db2e5ccddd934790b6cca415b3ccfd46ac5746af33d9d30f4400ffa9eb3a64fb1ce21b888c12c043bf6787d4a5c15ad10f21f6a6027ee3afe0",
"CHACHA20-POLY1305-HMAC-SHA256": "836d2ba87892711077adbdbe1452d3b2c590bbfdf6fd3387dc6810220a32ec19de862e1a4f865575e328424b5f178afac1b7eeff11494f719d119b7ebb924d1d0846a3",
// deprecated
"AES-128-CTR": "974c5c1782076e3de7255deabe8706a509b5772a8b7a8e7f83d01de7098c945934417071ec5351",
"AES-192-CTR": "1200e755ec14125e87136b5281957895eeb429be673b2241da261f949283aea59fd2fa64387764",
"AES-256-CTR": "39f13367828efb5fb22b97865ca0dbaad352d0c1a3083ff056bc771b812239445ed8af022f3760",
"SALSA20": "65ce12b14739aecbf9e6a9b9b9c4a72ffa8886fe0b071c0abdfb3d3e5c336b90f9af411ba69faf",
"SALSA20-HMAC": "a1dc47f250def4d97a422d505fb5e9a9a13699762cb32cfe7705982fa68ce71f54544ab932a1045fb0601087159954d563f0de0aaa15690d93ea63748bf91889e577daeeed5cf8",
},
},
}

30
repo/encryption/null_encryptor.go
View File

@@ -1,30 +0,0 @@
package encryption
// nullEncryptor implements non-encrypted format.
type nullEncryptor struct{}
func (fi nullEncryptor) Encrypt(output, plainText, contentID []byte) ([]byte, error) {
return append(output, plainText...), nil
}
func (fi nullEncryptor) Decrypt(output, cipherText, contentID []byte) ([]byte, error) {
return append(output, cipherText...), nil
}
func (fi nullEncryptor) IsAuthenticated() bool {
return false
}
func (fi nullEncryptor) IsDeprecated() bool {
return false
}
func (fi nullEncryptor) MaxOverhead() int {
return 0
}
func init() {
Register(DeprecatedNoneAlgorithm, "No encryption", true, func(p Parameters) (Encryptor, error) {
return nullEncryptor{}, nil
})
}

26
repo/maintenance/blob_gc_test.go
View File

@@ -11,6 +11,7 @@
"github.com/google/go-cmp/cmp"
"github.com/pkg/errors"
"github.com/stretchr/testify/require"
"github.com/kopia/kopia/internal/clock"
"github.com/kopia/kopia/internal/faketime"
@@ -20,11 +21,14 @@
"github.com/kopia/kopia/repo"
"github.com/kopia/kopia/repo/blob"
"github.com/kopia/kopia/repo/content"
"github.com/kopia/kopia/repo/encryption"
"github.com/kopia/kopia/repo/object"
)
var testHMACSecret = []byte{1, 2, 3}
var testMasterKey = []byte{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15}
func TestDeleteUnreferencedBlobs(t *testing.T) {
// set up fake clock which is initially synchronized to wall clock time
// and moved at the same speed but which can be moved forward.
@@ -35,7 +39,8 @@ func TestDeleteUnreferencedBlobs(t *testing.T) {
o.TimeNowFunc = ta.NowFunc()
},
NewRepositoryOptions: func(nro *repo.NewRepositoryOptions) {
nro.BlockFormat.Encryption = "NONE"
nro.BlockFormat.Encryption = encryption.DefaultAlgorithm
nro.BlockFormat.MasterKey = testMasterKey
nro.BlockFormat.Hash = "HMAC-SHA256"
nro.BlockFormat.HMACSecret = testHMACSecret
},
@@ -190,11 +195,22 @@ func mustPutDummySessionBlob(t *testing.T, st blob.Storage, sessionIDSuffix blob
h := hmac.New(sha256.New, testHMACSecret)
h.Write(j)
blobID := blob.ID(fmt.Sprintf("s%x-%v", h.Sum(nil)[16:32], sessionIDSuffix))
iv := h.Sum(nil)[16:32]
if err := st.PutBlob(testlogging.Context(t), blobID, gather.FromSlice(j)); err != nil {
t.Fatal(err)
}
blobID := blob.ID(fmt.Sprintf("s%x-%v", iv, sessionIDSuffix))
e, err := encryption.CreateEncryptor(&content.FormattingOptions{
Encryption: encryption.DefaultAlgorithm,
MasterKey: testMasterKey,
HMACSecret: testHMACSecret,
})
require.NoError(t, err)
enc, err := e.Encrypt(nil, j, iv)
require.NoError(t, err)
require.NoError(t, st.PutBlob(testlogging.Context(t), blobID, gather.FromSlice(enc)))
return blobID
}

3
tests/stress_test/stress_test.go
View File

@@ -14,6 +14,7 @@
"github.com/kopia/kopia/internal/testlogging"
"github.com/kopia/kopia/repo/blob"
"github.com/kopia/kopia/repo/content"
"github.com/kopia/kopia/repo/encryption"
)
const goroutineCount = 16
@@ -43,7 +44,7 @@ func stressTestWithStorage(t *testing.T, st blob.Storage, duration time.Duration
return content.NewManager(ctx, st, &content.FormattingOptions{
Version: 1,
Hash: "HMAC-SHA256-128",
Encryption: "AES-256-CTR",
Encryption: encryption.DefaultAlgorithm,
MaxPackSize: 20000000,
MasterKey: []byte{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15},
}, nil, nil)