diff --git a/cli/command_benchmark_crypto.go b/cli/command_benchmark_crypto.go index ba724dedd..684b70641 100644 --- a/cli/command_benchmark_crypto.go +++ b/cli/command_benchmark_crypto.go @@ -54,7 +54,7 @@ type benchResult struct { for _, ha := range hashing.SupportedAlgorithms() { for _, ea := range encryption.SupportedAlgorithms(c.deprecatedAlgorithms) { - h, e, err := content.CreateHashAndEncryptor(&content.FormattingOptions{ + cr, err := content.CreateCrypter(&content.FormattingOptions{ Encryption: ea, Hash: ha, MasterKey: make([]byte, 32), @@ -71,8 +71,8 @@ type benchResult struct { hashCount := c.repeat for i := 0; i < hashCount; i++ { - contentID := h(hashOutput[:0], data) - if _, encerr := e.Encrypt(encryptOutput[:0], data, contentID); encerr != nil { + contentID := cr.HashFunction(hashOutput[:0], data) + if _, encerr := cr.Encryptor.Encrypt(encryptOutput[:0], data, contentID); encerr != nil { log(ctx).Errorf("encryption failed: %v", encerr) break } diff --git a/cli/command_blob_show.go b/cli/command_blob_show.go index 92d289ad2..b5e7c3066 100644 --- a/cli/command_blob_show.go +++ b/cli/command_blob_show.go @@ -45,8 +45,10 @@ func (c *commandBlobShow) maybeDecryptBlob(ctx context.Context, w io.Writer, rep err error ) + d, err = rep.BlobReader().GetBlob(ctx, blobID, 0, -1) + if c.blobShowDecrypt && canDecryptBlob(blobID) { - d, err = rep.IndexBlobReader().DecryptBlob(ctx, blobID) + d, err = rep.Crypter().DecryptBLOB(d, blobID) if isJSONBlob(blobID) && err == nil { var b bytes.Buffer @@ -57,8 +59,6 @@ func (c *commandBlobShow) maybeDecryptBlob(ctx context.Context, w io.Writer, rep d = b.Bytes() } - } else { - d, err = rep.BlobReader().GetBlob(ctx, blobID, 0, -1) } if err != nil { diff --git a/repo/content/blob_crypto.go b/repo/content/blob_crypto.go index e9725c691..97f5cb2e5 100644 --- a/repo/content/blob_crypto.go +++ b/repo/content/blob_crypto.go @@ -1,7 +1,6 @@ package content import ( - "bytes" "crypto/aes" "encoding/hex" "strings" @@ -13,7 +12,25 @@ "github.com/kopia/kopia/repo/hashing" ) -func getIndexBlobIV(s blob.ID) ([]byte, error) { +// Crypter ecapsulates hashing and encryption and provides utilities for whole-BLOB encryption. +// Whole-BLOB encryption relies on BLOB identifiers formatted as: +// +// [-optionalSuffix] +// +// Where: +// 'prefix' is arbitrary string without dashes +// 'hash' is base16-encoded 128-bit hash of contents, used as initialization vector (IV) +// for the encryption. In case of longer hash functions, we use last 16 bytes of +// their outputs. +// 'optionalSuffix' can be any string +type Crypter struct { + HashFunction hashing.HashFunc + Encryptor encryption.Encryptor +} + +// getIndexBlobIV gets the initialization vector from the provided blob ID by taking +// 32 characters immediately preceding the first dash ('-') and decoding them using base16. +func (c *Crypter) getIndexBlobIV(s blob.ID) ([]byte, error) { if p := strings.Index(string(s), "-"); p >= 0 { // nolint:gocritic s = s[0:p] } @@ -30,58 +47,43 @@ func getIndexBlobIV(s blob.ID) ([]byte, error) { return v, nil } -func encryptFullBlob(h hashing.HashFunc, enc encryption.Encryptor, data []byte, prefix blob.ID, sessionID SessionID) (blob.ID, []byte, error) { +// EncryptBLOB encrypts the given data using crypter-defined key and returns a name that should +// be used to save the blob in thre repository. +func (c *Crypter) EncryptBLOB(data []byte, prefix blob.ID, sessionID SessionID) (blob.ID, []byte, error) { var hashOutput [maxHashSize]byte - hash := h(hashOutput[:0], data) + hash := c.HashFunction(hashOutput[:0], data) blobID := prefix + blob.ID(hex.EncodeToString(hash)) if sessionID != "" { blobID += blob.ID("-" + sessionID) } - iv, err := getIndexBlobIV(blobID) + iv, err := c.getIndexBlobIV(blobID) if err != nil { return "", nil, err } - data2, err := enc.Encrypt(nil, data, iv) + data2, err := c.Encryptor.Encrypt(nil, data, iv) if err != nil { - return "", nil, errors.Wrapf(err, "error encrypting blob %v", blobID) + return "", nil, errors.Wrapf(err, "error encrypting BLOB %v", blobID) } return blobID, data2, nil } -func decryptFullBlob(h hashing.HashFunc, enc encryption.Encryptor, payload []byte, blobID blob.ID) ([]byte, error) { - iv, err := getIndexBlobIV(blobID) +// DecryptBLOB decrypts the provided data using provided blobID to derive initialization vector. +func (c *Crypter) DecryptBLOB(payload []byte, blobID blob.ID) ([]byte, error) { + iv, err := c.getIndexBlobIV(blobID) if err != nil { return nil, errors.Wrap(err, "unable to get index blob IV") } - payload, err = enc.Decrypt(nil, payload, iv) + // Decrypt will verify the payload. + payload, err = c.Encryptor.Decrypt(nil, payload, iv) if err != nil { - return nil, errors.Wrap(err, "decrypt error") - } - - // Since the encryption key is a function of data, we must be able to generate exactly the same key - // after decrypting the content. This serves as a checksum. - if err := verifyChecksum(h, payload, iv); err != nil { - return nil, err + return nil, errors.Wrapf(err, "error decrypting BLOB %v", blobID) } return payload, nil } - -func verifyChecksum(h hashing.HashFunc, data, iv []byte) error { - var hashOutput [maxHashSize]byte - - expected := h(hashOutput[:0], data) - expected = expected[len(expected)-aes.BlockSize:] - - if !bytes.HasSuffix(iv, expected) { - return errors.Errorf("invalid checksum for blob %x, expected %x", iv, expected) - } - - return nil -} diff --git a/repo/content/committed_read_manager.go b/repo/content/committed_read_manager.go index ccc808dd8..a2eef01bc 100644 --- a/repo/content/committed_read_manager.go +++ b/repo/content/committed_read_manager.go @@ -15,8 +15,6 @@ "github.com/kopia/kopia/internal/clock" "github.com/kopia/kopia/repo/blob" "github.com/kopia/kopia/repo/compression" - "github.com/kopia/kopia/repo/encryption" - "github.com/kopia/kopia/repo/hashing" ) // number of bytes to read from each pack index when recovering the index. @@ -34,8 +32,7 @@ type SharedManager struct { contentCache contentCache metadataCache contentCache committedContents *committedContentIndex - hasher hashing.HashFunc - encryptor encryption.Encryptor + crypter *Crypter timeNow func() time.Time format FormattingOptions @@ -51,6 +48,11 @@ type SharedManager struct { encryptionBufferPool *buf.Pool } +// Crypter returns the crypter. +func (sm *SharedManager) Crypter() *Crypter { + return sm.crypter +} + func (sm *SharedManager) readPackFileLocalIndex(ctx context.Context, packFile blob.ID, packFileLength int64) ([]byte, error) { if packFileLength >= indexRecoverPostambleSize { data, err := sm.attemptReadPackFileLocalIndex(ctx, packFile, packFileLength-indexRecoverPostambleSize, indexRecoverPostambleSize) @@ -276,7 +278,7 @@ func (sm *SharedManager) decryptContentAndVerify(payload []byte, bi Info) ([]byt } func (sm *SharedManager) decryptAndVerify(encrypted, iv []byte) ([]byte, error) { - decrypted, err := sm.encryptor.Decrypt(nil, encrypted, iv) + decrypted, err := sm.crypter.Encryptor.Decrypt(nil, encrypted, iv) if err != nil { sm.Stats.foundInvalidContent() return nil, errors.Wrap(err, "decrypt") @@ -332,7 +334,7 @@ func (sm *SharedManager) setupReadManagerCaches(ctx context.Context, caching *Ca return errors.Wrap(err, "unable to initialize own writes cache") } - contentIndex := newCommittedContentIndex(caching, uint32(sm.encryptor.Overhead()), sm.indexVersion) + contentIndex := newCommittedContentIndex(caching, uint32(sm.crypter.Encryptor.Overhead()), sm.indexVersion) // once everything is ready, set it up sm.contentCache = dataCache @@ -341,8 +343,7 @@ func (sm *SharedManager) setupReadManagerCaches(ctx context.Context, caching *Ca sm.indexBlobManager = &indexBlobManagerImpl{ st: sm.st, - encryptor: sm.encryptor, - hasher: sm.hasher, + crypter: sm.crypter, timeNow: sm.timeNow, ownWritesCache: owc, listCache: listCache, @@ -405,7 +406,7 @@ func NewSharedManager(ctx context.Context, st blob.Storage, f *FormattingOptions return nil, errors.Errorf("can't handle repositories created using version %v (min supported %v, max supported %v)", f.Version, minSupportedWriteVersion, maxSupportedWriteVersion) } - hasher, encryptor, err := CreateHashAndEncryptor(f) + crypter, err := CreateCrypter(f) if err != nil { return nil, err } @@ -421,8 +422,7 @@ func NewSharedManager(ctx context.Context, st blob.Storage, f *FormattingOptions sm := &SharedManager{ st: st, - encryptor: encryptor, - hasher: hasher, + crypter: crypter, Stats: new(Stats), timeNow: opts.TimeNow, format: *f, @@ -433,7 +433,7 @@ func NewSharedManager(ctx context.Context, st blob.Storage, f *FormattingOptions repositoryFormatBytes: opts.RepositoryFormatBytes, checkInvariantsOnUnlock: os.Getenv("KOPIA_VERIFY_INVARIANTS") != "", writeFormatVersion: int32(f.Version), - encryptionBufferPool: buf.NewPool(ctx, defaultEncryptionBufferPoolSegmentSize+encryptor.Overhead()+maxCompressionOverheadPerContent, "content-manager-encryption"), + encryptionBufferPool: buf.NewPool(ctx, defaultEncryptionBufferPoolSegmentSize+crypter.Encryptor.Overhead()+maxCompressionOverheadPerContent, "content-manager-encryption"), indexVersion: actualIndexVersion, } diff --git a/repo/content/content_formatter_test.go b/repo/content/content_formatter_test.go index e0c23f694..0089cd29d 100644 --- a/repo/content/content_formatter_test.go +++ b/repo/content/content_formatter_test.go @@ -43,7 +43,7 @@ func TestFormatters(t *testing.T) { t.Run(encryptionAlgo, func(t *testing.T) { ctx := testlogging.Context(t) - h, e, err := CreateHashAndEncryptor(&FormattingOptions{ + cr, err := CreateCrypter(&FormattingOptions{ HMACSecret: secret, MasterKey: make([]byte, 32), Hash: hashAlgo, @@ -66,14 +66,14 @@ func TestFormatters(t *testing.T) { return } - contentID := h(nil, data) + contentID := cr.HashFunction(nil, data) - cipherText, err := e.Encrypt(nil, data, contentID) + cipherText, err := cr.Encryptor.Encrypt(nil, data, contentID) if err != nil || cipherText == nil { t.Errorf("invalid response from Encrypt: %v %v", cipherText, err) } - plainText, err := e.Decrypt(nil, cipherText, contentID) + plainText, err := cr.Encryptor.Decrypt(nil, cipherText, contentID) if err != nil || plainText == nil { t.Errorf("invalid response from Decrypt: %v %v", plainText, err) } diff --git a/repo/content/content_index_reader.go b/repo/content/content_index_reader.go index 469d5ff5a..763304422 100644 --- a/repo/content/content_index_reader.go +++ b/repo/content/content_index_reader.go @@ -9,6 +9,5 @@ // IndexBlobReader defines content read API. type IndexBlobReader interface { ParseIndexBlob(ctx context.Context, blobID blob.ID) ([]Info, error) - DecryptBlob(ctx context.Context, blobID blob.ID) ([]byte, error) IndexBlobs(ctx context.Context, includeInactive bool) ([]IndexBlobInfo, error) } diff --git a/repo/content/content_index_recovery.go b/repo/content/content_index_recovery.go index 51c835033..51cfee6ab 100644 --- a/repo/content/content_index_recovery.go +++ b/repo/content/content_index_recovery.go @@ -20,7 +20,7 @@ func (bm *WriteManager) RecoverIndexFromPackBlob(ctx context.Context, packFile b return nil, err } - ndx, err := openPackIndex(bytes.NewReader(localIndexBytes), uint32(bm.encryptor.Overhead())) + ndx, err := openPackIndex(bytes.NewReader(localIndexBytes), uint32(bm.crypter.Encryptor.Overhead())) if err != nil { return nil, errors.Errorf("unable to open index in file %v", packFile) } @@ -181,7 +181,7 @@ func (sm *SharedManager) writePackFileIndexRecoveryData(buf *gather.WriteBuffer, localIndexIV := sm.hashData(nil, localIndex) - encryptedLocalIndex, err := sm.encryptor.Encrypt(nil, localIndex, localIndexIV) + encryptedLocalIndex, err := sm.crypter.Encryptor.Encrypt(nil, localIndex, localIndexIV) if err != nil { return errors.Wrap(err, "encryption error") } diff --git a/repo/content/content_manager_indexes.go b/repo/content/content_manager_indexes.go index 2964953dd..487938ac2 100644 --- a/repo/content/content_manager_indexes.go +++ b/repo/content/content_manager_indexes.go @@ -178,7 +178,7 @@ func (sm *SharedManager) addIndexBlobsToBuilder(ctx context.Context, bld packInd return errors.Wrapf(err, "error getting index %q", indexBlob.BlobID) } - index, err := openPackIndex(bytes.NewReader(data), uint32(sm.encryptor.Overhead())) + index, err := openPackIndex(bytes.NewReader(data), uint32(sm.crypter.Encryptor.Overhead())) if err != nil { return errors.Wrapf(err, "unable to open index blob %q", indexBlob) } @@ -198,7 +198,7 @@ func (sm *SharedManager) ParseIndexBlob(ctx context.Context, blobID blob.ID) ([] return nil, errors.Wrapf(err, "error getting index %q", blobID) } - index, err := openPackIndex(bytes.NewReader(data), uint32(sm.encryptor.Overhead())) + index, err := openPackIndex(bytes.NewReader(data), uint32(sm.crypter.Encryptor.Overhead())) if err != nil { return nil, errors.Wrapf(err, "unable to open index blob") } diff --git a/repo/content/content_manager_lock_free.go b/repo/content/content_manager_lock_free.go index 4ea448fc0..5f32d40ce 100644 --- a/repo/content/content_manager_lock_free.go +++ b/repo/content/content_manager_lock_free.go @@ -60,10 +60,10 @@ func (sm *SharedManager) maybeCompressAndEncryptDataForPacking(output *gather.Wr } } - b := sm.encryptionBufferPool.Allocate(len(data) + sm.encryptor.Overhead()) + b := sm.encryptionBufferPool.Allocate(len(data) + sm.crypter.Encryptor.Overhead()) defer b.Release() - cipherText, err := sm.encryptor.Encrypt(b.Data[:0], data, iv) + cipherText, err := sm.crypter.Encryptor.Encrypt(b.Data[:0], data, iv) if err != nil { return NoCompression, errors.Wrap(err, "unable to encrypt") } @@ -180,31 +180,30 @@ func (bm *WriteManager) writePackFileNotLocked(ctx context.Context, packFile blo func (sm *SharedManager) hashData(output, data []byte) []byte { // Hash the content and compute encryption key. - contentID := sm.hasher(output, data) + contentID := sm.crypter.HashFunction(output, data) sm.Stats.hashedContent(len(data)) return contentID } -// CreateHashAndEncryptor returns new hashing and encrypting functions based on -// the specified formatting options. -func CreateHashAndEncryptor(f *FormattingOptions) (hashing.HashFunc, encryption.Encryptor, error) { +// CreateCrypter returns a Crypter based on the specified formatting options. +func CreateCrypter(f *FormattingOptions) (*Crypter, error) { h, err := hashing.CreateHashFunc(f) if err != nil { - return nil, nil, errors.Wrap(err, "unable to create hash") + return nil, errors.Wrap(err, "unable to create hash") } e, err := encryption.CreateEncryptor(f) if err != nil { - return nil, nil, errors.Wrap(err, "unable to create encryptor") + return nil, errors.Wrap(err, "unable to create encryptor") } contentID := h(nil, nil) _, err = e.Encrypt(nil, nil, contentID) if err != nil { - return nil, nil, errors.Wrap(err, "invalid encryptor") + return nil, errors.Wrap(err, "invalid encryptor") } - return h, e, nil + return &Crypter{h, e}, nil } diff --git a/repo/content/index_blob_manager.go b/repo/content/index_blob_manager.go index 23d6c8804..f2e6ee7c8 100644 --- a/repo/content/index_blob_manager.go +++ b/repo/content/index_blob_manager.go @@ -10,8 +10,6 @@ "github.com/kopia/kopia/internal/gather" "github.com/kopia/kopia/repo/blob" - "github.com/kopia/kopia/repo/encryption" - "github.com/kopia/kopia/repo/hashing" ) // indexBlobManager is the API of index blob manager as used by content manager. @@ -57,8 +55,7 @@ type cleanupEntry struct { type indexBlobManagerImpl struct { st blob.Storage - hasher hashing.HashFunc - encryptor encryption.Encryptor + crypter *Crypter listCache *listCache ownWritesCache ownWritesCache timeNow func() time.Time @@ -178,7 +175,7 @@ func (m *indexBlobManagerImpl) getEncryptedBlob(ctx context.Context, blobID blob return nil, errors.Wrap(err, "getContent") } - return decryptFullBlob(m.hasher, m.encryptor, payload, blobID) + return m.crypter.DecryptBLOB(payload, blobID) } func (m *indexBlobManagerImpl) writeIndexBlob(ctx context.Context, data []byte, sessionID SessionID) (blob.Metadata, error) { @@ -186,7 +183,7 @@ func (m *indexBlobManagerImpl) writeIndexBlob(ctx context.Context, data []byte, } func (m *indexBlobManagerImpl) encryptAndWriteBlob(ctx context.Context, data []byte, prefix blob.ID, sessionID SessionID) (blob.Metadata, error) { - blobID, data2, err := encryptFullBlob(m.hasher, m.encryptor, data, prefix, sessionID) + blobID, data2, err := m.crypter.EncryptBLOB(data, prefix, sessionID) if err != nil { return blob.Metadata{}, errors.Wrap(err, "error encrypting") } diff --git a/repo/content/index_blob_manager_test.go b/repo/content/index_blob_manager_test.go index 77fc7be7f..f2daadde1 100644 --- a/repo/content/index_blob_manager_test.go +++ b/repo/content/index_blob_manager_test.go @@ -774,10 +774,12 @@ func newIndexBlobManagerForTesting(t *testing.T, st blob.Storage, localTimeNow f localTimeNow, }, indexBlobCache: passthroughContentCache{st}, - encryptor: enc, - hasher: hf, - listCache: lc, - timeNow: localTimeNow, + crypter: &Crypter{ + HashFunction: hf, + Encryptor: enc, + }, + listCache: lc, + timeNow: localTimeNow, } return m diff --git a/repo/content/sessions.go b/repo/content/sessions.go index 3a4d2d739..d4d89b28c 100644 --- a/repo/content/sessions.go +++ b/repo/content/sessions.go @@ -105,7 +105,7 @@ func (bm *WriteManager) writeSessionMarkerLocked(ctx context.Context) error { return errors.Wrap(err, "unable to serialize session marker payload") } - sessionBlobID, encrypted, err := encryptFullBlob(bm.hasher, bm.encryptor, js, BlobIDPrefixSession, bm.currentSessionInfo.ID) + sessionBlobID, encrypted, err := bm.crypter.EncryptBLOB(js, BlobIDPrefixSession, bm.currentSessionInfo.ID) if err != nil { return errors.Wrap(err, "unable to encrypt session marker") } @@ -163,7 +163,7 @@ func (bm *WriteManager) ListActiveSessions(ctx context.Context) (map[SessionID]* return nil, errors.Wrapf(err, "error loading session: %v", b.BlobID) } - payload, err = decryptFullBlob(bm.hasher, bm.encryptor, payload, b.BlobID) + payload, err = bm.crypter.DecryptBLOB(payload, b.BlobID) if err != nil { return nil, errors.Wrapf(err, "error decrypting session: %v", b.BlobID) } diff --git a/repo/repository.go b/repo/repository.go index 9015517aa..53b1477dd 100644 --- a/repo/repository.go +++ b/repo/repository.go @@ -51,6 +51,7 @@ type DirectRepository interface { BlobReader() blob.Reader ContentReader() content.Reader IndexBlobReader() content.IndexBlobReader + Crypter() *content.Crypter NewDirectWriter(ctx context.Context, opt WriteSessionOptions) (DirectRepositoryWriter, error) @@ -119,6 +120,11 @@ func (r *directRepository) ConfigFilename() string { return r.configFile } +// Crypter returns a Crypter object. +func (r *directRepository) Crypter() *content.Crypter { + return r.sm.Crypter() +} + // NewObjectWriter creates an object writer. func (r *directRepository) NewObjectWriter(ctx context.Context, opt object.WriterOptions) object.Writer { return r.omgr.NewWriter(ctx, opt)