content: introduced content.Crypter (#1112)

cli/command_benchmark_crypto.go

@@ -54,7 +54,7 @@ type benchResult struct {
 
 	for _, ha := range hashing.SupportedAlgorithms() {
 		for _, ea := range encryption.SupportedAlgorithms(c.deprecatedAlgorithms) {
-			h, e, err := content.CreateHashAndEncryptor(&content.FormattingOptions{
+			cr, err := content.CreateCrypter(&content.FormattingOptions{
 				Encryption: ea,
 				Hash:       ha,
 				MasterKey:  make([]byte, 32),
@@ -71,8 +71,8 @@ type benchResult struct {
 			hashCount := c.repeat
 
 			for i := 0; i < hashCount; i++ {
-				contentID := h(hashOutput[:0], data)
-				if _, encerr := e.Encrypt(encryptOutput[:0], data, contentID); encerr != nil {
+				contentID := cr.HashFunction(hashOutput[:0], data)
+				if _, encerr := cr.Encryptor.Encrypt(encryptOutput[:0], data, contentID); encerr != nil {
 					log(ctx).Errorf("encryption failed: %v", encerr)
 					break
 				}

cli/command_blob_show.go

@@ -45,8 +45,10 @@ func (c *commandBlobShow) maybeDecryptBlob(ctx context.Context, w io.Writer, rep
 		err error
 	)
 
+	d, err = rep.BlobReader().GetBlob(ctx, blobID, 0, -1)
+
 	if c.blobShowDecrypt && canDecryptBlob(blobID) {
-		d, err = rep.IndexBlobReader().DecryptBlob(ctx, blobID)
+		d, err = rep.Crypter().DecryptBLOB(d, blobID)
 
 		if isJSONBlob(blobID) && err == nil {
 			var b bytes.Buffer
@@ -57,8 +59,6 @@ func (c *commandBlobShow) maybeDecryptBlob(ctx context.Context, w io.Writer, rep
 
 			d = b.Bytes()
 		}
-	} else {
-		d, err = rep.BlobReader().GetBlob(ctx, blobID, 0, -1)
 	}
 
 	if err != nil {

repo/content/blob_crypto.go

@@ -1,7 +1,6 @@
 package content
 
 import (
-	"bytes"
 	"crypto/aes"
 	"encoding/hex"
 	"strings"
@@ -13,7 +12,25 @@
 	"github.com/kopia/kopia/repo/hashing"
 )
 
-func getIndexBlobIV(s blob.ID) ([]byte, error) {
+// Crypter ecapsulates hashing and encryption and provides utilities for whole-BLOB encryption.
+// Whole-BLOB encryption relies on BLOB identifiers formatted as:
+//
+// <prefix><hash>[-optionalSuffix]
+//
+// Where:
+//   'prefix' is arbitrary string without dashes
+//   'hash' is base16-encoded 128-bit hash of contents, used as initialization vector (IV)
+//          for the encryption. In case of longer hash functions, we use last 16 bytes of
+//          their outputs.
+//   'optionalSuffix' can be any string
+type Crypter struct {
+	HashFunction hashing.HashFunc
+	Encryptor    encryption.Encryptor
+}
+
+// getIndexBlobIV gets the initialization vector from the provided blob ID by taking
+// 32 characters immediately preceding the first dash ('-') and decoding them using base16.
+func (c *Crypter) getIndexBlobIV(s blob.ID) ([]byte, error) {
 	if p := strings.Index(string(s), "-"); p >= 0 { // nolint:gocritic
 		s = s[0:p]
 	}
@@ -30,58 +47,43 @@ func getIndexBlobIV(s blob.ID) ([]byte, error) {
 	return v, nil
 }
 
-func encryptFullBlob(h hashing.HashFunc, enc encryption.Encryptor, data []byte, prefix blob.ID, sessionID SessionID) (blob.ID, []byte, error) {
+// EncryptBLOB encrypts the given data using crypter-defined key and returns a name that should
+// be used to save the blob in thre repository.
+func (c *Crypter) EncryptBLOB(data []byte, prefix blob.ID, sessionID SessionID) (blob.ID, []byte, error) {
 	var hashOutput [maxHashSize]byte
 
-	hash := h(hashOutput[:0], data)
+	hash := c.HashFunction(hashOutput[:0], data)
 	blobID := prefix + blob.ID(hex.EncodeToString(hash))
 
 	if sessionID != "" {
 		blobID += blob.ID("-" + sessionID)
 	}
 
-	iv, err := getIndexBlobIV(blobID)
+	iv, err := c.getIndexBlobIV(blobID)
 	if err != nil {
 		return "", nil, err
 	}
 
-	data2, err := enc.Encrypt(nil, data, iv)
+	data2, err := c.Encryptor.Encrypt(nil, data, iv)
 	if err != nil {
-		return "", nil, errors.Wrapf(err, "error encrypting blob %v", blobID)
+		return "", nil, errors.Wrapf(err, "error encrypting BLOB %v", blobID)
 	}
 
 	return blobID, data2, nil
 }
 
-func decryptFullBlob(h hashing.HashFunc, enc encryption.Encryptor, payload []byte, blobID blob.ID) ([]byte, error) {
-	iv, err := getIndexBlobIV(blobID)
+// DecryptBLOB decrypts the provided data using provided blobID to derive initialization vector.
+func (c *Crypter) DecryptBLOB(payload []byte, blobID blob.ID) ([]byte, error) {
+	iv, err := c.getIndexBlobIV(blobID)
 	if err != nil {
 		return nil, errors.Wrap(err, "unable to get index blob IV")
 	}
 
-	payload, err = enc.Decrypt(nil, payload, iv)
+	// Decrypt will verify the payload.
+	payload, err = c.Encryptor.Decrypt(nil, payload, iv)
 	if err != nil {
-		return nil, errors.Wrap(err, "decrypt error")
-	}
-
-	// Since the encryption key is a function of data, we must be able to generate exactly the same key
-	// after decrypting the content. This serves as a checksum.
-	if err := verifyChecksum(h, payload, iv); err != nil {
-		return nil, err
+		return nil, errors.Wrapf(err, "error decrypting BLOB %v", blobID)
 	}
 
 	return payload, nil
 }
-
-func verifyChecksum(h hashing.HashFunc, data, iv []byte) error {
-	var hashOutput [maxHashSize]byte
-
-	expected := h(hashOutput[:0], data)
-	expected = expected[len(expected)-aes.BlockSize:]
-
-	if !bytes.HasSuffix(iv, expected) {
-		return errors.Errorf("invalid checksum for blob %x, expected %x", iv, expected)
-	}
-
-	return nil
-}

repo/content/committed_read_manager.go

@@ -15,8 +15,6 @@
 	"github.com/kopia/kopia/internal/clock"
 	"github.com/kopia/kopia/repo/blob"
 	"github.com/kopia/kopia/repo/compression"
-	"github.com/kopia/kopia/repo/encryption"
-	"github.com/kopia/kopia/repo/hashing"
 )
 
 // number of bytes to read from each pack index when recovering the index.
@@ -34,8 +32,7 @@ type SharedManager struct {
 	contentCache      contentCache
 	metadataCache     contentCache
 	committedContents *committedContentIndex
-	hasher            hashing.HashFunc
-	encryptor         encryption.Encryptor
+	crypter           *Crypter
 	timeNow           func() time.Time
 
 	format                  FormattingOptions
@@ -51,6 +48,11 @@ type SharedManager struct {
 	encryptionBufferPool *buf.Pool
 }
 
+// Crypter returns the crypter.
+func (sm *SharedManager) Crypter() *Crypter {
+	return sm.crypter
+}
+
 func (sm *SharedManager) readPackFileLocalIndex(ctx context.Context, packFile blob.ID, packFileLength int64) ([]byte, error) {
 	if packFileLength >= indexRecoverPostambleSize {
 		data, err := sm.attemptReadPackFileLocalIndex(ctx, packFile, packFileLength-indexRecoverPostambleSize, indexRecoverPostambleSize)
@@ -276,7 +278,7 @@ func (sm *SharedManager) decryptContentAndVerify(payload []byte, bi Info) ([]byt
 }
 
 func (sm *SharedManager) decryptAndVerify(encrypted, iv []byte) ([]byte, error) {
-	decrypted, err := sm.encryptor.Decrypt(nil, encrypted, iv)
+	decrypted, err := sm.crypter.Encryptor.Decrypt(nil, encrypted, iv)
 	if err != nil {
 		sm.Stats.foundInvalidContent()
 		return nil, errors.Wrap(err, "decrypt")
@@ -332,7 +334,7 @@ func (sm *SharedManager) setupReadManagerCaches(ctx context.Context, caching *Ca
 		return errors.Wrap(err, "unable to initialize own writes cache")
 	}
 
-	contentIndex := newCommittedContentIndex(caching, uint32(sm.encryptor.Overhead()), sm.indexVersion)
+	contentIndex := newCommittedContentIndex(caching, uint32(sm.crypter.Encryptor.Overhead()), sm.indexVersion)
 
 	// once everything is ready, set it up
 	sm.contentCache = dataCache
@@ -341,8 +343,7 @@ func (sm *SharedManager) setupReadManagerCaches(ctx context.Context, caching *Ca
 
 	sm.indexBlobManager = &indexBlobManagerImpl{
 		st:             sm.st,
-		encryptor:      sm.encryptor,
-		hasher:         sm.hasher,
+		crypter:        sm.crypter,
 		timeNow:        sm.timeNow,
 		ownWritesCache: owc,
 		listCache:      listCache,
@@ -405,7 +406,7 @@ func NewSharedManager(ctx context.Context, st blob.Storage, f *FormattingOptions
 		return nil, errors.Errorf("can't handle repositories created using version %v (min supported %v, max supported %v)", f.Version, minSupportedWriteVersion, maxSupportedWriteVersion)
 	}
 
-	hasher, encryptor, err := CreateHashAndEncryptor(f)
+	crypter, err := CreateCrypter(f)
 	if err != nil {
 		return nil, err
 	}
@@ -421,8 +422,7 @@ func NewSharedManager(ctx context.Context, st blob.Storage, f *FormattingOptions
 
 	sm := &SharedManager{
 		st:                      st,
-		encryptor:               encryptor,
-		hasher:                  hasher,
+		crypter:                 crypter,
 		Stats:                   new(Stats),
 		timeNow:                 opts.TimeNow,
 		format:                  *f,
@@ -433,7 +433,7 @@ func NewSharedManager(ctx context.Context, st blob.Storage, f *FormattingOptions
 		repositoryFormatBytes:   opts.RepositoryFormatBytes,
 		checkInvariantsOnUnlock: os.Getenv("KOPIA_VERIFY_INVARIANTS") != "",
 		writeFormatVersion:      int32(f.Version),
-		encryptionBufferPool:    buf.NewPool(ctx, defaultEncryptionBufferPoolSegmentSize+encryptor.Overhead()+maxCompressionOverheadPerContent, "content-manager-encryption"),
+		encryptionBufferPool:    buf.NewPool(ctx, defaultEncryptionBufferPoolSegmentSize+crypter.Encryptor.Overhead()+maxCompressionOverheadPerContent, "content-manager-encryption"),
 		indexVersion:            actualIndexVersion,
 	}
 

repo/content/content_formatter_test.go

@@ -43,7 +43,7 @@ func TestFormatters(t *testing.T) {
 				t.Run(encryptionAlgo, func(t *testing.T) {
 					ctx := testlogging.Context(t)
 
-					h, e, err := CreateHashAndEncryptor(&FormattingOptions{
+					cr, err := CreateCrypter(&FormattingOptions{
 						HMACSecret: secret,
 						MasterKey:  make([]byte, 32),
 						Hash:       hashAlgo,
@@ -66,14 +66,14 @@ func TestFormatters(t *testing.T) {
 						return
 					}
 
-					contentID := h(nil, data)
+					contentID := cr.HashFunction(nil, data)
 
-					cipherText, err := e.Encrypt(nil, data, contentID)
+					cipherText, err := cr.Encryptor.Encrypt(nil, data, contentID)
 					if err != nil || cipherText == nil {
 						t.Errorf("invalid response from Encrypt: %v %v", cipherText, err)
 					}
 
-					plainText, err := e.Decrypt(nil, cipherText, contentID)
+					plainText, err := cr.Encryptor.Decrypt(nil, cipherText, contentID)
 					if err != nil || plainText == nil {
 						t.Errorf("invalid response from Decrypt: %v %v", plainText, err)
 					}

repo/content/content_index_reader.go

@@ -9,6 +9,5 @@
 // IndexBlobReader defines content read API.
 type IndexBlobReader interface {
 	ParseIndexBlob(ctx context.Context, blobID blob.ID) ([]Info, error)
-	DecryptBlob(ctx context.Context, blobID blob.ID) ([]byte, error)
 	IndexBlobs(ctx context.Context, includeInactive bool) ([]IndexBlobInfo, error)
 }

repo/content/content_index_recovery.go

@@ -20,7 +20,7 @@ func (bm *WriteManager) RecoverIndexFromPackBlob(ctx context.Context, packFile b
 		return nil, err
 	}
 
-	ndx, err := openPackIndex(bytes.NewReader(localIndexBytes), uint32(bm.encryptor.Overhead()))
+	ndx, err := openPackIndex(bytes.NewReader(localIndexBytes), uint32(bm.crypter.Encryptor.Overhead()))
 	if err != nil {
 		return nil, errors.Errorf("unable to open index in file %v", packFile)
 	}
@@ -181,7 +181,7 @@ func (sm *SharedManager) writePackFileIndexRecoveryData(buf *gather.WriteBuffer,
 
 	localIndexIV := sm.hashData(nil, localIndex)
 
-	encryptedLocalIndex, err := sm.encryptor.Encrypt(nil, localIndex, localIndexIV)
+	encryptedLocalIndex, err := sm.crypter.Encryptor.Encrypt(nil, localIndex, localIndexIV)
 	if err != nil {
 		return errors.Wrap(err, "encryption error")
 	}

repo/content/content_manager_indexes.go

@@ -178,7 +178,7 @@ func (sm *SharedManager) addIndexBlobsToBuilder(ctx context.Context, bld packInd
 		return errors.Wrapf(err, "error getting index %q", indexBlob.BlobID)
 	}
 
-	index, err := openPackIndex(bytes.NewReader(data), uint32(sm.encryptor.Overhead()))
+	index, err := openPackIndex(bytes.NewReader(data), uint32(sm.crypter.Encryptor.Overhead()))
 	if err != nil {
 		return errors.Wrapf(err, "unable to open index blob %q", indexBlob)
 	}
@@ -198,7 +198,7 @@ func (sm *SharedManager) ParseIndexBlob(ctx context.Context, blobID blob.ID) ([]
 		return nil, errors.Wrapf(err, "error getting index %q", blobID)
 	}
 
-	index, err := openPackIndex(bytes.NewReader(data), uint32(sm.encryptor.Overhead()))
+	index, err := openPackIndex(bytes.NewReader(data), uint32(sm.crypter.Encryptor.Overhead()))
 	if err != nil {
 		return nil, errors.Wrapf(err, "unable to open index blob")
 	}

repo/content/content_manager_lock_free.go

@@ -60,10 +60,10 @@ func (sm *SharedManager) maybeCompressAndEncryptDataForPacking(output *gather.Wr
 		}
 	}
 
-	b := sm.encryptionBufferPool.Allocate(len(data) + sm.encryptor.Overhead())
+	b := sm.encryptionBufferPool.Allocate(len(data) + sm.crypter.Encryptor.Overhead())
 	defer b.Release()
 
-	cipherText, err := sm.encryptor.Encrypt(b.Data[:0], data, iv)
+	cipherText, err := sm.crypter.Encryptor.Encrypt(b.Data[:0], data, iv)
 	if err != nil {
 		return NoCompression, errors.Wrap(err, "unable to encrypt")
 	}
@@ -180,31 +180,30 @@ func (bm *WriteManager) writePackFileNotLocked(ctx context.Context, packFile blo
 
 func (sm *SharedManager) hashData(output, data []byte) []byte {
 	// Hash the content and compute encryption key.
-	contentID := sm.hasher(output, data)
+	contentID := sm.crypter.HashFunction(output, data)
 	sm.Stats.hashedContent(len(data))
 
 	return contentID
 }
 
-// CreateHashAndEncryptor returns new hashing and encrypting functions based on
-// the specified formatting options.
-func CreateHashAndEncryptor(f *FormattingOptions) (hashing.HashFunc, encryption.Encryptor, error) {
+// CreateCrypter returns a Crypter based on the specified formatting options.
+func CreateCrypter(f *FormattingOptions) (*Crypter, error) {
 	h, err := hashing.CreateHashFunc(f)
 	if err != nil {
-		return nil, nil, errors.Wrap(err, "unable to create hash")
+		return nil, errors.Wrap(err, "unable to create hash")
 	}
 
 	e, err := encryption.CreateEncryptor(f)
 	if err != nil {
-		return nil, nil, errors.Wrap(err, "unable to create encryptor")
+		return nil, errors.Wrap(err, "unable to create encryptor")
 	}
 
 	contentID := h(nil, nil)
 
 	_, err = e.Encrypt(nil, nil, contentID)
 	if err != nil {
-		return nil, nil, errors.Wrap(err, "invalid encryptor")
+		return nil, errors.Wrap(err, "invalid encryptor")
 	}
 
-	return h, e, nil
+	return &Crypter{h, e}, nil
 }

repo/content/index_blob_manager.go

@@ -10,8 +10,6 @@
 
 	"github.com/kopia/kopia/internal/gather"
 	"github.com/kopia/kopia/repo/blob"
-	"github.com/kopia/kopia/repo/encryption"
-	"github.com/kopia/kopia/repo/hashing"
 )
 
 // indexBlobManager is the API of index blob manager as used by content manager.
@@ -57,8 +55,7 @@ type cleanupEntry struct {
 
 type indexBlobManagerImpl struct {
 	st             blob.Storage
-	hasher         hashing.HashFunc
-	encryptor      encryption.Encryptor
+	crypter        *Crypter
 	listCache      *listCache
 	ownWritesCache ownWritesCache
 	timeNow        func() time.Time
@@ -178,7 +175,7 @@ func (m *indexBlobManagerImpl) getEncryptedBlob(ctx context.Context, blobID blob
 		return nil, errors.Wrap(err, "getContent")
 	}
 
-	return decryptFullBlob(m.hasher, m.encryptor, payload, blobID)
+	return m.crypter.DecryptBLOB(payload, blobID)
 }
 
 func (m *indexBlobManagerImpl) writeIndexBlob(ctx context.Context, data []byte, sessionID SessionID) (blob.Metadata, error) {
@@ -186,7 +183,7 @@ func (m *indexBlobManagerImpl) writeIndexBlob(ctx context.Context, data []byte,
 }
 
 func (m *indexBlobManagerImpl) encryptAndWriteBlob(ctx context.Context, data []byte, prefix blob.ID, sessionID SessionID) (blob.Metadata, error) {
-	blobID, data2, err := encryptFullBlob(m.hasher, m.encryptor, data, prefix, sessionID)
+	blobID, data2, err := m.crypter.EncryptBLOB(data, prefix, sessionID)
 	if err != nil {
 		return blob.Metadata{}, errors.Wrap(err, "error encrypting")
 	}

repo/content/index_blob_manager_test.go

@@ -774,10 +774,12 @@ func newIndexBlobManagerForTesting(t *testing.T, st blob.Storage, localTimeNow f
 			localTimeNow,
 		},
 		indexBlobCache: passthroughContentCache{st},
-		encryptor:      enc,
-		hasher:         hf,
-		listCache:      lc,
-		timeNow:        localTimeNow,
+		crypter: &Crypter{
+			HashFunction: hf,
+			Encryptor:    enc,
+		},
+		listCache: lc,
+		timeNow:   localTimeNow,
 	}
 
 	return m

repo/content/sessions.go

@@ -105,7 +105,7 @@ func (bm *WriteManager) writeSessionMarkerLocked(ctx context.Context) error {
 		return errors.Wrap(err, "unable to serialize session marker payload")
 	}
 
-	sessionBlobID, encrypted, err := encryptFullBlob(bm.hasher, bm.encryptor, js, BlobIDPrefixSession, bm.currentSessionInfo.ID)
+	sessionBlobID, encrypted, err := bm.crypter.EncryptBLOB(js, BlobIDPrefixSession, bm.currentSessionInfo.ID)
 	if err != nil {
 		return errors.Wrap(err, "unable to encrypt session marker")
 	}
@@ -163,7 +163,7 @@ func (bm *WriteManager) ListActiveSessions(ctx context.Context) (map[SessionID]*
 			return nil, errors.Wrapf(err, "error loading session: %v", b.BlobID)
 		}
 
-		payload, err = decryptFullBlob(bm.hasher, bm.encryptor, payload, b.BlobID)
+		payload, err = bm.crypter.DecryptBLOB(payload, b.BlobID)
 		if err != nil {
 			return nil, errors.Wrapf(err, "error decrypting session: %v", b.BlobID)
 		}

repo/repository.go

@@ -51,6 +51,7 @@ type DirectRepository interface {
 	BlobReader() blob.Reader
 	ContentReader() content.Reader
 	IndexBlobReader() content.IndexBlobReader
+	Crypter() *content.Crypter
 
 	NewDirectWriter(ctx context.Context, opt WriteSessionOptions) (DirectRepositoryWriter, error)
 
@@ -119,6 +120,11 @@ func (r *directRepository) ConfigFilename() string {
 	return r.configFile
 }
 
+// Crypter returns a Crypter object.
+func (r *directRepository) Crypter() *content.Crypter {
+	return r.sm.Crypter()
+}
+
 // NewObjectWriter creates an object writer.
 func (r *directRepository) NewObjectWriter(ctx context.Context, opt object.WriterOptions) object.Writer {
 	return r.omgr.NewWriter(ctx, opt)