From 0cd6ebabb4792ba85bdade6175aa7c6032bc7950 Mon Sep 17 00:00:00 2001 From: Jarek Kowalski Date: Sun, 3 Apr 2016 10:16:07 -0700 Subject: [PATCH] added missing data encryption/decryption and validation --- cas/object_manager.go | 170 ++++++++++++++++++++++++++++++++++--- cas/object_manager_test.go | 105 ++++++++++++++++++++++- cas/object_reader.go | 81 ------------------ cas/objectid.go | 49 +---------- cas/objectid_test.go | 8 +- fs/dir.go | 1 + session/session.go | 100 ++++++++++++++++++++++ session/session_test.go | 51 +++++++++++ user/doc.go | 1 + 9 files changed, 417 insertions(+), 149 deletions(-) create mode 100644 session/session.go create mode 100644 session/session_test.go create mode 100644 user/doc.go diff --git a/cas/object_manager.go b/cas/object_manager.go index fc43aea68..58784a572 100644 --- a/cas/object_manager.go +++ b/cas/object_manager.go @@ -1,6 +1,7 @@ package cas import ( + "bufio" "bytes" "crypto/aes" "crypto/cipher" @@ -10,10 +11,13 @@ "crypto/sha256" "crypto/sha512" "encoding/hex" + "errors" "fmt" "hash" "io" "io/ioutil" + "log" + "strconv" "strings" "sync/atomic" @@ -41,9 +45,17 @@ type ObjectManager interface { // ObjectManagerStats exposes statistics about ObjectManager operation type ObjectManagerStats struct { - HashedBytes int64 - HashedBlocks int32 - UploadedBytes int64 + HashedBytes int64 + HashedBlocks int32 + + BytesReadFromStorage int64 + BytesWrittenToStorage int64 + + EncryptedBytes int64 + DecryptedBytes int64 + + InvalidBlobs int32 + ValidBlobs int32 } type keygenFunc func([]byte) (key []byte, locator []byte) @@ -222,25 +234,31 @@ func NewObjectManager( func splitHash(keySize int) keygenFunc { return func(b []byte) ([]byte, []byte) { p := len(b) - keySize - return b[p:], b[0:p] + return b[0:p], b[p:] } } +func (mgr *objectManager) hashBuffer(data []byte) ([]byte, []byte) { + h := mgr.hashFunc() + h.Write(data) + contentHash := h.Sum(nil) + + if mgr.keygen != nil { + return mgr.keygen(contentHash) + } + + return contentHash, nil +} + func (mgr *objectManager) hashBufferForWriting(buffer *bytes.Buffer, prefix string) (ObjectID, io.ReadCloser) { var data []byte if buffer != nil { data = buffer.Bytes() } - h := mgr.hashFunc() - h.Write(data) - contentHash := h.Sum(nil) - + contentHash, cryptoKey := mgr.hashBuffer(data) var objectID ObjectID - var cryptoKey []byte - - if mgr.createCipher != nil { - cryptoKey, contentHash = mgr.keygen(contentHash) + if cryptoKey != nil { objectID = ObjectID(prefix + hex.EncodeToString(contentHash) + ":" + hex.EncodeToString(cryptoKey)) } else { objectID = ObjectID(prefix + hex.EncodeToString(contentHash)) @@ -253,19 +271,143 @@ func (mgr *objectManager) hashBufferForWriting(buffer *bytes.Buffer, prefix stri } readCloser := mgr.bufferManager.returnBufferOnClose(buffer) + readCloser = newCountingReader(readCloser, &mgr.stats.BytesWrittenToStorage) if cryptoKey != nil { c, err := mgr.createCipher(cryptoKey) if err != nil { - panic("can't create cipher") + log.Printf("can't create cipher: %v", err) + panic("can't encrypt block") } // Since we're not sharing the key, all-zero IV is ok. // We don't need to worry about separate MAC either, since hashing content produces object ID. ctr := cipher.NewCTR(c, constantIV[0:c.BlockSize()]) - readCloser = newEncryptingReader(readCloser, nil, ctr, nil) + readCloser = newCountingReader( + newEncryptingReader(readCloser, nil, ctr, nil), + &mgr.stats.EncryptedBytes) } return objectID, readCloser } + +func (mgr *objectManager) flattenListChunk( + seekTable []seekTableEntry, + listObjectID ObjectID, + rawReader io.Reader) ([]seekTableEntry, error) { + + scanner := bufio.NewScanner(rawReader) + + for scanner.Scan() { + c := scanner.Text() + comma := strings.Index(c, ",") + if comma <= 0 { + return nil, fmt.Errorf("unsupported entry '%v' in list '%s'", c, listObjectID) + } + + length, err := strconv.ParseInt(c[0:comma], 10, 64) + + objectID, err := ParseObjectID(c[comma+1:]) + if err != nil { + return nil, fmt.Errorf("unsupported entry '%v' in list '%s': %#v", c, listObjectID, err) + } + + switch objectID.Type() { + case ObjectIDTypeList: + subreader, err := mgr.newRawReader(objectID) + if err != nil { + return nil, err + } + + seekTable, err = mgr.flattenListChunk(seekTable, objectID, subreader) + if err != nil { + return nil, err + } + + case ObjectIDTypeStored: + var startOffset int64 + if len(seekTable) > 0 { + startOffset = seekTable[len(seekTable)-1].endOffset() + } else { + startOffset = 0 + } + + seekTable = append( + seekTable, + seekTableEntry{ + blockID: objectID.BlockID(), + startOffset: startOffset, + length: length, + }) + + default: + return nil, fmt.Errorf("unsupported entry '%v' in list '%v'", objectID, listObjectID) + + } + } + + return seekTable, nil +} + +func (mgr *objectManager) newRawReader(objectID ObjectID) (io.ReadSeeker, error) { + inline := objectID.InlineData() + if inline != nil { + return bytes.NewReader(inline), nil + } + + blockID := objectID.BlockID() + payload, err := mgr.storage.GetBlock(blockID) + if err != nil { + return nil, err + } + + atomic.AddInt64(&mgr.stats.BytesReadFromStorage, int64(len(payload))) + + if objectID.EncryptionInfo() == NoEncryption { + if err := mgr.verifyChecksum(payload, objectID.BlockID()); err != nil { + return nil, err + } + return bytes.NewReader(payload), nil + } + + if mgr.createCipher == nil { + return nil, errors.New("encrypted object cannot be used with non-encrypted ObjectManager") + } + + cryptoKey, err := hex.DecodeString(string(objectID.EncryptionInfo())) + if err != nil { + return nil, errors.New("malformed encryption key") + } + + blockCipher, err := mgr.createCipher(cryptoKey) + if err != nil { + return nil, errors.New("cannot create cipher") + } + + iv := constantIV[0:blockCipher.BlockSize()] + ctr := cipher.NewCTR(blockCipher, iv) + ctr.XORKeyStream(payload, payload) + + // Since the encryption key is a function of data, we must be able to generate exactly the same key + // after decrypting the content. This serves as a checksum. + atomic.AddInt64(&mgr.stats.DecryptedBytes, int64(len(payload))) + + if err := mgr.verifyChecksum(payload, objectID.BlockID()); err != nil { + return nil, err + } + + return bytes.NewReader(payload), nil +} + +func (mgr *objectManager) verifyChecksum(data []byte, blockID blob.BlockID) error { + payloadHash, _ := mgr.hashBuffer(data) + checksum := hex.EncodeToString(payloadHash) + if !strings.HasSuffix(string(blockID), checksum) { + atomic.AddInt32(&mgr.stats.InvalidBlobs, 1) + return fmt.Errorf("invalid checksum for blob: '%v'", blockID) + } + + atomic.AddInt32(&mgr.stats.ValidBlobs, 1) + return nil +} diff --git a/cas/object_manager_test.go b/cas/object_manager_test.go index a51486124..c29f49984 100644 --- a/cas/object_manager_test.go +++ b/cas/object_manager_test.go @@ -254,7 +254,7 @@ func TestReader(t *testing.T) { data, mgr := setupTest(t) storedPayload := []byte("foo\nbar") - data["abcdef"] = storedPayload + data["a76999788386641a3ec798554f1fe7e6"] = storedPayload cases := []struct { text string @@ -264,7 +264,7 @@ func TestReader(t *testing.T) { {"BAQIDBA", []byte{1, 2, 3, 4}}, {"T", []byte{}}, {"Tfoo\nbar", []byte("foo\nbar")}, - {"Cabcdef", storedPayload}, + {"Ca76999788386641a3ec798554f1fe7e6", storedPayload}, } for _, c := range cases { @@ -292,6 +292,29 @@ func TestReader(t *testing.T) { } } +func TestMalformedStoredData(t *testing.T) { + data, mgr := setupTest(t) + + cases := [][]byte{ + []byte("foo\nba"), + []byte("foo\nbar1"), + } + + for _, c := range cases { + data["a76999788386641a3ec798554f1fe7e6"] = c + objectID, err := ParseObjectID("Ca76999788386641a3ec798554f1fe7e6") + if err != nil { + t.Errorf("cannot parse object ID: %v", err) + continue + } + + reader, err := mgr.Open(objectID) + if err == nil || reader != nil { + t.Errorf("expected error for %x", c) + } + } +} + func TestReaderStoredBlockNotFound(t *testing.T) { _, mgr := setupTest(t) @@ -456,6 +479,7 @@ func TestFormats(t *testing.T) { data := map[string][]byte{} st := blob.NewMapStorage(data) + t.Logf("verifying %#v", c.format) mgr, err := NewObjectManager(st, c.format) if err != nil { t.Errorf("cannot create manager: %v", err) @@ -463,8 +487,9 @@ func TestFormats(t *testing.T) { } for k, v := range c.oids { + bytesToWrite := []byte(k) w := mgr.NewWriter() - w.Write([]byte(k)) + w.Write(bytesToWrite) oid, err := w.Result(true) if err != nil { t.Errorf("error: %v", err) @@ -472,6 +497,80 @@ func TestFormats(t *testing.T) { if oid != v { t.Errorf("invalid oid for %v/%v: %v expected %v", c.format.Hash, k, oid, v) } + + rc, err := mgr.Open(oid) + if err != nil { + t.Errorf("open failed: %v", err) + continue + } + bytesRead, err := ioutil.ReadAll(rc) + if err != nil { + t.Errorf("error reading: %v", err) + } + if !bytes.Equal(bytesRead, bytesToWrite) { + t.Errorf("data mismatch, read:%x vs written:%v", bytesRead, bytesToWrite) + } } } } + +func TestInvalidEncryptionKey(t *testing.T) { + data := map[string][]byte{} + st := blob.NewMapStorage(data) + format := Format{ + Version: "1", + Hash: "hmac-sha512", + Secret: []byte("key"), + Encryption: "aes-256", + } + + mgr, err := NewObjectManager(st, format) + if err != nil { + t.Errorf("cannot create manager: %v", err) + } + + bytesToWrite := make([]byte, 1024) + for i := range bytesToWrite { + bytesToWrite[i] = byte(i) + } + + w := mgr.NewWriter() + w.Write(bytesToWrite) + oid, err := w.Result(true) + if err != nil { + t.Errorf("error: %v", err) + } + + rc, err := mgr.Open(oid) + if err != nil || rc == nil { + t.Errorf("error opening valid ObjectID: %v", err) + return + } + + // Key too short + rc, err = mgr.Open(oid[0 : len(oid)-2]) + if err == nil || rc != nil { + t.Errorf("expected error when opening malformed object") + } + + // Key too long + rc, err = mgr.Open(oid + "ff") + if err == nil || rc != nil { + t.Errorf("expected error when opening malformed object") + } + + // Invalid key + lastByte, _ := hex.DecodeString(string(oid[len(oid)-2:])) + lastByte[0]++ + rc, err = mgr.Open(oid[0:len(oid)-2] + ObjectID(hex.EncodeToString(lastByte))) + if err == nil || rc != nil { + t.Errorf("expected error when opening malformed object: %v", err) + } + + // Now corrupt the data + data[string(oid.BlockID())][0] ^= 1 + rc, err = mgr.Open(oid) + if err == nil || rc != nil { + t.Errorf("expected error when opening object with corrupt data") + } +} diff --git a/cas/object_reader.go b/cas/object_reader.go index 346d6a2ef..97e318dbe 100644 --- a/cas/object_reader.go +++ b/cas/object_reader.go @@ -1,12 +1,8 @@ package cas import ( - "bufio" - "bytes" "fmt" "io" - "strconv" - "strings" "github.com/kopia/kopia/blob" ) @@ -156,80 +152,3 @@ func (r *objectReader) Seek(offset int64, whence int) (int64, error) { return r.currentPosition, nil } - -func (mgr *objectManager) newRawReader(objectID ObjectID) (io.ReadSeeker, error) { - inline := objectID.InlineData() - if inline != nil { - return bytes.NewReader(inline), nil - } - - blockID := objectID.BlockID() - payload, err := mgr.storage.GetBlock(blockID) - if err != nil { - return nil, err - } - - if objectID.EncryptionInfo().Mode() == ObjectEncryptionNone { - return bytes.NewReader(payload), nil - } - - return nil, nil -} - -func (mgr *objectManager) flattenListChunk( - seekTable []seekTableEntry, - listObjectID ObjectID, - rawReader io.Reader) ([]seekTableEntry, error) { - - scanner := bufio.NewScanner(rawReader) - - for scanner.Scan() { - c := scanner.Text() - comma := strings.Index(c, ",") - if comma <= 0 { - return nil, fmt.Errorf("unsupported entry '%v' in list '%s'", c, listObjectID) - } - - length, err := strconv.ParseInt(c[0:comma], 10, 64) - - objectID, err := ParseObjectID(c[comma+1:]) - if err != nil { - return nil, fmt.Errorf("unsupported entry '%v' in list '%s': %#v", c, listObjectID, err) - } - - switch objectID.Type() { - case ObjectIDTypeList: - subreader, err := mgr.newRawReader(objectID) - if err != nil { - return nil, err - } - - seekTable, err = mgr.flattenListChunk(seekTable, objectID, subreader) - if err != nil { - return nil, err - } - - case ObjectIDTypeStored: - var startOffset int64 - if len(seekTable) > 0 { - startOffset = seekTable[len(seekTable)-1].endOffset() - } else { - startOffset = 0 - } - - seekTable = append( - seekTable, - seekTableEntry{ - blockID: objectID.BlockID(), - startOffset: startOffset, - length: length, - }) - - default: - return nil, fmt.Errorf("unsupported entry '%v' in list '%v'", objectID, listObjectID) - - } - } - - return seekTable, nil -} diff --git a/cas/objectid.go b/cas/objectid.go index 68b43684c..df4517fad 100644 --- a/cas/objectid.go +++ b/cas/objectid.go @@ -4,7 +4,6 @@ "encoding/base64" "encoding/hex" "fmt" - "strconv" "strings" "unicode/utf8" @@ -18,52 +17,12 @@ // ObjectIDType describes the type of the chunk. type ObjectIDType string -// EncryptionMode specifies encryption mode used to encrypt an object. -type EncryptionMode byte - -// Supported encryption modes. -const ( - ObjectEncryptionNone EncryptionMode = iota - ObjectEncryptionModeAES256 - objectEncryptionMax - objectEncryptionInvalid -) - // ObjectEncryptionInfo represents encryption info associated with ObjectID. type ObjectEncryptionInfo string // NoEncryption indicates that the object is not encrypted. var NoEncryption = ObjectEncryptionInfo("") -// Mode returns EncryptionMode for the object. -func (oei ObjectEncryptionInfo) Mode() EncryptionMode { - if len(oei) == 0 { - return ObjectEncryptionNone - } - - if len(oei)%2 != 0 { - return objectEncryptionInvalid - } - - v, err := strconv.ParseInt(string(oei[0:2]), 16, 8) - if err != nil { - return objectEncryptionInvalid - } - - m := EncryptionMode(v) - switch m { - case ObjectEncryptionModeAES256: - if len(oei) != 66 { - return objectEncryptionInvalid - } - - default: - return objectEncryptionInvalid - } - - return m -} - const ( // ObjectIDTypeText represents text-only inline object ID ObjectIDTypeText ObjectIDType = "T" @@ -195,13 +154,9 @@ func ParseObjectID(objectIDString string) (ObjectID, error) { if firstColon > 0 { b, err := hex.DecodeString(content[firstColon+1:]) - if err == nil && len(b) > 0 { + if err == nil && len(b) > 0 && len(b)%2 == 0 { // Valid chunk ID with encryption info. - oid := ObjectID(objectIDString) - - if oid.EncryptionInfo().Mode() < objectEncryptionMax { - return oid, nil - } + return ObjectID(objectIDString), nil } } } diff --git a/cas/objectid_test.go b/cas/objectid_test.go index 292fcec66..819156b70 100644 --- a/cas/objectid_test.go +++ b/cas/objectid_test.go @@ -49,12 +49,12 @@ func TestParseObjectIDEncryptionInfo(t *testing.T) { {"Cabcdef", NoEncryption}, {"Labcdef", NoEncryption}, { - "Cabcdef:0100112233445566778899aabbccddeeff00112233445566778899aabbccddeeff", - ObjectEncryptionInfo("0100112233445566778899aabbccddeeff00112233445566778899aabbccddeeff"), + "Cabcdef:00112233445566778899aabbccddeeff00112233445566778899aabbccddeeff", + ObjectEncryptionInfo("00112233445566778899aabbccddeeff00112233445566778899aabbccddeeff"), }, { - "Labcdef:0100112233445566778899aabbccddeeff00112233445566778899aabbccddeeff", - ObjectEncryptionInfo("0100112233445566778899aabbccddeeff00112233445566778899aabbccddeeff"), + "Labcdef:00112233445566778899aabbccddeeff00112233445566778899aabbccddeeff", + ObjectEncryptionInfo("00112233445566778899aabbccddeeff00112233445566778899aabbccddeeff"), }, } diff --git a/fs/dir.go b/fs/dir.go index cf8a1aad5..8aa50c63e 100644 --- a/fs/dir.go +++ b/fs/dir.go @@ -6,6 +6,7 @@ ) // Directory represents contents of a directory. +// All entries are sorted by name. type Directory struct { Entries []*Entry } diff --git a/session/session.go b/session/session.go new file mode 100644 index 000000000..0756d151e --- /dev/null +++ b/session/session.go @@ -0,0 +1,100 @@ +package session + +import ( + "bytes" + "encoding/json" + "fmt" + "io" + "io/ioutil" + + "github.com/kopia/kopia/auth" + "github.com/kopia/kopia/blob" + "github.com/kopia/kopia/cas" +) + +type Session interface { + io.Closer + + InitObjectManager(f cas.Format) (cas.ObjectManager, error) + OpenObjectManager() (cas.ObjectManager, error) +} + +type session struct { + storage blob.Storage + creds auth.Credentials + format cas.Format +} + +func (s *session) Close() error { + return nil +} + +func (s *session) getPrivateBlock(blkID blob.BlockID) ([]byte, error) { + b, err := s.storage.GetBlock(blkID) + if err != nil { + return nil, fmt.Errorf("unable to read block %v: %v", blkID, err) + } + + return b, err +} + +func (s *session) encryptBlockWithPublicKey(blkID blob.BlockID, data io.ReadCloser, options blob.PutOptions) error { + err := s.storage.PutBlock(blkID, data, options) + if err != nil { + return fmt.Errorf("unable to write block %v: %v", blkID, err) + } + + return err +} + +func (s *session) getConfigBlockID() blob.BlockID { + if s.creds == nil { + return blob.BlockID("config.json") + } + + return blob.BlockID("users." + s.creds.Username() + ".config.json") +} + +func (s *session) InitObjectManager(format cas.Format) (cas.ObjectManager, error) { + mgr, err := cas.NewObjectManager(s.storage, format) + if err != nil { + return nil, err + } + + b, err := json.Marshal(format) + if err != nil { + return nil, err + } + + if err := s.encryptBlockWithPublicKey( + s.getConfigBlockID(), + ioutil.NopCloser(bytes.NewBuffer(b)), + blob.PutOptions{}); err != nil { + return nil, err + } + + return mgr, nil +} + +func (s *session) OpenObjectManager() (cas.ObjectManager, error) { + b, err := s.getPrivateBlock(s.getConfigBlockID()) + if err != nil { + return nil, err + } + + var format cas.Format + err = json.Unmarshal(b, &format) + if err != nil { + return nil, err + } + + return cas.NewObjectManager(s.storage, format) +} + +func New(storage blob.Storage, creds auth.Credentials) (Session, error) { + sess := &session{ + storage: storage, + creds: creds, + } + return sess, nil +} diff --git a/session/session_test.go b/session/session_test.go new file mode 100644 index 000000000..dc9502ba8 --- /dev/null +++ b/session/session_test.go @@ -0,0 +1,51 @@ +package session + +import ( + "io/ioutil" + + "github.com/kopia/kopia/cas" + + "github.com/kopia/kopia/blob" + + "testing" +) + +func TestA(t *testing.T) { + tmpDir, err := ioutil.TempDir("", "kopia") + if err != nil { + t.Errorf("can't create temp directory: %v", err) + return + } + + // cfg := LoadConfig("kopia.config") + sc := blob.StorageConfiguration{ + Type: "fs", + Config: &blob.FSStorageOptions{ + Path: tmpDir, + }, + } + + storage, err := blob.NewStorage(sc) + if err != nil { + t.Errorf("cannot create storage: %v", err) + return + } + + sess, err := New(storage, nil) + defer sess.Close() + + om, err := sess.InitObjectManager(cas.Format{ + Version: "1", + Hash: "sha1", + }) + + if err != nil { + t.Errorf("unable to init object manager: %v", err) + return + } + + w := om.NewWriter() + w.Write([]byte{1, 2, 3}) + x, err := w.Result(true) + t.Logf("%v x: %v %v", tmpDir, x, err) +} diff --git a/user/doc.go b/user/doc.go new file mode 100644 index 000000000..a00006b65 --- /dev/null +++ b/user/doc.go @@ -0,0 +1 @@ +package user