added missing data encryption/decryption and validation

cas/object_manager.go

@@ -1,6 +1,7 @@
 package cas
 
 import (
+	"bufio"
 	"bytes"
 	"crypto/aes"
 	"crypto/cipher"
@@ -10,10 +11,13 @@
 	"crypto/sha256"
 	"crypto/sha512"
 	"encoding/hex"
+	"errors"
 	"fmt"
 	"hash"
 	"io"
 	"io/ioutil"
+	"log"
+	"strconv"
 	"strings"
 	"sync/atomic"
 
@@ -41,9 +45,17 @@ type ObjectManager interface {
 
 // ObjectManagerStats exposes statistics about ObjectManager operation
 type ObjectManagerStats struct {
-	HashedBytes   int64
-	HashedBlocks  int32
-	UploadedBytes int64
+	HashedBytes  int64
+	HashedBlocks int32
+
+	BytesReadFromStorage  int64
+	BytesWrittenToStorage int64
+
+	EncryptedBytes int64
+	DecryptedBytes int64
+
+	InvalidBlobs int32
+	ValidBlobs   int32
 }
 
 type keygenFunc func([]byte) (key []byte, locator []byte)
@@ -222,25 +234,31 @@ func NewObjectManager(
 func splitHash(keySize int) keygenFunc {
 	return func(b []byte) ([]byte, []byte) {
 		p := len(b) - keySize
-		return b[p:], b[0:p]
+		return b[0:p], b[p:]
 	}
 }
 
+func (mgr *objectManager) hashBuffer(data []byte) ([]byte, []byte) {
+	h := mgr.hashFunc()
+	h.Write(data)
+	contentHash := h.Sum(nil)
+
+	if mgr.keygen != nil {
+		return mgr.keygen(contentHash)
+	}
+
+	return contentHash, nil
+}
+
 func (mgr *objectManager) hashBufferForWriting(buffer *bytes.Buffer, prefix string) (ObjectID, io.ReadCloser) {
 	var data []byte
 	if buffer != nil {
 		data = buffer.Bytes()
 	}
 
-	h := mgr.hashFunc()
-	h.Write(data)
-	contentHash := h.Sum(nil)
-
+	contentHash, cryptoKey := mgr.hashBuffer(data)
 	var objectID ObjectID
-	var cryptoKey []byte
-
-	if mgr.createCipher != nil {
-		cryptoKey, contentHash = mgr.keygen(contentHash)
+	if cryptoKey != nil {
 		objectID = ObjectID(prefix + hex.EncodeToString(contentHash) + ":" + hex.EncodeToString(cryptoKey))
 	} else {
 		objectID = ObjectID(prefix + hex.EncodeToString(contentHash))
@@ -253,19 +271,143 @@ func (mgr *objectManager) hashBufferForWriting(buffer *bytes.Buffer, prefix stri
 	}
 
 	readCloser := mgr.bufferManager.returnBufferOnClose(buffer)
+	readCloser = newCountingReader(readCloser, &mgr.stats.BytesWrittenToStorage)
 
 	if cryptoKey != nil {
 		c, err := mgr.createCipher(cryptoKey)
 		if err != nil {
-			panic("can't create cipher")
+			log.Printf("can't create cipher: %v", err)
+			panic("can't encrypt block")
 		}
 
 		// Since we're not sharing the key, all-zero IV is ok.
 		// We don't need to worry about separate MAC either, since hashing content produces object ID.
 		ctr := cipher.NewCTR(c, constantIV[0:c.BlockSize()])
 
-		readCloser = newEncryptingReader(readCloser, nil, ctr, nil)
+		readCloser = newCountingReader(
+			newEncryptingReader(readCloser, nil, ctr, nil),
+			&mgr.stats.EncryptedBytes)
 	}
 
 	return objectID, readCloser
 }
+
+func (mgr *objectManager) flattenListChunk(
+	seekTable []seekTableEntry,
+	listObjectID ObjectID,
+	rawReader io.Reader) ([]seekTableEntry, error) {
+
+	scanner := bufio.NewScanner(rawReader)
+
+	for scanner.Scan() {
+		c := scanner.Text()
+		comma := strings.Index(c, ",")
+		if comma <= 0 {
+			return nil, fmt.Errorf("unsupported entry '%v' in list '%s'", c, listObjectID)
+		}
+
+		length, err := strconv.ParseInt(c[0:comma], 10, 64)
+
+		objectID, err := ParseObjectID(c[comma+1:])
+		if err != nil {
+			return nil, fmt.Errorf("unsupported entry '%v' in list '%s': %#v", c, listObjectID, err)
+		}
+
+		switch objectID.Type() {
+		case ObjectIDTypeList:
+			subreader, err := mgr.newRawReader(objectID)
+			if err != nil {
+				return nil, err
+			}
+
+			seekTable, err = mgr.flattenListChunk(seekTable, objectID, subreader)
+			if err != nil {
+				return nil, err
+			}
+
+		case ObjectIDTypeStored:
+			var startOffset int64
+			if len(seekTable) > 0 {
+				startOffset = seekTable[len(seekTable)-1].endOffset()
+			} else {
+				startOffset = 0
+			}
+
+			seekTable = append(
+				seekTable,
+				seekTableEntry{
+					blockID:     objectID.BlockID(),
+					startOffset: startOffset,
+					length:      length,
+				})
+
+		default:
+			return nil, fmt.Errorf("unsupported entry '%v' in list '%v'", objectID, listObjectID)
+
+		}
+	}
+
+	return seekTable, nil
+}
+
+func (mgr *objectManager) newRawReader(objectID ObjectID) (io.ReadSeeker, error) {
+	inline := objectID.InlineData()
+	if inline != nil {
+		return bytes.NewReader(inline), nil
+	}
+
+	blockID := objectID.BlockID()
+	payload, err := mgr.storage.GetBlock(blockID)
+	if err != nil {
+		return nil, err
+	}
+
+	atomic.AddInt64(&mgr.stats.BytesReadFromStorage, int64(len(payload)))
+
+	if objectID.EncryptionInfo() == NoEncryption {
+		if err := mgr.verifyChecksum(payload, objectID.BlockID()); err != nil {
+			return nil, err
+		}
+		return bytes.NewReader(payload), nil
+	}
+
+	if mgr.createCipher == nil {
+		return nil, errors.New("encrypted object cannot be used with non-encrypted ObjectManager")
+	}
+
+	cryptoKey, err := hex.DecodeString(string(objectID.EncryptionInfo()))
+	if err != nil {
+		return nil, errors.New("malformed encryption key")
+	}
+
+	blockCipher, err := mgr.createCipher(cryptoKey)
+	if err != nil {
+		return nil, errors.New("cannot create cipher")
+	}
+
+	iv := constantIV[0:blockCipher.BlockSize()]
+	ctr := cipher.NewCTR(blockCipher, iv)
+	ctr.XORKeyStream(payload, payload)
+
+	// Since the encryption key is a function of data, we must be able to generate exactly the same key
+	// after decrypting the content. This serves as a checksum.
+	atomic.AddInt64(&mgr.stats.DecryptedBytes, int64(len(payload)))
+
+	if err := mgr.verifyChecksum(payload, objectID.BlockID()); err != nil {
+		return nil, err
+	}
+
+	return bytes.NewReader(payload), nil
+}
+
+func (mgr *objectManager) verifyChecksum(data []byte, blockID blob.BlockID) error {
+	payloadHash, _ := mgr.hashBuffer(data)
+	checksum := hex.EncodeToString(payloadHash)
+	if !strings.HasSuffix(string(blockID), checksum) {
+		atomic.AddInt32(&mgr.stats.InvalidBlobs, 1)
+		return fmt.Errorf("invalid checksum for blob: '%v'", blockID)
+	}
+
+	atomic.AddInt32(&mgr.stats.ValidBlobs, 1)
+	return nil
+}

cas/object_manager_test.go

@@ -254,7 +254,7 @@ func TestReader(t *testing.T) {
 	data, mgr := setupTest(t)
 
 	storedPayload := []byte("foo\nbar")
-	data["abcdef"] = storedPayload
+	data["a76999788386641a3ec798554f1fe7e6"] = storedPayload
 
 	cases := []struct {
 		text    string
@@ -264,7 +264,7 @@ func TestReader(t *testing.T) {
 		{"BAQIDBA", []byte{1, 2, 3, 4}},
 		{"T", []byte{}},
 		{"Tfoo\nbar", []byte("foo\nbar")},
-		{"Cabcdef", storedPayload},
+		{"Ca76999788386641a3ec798554f1fe7e6", storedPayload},
 	}
 
 	for _, c := range cases {
@@ -292,6 +292,29 @@ func TestReader(t *testing.T) {
 	}
 }
 
+func TestMalformedStoredData(t *testing.T) {
+	data, mgr := setupTest(t)
+
+	cases := [][]byte{
+		[]byte("foo\nba"),
+		[]byte("foo\nbar1"),
+	}
+
+	for _, c := range cases {
+		data["a76999788386641a3ec798554f1fe7e6"] = c
+		objectID, err := ParseObjectID("Ca76999788386641a3ec798554f1fe7e6")
+		if err != nil {
+			t.Errorf("cannot parse object ID: %v", err)
+			continue
+		}
+
+		reader, err := mgr.Open(objectID)
+		if err == nil || reader != nil {
+			t.Errorf("expected error for %x", c)
+		}
+	}
+}
+
 func TestReaderStoredBlockNotFound(t *testing.T) {
 	_, mgr := setupTest(t)
 
@@ -456,6 +479,7 @@ func TestFormats(t *testing.T) {
 		data := map[string][]byte{}
 		st := blob.NewMapStorage(data)
 
+		t.Logf("verifying %#v", c.format)
 		mgr, err := NewObjectManager(st, c.format)
 		if err != nil {
 			t.Errorf("cannot create manager: %v", err)
@@ -463,8 +487,9 @@ func TestFormats(t *testing.T) {
 		}
 
 		for k, v := range c.oids {
+			bytesToWrite := []byte(k)
 			w := mgr.NewWriter()
-			w.Write([]byte(k))
+			w.Write(bytesToWrite)
 			oid, err := w.Result(true)
 			if err != nil {
 				t.Errorf("error: %v", err)
@@ -472,6 +497,80 @@ func TestFormats(t *testing.T) {
 			if oid != v {
 				t.Errorf("invalid oid for %v/%v: %v expected %v", c.format.Hash, k, oid, v)
 			}
+
+			rc, err := mgr.Open(oid)
+			if err != nil {
+				t.Errorf("open failed: %v", err)
+				continue
+			}
+			bytesRead, err := ioutil.ReadAll(rc)
+			if err != nil {
+				t.Errorf("error reading: %v", err)
+			}
+			if !bytes.Equal(bytesRead, bytesToWrite) {
+				t.Errorf("data mismatch, read:%x vs written:%v", bytesRead, bytesToWrite)
+			}
 		}
 	}
 }
+
+func TestInvalidEncryptionKey(t *testing.T) {
+	data := map[string][]byte{}
+	st := blob.NewMapStorage(data)
+	format := Format{
+		Version:    "1",
+		Hash:       "hmac-sha512",
+		Secret:     []byte("key"),
+		Encryption: "aes-256",
+	}
+
+	mgr, err := NewObjectManager(st, format)
+	if err != nil {
+		t.Errorf("cannot create manager: %v", err)
+	}
+
+	bytesToWrite := make([]byte, 1024)
+	for i := range bytesToWrite {
+		bytesToWrite[i] = byte(i)
+	}
+
+	w := mgr.NewWriter()
+	w.Write(bytesToWrite)
+	oid, err := w.Result(true)
+	if err != nil {
+		t.Errorf("error: %v", err)
+	}
+
+	rc, err := mgr.Open(oid)
+	if err != nil || rc == nil {
+		t.Errorf("error opening valid ObjectID: %v", err)
+		return
+	}
+
+	// Key too short
+	rc, err = mgr.Open(oid[0 : len(oid)-2])
+	if err == nil || rc != nil {
+		t.Errorf("expected error when opening malformed object")
+	}
+
+	// Key too long
+	rc, err = mgr.Open(oid + "ff")
+	if err == nil || rc != nil {
+		t.Errorf("expected error when opening malformed object")
+	}
+
+	// Invalid key
+	lastByte, _ := hex.DecodeString(string(oid[len(oid)-2:]))
+	lastByte[0]++
+	rc, err = mgr.Open(oid[0:len(oid)-2] + ObjectID(hex.EncodeToString(lastByte)))
+	if err == nil || rc != nil {
+		t.Errorf("expected error when opening malformed object: %v", err)
+	}
+
+	// Now corrupt the data
+	data[string(oid.BlockID())][0] ^= 1
+	rc, err = mgr.Open(oid)
+	if err == nil || rc != nil {
+		t.Errorf("expected error when opening object with corrupt data")
+	}
+}

cas/object_reader.go

@@ -1,12 +1,8 @@
 package cas
 
 import (
-	"bufio"
-	"bytes"
 	"fmt"
 	"io"
-	"strconv"
-	"strings"
 
 	"github.com/kopia/kopia/blob"
 )
@@ -156,80 +152,3 @@ func (r *objectReader) Seek(offset int64, whence int) (int64, error) {
 
 	return r.currentPosition, nil
 }
-
-func (mgr *objectManager) newRawReader(objectID ObjectID) (io.ReadSeeker, error) {
-	inline := objectID.InlineData()
-	if inline != nil {
-		return bytes.NewReader(inline), nil
-	}
-
-	blockID := objectID.BlockID()
-	payload, err := mgr.storage.GetBlock(blockID)
-	if err != nil {
-		return nil, err
-	}
-
-	if objectID.EncryptionInfo().Mode() == ObjectEncryptionNone {
-		return bytes.NewReader(payload), nil
-	}
-
-	return nil, nil
-}
-
-func (mgr *objectManager) flattenListChunk(
-	seekTable []seekTableEntry,
-	listObjectID ObjectID,
-	rawReader io.Reader) ([]seekTableEntry, error) {
-
-	scanner := bufio.NewScanner(rawReader)
-
-	for scanner.Scan() {
-		c := scanner.Text()
-		comma := strings.Index(c, ",")
-		if comma <= 0 {
-			return nil, fmt.Errorf("unsupported entry '%v' in list '%s'", c, listObjectID)
-		}
-
-		length, err := strconv.ParseInt(c[0:comma], 10, 64)
-
-		objectID, err := ParseObjectID(c[comma+1:])
-		if err != nil {
-			return nil, fmt.Errorf("unsupported entry '%v' in list '%s': %#v", c, listObjectID, err)
-		}
-
-		switch objectID.Type() {
-		case ObjectIDTypeList:
-			subreader, err := mgr.newRawReader(objectID)
-			if err != nil {
-				return nil, err
-			}
-
-			seekTable, err = mgr.flattenListChunk(seekTable, objectID, subreader)
-			if err != nil {
-				return nil, err
-			}
-
-		case ObjectIDTypeStored:
-			var startOffset int64
-			if len(seekTable) > 0 {
-				startOffset = seekTable[len(seekTable)-1].endOffset()
-			} else {
-				startOffset = 0
-			}
-
-			seekTable = append(
-				seekTable,
-				seekTableEntry{
-					blockID:     objectID.BlockID(),
-					startOffset: startOffset,
-					length:      length,
-				})
-
-		default:
-			return nil, fmt.Errorf("unsupported entry '%v' in list '%v'", objectID, listObjectID)
-
-		}
-	}
-
-	return seekTable, nil
-}

cas/objectid.go

@@ -4,7 +4,6 @@
 	"encoding/base64"
 	"encoding/hex"
 	"fmt"
-	"strconv"
 	"strings"
 	"unicode/utf8"
 
@@ -18,52 +17,12 @@
 // ObjectIDType describes the type of the chunk.
 type ObjectIDType string
 
-// EncryptionMode specifies encryption mode used to encrypt an object.
-type EncryptionMode byte
-
-// Supported encryption modes.
-const (
-	ObjectEncryptionNone EncryptionMode = iota
-	ObjectEncryptionModeAES256
-	objectEncryptionMax
-	objectEncryptionInvalid
-)
-
 // ObjectEncryptionInfo represents encryption info associated with ObjectID.
 type ObjectEncryptionInfo string
 
 // NoEncryption indicates that the object is not encrypted.
 var NoEncryption = ObjectEncryptionInfo("")
 
-// Mode returns EncryptionMode for the object.
-func (oei ObjectEncryptionInfo) Mode() EncryptionMode {
-	if len(oei) == 0 {
-		return ObjectEncryptionNone
-	}
-
-	if len(oei)%2 != 0 {
-		return objectEncryptionInvalid
-	}
-
-	v, err := strconv.ParseInt(string(oei[0:2]), 16, 8)
-	if err != nil {
-		return objectEncryptionInvalid
-	}
-
-	m := EncryptionMode(v)
-	switch m {
-	case ObjectEncryptionModeAES256:
-		if len(oei) != 66 {
-			return objectEncryptionInvalid
-		}
-
-	default:
-		return objectEncryptionInvalid
-	}
-
-	return m
-}
-
 const (
 	// ObjectIDTypeText represents text-only inline object ID
 	ObjectIDTypeText ObjectIDType = "T"
@@ -195,13 +154,9 @@ func ParseObjectID(objectIDString string) (ObjectID, error) {
 
 				if firstColon > 0 {
 					b, err := hex.DecodeString(content[firstColon+1:])
-					if err == nil && len(b) > 0 {
+					if err == nil && len(b) > 0 && len(b)%2 == 0 {
 						// Valid chunk ID with encryption info.
-						oid := ObjectID(objectIDString)
-
-						if oid.EncryptionInfo().Mode() < objectEncryptionMax {
-							return oid, nil
-						}
+						return ObjectID(objectIDString), nil
 					}
 				}
 			}

cas/objectid_test.go

@@ -49,12 +49,12 @@ func TestParseObjectIDEncryptionInfo(t *testing.T) {
 		{"Cabcdef", NoEncryption},
 		{"Labcdef", NoEncryption},
 		{
-			"Cabcdef:0100112233445566778899aabbccddeeff00112233445566778899aabbccddeeff",
-			ObjectEncryptionInfo("0100112233445566778899aabbccddeeff00112233445566778899aabbccddeeff"),
+			"Cabcdef:00112233445566778899aabbccddeeff00112233445566778899aabbccddeeff",
+			ObjectEncryptionInfo("00112233445566778899aabbccddeeff00112233445566778899aabbccddeeff"),
 		},
 		{
-			"Labcdef:0100112233445566778899aabbccddeeff00112233445566778899aabbccddeeff",
-			ObjectEncryptionInfo("0100112233445566778899aabbccddeeff00112233445566778899aabbccddeeff"),
+			"Labcdef:00112233445566778899aabbccddeeff00112233445566778899aabbccddeeff",
+			ObjectEncryptionInfo("00112233445566778899aabbccddeeff00112233445566778899aabbccddeeff"),
 		},
 	}
 

fs/dir.go

@@ -6,6 +6,7 @@
 )
 
 // Directory represents contents of a directory.
+// All entries are sorted by name.
 type Directory struct {
 	Entries []*Entry
 }

session/session.go

@@ -0,0 +1,100 @@
+package session
+
+import (
+	"bytes"
+	"encoding/json"
+	"fmt"
+	"io"
+	"io/ioutil"
+
+	"github.com/kopia/kopia/auth"
+	"github.com/kopia/kopia/blob"
+	"github.com/kopia/kopia/cas"
+)
+
+type Session interface {
+	io.Closer
+
+	InitObjectManager(f cas.Format) (cas.ObjectManager, error)
+	OpenObjectManager() (cas.ObjectManager, error)
+}
+
+type session struct {
+	storage blob.Storage
+	creds   auth.Credentials
+	format  cas.Format
+}
+
+func (s *session) Close() error {
+	return nil
+}
+
+func (s *session) getPrivateBlock(blkID blob.BlockID) ([]byte, error) {
+	b, err := s.storage.GetBlock(blkID)
+	if err != nil {
+		return nil, fmt.Errorf("unable to read block %v: %v", blkID, err)
+	}
+
+	return b, err
+}
+
+func (s *session) encryptBlockWithPublicKey(blkID blob.BlockID, data io.ReadCloser, options blob.PutOptions) error {
+	err := s.storage.PutBlock(blkID, data, options)
+	if err != nil {
+		return fmt.Errorf("unable to write block %v: %v", blkID, err)
+	}
+
+	return err
+}
+
+func (s *session) getConfigBlockID() blob.BlockID {
+	if s.creds == nil {
+		return blob.BlockID("config.json")
+	}
+
+	return blob.BlockID("users." + s.creds.Username() + ".config.json")
+}
+
+func (s *session) InitObjectManager(format cas.Format) (cas.ObjectManager, error) {
+	mgr, err := cas.NewObjectManager(s.storage, format)
+	if err != nil {
+		return nil, err
+	}
+
+	b, err := json.Marshal(format)
+	if err != nil {
+		return nil, err
+	}
+
+	if err := s.encryptBlockWithPublicKey(
+		s.getConfigBlockID(),
+		ioutil.NopCloser(bytes.NewBuffer(b)),
+		blob.PutOptions{}); err != nil {
+		return nil, err
+	}
+
+	return mgr, nil
+}
+
+func (s *session) OpenObjectManager() (cas.ObjectManager, error) {
+	b, err := s.getPrivateBlock(s.getConfigBlockID())
+	if err != nil {
+		return nil, err
+	}
+
+	var format cas.Format
+	err = json.Unmarshal(b, &format)
+	if err != nil {
+		return nil, err
+	}
+
+	return cas.NewObjectManager(s.storage, format)
+}
+
+func New(storage blob.Storage, creds auth.Credentials) (Session, error) {
+	sess := &session{
+		storage: storage,
+		creds:   creds,
+	}
+	return sess, nil
+}

session/session_test.go

@@ -0,0 +1,51 @@
+package session
+
+import (
+	"io/ioutil"
+
+	"github.com/kopia/kopia/cas"
+
+	"github.com/kopia/kopia/blob"
+
+	"testing"
+)
+
+func TestA(t *testing.T) {
+	tmpDir, err := ioutil.TempDir("", "kopia")
+	if err != nil {
+		t.Errorf("can't create temp directory: %v", err)
+		return
+	}
+
+	// cfg := LoadConfig("kopia.config")
+	sc := blob.StorageConfiguration{
+		Type: "fs",
+		Config: &blob.FSStorageOptions{
+			Path: tmpDir,
+		},
+	}
+
+	storage, err := blob.NewStorage(sc)
+	if err != nil {
+		t.Errorf("cannot create storage: %v", err)
+		return
+	}
+
+	sess, err := New(storage, nil)
+	defer sess.Close()
+
+	om, err := sess.InitObjectManager(cas.Format{
+		Version: "1",
+		Hash:    "sha1",
+	})
+
+	if err != nil {
+		t.Errorf("unable to init object manager: %v", err)
+		return
+	}
+
+	w := om.NewWriter()
+	w.Write([]byte{1, 2, 3})
+	x, err := w.Result(true)
+	t.Logf("%v x: %v %v", tmpDir, x, err)
+}

user/doc.go

@@ -0,0 +1 @@
+package user