From beb39e13457b7e86bf67e0810049cb0715513b9a Mon Sep 17 00:00:00 2001 From: Jarek Kowalski Date: Tue, 22 Mar 2016 22:21:43 -0700 Subject: [PATCH] fixed inline encoding --- Makefile | 10 +++++++++- auth/credentials.go | 5 +++++ auth/keys.go | 12 +++++++---- cas/format.go | 3 ++- cas/object_manager.go | 16 +-------------- cas/object_manager_test.go | 33 +++++++++++++++--------------- cas/object_writer.go | 24 ++-------------------- content/objectid.go | 41 ++++++++++++++++++++++++++++---------- content/objectid_test.go | 2 +- storage/repository.go | 13 ++---------- 10 files changed, 77 insertions(+), 82 deletions(-) diff --git a/Makefile b/Makefile index 8cc017b76..bb7f1a0fc 100644 --- a/Makefile +++ b/Makefile @@ -1,5 +1,13 @@ +all: lint vet build + build: - go install github.com/kopia/kopia/cmd/kopia + go build github.com/kopia/kopia/... + +lint: + golint github.com/kopia/kopia/... + +vet: + go vet github.com/kopia/kopia/... deps: go get -u -t -v github.com/kopia/kopia/... diff --git a/auth/credentials.go b/auth/credentials.go index a23b94a63..5f7564406 100644 --- a/auth/credentials.go +++ b/auth/credentials.go @@ -8,6 +8,7 @@ import ( "golang.org/x/crypto/pbkdf2" ) +// Credentials encapsulates user credentials. type Credentials interface { Username() string PrivateKey() *UserPrivateKey @@ -46,6 +47,7 @@ func (pc *credentials) deriveKeyFromPassword() { pc.privateKey = pk } +// Password returns Credentials object with static username and password. func Password(username, password string) Credentials { return &credentials{ username: username, @@ -55,6 +57,7 @@ func Password(username, password string) Credentials { } } +// PasswordPrompt returns Credentials object that will prompt user for password using the specified callback function. func PasswordPrompt(username string, prompt func() string) Credentials { return &credentials{ username: username, @@ -62,6 +65,7 @@ func PasswordPrompt(username string, prompt func() string) Credentials { } } +// Key returns Credentials object with specified username and key bytes. func Key(username string, key []byte) (Credentials, error) { pk, err := newPrivateKey(key) if err != nil { @@ -74,6 +78,7 @@ func Key(username string, key []byte) (Credentials, error) { }, nil } +// KeyFromFile returns Credentials object with specified username and with key read from the specified file. func KeyFromFile(username string, fileName string) (Credentials, error) { k, err := ioutil.ReadFile(fileName) if err != nil { diff --git a/auth/keys.go b/auth/keys.go index 544e4cd75..524b3d94e 100644 --- a/auth/keys.go +++ b/auth/keys.go @@ -11,16 +11,24 @@ const ( masterKeySize = 32 ) +// UserPrivateKey encapsulates secret key belonging to a user. type UserPrivateKey struct { key [32]byte } +// UserPublicKey encapsulates public key belonging to a user. +type UserPublicKey struct { + key [32]byte +} + +// Bytes returns the private key bytes. func (prv UserPrivateKey) Bytes() []byte { r := make([]byte, 32) copy(r, prv.key[:]) return r } +// PublicKey returns public key associated with the private key. func (prv UserPrivateKey) PublicKey() *UserPublicKey { pub := &UserPublicKey{} @@ -37,7 +45,3 @@ func newPrivateKey(key []byte) (*UserPrivateKey, error) { copy(k.key[:], key) return k, nil } - -type UserPublicKey struct { - key [32]byte -} diff --git a/cas/format.go b/cas/format.go index 7fa142cc3..78419be1c 100644 --- a/cas/format.go +++ b/cas/format.go @@ -2,7 +2,8 @@ package cas // Format describes the format of object data. type Format struct { - Algorithm string `json:"algorithm"` + Hash string `json:"hash"` + Encryption string `json:"encryption"` Secret []byte `json:"secret,omitempty"` MaxInlineBlobSize int `json:"maxInlineBlobSize"` MaxBlobSize int `json:"maxBlobSize"` diff --git a/cas/object_manager.go b/cas/object_manager.go index b34be2308..9fda5b837 100644 --- a/cas/object_manager.go +++ b/cas/object_manager.go @@ -121,20 +121,6 @@ func EnableLogging() ObjectManagerOption { } } -func MaxInlineBlobSize(sizeBytes int) ObjectManagerOption { - return func(o *objectManager) error { - o.maxInlineBlobSize = sizeBytes - return nil - } -} - -func MaxBlobSize(sizeBytes int) ObjectManagerOption { - return func(o *objectManager) error { - o.maxBlobSize = sizeBytes - return nil - } -} - // NewObjectManager creates new ObjectManager with the specified repository, options, and key provider. func NewObjectManager( r storage.Repository, @@ -153,7 +139,7 @@ func NewObjectManager( var hashFunc func() hash.Hash - hashAlgo := f.Algorithm + hashAlgo := f.Hash hf := strings.TrimPrefix(hashAlgo, "hmac-") switch hf { diff --git a/cas/object_manager_test.go b/cas/object_manager_test.go index dd44dbab6..9ab8a8dbb 100644 --- a/cas/object_manager_test.go +++ b/cas/object_manager_test.go @@ -20,7 +20,7 @@ func testFormat() Format { return Format{ MaxBlobSize: 200, MaxInlineBlobSize: 20, - Algorithm: "md5", + Hash: "md5", } } @@ -56,8 +56,9 @@ func TestWriters(t *testing.T) { }{ {[]byte{}, "B"}, {[]byte("quick brown fox"), "Tquick brown fox"}, - {[]byte{1, 2, 3, 4}, "BAQIDBA=="}, - {[]byte{0xc2, 0x28}, "Bwig="}, // invalid UTF-8, will be represented as binary + {[]byte{1, 2, 3, 4}, "BAQIDBA"}, + {[]byte{10, 13, 9}, "T\n\r\t"}, + {[]byte{0xc2, 0x28}, "Bwig"}, // invalid UTF-8, will be represented as binary { []byte("the quick brown fox jumps over the lazy dog"), "CX77add1d5f41223d5582fca736a5cb335", @@ -234,7 +235,7 @@ func TestHMAC(t *testing.T) { content := bytes.Repeat([]byte{0xcd}, 50) s := testFormat() - s.Algorithm = "hmac-md5" + s.Hash = "hmac-md5" s.Secret = []byte{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25} mgr, err := NewObjectManager(storage.NewMapRepository(data), s) @@ -260,7 +261,7 @@ func TestReader(t *testing.T) { payload []byte }{ {"B", []byte{}}, - {"BAQIDBA==", []byte{1, 2, 3, 4}}, + {"BAQIDBA", []byte{1, 2, 3, 4}}, {"T", []byte{}}, {"Tfoo\nbar", []byte("foo\nbar")}, {"Cabcdef", storedPayload}, @@ -358,7 +359,7 @@ func TestFormats(t *testing.T) { }{ { format: Format{ - Algorithm: "md5", + Hash: "md5", }, hashes: map[string]content.ObjectID{ "": "Cd41d8cd98f00b204e9800998ecf8427e", @@ -367,7 +368,7 @@ func TestFormats(t *testing.T) { }, { format: Format{ - Algorithm: "hmac-md5", + Hash: "hmac-md5", }, hashes: map[string]content.ObjectID{ "": "C74e6f7298a9c2d168935f58c001bad88", @@ -376,8 +377,8 @@ func TestFormats(t *testing.T) { }, { format: Format{ - Algorithm: "hmac-md5", - Secret: []byte("key"), + Hash: "hmac-md5", + Secret: []byte("key"), }, hashes: map[string]content.ObjectID{ "The quick brown fox jumps over the lazy dog": "C80070713463e7749b90c2dc24911e275", @@ -385,8 +386,8 @@ func TestFormats(t *testing.T) { }, { format: Format{ - Algorithm: "hmac-sha1", - Secret: []byte("key"), + Hash: "hmac-sha1", + Secret: []byte("key"), }, hashes: map[string]content.ObjectID{ "The quick brown fox jumps over the lazy dog": "Cde7c9b85b8b78aa6bc8a7a36f70a90701c9db4d9", @@ -394,8 +395,8 @@ func TestFormats(t *testing.T) { }, { format: Format{ - Algorithm: "hmac-sha256", - Secret: []byte("key"), + Hash: "hmac-sha256", + Secret: []byte("key"), }, hashes: map[string]content.ObjectID{ "The quick brown fox jumps over the lazy dog": "Cf7bc83f430538424b13298e6aa6fb143ef4d59a14946175997479dbc2d1a3cd8", @@ -403,8 +404,8 @@ func TestFormats(t *testing.T) { }, { format: Format{ - Algorithm: "hmac-sha512", - Secret: []byte("key"), + Hash: "hmac-sha512", + Secret: []byte("key"), }, hashes: map[string]content.ObjectID{ "The quick brown fox jumps over the lazy dog": "Cb42af09057bac1e2d41708e48a902e09b5ff7f12ab428a4fe86653c73dd248fb82f948a549f7b791a5b41915ee4d1ec3935357e4e2317250d0372afa2ebeeb3a", @@ -430,7 +431,7 @@ func TestFormats(t *testing.T) { t.Errorf("error: %v", err) } if oid != v { - t.Errorf("invalid oid for %v/%v: %v expected %v", c.format.Algorithm, k, oid, v) + t.Errorf("invalid oid for %v/%v: %v expected %v", c.format.Hash, k, oid, v) } } } diff --git a/cas/object_writer.go b/cas/object_writer.go index 6ef2b5080..37503daa0 100644 --- a/cas/object_writer.go +++ b/cas/object_writer.go @@ -2,12 +2,9 @@ package cas import ( "bytes" - "encoding/base64" - "encoding/json" "fmt" "io" "io/ioutil" - "unicode/utf8" "github.com/kopia/kopia/content" "github.com/kopia/kopia/storage" @@ -148,28 +145,11 @@ func newObjectWriter(cfg objectWriterConfig, objectType content.ObjectIDType) *o func (w *objectWriter) Result(forceStored bool) (content.ObjectID, error) { if !forceStored && w.flushedObjectCount == 0 { if w.buffer == nil { - return content.NewInlineBinaryObjectID(nil), nil + return "B", nil } if w.buffer.Len() < w.mgr.maxInlineBlobSize { - data := w.buffer.Bytes() - if !utf8.Valid(data) { - return content.NewInlineBinaryObjectID(data), nil - } - - // If the binary represents valid UTF8, try encoding it as text (JSON) or binary chunk - // and pick the one that has shorter representation. - dataString := string(data) - jsonData, _ := json.Marshal(dataString) - - jsonLen := len(jsonData) - base64Len := base64.StdEncoding.EncodedLen(len(data)) - - if jsonLen < base64Len { - return content.NewInlineTextObjectID(dataString), nil - } - - return content.NewInlineBinaryObjectID(data), nil + return content.NewInlineObjectID(w.buffer.Bytes()), nil } } diff --git a/content/objectid.go b/content/objectid.go index 69a1b6644..4220f4b8e 100644 --- a/content/objectid.go +++ b/content/objectid.go @@ -6,17 +6,22 @@ import ( "fmt" "strconv" "strings" + "unicode/utf8" "github.com/kopia/kopia/storage" ) +var ( + binaryEncoding = base64.RawURLEncoding +) + // ObjectIDType describes the type of the chunk. type ObjectIDType string -var NoEncryption = ObjectEncryptionInfo("") - +// EncryptionMode specifies encryption mode used to encrypt an object. type EncryptionMode byte +// Supported encryption modes. const ( ObjectEncryptionNone EncryptionMode = iota ObjectEncryptionModeAES256 @@ -24,8 +29,13 @@ const ( objectEncryptionInvalid ) +// ObjectEncryptionInfo represents encryption info associated with ObjectID. type ObjectEncryptionInfo string +// NoEncryption indicates that the object is not encrypted. +var NoEncryption = ObjectEncryptionInfo("") + +// Mode returns EncryptionMode for the object. func (oei ObjectEncryptionInfo) Mode() EncryptionMode { if len(oei) == 0 { return ObjectEncryptionNone @@ -106,7 +116,7 @@ func (c ObjectID) InlineData() []byte { return []byte(payload) case ObjectIDTypeBinary: - decodedPayload, err := base64.StdEncoding.DecodeString(payload) + decodedPayload, err := binaryEncoding.DecodeString(payload) if err == nil { return decodedPayload } @@ -122,14 +132,15 @@ func (c ObjectID) BlockID() storage.BlockID { firstColon := strings.Index(content, ":") if firstColon > 0 { return storage.BlockID(content[0:firstColon]) - } else { - return storage.BlockID(content) } + + return storage.BlockID(content) } return "" } +// EncryptionInfo returns ObjectEncryptionInfo for the ObjectID. func (c ObjectID) EncryptionInfo() ObjectEncryptionInfo { if c.Type().IsStored() { content := string(c[1:]) @@ -142,12 +153,20 @@ func (c ObjectID) EncryptionInfo() ObjectEncryptionInfo { return NoEncryption } -func NewInlineBinaryObjectID(data []byte) ObjectID { - return ObjectID("B" + base64.StdEncoding.EncodeToString(data)) -} +// NewInlineObjectID returns new ObjectID containing inline binary or text-encoded data. +func NewInlineObjectID(data []byte) ObjectID { + if !utf8.Valid(data) { + return ObjectID("B" + binaryEncoding.EncodeToString(data)) + } -func NewInlineTextObjectID(text string) ObjectID { - return ObjectID("T" + text) + for _, b := range data { + if b < 32 && (b != 9 && b != 10 && b != 13) { + // Any other character indicates binary content. + return ObjectID("B" + binaryEncoding.EncodeToString(data)) + } + } + + return ObjectID("T" + string(data)) } // ParseObjectID converts the specified string into ObjectID. @@ -161,7 +180,7 @@ func ParseObjectID(objectIDString string) (ObjectID, error) { return ObjectID(objectIDString), nil case "B": - if _, err := base64.StdEncoding.DecodeString(content); err == nil { + if _, err := binaryEncoding.DecodeString(content); err == nil { return ObjectID(objectIDString), nil } diff --git a/content/objectid_test.go b/content/objectid_test.go index b9895bcc0..00ad264e1 100644 --- a/content/objectid_test.go +++ b/content/objectid_test.go @@ -41,7 +41,7 @@ func TestParseObjectIDEncryptionInfo(t *testing.T) { expected ObjectEncryptionInfo }{ {"B", NoEncryption}, - {"BAQIDBA==", NoEncryption}, + {"BAQIDBA", NoEncryption}, {"T", NoEncryption}, {"T:foo", NoEncryption}, {"Tfoo\nbar", NoEncryption}, diff --git a/storage/repository.go b/storage/repository.go index cdb257999..8ef33ed2d 100644 --- a/storage/repository.go +++ b/storage/repository.go @@ -14,24 +14,15 @@ type PutOptions struct { IgnoreLimits bool } -// Repository encapsulates storage for blocks of data. -type RepositoryWriter interface { +// Repository encapsulates API for connecting to blob storage +type Repository interface { // BlockExists determines whether the specified block existts. PutBlock(id BlockID, data io.ReadCloser, options PutOptions) error DeleteBlock(id BlockID) error Flush() error -} - -// Repository encapsulates storage for blocks of data. -type RepositoryReader interface { BlockExists(id BlockID) (bool, error) GetBlock(id BlockID) ([]byte, error) ListBlocks(prefix BlockID) chan (BlockMetadata) -} - -type Repository interface { - RepositoryReader - RepositoryWriter Configuration() RepositoryConfiguration }