mirror/kopia
1
0 Fork 0
mirror of https://github.com/kopia/kopia.git synced 2026-10-22 06:04:47 -04:00
Code Issues Packages Projects Releases Wiki Activity

fixed inline encoding

Browse Source
This commit is contained in:
Jarek Kowalski committed 2016-03-22 22:21:43 -07:00
1 parent be05ab5457
commit beb39e1345
10 files changed
+77 -82
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

Makefile
+9 -1
View File
@@ -1,5 +1,13 @@
all: lint vet build
build:
go install github.com/kopia/kopia/cmd/kopia
go build github.com/kopia/kopia/...
lint:
golint github.com/kopia/kopia/...
vet:
go vet github.com/kopia/kopia/...
deps:
go get -u -t -v github.com/kopia/kopia/...
auth/credentials.go
+5
View File
@@ -8,6 +8,7 @@ import (
"golang.org/x/crypto/pbkdf2"
)
// Credentials encapsulates user credentials.
type Credentials interface {
Username() string
PrivateKey() *UserPrivateKey
@@ -46,6 +47,7 @@ func (pc *credentials) deriveKeyFromPassword() {
pc.privateKey = pk
}
// Password returns Credentials object with static username and password.
func Password(username, password string) Credentials {
return &credentials{
username: username,
@@ -55,6 +57,7 @@ func Password(username, password string) Credentials {
}
}
// PasswordPrompt returns Credentials object that will prompt user for password using the specified callback function.
func PasswordPrompt(username string, prompt func() string) Credentials {
return &credentials{
username: username,
@@ -62,6 +65,7 @@ func PasswordPrompt(username string, prompt func() string) Credentials {
}
}
// Key returns Credentials object with specified username and key bytes.
func Key(username string, key []byte) (Credentials, error) {
pk, err := newPrivateKey(key)
if err != nil {
@@ -74,6 +78,7 @@ func Key(username string, key []byte) (Credentials, error) {
}, nil
}
// KeyFromFile returns Credentials object with specified username and with key read from the specified file.
func KeyFromFile(username string, fileName string) (Credentials, error) {
k, err := ioutil.ReadFile(fileName)
if err != nil {
auth/keys.go
+8 -4
View File
@@ -11,16 +11,24 @@ const (
masterKeySize = 32
)
// UserPrivateKey encapsulates secret key belonging to a user.
type UserPrivateKey struct {
key [32]byte
}
// UserPublicKey encapsulates public key belonging to a user.
type UserPublicKey struct {
key [32]byte
}
// Bytes returns the private key bytes.
func (prv UserPrivateKey) Bytes() []byte {
r := make([]byte, 32)
copy(r, prv.key[:])
return r
}
// PublicKey returns public key associated with the private key.
func (prv UserPrivateKey) PublicKey() *UserPublicKey {
pub := &UserPublicKey{}
@@ -37,7 +45,3 @@ func newPrivateKey(key []byte) (*UserPrivateKey, error) {
copy(k.key[:], key)
return k, nil
}
type UserPublicKey struct {
key [32]byte
}
cas/format.go
+2 -1
View File
@@ -2,7 +2,8 @@ package cas
// Format describes the format of object data.
type Format struct {
Algorithm string `json:"algorithm"`
Hash string `json:"hash"`
Encryption string `json:"encryption"`
Secret []byte `json:"secret,omitempty"`
MaxInlineBlobSize int `json:"maxInlineBlobSize"`
MaxBlobSize int `json:"maxBlobSize"`
cas/object_manager.go
+1 -15
View File
@@ -121,20 +121,6 @@ func EnableLogging() ObjectManagerOption {
}
}
func MaxInlineBlobSize(sizeBytes int) ObjectManagerOption {
return func(o *objectManager) error {
o.maxInlineBlobSize = sizeBytes
return nil
}
}
func MaxBlobSize(sizeBytes int) ObjectManagerOption {
return func(o *objectManager) error {
o.maxBlobSize = sizeBytes
return nil
}
}
// NewObjectManager creates new ObjectManager with the specified repository, options, and key provider.
func NewObjectManager(
r storage.Repository,
@@ -153,7 +139,7 @@ func NewObjectManager(
var hashFunc func() hash.Hash
hashAlgo := f.Algorithm
hashAlgo := f.Hash
hf := strings.TrimPrefix(hashAlgo, "hmac-")
switch hf {
cas/object_manager_test.go
+17 -16
View File
@@ -20,7 +20,7 @@ func testFormat() Format {
return Format{
MaxBlobSize: 200,
MaxInlineBlobSize: 20,
Algorithm: "md5",
Hash: "md5",
}
}
@@ -56,8 +56,9 @@ func TestWriters(t *testing.T) {
}{
{[]byte{}, "B"},
{[]byte("quick brown fox"), "Tquick brown fox"},
{[]byte{1, 2, 3, 4}, "BAQIDBA=="},
{[]byte{0xc2, 0x28}, "Bwig="}, // invalid UTF-8, will be represented as binary
{[]byte{1, 2, 3, 4}, "BAQIDBA"},
{[]byte{10, 13, 9}, "T\n\r\t"},
{[]byte{0xc2, 0x28}, "Bwig"}, // invalid UTF-8, will be represented as binary
{
[]byte("the quick brown fox jumps over the lazy dog"),
"CX77add1d5f41223d5582fca736a5cb335",
@@ -234,7 +235,7 @@ func TestHMAC(t *testing.T) {
content := bytes.Repeat([]byte{0xcd}, 50)
s := testFormat()
s.Algorithm = "hmac-md5"
s.Hash = "hmac-md5"
s.Secret = []byte{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25}
mgr, err := NewObjectManager(storage.NewMapRepository(data), s)
@@ -260,7 +261,7 @@ func TestReader(t *testing.T) {
payload []byte
}{
{"B", []byte{}},
{"BAQIDBA==", []byte{1, 2, 3, 4}},
{"BAQIDBA", []byte{1, 2, 3, 4}},
{"T", []byte{}},
{"Tfoo\nbar", []byte("foo\nbar")},
{"Cabcdef", storedPayload},
@@ -358,7 +359,7 @@ func TestFormats(t *testing.T) {
}{
{
format: Format{
Algorithm: "md5",
Hash: "md5",
},
hashes: map[string]content.ObjectID{
"": "Cd41d8cd98f00b204e9800998ecf8427e",
@@ -367,7 +368,7 @@ func TestFormats(t *testing.T) {
},
{
format: Format{
Algorithm: "hmac-md5",
Hash: "hmac-md5",
},
hashes: map[string]content.ObjectID{
"": "C74e6f7298a9c2d168935f58c001bad88",
@@ -376,8 +377,8 @@ func TestFormats(t *testing.T) {
},
{
format: Format{
Algorithm: "hmac-md5",
Secret: []byte("key"),
Hash: "hmac-md5",
Secret: []byte("key"),
},
hashes: map[string]content.ObjectID{
"The quick brown fox jumps over the lazy dog": "C80070713463e7749b90c2dc24911e275",
@@ -385,8 +386,8 @@ func TestFormats(t *testing.T) {
},
{
format: Format{
Algorithm: "hmac-sha1",
Secret: []byte("key"),
Hash: "hmac-sha1",
Secret: []byte("key"),
},
hashes: map[string]content.ObjectID{
"The quick brown fox jumps over the lazy dog": "Cde7c9b85b8b78aa6bc8a7a36f70a90701c9db4d9",
@@ -394,8 +395,8 @@ func TestFormats(t *testing.T) {
},
{
format: Format{
Algorithm: "hmac-sha256",
Secret: []byte("key"),
Hash: "hmac-sha256",
Secret: []byte("key"),
},
hashes: map[string]content.ObjectID{
"The quick brown fox jumps over the lazy dog": "Cf7bc83f430538424b13298e6aa6fb143ef4d59a14946175997479dbc2d1a3cd8",
@@ -403,8 +404,8 @@ func TestFormats(t *testing.T) {
},
{
format: Format{
Algorithm: "hmac-sha512",
Secret: []byte("key"),
Hash: "hmac-sha512",
Secret: []byte("key"),
},
hashes: map[string]content.ObjectID{
"The quick brown fox jumps over the lazy dog": "Cb42af09057bac1e2d41708e48a902e09b5ff7f12ab428a4fe86653c73dd248fb82f948a549f7b791a5b41915ee4d1ec3935357e4e2317250d0372afa2ebeeb3a",
@@ -430,7 +431,7 @@ func TestFormats(t *testing.T) {
t.Errorf("error: %v", err)
}
if oid != v {
t.Errorf("invalid oid for %v/%v: %v expected %v", c.format.Algorithm, k, oid, v)
t.Errorf("invalid oid for %v/%v: %v expected %v", c.format.Hash, k, oid, v)
}
}
}
cas/object_writer.go
+2 -22
View File
@@ -2,12 +2,9 @@ package cas
import (
"bytes"
"encoding/base64"
"encoding/json"
"fmt"
"io"
"io/ioutil"
"unicode/utf8"
"github.com/kopia/kopia/content"
"github.com/kopia/kopia/storage"
@@ -148,28 +145,11 @@ func newObjectWriter(cfg objectWriterConfig, objectType content.ObjectIDType) *o
func (w *objectWriter) Result(forceStored bool) (content.ObjectID, error) {
if !forceStored && w.flushedObjectCount == 0 {
if w.buffer == nil {
return content.NewInlineBinaryObjectID(nil), nil
return "B", nil
}
if w.buffer.Len() < w.mgr.maxInlineBlobSize {
data := w.buffer.Bytes()
if !utf8.Valid(data) {
return content.NewInlineBinaryObjectID(data), nil
}
// If the binary represents valid UTF8, try encoding it as text (JSON) or binary chunk
// and pick the one that has shorter representation.
dataString := string(data)
jsonData, _ := json.Marshal(dataString)
jsonLen := len(jsonData)
base64Len := base64.StdEncoding.EncodedLen(len(data))
if jsonLen < base64Len {
return content.NewInlineTextObjectID(dataString), nil
}
return content.NewInlineBinaryObjectID(data), nil
return content.NewInlineObjectID(w.buffer.Bytes()), nil
}
}
content/objectid.go
+30 -11
View File
@@ -6,17 +6,22 @@ import (
"fmt"
"strconv"
"strings"
"unicode/utf8"
"github.com/kopia/kopia/storage"
)
var (
binaryEncoding = base64.RawURLEncoding
)
// ObjectIDType describes the type of the chunk.
type ObjectIDType string
var NoEncryption = ObjectEncryptionInfo("")
// EncryptionMode specifies encryption mode used to encrypt an object.
type EncryptionMode byte
// Supported encryption modes.
const (
ObjectEncryptionNone EncryptionMode = iota
ObjectEncryptionModeAES256
@@ -24,8 +29,13 @@ const (
objectEncryptionInvalid
)
// ObjectEncryptionInfo represents encryption info associated with ObjectID.
type ObjectEncryptionInfo string
// NoEncryption indicates that the object is not encrypted.
var NoEncryption = ObjectEncryptionInfo("")
// Mode returns EncryptionMode for the object.
func (oei ObjectEncryptionInfo) Mode() EncryptionMode {
if len(oei) == 0 {
return ObjectEncryptionNone
@@ -106,7 +116,7 @@ func (c ObjectID) InlineData() []byte {
return []byte(payload)
case ObjectIDTypeBinary:
decodedPayload, err := base64.StdEncoding.DecodeString(payload)
decodedPayload, err := binaryEncoding.DecodeString(payload)
if err == nil {
return decodedPayload
}
@@ -122,14 +132,15 @@ func (c ObjectID) BlockID() storage.BlockID {
firstColon := strings.Index(content, ":")
if firstColon > 0 {
return storage.BlockID(content[0:firstColon])
} else {
return storage.BlockID(content)
}
return storage.BlockID(content)
}
return ""
}
// EncryptionInfo returns ObjectEncryptionInfo for the ObjectID.
func (c ObjectID) EncryptionInfo() ObjectEncryptionInfo {
if c.Type().IsStored() {
content := string(c[1:])
@@ -142,12 +153,20 @@ func (c ObjectID) EncryptionInfo() ObjectEncryptionInfo {
return NoEncryption
}
func NewInlineBinaryObjectID(data []byte) ObjectID {
return ObjectID("B" + base64.StdEncoding.EncodeToString(data))
}
// NewInlineObjectID returns new ObjectID containing inline binary or text-encoded data.
func NewInlineObjectID(data []byte) ObjectID {
if !utf8.Valid(data) {
return ObjectID("B" + binaryEncoding.EncodeToString(data))
}
func NewInlineTextObjectID(text string) ObjectID {
return ObjectID("T" + text)
for _, b := range data {
if b < 32 && (b != 9 && b != 10 && b != 13) {
// Any other character indicates binary content.
return ObjectID("B" + binaryEncoding.EncodeToString(data))
}
}
return ObjectID("T" + string(data))
}
// ParseObjectID converts the specified string into ObjectID.
@@ -161,7 +180,7 @@ func ParseObjectID(objectIDString string) (ObjectID, error) {
return ObjectID(objectIDString), nil
case "B":
if _, err := base64.StdEncoding.DecodeString(content); err == nil {
if _, err := binaryEncoding.DecodeString(content); err == nil {
return ObjectID(objectIDString), nil
}
content/objectid_test.go
+1 -1
View File
@@ -41,7 +41,7 @@ func TestParseObjectIDEncryptionInfo(t *testing.T) {
expected ObjectEncryptionInfo
}{
{"B", NoEncryption},
{"BAQIDBA==", NoEncryption},
{"BAQIDBA", NoEncryption},
{"T", NoEncryption},
{"T:foo", NoEncryption},
{"Tfoo\nbar", NoEncryption},
storage/repository.go
+2 -11
View File
@@ -14,24 +14,15 @@ type PutOptions struct {
IgnoreLimits bool
}
// Repository encapsulates storage for blocks of data.
type RepositoryWriter interface {
// Repository encapsulates API for connecting to blob storage
type Repository interface {
// BlockExists determines whether the specified block existts.
PutBlock(id BlockID, data io.ReadCloser, options PutOptions) error
DeleteBlock(id BlockID) error
Flush() error
}
// Repository encapsulates storage for blocks of data.
type RepositoryReader interface {
BlockExists(id BlockID) (bool, error)
GetBlock(id BlockID) ([]byte, error)
ListBlocks(prefix BlockID) chan (BlockMetadata)
}
type Repository interface {
RepositoryReader
RepositoryWriter
Configuration() RepositoryConfiguration
}