mirror/kopia
1
0
Fork 0
mirror of https://github.com/kopia/kopia.git synced 2026-05-24 14:44:47 -04:00
Code Issues Packages Projects Releases Wiki Activity

moved content.ObjectID to cas.ObjectID

Browse Source
This commit is contained in:
Jarek Kowalski
2016-04-01 18:33:29 -07:00
parent 73258cd92d
commit 4f4275c8a5
10 changed files with 56 additions and 63 deletions
Download Patch File Download Diff File Expand all files Collapse all files

12
cas/formatter.go
View File

@@ -11,27 +11,25 @@
"hash"
"io"
"sync/atomic"
"github.com/kopia/kopia/content"
)
type streamTransformer func(io.ReadCloser) io.ReadCloser
type objectFormatter interface {
Do(b []byte, prefix string, stats *ObjectManagerStats) (content.ObjectID, streamTransformer)
Do(b []byte, prefix string, stats *ObjectManagerStats) (ObjectID, streamTransformer)
}
type nonEncryptingFormatter struct {
hash func() hash.Hash
}
func (f *nonEncryptingFormatter) Do(b []byte, prefix string, stats *ObjectManagerStats) (content.ObjectID, streamTransformer) {
func (f *nonEncryptingFormatter) Do(b []byte, prefix string, stats *ObjectManagerStats) (ObjectID, streamTransformer) {
h := f.hash()
h.Write(b)
blockID := hex.EncodeToString(h.Sum(nil))
atomic.AddInt64(&stats.HashedBytes, int64(len(b)))
return content.ObjectID(prefix + blockID), func(r io.ReadCloser) io.ReadCloser { return r }
return ObjectID(prefix + blockID), func(r io.ReadCloser) io.ReadCloser { return r }
}
func newNonEncryptingFormatter(hash func() hash.Hash) objectFormatter {
@@ -44,7 +42,7 @@ type aesEncryptingFormatter struct {
masterContentSecret []byte
}
func (f *aesEncryptingFormatter) Do(b []byte, prefix string, stats *ObjectManagerStats) (content.ObjectID, streamTransformer) {
func (f *aesEncryptingFormatter) Do(b []byte, prefix string, stats *ObjectManagerStats) (ObjectID, streamTransformer) {
// Compute HMAC-SHA512 of the content
s := hmac.New(sha512.New, f.masterContentSecret)
s.Write(b)
@@ -53,7 +51,7 @@ func (f *aesEncryptingFormatter) Do(b []byte, prefix string, stats *ObjectManage
// Split the hash into two portions - encryption key and content ID.
aesKey := contentHash[0:32]
return content.ObjectID(prefix + hex.EncodeToString(contentHash[32:64]) + ".e"),
return ObjectID(prefix + hex.EncodeToString(contentHash[32:64]) + ".e"),
func(r io.ReadCloser) io.ReadCloser {
var iv [aes.BlockSize]byte
rand.Read(iv[:])

17
cas/object_manager.go
View File

@@ -17,7 +17,6 @@
"strings"
"sync/atomic"
"github.com/kopia/kopia/content"
"github.com/kopia/kopia/storage"
)
@@ -31,7 +30,7 @@ type ObjectManager interface {
NewWriter(options ...WriterOption) ObjectWriter
// Open creates an io.ReadSeeker for reading object with a specified ID.
Open(objectID content.ObjectID) (io.ReadSeeker, error)
Open(objectID ObjectID) (io.ReadSeeker, error)
Flush() error
Repository() storage.Repository
@@ -87,7 +86,7 @@ func (mgr *objectManager) NewWriter(options ...WriterOption) ObjectWriter {
mgr: mgr,
putOptions: storage.PutOptions{},
},
content.ObjectIDTypeStored)
ObjectIDTypeStored)
for _, option := range options {
option(result)
@@ -96,13 +95,13 @@ func (mgr *objectManager) NewWriter(options ...WriterOption) ObjectWriter {
return result
}
func (mgr *objectManager) Open(objectID content.ObjectID) (io.ReadSeeker, error) {
func (mgr *objectManager) Open(objectID ObjectID) (io.ReadSeeker, error) {
r, err := mgr.newRawReader(objectID)
if err != nil {
return nil, err
}
if objectID.Type() == content.ObjectIDTypeList {
if objectID.Type() == ObjectIDTypeList {
seekTable := make([]seekTableEntry, 0, 100)
seekTable, err = mgr.flattenListChunk(seekTable, objectID, r)
@@ -228,7 +227,7 @@ func splitHash(keySize int) keygenFunc {
}
}
func (mgr *objectManager) hashBufferForWriting(buffer *bytes.Buffer, prefix string) (content.ObjectID, io.ReadCloser) {
func (mgr *objectManager) hashBufferForWriting(buffer *bytes.Buffer, prefix string) (ObjectID, io.ReadCloser) {
var data []byte
if buffer != nil {
data = buffer.Bytes()
@@ -238,14 +237,14 @@ func (mgr *objectManager) hashBufferForWriting(buffer *bytes.Buffer, prefix stri
h.Write(data)
contentHash := h.Sum(nil)
var objectID content.ObjectID
var objectID ObjectID
var cryptoKey []byte
if mgr.createCipher != nil {
cryptoKey, contentHash = mgr.keygen(contentHash)
objectID = content.ObjectID(prefix + hex.EncodeToString(contentHash) + ":" + hex.EncodeToString(cryptoKey))
objectID = ObjectID(prefix + hex.EncodeToString(contentHash) + ":" + hex.EncodeToString(cryptoKey))
} else {
objectID = content.ObjectID(prefix + hex.EncodeToString(contentHash))
objectID = ObjectID(prefix + hex.EncodeToString(contentHash))
}
atomic.AddInt64(&mgr.stats.HashedBytes, int64(len(data)))

27
cas/object_manager_test.go
View File

@@ -12,7 +12,6 @@
"strings"
"testing"
"github.com/kopia/kopia/content"
"github.com/kopia/kopia/storage"
)
@@ -53,7 +52,7 @@ func setupTest(t *testing.T) (data map[string][]byte, mgr ObjectManager) {
func TestWriters(t *testing.T) {
cases := []struct {
data []byte
objectID content.ObjectID
objectID ObjectID
}{
{[]byte{}, "B"},
{[]byte("quick brown fox"), "Tquick brown fox"},
@@ -269,7 +268,7 @@ func TestReader(t *testing.T) {
}
for _, c := range cases {
objectID, err := content.ParseObjectID(c.text)
objectID, err := ParseObjectID(c.text)
if err != nil {
t.Errorf("cannot parse object ID: %v", err)
continue
@@ -296,7 +295,7 @@ func TestReader(t *testing.T) {
func TestReaderStoredBlockNotFound(t *testing.T) {
_, mgr := setupTest(t)
objectID, err := content.ParseObjectID("Cno-such-block")
objectID, err := ParseObjectID("Cno-such-block")
if err != nil {
t.Errorf("cannot parse object ID: %v", err)
}
@@ -356,14 +355,14 @@ func TestEndToEndReadAndSeek(t *testing.T) {
func TestFormats(t *testing.T) {
cases := []struct {
format Format
oids map[string]content.ObjectID
oids map[string]ObjectID
}{
{
format: Format{
Version: "1",
Hash: "md5",
},
oids: map[string]content.ObjectID{
oids: map[string]ObjectID{
"": "Cd41d8cd98f00b204e9800998ecf8427e",
"The quick brown fox jumps over the lazy dog": "C9e107d9d372bb6826bd81d3542a419d6",
},
@@ -373,7 +372,7 @@ func TestFormats(t *testing.T) {
Version: "1",
Hash: "hmac-md5",
},
oids: map[string]content.ObjectID{
oids: map[string]ObjectID{
"": "C74e6f7298a9c2d168935f58c001bad88",
"The quick brown fox jumps over the lazy dog": "Cad262969c53bc16032f160081c4a07a0",
},
@@ -384,7 +383,7 @@ func TestFormats(t *testing.T) {
Hash: "hmac-md5",
Secret: []byte("key"),
},
oids: map[string]content.ObjectID{
oids: map[string]ObjectID{
"The quick brown fox jumps over the lazy dog": "C80070713463e7749b90c2dc24911e275",
},
},
@@ -394,7 +393,7 @@ func TestFormats(t *testing.T) {
Hash: "hmac-sha1",
Secret: []byte("key"),
},
oids: map[string]content.ObjectID{
oids: map[string]ObjectID{
"The quick brown fox jumps over the lazy dog": "Cde7c9b85b8b78aa6bc8a7a36f70a90701c9db4d9",
},
},
@@ -404,7 +403,7 @@ func TestFormats(t *testing.T) {
Hash: "hmac-sha256",
Secret: []byte("key"),
},
oids: map[string]content.ObjectID{
oids: map[string]ObjectID{
"The quick brown fox jumps over the lazy dog": "Cf7bc83f430538424b13298e6aa6fb143ef4d59a14946175997479dbc2d1a3cd8",
},
},
@@ -414,7 +413,7 @@ func TestFormats(t *testing.T) {
Hash: "hmac-sha512",
Secret: []byte("key"),
},
oids: map[string]content.ObjectID{
oids: map[string]ObjectID{
"The quick brown fox jumps over the lazy dog": "Cb42af09057bac1e2d41708e48a902e09b5ff7f12ab428a4fe86653c73dd248fb82f948a549f7b791a5b41915ee4d1ec3935357e4e2317250d0372afa2ebeeb3a",
},
},
@@ -425,7 +424,7 @@ func TestFormats(t *testing.T) {
Secret: []byte("key"),
Encryption: "aes-128",
},
oids: map[string]content.ObjectID{
oids: map[string]ObjectID{
"The quick brown fox jumps over the lazy dog": "Cb42af09057bac1e2d41708e48a902e09b5ff7f12ab428a4fe86653c73dd248fb82f948a549f7b791a5b41915ee4d1ec3:935357e4e2317250d0372afa2ebeeb3a",
},
},
@@ -436,7 +435,7 @@ func TestFormats(t *testing.T) {
Secret: []byte("key"),
Encryption: "aes-192",
},
oids: map[string]content.ObjectID{
oids: map[string]ObjectID{
"The quick brown fox jumps over the lazy dog": "Cb42af09057bac1e2d41708e48a902e09b5ff7f12ab428a4fe86653c73dd248fb82f948a549f7b791:a5b41915ee4d1ec3935357e4e2317250d0372afa2ebeeb3a",
},
},
@@ -447,7 +446,7 @@ func TestFormats(t *testing.T) {
Secret: []byte("key"),
Encryption: "aes-256",
},
oids: map[string]content.ObjectID{
oids: map[string]ObjectID{
"The quick brown fox jumps over the lazy dog": "Cb42af09057bac1e2d41708e48a902e09b5ff7f12ab428a4fe86653c73dd248fb:82f948a549f7b791a5b41915ee4d1ec3935357e4e2317250d0372afa2ebeeb3a",
},
},

13
cas/object_reader.go
View File

@@ -8,7 +8,6 @@
"strconv"
"strings"
"github.com/kopia/kopia/content"
"github.com/kopia/kopia/storage"
)
@@ -162,7 +161,7 @@ func (r *objectReader) Seek(offset int64, whence int) (int64, error) {
return r.currentPosition, nil
}
func (mgr *objectManager) newRawReader(objectID content.ObjectID) (io.ReadSeeker, error) {
func (mgr *objectManager) newRawReader(objectID ObjectID) (io.ReadSeeker, error) {
inline := objectID.InlineData()
if inline != nil {
return bytes.NewReader(inline), nil
@@ -174,7 +173,7 @@ func (mgr *objectManager) newRawReader(objectID content.ObjectID) (io.ReadSeeker
return nil, err
}
if objectID.EncryptionInfo().Mode() == content.ObjectEncryptionNone {
if objectID.EncryptionInfo().Mode() == ObjectEncryptionNone {
return bytes.NewReader(payload), nil
}
@@ -183,7 +182,7 @@ func (mgr *objectManager) newRawReader(objectID content.ObjectID) (io.ReadSeeker
func (mgr *objectManager) flattenListChunk(
seekTable []seekTableEntry,
listObjectID content.ObjectID,
listObjectID ObjectID,
rawReader io.Reader) ([]seekTableEntry, error) {
scanner := bufio.NewScanner(rawReader)
@@ -197,13 +196,13 @@ func (mgr *objectManager) flattenListChunk(
length, err := strconv.ParseInt(c[0:comma], 10, 64)
objectID, err := content.ParseObjectID(c[comma+1:])
objectID, err := ParseObjectID(c[comma+1:])
if err != nil {
return nil, fmt.Errorf("unsupported entry '%v' in list '%s': %#v", c, listObjectID, err)
}
switch objectID.Type() {
case content.ObjectIDTypeList:
case ObjectIDTypeList:
subreader, err := mgr.newRawReader(objectID)
if err != nil {
return nil, err
@@ -214,7 +213,7 @@ func (mgr *objectManager) flattenListChunk(
return nil, err
}
case content.ObjectIDTypeStored:
case ObjectIDTypeStored:
var startOffset int64
if len(seekTable) > 0 {
startOffset = seekTable[len(seekTable)-1].endOffset()

17
cas/object_writer.go
View File

@@ -5,7 +5,6 @@
"fmt"
"io"
"github.com/kopia/kopia/content"
"github.com/kopia/kopia/storage"
)
@@ -18,7 +17,7 @@ type blockHasher interface {
type ObjectWriter interface {
io.WriteCloser
Result(forceStored bool) (content.ObjectID, error)
Result(forceStored bool) (ObjectID, error)
}
// objectWriterConfig
@@ -36,10 +35,10 @@ type objectWriter struct {
prefix string
listWriter *objectWriter
flushedObjectCount int
lastFlushedObject content.ObjectID
lastFlushedObject ObjectID
description string
objectType content.ObjectIDType
objectType ObjectIDType
atomicWrites bool
}
@@ -116,7 +115,7 @@ func (w *objectWriter) flushBuffer(force bool) error {
w.flushedObjectCount++
w.lastFlushedObject = objectID
if w.listWriter == nil {
w.listWriter = newObjectWriter(w.objectWriterConfig, content.ObjectIDTypeList)
w.listWriter = newObjectWriter(w.objectWriterConfig, ObjectIDTypeList)
w.listWriter.description = "LIST(" + w.description + ")"
w.listWriter.atomicWrites = true
}
@@ -126,21 +125,21 @@ func (w *objectWriter) flushBuffer(force bool) error {
return nil
}
func newObjectWriter(cfg objectWriterConfig, objectType content.ObjectIDType) *objectWriter {
func newObjectWriter(cfg objectWriterConfig, objectType ObjectIDType) *objectWriter {
return &objectWriter{
objectWriterConfig: cfg,
objectType: objectType,
}
}
func (w *objectWriter) Result(forceStored bool) (content.ObjectID, error) {
func (w *objectWriter) Result(forceStored bool) (ObjectID, error) {
if !forceStored && w.flushedObjectCount == 0 {
if w.buffer == nil {
return "B", nil
}
if w.buffer.Len() < w.mgr.maxInlineBlobSize {
return content.NewInlineObjectID(w.buffer.Bytes()), nil
return NewInlineObjectID(w.buffer.Bytes()), nil
}
}
@@ -154,7 +153,7 @@ func (w *objectWriter) Result(forceStored bool) (content.ObjectID, error) {
if w.flushedObjectCount == 1 {
return w.lastFlushedObject, nil
} else if w.flushedObjectCount == 0 {
return content.NullObjectID, nil
return NullObjectID, nil
} else {
return w.listWriter.Result(true)
}

2
content/objectid.go → cas/objectid.go
View File

@@ -1,4 +1,4 @@
package content
package cas
import (
"encoding/base64"

2
content/objectid_test.go → cas/objectid_test.go
View File

@@ -1,4 +1,4 @@
package content
package cas
import (
"strings"

4
dir/entry.go
View File

@@ -5,7 +5,7 @@
"os"
"time"
"github.com/kopia/kopia/content"
"github.com/kopia/kopia/cas"
)
// EntryType describes the type of an backup entry.
@@ -69,7 +69,7 @@ type EntryMetadata struct {
type Entry struct {
EntryMetadata
ObjectID content.ObjectID
ObjectID cas.ObjectID
}
func (e *Entry) metadataEquals(other *Entry) bool {

21
dir/upload.go
View File

@@ -10,7 +10,6 @@
"sync/atomic"
"github.com/kopia/kopia/cas"
"github.com/kopia/kopia/content"
)
// ErrUploadCancelled is returned when the upload gets cancelled.
@@ -18,8 +17,8 @@
// Uploader supports efficient uploading files and directories to CAS storage.
type Uploader interface {
UploadFile(path string) (content.ObjectID, error)
UploadDir(path string, previousObjectID content.ObjectID) (content.ObjectID, error)
UploadFile(path string) (cas.ObjectID, error)
UploadDir(path string, previousObjectID cas.ObjectID) (cas.ObjectID, error)
Cancel()
}
@@ -34,10 +33,10 @@ func (u *uploader) isCancelled() bool {
return atomic.LoadInt32(&u.cancelled) != 0
}
func (u *uploader) UploadFile(path string) (content.ObjectID, error) {
func (u *uploader) UploadFile(path string) (cas.ObjectID, error) {
file, err := os.Open(path)
if err != nil {
return content.NullObjectID, fmt.Errorf("unable to open file %s: %v", path, err)
return cas.NullObjectID, fmt.Errorf("unable to open file %s: %v", path, err)
}
defer file.Close()
@@ -50,20 +49,20 @@ func (u *uploader) UploadFile(path string) (content.ObjectID, error) {
io.Copy(writer, file)
result, err := writer.Result(false)
if err != nil {
return content.NullObjectID, err
return cas.NullObjectID, err
}
return result, nil
}
func (u *uploader) UploadDir(path string, previous content.ObjectID) (content.ObjectID, error) {
func (u *uploader) UploadDir(path string, previous cas.ObjectID) (cas.ObjectID, error) {
if u.isCancelled() {
return previous, ErrUploadCancelled
}
listing, err := u.lister.List(path)
if err != nil {
return content.NullObjectID, err
return cas.NullObjectID, err
}
var cached Listing
@@ -91,14 +90,14 @@ func (u *uploader) UploadDir(path string, previous content.ObjectID) (content.Ob
directoryMatchesCache = directoryMatchesCache && cachedMetadataMatches
if e.Type == EntryTypeDirectory {
var previousSubdirObjectID content.ObjectID
var previousSubdirObjectID cas.ObjectID
if cachedEntry != nil {
previousSubdirObjectID = cachedEntry.ObjectID
}
e.ObjectID, err = u.UploadDir(fullPath, previousSubdirObjectID)
if err != nil {
return content.NullObjectID, err
return cas.NullObjectID, err
}
if cachedEntry != nil && e.ObjectID != cachedEntry.ObjectID {
@@ -110,7 +109,7 @@ func (u *uploader) UploadDir(path string, previous content.ObjectID) (content.Ob
} else {
e.ObjectID, err = u.UploadFile(fullPath)
if err != nil {
return content.NullObjectID, fmt.Errorf("unable to hash file: %s", err)
return cas.NullObjectID, fmt.Errorf("unable to hash file: %s", err)
}
}
}

4
dir/writer.go
View File

@@ -10,7 +10,7 @@
"strconv"
"time"
"github.com/kopia/kopia/content"
"github.com/kopia/kopia/cas"
)
var (
@@ -133,7 +133,7 @@ func ReadDir(r io.Reader) (Listing, error) {
e.Name = v.Name
e.UserID = v.UserID
e.GroupID = v.GroupID
e.ObjectID, err = content.ParseObjectID(v.ObjectID)
e.ObjectID, err = cas.ParseObjectID(v.ObjectID)
if err != nil {
return Listing{}, nil
}