From 4f4275c8a5779a0ba4d778e8295631f9a5686eff Mon Sep 17 00:00:00 2001 From: Jarek Kowalski Date: Fri, 1 Apr 2016 18:33:29 -0700 Subject: [PATCH] moved content.ObjectID to cas.ObjectID --- cas/formatter.go | 12 +++++------- cas/object_manager.go | 17 ++++++++--------- cas/object_manager_test.go | 27 +++++++++++++-------------- cas/object_reader.go | 13 ++++++------- cas/object_writer.go | 17 ++++++++--------- {content => cas}/objectid.go | 2 +- {content => cas}/objectid_test.go | 2 +- dir/entry.go | 4 ++-- dir/upload.go | 21 ++++++++++----------- dir/writer.go | 4 ++-- 10 files changed, 56 insertions(+), 63 deletions(-) rename {content => cas}/objectid.go (99%) rename {content => cas}/objectid_test.go (98%) diff --git a/cas/formatter.go b/cas/formatter.go index 422d34f58..56c61b4f6 100644 --- a/cas/formatter.go +++ b/cas/formatter.go @@ -11,27 +11,25 @@ "hash" "io" "sync/atomic" - - "github.com/kopia/kopia/content" ) type streamTransformer func(io.ReadCloser) io.ReadCloser type objectFormatter interface { - Do(b []byte, prefix string, stats *ObjectManagerStats) (content.ObjectID, streamTransformer) + Do(b []byte, prefix string, stats *ObjectManagerStats) (ObjectID, streamTransformer) } type nonEncryptingFormatter struct { hash func() hash.Hash } -func (f *nonEncryptingFormatter) Do(b []byte, prefix string, stats *ObjectManagerStats) (content.ObjectID, streamTransformer) { +func (f *nonEncryptingFormatter) Do(b []byte, prefix string, stats *ObjectManagerStats) (ObjectID, streamTransformer) { h := f.hash() h.Write(b) blockID := hex.EncodeToString(h.Sum(nil)) atomic.AddInt64(&stats.HashedBytes, int64(len(b))) - return content.ObjectID(prefix + blockID), func(r io.ReadCloser) io.ReadCloser { return r } + return ObjectID(prefix + blockID), func(r io.ReadCloser) io.ReadCloser { return r } } func newNonEncryptingFormatter(hash func() hash.Hash) objectFormatter { @@ -44,7 +42,7 @@ type aesEncryptingFormatter struct { masterContentSecret []byte } -func (f *aesEncryptingFormatter) Do(b []byte, prefix string, stats *ObjectManagerStats) (content.ObjectID, streamTransformer) { +func (f *aesEncryptingFormatter) Do(b []byte, prefix string, stats *ObjectManagerStats) (ObjectID, streamTransformer) { // Compute HMAC-SHA512 of the content s := hmac.New(sha512.New, f.masterContentSecret) s.Write(b) @@ -53,7 +51,7 @@ func (f *aesEncryptingFormatter) Do(b []byte, prefix string, stats *ObjectManage // Split the hash into two portions - encryption key and content ID. aesKey := contentHash[0:32] - return content.ObjectID(prefix + hex.EncodeToString(contentHash[32:64]) + ".e"), + return ObjectID(prefix + hex.EncodeToString(contentHash[32:64]) + ".e"), func(r io.ReadCloser) io.ReadCloser { var iv [aes.BlockSize]byte rand.Read(iv[:]) diff --git a/cas/object_manager.go b/cas/object_manager.go index 8dc43f2dc..b0652ca17 100644 --- a/cas/object_manager.go +++ b/cas/object_manager.go @@ -17,7 +17,6 @@ "strings" "sync/atomic" - "github.com/kopia/kopia/content" "github.com/kopia/kopia/storage" ) @@ -31,7 +30,7 @@ type ObjectManager interface { NewWriter(options ...WriterOption) ObjectWriter // Open creates an io.ReadSeeker for reading object with a specified ID. - Open(objectID content.ObjectID) (io.ReadSeeker, error) + Open(objectID ObjectID) (io.ReadSeeker, error) Flush() error Repository() storage.Repository @@ -87,7 +86,7 @@ func (mgr *objectManager) NewWriter(options ...WriterOption) ObjectWriter { mgr: mgr, putOptions: storage.PutOptions{}, }, - content.ObjectIDTypeStored) + ObjectIDTypeStored) for _, option := range options { option(result) @@ -96,13 +95,13 @@ func (mgr *objectManager) NewWriter(options ...WriterOption) ObjectWriter { return result } -func (mgr *objectManager) Open(objectID content.ObjectID) (io.ReadSeeker, error) { +func (mgr *objectManager) Open(objectID ObjectID) (io.ReadSeeker, error) { r, err := mgr.newRawReader(objectID) if err != nil { return nil, err } - if objectID.Type() == content.ObjectIDTypeList { + if objectID.Type() == ObjectIDTypeList { seekTable := make([]seekTableEntry, 0, 100) seekTable, err = mgr.flattenListChunk(seekTable, objectID, r) @@ -228,7 +227,7 @@ func splitHash(keySize int) keygenFunc { } } -func (mgr *objectManager) hashBufferForWriting(buffer *bytes.Buffer, prefix string) (content.ObjectID, io.ReadCloser) { +func (mgr *objectManager) hashBufferForWriting(buffer *bytes.Buffer, prefix string) (ObjectID, io.ReadCloser) { var data []byte if buffer != nil { data = buffer.Bytes() @@ -238,14 +237,14 @@ func (mgr *objectManager) hashBufferForWriting(buffer *bytes.Buffer, prefix stri h.Write(data) contentHash := h.Sum(nil) - var objectID content.ObjectID + var objectID ObjectID var cryptoKey []byte if mgr.createCipher != nil { cryptoKey, contentHash = mgr.keygen(contentHash) - objectID = content.ObjectID(prefix + hex.EncodeToString(contentHash) + ":" + hex.EncodeToString(cryptoKey)) + objectID = ObjectID(prefix + hex.EncodeToString(contentHash) + ":" + hex.EncodeToString(cryptoKey)) } else { - objectID = content.ObjectID(prefix + hex.EncodeToString(contentHash)) + objectID = ObjectID(prefix + hex.EncodeToString(contentHash)) } atomic.AddInt64(&mgr.stats.HashedBytes, int64(len(data))) diff --git a/cas/object_manager_test.go b/cas/object_manager_test.go index af2877e66..18b5958a0 100644 --- a/cas/object_manager_test.go +++ b/cas/object_manager_test.go @@ -12,7 +12,6 @@ "strings" "testing" - "github.com/kopia/kopia/content" "github.com/kopia/kopia/storage" ) @@ -53,7 +52,7 @@ func setupTest(t *testing.T) (data map[string][]byte, mgr ObjectManager) { func TestWriters(t *testing.T) { cases := []struct { data []byte - objectID content.ObjectID + objectID ObjectID }{ {[]byte{}, "B"}, {[]byte("quick brown fox"), "Tquick brown fox"}, @@ -269,7 +268,7 @@ func TestReader(t *testing.T) { } for _, c := range cases { - objectID, err := content.ParseObjectID(c.text) + objectID, err := ParseObjectID(c.text) if err != nil { t.Errorf("cannot parse object ID: %v", err) continue @@ -296,7 +295,7 @@ func TestReader(t *testing.T) { func TestReaderStoredBlockNotFound(t *testing.T) { _, mgr := setupTest(t) - objectID, err := content.ParseObjectID("Cno-such-block") + objectID, err := ParseObjectID("Cno-such-block") if err != nil { t.Errorf("cannot parse object ID: %v", err) } @@ -356,14 +355,14 @@ func TestEndToEndReadAndSeek(t *testing.T) { func TestFormats(t *testing.T) { cases := []struct { format Format - oids map[string]content.ObjectID + oids map[string]ObjectID }{ { format: Format{ Version: "1", Hash: "md5", }, - oids: map[string]content.ObjectID{ + oids: map[string]ObjectID{ "": "Cd41d8cd98f00b204e9800998ecf8427e", "The quick brown fox jumps over the lazy dog": "C9e107d9d372bb6826bd81d3542a419d6", }, @@ -373,7 +372,7 @@ func TestFormats(t *testing.T) { Version: "1", Hash: "hmac-md5", }, - oids: map[string]content.ObjectID{ + oids: map[string]ObjectID{ "": "C74e6f7298a9c2d168935f58c001bad88", "The quick brown fox jumps over the lazy dog": "Cad262969c53bc16032f160081c4a07a0", }, @@ -384,7 +383,7 @@ func TestFormats(t *testing.T) { Hash: "hmac-md5", Secret: []byte("key"), }, - oids: map[string]content.ObjectID{ + oids: map[string]ObjectID{ "The quick brown fox jumps over the lazy dog": "C80070713463e7749b90c2dc24911e275", }, }, @@ -394,7 +393,7 @@ func TestFormats(t *testing.T) { Hash: "hmac-sha1", Secret: []byte("key"), }, - oids: map[string]content.ObjectID{ + oids: map[string]ObjectID{ "The quick brown fox jumps over the lazy dog": "Cde7c9b85b8b78aa6bc8a7a36f70a90701c9db4d9", }, }, @@ -404,7 +403,7 @@ func TestFormats(t *testing.T) { Hash: "hmac-sha256", Secret: []byte("key"), }, - oids: map[string]content.ObjectID{ + oids: map[string]ObjectID{ "The quick brown fox jumps over the lazy dog": "Cf7bc83f430538424b13298e6aa6fb143ef4d59a14946175997479dbc2d1a3cd8", }, }, @@ -414,7 +413,7 @@ func TestFormats(t *testing.T) { Hash: "hmac-sha512", Secret: []byte("key"), }, - oids: map[string]content.ObjectID{ + oids: map[string]ObjectID{ "The quick brown fox jumps over the lazy dog": "Cb42af09057bac1e2d41708e48a902e09b5ff7f12ab428a4fe86653c73dd248fb82f948a549f7b791a5b41915ee4d1ec3935357e4e2317250d0372afa2ebeeb3a", }, }, @@ -425,7 +424,7 @@ func TestFormats(t *testing.T) { Secret: []byte("key"), Encryption: "aes-128", }, - oids: map[string]content.ObjectID{ + oids: map[string]ObjectID{ "The quick brown fox jumps over the lazy dog": "Cb42af09057bac1e2d41708e48a902e09b5ff7f12ab428a4fe86653c73dd248fb82f948a549f7b791a5b41915ee4d1ec3:935357e4e2317250d0372afa2ebeeb3a", }, }, @@ -436,7 +435,7 @@ func TestFormats(t *testing.T) { Secret: []byte("key"), Encryption: "aes-192", }, - oids: map[string]content.ObjectID{ + oids: map[string]ObjectID{ "The quick brown fox jumps over the lazy dog": "Cb42af09057bac1e2d41708e48a902e09b5ff7f12ab428a4fe86653c73dd248fb82f948a549f7b791:a5b41915ee4d1ec3935357e4e2317250d0372afa2ebeeb3a", }, }, @@ -447,7 +446,7 @@ func TestFormats(t *testing.T) { Secret: []byte("key"), Encryption: "aes-256", }, - oids: map[string]content.ObjectID{ + oids: map[string]ObjectID{ "The quick brown fox jumps over the lazy dog": "Cb42af09057bac1e2d41708e48a902e09b5ff7f12ab428a4fe86653c73dd248fb:82f948a549f7b791a5b41915ee4d1ec3935357e4e2317250d0372afa2ebeeb3a", }, }, diff --git a/cas/object_reader.go b/cas/object_reader.go index 84b49a68a..40d543aca 100644 --- a/cas/object_reader.go +++ b/cas/object_reader.go @@ -8,7 +8,6 @@ "strconv" "strings" - "github.com/kopia/kopia/content" "github.com/kopia/kopia/storage" ) @@ -162,7 +161,7 @@ func (r *objectReader) Seek(offset int64, whence int) (int64, error) { return r.currentPosition, nil } -func (mgr *objectManager) newRawReader(objectID content.ObjectID) (io.ReadSeeker, error) { +func (mgr *objectManager) newRawReader(objectID ObjectID) (io.ReadSeeker, error) { inline := objectID.InlineData() if inline != nil { return bytes.NewReader(inline), nil @@ -174,7 +173,7 @@ func (mgr *objectManager) newRawReader(objectID content.ObjectID) (io.ReadSeeker return nil, err } - if objectID.EncryptionInfo().Mode() == content.ObjectEncryptionNone { + if objectID.EncryptionInfo().Mode() == ObjectEncryptionNone { return bytes.NewReader(payload), nil } @@ -183,7 +182,7 @@ func (mgr *objectManager) newRawReader(objectID content.ObjectID) (io.ReadSeeker func (mgr *objectManager) flattenListChunk( seekTable []seekTableEntry, - listObjectID content.ObjectID, + listObjectID ObjectID, rawReader io.Reader) ([]seekTableEntry, error) { scanner := bufio.NewScanner(rawReader) @@ -197,13 +196,13 @@ func (mgr *objectManager) flattenListChunk( length, err := strconv.ParseInt(c[0:comma], 10, 64) - objectID, err := content.ParseObjectID(c[comma+1:]) + objectID, err := ParseObjectID(c[comma+1:]) if err != nil { return nil, fmt.Errorf("unsupported entry '%v' in list '%s': %#v", c, listObjectID, err) } switch objectID.Type() { - case content.ObjectIDTypeList: + case ObjectIDTypeList: subreader, err := mgr.newRawReader(objectID) if err != nil { return nil, err @@ -214,7 +213,7 @@ func (mgr *objectManager) flattenListChunk( return nil, err } - case content.ObjectIDTypeStored: + case ObjectIDTypeStored: var startOffset int64 if len(seekTable) > 0 { startOffset = seekTable[len(seekTable)-1].endOffset() diff --git a/cas/object_writer.go b/cas/object_writer.go index 97c786fdc..e6f2ac1f4 100644 --- a/cas/object_writer.go +++ b/cas/object_writer.go @@ -5,7 +5,6 @@ "fmt" "io" - "github.com/kopia/kopia/content" "github.com/kopia/kopia/storage" ) @@ -18,7 +17,7 @@ type blockHasher interface { type ObjectWriter interface { io.WriteCloser - Result(forceStored bool) (content.ObjectID, error) + Result(forceStored bool) (ObjectID, error) } // objectWriterConfig @@ -36,10 +35,10 @@ type objectWriter struct { prefix string listWriter *objectWriter flushedObjectCount int - lastFlushedObject content.ObjectID + lastFlushedObject ObjectID description string - objectType content.ObjectIDType + objectType ObjectIDType atomicWrites bool } @@ -116,7 +115,7 @@ func (w *objectWriter) flushBuffer(force bool) error { w.flushedObjectCount++ w.lastFlushedObject = objectID if w.listWriter == nil { - w.listWriter = newObjectWriter(w.objectWriterConfig, content.ObjectIDTypeList) + w.listWriter = newObjectWriter(w.objectWriterConfig, ObjectIDTypeList) w.listWriter.description = "LIST(" + w.description + ")" w.listWriter.atomicWrites = true } @@ -126,21 +125,21 @@ func (w *objectWriter) flushBuffer(force bool) error { return nil } -func newObjectWriter(cfg objectWriterConfig, objectType content.ObjectIDType) *objectWriter { +func newObjectWriter(cfg objectWriterConfig, objectType ObjectIDType) *objectWriter { return &objectWriter{ objectWriterConfig: cfg, objectType: objectType, } } -func (w *objectWriter) Result(forceStored bool) (content.ObjectID, error) { +func (w *objectWriter) Result(forceStored bool) (ObjectID, error) { if !forceStored && w.flushedObjectCount == 0 { if w.buffer == nil { return "B", nil } if w.buffer.Len() < w.mgr.maxInlineBlobSize { - return content.NewInlineObjectID(w.buffer.Bytes()), nil + return NewInlineObjectID(w.buffer.Bytes()), nil } } @@ -154,7 +153,7 @@ func (w *objectWriter) Result(forceStored bool) (content.ObjectID, error) { if w.flushedObjectCount == 1 { return w.lastFlushedObject, nil } else if w.flushedObjectCount == 0 { - return content.NullObjectID, nil + return NullObjectID, nil } else { return w.listWriter.Result(true) } diff --git a/content/objectid.go b/cas/objectid.go similarity index 99% rename from content/objectid.go rename to cas/objectid.go index 4220f4b8e..42b928192 100644 --- a/content/objectid.go +++ b/cas/objectid.go @@ -1,4 +1,4 @@ -package content +package cas import ( "encoding/base64" diff --git a/content/objectid_test.go b/cas/objectid_test.go similarity index 98% rename from content/objectid_test.go rename to cas/objectid_test.go index 00ad264e1..292fcec66 100644 --- a/content/objectid_test.go +++ b/cas/objectid_test.go @@ -1,4 +1,4 @@ -package content +package cas import ( "strings" diff --git a/dir/entry.go b/dir/entry.go index bedad6fe0..f68dbfca0 100644 --- a/dir/entry.go +++ b/dir/entry.go @@ -5,7 +5,7 @@ "os" "time" - "github.com/kopia/kopia/content" + "github.com/kopia/kopia/cas" ) // EntryType describes the type of an backup entry. @@ -69,7 +69,7 @@ type EntryMetadata struct { type Entry struct { EntryMetadata - ObjectID content.ObjectID + ObjectID cas.ObjectID } func (e *Entry) metadataEquals(other *Entry) bool { diff --git a/dir/upload.go b/dir/upload.go index 5b4afe51a..7a5dfc30a 100644 --- a/dir/upload.go +++ b/dir/upload.go @@ -10,7 +10,6 @@ "sync/atomic" "github.com/kopia/kopia/cas" - "github.com/kopia/kopia/content" ) // ErrUploadCancelled is returned when the upload gets cancelled. @@ -18,8 +17,8 @@ // Uploader supports efficient uploading files and directories to CAS storage. type Uploader interface { - UploadFile(path string) (content.ObjectID, error) - UploadDir(path string, previousObjectID content.ObjectID) (content.ObjectID, error) + UploadFile(path string) (cas.ObjectID, error) + UploadDir(path string, previousObjectID cas.ObjectID) (cas.ObjectID, error) Cancel() } @@ -34,10 +33,10 @@ func (u *uploader) isCancelled() bool { return atomic.LoadInt32(&u.cancelled) != 0 } -func (u *uploader) UploadFile(path string) (content.ObjectID, error) { +func (u *uploader) UploadFile(path string) (cas.ObjectID, error) { file, err := os.Open(path) if err != nil { - return content.NullObjectID, fmt.Errorf("unable to open file %s: %v", path, err) + return cas.NullObjectID, fmt.Errorf("unable to open file %s: %v", path, err) } defer file.Close() @@ -50,20 +49,20 @@ func (u *uploader) UploadFile(path string) (content.ObjectID, error) { io.Copy(writer, file) result, err := writer.Result(false) if err != nil { - return content.NullObjectID, err + return cas.NullObjectID, err } return result, nil } -func (u *uploader) UploadDir(path string, previous content.ObjectID) (content.ObjectID, error) { +func (u *uploader) UploadDir(path string, previous cas.ObjectID) (cas.ObjectID, error) { if u.isCancelled() { return previous, ErrUploadCancelled } listing, err := u.lister.List(path) if err != nil { - return content.NullObjectID, err + return cas.NullObjectID, err } var cached Listing @@ -91,14 +90,14 @@ func (u *uploader) UploadDir(path string, previous content.ObjectID) (content.Ob directoryMatchesCache = directoryMatchesCache && cachedMetadataMatches if e.Type == EntryTypeDirectory { - var previousSubdirObjectID content.ObjectID + var previousSubdirObjectID cas.ObjectID if cachedEntry != nil { previousSubdirObjectID = cachedEntry.ObjectID } e.ObjectID, err = u.UploadDir(fullPath, previousSubdirObjectID) if err != nil { - return content.NullObjectID, err + return cas.NullObjectID, err } if cachedEntry != nil && e.ObjectID != cachedEntry.ObjectID { @@ -110,7 +109,7 @@ func (u *uploader) UploadDir(path string, previous content.ObjectID) (content.Ob } else { e.ObjectID, err = u.UploadFile(fullPath) if err != nil { - return content.NullObjectID, fmt.Errorf("unable to hash file: %s", err) + return cas.NullObjectID, fmt.Errorf("unable to hash file: %s", err) } } } diff --git a/dir/writer.go b/dir/writer.go index 4e8edbc2c..99a11bd9a 100644 --- a/dir/writer.go +++ b/dir/writer.go @@ -10,7 +10,7 @@ "strconv" "time" - "github.com/kopia/kopia/content" + "github.com/kopia/kopia/cas" ) var ( @@ -133,7 +133,7 @@ func ReadDir(r io.Reader) (Listing, error) { e.Name = v.Name e.UserID = v.UserID e.GroupID = v.GroupID - e.ObjectID, err = content.ParseObjectID(v.ObjectID) + e.ObjectID, err = cas.ParseObjectID(v.ObjectID) if err != nil { return Listing{}, nil }