diff --git a/block/block_manager.go b/block/block_manager.go index 13ca50fda..6e22e3a51 100644 --- a/block/block_manager.go +++ b/block/block_manager.go @@ -18,7 +18,7 @@ import ( const parallelFetches = 5 const parallelDeletes = 20 const flushPackIndexTimeout = 10 * time.Minute -const packObjectPrefix = "P" +const packBlockPrefix = "P" const legacyUnpackedObjectsPackGroup = "_unpacked_" const nonPackedObjectsPackGroup = "raw" const packObjectsPackGroup = "packs" @@ -82,7 +82,7 @@ func (bm *Manager) BlockSize(blockID string) (int64, error) { } func (bm *Manager) blockIDToPackSection(blockID string) (Info, bool, error) { - if strings.HasPrefix(blockID, packObjectPrefix) { + if strings.HasPrefix(blockID, packBlockPrefix) { return Info{}, false, nil } @@ -120,7 +120,7 @@ func (bm *Manager) blockIDToPackSection(blockID string) (Info, bool, error) { } func (bm *Manager) registerUnpackedBlock(packGroupID string, blockID string, dataLength int64) error { - if strings.HasPrefix(blockID, packObjectPrefix) { + if strings.HasPrefix(blockID, packBlockPrefix) { return nil } @@ -153,7 +153,7 @@ func (bm *Manager) registerUnpackedBlockLockedNoFlush(groupID string, blockID st } func (bm *Manager) addToPack(packGroup string, blockID string, data []byte) error { - if strings.HasPrefix(blockID, packObjectPrefix) { + if strings.HasPrefix(blockID, packBlockPrefix) { return fmt.Errorf("pack objects can't be packed: %v", blockID) } @@ -254,7 +254,7 @@ func (bm *Manager) writePackIndexes(ndx packIndexes) (string, error) { } zw.Close() - return bm.writeUnpackedBlock(buf.Bytes(), packObjectPrefix, true) + return bm.writeUnpackedBlock(buf.Bytes(), packBlockPrefix, true) } func (bm *Manager) finishAllOpenPacksLocked() error { @@ -312,7 +312,7 @@ func isNonPacked(g string) bool { } func (bm *Manager) loadMergedPackIndexLocked(cutoffTime time.Time) (packIndexes, []string, error) { - ch, cancel := bm.storage.ListBlocks(packObjectPrefix) + ch, cancel := bm.storage.ListBlocks(packBlockPrefix) defer cancel() t0 := time.Now() @@ -614,15 +614,15 @@ func (bm *Manager) Flush() error { // WriteBlock saves a given block of data to a pack group with a provided name and returns a blockID // that's based on the contents of data written. -func (bm *Manager) WriteBlock(packGroup string, data []byte, prefix string) (string, error) { +func (bm *Manager) WriteBlock(packGroup string, data []byte) (string, error) { if bm.maxPackedContentLength > 0 && len(data) <= bm.maxPackedContentLength { - blockID := prefix + bm.hashData(data) + blockID := bm.hashData(data) err := bm.addToPack(packGroup, blockID, data) return blockID, err } - blockID, err := bm.writeUnpackedBlock(data, prefix, false) + blockID, err := bm.writeUnpackedBlock(data, "", false) if err != nil { return "", err } diff --git a/block/block_manager_test.go b/block/block_manager_test.go index 855e3744c..49ab432cf 100644 --- a/block/block_manager_test.go +++ b/block/block_manager_test.go @@ -160,52 +160,6 @@ func TestBlockManagerEmpty(t *testing.T) { } } -func TestBlockManagerMultiplePrefixes(t *testing.T) { - data1 := seededRandomData(1, 600) - h := md5hash(data1) - - data := map[string][]byte{} - bm := newTestBlockManager(data) - - prefixes := []string{"A", "B", "C", "D", "E"} - - for _, prefix := range prefixes { - b1, err := bm.WriteBlock("", data1, prefix) - if err != nil { - t.Errorf("error writing block with prefix %q", prefix) - continue - } - - verifyBlock(t, bm, b1, data1) - bm.Flush() - verifyBlock(t, bm, b1, data1) - - if got, want := b1, prefix+h; got != want { - t.Errorf("unexpected block ID: %v, want %v", got, want) - } - } - - bm = newTestBlockManager(data) - for _, prefix := range prefixes { - verifyBlock(t, bm, prefix+h, data1) - } - - if got, want := len(data), len(prefixes)+1; got != want { - t.Errorf("unexpected block count: %v, wanted %v", got, want) - } - - // should have 1 data block + indexes - if err := bm.CompactIndexes(fakeTime, nil); err != nil { - t.Errorf("error compacting indexes: %v", err) - } - - if got, want := len(data), 2; got != want { - t.Errorf("unexpected block count: %v, wanted %v", got, want) - } - - dumpBlockManagerData(data) -} - func TestBlockManagerPackIdentialToRawObject(t *testing.T) { data0 := []byte{} data1 := seededRandomData(1, 600) @@ -322,7 +276,7 @@ func TestBlockManagerWriteMultiple(t *testing.T) { //t.Logf("i=%v", i) b := seededRandomData(i, i%113) //t.Logf("writing block #%v with %x", i, b) - blkID, err := bm.WriteBlock("", b, "") + blkID, err := bm.WriteBlock("", b) //t.Logf("wrote %v=%v", i, blkID) if err != nil { t.Errorf("err: %v", err) @@ -462,7 +416,7 @@ func getIndexCount(d map[string][]byte) int { var cnt int for k := range d { - if strings.HasPrefix(k, packObjectPrefix) { + if strings.HasPrefix(k, packBlockPrefix) { cnt++ } } @@ -508,7 +462,7 @@ func verifyBlock(t *testing.T, bm *Manager, blockID string, b []byte) { func writeBlockAndVerify(t *testing.T, bm *Manager, packGroup string, b []byte) string { t.Helper() - blockID, err := bm.WriteBlock(packGroup, b, "") + blockID, err := bm.WriteBlock(packGroup, b) if err != nil { t.Errorf("err: %v", err) } diff --git a/object/object_manager.go b/object/object_manager.go index fbd3c0a89..0b9c24c03 100644 --- a/object/object_manager.go +++ b/object/object_manager.go @@ -24,7 +24,7 @@ type Reader interface { type blockManager interface { BlockInfo(blockID string) (block.Info, error) GetBlock(blockID string) ([]byte, error) - WriteBlock(packGroup string, data []byte, prefix string) (string, error) + WriteBlock(packGroup string, data []byte) (string, error) Flush() error } @@ -56,7 +56,6 @@ func (om *Manager) NewWriter(opt WriterOptions) Writer { repo: om, splitter: om.newSplitter(), description: opt.Description, - prefix: opt.BlockNamePrefix, packGroup: opt.PackGroup, } diff --git a/object/object_manager_test.go b/object/object_manager_test.go index 34793461b..9640a687a 100644 --- a/object/object_manager_test.go +++ b/object/object_manager_test.go @@ -4,6 +4,7 @@ import ( "bytes" "crypto/md5" cryptorand "crypto/rand" + "encoding/hex" "fmt" "io" "io/ioutil" @@ -36,10 +37,10 @@ func (f *fakeBlockManager) GetBlock(blockID string) ([]byte, error) { return nil, storage.ErrBlockNotFound } -func (f *fakeBlockManager) WriteBlock(groupID string, data []byte, prefix string) (string, error) { +func (f *fakeBlockManager) WriteBlock(groupID string, data []byte) (string, error) { h := md5.New() h.Write(data) - blockID := fmt.Sprintf("%v%x", prefix, h.Sum(nil)) + blockID := hex.EncodeToString(h.Sum(nil)) f.mu.Lock() defer f.mu.Unlock() @@ -87,17 +88,15 @@ func TestWriters(t *testing.T) { }{ { []byte("the quick brown fox jumps over the lazy dog"), - ID{StorageBlock: "X77add1d5f41223d5582fca736a5cb335"}, + ID{StorageBlock: "77add1d5f41223d5582fca736a5cb335"}, }, - {make([]byte, 100), ID{StorageBlock: "X6d0bb00954ceb7fbee436bb55a8397a9"}}, // 100 zero bytes + {make([]byte, 100), ID{StorageBlock: "6d0bb00954ceb7fbee436bb55a8397a9"}}, // 100 zero bytes } for _, c := range cases { data, om := setupTest(t) - writer := om.NewWriter(WriterOptions{ - BlockNamePrefix: "X", - }) + writer := om.NewWriter(WriterOptions{}) writer.Write(c.data) @@ -134,13 +133,11 @@ func TestWriterCompleteChunkInTwoWrites(t *testing.T) { _, om := setupTest(t) bytes := make([]byte, 100) - writer := om.NewWriter(WriterOptions{ - BlockNamePrefix: "X", - }) + writer := om.NewWriter(WriterOptions{}) writer.Write(bytes[0:50]) writer.Write(bytes[0:50]) result, err := writer.Result() - if !objectIDsEqual(result, ID{StorageBlock: "X6d0bb00954ceb7fbee436bb55a8397a9"}) { + if !objectIDsEqual(result, ID{StorageBlock: "6d0bb00954ceb7fbee436bb55a8397a9"}) { t.Errorf("unexpected result: %v err: %v", result, err) } } @@ -309,9 +306,7 @@ func TestEndToEndReadAndSeek(t *testing.T) { randomData := make([]byte, size) cryptorand.Read(randomData) - writer := om.NewWriter(WriterOptions{ - BlockNamePrefix: "X", - }) + writer := om.NewWriter(WriterOptions{}) writer.Write(randomData) objectID, err := writer.Result() writer.Close() diff --git a/object/object_writer.go b/object/object_writer.go index 7d9e99b5e..9429a5fb3 100644 --- a/object/object_writer.go +++ b/object/object_writer.go @@ -49,7 +49,6 @@ type objectWriter struct { buffer bytes.Buffer totalLength int64 - prefix string currentPosition int64 blockIndex []indirectObjectEntry @@ -98,7 +97,7 @@ func (w *objectWriter) flushBuffer() error { w.buffer.Reset() do := func() { - blockID, err := w.repo.blockMgr.WriteBlock(w.packGroup, b2.Bytes(), w.prefix) + blockID, err := w.repo.blockMgr.WriteBlock(w.packGroup, b2.Bytes()) w.repo.trace("OBJECT_WRITER(%q) stored %v (%v bytes)", w.description, blockID, length) if err != nil { w.err.add(fmt.Errorf("error when flushing chunk %d of %s: %v", chunkID, w.description, err)) @@ -145,7 +144,6 @@ func (w *objectWriter) Result() (ID, error) { iw := &objectWriter{ repo: w.repo, - prefix: w.prefix, description: "LIST(" + w.description + ")", splitter: w.repo.newSplitter(), packGroup: w.packGroup, @@ -165,9 +163,8 @@ func (w *objectWriter) Result() (ID, error) { // WriterOptions can be passed to Repository.NewWriter() type WriterOptions struct { - BlockNamePrefix string - Description string - PackGroup string + Description string + PackGroup string splitter objectSplitter } diff --git a/repo/repository_test.go b/repo/repository_test.go index 29ac0fcab..0105330e1 100644 --- a/repo/repository_test.go +++ b/repo/repository_test.go @@ -64,17 +64,15 @@ func TestWriters(t *testing.T) { }{ { []byte("the quick brown fox jumps over the lazy dog"), - object.ID{StorageBlock: "X77add1d5f41223d5582fca736a5cb335"}, + object.ID{StorageBlock: "77add1d5f41223d5582fca736a5cb335"}, }, - {make([]byte, 100), object.ID{StorageBlock: "X6d0bb00954ceb7fbee436bb55a8397a9"}}, // 100 zero bytes + {make([]byte, 100), object.ID{StorageBlock: "6d0bb00954ceb7fbee436bb55a8397a9"}}, // 100 zero bytes } for _, c := range cases { data, repo := setupTest(t) - writer := repo.Objects.NewWriter(object.WriterOptions{ - BlockNamePrefix: "X", - }) + writer := repo.Objects.NewWriter(object.WriterOptions{}) writer.Write(c.data) @@ -122,13 +120,11 @@ func TestWriterCompleteChunkInTwoWrites(t *testing.T) { _, repo := setupTest(t) bytes := make([]byte, 100) - writer := repo.Objects.NewWriter(object.WriterOptions{ - BlockNamePrefix: "X", - }) + writer := repo.Objects.NewWriter(object.WriterOptions{}) writer.Write(bytes[0:50]) writer.Write(bytes[0:50]) result, err := writer.Result() - if !objectIDsEqual(result, object.ID{StorageBlock: "X6d0bb00954ceb7fbee436bb55a8397a9"}) { + if !objectIDsEqual(result, object.ID{StorageBlock: "6d0bb00954ceb7fbee436bb55a8397a9"}) { t.Errorf("unexpected result: %v err: %v", result, err) } } @@ -321,9 +317,7 @@ func TestEndToEndReadAndSeek(t *testing.T) { randomData := make([]byte, size) cryptorand.Read(randomData) - writer := repo.Objects.NewWriter(object.WriterOptions{ - BlockNamePrefix: "X", - }) + writer := repo.Objects.NewWriter(object.WriterOptions{}) writer.Write(randomData) objectID, err := writer.Result() writer.Close() diff --git a/snapshot/upload.go b/snapshot/upload.go index c5c6682ed..8733cdff0 100644 --- a/snapshot/upload.go +++ b/snapshot/upload.go @@ -221,9 +221,8 @@ func (u *Uploader) uploadDir(dir fs.Directory) (object.ID, object.ID, error) { var err error mw := u.repo.Objects.NewWriter(object.WriterOptions{ - Description: "HASHCACHE:" + dir.Metadata().Name, - BlockNamePrefix: "H", - PackGroup: "HC", + Description: "HASHCACHE:" + dir.Metadata().Name, + PackGroup: "HC", }) defer mw.Close() u.cacheWriter = hashcache.NewWriter(mw)