removed support for object ID prefixes

block/block_manager.go

@@ -18,7 +18,7 @@ import (
 const parallelFetches = 5
 const parallelDeletes = 20
 const flushPackIndexTimeout = 10 * time.Minute
-const packObjectPrefix = "P"
+const packBlockPrefix = "P"
 const legacyUnpackedObjectsPackGroup = "_unpacked_"
 const nonPackedObjectsPackGroup = "raw"
 const packObjectsPackGroup = "packs"
@@ -82,7 +82,7 @@ func (bm *Manager) BlockSize(blockID string) (int64, error) {
 }
 
 func (bm *Manager) blockIDToPackSection(blockID string) (Info, bool, error) {
-	if strings.HasPrefix(blockID, packObjectPrefix) {
+	if strings.HasPrefix(blockID, packBlockPrefix) {
 		return Info{}, false, nil
 	}
 
@@ -120,7 +120,7 @@ func (bm *Manager) blockIDToPackSection(blockID string) (Info, bool, error) {
 }
 
 func (bm *Manager) registerUnpackedBlock(packGroupID string, blockID string, dataLength int64) error {
-	if strings.HasPrefix(blockID, packObjectPrefix) {
+	if strings.HasPrefix(blockID, packBlockPrefix) {
 		return nil
 	}
 
@@ -153,7 +153,7 @@ func (bm *Manager) registerUnpackedBlockLockedNoFlush(groupID string, blockID st
 }
 
 func (bm *Manager) addToPack(packGroup string, blockID string, data []byte) error {
-	if strings.HasPrefix(blockID, packObjectPrefix) {
+	if strings.HasPrefix(blockID, packBlockPrefix) {
 		return fmt.Errorf("pack objects can't be packed: %v", blockID)
 	}
 
@@ -254,7 +254,7 @@ func (bm *Manager) writePackIndexes(ndx packIndexes) (string, error) {
 	}
 	zw.Close()
 
-	return bm.writeUnpackedBlock(buf.Bytes(), packObjectPrefix, true)
+	return bm.writeUnpackedBlock(buf.Bytes(), packBlockPrefix, true)
 }
 
 func (bm *Manager) finishAllOpenPacksLocked() error {
@@ -312,7 +312,7 @@ func isNonPacked(g string) bool {
 }
 
 func (bm *Manager) loadMergedPackIndexLocked(cutoffTime time.Time) (packIndexes, []string, error) {
-	ch, cancel := bm.storage.ListBlocks(packObjectPrefix)
+	ch, cancel := bm.storage.ListBlocks(packBlockPrefix)
 	defer cancel()
 
 	t0 := time.Now()
@@ -614,15 +614,15 @@ func (bm *Manager) Flush() error {
 
 // WriteBlock saves a given block of data to a pack group with a provided name and returns a blockID
 // that's based on the contents of data written.
-func (bm *Manager) WriteBlock(packGroup string, data []byte, prefix string) (string, error) {
+func (bm *Manager) WriteBlock(packGroup string, data []byte) (string, error) {
 	if bm.maxPackedContentLength > 0 && len(data) <= bm.maxPackedContentLength {
-		blockID := prefix + bm.hashData(data)
+		blockID := bm.hashData(data)
 
 		err := bm.addToPack(packGroup, blockID, data)
 		return blockID, err
 	}
 
-	blockID, err := bm.writeUnpackedBlock(data, prefix, false)
+	blockID, err := bm.writeUnpackedBlock(data, "", false)
 	if err != nil {
 		return "", err
 	}

block/block_manager_test.go

@@ -160,52 +160,6 @@ func TestBlockManagerEmpty(t *testing.T) {
 	}
 }
 
-func TestBlockManagerMultiplePrefixes(t *testing.T) {
-	data1 := seededRandomData(1, 600)
-	h := md5hash(data1)
-
-	data := map[string][]byte{}
-	bm := newTestBlockManager(data)
-
-	prefixes := []string{"A", "B", "C", "D", "E"}
-
-	for _, prefix := range prefixes {
-		b1, err := bm.WriteBlock("", data1, prefix)
-		if err != nil {
-			t.Errorf("error writing block with prefix %q", prefix)
-			continue
-		}
-
-		verifyBlock(t, bm, b1, data1)
-		bm.Flush()
-		verifyBlock(t, bm, b1, data1)
-
-		if got, want := b1, prefix+h; got != want {
-			t.Errorf("unexpected block ID: %v, want %v", got, want)
-		}
-	}
-
-	bm = newTestBlockManager(data)
-	for _, prefix := range prefixes {
-		verifyBlock(t, bm, prefix+h, data1)
-	}
-
-	if got, want := len(data), len(prefixes)+1; got != want {
-		t.Errorf("unexpected block count: %v, wanted %v", got, want)
-	}
-
-	// should have 1 data block + indexes
-	if err := bm.CompactIndexes(fakeTime, nil); err != nil {
-		t.Errorf("error compacting indexes: %v", err)
-	}
-
-	if got, want := len(data), 2; got != want {
-		t.Errorf("unexpected block count: %v, wanted %v", got, want)
-	}
-
-	dumpBlockManagerData(data)
-}
-
 func TestBlockManagerPackIdentialToRawObject(t *testing.T) {
 	data0 := []byte{}
 	data1 := seededRandomData(1, 600)
@@ -322,7 +276,7 @@ func TestBlockManagerWriteMultiple(t *testing.T) {
 		//t.Logf("i=%v", i)
 		b := seededRandomData(i, i%113)
 		//t.Logf("writing block #%v with %x", i, b)
-		blkID, err := bm.WriteBlock("", b, "")
+		blkID, err := bm.WriteBlock("", b)
 		//t.Logf("wrote %v=%v", i, blkID)
 		if err != nil {
 			t.Errorf("err: %v", err)
@@ -462,7 +416,7 @@ func getIndexCount(d map[string][]byte) int {
 	var cnt int
 
 	for k := range d {
-		if strings.HasPrefix(k, packObjectPrefix) {
+		if strings.HasPrefix(k, packBlockPrefix) {
 			cnt++
 		}
 	}
@@ -508,7 +462,7 @@ func verifyBlock(t *testing.T, bm *Manager, blockID string, b []byte) {
 func writeBlockAndVerify(t *testing.T, bm *Manager, packGroup string, b []byte) string {
 	t.Helper()
 
-	blockID, err := bm.WriteBlock(packGroup, b, "")
+	blockID, err := bm.WriteBlock(packGroup, b)
 	if err != nil {
 		t.Errorf("err: %v", err)
 	}

object/object_manager.go

@@ -24,7 +24,7 @@ type Reader interface {
 type blockManager interface {
 	BlockInfo(blockID string) (block.Info, error)
 	GetBlock(blockID string) ([]byte, error)
-	WriteBlock(packGroup string, data []byte, prefix string) (string, error)
+	WriteBlock(packGroup string, data []byte) (string, error)
 	Flush() error
 }
 
@@ -56,7 +56,6 @@ func (om *Manager) NewWriter(opt WriterOptions) Writer {
 		repo:        om,
 		splitter:    om.newSplitter(),
 		description: opt.Description,
-		prefix:      opt.BlockNamePrefix,
 		packGroup:   opt.PackGroup,
 	}
 

object/object_manager_test.go

@@ -4,6 +4,7 @@ import (
 	"bytes"
 	"crypto/md5"
 	cryptorand "crypto/rand"
+	"encoding/hex"
 	"fmt"
 	"io"
 	"io/ioutil"
@@ -36,10 +37,10 @@ func (f *fakeBlockManager) GetBlock(blockID string) ([]byte, error) {
 	return nil, storage.ErrBlockNotFound
 }
 
-func (f *fakeBlockManager) WriteBlock(groupID string, data []byte, prefix string) (string, error) {
+func (f *fakeBlockManager) WriteBlock(groupID string, data []byte) (string, error) {
 	h := md5.New()
 	h.Write(data)
-	blockID := fmt.Sprintf("%v%x", prefix, h.Sum(nil))
+	blockID := hex.EncodeToString(h.Sum(nil))
 
 	f.mu.Lock()
 	defer f.mu.Unlock()
@@ -87,17 +88,15 @@ func TestWriters(t *testing.T) {
 	}{
 		{
 			[]byte("the quick brown fox jumps over the lazy dog"),
-			ID{StorageBlock: "X77add1d5f41223d5582fca736a5cb335"},
+			ID{StorageBlock: "77add1d5f41223d5582fca736a5cb335"},
 		},
-		{make([]byte, 100), ID{StorageBlock: "X6d0bb00954ceb7fbee436bb55a8397a9"}}, // 100 zero bytes
+		{make([]byte, 100), ID{StorageBlock: "6d0bb00954ceb7fbee436bb55a8397a9"}}, // 100 zero bytes
 	}
 
 	for _, c := range cases {
 		data, om := setupTest(t)
 
-		writer := om.NewWriter(WriterOptions{
-			BlockNamePrefix: "X",
-		})
+		writer := om.NewWriter(WriterOptions{})
 
 		writer.Write(c.data)
 
@@ -134,13 +133,11 @@ func TestWriterCompleteChunkInTwoWrites(t *testing.T) {
 	_, om := setupTest(t)
 
 	bytes := make([]byte, 100)
-	writer := om.NewWriter(WriterOptions{
-		BlockNamePrefix: "X",
-	})
+	writer := om.NewWriter(WriterOptions{})
 	writer.Write(bytes[0:50])
 	writer.Write(bytes[0:50])
 	result, err := writer.Result()
-	if !objectIDsEqual(result, ID{StorageBlock: "X6d0bb00954ceb7fbee436bb55a8397a9"}) {
+	if !objectIDsEqual(result, ID{StorageBlock: "6d0bb00954ceb7fbee436bb55a8397a9"}) {
 		t.Errorf("unexpected result: %v err: %v", result, err)
 	}
 }
@@ -309,9 +306,7 @@ func TestEndToEndReadAndSeek(t *testing.T) {
 		randomData := make([]byte, size)
 		cryptorand.Read(randomData)
 
-		writer := om.NewWriter(WriterOptions{
-			BlockNamePrefix: "X",
-		})
+		writer := om.NewWriter(WriterOptions{})
 		writer.Write(randomData)
 		objectID, err := writer.Result()
 		writer.Close()

object/object_writer.go

@@ -49,7 +49,6 @@ type objectWriter struct {
 	buffer      bytes.Buffer
 	totalLength int64
 
-	prefix          string
 	currentPosition int64
 	blockIndex      []indirectObjectEntry
 
@@ -98,7 +97,7 @@ func (w *objectWriter) flushBuffer() error {
 	w.buffer.Reset()
 
 	do := func() {
-		blockID, err := w.repo.blockMgr.WriteBlock(w.packGroup, b2.Bytes(), w.prefix)
+		blockID, err := w.repo.blockMgr.WriteBlock(w.packGroup, b2.Bytes())
 		w.repo.trace("OBJECT_WRITER(%q) stored %v (%v bytes)", w.description, blockID, length)
 		if err != nil {
 			w.err.add(fmt.Errorf("error when flushing chunk %d of %s: %v", chunkID, w.description, err))
@@ -145,7 +144,6 @@ func (w *objectWriter) Result() (ID, error) {
 
 	iw := &objectWriter{
 		repo:        w.repo,
-		prefix:      w.prefix,
 		description: "LIST(" + w.description + ")",
 		splitter:    w.repo.newSplitter(),
 		packGroup:   w.packGroup,
@@ -165,9 +163,8 @@ func (w *objectWriter) Result() (ID, error) {
 
 // WriterOptions can be passed to Repository.NewWriter()
 type WriterOptions struct {
-	BlockNamePrefix string
-	Description     string
-	PackGroup       string
+	Description string
+	PackGroup   string
 
 	splitter objectSplitter
 }

repo/repository_test.go

@@ -64,17 +64,15 @@ func TestWriters(t *testing.T) {
 	}{
 		{
 			[]byte("the quick brown fox jumps over the lazy dog"),
-			object.ID{StorageBlock: "X77add1d5f41223d5582fca736a5cb335"},
+			object.ID{StorageBlock: "77add1d5f41223d5582fca736a5cb335"},
 		},
-		{make([]byte, 100), object.ID{StorageBlock: "X6d0bb00954ceb7fbee436bb55a8397a9"}}, // 100 zero bytes
+		{make([]byte, 100), object.ID{StorageBlock: "6d0bb00954ceb7fbee436bb55a8397a9"}}, // 100 zero bytes
 	}
 
 	for _, c := range cases {
 		data, repo := setupTest(t)
 
-		writer := repo.Objects.NewWriter(object.WriterOptions{
-			BlockNamePrefix: "X",
-		})
+		writer := repo.Objects.NewWriter(object.WriterOptions{})
 
 		writer.Write(c.data)
 
@@ -122,13 +120,11 @@ func TestWriterCompleteChunkInTwoWrites(t *testing.T) {
 	_, repo := setupTest(t)
 
 	bytes := make([]byte, 100)
-	writer := repo.Objects.NewWriter(object.WriterOptions{
-		BlockNamePrefix: "X",
-	})
+	writer := repo.Objects.NewWriter(object.WriterOptions{})
 	writer.Write(bytes[0:50])
 	writer.Write(bytes[0:50])
 	result, err := writer.Result()
-	if !objectIDsEqual(result, object.ID{StorageBlock: "X6d0bb00954ceb7fbee436bb55a8397a9"}) {
+	if !objectIDsEqual(result, object.ID{StorageBlock: "6d0bb00954ceb7fbee436bb55a8397a9"}) {
 		t.Errorf("unexpected result: %v err: %v", result, err)
 	}
 }
@@ -321,9 +317,7 @@ func TestEndToEndReadAndSeek(t *testing.T) {
 		randomData := make([]byte, size)
 		cryptorand.Read(randomData)
 
-		writer := repo.Objects.NewWriter(object.WriterOptions{
-			BlockNamePrefix: "X",
-		})
+		writer := repo.Objects.NewWriter(object.WriterOptions{})
 		writer.Write(randomData)
 		objectID, err := writer.Result()
 		writer.Close()

snapshot/upload.go

@@ -221,9 +221,8 @@ func (u *Uploader) uploadDir(dir fs.Directory) (object.ID, object.ID, error) {
 	var err error
 
 	mw := u.repo.Objects.NewWriter(object.WriterOptions{
-		Description:     "HASHCACHE:" + dir.Metadata().Name,
-		BlockNamePrefix: "H",
-		PackGroup:       "HC",
+		Description: "HASHCACHE:" + dir.Metadata().Name,
+		PackGroup:   "HC",
 	})
 	defer mw.Close()
 	u.cacheWriter = hashcache.NewWriter(mw)