From 3a02ef24e09a6ead82d73ff52eaf0854312a8e5b Mon Sep 17 00:00:00 2001 From: Jarek Kowalski Date: Sat, 5 May 2018 10:34:39 -0700 Subject: [PATCH] retire block.ContentID and replace with string --- block/block_manager.go | 53 ++++++++++------------- block/block_manager_test.go | 16 +++---- block/committed_block_index.go | 6 +-- block/inmemory_committed_block_index.go | 6 +-- block/simple_committed_block_index.go | 4 +- cli/command_block_list.go | 2 +- cli/command_block_rm.go | 5 +-- cli/command_block_show.go | 5 +-- cli/command_object_ls.go | 3 +- internal/packindex/builder.go | 8 ++-- internal/packindex/content_id_to_bytes.go | 10 ++--- internal/packindex/index.go | 16 +++---- internal/packindex/info.go | 5 +-- internal/packindex/merged.go | 6 +-- internal/packindex/merged_test.go | 4 +- internal/packindex/packindex_test.go | 8 ++-- manifest/manifest_manager.go | 12 ++--- object/object_manager.go | 10 ++--- object/object_manager_test.go | 16 +++---- object/object_writer.go | 11 +++-- object/objectid.go | 8 ++-- tests/stress_test/stress_test.go | 2 +- 22 files changed, 102 insertions(+), 114 deletions(-) diff --git a/block/block_manager.go b/block/block_manager.go index 799b1fdd1..d4217edfb 100644 --- a/block/block_manager.go +++ b/block/block_manager.go @@ -43,11 +43,6 @@ const ( // Info is an information about a single block managed by Manager. type Info = packindex.Info -// ContentID uniquely identifies a block of content stored in repository. -// It consists of optional one-character prefix (which can't be 0..9 or a..f) followed by hexa-decimal -// digits representing hash of the content. -type ContentID = packindex.ContentID - // PhysicalBlockID identifies physical storage block. type PhysicalBlockID = packindex.PhysicalBlockID @@ -69,9 +64,9 @@ type Manager struct { locked bool checkInvariantsOnUnlock bool - currentPackItems map[ContentID]Info // blocks that are in the pack block currently being built (all inline) - currentPackDataLength int // total length of all items in the current pack block - packIndexBuilder packindex.Builder // blocks that are in index currently being built (current pack and all packs saved but not committed) + currentPackItems map[string]Info // blocks that are in the pack block currently being built (all inline) + currentPackDataLength int // total length of all items in the current pack block + packIndexBuilder packindex.Builder // blocks that are in index currently being built (current pack and all packs saved but not committed) committedBlocks committedBlockIndex flushPackIndexesAfter time.Time // time when those indexes should be flushed @@ -93,7 +88,7 @@ type Manager struct { // NOTE: To avoid race conditions only blocks that cannot be possibly re-created // should ever be deleted. That means that contents of such blocks should include some element // of randomness or a contemporaneous timestamp that will never reappear. -func (bm *Manager) DeleteBlock(blockID ContentID) error { +func (bm *Manager) DeleteBlock(blockID string) error { bm.lock() defer bm.unlock() @@ -121,7 +116,7 @@ func (bm *Manager) setPendingBlock(i Info) { bm.currentPackItems[i.BlockID] = i } -func (bm *Manager) addToPackLocked(ctx context.Context, blockID ContentID, data []byte) error { +func (bm *Manager) addToPackLocked(ctx context.Context, blockID string, data []byte) error { bm.assertLocked() bm.currentPackDataLength += len(data) @@ -234,7 +229,7 @@ func (bm *Manager) invariantViolated(msg string, arg ...interface{}) { } func (bm *Manager) startPackIndexLocked() { - bm.currentPackItems = make(map[ContentID]Info) + bm.currentPackItems = make(map[string]Info) bm.currentPackDataLength = 0 } @@ -306,13 +301,13 @@ func (bm *Manager) writePackBlockLocked(ctx context.Context) error { return nil } -func (bm *Manager) preparePackDataBlock() ([]byte, map[ContentID]Info, error) { +func (bm *Manager) preparePackDataBlock() ([]byte, map[string]Info, error) { blockData, err := appendRandomBytes(nil, rand.Intn(bm.maxPreambleLength-bm.minPreambleLength+1)+bm.minPreambleLength) if err != nil { return nil, nil, fmt.Errorf("unable to prepare block preamble: %v", err) } - pending := map[ContentID]Info{} + pending := map[string]Info{} for blockID, info := range bm.currentPackItems { if info.Deleted { continue @@ -351,7 +346,7 @@ func (bm *Manager) preparePackDataBlock() ([]byte, map[ContentID]Info, error) { } -func (bm *Manager) maybeEncryptBlockDataForPacking(data []byte, blockID ContentID) ([]byte, error) { +func (bm *Manager) maybeEncryptBlockDataForPacking(data []byte, blockID string) ([]byte, error) { if bm.writeFormatVersion == 0 { // in v0 the entire block is encrypted together later on return data, nil @@ -510,14 +505,14 @@ func (bm *Manager) CompactIndexes(ctx context.Context) error { } // ListBlocks returns the metadata about blocks with a given prefix and kind. -func (bm *Manager) ListBlocks(prefix ContentID) ([]Info, error) { +func (bm *Manager) ListBlocks(prefix string) ([]Info, error) { bm.lock() defer bm.unlock() var result []Info appendToResult := func(i Info) error { - if i.Deleted || !strings.HasPrefix(string(i.BlockID), string(prefix)) { + if i.Deleted || !strings.HasPrefix(i.BlockID, prefix) { return nil } if bi, ok := bm.packIndexBuilder[i.BlockID]; ok && bi.Deleted { @@ -587,11 +582,11 @@ func (bm *Manager) Flush(ctx context.Context) error { // WriteBlock saves a given block of data to a pack group with a provided name and returns a blockID // that's based on the contents of data written. -func (bm *Manager) WriteBlock(ctx context.Context, data []byte, prefix ContentID) (ContentID, error) { +func (bm *Manager) WriteBlock(ctx context.Context, data []byte, prefix string) (string, error) { if err := validatePrefix(prefix); err != nil { return "", err } - blockID := prefix + ContentID(hex.EncodeToString(bm.hashData(data))) + blockID := prefix + hex.EncodeToString(bm.hashData(data)) bm.lock() defer bm.unlock() @@ -609,7 +604,7 @@ func (bm *Manager) WriteBlock(ctx context.Context, data []byte, prefix ContentID return blockID, err } -func validatePrefix(prefix ContentID) error { +func validatePrefix(prefix string) error { switch len(prefix) { case 0: return nil @@ -672,7 +667,7 @@ func (bm *Manager) hashData(data []byte) []byte { return blockID } -func (bm *Manager) getPendingBlockLocked(blockID ContentID) ([]byte, error) { +func (bm *Manager) getPendingBlockLocked(blockID string) ([]byte, error) { bm.assertLocked() bi, ok := bm.currentPackItems[blockID] @@ -684,7 +679,7 @@ func (bm *Manager) getPendingBlockLocked(blockID ContentID) ([]byte, error) { } // GetBlock gets the contents of a given block. If the block is not found returns blob.ErrBlockNotFound. -func (bm *Manager) GetBlock(ctx context.Context, blockID ContentID) ([]byte, error) { +func (bm *Manager) GetBlock(ctx context.Context, blockID string) ([]byte, error) { bm.lock() defer bm.unlock() @@ -704,14 +699,14 @@ func (bm *Manager) GetIndexBlock(ctx context.Context, blockID PhysicalBlockID) ( } // BlockInfo returns information about a single block. -func (bm *Manager) BlockInfo(ctx context.Context, blockID ContentID) (Info, error) { +func (bm *Manager) BlockInfo(ctx context.Context, blockID string) (Info, error) { bm.lock() defer bm.unlock() return bm.packedBlockInfoLocked(blockID) } -func (bm *Manager) packedBlockInfoLocked(blockID ContentID) (Info, error) { +func (bm *Manager) packedBlockInfoLocked(blockID string) (Info, error) { bm.assertLocked() if bi, ok := bm.packIndexBuilder[blockID]; ok { @@ -725,7 +720,7 @@ func (bm *Manager) packedBlockInfoLocked(blockID ContentID) (Info, error) { return bm.committedBlocks.getBlock(blockID) } -func (bm *Manager) getPackedBlockInternalLocked(ctx context.Context, blockID ContentID) ([]byte, error) { +func (bm *Manager) getPackedBlockInternalLocked(ctx context.Context, blockID string) ([]byte, error) { bm.assertLocked() bi, err := bm.packedBlockInfoLocked(blockID) @@ -739,7 +734,7 @@ func (bm *Manager) getPackedBlockInternalLocked(ctx context.Context, blockID Con } packBlockID := bi.PackBlockID - payload, err := bm.cache.getBlock(ctx, string(blockID), packBlockID, int64(bi.PackOffset), int64(bi.Length)) + payload, err := bm.cache.getBlock(ctx, blockID, packBlockID, int64(bi.PackOffset), int64(bi.Length)) if err != nil { return nil, err } @@ -747,7 +742,7 @@ func (bm *Manager) getPackedBlockInternalLocked(ctx context.Context, blockID Con return bm.decryptAndVerifyPayload(bi.FormatVersion, payload, int(bi.PackOffset), blockID, packBlockID) } -func (bm *Manager) decryptAndVerifyPayload(formatVersion byte, payload []byte, offset int, blockID ContentID, packBlockID PhysicalBlockID) ([]byte, error) { +func (bm *Manager) decryptAndVerifyPayload(formatVersion byte, payload []byte, offset int, blockID string, packBlockID PhysicalBlockID) ([]byte, error) { atomic.AddInt32(&bm.stats.ReadBlocks, 1) atomic.AddInt64(&bm.stats.ReadBytes, int64(len(payload))) @@ -818,8 +813,8 @@ func (bm *Manager) getPhysicalBlockInternal(ctx context.Context, blockID Physica return payload, nil } -func getPackedBlockIV(s ContentID) ([]byte, error) { - return hex.DecodeString(string(s[len(s)-(aes.BlockSize*2):])) +func getPackedBlockIV(blockID string) ([]byte, error) { + return hex.DecodeString(blockID[len(blockID)-(aes.BlockSize*2):]) } func getPhysicalBlockIV(b PhysicalBlockID) ([]byte, error) { @@ -952,7 +947,7 @@ func newManagerWithOptions(ctx context.Context, st storage.Storage, f Formatting flushPackIndexesAfter: timeNow().Add(flushPackIndexTimeout), maxPackSize: f.MaxPackSize, formatter: formatter, - currentPackItems: make(map[ContentID]Info), + currentPackItems: make(map[string]Info), packIndexBuilder: packindex.NewBuilder(), committedBlocks: cbi, minPreambleLength: defaultMinPreambleLength, diff --git a/block/block_manager_test.go b/block/block_manager_test.go index ec4302947..971260ba8 100644 --- a/block/block_manager_test.go +++ b/block/block_manager_test.go @@ -146,7 +146,7 @@ func TestBlockManagerEmpty(t *testing.T) { keyTime := map[string]time.Time{} bm := newTestBlockManager(data, keyTime, nil) - noSuchBlockID := ContentID(md5hash([]byte("foo"))) + noSuchBlockID := string(md5hash([]byte("foo"))) b, err := bm.GetBlock(ctx, noSuchBlockID) if err != storage.ErrBlockNotFound { @@ -220,7 +220,7 @@ func TestBlockManagerWriteMultiple(t *testing.T) { keyTime := map[string]time.Time{} bm := newTestBlockManager(data, keyTime, nil) - var blockIDs []ContentID + var blockIDs []string for i := 0; i < 5000; i++ { //t.Logf("i=%v", i) @@ -439,7 +439,7 @@ func verifyVersionCompat(t *testing.T, writeVersion int) { mgr := newTestBlockManager(data, keyTime, nil) mgr.writeFormatVersion = int32(writeVersion) - dataSet := map[ContentID][]byte{} + dataSet := map[string][]byte{} for i := 0; i < 3000000; i = (i + 1) * 2 { data := make([]byte, i) @@ -487,7 +487,7 @@ func verifyVersionCompat(t *testing.T, writeVersion int) { verifyBlockManagerDataSet(ctx, t, mgr, dataSet) } -func verifyBlockManagerDataSet(ctx context.Context, t *testing.T, mgr *Manager, dataSet map[ContentID][]byte) { +func verifyBlockManagerDataSet(ctx context.Context, t *testing.T, mgr *Manager, dataSet map[string][]byte) { for blockID, originalPayload := range dataSet { v, err := mgr.GetBlock(ctx, blockID) if err != nil { @@ -546,7 +546,7 @@ func fakeTimeNowWithAutoAdvance(t time.Time, dt time.Duration) func() time.Time } } -func verifyBlockNotFound(ctx context.Context, t *testing.T, bm *Manager, blockID ContentID) { +func verifyBlockNotFound(ctx context.Context, t *testing.T, bm *Manager, blockID string) { t.Helper() b, err := bm.GetBlock(ctx, blockID) @@ -555,7 +555,7 @@ func verifyBlockNotFound(ctx context.Context, t *testing.T, bm *Manager, blockID } } -func verifyBlock(ctx context.Context, t *testing.T, bm *Manager, blockID ContentID, b []byte) { +func verifyBlock(ctx context.Context, t *testing.T, bm *Manager, blockID string, b []byte) { t.Helper() b2, err := bm.GetBlock(ctx, blockID) @@ -578,7 +578,7 @@ func verifyBlock(ctx context.Context, t *testing.T, bm *Manager, blockID Content } } -func writeBlockAndVerify(ctx context.Context, t *testing.T, bm *Manager, b []byte) ContentID { +func writeBlockAndVerify(ctx context.Context, t *testing.T, bm *Manager, b []byte) string { t.Helper() blockID, err := bm.WriteBlock(ctx, b, "") @@ -586,7 +586,7 @@ func writeBlockAndVerify(ctx context.Context, t *testing.T, bm *Manager, b []byt t.Errorf("err: %v", err) } - if got, want := blockID, ContentID(md5hash(b)); got != want { + if got, want := blockID, string(md5hash(b)); got != want { t.Errorf("invalid block ID for %x, got %v, want %v", b, got, want) } diff --git a/block/committed_block_index.go b/block/committed_block_index.go index dff258e19..af7cf1885 100644 --- a/block/committed_block_index.go +++ b/block/committed_block_index.go @@ -5,16 +5,16 @@ import ( ) type committedBlockIndex interface { - getBlock(blockID ContentID) (Info, error) + getBlock(blockID string) (Info, error) commit(indexBlockID PhysicalBlockID, pendingBlocks packindex.Builder) error load(indexBlockID PhysicalBlockID, data []byte) (int, error) - listBlocks(prefix ContentID, cb func(i Info) error) error + listBlocks(prefix string, cb func(i Info) error) error hasIndexBlockID(indexBlockID PhysicalBlockID) (bool, error) } func newCommittedBlockIndex() committedBlockIndex { return &inMemoryCommittedBlockIndex{ - blocks: make(map[ContentID]Info), + blocks: make(map[string]Info), physicalBlocks: make(map[PhysicalBlockID]bool), } } diff --git a/block/inmemory_committed_block_index.go b/block/inmemory_committed_block_index.go index 93170d806..7e864b02a 100644 --- a/block/inmemory_committed_block_index.go +++ b/block/inmemory_committed_block_index.go @@ -11,11 +11,11 @@ import ( type inMemoryCommittedBlockIndex struct { mu sync.Mutex - blocks map[ContentID]Info + blocks map[string]Info physicalBlocks map[PhysicalBlockID]bool } -func (b *inMemoryCommittedBlockIndex) getBlock(blockID ContentID) (Info, error) { +func (b *inMemoryCommittedBlockIndex) getBlock(blockID string) (Info, error) { b.mu.Lock() defer b.mu.Unlock() @@ -66,7 +66,7 @@ func (b *inMemoryCommittedBlockIndex) load(indexBlockID PhysicalBlockID, data [] return updated, nil } -func (b *inMemoryCommittedBlockIndex) listBlocks(prefix ContentID, cb func(i Info) error) error { +func (b *inMemoryCommittedBlockIndex) listBlocks(prefix string, cb func(i Info) error) error { b.mu.Lock() defer b.mu.Unlock() diff --git a/block/simple_committed_block_index.go b/block/simple_committed_block_index.go index 713a05893..ebcac4d96 100644 --- a/block/simple_committed_block_index.go +++ b/block/simple_committed_block_index.go @@ -23,7 +23,7 @@ type simpleCommittedBlockIndex struct { merged packindex.Merged } -func (b *simpleCommittedBlockIndex) getBlock(blockID ContentID) (Info, error) { +func (b *simpleCommittedBlockIndex) getBlock(blockID string) (Info, error) { info, err := b.merged.GetInfo(blockID) if info != nil { return *info, nil @@ -89,7 +89,7 @@ func (b *simpleCommittedBlockIndex) load(indexBlockID PhysicalBlockID, data []by return 0, nil } -func (b *simpleCommittedBlockIndex) listBlocks(prefix ContentID, cb func(i Info) error) error { +func (b *simpleCommittedBlockIndex) listBlocks(prefix string, cb func(i Info) error) error { return b.merged.Iterate(prefix, cb) } diff --git a/cli/command_block_list.go b/cli/command_block_list.go index 449d7395b..b80aeb34b 100644 --- a/cli/command_block_list.go +++ b/cli/command_block_list.go @@ -19,7 +19,7 @@ var ( ) func runListBlocksAction(ctx context.Context, rep *repo.Repository) error { - blocks, err := rep.Blocks.ListBlocks(block.ContentID(*blockListPrefix)) + blocks, err := rep.Blocks.ListBlocks(*blockListPrefix) if err != nil { return err } diff --git a/cli/command_block_rm.go b/cli/command_block_rm.go index 695847051..5e4d6835f 100644 --- a/cli/command_block_rm.go +++ b/cli/command_block_rm.go @@ -3,7 +3,6 @@ package cli import ( "context" - "github.com/kopia/kopia/block" "github.com/kopia/kopia/repo" ) @@ -15,7 +14,7 @@ var ( func runRemoveBlockCommand(ctx context.Context, rep *repo.Repository) error { for _, blockID := range *removeBlockIDs { - if err := removeBlock(rep, block.ContentID(blockID)); err != nil { + if err := removeBlock(rep, blockID); err != nil { return err } } @@ -23,7 +22,7 @@ func runRemoveBlockCommand(ctx context.Context, rep *repo.Repository) error { return nil } -func removeBlock(r *repo.Repository, blockID block.ContentID) error { +func removeBlock(r *repo.Repository, blockID string) error { return r.Blocks.DeleteBlock(blockID) } diff --git a/cli/command_block_show.go b/cli/command_block_show.go index 9d4352809..019dc4def 100644 --- a/cli/command_block_show.go +++ b/cli/command_block_show.go @@ -4,7 +4,6 @@ import ( "bytes" "context" - "github.com/kopia/kopia/block" "github.com/kopia/kopia/repo" ) @@ -16,7 +15,7 @@ var ( func runShowBlockCommand(ctx context.Context, rep *repo.Repository) error { for _, blockID := range *showBlockIDs { - if err := showBlock(ctx, rep, block.ContentID(blockID)); err != nil { + if err := showBlock(ctx, rep, blockID); err != nil { return err } } @@ -24,7 +23,7 @@ func runShowBlockCommand(ctx context.Context, rep *repo.Repository) error { return nil } -func showBlock(ctx context.Context, r *repo.Repository, blockID block.ContentID) error { +func showBlock(ctx context.Context, r *repo.Repository, blockID string) error { data, err := r.Blocks.GetBlock(ctx, blockID) if err != nil { return err diff --git a/cli/command_object_ls.go b/cli/command_object_ls.go index 66ddba92f..99b9009ea 100644 --- a/cli/command_object_ls.go +++ b/cli/command_object_ls.go @@ -4,7 +4,6 @@ import ( "context" "fmt" - "github.com/kopia/kopia/block" "github.com/kopia/kopia/repo" ) @@ -14,7 +13,7 @@ var ( ) func runListObjectsAction(ctx context.Context, rep *repo.Repository) error { - info, err := rep.Blocks.ListBlocks(block.ContentID(*objectListPrefix)) + info, err := rep.Blocks.ListBlocks(*objectListPrefix) if err != nil { return err } diff --git a/internal/packindex/builder.go b/internal/packindex/builder.go index 50fa2f4ce..30e5df7e7 100644 --- a/internal/packindex/builder.go +++ b/internal/packindex/builder.go @@ -9,7 +9,7 @@ import ( ) // Builder prepares and writes block index for writing. -type Builder map[ContentID]*Info +type Builder map[string]*Info // Add adds a new entry to the builder or conditionally replaces it if the timestamp is greater. func (b Builder) Add(i Info) { @@ -35,7 +35,7 @@ func (b Builder) sortedBlocks() []*Info { type indexLayout struct { packBlockIDOffsets map[PhysicalBlockID]uint32 - payloadOffsets map[ContentID]uint32 + payloadOffsets map[string]uint32 entryCount int keyLength int entryLength int @@ -47,7 +47,7 @@ func (b Builder) Build(output io.Writer) error { allBlocks := b.sortedBlocks() layout := &indexLayout{ packBlockIDOffsets: map[PhysicalBlockID]uint32{}, - payloadOffsets: map[ContentID]uint32{}, + payloadOffsets: map[string]uint32{}, keyLength: -1, entryLength: 20, entryCount: len(allBlocks), @@ -159,5 +159,5 @@ func formatEntry(entry []byte, it *Info, layout *indexLayout) error { // NewBuilder creates a new Builder. func NewBuilder() Builder { - return make(map[ContentID]*Info) + return make(map[string]*Info) } diff --git a/internal/packindex/content_id_to_bytes.go b/internal/packindex/content_id_to_bytes.go index bc82787d2..bc123a2a6 100644 --- a/internal/packindex/content_id_to_bytes.go +++ b/internal/packindex/content_id_to_bytes.go @@ -4,22 +4,22 @@ import ( "encoding/hex" ) -func bytesToContentID(b []byte) ContentID { +func bytesToContentID(b []byte) string { if len(b) == 0 { return "" } if b[0] == 0xff { - return ContentID(b[1:]) + return string(b[1:]) } prefix := "" if b[0] != 0 { prefix = string(b[0:1]) } - return ContentID(prefix + hex.EncodeToString(b[1:])) + return prefix + hex.EncodeToString(b[1:]) } -func contentIDToBytes(c ContentID) []byte { +func contentIDToBytes(c string) []byte { var prefix []byte if len(c)%2 == 1 { prefix = []byte(c[0:1]) @@ -28,7 +28,7 @@ func contentIDToBytes(c ContentID) []byte { prefix = []byte{0} } - b, err := hex.DecodeString(string(c)) + b, err := hex.DecodeString(c) if err != nil { return append([]byte{0xff}, []byte(c)...) } diff --git a/internal/packindex/index.go b/internal/packindex/index.go index 90e3c33d2..de3a2778e 100644 --- a/internal/packindex/index.go +++ b/internal/packindex/index.go @@ -15,8 +15,8 @@ type Index interface { io.Closer EntryCount() int - GetInfo(blockID ContentID) (*Info, error) - Iterate(prefix ContentID, cb func(Info) error) error + GetInfo(blockID string) (*Info, error) + Iterate(prefix string, cb func(Info) error) error } type index struct { @@ -57,7 +57,7 @@ func (b *index) EntryCount() int { // Iterate invokes the provided callback function for all blocks in the index, sorted alphabetically. // The iteration ends when the callback returns an error, which is propagated to the caller or when // all blocks have been visited. -func (b *index) Iterate(prefix ContentID, cb func(Info) error) error { +func (b *index) Iterate(prefix string, cb func(Info) error) error { b.mu.Lock() defer b.mu.Unlock() @@ -80,7 +80,7 @@ func (b *index) Iterate(prefix ContentID, cb func(Info) error) error { if err != nil { return fmt.Errorf("invalid index data: %v", err) } - if !strings.HasPrefix(string(i.BlockID), string(prefix)) { + if !strings.HasPrefix(i.BlockID, prefix) { break } if err := cb(i); err != nil { @@ -90,7 +90,7 @@ func (b *index) Iterate(prefix ContentID, cb func(Info) error) error { return nil } -func (b *index) findEntryPosition(blockID ContentID) (int, error) { +func (b *index) findEntryPosition(blockID string) (int, error) { stride := b.hdr.keySize + b.hdr.valueSize entryBuf := make([]byte, stride) var readErr error @@ -110,7 +110,7 @@ func (b *index) findEntryPosition(blockID ContentID) (int, error) { return pos, readErr } -func (b *index) findEntry(blockID ContentID) ([]byte, error) { +func (b *index) findEntry(blockID string) ([]byte, error) { key := contentIDToBytes(blockID) if len(key) != b.hdr.keySize { return nil, fmt.Errorf("invalid block ID: %q", blockID) @@ -138,7 +138,7 @@ func (b *index) findEntry(blockID ContentID) ([]byte, error) { } // GetInfo returns information about a given block. If a block is not found, nil is returned. -func (b *index) GetInfo(blockID ContentID) (*Info, error) { +func (b *index) GetInfo(blockID string) (*Info, error) { b.mu.Lock() defer b.mu.Unlock() @@ -158,7 +158,7 @@ func (b *index) GetInfo(blockID ContentID) (*Info, error) { return &i, err } -func (b *index) entryToInfo(blockID ContentID, entryData []byte) (Info, error) { +func (b *index) entryToInfo(blockID string, entryData []byte) (Info, error) { if len(entryData) < 20 { return Info{}, fmt.Errorf("invalid entry length: %v", len(entryData)) } diff --git a/internal/packindex/info.go b/internal/packindex/info.go index a58ab9043..8e126b9ca 100644 --- a/internal/packindex/info.go +++ b/internal/packindex/info.go @@ -5,13 +5,12 @@ import ( "time" ) -type ContentID string - +// PhysicalBlockID is the name of a physical block in storage. type PhysicalBlockID string // Info is an information about a single block managed by Manager. type Info struct { - BlockID ContentID `json:"blockID"` + BlockID string `json:"blockID"` Length uint32 `json:"length"` TimestampSeconds int64 `json:"time"` PackBlockID PhysicalBlockID `json:"packBlockID,omitempty"` diff --git a/internal/packindex/merged.go b/internal/packindex/merged.go index 74d40be36..d54b739d1 100644 --- a/internal/packindex/merged.go +++ b/internal/packindex/merged.go @@ -29,7 +29,7 @@ func (m Merged) EntryCount() int { } // GetInfo returns information about a single block. If a block is not found, returns (nil,nil) -func (m Merged) GetInfo(contentID ContentID) (*Info, error) { +func (m Merged) GetInfo(contentID string) (*Info, error) { var best *Info for _, ndx := range m { @@ -67,7 +67,7 @@ func (h *nextInfoHeap) Pop() interface{} { return x } -func iterateChan(prefix ContentID, ndx Index, done chan bool) <-chan Info { +func iterateChan(prefix string, ndx Index, done chan bool) <-chan Info { ch := make(chan Info) go func() { defer close(ch) @@ -86,7 +86,7 @@ func iterateChan(prefix ContentID, ndx Index, done chan bool) <-chan Info { // Iterate invokes the provided callback for all unique block IDs in the underlying sources until either // all blocks have been visited or until an error is returned by the callback. -func (m Merged) Iterate(prefix ContentID, cb func(i Info) error) error { +func (m Merged) Iterate(prefix string, cb func(i Info) error) error { var minHeap nextInfoHeap done := make(chan bool) defer close(done) diff --git a/internal/packindex/merged_test.go b/internal/packindex/merged_test.go index e4d2fa2f3..0b05e72ff 100644 --- a/internal/packindex/merged_test.go +++ b/internal/packindex/merged_test.go @@ -44,13 +44,13 @@ func TestMerged(t *testing.T) { t.Errorf("invalid pack offset %v, wanted %v", got, want) } - var inOrder []packindex.ContentID + var inOrder []string m.Iterate("", func(i packindex.Info) error { inOrder = append(inOrder, i.BlockID) return nil }) - expectedInOrder := []packindex.ContentID{ + expectedInOrder := []string{ "aabbcc", "ddeeff", "k010203", diff --git a/internal/packindex/packindex_test.go b/internal/packindex/packindex_test.go index 583192446..7f3b8ea69 100644 --- a/internal/packindex/packindex_test.go +++ b/internal/packindex/packindex_test.go @@ -17,7 +17,7 @@ func TestPackIndex(t *testing.T) { blockNumber := 0 - deterministicBlockID := func(prefix string, id int) packindex.ContentID { + deterministicBlockID := func(prefix string, id int) string { h := sha1.New() fmt.Fprintf(h, "%v%v", prefix, id) blockNumber++ @@ -32,7 +32,7 @@ func TestPackIndex(t *testing.T) { if id%5 == 0 { prefix2 = "m" } - return packindex.ContentID(fmt.Sprintf("%v%x", prefix2, h.Sum(nil))) + return string(fmt.Sprintf("%v%x", prefix2, h.Sum(nil))) } deterministicPackBlockID := func(id int) packindex.PhysicalBlockID { h := sha1.New() @@ -95,7 +95,7 @@ func TestPackIndex(t *testing.T) { }) } - infoMap := map[packindex.ContentID]packindex.Info{} + infoMap := map[string]packindex.Info{} for _, info := range infos { infoMap[info.BlockID] = info @@ -151,7 +151,7 @@ func TestPackIndex(t *testing.T) { for _, prefix := range prefixes { cnt2 := 0 - ndx.Iterate(packindex.ContentID(prefix), func(info2 packindex.Info) error { + ndx.Iterate(string(prefix), func(info2 packindex.Info) error { cnt2++ if !strings.HasPrefix(string(info2.BlockID), string(prefix)) { t.Errorf("unexpected item %v when iterating prefix %v", info2.BlockID, prefix) diff --git a/manifest/manifest_manager.go b/manifest/manifest_manager.go index eab419648..a81944ab9 100644 --- a/manifest/manifest_manager.go +++ b/manifest/manifest_manager.go @@ -28,7 +28,7 @@ type Manager struct { mu sync.Mutex b *block.Manager entries map[string]*manifestEntry - blockIDs []block.ContentID + blockIDs []string pendingEntries []*manifestEntry } @@ -165,7 +165,7 @@ func (m *Manager) Flush(ctx context.Context) error { return nil } -func (m *Manager) flushPendingEntriesLocked(ctx context.Context) (block.ContentID, error) { +func (m *Manager) flushPendingEntriesLocked(ctx context.Context) (string, error) { if len(m.pendingEntries) == 0 { return "", nil } @@ -279,7 +279,7 @@ func (m *Manager) loadManifestBlocks(ctx context.Context, blocks []block.Info) e func (m *Manager) loadBlocksInParallel(ctx context.Context, blocks []block.Info) ([]manifest, error) { errors := make(chan error, len(blocks)) manifests := make(chan manifest, len(blocks)) - blockIDs := make(chan block.ContentID, len(blocks)) + blockIDs := make(chan string, len(blocks)) var wg sync.WaitGroup for i := 0; i < 8; i++ { @@ -290,7 +290,7 @@ func (m *Manager) loadBlocksInParallel(ctx context.Context, blocks []block.Info) for blk := range blockIDs { t1 := time.Now() man, err := m.loadManifestBlock(ctx, blk) - log.Debug().Dur("duration", time.Since(t1)).Str("blk", string(blk)).Int("worker", workerID).Err(err).Msg("manifest block loaded") + log.Debug().Dur("duration", time.Since(t1)).Str("blk", blk).Int("worker", workerID).Err(err).Msg("manifest block loaded") if err != nil { errors <- err } else { @@ -324,7 +324,7 @@ func (m *Manager) loadBlocksInParallel(ctx context.Context, blocks []block.Info) return man, nil } -func (m *Manager) loadManifestBlock(ctx context.Context, blockID block.ContentID) (manifest, error) { +func (m *Manager) loadManifestBlock(ctx context.Context, blockID string) (manifest, error) { man := manifest{} blk, err := m.b.GetBlock(ctx, blockID) if err != nil { @@ -388,7 +388,7 @@ func (m *Manager) compactLocked(ctx context.Context) error { } // all previous blocks were deleted, now we have a new block - m.blockIDs = []block.ContentID{blockID} + m.blockIDs = []string{blockID} return nil } diff --git a/object/object_manager.go b/object/object_manager.go index 1d05b0e20..035603419 100644 --- a/object/object_manager.go +++ b/object/object_manager.go @@ -24,9 +24,9 @@ type Reader interface { } type blockManager interface { - BlockInfo(ctx context.Context, blockID block.ContentID) (block.Info, error) - GetBlock(ctx context.Context, blockID block.ContentID) ([]byte, error) - WriteBlock(ctx context.Context, data []byte, prefix block.ContentID) (block.ContentID, error) + BlockInfo(ctx context.Context, blockID string) (block.Info, error) + GetBlock(ctx context.Context, blockID string) ([]byte, error) + WriteBlock(ctx context.Context, data []byte, prefix string) (string, error) Flush(ctx context.Context) error } @@ -102,7 +102,7 @@ func (om *Manager) Open(ctx context.Context, objectID ID) (Reader, error) { // VerifyObject ensures that all objects backing ObjectID are present in the repository // and returns the total length of the object and storage blocks of which it is composed. -func (om *Manager) VerifyObject(ctx context.Context, oid ID) (int64, []block.ContentID, error) { +func (om *Manager) VerifyObject(ctx context.Context, oid ID) (int64, []string, error) { // Flush any pending writes. om.writeBackWG.Wait() @@ -157,7 +157,7 @@ func (om *Manager) verifyObjectInternal(ctx context.Context, oid ID, blocks *blo blocks.addBlock(oid.ContentBlockID) if p.PackBlockID != "" { - l, err := om.verifyObjectInternal(ctx, ID{ContentBlockID: block.ContentID(p.PackBlockID)}, blocks) + l, err := om.verifyObjectInternal(ctx, ID{ContentBlockID: string(p.PackBlockID)}, blocks) if err != nil { return 0, err } diff --git a/object/object_manager_test.go b/object/object_manager_test.go index ef3188000..6337e9a70 100644 --- a/object/object_manager_test.go +++ b/object/object_manager_test.go @@ -24,10 +24,10 @@ import ( type fakeBlockManager struct { mu sync.Mutex - data map[block.ContentID][]byte + data map[string][]byte } -func (f *fakeBlockManager) GetBlock(ctx context.Context, blockID block.ContentID) ([]byte, error) { +func (f *fakeBlockManager) GetBlock(ctx context.Context, blockID string) ([]byte, error) { f.mu.Lock() defer f.mu.Unlock() @@ -38,10 +38,10 @@ func (f *fakeBlockManager) GetBlock(ctx context.Context, blockID block.ContentID return nil, storage.ErrBlockNotFound } -func (f *fakeBlockManager) WriteBlock(ctx context.Context, data []byte, prefix block.ContentID) (block.ContentID, error) { +func (f *fakeBlockManager) WriteBlock(ctx context.Context, data []byte, prefix string) (string, error) { h := md5.New() h.Write(data) - blockID := prefix + block.ContentID(hex.EncodeToString(h.Sum(nil))) + blockID := prefix + string(hex.EncodeToString(h.Sum(nil))) f.mu.Lock() defer f.mu.Unlock() @@ -50,7 +50,7 @@ func (f *fakeBlockManager) WriteBlock(ctx context.Context, data []byte, prefix b return blockID, nil } -func (f *fakeBlockManager) BlockInfo(ctx context.Context, blockID block.ContentID) (block.Info, error) { +func (f *fakeBlockManager) BlockInfo(ctx context.Context, blockID string) (block.Info, error) { f.mu.Lock() defer f.mu.Unlock() @@ -65,11 +65,11 @@ func (f *fakeBlockManager) Flush(ctx context.Context) error { return nil } -func setupTest(t *testing.T) (map[block.ContentID][]byte, *Manager) { - return setupTestWithData(t, map[block.ContentID][]byte{}, ManagerOptions{}) +func setupTest(t *testing.T) (map[string][]byte, *Manager) { + return setupTestWithData(t, map[string][]byte{}, ManagerOptions{}) } -func setupTestWithData(t *testing.T, data map[block.ContentID][]byte, opts ManagerOptions) (map[block.ContentID][]byte, *Manager) { +func setupTestWithData(t *testing.T, data map[string][]byte, opts ManagerOptions) (map[string][]byte, *Manager) { r, err := NewObjectManager(context.Background(), &fakeBlockManager{data: data}, config.RepositoryObjectFormat{ FormattingOptions: block.FormattingOptions{ Version: 1, diff --git a/object/object_writer.go b/object/object_writer.go index 7df528187..ed4ef6a84 100644 --- a/object/object_writer.go +++ b/object/object_writer.go @@ -7,7 +7,6 @@ import ( "io" "sync" - "github.com/kopia/kopia/block" "github.com/kopia/kopia/internal/jsonstream" ) @@ -21,24 +20,24 @@ type Writer interface { type blockTracker struct { mu sync.Mutex - blocks map[block.ContentID]bool + blocks map[string]bool } -func (t *blockTracker) addBlock(blockID block.ContentID) { +func (t *blockTracker) addBlock(blockID string) { t.mu.Lock() defer t.mu.Unlock() if t.blocks == nil { - t.blocks = make(map[block.ContentID]bool) + t.blocks = make(map[string]bool) } t.blocks[blockID] = true } -func (t *blockTracker) blockIDs() []block.ContentID { +func (t *blockTracker) blockIDs() []string { t.mu.Lock() defer t.mu.Unlock() - result := make([]block.ContentID, 0, len(t.blocks)) + result := make([]string, 0, len(t.blocks)) for k := range t.blocks { result = append(result, k) } diff --git a/object/objectid.go b/object/objectid.go index 665d667e2..c8e5cedab 100644 --- a/object/objectid.go +++ b/object/objectid.go @@ -4,8 +4,6 @@ import ( "encoding/json" "fmt" - - "github.com/kopia/kopia/block" ) // ID is an identifier of a repository object. Repository objects can be stored: @@ -14,7 +12,7 @@ import ( // 2. In a series of content blocks with an indirect block pointing at them (multiple indirections are allowed). This is used for larger files. // type ID struct { - ContentBlockID block.ContentID + ContentBlockID string Indirect *ID } @@ -53,7 +51,7 @@ func (oid ID) String() string { } if oid.ContentBlockID != "" { - return "D" + string(oid.ContentBlockID) + return "D" + oid.ContentBlockID } return "B" @@ -98,7 +96,7 @@ func ParseID(s string) (ID, error) { return ID{Indirect: &base}, nil } - return ID{ContentBlockID: block.ContentID(content)}, nil + return ID{ContentBlockID: content}, nil } } diff --git a/tests/stress_test/stress_test.go b/tests/stress_test/stress_test.go index 45eed7bc6..4f1048c0e 100644 --- a/tests/stress_test/stress_test.go +++ b/tests/stress_test/stress_test.go @@ -72,7 +72,7 @@ func stressWorker(ctx context.Context, t *testing.T, workerID int, openMgr func( } type writtenBlock struct { - contentID block.ContentID + contentID string data []byte }