diff --git a/block/block_manager.go b/block/block_manager.go index cf2ceea14..44e769430 100644 --- a/block/block_manager.go +++ b/block/block_manager.go @@ -110,7 +110,6 @@ func (bm *Manager) DeleteBlock(blockID ContentID) error { // Add deletion to current pack. bm.currentPackIndex.deleteBlock(blockID) bm.blockIDToIndex[blockID] = bm.currentPackIndex - return nil } @@ -178,11 +177,12 @@ func (bm *Manager) verifyInvariantsLocked() { func (bm *Manager) verifyPendingPackIndexesAreRegisteredLocked() { // each pending pack index is registered for _, p := range bm.pendingPackIndexes { - for _, blkID := range p.activeBlockIDs() { - if _, ok := bm.blockIDToIndex[blkID]; !ok { - bm.invariantViolated("invariant violated - pending block %q not in index", blkID) + _ = p.iterate(func(blockID ContentID, info packBlockInfo) error { + if _, ok := bm.blockIDToIndex[blockID]; !ok { + bm.invariantViolated("invariant violated - pending block %q not in index", blockID) } - } + return nil + }) } } @@ -261,7 +261,7 @@ func (bm *Manager) writePackIndexes(ctx context.Context, ndx []packIndex, isComp } func (bm *Manager) finishPackLocked(ctx context.Context) error { - if !bm.currentPackIndex.isEmpty() { + if !isIndexEmpty(bm.currentPackIndex) { log.Debug().Msg("finishing pack") if len(bm.currentPackData) < bm.maxInlineContentLength { bm.currentPackIndex.packedToInline(bm.currentPackData) @@ -414,7 +414,6 @@ func (bm *Manager) initializeIndexes(ctx context.Context) error { bm.blockIDToIndex, bm.packBlockIDToIndex = dedupeBlockIDsAndIndex(merged) if len(blockIDs) >= autoCompactionBlockCount { log.Debug().Msgf("auto compacting block indexes (block count %v exceeds threshold of %v)", len(blockIDs), autoCompactionBlockCount) - merged = removeEmptyIndexes(merged) if _, err := bm.writePackIndexes(ctx, merged, true); err != nil { return err } @@ -434,28 +433,15 @@ func dedupeBlockIDsAndIndex(ndx []packIndex) (blockToIndex map[ContentID]packInd packToIndex = make(map[PhysicalBlockID]packIndex) for _, pck := range ndx { packToIndex[pck.packBlockID()] = pck - for _, blockID := range pck.activeBlockIDs() { + _ = pck.iterate(func(blockID ContentID, _ packBlockInfo) error { blockToIndex[blockID] = pck - } - for _, blockID := range pck.deletedBlockIDs() { - blockToIndex[blockID] = pck - } + return nil + }) } return } -func removeEmptyIndexes(ndx []packIndex) []packIndex { - var res []packIndex - for _, n := range ndx { - if !n.isEmpty() { - res = append(res, n) - } - } - - return res -} - // CompactIndexes performs compaction of index blocks. func (bm *Manager) CompactIndexes(ctx context.Context) error { bm.lock() @@ -628,16 +614,16 @@ func (bm *Manager) repackageBlock(ctx context.Context, m packIndex, done map[Con if err != nil { return fmt.Errorf("can't fetch block %q for repackaging: %v", m.packBlockID(), err) } + return m.iterate(func(blockID ContentID, bi packBlockInfo) error { + if bi.deleted { + return nil + } - for _, blockID := range m.activeBlockIDs() { if done[blockID] { - continue + return nil } done[blockID] = true - bi, ok := m.getBlock(blockID) - if !ok { - return fmt.Errorf("unable to get packed block to repackage") - } + var payload []byte if bi.payload == nil { payload = data[bi.offset : bi.offset+bi.size] @@ -647,9 +633,9 @@ func (bm *Manager) repackageBlock(ctx context.Context, m packIndex, done map[Con if err := bm.addToPackLocked(ctx, blockID, payload); err != nil { return fmt.Errorf("unable to re-package %q: %v", blockID, err) } - } - return nil + return nil + }) } func (bm *Manager) writeUnpackedBlockNotLocked(ctx context.Context, data []byte, prefix string, suffix string) (PhysicalBlockID, error) { diff --git a/block/pack_index.go b/block/pack_index.go index 20079f3ec..2d8871956 100644 --- a/block/pack_index.go +++ b/block/pack_index.go @@ -1,6 +1,8 @@ package block import ( + "errors" + "github.com/golang/protobuf/proto" "github.com/kopia/kopia/internal/blockmgrpb" ) @@ -11,9 +13,7 @@ type packIndex interface { createTimeNanos() uint64 getBlock(blockID ContentID) (packBlockInfo, bool) - isEmpty() bool - activeBlockIDs() []ContentID - deletedBlockIDs() []ContentID + iterate(func(blockID ContentID, info packBlockInfo) error) error addToIndexes(pb *blockmgrpb.Indexes) } @@ -59,3 +59,10 @@ func unpackOffsetAndSize(os uint64) (uint32, uint32) { return offset, size } + +func isIndexEmpty(ndx packIndex) bool { + return nil == ndx.iterate( + func(blockID ContentID, bi packBlockInfo) error { + return errors.New("have items") + }) +} diff --git a/block/pack_index_test.go b/block/pack_index_test.go index 228d11603..e36660a1f 100644 --- a/block/pack_index_test.go +++ b/block/pack_index_test.go @@ -16,15 +16,6 @@ func TestPackIndexV1(t *testing.T) { } func verifyPackIndex(t *testing.T, ndx packIndexBuilder, ts time.Time) { - if got, want := ndx.isEmpty(), true; got != want { - t.Errorf("unexpected isEmpty(): %v, wanted %v", got, want) - } - if got, want := len(ndx.activeBlockIDs()), 0; got != want { - t.Errorf("unexpected number of active block IDs: %v, wanted %v", got, want) - } - if got, want := len(ndx.deletedBlockIDs()), 0; got != want { - t.Errorf("unexpected number of active block IDs: %v, wanted %v", got, want) - } if got, want := ndx.packBlockID(), PhysicalBlockID(""); got != want { t.Errorf("unexpected pack block ID: %q, wanted %q", got, want) } @@ -75,10 +66,6 @@ func verifyPackIndex(t *testing.T, ndx packIndexBuilder, ts time.Time) { verifyIndexBlockInline(t, ndx, blockID, blockData[blockID]) } - if !blockIDSlicesEqual(ndx.activeBlockIDs(), blockIDs) { - t.Errorf("unexpected active blocks: %v wanted %v", ndx.activeBlockIDs(), blockIDs) - } - ndx.deleteBlock(blockIDs[0]) verifyIndexBlockNotFound(t, ndx, blockIDs[0]) verifyIndexBlockDeleted(t, ndx, blockIDs[0]) @@ -120,28 +107,26 @@ func verifyIndexBlockDeleted(t *testing.T, ndx packIndex, blockID ContentID) { t.Helper() verifyIndexBlockNotFound(t, ndx, blockID) - var found bool - for _, b := range ndx.deletedBlockIDs() { - if b == blockID { - found = true - } + bi, ok := ndx.getBlock(blockID) + if !ok { + t.Errorf("block %q not found in index", blockID) } - if !found { - t.Errorf("expected block %q to be amongst deleted, was not found: %v", blockID, ndx.deletedBlockIDs()) + + if !bi.deleted { + t.Errorf("expected block %q to be deleted", blockID) } } func verifyIndexBlockNotDeleted(t *testing.T, ndx packIndex, blockID ContentID) { t.Helper() - var found bool - for _, b := range ndx.deletedBlockIDs() { - if b == blockID { - found = true - } + bi, ok := ndx.getBlock(blockID) + if !ok { + t.Errorf("block %q not found in index", blockID) } - if found { - t.Errorf("expected block %q to not be amongst deleted, was not found: %v", blockID, ndx.deletedBlockIDs()) + + if bi.deleted { + t.Errorf("expected block %q to not be deleted", blockID) } } diff --git a/block/pack_index_v1.go b/block/pack_index_v1.go index 25416ba62..b7543747e 100644 --- a/block/pack_index_v1.go +++ b/block/pack_index_v1.go @@ -53,13 +53,25 @@ func (p protoPackIndexV1) getBlock(blockID ContentID) (packBlockInfo, bool) { return packBlockInfo{}, false } -func (p protoPackIndexV1) deletedBlockIDs() []ContentID { - var result []ContentID - - for _, d := range p.ndx.DeletedItems { - result = append(result, ContentID(d)) +func (p protoPackIndexV1) iterate(cb func(ContentID, packBlockInfo) error) error { + for k, v := range p.ndx.Items { + offset, size := unpackOffsetAndSize(v) + if err := cb(ContentID(k), packBlockInfo{offset: offset, size: size}); err != nil { + return err + } } - return result + for k, v := range p.ndx.InlineItems { + if err := cb(ContentID(k), packBlockInfo{size: uint32(len(v)), payload: v}); err != nil { + return err + } + } + for _, k := range p.ndx.DeletedItems { + if err := cb(ContentID(k), packBlockInfo{deleted: true}); err != nil { + return err + } + } + + return nil } func (p protoPackIndexV1) packBlockID() PhysicalBlockID { @@ -80,21 +92,6 @@ func (p protoPackIndexV1) deleteBlock(blockID ContentID) { p.ndx.DeletedItems = append(p.ndx.DeletedItems, string(blockID)) } -func (p protoPackIndexV1) activeBlockIDs() []ContentID { - var result []ContentID - for blkID := range p.ndx.Items { - result = append(result, ContentID(blkID)) - } - for blkID := range p.ndx.InlineItems { - result = append(result, ContentID(blkID)) - } - return result -} - -func (p protoPackIndexV1) isEmpty() bool { - return len(p.ndx.Items)+len(p.ndx.InlineItems)+len(p.ndx.DeletedItems) == 0 -} - func newPackIndexV1(t time.Time) packIndexBuilder { return protoPackIndexV1{&blockmgrpb.IndexV1{ Items: make(map[string]uint64),