removed packInfo.isEmpty

block/block_manager.go

@@ -110,7 +110,6 @@ func (bm *Manager) DeleteBlock(blockID ContentID) error {
 	// Add deletion to current pack.
 	bm.currentPackIndex.deleteBlock(blockID)
 	bm.blockIDToIndex[blockID] = bm.currentPackIndex
-
 	return nil
 }
 
@@ -178,11 +177,12 @@ func (bm *Manager) verifyInvariantsLocked() {
 func (bm *Manager) verifyPendingPackIndexesAreRegisteredLocked() {
 	// each pending pack index is registered
 	for _, p := range bm.pendingPackIndexes {
-		for _, blkID := range p.activeBlockIDs() {
-			if _, ok := bm.blockIDToIndex[blkID]; !ok {
-				bm.invariantViolated("invariant violated - pending block %q not in index", blkID)
+		_ = p.iterate(func(blockID ContentID, info packBlockInfo) error {
+			if _, ok := bm.blockIDToIndex[blockID]; !ok {
+				bm.invariantViolated("invariant violated - pending block %q not in index", blockID)
 			}
-		}
+			return nil
+		})
 	}
 }
 
@@ -261,7 +261,7 @@ func (bm *Manager) writePackIndexes(ctx context.Context, ndx []packIndex, isComp
 }
 
 func (bm *Manager) finishPackLocked(ctx context.Context) error {
-	if !bm.currentPackIndex.isEmpty() {
+	if !isIndexEmpty(bm.currentPackIndex) {
 		log.Debug().Msg("finishing pack")
 		if len(bm.currentPackData) < bm.maxInlineContentLength {
 			bm.currentPackIndex.packedToInline(bm.currentPackData)
@@ -414,7 +414,6 @@ func (bm *Manager) initializeIndexes(ctx context.Context) error {
 	bm.blockIDToIndex, bm.packBlockIDToIndex = dedupeBlockIDsAndIndex(merged)
 	if len(blockIDs) >= autoCompactionBlockCount {
 		log.Debug().Msgf("auto compacting block indexes (block count %v exceeds threshold of %v)", len(blockIDs), autoCompactionBlockCount)
-		merged = removeEmptyIndexes(merged)
 		if _, err := bm.writePackIndexes(ctx, merged, true); err != nil {
 			return err
 		}
@@ -434,28 +433,15 @@ func dedupeBlockIDsAndIndex(ndx []packIndex) (blockToIndex map[ContentID]packInd
 	packToIndex = make(map[PhysicalBlockID]packIndex)
 	for _, pck := range ndx {
 		packToIndex[pck.packBlockID()] = pck
-		for _, blockID := range pck.activeBlockIDs() {
+		_ = pck.iterate(func(blockID ContentID, _ packBlockInfo) error {
 			blockToIndex[blockID] = pck
-		}
-		for _, blockID := range pck.deletedBlockIDs() {
-			blockToIndex[blockID] = pck
-		}
+			return nil
+		})
 	}
 
 	return
 }
 
-func removeEmptyIndexes(ndx []packIndex) []packIndex {
-	var res []packIndex
-	for _, n := range ndx {
-		if !n.isEmpty() {
-			res = append(res, n)
-		}
-	}
-
-	return res
-}
-
 // CompactIndexes performs compaction of index blocks.
 func (bm *Manager) CompactIndexes(ctx context.Context) error {
 	bm.lock()
@@ -628,16 +614,16 @@ func (bm *Manager) repackageBlock(ctx context.Context, m packIndex, done map[Con
 	if err != nil {
 		return fmt.Errorf("can't fetch block %q for repackaging: %v", m.packBlockID(), err)
 	}
+	return m.iterate(func(blockID ContentID, bi packBlockInfo) error {
+		if bi.deleted {
+			return nil
+		}
 
-	for _, blockID := range m.activeBlockIDs() {
 		if done[blockID] {
-			continue
+			return nil
 		}
 		done[blockID] = true
-		bi, ok := m.getBlock(blockID)
-		if !ok {
-			return fmt.Errorf("unable to get packed block to repackage")
-		}
+
 		var payload []byte
 		if bi.payload == nil {
 			payload = data[bi.offset : bi.offset+bi.size]
@@ -647,9 +633,9 @@ func (bm *Manager) repackageBlock(ctx context.Context, m packIndex, done map[Con
 		if err := bm.addToPackLocked(ctx, blockID, payload); err != nil {
 			return fmt.Errorf("unable to re-package %q: %v", blockID, err)
 		}
-	}
 
-	return nil
+		return nil
+	})
 }
 
 func (bm *Manager) writeUnpackedBlockNotLocked(ctx context.Context, data []byte, prefix string, suffix string) (PhysicalBlockID, error) {

block/pack_index.go

@@ -1,6 +1,8 @@
 package block
 
 import (
+	"errors"
+
 	"github.com/golang/protobuf/proto"
 	"github.com/kopia/kopia/internal/blockmgrpb"
 )
@@ -11,9 +13,7 @@ type packIndex interface {
 	createTimeNanos() uint64
 
 	getBlock(blockID ContentID) (packBlockInfo, bool)
-	isEmpty() bool
-	activeBlockIDs() []ContentID
-	deletedBlockIDs() []ContentID
+	iterate(func(blockID ContentID, info packBlockInfo) error) error
 	addToIndexes(pb *blockmgrpb.Indexes)
 }
 
@@ -59,3 +59,10 @@ func unpackOffsetAndSize(os uint64) (uint32, uint32) {
 
 	return offset, size
 }
+
+func isIndexEmpty(ndx packIndex) bool {
+	return nil == ndx.iterate(
+		func(blockID ContentID, bi packBlockInfo) error {
+			return errors.New("have items")
+		})
+}

block/pack_index_test.go

@@ -16,15 +16,6 @@ func TestPackIndexV1(t *testing.T) {
 }
 
 func verifyPackIndex(t *testing.T, ndx packIndexBuilder, ts time.Time) {
-	if got, want := ndx.isEmpty(), true; got != want {
-		t.Errorf("unexpected isEmpty(): %v, wanted %v", got, want)
-	}
-	if got, want := len(ndx.activeBlockIDs()), 0; got != want {
-		t.Errorf("unexpected number of active block IDs: %v, wanted %v", got, want)
-	}
-	if got, want := len(ndx.deletedBlockIDs()), 0; got != want {
-		t.Errorf("unexpected number of active block IDs: %v, wanted %v", got, want)
-	}
 	if got, want := ndx.packBlockID(), PhysicalBlockID(""); got != want {
 		t.Errorf("unexpected pack block ID: %q, wanted %q", got, want)
 	}
@@ -75,10 +66,6 @@ func verifyPackIndex(t *testing.T, ndx packIndexBuilder, ts time.Time) {
 		verifyIndexBlockInline(t, ndx, blockID, blockData[blockID])
 	}
 
-	if !blockIDSlicesEqual(ndx.activeBlockIDs(), blockIDs) {
-		t.Errorf("unexpected active blocks: %v wanted %v", ndx.activeBlockIDs(), blockIDs)
-	}
-
 	ndx.deleteBlock(blockIDs[0])
 	verifyIndexBlockNotFound(t, ndx, blockIDs[0])
 	verifyIndexBlockDeleted(t, ndx, blockIDs[0])
@@ -120,28 +107,26 @@ func verifyIndexBlockDeleted(t *testing.T, ndx packIndex, blockID ContentID) {
 	t.Helper()
 
 	verifyIndexBlockNotFound(t, ndx, blockID)
-	var found bool
-	for _, b := range ndx.deletedBlockIDs() {
-		if b == blockID {
-			found = true
-		}
+	bi, ok := ndx.getBlock(blockID)
+	if !ok {
+		t.Errorf("block %q not found in index", blockID)
 	}
-	if !found {
-		t.Errorf("expected block %q to be amongst deleted, was not found: %v", blockID, ndx.deletedBlockIDs())
+
+	if !bi.deleted {
+		t.Errorf("expected block %q to be deleted", blockID)
 	}
 }
 
 func verifyIndexBlockNotDeleted(t *testing.T, ndx packIndex, blockID ContentID) {
 	t.Helper()
 
-	var found bool
-	for _, b := range ndx.deletedBlockIDs() {
-		if b == blockID {
-			found = true
-		}
+	bi, ok := ndx.getBlock(blockID)
+	if !ok {
+		t.Errorf("block %q not found in index", blockID)
 	}
-	if found {
-		t.Errorf("expected block %q to not be amongst deleted, was not found: %v", blockID, ndx.deletedBlockIDs())
+
+	if bi.deleted {
+		t.Errorf("expected block %q to not be deleted", blockID)
 	}
 }
 

block/pack_index_v1.go

@@ -53,13 +53,25 @@ func (p protoPackIndexV1) getBlock(blockID ContentID) (packBlockInfo, bool) {
 	return packBlockInfo{}, false
 }
 
-func (p protoPackIndexV1) deletedBlockIDs() []ContentID {
-	var result []ContentID
-
-	for _, d := range p.ndx.DeletedItems {
-		result = append(result, ContentID(d))
+func (p protoPackIndexV1) iterate(cb func(ContentID, packBlockInfo) error) error {
+	for k, v := range p.ndx.Items {
+		offset, size := unpackOffsetAndSize(v)
+		if err := cb(ContentID(k), packBlockInfo{offset: offset, size: size}); err != nil {
+			return err
+		}
 	}
-	return result
+	for k, v := range p.ndx.InlineItems {
+		if err := cb(ContentID(k), packBlockInfo{size: uint32(len(v)), payload: v}); err != nil {
+			return err
+		}
+	}
+	for _, k := range p.ndx.DeletedItems {
+		if err := cb(ContentID(k), packBlockInfo{deleted: true}); err != nil {
+			return err
+		}
+	}
+
+	return nil
 }
 
 func (p protoPackIndexV1) packBlockID() PhysicalBlockID {
@@ -80,21 +92,6 @@ func (p protoPackIndexV1) deleteBlock(blockID ContentID) {
 	p.ndx.DeletedItems = append(p.ndx.DeletedItems, string(blockID))
 }
 
-func (p protoPackIndexV1) activeBlockIDs() []ContentID {
-	var result []ContentID
-	for blkID := range p.ndx.Items {
-		result = append(result, ContentID(blkID))
-	}
-	for blkID := range p.ndx.InlineItems {
-		result = append(result, ContentID(blkID))
-	}
-	return result
-}
-
-func (p protoPackIndexV1) isEmpty() bool {
-	return len(p.ndx.Items)+len(p.ndx.InlineItems)+len(p.ndx.DeletedItems) == 0
-}
-
 func newPackIndexV1(t time.Time) packIndexBuilder {
 	return protoPackIndexV1{&blockmgrpb.IndexV1{
 		Items:           make(map[string]uint64),