support for array pack indexes instead of maps

cli/command_object_verify.go

@@ -43,7 +43,7 @@ func (v *verifier) verifyDirectory(oid repo.ObjectID, path string) error {
 	d := v.mgr.DirectoryEntry(oid)
 	entries, err := d.Readdir()
 	if err != nil {
-		if v.reportError(path, err) {
+		if v.reportError(path, fmt.Errorf("error reading directory %q %v: %v", path, oid, err)) {
 			return err
 		}
 	}

repo/object_manager_test.go

@@ -432,6 +432,7 @@ func writeObject(t *testing.T, repo *Repository, data []byte, testCaseID string)
 }
 
 func verify(t *testing.T, repo *Repository, objectID ObjectID, expectedData []byte, testCaseID string) {
+	t.Helper()
 	reader, err := repo.Objects.Open(objectID)
 	if err != nil {
 		t.Errorf("cannot get reader for %v (%v): %v %v", testCaseID, objectID, err, string(debug.Stack()))

repo/pack_index.go

@@ -1,12 +1,15 @@
 package repo
 
 import (
+	"bufio"
 	"encoding/json"
 	"io"
 	"time"
 )
 
-type packIndexes map[string]*packIndex
+type packIndexesOld map[string]*packIndex
+
+type packIndexes []*packIndex
 
 type packIndex struct {
 	PackBlockID string            `json:"packBlock,omitempty"`
@@ -16,20 +19,31 @@ type packIndex struct {
 }
 
 func loadPackIndexes(r io.Reader) (packIndexes, error) {
+	b := bufio.NewReader(r)
+	peek, err := b.Peek(1)
+	if err != nil {
+		return nil, err
+	}
+
+	if peek[0] == '{' {
+		var old packIndexesOld
+
+		if err := json.NewDecoder(b).Decode(&old); err != nil {
+			return nil, err
+		}
+
+		var pi packIndexes
+		for _, v := range old {
+			pi = append(pi, v)
+		}
+		return pi, nil
+	}
+
 	var pi packIndexes
 
-	if err := json.NewDecoder(r).Decode(&pi); err != nil {
+	if err := json.NewDecoder(b).Decode(&pi); err != nil {
 		return nil, err
 	}
 
 	return pi, nil
 }
-
-func (i packIndexes) merge(other packIndexes) {
-	for packID, ndx := range other {
-		old := i[packID]
-		if old == nil || ndx.CreateTime.After(old.CreateTime) {
-			i[packID] = ndx
-		}
-	}
-}

repo/pack_manager.go

@@ -13,8 +13,6 @@ import (
 	"strings"
 	"sync"
 	"time"
-
-	"github.com/kopia/kopia/blob"
 )
 
 const flushPackIndexTimeout = 10 * time.Minute
@@ -36,7 +34,6 @@ type blockLocation struct {
 
 type packManager struct {
 	objectManager *ObjectManager
-	storage       blob.Storage
 
 	mu           sync.Mutex
 	blockToIndex map[string]*packIndex
@@ -209,7 +206,7 @@ func (p *packManager) flushPackIndexesLocked() error {
 	}
 
 	p.flushPackIndexesAfter = time.Now().Add(flushPackIndexTimeout)
-	p.pendingPackIndexes = make(packIndexes)
+	p.pendingPackIndexes = nil
 	return nil
 }
 
@@ -267,13 +264,13 @@ func (p *packManager) finishPackLocked(g *packInfo) error {
 		g.currentPackIndex.PackBlockID = oid.StorageBlock
 	}
 
-	p.pendingPackIndexes[g.currentPackID] = g.currentPackIndex
+	p.pendingPackIndexes = append(p.pendingPackIndexes, g.currentPackIndex)
 	g.currentPackIndex = nil
 
 	return nil
 }
 
-func (p *packManager) loadMergedPackIndexLocked(olderThan *time.Time) (map[string]*packIndex, []string, error) {
+func (p *packManager) loadMergedPackIndexLocked(olderThan *time.Time) (packIndexes, []string, error) {
 	ch, cancel := p.objectManager.storage.ListBlocks(packObjectPrefix)
 	defer cancel()
 
@@ -374,9 +371,9 @@ func (p *packManager) loadMergedPackIndexLocked(olderThan *time.Time) (map[strin
 		}
 	}
 
-	merged := make(packIndexes)
+	var merged packIndexes
 	for _, pi := range indexes {
-		merged.merge(pi)
+		merged = append(merged, pi...)
 	}
 
 	return merged, blockIDs, nil
@@ -393,17 +390,40 @@ func (p *packManager) ensurePackIndexesLoaded() (map[string]*packIndex, error) {
 		return nil, err
 	}
 
-	pi = make(map[string]*packIndex)
-	for _, pck := range merged {
+	p.blockToIndex = dedupeBlockIDsAndIndex(merged)
+
+	return p.blockToIndex, nil
+}
+
+func dedupeBlockIDsAndIndex(ndx packIndexes) map[string]*packIndex {
+	pi := make(map[string]*packIndex)
+	for _, pck := range ndx {
 		for blockID := range pck.Items {
-			pi[blockID] = pck
+			if o := pi[blockID]; o != nil {
+				if !pck.CreateTime.Before(o.CreateTime) {
+					// this pack is same or newer.
+					delete(o.Items, blockID)
+					pi[blockID] = pck
+				} else {
+					// this pack is older.
+					delete(pck.Items, blockID)
+				}
+			} else {
+				pi[blockID] = pck
+			}
 		}
 	}
+	return pi
+}
 
-	p.blockToIndex = pi
-	// log.Printf("loaded pack index with %v entries", len(p.blockToIndex))
-
-	return pi, nil
+func removeEmptyIndexes(ndx packIndexes) packIndexes {
+	var res packIndexes
+	for _, n := range ndx {
+		if len(n.Items) > 0 {
+			res = append(res, n)
+		}
+	}
+	return res
 }
 
 func (p *packManager) Compact(cutoffTime time.Time) error {
@@ -415,8 +435,12 @@ func (p *packManager) Compact(cutoffTime time.Time) error {
 		return err
 	}
 
+	dedupeBlockIDsAndIndex(merged)
+
+	merged = removeEmptyIndexes(merged)
+
 	if len(blockIDs) <= 1 {
-		log.Printf("skipping index compaction - the number of segments %v is too low", len(blockIDs))
+		log.Printf("skipping index compaction - already compacted")
 		return nil
 	}
 
@@ -488,6 +512,6 @@ func newPackManager(om *ObjectManager) *packManager {
 		objectManager:         om,
 		openPackGroups:        make(map[string]*packInfo),
 		flushPackIndexesAfter: time.Now().Add(flushPackIndexTimeout),
-		pendingPackIndexes:    make(packIndexes),
+		pendingPackIndexes:    nil,
 	}
 }