diff --git a/cli/command_object_verify.go b/cli/command_object_verify.go index e55b1764b..f4415b77b 100644 --- a/cli/command_object_verify.go +++ b/cli/command_object_verify.go @@ -43,7 +43,7 @@ func (v *verifier) verifyDirectory(oid repo.ObjectID, path string) error { d := v.mgr.DirectoryEntry(oid) entries, err := d.Readdir() if err != nil { - if v.reportError(path, err) { + if v.reportError(path, fmt.Errorf("error reading directory %q %v: %v", path, oid, err)) { return err } } diff --git a/repo/object_manager_test.go b/repo/object_manager_test.go index 3cca342e6..df8c57dd9 100644 --- a/repo/object_manager_test.go +++ b/repo/object_manager_test.go @@ -432,6 +432,7 @@ func writeObject(t *testing.T, repo *Repository, data []byte, testCaseID string) } func verify(t *testing.T, repo *Repository, objectID ObjectID, expectedData []byte, testCaseID string) { + t.Helper() reader, err := repo.Objects.Open(objectID) if err != nil { t.Errorf("cannot get reader for %v (%v): %v %v", testCaseID, objectID, err, string(debug.Stack())) diff --git a/repo/pack_index.go b/repo/pack_index.go index 550554890..959cf9806 100644 --- a/repo/pack_index.go +++ b/repo/pack_index.go @@ -1,12 +1,15 @@ package repo import ( + "bufio" "encoding/json" "io" "time" ) -type packIndexes map[string]*packIndex +type packIndexesOld map[string]*packIndex + +type packIndexes []*packIndex type packIndex struct { PackBlockID string `json:"packBlock,omitempty"` @@ -16,20 +19,31 @@ type packIndex struct { } func loadPackIndexes(r io.Reader) (packIndexes, error) { + b := bufio.NewReader(r) + peek, err := b.Peek(1) + if err != nil { + return nil, err + } + + if peek[0] == '{' { + var old packIndexesOld + + if err := json.NewDecoder(b).Decode(&old); err != nil { + return nil, err + } + + var pi packIndexes + for _, v := range old { + pi = append(pi, v) + } + return pi, nil + } + var pi packIndexes - if err := json.NewDecoder(r).Decode(&pi); err != nil { + if err := json.NewDecoder(b).Decode(&pi); err != nil { return nil, err } return pi, nil } - -func (i packIndexes) merge(other packIndexes) { - for packID, ndx := range other { - old := i[packID] - if old == nil || ndx.CreateTime.After(old.CreateTime) { - i[packID] = ndx - } - } -} diff --git a/repo/pack_manager.go b/repo/pack_manager.go index 7f444f166..1cd0818b3 100644 --- a/repo/pack_manager.go +++ b/repo/pack_manager.go @@ -13,8 +13,6 @@ import ( "strings" "sync" "time" - - "github.com/kopia/kopia/blob" ) const flushPackIndexTimeout = 10 * time.Minute @@ -36,7 +34,6 @@ type blockLocation struct { type packManager struct { objectManager *ObjectManager - storage blob.Storage mu sync.Mutex blockToIndex map[string]*packIndex @@ -209,7 +206,7 @@ func (p *packManager) flushPackIndexesLocked() error { } p.flushPackIndexesAfter = time.Now().Add(flushPackIndexTimeout) - p.pendingPackIndexes = make(packIndexes) + p.pendingPackIndexes = nil return nil } @@ -267,13 +264,13 @@ func (p *packManager) finishPackLocked(g *packInfo) error { g.currentPackIndex.PackBlockID = oid.StorageBlock } - p.pendingPackIndexes[g.currentPackID] = g.currentPackIndex + p.pendingPackIndexes = append(p.pendingPackIndexes, g.currentPackIndex) g.currentPackIndex = nil return nil } -func (p *packManager) loadMergedPackIndexLocked(olderThan *time.Time) (map[string]*packIndex, []string, error) { +func (p *packManager) loadMergedPackIndexLocked(olderThan *time.Time) (packIndexes, []string, error) { ch, cancel := p.objectManager.storage.ListBlocks(packObjectPrefix) defer cancel() @@ -374,9 +371,9 @@ func (p *packManager) loadMergedPackIndexLocked(olderThan *time.Time) (map[strin } } - merged := make(packIndexes) + var merged packIndexes for _, pi := range indexes { - merged.merge(pi) + merged = append(merged, pi...) } return merged, blockIDs, nil @@ -393,17 +390,40 @@ func (p *packManager) ensurePackIndexesLoaded() (map[string]*packIndex, error) { return nil, err } - pi = make(map[string]*packIndex) - for _, pck := range merged { + p.blockToIndex = dedupeBlockIDsAndIndex(merged) + + return p.blockToIndex, nil +} + +func dedupeBlockIDsAndIndex(ndx packIndexes) map[string]*packIndex { + pi := make(map[string]*packIndex) + for _, pck := range ndx { for blockID := range pck.Items { - pi[blockID] = pck + if o := pi[blockID]; o != nil { + if !pck.CreateTime.Before(o.CreateTime) { + // this pack is same or newer. + delete(o.Items, blockID) + pi[blockID] = pck + } else { + // this pack is older. + delete(pck.Items, blockID) + } + } else { + pi[blockID] = pck + } } } + return pi +} - p.blockToIndex = pi - // log.Printf("loaded pack index with %v entries", len(p.blockToIndex)) - - return pi, nil +func removeEmptyIndexes(ndx packIndexes) packIndexes { + var res packIndexes + for _, n := range ndx { + if len(n.Items) > 0 { + res = append(res, n) + } + } + return res } func (p *packManager) Compact(cutoffTime time.Time) error { @@ -415,8 +435,12 @@ func (p *packManager) Compact(cutoffTime time.Time) error { return err } + dedupeBlockIDsAndIndex(merged) + + merged = removeEmptyIndexes(merged) + if len(blockIDs) <= 1 { - log.Printf("skipping index compaction - the number of segments %v is too low", len(blockIDs)) + log.Printf("skipping index compaction - already compacted") return nil } @@ -488,6 +512,6 @@ func newPackManager(om *ObjectManager) *packManager { objectManager: om, openPackGroups: make(map[string]*packInfo), flushPackIndexesAfter: time.Now().Add(flushPackIndexTimeout), - pendingPackIndexes: make(packIndexes), + pendingPackIndexes: nil, } }