diff --git a/block/block_cache.go b/block/block_cache.go index ab8f1fa67..27766d12b 100644 --- a/block/block_cache.go +++ b/block/block_cache.go @@ -11,9 +11,9 @@ import ( ) type blockCache interface { - getBlock(virtualBlockID, physicalBlockID string, offset, length int64) ([]byte, error) - putBlock(blockID string, data []byte) error - listIndexBlocks(full bool) ([]Info, error) + getBlock(ctx context.Context, virtualBlockID, physicalBlockID string, offset, length int64) ([]byte, error) + putBlock(ctx context.Context, blockID string, data []byte) error + listIndexBlocks(ctx context.Context, full bool) ([]Info, error) close() error } @@ -26,7 +26,7 @@ type CachingOptions struct { HMACSecret []byte `json:"-"` } -func newBlockCache(st storage.Storage, caching CachingOptions) (blockCache, error) { +func newBlockCache(ctx context.Context, st storage.Storage, caching CachingOptions) (blockCache, error) { if caching.MaxCacheSizeBytes == 0 || caching.CacheDirectory == "" { return nullBlockCache{st}, nil } @@ -53,13 +53,13 @@ func newBlockCache(st storage.Storage, caching CachingOptions) (blockCache, erro } if caching.IgnoreListCache { - c.deleteListCache() + c.deleteListCache(ctx) } - if err := c.sweepDirectory(); err != nil { + if err := c.sweepDirectory(ctx); err != nil { return nil, err } - go c.sweepDirectoryPeriodically() + go c.sweepDirectoryPeriodically(ctx) return c, nil } diff --git a/block/block_manager.go b/block/block_manager.go index 82d164f80..747b367a7 100644 --- a/block/block_manager.go +++ b/block/block_manager.go @@ -1,6 +1,7 @@ package block import ( + "context" cryptorand "crypto/rand" "fmt" "io" @@ -110,7 +111,7 @@ func (bm *Manager) addToIndexLocked(blockID string, ndx *blockmgrpb.Index, packe bm.blockIDToIndex[blockID] = ndx } -func (bm *Manager) addToPackLocked(blockID string, data []byte, force bool) error { +func (bm *Manager) addToPackLocked(ctx context.Context, blockID string, data []byte, force bool) error { bm.assertLocked() if !force { @@ -137,7 +138,7 @@ func (bm *Manager) addToPackLocked(blockID string, data []byte, force bool) erro bm.addToIndexLocked(blockID, bm.currentPackIndex, packOffsetAndSize(uint32(offset), uint32(len(data)))) if shouldFinish { - if err := bm.finishPackAndMaybeFlushIndexes(); err != nil { + if err := bm.finishPackAndMaybeFlushIndexes(ctx); err != nil { return err } } @@ -145,13 +146,13 @@ func (bm *Manager) addToPackLocked(blockID string, data []byte, force bool) erro return nil } -func (bm *Manager) finishPackAndMaybeFlushIndexes() error { - if err := bm.finishPackLocked(); err != nil { +func (bm *Manager) finishPackAndMaybeFlushIndexes(ctx context.Context) error { + if err := bm.finishPackLocked(ctx); err != nil { return err } if bm.timeNow().After(bm.flushPackIndexesAfter) { - if err := bm.flushPackIndexesLocked(); err != nil { + if err := bm.flushPackIndexesLocked(ctx); err != nil { return err } } @@ -239,12 +240,12 @@ func (bm *Manager) startPackIndexLocked() { bm.currentPackData = []byte{} } -func (bm *Manager) flushPackIndexesLocked() error { +func (bm *Manager) flushPackIndexesLocked(ctx context.Context) error { if len(bm.pendingPackIndexes) > 0 { if false { log.Printf("saving %v pack indexes", len(bm.pendingPackIndexes)) } - if _, err := bm.writePackIndexes(bm.pendingPackIndexes, nil); err != nil { + if _, err := bm.writePackIndexes(ctx, bm.pendingPackIndexes, nil); err != nil { return err } } @@ -254,7 +255,7 @@ func (bm *Manager) flushPackIndexesLocked() error { return nil } -func (bm *Manager) writePackIndexes(ndx []*blockmgrpb.Index, replacesBlockBeforeTime *time.Time) (string, error) { +func (bm *Manager) writePackIndexes(ctx context.Context, ndx []*blockmgrpb.Index, replacesBlockBeforeTime *time.Time) (string, error) { data, err := proto.Marshal(&blockmgrpb.Indexes{ Indexes: ndx, }) @@ -269,10 +270,10 @@ func (bm *Manager) writePackIndexes(ndx []*blockmgrpb.Index, replacesBlockBefore inverseTimePrefix := fmt.Sprintf("%016x", math.MaxInt64-time.Now().UnixNano()) - return bm.writeUnpackedBlockNotLocked(data, indexBlockPrefix+inverseTimePrefix, suffix, true) + return bm.writeUnpackedBlockNotLocked(ctx, data, indexBlockPrefix+inverseTimePrefix, suffix, true) } -func (bm *Manager) finishPackLocked() error { +func (bm *Manager) finishPackLocked(ctx context.Context) error { if len(bm.currentPackIndex.Items)+len(bm.currentPackIndex.DeletedItems)+len(bm.currentPackIndex.InlineItems) > 0 { log.Debug().Int("items", len(bm.currentPackIndex.Items)).Int("deleted", len(bm.currentPackIndex.DeletedItems)).Int("inline", len(bm.currentPackIndex.InlineItems)).Msg("finishing pack") if len(bm.currentPackData) < bm.maxInlineContentLength { @@ -295,7 +296,7 @@ func (bm *Manager) finishPackLocked() error { bm.currentPackData = append(bm.currentPackData, postamble...) } } - blockID, err := bm.writeUnpackedBlockNotLocked(bm.currentPackData, "", "", true) + blockID, err := bm.writeUnpackedBlockNotLocked(ctx, bm.currentPackData, "", "", true) if err != nil { return fmt.Errorf("can't save pack data block %q: %v", blockID, err) } @@ -316,8 +317,8 @@ func (bm *Manager) finishPackLocked() error { } // ListIndexBlocks returns the list of all index blocks, including inactive, sorted by time. -func (bm *Manager) ListIndexBlocks() ([]Info, error) { - blocks, err := bm.cache.listIndexBlocks(true) +func (bm *Manager) ListIndexBlocks(ctx context.Context) ([]Info, error) { + blocks, err := bm.cache.listIndexBlocks(ctx, true) if err != nil { return nil, fmt.Errorf("error listing index blocks: %v", err) } @@ -327,8 +328,8 @@ func (bm *Manager) ListIndexBlocks() ([]Info, error) { } // ActiveIndexBlocks returns the list of active index blocks, sorted by time. -func (bm *Manager) ActiveIndexBlocks() ([]Info, error) { - blocks, err := bm.cache.listIndexBlocks(false) +func (bm *Manager) ActiveIndexBlocks(ctx context.Context) ([]Info, error) { + blocks, err := bm.cache.listIndexBlocks(ctx, false) if err != nil { return nil, err } @@ -375,9 +376,9 @@ func findLatestCompactedTimestamp(blocks []Info) (time.Time, error) { return latestTime, nil } -func (bm *Manager) loadMergedPackIndexLocked() ([]*blockmgrpb.Index, []string, time.Time, error) { +func (bm *Manager) loadMergedPackIndexLocked(ctx context.Context) ([]*blockmgrpb.Index, []string, time.Time, error) { log.Debug().Msg("listing active index blocks") - blocks, err := bm.ActiveIndexBlocks() + blocks, err := bm.ActiveIndexBlocks(ctx) if err != nil { return nil, nil, zeroTime, err } @@ -412,7 +413,7 @@ func (bm *Manager) loadMergedPackIndexLocked() ([]*blockmgrpb.Index, []string, t defer wg.Done() for b := range ch { - data, err := bm.getBlockInternalLocked(b) + data, err := bm.getBlockInternalLocked(ctx, b) if err != nil { errors <- err return @@ -453,11 +454,11 @@ func (bm *Manager) loadMergedPackIndexLocked() ([]*blockmgrpb.Index, []string, t return merged, blockIDs, blocks[len(blocks)-1].Timestamp, nil } -func (bm *Manager) initializeIndexes() error { +func (bm *Manager) initializeIndexes(ctx context.Context) error { bm.lock() defer bm.unlock() - merged, blockIDs, latestBlockTime, err := bm.loadMergedPackIndexLocked() + merged, blockIDs, latestBlockTime, err := bm.loadMergedPackIndexLocked(ctx) if err != nil { return err } @@ -467,7 +468,7 @@ func (bm *Manager) initializeIndexes() error { if len(blockIDs) >= autoCompactionBlockCount && latestBlockTime.Before(time.Now().Add(-autoCompactionSafetyMargin)) { log.Debug().Msgf("auto compacting block indexes (block count %v exceeds threshold of %v)", len(blockIDs), autoCompactionBlockCount) merged = removeEmptyIndexes(merged) - if _, err := bm.writePackIndexes(merged, &latestBlockTime); err != nil { + if _, err := bm.writePackIndexes(ctx, merged, &latestBlockTime); err != nil { return err } } @@ -525,16 +526,16 @@ func removeEmptyIndexes(ndx []*blockmgrpb.Index) []*blockmgrpb.Index { } // CompactIndexes performs compaction of index blocks. -func (bm *Manager) CompactIndexes() error { +func (bm *Manager) CompactIndexes(ctx context.Context) error { bm.lock() defer bm.unlock() - merged, blockIDs, latestBlockTime, err := bm.loadMergedPackIndexLocked() + merged, blockIDs, latestBlockTime, err := bm.loadMergedPackIndexLocked(ctx) if err != nil { return err } - if err := bm.compactIndexes(merged, blockIDs, latestBlockTime); err != nil { + if err := bm.compactIndexes(ctx, merged, blockIDs, latestBlockTime); err != nil { return err } @@ -578,7 +579,7 @@ func newInfo(blockID string, ndx *blockmgrpb.Index) Info { } } -func (bm *Manager) compactIndexes(merged []*blockmgrpb.Index, blockIDs []string, latestBlockTime time.Time) error { +func (bm *Manager) compactIndexes(ctx context.Context, merged []*blockmgrpb.Index, blockIDs []string, latestBlockTime time.Time) error { dedupeBlockIDsAndIndex(merged) merged = removeEmptyIndexes(merged) if len(blockIDs) <= 1 { @@ -586,20 +587,20 @@ func (bm *Manager) compactIndexes(merged []*blockmgrpb.Index, blockIDs []string, return nil } - _, err := bm.writePackIndexes(merged, &latestBlockTime) + _, err := bm.writePackIndexes(ctx, merged, &latestBlockTime) return err } // Flush completes writing any pending packs and writes pack indexes to the underlyign storage. -func (bm *Manager) Flush() error { +func (bm *Manager) Flush(ctx context.Context) error { bm.lock() defer bm.unlock() - if err := bm.finishPackLocked(); err != nil { + if err := bm.finishPackLocked(ctx); err != nil { return err } - if err := bm.flushPackIndexesLocked(); err != nil { + if err := bm.flushPackIndexesLocked(ctx); err != nil { return err } @@ -608,13 +609,13 @@ func (bm *Manager) Flush() error { // WriteBlock saves a given block of data to a pack group with a provided name and returns a blockID // that's based on the contents of data written. -func (bm *Manager) WriteBlock(data []byte, prefix string) (string, error) { +func (bm *Manager) WriteBlock(ctx context.Context, data []byte, prefix string) (string, error) { blockID := prefix + bm.hashData(data) bm.lock() defer bm.unlock() - err := bm.addToPackLocked(blockID, data, false) + err := bm.addToPackLocked(ctx, blockID, data, false) return blockID, err } @@ -627,11 +628,11 @@ func (bm *Manager) IsStorageBlockInUse(storageBlockID string) bool { } // Repackage reorganizes all pack blocks belonging to a given group that are not bigger than given size. -func (bm *Manager) Repackage(maxLength uint64) error { +func (bm *Manager) Repackage(ctx context.Context, maxLength uint64) error { bm.lock() defer bm.unlock() - merged, _, _, err := bm.loadMergedPackIndexLocked() + merged, _, _, err := bm.loadMergedPackIndexLocked(ctx) if err != nil { return err } @@ -661,7 +662,7 @@ func (bm *Manager) Repackage(maxLength uint64) error { log.Printf("%v blocks to re-package (%v total bytes)", len(toRepackage), totalBytes) for _, m := range toRepackage { - if err := bm.repackageBlock(m, done); err != nil { + if err := bm.repackageBlock(ctx, m, done); err != nil { return err } } @@ -669,8 +670,8 @@ func (bm *Manager) Repackage(maxLength uint64) error { return nil } -func (bm *Manager) repackageBlock(m *blockmgrpb.Index, done map[string]bool) error { - data, err := bm.getBlockInternalLocked(m.PackBlockId) +func (bm *Manager) repackageBlock(ctx context.Context, m *blockmgrpb.Index, done map[string]bool) error { + data, err := bm.getBlockInternalLocked(ctx, m.PackBlockId) if err != nil { return fmt.Errorf("can't fetch block %q for repackaging: %v", m.PackBlockId, err) } @@ -684,7 +685,7 @@ func (bm *Manager) repackageBlock(m *blockmgrpb.Index, done map[string]bool) err offset, size := unpackOffsetAndSize(os) blockData := data[offset : offset+size] - if err := bm.addToPackLocked(blockID, blockData, true); err != nil { + if err := bm.addToPackLocked(ctx, blockID, blockData, true); err != nil { return fmt.Errorf("unable to re-package %q: %v", blockID, err) } } @@ -694,7 +695,7 @@ func (bm *Manager) repackageBlock(m *blockmgrpb.Index, done map[string]bool) err continue } done[blockID] = true - if err := bm.addToPackLocked(blockID, blockData, true); err != nil { + if err := bm.addToPackLocked(ctx, blockID, blockData, true); err != nil { return fmt.Errorf("unable to re-package %q: %v", blockID, err) } } @@ -702,12 +703,12 @@ func (bm *Manager) repackageBlock(m *blockmgrpb.Index, done map[string]bool) err return nil } -func (bm *Manager) writeUnpackedBlockNotLocked(data []byte, prefix string, suffix string, force bool) (string, error) { +func (bm *Manager) writeUnpackedBlockNotLocked(ctx context.Context, data []byte, prefix string, suffix string, force bool) (string, error) { blockID := prefix + bm.hashData(data) + suffix if !force { // Before performing encryption, check if the block is already there. - i, err := bm.BlockInfo(blockID) + i, err := bm.BlockInfo(ctx, blockID) atomic.AddInt32(&bm.stats.CheckedBlocks, 1) if err == nil && i.Length == int64(len(data)) { atomic.AddInt32(&bm.stats.PresentBlocks, 1) @@ -730,7 +731,7 @@ func (bm *Manager) writeUnpackedBlockNotLocked(data []byte, prefix string, suffi atomic.AddInt32(&bm.stats.WrittenBlocks, 1) atomic.AddInt64(&bm.stats.WrittenBytes, int64(len(data))) - if err := bm.cache.putBlock(blockID, data2); err != nil { + if err := bm.cache.putBlock(ctx, blockID, data2); err != nil { return "", err } @@ -761,7 +762,7 @@ func (bm *Manager) getPendingBlockLocked(blockID string) ([]byte, error) { } // GetBlock gets the contents of a given block. If the block is not found returns blob.ErrBlockNotFound. -func (bm *Manager) GetBlock(blockID string) ([]byte, error) { +func (bm *Manager) GetBlock(ctx context.Context, blockID string) ([]byte, error) { bm.lock() defer bm.unlock() @@ -769,11 +770,11 @@ func (bm *Manager) GetBlock(blockID string) ([]byte, error) { return b, nil } - return bm.getBlockInternalLocked(blockID) + return bm.getBlockInternalLocked(ctx, blockID) } // BlockInfo returns information about a single block. -func (bm *Manager) BlockInfo(blockID string) (Info, error) { +func (bm *Manager) BlockInfo(ctx context.Context, blockID string) (Info, error) { bm.lock() defer bm.unlock() @@ -828,7 +829,7 @@ func (bm *Manager) blockInfoLocked(blockID string) (Info, error) { }, nil } -func (bm *Manager) getBlockInternalLocked(blockID string) ([]byte, error) { +func (bm *Manager) getBlockInternalLocked(ctx context.Context, blockID string) ([]byte, error) { bm.assertLocked() if ndx, ok := bm.blockIDToIndex[blockID]; ok { @@ -848,10 +849,10 @@ func (bm *Manager) getBlockInternalLocked(blockID string) ([]byte, error) { if s.PackBlockID != "" { underlyingBlockID = s.PackBlockID - payload, err = bm.cache.getBlock(blockID, underlyingBlockID, s.PackOffset, s.Length) + payload, err = bm.cache.getBlock(ctx, blockID, underlyingBlockID, s.PackOffset, s.Length) decryptSkip = int(s.PackOffset) } else { - payload, err = bm.cache.getBlock(blockID, blockID, 0, -1) + payload, err = bm.cache.getBlock(ctx, blockID, blockID, 0, -1) } if err != nil { @@ -919,15 +920,17 @@ type cachedList struct { // If 'full' is set to true, this function lists and returns all blocks, // if 'full' is false, the function returns only blocks from the last 2 compactions. // The list of blocks is not guaranteed to be sorted. -func listIndexBlocksFromStorage(st storage.Storage, full bool) ([]Info, error) { +func listIndexBlocksFromStorage(ctx context.Context, st storage.Storage, full bool) ([]Info, error) { maxCompactions := 2 if full { maxCompactions = math.MaxInt32 } - ch, cancel := st.ListBlocks(indexBlockPrefix) + ctx, cancel := context.WithCancel(ctx) defer cancel() + ch := st.ListBlocks(ctx, indexBlockPrefix) + var results []Info numCompactions := 0 @@ -959,11 +962,11 @@ func listIndexBlocksFromStorage(st storage.Storage, full bool) ([]Info, error) { } // NewManager creates new block manager with given packing options and a formatter. -func NewManager(st storage.Storage, f FormattingOptions, caching CachingOptions) (*Manager, error) { - return newManagerWithTime(st, f, caching, time.Now) +func NewManager(ctx context.Context, st storage.Storage, f FormattingOptions, caching CachingOptions) (*Manager, error) { + return newManagerWithTime(ctx, st, f, caching, time.Now) } -func newManagerWithTime(st storage.Storage, f FormattingOptions, caching CachingOptions, timeNow func() time.Time) (*Manager, error) { +func newManagerWithTime(ctx context.Context, st storage.Storage, f FormattingOptions, caching CachingOptions, timeNow func() time.Time) (*Manager, error) { sf := FormatterFactories[f.BlockFormat] if sf == nil { return nil, fmt.Errorf("unsupported block format: %v", f.BlockFormat) @@ -974,7 +977,7 @@ func newManagerWithTime(st storage.Storage, f FormattingOptions, caching Caching return nil, err } - cache, err := newBlockCache(st, caching) + cache, err := newBlockCache(ctx, st, caching) if err != nil { return nil, fmt.Errorf("unable to initialize cache: %v", err) } @@ -1001,7 +1004,7 @@ func newManagerWithTime(st storage.Storage, f FormattingOptions, caching Caching m.startPackIndexLocked() - if err := m.initializeIndexes(); err != nil { + if err := m.initializeIndexes(ctx); err != nil { return nil, fmt.Errorf("unable initialize indexes: %v", err) } diff --git a/block/block_manager_test.go b/block/block_manager_test.go index 6bb40318f..7a8ef4c14 100644 --- a/block/block_manager_test.go +++ b/block/block_manager_test.go @@ -1,6 +1,7 @@ package block import ( + "context" "crypto/md5" "encoding/hex" "fmt" @@ -34,21 +35,23 @@ func init() { } func TestBlockManagerEmptyFlush(t *testing.T) { + ctx := context.Background() data := map[string][]byte{} keyTime := map[string]time.Time{} bm := newTestBlockManager(data, keyTime, nil) - bm.Flush() + bm.Flush(ctx) if got, want := len(data), 0; got != want { t.Errorf("unexpected number of blocks: %v, wanted %v", got, want) } } func TestBlockZeroBytes1(t *testing.T) { + ctx := context.Background() data := map[string][]byte{} keyTime := map[string]time.Time{} bm := newTestBlockManager(data, keyTime, nil) - writeBlockAndVerify(t, bm, []byte{}) - bm.Flush() + writeBlockAndVerify(ctx, t, bm, []byte{}) + bm.Flush(ctx) if got, want := len(data), 2; got != want { t.Errorf("unexpected number of blocks: %v, wanted %v", got, want) } @@ -56,12 +59,13 @@ func TestBlockZeroBytes1(t *testing.T) { } func TestBlockZeroBytes2(t *testing.T) { + ctx := context.Background() data := map[string][]byte{} keyTime := map[string]time.Time{} bm := newTestBlockManager(data, keyTime, nil) - writeBlockAndVerify(t, bm, seededRandomData(10, 10)) - writeBlockAndVerify(t, bm, []byte{}) - bm.Flush() + writeBlockAndVerify(ctx, t, bm, seededRandomData(10, 10)) + writeBlockAndVerify(ctx, t, bm, []byte{}) + bm.Flush(ctx) dumpBlockManagerData(data) if got, want := len(data), 2; got != want { t.Errorf("unexpected number of blocks: %v, wanted %v", got, want) @@ -70,60 +74,63 @@ func TestBlockZeroBytes2(t *testing.T) { } func TestBlockManagerSmallBlockWrites(t *testing.T) { + ctx := context.Background() data := map[string][]byte{} keyTime := map[string]time.Time{} bm := newTestBlockManager(data, keyTime, nil) for i := 0; i < 100; i++ { - writeBlockAndVerify(t, bm, seededRandomData(i, 10)) + writeBlockAndVerify(ctx, t, bm, seededRandomData(i, 10)) } if got, want := len(data), 0; got != want { t.Errorf("unexpected number of blocks: %v, wanted %v", got, want) } - bm.Flush() + bm.Flush(ctx) if got, want := len(data), 2; got != want { t.Errorf("unexpected number of blocks: %v, wanted %v", got, want) } } func TestBlockManagerDedupesPendingBlocks(t *testing.T) { + ctx := context.Background() data := map[string][]byte{} keyTime := map[string]time.Time{} bm := newTestBlockManager(data, keyTime, nil) for i := 0; i < 100; i++ { - writeBlockAndVerify(t, bm, seededRandomData(0, 999)) + writeBlockAndVerify(ctx, t, bm, seededRandomData(0, 999)) } if got, want := len(data), 0; got != want { t.Errorf("unexpected number of blocks: %v, wanted %v", got, want) } - bm.Flush() + bm.Flush(ctx) if got, want := len(data), 2; got != want { t.Errorf("unexpected number of blocks: %v, wanted %v", got, want) } } func TestBlockManagerDedupesPendingAndUncommittedBlocks(t *testing.T) { + ctx := context.Background() data := map[string][]byte{} keyTime := map[string]time.Time{} bm := newTestBlockManager(data, keyTime, nil) // no writes here, all data fits in a single pack. - writeBlockAndVerify(t, bm, seededRandomData(0, 950)) - writeBlockAndVerify(t, bm, seededRandomData(1, 950)) - writeBlockAndVerify(t, bm, seededRandomData(2, 10)) + writeBlockAndVerify(ctx, t, bm, seededRandomData(0, 950)) + writeBlockAndVerify(ctx, t, bm, seededRandomData(1, 950)) + writeBlockAndVerify(ctx, t, bm, seededRandomData(2, 10)) if got, want := len(data), 0; got != want { t.Errorf("unexpected number of blocks: %v, wanted %v", got, want) } // no writes here - writeBlockAndVerify(t, bm, seededRandomData(0, 950)) - writeBlockAndVerify(t, bm, seededRandomData(1, 950)) - writeBlockAndVerify(t, bm, seededRandomData(2, 10)) + writeBlockAndVerify(ctx, t, bm, seededRandomData(0, 950)) + writeBlockAndVerify(ctx, t, bm, seededRandomData(1, 950)) + writeBlockAndVerify(ctx, t, bm, seededRandomData(2, 10)) if got, want := len(data), 0; got != want { t.Errorf("unexpected number of blocks: %v, wanted %v", got, want) } - bm.Flush() + bm.Flush(ctx) // this flushes the pack block + index block if got, want := len(data), 2; got != want { @@ -133,18 +140,19 @@ func TestBlockManagerDedupesPendingAndUncommittedBlocks(t *testing.T) { } func TestBlockManagerEmpty(t *testing.T) { + ctx := context.Background() data := map[string][]byte{} keyTime := map[string]time.Time{} bm := newTestBlockManager(data, keyTime, nil) noSuchBlockID := md5hash([]byte("foo")) - b, err := bm.GetBlock(noSuchBlockID) + b, err := bm.GetBlock(ctx, noSuchBlockID) if err != storage.ErrBlockNotFound { t.Errorf("unexpected error when getting non-existent block: %v, %v", b, err) } - bi, err := bm.BlockInfo(noSuchBlockID) + bi, err := bm.BlockInfo(ctx, noSuchBlockID) if err != storage.ErrBlockNotFound { t.Errorf("unexpected error when getting non-existent block info: %v, %v", bi, err) } @@ -155,6 +163,7 @@ func TestBlockManagerEmpty(t *testing.T) { } func TestBlockManagerRepack(t *testing.T) { + ctx := context.Background() data := map[string][]byte{} keyTime := map[string]time.Time{} bm := newTestBlockManager(data, keyTime, nil) @@ -168,12 +177,12 @@ func TestBlockManagerRepack(t *testing.T) { d2 := seededRandomData(2, 20) d3 := seededRandomData(3, 30) - writeBlockAndVerify(t, bm, d1) - bm.Flush() - writeBlockAndVerify(t, bm, d2) - bm.Flush() - writeBlockAndVerify(t, bm, d3) - bm.Flush() + writeBlockAndVerify(ctx, t, bm, d1) + bm.Flush(ctx) + writeBlockAndVerify(ctx, t, bm, d2) + bm.Flush(ctx) + writeBlockAndVerify(ctx, t, bm, d3) + bm.Flush(ctx) // 3 data blocks, 3 index blocks. if got, want := len(data), 6; got != want { @@ -183,10 +192,10 @@ func TestBlockManagerRepack(t *testing.T) { log.Printf("before repackage") dumpBlockManagerData(data) - if err := bm.Repackage(5); err != nil { + if err := bm.Repackage(ctx, 5); err != nil { t.Errorf("repackage failure: %v", err) } - bm.Flush() + bm.Flush(ctx) // nothing happened, still 3 data blocks, 3 index blocks. if got, want := len(data), 6; got != want { @@ -195,10 +204,10 @@ func TestBlockManagerRepack(t *testing.T) { bm.timeNow = fakeTimeNowFrozen(fakeTime.Add(1 * time.Second)) - if err := bm.Repackage(30); err != nil { + if err := bm.Repackage(ctx, 30); err != nil { t.Errorf("repackage failure: %v", err) } - bm.Flush() + bm.Flush(ctx) log.Printf("after repackage") dumpBlockManagerData(data) @@ -207,7 +216,7 @@ func TestBlockManagerRepack(t *testing.T) { if got, want := len(data), 8; got != want { t.Errorf("unexpected block count: %v, wanted %v", got, want) } - if err := bm.CompactIndexes(); err != nil { + if err := bm.CompactIndexes(ctx); err != nil { t.Errorf("compaction failure: %v", err) } @@ -216,13 +225,13 @@ func TestBlockManagerRepack(t *testing.T) { dumpBlockManagerData(data) } - verifyActiveIndexBlockCount(t, bm, 1) + verifyActiveIndexBlockCount(ctx, t, bm, 1) } -func verifyActiveIndexBlockCount(t *testing.T, bm *Manager, expected int) { +func verifyActiveIndexBlockCount(ctx context.Context, t *testing.T, bm *Manager, expected int) { t.Helper() - blks, err := bm.ActiveIndexBlocks() + blks, err := bm.ActiveIndexBlocks(ctx) if err != nil { t.Errorf("error listing active index blocks: %v", err) return @@ -233,6 +242,7 @@ func verifyActiveIndexBlockCount(t *testing.T, bm *Manager, expected int) { } } func TestBlockManagerInternalFlush(t *testing.T) { + ctx := context.Background() data := map[string][]byte{} keyTime := map[string]time.Time{} bm := newTestBlockManager(data, keyTime, nil) @@ -240,7 +250,7 @@ func TestBlockManagerInternalFlush(t *testing.T) { for i := 0; i < 100; i++ { b := make([]byte, 25) rand.Read(b) - writeBlockAndVerify(t, bm, b) + writeBlockAndVerify(ctx, t, bm, b) } // 1 data block written, but no index yet. @@ -252,7 +262,7 @@ func TestBlockManagerInternalFlush(t *testing.T) { for i := 0; i < 100; i++ { b := make([]byte, 25) rand.Read(b) - writeBlockAndVerify(t, bm, b) + writeBlockAndVerify(ctx, t, bm, b) } // 2 data blocks written, but no index yet. @@ -260,7 +270,7 @@ func TestBlockManagerInternalFlush(t *testing.T) { t.Errorf("unexpected number of blocks: %v, wanted %v", got, want) } - bm.Flush() + bm.Flush(ctx) // third block gets written, followed by index. if got, want := len(data), 4; got != want { @@ -271,6 +281,7 @@ func TestBlockManagerInternalFlush(t *testing.T) { } func TestBlockManagerWriteMultiple(t *testing.T) { + ctx := context.Background() data := map[string][]byte{} keyTime := map[string]time.Time{} bm := newTestBlockManager(data, keyTime, nil) @@ -281,7 +292,7 @@ func TestBlockManagerWriteMultiple(t *testing.T) { //t.Logf("i=%v", i) b := seededRandomData(i, i%113) //t.Logf("writing block #%v with %x", i, b) - blkID, err := bm.WriteBlock(b, "") + blkID, err := bm.WriteBlock(ctx, b, "") //t.Logf("wrote %v=%v", i, blkID) if err != nil { t.Errorf("err: %v", err) @@ -291,13 +302,13 @@ func TestBlockManagerWriteMultiple(t *testing.T) { if i%17 == 0 { t.Logf("flushing %v", i) - bm.Flush() + bm.Flush(ctx) //dumpBlockManagerData(data) } if i%41 == 0 { t.Logf("opening new manager: %v", i) - bm.Flush() + bm.Flush(ctx) t.Logf("data block count: %v", len(data)) //dumpBlockManagerData(data) bm = newTestBlockManager(data, keyTime, nil) @@ -305,7 +316,7 @@ func TestBlockManagerWriteMultiple(t *testing.T) { } for _, blockID := range blockIDs { - _, err := bm.GetBlock(blockID) + _, err := bm.GetBlock(ctx, blockID) if err != nil { t.Errorf("can't read block %q: %v", blockID, err) continue @@ -314,63 +325,64 @@ func TestBlockManagerWriteMultiple(t *testing.T) { } func TestBlockManagerConcurrency(t *testing.T) { + ctx := context.Background() data := map[string][]byte{} keyTime := map[string]time.Time{} bm := newTestBlockManager(data, keyTime, nil) - preexistingBlock := writeBlockAndVerify(t, bm, seededRandomData(10, 100)) - bm.Flush() + preexistingBlock := writeBlockAndVerify(ctx, t, bm, seededRandomData(10, 100)) + bm.Flush(ctx) bm1 := newTestBlockManager(data, keyTime, nil) bm2 := newTestBlockManager(data, keyTime, nil) bm3 := newTestBlockManager(data, keyTime, fakeTimeNowWithAutoAdvance(fakeTime.Add(1), 1)) // all bm* can see pre-existing block - verifyBlock(t, bm1, preexistingBlock, seededRandomData(10, 100)) - verifyBlock(t, bm2, preexistingBlock, seededRandomData(10, 100)) - verifyBlock(t, bm3, preexistingBlock, seededRandomData(10, 100)) + verifyBlock(ctx, t, bm1, preexistingBlock, seededRandomData(10, 100)) + verifyBlock(ctx, t, bm2, preexistingBlock, seededRandomData(10, 100)) + verifyBlock(ctx, t, bm3, preexistingBlock, seededRandomData(10, 100)) // write the same block in all managers. - sharedBlock := writeBlockAndVerify(t, bm1, seededRandomData(20, 100)) - writeBlockAndVerify(t, bm2, seededRandomData(20, 100)) - writeBlockAndVerify(t, bm3, seededRandomData(20, 100)) + sharedBlock := writeBlockAndVerify(ctx, t, bm1, seededRandomData(20, 100)) + writeBlockAndVerify(ctx, t, bm2, seededRandomData(20, 100)) + writeBlockAndVerify(ctx, t, bm3, seededRandomData(20, 100)) // write unique block per manager. - bm1block := writeBlockAndVerify(t, bm1, seededRandomData(31, 100)) - bm2block := writeBlockAndVerify(t, bm2, seededRandomData(32, 100)) - bm3block := writeBlockAndVerify(t, bm3, seededRandomData(33, 100)) + bm1block := writeBlockAndVerify(ctx, t, bm1, seededRandomData(31, 100)) + bm2block := writeBlockAndVerify(ctx, t, bm2, seededRandomData(32, 100)) + bm3block := writeBlockAndVerify(ctx, t, bm3, seededRandomData(33, 100)) // make sure they can't see each other's unflushed blocks. - verifyBlockNotFound(t, bm1, bm2block) - verifyBlockNotFound(t, bm1, bm3block) - verifyBlockNotFound(t, bm2, bm1block) - verifyBlockNotFound(t, bm2, bm3block) - verifyBlockNotFound(t, bm3, bm1block) - verifyBlockNotFound(t, bm3, bm2block) + verifyBlockNotFound(ctx, t, bm1, bm2block) + verifyBlockNotFound(ctx, t, bm1, bm3block) + verifyBlockNotFound(ctx, t, bm2, bm1block) + verifyBlockNotFound(ctx, t, bm2, bm3block) + verifyBlockNotFound(ctx, t, bm3, bm1block) + verifyBlockNotFound(ctx, t, bm3, bm2block) // now flush all writers, they still can't see each others' data. - bm1.Flush() - bm2.Flush() - bm3.Flush() - verifyBlockNotFound(t, bm1, bm2block) - verifyBlockNotFound(t, bm1, bm3block) - verifyBlockNotFound(t, bm2, bm1block) - verifyBlockNotFound(t, bm2, bm3block) - verifyBlockNotFound(t, bm3, bm1block) - verifyBlockNotFound(t, bm3, bm2block) + bm1.Flush(ctx) + bm2.Flush(ctx) + bm3.Flush(ctx) + verifyBlockNotFound(ctx, t, bm1, bm2block) + verifyBlockNotFound(ctx, t, bm1, bm3block) + verifyBlockNotFound(ctx, t, bm2, bm1block) + verifyBlockNotFound(ctx, t, bm2, bm3block) + verifyBlockNotFound(ctx, t, bm3, bm1block) + verifyBlockNotFound(ctx, t, bm3, bm2block) // new block manager at this point can see all data. bm4 := newTestBlockManager(data, keyTime, nil) - verifyBlock(t, bm4, preexistingBlock, seededRandomData(10, 100)) - verifyBlock(t, bm4, sharedBlock, seededRandomData(20, 100)) - verifyBlock(t, bm4, bm1block, seededRandomData(31, 100)) - verifyBlock(t, bm4, bm2block, seededRandomData(32, 100)) - verifyBlock(t, bm4, bm3block, seededRandomData(33, 100)) + verifyBlock(ctx, t, bm4, preexistingBlock, seededRandomData(10, 100)) + verifyBlock(ctx, t, bm4, sharedBlock, seededRandomData(20, 100)) + verifyBlock(ctx, t, bm4, bm1block, seededRandomData(31, 100)) + verifyBlock(ctx, t, bm4, bm2block, seededRandomData(32, 100)) + verifyBlock(ctx, t, bm4, bm3block, seededRandomData(33, 100)) if got, want := getIndexCount(data), 4; got != want { t.Errorf("unexpected index count before compaction: %v, wanted %v", got, want) } - if err := bm4.CompactIndexes(); err != nil { + if err := bm4.CompactIndexes(ctx); err != nil { t.Errorf("compaction error: %v", err) } if got, want := getIndexCount(data), 5; got != want { @@ -379,39 +391,41 @@ func TestBlockManagerConcurrency(t *testing.T) { // new block manager at this point can see all data. bm5 := newTestBlockManager(data, keyTime, nil) - verifyBlock(t, bm5, preexistingBlock, seededRandomData(10, 100)) - verifyBlock(t, bm5, sharedBlock, seededRandomData(20, 100)) - verifyBlock(t, bm5, bm1block, seededRandomData(31, 100)) - verifyBlock(t, bm5, bm2block, seededRandomData(32, 100)) - verifyBlock(t, bm5, bm3block, seededRandomData(33, 100)) - if err := bm5.CompactIndexes(); err != nil { + verifyBlock(ctx, t, bm5, preexistingBlock, seededRandomData(10, 100)) + verifyBlock(ctx, t, bm5, sharedBlock, seededRandomData(20, 100)) + verifyBlock(ctx, t, bm5, bm1block, seededRandomData(31, 100)) + verifyBlock(ctx, t, bm5, bm2block, seededRandomData(32, 100)) + verifyBlock(ctx, t, bm5, bm3block, seededRandomData(33, 100)) + if err := bm5.CompactIndexes(ctx); err != nil { t.Errorf("compaction error: %v", err) } } func TestDeleteBlock(t *testing.T) { + ctx := context.Background() data := map[string][]byte{} keyTime := map[string]time.Time{} bm := newTestBlockManager(data, keyTime, nil) - block1 := writeBlockAndVerify(t, bm, seededRandomData(10, 100)) - bm.Flush() - block2 := writeBlockAndVerify(t, bm, seededRandomData(11, 100)) + block1 := writeBlockAndVerify(ctx, t, bm, seededRandomData(10, 100)) + bm.Flush(ctx) + block2 := writeBlockAndVerify(ctx, t, bm, seededRandomData(11, 100)) if err := bm.DeleteBlock(block1); err != nil { t.Errorf("unable to delete block: %v", block1) } if err := bm.DeleteBlock(block2); err != nil { t.Errorf("unable to delete block: %v", block1) } - verifyBlockNotFound(t, bm, block1) - verifyBlockNotFound(t, bm, block2) - bm.Flush() + verifyBlockNotFound(ctx, t, bm, block1) + verifyBlockNotFound(ctx, t, bm, block2) + bm.Flush(ctx) bm = newTestBlockManager(data, keyTime, nil) dumpBlockManagerData(data) - verifyBlockNotFound(t, bm, block1) - verifyBlockNotFound(t, bm, block2) + verifyBlockNotFound(ctx, t, bm, block1) + verifyBlockNotFound(ctx, t, bm, block2) } func TestDeleteAndRecreate(t *testing.T) { + ctx := context.Background() // simulate race between delete/recreate and delete // delete happens at t0+10, recreate at t0+20 and second delete time is parameterized. // depending on it, the second delete results will be visible. @@ -430,8 +444,8 @@ func TestDeleteAndRecreate(t *testing.T) { data := map[string][]byte{} keyTime := map[string]time.Time{} bm := newTestBlockManager(data, keyTime, fakeTimeNowFrozen(fakeTime)) - block1 := writeBlockAndVerify(t, bm, seededRandomData(10, 100)) - bm.Flush() + block1 := writeBlockAndVerify(ctx, t, bm, seededRandomData(10, 100)) + bm.Flush(ctx) // delete but at given timestamp but don't commit yet. bm0 := newTestBlockManager(data, keyTime, fakeTimeNowWithAutoAdvance(tc.deletionTime, 1)) @@ -439,17 +453,17 @@ func TestDeleteAndRecreate(t *testing.T) { // delete it at t0+10 bm1 := newTestBlockManager(data, keyTime, fakeTimeNowWithAutoAdvance(fakeTime.Add(10), 1)) - verifyBlock(t, bm1, block1, seededRandomData(10, 100)) + verifyBlock(ctx, t, bm1, block1, seededRandomData(10, 100)) bm1.DeleteBlock(block1) - bm1.Flush() + bm1.Flush(ctx) // recreate at t0+20 bm2 := newTestBlockManager(data, keyTime, fakeTimeNowWithAutoAdvance(fakeTime.Add(20), 1)) - block2 := writeBlockAndVerify(t, bm2, seededRandomData(10, 100)) - bm2.Flush() + block2 := writeBlockAndVerify(ctx, t, bm2, seededRandomData(10, 100)) + bm2.Flush(ctx) // commit deletion from bm0 (t0+5) - bm0.Flush() + bm0.Flush(ctx) dumpBlockManagerData(data) @@ -459,9 +473,9 @@ func TestDeleteAndRecreate(t *testing.T) { bm3 := newTestBlockManager(data, keyTime, nil) if tc.isVisible { - verifyBlock(t, bm3, block1, seededRandomData(10, 100)) + verifyBlock(ctx, t, bm3, block1, seededRandomData(10, 100)) } else { - verifyBlockNotFound(t, bm3, block1) + verifyBlockNotFound(ctx, t, bm3, block1) } }) } @@ -473,7 +487,7 @@ func newTestBlockManager(data map[string][]byte, keyTime map[string]time.Time, t if timeFunc == nil { timeFunc = fakeTimeNowWithAutoAdvance(fakeTime, 1) } - bm, err := newManagerWithTime(st, FormattingOptions{ + bm, err := newManagerWithTime(context.Background(), st, FormattingOptions{ BlockFormat: "TESTONLY_MD5", MaxPackSize: maxPackSize, }, CachingOptions{}, timeFunc) @@ -513,19 +527,19 @@ func fakeTimeNowWithAutoAdvance(t time.Time, dt time.Duration) func() time.Time } } -func verifyBlockNotFound(t *testing.T, bm *Manager, blockID string) { +func verifyBlockNotFound(ctx context.Context, t *testing.T, bm *Manager, blockID string) { t.Helper() - b, err := bm.GetBlock(blockID) + b, err := bm.GetBlock(ctx, blockID) if err != storage.ErrBlockNotFound { t.Errorf("unexpected response from GetBlock(%q), got %v,%v, expected %v", blockID, b, err, storage.ErrBlockNotFound) } } -func verifyBlock(t *testing.T, bm *Manager, blockID string, b []byte) { +func verifyBlock(ctx context.Context, t *testing.T, bm *Manager, blockID string, b []byte) { t.Helper() - b2, err := bm.GetBlock(blockID) + b2, err := bm.GetBlock(ctx, blockID) if err != nil { t.Errorf("unable to read block %q: %v", blockID, err) return @@ -535,7 +549,7 @@ func verifyBlock(t *testing.T, bm *Manager, blockID string, b []byte) { t.Errorf("block %q data mismatch: got %x (nil:%v), wanted %x (nil:%v)", blockID, got, got == nil, want, want == nil) } - bi, err := bm.BlockInfo(blockID) + bi, err := bm.BlockInfo(ctx, blockID) if err != nil { t.Errorf("error getting block info %q: %v", blockID, err) } @@ -545,10 +559,10 @@ func verifyBlock(t *testing.T, bm *Manager, blockID string, b []byte) { } } -func writeBlockAndVerify(t *testing.T, bm *Manager, b []byte) string { +func writeBlockAndVerify(ctx context.Context, t *testing.T, bm *Manager, b []byte) string { t.Helper() - blockID, err := bm.WriteBlock(b, "") + blockID, err := bm.WriteBlock(ctx, b, "") if err != nil { t.Errorf("err: %v", err) } @@ -557,7 +571,7 @@ func writeBlockAndVerify(t *testing.T, bm *Manager, b []byte) string { t.Errorf("invalid block ID for %x, got %v, want %v", b, got, want) } - verifyBlock(t, bm, blockID, b) + verifyBlock(ctx, t, bm, blockID, b) return blockID } diff --git a/block/local_storage_cache.go b/block/local_storage_cache.go index 6c76242a9..f680612b2 100644 --- a/block/local_storage_cache.go +++ b/block/local_storage_cache.go @@ -1,11 +1,14 @@ package block import ( + "bytes" + "context" "crypto/hmac" "crypto/sha256" "encoding/json" "errors" "fmt" + "io" "sort" "sync" "time" @@ -34,8 +37,8 @@ type diskBlockCache struct { closed chan struct{} } -func (c *diskBlockCache) getBlock(virtualBlockID, physicalBlockID string, offset, length int64) ([]byte, error) { - b, err := c.cacheStorage.GetBlock(virtualBlockID, 0, -1) +func (c *diskBlockCache) getBlock(ctx context.Context, virtualBlockID, physicalBlockID string, offset, length int64) ([]byte, error) { + b, err := c.cacheStorage.GetBlock(ctx, virtualBlockID, 0, -1) if err == nil { b, err = c.verifyHMAC(b) if err == nil { @@ -49,39 +52,40 @@ func (c *diskBlockCache) getBlock(virtualBlockID, physicalBlockID string, offset log.Warn().Msgf("unable to read cache %v: %v", virtualBlockID, err) } - b, err = c.st.GetBlock(physicalBlockID, offset, length) + b, err = c.st.GetBlock(ctx, physicalBlockID, offset, length) if err == storage.ErrBlockNotFound { // not found in underlying storage return nil, err } if err == nil { - c.writeToCache(virtualBlockID, c.appendHMAC(b)) + c.writeToCacheBestEffort(ctx, virtualBlockID, b) } return b, err } -func (c *diskBlockCache) writeToCache(virtualBlockID string, checksummedData []byte) { - if err := c.cacheStorage.PutBlock(virtualBlockID, checksummedData); err != nil { - log.Warn().Msgf("unable to write cache items %v: %v", virtualBlockID, err) +func (c *diskBlockCache) writeToCacheBestEffort(ctx context.Context, virtualBlockID string, data []byte) { + rdr := io.MultiReader( + bytes.NewReader(data), + bytes.NewReader(c.computeHMAC(data)), + ) + if err := c.cacheStorage.PutBlock(ctx, virtualBlockID, rdr); err != nil { + log.Warn().Msgf("unable to write cache item %v: %v", virtualBlockID, err) } } -func (c *diskBlockCache) putBlock(blockID string, data []byte) error { - if err := c.st.PutBlock(blockID, data); err != nil { +func (c *diskBlockCache) putBlock(ctx context.Context, blockID string, data []byte) error { + if err := c.st.PutBlock(ctx, blockID, bytes.NewReader(data)); err != nil { return err } - // now write to cache on a best-effort basis - if err := c.cacheStorage.PutBlock(blockID, c.appendHMAC(data)); err != nil { - log.Warn().Msgf("unable to write cache item: %v", err) - } - c.deleteListCache() + c.writeToCacheBestEffort(ctx, blockID, data) + c.deleteListCache(ctx) return nil } -func (c *diskBlockCache) listIndexBlocks(full bool) ([]Info, error) { +func (c *diskBlockCache) listIndexBlocks(ctx context.Context, full bool) ([]Info, error) { var cachedListBlockID string if full { @@ -90,7 +94,7 @@ func (c *diskBlockCache) listIndexBlocks(full bool) ([]Info, error) { cachedListBlockID = activeListCacheItem } - ci, err := c.readBlocksFromCacheBlock(cachedListBlockID) + ci, err := c.readBlocksFromCacheBlock(ctx, cachedListBlockID) if err == nil { expirationTime := ci.Timestamp.Add(c.listCacheDuration) if time.Now().Before(expirationTime) { @@ -102,9 +106,9 @@ func (c *diskBlockCache) listIndexBlocks(full bool) ([]Info, error) { } log.Debug().Bool("full", full).Msg("listing index blocks from source") - blocks, err := listIndexBlocksFromStorage(c.st, full) + blocks, err := listIndexBlocksFromStorage(ctx, c.st, full) if err == nil { - c.saveListToCache(cachedListBlockID, &cachedList{ + c.saveListToCache(ctx, cachedListBlockID, &cachedList{ Blocks: blocks, Timestamp: time.Now(), }) @@ -113,18 +117,16 @@ func (c *diskBlockCache) listIndexBlocks(full bool) ([]Info, error) { return blocks, err } -func (c *diskBlockCache) saveListToCache(cachedListBlockID string, ci *cachedList) { +func (c *diskBlockCache) saveListToCache(ctx context.Context, cachedListBlockID string, ci *cachedList) { log.Debug().Str("blockID", cachedListBlockID).Int("count", len(ci.Blocks)).Msg("saving index blocks to cache") if data, err := json.Marshal(ci); err == nil { - if err := c.cacheStorage.PutBlock(cachedListBlockID, c.appendHMAC(data)); err != nil { - log.Warn().Str("blockID", cachedListBlockID).Err(err).Msg("can't save list") - } + c.writeToCacheBestEffort(ctx, cachedListBlockID, data) } } -func (c *diskBlockCache) readBlocksFromCacheBlock(blockID string) (*cachedList, error) { +func (c *diskBlockCache) readBlocksFromCacheBlock(ctx context.Context, blockID string) (*cachedList, error) { ci := &cachedList{} - data, err := c.cacheStorage.GetBlock(blockID, 0, -1) + data, err := c.cacheStorage.GetBlock(ctx, blockID, 0, -1) if err != nil { return nil, err } @@ -142,11 +144,10 @@ func (c *diskBlockCache) readBlocksFromCacheBlock(blockID string) (*cachedList, } -func (c *diskBlockCache) appendHMAC(data []byte) []byte { +func (c *diskBlockCache) computeHMAC(data []byte) []byte { h := hmac.New(sha256.New, c.hmacSecret) h.Write(data) // nolint:errcheck - validSignature := h.Sum(nil) - return append(append([]byte(nil), data...), validSignature...) + return h.Sum(nil) } func (c *diskBlockCache) verifyHMAC(b []byte) ([]byte, error) { @@ -157,9 +158,8 @@ func (c *diskBlockCache) verifyHMAC(b []byte) ([]byte, error) { p := len(b) - sha256.Size data := b[0:p] signature := b[p:] - h := hmac.New(sha256.New, c.hmacSecret) - h.Write(data) // nolint:errcheck - validSignature := h.Sum(nil) + + validSignature := c.computeHMAC(data) if len(signature) != len(validSignature) { return nil, errors.New("invalid signature length") } @@ -175,14 +175,14 @@ func (c *diskBlockCache) close() error { return nil } -func (c *diskBlockCache) sweepDirectoryPeriodically() { +func (c *diskBlockCache) sweepDirectoryPeriodically(ctx context.Context) { for { select { case <-c.closed: return case <-time.After(sweepCacheFrequency): - err := c.sweepDirectory() + err := c.sweepDirectory(ctx) if err != nil { log.Printf("warning: blockCache sweep failed: %v", err) } @@ -190,7 +190,7 @@ func (c *diskBlockCache) sweepDirectoryPeriodically() { } } -func (c *diskBlockCache) sweepDirectory() (err error) { +func (c *diskBlockCache) sweepDirectory(ctx context.Context) (err error) { c.mu.Lock() defer c.mu.Unlock() @@ -201,7 +201,8 @@ func (c *diskBlockCache) sweepDirectory() (err error) { t0 := time.Now() log.Debug().Msg("sweeping cache") - ch, cancel := c.cacheStorage.ListBlocks("") + ctx, cancel := context.WithCancel(ctx) + ch := c.cacheStorage.ListBlocks(ctx, "") defer cancel() var items []storage.BlockMetadata @@ -220,7 +221,7 @@ func (c *diskBlockCache) sweepDirectory() (err error) { var totalRetainedSize int64 for _, it := range items { if totalRetainedSize > c.maxSizeBytes { - if err := c.cacheStorage.DeleteBlock(it.BlockID); err != nil { + if err := c.cacheStorage.DeleteBlock(ctx, it.BlockID); err != nil { log.Warn().Msgf("unable to remove %v: %v", it.BlockID, err) } } else { @@ -232,9 +233,9 @@ func (c *diskBlockCache) sweepDirectory() (err error) { return nil } -func (c *diskBlockCache) deleteListCache() { +func (c *diskBlockCache) deleteListCache(ctx context.Context) { for _, it := range []string{fullListCacheItem, activeListCacheItem} { - if err := c.cacheStorage.DeleteBlock(it); err != nil && err != storage.ErrBlockNotFound { + if err := c.cacheStorage.DeleteBlock(ctx, it); err != nil && err != storage.ErrBlockNotFound { log.Warn().Err(err).Msg("unable to delete cache item") } } diff --git a/block/null_block_cache.go b/block/null_block_cache.go index 9cc019b67..7068cfa5c 100644 --- a/block/null_block_cache.go +++ b/block/null_block_cache.go @@ -1,6 +1,9 @@ package block import ( + "bytes" + "context" + "github.com/kopia/kopia/storage" ) @@ -8,16 +11,16 @@ type nullBlockCache struct { st storage.Storage } -func (c nullBlockCache) getBlock(virtualBlockID string, blockID string, offset, length int64) ([]byte, error) { - return c.st.GetBlock(blockID, offset, length) +func (c nullBlockCache) getBlock(ctx context.Context, virtualBlockID string, blockID string, offset, length int64) ([]byte, error) { + return c.st.GetBlock(ctx, blockID, offset, length) } -func (c nullBlockCache) putBlock(blockID string, data []byte) error { - return c.st.PutBlock(blockID, data) +func (c nullBlockCache) putBlock(ctx context.Context, blockID string, data []byte) error { + return c.st.PutBlock(ctx, blockID, bytes.NewReader(data)) } -func (c nullBlockCache) listIndexBlocks(full bool) ([]Info, error) { - return listIndexBlocksFromStorage(c.st, full) +func (c nullBlockCache) listIndexBlocks(ctx context.Context, full bool) ([]Info, error) { + return listIndexBlocksFromStorage(ctx, c.st, full) } func (c nullBlockCache) close() error { diff --git a/cli/app.go b/cli/app.go index 8e715b93b..18d75ea0d 100644 --- a/cli/app.go +++ b/cli/app.go @@ -1,8 +1,12 @@ package cli import ( + "context" + "fmt" "os" + "github.com/kopia/kopia/repo" + kingpin "gopkg.in/alecthomas/kingpin.v2" ) @@ -27,6 +31,24 @@ func helpFullAction(ctx *kingpin.ParseContext) error { return nil } +func noRepositoryAction(act func(ctx context.Context) error) func(ctx *kingpin.ParseContext) error { + return func(_ *kingpin.ParseContext) error { + return act(context.Background()) + } +} + +func repositoryAction(act func(ctx context.Context, rep *repo.Repository) error) func(ctx *kingpin.ParseContext) error { + return func(_ *kingpin.ParseContext) error { + ctx := context.Background() + rep := mustOpenRepository(ctx, nil) + err := act(ctx, rep) + if cerr := rep.Close(ctx); cerr != nil { + return fmt.Errorf("unable to close repository: %v", cerr) + } + return err + } +} + // App returns an instance of command-line application object. func App() *kingpin.Application { return app diff --git a/cli/command_block_index_list.go b/cli/command_block_index_list.go index 7e2fdd561..3f0f82287 100644 --- a/cli/command_block_index_list.go +++ b/cli/command_block_index_list.go @@ -1,11 +1,11 @@ package cli import ( + "context" "fmt" "github.com/kopia/kopia/block" - - kingpin "gopkg.in/alecthomas/kingpin.v2" + "github.com/kopia/kopia/repo" ) var ( @@ -14,17 +14,14 @@ var ( blockIndexListSummary = blockIndexListCommand.Flag("summary", "Display block summary").Bool() ) -func runListBlockIndexesAction(context *kingpin.ParseContext) error { - rep := mustOpenRepository(nil) - defer rep.Close() //nolint: errcheck - +func runListBlockIndexesAction(ctx context.Context, rep *repo.Repository) error { var blks []block.Info var err error if !*blockIndexListAll { - blks, err = rep.Blocks.ActiveIndexBlocks() + blks, err = rep.Blocks.ActiveIndexBlocks(ctx) } else { - blks, err = rep.Blocks.ListIndexBlocks() + blks, err = rep.Blocks.ListIndexBlocks(ctx) } if err != nil { @@ -43,5 +40,5 @@ func runListBlockIndexesAction(context *kingpin.ParseContext) error { } func init() { - blockIndexListCommand.Action(runListBlockIndexesAction) + blockIndexListCommand.Action(repositoryAction(runListBlockIndexesAction)) } diff --git a/cli/command_block_index_optimize.go b/cli/command_block_index_optimize.go index c52ea56d8..f11a456cd 100644 --- a/cli/command_block_index_optimize.go +++ b/cli/command_block_index_optimize.go @@ -1,20 +1,19 @@ package cli import ( - kingpin "gopkg.in/alecthomas/kingpin.v2" + "context" + + "github.com/kopia/kopia/repo" ) var ( optimizeCommand = blockIndexCommands.Command("optimize", "Optimize block indexes.") ) -func runOptimizeCommand(context *kingpin.ParseContext) error { - rep := mustOpenRepository(nil) - defer rep.Close() //nolint: errcheck - - return rep.Blocks.CompactIndexes() +func runOptimizeCommand(ctx context.Context, rep *repo.Repository) error { + return rep.Blocks.CompactIndexes(ctx) } func init() { - optimizeCommand.Action(runOptimizeCommand) + optimizeCommand.Action(repositoryAction(runOptimizeCommand)) } diff --git a/cli/command_block_index_show.go b/cli/command_block_index_show.go index b2350f2dc..0aad8ef08 100644 --- a/cli/command_block_index_show.go +++ b/cli/command_block_index_show.go @@ -1,14 +1,14 @@ package cli import ( + "context" "fmt" "sort" "time" "github.com/gogo/protobuf/proto" "github.com/kopia/kopia/internal/blockmgrpb" - - kingpin "gopkg.in/alecthomas/kingpin.v2" + "github.com/kopia/kopia/repo" ) var ( @@ -24,13 +24,10 @@ type blockIndexEntryInfo struct { inline bool } -func runShowBlockIndexesAction(context *kingpin.ParseContext) error { - rep := mustOpenRepository(nil) - defer rep.Close() //nolint: errcheck - +func runShowBlockIndexesAction(ctx context.Context, rep *repo.Repository) error { blockIDs := *blockIndexShowIDs if len(blockIDs) == 1 && blockIDs[0] == "active" { - b, err := rep.Blocks.ActiveIndexBlocks() + b, err := rep.Blocks.ActiveIndexBlocks(ctx) if err != nil { return err } @@ -46,7 +43,7 @@ func runShowBlockIndexesAction(context *kingpin.ParseContext) error { } for _, blockID := range blockIDs { - data, err := rep.Blocks.GetBlock(blockID) + data, err := rep.Blocks.GetBlock(ctx, blockID) if err != nil { return fmt.Errorf("can't read block %q: %v", blockID, err) } @@ -105,5 +102,5 @@ func sortIndexBlocks(lines []blockIndexEntryInfo) { } func init() { - blockIndexShowCommand.Action(runShowBlockIndexesAction) + blockIndexShowCommand.Action(repositoryAction(runShowBlockIndexesAction)) } diff --git a/cli/command_block_list.go b/cli/command_block_list.go index 1816e6924..776990b62 100644 --- a/cli/command_block_list.go +++ b/cli/command_block_list.go @@ -1,11 +1,12 @@ package cli import ( + "context" "fmt" "sort" "github.com/kopia/kopia/block" - kingpin "gopkg.in/alecthomas/kingpin.v2" + "github.com/kopia/kopia/repo" ) var ( @@ -17,10 +18,7 @@ var ( blockListSummary = blockListCommand.Flag("summary", "Summarize the list").Short('s').Bool() ) -func runListBlocksAction(context *kingpin.ParseContext) error { - rep := mustOpenRepository(nil) - defer rep.Close() //nolint: errcheck - +func runListBlocksAction(ctx context.Context, rep *repo.Repository) error { blocks, err := rep.Blocks.ListBlocks(*blockListPrefix) if err != nil { return err @@ -83,5 +81,5 @@ func comparePacks(a, b block.Info) bool { } func init() { - blockListCommand.Action(runListBlocksAction) + blockListCommand.Action(repositoryAction(runListBlocksAction)) } diff --git a/cli/command_block_repack.go b/cli/command_block_repack.go index 8eb3519e2..34cb67e26 100644 --- a/cli/command_block_repack.go +++ b/cli/command_block_repack.go @@ -1,7 +1,9 @@ package cli import ( - kingpin "gopkg.in/alecthomas/kingpin.v2" + "context" + + "github.com/kopia/kopia/repo" ) var ( @@ -9,13 +11,10 @@ var ( blockRepackSizeThreshold = blockRepackCommand.Flag("max-size", "Max size of block to re-pack").Default("500000").Uint64() ) -func runBlockRepackAction(context *kingpin.ParseContext) error { - rep := mustOpenRepository(nil) - defer rep.Close() //nolint: errcheck - - return rep.Blocks.Repackage(*blockRepackSizeThreshold) +func runBlockRepackAction(ctx context.Context, rep *repo.Repository) error { + return rep.Blocks.Repackage(ctx, *blockRepackSizeThreshold) } func init() { - blockRepackCommand.Action(runBlockRepackAction) + blockRepackCommand.Action(repositoryAction(runBlockRepackAction)) } diff --git a/cli/command_block_rm.go b/cli/command_block_rm.go index 99f240315..5e4d6835f 100644 --- a/cli/command_block_rm.go +++ b/cli/command_block_rm.go @@ -1,9 +1,9 @@ package cli import ( - "github.com/kopia/kopia/repo" + "context" - kingpin "gopkg.in/alecthomas/kingpin.v2" + "github.com/kopia/kopia/repo" ) var ( @@ -12,10 +12,7 @@ var ( removeBlockIDs = removeBlockCommand.Arg("id", "IDs of blocks to remove").Required().Strings() ) -func runRemoveBlockCommand(context *kingpin.ParseContext) error { - rep := mustOpenRepository(nil) - defer rep.Close() //nolint: errcheck - +func runRemoveBlockCommand(ctx context.Context, rep *repo.Repository) error { for _, blockID := range *removeBlockIDs { if err := removeBlock(rep, blockID); err != nil { return err @@ -31,5 +28,5 @@ func removeBlock(r *repo.Repository, blockID string) error { func init() { setupShowCommand(removeBlockCommand) - removeBlockCommand.Action(runRemoveBlockCommand) + removeBlockCommand.Action(repositoryAction(runRemoveBlockCommand)) } diff --git a/cli/command_block_show.go b/cli/command_block_show.go index 8f551d90d..019dc4def 100644 --- a/cli/command_block_show.go +++ b/cli/command_block_show.go @@ -2,10 +2,9 @@ package cli import ( "bytes" + "context" "github.com/kopia/kopia/repo" - - kingpin "gopkg.in/alecthomas/kingpin.v2" ) var ( @@ -14,12 +13,9 @@ var ( showBlockIDs = showBlockCommand.Arg("id", "IDs of blocks to show").Required().Strings() ) -func runShowBlockCommand(context *kingpin.ParseContext) error { - rep := mustOpenRepository(nil) - defer rep.Close() //nolint: errcheck - +func runShowBlockCommand(ctx context.Context, rep *repo.Repository) error { for _, blockID := range *showBlockIDs { - if err := showBlock(rep, blockID); err != nil { + if err := showBlock(ctx, rep, blockID); err != nil { return err } } @@ -27,8 +23,8 @@ func runShowBlockCommand(context *kingpin.ParseContext) error { return nil } -func showBlock(r *repo.Repository, blockID string) error { - data, err := r.Blocks.GetBlock(blockID) +func showBlock(ctx context.Context, r *repo.Repository, blockID string) error { + data, err := r.Blocks.GetBlock(ctx, blockID) if err != nil { return err } @@ -38,5 +34,5 @@ func showBlock(r *repo.Repository, blockID string) error { func init() { setupShowCommand(showBlockCommand) - showBlockCommand.Action(runShowBlockCommand) + showBlockCommand.Action(repositoryAction(runShowBlockCommand)) } diff --git a/cli/command_block_stats.go b/cli/command_block_stats.go index 639401b55..a4e6c648b 100644 --- a/cli/command_block_stats.go +++ b/cli/command_block_stats.go @@ -1,13 +1,14 @@ package cli import ( + "context" "fmt" "sort" "strconv" "github.com/kopia/kopia/block" "github.com/kopia/kopia/internal/units" - kingpin "gopkg.in/alecthomas/kingpin.v2" + "github.com/kopia/kopia/repo" ) var ( @@ -15,10 +16,7 @@ var ( blockStatsRaw = blockStatsCommand.Flag("raw", "Raw numbers").Short('r').Bool() ) -func runBlockStatsAction(context *kingpin.ParseContext) error { - rep := mustOpenRepository(nil) - defer rep.Close() //nolint: errcheck - +func runBlockStatsAction(ctx context.Context, rep *repo.Repository) error { blocks, err := rep.Blocks.ListBlocks("") if err != nil { return err @@ -82,5 +80,5 @@ func percentileSize(p int, blocks []block.Info) int64 { } func init() { - blockStatsCommand.Action(runBlockStatsAction) + blockStatsCommand.Action(repositoryAction(runBlockStatsAction)) } diff --git a/cli/command_cat.go b/cli/command_cat.go index 49982e1ff..8ae3d160d 100644 --- a/cli/command_cat.go +++ b/cli/command_cat.go @@ -1,12 +1,12 @@ package cli import ( + "context" "io" "os" + "github.com/kopia/kopia/repo" "github.com/kopia/kopia/snapshot" - - kingpin "gopkg.in/alecthomas/kingpin.v2" ) var ( @@ -14,17 +14,14 @@ var ( catCommandPath = catCommand.Arg("path", "Path").Required().String() ) -func runCatCommand(context *kingpin.ParseContext) error { - rep := mustOpenRepository(nil) - defer rep.Close() //nolint: errcheck - +func runCatCommand(ctx context.Context, rep *repo.Repository) error { mgr := snapshot.NewManager(rep) - oid, err := parseObjectID(mgr, *catCommandPath) + oid, err := parseObjectID(ctx, mgr, *catCommandPath) if err != nil { return err } - r, err := rep.Objects.Open(oid) + r, err := rep.Objects.Open(ctx, oid) if err != nil { return err } @@ -34,5 +31,5 @@ func runCatCommand(context *kingpin.ParseContext) error { } func init() { - catCommand.Action(runCatCommand) + catCommand.Action(repositoryAction(runCatCommand)) } diff --git a/cli/command_ls.go b/cli/command_ls.go index af803c7b4..c5615eeb8 100644 --- a/cli/command_ls.go +++ b/cli/command_ls.go @@ -1,15 +1,15 @@ package cli import ( + "context" "fmt" "strconv" "strings" "github.com/kopia/kopia/fs" "github.com/kopia/kopia/object" + "github.com/kopia/kopia/repo" "github.com/kopia/kopia/snapshot" - - kingpin "gopkg.in/alecthomas/kingpin.v2" ) const timeFormat = "02 Jan 06 15:04:05" @@ -24,13 +24,10 @@ var ( lsCommandPath = lsCommand.Arg("path", "Path").Required().String() ) -func runLSCommand(context *kingpin.ParseContext) error { - rep := mustOpenRepository(nil) - defer rep.Close() //nolint: errcheck - +func runLSCommand(ctx context.Context, rep *repo.Repository) error { mgr := snapshot.NewManager(rep) - oid, err := parseObjectID(mgr, *lsCommandPath) + oid, err := parseObjectID(ctx, mgr, *lsCommandPath) if err != nil { return err } @@ -43,17 +40,17 @@ func runLSCommand(context *kingpin.ParseContext) error { } } - return listDirectory(mgr, prefix, oid, "") + return listDirectory(ctx, mgr, prefix, oid, "") } func init() { - lsCommand.Action(runLSCommand) + lsCommand.Action(repositoryAction(runLSCommand)) } -func listDirectory(mgr *snapshot.Manager, prefix string, oid object.ID, indent string) error { +func listDirectory(ctx context.Context, mgr *snapshot.Manager, prefix string, oid object.ID, indent string) error { d := mgr.DirectoryEntry(oid) - entries, err := d.Readdir() + entries, err := d.Readdir(ctx) if err != nil { return err } @@ -91,7 +88,7 @@ func listDirectory(mgr *snapshot.Manager, prefix string, oid object.ID, indent s } fmt.Println(info) if *lsCommandRecursive && m.FileMode().IsDir() { - if listerr := listDirectory(mgr, prefix+m.Name+"/", objectID, indent+" "); listerr != nil { + if listerr := listDirectory(ctx, mgr, prefix+m.Name+"/", objectID, indent+" "); listerr != nil { return listerr } } diff --git a/cli/command_manifest_ls.go b/cli/command_manifest_ls.go index 5a65ff1fc..7575928f3 100644 --- a/cli/command_manifest_ls.go +++ b/cli/command_manifest_ls.go @@ -1,11 +1,12 @@ package cli import ( + "context" "fmt" "sort" "strings" - "gopkg.in/alecthomas/kingpin.v2" + "github.com/kopia/kopia/repo" ) var ( @@ -15,12 +16,10 @@ var ( ) func init() { - manifestListCommand.Action(listManifestItems) + manifestListCommand.Action(repositoryAction(listManifestItems)) } -func listManifestItems(context *kingpin.ParseContext) error { - rep := mustOpenRepository(nil) - +func listManifestItems(ctx context.Context, rep *repo.Repository) error { filter := map[string]string{} for _, kv := range *manifestListFilter { diff --git a/cli/command_manifest_rm.go b/cli/command_manifest_rm.go index ba4ad7401..61f2168f2 100644 --- a/cli/command_manifest_rm.go +++ b/cli/command_manifest_rm.go @@ -1,6 +1,10 @@ package cli -import "gopkg.in/alecthomas/kingpin.v2" +import ( + "context" + + "github.com/kopia/kopia/repo" +) var ( manifestRemoveCommand = manifestCommands.Command("rm", "Remove manifest items") @@ -8,12 +12,10 @@ var ( ) func init() { - manifestRemoveCommand.Action(removeMetadataItem) + manifestRemoveCommand.Action(repositoryAction(removeMetadataItem)) } -func removeMetadataItem(context *kingpin.ParseContext) error { - rep := mustOpenRepository(nil) - +func removeMetadataItem(ctx context.Context, rep *repo.Repository) error { for _, it := range *manifestRemoveItems { rep.Manifests.Delete(it) } diff --git a/cli/command_manifest_show.go b/cli/command_manifest_show.go index 0656ea42d..75b774acd 100644 --- a/cli/command_manifest_show.go +++ b/cli/command_manifest_show.go @@ -2,10 +2,11 @@ package cli import ( "bytes" + "context" "fmt" "os" - "gopkg.in/alecthomas/kingpin.v2" + "github.com/kopia/kopia/repo" ) var ( @@ -14,12 +15,10 @@ var ( ) func init() { - manifestShowCommand.Action(showManifestItems) + manifestShowCommand.Action(repositoryAction(showManifestItems)) } -func showManifestItems(context *kingpin.ParseContext) error { - rep := mustOpenRepository(nil) - +func showManifestItems(ctx context.Context, rep *repo.Repository) error { for _, it := range *manifestShowItems { md, err := rep.Manifests.GetMetadata(it) if err != nil { diff --git a/cli/command_mount.go b/cli/command_mount.go index 5eec6b679..6adb58e67 100644 --- a/cli/command_mount.go +++ b/cli/command_mount.go @@ -1,14 +1,14 @@ package cli import ( + "context" "fmt" "github.com/kopia/kopia/fs" "github.com/kopia/kopia/fs/cachefs" "github.com/kopia/kopia/fs/loggingfs" + "github.com/kopia/kopia/repo" "github.com/kopia/kopia/snapshot" - - kingpin "gopkg.in/alecthomas/kingpin.v2" ) var ( @@ -19,16 +19,14 @@ var ( mountTraceFS = mountCommand.Flag("trace-fs", "Trace filesystem operations").Bool() ) -func runMountCommand(context *kingpin.ParseContext) error { - rep := mustOpenRepository(nil) - +func runMountCommand(ctx context.Context, rep *repo.Repository) error { mgr := snapshot.NewManager(rep) var entry fs.Directory if *mountObjectID == "all" { entry = mgr.AllSourcesEntry() } else { - oid, err := parseObjectID(mgr, *mountObjectID) + oid, err := parseObjectID(ctx, mgr, *mountObjectID) if err != nil { return err } @@ -53,5 +51,5 @@ func runMountCommand(context *kingpin.ParseContext) error { func init() { setupFSCacheFlags(mountCommand) - mountCommand.Action(runMountCommand) + mountCommand.Action(repositoryAction(runMountCommand)) } diff --git a/cli/command_object_ls.go b/cli/command_object_ls.go index 138eac32c..963a230d4 100644 --- a/cli/command_object_ls.go +++ b/cli/command_object_ls.go @@ -1,9 +1,10 @@ package cli import ( + "context" "fmt" - kingpin "gopkg.in/alecthomas/kingpin.v2" + "github.com/kopia/kopia/repo" ) var ( @@ -11,10 +12,7 @@ var ( objectListPrefix = objectListCommand.Flag("prefix", "Prefix").String() ) -func runListObjectsAction(context *kingpin.ParseContext) error { - rep := mustOpenRepository(nil) - defer rep.Close() //nolint: errcheck - +func runListObjectsAction(ctx context.Context, rep *repo.Repository) error { info, err := rep.Blocks.ListBlocks(*objectListPrefix) if err != nil { return err @@ -28,5 +26,5 @@ func runListObjectsAction(context *kingpin.ParseContext) error { } func init() { - objectListCommand.Action(runListObjectsAction) + objectListCommand.Action(repositoryAction(runListObjectsAction)) } diff --git a/cli/command_object_show.go b/cli/command_object_show.go index a5d59fd78..ee6e9a826 100644 --- a/cli/command_object_show.go +++ b/cli/command_object_show.go @@ -1,14 +1,13 @@ package cli import ( + "context" "io" "github.com/kopia/kopia/object" "github.com/kopia/kopia/snapshot" "github.com/kopia/kopia/repo" - - kingpin "gopkg.in/alecthomas/kingpin.v2" ) var ( @@ -17,19 +16,16 @@ var ( showObjectIDs = showCommand.Arg("id", "IDs of objects to show").Required().Strings() ) -func runShowCommand(context *kingpin.ParseContext) error { - rep := mustOpenRepository(nil) - defer rep.Close() //nolint: errcheck - +func runShowCommand(ctx context.Context, rep *repo.Repository) error { mgr := snapshot.NewManager(rep) for _, oidString := range *showObjectIDs { - oid, err := parseObjectID(mgr, oidString) + oid, err := parseObjectID(ctx, mgr, oidString) if err != nil { return err } - if err := showObject(rep, oid); err != nil { + if err := showObject(ctx, rep, oid); err != nil { return err } } @@ -37,10 +33,10 @@ func runShowCommand(context *kingpin.ParseContext) error { return nil } -func showObject(r *repo.Repository, oid object.ID) error { +func showObject(ctx context.Context, r *repo.Repository, oid object.ID) error { var rd io.ReadCloser - rd, err := r.Objects.Open(oid) + rd, err := r.Objects.Open(ctx, oid) if err != nil { return err } @@ -51,5 +47,5 @@ func showObject(r *repo.Repository, oid object.ID) error { func init() { setupShowCommand(showCommand) - showCommand.Action(runShowCommand) + showCommand.Action(repositoryAction(runShowCommand)) } diff --git a/cli/command_object_verify.go b/cli/command_object_verify.go index 230ca5bec..5ceaa8e21 100644 --- a/cli/command_object_verify.go +++ b/cli/command_object_verify.go @@ -1,6 +1,7 @@ package cli import ( + "context" "fmt" "os" @@ -8,9 +9,8 @@ import ( "github.com/kopia/kopia/fs" "github.com/kopia/kopia/object" + "github.com/kopia/kopia/repo" "github.com/kopia/kopia/snapshot" - - kingpin "gopkg.in/alecthomas/kingpin.v2" ) var ( @@ -34,7 +34,7 @@ func (v *verifier) reportError(path string, err error) bool { return len(v.errors) >= *verifyCommandErrorThreshold } -func (v *verifier) verifyDirectory(oid object.ID, path string) error { +func (v *verifier) verifyDirectory(ctx context.Context, oid object.ID, path string) error { if v.visited[oid.String()] { return nil } @@ -43,7 +43,7 @@ func (v *verifier) verifyDirectory(oid object.ID, path string) error { log.Printf("verifying directory %q (%v)", path, oid) d := v.mgr.DirectoryEntry(oid) - entries, err := d.Readdir() + entries, err := d.Readdir(ctx) if err != nil { if v.reportError(path, fmt.Errorf("error reading directory %q %v: %v", path, oid, err)) { return err @@ -51,7 +51,7 @@ func (v *verifier) verifyDirectory(oid object.ID, path string) error { } for _, e := range entries { - if err = v.verifyDirectoryEntry(path, e); err != nil { + if err = v.verifyDirectoryEntry(ctx, path, e); err != nil { return err } } @@ -59,13 +59,13 @@ func (v *verifier) verifyDirectory(oid object.ID, path string) error { return nil } -func (v *verifier) verifyDirectoryEntry(path string, e fs.Entry) error { +func (v *verifier) verifyDirectoryEntry(ctx context.Context, path string, e fs.Entry) error { m := e.Metadata() objectID := e.(object.HasObjectID).ObjectID() childPath := path + "/" + m.Name if m.FileMode().IsDir() { if *verifyCommandRecursive { - if err := v.verifyDirectory(objectID, childPath); err != nil { + if err := v.verifyDirectory(ctx, objectID, childPath); err != nil { if v.reportError(childPath, err) { return err } @@ -73,7 +73,7 @@ func (v *verifier) verifyDirectoryEntry(path string, e fs.Entry) error { } } - if err := v.verifyObject(objectID, childPath, m.FileSize); err != nil { + if err := v.verifyObject(ctx, objectID, childPath, m.FileSize); err != nil { if v.reportError(childPath, err) { return err } @@ -81,7 +81,7 @@ func (v *verifier) verifyDirectoryEntry(path string, e fs.Entry) error { return nil } -func (v *verifier) verifyObject(oid object.ID, path string, expectedLength int64) error { +func (v *verifier) verifyObject(ctx context.Context, oid object.ID, path string, expectedLength int64) error { if v.visited[oid.String()] { return nil } @@ -93,7 +93,7 @@ func (v *verifier) verifyObject(oid object.ID, path string, expectedLength int64 log.Printf("verifying object %v (%v) with length %v", path, oid, expectedLength) } - length, _, err := v.om.VerifyObject(oid) + length, _, err := v.om.VerifyObject(ctx, oid) if err != nil { return fmt.Errorf("invalid object %q: %v", oid, err) } @@ -107,13 +107,10 @@ func (v *verifier) verifyObject(oid object.ID, path string, expectedLength int64 return nil } -func runVerifyCommand(context *kingpin.ParseContext) error { - rep := mustOpenRepository(nil) - defer rep.Close() //nolint: errcheck - +func runVerifyCommand(ctx context.Context, rep *repo.Repository) error { mgr := snapshot.NewManager(rep) - oid, err := parseObjectID(mgr, *verifyCommandPath) + oid, err := parseObjectID(ctx, mgr, *verifyCommandPath) if err != nil { return err } @@ -126,10 +123,10 @@ func runVerifyCommand(context *kingpin.ParseContext) error { } if *verifyCommandRecursive { - _ = v.verifyDirectory(oid, oid.String()) + _ = v.verifyDirectory(ctx, oid, oid.String()) } - if err := v.verifyObject(oid, oid.String(), -1); err != nil { + if err := v.verifyObject(ctx, oid, oid.String(), -1); err != nil { if v.reportError(".", err) { return err } @@ -151,5 +148,5 @@ func runVerifyCommand(context *kingpin.ParseContext) error { } func init() { - verifyCommand.Action(runVerifyCommand) + verifyCommand.Action(repositoryAction(runVerifyCommand)) } diff --git a/cli/command_policy_ls.go b/cli/command_policy_ls.go index 4f90efcdc..3d5afddf7 100644 --- a/cli/command_policy_ls.go +++ b/cli/command_policy_ls.go @@ -1,10 +1,11 @@ package cli import ( + "context" "fmt" + "github.com/kopia/kopia/repo" "github.com/kopia/kopia/snapshot" - kingpin "gopkg.in/alecthomas/kingpin.v2" ) var ( @@ -12,11 +13,10 @@ var ( ) func init() { - policyListCommand.Action(listPolicies) + policyListCommand.Action(repositoryAction(listPolicies)) } -func listPolicies(context *kingpin.ParseContext) error { - rep := mustOpenRepository(nil) +func listPolicies(ctx context.Context, rep *repo.Repository) error { mgr := snapshot.NewPolicyManager(rep) policies, err := mgr.ListPolicies() diff --git a/cli/command_policy_remove.go b/cli/command_policy_remove.go index 9f787103c..d6964e135 100644 --- a/cli/command_policy_remove.go +++ b/cli/command_policy_remove.go @@ -1,10 +1,12 @@ package cli import ( + "context" + "github.com/rs/zerolog/log" + "github.com/kopia/kopia/repo" "github.com/kopia/kopia/snapshot" - kingpin "gopkg.in/alecthomas/kingpin.v2" ) var ( @@ -14,13 +16,10 @@ var ( ) func init() { - policyRemoveCommand.Action(removePolicy) + policyRemoveCommand.Action(repositoryAction(removePolicy)) } -func removePolicy(context *kingpin.ParseContext) error { - rep := mustOpenRepository(nil) - defer rep.Close() //nolint: errcheck - +func removePolicy(ctx context.Context, rep *repo.Repository) error { mgr := snapshot.NewPolicyManager(rep) targets, err := policyTargets(policyRemoveGlobal, policyRemoveTargets) diff --git a/cli/command_policy_set.go b/cli/command_policy_set.go index f96cb52ed..ae7cd950d 100644 --- a/cli/command_policy_set.go +++ b/cli/command_policy_set.go @@ -1,14 +1,15 @@ package cli import ( + "context" "fmt" "sort" "strconv" "github.com/rs/zerolog/log" + "github.com/kopia/kopia/repo" "github.com/kopia/kopia/snapshot" - kingpin "gopkg.in/alecthomas/kingpin.v2" ) var ( @@ -42,13 +43,10 @@ var ( ) func init() { - policySetCommand.Action(setPolicy) + policySetCommand.Action(repositoryAction(setPolicy)) } -func setPolicy(context *kingpin.ParseContext) error { - rep := mustOpenRepository(nil) - defer rep.Close() //nolint: errcheck - +func setPolicy(ctx context.Context, rep *repo.Repository) error { mgr := snapshot.NewPolicyManager(rep) targets, err := policyTargets(policySetGlobal, policySetTargets) diff --git a/cli/command_policy_show.go b/cli/command_policy_show.go index 9f76414f0..0dfd760f6 100644 --- a/cli/command_policy_show.go +++ b/cli/command_policy_show.go @@ -2,12 +2,13 @@ package cli import ( "bytes" + "context" "fmt" "os" "github.com/kopia/kopia/internal/units" + "github.com/kopia/kopia/repo" "github.com/kopia/kopia/snapshot" - kingpin "gopkg.in/alecthomas/kingpin.v2" ) var ( @@ -18,11 +19,10 @@ var ( ) func init() { - policyShowCommand.Action(showPolicy) + policyShowCommand.Action(repositoryAction(showPolicy)) } -func showPolicy(context *kingpin.ParseContext) error { - rep := mustOpenRepository(nil) +func showPolicy(ctx context.Context, rep *repo.Repository) error { pmgr := snapshot.NewPolicyManager(rep) targets, err := policyTargets(policyShowGlobal, policyShowTargets) diff --git a/cli/command_repository_cache.go b/cli/command_repository_cache.go index 3ee1cd985..760143d01 100644 --- a/cli/command_repository_cache.go +++ b/cli/command_repository_cache.go @@ -1,9 +1,10 @@ package cli import ( + "context" + "github.com/kopia/kopia/block" "github.com/kopia/kopia/repo" - kingpin "gopkg.in/alecthomas/kingpin.v2" ) var ( @@ -14,16 +15,16 @@ var ( cacheMaxListCacheDuration = cacheCommand.Flag("max-list-cache-duration", "Duration of index cache").Default("600s").Duration() ) -func runCacheCommand(context *kingpin.ParseContext) error { +func runCacheCommand(ctx context.Context, rep *repo.Repository) error { opts := block.CachingOptions{ CacheDirectory: *cacheDirectory, MaxCacheSizeBytes: *cacheMaxCacheSizeMB << 20, MaxListCacheDurationSec: int(cacheMaxListCacheDuration.Seconds()), } - return repo.SetCachingConfig(getContext(), repositoryConfigFileName(), opts) + return repo.SetCachingConfig(ctx, repositoryConfigFileName(), opts) } func init() { - cacheCommand.Action(runCacheCommand) + cacheCommand.Action(repositoryAction(runCacheCommand)) } diff --git a/cli/command_repository_connect.go b/cli/command_repository_connect.go index d19e8f746..817c7ee7d 100644 --- a/cli/command_repository_connect.go +++ b/cli/command_repository_connect.go @@ -46,13 +46,13 @@ func init() { setupConnectOptions(connectCommand) } -func runConnectCommandWithStorage(st storage.Storage) error { +func runConnectCommandWithStorage(ctx context.Context, st storage.Storage) error { creds, err := getRepositoryCredentials(false) if err != nil { return err } - err = repo.Connect(context.Background(), repositoryConfigFileName(), st, creds, connectOptions()) + err = repo.Connect(ctx, repositoryConfigFileName(), st, creds, connectOptions()) if err != nil { return err } diff --git a/cli/command_repository_create.go b/cli/command_repository_create.go index da421b1f6..279f87ab2 100644 --- a/cli/command_repository_create.go +++ b/cli/command_repository_create.go @@ -1,6 +1,7 @@ package cli import ( + "context" "fmt" "os" @@ -42,8 +43,9 @@ func newRepositoryOptionsFromFlags() *repo.NewRepositoryOptions { } } -func ensureEmpty(s storage.Storage) error { - ch, cancel := s.ListBlocks("") +func ensureEmpty(ctx context.Context, s storage.Storage) error { + ctx, cancel := context.WithCancel(ctx) + ch := s.ListBlocks(ctx, "") d, hasData := <-ch cancel() @@ -60,8 +62,8 @@ func ensureEmpty(s storage.Storage) error { return nil } -func runCreateCommandWithStorage(st storage.Storage) error { - err := ensureEmpty(st) +func runCreateCommandWithStorage(ctx context.Context, st storage.Storage) error { + err := ensureEmpty(ctx, st) if err != nil { return fmt.Errorf("unable to get repository storage: %v", err) } @@ -90,12 +92,12 @@ func runCreateCommandWithStorage(st storage.Storage) error { fmt.Fprintf(os.Stderr, " object splitter: NEVER\n") } - if err := repo.Initialize(st, options, creds); err != nil { + if err := repo.Initialize(ctx, st, options, creds); err != nil { return fmt.Errorf("cannot initialize repository: %v", err) } if !*createOnly { - err := repo.Connect(getContext(), repositoryConfigFileName(), st, creds, connectOptions()) + err := repo.Connect(ctx, repositoryConfigFileName(), st, creds, connectOptions()) if err != nil { return err } diff --git a/cli/command_repository_disconnect.go b/cli/command_repository_disconnect.go index 679375ec5..9ec933862 100644 --- a/cli/command_repository_disconnect.go +++ b/cli/command_repository_disconnect.go @@ -1,8 +1,9 @@ package cli import ( + "context" + "github.com/kopia/kopia/repo" - "gopkg.in/alecthomas/kingpin.v2" ) var ( @@ -10,9 +11,9 @@ var ( ) func init() { - disconnectCommand.Action(runDisconnectCommand) + disconnectCommand.Action(noRepositoryAction(runDisconnectCommand)) } -func runDisconnectCommand(context *kingpin.ParseContext) error { +func runDisconnectCommand(ctx context.Context) error { return repo.Disconnect(repositoryConfigFileName()) } diff --git a/cli/command_repository_migrate.go b/cli/command_repository_migrate.go index d50a36a28..69d96b5e4 100644 --- a/cli/command_repository_migrate.go +++ b/cli/command_repository_migrate.go @@ -1,6 +1,7 @@ package cli import ( + "context" "fmt" "github.com/rs/zerolog/log" @@ -8,7 +9,6 @@ import ( "github.com/kopia/kopia/object" "github.com/kopia/kopia/repo" "github.com/kopia/kopia/snapshot" - kingpin "gopkg.in/alecthomas/kingpin.v2" ) var ( @@ -21,8 +21,7 @@ var ( migrateIgnoreErrors = migrateCommand.Flag("ignore-errors", "Ignore errors when reading source backup").Bool() ) -func runMigrateCommand(context *kingpin.ParseContext) error { - destRepo := mustOpenRepository(nil) +func runMigrateCommand(ctx context.Context, destRepo *repo.Repository) error { destSM := snapshot.NewManager(destRepo) uploader := snapshot.NewUploader(destRepo) @@ -30,7 +29,7 @@ func runMigrateCommand(context *kingpin.ParseContext) error { uploader.IgnoreFileErrors = *migrateIgnoreErrors onCtrlC(uploader.Cancel) - sourceRepo, err := repo.Open(getContext(), *migrateSourceConfig, applyOptionsFromFlags(nil)) + sourceRepo, err := repo.Open(ctx, *migrateSourceConfig, applyOptionsFromFlags(nil)) if err != nil { return fmt.Errorf("can't open source repository: %v", err) } @@ -47,7 +46,7 @@ func runMigrateCommand(context *kingpin.ParseContext) error { break } - if err := migrateSingleSource(uploader, sourceSM, destSM, s); err != nil { + if err := migrateSingleSource(ctx, uploader, sourceSM, destSM, s); err != nil { return err } } @@ -58,7 +57,7 @@ func runMigrateCommand(context *kingpin.ParseContext) error { return err } d := sourceSM.DirectoryEntry(dirOID) - newm, err := uploader.Upload(d, snapshot.SourceInfo{Host: "temp"}, nil) + newm, err := uploader.Upload(ctx, d, snapshot.SourceInfo{Host: "temp"}, nil) if err != nil { return fmt.Errorf("error migrating directory %v: %v", dirOID, err) } @@ -69,7 +68,7 @@ func runMigrateCommand(context *kingpin.ParseContext) error { return nil } -func migrateSingleSource(uploader *snapshot.Uploader, sourceSM, destSM *snapshot.Manager, s snapshot.SourceInfo) error { +func migrateSingleSource(ctx context.Context, uploader *snapshot.Uploader, sourceSM, destSM *snapshot.Manager, s snapshot.SourceInfo) error { log.Printf("migrating source %v", s) manifests := sourceSM.ListSnapshotManifests(&s) @@ -80,7 +79,7 @@ func migrateSingleSource(uploader *snapshot.Uploader, sourceSM, destSM *snapshot for _, m := range filterSnapshotsToMigrate(snapshots) { d := sourceSM.DirectoryEntry(m.RootObjectID) - newm, err := uploader.Upload(d, m.Source, nil) + newm, err := uploader.Upload(ctx, d, m.Source, nil) if err != nil { return fmt.Errorf("error migrating shapshot %v @ %v: %v", m.Source, m.StartTime, err) } @@ -129,5 +128,5 @@ func getSourcesToMigrate(mgr *snapshot.Manager) ([]snapshot.SourceInfo, error) { } func init() { - migrateCommand.Action(runMigrateCommand) + migrateCommand.Action(repositoryAction(runMigrateCommand)) } diff --git a/cli/command_repository_status.go b/cli/command_repository_status.go index 34b5c941a..59cb11443 100644 --- a/cli/command_repository_status.go +++ b/cli/command_repository_status.go @@ -1,6 +1,7 @@ package cli import ( + "context" "encoding/json" "fmt" "io/ioutil" @@ -8,17 +9,14 @@ import ( "github.com/kopia/kopia/internal/scrubber" "github.com/kopia/kopia/internal/units" - kingpin "gopkg.in/alecthomas/kingpin.v2" + "github.com/kopia/kopia/repo" ) var ( statusCommand = repositoryCommands.Command("status", "Display the status of connected repository.") ) -func runStatusCommand(context *kingpin.ParseContext) error { - rep := mustOpenRepository(nil) - defer rep.Close() //nolint: errcheck - +func runStatusCommand(ctx context.Context, rep *repo.Repository) error { fmt.Printf("Config file: %v\n", rep.ConfigFile) entries, err := ioutil.ReadDir(rep.CacheDirectory) if err != nil { @@ -67,5 +65,5 @@ func runStatusCommand(context *kingpin.ParseContext) error { } func init() { - statusCommand.Action(runStatusCommand) + statusCommand.Action(repositoryAction(runStatusCommand)) } diff --git a/cli/command_snapshot_create.go b/cli/command_snapshot_create.go index 7fab8c03e..e4c1b7206 100644 --- a/cli/command_snapshot_create.go +++ b/cli/command_snapshot_create.go @@ -1,6 +1,7 @@ package cli import ( + "context" "encoding/json" "errors" "fmt" @@ -13,11 +14,8 @@ import ( "github.com/rs/zerolog/log" - "github.com/kopia/kopia/object" "github.com/kopia/kopia/repo" "github.com/kopia/kopia/snapshot" - - kingpin "gopkg.in/alecthomas/kingpin.v2" ) const ( @@ -33,18 +31,10 @@ var ( snapshotCreateDescription = snapshotCreateCommand.Flag("description", "Free-form snapshot description.").String() snapshotCreateForceHash = snapshotCreateCommand.Flag("force-hash", "Force hashing of source files for a given percentage of files [0..100]").Default("0").Int() snapshotCreateHashCacheMinAge = snapshotCreateCommand.Flag("hash-cache-min-age", "Do not hash-cache files below certain age").Default("1h").Duration() - snapshotCreateWriteBack = snapshotCreateCommand.Flag("async-write", "Perform updates asynchronously.").PlaceHolder("N").Default("0").Int() snapshotCreateParallelUploads = snapshotCreateCommand.Flag("parallel", "Upload N files in parallel").PlaceHolder("N").Default("0").Int() ) -func runBackupCommand(c *kingpin.ParseContext) error { - rep := mustOpenRepository(&repo.Options{ - ObjectManagerOptions: object.ManagerOptions{ - WriteBack: *snapshotCreateWriteBack, - }, - }) - defer rep.Close() //nolint: errcheck - +func runBackupCommand(ctx context.Context, rep *repo.Repository) error { mgr := snapshot.NewManager(rep) pmgr := snapshot.NewPolicyManager(rep) @@ -83,7 +73,7 @@ func runBackupCommand(c *kingpin.ParseContext) error { sourceInfo := snapshot.SourceInfo{Path: filepath.Clean(dir), Host: getHostName(), UserName: getUserName()} log.Info().Str("source", sourceInfo.String()).Msg("snapshotting") - if err := snapshotSingleSource(rep, mgr, pmgr, u, sourceInfo); err != nil { + if err := snapshotSingleSource(ctx, rep, mgr, pmgr, u, sourceInfo); err != nil { return err } } @@ -91,7 +81,7 @@ func runBackupCommand(c *kingpin.ParseContext) error { return nil } -func snapshotSingleSource(rep *repo.Repository, mgr *snapshot.Manager, pmgr *snapshot.PolicyManager, u *snapshot.Uploader, sourceInfo snapshot.SourceInfo) error { +func snapshotSingleSource(ctx context.Context, rep *repo.Repository, mgr *snapshot.Manager, pmgr *snapshot.PolicyManager, u *snapshot.Uploader, sourceInfo snapshot.SourceInfo) error { t0 := time.Now() rep.Blocks.ResetStats() policy, err := pmgr.GetEffectivePolicy(sourceInfo) @@ -112,7 +102,7 @@ func snapshotSingleSource(rep *repo.Repository, mgr *snapshot.Manager, pmgr *sna } log.Debug().Msgf("uploading %v using previous manifest %v", sourceInfo, previousManifest) - manifest, err := u.Upload(localEntry, sourceInfo, previousManifest) + manifest, err := u.Upload(ctx, localEntry, sourceInfo, previousManifest) if err != nil { return err } @@ -205,5 +195,5 @@ func getHostName() string { } func init() { - snapshotCreateCommand.Action(runBackupCommand) + snapshotCreateCommand.Action(repositoryAction(runBackupCommand)) } diff --git a/cli/command_snapshot_estimate.go b/cli/command_snapshot_estimate.go index 8a5b8b54c..80b4d5c72 100644 --- a/cli/command_snapshot_estimate.go +++ b/cli/command_snapshot_estimate.go @@ -1,17 +1,18 @@ package cli import ( + "context" "fmt" "path/filepath" "time" + "github.com/kopia/kopia/repo" + "github.com/rs/zerolog/log" "github.com/kopia/kopia/fs" "github.com/kopia/kopia/internal/units" "github.com/kopia/kopia/snapshot" - - kingpin "gopkg.in/alecthomas/kingpin.v2" ) var ( @@ -59,10 +60,7 @@ func makeBuckets() buckets { } } -func runSnapshotEstimateCommand(c *kingpin.ParseContext) error { - rep := mustOpenRepository(nil) - defer rep.Close() //nolint: errcheck - +func runSnapshotEstimateCommand(ctx context.Context, rep *repo.Repository) error { pmgr := snapshot.NewPolicyManager(rep) path, err := filepath.Abs(*snapshotEstimateSource) @@ -79,7 +77,7 @@ func runSnapshotEstimateCommand(c *kingpin.ParseContext) error { var stats snapshot.Stats ib := makeBuckets() eb := makeBuckets() - if err := estimate(".", mustGetLocalFSEntry(path), &policy.FilesPolicy, &stats, ib, eb); err != nil { + if err := estimate(ctx, ".", mustGetLocalFSEntry(path), &policy.FilesPolicy, &stats, ib, eb); err != nil { return err } @@ -112,7 +110,7 @@ func showBuckets(b buckets) { } } } -func estimate(relativePath string, entry fs.Entry, pol *snapshot.FilesPolicy, stats *snapshot.Stats, ib, eb buckets) error { +func estimate(ctx context.Context, relativePath string, entry fs.Entry, pol *snapshot.FilesPolicy, stats *snapshot.Stats, ib, eb buckets) error { if !pol.ShouldInclude(entry.Metadata()) { eb.add(relativePath, entry.Metadata().FileSize) stats.ExcludedFileCount++ @@ -125,13 +123,13 @@ func estimate(relativePath string, entry fs.Entry, pol *snapshot.FilesPolicy, st if !*snapshotEstimateQuiet { log.Printf("Scanning %q...\n", relativePath) } - children, err := entry.Readdir() + children, err := entry.Readdir(ctx) if err != nil { return err } for _, child := range children { - if err := estimate(filepath.Join(relativePath, child.Metadata().Name), child, pol, stats, ib, eb); err != nil { + if err := estimate(ctx, filepath.Join(relativePath, child.Metadata().Name), child, pol, stats, ib, eb); err != nil { return err } } @@ -145,5 +143,5 @@ func estimate(relativePath string, entry fs.Entry, pol *snapshot.FilesPolicy, st } func init() { - snapshotEstimate.Action(runSnapshotEstimateCommand) + snapshotEstimate.Action(repositoryAction(runSnapshotEstimateCommand)) } diff --git a/cli/command_snapshot_expire.go b/cli/command_snapshot_expire.go index 60b9b9e4d..b041798cb 100644 --- a/cli/command_snapshot_expire.go +++ b/cli/command_snapshot_expire.go @@ -1,6 +1,7 @@ package cli import ( + "context" "fmt" "os" "sort" @@ -9,9 +10,8 @@ import ( "github.com/rs/zerolog/log" + "github.com/kopia/kopia/repo" "github.com/kopia/kopia/snapshot" - - kingpin "gopkg.in/alecthomas/kingpin.v2" ) var ( @@ -226,10 +226,7 @@ func expireSnapshots(pmgr *snapshot.PolicyManager, snapshots []*snapshot.Manifes return toDelete, nil } -func runExpireCommand(context *kingpin.ParseContext) error { - rep := mustOpenRepository(nil) - defer rep.Close() //nolint: errcheck - +func runExpireCommand(ctx context.Context, rep *repo.Repository) error { mgr := snapshot.NewManager(rep) pmgr := snapshot.NewPolicyManager(rep) snapshotNames, err := getSnapshotNamesToExpire(mgr) @@ -288,5 +285,5 @@ func filterHostAndUser(snapshots []*snapshot.Manifest) []*snapshot.Manifest { } func init() { - snapshotExpireCommand.Action(runExpireCommand) + snapshotExpireCommand.Action(repositoryAction(runExpireCommand)) } diff --git a/cli/command_snapshot_list.go b/cli/command_snapshot_list.go index 1cab88486..6ad5e524f 100644 --- a/cli/command_snapshot_list.go +++ b/cli/command_snapshot_list.go @@ -1,6 +1,7 @@ package cli import ( + "context" "fmt" "path/filepath" "sort" @@ -9,9 +10,8 @@ import ( "github.com/rs/zerolog/log" "github.com/kopia/kopia/internal/units" + "github.com/kopia/kopia/repo" "github.com/kopia/kopia/snapshot" - - kingpin "gopkg.in/alecthomas/kingpin.v2" ) var ( @@ -67,10 +67,7 @@ func findManifestIDs(mgr *snapshot.Manager, source string) ([]string, string, er return manifestIDs, relPath, err } -func runBackupsCommand(context *kingpin.ParseContext) error { - rep := mustOpenRepository(nil) - defer rep.Close() //nolint: errcheck - +func runBackupsCommand(ctx context.Context, rep *repo.Repository) error { mgr := snapshot.NewManager(rep) manifestIDs, relPath, err := findManifestIDs(mgr, *snapshotListPath) @@ -160,5 +157,5 @@ func deltaBytes(b int64) string { } func init() { - snapshotListCommand.Action(runBackupsCommand) + snapshotListCommand.Action(repositoryAction(runBackupsCommand)) } diff --git a/cli/command_storage_list.go b/cli/command_storage_list.go index 6380cd89e..895fd0f7c 100644 --- a/cli/command_storage_list.go +++ b/cli/command_storage_list.go @@ -1,9 +1,10 @@ package cli import ( + "context" "fmt" - kingpin "gopkg.in/alecthomas/kingpin.v2" + "github.com/kopia/kopia/repo" ) var ( @@ -11,12 +12,10 @@ var ( storageListPrefix = storageListCommand.Flag("prefix", "Block prefix").String() ) -func runListStorageBlocks(context *kingpin.ParseContext) error { - rep := mustOpenRepository(nil) - defer rep.Close() //nolint: errcheck - - ch, cancel := rep.Storage.ListBlocks(*storageListPrefix) +func runListStorageBlocks(ctx context.Context, rep *repo.Repository) error { + ctx, cancel := context.WithCancel(ctx) defer cancel() + ch := rep.Storage.ListBlocks(ctx, *storageListPrefix) for b := range ch { if b.Error != nil { @@ -30,5 +29,5 @@ func runListStorageBlocks(context *kingpin.ParseContext) error { } func init() { - storageListCommand.Action(runListStorageBlocks) + storageListCommand.Action(repositoryAction(runListStorageBlocks)) } diff --git a/cli/command_storage_show.go b/cli/command_storage_show.go index 7775fd3eb..1885a81ab 100644 --- a/cli/command_storage_show.go +++ b/cli/command_storage_show.go @@ -2,11 +2,12 @@ package cli import ( "bytes" + "context" "fmt" "io" "os" - kingpin "gopkg.in/alecthomas/kingpin.v2" + "github.com/kopia/kopia/repo" ) var ( @@ -14,12 +15,9 @@ var ( storageShowBlockIDs = storageShowCommand.Arg("blockIDs", "Block IDs").Required().Strings() ) -func runShowStorageBlocks(context *kingpin.ParseContext) error { - rep := mustOpenRepository(nil) - defer rep.Close() //nolint: errcheck - +func runShowStorageBlocks(ctx context.Context, rep *repo.Repository) error { for _, b := range *storageShowBlockIDs { - d, err := rep.Storage.GetBlock(b, 0, -1) + d, err := rep.Storage.GetBlock(ctx, b, 0, -1) if err != nil { return fmt.Errorf("error getting %v: %v", b, err) } @@ -32,5 +30,5 @@ func runShowStorageBlocks(context *kingpin.ParseContext) error { } func init() { - storageShowCommand.Action(runShowStorageBlocks) + storageShowCommand.Action(repositoryAction(runShowStorageBlocks)) } diff --git a/cli/command_storage_sweep.go b/cli/command_storage_sweep.go index cb22c2d45..52ddaceb4 100644 --- a/cli/command_storage_sweep.go +++ b/cli/command_storage_sweep.go @@ -1,13 +1,12 @@ package cli import ( + "context" "fmt" "time" "github.com/kopia/kopia/repo" "github.com/kopia/kopia/storage" - - kingpin "gopkg.in/alecthomas/kingpin.v2" ) var ( @@ -16,11 +15,8 @@ var ( storageSweepPrintUnused = storageSweepCommand.Flag("print", "Do not delete blocks, but print which ones would be deleted").Short('n').Bool() ) -func runStorageSweepAction(context *kingpin.ParseContext) error { - rep := mustOpenRepository(nil) - defer rep.Close() //nolint: errcheck - - indexBlocks, err := rep.Blocks.ActiveIndexBlocks() +func runStorageSweepAction(ctx context.Context, rep *repo.Repository) error { + indexBlocks, err := rep.Blocks.ActiveIndexBlocks(ctx) if err != nil { return err } @@ -30,15 +26,16 @@ func runStorageSweepAction(context *kingpin.ParseContext) error { inUseIndexBlocks[ib.BlockID] = true } - ch, cancel := rep.Storage.ListBlocks("") + ctx, cancel := context.WithCancel(ctx) defer cancel() + ch := rep.Storage.ListBlocks(ctx, "") for bm := range ch { if bm.Error != nil { return bm.Error } - if err = sweepBlock(rep, bm, inUseIndexBlocks); err != nil { + if err = sweepBlock(ctx, rep, bm, inUseIndexBlocks); err != nil { return err } } @@ -46,7 +43,7 @@ func runStorageSweepAction(context *kingpin.ParseContext) error { return nil } -func sweepBlock(rep *repo.Repository, bm storage.BlockMetadata, inUseIndexBlocks map[string]bool) error { +func sweepBlock(ctx context.Context, rep *repo.Repository, bm storage.BlockMetadata, inUseIndexBlocks map[string]bool) error { age := time.Since(bm.TimeStamp) inUse := rep.Blocks.IsStorageBlockInUse(bm.BlockID) || inUseIndexBlocks[bm.BlockID] || bm.BlockID == repo.FormatBlockID keep := inUse || age < *storageSweepAgeThreshold @@ -60,9 +57,9 @@ func sweepBlock(rep *repo.Repository, bm storage.BlockMetadata, inUseIndexBlocks } fmt.Printf("deleting unused block %v age %v\n", bm.BlockID, age) - return rep.Storage.DeleteBlock(bm.BlockID) + return rep.Storage.DeleteBlock(ctx, bm.BlockID) } func init() { - storageSweepCommand.Action(runStorageSweepAction) + storageSweepCommand.Action(repositoryAction(runStorageSweepAction)) } diff --git a/cli/config.go b/cli/config.go index 3391e6fa0..8d4ff14e7 100644 --- a/cli/config.go +++ b/cli/config.go @@ -63,12 +63,8 @@ func waitForCtrlC() { <-done } -func getContext() context.Context { - return context.Background() -} - -func openRepository(opts *repo.Options) (*repo.Repository, error) { - r, err := repo.Open(getContext(), repositoryConfigFileName(), applyOptionsFromFlags(opts)) +func openRepository(ctx context.Context, opts *repo.Options) (*repo.Repository, error) { + r, err := repo.Open(ctx, repositoryConfigFileName(), applyOptionsFromFlags(opts)) if os.IsNotExist(err) { return nil, fmt.Errorf("not connected to a repository, use 'kopia connect'") } @@ -96,8 +92,8 @@ func applyOptionsFromFlags(opts *repo.Options) *repo.Options { return opts } -func mustOpenRepository(opts *repo.Options) *repo.Repository { - s, err := openRepository(opts) +func mustOpenRepository(ctx context.Context, opts *repo.Options) *repo.Repository { + s, err := openRepository(ctx, opts) failOnError(err) return s } diff --git a/cli/objref.go b/cli/objref.go index 1c951c2ba..3cfdd177e 100644 --- a/cli/objref.go +++ b/cli/objref.go @@ -1,6 +1,7 @@ package cli import ( + "context" "fmt" "strings" @@ -10,7 +11,7 @@ import ( ) // ParseObjectID interprets the given ID string and returns corresponding object.ID. -func parseObjectID(mgr *snapshot.Manager, id string) (object.ID, error) { +func parseObjectID(ctx context.Context, mgr *snapshot.Manager, id string) (object.ID, error) { head, tail := splitHeadTail(id) if len(head) == 0 { return object.NullID, fmt.Errorf("invalid object ID: %v", id) @@ -30,11 +31,11 @@ func parseObjectID(mgr *snapshot.Manager, id string) (object.ID, error) { return object.NullID, err } - return parseNestedObjectID(dir, tail) + return parseNestedObjectID(ctx, dir, tail) } //nolint:interfacer -func parseNestedObjectID(startingDir fs.Directory, id string) (object.ID, error) { +func parseNestedObjectID(ctx context.Context, startingDir fs.Directory, id string) (object.ID, error) { head, tail := splitHeadTail(id) var current fs.Entry = startingDir for head != "" { @@ -43,7 +44,7 @@ func parseNestedObjectID(startingDir fs.Directory, id string) (object.ID, error) return object.NullID, fmt.Errorf("entry not found '%v': parent is not a directory", head) } - entries, err := dir.Readdir() + entries, err := dir.Readdir(ctx) if err != nil { return object.NullID, err } diff --git a/cli/storage_providers.go b/cli/storage_providers.go index 383000b88..c3d7c5ba5 100644 --- a/cli/storage_providers.go +++ b/cli/storage_providers.go @@ -19,24 +19,26 @@ func RegisterStorageConnectFlags( // Set up 'create' subcommand cc := createCommand.Command(name, "Create repository in "+description) flags(cc) - cc.Action(func(ctx *kingpin.ParseContext) error { - st, err := connect(getContext()) + cc.Action(func(_ *kingpin.ParseContext) error { + ctx := context.Background() + st, err := connect(ctx) if err != nil { return fmt.Errorf("can't connect to storage: %v", err) } - return runCreateCommandWithStorage(st) + return runCreateCommandWithStorage(ctx, st) }) // Set up 'connect' subcommand cc = connectCommand.Command(name, "Connect to repository in "+description) flags(cc) - cc.Action(func(ctx *kingpin.ParseContext) error { - st, err := connect(getContext()) + cc.Action(func(_ *kingpin.ParseContext) error { + ctx := context.Background() + st, err := connect(ctx) if err != nil { return fmt.Errorf("can't connect to storage: %v", err) } - return runConnectCommandWithStorage(st) + return runConnectCommandWithStorage(ctx, st) }) } diff --git a/fs/cachefs/cache.go b/fs/cachefs/cache.go index f1f50a054..b9984b91f 100644 --- a/fs/cachefs/cache.go +++ b/fs/cachefs/cache.go @@ -1,6 +1,7 @@ package cachefs import ( + "context" "sync" "time" @@ -73,18 +74,18 @@ func (c *Cache) remove(e *cacheEntry) { } // Loader provides data to be stored in the cache. -type Loader func() (fs.Entries, error) +type Loader func(ctx context.Context) (fs.Entries, error) // Readdir reads the contents of a provided directory using ObjectID of a directory (if any) to cache // the results. -func (c *Cache) Readdir(d fs.Directory) (fs.Entries, error) { +func (c *Cache) Readdir(ctx context.Context, d fs.Directory) (fs.Entries, error) { if h, ok := d.(object.HasObjectID); ok { cacheID := h.ObjectID().String() cacheExpiration := 24 * time.Hour - return c.getEntries(cacheID, cacheExpiration, d.Readdir) + return c.getEntries(ctx, cacheID, cacheExpiration, d.Readdir) } - return d.Readdir() + return d.Readdir(ctx) } func (c *Cache) getEntriesFromCache(id string) fs.Entries { @@ -110,9 +111,9 @@ func (c *Cache) getEntriesFromCache(id string) fs.Entries { // getEntries consults the cache and either retrieves the contents of directory listing from the cache // or invokes the provides callback and adds the results to cache. -func (c *Cache) getEntries(id string, expirationTime time.Duration, cb Loader) (fs.Entries, error) { +func (c *Cache) getEntries(ctx context.Context, id string, expirationTime time.Duration, cb Loader) (fs.Entries, error) { if c == nil { - return cb() + return cb(ctx) } c.mu.Lock() @@ -123,7 +124,7 @@ func (c *Cache) getEntries(id string, expirationTime time.Duration, cb Loader) ( if c.debug { log.Printf("cache miss for %q", id) } - raw, err := cb() + raw, err := cb(ctx) if err != nil { return nil, err } diff --git a/fs/cachefs/cache_test.go b/fs/cachefs/cache_test.go index a2c6a7904..03064c42b 100644 --- a/fs/cachefs/cache_test.go +++ b/fs/cachefs/cache_test.go @@ -1,6 +1,7 @@ package cachefs import ( + "context" "errors" "fmt" "path/filepath" @@ -19,8 +20,8 @@ type cacheSource struct { callCounter map[string]int } -func (cs *cacheSource) get(id string) func() (fs.Entries, error) { - return func() (fs.Entries, error) { +func (cs *cacheSource) get(id string) func(ctx context.Context) (fs.Entries, error) { + return func(context.Context) (fs.Entries, error) { cs.callCounter[id]++ d, ok := cs.data[id] if !ok { @@ -119,6 +120,7 @@ func (cv *cacheVerifier) reset() { } func TestCache(t *testing.T) { + ctx := context.Background() c := NewCache(&Options{ MaxCachedDirectories: 4, MaxCachedEntries: 100, @@ -147,57 +149,57 @@ func TestCache(t *testing.T) { cv.verifyCacheOrdering(t) // fetch id1 - c.getEntries(id1, expirationTime, cs.get(id1)) + c.getEntries(ctx, id1, expirationTime, cs.get(id1)) cv.verifyCacheMiss(t, id1) cv.verifyCacheOrdering(t, id1) // fetch id1 again - cache hit, no change - c.getEntries(id1, expirationTime, cs.get(id1)) + c.getEntries(ctx, id1, expirationTime, cs.get(id1)) cv.verifyCacheHit(t, id1) cv.verifyCacheOrdering(t, id1) // fetch id2 - c.getEntries(id2, expirationTime, cs.get(id2)) + c.getEntries(ctx, id2, expirationTime, cs.get(id2)) cv.verifyCacheMiss(t, id2) cv.verifyCacheOrdering(t, id2, id1) // fetch id1 again - cache hit, id1 moved to the top of the LRU list - c.getEntries(id1, expirationTime, cs.get(id1)) + c.getEntries(ctx, id1, expirationTime, cs.get(id1)) cv.verifyCacheHit(t, id1) cv.verifyCacheOrdering(t, id1, id2) // fetch id2 again - c.getEntries(id2, expirationTime, cs.get(id2)) + c.getEntries(ctx, id2, expirationTime, cs.get(id2)) cv.verifyCacheHit(t, id2) cv.verifyCacheOrdering(t, id2, id1) // fetch id3 - c.getEntries(id3, expirationTime, cs.get(id3)) + c.getEntries(ctx, id3, expirationTime, cs.get(id3)) cv.verifyCacheMiss(t, id3) cv.verifyCacheOrdering(t, id3, id2, id1) // fetch id4 - c.getEntries(id4, expirationTime, cs.get(id4)) + c.getEntries(ctx, id4, expirationTime, cs.get(id4)) cv.verifyCacheMiss(t, id4) cv.verifyCacheOrdering(t, id4, id3) // fetch id1 again - c.getEntries(id1, expirationTime, cs.get(id1)) + c.getEntries(ctx, id1, expirationTime, cs.get(id1)) cv.verifyCacheMiss(t, id1) cv.verifyCacheOrdering(t, id1, id4) // fetch id5, it's a big one that expels all but one - c.getEntries(id5, expirationTime, cs.get(id5)) + c.getEntries(ctx, id5, expirationTime, cs.get(id5)) cv.verifyCacheMiss(t, id5) cv.verifyCacheOrdering(t, id5, id1) // fetch id6 - c.getEntries(id6, expirationTime, cs.get(id6)) + c.getEntries(ctx, id6, expirationTime, cs.get(id6)) cv.verifyCacheMiss(t, id6) cv.verifyCacheOrdering(t, id6) // fetch id7 - c.getEntries(id7, expirationTime, cs.get(id7)) + c.getEntries(ctx, id7, expirationTime, cs.get(id7)) cv.verifyCacheMiss(t, id7) cv.verifyCacheOrdering(t, id6) } diff --git a/fs/cachefs/cachefs.go b/fs/cachefs/cachefs.go index c79182cb6..cf699de04 100644 --- a/fs/cachefs/cachefs.go +++ b/fs/cachefs/cachefs.go @@ -2,25 +2,27 @@ package cachefs import ( + "context" + "github.com/kopia/kopia/fs" ) // DirectoryCacher reads and potentially caches directory entries for a given directory. type DirectoryCacher interface { - Readdir(d fs.Directory) (fs.Entries, error) + Readdir(ctx context.Context, d fs.Directory) (fs.Entries, error) } -type context struct { +type cacheContext struct { cacher DirectoryCacher } type directory struct { - ctx *context + ctx *cacheContext fs.Directory } -func (d *directory) Readdir() (fs.Entries, error) { - entries, err := d.ctx.cacher.Readdir(d.Directory) +func (d *directory) Readdir(ctx context.Context) (fs.Entries, error) { + entries, err := d.ctx.cacher.Readdir(ctx, d.Directory) if err != nil { return entries, err } @@ -33,21 +35,21 @@ func (d *directory) Readdir() (fs.Entries, error) { } type file struct { - ctx *context + ctx *cacheContext fs.File } type symlink struct { - ctx *context + ctx *cacheContext fs.Symlink } // Wrap returns an Entry that wraps another Entry and caches directory reads. func Wrap(e fs.Entry, cacher DirectoryCacher) fs.Entry { - return wrapWithContext(e, &context{cacher}) + return wrapWithContext(e, &cacheContext{cacher}) } -func wrapWithContext(e fs.Entry, opts *context) fs.Entry { +func wrapWithContext(e fs.Entry, opts *cacheContext) fs.Entry { switch e := e.(type) { case fs.Directory: return fs.Directory(&directory{opts, e}) diff --git a/fs/entry.go b/fs/entry.go index 2d0a6d005..9bb86646f 100644 --- a/fs/entry.go +++ b/fs/entry.go @@ -1,6 +1,7 @@ package fs import ( + "context" "io" "sort" "strings" @@ -25,19 +26,19 @@ type Reader interface { // File represents an entry that is a file. type File interface { Entry - Open() (Reader, error) + Open(ctx context.Context) (Reader, error) } // Directory represents contents of a directory. type Directory interface { Entry - Readdir() (Entries, error) + Readdir(ctx context.Context) (Entries, error) } // Symlink represents a symbolic link entry. type Symlink interface { Entry - Readlink() (string, error) + Readlink(ctx context.Context) (string, error) } // FindByName returns an entry with a given name, or nil if not found. diff --git a/fs/localfs/local_fs.go b/fs/localfs/local_fs.go index 0b2f81913..ef44012be 100644 --- a/fs/localfs/local_fs.go +++ b/fs/localfs/local_fs.go @@ -1,6 +1,7 @@ package localfs import ( + "context" "fmt" "io" "os" @@ -50,7 +51,7 @@ type filesystemFile struct { filesystemEntry } -func (fsd *filesystemDirectory) Readdir() (fs.Entries, error) { +func (fsd *filesystemDirectory) Readdir(ctx context.Context) (fs.Entries, error) { f, direrr := os.Open(fsd.path) if direrr != nil { return nil, direrr @@ -95,7 +96,7 @@ func (erc *fileWithMetadata) EntryMetadata() (*fs.EntryMetadata, error) { return entryMetadataFromFileInfo(fi), nil } -func (fsf *filesystemFile) Open() (fs.Reader, error) { +func (fsf *filesystemFile) Open(ctx context.Context) (fs.Reader, error) { f, err := os.Open(fsf.path) if err != nil { return nil, err @@ -104,7 +105,7 @@ func (fsf *filesystemFile) Open() (fs.Reader, error) { return &fileWithMetadata{f}, nil } -func (fsl *filesystemSymlink) Readlink() (string, error) { +func (fsl *filesystemSymlink) Readlink(ctx context.Context) (string, error) { return os.Readlink(fsl.path) } diff --git a/fs/localfs/local_fs_test.go b/fs/localfs/local_fs_test.go index bd862fa0f..70ce91ed4 100644 --- a/fs/localfs/local_fs_test.go +++ b/fs/localfs/local_fs_test.go @@ -1,6 +1,7 @@ package localfs import ( + "context" "fmt" "io/ioutil" "os" @@ -13,6 +14,7 @@ import ( ) func TestFiles(t *testing.T) { + ctx := context.Background() var err error tmp, err := ioutil.TempDir("", "kopia") @@ -37,7 +39,7 @@ func TestFiles(t *testing.T) { t.Errorf("error when dir empty directory: %v", err) } - entries, err := dir.Readdir() + entries, err := dir.Readdir(ctx) if err != nil { t.Errorf("error gettind dir Entries: %v", err) } @@ -59,7 +61,7 @@ func TestFiles(t *testing.T) { t.Errorf("error when dir directory with files: %v", err) } - entries, err = dir.Readdir() + entries, err = dir.Readdir(ctx) if err != nil { t.Errorf("error gettind dir Entries: %v", err) } diff --git a/fs/loggingfs/loggingfs.go b/fs/loggingfs/loggingfs.go index 2cf23e0e6..6a09879a2 100644 --- a/fs/loggingfs/loggingfs.go +++ b/fs/loggingfs/loggingfs.go @@ -2,6 +2,7 @@ package loggingfs import ( + "context" "time" "github.com/rs/zerolog/log" @@ -19,9 +20,9 @@ type loggingDirectory struct { fs.Directory } -func (ld *loggingDirectory) Readdir() (fs.Entries, error) { +func (ld *loggingDirectory) Readdir(ctx context.Context) (fs.Entries, error) { t0 := time.Now() - entries, err := ld.Directory.Readdir() + entries, err := ld.Directory.Readdir(ctx) dt := time.Since(t0) ld.options.printf(ld.options.prefix+"Readdir(%v) took %v and returned %v items", fs.EntryPath(ld), dt, len(entries)) loggingEntries := make(fs.Entries, len(entries)) diff --git a/internal/fusemount/fusefs.go b/internal/fusemount/fusefs.go index 6db6bded9..d7753f2a6 100644 --- a/internal/fusemount/fusefs.go +++ b/internal/fusemount/fusefs.go @@ -37,7 +37,7 @@ type fuseFileNode struct { } func (f *fuseFileNode) ReadAll(ctx context.Context) ([]byte, error) { - reader, err := f.entry.(fs.File).Open() + reader, err := f.entry.(fs.File).Open(ctx) if err != nil { return nil, err } @@ -55,7 +55,7 @@ func (dir *fuseDirectoryNode) directory() fs.Directory { } func (dir *fuseDirectoryNode) Lookup(ctx context.Context, fileName string) (fusefs.Node, error) { - entries, err := dir.directory().Readdir() + entries, err := dir.directory().Readdir(ctx) if err != nil { if os.IsNotExist(err) { return nil, fuse.ENOENT @@ -73,7 +73,7 @@ func (dir *fuseDirectoryNode) Lookup(ctx context.Context, fileName string) (fuse } func (dir *fuseDirectoryNode) ReadDirAll(ctx context.Context) ([]fuse.Dirent, error) { - entries, err := dir.directory().Readdir() + entries, err := dir.directory().Readdir(ctx) if err != nil { return nil, err } @@ -106,7 +106,7 @@ type fuseSymlinkNode struct { } func (sl *fuseSymlinkNode) Readlink(ctx context.Context, req *fuse.ReadlinkRequest) (string, error) { - return sl.entry.(fs.Symlink).Readlink() + return sl.entry.(fs.Symlink).Readlink(ctx) } func newFuseNode(e fs.Entry) (fusefs.Node, error) { diff --git a/internal/mockfs/mockfs.go b/internal/mockfs/mockfs.go index 2c54c5858..419eeae9d 100644 --- a/internal/mockfs/mockfs.go +++ b/internal/mockfs/mockfs.go @@ -3,6 +3,7 @@ package mockfs import ( "bytes" + "context" "fmt" "io" "sort" @@ -150,7 +151,7 @@ func (imd *Directory) FailReaddir(err error) { } // Readdir gets the contents of a directory. -func (imd *Directory) Readdir() (fs.Entries, error) { +func (imd *Directory) Readdir(ctx context.Context) (fs.Entries, error) { if imd.readdirError != nil { return nil, imd.readdirError } @@ -175,7 +176,7 @@ func (ifr *fileReader) EntryMetadata() (*fs.EntryMetadata, error) { } // Open opens the file for reading, optionally simulating error. -func (imf *File) Open() (fs.Reader, error) { +func (imf *File) Open(ctx context.Context) (fs.Reader, error) { r, err := imf.source() if err != nil { return nil, err @@ -191,7 +192,7 @@ type inmemorySymlink struct { entry } -func (imsl *inmemorySymlink) Readlink() (string, error) { +func (imsl *inmemorySymlink) Readlink(ctx context.Context) (string, error) { panic("not implemented yet") } diff --git a/internal/storagetesting/asserts.go b/internal/storagetesting/asserts.go index 973f48e2c..12c91e120 100644 --- a/internal/storagetesting/asserts.go +++ b/internal/storagetesting/asserts.go @@ -2,6 +2,7 @@ package storagetesting import ( "bytes" + "context" "fmt" "path/filepath" "reflect" @@ -12,8 +13,8 @@ import ( ) // AssertGetBlock asserts that the specified storage block has correct content. -func AssertGetBlock(t *testing.T, s storage.Storage, block string, expected []byte) { - b, err := s.GetBlock(block, 0, -1) +func AssertGetBlock(ctx context.Context, t *testing.T, s storage.Storage, block string, expected []byte) { + b, err := s.GetBlock(ctx, block, 0, -1) if err != nil { t.Errorf(errorPrefix()+"GetBlock(%v) returned error %v, expected data: %v", block, err, expected) return @@ -25,18 +26,19 @@ func AssertGetBlock(t *testing.T, s storage.Storage, block string, expected []by } // AssertGetBlockNotFound asserts that GetBlock() for specified storage block returns ErrBlockNotFound. -func AssertGetBlockNotFound(t *testing.T, s storage.Storage, block string) { - b, err := s.GetBlock(block, 0, -1) +func AssertGetBlockNotFound(ctx context.Context, t *testing.T, s storage.Storage, block string) { + b, err := s.GetBlock(ctx, block, 0, -1) if err != storage.ErrBlockNotFound || b != nil { t.Errorf(errorPrefix()+"GetBlock(%v) returned %v, %v but expected ErrBlockNotFound", block, b, err) } } // AssertListResults asserts that the list results with given prefix return the specified list of names in order. -func AssertListResults(t *testing.T, s storage.Storage, prefix string, expected ...string) { +func AssertListResults(ctx context.Context, t *testing.T, s storage.Storage, prefix string, expected ...string) { var names []string - blocks, cancel := s.ListBlocks(prefix) + ctx, cancel := context.WithCancel(ctx) + blocks := s.ListBlocks(ctx, prefix) defer cancel() for e := range blocks { names = append(names, e.BlockID) diff --git a/internal/storagetesting/map.go b/internal/storagetesting/map.go index 38b96c341..3c2bcd37e 100644 --- a/internal/storagetesting/map.go +++ b/internal/storagetesting/map.go @@ -1,6 +1,9 @@ package storagetesting import ( + "context" + "io" + "io/ioutil" "sort" "strings" "sync" @@ -15,7 +18,7 @@ type mapStorage struct { mutex sync.RWMutex } -func (s *mapStorage) GetBlock(id string, offset, length int64) ([]byte, error) { +func (s *mapStorage) GetBlock(ctx context.Context, id string, offset, length int64) ([]byte, error) { s.mutex.RLock() defer s.mutex.RUnlock() @@ -35,10 +38,15 @@ func (s *mapStorage) GetBlock(id string, offset, length int64) ([]byte, error) { return nil, storage.ErrBlockNotFound } -func (s *mapStorage) PutBlock(id string, data []byte) error { +func (s *mapStorage) PutBlock(ctx context.Context, id string, r io.Reader) error { s.mutex.Lock() defer s.mutex.Unlock() + data, err := ioutil.ReadAll(r) + if err != nil { + return err + } + if _, ok := s.data[id]; ok { return nil } @@ -48,7 +56,7 @@ func (s *mapStorage) PutBlock(id string, data []byte) error { return nil } -func (s *mapStorage) DeleteBlock(id string) error { +func (s *mapStorage) DeleteBlock(ctx context.Context, id string) error { s.mutex.Lock() defer s.mutex.Unlock() @@ -56,9 +64,8 @@ func (s *mapStorage) DeleteBlock(id string) error { return nil } -func (s *mapStorage) ListBlocks(prefix string) (<-chan storage.BlockMetadata, storage.CancelFunc) { +func (s *mapStorage) ListBlocks(ctx context.Context, prefix string) <-chan storage.BlockMetadata { ch := make(chan storage.BlockMetadata) - cancelled := make(chan bool) go func() { defer close(ch) s.mutex.RLock() @@ -76,7 +83,7 @@ func (s *mapStorage) ListBlocks(prefix string) (<-chan storage.BlockMetadata, st for _, k := range keys { v := s.data[k] select { - case <-cancelled: + case <-ctx.Done(): return case ch <- storage.BlockMetadata{ BlockID: k, @@ -86,12 +93,10 @@ func (s *mapStorage) ListBlocks(prefix string) (<-chan storage.BlockMetadata, st } } }() - return ch, func() { - close(cancelled) - } + return ch } -func (s *mapStorage) Close() error { +func (s *mapStorage) Close(ctx context.Context) error { return nil } diff --git a/internal/storagetesting/map_test.go b/internal/storagetesting/map_test.go index b9ac36b23..380a44fb6 100644 --- a/internal/storagetesting/map_test.go +++ b/internal/storagetesting/map_test.go @@ -1,6 +1,9 @@ package storagetesting -import "testing" +import ( + "context" + "testing" +) func TestMapStorage(t *testing.T) { data := map[string][]byte{} @@ -8,5 +11,5 @@ func TestMapStorage(t *testing.T) { if r == nil { t.Errorf("unexpected result: %v", r) } - VerifyStorage(t, r) + VerifyStorage(context.Background(), t, r) } diff --git a/internal/storagetesting/verify.go b/internal/storagetesting/verify.go index 32a37c81f..4faaa0498 100644 --- a/internal/storagetesting/verify.go +++ b/internal/storagetesting/verify.go @@ -2,13 +2,14 @@ package storagetesting import ( "bytes" + "context" "testing" "github.com/kopia/kopia/storage" ) // VerifyStorage verifies the behavior of the specified storage. -func VerifyStorage(t *testing.T, r storage.Storage) { +func VerifyStorage(ctx context.Context, t *testing.T, r storage.Storage) { blocks := []struct { blk string contents []byte @@ -21,20 +22,21 @@ func VerifyStorage(t *testing.T, r storage.Storage) { // First verify that blocks don't exist. for _, b := range blocks { - AssertGetBlockNotFound(t, r, b.blk) + AssertGetBlockNotFound(ctx, t, r, b.blk) } // Now add blocks. for _, b := range blocks { - if err := r.PutBlock(b.blk, b.contents); err != nil { + if err := r.PutBlock(ctx, b.blk, bytes.NewReader(b.contents)); err != nil { t.Errorf("can't put block: %v", err) } - AssertGetBlock(t, r, b.blk, b.contents) + AssertGetBlock(ctx, t, r, b.blk, b.contents) } // List - ch, cancel := r.ListBlocks(string("ab")) + ctx, cancel := context.WithCancel(ctx) + ch := r.ListBlocks(ctx, "ab") defer cancel() e1 := verifyNextBlock(t, blocks[0].blk, ch) e2 := verifyNextBlock(t, blocks[2].blk, ch) diff --git a/internal/webdavmount/webdavmount.go b/internal/webdavmount/webdavmount.go index aa6dd5886..1044227e1 100644 --- a/internal/webdavmount/webdavmount.go +++ b/internal/webdavmount/webdavmount.go @@ -20,6 +20,7 @@ var _ webdav.File = (*webdavFile)(nil) var _ webdav.File = (*webdavDir)(nil) type webdavFile struct { + ctx context.Context entry fs.File mu sync.Mutex @@ -38,7 +39,7 @@ func (f *webdavFile) getReader() (fs.Reader, error) { f.mu.Lock() defer f.mu.Unlock() if f.r == nil { - r, err := f.entry.Open() + r, err := f.entry.Open(f.ctx) if err != nil { return nil, err } @@ -83,12 +84,13 @@ func (f *webdavFile) Close() error { } type webdavDir struct { + ctx context.Context w *webdavFS entry fs.Directory } func (d *webdavDir) Readdir(n int) ([]os.FileInfo, error) { - entries, err := d.entry.Readdir() + entries, err := d.entry.Readdir(d.ctx) if err != nil { return nil, err } @@ -168,7 +170,7 @@ func (w *webdavFS) Rename(ctx context.Context, oldPath, newPath string) error { } func (w *webdavFS) OpenFile(ctx context.Context, path string, flags int, mode os.FileMode) (webdav.File, error) { - f, err := w.findEntry(path) + f, err := w.findEntry(ctx, path) if err != nil { log.Printf("OpenFile(%q) failed with %v", path, err) return nil, err @@ -176,16 +178,16 @@ func (w *webdavFS) OpenFile(ctx context.Context, path string, flags int, mode os switch f := f.(type) { case fs.Directory: - return &webdavDir{w, f}, nil + return &webdavDir{ctx, w, f}, nil case fs.File: - return &webdavFile{entry: f}, nil + return &webdavFile{ctx: ctx, entry: f}, nil } return nil, fmt.Errorf("can't open %q: not implemented", path) } func (w *webdavFS) Stat(ctx context.Context, path string) (os.FileInfo, error) { - e, err := w.findEntry(path) + e, err := w.findEntry(ctx, path) if err != nil { return nil, err } @@ -193,7 +195,7 @@ func (w *webdavFS) Stat(ctx context.Context, path string) (os.FileInfo, error) { return webdavFileInfo{e.Metadata()}, nil } -func (w *webdavFS) findEntry(path string) (fs.Entry, error) { +func (w *webdavFS) findEntry(ctx context.Context, path string) (fs.Entry, error) { parts := removeEmpty(strings.Split(path, "/")) var e fs.Entry = w.dir for i, p := range parts { @@ -202,7 +204,7 @@ func (w *webdavFS) findEntry(path string) (fs.Entry, error) { return nil, fmt.Errorf("%q not found in %q (not a directory)", p, strings.Join(parts[0:i], "/")) } - entries, err := d.Readdir() + entries, err := d.Readdir(ctx) if err != nil { return nil, err } diff --git a/manifest/manifest_manager.go b/manifest/manifest_manager.go index 067719acd..a5fbf4c31 100644 --- a/manifest/manifest_manager.go +++ b/manifest/manifest_manager.go @@ -3,6 +3,7 @@ package manifest import ( "bytes" "compress/gzip" + "context" "crypto/rand" "encoding/hex" "encoding/json" @@ -147,11 +148,11 @@ func matchesLabels(a, b map[string]string) bool { } // Flush persists changes to manifest manager. -func (m *Manager) Flush() error { +func (m *Manager) Flush(ctx context.Context) error { m.mu.Lock() defer m.mu.Unlock() - blockID, err := m.flushPendingEntriesLocked() + blockID, err := m.flushPendingEntriesLocked(ctx) if err != nil { return err } @@ -164,7 +165,7 @@ func (m *Manager) Flush() error { return nil } -func (m *Manager) flushPendingEntriesLocked() (string, error) { +func (m *Manager) flushPendingEntriesLocked(ctx context.Context) (string, error) { if len(m.pendingEntries) == 0 { return "", nil } @@ -186,7 +187,7 @@ func (m *Manager) flushPendingEntriesLocked() (string, error) { return "", fmt.Errorf("unable to close: %v", err) } - blockID, err := m.b.WriteBlock(buf.Bytes(), manifestBlockPrefix) + blockID, err := m.b.WriteBlock(ctx, buf.Bytes(), manifestBlockPrefix) if err != nil { return "", err } @@ -209,8 +210,8 @@ func (m *Manager) Delete(id string) { }) } -func (m *Manager) load() error { - if err := m.Flush(); err != nil { +func (m *Manager) load(ctx context.Context) error { + if err := m.Flush(ctx); err != nil { return err } @@ -227,17 +228,17 @@ func (m *Manager) load() error { log.Printf("found %v manifest blocks", len(blocks)) - if err := m.loadManifestBlocks(blocks); err != nil { + if err := m.loadManifestBlocks(ctx, blocks); err != nil { return fmt.Errorf("unable to load manifest blocks: %v", err) } if len(blocks) > autoCompactionBlockCount { log.Debug().Int("blocks", len(blocks)).Msg("performing automatic compaction") - if err := m.compactLocked(); err != nil { + if err := m.compactLocked(ctx); err != nil { return fmt.Errorf("unable to compact manifest blocks: %v", err) } - if err := m.b.Flush(); err != nil { + if err := m.b.Flush(ctx); err != nil { return fmt.Errorf("unable to flush blocks after auto-compaction: %v", err) } } @@ -245,7 +246,7 @@ func (m *Manager) load() error { return nil } -func (m *Manager) loadManifestBlocks(blocks []block.Info) error { +func (m *Manager) loadManifestBlocks(ctx context.Context, blocks []block.Info) error { t0 := time.Now() log.Debug().Dur("duration_ms", time.Since(t0)).Msgf("finished loading manifest blocks.") @@ -254,7 +255,7 @@ func (m *Manager) loadManifestBlocks(blocks []block.Info) error { m.blockIDs = append(m.blockIDs, b.BlockID) } - manifests, err := m.loadBlocksInParallel(blocks) + manifests, err := m.loadBlocksInParallel(ctx, blocks) if err != nil { return err } @@ -275,7 +276,7 @@ func (m *Manager) loadManifestBlocks(blocks []block.Info) error { return nil } -func (m *Manager) loadBlocksInParallel(blocks []block.Info) ([]manifest, error) { +func (m *Manager) loadBlocksInParallel(ctx context.Context, blocks []block.Info) ([]manifest, error) { errors := make(chan error, len(blocks)) manifests := make(chan manifest, len(blocks)) blockIDs := make(chan string, len(blocks)) @@ -288,7 +289,7 @@ func (m *Manager) loadBlocksInParallel(blocks []block.Info) ([]manifest, error) for blk := range blockIDs { t1 := time.Now() - man, err := m.loadManifestBlock(blk) + man, err := m.loadManifestBlock(ctx, blk) log.Debug().Dur("duration", time.Since(t1)).Str("blk", blk).Int("worker", workerID).Msg("manifest block loaded") if err != nil { errors <- err @@ -323,9 +324,9 @@ func (m *Manager) loadBlocksInParallel(blocks []block.Info) ([]manifest, error) return man, nil } -func (m *Manager) loadManifestBlock(blockID string) (manifest, error) { +func (m *Manager) loadManifestBlock(ctx context.Context, blockID string) (manifest, error) { man := manifest{} - blk, err := m.b.GetBlock(blockID) + blk, err := m.b.GetBlock(ctx, blockID) if err != nil { return man, fmt.Errorf("unable to read block %q: %v", blockID, err) } @@ -349,14 +350,14 @@ func (m *Manager) loadManifestBlock(blockID string) (manifest, error) { } // Compact performs compaction of manifest blocks. -func (m *Manager) Compact() error { +func (m *Manager) Compact(ctx context.Context) error { m.mu.Lock() defer m.mu.Unlock() - return m.compactLocked() + return m.compactLocked(ctx) } -func (m *Manager) compactLocked() error { +func (m *Manager) compactLocked(ctx context.Context) error { log.Printf("compactLocked: pendingEntries=%v blockIDs=%v", len(m.pendingEntries), len(m.blockIDs)) if len(m.blockIDs) == 1 && len(m.pendingEntries) == 0 { @@ -367,7 +368,7 @@ func (m *Manager) compactLocked() error { m.pendingEntries = append(m.pendingEntries, e) } - blockID, err := m.flushPendingEntriesLocked() + blockID, err := m.flushPendingEntriesLocked(ctx) if err != nil { return err } @@ -412,13 +413,13 @@ func copyLabels(m map[string]string) map[string]string { } // NewManager returns new manifest manager for the provided block manager. -func NewManager(b *block.Manager) (*Manager, error) { +func NewManager(ctx context.Context, b *block.Manager) (*Manager, error) { m := &Manager{ b: b, entries: map[string]*manifestEntry{}, } - if err := m.load(); err != nil { + if err := m.load(ctx); err != nil { return nil, err } diff --git a/manifest/manifest_manager_test.go b/manifest/manifest_manager_test.go index b0a58654b..f9eee09cd 100644 --- a/manifest/manifest_manager_test.go +++ b/manifest/manifest_manager_test.go @@ -1,6 +1,7 @@ package manifest import ( + "context" "fmt" "reflect" "sort" @@ -13,9 +14,10 @@ import ( ) func TestManifest(t *testing.T) { + ctx := context.Background() data := map[string][]byte{} keyTime := map[string]time.Time{} - mgr, setupErr := newManagerForTesting(t, data, keyTime) + mgr, setupErr := newManagerForTesting(ctx, t, data, keyTime) if setupErr != nil { t.Fatalf("unable to open block manager: %v", setupErr) } @@ -52,10 +54,10 @@ func TestManifest(t *testing.T) { verifyItem(t, mgr, id2, labels2, item2) verifyItem(t, mgr, id3, labels3, item3) - if err := mgr.Flush(); err != nil { + if err := mgr.Flush(ctx); err != nil { t.Errorf("flush error: %v", err) } - if err := mgr.Flush(); err != nil { + if err := mgr.Flush(ctx); err != nil { t.Errorf("flush error: %v", err) } @@ -68,8 +70,8 @@ func TestManifest(t *testing.T) { verifyItem(t, mgr, id3, labels3, item3) // flush underlying block manager and verify in new manifest manager. - mgr.b.Flush() - mgr2, setupErr := newManagerForTesting(t, data, keyTime) + mgr.b.Flush(ctx) + mgr2, setupErr := newManagerForTesting(ctx, t, data, keyTime) if setupErr != nil { t.Fatalf("can't open block manager: %v", setupErr) } @@ -79,23 +81,23 @@ func TestManifest(t *testing.T) { verifyItem(t, mgr2, id1, labels1, item1) verifyItem(t, mgr2, id2, labels2, item2) verifyItem(t, mgr2, id3, labels3, item3) - if err := mgr2.Flush(); err != nil { + if err := mgr2.Flush(ctx); err != nil { t.Errorf("flush error: %v", err) } // delete from one mgr.Delete(id3) verifyItemNotFound(t, mgr, id3) - mgr.Flush() + mgr.Flush(ctx) verifyItemNotFound(t, mgr, id3) // still found in another verifyItem(t, mgr2, id3, labels3, item3) - if err := mgr2.load(); err != nil { + if err := mgr2.load(ctx); err != nil { t.Errorf("unable to load: %v", err) } - if err := mgr.Compact(); err != nil { + if err := mgr.Compact(ctx); err != nil { t.Errorf("can't compact: %v", err) } @@ -107,9 +109,9 @@ func TestManifest(t *testing.T) { t.Errorf("unexpected number of blocks: %v, want %v", got, want) } - mgr.b.Flush() + mgr.b.Flush(ctx) - mgr3, err := newManagerForTesting(t, data, keyTime) + mgr3, err := newManagerForTesting(ctx, t, data, keyTime) if err != nil { t.Fatalf("can't open manager: %v", err) } @@ -170,10 +172,10 @@ func verifyMatches(t *testing.T, mgr *Manager, labels map[string]string, expecte } } -func newManagerForTesting(t *testing.T, data map[string][]byte, keyTime map[string]time.Time) (*Manager, error) { +func newManagerForTesting(ctx context.Context, t *testing.T, data map[string][]byte, keyTime map[string]time.Time) (*Manager, error) { st := storagetesting.NewMapStorage(data, keyTime) - bm, err := block.NewManager(st, block.FormattingOptions{ + bm, err := block.NewManager(ctx, st, block.FormattingOptions{ BlockFormat: "TESTONLY_MD5", MaxPackSize: 100000, }, block.CachingOptions{}) @@ -181,5 +183,5 @@ func newManagerForTesting(t *testing.T, data map[string][]byte, keyTime map[stri return nil, fmt.Errorf("can't create block manager: %v", err) } - return NewManager(bm) + return NewManager(ctx, bm) } diff --git a/object/object_manager.go b/object/object_manager.go index 1f8ce957e..5fa973b60 100644 --- a/object/object_manager.go +++ b/object/object_manager.go @@ -3,6 +3,7 @@ package object import ( "bufio" "bytes" + "context" "fmt" "io" "sync" @@ -23,10 +24,10 @@ type Reader interface { } type blockManager interface { - BlockInfo(blockID string) (block.Info, error) - GetBlock(blockID string) ([]byte, error) - WriteBlock(data []byte, prefix string) (string, error) - Flush() error + BlockInfo(ctx context.Context, blockID string) (block.Info, error) + GetBlock(ctx context.Context, blockID string) ([]byte, error) + WriteBlock(ctx context.Context, data []byte, prefix string) (string, error) + Flush(ctx context.Context) error } // Manager implements a content-addressable storage on top of blob storage. @@ -45,14 +46,15 @@ type Manager struct { } // Close closes the connection to the underlying blob storage and releases any resources. -func (om *Manager) Close() error { +func (om *Manager) Close(ctx context.Context) error { om.writeBackWG.Wait() - return om.Flush() + return om.Flush(ctx) } // NewWriter creates an ObjectWriter for writing to the repository. -func (om *Manager) NewWriter(opt WriterOptions) Writer { +func (om *Manager) NewWriter(ctx context.Context, opt WriterOptions) Writer { w := &objectWriter{ + ctx: ctx, repo: om, splitter: om.newSplitter(), description: opt.Description, @@ -66,7 +68,7 @@ func (om *Manager) NewWriter(opt WriterOptions) Writer { } // Open creates new ObjectReader for reading given object from a repository. -func (om *Manager) Open(objectID ID) (Reader, error) { +func (om *Manager) Open(ctx context.Context, objectID ID) (Reader, error) { // log.Printf("Repository::Open %v", objectID.String()) // defer log.Printf("finished Repository::Open() %v", objectID.String()) @@ -74,7 +76,7 @@ func (om *Manager) Open(objectID ID) (Reader, error) { om.writeBackWG.Wait() if objectID.Indirect != nil { - rd, err := om.Open(*objectID.Indirect) + rd, err := om.Open(ctx, *objectID.Indirect) if err != nil { return nil, err } @@ -88,23 +90,24 @@ func (om *Manager) Open(objectID ID) (Reader, error) { totalLength := seekTable[len(seekTable)-1].endOffset() return &objectReader{ + ctx: ctx, repo: om, seekTable: seekTable, totalLength: totalLength, }, nil } - return om.newRawReader(objectID) + return om.newRawReader(ctx, objectID) } // VerifyObject ensures that all objects backing ObjectID are present in the repository // and returns the total length of the object and storage blocks of which it is composed. -func (om *Manager) VerifyObject(oid ID) (int64, []string, error) { +func (om *Manager) VerifyObject(ctx context.Context, oid ID) (int64, []string, error) { // Flush any pending writes. om.writeBackWG.Wait() blocks := &blockTracker{} - l, err := om.verifyObjectInternal(oid, blocks) + l, err := om.verifyObjectInternal(ctx, oid, blocks) if err != nil { return 0, nil, err } @@ -112,11 +115,11 @@ func (om *Manager) VerifyObject(oid ID) (int64, []string, error) { return l, blocks.blockIDs(), nil } -func (om *Manager) verifyIndirectObjectInternal(oid ID, blocks *blockTracker) (int64, error) { - if _, err := om.verifyObjectInternal(*oid.Indirect, blocks); err != nil { +func (om *Manager) verifyIndirectObjectInternal(ctx context.Context, oid ID, blocks *blockTracker) (int64, error) { + if _, err := om.verifyObjectInternal(ctx, *oid.Indirect, blocks); err != nil { return 0, fmt.Errorf("unable to read index: %v", err) } - rd, err := om.Open(*oid.Indirect) + rd, err := om.Open(ctx, *oid.Indirect) if err != nil { return 0, err } @@ -128,7 +131,7 @@ func (om *Manager) verifyIndirectObjectInternal(oid ID, blocks *blockTracker) (i } for i, m := range seekTable { - l, err := om.verifyObjectInternal(m.Object, blocks) + l, err := om.verifyObjectInternal(ctx, m.Object, blocks) if err != nil { return 0, err } @@ -142,19 +145,19 @@ func (om *Manager) verifyIndirectObjectInternal(oid ID, blocks *blockTracker) (i return totalLength, nil } -func (om *Manager) verifyObjectInternal(oid ID, blocks *blockTracker) (int64, error) { +func (om *Manager) verifyObjectInternal(ctx context.Context, oid ID, blocks *blockTracker) (int64, error) { if oid.Indirect != nil { - return om.verifyIndirectObjectInternal(oid, blocks) + return om.verifyIndirectObjectInternal(ctx, oid, blocks) } - p, err := om.blockMgr.BlockInfo(oid.StorageBlock) + p, err := om.blockMgr.BlockInfo(ctx, oid.StorageBlock) if err != nil { return 0, err } blocks.addBlock(oid.StorageBlock) if p.PackBlockID != "" { - l, err := om.verifyObjectInternal(ID{StorageBlock: p.PackBlockID}, blocks) + l, err := om.verifyObjectInternal(ctx, ID{StorageBlock: p.PackBlockID}, blocks) if err != nil { return 0, err } @@ -171,7 +174,7 @@ func (om *Manager) verifyObjectInternal(oid ID, blocks *blockTracker) (int64, er // Flush closes any pending pack files. Once this method returns, ObjectIDs returned by ObjectManager are // ok to be used. -func (om *Manager) Flush() error { +func (om *Manager) Flush(ctx context.Context) error { om.writeBackWG.Wait() return nil } @@ -195,7 +198,7 @@ type ManagerOptions struct { } // NewObjectManager creates an ObjectManager with the specified block manager and format. -func NewObjectManager(bm blockManager, f config.RepositoryObjectFormat, opts ManagerOptions) (*Manager, error) { +func NewObjectManager(ctx context.Context, bm blockManager, f config.RepositoryObjectFormat, opts ManagerOptions) (*Manager, error) { if err := validateFormat(&f); err != nil { return nil, err } @@ -260,8 +263,8 @@ func (om *Manager) flattenListChunk(rawReader io.Reader) ([]indirectObjectEntry, return seekTable, nil } -func (om *Manager) newRawReader(objectID ID) (Reader, error) { - payload, err := om.blockMgr.GetBlock(objectID.StorageBlock) +func (om *Manager) newRawReader(ctx context.Context, objectID ID) (Reader, error) { + payload, err := om.blockMgr.GetBlock(ctx, objectID.StorageBlock) if err != nil { return nil, err } diff --git a/object/object_manager_test.go b/object/object_manager_test.go index 1af1de421..cfd91547e 100644 --- a/object/object_manager_test.go +++ b/object/object_manager_test.go @@ -2,6 +2,7 @@ package object import ( "bytes" + "context" "crypto/md5" cryptorand "crypto/rand" "encoding/hex" @@ -26,7 +27,7 @@ type fakeBlockManager struct { data map[string][]byte } -func (f *fakeBlockManager) GetBlock(blockID string) ([]byte, error) { +func (f *fakeBlockManager) GetBlock(ctx context.Context, blockID string) ([]byte, error) { f.mu.Lock() defer f.mu.Unlock() @@ -37,7 +38,7 @@ func (f *fakeBlockManager) GetBlock(blockID string) ([]byte, error) { return nil, storage.ErrBlockNotFound } -func (f *fakeBlockManager) WriteBlock(data []byte, prefix string) (string, error) { +func (f *fakeBlockManager) WriteBlock(ctx context.Context, data []byte, prefix string) (string, error) { h := md5.New() h.Write(data) blockID := prefix + hex.EncodeToString(h.Sum(nil)) @@ -49,7 +50,7 @@ func (f *fakeBlockManager) WriteBlock(data []byte, prefix string) (string, error return blockID, nil } -func (f *fakeBlockManager) BlockInfo(blockID string) (block.Info, error) { +func (f *fakeBlockManager) BlockInfo(ctx context.Context, blockID string) (block.Info, error) { f.mu.Lock() defer f.mu.Unlock() @@ -60,7 +61,7 @@ func (f *fakeBlockManager) BlockInfo(blockID string) (block.Info, error) { return block.Info{}, storage.ErrBlockNotFound } -func (f *fakeBlockManager) Flush() error { +func (f *fakeBlockManager) Flush(ctx context.Context) error { return nil } @@ -69,7 +70,7 @@ func setupTest(t *testing.T) (map[string][]byte, *Manager) { } func setupTestWithData(t *testing.T, data map[string][]byte, opts ManagerOptions) (map[string][]byte, *Manager) { - r, err := NewObjectManager(&fakeBlockManager{data: data}, config.RepositoryObjectFormat{ + r, err := NewObjectManager(context.Background(), &fakeBlockManager{data: data}, config.RepositoryObjectFormat{ FormattingOptions: block.FormattingOptions{ Version: 1, }, @@ -84,6 +85,7 @@ func setupTestWithData(t *testing.T, data map[string][]byte, opts ManagerOptions } func TestWriters(t *testing.T) { + ctx := context.Background() cases := []struct { data []byte objectID ID @@ -98,7 +100,7 @@ func TestWriters(t *testing.T) { for _, c := range cases { data, om := setupTest(t) - writer := om.NewWriter(WriterOptions{}) + writer := om.NewWriter(ctx, WriterOptions{}) writer.Write(c.data) @@ -132,10 +134,11 @@ func objectIDsEqual(o1 ID, o2 ID) bool { } func TestWriterCompleteChunkInTwoWrites(t *testing.T) { + ctx := context.Background() _, om := setupTest(t) bytes := make([]byte, 100) - writer := om.NewWriter(WriterOptions{}) + writer := om.NewWriter(ctx, WriterOptions{}) writer.Write(bytes[0:50]) writer.Write(bytes[0:50]) result, err := writer.Result() @@ -144,12 +147,12 @@ func TestWriterCompleteChunkInTwoWrites(t *testing.T) { } } -func verifyIndirectBlock(t *testing.T, r *Manager, oid ID) { +func verifyIndirectBlock(ctx context.Context, t *testing.T, r *Manager, oid ID) { for oid.Indirect != nil { direct := *oid.Indirect oid = direct - rd, err := r.Open(direct) + rd, err := r.Open(ctx, direct) if err != nil { t.Errorf("unable to open %v: %v", oid.String(), err) return @@ -175,6 +178,7 @@ func verifyIndirectBlock(t *testing.T, r *Manager, oid ID) { } func TestIndirection(t *testing.T) { + ctx := context.Background() cases := []struct { dataLength int expectedBlockCount int @@ -194,7 +198,7 @@ func TestIndirection(t *testing.T) { contentBytes := make([]byte, c.dataLength) - writer := om.NewWriter(WriterOptions{}) + writer := om.NewWriter(ctx, WriterOptions{}) writer.Write(contentBytes) result, err := writer.Result() if err != nil { @@ -209,9 +213,9 @@ func TestIndirection(t *testing.T) { t.Errorf("unexpected block count for %v: %v, expected %v", c.dataLength, got, want) } - om.Flush() + om.Flush(ctx) - l, b, err := om.VerifyObject(result) + l, b, err := om.VerifyObject(ctx, result) if err != nil { t.Errorf("error verifying %q: %v", result, err) } @@ -224,7 +228,7 @@ func TestIndirection(t *testing.T) { t.Errorf("invalid block count for %v, got %v, wanted %v", result, got, want) } - verifyIndirectBlock(t, om, result) + verifyIndirectBlock(ctx, t, om, result) } } @@ -237,11 +241,12 @@ func indirectionLevel(oid ID) int { } func TestHMAC(t *testing.T) { + ctx := context.Background() content := bytes.Repeat([]byte{0xcd}, 50) _, om := setupTest(t) - w := om.NewWriter(WriterOptions{}) + w := om.NewWriter(ctx, WriterOptions{}) w.Write(content) result, err := w.Result() if result.String() != "D999732b72ceff665b3f7608411db66a4" { @@ -250,6 +255,7 @@ func TestHMAC(t *testing.T) { } func TestReader(t *testing.T) { + ctx := context.Background() data, om := setupTest(t) storedPayload := []byte("foo\nbar") @@ -269,7 +275,7 @@ func TestReader(t *testing.T) { continue } - reader, err := om.Open(objectID) + reader, err := om.Open(ctx, objectID) if err != nil { t.Errorf("cannot create reader for %v: %v", objectID, err) continue @@ -288,19 +294,21 @@ func TestReader(t *testing.T) { } func TestReaderStoredBlockNotFound(t *testing.T) { + ctx := context.Background() _, om := setupTest(t) objectID, err := ParseID("Dno-such-block") if err != nil { t.Errorf("cannot parse object ID: %v", err) } - reader, err := om.Open(objectID) + reader, err := om.Open(ctx, objectID) if err != storage.ErrBlockNotFound || reader != nil { t.Errorf("unexpected result: reader: %v err: %v", reader, err) } } func TestEndToEndReadAndSeek(t *testing.T) { + ctx := context.Background() _, om := setupTest(t) for _, size := range []int{1, 199, 200, 201, 9999, 512434} { @@ -308,7 +316,7 @@ func TestEndToEndReadAndSeek(t *testing.T) { randomData := make([]byte, size) cryptorand.Read(randomData) - writer := om.NewWriter(WriterOptions{}) + writer := om.NewWriter(ctx, WriterOptions{}) writer.Write(randomData) objectID, err := writer.Result() writer.Close() @@ -317,13 +325,13 @@ func TestEndToEndReadAndSeek(t *testing.T) { continue } - verify(t, om, objectID, randomData, fmt.Sprintf("%v %v", objectID, size)) + verify(ctx, t, om, objectID, randomData, fmt.Sprintf("%v %v", objectID, size)) } } -func verify(t *testing.T, om *Manager, objectID ID, expectedData []byte, testCaseID string) { +func verify(ctx context.Context, t *testing.T, om *Manager, objectID ID, expectedData []byte, testCaseID string) { t.Helper() - reader, err := om.Open(objectID) + reader, err := om.Open(ctx, objectID) if err != nil { t.Errorf("cannot get reader for %v (%v): %v %v", testCaseID, objectID, err, string(debug.Stack())) return diff --git a/object/object_reader.go b/object/object_reader.go index c0fbee693..5097d5fff 100644 --- a/object/object_reader.go +++ b/object/object_reader.go @@ -1,6 +1,7 @@ package object import ( + "context" "fmt" "io" ) @@ -10,6 +11,7 @@ func (i *indirectObjectEntry) endOffset() int64 { } type objectReader struct { + ctx context.Context repo *Manager seekTable []indirectObjectEntry @@ -65,7 +67,7 @@ func (r *objectReader) Read(buffer []byte) (int, error) { func (r *objectReader) openCurrentChunk() error { st := r.seekTable[r.currentChunkIndex] - blockData, err := r.repo.Open(st.Object) + blockData, err := r.repo.Open(r.ctx, st.Object) if err != nil { return err } diff --git a/object/object_writer.go b/object/object_writer.go index fa5b66eae..20b2f9d57 100644 --- a/object/object_writer.go +++ b/object/object_writer.go @@ -2,6 +2,7 @@ package object import ( "bytes" + "context" "fmt" "io" "sync" @@ -44,6 +45,7 @@ func (t *blockTracker) blockIDs() []string { } type objectWriter struct { + ctx context.Context repo *Manager buffer bytes.Buffer @@ -95,7 +97,7 @@ func (w *objectWriter) flushBuffer() error { w.buffer.Reset() do := func() { - blockID, err := w.repo.blockMgr.WriteBlock(b2.Bytes(), "") + blockID, err := w.repo.blockMgr.WriteBlock(w.ctx, b2.Bytes(), "") w.repo.trace("OBJECT_WRITER(%q) stored %v (%v bytes)", w.description, blockID, length) if err != nil { w.err.add(fmt.Errorf("error when flushing chunk %d of %s: %v", chunkID, w.description, err)) @@ -143,6 +145,7 @@ func (w *objectWriter) Result() (ID, error) { } iw := &objectWriter{ + ctx: w.ctx, repo: w.repo, description: "LIST(" + w.description + ")", splitter: w.repo.newSplitter(), diff --git a/repo/connect.go b/repo/connect.go index be48e7079..d9fc84000 100644 --- a/repo/connect.go +++ b/repo/connect.go @@ -31,7 +31,7 @@ type ConnectOptions struct { // Connect connects to the repository in the specified storage and persists the configuration and credentials in the file provided. func Connect(ctx context.Context, configFile string, st storage.Storage, creds auth.Credentials, opt ConnectOptions) error { - formatBytes, err := st.GetBlock(FormatBlockID, 0, -1) + formatBytes, err := st.GetBlock(ctx, FormatBlockID, 0, -1) if err != nil { return fmt.Errorf("unable to read format block: %v", err) } @@ -85,7 +85,7 @@ func Connect(ctx context.Context, configFile string, st storage.Storage, creds a return err } - return r.Close() + return r.Close(ctx) } func setupCaching(lc *config.LocalConfig, opt block.CachingOptions, uniqueID []byte) error { diff --git a/repo/format_block.go b/repo/format_block.go index 427a52192..18ce3b6f1 100644 --- a/repo/format_block.go +++ b/repo/format_block.go @@ -2,6 +2,7 @@ package repo import ( "bytes" + "context" "crypto/aes" "crypto/cipher" "crypto/rand" @@ -50,7 +51,7 @@ func parseFormatBlock(b []byte) (*formatBlock, error) { return f, nil } -func writeFormatBlock(st storage.Storage, f *formatBlock) error { +func writeFormatBlock(ctx context.Context, st storage.Storage, f *formatBlock) error { var buf bytes.Buffer e := json.NewEncoder(&buf) e.SetIndent("", " ") @@ -58,7 +59,7 @@ func writeFormatBlock(st storage.Storage, f *formatBlock) error { return fmt.Errorf("unable to marshal format block: %v", err) } - if err := st.PutBlock(FormatBlockID, buf.Bytes()); err != nil { + if err := st.PutBlock(ctx, FormatBlockID, &buf); err != nil { return fmt.Errorf("unable to write format block: %v", err) } diff --git a/repo/initialize.go b/repo/initialize.go index 70dc73d3d..9a3332b40 100644 --- a/repo/initialize.go +++ b/repo/initialize.go @@ -1,6 +1,7 @@ package repo import ( + "context" "crypto/rand" "io" @@ -44,7 +45,7 @@ type NewRepositoryOptions struct { } // Initialize creates initial repository data structures in the specified storage with given credentials. -func Initialize(st storage.Storage, opt *NewRepositoryOptions, creds auth.Credentials) error { +func Initialize(ctx context.Context, st storage.Storage, opt *NewRepositoryOptions, creds auth.Credentials) error { if opt == nil { opt = &NewRepositoryOptions{} } @@ -60,7 +61,7 @@ func Initialize(st storage.Storage, opt *NewRepositoryOptions, creds auth.Creden return err } - if err := writeFormatBlock(st, format); err != nil { + if err := writeFormatBlock(ctx, st, format); err != nil { return err } diff --git a/repo/open.go b/repo/open.go index e0d9af7b2..f6c1c10c6 100644 --- a/repo/open.go +++ b/repo/open.go @@ -80,7 +80,7 @@ func Open(ctx context.Context, configFile string, options *Options) (rep *Reposi r, err := connect(ctx, st, creds, options, caching) if err != nil { - st.Close() //nolint:errcheck + st.Close(ctx) //nolint:errcheck return nil, err } @@ -121,7 +121,7 @@ func SetCachingConfig(ctx context.Context, configFile string, opt block.CachingO return fmt.Errorf("cannot open storage: %v", err) } - f, err := readAndCacheFormatBlock(st, "") + f, err := readAndCacheFormatBlock(ctx, st, "") if err != nil { return fmt.Errorf("can't read format block: %v", err) } @@ -142,7 +142,7 @@ func SetCachingConfig(ctx context.Context, configFile string, opt block.CachingO return nil } -func readAndCacheFormatBlock(st storage.Storage, cacheDirectory string) (*formatBlock, error) { +func readAndCacheFormatBlock(ctx context.Context, st storage.Storage, cacheDirectory string) (*formatBlock, error) { cachedFile := filepath.Join(cacheDirectory, "kopia.repository") if cacheDirectory != "" { b, err := ioutil.ReadFile(cachedFile) @@ -152,7 +152,7 @@ func readAndCacheFormatBlock(st storage.Storage, cacheDirectory string) (*format } } - b, err := st.GetBlock(FormatBlockID, 0, -1) + b, err := st.GetBlock(ctx, FormatBlockID, 0, -1) if err != nil { return nil, err } @@ -182,7 +182,7 @@ func connect(ctx context.Context, st storage.Storage, creds auth.Credentials, op log.Debug().Msg("reading encrypted format block") // Read cache block, potentially from cache. - f, err := readAndCacheFormatBlock(st, caching.CacheDirectory) + f, err := readAndCacheFormatBlock(ctx, st, caching.CacheDirectory) if err != nil { return nil, fmt.Errorf("unable to read format block: %v", err) } @@ -205,19 +205,19 @@ func connect(ctx context.Context, st storage.Storage, creds auth.Credentials, op caching.HMACSecret = km.DeriveKey([]byte("local-cache-integrity"), 16) log.Debug().Msg("initializing block manager") - bm, err := block.NewManager(st, fo, caching) + bm, err := block.NewManager(ctx, st, fo, caching) if err != nil { return nil, fmt.Errorf("unable to open block manager: %v", err) } log.Debug().Msg("initializing object manager") - om, err := object.NewObjectManager(bm, *repoConfig, options.ObjectManagerOptions) + om, err := object.NewObjectManager(ctx, bm, *repoConfig, options.ObjectManagerOptions) if err != nil { return nil, fmt.Errorf("unable to open object manager: %v", err) } log.Debug().Msg("initializing manifest manager") - manifests, err := manifest.NewManager(bm) + manifests, err := manifest.NewManager(ctx, bm) if err != nil { return nil, fmt.Errorf("unable to open manifests: %v", err) } diff --git a/repo/repository.go b/repo/repository.go index d5b50795c..c1609fbbc 100644 --- a/repo/repository.go +++ b/repo/repository.go @@ -1,6 +1,8 @@ package repo import ( + "context" + "github.com/kopia/kopia/auth" "github.com/kopia/kopia/block" "github.com/kopia/kopia/manifest" @@ -22,30 +24,30 @@ type Repository struct { } // Close closes the repository and releases all resources. -func (r *Repository) Close() error { - if err := r.Manifests.Flush(); err != nil { +func (r *Repository) Close(ctx context.Context) error { + if err := r.Manifests.Flush(ctx); err != nil { return err } - if err := r.Objects.Close(); err != nil { + if err := r.Objects.Close(ctx); err != nil { return err } - if err := r.Blocks.Flush(); err != nil { + if err := r.Blocks.Flush(ctx); err != nil { return err } - if err := r.Storage.Close(); err != nil { + if err := r.Storage.Close(ctx); err != nil { return err } return nil } // Flush waits for all in-flight writes to complete. -func (r *Repository) Flush() error { - if err := r.Manifests.Flush(); err != nil { +func (r *Repository) Flush(ctx context.Context) error { + if err := r.Manifests.Flush(ctx); err != nil { return err } - if err := r.Objects.Flush(); err != nil { + if err := r.Objects.Flush(ctx); err != nil { return err } - return r.Blocks.Flush() + return r.Blocks.Flush(ctx) } diff --git a/repo/repository_test.go b/repo/repository_test.go index d21113d16..681f79651 100644 --- a/repo/repository_test.go +++ b/repo/repository_test.go @@ -42,9 +42,9 @@ func setupTestWithData(t *testing.T, data map[string][]byte, keyTime map[string] for _, m := range mods { m(opt) } - Initialize(st, opt, creds) ctx := context.Background() + Initialize(ctx, st, opt, creds) r, err := connect(ctx, st, creds, &Options{ //TraceStorage: log.Printf, @@ -68,10 +68,12 @@ func TestWriters(t *testing.T) { {make([]byte, 100), object.ID{StorageBlock: "6d0bb00954ceb7fbee436bb55a8397a9"}}, // 100 zero bytes } + ctx := context.Background() + for _, c := range cases { data, _, repo := setupTest(t) - writer := repo.Objects.NewWriter(object.WriterOptions{}) + writer := repo.Objects.NewWriter(ctx, object.WriterOptions{}) writer.Write(c.data) @@ -81,13 +83,13 @@ func TestWriters(t *testing.T) { continue } - repo.Objects.Flush() + repo.Objects.Flush(ctx) if !objectIDsEqual(result, c.objectID) { t.Errorf("incorrect result for %v, expected: %v got: %v", c.data, c.objectID.String(), result.String()) } - repo.Blocks.Flush() + repo.Blocks.Flush(ctx) if got, want := len(data), 2; got != want { // 1 format block + 1 pack index block (including inline data blocks) @@ -118,10 +120,11 @@ func objectIDsEqual(o1 object.ID, o2 object.ID) bool { } func TestWriterCompleteChunkInTwoWrites(t *testing.T) { + ctx := context.Background() _, _, repo := setupTest(t) bytes := make([]byte, 100) - writer := repo.Objects.NewWriter(object.WriterOptions{}) + writer := repo.Objects.NewWriter(ctx, object.WriterOptions{}) writer.Write(bytes[0:50]) writer.Write(bytes[0:50]) result, err := writer.Result() @@ -133,25 +136,26 @@ func TestWriterCompleteChunkInTwoWrites(t *testing.T) { func TestPackingSimple(t *testing.T) { data, keyTime, repo := setupTest(t, func(n *NewRepositoryOptions) { }) + ctx := context.Background() content1 := "hello, how do you do?" content2 := "hi, how are you?" content3 := "thank you!" - oid1a := writeObject(t, repo, []byte(content1), "packed-object-1a") - oid1b := writeObject(t, repo, []byte(content1), "packed-object-1b") - oid2a := writeObject(t, repo, []byte(content2), "packed-object-2a") - oid2b := writeObject(t, repo, []byte(content2), "packed-object-2b") + oid1a := writeObject(ctx, t, repo, []byte(content1), "packed-object-1a") + oid1b := writeObject(ctx, t, repo, []byte(content1), "packed-object-1b") + oid2a := writeObject(ctx, t, repo, []byte(content2), "packed-object-2a") + oid2b := writeObject(ctx, t, repo, []byte(content2), "packed-object-2b") - oid3a := writeObject(t, repo, []byte(content3), "packed-object-3a") - oid3b := writeObject(t, repo, []byte(content3), "packed-object-3b") - verify(t, repo, oid1a, []byte(content1), "packed-object-1") - verify(t, repo, oid2a, []byte(content2), "packed-object-2") - oid2c := writeObject(t, repo, []byte(content2), "packed-object-2c") - oid1c := writeObject(t, repo, []byte(content1), "packed-object-1c") + oid3a := writeObject(ctx, t, repo, []byte(content3), "packed-object-3a") + oid3b := writeObject(ctx, t, repo, []byte(content3), "packed-object-3b") + verify(ctx, t, repo, oid1a, []byte(content1), "packed-object-1") + verify(ctx, t, repo, oid2a, []byte(content2), "packed-object-2") + oid2c := writeObject(ctx, t, repo, []byte(content2), "packed-object-2c") + oid1c := writeObject(ctx, t, repo, []byte(content1), "packed-object-1c") - repo.Objects.Flush() - repo.Blocks.Flush() + repo.Objects.Flush(ctx) + repo.Blocks.Flush(ctx) if got, want := oid1a.String(), oid1b.String(); got != want { t.Errorf("oid1a(%q) != oid1b(%q)", got, want) @@ -173,34 +177,34 @@ func TestPackingSimple(t *testing.T) { if got, want := len(data), 2; got != want { t.Errorf("got unexpected repository contents %v items, wanted %v", got, want) } - repo.Close() + repo.Close(ctx) data, _, repo = setupTestWithData(t, data, keyTime, func(n *NewRepositoryOptions) { }) - verify(t, repo, oid1a, []byte(content1), "packed-object-1") - verify(t, repo, oid2a, []byte(content2), "packed-object-2") - verify(t, repo, oid3a, []byte(content3), "packed-object-3") + verify(ctx, t, repo, oid1a, []byte(content1), "packed-object-1") + verify(ctx, t, repo, oid2a, []byte(content2), "packed-object-2") + verify(ctx, t, repo, oid3a, []byte(content3), "packed-object-3") - if err := repo.Blocks.CompactIndexes(); err != nil { + if err := repo.Blocks.CompactIndexes(ctx); err != nil { t.Errorf("optimize error: %v", err) } data, _, repo = setupTestWithData(t, data, keyTime, func(n *NewRepositoryOptions) { }) - verify(t, repo, oid1a, []byte(content1), "packed-object-1") - verify(t, repo, oid2a, []byte(content2), "packed-object-2") - verify(t, repo, oid3a, []byte(content3), "packed-object-3") + verify(ctx, t, repo, oid1a, []byte(content1), "packed-object-1") + verify(ctx, t, repo, oid2a, []byte(content2), "packed-object-2") + verify(ctx, t, repo, oid3a, []byte(content3), "packed-object-3") - if err := repo.Blocks.CompactIndexes(); err != nil { + if err := repo.Blocks.CompactIndexes(ctx); err != nil { t.Errorf("optimize error: %v", err) } _, _, repo = setupTestWithData(t, data, keyTime, func(n *NewRepositoryOptions) { }) - verify(t, repo, oid1a, []byte(content1), "packed-object-1") - verify(t, repo, oid2a, []byte(content2), "packed-object-2") - verify(t, repo, oid3a, []byte(content3), "packed-object-3") + verify(ctx, t, repo, oid1a, []byte(content1), "packed-object-1") + verify(ctx, t, repo, oid2a, []byte(content2), "packed-object-2") + verify(ctx, t, repo, oid3a, []byte(content3), "packed-object-3") } func TestHMAC(t *testing.T) { @@ -208,7 +212,9 @@ func TestHMAC(t *testing.T) { _, _, repo := setupTest(t) - w := repo.Objects.NewWriter(object.WriterOptions{}) + ctx := context.Background() + + w := repo.Objects.NewWriter(ctx, object.WriterOptions{}) w.Write(content) result, err := w.Result() if result.String() != "D999732b72ceff665b3f7608411db66a4" { @@ -217,6 +223,7 @@ func TestHMAC(t *testing.T) { } func TestMalformedStoredData(t *testing.T) { data, _, repo := setupTest(t) + ctx := context.Background() cases := [][]byte{ []byte("foo\nba"), @@ -231,7 +238,7 @@ func TestMalformedStoredData(t *testing.T) { continue } - reader, err := repo.Objects.Open(objectID) + reader, err := repo.Objects.Open(ctx, objectID) if err == nil || reader != nil { t.Errorf("expected error for %x", c) } @@ -240,12 +247,13 @@ func TestMalformedStoredData(t *testing.T) { func TestReaderStoredBlockNotFound(t *testing.T) { _, _, repo := setupTest(t) + ctx := context.Background() objectID, err := object.ParseID("Dno-such-block") if err != nil { t.Errorf("cannot parse object ID: %v", err) } - reader, err := repo.Objects.Open(objectID) + reader, err := repo.Objects.Open(ctx, objectID) if err != storage.ErrBlockNotFound || reader != nil { t.Errorf("unexpected result: reader: %v err: %v", reader, err) } @@ -253,13 +261,14 @@ func TestReaderStoredBlockNotFound(t *testing.T) { func TestEndToEndReadAndSeek(t *testing.T) { _, _, repo := setupTest(t) + ctx := context.Background() for _, size := range []int{1, 199, 200, 201, 9999, 512434} { // Create some random data sample of the specified size. randomData := make([]byte, size) cryptorand.Read(randomData) - writer := repo.Objects.NewWriter(object.WriterOptions{}) + writer := repo.Objects.NewWriter(ctx, object.WriterOptions{}) writer.Write(randomData) objectID, err := writer.Result() writer.Close() @@ -268,12 +277,12 @@ func TestEndToEndReadAndSeek(t *testing.T) { continue } - verify(t, repo, objectID, randomData, fmt.Sprintf("%v %v", objectID, size)) + verify(ctx, t, repo, objectID, randomData, fmt.Sprintf("%v %v", objectID, size)) } } -func writeObject(t *testing.T, repo *Repository, data []byte, testCaseID string) object.ID { - w := repo.Objects.NewWriter(object.WriterOptions{}) +func writeObject(ctx context.Context, t *testing.T, repo *Repository, data []byte, testCaseID string) object.ID { + w := repo.Objects.NewWriter(ctx, object.WriterOptions{}) if _, err := w.Write(data); err != nil { t.Fatalf("can't write object %q - write failed: %v", testCaseID, err) @@ -286,9 +295,9 @@ func writeObject(t *testing.T, repo *Repository, data []byte, testCaseID string) return oid } -func verify(t *testing.T, repo *Repository, objectID object.ID, expectedData []byte, testCaseID string) { +func verify(ctx context.Context, t *testing.T, repo *Repository, objectID object.ID, expectedData []byte, testCaseID string) { t.Helper() - reader, err := repo.Objects.Open(objectID) + reader, err := repo.Objects.Open(ctx, objectID) if err != nil { t.Errorf("cannot get reader for %v (%v): %v %v", testCaseID, objectID, err, string(debug.Stack())) return @@ -319,6 +328,7 @@ func verify(t *testing.T, repo *Repository, objectID object.ID, expectedData []b } func TestFormats(t *testing.T) { + ctx := context.Background() makeFormat := func(objectFormat string) func(*NewRepositoryOptions) { return func(n *NewRepositoryOptions) { n.BlockFormat = objectFormat @@ -368,7 +378,7 @@ func TestFormats(t *testing.T) { for k, v := range c.oids { bytesToWrite := []byte(k) - w := repo.Objects.NewWriter(object.WriterOptions{}) + w := repo.Objects.NewWriter(ctx, object.WriterOptions{}) w.Write(bytesToWrite) oid, err := w.Result() if err != nil { @@ -378,7 +388,7 @@ func TestFormats(t *testing.T) { t.Errorf("invalid oid for #%v\ngot:\n%#v\nexpected:\n%#v", caseIndex, oid.String(), v.String()) } - rc, err := repo.Objects.Open(oid) + rc, err := repo.Objects.Open(ctx, oid) if err != nil { t.Errorf("open failed: %v", err) continue diff --git a/snapshot/all_sources.go b/snapshot/all_sources.go index 140e2cbc6..8aabc7bc5 100644 --- a/snapshot/all_sources.go +++ b/snapshot/all_sources.go @@ -1,6 +1,7 @@ package snapshot import ( + "context" "fmt" "time" @@ -26,7 +27,7 @@ func (s *repositoryAllSources) Metadata() *fs.EntryMetadata { } } -func (s *repositoryAllSources) Readdir() (fs.Entries, error) { +func (s *repositoryAllSources) Readdir(ctx context.Context) (fs.Entries, error) { srcs := s.snapshotManager.ListSources() users := map[string]bool{} diff --git a/snapshot/repofs.go b/snapshot/repofs.go index f0ddf9e2b..e1be4b6f8 100644 --- a/snapshot/repofs.go +++ b/snapshot/repofs.go @@ -1,6 +1,7 @@ package snapshot import ( + "context" "fmt" "io/ioutil" @@ -40,8 +41,8 @@ type repositorySymlink struct { repositoryEntry } -func (rd *repositoryDirectory) Readdir() (fs.Entries, error) { - r, err := rd.repo.Objects.Open(rd.metadata.ObjectID) +func (rd *repositoryDirectory) Readdir(ctx context.Context) (fs.Entries, error) { + r, err := rd.repo.Objects.Open(ctx, rd.metadata.ObjectID) if err != nil { return nil, err } @@ -60,8 +61,8 @@ func (rd *repositoryDirectory) Readdir() (fs.Entries, error) { return entries, nil } -func (rf *repositoryFile) Open() (fs.Reader, error) { - r, err := rf.repo.Objects.Open(rf.metadata.ObjectID) +func (rf *repositoryFile) Open(ctx context.Context) (fs.Reader, error) { + r, err := rf.repo.Objects.Open(ctx, rf.metadata.ObjectID) if err != nil { return nil, err } @@ -69,8 +70,8 @@ func (rf *repositoryFile) Open() (fs.Reader, error) { return withMetadata(r, &rf.metadata.EntryMetadata), nil } -func (rsl *repositorySymlink) Readlink() (string, error) { - r, err := rsl.repo.Objects.Open(rsl.metadata.ObjectID) +func (rsl *repositorySymlink) Readlink(ctx context.Context) (string, error) { + r, err := rsl.repo.Objects.Open(ctx, rsl.metadata.ObjectID) if err != nil { return "", err } diff --git a/snapshot/source_directories.go b/snapshot/source_directories.go index 4671cecac..db04706e1 100644 --- a/snapshot/source_directories.go +++ b/snapshot/source_directories.go @@ -1,6 +1,7 @@ package snapshot import ( + "context" "time" "github.com/kopia/kopia/fs" @@ -27,7 +28,7 @@ func (s *sourceDirectories) Metadata() *fs.EntryMetadata { } } -func (s *sourceDirectories) Readdir() (fs.Entries, error) { +func (s *sourceDirectories) Readdir(ctx context.Context) (fs.Entries, error) { sources := s.snapshotManager.ListSources() var result fs.Entries diff --git a/snapshot/source_snapshots.go b/snapshot/source_snapshots.go index 02495a236..63d76e287 100644 --- a/snapshot/source_snapshots.go +++ b/snapshot/source_snapshots.go @@ -1,6 +1,7 @@ package snapshot import ( + "context" "fmt" "strings" @@ -33,7 +34,7 @@ func safeName(path string) string { return strings.Replace(path, "/", "_", -1) } -func (s *sourceSnapshots) Readdir() (fs.Entries, error) { +func (s *sourceSnapshots) Readdir(ctx context.Context) (fs.Entries, error) { manifests, err := s.snapshotManager.ListSnapshots(s.src) if err != nil { return nil, err diff --git a/snapshot/upload.go b/snapshot/upload.go index a7afccdb6..2164779fb 100644 --- a/snapshot/upload.go +++ b/snapshot/upload.go @@ -2,6 +2,7 @@ package snapshot import ( "bytes" + "context" "encoding/binary" "errors" "fmt" @@ -95,14 +96,14 @@ func (u *Uploader) cancelReason() string { return "" } -func (u *Uploader) uploadFileInternal(f fs.File, relativePath string) entryResult { - file, err := f.Open() +func (u *Uploader) uploadFileInternal(ctx context.Context, f fs.File, relativePath string) entryResult { + file, err := f.Open(ctx) if err != nil { return entryResult{err: fmt.Errorf("unable to open file: %v", err)} } defer file.Close() //nolint:errcheck - writer := u.repo.Objects.NewWriter(object.WriterOptions{ + writer := u.repo.Objects.NewWriter(ctx, object.WriterOptions{ Description: "FILE:" + f.Metadata().Name, }) defer writer.Close() //nolint:errcheck @@ -128,13 +129,13 @@ func (u *Uploader) uploadFileInternal(f fs.File, relativePath string) entryResul return entryResult{de: de, hash: metadataHash(&de.EntryMetadata)} } -func (u *Uploader) uploadSymlinkInternal(f fs.Symlink, relativePath string) entryResult { - target, err := f.Readlink() +func (u *Uploader) uploadSymlinkInternal(ctx context.Context, f fs.Symlink, relativePath string) entryResult { + target, err := f.Readlink(ctx) if err != nil { return entryResult{err: fmt.Errorf("unable to read symlink: %v", err)} } - writer := u.repo.Objects.NewWriter(object.WriterOptions{ + writer := u.repo.Objects.NewWriter(ctx, object.WriterOptions{ Description: "SYMLINK:" + f.Metadata().Name, }) defer writer.Close() //nolint:errcheck @@ -222,8 +223,8 @@ func newDirEntry(md *fs.EntryMetadata, oid object.ID) *dir.Entry { } // uploadFile uploads the specified File to the repository. -func (u *Uploader) uploadFile(file fs.File) (object.ID, error) { - res := u.uploadFileInternal(file, file.Metadata().Name) +func (u *Uploader) uploadFile(ctx context.Context, file fs.File) (object.ID, error) { + res := u.uploadFileInternal(ctx, file, file.Metadata().Name) if res.err != nil { return object.NullID, res.err } @@ -233,13 +234,13 @@ func (u *Uploader) uploadFile(file fs.File) (object.ID, error) { // uploadDir uploads the specified Directory to the repository. // An optional ID of a hash-cache object may be provided, in which case the Uploader will use its // contents to avoid hashing -func (u *Uploader) uploadDir(dir fs.Directory) (object.ID, object.ID, error) { - mw := u.repo.Objects.NewWriter(object.WriterOptions{ +func (u *Uploader) uploadDir(ctx context.Context, dir fs.Directory) (object.ID, object.ID, error) { + mw := u.repo.Objects.NewWriter(ctx, object.WriterOptions{ Description: "HASHCACHE:" + dir.Metadata().Name, }) defer mw.Close() //nolint:errcheck u.cacheWriter = hashcache.NewWriter(mw) - oid, _, err := uploadDirInternal(u, dir, ".") + oid, _, err := uploadDirInternal(ctx, u, dir, ".") if u.IsCancelled() { if err2 := u.cacheReader.CopyTo(u.cacheWriter); err != nil { return object.NullID, object.NullID, err2 @@ -253,7 +254,7 @@ func (u *Uploader) uploadDir(dir fs.Directory) (object.ID, object.ID, error) { } hcid, err := mw.Result() - if flushErr := u.repo.Objects.Flush(); flushErr != nil { + if flushErr := u.repo.Objects.Flush(ctx); flushErr != nil { return object.NullID, object.NullID, fmt.Errorf("can't flush pending objects: %v", flushErr) } return oid, hcid, err @@ -289,7 +290,7 @@ type entryResult struct { hash uint64 } -func (u *Uploader) processSubdirectories(relativePath string, entries fs.Entries, dw *dir.Writer, summ *dir.Summary) error { +func (u *Uploader) processSubdirectories(ctx context.Context, relativePath string, entries fs.Entries, dw *dir.Writer, summ *dir.Summary) error { return u.foreachEntryUnlessCancelled(relativePath, entries, func(entry fs.Entry, entryRelativePath string) error { dir, ok := entry.(fs.Directory) if !ok { @@ -298,7 +299,7 @@ func (u *Uploader) processSubdirectories(relativePath string, entries fs.Entries } e := dir.Metadata() - oid, subdirsumm, err := uploadDirInternal(u, dir, entryRelativePath) + oid, subdirsumm, err := uploadDirInternal(ctx, u, dir, entryRelativePath) if err == errCancelled { return err } @@ -342,7 +343,7 @@ type uploadWorkItem struct { resultChan chan entryResult } -func (u *Uploader) prepareWorkItems(dirRelativePath string, entries fs.Entries, summ *dir.Summary) ([]*uploadWorkItem, error) { +func (u *Uploader) prepareWorkItems(ctx context.Context, dirRelativePath string, entries fs.Entries, summ *dir.Summary) ([]*uploadWorkItem, error) { var result []*uploadWorkItem resultErr := u.foreachEntryUnlessCancelled(dirRelativePath, entries, func(entry fs.Entry, entryRelativePath string) error { @@ -400,7 +401,7 @@ func (u *Uploader) prepareWorkItems(dirRelativePath string, entries fs.Entries, entry: entry, entryRelativePath: entryRelativePath, uploadFunc: func() entryResult { - return u.uploadSymlinkInternal(entry, entryRelativePath) + return u.uploadSymlinkInternal(ctx, entry, entryRelativePath) }, }) @@ -410,7 +411,7 @@ func (u *Uploader) prepareWorkItems(dirRelativePath string, entries fs.Entries, entry: entry, entryRelativePath: entryRelativePath, uploadFunc: func() entryResult { - return u.uploadFileInternal(entry, entryRelativePath) + return u.uploadFileInternal(ctx, entry, entryRelativePath) }, }) @@ -507,6 +508,7 @@ func (u *Uploader) processUploadWorkItems(workItems []*uploadWorkItem, dw *dir.W } func uploadDirInternal( + ctx context.Context, u *Uploader, directory fs.Directory, dirRelativePath string, @@ -515,24 +517,24 @@ func uploadDirInternal( var summ dir.Summary - entries, direrr := directory.Readdir() + entries, direrr := directory.Readdir(ctx) if direrr != nil { return object.NullID, dir.Summary{}, direrr } - writer := u.repo.Objects.NewWriter(object.WriterOptions{ + writer := u.repo.Objects.NewWriter(ctx, object.WriterOptions{ Description: "DIR:" + dirRelativePath, }) dw := dir.NewWriter(writer) defer writer.Close() //nolint:errcheck - if err := u.processSubdirectories(dirRelativePath, entries, dw, &summ); err != nil { + if err := u.processSubdirectories(ctx, dirRelativePath, entries, dw, &summ); err != nil { return object.NullID, dir.Summary{}, err } u.prepareProgress(dirRelativePath, entries) - workItems, workItemErr := u.prepareWorkItems(dirRelativePath, entries, &summ) + workItems, workItemErr := u.prepareWorkItems(ctx, dirRelativePath, entries, &summ) if workItemErr != nil { return object.NullID, dir.Summary{}, workItemErr } @@ -574,6 +576,7 @@ func (u *Uploader) Cancel() { // Upload uploads contents of the specified filesystem entry (file or directory) to the repository and returns snapshot.Manifest with statistics. // Old snapshot manifest, when provided can be used to speed up uploads by utilizing hash cache. func (u *Uploader) Upload( + ctx context.Context, source fs.Entry, sourceInfo SourceInfo, old *Manifest, @@ -588,7 +591,7 @@ func (u *Uploader) Upload( u.stats = Stats{} if old != nil { log.Debug().Msgf("opening hash cache: %v", old.HashCacheID) - if r, err := u.repo.Objects.Open(old.HashCacheID); err == nil { + if r, err := u.repo.Objects.Open(ctx, old.HashCacheID); err == nil { u.cacheReader = hashcache.Open(r) log.Debug().Msgf("opened hash cache: %v", old.HashCacheID) } else { @@ -605,10 +608,10 @@ func (u *Uploader) Upload( switch entry := source.(type) { case fs.Directory: - s.RootObjectID, s.HashCacheID, err = u.uploadDir(entry) + s.RootObjectID, s.HashCacheID, err = u.uploadDir(ctx, entry) case fs.File: - s.RootObjectID, err = u.uploadFile(entry) + s.RootObjectID, err = u.uploadFile(ctx, entry) default: return nil, fmt.Errorf("unsupported source: %v", s.Source) diff --git a/snapshot/upload_test.go b/snapshot/upload_test.go index 1626ec448..1aaea5585 100644 --- a/snapshot/upload_test.go +++ b/snapshot/upload_test.go @@ -52,7 +52,7 @@ func newUploadTestHarness() *uploadTestHarness { panic("unable to create credentials: " + err.Error()) } - if initerr := repo.Initialize(storage, &repo.NewRepositoryOptions{}, creds); initerr != nil { + if initerr := repo.Initialize(ctx, storage, &repo.NewRepositoryOptions{}, creds); initerr != nil { panic("unable to create repository: " + initerr.Error()) } @@ -100,16 +100,17 @@ func newUploadTestHarness() *uploadTestHarness { } func TestUpload(t *testing.T) { + ctx := context.Background() th := newUploadTestHarness() defer th.cleanup() u := NewUploader(th.repo) - s1, err := u.Upload(th.sourceDir, SourceInfo{}, nil) + s1, err := u.Upload(ctx, th.sourceDir, SourceInfo{}, nil) if err != nil { t.Errorf("Upload error: %v", err) } - s2, err := u.Upload(th.sourceDir, SourceInfo{}, s1) + s2, err := u.Upload(ctx, th.sourceDir, SourceInfo{}, s1) if err != nil { t.Errorf("Upload error: %v", err) } @@ -133,7 +134,7 @@ func TestUpload(t *testing.T) { // Add one more file, the s1.RootObjectID should change. th.sourceDir.AddFile("d2/d1/f3", []byte{1, 2, 3, 4, 5}, 0777) - s3, err := u.Upload(th.sourceDir, SourceInfo{}, s1) + s3, err := u.Upload(ctx, th.sourceDir, SourceInfo{}, s1) if err != nil { t.Errorf("upload failed: %v", err) } @@ -154,7 +155,7 @@ func TestUpload(t *testing.T) { // Now remove the added file, OID should be identical to the original before the file got added. th.sourceDir.Subdir("d2", "d1").Remove("f3") - s4, err := u.Upload(th.sourceDir, SourceInfo{}, s1) + s4, err := u.Upload(ctx, th.sourceDir, SourceInfo{}, s1) if err != nil { t.Errorf("upload failed: %v", err) } @@ -171,7 +172,7 @@ func TestUpload(t *testing.T) { t.Errorf("unexpected s4 stats: %+v", s4.Stats) } - s5, err := u.Upload(th.sourceDir, SourceInfo{}, s3) + s5, err := u.Upload(ctx, th.sourceDir, SourceInfo{}, s3) if err != nil { t.Errorf("upload failed: %v", err) } @@ -193,13 +194,14 @@ func TestUpload_Cancel(t *testing.T) { } func TestUpload_TopLevelDirectoryReadFailure(t *testing.T) { + ctx := context.Background() th := newUploadTestHarness() defer th.cleanup() th.sourceDir.FailReaddir(errTest) u := NewUploader(th.repo) - s, err := u.Upload(th.sourceDir, SourceInfo{}, nil) + s, err := u.Upload(ctx, th.sourceDir, SourceInfo{}, nil) if err != errTest { t.Errorf("expected error: %v", err) } @@ -210,6 +212,7 @@ func TestUpload_TopLevelDirectoryReadFailure(t *testing.T) { } func TestUpload_SubDirectoryReadFailure(t *testing.T) { + ctx := context.Background() th := newUploadTestHarness() defer th.cleanup() @@ -217,7 +220,7 @@ func TestUpload_SubDirectoryReadFailure(t *testing.T) { u := NewUploader(th.repo) u.IgnoreFileErrors = false - _, err := u.Upload(th.sourceDir, SourceInfo{}, nil) + _, err := u.Upload(ctx, th.sourceDir, SourceInfo{}, nil) if err == nil { t.Errorf("expected error") } diff --git a/storage/filesystem/filesystem_storage.go b/storage/filesystem/filesystem_storage.go index cd4632f1f..5664e0f99 100644 --- a/storage/filesystem/filesystem_storage.go +++ b/storage/filesystem/filesystem_storage.go @@ -30,7 +30,7 @@ type fsStorage struct { Options } -func (fs *fsStorage) GetBlock(blockID string, offset, length int64) ([]byte, error) { +func (fs *fsStorage) GetBlock(ctx context.Context, blockID string, offset, length int64) ([]byte, error) { _, path := fs.getShardedPathAndFilePath(blockID) f, err := os.Open(path) @@ -65,9 +65,8 @@ func makeFileName(blockID string) string { return blockID + fsStorageChunkSuffix } -func (fs *fsStorage) ListBlocks(prefix string) (<-chan storage.BlockMetadata, storage.CancelFunc) { +func (fs *fsStorage) ListBlocks(ctx context.Context, prefix string) <-chan storage.BlockMetadata { result := make(chan storage.BlockMetadata) - cancelled := make(chan bool) var walkDir func(string, string) @@ -92,7 +91,7 @@ func (fs *fsStorage) ListBlocks(prefix string) (<-chan storage.BlockMetadata, st } else if fullID, ok := getstringFromFileName(currentPrefix + e.Name()); ok { if strings.HasPrefix(fullID, prefix) { select { - case <-cancelled: + case <-ctx.Done(): return case result <- storage.BlockMetadata{ BlockID: fullID, @@ -112,12 +111,10 @@ func (fs *fsStorage) ListBlocks(prefix string) (<-chan storage.BlockMetadata, st } go walkDirAndClose(fs.Path) - return result, func() { - close(cancelled) - } + return result } -func (fs *fsStorage) PutBlock(blockID string, data []byte) error { +func (fs *fsStorage) PutBlock(ctx context.Context, blockID string, r io.Reader) error { _, path := fs.getShardedPathAndFilePath(blockID) tempFile := fmt.Sprintf("%s.tmp.%d", path, rand.Int()) @@ -125,7 +122,7 @@ func (fs *fsStorage) PutBlock(blockID string, data []byte) error { if err != nil { return fmt.Errorf("cannot create temporary file: %v", err) } - if _, err = f.Write(data); err != nil { + if _, err = io.Copy(f, r); err != nil { return fmt.Errorf("can't write temporary file: %v", err) } if err = f.Close(); err != nil { @@ -162,7 +159,7 @@ func (fs *fsStorage) createTempFileAndDir(tempFile string) (*os.File, error) { return f, err } -func (fs *fsStorage) DeleteBlock(blockID string) error { +func (fs *fsStorage) DeleteBlock(ctx context.Context, blockID string) error { _, path := fs.getShardedPathAndFilePath(blockID) err := os.Remove(path) if err == nil || os.IsNotExist(err) { @@ -198,7 +195,7 @@ func (fs *fsStorage) ConnectionInfo() storage.ConnectionInfo { } } -func (fs *fsStorage) Close() error { +func (fs *fsStorage) Close(ctx context.Context) error { return nil } diff --git a/storage/filesystem/filesystem_storage_test.go b/storage/filesystem/filesystem_storage_test.go index d70ae0696..ba8f5982d 100644 --- a/storage/filesystem/filesystem_storage_test.go +++ b/storage/filesystem/filesystem_storage_test.go @@ -10,6 +10,8 @@ import ( ) func TestFileStorage(t *testing.T) { + ctx := context.Background() + // Test varioush shard configurations. for _, shardSpec := range [][]int{ []int{0}, @@ -23,7 +25,7 @@ func TestFileStorage(t *testing.T) { path, _ := ioutil.TempDir("", "r-fs") defer os.RemoveAll(path) - r, err := New(context.Background(), &Options{ + r, err := New(ctx, &Options{ Path: path, DirectoryShards: shardSpec, }) @@ -32,6 +34,6 @@ func TestFileStorage(t *testing.T) { t.Errorf("unexpected result: %v %v", r, err) } - storagetesting.VerifyStorage(t, r) + storagetesting.VerifyStorage(ctx, t, r) } } diff --git a/storage/gcs/gcs_storage.go b/storage/gcs/gcs_storage.go index a370e40ea..6fc7520f2 100644 --- a/storage/gcs/gcs_storage.go +++ b/storage/gcs/gcs_storage.go @@ -5,6 +5,7 @@ import ( "context" "errors" "fmt" + "io" "io/ioutil" "github.com/kopia/kopia/internal/retry" @@ -37,7 +38,7 @@ type gcsStorage struct { uploadThrottler *iothrottler.IOThrottlerPool } -func (gcs *gcsStorage) GetBlock(b string, offset, length int64) ([]byte, error) { +func (gcs *gcsStorage) GetBlock(ctx context.Context, b string, offset, length int64) ([]byte, error) { attempt := func() (interface{}, error) { reader, err := gcs.bucket.Object(gcs.getObjectNameString(b)).NewRangeReader(gcs.ctx, offset, length) if err != nil { @@ -85,31 +86,27 @@ func translateError(err error) error { return fmt.Errorf("unexpected GCS error: %v", err) } } +func (gcs *gcsStorage) PutBlock(ctx context.Context, b string, r io.Reader) error { + ctx, cancel := context.WithCancel(ctx) -func (gcs *gcsStorage) PutBlock(b string, data []byte) error { - attempt := func() (interface{}, error) { - ctx, cancel := context.WithCancel(gcs.ctx) + writer := gcs.bucket.Object(gcs.getObjectNameString(b)).NewWriter(ctx) + writer.ChunkSize = 1 << 20 + writer.ContentType = "application/x-kopia" - writer := gcs.bucket.Object(gcs.getObjectNameString(b)).NewWriter(ctx) - n, err := writer.Write(data) - if err != nil { - cancel() - return nil, err - } - if n != len(data) { - cancel() - return nil, fmt.Errorf("truncated write %v of %v bytes", n, len(data)) - } - - defer cancel() - return nil, writer.Close() + _, err := io.Copy(writer, r) + if err != nil { + // cancel context before closing the writer causes it to abandon the upload. + cancel() + writer.Close() //nolint:errcheck + return translateError(err) } + defer cancel() - _, err := exponentialBackoff(fmt.Sprintf("PutBlock(%q)", b), attempt) - return translateError(err) + // calling close before cancel() causes it to commit the upload. + return translateError(writer.Close()) } -func (gcs *gcsStorage) DeleteBlock(b string) error { +func (gcs *gcsStorage) DeleteBlock(ctx context.Context, b string) error { attempt := func() (interface{}, error) { return nil, gcs.bucket.Object(gcs.getObjectNameString(b)).Delete(gcs.ctx) } @@ -122,9 +119,8 @@ func (gcs *gcsStorage) getObjectNameString(blockID string) string { return gcs.Prefix + blockID } -func (gcs *gcsStorage) ListBlocks(prefix string) (<-chan storage.BlockMetadata, storage.CancelFunc) { +func (gcs *gcsStorage) ListBlocks(ctx context.Context, prefix string) <-chan storage.BlockMetadata { ch := make(chan storage.BlockMetadata, 100) - cancelled := make(chan bool) go func() { defer close(ch) @@ -142,7 +138,7 @@ func (gcs *gcsStorage) ListBlocks(prefix string) (<-chan storage.BlockMetadata, } select { case ch <- bm: - case <-cancelled: + case <-ctx.Done(): return } oa, err = lst.Next() @@ -152,15 +148,13 @@ func (gcs *gcsStorage) ListBlocks(prefix string) (<-chan storage.BlockMetadata, select { case ch <- storage.BlockMetadata{Error: translateError(err)}: return - case <-cancelled: + case <-ctx.Done(): return } } }() - return ch, func() { - close(cancelled) - } + return ch } func (gcs *gcsStorage) ConnectionInfo() storage.ConnectionInfo { @@ -170,7 +164,7 @@ func (gcs *gcsStorage) ConnectionInfo() storage.ConnectionInfo { } } -func (gcs *gcsStorage) Close() error { +func (gcs *gcsStorage) Close(ctx context.Context) error { gcs.storageClient.Close() //nolint:errcheck return nil } diff --git a/storage/logging/logging_storage.go b/storage/logging/logging_storage.go index 008b57fd7..3ed303c22 100644 --- a/storage/logging/logging_storage.go +++ b/storage/logging/logging_storage.go @@ -2,6 +2,8 @@ package logging import ( + "context" + "io" "time" "github.com/kopia/kopia/storage" @@ -14,9 +16,9 @@ type loggingStorage struct { prefix string } -func (s *loggingStorage) GetBlock(id string, offset, length int64) ([]byte, error) { +func (s *loggingStorage) GetBlock(ctx context.Context, id string, offset, length int64) ([]byte, error) { t0 := time.Now() - result, err := s.base.GetBlock(id, offset, length) + result, err := s.base.GetBlock(ctx, id, offset, length) dt := time.Since(t0) if len(result) < 20 { s.printf(s.prefix+"GetBlock(%q,%v,%v)=(%#v, %#v) took %v", id, offset, length, result, err, dt) @@ -26,35 +28,32 @@ func (s *loggingStorage) GetBlock(id string, offset, length int64) ([]byte, erro return result, err } -func (s *loggingStorage) PutBlock(id string, data []byte) error { +func (s *loggingStorage) PutBlock(ctx context.Context, id string, r io.Reader) error { t0 := time.Now() - err := s.base.PutBlock(id, data) + err := s.base.PutBlock(ctx, id, r) dt := time.Since(t0) - s.printf(s.prefix+"PutBlock(%q, len=%v)=%#v took %v", id, len(data), err, dt) + s.printf(s.prefix+"PutBlock(%q)=%#v took %v", id, err, dt) return err } -func (s *loggingStorage) DeleteBlock(id string) error { +func (s *loggingStorage) DeleteBlock(ctx context.Context, id string) error { t0 := time.Now() - err := s.base.DeleteBlock(id) + err := s.base.DeleteBlock(ctx, id) dt := time.Since(t0) s.printf(s.prefix+"DeleteBlock(%q)=%#v took %v", id, err, dt) return err } -func (s *loggingStorage) ListBlocks(prefix string) (<-chan storage.BlockMetadata, storage.CancelFunc) { +func (s *loggingStorage) ListBlocks(ctx context.Context, prefix string) <-chan storage.BlockMetadata { t0 := time.Now() - ch, cf := s.base.ListBlocks(prefix) + ch := s.base.ListBlocks(ctx, prefix) s.printf(s.prefix+"ListBlocks(%q) took %v", prefix, time.Since(t0)) - return ch, func() { - s.printf(s.prefix+"Cancelled ListBlocks(%q)after %v", prefix, time.Since(t0)) - cf() - } + return ch } -func (s *loggingStorage) Close() error { +func (s *loggingStorage) Close(ctx context.Context) error { t0 := time.Now() - err := s.base.Close() + err := s.base.Close(ctx) dt := time.Since(t0) s.printf(s.prefix+"Close()=%#v took %v", err, dt) return err diff --git a/storage/logging/logging_storage_test.go b/storage/logging/logging_storage_test.go index e40544ce3..281673f40 100644 --- a/storage/logging/logging_storage_test.go +++ b/storage/logging/logging_storage_test.go @@ -1,6 +1,7 @@ package logging import ( + "context" "testing" "github.com/kopia/kopia/internal/storagetesting" @@ -12,5 +13,5 @@ func TestLoggingStorage(t *testing.T) { if r == nil { t.Errorf("unexpected result: %v", r) } - storagetesting.VerifyStorage(t, r) + storagetesting.VerifyStorage(context.Background(), t, r) } diff --git a/storage/s3/s3_storage.go b/storage/s3/s3_storage.go index 8afed9ca2..0b26bf163 100644 --- a/storage/s3/s3_storage.go +++ b/storage/s3/s3_storage.go @@ -5,6 +5,7 @@ import ( "context" "errors" "fmt" + "io" "io/ioutil" "github.com/efarrer/iothrottler" @@ -28,7 +29,7 @@ type s3Storage struct { uploadThrottler *iothrottler.IOThrottlerPool } -func (s *s3Storage) GetBlock(b string, offset, length int64) ([]byte, error) { +func (s *s3Storage) GetBlock(ctx context.Context, b string, offset, length int64) ([]byte, error) { attempt := func() (interface{}, error) { var opt minio.GetObjectOptions if length > 0 { @@ -92,31 +93,21 @@ func translateError(err error) error { } } -func (s *s3Storage) PutBlock(b string, data []byte) error { - attempt := func() (interface{}, error) { - rc := ioutil.NopCloser(bytes.NewReader(data)) - throttled, err := s.uploadThrottler.AddReader(rc) - if err != nil { - return nil, err - } - - n, err := s.cli.PutObject(s.BucketName, s.getObjectNameString(b), throttled, int64(len(data)), minio.PutObjectOptions{}) - if err != nil { - return nil, err - } - - if n != int64(len(data)) { - return nil, fmt.Errorf("truncated write %v of %v bytes", n, len(data)) - } - - return nil, nil +func (s *s3Storage) PutBlock(ctx context.Context, b string, r io.Reader) error { + throttled, err := s.uploadThrottler.AddReader(ioutil.NopCloser(r)) + if err != nil { + return err } - _, err := exponentialBackoff(fmt.Sprintf("PutBlock(%q)", b), attempt) + n, err := s.cli.PutObject(s.BucketName, s.getObjectNameString(b), throttled, -1, minio.PutObjectOptions{}) + if err == io.EOF && n == 0 { + // special case empty stream + _, err = s.cli.PutObject(s.BucketName, s.getObjectNameString(b), bytes.NewBuffer(nil), 0, minio.PutObjectOptions{}) + } return translateError(err) } -func (s *s3Storage) DeleteBlock(b string) error { +func (s *s3Storage) DeleteBlock(ctx context.Context, b string) error { attempt := func() (interface{}, error) { return nil, s.cli.RemoveObject(s.BucketName, s.getObjectNameString(b)) } @@ -129,20 +120,19 @@ func (s *s3Storage) getObjectNameString(b string) string { return s.Prefix + b } -func (s *s3Storage) ListBlocks(prefix string) (<-chan storage.BlockMetadata, storage.CancelFunc) { +func (s *s3Storage) ListBlocks(ctx context.Context, prefix string) <-chan storage.BlockMetadata { ch := make(chan storage.BlockMetadata, 100) - cancelled := make(chan struct{}) go func() { defer close(ch) - oi := s.cli.ListObjects(s.BucketName, s.Prefix+prefix, false, cancelled) + oi := s.cli.ListObjects(s.BucketName, s.Prefix+prefix, false, ctx.Done()) for o := range oi { if err := o.Err; err != nil { select { case ch <- storage.BlockMetadata{Error: translateError(err)}: return - case <-cancelled: + case <-ctx.Done(): return } } @@ -155,15 +145,13 @@ func (s *s3Storage) ListBlocks(prefix string) (<-chan storage.BlockMetadata, sto select { case ch <- bm: - case <-cancelled: + case <-ctx.Done(): return } } }() - return ch, func() { - close(cancelled) - } + return ch } func (s *s3Storage) ConnectionInfo() storage.ConnectionInfo { @@ -173,7 +161,7 @@ func (s *s3Storage) ConnectionInfo() storage.ConnectionInfo { } } -func (s *s3Storage) Close() error { +func (s *s3Storage) Close(ctx context.Context) error { return nil } diff --git a/storage/s3/s3_storage_test.go b/storage/s3/s3_storage_test.go index ee46cb248..19736319d 100644 --- a/storage/s3/s3_storage_test.go +++ b/storage/s3/s3_storage_test.go @@ -42,9 +42,11 @@ func TestS3Storage(t *testing.T) { return } + ctx := context.Background() + // recreate per-host bucket, which sometimes get cleaned up by play.minio.io createBucket(t) - cleanupOldData(t) + cleanupOldData(ctx, t) data := make([]byte, 8) rand.Read(data) @@ -60,7 +62,7 @@ func TestS3Storage(t *testing.T) { t.Fatalf("err: %v", err) } - storagetesting.VerifyStorage(t, st) + storagetesting.VerifyStorage(ctx, t, st) } func createBucket(t *testing.T) { @@ -71,7 +73,7 @@ func createBucket(t *testing.T) { minioClient.MakeBucket(bucketName, "us-east-1") } -func cleanupOldData(t *testing.T) { +func cleanupOldData(ctx context.Context, t *testing.T) { // cleanup old data from the bucket st, err := New(context.Background(), &Options{ AccessKeyID: accessKeyID, @@ -83,7 +85,8 @@ func cleanupOldData(t *testing.T) { t.Fatalf("err: %v", err) } - items, cancel := st.ListBlocks("") + ctx, cancel := context.WithCancel(ctx) + items := st.ListBlocks(ctx, "") defer cancel() for it := range items { if it.Error != nil { @@ -93,7 +96,7 @@ func cleanupOldData(t *testing.T) { age := time.Since(it.TimeStamp) if age > cleanupAge { - if err := st.DeleteBlock(it.BlockID); err != nil { + if err := st.DeleteBlock(ctx, it.BlockID); err != nil { t.Errorf("warning: unable to delete %q: %v", it.BlockID, err) } } else { diff --git a/storage/storage.go b/storage/storage.go index de482affd..d311e9ccb 100644 --- a/storage/storage.go +++ b/storage/storage.go @@ -1,6 +1,7 @@ package storage import ( + "context" "errors" "io" "time" @@ -21,27 +22,28 @@ type CancelFunc func() // // The required semantics are provided by existing commercial cloud storage products (Google Cloud, AWS, Azure). type Storage interface { - io.Closer - // PutBlock uploads the block with given data to the repository or replaces existing block with the provided // id with given contents. - PutBlock(id string, data []byte) error + PutBlock(ctx context.Context, id string, data io.Reader) error // DeleteBlock removes the block from storage. Future GetBlock() operations will fail with ErrBlockNotFound. - DeleteBlock(id string) error + DeleteBlock(ctx context.Context, id string) error // GetBlock returns full or partial contents of a block with given ID. // If length>0, the the function retrieves a range of bytes [offset,offset+length) // If length<0, the entire block must be fetched. - GetBlock(id string, offset, length int64) ([]byte, error) + GetBlock(ctx context.Context, id string, offset, length int64) ([]byte, error) // ListBlocks returns a channel of BlockMetadata that describes storage blocks with existing name prefixes. // Iteration continues until all blocks have been listed or until client code invokes the returned cancellation function. - ListBlocks(prefix string) (<-chan BlockMetadata, CancelFunc) + ListBlocks(ctx context.Context, prefix string) <-chan BlockMetadata // ConnectionInfo returns JSON-serializable data structure containing information required to // connect to storage. ConnectionInfo() ConnectionInfo + + // Close releases all resources associated with storage. + Close(ctx context.Context) error } // BlockMetadata represents metadata about a single block in a storage. diff --git a/storage/webdav/webdav_storage.go b/storage/webdav/webdav_storage.go index 1d39409f1..e33993b16 100644 --- a/storage/webdav/webdav_storage.go +++ b/storage/webdav/webdav_storage.go @@ -2,8 +2,10 @@ package webdav import ( + "bytes" "context" "fmt" + "io" "io/ioutil" "net/http" "os" @@ -33,7 +35,7 @@ type davStorage struct { Client *http.Client // HTTP client used when making all calls, may be overridden to use custom auth } -func (d *davStorage) GetBlock(blockID string, offset, length int64) ([]byte, error) { +func (d *davStorage) GetBlock(ctx context.Context, blockID string, offset, length int64) ([]byte, error) { _, urlStr := d.getCollectionAndFileURL(blockID) req, err := http.NewRequest("GET", urlStr, nil) @@ -74,9 +76,8 @@ func makeFileName(blockID string) string { return blockID + fsStorageChunkSuffix } -func (d *davStorage) ListBlocks(prefix string) (<-chan storage.BlockMetadata, storage.CancelFunc) { +func (d *davStorage) ListBlocks(ctx context.Context, prefix string) <-chan storage.BlockMetadata { result := make(chan storage.BlockMetadata) - cancelled := make(chan bool) var walkDir func(string, string) @@ -99,7 +100,7 @@ func (d *davStorage) ListBlocks(prefix string) (<-chan storage.BlockMetadata, st } else if fullID, ok := getBlockIDFromFileName(currentPrefix + e.name); ok { if strings.HasPrefix(fullID, prefix) { select { - case <-cancelled: + case <-ctx.Done(): return case result <- storage.BlockMetadata{ BlockID: fullID, @@ -119,9 +120,7 @@ func (d *davStorage) ListBlocks(prefix string) (<-chan storage.BlockMetadata, st } go walkDirAndClose(d.URL) - return result, func() { - close(cancelled) - } + return result } func (d *davStorage) makeCollectionAll(urlStr string) error { @@ -244,11 +243,19 @@ func (d *davStorage) putBlockInternal(urlStr string, data []byte) error { } } -func (d *davStorage) PutBlock(blockID string, data []byte) error { +func (d *davStorage) PutBlock(ctx context.Context, blockID string, r io.Reader) error { shardPath, url := d.getCollectionAndFileURL(blockID) + var buf bytes.Buffer + _, err := buf.ReadFrom(r) + if err != nil { + return err + } + + data := buf.Bytes() + tmpURL := url + "-" + makeClientNonce() - err := d.putBlockInternal(tmpURL, data) + err = d.putBlockInternal(tmpURL, data) if err == storage.ErrBlockNotFound { if err = d.makeCollectionAll(shardPath); err != nil { @@ -272,7 +279,7 @@ func (d *davStorage) PutBlock(blockID string, data []byte) error { return nil } -func (d *davStorage) DeleteBlock(blockID string) error { +func (d *davStorage) DeleteBlock(ctx context.Context, blockID string) error { _, url := d.getCollectionAndFileURL(blockID) err := os.Remove(url) if err == nil || os.IsNotExist(err) { @@ -308,7 +315,7 @@ func (d *davStorage) ConnectionInfo() storage.ConnectionInfo { } } -func (d *davStorage) Close() error { +func (d *davStorage) Close(ctx context.Context) error { return nil } diff --git a/storage/webdav/webdav_storage_test.go b/storage/webdav/webdav_storage_test.go index 0c693bf84..2966c40c8 100644 --- a/storage/webdav/webdav_storage_test.go +++ b/storage/webdav/webdav_storage_test.go @@ -27,6 +27,8 @@ func TestWebDAVStorage(t *testing.T) { server := httptest.NewServer(mux) defer server.Close() + ctx := context.Background() + // Test varioush shard configurations. for _, shardSpec := range [][]int{ []int{1}, @@ -50,6 +52,6 @@ func TestWebDAVStorage(t *testing.T) { t.Errorf("unexpected result: %v %v", r, err) } - storagetesting.VerifyStorage(t, r) + storagetesting.VerifyStorage(ctx, t, r) } }