plumbed through ctx in storage.Storage APIs and all uses

block/block_cache.go

@@ -11,9 +11,9 @@ import (
 )
 
 type blockCache interface {
-	getBlock(virtualBlockID, physicalBlockID string, offset, length int64) ([]byte, error)
-	putBlock(blockID string, data []byte) error
-	listIndexBlocks(full bool) ([]Info, error)
+	getBlock(ctx context.Context, virtualBlockID, physicalBlockID string, offset, length int64) ([]byte, error)
+	putBlock(ctx context.Context, blockID string, data []byte) error
+	listIndexBlocks(ctx context.Context, full bool) ([]Info, error)
 	close() error
 }
 
@@ -26,7 +26,7 @@ type CachingOptions struct {
 	HMACSecret              []byte `json:"-"`
 }
 
-func newBlockCache(st storage.Storage, caching CachingOptions) (blockCache, error) {
+func newBlockCache(ctx context.Context, st storage.Storage, caching CachingOptions) (blockCache, error) {
 	if caching.MaxCacheSizeBytes == 0 || caching.CacheDirectory == "" {
 		return nullBlockCache{st}, nil
 	}
@@ -53,13 +53,13 @@ func newBlockCache(st storage.Storage, caching CachingOptions) (blockCache, erro
 	}
 
 	if caching.IgnoreListCache {
-		c.deleteListCache()
+		c.deleteListCache(ctx)
 	}
 
-	if err := c.sweepDirectory(); err != nil {
+	if err := c.sweepDirectory(ctx); err != nil {
 		return nil, err
 	}
-	go c.sweepDirectoryPeriodically()
+	go c.sweepDirectoryPeriodically(ctx)
 
 	return c, nil
 }

block/block_manager.go

@@ -1,6 +1,7 @@
 package block
 
 import (
+	"context"
 	cryptorand "crypto/rand"
 	"fmt"
 	"io"
@@ -110,7 +111,7 @@ func (bm *Manager) addToIndexLocked(blockID string, ndx *blockmgrpb.Index, packe
 	bm.blockIDToIndex[blockID] = ndx
 }
 
-func (bm *Manager) addToPackLocked(blockID string, data []byte, force bool) error {
+func (bm *Manager) addToPackLocked(ctx context.Context, blockID string, data []byte, force bool) error {
 	bm.assertLocked()
 
 	if !force {
@@ -137,7 +138,7 @@ func (bm *Manager) addToPackLocked(blockID string, data []byte, force bool) erro
 	bm.addToIndexLocked(blockID, bm.currentPackIndex, packOffsetAndSize(uint32(offset), uint32(len(data))))
 
 	if shouldFinish {
-		if err := bm.finishPackAndMaybeFlushIndexes(); err != nil {
+		if err := bm.finishPackAndMaybeFlushIndexes(ctx); err != nil {
 			return err
 		}
 	}
@@ -145,13 +146,13 @@ func (bm *Manager) addToPackLocked(blockID string, data []byte, force bool) erro
 	return nil
 }
 
-func (bm *Manager) finishPackAndMaybeFlushIndexes() error {
-	if err := bm.finishPackLocked(); err != nil {
+func (bm *Manager) finishPackAndMaybeFlushIndexes(ctx context.Context) error {
+	if err := bm.finishPackLocked(ctx); err != nil {
 		return err
 	}
 
 	if bm.timeNow().After(bm.flushPackIndexesAfter) {
-		if err := bm.flushPackIndexesLocked(); err != nil {
+		if err := bm.flushPackIndexesLocked(ctx); err != nil {
 			return err
 		}
 	}
@@ -239,12 +240,12 @@ func (bm *Manager) startPackIndexLocked() {
 	bm.currentPackData = []byte{}
 }
 
-func (bm *Manager) flushPackIndexesLocked() error {
+func (bm *Manager) flushPackIndexesLocked(ctx context.Context) error {
 	if len(bm.pendingPackIndexes) > 0 {
 		if false {
 			log.Printf("saving %v pack indexes", len(bm.pendingPackIndexes))
 		}
-		if _, err := bm.writePackIndexes(bm.pendingPackIndexes, nil); err != nil {
+		if _, err := bm.writePackIndexes(ctx, bm.pendingPackIndexes, nil); err != nil {
 			return err
 		}
 	}
@@ -254,7 +255,7 @@ func (bm *Manager) flushPackIndexesLocked() error {
 	return nil
 }
 
-func (bm *Manager) writePackIndexes(ndx []*blockmgrpb.Index, replacesBlockBeforeTime *time.Time) (string, error) {
+func (bm *Manager) writePackIndexes(ctx context.Context, ndx []*blockmgrpb.Index, replacesBlockBeforeTime *time.Time) (string, error) {
 	data, err := proto.Marshal(&blockmgrpb.Indexes{
 		Indexes: ndx,
 	})
@@ -269,10 +270,10 @@ func (bm *Manager) writePackIndexes(ndx []*blockmgrpb.Index, replacesBlockBefore
 
 	inverseTimePrefix := fmt.Sprintf("%016x", math.MaxInt64-time.Now().UnixNano())
 
-	return bm.writeUnpackedBlockNotLocked(data, indexBlockPrefix+inverseTimePrefix, suffix, true)
+	return bm.writeUnpackedBlockNotLocked(ctx, data, indexBlockPrefix+inverseTimePrefix, suffix, true)
 }
 
-func (bm *Manager) finishPackLocked() error {
+func (bm *Manager) finishPackLocked(ctx context.Context) error {
 	if len(bm.currentPackIndex.Items)+len(bm.currentPackIndex.DeletedItems)+len(bm.currentPackIndex.InlineItems) > 0 {
 		log.Debug().Int("items", len(bm.currentPackIndex.Items)).Int("deleted", len(bm.currentPackIndex.DeletedItems)).Int("inline", len(bm.currentPackIndex.InlineItems)).Msg("finishing pack")
 		if len(bm.currentPackData) < bm.maxInlineContentLength {
@@ -295,7 +296,7 @@ func (bm *Manager) finishPackLocked() error {
 					bm.currentPackData = append(bm.currentPackData, postamble...)
 				}
 			}
-			blockID, err := bm.writeUnpackedBlockNotLocked(bm.currentPackData, "", "", true)
+			blockID, err := bm.writeUnpackedBlockNotLocked(ctx, bm.currentPackData, "", "", true)
 			if err != nil {
 				return fmt.Errorf("can't save pack data block %q: %v", blockID, err)
 			}
@@ -316,8 +317,8 @@ func (bm *Manager) finishPackLocked() error {
 }
 
 // ListIndexBlocks returns the list of all index blocks, including inactive, sorted by time.
-func (bm *Manager) ListIndexBlocks() ([]Info, error) {
-	blocks, err := bm.cache.listIndexBlocks(true)
+func (bm *Manager) ListIndexBlocks(ctx context.Context) ([]Info, error) {
+	blocks, err := bm.cache.listIndexBlocks(ctx, true)
 	if err != nil {
 		return nil, fmt.Errorf("error listing index blocks: %v", err)
 	}
@@ -327,8 +328,8 @@ func (bm *Manager) ListIndexBlocks() ([]Info, error) {
 }
 
 // ActiveIndexBlocks returns the list of active index blocks, sorted by time.
-func (bm *Manager) ActiveIndexBlocks() ([]Info, error) {
-	blocks, err := bm.cache.listIndexBlocks(false)
+func (bm *Manager) ActiveIndexBlocks(ctx context.Context) ([]Info, error) {
+	blocks, err := bm.cache.listIndexBlocks(ctx, false)
 	if err != nil {
 		return nil, err
 	}
@@ -375,9 +376,9 @@ func findLatestCompactedTimestamp(blocks []Info) (time.Time, error) {
 	return latestTime, nil
 }
 
-func (bm *Manager) loadMergedPackIndexLocked() ([]*blockmgrpb.Index, []string, time.Time, error) {
+func (bm *Manager) loadMergedPackIndexLocked(ctx context.Context) ([]*blockmgrpb.Index, []string, time.Time, error) {
 	log.Debug().Msg("listing active index blocks")
-	blocks, err := bm.ActiveIndexBlocks()
+	blocks, err := bm.ActiveIndexBlocks(ctx)
 	if err != nil {
 		return nil, nil, zeroTime, err
 	}
@@ -412,7 +413,7 @@ func (bm *Manager) loadMergedPackIndexLocked() ([]*blockmgrpb.Index, []string, t
 			defer wg.Done()
 
 			for b := range ch {
-				data, err := bm.getBlockInternalLocked(b)
+				data, err := bm.getBlockInternalLocked(ctx, b)
 				if err != nil {
 					errors <- err
 					return
@@ -453,11 +454,11 @@ func (bm *Manager) loadMergedPackIndexLocked() ([]*blockmgrpb.Index, []string, t
 	return merged, blockIDs, blocks[len(blocks)-1].Timestamp, nil
 }
 
-func (bm *Manager) initializeIndexes() error {
+func (bm *Manager) initializeIndexes(ctx context.Context) error {
 	bm.lock()
 	defer bm.unlock()
 
-	merged, blockIDs, latestBlockTime, err := bm.loadMergedPackIndexLocked()
+	merged, blockIDs, latestBlockTime, err := bm.loadMergedPackIndexLocked(ctx)
 	if err != nil {
 		return err
 	}
@@ -467,7 +468,7 @@ func (bm *Manager) initializeIndexes() error {
 	if len(blockIDs) >= autoCompactionBlockCount && latestBlockTime.Before(time.Now().Add(-autoCompactionSafetyMargin)) {
 		log.Debug().Msgf("auto compacting block indexes (block count %v exceeds threshold of %v)", len(blockIDs), autoCompactionBlockCount)
 		merged = removeEmptyIndexes(merged)
-		if _, err := bm.writePackIndexes(merged, &latestBlockTime); err != nil {
+		if _, err := bm.writePackIndexes(ctx, merged, &latestBlockTime); err != nil {
 			return err
 		}
 	}
@@ -525,16 +526,16 @@ func removeEmptyIndexes(ndx []*blockmgrpb.Index) []*blockmgrpb.Index {
 }
 
 // CompactIndexes performs compaction of index blocks.
-func (bm *Manager) CompactIndexes() error {
+func (bm *Manager) CompactIndexes(ctx context.Context) error {
 	bm.lock()
 	defer bm.unlock()
 
-	merged, blockIDs, latestBlockTime, err := bm.loadMergedPackIndexLocked()
+	merged, blockIDs, latestBlockTime, err := bm.loadMergedPackIndexLocked(ctx)
 	if err != nil {
 		return err
 	}
 
-	if err := bm.compactIndexes(merged, blockIDs, latestBlockTime); err != nil {
+	if err := bm.compactIndexes(ctx, merged, blockIDs, latestBlockTime); err != nil {
 		return err
 	}
 
@@ -578,7 +579,7 @@ func newInfo(blockID string, ndx *blockmgrpb.Index) Info {
 	}
 }
 
-func (bm *Manager) compactIndexes(merged []*blockmgrpb.Index, blockIDs []string, latestBlockTime time.Time) error {
+func (bm *Manager) compactIndexes(ctx context.Context, merged []*blockmgrpb.Index, blockIDs []string, latestBlockTime time.Time) error {
 	dedupeBlockIDsAndIndex(merged)
 	merged = removeEmptyIndexes(merged)
 	if len(blockIDs) <= 1 {
@@ -586,20 +587,20 @@ func (bm *Manager) compactIndexes(merged []*blockmgrpb.Index, blockIDs []string,
 		return nil
 	}
 
-	_, err := bm.writePackIndexes(merged, &latestBlockTime)
+	_, err := bm.writePackIndexes(ctx, merged, &latestBlockTime)
 	return err
 }
 
 // Flush completes writing any pending packs and writes pack indexes to the underlyign storage.
-func (bm *Manager) Flush() error {
+func (bm *Manager) Flush(ctx context.Context) error {
 	bm.lock()
 	defer bm.unlock()
 
-	if err := bm.finishPackLocked(); err != nil {
+	if err := bm.finishPackLocked(ctx); err != nil {
 		return err
 	}
 
-	if err := bm.flushPackIndexesLocked(); err != nil {
+	if err := bm.flushPackIndexesLocked(ctx); err != nil {
 		return err
 	}
 
@@ -608,13 +609,13 @@ func (bm *Manager) Flush() error {
 
 // WriteBlock saves a given block of data to a pack group with a provided name and returns a blockID
 // that's based on the contents of data written.
-func (bm *Manager) WriteBlock(data []byte, prefix string) (string, error) {
+func (bm *Manager) WriteBlock(ctx context.Context, data []byte, prefix string) (string, error) {
 	blockID := prefix + bm.hashData(data)
 
 	bm.lock()
 	defer bm.unlock()
 
-	err := bm.addToPackLocked(blockID, data, false)
+	err := bm.addToPackLocked(ctx, blockID, data, false)
 	return blockID, err
 }
 
@@ -627,11 +628,11 @@ func (bm *Manager) IsStorageBlockInUse(storageBlockID string) bool {
 }
 
 // Repackage reorganizes all pack blocks belonging to a given group that are not bigger than given size.
-func (bm *Manager) Repackage(maxLength uint64) error {
+func (bm *Manager) Repackage(ctx context.Context, maxLength uint64) error {
 	bm.lock()
 	defer bm.unlock()
 
-	merged, _, _, err := bm.loadMergedPackIndexLocked()
+	merged, _, _, err := bm.loadMergedPackIndexLocked(ctx)
 	if err != nil {
 		return err
 	}
@@ -661,7 +662,7 @@ func (bm *Manager) Repackage(maxLength uint64) error {
 	log.Printf("%v blocks to re-package (%v total bytes)", len(toRepackage), totalBytes)
 
 	for _, m := range toRepackage {
-		if err := bm.repackageBlock(m, done); err != nil {
+		if err := bm.repackageBlock(ctx, m, done); err != nil {
 			return err
 		}
 	}
@@ -669,8 +670,8 @@ func (bm *Manager) Repackage(maxLength uint64) error {
 	return nil
 }
 
-func (bm *Manager) repackageBlock(m *blockmgrpb.Index, done map[string]bool) error {
-	data, err := bm.getBlockInternalLocked(m.PackBlockId)
+func (bm *Manager) repackageBlock(ctx context.Context, m *blockmgrpb.Index, done map[string]bool) error {
+	data, err := bm.getBlockInternalLocked(ctx, m.PackBlockId)
 	if err != nil {
 		return fmt.Errorf("can't fetch block %q for repackaging: %v", m.PackBlockId, err)
 	}
@@ -684,7 +685,7 @@ func (bm *Manager) repackageBlock(m *blockmgrpb.Index, done map[string]bool) err
 
 		offset, size := unpackOffsetAndSize(os)
 		blockData := data[offset : offset+size]
-		if err := bm.addToPackLocked(blockID, blockData, true); err != nil {
+		if err := bm.addToPackLocked(ctx, blockID, blockData, true); err != nil {
 			return fmt.Errorf("unable to re-package %q: %v", blockID, err)
 		}
 	}
@@ -694,7 +695,7 @@ func (bm *Manager) repackageBlock(m *blockmgrpb.Index, done map[string]bool) err
 			continue
 		}
 		done[blockID] = true
-		if err := bm.addToPackLocked(blockID, blockData, true); err != nil {
+		if err := bm.addToPackLocked(ctx, blockID, blockData, true); err != nil {
 			return fmt.Errorf("unable to re-package %q: %v", blockID, err)
 		}
 	}
@@ -702,12 +703,12 @@ func (bm *Manager) repackageBlock(m *blockmgrpb.Index, done map[string]bool) err
 	return nil
 }
 
-func (bm *Manager) writeUnpackedBlockNotLocked(data []byte, prefix string, suffix string, force bool) (string, error) {
+func (bm *Manager) writeUnpackedBlockNotLocked(ctx context.Context, data []byte, prefix string, suffix string, force bool) (string, error) {
 	blockID := prefix + bm.hashData(data) + suffix
 
 	if !force {
 		// Before performing encryption, check if the block is already there.
-		i, err := bm.BlockInfo(blockID)
+		i, err := bm.BlockInfo(ctx, blockID)
 		atomic.AddInt32(&bm.stats.CheckedBlocks, 1)
 		if err == nil && i.Length == int64(len(data)) {
 			atomic.AddInt32(&bm.stats.PresentBlocks, 1)
@@ -730,7 +731,7 @@ func (bm *Manager) writeUnpackedBlockNotLocked(data []byte, prefix string, suffi
 
 	atomic.AddInt32(&bm.stats.WrittenBlocks, 1)
 	atomic.AddInt64(&bm.stats.WrittenBytes, int64(len(data)))
-	if err := bm.cache.putBlock(blockID, data2); err != nil {
+	if err := bm.cache.putBlock(ctx, blockID, data2); err != nil {
 		return "", err
 	}
 
@@ -761,7 +762,7 @@ func (bm *Manager) getPendingBlockLocked(blockID string) ([]byte, error) {
 }
 
 // GetBlock gets the contents of a given block. If the block is not found returns blob.ErrBlockNotFound.
-func (bm *Manager) GetBlock(blockID string) ([]byte, error) {
+func (bm *Manager) GetBlock(ctx context.Context, blockID string) ([]byte, error) {
 	bm.lock()
 	defer bm.unlock()
 
@@ -769,11 +770,11 @@ func (bm *Manager) GetBlock(blockID string) ([]byte, error) {
 		return b, nil
 	}
 
-	return bm.getBlockInternalLocked(blockID)
+	return bm.getBlockInternalLocked(ctx, blockID)
 }
 
 // BlockInfo returns information about a single block.
-func (bm *Manager) BlockInfo(blockID string) (Info, error) {
+func (bm *Manager) BlockInfo(ctx context.Context, blockID string) (Info, error) {
 	bm.lock()
 	defer bm.unlock()
 
@@ -828,7 +829,7 @@ func (bm *Manager) blockInfoLocked(blockID string) (Info, error) {
 	}, nil
 }
 
-func (bm *Manager) getBlockInternalLocked(blockID string) ([]byte, error) {
+func (bm *Manager) getBlockInternalLocked(ctx context.Context, blockID string) ([]byte, error) {
 	bm.assertLocked()
 
 	if ndx, ok := bm.blockIDToIndex[blockID]; ok {
@@ -848,10 +849,10 @@ func (bm *Manager) getBlockInternalLocked(blockID string) ([]byte, error) {
 
 	if s.PackBlockID != "" {
 		underlyingBlockID = s.PackBlockID
-		payload, err = bm.cache.getBlock(blockID, underlyingBlockID, s.PackOffset, s.Length)
+		payload, err = bm.cache.getBlock(ctx, blockID, underlyingBlockID, s.PackOffset, s.Length)
 		decryptSkip = int(s.PackOffset)
 	} else {
-		payload, err = bm.cache.getBlock(blockID, blockID, 0, -1)
+		payload, err = bm.cache.getBlock(ctx, blockID, blockID, 0, -1)
 	}
 
 	if err != nil {
@@ -919,15 +920,17 @@ type cachedList struct {
 // If 'full' is set to true, this function lists and returns all blocks,
 // if 'full' is false, the function returns only blocks from the last 2 compactions.
 // The list of blocks is not guaranteed to be sorted.
-func listIndexBlocksFromStorage(st storage.Storage, full bool) ([]Info, error) {
+func listIndexBlocksFromStorage(ctx context.Context, st storage.Storage, full bool) ([]Info, error) {
 	maxCompactions := 2
 	if full {
 		maxCompactions = math.MaxInt32
 	}
 
-	ch, cancel := st.ListBlocks(indexBlockPrefix)
+	ctx, cancel := context.WithCancel(ctx)
 	defer cancel()
 
+	ch := st.ListBlocks(ctx, indexBlockPrefix)
+
 	var results []Info
 	numCompactions := 0
 
@@ -959,11 +962,11 @@ func listIndexBlocksFromStorage(st storage.Storage, full bool) ([]Info, error) {
 }
 
 // NewManager creates new block manager with given packing options and a formatter.
-func NewManager(st storage.Storage, f FormattingOptions, caching CachingOptions) (*Manager, error) {
-	return newManagerWithTime(st, f, caching, time.Now)
+func NewManager(ctx context.Context, st storage.Storage, f FormattingOptions, caching CachingOptions) (*Manager, error) {
+	return newManagerWithTime(ctx, st, f, caching, time.Now)
 }
 
-func newManagerWithTime(st storage.Storage, f FormattingOptions, caching CachingOptions, timeNow func() time.Time) (*Manager, error) {
+func newManagerWithTime(ctx context.Context, st storage.Storage, f FormattingOptions, caching CachingOptions, timeNow func() time.Time) (*Manager, error) {
 	sf := FormatterFactories[f.BlockFormat]
 	if sf == nil {
 		return nil, fmt.Errorf("unsupported block format: %v", f.BlockFormat)
@@ -974,7 +977,7 @@ func newManagerWithTime(st storage.Storage, f FormattingOptions, caching Caching
 		return nil, err
 	}
 
-	cache, err := newBlockCache(st, caching)
+	cache, err := newBlockCache(ctx, st, caching)
 	if err != nil {
 		return nil, fmt.Errorf("unable to initialize cache: %v", err)
 	}
@@ -1001,7 +1004,7 @@ func newManagerWithTime(st storage.Storage, f FormattingOptions, caching Caching
 
 	m.startPackIndexLocked()
 
-	if err := m.initializeIndexes(); err != nil {
+	if err := m.initializeIndexes(ctx); err != nil {
 		return nil, fmt.Errorf("unable initialize indexes: %v", err)
 	}
 

block/block_manager_test.go

@@ -1,6 +1,7 @@
 package block
 
 import (
+	"context"
 	"crypto/md5"
 	"encoding/hex"
 	"fmt"
@@ -34,21 +35,23 @@ func init() {
 }
 
 func TestBlockManagerEmptyFlush(t *testing.T) {
+	ctx := context.Background()
 	data := map[string][]byte{}
 	keyTime := map[string]time.Time{}
 	bm := newTestBlockManager(data, keyTime, nil)
-	bm.Flush()
+	bm.Flush(ctx)
 	if got, want := len(data), 0; got != want {
 		t.Errorf("unexpected number of blocks: %v, wanted %v", got, want)
 	}
 }
 
 func TestBlockZeroBytes1(t *testing.T) {
+	ctx := context.Background()
 	data := map[string][]byte{}
 	keyTime := map[string]time.Time{}
 	bm := newTestBlockManager(data, keyTime, nil)
-	writeBlockAndVerify(t, bm, []byte{})
-	bm.Flush()
+	writeBlockAndVerify(ctx, t, bm, []byte{})
+	bm.Flush(ctx)
 	if got, want := len(data), 2; got != want {
 		t.Errorf("unexpected number of blocks: %v, wanted %v", got, want)
 	}
@@ -56,12 +59,13 @@ func TestBlockZeroBytes1(t *testing.T) {
 }
 
 func TestBlockZeroBytes2(t *testing.T) {
+	ctx := context.Background()
 	data := map[string][]byte{}
 	keyTime := map[string]time.Time{}
 	bm := newTestBlockManager(data, keyTime, nil)
-	writeBlockAndVerify(t, bm, seededRandomData(10, 10))
-	writeBlockAndVerify(t, bm, []byte{})
-	bm.Flush()
+	writeBlockAndVerify(ctx, t, bm, seededRandomData(10, 10))
+	writeBlockAndVerify(ctx, t, bm, []byte{})
+	bm.Flush(ctx)
 	dumpBlockManagerData(data)
 	if got, want := len(data), 2; got != want {
 		t.Errorf("unexpected number of blocks: %v, wanted %v", got, want)
@@ -70,60 +74,63 @@ func TestBlockZeroBytes2(t *testing.T) {
 }
 
 func TestBlockManagerSmallBlockWrites(t *testing.T) {
+	ctx := context.Background()
 	data := map[string][]byte{}
 	keyTime := map[string]time.Time{}
 	bm := newTestBlockManager(data, keyTime, nil)
 
 	for i := 0; i < 100; i++ {
-		writeBlockAndVerify(t, bm, seededRandomData(i, 10))
+		writeBlockAndVerify(ctx, t, bm, seededRandomData(i, 10))
 	}
 	if got, want := len(data), 0; got != want {
 		t.Errorf("unexpected number of blocks: %v, wanted %v", got, want)
 	}
-	bm.Flush()
+	bm.Flush(ctx)
 	if got, want := len(data), 2; got != want {
 		t.Errorf("unexpected number of blocks: %v, wanted %v", got, want)
 	}
 }
 
 func TestBlockManagerDedupesPendingBlocks(t *testing.T) {
+	ctx := context.Background()
 	data := map[string][]byte{}
 	keyTime := map[string]time.Time{}
 	bm := newTestBlockManager(data, keyTime, nil)
 
 	for i := 0; i < 100; i++ {
-		writeBlockAndVerify(t, bm, seededRandomData(0, 999))
+		writeBlockAndVerify(ctx, t, bm, seededRandomData(0, 999))
 	}
 	if got, want := len(data), 0; got != want {
 		t.Errorf("unexpected number of blocks: %v, wanted %v", got, want)
 	}
-	bm.Flush()
+	bm.Flush(ctx)
 	if got, want := len(data), 2; got != want {
 		t.Errorf("unexpected number of blocks: %v, wanted %v", got, want)
 	}
 }
 
 func TestBlockManagerDedupesPendingAndUncommittedBlocks(t *testing.T) {
+	ctx := context.Background()
 	data := map[string][]byte{}
 	keyTime := map[string]time.Time{}
 	bm := newTestBlockManager(data, keyTime, nil)
 
 	// no writes here, all data fits in a single pack.
-	writeBlockAndVerify(t, bm, seededRandomData(0, 950))
-	writeBlockAndVerify(t, bm, seededRandomData(1, 950))
-	writeBlockAndVerify(t, bm, seededRandomData(2, 10))
+	writeBlockAndVerify(ctx, t, bm, seededRandomData(0, 950))
+	writeBlockAndVerify(ctx, t, bm, seededRandomData(1, 950))
+	writeBlockAndVerify(ctx, t, bm, seededRandomData(2, 10))
 	if got, want := len(data), 0; got != want {
 		t.Errorf("unexpected number of blocks: %v, wanted %v", got, want)
 	}
 
 	// no writes here
-	writeBlockAndVerify(t, bm, seededRandomData(0, 950))
-	writeBlockAndVerify(t, bm, seededRandomData(1, 950))
-	writeBlockAndVerify(t, bm, seededRandomData(2, 10))
+	writeBlockAndVerify(ctx, t, bm, seededRandomData(0, 950))
+	writeBlockAndVerify(ctx, t, bm, seededRandomData(1, 950))
+	writeBlockAndVerify(ctx, t, bm, seededRandomData(2, 10))
 	if got, want := len(data), 0; got != want {
 		t.Errorf("unexpected number of blocks: %v, wanted %v", got, want)
 	}
-	bm.Flush()
+	bm.Flush(ctx)
 
 	// this flushes the pack block + index block
 	if got, want := len(data), 2; got != want {
@@ -133,18 +140,19 @@ func TestBlockManagerDedupesPendingAndUncommittedBlocks(t *testing.T) {
 }
 
 func TestBlockManagerEmpty(t *testing.T) {
+	ctx := context.Background()
 	data := map[string][]byte{}
 	keyTime := map[string]time.Time{}
 	bm := newTestBlockManager(data, keyTime, nil)
 
 	noSuchBlockID := md5hash([]byte("foo"))
 
-	b, err := bm.GetBlock(noSuchBlockID)
+	b, err := bm.GetBlock(ctx, noSuchBlockID)
 	if err != storage.ErrBlockNotFound {
 		t.Errorf("unexpected error when getting non-existent block: %v, %v", b, err)
 	}
 
-	bi, err := bm.BlockInfo(noSuchBlockID)
+	bi, err := bm.BlockInfo(ctx, noSuchBlockID)
 	if err != storage.ErrBlockNotFound {
 		t.Errorf("unexpected error when getting non-existent block info: %v, %v", bi, err)
 	}
@@ -155,6 +163,7 @@ func TestBlockManagerEmpty(t *testing.T) {
 }
 
 func TestBlockManagerRepack(t *testing.T) {
+	ctx := context.Background()
 	data := map[string][]byte{}
 	keyTime := map[string]time.Time{}
 	bm := newTestBlockManager(data, keyTime, nil)
@@ -168,12 +177,12 @@ func TestBlockManagerRepack(t *testing.T) {
 	d2 := seededRandomData(2, 20)
 	d3 := seededRandomData(3, 30)
 
-	writeBlockAndVerify(t, bm, d1)
-	bm.Flush()
-	writeBlockAndVerify(t, bm, d2)
-	bm.Flush()
-	writeBlockAndVerify(t, bm, d3)
-	bm.Flush()
+	writeBlockAndVerify(ctx, t, bm, d1)
+	bm.Flush(ctx)
+	writeBlockAndVerify(ctx, t, bm, d2)
+	bm.Flush(ctx)
+	writeBlockAndVerify(ctx, t, bm, d3)
+	bm.Flush(ctx)
 
 	// 3 data blocks, 3 index blocks.
 	if got, want := len(data), 6; got != want {
@@ -183,10 +192,10 @@ func TestBlockManagerRepack(t *testing.T) {
 	log.Printf("before repackage")
 	dumpBlockManagerData(data)
 
-	if err := bm.Repackage(5); err != nil {
+	if err := bm.Repackage(ctx, 5); err != nil {
 		t.Errorf("repackage failure: %v", err)
 	}
-	bm.Flush()
+	bm.Flush(ctx)
 
 	// nothing happened, still 3 data blocks, 3 index blocks.
 	if got, want := len(data), 6; got != want {
@@ -195,10 +204,10 @@ func TestBlockManagerRepack(t *testing.T) {
 
 	bm.timeNow = fakeTimeNowFrozen(fakeTime.Add(1 * time.Second))
 
-	if err := bm.Repackage(30); err != nil {
+	if err := bm.Repackage(ctx, 30); err != nil {
 		t.Errorf("repackage failure: %v", err)
 	}
-	bm.Flush()
+	bm.Flush(ctx)
 
 	log.Printf("after repackage")
 	dumpBlockManagerData(data)
@@ -207,7 +216,7 @@ func TestBlockManagerRepack(t *testing.T) {
 	if got, want := len(data), 8; got != want {
 		t.Errorf("unexpected block count: %v, wanted %v", got, want)
 	}
-	if err := bm.CompactIndexes(); err != nil {
+	if err := bm.CompactIndexes(ctx); err != nil {
 		t.Errorf("compaction failure: %v", err)
 	}
 
@@ -216,13 +225,13 @@ func TestBlockManagerRepack(t *testing.T) {
 		dumpBlockManagerData(data)
 	}
 
-	verifyActiveIndexBlockCount(t, bm, 1)
+	verifyActiveIndexBlockCount(ctx, t, bm, 1)
 }
 
-func verifyActiveIndexBlockCount(t *testing.T, bm *Manager, expected int) {
+func verifyActiveIndexBlockCount(ctx context.Context, t *testing.T, bm *Manager, expected int) {
 	t.Helper()
 
-	blks, err := bm.ActiveIndexBlocks()
+	blks, err := bm.ActiveIndexBlocks(ctx)
 	if err != nil {
 		t.Errorf("error listing active index blocks: %v", err)
 		return
@@ -233,6 +242,7 @@ func verifyActiveIndexBlockCount(t *testing.T, bm *Manager, expected int) {
 	}
 }
 func TestBlockManagerInternalFlush(t *testing.T) {
+	ctx := context.Background()
 	data := map[string][]byte{}
 	keyTime := map[string]time.Time{}
 	bm := newTestBlockManager(data, keyTime, nil)
@@ -240,7 +250,7 @@ func TestBlockManagerInternalFlush(t *testing.T) {
 	for i := 0; i < 100; i++ {
 		b := make([]byte, 25)
 		rand.Read(b)
-		writeBlockAndVerify(t, bm, b)
+		writeBlockAndVerify(ctx, t, bm, b)
 	}
 
 	// 1 data block written, but no index yet.
@@ -252,7 +262,7 @@ func TestBlockManagerInternalFlush(t *testing.T) {
 	for i := 0; i < 100; i++ {
 		b := make([]byte, 25)
 		rand.Read(b)
-		writeBlockAndVerify(t, bm, b)
+		writeBlockAndVerify(ctx, t, bm, b)
 	}
 
 	// 2 data blocks written, but no index yet.
@@ -260,7 +270,7 @@ func TestBlockManagerInternalFlush(t *testing.T) {
 		t.Errorf("unexpected number of blocks: %v, wanted %v", got, want)
 	}
 
-	bm.Flush()
+	bm.Flush(ctx)
 
 	// third block gets written, followed by index.
 	if got, want := len(data), 4; got != want {
@@ -271,6 +281,7 @@ func TestBlockManagerInternalFlush(t *testing.T) {
 }
 
 func TestBlockManagerWriteMultiple(t *testing.T) {
+	ctx := context.Background()
 	data := map[string][]byte{}
 	keyTime := map[string]time.Time{}
 	bm := newTestBlockManager(data, keyTime, nil)
@@ -281,7 +292,7 @@ func TestBlockManagerWriteMultiple(t *testing.T) {
 		//t.Logf("i=%v", i)
 		b := seededRandomData(i, i%113)
 		//t.Logf("writing block #%v with %x", i, b)
-		blkID, err := bm.WriteBlock(b, "")
+		blkID, err := bm.WriteBlock(ctx, b, "")
 		//t.Logf("wrote %v=%v", i, blkID)
 		if err != nil {
 			t.Errorf("err: %v", err)
@@ -291,13 +302,13 @@ func TestBlockManagerWriteMultiple(t *testing.T) {
 
 		if i%17 == 0 {
 			t.Logf("flushing %v", i)
-			bm.Flush()
+			bm.Flush(ctx)
 			//dumpBlockManagerData(data)
 		}
 
 		if i%41 == 0 {
 			t.Logf("opening new manager: %v", i)
-			bm.Flush()
+			bm.Flush(ctx)
 			t.Logf("data block count: %v", len(data))
 			//dumpBlockManagerData(data)
 			bm = newTestBlockManager(data, keyTime, nil)
@@ -305,7 +316,7 @@ func TestBlockManagerWriteMultiple(t *testing.T) {
 	}
 
 	for _, blockID := range blockIDs {
-		_, err := bm.GetBlock(blockID)
+		_, err := bm.GetBlock(ctx, blockID)
 		if err != nil {
 			t.Errorf("can't read block %q: %v", blockID, err)
 			continue
@@ -314,63 +325,64 @@ func TestBlockManagerWriteMultiple(t *testing.T) {
 }
 
 func TestBlockManagerConcurrency(t *testing.T) {
+	ctx := context.Background()
 	data := map[string][]byte{}
 	keyTime := map[string]time.Time{}
 	bm := newTestBlockManager(data, keyTime, nil)
-	preexistingBlock := writeBlockAndVerify(t, bm, seededRandomData(10, 100))
-	bm.Flush()
+	preexistingBlock := writeBlockAndVerify(ctx, t, bm, seededRandomData(10, 100))
+	bm.Flush(ctx)
 
 	bm1 := newTestBlockManager(data, keyTime, nil)
 	bm2 := newTestBlockManager(data, keyTime, nil)
 	bm3 := newTestBlockManager(data, keyTime, fakeTimeNowWithAutoAdvance(fakeTime.Add(1), 1))
 
 	// all bm* can see pre-existing block
-	verifyBlock(t, bm1, preexistingBlock, seededRandomData(10, 100))
-	verifyBlock(t, bm2, preexistingBlock, seededRandomData(10, 100))
-	verifyBlock(t, bm3, preexistingBlock, seededRandomData(10, 100))
+	verifyBlock(ctx, t, bm1, preexistingBlock, seededRandomData(10, 100))
+	verifyBlock(ctx, t, bm2, preexistingBlock, seededRandomData(10, 100))
+	verifyBlock(ctx, t, bm3, preexistingBlock, seededRandomData(10, 100))
 
 	// write the same block in all managers.
-	sharedBlock := writeBlockAndVerify(t, bm1, seededRandomData(20, 100))
-	writeBlockAndVerify(t, bm2, seededRandomData(20, 100))
-	writeBlockAndVerify(t, bm3, seededRandomData(20, 100))
+	sharedBlock := writeBlockAndVerify(ctx, t, bm1, seededRandomData(20, 100))
+	writeBlockAndVerify(ctx, t, bm2, seededRandomData(20, 100))
+	writeBlockAndVerify(ctx, t, bm3, seededRandomData(20, 100))
 
 	// write unique block per manager.
-	bm1block := writeBlockAndVerify(t, bm1, seededRandomData(31, 100))
-	bm2block := writeBlockAndVerify(t, bm2, seededRandomData(32, 100))
-	bm3block := writeBlockAndVerify(t, bm3, seededRandomData(33, 100))
+	bm1block := writeBlockAndVerify(ctx, t, bm1, seededRandomData(31, 100))
+	bm2block := writeBlockAndVerify(ctx, t, bm2, seededRandomData(32, 100))
+	bm3block := writeBlockAndVerify(ctx, t, bm3, seededRandomData(33, 100))
 
 	// make sure they can't see each other's unflushed blocks.
-	verifyBlockNotFound(t, bm1, bm2block)
-	verifyBlockNotFound(t, bm1, bm3block)
-	verifyBlockNotFound(t, bm2, bm1block)
-	verifyBlockNotFound(t, bm2, bm3block)
-	verifyBlockNotFound(t, bm3, bm1block)
-	verifyBlockNotFound(t, bm3, bm2block)
+	verifyBlockNotFound(ctx, t, bm1, bm2block)
+	verifyBlockNotFound(ctx, t, bm1, bm3block)
+	verifyBlockNotFound(ctx, t, bm2, bm1block)
+	verifyBlockNotFound(ctx, t, bm2, bm3block)
+	verifyBlockNotFound(ctx, t, bm3, bm1block)
+	verifyBlockNotFound(ctx, t, bm3, bm2block)
 
 	// now flush all writers, they still can't see each others' data.
-	bm1.Flush()
-	bm2.Flush()
-	bm3.Flush()
-	verifyBlockNotFound(t, bm1, bm2block)
-	verifyBlockNotFound(t, bm1, bm3block)
-	verifyBlockNotFound(t, bm2, bm1block)
-	verifyBlockNotFound(t, bm2, bm3block)
-	verifyBlockNotFound(t, bm3, bm1block)
-	verifyBlockNotFound(t, bm3, bm2block)
+	bm1.Flush(ctx)
+	bm2.Flush(ctx)
+	bm3.Flush(ctx)
+	verifyBlockNotFound(ctx, t, bm1, bm2block)
+	verifyBlockNotFound(ctx, t, bm1, bm3block)
+	verifyBlockNotFound(ctx, t, bm2, bm1block)
+	verifyBlockNotFound(ctx, t, bm2, bm3block)
+	verifyBlockNotFound(ctx, t, bm3, bm1block)
+	verifyBlockNotFound(ctx, t, bm3, bm2block)
 
 	// new block manager at this point can see all data.
 	bm4 := newTestBlockManager(data, keyTime, nil)
-	verifyBlock(t, bm4, preexistingBlock, seededRandomData(10, 100))
-	verifyBlock(t, bm4, sharedBlock, seededRandomData(20, 100))
-	verifyBlock(t, bm4, bm1block, seededRandomData(31, 100))
-	verifyBlock(t, bm4, bm2block, seededRandomData(32, 100))
-	verifyBlock(t, bm4, bm3block, seededRandomData(33, 100))
+	verifyBlock(ctx, t, bm4, preexistingBlock, seededRandomData(10, 100))
+	verifyBlock(ctx, t, bm4, sharedBlock, seededRandomData(20, 100))
+	verifyBlock(ctx, t, bm4, bm1block, seededRandomData(31, 100))
+	verifyBlock(ctx, t, bm4, bm2block, seededRandomData(32, 100))
+	verifyBlock(ctx, t, bm4, bm3block, seededRandomData(33, 100))
 
 	if got, want := getIndexCount(data), 4; got != want {
 		t.Errorf("unexpected index count before compaction: %v, wanted %v", got, want)
 	}
 
-	if err := bm4.CompactIndexes(); err != nil {
+	if err := bm4.CompactIndexes(ctx); err != nil {
 		t.Errorf("compaction error: %v", err)
 	}
 	if got, want := getIndexCount(data), 5; got != want {
@@ -379,39 +391,41 @@ func TestBlockManagerConcurrency(t *testing.T) {
 
 	// new block manager at this point can see all data.
 	bm5 := newTestBlockManager(data, keyTime, nil)
-	verifyBlock(t, bm5, preexistingBlock, seededRandomData(10, 100))
-	verifyBlock(t, bm5, sharedBlock, seededRandomData(20, 100))
-	verifyBlock(t, bm5, bm1block, seededRandomData(31, 100))
-	verifyBlock(t, bm5, bm2block, seededRandomData(32, 100))
-	verifyBlock(t, bm5, bm3block, seededRandomData(33, 100))
-	if err := bm5.CompactIndexes(); err != nil {
+	verifyBlock(ctx, t, bm5, preexistingBlock, seededRandomData(10, 100))
+	verifyBlock(ctx, t, bm5, sharedBlock, seededRandomData(20, 100))
+	verifyBlock(ctx, t, bm5, bm1block, seededRandomData(31, 100))
+	verifyBlock(ctx, t, bm5, bm2block, seededRandomData(32, 100))
+	verifyBlock(ctx, t, bm5, bm3block, seededRandomData(33, 100))
+	if err := bm5.CompactIndexes(ctx); err != nil {
 		t.Errorf("compaction error: %v", err)
 	}
 }
 
 func TestDeleteBlock(t *testing.T) {
+	ctx := context.Background()
 	data := map[string][]byte{}
 	keyTime := map[string]time.Time{}
 	bm := newTestBlockManager(data, keyTime, nil)
-	block1 := writeBlockAndVerify(t, bm, seededRandomData(10, 100))
-	bm.Flush()
-	block2 := writeBlockAndVerify(t, bm, seededRandomData(11, 100))
+	block1 := writeBlockAndVerify(ctx, t, bm, seededRandomData(10, 100))
+	bm.Flush(ctx)
+	block2 := writeBlockAndVerify(ctx, t, bm, seededRandomData(11, 100))
 	if err := bm.DeleteBlock(block1); err != nil {
 		t.Errorf("unable to delete block: %v", block1)
 	}
 	if err := bm.DeleteBlock(block2); err != nil {
 		t.Errorf("unable to delete block: %v", block1)
 	}
-	verifyBlockNotFound(t, bm, block1)
-	verifyBlockNotFound(t, bm, block2)
-	bm.Flush()
+	verifyBlockNotFound(ctx, t, bm, block1)
+	verifyBlockNotFound(ctx, t, bm, block2)
+	bm.Flush(ctx)
 	bm = newTestBlockManager(data, keyTime, nil)
 	dumpBlockManagerData(data)
-	verifyBlockNotFound(t, bm, block1)
-	verifyBlockNotFound(t, bm, block2)
+	verifyBlockNotFound(ctx, t, bm, block1)
+	verifyBlockNotFound(ctx, t, bm, block2)
 }
 
 func TestDeleteAndRecreate(t *testing.T) {
+	ctx := context.Background()
 	// simulate race between delete/recreate and delete
 	// delete happens at t0+10, recreate at t0+20 and second delete time is parameterized.
 	// depending on it, the second delete results will be visible.
@@ -430,8 +444,8 @@ func TestDeleteAndRecreate(t *testing.T) {
 			data := map[string][]byte{}
 			keyTime := map[string]time.Time{}
 			bm := newTestBlockManager(data, keyTime, fakeTimeNowFrozen(fakeTime))
-			block1 := writeBlockAndVerify(t, bm, seededRandomData(10, 100))
-			bm.Flush()
+			block1 := writeBlockAndVerify(ctx, t, bm, seededRandomData(10, 100))
+			bm.Flush(ctx)
 
 			// delete but at given timestamp but don't commit yet.
 			bm0 := newTestBlockManager(data, keyTime, fakeTimeNowWithAutoAdvance(tc.deletionTime, 1))
@@ -439,17 +453,17 @@ func TestDeleteAndRecreate(t *testing.T) {
 
 			// delete it at t0+10
 			bm1 := newTestBlockManager(data, keyTime, fakeTimeNowWithAutoAdvance(fakeTime.Add(10), 1))
-			verifyBlock(t, bm1, block1, seededRandomData(10, 100))
+			verifyBlock(ctx, t, bm1, block1, seededRandomData(10, 100))
 			bm1.DeleteBlock(block1)
-			bm1.Flush()
+			bm1.Flush(ctx)
 
 			// recreate at t0+20
 			bm2 := newTestBlockManager(data, keyTime, fakeTimeNowWithAutoAdvance(fakeTime.Add(20), 1))
-			block2 := writeBlockAndVerify(t, bm2, seededRandomData(10, 100))
-			bm2.Flush()
+			block2 := writeBlockAndVerify(ctx, t, bm2, seededRandomData(10, 100))
+			bm2.Flush(ctx)
 
 			// commit deletion from bm0 (t0+5)
-			bm0.Flush()
+			bm0.Flush(ctx)
 
 			dumpBlockManagerData(data)
 
@@ -459,9 +473,9 @@ func TestDeleteAndRecreate(t *testing.T) {
 
 			bm3 := newTestBlockManager(data, keyTime, nil)
 			if tc.isVisible {
-				verifyBlock(t, bm3, block1, seededRandomData(10, 100))
+				verifyBlock(ctx, t, bm3, block1, seededRandomData(10, 100))
 			} else {
-				verifyBlockNotFound(t, bm3, block1)
+				verifyBlockNotFound(ctx, t, bm3, block1)
 			}
 		})
 	}
@@ -473,7 +487,7 @@ func newTestBlockManager(data map[string][]byte, keyTime map[string]time.Time, t
 	if timeFunc == nil {
 		timeFunc = fakeTimeNowWithAutoAdvance(fakeTime, 1)
 	}
-	bm, err := newManagerWithTime(st, FormattingOptions{
+	bm, err := newManagerWithTime(context.Background(), st, FormattingOptions{
 		BlockFormat: "TESTONLY_MD5",
 		MaxPackSize: maxPackSize,
 	}, CachingOptions{}, timeFunc)
@@ -513,19 +527,19 @@ func fakeTimeNowWithAutoAdvance(t time.Time, dt time.Duration) func() time.Time
 	}
 }
 
-func verifyBlockNotFound(t *testing.T, bm *Manager, blockID string) {
+func verifyBlockNotFound(ctx context.Context, t *testing.T, bm *Manager, blockID string) {
 	t.Helper()
 
-	b, err := bm.GetBlock(blockID)
+	b, err := bm.GetBlock(ctx, blockID)
 	if err != storage.ErrBlockNotFound {
 		t.Errorf("unexpected response from GetBlock(%q), got %v,%v, expected %v", blockID, b, err, storage.ErrBlockNotFound)
 	}
 }
 
-func verifyBlock(t *testing.T, bm *Manager, blockID string, b []byte) {
+func verifyBlock(ctx context.Context, t *testing.T, bm *Manager, blockID string, b []byte) {
 	t.Helper()
 
-	b2, err := bm.GetBlock(blockID)
+	b2, err := bm.GetBlock(ctx, blockID)
 	if err != nil {
 		t.Errorf("unable to read block %q: %v", blockID, err)
 		return
@@ -535,7 +549,7 @@ func verifyBlock(t *testing.T, bm *Manager, blockID string, b []byte) {
 		t.Errorf("block %q data mismatch: got %x (nil:%v), wanted %x (nil:%v)", blockID, got, got == nil, want, want == nil)
 	}
 
-	bi, err := bm.BlockInfo(blockID)
+	bi, err := bm.BlockInfo(ctx, blockID)
 	if err != nil {
 		t.Errorf("error getting block info %q: %v", blockID, err)
 	}
@@ -545,10 +559,10 @@ func verifyBlock(t *testing.T, bm *Manager, blockID string, b []byte) {
 	}
 
 }
-func writeBlockAndVerify(t *testing.T, bm *Manager, b []byte) string {
+func writeBlockAndVerify(ctx context.Context, t *testing.T, bm *Manager, b []byte) string {
 	t.Helper()
 
-	blockID, err := bm.WriteBlock(b, "")
+	blockID, err := bm.WriteBlock(ctx, b, "")
 	if err != nil {
 		t.Errorf("err: %v", err)
 	}
@@ -557,7 +571,7 @@ func writeBlockAndVerify(t *testing.T, bm *Manager, b []byte) string {
 		t.Errorf("invalid block ID for %x, got %v, want %v", b, got, want)
 	}
 
-	verifyBlock(t, bm, blockID, b)
+	verifyBlock(ctx, t, bm, blockID, b)
 
 	return blockID
 }

block/local_storage_cache.go

@@ -1,11 +1,14 @@
 package block
 
 import (
+	"bytes"
+	"context"
 	"crypto/hmac"
 	"crypto/sha256"
 	"encoding/json"
 	"errors"
 	"fmt"
+	"io"
 	"sort"
 	"sync"
 	"time"
@@ -34,8 +37,8 @@ type diskBlockCache struct {
 	closed chan struct{}
 }
 
-func (c *diskBlockCache) getBlock(virtualBlockID, physicalBlockID string, offset, length int64) ([]byte, error) {
-	b, err := c.cacheStorage.GetBlock(virtualBlockID, 0, -1)
+func (c *diskBlockCache) getBlock(ctx context.Context, virtualBlockID, physicalBlockID string, offset, length int64) ([]byte, error) {
+	b, err := c.cacheStorage.GetBlock(ctx, virtualBlockID, 0, -1)
 	if err == nil {
 		b, err = c.verifyHMAC(b)
 		if err == nil {
@@ -49,39 +52,40 @@ func (c *diskBlockCache) getBlock(virtualBlockID, physicalBlockID string, offset
 		log.Warn().Msgf("unable to read cache %v: %v", virtualBlockID, err)
 	}
 
-	b, err = c.st.GetBlock(physicalBlockID, offset, length)
+	b, err = c.st.GetBlock(ctx, physicalBlockID, offset, length)
 	if err == storage.ErrBlockNotFound {
 		// not found in underlying storage
 		return nil, err
 	}
 
 	if err == nil {
-		c.writeToCache(virtualBlockID, c.appendHMAC(b))
+		c.writeToCacheBestEffort(ctx, virtualBlockID, b)
 	}
 
 	return b, err
 }
 
-func (c *diskBlockCache) writeToCache(virtualBlockID string, checksummedData []byte) {
-	if err := c.cacheStorage.PutBlock(virtualBlockID, checksummedData); err != nil {
-		log.Warn().Msgf("unable to write cache items %v: %v", virtualBlockID, err)
+func (c *diskBlockCache) writeToCacheBestEffort(ctx context.Context, virtualBlockID string, data []byte) {
+	rdr := io.MultiReader(
+		bytes.NewReader(data),
+		bytes.NewReader(c.computeHMAC(data)),
+	)
+	if err := c.cacheStorage.PutBlock(ctx, virtualBlockID, rdr); err != nil {
+		log.Warn().Msgf("unable to write cache item %v: %v", virtualBlockID, err)
 	}
 }
 
-func (c *diskBlockCache) putBlock(blockID string, data []byte) error {
-	if err := c.st.PutBlock(blockID, data); err != nil {
+func (c *diskBlockCache) putBlock(ctx context.Context, blockID string, data []byte) error {
+	if err := c.st.PutBlock(ctx, blockID, bytes.NewReader(data)); err != nil {
 		return err
 	}
 
-	// now write to cache on a best-effort basis
-	if err := c.cacheStorage.PutBlock(blockID, c.appendHMAC(data)); err != nil {
-		log.Warn().Msgf("unable to write cache item: %v", err)
-	}
-	c.deleteListCache()
+	c.writeToCacheBestEffort(ctx, blockID, data)
+	c.deleteListCache(ctx)
 	return nil
 }
 
-func (c *diskBlockCache) listIndexBlocks(full bool) ([]Info, error) {
+func (c *diskBlockCache) listIndexBlocks(ctx context.Context, full bool) ([]Info, error) {
 	var cachedListBlockID string
 
 	if full {
@@ -90,7 +94,7 @@ func (c *diskBlockCache) listIndexBlocks(full bool) ([]Info, error) {
 		cachedListBlockID = activeListCacheItem
 	}
 
-	ci, err := c.readBlocksFromCacheBlock(cachedListBlockID)
+	ci, err := c.readBlocksFromCacheBlock(ctx, cachedListBlockID)
 	if err == nil {
 		expirationTime := ci.Timestamp.Add(c.listCacheDuration)
 		if time.Now().Before(expirationTime) {
@@ -102,9 +106,9 @@ func (c *diskBlockCache) listIndexBlocks(full bool) ([]Info, error) {
 	}
 
 	log.Debug().Bool("full", full).Msg("listing index blocks from source")
-	blocks, err := listIndexBlocksFromStorage(c.st, full)
+	blocks, err := listIndexBlocksFromStorage(ctx, c.st, full)
 	if err == nil {
-		c.saveListToCache(cachedListBlockID, &cachedList{
+		c.saveListToCache(ctx, cachedListBlockID, &cachedList{
 			Blocks:    blocks,
 			Timestamp: time.Now(),
 		})
@@ -113,18 +117,16 @@ func (c *diskBlockCache) listIndexBlocks(full bool) ([]Info, error) {
 	return blocks, err
 }
 
-func (c *diskBlockCache) saveListToCache(cachedListBlockID string, ci *cachedList) {
+func (c *diskBlockCache) saveListToCache(ctx context.Context, cachedListBlockID string, ci *cachedList) {
 	log.Debug().Str("blockID", cachedListBlockID).Int("count", len(ci.Blocks)).Msg("saving index blocks to cache")
 	if data, err := json.Marshal(ci); err == nil {
-		if err := c.cacheStorage.PutBlock(cachedListBlockID, c.appendHMAC(data)); err != nil {
-			log.Warn().Str("blockID", cachedListBlockID).Err(err).Msg("can't save list")
-		}
+		c.writeToCacheBestEffort(ctx, cachedListBlockID, data)
 	}
 }
 
-func (c *diskBlockCache) readBlocksFromCacheBlock(blockID string) (*cachedList, error) {
+func (c *diskBlockCache) readBlocksFromCacheBlock(ctx context.Context, blockID string) (*cachedList, error) {
 	ci := &cachedList{}
-	data, err := c.cacheStorage.GetBlock(blockID, 0, -1)
+	data, err := c.cacheStorage.GetBlock(ctx, blockID, 0, -1)
 	if err != nil {
 		return nil, err
 	}
@@ -142,11 +144,10 @@ func (c *diskBlockCache) readBlocksFromCacheBlock(blockID string) (*cachedList,
 
 }
 
-func (c *diskBlockCache) appendHMAC(data []byte) []byte {
+func (c *diskBlockCache) computeHMAC(data []byte) []byte {
 	h := hmac.New(sha256.New, c.hmacSecret)
 	h.Write(data) // nolint:errcheck
-	validSignature := h.Sum(nil)
-	return append(append([]byte(nil), data...), validSignature...)
+	return h.Sum(nil)
 }
 
 func (c *diskBlockCache) verifyHMAC(b []byte) ([]byte, error) {
@@ -157,9 +158,8 @@ func (c *diskBlockCache) verifyHMAC(b []byte) ([]byte, error) {
 	p := len(b) - sha256.Size
 	data := b[0:p]
 	signature := b[p:]
-	h := hmac.New(sha256.New, c.hmacSecret)
-	h.Write(data) // nolint:errcheck
-	validSignature := h.Sum(nil)
+
+	validSignature := c.computeHMAC(data)
 	if len(signature) != len(validSignature) {
 		return nil, errors.New("invalid signature length")
 	}
@@ -175,14 +175,14 @@ func (c *diskBlockCache) close() error {
 	return nil
 }
 
-func (c *diskBlockCache) sweepDirectoryPeriodically() {
+func (c *diskBlockCache) sweepDirectoryPeriodically(ctx context.Context) {
 	for {
 		select {
 		case <-c.closed:
 			return
 
 		case <-time.After(sweepCacheFrequency):
-			err := c.sweepDirectory()
+			err := c.sweepDirectory(ctx)
 			if err != nil {
 				log.Printf("warning: blockCache sweep failed: %v", err)
 			}
@@ -190,7 +190,7 @@ func (c *diskBlockCache) sweepDirectoryPeriodically() {
 	}
 }
 
-func (c *diskBlockCache) sweepDirectory() (err error) {
+func (c *diskBlockCache) sweepDirectory(ctx context.Context) (err error) {
 	c.mu.Lock()
 	defer c.mu.Unlock()
 
@@ -201,7 +201,8 @@ func (c *diskBlockCache) sweepDirectory() (err error) {
 	t0 := time.Now()
 	log.Debug().Msg("sweeping cache")
 
-	ch, cancel := c.cacheStorage.ListBlocks("")
+	ctx, cancel := context.WithCancel(ctx)
+	ch := c.cacheStorage.ListBlocks(ctx, "")
 	defer cancel()
 
 	var items []storage.BlockMetadata
@@ -220,7 +221,7 @@ func (c *diskBlockCache) sweepDirectory() (err error) {
 	var totalRetainedSize int64
 	for _, it := range items {
 		if totalRetainedSize > c.maxSizeBytes {
-			if err := c.cacheStorage.DeleteBlock(it.BlockID); err != nil {
+			if err := c.cacheStorage.DeleteBlock(ctx, it.BlockID); err != nil {
 				log.Warn().Msgf("unable to remove %v: %v", it.BlockID, err)
 			}
 		} else {
@@ -232,9 +233,9 @@ func (c *diskBlockCache) sweepDirectory() (err error) {
 	return nil
 }
 
-func (c *diskBlockCache) deleteListCache() {
+func (c *diskBlockCache) deleteListCache(ctx context.Context) {
 	for _, it := range []string{fullListCacheItem, activeListCacheItem} {
-		if err := c.cacheStorage.DeleteBlock(it); err != nil && err != storage.ErrBlockNotFound {
+		if err := c.cacheStorage.DeleteBlock(ctx, it); err != nil && err != storage.ErrBlockNotFound {
 			log.Warn().Err(err).Msg("unable to delete cache item")
 		}
 	}

block/null_block_cache.go

@@ -1,6 +1,9 @@
 package block
 
 import (
+	"bytes"
+	"context"
+
 	"github.com/kopia/kopia/storage"
 )
 
@@ -8,16 +11,16 @@ type nullBlockCache struct {
 	st storage.Storage
 }
 
-func (c nullBlockCache) getBlock(virtualBlockID string, blockID string, offset, length int64) ([]byte, error) {
-	return c.st.GetBlock(blockID, offset, length)
+func (c nullBlockCache) getBlock(ctx context.Context, virtualBlockID string, blockID string, offset, length int64) ([]byte, error) {
+	return c.st.GetBlock(ctx, blockID, offset, length)
 }
 
-func (c nullBlockCache) putBlock(blockID string, data []byte) error {
-	return c.st.PutBlock(blockID, data)
+func (c nullBlockCache) putBlock(ctx context.Context, blockID string, data []byte) error {
+	return c.st.PutBlock(ctx, blockID, bytes.NewReader(data))
 }
 
-func (c nullBlockCache) listIndexBlocks(full bool) ([]Info, error) {
-	return listIndexBlocksFromStorage(c.st, full)
+func (c nullBlockCache) listIndexBlocks(ctx context.Context, full bool) ([]Info, error) {
+	return listIndexBlocksFromStorage(ctx, c.st, full)
 }
 
 func (c nullBlockCache) close() error {

cli/app.go

@@ -1,8 +1,12 @@
 package cli
 
 import (
+	"context"
+	"fmt"
 	"os"
 
+	"github.com/kopia/kopia/repo"
+
 	kingpin "gopkg.in/alecthomas/kingpin.v2"
 )
 
@@ -27,6 +31,24 @@ func helpFullAction(ctx *kingpin.ParseContext) error {
 	return nil
 }
 
+func noRepositoryAction(act func(ctx context.Context) error) func(ctx *kingpin.ParseContext) error {
+	return func(_ *kingpin.ParseContext) error {
+		return act(context.Background())
+	}
+}
+
+func repositoryAction(act func(ctx context.Context, rep *repo.Repository) error) func(ctx *kingpin.ParseContext) error {
+	return func(_ *kingpin.ParseContext) error {
+		ctx := context.Background()
+		rep := mustOpenRepository(ctx, nil)
+		err := act(ctx, rep)
+		if cerr := rep.Close(ctx); cerr != nil {
+			return fmt.Errorf("unable to close repository: %v", cerr)
+		}
+		return err
+	}
+}
+
 // App returns an instance of command-line application object.
 func App() *kingpin.Application {
 	return app

cli/command_block_index_list.go

@@ -1,11 +1,11 @@
 package cli
 
 import (
+	"context"
 	"fmt"
 
 	"github.com/kopia/kopia/block"
-
-	kingpin "gopkg.in/alecthomas/kingpin.v2"
+	"github.com/kopia/kopia/repo"
 )
 
 var (
@@ -14,17 +14,14 @@ var (
 	blockIndexListSummary = blockIndexListCommand.Flag("summary", "Display block summary").Bool()
 )
 
-func runListBlockIndexesAction(context *kingpin.ParseContext) error {
-	rep := mustOpenRepository(nil)
-	defer rep.Close() //nolint: errcheck
-
+func runListBlockIndexesAction(ctx context.Context, rep *repo.Repository) error {
 	var blks []block.Info
 	var err error
 
 	if !*blockIndexListAll {
-		blks, err = rep.Blocks.ActiveIndexBlocks()
+		blks, err = rep.Blocks.ActiveIndexBlocks(ctx)
 	} else {
-		blks, err = rep.Blocks.ListIndexBlocks()
+		blks, err = rep.Blocks.ListIndexBlocks(ctx)
 	}
 
 	if err != nil {
@@ -43,5 +40,5 @@ func runListBlockIndexesAction(context *kingpin.ParseContext) error {
 }
 
 func init() {
-	blockIndexListCommand.Action(runListBlockIndexesAction)
+	blockIndexListCommand.Action(repositoryAction(runListBlockIndexesAction))
 }

cli/command_block_index_optimize.go

@@ -1,20 +1,19 @@
 package cli
 
 import (
-	kingpin "gopkg.in/alecthomas/kingpin.v2"
+	"context"
+
+	"github.com/kopia/kopia/repo"
 )
 
 var (
 	optimizeCommand = blockIndexCommands.Command("optimize", "Optimize block indexes.")
 )
 
-func runOptimizeCommand(context *kingpin.ParseContext) error {
-	rep := mustOpenRepository(nil)
-	defer rep.Close() //nolint: errcheck
-
-	return rep.Blocks.CompactIndexes()
+func runOptimizeCommand(ctx context.Context, rep *repo.Repository) error {
+	return rep.Blocks.CompactIndexes(ctx)
 }
 
 func init() {
-	optimizeCommand.Action(runOptimizeCommand)
+	optimizeCommand.Action(repositoryAction(runOptimizeCommand))
 }

cli/command_block_index_show.go

@@ -1,14 +1,14 @@
 package cli
 
 import (
+	"context"
 	"fmt"
 	"sort"
 	"time"
 
 	"github.com/gogo/protobuf/proto"
 	"github.com/kopia/kopia/internal/blockmgrpb"
-
-	kingpin "gopkg.in/alecthomas/kingpin.v2"
+	"github.com/kopia/kopia/repo"
 )
 
 var (
@@ -24,13 +24,10 @@ type blockIndexEntryInfo struct {
 	inline  bool
 }
 
-func runShowBlockIndexesAction(context *kingpin.ParseContext) error {
-	rep := mustOpenRepository(nil)
-	defer rep.Close() //nolint: errcheck
-
+func runShowBlockIndexesAction(ctx context.Context, rep *repo.Repository) error {
 	blockIDs := *blockIndexShowIDs
 	if len(blockIDs) == 1 && blockIDs[0] == "active" {
-		b, err := rep.Blocks.ActiveIndexBlocks()
+		b, err := rep.Blocks.ActiveIndexBlocks(ctx)
 		if err != nil {
 			return err
 		}
@@ -46,7 +43,7 @@ func runShowBlockIndexesAction(context *kingpin.ParseContext) error {
 	}
 
 	for _, blockID := range blockIDs {
-		data, err := rep.Blocks.GetBlock(blockID)
+		data, err := rep.Blocks.GetBlock(ctx, blockID)
 		if err != nil {
 			return fmt.Errorf("can't read block %q: %v", blockID, err)
 		}
@@ -105,5 +102,5 @@ func sortIndexBlocks(lines []blockIndexEntryInfo) {
 }
 
 func init() {
-	blockIndexShowCommand.Action(runShowBlockIndexesAction)
+	blockIndexShowCommand.Action(repositoryAction(runShowBlockIndexesAction))
 }

cli/command_block_list.go

@@ -1,11 +1,12 @@
 package cli
 
 import (
+	"context"
 	"fmt"
 	"sort"
 
 	"github.com/kopia/kopia/block"
-	kingpin "gopkg.in/alecthomas/kingpin.v2"
+	"github.com/kopia/kopia/repo"
 )
 
 var (
@@ -17,10 +18,7 @@ var (
 	blockListSummary = blockListCommand.Flag("summary", "Summarize the list").Short('s').Bool()
 )
 
-func runListBlocksAction(context *kingpin.ParseContext) error {
-	rep := mustOpenRepository(nil)
-	defer rep.Close() //nolint: errcheck
-
+func runListBlocksAction(ctx context.Context, rep *repo.Repository) error {
 	blocks, err := rep.Blocks.ListBlocks(*blockListPrefix)
 	if err != nil {
 		return err
@@ -83,5 +81,5 @@ func comparePacks(a, b block.Info) bool {
 }
 
 func init() {
-	blockListCommand.Action(runListBlocksAction)
+	blockListCommand.Action(repositoryAction(runListBlocksAction))
 }

cli/command_block_repack.go

@@ -1,7 +1,9 @@
 package cli
 
 import (
-	kingpin "gopkg.in/alecthomas/kingpin.v2"
+	"context"
+
+	"github.com/kopia/kopia/repo"
 )
 
 var (
@@ -9,13 +11,10 @@ var (
 	blockRepackSizeThreshold = blockRepackCommand.Flag("max-size", "Max size of block to re-pack").Default("500000").Uint64()
 )
 
-func runBlockRepackAction(context *kingpin.ParseContext) error {
-	rep := mustOpenRepository(nil)
-	defer rep.Close() //nolint: errcheck
-
-	return rep.Blocks.Repackage(*blockRepackSizeThreshold)
+func runBlockRepackAction(ctx context.Context, rep *repo.Repository) error {
+	return rep.Blocks.Repackage(ctx, *blockRepackSizeThreshold)
 }
 
 func init() {
-	blockRepackCommand.Action(runBlockRepackAction)
+	blockRepackCommand.Action(repositoryAction(runBlockRepackAction))
 }

cli/command_block_rm.go

@@ -1,9 +1,9 @@
 package cli
 
 import (
-	"github.com/kopia/kopia/repo"
+	"context"
 
-	kingpin "gopkg.in/alecthomas/kingpin.v2"
+	"github.com/kopia/kopia/repo"
 )
 
 var (
@@ -12,10 +12,7 @@ var (
 	removeBlockIDs = removeBlockCommand.Arg("id", "IDs of blocks to remove").Required().Strings()
 )
 
-func runRemoveBlockCommand(context *kingpin.ParseContext) error {
-	rep := mustOpenRepository(nil)
-	defer rep.Close() //nolint: errcheck
-
+func runRemoveBlockCommand(ctx context.Context, rep *repo.Repository) error {
 	for _, blockID := range *removeBlockIDs {
 		if err := removeBlock(rep, blockID); err != nil {
 			return err
@@ -31,5 +28,5 @@ func removeBlock(r *repo.Repository, blockID string) error {
 
 func init() {
 	setupShowCommand(removeBlockCommand)
-	removeBlockCommand.Action(runRemoveBlockCommand)
+	removeBlockCommand.Action(repositoryAction(runRemoveBlockCommand))
 }

cli/command_block_show.go

@@ -2,10 +2,9 @@ package cli
 
 import (
 	"bytes"
+	"context"
 
 	"github.com/kopia/kopia/repo"
-
-	kingpin "gopkg.in/alecthomas/kingpin.v2"
 )
 
 var (
@@ -14,12 +13,9 @@ var (
 	showBlockIDs = showBlockCommand.Arg("id", "IDs of blocks to show").Required().Strings()
 )
 
-func runShowBlockCommand(context *kingpin.ParseContext) error {
-	rep := mustOpenRepository(nil)
-	defer rep.Close() //nolint: errcheck
-
+func runShowBlockCommand(ctx context.Context, rep *repo.Repository) error {
 	for _, blockID := range *showBlockIDs {
-		if err := showBlock(rep, blockID); err != nil {
+		if err := showBlock(ctx, rep, blockID); err != nil {
 			return err
 		}
 	}
@@ -27,8 +23,8 @@ func runShowBlockCommand(context *kingpin.ParseContext) error {
 	return nil
 }
 
-func showBlock(r *repo.Repository, blockID string) error {
-	data, err := r.Blocks.GetBlock(blockID)
+func showBlock(ctx context.Context, r *repo.Repository, blockID string) error {
+	data, err := r.Blocks.GetBlock(ctx, blockID)
 	if err != nil {
 		return err
 	}
@@ -38,5 +34,5 @@ func showBlock(r *repo.Repository, blockID string) error {
 
 func init() {
 	setupShowCommand(showBlockCommand)
-	showBlockCommand.Action(runShowBlockCommand)
+	showBlockCommand.Action(repositoryAction(runShowBlockCommand))
 }

cli/command_block_stats.go

@@ -1,13 +1,14 @@
 package cli
 
 import (
+	"context"
 	"fmt"
 	"sort"
 	"strconv"
 
 	"github.com/kopia/kopia/block"
 	"github.com/kopia/kopia/internal/units"
-	kingpin "gopkg.in/alecthomas/kingpin.v2"
+	"github.com/kopia/kopia/repo"
 )
 
 var (
@@ -15,10 +16,7 @@ var (
 	blockStatsRaw     = blockStatsCommand.Flag("raw", "Raw numbers").Short('r').Bool()
 )
 
-func runBlockStatsAction(context *kingpin.ParseContext) error {
-	rep := mustOpenRepository(nil)
-	defer rep.Close() //nolint: errcheck
-
+func runBlockStatsAction(ctx context.Context, rep *repo.Repository) error {
 	blocks, err := rep.Blocks.ListBlocks("")
 	if err != nil {
 		return err
@@ -82,5 +80,5 @@ func percentileSize(p int, blocks []block.Info) int64 {
 }
 
 func init() {
-	blockStatsCommand.Action(runBlockStatsAction)
+	blockStatsCommand.Action(repositoryAction(runBlockStatsAction))
 }

cli/command_cat.go

@@ -1,12 +1,12 @@
 package cli
 
 import (
+	"context"
 	"io"
 	"os"
 
+	"github.com/kopia/kopia/repo"
 	"github.com/kopia/kopia/snapshot"
-
-	kingpin "gopkg.in/alecthomas/kingpin.v2"
 )
 
 var (
@@ -14,17 +14,14 @@ var (
 	catCommandPath = catCommand.Arg("path", "Path").Required().String()
 )
 
-func runCatCommand(context *kingpin.ParseContext) error {
-	rep := mustOpenRepository(nil)
-	defer rep.Close() //nolint: errcheck
-
+func runCatCommand(ctx context.Context, rep *repo.Repository) error {
 	mgr := snapshot.NewManager(rep)
 
-	oid, err := parseObjectID(mgr, *catCommandPath)
+	oid, err := parseObjectID(ctx, mgr, *catCommandPath)
 	if err != nil {
 		return err
 	}
-	r, err := rep.Objects.Open(oid)
+	r, err := rep.Objects.Open(ctx, oid)
 	if err != nil {
 		return err
 	}
@@ -34,5 +31,5 @@ func runCatCommand(context *kingpin.ParseContext) error {
 }
 
 func init() {
-	catCommand.Action(runCatCommand)
+	catCommand.Action(repositoryAction(runCatCommand))
 }

cli/command_ls.go

@@ -1,15 +1,15 @@
 package cli
 
 import (
+	"context"
 	"fmt"
 	"strconv"
 	"strings"
 
 	"github.com/kopia/kopia/fs"
 	"github.com/kopia/kopia/object"
+	"github.com/kopia/kopia/repo"
 	"github.com/kopia/kopia/snapshot"
-
-	kingpin "gopkg.in/alecthomas/kingpin.v2"
 )
 
 const timeFormat = "02 Jan 06 15:04:05"
@@ -24,13 +24,10 @@ var (
 	lsCommandPath      = lsCommand.Arg("path", "Path").Required().String()
 )
 
-func runLSCommand(context *kingpin.ParseContext) error {
-	rep := mustOpenRepository(nil)
-	defer rep.Close() //nolint: errcheck
-
+func runLSCommand(ctx context.Context, rep *repo.Repository) error {
 	mgr := snapshot.NewManager(rep)
 
-	oid, err := parseObjectID(mgr, *lsCommandPath)
+	oid, err := parseObjectID(ctx, mgr, *lsCommandPath)
 	if err != nil {
 		return err
 	}
@@ -43,17 +40,17 @@ func runLSCommand(context *kingpin.ParseContext) error {
 		}
 	}
 
-	return listDirectory(mgr, prefix, oid, "")
+	return listDirectory(ctx, mgr, prefix, oid, "")
 }
 
 func init() {
-	lsCommand.Action(runLSCommand)
+	lsCommand.Action(repositoryAction(runLSCommand))
 }
 
-func listDirectory(mgr *snapshot.Manager, prefix string, oid object.ID, indent string) error {
+func listDirectory(ctx context.Context, mgr *snapshot.Manager, prefix string, oid object.ID, indent string) error {
 	d := mgr.DirectoryEntry(oid)
 
-	entries, err := d.Readdir()
+	entries, err := d.Readdir(ctx)
 	if err != nil {
 		return err
 	}
@@ -91,7 +88,7 @@ func listDirectory(mgr *snapshot.Manager, prefix string, oid object.ID, indent s
 		}
 		fmt.Println(info)
 		if *lsCommandRecursive && m.FileMode().IsDir() {
-			if listerr := listDirectory(mgr, prefix+m.Name+"/", objectID, indent+"  "); listerr != nil {
+			if listerr := listDirectory(ctx, mgr, prefix+m.Name+"/", objectID, indent+"  "); listerr != nil {
 				return listerr
 			}
 		}

cli/command_manifest_ls.go

@@ -1,11 +1,12 @@
 package cli
 
 import (
+	"context"
 	"fmt"
 	"sort"
 	"strings"
 
-	"gopkg.in/alecthomas/kingpin.v2"
+	"github.com/kopia/kopia/repo"
 )
 
 var (
@@ -15,12 +16,10 @@ var (
 )
 
 func init() {
-	manifestListCommand.Action(listManifestItems)
+	manifestListCommand.Action(repositoryAction(listManifestItems))
 }
 
-func listManifestItems(context *kingpin.ParseContext) error {
-	rep := mustOpenRepository(nil)
-
+func listManifestItems(ctx context.Context, rep *repo.Repository) error {
 	filter := map[string]string{}
 
 	for _, kv := range *manifestListFilter {

cli/command_manifest_rm.go

@@ -1,6 +1,10 @@
 package cli
 
-import "gopkg.in/alecthomas/kingpin.v2"
+import (
+	"context"
+
+	"github.com/kopia/kopia/repo"
+)
 
 var (
 	manifestRemoveCommand = manifestCommands.Command("rm", "Remove manifest items")
@@ -8,12 +12,10 @@ var (
 )
 
 func init() {
-	manifestRemoveCommand.Action(removeMetadataItem)
+	manifestRemoveCommand.Action(repositoryAction(removeMetadataItem))
 }
 
-func removeMetadataItem(context *kingpin.ParseContext) error {
-	rep := mustOpenRepository(nil)
-
+func removeMetadataItem(ctx context.Context, rep *repo.Repository) error {
 	for _, it := range *manifestRemoveItems {
 		rep.Manifests.Delete(it)
 	}

cli/command_manifest_show.go

@@ -2,10 +2,11 @@ package cli
 
 import (
 	"bytes"
+	"context"
 	"fmt"
 	"os"
 
-	"gopkg.in/alecthomas/kingpin.v2"
+	"github.com/kopia/kopia/repo"
 )
 
 var (
@@ -14,12 +15,10 @@ var (
 )
 
 func init() {
-	manifestShowCommand.Action(showManifestItems)
+	manifestShowCommand.Action(repositoryAction(showManifestItems))
 }
 
-func showManifestItems(context *kingpin.ParseContext) error {
-	rep := mustOpenRepository(nil)
-
+func showManifestItems(ctx context.Context, rep *repo.Repository) error {
 	for _, it := range *manifestShowItems {
 		md, err := rep.Manifests.GetMetadata(it)
 		if err != nil {

cli/command_mount.go

@@ -1,14 +1,14 @@
 package cli
 
 import (
+	"context"
 	"fmt"
 
 	"github.com/kopia/kopia/fs"
 	"github.com/kopia/kopia/fs/cachefs"
 	"github.com/kopia/kopia/fs/loggingfs"
+	"github.com/kopia/kopia/repo"
 	"github.com/kopia/kopia/snapshot"
-
-	kingpin "gopkg.in/alecthomas/kingpin.v2"
 )
 
 var (
@@ -19,16 +19,14 @@ var (
 	mountTraceFS  = mountCommand.Flag("trace-fs", "Trace filesystem operations").Bool()
 )
 
-func runMountCommand(context *kingpin.ParseContext) error {
-	rep := mustOpenRepository(nil)
-
+func runMountCommand(ctx context.Context, rep *repo.Repository) error {
 	mgr := snapshot.NewManager(rep)
 	var entry fs.Directory
 
 	if *mountObjectID == "all" {
 		entry = mgr.AllSourcesEntry()
 	} else {
-		oid, err := parseObjectID(mgr, *mountObjectID)
+		oid, err := parseObjectID(ctx, mgr, *mountObjectID)
 		if err != nil {
 			return err
 		}
@@ -53,5 +51,5 @@ func runMountCommand(context *kingpin.ParseContext) error {
 
 func init() {
 	setupFSCacheFlags(mountCommand)
-	mountCommand.Action(runMountCommand)
+	mountCommand.Action(repositoryAction(runMountCommand))
 }

cli/command_object_ls.go

@@ -1,9 +1,10 @@
 package cli
 
 import (
+	"context"
 	"fmt"
 
-	kingpin "gopkg.in/alecthomas/kingpin.v2"
+	"github.com/kopia/kopia/repo"
 )
 
 var (
@@ -11,10 +12,7 @@ var (
 	objectListPrefix  = objectListCommand.Flag("prefix", "Prefix").String()
 )
 
-func runListObjectsAction(context *kingpin.ParseContext) error {
-	rep := mustOpenRepository(nil)
-	defer rep.Close() //nolint: errcheck
-
+func runListObjectsAction(ctx context.Context, rep *repo.Repository) error {
 	info, err := rep.Blocks.ListBlocks(*objectListPrefix)
 	if err != nil {
 		return err
@@ -28,5 +26,5 @@ func runListObjectsAction(context *kingpin.ParseContext) error {
 }
 
 func init() {
-	objectListCommand.Action(runListObjectsAction)
+	objectListCommand.Action(repositoryAction(runListObjectsAction))
 }

cli/command_object_show.go

@@ -1,14 +1,13 @@
 package cli
 
 import (
+	"context"
 	"io"
 
 	"github.com/kopia/kopia/object"
 	"github.com/kopia/kopia/snapshot"
 
 	"github.com/kopia/kopia/repo"
-
-	kingpin "gopkg.in/alecthomas/kingpin.v2"
 )
 
 var (
@@ -17,19 +16,16 @@ var (
 	showObjectIDs = showCommand.Arg("id", "IDs of objects to show").Required().Strings()
 )
 
-func runShowCommand(context *kingpin.ParseContext) error {
-	rep := mustOpenRepository(nil)
-	defer rep.Close() //nolint: errcheck
-
+func runShowCommand(ctx context.Context, rep *repo.Repository) error {
 	mgr := snapshot.NewManager(rep)
 
 	for _, oidString := range *showObjectIDs {
-		oid, err := parseObjectID(mgr, oidString)
+		oid, err := parseObjectID(ctx, mgr, oidString)
 		if err != nil {
 			return err
 		}
 
-		if err := showObject(rep, oid); err != nil {
+		if err := showObject(ctx, rep, oid); err != nil {
 			return err
 		}
 	}
@@ -37,10 +33,10 @@ func runShowCommand(context *kingpin.ParseContext) error {
 	return nil
 }
 
-func showObject(r *repo.Repository, oid object.ID) error {
+func showObject(ctx context.Context, r *repo.Repository, oid object.ID) error {
 	var rd io.ReadCloser
 
-	rd, err := r.Objects.Open(oid)
+	rd, err := r.Objects.Open(ctx, oid)
 	if err != nil {
 		return err
 	}
@@ -51,5 +47,5 @@ func showObject(r *repo.Repository, oid object.ID) error {
 
 func init() {
 	setupShowCommand(showCommand)
-	showCommand.Action(runShowCommand)
+	showCommand.Action(repositoryAction(runShowCommand))
 }

cli/command_object_verify.go

@@ -1,6 +1,7 @@
 package cli
 
 import (
+	"context"
 	"fmt"
 	"os"
 
@@ -8,9 +9,8 @@ import (
 
 	"github.com/kopia/kopia/fs"
 	"github.com/kopia/kopia/object"
+	"github.com/kopia/kopia/repo"
 	"github.com/kopia/kopia/snapshot"
-
-	kingpin "gopkg.in/alecthomas/kingpin.v2"
 )
 
 var (
@@ -34,7 +34,7 @@ func (v *verifier) reportError(path string, err error) bool {
 	return len(v.errors) >= *verifyCommandErrorThreshold
 }
 
-func (v *verifier) verifyDirectory(oid object.ID, path string) error {
+func (v *verifier) verifyDirectory(ctx context.Context, oid object.ID, path string) error {
 	if v.visited[oid.String()] {
 		return nil
 	}
@@ -43,7 +43,7 @@ func (v *verifier) verifyDirectory(oid object.ID, path string) error {
 	log.Printf("verifying directory %q (%v)", path, oid)
 
 	d := v.mgr.DirectoryEntry(oid)
-	entries, err := d.Readdir()
+	entries, err := d.Readdir(ctx)
 	if err != nil {
 		if v.reportError(path, fmt.Errorf("error reading directory %q %v: %v", path, oid, err)) {
 			return err
@@ -51,7 +51,7 @@ func (v *verifier) verifyDirectory(oid object.ID, path string) error {
 	}
 
 	for _, e := range entries {
-		if err = v.verifyDirectoryEntry(path, e); err != nil {
+		if err = v.verifyDirectoryEntry(ctx, path, e); err != nil {
 			return err
 		}
 	}
@@ -59,13 +59,13 @@ func (v *verifier) verifyDirectory(oid object.ID, path string) error {
 	return nil
 }
 
-func (v *verifier) verifyDirectoryEntry(path string, e fs.Entry) error {
+func (v *verifier) verifyDirectoryEntry(ctx context.Context, path string, e fs.Entry) error {
 	m := e.Metadata()
 	objectID := e.(object.HasObjectID).ObjectID()
 	childPath := path + "/" + m.Name
 	if m.FileMode().IsDir() {
 		if *verifyCommandRecursive {
-			if err := v.verifyDirectory(objectID, childPath); err != nil {
+			if err := v.verifyDirectory(ctx, objectID, childPath); err != nil {
 				if v.reportError(childPath, err) {
 					return err
 				}
@@ -73,7 +73,7 @@ func (v *verifier) verifyDirectoryEntry(path string, e fs.Entry) error {
 		}
 	}
 
-	if err := v.verifyObject(objectID, childPath, m.FileSize); err != nil {
+	if err := v.verifyObject(ctx, objectID, childPath, m.FileSize); err != nil {
 		if v.reportError(childPath, err) {
 			return err
 		}
@@ -81,7 +81,7 @@ func (v *verifier) verifyDirectoryEntry(path string, e fs.Entry) error {
 	return nil
 }
 
-func (v *verifier) verifyObject(oid object.ID, path string, expectedLength int64) error {
+func (v *verifier) verifyObject(ctx context.Context, oid object.ID, path string, expectedLength int64) error {
 	if v.visited[oid.String()] {
 		return nil
 	}
@@ -93,7 +93,7 @@ func (v *verifier) verifyObject(oid object.ID, path string, expectedLength int64
 		log.Printf("verifying object %v (%v) with length %v", path, oid, expectedLength)
 	}
 
-	length, _, err := v.om.VerifyObject(oid)
+	length, _, err := v.om.VerifyObject(ctx, oid)
 	if err != nil {
 		return fmt.Errorf("invalid object %q: %v", oid, err)
 	}
@@ -107,13 +107,10 @@ func (v *verifier) verifyObject(oid object.ID, path string, expectedLength int64
 	return nil
 }
 
-func runVerifyCommand(context *kingpin.ParseContext) error {
-	rep := mustOpenRepository(nil)
-	defer rep.Close() //nolint: errcheck
-
+func runVerifyCommand(ctx context.Context, rep *repo.Repository) error {
 	mgr := snapshot.NewManager(rep)
 
-	oid, err := parseObjectID(mgr, *verifyCommandPath)
+	oid, err := parseObjectID(ctx, mgr, *verifyCommandPath)
 	if err != nil {
 		return err
 	}
@@ -126,10 +123,10 @@ func runVerifyCommand(context *kingpin.ParseContext) error {
 	}
 
 	if *verifyCommandRecursive {
-		_ = v.verifyDirectory(oid, oid.String())
+		_ = v.verifyDirectory(ctx, oid, oid.String())
 	}
 
-	if err := v.verifyObject(oid, oid.String(), -1); err != nil {
+	if err := v.verifyObject(ctx, oid, oid.String(), -1); err != nil {
 		if v.reportError(".", err) {
 			return err
 		}
@@ -151,5 +148,5 @@ func runVerifyCommand(context *kingpin.ParseContext) error {
 }
 
 func init() {
-	verifyCommand.Action(runVerifyCommand)
+	verifyCommand.Action(repositoryAction(runVerifyCommand))
 }

cli/command_policy_ls.go

@@ -1,10 +1,11 @@
 package cli
 
 import (
+	"context"
 	"fmt"
 
+	"github.com/kopia/kopia/repo"
 	"github.com/kopia/kopia/snapshot"
-	kingpin "gopkg.in/alecthomas/kingpin.v2"
 )
 
 var (
@@ -12,11 +13,10 @@ var (
 )
 
 func init() {
-	policyListCommand.Action(listPolicies)
+	policyListCommand.Action(repositoryAction(listPolicies))
 }
 
-func listPolicies(context *kingpin.ParseContext) error {
-	rep := mustOpenRepository(nil)
+func listPolicies(ctx context.Context, rep *repo.Repository) error {
 	mgr := snapshot.NewPolicyManager(rep)
 
 	policies, err := mgr.ListPolicies()

cli/command_policy_remove.go

@@ -1,10 +1,12 @@
 package cli
 
 import (
+	"context"
+
 	"github.com/rs/zerolog/log"
 
+	"github.com/kopia/kopia/repo"
 	"github.com/kopia/kopia/snapshot"
-	kingpin "gopkg.in/alecthomas/kingpin.v2"
 )
 
 var (
@@ -14,13 +16,10 @@ var (
 )
 
 func init() {
-	policyRemoveCommand.Action(removePolicy)
+	policyRemoveCommand.Action(repositoryAction(removePolicy))
 }
 
-func removePolicy(context *kingpin.ParseContext) error {
-	rep := mustOpenRepository(nil)
-	defer rep.Close() //nolint: errcheck
-
+func removePolicy(ctx context.Context, rep *repo.Repository) error {
 	mgr := snapshot.NewPolicyManager(rep)
 
 	targets, err := policyTargets(policyRemoveGlobal, policyRemoveTargets)

cli/command_policy_set.go

@@ -1,14 +1,15 @@
 package cli
 
 import (
+	"context"
 	"fmt"
 	"sort"
 	"strconv"
 
 	"github.com/rs/zerolog/log"
 
+	"github.com/kopia/kopia/repo"
 	"github.com/kopia/kopia/snapshot"
-	kingpin "gopkg.in/alecthomas/kingpin.v2"
 )
 
 var (
@@ -42,13 +43,10 @@ var (
 )
 
 func init() {
-	policySetCommand.Action(setPolicy)
+	policySetCommand.Action(repositoryAction(setPolicy))
 }
 
-func setPolicy(context *kingpin.ParseContext) error {
-	rep := mustOpenRepository(nil)
-	defer rep.Close() //nolint: errcheck
-
+func setPolicy(ctx context.Context, rep *repo.Repository) error {
 	mgr := snapshot.NewPolicyManager(rep)
 
 	targets, err := policyTargets(policySetGlobal, policySetTargets)

cli/command_policy_show.go

@@ -2,12 +2,13 @@ package cli
 
 import (
 	"bytes"
+	"context"
 	"fmt"
 	"os"
 
 	"github.com/kopia/kopia/internal/units"
+	"github.com/kopia/kopia/repo"
 	"github.com/kopia/kopia/snapshot"
-	kingpin "gopkg.in/alecthomas/kingpin.v2"
 )
 
 var (
@@ -18,11 +19,10 @@ var (
 )
 
 func init() {
-	policyShowCommand.Action(showPolicy)
+	policyShowCommand.Action(repositoryAction(showPolicy))
 }
 
-func showPolicy(context *kingpin.ParseContext) error {
-	rep := mustOpenRepository(nil)
+func showPolicy(ctx context.Context, rep *repo.Repository) error {
 	pmgr := snapshot.NewPolicyManager(rep)
 
 	targets, err := policyTargets(policyShowGlobal, policyShowTargets)

cli/command_repository_cache.go

@@ -1,9 +1,10 @@
 package cli
 
 import (
+	"context"
+
 	"github.com/kopia/kopia/block"
 	"github.com/kopia/kopia/repo"
-	kingpin "gopkg.in/alecthomas/kingpin.v2"
 )
 
 var (
@@ -14,16 +15,16 @@ var (
 	cacheMaxListCacheDuration = cacheCommand.Flag("max-list-cache-duration", "Duration of index cache").Default("600s").Duration()
 )
 
-func runCacheCommand(context *kingpin.ParseContext) error {
+func runCacheCommand(ctx context.Context, rep *repo.Repository) error {
 	opts := block.CachingOptions{
 		CacheDirectory:          *cacheDirectory,
 		MaxCacheSizeBytes:       *cacheMaxCacheSizeMB << 20,
 		MaxListCacheDurationSec: int(cacheMaxListCacheDuration.Seconds()),
 	}
 
-	return repo.SetCachingConfig(getContext(), repositoryConfigFileName(), opts)
+	return repo.SetCachingConfig(ctx, repositoryConfigFileName(), opts)
 }
 
 func init() {
-	cacheCommand.Action(runCacheCommand)
+	cacheCommand.Action(repositoryAction(runCacheCommand))
 }

cli/command_repository_connect.go

@@ -46,13 +46,13 @@ func init() {
 	setupConnectOptions(connectCommand)
 }
 
-func runConnectCommandWithStorage(st storage.Storage) error {
+func runConnectCommandWithStorage(ctx context.Context, st storage.Storage) error {
 	creds, err := getRepositoryCredentials(false)
 	if err != nil {
 		return err
 	}
 
-	err = repo.Connect(context.Background(), repositoryConfigFileName(), st, creds, connectOptions())
+	err = repo.Connect(ctx, repositoryConfigFileName(), st, creds, connectOptions())
 	if err != nil {
 		return err
 	}

cli/command_repository_create.go

@@ -1,6 +1,7 @@
 package cli
 
 import (
+	"context"
 	"fmt"
 	"os"
 
@@ -42,8 +43,9 @@ func newRepositoryOptionsFromFlags() *repo.NewRepositoryOptions {
 	}
 }
 
-func ensureEmpty(s storage.Storage) error {
-	ch, cancel := s.ListBlocks("")
+func ensureEmpty(ctx context.Context, s storage.Storage) error {
+	ctx, cancel := context.WithCancel(ctx)
+	ch := s.ListBlocks(ctx, "")
 	d, hasData := <-ch
 	cancel()
 
@@ -60,8 +62,8 @@ func ensureEmpty(s storage.Storage) error {
 	return nil
 }
 
-func runCreateCommandWithStorage(st storage.Storage) error {
-	err := ensureEmpty(st)
+func runCreateCommandWithStorage(ctx context.Context, st storage.Storage) error {
+	err := ensureEmpty(ctx, st)
 	if err != nil {
 		return fmt.Errorf("unable to get repository storage: %v", err)
 	}
@@ -90,12 +92,12 @@ func runCreateCommandWithStorage(st storage.Storage) error {
 		fmt.Fprintf(os.Stderr, "  object splitter:     NEVER\n")
 	}
 
-	if err := repo.Initialize(st, options, creds); err != nil {
+	if err := repo.Initialize(ctx, st, options, creds); err != nil {
 		return fmt.Errorf("cannot initialize repository: %v", err)
 	}
 
 	if !*createOnly {
-		err := repo.Connect(getContext(), repositoryConfigFileName(), st, creds, connectOptions())
+		err := repo.Connect(ctx, repositoryConfigFileName(), st, creds, connectOptions())
 		if err != nil {
 			return err
 		}

cli/command_repository_disconnect.go

@@ -1,8 +1,9 @@
 package cli
 
 import (
+	"context"
+
 	"github.com/kopia/kopia/repo"
-	"gopkg.in/alecthomas/kingpin.v2"
 )
 
 var (
@@ -10,9 +11,9 @@ var (
 )
 
 func init() {
-	disconnectCommand.Action(runDisconnectCommand)
+	disconnectCommand.Action(noRepositoryAction(runDisconnectCommand))
 }
 
-func runDisconnectCommand(context *kingpin.ParseContext) error {
+func runDisconnectCommand(ctx context.Context) error {
 	return repo.Disconnect(repositoryConfigFileName())
 }

cli/command_repository_migrate.go

@@ -1,6 +1,7 @@
 package cli
 
 import (
+	"context"
 	"fmt"
 
 	"github.com/rs/zerolog/log"
@@ -8,7 +9,6 @@ import (
 	"github.com/kopia/kopia/object"
 	"github.com/kopia/kopia/repo"
 	"github.com/kopia/kopia/snapshot"
-	kingpin "gopkg.in/alecthomas/kingpin.v2"
 )
 
 var (
@@ -21,8 +21,7 @@ var (
 	migrateIgnoreErrors = migrateCommand.Flag("ignore-errors", "Ignore errors when reading source backup").Bool()
 )
 
-func runMigrateCommand(context *kingpin.ParseContext) error {
-	destRepo := mustOpenRepository(nil)
+func runMigrateCommand(ctx context.Context, destRepo *repo.Repository) error {
 	destSM := snapshot.NewManager(destRepo)
 
 	uploader := snapshot.NewUploader(destRepo)
@@ -30,7 +29,7 @@ func runMigrateCommand(context *kingpin.ParseContext) error {
 	uploader.IgnoreFileErrors = *migrateIgnoreErrors
 	onCtrlC(uploader.Cancel)
 
-	sourceRepo, err := repo.Open(getContext(), *migrateSourceConfig, applyOptionsFromFlags(nil))
+	sourceRepo, err := repo.Open(ctx, *migrateSourceConfig, applyOptionsFromFlags(nil))
 	if err != nil {
 		return fmt.Errorf("can't open source repository: %v", err)
 	}
@@ -47,7 +46,7 @@ func runMigrateCommand(context *kingpin.ParseContext) error {
 			break
 		}
 
-		if err := migrateSingleSource(uploader, sourceSM, destSM, s); err != nil {
+		if err := migrateSingleSource(ctx, uploader, sourceSM, destSM, s); err != nil {
 			return err
 		}
 	}
@@ -58,7 +57,7 @@ func runMigrateCommand(context *kingpin.ParseContext) error {
 			return err
 		}
 		d := sourceSM.DirectoryEntry(dirOID)
-		newm, err := uploader.Upload(d, snapshot.SourceInfo{Host: "temp"}, nil)
+		newm, err := uploader.Upload(ctx, d, snapshot.SourceInfo{Host: "temp"}, nil)
 		if err != nil {
 			return fmt.Errorf("error migrating directory %v: %v", dirOID, err)
 		}
@@ -69,7 +68,7 @@ func runMigrateCommand(context *kingpin.ParseContext) error {
 	return nil
 }
 
-func migrateSingleSource(uploader *snapshot.Uploader, sourceSM, destSM *snapshot.Manager, s snapshot.SourceInfo) error {
+func migrateSingleSource(ctx context.Context, uploader *snapshot.Uploader, sourceSM, destSM *snapshot.Manager, s snapshot.SourceInfo) error {
 	log.Printf("migrating source %v", s)
 
 	manifests := sourceSM.ListSnapshotManifests(&s)
@@ -80,7 +79,7 @@ func migrateSingleSource(uploader *snapshot.Uploader, sourceSM, destSM *snapshot
 
 	for _, m := range filterSnapshotsToMigrate(snapshots) {
 		d := sourceSM.DirectoryEntry(m.RootObjectID)
-		newm, err := uploader.Upload(d, m.Source, nil)
+		newm, err := uploader.Upload(ctx, d, m.Source, nil)
 		if err != nil {
 			return fmt.Errorf("error migrating shapshot %v @ %v: %v", m.Source, m.StartTime, err)
 		}
@@ -129,5 +128,5 @@ func getSourcesToMigrate(mgr *snapshot.Manager) ([]snapshot.SourceInfo, error) {
 }
 
 func init() {
-	migrateCommand.Action(runMigrateCommand)
+	migrateCommand.Action(repositoryAction(runMigrateCommand))
 }

cli/command_repository_status.go

@@ -1,6 +1,7 @@
 package cli
 
 import (
+	"context"
 	"encoding/json"
 	"fmt"
 	"io/ioutil"
@@ -8,17 +9,14 @@ import (
 
 	"github.com/kopia/kopia/internal/scrubber"
 	"github.com/kopia/kopia/internal/units"
-	kingpin "gopkg.in/alecthomas/kingpin.v2"
+	"github.com/kopia/kopia/repo"
 )
 
 var (
 	statusCommand = repositoryCommands.Command("status", "Display the status of connected repository.")
 )
 
-func runStatusCommand(context *kingpin.ParseContext) error {
-	rep := mustOpenRepository(nil)
-	defer rep.Close() //nolint: errcheck
-
+func runStatusCommand(ctx context.Context, rep *repo.Repository) error {
 	fmt.Printf("Config file:         %v\n", rep.ConfigFile)
 	entries, err := ioutil.ReadDir(rep.CacheDirectory)
 	if err != nil {
@@ -67,5 +65,5 @@ func runStatusCommand(context *kingpin.ParseContext) error {
 }
 
 func init() {
-	statusCommand.Action(runStatusCommand)
+	statusCommand.Action(repositoryAction(runStatusCommand))
 }

cli/command_snapshot_create.go

@@ -1,6 +1,7 @@
 package cli
 
 import (
+	"context"
 	"encoding/json"
 	"errors"
 	"fmt"
@@ -13,11 +14,8 @@ import (
 
 	"github.com/rs/zerolog/log"
 
-	"github.com/kopia/kopia/object"
 	"github.com/kopia/kopia/repo"
 	"github.com/kopia/kopia/snapshot"
-
-	kingpin "gopkg.in/alecthomas/kingpin.v2"
 )
 
 const (
@@ -33,18 +31,10 @@ var (
 	snapshotCreateDescription             = snapshotCreateCommand.Flag("description", "Free-form snapshot description.").String()
 	snapshotCreateForceHash               = snapshotCreateCommand.Flag("force-hash", "Force hashing of source files for a given percentage of files [0..100]").Default("0").Int()
 	snapshotCreateHashCacheMinAge         = snapshotCreateCommand.Flag("hash-cache-min-age", "Do not hash-cache files below certain age").Default("1h").Duration()
-	snapshotCreateWriteBack               = snapshotCreateCommand.Flag("async-write", "Perform updates asynchronously.").PlaceHolder("N").Default("0").Int()
 	snapshotCreateParallelUploads         = snapshotCreateCommand.Flag("parallel", "Upload N files in parallel").PlaceHolder("N").Default("0").Int()
 )
 
-func runBackupCommand(c *kingpin.ParseContext) error {
-	rep := mustOpenRepository(&repo.Options{
-		ObjectManagerOptions: object.ManagerOptions{
-			WriteBack: *snapshotCreateWriteBack,
-		},
-	})
-	defer rep.Close() //nolint: errcheck
-
+func runBackupCommand(ctx context.Context, rep *repo.Repository) error {
 	mgr := snapshot.NewManager(rep)
 	pmgr := snapshot.NewPolicyManager(rep)
 
@@ -83,7 +73,7 @@ func runBackupCommand(c *kingpin.ParseContext) error {
 
 		sourceInfo := snapshot.SourceInfo{Path: filepath.Clean(dir), Host: getHostName(), UserName: getUserName()}
 		log.Info().Str("source", sourceInfo.String()).Msg("snapshotting")
-		if err := snapshotSingleSource(rep, mgr, pmgr, u, sourceInfo); err != nil {
+		if err := snapshotSingleSource(ctx, rep, mgr, pmgr, u, sourceInfo); err != nil {
 			return err
 		}
 	}
@@ -91,7 +81,7 @@ func runBackupCommand(c *kingpin.ParseContext) error {
 	return nil
 }
 
-func snapshotSingleSource(rep *repo.Repository, mgr *snapshot.Manager, pmgr *snapshot.PolicyManager, u *snapshot.Uploader, sourceInfo snapshot.SourceInfo) error {
+func snapshotSingleSource(ctx context.Context, rep *repo.Repository, mgr *snapshot.Manager, pmgr *snapshot.PolicyManager, u *snapshot.Uploader, sourceInfo snapshot.SourceInfo) error {
 	t0 := time.Now()
 	rep.Blocks.ResetStats()
 	policy, err := pmgr.GetEffectivePolicy(sourceInfo)
@@ -112,7 +102,7 @@ func snapshotSingleSource(rep *repo.Repository, mgr *snapshot.Manager, pmgr *sna
 	}
 
 	log.Debug().Msgf("uploading %v using previous manifest %v", sourceInfo, previousManifest)
-	manifest, err := u.Upload(localEntry, sourceInfo, previousManifest)
+	manifest, err := u.Upload(ctx, localEntry, sourceInfo, previousManifest)
 	if err != nil {
 		return err
 	}
@@ -205,5 +195,5 @@ func getHostName() string {
 }
 
 func init() {
-	snapshotCreateCommand.Action(runBackupCommand)
+	snapshotCreateCommand.Action(repositoryAction(runBackupCommand))
 }

cli/command_snapshot_estimate.go

@@ -1,17 +1,18 @@
 package cli
 
 import (
+	"context"
 	"fmt"
 	"path/filepath"
 	"time"
 
+	"github.com/kopia/kopia/repo"
+
 	"github.com/rs/zerolog/log"
 
 	"github.com/kopia/kopia/fs"
 	"github.com/kopia/kopia/internal/units"
 	"github.com/kopia/kopia/snapshot"
-
-	kingpin "gopkg.in/alecthomas/kingpin.v2"
 )
 
 var (
@@ -59,10 +60,7 @@ func makeBuckets() buckets {
 	}
 }
 
-func runSnapshotEstimateCommand(c *kingpin.ParseContext) error {
-	rep := mustOpenRepository(nil)
-	defer rep.Close() //nolint: errcheck
-
+func runSnapshotEstimateCommand(ctx context.Context, rep *repo.Repository) error {
 	pmgr := snapshot.NewPolicyManager(rep)
 
 	path, err := filepath.Abs(*snapshotEstimateSource)
@@ -79,7 +77,7 @@ func runSnapshotEstimateCommand(c *kingpin.ParseContext) error {
 	var stats snapshot.Stats
 	ib := makeBuckets()
 	eb := makeBuckets()
-	if err := estimate(".", mustGetLocalFSEntry(path), &policy.FilesPolicy, &stats, ib, eb); err != nil {
+	if err := estimate(ctx, ".", mustGetLocalFSEntry(path), &policy.FilesPolicy, &stats, ib, eb); err != nil {
 		return err
 	}
 
@@ -112,7 +110,7 @@ func showBuckets(b buckets) {
 		}
 	}
 }
-func estimate(relativePath string, entry fs.Entry, pol *snapshot.FilesPolicy, stats *snapshot.Stats, ib, eb buckets) error {
+func estimate(ctx context.Context, relativePath string, entry fs.Entry, pol *snapshot.FilesPolicy, stats *snapshot.Stats, ib, eb buckets) error {
 	if !pol.ShouldInclude(entry.Metadata()) {
 		eb.add(relativePath, entry.Metadata().FileSize)
 		stats.ExcludedFileCount++
@@ -125,13 +123,13 @@ func estimate(relativePath string, entry fs.Entry, pol *snapshot.FilesPolicy, st
 		if !*snapshotEstimateQuiet {
 			log.Printf("Scanning %q...\n", relativePath)
 		}
-		children, err := entry.Readdir()
+		children, err := entry.Readdir(ctx)
 		if err != nil {
 			return err
 		}
 
 		for _, child := range children {
-			if err := estimate(filepath.Join(relativePath, child.Metadata().Name), child, pol, stats, ib, eb); err != nil {
+			if err := estimate(ctx, filepath.Join(relativePath, child.Metadata().Name), child, pol, stats, ib, eb); err != nil {
 				return err
 			}
 		}
@@ -145,5 +143,5 @@ func estimate(relativePath string, entry fs.Entry, pol *snapshot.FilesPolicy, st
 }
 
 func init() {
-	snapshotEstimate.Action(runSnapshotEstimateCommand)
+	snapshotEstimate.Action(repositoryAction(runSnapshotEstimateCommand))
 }

cli/command_snapshot_expire.go

@@ -1,6 +1,7 @@
 package cli
 
 import (
+	"context"
 	"fmt"
 	"os"
 	"sort"
@@ -9,9 +10,8 @@ import (
 
 	"github.com/rs/zerolog/log"
 
+	"github.com/kopia/kopia/repo"
 	"github.com/kopia/kopia/snapshot"
-
-	kingpin "gopkg.in/alecthomas/kingpin.v2"
 )
 
 var (
@@ -226,10 +226,7 @@ func expireSnapshots(pmgr *snapshot.PolicyManager, snapshots []*snapshot.Manifes
 	return toDelete, nil
 }
 
-func runExpireCommand(context *kingpin.ParseContext) error {
-	rep := mustOpenRepository(nil)
-	defer rep.Close() //nolint: errcheck
-
+func runExpireCommand(ctx context.Context, rep *repo.Repository) error {
 	mgr := snapshot.NewManager(rep)
 	pmgr := snapshot.NewPolicyManager(rep)
 	snapshotNames, err := getSnapshotNamesToExpire(mgr)
@@ -288,5 +285,5 @@ func filterHostAndUser(snapshots []*snapshot.Manifest) []*snapshot.Manifest {
 }
 
 func init() {
-	snapshotExpireCommand.Action(runExpireCommand)
+	snapshotExpireCommand.Action(repositoryAction(runExpireCommand))
 }

cli/command_snapshot_list.go

@@ -1,6 +1,7 @@
 package cli
 
 import (
+	"context"
 	"fmt"
 	"path/filepath"
 	"sort"
@@ -9,9 +10,8 @@ import (
 	"github.com/rs/zerolog/log"
 
 	"github.com/kopia/kopia/internal/units"
+	"github.com/kopia/kopia/repo"
 	"github.com/kopia/kopia/snapshot"
-
-	kingpin "gopkg.in/alecthomas/kingpin.v2"
 )
 
 var (
@@ -67,10 +67,7 @@ func findManifestIDs(mgr *snapshot.Manager, source string) ([]string, string, er
 	return manifestIDs, relPath, err
 }
 
-func runBackupsCommand(context *kingpin.ParseContext) error {
-	rep := mustOpenRepository(nil)
-	defer rep.Close() //nolint: errcheck
-
+func runBackupsCommand(ctx context.Context, rep *repo.Repository) error {
 	mgr := snapshot.NewManager(rep)
 
 	manifestIDs, relPath, err := findManifestIDs(mgr, *snapshotListPath)
@@ -160,5 +157,5 @@ func deltaBytes(b int64) string {
 }
 
 func init() {
-	snapshotListCommand.Action(runBackupsCommand)
+	snapshotListCommand.Action(repositoryAction(runBackupsCommand))
 }

cli/command_storage_list.go

@@ -1,9 +1,10 @@
 package cli
 
 import (
+	"context"
 	"fmt"
 
-	kingpin "gopkg.in/alecthomas/kingpin.v2"
+	"github.com/kopia/kopia/repo"
 )
 
 var (
@@ -11,12 +12,10 @@ var (
 	storageListPrefix  = storageListCommand.Flag("prefix", "Block prefix").String()
 )
 
-func runListStorageBlocks(context *kingpin.ParseContext) error {
-	rep := mustOpenRepository(nil)
-	defer rep.Close() //nolint: errcheck
-
-	ch, cancel := rep.Storage.ListBlocks(*storageListPrefix)
+func runListStorageBlocks(ctx context.Context, rep *repo.Repository) error {
+	ctx, cancel := context.WithCancel(ctx)
 	defer cancel()
+	ch := rep.Storage.ListBlocks(ctx, *storageListPrefix)
 
 	for b := range ch {
 		if b.Error != nil {
@@ -30,5 +29,5 @@ func runListStorageBlocks(context *kingpin.ParseContext) error {
 }
 
 func init() {
-	storageListCommand.Action(runListStorageBlocks)
+	storageListCommand.Action(repositoryAction(runListStorageBlocks))
 }

cli/command_storage_show.go

@@ -2,11 +2,12 @@ package cli
 
 import (
 	"bytes"
+	"context"
 	"fmt"
 	"io"
 	"os"
 
-	kingpin "gopkg.in/alecthomas/kingpin.v2"
+	"github.com/kopia/kopia/repo"
 )
 
 var (
@@ -14,12 +15,9 @@ var (
 	storageShowBlockIDs = storageShowCommand.Arg("blockIDs", "Block IDs").Required().Strings()
 )
 
-func runShowStorageBlocks(context *kingpin.ParseContext) error {
-	rep := mustOpenRepository(nil)
-	defer rep.Close() //nolint: errcheck
-
+func runShowStorageBlocks(ctx context.Context, rep *repo.Repository) error {
 	for _, b := range *storageShowBlockIDs {
-		d, err := rep.Storage.GetBlock(b, 0, -1)
+		d, err := rep.Storage.GetBlock(ctx, b, 0, -1)
 		if err != nil {
 			return fmt.Errorf("error getting %v: %v", b, err)
 		}
@@ -32,5 +30,5 @@ func runShowStorageBlocks(context *kingpin.ParseContext) error {
 }
 
 func init() {
-	storageShowCommand.Action(runShowStorageBlocks)
+	storageShowCommand.Action(repositoryAction(runShowStorageBlocks))
 }

cli/command_storage_sweep.go

@@ -1,13 +1,12 @@
 package cli
 
 import (
+	"context"
 	"fmt"
 	"time"
 
 	"github.com/kopia/kopia/repo"
 	"github.com/kopia/kopia/storage"
-
-	kingpin "gopkg.in/alecthomas/kingpin.v2"
 )
 
 var (
@@ -16,11 +15,8 @@ var (
 	storageSweepPrintUnused  = storageSweepCommand.Flag("print", "Do not delete blocks, but print which ones would be deleted").Short('n').Bool()
 )
 
-func runStorageSweepAction(context *kingpin.ParseContext) error {
-	rep := mustOpenRepository(nil)
-	defer rep.Close() //nolint: errcheck
-
-	indexBlocks, err := rep.Blocks.ActiveIndexBlocks()
+func runStorageSweepAction(ctx context.Context, rep *repo.Repository) error {
+	indexBlocks, err := rep.Blocks.ActiveIndexBlocks(ctx)
 	if err != nil {
 		return err
 	}
@@ -30,15 +26,16 @@ func runStorageSweepAction(context *kingpin.ParseContext) error {
 		inUseIndexBlocks[ib.BlockID] = true
 	}
 
-	ch, cancel := rep.Storage.ListBlocks("")
+	ctx, cancel := context.WithCancel(ctx)
 	defer cancel()
+	ch := rep.Storage.ListBlocks(ctx, "")
 
 	for bm := range ch {
 		if bm.Error != nil {
 			return bm.Error
 		}
 
-		if err = sweepBlock(rep, bm, inUseIndexBlocks); err != nil {
+		if err = sweepBlock(ctx, rep, bm, inUseIndexBlocks); err != nil {
 			return err
 		}
 	}
@@ -46,7 +43,7 @@ func runStorageSweepAction(context *kingpin.ParseContext) error {
 	return nil
 }
 
-func sweepBlock(rep *repo.Repository, bm storage.BlockMetadata, inUseIndexBlocks map[string]bool) error {
+func sweepBlock(ctx context.Context, rep *repo.Repository, bm storage.BlockMetadata, inUseIndexBlocks map[string]bool) error {
 	age := time.Since(bm.TimeStamp)
 	inUse := rep.Blocks.IsStorageBlockInUse(bm.BlockID) || inUseIndexBlocks[bm.BlockID] || bm.BlockID == repo.FormatBlockID
 	keep := inUse || age < *storageSweepAgeThreshold
@@ -60,9 +57,9 @@ func sweepBlock(rep *repo.Repository, bm storage.BlockMetadata, inUseIndexBlocks
 	}
 
 	fmt.Printf("deleting unused block %v age %v\n", bm.BlockID, age)
-	return rep.Storage.DeleteBlock(bm.BlockID)
+	return rep.Storage.DeleteBlock(ctx, bm.BlockID)
 }
 
 func init() {
-	storageSweepCommand.Action(runStorageSweepAction)
+	storageSweepCommand.Action(repositoryAction(runStorageSweepAction))
 }

cli/config.go

@@ -63,12 +63,8 @@ func waitForCtrlC() {
 	<-done
 }
 
-func getContext() context.Context {
-	return context.Background()
-}
-
-func openRepository(opts *repo.Options) (*repo.Repository, error) {
-	r, err := repo.Open(getContext(), repositoryConfigFileName(), applyOptionsFromFlags(opts))
+func openRepository(ctx context.Context, opts *repo.Options) (*repo.Repository, error) {
+	r, err := repo.Open(ctx, repositoryConfigFileName(), applyOptionsFromFlags(opts))
 	if os.IsNotExist(err) {
 		return nil, fmt.Errorf("not connected to a repository, use 'kopia connect'")
 	}
@@ -96,8 +92,8 @@ func applyOptionsFromFlags(opts *repo.Options) *repo.Options {
 	return opts
 }
 
-func mustOpenRepository(opts *repo.Options) *repo.Repository {
-	s, err := openRepository(opts)
+func mustOpenRepository(ctx context.Context, opts *repo.Options) *repo.Repository {
+	s, err := openRepository(ctx, opts)
 	failOnError(err)
 	return s
 }

cli/objref.go

@@ -1,6 +1,7 @@
 package cli
 
 import (
+	"context"
 	"fmt"
 	"strings"
 
@@ -10,7 +11,7 @@ import (
 )
 
 // ParseObjectID interprets the given ID string and returns corresponding object.ID.
-func parseObjectID(mgr *snapshot.Manager, id string) (object.ID, error) {
+func parseObjectID(ctx context.Context, mgr *snapshot.Manager, id string) (object.ID, error) {
 	head, tail := splitHeadTail(id)
 	if len(head) == 0 {
 		return object.NullID, fmt.Errorf("invalid object ID: %v", id)
@@ -30,11 +31,11 @@ func parseObjectID(mgr *snapshot.Manager, id string) (object.ID, error) {
 		return object.NullID, err
 	}
 
-	return parseNestedObjectID(dir, tail)
+	return parseNestedObjectID(ctx, dir, tail)
 }
 
 //nolint:interfacer
-func parseNestedObjectID(startingDir fs.Directory, id string) (object.ID, error) {
+func parseNestedObjectID(ctx context.Context, startingDir fs.Directory, id string) (object.ID, error) {
 	head, tail := splitHeadTail(id)
 	var current fs.Entry = startingDir
 	for head != "" {
@@ -43,7 +44,7 @@ func parseNestedObjectID(startingDir fs.Directory, id string) (object.ID, error)
 			return object.NullID, fmt.Errorf("entry not found '%v': parent is not a directory", head)
 		}
 
-		entries, err := dir.Readdir()
+		entries, err := dir.Readdir(ctx)
 		if err != nil {
 			return object.NullID, err
 		}

cli/storage_providers.go

@@ -19,24 +19,26 @@ func RegisterStorageConnectFlags(
 	// Set up 'create' subcommand
 	cc := createCommand.Command(name, "Create repository in "+description)
 	flags(cc)
-	cc.Action(func(ctx *kingpin.ParseContext) error {
-		st, err := connect(getContext())
+	cc.Action(func(_ *kingpin.ParseContext) error {
+		ctx := context.Background()
+		st, err := connect(ctx)
 		if err != nil {
 			return fmt.Errorf("can't connect to storage: %v", err)
 		}
 
-		return runCreateCommandWithStorage(st)
+		return runCreateCommandWithStorage(ctx, st)
 	})
 
 	// Set up 'connect' subcommand
 	cc = connectCommand.Command(name, "Connect to repository in "+description)
 	flags(cc)
-	cc.Action(func(ctx *kingpin.ParseContext) error {
-		st, err := connect(getContext())
+	cc.Action(func(_ *kingpin.ParseContext) error {
+		ctx := context.Background()
+		st, err := connect(ctx)
 		if err != nil {
 			return fmt.Errorf("can't connect to storage: %v", err)
 		}
 
-		return runConnectCommandWithStorage(st)
+		return runConnectCommandWithStorage(ctx, st)
 	})
 }

fs/cachefs/cache.go

@@ -1,6 +1,7 @@
 package cachefs
 
 import (
+	"context"
 	"sync"
 	"time"
 
@@ -73,18 +74,18 @@ func (c *Cache) remove(e *cacheEntry) {
 }
 
 // Loader provides data to be stored in the cache.
-type Loader func() (fs.Entries, error)
+type Loader func(ctx context.Context) (fs.Entries, error)
 
 // Readdir reads the contents of a provided directory using ObjectID of a directory (if any) to cache
 // the results.
-func (c *Cache) Readdir(d fs.Directory) (fs.Entries, error) {
+func (c *Cache) Readdir(ctx context.Context, d fs.Directory) (fs.Entries, error) {
 	if h, ok := d.(object.HasObjectID); ok {
 		cacheID := h.ObjectID().String()
 		cacheExpiration := 24 * time.Hour
-		return c.getEntries(cacheID, cacheExpiration, d.Readdir)
+		return c.getEntries(ctx, cacheID, cacheExpiration, d.Readdir)
 	}
 
-	return d.Readdir()
+	return d.Readdir(ctx)
 }
 
 func (c *Cache) getEntriesFromCache(id string) fs.Entries {
@@ -110,9 +111,9 @@ func (c *Cache) getEntriesFromCache(id string) fs.Entries {
 
 // getEntries consults the cache and either retrieves the contents of directory listing from the cache
 // or invokes the provides callback and adds the results to cache.
-func (c *Cache) getEntries(id string, expirationTime time.Duration, cb Loader) (fs.Entries, error) {
+func (c *Cache) getEntries(ctx context.Context, id string, expirationTime time.Duration, cb Loader) (fs.Entries, error) {
 	if c == nil {
-		return cb()
+		return cb(ctx)
 	}
 
 	c.mu.Lock()
@@ -123,7 +124,7 @@ func (c *Cache) getEntries(id string, expirationTime time.Duration, cb Loader) (
 	if c.debug {
 		log.Printf("cache miss for %q", id)
 	}
-	raw, err := cb()
+	raw, err := cb(ctx)
 	if err != nil {
 		return nil, err
 	}

fs/cachefs/cache_test.go

@@ -1,6 +1,7 @@
 package cachefs
 
 import (
+	"context"
 	"errors"
 	"fmt"
 	"path/filepath"
@@ -19,8 +20,8 @@ type cacheSource struct {
 	callCounter map[string]int
 }
 
-func (cs *cacheSource) get(id string) func() (fs.Entries, error) {
-	return func() (fs.Entries, error) {
+func (cs *cacheSource) get(id string) func(ctx context.Context) (fs.Entries, error) {
+	return func(context.Context) (fs.Entries, error) {
 		cs.callCounter[id]++
 		d, ok := cs.data[id]
 		if !ok {
@@ -119,6 +120,7 @@ func (cv *cacheVerifier) reset() {
 }
 
 func TestCache(t *testing.T) {
+	ctx := context.Background()
 	c := NewCache(&Options{
 		MaxCachedDirectories: 4,
 		MaxCachedEntries:     100,
@@ -147,57 +149,57 @@ func TestCache(t *testing.T) {
 	cv.verifyCacheOrdering(t)
 
 	// fetch id1
-	c.getEntries(id1, expirationTime, cs.get(id1))
+	c.getEntries(ctx, id1, expirationTime, cs.get(id1))
 	cv.verifyCacheMiss(t, id1)
 	cv.verifyCacheOrdering(t, id1)
 
 	// fetch id1 again - cache hit, no change
-	c.getEntries(id1, expirationTime, cs.get(id1))
+	c.getEntries(ctx, id1, expirationTime, cs.get(id1))
 	cv.verifyCacheHit(t, id1)
 	cv.verifyCacheOrdering(t, id1)
 
 	// fetch id2
-	c.getEntries(id2, expirationTime, cs.get(id2))
+	c.getEntries(ctx, id2, expirationTime, cs.get(id2))
 	cv.verifyCacheMiss(t, id2)
 	cv.verifyCacheOrdering(t, id2, id1)
 
 	// fetch id1 again - cache hit, id1 moved to the top of the LRU list
-	c.getEntries(id1, expirationTime, cs.get(id1))
+	c.getEntries(ctx, id1, expirationTime, cs.get(id1))
 	cv.verifyCacheHit(t, id1)
 	cv.verifyCacheOrdering(t, id1, id2)
 
 	// fetch id2 again
-	c.getEntries(id2, expirationTime, cs.get(id2))
+	c.getEntries(ctx, id2, expirationTime, cs.get(id2))
 	cv.verifyCacheHit(t, id2)
 	cv.verifyCacheOrdering(t, id2, id1)
 
 	// fetch id3
-	c.getEntries(id3, expirationTime, cs.get(id3))
+	c.getEntries(ctx, id3, expirationTime, cs.get(id3))
 	cv.verifyCacheMiss(t, id3)
 	cv.verifyCacheOrdering(t, id3, id2, id1)
 
 	// fetch id4
-	c.getEntries(id4, expirationTime, cs.get(id4))
+	c.getEntries(ctx, id4, expirationTime, cs.get(id4))
 	cv.verifyCacheMiss(t, id4)
 	cv.verifyCacheOrdering(t, id4, id3)
 
 	// fetch id1 again
-	c.getEntries(id1, expirationTime, cs.get(id1))
+	c.getEntries(ctx, id1, expirationTime, cs.get(id1))
 	cv.verifyCacheMiss(t, id1)
 	cv.verifyCacheOrdering(t, id1, id4)
 
 	// fetch id5, it's a big one that expels all but one
-	c.getEntries(id5, expirationTime, cs.get(id5))
+	c.getEntries(ctx, id5, expirationTime, cs.get(id5))
 	cv.verifyCacheMiss(t, id5)
 	cv.verifyCacheOrdering(t, id5, id1)
 
 	// fetch id6
-	c.getEntries(id6, expirationTime, cs.get(id6))
+	c.getEntries(ctx, id6, expirationTime, cs.get(id6))
 	cv.verifyCacheMiss(t, id6)
 	cv.verifyCacheOrdering(t, id6)
 
 	// fetch id7
-	c.getEntries(id7, expirationTime, cs.get(id7))
+	c.getEntries(ctx, id7, expirationTime, cs.get(id7))
 	cv.verifyCacheMiss(t, id7)
 	cv.verifyCacheOrdering(t, id6)
 }

fs/cachefs/cachefs.go

@@ -2,25 +2,27 @@
 package cachefs
 
 import (
+	"context"
+
 	"github.com/kopia/kopia/fs"
 )
 
 // DirectoryCacher reads and potentially caches directory entries for a given directory.
 type DirectoryCacher interface {
-	Readdir(d fs.Directory) (fs.Entries, error)
+	Readdir(ctx context.Context, d fs.Directory) (fs.Entries, error)
 }
 
-type context struct {
+type cacheContext struct {
 	cacher DirectoryCacher
 }
 
 type directory struct {
-	ctx *context
+	ctx *cacheContext
 	fs.Directory
 }
 
-func (d *directory) Readdir() (fs.Entries, error) {
-	entries, err := d.ctx.cacher.Readdir(d.Directory)
+func (d *directory) Readdir(ctx context.Context) (fs.Entries, error) {
+	entries, err := d.ctx.cacher.Readdir(ctx, d.Directory)
 	if err != nil {
 		return entries, err
 	}
@@ -33,21 +35,21 @@ func (d *directory) Readdir() (fs.Entries, error) {
 }
 
 type file struct {
-	ctx *context
+	ctx *cacheContext
 	fs.File
 }
 
 type symlink struct {
-	ctx *context
+	ctx *cacheContext
 	fs.Symlink
 }
 
 // Wrap returns an Entry that wraps another Entry and caches directory reads.
 func Wrap(e fs.Entry, cacher DirectoryCacher) fs.Entry {
-	return wrapWithContext(e, &context{cacher})
+	return wrapWithContext(e, &cacheContext{cacher})
 }
 
-func wrapWithContext(e fs.Entry, opts *context) fs.Entry {
+func wrapWithContext(e fs.Entry, opts *cacheContext) fs.Entry {
 	switch e := e.(type) {
 	case fs.Directory:
 		return fs.Directory(&directory{opts, e})

fs/entry.go

@@ -1,6 +1,7 @@
 package fs
 
 import (
+	"context"
 	"io"
 	"sort"
 	"strings"
@@ -25,19 +26,19 @@ type Reader interface {
 // File represents an entry that is a file.
 type File interface {
 	Entry
-	Open() (Reader, error)
+	Open(ctx context.Context) (Reader, error)
 }
 
 // Directory represents contents of a directory.
 type Directory interface {
 	Entry
-	Readdir() (Entries, error)
+	Readdir(ctx context.Context) (Entries, error)
 }
 
 // Symlink represents a symbolic link entry.
 type Symlink interface {
 	Entry
-	Readlink() (string, error)
+	Readlink(ctx context.Context) (string, error)
 }
 
 // FindByName returns an entry with a given name, or nil if not found.

fs/localfs/local_fs.go

@@ -1,6 +1,7 @@
 package localfs
 
 import (
+	"context"
 	"fmt"
 	"io"
 	"os"
@@ -50,7 +51,7 @@ type filesystemFile struct {
 	filesystemEntry
 }
 
-func (fsd *filesystemDirectory) Readdir() (fs.Entries, error) {
+func (fsd *filesystemDirectory) Readdir(ctx context.Context) (fs.Entries, error) {
 	f, direrr := os.Open(fsd.path)
 	if direrr != nil {
 		return nil, direrr
@@ -95,7 +96,7 @@ func (erc *fileWithMetadata) EntryMetadata() (*fs.EntryMetadata, error) {
 	return entryMetadataFromFileInfo(fi), nil
 }
 
-func (fsf *filesystemFile) Open() (fs.Reader, error) {
+func (fsf *filesystemFile) Open(ctx context.Context) (fs.Reader, error) {
 	f, err := os.Open(fsf.path)
 	if err != nil {
 		return nil, err
@@ -104,7 +105,7 @@ func (fsf *filesystemFile) Open() (fs.Reader, error) {
 	return &fileWithMetadata{f}, nil
 }
 
-func (fsl *filesystemSymlink) Readlink() (string, error) {
+func (fsl *filesystemSymlink) Readlink(ctx context.Context) (string, error) {
 	return os.Readlink(fsl.path)
 }
 

fs/localfs/local_fs_test.go

@@ -1,6 +1,7 @@
 package localfs
 
 import (
+	"context"
 	"fmt"
 	"io/ioutil"
 	"os"
@@ -13,6 +14,7 @@ import (
 )
 
 func TestFiles(t *testing.T) {
+	ctx := context.Background()
 	var err error
 
 	tmp, err := ioutil.TempDir("", "kopia")
@@ -37,7 +39,7 @@ func TestFiles(t *testing.T) {
 		t.Errorf("error when dir empty directory: %v", err)
 	}
 
-	entries, err := dir.Readdir()
+	entries, err := dir.Readdir(ctx)
 	if err != nil {
 		t.Errorf("error gettind dir Entries: %v", err)
 	}
@@ -59,7 +61,7 @@ func TestFiles(t *testing.T) {
 		t.Errorf("error when dir directory with files: %v", err)
 	}
 
-	entries, err = dir.Readdir()
+	entries, err = dir.Readdir(ctx)
 	if err != nil {
 		t.Errorf("error gettind dir Entries: %v", err)
 	}

fs/loggingfs/loggingfs.go

@@ -2,6 +2,7 @@
 package loggingfs
 
 import (
+	"context"
 	"time"
 
 	"github.com/rs/zerolog/log"
@@ -19,9 +20,9 @@ type loggingDirectory struct {
 	fs.Directory
 }
 
-func (ld *loggingDirectory) Readdir() (fs.Entries, error) {
+func (ld *loggingDirectory) Readdir(ctx context.Context) (fs.Entries, error) {
 	t0 := time.Now()
-	entries, err := ld.Directory.Readdir()
+	entries, err := ld.Directory.Readdir(ctx)
 	dt := time.Since(t0)
 	ld.options.printf(ld.options.prefix+"Readdir(%v) took %v and returned %v items", fs.EntryPath(ld), dt, len(entries))
 	loggingEntries := make(fs.Entries, len(entries))

internal/fusemount/fusefs.go

@@ -37,7 +37,7 @@ type fuseFileNode struct {
 }
 
 func (f *fuseFileNode) ReadAll(ctx context.Context) ([]byte, error) {
-	reader, err := f.entry.(fs.File).Open()
+	reader, err := f.entry.(fs.File).Open(ctx)
 	if err != nil {
 		return nil, err
 	}
@@ -55,7 +55,7 @@ func (dir *fuseDirectoryNode) directory() fs.Directory {
 }
 
 func (dir *fuseDirectoryNode) Lookup(ctx context.Context, fileName string) (fusefs.Node, error) {
-	entries, err := dir.directory().Readdir()
+	entries, err := dir.directory().Readdir(ctx)
 	if err != nil {
 		if os.IsNotExist(err) {
 			return nil, fuse.ENOENT
@@ -73,7 +73,7 @@ func (dir *fuseDirectoryNode) Lookup(ctx context.Context, fileName string) (fuse
 }
 
 func (dir *fuseDirectoryNode) ReadDirAll(ctx context.Context) ([]fuse.Dirent, error) {
-	entries, err := dir.directory().Readdir()
+	entries, err := dir.directory().Readdir(ctx)
 	if err != nil {
 		return nil, err
 	}
@@ -106,7 +106,7 @@ type fuseSymlinkNode struct {
 }
 
 func (sl *fuseSymlinkNode) Readlink(ctx context.Context, req *fuse.ReadlinkRequest) (string, error) {
-	return sl.entry.(fs.Symlink).Readlink()
+	return sl.entry.(fs.Symlink).Readlink(ctx)
 }
 
 func newFuseNode(e fs.Entry) (fusefs.Node, error) {

internal/mockfs/mockfs.go

@@ -3,6 +3,7 @@ package mockfs
 
 import (
 	"bytes"
+	"context"
 	"fmt"
 	"io"
 	"sort"
@@ -150,7 +151,7 @@ func (imd *Directory) FailReaddir(err error) {
 }
 
 // Readdir gets the contents of a directory.
-func (imd *Directory) Readdir() (fs.Entries, error) {
+func (imd *Directory) Readdir(ctx context.Context) (fs.Entries, error) {
 	if imd.readdirError != nil {
 		return nil, imd.readdirError
 	}
@@ -175,7 +176,7 @@ func (ifr *fileReader) EntryMetadata() (*fs.EntryMetadata, error) {
 }
 
 // Open opens the file for reading, optionally simulating error.
-func (imf *File) Open() (fs.Reader, error) {
+func (imf *File) Open(ctx context.Context) (fs.Reader, error) {
 	r, err := imf.source()
 	if err != nil {
 		return nil, err
@@ -191,7 +192,7 @@ type inmemorySymlink struct {
 	entry
 }
 
-func (imsl *inmemorySymlink) Readlink() (string, error) {
+func (imsl *inmemorySymlink) Readlink(ctx context.Context) (string, error) {
 	panic("not implemented yet")
 }
 

internal/storagetesting/asserts.go

@@ -2,6 +2,7 @@ package storagetesting
 
 import (
 	"bytes"
+	"context"
 	"fmt"
 	"path/filepath"
 	"reflect"
@@ -12,8 +13,8 @@ import (
 )
 
 // AssertGetBlock asserts that the specified storage block has correct content.
-func AssertGetBlock(t *testing.T, s storage.Storage, block string, expected []byte) {
-	b, err := s.GetBlock(block, 0, -1)
+func AssertGetBlock(ctx context.Context, t *testing.T, s storage.Storage, block string, expected []byte) {
+	b, err := s.GetBlock(ctx, block, 0, -1)
 	if err != nil {
 		t.Errorf(errorPrefix()+"GetBlock(%v) returned error %v, expected data: %v", block, err, expected)
 		return
@@ -25,18 +26,19 @@ func AssertGetBlock(t *testing.T, s storage.Storage, block string, expected []by
 }
 
 // AssertGetBlockNotFound asserts that GetBlock() for specified storage block returns ErrBlockNotFound.
-func AssertGetBlockNotFound(t *testing.T, s storage.Storage, block string) {
-	b, err := s.GetBlock(block, 0, -1)
+func AssertGetBlockNotFound(ctx context.Context, t *testing.T, s storage.Storage, block string) {
+	b, err := s.GetBlock(ctx, block, 0, -1)
 	if err != storage.ErrBlockNotFound || b != nil {
 		t.Errorf(errorPrefix()+"GetBlock(%v) returned %v, %v but expected ErrBlockNotFound", block, b, err)
 	}
 }
 
 // AssertListResults asserts that the list results with given prefix return the specified list of names in order.
-func AssertListResults(t *testing.T, s storage.Storage, prefix string, expected ...string) {
+func AssertListResults(ctx context.Context, t *testing.T, s storage.Storage, prefix string, expected ...string) {
 	var names []string
 
-	blocks, cancel := s.ListBlocks(prefix)
+	ctx, cancel := context.WithCancel(ctx)
+	blocks := s.ListBlocks(ctx, prefix)
 	defer cancel()
 	for e := range blocks {
 		names = append(names, e.BlockID)

internal/storagetesting/map.go

@@ -1,6 +1,9 @@
 package storagetesting
 
 import (
+	"context"
+	"io"
+	"io/ioutil"
 	"sort"
 	"strings"
 	"sync"
@@ -15,7 +18,7 @@ type mapStorage struct {
 	mutex   sync.RWMutex
 }
 
-func (s *mapStorage) GetBlock(id string, offset, length int64) ([]byte, error) {
+func (s *mapStorage) GetBlock(ctx context.Context, id string, offset, length int64) ([]byte, error) {
 	s.mutex.RLock()
 	defer s.mutex.RUnlock()
 
@@ -35,10 +38,15 @@ func (s *mapStorage) GetBlock(id string, offset, length int64) ([]byte, error) {
 	return nil, storage.ErrBlockNotFound
 }
 
-func (s *mapStorage) PutBlock(id string, data []byte) error {
+func (s *mapStorage) PutBlock(ctx context.Context, id string, r io.Reader) error {
 	s.mutex.Lock()
 	defer s.mutex.Unlock()
 
+	data, err := ioutil.ReadAll(r)
+	if err != nil {
+		return err
+	}
+
 	if _, ok := s.data[id]; ok {
 		return nil
 	}
@@ -48,7 +56,7 @@ func (s *mapStorage) PutBlock(id string, data []byte) error {
 	return nil
 }
 
-func (s *mapStorage) DeleteBlock(id string) error {
+func (s *mapStorage) DeleteBlock(ctx context.Context, id string) error {
 	s.mutex.Lock()
 	defer s.mutex.Unlock()
 
@@ -56,9 +64,8 @@ func (s *mapStorage) DeleteBlock(id string) error {
 	return nil
 }
 
-func (s *mapStorage) ListBlocks(prefix string) (<-chan storage.BlockMetadata, storage.CancelFunc) {
+func (s *mapStorage) ListBlocks(ctx context.Context, prefix string) <-chan storage.BlockMetadata {
 	ch := make(chan storage.BlockMetadata)
-	cancelled := make(chan bool)
 	go func() {
 		defer close(ch)
 		s.mutex.RLock()
@@ -76,7 +83,7 @@ func (s *mapStorage) ListBlocks(prefix string) (<-chan storage.BlockMetadata, st
 		for _, k := range keys {
 			v := s.data[k]
 			select {
-			case <-cancelled:
+			case <-ctx.Done():
 				return
 			case ch <- storage.BlockMetadata{
 				BlockID:   k,
@@ -86,12 +93,10 @@ func (s *mapStorage) ListBlocks(prefix string) (<-chan storage.BlockMetadata, st
 			}
 		}
 	}()
-	return ch, func() {
-		close(cancelled)
-	}
+	return ch
 }
 
-func (s *mapStorage) Close() error {
+func (s *mapStorage) Close(ctx context.Context) error {
 	return nil
 }
 

internal/storagetesting/map_test.go

@@ -1,6 +1,9 @@
 package storagetesting
 
-import "testing"
+import (
+	"context"
+	"testing"
+)
 
 func TestMapStorage(t *testing.T) {
 	data := map[string][]byte{}
@@ -8,5 +11,5 @@ func TestMapStorage(t *testing.T) {
 	if r == nil {
 		t.Errorf("unexpected result: %v", r)
 	}
-	VerifyStorage(t, r)
+	VerifyStorage(context.Background(), t, r)
 }

internal/storagetesting/verify.go

@@ -2,13 +2,14 @@ package storagetesting
 
 import (
 	"bytes"
+	"context"
 	"testing"
 
 	"github.com/kopia/kopia/storage"
 )
 
 // VerifyStorage verifies the behavior of the specified storage.
-func VerifyStorage(t *testing.T, r storage.Storage) {
+func VerifyStorage(ctx context.Context, t *testing.T, r storage.Storage) {
 	blocks := []struct {
 		blk      string
 		contents []byte
@@ -21,20 +22,21 @@ func VerifyStorage(t *testing.T, r storage.Storage) {
 
 	// First verify that blocks don't exist.
 	for _, b := range blocks {
-		AssertGetBlockNotFound(t, r, b.blk)
+		AssertGetBlockNotFound(ctx, t, r, b.blk)
 	}
 
 	// Now add blocks.
 	for _, b := range blocks {
-		if err := r.PutBlock(b.blk, b.contents); err != nil {
+		if err := r.PutBlock(ctx, b.blk, bytes.NewReader(b.contents)); err != nil {
 			t.Errorf("can't put block: %v", err)
 		}
 
-		AssertGetBlock(t, r, b.blk, b.contents)
+		AssertGetBlock(ctx, t, r, b.blk, b.contents)
 	}
 
 	// List
-	ch, cancel := r.ListBlocks(string("ab"))
+	ctx, cancel := context.WithCancel(ctx)
+	ch := r.ListBlocks(ctx, "ab")
 	defer cancel()
 	e1 := verifyNextBlock(t, blocks[0].blk, ch)
 	e2 := verifyNextBlock(t, blocks[2].blk, ch)

internal/webdavmount/webdavmount.go

@@ -20,6 +20,7 @@ var _ webdav.File = (*webdavFile)(nil)
 var _ webdav.File = (*webdavDir)(nil)
 
 type webdavFile struct {
+	ctx   context.Context
 	entry fs.File
 
 	mu sync.Mutex
@@ -38,7 +39,7 @@ func (f *webdavFile) getReader() (fs.Reader, error) {
 	f.mu.Lock()
 	defer f.mu.Unlock()
 	if f.r == nil {
-		r, err := f.entry.Open()
+		r, err := f.entry.Open(f.ctx)
 		if err != nil {
 			return nil, err
 		}
@@ -83,12 +84,13 @@ func (f *webdavFile) Close() error {
 }
 
 type webdavDir struct {
+	ctx   context.Context
 	w     *webdavFS
 	entry fs.Directory
 }
 
 func (d *webdavDir) Readdir(n int) ([]os.FileInfo, error) {
-	entries, err := d.entry.Readdir()
+	entries, err := d.entry.Readdir(d.ctx)
 	if err != nil {
 		return nil, err
 	}
@@ -168,7 +170,7 @@ func (w *webdavFS) Rename(ctx context.Context, oldPath, newPath string) error {
 }
 
 func (w *webdavFS) OpenFile(ctx context.Context, path string, flags int, mode os.FileMode) (webdav.File, error) {
-	f, err := w.findEntry(path)
+	f, err := w.findEntry(ctx, path)
 	if err != nil {
 		log.Printf("OpenFile(%q) failed with %v", path, err)
 		return nil, err
@@ -176,16 +178,16 @@ func (w *webdavFS) OpenFile(ctx context.Context, path string, flags int, mode os
 
 	switch f := f.(type) {
 	case fs.Directory:
-		return &webdavDir{w, f}, nil
+		return &webdavDir{ctx, w, f}, nil
 	case fs.File:
-		return &webdavFile{entry: f}, nil
+		return &webdavFile{ctx: ctx, entry: f}, nil
 	}
 
 	return nil, fmt.Errorf("can't open %q: not implemented", path)
 }
 
 func (w *webdavFS) Stat(ctx context.Context, path string) (os.FileInfo, error) {
-	e, err := w.findEntry(path)
+	e, err := w.findEntry(ctx, path)
 	if err != nil {
 		return nil, err
 	}
@@ -193,7 +195,7 @@ func (w *webdavFS) Stat(ctx context.Context, path string) (os.FileInfo, error) {
 	return webdavFileInfo{e.Metadata()}, nil
 }
 
-func (w *webdavFS) findEntry(path string) (fs.Entry, error) {
+func (w *webdavFS) findEntry(ctx context.Context, path string) (fs.Entry, error) {
 	parts := removeEmpty(strings.Split(path, "/"))
 	var e fs.Entry = w.dir
 	for i, p := range parts {
@@ -202,7 +204,7 @@ func (w *webdavFS) findEntry(path string) (fs.Entry, error) {
 			return nil, fmt.Errorf("%q not found in %q (not a directory)", p, strings.Join(parts[0:i], "/"))
 		}
 
-		entries, err := d.Readdir()
+		entries, err := d.Readdir(ctx)
 		if err != nil {
 			return nil, err
 		}

manifest/manifest_manager.go

@@ -3,6 +3,7 @@ package manifest
 import (
 	"bytes"
 	"compress/gzip"
+	"context"
 	"crypto/rand"
 	"encoding/hex"
 	"encoding/json"
@@ -147,11 +148,11 @@ func matchesLabels(a, b map[string]string) bool {
 }
 
 // Flush persists changes to manifest manager.
-func (m *Manager) Flush() error {
+func (m *Manager) Flush(ctx context.Context) error {
 	m.mu.Lock()
 	defer m.mu.Unlock()
 
-	blockID, err := m.flushPendingEntriesLocked()
+	blockID, err := m.flushPendingEntriesLocked(ctx)
 	if err != nil {
 		return err
 	}
@@ -164,7 +165,7 @@ func (m *Manager) Flush() error {
 	return nil
 }
 
-func (m *Manager) flushPendingEntriesLocked() (string, error) {
+func (m *Manager) flushPendingEntriesLocked(ctx context.Context) (string, error) {
 	if len(m.pendingEntries) == 0 {
 		return "", nil
 	}
@@ -186,7 +187,7 @@ func (m *Manager) flushPendingEntriesLocked() (string, error) {
 		return "", fmt.Errorf("unable to close: %v", err)
 	}
 
-	blockID, err := m.b.WriteBlock(buf.Bytes(), manifestBlockPrefix)
+	blockID, err := m.b.WriteBlock(ctx, buf.Bytes(), manifestBlockPrefix)
 	if err != nil {
 		return "", err
 	}
@@ -209,8 +210,8 @@ func (m *Manager) Delete(id string) {
 	})
 }
 
-func (m *Manager) load() error {
-	if err := m.Flush(); err != nil {
+func (m *Manager) load(ctx context.Context) error {
+	if err := m.Flush(ctx); err != nil {
 		return err
 	}
 
@@ -227,17 +228,17 @@ func (m *Manager) load() error {
 
 	log.Printf("found %v manifest blocks", len(blocks))
 
-	if err := m.loadManifestBlocks(blocks); err != nil {
+	if err := m.loadManifestBlocks(ctx, blocks); err != nil {
 		return fmt.Errorf("unable to load manifest blocks: %v", err)
 	}
 
 	if len(blocks) > autoCompactionBlockCount {
 		log.Debug().Int("blocks", len(blocks)).Msg("performing automatic compaction")
-		if err := m.compactLocked(); err != nil {
+		if err := m.compactLocked(ctx); err != nil {
 			return fmt.Errorf("unable to compact manifest blocks: %v", err)
 		}
 
-		if err := m.b.Flush(); err != nil {
+		if err := m.b.Flush(ctx); err != nil {
 			return fmt.Errorf("unable to flush blocks after auto-compaction: %v", err)
 		}
 	}
@@ -245,7 +246,7 @@ func (m *Manager) load() error {
 	return nil
 }
 
-func (m *Manager) loadManifestBlocks(blocks []block.Info) error {
+func (m *Manager) loadManifestBlocks(ctx context.Context, blocks []block.Info) error {
 	t0 := time.Now()
 
 	log.Debug().Dur("duration_ms", time.Since(t0)).Msgf("finished loading manifest blocks.")
@@ -254,7 +255,7 @@ func (m *Manager) loadManifestBlocks(blocks []block.Info) error {
 		m.blockIDs = append(m.blockIDs, b.BlockID)
 	}
 
-	manifests, err := m.loadBlocksInParallel(blocks)
+	manifests, err := m.loadBlocksInParallel(ctx, blocks)
 	if err != nil {
 		return err
 	}
@@ -275,7 +276,7 @@ func (m *Manager) loadManifestBlocks(blocks []block.Info) error {
 	return nil
 }
 
-func (m *Manager) loadBlocksInParallel(blocks []block.Info) ([]manifest, error) {
+func (m *Manager) loadBlocksInParallel(ctx context.Context, blocks []block.Info) ([]manifest, error) {
 	errors := make(chan error, len(blocks))
 	manifests := make(chan manifest, len(blocks))
 	blockIDs := make(chan string, len(blocks))
@@ -288,7 +289,7 @@ func (m *Manager) loadBlocksInParallel(blocks []block.Info) ([]manifest, error)
 
 			for blk := range blockIDs {
 				t1 := time.Now()
-				man, err := m.loadManifestBlock(blk)
+				man, err := m.loadManifestBlock(ctx, blk)
 				log.Debug().Dur("duration", time.Since(t1)).Str("blk", blk).Int("worker", workerID).Msg("manifest block loaded")
 				if err != nil {
 					errors <- err
@@ -323,9 +324,9 @@ func (m *Manager) loadBlocksInParallel(blocks []block.Info) ([]manifest, error)
 	return man, nil
 }
 
-func (m *Manager) loadManifestBlock(blockID string) (manifest, error) {
+func (m *Manager) loadManifestBlock(ctx context.Context, blockID string) (manifest, error) {
 	man := manifest{}
-	blk, err := m.b.GetBlock(blockID)
+	blk, err := m.b.GetBlock(ctx, blockID)
 	if err != nil {
 		return man, fmt.Errorf("unable to read block %q: %v", blockID, err)
 	}
@@ -349,14 +350,14 @@ func (m *Manager) loadManifestBlock(blockID string) (manifest, error) {
 }
 
 // Compact performs compaction of manifest blocks.
-func (m *Manager) Compact() error {
+func (m *Manager) Compact(ctx context.Context) error {
 	m.mu.Lock()
 	defer m.mu.Unlock()
 
-	return m.compactLocked()
+	return m.compactLocked(ctx)
 }
 
-func (m *Manager) compactLocked() error {
+func (m *Manager) compactLocked(ctx context.Context) error {
 	log.Printf("compactLocked: pendingEntries=%v blockIDs=%v", len(m.pendingEntries), len(m.blockIDs))
 
 	if len(m.blockIDs) == 1 && len(m.pendingEntries) == 0 {
@@ -367,7 +368,7 @@ func (m *Manager) compactLocked() error {
 		m.pendingEntries = append(m.pendingEntries, e)
 	}
 
-	blockID, err := m.flushPendingEntriesLocked()
+	blockID, err := m.flushPendingEntriesLocked(ctx)
 	if err != nil {
 		return err
 	}
@@ -412,13 +413,13 @@ func copyLabels(m map[string]string) map[string]string {
 }
 
 // NewManager returns new manifest manager for the provided block manager.
-func NewManager(b *block.Manager) (*Manager, error) {
+func NewManager(ctx context.Context, b *block.Manager) (*Manager, error) {
 	m := &Manager{
 		b:       b,
 		entries: map[string]*manifestEntry{},
 	}
 
-	if err := m.load(); err != nil {
+	if err := m.load(ctx); err != nil {
 		return nil, err
 	}
 

manifest/manifest_manager_test.go

@@ -1,6 +1,7 @@
 package manifest
 
 import (
+	"context"
 	"fmt"
 	"reflect"
 	"sort"
@@ -13,9 +14,10 @@ import (
 )
 
 func TestManifest(t *testing.T) {
+	ctx := context.Background()
 	data := map[string][]byte{}
 	keyTime := map[string]time.Time{}
-	mgr, setupErr := newManagerForTesting(t, data, keyTime)
+	mgr, setupErr := newManagerForTesting(ctx, t, data, keyTime)
 	if setupErr != nil {
 		t.Fatalf("unable to open block manager: %v", setupErr)
 	}
@@ -52,10 +54,10 @@ func TestManifest(t *testing.T) {
 	verifyItem(t, mgr, id2, labels2, item2)
 	verifyItem(t, mgr, id3, labels3, item3)
 
-	if err := mgr.Flush(); err != nil {
+	if err := mgr.Flush(ctx); err != nil {
 		t.Errorf("flush error: %v", err)
 	}
-	if err := mgr.Flush(); err != nil {
+	if err := mgr.Flush(ctx); err != nil {
 		t.Errorf("flush error: %v", err)
 	}
 
@@ -68,8 +70,8 @@ func TestManifest(t *testing.T) {
 	verifyItem(t, mgr, id3, labels3, item3)
 
 	// flush underlying block manager and verify in new manifest manager.
-	mgr.b.Flush()
-	mgr2, setupErr := newManagerForTesting(t, data, keyTime)
+	mgr.b.Flush(ctx)
+	mgr2, setupErr := newManagerForTesting(ctx, t, data, keyTime)
 	if setupErr != nil {
 		t.Fatalf("can't open block manager: %v", setupErr)
 	}
@@ -79,23 +81,23 @@ func TestManifest(t *testing.T) {
 	verifyItem(t, mgr2, id1, labels1, item1)
 	verifyItem(t, mgr2, id2, labels2, item2)
 	verifyItem(t, mgr2, id3, labels3, item3)
-	if err := mgr2.Flush(); err != nil {
+	if err := mgr2.Flush(ctx); err != nil {
 		t.Errorf("flush error: %v", err)
 	}
 
 	// delete from one
 	mgr.Delete(id3)
 	verifyItemNotFound(t, mgr, id3)
-	mgr.Flush()
+	mgr.Flush(ctx)
 	verifyItemNotFound(t, mgr, id3)
 
 	// still found in another
 	verifyItem(t, mgr2, id3, labels3, item3)
-	if err := mgr2.load(); err != nil {
+	if err := mgr2.load(ctx); err != nil {
 		t.Errorf("unable to load: %v", err)
 	}
 
-	if err := mgr.Compact(); err != nil {
+	if err := mgr.Compact(ctx); err != nil {
 		t.Errorf("can't compact: %v", err)
 	}
 
@@ -107,9 +109,9 @@ func TestManifest(t *testing.T) {
 		t.Errorf("unexpected number of blocks: %v, want %v", got, want)
 	}
 
-	mgr.b.Flush()
+	mgr.b.Flush(ctx)
 
-	mgr3, err := newManagerForTesting(t, data, keyTime)
+	mgr3, err := newManagerForTesting(ctx, t, data, keyTime)
 	if err != nil {
 		t.Fatalf("can't open manager: %v", err)
 	}
@@ -170,10 +172,10 @@ func verifyMatches(t *testing.T, mgr *Manager, labels map[string]string, expecte
 	}
 }
 
-func newManagerForTesting(t *testing.T, data map[string][]byte, keyTime map[string]time.Time) (*Manager, error) {
+func newManagerForTesting(ctx context.Context, t *testing.T, data map[string][]byte, keyTime map[string]time.Time) (*Manager, error) {
 	st := storagetesting.NewMapStorage(data, keyTime)
 
-	bm, err := block.NewManager(st, block.FormattingOptions{
+	bm, err := block.NewManager(ctx, st, block.FormattingOptions{
 		BlockFormat: "TESTONLY_MD5",
 		MaxPackSize: 100000,
 	}, block.CachingOptions{})
@@ -181,5 +183,5 @@ func newManagerForTesting(t *testing.T, data map[string][]byte, keyTime map[stri
 		return nil, fmt.Errorf("can't create block manager: %v", err)
 	}
 
-	return NewManager(bm)
+	return NewManager(ctx, bm)
 }

object/object_manager.go

@@ -3,6 +3,7 @@ package object
 import (
 	"bufio"
 	"bytes"
+	"context"
 	"fmt"
 	"io"
 	"sync"
@@ -23,10 +24,10 @@ type Reader interface {
 }
 
 type blockManager interface {
-	BlockInfo(blockID string) (block.Info, error)
-	GetBlock(blockID string) ([]byte, error)
-	WriteBlock(data []byte, prefix string) (string, error)
-	Flush() error
+	BlockInfo(ctx context.Context, blockID string) (block.Info, error)
+	GetBlock(ctx context.Context, blockID string) ([]byte, error)
+	WriteBlock(ctx context.Context, data []byte, prefix string) (string, error)
+	Flush(ctx context.Context) error
 }
 
 // Manager implements a content-addressable storage on top of blob storage.
@@ -45,14 +46,15 @@ type Manager struct {
 }
 
 // Close closes the connection to the underlying blob storage and releases any resources.
-func (om *Manager) Close() error {
+func (om *Manager) Close(ctx context.Context) error {
 	om.writeBackWG.Wait()
-	return om.Flush()
+	return om.Flush(ctx)
 }
 
 // NewWriter creates an ObjectWriter for writing to the repository.
-func (om *Manager) NewWriter(opt WriterOptions) Writer {
+func (om *Manager) NewWriter(ctx context.Context, opt WriterOptions) Writer {
 	w := &objectWriter{
+		ctx:         ctx,
 		repo:        om,
 		splitter:    om.newSplitter(),
 		description: opt.Description,
@@ -66,7 +68,7 @@ func (om *Manager) NewWriter(opt WriterOptions) Writer {
 }
 
 // Open creates new ObjectReader for reading given object from a repository.
-func (om *Manager) Open(objectID ID) (Reader, error) {
+func (om *Manager) Open(ctx context.Context, objectID ID) (Reader, error) {
 	// log.Printf("Repository::Open %v", objectID.String())
 	// defer log.Printf("finished Repository::Open() %v", objectID.String())
 
@@ -74,7 +76,7 @@ func (om *Manager) Open(objectID ID) (Reader, error) {
 	om.writeBackWG.Wait()
 
 	if objectID.Indirect != nil {
-		rd, err := om.Open(*objectID.Indirect)
+		rd, err := om.Open(ctx, *objectID.Indirect)
 		if err != nil {
 			return nil, err
 		}
@@ -88,23 +90,24 @@ func (om *Manager) Open(objectID ID) (Reader, error) {
 		totalLength := seekTable[len(seekTable)-1].endOffset()
 
 		return &objectReader{
+			ctx:         ctx,
 			repo:        om,
 			seekTable:   seekTable,
 			totalLength: totalLength,
 		}, nil
 	}
 
-	return om.newRawReader(objectID)
+	return om.newRawReader(ctx, objectID)
 }
 
 // VerifyObject ensures that all objects backing ObjectID are present in the repository
 // and returns the total length of the object and storage blocks of which it is composed.
-func (om *Manager) VerifyObject(oid ID) (int64, []string, error) {
+func (om *Manager) VerifyObject(ctx context.Context, oid ID) (int64, []string, error) {
 	// Flush any pending writes.
 	om.writeBackWG.Wait()
 
 	blocks := &blockTracker{}
-	l, err := om.verifyObjectInternal(oid, blocks)
+	l, err := om.verifyObjectInternal(ctx, oid, blocks)
 	if err != nil {
 		return 0, nil, err
 	}
@@ -112,11 +115,11 @@ func (om *Manager) VerifyObject(oid ID) (int64, []string, error) {
 	return l, blocks.blockIDs(), nil
 }
 
-func (om *Manager) verifyIndirectObjectInternal(oid ID, blocks *blockTracker) (int64, error) {
-	if _, err := om.verifyObjectInternal(*oid.Indirect, blocks); err != nil {
+func (om *Manager) verifyIndirectObjectInternal(ctx context.Context, oid ID, blocks *blockTracker) (int64, error) {
+	if _, err := om.verifyObjectInternal(ctx, *oid.Indirect, blocks); err != nil {
 		return 0, fmt.Errorf("unable to read index: %v", err)
 	}
-	rd, err := om.Open(*oid.Indirect)
+	rd, err := om.Open(ctx, *oid.Indirect)
 	if err != nil {
 		return 0, err
 	}
@@ -128,7 +131,7 @@ func (om *Manager) verifyIndirectObjectInternal(oid ID, blocks *blockTracker) (i
 	}
 
 	for i, m := range seekTable {
-		l, err := om.verifyObjectInternal(m.Object, blocks)
+		l, err := om.verifyObjectInternal(ctx, m.Object, blocks)
 		if err != nil {
 			return 0, err
 		}
@@ -142,19 +145,19 @@ func (om *Manager) verifyIndirectObjectInternal(oid ID, blocks *blockTracker) (i
 	return totalLength, nil
 }
 
-func (om *Manager) verifyObjectInternal(oid ID, blocks *blockTracker) (int64, error) {
+func (om *Manager) verifyObjectInternal(ctx context.Context, oid ID, blocks *blockTracker) (int64, error) {
 	if oid.Indirect != nil {
-		return om.verifyIndirectObjectInternal(oid, blocks)
+		return om.verifyIndirectObjectInternal(ctx, oid, blocks)
 	}
 
-	p, err := om.blockMgr.BlockInfo(oid.StorageBlock)
+	p, err := om.blockMgr.BlockInfo(ctx, oid.StorageBlock)
 	if err != nil {
 		return 0, err
 	}
 	blocks.addBlock(oid.StorageBlock)
 
 	if p.PackBlockID != "" {
-		l, err := om.verifyObjectInternal(ID{StorageBlock: p.PackBlockID}, blocks)
+		l, err := om.verifyObjectInternal(ctx, ID{StorageBlock: p.PackBlockID}, blocks)
 		if err != nil {
 			return 0, err
 		}
@@ -171,7 +174,7 @@ func (om *Manager) verifyObjectInternal(oid ID, blocks *blockTracker) (int64, er
 
 // Flush closes any pending pack files. Once this method returns, ObjectIDs returned by ObjectManager are
 // ok to be used.
-func (om *Manager) Flush() error {
+func (om *Manager) Flush(ctx context.Context) error {
 	om.writeBackWG.Wait()
 	return nil
 }
@@ -195,7 +198,7 @@ type ManagerOptions struct {
 }
 
 // NewObjectManager creates an ObjectManager with the specified block manager and format.
-func NewObjectManager(bm blockManager, f config.RepositoryObjectFormat, opts ManagerOptions) (*Manager, error) {
+func NewObjectManager(ctx context.Context, bm blockManager, f config.RepositoryObjectFormat, opts ManagerOptions) (*Manager, error) {
 	if err := validateFormat(&f); err != nil {
 		return nil, err
 	}
@@ -260,8 +263,8 @@ func (om *Manager) flattenListChunk(rawReader io.Reader) ([]indirectObjectEntry,
 	return seekTable, nil
 }
 
-func (om *Manager) newRawReader(objectID ID) (Reader, error) {
-	payload, err := om.blockMgr.GetBlock(objectID.StorageBlock)
+func (om *Manager) newRawReader(ctx context.Context, objectID ID) (Reader, error) {
+	payload, err := om.blockMgr.GetBlock(ctx, objectID.StorageBlock)
 	if err != nil {
 		return nil, err
 	}

object/object_manager_test.go

@@ -2,6 +2,7 @@ package object
 
 import (
 	"bytes"
+	"context"
 	"crypto/md5"
 	cryptorand "crypto/rand"
 	"encoding/hex"
@@ -26,7 +27,7 @@ type fakeBlockManager struct {
 	data map[string][]byte
 }
 
-func (f *fakeBlockManager) GetBlock(blockID string) ([]byte, error) {
+func (f *fakeBlockManager) GetBlock(ctx context.Context, blockID string) ([]byte, error) {
 	f.mu.Lock()
 	defer f.mu.Unlock()
 
@@ -37,7 +38,7 @@ func (f *fakeBlockManager) GetBlock(blockID string) ([]byte, error) {
 	return nil, storage.ErrBlockNotFound
 }
 
-func (f *fakeBlockManager) WriteBlock(data []byte, prefix string) (string, error) {
+func (f *fakeBlockManager) WriteBlock(ctx context.Context, data []byte, prefix string) (string, error) {
 	h := md5.New()
 	h.Write(data)
 	blockID := prefix + hex.EncodeToString(h.Sum(nil))
@@ -49,7 +50,7 @@ func (f *fakeBlockManager) WriteBlock(data []byte, prefix string) (string, error
 	return blockID, nil
 }
 
-func (f *fakeBlockManager) BlockInfo(blockID string) (block.Info, error) {
+func (f *fakeBlockManager) BlockInfo(ctx context.Context, blockID string) (block.Info, error) {
 	f.mu.Lock()
 	defer f.mu.Unlock()
 
@@ -60,7 +61,7 @@ func (f *fakeBlockManager) BlockInfo(blockID string) (block.Info, error) {
 	return block.Info{}, storage.ErrBlockNotFound
 }
 
-func (f *fakeBlockManager) Flush() error {
+func (f *fakeBlockManager) Flush(ctx context.Context) error {
 	return nil
 }
 
@@ -69,7 +70,7 @@ func setupTest(t *testing.T) (map[string][]byte, *Manager) {
 }
 
 func setupTestWithData(t *testing.T, data map[string][]byte, opts ManagerOptions) (map[string][]byte, *Manager) {
-	r, err := NewObjectManager(&fakeBlockManager{data: data}, config.RepositoryObjectFormat{
+	r, err := NewObjectManager(context.Background(), &fakeBlockManager{data: data}, config.RepositoryObjectFormat{
 		FormattingOptions: block.FormattingOptions{
 			Version: 1,
 		},
@@ -84,6 +85,7 @@ func setupTestWithData(t *testing.T, data map[string][]byte, opts ManagerOptions
 }
 
 func TestWriters(t *testing.T) {
+	ctx := context.Background()
 	cases := []struct {
 		data     []byte
 		objectID ID
@@ -98,7 +100,7 @@ func TestWriters(t *testing.T) {
 	for _, c := range cases {
 		data, om := setupTest(t)
 
-		writer := om.NewWriter(WriterOptions{})
+		writer := om.NewWriter(ctx, WriterOptions{})
 
 		writer.Write(c.data)
 
@@ -132,10 +134,11 @@ func objectIDsEqual(o1 ID, o2 ID) bool {
 }
 
 func TestWriterCompleteChunkInTwoWrites(t *testing.T) {
+	ctx := context.Background()
 	_, om := setupTest(t)
 
 	bytes := make([]byte, 100)
-	writer := om.NewWriter(WriterOptions{})
+	writer := om.NewWriter(ctx, WriterOptions{})
 	writer.Write(bytes[0:50])
 	writer.Write(bytes[0:50])
 	result, err := writer.Result()
@@ -144,12 +147,12 @@ func TestWriterCompleteChunkInTwoWrites(t *testing.T) {
 	}
 }
 
-func verifyIndirectBlock(t *testing.T, r *Manager, oid ID) {
+func verifyIndirectBlock(ctx context.Context, t *testing.T, r *Manager, oid ID) {
 	for oid.Indirect != nil {
 		direct := *oid.Indirect
 		oid = direct
 
-		rd, err := r.Open(direct)
+		rd, err := r.Open(ctx, direct)
 		if err != nil {
 			t.Errorf("unable to open %v: %v", oid.String(), err)
 			return
@@ -175,6 +178,7 @@ func verifyIndirectBlock(t *testing.T, r *Manager, oid ID) {
 }
 
 func TestIndirection(t *testing.T) {
+	ctx := context.Background()
 	cases := []struct {
 		dataLength          int
 		expectedBlockCount  int
@@ -194,7 +198,7 @@ func TestIndirection(t *testing.T) {
 
 		contentBytes := make([]byte, c.dataLength)
 
-		writer := om.NewWriter(WriterOptions{})
+		writer := om.NewWriter(ctx, WriterOptions{})
 		writer.Write(contentBytes)
 		result, err := writer.Result()
 		if err != nil {
@@ -209,9 +213,9 @@ func TestIndirection(t *testing.T) {
 			t.Errorf("unexpected block count for %v: %v, expected %v", c.dataLength, got, want)
 		}
 
-		om.Flush()
+		om.Flush(ctx)
 
-		l, b, err := om.VerifyObject(result)
+		l, b, err := om.VerifyObject(ctx, result)
 		if err != nil {
 			t.Errorf("error verifying %q: %v", result, err)
 		}
@@ -224,7 +228,7 @@ func TestIndirection(t *testing.T) {
 			t.Errorf("invalid block count for %v, got %v, wanted %v", result, got, want)
 		}
 
-		verifyIndirectBlock(t, om, result)
+		verifyIndirectBlock(ctx, t, om, result)
 	}
 }
 
@@ -237,11 +241,12 @@ func indirectionLevel(oid ID) int {
 }
 
 func TestHMAC(t *testing.T) {
+	ctx := context.Background()
 	content := bytes.Repeat([]byte{0xcd}, 50)
 
 	_, om := setupTest(t)
 
-	w := om.NewWriter(WriterOptions{})
+	w := om.NewWriter(ctx, WriterOptions{})
 	w.Write(content)
 	result, err := w.Result()
 	if result.String() != "D999732b72ceff665b3f7608411db66a4" {
@@ -250,6 +255,7 @@ func TestHMAC(t *testing.T) {
 }
 
 func TestReader(t *testing.T) {
+	ctx := context.Background()
 	data, om := setupTest(t)
 
 	storedPayload := []byte("foo\nbar")
@@ -269,7 +275,7 @@ func TestReader(t *testing.T) {
 			continue
 		}
 
-		reader, err := om.Open(objectID)
+		reader, err := om.Open(ctx, objectID)
 		if err != nil {
 			t.Errorf("cannot create reader for %v: %v", objectID, err)
 			continue
@@ -288,19 +294,21 @@ func TestReader(t *testing.T) {
 }
 
 func TestReaderStoredBlockNotFound(t *testing.T) {
+	ctx := context.Background()
 	_, om := setupTest(t)
 
 	objectID, err := ParseID("Dno-such-block")
 	if err != nil {
 		t.Errorf("cannot parse object ID: %v", err)
 	}
-	reader, err := om.Open(objectID)
+	reader, err := om.Open(ctx, objectID)
 	if err != storage.ErrBlockNotFound || reader != nil {
 		t.Errorf("unexpected result: reader: %v err: %v", reader, err)
 	}
 }
 
 func TestEndToEndReadAndSeek(t *testing.T) {
+	ctx := context.Background()
 	_, om := setupTest(t)
 
 	for _, size := range []int{1, 199, 200, 201, 9999, 512434} {
@@ -308,7 +316,7 @@ func TestEndToEndReadAndSeek(t *testing.T) {
 		randomData := make([]byte, size)
 		cryptorand.Read(randomData)
 
-		writer := om.NewWriter(WriterOptions{})
+		writer := om.NewWriter(ctx, WriterOptions{})
 		writer.Write(randomData)
 		objectID, err := writer.Result()
 		writer.Close()
@@ -317,13 +325,13 @@ func TestEndToEndReadAndSeek(t *testing.T) {
 			continue
 		}
 
-		verify(t, om, objectID, randomData, fmt.Sprintf("%v %v", objectID, size))
+		verify(ctx, t, om, objectID, randomData, fmt.Sprintf("%v %v", objectID, size))
 	}
 }
 
-func verify(t *testing.T, om *Manager, objectID ID, expectedData []byte, testCaseID string) {
+func verify(ctx context.Context, t *testing.T, om *Manager, objectID ID, expectedData []byte, testCaseID string) {
 	t.Helper()
-	reader, err := om.Open(objectID)
+	reader, err := om.Open(ctx, objectID)
 	if err != nil {
 		t.Errorf("cannot get reader for %v (%v): %v %v", testCaseID, objectID, err, string(debug.Stack()))
 		return

object/object_reader.go

@@ -1,6 +1,7 @@
 package object
 
 import (
+	"context"
 	"fmt"
 	"io"
 )
@@ -10,6 +11,7 @@ func (i *indirectObjectEntry) endOffset() int64 {
 }
 
 type objectReader struct {
+	ctx  context.Context
 	repo *Manager
 
 	seekTable []indirectObjectEntry
@@ -65,7 +67,7 @@ func (r *objectReader) Read(buffer []byte) (int, error) {
 
 func (r *objectReader) openCurrentChunk() error {
 	st := r.seekTable[r.currentChunkIndex]
-	blockData, err := r.repo.Open(st.Object)
+	blockData, err := r.repo.Open(r.ctx, st.Object)
 	if err != nil {
 		return err
 	}

object/object_writer.go

@@ -2,6 +2,7 @@ package object
 
 import (
 	"bytes"
+	"context"
 	"fmt"
 	"io"
 	"sync"
@@ -44,6 +45,7 @@ func (t *blockTracker) blockIDs() []string {
 }
 
 type objectWriter struct {
+	ctx  context.Context
 	repo *Manager
 
 	buffer      bytes.Buffer
@@ -95,7 +97,7 @@ func (w *objectWriter) flushBuffer() error {
 	w.buffer.Reset()
 
 	do := func() {
-		blockID, err := w.repo.blockMgr.WriteBlock(b2.Bytes(), "")
+		blockID, err := w.repo.blockMgr.WriteBlock(w.ctx, b2.Bytes(), "")
 		w.repo.trace("OBJECT_WRITER(%q) stored %v (%v bytes)", w.description, blockID, length)
 		if err != nil {
 			w.err.add(fmt.Errorf("error when flushing chunk %d of %s: %v", chunkID, w.description, err))
@@ -143,6 +145,7 @@ func (w *objectWriter) Result() (ID, error) {
 	}
 
 	iw := &objectWriter{
+		ctx:         w.ctx,
 		repo:        w.repo,
 		description: "LIST(" + w.description + ")",
 		splitter:    w.repo.newSplitter(),

repo/connect.go

@@ -31,7 +31,7 @@ type ConnectOptions struct {
 
 // Connect connects to the repository in the specified storage and persists the configuration and credentials in the file provided.
 func Connect(ctx context.Context, configFile string, st storage.Storage, creds auth.Credentials, opt ConnectOptions) error {
-	formatBytes, err := st.GetBlock(FormatBlockID, 0, -1)
+	formatBytes, err := st.GetBlock(ctx, FormatBlockID, 0, -1)
 	if err != nil {
 		return fmt.Errorf("unable to read format block: %v", err)
 	}
@@ -85,7 +85,7 @@ func Connect(ctx context.Context, configFile string, st storage.Storage, creds a
 		return err
 	}
 
-	return r.Close()
+	return r.Close(ctx)
 }
 
 func setupCaching(lc *config.LocalConfig, opt block.CachingOptions, uniqueID []byte) error {

repo/format_block.go

@@ -2,6 +2,7 @@ package repo
 
 import (
 	"bytes"
+	"context"
 	"crypto/aes"
 	"crypto/cipher"
 	"crypto/rand"
@@ -50,7 +51,7 @@ func parseFormatBlock(b []byte) (*formatBlock, error) {
 	return f, nil
 }
 
-func writeFormatBlock(st storage.Storage, f *formatBlock) error {
+func writeFormatBlock(ctx context.Context, st storage.Storage, f *formatBlock) error {
 	var buf bytes.Buffer
 	e := json.NewEncoder(&buf)
 	e.SetIndent("", "  ")
@@ -58,7 +59,7 @@ func writeFormatBlock(st storage.Storage, f *formatBlock) error {
 		return fmt.Errorf("unable to marshal format block: %v", err)
 	}
 
-	if err := st.PutBlock(FormatBlockID, buf.Bytes()); err != nil {
+	if err := st.PutBlock(ctx, FormatBlockID, &buf); err != nil {
 		return fmt.Errorf("unable to write format block: %v", err)
 	}
 

repo/initialize.go

@@ -1,6 +1,7 @@
 package repo
 
 import (
+	"context"
 	"crypto/rand"
 	"io"
 
@@ -44,7 +45,7 @@ type NewRepositoryOptions struct {
 }
 
 // Initialize creates initial repository data structures in the specified storage with given credentials.
-func Initialize(st storage.Storage, opt *NewRepositoryOptions, creds auth.Credentials) error {
+func Initialize(ctx context.Context, st storage.Storage, opt *NewRepositoryOptions, creds auth.Credentials) error {
 	if opt == nil {
 		opt = &NewRepositoryOptions{}
 	}
@@ -60,7 +61,7 @@ func Initialize(st storage.Storage, opt *NewRepositoryOptions, creds auth.Creden
 		return err
 	}
 
-	if err := writeFormatBlock(st, format); err != nil {
+	if err := writeFormatBlock(ctx, st, format); err != nil {
 		return err
 	}
 

repo/open.go

@@ -80,7 +80,7 @@ func Open(ctx context.Context, configFile string, options *Options) (rep *Reposi
 
 	r, err := connect(ctx, st, creds, options, caching)
 	if err != nil {
-		st.Close() //nolint:errcheck
+		st.Close(ctx) //nolint:errcheck
 		return nil, err
 	}
 
@@ -121,7 +121,7 @@ func SetCachingConfig(ctx context.Context, configFile string, opt block.CachingO
 		return fmt.Errorf("cannot open storage: %v", err)
 	}
 
-	f, err := readAndCacheFormatBlock(st, "")
+	f, err := readAndCacheFormatBlock(ctx, st, "")
 	if err != nil {
 		return fmt.Errorf("can't read format block: %v", err)
 	}
@@ -142,7 +142,7 @@ func SetCachingConfig(ctx context.Context, configFile string, opt block.CachingO
 	return nil
 }
 
-func readAndCacheFormatBlock(st storage.Storage, cacheDirectory string) (*formatBlock, error) {
+func readAndCacheFormatBlock(ctx context.Context, st storage.Storage, cacheDirectory string) (*formatBlock, error) {
 	cachedFile := filepath.Join(cacheDirectory, "kopia.repository")
 	if cacheDirectory != "" {
 		b, err := ioutil.ReadFile(cachedFile)
@@ -152,7 +152,7 @@ func readAndCacheFormatBlock(st storage.Storage, cacheDirectory string) (*format
 		}
 	}
 
-	b, err := st.GetBlock(FormatBlockID, 0, -1)
+	b, err := st.GetBlock(ctx, FormatBlockID, 0, -1)
 	if err != nil {
 		return nil, err
 	}
@@ -182,7 +182,7 @@ func connect(ctx context.Context, st storage.Storage, creds auth.Credentials, op
 
 	log.Debug().Msg("reading encrypted format block")
 	// Read cache block, potentially from cache.
-	f, err := readAndCacheFormatBlock(st, caching.CacheDirectory)
+	f, err := readAndCacheFormatBlock(ctx, st, caching.CacheDirectory)
 	if err != nil {
 		return nil, fmt.Errorf("unable to read format block: %v", err)
 	}
@@ -205,19 +205,19 @@ func connect(ctx context.Context, st storage.Storage, creds auth.Credentials, op
 	caching.HMACSecret = km.DeriveKey([]byte("local-cache-integrity"), 16)
 
 	log.Debug().Msg("initializing block manager")
-	bm, err := block.NewManager(st, fo, caching)
+	bm, err := block.NewManager(ctx, st, fo, caching)
 	if err != nil {
 		return nil, fmt.Errorf("unable to open block manager: %v", err)
 	}
 
 	log.Debug().Msg("initializing object manager")
-	om, err := object.NewObjectManager(bm, *repoConfig, options.ObjectManagerOptions)
+	om, err := object.NewObjectManager(ctx, bm, *repoConfig, options.ObjectManagerOptions)
 	if err != nil {
 		return nil, fmt.Errorf("unable to open object manager: %v", err)
 	}
 
 	log.Debug().Msg("initializing manifest manager")
-	manifests, err := manifest.NewManager(bm)
+	manifests, err := manifest.NewManager(ctx, bm)
 	if err != nil {
 		return nil, fmt.Errorf("unable to open manifests: %v", err)
 	}

repo/repository.go

@@ -1,6 +1,8 @@
 package repo
 
 import (
+	"context"
+
 	"github.com/kopia/kopia/auth"
 	"github.com/kopia/kopia/block"
 	"github.com/kopia/kopia/manifest"
@@ -22,30 +24,30 @@ type Repository struct {
 }
 
 // Close closes the repository and releases all resources.
-func (r *Repository) Close() error {
-	if err := r.Manifests.Flush(); err != nil {
+func (r *Repository) Close(ctx context.Context) error {
+	if err := r.Manifests.Flush(ctx); err != nil {
 		return err
 	}
-	if err := r.Objects.Close(); err != nil {
+	if err := r.Objects.Close(ctx); err != nil {
 		return err
 	}
-	if err := r.Blocks.Flush(); err != nil {
+	if err := r.Blocks.Flush(ctx); err != nil {
 		return err
 	}
-	if err := r.Storage.Close(); err != nil {
+	if err := r.Storage.Close(ctx); err != nil {
 		return err
 	}
 	return nil
 }
 
 // Flush waits for all in-flight writes to complete.
-func (r *Repository) Flush() error {
-	if err := r.Manifests.Flush(); err != nil {
+func (r *Repository) Flush(ctx context.Context) error {
+	if err := r.Manifests.Flush(ctx); err != nil {
 		return err
 	}
-	if err := r.Objects.Flush(); err != nil {
+	if err := r.Objects.Flush(ctx); err != nil {
 		return err
 	}
 
-	return r.Blocks.Flush()
+	return r.Blocks.Flush(ctx)
 }

repo/repository_test.go

@@ -42,9 +42,9 @@ func setupTestWithData(t *testing.T, data map[string][]byte, keyTime map[string]
 	for _, m := range mods {
 		m(opt)
 	}
-	Initialize(st, opt, creds)
 
 	ctx := context.Background()
+	Initialize(ctx, st, opt, creds)
 
 	r, err := connect(ctx, st, creds, &Options{
 		//TraceStorage: log.Printf,
@@ -68,10 +68,12 @@ func TestWriters(t *testing.T) {
 		{make([]byte, 100), object.ID{StorageBlock: "6d0bb00954ceb7fbee436bb55a8397a9"}}, // 100 zero bytes
 	}
 
+	ctx := context.Background()
+
 	for _, c := range cases {
 		data, _, repo := setupTest(t)
 
-		writer := repo.Objects.NewWriter(object.WriterOptions{})
+		writer := repo.Objects.NewWriter(ctx, object.WriterOptions{})
 
 		writer.Write(c.data)
 
@@ -81,13 +83,13 @@ func TestWriters(t *testing.T) {
 			continue
 		}
 
-		repo.Objects.Flush()
+		repo.Objects.Flush(ctx)
 
 		if !objectIDsEqual(result, c.objectID) {
 			t.Errorf("incorrect result for %v, expected: %v got: %v", c.data, c.objectID.String(), result.String())
 		}
 
-		repo.Blocks.Flush()
+		repo.Blocks.Flush(ctx)
 
 		if got, want := len(data), 2; got != want {
 			// 1 format block + 1 pack index block (including inline data blocks)
@@ -118,10 +120,11 @@ func objectIDsEqual(o1 object.ID, o2 object.ID) bool {
 }
 
 func TestWriterCompleteChunkInTwoWrites(t *testing.T) {
+	ctx := context.Background()
 	_, _, repo := setupTest(t)
 
 	bytes := make([]byte, 100)
-	writer := repo.Objects.NewWriter(object.WriterOptions{})
+	writer := repo.Objects.NewWriter(ctx, object.WriterOptions{})
 	writer.Write(bytes[0:50])
 	writer.Write(bytes[0:50])
 	result, err := writer.Result()
@@ -133,25 +136,26 @@ func TestWriterCompleteChunkInTwoWrites(t *testing.T) {
 func TestPackingSimple(t *testing.T) {
 	data, keyTime, repo := setupTest(t, func(n *NewRepositoryOptions) {
 	})
+	ctx := context.Background()
 
 	content1 := "hello, how do you do?"
 	content2 := "hi, how are you?"
 	content3 := "thank you!"
 
-	oid1a := writeObject(t, repo, []byte(content1), "packed-object-1a")
-	oid1b := writeObject(t, repo, []byte(content1), "packed-object-1b")
-	oid2a := writeObject(t, repo, []byte(content2), "packed-object-2a")
-	oid2b := writeObject(t, repo, []byte(content2), "packed-object-2b")
+	oid1a := writeObject(ctx, t, repo, []byte(content1), "packed-object-1a")
+	oid1b := writeObject(ctx, t, repo, []byte(content1), "packed-object-1b")
+	oid2a := writeObject(ctx, t, repo, []byte(content2), "packed-object-2a")
+	oid2b := writeObject(ctx, t, repo, []byte(content2), "packed-object-2b")
 
-	oid3a := writeObject(t, repo, []byte(content3), "packed-object-3a")
-	oid3b := writeObject(t, repo, []byte(content3), "packed-object-3b")
-	verify(t, repo, oid1a, []byte(content1), "packed-object-1")
-	verify(t, repo, oid2a, []byte(content2), "packed-object-2")
-	oid2c := writeObject(t, repo, []byte(content2), "packed-object-2c")
-	oid1c := writeObject(t, repo, []byte(content1), "packed-object-1c")
+	oid3a := writeObject(ctx, t, repo, []byte(content3), "packed-object-3a")
+	oid3b := writeObject(ctx, t, repo, []byte(content3), "packed-object-3b")
+	verify(ctx, t, repo, oid1a, []byte(content1), "packed-object-1")
+	verify(ctx, t, repo, oid2a, []byte(content2), "packed-object-2")
+	oid2c := writeObject(ctx, t, repo, []byte(content2), "packed-object-2c")
+	oid1c := writeObject(ctx, t, repo, []byte(content1), "packed-object-1c")
 
-	repo.Objects.Flush()
-	repo.Blocks.Flush()
+	repo.Objects.Flush(ctx)
+	repo.Blocks.Flush(ctx)
 
 	if got, want := oid1a.String(), oid1b.String(); got != want {
 		t.Errorf("oid1a(%q) != oid1b(%q)", got, want)
@@ -173,34 +177,34 @@ func TestPackingSimple(t *testing.T) {
 	if got, want := len(data), 2; got != want {
 		t.Errorf("got unexpected repository contents %v items, wanted %v", got, want)
 	}
-	repo.Close()
+	repo.Close(ctx)
 
 	data, _, repo = setupTestWithData(t, data, keyTime, func(n *NewRepositoryOptions) {
 	})
 
-	verify(t, repo, oid1a, []byte(content1), "packed-object-1")
-	verify(t, repo, oid2a, []byte(content2), "packed-object-2")
-	verify(t, repo, oid3a, []byte(content3), "packed-object-3")
+	verify(ctx, t, repo, oid1a, []byte(content1), "packed-object-1")
+	verify(ctx, t, repo, oid2a, []byte(content2), "packed-object-2")
+	verify(ctx, t, repo, oid3a, []byte(content3), "packed-object-3")
 
-	if err := repo.Blocks.CompactIndexes(); err != nil {
+	if err := repo.Blocks.CompactIndexes(ctx); err != nil {
 		t.Errorf("optimize error: %v", err)
 	}
 	data, _, repo = setupTestWithData(t, data, keyTime, func(n *NewRepositoryOptions) {
 	})
 
-	verify(t, repo, oid1a, []byte(content1), "packed-object-1")
-	verify(t, repo, oid2a, []byte(content2), "packed-object-2")
-	verify(t, repo, oid3a, []byte(content3), "packed-object-3")
+	verify(ctx, t, repo, oid1a, []byte(content1), "packed-object-1")
+	verify(ctx, t, repo, oid2a, []byte(content2), "packed-object-2")
+	verify(ctx, t, repo, oid3a, []byte(content3), "packed-object-3")
 
-	if err := repo.Blocks.CompactIndexes(); err != nil {
+	if err := repo.Blocks.CompactIndexes(ctx); err != nil {
 		t.Errorf("optimize error: %v", err)
 	}
 	_, _, repo = setupTestWithData(t, data, keyTime, func(n *NewRepositoryOptions) {
 	})
 
-	verify(t, repo, oid1a, []byte(content1), "packed-object-1")
-	verify(t, repo, oid2a, []byte(content2), "packed-object-2")
-	verify(t, repo, oid3a, []byte(content3), "packed-object-3")
+	verify(ctx, t, repo, oid1a, []byte(content1), "packed-object-1")
+	verify(ctx, t, repo, oid2a, []byte(content2), "packed-object-2")
+	verify(ctx, t, repo, oid3a, []byte(content3), "packed-object-3")
 }
 
 func TestHMAC(t *testing.T) {
@@ -208,7 +212,9 @@ func TestHMAC(t *testing.T) {
 
 	_, _, repo := setupTest(t)
 
-	w := repo.Objects.NewWriter(object.WriterOptions{})
+	ctx := context.Background()
+
+	w := repo.Objects.NewWriter(ctx, object.WriterOptions{})
 	w.Write(content)
 	result, err := w.Result()
 	if result.String() != "D999732b72ceff665b3f7608411db66a4" {
@@ -217,6 +223,7 @@ func TestHMAC(t *testing.T) {
 }
 func TestMalformedStoredData(t *testing.T) {
 	data, _, repo := setupTest(t)
+	ctx := context.Background()
 
 	cases := [][]byte{
 		[]byte("foo\nba"),
@@ -231,7 +238,7 @@ func TestMalformedStoredData(t *testing.T) {
 			continue
 		}
 
-		reader, err := repo.Objects.Open(objectID)
+		reader, err := repo.Objects.Open(ctx, objectID)
 		if err == nil || reader != nil {
 			t.Errorf("expected error for %x", c)
 		}
@@ -240,12 +247,13 @@ func TestMalformedStoredData(t *testing.T) {
 
 func TestReaderStoredBlockNotFound(t *testing.T) {
 	_, _, repo := setupTest(t)
+	ctx := context.Background()
 
 	objectID, err := object.ParseID("Dno-such-block")
 	if err != nil {
 		t.Errorf("cannot parse object ID: %v", err)
 	}
-	reader, err := repo.Objects.Open(objectID)
+	reader, err := repo.Objects.Open(ctx, objectID)
 	if err != storage.ErrBlockNotFound || reader != nil {
 		t.Errorf("unexpected result: reader: %v err: %v", reader, err)
 	}
@@ -253,13 +261,14 @@ func TestReaderStoredBlockNotFound(t *testing.T) {
 
 func TestEndToEndReadAndSeek(t *testing.T) {
 	_, _, repo := setupTest(t)
+	ctx := context.Background()
 
 	for _, size := range []int{1, 199, 200, 201, 9999, 512434} {
 		// Create some random data sample of the specified size.
 		randomData := make([]byte, size)
 		cryptorand.Read(randomData)
 
-		writer := repo.Objects.NewWriter(object.WriterOptions{})
+		writer := repo.Objects.NewWriter(ctx, object.WriterOptions{})
 		writer.Write(randomData)
 		objectID, err := writer.Result()
 		writer.Close()
@@ -268,12 +277,12 @@ func TestEndToEndReadAndSeek(t *testing.T) {
 			continue
 		}
 
-		verify(t, repo, objectID, randomData, fmt.Sprintf("%v %v", objectID, size))
+		verify(ctx, t, repo, objectID, randomData, fmt.Sprintf("%v %v", objectID, size))
 	}
 }
 
-func writeObject(t *testing.T, repo *Repository, data []byte, testCaseID string) object.ID {
-	w := repo.Objects.NewWriter(object.WriterOptions{})
+func writeObject(ctx context.Context, t *testing.T, repo *Repository, data []byte, testCaseID string) object.ID {
+	w := repo.Objects.NewWriter(ctx, object.WriterOptions{})
 	if _, err := w.Write(data); err != nil {
 		t.Fatalf("can't write object %q - write failed: %v", testCaseID, err)
 
@@ -286,9 +295,9 @@ func writeObject(t *testing.T, repo *Repository, data []byte, testCaseID string)
 	return oid
 }
 
-func verify(t *testing.T, repo *Repository, objectID object.ID, expectedData []byte, testCaseID string) {
+func verify(ctx context.Context, t *testing.T, repo *Repository, objectID object.ID, expectedData []byte, testCaseID string) {
 	t.Helper()
-	reader, err := repo.Objects.Open(objectID)
+	reader, err := repo.Objects.Open(ctx, objectID)
 	if err != nil {
 		t.Errorf("cannot get reader for %v (%v): %v %v", testCaseID, objectID, err, string(debug.Stack()))
 		return
@@ -319,6 +328,7 @@ func verify(t *testing.T, repo *Repository, objectID object.ID, expectedData []b
 }
 
 func TestFormats(t *testing.T) {
+	ctx := context.Background()
 	makeFormat := func(objectFormat string) func(*NewRepositoryOptions) {
 		return func(n *NewRepositoryOptions) {
 			n.BlockFormat = objectFormat
@@ -368,7 +378,7 @@ func TestFormats(t *testing.T) {
 
 		for k, v := range c.oids {
 			bytesToWrite := []byte(k)
-			w := repo.Objects.NewWriter(object.WriterOptions{})
+			w := repo.Objects.NewWriter(ctx, object.WriterOptions{})
 			w.Write(bytesToWrite)
 			oid, err := w.Result()
 			if err != nil {
@@ -378,7 +388,7 @@ func TestFormats(t *testing.T) {
 				t.Errorf("invalid oid for #%v\ngot:\n%#v\nexpected:\n%#v", caseIndex, oid.String(), v.String())
 			}
 
-			rc, err := repo.Objects.Open(oid)
+			rc, err := repo.Objects.Open(ctx, oid)
 			if err != nil {
 				t.Errorf("open failed: %v", err)
 				continue

snapshot/all_sources.go

@@ -1,6 +1,7 @@
 package snapshot
 
 import (
+	"context"
 	"fmt"
 	"time"
 
@@ -26,7 +27,7 @@ func (s *repositoryAllSources) Metadata() *fs.EntryMetadata {
 	}
 }
 
-func (s *repositoryAllSources) Readdir() (fs.Entries, error) {
+func (s *repositoryAllSources) Readdir(ctx context.Context) (fs.Entries, error) {
 	srcs := s.snapshotManager.ListSources()
 
 	users := map[string]bool{}

snapshot/repofs.go

@@ -1,6 +1,7 @@
 package snapshot
 
 import (
+	"context"
 	"fmt"
 	"io/ioutil"
 
@@ -40,8 +41,8 @@ type repositorySymlink struct {
 	repositoryEntry
 }
 
-func (rd *repositoryDirectory) Readdir() (fs.Entries, error) {
-	r, err := rd.repo.Objects.Open(rd.metadata.ObjectID)
+func (rd *repositoryDirectory) Readdir(ctx context.Context) (fs.Entries, error) {
+	r, err := rd.repo.Objects.Open(ctx, rd.metadata.ObjectID)
 	if err != nil {
 		return nil, err
 	}
@@ -60,8 +61,8 @@ func (rd *repositoryDirectory) Readdir() (fs.Entries, error) {
 	return entries, nil
 }
 
-func (rf *repositoryFile) Open() (fs.Reader, error) {
-	r, err := rf.repo.Objects.Open(rf.metadata.ObjectID)
+func (rf *repositoryFile) Open(ctx context.Context) (fs.Reader, error) {
+	r, err := rf.repo.Objects.Open(ctx, rf.metadata.ObjectID)
 	if err != nil {
 		return nil, err
 	}
@@ -69,8 +70,8 @@ func (rf *repositoryFile) Open() (fs.Reader, error) {
 	return withMetadata(r, &rf.metadata.EntryMetadata), nil
 }
 
-func (rsl *repositorySymlink) Readlink() (string, error) {
-	r, err := rsl.repo.Objects.Open(rsl.metadata.ObjectID)
+func (rsl *repositorySymlink) Readlink(ctx context.Context) (string, error) {
+	r, err := rsl.repo.Objects.Open(ctx, rsl.metadata.ObjectID)
 	if err != nil {
 		return "", err
 	}

snapshot/source_directories.go

@@ -1,6 +1,7 @@
 package snapshot
 
 import (
+	"context"
 	"time"
 
 	"github.com/kopia/kopia/fs"
@@ -27,7 +28,7 @@ func (s *sourceDirectories) Metadata() *fs.EntryMetadata {
 	}
 }
 
-func (s *sourceDirectories) Readdir() (fs.Entries, error) {
+func (s *sourceDirectories) Readdir(ctx context.Context) (fs.Entries, error) {
 	sources := s.snapshotManager.ListSources()
 	var result fs.Entries
 

snapshot/source_snapshots.go

@@ -1,6 +1,7 @@
 package snapshot
 
 import (
+	"context"
 	"fmt"
 	"strings"
 
@@ -33,7 +34,7 @@ func safeName(path string) string {
 	return strings.Replace(path, "/", "_", -1)
 }
 
-func (s *sourceSnapshots) Readdir() (fs.Entries, error) {
+func (s *sourceSnapshots) Readdir(ctx context.Context) (fs.Entries, error) {
 	manifests, err := s.snapshotManager.ListSnapshots(s.src)
 	if err != nil {
 		return nil, err

snapshot/upload.go

@@ -2,6 +2,7 @@ package snapshot
 
 import (
 	"bytes"
+	"context"
 	"encoding/binary"
 	"errors"
 	"fmt"
@@ -95,14 +96,14 @@ func (u *Uploader) cancelReason() string {
 	return ""
 }
 
-func (u *Uploader) uploadFileInternal(f fs.File, relativePath string) entryResult {
-	file, err := f.Open()
+func (u *Uploader) uploadFileInternal(ctx context.Context, f fs.File, relativePath string) entryResult {
+	file, err := f.Open(ctx)
 	if err != nil {
 		return entryResult{err: fmt.Errorf("unable to open file: %v", err)}
 	}
 	defer file.Close() //nolint:errcheck
 
-	writer := u.repo.Objects.NewWriter(object.WriterOptions{
+	writer := u.repo.Objects.NewWriter(ctx, object.WriterOptions{
 		Description: "FILE:" + f.Metadata().Name,
 	})
 	defer writer.Close() //nolint:errcheck
@@ -128,13 +129,13 @@ func (u *Uploader) uploadFileInternal(f fs.File, relativePath string) entryResul
 	return entryResult{de: de, hash: metadataHash(&de.EntryMetadata)}
 }
 
-func (u *Uploader) uploadSymlinkInternal(f fs.Symlink, relativePath string) entryResult {
-	target, err := f.Readlink()
+func (u *Uploader) uploadSymlinkInternal(ctx context.Context, f fs.Symlink, relativePath string) entryResult {
+	target, err := f.Readlink(ctx)
 	if err != nil {
 		return entryResult{err: fmt.Errorf("unable to read symlink: %v", err)}
 	}
 
-	writer := u.repo.Objects.NewWriter(object.WriterOptions{
+	writer := u.repo.Objects.NewWriter(ctx, object.WriterOptions{
 		Description: "SYMLINK:" + f.Metadata().Name,
 	})
 	defer writer.Close() //nolint:errcheck
@@ -222,8 +223,8 @@ func newDirEntry(md *fs.EntryMetadata, oid object.ID) *dir.Entry {
 }
 
 // uploadFile uploads the specified File to the repository.
-func (u *Uploader) uploadFile(file fs.File) (object.ID, error) {
-	res := u.uploadFileInternal(file, file.Metadata().Name)
+func (u *Uploader) uploadFile(ctx context.Context, file fs.File) (object.ID, error) {
+	res := u.uploadFileInternal(ctx, file, file.Metadata().Name)
 	if res.err != nil {
 		return object.NullID, res.err
 	}
@@ -233,13 +234,13 @@ func (u *Uploader) uploadFile(file fs.File) (object.ID, error) {
 // uploadDir uploads the specified Directory to the repository.
 // An optional ID of a hash-cache object may be provided, in which case the Uploader will use its
 // contents to avoid hashing
-func (u *Uploader) uploadDir(dir fs.Directory) (object.ID, object.ID, error) {
-	mw := u.repo.Objects.NewWriter(object.WriterOptions{
+func (u *Uploader) uploadDir(ctx context.Context, dir fs.Directory) (object.ID, object.ID, error) {
+	mw := u.repo.Objects.NewWriter(ctx, object.WriterOptions{
 		Description: "HASHCACHE:" + dir.Metadata().Name,
 	})
 	defer mw.Close() //nolint:errcheck
 	u.cacheWriter = hashcache.NewWriter(mw)
-	oid, _, err := uploadDirInternal(u, dir, ".")
+	oid, _, err := uploadDirInternal(ctx, u, dir, ".")
 	if u.IsCancelled() {
 		if err2 := u.cacheReader.CopyTo(u.cacheWriter); err != nil {
 			return object.NullID, object.NullID, err2
@@ -253,7 +254,7 @@ func (u *Uploader) uploadDir(dir fs.Directory) (object.ID, object.ID, error) {
 	}
 
 	hcid, err := mw.Result()
-	if flushErr := u.repo.Objects.Flush(); flushErr != nil {
+	if flushErr := u.repo.Objects.Flush(ctx); flushErr != nil {
 		return object.NullID, object.NullID, fmt.Errorf("can't flush pending objects: %v", flushErr)
 	}
 	return oid, hcid, err
@@ -289,7 +290,7 @@ type entryResult struct {
 	hash uint64
 }
 
-func (u *Uploader) processSubdirectories(relativePath string, entries fs.Entries, dw *dir.Writer, summ *dir.Summary) error {
+func (u *Uploader) processSubdirectories(ctx context.Context, relativePath string, entries fs.Entries, dw *dir.Writer, summ *dir.Summary) error {
 	return u.foreachEntryUnlessCancelled(relativePath, entries, func(entry fs.Entry, entryRelativePath string) error {
 		dir, ok := entry.(fs.Directory)
 		if !ok {
@@ -298,7 +299,7 @@ func (u *Uploader) processSubdirectories(relativePath string, entries fs.Entries
 		}
 
 		e := dir.Metadata()
-		oid, subdirsumm, err := uploadDirInternal(u, dir, entryRelativePath)
+		oid, subdirsumm, err := uploadDirInternal(ctx, u, dir, entryRelativePath)
 		if err == errCancelled {
 			return err
 		}
@@ -342,7 +343,7 @@ type uploadWorkItem struct {
 	resultChan        chan entryResult
 }
 
-func (u *Uploader) prepareWorkItems(dirRelativePath string, entries fs.Entries, summ *dir.Summary) ([]*uploadWorkItem, error) {
+func (u *Uploader) prepareWorkItems(ctx context.Context, dirRelativePath string, entries fs.Entries, summ *dir.Summary) ([]*uploadWorkItem, error) {
 	var result []*uploadWorkItem
 
 	resultErr := u.foreachEntryUnlessCancelled(dirRelativePath, entries, func(entry fs.Entry, entryRelativePath string) error {
@@ -400,7 +401,7 @@ func (u *Uploader) prepareWorkItems(dirRelativePath string, entries fs.Entries,
 					entry:             entry,
 					entryRelativePath: entryRelativePath,
 					uploadFunc: func() entryResult {
-						return u.uploadSymlinkInternal(entry, entryRelativePath)
+						return u.uploadSymlinkInternal(ctx, entry, entryRelativePath)
 					},
 				})
 
@@ -410,7 +411,7 @@ func (u *Uploader) prepareWorkItems(dirRelativePath string, entries fs.Entries,
 					entry:             entry,
 					entryRelativePath: entryRelativePath,
 					uploadFunc: func() entryResult {
-						return u.uploadFileInternal(entry, entryRelativePath)
+						return u.uploadFileInternal(ctx, entry, entryRelativePath)
 					},
 				})
 
@@ -507,6 +508,7 @@ func (u *Uploader) processUploadWorkItems(workItems []*uploadWorkItem, dw *dir.W
 }
 
 func uploadDirInternal(
+	ctx context.Context,
 	u *Uploader,
 	directory fs.Directory,
 	dirRelativePath string,
@@ -515,24 +517,24 @@ func uploadDirInternal(
 
 	var summ dir.Summary
 
-	entries, direrr := directory.Readdir()
+	entries, direrr := directory.Readdir(ctx)
 	if direrr != nil {
 		return object.NullID, dir.Summary{}, direrr
 	}
 
-	writer := u.repo.Objects.NewWriter(object.WriterOptions{
+	writer := u.repo.Objects.NewWriter(ctx, object.WriterOptions{
 		Description: "DIR:" + dirRelativePath,
 	})
 
 	dw := dir.NewWriter(writer)
 	defer writer.Close() //nolint:errcheck
 
-	if err := u.processSubdirectories(dirRelativePath, entries, dw, &summ); err != nil {
+	if err := u.processSubdirectories(ctx, dirRelativePath, entries, dw, &summ); err != nil {
 		return object.NullID, dir.Summary{}, err
 	}
 	u.prepareProgress(dirRelativePath, entries)
 
-	workItems, workItemErr := u.prepareWorkItems(dirRelativePath, entries, &summ)
+	workItems, workItemErr := u.prepareWorkItems(ctx, dirRelativePath, entries, &summ)
 	if workItemErr != nil {
 		return object.NullID, dir.Summary{}, workItemErr
 	}
@@ -574,6 +576,7 @@ func (u *Uploader) Cancel() {
 // Upload uploads contents of the specified filesystem entry (file or directory) to the repository and returns snapshot.Manifest with statistics.
 // Old snapshot manifest, when provided can be used to speed up uploads by utilizing hash cache.
 func (u *Uploader) Upload(
+	ctx context.Context,
 	source fs.Entry,
 	sourceInfo SourceInfo,
 	old *Manifest,
@@ -588,7 +591,7 @@ func (u *Uploader) Upload(
 	u.stats = Stats{}
 	if old != nil {
 		log.Debug().Msgf("opening hash cache: %v", old.HashCacheID)
-		if r, err := u.repo.Objects.Open(old.HashCacheID); err == nil {
+		if r, err := u.repo.Objects.Open(ctx, old.HashCacheID); err == nil {
 			u.cacheReader = hashcache.Open(r)
 			log.Debug().Msgf("opened hash cache: %v", old.HashCacheID)
 		} else {
@@ -605,10 +608,10 @@ func (u *Uploader) Upload(
 
 	switch entry := source.(type) {
 	case fs.Directory:
-		s.RootObjectID, s.HashCacheID, err = u.uploadDir(entry)
+		s.RootObjectID, s.HashCacheID, err = u.uploadDir(ctx, entry)
 
 	case fs.File:
-		s.RootObjectID, err = u.uploadFile(entry)
+		s.RootObjectID, err = u.uploadFile(ctx, entry)
 
 	default:
 		return nil, fmt.Errorf("unsupported source: %v", s.Source)

snapshot/upload_test.go

@@ -52,7 +52,7 @@ func newUploadTestHarness() *uploadTestHarness {
 		panic("unable to create credentials: " + err.Error())
 	}
 
-	if initerr := repo.Initialize(storage, &repo.NewRepositoryOptions{}, creds); initerr != nil {
+	if initerr := repo.Initialize(ctx, storage, &repo.NewRepositoryOptions{}, creds); initerr != nil {
 		panic("unable to create repository: " + initerr.Error())
 	}
 
@@ -100,16 +100,17 @@ func newUploadTestHarness() *uploadTestHarness {
 }
 
 func TestUpload(t *testing.T) {
+	ctx := context.Background()
 	th := newUploadTestHarness()
 	defer th.cleanup()
 
 	u := NewUploader(th.repo)
-	s1, err := u.Upload(th.sourceDir, SourceInfo{}, nil)
+	s1, err := u.Upload(ctx, th.sourceDir, SourceInfo{}, nil)
 	if err != nil {
 		t.Errorf("Upload error: %v", err)
 	}
 
-	s2, err := u.Upload(th.sourceDir, SourceInfo{}, s1)
+	s2, err := u.Upload(ctx, th.sourceDir, SourceInfo{}, s1)
 	if err != nil {
 		t.Errorf("Upload error: %v", err)
 	}
@@ -133,7 +134,7 @@ func TestUpload(t *testing.T) {
 
 	// Add one more file, the s1.RootObjectID should change.
 	th.sourceDir.AddFile("d2/d1/f3", []byte{1, 2, 3, 4, 5}, 0777)
-	s3, err := u.Upload(th.sourceDir, SourceInfo{}, s1)
+	s3, err := u.Upload(ctx, th.sourceDir, SourceInfo{}, s1)
 	if err != nil {
 		t.Errorf("upload failed: %v", err)
 	}
@@ -154,7 +155,7 @@ func TestUpload(t *testing.T) {
 	// Now remove the added file, OID should be identical to the original before the file got added.
 	th.sourceDir.Subdir("d2", "d1").Remove("f3")
 
-	s4, err := u.Upload(th.sourceDir, SourceInfo{}, s1)
+	s4, err := u.Upload(ctx, th.sourceDir, SourceInfo{}, s1)
 	if err != nil {
 		t.Errorf("upload failed: %v", err)
 	}
@@ -171,7 +172,7 @@ func TestUpload(t *testing.T) {
 		t.Errorf("unexpected s4 stats: %+v", s4.Stats)
 	}
 
-	s5, err := u.Upload(th.sourceDir, SourceInfo{}, s3)
+	s5, err := u.Upload(ctx, th.sourceDir, SourceInfo{}, s3)
 	if err != nil {
 		t.Errorf("upload failed: %v", err)
 	}
@@ -193,13 +194,14 @@ func TestUpload_Cancel(t *testing.T) {
 }
 
 func TestUpload_TopLevelDirectoryReadFailure(t *testing.T) {
+	ctx := context.Background()
 	th := newUploadTestHarness()
 	defer th.cleanup()
 
 	th.sourceDir.FailReaddir(errTest)
 
 	u := NewUploader(th.repo)
-	s, err := u.Upload(th.sourceDir, SourceInfo{}, nil)
+	s, err := u.Upload(ctx, th.sourceDir, SourceInfo{}, nil)
 	if err != errTest {
 		t.Errorf("expected error: %v", err)
 	}
@@ -210,6 +212,7 @@ func TestUpload_TopLevelDirectoryReadFailure(t *testing.T) {
 }
 
 func TestUpload_SubDirectoryReadFailure(t *testing.T) {
+	ctx := context.Background()
 	th := newUploadTestHarness()
 	defer th.cleanup()
 
@@ -217,7 +220,7 @@ func TestUpload_SubDirectoryReadFailure(t *testing.T) {
 
 	u := NewUploader(th.repo)
 	u.IgnoreFileErrors = false
-	_, err := u.Upload(th.sourceDir, SourceInfo{}, nil)
+	_, err := u.Upload(ctx, th.sourceDir, SourceInfo{}, nil)
 	if err == nil {
 		t.Errorf("expected error")
 	}

storage/filesystem/filesystem_storage.go

@@ -30,7 +30,7 @@ type fsStorage struct {
 	Options
 }
 
-func (fs *fsStorage) GetBlock(blockID string, offset, length int64) ([]byte, error) {
+func (fs *fsStorage) GetBlock(ctx context.Context, blockID string, offset, length int64) ([]byte, error) {
 	_, path := fs.getShardedPathAndFilePath(blockID)
 
 	f, err := os.Open(path)
@@ -65,9 +65,8 @@ func makeFileName(blockID string) string {
 	return blockID + fsStorageChunkSuffix
 }
 
-func (fs *fsStorage) ListBlocks(prefix string) (<-chan storage.BlockMetadata, storage.CancelFunc) {
+func (fs *fsStorage) ListBlocks(ctx context.Context, prefix string) <-chan storage.BlockMetadata {
 	result := make(chan storage.BlockMetadata)
-	cancelled := make(chan bool)
 
 	var walkDir func(string, string)
 
@@ -92,7 +91,7 @@ func (fs *fsStorage) ListBlocks(prefix string) (<-chan storage.BlockMetadata, st
 				} else if fullID, ok := getstringFromFileName(currentPrefix + e.Name()); ok {
 					if strings.HasPrefix(fullID, prefix) {
 						select {
-						case <-cancelled:
+						case <-ctx.Done():
 							return
 						case result <- storage.BlockMetadata{
 							BlockID:   fullID,
@@ -112,12 +111,10 @@ func (fs *fsStorage) ListBlocks(prefix string) (<-chan storage.BlockMetadata, st
 	}
 
 	go walkDirAndClose(fs.Path)
-	return result, func() {
-		close(cancelled)
-	}
+	return result
 }
 
-func (fs *fsStorage) PutBlock(blockID string, data []byte) error {
+func (fs *fsStorage) PutBlock(ctx context.Context, blockID string, r io.Reader) error {
 	_, path := fs.getShardedPathAndFilePath(blockID)
 
 	tempFile := fmt.Sprintf("%s.tmp.%d", path, rand.Int())
@@ -125,7 +122,7 @@ func (fs *fsStorage) PutBlock(blockID string, data []byte) error {
 	if err != nil {
 		return fmt.Errorf("cannot create temporary file: %v", err)
 	}
-	if _, err = f.Write(data); err != nil {
+	if _, err = io.Copy(f, r); err != nil {
 		return fmt.Errorf("can't write temporary file: %v", err)
 	}
 	if err = f.Close(); err != nil {
@@ -162,7 +159,7 @@ func (fs *fsStorage) createTempFileAndDir(tempFile string) (*os.File, error) {
 	return f, err
 }
 
-func (fs *fsStorage) DeleteBlock(blockID string) error {
+func (fs *fsStorage) DeleteBlock(ctx context.Context, blockID string) error {
 	_, path := fs.getShardedPathAndFilePath(blockID)
 	err := os.Remove(path)
 	if err == nil || os.IsNotExist(err) {
@@ -198,7 +195,7 @@ func (fs *fsStorage) ConnectionInfo() storage.ConnectionInfo {
 	}
 }
 
-func (fs *fsStorage) Close() error {
+func (fs *fsStorage) Close(ctx context.Context) error {
 	return nil
 }
 

storage/filesystem/filesystem_storage_test.go

@@ -10,6 +10,8 @@ import (
 )
 
 func TestFileStorage(t *testing.T) {
+	ctx := context.Background()
+
 	// Test varioush shard configurations.
 	for _, shardSpec := range [][]int{
 		[]int{0},
@@ -23,7 +25,7 @@ func TestFileStorage(t *testing.T) {
 		path, _ := ioutil.TempDir("", "r-fs")
 		defer os.RemoveAll(path)
 
-		r, err := New(context.Background(), &Options{
+		r, err := New(ctx, &Options{
 			Path:            path,
 			DirectoryShards: shardSpec,
 		})
@@ -32,6 +34,6 @@ func TestFileStorage(t *testing.T) {
 			t.Errorf("unexpected result: %v %v", r, err)
 		}
 
-		storagetesting.VerifyStorage(t, r)
+		storagetesting.VerifyStorage(ctx, t, r)
 	}
 }

storage/gcs/gcs_storage.go

@@ -5,6 +5,7 @@ import (
 	"context"
 	"errors"
 	"fmt"
+	"io"
 	"io/ioutil"
 
 	"github.com/kopia/kopia/internal/retry"
@@ -37,7 +38,7 @@ type gcsStorage struct {
 	uploadThrottler   *iothrottler.IOThrottlerPool
 }
 
-func (gcs *gcsStorage) GetBlock(b string, offset, length int64) ([]byte, error) {
+func (gcs *gcsStorage) GetBlock(ctx context.Context, b string, offset, length int64) ([]byte, error) {
 	attempt := func() (interface{}, error) {
 		reader, err := gcs.bucket.Object(gcs.getObjectNameString(b)).NewRangeReader(gcs.ctx, offset, length)
 		if err != nil {
@@ -85,31 +86,27 @@ func translateError(err error) error {
 		return fmt.Errorf("unexpected GCS error: %v", err)
 	}
 }
+func (gcs *gcsStorage) PutBlock(ctx context.Context, b string, r io.Reader) error {
+	ctx, cancel := context.WithCancel(ctx)
 
-func (gcs *gcsStorage) PutBlock(b string, data []byte) error {
-	attempt := func() (interface{}, error) {
-		ctx, cancel := context.WithCancel(gcs.ctx)
+	writer := gcs.bucket.Object(gcs.getObjectNameString(b)).NewWriter(ctx)
+	writer.ChunkSize = 1 << 20
+	writer.ContentType = "application/x-kopia"
 
-		writer := gcs.bucket.Object(gcs.getObjectNameString(b)).NewWriter(ctx)
-		n, err := writer.Write(data)
-		if err != nil {
-			cancel()
-			return nil, err
-		}
-		if n != len(data) {
-			cancel()
-			return nil, fmt.Errorf("truncated write %v of %v bytes", n, len(data))
-		}
-
-		defer cancel()
-		return nil, writer.Close()
+	_, err := io.Copy(writer, r)
+	if err != nil {
+		// cancel context before closing the writer causes it to abandon the upload.
+		cancel()
+		writer.Close() //nolint:errcheck
+		return translateError(err)
 	}
+	defer cancel()
 
-	_, err := exponentialBackoff(fmt.Sprintf("PutBlock(%q)", b), attempt)
-	return translateError(err)
+	// calling close before cancel() causes it to commit the upload.
+	return translateError(writer.Close())
 }
 
-func (gcs *gcsStorage) DeleteBlock(b string) error {
+func (gcs *gcsStorage) DeleteBlock(ctx context.Context, b string) error {
 	attempt := func() (interface{}, error) {
 		return nil, gcs.bucket.Object(gcs.getObjectNameString(b)).Delete(gcs.ctx)
 	}
@@ -122,9 +119,8 @@ func (gcs *gcsStorage) getObjectNameString(blockID string) string {
 	return gcs.Prefix + blockID
 }
 
-func (gcs *gcsStorage) ListBlocks(prefix string) (<-chan storage.BlockMetadata, storage.CancelFunc) {
+func (gcs *gcsStorage) ListBlocks(ctx context.Context, prefix string) <-chan storage.BlockMetadata {
 	ch := make(chan storage.BlockMetadata, 100)
-	cancelled := make(chan bool)
 
 	go func() {
 		defer close(ch)
@@ -142,7 +138,7 @@ func (gcs *gcsStorage) ListBlocks(prefix string) (<-chan storage.BlockMetadata,
 			}
 			select {
 			case ch <- bm:
-			case <-cancelled:
+			case <-ctx.Done():
 				return
 			}
 			oa, err = lst.Next()
@@ -152,15 +148,13 @@ func (gcs *gcsStorage) ListBlocks(prefix string) (<-chan storage.BlockMetadata,
 			select {
 			case ch <- storage.BlockMetadata{Error: translateError(err)}:
 				return
-			case <-cancelled:
+			case <-ctx.Done():
 				return
 			}
 		}
 	}()
 
-	return ch, func() {
-		close(cancelled)
-	}
+	return ch
 }
 
 func (gcs *gcsStorage) ConnectionInfo() storage.ConnectionInfo {
@@ -170,7 +164,7 @@ func (gcs *gcsStorage) ConnectionInfo() storage.ConnectionInfo {
 	}
 }
 
-func (gcs *gcsStorage) Close() error {
+func (gcs *gcsStorage) Close(ctx context.Context) error {
 	gcs.storageClient.Close() //nolint:errcheck
 	return nil
 }

storage/logging/logging_storage.go

@@ -2,6 +2,8 @@
 package logging
 
 import (
+	"context"
+	"io"
 	"time"
 
 	"github.com/kopia/kopia/storage"
@@ -14,9 +16,9 @@ type loggingStorage struct {
 	prefix string
 }
 
-func (s *loggingStorage) GetBlock(id string, offset, length int64) ([]byte, error) {
+func (s *loggingStorage) GetBlock(ctx context.Context, id string, offset, length int64) ([]byte, error) {
 	t0 := time.Now()
-	result, err := s.base.GetBlock(id, offset, length)
+	result, err := s.base.GetBlock(ctx, id, offset, length)
 	dt := time.Since(t0)
 	if len(result) < 20 {
 		s.printf(s.prefix+"GetBlock(%q,%v,%v)=(%#v, %#v) took %v", id, offset, length, result, err, dt)
@@ -26,35 +28,32 @@ func (s *loggingStorage) GetBlock(id string, offset, length int64) ([]byte, erro
 	return result, err
 }
 
-func (s *loggingStorage) PutBlock(id string, data []byte) error {
+func (s *loggingStorage) PutBlock(ctx context.Context, id string, r io.Reader) error {
 	t0 := time.Now()
-	err := s.base.PutBlock(id, data)
+	err := s.base.PutBlock(ctx, id, r)
 	dt := time.Since(t0)
-	s.printf(s.prefix+"PutBlock(%q, len=%v)=%#v took %v", id, len(data), err, dt)
+	s.printf(s.prefix+"PutBlock(%q)=%#v took %v", id, err, dt)
 	return err
 }
 
-func (s *loggingStorage) DeleteBlock(id string) error {
+func (s *loggingStorage) DeleteBlock(ctx context.Context, id string) error {
 	t0 := time.Now()
-	err := s.base.DeleteBlock(id)
+	err := s.base.DeleteBlock(ctx, id)
 	dt := time.Since(t0)
 	s.printf(s.prefix+"DeleteBlock(%q)=%#v took %v", id, err, dt)
 	return err
 }
 
-func (s *loggingStorage) ListBlocks(prefix string) (<-chan storage.BlockMetadata, storage.CancelFunc) {
+func (s *loggingStorage) ListBlocks(ctx context.Context, prefix string) <-chan storage.BlockMetadata {
 	t0 := time.Now()
-	ch, cf := s.base.ListBlocks(prefix)
+	ch := s.base.ListBlocks(ctx, prefix)
 	s.printf(s.prefix+"ListBlocks(%q) took %v", prefix, time.Since(t0))
-	return ch, func() {
-		s.printf(s.prefix+"Cancelled ListBlocks(%q)after %v", prefix, time.Since(t0))
-		cf()
-	}
+	return ch
 }
 
-func (s *loggingStorage) Close() error {
+func (s *loggingStorage) Close(ctx context.Context) error {
 	t0 := time.Now()
-	err := s.base.Close()
+	err := s.base.Close(ctx)
 	dt := time.Since(t0)
 	s.printf(s.prefix+"Close()=%#v took %v", err, dt)
 	return err

storage/logging/logging_storage_test.go

@@ -1,6 +1,7 @@
 package logging
 
 import (
+	"context"
 	"testing"
 
 	"github.com/kopia/kopia/internal/storagetesting"
@@ -12,5 +13,5 @@ func TestLoggingStorage(t *testing.T) {
 	if r == nil {
 		t.Errorf("unexpected result: %v", r)
 	}
-	storagetesting.VerifyStorage(t, r)
+	storagetesting.VerifyStorage(context.Background(), t, r)
 }

storage/s3/s3_storage.go

@@ -5,6 +5,7 @@ import (
 	"context"
 	"errors"
 	"fmt"
+	"io"
 	"io/ioutil"
 
 	"github.com/efarrer/iothrottler"
@@ -28,7 +29,7 @@ type s3Storage struct {
 	uploadThrottler   *iothrottler.IOThrottlerPool
 }
 
-func (s *s3Storage) GetBlock(b string, offset, length int64) ([]byte, error) {
+func (s *s3Storage) GetBlock(ctx context.Context, b string, offset, length int64) ([]byte, error) {
 	attempt := func() (interface{}, error) {
 		var opt minio.GetObjectOptions
 		if length > 0 {
@@ -92,31 +93,21 @@ func translateError(err error) error {
 	}
 }
 
-func (s *s3Storage) PutBlock(b string, data []byte) error {
-	attempt := func() (interface{}, error) {
-		rc := ioutil.NopCloser(bytes.NewReader(data))
-		throttled, err := s.uploadThrottler.AddReader(rc)
-		if err != nil {
-			return nil, err
-		}
-
-		n, err := s.cli.PutObject(s.BucketName, s.getObjectNameString(b), throttled, int64(len(data)), minio.PutObjectOptions{})
-		if err != nil {
-			return nil, err
-		}
-
-		if n != int64(len(data)) {
-			return nil, fmt.Errorf("truncated write %v of %v bytes", n, len(data))
-		}
-
-		return nil, nil
+func (s *s3Storage) PutBlock(ctx context.Context, b string, r io.Reader) error {
+	throttled, err := s.uploadThrottler.AddReader(ioutil.NopCloser(r))
+	if err != nil {
+		return err
 	}
 
-	_, err := exponentialBackoff(fmt.Sprintf("PutBlock(%q)", b), attempt)
+	n, err := s.cli.PutObject(s.BucketName, s.getObjectNameString(b), throttled, -1, minio.PutObjectOptions{})
+	if err == io.EOF && n == 0 {
+		// special case empty stream
+		_, err = s.cli.PutObject(s.BucketName, s.getObjectNameString(b), bytes.NewBuffer(nil), 0, minio.PutObjectOptions{})
+	}
 	return translateError(err)
 }
 
-func (s *s3Storage) DeleteBlock(b string) error {
+func (s *s3Storage) DeleteBlock(ctx context.Context, b string) error {
 	attempt := func() (interface{}, error) {
 		return nil, s.cli.RemoveObject(s.BucketName, s.getObjectNameString(b))
 	}
@@ -129,20 +120,19 @@ func (s *s3Storage) getObjectNameString(b string) string {
 	return s.Prefix + b
 }
 
-func (s *s3Storage) ListBlocks(prefix string) (<-chan storage.BlockMetadata, storage.CancelFunc) {
+func (s *s3Storage) ListBlocks(ctx context.Context, prefix string) <-chan storage.BlockMetadata {
 	ch := make(chan storage.BlockMetadata, 100)
-	cancelled := make(chan struct{})
 
 	go func() {
 		defer close(ch)
 
-		oi := s.cli.ListObjects(s.BucketName, s.Prefix+prefix, false, cancelled)
+		oi := s.cli.ListObjects(s.BucketName, s.Prefix+prefix, false, ctx.Done())
 		for o := range oi {
 			if err := o.Err; err != nil {
 				select {
 				case ch <- storage.BlockMetadata{Error: translateError(err)}:
 					return
-				case <-cancelled:
+				case <-ctx.Done():
 					return
 				}
 			}
@@ -155,15 +145,13 @@ func (s *s3Storage) ListBlocks(prefix string) (<-chan storage.BlockMetadata, sto
 
 			select {
 			case ch <- bm:
-			case <-cancelled:
+			case <-ctx.Done():
 				return
 			}
 		}
 	}()
 
-	return ch, func() {
-		close(cancelled)
-	}
+	return ch
 }
 
 func (s *s3Storage) ConnectionInfo() storage.ConnectionInfo {
@@ -173,7 +161,7 @@ func (s *s3Storage) ConnectionInfo() storage.ConnectionInfo {
 	}
 }
 
-func (s *s3Storage) Close() error {
+func (s *s3Storage) Close(ctx context.Context) error {
 	return nil
 }
 

storage/s3/s3_storage_test.go

@@ -42,9 +42,11 @@ func TestS3Storage(t *testing.T) {
 		return
 	}
 
+	ctx := context.Background()
+
 	// recreate per-host bucket, which sometimes get cleaned up by play.minio.io
 	createBucket(t)
-	cleanupOldData(t)
+	cleanupOldData(ctx, t)
 
 	data := make([]byte, 8)
 	rand.Read(data)
@@ -60,7 +62,7 @@ func TestS3Storage(t *testing.T) {
 		t.Fatalf("err: %v", err)
 	}
 
-	storagetesting.VerifyStorage(t, st)
+	storagetesting.VerifyStorage(ctx, t, st)
 }
 
 func createBucket(t *testing.T) {
@@ -71,7 +73,7 @@ func createBucket(t *testing.T) {
 	minioClient.MakeBucket(bucketName, "us-east-1")
 }
 
-func cleanupOldData(t *testing.T) {
+func cleanupOldData(ctx context.Context, t *testing.T) {
 	// cleanup old data from the bucket
 	st, err := New(context.Background(), &Options{
 		AccessKeyID:     accessKeyID,
@@ -83,7 +85,8 @@ func cleanupOldData(t *testing.T) {
 		t.Fatalf("err: %v", err)
 	}
 
-	items, cancel := st.ListBlocks("")
+	ctx, cancel := context.WithCancel(ctx)
+	items := st.ListBlocks(ctx, "")
 	defer cancel()
 	for it := range items {
 		if it.Error != nil {
@@ -93,7 +96,7 @@ func cleanupOldData(t *testing.T) {
 
 		age := time.Since(it.TimeStamp)
 		if age > cleanupAge {
-			if err := st.DeleteBlock(it.BlockID); err != nil {
+			if err := st.DeleteBlock(ctx, it.BlockID); err != nil {
 				t.Errorf("warning: unable to delete %q: %v", it.BlockID, err)
 			}
 		} else {

storage/storage.go

@@ -1,6 +1,7 @@
 package storage
 
 import (
+	"context"
 	"errors"
 	"io"
 	"time"
@@ -21,27 +22,28 @@ type CancelFunc func()
 //
 // The required semantics are provided by existing commercial cloud storage products (Google Cloud, AWS, Azure).
 type Storage interface {
-	io.Closer
-
 	// PutBlock uploads the block with given data to the repository or replaces existing block with the provided
 	// id with given contents.
-	PutBlock(id string, data []byte) error
+	PutBlock(ctx context.Context, id string, data io.Reader) error
 
 	// DeleteBlock removes the block from storage. Future GetBlock() operations will fail with ErrBlockNotFound.
-	DeleteBlock(id string) error
+	DeleteBlock(ctx context.Context, id string) error
 
 	// GetBlock returns full or partial contents of a block with given ID.
 	// If length>0, the the function retrieves a range of bytes [offset,offset+length)
 	// If length<0, the entire block must be fetched.
-	GetBlock(id string, offset, length int64) ([]byte, error)
+	GetBlock(ctx context.Context, id string, offset, length int64) ([]byte, error)
 
 	// ListBlocks returns a channel of BlockMetadata that describes storage blocks with existing name prefixes.
 	// Iteration continues until all blocks have been listed or until client code invokes the returned cancellation function.
-	ListBlocks(prefix string) (<-chan BlockMetadata, CancelFunc)
+	ListBlocks(ctx context.Context, prefix string) <-chan BlockMetadata
 
 	// ConnectionInfo returns JSON-serializable data structure containing information required to
 	// connect to storage.
 	ConnectionInfo() ConnectionInfo
+
+	// Close releases all resources associated with storage.
+	Close(ctx context.Context) error
 }
 
 // BlockMetadata represents metadata about a single block in a storage.

storage/webdav/webdav_storage.go

@@ -2,8 +2,10 @@
 package webdav
 
 import (
+	"bytes"
 	"context"
 	"fmt"
+	"io"
 	"io/ioutil"
 	"net/http"
 	"os"
@@ -33,7 +35,7 @@ type davStorage struct {
 	Client *http.Client // HTTP client used when making all calls, may be overridden to use custom auth
 }
 
-func (d *davStorage) GetBlock(blockID string, offset, length int64) ([]byte, error) {
+func (d *davStorage) GetBlock(ctx context.Context, blockID string, offset, length int64) ([]byte, error) {
 	_, urlStr := d.getCollectionAndFileURL(blockID)
 
 	req, err := http.NewRequest("GET", urlStr, nil)
@@ -74,9 +76,8 @@ func makeFileName(blockID string) string {
 	return blockID + fsStorageChunkSuffix
 }
 
-func (d *davStorage) ListBlocks(prefix string) (<-chan storage.BlockMetadata, storage.CancelFunc) {
+func (d *davStorage) ListBlocks(ctx context.Context, prefix string) <-chan storage.BlockMetadata {
 	result := make(chan storage.BlockMetadata)
-	cancelled := make(chan bool)
 
 	var walkDir func(string, string)
 
@@ -99,7 +100,7 @@ func (d *davStorage) ListBlocks(prefix string) (<-chan storage.BlockMetadata, st
 				} else if fullID, ok := getBlockIDFromFileName(currentPrefix + e.name); ok {
 					if strings.HasPrefix(fullID, prefix) {
 						select {
-						case <-cancelled:
+						case <-ctx.Done():
 							return
 						case result <- storage.BlockMetadata{
 							BlockID:   fullID,
@@ -119,9 +120,7 @@ func (d *davStorage) ListBlocks(prefix string) (<-chan storage.BlockMetadata, st
 	}
 
 	go walkDirAndClose(d.URL)
-	return result, func() {
-		close(cancelled)
-	}
+	return result
 }
 
 func (d *davStorage) makeCollectionAll(urlStr string) error {
@@ -244,11 +243,19 @@ func (d *davStorage) putBlockInternal(urlStr string, data []byte) error {
 	}
 }
 
-func (d *davStorage) PutBlock(blockID string, data []byte) error {
+func (d *davStorage) PutBlock(ctx context.Context, blockID string, r io.Reader) error {
 	shardPath, url := d.getCollectionAndFileURL(blockID)
 
+	var buf bytes.Buffer
+	_, err := buf.ReadFrom(r)
+	if err != nil {
+		return err
+	}
+
+	data := buf.Bytes()
+
 	tmpURL := url + "-" + makeClientNonce()
-	err := d.putBlockInternal(tmpURL, data)
+	err = d.putBlockInternal(tmpURL, data)
 
 	if err == storage.ErrBlockNotFound {
 		if err = d.makeCollectionAll(shardPath); err != nil {
@@ -272,7 +279,7 @@ func (d *davStorage) PutBlock(blockID string, data []byte) error {
 	return nil
 }
 
-func (d *davStorage) DeleteBlock(blockID string) error {
+func (d *davStorage) DeleteBlock(ctx context.Context, blockID string) error {
 	_, url := d.getCollectionAndFileURL(blockID)
 	err := os.Remove(url)
 	if err == nil || os.IsNotExist(err) {
@@ -308,7 +315,7 @@ func (d *davStorage) ConnectionInfo() storage.ConnectionInfo {
 	}
 }
 
-func (d *davStorage) Close() error {
+func (d *davStorage) Close(ctx context.Context) error {
 	return nil
 }
 

storage/webdav/webdav_storage_test.go

@@ -27,6 +27,8 @@ func TestWebDAVStorage(t *testing.T) {
 	server := httptest.NewServer(mux)
 	defer server.Close()
 
+	ctx := context.Background()
+
 	// Test varioush shard configurations.
 	for _, shardSpec := range [][]int{
 		[]int{1},
@@ -50,6 +52,6 @@ func TestWebDAVStorage(t *testing.T) {
 			t.Errorf("unexpected result: %v %v", r, err)
 		}
 
-		storagetesting.VerifyStorage(t, r)
+		storagetesting.VerifyStorage(ctx, t, r)
 	}
 }