diff --git a/internal/cache/content_cache.go b/internal/cache/content_cache.go index 10a04e014..61180c5ab 100644 --- a/internal/cache/content_cache.go +++ b/internal/cache/content_cache.go @@ -2,8 +2,12 @@ import ( "context" + "path/filepath" + + "github.com/pkg/errors" "github.com/kopia/kopia/internal/gather" + "github.com/kopia/kopia/internal/impossible" "github.com/kopia/kopia/repo/blob" ) @@ -15,9 +19,186 @@ type ContentCache interface { CacheStorage() Storage } -// SyncableContentCache caches contents stored in pack blobs and supports synchronizing. -type SyncableContentCache interface { - ContentCache - - Sync(ctx context.Context, blobPrefix blob.ID) error +// Options encapsulates all content cache options. +type Options struct { + BaseCacheDirectory string + CacheSubDir string + Storage Storage // force particular storage, used for testing + HMACSecret []byte + FetchFullBlobs bool + Sweep SweepSettings +} + +type contentCacheImpl struct { + pc *PersistentCache + st blob.Storage + fetchFullBlobs bool +} + +// ContentIDCacheKey computes the cache key for the provided content ID. +func ContentIDCacheKey(contentID string) string { + // move the prefix to the end of cache key to make sure the top level shard is spread 256 ways. + if contentID[0] >= 'g' && contentID[0] <= 'z' { + return contentID[1:] + contentID[0:1] + } + + return contentID +} + +// BlobIDCacheKey computes the cache key for the provided blob ID. +func BlobIDCacheKey(id blob.ID) string { + return string(id[1:] + id[0:1]) +} + +func (c *contentCacheImpl) GetContent(ctx context.Context, contentID string, blobID blob.ID, offset, length int64, output *gather.WriteBuffer) error { + if c.fetchFullBlobs { + return c.getContentFromFullBlob(ctx, blobID, offset, length, output) + } + + return c.getContentFromFullOrPartialBlob(ctx, contentID, blobID, offset, length, output) +} + +func (c *contentCacheImpl) getContentFromFullBlob(ctx context.Context, blobID blob.ID, offset, length int64, output *gather.WriteBuffer) error { + // acquire exclusive lock + mut := c.pc.GetFetchingMutex(string(blobID)) + mut.Lock() + defer mut.Unlock() + + // check again to see if we perhaps lost the race and the data is now in cache. + if c.pc.GetPartial(ctx, BlobIDCacheKey(blobID), offset, length, output) { + return nil + } + + var blobData gather.WriteBuffer + defer blobData.Close() + + if err := c.fetchBlobInternal(ctx, blobID, &blobData); err != nil { + return err + } + + if offset == 0 && length == -1 { + _, err := blobData.Bytes().WriteTo(output) + + return errors.Wrap(err, "error copying results") + } + + if offset < 0 || offset+length > int64(blobData.Length()) { + return errors.Errorf("invalid (offset=%v,length=%v) for blob %q of size %v", offset, length, blobID, blobData.Length()) + } + + output.Reset() + + impossible.PanicOnError(blobData.AppendSectionTo(output, int(offset), int(length))) + + return nil +} + +func (c *contentCacheImpl) fetchBlobInternal(ctx context.Context, blobID blob.ID, blobData *gather.WriteBuffer) error { + // read the entire blob + if err := c.st.GetBlob(ctx, blobID, 0, -1, blobData); err != nil { + reportMissError() + + // nolint:wrapcheck + return err + } + + reportMissBytes(int64(blobData.Length())) + + // store the whole blob in the cache. + c.pc.Put(ctx, BlobIDCacheKey(blobID), blobData.Bytes()) + + return nil +} + +func (c *contentCacheImpl) getContentFromFullOrPartialBlob(ctx context.Context, contentID string, blobID blob.ID, offset, length int64, output *gather.WriteBuffer) error { + // acquire shared lock on a blob, PrefetchBlob will acquire exclusive lock here. + mut := c.pc.GetFetchingMutex(string(blobID)) + mut.RLock() + defer mut.RUnlock() + + // see if we have the full blob cached by extracting a partial range. + if c.pc.GetPartial(ctx, BlobIDCacheKey(blobID), offset, length, output) { + return nil + } + + // acquire exclusive lock on the content + mut2 := c.pc.GetFetchingMutex(contentID) + mut2.Lock() + defer mut2.Unlock() + + output.Reset() + + if c.pc.GetFull(ctx, ContentIDCacheKey(contentID), output) { + return nil + } + + if err := c.st.GetBlob(ctx, blobID, offset, length, output); err != nil { + reportMissError() + + // nolint:wrapcheck + return err + } + + reportMissBytes(int64(output.Length())) + + c.pc.Put(ctx, ContentIDCacheKey(contentID), output.Bytes()) + + return nil +} + +func (c *contentCacheImpl) Close(ctx context.Context) { + c.pc.Close(ctx) +} + +func (c *contentCacheImpl) PrefetchBlob(ctx context.Context, blobID blob.ID) error { + var blobData gather.WriteBuffer + defer blobData.Close() + + // see if it's already cached before taking a lock + if c.pc.GetPartial(ctx, BlobIDCacheKey(blobID), 0, 1, &blobData) { + return nil + } + + // acquire exclusive lock for the blob. + mut := c.pc.GetFetchingMutex(string(blobID)) + mut.Lock() + defer mut.Unlock() + + if c.pc.GetPartial(ctx, BlobIDCacheKey(blobID), 0, 1, &blobData) { + return nil + } + + return c.fetchBlobInternal(ctx, blobID, &blobData) +} + +func (c *contentCacheImpl) CacheStorage() Storage { + return c.pc.cacheStorage +} + +// NewContentCache creates new content cache for data contents. +func NewContentCache(ctx context.Context, st blob.Storage, opt Options) (ContentCache, error) { + cacheStorage := opt.Storage + if cacheStorage == nil { + if opt.BaseCacheDirectory == "" { + return passthroughContentCache{st}, nil + } + + var err error + + cacheStorage, err = NewStorageOrNil(ctx, filepath.Join(opt.BaseCacheDirectory, opt.CacheSubDir), opt.Sweep.MaxSizeBytes, opt.CacheSubDir) + if err != nil { + return nil, errors.Wrap(err, "error initializing cache storage") + } + } + + pc, err := NewPersistentCache(ctx, opt.CacheSubDir, cacheStorage, ChecksumProtection(opt.HMACSecret), opt.Sweep) + if err != nil { + return nil, errors.Wrap(err, "unable to create base cache") + } + + return &contentCacheImpl{ + st: st, + pc: pc, + fetchFullBlobs: opt.FetchFullBlobs, + }, nil } diff --git a/internal/cache/content_cache_concurrency_test.go b/internal/cache/content_cache_concurrency_test.go index ad21fa3ea..48fc7aaf5 100644 --- a/internal/cache/content_cache_concurrency_test.go +++ b/internal/cache/content_cache_concurrency_test.go @@ -21,42 +21,58 @@ type newContentCacheFunc func(ctx context.Context, st blob.Storage, cacheStorage cache.Storage) (cache.ContentCache, error) func newContentDataCache(ctx context.Context, st blob.Storage, cacheStorage cache.Storage) (cache.ContentCache, error) { - return cache.NewContentCacheForData(ctx, st, cacheStorage, cache.SweepSettings{ - MaxSizeBytes: 100, - }, []byte{1, 2, 3, 4}) + return cache.NewContentCache(ctx, st, cache.Options{ + Storage: cacheStorage, + HMACSecret: []byte{1, 2, 3, 4}, + Sweep: cache.SweepSettings{ + MaxSizeBytes: 100, + }, + }) } func newContentMetadataCache(ctx context.Context, st blob.Storage, cacheStorage cache.Storage) (cache.ContentCache, error) { - return cache.NewContentCacheForMetadata(ctx, st, cacheStorage, cache.SweepSettings{ - MaxSizeBytes: 100, + return cache.NewContentCache(ctx, st, cache.Options{ + Storage: cacheStorage, + HMACSecret: []byte{1, 2, 3, 4}, + FetchFullBlobs: true, + Sweep: cache.SweepSettings{ + MaxSizeBytes: 100, + }, }) } func TestPrefetchBlocksGetContent_DataCache(t *testing.T) { + t.Parallel() testContentCachePrefetchBlocksGetContent(t, newContentDataCache) } func TestPrefetchBlocksGetContent_MetadataCache(t *testing.T) { + t.Parallel() testContentCachePrefetchBlocksGetContent(t, newContentMetadataCache) } func TestGetContentForDifferentContentIDsExecutesInParallel_DataCache(t *testing.T) { + t.Parallel() testGetContentForDifferentContentIDsExecutesInParallel(t, newContentDataCache, 2) } func TestGetContentForDifferentContentIDsExecutesInParallel_MetadataCache(t *testing.T) { + t.Parallel() testGetContentForDifferentContentIDsExecutesInParallel(t, newContentMetadataCache, 1) } func TestGetContentForDifferentBlobsExecutesInParallel_DataCache(t *testing.T) { + t.Parallel() testGetContentForDifferentBlobsExecutesInParallel(t, newContentDataCache) } func TestGetContentForDifferentBlobsExecutesInParallel_MetadataCache(t *testing.T) { + t.Parallel() testGetContentForDifferentBlobsExecutesInParallel(t, newContentMetadataCache) } func TestGetContentRaceFetchesOnce_DataCache(t *testing.T) { + t.Parallel() testGetContentRaceFetchesOnce(t, newContentDataCache) } @@ -249,7 +265,6 @@ func testGetContentRaceFetchesOnce(t *testing.T, newCache newContentCacheFunc) { require.NoError(t, underlying.PutBlob(ctx, "blob1", gather.FromSlice([]byte{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10}), blob.PutOptions{})) faulty.AddFault(blobtesting.MethodGetBlob).Before(func() { - t.Logf("stack1: %s", debug.Stack()) time.Sleep(time.Second) }) diff --git a/internal/cache/content_cache_data.go b/internal/cache/content_cache_data.go deleted file mode 100644 index a48866d11..000000000 --- a/internal/cache/content_cache_data.go +++ /dev/null @@ -1,106 +0,0 @@ -package cache - -import ( - "context" - - "github.com/pkg/errors" - - "github.com/kopia/kopia/internal/gather" - "github.com/kopia/kopia/repo/blob" -) - -type contentCacheForData struct { - pc *PersistentCache - st blob.Storage -} - -// ContentIDCacheKey computes the cache key for the provided content ID. -func ContentIDCacheKey(contentID string) string { - // move the prefix to the end of cache key to make sure the top level shard is spread 256 ways. - if contentID[0] >= 'g' && contentID[0] <= 'z' { - return contentID[1:] + contentID[0:1] - } - - return contentID -} - -// BlobIDCacheKey computes the cache key for the provided blob ID. -func BlobIDCacheKey(id blob.ID) string { - return string(id[1:] + id[0:1]) -} - -func (c *contentCacheForData) GetContent(ctx context.Context, contentID string, blobID blob.ID, offset, length int64, output *gather.WriteBuffer) error { - // acquire shared lock - mut := c.pc.GetFetchingMutex(string(blobID)) - mut.RLock() - defer mut.RUnlock() - - if c.pc.GetPartial(ctx, BlobIDCacheKey(blobID), offset, length, output) { - return nil - } - - output.Reset() - - // nolint:wrapcheck - return c.pc.GetOrLoad(ctx, ContentIDCacheKey(contentID), func(output *gather.WriteBuffer) error { - // nolint:wrapcheck - return c.st.GetBlob(ctx, blobID, offset, length, output) - }, output) -} - -func (c *contentCacheForData) Close(ctx context.Context) { - c.pc.Close(ctx) -} - -func (c *contentCacheForData) PrefetchBlob(ctx context.Context, blobID blob.ID) error { - var blobData gather.WriteBuffer - defer blobData.Close() - - if c.pc.GetPartial(ctx, BlobIDCacheKey(blobID), 0, 1, &blobData) { - return nil - } - - // acquire exclusive lock - mut := c.pc.GetFetchingMutex(string(blobID)) - mut.Lock() - defer mut.Unlock() - - if c.pc.GetPartial(ctx, BlobIDCacheKey(blobID), 0, 1, &blobData) { - return nil - } - - // read the entire blob - if err := c.st.GetBlob(ctx, blobID, 0, -1, &blobData); err != nil { - reportMissError() - // nolint:wrapcheck - return err - } - - reportMissBytes(int64(blobData.Length())) - - // store the whole blob in the - c.pc.Put(ctx, BlobIDCacheKey(blobID), blobData.Bytes()) - - return nil -} - -func (c *contentCacheForData) CacheStorage() Storage { - return c.pc.cacheStorage -} - -// NewContentCacheForData creates new content cache for data contents. -func NewContentCacheForData(ctx context.Context, st blob.Storage, cacheStorage Storage, sweep SweepSettings, hmacSecret []byte) (ContentCache, error) { - if cacheStorage == nil { - return passthroughContentCache{st}, nil - } - - pc, err := NewPersistentCache(ctx, "content cache", cacheStorage, ChecksumProtection(hmacSecret), sweep) - if err != nil { - return nil, errors.Wrap(err, "unable to create base cache") - } - - return &contentCacheForData{ - st: st, - pc: pc, - }, nil -} diff --git a/internal/cache/content_cache_data_test.go b/internal/cache/content_cache_data_test.go index 835ff2892..64c693854 100644 --- a/internal/cache/content_cache_data_test.go +++ b/internal/cache/content_cache_data_test.go @@ -19,11 +19,15 @@ func TestContentCacheForData(t *testing.T) { underlying := blobtesting.NewMapStorage(underlyingData, nil, nil) cacheData := blobtesting.DataMap{} - metadataCacheStorage := blobtesting.NewMapStorage(cacheData, nil, nil).(cache.Storage) + cacheStorage := blobtesting.NewMapStorage(cacheData, nil, nil).(cache.Storage) - dataCache, err := cache.NewContentCacheForData(ctx, underlying, metadataCacheStorage, cache.SweepSettings{ - MaxSizeBytes: 100, - }, []byte{1, 2, 3, 4}) + dataCache, err := cache.NewContentCache(ctx, underlying, cache.Options{ + Storage: cacheStorage, + HMACSecret: []byte{1, 2, 3, 4}, + Sweep: cache.SweepSettings{ + MaxSizeBytes: 100, + }, + }) require.NoError(t, err) var tmp gather.WriteBuffer @@ -74,9 +78,7 @@ func TestContentCacheForData_Passthrough(t *testing.T) { ctx := testlogging.Context(t) - dataCache, err := cache.NewContentCacheForData(ctx, underlying, nil, cache.SweepSettings{ - MaxSizeBytes: 100, - }, []byte{1, 2, 3, 4}) + dataCache, err := cache.NewContentCache(ctx, underlying, cache.Options{}) require.NoError(t, err) require.NoError(t, underlying.PutBlob(ctx, "blob1", gather.FromSlice([]byte{1, 2, 3, 4, 5, 6}), blob.PutOptions{})) diff --git a/internal/cache/content_cache_metadata.go b/internal/cache/content_cache_metadata.go deleted file mode 100644 index edace1480..000000000 --- a/internal/cache/content_cache_metadata.go +++ /dev/null @@ -1,116 +0,0 @@ -package cache - -import ( - "context" - - "github.com/pkg/errors" - - "github.com/kopia/kopia/internal/gather" - "github.com/kopia/kopia/internal/impossible" - "github.com/kopia/kopia/repo/blob" -) - -type contentCacheForMetadata struct { - pc *PersistentCache - - st blob.Storage -} - -func (c *contentCacheForMetadata) GetContent(ctx context.Context, contentID string, blobID blob.ID, offset, length int64, output *gather.WriteBuffer) error { - // try getting from cache first - if c.pc.GetPartial(ctx, string(blobID), offset, length, output) { - return nil - } - - // acquire exclusive lock - mut := c.pc.GetFetchingMutex(string(blobID)) - mut.Lock() - defer mut.Unlock() - - // check again to see if we perhaps lost the race and the data is now in cache. - if c.pc.GetPartial(ctx, string(blobID), offset, length, output) { - return nil - } - - var blobData gather.WriteBuffer - defer blobData.Close() - - if err := c.fetchBlobInternal(ctx, blobID, &blobData); err != nil { - return err - } - - if offset == 0 && length == -1 { - _, err := blobData.Bytes().WriteTo(output) - - return errors.Wrap(err, "error copying results") - } - - if offset < 0 || offset+length > int64(blobData.Length()) { - return errors.Errorf("invalid (offset=%v,length=%v) for blob %q of size %v", offset, length, blobID, blobData.Length()) - } - - output.Reset() - - impossible.PanicOnError(blobData.AppendSectionTo(output, int(offset), int(length))) - - return nil -} - -func (c *contentCacheForMetadata) PrefetchBlob(ctx context.Context, blobID blob.ID) error { - var blobData gather.WriteBuffer - defer blobData.Close() - - // acquire exclusive lock - mut := c.pc.GetFetchingMutex(string(blobID)) - mut.Lock() - defer mut.Unlock() - - // check to see if the data is now in cache. - if c.pc.GetPartial(ctx, string(blobID), 0, 1, &blobData) { - return nil - } - - return c.fetchBlobInternal(ctx, blobID, &blobData) -} - -func (c *contentCacheForMetadata) fetchBlobInternal(ctx context.Context, blobID blob.ID, blobData *gather.WriteBuffer) error { - // read the entire blob - if err := c.st.GetBlob(ctx, blobID, 0, -1, blobData); err != nil { - reportMissError() - - // nolint:wrapcheck - return err - } - - reportMissBytes(int64(blobData.Length())) - - // store the whole blob in the cache. - c.pc.Put(ctx, string(blobID), blobData.Bytes()) - - return nil -} - -func (c *contentCacheForMetadata) Close(ctx context.Context) { - c.pc.Close(ctx) -} - -func (c *contentCacheForMetadata) CacheStorage() Storage { - return c.pc.cacheStorage -} - -// NewContentCacheForMetadata creates new content cache for metadata contents. -func NewContentCacheForMetadata(ctx context.Context, st blob.Storage, cacheStorage Storage, sweep SweepSettings) (ContentCache, error) { - if cacheStorage == nil { - return passthroughContentCache{st}, nil - } - - pc, err := NewPersistentCache(ctx, "metadata cache", cacheStorage, NoProtection(), sweep) - if err != nil { - return nil, errors.Wrap(err, "unable to create base cache") - } - - return &contentCacheForMetadata{ - st: st, - pc: pc, - }, nil -} diff --git a/internal/cache/content_cache_metadata_test.go b/internal/cache/content_cache_metadata_test.go index 75b72dd30..069c2cd37 100644 --- a/internal/cache/content_cache_metadata_test.go +++ b/internal/cache/content_cache_metadata_test.go @@ -9,6 +9,7 @@ "github.com/kopia/kopia/internal/cache" "github.com/kopia/kopia/internal/gather" "github.com/kopia/kopia/internal/testlogging" + "github.com/kopia/kopia/internal/testutil" "github.com/kopia/kopia/repo/blob" ) @@ -18,14 +19,21 @@ func TestContentCacheForMetadata(t *testing.T) { underlyingData := blobtesting.DataMap{} underlying := blobtesting.NewMapStorage(underlyingData, nil, nil) - cacheData := blobtesting.DataMap{} - metadataCacheStorage := blobtesting.NewMapStorage(cacheData, nil, nil).(cache.Storage) + td := testutil.TempDirectory(t) - metadataCache, err := cache.NewContentCacheForMetadata(ctx, underlying, metadataCacheStorage, cache.SweepSettings{ - MaxSizeBytes: 100, + metadataCache, err := cache.NewContentCache(ctx, underlying, cache.Options{ + BaseCacheDirectory: td, + CacheSubDir: "subdir", + HMACSecret: []byte{1, 2, 3}, + FetchFullBlobs: true, + Sweep: cache.SweepSettings{ + MaxSizeBytes: 100, + }, }) require.NoError(t, err) + cacheStorage := metadataCache.CacheStorage() + var tmp gather.WriteBuffer defer tmp.Close() @@ -40,17 +48,18 @@ func TestContentCacheForMetadata(t *testing.T) { require.NoError(t, metadataCache.GetContent(ctx, "key1", "blob1", 0, 3, &tmp)) require.Equal(t, []byte{1, 2, 3}, tmp.ToByteSlice()) + cacheEntries, err := blob.ListAllBlobs(ctx, cacheStorage, "") + require.NoError(t, err) // cache has the entire blob - require.Len(t, cacheData, 1) + require.Len(t, cacheEntries, 1) require.NoError(t, metadataCache.GetContent(ctx, "key1", "blob1", 3, 3, &tmp)) require.Equal(t, []byte{4, 5, 6}, tmp.ToByteSlice()) - // out of bounds - require.Error(t, metadataCache.GetContent(ctx, "key1", "blob1", 3, 9, &tmp)) - + cacheEntries, err = blob.ListAllBlobs(ctx, cacheStorage, "") + require.NoError(t, err) // cache has the entire blob - require.Len(t, cacheData, 1) + require.Len(t, cacheEntries, 1) metadataCache.Close(ctx) @@ -66,8 +75,11 @@ func TestContentCacheForMetadata_Passthrough(t *testing.T) { ctx := testlogging.Context(t) - metadataCache, err := cache.NewContentCacheForMetadata(ctx, underlying, nil, cache.SweepSettings{ - MaxSizeBytes: 100, + metadataCache, err := cache.NewContentCache(ctx, underlying, cache.Options{ + BaseCacheDirectory: "", + Sweep: cache.SweepSettings{ + MaxSizeBytes: 100, + }, }) require.NoError(t, err) diff --git a/internal/cache/content_cache_test.go b/internal/cache/content_cache_test.go index 22236569d..fbe3e52ae 100644 --- a/internal/cache/content_cache_test.go +++ b/internal/cache/content_cache_test.go @@ -56,11 +56,14 @@ func TestCacheExpiration(t *testing.T) { underlyingStorage := newUnderlyingStorageForContentCacheTesting(t) ctx := testlogging.Context(t) - cc, err := cache.NewContentCacheForData(ctx, underlyingStorage, cacheStorage.(cache.Storage), cache.SweepSettings{ - MaxSizeBytes: 10000, - SweepFrequency: 500 * time.Millisecond, - TouchThreshold: -1, - }, nil) + cc, err := cache.NewContentCache(ctx, underlyingStorage, cache.Options{ + Storage: cacheStorage.(cache.Storage), + Sweep: cache.SweepSettings{ + MaxSizeBytes: 10000, + SweepFrequency: 500 * time.Millisecond, + TouchThreshold: -1, + }, + }) require.NoError(t, err) @@ -118,9 +121,12 @@ func TestDiskContentCache(t *testing.T) { t.Fatal(err) } - cc, err := cache.NewContentCacheForData(ctx, newUnderlyingStorageForContentCacheTesting(t), cacheStorage, cache.SweepSettings{ - MaxSizeBytes: maxBytes, - }, nil) + cc, err := cache.NewContentCache(ctx, newUnderlyingStorageForContentCacheTesting(t), cache.Options{ + Storage: cacheStorage, + Sweep: cache.SweepSettings{ + MaxSizeBytes: maxBytes, + }, + }) if err != nil { t.Fatalf("err: %v", err) } @@ -209,7 +215,10 @@ func TestCacheFailureToOpen(t *testing.T) { faultyCache.AddFault(blobtesting.MethodGetMetadata).ErrorInstead(someError) // Will fail because of ListBlobs failure. - _, err := cache.NewContentCacheForData(testlogging.Context(t), underlyingStorage, withoutTouchBlob{faultyCache}, cache.SweepSettings{MaxSizeBytes: 10000}, nil) + _, err := cache.NewContentCache(testlogging.Context(t), underlyingStorage, cache.Options{ + Storage: withoutTouchBlob{faultyCache}, + Sweep: cache.SweepSettings{MaxSizeBytes: 10000}, + }) if err == nil || !strings.Contains(err.Error(), someError.Error()) { t.Errorf("invalid error %v, wanted: %v", err, someError) } @@ -217,7 +226,10 @@ func TestCacheFailureToOpen(t *testing.T) { // ListBlobs fails only once, next time it succeeds. ctx := testlogging.Context(t) - cc, err := cache.NewContentCacheForData(ctx, underlyingStorage, withoutTouchBlob{faultyCache}, cache.SweepSettings{MaxSizeBytes: 10000}, nil) + cc, err := cache.NewContentCache(ctx, underlyingStorage, cache.Options{ + Storage: withoutTouchBlob{faultyCache}, + Sweep: cache.SweepSettings{MaxSizeBytes: 10000}, + }) if err != nil { t.Fatalf("err: %v", err) } @@ -233,7 +245,10 @@ func TestCacheFailureToWrite(t *testing.T) { underlyingStorage := newUnderlyingStorageForContentCacheTesting(t) faultyCache := blobtesting.NewFaultyStorage(cacheStorage) - cc, err := cache.NewContentCacheForData(testlogging.Context(t), underlyingStorage, withoutTouchBlob{faultyCache}, cache.SweepSettings{MaxSizeBytes: 10000}, nil) + cc, err := cache.NewContentCache(testlogging.Context(t), underlyingStorage, cache.Options{ + Storage: withoutTouchBlob{faultyCache}, + Sweep: cache.SweepSettings{MaxSizeBytes: 10000}, + }) if err != nil { t.Fatalf("err: %v", err) } @@ -273,7 +288,10 @@ func TestCacheFailureToRead(t *testing.T) { underlyingStorage := newUnderlyingStorageForContentCacheTesting(t) faultyCache := blobtesting.NewFaultyStorage(cacheStorage) - cc, err := cache.NewContentCacheForData(testlogging.Context(t), underlyingStorage, withoutTouchBlob{faultyCache}, cache.SweepSettings{MaxSizeBytes: 10000}, nil) + cc, err := cache.NewContentCache(testlogging.Context(t), underlyingStorage, cache.Options{ + Storage: withoutTouchBlob{faultyCache}, + Sweep: cache.SweepSettings{MaxSizeBytes: 10000}, + }) if err != nil { t.Fatalf("err: %v", err) } diff --git a/repo/content/committed_read_manager.go b/repo/content/committed_read_manager.go index ce807517b..161829374 100644 --- a/repo/content/committed_read_manager.go +++ b/repo/content/committed_read_manager.go @@ -372,15 +372,15 @@ func (sm *SharedManager) namedLogger(n string) logging.Logger { } func (sm *SharedManager) setupReadManagerCaches(ctx context.Context, caching *CachingOptions) error { - dataCacheStorage, err := cache.NewStorageOrNil(ctx, caching.CacheDirectory, caching.MaxCacheSizeBytes, "contents") - if err != nil { - return errors.Wrap(err, "unable to initialize data cache storage") - } - - dataCache, err := cache.NewContentCacheForData(ctx, sm.st, dataCacheStorage, cache.SweepSettings{ - MaxSizeBytes: caching.MaxCacheSizeBytes, - MinSweepAge: caching.MinContentSweepAge.DurationOrDefault(DefaultDataCacheSweepAge), - }, caching.HMACSecret) + dataCache, err := cache.NewContentCache(ctx, sm.st, cache.Options{ + BaseCacheDirectory: caching.CacheDirectory, + CacheSubDir: "contents", + HMACSecret: caching.HMACSecret, + Sweep: cache.SweepSettings{ + MaxSizeBytes: caching.MaxCacheSizeBytes, + MinSweepAge: caching.MinContentSweepAge.DurationOrDefault(DefaultDataCacheSweepAge), + }, + }) if err != nil { return errors.Wrap(err, "unable to initialize content cache") } @@ -390,14 +390,14 @@ func (sm *SharedManager) setupReadManagerCaches(ctx context.Context, caching *Ca metadataCacheSize = caching.MaxCacheSizeBytes } - metadataCacheStorage, err := cache.NewStorageOrNil(ctx, caching.CacheDirectory, metadataCacheSize, "metadata") - if err != nil { - return errors.Wrap(err, "unable to initialize data cache storage") - } - - metadataCache, err := cache.NewContentCacheForMetadata(ctx, sm.st, metadataCacheStorage, cache.SweepSettings{ - MaxSizeBytes: metadataCacheSize, - MinSweepAge: caching.MinMetadataSweepAge.DurationOrDefault(DefaultMetadataCacheSweepAge), + metadataCache, err := cache.NewContentCache(ctx, sm.st, cache.Options{ + BaseCacheDirectory: caching.CacheDirectory, + CacheSubDir: "metadata", + HMACSecret: caching.HMACSecret, + Sweep: cache.SweepSettings{ + MaxSizeBytes: metadataCacheSize, + MinSweepAge: caching.MinMetadataSweepAge.DurationOrDefault(DefaultMetadataCacheSweepAge), + }, }) if err != nil { return errors.Wrap(err, "unable to initialize metadata cache")