mirror/kopia
1
0
Fork 0
mirror of https://github.com/kopia/kopia.git synced 2026-03-07 15:58:25 -05:00
Code Issues Packages Projects Releases Wiki Activity

refactor(repository): moved cache protection to separate package (#2621)

Browse Source
This commit is contained in:
Jarek Kowalski
2022-12-03 10:47:01 -08:00
committed by GitHub
parent c44c60d376
commit 7cc93265ef
7 changed files with 28 additions and 22 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
internal/cache/content_cache.go vendored
View File

@@ -5,6 +5,7 @@
"github.com/pkg/errors"
"github.com/kopia/kopia/internal/cacheprot"
"github.com/kopia/kopia/internal/gather"
"github.com/kopia/kopia/internal/impossible"
"github.com/kopia/kopia/internal/metrics"
@@ -189,7 +190,7 @@ func NewContentCache(ctx context.Context, st blob.Storage, opt Options, mr *metr
}
}
pc, err := NewPersistentCache(ctx, opt.CacheSubDir, cacheStorage, ChecksumProtection(opt.HMACSecret), opt.Sweep, mr)
pc, err := NewPersistentCache(ctx, opt.CacheSubDir, cacheStorage, cacheprot.ChecksumProtection(opt.HMACSecret), opt.Sweep, mr)
if err != nil {
return nil, errors.Wrap(err, "unable to create base cache")
}

9
internal/cache/persistent_lru_cache.go vendored
View File

@@ -11,6 +11,7 @@
lru "github.com/hashicorp/golang-lru"
"github.com/pkg/errors"
"github.com/kopia/kopia/internal/cacheprot"
"github.com/kopia/kopia/internal/clock"
"github.com/kopia/kopia/internal/ctxutil"
"github.com/kopia/kopia/internal/gather"
@@ -42,7 +43,7 @@ type PersistentCache struct {
anyChange atomic.Bool
cacheStorage Storage
storageProtection StorageProtection
storageProtection cacheprot.StorageProtection
sweep SweepSettings
description string
@@ -137,7 +138,7 @@ func (c *PersistentCache) GetPartial(ctx context.Context, key string, offset, le
prot := c.storageProtection
if length >= 0 {
// only full items have protection.
prot = nullStorageProtection{}
prot = cacheprot.NoProtection()
}
if err := prot.Verify(key, tmp.Bytes(), output); err == nil {
@@ -333,7 +334,7 @@ func (s SweepSettings) applyDefaults() SweepSettings {
}
// NewPersistentCache creates the persistent cache in the provided storage.
func NewPersistentCache(ctx context.Context, description string, cacheStorage Storage, storageProtection StorageProtection, sweep SweepSettings, mr *metrics.Registry) (*PersistentCache, error) {
func NewPersistentCache(ctx context.Context, description string, cacheStorage Storage, storageProtection cacheprot.StorageProtection, sweep SweepSettings, mr *metrics.Registry) (*PersistentCache, error) {
if cacheStorage == nil {
return nil, nil
}
@@ -341,7 +342,7 @@ func NewPersistentCache(ctx context.Context, description string, cacheStorage St
sweep = sweep.applyDefaults()
if storageProtection == nil {
storageProtection = NoProtection()
storageProtection = cacheprot.NoProtection()
}
c := &PersistentCache{

17
internal/cache/persistent_lru_cache_test.go vendored
View File

@@ -11,6 +11,7 @@
"github.com/kopia/kopia/internal/blobtesting"
"github.com/kopia/kopia/internal/cache"
"github.com/kopia/kopia/internal/cacheprot"
"github.com/kopia/kopia/internal/fault"
"github.com/kopia/kopia/internal/gather"
"github.com/kopia/kopia/internal/testlogging"
@@ -25,7 +26,7 @@ func TestPersistentLRUCache(t *testing.T) {
cs := blobtesting.NewMapStorage(blobtesting.DataMap{}, nil, nil).(cache.Storage)
pc, err := cache.NewPersistentCache(ctx, "testing", cs, cache.ChecksumProtection([]byte{1, 2, 3}), cache.SweepSettings{
pc, err := cache.NewPersistentCache(ctx, "testing", cs, cacheprot.ChecksumProtection([]byte{1, 2, 3}), cache.SweepSettings{
MaxSizeBytes: maxSizeBytes,
TouchThreshold: cache.DefaultTouchThreshold,
SweepFrequency: cache.DefaultSweepFrequency,
@@ -67,7 +68,7 @@ func TestPersistentLRUCache(t *testing.T) {
verifyBlobExists(ctx, t, cs, "key3")
verifyBlobExists(ctx, t, cs, "key4")
pc, err = cache.NewPersistentCache(ctx, "testing", cs, cache.ChecksumProtection([]byte{1, 2, 3}), cache.SweepSettings{
pc, err = cache.NewPersistentCache(ctx, "testing", cs, cacheprot.ChecksumProtection([]byte{1, 2, 3}), cache.SweepSettings{
MaxSizeBytes: maxSizeBytes,
TouchThreshold: cache.DefaultTouchThreshold,
SweepFrequency: cache.DefaultSweepFrequency,
@@ -81,7 +82,7 @@ func TestPersistentLRUCache(t *testing.T) {
// create another persistent cache based on the same storage but wrong protection key.
// all reads from cache will be invalid, which means GetOrLoad will fetch them from the source.
pc2, err := cache.NewPersistentCache(ctx, "testing", cs, cache.ChecksumProtection([]byte{3, 2, 1}), cache.SweepSettings{
pc2, err := cache.NewPersistentCache(ctx, "testing", cs, cacheprot.ChecksumProtection([]byte{3, 2, 1}), cache.SweepSettings{
MaxSizeBytes: maxSizeBytes,
TouchThreshold: cache.DefaultTouchThreshold,
SweepFrequency: cache.DefaultSweepFrequency,
@@ -153,7 +154,7 @@ func TestPersistentLRUCache_GetDeletesInvalidBlob(t *testing.T) {
fs := blobtesting.NewFaultyStorage(st)
fc := faultyCache{fs}
pc, err := cache.NewPersistentCache(ctx, "test", fc, cache.ChecksumProtection([]byte{1, 2, 3}), cache.SweepSettings{}, nil)
pc, err := cache.NewPersistentCache(ctx, "test", fc, cacheprot.ChecksumProtection([]byte{1, 2, 3}), cache.SweepSettings{}, nil)
require.NoError(t, err)
pc.Put(ctx, "key", gather.FromSlice([]byte{1, 2, 3}))
@@ -183,7 +184,7 @@ func TestPersistentLRUCache_PutIgnoresStorageFailure(t *testing.T) {
fs := blobtesting.NewFaultyStorage(st)
fc := faultyCache{fs}
pc, err := cache.NewPersistentCache(ctx, "test", fc, cache.ChecksumProtection([]byte{1, 2, 3}), cache.SweepSettings{}, nil)
pc, err := cache.NewPersistentCache(ctx, "test", fc, cacheprot.ChecksumProtection([]byte{1, 2, 3}), cache.SweepSettings{}, nil)
require.NoError(t, err)
fs.AddFault(blobtesting.MethodPutBlob).ErrorInstead(someError)
@@ -209,7 +210,7 @@ func TestPersistentLRUCache_SweepMinSweepAge(t *testing.T) {
fs := blobtesting.NewFaultyStorage(st)
fc := faultyCache{fs}
pc, err := cache.NewPersistentCache(ctx, "test", fc, cache.ChecksumProtection([]byte{1, 2, 3}), cache.SweepSettings{
pc, err := cache.NewPersistentCache(ctx, "test", fc, cacheprot.ChecksumProtection([]byte{1, 2, 3}), cache.SweepSettings{
SweepFrequency: 100 * time.Millisecond,
MaxSizeBytes: 1000,
MinSweepAge: 10 * time.Second,
@@ -238,7 +239,7 @@ func TestPersistentLRUCache_SweepIgnoresErrors(t *testing.T) {
fs := blobtesting.NewFaultyStorage(st)
fc := faultyCache{fs}
pc, err := cache.NewPersistentCache(ctx, "test", fc, cache.ChecksumProtection([]byte{1, 2, 3}), cache.SweepSettings{
pc, err := cache.NewPersistentCache(ctx, "test", fc, cacheprot.ChecksumProtection([]byte{1, 2, 3}), cache.SweepSettings{
SweepFrequency: 100 * time.Millisecond,
MaxSizeBytes: 1000,
}, nil)
@@ -271,7 +272,7 @@ func TestPersistentLRUCache_Sweep1(t *testing.T) {
fs := blobtesting.NewFaultyStorage(st)
fc := faultyCache{fs}
pc, err := cache.NewPersistentCache(ctx, "test", fc, cache.ChecksumProtection([]byte{1, 2, 3}), cache.SweepSettings{
pc, err := cache.NewPersistentCache(ctx, "test", fc, cacheprot.ChecksumProtection([]byte{1, 2, 3}), cache.SweepSettings{
SweepFrequency: 10 * time.Millisecond,
MaxSizeBytes: 1,
MinSweepAge: 0 * time.Second,

3
internal/cache/storage_protection.go → internal/cacheprot/storage_protection.go
View File

@@ -1,4 +1,5 @@
package cache
// Package cacheprot provides utilities for protection of cache entries.
package cacheprot
import (
"crypto/sha256"

12
internal/cache/storage_protection_test.go → internal/cacheprot/storage_protection_test.go
View File

@@ -1,4 +1,4 @@
package cache_test
package cacheprot_test
import (
"bytes"
@@ -6,27 +6,27 @@
"github.com/stretchr/testify/require"
"github.com/kopia/kopia/internal/cache"
"github.com/kopia/kopia/internal/cacheprot"
"github.com/kopia/kopia/internal/gather"
)
func TestNoStorageProection(t *testing.T) {
testStorageProtection(t, cache.NoProtection(), false)
testStorageProtection(t, cacheprot.NoProtection(), false)
}
func TestHMACStorageProtection(t *testing.T) {
testStorageProtection(t, cache.ChecksumProtection([]byte{1, 2, 3, 4}), true)
testStorageProtection(t, cacheprot.ChecksumProtection([]byte{1, 2, 3, 4}), true)
}
func TestEncryptionStorageProtection(t *testing.T) {
e, err := cache.AuthenticatedEncryptionProtection([]byte{1})
e, err := cacheprot.AuthenticatedEncryptionProtection([]byte{1})
require.NoError(t, err)
testStorageProtection(t, e, true)
}
//nolint:thelper
func testStorageProtection(t *testing.T, sp cache.StorageProtection, protectsFromBitFlips bool) {
func testStorageProtection(t *testing.T, sp cacheprot.StorageProtection, protectsFromBitFlips bool) {
payload := []byte{0, 1, 2, 3, 4}
var protected gather.WriteBuffer

3
repo/content/committed_read_manager.go
View File

@@ -11,6 +11,7 @@
"go.uber.org/zap"
"github.com/kopia/kopia/internal/cache"
"github.com/kopia/kopia/internal/cacheprot"
"github.com/kopia/kopia/internal/clock"
"github.com/kopia/kopia/internal/epoch"
"github.com/kopia/kopia/internal/gather"
@@ -427,7 +428,7 @@ func (sm *SharedManager) setupReadManagerCaches(ctx context.Context, caching *Ca
return errors.Wrap(err, "unable to initialize index blob cache storage")
}
indexBlobCache, err := cache.NewPersistentCache(ctx, "index-blobs", indexBlobStorage, cache.ChecksumProtection(caching.HMACSecret), cache.SweepSettings{
indexBlobCache, err := cache.NewPersistentCache(ctx, "index-blobs", indexBlobStorage, cacheprot.ChecksumProtection(caching.HMACSecret), cache.SweepSettings{
MaxSizeBytes: metadataCacheSize,
MinSweepAge: caching.MinMetadataSweepAge.DurationOrDefault(DefaultMetadataCacheSweepAge),
}, mr)

3
repo/open.go
View File

@@ -13,6 +13,7 @@
"golang.org/x/crypto/scrypt"
"github.com/kopia/kopia/internal/cache"
"github.com/kopia/kopia/internal/cacheprot"
"github.com/kopia/kopia/internal/epoch"
"github.com/kopia/kopia/internal/feature"
"github.com/kopia/kopia/internal/metrics"
@@ -143,7 +144,7 @@ func getContentCacheOrNil(ctx context.Context, opt *content.CachingOptions, pass
return nil, errors.Wrap(err, "unable to derive cache encryption key from password")
}
prot, err := cache.AuthenticatedEncryptionProtection(cacheEncryptionKey)
prot, err := cacheprot.AuthenticatedEncryptionProtection(cacheEncryptionKey)
if err != nil {
return nil, errors.Wrap(err, "unable to initialize protection")
}