mirror of
https://github.com/kopia/kopia.git
synced 2026-03-30 03:53:09 -04:00
524ffaf4b8
Primarily for wiring a context.Context to a call to content.Manager.refresh, which was using a detached context.
466 lines
14 KiB
Go
466 lines
14 KiB
Go
package format_test
|
|
|
|
import (
|
|
"bytes"
|
|
"testing"
|
|
"time"
|
|
|
|
"github.com/pkg/errors"
|
|
"github.com/stretchr/testify/assert"
|
|
"github.com/stretchr/testify/require"
|
|
|
|
"github.com/kopia/kopia/internal/blobtesting"
|
|
"github.com/kopia/kopia/internal/epoch"
|
|
"github.com/kopia/kopia/internal/faketime"
|
|
"github.com/kopia/kopia/internal/feature"
|
|
"github.com/kopia/kopia/internal/gather"
|
|
"github.com/kopia/kopia/internal/testlogging"
|
|
"github.com/kopia/kopia/repo/blob"
|
|
"github.com/kopia/kopia/repo/encryption"
|
|
"github.com/kopia/kopia/repo/format"
|
|
"github.com/kopia/kopia/repo/hashing"
|
|
)
|
|
|
|
var (
|
|
errSomeError = errors.Errorf("some error")
|
|
|
|
cf = format.ContentFormat{
|
|
MutableParameters: format.MutableParameters{
|
|
Version: format.FormatVersion1,
|
|
EpochParameters: epoch.DefaultParameters(),
|
|
MaxPackSize: 20e6,
|
|
IndexVersion: 2,
|
|
},
|
|
Hash: hashing.DefaultAlgorithm,
|
|
Encryption: encryption.DefaultAlgorithm,
|
|
HMACSecret: []byte{1, 2, 3, 4, 5},
|
|
}
|
|
|
|
uli = &format.UpgradeLockIntent{
|
|
OwnerID: "foo@bar",
|
|
}
|
|
|
|
rc = &format.RepositoryConfig{
|
|
ContentFormat: cf,
|
|
UpgradeLock: uli,
|
|
}
|
|
|
|
cacheDuration = 10 * time.Minute
|
|
)
|
|
|
|
func TestFormatManager(t *testing.T) {
|
|
ctx := testlogging.Context(t)
|
|
|
|
startTime := time.Date(2020, 1, 1, 12, 0, 0, 0, time.UTC)
|
|
ta := faketime.NewTimeAdvance(startTime)
|
|
nowFunc := ta.NowFunc()
|
|
blobCache := format.NewMemoryBlobCache(nowFunc)
|
|
|
|
st := blobtesting.NewMapStorage(blobtesting.DataMap{}, nil, nil)
|
|
fst := blobtesting.NewFaultyStorage(st)
|
|
require.NoError(t, format.Initialize(ctx, fst, &format.KopiaRepositoryJSON{}, rc, format.BlobStorageConfiguration{}, "some-password"))
|
|
|
|
rawBytes := mustGetBytes(t, st, "kopia.repository")
|
|
|
|
mgr, err := format.NewManagerWithCache(ctx, fst, cacheDuration, "some-password", nowFunc, blobCache)
|
|
require.NoError(t, err)
|
|
|
|
require.Equal(t, cf.HMACSecret, mgr.GetHmacSecret())
|
|
require.Equal(t, cf.Encryption, mgr.GetEncryptionAlgorithm())
|
|
require.Equal(t, cf.Hash, mgr.GetHashFunction())
|
|
require.NotNil(t, mgr.HashFunc())
|
|
require.NotNil(t, mgr.Encryptor())
|
|
require.Equal(t, cf.MasterKey, mgr.GetMasterKey())
|
|
require.False(t, mgr.SupportsPasswordChange())
|
|
require.Equal(t, startTime, mgr.LoadedTime())
|
|
require.Equal(t, cf.MutableParameters, mustGetMutableParameters(t, mgr))
|
|
require.True(t, bytes.Contains(mustGetRepositoryFormatBytes(t, mgr), rawBytes))
|
|
require.Equal(t, uli, mustGetUpgradeLockIntent(t, mgr))
|
|
|
|
// move time to be 1ns shy of when the cache expires
|
|
fst.AddFault(blobtesting.MethodGetBlob).ErrorInstead(errSomeError)
|
|
ta.Advance(cacheDuration - 1)
|
|
|
|
// despite the failure, we still trust the cache
|
|
mustGetMutableParameters(t, mgr)
|
|
|
|
// now move the final nanosecond, this will trigger a load and storage errors
|
|
ta.Advance(1)
|
|
|
|
// error on first read, subsequent reads are ok
|
|
require.ErrorIs(t, expectMutableParametersError(t, mgr), errSomeError)
|
|
mustGetMutableParameters(t, mgr)
|
|
mustGetMutableParameters(t, mgr)
|
|
|
|
n := mgr.LoadedTime()
|
|
|
|
require.Equal(t, 2, mgr.RefreshCount())
|
|
|
|
// open another manager when cache is still valid, it will reuse old cached time
|
|
ta.Advance(5)
|
|
|
|
mgr2, err := format.NewManagerWithCache(ctx, fst, cacheDuration, "some-password", nowFunc, blobCache)
|
|
require.NoError(t, err)
|
|
|
|
mustGetMutableParameters(t, mgr2)
|
|
|
|
require.Equal(t, n, mgr2.LoadedTime())
|
|
// open another manager when cache has already expired
|
|
ta.Advance(2 * cacheDuration)
|
|
|
|
n = ta.NowFunc()()
|
|
|
|
mgr3, err := format.NewManagerWithCache(ctx, fst, cacheDuration, "some-password", nowFunc, blobCache)
|
|
require.NoError(t, err)
|
|
|
|
// make sure we're using current time
|
|
require.Equal(t, n, mgr3.LoadedTime())
|
|
|
|
// update using mgr3
|
|
mp := mustGetMutableParameters(t, mgr3)
|
|
bc2 := mustGetBlobStorageConfiguration(t, mgr3)
|
|
rf2 := mustGetRequiredFeatures(t, mgr3)
|
|
|
|
// make some changes
|
|
mp.MaxPackSize++
|
|
|
|
require.NoError(t, mgr3.SetParameters(ctx, mp, bc2, rf2))
|
|
|
|
// enough time has passed since last read, so mgr will notice the update immediately
|
|
require.Equal(t, mp, mustGetMutableParameters(t, mgr))
|
|
|
|
// update again
|
|
oldmp := mp
|
|
mp.MaxPackSize++
|
|
require.NoError(t, mgr3.SetParameters(ctx, mp, bc2, rf2))
|
|
|
|
// mgr still sees old mp
|
|
require.Equal(t, oldmp, mustGetMutableParameters(t, mgr))
|
|
|
|
// advance time, the now update is now visible
|
|
ta.Advance(cacheDuration)
|
|
require.Equal(t, mp, mustGetMutableParameters(t, mgr))
|
|
}
|
|
|
|
func TestInitialize(t *testing.T) {
|
|
ctx := testlogging.Context(t)
|
|
|
|
st := blobtesting.NewMapStorage(blobtesting.DataMap{}, nil, nil)
|
|
fst := blobtesting.NewFaultyStorage(st)
|
|
|
|
// error fetching first blob - kopia.repository
|
|
fst.AddFault(blobtesting.MethodGetBlob).ErrorInstead(errSomeError)
|
|
require.ErrorIs(t,
|
|
format.Initialize(ctx, fst, &format.KopiaRepositoryJSON{}, rc, format.BlobStorageConfiguration{}, "some-password"),
|
|
errSomeError)
|
|
|
|
// error fetching second blob - kopia.blobcfg
|
|
fst.AddFault(blobtesting.MethodGetBlob)
|
|
fst.AddFault(blobtesting.MethodGetBlob).ErrorInstead(errSomeError)
|
|
require.ErrorIs(t,
|
|
format.Initialize(ctx, fst, &format.KopiaRepositoryJSON{}, rc, format.BlobStorageConfiguration{}, "some-password"),
|
|
errSomeError)
|
|
|
|
// success
|
|
require.NoError(t, format.Initialize(ctx, fst, &format.KopiaRepositoryJSON{}, rc, format.BlobStorageConfiguration{}, "some-password"))
|
|
|
|
// already initialized
|
|
require.ErrorIs(t,
|
|
format.Initialize(ctx, fst, &format.KopiaRepositoryJSON{}, rc, format.BlobStorageConfiguration{}, "some-password"),
|
|
format.ErrAlreadyInitialized)
|
|
}
|
|
|
|
func TestInitializeWithRetention(t *testing.T) {
|
|
ctx := testlogging.Context(t)
|
|
|
|
mode := blob.Governance
|
|
period := time.Hour * 48
|
|
|
|
ta := faketime.NewClockTimeWithOffset(0)
|
|
nowFunc := ta.NowFunc()
|
|
earliestExpiry := nowFunc().Add(period)
|
|
|
|
st := blobtesting.NewVersionedMapStorage(nowFunc)
|
|
blobCache := format.NewMemoryBlobCache(nowFunc)
|
|
|
|
// success
|
|
require.NoError(t, format.Initialize(
|
|
ctx,
|
|
st,
|
|
&format.KopiaRepositoryJSON{},
|
|
rc,
|
|
format.BlobStorageConfiguration{
|
|
RetentionMode: mode,
|
|
RetentionPeriod: period,
|
|
},
|
|
"some-password",
|
|
))
|
|
|
|
mgr, err := format.NewManagerWithCache(ctx, st, cacheDuration, "some-password", nowFunc, blobCache)
|
|
require.NoError(t, err, "getting format manager")
|
|
|
|
// New retention parameters should be available from the format manager.
|
|
blobCfg := mustGetBlobStorageConfiguration(t, mgr)
|
|
assert.Equal(t, mode, blobCfg.RetentionMode)
|
|
assert.Equal(t, period, blobCfg.RetentionPeriod)
|
|
|
|
// Get the retention configuration that was added to the blob. Allow up to a
|
|
// minute difference between the expected and returned values since that
|
|
// should be large enough to avoid test flakes.
|
|
gotMode, expiry, err := st.GetRetention(ctx, format.KopiaRepositoryBlobID)
|
|
require.NoError(t, err, "getting repo blob retention info")
|
|
|
|
assert.Equal(t, mode, gotMode)
|
|
assert.WithinDuration(t, earliestExpiry, expiry, time.Minute)
|
|
|
|
gotMode, expiry, err = st.GetRetention(ctx, format.KopiaBlobCfgBlobID)
|
|
require.NoError(t, err, "getting storage blob config retention info")
|
|
|
|
assert.Equal(t, mode, gotMode)
|
|
assert.WithinDuration(t, earliestExpiry, expiry, time.Minute)
|
|
}
|
|
|
|
func TestUpdateRetention(t *testing.T) {
|
|
ctx := testlogging.Context(t)
|
|
|
|
mode := blob.Governance
|
|
period := time.Hour * 48
|
|
|
|
ta := faketime.NewClockTimeWithOffset(0)
|
|
nowFunc := ta.NowFunc()
|
|
earliestExpiry := nowFunc().Add(period)
|
|
|
|
st := blobtesting.NewVersionedMapStorage(nowFunc)
|
|
blobCache := format.NewMemoryBlobCache(nowFunc)
|
|
|
|
// success
|
|
require.NoError(t, format.Initialize(ctx, st, &format.KopiaRepositoryJSON{}, rc, format.BlobStorageConfiguration{}, "some-password"))
|
|
|
|
mgr, err := format.NewManagerWithCache(ctx, st, cacheDuration, "some-password", nowFunc, blobCache)
|
|
require.NoError(t, err, "getting format manager")
|
|
|
|
mp := mustGetMutableParameters(t, mgr)
|
|
rf := mustGetRequiredFeatures(t, mgr)
|
|
|
|
err = mgr.SetParameters(
|
|
ctx,
|
|
mp,
|
|
format.BlobStorageConfiguration{
|
|
RetentionMode: mode,
|
|
RetentionPeriod: period,
|
|
},
|
|
rf,
|
|
)
|
|
require.NoError(t, err, "setting repo parameters")
|
|
|
|
// New retention parameters should be available from the format manager.
|
|
blobCfg := mustGetBlobStorageConfiguration(t, mgr)
|
|
assert.Equal(t, mode, blobCfg.RetentionMode)
|
|
assert.Equal(t, period, blobCfg.RetentionPeriod)
|
|
|
|
// Get the retention configuration that was added to the blob. Allow up to a
|
|
// minute difference between the expected and returned values since that
|
|
// should be large enough to avoid test flakes.
|
|
gotMode, expiry, err := st.GetRetention(ctx, format.KopiaRepositoryBlobID)
|
|
require.NoError(t, err, "getting repo blob retention info")
|
|
|
|
assert.Equal(t, mode, gotMode)
|
|
assert.WithinDuration(t, earliestExpiry, expiry, time.Minute)
|
|
|
|
gotMode, expiry, err = st.GetRetention(ctx, format.KopiaBlobCfgBlobID)
|
|
require.NoError(t, err, "getting storage blob config retention info")
|
|
|
|
assert.Equal(t, mode, gotMode)
|
|
assert.WithinDuration(t, earliestExpiry, expiry, time.Minute)
|
|
}
|
|
|
|
func TestUpdateRetentionNegativeValue(t *testing.T) {
|
|
ctx := testlogging.Context(t)
|
|
|
|
startTime := time.Date(2020, 1, 1, 12, 0, 0, 0, time.UTC)
|
|
ta := faketime.NewTimeAdvance(startTime)
|
|
nowFunc := ta.NowFunc()
|
|
|
|
st := blobtesting.NewVersionedMapStorage(nowFunc)
|
|
blobCache := format.NewMemoryBlobCache(nowFunc)
|
|
mode := blob.Governance
|
|
period := -time.Hour * 48
|
|
|
|
// success
|
|
require.NoError(t, format.Initialize(ctx, st, &format.KopiaRepositoryJSON{}, rc, format.BlobStorageConfiguration{}, "some-password"))
|
|
|
|
mgr, err := format.NewManagerWithCache(ctx, st, cacheDuration, "some-password", nowFunc, blobCache)
|
|
require.NoError(t, err, "getting format manager")
|
|
|
|
mp := mustGetMutableParameters(t, mgr)
|
|
rf := mustGetRequiredFeatures(t, mgr)
|
|
|
|
err = mgr.SetParameters(
|
|
ctx,
|
|
mp,
|
|
format.BlobStorageConfiguration{
|
|
RetentionMode: mode,
|
|
RetentionPeriod: period,
|
|
},
|
|
rf,
|
|
)
|
|
require.Error(t, err, "setting repo parameters")
|
|
|
|
// Old retention parameters should be available from the format manager.
|
|
blobCfg := mustGetBlobStorageConfiguration(t, mgr)
|
|
assert.Empty(t, blobCfg.RetentionMode)
|
|
assert.Zero(t, blobCfg.RetentionPeriod)
|
|
|
|
// Retention wasn't set so everything should be zero/empty.
|
|
gotMode, expiry, err := st.GetRetention(ctx, format.KopiaRepositoryBlobID)
|
|
require.NoError(t, err, "getting repo blob retention info")
|
|
|
|
assert.Empty(t, gotMode)
|
|
assert.Zero(t, expiry)
|
|
|
|
gotMode, expiry, err = st.GetRetention(ctx, format.KopiaBlobCfgBlobID)
|
|
require.NoError(t, err, "getting storage blob config retention info")
|
|
|
|
assert.Empty(t, gotMode)
|
|
assert.Zero(t, expiry)
|
|
}
|
|
|
|
func TestChangePassword(t *testing.T) {
|
|
ctx := testlogging.Context(t)
|
|
|
|
startTime := time.Date(2020, 1, 1, 12, 0, 0, 0, time.UTC)
|
|
ta := faketime.NewTimeAdvance(startTime)
|
|
nowFunc := ta.NowFunc()
|
|
blobCache := format.NewMemoryBlobCache(nowFunc)
|
|
|
|
cf2 := cf
|
|
cf2.Version = format.FormatVersion3
|
|
cf2.EnablePasswordChange = true
|
|
|
|
rc = &format.RepositoryConfig{
|
|
ContentFormat: cf2,
|
|
UpgradeLock: uli,
|
|
}
|
|
|
|
st := blobtesting.NewMapStorage(blobtesting.DataMap{}, nil, nil)
|
|
fst := blobtesting.NewFaultyStorage(st)
|
|
require.NoError(t, format.Initialize(ctx, fst, &format.KopiaRepositoryJSON{}, rc, format.BlobStorageConfiguration{}, "some-password"))
|
|
|
|
mgr, err := format.NewManagerWithCache(ctx, fst, cacheDuration, "some-password", nowFunc, blobCache)
|
|
require.NoError(t, err)
|
|
|
|
mgr2, err := format.NewManagerWithCache(ctx, fst, cacheDuration, "some-password", nowFunc, blobCache)
|
|
require.NoError(t, err)
|
|
|
|
require.NoError(t, mgr2.ChangePassword(ctx, "new-password"))
|
|
|
|
// immediately after changing the password, both managers can still read the repo
|
|
mustGetMutableParameters(t, mgr)
|
|
mustGetMutableParameters(t, mgr2)
|
|
|
|
ta.Advance(cacheDuration)
|
|
|
|
require.ErrorIs(t, expectMutableParametersError(t, mgr), format.ErrInvalidPassword)
|
|
mustGetMutableParameters(t, mgr2)
|
|
|
|
_, err = format.NewManagerWithCache(ctx, fst, cacheDuration, "some-password", nowFunc, blobCache)
|
|
require.ErrorIs(t, err, format.ErrInvalidPassword)
|
|
}
|
|
|
|
func TestFormatManagerValidDuration(t *testing.T) {
|
|
cases := map[time.Duration]time.Duration{
|
|
-1: 15 * time.Minute,
|
|
time.Second: time.Second,
|
|
30 * time.Minute: 15 * time.Minute,
|
|
10 * time.Minute: 10 * time.Minute,
|
|
}
|
|
|
|
for requestedCacheDuration, actualCacheDuration := range cases {
|
|
ctx := testlogging.Context(t)
|
|
|
|
startTime := time.Date(2020, 1, 1, 12, 0, 0, 0, time.UTC)
|
|
ta := faketime.NewTimeAdvance(startTime)
|
|
nowFunc := ta.NowFunc()
|
|
blobCache := format.NewMemoryBlobCache(nowFunc)
|
|
|
|
st := blobtesting.NewMapStorage(blobtesting.DataMap{}, nil, nil)
|
|
fst := blobtesting.NewFaultyStorage(st)
|
|
require.NoError(t, format.Initialize(ctx, fst, &format.KopiaRepositoryJSON{}, rc, format.BlobStorageConfiguration{}, "some-password"))
|
|
|
|
if requestedCacheDuration < 0 {
|
|
// plant a malformed cache entry to ensure it's not being used
|
|
blobCache.Put(ctx, "kopia.repository", []byte("malformed"))
|
|
}
|
|
|
|
mgr, err := format.NewManagerWithCache(ctx, fst, requestedCacheDuration, "some-password", nowFunc, blobCache)
|
|
require.NoError(t, err)
|
|
|
|
require.Equal(t, actualCacheDuration, mgr.ValidCacheDuration())
|
|
}
|
|
}
|
|
|
|
func mustGetMutableParameters(t *testing.T, mgr *format.Manager) format.MutableParameters {
|
|
t.Helper()
|
|
|
|
mp, err := mgr.GetMutableParameters(testlogging.Context(t))
|
|
require.NoError(t, err)
|
|
|
|
return mp
|
|
}
|
|
|
|
func mustGetUpgradeLockIntent(t *testing.T, mgr *format.Manager) *format.UpgradeLockIntent {
|
|
t.Helper()
|
|
|
|
uli, err := mgr.GetUpgradeLockIntent(testlogging.Context(t))
|
|
require.NoError(t, err)
|
|
|
|
return uli
|
|
}
|
|
|
|
func mustGetRepositoryFormatBytes(t *testing.T, mgr *format.Manager) []byte {
|
|
t.Helper()
|
|
|
|
b, err := mgr.RepositoryFormatBytes(testlogging.Context(t))
|
|
require.NoError(t, err)
|
|
|
|
return b
|
|
}
|
|
|
|
func mustGetRequiredFeatures(t *testing.T, mgr *format.Manager) []feature.Required {
|
|
t.Helper()
|
|
|
|
rf, err := mgr.RequiredFeatures(testlogging.Context(t))
|
|
require.NoError(t, err)
|
|
|
|
return rf
|
|
}
|
|
|
|
func mustGetBlobStorageConfiguration(t *testing.T, mgr *format.Manager) format.BlobStorageConfiguration {
|
|
t.Helper()
|
|
|
|
cfg, err := mgr.BlobCfgBlob(testlogging.Context(t))
|
|
require.NoError(t, err)
|
|
|
|
return cfg
|
|
}
|
|
|
|
func expectMutableParametersError(t *testing.T, mgr *format.Manager) error {
|
|
t.Helper()
|
|
|
|
_, err := mgr.GetMutableParameters(testlogging.Context(t))
|
|
require.Error(t, err)
|
|
|
|
return err
|
|
}
|
|
|
|
func mustGetBytes(t *testing.T, st blob.Storage, blobID blob.ID) []byte {
|
|
t.Helper()
|
|
|
|
var tmp gather.WriteBuffer
|
|
defer tmp.Close()
|
|
|
|
require.NoError(t, st.GetBlob(testlogging.Context(t), blobID, 0, -1, &tmp))
|
|
|
|
return tmp.ToByteSlice()
|
|
}
|