mirror of
https://github.com/kopia/kopia.git
synced 2026-01-28 00:08:04 -05:00
[Robustness] Add test engine to manage snapshot verification testing (#468)
* Add test engine to manage snapshot verification testing Test engine manages the test and metadata repositories, snapshot checker, metadata storage persistence, and file writer. It is the high level helper that will be invoked in the snapshot verification testing suite. - modify data directory file structure - issue snapshot/restore/delete to the data directory - accumulate metadata over the course of the test suite - flush accumulated metadata to the metadata repository - load historical metadata from the repository on initialization - perform automatic data integrity verification on snap restore This change corresponds to the robustness execution engine component from the design documentation.
This commit is contained in:
Nick
@@ -20,4 +20,5 @@ type Persister interface {
|
||||
snap.RepoManager
|
||||
LoadMetadata() error
|
||||
FlushMetadata() error
|
||||
Cleanup()
|
||||
}
|
||||
|
||||
194
tests/robustness/test_engine/engine.go
Normal file
194
tests/robustness/test_engine/engine.go
Normal file
@@ -0,0 +1,194 @@
|
||||
// +build darwin linux
|
||||
|
||||
// Package engine provides the framework for a snapshot repository testing engine
|
||||
package engine
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"math/rand"
|
||||
"os"
|
||||
"strconv"
|
||||
|
||||
"github.com/kopia/kopia/tests/robustness/checker"
|
||||
"github.com/kopia/kopia/tests/robustness/snap"
|
||||
"github.com/kopia/kopia/tests/robustness/snapmeta"
|
||||
"github.com/kopia/kopia/tests/tools/fio"
|
||||
"github.com/kopia/kopia/tests/tools/fswalker"
|
||||
"github.com/kopia/kopia/tests/tools/kopiarunner"
|
||||
)
|
||||
|
||||
const (
|
||||
// S3BucketNameEnvKey is the environment variable required to connect to a repo on S3
|
||||
S3BucketNameEnvKey = "S3_BUCKET_NAME"
|
||||
)
|
||||
|
||||
// ErrS3BucketNameEnvUnset is the error returned when the S3BucketNameEnvKey environment variable is not set
|
||||
var ErrS3BucketNameEnvUnset = fmt.Errorf("environment variable required: %v", S3BucketNameEnvKey)
|
||||
|
||||
// Engine is the outer level testing framework for robustness testing
|
||||
type Engine struct {
|
||||
FileWriter *fio.Runner
|
||||
TestRepo snap.Snapshotter
|
||||
MetaStore snapmeta.Persister
|
||||
Checker *checker.Checker
|
||||
cleanupRoutines []func()
|
||||
}
|
||||
|
||||
// NewEngine instantiates a new Engine and returns its pointer. It is
|
||||
// currently created with:
|
||||
// - FIO file writer
|
||||
// - Kopia test repo snapshotter
|
||||
// - Kopia metadata storage repo
|
||||
// - FSWalker data integrity checker
|
||||
func NewEngine() (*Engine, error) {
|
||||
e := new(Engine)
|
||||
|
||||
var err error
|
||||
|
||||
// Fill the file writer
|
||||
e.FileWriter, err = fio.NewRunner()
|
||||
if err != nil {
|
||||
e.Cleanup() //nolint:errcheck
|
||||
return nil, err
|
||||
}
|
||||
|
||||
e.cleanupRoutines = append(e.cleanupRoutines, e.FileWriter.Cleanup)
|
||||
|
||||
// Fill Snapshotter interface
|
||||
kopiaSnapper, err := kopiarunner.NewKopiaSnapshotter()
|
||||
if err != nil {
|
||||
e.Cleanup() //nolint:errcheck
|
||||
return nil, err
|
||||
}
|
||||
|
||||
e.cleanupRoutines = append(e.cleanupRoutines, kopiaSnapper.Cleanup)
|
||||
e.TestRepo = kopiaSnapper
|
||||
|
||||
// Fill the snapshot store interface
|
||||
snapStore, err := snapmeta.New()
|
||||
if err != nil {
|
||||
e.Cleanup() //nolint:errcheck
|
||||
return nil, err
|
||||
}
|
||||
|
||||
e.cleanupRoutines = append(e.cleanupRoutines, snapStore.Cleanup)
|
||||
|
||||
e.MetaStore = snapStore
|
||||
|
||||
// Create the data integrity checker
|
||||
chk, err := checker.NewChecker(kopiaSnapper, snapStore, fswalker.NewWalkCompare())
|
||||
e.cleanupRoutines = append(e.cleanupRoutines, chk.Cleanup)
|
||||
|
||||
if err != nil {
|
||||
e.Cleanup() //nolint:errcheck
|
||||
return nil, err
|
||||
}
|
||||
|
||||
e.Checker = chk
|
||||
|
||||
return e, nil
|
||||
}
|
||||
|
||||
// Cleanup cleans up after each component of the test engine
|
||||
func (e *Engine) Cleanup() error {
|
||||
defer e.cleanup()
|
||||
|
||||
if e.MetaStore != nil {
|
||||
return e.MetaStore.FlushMetadata()
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (e *Engine) cleanup() {
|
||||
for _, f := range e.cleanupRoutines {
|
||||
f()
|
||||
}
|
||||
}
|
||||
|
||||
// InitS3 attempts to connect to a test repo and metadata repo on S3. If connection
|
||||
// is successful, the engine is populated with the metadata associated with the
|
||||
// snapshot in that repo. A new repo will be created if one does not already
|
||||
// exist.
|
||||
func (e *Engine) InitS3(ctx context.Context, testRepoPath, metaRepoPath string) error {
|
||||
bucketName := os.Getenv(S3BucketNameEnvKey)
|
||||
if bucketName == "" {
|
||||
return ErrS3BucketNameEnvUnset
|
||||
}
|
||||
|
||||
err := e.MetaStore.ConnectOrCreateS3(bucketName, metaRepoPath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
err = e.MetaStore.LoadMetadata()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
err = e.TestRepo.ConnectOrCreateS3(bucketName, testRepoPath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
_, _, err = e.TestRepo.Run("policy", "set", "--global", "--keep-latest", strconv.Itoa(1<<31-1))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
err = e.Checker.VerifySnapshotMetadata()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
snapIDs := e.Checker.GetLiveSnapIDs()
|
||||
if len(snapIDs) > 0 {
|
||||
randSnapID := snapIDs[rand.Intn(len(snapIDs))]
|
||||
|
||||
err = e.Checker.RestoreSnapshotToPath(ctx, randSnapID, e.FileWriter.LocalDataDir, os.Stdout)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// InitFilesystem attempts to connect to a test repo and metadata repo on the local
|
||||
// filesystem. If connection is successful, the engine is populated with the
|
||||
// metadata associated with the snapshot in that repo. A new repo will be created if
|
||||
// one does not already exist.
|
||||
func (e *Engine) InitFilesystem(ctx context.Context, testRepoPath, metaRepoPath string) error {
|
||||
err := e.MetaStore.ConnectOrCreateFilesystem(metaRepoPath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
err = e.MetaStore.LoadMetadata()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
err = e.TestRepo.ConnectOrCreateFilesystem(testRepoPath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
err = e.Checker.VerifySnapshotMetadata()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
snapIDs := e.Checker.GetSnapIDs()
|
||||
if len(snapIDs) > 0 {
|
||||
randSnapID := snapIDs[rand.Intn(len(snapIDs))]
|
||||
|
||||
err = e.Checker.RestoreSnapshotToPath(ctx, randSnapID, e.FileWriter.LocalDataDir, os.Stdout)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
270
tests/robustness/test_engine/engine_test.go
Normal file
270
tests/robustness/test_engine/engine_test.go
Normal file
@@ -0,0 +1,270 @@
|
||||
// +build darwin linux
|
||||
|
||||
// Package engine provides the framework for a snapshot repository testing engine
|
||||
package engine
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io/ioutil"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
"github.com/kopia/kopia/tests/testenv"
|
||||
"github.com/kopia/kopia/tests/tools/fio"
|
||||
"github.com/kopia/kopia/tests/tools/fswalker"
|
||||
"github.com/kopia/kopia/tests/tools/kopiarunner"
|
||||
)
|
||||
|
||||
var (
|
||||
fsMetadataRepoPath = filepath.Join("/tmp", "metadata-repo")
|
||||
s3MetadataRepoPath = filepath.Join("some/path", "metadata-repo")
|
||||
fsDataRepoPath = filepath.Join("/tmp", "data-repo")
|
||||
s3DataRepoPath = filepath.Join("some/path", "data-repo")
|
||||
)
|
||||
|
||||
func TestEngineWritefilesBasicFS(t *testing.T) {
|
||||
eng, err := NewEngine()
|
||||
if err == kopiarunner.ErrExeVariableNotSet || errors.Is(err, fio.ErrEnvNotSet) {
|
||||
t.Skip(err)
|
||||
}
|
||||
|
||||
testenv.AssertNoError(t, err)
|
||||
|
||||
defer func() {
|
||||
cleanupErr := eng.Cleanup()
|
||||
testenv.AssertNoError(t, cleanupErr)
|
||||
}()
|
||||
|
||||
ctx := context.TODO()
|
||||
err = eng.InitFilesystem(ctx, fsDataRepoPath, fsMetadataRepoPath)
|
||||
testenv.AssertNoError(t, err)
|
||||
|
||||
fileSize := int64(256 * 1024 * 1024)
|
||||
numFiles := 10
|
||||
|
||||
fioOpt := fio.Options{}.WithFileSize(fileSize).WithNumFiles(numFiles)
|
||||
|
||||
err = eng.FileWriter.WriteFiles("", fioOpt)
|
||||
testenv.AssertNoError(t, err)
|
||||
|
||||
snapIDs := eng.Checker.GetSnapIDs()
|
||||
|
||||
snapID, err := eng.Checker.TakeSnapshot(ctx, eng.FileWriter.LocalDataDir)
|
||||
testenv.AssertNoError(t, err)
|
||||
|
||||
err = eng.Checker.RestoreSnapshot(ctx, snapID, os.Stdout)
|
||||
testenv.AssertNoError(t, err)
|
||||
|
||||
for _, sID := range snapIDs {
|
||||
err = eng.Checker.RestoreSnapshot(ctx, sID, os.Stdout)
|
||||
testenv.AssertNoError(t, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestWriteFilesBasicS3(t *testing.T) {
|
||||
eng, err := NewEngine()
|
||||
if err == kopiarunner.ErrExeVariableNotSet || errors.Is(err, fio.ErrEnvNotSet) {
|
||||
t.Skip(err)
|
||||
}
|
||||
|
||||
testenv.AssertNoError(t, err)
|
||||
|
||||
defer func() {
|
||||
cleanupErr := eng.Cleanup()
|
||||
testenv.AssertNoError(t, cleanupErr)
|
||||
}()
|
||||
|
||||
ctx := context.TODO()
|
||||
err = eng.InitS3(ctx, s3DataRepoPath, s3MetadataRepoPath)
|
||||
testenv.AssertNoError(t, err)
|
||||
|
||||
fileSize := int64(256 * 1024 * 1024)
|
||||
numFiles := 10
|
||||
|
||||
fioOpt := fio.Options{}.WithFileSize(fileSize).WithNumFiles(numFiles)
|
||||
|
||||
err = eng.FileWriter.WriteFiles("", fioOpt)
|
||||
testenv.AssertNoError(t, err)
|
||||
|
||||
snapIDs := eng.Checker.GetLiveSnapIDs()
|
||||
|
||||
snapID, err := eng.Checker.TakeSnapshot(ctx, eng.FileWriter.LocalDataDir)
|
||||
testenv.AssertNoError(t, err)
|
||||
|
||||
err = eng.Checker.RestoreSnapshot(ctx, snapID, os.Stdout)
|
||||
testenv.AssertNoError(t, err)
|
||||
|
||||
for _, sID := range snapIDs {
|
||||
err = eng.Checker.RestoreSnapshot(ctx, sID, os.Stdout)
|
||||
testenv.AssertNoError(t, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDeleteSnapshotS3(t *testing.T) {
|
||||
eng, err := NewEngine()
|
||||
if err == kopiarunner.ErrExeVariableNotSet || errors.Is(err, fio.ErrEnvNotSet) {
|
||||
t.Skip(err)
|
||||
}
|
||||
|
||||
testenv.AssertNoError(t, err)
|
||||
|
||||
defer func() {
|
||||
cleanupErr := eng.Cleanup()
|
||||
testenv.AssertNoError(t, cleanupErr)
|
||||
}()
|
||||
|
||||
ctx := context.TODO()
|
||||
err = eng.InitS3(ctx, s3DataRepoPath, s3MetadataRepoPath)
|
||||
testenv.AssertNoError(t, err)
|
||||
|
||||
fileSize := int64(256 * 1024 * 1024)
|
||||
numFiles := 10
|
||||
|
||||
fioOpt := fio.Options{}.WithFileSize(fileSize).WithNumFiles(numFiles)
|
||||
|
||||
err = eng.FileWriter.WriteFiles("", fioOpt)
|
||||
testenv.AssertNoError(t, err)
|
||||
|
||||
snapID, err := eng.Checker.TakeSnapshot(ctx, eng.FileWriter.LocalDataDir)
|
||||
testenv.AssertNoError(t, err)
|
||||
|
||||
err = eng.Checker.RestoreSnapshot(ctx, snapID, os.Stdout)
|
||||
testenv.AssertNoError(t, err)
|
||||
|
||||
err = eng.Checker.DeleteSnapshot(ctx, snapID)
|
||||
testenv.AssertNoError(t, err)
|
||||
|
||||
err = eng.Checker.RestoreSnapshot(ctx, snapID, os.Stdout)
|
||||
if err == nil {
|
||||
t.Fatalf("Expected an error when trying to restore a deleted snapshot")
|
||||
}
|
||||
}
|
||||
|
||||
func TestSnapshotVerificationFail(t *testing.T) {
|
||||
eng, err := NewEngine()
|
||||
if err == kopiarunner.ErrExeVariableNotSet || errors.Is(err, fio.ErrEnvNotSet) {
|
||||
t.Skip(err)
|
||||
}
|
||||
|
||||
testenv.AssertNoError(t, err)
|
||||
|
||||
defer func() {
|
||||
cleanupErr := eng.Cleanup()
|
||||
testenv.AssertNoError(t, cleanupErr)
|
||||
}()
|
||||
|
||||
ctx := context.TODO()
|
||||
err = eng.InitS3(ctx, s3DataRepoPath, s3MetadataRepoPath)
|
||||
testenv.AssertNoError(t, err)
|
||||
|
||||
// Perform writes
|
||||
fileSize := int64(256 * 1024 * 1024)
|
||||
numFiles := 10
|
||||
|
||||
fioOpt := fio.Options{}.WithFileSize(fileSize).WithNumFiles(numFiles)
|
||||
|
||||
err = eng.FileWriter.WriteFiles("", fioOpt)
|
||||
testenv.AssertNoError(t, err)
|
||||
|
||||
// Take a first snapshot
|
||||
snapID1, err := eng.Checker.TakeSnapshot(ctx, eng.FileWriter.LocalDataDir)
|
||||
testenv.AssertNoError(t, err)
|
||||
|
||||
// Get the metadata collected on that snapshot
|
||||
ssMeta1, err := eng.Checker.GetSnapshotMetadata(snapID1)
|
||||
testenv.AssertNoError(t, err)
|
||||
|
||||
// Do additional writes, writing 1 extra byte than before
|
||||
err = eng.FileWriter.WriteFiles("", fioOpt.WithFileSize(fileSize+1))
|
||||
testenv.AssertNoError(t, err)
|
||||
|
||||
// Take a second snapshot
|
||||
snapID2, err := eng.Checker.TakeSnapshot(ctx, eng.FileWriter.LocalDataDir)
|
||||
testenv.AssertNoError(t, err)
|
||||
|
||||
// Get the second snapshot's metadata
|
||||
ssMeta2, err := eng.Checker.GetSnapshotMetadata(snapID2)
|
||||
testenv.AssertNoError(t, err)
|
||||
|
||||
// Swap second snapshot's validation data into the first's metadata
|
||||
ssMeta1.ValidationData = ssMeta2.ValidationData
|
||||
|
||||
restoreDir, err := ioutil.TempDir(eng.Checker.RestoreDir, fmt.Sprintf("restore-snap-%v", snapID1))
|
||||
testenv.AssertNoError(t, err)
|
||||
|
||||
defer os.RemoveAll(restoreDir) //nolint:errcheck
|
||||
|
||||
// Restore snapshot ID 1 with snapshot 2's validation data in metadata, expect error
|
||||
err = eng.Checker.RestoreVerifySnapshot(ctx, snapID1, restoreDir, ssMeta1, os.Stdout)
|
||||
if err == nil {
|
||||
t.Fatalf("Expected an integrity error when trying to restore a snapshot with incorrect metadata")
|
||||
}
|
||||
}
|
||||
|
||||
func TestDataPersistency(t *testing.T) {
|
||||
tempDir, err := ioutil.TempDir("", "")
|
||||
testenv.AssertNoError(t, err)
|
||||
|
||||
defer os.RemoveAll(tempDir) //nolint:errcheck
|
||||
|
||||
eng, err := NewEngine()
|
||||
if err == kopiarunner.ErrExeVariableNotSet || errors.Is(err, fio.ErrEnvNotSet) {
|
||||
t.Skip(err)
|
||||
}
|
||||
|
||||
testenv.AssertNoError(t, err)
|
||||
|
||||
defer func() {
|
||||
cleanupErr := eng.Cleanup()
|
||||
testenv.AssertNoError(t, cleanupErr)
|
||||
}()
|
||||
|
||||
dataRepoPath := filepath.Join(tempDir, "data-repo-")
|
||||
metadataRepoPath := filepath.Join(tempDir, "metadata-repo-")
|
||||
|
||||
ctx := context.TODO()
|
||||
err = eng.InitFilesystem(ctx, dataRepoPath, metadataRepoPath)
|
||||
testenv.AssertNoError(t, err)
|
||||
|
||||
// Perform writes
|
||||
fileSize := int64(256 * 1024 * 1024)
|
||||
numFiles := 10
|
||||
|
||||
fioOpt := fio.Options{}.WithFileSize(fileSize).WithNumFiles(numFiles)
|
||||
|
||||
err = eng.FileWriter.WriteFiles("", fioOpt)
|
||||
testenv.AssertNoError(t, err)
|
||||
|
||||
// Take a snapshot
|
||||
snapID, err := eng.Checker.TakeSnapshot(ctx, eng.FileWriter.LocalDataDir)
|
||||
testenv.AssertNoError(t, err)
|
||||
|
||||
// Get the walk data associated with the snapshot that was taken
|
||||
dataDirWalk, err := eng.Checker.GetSnapshotMetadata(snapID)
|
||||
testenv.AssertNoError(t, err)
|
||||
|
||||
// Flush the snapshot metadata to persistent storage
|
||||
err = eng.MetaStore.FlushMetadata()
|
||||
testenv.AssertNoError(t, err)
|
||||
|
||||
// Create a new engine
|
||||
eng2, err := NewEngine()
|
||||
testenv.AssertNoError(t, err)
|
||||
|
||||
defer eng2.cleanup()
|
||||
|
||||
// Connect this engine to the same data and metadata repositories -
|
||||
// expect that the snapshot taken above will be found in metadata,
|
||||
// and the data will be chosen to be restored to this engine's DataDir
|
||||
// as a starting point.
|
||||
err = eng2.InitFilesystem(ctx, dataRepoPath, metadataRepoPath)
|
||||
testenv.AssertNoError(t, err)
|
||||
|
||||
// Compare the data directory of the second engine with the fingerprint
|
||||
// of the snapshot taken earlier. They should match.
|
||||
err = fswalker.NewWalkCompare().Compare(ctx, eng2.FileWriter.LocalDataDir, dataDirWalk.ValidationData, os.Stdout)
|
||||
testenv.AssertNoError(t, err)
|
||||
}
|
||||
Reference in New Issue
Block a user