diff --git a/tests/robustness/snapmeta/snapmeta.go b/tests/robustness/snapmeta/snapmeta.go index a4dd5841a..85b34f9f1 100644 --- a/tests/robustness/snapmeta/snapmeta.go +++ b/tests/robustness/snapmeta/snapmeta.go @@ -20,4 +20,5 @@ type Persister interface { snap.RepoManager LoadMetadata() error FlushMetadata() error + Cleanup() } diff --git a/tests/robustness/test_engine/engine.go b/tests/robustness/test_engine/engine.go new file mode 100644 index 000000000..8e0ce73b5 --- /dev/null +++ b/tests/robustness/test_engine/engine.go @@ -0,0 +1,194 @@ +// +build darwin linux + +// Package engine provides the framework for a snapshot repository testing engine +package engine + +import ( + "context" + "fmt" + "math/rand" + "os" + "strconv" + + "github.com/kopia/kopia/tests/robustness/checker" + "github.com/kopia/kopia/tests/robustness/snap" + "github.com/kopia/kopia/tests/robustness/snapmeta" + "github.com/kopia/kopia/tests/tools/fio" + "github.com/kopia/kopia/tests/tools/fswalker" + "github.com/kopia/kopia/tests/tools/kopiarunner" +) + +const ( + // S3BucketNameEnvKey is the environment variable required to connect to a repo on S3 + S3BucketNameEnvKey = "S3_BUCKET_NAME" +) + +// ErrS3BucketNameEnvUnset is the error returned when the S3BucketNameEnvKey environment variable is not set +var ErrS3BucketNameEnvUnset = fmt.Errorf("environment variable required: %v", S3BucketNameEnvKey) + +// Engine is the outer level testing framework for robustness testing +type Engine struct { + FileWriter *fio.Runner + TestRepo snap.Snapshotter + MetaStore snapmeta.Persister + Checker *checker.Checker + cleanupRoutines []func() +} + +// NewEngine instantiates a new Engine and returns its pointer. It is +// currently created with: +// - FIO file writer +// - Kopia test repo snapshotter +// - Kopia metadata storage repo +// - FSWalker data integrity checker +func NewEngine() (*Engine, error) { + e := new(Engine) + + var err error + + // Fill the file writer + e.FileWriter, err = fio.NewRunner() + if err != nil { + e.Cleanup() //nolint:errcheck + return nil, err + } + + e.cleanupRoutines = append(e.cleanupRoutines, e.FileWriter.Cleanup) + + // Fill Snapshotter interface + kopiaSnapper, err := kopiarunner.NewKopiaSnapshotter() + if err != nil { + e.Cleanup() //nolint:errcheck + return nil, err + } + + e.cleanupRoutines = append(e.cleanupRoutines, kopiaSnapper.Cleanup) + e.TestRepo = kopiaSnapper + + // Fill the snapshot store interface + snapStore, err := snapmeta.New() + if err != nil { + e.Cleanup() //nolint:errcheck + return nil, err + } + + e.cleanupRoutines = append(e.cleanupRoutines, snapStore.Cleanup) + + e.MetaStore = snapStore + + // Create the data integrity checker + chk, err := checker.NewChecker(kopiaSnapper, snapStore, fswalker.NewWalkCompare()) + e.cleanupRoutines = append(e.cleanupRoutines, chk.Cleanup) + + if err != nil { + e.Cleanup() //nolint:errcheck + return nil, err + } + + e.Checker = chk + + return e, nil +} + +// Cleanup cleans up after each component of the test engine +func (e *Engine) Cleanup() error { + defer e.cleanup() + + if e.MetaStore != nil { + return e.MetaStore.FlushMetadata() + } + + return nil +} + +func (e *Engine) cleanup() { + for _, f := range e.cleanupRoutines { + f() + } +} + +// InitS3 attempts to connect to a test repo and metadata repo on S3. If connection +// is successful, the engine is populated with the metadata associated with the +// snapshot in that repo. A new repo will be created if one does not already +// exist. +func (e *Engine) InitS3(ctx context.Context, testRepoPath, metaRepoPath string) error { + bucketName := os.Getenv(S3BucketNameEnvKey) + if bucketName == "" { + return ErrS3BucketNameEnvUnset + } + + err := e.MetaStore.ConnectOrCreateS3(bucketName, metaRepoPath) + if err != nil { + return err + } + + err = e.MetaStore.LoadMetadata() + if err != nil { + return err + } + + err = e.TestRepo.ConnectOrCreateS3(bucketName, testRepoPath) + if err != nil { + return err + } + + _, _, err = e.TestRepo.Run("policy", "set", "--global", "--keep-latest", strconv.Itoa(1<<31-1)) + if err != nil { + return err + } + + err = e.Checker.VerifySnapshotMetadata() + if err != nil { + return err + } + + snapIDs := e.Checker.GetLiveSnapIDs() + if len(snapIDs) > 0 { + randSnapID := snapIDs[rand.Intn(len(snapIDs))] + + err = e.Checker.RestoreSnapshotToPath(ctx, randSnapID, e.FileWriter.LocalDataDir, os.Stdout) + if err != nil { + return err + } + } + + return nil +} + +// InitFilesystem attempts to connect to a test repo and metadata repo on the local +// filesystem. If connection is successful, the engine is populated with the +// metadata associated with the snapshot in that repo. A new repo will be created if +// one does not already exist. +func (e *Engine) InitFilesystem(ctx context.Context, testRepoPath, metaRepoPath string) error { + err := e.MetaStore.ConnectOrCreateFilesystem(metaRepoPath) + if err != nil { + return err + } + + err = e.MetaStore.LoadMetadata() + if err != nil { + return err + } + + err = e.TestRepo.ConnectOrCreateFilesystem(testRepoPath) + if err != nil { + return err + } + + err = e.Checker.VerifySnapshotMetadata() + if err != nil { + return err + } + + snapIDs := e.Checker.GetSnapIDs() + if len(snapIDs) > 0 { + randSnapID := snapIDs[rand.Intn(len(snapIDs))] + + err = e.Checker.RestoreSnapshotToPath(ctx, randSnapID, e.FileWriter.LocalDataDir, os.Stdout) + if err != nil { + return err + } + } + + return nil +} diff --git a/tests/robustness/test_engine/engine_test.go b/tests/robustness/test_engine/engine_test.go new file mode 100644 index 000000000..c34fc5b43 --- /dev/null +++ b/tests/robustness/test_engine/engine_test.go @@ -0,0 +1,270 @@ +// +build darwin linux + +// Package engine provides the framework for a snapshot repository testing engine +package engine + +import ( + "context" + "errors" + "fmt" + "io/ioutil" + "os" + "path/filepath" + "testing" + + "github.com/kopia/kopia/tests/testenv" + "github.com/kopia/kopia/tests/tools/fio" + "github.com/kopia/kopia/tests/tools/fswalker" + "github.com/kopia/kopia/tests/tools/kopiarunner" +) + +var ( + fsMetadataRepoPath = filepath.Join("/tmp", "metadata-repo") + s3MetadataRepoPath = filepath.Join("some/path", "metadata-repo") + fsDataRepoPath = filepath.Join("/tmp", "data-repo") + s3DataRepoPath = filepath.Join("some/path", "data-repo") +) + +func TestEngineWritefilesBasicFS(t *testing.T) { + eng, err := NewEngine() + if err == kopiarunner.ErrExeVariableNotSet || errors.Is(err, fio.ErrEnvNotSet) { + t.Skip(err) + } + + testenv.AssertNoError(t, err) + + defer func() { + cleanupErr := eng.Cleanup() + testenv.AssertNoError(t, cleanupErr) + }() + + ctx := context.TODO() + err = eng.InitFilesystem(ctx, fsDataRepoPath, fsMetadataRepoPath) + testenv.AssertNoError(t, err) + + fileSize := int64(256 * 1024 * 1024) + numFiles := 10 + + fioOpt := fio.Options{}.WithFileSize(fileSize).WithNumFiles(numFiles) + + err = eng.FileWriter.WriteFiles("", fioOpt) + testenv.AssertNoError(t, err) + + snapIDs := eng.Checker.GetSnapIDs() + + snapID, err := eng.Checker.TakeSnapshot(ctx, eng.FileWriter.LocalDataDir) + testenv.AssertNoError(t, err) + + err = eng.Checker.RestoreSnapshot(ctx, snapID, os.Stdout) + testenv.AssertNoError(t, err) + + for _, sID := range snapIDs { + err = eng.Checker.RestoreSnapshot(ctx, sID, os.Stdout) + testenv.AssertNoError(t, err) + } +} + +func TestWriteFilesBasicS3(t *testing.T) { + eng, err := NewEngine() + if err == kopiarunner.ErrExeVariableNotSet || errors.Is(err, fio.ErrEnvNotSet) { + t.Skip(err) + } + + testenv.AssertNoError(t, err) + + defer func() { + cleanupErr := eng.Cleanup() + testenv.AssertNoError(t, cleanupErr) + }() + + ctx := context.TODO() + err = eng.InitS3(ctx, s3DataRepoPath, s3MetadataRepoPath) + testenv.AssertNoError(t, err) + + fileSize := int64(256 * 1024 * 1024) + numFiles := 10 + + fioOpt := fio.Options{}.WithFileSize(fileSize).WithNumFiles(numFiles) + + err = eng.FileWriter.WriteFiles("", fioOpt) + testenv.AssertNoError(t, err) + + snapIDs := eng.Checker.GetLiveSnapIDs() + + snapID, err := eng.Checker.TakeSnapshot(ctx, eng.FileWriter.LocalDataDir) + testenv.AssertNoError(t, err) + + err = eng.Checker.RestoreSnapshot(ctx, snapID, os.Stdout) + testenv.AssertNoError(t, err) + + for _, sID := range snapIDs { + err = eng.Checker.RestoreSnapshot(ctx, sID, os.Stdout) + testenv.AssertNoError(t, err) + } +} + +func TestDeleteSnapshotS3(t *testing.T) { + eng, err := NewEngine() + if err == kopiarunner.ErrExeVariableNotSet || errors.Is(err, fio.ErrEnvNotSet) { + t.Skip(err) + } + + testenv.AssertNoError(t, err) + + defer func() { + cleanupErr := eng.Cleanup() + testenv.AssertNoError(t, cleanupErr) + }() + + ctx := context.TODO() + err = eng.InitS3(ctx, s3DataRepoPath, s3MetadataRepoPath) + testenv.AssertNoError(t, err) + + fileSize := int64(256 * 1024 * 1024) + numFiles := 10 + + fioOpt := fio.Options{}.WithFileSize(fileSize).WithNumFiles(numFiles) + + err = eng.FileWriter.WriteFiles("", fioOpt) + testenv.AssertNoError(t, err) + + snapID, err := eng.Checker.TakeSnapshot(ctx, eng.FileWriter.LocalDataDir) + testenv.AssertNoError(t, err) + + err = eng.Checker.RestoreSnapshot(ctx, snapID, os.Stdout) + testenv.AssertNoError(t, err) + + err = eng.Checker.DeleteSnapshot(ctx, snapID) + testenv.AssertNoError(t, err) + + err = eng.Checker.RestoreSnapshot(ctx, snapID, os.Stdout) + if err == nil { + t.Fatalf("Expected an error when trying to restore a deleted snapshot") + } +} + +func TestSnapshotVerificationFail(t *testing.T) { + eng, err := NewEngine() + if err == kopiarunner.ErrExeVariableNotSet || errors.Is(err, fio.ErrEnvNotSet) { + t.Skip(err) + } + + testenv.AssertNoError(t, err) + + defer func() { + cleanupErr := eng.Cleanup() + testenv.AssertNoError(t, cleanupErr) + }() + + ctx := context.TODO() + err = eng.InitS3(ctx, s3DataRepoPath, s3MetadataRepoPath) + testenv.AssertNoError(t, err) + + // Perform writes + fileSize := int64(256 * 1024 * 1024) + numFiles := 10 + + fioOpt := fio.Options{}.WithFileSize(fileSize).WithNumFiles(numFiles) + + err = eng.FileWriter.WriteFiles("", fioOpt) + testenv.AssertNoError(t, err) + + // Take a first snapshot + snapID1, err := eng.Checker.TakeSnapshot(ctx, eng.FileWriter.LocalDataDir) + testenv.AssertNoError(t, err) + + // Get the metadata collected on that snapshot + ssMeta1, err := eng.Checker.GetSnapshotMetadata(snapID1) + testenv.AssertNoError(t, err) + + // Do additional writes, writing 1 extra byte than before + err = eng.FileWriter.WriteFiles("", fioOpt.WithFileSize(fileSize+1)) + testenv.AssertNoError(t, err) + + // Take a second snapshot + snapID2, err := eng.Checker.TakeSnapshot(ctx, eng.FileWriter.LocalDataDir) + testenv.AssertNoError(t, err) + + // Get the second snapshot's metadata + ssMeta2, err := eng.Checker.GetSnapshotMetadata(snapID2) + testenv.AssertNoError(t, err) + + // Swap second snapshot's validation data into the first's metadata + ssMeta1.ValidationData = ssMeta2.ValidationData + + restoreDir, err := ioutil.TempDir(eng.Checker.RestoreDir, fmt.Sprintf("restore-snap-%v", snapID1)) + testenv.AssertNoError(t, err) + + defer os.RemoveAll(restoreDir) //nolint:errcheck + + // Restore snapshot ID 1 with snapshot 2's validation data in metadata, expect error + err = eng.Checker.RestoreVerifySnapshot(ctx, snapID1, restoreDir, ssMeta1, os.Stdout) + if err == nil { + t.Fatalf("Expected an integrity error when trying to restore a snapshot with incorrect metadata") + } +} + +func TestDataPersistency(t *testing.T) { + tempDir, err := ioutil.TempDir("", "") + testenv.AssertNoError(t, err) + + defer os.RemoveAll(tempDir) //nolint:errcheck + + eng, err := NewEngine() + if err == kopiarunner.ErrExeVariableNotSet || errors.Is(err, fio.ErrEnvNotSet) { + t.Skip(err) + } + + testenv.AssertNoError(t, err) + + defer func() { + cleanupErr := eng.Cleanup() + testenv.AssertNoError(t, cleanupErr) + }() + + dataRepoPath := filepath.Join(tempDir, "data-repo-") + metadataRepoPath := filepath.Join(tempDir, "metadata-repo-") + + ctx := context.TODO() + err = eng.InitFilesystem(ctx, dataRepoPath, metadataRepoPath) + testenv.AssertNoError(t, err) + + // Perform writes + fileSize := int64(256 * 1024 * 1024) + numFiles := 10 + + fioOpt := fio.Options{}.WithFileSize(fileSize).WithNumFiles(numFiles) + + err = eng.FileWriter.WriteFiles("", fioOpt) + testenv.AssertNoError(t, err) + + // Take a snapshot + snapID, err := eng.Checker.TakeSnapshot(ctx, eng.FileWriter.LocalDataDir) + testenv.AssertNoError(t, err) + + // Get the walk data associated with the snapshot that was taken + dataDirWalk, err := eng.Checker.GetSnapshotMetadata(snapID) + testenv.AssertNoError(t, err) + + // Flush the snapshot metadata to persistent storage + err = eng.MetaStore.FlushMetadata() + testenv.AssertNoError(t, err) + + // Create a new engine + eng2, err := NewEngine() + testenv.AssertNoError(t, err) + + defer eng2.cleanup() + + // Connect this engine to the same data and metadata repositories - + // expect that the snapshot taken above will be found in metadata, + // and the data will be chosen to be restored to this engine's DataDir + // as a starting point. + err = eng2.InitFilesystem(ctx, dataRepoPath, metadataRepoPath) + testenv.AssertNoError(t, err) + + // Compare the data directory of the second engine with the fingerprint + // of the snapshot taken earlier. They should match. + err = fswalker.NewWalkCompare().Compare(ctx, eng2.FileWriter.LocalDataDir, dataDirWalk.ValidationData, os.Stdout) + testenv.AssertNoError(t, err) +}