From c23d42f84db549fc0f903791ce652a9c4993c009 Mon Sep 17 00:00:00 2001 From: carlbraganza Date: Wed, 3 Feb 2021 11:19:33 -0800 Subject: [PATCH] Refactored the robustness engine constructor to use externally specified interfaces. (#815) --- tests/robustness/checker/checker.go | 28 +- tests/robustness/comparer.go | 4 +- tests/robustness/engine/action.go | 10 +- tests/robustness/engine/engine.go | 300 +++++------------- tests/robustness/engine/engine_test.go | 261 +++++++++++---- tests/robustness/engine/metadata.go | 16 +- tests/robustness/filewriter.go | 4 - .../fiofilewriter/fio_filewriter.go | 4 +- tests/robustness/persister.go | 2 - tests/robustness/robustness_test/main_test.go | 224 +++++++++++-- tests/robustness/snapmeta/kopia_connector.go | 104 ++++++ .../snapmeta/kopia_connector_test.go | 117 +++++++ .../{kopia_meta.go => kopia_persister.go} | 72 +++-- .../robustness/snapmeta/kopia_snapshotter.go | 98 ++++++ tests/robustness/snapshotter.go | 22 +- tests/tools/fswalker/fswalker.go | 4 +- tests/tools/fswalker/fswalker_test.go | 4 +- tests/tools/kopiarunner/kopia_snapshotter.go | 3 - 18 files changed, 871 insertions(+), 406 deletions(-) create mode 100644 tests/robustness/snapmeta/kopia_connector.go create mode 100644 tests/robustness/snapmeta/kopia_connector_test.go rename tests/robustness/snapmeta/{kopia_meta.go => kopia_persister.go} (63%) create mode 100644 tests/robustness/snapmeta/kopia_snapshotter.go diff --git a/tests/robustness/checker/checker.go b/tests/robustness/checker/checker.go index e7796050b..c1af6b056 100644 --- a/tests/robustness/checker/checker.go +++ b/tests/robustness/checker/checker.go @@ -161,7 +161,7 @@ func (chk *Checker) VerifySnapshotMetadata() error { // Might as well delete the snapshot since we don't have metadata for it log.Printf("Deleting snapshot ID %s", liveSnapID) - err = chk.snapshotIssuer.DeleteSnapshot(liveSnapID) + err = chk.snapshotIssuer.DeleteSnapshot(liveSnapID, map[string]string{}) if err != nil { log.Printf("error deleting snapshot: %s", err) errCount++ @@ -182,15 +182,15 @@ func (chk *Checker) VerifySnapshotMetadata() error { // TakeSnapshot gathers state information on the requested snapshot path, then // performs the snapshot action defined by the Checker's Snapshotter. -func (chk *Checker) TakeSnapshot(ctx context.Context, sourceDir string) (snapID string, err error) { - b, err := chk.validator.Gather(ctx, sourceDir) +func (chk *Checker) TakeSnapshot(ctx context.Context, sourceDir string, opts map[string]string) (snapID string, err error) { + b, err := chk.validator.Gather(ctx, sourceDir, opts) if err != nil { return "", err } ssStart := clock.Now() - snapID, err = chk.snapshotIssuer.CreateSnapshot(sourceDir) + snapID, err = chk.snapshotIssuer.CreateSnapshot(sourceDir, opts) if err != nil { return snapID, err } @@ -218,7 +218,7 @@ func (chk *Checker) TakeSnapshot(ctx context.Context, sourceDir string) (snapID // RestoreSnapshot restores a snapshot to the Checker's temporary restore directory // using the Checker's Snapshotter, and performs a data consistency check on the // resulting tree using the saved snapshot data. -func (chk *Checker) RestoreSnapshot(ctx context.Context, snapID string, reportOut io.Writer) error { +func (chk *Checker) RestoreSnapshot(ctx context.Context, snapID string, reportOut io.Writer, opts map[string]string) error { // Make an independent directory for the restore restoreSubDir, err := ioutil.TempDir(chk.RestoreDir, fmt.Sprintf("restore-snap-%v", snapID)) if err != nil { @@ -227,25 +227,25 @@ func (chk *Checker) RestoreSnapshot(ctx context.Context, snapID string, reportOu defer os.RemoveAll(restoreSubDir) //nolint:errcheck - return chk.RestoreSnapshotToPath(ctx, snapID, restoreSubDir, reportOut) + return chk.RestoreSnapshotToPath(ctx, snapID, restoreSubDir, reportOut, opts) } // RestoreSnapshotToPath restores a snapshot to the requested path // using the Checker's Snapshotter, and performs a data consistency check on the // resulting tree using the saved snapshot data. -func (chk *Checker) RestoreSnapshotToPath(ctx context.Context, snapID, destPath string, reportOut io.Writer) error { +func (chk *Checker) RestoreSnapshotToPath(ctx context.Context, snapID, destPath string, reportOut io.Writer, opts map[string]string) error { ssMeta, err := chk.loadSnapshotMetadata(snapID) if err != nil { return err } - return chk.RestoreVerifySnapshot(ctx, snapID, destPath, ssMeta, reportOut) + return chk.RestoreVerifySnapshot(ctx, snapID, destPath, ssMeta, reportOut, opts) } // RestoreVerifySnapshot restores a snapshot and verifies its integrity against // the metadata provided. -func (chk *Checker) RestoreVerifySnapshot(ctx context.Context, snapID, destPath string, ssMeta *SnapshotMetadata, reportOut io.Writer) error { - err := chk.snapshotIssuer.RestoreSnapshot(snapID, destPath) +func (chk *Checker) RestoreVerifySnapshot(ctx context.Context, snapID, destPath string, ssMeta *SnapshotMetadata, reportOut io.Writer, opts map[string]string) error { + err := chk.snapshotIssuer.RestoreSnapshot(snapID, destPath, opts) if err != nil { return err } @@ -253,7 +253,7 @@ func (chk *Checker) RestoreVerifySnapshot(ctx context.Context, snapID, destPath if ssMeta == nil && chk.RecoveryMode { var b []byte - b, err = chk.validator.Gather(ctx, destPath) + b, err = chk.validator.Gather(ctx, destPath, opts) if err != nil { return err } @@ -266,7 +266,7 @@ func (chk *Checker) RestoreVerifySnapshot(ctx context.Context, snapID, destPath return chk.saveSnapshotMetadata(ssMeta) } - err = chk.validator.Compare(ctx, destPath, ssMeta.ValidationData, reportOut) + err = chk.validator.Compare(ctx, destPath, ssMeta.ValidationData, reportOut, opts) if err != nil { return err } @@ -282,8 +282,8 @@ const ( // DeleteSnapshot performs the Snapshotter's DeleteSnapshot action, and // marks the snapshot with the given snapshot ID as deleted. -func (chk *Checker) DeleteSnapshot(ctx context.Context, snapID string) error { - err := chk.snapshotIssuer.DeleteSnapshot(snapID) +func (chk *Checker) DeleteSnapshot(ctx context.Context, snapID string, opts map[string]string) error { + err := chk.snapshotIssuer.DeleteSnapshot(snapID, opts) if err != nil { return err } diff --git a/tests/robustness/comparer.go b/tests/robustness/comparer.go index d9ab13d24..66ef1cd8e 100644 --- a/tests/robustness/comparer.go +++ b/tests/robustness/comparer.go @@ -8,6 +8,6 @@ import ( // Comparer describes an interface that gathers state data on a provided // path, and compares that data to the state on another path. type Comparer interface { - Gather(ctx context.Context, path string) ([]byte, error) - Compare(ctx context.Context, path string, data []byte, reportOut io.Writer) error + Gather(ctx context.Context, path string, opts map[string]string) ([]byte, error) + Compare(ctx context.Context, path string, data []byte, reportOut io.Writer, opts map[string]string) error } diff --git a/tests/robustness/engine/action.go b/tests/robustness/engine/action.go index 7dfbc4170..9f2a6f70d 100644 --- a/tests/robustness/engine/action.go +++ b/tests/robustness/engine/action.go @@ -175,7 +175,7 @@ var actions = map[ActionKey]Action{ log.Printf("Creating snapshot of root directory %s", e.FileWriter.DataDirectory()) ctx := context.TODO() - snapID, err := e.Checker.TakeSnapshot(ctx, e.FileWriter.DataDirectory()) + snapID, err := e.Checker.TakeSnapshot(ctx, e.FileWriter.DataDirectory(), opts) setLogEntryCmdOpts(l, map[string]string{ "snap-dir": e.FileWriter.DataDirectory(), @@ -201,7 +201,7 @@ var actions = map[ActionKey]Action{ ctx := context.Background() b := &bytes.Buffer{} - err = e.Checker.RestoreSnapshot(ctx, snapID, b) + err = e.Checker.RestoreSnapshot(ctx, snapID, b, opts) if err != nil { log.Print(b.String()) } @@ -221,13 +221,13 @@ var actions = map[ActionKey]Action{ setLogEntryCmdOpts(l, map[string]string{"snapID": snapID}) ctx := context.Background() - err = e.Checker.DeleteSnapshot(ctx, snapID) + err = e.Checker.DeleteSnapshot(ctx, snapID, opts) return nil, err }, }, GCActionKey: { f: func(e *Engine, opts map[string]string, l *LogEntry) (out map[string]string, err error) { - return nil, e.TestRepo.RunGC() + return nil, e.TestRepo.RunGC(opts) }, }, WriteRandomFilesActionKey: { @@ -266,7 +266,7 @@ var actions = map[ActionKey]Action{ setLogEntryCmdOpts(l, map[string]string{"snapID": snapID}) b := &bytes.Buffer{} - err = e.Checker.RestoreSnapshotToPath(context.Background(), snapID, e.FileWriter.DataDirectory(), b) + err = e.Checker.RestoreSnapshotToPath(context.Background(), snapID, e.FileWriter.DataDirectory(), b, opts) if err != nil { log.Print(b.String()) return nil, err diff --git a/tests/robustness/engine/engine.go b/tests/robustness/engine/engine.go index a8ec37284..565ba5af6 100644 --- a/tests/robustness/engine/engine.go +++ b/tests/robustness/engine/engine.go @@ -7,133 +7,104 @@ import ( "context" "fmt" "io" - "io/ioutil" "log" "os" - "os/exec" "path/filepath" - "strconv" "strings" - "syscall" "time" "github.com/kopia/kopia/tests/robustness" "github.com/kopia/kopia/tests/robustness/checker" - "github.com/kopia/kopia/tests/robustness/fiofilewriter" - "github.com/kopia/kopia/tests/robustness/snapmeta" - "github.com/kopia/kopia/tests/tools/fswalker" - "github.com/kopia/kopia/tests/tools/kopiarunner" -) - -const ( - // S3BucketNameEnvKey is the environment variable required to connect to a repo on S3. - S3BucketNameEnvKey = "S3_BUCKET_NAME" - // EngineModeEnvKey is the environment variable required to switch between basic and server/client model. - EngineModeEnvKey = "ENGINE_MODE" - // EngineModeBasic is a constant used to check the engineMode. - EngineModeBasic = "BASIC" - // EngineModeServer is a constant used to check the engineMode. - EngineModeServer = "SERVER" - // defaultAddr is used for setting the address of Kopia Server. - defaultAddr = "localhost:51515" ) var ( - // ErrS3BucketNameEnvUnset is the error returned when the S3BucketNameEnvKey environment variable is not set. - ErrS3BucketNameEnvUnset = fmt.Errorf("environment variable required: %v", S3BucketNameEnvKey) + // ErrorInvalidArgs is returned if the constructor arguments are incorrect. + ErrorInvalidArgs = fmt.Errorf("invalid arguments") + noSpaceOnDeviceMatchStr = "no space left on device" ) +// Args contain the parameters for the engine constructor. +type Args struct { + // Interfaces used by the engine. + MetaStore robustness.Persister + TestRepo robustness.Snapshotter + Validator robustness.Comparer + FileWriter robustness.FileWriter + + // WorkingDir is a directory to use for temporary data. + WorkingDir string + + // SyncRepositories should be set to true to reconcile differences. + SyncRepositories bool +} + +// Validate checks the arguments for correctness. +func (a *Args) Validate() error { + if a.MetaStore == nil || a.TestRepo == nil || a.Validator == nil || a.FileWriter == nil || a.WorkingDir == "" { + return ErrorInvalidArgs + } + + return nil +} + +// New creates an Engine. +func New(args *Args) (*Engine, error) { + if err := args.Validate(); err != nil { + return nil, err + } + + var ( + e = &Engine{ + MetaStore: args.MetaStore, + TestRepo: args.TestRepo, + Validator: args.Validator, + FileWriter: args.FileWriter, + baseDirPath: args.WorkingDir, + RunStats: Stats{ + RunCounter: 1, + CreationTime: time.Now(), + PerActionStats: make(map[ActionKey]*ActionStats), + }, + } + err error + ) + + if err = e.setupLogging(); err != nil { + e.cleanComponents() + return nil, err + } + + e.Checker, err = checker.NewChecker(e.TestRepo, e.MetaStore, e.Validator, e.baseDirPath) + if err != nil { + e.cleanComponents() + return nil, err + } + + e.Checker.RecoveryMode = args.SyncRepositories + e.cleanupRoutines = append(e.cleanupRoutines, e.Checker.Cleanup) + + return e, nil +} + // Engine is the outer level testing framework for robustness testing. type Engine struct { - FileWriter robustness.FileWriter - TestRepo robustness.Snapshotter - MetaStore robustness.Persister + FileWriter robustness.FileWriter + TestRepo robustness.Snapshotter + MetaStore robustness.Persister + Validator robustness.Comparer + Checker *checker.Checker cleanupRoutines []func() baseDirPath string - serverCmd *exec.Cmd RunStats Stats CumulativeStats Stats EngineLog Log } -// NewEngine instantiates a new Engine and returns its pointer. It is -// currently created with: -// - FIO file writer -// - Kopia test repo snapshotter -// - Kopia metadata storage repo -// - FSWalker data integrity checker. -func NewEngine(workingDir string) (*Engine, error) { - baseDirPath, err := ioutil.TempDir(workingDir, "engine-data-") - if err != nil { - return nil, err - } - - e := &Engine{ - baseDirPath: baseDirPath, - RunStats: Stats{ - RunCounter: 1, - CreationTime: time.Now(), - PerActionStats: make(map[ActionKey]*ActionStats), - }, - } - - // Create an FIO file writer - e.FileWriter, err = fiofilewriter.New() - if err != nil { - e.CleanComponents() - return nil, err - } - - e.cleanupRoutines = append(e.cleanupRoutines, e.FileWriter.Cleanup) - - // Fill Snapshotter interface - kopiaSnapper, err := kopiarunner.NewKopiaSnapshotter(baseDirPath) - if err != nil { - e.CleanComponents() - return nil, err - } - - e.cleanupRoutines = append(e.cleanupRoutines, kopiaSnapper.Cleanup) - e.TestRepo = kopiaSnapper - - // Fill the snapshot store interface - snapStore, err := snapmeta.New(baseDirPath) - if err != nil { - e.CleanComponents() - return nil, err - } - - e.cleanupRoutines = append(e.cleanupRoutines, snapStore.Cleanup) - - e.MetaStore = snapStore - - err = e.setupLogging() - if err != nil { - e.CleanComponents() - return nil, err - } - - // Create the data integrity checker - chk, err := checker.NewChecker(kopiaSnapper, snapStore, fswalker.NewWalkCompare(), baseDirPath) - e.cleanupRoutines = append(e.cleanupRoutines, chk.Cleanup) - - if err != nil { - e.CleanComponents() - return nil, err - } - - e.cleanupRoutines = append(e.cleanupRoutines, e.cleanUpServer) - - e.Checker = chk - - return e, nil -} - -// Cleanup cleans up after each component of the test engine. -func (e *Engine) Cleanup() error { +// Shutdown makes a last snapshot then flushes the metadata and prints the final statistics. +func (e *Engine) Shutdown() error { // Perform a snapshot action to capture the state of the data directory // at the end of the run lastWriteEntry := e.EngineLog.FindLastThisRun(WriteRandomFilesActionKey) @@ -159,15 +130,15 @@ func (e *Engine) Cleanup() error { e.RunStats.RunTime = time.Since(e.RunStats.CreationTime) e.CumulativeStats.RunTime += e.RunStats.RunTime - defer e.CleanComponents() + defer e.cleanComponents() if e.MetaStore != nil { - err := e.SaveLog() + err := e.saveLog() if err != nil { return err } - err = e.SaveStats() + err = e.saveStats() if err != nil { return err } @@ -200,140 +171,33 @@ func (e *Engine) formatLogName() string { return fmt.Sprintf("Log_%s", st.Format("2006_01_02_15_04_05")) } -// CleanComponents cleans up each component part of the test engine. -func (e *Engine) CleanComponents() { +// cleanComponents cleans up each component part of the test engine. +func (e *Engine) cleanComponents() { for _, f := range e.cleanupRoutines { if f != nil { f() } } - - os.RemoveAll(e.baseDirPath) //nolint:errcheck } -// Init initializes the Engine to a repository location according to the environment setup. -// - If S3_BUCKET_NAME is set, initialize S3 -// - Else initialize filesystem. -func (e *Engine) Init(ctx context.Context, testRepoPath, metaRepoPath string) error { - bucketName := os.Getenv(S3BucketNameEnvKey) - engineMode := os.Getenv(EngineModeEnvKey) - - switch { - case bucketName != "" && engineMode == EngineModeBasic: - return e.InitS3(ctx, bucketName, testRepoPath, metaRepoPath) - - case bucketName != "" && engineMode == EngineModeServer: - return e.InitS3WithServer(ctx, bucketName, testRepoPath, metaRepoPath, defaultAddr) - - case bucketName == "" && engineMode == EngineModeServer: - return e.InitFilesystemWithServer(ctx, testRepoPath, metaRepoPath, defaultAddr) - - default: - return e.InitFilesystem(ctx, testRepoPath, metaRepoPath) - } -} - -// InitS3 attempts to connect to a test repo and metadata repo on S3. If connection -// is successful, the engine is populated with the metadata associated with the -// snapshot in that repo. A new repo will be created if one does not already -// exist. -func (e *Engine) InitS3(ctx context.Context, bucketName, testRepoPath, metaRepoPath string) error { - err := e.MetaStore.ConnectOrCreateS3(bucketName, metaRepoPath) - if err != nil { - return err - } - - err = e.TestRepo.ConnectOrCreateS3(bucketName, testRepoPath) - if err != nil { - return err - } - - return e.init(ctx) -} - -// InitFilesystem attempts to connect to a test repo and metadata repo on the local -// filesystem. If connection is successful, the engine is populated with the -// metadata associated with the snapshot in that repo. A new repo will be created if -// one does not already exist. -func (e *Engine) InitFilesystem(ctx context.Context, testRepoPath, metaRepoPath string) error { - err := e.MetaStore.ConnectOrCreateFilesystem(metaRepoPath) - if err != nil { - return err - } - - err = e.TestRepo.ConnectOrCreateFilesystem(testRepoPath) - if err != nil { - return err - } - - return e.init(ctx) -} - -func (e *Engine) init(ctx context.Context) error { +// Init initializes the Engine and performs a consistency check. +func (e *Engine) Init(ctx context.Context) error { err := e.MetaStore.LoadMetadata() if err != nil { return err } - err = e.LoadStats() + err = e.loadStats() if err != nil { return err } e.CumulativeStats.RunCounter++ - err = e.LoadLog() - if err != nil { - return err - } - - _, _, err = e.TestRepo.Run("policy", "set", "--global", "--keep-latest", strconv.Itoa(1<<31-1), "--compression", "s2-default") + err = e.loadLog() if err != nil { return err } return e.Checker.VerifySnapshotMetadata() } - -// InitS3WithServer initializes the Engine with InitS3 for use with the server/client model. -func (e *Engine) InitS3WithServer(ctx context.Context, bucketName, testRepoPath, metaRepoPath, addr string) error { - if err := e.MetaStore.ConnectOrCreateS3(bucketName, metaRepoPath); err != nil { - return err - } - - cmd, err := e.TestRepo.ConnectOrCreateS3WithServer(addr, bucketName, testRepoPath) - if err != nil { - return err - } - - e.serverCmd = cmd - - return e.init(ctx) -} - -// InitFilesystemWithServer initializes the Engine for testing the server/client model with a local filesystem repository. -func (e *Engine) InitFilesystemWithServer(ctx context.Context, testRepoPath, metaRepoPath, addr string) error { - if err := e.MetaStore.ConnectOrCreateFilesystem(metaRepoPath); err != nil { - return err - } - - cmd, err := e.TestRepo.ConnectOrCreateFilesystemWithServer(addr, testRepoPath) - if err != nil { - return err - } - - e.serverCmd = cmd - - return e.init(ctx) -} - -// cleanUpServer cleans up the server process. -func (e *Engine) cleanUpServer() { - if e.serverCmd == nil { - return - } - - if err := e.serverCmd.Process.Signal(syscall.SIGTERM); err != nil { - log.Println("Failed to send termination signal to kopia server process:", err) - } -} diff --git a/tests/robustness/engine/engine_test.go b/tests/robustness/engine/engine_test.go index 04df637e6..936684823 100644 --- a/tests/robustness/engine/engine_test.go +++ b/tests/robustness/engine/engine_test.go @@ -11,10 +11,12 @@ import ( "fmt" "io" "io/ioutil" + "log" "math" "os" "path/filepath" "strconv" + "syscall" "testing" "time" @@ -45,7 +47,10 @@ var ( ) func TestEngineWritefilesBasicFS(t *testing.T) { - eng, err := NewEngine("") + os.Setenv(snapmeta.EngineModeEnvKey, snapmeta.EngineModeBasic) + os.Setenv(snapmeta.S3BucketNameEnvKey, "") + + th, eng, err := newTestHarness(t, fsDataRepoPath, fsMetadataRepoPath) if errors.Is(err, kopiarunner.ErrExeVariableNotSet) || errors.Is(err, fio.ErrEnvNotSet) { t.Skip(err) } @@ -53,34 +58,35 @@ func TestEngineWritefilesBasicFS(t *testing.T) { testenv.AssertNoError(t, err) defer func() { - cleanupErr := eng.Cleanup() + cleanupErr := th.Cleanup() testenv.AssertNoError(t, cleanupErr) os.RemoveAll(fsRepoBaseDirPath) }() + opts := map[string]string{} ctx := context.TODO() - err = eng.InitFilesystem(ctx, fsDataRepoPath, fsMetadataRepoPath) + err = eng.Init(ctx) testenv.AssertNoError(t, err) fileSize := int64(256 * 1024) numFiles := 10 fioOpts := fio.Options{}.WithFileSize(fileSize).WithNumFiles(numFiles) - fioRunner := engineFioRunner(t, eng) + fioRunner := th.FioRunner() err = fioRunner.WriteFiles("", fioOpts) testenv.AssertNoError(t, err) snapIDs := eng.Checker.GetSnapIDs() - snapID, err := eng.Checker.TakeSnapshot(ctx, fioRunner.LocalDataDir) + snapID, err := eng.Checker.TakeSnapshot(ctx, fioRunner.LocalDataDir, opts) testenv.AssertNoError(t, err) - err = eng.Checker.RestoreSnapshot(ctx, snapID, os.Stdout) + err = eng.Checker.RestoreSnapshot(ctx, snapID, os.Stdout, opts) testenv.AssertNoError(t, err) for _, sID := range snapIDs { - err = eng.Checker.RestoreSnapshot(ctx, sID, os.Stdout) + err = eng.Checker.RestoreSnapshot(ctx, sID, os.Stdout, opts) testenv.AssertNoError(t, err) } } @@ -157,7 +163,10 @@ func TestWriteFilesBasicS3(t *testing.T) { bucketName, cleanupCB := makeTempS3Bucket(t) defer cleanupCB() - eng, err := NewEngine("") + os.Setenv(snapmeta.EngineModeEnvKey, snapmeta.EngineModeBasic) + os.Setenv(snapmeta.S3BucketNameEnvKey, bucketName) + + th, eng, err := newTestHarness(t, s3DataRepoPath, s3MetadataRepoPath) if errors.Is(err, kopiarunner.ErrExeVariableNotSet) || errors.Is(err, fio.ErrEnvNotSet) { t.Skip(err) } @@ -165,32 +174,33 @@ func TestWriteFilesBasicS3(t *testing.T) { testenv.AssertNoError(t, err) defer func() { - cleanupErr := eng.Cleanup() + cleanupErr := th.Cleanup() testenv.AssertNoError(t, cleanupErr) }() + opts := map[string]string{} ctx := context.TODO() - err = eng.InitS3(ctx, bucketName, s3DataRepoPath, s3MetadataRepoPath) + err = eng.Init(ctx) testenv.AssertNoError(t, err) fileSize := int64(256 * 1024) numFiles := 10 fioOpts := fio.Options{}.WithFileSize(fileSize).WithNumFiles(numFiles) - fioRunner := engineFioRunner(t, eng) + fioRunner := th.FioRunner() err = fioRunner.WriteFiles("", fioOpts) testenv.AssertNoError(t, err) snapIDs := eng.Checker.GetLiveSnapIDs() - snapID, err := eng.Checker.TakeSnapshot(ctx, fioRunner.LocalDataDir) + snapID, err := eng.Checker.TakeSnapshot(ctx, fioRunner.LocalDataDir, opts) testenv.AssertNoError(t, err) - err = eng.Checker.RestoreSnapshot(ctx, snapID, os.Stdout) + err = eng.Checker.RestoreSnapshot(ctx, snapID, os.Stdout, opts) testenv.AssertNoError(t, err) for _, sID := range snapIDs { - err = eng.Checker.RestoreSnapshot(ctx, sID, os.Stdout) + err = eng.Checker.RestoreSnapshot(ctx, sID, os.Stdout, opts) testenv.AssertNoError(t, err) } } @@ -199,7 +209,10 @@ func TestDeleteSnapshotS3(t *testing.T) { bucketName, cleanupCB := makeTempS3Bucket(t) defer cleanupCB() - eng, err := NewEngine("") + os.Setenv(snapmeta.EngineModeEnvKey, snapmeta.EngineModeBasic) + os.Setenv(snapmeta.S3BucketNameEnvKey, bucketName) + + th, eng, err := newTestHarness(t, s3DataRepoPath, s3MetadataRepoPath) if errors.Is(err, kopiarunner.ErrExeVariableNotSet) || errors.Is(err, fio.ErrEnvNotSet) { t.Skip(err) } @@ -207,32 +220,33 @@ func TestDeleteSnapshotS3(t *testing.T) { testenv.AssertNoError(t, err) defer func() { - cleanupErr := eng.Cleanup() + cleanupErr := th.Cleanup() testenv.AssertNoError(t, cleanupErr) }() + opts := map[string]string{} ctx := context.TODO() - err = eng.InitS3(ctx, bucketName, s3DataRepoPath, s3MetadataRepoPath) + err = eng.Init(ctx) testenv.AssertNoError(t, err) fileSize := int64(256 * 1024) numFiles := 10 fioOpts := fio.Options{}.WithFileSize(fileSize).WithNumFiles(numFiles) - fioRunner := engineFioRunner(t, eng) + fioRunner := th.FioRunner() err = fioRunner.WriteFiles("", fioOpts) testenv.AssertNoError(t, err) - snapID, err := eng.Checker.TakeSnapshot(ctx, fioRunner.LocalDataDir) + snapID, err := eng.Checker.TakeSnapshot(ctx, fioRunner.LocalDataDir, opts) testenv.AssertNoError(t, err) - err = eng.Checker.RestoreSnapshot(ctx, snapID, os.Stdout) + err = eng.Checker.RestoreSnapshot(ctx, snapID, os.Stdout, opts) testenv.AssertNoError(t, err) - err = eng.Checker.DeleteSnapshot(ctx, snapID) + err = eng.Checker.DeleteSnapshot(ctx, snapID, opts) testenv.AssertNoError(t, err) - err = eng.Checker.RestoreSnapshot(ctx, snapID, os.Stdout) + err = eng.Checker.RestoreSnapshot(ctx, snapID, os.Stdout, opts) if err == nil { t.Fatalf("Expected an error when trying to restore a deleted snapshot") } @@ -242,7 +256,10 @@ func TestSnapshotVerificationFail(t *testing.T) { bucketName, cleanupCB := makeTempS3Bucket(t) defer cleanupCB() - eng, err := NewEngine("") + os.Setenv(snapmeta.EngineModeEnvKey, snapmeta.EngineModeBasic) + os.Setenv(snapmeta.S3BucketNameEnvKey, bucketName) + + th, eng, err := newTestHarness(t, s3DataRepoPath, s3MetadataRepoPath) if errors.Is(err, kopiarunner.ErrExeVariableNotSet) || errors.Is(err, fio.ErrEnvNotSet) { t.Skip(err) } @@ -250,12 +267,13 @@ func TestSnapshotVerificationFail(t *testing.T) { testenv.AssertNoError(t, err) defer func() { - cleanupErr := eng.Cleanup() + cleanupErr := th.Cleanup() testenv.AssertNoError(t, cleanupErr) }() + opts := map[string]string{} ctx := context.TODO() - err = eng.InitS3(ctx, bucketName, s3DataRepoPath, s3MetadataRepoPath) + err = eng.Init(ctx) testenv.AssertNoError(t, err) // Perform writes @@ -263,12 +281,12 @@ func TestSnapshotVerificationFail(t *testing.T) { numFiles := 10 fioOpt := fio.Options{}.WithFileSize(fileSize).WithNumFiles(numFiles) - fioRunner := engineFioRunner(t, eng) + fioRunner := th.FioRunner() err = fioRunner.WriteFiles("", fioOpt) testenv.AssertNoError(t, err) // Take a first snapshot - snapID1, err := eng.Checker.TakeSnapshot(ctx, fioRunner.LocalDataDir) + snapID1, err := eng.Checker.TakeSnapshot(ctx, fioRunner.LocalDataDir, opts) testenv.AssertNoError(t, err) // Get the metadata collected on that snapshot @@ -280,7 +298,7 @@ func TestSnapshotVerificationFail(t *testing.T) { testenv.AssertNoError(t, err) // Take a second snapshot - snapID2, err := eng.Checker.TakeSnapshot(ctx, fioRunner.LocalDataDir) + snapID2, err := eng.Checker.TakeSnapshot(ctx, fioRunner.LocalDataDir, opts) testenv.AssertNoError(t, err) // Get the second snapshot's metadata @@ -296,19 +314,25 @@ func TestSnapshotVerificationFail(t *testing.T) { defer os.RemoveAll(restoreDir) // Restore snapshot ID 1 with snapshot 2's validation data in metadata, expect error - err = eng.Checker.RestoreVerifySnapshot(ctx, snapID1, restoreDir, ssMeta1, os.Stdout) + err = eng.Checker.RestoreVerifySnapshot(ctx, snapID1, restoreDir, ssMeta1, os.Stdout, opts) if err == nil { t.Fatalf("Expected an integrity error when trying to restore a snapshot with incorrect metadata") } } func TestDataPersistency(t *testing.T) { + os.Setenv(snapmeta.EngineModeEnvKey, snapmeta.EngineModeBasic) + os.Setenv(snapmeta.S3BucketNameEnvKey, "") + tempDir, err := ioutil.TempDir("", "") testenv.AssertNoError(t, err) defer os.RemoveAll(tempDir) - eng, err := NewEngine("") + dataRepoPath := filepath.Join(tempDir, "data-repo-") + metadataRepoPath := filepath.Join(tempDir, "metadata-repo-") + + th, eng, err := newTestHarness(t, dataRepoPath, metadataRepoPath) if errors.Is(err, kopiarunner.ErrExeVariableNotSet) || errors.Is(err, fio.ErrEnvNotSet) { t.Skip(err) } @@ -316,15 +340,13 @@ func TestDataPersistency(t *testing.T) { testenv.AssertNoError(t, err) defer func() { - cleanupErr := eng.Cleanup() + cleanupErr := th.Cleanup() testenv.AssertNoError(t, cleanupErr) }() - dataRepoPath := filepath.Join(tempDir, "data-repo-") - metadataRepoPath := filepath.Join(tempDir, "metadata-repo-") - + opts := map[string]string{} ctx := context.TODO() - err = eng.InitFilesystem(ctx, dataRepoPath, metadataRepoPath) + err = eng.Init(ctx) testenv.AssertNoError(t, err) // Perform writes @@ -332,12 +354,12 @@ func TestDataPersistency(t *testing.T) { numFiles := 10 fioOpt := fio.Options{}.WithFileSize(fileSize).WithNumFiles(numFiles) - fioRunner := engineFioRunner(t, eng) + fioRunner := th.FioRunner() err = fioRunner.WriteFiles("", fioOpt) testenv.AssertNoError(t, err) // Take a snapshot - snapID, err := eng.Checker.TakeSnapshot(ctx, fioRunner.LocalDataDir) + snapID, err := eng.Checker.TakeSnapshot(ctx, fioRunner.LocalDataDir, opts) testenv.AssertNoError(t, err) // Get the walk data associated with the snapshot that was taken @@ -349,25 +371,29 @@ func TestDataPersistency(t *testing.T) { testenv.AssertNoError(t, err) // Create a new engine - eng2, err := NewEngine("") + th2, eng2, err := newTestHarness(t, dataRepoPath, metadataRepoPath) testenv.AssertNoError(t, err) - defer eng2.CleanComponents() + defer func() { + th2.eng.cleanComponents() + th2.eng = nil + th2.Cleanup() + }() // Connect this engine to the same data and metadata repositories - // expect that the snapshot taken above will be found in metadata, // and the data will be chosen to be restored to this engine's DataDir // as a starting point. - err = eng2.InitFilesystem(ctx, dataRepoPath, metadataRepoPath) + err = eng2.Init(ctx) testenv.AssertNoError(t, err) - fioRunner2 := engineFioRunner(t, eng2) - err = eng2.Checker.RestoreSnapshotToPath(ctx, snapID, fioRunner2.LocalDataDir, os.Stdout) + fioRunner2 := th2.FioRunner() + err = eng2.Checker.RestoreSnapshotToPath(ctx, snapID, fioRunner2.LocalDataDir, os.Stdout, opts) testenv.AssertNoError(t, err) // Compare the data directory of the second engine with the fingerprint // of the snapshot taken earlier. They should match. - err = fswalker.NewWalkCompare().Compare(ctx, fioRunner2.LocalDataDir, dataDirWalk.ValidationData, os.Stdout) + err = fswalker.NewWalkCompare().Compare(ctx, fioRunner2.LocalDataDir, dataDirWalk.ValidationData, os.Stdout, opts) testenv.AssertNoError(t, err) } @@ -466,7 +492,10 @@ func TestPickActionWeighted(t *testing.T) { } func TestActionsFilesystem(t *testing.T) { - eng, err := NewEngine("") + os.Setenv(snapmeta.EngineModeEnvKey, snapmeta.EngineModeBasic) + os.Setenv(snapmeta.S3BucketNameEnvKey, "") + + th, eng, err := newTestHarness(t, fsDataRepoPath, fsMetadataRepoPath) if errors.Is(err, kopiarunner.ErrExeVariableNotSet) || errors.Is(err, fio.ErrEnvNotSet) { t.Skip(err) } @@ -474,14 +503,14 @@ func TestActionsFilesystem(t *testing.T) { testenv.AssertNoError(t, err) defer func() { - cleanupErr := eng.Cleanup() + cleanupErr := th.Cleanup() testenv.AssertNoError(t, cleanupErr) os.RemoveAll(fsRepoBaseDirPath) }() ctx := context.TODO() - err = eng.InitFilesystem(ctx, fsDataRepoPath, fsMetadataRepoPath) + err = eng.Init(ctx) testenv.AssertNoError(t, err) actionOpts := ActionOpts{ @@ -511,7 +540,10 @@ func TestActionsS3(t *testing.T) { bucketName, cleanupCB := makeTempS3Bucket(t) defer cleanupCB() - eng, err := NewEngine("") + os.Setenv(snapmeta.EngineModeEnvKey, snapmeta.EngineModeBasic) + os.Setenv(snapmeta.S3BucketNameEnvKey, bucketName) + + th, eng, err := newTestHarness(t, s3DataRepoPath, s3MetadataRepoPath) if errors.Is(err, kopiarunner.ErrExeVariableNotSet) || errors.Is(err, fio.ErrEnvNotSet) { t.Skip(err) } @@ -519,12 +551,12 @@ func TestActionsS3(t *testing.T) { testenv.AssertNoError(t, err) defer func() { - cleanupErr := eng.Cleanup() + cleanupErr := th.Cleanup() testenv.AssertNoError(t, cleanupErr) }() ctx := context.TODO() - err = eng.InitS3(ctx, bucketName, s3DataRepoPath, s3MetadataRepoPath) + err = eng.Init(ctx) testenv.AssertNoError(t, err) actionOpts := ActionOpts{ @@ -558,7 +590,10 @@ func TestIOLimitPerWriteAction(t *testing.T) { // set to 1 MB. const timeout = 10 * time.Second - eng, err := NewEngine("") + os.Setenv(snapmeta.EngineModeEnvKey, snapmeta.EngineModeBasic) + os.Setenv(snapmeta.S3BucketNameEnvKey, "") + + th, eng, err := newTestHarness(t, fsDataRepoPath, fsMetadataRepoPath) if errors.Is(err, kopiarunner.ErrExeVariableNotSet) || errors.Is(err, fio.ErrEnvNotSet) { t.Skip(err) } @@ -566,14 +601,14 @@ func TestIOLimitPerWriteAction(t *testing.T) { testenv.AssertNoError(t, err) defer func() { - cleanupErr := eng.Cleanup() + cleanupErr := th.Cleanup() testenv.AssertNoError(t, cleanupErr) os.RemoveAll(fsRepoBaseDirPath) }() ctx := context.TODO() - err = eng.InitFilesystem(ctx, fsDataRepoPath, fsMetadataRepoPath) + err = eng.Init(ctx) testenv.AssertNoError(t, err) actionOpts := ActionOpts{ @@ -613,7 +648,7 @@ func TestStatsPersist(t *testing.T) { defer os.RemoveAll(tmpDir) - snapStore, err := snapmeta.New(tmpDir) + snapStore, err := snapmeta.NewPersister(tmpDir) if errors.Is(err, kopiarunner.ErrExeVariableNotSet) { t.Skip(err) } @@ -644,13 +679,13 @@ func TestStatsPersist(t *testing.T) { }, } - err = eng.SaveStats() + err = eng.saveStats() testenv.AssertNoError(t, err) err = eng.MetaStore.FlushMetadata() testenv.AssertNoError(t, err) - snapStoreNew, err := snapmeta.New(tmpDir) + snapStoreNew, err := snapmeta.NewPersister(tmpDir) testenv.AssertNoError(t, err) // Connect to the same metadata store @@ -664,7 +699,7 @@ func TestStatsPersist(t *testing.T) { MetaStore: snapStoreNew, } - err = engNew.LoadStats() + err = engNew.loadStats() testenv.AssertNoError(t, err) if got, want := engNew.Stats(), eng.Stats(); got != want { @@ -681,7 +716,7 @@ func TestLogsPersist(t *testing.T) { defer os.RemoveAll(tmpDir) - snapStore, err := snapmeta.New(tmpDir) + snapStore, err := snapmeta.NewPersister(tmpDir) if errors.Is(err, kopiarunner.ErrExeVariableNotSet) { t.Skip(err) } @@ -691,7 +726,7 @@ func TestLogsPersist(t *testing.T) { err = snapStore.ConnectOrCreateFilesystem(tmpDir) testenv.AssertNoError(t, err) - log := Log{ + logData := Log{ Log: []*LogEntry{ { StartTime: time.Now().Add(-time.Hour), @@ -711,16 +746,16 @@ func TestLogsPersist(t *testing.T) { eng := &Engine{ MetaStore: snapStore, - EngineLog: log, + EngineLog: logData, } - err = eng.SaveLog() + err = eng.saveLog() testenv.AssertNoError(t, err) err = eng.MetaStore.FlushMetadata() testenv.AssertNoError(t, err) - snapStoreNew, err := snapmeta.New(tmpDir) + snapStoreNew, err := snapmeta.NewPersister(tmpDir) testenv.AssertNoError(t, err) // Connect to the same metadata store @@ -734,7 +769,7 @@ func TestLogsPersist(t *testing.T) { MetaStore: snapStoreNew, } - err = engNew.LoadLog() + err = engNew.loadLog() testenv.AssertNoError(t, err) if got, want := engNew.EngineLog.String(), eng.EngineLog.String(); got != want { @@ -742,14 +777,104 @@ func TestLogsPersist(t *testing.T) { } } -// engineFioRunner extracts the fio.Runner used by the engine. -func engineFioRunner(t *testing.T, eng *Engine) *fio.Runner { +type testHarness struct { + fw *fiofilewriter.FileWriter + ks *snapmeta.KopiaSnapshotter + kp *snapmeta.KopiaPersister + wc *fswalker.WalkCompare + + baseDir string + + eng *Engine +} + +func newTestHarness(t *testing.T, dataRepoPath, metaRepoPath string) (*testHarness, *Engine, error) { t.Helper() - fiofw, ok := eng.FileWriter.(*fiofilewriter.FileWriter) - if !ok { - t.Fatal("engine does not have a fiofilewriter.FileWriter") + var ( + th = &testHarness{} + err error + ) + + if th.baseDir, err = ioutil.TempDir("", "engine-data-"); err != nil { + return nil, nil, err } - return fiofw.Runner + if th.fw, err = fiofilewriter.New(); err != nil { + th.Cleanup() + return nil, nil, err + } + + if th.ks, err = snapmeta.NewSnapshotter(th.baseDir); err != nil { + th.Cleanup() + return nil, nil, err + } + + if err = th.ks.ConnectOrCreateRepo(dataRepoPath); err != nil { + th.Cleanup() + return nil, nil, err + } + + if th.kp, err = snapmeta.NewPersister(th.baseDir); err != nil { + th.Cleanup() + return nil, nil, err + } + + if err = th.kp.ConnectOrCreateRepo(metaRepoPath); err != nil { + th.Cleanup() + return nil, nil, err + } + + th.wc = fswalker.NewWalkCompare() + + if th.eng, err = New(th.args()); err != nil { + th.Cleanup() + return nil, nil, err + } + + return th, th.eng, err +} + +func (th *testHarness) args() *Args { + return &Args{ + MetaStore: th.kp, + TestRepo: th.ks, + Validator: th.wc, + FileWriter: th.fw, + WorkingDir: th.baseDir, + } +} + +func (th *testHarness) FioRunner() *fio.Runner { + return th.fw.Runner +} + +func (th *testHarness) Cleanup() error { + var err error + + if th.eng != nil { + err = th.eng.Shutdown() + } + + if th.fw != nil { + th.fw.Cleanup() + } + + if th.ks != nil { + if sc := th.ks.ServerCmd(); sc != nil { + if err = sc.Process.Signal(syscall.SIGTERM); err != nil { + log.Println("Failed to send termination signal to kopia server process:", err) + } + } + + th.ks.Cleanup() + } + + if th.kp != nil { + th.kp.Cleanup() + } + + os.RemoveAll(th.baseDir) + + return err } diff --git a/tests/robustness/engine/metadata.go b/tests/robustness/engine/metadata.go index 20e9cdb0b..5ed261317 100644 --- a/tests/robustness/engine/metadata.go +++ b/tests/robustness/engine/metadata.go @@ -15,8 +15,8 @@ const ( engineLogsStoreKey = "engine-logs" ) -// SaveLog saves the engine Log in the metadata store. -func (e *Engine) SaveLog() error { +// saveLog saves the engine Log in the metadata store. +func (e *Engine) saveLog() error { b, err := json.Marshal(e.EngineLog) if err != nil { return err @@ -25,8 +25,8 @@ func (e *Engine) SaveLog() error { return e.MetaStore.Store(engineLogsStoreKey, b) } -// LoadLog loads the engine log from the metadata store. -func (e *Engine) LoadLog() error { +// loadLog loads the engine log from the metadata store. +func (e *Engine) loadLog() error { b, err := e.MetaStore.Load(engineLogsStoreKey) if err != nil { if errors.Is(err, snapmeta.ErrKeyNotFound) { @@ -47,8 +47,8 @@ func (e *Engine) LoadLog() error { return err } -// SaveStats saves the engine Stats in the metadata store. -func (e *Engine) SaveStats() error { +// saveStats saves the engine Stats in the metadata store. +func (e *Engine) saveStats() error { cumulStatRaw, err := json.Marshal(e.CumulativeStats) if err != nil { return err @@ -57,8 +57,8 @@ func (e *Engine) SaveStats() error { return e.MetaStore.Store(engineStatsStoreKey, cumulStatRaw) } -// LoadStats loads the engine Stats from the metadata store. -func (e *Engine) LoadStats() error { +// loadStats loads the engine Stats from the metadata store. +func (e *Engine) loadStats() error { b, err := e.MetaStore.Load(engineStatsStoreKey) if err != nil { if errors.Is(err, snapmeta.ErrKeyNotFound) { diff --git a/tests/robustness/filewriter.go b/tests/robustness/filewriter.go index 3c3c90c85..d272ce4d3 100644 --- a/tests/robustness/filewriter.go +++ b/tests/robustness/filewriter.go @@ -2,10 +2,6 @@ package robustness // FileWriter is an interface used for filesystem related actions. type FileWriter interface { - // Cleanup is called prior to termination. - // TBD: Will be removed when initialization refactored. - Cleanup() - // DataDirectory returns the absolute path of the data directory configured. DataDirectory() string diff --git a/tests/robustness/fiofilewriter/fio_filewriter.go b/tests/robustness/fiofilewriter/fio_filewriter.go index ed6a7c80f..d0cc25fe4 100644 --- a/tests/robustness/fiofilewriter/fio_filewriter.go +++ b/tests/robustness/fiofilewriter/fio_filewriter.go @@ -46,7 +46,7 @@ const ( // New returns a FileWriter based on FIO. // See tests/tools/fio for configuration details. -func New() (robustness.FileWriter, error) { +func New() (*FileWriter, error) { runner, err := fio.NewRunner() if err != nil { return nil, err @@ -60,6 +60,8 @@ type FileWriter struct { Runner *fio.Runner } +var _ robustness.FileWriter = (*FileWriter)(nil) + // DataDirectory returns the data directory configured. // See tests/tools/fio for details. func (fw *FileWriter) DataDirectory() string { diff --git a/tests/robustness/persister.go b/tests/robustness/persister.go index 0a02f03d3..649fcaeac 100644 --- a/tests/robustness/persister.go +++ b/tests/robustness/persister.go @@ -20,9 +20,7 @@ type Indexer interface { // to, and load it again, from a repository. type Persister interface { Store - RepoManager // TBD: may not be needed once initialization refactored LoadMetadata() error FlushMetadata() error GetPersistDir() string - Cleanup() } diff --git a/tests/robustness/robustness_test/main_test.go b/tests/robustness/robustness_test/main_test.go index cc53a4e00..80f2b0fad 100644 --- a/tests/robustness/robustness_test/main_test.go +++ b/tests/robustness/robustness_test/main_test.go @@ -6,19 +6,24 @@ import ( "context" "errors" "flag" + "io/ioutil" "log" "os" "path" + "syscall" "testing" "time" "github.com/kopia/kopia/tests/robustness" "github.com/kopia/kopia/tests/robustness/engine" + "github.com/kopia/kopia/tests/robustness/fiofilewriter" + "github.com/kopia/kopia/tests/robustness/snapmeta" "github.com/kopia/kopia/tests/tools/fio" + "github.com/kopia/kopia/tests/tools/fswalker" "github.com/kopia/kopia/tests/tools/kopiarunner" ) -var eng *engine.Engine +var eng *engine.Engine // for use in the test functions const ( dataSubPath = "robustness-data" @@ -34,50 +39,205 @@ var ( func TestMain(m *testing.M) { flag.Parse() - var err error - - eng, err = engine.NewEngine("") - - switch { - case errors.Is(err, kopiarunner.ErrExeVariableNotSet): - log.Println("Skipping robustness tests because KOPIA_EXE is not set") - os.Exit(0) - case errors.Is(err, fio.ErrEnvNotSet): - log.Println("Skipping robustness tests because FIO environment is not set") - os.Exit(0) - case err != nil: - log.Fatalln("error on engine creation:", err) - } - dataRepoPath := path.Join(*repoPathPrefix, dataSubPath) metadataRepoPath := path.Join(*repoPathPrefix, metadataSubPath) - // Try to reconcile metadata if it is out of sync with the repo state - eng.Checker.RecoveryMode = true - - // Initialize the engine, connecting it to the repositories - err = eng.Init(context.Background(), dataRepoPath, metadataRepoPath) - if err != nil { - // Clean the temporary dirs from the file system, don't write out the - // metadata, in case there was an issue loading it - eng.CleanComponents() - log.Fatalln("error initializing engine for S3:", err) - } + th := &kopiaRobustnessTestHarness{} + th.init(dataRepoPath, metadataRepoPath) + eng = th.engine // Restore a random snapshot into the data directory - _, err = eng.ExecAction(engine.RestoreIntoDataDirectoryActionKey, nil) + _, err := eng.ExecAction(engine.RestoreIntoDataDirectoryActionKey, nil) if err != nil && !errors.Is(err, robustness.ErrNoOp) { - eng.Cleanup() - log.Fatalln("error restoring into the data directory:", err) + th.cleanup() + log.Fatalln("Error restoring into the data directory:", err) } + // run the tests result := m.Run() - err = eng.Cleanup() + err = th.cleanup() if err != nil { - log.Printf("error cleaning up the engine: %s\n", err.Error()) + log.Printf("Error cleaning up the engine: %s\n", err.Error()) os.Exit(2) } os.Exit(result) } + +type kopiaRobustnessTestHarness struct { + dataRepoPath string + metaRepoPath string + + baseDirPath string + fileWriter *fiofilewriter.FileWriter + snapshotter *snapmeta.KopiaSnapshotter + persister *snapmeta.KopiaPersister + comparer *fswalker.WalkCompare + engine *engine.Engine + + skipTest bool +} + +func (th *kopiaRobustnessTestHarness) init(dataRepoPath, metaRepoPath string) { + th.dataRepoPath = dataRepoPath + th.metaRepoPath = metaRepoPath + + // the initialization state machine is linear and bails out on first failure + if th.makeBaseDir() && th.getFileWriter() && th.getSnapshotter() && + th.getPersister() && th.getComparer() && th.getEngine() { + return // success! + } + + th.cleanup() + + if th.skipTest { + os.Exit(0) + } + + os.Exit(1) +} + +func (th *kopiaRobustnessTestHarness) makeBaseDir() bool { + baseDir, err := ioutil.TempDir("", "engine-data-") + if err != nil { + log.Println("Error creating temp dir:", err) + return false + } + + th.baseDirPath = baseDir + + return true +} + +func (th *kopiaRobustnessTestHarness) getFileWriter() bool { + fw, err := fiofilewriter.New() + if err != nil { + if errors.Is(err, fio.ErrEnvNotSet) { + log.Println("Skipping robustness tests because FIO environment is not set") + + th.skipTest = true + } else { + log.Println("Error creating fio FileWriter:", err) + } + + return false + } + + th.fileWriter = fw + + return true +} + +func (th *kopiaRobustnessTestHarness) getSnapshotter() bool { + ks, err := snapmeta.NewSnapshotter(th.baseDirPath) + if err != nil { + if errors.Is(err, kopiarunner.ErrExeVariableNotSet) { + log.Println("Skipping robustness tests because KOPIA_EXE is not set") + + th.skipTest = true + } else { + log.Println("Error creating kopia Snapshotter:", err) + } + + return false + } + + th.snapshotter = ks + + if err = ks.ConnectOrCreateRepo(th.dataRepoPath); err != nil { + log.Println("Error initializing kopia Snapshotter:", err) + return false + } + + return true +} + +func (th *kopiaRobustnessTestHarness) getPersister() bool { + kp, err := snapmeta.NewPersister(th.baseDirPath) + if err != nil { + if errors.Is(err, kopiarunner.ErrExeVariableNotSet) { + log.Println("Skipping robustness tests because KOPIA_EXE is not set") + + th.skipTest = true + } else { + log.Println("Error creating kopia Persister:", err) + } + + return false + } + + th.persister = kp + + if err = kp.ConnectOrCreateRepo(th.metaRepoPath); err != nil { + log.Println("Error initializing kopia Persister:", err) + return false + } + + return true +} + +func (th *kopiaRobustnessTestHarness) getComparer() bool { + th.comparer = fswalker.NewWalkCompare() + return true +} + +func (th *kopiaRobustnessTestHarness) getEngine() bool { + args := &engine.Args{ + MetaStore: th.persister, + TestRepo: th.snapshotter, + Validator: th.comparer, + FileWriter: th.fileWriter, + WorkingDir: th.baseDirPath, + SyncRepositories: true, + } + + eng, err := engine.New(args) // nolint:govet + if err != nil { + log.Println("Error on engine creation:", err) + return false + } + + // Initialize the engine, connecting it to the repositories. + // Note that th.engine is not yet set so that metadata will not be + // flushed on cleanup in case there was an issue while loading. + err = eng.Init(context.Background()) + if err != nil { + log.Println("Error initializing engine for S3:", err) + return false + } + + th.engine = eng + + return true +} + +func (th *kopiaRobustnessTestHarness) cleanup() (retErr error) { + if th.engine != nil { + retErr = th.engine.Shutdown() + } + + if th.persister != nil { + th.persister.Cleanup() + } + + if th.snapshotter != nil { + if sc := th.snapshotter.ServerCmd(); sc != nil { + if err := sc.Process.Signal(syscall.SIGTERM); err != nil { + log.Println("Warning: Failed to send termination signal to kopia server process:", err) + } + } + + th.snapshotter.Cleanup() + } + + if th.fileWriter != nil { + th.fileWriter.Cleanup() + } + + if th.baseDirPath != "" { + os.RemoveAll(th.baseDirPath) + } + + return +} diff --git a/tests/robustness/snapmeta/kopia_connector.go b/tests/robustness/snapmeta/kopia_connector.go new file mode 100644 index 000000000..de39c1df7 --- /dev/null +++ b/tests/robustness/snapmeta/kopia_connector.go @@ -0,0 +1,104 @@ +package snapmeta + +import ( + "os" + "os/exec" + + "github.com/kopia/kopia/tests/tools/kopiarunner" +) + +const ( + // S3BucketNameEnvKey is the environment variable required to connect to a repo on S3. + S3BucketNameEnvKey = "S3_BUCKET_NAME" + // EngineModeEnvKey is the environment variable required to switch between basic and server/client model. + EngineModeEnvKey = "ENGINE_MODE" + // EngineModeBasic is a constant used to check the engineMode. + EngineModeBasic = "BASIC" + // EngineModeServer is a constant used to check the engineMode. + EngineModeServer = "SERVER" + // defaultAddr is used for setting the address of Kopia Server. + defaultAddr = "localhost:51515" +) + +// kopiaConnector is a base type for Persister and Snapshotter. +// It provides a kopiarunner.KopiaSnapshotter and common initialization +// behavior based on the values of the EngineModeEnvKey and +// S3BucketNameEnvKey environment variables. +// +// Derived types can customize the initialization behavior by overriding +// the default handler functions. +type kopiaConnector struct { + // properties set by initializeConnector() + snap *kopiarunner.KopiaSnapshotter + initS3Fn func(repoPath, bucketName string) error + initS3WithServerFn func(repoPath, bucketName, addr string) error + initFilesystemFn func(repoPath string) error + initFilesystemWithServerFn func(repoPath, addr string) error + + // properties that may be set by connectOrCreateRepo() + serverCmd *exec.Cmd +} + +// initializeConnector initializes the connector object and enables use of the +// connectOrCreateRepo method. +func (ki *kopiaConnector) initializeConnector(baseDirPath string) error { + snap, err := kopiarunner.NewKopiaSnapshotter(baseDirPath) + if err != nil { + return err + } + + ki.snap = snap + ki.initS3Fn = ki.initS3 + ki.initFilesystemFn = ki.initFilesystem + ki.initS3WithServerFn = ki.initS3WithServer + ki.initFilesystemWithServerFn = ki.initFilesystemWithServer + + return nil +} + +// connectOrCreateRepo makes the connector ready for use. +// It invokes the appropriate initialization routine based on the environment variables set. +func (ki *kopiaConnector) connectOrCreateRepo(repoPath string) error { + bucketName := os.Getenv(S3BucketNameEnvKey) + engineMode := os.Getenv(EngineModeEnvKey) + + switch { + case bucketName != "" && engineMode == EngineModeBasic: + return ki.initS3Fn(repoPath, bucketName) + + case bucketName != "" && engineMode == EngineModeServer: + return ki.initS3WithServerFn(repoPath, bucketName, defaultAddr) + + case bucketName == "" && engineMode == EngineModeServer: + return ki.initFilesystemWithServerFn(repoPath, defaultAddr) + + default: + return ki.initFilesystemFn(repoPath) + } +} + +// initS3 initializes basic mode with an S3 repository. +func (ki *kopiaConnector) initS3(repoPath, bucketName string) error { + return ki.snap.ConnectOrCreateS3(bucketName, repoPath) +} + +// initFilesystem initializes basic mode with a filesystem repository. +func (ki *kopiaConnector) initFilesystem(repoPath string) error { + return ki.snap.ConnectOrCreateFilesystem(repoPath) +} + +// initS3WithServerFn initializes server mode with an S3 repository. +func (ki *kopiaConnector) initS3WithServer(repoPath, bucketName, addr string) error { + cmd, err := ki.snap.ConnectOrCreateS3WithServer(addr, bucketName, repoPath) + ki.serverCmd = cmd + + return err +} + +// initFilesystemWithServer initializes server mode with a filesystem repository. +func (ki *kopiaConnector) initFilesystemWithServer(repoPath, addr string) error { + cmd, err := ki.snap.ConnectOrCreateFilesystemWithServer(addr, repoPath) + ki.serverCmd = cmd + + return err +} diff --git a/tests/robustness/snapmeta/kopia_connector_test.go b/tests/robustness/snapmeta/kopia_connector_test.go new file mode 100644 index 000000000..4364eb6db --- /dev/null +++ b/tests/robustness/snapmeta/kopia_connector_test.go @@ -0,0 +1,117 @@ +package snapmeta + +import ( + "os" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestKopiaConnector(t *testing.T) { + assert := assert.New(t) // nolint:gocritic + + os.Setenv("KOPIA_EXE", "kopia.exe") + + tc := &testConnector{} + + err := tc.initializeConnector("") + assert.NoError(err) + assert.NotNil(tc.snap) + assert.NotNil(tc.initS3Fn) + assert.NotNil(tc.initS3WithServerFn) + assert.NotNil(tc.initFilesystemFn) + assert.NotNil(tc.initFilesystemWithServerFn) + + tc.initS3Fn = tc.testInitS3 + tc.initFilesystemFn = tc.testInitFilesystem + tc.initS3WithServerFn = tc.testInitS3WithServer + tc.initFilesystemWithServerFn = tc.testInitFilesystemWithServer + + repoPath := "repoPath" + bucketName := "bucketName" + + os.Setenv(EngineModeEnvKey, EngineModeBasic) + os.Setenv(S3BucketNameEnvKey, "") + tc.reset() + assert.NoError(tc.connectOrCreateRepo(repoPath)) + assert.True(tc.initFilesystemCalled) + assert.Equal(repoPath, tc.tcRepoPath) + + os.Setenv(EngineModeEnvKey, EngineModeBasic) + os.Setenv(S3BucketNameEnvKey, bucketName) + tc.reset() + assert.NoError(tc.connectOrCreateRepo(repoPath)) + assert.True(tc.initS3Called) + assert.Equal(repoPath, tc.tcRepoPath) + assert.Equal(bucketName, tc.tcBucketName) + + os.Setenv(EngineModeEnvKey, EngineModeServer) + os.Setenv(S3BucketNameEnvKey, "") + tc.reset() + assert.NoError(tc.connectOrCreateRepo(repoPath)) + assert.True(tc.initFilesystemWithServerCalled) + assert.Equal(repoPath, tc.tcRepoPath) + assert.Equal(defaultAddr, tc.tcAddr) + + os.Setenv(EngineModeEnvKey, EngineModeServer) + os.Setenv(S3BucketNameEnvKey, bucketName) + tc.reset() + assert.NoError(tc.connectOrCreateRepo(repoPath)) + assert.True(tc.initS3WithServerCalled) + assert.Equal(repoPath, tc.tcRepoPath) + assert.Equal(bucketName, tc.tcBucketName) + assert.Equal(defaultAddr, tc.tcAddr) +} + +type testConnector struct { + kopiaConnector + tcRepoPath string + tcBucketName string + tcAddr string + initS3Called bool + initFilesystemCalled bool + initS3WithServerCalled bool + initFilesystemWithServerCalled bool +} + +func (tc *testConnector) reset() { + tc.tcRepoPath = "" + tc.tcBucketName = "" + tc.tcAddr = "" + tc.initS3Called = false + tc.initFilesystemCalled = false + tc.initS3WithServerCalled = false + tc.initFilesystemWithServerCalled = false +} + +func (tc *testConnector) testInitS3(repoPath, bucketName string) error { + tc.tcRepoPath = repoPath + tc.tcBucketName = bucketName + tc.initS3Called = true + + return nil +} + +func (tc *testConnector) testInitFilesystem(repoPath string) error { + tc.tcRepoPath = repoPath + tc.initFilesystemCalled = true + + return nil +} + +func (tc *testConnector) testInitS3WithServer(repoPath, bucketName, addr string) error { + tc.tcRepoPath = repoPath + tc.tcBucketName = bucketName + tc.tcAddr = addr + tc.initS3WithServerCalled = true + + return nil +} + +func (tc *testConnector) testInitFilesystemWithServer(repoPath, addr string) error { + tc.tcRepoPath = repoPath + tc.tcAddr = addr + tc.initFilesystemWithServerCalled = true + + return nil +} diff --git a/tests/robustness/snapmeta/kopia_meta.go b/tests/robustness/snapmeta/kopia_persister.go similarity index 63% rename from tests/robustness/snapmeta/kopia_meta.go rename to tests/robustness/snapmeta/kopia_persister.go index 83caf74a8..88239a060 100644 --- a/tests/robustness/snapmeta/kopia_meta.go +++ b/tests/robustness/snapmeta/kopia_persister.go @@ -1,5 +1,4 @@ -// Package snapmeta describes entities that can accept -// arbitrary metadata and flush it to a persistent repository. +// Package snapmeta provides Kopia implementations of Persister and Snapshotter. package snapmeta import ( @@ -10,28 +9,22 @@ import ( "path/filepath" "github.com/kopia/kopia/tests/robustness" - "github.com/kopia/kopia/tests/tools/kopiarunner" ) -var _ robustness.Persister = &kopiaMetadata{} - -// kopiaMetadata handles metadata persistency of a snapshot store, using a Kopia -// repository as the persistency mechanism. -type kopiaMetadata struct { +// KopiaPersister implements robustness.Persister. +type KopiaPersister struct { *Simple localMetadataDir string persistenceDir string - snap *kopiarunner.KopiaSnapshotter + kopiaConnector } -// New instantiates a new Persister and returns it. -func New(baseDir string) (robustness.Persister, error) { - localDir, err := ioutil.TempDir(baseDir, "kopia-local-metadata-") - if err != nil { - return nil, err - } +var _ robustness.Persister = (*KopiaPersister)(nil) - snap, err := kopiarunner.NewKopiaSnapshotter(localDir) +// NewPersister returns a Kopia based Persister. +// ConnectOrCreateRepo must be invoked to enable the interface. +func NewPersister(baseDir string) (*KopiaPersister, error) { + localDir, err := ioutil.TempDir(baseDir, "kopia-local-metadata-") if err != nil { return nil, err } @@ -41,16 +34,41 @@ func New(baseDir string) (robustness.Persister, error) { return nil, err } - return &kopiaMetadata{ + km := &KopiaPersister{ localMetadataDir: localDir, persistenceDir: persistenceDir, Simple: NewSimple(), - snap: snap, - }, nil + } + + if err := km.initializeConnector(localDir); err != nil { + return nil, err + } + + km.initS3WithServerFn = km.persisterInitS3WithServer + km.initFilesystemWithServerFn = km.persisterInitFilesystemWithServer + + return km, nil +} + +// persisterInitS3WithServer is an adaptor for initS3() as the persister +// does not support the server configuration. +func (store *KopiaPersister) persisterInitS3WithServer(repoPath, bucketName, addr string) error { + return store.initS3(repoPath, bucketName) +} + +// persisterInitFilesystemWithServer is an adaptor for initFilesystem() as the persister +// does not support the server configuration. +func (store *KopiaPersister) persisterInitFilesystemWithServer(repoPath, addr string) error { + return store.initFilesystem(repoPath) +} + +// ConnectOrCreateRepo makes the Persister ready for use. +func (store *KopiaPersister) ConnectOrCreateRepo(repoPath string) error { + return store.connectOrCreateRepo(repoPath) } // Cleanup cleans up the local temporary files used by a KopiaMetadata. -func (store *kopiaMetadata) Cleanup() { +func (store *KopiaPersister) Cleanup() { if store.localMetadataDir != "" { os.RemoveAll(store.localMetadataDir) //nolint:errcheck } @@ -62,13 +80,13 @@ func (store *kopiaMetadata) Cleanup() { // ConnectOrCreateS3 implements the RepoManager interface, connects to a repo in an S3 // bucket or attempts to create one if connection is unsuccessful. -func (store *kopiaMetadata) ConnectOrCreateS3(bucketName, pathPrefix string) error { +func (store *KopiaPersister) ConnectOrCreateS3(bucketName, pathPrefix string) error { return store.snap.ConnectOrCreateS3(bucketName, pathPrefix) } // ConnectOrCreateFilesystem implements the RepoManager interface, connects to a repo in the filesystem // or attempts to create one if connection is unsuccessful. -func (store *kopiaMetadata) ConnectOrCreateFilesystem(path string) error { +func (store *KopiaPersister) ConnectOrCreateFilesystem(path string) error { return store.snap.ConnectOrCreateFilesystem(path) } @@ -76,20 +94,20 @@ const metadataStoreFileName = "metadata-store-latest" // ConnectOrCreateS3WithServer implements the RepoManager interface, creates a server // connects it a repo in an S3 bucket and creates a client to perform operations. -func (store *kopiaMetadata) ConnectOrCreateS3WithServer(serverAddr, bucketName, pathPrefix string) (*exec.Cmd, error) { +func (store *KopiaPersister) ConnectOrCreateS3WithServer(serverAddr, bucketName, pathPrefix string) (*exec.Cmd, error) { return store.snap.ConnectOrCreateS3WithServer(serverAddr, bucketName, pathPrefix) } // ConnectOrCreateFilesystemWithServer implements the RepoManager interface, creates a server // connects it a repo in the filesystem and creates a client to perform operations. -func (store *kopiaMetadata) ConnectOrCreateFilesystemWithServer(repoPath, serverAddr string) (*exec.Cmd, error) { +func (store *KopiaPersister) ConnectOrCreateFilesystemWithServer(repoPath, serverAddr string) (*exec.Cmd, error) { return store.snap.ConnectOrCreateFilesystemWithServer(repoPath, serverAddr) } // LoadMetadata implements the DataPersister interface, restores the latest // snapshot from the kopia repository and decodes its contents, populating // its metadata on the snapshots residing in the target test repository. -func (store *kopiaMetadata) LoadMetadata() error { +func (store *KopiaPersister) LoadMetadata() error { snapIDs, err := store.snap.ListSnapshots() if err != nil { return err @@ -125,14 +143,14 @@ func (store *kopiaMetadata) LoadMetadata() error { // GetPersistDir returns the path to the directory that will be persisted // as a snapshot to the kopia repository. -func (store *kopiaMetadata) GetPersistDir() string { +func (store *KopiaPersister) GetPersistDir() string { return store.persistenceDir } // FlushMetadata implements the DataPersister interface, flushing the local // metadata on the target test repo's snapshots to the metadata Kopia repository // as a snapshot create. -func (store *kopiaMetadata) FlushMetadata() error { +func (store *KopiaPersister) FlushMetadata() error { metadataPath := filepath.Join(store.persistenceDir, metadataStoreFileName) f, err := os.Create(metadataPath) diff --git a/tests/robustness/snapmeta/kopia_snapshotter.go b/tests/robustness/snapmeta/kopia_snapshotter.go new file mode 100644 index 000000000..be4607e95 --- /dev/null +++ b/tests/robustness/snapmeta/kopia_snapshotter.go @@ -0,0 +1,98 @@ +package snapmeta + +import ( + "os/exec" + "strconv" + + "github.com/kopia/kopia/tests/robustness" +) + +// KopiaSnapshotter implements robustness.Snapshotter. +type KopiaSnapshotter struct { + kopiaConnector +} + +var _ robustness.Snapshotter = (*KopiaSnapshotter)(nil) + +// NewSnapshotter returns a Kopia based Snapshotter. +// ConnectOrCreateRepo must be invoked to enable the interface. +func NewSnapshotter(baseDirPath string) (*KopiaSnapshotter, error) { + ks := &KopiaSnapshotter{} + + if err := ks.initializeConnector(baseDirPath); err != nil { + return nil, err + } + + return ks, nil +} + +// ConnectOrCreateRepo makes the Snapshotter ready for use. +func (ks *KopiaSnapshotter) ConnectOrCreateRepo(repoPath string) error { + if err := ks.connectOrCreateRepo(repoPath); err != nil { + return err + } + + _, _, err := ks.snap.Run("policy", "set", "--global", "--keep-latest", strconv.Itoa(1<<31-1), "--compression", "s2-default") + + return err +} + +// ServerCmd returns the server command. +func (ks *KopiaSnapshotter) ServerCmd() *exec.Cmd { + return ks.serverCmd +} + +// CreateSnapshot is part of Snapshotter. +func (ks *KopiaSnapshotter) CreateSnapshot(sourceDir string, opts map[string]string) (snapID string, err error) { + return ks.snap.CreateSnapshot(sourceDir) +} + +// RestoreSnapshot is part of Snapshotter. +func (ks *KopiaSnapshotter) RestoreSnapshot(snapID, restoreDir string, opts map[string]string) error { + return ks.snap.RestoreSnapshot(snapID, restoreDir) +} + +// DeleteSnapshot is part of Snapshotter. +func (ks *KopiaSnapshotter) DeleteSnapshot(snapID string, opts map[string]string) error { + return ks.snap.DeleteSnapshot(snapID) +} + +// RunGC is part of Snapshotter. +func (ks *KopiaSnapshotter) RunGC(opts map[string]string) error { + return ks.snap.RunGC() +} + +// ListSnapshots is part of Snapshotter. +func (ks *KopiaSnapshotter) ListSnapshots() ([]string, error) { + return ks.snap.ListSnapshots() +} + +// Run is part of Snapshotter. +func (ks *KopiaSnapshotter) Run(args ...string) (stdout, stderr string, err error) { + return ks.snap.Run(args...) +} + +// ConnectOrCreateS3 TBD: remove this. +func (ks *KopiaSnapshotter) ConnectOrCreateS3(bucketName, pathPrefix string) error { + return nil +} + +// ConnectOrCreateFilesystem TBD: remove this. +func (ks *KopiaSnapshotter) ConnectOrCreateFilesystem(path string) error { + return nil +} + +// ConnectOrCreateS3WithServer TBD: remove this. +func (ks *KopiaSnapshotter) ConnectOrCreateS3WithServer(serverAddr, bucketName, pathPrefix string) (*exec.Cmd, error) { + return nil, nil +} + +// ConnectOrCreateFilesystemWithServer TBD: remove this. +func (ks *KopiaSnapshotter) ConnectOrCreateFilesystemWithServer(serverAddr, repoPath string) (*exec.Cmd, error) { + return nil, nil +} + +// Cleanup should be called before termination. +func (ks *KopiaSnapshotter) Cleanup() { + ks.snap.Cleanup() +} diff --git a/tests/robustness/snapshotter.go b/tests/robustness/snapshotter.go index a7fdda01a..37819aed6 100644 --- a/tests/robustness/snapshotter.go +++ b/tests/robustness/snapshotter.go @@ -3,27 +3,13 @@ // can be used to test other environments. package robustness -import "os/exec" - // Snapshotter is an interface that describes methods // for taking, restoring, deleting snapshots, and // tracking them by a string snapshot ID. type Snapshotter interface { - RepoManager // TBD: may not be needed once initialization refactored - CreateSnapshot(sourceDir string) (snapID string, err error) - RestoreSnapshot(snapID string, restoreDir string) error - DeleteSnapshot(snapID string) error - RunGC() error + CreateSnapshot(sourceDir string, opts map[string]string) (snapID string, err error) + RestoreSnapshot(snapID string, restoreDir string, opts map[string]string) error + DeleteSnapshot(snapID string, opts map[string]string) error + RunGC(opts map[string]string) error ListSnapshots() ([]string, error) - Run(args ...string) (stdout, stderr string, err error) // TBD: remove once initialization refactored -} - -// RepoManager is an interface that describes connecting to -// a repository. -// TBD: may not be needed once initialization refactored. -type RepoManager interface { - ConnectOrCreateS3(bucketName, pathPrefix string) error - ConnectOrCreateFilesystem(path string) error - ConnectOrCreateS3WithServer(serverAddr, bucketName, pathPrefix string) (*exec.Cmd, error) - ConnectOrCreateFilesystemWithServer(serverAddr, repoPath string) (*exec.Cmd, error) } diff --git a/tests/tools/fswalker/fswalker.go b/tests/tools/fswalker/fswalker.go index 769d46d47..a774f9fae 100644 --- a/tests/tools/fswalker/fswalker.go +++ b/tests/tools/fswalker/fswalker.go @@ -46,7 +46,7 @@ func NewWalkCompare() *WalkCompare { // Gather meets the checker.Comparer interface. It performs a fswalker Walk // and returns the resulting Walk as a protobuf Marshaled buffer. -func (chk *WalkCompare) Gather(ctx context.Context, path string) ([]byte, error) { +func (chk *WalkCompare) Gather(ctx context.Context, path string, opts map[string]string) ([]byte, error) { walkData, err := performWalk(ctx, path) if err != nil { return nil, errors.Wrap(err, "walk with hashing error during gather phase") @@ -91,7 +91,7 @@ func clearHostname(walk *fspb.Walk) { // and generates a fswalker report comparing the two Walks. If there are any differences // an error is returned, and the full report will be written to the provided writer // as JSON. -func (chk *WalkCompare) Compare(ctx context.Context, path string, data []byte, reportOut io.Writer) error { +func (chk *WalkCompare) Compare(ctx context.Context, path string, data []byte, reportOut io.Writer, opts map[string]string) error { beforeWalk := &fspb.Walk{} err := proto.Unmarshal(data, beforeWalk) diff --git a/tests/tools/fswalker/fswalker_test.go b/tests/tools/fswalker/fswalker_test.go index 34dc7c59b..4b520b39a 100644 --- a/tests/tools/fswalker/fswalker_test.go +++ b/tests/tools/fswalker/fswalker_test.go @@ -213,7 +213,7 @@ func TestWalkChecker_GatherCompare(t *testing.T) { ctx := context.TODO() - walk, err := chk.Gather(ctx, tmpDir) + walk, err := chk.Gather(ctx, tmpDir, nil) if err != nil { t.Fatal(err) } @@ -221,7 +221,7 @@ func TestWalkChecker_GatherCompare(t *testing.T) { tt.fileTreeModifier(tmpDir) reportOut := &bytes.Buffer{} - if err := chk.Compare(ctx, tmpDir, walk, reportOut); (err != nil) != tt.wantErr { + if err := chk.Compare(ctx, tmpDir, walk, reportOut, nil); (err != nil) != tt.wantErr { t.Errorf("Compare error = %v, wantErr %v", err, tt.wantErr) return } diff --git a/tests/tools/kopiarunner/kopia_snapshotter.go b/tests/tools/kopiarunner/kopia_snapshotter.go index 31e11ee2a..0b97ce97d 100644 --- a/tests/tools/kopiarunner/kopia_snapshotter.go +++ b/tests/tools/kopiarunner/kopia_snapshotter.go @@ -19,11 +19,8 @@ import ( "github.com/pkg/errors" "github.com/kopia/kopia/internal/retry" - "github.com/kopia/kopia/tests/robustness" ) -var _ robustness.Snapshotter = &KopiaSnapshotter{} - const ( contentCacheSizeMBFlag = "--content-cache-size-mb" metadataCacheSizeMBFlag = "--metadata-cache-size-mb"