From 6fd4409bb320724a3c3dc78afa8dfd461ab1d68d Mon Sep 17 00:00:00 2001 From: carlbraganza Date: Mon, 25 Jan 2021 19:57:27 -0800 Subject: [PATCH] Introduced a FileWriter interface in the robustness test and refactored accordingly. (#797) --- tests/robustness/engine/action.go | 193 ++----------- tests/robustness/engine/engine.go | 13 +- tests/robustness/engine/engine_test.go | 104 ++++--- tests/robustness/engine/sys.go | 19 -- tests/robustness/errors.go | 21 ++ tests/robustness/filewriter.go | 31 +++ .../fiofilewriter/fio_filewriter.go | 255 ++++++++++++++++++ tests/robustness/options.go | 22 ++ tests/robustness/robustness_test/main_test.go | 3 +- .../robustness_test/robustness_test.go | 44 +-- 10 files changed, 442 insertions(+), 263 deletions(-) delete mode 100644 tests/robustness/engine/sys.go create mode 100644 tests/robustness/errors.go create mode 100644 tests/robustness/filewriter.go create mode 100644 tests/robustness/fiofilewriter/fio_filewriter.go create mode 100644 tests/robustness/options.go diff --git a/tests/robustness/engine/action.go b/tests/robustness/engine/action.go index 821dffba9..7dfbc4170 100644 --- a/tests/robustness/engine/action.go +++ b/tests/robustness/engine/action.go @@ -12,13 +12,7 @@ "strings" "time" - "github.com/kopia/kopia/tests/tools/fio" -) - -// Errors associated with action-picking. -var ( - ErrNoActionPicked = errors.New("unable to pick an action with the action control options provided") - ErrInvalidOption = errors.New("invalid option setting") + "github.com/kopia/kopia/tests/robustness" ) // ExecAction executes the action denoted by the provided ActionKey. @@ -42,12 +36,12 @@ func (e *Engine) ExecAction(actionKey ActionKey, opts map[string]string) (map[st } // Execute the action n times - err := ErrNoOp // Default to no-op error + err := robustness.ErrNoOp // Default to no-op error // TODO: return more than the last output var out map[string]string - n := getOptAsIntOrDefault(ActionRepeaterField, opts, defaultActionRepeats) + n := robustness.GetOptAsIntOrDefault(ActionRepeaterField, opts, defaultActionRepeats) for i := 0; i < n; i++ { out, err = action.f(e, opts, logEntry) if err != nil { @@ -57,7 +51,7 @@ func (e *Engine) ExecAction(actionKey ActionKey, opts map[string]string) (map[st // If error was just a no-op, don't bother logging the action switch { - case errors.Is(err, ErrNoOp): + case errors.Is(err, robustness.ErrNoOp): e.RunStats.NoOpCount++ e.CumulativeStats.NoOpCount++ @@ -91,7 +85,7 @@ func (e *Engine) RandomAction(actionOpts ActionOpts) error { actionName := pickActionWeighted(actionControlOpts, actions) if string(actionName) == "" { - return ErrNoActionPicked + return robustness.ErrNoActionPicked } _, err := e.ExecAction(actionName, actionOpts[actionName]) @@ -112,15 +106,7 @@ func (e *Engine) checkErrRecovery(incomingErr error, actionOpts ActionOpts) (out if errIsNotEnoughSpace(incomingErr) && ctrl[ThrowNoSpaceOnDeviceErrField] == "" { // no space left on device // Delete everything in the data directory - const hundredPcnt = 100 - - deleteDirActionKey := DeleteDirectoryContentsActionKey - deleteRootOpts := map[string]string{ - MaxDirDepthField: strconv.Itoa(0), - DeletePercentOfContentsField: strconv.Itoa(hundredPcnt), - } - - _, outgoingErr = e.ExecAction(deleteDirActionKey, deleteRootOpts) + outgoingErr = e.FileWriter.DeleteEverything() if outgoingErr != nil { return outgoingErr } @@ -132,7 +118,7 @@ func (e *Engine) checkErrRecovery(incomingErr error, actionOpts ActionOpts) (out restoreActionKey := RestoreIntoDataDirectoryActionKey _, outgoingErr = e.ExecAction(restoreActionKey, actionOpts[restoreActionKey]) - if errors.Is(outgoingErr, ErrNoOp) { + if errors.Is(outgoingErr, robustness.ErrNoOp) { outgoingErr = nil } else { e.RunStats.DataRestoreCount++ @@ -186,13 +172,13 @@ type Action struct { var actions = map[ActionKey]Action{ SnapshotRootDirActionKey: { f: func(e *Engine, opts map[string]string, l *LogEntry) (out map[string]string, err error) { - log.Printf("Creating snapshot of root directory %s", e.FileWriter.LocalDataDir) + log.Printf("Creating snapshot of root directory %s", e.FileWriter.DataDirectory()) ctx := context.TODO() - snapID, err := e.Checker.TakeSnapshot(ctx, e.FileWriter.LocalDataDir) + snapID, err := e.Checker.TakeSnapshot(ctx, e.FileWriter.DataDirectory()) setLogEntryCmdOpts(l, map[string]string{ - "snap-dir": e.FileWriter.LocalDataDir, + "snap-dir": e.FileWriter.DataDirectory(), "snapID": snapID, }) @@ -246,118 +232,26 @@ type Action struct { }, WriteRandomFilesActionKey: { f: func(e *Engine, opts map[string]string, l *LogEntry) (out map[string]string, err error) { - // Directory depth - maxDirDepth := getOptAsIntOrDefault(MaxDirDepthField, opts, defaultMaxDirDepth) - dirDepth := rand.Intn(maxDirDepth + 1) + out, err = e.FileWriter.WriteRandomFiles(opts) + setLogEntryCmdOpts(l, out) - // File size range - maxFileSizeB := getOptAsIntOrDefault(MaxFileSizeField, opts, defaultMaxFileSize) - minFileSizeB := getOptAsIntOrDefault(MinFileSizeField, opts, defaultMinFileSize) - - // Number of files to write - maxNumFiles := getOptAsIntOrDefault(MaxNumFilesPerWriteField, opts, defaultMaxNumFilesPerWrite) - minNumFiles := getOptAsIntOrDefault(MinNumFilesPerWriteField, opts, defaultMinNumFilesPerWrite) - - numFiles := rand.Intn(maxNumFiles-minNumFiles+1) + minNumFiles //nolint:gosec - - // Dedup Percentage - maxDedupPcnt := getOptAsIntOrDefault(MaxDedupePercentField, opts, defaultMaxDedupePercent) - minDedupPcnt := getOptAsIntOrDefault(MinDedupePercentField, opts, defaultMinDedupePercent) - - dedupStep := getOptAsIntOrDefault(DedupePercentStepField, opts, defaultDedupePercentStep) - - dedupPcnt := dedupStep * (rand.Intn(maxDedupPcnt/dedupStep-minDedupPcnt/dedupStep+1) + minDedupPcnt/dedupStep) //nolint:gosec - - blockSize := int64(defaultMinFileSize) - - fioOpts := fio.Options{}. - WithFileSizeRange(int64(minFileSizeB), int64(maxFileSizeB)). - WithNumFiles(numFiles). - WithBlockSize(blockSize). - WithDedupePercentage(dedupPcnt). - WithNoFallocate() - - ioLimit := getOptAsIntOrDefault(IOLimitPerWriteAction, opts, defaultIOLimitPerWriteAction) - - if ioLimit > 0 { - freeSpaceLimitB := getOptAsIntOrDefault(FreeSpaceLimitField, opts, defaultFreeSpaceLimit) - - freeSpaceB, err := getFreeSpaceB(e.FileWriter.LocalDataDir) - if err != nil { - return nil, err - } - log.Printf("Free Space %v B, limit %v B, ioLimit %v B\n", freeSpaceB, freeSpaceLimitB, ioLimit) - - if int(freeSpaceB)-ioLimit < freeSpaceLimitB { - ioLimit = int(freeSpaceB) - freeSpaceLimitB - log.Printf("Cutting down I/O limit for space %v", ioLimit) - if ioLimit <= 0 { - return nil, ErrCannotPerformIO - } - } - - fioOpts = fioOpts.WithIOLimit(int64(ioLimit)) - } - - relBasePath := "." - - log.Printf("Writing files at depth %v (fileSize: %v-%v, numFiles: %v, blockSize: %v, dedupPcnt: %v, ioLimit: %v)\n", dirDepth, minFileSizeB, maxFileSizeB, numFiles, blockSize, dedupPcnt, ioLimit) - - setLogEntryCmdOpts(l, map[string]string{ - "dirDepth": strconv.Itoa(dirDepth), - "relBasePath": relBasePath, - }) - - for k, v := range fioOpts { - l.CmdOpts[k] = v - } - - return nil, e.FileWriter.WriteFilesAtDepthRandomBranch(relBasePath, dirDepth, fioOpts) + return }, }, DeleteRandomSubdirectoryActionKey: { f: func(e *Engine, opts map[string]string, l *LogEntry) (out map[string]string, err error) { - maxDirDepth := getOptAsIntOrDefault(MaxDirDepthField, opts, defaultMaxDirDepth) - if maxDirDepth <= 0 { - return nil, ErrInvalidOption - } - dirDepth := rand.Intn(maxDirDepth) + 1 //nolint:gosec + out, err = e.FileWriter.DeleteRandomSubdirectory(opts) + setLogEntryCmdOpts(l, out) - log.Printf("Deleting directory at depth %v\n", dirDepth) - - setLogEntryCmdOpts(l, map[string]string{"dirDepth": strconv.Itoa(dirDepth)}) - - err = e.FileWriter.DeleteDirAtDepth("", dirDepth) - if errors.Is(err, fio.ErrNoDirFound) { - log.Print(err) - return nil, ErrNoOp - } - - return nil, err + return }, }, DeleteDirectoryContentsActionKey: { f: func(e *Engine, opts map[string]string, l *LogEntry) (out map[string]string, err error) { - maxDirDepth := getOptAsIntOrDefault(MaxDirDepthField, opts, defaultMaxDirDepth) - dirDepth := rand.Intn(maxDirDepth + 1) //nolint:gosec + out, err = e.FileWriter.DeleteDirectoryContents(opts) + setLogEntryCmdOpts(l, out) - pcnt := getOptAsIntOrDefault(DeletePercentOfContentsField, opts, defaultDeletePercentOfContents) - - log.Printf("Deleting %d%% of directory contents at depth %v\n", pcnt, dirDepth) - - setLogEntryCmdOpts(l, map[string]string{ - "dirDepth": strconv.Itoa(dirDepth), - "percent": strconv.Itoa(pcnt), - }) - - const pcntConv = 100 - err = e.FileWriter.DeleteContentsAtDepth("", dirDepth, float32(pcnt)/pcntConv) - if errors.Is(err, fio.ErrNoDirFound) { - log.Print(err) - return nil, ErrNoOp - } - - return nil, err + return }, }, RestoreIntoDataDirectoryActionKey: { @@ -372,7 +266,7 @@ type Action struct { setLogEntryCmdOpts(l, map[string]string{"snapID": snapID}) b := &bytes.Buffer{} - err = e.Checker.RestoreSnapshotToPath(context.Background(), snapID, e.FileWriter.LocalDataDir, b) + err = e.Checker.RestoreSnapshotToPath(context.Background(), snapID, e.FileWriter.DataDirectory(), b) if err != nil { log.Print(b.String()) return nil, err @@ -385,55 +279,16 @@ type Action struct { // Action constants. const ( - defaultMaxDirDepth = 20 - defaultMaxFileSize = 1 * 1024 * 1024 * 1024 // 1GB - defaultMinFileSize = 4096 - defaultMaxNumFilesPerWrite = 10000 - defaultMinNumFilesPerWrite = 1 - defaultIOLimitPerWriteAction = 0 // A zero value does not impose any limit on IO - defaultFreeSpaceLimit = 100 * 1024 * 1024 // 100 MB - defaultMaxDedupePercent = 100 - defaultMinDedupePercent = 0 - defaultDedupePercentStep = 25 - defaultDeletePercentOfContents = 20 - defaultActionRepeats = 1 + defaultActionRepeats = 1 ) // Option field names. const ( - MaxDirDepthField = "max-dir-depth" - MaxFileSizeField = "max-file-size" - MinFileSizeField = "min-file-size" - MaxNumFilesPerWriteField = "max-num-files-per-write" - MinNumFilesPerWriteField = "min-num-files-per-write" - IOLimitPerWriteAction = "io-limit-per-write" - FreeSpaceLimitField = "free-space-limit" - MaxDedupePercentField = "max-dedupe-percent" - MinDedupePercentField = "min-dedupe-percent" - DedupePercentStepField = "dedupe-percent" ActionRepeaterField = "repeat-action" ThrowNoSpaceOnDeviceErrField = "throw-no-space-error" - DeletePercentOfContentsField = "delete-contents-percent" SnapshotIDField = "snapshot-ID" ) -func getOptAsIntOrDefault(key string, opts map[string]string, def int) int { - if opts == nil { - return def - } - - if opts[key] == "" { - return def - } - - retInt, err := strconv.Atoi(opts[key]) - if err != nil { - return def - } - - return retInt -} - func defaultActionControls() map[string]string { ret := make(map[string]string, len(actions)) @@ -456,7 +311,7 @@ func pickActionWeighted(actionControlOpts map[string]string, actionList map[Acti sum := 0 for actionName := range actionList { - weight := getOptAsIntOrDefault(string(actionName), actionControlOpts, 0) + weight := robustness.GetOptAsIntOrDefault(string(actionName), actionControlOpts, 0) if weight == 0 { continue } @@ -471,7 +326,7 @@ func pickActionWeighted(actionControlOpts map[string]string, actionList map[Acti } func errIsNotEnoughSpace(err error) bool { - return errors.Is(err, ErrCannotPerformIO) || strings.Contains(err.Error(), noSpaceOnDeviceMatchStr) + return errors.Is(err, robustness.ErrCannotPerformIO) || strings.Contains(err.Error(), noSpaceOnDeviceMatchStr) } func (e *Engine) getSnapIDOptOrRandLive(opts map[string]string) (snapID string, err error) { @@ -482,7 +337,7 @@ func (e *Engine) getSnapIDOptOrRandLive(opts map[string]string) (snapID string, snapIDList := e.Checker.GetLiveSnapIDs() if len(snapIDList) == 0 { - return "", ErrNoOp + return "", robustness.ErrNoOp } return snapIDList[rand.Intn(len(snapIDList))], nil //nolint:gosec diff --git a/tests/robustness/engine/engine.go b/tests/robustness/engine/engine.go index d0bf8b906..a8ec37284 100644 --- a/tests/robustness/engine/engine.go +++ b/tests/robustness/engine/engine.go @@ -19,8 +19,8 @@ "github.com/kopia/kopia/tests/robustness" "github.com/kopia/kopia/tests/robustness/checker" + "github.com/kopia/kopia/tests/robustness/fiofilewriter" "github.com/kopia/kopia/tests/robustness/snapmeta" - "github.com/kopia/kopia/tests/tools/fio" "github.com/kopia/kopia/tests/tools/fswalker" "github.com/kopia/kopia/tests/tools/kopiarunner" ) @@ -39,11 +39,6 @@ ) var ( - // ErrNoOp is thrown when an action could not do anything useful. - ErrNoOp = fmt.Errorf("no-op") - // ErrCannotPerformIO is returned if the engine determines there is not enough space - // to write files. - ErrCannotPerformIO = fmt.Errorf("cannot perform i/o") // ErrS3BucketNameEnvUnset is the error returned when the S3BucketNameEnvKey environment variable is not set. ErrS3BucketNameEnvUnset = fmt.Errorf("environment variable required: %v", S3BucketNameEnvKey) noSpaceOnDeviceMatchStr = "no space left on device" @@ -51,7 +46,7 @@ // Engine is the outer level testing framework for robustness testing. type Engine struct { - FileWriter *fio.Runner + FileWriter robustness.FileWriter TestRepo robustness.Snapshotter MetaStore robustness.Persister Checker *checker.Checker @@ -85,8 +80,8 @@ func NewEngine(workingDir string) (*Engine, error) { }, } - // Fill the file writer - e.FileWriter, err = fio.NewRunner() + // Create an FIO file writer + e.FileWriter, err = fiofilewriter.New() if err != nil { e.CleanComponents() return nil, err diff --git a/tests/robustness/engine/engine_test.go b/tests/robustness/engine/engine_test.go index f90df92b4..04df637e6 100644 --- a/tests/robustness/engine/engine_test.go +++ b/tests/robustness/engine/engine_test.go @@ -21,6 +21,8 @@ "github.com/minio/minio-go/v7" "github.com/minio/minio-go/v7/pkg/credentials" + "github.com/kopia/kopia/tests/robustness" + "github.com/kopia/kopia/tests/robustness/fiofilewriter" "github.com/kopia/kopia/tests/robustness/snapmeta" "github.com/kopia/kopia/tests/testenv" "github.com/kopia/kopia/tests/tools/fio" @@ -65,13 +67,13 @@ func TestEngineWritefilesBasicFS(t *testing.T) { numFiles := 10 fioOpts := fio.Options{}.WithFileSize(fileSize).WithNumFiles(numFiles) - - err = eng.FileWriter.WriteFiles("", fioOpts) + fioRunner := engineFioRunner(t, eng) + err = fioRunner.WriteFiles("", fioOpts) testenv.AssertNoError(t, err) snapIDs := eng.Checker.GetSnapIDs() - snapID, err := eng.Checker.TakeSnapshot(ctx, eng.FileWriter.LocalDataDir) + snapID, err := eng.Checker.TakeSnapshot(ctx, fioRunner.LocalDataDir) testenv.AssertNoError(t, err) err = eng.Checker.RestoreSnapshot(ctx, snapID, os.Stdout) @@ -175,13 +177,13 @@ func TestWriteFilesBasicS3(t *testing.T) { numFiles := 10 fioOpts := fio.Options{}.WithFileSize(fileSize).WithNumFiles(numFiles) - - err = eng.FileWriter.WriteFiles("", fioOpts) + fioRunner := engineFioRunner(t, eng) + err = fioRunner.WriteFiles("", fioOpts) testenv.AssertNoError(t, err) snapIDs := eng.Checker.GetLiveSnapIDs() - snapID, err := eng.Checker.TakeSnapshot(ctx, eng.FileWriter.LocalDataDir) + snapID, err := eng.Checker.TakeSnapshot(ctx, fioRunner.LocalDataDir) testenv.AssertNoError(t, err) err = eng.Checker.RestoreSnapshot(ctx, snapID, os.Stdout) @@ -217,11 +219,11 @@ func TestDeleteSnapshotS3(t *testing.T) { numFiles := 10 fioOpts := fio.Options{}.WithFileSize(fileSize).WithNumFiles(numFiles) - - err = eng.FileWriter.WriteFiles("", fioOpts) + fioRunner := engineFioRunner(t, eng) + err = fioRunner.WriteFiles("", fioOpts) testenv.AssertNoError(t, err) - snapID, err := eng.Checker.TakeSnapshot(ctx, eng.FileWriter.LocalDataDir) + snapID, err := eng.Checker.TakeSnapshot(ctx, fioRunner.LocalDataDir) testenv.AssertNoError(t, err) err = eng.Checker.RestoreSnapshot(ctx, snapID, os.Stdout) @@ -261,11 +263,12 @@ func TestSnapshotVerificationFail(t *testing.T) { numFiles := 10 fioOpt := fio.Options{}.WithFileSize(fileSize).WithNumFiles(numFiles) - err = eng.FileWriter.WriteFiles("", fioOpt) + fioRunner := engineFioRunner(t, eng) + err = fioRunner.WriteFiles("", fioOpt) testenv.AssertNoError(t, err) // Take a first snapshot - snapID1, err := eng.Checker.TakeSnapshot(ctx, eng.FileWriter.LocalDataDir) + snapID1, err := eng.Checker.TakeSnapshot(ctx, fioRunner.LocalDataDir) testenv.AssertNoError(t, err) // Get the metadata collected on that snapshot @@ -273,11 +276,11 @@ func TestSnapshotVerificationFail(t *testing.T) { testenv.AssertNoError(t, err) // Do additional writes, writing 1 extra byte than before - err = eng.FileWriter.WriteFiles("", fioOpt.WithFileSize(fileSize+1)) + err = fioRunner.WriteFiles("", fioOpt.WithFileSize(fileSize+1)) testenv.AssertNoError(t, err) // Take a second snapshot - snapID2, err := eng.Checker.TakeSnapshot(ctx, eng.FileWriter.LocalDataDir) + snapID2, err := eng.Checker.TakeSnapshot(ctx, fioRunner.LocalDataDir) testenv.AssertNoError(t, err) // Get the second snapshot's metadata @@ -329,12 +332,12 @@ func TestDataPersistency(t *testing.T) { numFiles := 10 fioOpt := fio.Options{}.WithFileSize(fileSize).WithNumFiles(numFiles) - - err = eng.FileWriter.WriteFiles("", fioOpt) + fioRunner := engineFioRunner(t, eng) + err = fioRunner.WriteFiles("", fioOpt) testenv.AssertNoError(t, err) // Take a snapshot - snapID, err := eng.Checker.TakeSnapshot(ctx, eng.FileWriter.LocalDataDir) + snapID, err := eng.Checker.TakeSnapshot(ctx, fioRunner.LocalDataDir) testenv.AssertNoError(t, err) // Get the walk data associated with the snapshot that was taken @@ -358,12 +361,13 @@ func TestDataPersistency(t *testing.T) { err = eng2.InitFilesystem(ctx, dataRepoPath, metadataRepoPath) testenv.AssertNoError(t, err) - err = eng2.Checker.RestoreSnapshotToPath(ctx, snapID, eng2.FileWriter.LocalDataDir, os.Stdout) + fioRunner2 := engineFioRunner(t, eng2) + err = eng2.Checker.RestoreSnapshotToPath(ctx, snapID, fioRunner2.LocalDataDir, os.Stdout) testenv.AssertNoError(t, err) // Compare the data directory of the second engine with the fingerprint // of the snapshot taken earlier. They should match. - err = fswalker.NewWalkCompare().Compare(ctx, eng2.FileWriter.LocalDataDir, dataDirWalk.ValidationData, os.Stdout) + err = fswalker.NewWalkCompare().Compare(ctx, fioRunner2.LocalDataDir, dataDirWalk.ValidationData, os.Stdout) testenv.AssertNoError(t, err) } @@ -482,22 +486,22 @@ func TestActionsFilesystem(t *testing.T) { actionOpts := ActionOpts{ WriteRandomFilesActionKey: map[string]string{ - MaxDirDepthField: "20", - MaxFileSizeField: strconv.Itoa(10 * 1024 * 1024), - MinFileSizeField: strconv.Itoa(10 * 1024 * 1024), - MaxNumFilesPerWriteField: "10", - MinNumFilesPerWriteField: "10", - MaxDedupePercentField: "100", - MinDedupePercentField: "100", - DedupePercentStepField: "1", - IOLimitPerWriteAction: "0", + fiofilewriter.MaxDirDepthField: "20", + fiofilewriter.MaxFileSizeField: strconv.Itoa(10 * 1024 * 1024), + fiofilewriter.MinFileSizeField: strconv.Itoa(10 * 1024 * 1024), + fiofilewriter.MaxNumFilesPerWriteField: "10", + fiofilewriter.MinNumFilesPerWriteField: "10", + fiofilewriter.MaxDedupePercentField: "100", + fiofilewriter.MinDedupePercentField: "100", + fiofilewriter.DedupePercentStepField: "1", + fiofilewriter.IOLimitPerWriteAction: "0", }, } numActions := 10 for loop := 0; loop < numActions; loop++ { err := eng.RandomAction(actionOpts) - if !(err == nil || errors.Is(err, ErrNoOp)) { + if !(err == nil || errors.Is(err, robustness.ErrNoOp)) { t.Error("Hit error", err) } } @@ -525,22 +529,22 @@ func TestActionsS3(t *testing.T) { actionOpts := ActionOpts{ WriteRandomFilesActionKey: map[string]string{ - MaxDirDepthField: "20", - MaxFileSizeField: strconv.Itoa(10 * 1024 * 1024), - MinFileSizeField: strconv.Itoa(10 * 1024 * 1024), - MaxNumFilesPerWriteField: "10", - MinNumFilesPerWriteField: "10", - MaxDedupePercentField: "100", - MinDedupePercentField: "100", - DedupePercentStepField: "1", - IOLimitPerWriteAction: "0", + fiofilewriter.MaxDirDepthField: "20", + fiofilewriter.MaxFileSizeField: strconv.Itoa(10 * 1024 * 1024), + fiofilewriter.MinFileSizeField: strconv.Itoa(10 * 1024 * 1024), + fiofilewriter.MaxNumFilesPerWriteField: "10", + fiofilewriter.MinNumFilesPerWriteField: "10", + fiofilewriter.MaxDedupePercentField: "100", + fiofilewriter.MinDedupePercentField: "100", + fiofilewriter.DedupePercentStepField: "1", + fiofilewriter.IOLimitPerWriteAction: "0", }, } numActions := 10 for loop := 0; loop < numActions; loop++ { err := eng.RandomAction(actionOpts) - if !(err == nil || errors.Is(err, ErrNoOp)) { + if !(err == nil || errors.Is(err, robustness.ErrNoOp)) { t.Error("Hit error", err) } } @@ -581,12 +585,12 @@ func TestIOLimitPerWriteAction(t *testing.T) { string(DeleteRandomSubdirectoryActionKey): strconv.Itoa(0), }, WriteRandomFilesActionKey: map[string]string{ - MaxDirDepthField: "2", - MaxFileSizeField: strconv.Itoa(10 * 1024 * 1024), - MinFileSizeField: strconv.Itoa(10 * 1024 * 1024), - MaxNumFilesPerWriteField: "100", - MinNumFilesPerWriteField: "100", - IOLimitPerWriteAction: strconv.Itoa(1 * 1024 * 1024), + fiofilewriter.MaxDirDepthField: "2", + fiofilewriter.MaxFileSizeField: strconv.Itoa(10 * 1024 * 1024), + fiofilewriter.MinFileSizeField: strconv.Itoa(10 * 1024 * 1024), + fiofilewriter.MaxNumFilesPerWriteField: "100", + fiofilewriter.MinNumFilesPerWriteField: "100", + fiofilewriter.IOLimitPerWriteAction: strconv.Itoa(1 * 1024 * 1024), }, } @@ -737,3 +741,15 @@ func TestLogsPersist(t *testing.T) { t.Errorf("Logs do not match\n%v\n%v", got, want) } } + +// engineFioRunner extracts the fio.Runner used by the engine. +func engineFioRunner(t *testing.T, eng *Engine) *fio.Runner { + t.Helper() + + fiofw, ok := eng.FileWriter.(*fiofilewriter.FileWriter) + if !ok { + t.Fatal("engine does not have a fiofilewriter.FileWriter") + } + + return fiofw.Runner +} diff --git a/tests/robustness/engine/sys.go b/tests/robustness/engine/sys.go deleted file mode 100644 index 5a4032e90..000000000 --- a/tests/robustness/engine/sys.go +++ /dev/null @@ -1,19 +0,0 @@ -// +build darwin,amd64 linux,amd64 - -package engine - -import ( - "syscall" -) - -func getFreeSpaceB(path string) (uint64, error) { - var stat syscall.Statfs_t - - err := syscall.Statfs(path, &stat) - if err != nil { - return 0, err - } - - // Available blocks * size per block = available space in bytes - return stat.Bavail * uint64(stat.Bsize), nil -} diff --git a/tests/robustness/errors.go b/tests/robustness/errors.go new file mode 100644 index 000000000..08b5edc0e --- /dev/null +++ b/tests/robustness/errors.go @@ -0,0 +1,21 @@ +package robustness + +import ( + "errors" + "fmt" +) + +var ( + // ErrNoOp is thrown when an action could not do anything useful. + ErrNoOp = fmt.Errorf("no-op") + + // ErrCannotPerformIO is returned if the engine determines there is not enough space + // to write files. + ErrCannotPerformIO = fmt.Errorf("cannot perform i/o") + + // ErrNoActionPicked is returned if a random action could not be selected. + ErrNoActionPicked = errors.New("unable to pick an action with the action control options provided") + + // ErrInvalidOption is returned if an option value is invalid or missing. + ErrInvalidOption = errors.New("invalid option setting") +) diff --git a/tests/robustness/filewriter.go b/tests/robustness/filewriter.go new file mode 100644 index 000000000..3c3c90c85 --- /dev/null +++ b/tests/robustness/filewriter.go @@ -0,0 +1,31 @@ +package robustness + +// FileWriter is an interface used for filesystem related actions. +type FileWriter interface { + // Cleanup is called prior to termination. + // TBD: Will be removed when initialization refactored. + Cleanup() + + // DataDirectory returns the absolute path of the data directory configured. + DataDirectory() string + + // DeleteDirectoryContents deletes some of the content of a random directory, + // based on its input option values (none of which are required). + // The method returns the effective option values used and the error if any. + // ErrNoOp is returned if no directory is found. + DeleteDirectoryContents(opts map[string]string) (map[string]string, error) + + // DeleteEverything deletes all content. + DeleteEverything() error + + // DeleteRandomSubdirectory deletes a random directory, based + // on its input option values (none of which are required). + // The method returns the effective option values used and the error if any. + // ErrNoOp is returned if no directory is found. + DeleteRandomSubdirectory(opts map[string]string) (map[string]string, error) + + // WriteRandomFiles writes a number of files in a random directory, based + // on its input option values (none of which are required). + // The method returns the effective option values used and the error if any. + WriteRandomFiles(opts map[string]string) (map[string]string, error) +} diff --git a/tests/robustness/fiofilewriter/fio_filewriter.go b/tests/robustness/fiofilewriter/fio_filewriter.go new file mode 100644 index 000000000..ed6a7c80f --- /dev/null +++ b/tests/robustness/fiofilewriter/fio_filewriter.go @@ -0,0 +1,255 @@ +// +build darwin,amd64 linux,amd64 + +// Package fiofilewriter provides a FileWriter based on FIO. +package fiofilewriter + +import ( + "errors" + "log" + "math/rand" + "strconv" + "syscall" + + "github.com/kopia/kopia/tests/robustness" + "github.com/kopia/kopia/tests/tools/fio" +) + +// Option field names. +const ( + DedupePercentStepField = "dedupe-percent" + DeletePercentOfContentsField = "delete-contents-percent" + FreeSpaceLimitField = "free-space-limit" + IOLimitPerWriteAction = "io-limit-per-write" + MaxDedupePercentField = "max-dedupe-percent" + MaxDirDepthField = "max-dir-depth" + MaxFileSizeField = "max-file-size" + MaxNumFilesPerWriteField = "max-num-files-per-write" + MinDedupePercentField = "min-dedupe-percent" + MinFileSizeField = "min-file-size" + MinNumFilesPerWriteField = "min-num-files-per-write" +) + +// Option defaults. +const ( + defaultDedupePercentStep = 25 + defaultDeletePercentOfContents = 20 + defaultFreeSpaceLimit = 100 * 1024 * 1024 // 100 MB + defaultIOLimitPerWriteAction = 0 // A zero value does not impose any limit on IO + defaultMaxDedupePercent = 100 + defaultMaxDirDepth = 20 + defaultMaxFileSize = 1 * 1024 * 1024 * 1024 // 1GB + defaultMaxNumFilesPerWrite = 10000 + defaultMinDedupePercent = 0 + defaultMinFileSize = 4096 + defaultMinNumFilesPerWrite = 1 +) + +// New returns a FileWriter based on FIO. +// See tests/tools/fio for configuration details. +func New() (robustness.FileWriter, error) { + runner, err := fio.NewRunner() + if err != nil { + return nil, err + } + + return &FileWriter{Runner: runner}, nil +} + +// FileWriter implements a FileWriter over tools/fio.Runner. +type FileWriter struct { + Runner *fio.Runner +} + +// DataDirectory returns the data directory configured. +// See tests/tools/fio for details. +func (fw *FileWriter) DataDirectory() string { + return fw.Runner.LocalDataDir +} + +// WriteRandomFiles writes a number of files at some filesystem depth, based +// on its input options. +// +// - MaxDirDepthField +// - MaxFileSizeField +// - MinFileSizeField +// - MaxNumFilesPerWriteField +// - MinNumFilesPerWriteField +// - MaxDedupePercentField +// - MinDedupePercentField +// - DedupePercentStepField +// +// Default values are used for missing options. The method +// returns the effective options used along with the selected depth +// and the error if any. +func (fw *FileWriter) WriteRandomFiles(opts map[string]string) (map[string]string, error) { + // Directory depth + maxDirDepth := robustness.GetOptAsIntOrDefault(MaxDirDepthField, opts, defaultMaxDirDepth) + dirDepth := rand.Intn(maxDirDepth + 1) + + // File size range + maxFileSizeB := robustness.GetOptAsIntOrDefault(MaxFileSizeField, opts, defaultMaxFileSize) + minFileSizeB := robustness.GetOptAsIntOrDefault(MinFileSizeField, opts, defaultMinFileSize) + + // Number of files to write + maxNumFiles := robustness.GetOptAsIntOrDefault(MaxNumFilesPerWriteField, opts, defaultMaxNumFilesPerWrite) + minNumFiles := robustness.GetOptAsIntOrDefault(MinNumFilesPerWriteField, opts, defaultMinNumFilesPerWrite) + + numFiles := rand.Intn(maxNumFiles-minNumFiles+1) + minNumFiles //nolint:gosec + + // Dedup Percentage + maxDedupPcnt := robustness.GetOptAsIntOrDefault(MaxDedupePercentField, opts, defaultMaxDedupePercent) + minDedupPcnt := robustness.GetOptAsIntOrDefault(MinDedupePercentField, opts, defaultMinDedupePercent) + + dedupStep := robustness.GetOptAsIntOrDefault(DedupePercentStepField, opts, defaultDedupePercentStep) + + dedupPcnt := dedupStep * (rand.Intn(maxDedupPcnt/dedupStep-minDedupPcnt/dedupStep+1) + minDedupPcnt/dedupStep) //nolint:gosec + + blockSize := int64(defaultMinFileSize) + + fioOpts := fio.Options{}. + WithFileSizeRange(int64(minFileSizeB), int64(maxFileSizeB)). + WithNumFiles(numFiles). + WithBlockSize(blockSize). + WithDedupePercentage(dedupPcnt). + WithNoFallocate() + + ioLimit := robustness.GetOptAsIntOrDefault(IOLimitPerWriteAction, opts, defaultIOLimitPerWriteAction) + + if ioLimit > 0 { + freeSpaceLimitB := robustness.GetOptAsIntOrDefault(FreeSpaceLimitField, opts, defaultFreeSpaceLimit) + + freeSpaceB, err := getFreeSpaceB(fw.Runner.LocalDataDir) + if err != nil { + return nil, err + } + + log.Printf("Free Space %v B, limit %v B, ioLimit %v B\n", freeSpaceB, freeSpaceLimitB, ioLimit) + + if int(freeSpaceB)-ioLimit < freeSpaceLimitB { + ioLimit = int(freeSpaceB) - freeSpaceLimitB + + log.Printf("Cutting down I/O limit for space %v", ioLimit) + + if ioLimit <= 0 { + return nil, robustness.ErrCannotPerformIO + } + } + + fioOpts = fioOpts.WithIOLimit(int64(ioLimit)) + } + + relBasePath := "." + + log.Printf("Writing files at depth %v (fileSize: %v-%v, numFiles: %v, blockSize: %v, dedupPcnt: %v, ioLimit: %v)\n", dirDepth, minFileSizeB, maxFileSizeB, numFiles, blockSize, dedupPcnt, ioLimit) + + retOpts := make(map[string]string, len(opts)) + for k, v := range opts { + retOpts[k] = v + } + + for k, v := range fioOpts { + retOpts[k] = v + } + + retOpts["dirDepth"] = strconv.Itoa(dirDepth) + retOpts["relBasePath"] = relBasePath + + return retOpts, fw.Runner.WriteFilesAtDepthRandomBranch(relBasePath, dirDepth, fioOpts) +} + +// DeleteRandomSubdirectory deletes a random directory up to a specified depth, +// based on its input options: +// +// - MaxDirDepthField +// +// Default values are used for missing options. The method +// returns the effective options used along with the selected depth +// and the error if any. ErrNoOp is returned if no directory is found. +func (fw *FileWriter) DeleteRandomSubdirectory(opts map[string]string) (map[string]string, error) { + maxDirDepth := robustness.GetOptAsIntOrDefault(MaxDirDepthField, opts, defaultMaxDirDepth) + if maxDirDepth <= 0 { + return nil, robustness.ErrInvalidOption + } + + dirDepth := rand.Intn(maxDirDepth) + 1 //nolint:gosec + + log.Printf("Deleting directory at depth %v\n", dirDepth) + + retOpts := make(map[string]string, len(opts)) + for k, v := range opts { + retOpts[k] = v + } + + retOpts["dirDepth"] = strconv.Itoa(dirDepth) + + err := fw.Runner.DeleteDirAtDepth("", dirDepth) + if errors.Is(err, fio.ErrNoDirFound) { + log.Print(err) + err = robustness.ErrNoOp + } + + return retOpts, err +} + +// DeleteDirectoryContents deletes some of the contents of random directory up to a specified depth, +// based on its input options: +// +// - MaxDirDepthField +// - DeletePercentOfContentsField +// +// Default values are used for missing options. The method +// returns the effective options used along with the selected depth +// and the error if any. ErrNoOp is returned if no directory is found. +func (fw *FileWriter) DeleteDirectoryContents(opts map[string]string) (map[string]string, error) { + maxDirDepth := robustness.GetOptAsIntOrDefault(MaxDirDepthField, opts, defaultMaxDirDepth) + dirDepth := rand.Intn(maxDirDepth + 1) //nolint:gosec + + pcnt := robustness.GetOptAsIntOrDefault(DeletePercentOfContentsField, opts, defaultDeletePercentOfContents) + + log.Printf("Deleting %d%% of directory contents at depth %v\n", pcnt, dirDepth) + + retOpts := make(map[string]string, len(opts)) + for k, v := range opts { + retOpts[k] = v + } + + retOpts["dirDepth"] = strconv.Itoa(dirDepth) + retOpts["percent"] = strconv.Itoa(pcnt) + + const pcntConv = 100 + + err := fw.Runner.DeleteContentsAtDepth("", dirDepth, float32(pcnt)/pcntConv) + if errors.Is(err, fio.ErrNoDirFound) { + log.Print(err) + err = robustness.ErrNoOp + } + + return retOpts, err +} + +// DeleteEverything deletes all content. +func (fw *FileWriter) DeleteEverything() error { + _, err := fw.DeleteDirectoryContents(map[string]string{ + MaxDirDepthField: strconv.Itoa(0), + DeletePercentOfContentsField: strconv.Itoa(100), + }) + + return err +} + +// Cleanup is part of FileWriter. +func (fw *FileWriter) Cleanup() { + fw.Runner.Cleanup() +} + +func getFreeSpaceB(path string) (uint64, error) { + var stat syscall.Statfs_t + + err := syscall.Statfs(path, &stat) + if err != nil { + return 0, err + } + + // Available blocks * size per block = available space in bytes + return stat.Bavail * uint64(stat.Bsize), nil +} diff --git a/tests/robustness/options.go b/tests/robustness/options.go new file mode 100644 index 000000000..3d57c8291 --- /dev/null +++ b/tests/robustness/options.go @@ -0,0 +1,22 @@ +package robustness + +import "strconv" + +// GetOptAsIntOrDefault extracts an integer value from a configuration map +// if present, or else returns a default. +func GetOptAsIntOrDefault(key string, opts map[string]string, def int) int { + if opts == nil { + return def + } + + if opts[key] == "" { + return def + } + + retInt, err := strconv.Atoi(opts[key]) + if err != nil { + return def + } + + return retInt +} diff --git a/tests/robustness/robustness_test/main_test.go b/tests/robustness/robustness_test/main_test.go index 5b95bafc4..cc53a4e00 100644 --- a/tests/robustness/robustness_test/main_test.go +++ b/tests/robustness/robustness_test/main_test.go @@ -12,6 +12,7 @@ "testing" "time" + "github.com/kopia/kopia/tests/robustness" "github.com/kopia/kopia/tests/robustness/engine" "github.com/kopia/kopia/tests/tools/fio" "github.com/kopia/kopia/tests/tools/kopiarunner" @@ -65,7 +66,7 @@ func TestMain(m *testing.M) { // Restore a random snapshot into the data directory _, err = eng.ExecAction(engine.RestoreIntoDataDirectoryActionKey, nil) - if err != nil && !errors.Is(err, engine.ErrNoOp) { + if err != nil && !errors.Is(err, robustness.ErrNoOp) { eng.Cleanup() log.Fatalln("error restoring into the data directory:", err) } diff --git a/tests/robustness/robustness_test/robustness_test.go b/tests/robustness/robustness_test/robustness_test.go index 7aab3eac9..44c33c316 100644 --- a/tests/robustness/robustness_test/robustness_test.go +++ b/tests/robustness/robustness_test/robustness_test.go @@ -9,7 +9,9 @@ "testing" "time" + "github.com/kopia/kopia/tests/robustness" "github.com/kopia/kopia/tests/robustness/engine" + "github.com/kopia/kopia/tests/robustness/fiofilewriter" "github.com/kopia/kopia/tests/testenv" ) @@ -18,11 +20,11 @@ func TestManySmallFiles(t *testing.T) { numFiles := 10000 fileWriteOpts := map[string]string{ - engine.MaxDirDepthField: strconv.Itoa(1), - engine.MaxFileSizeField: strconv.Itoa(fileSize), - engine.MinFileSizeField: strconv.Itoa(fileSize), - engine.MaxNumFilesPerWriteField: strconv.Itoa(numFiles), - engine.MinNumFilesPerWriteField: strconv.Itoa(numFiles), + fiofilewriter.MaxDirDepthField: strconv.Itoa(1), + fiofilewriter.MaxFileSizeField: strconv.Itoa(fileSize), + fiofilewriter.MinFileSizeField: strconv.Itoa(fileSize), + fiofilewriter.MaxNumFilesPerWriteField: strconv.Itoa(numFiles), + fiofilewriter.MinNumFilesPerWriteField: strconv.Itoa(numFiles), } _, err := eng.ExecAction(engine.WriteRandomFilesActionKey, fileWriteOpts) @@ -40,11 +42,11 @@ func TestOneLargeFile(t *testing.T) { numFiles := 1 fileWriteOpts := map[string]string{ - engine.MaxDirDepthField: strconv.Itoa(1), - engine.MaxFileSizeField: strconv.Itoa(fileSize), - engine.MinFileSizeField: strconv.Itoa(fileSize), - engine.MaxNumFilesPerWriteField: strconv.Itoa(numFiles), - engine.MinNumFilesPerWriteField: strconv.Itoa(numFiles), + fiofilewriter.MaxDirDepthField: strconv.Itoa(1), + fiofilewriter.MaxFileSizeField: strconv.Itoa(fileSize), + fiofilewriter.MinFileSizeField: strconv.Itoa(fileSize), + fiofilewriter.MaxNumFilesPerWriteField: strconv.Itoa(numFiles), + fiofilewriter.MinNumFilesPerWriteField: strconv.Itoa(numFiles), } _, err := eng.ExecAction(engine.WriteRandomFilesActionKey, fileWriteOpts) @@ -65,12 +67,12 @@ func TestManySmallFilesAcrossDirecoryTree(t *testing.T) { actionRepeats := numFiles / filesPerWrite fileWriteOpts := map[string]string{ - engine.MaxDirDepthField: strconv.Itoa(15), - engine.MaxFileSizeField: strconv.Itoa(fileSize), - engine.MinFileSizeField: strconv.Itoa(fileSize), - engine.MaxNumFilesPerWriteField: strconv.Itoa(filesPerWrite), - engine.MinNumFilesPerWriteField: strconv.Itoa(filesPerWrite), - engine.ActionRepeaterField: strconv.Itoa(actionRepeats), + fiofilewriter.MaxDirDepthField: strconv.Itoa(15), + fiofilewriter.MaxFileSizeField: strconv.Itoa(fileSize), + fiofilewriter.MinFileSizeField: strconv.Itoa(fileSize), + fiofilewriter.MaxNumFilesPerWriteField: strconv.Itoa(filesPerWrite), + fiofilewriter.MinNumFilesPerWriteField: strconv.Itoa(filesPerWrite), + engine.ActionRepeaterField: strconv.Itoa(actionRepeats), } _, err := eng.ExecAction(engine.WriteRandomFilesActionKey, fileWriteOpts) @@ -95,16 +97,16 @@ func TestRandomizedSmall(t *testing.T) { string(engine.DeleteRandomSubdirectoryActionKey): strconv.Itoa(1), }, engine.WriteRandomFilesActionKey: map[string]string{ - engine.IOLimitPerWriteAction: fmt.Sprintf("%d", 512*1024*1024), - engine.MaxNumFilesPerWriteField: strconv.Itoa(100), - engine.MaxFileSizeField: strconv.Itoa(64 * 1024 * 1024), - engine.MaxDirDepthField: strconv.Itoa(3), + fiofilewriter.IOLimitPerWriteAction: fmt.Sprintf("%d", 512*1024*1024), + fiofilewriter.MaxNumFilesPerWriteField: strconv.Itoa(100), + fiofilewriter.MaxFileSizeField: strconv.Itoa(64 * 1024 * 1024), + fiofilewriter.MaxDirDepthField: strconv.Itoa(3), }, } for time.Since(st) <= *randomizedTestDur { err := eng.RandomAction(opts) - if errors.Is(err, engine.ErrNoOp) { + if errors.Is(err, robustness.ErrNoOp) { t.Log("Random action resulted in no-op") err = nil