Introduced a FileWriter interface in the robustness test and refactored accordingly. (#797)

2026-03-05 14:57:36 -05:00 · 2021-01-25 19:57:27 -08:00
parent c3e6aebfb6
commit 6fd4409bb3
10 changed files with 442 additions and 263 deletions
--- a/tests/robustness/engine/action.go
+++ b/tests/robustness/engine/action.go
@@ -12,13 +12,7 @@
 	"strings"
 	"time"

-	"github.com/kopia/kopia/tests/tools/fio"
-)
-
-// Errors associated with action-picking.
-var (
-	ErrNoActionPicked = errors.New("unable to pick an action with the action control options provided")
-	ErrInvalidOption  = errors.New("invalid option setting")
+	"github.com/kopia/kopia/tests/robustness"
 )

 // ExecAction executes the action denoted by the provided ActionKey.
@@ -42,12 +36,12 @@ func (e *Engine) ExecAction(actionKey ActionKey, opts map[string]string) (map[st
 	}

 	// Execute the action n times
-	err := ErrNoOp // Default to no-op error
+	err := robustness.ErrNoOp // Default to no-op error

 	// TODO: return more than the last output
 	var out map[string]string

-	n := getOptAsIntOrDefault(ActionRepeaterField, opts, defaultActionRepeats)
+	n := robustness.GetOptAsIntOrDefault(ActionRepeaterField, opts, defaultActionRepeats)
 	for i := 0; i < n; i++ {
 		out, err = action.f(e, opts, logEntry)
 		if err != nil {
@@ -57,7 +51,7 @@ func (e *Engine) ExecAction(actionKey ActionKey, opts map[string]string) (map[st

 	// If error was just a no-op, don't bother logging the action
 	switch {
-	case errors.Is(err, ErrNoOp):
+	case errors.Is(err, robustness.ErrNoOp):
 		e.RunStats.NoOpCount++
 		e.CumulativeStats.NoOpCount++

@@ -91,7 +85,7 @@ func (e *Engine) RandomAction(actionOpts ActionOpts) error {

 	actionName := pickActionWeighted(actionControlOpts, actions)
 	if string(actionName) == "" {
-		return ErrNoActionPicked
+		return robustness.ErrNoActionPicked
 	}

 	_, err := e.ExecAction(actionName, actionOpts[actionName])
@@ -112,15 +106,7 @@ func (e *Engine) checkErrRecovery(incomingErr error, actionOpts ActionOpts) (out
 	if errIsNotEnoughSpace(incomingErr) && ctrl[ThrowNoSpaceOnDeviceErrField] == "" {
 		// no space left on device
 		// Delete everything in the data directory
-		const hundredPcnt = 100
-
-		deleteDirActionKey := DeleteDirectoryContentsActionKey
-		deleteRootOpts := map[string]string{
-			MaxDirDepthField:             strconv.Itoa(0),
-			DeletePercentOfContentsField: strconv.Itoa(hundredPcnt),
-		}
-
-		_, outgoingErr = e.ExecAction(deleteDirActionKey, deleteRootOpts)
+		outgoingErr = e.FileWriter.DeleteEverything()
 		if outgoingErr != nil {
 			return outgoingErr
 		}
@@ -132,7 +118,7 @@ func (e *Engine) checkErrRecovery(incomingErr error, actionOpts ActionOpts) (out
 		restoreActionKey := RestoreIntoDataDirectoryActionKey
 		_, outgoingErr = e.ExecAction(restoreActionKey, actionOpts[restoreActionKey])

-		if errors.Is(outgoingErr, ErrNoOp) {
+		if errors.Is(outgoingErr, robustness.ErrNoOp) {
 			outgoingErr = nil
 		} else {
 			e.RunStats.DataRestoreCount++
@@ -186,13 +172,13 @@ type Action struct {
 var actions = map[ActionKey]Action{
 	SnapshotRootDirActionKey: {
 		f: func(e *Engine, opts map[string]string, l *LogEntry) (out map[string]string, err error) {
-			log.Printf("Creating snapshot of root directory %s", e.FileWriter.LocalDataDir)
+			log.Printf("Creating snapshot of root directory %s", e.FileWriter.DataDirectory())

 			ctx := context.TODO()
-			snapID, err := e.Checker.TakeSnapshot(ctx, e.FileWriter.LocalDataDir)
+			snapID, err := e.Checker.TakeSnapshot(ctx, e.FileWriter.DataDirectory())

 			setLogEntryCmdOpts(l, map[string]string{
-				"snap-dir": e.FileWriter.LocalDataDir,
+				"snap-dir": e.FileWriter.DataDirectory(),
 				"snapID":   snapID,
 			})

@@ -246,118 +232,26 @@ type Action struct {
 	},
 	WriteRandomFilesActionKey: {
 		f: func(e *Engine, opts map[string]string, l *LogEntry) (out map[string]string, err error) {
-			// Directory depth
-			maxDirDepth := getOptAsIntOrDefault(MaxDirDepthField, opts, defaultMaxDirDepth)
-			dirDepth := rand.Intn(maxDirDepth + 1)
+			out, err = e.FileWriter.WriteRandomFiles(opts)
+			setLogEntryCmdOpts(l, out)

-			// File size range
-			maxFileSizeB := getOptAsIntOrDefault(MaxFileSizeField, opts, defaultMaxFileSize)
-			minFileSizeB := getOptAsIntOrDefault(MinFileSizeField, opts, defaultMinFileSize)
-
-			// Number of files to write
-			maxNumFiles := getOptAsIntOrDefault(MaxNumFilesPerWriteField, opts, defaultMaxNumFilesPerWrite)
-			minNumFiles := getOptAsIntOrDefault(MinNumFilesPerWriteField, opts, defaultMinNumFilesPerWrite)
-
-			numFiles := rand.Intn(maxNumFiles-minNumFiles+1) + minNumFiles //nolint:gosec
-
-			// Dedup Percentage
-			maxDedupPcnt := getOptAsIntOrDefault(MaxDedupePercentField, opts, defaultMaxDedupePercent)
-			minDedupPcnt := getOptAsIntOrDefault(MinDedupePercentField, opts, defaultMinDedupePercent)
-
-			dedupStep := getOptAsIntOrDefault(DedupePercentStepField, opts, defaultDedupePercentStep)
-
-			dedupPcnt := dedupStep * (rand.Intn(maxDedupPcnt/dedupStep-minDedupPcnt/dedupStep+1) + minDedupPcnt/dedupStep) //nolint:gosec
-
-			blockSize := int64(defaultMinFileSize)
-
-			fioOpts := fio.Options{}.
-				WithFileSizeRange(int64(minFileSizeB), int64(maxFileSizeB)).
-				WithNumFiles(numFiles).
-				WithBlockSize(blockSize).
-				WithDedupePercentage(dedupPcnt).
-				WithNoFallocate()
-
-			ioLimit := getOptAsIntOrDefault(IOLimitPerWriteAction, opts, defaultIOLimitPerWriteAction)
-
-			if ioLimit > 0 {
-				freeSpaceLimitB := getOptAsIntOrDefault(FreeSpaceLimitField, opts, defaultFreeSpaceLimit)
-
-				freeSpaceB, err := getFreeSpaceB(e.FileWriter.LocalDataDir)
-				if err != nil {
-					return nil, err
-				}
-				log.Printf("Free Space %v B, limit %v B, ioLimit %v B\n", freeSpaceB, freeSpaceLimitB, ioLimit)
-
-				if int(freeSpaceB)-ioLimit < freeSpaceLimitB {
-					ioLimit = int(freeSpaceB) - freeSpaceLimitB
-					log.Printf("Cutting down I/O limit for space %v", ioLimit)
-					if ioLimit <= 0 {
-						return nil, ErrCannotPerformIO
-					}
-				}
-
-				fioOpts = fioOpts.WithIOLimit(int64(ioLimit))
-			}
-
-			relBasePath := "."
-
-			log.Printf("Writing files at depth %v (fileSize: %v-%v, numFiles: %v, blockSize: %v, dedupPcnt: %v, ioLimit: %v)\n", dirDepth, minFileSizeB, maxFileSizeB, numFiles, blockSize, dedupPcnt, ioLimit)
-
-			setLogEntryCmdOpts(l, map[string]string{
-				"dirDepth":    strconv.Itoa(dirDepth),
-				"relBasePath": relBasePath,
-			})
-
-			for k, v := range fioOpts {
-				l.CmdOpts[k] = v
-			}
-
-			return nil, e.FileWriter.WriteFilesAtDepthRandomBranch(relBasePath, dirDepth, fioOpts)
+			return
 		},
 	},
 	DeleteRandomSubdirectoryActionKey: {
 		f: func(e *Engine, opts map[string]string, l *LogEntry) (out map[string]string, err error) {
-			maxDirDepth := getOptAsIntOrDefault(MaxDirDepthField, opts, defaultMaxDirDepth)
-			if maxDirDepth <= 0 {
-				return nil, ErrInvalidOption
-			}
-			dirDepth := rand.Intn(maxDirDepth) + 1 //nolint:gosec
+			out, err = e.FileWriter.DeleteRandomSubdirectory(opts)
+			setLogEntryCmdOpts(l, out)

-			log.Printf("Deleting directory at depth %v\n", dirDepth)
-
-			setLogEntryCmdOpts(l, map[string]string{"dirDepth": strconv.Itoa(dirDepth)})
-
-			err = e.FileWriter.DeleteDirAtDepth("", dirDepth)
-			if errors.Is(err, fio.ErrNoDirFound) {
-				log.Print(err)
-				return nil, ErrNoOp
-			}
-
-			return nil, err
+			return
 		},
 	},
 	DeleteDirectoryContentsActionKey: {
 		f: func(e *Engine, opts map[string]string, l *LogEntry) (out map[string]string, err error) {
-			maxDirDepth := getOptAsIntOrDefault(MaxDirDepthField, opts, defaultMaxDirDepth)
-			dirDepth := rand.Intn(maxDirDepth + 1) //nolint:gosec
+			out, err = e.FileWriter.DeleteDirectoryContents(opts)
+			setLogEntryCmdOpts(l, out)

-			pcnt := getOptAsIntOrDefault(DeletePercentOfContentsField, opts, defaultDeletePercentOfContents)
-
-			log.Printf("Deleting %d%% of directory contents at depth %v\n", pcnt, dirDepth)
-
-			setLogEntryCmdOpts(l, map[string]string{
-				"dirDepth": strconv.Itoa(dirDepth),
-				"percent":  strconv.Itoa(pcnt),
-			})
-
-			const pcntConv = 100
-			err = e.FileWriter.DeleteContentsAtDepth("", dirDepth, float32(pcnt)/pcntConv)
-			if errors.Is(err, fio.ErrNoDirFound) {
-				log.Print(err)
-				return nil, ErrNoOp
-			}
-
-			return nil, err
+			return
 		},
 	},
 	RestoreIntoDataDirectoryActionKey: {
@@ -372,7 +266,7 @@ type Action struct {
 			setLogEntryCmdOpts(l, map[string]string{"snapID": snapID})

 			b := &bytes.Buffer{}
-			err = e.Checker.RestoreSnapshotToPath(context.Background(), snapID, e.FileWriter.LocalDataDir, b)
+			err = e.Checker.RestoreSnapshotToPath(context.Background(), snapID, e.FileWriter.DataDirectory(), b)
 			if err != nil {
 				log.Print(b.String())
 				return nil, err
@@ -385,55 +279,16 @@ type Action struct {

 // Action constants.
 const (
-	defaultMaxDirDepth             = 20
-	defaultMaxFileSize             = 1 * 1024 * 1024 * 1024 // 1GB
-	defaultMinFileSize             = 4096
-	defaultMaxNumFilesPerWrite     = 10000
-	defaultMinNumFilesPerWrite     = 1
-	defaultIOLimitPerWriteAction   = 0                 // A zero value does not impose any limit on IO
-	defaultFreeSpaceLimit          = 100 * 1024 * 1024 // 100 MB
-	defaultMaxDedupePercent        = 100
-	defaultMinDedupePercent        = 0
-	defaultDedupePercentStep       = 25
-	defaultDeletePercentOfContents = 20
-	defaultActionRepeats           = 1
+	defaultActionRepeats = 1
 )

 // Option field names.
 const (
-	MaxDirDepthField             = "max-dir-depth"
-	MaxFileSizeField             = "max-file-size"
-	MinFileSizeField             = "min-file-size"
-	MaxNumFilesPerWriteField     = "max-num-files-per-write"
-	MinNumFilesPerWriteField     = "min-num-files-per-write"
-	IOLimitPerWriteAction        = "io-limit-per-write"
-	FreeSpaceLimitField          = "free-space-limit"
-	MaxDedupePercentField        = "max-dedupe-percent"
-	MinDedupePercentField        = "min-dedupe-percent"
-	DedupePercentStepField       = "dedupe-percent"
 	ActionRepeaterField          = "repeat-action"
 	ThrowNoSpaceOnDeviceErrField = "throw-no-space-error"
-	DeletePercentOfContentsField = "delete-contents-percent"
 	SnapshotIDField              = "snapshot-ID"
 )

-func getOptAsIntOrDefault(key string, opts map[string]string, def int) int {
-	if opts == nil {
-		return def
-	}
-
-	if opts[key] == "" {
-		return def
-	}
-
-	retInt, err := strconv.Atoi(opts[key])
-	if err != nil {
-		return def
-	}
-
-	return retInt
-}
-
 func defaultActionControls() map[string]string {
 	ret := make(map[string]string, len(actions))

@@ -456,7 +311,7 @@ func pickActionWeighted(actionControlOpts map[string]string, actionList map[Acti
 	sum := 0

 	for actionName := range actionList {
-		weight := getOptAsIntOrDefault(string(actionName), actionControlOpts, 0)
+		weight := robustness.GetOptAsIntOrDefault(string(actionName), actionControlOpts, 0)
 		if weight == 0 {
 			continue
 		}
@@ -471,7 +326,7 @@ func pickActionWeighted(actionControlOpts map[string]string, actionList map[Acti
 }

 func errIsNotEnoughSpace(err error) bool {
-	return errors.Is(err, ErrCannotPerformIO) || strings.Contains(err.Error(), noSpaceOnDeviceMatchStr)
+	return errors.Is(err, robustness.ErrCannotPerformIO) || strings.Contains(err.Error(), noSpaceOnDeviceMatchStr)
 }

 func (e *Engine) getSnapIDOptOrRandLive(opts map[string]string) (snapID string, err error) {
@@ -482,7 +337,7 @@ func (e *Engine) getSnapIDOptOrRandLive(opts map[string]string) (snapID string,

 	snapIDList := e.Checker.GetLiveSnapIDs()
 	if len(snapIDList) == 0 {
-		return "", ErrNoOp
+		return "", robustness.ErrNoOp
 	}

 	return snapIDList[rand.Intn(len(snapIDList))], nil //nolint:gosec
--- a/tests/robustness/engine/engine.go
+++ b/tests/robustness/engine/engine.go
@@ -19,8 +19,8 @@

 	"github.com/kopia/kopia/tests/robustness"
 	"github.com/kopia/kopia/tests/robustness/checker"
+	"github.com/kopia/kopia/tests/robustness/fiofilewriter"
 	"github.com/kopia/kopia/tests/robustness/snapmeta"
-	"github.com/kopia/kopia/tests/tools/fio"
 	"github.com/kopia/kopia/tests/tools/fswalker"
 	"github.com/kopia/kopia/tests/tools/kopiarunner"
 )
@@ -39,11 +39,6 @@
 )

 var (
-	// ErrNoOp is thrown when an action could not do anything useful.
-	ErrNoOp = fmt.Errorf("no-op")
-	// ErrCannotPerformIO is returned if the engine determines there is not enough space
-	// to write files.
-	ErrCannotPerformIO = fmt.Errorf("cannot perform i/o")
 	// ErrS3BucketNameEnvUnset is the error returned when the S3BucketNameEnvKey environment variable is not set.
 	ErrS3BucketNameEnvUnset = fmt.Errorf("environment variable required: %v", S3BucketNameEnvKey)
 	noSpaceOnDeviceMatchStr = "no space left on device"
@@ -51,7 +46,7 @@

 // Engine is the outer level testing framework for robustness testing.
 type Engine struct {
-	FileWriter      *fio.Runner
+	FileWriter      robustness.FileWriter
 	TestRepo        robustness.Snapshotter
 	MetaStore       robustness.Persister
 	Checker         *checker.Checker
@@ -85,8 +80,8 @@ func NewEngine(workingDir string) (*Engine, error) {
 		},
 	}

-	// Fill the file writer
-	e.FileWriter, err = fio.NewRunner()
+	// Create an FIO file writer
+	e.FileWriter, err = fiofilewriter.New()
 	if err != nil {
 		e.CleanComponents()
 		return nil, err
--- a/tests/robustness/engine/engine_test.go
+++ b/tests/robustness/engine/engine_test.go
@@ -21,6 +21,8 @@
 	"github.com/minio/minio-go/v7"
 	"github.com/minio/minio-go/v7/pkg/credentials"

+	"github.com/kopia/kopia/tests/robustness"
+	"github.com/kopia/kopia/tests/robustness/fiofilewriter"
 	"github.com/kopia/kopia/tests/robustness/snapmeta"
 	"github.com/kopia/kopia/tests/testenv"
 	"github.com/kopia/kopia/tests/tools/fio"
@@ -65,13 +67,13 @@ func TestEngineWritefilesBasicFS(t *testing.T) {
 	numFiles := 10

 	fioOpts := fio.Options{}.WithFileSize(fileSize).WithNumFiles(numFiles)
-
-	err = eng.FileWriter.WriteFiles("", fioOpts)
+	fioRunner := engineFioRunner(t, eng)
+	err = fioRunner.WriteFiles("", fioOpts)
 	testenv.AssertNoError(t, err)

 	snapIDs := eng.Checker.GetSnapIDs()

-	snapID, err := eng.Checker.TakeSnapshot(ctx, eng.FileWriter.LocalDataDir)
+	snapID, err := eng.Checker.TakeSnapshot(ctx, fioRunner.LocalDataDir)
 	testenv.AssertNoError(t, err)

 	err = eng.Checker.RestoreSnapshot(ctx, snapID, os.Stdout)
@@ -175,13 +177,13 @@ func TestWriteFilesBasicS3(t *testing.T) {
 	numFiles := 10

 	fioOpts := fio.Options{}.WithFileSize(fileSize).WithNumFiles(numFiles)
-
-	err = eng.FileWriter.WriteFiles("", fioOpts)
+	fioRunner := engineFioRunner(t, eng)
+	err = fioRunner.WriteFiles("", fioOpts)
 	testenv.AssertNoError(t, err)

 	snapIDs := eng.Checker.GetLiveSnapIDs()

-	snapID, err := eng.Checker.TakeSnapshot(ctx, eng.FileWriter.LocalDataDir)
+	snapID, err := eng.Checker.TakeSnapshot(ctx, fioRunner.LocalDataDir)
 	testenv.AssertNoError(t, err)

 	err = eng.Checker.RestoreSnapshot(ctx, snapID, os.Stdout)
@@ -217,11 +219,11 @@ func TestDeleteSnapshotS3(t *testing.T) {
 	numFiles := 10

 	fioOpts := fio.Options{}.WithFileSize(fileSize).WithNumFiles(numFiles)
-
-	err = eng.FileWriter.WriteFiles("", fioOpts)
+	fioRunner := engineFioRunner(t, eng)
+	err = fioRunner.WriteFiles("", fioOpts)
 	testenv.AssertNoError(t, err)

-	snapID, err := eng.Checker.TakeSnapshot(ctx, eng.FileWriter.LocalDataDir)
+	snapID, err := eng.Checker.TakeSnapshot(ctx, fioRunner.LocalDataDir)
 	testenv.AssertNoError(t, err)

 	err = eng.Checker.RestoreSnapshot(ctx, snapID, os.Stdout)
@@ -261,11 +263,12 @@ func TestSnapshotVerificationFail(t *testing.T) {
 	numFiles := 10
 	fioOpt := fio.Options{}.WithFileSize(fileSize).WithNumFiles(numFiles)

-	err = eng.FileWriter.WriteFiles("", fioOpt)
+	fioRunner := engineFioRunner(t, eng)
+	err = fioRunner.WriteFiles("", fioOpt)
 	testenv.AssertNoError(t, err)

 	// Take a first snapshot
-	snapID1, err := eng.Checker.TakeSnapshot(ctx, eng.FileWriter.LocalDataDir)
+	snapID1, err := eng.Checker.TakeSnapshot(ctx, fioRunner.LocalDataDir)
 	testenv.AssertNoError(t, err)

 	// Get the metadata collected on that snapshot
@@ -273,11 +276,11 @@ func TestSnapshotVerificationFail(t *testing.T) {
 	testenv.AssertNoError(t, err)

 	// Do additional writes, writing 1 extra byte than before
-	err = eng.FileWriter.WriteFiles("", fioOpt.WithFileSize(fileSize+1))
+	err = fioRunner.WriteFiles("", fioOpt.WithFileSize(fileSize+1))
 	testenv.AssertNoError(t, err)

 	// Take a second snapshot
-	snapID2, err := eng.Checker.TakeSnapshot(ctx, eng.FileWriter.LocalDataDir)
+	snapID2, err := eng.Checker.TakeSnapshot(ctx, fioRunner.LocalDataDir)
 	testenv.AssertNoError(t, err)

 	// Get the second snapshot's metadata
@@ -329,12 +332,12 @@ func TestDataPersistency(t *testing.T) {
 	numFiles := 10

 	fioOpt := fio.Options{}.WithFileSize(fileSize).WithNumFiles(numFiles)
-
-	err = eng.FileWriter.WriteFiles("", fioOpt)
+	fioRunner := engineFioRunner(t, eng)
+	err = fioRunner.WriteFiles("", fioOpt)
 	testenv.AssertNoError(t, err)

 	// Take a snapshot
-	snapID, err := eng.Checker.TakeSnapshot(ctx, eng.FileWriter.LocalDataDir)
+	snapID, err := eng.Checker.TakeSnapshot(ctx, fioRunner.LocalDataDir)
 	testenv.AssertNoError(t, err)

 	// Get the walk data associated with the snapshot that was taken
@@ -358,12 +361,13 @@ func TestDataPersistency(t *testing.T) {
 	err = eng2.InitFilesystem(ctx, dataRepoPath, metadataRepoPath)
 	testenv.AssertNoError(t, err)

-	err = eng2.Checker.RestoreSnapshotToPath(ctx, snapID, eng2.FileWriter.LocalDataDir, os.Stdout)
+	fioRunner2 := engineFioRunner(t, eng2)
+	err = eng2.Checker.RestoreSnapshotToPath(ctx, snapID, fioRunner2.LocalDataDir, os.Stdout)
 	testenv.AssertNoError(t, err)

 	// Compare the data directory of the second engine with the fingerprint
 	// of the snapshot taken earlier. They should match.
-	err = fswalker.NewWalkCompare().Compare(ctx, eng2.FileWriter.LocalDataDir, dataDirWalk.ValidationData, os.Stdout)
+	err = fswalker.NewWalkCompare().Compare(ctx, fioRunner2.LocalDataDir, dataDirWalk.ValidationData, os.Stdout)
 	testenv.AssertNoError(t, err)
 }

@@ -482,22 +486,22 @@ func TestActionsFilesystem(t *testing.T) {

 	actionOpts := ActionOpts{
 		WriteRandomFilesActionKey: map[string]string{
-			MaxDirDepthField:         "20",
-			MaxFileSizeField:         strconv.Itoa(10 * 1024 * 1024),
-			MinFileSizeField:         strconv.Itoa(10 * 1024 * 1024),
-			MaxNumFilesPerWriteField: "10",
-			MinNumFilesPerWriteField: "10",
-			MaxDedupePercentField:    "100",
-			MinDedupePercentField:    "100",
-			DedupePercentStepField:   "1",
-			IOLimitPerWriteAction:    "0",
+			fiofilewriter.MaxDirDepthField:         "20",
+			fiofilewriter.MaxFileSizeField:         strconv.Itoa(10 * 1024 * 1024),
+			fiofilewriter.MinFileSizeField:         strconv.Itoa(10 * 1024 * 1024),
+			fiofilewriter.MaxNumFilesPerWriteField: "10",
+			fiofilewriter.MinNumFilesPerWriteField: "10",
+			fiofilewriter.MaxDedupePercentField:    "100",
+			fiofilewriter.MinDedupePercentField:    "100",
+			fiofilewriter.DedupePercentStepField:   "1",
+			fiofilewriter.IOLimitPerWriteAction:    "0",
 		},
 	}

 	numActions := 10
 	for loop := 0; loop < numActions; loop++ {
 		err := eng.RandomAction(actionOpts)
-		if !(err == nil || errors.Is(err, ErrNoOp)) {
+		if !(err == nil || errors.Is(err, robustness.ErrNoOp)) {
 			t.Error("Hit error", err)
 		}
 	}
@@ -525,22 +529,22 @@ func TestActionsS3(t *testing.T) {

 	actionOpts := ActionOpts{
 		WriteRandomFilesActionKey: map[string]string{
-			MaxDirDepthField:         "20",
-			MaxFileSizeField:         strconv.Itoa(10 * 1024 * 1024),
-			MinFileSizeField:         strconv.Itoa(10 * 1024 * 1024),
-			MaxNumFilesPerWriteField: "10",
-			MinNumFilesPerWriteField: "10",
-			MaxDedupePercentField:    "100",
-			MinDedupePercentField:    "100",
-			DedupePercentStepField:   "1",
-			IOLimitPerWriteAction:    "0",
+			fiofilewriter.MaxDirDepthField:         "20",
+			fiofilewriter.MaxFileSizeField:         strconv.Itoa(10 * 1024 * 1024),
+			fiofilewriter.MinFileSizeField:         strconv.Itoa(10 * 1024 * 1024),
+			fiofilewriter.MaxNumFilesPerWriteField: "10",
+			fiofilewriter.MinNumFilesPerWriteField: "10",
+			fiofilewriter.MaxDedupePercentField:    "100",
+			fiofilewriter.MinDedupePercentField:    "100",
+			fiofilewriter.DedupePercentStepField:   "1",
+			fiofilewriter.IOLimitPerWriteAction:    "0",
 		},
 	}

 	numActions := 10
 	for loop := 0; loop < numActions; loop++ {
 		err := eng.RandomAction(actionOpts)
-		if !(err == nil || errors.Is(err, ErrNoOp)) {
+		if !(err == nil || errors.Is(err, robustness.ErrNoOp)) {
 			t.Error("Hit error", err)
 		}
 	}
@@ -581,12 +585,12 @@ func TestIOLimitPerWriteAction(t *testing.T) {
 			string(DeleteRandomSubdirectoryActionKey): strconv.Itoa(0),
 		},
 		WriteRandomFilesActionKey: map[string]string{
-			MaxDirDepthField:         "2",
-			MaxFileSizeField:         strconv.Itoa(10 * 1024 * 1024),
-			MinFileSizeField:         strconv.Itoa(10 * 1024 * 1024),
-			MaxNumFilesPerWriteField: "100",
-			MinNumFilesPerWriteField: "100",
-			IOLimitPerWriteAction:    strconv.Itoa(1 * 1024 * 1024),
+			fiofilewriter.MaxDirDepthField:         "2",
+			fiofilewriter.MaxFileSizeField:         strconv.Itoa(10 * 1024 * 1024),
+			fiofilewriter.MinFileSizeField:         strconv.Itoa(10 * 1024 * 1024),
+			fiofilewriter.MaxNumFilesPerWriteField: "100",
+			fiofilewriter.MinNumFilesPerWriteField: "100",
+			fiofilewriter.IOLimitPerWriteAction:    strconv.Itoa(1 * 1024 * 1024),
 		},
 	}

@@ -737,3 +741,15 @@ func TestLogsPersist(t *testing.T) {
 		t.Errorf("Logs do not match\n%v\n%v", got, want)
 	}
 }
+
+// engineFioRunner extracts the fio.Runner used by the engine.
+func engineFioRunner(t *testing.T, eng *Engine) *fio.Runner {
+	t.Helper()
+
+	fiofw, ok := eng.FileWriter.(*fiofilewriter.FileWriter)
+	if !ok {
+		t.Fatal("engine does not have a fiofilewriter.FileWriter")
+	}
+
+	return fiofw.Runner
+}
--- a/tests/robustness/engine/sys.go
+++ b/tests/robustness/engine/sys.go
@@ -1,19 +0,0 @@
-// +build darwin,amd64 linux,amd64
-
-package engine
-
-import (
-	"syscall"
-)
-
-func getFreeSpaceB(path string) (uint64, error) {
-	var stat syscall.Statfs_t
-
-	err := syscall.Statfs(path, &stat)
-	if err != nil {
-		return 0, err
-	}
-
-	// Available blocks * size per block = available space in bytes
-	return stat.Bavail * uint64(stat.Bsize), nil
-}
--- a/tests/robustness/errors.go
+++ b/tests/robustness/errors.go
@@ -0,0 +1,21 @@
+package robustness
+
+import (
+	"errors"
+	"fmt"
+)
+
+var (
+	// ErrNoOp is thrown when an action could not do anything useful.
+	ErrNoOp = fmt.Errorf("no-op")
+
+	// ErrCannotPerformIO is returned if the engine determines there is not enough space
+	// to write files.
+	ErrCannotPerformIO = fmt.Errorf("cannot perform i/o")
+
+	// ErrNoActionPicked is returned if a random action could not be selected.
+	ErrNoActionPicked = errors.New("unable to pick an action with the action control options provided")
+
+	// ErrInvalidOption is returned if an option value is invalid or missing.
+	ErrInvalidOption = errors.New("invalid option setting")
+)
--- a/tests/robustness/filewriter.go
+++ b/tests/robustness/filewriter.go
@@ -0,0 +1,31 @@
+package robustness
+
+// FileWriter is an interface used for filesystem related actions.
+type FileWriter interface {
+	// Cleanup is called prior to termination.
+	// TBD: Will be removed when initialization refactored.
+	Cleanup()
+
+	// DataDirectory returns the absolute path of the data directory configured.
+	DataDirectory() string
+
+	// DeleteDirectoryContents deletes some of the content of a random directory,
+	// based on its input option values (none of which are required).
+	// The method returns the effective option values used and the error if any.
+	// ErrNoOp is returned if no directory is found.
+	DeleteDirectoryContents(opts map[string]string) (map[string]string, error)
+
+	// DeleteEverything deletes all content.
+	DeleteEverything() error
+
+	// DeleteRandomSubdirectory deletes a random directory, based
+	// on its input option values (none of which are required).
+	// The method returns the effective option values used and the error if any.
+	// ErrNoOp is returned if no directory is found.
+	DeleteRandomSubdirectory(opts map[string]string) (map[string]string, error)
+
+	// WriteRandomFiles writes a number of files in a random directory, based
+	// on its input option values (none of which are required).
+	// The method returns the effective option values used and the error if any.
+	WriteRandomFiles(opts map[string]string) (map[string]string, error)
+}
--- a/tests/robustness/fiofilewriter/fio_filewriter.go
+++ b/tests/robustness/fiofilewriter/fio_filewriter.go
@@ -0,0 +1,255 @@
+// +build darwin,amd64 linux,amd64
+
+// Package fiofilewriter provides a FileWriter based on FIO.
+package fiofilewriter
+
+import (
+	"errors"
+	"log"
+	"math/rand"
+	"strconv"
+	"syscall"
+
+	"github.com/kopia/kopia/tests/robustness"
+	"github.com/kopia/kopia/tests/tools/fio"
+)
+
+// Option field names.
+const (
+	DedupePercentStepField       = "dedupe-percent"
+	DeletePercentOfContentsField = "delete-contents-percent"
+	FreeSpaceLimitField          = "free-space-limit"
+	IOLimitPerWriteAction        = "io-limit-per-write"
+	MaxDedupePercentField        = "max-dedupe-percent"
+	MaxDirDepthField             = "max-dir-depth"
+	MaxFileSizeField             = "max-file-size"
+	MaxNumFilesPerWriteField     = "max-num-files-per-write"
+	MinDedupePercentField        = "min-dedupe-percent"
+	MinFileSizeField             = "min-file-size"
+	MinNumFilesPerWriteField     = "min-num-files-per-write"
+)
+
+// Option defaults.
+const (
+	defaultDedupePercentStep       = 25
+	defaultDeletePercentOfContents = 20
+	defaultFreeSpaceLimit          = 100 * 1024 * 1024 // 100 MB
+	defaultIOLimitPerWriteAction   = 0                 // A zero value does not impose any limit on IO
+	defaultMaxDedupePercent        = 100
+	defaultMaxDirDepth             = 20
+	defaultMaxFileSize             = 1 * 1024 * 1024 * 1024 // 1GB
+	defaultMaxNumFilesPerWrite     = 10000
+	defaultMinDedupePercent        = 0
+	defaultMinFileSize             = 4096
+	defaultMinNumFilesPerWrite     = 1
+)
+
+// New returns a FileWriter based on FIO.
+// See tests/tools/fio for configuration details.
+func New() (robustness.FileWriter, error) {
+	runner, err := fio.NewRunner()
+	if err != nil {
+		return nil, err
+	}
+
+	return &FileWriter{Runner: runner}, nil
+}
+
+// FileWriter implements a FileWriter over tools/fio.Runner.
+type FileWriter struct {
+	Runner *fio.Runner
+}
+
+// DataDirectory returns the data directory configured.
+// See tests/tools/fio for details.
+func (fw *FileWriter) DataDirectory() string {
+	return fw.Runner.LocalDataDir
+}
+
+// WriteRandomFiles writes a number of files at some filesystem depth, based
+// on its input options.
+//
+//  - MaxDirDepthField
+//  - MaxFileSizeField
+//  - MinFileSizeField
+//  - MaxNumFilesPerWriteField
+//  - MinNumFilesPerWriteField
+//  - MaxDedupePercentField
+//  - MinDedupePercentField
+//  - DedupePercentStepField
+//
+// Default values are used for missing options. The method
+// returns the effective options used along with the selected depth
+// and the error if any.
+func (fw *FileWriter) WriteRandomFiles(opts map[string]string) (map[string]string, error) {
+	// Directory depth
+	maxDirDepth := robustness.GetOptAsIntOrDefault(MaxDirDepthField, opts, defaultMaxDirDepth)
+	dirDepth := rand.Intn(maxDirDepth + 1)
+
+	// File size range
+	maxFileSizeB := robustness.GetOptAsIntOrDefault(MaxFileSizeField, opts, defaultMaxFileSize)
+	minFileSizeB := robustness.GetOptAsIntOrDefault(MinFileSizeField, opts, defaultMinFileSize)
+
+	// Number of files to write
+	maxNumFiles := robustness.GetOptAsIntOrDefault(MaxNumFilesPerWriteField, opts, defaultMaxNumFilesPerWrite)
+	minNumFiles := robustness.GetOptAsIntOrDefault(MinNumFilesPerWriteField, opts, defaultMinNumFilesPerWrite)
+
+	numFiles := rand.Intn(maxNumFiles-minNumFiles+1) + minNumFiles //nolint:gosec
+
+	// Dedup Percentage
+	maxDedupPcnt := robustness.GetOptAsIntOrDefault(MaxDedupePercentField, opts, defaultMaxDedupePercent)
+	minDedupPcnt := robustness.GetOptAsIntOrDefault(MinDedupePercentField, opts, defaultMinDedupePercent)
+
+	dedupStep := robustness.GetOptAsIntOrDefault(DedupePercentStepField, opts, defaultDedupePercentStep)
+
+	dedupPcnt := dedupStep * (rand.Intn(maxDedupPcnt/dedupStep-minDedupPcnt/dedupStep+1) + minDedupPcnt/dedupStep) //nolint:gosec
+
+	blockSize := int64(defaultMinFileSize)
+
+	fioOpts := fio.Options{}.
+		WithFileSizeRange(int64(minFileSizeB), int64(maxFileSizeB)).
+		WithNumFiles(numFiles).
+		WithBlockSize(blockSize).
+		WithDedupePercentage(dedupPcnt).
+		WithNoFallocate()
+
+	ioLimit := robustness.GetOptAsIntOrDefault(IOLimitPerWriteAction, opts, defaultIOLimitPerWriteAction)
+
+	if ioLimit > 0 {
+		freeSpaceLimitB := robustness.GetOptAsIntOrDefault(FreeSpaceLimitField, opts, defaultFreeSpaceLimit)
+
+		freeSpaceB, err := getFreeSpaceB(fw.Runner.LocalDataDir)
+		if err != nil {
+			return nil, err
+		}
+
+		log.Printf("Free Space %v B, limit %v B, ioLimit %v B\n", freeSpaceB, freeSpaceLimitB, ioLimit)
+
+		if int(freeSpaceB)-ioLimit < freeSpaceLimitB {
+			ioLimit = int(freeSpaceB) - freeSpaceLimitB
+
+			log.Printf("Cutting down I/O limit for space %v", ioLimit)
+
+			if ioLimit <= 0 {
+				return nil, robustness.ErrCannotPerformIO
+			}
+		}
+
+		fioOpts = fioOpts.WithIOLimit(int64(ioLimit))
+	}
+
+	relBasePath := "."
+
+	log.Printf("Writing files at depth %v (fileSize: %v-%v, numFiles: %v, blockSize: %v, dedupPcnt: %v, ioLimit: %v)\n", dirDepth, minFileSizeB, maxFileSizeB, numFiles, blockSize, dedupPcnt, ioLimit)
+
+	retOpts := make(map[string]string, len(opts))
+	for k, v := range opts {
+		retOpts[k] = v
+	}
+
+	for k, v := range fioOpts {
+		retOpts[k] = v
+	}
+
+	retOpts["dirDepth"] = strconv.Itoa(dirDepth)
+	retOpts["relBasePath"] = relBasePath
+
+	return retOpts, fw.Runner.WriteFilesAtDepthRandomBranch(relBasePath, dirDepth, fioOpts)
+}
+
+// DeleteRandomSubdirectory deletes a random directory up to a specified depth,
+// based on its input options:
+//
+//  - MaxDirDepthField
+//
+// Default values are used for missing options. The method
+// returns the effective options used along with the selected depth
+// and the error if any. ErrNoOp is returned if no directory is found.
+func (fw *FileWriter) DeleteRandomSubdirectory(opts map[string]string) (map[string]string, error) {
+	maxDirDepth := robustness.GetOptAsIntOrDefault(MaxDirDepthField, opts, defaultMaxDirDepth)
+	if maxDirDepth <= 0 {
+		return nil, robustness.ErrInvalidOption
+	}
+
+	dirDepth := rand.Intn(maxDirDepth) + 1 //nolint:gosec
+
+	log.Printf("Deleting directory at depth %v\n", dirDepth)
+
+	retOpts := make(map[string]string, len(opts))
+	for k, v := range opts {
+		retOpts[k] = v
+	}
+
+	retOpts["dirDepth"] = strconv.Itoa(dirDepth)
+
+	err := fw.Runner.DeleteDirAtDepth("", dirDepth)
+	if errors.Is(err, fio.ErrNoDirFound) {
+		log.Print(err)
+		err = robustness.ErrNoOp
+	}
+
+	return retOpts, err
+}
+
+// DeleteDirectoryContents deletes some of the contents of random directory up to a specified depth,
+// based on its input options:
+//
+//  - MaxDirDepthField
+//  - DeletePercentOfContentsField
+//
+// Default values are used for missing options. The method
+// returns the effective options used along with the selected depth
+// and the error if any. ErrNoOp is returned if no directory is found.
+func (fw *FileWriter) DeleteDirectoryContents(opts map[string]string) (map[string]string, error) {
+	maxDirDepth := robustness.GetOptAsIntOrDefault(MaxDirDepthField, opts, defaultMaxDirDepth)
+	dirDepth := rand.Intn(maxDirDepth + 1) //nolint:gosec
+
+	pcnt := robustness.GetOptAsIntOrDefault(DeletePercentOfContentsField, opts, defaultDeletePercentOfContents)
+
+	log.Printf("Deleting %d%% of directory contents at depth %v\n", pcnt, dirDepth)
+
+	retOpts := make(map[string]string, len(opts))
+	for k, v := range opts {
+		retOpts[k] = v
+	}
+
+	retOpts["dirDepth"] = strconv.Itoa(dirDepth)
+	retOpts["percent"] = strconv.Itoa(pcnt)
+
+	const pcntConv = 100
+
+	err := fw.Runner.DeleteContentsAtDepth("", dirDepth, float32(pcnt)/pcntConv)
+	if errors.Is(err, fio.ErrNoDirFound) {
+		log.Print(err)
+		err = robustness.ErrNoOp
+	}
+
+	return retOpts, err
+}
+
+// DeleteEverything deletes all content.
+func (fw *FileWriter) DeleteEverything() error {
+	_, err := fw.DeleteDirectoryContents(map[string]string{
+		MaxDirDepthField:             strconv.Itoa(0),
+		DeletePercentOfContentsField: strconv.Itoa(100),
+	})
+
+	return err
+}
+
+// Cleanup is part of FileWriter.
+func (fw *FileWriter) Cleanup() {
+	fw.Runner.Cleanup()
+}
+
+func getFreeSpaceB(path string) (uint64, error) {
+	var stat syscall.Statfs_t
+
+	err := syscall.Statfs(path, &stat)
+	if err != nil {
+		return 0, err
+	}
+
+	// Available blocks * size per block = available space in bytes
+	return stat.Bavail * uint64(stat.Bsize), nil
+}
--- a/tests/robustness/options.go
+++ b/tests/robustness/options.go
@@ -0,0 +1,22 @@
+package robustness
+
+import "strconv"
+
+// GetOptAsIntOrDefault extracts an integer value from a configuration map
+// if present, or else returns a default.
+func GetOptAsIntOrDefault(key string, opts map[string]string, def int) int {
+	if opts == nil {
+		return def
+	}
+
+	if opts[key] == "" {
+		return def
+	}
+
+	retInt, err := strconv.Atoi(opts[key])
+	if err != nil {
+		return def
+	}
+
+	return retInt
+}
--- a/tests/robustness/robustness_test/main_test.go
+++ b/tests/robustness/robustness_test/main_test.go
@@ -12,6 +12,7 @@
 	"testing"
 	"time"

+	"github.com/kopia/kopia/tests/robustness"
 	"github.com/kopia/kopia/tests/robustness/engine"
 	"github.com/kopia/kopia/tests/tools/fio"
 	"github.com/kopia/kopia/tests/tools/kopiarunner"
@@ -65,7 +66,7 @@ func TestMain(m *testing.M) {

 	// Restore a random snapshot into the data directory
 	_, err = eng.ExecAction(engine.RestoreIntoDataDirectoryActionKey, nil)
-	if err != nil && !errors.Is(err, engine.ErrNoOp) {
+	if err != nil && !errors.Is(err, robustness.ErrNoOp) {
 		eng.Cleanup()
 		log.Fatalln("error restoring into the data directory:", err)
 	}
--- a/tests/robustness/robustness_test/robustness_test.go
+++ b/tests/robustness/robustness_test/robustness_test.go
@@ -9,7 +9,9 @@
 	"testing"
 	"time"

+	"github.com/kopia/kopia/tests/robustness"
 	"github.com/kopia/kopia/tests/robustness/engine"
+	"github.com/kopia/kopia/tests/robustness/fiofilewriter"
 	"github.com/kopia/kopia/tests/testenv"
 )

@@ -18,11 +20,11 @@ func TestManySmallFiles(t *testing.T) {
 	numFiles := 10000

 	fileWriteOpts := map[string]string{
-		engine.MaxDirDepthField:         strconv.Itoa(1),
-		engine.MaxFileSizeField:         strconv.Itoa(fileSize),
-		engine.MinFileSizeField:         strconv.Itoa(fileSize),
-		engine.MaxNumFilesPerWriteField: strconv.Itoa(numFiles),
-		engine.MinNumFilesPerWriteField: strconv.Itoa(numFiles),
+		fiofilewriter.MaxDirDepthField:         strconv.Itoa(1),
+		fiofilewriter.MaxFileSizeField:         strconv.Itoa(fileSize),
+		fiofilewriter.MinFileSizeField:         strconv.Itoa(fileSize),
+		fiofilewriter.MaxNumFilesPerWriteField: strconv.Itoa(numFiles),
+		fiofilewriter.MinNumFilesPerWriteField: strconv.Itoa(numFiles),
 	}

 	_, err := eng.ExecAction(engine.WriteRandomFilesActionKey, fileWriteOpts)
@@ -40,11 +42,11 @@ func TestOneLargeFile(t *testing.T) {
 	numFiles := 1

 	fileWriteOpts := map[string]string{
-		engine.MaxDirDepthField:         strconv.Itoa(1),
-		engine.MaxFileSizeField:         strconv.Itoa(fileSize),
-		engine.MinFileSizeField:         strconv.Itoa(fileSize),
-		engine.MaxNumFilesPerWriteField: strconv.Itoa(numFiles),
-		engine.MinNumFilesPerWriteField: strconv.Itoa(numFiles),
+		fiofilewriter.MaxDirDepthField:         strconv.Itoa(1),
+		fiofilewriter.MaxFileSizeField:         strconv.Itoa(fileSize),
+		fiofilewriter.MinFileSizeField:         strconv.Itoa(fileSize),
+		fiofilewriter.MaxNumFilesPerWriteField: strconv.Itoa(numFiles),
+		fiofilewriter.MinNumFilesPerWriteField: strconv.Itoa(numFiles),
 	}

 	_, err := eng.ExecAction(engine.WriteRandomFilesActionKey, fileWriteOpts)
@@ -65,12 +67,12 @@ func TestManySmallFilesAcrossDirecoryTree(t *testing.T) {
 	actionRepeats := numFiles / filesPerWrite

 	fileWriteOpts := map[string]string{
-		engine.MaxDirDepthField:         strconv.Itoa(15),
-		engine.MaxFileSizeField:         strconv.Itoa(fileSize),
-		engine.MinFileSizeField:         strconv.Itoa(fileSize),
-		engine.MaxNumFilesPerWriteField: strconv.Itoa(filesPerWrite),
-		engine.MinNumFilesPerWriteField: strconv.Itoa(filesPerWrite),
-		engine.ActionRepeaterField:      strconv.Itoa(actionRepeats),
+		fiofilewriter.MaxDirDepthField:         strconv.Itoa(15),
+		fiofilewriter.MaxFileSizeField:         strconv.Itoa(fileSize),
+		fiofilewriter.MinFileSizeField:         strconv.Itoa(fileSize),
+		fiofilewriter.MaxNumFilesPerWriteField: strconv.Itoa(filesPerWrite),
+		fiofilewriter.MinNumFilesPerWriteField: strconv.Itoa(filesPerWrite),
+		engine.ActionRepeaterField:             strconv.Itoa(actionRepeats),
 	}

 	_, err := eng.ExecAction(engine.WriteRandomFilesActionKey, fileWriteOpts)
@@ -95,16 +97,16 @@ func TestRandomizedSmall(t *testing.T) {
 			string(engine.DeleteRandomSubdirectoryActionKey): strconv.Itoa(1),
 		},
 		engine.WriteRandomFilesActionKey: map[string]string{
-			engine.IOLimitPerWriteAction:    fmt.Sprintf("%d", 512*1024*1024),
-			engine.MaxNumFilesPerWriteField: strconv.Itoa(100),
-			engine.MaxFileSizeField:         strconv.Itoa(64 * 1024 * 1024),
-			engine.MaxDirDepthField:         strconv.Itoa(3),
+			fiofilewriter.IOLimitPerWriteAction:    fmt.Sprintf("%d", 512*1024*1024),
+			fiofilewriter.MaxNumFilesPerWriteField: strconv.Itoa(100),
+			fiofilewriter.MaxFileSizeField:         strconv.Itoa(64 * 1024 * 1024),
+			fiofilewriter.MaxDirDepthField:         strconv.Itoa(3),
 		},
 	}

 	for time.Since(st) <= *randomizedTestDur {
 		err := eng.RandomAction(opts)
-		if errors.Is(err, engine.ErrNoOp) {
+		if errors.Is(err, robustness.ErrNoOp) {
 			t.Log("Random action resulted in no-op")

 			err = nil