Path protection between robustness engine FileWriter and Snapshotter (#865)

Protect filesystem subtrees from concurrent manipulation during critical sections
if engine actions are called asynchronously. This change provides coordination
between the `Snapshotter` and the `FileWriter`. For example, the `FileWriter`
should be blocked from perturbing the same directory tree if a
Gather-Snapshot is taking place along that tree simultaneously.
This will ensure the fingerprint data accumulated during the `Gather` phase
will correspond unambiguously to the data included in the snapshot.

Extend build flags to kopia snapshotter
  This package now imports fswalker which can only be built for
  darwin,amd64 or linux,amd64

tests/robustness/checker/checker.go

@@ -32,7 +32,6 @@ type Checker struct {
 	RestoreDir            string
 	snapshotIssuer        robustness.Snapshotter
 	snapshotMetadataStore robustness.Store
-	validator             robustness.Comparer
 	RecoveryMode          bool
 	DeleteLimit           int
 
@@ -42,7 +41,7 @@ type Checker struct {
 
 // NewChecker instantiates a new Checker, returning its pointer. A temporary
 // directory is created to mount restored data.
-func NewChecker(snapIssuer robustness.Snapshotter, snapmetaStore robustness.Store, validator robustness.Comparer, restoreDir string) (*Checker, error) {
+func NewChecker(snapIssuer robustness.Snapshotter, snapmetaStore robustness.Store, restoreDir string) (*Checker, error) {
 	restoreDir, err := ioutil.TempDir(restoreDir, "restore-data-")
 	if err != nil {
 		return nil, err
@@ -60,7 +59,6 @@ func NewChecker(snapIssuer robustness.Snapshotter, snapmetaStore robustness.Stor
 		RestoreDir:            restoreDir,
 		snapshotIssuer:        snapIssuer,
 		snapshotMetadataStore: snapmetaStore,
-		validator:             validator,
 		RecoveryMode:          false,
 		DeleteLimit:           delLimit,
 		SnapIDIndex:           make(snapmeta.Index),
@@ -79,7 +77,7 @@ func (chk *Checker) GetSnapIDs() []string {
 	chk.mu.RLock()
 	defer chk.mu.RUnlock()
 
-	return chk.SnapIDIndex.GetKeys(allSnapshotsIdxName)
+	return chk.SnapIDIndex.GetKeys(AllSnapshotsIdxName)
 }
 
 // SnapshotMetadata holds metadata associated with a given snapshot.
@@ -91,6 +89,12 @@ type SnapshotMetadata struct {
 	ValidationData []byte    `json:"validationData"`
 }
 
+// IsDeleted returns true if the SnapshotMetadata references a snapshot ID that
+// has been deleted.
+func (ssMeta *SnapshotMetadata) IsDeleted() bool {
+	return !ssMeta.DeletionTime.IsZero()
+}
+
 // GetSnapshotMetadata gets the metadata associated with the given snapshot ID.
 func (chk *Checker) GetSnapshotMetadata(snapID string) (*SnapshotMetadata, error) {
 	chk.mu.RLock()
@@ -105,7 +109,18 @@ func (chk *Checker) GetLiveSnapIDs() []string {
 	chk.mu.RLock()
 	defer chk.mu.RUnlock()
 
-	return chk.SnapIDIndex.GetKeys(liveSnapshotsIdxName)
+	return chk.SnapIDIndex.GetKeys(LiveSnapshotsIdxName)
+}
+
+// IsSnapshotIDDeleted reports whether the metadata associated with the provided snapshot ID
+// has it marked as deleted.
+func (chk *Checker) IsSnapshotIDDeleted(snapID string) (bool, error) {
+	md, err := chk.loadSnapshotMetadata(snapID)
+	if err != nil {
+		return false, err
+	}
+
+	return md.IsDeleted(), nil
 }
 
 // VerifySnapshotMetadata compares the list of live snapshot IDs present in
@@ -141,7 +156,7 @@ func (chk *Checker) VerifySnapshotMetadata() error {
 			if chk.RecoveryMode {
 				chk.mu.Lock()
 				chk.snapshotMetadataStore.Delete(metaSnapID)
-				chk.SnapIDIndex.RemoveFromIndex(metaSnapID, liveSnapshotsIdxName)
+				chk.SnapIDIndex.RemoveFromIndex(metaSnapID, LiveSnapshotsIdxName)
 				chk.mu.Unlock()
 			} else {
 				errCount++
@@ -190,25 +205,13 @@ func (chk *Checker) VerifySnapshotMetadata() error {
 // TakeSnapshot gathers state information on the requested snapshot path, then
 // performs the snapshot action defined by the Checker's Snapshotter.
 func (chk *Checker) TakeSnapshot(ctx context.Context, sourceDir string, opts map[string]string) (snapID string, err error) {
-	b, err := chk.validator.Gather(ctx, sourceDir, opts)
-	if err != nil {
-		return "", err
-	}
-
-	ssStart := clock.Now()
-
-	snapID, err = chk.snapshotIssuer.CreateSnapshot(sourceDir, opts)
-	if err != nil {
-		return snapID, err
-	}
-
-	ssEnd := clock.Now()
+	snapID, fingerprint, stats, err := chk.snapshotIssuer.CreateSnapshot(sourceDir, opts)
 
 	ssMeta := &SnapshotMetadata{
 		SnapID:         snapID,
-		SnapStartTime:  ssStart,
-		SnapEndTime:    ssEnd,
-		ValidationData: b,
+		SnapStartTime:  stats.SnapStartTime,
+		SnapEndTime:    stats.SnapEndTime,
+		ValidationData: fingerprint,
 	}
 
 	chk.mu.Lock()
@@ -219,8 +222,8 @@ func (chk *Checker) TakeSnapshot(ctx context.Context, sourceDir string, opts map
 		return snapID, err
 	}
 
-	chk.SnapIDIndex.AddToIndex(snapID, allSnapshotsIdxName)
-	chk.SnapIDIndex.AddToIndex(snapID, liveSnapshotsIdxName)
+	chk.SnapIDIndex.AddToIndex(snapID, AllSnapshotsIdxName)
+	chk.SnapIDIndex.AddToIndex(snapID, LiveSnapshotsIdxName)
 
 	return snapID, nil
 }
@@ -258,10 +261,10 @@ func (chk *Checker) safeRestorePrepare(snapID string) (*SnapshotMetadata, error)
 	chk.mu.RLock()
 	defer chk.mu.RUnlock()
 
-	if !chk.SnapIDIndex.IsKeyInIndex(snapID, liveSnapshotsIdxName) {
+	if !chk.SnapIDIndex.IsKeyInIndex(snapID, LiveSnapshotsIdxName) {
 		// Preventing restore of a snapshot ID that has been (or is currently
 		// being) deleted.
-		log.Printf("Snapshot ID %s is flagged for deletion", snapID)
+		log.Printf("Snapshot ID %s could not be found as a live snapshot", snapID)
 		return nil, robustness.ErrNoOp
 	}
 
@@ -276,50 +279,47 @@ func (chk *Checker) safeRestorePrepare(snapID string) (*SnapshotMetadata, error)
 // RestoreVerifySnapshot restores a snapshot and verifies its integrity against
 // the metadata provided.
 func (chk *Checker) RestoreVerifySnapshot(ctx context.Context, snapID, destPath string, ssMeta *SnapshotMetadata, reportOut io.Writer, opts map[string]string) error {
-	err := chk.snapshotIssuer.RestoreSnapshot(snapID, destPath, opts)
+	if ssMeta != nil {
+		return chk.snapshotIssuer.RestoreSnapshotCompare(snapID, destPath, ssMeta.ValidationData, reportOut, opts)
+	}
+
+	// We have no metadata for this snapshot ID.
+	if !chk.RecoveryMode {
+		return robustness.ErrMetadataMissing
+	}
+
+	// Recovery path:
+	// If in recovery mode, restore the snapshot by snapshot ID and gather
+	// its fingerprint (i.e. assume the snapshot will restore properly).
+	fingerprint, err := chk.snapshotIssuer.RestoreSnapshot(snapID, destPath, opts)
 	if err != nil {
 		return err
 	}
 
-	if ssMeta == nil && chk.RecoveryMode {
-		var b []byte
-
-		b, err = chk.validator.Gather(ctx, destPath, opts)
-		if err != nil {
-			return err
-		}
-
-		ssMeta = &SnapshotMetadata{
-			SnapID:         snapID,
-			ValidationData: b,
-		}
-
-		chk.mu.Lock()
-		defer chk.mu.Unlock()
-
-		err = chk.saveSnapshotMetadata(ssMeta)
-		if err != nil {
-			return err
-		}
-
-		chk.SnapIDIndex.AddToIndex(snapID, allSnapshotsIdxName)
-		chk.SnapIDIndex.AddToIndex(snapID, liveSnapshotsIdxName)
-
-		return nil
+	ssMeta = &SnapshotMetadata{
+		SnapID:         snapID,
+		ValidationData: fingerprint,
 	}
 
-	err = chk.validator.Compare(ctx, destPath, ssMeta.ValidationData, reportOut, opts)
+	chk.mu.Lock()
+	defer chk.mu.Unlock()
+
+	err = chk.saveSnapshotMetadata(ssMeta)
 	if err != nil {
 		return err
 	}
 
+	chk.SnapIDIndex.AddToIndex(snapID, AllSnapshotsIdxName)
+	chk.SnapIDIndex.AddToIndex(snapID, LiveSnapshotsIdxName)
+
 	return nil
 }
 
+// Index names for categorizing snapshot lookups.
 const (
-	deletedSnapshotsIdxName = "deleted-snapshots-idx"
-	liveSnapshotsIdxName    = "live-snapshots-idx"
-	allSnapshotsIdxName     = "all-snapshots-idx"
+	DeletedSnapshotsIdxName = "deleted-snapshots-idx"
+	LiveSnapshotsIdxName    = "live-snapshots-idx"
+	AllSnapshotsIdxName     = "all-snapshots-idx"
 )
 
 // DeleteSnapshot performs the Snapshotter's DeleteSnapshot action, and
@@ -353,7 +353,7 @@ func (chk *Checker) safeDeletePrepare(snapID string) (*SnapshotMetadata, error)
 	chk.mu.Lock()
 	defer chk.mu.Unlock()
 
-	if !chk.SnapIDIndex.IsKeyInIndex(snapID, liveSnapshotsIdxName) {
+	if !chk.SnapIDIndex.IsKeyInIndex(snapID, LiveSnapshotsIdxName) {
 		// Preventing restore of a snapshot ID that has been already deleted.
 		log.Printf("Cannot delete snapshot ID %s as it is not a live snapshot", snapID)
 		return nil, robustness.ErrNoOp
@@ -369,7 +369,7 @@ func (chk *Checker) safeDeletePrepare(snapID string) (*SnapshotMetadata, error)
 	// Place the snapshot ID in the list of IDs with unknown state. If we
 	// hit an error during delete, we don't know whether to expect it to
 	// still be valid or not.
-	chk.SnapIDIndex.RemoveFromIndex(snapID, liveSnapshotsIdxName)
+	chk.SnapIDIndex.RemoveFromIndex(snapID, LiveSnapshotsIdxName)
 
 	return ssMeta, nil
 }
@@ -380,7 +380,7 @@ func (chk *Checker) safeDeleteFinish(ssMeta *SnapshotMetadata) error {
 	chk.mu.Lock()
 	defer chk.mu.Unlock()
 
-	chk.SnapIDIndex.AddToIndex(ssMeta.SnapID, deletedSnapshotsIdxName)
+	chk.SnapIDIndex.AddToIndex(ssMeta.SnapID, DeletedSnapshotsIdxName)
 
 	err := chk.saveSnapshotMetadata(ssMeta)
 	if err != nil {

tests/robustness/comparer.go

@@ -1,13 +0,0 @@
-package robustness
-
-import (
-	"context"
-	"io"
-)
-
-// Comparer describes an interface that gathers state data on a provided
-// path, and compares that data to the state on another path.
-type Comparer interface {
-	Gather(ctx context.Context, path string, opts map[string]string) ([]byte, error)
-	Compare(ctx context.Context, path string, data []byte, reportOut io.Writer, opts map[string]string) error
-}

tests/robustness/engine/action.go

@@ -8,6 +8,7 @@ import (
 	"errors"
 	"log"
 	"math/rand"
+	"path/filepath"
 	"strconv"
 	"strings"
 
@@ -137,7 +138,7 @@ func (e *Engine) checkErrRecovery(incomingErr error, actionOpts ActionOpts) (out
 // List of action keys.
 const (
 	ActionControlActionKey            ActionKey = "action-control"
-	SnapshotRootDirActionKey          ActionKey = "snapshot-root"
+	SnapshotDirActionKey              ActionKey = "snapshot-root"
 	RestoreSnapshotActionKey          ActionKey = "restore-random-snapID"
 	DeleteRandomSnapshotActionKey     ActionKey = "delete-random-snapID"
 	WriteRandomFilesActionKey         ActionKey = "write-random-files"
@@ -170,15 +171,20 @@ type Action struct {
 type ActionKey string
 
 var actions = map[ActionKey]Action{
-	SnapshotRootDirActionKey: {
+	SnapshotDirActionKey: {
 		f: func(e *Engine, opts map[string]string, l *LogEntry) (out map[string]string, err error) {
-			log.Printf("Creating snapshot of root directory %s", e.FileWriter.DataDirectory())
+			snapPath := e.FileWriter.DataDirectory()
+			if opts != nil && opts[SubPathOptionName] != "" {
+				snapPath = filepath.Join(snapPath, opts[SubPathOptionName])
+			}
+
+			log.Printf("Creating snapshot of directory %s", snapPath)
 
 			ctx := context.TODO()
-			snapID, err := e.Checker.TakeSnapshot(ctx, e.FileWriter.DataDirectory(), opts)
+			snapID, err := e.Checker.TakeSnapshot(ctx, snapPath, opts)
 
 			setLogEntryCmdOpts(l, map[string]string{
-				"snap-dir": e.FileWriter.DataDirectory(),
+				"snap-dir": snapPath,
 				"snapID":   snapID,
 			})
 
@@ -287,6 +293,7 @@ const (
 	ActionRepeaterField          = "repeat-action"
 	ThrowNoSpaceOnDeviceErrField = "throw-no-space-error"
 	SnapshotIDField              = "snapshot-ID"
+	SubPathOptionName            = "sub-path"
 )
 
 func defaultActionControls() map[string]string {

tests/robustness/engine/engine.go

@@ -29,7 +29,6 @@ type Args struct {
 	// Interfaces used by the engine.
 	MetaStore  robustness.Persister
 	TestRepo   robustness.Snapshotter
-	Validator  robustness.Comparer
 	FileWriter robustness.FileWriter
 
 	// WorkingDir is a directory to use for temporary data.
@@ -41,7 +40,7 @@ type Args struct {
 
 // Validate checks the arguments for correctness.
 func (a *Args) Validate() error {
-	if a.MetaStore == nil || a.TestRepo == nil || a.Validator == nil || a.FileWriter == nil || a.WorkingDir == "" {
+	if a.MetaStore == nil || a.TestRepo == nil || a.FileWriter == nil || a.WorkingDir == "" {
 		return ErrorInvalidArgs
 	}
 
@@ -58,7 +57,6 @@ func New(args *Args) (*Engine, error) {
 		e = &Engine{
 			MetaStore:   args.MetaStore,
 			TestRepo:    args.TestRepo,
-			Validator:   args.Validator,
 			FileWriter:  args.FileWriter,
 			baseDirPath: args.WorkingDir,
 			RunStats: Stats{
@@ -75,7 +73,7 @@ func New(args *Args) (*Engine, error) {
 		return nil, err
 	}
 
-	e.Checker, err = checker.NewChecker(e.TestRepo, e.MetaStore, e.Validator, e.baseDirPath)
+	e.Checker, err = checker.NewChecker(e.TestRepo, e.MetaStore, e.baseDirPath)
 	if err != nil {
 		e.cleanComponents()
 		return nil, err
@@ -92,7 +90,6 @@ type Engine struct {
 	FileWriter robustness.FileWriter
 	TestRepo   robustness.Snapshotter
 	MetaStore  robustness.Persister
-	Validator  robustness.Comparer
 
 	Checker         *checker.Checker
 	cleanupRoutines []func()
@@ -108,12 +105,12 @@ func (e *Engine) Shutdown() error {
 	// Perform a snapshot action to capture the state of the data directory
 	// at the end of the run
 	lastWriteEntry := e.EngineLog.FindLastThisRun(WriteRandomFilesActionKey)
-	lastSnapEntry := e.EngineLog.FindLastThisRun(SnapshotRootDirActionKey)
+	lastSnapEntry := e.EngineLog.FindLastThisRun(SnapshotDirActionKey)
 
 	if lastWriteEntry != nil {
 		if lastSnapEntry == nil || lastSnapEntry.Idx < lastWriteEntry.Idx {
 			// Only force a final snapshot if the data tree has been modified since the last snapshot
-			e.ExecAction(SnapshotRootDirActionKey, make(map[string]string)) //nolint:errcheck
+			e.ExecAction(SnapshotDirActionKey, make(map[string]string)) //nolint:errcheck
 		}
 	}
 

tests/robustness/engine/engine_test.go

@@ -615,7 +615,7 @@ func TestIOLimitPerWriteAction(t *testing.T) {
 
 	actionOpts := ActionOpts{
 		ActionControlActionKey: map[string]string{
-			string(SnapshotRootDirActionKey):          strconv.Itoa(0),
+			string(SnapshotDirActionKey):              strconv.Itoa(0),
 			string(RestoreSnapshotActionKey):          strconv.Itoa(0),
 			string(DeleteRandomSnapshotActionKey):     strconv.Itoa(0),
 			string(WriteRandomFilesActionKey):         strconv.Itoa(1),
@@ -822,7 +822,6 @@ type testHarness struct {
 	fw *fiofilewriter.FileWriter
 	ks *snapmeta.KopiaSnapshotter
 	kp *snapmeta.KopiaPersister
-	wc *fswalker.WalkCompare
 
 	baseDir string
 
@@ -866,8 +865,6 @@ func newTestHarness(t *testing.T, dataRepoPath, metaRepoPath string) (*testHarne
 		return nil, nil, err
 	}
 
-	th.wc = fswalker.NewWalkCompare()
-
 	if th.eng, err = New(th.args()); err != nil {
 		th.Cleanup()
 		return nil, nil, err
@@ -880,7 +877,6 @@ func (th *testHarness) args() *Args {
 	return &Args{
 		MetaStore:  th.kp,
 		TestRepo:   th.ks,
-		Validator:  th.wc,
 		FileWriter: th.fw,
 		WorkingDir: th.baseDir,
 	}

tests/robustness/errors.go

@@ -21,4 +21,7 @@ var (
 
 	// ErrKeyNotFound is returned when the store can't find the key provided.
 	ErrKeyNotFound = errors.New("key not found")
+
+	// ErrMetadataMissing is returned when the metadata can't be found.
+	ErrMetadataMissing = errors.New("metadata missing")
 )

tests/robustness/robustness_test/main_test.go

@@ -19,7 +19,6 @@ import (
 	"github.com/kopia/kopia/tests/robustness/fiofilewriter"
 	"github.com/kopia/kopia/tests/robustness/snapmeta"
 	"github.com/kopia/kopia/tests/tools/fio"
-	"github.com/kopia/kopia/tests/tools/fswalker"
 	"github.com/kopia/kopia/tests/tools/kopiarunner"
 )
 
@@ -73,7 +72,6 @@ type kopiaRobustnessTestHarness struct {
 	fileWriter  *fiofilewriter.FileWriter
 	snapshotter *snapmeta.KopiaSnapshotter
 	persister   *snapmeta.KopiaPersister
-	comparer    *fswalker.WalkCompare
 	engine      *engine.Engine
 
 	skipTest bool
@@ -85,7 +83,7 @@ func (th *kopiaRobustnessTestHarness) init(dataRepoPath, metaRepoPath string) {
 
 	// the initialization state machine is linear and bails out on first failure
 	if th.makeBaseDir() && th.getFileWriter() && th.getSnapshotter() &&
-		th.getPersister() && th.getComparer() && th.getEngine() {
+		th.getPersister() && th.getEngine() {
 		return // success!
 	}
 
@@ -177,16 +175,10 @@ func (th *kopiaRobustnessTestHarness) getPersister() bool {
 	return true
 }
 
-func (th *kopiaRobustnessTestHarness) getComparer() bool {
-	th.comparer = fswalker.NewWalkCompare()
-	return true
-}
-
 func (th *kopiaRobustnessTestHarness) getEngine() bool {
 	args := &engine.Args{
 		MetaStore:        th.persister,
 		TestRepo:         th.snapshotter,
-		Validator:        th.comparer,
 		FileWriter:       th.fileWriter,
 		WorkingDir:       th.baseDirPath,
 		SyncRepositories: true,

tests/robustness/robustness_test/robustness_test.go

@@ -30,7 +30,7 @@ func TestManySmallFiles(t *testing.T) {
 	_, err := eng.ExecAction(engine.WriteRandomFilesActionKey, fileWriteOpts)
 	testenv.AssertNoError(t, err)
 
-	snapOut, err := eng.ExecAction(engine.SnapshotRootDirActionKey, nil)
+	snapOut, err := eng.ExecAction(engine.SnapshotDirActionKey, nil)
 	testenv.AssertNoError(t, err)
 
 	_, err = eng.ExecAction(engine.RestoreSnapshotActionKey, snapOut)
@@ -52,7 +52,7 @@ func TestOneLargeFile(t *testing.T) {
 	_, err := eng.ExecAction(engine.WriteRandomFilesActionKey, fileWriteOpts)
 	testenv.AssertNoError(t, err)
 
-	snapOut, err := eng.ExecAction(engine.SnapshotRootDirActionKey, nil)
+	snapOut, err := eng.ExecAction(engine.SnapshotDirActionKey, nil)
 	testenv.AssertNoError(t, err)
 
 	_, err = eng.ExecAction(engine.RestoreSnapshotActionKey, snapOut)
@@ -78,7 +78,7 @@ func TestManySmallFilesAcrossDirecoryTree(t *testing.T) {
 	_, err := eng.ExecAction(engine.WriteRandomFilesActionKey, fileWriteOpts)
 	testenv.AssertNoError(t, err)
 
-	snapOut, err := eng.ExecAction(engine.SnapshotRootDirActionKey, nil)
+	snapOut, err := eng.ExecAction(engine.SnapshotDirActionKey, nil)
 	testenv.AssertNoError(t, err)
 
 	_, err = eng.ExecAction(engine.RestoreSnapshotActionKey, snapOut)
@@ -90,7 +90,7 @@ func TestRandomizedSmall(t *testing.T) {
 
 	opts := engine.ActionOpts{
 		engine.ActionControlActionKey: map[string]string{
-			string(engine.SnapshotRootDirActionKey):          strconv.Itoa(2),
+			string(engine.SnapshotDirActionKey):              strconv.Itoa(2),
 			string(engine.RestoreSnapshotActionKey):          strconv.Itoa(2),
 			string(engine.DeleteRandomSnapshotActionKey):     strconv.Itoa(1),
 			string(engine.WriteRandomFilesActionKey):         strconv.Itoa(8),

tests/robustness/snapmeta/kopia_snapshotter.go

@@ -1,23 +1,34 @@
+// +build darwin,amd64 linux,amd64
+
 package snapmeta
 
 import (
+	"context"
+	"io"
 	"os/exec"
 	"strconv"
 
+	"github.com/kopia/kopia/internal/clock"
 	"github.com/kopia/kopia/tests/robustness"
+	"github.com/kopia/kopia/tests/tools/fswalker"
 )
 
-// KopiaSnapshotter implements robustness.Snapshotter.
+// KopiaSnapshotter wraps the functionality to connect to a kopia repository with
+// the fswalker WalkCompare.
 type KopiaSnapshotter struct {
+	comparer *fswalker.WalkCompare
 	kopiaConnector
 }
 
+// KopiaSnapshotter implements robustness.Snapshotter.
 var _ robustness.Snapshotter = (*KopiaSnapshotter)(nil)
 
 // NewSnapshotter returns a Kopia based Snapshotter.
 // ConnectOrCreateRepo must be invoked to enable the interface.
 func NewSnapshotter(baseDirPath string) (*KopiaSnapshotter, error) {
-	ks := &KopiaSnapshotter{}
+	ks := &KopiaSnapshotter{
+		comparer: fswalker.NewWalkCompare(),
+	}
 
 	if err := ks.initializeConnector(baseDirPath); err != nil {
 		return nil, err
@@ -43,13 +54,49 @@ func (ks *KopiaSnapshotter) ServerCmd() *exec.Cmd {
 }
 
 // CreateSnapshot is part of Snapshotter.
-func (ks *KopiaSnapshotter) CreateSnapshot(sourceDir string, opts map[string]string) (snapID string, err error) {
-	return ks.snap.CreateSnapshot(sourceDir)
+func (ks *KopiaSnapshotter) CreateSnapshot(sourceDir string, opts map[string]string) (snapID string, fingerprint []byte, snapStats *robustness.CreateSnapshotStats, err error) {
+	fingerprint, err = ks.comparer.Gather(context.TODO(), sourceDir, opts)
+	if err != nil {
+		return
+	}
+
+	ssStart := clock.Now()
+
+	snapID, err = ks.snap.CreateSnapshot(sourceDir)
+	if err != nil {
+		return
+	}
+
+	ssEnd := clock.Now()
+
+	snapStats = &robustness.CreateSnapshotStats{
+		SnapStartTime: ssStart,
+		SnapEndTime:   ssEnd,
+	}
+
+	return
 }
 
-// RestoreSnapshot is part of Snapshotter.
-func (ks *KopiaSnapshotter) RestoreSnapshot(snapID, restoreDir string, opts map[string]string) error {
-	return ks.snap.RestoreSnapshot(snapID, restoreDir)
+// RestoreSnapshot restores the snapshot with the given ID to the provided restore directory. It returns
+// fingerprint verification data of the restored snapshot directory.
+func (ks *KopiaSnapshotter) RestoreSnapshot(snapID, restoreDir string, opts map[string]string) (fingerprint []byte, err error) {
+	err = ks.snap.RestoreSnapshot(snapID, restoreDir)
+	if err != nil {
+		return
+	}
+
+	return ks.comparer.Gather(context.TODO(), restoreDir, opts)
+}
+
+// RestoreSnapshotCompare restores the snapshot with the given ID to the provided restore directory, then verifies the data
+// that has been restored against the provided fingerprint validation data.
+func (ks *KopiaSnapshotter) RestoreSnapshotCompare(snapID, restoreDir string, validationData []byte, reportOut io.Writer, opts map[string]string) (err error) {
+	err = ks.snap.RestoreSnapshot(snapID, restoreDir)
+	if err != nil {
+		return err
+	}
+
+	return ks.comparer.Compare(context.TODO(), restoreDir, validationData, reportOut, opts)
 }
 
 // DeleteSnapshot is part of Snapshotter.

tests/robustness/snapshotter.go

@@ -3,13 +3,27 @@
 // can be used to test other environments.
 package robustness
 
+import (
+	"io"
+	"time"
+)
+
 // Snapshotter is an interface that describes methods
 // for taking, restoring, deleting snapshots, and
 // tracking them by a string snapshot ID.
 type Snapshotter interface {
-	CreateSnapshot(sourceDir string, opts map[string]string) (snapID string, err error)
-	RestoreSnapshot(snapID string, restoreDir string, opts map[string]string) error
 	DeleteSnapshot(snapID string, opts map[string]string) error
 	RunGC(opts map[string]string) error
 	ListSnapshots() ([]string, error)
+
+	CreateSnapshot(sourceDir string, opts map[string]string) (snapID string, fingerprint []byte, stats *CreateSnapshotStats, err error)
+	RestoreSnapshot(snapID, restoreDir string, opts map[string]string) ([]byte, error)
+	RestoreSnapshotCompare(snapID, restoreDir string, validationData []byte, reportOut io.Writer, opts map[string]string) error
+}
+
+// CreateSnapshotStats is a struct for returning various stats from the snapshot execution.
+type CreateSnapshotStats struct {
+	SnapStartTime time.Time
+	SnapEndTime   time.Time
+	Raw           []byte
 }

tests/tools/fio/fio.go

@@ -17,6 +17,8 @@ import (
 	"strings"
 
 	"github.com/pkg/errors"
+
+	"github.com/kopia/kopia/tests/robustness/pathlock"
 )
 
 // List of fio flags.
@@ -62,8 +64,23 @@ type Runner struct {
 	FioWriteBaseDir string
 	Global          Config
 	Debug           bool
+
+	PathLock pathlock.Locker
 }
 
+// NullPathLocker satisfies the pathlock.Locker interface but is a no-op.
+type NullPathLocker struct{}
+
+var _ pathlock.Locker = (*NullPathLocker)(nil)
+
+// Lock implements the pathlock.Locker interface.
+func (l *NullPathLocker) Lock(lockPath string) (pathlock.Unlocker, error) {
+	return l, nil
+}
+
+// Unlock satisfies the pathlock.Unlocker interface.
+func (l *NullPathLocker) Unlock() {}
+
 // NewRunner creates a new fio runner.
 func NewRunner() (fr *Runner, err error) {
 	exeStr := os.Getenv(FioExeEnvKey)
@@ -136,6 +153,7 @@ func NewRunner() (fr *Runner, err error) {
 				}.WithDirectory(fioWriteBaseDir),
 			},
 		},
+		PathLock: &NullPathLocker{},
 	}
 
 	err = fr.verifySetupWithTestWrites()

tests/tools/fio/workload.go

@@ -13,7 +13,14 @@ import (
 // WriteFiles writes files to the directory specified by path, up to the
 // provided size and number of files.
 func (fr *Runner) WriteFiles(relPath string, opt Options) error {
+	lock, err := fr.PathLock.Lock(relPath)
+	if err != nil {
+		return err
+	}
+	defer lock.Unlock()
+
 	fullPath := filepath.Join(fr.LocalDataDir, relPath)
+
 	return fr.writeFiles(fullPath, opt)
 }
 
@@ -48,9 +55,15 @@ func (fr *Runner) writeFiles(fullPath string, opt Options) error {
 // WriteFilesAtDepth writes files to a directory "depth" layers deep below
 // the base data directory.
 func (fr *Runner) WriteFilesAtDepth(relBasePath string, depth int, opt Options) error {
+	lock, err := fr.PathLock.Lock(relBasePath)
+	if err != nil {
+		return err
+	}
+	defer lock.Unlock()
+
 	fullBasePath := filepath.Join(fr.LocalDataDir, relBasePath)
 
-	err := os.MkdirAll(fullBasePath, 0700)
+	err = os.MkdirAll(fullBasePath, 0700)
 	if err != nil {
 		return errors.Wrapf(err, "unable to make base dir %v for writing at depth", fullBasePath)
 	}
@@ -61,9 +74,15 @@ func (fr *Runner) WriteFilesAtDepth(relBasePath string, depth int, opt Options)
 // WriteFilesAtDepthRandomBranch writes files to a directory "depth" layers deep below
 // the base data directory and branches at a random depth.
 func (fr *Runner) WriteFilesAtDepthRandomBranch(relBasePath string, depth int, opt Options) error {
+	lock, err := fr.PathLock.Lock(relBasePath)
+	if err != nil {
+		return err
+	}
+	defer lock.Unlock()
+
 	fullBasePath := filepath.Join(fr.LocalDataDir, relBasePath)
 
-	err := os.MkdirAll(fullBasePath, 0700)
+	err = os.MkdirAll(fullBasePath, 0700)
 	if err != nil {
 		return errors.Wrapf(err, "unable to make base dir %v for writing at depth with a branch", fullBasePath)
 	}
@@ -73,11 +92,23 @@ func (fr *Runner) WriteFilesAtDepthRandomBranch(relBasePath string, depth int, o
 
 // DeleteRelDir deletes a relative directory in the runner's data directory.
 func (fr *Runner) DeleteRelDir(relDirPath string) error {
+	lock, err := fr.PathLock.Lock(relDirPath)
+	if err != nil {
+		return err
+	}
+	defer lock.Unlock()
+
 	return os.RemoveAll(filepath.Join(fr.LocalDataDir, relDirPath))
 }
 
 // DeleteDirAtDepth deletes a random directory at the given depth.
 func (fr *Runner) DeleteDirAtDepth(relBasePath string, depth int) error {
+	lock, err := fr.PathLock.Lock(relBasePath)
+	if err != nil {
+		return err
+	}
+	defer lock.Unlock()
+
 	if depth == 0 {
 		return ErrCanNotDeleteRoot
 	}
@@ -93,6 +124,12 @@ func (fr *Runner) DeleteDirAtDepth(relBasePath string, depth int) error {
 // deleted. Probability set to 0 will delete nothing. Probability set to 1 will delete
 // everything.
 func (fr *Runner) DeleteContentsAtDepth(relBasePath string, depth int, prob float32) error {
+	lock, err := fr.PathLock.Lock(relBasePath)
+	if err != nil {
+		return err
+	}
+	defer lock.Unlock()
+
 	fullBasePath := filepath.Join(fr.LocalDataDir, relBasePath)
 
 	return fr.operateAtDepth(fullBasePath, depth, func(dirPath string) error {

tests/tools/fswalker/fswalker.go

@@ -18,13 +18,10 @@ import (
 	fspb "github.com/google/fswalker/proto/fswalker"
 	"github.com/pkg/errors"
 
-	"github.com/kopia/kopia/tests/robustness"
 	"github.com/kopia/kopia/tests/tools/fswalker/reporter"
 	"github.com/kopia/kopia/tests/tools/fswalker/walker"
 )
 
-var _ robustness.Comparer = &WalkCompare{}
-
 // WalkCompare is a checker.Comparer that utilizes the fswalker
 // libraries to perform the data consistency check.
 type WalkCompare struct {