[robustness testing] Adding library for file generation using 'fio'

Adding a helper library that wraps fio execution. This is the basic initial check-in that implements the runners, configs, and a single WriteFiles helper. It should be enough to unblock subsequent tasks that will use fio to generate data sets for kopia snapshot verification. More helper workloads can be added as needed.

In this implementation the tests will all skip from test main if the `FIO_EXE` env variable is not set. Adding fio to the CI environment will be addressed as a separate PR.

Tracking progress in issue https://github.com/kopia/kopia/issues/179

tests/tools/fio/config.go

@@ -0,0 +1,17 @@
+package fio
+
+import "strings"
+
+// Config structures the fields of a FIO job run configuration
+type Config []Job
+
+// String implements the stringer interface, formats the Config
+// as it would appear in a well-formed fio config file.
+func (cfg Config) String() string {
+	ret := []string{}
+	for _, job := range cfg {
+		ret = append(ret, job.String())
+	}
+
+	return strings.Join(ret, "\n")
+}

tests/tools/fio/fio.go

@@ -0,0 +1,126 @@
+// Package fio wraps calls to the fio tool.
+// It assumes the tool is executable by "fio", but
+// gives the option to specify another executable
+// path by setting environment variable FIO_EXE.
+package fio
+
+import (
+	"io/ioutil"
+	"log"
+	"os"
+	"os/exec"
+	"strings"
+	"sync"
+)
+
+// List of fio flags
+const (
+	JobNameFlag = "--name"
+)
+
+// Runner is a helper for running fio commands
+type Runner struct {
+	Exe     string
+	DataDir string
+	Global  Config
+}
+
+// NewRunner creates a new fio runner
+func NewRunner() (*Runner, error) {
+	Exe := os.Getenv("FIO_EXE")
+	if Exe == "" {
+		Exe = "fio"
+	}
+
+	dataDir, err := ioutil.TempDir("", "fio-data")
+	if err != nil {
+		return nil, err
+	}
+
+	return &Runner{
+		Exe:     Exe,
+		DataDir: dataDir,
+		Global: Config{
+			{
+				Name: "global",
+				Options: map[string]string{
+					"openfiles":         "10",
+					"create_fsync":      "0",
+					"create_serialize":  "1",
+					"file_service_type": "sequential",
+					"ioengine":          "libaio",
+					"direct":            "1",
+					"iodepth":           "32",
+					"blocksize":         "1m",
+					"refill_buffers":    "",
+					"rw":                "write",
+					"unique_filename":   "1",
+					"directory":         dataDir,
+				},
+			},
+		},
+	}, nil
+}
+
+// Cleanup cleans up the data directory
+func (fr *Runner) Cleanup() {
+	if fr.DataDir != "" {
+		os.RemoveAll(fr.DataDir) //nolint:errcheck
+	}
+}
+
+// RunConfigs runs fio using the provided Configs
+func (fr *Runner) RunConfigs(cfgs ...Config) (stdout, stderr string, err error) {
+	var args []string
+
+	// Apply global config before any other configs
+	for _, cfg := range append([]Config{fr.Global}, cfgs...) {
+		log.Printf("Applying config:\n%s", cfg)
+
+		for _, job := range cfg {
+			args = append(args, JobNameFlag, job.Name)
+			for flagK, flagV := range job.Options {
+				args = append(args, "--"+flagK)
+
+				if flagV != "" {
+					args = append(args, flagV)
+				}
+			}
+		}
+	}
+
+	return fr.Run(args...)
+}
+
+// Run will execute the fio command with the given args
+func (fr *Runner) Run(args ...string) (stdout, stderr string, err error) {
+	argsStr := strings.Join(args, " ")
+	log.Printf("running '%s %v'", fr.Exe, argsStr)
+	// nolint:gosec
+	c := exec.Command(fr.Exe, args...)
+
+	stderrPipe, err := c.StderrPipe()
+	if err != nil {
+		return stdout, stderr, err
+	}
+
+	var errOut []byte
+
+	var wg sync.WaitGroup
+
+	wg.Add(1)
+
+	go func() {
+		defer wg.Done()
+
+		errOut, err = ioutil.ReadAll(stderrPipe)
+	}()
+
+	o, err := c.Output()
+
+	wg.Wait()
+
+	log.Printf("finished '%s %v' with err=%v and output:\n%v\n%v", fr.Exe, argsStr, err, string(o), string(errOut))
+
+	return string(o), string(errOut), err
+}

tests/tools/fio/fio_test.go

@@ -0,0 +1,87 @@
+package fio
+
+import (
+	"strings"
+	"testing"
+
+	"github.com/kopia/kopia/tests/testenv"
+)
+
+func TestFIORun(t *testing.T) {
+	r, err := NewRunner()
+	testenv.AssertNoError(t, err)
+
+	defer r.Cleanup()
+
+	stdout, stderr, err := r.Run()
+	if err == nil {
+		t.Fatal("Expected error to be set as no params were passed")
+	}
+
+	if !strings.Contains(stderr, "No job(s) defined") {
+		t.Fatal("Expected an error indicating no jobs were defined")
+	}
+
+	if !strings.Contains(stdout, "Print this page") {
+		// Indicates the --help page has been printed
+		t.Fatal("Expected --help page when running fio with no args")
+	}
+}
+
+func TestFIORunConfig(t *testing.T) {
+	r, err := NewRunner()
+	testenv.AssertNoError(t, err)
+
+	defer r.Cleanup()
+
+	cfg := Config{
+		{
+			Name: "write-10g",
+			Options: map[string]string{
+				"size":    "1g",
+				"nrfiles": "10",
+			},
+		},
+	}
+	stdout, stderr, err := r.RunConfigs(cfg)
+	testenv.AssertNoError(t, err)
+
+	if stderr != "" {
+		t.Error("Stderr was not empty")
+	}
+
+	if !strings.Contains(stdout, "rw=write") {
+		t.Error("Expected the output to indicate writes took place")
+	}
+}
+
+func TestFIOGlobalConfigOverride(t *testing.T) {
+	r, err := NewRunner()
+	testenv.AssertNoError(t, err)
+
+	defer r.Cleanup()
+
+	cfgs := []Config{
+		{
+			{
+				Name: "global",
+				Options: map[string]string{
+					"rw": "read",
+				},
+			},
+			{
+				Name: "write-10g",
+				Options: map[string]string{
+					"size":    "1g",
+					"nrfiles": "10",
+				},
+			},
+		},
+	}
+	stdout, _, err := r.RunConfigs(cfgs...)
+	testenv.AssertNoError(t, err)
+
+	if !strings.Contains(stdout, "rw=read") {
+		t.Fatal("Expected the global config 'rw' flag to be overwritten by the passed config")
+	}
+}

tests/tools/fio/job.go

@@ -0,0 +1,29 @@
+package fio
+
+import (
+	"fmt"
+	"strings"
+)
+
+// Job represents the configuration for running a FIO job
+type Job struct {
+	Name    string
+	Options Options
+}
+
+// String implements the stringer interface, formats the Job
+// as it would appear in a well-formed fio config file.
+func (job Job) String() string {
+	ret := []string{fmt.Sprintf("[%s]", job.Name)}
+
+	for k, v := range job.Options {
+		if v == "" {
+			ret = append(ret, k)
+			continue
+		}
+
+		ret = append(ret, fmt.Sprintf("%s=%s", k, v))
+	}
+
+	return strings.Join(ret, "\n")
+}

tests/tools/fio/main_test.go

@@ -0,0 +1,19 @@
+package fio
+
+import (
+	"fmt"
+	"os"
+	"testing"
+)
+
+func TestMain(m *testing.M) {
+	Exe := os.Getenv("FIO_EXE")
+	if Exe == "" {
+		fmt.Println("Skipping fio tests if FIO_EXE is not set")
+		os.Exit(0)
+	}
+
+	result := m.Run()
+
+	os.Exit(result)
+}

tests/tools/fio/options.go

@@ -0,0 +1,20 @@
+package fio
+
+// Options are flags to be set when running fio
+type Options map[string]string
+
+// Merge will merge two Options, overwriting common option keys
+// with the incoming option values. Returns the merged result
+func (o Options) Merge(other Options) map[string]string {
+	out := make(map[string]string, len(o)+len(other))
+
+	for k, v := range o {
+		out[k] = v
+	}
+
+	for k, v := range other {
+		out[k] = v
+	}
+
+	return out
+}

tests/tools/fio/workload.go

@@ -0,0 +1,32 @@
+package fio
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+	"strconv"
+)
+
+// WriteFiles writes files to the directory specified by path, up to the
+// provided size and number of files
+func (fr *Runner) WriteFiles(path string, sizeB int64, numFiles int, opt Options) error {
+	fullPath := filepath.Join(fr.DataDir, path)
+
+	err := os.MkdirAll(fullPath, 0700)
+	if err != nil {
+		return err
+	}
+
+	_, _, err = fr.RunConfigs(Config{
+		{
+			Name: fmt.Sprintf("write-%vB-%v", sizeB, numFiles),
+			Options: opt.Merge(Options{
+				"size":      strconv.Itoa(int(sizeB)),
+				"nrfiles":   strconv.Itoa(numFiles),
+				"directory": fullPath,
+			}),
+		},
+	})
+
+	return err
+}

tests/tools/fio/workload_test.go

@@ -0,0 +1,45 @@
+package fio
+
+import (
+	"fmt"
+	"io/ioutil"
+	"path/filepath"
+	"testing"
+
+	"github.com/kopia/kopia/tests/testenv"
+)
+
+func TestWriteFiles(t *testing.T) {
+	r, err := NewRunner()
+	testenv.AssertNoError(t, err)
+
+	defer r.Cleanup()
+
+	relativeWritePath := "some/path/to/check"
+	writeSizeB := int64(256 * 1024 * 1024) // 256 MiB
+	numFiles := 13
+
+	// Test a call to WriteFiles
+	err = r.WriteFiles(relativeWritePath, writeSizeB, numFiles, Options{})
+	testenv.AssertNoError(t, err)
+
+	fullPath := filepath.Join(r.DataDir, relativeWritePath)
+	dir, err := ioutil.ReadDir(fullPath)
+	testenv.AssertNoError(t, err)
+
+	if got, want := len(dir), numFiles; got != want {
+		t.Errorf("Did not get expected number of files %v (actual) != %v (expected", got, want)
+	}
+
+	sizeTot := int64(0)
+
+	for _, fi := range dir {
+		fmt.Println(fi.Name(), fi.Size())
+		sizeTot += fi.Size()
+	}
+
+	want := (writeSizeB / int64(numFiles)) * int64(numFiles)
+	if got := sizeTot; got != want {
+		t.Errorf("Did not get the expected amount of data written %v (actual) != %v (expected)", got, want)
+	}
+}