diff --git a/tests/tools/fio/config.go b/tests/tools/fio/config.go new file mode 100644 index 000000000..f39d76cec --- /dev/null +++ b/tests/tools/fio/config.go @@ -0,0 +1,17 @@ +package fio + +import "strings" + +// Config structures the fields of a FIO job run configuration +type Config []Job + +// String implements the stringer interface, formats the Config +// as it would appear in a well-formed fio config file. +func (cfg Config) String() string { + ret := []string{} + for _, job := range cfg { + ret = append(ret, job.String()) + } + + return strings.Join(ret, "\n") +} diff --git a/tests/tools/fio/fio.go b/tests/tools/fio/fio.go new file mode 100644 index 000000000..ec18db1a8 --- /dev/null +++ b/tests/tools/fio/fio.go @@ -0,0 +1,126 @@ +// Package fio wraps calls to the fio tool. +// It assumes the tool is executable by "fio", but +// gives the option to specify another executable +// path by setting environment variable FIO_EXE. +package fio + +import ( + "io/ioutil" + "log" + "os" + "os/exec" + "strings" + "sync" +) + +// List of fio flags +const ( + JobNameFlag = "--name" +) + +// Runner is a helper for running fio commands +type Runner struct { + Exe string + DataDir string + Global Config +} + +// NewRunner creates a new fio runner +func NewRunner() (*Runner, error) { + Exe := os.Getenv("FIO_EXE") + if Exe == "" { + Exe = "fio" + } + + dataDir, err := ioutil.TempDir("", "fio-data") + if err != nil { + return nil, err + } + + return &Runner{ + Exe: Exe, + DataDir: dataDir, + Global: Config{ + { + Name: "global", + Options: map[string]string{ + "openfiles": "10", + "create_fsync": "0", + "create_serialize": "1", + "file_service_type": "sequential", + "ioengine": "libaio", + "direct": "1", + "iodepth": "32", + "blocksize": "1m", + "refill_buffers": "", + "rw": "write", + "unique_filename": "1", + "directory": dataDir, + }, + }, + }, + }, nil +} + +// Cleanup cleans up the data directory +func (fr *Runner) Cleanup() { + if fr.DataDir != "" { + os.RemoveAll(fr.DataDir) //nolint:errcheck + } +} + +// RunConfigs runs fio using the provided Configs +func (fr *Runner) RunConfigs(cfgs ...Config) (stdout, stderr string, err error) { + var args []string + + // Apply global config before any other configs + for _, cfg := range append([]Config{fr.Global}, cfgs...) { + log.Printf("Applying config:\n%s", cfg) + + for _, job := range cfg { + args = append(args, JobNameFlag, job.Name) + for flagK, flagV := range job.Options { + args = append(args, "--"+flagK) + + if flagV != "" { + args = append(args, flagV) + } + } + } + } + + return fr.Run(args...) +} + +// Run will execute the fio command with the given args +func (fr *Runner) Run(args ...string) (stdout, stderr string, err error) { + argsStr := strings.Join(args, " ") + log.Printf("running '%s %v'", fr.Exe, argsStr) + // nolint:gosec + c := exec.Command(fr.Exe, args...) + + stderrPipe, err := c.StderrPipe() + if err != nil { + return stdout, stderr, err + } + + var errOut []byte + + var wg sync.WaitGroup + + wg.Add(1) + + go func() { + defer wg.Done() + + errOut, err = ioutil.ReadAll(stderrPipe) + }() + + o, err := c.Output() + + wg.Wait() + + log.Printf("finished '%s %v' with err=%v and output:\n%v\n%v", fr.Exe, argsStr, err, string(o), string(errOut)) + + return string(o), string(errOut), err +} diff --git a/tests/tools/fio/fio_test.go b/tests/tools/fio/fio_test.go new file mode 100644 index 000000000..13883dd26 --- /dev/null +++ b/tests/tools/fio/fio_test.go @@ -0,0 +1,87 @@ +package fio + +import ( + "strings" + "testing" + + "github.com/kopia/kopia/tests/testenv" +) + +func TestFIORun(t *testing.T) { + r, err := NewRunner() + testenv.AssertNoError(t, err) + + defer r.Cleanup() + + stdout, stderr, err := r.Run() + if err == nil { + t.Fatal("Expected error to be set as no params were passed") + } + + if !strings.Contains(stderr, "No job(s) defined") { + t.Fatal("Expected an error indicating no jobs were defined") + } + + if !strings.Contains(stdout, "Print this page") { + // Indicates the --help page has been printed + t.Fatal("Expected --help page when running fio with no args") + } +} + +func TestFIORunConfig(t *testing.T) { + r, err := NewRunner() + testenv.AssertNoError(t, err) + + defer r.Cleanup() + + cfg := Config{ + { + Name: "write-10g", + Options: map[string]string{ + "size": "1g", + "nrfiles": "10", + }, + }, + } + stdout, stderr, err := r.RunConfigs(cfg) + testenv.AssertNoError(t, err) + + if stderr != "" { + t.Error("Stderr was not empty") + } + + if !strings.Contains(stdout, "rw=write") { + t.Error("Expected the output to indicate writes took place") + } +} + +func TestFIOGlobalConfigOverride(t *testing.T) { + r, err := NewRunner() + testenv.AssertNoError(t, err) + + defer r.Cleanup() + + cfgs := []Config{ + { + { + Name: "global", + Options: map[string]string{ + "rw": "read", + }, + }, + { + Name: "write-10g", + Options: map[string]string{ + "size": "1g", + "nrfiles": "10", + }, + }, + }, + } + stdout, _, err := r.RunConfigs(cfgs...) + testenv.AssertNoError(t, err) + + if !strings.Contains(stdout, "rw=read") { + t.Fatal("Expected the global config 'rw' flag to be overwritten by the passed config") + } +} diff --git a/tests/tools/fio/job.go b/tests/tools/fio/job.go new file mode 100644 index 000000000..17fc94dda --- /dev/null +++ b/tests/tools/fio/job.go @@ -0,0 +1,29 @@ +package fio + +import ( + "fmt" + "strings" +) + +// Job represents the configuration for running a FIO job +type Job struct { + Name string + Options Options +} + +// String implements the stringer interface, formats the Job +// as it would appear in a well-formed fio config file. +func (job Job) String() string { + ret := []string{fmt.Sprintf("[%s]", job.Name)} + + for k, v := range job.Options { + if v == "" { + ret = append(ret, k) + continue + } + + ret = append(ret, fmt.Sprintf("%s=%s", k, v)) + } + + return strings.Join(ret, "\n") +} diff --git a/tests/tools/fio/main_test.go b/tests/tools/fio/main_test.go new file mode 100644 index 000000000..1910a96b4 --- /dev/null +++ b/tests/tools/fio/main_test.go @@ -0,0 +1,19 @@ +package fio + +import ( + "fmt" + "os" + "testing" +) + +func TestMain(m *testing.M) { + Exe := os.Getenv("FIO_EXE") + if Exe == "" { + fmt.Println("Skipping fio tests if FIO_EXE is not set") + os.Exit(0) + } + + result := m.Run() + + os.Exit(result) +} diff --git a/tests/tools/fio/options.go b/tests/tools/fio/options.go new file mode 100644 index 000000000..c8f3a6c7b --- /dev/null +++ b/tests/tools/fio/options.go @@ -0,0 +1,20 @@ +package fio + +// Options are flags to be set when running fio +type Options map[string]string + +// Merge will merge two Options, overwriting common option keys +// with the incoming option values. Returns the merged result +func (o Options) Merge(other Options) map[string]string { + out := make(map[string]string, len(o)+len(other)) + + for k, v := range o { + out[k] = v + } + + for k, v := range other { + out[k] = v + } + + return out +} diff --git a/tests/tools/fio/workload.go b/tests/tools/fio/workload.go new file mode 100644 index 000000000..5ee021893 --- /dev/null +++ b/tests/tools/fio/workload.go @@ -0,0 +1,32 @@ +package fio + +import ( + "fmt" + "os" + "path/filepath" + "strconv" +) + +// WriteFiles writes files to the directory specified by path, up to the +// provided size and number of files +func (fr *Runner) WriteFiles(path string, sizeB int64, numFiles int, opt Options) error { + fullPath := filepath.Join(fr.DataDir, path) + + err := os.MkdirAll(fullPath, 0700) + if err != nil { + return err + } + + _, _, err = fr.RunConfigs(Config{ + { + Name: fmt.Sprintf("write-%vB-%v", sizeB, numFiles), + Options: opt.Merge(Options{ + "size": strconv.Itoa(int(sizeB)), + "nrfiles": strconv.Itoa(numFiles), + "directory": fullPath, + }), + }, + }) + + return err +} diff --git a/tests/tools/fio/workload_test.go b/tests/tools/fio/workload_test.go new file mode 100644 index 000000000..17ab0e0c8 --- /dev/null +++ b/tests/tools/fio/workload_test.go @@ -0,0 +1,45 @@ +package fio + +import ( + "fmt" + "io/ioutil" + "path/filepath" + "testing" + + "github.com/kopia/kopia/tests/testenv" +) + +func TestWriteFiles(t *testing.T) { + r, err := NewRunner() + testenv.AssertNoError(t, err) + + defer r.Cleanup() + + relativeWritePath := "some/path/to/check" + writeSizeB := int64(256 * 1024 * 1024) // 256 MiB + numFiles := 13 + + // Test a call to WriteFiles + err = r.WriteFiles(relativeWritePath, writeSizeB, numFiles, Options{}) + testenv.AssertNoError(t, err) + + fullPath := filepath.Join(r.DataDir, relativeWritePath) + dir, err := ioutil.ReadDir(fullPath) + testenv.AssertNoError(t, err) + + if got, want := len(dir), numFiles; got != want { + t.Errorf("Did not get expected number of files %v (actual) != %v (expected", got, want) + } + + sizeTot := int64(0) + + for _, fi := range dir { + fmt.Println(fi.Name(), fi.Size()) + sizeTot += fi.Size() + } + + want := (writeSizeB / int64(numFiles)) * int64(numFiles) + if got := sizeTot; got != want { + t.Errorf("Did not get the expected amount of data written %v (actual) != %v (expected)", got, want) + } +}