From edbc7591d7c36056d22fbe7e972bf1b810e312ed Mon Sep 17 00:00:00 2001 From: Nick Date: Wed, 29 Jan 2020 11:36:20 -0800 Subject: [PATCH] [robustness testing] Adding library for file generation using 'fio' Adding a helper library that wraps fio execution. This is the basic initial check-in that implements the runners, configs, and a single WriteFiles helper. It should be enough to unblock subsequent tasks that will use fio to generate data sets for kopia snapshot verification. More helper workloads can be added as needed. In this implementation the tests will all skip from test main if the `FIO_EXE` env variable is not set. Adding fio to the CI environment will be addressed as a separate PR. Tracking progress in issue https://github.com/kopia/kopia/issues/179 --- tests/tools/fio/config.go | 17 +++++ tests/tools/fio/fio.go | 126 +++++++++++++++++++++++++++++++ tests/tools/fio/fio_test.go | 87 +++++++++++++++++++++ tests/tools/fio/job.go | 29 +++++++ tests/tools/fio/main_test.go | 19 +++++ tests/tools/fio/options.go | 20 +++++ tests/tools/fio/workload.go | 32 ++++++++ tests/tools/fio/workload_test.go | 45 +++++++++++ 8 files changed, 375 insertions(+) create mode 100644 tests/tools/fio/config.go create mode 100644 tests/tools/fio/fio.go create mode 100644 tests/tools/fio/fio_test.go create mode 100644 tests/tools/fio/job.go create mode 100644 tests/tools/fio/main_test.go create mode 100644 tests/tools/fio/options.go create mode 100644 tests/tools/fio/workload.go create mode 100644 tests/tools/fio/workload_test.go diff --git a/tests/tools/fio/config.go b/tests/tools/fio/config.go new file mode 100644 index 000000000..f39d76cec --- /dev/null +++ b/tests/tools/fio/config.go @@ -0,0 +1,17 @@ +package fio + +import "strings" + +// Config structures the fields of a FIO job run configuration +type Config []Job + +// String implements the stringer interface, formats the Config +// as it would appear in a well-formed fio config file. +func (cfg Config) String() string { + ret := []string{} + for _, job := range cfg { + ret = append(ret, job.String()) + } + + return strings.Join(ret, "\n") +} diff --git a/tests/tools/fio/fio.go b/tests/tools/fio/fio.go new file mode 100644 index 000000000..ec18db1a8 --- /dev/null +++ b/tests/tools/fio/fio.go @@ -0,0 +1,126 @@ +// Package fio wraps calls to the fio tool. +// It assumes the tool is executable by "fio", but +// gives the option to specify another executable +// path by setting environment variable FIO_EXE. +package fio + +import ( + "io/ioutil" + "log" + "os" + "os/exec" + "strings" + "sync" +) + +// List of fio flags +const ( + JobNameFlag = "--name" +) + +// Runner is a helper for running fio commands +type Runner struct { + Exe string + DataDir string + Global Config +} + +// NewRunner creates a new fio runner +func NewRunner() (*Runner, error) { + Exe := os.Getenv("FIO_EXE") + if Exe == "" { + Exe = "fio" + } + + dataDir, err := ioutil.TempDir("", "fio-data") + if err != nil { + return nil, err + } + + return &Runner{ + Exe: Exe, + DataDir: dataDir, + Global: Config{ + { + Name: "global", + Options: map[string]string{ + "openfiles": "10", + "create_fsync": "0", + "create_serialize": "1", + "file_service_type": "sequential", + "ioengine": "libaio", + "direct": "1", + "iodepth": "32", + "blocksize": "1m", + "refill_buffers": "", + "rw": "write", + "unique_filename": "1", + "directory": dataDir, + }, + }, + }, + }, nil +} + +// Cleanup cleans up the data directory +func (fr *Runner) Cleanup() { + if fr.DataDir != "" { + os.RemoveAll(fr.DataDir) //nolint:errcheck + } +} + +// RunConfigs runs fio using the provided Configs +func (fr *Runner) RunConfigs(cfgs ...Config) (stdout, stderr string, err error) { + var args []string + + // Apply global config before any other configs + for _, cfg := range append([]Config{fr.Global}, cfgs...) { + log.Printf("Applying config:\n%s", cfg) + + for _, job := range cfg { + args = append(args, JobNameFlag, job.Name) + for flagK, flagV := range job.Options { + args = append(args, "--"+flagK) + + if flagV != "" { + args = append(args, flagV) + } + } + } + } + + return fr.Run(args...) +} + +// Run will execute the fio command with the given args +func (fr *Runner) Run(args ...string) (stdout, stderr string, err error) { + argsStr := strings.Join(args, " ") + log.Printf("running '%s %v'", fr.Exe, argsStr) + // nolint:gosec + c := exec.Command(fr.Exe, args...) + + stderrPipe, err := c.StderrPipe() + if err != nil { + return stdout, stderr, err + } + + var errOut []byte + + var wg sync.WaitGroup + + wg.Add(1) + + go func() { + defer wg.Done() + + errOut, err = ioutil.ReadAll(stderrPipe) + }() + + o, err := c.Output() + + wg.Wait() + + log.Printf("finished '%s %v' with err=%v and output:\n%v\n%v", fr.Exe, argsStr, err, string(o), string(errOut)) + + return string(o), string(errOut), err +} diff --git a/tests/tools/fio/fio_test.go b/tests/tools/fio/fio_test.go new file mode 100644 index 000000000..13883dd26 --- /dev/null +++ b/tests/tools/fio/fio_test.go @@ -0,0 +1,87 @@ +package fio + +import ( + "strings" + "testing" + + "github.com/kopia/kopia/tests/testenv" +) + +func TestFIORun(t *testing.T) { + r, err := NewRunner() + testenv.AssertNoError(t, err) + + defer r.Cleanup() + + stdout, stderr, err := r.Run() + if err == nil { + t.Fatal("Expected error to be set as no params were passed") + } + + if !strings.Contains(stderr, "No job(s) defined") { + t.Fatal("Expected an error indicating no jobs were defined") + } + + if !strings.Contains(stdout, "Print this page") { + // Indicates the --help page has been printed + t.Fatal("Expected --help page when running fio with no args") + } +} + +func TestFIORunConfig(t *testing.T) { + r, err := NewRunner() + testenv.AssertNoError(t, err) + + defer r.Cleanup() + + cfg := Config{ + { + Name: "write-10g", + Options: map[string]string{ + "size": "1g", + "nrfiles": "10", + }, + }, + } + stdout, stderr, err := r.RunConfigs(cfg) + testenv.AssertNoError(t, err) + + if stderr != "" { + t.Error("Stderr was not empty") + } + + if !strings.Contains(stdout, "rw=write") { + t.Error("Expected the output to indicate writes took place") + } +} + +func TestFIOGlobalConfigOverride(t *testing.T) { + r, err := NewRunner() + testenv.AssertNoError(t, err) + + defer r.Cleanup() + + cfgs := []Config{ + { + { + Name: "global", + Options: map[string]string{ + "rw": "read", + }, + }, + { + Name: "write-10g", + Options: map[string]string{ + "size": "1g", + "nrfiles": "10", + }, + }, + }, + } + stdout, _, err := r.RunConfigs(cfgs...) + testenv.AssertNoError(t, err) + + if !strings.Contains(stdout, "rw=read") { + t.Fatal("Expected the global config 'rw' flag to be overwritten by the passed config") + } +} diff --git a/tests/tools/fio/job.go b/tests/tools/fio/job.go new file mode 100644 index 000000000..17fc94dda --- /dev/null +++ b/tests/tools/fio/job.go @@ -0,0 +1,29 @@ +package fio + +import ( + "fmt" + "strings" +) + +// Job represents the configuration for running a FIO job +type Job struct { + Name string + Options Options +} + +// String implements the stringer interface, formats the Job +// as it would appear in a well-formed fio config file. +func (job Job) String() string { + ret := []string{fmt.Sprintf("[%s]", job.Name)} + + for k, v := range job.Options { + if v == "" { + ret = append(ret, k) + continue + } + + ret = append(ret, fmt.Sprintf("%s=%s", k, v)) + } + + return strings.Join(ret, "\n") +} diff --git a/tests/tools/fio/main_test.go b/tests/tools/fio/main_test.go new file mode 100644 index 000000000..1910a96b4 --- /dev/null +++ b/tests/tools/fio/main_test.go @@ -0,0 +1,19 @@ +package fio + +import ( + "fmt" + "os" + "testing" +) + +func TestMain(m *testing.M) { + Exe := os.Getenv("FIO_EXE") + if Exe == "" { + fmt.Println("Skipping fio tests if FIO_EXE is not set") + os.Exit(0) + } + + result := m.Run() + + os.Exit(result) +} diff --git a/tests/tools/fio/options.go b/tests/tools/fio/options.go new file mode 100644 index 000000000..c8f3a6c7b --- /dev/null +++ b/tests/tools/fio/options.go @@ -0,0 +1,20 @@ +package fio + +// Options are flags to be set when running fio +type Options map[string]string + +// Merge will merge two Options, overwriting common option keys +// with the incoming option values. Returns the merged result +func (o Options) Merge(other Options) map[string]string { + out := make(map[string]string, len(o)+len(other)) + + for k, v := range o { + out[k] = v + } + + for k, v := range other { + out[k] = v + } + + return out +} diff --git a/tests/tools/fio/workload.go b/tests/tools/fio/workload.go new file mode 100644 index 000000000..5ee021893 --- /dev/null +++ b/tests/tools/fio/workload.go @@ -0,0 +1,32 @@ +package fio + +import ( + "fmt" + "os" + "path/filepath" + "strconv" +) + +// WriteFiles writes files to the directory specified by path, up to the +// provided size and number of files +func (fr *Runner) WriteFiles(path string, sizeB int64, numFiles int, opt Options) error { + fullPath := filepath.Join(fr.DataDir, path) + + err := os.MkdirAll(fullPath, 0700) + if err != nil { + return err + } + + _, _, err = fr.RunConfigs(Config{ + { + Name: fmt.Sprintf("write-%vB-%v", sizeB, numFiles), + Options: opt.Merge(Options{ + "size": strconv.Itoa(int(sizeB)), + "nrfiles": strconv.Itoa(numFiles), + "directory": fullPath, + }), + }, + }) + + return err +} diff --git a/tests/tools/fio/workload_test.go b/tests/tools/fio/workload_test.go new file mode 100644 index 000000000..17ab0e0c8 --- /dev/null +++ b/tests/tools/fio/workload_test.go @@ -0,0 +1,45 @@ +package fio + +import ( + "fmt" + "io/ioutil" + "path/filepath" + "testing" + + "github.com/kopia/kopia/tests/testenv" +) + +func TestWriteFiles(t *testing.T) { + r, err := NewRunner() + testenv.AssertNoError(t, err) + + defer r.Cleanup() + + relativeWritePath := "some/path/to/check" + writeSizeB := int64(256 * 1024 * 1024) // 256 MiB + numFiles := 13 + + // Test a call to WriteFiles + err = r.WriteFiles(relativeWritePath, writeSizeB, numFiles, Options{}) + testenv.AssertNoError(t, err) + + fullPath := filepath.Join(r.DataDir, relativeWritePath) + dir, err := ioutil.ReadDir(fullPath) + testenv.AssertNoError(t, err) + + if got, want := len(dir), numFiles; got != want { + t.Errorf("Did not get expected number of files %v (actual) != %v (expected", got, want) + } + + sizeTot := int64(0) + + for _, fi := range dir { + fmt.Println(fi.Name(), fi.Size()) + sizeTot += fi.Size() + } + + want := (writeSizeB / int64(numFiles)) * int64(numFiles) + if got := sizeTot; got != want { + t.Errorf("Did not get the expected amount of data written %v (actual) != %v (expected)", got, want) + } +}