diff --git a/.github/workflows/endurance-test.yml b/.github/workflows/endurance-test.yml new file mode 100644 index 000000000..8cf879acb --- /dev/null +++ b/.github/workflows/endurance-test.yml @@ -0,0 +1,25 @@ +name: Endurance Test +on: + push: + branches: [ master ] + tags: + - v* + schedule: + # run on Mondays at 8AM + - cron: '0 8 * * 1' +jobs: + endurance-test: + name: Endurance Test + runs-on: ubuntu-latest + steps: + - name: Set up Go. + uses: actions/setup-go@v2 + with: + go-version: ^1.16 + id: go + - name: Check out code into the Go module directory + uses: actions/checkout@v2 + with: + fetch-depth: 0 + - name: Endurance Tests + run: make endurance-tests diff --git a/Makefile b/Makefile index a5f473e58..55724b9a1 100644 --- a/Makefile +++ b/Makefile @@ -223,7 +223,7 @@ integration-tests: build-integration-test-binary $(gotestsum) $(TESTING_ACTION_E endurance-tests: export KOPIA_EXE ?= $(KOPIA_INTEGRATION_EXE) endurance-tests: build-integration-test-binary $(gotestsum) - $(GO_TEST) $(TEST_FLAGS) -count=$(REPEAT_TEST) -parallel $(PARALLEL) -timeout 3600s github.com/kopia/kopia/tests/endurance_test + go test -v $(TEST_FLAGS) -count=$(REPEAT_TEST) -parallel $(PARALLEL) -timeout 3600s github.com/kopia/kopia/tests/endurance_test robustness-tests: export KOPIA_EXE ?= $(KOPIA_INTEGRATION_EXE) robustness-tests: GOTESTSUM_FORMAT=testname diff --git a/tests/endurance_test/endurance_test.go b/tests/endurance_test/endurance_test.go index 56f9c7a6d..7db31a7df 100644 --- a/tests/endurance_test/endurance_test.go +++ b/tests/endurance_test/endurance_test.go @@ -8,6 +8,7 @@ "math/rand" "net/http/httptest" "os" + "sync/atomic" "testing" "time" @@ -19,7 +20,14 @@ const ( maxSourcesPerEnduranceRunner = 3 enduranceRunnerCount = 3 - runnerIterations = 1000 +) + +var ( + // We will simulate 2 weeks of running with clock moving by a lot every time it's read. + startTime = time.Date(2000, 1, 1, 0, 0, 0, 0, time.UTC) + simulatedDuration = 14 * 24 * time.Hour + endTime = startTime.Add(simulatedDuration) + tickIncrement = 350 * time.Millisecond ) type webdavDirWithFakeClock struct { @@ -60,7 +68,7 @@ func TestEndurance(t *testing.T) { defer os.RemoveAll(tmpDir) - fts := testenv.NewFakeTimeServer(time.Date(2000, 1, 1, 0, 0, 0, 0, time.Local), 100*time.Millisecond) + fts := testenv.NewFakeTimeServer(startTime, tickIncrement) ft := httptest.NewServer(fts) defer ft.Close() @@ -75,14 +83,21 @@ func TestEndurance(t *testing.T) { e.RunAndExpectSuccess(t, "repo", "create", "webdav", "--url", sts.URL) + failureCount := new(int32) + t.Run("Runners", func(t *testing.T) { for i := 0; i < enduranceRunnerCount; i++ { i := i t.Run(fmt.Sprintf("Runner-%v", i), func(t *testing.T) { t.Parallel() + defer func() { + if t.Failed() { + atomic.AddInt32(failureCount, 1) + } + }() - enduranceRunner(t, i, ft.URL, sts.URL) + enduranceRunner(t, i, ft.URL, sts.URL, failureCount, fts.Now) }) } }) @@ -94,6 +109,7 @@ func TestEndurance(t *testing.T) { type runnerState struct { dirs []string snapshottedAnything bool + runnerID int } type action func(t *testing.T, e *testenv.CLITest, s *runnerState) @@ -109,6 +125,7 @@ type runnerState struct { {actionMutateDirectoryTree, 1}, {actionSnapshotVerify, 10}, {actionContentVerify, 5}, + {actionMaintenance, 5}, } func actionSnapshotExisting(t *testing.T, e *testenv.CLITest, s *runnerState) { @@ -150,6 +167,14 @@ func actionContentVerify(t *testing.T, e *testenv.CLITest, s *runnerState) { e.RunAndExpectSuccess(t, "content", "verify") } +func actionMaintenance(t *testing.T, e *testenv.CLITest, s *runnerState) { + t.Helper() + + if s.runnerID == 0 { + e.RunAndExpectSuccess(t, "maintenance", "run", "--full") + } +} + func actionAddNewSource(t *testing.T, e *testenv.CLITest, s *runnerState) { t.Helper() @@ -203,7 +228,7 @@ func pickRandomEnduranceTestAction() action { panic("impossible") } -func enduranceRunner(t *testing.T, runnerID int, fakeTimeServer, webdavServer string) { +func enduranceRunner(t *testing.T, runnerID int, fakeTimeServer, webdavServer string, failureCount *int32, nowFunc func() time.Time) { t.Helper() e := testenv.NewCLITest(t) @@ -221,10 +246,19 @@ func enduranceRunner(t *testing.T, runnerID int, fakeTimeServer, webdavServer st var s runnerState + s.runnerID = runnerID + actionAddNewSource(t, e, &s) - for k := 0; k < runnerIterations; k++ { - t.Logf("ITERATION %v / %v", k, runnerIterations) + for now, k := nowFunc(), 0; now.Before(endTime); now, k = nowFunc(), k+1 { + if atomic.LoadInt32(failureCount) != 0 { + t.Logf("Aborting early because of failures.") + break + } + + percent := 100 * now.Sub(startTime).Seconds() / endTime.Sub(startTime).Seconds() + + t.Logf("ITERATION %v NOW=%v (%.2f %%)", k, now, percent) act := pickRandomEnduranceTestAction() act(t, e, &s)