From 5179ad2cd204de67a6f3b5fd130928d768030c13 Mon Sep 17 00:00:00 2001 From: Jarek Kowalski Date: Mon, 17 May 2021 21:47:11 -0700 Subject: [PATCH] cli: test + misc improvements (#1083) * cli: Added --max-examples-per-bucket flag to 'kopia snapshot estimate' Added and cleaned up a bunch of unit tests. Fixes #1054 * cli: misc tests to increase code coverage of the cli package * ci: move code coverage run into separate GH job --- .github/workflows/code-coverage.yml | 20 +++ .github/workflows/make.yml | 2 - Makefile | 2 +- cli/command_blob_show_test.go | 22 ++++ cli/command_cache_set_test.go | 49 +++++++ cli/command_cache_sync_test.go | 41 ++++++ cli/command_content_verify_test.go | 40 ++++++ cli/command_index_inspect_test.go | 17 +++ cli/command_server_start.go | 21 +-- cli/command_snapshot_estimate.go | 16 ++- cli/command_snapshot_estimate_test.go | 58 +++++++++ htmlui/src/NewSnapshot.js | 1 + internal/server/api_estimate.go | 49 ++++++- internal/serverapi/client_wrappers.go | 21 +++ internal/serverapi/serverapi.go | 3 +- internal/uitask/uitask.go | 10 ++ snapshot/snapshotfs/estimate.go | 20 ++- snapshot/snapshotfs/upload_scan.go | 2 +- tests/end_to_end_test/server_start_test.go | 141 +++++++++------------ 19 files changed, 423 insertions(+), 112 deletions(-) create mode 100644 .github/workflows/code-coverage.yml create mode 100644 cli/command_blob_show_test.go create mode 100644 cli/command_cache_set_test.go create mode 100644 cli/command_cache_sync_test.go create mode 100644 cli/command_content_verify_test.go create mode 100644 cli/command_index_inspect_test.go create mode 100644 cli/command_snapshot_estimate_test.go diff --git a/.github/workflows/code-coverage.yml b/.github/workflows/code-coverage.yml new file mode 100644 index 000000000..a9e64eddd --- /dev/null +++ b/.github/workflows/code-coverage.yml @@ -0,0 +1,20 @@ +name: Code Coverage +on: [push,pull_request] +jobs: + tests: + name: Code Coverage Tests + runs-on: ubuntu-latest + steps: + - name: Set up Go. + uses: actions/setup-go@v2 + with: + go-version: ^1.16 + id: go + - name: Check out code into the Go module directory + uses: actions/checkout@v2 + with: + fetch-depth: 0 + - name: Run Tests + run: make test-with-coverage + - name: Publish Coverage Results + run: make ci-publish-coverage diff --git a/.github/workflows/make.yml b/.github/workflows/make.yml index b336e2915..8cb057d7c 100644 --- a/.github/workflows/make.yml +++ b/.github/workflows/make.yml @@ -103,8 +103,6 @@ jobs: - name: Integration Tests run: make -j2 ci-integration-tests continue-on-error: ${{ github.event_name != 'pull_request' }} - - name: Publish Coverage Results - run: make ci-publish-coverage - name: Upload Kopia Artifacts uses: actions/upload-artifact@v2 with: diff --git a/Makefile b/Makefile index 2da80a9be..09faf4765 100644 --- a/Makefile +++ b/Makefile @@ -151,7 +151,7 @@ ifeq ($(GOARCH),amd64) $(retry) $(MAKE) kopia-ui endif -ci-tests: lint vet test-with-coverage +ci-tests: lint vet test ci-integration-tests: integration-tests robustness-tool-tests $(MAKE) stress-test diff --git a/cli/command_blob_show_test.go b/cli/command_blob_show_test.go new file mode 100644 index 000000000..ebf0880e8 --- /dev/null +++ b/cli/command_blob_show_test.go @@ -0,0 +1,22 @@ +package cli_test + +import ( + "strings" + "testing" + + "github.com/kopia/kopia/tests/testenv" +) + +func TestBlobShow(t *testing.T) { + env := testenv.NewCLITest(t, testenv.NewInProcRunner(t)) + + env.RunAndExpectSuccess(t, "repo", "create", "filesystem", "--path", env.RepoDir) + + someNBlob := strings.Split(env.RunAndExpectSuccess(t, "blob", "list", "--prefix=n")[0], " ")[0] + someQBlob := strings.Split(env.RunAndExpectSuccess(t, "blob", "list", "--prefix=q")[0], " ")[0] + env.RunAndExpectSuccess(t, "blob", "show", someNBlob) + env.RunAndExpectSuccess(t, "blob", "show", someQBlob) + env.RunAndExpectSuccess(t, "blob", "show", "--decrypt", someNBlob) + // --decrypt will be ignored + env.RunAndExpectSuccess(t, "blob", "show", "--decrypt", someQBlob) +} diff --git a/cli/command_cache_set_test.go b/cli/command_cache_set_test.go new file mode 100644 index 000000000..46a009f98 --- /dev/null +++ b/cli/command_cache_set_test.go @@ -0,0 +1,49 @@ +package cli_test + +import ( + "strings" + "testing" + + "github.com/stretchr/testify/require" + + "github.com/kopia/kopia/internal/testutil" + "github.com/kopia/kopia/tests/testenv" +) + +func TestCacheSet(t *testing.T) { + env := testenv.NewCLITest(t, testenv.NewInProcRunner(t)) + + env.RunAndExpectSuccess(t, "repo", "create", "filesystem", "--path", env.RepoDir) + + // no changes + env.RunAndExpectFailure(t, "cache", "set") + + ncd := testutil.TempDirectory(t) + env.RunAndExpectSuccess(t, + "cache", "set", + "--cache-directory", ncd, + "--content-cache-size-mb=33", + "--metadata-cache-size-mb=44", + "--max-list-cache-duration=55s", + ) + + out := env.RunAndExpectSuccess(t, "cache", "info") + require.Contains(t, mustGetLineContaining(t, out, "33 MB"), ncd) + require.Contains(t, mustGetLineContaining(t, out, "33 MB"), "contents") + require.Contains(t, mustGetLineContaining(t, out, "44 MB"), "metadata") + require.Contains(t, mustGetLineContaining(t, out, "55s"), "blob-list") +} + +func mustGetLineContaining(t *testing.T, lines []string, containing string) string { + t.Helper() + + for _, l := range lines { + if strings.Contains(l, containing) { + return l + } + } + + t.Fatalf("no line containing %q found in %v", containing, lines) + + return "" +} diff --git a/cli/command_cache_sync_test.go b/cli/command_cache_sync_test.go new file mode 100644 index 000000000..e33b0bd33 --- /dev/null +++ b/cli/command_cache_sync_test.go @@ -0,0 +1,41 @@ +package cli_test + +import ( + "testing" + + "github.com/stretchr/testify/require" + + "github.com/kopia/kopia/internal/testutil" + "github.com/kopia/kopia/tests/testenv" +) + +func TestCacheClearSync(t *testing.T) { + env := testenv.NewCLITest(t, testenv.NewInProcRunner(t)) + + emptyDir := testutil.TempDirectory(t) + + env.RunAndExpectSuccess(t, "repo", "create", "filesystem", "--path", env.RepoDir) + env.RunAndExpectSuccess(t, "snapshot", "create", emptyDir) + env.RunAndExpectSuccess(t, "snapshot", "create", emptyDir) + env.RunAndExpectSuccess(t, "snapshot", "create", emptyDir) + env.RunAndExpectSuccess(t, "snapshot", "create", emptyDir) + env.RunAndExpectSuccess(t, "snapshot", "create", emptyDir) + + oldMetadataLine := mustGetLineContaining(t, env.RunAndExpectSuccess(t, "cache", "info"), "metadata") + oldMetadataLine2 := mustGetLineContaining(t, env.RunAndExpectSuccess(t, "cache", "info"), "metadata") + require.Equal(t, oldMetadataLine, oldMetadataLine2) + + env.RunAndExpectSuccess(t, "cache", "clear") + + newMetadataLine := mustGetLineContaining(t, env.RunAndExpectSuccess(t, "cache", "info"), "metadata") + require.NotEqual(t, oldMetadataLine, newMetadataLine) + + env.RunAndExpectSuccess(t, "cache", "sync") + newerMetadataLine := mustGetLineContaining(t, env.RunAndExpectSuccess(t, "cache", "info"), "metadata") + + env.RunAndExpectSuccess(t, "cache", "sync") + newerMetadataLine2 := mustGetLineContaining(t, env.RunAndExpectSuccess(t, "cache", "info"), "metadata") + + require.NotEqual(t, newerMetadataLine, newMetadataLine) + require.Equal(t, newerMetadataLine, newerMetadataLine2) +} diff --git a/cli/command_content_verify_test.go b/cli/command_content_verify_test.go new file mode 100644 index 000000000..91061f82b --- /dev/null +++ b/cli/command_content_verify_test.go @@ -0,0 +1,40 @@ +package cli_test + +import ( + "bytes" + "io/ioutil" + "path/filepath" + "strings" + "testing" + + "github.com/stretchr/testify/require" + + "github.com/kopia/kopia/internal/testutil" + "github.com/kopia/kopia/tests/testenv" +) + +func TestContentVerify(t *testing.T) { + env := testenv.NewCLITest(t, testenv.NewInProcRunner(t)) + + dir := testutil.TempDirectory(t) + require.NoError(t, ioutil.WriteFile(filepath.Join(dir, "file1.txt"), bytes.Repeat([]byte{1, 2, 3, 4, 5}, 15000), 0o600)) + + env.RunAndExpectSuccess(t, "repo", "create", "filesystem", "--path", env.RepoDir) + env.RunAndExpectSuccess(t, "content", "verify") + env.RunAndExpectSuccess(t, "snapshot", "create", dir) + env.RunAndExpectSuccess(t, "content", "verify") + + // delete one of 'p' blobs. + blobIDToDelete := strings.Split(env.RunAndExpectSuccess(t, "blob", "list", "--prefix=p")[0], " ")[0] + blobList := env.RunAndExpectSuccess(t, "blob", "list") + t.Logf("blob list: %v", strings.Join(blobList, "\n")) + env.RunAndExpectSuccess(t, "blob", "delete", blobIDToDelete) + + _, verifyStderr, err := env.Run(t, true, "content", "verify") + require.Error(t, err) + + // this fails if not found + mustGetLineContaining(t, verifyStderr, "missing blob "+blobIDToDelete) + + env.RunAndExpectFailure(t, "content", "verify", "--full") +} diff --git a/cli/command_index_inspect_test.go b/cli/command_index_inspect_test.go new file mode 100644 index 000000000..89fb1e2ff --- /dev/null +++ b/cli/command_index_inspect_test.go @@ -0,0 +1,17 @@ +package cli_test + +import ( + "strings" + "testing" + + "github.com/kopia/kopia/tests/testenv" +) + +func TestIndexInspect(t *testing.T) { + env := testenv.NewCLITest(t, testenv.NewInProcRunner(t)) + + env.RunAndExpectSuccess(t, "repo", "create", "filesystem", "--path", env.RepoDir) + + someIndex := strings.Split(env.RunAndExpectSuccess(t, "index", "list")[0], " ")[0] + env.RunAndExpectSuccess(t, "index", "inspect", someIndex) +} diff --git a/cli/command_server_start.go b/cli/command_server_start.go index d241d69e2..a6fc1214c 100644 --- a/cli/command_server_start.go +++ b/cli/command_server_start.go @@ -54,6 +54,7 @@ type commandServerStart struct { serverStartTLSGenerateCertValidDays int serverStartTLSGenerateCertNames []string serverStartTLSPrintFullServerCert bool + uiTitlePrefix string sf serverFlags svc advancedAppServices @@ -88,6 +89,8 @@ func (c *commandServerStart) setup(svc advancedAppServices, parent commandParent cmd.Flag("tls-generate-cert-name", "Host names/IP addresses to generate TLS certificate for").Default("127.0.0.1").Hidden().StringsVar(&c.serverStartTLSGenerateCertNames) cmd.Flag("tls-print-server-cert", "Print server certificate").Hidden().BoolVar(&c.serverStartTLSPrintFullServerCert) + cmd.Flag("ui-title-prefix", "UI title prefix").Hidden().Envar("KOPIA_UI_TITLE_PREFIX").StringVar(&c.uiTitlePrefix) + c.sf.setup(cmd) c.co.setup(cmd) c.svc = svc @@ -133,10 +136,10 @@ func (c *commandServerStart) run(ctx context.Context, rep repo.Repository) error mux.Handle("/api/", srv.APIHandlers(c.serverStartLegacyRepositoryAPI)) if c.serverStartHTMLPath != "" { - fileServer := srv.RequireUIUserAuth(serveIndexFileForKnownUIRoutes(http.Dir(c.serverStartHTMLPath))) + fileServer := srv.RequireUIUserAuth(c.serveIndexFileForKnownUIRoutes(http.Dir(c.serverStartHTMLPath))) mux.Handle("/", fileServer) } else if c.serverStartUI { - mux.Handle("/", srv.RequireUIUserAuth(serveIndexFileForKnownUIRoutes(server.AssetFile()))) + mux.Handle("/", srv.RequireUIUserAuth(c.serveIndexFileForKnownUIRoutes(server.AssetFile()))) } httpServer := &http.Server{Addr: stripProtocol(c.sf.serverAddress)} @@ -215,16 +218,16 @@ func stripProtocol(addr string) string { return strings.TrimPrefix(strings.TrimPrefix(addr, "https://"), "http://") } -func isKnownUIRoute(path string) bool { +func (c *commandServerStart) isKnownUIRoute(path string) bool { return strings.HasPrefix(path, "/snapshots") || strings.HasPrefix(path, "/policies") || strings.HasPrefix(path, "/tasks") || strings.HasPrefix(path, "/repo") } -func patchIndexBytes(b []byte) []byte { - if prefix := os.Getenv("KOPIA_UI_TITLE_PREFIX"); prefix != "" { - b = bytes.ReplaceAll(b, []byte(""), []byte("<title>"+html.EscapeString(prefix))) +func (c *commandServerStart) patchIndexBytes(b []byte) []byte { + if c.uiTitlePrefix != "" { + b = bytes.ReplaceAll(b, []byte("<title>"), []byte("<title>"+html.EscapeString(c.uiTitlePrefix))) } return b @@ -246,14 +249,14 @@ func maybeReadIndexBytes(fs http.FileSystem) []byte { return rd } -func serveIndexFileForKnownUIRoutes(fs http.FileSystem) http.Handler { +func (c *commandServerStart) serveIndexFileForKnownUIRoutes(fs http.FileSystem) http.Handler { h := http.FileServer(fs) // read bytes from 'index.html' and patch based on optional environment variables. - indexBytes := patchIndexBytes(maybeReadIndexBytes(fs)) + indexBytes := c.patchIndexBytes(maybeReadIndexBytes(fs)) return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - if isKnownUIRoute(r.URL.Path) { + if c.isKnownUIRoute(r.URL.Path) { r2 := new(http.Request) *r2 = *r r2.URL = new(url.URL) diff --git a/cli/command_snapshot_estimate.go b/cli/command_snapshot_estimate.go index 004b1347c..6bd7b120a 100644 --- a/cli/command_snapshot_estimate.go +++ b/cli/command_snapshot_estimate.go @@ -21,6 +21,7 @@ type commandSnapshotEstimate struct { snapshotEstimateShowFiles bool snapshotEstimateQuiet bool snapshotEstimateUploadSpeed float64 + maxExamplesPerBucket int out textOutput } @@ -31,6 +32,7 @@ func (c *commandSnapshotEstimate) setup(svc appServices, parent commandParent) { cmd.Flag("show-files", "Show files").BoolVar(&c.snapshotEstimateShowFiles) cmd.Flag("quiet", "Do not display scanning progress").Short('q').BoolVar(&c.snapshotEstimateQuiet) cmd.Flag("upload-speed", "Upload speed to use for estimation").Default("10").PlaceHolder("mbit/s").Float64Var(&c.snapshotEstimateUploadSpeed) + cmd.Flag("max-examples-per-bucket", "Max examples per bucket").Default("10").IntVar(&c.maxExamplesPerBucket) cmd.Action(svc.repositoryReaderAction(c.run)) c.out.setup(svc) } @@ -95,33 +97,33 @@ func (c *commandSnapshotEstimate) run(ctx context.Context, rep repo.Repository) return errors.Wrapf(err, "error creating policy tree for %v", sourceInfo) } - if err := snapshotfs.Estimate(ctx, rep, dir, policyTree, &ep); err != nil { + if err := snapshotfs.Estimate(ctx, rep, dir, policyTree, &ep, c.maxExamplesPerBucket); err != nil { return errors.Wrap(err, "error estimating") } - c.out.printStdout("Snapshot includes %v files, total size %v\n", ep.stats.TotalFileCount, units.BytesStringBase10(ep.stats.TotalFileSize)) + c.out.printStdout("Snapshot includes %v file(s), total size %v\n", ep.stats.TotalFileCount, units.BytesStringBase10(ep.stats.TotalFileSize)) c.showBuckets(ep.included, c.snapshotEstimateShowFiles) c.out.printStdout("\n") if ep.stats.ExcludedFileCount > 0 { - c.out.printStdout("Snapshot excludes %v files, total size %v\n", ep.stats.ExcludedFileCount, ep.stats.ExcludedTotalFileSize) + c.out.printStdout("Snapshot excludes %v file(s), total size %v\n", ep.stats.ExcludedFileCount, units.BytesStringBase10(ep.stats.ExcludedTotalFileSize)) c.showBuckets(ep.excluded, true) } else { - c.out.printStdout("Snapshots excludes no files.\n") + c.out.printStdout("Snapshot excludes no files.\n") } if ep.stats.ExcludedDirCount > 0 { - c.out.printStdout("Snapshots excludes %v directories. Examples:\n", ep.stats.ExcludedDirCount) + c.out.printStdout("Snapshot excludes %v directories. Examples:\n", ep.stats.ExcludedDirCount) for _, ed := range ep.excludedDirs { c.out.printStdout(" - %v\n", ed) } } else { - c.out.printStdout("Snapshots excludes no directories.\n") + c.out.printStdout("Snapshot excludes no directories.\n") } if ep.stats.ErrorCount > 0 { - c.out.printStdout("Encountered %v errors.\n", ep.stats.ErrorCount) + c.out.printStdout("Encountered %v error(s).\n", ep.stats.ErrorCount) } megabits := float64(ep.stats.TotalFileSize) * 8 / 1000000 //nolint:gomnd diff --git a/cli/command_snapshot_estimate_test.go b/cli/command_snapshot_estimate_test.go new file mode 100644 index 000000000..c7a791ea3 --- /dev/null +++ b/cli/command_snapshot_estimate_test.go @@ -0,0 +1,58 @@ +package cli_test + +import ( + "bytes" + "io/ioutil" + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/require" + + "github.com/kopia/kopia/internal/testutil" + "github.com/kopia/kopia/tests/testenv" +) + +func TestSnapshotEstimate(t *testing.T) { + env := testenv.NewCLITest(t, testenv.NewInProcRunner(t)) + + dir := testutil.TempDirectory(t) + require.NoError(t, ioutil.WriteFile(filepath.Join(dir, "file1.txt"), bytes.Repeat([]byte{1, 2, 3, 4, 5}, 15000), 0o600)) + require.NoError(t, ioutil.WriteFile(filepath.Join(dir, "file2.txt"), bytes.Repeat([]byte{2, 3, 4, 5, 6}, 10000), 0o600)) + require.NoError(t, os.MkdirAll(filepath.Join(dir, "subdir"), 0o755)) + require.NoError(t, ioutil.WriteFile(filepath.Join(dir, "subdir", "file2.txt"), bytes.Repeat([]byte{3, 4, 5, 6, 7}, 5000), 0o600)) + + env.RunAndExpectSuccess(t, "repo", "create", "filesystem", "--path", env.RepoDir) + out := env.RunAndExpectSuccess(t, "snapshot", "estimate", dir) + require.Contains(t, out, "Snapshot includes 3 file(s), total size 150 KB") + require.Contains(t, out, "Snapshot excludes no files.") + require.Contains(t, out, "Snapshot excludes no directories.") + + // ignore some files + env.RunAndExpectSuccess(t, "policy", "set", "--add-ignore", "*2.txt", dir) + out = env.RunAndExpectSuccess(t, "snapshot", "estimate", dir) + require.Contains(t, out, "Snapshot includes 1 file(s), total size 75 KB") + require.Contains(t, out, "Snapshot excludes 2 file(s), total size 75 KB") + require.Contains(t, out, " - ./file2.txt - 50 KB") + require.Contains(t, out, " - ./subdir/file2.txt - 25 KB") + require.Contains(t, out, "Snapshot excludes no directories.") + + // ignore a dir + env.RunAndExpectSuccess(t, "policy", "set", "--add-ignore", "subdir", dir) + out = env.RunAndExpectSuccess(t, "snapshot", "estimate", dir) + require.Contains(t, out, "Snapshot includes 1 file(s), total size 75 KB") + require.Contains(t, out, "Snapshot excludes 1 file(s), total size 50 KB") + require.Contains(t, out, " - ./file2.txt - 50 KB") + require.Contains(t, out, " - ./subdir") + require.Contains(t, out, "Snapshot excludes 1 directories. Examples:") +} + +func TestSnapshotEstimate_NotADirectory(t *testing.T) { + env := testenv.NewCLITest(t, testenv.NewInProcRunner(t)) + + dir := testutil.TempDirectory(t) + require.NoError(t, ioutil.WriteFile(filepath.Join(dir, "file1.txt"), []byte{1, 2, 3}, 0o600)) + + env.RunAndExpectSuccess(t, "repo", "create", "filesystem", "--path", env.RepoDir) + env.RunAndExpectFailure(t, "snapshot", "estimate", filepath.Join(dir, "file1.txt")) +} diff --git a/htmlui/src/NewSnapshot.js b/htmlui/src/NewSnapshot.js index 629643512..1b5909ed4 100644 --- a/htmlui/src/NewSnapshot.js +++ b/htmlui/src/NewSnapshot.js @@ -65,6 +65,7 @@ export class NewSnapshot extends Component { let req = { root: this.state.path, + maxExamplesPerBucket: 10, } axios.post('/api/v1/estimate', req).then(result => { diff --git a/internal/server/api_estimate.go b/internal/server/api_estimate.go index 6a248b88e..f750644b8 100644 --- a/internal/server/api_estimate.go +++ b/internal/server/api_estimate.go @@ -3,6 +3,7 @@ package server import ( "context" "encoding/json" + "fmt" "net/http" "path/filepath" @@ -14,6 +15,7 @@ import ( "github.com/kopia/kopia/internal/ospath" "github.com/kopia/kopia/internal/serverapi" "github.com/kopia/kopia/internal/uitask" + "github.com/kopia/kopia/internal/units" "github.com/kopia/kopia/snapshot" "github.com/kopia/kopia/snapshot/policy" "github.com/kopia/kopia/snapshot/snapshotfs" @@ -45,6 +47,51 @@ func (p estimateTaskProgress) Stats(ctx context.Context, st *snapshot.Stats, inc "Errors": uitask.ErrorCounter(int64(st.ErrorCount)), "Ignored Errors": uitask.ErrorCounter(int64(st.IgnoredErrorCount)), }) + + if final { + logBucketSamples(ctx, included, "Included", false) + logBucketSamples(ctx, excluded, "Excluded", true) + } +} + +func logBucketSamples(ctx context.Context, buckets snapshotfs.SampleBuckets, prefix string, showExamples bool) { + any := false + + for i, bucket := range buckets { + if bucket.Count == 0 { + continue + } + + var sizeRange string + + if i == 0 { + sizeRange = fmt.Sprintf("< %-6v", + units.BytesStringBase10(bucket.MinSize)) + } else { + sizeRange = fmt.Sprintf("%-6v...%6v", + units.BytesStringBase10(bucket.MinSize), + units.BytesStringBase10(buckets[i-1].MinSize)) + } + + log(ctx).Infof("%v files %v: %7v files, total size %v\n", + prefix, + sizeRange, + bucket.Count, units.BytesStringBase10(bucket.TotalSize)) + + any = true + + if showExamples && len(bucket.Examples) > 0 { + log(ctx).Infof("Examples:") + + for _, sample := range bucket.Examples { + log(ctx).Infof(" - %v\n", sample) + } + } + } + + if !any { + log(ctx).Infof("%v files: None", prefix) + } } var _ snapshotfs.EstimateProgress = estimateTaskProgress{} @@ -94,7 +141,7 @@ func (s *Server) handleEstimate(ctx context.Context, r *http.Request, body []byt } // nolint:wrapcheck - return snapshotfs.Estimate(estimatectx, rep, dir, policyTree, estimateTaskProgress{ctrl}) + return snapshotfs.Estimate(estimatectx, rep, dir, policyTree, estimateTaskProgress{ctrl}, req.MaxExamplesPerBucket) }) taskID := <-taskIDChan diff --git a/internal/serverapi/client_wrappers.go b/internal/serverapi/client_wrappers.go index cb8a81e0d..18608359c 100644 --- a/internal/serverapi/client_wrappers.go +++ b/internal/serverapi/client_wrappers.go @@ -7,6 +7,7 @@ import ( "github.com/pkg/errors" "github.com/kopia/kopia/internal/apiclient" + "github.com/kopia/kopia/internal/uitask" "github.com/kopia/kopia/repo/object" "github.com/kopia/kopia/snapshot" ) @@ -21,6 +22,26 @@ func CreateSnapshotSource(ctx context.Context, c *apiclient.KopiaAPIClient, req return resp, nil } +// Estimate starts snapshot estimation task for a given directory. +func Estimate(ctx context.Context, c *apiclient.KopiaAPIClient, req *EstimateRequest) (*uitask.Info, error) { + resp := &uitask.Info{} + if err := c.Post(ctx, "estimate", req, resp); err != nil { + return nil, errors.Wrap(err, "Estimate") + } + + return resp, nil +} + +// GetTask starts snapshot estimation task for a given directory. +func GetTask(ctx context.Context, c *apiclient.KopiaAPIClient, taskID string) (*uitask.Info, error) { + resp := &uitask.Info{} + if err := c.Get(ctx, "tasks/"+taskID, nil, resp); err != nil { + return nil, errors.Wrap(err, "GetTask") + } + + return resp, nil +} + // UploadSnapshots triggers snapshot upload on matching snapshots. func UploadSnapshots(ctx context.Context, c *apiclient.KopiaAPIClient, match *snapshot.SourceInfo) (*MultipleSourceActionResponse, error) { resp := &MultipleSourceActionResponse{} diff --git a/internal/serverapi/serverapi.go b/internal/serverapi/serverapi.go index a99d9a181..21bc8e8ef 100644 --- a/internal/serverapi/serverapi.go +++ b/internal/serverapi/serverapi.go @@ -216,5 +216,6 @@ type RestoreRequest struct { // EstimateRequest contains request to estimate the size of the snapshot in a given root. type EstimateRequest struct { - Root string `json:"root"` + Root string `json:"root"` + MaxExamplesPerBucket int `json:"maxExamplesPerBucket"` } diff --git a/internal/uitask/uitask.go b/internal/uitask/uitask.go index 3651d9724..ecd8e2f10 100644 --- a/internal/uitask/uitask.go +++ b/internal/uitask/uitask.go @@ -23,6 +23,16 @@ const ( StatusFailed Status = "FAILED" ) +// IsFinished returns true if the given status is finished. +func (s Status) IsFinished() bool { + switch s { + case StatusCanceled, StatusSuccess, StatusFailed: + return true + default: + return false + } +} + // LogLevel represents the log level associated with LogEntry. type LogLevel int diff --git a/snapshot/snapshotfs/estimate.go b/snapshot/snapshotfs/estimate.go index e6f3957af..fa299a600 100644 --- a/snapshot/snapshotfs/estimate.go +++ b/snapshot/snapshotfs/estimate.go @@ -13,8 +13,6 @@ import ( "github.com/kopia/kopia/snapshot/policy" ) -const maxExamplesPerBucket = 10 - // SampleBucket keeps track of count and total size of files above in certain size range and // includes small number of examples of such files. type SampleBucket struct { @@ -24,7 +22,7 @@ type SampleBucket struct { Examples []string `json:"examples,omitempty"` } -func (b *SampleBucket) add(fname string, size int64) { +func (b *SampleBucket) add(fname string, size int64, maxExamplesPerBucket int) { b.Count++ b.TotalSize += size @@ -36,10 +34,10 @@ func (b *SampleBucket) add(fname string, size int64) { // SampleBuckets is a collection of buckets for interesting file sizes sorted in descending order. type SampleBuckets []*SampleBucket -func (b SampleBuckets) add(fname string, size int64) { +func (b SampleBuckets) add(fname string, size int64, maxExamplesPerBucket int) { for _, bucket := range b { if size >= bucket.MinSize { - bucket.add(fname, size) + bucket.add(fname, size, maxExamplesPerBucket) break } } @@ -73,7 +71,7 @@ type EstimateProgress interface { // Estimate walks the provided directory tree and invokes provided progress callback as it discovers // items to be snapshotted. -func Estimate(ctx context.Context, rep repo.Repository, entry fs.Directory, policyTree *policy.Tree, progress EstimateProgress) error { +func Estimate(ctx context.Context, rep repo.Repository, entry fs.Directory, policyTree *policy.Tree, progress EstimateProgress, maxExamplesPerBucket int) error { stats := &snapshot.Stats{} ed := []string{} ib := makeBuckets() @@ -97,16 +95,16 @@ func Estimate(ctx context.Context, rep repo.Repository, entry fs.Directory, poli log(ctx).Debugf("excluded file %v (%v)", relativePath, units.BytesStringBase10(e.Size())) stats.ExcludedFileCount++ stats.ExcludedTotalFileSize += e.Size() - eb.add(relativePath, e.Size()) + eb.add(relativePath, e.Size(), maxExamplesPerBucket) } } entry = ignorefs.New(entry, policyTree, ignorefs.ReportIgnoredFiles(onIgnoredFile)) - return estimate(ctx, ".", entry, policyTree, stats, ib, eb, &ed, progress) + return estimate(ctx, ".", entry, policyTree, stats, ib, eb, &ed, progress, maxExamplesPerBucket) } -func estimate(ctx context.Context, relativePath string, entry fs.Entry, policyTree *policy.Tree, stats *snapshot.Stats, ib, eb SampleBuckets, ed *[]string, progress EstimateProgress) error { +func estimate(ctx context.Context, relativePath string, entry fs.Entry, policyTree *policy.Tree, stats *snapshot.Stats, ib, eb SampleBuckets, ed *[]string, progress EstimateProgress, maxExamplesPerBucket int) error { // see if the context got canceled select { case <-ctx.Done(): @@ -135,7 +133,7 @@ func estimate(ctx context.Context, relativePath string, entry fs.Entry, policyTr progress.Error(ctx, relativePath, err, isIgnored) } else { for _, child := range children { - if err := estimate(ctx, filepath.Join(relativePath, child.Name()), child, policyTree.Child(child.Name()), stats, ib, eb, ed, progress); err != nil { + if err := estimate(ctx, filepath.Join(relativePath, child.Name()), child, policyTree.Child(child.Name()), stats, ib, eb, ed, progress, maxExamplesPerBucket); err != nil { return err } } @@ -144,7 +142,7 @@ func estimate(ctx context.Context, relativePath string, entry fs.Entry, policyTr progress.Stats(ctx, stats, ib, eb, *ed, false) case fs.File: - ib.add(relativePath, entry.Size()) + ib.add(relativePath, entry.Size(), maxExamplesPerBucket) stats.TotalFileCount++ stats.TotalFileSize += entry.Size() } diff --git a/snapshot/snapshotfs/upload_scan.go b/snapshot/snapshotfs/upload_scan.go index 96cf872ca..4c6da5a1e 100644 --- a/snapshot/snapshotfs/upload_scan.go +++ b/snapshot/snapshotfs/upload_scan.go @@ -35,7 +35,7 @@ func (u *Uploader) scanDirectory(ctx context.Context, dir fs.Directory, policyTr return res, nil } - err := Estimate(ctx, u.repo, dir, policyTree, &res) + err := Estimate(ctx, u.repo, dir, policyTree, &res, 1) return res, err } diff --git a/tests/end_to_end_test/server_start_test.go b/tests/end_to_end_test/server_start_test.go index 70c6c81e7..850f3dbe3 100644 --- a/tests/end_to_end_test/server_start_test.go +++ b/tests/end_to_end_test/server_start_test.go @@ -11,11 +11,13 @@ import ( "time" "github.com/pkg/errors" + "github.com/stretchr/testify/require" "github.com/kopia/kopia/internal/apiclient" "github.com/kopia/kopia/internal/retry" "github.com/kopia/kopia/internal/serverapi" "github.com/kopia/kopia/internal/testlogging" + "github.com/kopia/kopia/internal/uitask" "github.com/kopia/kopia/repo/blob" "github.com/kopia/kopia/repo/blob/filesystem" "github.com/kopia/kopia/snapshot" @@ -56,7 +58,7 @@ func (s *serverParameters) ProcessOutput(l string) bool { func TestServerStart(t *testing.T) { ctx := testlogging.Context(t) - runner := testenv.NewExeRunner(t) + runner := testenv.NewInProcRunner(t) e := testenv.NewCLITest(t, runner) defer e.RunAndExpectSuccess(t, "repo", "disconnect") @@ -68,8 +70,6 @@ func TestServerStart(t *testing.T) { var sp serverParameters - runner.Environment = append(runner.Environment, `KOPIA_UI_TITLE_PREFIX=Blah: <script>bleh</script> `) - e.RunAndProcessStderr(t, sp.ProcessOutput, "server", "start", "--ui", @@ -79,6 +79,7 @@ func TestServerStart(t *testing.T) { "--tls-generate-rsa-key-size=2048", // use shorter key size to speed up generation "--override-hostname=fake-hostname", "--override-username=fake-username", + "--ui-title-prefix", "Blah: <script>bleh</script> ", ) t.Logf("detected server parameters %#v", sp) @@ -89,9 +90,7 @@ func TestServerStart(t *testing.T) { TrustedServerCertificateFingerprint: sp.sha256Fingerprint, LogRequests: true, }) - if err != nil { - t.Fatalf("unable to create API apiclient") - } + require.NoError(t, err) defer serverapi.Shutdown(ctx, cli) @@ -99,29 +98,27 @@ func TestServerStart(t *testing.T) { verifyUIServedWithCorrectTitle(t, cli, sp) st := verifyServerConnected(t, cli, true) - if got, want := st.Storage, "filesystem"; got != want { - t.Errorf("unexpected storage type: %v, want %v", got, want) - } + require.Equal(t, "filesystem", st.Storage) sources := verifySourceCount(t, cli, nil, 1) - if got, want := sources[0].Source.Path, sharedTestDataDir1; got != want { - t.Errorf("unexpected source path: %v, want %v", got, want) - } + require.Equal(t, sharedTestDataDir1, sources[0].Source.Path) + + et := estimateSnapshotSize(ctx, t, cli, sharedTestDataDir3) + require.NotEqual(t, int64(0), et.Counters["Bytes"].Value) + require.NotEqual(t, int64(0), et.Counters["Directories"].Value) + require.NotEqual(t, int64(0), et.Counters["Files"].Value) + require.Equal(t, int64(0), et.Counters["Excluded Directories"].Value) + require.Equal(t, int64(0), et.Counters["Excluded Files"].Value) + require.Equal(t, int64(0), et.Counters["Errors"].Value) + require.Equal(t, int64(0), et.Counters["Ignored Errors"].Value) createResp, err := serverapi.CreateSnapshotSource(ctx, cli, &serverapi.CreateSnapshotSourceRequest{ Path: sharedTestDataDir2, }) - if err != nil { - t.Fatalf("create snapshot source error: %v", err) - } + require.NoError(t, err) - if !createResp.Created { - t.Errorf("unexpected value of 'created': %v", createResp.Created) - } - - if createResp.SnapshotStarted { - t.Errorf("unexpected value of 'snapshotStarted': %v", createResp.SnapshotStarted) - } + require.True(t, createResp.Created) + require.False(t, createResp.SnapshotStarted) verifySourceCount(t, cli, nil, 2) verifySourceCount(t, cli, &snapshot.SourceInfo{Host: "no-such-host"}, 0) @@ -135,22 +132,18 @@ func TestServerStart(t *testing.T) { uploadMatchingSnapshots(t, cli, &snapshot.SourceInfo{Host: "fake-hostname", UserName: "fake-username", Path: sharedTestDataDir2}) waitForSnapshotCount(ctx, t, cli, &snapshot.SourceInfo{Host: "fake-hostname", UserName: "fake-username", Path: sharedTestDataDir2}, 1) - if _, err = serverapi.CancelUpload(ctx, cli, nil); err != nil { - t.Fatalf("cancel failed: %v", err) - } + _, err = serverapi.CancelUpload(ctx, cli, nil) + require.NoError(t, err) snaps := verifySnapshotCount(t, cli, &snapshot.SourceInfo{Host: "fake-hostname", UserName: "fake-username", Path: sharedTestDataDir2}, 1) rootPayload, err := serverapi.GetObject(ctx, cli, snaps[0].RootEntry) - if err != nil { - t.Fatalf("getObject %v", err) - } + require.NoError(t, err) // make sure root payload is valid JSON for the directory. var dummy map[string]interface{} - if err = json.Unmarshal(rootPayload, &dummy); err != nil { - t.Fatalf("invalid JSON received: %v", err) - } + err = json.Unmarshal(rootPayload, &dummy) + require.NoError(t, err) keepDaily := 77 @@ -163,27 +156,15 @@ func TestServerStart(t *testing.T) { }, }, }) + require.NoError(t, err) - if err != nil { - t.Fatalf("unable to create source") - } - - if !createResp.SnapshotStarted { - t.Errorf("unexpected value of 'snapshotStarted': %v", createResp.SnapshotStarted) - } + require.True(t, createResp.SnapshotStarted) policies, err := serverapi.ListPolicies(ctx, cli, &snapshot.SourceInfo{Host: "fake-hostname", UserName: "fake-username", Path: sharedTestDataDir3}) - if err != nil { - t.Errorf("aaa") - } + require.NoError(t, err) - if len(policies.Policies) != 1 { - t.Fatalf("unexpected number of policies") - } - - if got, want := *policies.Policies[0].Policy.RetentionPolicy.KeepDaily, keepDaily; got != want { - t.Errorf("initial policy not persisted") - } + require.Len(t, policies.Policies, 1) + require.Equal(t, keepDaily, *policies.Policies[0].Policy.RetentionPolicy.KeepDaily) waitForSnapshotCount(ctx, t, cli, &snapshot.SourceInfo{Host: "fake-hostname", UserName: "fake-username", Path: sharedTestDataDir3}, 1) } @@ -221,9 +202,7 @@ func TestServerCreateAndConnectViaAPI(t *testing.T) { Password: sp.password, TrustedServerCertificateFingerprint: sp.sha256Fingerprint, }) - if err != nil { - t.Fatalf("unable to create API apiclient") - } + require.NoError(t, err) defer serverapi.Shutdown(ctx, cli) @@ -292,9 +271,7 @@ func TestConnectToExistingRepositoryViaAPI(t *testing.T) { Password: sp.password, TrustedServerCertificateFingerprint: sp.sha256Fingerprint, }) - if err != nil { - t.Fatalf("unable to create API apiclient") - } + require.NoError(t, err) defer serverapi.Shutdown(ctx, cli) @@ -365,9 +342,7 @@ func TestServerStartInsecure(t *testing.T) { cli, err := apiclient.NewKopiaAPIClient(apiclient.Options{ BaseURL: sp.baseURL, }) - if err != nil { - t.Fatalf("unable to create API apiclient") - } + require.NoError(t, err) defer serverapi.Shutdown(ctx, cli) @@ -385,9 +360,7 @@ func verifyServerConnected(t *testing.T, cli *apiclient.KopiaAPIClient, want boo t.Helper() st, err := serverapi.Status(testlogging.Context(t), cli) - if err != nil { - t.Fatalf("status error: %v", err) - } + require.NoError(t, err) if got := st.Connected; got != want { t.Errorf("invalid status connected %v, want %v", st.Connected, want) @@ -415,13 +388,33 @@ func waitForSnapshotCount(ctx context.Context, t *testing.T, cli *apiclient.Kopi return nil }, retry.Always) - if err != nil { - t.Fatal(err) - } + + require.NoError(t, err) return result } +func estimateSnapshotSize(ctx context.Context, t *testing.T, cli *apiclient.KopiaAPIClient, dir string) *uitask.Info { + t.Helper() + + estimateTask, err := serverapi.Estimate(ctx, cli, &serverapi.EstimateRequest{ + Root: dir, + MaxExamplesPerBucket: 3, + }) + require.NoError(t, err) + + estimateTaskID := estimateTask.TaskID + + for !estimateTask.Status.IsFinished() { + time.Sleep(1 * time.Second) + + estimateTask, err = serverapi.GetTask(ctx, cli, estimateTaskID) + require.NoError(t, err) + } + + return estimateTask +} + func uploadMatchingSnapshots(t *testing.T, cli *apiclient.KopiaAPIClient, match *snapshot.SourceInfo) { t.Helper() @@ -434,9 +427,7 @@ func verifySnapshotCount(t *testing.T, cli *apiclient.KopiaAPIClient, match *sna t.Helper() snapshots, err := serverapi.ListSnapshots(testlogging.Context(t), cli, match) - if err != nil { - t.Fatalf("error listing sources: %v", err) - } + require.NoError(t, err) if got := len(snapshots.Snapshots); got != want { t.Errorf("unexpected number of snapshots %v, want %v", got, want) @@ -449,9 +440,7 @@ func verifySourceCount(t *testing.T, cli *apiclient.KopiaAPIClient, match *snaps t.Helper() sources, err := serverapi.ListSources(testlogging.Context(t), cli, match) - if err != nil { - t.Fatalf("error listing sources: %v", err) - } + require.NoError(t, err) if got, want := sources.LocalHost, "fake-hostname"; got != want { t.Errorf("unexpected local host: %v, want %v", got, want) @@ -472,28 +461,22 @@ func verifyUIServedWithCorrectTitle(t *testing.T, cli *apiclient.KopiaAPIClient, t.Helper() req, err := http.NewRequestWithContext(context.Background(), "GET", sp.baseURL, nil) - if err != nil { - t.Fatalf("unable to create HTTP request: %v", err) - } + require.NoError(t, err) req.SetBasicAuth("kopia", sp.password) resp, err := cli.HTTPClient.Do(req) - if err != nil { - t.Fatalf("unable to get HTML root: %v", err) - } + require.NoError(t, err) defer resp.Body.Close() b, err := ioutil.ReadAll(resp.Body) - if err != nil { - t.Fatalf("error reading response body: %v", err) - } + require.NoError(t, err) // make sure the UI correctly inserts prefix from KOPIA_UI_TITLE_PREFIX // and it's correctly HTML-escaped. if !bytes.Contains(b, []byte(`<title>Blah: <script>bleh</script> Kopia UI`)) { - t.Errorf("invalid title served by the UI: %v.", string(b)) + t.Fatalf("invalid title served by the UI: %v.", string(b)) } }