diff --git a/Makefile b/Makefile index 593260df9..8903f9dcc 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -COVERAGE_PACKAGES=github.com/kopia/kopia/repo/...,github.com/kopia/kopia/fs/...,github.com/kopia/kopia/snapshot/... +COVERAGE_PACKAGES=./repo/...,./fs/...,./snapshot/...,./cli/...,./internal/... TEST_FLAGS?= KOPIA_INTEGRATION_EXE=$(CURDIR)/dist/testing_$(GOOS)_$(GOARCH)/kopia.exe @@ -153,7 +153,7 @@ ci-integration-tests: integration-tests robustness-tool-tests $(MAKE) stress-test ci-publish-coverage: -ifeq ($(GOOS)/$(GOARCH)/$(IS_PULL_REQUEST),linux/amd64/false) +ifeq ($(GOOS)/$(GOARCH),linux/amd64) -bash -c "bash <(curl -s https://codecov.io/bash) -f coverage.txt" endif @@ -186,11 +186,11 @@ dev-deps: GO111MODULE=off go get -u github.com/sqs/goreturns test-with-coverage: $(gotestsum) - $(GO_TEST) $(UNIT_TEST_RACE_FLAGS) -count=$(REPEAT_TEST) -covermode=atomic -coverprofile=coverage.txt --coverpkg $(COVERAGE_PACKAGES) -timeout 300s ./... + $(GO_TEST) $(UNIT_TEST_RACE_FLAGS) -tags testing -count=$(REPEAT_TEST) -covermode=atomic -coverprofile=coverage.txt --coverpkg $(COVERAGE_PACKAGES) -timeout 300s ./... test: GOTESTSUM_FLAGS=--format=$(GOTESTSUM_FORMAT) --no-summary=skipped --jsonfile=.tmp.unit-tests.json test: $(gotestsum) - $(GO_TEST) $(UNIT_TEST_RACE_FLAGS) -count=$(REPEAT_TEST) -timeout $(UNIT_TESTS_TIMEOUT) ./... + $(GO_TEST) $(UNIT_TEST_RACE_FLAGS) -tags testing -count=$(REPEAT_TEST) -timeout $(UNIT_TESTS_TIMEOUT) ./... -$(gotestsum) tool slowest --jsonfile .tmp.unit-tests.json --threshold 1000ms provider-tests: export KOPIA_PROVIDER_TEST=true diff --git a/cli/app.go b/cli/app.go index bf482cf23..76dd43896 100644 --- a/cli/app.go +++ b/cli/app.go @@ -79,6 +79,7 @@ type appServices interface { repositoryWriterAction(act func(ctx context.Context, rep repo.RepositoryWriter) error) func(ctx *kingpin.ParseContext) error maybeRepositoryAction(act func(ctx context.Context, rep repo.Repository) error, mode repositoryAccessMode) func(ctx *kingpin.ParseContext) error + advancedCommand(ctx context.Context) repositoryConfigFileName() string getProgress() *cliProgress @@ -99,6 +100,8 @@ type advancedAppServices interface { passwordPersistenceStrategy() passwordpersist.Strategy getPasswordFromFlags(ctx context.Context, isNew, allowPersistent bool) (string, error) optionsFromFlags(ctx context.Context) *repo.Options + + rootContext() context.Context } // App contains per-invocation flags and state of Kopia CLI. @@ -116,6 +119,7 @@ type App struct { metricsListenAddr string keyRingEnabled bool persistCredentials bool + AdvancedCommands string // subcommands blob commandBlob @@ -140,6 +144,7 @@ type App struct { osExit func(int) // allows replacing os.Exit() with custom code stdoutWriter io.Writer stderrWriter io.Writer + rootctx context.Context } func (c *App) getProgress() *cliProgress { @@ -185,9 +190,10 @@ func (c *App) setup(app *kingpin.Application) { app.Flag("config-file", "Specify the config file to use.").Default(defaultConfigFileName()).Envar("KOPIA_CONFIG_PATH").StringVar(&c.configPath) app.Flag("trace-storage", "Enables tracing of storage operations.").Default("true").Hidden().BoolVar(&c.traceStorage) app.Flag("metrics-listen-addr", "Expose Prometheus metrics on a given host:port").Hidden().StringVar(&c.metricsListenAddr) - app.Flag("timezone", "Format time according to specified time zone (local, utc, original or time zone name)").Default("local").Hidden().StringVar(&timeZone) + app.Flag("timezone", "Format time according to specified time zone (local, utc, original or time zone name)").Hidden().StringVar(&timeZone) app.Flag("password", "Repository password.").Envar("KOPIA_PASSWORD").Short('p').StringVar(&c.password) app.Flag("persist-credentials", "Persist credentials").Default("true").Envar("KOPIA_PERSIST_CREDENTIALS_ON_CONNECT").BoolVar(&c.persistCredentials) + app.Flag("advanced-commands", "Enable advanced (and potentially dangerous) commands.").Hidden().Envar("KOPIA_ADVANCED_COMMANDS").StringVar(&c.AdvancedCommands) c.setupOSSpecificKeychainFlags(app) @@ -235,6 +241,7 @@ func NewApp() *App { osExit: os.Exit, stdoutWriter: os.Stdout, stderrWriter: os.Stderr, + rootctx: context.Background(), } } @@ -268,7 +275,7 @@ func safetyFlagVar(cmd *kingpin.CmdClause, result *maintenance.SafetyParameters) func (c *App) noRepositoryAction(act func(ctx context.Context) error) func(ctx *kingpin.ParseContext) error { return func(_ *kingpin.ParseContext) error { - return act(rootContext()) + return act(c.rootContext()) } } @@ -284,7 +291,7 @@ func (c *App) serverAction(sf *serverClientFlags, act func(ctx context.Context, return errors.Wrap(err, "unable to create API client") } - return act(rootContext(), apiClient) + return act(c.rootContext(), apiClient) } } @@ -348,8 +355,8 @@ func (c *App) repositoryWriterAction(act func(ctx context.Context, rep repo.Repo }) } -func rootContext() context.Context { - return context.Background() +func (c *App) rootContext() context.Context { + return c.rootctx } type repositoryAccessMode struct { @@ -359,7 +366,7 @@ type repositoryAccessMode struct { func (c *App) maybeRepositoryAction(act func(ctx context.Context, rep repo.Repository) error, mode repositoryAccessMode) func(ctx *kingpin.ParseContext) error { return func(kpc *kingpin.ParseContext) error { - ctx := rootContext() + ctx := c.rootContext() if err := withProfiling(func() error { c.mt.startMemoryTracking(ctx) @@ -398,7 +405,7 @@ func (c *App) maybeRepositoryAction(act func(ctx context.Context, rep repo.Repos }); err != nil { // print error in red log(ctx).Errorf("ERROR: %v", err.Error()) - os.Exit(1) + c.osExit(1) } return nil @@ -436,15 +443,20 @@ func (c *App) maybeRunMaintenance(ctx context.Context, rep repo.Repository) erro return errors.Wrap(err, "error running maintenance") } -func advancedCommand(ctx context.Context) { - if os.Getenv("KOPIA_ADVANCED_COMMANDS") != "enabled" { - log(ctx).Errorf(` +func (c *App) advancedCommand(ctx context.Context) { + if c.AdvancedCommands != "enabled" { + _, _ = errorColor.Fprintf(c.stderrWriter, ` This command could be dangerous or lead to repository corruption when used improperly. Running this command is not needed for using Kopia. Instead, most users should rely on periodic repository maintenance. See https://kopia.io/docs/advanced/maintenance/ for more information. To run this command despite the warning, set KOPIA_ADVANCED_COMMANDS=enabled `) - os.Exit(1) + + c.osExit(1) } } + +func init() { + kingpin.EnableFileExpansion = false +} diff --git a/cli/command_blob_delete.go b/cli/command_blob_delete.go index 0981ad19e..3cca3d061 100644 --- a/cli/command_blob_delete.go +++ b/cli/command_blob_delete.go @@ -11,16 +11,20 @@ type commandBlobDelete struct { blobIDs []string + + svc appServices } func (c *commandBlobDelete) setup(svc appServices, parent commandParent) { cmd := parent.Command("delete", "Delete blobs by ID").Alias("remove").Alias("rm") cmd.Arg("blobIDs", "Blob IDs").Required().StringsVar(&c.blobIDs) cmd.Action(svc.directRepositoryWriteAction(c.run)) + + c.svc = svc } func (c *commandBlobDelete) run(ctx context.Context, rep repo.DirectRepositoryWriter) error { - advancedCommand(ctx) + c.svc.advancedCommand(ctx) for _, b := range c.blobIDs { err := rep.BlobStorage().DeleteBlob(ctx, blob.ID(b)) diff --git a/cli/command_blob_gc.go b/cli/command_blob_gc.go index f701f445c..ee8aaa21e 100644 --- a/cli/command_blob_gc.go +++ b/cli/command_blob_gc.go @@ -15,6 +15,8 @@ type commandBlobGC struct { parallel int prefix string safety maintenance.SafetyParameters + + svc appServices } func (c *commandBlobGC) setup(svc appServices, parent commandParent) { @@ -24,10 +26,12 @@ func (c *commandBlobGC) setup(svc appServices, parent commandParent) { cmd.Flag("prefix", "Only GC blobs with given prefix").StringVar(&c.prefix) safetyFlagVar(cmd, &c.safety) cmd.Action(svc.directRepositoryWriteAction(c.run)) + + c.svc = svc } func (c *commandBlobGC) run(ctx context.Context, rep repo.DirectRepositoryWriter) error { - advancedCommand(ctx) + c.svc.advancedCommand(ctx) opts := maintenance.DeleteUnreferencedBlobsOptions{ DryRun: c.delete != "yes", diff --git a/cli/command_blob_show.go b/cli/command_blob_show.go index 583766b43..92d289ad2 100644 --- a/cli/command_blob_show.go +++ b/cli/command_blob_show.go @@ -5,7 +5,6 @@ "context" "encoding/json" "io" - "os" "github.com/pkg/errors" @@ -17,6 +16,8 @@ type commandBlobShow struct { blobShowDecrypt bool blobShowIDs []string + + out textOutput } func (c *commandBlobShow) setup(svc appServices, parent commandParent) { @@ -24,11 +25,13 @@ func (c *commandBlobShow) setup(svc appServices, parent commandParent) { cmd.Flag("decrypt", "Decrypt blob if possible").BoolVar(&c.blobShowDecrypt) cmd.Arg("blobID", "Blob IDs").Required().StringsVar(&c.blobShowIDs) cmd.Action(svc.directRepositoryReadAction(c.run)) + + c.out.setup(svc) } func (c *commandBlobShow) run(ctx context.Context, rep repo.DirectRepository) error { for _, blobID := range c.blobShowIDs { - if err := c.maybeDecryptBlob(ctx, os.Stdout, rep, blob.ID(blobID)); err != nil { + if err := c.maybeDecryptBlob(ctx, c.out.stdout(), rep, blob.ID(blobID)); err != nil { return errors.Wrap(err, "error presenting blob") } } diff --git a/cli/command_content_delete.go b/cli/command_content_delete.go index 83ecb5160..5078d2ef3 100644 --- a/cli/command_content_delete.go +++ b/cli/command_content_delete.go @@ -10,16 +10,20 @@ type commandContentDelete struct { ids []string + + svc appServices } func (c *commandContentDelete) setup(svc appServices, parent commandParent) { cmd := parent.Command("delete", "Remove content").Alias("remove").Alias("rm") cmd.Arg("id", "IDs of content to remove").Required().StringsVar(&c.ids) cmd.Action(svc.directRepositoryWriteAction(c.run)) + + c.svc = svc } func (c *commandContentDelete) run(ctx context.Context, rep repo.DirectRepositoryWriter) error { - advancedCommand(ctx) + c.svc.advancedCommand(ctx) for _, contentID := range toContentIDs(c.ids) { if err := rep.ContentManager().DeleteContent(ctx, contentID); err != nil { diff --git a/cli/command_content_rewrite.go b/cli/command_content_rewrite.go index 7d8e3dc8f..f2364bafd 100644 --- a/cli/command_content_rewrite.go +++ b/cli/command_content_rewrite.go @@ -19,6 +19,7 @@ type commandContentRewrite struct { contentRewriteSafety maintenance.SafetyParameters contentRange contentRangeFlags + svc appServices } func (c *commandContentRewrite) setup(svc appServices, parent commandParent) { @@ -33,10 +34,12 @@ func (c *commandContentRewrite) setup(svc appServices, parent commandParent) { c.contentRange.setup(cmd) safetyFlagVar(cmd, &c.contentRewriteSafety) cmd.Action(svc.directRepositoryWriteAction(c.runContentRewriteCommand)) + + c.svc = svc } func (c *commandContentRewrite) runContentRewriteCommand(ctx context.Context, rep repo.DirectRepositoryWriter) error { - advancedCommand(ctx) + c.svc.advancedCommand(ctx) return maintenance.RewriteContents(ctx, rep, &maintenance.RewriteContentsOptions{ ContentIDRange: c.contentRange.contentIDRange(), diff --git a/cli/command_content_show.go b/cli/command_content_show.go index bc370055b..e329f8906 100644 --- a/cli/command_content_show.go +++ b/cli/command_content_show.go @@ -14,6 +14,8 @@ type commandContentShow struct { ids []string indentJSON bool decompress bool + + out textOutput } func (c *commandContentShow) setup(svc appServices, parent commandParent) { @@ -23,6 +25,8 @@ func (c *commandContentShow) setup(svc appServices, parent commandParent) { cmd.Flag("json", "Pretty-print JSON content").Short('j').BoolVar(&c.indentJSON) cmd.Flag("unzip", "Transparently decompress the content").Short('z').BoolVar(&c.decompress) cmd.Action(svc.directRepositoryReadAction(c.run)) + + c.out.setup(svc) } func (c *commandContentShow) run(ctx context.Context, rep repo.DirectRepository) error { @@ -41,5 +45,5 @@ func (c *commandContentShow) contentShow(ctx context.Context, r repo.DirectRepos return errors.Wrapf(err, "error getting content %v", contentID) } - return showContentWithFlags(bytes.NewReader(data), c.decompress, c.indentJSON) + return showContentWithFlags(c.out.stdout(), bytes.NewReader(data), c.decompress, c.indentJSON) } diff --git a/cli/command_diff.go b/cli/command_diff.go index 8be909c54..be74a1e0d 100644 --- a/cli/command_diff.go +++ b/cli/command_diff.go @@ -2,7 +2,6 @@ import ( "context" - "os" "strings" "github.com/pkg/errors" @@ -18,6 +17,8 @@ type commandDiff struct { diffSecondObjectPath string diffCompareFiles bool diffCommandCommand string + + out textOutput } func (c *commandDiff) setup(svc appServices, parent commandParent) { @@ -27,6 +28,8 @@ func (c *commandDiff) setup(svc appServices, parent commandParent) { cmd.Flag("files", "Compare files by launching diff command for all pairs of (old,new)").Short('f').BoolVar(&c.diffCompareFiles) cmd.Flag("diff-command", "Displays differences between two repository objects (files or directories)").Default(defaultDiffCommand()).Envar("KOPIA_DIFF").StringVar(&c.diffCommandCommand) cmd.Action(svc.repositoryReaderAction(c.run)) + + c.out.setup(svc) } func (c *commandDiff) run(ctx context.Context, rep repo.Repository) error { @@ -47,7 +50,7 @@ func (c *commandDiff) run(ctx context.Context, rep repo.Repository) error { return errors.New("arguments do diff must both be directories or both non-directories") } - d, err := diff.NewComparer(os.Stdout) + d, err := diff.NewComparer(c.out.stdout()) if err != nil { return errors.Wrap(err, "error creating comparer") } diff --git a/cli/command_index_optimize.go b/cli/command_index_optimize.go index 7685837f9..3b144af46 100644 --- a/cli/command_index_optimize.go +++ b/cli/command_index_optimize.go @@ -13,6 +13,8 @@ type commandIndexOptimize struct { optimizeDropDeletedOlderThan time.Duration optimizeDropContents []string optimizeAllIndexes bool + + svc appServices } func (c *commandIndexOptimize) setup(svc appServices, parent commandParent) { @@ -22,10 +24,12 @@ func (c *commandIndexOptimize) setup(svc appServices, parent commandParent) { cmd.Flag("drop-contents", "Drop contents with given IDs").StringsVar(&c.optimizeDropContents) cmd.Flag("all", "Optimize all indexes, even those above maximum size.").BoolVar(&c.optimizeAllIndexes) cmd.Action(svc.directRepositoryWriteAction(c.runOptimizeCommand)) + + c.svc = svc } func (c *commandIndexOptimize) runOptimizeCommand(ctx context.Context, rep repo.DirectRepositoryWriter) error { - advancedCommand(ctx) + c.svc.advancedCommand(ctx) opt := content.CompactOptions{ MaxSmallBlobs: c.optimizeMaxSmallBlobs, diff --git a/cli/command_index_recover.go b/cli/command_index_recover.go index 08b311432..591a03e71 100644 --- a/cli/command_index_recover.go +++ b/cli/command_index_recover.go @@ -13,6 +13,8 @@ type commandIndexRecover struct { blobIDs []string commit bool + + svc appServices } func (c *commandIndexRecover) setup(svc appServices, parent commandParent) { @@ -20,10 +22,12 @@ func (c *commandIndexRecover) setup(svc appServices, parent commandParent) { cmd.Flag("blobs", "Names of pack blobs to recover from (default=all packs)").StringsVar(&c.blobIDs) cmd.Flag("commit", "Commit recovered content").BoolVar(&c.commit) cmd.Action(svc.directRepositoryWriteAction(c.run)) + + c.svc = svc } func (c *commandIndexRecover) run(ctx context.Context, rep repo.DirectRepositoryWriter) error { - advancedCommand(ctx) + c.svc.advancedCommand(ctx) var totalCount int diff --git a/cli/command_ls.go b/cli/command_ls.go index bd3a969c4..e277cf234 100644 --- a/cli/command_ls.go +++ b/cli/command_ls.go @@ -113,7 +113,7 @@ func (c *commandList) printDirectoryEntry(ctx context.Context, e fs.Entry, prefi info = fmt.Sprintf("%v%v", c.nameToDisplay(prefix, e), errorSummary) } - col.Println(info) //nolint:errcheck + col.Fprintln(c.out.stdout(), info) //nolint:errcheck if c.recursive { if subdir, ok := e.(fs.Directory); ok { diff --git a/cli/command_manifest_delete.go b/cli/command_manifest_delete.go index daddbe5c1..44e5abf32 100644 --- a/cli/command_manifest_delete.go +++ b/cli/command_manifest_delete.go @@ -10,16 +10,20 @@ type commandManifestDelete struct { manifestRemoveItems []string + + svc appServices } func (c *commandManifestDelete) setup(svc appServices, parent commandParent) { cmd := parent.Command("delete", "Remove manifest items").Alias("remove").Alias("rm") cmd.Arg("item", "Items to remove").Required().StringsVar(&c.manifestRemoveItems) cmd.Action(svc.repositoryWriterAction(c.run)) + + c.svc = svc } func (c *commandManifestDelete) run(ctx context.Context, rep repo.RepositoryWriter) error { - advancedCommand(ctx) + c.svc.advancedCommand(ctx) for _, it := range toManifestIDs(c.manifestRemoveItems) { if err := rep.DeleteManifest(ctx, it); err != nil { diff --git a/cli/command_manifest_show.go b/cli/command_manifest_show.go index 00a82e004..e0a7d8aed 100644 --- a/cli/command_manifest_show.go +++ b/cli/command_manifest_show.go @@ -51,7 +51,7 @@ func (c *commandManifestShow) showManifestItems(ctx context.Context, rep repo.Re c.out.printStdout("// label %v:%v\n", k, v) } - if showerr := showContentWithFlags(bytes.NewReader(b), false, true); showerr != nil { + if showerr := showContentWithFlags(c.out.stdout(), bytes.NewReader(b), false, true); showerr != nil { return showerr } } diff --git a/cli/command_repository_connect.go b/cli/command_repository_connect.go index b853a822d..33e51d22c 100644 --- a/cli/command_repository_connect.go +++ b/cli/command_repository_connect.go @@ -31,7 +31,7 @@ func (c *commandRepositoryConnect) setup(svc advancedAppServices, parent command cc := cmd.Command(prov.name, "Connect to repository in "+prov.description) f.setup(svc, cc) cc.Action(func(_ *kingpin.ParseContext) error { - ctx := rootContext() + ctx := svc.rootContext() st, err := f.connect(ctx, false) if err != nil { return errors.Wrap(err, "can't connect to storage") diff --git a/cli/command_repository_create.go b/cli/command_repository_create.go index 77d81f91f..c7c36ae02 100644 --- a/cli/command_repository_create.go +++ b/cli/command_repository_create.go @@ -49,7 +49,7 @@ func (c *commandRepositoryCreate) setup(svc advancedAppServices, parent commandP cc := cmd.Command(prov.name, "Create repository in "+prov.description) f.setup(svc, cc) cc.Action(func(_ *kingpin.ParseContext) error { - ctx := rootContext() + ctx := svc.rootContext() st, err := f.connect(ctx, true) if err != nil { return errors.Wrap(err, "can't connect to storage") diff --git a/cli/command_repository_repair.go b/cli/command_repository_repair.go index 699bc6427..e4ccac8bf 100644 --- a/cli/command_repository_repair.go +++ b/cli/command_repository_repair.go @@ -30,7 +30,7 @@ func (c *commandRepositoryRepair) setup(svc advancedAppServices, parent commandP cc := cmd.Command(prov.name, "Repair repository in "+prov.description) f.setup(svc, cc) cc.Action(func(_ *kingpin.ParseContext) error { - ctx := rootContext() + ctx := svc.rootContext() st, err := f.connect(ctx, false) if err != nil { return errors.Wrap(err, "can't connect to storage") diff --git a/cli/command_repository_sync.go b/cli/command_repository_sync.go index 77fda559f..e0359a11f 100644 --- a/cli/command_repository_sync.go +++ b/cli/command_repository_sync.go @@ -21,6 +21,8 @@ ) type commandRepositorySyncTo struct { + nextSyncOutputTime *timetrack.Throttle + repositorySyncUpdate bool repositorySyncDelete bool repositorySyncDryRun bool @@ -30,7 +32,6 @@ type commandRepositorySyncTo struct { lastSyncProgress string syncProgressMutex sync.Mutex - nextSyncOutputTime timetrack.Throttle setTimeUnsupportedOnce sync.Once out textOutput @@ -47,13 +48,16 @@ func (c *commandRepositorySyncTo) setup(svc advancedAppServices, parent commandP c.out.setup(svc) + // needs to be 64-bit aligned on ARM + c.nextSyncOutputTime = new(timetrack.Throttle) + for _, prov := range storageProviders { // Set up 'sync-to' subcommand f := prov.newFlags() cc := cmd.Command(prov.name, "Synchronize repository data to another repository in "+prov.description) f.setup(svc, cc) cc.Action(func(_ *kingpin.ParseContext) error { - ctx := rootContext() + ctx := svc.rootContext() st, err := f.connect(ctx, false) if err != nil { return errors.Wrap(err, "can't connect to storage") diff --git a/cli/command_show.go b/cli/command_show.go index 6f56dc09c..5e4e62c9b 100644 --- a/cli/command_show.go +++ b/cli/command_show.go @@ -2,7 +2,6 @@ import ( "context" - "os" "github.com/pkg/errors" @@ -13,12 +12,16 @@ type commandShow struct { path string + + out textOutput } func (c *commandShow) setup(svc appServices, parent commandParent) { cmd := parent.Command("show", "Displays contents of a repository object.").Alias("cat") cmd.Arg("object-path", "Path").Required().StringVar(&c.path) cmd.Action(svc.repositoryReaderAction(c.run)) + + c.out.setup(svc) } func (c *commandShow) run(ctx context.Context, rep repo.Repository) error { @@ -34,7 +37,7 @@ func (c *commandShow) run(ctx context.Context, rep repo.Repository) error { defer r.Close() //nolint:errcheck - _, err = iocopy.Copy(os.Stdout, r) + _, err = iocopy.Copy(c.out.stdout(), r) return errors.Wrap(err, "unable to copy data") } diff --git a/cli/command_snapshot_list.go b/cli/command_snapshot_list.go index eeaf9e70f..6d786078d 100644 --- a/cli/command_snapshot_list.go +++ b/cli/command_snapshot_list.go @@ -252,7 +252,7 @@ func (c *commandSnapshotList) outputManifestFromSingleSource(ctx context.Context elidedCount = 0 previousOID = oid - col.Print(fmt.Sprintf(" %v %v %v\n", formatTimestamp(m.StartTime), oid, strings.Join(bits, " "))) //nolint:errcheck + col.Fprint(c.out.stdout(), fmt.Sprintf(" %v %v %v\n", formatTimestamp(m.StartTime), oid, strings.Join(bits, " "))) //nolint:errcheck count++ diff --git a/cli/inproc.go b/cli/inproc.go new file mode 100644 index 000000000..82ec73d5e --- /dev/null +++ b/cli/inproc.go @@ -0,0 +1,55 @@ +package cli + +import ( + "context" + "io" + + "github.com/alecthomas/kingpin" + "github.com/pkg/errors" + + "github.com/kopia/kopia/repo/logging" +) + +// RunSubcommand executes the subcommand asynchronously in current process +// with flags in an isolated CLI environment and returns standard output and standard error. +func (c *App) RunSubcommand(ctx context.Context, argsAndFlags []string) (stdout, stderr io.Reader, wait func() error, kill func()) { + kpapp := kingpin.New("test", "test") + + stdoutReader, stdoutWriter := io.Pipe() + stderrReader, stderrWriter := io.Pipe() + + c.stdoutWriter = stdoutWriter + c.stderrWriter = stderrWriter + c.rootctx = logging.WithLogger(ctx, logging.Writer(stderrWriter)) + + c.Attach(kpapp) + + var exitCode int + + resultErr := make(chan error, 1) + + c.osExit = func(ec int) { + exitCode = ec + } + + go func() { + defer close(resultErr) + defer stderrWriter.Close() //nolint:errcheck + defer stdoutWriter.Close() //nolint:errcheck + + _, err := kpapp.Parse(argsAndFlags) + if err != nil { + resultErr <- err + return + } + + if exitCode != 0 { + resultErr <- errors.Errorf("exit code %v", exitCode) + return + } + }() + + return stdoutReader, stderrReader, func() error { + return <-resultErr + }, func() {} +} diff --git a/cli/show_utils.go b/cli/show_utils.go index 79fb41926..86c133b78 100644 --- a/cli/show_utils.go +++ b/cli/show_utils.go @@ -7,7 +7,6 @@ "fmt" "io" "io/ioutil" - "os" "time" "github.com/pkg/errors" @@ -17,9 +16,9 @@ ) // TODO - remove this global. -var timeZone string +var timeZone = "local" -func showContentWithFlags(rd io.Reader, unzip, indentJSON bool) error { +func showContentWithFlags(w io.Writer, rd io.Reader, unzip, indentJSON bool) error { if unzip { gz, err := gzip.NewReader(rd) if err != nil { @@ -43,7 +42,7 @@ func showContentWithFlags(rd io.Reader, unzip, indentJSON bool) error { rd = ioutil.NopCloser(&buf2) } - if _, err := iocopy.Copy(os.Stdout, rd); err != nil { + if _, err := iocopy.Copy(w, rd); err != nil { return errors.Wrap(err, "error copying data") } diff --git a/internal/buf/pool.go b/internal/buf/pool.go index 96d3c12ab..ecda6f975 100644 --- a/internal/buf/pool.go +++ b/internal/buf/pool.go @@ -12,6 +12,10 @@ "go.opencensus.io/tag" ) +// DisableBufferManagement is a global flag that disables memory buffer reuse, +// which can be useful in tests to reduce overall memory usage. +var DisableBufferManagement = false + type segment struct { mu sync.RWMutex @@ -103,7 +107,7 @@ func (s *segment) allocate(n int) (Buf, bool) { // // The pool uses N segments, with each segment tracking its high water mark usage. // -// Allocation simply advances the high water mark within first segment that has capacity +// ation simply advances the high water mark within first segment that has capacity // and increments per-segment refcount. // // On Buf.Release() the refcount is decremented and when it hits zero, the entire segment becomes instantly @@ -243,7 +247,7 @@ func (p *Pool) AddSegments(n int) { // Allocate allocates from the buffer a slice of size n. func (p *Pool) Allocate(n int) Buf { // requested more than the pool can cache, allocate throw-away buffer. - if p == nil || n > p.segmentSize { + if p == nil || n > p.segmentSize || DisableBufferManagement { return Buf{make([]byte, n), 0, 0, nil} } diff --git a/main.go b/main.go index 24e6c4c98..09e09b071 100644 --- a/main.go +++ b/main.go @@ -69,8 +69,6 @@ func main() { app := cli.NewApp() kp := kingpin.New("kopia", "Kopia - Fast And Secure Open-Source Backup").Author("http://kopia.github.io/") - kingpin.EnableFileExpansion = false - logging.SetDefault(func(module string) logging.Logger { return gologging.MustGetLogger(module) }) diff --git a/repo/logging/printf_logger.go b/repo/logging/printf_logger.go index 547a43f5d..7f86555ae 100644 --- a/repo/logging/printf_logger.go +++ b/repo/logging/printf_logger.go @@ -1,5 +1,11 @@ package logging +import ( + "fmt" + "io" + "strings" +) + type printfLogger struct { printf func(msg string, args ...interface{}) prefix string @@ -15,3 +21,20 @@ func Printf(printf func(msg string, args ...interface{})) LoggerForModuleFunc { return &printfLogger{printf, "[" + module + "]"} } } + +// Writer returns LoggerForModuleFunc that uses given writer for log output. +func Writer(w io.Writer) LoggerForModuleFunc { + printf := func(msg string, args ...interface{}) { + msg = fmt.Sprintf(msg, args...) + + if !strings.HasSuffix(msg, "\n") { + msg += "\n" + } + + io.WriteString(w, msg) //nolint:errcheck + } + + return func(module string) Logger { + return &printfLogger{printf, ""} + } +} diff --git a/tests/end_to_end_test/acl_test.go b/tests/end_to_end_test/acl_test.go index c0adc221d..c4cc885aa 100644 --- a/tests/end_to_end_test/acl_test.go +++ b/tests/end_to_end_test/acl_test.go @@ -11,7 +11,9 @@ func TestACL(t *testing.T) { t.Parallel() - serverEnvironment := testenv.NewCLITest(t) + serverRunner := testenv.NewExeRunner(t) + serverEnvironment := testenv.NewCLITest(t, serverRunner) + defer serverEnvironment.RunAndExpectSuccess(t, "repo", "disconnect") serverEnvironment.RunAndExpectSuccess(t, "repo", "create", "filesystem", "--path", serverEnvironment.RepoDir, "--override-hostname=foo", "--override-username=foo", "--enable-actions") @@ -39,7 +41,7 @@ func TestACL(t *testing.T) { var sp serverParameters - srv := serverEnvironment.RunAndProcessStderr(t, sp.ProcessOutput, + kill := serverEnvironment.RunAndProcessStderr(t, sp.ProcessOutput, "server", "start", "--address=localhost:0", "--server-username=admin-user", @@ -50,12 +52,14 @@ func TestACL(t *testing.T) { t.Logf("detected server parameters %#v", sp) - defer srv.Process.Kill() + defer kill() + + fooBarRunner := testenv.NewExeRunner(t) + foobarClientEnvironment := testenv.NewCLITest(t, fooBarRunner) - foobarClientEnvironment := testenv.NewCLITest(t) defer foobarClientEnvironment.RunAndExpectSuccess(t, "repo", "disconnect") - foobarClientEnvironment.RemoveDefaultPassword() + fooBarRunner.RemoveDefaultPassword() // connect as foo@bar with password baz foobarClientEnvironment.RunAndExpectSuccess(t, "repo", "connect", "server", @@ -66,10 +70,12 @@ func TestACL(t *testing.T) { "--password", "baz", ) - aliceInWonderlandClientEnvironment := testenv.NewCLITest(t) + aliceInWonderlandRunner := testenv.NewExeRunner(t) + aliceInWonderlandClientEnvironment := testenv.NewCLITest(t, aliceInWonderlandRunner) + defer aliceInWonderlandClientEnvironment.RunAndExpectSuccess(t, "repo", "disconnect") - aliceInWonderlandClientEnvironment.RemoveDefaultPassword() + aliceInWonderlandRunner.RemoveDefaultPassword() // connect as alice@wonderland with password baz aliceInWonderlandClientEnvironment.RunAndExpectSuccess(t, "repo", "connect", "server", diff --git a/tests/end_to_end_test/all_formats_test.go b/tests/end_to_end_test/all_formats_test.go index da24f9b71..9dd2d53be 100644 --- a/tests/end_to_end_test/all_formats_test.go +++ b/tests/end_to_end_test/all_formats_test.go @@ -14,6 +14,8 @@ func TestAllFormatsSmokeTest(t *testing.T) { srcDir := testutil.TempDirectory(t) + runner := testenv.NewInProcRunner(t) + // 3-level directory with <=10 files and <=10 subdirectories at each level testdirtree.CreateDirectoryTree(srcDir, testdirtree.DirectoryTreeOptions{ Depth: 2, @@ -32,7 +34,7 @@ func TestAllFormatsSmokeTest(t *testing.T) { t.Run(hashAlgo, func(t *testing.T) { t.Parallel() - e := testenv.NewCLITest(t) + e := testenv.NewCLITest(t, runner) defer e.RunAndExpectSuccess(t, "repo", "disconnect") e.DefaultRepositoryCreateFlags = nil diff --git a/tests/end_to_end_test/api_server_repository_test.go b/tests/end_to_end_test/api_server_repository_test.go index f210667e3..2c878f0c4 100644 --- a/tests/end_to_end_test/api_server_repository_test.go +++ b/tests/end_to_end_test/api_server_repository_test.go @@ -55,14 +55,16 @@ func testAPIServerRepository(t *testing.T, serverStartArgs []string, useGRPC, al connectArgs = []string{"--no-grpc"} } - e := testenv.NewCLITest(t) + runner := testenv.NewExeRunner(t) + e := testenv.NewCLITest(t, runner) + defer e.RunAndExpectSuccess(t, "repo", "disconnect") // create one snapshot as foo@bar e.RunAndExpectSuccess(t, "repo", "create", "filesystem", "--path", e.RepoDir, "--override-username", "foo", "--override-hostname", "bar") e.RunAndExpectSuccess(t, "snapshot", "create", sharedTestDataDir1) - e1 := testenv.NewCLITest(t) + e1 := testenv.NewCLITest(t, runner) defer e1.RunAndExpectSuccess(t, "repo", "disconnect") // create one snapshot as not-foo@bar @@ -201,7 +203,9 @@ func testAPIServerRepository(t *testing.T, serverStartArgs []string, useGRPC, al verifyFindManifestCount(ctx, t, writeSess, someLabels, 1) } - e2 := testenv.NewCLITest(t) + runner2 := testenv.NewExeRunner(t) + e2 := testenv.NewCLITest(t, runner2) + defer e2.RunAndExpectSuccess(t, "repo", "disconnect") e2.RunAndExpectSuccess(t, append([]string{ @@ -216,7 +220,7 @@ func testAPIServerRepository(t *testing.T, serverStartArgs []string, useGRPC, al // we are providing custom password to connect, make sure we won't be providing // (different) default password via environment variable, as command-line password // takes precedence over persisted password. - e2.RemoveDefaultPassword() + runner2.RemoveDefaultPassword() // should see one snapshot snapshots := clitestutil.ListSnapshotsAndExpectSuccess(t, e2) diff --git a/tests/end_to_end_test/auto_update_test.go b/tests/end_to_end_test/auto_update_test.go index d5616c195..b0dbbf7ec 100644 --- a/tests/end_to_end_test/auto_update_test.go +++ b/tests/end_to_end_test/auto_update_test.go @@ -40,14 +40,15 @@ func TestAutoUpdateEnableTest(t *testing.T) { t.Run(tc.desc, func(t *testing.T) { t.Parallel() - e := testenv.NewCLITest(t) + runner := testenv.NewExeRunner(t) + e := testenv.NewCLITest(t, runner) // create repo args := append([]string{ "repo", "create", "filesystem", "--path", e.RepoDir, }, tc.extraArgs...) - e.Environment = append(e.Environment, tc.extraEnv...) + runner.Environment = append(runner.Environment, tc.extraEnv...) e.RunAndExpectSuccess(t, args...) updateInfoFile := filepath.Join(e.ConfigDir, ".kopia.config.update-info.json") diff --git a/tests/end_to_end_test/compression_test.go b/tests/end_to_end_test/compression_test.go index 4719dc075..9d06298f0 100644 --- a/tests/end_to_end_test/compression_test.go +++ b/tests/end_to_end_test/compression_test.go @@ -17,7 +17,9 @@ func TestCompression(t *testing.T) { t.Parallel() - e := testenv.NewCLITest(t) + runner := testenv.NewInProcRunner(t) + e := testenv.NewCLITest(t, runner) + defer e.RunAndExpectSuccess(t, "repo", "disconnect") e.RunAndExpectSuccess(t, "repo", "create", "filesystem", "--path", e.RepoDir) diff --git a/tests/end_to_end_test/diff_test.go b/tests/end_to_end_test/diff_test.go index 28794f79f..53f4bd728 100644 --- a/tests/end_to_end_test/diff_test.go +++ b/tests/end_to_end_test/diff_test.go @@ -16,7 +16,9 @@ func TestDiff(t *testing.T) { t.Parallel() - e := testenv.NewCLITest(t) + runner := testenv.NewInProcRunner(t) + e := testenv.NewCLITest(t, runner) + defer e.RunAndExpectSuccess(t, "repo", "disconnect") e.RunAndExpectSuccess(t, "repo", "create", "filesystem", "--path", e.RepoDir) diff --git a/tests/end_to_end_test/index_optimize_test.go b/tests/end_to_end_test/index_optimize_test.go index 403aebba3..5fbeea250 100644 --- a/tests/end_to_end_test/index_optimize_test.go +++ b/tests/end_to_end_test/index_optimize_test.go @@ -9,7 +9,9 @@ func TestIndexOptimize(t *testing.T) { t.Parallel() - e := testenv.NewCLITest(t) + runner := testenv.NewInProcRunner(t) + e := testenv.NewCLITest(t, runner) + defer e.RunAndExpectSuccess(t, "repo", "disconnect") e.RunAndExpectSuccess(t, "repo", "create", "filesystem", "--path", e.RepoDir) diff --git a/tests/end_to_end_test/index_recover_test.go b/tests/end_to_end_test/index_recover_test.go index 807cef065..aa606af23 100644 --- a/tests/end_to_end_test/index_recover_test.go +++ b/tests/end_to_end_test/index_recover_test.go @@ -13,7 +13,8 @@ func TestIndexRecover(t *testing.T) { t.Parallel() - e := testenv.NewCLITest(t) + runner := testenv.NewInProcRunner(t) + e := testenv.NewCLITest(t, runner) e.RunAndExpectSuccess(t, "repo", "create", "filesystem", "--path", e.RepoDir) diff --git a/tests/end_to_end_test/maintenance_test.go b/tests/end_to_end_test/maintenance_test.go index cfca3470d..50f175dae 100644 --- a/tests/end_to_end_test/maintenance_test.go +++ b/tests/end_to_end_test/maintenance_test.go @@ -12,7 +12,8 @@ func TestFullMaintenance(t *testing.T) { t.Parallel() - e := testenv.NewCLITest(t) + runner := testenv.NewInProcRunner(t) + e := testenv.NewCLITest(t, runner) e.RunAndExpectSuccess(t, "repo", "create", "filesystem", "--path", e.RepoDir) defer e.RunAndExpectSuccess(t, "repo", "disconnect") diff --git a/tests/end_to_end_test/policy_test.go b/tests/end_to_end_test/policy_test.go index ae140157e..26e0172d9 100644 --- a/tests/end_to_end_test/policy_test.go +++ b/tests/end_to_end_test/policy_test.go @@ -13,7 +13,8 @@ func TestDefaultGlobalPolicy(t *testing.T) { t.Parallel() - e := testenv.NewCLITest(t) + runner := testenv.NewInProcRunner(t) + e := testenv.NewCLITest(t, runner) e.RunAndExpectSuccess(t, "repo", "create", "filesystem", "--path", e.RepoDir) diff --git a/tests/end_to_end_test/repository_connect_test.go b/tests/end_to_end_test/repository_connect_test.go index a36b08c3b..4065cde64 100644 --- a/tests/end_to_end_test/repository_connect_test.go +++ b/tests/end_to_end_test/repository_connect_test.go @@ -14,7 +14,8 @@ func TestFilesystemRequiresAbsolutePaths(t *testing.T) { t.Parallel() - e := testenv.NewCLITest(t) + runner := testenv.NewInProcRunner(t) + e := testenv.NewCLITest(t, runner) e.RunAndExpectFailure(t, "repo", "create", "filesystem", "--path", "./relative-path") } @@ -27,7 +28,8 @@ func TestFilesystemSupportsTildeToReferToHome(t *testing.T) { t.Skip("home directory not available") } - e := testenv.NewCLITest(t) + runner := testenv.NewInProcRunner(t) + e := testenv.NewCLITest(t, runner) subdir := "repo-" + uuid.NewString() fullPath := filepath.Join(home, subdir) @@ -45,7 +47,8 @@ func TestFilesystemSupportsTildeToReferToHome(t *testing.T) { func TestReconnect(t *testing.T) { t.Parallel() - e := testenv.NewCLITest(t) + runner := testenv.NewInProcRunner(t) + e := testenv.NewCLITest(t, runner) e.RunAndExpectSuccess(t, "repo", "create", "filesystem", "--path", e.RepoDir) e.RunAndExpectSuccess(t, "repo", "disconnect") @@ -56,7 +59,8 @@ func TestReconnect(t *testing.T) { func TestReconnectUsingToken(t *testing.T) { t.Parallel() - e := testenv.NewCLITest(t) + runner := testenv.NewInProcRunner(t) + e := testenv.NewCLITest(t, runner) e.RunAndExpectSuccess(t, "repo", "create", "filesystem", "--path", e.RepoDir) lines := e.RunAndExpectSuccess(t, "repo", "status", "-t", "-s") diff --git a/tests/end_to_end_test/repository_repair_test.go b/tests/end_to_end_test/repository_repair_test.go index 92feca843..15813328d 100644 --- a/tests/end_to_end_test/repository_repair_test.go +++ b/tests/end_to_end_test/repository_repair_test.go @@ -9,7 +9,9 @@ func TestRepositoryRepair(t *testing.T) { t.Parallel() - e := testenv.NewCLITest(t) + runner := testenv.NewInProcRunner(t) + e := testenv.NewCLITest(t, runner) + defer e.RunAndExpectSuccess(t, "repo", "disconnect") e.RunAndExpectSuccess(t, "repo", "create", "filesystem", "--path", e.RepoDir) @@ -31,7 +33,7 @@ func TestRepositoryRepair(t *testing.T) { e.RunAndExpectFailure(t, "repo", "connect", "filesystem", "--path", e.RepoDir) // now run repair, which will recover the format blob from one of the pack blobs. - e.RunAndExpectSuccess(t, "repo", "repair", "--log-level=debug", "--trace-storage", "filesystem", "--path", e.RepoDir) + e.RunAndExpectSuccess(t, "repo", "repair", "filesystem", "--path", e.RepoDir) // now connect can succeed e.RunAndExpectSuccess(t, "repo", "connect", "filesystem", "--path", e.RepoDir) diff --git a/tests/end_to_end_test/repository_set_client_test.go b/tests/end_to_end_test/repository_set_client_test.go index 2300fea30..b53a9ac0d 100644 --- a/tests/end_to_end_test/repository_set_client_test.go +++ b/tests/end_to_end_test/repository_set_client_test.go @@ -10,7 +10,8 @@ func TestRepositorySetClient(t *testing.T) { t.Parallel() - e := testenv.NewCLITest(t) + runner := testenv.NewInProcRunner(t) + e := testenv.NewCLITest(t, runner) defer e.RunAndExpectSuccess(t, "repo", "disconnect") diff --git a/tests/end_to_end_test/repository_sync_test.go b/tests/end_to_end_test/repository_sync_test.go index 7a3764907..af2ac6b8e 100644 --- a/tests/end_to_end_test/repository_sync_test.go +++ b/tests/end_to_end_test/repository_sync_test.go @@ -11,7 +11,8 @@ func TestRepositorySync(t *testing.T) { t.Parallel() - e := testenv.NewCLITest(t) + runner := testenv.NewInProcRunner(t) + e := testenv.NewCLITest(t, runner) defer e.RunAndExpectSuccess(t, "repo", "disconnect") @@ -40,7 +41,7 @@ func TestRepositorySync(t *testing.T) { } // now create a whole new repository - e2 := testenv.NewCLITest(t) + e2 := testenv.NewCLITest(t, runner) defer e2.RunAndExpectSuccess(t, "repo", "disconnect") diff --git a/tests/end_to_end_test/restore_fail_test.go b/tests/end_to_end_test/restore_fail_test.go index 6f223c00d..055d81f40 100644 --- a/tests/end_to_end_test/restore_fail_test.go +++ b/tests/end_to_end_test/restore_fail_test.go @@ -31,7 +31,8 @@ func TestRestoreFail(t *testing.T) { t.Parallel() - e := testenv.NewCLITest(t) + runner := testenv.NewExeRunner(t) + e := testenv.NewCLITest(t, runner) defer e.RunAndExpectSuccess(t, "repo", "disconnect") diff --git a/tests/end_to_end_test/restore_test.go b/tests/end_to_end_test/restore_test.go index 2934c6fea..5c2c8798a 100644 --- a/tests/end_to_end_test/restore_test.go +++ b/tests/end_to_end_test/restore_test.go @@ -39,7 +39,8 @@ func TestRestoreCommand(t *testing.T) { t.Parallel() - e := testenv.NewCLITest(t) + runner := testenv.NewInProcRunner(t) + e := testenv.NewCLITest(t, runner) e.RunAndExpectSuccess(t, "repo", "create", "filesystem", "--path", e.RepoDir) @@ -154,7 +155,9 @@ func compareDirs(t *testing.T, source, restoreDir string) { func TestSnapshotRestore(t *testing.T) { t.Parallel() - e := testenv.NewCLITest(t) + runner := testenv.NewInProcRunner(t) + e := testenv.NewCLITest(t, runner) + defer e.RunAndExpectSuccess(t, "repo", "disconnect") e.RunAndExpectSuccess(t, "repo", "create", "filesystem", "--path", e.RepoDir) @@ -329,7 +332,9 @@ func TestSnapshotRestore(t *testing.T) { func TestRestoreSymlinkWithoutTarget(t *testing.T) { t.Parallel() - e := testenv.NewCLITest(t) + runner := testenv.NewInProcRunner(t) + e := testenv.NewCLITest(t, runner) + defer e.RunAndExpectSuccess(t, "repo", "disconnect") e.RunAndExpectSuccess(t, "repo", "create", "filesystem", "--path", e.RepoDir) @@ -363,7 +368,9 @@ func TestRestoreSymlinkWithoutTarget(t *testing.T) { func TestRestoreSymlinkWithNonSymlinkOverwrite(t *testing.T) { t.Parallel() - e := testenv.NewCLITest(t) + runner := testenv.NewInProcRunner(t) + e := testenv.NewCLITest(t, runner) + defer e.RunAndExpectSuccess(t, "repo", "disconnect") e.RunAndExpectSuccess(t, "repo", "create", "filesystem", "--path", e.RepoDir) @@ -406,7 +413,9 @@ func TestRestoreSymlinkWithNonSymlinkOverwrite(t *testing.T) { func TestRestoreSnapshotOfSingleFile(t *testing.T) { t.Parallel() - e := testenv.NewCLITest(t) + runner := testenv.NewInProcRunner(t) + e := testenv.NewCLITest(t, runner) + defer e.RunAndExpectSuccess(t, "repo", "disconnect") e.RunAndExpectSuccess(t, "repo", "create", "filesystem", "--path", e.RepoDir) diff --git a/tests/end_to_end_test/server_start_test.go b/tests/end_to_end_test/server_start_test.go index 6c2ec9735..70c6c81e7 100644 --- a/tests/end_to_end_test/server_start_test.go +++ b/tests/end_to_end_test/server_start_test.go @@ -56,7 +56,9 @@ func (s *serverParameters) ProcessOutput(l string) bool { func TestServerStart(t *testing.T) { ctx := testlogging.Context(t) - e := testenv.NewCLITest(t) + runner := testenv.NewExeRunner(t) + e := testenv.NewCLITest(t, runner) + defer e.RunAndExpectSuccess(t, "repo", "disconnect") e.RunAndExpectSuccess(t, "repo", "create", "filesystem", "--path", e.RepoDir, "--override-hostname=fake-hostname", "--override-username=fake-username") @@ -66,7 +68,8 @@ func TestServerStart(t *testing.T) { var sp serverParameters - e.Environment = append(e.Environment, `KOPIA_UI_TITLE_PREFIX=Blah: `) + runner.Environment = append(runner.Environment, `KOPIA_UI_TITLE_PREFIX=Blah: `) + e.RunAndProcessStderr(t, sp.ProcessOutput, "server", "start", "--ui", @@ -190,7 +193,8 @@ func TestServerCreateAndConnectViaAPI(t *testing.T) { ctx := testlogging.Context(t) - e := testenv.NewCLITest(t) + runner := testenv.NewInProcRunner(t) + e := testenv.NewCLITest(t, runner) defer e.RunAndExpectSuccess(t, "repo", "disconnect") @@ -258,7 +262,8 @@ func TestConnectToExistingRepositoryViaAPI(t *testing.T) { ctx := testlogging.Context(t) - e := testenv.NewCLITest(t) + runner := testenv.NewInProcRunner(t) + e := testenv.NewCLITest(t, runner) e.RunAndExpectSuccess(t, "repo", "create", "filesystem", "--path", e.RepoDir, "--override-hostname=fake-hostname", "--override-username=fake-username") e.RunAndExpectSuccess(t, "snapshot", "create", sharedTestDataDir1) e.RunAndExpectSuccess(t, "snapshot", "create", sharedTestDataDir1) @@ -337,7 +342,9 @@ func TestServerStartInsecure(t *testing.T) { ctx := testlogging.Context(t) - e := testenv.NewCLITest(t) + runner := testenv.NewInProcRunner(t) + e := testenv.NewCLITest(t, runner) + defer e.RunAndExpectSuccess(t, "repo", "disconnect") e.RunAndExpectSuccess(t, "repo", "create", "filesystem", "--path", e.RepoDir, "--override-hostname=fake-hostname", "--override-username=fake-username") @@ -347,7 +354,7 @@ func TestServerStartInsecure(t *testing.T) { var sp serverParameters // server starts without password and no TLS when --insecure is provided. - c := e.RunAndProcessStderr(t, sp.ProcessOutput, + e.RunAndProcessStderr(t, sp.ProcessOutput, "server", "start", "--ui", "--address=localhost:0", @@ -355,8 +362,6 @@ func TestServerStartInsecure(t *testing.T) { "--insecure", ) - defer c.Process.Kill() - cli, err := apiclient.NewKopiaAPIClient(apiclient.Options{ BaseURL: sp.baseURL, }) diff --git a/tests/end_to_end_test/snapshot_actions_test.go b/tests/end_to_end_test/snapshot_actions_test.go index 3e7ce84ed..5c33f67a5 100644 --- a/tests/end_to_end_test/snapshot_actions_test.go +++ b/tests/end_to_end_test/snapshot_actions_test.go @@ -21,17 +21,18 @@ func TestSnapshotActionsBeforeSnapshotRoot(t *testing.T) { th := os.Getenv("TESTING_ACTION_EXE") if th == "" { - t.Skip("TESTING_ACTION_EXE verifyNoError be set") + t.Skip("TESTING_ACTION_EXE must be set") } - e := testenv.NewCLITest(t) + runner := testenv.NewExeRunner(t) + e := testenv.NewCLITest(t, runner) defer e.RunAndExpectSuccess(t, "repo", "disconnect") e.RunAndExpectSuccess(t, "repo", "create", "filesystem", "--path", e.RepoDir, "--override-hostname=foo", "--override-username=foo", "--enable-actions") e.RunAndExpectSuccess(t, "snapshot", "create", sharedTestDataDir2) - envFile1 := filepath.Join(e.LogsDir, "env1.txt") + envFile1 := filepath.Join(runner.LogsDir, "env1.txt") // set a action before-snapshot-root that fails and which saves the environment to a file. e.RunAndExpectSuccess(t, @@ -42,7 +43,7 @@ func TestSnapshotActionsBeforeSnapshotRoot(t *testing.T) { // this prevents the snapshot from being created e.RunAndExpectFailure(t, "snapshot", "create", sharedTestDataDir1) - envFile2 := filepath.Join(e.LogsDir, "env2.txt") + envFile2 := filepath.Join(runner.LogsDir, "env2.txt") // now set a action before-snapshot-root that succeeds and saves environment to a different file e.RunAndExpectSuccess(t, @@ -155,10 +156,11 @@ func TestSnapshotActionsBeforeAfterFolder(t *testing.T) { th := os.Getenv("TESTING_ACTION_EXE") if th == "" { - t.Skip("TESTING_ACTION_EXE verifyNoError be set") + t.Skip("TESTING_ACTION_EXE must be set") } - e := testenv.NewCLITest(t) + runner := testenv.NewExeRunner(t) + e := testenv.NewCLITest(t, runner) e.RunAndExpectSuccess(t, "repo", "create", "filesystem", "--path", e.RepoDir, "--enable-actions") defer e.RunAndExpectSuccess(t, "repo", "disconnect") @@ -226,7 +228,8 @@ func TestSnapshotActionsBeforeAfterFolder(t *testing.T) { func TestSnapshotActionsEmbeddedScript(t *testing.T) { t.Parallel() - e := testenv.NewCLITest(t) + runner := testenv.NewInProcRunner(t) + e := testenv.NewCLITest(t, runner) e.RunAndExpectSuccess(t, "repo", "create", "filesystem", "--path", e.RepoDir, "--enable-actions") defer e.RunAndExpectSuccess(t, "repo", "disconnect") @@ -273,7 +276,7 @@ func TestSnapshotActionsEnable(t *testing.T) { th := os.Getenv("TESTING_ACTION_EXE") if th == "" { - t.Skip("TESTING_ACTION_EXE verifyNoError be set") + t.Skip("TESTING_ACTION_EXE must be set") } cases := []struct { @@ -297,13 +300,14 @@ func TestSnapshotActionsEnable(t *testing.T) { t.Run(tc.desc, func(t *testing.T) { t.Parallel() - e := testenv.NewCLITest(t) + runner := testenv.NewExeRunner(t) + e := testenv.NewCLITest(t, runner) defer e.RunAndExpectSuccess(t, "repo", "disconnect") e.RunAndExpectSuccess(t, append([]string{"repo", "create", "filesystem", "--path", e.RepoDir}, tc.connectFlags...)...) - envFile := filepath.Join(e.LogsDir, "env1.txt") + envFile := filepath.Join(runner.LogsDir, "env1.txt") // set an action before-snapshot-root that fails and which saves the environment to a file. e.RunAndExpectSuccess(t, diff --git a/tests/end_to_end_test/snapshot_copy_move_history_test.go b/tests/end_to_end_test/snapshot_copy_move_history_test.go index 88f7362b6..77f98db83 100644 --- a/tests/end_to_end_test/snapshot_copy_move_history_test.go +++ b/tests/end_to_end_test/snapshot_copy_move_history_test.go @@ -11,7 +11,8 @@ func TestSnapshotCopy(t *testing.T) { t.Parallel() - e := testenv.NewCLITest(t) + runner := testenv.NewInProcRunner(t) + e := testenv.NewCLITest(t, runner) defer e.RunAndExpectSuccess(t, "repo", "disconnect") diff --git a/tests/end_to_end_test/snapshot_create_test.go b/tests/end_to_end_test/snapshot_create_test.go index 15eab9ad0..4dfbce3f1 100644 --- a/tests/end_to_end_test/snapshot_create_test.go +++ b/tests/end_to_end_test/snapshot_create_test.go @@ -23,7 +23,8 @@ func TestSnapshotCreate(t *testing.T) { t.Parallel() - e := testenv.NewCLITest(t) + runner := testenv.NewInProcRunner(t) + e := testenv.NewCLITest(t, runner) defer e.RunAndExpectSuccess(t, "repo", "disconnect") @@ -84,7 +85,8 @@ func TestSnapshotCreate(t *testing.T) { func TestTagging(t *testing.T) { t.Parallel() - e := testenv.NewCLITest(t) + runner := testenv.NewInProcRunner(t) + e := testenv.NewCLITest(t, runner) defer e.RunAndExpectSuccess(t, "repo", "disconnect") @@ -110,7 +112,8 @@ func TestTagging(t *testing.T) { func TestTaggingBadTags(t *testing.T) { t.Parallel() - e := testenv.NewCLITest(t) + runner := testenv.NewInProcRunner(t) + e := testenv.NewCLITest(t, runner) defer e.RunAndExpectSuccess(t, "repo", "disconnect") @@ -129,7 +132,8 @@ func TestTaggingBadTags(t *testing.T) { func TestStartTimeOverride(t *testing.T) { t.Parallel() - e := testenv.NewCLITest(t) + runner := testenv.NewInProcRunner(t) + e := testenv.NewCLITest(t, runner) e.RunAndExpectSuccess(t, "repo", "create", "filesystem", "--path", e.RepoDir) e.RunAndExpectSuccess(t, "snapshot", "create", sharedTestDataDir1, "--start-time", "2000-01-01 01:01:00 UTC") @@ -148,7 +152,8 @@ func TestStartTimeOverride(t *testing.T) { func TestEndTimeOverride(t *testing.T) { t.Parallel() - e := testenv.NewCLITest(t) + runner := testenv.NewInProcRunner(t) + e := testenv.NewCLITest(t, runner) e.RunAndExpectSuccess(t, "repo", "create", "filesystem", "--path", e.RepoDir) e.RunAndExpectSuccess(t, "snapshot", "create", sharedTestDataDir1, "--end-time", "2000-01-01 01:01:00 UTC") @@ -168,7 +173,8 @@ func TestEndTimeOverride(t *testing.T) { func TestInvalidTimeOverride(t *testing.T) { t.Parallel() - e := testenv.NewCLITest(t) + runner := testenv.NewInProcRunner(t) + e := testenv.NewCLITest(t, runner) e.RunAndExpectSuccess(t, "repo", "create", "filesystem", "--path", e.RepoDir) e.RunAndExpectFailure(t, "snapshot", "create", sharedTestDataDir1, "--start-time", "2000-01-01 01:01:00 UTC", "--end-time", "1999-01-01 01:01:00 UTC") @@ -177,7 +183,8 @@ func TestInvalidTimeOverride(t *testing.T) { func TestSnapshottingCacheDirectory(t *testing.T) { t.Parallel() - e := testenv.NewCLITest(t) + runner := testenv.NewInProcRunner(t) + e := testenv.NewCLITest(t, runner) e.RunAndExpectSuccess(t, "repo", "create", "filesystem", "--path", e.RepoDir) e.RunAndExpectSuccess(t, "snapshot", "create", sharedTestDataDir1) @@ -478,7 +485,8 @@ func TestSnapshotCreateWithIgnore(t *testing.T) { for _, tc := range cases { tc := tc t.Run(tc.desc, func(t *testing.T) { - e := testenv.NewCLITest(t) + runner := testenv.NewInProcRunner(t) + e := testenv.NewCLITest(t, runner) baseDir := testutil.TempDirectory(t) @@ -525,7 +533,8 @@ func TestSnapshotCreateWithIgnore(t *testing.T) { func TestSnapshotCreateAllWithManualSnapshot(t *testing.T) { t.Parallel() - e := testenv.NewCLITest(t) + runner := testenv.NewInProcRunner(t) + e := testenv.NewCLITest(t, runner) defer e.RunAndExpectSuccess(t, "repo", "disconnect") e.RunAndExpectSuccess(t, "repo", "create", "filesystem", "--path", e.RepoDir) @@ -552,7 +561,8 @@ func TestSnapshotCreateAllWithManualSnapshot(t *testing.T) { func TestSnapshotCreateWithStdinStream(t *testing.T) { t.Parallel() - e := testenv.NewCLITest(t) + runner := testenv.NewExeRunner(t) + e := testenv.NewCLITest(t, runner) defer e.RunAndExpectSuccess(t, "repo", "disconnect") e.RunAndExpectSuccess(t, "repo", "create", "filesystem", "--path", e.RepoDir) @@ -572,7 +582,7 @@ func TestSnapshotCreateWithStdinStream(t *testing.T) { w.Close() streamFileName := "stream-file" - e.NextCommandStdin = r + runner.NextCommandStdin = r e.RunAndExpectSuccess(t, "snapshot", "create", "rootdir", "--stdin-file", streamFileName) diff --git a/tests/end_to_end_test/snapshot_delete_test.go b/tests/end_to_end_test/snapshot_delete_test.go index dc4f2cb49..df9ddb393 100644 --- a/tests/end_to_end_test/snapshot_delete_test.go +++ b/tests/end_to_end_test/snapshot_delete_test.go @@ -109,7 +109,9 @@ func(manifestID, objectID string, source clitestutil.SourceInfo) []string { func testSnapshotDelete(t *testing.T, argMaker deleteArgMaker, expectDeleteSucceeds bool) { t.Helper() - e := testenv.NewCLITest(t) + runner := testenv.NewInProcRunner(t) + e := testenv.NewCLITest(t, runner) + defer e.RunAndExpectSuccess(t, "repo", "disconnect") e.RunAndExpectSuccess(t, "repo", "create", "filesystem", "--path", e.RepoDir) @@ -143,7 +145,8 @@ func testSnapshotDelete(t *testing.T, argMaker deleteArgMaker, expectDeleteSucce func TestSnapshotDeleteTypeCheck(t *testing.T) { t.Parallel() - e := testenv.NewCLITest(t) + runner := testenv.NewInProcRunner(t) + e := testenv.NewCLITest(t, runner) defer e.RunAndExpectSuccess(t, "repo", "disconnect") @@ -175,7 +178,8 @@ func TestSnapshotDeleteTypeCheck(t *testing.T) { func TestSnapshotDeleteRestore(t *testing.T) { t.Parallel() - e := testenv.NewCLITest(t) + runner := testenv.NewInProcRunner(t) + e := testenv.NewCLITest(t, runner) defer e.RunAndExpectSuccess(t, "repo", "disconnect") diff --git a/tests/end_to_end_test/snapshot_fail_test.go b/tests/end_to_end_test/snapshot_fail_test.go index 450e08af6..a9bfaa25c 100644 --- a/tests/end_to_end_test/snapshot_fail_test.go +++ b/tests/end_to_end_test/snapshot_fail_test.go @@ -21,7 +21,8 @@ func TestSnapshotNonexistent(t *testing.T) { t.Parallel() - e := testenv.NewCLITest(t) + runner := testenv.NewInProcRunner(t) + e := testenv.NewCLITest(t, runner) defer e.RunAndExpectSuccess(t, "repo", "disconnect") @@ -239,7 +240,8 @@ func testSnapshotFail(t *testing.T, isFailFast bool, snapshotCreateFlags, snapsh t.Run(tname, func(t *testing.T) { t.Parallel() - e := testenv.NewCLITest(t) + runner := testenv.NewExeRunner(t) + e := testenv.NewCLITest(t, runner) defer e.RunAndExpectSuccess(t, "repo", "disconnect") @@ -258,7 +260,7 @@ func testSnapshotFail(t *testing.T, isFailFast bool, snapshotCreateFlags, snapsh e.RunAndExpectSuccess(t, "policy", "set", snapSource, "--ignore-dir-errors", tcIgnoreDirErr, "--ignore-file-errors", tcIgnoreFileErr) restoreDir := fmt.Sprintf("%s%d_%v_%v", restoreDirPrefix, tcIdx, tcIgnoreDirErr, tcIgnoreFileErr) - testPermissions(t, e, snapSource, modifyEntry, restoreDir, tc.expectSuccess, snapshotCreateFlags, snapshotCreateEnv) + testPermissions(t, runner, e, snapSource, modifyEntry, restoreDir, tc.expectSuccess, snapshotCreateFlags, snapshotCreateEnv) e.RunAndExpectSuccess(t, "policy", "remove", snapSource) }) @@ -297,7 +299,7 @@ func createSimplestFileTree(t *testing.T, dirDepth, currDepth int, currPath stri // against "source" and will test permissions against all entries in "parentDir". // It returns the number of successful snapshot operations. // nolint:thelper -func testPermissions(t *testing.T, e *testenv.CLITest, source, modifyEntry, restoreDir string, expect map[os.FileMode]expectedSnapshotResult, snapshotCreateFlags, snapshotCreateEnv []string) int { +func testPermissions(t *testing.T, runner *testenv.CLIExeRunner, e *testenv.CLITest, source, modifyEntry, restoreDir string, expect map[os.FileMode]expectedSnapshotResult, snapshotCreateFlags, snapshotCreateEnv []string) int { var numSuccessfulSnapshots int changeFile, err := os.Stat(modifyEntry) @@ -321,14 +323,14 @@ func() { require.NoError(t, err) // set up environment for the child process. - oldEnv := e.Environment - e.Environment = append(append([]string{}, e.Environment...), snapshotCreateEnv...) + oldEnv := runner.Environment + runner.Environment = append(append([]string{}, runner.Environment...), snapshotCreateEnv...) - defer func() { e.Environment = oldEnv }() + defer func() { runner.Environment = oldEnv }() snapshotCreateWithArgs := append([]string{"snapshot", "create", source}, snapshotCreateFlags...) - _, errOut, runErr := e.Run(t, expected.success, snapshotCreateWithArgs...) + _, errOut, runErr := e.Run(t, !expected.success, snapshotCreateWithArgs...) if got, want := (runErr == nil), expected.success; got != want { t.Fatalf("unexpected success %v, want %v", got, want) diff --git a/tests/end_to_end_test/snapshot_gc_test.go b/tests/end_to_end_test/snapshot_gc_test.go index 629231946..70c3d971a 100644 --- a/tests/end_to_end_test/snapshot_gc_test.go +++ b/tests/end_to_end_test/snapshot_gc_test.go @@ -18,7 +18,8 @@ func TestSnapshotGC(t *testing.T) { t.Parallel() - e := testenv.NewCLITest(t) + runner := testenv.NewInProcRunner(t) + e := testenv.NewCLITest(t, runner) e.RunAndExpectSuccess(t, "repo", "create", "filesystem", "--path", e.RepoDir) diff --git a/tests/end_to_end_test/snapshot_migrate_test.go b/tests/end_to_end_test/snapshot_migrate_test.go index f1efc67e0..e697221ee 100644 --- a/tests/end_to_end_test/snapshot_migrate_test.go +++ b/tests/end_to_end_test/snapshot_migrate_test.go @@ -10,7 +10,8 @@ func TestSnapshotMigrate(t *testing.T) { t.Parallel() - e := testenv.NewCLITest(t) + runner := testenv.NewInProcRunner(t) + e := testenv.NewCLITest(t, runner) defer e.RunAndExpectSuccess(t, "repo", "disconnect") @@ -30,7 +31,7 @@ func TestSnapshotMigrate(t *testing.T) { sourceSnapshotCount := len(e.RunAndExpectSuccess(t, "snapshot", "list", ".", "-a")) sourcePolicyCount := len(e.RunAndExpectSuccess(t, "policy", "list")) - dstenv := testenv.NewCLITest(t) + dstenv := testenv.NewCLITest(t, runner) dstenv.RunAndExpectSuccess(t, "repo", "create", "filesystem", "--path", dstenv.RepoDir) diff --git a/tests/end_to_end_test/snapshot_verify_test.go b/tests/end_to_end_test/snapshot_verify_test.go index dcc390528..32a70c1aa 100644 --- a/tests/end_to_end_test/snapshot_verify_test.go +++ b/tests/end_to_end_test/snapshot_verify_test.go @@ -10,7 +10,8 @@ func TestSnapshotVerifyTest(t *testing.T) { t.Parallel() - e := testenv.NewCLITest(t) + runner := testenv.NewInProcRunner(t) + e := testenv.NewCLITest(t, runner) e.RunAndExpectSuccess(t, "repo", "create", "filesystem", "--path", e.RepoDir) diff --git a/tests/endurance_test/endurance_test.go b/tests/endurance_test/endurance_test.go index 12c1be687..67db32cb3 100644 --- a/tests/endurance_test/endurance_test.go +++ b/tests/endurance_test/endurance_test.go @@ -60,7 +60,8 @@ func (d webdavDirWithFakeClock) OpenFile(ctx context.Context, fname string, flag } func TestEndurance(t *testing.T) { - e := testenv.NewCLITest(t) + runner := testenv.NewExeRunner(t) + e := testenv.NewCLITest(t, runner) tmpDir, err := ioutil.TempDir("", "endurance") if err != nil { @@ -74,7 +75,7 @@ func TestEndurance(t *testing.T) { ft := httptest.NewServer(fts) defer ft.Close() - e.Environment = append(e.Environment, "KOPIA_FAKE_CLOCK_ENDPOINT="+ft.URL) + runner.Environment = append(runner.Environment, "KOPIA_FAKE_CLOCK_ENDPOINT="+ft.URL) sts := httptest.NewServer(&webdav.Handler{ FileSystem: webdavDirWithFakeClock{webdav.Dir(tmpDir), fts}, @@ -232,9 +233,10 @@ func pickRandomEnduranceTestAction() action { func enduranceRunner(t *testing.T, runnerID int, fakeTimeServer, webdavServer string, failureCount *int32, nowFunc func() time.Time) { t.Helper() - e := testenv.NewCLITest(t) + runner := testenv.NewExeRunner(t) + e := testenv.NewCLITest(t, runner) - e.Environment = append(e.Environment, + runner.Environment = append(runner.Environment, "KOPIA_FAKE_CLOCK_ENDPOINT="+fakeTimeServer, "KOPIA_CHECK_FOR_UPDATES=false", ) diff --git a/tests/testenv/cli_exe_runner.go b/tests/testenv/cli_exe_runner.go new file mode 100644 index 000000000..701a3357a --- /dev/null +++ b/tests/testenv/cli_exe_runner.go @@ -0,0 +1,126 @@ +package testenv + +import ( + "io" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" + + "github.com/kopia/kopia/internal/clock" +) + +// CLIExeRunner is a CLIExeRunner that invokes the commands via external executable. +type CLIExeRunner struct { + Exe string + Environment []string + PassthroughStderr bool // this is for debugging only + NextCommandStdin io.Reader // this is used for stdin source tests + LogsDir string +} + +// Start implements CLIRunner. +func (e *CLIExeRunner) Start(t *testing.T, args []string) (stdout, stderr io.Reader, wait func() error, kill func()) { + t.Helper() + + c := exec.Command(e.Exe, append([]string{ + "--log-dir", e.LogsDir, + }, args...)...) + + c.Env = append(os.Environ(), e.Environment...) + + stdoutPipe, err := c.StdoutPipe() + if err != nil { + t.Fatalf("can't set up stdout pipe reader: %v", err) + } + + stderrPipe, err := c.StderrPipe() + if err != nil { + t.Fatalf("can't set up stderr pipe reader: %v", err) + } + + c.Stdin = e.NextCommandStdin + e.NextCommandStdin = nil + + if err := c.Start(); err != nil { + t.Fatalf("unable to start: %v", err) + } + + return stdoutPipe, stderrPipe, c.Wait, func() { + c.Process.Kill() + } +} + +// RemoveDefaultPassword prevents KOPIA_PASSWORD from being passed to kopia. +func (e *CLIExeRunner) RemoveDefaultPassword() { + var newEnv []string + + for _, s := range e.Environment { + if !strings.HasPrefix(s, "KOPIA_PASSWORD=") { + newEnv = append(newEnv, s) + } + } + + e.Environment = newEnv +} + +// NewExeRunner resutns a CLIRunner that will execute kopia commands by launching subprocesses +// for each. The kopia executable must be passed via KOPIA_EXE environment variable. The test +// will be skipped if it's not provided (unless running inside an IDE in which case system-wide +// `kopia` will be used by default). +func NewExeRunner(t *testing.T) *CLIExeRunner { + t.Helper() + + exe := os.Getenv("KOPIA_EXE") + if exe == "" { + if os.Getenv("VSCODE_PID") != "" { + // we're launched from VSCode, use system-installed kopia executable. + exe = "kopia" + } else { + t.Skip() + } + } + + // unset environment variables that disrupt tests when passed to subprocesses. + os.Unsetenv("KOPIA_PASSWORD") + + cleanName := strings.ReplaceAll(strings.ReplaceAll(strings.ReplaceAll( + t.Name(), + "/", "_"), "\\", "_"), ":", "_") + + logsBaseDir := os.Getenv("KOPIA_LOGS_DIR") + if logsBaseDir == "" { + logsBaseDir = filepath.Join(os.TempDir(), "kopia-logs") + } + + logsDir := filepath.Join(logsBaseDir, cleanName+"."+clock.Now().Local().Format("20060102150405")) + + t.Cleanup(func() { + if t.Failed() { + t.Logf("FAULURE ABOVE ^^^^") + } + + if os.Getenv("KOPIA_KEEP_LOGS") != "" { + t.Logf("logs preserved in %v", logsDir) + return + } + + if t.Failed() && os.Getenv("KOPIA_DISABLE_LOG_DUMP_ON_FAILURE") == "" { + dumpLogs(t, logsDir) + } + + os.RemoveAll(logsDir) + }) + + return &CLIExeRunner{ + Exe: filepath.FromSlash(exe), + Environment: []string{ + "KOPIA_PASSWORD=" + TestRepoPassword, + "KOPIA_ADVANCED_COMMANDS=enabled", + }, + LogsDir: logsDir, + } +} + +var _ CLIRunner = (*CLIExeRunner)(nil) diff --git a/tests/testenv/cli_inproc_runner.go b/tests/testenv/cli_inproc_runner.go new file mode 100644 index 000000000..b122a6c50 --- /dev/null +++ b/tests/testenv/cli_inproc_runner.go @@ -0,0 +1,47 @@ +package testenv + +import ( + "io" + "os" + "testing" + + "github.com/kopia/kopia/cli" + "github.com/kopia/kopia/internal/buf" + "github.com/kopia/kopia/internal/testlogging" +) + +// CLIInProcRunner is a CLIRunner that invokes provided commands in the current process. +type CLIInProcRunner struct{} + +// Start implements CLIRunner. +func (e *CLIInProcRunner) Start(t *testing.T, args []string) (stdout, stderr io.Reader, wait func() error, kill func()) { + t.Helper() + + ctx := testlogging.Context(t) + + a := cli.NewApp() + a.AdvancedCommands = "enabled" + + return a.RunSubcommand(ctx, append([]string{ + "--password", TestRepoPassword, + }, args...)) +} + +// NewInProcRunner returns a runner that executes CLI subcommands in the current process using cli.RunSubcommand(). +func NewInProcRunner(t *testing.T) *CLIInProcRunner { + t.Helper() + + if os.Getenv("KOPIA_EXE") != "" { + t.Skip("not running test since it's also included in the unit tests") + } + + return &CLIInProcRunner{} +} + +var _ CLIRunner = (*CLIInProcRunner)(nil) + +func init() { + // disable buffer management in end-to-end tests as running too many of them in parallel causes too + // much memory usage on low-end platforms. + buf.DisableBufferManagement = true +} diff --git a/tests/testenv/cli_test_env.go b/tests/testenv/cli_test_env.go index a891d22fe..bcbe6ffe3 100644 --- a/tests/testenv/cli_test_env.go +++ b/tests/testenv/cli_test_env.go @@ -3,18 +3,18 @@ import ( "bufio" - "bytes" "fmt" "io" "io/ioutil" - "os" - "os/exec" "path/filepath" "runtime" "strings" + "sync" "testing" "time" + "github.com/stretchr/testify/require" + "github.com/kopia/kopia/internal/clock" "github.com/kopia/kopia/internal/testutil" ) @@ -26,77 +26,34 @@ maxOutputLinesToLog = 4000 ) +// CLIRunner encapsulates running kopia subcommands for testing purposes. +// It supports implementations that use subprocesses or in-process invocations. +type CLIRunner interface { + Start(t *testing.T, args []string) (stdout, stderr io.Reader, wait func() error, kill func()) +} + // CLITest encapsulates state for a CLI-based test. type CLITest struct { startTime time.Time RepoDir string ConfigDir string - Exe string - fixedArgs []string - Environment []string + Runner CLIRunner + + fixedArgs []string DefaultRepositoryCreateFlags []string - - PassthroughStderr bool // this is for debugging only - - NextCommandStdin io.Reader // this is used for stdin source tests - - LogsDir string } // NewCLITest creates a new instance of *CLITest. -func NewCLITest(t *testing.T) *CLITest { +func NewCLITest(t *testing.T, runner CLIRunner) *CLITest { t.Helper() - - exe := os.Getenv("KOPIA_EXE") - if exe == "" { - if os.Getenv("VSCODE_PID") != "" { - // we're launched from VSCode, use system-installed kopia executable. - exe = "kopia" - } else { - t.Skip() - } - } - - // unset environment variables that disrupt tests when passed to subprocesses. - os.Unsetenv("KOPIA_PASSWORD") - configDir := testutil.TempDirectory(t) - cleanName := strings.ReplaceAll(strings.ReplaceAll(strings.ReplaceAll( - t.Name(), - "/", "_"), "\\", "_"), ":", "_") - - logsBaseDir := os.Getenv("KOPIA_LOGS_DIR") - if logsBaseDir == "" { - logsBaseDir = filepath.Join(os.TempDir(), "kopia-logs") - } - - logsDir := filepath.Join(logsBaseDir, cleanName+"."+clock.Now().Local().Format("20060102150405")) - - t.Cleanup(func() { - if t.Failed() { - t.Logf("FAULURE ABOVE ^^^^") - } - - if os.Getenv("KOPIA_KEEP_LOGS") != "" { - t.Logf("logs preserved in %v", logsDir) - return - } - - if t.Failed() && os.Getenv("KOPIA_DISABLE_LOG_DUMP_ON_FAILURE") == "" { - dumpLogs(t, logsDir) - } - - os.RemoveAll(logsDir) - }) - fixedArgs := []string{ // use per-test config file, to avoid clobbering current user's setup. "--config-file", filepath.Join(configDir, ".kopia.config"), - "--log-dir", logsDir, } // disable the use of keyring @@ -122,14 +79,9 @@ func NewCLITest(t *testing.T) *CLITest { startTime: clock.Now(), RepoDir: testutil.TempDirectory(t), ConfigDir: configDir, - Exe: filepath.FromSlash(exe), fixedArgs: fixedArgs, DefaultRepositoryCreateFlags: formatFlags, - LogsDir: logsDir, - Environment: []string{ - "KOPIA_PASSWORD=" + TestRepoPassword, - "KOPIA_ADVANCED_COMMANDS=enabled", - }, + Runner: runner, } } @@ -165,19 +117,6 @@ func dumpLogFile(t *testing.T, fname string) { t.Logf("LOG FILE: %v %v", fname, trimOutput(string(data))) } -// RemoveDefaultPassword prevents KOPIA_PASSWORD from being passed to kopia. -func (e *CLITest) RemoveDefaultPassword() { - var newEnv []string - - for _, s := range e.Environment { - if !strings.HasPrefix(s, "KOPIA_PASSWORD=") { - newEnv = append(newEnv, s) - } - } - - e.Environment = newEnv -} - // RunAndExpectSuccess runs the given command, expects it to succeed and returns its output lines. func (e *CLITest) RunAndExpectSuccess(t *testing.T, args ...string) []string { t.Helper() @@ -191,23 +130,13 @@ func (e *CLITest) RunAndExpectSuccess(t *testing.T, args ...string) []string { } // RunAndProcessStderr runs the given command, and streams its output line-by-line to a given function until it returns false. -func (e *CLITest) RunAndProcessStderr(t *testing.T, callback func(line string) bool, args ...string) *exec.Cmd { +func (e *CLITest) RunAndProcessStderr(t *testing.T, callback func(line string) bool, args ...string) (kill func()) { t.Helper() - c := exec.Command(e.Exe, e.cmdArgs(args)...) - c.Env = append(os.Environ(), e.Environment...) - t.Logf("running '%v %v'", c.Path, c.Args) + stdout, stderr, _, kill := e.Runner.Start(t, e.cmdArgs(args)) + go io.Copy(io.Discard, stdout) - stderrPipe, err := c.StderrPipe() - if err != nil { - t.Fatalf("can't set up stderr pipe reader") - } - - if err := c.Start(); err != nil { - t.Fatalf("unable to start") - } - - scanner := bufio.NewScanner(stderrPipe) + scanner := bufio.NewScanner(stderr) for scanner.Scan() { if !callback(scanner.Text()) { break @@ -221,7 +150,7 @@ func (e *CLITest) RunAndProcessStderr(t *testing.T, callback func(line string) b } }() - return c + return kill } // RunAndExpectSuccessWithErrOut runs the given command, expects it to succeed and returns its stdout and stderr lines. @@ -276,28 +205,50 @@ func (e *CLITest) cmdArgs(args []string) []string { func (e *CLITest) Run(t *testing.T, expectedError bool, args ...string) (stdout, stderr []string, err error) { t.Helper() - c := exec.Command(e.Exe, e.cmdArgs(args)...) - c.Env = append(os.Environ(), e.Environment...) + t.Logf("running 'kopia %v'", strings.Join(args, " ")) - t.Logf("running '%v %v'", c.Path, c.Args) + args = e.cmdArgs(args) + t0 := clock.Now() - errOut := &bytes.Buffer{} - c.Stderr = errOut + stdoutReader, stderrReader, wait, _ := e.Runner.Start(t, args) - if e.PassthroughStderr { - c.Stderr = os.Stderr + var wg sync.WaitGroup + + wg.Add(1) + + go func() { + defer wg.Done() + + scanner := bufio.NewScanner(stdoutReader) + for scanner.Scan() { + stdout = append(stdout, scanner.Text()) + } + }() + + wg.Add(1) + + go func() { + defer wg.Done() + + scanner := bufio.NewScanner(stderrReader) + for scanner.Scan() { + stderr = append(stderr, scanner.Text()) + } + }() + + wg.Wait() + + gotErr := wait() + + if expectedError { + require.Error(t, gotErr, "unexpected success when running 'kopia %v' (stdout:\n%v\nstderr:\n%v", strings.Join(args, " "), strings.Join(stdout, "\n"), strings.Join(stderr, "\n")) + } else { + require.NoError(t, gotErr, "unexpected error when running 'kopia %v' (stdout:\n%v\nstderr:\n%v", strings.Join(args, " "), strings.Join(stdout, "\n"), strings.Join(stderr, "\n")) } - c.Stdin = e.NextCommandStdin - e.NextCommandStdin = nil + t.Logf("finished in %v: 'kopia %v'", clock.Since(t0).Milliseconds(), strings.Join(args, " ")) - o, err := c.Output() - - if err != nil && !expectedError { - t.Logf("finished 'kopia %v' with err=%v (expected=%v) and output:\n%v\nstderr:\n%v\n", strings.Join(args, " "), err, expectedError, trimOutput(string(o)), trimOutput(errOut.String())) - } - - return splitLines(string(o)), splitLines(errOut.String()), err + return stdout, stderr, gotErr } func trimOutput(s string) string {