Refactored most of the CLI tests to run in-process as opposed to using sub-processes (#1059)

* cli: fixed remaining testability indirections for output and logging

* cli: added cli.RunSubcommand() which is used in testing to execute a subcommand in the same process

* tests: refactored most e2e tests to invoke kopia subcommands in-process

* Makefile: enable code coverage for cli/ and internal/

* testing: pass 'testing' tag to unit tests which uses much faster (insecure) password hashing scheme

* Makefile: push coverage from PRs again

* tests: disable buffer management to reduce memory usage on ARM

* cli: fixed misaligned atomic field on ARMHF

also temporarily fixed statup-time benign race condition when setting
default on the timeZone variable, which is the last global variable.

Makefile

@@ -1,4 +1,4 @@
-COVERAGE_PACKAGES=github.com/kopia/kopia/repo/...,github.com/kopia/kopia/fs/...,github.com/kopia/kopia/snapshot/...
+COVERAGE_PACKAGES=./repo/...,./fs/...,./snapshot/...,./cli/...,./internal/...
 TEST_FLAGS?=
 
 KOPIA_INTEGRATION_EXE=$(CURDIR)/dist/testing_$(GOOS)_$(GOARCH)/kopia.exe
@@ -153,7 +153,7 @@ ci-integration-tests: integration-tests robustness-tool-tests
 	$(MAKE) stress-test
 
 ci-publish-coverage:
-ifeq ($(GOOS)/$(GOARCH)/$(IS_PULL_REQUEST),linux/amd64/false)
+ifeq ($(GOOS)/$(GOARCH),linux/amd64)
 	-bash -c "bash <(curl -s https://codecov.io/bash) -f coverage.txt"
 endif
 
@@ -186,11 +186,11 @@ dev-deps:
 	GO111MODULE=off go get -u github.com/sqs/goreturns
 
 test-with-coverage: $(gotestsum)
-	$(GO_TEST) $(UNIT_TEST_RACE_FLAGS) -count=$(REPEAT_TEST) -covermode=atomic -coverprofile=coverage.txt --coverpkg $(COVERAGE_PACKAGES) -timeout 300s ./...
+	$(GO_TEST) $(UNIT_TEST_RACE_FLAGS) -tags testing -count=$(REPEAT_TEST) -covermode=atomic -coverprofile=coverage.txt --coverpkg $(COVERAGE_PACKAGES) -timeout 300s ./...
 
 test: GOTESTSUM_FLAGS=--format=$(GOTESTSUM_FORMAT) --no-summary=skipped --jsonfile=.tmp.unit-tests.json
 test: $(gotestsum)
-	$(GO_TEST) $(UNIT_TEST_RACE_FLAGS) -count=$(REPEAT_TEST) -timeout $(UNIT_TESTS_TIMEOUT) ./...
+	$(GO_TEST) $(UNIT_TEST_RACE_FLAGS) -tags testing -count=$(REPEAT_TEST) -timeout $(UNIT_TESTS_TIMEOUT) ./...
 	-$(gotestsum) tool slowest --jsonfile .tmp.unit-tests.json  --threshold 1000ms
 
 provider-tests: export KOPIA_PROVIDER_TEST=true

cli/app.go

@@ -79,6 +79,7 @@ type appServices interface {
 	repositoryWriterAction(act func(ctx context.Context, rep repo.RepositoryWriter) error) func(ctx *kingpin.ParseContext) error
 	maybeRepositoryAction(act func(ctx context.Context, rep repo.Repository) error, mode repositoryAccessMode) func(ctx *kingpin.ParseContext) error
 
+	advancedCommand(ctx context.Context)
 	repositoryConfigFileName() string
 	getProgress() *cliProgress
 
@@ -99,6 +100,8 @@ type advancedAppServices interface {
 	passwordPersistenceStrategy() passwordpersist.Strategy
 	getPasswordFromFlags(ctx context.Context, isNew, allowPersistent bool) (string, error)
 	optionsFromFlags(ctx context.Context) *repo.Options
+
+	rootContext() context.Context
 }
 
 // App contains per-invocation flags and state of Kopia CLI.
@@ -116,6 +119,7 @@ type App struct {
 	metricsListenAddr             string
 	keyRingEnabled                bool
 	persistCredentials            bool
+	AdvancedCommands              string
 
 	// subcommands
 	blob        commandBlob
@@ -140,6 +144,7 @@ type App struct {
 	osExit       func(int) // allows replacing os.Exit() with custom code
 	stdoutWriter io.Writer
 	stderrWriter io.Writer
+	rootctx      context.Context
 }
 
 func (c *App) getProgress() *cliProgress {
@@ -185,9 +190,10 @@ func (c *App) setup(app *kingpin.Application) {
 	app.Flag("config-file", "Specify the config file to use.").Default(defaultConfigFileName()).Envar("KOPIA_CONFIG_PATH").StringVar(&c.configPath)
 	app.Flag("trace-storage", "Enables tracing of storage operations.").Default("true").Hidden().BoolVar(&c.traceStorage)
 	app.Flag("metrics-listen-addr", "Expose Prometheus metrics on a given host:port").Hidden().StringVar(&c.metricsListenAddr)
-	app.Flag("timezone", "Format time according to specified time zone (local, utc, original or time zone name)").Default("local").Hidden().StringVar(&timeZone)
+	app.Flag("timezone", "Format time according to specified time zone (local, utc, original or time zone name)").Hidden().StringVar(&timeZone)
 	app.Flag("password", "Repository password.").Envar("KOPIA_PASSWORD").Short('p').StringVar(&c.password)
 	app.Flag("persist-credentials", "Persist credentials").Default("true").Envar("KOPIA_PERSIST_CREDENTIALS_ON_CONNECT").BoolVar(&c.persistCredentials)
+	app.Flag("advanced-commands", "Enable advanced (and potentially dangerous) commands.").Hidden().Envar("KOPIA_ADVANCED_COMMANDS").StringVar(&c.AdvancedCommands)
 
 	c.setupOSSpecificKeychainFlags(app)
 
@@ -235,6 +241,7 @@ func NewApp() *App {
 		osExit:       os.Exit,
 		stdoutWriter: os.Stdout,
 		stderrWriter: os.Stderr,
+		rootctx:      context.Background(),
 	}
 }
 
@@ -268,7 +275,7 @@ func safetyFlagVar(cmd *kingpin.CmdClause, result *maintenance.SafetyParameters)
 
 func (c *App) noRepositoryAction(act func(ctx context.Context) error) func(ctx *kingpin.ParseContext) error {
 	return func(_ *kingpin.ParseContext) error {
-		return act(rootContext())
+		return act(c.rootContext())
 	}
 }
 
@@ -284,7 +291,7 @@ func (c *App) serverAction(sf *serverClientFlags, act func(ctx context.Context,
 			return errors.Wrap(err, "unable to create API client")
 		}
 
-		return act(rootContext(), apiClient)
+		return act(c.rootContext(), apiClient)
 	}
 }
 
@@ -348,8 +355,8 @@ func (c *App) repositoryWriterAction(act func(ctx context.Context, rep repo.Repo
 	})
 }
 
-func rootContext() context.Context {
-	return context.Background()
+func (c *App) rootContext() context.Context {
+	return c.rootctx
 }
 
 type repositoryAccessMode struct {
@@ -359,7 +366,7 @@ type repositoryAccessMode struct {
 
 func (c *App) maybeRepositoryAction(act func(ctx context.Context, rep repo.Repository) error, mode repositoryAccessMode) func(ctx *kingpin.ParseContext) error {
 	return func(kpc *kingpin.ParseContext) error {
-		ctx := rootContext()
+		ctx := c.rootContext()
 
 		if err := withProfiling(func() error {
 			c.mt.startMemoryTracking(ctx)
@@ -398,7 +405,7 @@ func (c *App) maybeRepositoryAction(act func(ctx context.Context, rep repo.Repos
 		}); err != nil {
 			// print error in red
 			log(ctx).Errorf("ERROR: %v", err.Error())
-			os.Exit(1)
+			c.osExit(1)
 		}
 
 		return nil
@@ -436,15 +443,20 @@ func (c *App) maybeRunMaintenance(ctx context.Context, rep repo.Repository) erro
 	return errors.Wrap(err, "error running maintenance")
 }
 
-func advancedCommand(ctx context.Context) {
-	if os.Getenv("KOPIA_ADVANCED_COMMANDS") != "enabled" {
-		log(ctx).Errorf(`
+func (c *App) advancedCommand(ctx context.Context) {
+	if c.AdvancedCommands != "enabled" {
+		_, _ = errorColor.Fprintf(c.stderrWriter, `
 This command could be dangerous or lead to repository corruption when used improperly.
 
 Running this command is not needed for using Kopia. Instead, most users should rely on periodic repository maintenance. See https://kopia.io/docs/advanced/maintenance/ for more information.
 To run this command despite the warning, set KOPIA_ADVANCED_COMMANDS=enabled
 
 `)
-		os.Exit(1)
+
+		c.osExit(1)
 	}
 }
+
+func init() {
+	kingpin.EnableFileExpansion = false
+}

cli/command_blob_delete.go

@@ -11,16 +11,20 @@
 
 type commandBlobDelete struct {
 	blobIDs []string
+
+	svc appServices
 }
 
 func (c *commandBlobDelete) setup(svc appServices, parent commandParent) {
 	cmd := parent.Command("delete", "Delete blobs by ID").Alias("remove").Alias("rm")
 	cmd.Arg("blobIDs", "Blob IDs").Required().StringsVar(&c.blobIDs)
 	cmd.Action(svc.directRepositoryWriteAction(c.run))
+
+	c.svc = svc
 }
 
 func (c *commandBlobDelete) run(ctx context.Context, rep repo.DirectRepositoryWriter) error {
-	advancedCommand(ctx)
+	c.svc.advancedCommand(ctx)
 
 	for _, b := range c.blobIDs {
 		err := rep.BlobStorage().DeleteBlob(ctx, blob.ID(b))

cli/command_blob_gc.go

@@ -15,6 +15,8 @@ type commandBlobGC struct {
 	parallel int
 	prefix   string
 	safety   maintenance.SafetyParameters
+
+	svc appServices
 }
 
 func (c *commandBlobGC) setup(svc appServices, parent commandParent) {
@@ -24,10 +26,12 @@ func (c *commandBlobGC) setup(svc appServices, parent commandParent) {
 	cmd.Flag("prefix", "Only GC blobs with given prefix").StringVar(&c.prefix)
 	safetyFlagVar(cmd, &c.safety)
 	cmd.Action(svc.directRepositoryWriteAction(c.run))
+
+	c.svc = svc
 }
 
 func (c *commandBlobGC) run(ctx context.Context, rep repo.DirectRepositoryWriter) error {
-	advancedCommand(ctx)
+	c.svc.advancedCommand(ctx)
 
 	opts := maintenance.DeleteUnreferencedBlobsOptions{
 		DryRun:   c.delete != "yes",

cli/command_blob_show.go

@@ -5,7 +5,6 @@
 	"context"
 	"encoding/json"
 	"io"
-	"os"
 
 	"github.com/pkg/errors"
 
@@ -17,6 +16,8 @@
 type commandBlobShow struct {
 	blobShowDecrypt bool
 	blobShowIDs     []string
+
+	out textOutput
 }
 
 func (c *commandBlobShow) setup(svc appServices, parent commandParent) {
@@ -24,11 +25,13 @@ func (c *commandBlobShow) setup(svc appServices, parent commandParent) {
 	cmd.Flag("decrypt", "Decrypt blob if possible").BoolVar(&c.blobShowDecrypt)
 	cmd.Arg("blobID", "Blob IDs").Required().StringsVar(&c.blobShowIDs)
 	cmd.Action(svc.directRepositoryReadAction(c.run))
+
+	c.out.setup(svc)
 }
 
 func (c *commandBlobShow) run(ctx context.Context, rep repo.DirectRepository) error {
 	for _, blobID := range c.blobShowIDs {
-		if err := c.maybeDecryptBlob(ctx, os.Stdout, rep, blob.ID(blobID)); err != nil {
+		if err := c.maybeDecryptBlob(ctx, c.out.stdout(), rep, blob.ID(blobID)); err != nil {
 			return errors.Wrap(err, "error presenting blob")
 		}
 	}

cli/command_content_delete.go

@@ -10,16 +10,20 @@
 
 type commandContentDelete struct {
 	ids []string
+
+	svc appServices
 }
 
 func (c *commandContentDelete) setup(svc appServices, parent commandParent) {
 	cmd := parent.Command("delete", "Remove content").Alias("remove").Alias("rm")
 	cmd.Arg("id", "IDs of content to remove").Required().StringsVar(&c.ids)
 	cmd.Action(svc.directRepositoryWriteAction(c.run))
+
+	c.svc = svc
 }
 
 func (c *commandContentDelete) run(ctx context.Context, rep repo.DirectRepositoryWriter) error {
-	advancedCommand(ctx)
+	c.svc.advancedCommand(ctx)
 
 	for _, contentID := range toContentIDs(c.ids) {
 		if err := rep.ContentManager().DeleteContent(ctx, contentID); err != nil {

cli/command_content_rewrite.go

@@ -19,6 +19,7 @@ type commandContentRewrite struct {
 	contentRewriteSafety        maintenance.SafetyParameters
 
 	contentRange contentRangeFlags
+	svc          appServices
 }
 
 func (c *commandContentRewrite) setup(svc appServices, parent commandParent) {
@@ -33,10 +34,12 @@ func (c *commandContentRewrite) setup(svc appServices, parent commandParent) {
 	c.contentRange.setup(cmd)
 	safetyFlagVar(cmd, &c.contentRewriteSafety)
 	cmd.Action(svc.directRepositoryWriteAction(c.runContentRewriteCommand))
+
+	c.svc = svc
 }
 
 func (c *commandContentRewrite) runContentRewriteCommand(ctx context.Context, rep repo.DirectRepositoryWriter) error {
-	advancedCommand(ctx)
+	c.svc.advancedCommand(ctx)
 
 	return maintenance.RewriteContents(ctx, rep, &maintenance.RewriteContentsOptions{
 		ContentIDRange: c.contentRange.contentIDRange(),

cli/command_content_show.go

@@ -14,6 +14,8 @@ type commandContentShow struct {
 	ids        []string
 	indentJSON bool
 	decompress bool
+
+	out textOutput
 }
 
 func (c *commandContentShow) setup(svc appServices, parent commandParent) {
@@ -23,6 +25,8 @@ func (c *commandContentShow) setup(svc appServices, parent commandParent) {
 	cmd.Flag("json", "Pretty-print JSON content").Short('j').BoolVar(&c.indentJSON)
 	cmd.Flag("unzip", "Transparently decompress the content").Short('z').BoolVar(&c.decompress)
 	cmd.Action(svc.directRepositoryReadAction(c.run))
+
+	c.out.setup(svc)
 }
 
 func (c *commandContentShow) run(ctx context.Context, rep repo.DirectRepository) error {
@@ -41,5 +45,5 @@ func (c *commandContentShow) contentShow(ctx context.Context, r repo.DirectRepos
 		return errors.Wrapf(err, "error getting content %v", contentID)
 	}
 
-	return showContentWithFlags(bytes.NewReader(data), c.decompress, c.indentJSON)
+	return showContentWithFlags(c.out.stdout(), bytes.NewReader(data), c.decompress, c.indentJSON)
 }

cli/command_diff.go

@@ -2,7 +2,6 @@
 
 import (
 	"context"
-	"os"
 	"strings"
 
 	"github.com/pkg/errors"
@@ -18,6 +17,8 @@ type commandDiff struct {
 	diffSecondObjectPath string
 	diffCompareFiles     bool
 	diffCommandCommand   string
+
+	out textOutput
 }
 
 func (c *commandDiff) setup(svc appServices, parent commandParent) {
@@ -27,6 +28,8 @@ func (c *commandDiff) setup(svc appServices, parent commandParent) {
 	cmd.Flag("files", "Compare files by launching diff command for all pairs of (old,new)").Short('f').BoolVar(&c.diffCompareFiles)
 	cmd.Flag("diff-command", "Displays differences between two repository objects (files or directories)").Default(defaultDiffCommand()).Envar("KOPIA_DIFF").StringVar(&c.diffCommandCommand)
 	cmd.Action(svc.repositoryReaderAction(c.run))
+
+	c.out.setup(svc)
 }
 
 func (c *commandDiff) run(ctx context.Context, rep repo.Repository) error {
@@ -47,7 +50,7 @@ func (c *commandDiff) run(ctx context.Context, rep repo.Repository) error {
 		return errors.New("arguments do diff must both be directories or both non-directories")
 	}
 
-	d, err := diff.NewComparer(os.Stdout)
+	d, err := diff.NewComparer(c.out.stdout())
 	if err != nil {
 		return errors.Wrap(err, "error creating comparer")
 	}

cli/command_index_optimize.go

@@ -13,6 +13,8 @@ type commandIndexOptimize struct {
 	optimizeDropDeletedOlderThan time.Duration
 	optimizeDropContents         []string
 	optimizeAllIndexes           bool
+
+	svc appServices
 }
 
 func (c *commandIndexOptimize) setup(svc appServices, parent commandParent) {
@@ -22,10 +24,12 @@ func (c *commandIndexOptimize) setup(svc appServices, parent commandParent) {
 	cmd.Flag("drop-contents", "Drop contents with given IDs").StringsVar(&c.optimizeDropContents)
 	cmd.Flag("all", "Optimize all indexes, even those above maximum size.").BoolVar(&c.optimizeAllIndexes)
 	cmd.Action(svc.directRepositoryWriteAction(c.runOptimizeCommand))
+
+	c.svc = svc
 }
 
 func (c *commandIndexOptimize) runOptimizeCommand(ctx context.Context, rep repo.DirectRepositoryWriter) error {
-	advancedCommand(ctx)
+	c.svc.advancedCommand(ctx)
 
 	opt := content.CompactOptions{
 		MaxSmallBlobs: c.optimizeMaxSmallBlobs,

cli/command_index_recover.go

@@ -13,6 +13,8 @@
 type commandIndexRecover struct {
 	blobIDs []string
 	commit  bool
+
+	svc appServices
 }
 
 func (c *commandIndexRecover) setup(svc appServices, parent commandParent) {
@@ -20,10 +22,12 @@ func (c *commandIndexRecover) setup(svc appServices, parent commandParent) {
 	cmd.Flag("blobs", "Names of pack blobs to recover from (default=all packs)").StringsVar(&c.blobIDs)
 	cmd.Flag("commit", "Commit recovered content").BoolVar(&c.commit)
 	cmd.Action(svc.directRepositoryWriteAction(c.run))
+
+	c.svc = svc
 }
 
 func (c *commandIndexRecover) run(ctx context.Context, rep repo.DirectRepositoryWriter) error {
-	advancedCommand(ctx)
+	c.svc.advancedCommand(ctx)
 
 	var totalCount int
 

cli/command_ls.go

@@ -113,7 +113,7 @@ func (c *commandList) printDirectoryEntry(ctx context.Context, e fs.Entry, prefi
 		info = fmt.Sprintf("%v%v", c.nameToDisplay(prefix, e), errorSummary)
 	}
 
-	col.Println(info) //nolint:errcheck
+	col.Fprintln(c.out.stdout(), info) //nolint:errcheck
 
 	if c.recursive {
 		if subdir, ok := e.(fs.Directory); ok {

cli/command_manifest_delete.go

@@ -10,16 +10,20 @@
 
 type commandManifestDelete struct {
 	manifestRemoveItems []string
+
+	svc appServices
 }
 
 func (c *commandManifestDelete) setup(svc appServices, parent commandParent) {
 	cmd := parent.Command("delete", "Remove manifest items").Alias("remove").Alias("rm")
 	cmd.Arg("item", "Items to remove").Required().StringsVar(&c.manifestRemoveItems)
 	cmd.Action(svc.repositoryWriterAction(c.run))
+
+	c.svc = svc
 }
 
 func (c *commandManifestDelete) run(ctx context.Context, rep repo.RepositoryWriter) error {
-	advancedCommand(ctx)
+	c.svc.advancedCommand(ctx)
 
 	for _, it := range toManifestIDs(c.manifestRemoveItems) {
 		if err := rep.DeleteManifest(ctx, it); err != nil {

cli/command_manifest_show.go

@@ -51,7 +51,7 @@ func (c *commandManifestShow) showManifestItems(ctx context.Context, rep repo.Re
 			c.out.printStdout("// label %v:%v\n", k, v)
 		}
 
-		if showerr := showContentWithFlags(bytes.NewReader(b), false, true); showerr != nil {
+		if showerr := showContentWithFlags(c.out.stdout(), bytes.NewReader(b), false, true); showerr != nil {
 			return showerr
 		}
 	}

cli/command_repository_connect.go

@@ -31,7 +31,7 @@ func (c *commandRepositoryConnect) setup(svc advancedAppServices, parent command
 		cc := cmd.Command(prov.name, "Connect to repository in "+prov.description)
 		f.setup(svc, cc)
 		cc.Action(func(_ *kingpin.ParseContext) error {
-			ctx := rootContext()
+			ctx := svc.rootContext()
 			st, err := f.connect(ctx, false)
 			if err != nil {
 				return errors.Wrap(err, "can't connect to storage")

cli/command_repository_create.go

@@ -49,7 +49,7 @@ func (c *commandRepositoryCreate) setup(svc advancedAppServices, parent commandP
 		cc := cmd.Command(prov.name, "Create repository in "+prov.description)
 		f.setup(svc, cc)
 		cc.Action(func(_ *kingpin.ParseContext) error {
-			ctx := rootContext()
+			ctx := svc.rootContext()
 			st, err := f.connect(ctx, true)
 			if err != nil {
 				return errors.Wrap(err, "can't connect to storage")

cli/command_repository_repair.go

@@ -30,7 +30,7 @@ func (c *commandRepositoryRepair) setup(svc advancedAppServices, parent commandP
 		cc := cmd.Command(prov.name, "Repair repository in "+prov.description)
 		f.setup(svc, cc)
 		cc.Action(func(_ *kingpin.ParseContext) error {
-			ctx := rootContext()
+			ctx := svc.rootContext()
 			st, err := f.connect(ctx, false)
 			if err != nil {
 				return errors.Wrap(err, "can't connect to storage")

cli/command_repository_sync.go

@@ -21,6 +21,8 @@
 )
 
 type commandRepositorySyncTo struct {
+	nextSyncOutputTime *timetrack.Throttle
+
 	repositorySyncUpdate               bool
 	repositorySyncDelete               bool
 	repositorySyncDryRun               bool
@@ -30,7 +32,6 @@ type commandRepositorySyncTo struct {
 
 	lastSyncProgress       string
 	syncProgressMutex      sync.Mutex
-	nextSyncOutputTime     timetrack.Throttle
 	setTimeUnsupportedOnce sync.Once
 
 	out textOutput
@@ -47,13 +48,16 @@ func (c *commandRepositorySyncTo) setup(svc advancedAppServices, parent commandP
 
 	c.out.setup(svc)
 
+	// needs to be 64-bit aligned on ARM
+	c.nextSyncOutputTime = new(timetrack.Throttle)
+
 	for _, prov := range storageProviders {
 		// Set up 'sync-to' subcommand
 		f := prov.newFlags()
 		cc := cmd.Command(prov.name, "Synchronize repository data to another repository in "+prov.description)
 		f.setup(svc, cc)
 		cc.Action(func(_ *kingpin.ParseContext) error {
-			ctx := rootContext()
+			ctx := svc.rootContext()
 			st, err := f.connect(ctx, false)
 			if err != nil {
 				return errors.Wrap(err, "can't connect to storage")

cli/command_show.go

@@ -2,7 +2,6 @@
 
 import (
 	"context"
-	"os"
 
 	"github.com/pkg/errors"
 
@@ -13,12 +12,16 @@
 
 type commandShow struct {
 	path string
+
+	out textOutput
 }
 
 func (c *commandShow) setup(svc appServices, parent commandParent) {
 	cmd := parent.Command("show", "Displays contents of a repository object.").Alias("cat")
 	cmd.Arg("object-path", "Path").Required().StringVar(&c.path)
 	cmd.Action(svc.repositoryReaderAction(c.run))
+
+	c.out.setup(svc)
 }
 
 func (c *commandShow) run(ctx context.Context, rep repo.Repository) error {
@@ -34,7 +37,7 @@ func (c *commandShow) run(ctx context.Context, rep repo.Repository) error {
 
 	defer r.Close() //nolint:errcheck
 
-	_, err = iocopy.Copy(os.Stdout, r)
+	_, err = iocopy.Copy(c.out.stdout(), r)
 
 	return errors.Wrap(err, "unable to copy data")
 }

cli/command_snapshot_list.go

@@ -252,7 +252,7 @@ func (c *commandSnapshotList) outputManifestFromSingleSource(ctx context.Context
 		elidedCount = 0
 		previousOID = oid
 
-		col.Print(fmt.Sprintf("  %v %v %v\n", formatTimestamp(m.StartTime), oid, strings.Join(bits, " "))) //nolint:errcheck
+		col.Fprint(c.out.stdout(), fmt.Sprintf("  %v %v %v\n", formatTimestamp(m.StartTime), oid, strings.Join(bits, " "))) //nolint:errcheck
 
 		count++
 

cli/inproc.go

@@ -0,0 +1,55 @@
+package cli
+
+import (
+	"context"
+	"io"
+
+	"github.com/alecthomas/kingpin"
+	"github.com/pkg/errors"
+
+	"github.com/kopia/kopia/repo/logging"
+)
+
+// RunSubcommand executes the subcommand asynchronously in current process
+// with flags in an isolated CLI environment and returns standard output and standard error.
+func (c *App) RunSubcommand(ctx context.Context, argsAndFlags []string) (stdout, stderr io.Reader, wait func() error, kill func()) {
+	kpapp := kingpin.New("test", "test")
+
+	stdoutReader, stdoutWriter := io.Pipe()
+	stderrReader, stderrWriter := io.Pipe()
+
+	c.stdoutWriter = stdoutWriter
+	c.stderrWriter = stderrWriter
+	c.rootctx = logging.WithLogger(ctx, logging.Writer(stderrWriter))
+
+	c.Attach(kpapp)
+
+	var exitCode int
+
+	resultErr := make(chan error, 1)
+
+	c.osExit = func(ec int) {
+		exitCode = ec
+	}
+
+	go func() {
+		defer close(resultErr)
+		defer stderrWriter.Close() //nolint:errcheck
+		defer stdoutWriter.Close() //nolint:errcheck
+
+		_, err := kpapp.Parse(argsAndFlags)
+		if err != nil {
+			resultErr <- err
+			return
+		}
+
+		if exitCode != 0 {
+			resultErr <- errors.Errorf("exit code %v", exitCode)
+			return
+		}
+	}()
+
+	return stdoutReader, stderrReader, func() error {
+		return <-resultErr
+	}, func() {}
+}

cli/show_utils.go

@@ -7,7 +7,6 @@
 	"fmt"
 	"io"
 	"io/ioutil"
-	"os"
 	"time"
 
 	"github.com/pkg/errors"
@@ -17,9 +16,9 @@
 )
 
 // TODO - remove this global.
-var timeZone string
+var timeZone = "local"
 
-func showContentWithFlags(rd io.Reader, unzip, indentJSON bool) error {
+func showContentWithFlags(w io.Writer, rd io.Reader, unzip, indentJSON bool) error {
 	if unzip {
 		gz, err := gzip.NewReader(rd)
 		if err != nil {
@@ -43,7 +42,7 @@ func showContentWithFlags(rd io.Reader, unzip, indentJSON bool) error {
 		rd = ioutil.NopCloser(&buf2)
 	}
 
-	if _, err := iocopy.Copy(os.Stdout, rd); err != nil {
+	if _, err := iocopy.Copy(w, rd); err != nil {
 		return errors.Wrap(err, "error copying data")
 	}
 

internal/buf/pool.go

@@ -12,6 +12,10 @@
 	"go.opencensus.io/tag"
 )
 
+// DisableBufferManagement is a global flag that disables memory buffer reuse,
+// which can be useful in tests to reduce overall memory usage.
+var DisableBufferManagement = false
+
 type segment struct {
 	mu sync.RWMutex
 
@@ -103,7 +107,7 @@ func (s *segment) allocate(n int) (Buf, bool) {
 //
 // The pool uses N segments, with each segment tracking its high water mark usage.
 //
-// Allocation simply advances the high water mark within first segment that has capacity
+// ation simply advances the high water mark within first segment that has capacity
 // and increments per-segment refcount.
 //
 // On Buf.Release() the refcount is decremented and when it hits zero, the entire segment becomes instantly
@@ -243,7 +247,7 @@ func (p *Pool) AddSegments(n int) {
 // Allocate allocates from the buffer a slice of size n.
 func (p *Pool) Allocate(n int) Buf {
 	// requested more than the pool can cache, allocate throw-away buffer.
-	if p == nil || n > p.segmentSize {
+	if p == nil || n > p.segmentSize || DisableBufferManagement {
 		return Buf{make([]byte, n), 0, 0, nil}
 	}
 

main.go

@@ -69,8 +69,6 @@ func main() {
 	app := cli.NewApp()
 	kp := kingpin.New("kopia", "Kopia - Fast And Secure Open-Source Backup").Author("http://kopia.github.io/")
 
-	kingpin.EnableFileExpansion = false
-
 	logging.SetDefault(func(module string) logging.Logger {
 		return gologging.MustGetLogger(module)
 	})

repo/logging/printf_logger.go

@@ -1,5 +1,11 @@
 package logging
 
+import (
+	"fmt"
+	"io"
+	"strings"
+)
+
 type printfLogger struct {
 	printf func(msg string, args ...interface{})
 	prefix string
@@ -15,3 +21,20 @@ func Printf(printf func(msg string, args ...interface{})) LoggerForModuleFunc {
 		return &printfLogger{printf, "[" + module + "]"}
 	}
 }
+
+// Writer returns LoggerForModuleFunc that uses given writer for log output.
+func Writer(w io.Writer) LoggerForModuleFunc {
+	printf := func(msg string, args ...interface{}) {
+		msg = fmt.Sprintf(msg, args...)
+
+		if !strings.HasSuffix(msg, "\n") {
+			msg += "\n"
+		}
+
+		io.WriteString(w, msg) //nolint:errcheck
+	}
+
+	return func(module string) Logger {
+		return &printfLogger{printf, ""}
+	}
+}

tests/end_to_end_test/acl_test.go

@@ -11,7 +11,9 @@
 func TestACL(t *testing.T) {
 	t.Parallel()
 
-	serverEnvironment := testenv.NewCLITest(t)
+	serverRunner := testenv.NewExeRunner(t)
+	serverEnvironment := testenv.NewCLITest(t, serverRunner)
+
 	defer serverEnvironment.RunAndExpectSuccess(t, "repo", "disconnect")
 
 	serverEnvironment.RunAndExpectSuccess(t, "repo", "create", "filesystem", "--path", serverEnvironment.RepoDir, "--override-hostname=foo", "--override-username=foo", "--enable-actions")
@@ -39,7 +41,7 @@ func TestACL(t *testing.T) {
 
 	var sp serverParameters
 
-	srv := serverEnvironment.RunAndProcessStderr(t, sp.ProcessOutput,
+	kill := serverEnvironment.RunAndProcessStderr(t, sp.ProcessOutput,
 		"server", "start",
 		"--address=localhost:0",
 		"--server-username=admin-user",
@@ -50,12 +52,14 @@ func TestACL(t *testing.T) {
 
 	t.Logf("detected server parameters %#v", sp)
 
-	defer srv.Process.Kill()
+	defer kill()
+
+	fooBarRunner := testenv.NewExeRunner(t)
+	foobarClientEnvironment := testenv.NewCLITest(t, fooBarRunner)
 
-	foobarClientEnvironment := testenv.NewCLITest(t)
 	defer foobarClientEnvironment.RunAndExpectSuccess(t, "repo", "disconnect")
 
-	foobarClientEnvironment.RemoveDefaultPassword()
+	fooBarRunner.RemoveDefaultPassword()
 
 	// connect as foo@bar with password baz
 	foobarClientEnvironment.RunAndExpectSuccess(t, "repo", "connect", "server",
@@ -66,10 +70,12 @@ func TestACL(t *testing.T) {
 		"--password", "baz",
 	)
 
-	aliceInWonderlandClientEnvironment := testenv.NewCLITest(t)
+	aliceInWonderlandRunner := testenv.NewExeRunner(t)
+	aliceInWonderlandClientEnvironment := testenv.NewCLITest(t, aliceInWonderlandRunner)
+
 	defer aliceInWonderlandClientEnvironment.RunAndExpectSuccess(t, "repo", "disconnect")
 
-	aliceInWonderlandClientEnvironment.RemoveDefaultPassword()
+	aliceInWonderlandRunner.RemoveDefaultPassword()
 
 	// connect as alice@wonderland with password baz
 	aliceInWonderlandClientEnvironment.RunAndExpectSuccess(t, "repo", "connect", "server",

tests/end_to_end_test/all_formats_test.go

@@ -14,6 +14,8 @@
 func TestAllFormatsSmokeTest(t *testing.T) {
 	srcDir := testutil.TempDirectory(t)
 
+	runner := testenv.NewInProcRunner(t)
+
 	// 3-level directory with <=10 files and <=10 subdirectories at each level
 	testdirtree.CreateDirectoryTree(srcDir, testdirtree.DirectoryTreeOptions{
 		Depth:                  2,
@@ -32,7 +34,7 @@ func TestAllFormatsSmokeTest(t *testing.T) {
 				t.Run(hashAlgo, func(t *testing.T) {
 					t.Parallel()
 
-					e := testenv.NewCLITest(t)
+					e := testenv.NewCLITest(t, runner)
 					defer e.RunAndExpectSuccess(t, "repo", "disconnect")
 
 					e.DefaultRepositoryCreateFlags = nil

tests/end_to_end_test/api_server_repository_test.go

@@ -55,14 +55,16 @@ func testAPIServerRepository(t *testing.T, serverStartArgs []string, useGRPC, al
 		connectArgs = []string{"--no-grpc"}
 	}
 
-	e := testenv.NewCLITest(t)
+	runner := testenv.NewExeRunner(t)
+	e := testenv.NewCLITest(t, runner)
+
 	defer e.RunAndExpectSuccess(t, "repo", "disconnect")
 
 	// create one snapshot as foo@bar
 	e.RunAndExpectSuccess(t, "repo", "create", "filesystem", "--path", e.RepoDir, "--override-username", "foo", "--override-hostname", "bar")
 	e.RunAndExpectSuccess(t, "snapshot", "create", sharedTestDataDir1)
 
-	e1 := testenv.NewCLITest(t)
+	e1 := testenv.NewCLITest(t, runner)
 	defer e1.RunAndExpectSuccess(t, "repo", "disconnect")
 
 	// create one snapshot as not-foo@bar
@@ -201,7 +203,9 @@ func testAPIServerRepository(t *testing.T, serverStartArgs []string, useGRPC, al
 		verifyFindManifestCount(ctx, t, writeSess, someLabels, 1)
 	}
 
-	e2 := testenv.NewCLITest(t)
+	runner2 := testenv.NewExeRunner(t)
+	e2 := testenv.NewCLITest(t, runner2)
+
 	defer e2.RunAndExpectSuccess(t, "repo", "disconnect")
 
 	e2.RunAndExpectSuccess(t, append([]string{
@@ -216,7 +220,7 @@ func testAPIServerRepository(t *testing.T, serverStartArgs []string, useGRPC, al
 	// we are providing custom password to connect, make sure we won't be providing
 	// (different) default password via environment variable, as command-line password
 	// takes precedence over persisted password.
-	e2.RemoveDefaultPassword()
+	runner2.RemoveDefaultPassword()
 
 	// should see one snapshot
 	snapshots := clitestutil.ListSnapshotsAndExpectSuccess(t, e2)

tests/end_to_end_test/auto_update_test.go

@@ -40,14 +40,15 @@ func TestAutoUpdateEnableTest(t *testing.T) {
 
 		t.Run(tc.desc, func(t *testing.T) {
 			t.Parallel()
-			e := testenv.NewCLITest(t)
+			runner := testenv.NewExeRunner(t)
+			e := testenv.NewCLITest(t, runner)
 
 			// create repo
 			args := append([]string{
 				"repo", "create", "filesystem", "--path", e.RepoDir,
 			}, tc.extraArgs...)
 
-			e.Environment = append(e.Environment, tc.extraEnv...)
+			runner.Environment = append(runner.Environment, tc.extraEnv...)
 			e.RunAndExpectSuccess(t, args...)
 
 			updateInfoFile := filepath.Join(e.ConfigDir, ".kopia.config.update-info.json")

tests/end_to_end_test/compression_test.go

@@ -17,7 +17,9 @@
 func TestCompression(t *testing.T) {
 	t.Parallel()
 
-	e := testenv.NewCLITest(t)
+	runner := testenv.NewInProcRunner(t)
+	e := testenv.NewCLITest(t, runner)
+
 	defer e.RunAndExpectSuccess(t, "repo", "disconnect")
 
 	e.RunAndExpectSuccess(t, "repo", "create", "filesystem", "--path", e.RepoDir)

tests/end_to_end_test/diff_test.go

@@ -16,7 +16,9 @@
 func TestDiff(t *testing.T) {
 	t.Parallel()
 
-	e := testenv.NewCLITest(t)
+	runner := testenv.NewInProcRunner(t)
+	e := testenv.NewCLITest(t, runner)
+
 	defer e.RunAndExpectSuccess(t, "repo", "disconnect")
 
 	e.RunAndExpectSuccess(t, "repo", "create", "filesystem", "--path", e.RepoDir)

tests/end_to_end_test/index_optimize_test.go

@@ -9,7 +9,9 @@
 func TestIndexOptimize(t *testing.T) {
 	t.Parallel()
 
-	e := testenv.NewCLITest(t)
+	runner := testenv.NewInProcRunner(t)
+	e := testenv.NewCLITest(t, runner)
+
 	defer e.RunAndExpectSuccess(t, "repo", "disconnect")
 
 	e.RunAndExpectSuccess(t, "repo", "create", "filesystem", "--path", e.RepoDir)

tests/end_to_end_test/index_recover_test.go

@@ -13,7 +13,8 @@
 func TestIndexRecover(t *testing.T) {
 	t.Parallel()
 
-	e := testenv.NewCLITest(t)
+	runner := testenv.NewInProcRunner(t)
+	e := testenv.NewCLITest(t, runner)
 
 	e.RunAndExpectSuccess(t, "repo", "create", "filesystem", "--path", e.RepoDir)
 

tests/end_to_end_test/maintenance_test.go

@@ -12,7 +12,8 @@
 func TestFullMaintenance(t *testing.T) {
 	t.Parallel()
 
-	e := testenv.NewCLITest(t)
+	runner := testenv.NewInProcRunner(t)
+	e := testenv.NewCLITest(t, runner)
 
 	e.RunAndExpectSuccess(t, "repo", "create", "filesystem", "--path", e.RepoDir)
 	defer e.RunAndExpectSuccess(t, "repo", "disconnect")

tests/end_to_end_test/policy_test.go

@@ -13,7 +13,8 @@
 func TestDefaultGlobalPolicy(t *testing.T) {
 	t.Parallel()
 
-	e := testenv.NewCLITest(t)
+	runner := testenv.NewInProcRunner(t)
+	e := testenv.NewCLITest(t, runner)
 
 	e.RunAndExpectSuccess(t, "repo", "create", "filesystem", "--path", e.RepoDir)
 

tests/end_to_end_test/repository_connect_test.go

@@ -14,7 +14,8 @@
 func TestFilesystemRequiresAbsolutePaths(t *testing.T) {
 	t.Parallel()
 
-	e := testenv.NewCLITest(t)
+	runner := testenv.NewInProcRunner(t)
+	e := testenv.NewCLITest(t, runner)
 
 	e.RunAndExpectFailure(t, "repo", "create", "filesystem", "--path", "./relative-path")
 }
@@ -27,7 +28,8 @@ func TestFilesystemSupportsTildeToReferToHome(t *testing.T) {
 		t.Skip("home directory not available")
 	}
 
-	e := testenv.NewCLITest(t)
+	runner := testenv.NewInProcRunner(t)
+	e := testenv.NewCLITest(t, runner)
 
 	subdir := "repo-" + uuid.NewString()
 	fullPath := filepath.Join(home, subdir)
@@ -45,7 +47,8 @@ func TestFilesystemSupportsTildeToReferToHome(t *testing.T) {
 func TestReconnect(t *testing.T) {
 	t.Parallel()
 
-	e := testenv.NewCLITest(t)
+	runner := testenv.NewInProcRunner(t)
+	e := testenv.NewCLITest(t, runner)
 
 	e.RunAndExpectSuccess(t, "repo", "create", "filesystem", "--path", e.RepoDir)
 	e.RunAndExpectSuccess(t, "repo", "disconnect")
@@ -56,7 +59,8 @@ func TestReconnect(t *testing.T) {
 func TestReconnectUsingToken(t *testing.T) {
 	t.Parallel()
 
-	e := testenv.NewCLITest(t)
+	runner := testenv.NewInProcRunner(t)
+	e := testenv.NewCLITest(t, runner)
 
 	e.RunAndExpectSuccess(t, "repo", "create", "filesystem", "--path", e.RepoDir)
 	lines := e.RunAndExpectSuccess(t, "repo", "status", "-t", "-s")

tests/end_to_end_test/repository_repair_test.go

@@ -9,7 +9,9 @@
 func TestRepositoryRepair(t *testing.T) {
 	t.Parallel()
 
-	e := testenv.NewCLITest(t)
+	runner := testenv.NewInProcRunner(t)
+	e := testenv.NewCLITest(t, runner)
+
 	defer e.RunAndExpectSuccess(t, "repo", "disconnect")
 
 	e.RunAndExpectSuccess(t, "repo", "create", "filesystem", "--path", e.RepoDir)
@@ -31,7 +33,7 @@ func TestRepositoryRepair(t *testing.T) {
 	e.RunAndExpectFailure(t, "repo", "connect", "filesystem", "--path", e.RepoDir)
 
 	// now run repair, which will recover the format blob from one of the pack blobs.
-	e.RunAndExpectSuccess(t, "repo", "repair", "--log-level=debug", "--trace-storage", "filesystem", "--path", e.RepoDir)
+	e.RunAndExpectSuccess(t, "repo", "repair", "filesystem", "--path", e.RepoDir)
 
 	// now connect can succeed
 	e.RunAndExpectSuccess(t, "repo", "connect", "filesystem", "--path", e.RepoDir)

tests/end_to_end_test/repository_set_client_test.go

@@ -10,7 +10,8 @@
 func TestRepositorySetClient(t *testing.T) {
 	t.Parallel()
 
-	e := testenv.NewCLITest(t)
+	runner := testenv.NewInProcRunner(t)
+	e := testenv.NewCLITest(t, runner)
 
 	defer e.RunAndExpectSuccess(t, "repo", "disconnect")
 

tests/end_to_end_test/repository_sync_test.go

@@ -11,7 +11,8 @@
 func TestRepositorySync(t *testing.T) {
 	t.Parallel()
 
-	e := testenv.NewCLITest(t)
+	runner := testenv.NewInProcRunner(t)
+	e := testenv.NewCLITest(t, runner)
 
 	defer e.RunAndExpectSuccess(t, "repo", "disconnect")
 
@@ -40,7 +41,7 @@ func TestRepositorySync(t *testing.T) {
 	}
 
 	// now create a whole new repository
-	e2 := testenv.NewCLITest(t)
+	e2 := testenv.NewCLITest(t, runner)
 
 	defer e2.RunAndExpectSuccess(t, "repo", "disconnect")
 

tests/end_to_end_test/restore_fail_test.go

@@ -31,7 +31,8 @@
 func TestRestoreFail(t *testing.T) {
 	t.Parallel()
 
-	e := testenv.NewCLITest(t)
+	runner := testenv.NewExeRunner(t)
+	e := testenv.NewCLITest(t, runner)
 
 	defer e.RunAndExpectSuccess(t, "repo", "disconnect")
 

tests/end_to_end_test/restore_test.go

@@ -39,7 +39,8 @@
 func TestRestoreCommand(t *testing.T) {
 	t.Parallel()
 
-	e := testenv.NewCLITest(t)
+	runner := testenv.NewInProcRunner(t)
+	e := testenv.NewCLITest(t, runner)
 
 	e.RunAndExpectSuccess(t, "repo", "create", "filesystem", "--path", e.RepoDir)
 
@@ -154,7 +155,9 @@ func compareDirs(t *testing.T, source, restoreDir string) {
 func TestSnapshotRestore(t *testing.T) {
 	t.Parallel()
 
-	e := testenv.NewCLITest(t)
+	runner := testenv.NewInProcRunner(t)
+	e := testenv.NewCLITest(t, runner)
+
 	defer e.RunAndExpectSuccess(t, "repo", "disconnect")
 
 	e.RunAndExpectSuccess(t, "repo", "create", "filesystem", "--path", e.RepoDir)
@@ -329,7 +332,9 @@ func TestSnapshotRestore(t *testing.T) {
 func TestRestoreSymlinkWithoutTarget(t *testing.T) {
 	t.Parallel()
 
-	e := testenv.NewCLITest(t)
+	runner := testenv.NewInProcRunner(t)
+	e := testenv.NewCLITest(t, runner)
+
 	defer e.RunAndExpectSuccess(t, "repo", "disconnect")
 
 	e.RunAndExpectSuccess(t, "repo", "create", "filesystem", "--path", e.RepoDir)
@@ -363,7 +368,9 @@ func TestRestoreSymlinkWithoutTarget(t *testing.T) {
 func TestRestoreSymlinkWithNonSymlinkOverwrite(t *testing.T) {
 	t.Parallel()
 
-	e := testenv.NewCLITest(t)
+	runner := testenv.NewInProcRunner(t)
+	e := testenv.NewCLITest(t, runner)
+
 	defer e.RunAndExpectSuccess(t, "repo", "disconnect")
 
 	e.RunAndExpectSuccess(t, "repo", "create", "filesystem", "--path", e.RepoDir)
@@ -406,7 +413,9 @@ func TestRestoreSymlinkWithNonSymlinkOverwrite(t *testing.T) {
 func TestRestoreSnapshotOfSingleFile(t *testing.T) {
 	t.Parallel()
 
-	e := testenv.NewCLITest(t)
+	runner := testenv.NewInProcRunner(t)
+	e := testenv.NewCLITest(t, runner)
+
 	defer e.RunAndExpectSuccess(t, "repo", "disconnect")
 
 	e.RunAndExpectSuccess(t, "repo", "create", "filesystem", "--path", e.RepoDir)

tests/end_to_end_test/server_start_test.go

@@ -56,7 +56,9 @@ func (s *serverParameters) ProcessOutput(l string) bool {
 func TestServerStart(t *testing.T) {
 	ctx := testlogging.Context(t)
 
-	e := testenv.NewCLITest(t)
+	runner := testenv.NewExeRunner(t)
+	e := testenv.NewCLITest(t, runner)
+
 	defer e.RunAndExpectSuccess(t, "repo", "disconnect")
 
 	e.RunAndExpectSuccess(t, "repo", "create", "filesystem", "--path", e.RepoDir, "--override-hostname=fake-hostname", "--override-username=fake-username")
@@ -66,7 +68,8 @@ func TestServerStart(t *testing.T) {
 
 	var sp serverParameters
 
-	e.Environment = append(e.Environment, `KOPIA_UI_TITLE_PREFIX=Blah: <script>bleh</script> `)
+	runner.Environment = append(runner.Environment, `KOPIA_UI_TITLE_PREFIX=Blah: <script>bleh</script> `)
+
 	e.RunAndProcessStderr(t, sp.ProcessOutput,
 		"server", "start",
 		"--ui",
@@ -190,7 +193,8 @@ func TestServerCreateAndConnectViaAPI(t *testing.T) {
 
 	ctx := testlogging.Context(t)
 
-	e := testenv.NewCLITest(t)
+	runner := testenv.NewInProcRunner(t)
+	e := testenv.NewCLITest(t, runner)
 
 	defer e.RunAndExpectSuccess(t, "repo", "disconnect")
 
@@ -258,7 +262,8 @@ func TestConnectToExistingRepositoryViaAPI(t *testing.T) {
 
 	ctx := testlogging.Context(t)
 
-	e := testenv.NewCLITest(t)
+	runner := testenv.NewInProcRunner(t)
+	e := testenv.NewCLITest(t, runner)
 	e.RunAndExpectSuccess(t, "repo", "create", "filesystem", "--path", e.RepoDir, "--override-hostname=fake-hostname", "--override-username=fake-username")
 	e.RunAndExpectSuccess(t, "snapshot", "create", sharedTestDataDir1)
 	e.RunAndExpectSuccess(t, "snapshot", "create", sharedTestDataDir1)
@@ -337,7 +342,9 @@ func TestServerStartInsecure(t *testing.T) {
 
 	ctx := testlogging.Context(t)
 
-	e := testenv.NewCLITest(t)
+	runner := testenv.NewInProcRunner(t)
+	e := testenv.NewCLITest(t, runner)
+
 	defer e.RunAndExpectSuccess(t, "repo", "disconnect")
 
 	e.RunAndExpectSuccess(t, "repo", "create", "filesystem", "--path", e.RepoDir, "--override-hostname=fake-hostname", "--override-username=fake-username")
@@ -347,7 +354,7 @@ func TestServerStartInsecure(t *testing.T) {
 	var sp serverParameters
 
 	// server starts without password and no TLS when --insecure is provided.
-	c := e.RunAndProcessStderr(t, sp.ProcessOutput,
+	e.RunAndProcessStderr(t, sp.ProcessOutput,
 		"server", "start",
 		"--ui",
 		"--address=localhost:0",
@@ -355,8 +362,6 @@ func TestServerStartInsecure(t *testing.T) {
 		"--insecure",
 	)
 
-	defer c.Process.Kill()
-
 	cli, err := apiclient.NewKopiaAPIClient(apiclient.Options{
 		BaseURL: sp.baseURL,
 	})

tests/end_to_end_test/snapshot_actions_test.go

@@ -21,17 +21,18 @@ func TestSnapshotActionsBeforeSnapshotRoot(t *testing.T) {
 
 	th := os.Getenv("TESTING_ACTION_EXE")
 	if th == "" {
-		t.Skip("TESTING_ACTION_EXE verifyNoError be set")
+		t.Skip("TESTING_ACTION_EXE must be set")
 	}
 
-	e := testenv.NewCLITest(t)
+	runner := testenv.NewExeRunner(t)
+	e := testenv.NewCLITest(t, runner)
 
 	defer e.RunAndExpectSuccess(t, "repo", "disconnect")
 
 	e.RunAndExpectSuccess(t, "repo", "create", "filesystem", "--path", e.RepoDir, "--override-hostname=foo", "--override-username=foo", "--enable-actions")
 	e.RunAndExpectSuccess(t, "snapshot", "create", sharedTestDataDir2)
 
-	envFile1 := filepath.Join(e.LogsDir, "env1.txt")
+	envFile1 := filepath.Join(runner.LogsDir, "env1.txt")
 
 	// set a action before-snapshot-root that fails and which saves the environment to a file.
 	e.RunAndExpectSuccess(t,
@@ -42,7 +43,7 @@ func TestSnapshotActionsBeforeSnapshotRoot(t *testing.T) {
 	// this prevents the snapshot from being created
 	e.RunAndExpectFailure(t, "snapshot", "create", sharedTestDataDir1)
 
-	envFile2 := filepath.Join(e.LogsDir, "env2.txt")
+	envFile2 := filepath.Join(runner.LogsDir, "env2.txt")
 
 	// now set a action before-snapshot-root that succeeds and saves environment to a different file
 	e.RunAndExpectSuccess(t,
@@ -155,10 +156,11 @@ func TestSnapshotActionsBeforeAfterFolder(t *testing.T) {
 
 	th := os.Getenv("TESTING_ACTION_EXE")
 	if th == "" {
-		t.Skip("TESTING_ACTION_EXE verifyNoError be set")
+		t.Skip("TESTING_ACTION_EXE must be set")
 	}
 
-	e := testenv.NewCLITest(t)
+	runner := testenv.NewExeRunner(t)
+	e := testenv.NewCLITest(t, runner)
 
 	e.RunAndExpectSuccess(t, "repo", "create", "filesystem", "--path", e.RepoDir, "--enable-actions")
 	defer e.RunAndExpectSuccess(t, "repo", "disconnect")
@@ -226,7 +228,8 @@ func TestSnapshotActionsBeforeAfterFolder(t *testing.T) {
 func TestSnapshotActionsEmbeddedScript(t *testing.T) {
 	t.Parallel()
 
-	e := testenv.NewCLITest(t)
+	runner := testenv.NewInProcRunner(t)
+	e := testenv.NewCLITest(t, runner)
 
 	e.RunAndExpectSuccess(t, "repo", "create", "filesystem", "--path", e.RepoDir, "--enable-actions")
 	defer e.RunAndExpectSuccess(t, "repo", "disconnect")
@@ -273,7 +276,7 @@ func TestSnapshotActionsEnable(t *testing.T) {
 
 	th := os.Getenv("TESTING_ACTION_EXE")
 	if th == "" {
-		t.Skip("TESTING_ACTION_EXE verifyNoError be set")
+		t.Skip("TESTING_ACTION_EXE must be set")
 	}
 
 	cases := []struct {
@@ -297,13 +300,14 @@ func TestSnapshotActionsEnable(t *testing.T) {
 		t.Run(tc.desc, func(t *testing.T) {
 			t.Parallel()
 
-			e := testenv.NewCLITest(t)
+			runner := testenv.NewExeRunner(t)
+			e := testenv.NewCLITest(t, runner)
 
 			defer e.RunAndExpectSuccess(t, "repo", "disconnect")
 
 			e.RunAndExpectSuccess(t, append([]string{"repo", "create", "filesystem", "--path", e.RepoDir}, tc.connectFlags...)...)
 
-			envFile := filepath.Join(e.LogsDir, "env1.txt")
+			envFile := filepath.Join(runner.LogsDir, "env1.txt")
 
 			// set an action before-snapshot-root that fails and which saves the environment to a file.
 			e.RunAndExpectSuccess(t,

tests/end_to_end_test/snapshot_copy_move_history_test.go

@@ -11,7 +11,8 @@
 func TestSnapshotCopy(t *testing.T) {
 	t.Parallel()
 
-	e := testenv.NewCLITest(t)
+	runner := testenv.NewInProcRunner(t)
+	e := testenv.NewCLITest(t, runner)
 
 	defer e.RunAndExpectSuccess(t, "repo", "disconnect")
 

tests/end_to_end_test/snapshot_create_test.go

@@ -23,7 +23,8 @@
 func TestSnapshotCreate(t *testing.T) {
 	t.Parallel()
 
-	e := testenv.NewCLITest(t)
+	runner := testenv.NewInProcRunner(t)
+	e := testenv.NewCLITest(t, runner)
 
 	defer e.RunAndExpectSuccess(t, "repo", "disconnect")
 
@@ -84,7 +85,8 @@ func TestSnapshotCreate(t *testing.T) {
 func TestTagging(t *testing.T) {
 	t.Parallel()
 
-	e := testenv.NewCLITest(t)
+	runner := testenv.NewInProcRunner(t)
+	e := testenv.NewCLITest(t, runner)
 
 	defer e.RunAndExpectSuccess(t, "repo", "disconnect")
 
@@ -110,7 +112,8 @@ func TestTagging(t *testing.T) {
 func TestTaggingBadTags(t *testing.T) {
 	t.Parallel()
 
-	e := testenv.NewCLITest(t)
+	runner := testenv.NewInProcRunner(t)
+	e := testenv.NewCLITest(t, runner)
 
 	defer e.RunAndExpectSuccess(t, "repo", "disconnect")
 
@@ -129,7 +132,8 @@ func TestTaggingBadTags(t *testing.T) {
 func TestStartTimeOverride(t *testing.T) {
 	t.Parallel()
 
-	e := testenv.NewCLITest(t)
+	runner := testenv.NewInProcRunner(t)
+	e := testenv.NewCLITest(t, runner)
 
 	e.RunAndExpectSuccess(t, "repo", "create", "filesystem", "--path", e.RepoDir)
 	e.RunAndExpectSuccess(t, "snapshot", "create", sharedTestDataDir1, "--start-time", "2000-01-01 01:01:00 UTC")
@@ -148,7 +152,8 @@ func TestStartTimeOverride(t *testing.T) {
 func TestEndTimeOverride(t *testing.T) {
 	t.Parallel()
 
-	e := testenv.NewCLITest(t)
+	runner := testenv.NewInProcRunner(t)
+	e := testenv.NewCLITest(t, runner)
 
 	e.RunAndExpectSuccess(t, "repo", "create", "filesystem", "--path", e.RepoDir)
 	e.RunAndExpectSuccess(t, "snapshot", "create", sharedTestDataDir1, "--end-time", "2000-01-01 01:01:00 UTC")
@@ -168,7 +173,8 @@ func TestEndTimeOverride(t *testing.T) {
 func TestInvalidTimeOverride(t *testing.T) {
 	t.Parallel()
 
-	e := testenv.NewCLITest(t)
+	runner := testenv.NewInProcRunner(t)
+	e := testenv.NewCLITest(t, runner)
 
 	e.RunAndExpectSuccess(t, "repo", "create", "filesystem", "--path", e.RepoDir)
 	e.RunAndExpectFailure(t, "snapshot", "create", sharedTestDataDir1, "--start-time", "2000-01-01 01:01:00 UTC", "--end-time", "1999-01-01 01:01:00 UTC")
@@ -177,7 +183,8 @@ func TestInvalidTimeOverride(t *testing.T) {
 func TestSnapshottingCacheDirectory(t *testing.T) {
 	t.Parallel()
 
-	e := testenv.NewCLITest(t)
+	runner := testenv.NewInProcRunner(t)
+	e := testenv.NewCLITest(t, runner)
 
 	e.RunAndExpectSuccess(t, "repo", "create", "filesystem", "--path", e.RepoDir)
 	e.RunAndExpectSuccess(t, "snapshot", "create", sharedTestDataDir1)
@@ -478,7 +485,8 @@ func TestSnapshotCreateWithIgnore(t *testing.T) {
 	for _, tc := range cases {
 		tc := tc
 		t.Run(tc.desc, func(t *testing.T) {
-			e := testenv.NewCLITest(t)
+			runner := testenv.NewInProcRunner(t)
+			e := testenv.NewCLITest(t, runner)
 
 			baseDir := testutil.TempDirectory(t)
 
@@ -525,7 +533,8 @@ func TestSnapshotCreateWithIgnore(t *testing.T) {
 func TestSnapshotCreateAllWithManualSnapshot(t *testing.T) {
 	t.Parallel()
 
-	e := testenv.NewCLITest(t)
+	runner := testenv.NewInProcRunner(t)
+	e := testenv.NewCLITest(t, runner)
 
 	defer e.RunAndExpectSuccess(t, "repo", "disconnect")
 	e.RunAndExpectSuccess(t, "repo", "create", "filesystem", "--path", e.RepoDir)
@@ -552,7 +561,8 @@ func TestSnapshotCreateAllWithManualSnapshot(t *testing.T) {
 func TestSnapshotCreateWithStdinStream(t *testing.T) {
 	t.Parallel()
 
-	e := testenv.NewCLITest(t)
+	runner := testenv.NewExeRunner(t)
+	e := testenv.NewCLITest(t, runner)
 
 	defer e.RunAndExpectSuccess(t, "repo", "disconnect")
 	e.RunAndExpectSuccess(t, "repo", "create", "filesystem", "--path", e.RepoDir)
@@ -572,7 +582,7 @@ func TestSnapshotCreateWithStdinStream(t *testing.T) {
 	w.Close()
 
 	streamFileName := "stream-file"
-	e.NextCommandStdin = r
+	runner.NextCommandStdin = r
 
 	e.RunAndExpectSuccess(t, "snapshot", "create", "rootdir", "--stdin-file", streamFileName)
 

tests/end_to_end_test/snapshot_delete_test.go

@@ -109,7 +109,9 @@ func(manifestID, objectID string, source clitestutil.SourceInfo) []string {
 func testSnapshotDelete(t *testing.T, argMaker deleteArgMaker, expectDeleteSucceeds bool) {
 	t.Helper()
 
-	e := testenv.NewCLITest(t)
+	runner := testenv.NewInProcRunner(t)
+	e := testenv.NewCLITest(t, runner)
+
 	defer e.RunAndExpectSuccess(t, "repo", "disconnect")
 
 	e.RunAndExpectSuccess(t, "repo", "create", "filesystem", "--path", e.RepoDir)
@@ -143,7 +145,8 @@ func testSnapshotDelete(t *testing.T, argMaker deleteArgMaker, expectDeleteSucce
 func TestSnapshotDeleteTypeCheck(t *testing.T) {
 	t.Parallel()
 
-	e := testenv.NewCLITest(t)
+	runner := testenv.NewInProcRunner(t)
+	e := testenv.NewCLITest(t, runner)
 
 	defer e.RunAndExpectSuccess(t, "repo", "disconnect")
 
@@ -175,7 +178,8 @@ func TestSnapshotDeleteTypeCheck(t *testing.T) {
 func TestSnapshotDeleteRestore(t *testing.T) {
 	t.Parallel()
 
-	e := testenv.NewCLITest(t)
+	runner := testenv.NewInProcRunner(t)
+	e := testenv.NewCLITest(t, runner)
 
 	defer e.RunAndExpectSuccess(t, "repo", "disconnect")
 

tests/end_to_end_test/snapshot_fail_test.go

@@ -21,7 +21,8 @@
 func TestSnapshotNonexistent(t *testing.T) {
 	t.Parallel()
 
-	e := testenv.NewCLITest(t)
+	runner := testenv.NewInProcRunner(t)
+	e := testenv.NewCLITest(t, runner)
 
 	defer e.RunAndExpectSuccess(t, "repo", "disconnect")
 
@@ -239,7 +240,8 @@ func testSnapshotFail(t *testing.T, isFailFast bool, snapshotCreateFlags, snapsh
 				t.Run(tname, func(t *testing.T) {
 					t.Parallel()
 
-					e := testenv.NewCLITest(t)
+					runner := testenv.NewExeRunner(t)
+					e := testenv.NewCLITest(t, runner)
 
 					defer e.RunAndExpectSuccess(t, "repo", "disconnect")
 
@@ -258,7 +260,7 @@ func testSnapshotFail(t *testing.T, isFailFast bool, snapshotCreateFlags, snapsh
 
 					e.RunAndExpectSuccess(t, "policy", "set", snapSource, "--ignore-dir-errors", tcIgnoreDirErr, "--ignore-file-errors", tcIgnoreFileErr)
 					restoreDir := fmt.Sprintf("%s%d_%v_%v", restoreDirPrefix, tcIdx, tcIgnoreDirErr, tcIgnoreFileErr)
-					testPermissions(t, e, snapSource, modifyEntry, restoreDir, tc.expectSuccess, snapshotCreateFlags, snapshotCreateEnv)
+					testPermissions(t, runner, e, snapSource, modifyEntry, restoreDir, tc.expectSuccess, snapshotCreateFlags, snapshotCreateEnv)
 
 					e.RunAndExpectSuccess(t, "policy", "remove", snapSource)
 				})
@@ -297,7 +299,7 @@ func createSimplestFileTree(t *testing.T, dirDepth, currDepth int, currPath stri
 // against "source" and will test permissions against all entries in "parentDir".
 // It returns the number of successful snapshot operations.
 // nolint:thelper
-func testPermissions(t *testing.T, e *testenv.CLITest, source, modifyEntry, restoreDir string, expect map[os.FileMode]expectedSnapshotResult, snapshotCreateFlags, snapshotCreateEnv []string) int {
+func testPermissions(t *testing.T, runner *testenv.CLIExeRunner, e *testenv.CLITest, source, modifyEntry, restoreDir string, expect map[os.FileMode]expectedSnapshotResult, snapshotCreateFlags, snapshotCreateEnv []string) int {
 	var numSuccessfulSnapshots int
 
 	changeFile, err := os.Stat(modifyEntry)
@@ -321,14 +323,14 @@ func() {
 			require.NoError(t, err)
 
 			// set up environment for the child process.
-			oldEnv := e.Environment
-			e.Environment = append(append([]string{}, e.Environment...), snapshotCreateEnv...)
+			oldEnv := runner.Environment
+			runner.Environment = append(append([]string{}, runner.Environment...), snapshotCreateEnv...)
 
-			defer func() { e.Environment = oldEnv }()
+			defer func() { runner.Environment = oldEnv }()
 
 			snapshotCreateWithArgs := append([]string{"snapshot", "create", source}, snapshotCreateFlags...)
 
-			_, errOut, runErr := e.Run(t, expected.success, snapshotCreateWithArgs...)
+			_, errOut, runErr := e.Run(t, !expected.success, snapshotCreateWithArgs...)
 
 			if got, want := (runErr == nil), expected.success; got != want {
 				t.Fatalf("unexpected success %v, want %v", got, want)

tests/end_to_end_test/snapshot_gc_test.go

@@ -18,7 +18,8 @@
 func TestSnapshotGC(t *testing.T) {
 	t.Parallel()
 
-	e := testenv.NewCLITest(t)
+	runner := testenv.NewInProcRunner(t)
+	e := testenv.NewCLITest(t, runner)
 
 	e.RunAndExpectSuccess(t, "repo", "create", "filesystem", "--path", e.RepoDir)
 

tests/end_to_end_test/snapshot_migrate_test.go

@@ -10,7 +10,8 @@
 func TestSnapshotMigrate(t *testing.T) {
 	t.Parallel()
 
-	e := testenv.NewCLITest(t)
+	runner := testenv.NewInProcRunner(t)
+	e := testenv.NewCLITest(t, runner)
 
 	defer e.RunAndExpectSuccess(t, "repo", "disconnect")
 
@@ -30,7 +31,7 @@ func TestSnapshotMigrate(t *testing.T) {
 	sourceSnapshotCount := len(e.RunAndExpectSuccess(t, "snapshot", "list", ".", "-a"))
 	sourcePolicyCount := len(e.RunAndExpectSuccess(t, "policy", "list"))
 
-	dstenv := testenv.NewCLITest(t)
+	dstenv := testenv.NewCLITest(t, runner)
 
 	dstenv.RunAndExpectSuccess(t, "repo", "create", "filesystem", "--path", dstenv.RepoDir)
 

tests/end_to_end_test/snapshot_verify_test.go

@@ -10,7 +10,8 @@
 func TestSnapshotVerifyTest(t *testing.T) {
 	t.Parallel()
 
-	e := testenv.NewCLITest(t)
+	runner := testenv.NewInProcRunner(t)
+	e := testenv.NewCLITest(t, runner)
 
 	e.RunAndExpectSuccess(t, "repo", "create", "filesystem", "--path", e.RepoDir)
 

tests/endurance_test/endurance_test.go

@@ -60,7 +60,8 @@ func (d webdavDirWithFakeClock) OpenFile(ctx context.Context, fname string, flag
 }
 
 func TestEndurance(t *testing.T) {
-	e := testenv.NewCLITest(t)
+	runner := testenv.NewExeRunner(t)
+	e := testenv.NewCLITest(t, runner)
 
 	tmpDir, err := ioutil.TempDir("", "endurance")
 	if err != nil {
@@ -74,7 +75,7 @@ func TestEndurance(t *testing.T) {
 	ft := httptest.NewServer(fts)
 	defer ft.Close()
 
-	e.Environment = append(e.Environment, "KOPIA_FAKE_CLOCK_ENDPOINT="+ft.URL)
+	runner.Environment = append(runner.Environment, "KOPIA_FAKE_CLOCK_ENDPOINT="+ft.URL)
 
 	sts := httptest.NewServer(&webdav.Handler{
 		FileSystem: webdavDirWithFakeClock{webdav.Dir(tmpDir), fts},
@@ -232,9 +233,10 @@ func pickRandomEnduranceTestAction() action {
 func enduranceRunner(t *testing.T, runnerID int, fakeTimeServer, webdavServer string, failureCount *int32, nowFunc func() time.Time) {
 	t.Helper()
 
-	e := testenv.NewCLITest(t)
+	runner := testenv.NewExeRunner(t)
+	e := testenv.NewCLITest(t, runner)
 
-	e.Environment = append(e.Environment,
+	runner.Environment = append(runner.Environment,
 		"KOPIA_FAKE_CLOCK_ENDPOINT="+fakeTimeServer,
 		"KOPIA_CHECK_FOR_UPDATES=false",
 	)

tests/testenv/cli_exe_runner.go

@@ -0,0 +1,126 @@
+package testenv
+
+import (
+	"io"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"strings"
+	"testing"
+
+	"github.com/kopia/kopia/internal/clock"
+)
+
+// CLIExeRunner is a CLIExeRunner that invokes the commands via external executable.
+type CLIExeRunner struct {
+	Exe               string
+	Environment       []string
+	PassthroughStderr bool      // this is for debugging only
+	NextCommandStdin  io.Reader // this is used for stdin source tests
+	LogsDir           string
+}
+
+// Start implements CLIRunner.
+func (e *CLIExeRunner) Start(t *testing.T, args []string) (stdout, stderr io.Reader, wait func() error, kill func()) {
+	t.Helper()
+
+	c := exec.Command(e.Exe, append([]string{
+		"--log-dir", e.LogsDir,
+	}, args...)...)
+
+	c.Env = append(os.Environ(), e.Environment...)
+
+	stdoutPipe, err := c.StdoutPipe()
+	if err != nil {
+		t.Fatalf("can't set up stdout pipe reader: %v", err)
+	}
+
+	stderrPipe, err := c.StderrPipe()
+	if err != nil {
+		t.Fatalf("can't set up stderr pipe reader: %v", err)
+	}
+
+	c.Stdin = e.NextCommandStdin
+	e.NextCommandStdin = nil
+
+	if err := c.Start(); err != nil {
+		t.Fatalf("unable to start: %v", err)
+	}
+
+	return stdoutPipe, stderrPipe, c.Wait, func() {
+		c.Process.Kill()
+	}
+}
+
+// RemoveDefaultPassword prevents KOPIA_PASSWORD from being passed to kopia.
+func (e *CLIExeRunner) RemoveDefaultPassword() {
+	var newEnv []string
+
+	for _, s := range e.Environment {
+		if !strings.HasPrefix(s, "KOPIA_PASSWORD=") {
+			newEnv = append(newEnv, s)
+		}
+	}
+
+	e.Environment = newEnv
+}
+
+// NewExeRunner resutns a CLIRunner that will execute kopia commands by launching subprocesses
+// for each. The kopia executable must be passed via KOPIA_EXE environment variable. The test
+// will be skipped if it's not provided (unless running inside an IDE in which case system-wide
+// `kopia` will be used by default).
+func NewExeRunner(t *testing.T) *CLIExeRunner {
+	t.Helper()
+
+	exe := os.Getenv("KOPIA_EXE")
+	if exe == "" {
+		if os.Getenv("VSCODE_PID") != "" {
+			// we're launched from VSCode, use system-installed kopia executable.
+			exe = "kopia"
+		} else {
+			t.Skip()
+		}
+	}
+
+	// unset environment variables that disrupt tests when passed to subprocesses.
+	os.Unsetenv("KOPIA_PASSWORD")
+
+	cleanName := strings.ReplaceAll(strings.ReplaceAll(strings.ReplaceAll(
+		t.Name(),
+		"/", "_"), "\\", "_"), ":", "_")
+
+	logsBaseDir := os.Getenv("KOPIA_LOGS_DIR")
+	if logsBaseDir == "" {
+		logsBaseDir = filepath.Join(os.TempDir(), "kopia-logs")
+	}
+
+	logsDir := filepath.Join(logsBaseDir, cleanName+"."+clock.Now().Local().Format("20060102150405"))
+
+	t.Cleanup(func() {
+		if t.Failed() {
+			t.Logf("FAULURE ABOVE ^^^^")
+		}
+
+		if os.Getenv("KOPIA_KEEP_LOGS") != "" {
+			t.Logf("logs preserved in %v", logsDir)
+			return
+		}
+
+		if t.Failed() && os.Getenv("KOPIA_DISABLE_LOG_DUMP_ON_FAILURE") == "" {
+			dumpLogs(t, logsDir)
+		}
+
+		os.RemoveAll(logsDir)
+	})
+
+	return &CLIExeRunner{
+		Exe: filepath.FromSlash(exe),
+		Environment: []string{
+			"KOPIA_PASSWORD=" + TestRepoPassword,
+			"KOPIA_ADVANCED_COMMANDS=enabled",
+		},
+		LogsDir: logsDir,
+	}
+}
+
+var _ CLIRunner = (*CLIExeRunner)(nil)

tests/testenv/cli_inproc_runner.go

@@ -0,0 +1,47 @@
+package testenv
+
+import (
+	"io"
+	"os"
+	"testing"
+
+	"github.com/kopia/kopia/cli"
+	"github.com/kopia/kopia/internal/buf"
+	"github.com/kopia/kopia/internal/testlogging"
+)
+
+// CLIInProcRunner is a CLIRunner that invokes provided commands in the current process.
+type CLIInProcRunner struct{}
+
+// Start implements CLIRunner.
+func (e *CLIInProcRunner) Start(t *testing.T, args []string) (stdout, stderr io.Reader, wait func() error, kill func()) {
+	t.Helper()
+
+	ctx := testlogging.Context(t)
+
+	a := cli.NewApp()
+	a.AdvancedCommands = "enabled"
+
+	return a.RunSubcommand(ctx, append([]string{
+		"--password", TestRepoPassword,
+	}, args...))
+}
+
+// NewInProcRunner returns a runner that executes CLI subcommands in the current process using cli.RunSubcommand().
+func NewInProcRunner(t *testing.T) *CLIInProcRunner {
+	t.Helper()
+
+	if os.Getenv("KOPIA_EXE") != "" {
+		t.Skip("not running test since it's also included in the unit tests")
+	}
+
+	return &CLIInProcRunner{}
+}
+
+var _ CLIRunner = (*CLIInProcRunner)(nil)
+
+func init() {
+	// disable buffer management in end-to-end tests as running too many of them in parallel causes too
+	// much memory usage on low-end platforms.
+	buf.DisableBufferManagement = true
+}

tests/testenv/cli_test_env.go

@@ -3,18 +3,18 @@
 
 import (
 	"bufio"
-	"bytes"
 	"fmt"
 	"io"
 	"io/ioutil"
-	"os"
-	"os/exec"
 	"path/filepath"
 	"runtime"
 	"strings"
+	"sync"
 	"testing"
 	"time"
 
+	"github.com/stretchr/testify/require"
+
 	"github.com/kopia/kopia/internal/clock"
 	"github.com/kopia/kopia/internal/testutil"
 )
@@ -26,77 +26,34 @@
 	maxOutputLinesToLog = 4000
 )
 
+// CLIRunner encapsulates running kopia subcommands for testing purposes.
+// It supports implementations that use subprocesses or in-process invocations.
+type CLIRunner interface {
+	Start(t *testing.T, args []string) (stdout, stderr io.Reader, wait func() error, kill func())
+}
+
 // CLITest encapsulates state for a CLI-based test.
 type CLITest struct {
 	startTime time.Time
 
 	RepoDir   string
 	ConfigDir string
-	Exe       string
 
-	fixedArgs   []string
-	Environment []string
+	Runner CLIRunner
+
+	fixedArgs []string
 
 	DefaultRepositoryCreateFlags []string
-
-	PassthroughStderr bool // this is for debugging only
-
-	NextCommandStdin io.Reader // this is used for stdin source tests
-
-	LogsDir string
 }
 
 // NewCLITest creates a new instance of *CLITest.
-func NewCLITest(t *testing.T) *CLITest {
+func NewCLITest(t *testing.T, runner CLIRunner) *CLITest {
 	t.Helper()
-
-	exe := os.Getenv("KOPIA_EXE")
-	if exe == "" {
-		if os.Getenv("VSCODE_PID") != "" {
-			// we're launched from VSCode, use system-installed kopia executable.
-			exe = "kopia"
-		} else {
-			t.Skip()
-		}
-	}
-
-	// unset environment variables that disrupt tests when passed to subprocesses.
-	os.Unsetenv("KOPIA_PASSWORD")
-
 	configDir := testutil.TempDirectory(t)
 
-	cleanName := strings.ReplaceAll(strings.ReplaceAll(strings.ReplaceAll(
-		t.Name(),
-		"/", "_"), "\\", "_"), ":", "_")
-
-	logsBaseDir := os.Getenv("KOPIA_LOGS_DIR")
-	if logsBaseDir == "" {
-		logsBaseDir = filepath.Join(os.TempDir(), "kopia-logs")
-	}
-
-	logsDir := filepath.Join(logsBaseDir, cleanName+"."+clock.Now().Local().Format("20060102150405"))
-
-	t.Cleanup(func() {
-		if t.Failed() {
-			t.Logf("FAULURE ABOVE ^^^^")
-		}
-
-		if os.Getenv("KOPIA_KEEP_LOGS") != "" {
-			t.Logf("logs preserved in %v", logsDir)
-			return
-		}
-
-		if t.Failed() && os.Getenv("KOPIA_DISABLE_LOG_DUMP_ON_FAILURE") == "" {
-			dumpLogs(t, logsDir)
-		}
-
-		os.RemoveAll(logsDir)
-	})
-
 	fixedArgs := []string{
 		// use per-test config file, to avoid clobbering current user's setup.
 		"--config-file", filepath.Join(configDir, ".kopia.config"),
-		"--log-dir", logsDir,
 	}
 
 	// disable the use of keyring
@@ -122,14 +79,9 @@ func NewCLITest(t *testing.T) *CLITest {
 		startTime:                    clock.Now(),
 		RepoDir:                      testutil.TempDirectory(t),
 		ConfigDir:                    configDir,
-		Exe:                          filepath.FromSlash(exe),
 		fixedArgs:                    fixedArgs,
 		DefaultRepositoryCreateFlags: formatFlags,
-		LogsDir:                      logsDir,
-		Environment: []string{
-			"KOPIA_PASSWORD=" + TestRepoPassword,
-			"KOPIA_ADVANCED_COMMANDS=enabled",
-		},
+		Runner:                       runner,
 	}
 }
 
@@ -165,19 +117,6 @@ func dumpLogFile(t *testing.T, fname string) {
 	t.Logf("LOG FILE: %v %v", fname, trimOutput(string(data)))
 }
 
-// RemoveDefaultPassword prevents KOPIA_PASSWORD from being passed to kopia.
-func (e *CLITest) RemoveDefaultPassword() {
-	var newEnv []string
-
-	for _, s := range e.Environment {
-		if !strings.HasPrefix(s, "KOPIA_PASSWORD=") {
-			newEnv = append(newEnv, s)
-		}
-	}
-
-	e.Environment = newEnv
-}
-
 // RunAndExpectSuccess runs the given command, expects it to succeed and returns its output lines.
 func (e *CLITest) RunAndExpectSuccess(t *testing.T, args ...string) []string {
 	t.Helper()
@@ -191,23 +130,13 @@ func (e *CLITest) RunAndExpectSuccess(t *testing.T, args ...string) []string {
 }
 
 // RunAndProcessStderr runs the given command, and streams its output line-by-line to a given function until it returns false.
-func (e *CLITest) RunAndProcessStderr(t *testing.T, callback func(line string) bool, args ...string) *exec.Cmd {
+func (e *CLITest) RunAndProcessStderr(t *testing.T, callback func(line string) bool, args ...string) (kill func()) {
 	t.Helper()
 
-	c := exec.Command(e.Exe, e.cmdArgs(args)...)
-	c.Env = append(os.Environ(), e.Environment...)
-	t.Logf("running '%v %v'", c.Path, c.Args)
+	stdout, stderr, _, kill := e.Runner.Start(t, e.cmdArgs(args))
+	go io.Copy(io.Discard, stdout)
 
-	stderrPipe, err := c.StderrPipe()
-	if err != nil {
-		t.Fatalf("can't set up stderr pipe reader")
-	}
-
-	if err := c.Start(); err != nil {
-		t.Fatalf("unable to start")
-	}
-
-	scanner := bufio.NewScanner(stderrPipe)
+	scanner := bufio.NewScanner(stderr)
 	for scanner.Scan() {
 		if !callback(scanner.Text()) {
 			break
@@ -221,7 +150,7 @@ func (e *CLITest) RunAndProcessStderr(t *testing.T, callback func(line string) b
 		}
 	}()
 
-	return c
+	return kill
 }
 
 // RunAndExpectSuccessWithErrOut runs the given command, expects it to succeed and returns its stdout and stderr lines.
@@ -276,28 +205,50 @@ func (e *CLITest) cmdArgs(args []string) []string {
 func (e *CLITest) Run(t *testing.T, expectedError bool, args ...string) (stdout, stderr []string, err error) {
 	t.Helper()
 
-	c := exec.Command(e.Exe, e.cmdArgs(args)...)
-	c.Env = append(os.Environ(), e.Environment...)
+	t.Logf("running 'kopia %v'", strings.Join(args, " "))
 
-	t.Logf("running '%v %v'", c.Path, c.Args)
+	args = e.cmdArgs(args)
+	t0 := clock.Now()
 
-	errOut := &bytes.Buffer{}
-	c.Stderr = errOut
+	stdoutReader, stderrReader, wait, _ := e.Runner.Start(t, args)
 
-	if e.PassthroughStderr {
-		c.Stderr = os.Stderr
+	var wg sync.WaitGroup
+
+	wg.Add(1)
+
+	go func() {
+		defer wg.Done()
+
+		scanner := bufio.NewScanner(stdoutReader)
+		for scanner.Scan() {
+			stdout = append(stdout, scanner.Text())
+		}
+	}()
+
+	wg.Add(1)
+
+	go func() {
+		defer wg.Done()
+
+		scanner := bufio.NewScanner(stderrReader)
+		for scanner.Scan() {
+			stderr = append(stderr, scanner.Text())
+		}
+	}()
+
+	wg.Wait()
+
+	gotErr := wait()
+
+	if expectedError {
+		require.Error(t, gotErr, "unexpected success when running 'kopia %v' (stdout:\n%v\nstderr:\n%v", strings.Join(args, " "), strings.Join(stdout, "\n"), strings.Join(stderr, "\n"))
+	} else {
+		require.NoError(t, gotErr, "unexpected error when running 'kopia %v' (stdout:\n%v\nstderr:\n%v", strings.Join(args, " "), strings.Join(stdout, "\n"), strings.Join(stderr, "\n"))
 	}
 
-	c.Stdin = e.NextCommandStdin
-	e.NextCommandStdin = nil
+	t.Logf("finished in %v: 'kopia %v'", clock.Since(t0).Milliseconds(), strings.Join(args, " "))
 
-	o, err := c.Output()
-
-	if err != nil && !expectedError {
-		t.Logf("finished 'kopia %v' with err=%v (expected=%v) and output:\n%v\nstderr:\n%v\n", strings.Join(args, " "), err, expectedError, trimOutput(string(o)), trimOutput(errOut.String()))
-	}
-
-	return splitLines(string(o)), splitLines(errOut.String()), err
+	return stdout, stderr, gotErr
 }
 
 func trimOutput(s string) string {