cli: added profile flags, built only with 'profiling' tag

cli/app.go

@@ -56,23 +56,25 @@ func serverAction(act func(ctx context.Context, cli *serverapi.Client) error) fu
 
 func repositoryAction(act func(ctx context.Context, rep *repo.Repository) error) func(ctx *kingpin.ParseContext) error {
 	return func(kpc *kingpin.ParseContext) error {
-		ctx := context.Background()
-		ctx = content.UsingContentCache(ctx, *enableCaching)
-		ctx = content.UsingListCache(ctx, *enableListCaching)
-		ctx = blob.WithUploadProgressCallback(ctx, func(desc string, progress, total int64) {
-			cliProgress.Report("upload '"+desc+"'", progress, total)
+		return withProfiling(func() error {
+			ctx := context.Background()
+			ctx = content.UsingContentCache(ctx, *enableCaching)
+			ctx = content.UsingListCache(ctx, *enableListCaching)
+			ctx = blob.WithUploadProgressCallback(ctx, func(desc string, progress, total int64) {
+				cliProgress.Report("upload '"+desc+"'", progress, total)
+			})
+
+			rep, err := openRepository(ctx, nil)
+			if err != nil {
+				return errors.Wrap(err, "open repository")
+			}
+
+			err = act(ctx, rep)
+			if cerr := rep.Close(ctx); cerr != nil {
+				return errors.Wrap(cerr, "unable to close repository")
+			}
+			return err
 		})
-
-		rep, err := openRepository(ctx, nil)
-		if err != nil {
-			return errors.Wrap(err, "open repository")
-		}
-
-		err = act(ctx, rep)
-		if cerr := rep.Close(ctx); cerr != nil {
-			return errors.Wrap(cerr, "unable to close repository")
-		}
-		return err
 	}
 }
 

cli/profile.go

@@ -0,0 +1,39 @@
+// +build profiling
+
+package cli
+
+import "github.com/pkg/profile"
+
+var (
+	profileDir      = app.Flag("profile-dir", "Write profile to the specified directory").Hidden().String()
+	profileCPU      = app.Flag("profile-cpu", "Enable CPU profiling").Hidden().Bool()
+	profileMemory   = app.Flag("profile-memory", "Enable memory profiling").Hidden().Bool()
+	profileBlocking = app.Flag("profile-blocking", "Enable block profiling").Hidden().Bool()
+	profileMutex    = app.Flag("profile-mutex", "Enable mutex profiling").Hidden().Bool()
+)
+
+// withProfiling runs the given callback with profiling enabled, configured according to command line flags
+func withProfiling(callback func() error) error {
+	if *profileDir != "" {
+		profileOpts := []func(*profile.Profile){
+			profile.ProfilePath(*profileDir),
+		}
+
+		if *profileMemory {
+			profileOpts = append(profileOpts, profile.MemProfile)
+		}
+		if *profileCPU {
+			profileOpts = append(profileOpts, profile.CPUProfile)
+		}
+		if *profileBlocking {
+			profileOpts = append(profileOpts, profile.BlockProfile)
+		}
+		if *profileMutex {
+			profileOpts = append(profileOpts, profile.MutexProfile)
+		}
+
+		defer profile.Start(profileOpts...).Stop()
+	}
+
+	return callback()
+}

cli/profile_disabled.go

@@ -0,0 +1,8 @@
+// +build !profiling
+
+package cli
+
+// withProfiling runs the given callback with profiling enabled, configured according to command line flags
+func withProfiling(callback func() error) error {
+	return callback()
+}

go.mod

@@ -21,6 +21,7 @@ require (
 	github.com/mitchellh/go-homedir v1.1.0 // indirect
 	github.com/op/go-logging v0.0.0-20160315200505-970db520ece7
 	github.com/pkg/errors v0.8.1
+	github.com/pkg/profile v1.3.0
 	github.com/pkg/sftp v1.10.0
 	github.com/skratchdot/open-golang v0.0.0-20190402232053-79abb63cd66e
 	github.com/smartystreets/goconvey v0.0.0-20190330032615-68dc04aab96a // indirect

go.sum

@@ -66,6 +66,8 @@ github.com/op/go-logging v0.0.0-20160315200505-970db520ece7 h1:lDH9UUVJtmYCjyT0C
 github.com/op/go-logging v0.0.0-20160315200505-970db520ece7/go.mod h1:HzydrMdWErDVzsI23lYNej1Htcns9BCg93Dk0bBINWk=
 github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I=
 github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pkg/profile v1.3.0 h1:OQIvuDgm00gWVWGTf4m4mCt6W1/0YqU7Ntg0mySWgaI=
+github.com/pkg/profile v1.3.0/go.mod h1:hJw3o1OdXxsrSjjVksARp5W95eeEaEfptyVZyv6JUPA=
 github.com/pkg/sftp v1.10.0 h1:DGA1KlA9esU6WcicH+P8PxFZOl15O6GYtab1cIJdOlE=
 github.com/pkg/sftp v1.10.0/go.mod h1:NxmoDg/QLVWluQDUYG7XBZTLUpKeFa8e3aMf1BfjyHk=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=

main.go

@@ -65,6 +65,7 @@
 
 func main() {
 	app := cli.App()
+
 	app.Version(repo.BuildVersion + " build: " + repo.BuildInfo)
 	app.PreAction(logfile.Initialize)
 	app.UsageTemplate(usageTemplate)

snapshot/snapshotfs/upload.go

@@ -589,7 +589,9 @@ func uploadDirInternal(
 		summ.IncompleteReason = u.cancelReason()
 	}()
 
+	log.Infof("reading directory %v", dirRelativePath)
 	entries, direrr := directory.Readdir(ctx)
+	log.Infof("finished reading directory %v", dirRelativePath)
 	if direrr != nil {
 		return "", fs.DirectorySummary{}, direrr
 	}
@@ -599,7 +601,6 @@ func uploadDirInternal(
 		if ent := maybeReadDirectoryEntries(ctx, d); ent != nil {
 			prevEntries = append(prevEntries, ent)
 		}
-
 	}
 	if len(entries) == 0 {
 		summ.MaxModTime = directory.ModTime()
@@ -614,13 +615,16 @@ func uploadDirInternal(
 	}
 	u.prepareProgress(dirRelativePath, entries)
 
+	log.Infof("preparing work items %v", dirRelativePath)
 	workItems, workItemErr := u.prepareWorkItems(ctx, dirRelativePath, entries, prevEntries, &summ)
+	log.Infof("finished preparing work items %v", dirRelativePath)
 	if workItemErr != nil && workItemErr != errCancelled {
 		return "", fs.DirectorySummary{}, workItemErr
 	}
 	if err := u.processUploadWorkItems(workItems, dirManifest); err != nil && err != errCancelled {
 		return "", fs.DirectorySummary{}, err
 	}
+	log.Infof("finished processing uploads %v", dirRelativePath)
 	dirManifest.Summary = &summ
 
 	writer := u.repo.Objects.NewWriter(ctx, object.WriterOptions{