From 81ea751c0c82eefbe3b78bb4b44e096c42f85e7a Mon Sep 17 00:00:00 2001 From: Jarek Kowalski Date: Sat, 24 Aug 2019 10:37:01 -0700 Subject: [PATCH] cli: added profile flags, built only with 'profiling' tag --- cli/app.go | 34 ++++++++++++++++-------------- cli/profile.go | 39 +++++++++++++++++++++++++++++++++++ cli/profile_disabled.go | 8 +++++++ go.mod | 1 + go.sum | 2 ++ main.go | 1 + snapshot/snapshotfs/upload.go | 6 +++++- 7 files changed, 74 insertions(+), 17 deletions(-) create mode 100644 cli/profile.go create mode 100644 cli/profile_disabled.go diff --git a/cli/app.go b/cli/app.go index 8dd608508..dfea4055c 100644 --- a/cli/app.go +++ b/cli/app.go @@ -56,23 +56,25 @@ func serverAction(act func(ctx context.Context, cli *serverapi.Client) error) fu func repositoryAction(act func(ctx context.Context, rep *repo.Repository) error) func(ctx *kingpin.ParseContext) error { return func(kpc *kingpin.ParseContext) error { - ctx := context.Background() - ctx = content.UsingContentCache(ctx, *enableCaching) - ctx = content.UsingListCache(ctx, *enableListCaching) - ctx = blob.WithUploadProgressCallback(ctx, func(desc string, progress, total int64) { - cliProgress.Report("upload '"+desc+"'", progress, total) + return withProfiling(func() error { + ctx := context.Background() + ctx = content.UsingContentCache(ctx, *enableCaching) + ctx = content.UsingListCache(ctx, *enableListCaching) + ctx = blob.WithUploadProgressCallback(ctx, func(desc string, progress, total int64) { + cliProgress.Report("upload '"+desc+"'", progress, total) + }) + + rep, err := openRepository(ctx, nil) + if err != nil { + return errors.Wrap(err, "open repository") + } + + err = act(ctx, rep) + if cerr := rep.Close(ctx); cerr != nil { + return errors.Wrap(cerr, "unable to close repository") + } + return err }) - - rep, err := openRepository(ctx, nil) - if err != nil { - return errors.Wrap(err, "open repository") - } - - err = act(ctx, rep) - if cerr := rep.Close(ctx); cerr != nil { - return errors.Wrap(cerr, "unable to close repository") - } - return err } } diff --git a/cli/profile.go b/cli/profile.go new file mode 100644 index 000000000..7c4a7bd16 --- /dev/null +++ b/cli/profile.go @@ -0,0 +1,39 @@ +// +build profiling + +package cli + +import "github.com/pkg/profile" + +var ( + profileDir = app.Flag("profile-dir", "Write profile to the specified directory").Hidden().String() + profileCPU = app.Flag("profile-cpu", "Enable CPU profiling").Hidden().Bool() + profileMemory = app.Flag("profile-memory", "Enable memory profiling").Hidden().Bool() + profileBlocking = app.Flag("profile-blocking", "Enable block profiling").Hidden().Bool() + profileMutex = app.Flag("profile-mutex", "Enable mutex profiling").Hidden().Bool() +) + +// withProfiling runs the given callback with profiling enabled, configured according to command line flags +func withProfiling(callback func() error) error { + if *profileDir != "" { + profileOpts := []func(*profile.Profile){ + profile.ProfilePath(*profileDir), + } + + if *profileMemory { + profileOpts = append(profileOpts, profile.MemProfile) + } + if *profileCPU { + profileOpts = append(profileOpts, profile.CPUProfile) + } + if *profileBlocking { + profileOpts = append(profileOpts, profile.BlockProfile) + } + if *profileMutex { + profileOpts = append(profileOpts, profile.MutexProfile) + } + + defer profile.Start(profileOpts...).Stop() + } + + return callback() +} diff --git a/cli/profile_disabled.go b/cli/profile_disabled.go new file mode 100644 index 000000000..ecf9b4806 --- /dev/null +++ b/cli/profile_disabled.go @@ -0,0 +1,8 @@ +// +build !profiling + +package cli + +// withProfiling runs the given callback with profiling enabled, configured according to command line flags +func withProfiling(callback func() error) error { + return callback() +} diff --git a/go.mod b/go.mod index 494fa4ac8..142caa348 100644 --- a/go.mod +++ b/go.mod @@ -21,6 +21,7 @@ require ( github.com/mitchellh/go-homedir v1.1.0 // indirect github.com/op/go-logging v0.0.0-20160315200505-970db520ece7 github.com/pkg/errors v0.8.1 + github.com/pkg/profile v1.3.0 github.com/pkg/sftp v1.10.0 github.com/skratchdot/open-golang v0.0.0-20190402232053-79abb63cd66e github.com/smartystreets/goconvey v0.0.0-20190330032615-68dc04aab96a // indirect diff --git a/go.sum b/go.sum index 02d3288b8..6bbae3176 100644 --- a/go.sum +++ b/go.sum @@ -66,6 +66,8 @@ github.com/op/go-logging v0.0.0-20160315200505-970db520ece7 h1:lDH9UUVJtmYCjyT0C github.com/op/go-logging v0.0.0-20160315200505-970db520ece7/go.mod h1:HzydrMdWErDVzsI23lYNej1Htcns9BCg93Dk0bBINWk= github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/profile v1.3.0 h1:OQIvuDgm00gWVWGTf4m4mCt6W1/0YqU7Ntg0mySWgaI= +github.com/pkg/profile v1.3.0/go.mod h1:hJw3o1OdXxsrSjjVksARp5W95eeEaEfptyVZyv6JUPA= github.com/pkg/sftp v1.10.0 h1:DGA1KlA9esU6WcicH+P8PxFZOl15O6GYtab1cIJdOlE= github.com/pkg/sftp v1.10.0/go.mod h1:NxmoDg/QLVWluQDUYG7XBZTLUpKeFa8e3aMf1BfjyHk= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= diff --git a/main.go b/main.go index a20942617..59f882f94 100644 --- a/main.go +++ b/main.go @@ -65,6 +65,7 @@ func main() { app := cli.App() + app.Version(repo.BuildVersion + " build: " + repo.BuildInfo) app.PreAction(logfile.Initialize) app.UsageTemplate(usageTemplate) diff --git a/snapshot/snapshotfs/upload.go b/snapshot/snapshotfs/upload.go index 047098f1a..7d360815b 100644 --- a/snapshot/snapshotfs/upload.go +++ b/snapshot/snapshotfs/upload.go @@ -589,7 +589,9 @@ func uploadDirInternal( summ.IncompleteReason = u.cancelReason() }() + log.Infof("reading directory %v", dirRelativePath) entries, direrr := directory.Readdir(ctx) + log.Infof("finished reading directory %v", dirRelativePath) if direrr != nil { return "", fs.DirectorySummary{}, direrr } @@ -599,7 +601,6 @@ func uploadDirInternal( if ent := maybeReadDirectoryEntries(ctx, d); ent != nil { prevEntries = append(prevEntries, ent) } - } if len(entries) == 0 { summ.MaxModTime = directory.ModTime() @@ -614,13 +615,16 @@ func uploadDirInternal( } u.prepareProgress(dirRelativePath, entries) + log.Infof("preparing work items %v", dirRelativePath) workItems, workItemErr := u.prepareWorkItems(ctx, dirRelativePath, entries, prevEntries, &summ) + log.Infof("finished preparing work items %v", dirRelativePath) if workItemErr != nil && workItemErr != errCancelled { return "", fs.DirectorySummary{}, workItemErr } if err := u.processUploadWorkItems(workItems, dirManifest); err != nil && err != errCancelled { return "", fs.DirectorySummary{}, err } + log.Infof("finished processing uploads %v", dirRelativePath) dirManifest.Summary = &summ writer := u.repo.Objects.NewWriter(ctx, object.WriterOptions{