diff --git a/cli/observability_flags.go b/cli/observability_flags.go index 442d59acd..b75637bae 100644 --- a/cli/observability_flags.go +++ b/cli/observability_flags.go @@ -29,8 +29,8 @@ "github.com/kopia/kopia/repo" ) -// DirMode is the directory mode for output directories. -const DirMode = 0o700 +// dirMode is the directory mode for output directories. +const dirMode = 0o700 //nolint:gochecknoglobals var metricsPushFormats = map[string]expfmt.Format{ @@ -165,7 +165,7 @@ func (c *observabilityFlags) start(ctx context.Context) error { func mkSubdirectories(directoryNames ...string) (dirName string, err error) { dirName = filepath.Join(directoryNames...) - if err := os.MkdirAll(dirName, DirMode); err != nil { + if err := os.MkdirAll(dirName, dirMode); err != nil { return "", errors.Wrapf(err, "could not create '%q' subdirectory to save diagnostics output", dirName) } diff --git a/go.mod b/go.mod index 562d64784..536dad19b 100644 --- a/go.mod +++ b/go.mod @@ -39,7 +39,6 @@ require ( github.com/natefinch/atomic v1.0.1 github.com/petar/GoLLRB v0.0.0-20210522233825-ae3b015fd3e9 github.com/pkg/errors v0.9.1 - github.com/pkg/profile v1.7.0 github.com/pkg/sftp v1.13.10 github.com/prometheus/client_golang v1.23.2 github.com/prometheus/client_model v0.6.2 @@ -96,7 +95,6 @@ require ( github.com/dustin/go-humanize v1.0.1 // indirect github.com/envoyproxy/go-control-plane/envoy v1.36.0 // indirect github.com/envoyproxy/protoc-gen-validate v1.3.0 // indirect - github.com/felixge/fgprof v0.9.3 // indirect github.com/felixge/httpsnoop v1.0.4 // indirect github.com/go-ini/ini v1.67.0 // indirect github.com/go-jose/go-jose/v4 v4.1.4 // indirect @@ -109,7 +107,6 @@ require ( github.com/gobwas/ws v1.4.0 // indirect github.com/godbus/dbus/v5 v5.2.2 // indirect github.com/golang/glog v1.2.5 // indirect - github.com/google/pprof v0.0.0-20230602150820-91b7bce49751 // indirect github.com/google/readahead v0.0.0-20161222183148-eaceba169032 // indirect github.com/google/s2a-go v0.1.9 // indirect github.com/googleapis/enterprise-certificate-proxy v0.3.14 // indirect diff --git a/go.sum b/go.sum index bcf4b4be9..0333dc755 100644 --- a/go.sum +++ b/go.sum @@ -65,9 +65,6 @@ github.com/chromedp/chromedp v0.14.2 h1:r3b/WtwM50RsBZHMUm9fsNhhzRStTHrKdr2zmwbZ github.com/chromedp/chromedp v0.14.2/go.mod h1:rHzAv60xDE7VNy/MYtTUrYreSc0ujt2O1/C3bzctYBo= github.com/chromedp/sysutil v1.1.0 h1:PUFNv5EcprjqXZD9nJb9b/c9ibAbxiYo4exNWZyipwM= github.com/chromedp/sysutil v1.1.0/go.mod h1:WiThHUdltqCNKGc4gaU50XgYjwjYIhKWoHGPTUfWTJ8= -github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= -github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= -github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= github.com/cncf/xds/go v0.0.0-20260202195803-dba9d589def2 h1:aBangftG7EVZoUb69Os8IaYg++6uMOdKK83QtkkvJik= github.com/cncf/xds/go v0.0.0-20260202195803-dba9d589def2/go.mod h1:qwXFYgsP6T7XnJtbKlf1HP8AjxZZyzxMmc+Lq5GjlU4= github.com/coreos/go-systemd/v22 v22.7.0 h1:LAEzFkke61DFROc7zNLX/WA2i5J8gYqe0rSj9KI28KA= @@ -95,8 +92,6 @@ github.com/envoyproxy/protoc-gen-validate v1.3.0 h1:TvGH1wof4H33rezVKWSpqKz5NXWg github.com/envoyproxy/protoc-gen-validate v1.3.0/go.mod h1:HvYl7zwPa5mffgyeTUHA9zHIH36nmrm7oCbo4YKoSWA= github.com/fatih/color v1.19.0 h1:Zp3PiM21/9Ld6FzSKyL5c/BULoe/ONr9KlbYVOfG8+w= github.com/fatih/color v1.19.0/go.mod h1:zNk67I0ZUT1bEGsSGyCZYZNrHuTkJJB+r6Q9VuMi0LE= -github.com/felixge/fgprof v0.9.3 h1:VvyZxILNuCiUCSXtPtYmmtGvb65nqXh2QFWc0Wpf2/g= -github.com/felixge/fgprof v0.9.3/go.mod h1:RdbpDgzqYVh/T9fPELJyV7EYJuHB55UTEULNun8eiPw= github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= github.com/foomo/htpasswd v0.0.0-20200116085101-e3a90e78da9c h1:DBGU7zCwrrPPDsD6+gqKG8UfMxenWg9BOJE/Nmfph+4= @@ -136,9 +131,6 @@ github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= github.com/google/martian/v3 v3.3.3 h1:DIhPTQrbPkgs2yJYdXU/eNACCG5DVQjySNRNlflZ9Fc= github.com/google/martian/v3 v3.3.3/go.mod h1:iEPrYcgCF7jA9OtScMFQyAlZZ4YXTKEtJ1E6RWzmBA0= -github.com/google/pprof v0.0.0-20211214055906-6f57359322fd/go.mod h1:KgnwoLYCZ8IQu3XUZ8Nc/bM9CCZFOyjUNOSygVozoDg= -github.com/google/pprof v0.0.0-20230602150820-91b7bce49751 h1:hR7/MlvK23p6+lIw9SN1TigNLn9ZnF3W4SYRKq2gAHs= -github.com/google/pprof v0.0.0-20230602150820-91b7bce49751/go.mod h1:Jh3hGz2jkYak8qXPD19ryItVnUgpgeqzdkY/D0EaeuA= github.com/google/readahead v0.0.0-20161222183148-eaceba169032 h1:6Be3nkuJFyRfCgr6qTIzmRp8y9QwDIbqy/nYr9WDPos= github.com/google/readahead v0.0.0-20161222183148-eaceba169032/go.mod h1:qYysrqQXuV4tzsizt4oOQ6mrBZQ0xnQXP3ylXX8Jk5Y= github.com/google/s2a-go v0.1.9 h1:LGD7gtMgezd8a/Xak7mEWL0PjoTQFvpRudN895yqKW0= @@ -157,7 +149,6 @@ github.com/hanwen/go-fuse/v2 v2.9.0 h1:0AOGUkHtbOVeyGLr0tXupiid1Vg7QB7M6YUcdmVdC github.com/hanwen/go-fuse/v2 v2.9.0/go.mod h1:yE6D2PqWwm3CbYRxFXV9xUd8Md5d6NG0WBs5spCswmI= github.com/hashicorp/cronexpr v1.1.3 h1:rl5IkxXN2m681EfivTlccqIryzYJSXRGRNa0xeG7NA4= github.com/hashicorp/cronexpr v1.1.3/go.mod h1:P4wA0KBl9C5q2hABiMO7cp6jcIg96CDh1Efb3g1PWA4= -github.com/ianlancetaylor/demangle v0.0.0-20210905161508-09a460cdf81d/go.mod h1:aYm2/VgdVmcIU8iMfdMvDMsRAQjcfZSKFby6HOFvi/w= github.com/keybase/go-keychain v0.0.1 h1:way+bWYa6lDppZoZcgMbYsvC7GxljxrskdNInRtuthU= github.com/keybase/go-keychain v0.0.1/go.mod h1:PdEILRW3i9D8JcdM+FmY6RwkHGnhHxXwkPPMeUgOK1k= github.com/klauspost/compress v1.18.5 h1:/h1gH5Ce+VWNLSWqPzOVn6XBO+vJbCNGvjoaGBFW2IE= @@ -213,8 +204,6 @@ github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c h1:+mdjkGKdHQG3305AYmd github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c/go.mod h1:7rwL4CYBLnjLxUqIJNnCWiEdr3bn6IUYi15bNlnbCCU= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= -github.com/pkg/profile v1.7.0 h1:hnbDkaNWPCLMO9wGLdBFTIZvzDrDfBM2072E1S9gJkA= -github.com/pkg/profile v1.7.0/go.mod h1:8Uer0jas47ZQMJ7VD+OHknK4YDY07LPUC6dEvqDjvNo= github.com/pkg/sftp v1.13.10 h1:+5FbKNTe5Z9aspU88DPIKJ9z2KZoaGCu6Sr6kKR/5mU= github.com/pkg/sftp v1.13.10/go.mod h1:bJ1a7uDhrX/4OII+agvy28lzRvQrmIQuaHrcI1HbeGA= github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 h1:GFCKgmp0tecUJ0sJuv4pzYCqS9+RGSn52M3FUwPs+uo= @@ -324,7 +313,6 @@ golang.org/x/sync v0.20.0 h1:e0PTpb7pjO8GAtTs2dQ6jYa5BWYlMuX047Dco/pItO4= golang.org/x/sync v0.20.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20211007075335-d3039528d8ac/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.43.0 h1:Rlag2XtaFTxp19wS8MXlJwTvoh8ArU6ezoyFsMyCTNI= diff --git a/internal/bigmap/bigmapbench/main.go b/internal/bigmap/bigmapbench/main.go index 760ada05f..aa99ffeda 100644 --- a/internal/bigmap/bigmapbench/main.go +++ b/internal/bigmap/bigmapbench/main.go @@ -7,13 +7,14 @@ "encoding/binary" "flag" "fmt" + "log" "os" + "path/filepath" "runtime" + "runtime/pprof" "sync" "time" - "github.com/pkg/profile" - "github.com/kopia/kopia/internal/bigmap" "github.com/kopia/kopia/internal/clock" "github.com/kopia/kopia/repo/logging" @@ -27,10 +28,11 @@ //nolint:gochecknoglobals var ( - impl = flag.Int("impl", implMapWithEmptyValue, "Select implementation") - profileDir = flag.String("profile-dir", "", "Profile directory") - profileCPU = flag.Bool("profile-cpu", false, "Profile CPU") - profileMemory = flag.Bool("profile-memory", false, "Profile RAM") + impl = flag.Int("impl", implMapWithEmptyValue, "Select implementation") + profileDir = flag.String("profile-dir", "", "Profile directory") + profileCPU = flag.Bool("profile-cpu", false, "Profile CPU") + profileMemory = flag.Bool("profile-memory", false, "Profile memory usage") + profileMemoryRate = flag.Int("profile-memory-rate", -1, "Profile memory rate") ) func main() { @@ -46,17 +48,7 @@ func main() { ms0 runtime.MemStats ) - if *profileDir != "" { - pp := profile.ProfilePath(*profileDir) - - if *profileCPU { - defer profile.Start(pp, profile.CPUProfile).Stop() - } - - if *profileMemory { - defer profile.Start(pp, profile.MemProfile).Stop() - } - } + defer maybeStartProfiling().stop() switch *impl { case implSyncMap: @@ -80,22 +72,6 @@ func main() { t0 := clock.Now() for i := range 300_000_000 { - if i%1_000_000 == 0 && i > 0 { - var ms runtime.MemStats - - runtime.ReadMemStats(&ms) - - alloc := ms.HeapAlloc - ms0.HeapAlloc - dur := clock.Now().Sub(t0).Truncate(time.Second) - - fmt.Printf("elapsed %v count: %v M bytes: %v MB bytes/item: %v Mitems/sec: %.1f\n", - dur, - float64(i)/1e6, - alloc/1e6, - alloc/uint64(i), - float64(i)/dur.Seconds()/1e6) - } - // generate key=sha256(i) without allocations. h.Reset() binary.LittleEndian.PutUint64(num[:], uint64(i)) //nolint:gosec @@ -110,5 +86,121 @@ func main() { case implMapWithValues: bm.PutIfAbsent(ctx, keyBuf[:], keyBuf[:]) } + + count := uint64(i + 1) //nolint:gosec + + if count%1_000_000 == 0 { + var ms runtime.MemStats + + runtime.ReadMemStats(&ms) + + alloc := ms.HeapAlloc - ms0.HeapAlloc + dur := clock.Now().Sub(t0) + + fmt.Printf("elapsed %v, count: %v M, bytes: %v MB, bytes/item: %v, Mitems/sec: %.1f\n", + dur.Truncate(time.Second), + float64(count)/1e6, + alloc/1e6, + alloc/count, + float64(count)/dur.Seconds()/1e6) + } + } +} + +// dirMode is the directory mode for output directories. +const dirMode = 0o700 + +type stopperFn func() + +func (f stopperFn) stop() { + f() +} + +func maybeStartProfiling() stopperFn { + if *profileDir == "" { + return func() {} + } + + // ensure upfront that the pprof output dir can be created. + if err := os.MkdirAll(*profileDir, dirMode); err != nil { + log.Fatalln("could not create directory for profile output:", err) + } + + var cpuProfileStopper stopperFn + + if *profileCPU { + cpuProfileStopper = startCPUProfiling(*profileDir) + } + + if *profileMemory && *profileMemoryRate >= 0 { + runtime.MemProfileRate = *profileMemoryRate + } + + return func() { + if cpuProfileStopper != nil { + cpuProfileStopper() + } + + if *profileMemory { + dumpProfiles(*profileDir) + } + } +} + +func startCPUProfiling(profDir string) stopperFn { + // start CPU profile dumper + f, err := os.Create(filepath.Join(profDir, "cpu.pprof")) //nolint:gosec + if err != nil { + log.Fatalln("could not create CPU profile output file:", err) + } + + // CPU profile profStopper + profStopper := func() { + pprof.StopCPUProfile() + + if err := f.Close(); err != nil { + log.Println("error closing CPU profile output file:", err) + } + } + + if err := pprof.StartCPUProfile(f); err != nil { + profStopper() + + log.Fatalln("could not start CPU profile:", err) + } + + return profStopper +} + +func dumpProfiles(profDir string) { + if err := os.MkdirAll(profDir, dirMode); err != nil { + log.Println("could not create directory for profile output:", err) + + return + } + + runtime.GC() // force GC to include stats since last GC + + for _, p := range pprof.Profiles() { + func() { + fname := filepath.Join(profDir, p.Name()+".pprof") + + f, err := os.Create(fname) //nolint:gosec + if err != nil { + log.Printf("unable to create profile output file '%s': %v", fname, err) + + return + } + + defer func() { + if err := f.Close(); err != nil { + log.Printf("unable to close profile output file '%s': %v", fname, err) + } + }() + + if err := p.WriteTo(f, 0); err != nil { + log.Printf("unable to write profile to file '%s': %v", fname, err) + } + }() } }