diff --git a/cli/observability_flags.go b/cli/observability_flags.go index 5149ce0a5..033aae07f 100644 --- a/cli/observability_flags.go +++ b/cli/observability_flags.go @@ -4,6 +4,8 @@ "context" "net/http" "net/http/pprof" + "os" + "path/filepath" "sort" "strings" "sync" @@ -21,9 +23,13 @@ "go.opentelemetry.io/otel/sdk/trace" semconv "go.opentelemetry.io/otel/semconv/v1.4.0" + "github.com/kopia/kopia/internal/clock" "github.com/kopia/kopia/repo" ) +// DirMode is the directory mode for output directories. +const DirMode = 0o700 + //nolint:gochecknoglobals var metricsPushFormats = map[string]expfmt.Format{ "text": expfmt.FmtText, @@ -44,6 +50,8 @@ type observabilityFlags struct { metricsPushUsername string metricsPushPassword string metricsPushFormat string + metricsOutputDir string + metricsFileName string enableJaeger bool @@ -76,6 +84,36 @@ func (c *observabilityFlags) setup(svc appServices, app *kingpin.Application) { sort.Strings(formats) app.Flag("metrics-push-format", "Format to use for push gateway").Envar(svc.EnvName("KOPIA_METRICS_FORMAT")).Hidden().EnumVar(&c.metricsPushFormat, formats...) + + app.Flag("metrics-directory", "Directory where the metrics should be saved when kopia exits. A file per process execution will be created in this directory").Hidden().StringVar(&c.metricsOutputDir) + + app.PreAction(c.initialize) +} + +func (c *observabilityFlags) initialize(ctx *kingpin.ParseContext) error { + if c.metricsOutputDir == "" { + return nil + } + + c.metricsOutputDir = filepath.Clean(c.metricsOutputDir) + + // ensure the metrics output dir can be created + if err := os.MkdirAll(c.metricsOutputDir, DirMode); err != nil { + return errors.Wrapf(err, "could not create metrics output directory: %s", c.metricsOutputDir) + } + + // write metrics in a separate file per command and process execution + // to avoid conflicts with previously created profiles + c.metricsFileName = clock.Now().Format("20060102-150405") + if cmd := ctx.SelectedCommand; cmd != nil { + c.metricsFileName += "-" + strings.ReplaceAll(cmd.FullCommand(), " ", "-") + } else { + c.metricsFileName += "-unknown" + } + + c.metricsFileName += ".prom" + + return nil } func (c *observabilityFlags) startMetrics(ctx context.Context) error { @@ -196,6 +234,12 @@ func (c *observabilityFlags) stopMetrics(ctx context.Context) { log(ctx).Warnf("unable to shutdown trace provicer: %v", err) } } + + if c.metricsOutputDir != "" { + if err := prometheus.WriteToTextfile(filepath.Join(c.metricsOutputDir, c.metricsFileName), prometheus.DefaultGatherer); err != nil { + log(ctx).Warnf("unable to write metrics file '%s': %v", c.metricsFileName, err) + } + } } func (c *observabilityFlags) pushPeriodically(ctx context.Context, p *push.Pusher) { diff --git a/cli/observability_flags_test.go b/cli/observability_flags_test.go index 9eb58c01a..a218c57c2 100644 --- a/cli/observability_flags_test.go +++ b/cli/observability_flags_test.go @@ -4,6 +4,7 @@ "io" "net/http" "net/http/httptest" + "os" "sync" "testing" @@ -77,3 +78,19 @@ func TestMetricsPushFlags(t *testing.T) { "--metrics-push-grouping=a=s", ) } + +func TestMetricsSaveToOutputDirFlags(t *testing.T) { + env := testenv.NewCLITest(t, testenv.RepoFormatNotImportant, testenv.NewInProcRunner(t)) + + tmp1 := testutil.TempDirectory(t) + + env.RunAndExpectSuccess(t, "repo", "create", "filesystem", "--path", tmp1) + + tmp2 := testutil.TempDirectory(t) + + env.RunAndExpectSuccess(t, "repo", "status", "--metrics-directory", tmp2) + + entries, err := os.ReadDir(tmp2) + require.NoError(t, err) + require.Len(t, entries, 1, "a metrics output file should have been created") +}