From 8b8d4a574b779682d130937ab91cee5dcb5807bd Mon Sep 17 00:00:00 2001
From: Julio Lopez <1953782+julio-lopez@users.noreply.github.com>
Date: Wed, 9 Aug 2023 17:40:45 -0700
Subject: [PATCH] feat(cli): save observability metric before exiting the
 process (#3201)

Objective: to facilitate capturing metrics for relatively short-lived kopia
executions.

Motivation: Currently, kopia allows a. exposing metrics via a Prometheus
exporter; and b. pushing metrics to a Prometheus gateway. In certain
scenarios, such as short lived executions, it is not possible to reliably
scrape the metrics from the exporter endpoint. And, while the pusher
approach would work, it requires starting a separate push gateway.

This change allows saving the metrics to a local file before kopia exists
by specifying the metrics output directory. For example,

`kopia --metrics-directory /tmp/kopia-metrics/ snapshot create ....`

In this case, after kopia exists, the `/tmp/kopia-metrics/` directory will
contain a file with a named of the form:
`<date>-<time>-<command_subcommand>.prom`
---
 cli/observability_flags.go      | 44 +++++++++++++++++++++++++++++++++
 cli/observability_flags_test.go | 17 +++++++++++++
 2 files changed, 61 insertions(+)

diff --git a/cli/observability_flags.go b/cli/observability_flags.go
index 5149ce0a5..033aae07f 100644
--- a/cli/observability_flags.go
+++ b/cli/observability_flags.go
@@ -4,6 +4,8 @@
 	"context"
 	"net/http"
 	"net/http/pprof"
+	"os"
+	"path/filepath"
 	"sort"
 	"strings"
 	"sync"
@@ -21,9 +23,13 @@
 	"go.opentelemetry.io/otel/sdk/trace"
 	semconv "go.opentelemetry.io/otel/semconv/v1.4.0"
 
+	"github.com/kopia/kopia/internal/clock"
 	"github.com/kopia/kopia/repo"
 )
 
+// DirMode is the directory mode for output directories.
+const DirMode = 0o700
+
 //nolint:gochecknoglobals
 var metricsPushFormats = map[string]expfmt.Format{
 	"text":               expfmt.FmtText,
@@ -44,6 +50,8 @@ type observabilityFlags struct {
 	metricsPushUsername string
 	metricsPushPassword string
 	metricsPushFormat   string
+	metricsOutputDir    string
+	metricsFileName     string
 
 	enableJaeger bool
 
@@ -76,6 +84,36 @@ func (c *observabilityFlags) setup(svc appServices, app *kingpin.Application) {
 	sort.Strings(formats)
 
 	app.Flag("metrics-push-format", "Format to use for push gateway").Envar(svc.EnvName("KOPIA_METRICS_FORMAT")).Hidden().EnumVar(&c.metricsPushFormat, formats...)
+
+	app.Flag("metrics-directory", "Directory where the metrics should be saved when kopia exits. A file per process execution will be created in this directory").Hidden().StringVar(&c.metricsOutputDir)
+
+	app.PreAction(c.initialize)
+}
+
+func (c *observabilityFlags) initialize(ctx *kingpin.ParseContext) error {
+	if c.metricsOutputDir == "" {
+		return nil
+	}
+
+	c.metricsOutputDir = filepath.Clean(c.metricsOutputDir)
+
+	// ensure the metrics output dir can be created
+	if err := os.MkdirAll(c.metricsOutputDir, DirMode); err != nil {
+		return errors.Wrapf(err, "could not create metrics output directory: %s", c.metricsOutputDir)
+	}
+
+	// write metrics in a separate file per command and process execution
+	// to avoid conflicts with previously created profiles
+	c.metricsFileName = clock.Now().Format("20060102-150405")
+	if cmd := ctx.SelectedCommand; cmd != nil {
+		c.metricsFileName += "-" + strings.ReplaceAll(cmd.FullCommand(), " ", "-")
+	} else {
+		c.metricsFileName += "-unknown"
+	}
+
+	c.metricsFileName += ".prom"
+
+	return nil
 }
 
 func (c *observabilityFlags) startMetrics(ctx context.Context) error {
@@ -196,6 +234,12 @@ func (c *observabilityFlags) stopMetrics(ctx context.Context) {
 			log(ctx).Warnf("unable to shutdown trace provicer: %v", err)
 		}
 	}
+
+	if c.metricsOutputDir != "" {
+		if err := prometheus.WriteToTextfile(filepath.Join(c.metricsOutputDir, c.metricsFileName), prometheus.DefaultGatherer); err != nil {
+			log(ctx).Warnf("unable to write metrics file '%s': %v", c.metricsFileName, err)
+		}
+	}
 }
 
 func (c *observabilityFlags) pushPeriodically(ctx context.Context, p *push.Pusher) {
diff --git a/cli/observability_flags_test.go b/cli/observability_flags_test.go
index 9eb58c01a..a218c57c2 100644
--- a/cli/observability_flags_test.go
+++ b/cli/observability_flags_test.go
@@ -4,6 +4,7 @@
 	"io"
 	"net/http"
 	"net/http/httptest"
+	"os"
 	"sync"
 	"testing"
 
@@ -77,3 +78,19 @@ func TestMetricsPushFlags(t *testing.T) {
 		"--metrics-push-grouping=a=s",
 	)
 }
+
+func TestMetricsSaveToOutputDirFlags(t *testing.T) {
+	env := testenv.NewCLITest(t, testenv.RepoFormatNotImportant, testenv.NewInProcRunner(t))
+
+	tmp1 := testutil.TempDirectory(t)
+
+	env.RunAndExpectSuccess(t, "repo", "create", "filesystem", "--path", tmp1)
+
+	tmp2 := testutil.TempDirectory(t)
+
+	env.RunAndExpectSuccess(t, "repo", "status", "--metrics-directory", tmp2)
+
+	entries, err := os.ReadDir(tmp2)
+	require.NoError(t, err)
+	require.Len(t, entries, 1, "a metrics output file should have been created")
+}