feat(general) maintenance stats for blob retention extension (#4956)

repo/maintenance/blob_retain.go

@@ -15,6 +15,7 @@
 	"github.com/kopia/kopia/repo"
 	"github.com/kopia/kopia/repo/blob"
 	"github.com/kopia/kopia/repo/format"
+	"github.com/kopia/kopia/repo/maintenancestats"
 )
 
 const parallelBlobRetainCPUMultiplier = 2
@@ -28,7 +29,9 @@ type ExtendBlobRetentionTimeOptions struct {
 }
 
 // ExtendBlobRetentionTime extends the retention time of all relevant blobs managed by storage engine with Object Locking enabled.
-func ExtendBlobRetentionTime(ctx context.Context, rep repo.DirectRepositoryWriter, opt ExtendBlobRetentionTimeOptions) (int, error) {
+//
+//nolint:funlen
+func ExtendBlobRetentionTime(ctx context.Context, rep repo.DirectRepositoryWriter, opt ExtendBlobRetentionTimeOptions) (*maintenancestats.ExtendBlobRetentionStats, error) {
 	ctx = contentlog.WithParams(ctx,
 		logparam.String("span:blob-retain", contentlog.RandomSpanID()))
 
@@ -50,14 +53,14 @@ func ExtendBlobRetentionTime(ctx context.Context, rep repo.DirectRepositoryWrite
 
 	blobCfg, err := rep.FormatManager().BlobCfgBlob(ctx)
 	if err != nil {
-		return 0, errors.Wrap(err, "blob configuration")
+		return nil, errors.Wrap(err, "blob configuration")
 	}
 
 	if !blobCfg.IsRetentionEnabled() {
 		// Blob retention is disabled
 		contentlog.Log(ctx, log, "Object lock retention is disabled.")
 
-		return 0, nil
+		return nil, nil
 	}
 
 	extend := make(chan blob.Metadata, extendQueueSize)
@@ -114,26 +117,34 @@ func ExtendBlobRetentionTime(ctx context.Context, rep repo.DirectRepositoryWrite
 	})
 
 	close(extend)
-	contentlog.Log1(ctx, log, "Found blobs to extend", logparam.UInt32("count", *toExtend))
+
+	result := &maintenancestats.ExtendBlobRetentionStats{
+		BlobsToExtend:   atomic.LoadUint32(toExtend),
+		RetentionPeriod: extendOpts.RetentionPeriod.String(),
+	}
+
+	contentlog.Log1(ctx, log, "Found blobs to extend retention time", result)
 
 	// wait for all extend workers to finish.
 	wg.Wait()
 
 	if *failedCnt > 0 {
-		return 0, errors.Errorf("Failed to extend %v blobs", *failedCnt)
+		return nil, errors.Errorf("Failed to extend %v blobs", *failedCnt)
 	}
 
 	if err != nil {
-		return 0, errors.Wrap(err, "error iterating packs")
+		return nil, errors.Wrap(err, "error iterating packs")
 	}
 
 	if opt.DryRun {
-		return int(*toExtend), nil
+		return result, nil
 	}
 
-	contentlog.Log1(ctx, log, "Extended total blobs", logparam.UInt32("count", *cnt))
+	result.BlobsExtended = atomic.LoadUint32(cnt)
 
-	return int(*cnt), nil
+	contentlog.Log1(ctx, log, "Extended retention time for blobs", result)
+
+	return result, nil
 }
 
 // CheckExtendRetention verifies if extension can be enabled due to maintenance and blob parameters.

repo/maintenance/blob_retain_test.go

@@ -71,8 +71,11 @@ func (s *formatSpecificTestSuite) TestExtendBlobRetentionTime(t *testing.T) {
 	earliestExpiry = ta.NowFunc()().Add(period)
 
 	// extend retention time of all blobs
-	_, err = maintenance.ExtendBlobRetentionTime(ctx, env.RepositoryWriter, maintenance.ExtendBlobRetentionTimeOptions{})
+	stats, err := maintenance.ExtendBlobRetentionTime(ctx, env.RepositoryWriter, maintenance.ExtendBlobRetentionTimeOptions{})
 	require.NoError(t, err)
+	require.Equal(t, uint32(4), stats.BlobsExtended)
+	require.Equal(t, uint32(4), stats.BlobsExtended)
+	require.Equal(t, "24h0m0s", stats.RetentionPeriod)
 
 	gotMode, expiry, err = st.GetRetention(ctx, blobsBefore[lastBlobIdx].BlobID)
 	require.NoError(t, err, "getting blob retention info")
@@ -120,8 +123,9 @@ func (s *formatSpecificTestSuite) TestExtendBlobRetentionTimeDisabled(t *testing
 	require.NoError(t, err, "Altering expired object failed")
 
 	// extend retention time of all blobs
-	_, err = maintenance.ExtendBlobRetentionTime(ctx, env.RepositoryWriter, maintenance.ExtendBlobRetentionTimeOptions{})
+	stats, err := maintenance.ExtendBlobRetentionTime(ctx, env.RepositoryWriter, maintenance.ExtendBlobRetentionTimeOptions{})
 	require.NoError(t, err)
+	require.Nil(t, stats)
 
 	_, err = st.TouchBlob(ctx, blobsBefore[lastBlobIdx].BlobID, time.Hour)
 	require.NoError(t, err, "Altering expired object failed")

repo/maintenance/maintenance_run.go

@@ -488,8 +488,7 @@ func runTaskDeleteOrphanedBlobsQuick(ctx context.Context, runParams RunParameter
 
 func runTaskExtendBlobRetentionTimeFull(ctx context.Context, runParams RunParameters, s *Schedule) error {
 	return ReportRun(ctx, runParams.rep, TaskExtendBlobRetentionTimeFull, s, func() (maintenancestats.Kind, error) {
-		_, err := ExtendBlobRetentionTime(ctx, runParams.rep, ExtendBlobRetentionTimeOptions{})
-		return nil, err
+		return ExtendBlobRetentionTime(ctx, runParams.rep, ExtendBlobRetentionTimeOptions{})
 	})
 }
 

repo/maintenancestats/builder.go

@@ -62,6 +62,8 @@ func BuildFromExtra(stats Extra) (Summarizer, error) {
 		result = &CompactIndexesStats{}
 	case deleteUnreferencedPacksStatsKind:
 		result = &DeleteUnreferencedPacksStats{}
+	case extendBlobRetentionStatsKind:
+		result = &ExtendBlobRetentionStats{}
 	default:
 		return nil, errors.Wrapf(ErrUnSupportedStatKindError, "invalid kind for stats %v", stats)
 	}

repo/maintenancestats/builder_test.go

@@ -92,6 +92,18 @@ func TestBuildExtraSuccess(t *testing.T) {
 				Data: []byte(`{"unreferencedPackCount":50,"unreferencedTotalSize":4096,"deletedPackCount":20,"deletedTotalSize":2048,"retainedPackCount":30,"retainedTotalSize":2048}`),
 			},
 		},
+		{
+			name: "ExtendBlobRetentionStats",
+			stats: &ExtendBlobRetentionStats{
+				BlobsToExtend:   10,
+				BlobsExtended:   10,
+				RetentionPeriod: (time.Hour * 24 * 15).String(),
+			},
+			expected: Extra{
+				Kind: extendBlobRetentionStatsKind,
+				Data: []byte(`{"blobsToExtend":10,"blobsExtended":10,"retentionPeriod":"360h0m0s"}`),
+			},
+		},
 	}
 
 	for _, tc := range cases {
@@ -219,6 +231,18 @@ func TestBuildFromExtraSuccess(t *testing.T) {
 				RetainedTotalSize:     2048,
 			},
 		},
+		{
+			name: "ExtendBlobRetentionStats",
+			stats: Extra{
+				Kind: extendBlobRetentionStatsKind,
+				Data: []byte(`{"blobsToExtend":10,"blobsExtended":10,"retentionPeriod":"360h0m0s"}`),
+			},
+			expected: &ExtendBlobRetentionStats{
+				BlobsToExtend:   10,
+				BlobsExtended:   10,
+				RetentionPeriod: (time.Hour * 24 * 15).String(),
+			},
+		},
 	}
 
 	for _, tc := range cases {

repo/maintenancestats/stats_extend_blob_retention.go

@@ -0,0 +1,35 @@
+package maintenancestats
+
+import (
+	"fmt"
+
+	"github.com/kopia/kopia/internal/contentlog"
+)
+
+const extendBlobRetentionStatsKind = "extendBlobRetentionStats"
+
+// ExtendBlobRetentionStats are the stats for extending blob retention time.
+type ExtendBlobRetentionStats struct {
+	BlobsToExtend   uint32 `json:"blobsToExtend"`
+	BlobsExtended   uint32 `json:"blobsExtended"`
+	RetentionPeriod string `json:"retentionPeriod"`
+}
+
+// WriteValueTo writes the stats to JSONWriter.
+func (es *ExtendBlobRetentionStats) WriteValueTo(jw *contentlog.JSONWriter) {
+	jw.BeginObjectField(es.Kind())
+	jw.UInt32Field("blobsToExtend", es.BlobsToExtend)
+	jw.UInt32Field("blobsExtended", es.BlobsExtended)
+	jw.StringField("retentionPeriod", es.RetentionPeriod)
+	jw.EndObject()
+}
+
+// Summary generates a human readable summary for the stats.
+func (es *ExtendBlobRetentionStats) Summary() string {
+	return fmt.Sprintf("Blob retention extension found %v blobs and extended for %v blobs, retention period %v", es.BlobsToExtend, es.BlobsExtended, es.RetentionPeriod)
+}
+
+// Kind returns the kind name for the stats.
+func (es *ExtendBlobRetentionStats) Kind() string {
+	return extendBlobRetentionStatsKind
+}