From af38dfc69d02241118bf7473252ca76c842cddde Mon Sep 17 00:00:00 2001 From: lyndon-li <98304688+Lyndon-Li@users.noreply.github.com> Date: Thu, 6 Nov 2025 08:16:44 +0800 Subject: [PATCH] feat(general) maintenance stats for blob retention extension (#4956) --- repo/maintenance/blob_retain.go | 29 ++++++++++----- repo/maintenance/blob_retain_test.go | 8 +++-- repo/maintenance/maintenance_run.go | 3 +- repo/maintenancestats/builder.go | 2 ++ repo/maintenancestats/builder_test.go | 24 +++++++++++++ .../stats_extend_blob_retention.go | 35 +++++++++++++++++++ 6 files changed, 88 insertions(+), 13 deletions(-) create mode 100644 repo/maintenancestats/stats_extend_blob_retention.go diff --git a/repo/maintenance/blob_retain.go b/repo/maintenance/blob_retain.go index b2c6f9a9e..a87c1ec61 100644 --- a/repo/maintenance/blob_retain.go +++ b/repo/maintenance/blob_retain.go @@ -15,6 +15,7 @@ "github.com/kopia/kopia/repo" "github.com/kopia/kopia/repo/blob" "github.com/kopia/kopia/repo/format" + "github.com/kopia/kopia/repo/maintenancestats" ) const parallelBlobRetainCPUMultiplier = 2 @@ -28,7 +29,9 @@ type ExtendBlobRetentionTimeOptions struct { } // ExtendBlobRetentionTime extends the retention time of all relevant blobs managed by storage engine with Object Locking enabled. -func ExtendBlobRetentionTime(ctx context.Context, rep repo.DirectRepositoryWriter, opt ExtendBlobRetentionTimeOptions) (int, error) { +// +//nolint:funlen +func ExtendBlobRetentionTime(ctx context.Context, rep repo.DirectRepositoryWriter, opt ExtendBlobRetentionTimeOptions) (*maintenancestats.ExtendBlobRetentionStats, error) { ctx = contentlog.WithParams(ctx, logparam.String("span:blob-retain", contentlog.RandomSpanID())) @@ -50,14 +53,14 @@ func ExtendBlobRetentionTime(ctx context.Context, rep repo.DirectRepositoryWrite blobCfg, err := rep.FormatManager().BlobCfgBlob(ctx) if err != nil { - return 0, errors.Wrap(err, "blob configuration") + return nil, errors.Wrap(err, "blob configuration") } if !blobCfg.IsRetentionEnabled() { // Blob retention is disabled contentlog.Log(ctx, log, "Object lock retention is disabled.") - return 0, nil + return nil, nil } extend := make(chan blob.Metadata, extendQueueSize) @@ -114,26 +117,34 @@ func ExtendBlobRetentionTime(ctx context.Context, rep repo.DirectRepositoryWrite }) close(extend) - contentlog.Log1(ctx, log, "Found blobs to extend", logparam.UInt32("count", *toExtend)) + + result := &maintenancestats.ExtendBlobRetentionStats{ + BlobsToExtend: atomic.LoadUint32(toExtend), + RetentionPeriod: extendOpts.RetentionPeriod.String(), + } + + contentlog.Log1(ctx, log, "Found blobs to extend retention time", result) // wait for all extend workers to finish. wg.Wait() if *failedCnt > 0 { - return 0, errors.Errorf("Failed to extend %v blobs", *failedCnt) + return nil, errors.Errorf("Failed to extend %v blobs", *failedCnt) } if err != nil { - return 0, errors.Wrap(err, "error iterating packs") + return nil, errors.Wrap(err, "error iterating packs") } if opt.DryRun { - return int(*toExtend), nil + return result, nil } - contentlog.Log1(ctx, log, "Extended total blobs", logparam.UInt32("count", *cnt)) + result.BlobsExtended = atomic.LoadUint32(cnt) - return int(*cnt), nil + contentlog.Log1(ctx, log, "Extended retention time for blobs", result) + + return result, nil } // CheckExtendRetention verifies if extension can be enabled due to maintenance and blob parameters. diff --git a/repo/maintenance/blob_retain_test.go b/repo/maintenance/blob_retain_test.go index c4f657265..824cb85ca 100644 --- a/repo/maintenance/blob_retain_test.go +++ b/repo/maintenance/blob_retain_test.go @@ -71,8 +71,11 @@ func (s *formatSpecificTestSuite) TestExtendBlobRetentionTime(t *testing.T) { earliestExpiry = ta.NowFunc()().Add(period) // extend retention time of all blobs - _, err = maintenance.ExtendBlobRetentionTime(ctx, env.RepositoryWriter, maintenance.ExtendBlobRetentionTimeOptions{}) + stats, err := maintenance.ExtendBlobRetentionTime(ctx, env.RepositoryWriter, maintenance.ExtendBlobRetentionTimeOptions{}) require.NoError(t, err) + require.Equal(t, uint32(4), stats.BlobsExtended) + require.Equal(t, uint32(4), stats.BlobsExtended) + require.Equal(t, "24h0m0s", stats.RetentionPeriod) gotMode, expiry, err = st.GetRetention(ctx, blobsBefore[lastBlobIdx].BlobID) require.NoError(t, err, "getting blob retention info") @@ -120,8 +123,9 @@ func (s *formatSpecificTestSuite) TestExtendBlobRetentionTimeDisabled(t *testing require.NoError(t, err, "Altering expired object failed") // extend retention time of all blobs - _, err = maintenance.ExtendBlobRetentionTime(ctx, env.RepositoryWriter, maintenance.ExtendBlobRetentionTimeOptions{}) + stats, err := maintenance.ExtendBlobRetentionTime(ctx, env.RepositoryWriter, maintenance.ExtendBlobRetentionTimeOptions{}) require.NoError(t, err) + require.Nil(t, stats) _, err = st.TouchBlob(ctx, blobsBefore[lastBlobIdx].BlobID, time.Hour) require.NoError(t, err, "Altering expired object failed") diff --git a/repo/maintenance/maintenance_run.go b/repo/maintenance/maintenance_run.go index 5c6deb139..588f98c2e 100644 --- a/repo/maintenance/maintenance_run.go +++ b/repo/maintenance/maintenance_run.go @@ -488,8 +488,7 @@ func runTaskDeleteOrphanedBlobsQuick(ctx context.Context, runParams RunParameter func runTaskExtendBlobRetentionTimeFull(ctx context.Context, runParams RunParameters, s *Schedule) error { return ReportRun(ctx, runParams.rep, TaskExtendBlobRetentionTimeFull, s, func() (maintenancestats.Kind, error) { - _, err := ExtendBlobRetentionTime(ctx, runParams.rep, ExtendBlobRetentionTimeOptions{}) - return nil, err + return ExtendBlobRetentionTime(ctx, runParams.rep, ExtendBlobRetentionTimeOptions{}) }) } diff --git a/repo/maintenancestats/builder.go b/repo/maintenancestats/builder.go index c1263bc09..cfebb54be 100644 --- a/repo/maintenancestats/builder.go +++ b/repo/maintenancestats/builder.go @@ -62,6 +62,8 @@ func BuildFromExtra(stats Extra) (Summarizer, error) { result = &CompactIndexesStats{} case deleteUnreferencedPacksStatsKind: result = &DeleteUnreferencedPacksStats{} + case extendBlobRetentionStatsKind: + result = &ExtendBlobRetentionStats{} default: return nil, errors.Wrapf(ErrUnSupportedStatKindError, "invalid kind for stats %v", stats) } diff --git a/repo/maintenancestats/builder_test.go b/repo/maintenancestats/builder_test.go index b500804c5..6adc144a8 100644 --- a/repo/maintenancestats/builder_test.go +++ b/repo/maintenancestats/builder_test.go @@ -92,6 +92,18 @@ func TestBuildExtraSuccess(t *testing.T) { Data: []byte(`{"unreferencedPackCount":50,"unreferencedTotalSize":4096,"deletedPackCount":20,"deletedTotalSize":2048,"retainedPackCount":30,"retainedTotalSize":2048}`), }, }, + { + name: "ExtendBlobRetentionStats", + stats: &ExtendBlobRetentionStats{ + BlobsToExtend: 10, + BlobsExtended: 10, + RetentionPeriod: (time.Hour * 24 * 15).String(), + }, + expected: Extra{ + Kind: extendBlobRetentionStatsKind, + Data: []byte(`{"blobsToExtend":10,"blobsExtended":10,"retentionPeriod":"360h0m0s"}`), + }, + }, } for _, tc := range cases { @@ -219,6 +231,18 @@ func TestBuildFromExtraSuccess(t *testing.T) { RetainedTotalSize: 2048, }, }, + { + name: "ExtendBlobRetentionStats", + stats: Extra{ + Kind: extendBlobRetentionStatsKind, + Data: []byte(`{"blobsToExtend":10,"blobsExtended":10,"retentionPeriod":"360h0m0s"}`), + }, + expected: &ExtendBlobRetentionStats{ + BlobsToExtend: 10, + BlobsExtended: 10, + RetentionPeriod: (time.Hour * 24 * 15).String(), + }, + }, } for _, tc := range cases { diff --git a/repo/maintenancestats/stats_extend_blob_retention.go b/repo/maintenancestats/stats_extend_blob_retention.go new file mode 100644 index 000000000..723a7bf9c --- /dev/null +++ b/repo/maintenancestats/stats_extend_blob_retention.go @@ -0,0 +1,35 @@ +package maintenancestats + +import ( + "fmt" + + "github.com/kopia/kopia/internal/contentlog" +) + +const extendBlobRetentionStatsKind = "extendBlobRetentionStats" + +// ExtendBlobRetentionStats are the stats for extending blob retention time. +type ExtendBlobRetentionStats struct { + BlobsToExtend uint32 `json:"blobsToExtend"` + BlobsExtended uint32 `json:"blobsExtended"` + RetentionPeriod string `json:"retentionPeriod"` +} + +// WriteValueTo writes the stats to JSONWriter. +func (es *ExtendBlobRetentionStats) WriteValueTo(jw *contentlog.JSONWriter) { + jw.BeginObjectField(es.Kind()) + jw.UInt32Field("blobsToExtend", es.BlobsToExtend) + jw.UInt32Field("blobsExtended", es.BlobsExtended) + jw.StringField("retentionPeriod", es.RetentionPeriod) + jw.EndObject() +} + +// Summary generates a human readable summary for the stats. +func (es *ExtendBlobRetentionStats) Summary() string { + return fmt.Sprintf("Blob retention extension found %v blobs and extended for %v blobs, retention period %v", es.BlobsToExtend, es.BlobsExtended, es.RetentionPeriod) +} + +// Kind returns the kind name for the stats. +func (es *ExtendBlobRetentionStats) Kind() string { + return extendBlobRetentionStatsKind +}