mirror/kopia
1
0
Fork 0
mirror of https://github.com/kopia/kopia.git synced 2025-12-23 22:57:50 -05:00
Code Issues Packages Projects Releases Wiki Activity

feat(general): maintenance stats for clean up logs phase (#4959)

Browse Source
This commit is contained in:
lyndon-li
2025-11-06 15:23:47 +08:00
committed by GitHub
parent c7ea2c6e09
commit 3446837bed
6 changed files with 104 additions and 16 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
cli/command_logs_cleanup.go
View File

@@ -31,7 +31,7 @@ func (c *commandLogsCleanup) setup(svc appServices, parent commandParent) {
func (c *commandLogsCleanup) run(ctx context.Context, rep repo.DirectRepositoryWriter) error {
rep.LogManager().Disable()
toDelete, err := maintenance.CleanupLogs(ctx, rep, maintenance.LogRetentionOptions{
stats, err := maintenance.CleanupLogs(ctx, rep, maintenance.LogRetentionOptions{
MaxTotalSize: c.maxTotalSizeMB << 20, //nolint:mnd
MaxCount: c.maxCount,
MaxAge: c.maxAge,
@@ -41,11 +41,11 @@ func (c *commandLogsCleanup) run(ctx context.Context, rep repo.DirectRepositoryW
return errors.Wrap(err, "error expiring logs")
}
if len(toDelete) > 0 {
if stats.ToDeleteBlobCount > 0 {
if c.dryRun {
log(ctx).Infof("Would delete %v logs.", len(toDelete))
log(ctx).Infof("Would delete %v logs.", stats.ToDeleteBlobCount)
} else {
log(ctx).Infof("Deleted %v logs.", len(toDelete))
log(ctx).Infof("Deleted %v logs.", stats.DeletedBlobCount)
}
} else {
log(ctx).Info("No logs found to delete.")

33
repo/maintenance/cleanup_logs.go
View File

@@ -12,6 +12,7 @@
"github.com/kopia/kopia/internal/contentlog/logparam"
"github.com/kopia/kopia/repo"
"github.com/kopia/kopia/repo/blob"
"github.com/kopia/kopia/repo/maintenancestats"
)
// LogRetentionOptions provides options for logs retention.
@@ -43,7 +44,7 @@ func defaultLogRetention() LogRetentionOptions {
}
// CleanupLogs deletes old logs blobs beyond certain age, total size or count.
func CleanupLogs(ctx context.Context, rep repo.DirectRepositoryWriter, opt LogRetentionOptions) ([]blob.Metadata, error) {
func CleanupLogs(ctx context.Context, rep repo.DirectRepositoryWriter, opt LogRetentionOptions) (*maintenancestats.CleanupLogsStats, error) {
ctx = contentlog.WithParams(ctx,
logparam.String("span:cleanup-logs", contentlog.RandomSpanID()))
@@ -63,14 +64,14 @@ func CleanupLogs(ctx context.Context, rep repo.DirectRepositoryWriter, opt LogRe
return allLogBlobs[i].Timestamp.After(allLogBlobs[j].Timestamp)
})
var totalSize int64
var retainedSize int64
deletePosition := len(allLogBlobs)
for i, bm := range allLogBlobs {
totalSize += bm.Length
retainedSize += bm.Length
if totalSize > opt.MaxTotalSize && opt.MaxTotalSize > 0 {
if retainedSize > opt.MaxTotalSize && opt.MaxTotalSize > 0 {
deletePosition = i
break
}
@@ -88,10 +89,21 @@ func CleanupLogs(ctx context.Context, rep repo.DirectRepositoryWriter, opt LogRe
toDelete := allLogBlobs[deletePosition:]
contentlog.Log2(ctx, log,
"Keeping logs",
logparam.Int("count", deletePosition),
logparam.Int64("bytes", totalSize))
var toDeleteSize int64
for _, bm := range toDelete {
toDeleteSize += bm.Length
}
result := &maintenancestats.CleanupLogsStats{
RetainedBlobCount: deletePosition,
RetainedBlobSize: retainedSize,
ToDeleteBlobCount: len(toDelete),
ToDeleteBlobSize: toDeleteSize,
DeletedBlobCount: 0,
DeletedBlobSize: 0,
}
contentlog.Log1(ctx, log, "Clean up logs", result)
if !opt.DryRun {
for _, bm := range toDelete {
@@ -99,7 +111,10 @@ func CleanupLogs(ctx context.Context, rep repo.DirectRepositoryWriter, opt LogRe
return nil, errors.Wrapf(err, "error deleting log %v", bm.BlobID)
}
}
result.DeletedBlobCount = result.ToDeleteBlobCount
result.DeletedBlobSize = result.ToDeleteBlobSize
}
return toDelete, nil
return result, nil
}

6
repo/maintenance/maintenance_run.go
View File

@@ -332,11 +332,11 @@ func notDeletingOrphanedPacks(ctx context.Context, log *contentlog.Logger, s *Sc
func runTaskCleanupLogs(ctx context.Context, runParams RunParameters, s *Schedule) error {
return ReportRun(ctx, runParams.rep, TaskCleanupLogs, s, func() (maintenancestats.Kind, error) {
deleted, err := CleanupLogs(ctx, runParams.rep, runParams.Params.LogRetention.OrDefault())
stats, err := CleanupLogs(ctx, runParams.rep, runParams.Params.LogRetention.OrDefault())
userLog(ctx).Infof("Cleaned up %v logs.", len(deleted))
userLog(ctx).Infof("Cleaned up %v logs.", stats.DeletedBlobCount)
return nil, err
return stats, err
})
}

2
repo/maintenancestats/builder.go
View File

@@ -64,6 +64,8 @@ func BuildFromExtra(stats Extra) (Summarizer, error) {
result = &DeleteUnreferencedPacksStats{}
case extendBlobRetentionStatsKind:
result = &ExtendBlobRetentionStats{}
case cleanupLogsStatsKind:
result = &CleanupLogsStats{}
default:
return nil, errors.Wrapf(ErrUnSupportedStatKindError, "invalid kind for stats %v", stats)
}

30
repo/maintenancestats/builder_test.go
View File

@@ -104,6 +104,21 @@ func TestBuildExtraSuccess(t *testing.T) {
Data: []byte(`{"toExtendBlobCount":10,"extendedBlobCount":10,"retentionPeriod":"360h0m0s"}`),
},
},
{
name: "CleanupLogsStats",
stats: &CleanupLogsStats{
ToDeleteBlobCount: 10,
ToDeleteBlobSize: 1024,
DeletedBlobCount: 5,
DeletedBlobSize: 512,
RetainedBlobCount: 20,
RetainedBlobSize: 2048,
},
expected: Extra{
Kind: cleanupLogsStatsKind,
Data: []byte(`{"toDeleteBlobCount":10,"toDeleteBlobSize":1024,"deletedBlobCount":5,"deletedBlobSize":512,"retainedBlobCount":20,"retainedBlobSize":2048}`),
},
},
}
for _, tc := range cases {
@@ -243,6 +258,21 @@ func TestBuildFromExtraSuccess(t *testing.T) {
RetentionPeriod: (time.Hour * 24 * 15).String(),
},
},
{
name: "CleanupLogsStats",
stats: Extra{
Kind: cleanupLogsStatsKind,
Data: []byte(`{"toDeleteBlobCount":10,"toDeleteBlobSize":1024,"retainedBlobCount":20,"retainedBlobSize":2048,"deletedBlobCount":5,"deletedBlobSize":512}`),
},
expected: &CleanupLogsStats{
ToDeleteBlobCount: 10,
ToDeleteBlobSize: 1024,
RetainedBlobCount: 20,
RetainedBlobSize: 2048,
DeletedBlobCount: 5,
DeletedBlobSize: 512,
},
},
}
for _, tc := range cases {

41
repo/maintenancestats/stats_clean_up_log.go Normal file
View File

@@ -0,0 +1,41 @@
package maintenancestats
import (
"fmt"
"github.com/kopia/kopia/internal/contentlog"
)
const cleanupLogsStatsKind = "cleanupLogsStats"
// CleanupLogsStats are the stats for cleanning up logs.
type CleanupLogsStats struct {
ToDeleteBlobCount int `json:"toDeleteBlobCount"`
ToDeleteBlobSize int64 `json:"toDeleteBlobSize"`
DeletedBlobCount int `json:"deletedBlobCount"`
DeletedBlobSize int64 `json:"deletedBlobSize"`
RetainedBlobCount int `json:"retainedBlobCount"`
RetainedBlobSize int64 `json:"retainedBlobSize"`
}
// WriteValueTo writes the stats to JSONWriter.
func (cs *CleanupLogsStats) WriteValueTo(jw *contentlog.JSONWriter) {
jw.BeginObjectField(cs.Kind())
jw.IntField("toDeleteBlobCount", cs.ToDeleteBlobCount)
jw.Int64Field("toDeleteBlobSize", cs.ToDeleteBlobSize)
jw.IntField("deletedBlobCount", cs.DeletedBlobCount)
jw.Int64Field("deletedBlobSize", cs.DeletedBlobSize)
jw.IntField("retainedBlobCount", cs.RetainedBlobCount)
jw.Int64Field("retainedBlobSize", cs.RetainedBlobSize)
jw.EndObject()
}
// Summary generates a human readable summary for the stats.
func (cs *CleanupLogsStats) Summary() string {
return fmt.Sprintf("Found %v(%v) logs blobs for deletion and deleted %v(%v) of them. Retained %v(%v) log blobs.", cs.ToDeleteBlobCount, cs.ToDeleteBlobSize, cs.DeletedBlobCount, cs.DeletedBlobSize, cs.RetainedBlobCount, cs.RetainedBlobSize)
}
// Kind returns the kind name for the stats.
func (cs *CleanupLogsStats) Kind() string {
return cleanupLogsStatsKind
}