From 960c33475ea02b56f327f74b564c14feedf2c7dc Mon Sep 17 00:00:00 2001 From: Jarek Kowalski Date: Sat, 30 May 2020 22:49:24 -0700 Subject: [PATCH] maintenance: disabled automatic compaction on repository opening instead moved to run as part of maintenance ('kopia maintenance run') added 'kopia maintenance run --force' flag which runs maintenance even if not owned --- cli/app.go | 12 ++++++++- cli/command_maintenance_run.go | 3 ++- internal/server/server.go | 2 +- repo/content/content_manager.go | 4 +-- repo/content/content_manager_indexes.go | 4 --- repo/maintenance/index_compaction.go | 18 +++++++++++++ repo/maintenance/maintenance_run.go | 25 ++++++++++++++++--- .../snapshotmaintenance.go | 4 +-- 8 files changed, 57 insertions(+), 15 deletions(-) create mode 100644 repo/maintenance/index_compaction.go diff --git a/cli/app.go b/cli/app.go index c7d1556fc..ad54bf132 100644 --- a/cli/app.go +++ b/cli/app.go @@ -165,7 +165,17 @@ func maybeRunMaintenance(ctx context.Context, rep repo.Repository) error { return nil } - return snapshotmaintenance.Run(ctx, rep, maintenance.ModeAuto) + err := snapshotmaintenance.Run(ctx, rep, maintenance.ModeAuto, false) + if err == nil { + return nil + } + + if _, ok := err.(maintenance.NotOwnedError); ok { + // do not report the NotOwnedError to the user since this is automatic maintenance. + return nil + } + + return err } // App returns an instance of command-line application object. diff --git a/cli/command_maintenance_run.go b/cli/command_maintenance_run.go index 533846348..e58495aec 100644 --- a/cli/command_maintenance_run.go +++ b/cli/command_maintenance_run.go @@ -11,6 +11,7 @@ var ( maintenanceRunCommand = maintenanceCommands.Command("run", "Run repository maintenance").Default() maintenanceRunFull = maintenanceRunCommand.Flag("full", "Full maintenance").Bool() + maintenanceRunForce = maintenanceRunCommand.Flag("force", "Run maintenance even if not owned (unsafe)").Hidden().Bool() ) func runMaintenanceCommand(ctx context.Context, rep *repo.DirectRepository) error { @@ -19,7 +20,7 @@ func runMaintenanceCommand(ctx context.Context, rep *repo.DirectRepository) erro mode = maintenance.ModeFull } - return snapshotmaintenance.Run(ctx, rep, mode) + return snapshotmaintenance.Run(ctx, rep, mode, *maintenanceRunForce) } func init() { diff --git a/internal/server/server.go b/internal/server/server.go index 11c8913cd..22ca2ee40 100644 --- a/internal/server/server.go +++ b/internal/server/server.go @@ -267,7 +267,7 @@ func (s *Server) periodicMaintenance(ctx context.Context, r repo.Repository) { return case <-time.After(maintenanceAttemptFrequency): - if err := snapshotmaintenance.Run(ctx, r, maintenance.ModeAuto); err != nil { + if err := snapshotmaintenance.Run(ctx, r, maintenance.ModeAuto, false); err != nil { log(ctx).Warningf("unable to run maintenance: %v", err) } } diff --git a/repo/content/content_manager.go b/repo/content/content_manager.go index 353916f3e..76ce08097 100644 --- a/repo/content/content_manager.go +++ b/repo/content/content_manager.go @@ -722,8 +722,8 @@ func newManagerWithOptions(ctx context.Context, st blob.Storage, f *FormattingOp return nil, errors.Wrap(err, "unable to set up caches") } - if err := m.CompactIndexes(ctx, autoCompactionOptions); err != nil { - return nil, errors.Wrap(err, "error initializing content manager") + if _, _, err := m.loadPackIndexesUnlocked(ctx); err != nil { + return nil, errors.Wrap(err, "error loading indexes") } return m, nil diff --git a/repo/content/content_manager_indexes.go b/repo/content/content_manager_indexes.go index 043458b18..5bbd6fa14 100644 --- a/repo/content/content_manager_indexes.go +++ b/repo/content/content_manager_indexes.go @@ -12,10 +12,6 @@ const verySmallContentFraction = 20 // blobs less than 1/verySmallContentFraction of maxPackSize are considered 'very small' -var autoCompactionOptions = CompactOptions{ - MaxSmallBlobs: 4 * parallelFetches, // nolint:gomnd -} - // CompactOptions provides options for compaction type CompactOptions struct { MaxSmallBlobs int diff --git a/repo/maintenance/index_compaction.go b/repo/maintenance/index_compaction.go new file mode 100644 index 000000000..96d0571d3 --- /dev/null +++ b/repo/maintenance/index_compaction.go @@ -0,0 +1,18 @@ +package maintenance + +import ( + "context" + + "github.com/kopia/kopia/repo/content" +) + +const maxSmallBlobsForIndexCompaction = 8 + +// IndexCompaction rewrites index blobs to reduce their count but does not drop any contents. +func IndexCompaction(ctx context.Context, rep MaintainableRepository) error { + log(ctx).Infof("Compacting indexes...") + + return rep.ContentManager().CompactIndexes(ctx, content.CompactOptions{ + MaxSmallBlobs: maxSmallBlobsForIndexCompaction, + }) +} diff --git a/repo/maintenance/maintenance_run.go b/repo/maintenance/maintenance_run.go index ff642db80..3dbfea7aa 100644 --- a/repo/maintenance/maintenance_run.go +++ b/repo/maintenance/maintenance_run.go @@ -136,18 +136,28 @@ type RunParameters struct { Params *Params } +// NotOwnedError is returned when maintenance cannot run because it is owned by another user. +type NotOwnedError struct { + Owner string +} + +func (e NotOwnedError) Error() string { + return "maintenance must be run by designated user: " + e.Owner +} + // RunExclusive runs the provided callback if the maintenance is owned by local user and // lock can be acquired. Lock is passed to the function, which ensures that every call to Run() // is within the exclusive context. -func RunExclusive(ctx context.Context, rep MaintainableRepository, mode Mode, cb func(runParams RunParameters) error) error { +func RunExclusive(ctx context.Context, rep MaintainableRepository, mode Mode, force bool, cb func(runParams RunParameters) error) error { p, err := GetParams(ctx, rep) if err != nil { return errors.Wrap(err, "unable to get maintenance params") } - if myUsername := rep.Username() + "@" + rep.Hostname(); p.Owner != myUsername { - log(ctx).Debugf("maintenance owned by another user '%v'", p.Owner) - return nil + if !force { + if myUsername := rep.Username() + "@" + rep.Hostname(); p.Owner != myUsername { + return NotOwnedError{p.Owner} + } } if mode == ModeAuto { @@ -231,6 +241,13 @@ func runQuickMaintenance(ctx context.Context, runParams RunParameters) error { return errors.Wrap(err, "error deleting unreferenced metadata blobs") } + // consolidate many smaller indexes into fewer larger ones. + if err := ReportRun(ctx, runParams.rep, "index-compaction", func() error { + return IndexCompaction(ctx, runParams.rep) + }); err != nil { + return errors.Wrap(err, "error performing index compaction") + } + return nil } diff --git a/snapshot/snapshotmaintenance/snapshotmaintenance.go b/snapshot/snapshotmaintenance/snapshotmaintenance.go index ee6f41fe0..57e872ce2 100644 --- a/snapshot/snapshotmaintenance/snapshotmaintenance.go +++ b/snapshot/snapshotmaintenance/snapshotmaintenance.go @@ -12,13 +12,13 @@ ) // Run runs the complete snapshot and repository maintenance. -func Run(ctx context.Context, rep repo.Repository, mode maintenance.Mode) error { +func Run(ctx context.Context, rep repo.Repository, mode maintenance.Mode, force bool) error { dr, ok := rep.(*repo.DirectRepository) if !ok { return nil } - return maintenance.RunExclusive(ctx, dr, mode, + return maintenance.RunExclusive(ctx, dr, mode, force, func(runParams maintenance.RunParameters) error { // run snapshot GC before full maintenance if runParams.Mode == maintenance.ModeFull {