From fa7976599ca5c82448f51cb3be658166959d5671 Mon Sep 17 00:00:00 2001 From: Jarek Kowalski Date: Wed, 20 Jan 2021 11:41:47 -0800 Subject: [PATCH] repo: refactored repository interfaces (#780) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - `repo.Repository` is now read-only and only has methods that can be supported over kopia server - `repo.RepositoryWriter` has read-write methods that can be supported over kopia server - `repo.DirectRepository` is read-only and contains all methods of `repo.Repository` plus some low-level methods for data inspection - `repo.DirectRepositoryWriter` contains write methods for `repo.DirectRepository` - `repo.Reader` removed and merged with `repo.Repository` - `repo.Writer` became `repo.RepositoryWriter` - `*repo.DirectRepository` struct became `repo.DirectRepository` interface Getting `{Direct}RepositoryWriter` requires using `NewWriter()` or `NewDirectWriter()` on a read-only repository and multiple simultaneous writers are supported at the same time, each writing to their own indexes and pack blobs. `repo.Open` returns `repo.Repository` (which is also `repo.RepositoryWriter`). * content: removed implicit flush on content manager close * repo: added tests for WriteSession() and implicit flush behavior * invalidate manifest manager after write session * cli: disable maintenance in 'kopia server start' Server will close the repository before completing. * repo: unconditionally close RepositoryWriter in {Direct,}WriteSession * repo: added panic in case somebody tries to create RepositoryWriter after closing repository - used atomic to manage SharedManager.closed * removed stale example * linter: fixed spurious failures Co-authored-by: Julio López --- cli/app.go | 49 +-- cli/auto_upgrade.go | 31 +- cli/command_blob_delete.go | 6 +- cli/command_blob_gc.go | 4 +- cli/command_blob_list.go | 4 +- cli/command_blob_show.go | 8 +- cli/command_blob_stats.go | 4 +- cli/command_cache_clear.go | 6 +- cli/command_cache_info.go | 12 +- cli/command_cache_set.go | 8 +- cli/command_cache_sync.go | 6 +- cli/command_content_list.go | 4 +- cli/command_content_rewrite.go | 4 +- cli/command_content_rm.go | 6 +- cli/command_content_show.go | 6 +- cli/command_content_stats.go | 4 +- cli/command_content_verify.go | 16 +- cli/command_diff.go | 2 +- cli/command_index_inspect.go | 8 +- cli/command_index_list.go | 4 +- cli/command_index_optimize.go | 6 +- cli/command_index_recover.go | 10 +- cli/command_ls.go | 2 +- cli/command_maintenance_info.go | 4 +- cli/command_maintenance_run.go | 4 +- cli/command_maintenance_set.go | 8 +- cli/command_manifest_ls.go | 2 +- cli/command_manifest_rm.go | 2 +- cli/command_manifest_show.go | 2 +- cli/command_mount.go | 2 +- cli/command_policy.go | 2 +- cli/command_policy_edit.go | 2 +- cli/command_policy_ls.go | 2 +- cli/command_policy_remove.go | 2 +- cli/command_policy_set.go | 2 +- cli/command_policy_show.go | 2 +- cli/command_repository_create.go | 25 +- cli/command_repository_set_client.go | 2 +- cli/command_repository_status.go | 18 +- cli/command_repository_sync.go | 8 +- cli/command_repository_upgrade.go | 4 +- cli/command_restore.go | 2 +- cli/command_server_start.go | 9 +- cli/command_session_list.go | 6 +- cli/command_show.go | 2 +- cli/command_snapshot_copy_move_history.go | 8 +- cli/command_snapshot_create.go | 14 +- cli/command_snapshot_delete.go | 6 +- cli/command_snapshot_estimate.go | 2 +- cli/command_snapshot_expire.go | 4 +- cli/command_snapshot_gc.go | 4 +- cli/command_snapshot_list.go | 12 +- cli/command_snapshot_migrate.go | 18 +- cli/command_snapshot_verify.go | 16 +- cli/storage_providers.go | 4 +- examples/upload_download/main.go | 30 -- examples/upload_download/setup_repository.go | 59 ---- .../upload_download_objects.go | 67 ---- internal/repotesting/repotesting.go | 40 ++- internal/repotesting/repotesting_test.go | 25 +- internal/server/api_content.go | 14 +- internal/server/api_error.go | 6 + internal/server/api_manifest.go | 17 +- internal/server/api_policies.go | 22 +- internal/server/api_repo.go | 54 ++-- internal/server/api_sources.go | 12 +- internal/server/server.go | 24 +- internal/server/source_manager.go | 55 ++-- repo/api_server_repository.go | 39 ++- repo/blob/storage.go | 55 ++-- ...d_manager.go => committed_read_manager.go} | 36 +++ repo/content/content_index_reader.go | 14 + repo/content/content_manager.go | 20 +- repo/content/content_manager_test.go | 13 +- repo/content/content_reader.go | 15 + repo/content/index_blob_manager_test.go | 1 - repo/local_config.go | 5 + repo/maintenance/blob_gc.go | 5 +- repo/maintenance/blob_gc_test.go | 76 ++--- repo/maintenance/content_rewrite.go | 19 +- repo/maintenance/drop_deleted_contents.go | 3 +- repo/maintenance/index_compaction.go | 3 +- repo/maintenance/maintenance_params.go | 9 +- repo/maintenance/maintenance_run.go | 27 +- repo/maintenance/maintenance_schedule.go | 11 +- repo/maintenance/maintenance_schedule_test.go | 6 +- repo/manifest/committed_manifest_manager.go | 8 +- repo/manifest/manifest_manager.go | 2 +- repo/open.go | 50 ++- repo/repository.go | 298 ++++++++++++------ repo/repository_test.go | 238 ++++++++++++-- repo/token.go | 4 +- repo/upgrade.go | 4 +- snapshot/manager.go | 14 +- snapshot/policy/expire.go | 6 +- snapshot/policy/policy_manager.go | 18 +- snapshot/policy/policy_manager_test.go | 14 +- snapshot/restore/restore.go | 2 +- snapshot/snapshot_test.go | 52 +-- snapshot/snapshotfs/all_sources.go | 4 +- snapshot/snapshotfs/objref.go | 8 +- snapshot/snapshotfs/repofs.go | 12 +- snapshot/snapshotfs/source_directories.go | 2 +- snapshot/snapshotfs/source_snapshots.go | 2 +- snapshot/snapshotfs/upload.go | 4 +- snapshot/snapshotfs/upload_test.go | 9 +- snapshot/snapshotgc/gc.go | 12 +- snapshot/snapshotmaintenance/helper_test.go | 4 +- .../snapshotmaintenance.go | 7 +- .../snapshotmaintenance_test.go | 62 ++-- .../repository_stress_test.go | 42 +-- 111 files changed, 1199 insertions(+), 871 deletions(-) delete mode 100644 examples/upload_download/main.go delete mode 100644 examples/upload_download/setup_repository.go delete mode 100644 examples/upload_download/upload_download_objects.go rename repo/content/{content_read_manager.go => committed_read_manager.go} (91%) create mode 100644 repo/content/content_index_reader.go create mode 100644 repo/content/content_reader.go diff --git a/cli/app.go b/cli/app.go index b719857c5..30b895ae7 100644 --- a/cli/app.go +++ b/cli/app.go @@ -78,7 +78,7 @@ func serverAction(act func(ctx context.Context, cli *apiclient.KopiaAPIClient) e } } -func assertDirectRepository(act func(ctx context.Context, rep *repo.DirectRepository) error) func(ctx context.Context, rep repo.Repository) error { +func assertDirectRepository(act func(ctx context.Context, rep repo.DirectRepository) error) func(ctx context.Context, rep repo.Repository) error { return func(ctx context.Context, rep repo.Repository) error { if rep == nil { return act(ctx, nil) @@ -86,7 +86,7 @@ func assertDirectRepository(act func(ctx context.Context, rep *repo.DirectReposi // right now this assertion never fails, // but will fail in the future when we have remote repository implementation - lr, ok := rep.(*repo.DirectRepository) + lr, ok := rep.(repo.DirectRepository) if !ok { return errors.Errorf("operation supported only on direct repository") } @@ -95,26 +95,27 @@ func assertDirectRepository(act func(ctx context.Context, rep *repo.DirectReposi } } -func directRepositoryAction(act func(ctx context.Context, rep *repo.DirectRepository) error) func(ctx *kingpin.ParseContext) error { - return maybeRepositoryAction(assertDirectRepository(act), repositoryAccessMode{ - mustBeConnected: true, - }) -} - -func directRepositoryReadAction(act func(ctx context.Context, rep *repo.DirectRepository) error) func(ctx *kingpin.ParseContext) error { - return maybeRepositoryAction(assertDirectRepository(act), repositoryAccessMode{ +func directRepositoryWriteAction(act func(ctx context.Context, rep repo.DirectRepositoryWriter) error) func(ctx *kingpin.ParseContext) error { + return maybeRepositoryAction(assertDirectRepository(func(ctx context.Context, rep repo.DirectRepository) error { + return repo.DirectWriteSession(ctx, rep, repo.WriteSessionOptions{ + Purpose: "directRepositoryWriteAction", + }, func(dw repo.DirectRepositoryWriter) error { return act(ctx, dw) }) + }), repositoryAccessMode{ mustBeConnected: true, disableMaintenance: true, }) } -func optionalRepositoryAction(act func(ctx context.Context, rep repo.Repository) error) func(ctx *kingpin.ParseContext) error { - return maybeRepositoryAction(act, repositoryAccessMode{ - mustBeConnected: false, +func directRepositoryReadAction(act func(ctx context.Context, rep repo.DirectRepository) error) func(ctx *kingpin.ParseContext) error { + return maybeRepositoryAction(assertDirectRepository(func(ctx context.Context, rep repo.DirectRepository) error { + return act(ctx, rep) + }), repositoryAccessMode{ + mustBeConnected: true, + disableMaintenance: true, }) } -func repositoryReaderAction(act func(ctx context.Context, rep repo.Reader) error) func(ctx *kingpin.ParseContext) error { +func repositoryReaderAction(act func(ctx context.Context, rep repo.Repository) error) func(ctx *kingpin.ParseContext) error { return maybeRepositoryAction(func(ctx context.Context, rep repo.Repository) error { return act(ctx, rep) }, repositoryAccessMode{ @@ -123,9 +124,13 @@ func repositoryReaderAction(act func(ctx context.Context, rep repo.Reader) error }) } -func repositoryWriterAction(act func(ctx context.Context, rep repo.Writer) error) func(ctx *kingpin.ParseContext) error { +func repositoryWriterAction(act func(ctx context.Context, rep repo.RepositoryWriter) error) func(ctx *kingpin.ParseContext) error { return maybeRepositoryAction(func(ctx context.Context, rep repo.Repository) error { - return act(ctx, rep) + return repo.WriteSession(ctx, rep, repo.WriteSessionOptions{ + Purpose: "repositoryWriterAction", + }, func(w repo.RepositoryWriter) error { + return act(ctx, w) + }) }, repositoryAccessMode{ mustBeConnected: true, }) @@ -192,7 +197,7 @@ func maybeRepositoryAction(act func(ctx context.Context, rep repo.Repository) er } } -func maybeRunMaintenance(ctx context.Context, rep repo.Writer) error { +func maybeRunMaintenance(ctx context.Context, rep repo.Repository) error { if !*enableAutomaticMaintenance { return nil } @@ -201,11 +206,17 @@ func maybeRunMaintenance(ctx context.Context, rep repo.Writer) error { return nil } - err := snapshotmaintenance.Run(ctx, rep, maintenance.ModeAuto, false) - if err == nil { + dr, ok := rep.(repo.DirectRepository) + if !ok { return nil } + err := repo.DirectWriteSession(ctx, dr, repo.WriteSessionOptions{ + Purpose: "maybeRunMaintenance", + }, func(w repo.DirectRepositoryWriter) error { + return snapshotmaintenance.Run(ctx, w, maintenance.ModeAuto, false) + }) + var noe maintenance.NotOwnedError if errors.As(err, &noe) { diff --git a/cli/auto_upgrade.go b/cli/auto_upgrade.go index 3dc7f3ad5..d166a566c 100644 --- a/cli/auto_upgrade.go +++ b/cli/auto_upgrade.go @@ -9,29 +9,32 @@ "github.com/kopia/kopia/repo/maintenance" ) -func maybeAutoUpgradeRepository(ctx context.Context, r repo.Writer) { +func maybeAutoUpgradeRepository(ctx context.Context, r repo.Repository) error { if r == nil { - return + return nil } - mr, ok := r.(maintenance.MaintainableRepository) - if !ok { - return + has, err := maintenance.HasParams(ctx, r) + if err != nil { + return errors.Wrap(err, "error looking for maintenance parameters") } - has, err := maintenance.HasParams(ctx, mr) - if err == nil && !has { - log(ctx).Noticef("Setting default maintenance parameters...") - - if err := setDefaultMaintenanceParameters(ctx, mr); err != nil { - log(ctx).Warningf("unable to set default maintenance parameters: %v", err) - } + if has { + return nil } + + log(ctx).Noticef("Setting default maintenance parameters...") + + return repo.WriteSession(ctx, r, repo.WriteSessionOptions{ + Purpose: "setDefaultMaintenanceParameters", + }, func(w repo.RepositoryWriter) error { + return setDefaultMaintenanceParameters(ctx, w) + }) } -func setDefaultMaintenanceParameters(ctx context.Context, rep maintenance.MaintainableRepository) error { +func setDefaultMaintenanceParameters(ctx context.Context, rep repo.RepositoryWriter) error { p := maintenance.DefaultParams() - p.Owner = rep.Username() + "@" + rep.Hostname() + p.Owner = rep.ClientOptions().UsernameAtHost() if err := maintenance.SetParams(ctx, rep, &p); err != nil { return errors.Wrap(err, "unable to set maintenance params") diff --git a/cli/command_blob_delete.go b/cli/command_blob_delete.go index ace52d855..79cbd4dc2 100644 --- a/cli/command_blob_delete.go +++ b/cli/command_blob_delete.go @@ -14,11 +14,11 @@ blobDeleteBlobIDs = blobDeleteCommand.Arg("blobIDs", "Blob IDs").Required().Strings() ) -func runDeleteBlobs(ctx context.Context, rep *repo.DirectRepository) error { +func runDeleteBlobs(ctx context.Context, rep repo.DirectRepositoryWriter) error { advancedCommand(ctx) for _, b := range *blobDeleteBlobIDs { - err := rep.Blobs.DeleteBlob(ctx, blob.ID(b)) + err := rep.BlobStorage().DeleteBlob(ctx, blob.ID(b)) if err != nil { return errors.Wrapf(err, "error deleting %v", b) } @@ -28,5 +28,5 @@ func runDeleteBlobs(ctx context.Context, rep *repo.DirectRepository) error { } func init() { - blobDeleteCommand.Action(directRepositoryAction(runDeleteBlobs)) + blobDeleteCommand.Action(directRepositoryWriteAction(runDeleteBlobs)) } diff --git a/cli/command_blob_gc.go b/cli/command_blob_gc.go index 69885da37..5b03d201d 100644 --- a/cli/command_blob_gc.go +++ b/cli/command_blob_gc.go @@ -19,7 +19,7 @@ blobGarbageCollectPrefix = blobGarbageCollectCommand.Flag("prefix", "Only GC blobs with given prefix").String() ) -func runBlobGarbageCollectCommand(ctx context.Context, rep *repo.DirectRepository) error { +func runBlobGarbageCollectCommand(ctx context.Context, rep repo.DirectRepositoryWriter) error { advancedCommand(ctx) opts := maintenance.DeleteUnreferencedBlobsOptions{ @@ -43,5 +43,5 @@ func runBlobGarbageCollectCommand(ctx context.Context, rep *repo.DirectRepositor } func init() { - blobGarbageCollectCommand.Action(directRepositoryAction(runBlobGarbageCollectCommand)) + blobGarbageCollectCommand.Action(directRepositoryWriteAction(runBlobGarbageCollectCommand)) } diff --git a/cli/command_blob_list.go b/cli/command_blob_list.go index 87f900803..f3a2c0b7d 100644 --- a/cli/command_blob_list.go +++ b/cli/command_blob_list.go @@ -15,8 +15,8 @@ blobListMaxSize = blobListCommand.Flag("max-size", "Maximum size").Int64() ) -func runBlobList(ctx context.Context, rep *repo.DirectRepository) error { - return rep.Blobs.ListBlobs(ctx, blob.ID(*blobListPrefix), func(b blob.Metadata) error { +func runBlobList(ctx context.Context, rep repo.DirectRepository) error { + return rep.BlobReader().ListBlobs(ctx, blob.ID(*blobListPrefix), func(b blob.Metadata) error { if *blobListMaxSize != 0 && b.Length > *blobListMaxSize { return nil } diff --git a/cli/command_blob_show.go b/cli/command_blob_show.go index 3f03284b3..b753f94cd 100644 --- a/cli/command_blob_show.go +++ b/cli/command_blob_show.go @@ -20,7 +20,7 @@ blobShowIDs = blobShowCommand.Arg("blobID", "Blob IDs").Required().Strings() ) -func runBlobShow(ctx context.Context, rep *repo.DirectRepository) error { +func runBlobShow(ctx context.Context, rep repo.DirectRepository) error { for _, blobID := range *blobShowIDs { if err := maybeDecryptBlob(ctx, os.Stdout, rep, blob.ID(blobID)); err != nil { return errors.Wrap(err, "error presenting blob") @@ -30,14 +30,14 @@ func runBlobShow(ctx context.Context, rep *repo.DirectRepository) error { return nil } -func maybeDecryptBlob(ctx context.Context, w io.Writer, rep *repo.DirectRepository, blobID blob.ID) error { +func maybeDecryptBlob(ctx context.Context, w io.Writer, rep repo.DirectRepository, blobID blob.ID) error { var ( d []byte err error ) if *blobShowDecrypt && canDecryptBlob(blobID) { - d, err = rep.Content.DecryptBlob(ctx, blobID) + d, err = rep.IndexBlobReader().DecryptBlob(ctx, blobID) if isJSONBlob(blobID) && err == nil { var b bytes.Buffer @@ -49,7 +49,7 @@ func maybeDecryptBlob(ctx context.Context, w io.Writer, rep *repo.DirectReposito d = b.Bytes() } } else { - d, err = rep.Blobs.GetBlob(ctx, blobID, 0, -1) + d, err = rep.BlobReader().GetBlob(ctx, blobID, 0, -1) } if err != nil { diff --git a/cli/command_blob_stats.go b/cli/command_blob_stats.go index 56f8d4b6c..43c8d55b4 100644 --- a/cli/command_blob_stats.go +++ b/cli/command_blob_stats.go @@ -18,7 +18,7 @@ blobStatsPrefix = blobStatsCommand.Flag("prefix", "Blob name prefix").String() ) -func runBlobStatsCommand(ctx context.Context, rep *repo.DirectRepository) error { +func runBlobStatsCommand(ctx context.Context, rep repo.DirectRepository) error { var sizeThreshold int64 = 10 countMap := map[int64]int{} @@ -34,7 +34,7 @@ func runBlobStatsCommand(ctx context.Context, rep *repo.DirectRepository) error var totalSize, count int64 - if err := rep.Blobs.ListBlobs( + if err := rep.BlobReader().ListBlobs( ctx, blob.ID(*blobStatsPrefix), func(b blob.Metadata) error { diff --git a/cli/command_cache_clear.go b/cli/command_cache_clear.go index 704089aac..6a330bcae 100644 --- a/cli/command_cache_clear.go +++ b/cli/command_cache_clear.go @@ -12,8 +12,8 @@ var cacheClearCommand = cacheCommands.Command("clear", "Clears the cache") -func runCacheClearCommand(ctx context.Context, rep *repo.DirectRepository) error { - if d := rep.Cache.CacheDirectory; d != "" { +func runCacheClearCommand(ctx context.Context, rep repo.DirectRepository) error { + if d := rep.CachingOptions().CacheDirectory; d != "" { log(ctx).Infof("Clearing cache directory: %v.", d) // close repository before removing cache @@ -41,5 +41,5 @@ func runCacheClearCommand(ctx context.Context, rep *repo.DirectRepository) error } func init() { - cacheClearCommand.Action(directRepositoryAction(runCacheClearCommand)) + cacheClearCommand.Action(directRepositoryReadAction(runCacheClearCommand)) } diff --git a/cli/command_cache_info.go b/cli/command_cache_info.go index 63c5ee9ea..a9aa9b427 100644 --- a/cli/command_cache_info.go +++ b/cli/command_cache_info.go @@ -17,20 +17,20 @@ cacheInfoPathOnly = cacheInfoCommand.Flag("path", "Only display cache path").Bool() ) -func runCacheInfoCommand(ctx context.Context, rep *repo.DirectRepository) error { +func runCacheInfoCommand(ctx context.Context, rep repo.DirectRepository) error { if *cacheInfoPathOnly { - fmt.Println(rep.Cache.CacheDirectory) + fmt.Println(rep.CachingOptions().CacheDirectory) return nil } - entries, err := ioutil.ReadDir(rep.Cache.CacheDirectory) + entries, err := ioutil.ReadDir(rep.CachingOptions().CacheDirectory) if err != nil { return errors.Wrap(err, "unable to scan cache directory") } path2Limit := map[string]int64{ - "contents": rep.Cache.MaxCacheSizeBytes, - "metadata": rep.Cache.MaxMetadataCacheSizeBytes, + "contents": rep.CachingOptions().MaxCacheSizeBytes, + "metadata": rep.CachingOptions().MaxMetadataCacheSizeBytes, } for _, ent := range entries { @@ -38,7 +38,7 @@ func runCacheInfoCommand(ctx context.Context, rep *repo.DirectRepository) error continue } - subdir := filepath.Join(rep.Cache.CacheDirectory, ent.Name()) + subdir := filepath.Join(rep.CachingOptions().CacheDirectory, ent.Name()) fileCount, totalFileSize, err := scanCacheDir(subdir) if err != nil { diff --git a/cli/command_cache_set.go b/cli/command_cache_set.go index c0132fae4..9278bc221 100644 --- a/cli/command_cache_set.go +++ b/cli/command_cache_set.go @@ -18,8 +18,8 @@ cacheSetMaxListCacheDuration = cacheSetParamsCommand.Flag("max-list-cache-duration", "Duration of index cache").Default("-1ns").Duration() ) -func runCacheSetCommand(ctx context.Context, rep *repo.DirectRepository) error { - opts := rep.Cache.CloneOrDefault() +func runCacheSetCommand(ctx context.Context, rep repo.DirectRepositoryWriter) error { + opts := rep.CachingOptions().CloneOrDefault() changed := 0 @@ -53,9 +53,9 @@ func runCacheSetCommand(ctx context.Context, rep *repo.DirectRepository) error { return errors.Errorf("no changes") } - return rep.SetCachingConfig(ctx, opts) + return rep.SetCachingOptions(ctx, opts) } func init() { - cacheSetParamsCommand.Action(directRepositoryAction(runCacheSetCommand)) + cacheSetParamsCommand.Action(directRepositoryWriteAction(runCacheSetCommand)) } diff --git a/cli/command_cache_sync.go b/cli/command_cache_sync.go index f621e22a1..4c6b0f223 100644 --- a/cli/command_cache_sync.go +++ b/cli/command_cache_sync.go @@ -8,10 +8,10 @@ var cacheSyncCommand = cacheCommands.Command("sync", "Synchronizes the metadata cache with blobs in storage") -func runCacheSyncCommand(ctx context.Context, rep *repo.DirectRepository) error { - return rep.Content.SyncMetadataCache(ctx) +func runCacheSyncCommand(ctx context.Context, rep repo.DirectRepositoryWriter) error { + return rep.ContentManager().SyncMetadataCache(ctx) } func init() { - cacheSyncCommand.Action(directRepositoryAction(runCacheSyncCommand)) + cacheSyncCommand.Action(directRepositoryWriteAction(runCacheSyncCommand)) } diff --git a/cli/command_content_list.go b/cli/command_content_list.go index 8272e6f32..e740c1a3c 100644 --- a/cli/command_content_list.go +++ b/cli/command_content_list.go @@ -20,10 +20,10 @@ contentListHuman = contentListCommand.Flag("human", "Human-readable output").Short('h').Bool() ) -func runContentListCommand(ctx context.Context, rep *repo.DirectRepository) error { +func runContentListCommand(ctx context.Context, rep repo.DirectRepository) error { var totalSize stats.CountSum - err := rep.Content.IterateContents( + err := rep.ContentReader().IterateContents( ctx, content.IterateOptions{ Range: contentIDRange(), diff --git a/cli/command_content_rewrite.go b/cli/command_content_rewrite.go index 701a595c9..b1ce08b86 100644 --- a/cli/command_content_rewrite.go +++ b/cli/command_content_rewrite.go @@ -21,7 +21,7 @@ contentRewriteMinAge = contentRewriteCommand.Flag("min-age", "Only rewrite contents above given age").Default("1h").Duration() ) -func runContentRewriteCommand(ctx context.Context, rep *repo.DirectRepository) error { +func runContentRewriteCommand(ctx context.Context, rep repo.DirectRepositoryWriter) error { advancedCommand(ctx) return maintenance.RewriteContents(ctx, rep, &maintenance.RewriteContentsOptions{ @@ -46,6 +46,6 @@ func toContentIDs(s []string) []content.ID { } func init() { - contentRewriteCommand.Action(directRepositoryAction(runContentRewriteCommand)) + contentRewriteCommand.Action(directRepositoryWriteAction(runContentRewriteCommand)) setupContentIDRangeFlags(contentRewriteCommand) } diff --git a/cli/command_content_rm.go b/cli/command_content_rm.go index 34d25aefb..d6104af6e 100644 --- a/cli/command_content_rm.go +++ b/cli/command_content_rm.go @@ -14,11 +14,11 @@ contentRemoveIDs = contentRemoveCommand.Arg("id", "IDs of content to remove").Required().Strings() ) -func runContentRemoveCommand(ctx context.Context, rep *repo.DirectRepository) error { +func runContentRemoveCommand(ctx context.Context, rep repo.DirectRepositoryWriter) error { advancedCommand(ctx) for _, contentID := range toContentIDs(*contentRemoveIDs) { - if err := rep.Content.DeleteContent(ctx, contentID); err != nil { + if err := rep.ContentManager().DeleteContent(ctx, contentID); err != nil { return errors.Wrapf(err, "error deleting content %v", contentID) } } @@ -28,5 +28,5 @@ func runContentRemoveCommand(ctx context.Context, rep *repo.DirectRepository) er func init() { setupShowCommand(contentRemoveCommand) - contentRemoveCommand.Action(directRepositoryAction(runContentRemoveCommand)) + contentRemoveCommand.Action(directRepositoryWriteAction(runContentRemoveCommand)) } diff --git a/cli/command_content_show.go b/cli/command_content_show.go index 6b66e140c..dc6c4da14 100644 --- a/cli/command_content_show.go +++ b/cli/command_content_show.go @@ -16,7 +16,7 @@ contentShowIDs = contentShowCommand.Arg("id", "IDs of contents to show").Required().Strings() ) -func runContentShowCommand(ctx context.Context, rep *repo.DirectRepository) error { +func runContentShowCommand(ctx context.Context, rep repo.DirectRepository) error { for _, contentID := range toContentIDs(*contentShowIDs) { if err := contentShow(ctx, rep, contentID); err != nil { return err @@ -26,8 +26,8 @@ func runContentShowCommand(ctx context.Context, rep *repo.DirectRepository) erro return nil } -func contentShow(ctx context.Context, r *repo.DirectRepository, contentID content.ID) error { - data, err := r.Content.GetContent(ctx, contentID) +func contentShow(ctx context.Context, r repo.DirectRepository, contentID content.ID) error { + data, err := r.ContentReader().GetContent(ctx, contentID) if err != nil { return errors.Wrapf(err, "error getting content %v", contentID) } diff --git a/cli/command_content_stats.go b/cli/command_content_stats.go index 40be45777..f19b403df 100644 --- a/cli/command_content_stats.go +++ b/cli/command_content_stats.go @@ -17,7 +17,7 @@ contentStatsRaw = contentStatsCommand.Flag("raw", "Raw numbers").Short('r').Bool() ) -func runContentStatsCommand(ctx context.Context, rep *repo.DirectRepository) error { +func runContentStatsCommand(ctx context.Context, rep repo.DirectRepository) error { var sizeThreshold uint32 = 10 countMap := map[uint32]int{} @@ -33,7 +33,7 @@ func runContentStatsCommand(ctx context.Context, rep *repo.DirectRepository) err var totalSize, count int64 - if err := rep.Content.IterateContents( + if err := rep.ContentReader().IterateContents( ctx, content.IterateOptions{ Range: contentIDRange(), diff --git a/cli/command_content_verify.go b/cli/command_content_verify.go index c8131419f..3b66b33c5 100644 --- a/cli/command_content_verify.go +++ b/cli/command_content_verify.go @@ -19,12 +19,12 @@ contentVerifyIncludeDeleted = contentVerifyCommand.Flag("include-deleted", "Include deleted contents").Bool() ) -func readBlobMap(ctx context.Context, rep *repo.DirectRepository) (map[blob.ID]blob.Metadata, error) { +func readBlobMap(ctx context.Context, br blob.Reader) (map[blob.ID]blob.Metadata, error) { blobMap := map[blob.ID]blob.Metadata{} log(ctx).Infof("Listing blobs...") - if err := rep.Blobs.ListBlobs(ctx, "", func(bm blob.Metadata) error { + if err := br.ListBlobs(ctx, "", func(bm blob.Metadata) error { blobMap[bm.BlobID] = bm if len(blobMap)%10000 == 0 { log(ctx).Infof(" %v blobs...", len(blobMap)) @@ -39,11 +39,11 @@ func readBlobMap(ctx context.Context, rep *repo.DirectRepository) (map[blob.ID]b return blobMap, nil } -func runContentVerifyCommand(ctx context.Context, rep *repo.DirectRepository) error { +func runContentVerifyCommand(ctx context.Context, rep repo.DirectRepository) error { blobMap := map[blob.ID]blob.Metadata{} if !*contentVerifyFull { - m, err := readBlobMap(ctx, rep) + m, err := readBlobMap(ctx, rep.BlobReader()) if err != nil { return err } @@ -55,12 +55,12 @@ func runContentVerifyCommand(ctx context.Context, rep *repo.DirectRepository) er log(ctx).Infof("Verifying all contents...") - err := rep.Content.IterateContents(ctx, content.IterateOptions{ + err := rep.ContentReader().IterateContents(ctx, content.IterateOptions{ Range: contentIDRange(), Parallel: *contentVerifyParallel, IncludeDeleted: *contentVerifyIncludeDeleted, }, func(ci content.Info) error { - if err := contentVerify(ctx, rep, &ci, blobMap); err != nil { + if err := contentVerify(ctx, rep.ContentReader(), &ci, blobMap); err != nil { log(ctx).Errorf("error %v", err) atomic.AddInt32(&errorCount, 1) } else { @@ -86,9 +86,9 @@ func runContentVerifyCommand(ctx context.Context, rep *repo.DirectRepository) er return errors.Errorf("encountered %v errors", errorCount) } -func contentVerify(ctx context.Context, r *repo.DirectRepository, ci *content.Info, blobMap map[blob.ID]blob.Metadata) error { +func contentVerify(ctx context.Context, r content.Reader, ci *content.Info, blobMap map[blob.ID]blob.Metadata) error { if *contentVerifyFull { - if _, err := r.Content.GetContent(ctx, ci.ID); err != nil { + if _, err := r.GetContent(ctx, ci.ID); err != nil { return errors.Wrapf(err, "content %v is invalid", ci.ID) } diff --git a/cli/command_diff.go b/cli/command_diff.go index e406013bf..b222007a1 100644 --- a/cli/command_diff.go +++ b/cli/command_diff.go @@ -21,7 +21,7 @@ diffCommandCommand = diffCommand.Flag("diff-command", "Displays differences between two repository objects (files or directories)").Default(defaultDiffCommand()).Envar("KOPIA_DIFF").String() ) -func runDiffCommand(ctx context.Context, rep repo.Reader) error { +func runDiffCommand(ctx context.Context, rep repo.Repository) error { ent1, err := snapshotfs.FilesystemEntryFromIDWithPath(ctx, rep, *diffFirstObjectPath, false) if err != nil { return errors.Wrapf(err, "error getting filesystem entry for %v", *diffFirstObjectPath) diff --git a/cli/command_index_inspect.go b/cli/command_index_inspect.go index 316eb9232..3edde0dce 100644 --- a/cli/command_index_inspect.go +++ b/cli/command_index_inspect.go @@ -15,7 +15,7 @@ indexInspectBlobIDs = indexInspectCommand.Arg("blobs", "Names of index blobs to inspect").Strings() ) -func runInspectIndexAction(ctx context.Context, rep *repo.DirectRepository) error { +func runInspectIndexAction(ctx context.Context, rep repo.DirectRepository) error { for _, indexBlobID := range *indexInspectBlobIDs { if err := inspectSingleIndexBlob(ctx, rep, blob.ID(indexBlobID)); err != nil { return err @@ -38,13 +38,13 @@ func dumpIndexBlobEntries(bm blob.Metadata, entries []content.Info) { } } -func inspectSingleIndexBlob(ctx context.Context, rep *repo.DirectRepository, blobID blob.ID) error { - bm, err := rep.Blobs.GetMetadata(ctx, blobID) +func inspectSingleIndexBlob(ctx context.Context, rep repo.DirectRepository, blobID blob.ID) error { + bm, err := rep.BlobReader().GetMetadata(ctx, blobID) if err != nil { return errors.Wrapf(err, "unable to get metadata for %v", blobID) } - entries, err := rep.Content.ParseIndexBlob(ctx, blobID) + entries, err := rep.IndexBlobReader().ParseIndexBlob(ctx, blobID) if err != nil { return errors.Wrapf(err, "unable to recover index from %v", blobID) } diff --git a/cli/command_index_list.go b/cli/command_index_list.go index c4025b119..283a0b68d 100644 --- a/cli/command_index_list.go +++ b/cli/command_index_list.go @@ -17,8 +17,8 @@ blockIndexListSort = blockIndexListCommand.Flag("sort", "Index blob sort order").Default("time").Enum("time", "size", "name") ) -func runListBlockIndexesAction(ctx context.Context, rep *repo.DirectRepository) error { - blks, err := rep.Content.IndexBlobs(ctx, *blockIndexListIncludeSuperseded) +func runListBlockIndexesAction(ctx context.Context, rep repo.DirectRepository) error { + blks, err := rep.IndexBlobReader().IndexBlobs(ctx, *blockIndexListIncludeSuperseded) if err != nil { return errors.Wrap(err, "error listing index blobs") } diff --git a/cli/command_index_optimize.go b/cli/command_index_optimize.go index 43107ae8e..c7cee8ae9 100644 --- a/cli/command_index_optimize.go +++ b/cli/command_index_optimize.go @@ -16,7 +16,7 @@ optimizeAllIndexes = optimizeCommand.Flag("all", "Optimize all indexes, even those above maximum size.").Bool() ) -func runOptimizeCommand(ctx context.Context, rep *repo.DirectRepository) error { +func runOptimizeCommand(ctx context.Context, rep repo.DirectRepositoryWriter) error { advancedCommand(ctx) opt := content.CompactOptions{ @@ -29,9 +29,9 @@ func runOptimizeCommand(ctx context.Context, rep *repo.DirectRepository) error { opt.DropDeletedBefore = clock.Now().Add(-age) } - return rep.Content.CompactIndexes(ctx, opt) + return rep.ContentManager().CompactIndexes(ctx, opt) } func init() { - optimizeCommand.Action(directRepositoryAction(runOptimizeCommand)) + optimizeCommand.Action(directRepositoryWriteAction(runOptimizeCommand)) } diff --git a/cli/command_index_recover.go b/cli/command_index_recover.go index d1eaecb5d..c8502da9d 100644 --- a/cli/command_index_recover.go +++ b/cli/command_index_recover.go @@ -16,7 +16,7 @@ blockIndexRecoverCommit = blockIndexRecoverCommand.Flag("commit", "Commit recovered content").Bool() ) -func runRecoverBlockIndexesAction(ctx context.Context, rep *repo.DirectRepository) error { +func runRecoverBlockIndexesAction(ctx context.Context, rep repo.DirectRepositoryWriter) error { advancedCommand(ctx) var totalCount int @@ -36,7 +36,7 @@ func runRecoverBlockIndexesAction(ctx context.Context, rep *repo.DirectRepositor if len(*blockIndexRecoverBlobIDs) == 0 { for _, prefix := range content.PackBlobIDPrefixes { - err := rep.Blobs.ListBlobs(ctx, prefix, func(bm blob.Metadata) error { + err := rep.BlobStorage().ListBlobs(ctx, prefix, func(bm blob.Metadata) error { recoverIndexFromSinglePackFile(ctx, rep, bm.BlobID, bm.Length, &totalCount) return nil }) @@ -53,8 +53,8 @@ func runRecoverBlockIndexesAction(ctx context.Context, rep *repo.DirectRepositor return nil } -func recoverIndexFromSinglePackFile(ctx context.Context, rep *repo.DirectRepository, blobID blob.ID, length int64, totalCount *int) { - recovered, err := rep.Content.RecoverIndexFromPackBlob(ctx, blobID, length, *blockIndexRecoverCommit) +func recoverIndexFromSinglePackFile(ctx context.Context, rep repo.DirectRepositoryWriter, blobID blob.ID, length int64, totalCount *int) { + recovered, err := rep.ContentManager().RecoverIndexFromPackBlob(ctx, blobID, length, *blockIndexRecoverCommit) if err != nil { log(ctx).Warningf("unable to recover index from %v: %v", blobID, err) return @@ -65,5 +65,5 @@ func recoverIndexFromSinglePackFile(ctx context.Context, rep *repo.DirectReposit } func init() { - blockIndexRecoverCommand.Action(directRepositoryAction(runRecoverBlockIndexesAction)) + blockIndexRecoverCommand.Action(directRepositoryWriteAction(runRecoverBlockIndexesAction)) } diff --git a/cli/command_ls.go b/cli/command_ls.go index 5d5fa4f1c..ed7289e14 100644 --- a/cli/command_ls.go +++ b/cli/command_ls.go @@ -24,7 +24,7 @@ lsCommandPath = lsCommand.Arg("object-path", "Path").Required().String() ) -func runLSCommand(ctx context.Context, rep repo.Reader) error { +func runLSCommand(ctx context.Context, rep repo.Repository) error { dir, err := snapshotfs.FilesystemDirectoryFromIDWithPath(ctx, rep, *lsCommandPath, false) if err != nil { return errors.Wrap(err, "unable to get filesystem directory entry") diff --git a/cli/command_maintenance_info.go b/cli/command_maintenance_info.go index cd1d24eed..ee7384c77 100644 --- a/cli/command_maintenance_info.go +++ b/cli/command_maintenance_info.go @@ -18,7 +18,7 @@ maintenanceInfoJSON = maintenanceInfoCommand.Flag("json", "Show raw JSON data").Short('j').Bool() ) -func runMaintenanceInfoCommand(ctx context.Context, rep *repo.DirectRepository) error { +func runMaintenanceInfoCommand(ctx context.Context, rep repo.DirectRepository) error { p, err := maintenance.GetParams(ctx, rep) if err != nil { return errors.Wrap(err, "unable to get maintenance params") @@ -68,7 +68,7 @@ func runMaintenanceInfoCommand(ctx context.Context, rep *repo.DirectRepository) return nil } -func displayCycleInfo(c *maintenance.CycleParams, t time.Time, rep *repo.DirectRepository) { +func displayCycleInfo(c *maintenance.CycleParams, t time.Time, rep repo.DirectRepository) { printStdout(" scheduled: %v\n", c.Enabled) if c.Enabled { diff --git a/cli/command_maintenance_run.go b/cli/command_maintenance_run.go index e58495aec..84c775cff 100644 --- a/cli/command_maintenance_run.go +++ b/cli/command_maintenance_run.go @@ -14,7 +14,7 @@ maintenanceRunForce = maintenanceRunCommand.Flag("force", "Run maintenance even if not owned (unsafe)").Hidden().Bool() ) -func runMaintenanceCommand(ctx context.Context, rep *repo.DirectRepository) error { +func runMaintenanceCommand(ctx context.Context, rep repo.DirectRepositoryWriter) error { mode := maintenance.ModeQuick if *maintenanceRunFull { mode = maintenance.ModeFull @@ -24,5 +24,5 @@ func runMaintenanceCommand(ctx context.Context, rep *repo.DirectRepository) erro } func init() { - maintenanceRunCommand.Action(directRepositoryAction(runMaintenanceCommand)) + maintenanceRunCommand.Action(directRepositoryWriteAction(runMaintenanceCommand)) } diff --git a/cli/command_maintenance_set.go b/cli/command_maintenance_set.go index 98a47e005..314177987 100644 --- a/cli/command_maintenance_set.go +++ b/cli/command_maintenance_set.go @@ -25,10 +25,10 @@ maintenanceSetPauseFull = maintenanceSetCommand.Flag("pause-full", "Pause full maintenance for a specified duration").DurationList() ) -func setMaintenanceOwnerFromFlags(ctx context.Context, p *maintenance.Params, rep *repo.DirectRepository, changed *bool) { +func setMaintenanceOwnerFromFlags(ctx context.Context, p *maintenance.Params, rep repo.DirectRepositoryWriter, changed *bool) { if v := *maintenanceSetOwner; v != "" { if v == "me" { - p.Owner = rep.Username() + "@" + rep.Hostname() + p.Owner = rep.ClientOptions().UsernameAtHost() } else { p.Owner = v } @@ -63,7 +63,7 @@ func setMaintenanceEnabledAndIntervalFromFlags(ctx context.Context, c *maintenan } } -func runMaintenanceSetParams(ctx context.Context, rep *repo.DirectRepository) error { +func runMaintenanceSetParams(ctx context.Context, rep repo.DirectRepositoryWriter) error { p, err := maintenance.GetParams(ctx, rep) if err != nil { return errors.Wrap(err, "unable to get current parameters") @@ -116,5 +116,5 @@ func runMaintenanceSetParams(ctx context.Context, rep *repo.DirectRepository) er } func init() { - maintenanceSetCommand.Action(directRepositoryAction(runMaintenanceSetParams)) + maintenanceSetCommand.Action(directRepositoryWriteAction(runMaintenanceSetParams)) } diff --git a/cli/command_manifest_ls.go b/cli/command_manifest_ls.go index 1fbc0e55c..5243b2ced 100644 --- a/cli/command_manifest_ls.go +++ b/cli/command_manifest_ls.go @@ -21,7 +21,7 @@ func init() { manifestListCommand.Action(repositoryReaderAction(listManifestItems)) } -func listManifestItems(ctx context.Context, rep repo.Reader) error { +func listManifestItems(ctx context.Context, rep repo.Repository) error { filter := map[string]string{} for _, kv := range *manifestListFilter { diff --git a/cli/command_manifest_rm.go b/cli/command_manifest_rm.go index 93043c215..e056a77da 100644 --- a/cli/command_manifest_rm.go +++ b/cli/command_manifest_rm.go @@ -13,7 +13,7 @@ manifestRemoveItems = manifestRemoveCommand.Arg("item", "Items to remove").Required().Strings() ) -func runManifestRemoveCommand(ctx context.Context, rep repo.Writer) error { +func runManifestRemoveCommand(ctx context.Context, rep repo.RepositoryWriter) error { advancedCommand(ctx) for _, it := range toManifestIDs(*manifestRemoveItems) { diff --git a/cli/command_manifest_show.go b/cli/command_manifest_show.go index 5cff75524..61efbc47f 100644 --- a/cli/command_manifest_show.go +++ b/cli/command_manifest_show.go @@ -30,7 +30,7 @@ func toManifestIDs(s []string) []manifest.ID { return result } -func showManifestItems(ctx context.Context, rep repo.Reader) error { +func showManifestItems(ctx context.Context, rep repo.Repository) error { for _, it := range toManifestIDs(*manifestShowItems) { var b json.RawMessage diff --git a/cli/command_mount.go b/cli/command_mount.go index 0d6d93303..828b7a811 100644 --- a/cli/command_mount.go +++ b/cli/command_mount.go @@ -26,7 +26,7 @@ mountFuseAllowNonEmptyMount = mountCommand.Flag("fuse-allow-non-empty-mount", "Allows the mounting over a non-empty directory. The files in it will be shadowed by the freshly created mount.").Bool() ) -func runMountCommand(ctx context.Context, rep repo.Reader) error { +func runMountCommand(ctx context.Context, rep repo.Repository) error { var entry fs.Directory if *mountObjectID == "all" { diff --git a/cli/command_policy.go b/cli/command_policy.go index 672573653..5f045b5f5 100644 --- a/cli/command_policy.go +++ b/cli/command_policy.go @@ -11,7 +11,7 @@ "github.com/kopia/kopia/snapshot/policy" ) -func policyTargets(ctx context.Context, rep repo.Reader, globalFlag *bool, targetsFlag *[]string) ([]snapshot.SourceInfo, error) { +func policyTargets(ctx context.Context, rep repo.Repository, globalFlag *bool, targetsFlag *[]string) ([]snapshot.SourceInfo, error) { if *globalFlag == (len(*targetsFlag) > 0) { return nil, errors.New("must pass either '--global' or a list of path targets") } diff --git a/cli/command_policy_edit.go b/cli/command_policy_edit.go index 9037ebfe2..964bb454a 100644 --- a/cli/command_policy_edit.go +++ b/cli/command_policy_edit.go @@ -59,7 +59,7 @@ func init() { policyEditCommand.Action(repositoryWriterAction(editPolicy)) } -func editPolicy(ctx context.Context, rep repo.Writer) error { +func editPolicy(ctx context.Context, rep repo.RepositoryWriter) error { targets, err := policyTargets(ctx, rep, policyEditGlobal, policyEditTargets) if err != nil { return err diff --git a/cli/command_policy_ls.go b/cli/command_policy_ls.go index 20e31c538..7ed000ed1 100644 --- a/cli/command_policy_ls.go +++ b/cli/command_policy_ls.go @@ -17,7 +17,7 @@ func init() { policyListCommand.Action(repositoryReaderAction(listPolicies)) } -func listPolicies(ctx context.Context, rep repo.Reader) error { +func listPolicies(ctx context.Context, rep repo.Repository) error { policies, err := policy.ListPolicies(ctx, rep) if err != nil { return errors.Wrap(err, "error listing policies") diff --git a/cli/command_policy_remove.go b/cli/command_policy_remove.go index 58c0ce7c4..86d4b98cb 100644 --- a/cli/command_policy_remove.go +++ b/cli/command_policy_remove.go @@ -20,7 +20,7 @@ func init() { policyRemoveCommand.Action(repositoryWriterAction(removePolicy)) } -func removePolicy(ctx context.Context, rep repo.Writer) error { +func removePolicy(ctx context.Context, rep repo.RepositoryWriter) error { targets, err := policyTargets(ctx, rep, policyRemoveGlobal, policyRemoveTargets) if err != nil { return err diff --git a/cli/command_policy_set.go b/cli/command_policy_set.go index 72d6062b6..83bb7e40e 100644 --- a/cli/command_policy_set.go +++ b/cli/command_policy_set.go @@ -30,7 +30,7 @@ func init() { policySetCommand.Action(repositoryWriterAction(setPolicy)) } -func setPolicy(ctx context.Context, rep repo.Writer) error { +func setPolicy(ctx context.Context, rep repo.RepositoryWriter) error { targets, err := policyTargets(ctx, rep, policySetGlobal, policySetTargets) if err != nil { return err diff --git a/cli/command_policy_show.go b/cli/command_policy_show.go index ab96982b8..6557231fb 100644 --- a/cli/command_policy_show.go +++ b/cli/command_policy_show.go @@ -24,7 +24,7 @@ func init() { policyShowCommand.Action(repositoryReaderAction(showPolicy)) } -func showPolicy(ctx context.Context, rep repo.Reader) error { +func showPolicy(ctx context.Context, rep repo.Repository) error { targets, err := policyTargets(ctx, rep, policyShowGlobal, policyShowTargets) if err != nil { return err diff --git a/cli/command_repository_create.go b/cli/command_repository_create.go index 3946f9204..d0f4d7aa9 100644 --- a/cli/command_repository_create.go +++ b/cli/command_repository_create.go @@ -10,7 +10,6 @@ "github.com/kopia/kopia/repo/content" "github.com/kopia/kopia/repo/encryption" "github.com/kopia/kopia/repo/hashing" - "github.com/kopia/kopia/repo/maintenance" "github.com/kopia/kopia/repo/object" "github.com/kopia/kopia/repo/splitter" "github.com/kopia/kopia/snapshot/policy" @@ -98,14 +97,22 @@ func populateRepository(ctx context.Context, password string) error { } defer rep.Close(ctx) //nolint:errcheck - if err := policy.SetPolicy(ctx, rep, policy.GlobalPolicySourceInfo, policy.DefaultPolicy); err != nil { - return errors.Wrap(err, "unable to set global policy") - } + return repo.WriteSession(ctx, rep, repo.WriteSessionOptions{ + Purpose: "populateRepository", + }, func(w repo.RepositoryWriter) error { + if err := policy.SetPolicy(ctx, w, policy.GlobalPolicySourceInfo, policy.DefaultPolicy); err != nil { + return errors.Wrap(err, "unable to set global policy") + } - printPolicy(policy.DefaultPolicy, nil) - printStdout("\n") - printStdout("To change the policy use:\n kopia policy set --global \n") - printStdout("or\n kopia policy set \n") + printPolicy(policy.DefaultPolicy, nil) + printStdout("\n") + printStdout("To change the policy use:\n kopia policy set --global \n") + printStdout("or\n kopia policy set \n") - return setDefaultMaintenanceParameters(ctx, rep.(maintenance.MaintainableRepository)) + if err := setDefaultMaintenanceParameters(ctx, w); err != nil { + return errors.Wrap(err, "unable to set maintenance parameters") + } + + return nil + }) } diff --git a/cli/command_repository_set_client.go b/cli/command_repository_set_client.go index c24559235..4867e98dd 100644 --- a/cli/command_repository_set_client.go +++ b/cli/command_repository_set_client.go @@ -18,7 +18,7 @@ repoClientOptionsHostname = repoClientOptionsCommand.Flag("hostname", "Change hostname").Strings() ) -func runRepoClientOptionsCommand(ctx context.Context, rep repo.Reader) error { +func runRepoClientOptionsCommand(ctx context.Context, rep repo.Repository) error { var anyChange bool opt := rep.ClientOptions() diff --git a/cli/command_repository_status.go b/cli/command_repository_status.go index b367cc9ca..495986c87 100644 --- a/cli/command_repository_status.go +++ b/cli/command_repository_status.go @@ -21,7 +21,7 @@ statusReconnectTokenIncludePassword = statusCommand.Flag("reconnect-token-with-password", "Include password in reconnect token").Short('s').Bool() ) -func runStatusCommand(ctx context.Context, rep repo.Reader) error { +func runStatusCommand(ctx context.Context, rep repo.Repository) error { fmt.Printf("Config file: %v\n", repositoryConfigFileName()) fmt.Println() fmt.Printf("Description: %v\n", rep.ClientOptions().Description) @@ -29,14 +29,14 @@ func runStatusCommand(ctx context.Context, rep repo.Reader) error { fmt.Printf("Username: %v\n", rep.ClientOptions().Username) fmt.Printf("Read-only: %v\n", rep.ClientOptions().ReadOnly) - dr, ok := rep.(*repo.DirectRepository) + dr, ok := rep.(repo.DirectRepository) if !ok { return nil } fmt.Println() - ci := dr.Blobs.ConnectionInfo() + ci := dr.BlobReader().ConnectionInfo() fmt.Printf("Storage type: %v\n", ci.Type) if cjson, err := json.MarshalIndent(scrubber.ScrubSensitiveData(reflect.ValueOf(ci.Config)).Interface(), " ", " "); err == nil { @@ -44,12 +44,12 @@ func runStatusCommand(ctx context.Context, rep repo.Reader) error { } fmt.Println() - fmt.Printf("Unique ID: %x\n", dr.UniqueID) - fmt.Printf("Hash: %v\n", dr.Content.Format().Hash) - fmt.Printf("Encryption: %v\n", dr.Content.Format().Encryption) - fmt.Printf("Splitter: %v\n", dr.Objects.Format.Splitter) - fmt.Printf("Format version: %v\n", dr.Content.Format().Version) - fmt.Printf("Max pack length: %v\n", units.BytesStringBase2(int64(dr.Content.Format().MaxPackSize))) + fmt.Printf("Unique ID: %x\n", dr.UniqueID()) + fmt.Printf("Hash: %v\n", dr.ContentReader().ContentFormat().Hash) + fmt.Printf("Encryption: %v\n", dr.ContentReader().ContentFormat().Encryption) + fmt.Printf("Splitter: %v\n", dr.ObjectFormat().Splitter) + fmt.Printf("Format version: %v\n", dr.ContentReader().ContentFormat().Version) + fmt.Printf("Max pack length: %v\n", units.BytesStringBase2(int64(dr.ContentReader().ContentFormat().MaxPackSize))) if !*statusReconnectToken { return nil diff --git a/cli/command_repository_sync.go b/cli/command_repository_sync.go index 8ea36f5c3..a1ecb0aac 100644 --- a/cli/command_repository_sync.go +++ b/cli/command_repository_sync.go @@ -31,7 +31,7 @@ const syncProgressInterval = 300 * time.Millisecond -func runSyncWithStorage(ctx context.Context, src, dst blob.Storage) error { +func runSyncWithStorage(ctx context.Context, src blob.Reader, dst blob.Storage) error { var () log(ctx).Infof("Synchronizing repositories:") @@ -178,7 +178,7 @@ func finishSyncProcess() { printStderr("\r%v\n", lastSyncProgress) } -func runSyncBlobs(ctx context.Context, src, dst blob.Storage, blobsToCopy, blobsToDelete []blob.Metadata, totalBytes int64) error { +func runSyncBlobs(ctx context.Context, src blob.Reader, dst blob.Storage, blobsToCopy, blobsToDelete []blob.Metadata, totalBytes int64) error { eg, ctx := errgroup.WithContext(ctx) copyCh := sliceToChannel(ctx, blobsToCopy) deleteCh := sliceToChannel(ctx, blobsToDelete) @@ -259,7 +259,7 @@ func sliceToChannel(ctx context.Context, md []blob.Metadata) chan blob.Metadata var setTimeUnsupportedOnce sync.Once -func syncCopyBlob(ctx context.Context, m blob.Metadata, src, dst blob.Storage) error { +func syncCopyBlob(ctx context.Context, m blob.Metadata, src blob.Reader, dst blob.Storage) error { data, err := src.GetBlob(ctx, m.BlobID, 0, -1) if err != nil { if errors.Is(err, blob.ErrBlobNotFound) { @@ -299,7 +299,7 @@ func syncDeleteBlob(ctx context.Context, m blob.Metadata, dst blob.Storage) erro return errors.Wrap(err, "error deleting blob") } -func ensureRepositoriesHaveSameFormatBlob(ctx context.Context, src, dst blob.Storage) error { +func ensureRepositoriesHaveSameFormatBlob(ctx context.Context, src blob.Reader, dst blob.Storage) error { srcData, err := src.GetBlob(ctx, repo.FormatBlobID, 0, -1) if err != nil { return errors.Wrap(err, "error reading format blob") diff --git a/cli/command_repository_upgrade.go b/cli/command_repository_upgrade.go index 300cb14a3..d28a6cc2f 100644 --- a/cli/command_repository_upgrade.go +++ b/cli/command_repository_upgrade.go @@ -8,10 +8,10 @@ var upgradeCommand = repositoryCommands.Command("upgrade", "Upgrade repository format.") -func runUpgradeCommand(ctx context.Context, rep *repo.DirectRepository) error { +func runUpgradeCommand(ctx context.Context, rep repo.DirectRepositoryWriter) error { return rep.Upgrade(ctx) } func init() { - upgradeCommand.Action(directRepositoryAction(runUpgradeCommand)) + upgradeCommand.Action(directRepositoryWriteAction(runUpgradeCommand)) } diff --git a/cli/command_restore.go b/cli/command_restore.go index e00221025..ea0716396 100644 --- a/cli/command_restore.go +++ b/cli/command_restore.go @@ -169,7 +169,7 @@ func printRestoreStats(ctx context.Context, st restore.Stats) { log(ctx).Infof("Restored %v files, %v directories and %v symbolic links (%v)\n", st.RestoredFileCount, st.RestoredDirCount, st.RestoredSymlinkCount, units.BytesStringBase10(st.RestoredTotalFileSize)) } -func runRestoreCommand(ctx context.Context, rep repo.Reader) error { +func runRestoreCommand(ctx context.Context, rep repo.Repository) error { output, err := restoreOutput(ctx) if err != nil { return errors.Wrap(err, "unable to initialize output") diff --git a/cli/command_server_start.go b/cli/command_server_start.go index a77effe20..b7002c2dd 100644 --- a/cli/command_server_start.go +++ b/cli/command_server_start.go @@ -38,7 +38,10 @@ func init() { setupConnectOptions(serverStartCommand) - serverStartCommand.Action(optionalRepositoryAction(runServer)) + serverStartCommand.Action(maybeRepositoryAction(runServer, repositoryAccessMode{ + mustBeConnected: false, + disableMaintenance: true, // server closes the repository so maintenance can't run. + })) } func runServer(ctx context.Context, rep repo.Repository) error { @@ -51,7 +54,9 @@ func runServer(ctx context.Context, rep repo.Repository) error { return errors.Wrap(err, "unable to initialize server") } - maybeAutoUpgradeRepository(ctx, rep) + if err = maybeAutoUpgradeRepository(ctx, rep); err != nil { + return errors.Wrap(err, "error upgrading repository") + } if err = srv.SetRepository(ctx, rep); err != nil { return errors.Wrap(err, "error connecting to repository") diff --git a/cli/command_session_list.go b/cli/command_session_list.go index 0f6e4d4f9..141a9ef1f 100644 --- a/cli/command_session_list.go +++ b/cli/command_session_list.go @@ -10,8 +10,8 @@ var sessionListCommand = sessionCommands.Command("list", "List sessions").Alias("ls") -func runSessionList(ctx context.Context, rep *repo.DirectRepository) error { - sessions, err := rep.ListActiveSessions(ctx) +func runSessionList(ctx context.Context, rep repo.DirectRepository) error { + sessions, err := rep.ContentReader().ListActiveSessions(ctx) if err != nil { return errors.Wrap(err, "error listing sessions") } @@ -24,5 +24,5 @@ func runSessionList(ctx context.Context, rep *repo.DirectRepository) error { } func init() { - sessionListCommand.Action(directRepositoryAction(runSessionList)) + sessionListCommand.Action(directRepositoryReadAction(runSessionList)) } diff --git a/cli/command_show.go b/cli/command_show.go index f127b6a36..11ac51892 100644 --- a/cli/command_show.go +++ b/cli/command_show.go @@ -16,7 +16,7 @@ catCommandPath = catCommand.Arg("object-path", "Path").Required().String() ) -func runCatCommand(ctx context.Context, rep repo.Reader) error { +func runCatCommand(ctx context.Context, rep repo.Repository) error { oid, err := snapshotfs.ParseObjectIDWithPath(ctx, rep, *catCommandPath) if err != nil { return errors.Wrapf(err, "unable to parse ID: %v", *catCommandPath) diff --git a/cli/command_snapshot_copy_move_history.go b/cli/command_snapshot_copy_move_history.go index 57460d348..cdec8dd9b 100644 --- a/cli/command_snapshot_copy_move_history.go +++ b/cli/command_snapshot_copy_move_history.go @@ -71,7 +71,7 @@ func registerSnapshotCopyFlags(cmd *kingpin.CmdClause) { // user1@host1:/path1 @host2 copy to user1@host2:/path1 // user1@host1:/path1 user2@host2 copy to user2@host2:/path1 // user1@host1:/path1 user2@host2:/path2 copy snapshots from single path. -func runSnapshotCopyCommand(ctx context.Context, rep repo.Writer, isMoveCommand bool) error { +func runSnapshotCopyCommand(ctx context.Context, rep repo.RepositoryWriter, isMoveCommand bool) error { si, di, err := getCopySourceAndDestination(rep) if err != nil { return err @@ -150,7 +150,7 @@ func getCopySnapshotAction(isMoveCommand bool) string { return action } -func getCopySourceAndDestination(rep repo.Writer) (si, di snapshot.SourceInfo, err error) { +func getCopySourceAndDestination(rep repo.RepositoryWriter) (si, di snapshot.SourceInfo, err error) { si, err = snapshot.ParseSourceInfo(snapshotCopyOrMoveSource, rep.ClientOptions().Hostname, rep.ClientOptions().Username) if err != nil { return si, di, errors.Wrap(err, "invalid source") @@ -231,12 +231,12 @@ func getCopyDestination(source, overrides snapshot.SourceInfo) snapshot.SourceIn func init() { registerSnapshotCopyFlags(snapshotCopyCommand) - snapshotCopyCommand.Action(repositoryWriterAction(func(ctx context.Context, rep repo.Writer) error { + snapshotCopyCommand.Action(repositoryWriterAction(func(ctx context.Context, rep repo.RepositoryWriter) error { return runSnapshotCopyCommand(ctx, rep, false) })) registerSnapshotCopyFlags(snapshotMoveCommand) - snapshotMoveCommand.Action(repositoryWriterAction(func(ctx context.Context, rep repo.Writer) error { + snapshotMoveCommand.Action(repositoryWriterAction(func(ctx context.Context, rep repo.RepositoryWriter) error { return runSnapshotCopyCommand(ctx, rep, true) })) } diff --git a/cli/command_snapshot_create.go b/cli/command_snapshot_create.go index 1edeef030..a4aca2534 100644 --- a/cli/command_snapshot_create.go +++ b/cli/command_snapshot_create.go @@ -36,10 +36,12 @@ snapshotCreateForceDisableActions = snapshotCreateCommand.Flag("force-disable-actions", "Disable snapshot actions even if globally enabled on this client").Hidden().Bool() ) -func runSnapshotCommand(ctx context.Context, rep repo.Writer) error { +func runSnapshotCommand(ctx context.Context, rep repo.RepositoryWriter) error { sources := *snapshotCreateSources - maybeAutoUpgradeRepository(ctx, rep) + if err := maybeAutoUpgradeRepository(ctx, rep); err != nil { + return errors.Wrap(err, "error upgrading repository") + } if *snapshotCreateAll { local, err := getLocalBackupPaths(ctx, rep) @@ -113,7 +115,7 @@ func validateStartEndTime(st, et string) error { return nil } -func setupUploader(rep repo.Writer) *snapshotfs.Uploader { +func setupUploader(rep repo.RepositoryWriter) *snapshotfs.Uploader { u := snapshotfs.NewUploader(rep) u.MaxUploadBytes = *snapshotCreateCheckpointUploadLimitMB << 20 //nolint:gomnd @@ -152,7 +154,7 @@ func startTimeAfterEndTime(startTime, endTime time.Time) bool { startTime.After(endTime) } -func snapshotSingleSource(ctx context.Context, rep repo.Writer, u *snapshotfs.Uploader, sourceInfo snapshot.SourceInfo) error { +func snapshotSingleSource(ctx context.Context, rep repo.RepositoryWriter, u *snapshotfs.Uploader, sourceInfo snapshot.SourceInfo) error { log(ctx).Infof("Snapshotting %v ...", sourceInfo) t0 := clock.Now() @@ -235,7 +237,7 @@ func snapshotSingleSource(ctx context.Context, rep repo.Writer, u *snapshotfs.Up // findPreviousSnapshotManifest returns the list of previous snapshots for a given source, including // last complete snapshot and possibly some number of incomplete snapshots following it. -func findPreviousSnapshotManifest(ctx context.Context, rep repo.Reader, sourceInfo snapshot.SourceInfo, noLaterThan *time.Time) ([]*snapshot.Manifest, error) { +func findPreviousSnapshotManifest(ctx context.Context, rep repo.Repository, sourceInfo snapshot.SourceInfo, noLaterThan *time.Time) ([]*snapshot.Manifest, error) { man, err := snapshot.ListSnapshots(ctx, rep, sourceInfo) if err != nil { return nil, errors.Wrap(err, "error listing previous snapshots") @@ -277,7 +279,7 @@ func findPreviousSnapshotManifest(ctx context.Context, rep repo.Reader, sourceIn return result, nil } -func getLocalBackupPaths(ctx context.Context, rep repo.Reader) ([]string, error) { +func getLocalBackupPaths(ctx context.Context, rep repo.Repository) ([]string, error) { log(ctx).Debugf("Looking for previous backups of '%v@%v'...", rep.ClientOptions().Hostname, rep.ClientOptions().Username) sources, err := snapshot.ListSources(ctx, rep) diff --git a/cli/command_snapshot_delete.go b/cli/command_snapshot_delete.go index ceef1f75d..bfb377f95 100644 --- a/cli/command_snapshot_delete.go +++ b/cli/command_snapshot_delete.go @@ -18,7 +18,7 @@ snapshotDeleteConfirm = snapshotDeleteCommand.Flag("delete", "Confirm deletion").Bool() ) -func runDeleteCommand(ctx context.Context, rep repo.Writer) error { +func runDeleteCommand(ctx context.Context, rep repo.RepositoryWriter) error { for _, id := range *snapshotDeleteIDs { m, err := snapshot.LoadSnapshot(ctx, rep, manifest.ID(id)) if err == nil { @@ -36,7 +36,7 @@ func runDeleteCommand(ctx context.Context, rep repo.Writer) error { return nil } -func deleteSnapshot(ctx context.Context, rep repo.Writer, m *snapshot.Manifest) error { +func deleteSnapshot(ctx context.Context, rep repo.RepositoryWriter, m *snapshot.Manifest) error { desc := fmt.Sprintf("snapshot %v of %v at %v", m.ID, m.Source, formatTimestamp(m.StartTime)) if !*snapshotDeleteConfirm { @@ -49,7 +49,7 @@ func deleteSnapshot(ctx context.Context, rep repo.Writer, m *snapshot.Manifest) return rep.DeleteManifest(ctx, m.ID) } -func deleteSnapshotsByRootObjectID(ctx context.Context, rep repo.Writer, rootID object.ID) error { +func deleteSnapshotsByRootObjectID(ctx context.Context, rep repo.RepositoryWriter, rootID object.ID) error { manifests, err := snapshot.FindSnapshotsByRootObjectID(ctx, rep, rootID) if err != nil { return errors.Wrapf(err, "unable to find snapshots by root %v", rootID) diff --git a/cli/command_snapshot_estimate.go b/cli/command_snapshot_estimate.go index d74f6916f..6c4c45e25 100644 --- a/cli/command_snapshot_estimate.go +++ b/cli/command_snapshot_estimate.go @@ -64,7 +64,7 @@ func makeBuckets() buckets { } } -func runSnapshotEstimateCommand(ctx context.Context, rep repo.Reader) error { +func runSnapshotEstimateCommand(ctx context.Context, rep repo.Repository) error { path, err := filepath.Abs(*snapshotEstimateSource) if err != nil { return errors.Errorf("invalid path: '%s': %s", path, err) diff --git a/cli/command_snapshot_expire.go b/cli/command_snapshot_expire.go index 8cc46fc5a..94c07a66e 100644 --- a/cli/command_snapshot_expire.go +++ b/cli/command_snapshot_expire.go @@ -19,7 +19,7 @@ snapshotExpireDelete = snapshotExpireCommand.Flag("delete", "Whether to actually delete snapshots").Bool() ) -func getSnapshotSourcesToExpire(ctx context.Context, rep repo.Reader) ([]snapshot.SourceInfo, error) { +func getSnapshotSourcesToExpire(ctx context.Context, rep repo.Repository) ([]snapshot.SourceInfo, error) { if *snapshotExpireAll { return snapshot.ListSources(ctx, rep) } @@ -38,7 +38,7 @@ func getSnapshotSourcesToExpire(ctx context.Context, rep repo.Reader) ([]snapsho return result, nil } -func runExpireCommand(ctx context.Context, rep repo.Writer) error { +func runExpireCommand(ctx context.Context, rep repo.RepositoryWriter) error { sources, err := getSnapshotSourcesToExpire(ctx, rep) if err != nil { return err diff --git a/cli/command_snapshot_gc.go b/cli/command_snapshot_gc.go index 0db27aac8..f5ad2dd0b 100644 --- a/cli/command_snapshot_gc.go +++ b/cli/command_snapshot_gc.go @@ -17,7 +17,7 @@ snapshotGCDelete = snapshotGCCommand.Flag("delete", "Delete unreferenced contents").Bool() ) -func runSnapshotGCCommand(ctx context.Context, rep *repo.DirectRepository) error { +func runSnapshotGCCommand(ctx context.Context, rep repo.DirectRepositoryWriter) error { st, err := snapshotgc.Run(ctx, rep, maintenance.SnapshotGCParams{ MinContentAge: *snapshotGCMinContentAge, }, *snapshotGCDelete) @@ -31,5 +31,5 @@ func runSnapshotGCCommand(ctx context.Context, rep *repo.DirectRepository) error } func init() { - snapshotGCCommand.Action(directRepositoryAction(runSnapshotGCCommand)) + snapshotGCCommand.Action(directRepositoryWriteAction(runSnapshotGCCommand)) } diff --git a/cli/command_snapshot_list.go b/cli/command_snapshot_list.go index 5f9c92f14..c0a00d417 100644 --- a/cli/command_snapshot_list.go +++ b/cli/command_snapshot_list.go @@ -35,7 +35,7 @@ maxResultsPerPath = snapshotListCommand.Flag("max-results", "Maximum number of entries per source.").Short('n').Int() ) -func findSnapshotsForSource(ctx context.Context, rep repo.Reader, sourceInfo snapshot.SourceInfo) (manifestIDs []manifest.ID, relPath string, err error) { +func findSnapshotsForSource(ctx context.Context, rep repo.Repository, sourceInfo snapshot.SourceInfo) (manifestIDs []manifest.ID, relPath string, err error) { for len(sourceInfo.Path) > 0 { list, err := snapshot.ListSnapshotManifests(ctx, rep, &sourceInfo) if err != nil { @@ -65,7 +65,7 @@ func findSnapshotsForSource(ctx context.Context, rep repo.Reader, sourceInfo sna return nil, "", nil } -func findManifestIDs(ctx context.Context, rep repo.Reader, source string) ([]manifest.ID, string, error) { +func findManifestIDs(ctx context.Context, rep repo.Repository, source string) ([]manifest.ID, string, error) { if source == "" { man, err := snapshot.ListSnapshotManifests(ctx, rep, nil) return man, "", errors.Wrap(err, "error listing all snapshot manifests") @@ -84,7 +84,7 @@ func findManifestIDs(ctx context.Context, rep repo.Reader, source string) ([]man return manifestIDs, relPath, err } -func runSnapshotsCommand(ctx context.Context, rep repo.Reader) error { +func runSnapshotsCommand(ctx context.Context, rep repo.Repository) error { manifestIDs, relPath, err := findManifestIDs(ctx, rep, *snapshotListPath) if err != nil { return err @@ -98,7 +98,7 @@ func runSnapshotsCommand(ctx context.Context, rep repo.Reader) error { return outputManifestGroups(ctx, rep, manifests, strings.Split(relPath, "/")) } -func shouldOutputSnapshotSource(rep repo.Reader, src snapshot.SourceInfo) bool { +func shouldOutputSnapshotSource(rep repo.Repository, src snapshot.SourceInfo) bool { if *snapshotListShowAll { return true } @@ -112,7 +112,7 @@ func shouldOutputSnapshotSource(rep repo.Reader, src snapshot.SourceInfo) bool { return src.UserName == co.Username } -func outputManifestGroups(ctx context.Context, rep repo.Reader, manifests []*snapshot.Manifest, relPathParts []string) error { +func outputManifestGroups(ctx context.Context, rep repo.Repository, manifests []*snapshot.Manifest, relPathParts []string) error { separator := "" var anyOutput bool @@ -148,7 +148,7 @@ func outputManifestGroups(ctx context.Context, rep repo.Reader, manifests []*sna return nil } -func outputManifestFromSingleSource(ctx context.Context, rep repo.Reader, manifests []*snapshot.Manifest, parts []string) error { +func outputManifestFromSingleSource(ctx context.Context, rep repo.Repository, manifests []*snapshot.Manifest, parts []string) error { var ( count int lastTotalFileSize int64 diff --git a/cli/command_snapshot_migrate.go b/cli/command_snapshot_migrate.go index 9fc95fe3d..894f6c7cd 100644 --- a/cli/command_snapshot_migrate.go +++ b/cli/command_snapshot_migrate.go @@ -26,7 +26,7 @@ migrateParallel = migrateCommand.Flag("parallel", "Number of sources to migrate in parallel").Default("1").Int() ) -func runMigrateCommand(ctx context.Context, destRepo repo.Writer) error { +func runMigrateCommand(ctx context.Context, destRepo repo.RepositoryWriter) error { sourceRepo, err := openSourceRepo(ctx) if err != nil { return errors.Wrap(err, "can't open source repository") @@ -114,7 +114,7 @@ func runMigrateCommand(ctx context.Context, destRepo repo.Writer) error { return nil } -func openSourceRepo(ctx context.Context) (repo.Reader, error) { +func openSourceRepo(ctx context.Context) (repo.Repository, error) { pass, ok := repo.GetPersistedPassword(ctx, *migrateSourceConfig) if !ok { var err error @@ -132,7 +132,7 @@ func openSourceRepo(ctx context.Context) (repo.Reader, error) { return sourceRepo, nil } -func migratePoliciesForSources(ctx context.Context, sourceRepo repo.Reader, destRepo repo.Writer, sources []snapshot.SourceInfo) error { +func migratePoliciesForSources(ctx context.Context, sourceRepo repo.Repository, destRepo repo.RepositoryWriter, sources []snapshot.SourceInfo) error { for _, si := range sources { if err := migrateSinglePolicy(ctx, sourceRepo, destRepo, si); err != nil { return errors.Wrapf(err, "unable to migrate policy for %v", si) @@ -142,7 +142,7 @@ func migratePoliciesForSources(ctx context.Context, sourceRepo repo.Reader, dest return nil } -func migrateAllPolicies(ctx context.Context, sourceRepo repo.Reader, destRepo repo.Writer) error { +func migrateAllPolicies(ctx context.Context, sourceRepo repo.Repository, destRepo repo.RepositoryWriter) error { policies, err := policy.ListPolicies(ctx, sourceRepo) if err != nil { return errors.Wrap(err, "unable to list source policies") @@ -157,7 +157,7 @@ func migrateAllPolicies(ctx context.Context, sourceRepo repo.Reader, destRepo re return nil } -func migrateSinglePolicy(ctx context.Context, sourceRepo repo.Reader, destRepo repo.Writer, si snapshot.SourceInfo) error { +func migrateSinglePolicy(ctx context.Context, sourceRepo repo.Repository, destRepo repo.RepositoryWriter, si snapshot.SourceInfo) error { pol, err := policy.GetDefinedPolicy(ctx, sourceRepo, si) if errors.Is(err, policy.ErrPolicyNotFound) { return nil @@ -183,7 +183,7 @@ func migrateSinglePolicy(ctx context.Context, sourceRepo repo.Reader, destRepo r return policy.SetPolicy(ctx, destRepo, si, pol) } -func findPreviousSnapshotManifestWithStartTime(ctx context.Context, rep repo.Reader, sourceInfo snapshot.SourceInfo, startTime time.Time) (*snapshot.Manifest, error) { +func findPreviousSnapshotManifestWithStartTime(ctx context.Context, rep repo.Repository, sourceInfo snapshot.SourceInfo, startTime time.Time) (*snapshot.Manifest, error) { previous, err := snapshot.ListSnapshots(ctx, rep, sourceInfo) if err != nil { return nil, errors.Wrap(err, "error listing previous snapshots") @@ -198,7 +198,7 @@ func findPreviousSnapshotManifestWithStartTime(ctx context.Context, rep repo.Rea return nil, nil } -func migrateSingleSource(ctx context.Context, uploader *snapshotfs.Uploader, sourceRepo repo.Reader, destRepo repo.Writer, s snapshot.SourceInfo) error { +func migrateSingleSource(ctx context.Context, uploader *snapshotfs.Uploader, sourceRepo repo.Repository, destRepo repo.RepositoryWriter, s snapshot.SourceInfo) error { manifests, err := snapshot.ListSnapshotManifests(ctx, sourceRepo, &s) if err != nil { return errors.Wrapf(err, "error listing snapshot manifests for %v", s) @@ -226,7 +226,7 @@ func migrateSingleSource(ctx context.Context, uploader *snapshotfs.Uploader, sou return nil } -func migrateSingleSourceSnapshot(ctx context.Context, uploader *snapshotfs.Uploader, sourceRepo repo.Reader, destRepo repo.Writer, s snapshot.SourceInfo, m *snapshot.Manifest) error { +func migrateSingleSourceSnapshot(ctx context.Context, uploader *snapshotfs.Uploader, sourceRepo repo.Repository, destRepo repo.RepositoryWriter, s snapshot.SourceInfo, m *snapshot.Manifest) error { if m.IncompleteReason != "" { log(ctx).Debugf("ignoring incomplete %v at %v", s, formatTimestamp(m.StartTime)) return nil @@ -282,7 +282,7 @@ func filterSnapshotsToMigrate(s []*snapshot.Manifest) []*snapshot.Manifest { return s } -func getSourcesToMigrate(ctx context.Context, rep repo.Reader) ([]snapshot.SourceInfo, error) { +func getSourcesToMigrate(ctx context.Context, rep repo.Repository) ([]snapshot.SourceInfo, error) { if len(*migrateSources) > 0 { var result []snapshot.SourceInfo diff --git a/cli/command_snapshot_verify.go b/cli/command_snapshot_verify.go index 96904dcab..8f4356ddc 100644 --- a/cli/command_snapshot_verify.go +++ b/cli/command_snapshot_verify.go @@ -34,7 +34,7 @@ ) type verifier struct { - rep repo.Reader + rep repo.Repository workQueue *parallelwork.Queue startTime time.Time @@ -155,9 +155,9 @@ func (v *verifier) doVerifyObject(ctx context.Context, oid object.ID, path strin v.reportError(ctx, path, errors.Wrapf(err, "error verifying %v", oid)) } - if dr, ok := v.rep.(*repo.DirectRepository); v.blobMap != nil && ok { + if dr, ok := v.rep.(repo.DirectRepository); v.blobMap != nil && ok { for _, cid := range contentIDs { - ci, err := dr.Content.ContentInfo(ctx, cid) + ci, err := dr.ContentReader().ContentInfo(ctx, cid) if err != nil { v.reportError(ctx, path, errors.Wrapf(err, "error verifying content %v: %v", cid, err)) continue @@ -197,7 +197,7 @@ func (v *verifier) readEntireObject(ctx context.Context, oid object.ID, path str return errors.Wrap(err, "unable to read data") } -func runVerifyCommand(ctx context.Context, rep repo.Reader) error { +func runVerifyCommand(ctx context.Context, rep repo.Repository) error { if *verifyCommandAllSources { log(ctx).Noticef("DEPRECATED: --all-sources flag has no effect and is the default when no sources are provided.") } @@ -209,8 +209,8 @@ func runVerifyCommand(ctx context.Context, rep repo.Reader) error { seen: map[object.ID]bool{}, } - if dr, ok := rep.(*repo.DirectRepository); ok { - blobMap, err := readBlobMap(ctx, dr) + if dr, ok := rep.(repo.DirectRepository); ok { + blobMap, err := readBlobMap(ctx, dr.BlobReader()) if err != nil { return err } @@ -234,7 +234,7 @@ func runVerifyCommand(ctx context.Context, rep repo.Reader) error { return errors.Errorf("encountered %v errors", len(v.errors)) } -func enqueueRootsToVerify(ctx context.Context, v *verifier, rep repo.Reader) error { +func enqueueRootsToVerify(ctx context.Context, v *verifier, rep repo.Repository) error { manifests, err := loadSourceManifests(ctx, rep, *verifyCommandSources) if err != nil { return err @@ -275,7 +275,7 @@ func enqueueRootsToVerify(ctx context.Context, v *verifier, rep repo.Reader) err return nil } -func loadSourceManifests(ctx context.Context, rep repo.Reader, sources []string) ([]*snapshot.Manifest, error) { +func loadSourceManifests(ctx context.Context, rep repo.Repository, sources []string) ([]*snapshot.Manifest, error) { var manifestIDs []manifest.ID if len(sources)+len(*verifyCommandDirObjectIDs)+len(*verifyCommandFileObjectIDs) == 0 { diff --git a/cli/storage_providers.go b/cli/storage_providers.go index b762476ca..0919c8cf7 100644 --- a/cli/storage_providers.go +++ b/cli/storage_providers.go @@ -73,11 +73,11 @@ func RegisterStorageConnectFlags( return errors.Wrap(err, "open repository") } - dr, ok := rep.(*repo.DirectRepository) + dr, ok := rep.(repo.DirectRepository) if !ok { return errors.Errorf("sync only supports directly-connected repositories") } - return runSyncWithStorage(ctx, dr.BlobStorage(), st) + return runSyncWithStorage(ctx, dr.BlobReader(), st) }) } diff --git a/examples/upload_download/main.go b/examples/upload_download/main.go deleted file mode 100644 index 524c181b1..000000000 --- a/examples/upload_download/main.go +++ /dev/null @@ -1,30 +0,0 @@ -//+build !test - -// Command repository_api demonstrates the use of Kopia's Repository API. -package main - -import ( - "context" - "log" - "os" - - "github.com/kopia/kopia/repo" -) - -func main() { - ctx := context.Background() - - if err := setupRepositoryAndConnect(ctx, masterPassword); err != nil { - log.Printf("unable to set up repository: %v", err) - os.Exit(1) - } - - r, err := repo.Open(ctx, configFile, masterPassword, nil) - if err != nil { - log.Printf("unable to open repository: %v", err) - os.Exit(1) - } - defer r.Close(ctx) //nolint:errcheck - - uploadAndDownloadObjects(ctx, r) -} diff --git a/examples/upload_download/setup_repository.go b/examples/upload_download/setup_repository.go deleted file mode 100644 index cb27a353f..000000000 --- a/examples/upload_download/setup_repository.go +++ /dev/null @@ -1,59 +0,0 @@ -//+build !test - -package main - -import ( - "context" - "log" - "os" - - "github.com/pkg/errors" - - "github.com/kopia/kopia/repo" - "github.com/kopia/kopia/repo/blob/filesystem" - "github.com/kopia/kopia/repo/blob/logging" - "github.com/kopia/kopia/repo/content" -) - -const ( - masterPassword = "my-password$!@#!@" - storageDir = "/tmp/kopia-example/storage" - configFile = "/tmp/kopia-example/config" - cacheDirectory = "/tmp/kopia-example/cache" -) - -func setupRepositoryAndConnect(ctx context.Context, password string) error { - if err := os.MkdirAll(storageDir, 0o700); err != nil { - return errors.Wrap(err, "unable to create directory") - } - - st, err := filesystem.New(ctx, &filesystem.Options{ - Path: storageDir, - }) - if err != nil { - return errors.Wrap(err, "unable to connect to storage") - } - - // set up logging so we can see what's going on - st = logging.NewWrapper(st, log.Printf, "") - - // see if we already have the config file, if not connect. - if _, err := os.Stat(configFile); os.IsNotExist(err) { - // initialize repository - if err := repo.Initialize(ctx, st, &repo.NewRepositoryOptions{}, password); err != nil { - return errors.Wrap(err, "unable to initialize repository") - } - - // now establish connection to repository and create configuration file. - if err := repo.Connect(ctx, configFile, st, password, &repo.ConnectOptions{ - CachingOptions: content.CachingOptions{ - CacheDirectory: cacheDirectory, - MaxCacheSizeBytes: 100000000, - }, - }); err != nil { - return errors.Wrap(err, "unable to connect to repository") - } - } - - return nil -} diff --git a/examples/upload_download/upload_download_objects.go b/examples/upload_download/upload_download_objects.go deleted file mode 100644 index d2feb161d..000000000 --- a/examples/upload_download/upload_download_objects.go +++ /dev/null @@ -1,67 +0,0 @@ -//+build !test - -package main - -import ( - "context" - "crypto/rand" - "io/ioutil" - "log" - "os" - - "github.com/kopia/kopia/repo" - "github.com/kopia/kopia/repo/object" -) - -func uploadRandomObject(ctx context.Context, r repo.Repository, length int) (object.ID, error) { - w := r.NewObjectWriter(ctx, object.WriterOptions{}) - defer w.Close() //nolint:errcheck - - buf := make([]byte, 256*1024) - for length > 0 { - todo := length - if todo > len(buf) { - todo = len(buf) - } - rand.Read(buf[0:todo]) //nolint:errcheck - if _, err := w.Write(buf[0:todo]); err != nil { - return "", err - } - length -= todo - } - return w.Result() -} - -func downloadObject(ctx context.Context, r repo.Repository, oid object.ID) ([]byte, error) { - rd, err := r.OpenObject(ctx, oid) - if err != nil { - return nil, err - } - defer rd.Close() //nolint:errcheck - - return ioutil.ReadAll(rd) -} - -func uploadAndDownloadObjects(ctx context.Context, r repo.Repository) { - var oids []object.ID - - for size := 100; size < 100000000; size *= 2 { - log.Printf("uploading file with %v bytes", size) - oid, err := uploadRandomObject(ctx, r, size) - if err != nil { - log.Printf("unable to upload: %v", err) - os.Exit(1) - } - log.Printf("uploaded %v bytes as %v", size, oid) - oids = append(oids, oid) - } - - for _, oid := range oids { - log.Printf("downloading %q", oid) - b, err := downloadObject(ctx, r, oid) - if err != nil { - log.Printf("unable to read object: %v", err) - } - log.Printf("downloaded %v", len(b)) - } -} diff --git a/internal/repotesting/repotesting.go b/internal/repotesting/repotesting.go index d6efc6fb7..afffb9b05 100644 --- a/internal/repotesting/repotesting.go +++ b/internal/repotesting/repotesting.go @@ -20,7 +20,8 @@ // Environment encapsulates details of a test environment. type Environment struct { - Repository *repo.DirectRepository + Repository repo.Repository + RepositoryWriter repo.DirectRepositoryWriter configDir string storageDir string @@ -85,7 +86,12 @@ func (e *Environment) Setup(t *testing.T, opts ...Options) *Environment { t.Fatalf("can't open: %v", err) } - e.Repository = rep.(*repo.DirectRepository) + e.Repository = rep + + e.RepositoryWriter, err = rep.(repo.DirectRepository).NewDirectWriter(ctx, "test") + if err != nil { + t.Fatal(err) + } return e } @@ -94,7 +100,7 @@ func (e *Environment) Setup(t *testing.T, opts ...Options) *Environment { func (e *Environment) Close(ctx context.Context, t *testing.T) { t.Helper() - if err := e.Repository.Close(ctx); err != nil { + if err := e.RepositoryWriter.Close(ctx); err != nil { t.Fatalf("unable to close: %v", err) } @@ -118,33 +124,45 @@ func (e *Environment) configFile() string { func (e *Environment) MustReopen(t *testing.T, openOpts ...func(*repo.Options)) { t.Helper() - err := e.Repository.Close(testlogging.Context(t)) + ctx := testlogging.Context(t) + + err := e.RepositoryWriter.Close(ctx) if err != nil { t.Fatalf("close error: %v", err) } - rep, err := repo.Open(testlogging.Context(t), e.configFile(), masterPassword, repoOptions(openOpts)) + rep, err := repo.Open(ctx, e.configFile(), masterPassword, repoOptions(openOpts)) if err != nil { t.Fatalf("err: %v", err) } - e.Repository = rep.(*repo.DirectRepository) + e.RepositoryWriter, err = rep.(repo.DirectRepository).NewDirectWriter(ctx, "test") + if err != nil { + t.Fatalf("err: %v", err) + } } // MustOpenAnother opens another repository backend by the same storage. -func (e *Environment) MustOpenAnother(t *testing.T) repo.Repository { +func (e *Environment) MustOpenAnother(t *testing.T) repo.RepositoryWriter { t.Helper() - rep2, err := repo.Open(testlogging.Context(t), e.configFile(), masterPassword, &repo.Options{}) + ctx := testlogging.Context(t) + + rep2, err := repo.Open(ctx, e.configFile(), masterPassword, &repo.Options{}) if err != nil { t.Fatalf("err: %v", err) } t.Cleanup(func() { - rep2.Close(testlogging.Context(t)) + rep2.Close(ctx) }) - return rep2 + w, err := rep2.NewWriter(ctx, "test") + if err != nil { + t.Fatal(err) + } + + return w } // MustConnectOpenAnother opens another repository backend by the same storage, @@ -186,7 +204,7 @@ func (e *Environment) VerifyBlobCount(t *testing.T, want int) { var got int - _ = e.Repository.Blobs.ListBlobs(testlogging.Context(t), "", func(_ blob.Metadata) error { + _ = e.RepositoryWriter.BlobReader().ListBlobs(testlogging.Context(t), "", func(_ blob.Metadata) error { got++ return nil }) diff --git a/internal/repotesting/repotesting_test.go b/internal/repotesting/repotesting_test.go index 165854ba8..b18258c67 100644 --- a/internal/repotesting/repotesting_test.go +++ b/internal/repotesting/repotesting_test.go @@ -19,38 +19,41 @@ func TestTimeFuncWiring(t *testing.T) { ctx := testlogging.Context(t) defer env.Setup(t).Close(ctx, t) - env.Repository.Close(ctx) + env.RepositoryWriter.Close(ctx) ft := faketime.NewTimeAdvance(time.Date(2018, time.February, 6, 0, 0, 0, 0, time.UTC), 0) // Re open with injected time - rep, err := repo.Open(ctx, env.Repository.ConfigFile, masterPassword, &repo.Options{TimeNowFunc: ft.NowFunc()}) + rep, err := repo.Open(ctx, env.RepositoryWriter.ConfigFilename(), masterPassword, &repo.Options{TimeNowFunc: ft.NowFunc()}) if err != nil { t.Fatal("Failed to open repo:", err) } - r := rep.(*repo.DirectRepository) + r0 := rep.(repo.DirectRepository) - env.Repository = r + env.RepositoryWriter, err = r0.NewDirectWriter(ctx, "test") + if err != nil { + t.Fatal(err) + } // verify wiring for the repo layer - if got, want := r.Time(), ft.NowFunc()(); !got.Equal(want) { + if got, want := env.RepositoryWriter.Time(), ft.NowFunc()(); !got.Equal(want) { t.Errorf("times don't match, got %v, want %v", got, want) } - if want, got := ft.Advance(10*time.Minute), r.Time(); !got.Equal(want) { + if want, got := ft.Advance(10*time.Minute), env.RepositoryWriter.Time(); !got.Equal(want) { t.Errorf("times don't match, got %v, want %v", got, want) } // verify wiring for the content layer nt := ft.Advance(20 * time.Second) - cid, err := r.Content.WriteContent(ctx, []byte("foo"), "") + cid, err := env.RepositoryWriter.ContentManager().WriteContent(ctx, []byte("foo"), "") if err != nil { t.Fatal("failed to write content:", err) } - info, err := r.Content.ContentInfo(ctx, cid) + info, err := env.RepositoryWriter.ContentReader().ContentInfo(ctx, cid) if err != nil { t.Fatal("failed to get content info for", cid, err) } @@ -64,12 +67,12 @@ func TestTimeFuncWiring(t *testing.T) { labels := map[string]string{"l1": "v1", "l2": "v2", "type": "my-manifest"} - mid, err := r.Manifests.Put(ctx, labels, "manifest content") + mid, err := env.RepositoryWriter.PutManifest(ctx, labels, "manifest content") if err != nil { t.Fatal("failed to put manifest:", err) } - meta, err := r.Manifests.GetMetadata(ctx, mid) + meta, err := env.RepositoryWriter.GetManifest(ctx, mid, nil) if err != nil { t.Fatal("failed to get manifest metadata:", err) } @@ -85,7 +88,7 @@ func TestTimeFuncWiring(t *testing.T) { sourceDir.AddFile("f1", []byte{1, 2, 3}, defaultPermissions) nt = ft.Advance(1 * time.Hour) - u := snapshotfs.NewUploader(r) + u := snapshotfs.NewUploader(env.RepositoryWriter) policyTree := policy.BuildTree(nil, policy.DefaultPolicy) s1, err := u.Upload(ctx, sourceDir, policyTree, snapshot.SourceInfo{}) diff --git a/internal/server/api_content.go b/internal/server/api_content.go index 852edbd35..4c95dbde3 100644 --- a/internal/server/api_content.go +++ b/internal/server/api_content.go @@ -13,14 +13,14 @@ ) func (s *Server) handleContentGet(ctx context.Context, r *http.Request, body []byte) (interface{}, *apiError) { - dr, ok := s.rep.(*repo.DirectRepository) + dr, ok := s.rep.(repo.DirectRepository) if !ok { return nil, notFoundError("content not found") } cid := content.ID(mux.Vars(r)["contentID"]) - data, err := dr.Content.GetContent(ctx, cid) + data, err := dr.ContentReader().GetContent(ctx, cid) if errors.Is(err, content.ErrContentNotFound) { return nil, notFoundError("content not found") } @@ -29,14 +29,14 @@ func (s *Server) handleContentGet(ctx context.Context, r *http.Request, body []b } func (s *Server) handleContentInfo(ctx context.Context, r *http.Request, body []byte) (interface{}, *apiError) { - dr, ok := s.rep.(*repo.DirectRepository) + dr, ok := s.rep.(repo.DirectRepository) if !ok { return nil, notFoundError("content not found") } cid := content.ID(mux.Vars(r)["contentID"]) - ci, err := dr.Content.ContentInfo(ctx, cid) + ci, err := dr.ContentReader().ContentInfo(ctx, cid) switch { case err == nil: @@ -51,15 +51,15 @@ func (s *Server) handleContentInfo(ctx context.Context, r *http.Request, body [] } func (s *Server) handleContentPut(ctx context.Context, r *http.Request, data []byte) (interface{}, *apiError) { - dr, ok := s.rep.(*repo.DirectRepository) + dr, ok := s.rep.(repo.DirectRepositoryWriter) if !ok { - return nil, notFoundError("content not found") + return nil, repositoryNotWritableError() } cid := content.ID(mux.Vars(r)["contentID"]) prefix := cid.Prefix() - actualCID, err := dr.Content.WriteContent(ctx, data, prefix) + actualCID, err := dr.ContentManager().WriteContent(ctx, data, prefix) if err != nil { return nil, internalServerError(err) } diff --git a/internal/server/api_error.go b/internal/server/api_error.go index fc2a4b134..602cbc4e4 100644 --- a/internal/server/api_error.go +++ b/internal/server/api_error.go @@ -3,6 +3,8 @@ import ( "fmt" + "github.com/pkg/errors" + "github.com/kopia/kopia/internal/serverapi" ) @@ -20,6 +22,10 @@ func notFoundError(message string) *apiError { return &apiError{404, serverapi.ErrorNotFound, message} } +func repositoryNotWritableError() *apiError { + return internalServerError(errors.Errorf("repository is not writable")) +} + func internalServerError(err error) *apiError { return &apiError{500, serverapi.ErrorInternal, fmt.Sprintf("internal server error: %v", err)} } diff --git a/internal/server/api_manifest.go b/internal/server/api_manifest.go index 02e3ac7ac..950bce794 100644 --- a/internal/server/api_manifest.go +++ b/internal/server/api_manifest.go @@ -3,13 +3,14 @@ import ( "context" "encoding/json" - "errors" "net/http" "github.com/gorilla/mux" + "github.com/pkg/errors" "github.com/kopia/kopia/internal/remoterepoapi" "github.com/kopia/kopia/internal/serverapi" + "github.com/kopia/kopia/repo" "github.com/kopia/kopia/repo/manifest" ) @@ -41,9 +42,14 @@ func (s *Server) handleManifestGet(ctx context.Context, r *http.Request, body [] } func (s *Server) handleManifestDelete(ctx context.Context, r *http.Request, body []byte) (interface{}, *apiError) { + rw, ok := s.rep.(repo.RepositoryWriter) + if !ok { + return nil, repositoryNotWritableError() + } + mid := manifest.ID(mux.Vars(r)["manifestID"]) - err := s.rep.DeleteManifest(ctx, mid) + err := rw.DeleteManifest(ctx, mid) if errors.Is(err, manifest.ErrNotFound) { return nil, notFoundError("manifest not found") } @@ -96,13 +102,18 @@ func filterManifests(manifests []*manifest.EntryMetadata, userAtHost string) []* } func (s *Server) handleManifestCreate(ctx context.Context, r *http.Request, body []byte) (interface{}, *apiError) { + rw, ok := s.rep.(repo.RepositoryWriter) + if !ok { + return nil, repositoryNotWritableError() + } + var req remoterepoapi.ManifestWithMetadata if err := json.Unmarshal(body, &req); err != nil { return nil, requestError(serverapi.ErrorMalformedRequest, "malformed request") } - id, err := s.rep.PutManifest(ctx, req.Metadata.Labels, req.Payload) + id, err := rw.PutManifest(ctx, req.Metadata.Labels, req.Payload) if err != nil { return nil, internalServerError(err) } diff --git a/internal/server/api_policies.go b/internal/server/api_policies.go index 4758fcebe..64303df00 100644 --- a/internal/server/api_policies.go +++ b/internal/server/api_policies.go @@ -3,11 +3,13 @@ import ( "context" "encoding/json" - "errors" "net/http" "net/url" + "github.com/pkg/errors" + "github.com/kopia/kopia/internal/serverapi" + "github.com/kopia/kopia/repo" "github.com/kopia/kopia/snapshot" "github.com/kopia/kopia/snapshot/policy" ) @@ -60,11 +62,16 @@ func (s *Server) handlePolicyGet(ctx context.Context, r *http.Request, body []by } func (s *Server) handlePolicyDelete(ctx context.Context, r *http.Request, body []byte) (interface{}, *apiError) { - if err := policy.RemovePolicy(ctx, s.rep, getPolicyTargetFromURL(r.URL)); err != nil { + w, ok := s.rep.(repo.RepositoryWriter) + if !ok { + return nil, repositoryNotWritableError() + } + + if err := policy.RemovePolicy(ctx, w, getPolicyTargetFromURL(r.URL)); err != nil { return nil, internalServerError(err) } - if err := s.rep.Flush(ctx); err != nil { + if err := w.Flush(ctx); err != nil { return nil, internalServerError(err) } @@ -77,11 +84,16 @@ func (s *Server) handlePolicyPut(ctx context.Context, r *http.Request, body []by return nil, requestError(serverapi.ErrorMalformedRequest, "malformed request body") } - if err := policy.SetPolicy(ctx, s.rep, getPolicyTargetFromURL(r.URL), newPolicy); err != nil { + w, ok := s.rep.(repo.RepositoryWriter) + if !ok { + return nil, repositoryNotWritableError() + } + + if err := policy.SetPolicy(ctx, w, getPolicyTargetFromURL(r.URL), newPolicy); err != nil { return nil, internalServerError(err) } - if err := s.rep.Flush(ctx); err != nil { + if err := w.Flush(ctx); err != nil { return nil, internalServerError(err) } diff --git a/internal/server/api_repo.go b/internal/server/api_repo.go index 55f3af602..66526ba28 100644 --- a/internal/server/api_repo.go +++ b/internal/server/api_repo.go @@ -21,7 +21,7 @@ ) func (s *Server) handleRepoParameters(ctx context.Context, r *http.Request, body []byte) (interface{}, *apiError) { - dr, ok := s.rep.(*repo.DirectRepository) + dr, ok := s.rep.(repo.DirectRepository) if !ok { return &serverapi.StatusResponse{ Connected: false, @@ -29,9 +29,9 @@ func (s *Server) handleRepoParameters(ctx context.Context, r *http.Request, body } rp := &remoterepoapi.Parameters{ - HashFunction: dr.Content.Format().Hash, - HMACSecret: dr.Content.Format().HMACSecret, - Format: dr.Objects.Format, + HashFunction: dr.ContentReader().ContentFormat().Hash, + HMACSecret: dr.ContentReader().ContentFormat().HMACSecret, + Format: dr.ObjectFormat(), } return rp, nil @@ -44,17 +44,17 @@ func (s *Server) handleRepoStatus(ctx context.Context, r *http.Request, body []b }, nil } - dr, ok := s.rep.(*repo.DirectRepository) + dr, ok := s.rep.(repo.DirectRepository) if ok { return &serverapi.StatusResponse{ Connected: true, - ConfigFile: dr.ConfigFile, - CacheDir: dr.Cache.CacheDirectory, - Hash: dr.Content.Format().Hash, - Encryption: dr.Content.Format().Encryption, - MaxPackSize: dr.Content.Format().MaxPackSize, - Splitter: dr.Objects.Format.Splitter, - Storage: dr.Blobs.ConnectionInfo().Type, + ConfigFile: dr.ConfigFilename(), + CacheDir: dr.CachingOptions().CacheDirectory, + Hash: dr.ContentReader().ContentFormat().Hash, + Encryption: dr.ContentReader().ContentFormat().Encryption, + MaxPackSize: dr.ContentReader().ContentFormat().MaxPackSize, + Splitter: dr.ObjectFormat().Splitter, + Storage: dr.BlobReader().ConnectionInfo().Type, ClientOptions: dr.ClientOptions(), }, nil } @@ -112,7 +112,7 @@ func (s *Server) handleRepoCreate(ctx context.Context, r *http.Request, body []b } defer st.Close(ctx) //nolint:errcheck - if err := repo.Initialize(ctx, st, &req.NewRepositoryOptions, req.Password); err != nil { + if err = repo.Initialize(ctx, st, &req.NewRepositoryOptions, req.Password); err != nil { return nil, repoErrorToAPIError(err) } @@ -120,21 +120,23 @@ func (s *Server) handleRepoCreate(ctx context.Context, r *http.Request, body []b return nil, err } - if err := policy.SetPolicy(ctx, s.rep, policy.GlobalPolicySourceInfo, policy.DefaultPolicy); err != nil { - return nil, internalServerError(errors.Wrap(err, "set global policy")) - } - - if dr, ok := s.rep.(*repo.DirectRepository); ok { - p := maintenance.DefaultParams() - p.Owner = dr.Username() + "@" + dr.Hostname() - - if err := maintenance.SetParams(ctx, dr, &p); err != nil { - return nil, internalServerError(errors.Wrap(err, "unable to set maintenance params")) + if err := repo.WriteSession(ctx, s.rep, repo.WriteSessionOptions{ + Purpose: "handleRepoCreate", + }, func(w repo.RepositoryWriter) error { + if err := policy.SetPolicy(ctx, w, policy.GlobalPolicySourceInfo, policy.DefaultPolicy); err != nil { + return errors.Wrap(err, "set global policy") } - } - if err := s.rep.Flush(ctx); err != nil { - return nil, internalServerError(errors.Wrap(err, "flush")) + p := maintenance.DefaultParams() + p.Owner = w.ClientOptions().UsernameAtHost() + + if err := maintenance.SetParams(ctx, w, &p); err != nil { + return errors.Wrap(err, "unable to set maintenance params") + } + + return nil + }); err != nil { + return nil, internalServerError(err) } return s.handleRepoStatus(ctx, r, nil) diff --git a/internal/server/api_sources.go b/internal/server/api_sources.go index f06b84933..7895a0c68 100644 --- a/internal/server/api_sources.go +++ b/internal/server/api_sources.go @@ -16,7 +16,7 @@ ) func (s *Server) handleSourcesList(ctx context.Context, r *http.Request, body []byte) (interface{}, *apiError) { - _, multiUser := s.rep.(*repo.DirectRepository) + _, multiUser := s.rep.(repo.DirectRepository) resp := &serverapi.SourcesResponse{ Sources: []*serverapi.SourceStatus{}, @@ -84,14 +84,14 @@ func (s *Server) handleSourcesCreate(ctx context.Context, r *http.Request, body // don't have policy - create an empty one log(ctx).Debugf("policy for %v not found, creating empty one", sourceInfo) - if err = policy.SetPolicy(ctx, s.rep, sourceInfo, &req.InitialPolicy); err != nil { + if err = repo.WriteSession(ctx, s.rep, repo.WriteSessionOptions{ + Purpose: "handleSourcesCreate", + }, func(w repo.RepositoryWriter) error { + return policy.SetPolicy(ctx, w, sourceInfo, &req.InitialPolicy) + }); err != nil { return nil, internalServerError(errors.Wrap(err, "unable to set initial policy")) } - if err = s.rep.Flush(ctx); err != nil { - return nil, internalServerError(errors.Wrap(err, "unable to flush")) - } - default: return nil, internalServerError(err) } diff --git a/internal/server/server.go b/internal/server/server.go index 6bd689c31..b4fd12918 100644 --- a/internal/server/server.go +++ b/internal/server/server.go @@ -166,7 +166,12 @@ func (s *Server) handleRefresh(ctx context.Context, r *http.Request, body []byte } func (s *Server) handleFlush(ctx context.Context, r *http.Request, body []byte) (interface{}, *apiError) { - if err := s.rep.Flush(ctx); err != nil { + rw, ok := s.rep.(repo.RepositoryWriter) + if !ok { + return nil, repositoryNotWritableError() + } + + if err := rw.Flush(ctx); err != nil { return nil, internalServerError(err) } @@ -291,20 +296,33 @@ func (s *Server) refreshPeriodically(ctx context.Context, r repo.Repository) { } } -func (s *Server) periodicMaintenance(ctx context.Context, r repo.Writer) { +func (s *Server) periodicMaintenance(ctx context.Context, rep repo.Repository) { for { select { case <-ctx.Done(): return case <-time.After(maintenanceAttemptFrequency): - if err := snapshotmaintenance.Run(ctx, r, maintenance.ModeAuto, false); err != nil { + if err := periodicMaintenanceOnce(ctx, rep); err != nil { log(ctx).Warningf("unable to run maintenance: %v", err) } } } } +func periodicMaintenanceOnce(ctx context.Context, rep repo.Repository) error { + dr, ok := rep.(repo.DirectRepository) + if !ok { + return errors.Errorf("not a direct repository") + } + + return repo.DirectWriteSession(ctx, dr, repo.WriteSessionOptions{ + Purpose: "periodicMaintenanceOnce", + }, func(w repo.DirectRepositoryWriter) error { + return snapshotmaintenance.Run(ctx, w, maintenance.ModeAuto, false) + }) +} + // SyncSources synchronizes the repository and source managers. func (s *Server) SyncSources(ctx context.Context) error { s.mu.Lock() diff --git a/internal/server/source_manager.go b/internal/server/source_manager.go index 15f870cd6..07464b630 100644 --- a/internal/server/source_manager.go +++ b/internal/server/source_manager.go @@ -11,6 +11,7 @@ "github.com/kopia/kopia/internal/clock" "github.com/kopia/kopia/internal/ctxutil" "github.com/kopia/kopia/internal/serverapi" + "github.com/kopia/kopia/repo" "github.com/kopia/kopia/snapshot" "github.com/kopia/kopia/snapshot/policy" "github.com/kopia/kopia/snapshot/snapshotfs" @@ -240,40 +241,40 @@ func (s *sourceManager) snapshot(ctx context.Context) error { return errors.Wrap(err, "unable to create local filesystem") } - u := snapshotfs.NewUploader(s.server.rep) + return repo.WriteSession(ctx, s.server.rep, repo.WriteSessionOptions{ + Purpose: "Source Manager Uploader", + }, func(w repo.RepositoryWriter) error { + log(ctx).Debugf("uploading %v", s.src) + u := snapshotfs.NewUploader(w) - policyTree, err := policy.TreeForSource(ctx, s.server.rep, s.src) - if err != nil { - return errors.Wrap(err, "unable to create policy getter") - } + policyTree, err := policy.TreeForSource(ctx, w, s.src) + if err != nil { + return errors.Wrap(err, "unable to create policy getter") + } - u.Progress = s.progress + u.Progress = s.progress - log(ctx).Debugf("starting upload of %v", s.src) - s.setUploader(u) - manifest, err := u.Upload(ctx, localEntry, policyTree, s.src, s.manifestsSinceLastCompleteSnapshot...) - s.setUploader(nil) + log(ctx).Debugf("starting upload of %v", s.src) + s.setUploader(u) + manifest, err := u.Upload(ctx, localEntry, policyTree, s.src, s.manifestsSinceLastCompleteSnapshot...) + s.setUploader(nil) - if err != nil { - return errors.Wrap(err, "upload error") - } + if err != nil { + return errors.Wrap(err, "upload error") + } - snapshotID, err := snapshot.SaveSnapshot(ctx, s.server.rep, manifest) - if err != nil { - return errors.Wrap(err, "unable to save snapshot") - } + snapshotID, err := snapshot.SaveSnapshot(ctx, w, manifest) + if err != nil { + return errors.Wrap(err, "unable to save snapshot") + } - if _, err := policy.ApplyRetentionPolicy(ctx, s.server.rep, s.src, true); err != nil { - return errors.Wrap(err, "unable to apply retention policy") - } + if _, err := policy.ApplyRetentionPolicy(ctx, w, s.src, true); err != nil { + return errors.Wrap(err, "unable to apply retention policy") + } - log(ctx).Debugf("created snapshot %v", snapshotID) - - if err := s.server.rep.Flush(ctx); err != nil { - return errors.Wrap(err, "unable to flush") - } - - return nil + log(ctx).Debugf("created snapshot %v", snapshotID) + return nil + }) } func (s *sourceManager) findClosestNextSnapshotTime() *time.Time { diff --git a/repo/api_server_repository.go b/repo/api_server_repository.go index ab05e7409..8d90106aa 100644 --- a/repo/api_server_repository.go +++ b/repo/api_server_repository.go @@ -31,11 +31,11 @@ type APIServerInfo struct { // remoteRepository is an implementation of Repository that connects to an instance of // API server hosted by `kopia server`, instead of directly manipulating files in the BLOB storage. type apiServerRepository struct { - cli *apiclient.KopiaAPIClient - h hashing.HashFunc - - omgr *object.Manager - cliOpts ClientOptions + cli *apiclient.KopiaAPIClient + h hashing.HashFunc + objectFormat object.Format + cliOpts ClientOptions + omgr *object.Manager } func (r *apiServerRepository) APIServerURL() string { @@ -130,12 +130,20 @@ func (r *apiServerRepository) Flush(ctx context.Context) error { return errors.Wrap(r.cli.Post(ctx, "flush", nil, nil), "Flush") } -func (r *apiServerRepository) Close(ctx context.Context) error { - if err := r.omgr.Close(); err != nil { - return errors.Wrap(err, "error closing object manager") +func (r *apiServerRepository) NewWriter(ctx context.Context, purpose string) (RepositoryWriter, error) { + // apiServerRepository is stateless except object manager. + r2 := *r + w := &r2 + + // create object manager using a remote repo as contentManager implementation. + omgr, err := object.NewObjectManager(ctx, w, r.objectFormat) + if err != nil { + return nil, errors.Wrap(err, "error initializing object manager") } - return errors.Wrap(r.Flush(ctx), "Close") + w.omgr = omgr + + return w, nil } func (r *apiServerRepository) ContentInfo(ctx context.Context, contentID content.ID) (content.Info, error) { @@ -179,6 +187,14 @@ func (r *apiServerRepository) UpdateDescription(d string) { r.cliOpts.Description = d } +func (r *apiServerRepository) Close(ctx context.Context) error { + if err := r.omgr.Close(); err != nil { + return errors.Wrap(err, "error closing object manager") + } + + return nil +} + var _ Repository = (*apiServerRepository)(nil) // openAPIServer connects remote repository over Kopia API. @@ -186,7 +202,7 @@ func openAPIServer(ctx context.Context, si *APIServerInfo, cliOpts ClientOptions cli, err := apiclient.NewKopiaAPIClient(apiclient.Options{ BaseURL: si.BaseURL, TrustedServerCertificateFingerprint: si.TrustedServerCertificateFingerprint, - Username: cliOpts.Username + "@" + cliOpts.Hostname, + Username: cliOpts.UsernameAtHost(), Password: password, LogRequests: true, }) @@ -211,9 +227,10 @@ func openAPIServer(ctx context.Context, si *APIServerInfo, cliOpts ClientOptions } rr.h = hf + rr.objectFormat = p.Format // create object manager using rr as contentManager implementation. - omgr, err := object.NewObjectManager(ctx, rr, p.Format) + omgr, err := object.NewObjectManager(ctx, rr, rr.objectFormat) if err != nil { return nil, errors.Wrap(err, "error initializing object manager") } diff --git a/repo/blob/storage.go b/repo/blob/storage.go index b13a3f78c..0eac57cbd 100644 --- a/repo/blob/storage.go +++ b/repo/blob/storage.go @@ -22,28 +22,8 @@ type Bytes interface { Reader() io.Reader } -// Storage encapsulates API for connecting to blob storage. -// -// The underlying storage system must provide: -// -// * high durability, availability and bit-rot protection -// * read-after-write - blob written using PubBlob() must be immediately readable using GetBlob() and ListBlobs() -// * atomicity - it mustn't be possible to observe partial results of PubBlob() via either GetBlob() or ListBlobs() -// * timestamps that don't go back in time (small clock skew up to minutes is allowed) -// * reasonably low latency for retrievals -// -// The required semantics are provided by existing commercial cloud storage products (Google Cloud, AWS, Azure). -type Storage interface { - // PutBlob uploads the blob with given data to the repository or replaces existing blob with the provided - // id with contents gathered from the specified list of slices. - PutBlob(ctx context.Context, blobID ID, data Bytes) error - - // SetTime changes last modification time of a given blob, if supported, returns ErrSetTimeUnsupported otherwise. - SetTime(ctx context.Context, blobID ID, t time.Time) error - - // DeleteBlob removes the blob from storage. Future Get() operations will fail with ErrNotFound. - DeleteBlob(ctx context.Context, blobID ID) error - +// Reader defines read access API to blob storage. +type Reader interface { // GetBlob returns full or partial contents of a blob with given ID. // If length>0, the the function retrieves a range of bytes [offset,offset+length) // If length<0, the entire blob must be fetched. @@ -60,13 +40,38 @@ type Storage interface { // connect to storage. ConnectionInfo() ConnectionInfo - // Close releases all resources associated with storage. - Close(ctx context.Context) error - // Name of the storage used for quick identification by humans. DisplayName() string } +// Storage encapsulates API for connecting to blob storage. +// +// The underlying storage system must provide: +// +// * high durability, availability and bit-rot protection +// * read-after-write - blob written using PubBlob() must be immediately readable using GetBlob() and ListBlobs() +// * atomicity - it mustn't be possible to observe partial results of PubBlob() via either GetBlob() or ListBlobs() +// * timestamps that don't go back in time (small clock skew up to minutes is allowed) +// * reasonably low latency for retrievals +// +// The required semantics are provided by existing commercial cloud storage products (Google Cloud, AWS, Azure). +type Storage interface { + Reader + + // PutBlob uploads the blob with given data to the repository or replaces existing blob with the provided + // id with contents gathered from the specified list of slices. + PutBlob(ctx context.Context, blobID ID, data Bytes) error + + // SetTime changes last modification time of a given blob, if supported, returns ErrSetTimeUnsupported otherwise. + SetTime(ctx context.Context, blobID ID, t time.Time) error + + // DeleteBlob removes the blob from storage. Future Get() operations will fail with ErrNotFound. + DeleteBlob(ctx context.Context, blobID ID) error + + // Close releases all resources associated with storage. + Close(ctx context.Context) error +} + // ID is a string that represents blob identifier. type ID string diff --git a/repo/content/content_read_manager.go b/repo/content/committed_read_manager.go similarity index 91% rename from repo/content/content_read_manager.go rename to repo/content/committed_read_manager.go index 02ab843ec..4647abd3f 100644 --- a/repo/content/content_read_manager.go +++ b/repo/content/committed_read_manager.go @@ -6,6 +6,7 @@ "crypto/aes" "os" "sync" + "sync/atomic" "time" "github.com/pkg/errors" @@ -19,6 +20,9 @@ // SharedManager is responsible for read-only access to committed data. type SharedManager struct { + refCount int32 // number of Manager objects that refer to this SharedManager + closed int32 // set to 1 if shared manager has been closed + Stats *Stats st blob.Storage indexBlobManager indexBlobManager @@ -319,6 +323,38 @@ func (sm *SharedManager) setupReadManagerCaches(ctx context.Context, caching *Ca return nil } +// AddRef adds a reference to shared manager to prevents its closing on Release(). +func (sm *SharedManager) addRef() { + if atomic.LoadInt32(&sm.closed) != 0 { + panic("attempted to re-use closed SharedManager") + } + + atomic.AddInt32(&sm.refCount, 1) +} + +// release removes a reference to the shared manager and destroys it if no more references are remaining. +func (sm *SharedManager) release(ctx context.Context) error { + remaining := atomic.AddInt32(&sm.refCount, -1) + if remaining != 0 { + log(ctx).Debugf("not closing shared manager, remaining = %v", remaining) + return nil + } + + atomic.StoreInt32(&sm.closed, 1) + + log(ctx).Debugf("closing shared manager") + + if err := sm.committedContents.close(); err != nil { + return errors.Wrap(err, "error closed committed content index") + } + + sm.contentCache.close() + sm.metadataCache.close() + sm.encryptionBufferPool.Close() + + return sm.st.Close(ctx) +} + // NewSharedManager returns SharedManager that is used by SessionWriteManagers on top of a repository. func NewSharedManager(ctx context.Context, st blob.Storage, f *FormattingOptions, caching *CachingOptions, opts *ManagerOptions) (*SharedManager, error) { opts = opts.CloneOrDefault() diff --git a/repo/content/content_index_reader.go b/repo/content/content_index_reader.go new file mode 100644 index 000000000..469d5ff5a --- /dev/null +++ b/repo/content/content_index_reader.go @@ -0,0 +1,14 @@ +package content + +import ( + "context" + + "github.com/kopia/kopia/repo/blob" +) + +// IndexBlobReader defines content read API. +type IndexBlobReader interface { + ParseIndexBlob(ctx context.Context, blobID blob.ID) ([]Info, error) + DecryptBlob(ctx context.Context, blobID blob.ID) ([]byte, error) + IndexBlobs(ctx context.Context, includeInactive bool) ([]IndexBlobInfo, error) +} diff --git a/repo/content/content_manager.go b/repo/content/content_manager.go index 2dec4b2b7..76d101ed7 100644 --- a/repo/content/content_manager.go +++ b/repo/content/content_manager.go @@ -451,26 +451,14 @@ func removePendingPack(slice []*pendingPackInfo, pp *pendingPackInfo) []*pending return result } -// Format returns formatting options. -func (bm *WriteManager) Format() FormattingOptions { +// ContentFormat returns formatting options. +func (bm *WriteManager) ContentFormat() FormattingOptions { return bm.format } // Close closes the content manager. func (bm *WriteManager) Close(ctx context.Context) error { - if err := bm.Flush(ctx); err != nil { - return errors.Wrap(err, "error flushing") - } - - if err := bm.committedContents.close(); err != nil { - return errors.Wrap(err, "error closed committed content index") - } - - bm.contentCache.close() - bm.metadataCache.close() - bm.encryptionBufferPool.Close() - - return nil + return bm.SharedManager.release(ctx) } // Flush completes writing any pending packs and writes pack indexes to the underlying storage. @@ -799,6 +787,8 @@ type SessionOptions struct { func NewWriteManager(sm *SharedManager, options SessionOptions) *WriteManager { mu := &sync.RWMutex{} + sm.addRef() + return &WriteManager{ SharedManager: sm, diff --git a/repo/content/content_manager_test.go b/repo/content/content_manager_test.go index d2619b56f..7c40aa1f3 100644 --- a/repo/content/content_manager_test.go +++ b/repo/content/content_manager_test.go @@ -364,14 +364,16 @@ func TestIndexCompactionDropsContent(t *testing.T) { // create record in index #1 bm := newTestContentManager(t, data, keyTime, timeFunc) content1 := writeContentAndVerify(ctx, t, bm, seededRandomData(10, 100)) - bm.Close(ctx) + must(t, bm.Flush(ctx)) + must(t, bm.Close(ctx)) timeFunc() // create record in index #2 bm = newTestContentManager(t, data, keyTime, timeFunc) deleteContent(ctx, t, bm, content1) - bm.Close(ctx) + must(t, bm.Flush(ctx)) + must(t, bm.Close(ctx)) timeFunc() @@ -385,7 +387,8 @@ func TestIndexCompactionDropsContent(t *testing.T) { DropDeletedBefore: deleteThreshold, AllIndexes: true, })) - bm.Close(ctx) + must(t, bm.Flush(ctx)) + must(t, bm.Close(ctx)) bm = newTestContentManager(t, data, keyTime, timeFunc) verifyContentNotFound(ctx, t, bm, content1) @@ -1907,12 +1910,14 @@ func verifyReadsOwnWrites(t *testing.T, st blob.Storage, timeNow func() time.Tim // every 10 contents, create new content manager if i%10 == 0 { - t.Logf("------- reopening -----") + t.Logf("------- flushing & reopening -----") + must(t, bm.Flush(ctx)) must(t, bm.Close(ctx)) bm = newTestContentManagerWithStorageAndCaching(t, st, cachingOptions, timeNow) } } + must(t, bm.Flush(ctx)) must(t, bm.Close(ctx)) bm = newTestContentManagerWithStorageAndCaching(t, st, cachingOptions, timeNow) diff --git a/repo/content/content_reader.go b/repo/content/content_reader.go new file mode 100644 index 000000000..406bc6b0f --- /dev/null +++ b/repo/content/content_reader.go @@ -0,0 +1,15 @@ +package content + +import ( + "context" +) + +// Reader defines content read API. +type Reader interface { + ContentFormat() FormattingOptions + GetContent(ctx context.Context, id ID) ([]byte, error) + ContentInfo(ctx context.Context, id ID) (Info, error) + IterateContents(ctx context.Context, opts IterateOptions, callback IterateCallback) error + IteratePacks(ctx context.Context, opts IteratePackOptions, callback IteratePacksCallback) error + ListActiveSessions(ctx context.Context) (map[SessionID]*SessionInfo, error) +} diff --git a/repo/content/index_blob_manager_test.go b/repo/content/index_blob_manager_test.go index b8c3c621a..4de2a2391 100644 --- a/repo/content/index_blob_manager_test.go +++ b/repo/content/index_blob_manager_test.go @@ -156,7 +156,6 @@ func pickRandomActionTestIndexBlobManagerStress() action { // nolint:gocyclo func TestIndexBlobManagerStress(t *testing.T) { t.Parallel() - rand.Seed(clock.Now().UnixNano()) for i := range actionsTestIndexBlobManagerStress { diff --git a/repo/local_config.go b/repo/local_config.go index 96990a6a9..8a4df85f6 100644 --- a/repo/local_config.go +++ b/repo/local_config.go @@ -64,6 +64,11 @@ func (o ClientOptions) Override(other ClientOptions) ClientOptions { return o } +// UsernameAtHost returns 'username@hostname' string. +func (o ClientOptions) UsernameAtHost() string { + return o.Username + "@" + o.Hostname +} + // LocalConfig is a configuration of Kopia stored in a configuration file. type LocalConfig struct { // APIServer is only provided for remote repository. diff --git a/repo/maintenance/blob_gc.go b/repo/maintenance/blob_gc.go index 896fa6e50..45d94d338 100644 --- a/repo/maintenance/blob_gc.go +++ b/repo/maintenance/blob_gc.go @@ -9,6 +9,7 @@ "github.com/kopia/kopia/internal/stats" "github.com/kopia/kopia/internal/units" + "github.com/kopia/kopia/repo" "github.com/kopia/kopia/repo/blob" "github.com/kopia/kopia/repo/content" ) @@ -32,7 +33,7 @@ type DeleteUnreferencedBlobsOptions struct { // DeleteUnreferencedBlobs deletes old blobs that are no longer referenced by index entries. // nolint:gocyclo -func DeleteUnreferencedBlobs(ctx context.Context, rep MaintainableRepository, opt DeleteUnreferencedBlobsOptions) (int, error) { +func DeleteUnreferencedBlobs(ctx context.Context, rep repo.DirectRepositoryWriter, opt DeleteUnreferencedBlobsOptions) (int, error) { if opt.Parallel == 0 { opt.Parallel = 16 } @@ -82,7 +83,7 @@ func DeleteUnreferencedBlobs(ctx context.Context, rep MaintainableRepository, op prefixes = append(prefixes, content.PackBlobIDPrefixRegular, content.PackBlobIDPrefixSpecial, content.BlobIDPrefixSession) } - activeSessions, err := rep.ListActiveSessions(ctx) + activeSessions, err := rep.ContentManager().ListActiveSessions(ctx) if err != nil { return 0, errors.Wrap(err, "unable to load active sessions") } diff --git a/repo/maintenance/blob_gc_test.go b/repo/maintenance/blob_gc_test.go index 04bff20be..29b64870b 100644 --- a/repo/maintenance/blob_gc_test.go +++ b/repo/maintenance/blob_gc_test.go @@ -43,14 +43,14 @@ func TestDeleteUnreferencedBlobs(t *testing.T) { }, }).Close(ctx, t) - w := env.Repository.NewObjectWriter(ctx, object.WriterOptions{}) + w := env.RepositoryWriter.NewObjectWriter(ctx, object.WriterOptions{}) io.WriteString(w, "hello world!") w.Result() w.Close() - env.Repository.Flush(ctx) + env.RepositoryWriter.Flush(ctx) - blobsBefore, err := blob.ListAllBlobs(ctx, env.Repository.Blobs, "") + blobsBefore, err := blob.ListAllBlobs(ctx, env.RepositoryWriter.BlobStorage(), "") if err != nil { t.Fatal(err) } @@ -65,30 +65,30 @@ func TestDeleteUnreferencedBlobs(t *testing.T) { extraBlobID2 blob.ID = "pdeadbeef2" ) - mustPutDummyBlob(t, env.Repository.Blobs, extraBlobID1) - mustPutDummyBlob(t, env.Repository.Blobs, extraBlobID2) - verifyBlobExists(t, env.Repository.Blobs, extraBlobID1) - verifyBlobExists(t, env.Repository.Blobs, extraBlobID2) + mustPutDummyBlob(t, env.RepositoryWriter.BlobStorage(), extraBlobID1) + mustPutDummyBlob(t, env.RepositoryWriter.BlobStorage(), extraBlobID2) + verifyBlobExists(t, env.RepositoryWriter.BlobStorage(), extraBlobID1) + verifyBlobExists(t, env.RepositoryWriter.BlobStorage(), extraBlobID2) // new blobs not will be deleted because of minimum age requirement - if _, err = DeleteUnreferencedBlobs(ctx, env.Repository, DeleteUnreferencedBlobsOptions{ + if _, err = DeleteUnreferencedBlobs(ctx, env.RepositoryWriter, DeleteUnreferencedBlobsOptions{ MinAge: 1 * time.Hour, }); err != nil { t.Fatal(err) } - verifyBlobExists(t, env.Repository.Blobs, extraBlobID1) - verifyBlobExists(t, env.Repository.Blobs, extraBlobID2) + verifyBlobExists(t, env.RepositoryWriter.BlobStorage(), extraBlobID1) + verifyBlobExists(t, env.RepositoryWriter.BlobStorage(), extraBlobID2) // new blobs will be deleted - if _, err = DeleteUnreferencedBlobs(ctx, env.Repository, DeleteUnreferencedBlobsOptions{ + if _, err = DeleteUnreferencedBlobs(ctx, env.RepositoryWriter, DeleteUnreferencedBlobsOptions{ MinAge: 1, }); err != nil { t.Fatal(err) } - verifyBlobNotFound(t, env.Repository.Blobs, extraBlobID1) - verifyBlobNotFound(t, env.Repository.Blobs, extraBlobID2) + verifyBlobNotFound(t, env.RepositoryWriter.BlobStorage(), extraBlobID1) + verifyBlobNotFound(t, env.RepositoryWriter.BlobStorage(), extraBlobID2) // add blobs again and const ( @@ -97,62 +97,62 @@ func TestDeleteUnreferencedBlobs(t *testing.T) { extraBlobIDWithSession3 blob.ID = "pdeadbeef3-s02" ) - mustPutDummyBlob(t, env.Repository.Blobs, extraBlobIDWithSession1) - mustPutDummyBlob(t, env.Repository.Blobs, extraBlobIDWithSession2) - mustPutDummyBlob(t, env.Repository.Blobs, extraBlobIDWithSession3) + mustPutDummyBlob(t, env.RepositoryWriter.BlobStorage(), extraBlobIDWithSession1) + mustPutDummyBlob(t, env.RepositoryWriter.BlobStorage(), extraBlobIDWithSession2) + mustPutDummyBlob(t, env.RepositoryWriter.BlobStorage(), extraBlobIDWithSession3) - session1Marker := mustPutDummySessionBlob(t, env.Repository.Blobs, "s01", &content.SessionInfo{ + session1Marker := mustPutDummySessionBlob(t, env.RepositoryWriter.BlobStorage(), "s01", &content.SessionInfo{ CheckpointTime: ta.NowFunc()(), }) - session2Marker := mustPutDummySessionBlob(t, env.Repository.Blobs, "s02", &content.SessionInfo{ + session2Marker := mustPutDummySessionBlob(t, env.RepositoryWriter.BlobStorage(), "s02", &content.SessionInfo{ CheckpointTime: ta.NowFunc()(), }) - if _, err = DeleteUnreferencedBlobs(ctx, env.Repository, DeleteUnreferencedBlobsOptions{ + if _, err = DeleteUnreferencedBlobs(ctx, env.RepositoryWriter, DeleteUnreferencedBlobsOptions{ MinAge: 1, }); err != nil { t.Fatal(err) } - verifyBlobExists(t, env.Repository.Blobs, extraBlobIDWithSession1) - verifyBlobExists(t, env.Repository.Blobs, extraBlobIDWithSession2) - verifyBlobExists(t, env.Repository.Blobs, extraBlobIDWithSession3) - verifyBlobExists(t, env.Repository.Blobs, session1Marker) - verifyBlobExists(t, env.Repository.Blobs, session2Marker) + verifyBlobExists(t, env.RepositoryWriter.BlobStorage(), extraBlobIDWithSession1) + verifyBlobExists(t, env.RepositoryWriter.BlobStorage(), extraBlobIDWithSession2) + verifyBlobExists(t, env.RepositoryWriter.BlobStorage(), extraBlobIDWithSession3) + verifyBlobExists(t, env.RepositoryWriter.BlobStorage(), session1Marker) + verifyBlobExists(t, env.RepositoryWriter.BlobStorage(), session2Marker) // now finish session 2 - env.Repository.Blobs.DeleteBlob(ctx, session2Marker) + env.RepositoryWriter.BlobStorage().DeleteBlob(ctx, session2Marker) - if _, err = DeleteUnreferencedBlobs(ctx, env.Repository, DeleteUnreferencedBlobsOptions{ + if _, err = DeleteUnreferencedBlobs(ctx, env.RepositoryWriter, DeleteUnreferencedBlobsOptions{ MinAge: 1, }); err != nil { t.Fatal(err) } - verifyBlobExists(t, env.Repository.Blobs, extraBlobIDWithSession1) - verifyBlobExists(t, env.Repository.Blobs, extraBlobIDWithSession2) - verifyBlobNotFound(t, env.Repository.Blobs, extraBlobIDWithSession3) - verifyBlobExists(t, env.Repository.Blobs, session1Marker) - verifyBlobNotFound(t, env.Repository.Blobs, session2Marker) + verifyBlobExists(t, env.RepositoryWriter.BlobStorage(), extraBlobIDWithSession1) + verifyBlobExists(t, env.RepositoryWriter.BlobStorage(), extraBlobIDWithSession2) + verifyBlobNotFound(t, env.RepositoryWriter.BlobStorage(), extraBlobIDWithSession3) + verifyBlobExists(t, env.RepositoryWriter.BlobStorage(), session1Marker) + verifyBlobNotFound(t, env.RepositoryWriter.BlobStorage(), session2Marker) // now move time into the future making session 1 timed out ta.Advance(7 * 24 * time.Hour) - if _, err = DeleteUnreferencedBlobs(ctx, env.Repository, DeleteUnreferencedBlobsOptions{ + if _, err = DeleteUnreferencedBlobs(ctx, env.RepositoryWriter, DeleteUnreferencedBlobsOptions{ MinAge: 1, }); err != nil { t.Fatal(err) } - verifyBlobNotFound(t, env.Repository.Blobs, extraBlobIDWithSession1) - verifyBlobNotFound(t, env.Repository.Blobs, extraBlobIDWithSession2) - verifyBlobNotFound(t, env.Repository.Blobs, extraBlobIDWithSession3) - verifyBlobNotFound(t, env.Repository.Blobs, session1Marker) - verifyBlobNotFound(t, env.Repository.Blobs, session2Marker) + verifyBlobNotFound(t, env.RepositoryWriter.BlobStorage(), extraBlobIDWithSession1) + verifyBlobNotFound(t, env.RepositoryWriter.BlobStorage(), extraBlobIDWithSession2) + verifyBlobNotFound(t, env.RepositoryWriter.BlobStorage(), extraBlobIDWithSession3) + verifyBlobNotFound(t, env.RepositoryWriter.BlobStorage(), session1Marker) + verifyBlobNotFound(t, env.RepositoryWriter.BlobStorage(), session2Marker) // make sure we're back to the starting point. - blobsAfter, err := blob.ListAllBlobs(ctx, env.Repository.Blobs, "") + blobsAfter, err := blob.ListAllBlobs(ctx, env.RepositoryWriter.BlobStorage(), "") if err != nil { t.Fatal(err) } diff --git a/repo/maintenance/content_rewrite.go b/repo/maintenance/content_rewrite.go index 1fdb23970..f343eeda3 100644 --- a/repo/maintenance/content_rewrite.go +++ b/repo/maintenance/content_rewrite.go @@ -10,6 +10,7 @@ "github.com/pkg/errors" "github.com/kopia/kopia/internal/units" + "github.com/kopia/kopia/repo" "github.com/kopia/kopia/repo/blob" "github.com/kopia/kopia/repo/content" ) @@ -39,7 +40,7 @@ type contentInfoOrError struct { // RewriteContents rewrites contents according to provided criteria and creates new // blobs and index entries to point at the. -func RewriteContents(ctx context.Context, rep MaintainableRepository, opt *RewriteContentsOptions) error { +func RewriteContents(ctx context.Context, rep repo.DirectRepositoryWriter, opt *RewriteContentsOptions) error { if opt == nil { return errors.Errorf("missing options") } @@ -124,7 +125,7 @@ func RewriteContents(ctx context.Context, rep MaintainableRepository, opt *Rewri return errors.Errorf("failed to rewrite %v contents", failedCount) } -func getContentToRewrite(ctx context.Context, rep MaintainableRepository, opt *RewriteContentsOptions) <-chan contentInfoOrError { +func getContentToRewrite(ctx context.Context, rep repo.DirectRepository, opt *RewriteContentsOptions) <-chan contentInfoOrError { ch := make(chan contentInfoOrError) go func() { @@ -135,7 +136,7 @@ func getContentToRewrite(ctx context.Context, rep MaintainableRepository, opt *R // add all content IDs from short packs if opt.ShortPacks { - threshold := int64(rep.ContentManager().Format().MaxPackSize * shortPackThresholdPercent / 100) //nolint:gomnd + threshold := int64(rep.ContentReader().ContentFormat().MaxPackSize * shortPackThresholdPercent / 100) //nolint:gomnd findContentInShortPacks(ctx, rep, ch, threshold, opt) } @@ -148,9 +149,9 @@ func getContentToRewrite(ctx context.Context, rep MaintainableRepository, opt *R return ch } -func findContentInfos(ctx context.Context, rep MaintainableRepository, ch chan contentInfoOrError, contentIDs []content.ID) { +func findContentInfos(ctx context.Context, rep repo.DirectRepository, ch chan contentInfoOrError, contentIDs []content.ID) { for _, contentID := range contentIDs { - i, err := rep.ContentManager().ContentInfo(ctx, contentID) + i, err := rep.ContentReader().ContentInfo(ctx, contentID) if err != nil { ch <- contentInfoOrError{err: errors.Wrapf(err, "unable to get info for content %q", contentID)} } else { @@ -159,8 +160,8 @@ func findContentInfos(ctx context.Context, rep MaintainableRepository, ch chan c } } -func findContentWithFormatVersion(ctx context.Context, rep MaintainableRepository, ch chan contentInfoOrError, opt *RewriteContentsOptions) { - _ = rep.ContentManager().IterateContents( +func findContentWithFormatVersion(ctx context.Context, rep repo.DirectRepository, ch chan contentInfoOrError, opt *RewriteContentsOptions) { + _ = rep.ContentReader().IterateContents( ctx, content.IterateOptions{ Range: opt.ContentIDRange, @@ -174,14 +175,14 @@ func(b content.Info) error { }) } -func findContentInShortPacks(ctx context.Context, rep MaintainableRepository, ch chan contentInfoOrError, threshold int64, opt *RewriteContentsOptions) { +func findContentInShortPacks(ctx context.Context, rep repo.DirectRepository, ch chan contentInfoOrError, threshold int64, opt *RewriteContentsOptions) { var prefixes []blob.ID if opt.PackPrefix != "" { prefixes = append(prefixes, opt.PackPrefix) } - err := rep.ContentManager().IteratePacks( + err := rep.ContentReader().IteratePacks( ctx, content.IteratePackOptions{ Prefixes: prefixes, diff --git a/repo/maintenance/drop_deleted_contents.go b/repo/maintenance/drop_deleted_contents.go index 581418564..0ccfde893 100644 --- a/repo/maintenance/drop_deleted_contents.go +++ b/repo/maintenance/drop_deleted_contents.go @@ -4,11 +4,12 @@ "context" "time" + "github.com/kopia/kopia/repo" "github.com/kopia/kopia/repo/content" ) // DropDeletedContents rewrites indexes while dropping deleted contents above certain age. -func DropDeletedContents(ctx context.Context, rep MaintainableRepository, dropDeletedBefore time.Time) error { +func DropDeletedContents(ctx context.Context, rep repo.DirectRepositoryWriter, dropDeletedBefore time.Time) error { log(ctx).Infof("Dropping contents deleted before %v", dropDeletedBefore) return rep.ContentManager().CompactIndexes(ctx, content.CompactOptions{ diff --git a/repo/maintenance/index_compaction.go b/repo/maintenance/index_compaction.go index 96d0571d3..ee5a3a595 100644 --- a/repo/maintenance/index_compaction.go +++ b/repo/maintenance/index_compaction.go @@ -3,13 +3,14 @@ import ( "context" + "github.com/kopia/kopia/repo" "github.com/kopia/kopia/repo/content" ) const maxSmallBlobsForIndexCompaction = 8 // IndexCompaction rewrites index blobs to reduce their count but does not drop any contents. -func IndexCompaction(ctx context.Context, rep MaintainableRepository) error { +func IndexCompaction(ctx context.Context, rep repo.DirectRepositoryWriter) error { log(ctx).Infof("Compacting indexes...") return rep.ContentManager().CompactIndexes(ctx, content.CompactOptions{ diff --git a/repo/maintenance/maintenance_params.go b/repo/maintenance/maintenance_params.go index 50b736108..b08d4c4a5 100644 --- a/repo/maintenance/maintenance_params.go +++ b/repo/maintenance/maintenance_params.go @@ -6,6 +6,7 @@ "github.com/pkg/errors" + "github.com/kopia/kopia/repo" "github.com/kopia/kopia/repo/manifest" ) @@ -55,7 +56,7 @@ type CycleParams struct { } // HasParams determines whether repository-wide maintenance parameters have been set. -func HasParams(ctx context.Context, rep MaintainableRepository) (bool, error) { +func HasParams(ctx context.Context, rep repo.Repository) (bool, error) { md, err := manifestIDs(ctx, rep) if err != nil { return false, err @@ -65,7 +66,7 @@ func HasParams(ctx context.Context, rep MaintainableRepository) (bool, error) { } // GetParams returns repository-wide maintenance parameters. -func GetParams(ctx context.Context, rep MaintainableRepository) (*Params, error) { +func GetParams(ctx context.Context, rep repo.Repository) (*Params, error) { md, err := manifestIDs(ctx, rep) if err != nil { return nil, err @@ -92,7 +93,7 @@ func GetParams(ctx context.Context, rep MaintainableRepository) (*Params, error) } // SetParams sets the maintenance parameters. -func SetParams(ctx context.Context, rep MaintainableRepository, par *Params) error { +func SetParams(ctx context.Context, rep repo.RepositoryWriter, par *Params) error { md, err := manifestIDs(ctx, rep) if err != nil { return err @@ -111,7 +112,7 @@ func SetParams(ctx context.Context, rep MaintainableRepository, par *Params) err return nil } -func manifestIDs(ctx context.Context, rep MaintainableRepository) ([]*manifest.EntryMetadata, error) { +func manifestIDs(ctx context.Context, rep repo.Repository) ([]*manifest.EntryMetadata, error) { md, err := rep.FindManifests(ctx, manifestLabels) if err != nil { return nil, errors.Wrap(err, "error looking for maintenance manifest") diff --git a/repo/maintenance/maintenance_run.go b/repo/maintenance/maintenance_run.go index 9ff736425..facdcb6df 100644 --- a/repo/maintenance/maintenance_run.go +++ b/repo/maintenance/maintenance_run.go @@ -10,7 +10,6 @@ "github.com/pkg/errors" "github.com/kopia/kopia/repo" - "github.com/kopia/kopia/repo/blob" "github.com/kopia/kopia/repo/content" "github.com/kopia/kopia/repo/logging" ) @@ -30,22 +29,6 @@ // Mode describes the mode of maintenance to perfor. type Mode string -// MaintainableRepository is a subset of Repository required for maintenance tasks. -type MaintainableRepository interface { - Username() string - Hostname() string - ConfigFilename() string - - BlobStorage() blob.Storage - ContentManager() *content.WriteManager - - repo.Writer - - DeriveKey(purpose []byte, keyLength int) []byte - - ListActiveSessions(ctx context.Context) (map[content.SessionID]*content.SessionInfo, error) -} - // Supported maintenance modes. const ( ModeNone Mode = "none" @@ -55,8 +38,8 @@ type MaintainableRepository interface { ) // shouldRun returns Mode if repository is due for periodic maintenance. -func shouldRun(ctx context.Context, rep MaintainableRepository, p *Params) (Mode, error) { - if myUsername := rep.Username() + "@" + rep.Hostname(); p.Owner != myUsername { +func shouldRun(ctx context.Context, rep repo.DirectRepository, p *Params) (Mode, error) { + if myUsername := rep.ClientOptions().UsernameAtHost(); p.Owner != myUsername { log(ctx).Debugf("maintenance owned by another user '%v'", p.Owner) return ModeNone, nil } @@ -127,7 +110,7 @@ func updateSchedule(ctx context.Context, runParams RunParameters) error { // It is generated by RunExclusive and can't be create outside of its package and // is required to ensure all maintenance tasks run under an exclusive lock. type RunParameters struct { - rep MaintainableRepository + rep repo.DirectRepositoryWriter Mode Mode @@ -146,14 +129,14 @@ func (e NotOwnedError) Error() string { // RunExclusive runs the provided callback if the maintenance is owned by local user and // lock can be acquired. Lock is passed to the function, which ensures that every call to Run() // is within the exclusive context. -func RunExclusive(ctx context.Context, rep MaintainableRepository, mode Mode, force bool, cb func(runParams RunParameters) error) error { +func RunExclusive(ctx context.Context, rep repo.DirectRepositoryWriter, mode Mode, force bool, cb func(runParams RunParameters) error) error { p, err := GetParams(ctx, rep) if err != nil { return errors.Wrap(err, "unable to get maintenance params") } if !force { - if myUsername := rep.Username() + "@" + rep.Hostname(); p.Owner != myUsername { + if myUsername := rep.ClientOptions().UsernameAtHost(); p.Owner != myUsername { return NotOwnedError{p.Owner} } } diff --git a/repo/maintenance/maintenance_schedule.go b/repo/maintenance/maintenance_schedule.go index 0a5889428..7d2e7a65f 100644 --- a/repo/maintenance/maintenance_schedule.go +++ b/repo/maintenance/maintenance_schedule.go @@ -11,6 +11,7 @@ "github.com/pkg/errors" "github.com/kopia/kopia/internal/gather" + "github.com/kopia/kopia/repo" "github.com/kopia/kopia/repo/blob" ) @@ -59,7 +60,7 @@ func (s *Schedule) ReportRun(runType string, info RunInfo) { s.Runs[runType] = history } -func getAES256GCM(rep MaintainableRepository) (cipher.AEAD, error) { +func getAES256GCM(rep repo.DirectRepository) (cipher.AEAD, error) { c, err := aes.NewCipher(rep.DeriveKey(maintenanceScheduleKeyPurpose, maintenanceScheduleKeySize)) if err != nil { return nil, errors.Wrap(err, "unable to create AES-256 cipher") @@ -69,9 +70,9 @@ func getAES256GCM(rep MaintainableRepository) (cipher.AEAD, error) { } // GetSchedule gets the scheduled maintenance times. -func GetSchedule(ctx context.Context, rep MaintainableRepository) (*Schedule, error) { +func GetSchedule(ctx context.Context, rep repo.DirectRepository) (*Schedule, error) { // read - v, err := rep.BlobStorage().GetBlob(ctx, maintenanceScheduleBlobID, 0, -1) + v, err := rep.BlobReader().GetBlob(ctx, maintenanceScheduleBlobID, 0, -1) if errors.Is(err, blob.ErrBlobNotFound) { return &Schedule{}, nil } @@ -105,7 +106,7 @@ func GetSchedule(ctx context.Context, rep MaintainableRepository) (*Schedule, er } // SetSchedule updates scheduled maintenance times. -func SetSchedule(ctx context.Context, rep MaintainableRepository, s *Schedule) error { +func SetSchedule(ctx context.Context, rep repo.DirectRepositoryWriter, s *Schedule) error { // encode JSON v, err := json.Marshal(s) if err != nil { @@ -131,7 +132,7 @@ func SetSchedule(ctx context.Context, rep MaintainableRepository, s *Schedule) e } // ReportRun reports timing of a maintenance run and persists it in repository. -func ReportRun(ctx context.Context, rep MaintainableRepository, runType string, run func() error) error { +func ReportRun(ctx context.Context, rep repo.DirectRepositoryWriter, runType string, run func() error) error { ri := RunInfo{ Start: rep.Time(), } diff --git a/repo/maintenance/maintenance_schedule_test.go b/repo/maintenance/maintenance_schedule_test.go index f58016d51..a58c16262 100644 --- a/repo/maintenance/maintenance_schedule_test.go +++ b/repo/maintenance/maintenance_schedule_test.go @@ -17,7 +17,7 @@ func TestMaintenanceSchedule(t *testing.T) { var env repotesting.Environment defer env.Setup(t).Close(ctx, t) - s, err := GetSchedule(ctx, env.Repository) + s, err := GetSchedule(ctx, env.RepositoryWriter) if err != nil { t.Fatalf("err: %v", err) } @@ -38,11 +38,11 @@ func TestMaintenanceSchedule(t *testing.T) { Success: true, }) - if err = SetSchedule(ctx, env.Repository, s); err != nil { + if err = SetSchedule(ctx, env.RepositoryWriter, s); err != nil { t.Fatalf("unable to set schedule: %v", err) } - s2, err := GetSchedule(ctx, env.Repository) + s2, err := GetSchedule(ctx, env.RepositoryWriter) if err != nil { t.Fatalf("unable to get schedule: %v", err) } diff --git a/repo/manifest/committed_manifest_manager.go b/repo/manifest/committed_manifest_manager.go index 562c681e9..b10d673d0 100644 --- a/repo/manifest/committed_manifest_manager.go +++ b/repo/manifest/committed_manifest_manager.go @@ -91,11 +91,15 @@ func (m *committedManifestManager) writeEntriesLocked(ctx context.Context, entri return map[content.ID]bool{contentID: true}, nil } -func (m *committedManifestManager) refresh(ctx context.Context) error { +func (m *committedManifestManager) invalidate() error { m.lock() defer m.unlock() - return m.loadCommittedContentsLocked(ctx) + m.initialized = false + m.committedContentIDs = map[content.ID]bool{} + m.committedEntries = map[ID]*manifestEntry{} + + return nil } func (m *committedManifestManager) loadCommittedContentsLocked(ctx context.Context) error { diff --git a/repo/manifest/manifest_manager.go b/repo/manifest/manifest_manager.go index 550105f63..06d1f43b8 100644 --- a/repo/manifest/manifest_manager.go +++ b/repo/manifest/manifest_manager.go @@ -241,7 +241,7 @@ func (m *Manager) Delete(ctx context.Context, id ID) error { // Refresh updates the committed contents from the underlying storage. func (m *Manager) Refresh(ctx context.Context) error { - return m.committed.refresh(ctx) + return m.committed.invalidate() } // Compact performs compaction of manifest contents. diff --git a/repo/open.go b/repo/open.go index f91646c29..248fb0029 100644 --- a/repo/open.go +++ b/repo/open.go @@ -83,7 +83,7 @@ func Open(ctx context.Context, configFile, password string, options *Options) (r } // openDirect opens the repository that directly manipulates blob storage.. -func openDirect(ctx context.Context, configFile string, lc *LocalConfig, password string, options *Options) (rep *DirectRepository, err error) { +func openDirect(ctx context.Context, configFile string, lc *LocalConfig, password string, options *Options) (rep Repository, err error) { if lc.Caching.CacheDirectory != "" && !filepath.IsAbs(lc.Caching.CacheDirectory) { lc.Caching.CacheDirectory = filepath.Join(filepath.Dir(configFile), lc.Caching.CacheDirectory) } @@ -105,20 +105,17 @@ func openDirect(ctx context.Context, configFile string, lc *LocalConfig, passwor st = readonly.NewWrapper(st) } - r, err := OpenWithConfig(ctx, st, lc, password, options, lc.Caching) + r, err := openWithConfig(ctx, st, lc, password, options, lc.Caching, configFile) if err != nil { st.Close(ctx) //nolint:errcheck return nil, err } - r.cliOpts = lc.ClientOptions.ApplyDefaults(ctx, "Repository in "+st.DisplayName()) - r.ConfigFile = configFile - return r, nil } -// OpenWithConfig opens the repository with a given configuration, avoiding the need for a config file. -func OpenWithConfig(ctx context.Context, st blob.Storage, lc *LocalConfig, password string, options *Options, caching *content.CachingOptions) (*DirectRepository, error) { +// openWithConfig opens the repository with a given configuration, avoiding the need for a config file. +func openWithConfig(ctx context.Context, st blob.Storage, lc *LocalConfig, password string, options *Options, caching *content.CachingOptions, configFile string) (DirectRepository, error) { caching = caching.CloneOrDefault() // Read format blob, potentially from cache. @@ -186,20 +183,21 @@ func OpenWithConfig(ctx context.Context, st blob.Storage, lc *LocalConfig, passw return nil, errors.Wrap(err, "unable to open manifests") } - dr := &DirectRepository{ - Cache: *caching, - Content: cm, - Objects: om, - Blobs: st, - Manifests: manifests, - UniqueID: f.UniqueID, - - sharedContentManager: scm, - - formatBlob: f, - masterKey: masterKey, - timeNow: cmOpts.TimeNow, - + dr := &directRepository{ + cmgr: cm, + omgr: om, + blobs: st, + mmgr: manifests, + sm: scm, + directRepositoryParameters: directRepositoryParameters{ + uniqueID: f.UniqueID, + cachingOptions: *caching, + formatBlob: f, + masterKey: masterKey, + timeNow: cmOpts.TimeNow, + cliOpts: lc.ClientOptions.ApplyDefaults(ctx, "Repository in "+st.DisplayName()), + configFile: configFile, + }, closed: make(chan struct{}), } @@ -237,14 +235,14 @@ func writeCacheMarker(cacheDir string) error { return f.Close() } -// SetCachingConfig changes caching configuration for a given repository. -func (r *DirectRepository) SetCachingConfig(ctx context.Context, opt *content.CachingOptions) error { - lc, err := loadConfigFromFile(r.ConfigFile) +// SetCachingOptions changes caching configuration for a given repository. +func (r *directRepository) SetCachingOptions(ctx context.Context, opt *content.CachingOptions) error { + lc, err := loadConfigFromFile(r.configFile) if err != nil { return err } - if err = setupCaching(ctx, r.ConfigFile, lc, opt, r.UniqueID); err != nil { + if err = setupCaching(ctx, r.configFile, lc, opt, r.uniqueID); err != nil { return errors.Wrap(err, "unable to set up caching") } @@ -253,7 +251,7 @@ func (r *DirectRepository) SetCachingConfig(ctx context.Context, opt *content.Ca return errors.Wrap(err, "error marshaling JSON") } - if err := ioutil.WriteFile(r.ConfigFile, d, 0o600); err != nil { + if err := ioutil.WriteFile(r.configFile, d, 0o600); err != nil { return nil } diff --git a/repo/repository.go b/repo/repository.go index 6d771e99d..bb5f37c4e 100644 --- a/repo/repository.go +++ b/repo/repository.go @@ -13,8 +13,8 @@ "github.com/kopia/kopia/repo/object" ) -// Reader provides methods to read from a repository. -type Reader interface { +// Repository exposes public API of Kopia repository, including objects and manifests. +type Repository interface { OpenObject(ctx context.Context, id object.ID) (object.Reader, error) VerifyObject(ctx context.Context, id object.ID) ([]content.ID, error) @@ -23,11 +23,18 @@ type Reader interface { Time() time.Time ClientOptions() ClientOptions + + NewWriter(ctx context.Context, purpose string) (RepositoryWriter, error) + + UpdateDescription(d string) + + Refresh(ctx context.Context) error + Close(ctx context.Context) error } -// Writer provides methods to write to a repository. -type Writer interface { - Reader +// RepositoryWriter provides methods to write to a repository. +type RepositoryWriter interface { + Repository NewObjectWriter(ctx context.Context, opt object.WriterOptions) object.Writer PutManifest(ctx context.Context, labels map[string]string, payload interface{}) (manifest.ID, error) @@ -36,117 +43,169 @@ type Writer interface { Flush(ctx context.Context) error } -// Repository exposes public API of Kopia repository, including objects and manifests. -type Repository interface { - Reader - Writer +// DirectRepository provides additional low-level repository functionality. +type DirectRepository interface { + Repository - UpdateDescription(d string) + ObjectFormat() object.Format + BlobReader() blob.Reader + ContentReader() content.Reader + IndexBlobReader() content.IndexBlobReader - Refresh(ctx context.Context) error - Close(ctx context.Context) error + NewDirectWriter(ctx context.Context, purpose string) (DirectRepositoryWriter, error) + + // misc + UniqueID() []byte + ConfigFilename() string + DeriveKey(purpose []byte, keyLength int) []byte + Token(password string) (string, error) + + CachingOptions() *content.CachingOptions + SetCachingOptions(ctx context.Context, opt *content.CachingOptions) error } -// DirectRepository is an implementation of repository that directly manipulates underlying storage. -type DirectRepository struct { - Blobs blob.Storage - Content *content.WriteManager - Objects *object.Manager - Manifests *manifest.Manager - Cache content.CachingOptions - UniqueID []byte +// DirectRepositoryWriter provides low-level write access to the repository. +type DirectRepositoryWriter interface { + RepositoryWriter + DirectRepository - sharedContentManager *content.SharedManager + BlobStorage() blob.Storage + ContentManager() *content.WriteManager + Upgrade(ctx context.Context) error +} - ConfigFile string +type directRepositoryParameters struct { + uniqueID []byte + configFile string + cachingOptions content.CachingOptions + cliOpts ClientOptions + timeNow func() time.Time + formatBlob *formatBlob + masterKey []byte +} - cliOpts ClientOptions +// directRepository is an implementation of repository that directly manipulates underlying storage. +type directRepository struct { + directRepositoryParameters - timeNow func() time.Time - formatBlob *formatBlob - masterKey []byte + blobs blob.Storage + cmgr *content.WriteManager + omgr *object.Manager + mmgr *manifest.Manager + sm *content.SharedManager closed chan struct{} } // DeriveKey derives encryption key of the provided length from the master key. -func (r *DirectRepository) DeriveKey(purpose []byte, keyLength int) []byte { - return deriveKeyFromMasterKey(r.masterKey, r.UniqueID, purpose, keyLength) +func (r *directRepository) DeriveKey(purpose []byte, keyLength int) []byte { + return deriveKeyFromMasterKey(r.masterKey, r.uniqueID, purpose, keyLength) } // ClientOptions returns client options. -func (r *DirectRepository) ClientOptions() ClientOptions { +func (r *directRepository) ClientOptions() ClientOptions { return r.cliOpts } -// Hostname returns the hostname that connected to the repository. -func (r *DirectRepository) Hostname() string { return r.cliOpts.Hostname } - -// Username returns the username that's connect to the repository. -func (r *DirectRepository) Username() string { return r.cliOpts.Username } - // BlobStorage returns the blob storage. -func (r *DirectRepository) BlobStorage() blob.Storage { - return r.Blobs +func (r *directRepository) BlobStorage() blob.Storage { + return r.blobs } // ContentManager returns the content manager. -func (r *DirectRepository) ContentManager() *content.WriteManager { - return r.Content +func (r *directRepository) ContentManager() *content.WriteManager { + return r.cmgr } // ConfigFilename returns the name of the configuration file. -func (r *DirectRepository) ConfigFilename() string { - return r.ConfigFile -} - -// OpenObject opens the reader for a given object, returns object.ErrNotFound. -func (r *DirectRepository) OpenObject(ctx context.Context, id object.ID) (object.Reader, error) { - return object.Open(ctx, r.Content, id) +func (r *directRepository) ConfigFilename() string { + return r.configFile } // NewObjectWriter creates an object writer. -func (r *DirectRepository) NewObjectWriter(ctx context.Context, opt object.WriterOptions) object.Writer { - return r.Objects.NewWriter(ctx, opt) +func (r *directRepository) NewObjectWriter(ctx context.Context, opt object.WriterOptions) object.Writer { + return r.omgr.NewWriter(ctx, opt) +} + +// OpenObject opens the reader for a given object, returns object.ErrNotFound. +func (r *directRepository) OpenObject(ctx context.Context, id object.ID) (object.Reader, error) { + return object.Open(ctx, r.cmgr, id) } // VerifyObject verifies that the given object is stored properly in a repository and returns backing content IDs. -func (r *DirectRepository) VerifyObject(ctx context.Context, id object.ID) ([]content.ID, error) { - return object.VerifyObject(ctx, r.Content, id) +func (r *directRepository) VerifyObject(ctx context.Context, id object.ID) ([]content.ID, error) { + return object.VerifyObject(ctx, r.cmgr, id) } // GetManifest returns the given manifest data and metadata. -func (r *DirectRepository) GetManifest(ctx context.Context, id manifest.ID, data interface{}) (*manifest.EntryMetadata, error) { - return r.Manifests.Get(ctx, id, data) +func (r *directRepository) GetManifest(ctx context.Context, id manifest.ID, data interface{}) (*manifest.EntryMetadata, error) { + return r.mmgr.Get(ctx, id, data) } // PutManifest saves the given manifest payload with a set of labels. -func (r *DirectRepository) PutManifest(ctx context.Context, labels map[string]string, payload interface{}) (manifest.ID, error) { - return r.Manifests.Put(ctx, labels, payload) +func (r *directRepository) PutManifest(ctx context.Context, labels map[string]string, payload interface{}) (manifest.ID, error) { + return r.mmgr.Put(ctx, labels, payload) } // FindManifests returns metadata for manifests matching given set of labels. -func (r *DirectRepository) FindManifests(ctx context.Context, labels map[string]string) ([]*manifest.EntryMetadata, error) { - return r.Manifests.Find(ctx, labels) +func (r *directRepository) FindManifests(ctx context.Context, labels map[string]string) ([]*manifest.EntryMetadata, error) { + return r.mmgr.Find(ctx, labels) } // DeleteManifest deletes the manifest with a given ID. -func (r *DirectRepository) DeleteManifest(ctx context.Context, id manifest.ID) error { - return r.Manifests.Delete(ctx, id) +func (r *directRepository) DeleteManifest(ctx context.Context, id manifest.ID) error { + return r.mmgr.Delete(ctx, id) } // ListActiveSessions returns the map of active sessions. -func (r *DirectRepository) ListActiveSessions(ctx context.Context) (map[content.SessionID]*content.SessionInfo, error) { - return r.Content.ListActiveSessions(ctx) +func (r *directRepository) ListActiveSessions(ctx context.Context) (map[content.SessionID]*content.SessionInfo, error) { + return r.cmgr.ListActiveSessions(ctx) } // UpdateDescription updates the description of a connected repository. -func (r *DirectRepository) UpdateDescription(d string) { +func (r *directRepository) UpdateDescription(d string) { r.cliOpts.Description = d } +// NewWriter returns new RepositoryWriter session for repository. +func (r *directRepository) NewWriter(ctx context.Context, purpose string) (RepositoryWriter, error) { + return r.NewDirectWriter(ctx, purpose) +} + +// NewDirectWriter returns new DirectRepositoryWriter session for repository. +func (r *directRepository) NewDirectWriter(ctx context.Context, purpose string) (DirectRepositoryWriter, error) { + cmgr := content.NewWriteManager(r.sm, content.SessionOptions{ + SessionUser: r.cliOpts.Username, + SessionHost: r.cliOpts.Hostname, + }) + + mmgr, err := manifest.NewManager(ctx, cmgr, manifest.ManagerOptions{ + TimeNow: r.timeNow, + }) + if err != nil { + return nil, errors.Wrap(err, "error creating manifest manager") + } + + omgr, err := object.NewObjectManager(ctx, cmgr, r.omgr.Format) + if err != nil { + return nil, errors.Wrap(err, "error creating object manager") + } + + w := &directRepository{ + directRepositoryParameters: r.directRepositoryParameters, + blobs: r.blobs, + cmgr: cmgr, + omgr: omgr, + mmgr: mmgr, + sm: r.sm, + closed: make(chan struct{}), + } + + return w, nil +} + // Close closes the repository and releases all resources. -func (r *DirectRepository) Close(ctx context.Context) error { +func (r *directRepository) Close(ctx context.Context) error { select { case <-r.closed: // already closed @@ -155,60 +214,75 @@ func (r *DirectRepository) Close(ctx context.Context) error { default: } - if err := r.Flush(ctx); err != nil { - return errors.Wrap(err, "error flushing") - } - - if err := r.Objects.Close(); err != nil { + if err := r.omgr.Close(); err != nil { return errors.Wrap(err, "error closing object manager") } - if err := r.Content.Close(ctx); err != nil { + // this will release shared manager and MAY release blob.Store (on last outstanding reference). + if err := r.cmgr.Close(ctx); err != nil { return errors.Wrap(err, "error closing content-addressable storage manager") } - if err := r.Blobs.Close(ctx); err != nil { - return errors.Wrap(err, "error closing blob storage") - } - close(r.closed) return nil } // Flush waits for all in-flight writes to complete. -func (r *DirectRepository) Flush(ctx context.Context) error { - if err := r.Manifests.Flush(ctx); err != nil { +func (r *directRepository) Flush(ctx context.Context) error { + if err := r.mmgr.Flush(ctx); err != nil { return errors.Wrap(err, "error flushing manifests") } - return r.Content.Flush(ctx) + return r.cmgr.Flush(ctx) +} + +// CachingOptions returns caching options. +func (r *directRepository) CachingOptions() *content.CachingOptions { + return r.cachingOptions.CloneOrDefault() +} + +// ObjectFormat returns the object format. +func (r *directRepository) ObjectFormat() object.Format { + return r.omgr.Format +} + +// UniqueID returns unique repository ID from which many keys and secrets are derived. +func (r *directRepository) UniqueID() []byte { + return r.uniqueID +} + +// BlobReader returns the blob reader. +func (r *directRepository) BlobReader() blob.Reader { + return r.blobs +} + +// ContentReader returns the content reader. +func (r *directRepository) ContentReader() content.Reader { + return r.cmgr +} + +// IndexBlobReader returns the index blob reader. +func (r *directRepository) IndexBlobReader() content.IndexBlobReader { + return r.cmgr } // Refresh periodically makes external changes visible to repository. -func (r *DirectRepository) Refresh(ctx context.Context) error { - updated, err := r.Content.Refresh(ctx) +func (r *directRepository) Refresh(ctx context.Context) error { + _, err := r.cmgr.Refresh(ctx) if err != nil { return errors.Wrap(err, "error refreshing content index") } - if !updated { - return nil - } - - log(ctx).Debugf("content index refreshed") - - if err := r.Manifests.Refresh(ctx); err != nil { + if err := r.mmgr.Refresh(ctx); err != nil { return errors.Wrap(err, "error reloading manifests") } - log(ctx).Debugf("manifests refreshed") - return nil } // RefreshPeriodically periodically refreshes the repository to reflect the changes made by other hosts. -func (r *DirectRepository) RefreshPeriodically(ctx context.Context, interval time.Duration) { +func (r *directRepository) RefreshPeriodically(ctx context.Context, interval time.Duration) { for { select { case <-r.closed: @@ -227,10 +301,56 @@ func (r *DirectRepository) RefreshPeriodically(ctx context.Context, interval tim } // Time returns the current local time for the repo. -func (r *DirectRepository) Time() time.Time { +func (r *directRepository) Time() time.Time { return defaultTime(r.timeNow)() } +// WriteSessionOptions describes options for a write session. +type WriteSessionOptions struct { + Purpose string + FlushOnFailure bool // whether to flush regardless of write sessionr result. +} + +// WriteSession executes the provided callback in a repository writer created for the purpose and flushes writes. +func WriteSession(ctx context.Context, r Repository, opt WriteSessionOptions, cb func(w RepositoryWriter) error) error { + w, err := r.NewWriter(ctx, opt.Purpose) + if err != nil { + return errors.Wrap(err, "unable to create writer") + } + + return handleWriteSessionResult(ctx, r, w, opt, cb(w)) +} + +// DirectWriteSession executes the provided callback in a DirectRepositoryWriter created for the purpose and flushes writes. +func DirectWriteSession(ctx context.Context, r DirectRepository, opt WriteSessionOptions, cb func(dw DirectRepositoryWriter) error) error { + w, err := r.NewDirectWriter(ctx, opt.Purpose) + if err != nil { + return errors.Wrap(err, "unable to create direct writer") + } + + return handleWriteSessionResult(ctx, r, w, opt, cb(w)) +} + +func handleWriteSessionResult(ctx context.Context, r Repository, w RepositoryWriter, opt WriteSessionOptions, resultErr error) error { + defer func() { + if err := w.Close(ctx); err != nil { + log(ctx).Warningf("error closing writer: %v", err) + } + }() + + if resultErr == nil || opt.FlushOnFailure { + if err := w.Flush(ctx); err != nil { + return errors.Wrap(err, "error flushing writer") + } + } + + if err := r.Refresh(ctx); err != nil { + return errors.Wrap(err, "error refreshing repository") + } + + return resultErr +} + func defaultTime(f func() time.Time) func() time.Time { if f != nil { return f @@ -238,3 +358,5 @@ func defaultTime(f func() time.Time) func() time.Time { return clock.Now } + +var _ DirectRepositoryWriter = (*directRepository)(nil) diff --git a/repo/repository_test.go b/repo/repository_test.go index 0b060bb19..427e7f0a1 100644 --- a/repo/repository_test.go +++ b/repo/repository_test.go @@ -35,7 +35,7 @@ func TestWriters(t *testing.T) { var env repotesting.Environment defer env.Setup(t).Close(ctx, t) - writer := env.Repository.NewObjectWriter(ctx, object.WriterOptions{}) + writer := env.RepositoryWriter.NewObjectWriter(ctx, object.WriterOptions{}) if _, err := writer.Write(c.data); err != nil { t.Fatalf("write error: %v", err) } @@ -50,7 +50,7 @@ func TestWriters(t *testing.T) { t.Errorf("incorrect result for %v, expected: %v got: %v", c.data, c.objectID.String(), result.String()) } - env.Repository.Content.Flush(ctx) + env.RepositoryWriter.ContentManager().Flush(ctx) } } @@ -65,7 +65,7 @@ func TestWriterCompleteChunkInTwoWrites(t *testing.T) { defer env.Setup(t).Close(ctx, t) b := make([]byte, 100) - writer := env.Repository.NewObjectWriter(ctx, object.WriterOptions{}) + writer := env.RepositoryWriter.NewObjectWriter(ctx, object.WriterOptions{}) writer.Write(b[0:50]) writer.Write(b[0:50]) result, err := writer.Result() @@ -85,19 +85,19 @@ func TestPackingSimple(t *testing.T) { content2 := "hi, how are you?" content3 := "thank you!" - oid1a := writeObject(ctx, t, env.Repository, []byte(content1), "packed-object-1a") - oid1b := writeObject(ctx, t, env.Repository, []byte(content1), "packed-object-1b") - oid2a := writeObject(ctx, t, env.Repository, []byte(content2), "packed-object-2a") - oid2b := writeObject(ctx, t, env.Repository, []byte(content2), "packed-object-2b") + oid1a := writeObject(ctx, t, env.RepositoryWriter, []byte(content1), "packed-object-1a") + oid1b := writeObject(ctx, t, env.RepositoryWriter, []byte(content1), "packed-object-1b") + oid2a := writeObject(ctx, t, env.RepositoryWriter, []byte(content2), "packed-object-2a") + oid2b := writeObject(ctx, t, env.RepositoryWriter, []byte(content2), "packed-object-2b") - oid3a := writeObject(ctx, t, env.Repository, []byte(content3), "packed-object-3a") - oid3b := writeObject(ctx, t, env.Repository, []byte(content3), "packed-object-3b") - verify(ctx, t, env.Repository, oid1a, []byte(content1), "packed-object-1") - verify(ctx, t, env.Repository, oid2a, []byte(content2), "packed-object-2") - oid2c := writeObject(ctx, t, env.Repository, []byte(content2), "packed-object-2c") - oid1c := writeObject(ctx, t, env.Repository, []byte(content1), "packed-object-1c") + oid3a := writeObject(ctx, t, env.RepositoryWriter, []byte(content3), "packed-object-3a") + oid3b := writeObject(ctx, t, env.RepositoryWriter, []byte(content3), "packed-object-3b") + verify(ctx, t, env.RepositoryWriter, oid1a, []byte(content1), "packed-object-1") + verify(ctx, t, env.RepositoryWriter, oid2a, []byte(content2), "packed-object-2") + oid2c := writeObject(ctx, t, env.RepositoryWriter, []byte(content2), "packed-object-2c") + oid1c := writeObject(ctx, t, env.RepositoryWriter, []byte(content1), "packed-object-1c") - env.Repository.Content.Flush(ctx) + env.RepositoryWriter.ContentManager().Flush(ctx) if got, want := oid1a.String(), oid1b.String(); got != want { t.Errorf("oid1a(%q) != oid1b(%q)", got, want) @@ -123,29 +123,29 @@ func TestPackingSimple(t *testing.T) { env.MustReopen(t) - verify(ctx, t, env.Repository, oid1a, []byte(content1), "packed-object-1") - verify(ctx, t, env.Repository, oid2a, []byte(content2), "packed-object-2") - verify(ctx, t, env.Repository, oid3a, []byte(content3), "packed-object-3") + verify(ctx, t, env.RepositoryWriter, oid1a, []byte(content1), "packed-object-1") + verify(ctx, t, env.RepositoryWriter, oid2a, []byte(content2), "packed-object-2") + verify(ctx, t, env.RepositoryWriter, oid3a, []byte(content3), "packed-object-3") - if err := env.Repository.Content.CompactIndexes(ctx, content.CompactOptions{MaxSmallBlobs: 1}); err != nil { + if err := env.RepositoryWriter.ContentManager().CompactIndexes(ctx, content.CompactOptions{MaxSmallBlobs: 1}); err != nil { t.Errorf("optimize error: %v", err) } env.MustReopen(t) - verify(ctx, t, env.Repository, oid1a, []byte(content1), "packed-object-1") - verify(ctx, t, env.Repository, oid2a, []byte(content2), "packed-object-2") - verify(ctx, t, env.Repository, oid3a, []byte(content3), "packed-object-3") + verify(ctx, t, env.RepositoryWriter, oid1a, []byte(content1), "packed-object-1") + verify(ctx, t, env.RepositoryWriter, oid2a, []byte(content2), "packed-object-2") + verify(ctx, t, env.RepositoryWriter, oid3a, []byte(content3), "packed-object-3") - if err := env.Repository.Content.CompactIndexes(ctx, content.CompactOptions{MaxSmallBlobs: 1}); err != nil { + if err := env.RepositoryWriter.ContentManager().CompactIndexes(ctx, content.CompactOptions{MaxSmallBlobs: 1}); err != nil { t.Errorf("optimize error: %v", err) } env.MustReopen(t) - verify(ctx, t, env.Repository, oid1a, []byte(content1), "packed-object-1") - verify(ctx, t, env.Repository, oid2a, []byte(content2), "packed-object-2") - verify(ctx, t, env.Repository, oid3a, []byte(content3), "packed-object-3") + verify(ctx, t, env.RepositoryWriter, oid1a, []byte(content1), "packed-object-1") + verify(ctx, t, env.RepositoryWriter, oid2a, []byte(content2), "packed-object-2") + verify(ctx, t, env.RepositoryWriter, oid3a, []byte(content3), "packed-object-3") } func TestHMAC(t *testing.T) { @@ -156,7 +156,7 @@ func TestHMAC(t *testing.T) { c := bytes.Repeat([]byte{0xcd}, 50) - w := env.Repository.NewObjectWriter(ctx, object.WriterOptions{}) + w := env.RepositoryWriter.NewObjectWriter(ctx, object.WriterOptions{}) w.Write(c) result, err := w.Result() @@ -171,11 +171,11 @@ func TestUpgrade(t *testing.T) { ctx := testlogging.Context(t) defer env.Setup(t).Close(ctx, t) - if err := env.Repository.Upgrade(ctx); err != nil { + if err := env.RepositoryWriter.Upgrade(ctx); err != nil { t.Errorf("upgrade error: %v", err) } - if err := env.Repository.Upgrade(ctx); err != nil { + if err := env.RepositoryWriter.Upgrade(ctx); err != nil { t.Errorf("2nd upgrade error: %v", err) } } @@ -191,13 +191,13 @@ func TestReaderStoredBlockNotFound(t *testing.T) { t.Errorf("cannot parse object ID: %v", err) } - reader, err := env.Repository.OpenObject(ctx, objectID) + reader, err := env.RepositoryWriter.OpenObject(ctx, objectID) if !errors.Is(err, object.ErrObjectNotFound) || reader != nil { t.Errorf("unexpected result: reader: %v err: %v", reader, err) } } -func writeObject(ctx context.Context, t *testing.T, rep repo.Repository, data []byte, testCaseID string) object.ID { +func writeObject(ctx context.Context, t *testing.T, rep repo.RepositoryWriter, data []byte, testCaseID string) object.ID { t.Helper() w := rep.NewObjectWriter(ctx, object.WriterOptions{}) @@ -295,7 +295,7 @@ func TestFormats(t *testing.T) { for k, v := range c.oids { bytesToWrite := []byte(k) - w := env.Repository.NewObjectWriter(ctx, object.WriterOptions{}) + w := env.RepositoryWriter.NewObjectWriter(ctx, object.WriterOptions{}) w.Write(bytesToWrite) oid, err := w.Result() @@ -307,7 +307,7 @@ func TestFormats(t *testing.T) { t.Errorf("invalid oid for #%v\ngot:\n%#v\nexpected:\n%#v", caseIndex, oid.String(), v.String()) } - rc, err := env.Repository.OpenObject(ctx, oid) + rc, err := env.RepositoryWriter.OpenObject(ctx, oid) if err != nil { t.Errorf("open failed: %v", err) continue @@ -324,3 +324,177 @@ func TestFormats(t *testing.T) { } } } + +func TestWriterScope(t *testing.T) { + var env repotesting.Environment + + ctx := context.Background() + defer env.Setup(t, repotesting.Options{}).Close(ctx, t) + + rep := env.Repository // read-only + + lw := rep.(repo.RepositoryWriter) + + // w1, w2, w3 are indepdendent sessions. + w1, err := rep.NewWriter(ctx, "writer1") + must(t, err) + + defer w1.Close(ctx) + + w2, err := rep.NewWriter(ctx, "writer2") + must(t, err) + + defer w2.Close(ctx) + + w3, err := rep.NewWriter(ctx, "writer3") + must(t, err) + + defer w3.Close(ctx) + + o1Data := []byte{1, 2, 3} + o2Data := []byte{2, 3, 4} + o3Data := []byte{3, 4, 5} + o4Data := []byte{4, 5, 6} + + o1 := writeObject(ctx, t, w1, o1Data, "o1") + o2 := writeObject(ctx, t, w2, o2Data, "o2") + o3 := writeObject(ctx, t, w3, o3Data, "o3") + o4 := writeObject(ctx, t, lw, o4Data, "o4") + + // each writer can see their own data but not others, except for 'lw' and 'rep' which are + // one and the same. + verify(ctx, t, w1, o1, o1Data, "o1-w1") + verifyNotFound(ctx, t, w2, o1, "o1-w2") + verifyNotFound(ctx, t, w3, o1, "o1-w3") + verifyNotFound(ctx, t, lw, o1, "o1-lw") + verifyNotFound(ctx, t, rep, o1, "o1-rep") + + verifyNotFound(ctx, t, w1, o2, "o2-w1") + verify(ctx, t, w2, o2, o2Data, "o2-w2") + verifyNotFound(ctx, t, w3, o2, "o2-w3") + verifyNotFound(ctx, t, lw, o2, "o2-lw") + verifyNotFound(ctx, t, rep, o2, "o2-rep") + + verifyNotFound(ctx, t, w1, o3, "o3-w1") + verifyNotFound(ctx, t, w2, o3, "o3-w2") + verify(ctx, t, w3, o3, o3Data, "o3-w3") + verifyNotFound(ctx, t, lw, o3, "o3-lw") + verifyNotFound(ctx, t, rep, o3, "o3-rep") + + verifyNotFound(ctx, t, w1, o4, "o4-w1") + verifyNotFound(ctx, t, w2, o4, "o4-w2") + verifyNotFound(ctx, t, w2, o3, "o4-ww") + verify(ctx, t, lw, o4, o4Data, "o4-lw") + verify(ctx, t, rep, o4, o4Data, "o4-rep") // rep == lw so read is immediately visible + + must(t, w1.Flush(ctx)) + + // after flushing w1, everybody else can now see o1 + verify(ctx, t, w1, o1, o1Data, "o1-w1") + verify(ctx, t, w2, o1, o1Data, "o1-w2") + verify(ctx, t, w3, o1, o1Data, "o1-w3") + verify(ctx, t, lw, o1, o1Data, "o1-lw") + verify(ctx, t, rep, o1, o1Data, "o1-rep") + + must(t, w2.Flush(ctx)) + + // after flushing w2, everybody else can now see o2 + verify(ctx, t, w1, o2, o2Data, "o2-w1") + verify(ctx, t, w2, o2, o2Data, "o2-w2") + verify(ctx, t, w3, o2, o2Data, "o2-w3") + verify(ctx, t, lw, o2, o2Data, "o2-lw") + verify(ctx, t, rep, o2, o2Data, "o2-rep") + + must(t, w3.Flush(ctx)) + must(t, lw.Flush(ctx)) + + verify(ctx, t, w1, o3, o3Data, "o3-w1") + verify(ctx, t, w2, o3, o3Data, "o3-w2") + verify(ctx, t, w3, o3, o3Data, "o3-w3") + verify(ctx, t, lw, o3, o3Data, "o3-lw") + verify(ctx, t, rep, o3, o3Data, "o3-rep") + + verify(ctx, t, w1, o4, o4Data, "o4-w1") + verify(ctx, t, w2, o4, o4Data, "o4-w2") + verify(ctx, t, w3, o4, o4Data, "o4-w3") + verify(ctx, t, lw, o4, o4Data, "o4-lw") + verify(ctx, t, rep, o4, o4Data, "o3-rep") +} + +func TestWriteSessionFlushOnSuccess(t *testing.T) { + var env repotesting.Environment + + ctx := context.Background() + defer env.Setup(t, repotesting.Options{}).Close(ctx, t) + + var oid object.ID + + must(t, repo.WriteSession(ctx, env.Repository, repo.WriteSessionOptions{}, func(w repo.RepositoryWriter) error { + oid = writeObject(ctx, t, w, []byte{1, 2, 3}, "test-1") + return nil + })) + + verify(ctx, t, env.Repository, oid, []byte{1, 2, 3}, "test-1") +} + +func TestWriteSessionNoFlushOnFailure(t *testing.T) { + var env repotesting.Environment + + ctx := context.Background() + defer env.Setup(t, repotesting.Options{}).Close(ctx, t) + + var oid object.ID + + someErr := errors.New("some error") + err := repo.WriteSession(ctx, env.Repository, repo.WriteSessionOptions{}, func(w repo.RepositoryWriter) error { + oid = writeObject(ctx, t, w, []byte{1, 2, 3}, "test-1") + return someErr + }) + + if !errors.Is(err, someErr) { + t.Fatalf("invalid error: %v want %v", err, someErr) + } + + verifyNotFound(ctx, t, env.Repository, oid, "test-1") +} + +func TestWriteSessionFlushOnFailure(t *testing.T) { + var env repotesting.Environment + + ctx := context.Background() + defer env.Setup(t, repotesting.Options{}).Close(ctx, t) + + var oid object.ID + + someErr := errors.New("some error") + err := repo.WriteSession(ctx, env.Repository, repo.WriteSessionOptions{ + FlushOnFailure: true, + }, func(w repo.RepositoryWriter) error { + oid = writeObject(ctx, t, w, []byte{1, 2, 3}, "test-1") + return someErr + }) + + if !errors.Is(err, someErr) { + t.Fatalf("invalid error: %v want %v", err, someErr) + } + + verify(ctx, t, env.Repository, oid, []byte{1, 2, 3}, "test-1") +} + +func verifyNotFound(ctx context.Context, t *testing.T, rep repo.Repository, objectID object.ID, testCaseID string) { + t.Helper() + + _, err := rep.OpenObject(ctx, objectID) + if !errors.Is(err, object.ErrObjectNotFound) { + t.Fatalf("expected not found for %v, got %v", testCaseID, err) + return + } +} + +func must(t *testing.T, err error) { + t.Helper() + + if err != nil { + t.Fatal(err) + } +} diff --git a/repo/token.go b/repo/token.go index 54d97f42b..ed477955f 100644 --- a/repo/token.go +++ b/repo/token.go @@ -17,10 +17,10 @@ type tokenInfo struct { // Token returns an opaque token that contains repository connection information // and optionally the provided password. -func (r *DirectRepository) Token(password string) (string, error) { +func (r *directRepository) Token(password string) (string, error) { ti := &tokenInfo{ Version: "1", - Storage: r.Blobs.ConnectionInfo(), + Storage: r.blobs.ConnectionInfo(), Password: password, } diff --git a/repo/upgrade.go b/repo/upgrade.go index 4f6b8b97e..fa810fbbe 100644 --- a/repo/upgrade.go +++ b/repo/upgrade.go @@ -7,7 +7,7 @@ ) // Upgrade upgrades repository data structures to the latest version. -func (r *DirectRepository) Upgrade(ctx context.Context) error { +func (r *directRepository) Upgrade(ctx context.Context) error { f := r.formatBlob repoConfig, err := f.decryptFormatBytes(r.masterKey) @@ -29,5 +29,5 @@ func (r *DirectRepository) Upgrade(ctx context.Context) error { log(ctx).Infof("writing updated format content...") - return writeFormatBlob(ctx, r.Blobs, f) + return writeFormatBlob(ctx, r.blobs, f) } diff --git a/snapshot/manager.go b/snapshot/manager.go index de775dab2..9fd69d7f5 100644 --- a/snapshot/manager.go +++ b/snapshot/manager.go @@ -27,7 +27,7 @@ var log = logging.GetContextLoggerFunc("kopia/snapshot") // ListSources lists all snapshot sources in a given repository. -func ListSources(ctx context.Context, rep repo.Reader) ([]SourceInfo, error) { +func ListSources(ctx context.Context, rep repo.Repository) ([]SourceInfo, error) { items, err := rep.FindManifests(ctx, map[string]string{ typeKey: ManifestType, }) @@ -70,7 +70,7 @@ func sourceInfoToLabels(si SourceInfo) map[string]string { } // ListSnapshots lists all snapshots for a given source. -func ListSnapshots(ctx context.Context, rep repo.Reader, si SourceInfo) ([]*Manifest, error) { +func ListSnapshots(ctx context.Context, rep repo.Repository, si SourceInfo) ([]*Manifest, error) { entries, err := rep.FindManifests(ctx, sourceInfoToLabels(si)) if err != nil { return nil, errors.Wrap(err, "unable to find manifest entries") @@ -80,7 +80,7 @@ func ListSnapshots(ctx context.Context, rep repo.Reader, si SourceInfo) ([]*Mani } // LoadSnapshot loads and parses a snapshot with a given ID. -func LoadSnapshot(ctx context.Context, rep repo.Reader, manifestID manifest.ID) (*Manifest, error) { +func LoadSnapshot(ctx context.Context, rep repo.Repository, manifestID manifest.ID) (*Manifest, error) { sm := &Manifest{} em, err := rep.GetManifest(ctx, manifestID, sm) @@ -103,7 +103,7 @@ func LoadSnapshot(ctx context.Context, rep repo.Reader, manifestID manifest.ID) } // SaveSnapshot persists given snapshot manifest and returns manifest ID. -func SaveSnapshot(ctx context.Context, rep repo.Writer, man *Manifest) (manifest.ID, error) { +func SaveSnapshot(ctx context.Context, rep repo.RepositoryWriter, man *Manifest) (manifest.ID, error) { if man.Source.Host == "" { return "", errors.New("missing host") } @@ -127,7 +127,7 @@ func SaveSnapshot(ctx context.Context, rep repo.Writer, man *Manifest) (manifest } // LoadSnapshots efficiently loads and parses a given list of snapshot IDs. -func LoadSnapshots(ctx context.Context, rep repo.Reader, manifestIDs []manifest.ID) ([]*Manifest, error) { +func LoadSnapshots(ctx context.Context, rep repo.Repository, manifestIDs []manifest.ID) ([]*Manifest, error) { result := make([]*Manifest, len(manifestIDs)) sem := make(chan bool, loadSnapshotsConcurrency) @@ -164,7 +164,7 @@ func LoadSnapshots(ctx context.Context, rep repo.Reader, manifestIDs []manifest. } // ListSnapshotManifests returns the list of snapshot manifests for a given source or all sources if nil. -func ListSnapshotManifests(ctx context.Context, rep repo.Reader, src *SourceInfo) ([]manifest.ID, error) { +func ListSnapshotManifests(ctx context.Context, rep repo.Repository, src *SourceInfo) ([]manifest.ID, error) { labels := map[string]string{ typeKey: ManifestType, } @@ -182,7 +182,7 @@ func ListSnapshotManifests(ctx context.Context, rep repo.Reader, src *SourceInfo } // FindSnapshotsByRootObjectID returns the list of matching snapshots for a given rootID. -func FindSnapshotsByRootObjectID(ctx context.Context, rep repo.Reader, rootID object.ID) ([]*Manifest, error) { +func FindSnapshotsByRootObjectID(ctx context.Context, rep repo.Repository, rootID object.ID) ([]*Manifest, error) { ids, err := ListSnapshotManifests(ctx, rep, nil) if err != nil { return nil, errors.Wrap(err, "error listing snapshot manifests") diff --git a/snapshot/policy/expire.go b/snapshot/policy/expire.go index 7fa5f0d2b..87e9b026a 100644 --- a/snapshot/policy/expire.go +++ b/snapshot/policy/expire.go @@ -11,7 +11,7 @@ ) // ApplyRetentionPolicy applies retention policy to a given source by deleting expired snapshots. -func ApplyRetentionPolicy(ctx context.Context, rep repo.Writer, sourceInfo snapshot.SourceInfo, reallyDelete bool) ([]*snapshot.Manifest, error) { +func ApplyRetentionPolicy(ctx context.Context, rep repo.RepositoryWriter, sourceInfo snapshot.SourceInfo, reallyDelete bool) ([]*snapshot.Manifest, error) { snapshots, err := snapshot.ListSnapshots(ctx, rep, sourceInfo) if err != nil { return nil, errors.Wrap(err, "error listing snapshots") @@ -33,7 +33,7 @@ func ApplyRetentionPolicy(ctx context.Context, rep repo.Writer, sourceInfo snaps return toDelete, nil } -func getExpiredSnapshots(ctx context.Context, rep repo.Reader, snapshots []*snapshot.Manifest) ([]*snapshot.Manifest, error) { +func getExpiredSnapshots(ctx context.Context, rep repo.Repository, snapshots []*snapshot.Manifest) ([]*snapshot.Manifest, error) { var toDelete []*snapshot.Manifest for _, snapshotGroup := range snapshot.GroupBySource(snapshots) { @@ -48,7 +48,7 @@ func getExpiredSnapshots(ctx context.Context, rep repo.Reader, snapshots []*snap return toDelete, nil } -func getExpiredSnapshotsForSource(ctx context.Context, rep repo.Reader, snapshots []*snapshot.Manifest) ([]*snapshot.Manifest, error) { +func getExpiredSnapshotsForSource(ctx context.Context, rep repo.Repository, snapshots []*snapshot.Manifest) ([]*snapshot.Manifest, error) { src := snapshots[0].Source pol, _, err := GetEffectivePolicy(ctx, rep, src) diff --git a/snapshot/policy/policy_manager.go b/snapshot/policy/policy_manager.go index 697abdc1f..cf163fc57 100644 --- a/snapshot/policy/policy_manager.go +++ b/snapshot/policy/policy_manager.go @@ -23,7 +23,7 @@ // GetEffectivePolicy calculates effective snapshot policy for a given source by combining the source-specifc policy (if any) // with parent policies. The source must contain a path. // Returns the effective policies and all source policies that contributed to that (most specific first). -func GetEffectivePolicy(ctx context.Context, rep repo.Reader, si snapshot.SourceInfo) (effective *Policy, sources []*Policy, e error) { +func GetEffectivePolicy(ctx context.Context, rep repo.Repository, si snapshot.SourceInfo) (effective *Policy, sources []*Policy, e error) { var md []*manifest.EntryMetadata // Find policies applying to paths all the way up to the root. @@ -86,7 +86,7 @@ func GetEffectivePolicy(ctx context.Context, rep repo.Reader, si snapshot.Source } // GetDefinedPolicy returns the policy defined on the provided snapshot.SourceInfo or ErrPolicyNotFound if not present. -func GetDefinedPolicy(ctx context.Context, rep repo.Reader, si snapshot.SourceInfo) (*Policy, error) { +func GetDefinedPolicy(ctx context.Context, rep repo.Repository, si snapshot.SourceInfo) (*Policy, error) { md, err := rep.FindManifests(ctx, labelsForSource(si)) if err != nil { return nil, errors.Wrap(err, "unable to find policy for source") @@ -112,7 +112,7 @@ func GetDefinedPolicy(ctx context.Context, rep repo.Reader, si snapshot.SourceIn } // SetPolicy sets the policy on a given source. -func SetPolicy(ctx context.Context, rep repo.Writer, si snapshot.SourceInfo, pol *Policy) error { +func SetPolicy(ctx context.Context, rep repo.RepositoryWriter, si snapshot.SourceInfo, pol *Policy) error { md, err := rep.FindManifests(ctx, labelsForSource(si)) if err != nil { return errors.Wrapf(err, "unable to load manifests for %v", si) @@ -132,7 +132,7 @@ func SetPolicy(ctx context.Context, rep repo.Writer, si snapshot.SourceInfo, pol } // RemovePolicy removes the policy for a given source. -func RemovePolicy(ctx context.Context, rep repo.Writer, si snapshot.SourceInfo) error { +func RemovePolicy(ctx context.Context, rep repo.RepositoryWriter, si snapshot.SourceInfo) error { md, err := rep.FindManifests(ctx, labelsForSource(si)) if err != nil { return errors.Wrapf(err, "unable to load manifests for %v", si) @@ -148,7 +148,7 @@ func RemovePolicy(ctx context.Context, rep repo.Writer, si snapshot.SourceInfo) } // GetPolicyByID gets the policy for a given unique ID or ErrPolicyNotFound if not found. -func GetPolicyByID(ctx context.Context, rep repo.Reader, id manifest.ID) (*Policy, error) { +func GetPolicyByID(ctx context.Context, rep repo.Repository, id manifest.ID) (*Policy, error) { p := &Policy{} if err := loadPolicyFromManifest(ctx, rep, id, p); err != nil { return nil, err @@ -158,7 +158,7 @@ func GetPolicyByID(ctx context.Context, rep repo.Reader, id manifest.ID) (*Polic } // ListPolicies returns a list of all policies. -func ListPolicies(ctx context.Context, rep repo.Reader) ([]*Policy, error) { +func ListPolicies(ctx context.Context, rep repo.Repository) ([]*Policy, error) { ids, err := rep.FindManifests(ctx, map[string]string{ typeKey: "policy", }) @@ -190,7 +190,7 @@ func (m SubdirectoryPolicyMap) GetPolicyForPath(relativePath string) (*Policy, e } // TreeForSource returns policy Tree for a given source. -func TreeForSource(ctx context.Context, rep repo.Reader, si snapshot.SourceInfo) (*Tree, error) { +func TreeForSource(ctx context.Context, rep repo.Repository, si snapshot.SourceInfo) (*Tree, error) { pols, err := applicablePoliciesForSource(ctx, rep, si) if err != nil { return nil, errors.Wrap(err, "unable to get policies") @@ -199,7 +199,7 @@ func TreeForSource(ctx context.Context, rep repo.Reader, si snapshot.SourceInfo) return BuildTree(pols, DefaultPolicy), nil } -func applicablePoliciesForSource(ctx context.Context, rep repo.Reader, si snapshot.SourceInfo) (map[string]*Policy, error) { +func applicablePoliciesForSource(ctx context.Context, rep repo.Repository, si snapshot.SourceInfo) (map[string]*Policy, error) { result := map[string]*Policy{} pol, _, err := GetEffectivePolicy(ctx, rep, si) @@ -247,7 +247,7 @@ func applicablePoliciesForSource(ctx context.Context, rep repo.Reader, si snapsh return result, nil } -func loadPolicyFromManifest(ctx context.Context, rep repo.Reader, id manifest.ID, pol *Policy) error { +func loadPolicyFromManifest(ctx context.Context, rep repo.Repository, id manifest.ID, pol *Policy) error { md, err := rep.GetManifest(ctx, id, pol) if err != nil { if errors.Is(err, manifest.ErrNotFound) { diff --git a/snapshot/policy/policy_manager_test.go b/snapshot/policy/policy_manager_test.go index 7ca68fbb5..ed4110a5c 100644 --- a/snapshot/policy/policy_manager_test.go +++ b/snapshot/policy/policy_manager_test.go @@ -25,7 +25,7 @@ func TestPolicyManagerInheritanceTest(t *testing.T) { "type": "policy", }) - must(t, SetPolicy(ctx, env.Repository, snapshot.SourceInfo{ + must(t, SetPolicy(ctx, env.RepositoryWriter, snapshot.SourceInfo{ Host: "host-a", }, &Policy{ RetentionPolicy: RetentionPolicy{ @@ -33,7 +33,7 @@ func TestPolicyManagerInheritanceTest(t *testing.T) { }, })) - must(t, SetPolicy(ctx, env.Repository, snapshot.SourceInfo{ + must(t, SetPolicy(ctx, env.RepositoryWriter, snapshot.SourceInfo{ Host: "host-a", UserName: "myuser", Path: "/some/path2", @@ -43,7 +43,7 @@ func TestPolicyManagerInheritanceTest(t *testing.T) { }, })) - must(t, SetPolicy(ctx, env.Repository, snapshot.SourceInfo{ + must(t, SetPolicy(ctx, env.RepositoryWriter, snapshot.SourceInfo{ Host: "host-b", }, &Policy{ RetentionPolicy: RetentionPolicy{ @@ -114,7 +114,7 @@ func TestPolicyManagerInheritanceTest(t *testing.T) { for _, tc := range cases { tc := tc t.Run(fmt.Sprintf("%v", tc.sourceInfo), func(t *testing.T) { - pol, src, err := GetEffectivePolicy(ctx, env.Repository, tc.sourceInfo) + pol, src, err := GetEffectivePolicy(ctx, env.RepositoryWriter, tc.sourceInfo) if err != nil { t.Fatalf("err: %v", err) } @@ -175,7 +175,7 @@ func TestPolicyManagerResolvesConflicts(t *testing.T) { defer env.Setup(t).Close(ctx, t) - r1 := env.Repository + r1 := env.RepositoryWriter r2 := env.MustOpenAnother(t) sourceInfo := GlobalPolicySourceInfo @@ -333,7 +333,7 @@ func TestApplicablePoliciesForSource(t *testing.T) { } for si, pol := range setPols { - must(t, SetPolicy(ctx, env.Repository, si, pol)) + must(t, SetPolicy(ctx, env.RepositoryWriter, si, pol)) } cases := []struct { @@ -405,7 +405,7 @@ func TestApplicablePoliciesForSource(t *testing.T) { for _, tc := range cases { tc := tc t.Run(fmt.Sprintf("%v", tc.si), func(t *testing.T) { - res, err := applicablePoliciesForSource(ctx, env.Repository, tc.si) + res, err := applicablePoliciesForSource(ctx, env.RepositoryWriter, tc.si) if err != nil { t.Fatalf("error in applicablePoliciesForSource(%v): %v", tc.si, err) } diff --git a/snapshot/restore/restore.go b/snapshot/restore/restore.go index 4c45e1ec8..66f06f359 100644 --- a/snapshot/restore/restore.go +++ b/snapshot/restore/restore.go @@ -59,7 +59,7 @@ type Options struct { } // Entry walks a snapshot root with given root entry and restores it to the provided output. -func Entry(ctx context.Context, rep repo.Reader, output Output, rootEntry fs.Entry, options Options) (Stats, error) { +func Entry(ctx context.Context, rep repo.Repository, output Output, rootEntry fs.Entry, options Options) (Stats, error) { c := copier{output: output, q: parallelwork.NewQueue()} c.q.ProgressCallback = func(ctx context.Context, enqueued, active, completed int64) { diff --git a/snapshot/snapshot_test.go b/snapshot/snapshot_test.go index bf8e9de1f..f1461fcc0 100644 --- a/snapshot/snapshot_test.go +++ b/snapshot/snapshot_test.go @@ -33,55 +33,55 @@ func TestSnapshotsAPI(t *testing.T) { Path: "/some/other/path", } - if _, err := snapshot.LoadSnapshot(ctx, env.Repository, "no-such-manifest-id"); !errors.Is(err, snapshot.ErrSnapshotNotFound) { + if _, err := snapshot.LoadSnapshot(ctx, env.RepositoryWriter, "no-such-manifest-id"); !errors.Is(err, snapshot.ErrSnapshotNotFound) { t.Errorf("unexpected error when deleting snapshot for manifest that does not exist: %v", err) } - verifySnapshotManifestIDs(t, env.Repository, nil, nil) - verifySnapshotManifestIDs(t, env.Repository, &src1, nil) - verifySnapshotManifestIDs(t, env.Repository, &src2, nil) - verifyListSnapshots(t, env.Repository, src1, []*snapshot.Manifest{}) - verifyListSnapshots(t, env.Repository, src2, []*snapshot.Manifest{}) + verifySnapshotManifestIDs(t, env.RepositoryWriter, nil, nil) + verifySnapshotManifestIDs(t, env.RepositoryWriter, &src1, nil) + verifySnapshotManifestIDs(t, env.RepositoryWriter, &src2, nil) + verifyListSnapshots(t, env.RepositoryWriter, src1, []*snapshot.Manifest{}) + verifyListSnapshots(t, env.RepositoryWriter, src2, []*snapshot.Manifest{}) manifest1 := &snapshot.Manifest{ Source: src1, Description: "some-description", } - id1 := mustSaveSnapshot(t, env.Repository, manifest1) - verifySnapshotManifestIDs(t, env.Repository, nil, []manifest.ID{id1}) - verifySnapshotManifestIDs(t, env.Repository, &src1, []manifest.ID{id1}) - verifySnapshotManifestIDs(t, env.Repository, &src2, nil) - verifyListSnapshots(t, env.Repository, src1, []*snapshot.Manifest{manifest1}) + id1 := mustSaveSnapshot(t, env.RepositoryWriter, manifest1) + verifySnapshotManifestIDs(t, env.RepositoryWriter, nil, []manifest.ID{id1}) + verifySnapshotManifestIDs(t, env.RepositoryWriter, &src1, []manifest.ID{id1}) + verifySnapshotManifestIDs(t, env.RepositoryWriter, &src2, nil) + verifyListSnapshots(t, env.RepositoryWriter, src1, []*snapshot.Manifest{manifest1}) manifest2 := &snapshot.Manifest{ Source: src1, Description: "some-other-description", } - id2 := mustSaveSnapshot(t, env.Repository, manifest2) + id2 := mustSaveSnapshot(t, env.RepositoryWriter, manifest2) if id1 == id2 { t.Errorf("expected different manifest IDs, got same: %v", id1) } - verifySnapshotManifestIDs(t, env.Repository, nil, []manifest.ID{id1, id2}) - verifySnapshotManifestIDs(t, env.Repository, &src1, []manifest.ID{id1, id2}) - verifySnapshotManifestIDs(t, env.Repository, &src2, nil) + verifySnapshotManifestIDs(t, env.RepositoryWriter, nil, []manifest.ID{id1, id2}) + verifySnapshotManifestIDs(t, env.RepositoryWriter, &src1, []manifest.ID{id1, id2}) + verifySnapshotManifestIDs(t, env.RepositoryWriter, &src2, nil) manifest3 := &snapshot.Manifest{ Source: src2, Description: "some-other-description", } - id3 := mustSaveSnapshot(t, env.Repository, manifest3) - verifySnapshotManifestIDs(t, env.Repository, nil, []manifest.ID{id1, id2, id3}) - verifySnapshotManifestIDs(t, env.Repository, &src1, []manifest.ID{id1, id2}) - verifySnapshotManifestIDs(t, env.Repository, &src2, []manifest.ID{id3}) - verifySources(t, env.Repository, src1, src2) - verifyLoadSnapshots(t, env.Repository, []manifest.ID{id1, id2, id3}, []*snapshot.Manifest{manifest1, manifest2, manifest3}) + id3 := mustSaveSnapshot(t, env.RepositoryWriter, manifest3) + verifySnapshotManifestIDs(t, env.RepositoryWriter, nil, []manifest.ID{id1, id2, id3}) + verifySnapshotManifestIDs(t, env.RepositoryWriter, &src1, []manifest.ID{id1, id2}) + verifySnapshotManifestIDs(t, env.RepositoryWriter, &src2, []manifest.ID{id3}) + verifySources(t, env.RepositoryWriter, src1, src2) + verifyLoadSnapshots(t, env.RepositoryWriter, []manifest.ID{id1, id2, id3}, []*snapshot.Manifest{manifest1, manifest2, manifest3}) } -func verifySnapshotManifestIDs(t *testing.T, rep repo.Reader, src *snapshot.SourceInfo, expected []manifest.ID) { +func verifySnapshotManifestIDs(t *testing.T, rep repo.Repository, src *snapshot.SourceInfo, expected []manifest.ID) { t.Helper() res, err := snapshot.ListSnapshotManifests(testlogging.Context(t), rep, src) @@ -103,7 +103,7 @@ func sortManifestIDs(s []manifest.ID) { }) } -func mustSaveSnapshot(t *testing.T, rep repo.Writer, man *snapshot.Manifest) manifest.ID { +func mustSaveSnapshot(t *testing.T, rep repo.RepositoryWriter, man *snapshot.Manifest) manifest.ID { t.Helper() id, err := snapshot.SaveSnapshot(testlogging.Context(t), rep, man) @@ -114,7 +114,7 @@ func mustSaveSnapshot(t *testing.T, rep repo.Writer, man *snapshot.Manifest) man return id } -func verifySources(t *testing.T, rep repo.Reader, sources ...snapshot.SourceInfo) { +func verifySources(t *testing.T, rep repo.Repository, sources ...snapshot.SourceInfo) { t.Helper() actualSources, err := snapshot.ListSources(testlogging.Context(t), rep) @@ -127,7 +127,7 @@ func verifySources(t *testing.T, rep repo.Reader, sources ...snapshot.SourceInfo } } -func verifyListSnapshots(t *testing.T, rep repo.Reader, src snapshot.SourceInfo, expected []*snapshot.Manifest) { +func verifyListSnapshots(t *testing.T, rep repo.Repository, src snapshot.SourceInfo, expected []*snapshot.Manifest) { t.Helper() got, err := snapshot.ListSnapshots(testlogging.Context(t), rep, src) @@ -149,7 +149,7 @@ func verifyListSnapshots(t *testing.T, rep repo.Reader, src snapshot.SourceInfo, } } -func verifyLoadSnapshots(t *testing.T, rep repo.Reader, ids []manifest.ID, expected []*snapshot.Manifest) { +func verifyLoadSnapshots(t *testing.T, rep repo.Repository, ids []manifest.ID, expected []*snapshot.Manifest) { t.Helper() got, err := snapshot.LoadSnapshots(testlogging.Context(t), rep, ids) diff --git a/snapshot/snapshotfs/all_sources.go b/snapshot/snapshotfs/all_sources.go index a5e69bb21..364e302a5 100644 --- a/snapshot/snapshotfs/all_sources.go +++ b/snapshot/snapshotfs/all_sources.go @@ -14,7 +14,7 @@ ) type repositoryAllSources struct { - rep repo.Reader + rep repo.Repository } func (s *repositoryAllSources) IsDir() bool { @@ -82,6 +82,6 @@ func (s *repositoryAllSources) Readdir(ctx context.Context) (fs.Entries, error) } // AllSourcesEntry returns fs.Directory that contains the list of all snapshot sources found in the repository. -func AllSourcesEntry(rep repo.Reader) fs.Directory { +func AllSourcesEntry(rep repo.Repository) fs.Directory { return &repositoryAllSources{rep: rep} } diff --git a/snapshot/snapshotfs/objref.go b/snapshot/snapshotfs/objref.go index 6577dbace..3837851b4 100644 --- a/snapshot/snapshotfs/objref.go +++ b/snapshot/snapshotfs/objref.go @@ -16,7 +16,7 @@ // ParseObjectIDWithPath interprets the given ID string (which could be an object ID optionally followed by // nested path specification) and returns corresponding object.ID. -func ParseObjectIDWithPath(ctx context.Context, rep repo.Reader, objectIDWithPath string) (object.ID, error) { +func ParseObjectIDWithPath(ctx context.Context, rep repo.Repository, objectIDWithPath string) (object.ID, error) { parts := strings.Split(objectIDWithPath, "/") oid, err := object.ParseID(parts[0]) @@ -75,7 +75,7 @@ func parseNestedObjectID(ctx context.Context, startingDir fs.Entry, parts []stri // or the root object ID (which can match arbitrary number of snapshots). // If multiple snapshots match and they don't agree on root object attributes and consistentAttributes==true // the function fails, otherwise it returns the latest of the snapshots. -func findSnapshotByRootObjectIDOrManifestID(ctx context.Context, rep repo.Reader, rootID string, consistentAttributes bool) (*snapshot.Manifest, error) { +func findSnapshotByRootObjectIDOrManifestID(ctx context.Context, rep repo.Repository, rootID string, consistentAttributes bool) (*snapshot.Manifest, error) { m, err := snapshot.LoadSnapshot(ctx, rep, manifest.ID(rootID)) if err == nil { return m, nil @@ -133,7 +133,7 @@ func latestManifest(mans []*snapshot.Manifest) *snapshot.Manifest { // can be a snapshot manifest ID or an object ID with path. // If multiple snapshots match and they don't agree on root object attributes and consistentAttributes==true // the function fails, otherwise it returns the latest of the snapshots. -func FilesystemEntryFromIDWithPath(ctx context.Context, rep repo.Reader, rootID string, consistentAttributes bool) (fs.Entry, error) { +func FilesystemEntryFromIDWithPath(ctx context.Context, rep repo.Repository, rootID string, consistentAttributes bool) (fs.Entry, error) { pathElements := strings.Split(rootID, "/") if len(pathElements) > 1 { @@ -169,7 +169,7 @@ func FilesystemEntryFromIDWithPath(ctx context.Context, rep repo.Reader, rootID // FilesystemDirectoryFromIDWithPath returns a filesystem directory entry for the provided object ID, which // can be a snapshot manifest ID or an object ID with path. -func FilesystemDirectoryFromIDWithPath(ctx context.Context, rep repo.Reader, rootID string, consistentAttributes bool) (fs.Directory, error) { +func FilesystemDirectoryFromIDWithPath(ctx context.Context, rep repo.Repository, rootID string, consistentAttributes bool) (fs.Directory, error) { e, err := FilesystemEntryFromIDWithPath(ctx, rep, rootID, consistentAttributes) if err != nil { return nil, err diff --git a/snapshot/snapshotfs/repofs.go b/snapshot/snapshotfs/repofs.go index 961b29437..a57c12f55 100644 --- a/snapshot/snapshotfs/repofs.go +++ b/snapshot/snapshotfs/repofs.go @@ -1,4 +1,4 @@ -// Package snapshotfs implements virtual filesystem on top of snapshots in repo.Reader. +// Package snapshotfs implements virtual filesystem on top of snapshots in repo.Repository. package snapshotfs import ( @@ -22,7 +22,7 @@ type repositoryEntry struct { metadata *snapshot.DirEntry - repo repo.Reader + repo repo.Repository } func (e *repositoryEntry) IsDir() bool { @@ -170,7 +170,7 @@ func (rsl *repositorySymlink) Readlink(ctx context.Context) (string, error) { } // EntryFromDirEntry returns a filesystem entry based on the directory entry. -func EntryFromDirEntry(r repo.Reader, md *snapshot.DirEntry) (fs.Entry, error) { +func EntryFromDirEntry(r repo.Repository, md *snapshot.DirEntry) (fs.Entry, error) { re := repositoryEntry{ metadata: md, repo: r, @@ -214,7 +214,7 @@ func withFileInfo(r object.Reader, e fs.Entry) fs.Reader { // DirectoryEntry returns fs.Directory based on repository object with the specified ID. // The existence or validity of the directory object is not validated until its contents are read. -func DirectoryEntry(rep repo.Reader, objectID object.ID, dirSummary *fs.DirectorySummary) fs.Directory { +func DirectoryEntry(rep repo.Repository, objectID object.ID, dirSummary *fs.DirectorySummary) fs.Directory { d, _ := EntryFromDirEntry(rep, &snapshot.DirEntry{ Name: "/", Permissions: 0o555, //nolint:gomnd @@ -227,7 +227,7 @@ func DirectoryEntry(rep repo.Reader, objectID object.ID, dirSummary *fs.Director } // SnapshotRoot returns fs.Entry representing the root of a snapshot. -func SnapshotRoot(rep repo.Reader, man *snapshot.Manifest) (fs.Entry, error) { +func SnapshotRoot(rep repo.Repository, man *snapshot.Manifest) (fs.Entry, error) { oid := man.RootObjectID() if oid == "" { return nil, errors.New("manifest root object ID") @@ -238,7 +238,7 @@ func SnapshotRoot(rep repo.Reader, man *snapshot.Manifest) (fs.Entry, error) { // AutoDetectEntryFromObjectID returns fs.Entry (either file or directory) for the provided object ID. // It uses heuristics to determine whether object ID is possibly a directory and treats it as such. -func AutoDetectEntryFromObjectID(ctx context.Context, rep repo.Reader, oid object.ID, maybeName string) fs.Entry { +func AutoDetectEntryFromObjectID(ctx context.Context, rep repo.Repository, oid object.ID, maybeName string) fs.Entry { if IsDirectoryID(oid) { dirEntry := DirectoryEntry(rep, oid, nil) if _, err := dirEntry.Readdir(ctx); err == nil { diff --git a/snapshot/snapshotfs/source_directories.go b/snapshot/snapshotfs/source_directories.go index fccd3ea13..892771fd9 100644 --- a/snapshot/snapshotfs/source_directories.go +++ b/snapshot/snapshotfs/source_directories.go @@ -13,7 +13,7 @@ ) type sourceDirectories struct { - rep repo.Reader + rep repo.Repository userHost string } diff --git a/snapshot/snapshotfs/source_snapshots.go b/snapshot/snapshotfs/source_snapshots.go index 4d206b61b..ace4734a1 100644 --- a/snapshot/snapshotfs/source_snapshots.go +++ b/snapshot/snapshotfs/source_snapshots.go @@ -15,7 +15,7 @@ ) type sourceSnapshots struct { - rep repo.Reader + rep repo.Repository src snapshot.SourceInfo } diff --git a/snapshot/snapshotfs/upload.go b/snapshot/snapshotfs/upload.go index a2df3c7c9..22caf9a52 100644 --- a/snapshot/snapshotfs/upload.go +++ b/snapshot/snapshotfs/upload.go @@ -70,7 +70,7 @@ type Uploader struct { // How frequently to create checkpoint snapshot entries. CheckpointInterval time.Duration - repo repo.Writer + repo repo.RepositoryWriter // stats must be allocated on heap to enforce 64-bit alignment due to atomic access on ARM. stats *snapshot.Stats @@ -967,7 +967,7 @@ func (u *Uploader) shouldIgnoreDirectoryReadErrors(policyTree *policy.Tree) bool } // NewUploader creates new Uploader object for a given repository. -func NewUploader(r repo.Writer) *Uploader { +func NewUploader(r repo.RepositoryWriter) *Uploader { return &Uploader{ repo: r, Progress: &NullUploadProgress{}, diff --git a/snapshot/snapshotfs/upload_test.go b/snapshot/snapshotfs/upload_test.go index 4f06e8f7f..dbc7713c7 100644 --- a/snapshot/snapshotfs/upload_test.go +++ b/snapshot/snapshotfs/upload_test.go @@ -32,7 +32,7 @@ type uploadTestHarness struct { sourceDir *mockfs.Directory repoDir string - repo repo.Writer + repo repo.RepositoryWriter ft *faketime.TimeAdvance } @@ -95,10 +95,15 @@ func newUploadTestHarness(ctx context.Context) *uploadTestHarness { sourceDir.AddFile("d2/d1/f1", []byte{1, 2, 3}, defaultPermissions) sourceDir.AddFile("d2/d1/f2", []byte{1, 2, 3, 4}, defaultPermissions) + w, err := rep.NewWriter(ctx, "test") + if err != nil { + panic("writer creation error: " + err.Error()) + } + th := &uploadTestHarness{ sourceDir: sourceDir, repoDir: repoDir, - repo: rep, + repo: w, ft: ft, } diff --git a/snapshot/snapshotgc/gc.go b/snapshot/snapshotgc/gc.go index 490992bd9..d515fe490 100644 --- a/snapshot/snapshotgc/gc.go +++ b/snapshot/snapshotgc/gc.go @@ -26,7 +26,7 @@ func oidOf(entry fs.Entry) object.ID { return entry.(object.HasObjectID).ObjectID() } -func findInUseContentIDs(ctx context.Context, rep repo.Reader, used *sync.Map) error { +func findInUseContentIDs(ctx context.Context, rep repo.Repository, used *sync.Map) error { ids, err := snapshot.ListSnapshotManifests(ctx, rep, nil) if err != nil { return errors.Wrap(err, "unable to list snapshot manifest IDs") @@ -74,7 +74,7 @@ func findInUseContentIDs(ctx context.Context, rep repo.Reader, used *sync.Map) e } // Run performs garbage collection on all the snapshots in the repository. -func Run(ctx context.Context, rep *repo.DirectRepository, params maintenance.SnapshotGCParams, gcDelete bool) (Stats, error) { +func Run(ctx context.Context, rep repo.DirectRepositoryWriter, params maintenance.SnapshotGCParams, gcDelete bool) (Stats, error) { var st Stats err := maintenance.ReportRun(ctx, rep, "snapshot-gc", func() error { @@ -84,7 +84,7 @@ func Run(ctx context.Context, rep *repo.DirectRepository, params maintenance.Sna return st, errors.Wrap(err, "error running snapshot gc") } -func runInternal(ctx context.Context, rep *repo.DirectRepository, params maintenance.SnapshotGCParams, gcDelete bool, st *Stats) error { +func runInternal(ctx context.Context, rep repo.DirectRepositoryWriter, params maintenance.SnapshotGCParams, gcDelete bool, st *Stats) error { var ( used sync.Map @@ -99,7 +99,7 @@ func runInternal(ctx context.Context, rep *repo.DirectRepository, params mainten // Ensure that the iteration includes deleted contents, so those can be // undeleted (recovered). - err := rep.Content.IterateContents(ctx, content.IterateOptions{IncludeDeleted: true}, func(ci content.Info) error { + err := rep.ContentReader().IterateContents(ctx, content.IterateOptions{IncludeDeleted: true}, func(ci content.Info) error { if manifest.ContentPrefix == ci.ID.Prefix() { system.Add(int64(ci.Length)) return nil @@ -107,7 +107,7 @@ func runInternal(ctx context.Context, rep *repo.DirectRepository, params mainten if _, ok := used.Load(ci.ID); ok { if ci.Deleted { - if err := rep.Content.UndeleteContent(ctx, ci.ID); err != nil { + if err := rep.ContentManager().UndeleteContent(ctx, ci.ID); err != nil { return errors.Wrapf(err, "Could not undelete referenced content: %v", ci) } undeleted.Add(int64(ci.Length)) @@ -127,7 +127,7 @@ func runInternal(ctx context.Context, rep *repo.DirectRepository, params mainten cnt, totalSize := unused.Add(int64(ci.Length)) if gcDelete { - if err := rep.Content.DeleteContent(ctx, ci.ID); err != nil { + if err := rep.ContentManager().DeleteContent(ctx, ci.ID); err != nil { return errors.Wrap(err, "error deleting content") } } diff --git a/snapshot/snapshotmaintenance/helper_test.go b/snapshot/snapshotmaintenance/helper_test.go index 197278675..0ff0e0978 100644 --- a/snapshot/snapshotmaintenance/helper_test.go +++ b/snapshot/snapshotmaintenance/helper_test.go @@ -14,7 +14,7 @@ ) // Create snapshots an FS entry. -func createSnapshot(ctx context.Context, rep repo.Writer, e fs.Entry, si snapshot.SourceInfo, description string) (*snapshot.Manifest, error) { +func createSnapshot(ctx context.Context, rep repo.RepositoryWriter, e fs.Entry, si snapshot.SourceInfo, description string) (*snapshot.Manifest, error) { // sanitize source path si.Path = filepath.Clean(si.Path) @@ -46,7 +46,7 @@ func createSnapshot(ctx context.Context, rep repo.Writer, e fs.Entry, si snapsho // findPreviousSnapshotManifest returns the list of previous snapshots for a given source, including // last complete snapshot and possibly some number of incomplete snapshots following it. // this would belong in the snapshot package. -func findPreviousSnapshotManifest(ctx context.Context, rep repo.Reader, sourceInfo snapshot.SourceInfo) ([]*snapshot.Manifest, error) { +func findPreviousSnapshotManifest(ctx context.Context, rep repo.Repository, sourceInfo snapshot.SourceInfo) ([]*snapshot.Manifest, error) { man, err := snapshot.ListSnapshots(ctx, rep, sourceInfo) if err != nil { return nil, errors.Wrap(err, "error listing previous snapshots") diff --git a/snapshot/snapshotmaintenance/snapshotmaintenance.go b/snapshot/snapshotmaintenance/snapshotmaintenance.go index 107844b1b..d379e6251 100644 --- a/snapshot/snapshotmaintenance/snapshotmaintenance.go +++ b/snapshot/snapshotmaintenance/snapshotmaintenance.go @@ -12,12 +12,7 @@ ) // Run runs the complete snapshot and repository maintenance. -func Run(ctx context.Context, rep repo.Writer, mode maintenance.Mode, force bool) error { - dr, ok := rep.(*repo.DirectRepository) - if !ok { - return nil - } - +func Run(ctx context.Context, dr repo.DirectRepositoryWriter, mode maintenance.Mode, force bool) error { return maintenance.RunExclusive(ctx, dr, mode, force, func(runParams maintenance.RunParameters) error { // run snapshot GC before full maintenance diff --git a/snapshot/snapshotmaintenance/snapshotmaintenance_test.go b/snapshot/snapshotmaintenance/snapshotmaintenance_test.go index 7d36696d8..ecbd505b4 100644 --- a/snapshot/snapshotmaintenance/snapshotmaintenance_test.go +++ b/snapshot/snapshotmaintenance/snapshotmaintenance_test.go @@ -44,30 +44,30 @@ func TestSnapshotGCSimple(t *testing.T) { UserName: "user", Path: "/foo", } - s1 := mustSnapshot(t, th.Repository, th.sourceDir, si) + s1 := mustSnapshot(t, th.RepositoryWriter, th.sourceDir, si) t.Log("snap 1:", pretty.Sprint(s1)) - mustFlush(t, th.Repository) + mustFlush(t, th.RepositoryWriter) // Delete snapshot - err := th.Repository.Manifests.Delete(ctx, s1.ID) + err := th.RepositoryWriter.DeleteManifest(ctx, s1.ID) require.NoError(t, err) - mustFlush(t, th.Repository) + mustFlush(t, th.RepositoryWriter) // Advance time to force GC to mark as deleted the contents from the previous snapshot th.fakeTime.Advance(maintenance.DefaultParams().SnapshotGC.MinContentAge + time.Hour) - err = snapshotmaintenance.Run(ctx, th.Repository, maintenance.ModeFull, true) + err = snapshotmaintenance.Run(ctx, th.RepositoryWriter, maintenance.ModeFull, true) require.NoError(t, err) - mustFlush(t, th.Repository) + mustFlush(t, th.RepositoryWriter) - s2 := mustSnapshot(t, th.Repository, th.sourceDir, si) + s2 := mustSnapshot(t, th.RepositoryWriter, th.sourceDir, si) t.Log("snap 2:", pretty.Sprint(s2)) - mustFlush(t, th.Repository) + mustFlush(t, th.RepositoryWriter) - info, err := th.Repository.Content.ContentInfo(ctx, content.ID(s2.RootObjectID())) + info, err := th.RepositoryWriter.ContentReader().ContentInfo(ctx, content.ID(s2.RootObjectID())) require.NoError(t, err) t.Log("root info:", pretty.Sprint(info)) @@ -98,16 +98,16 @@ func TestMaintenanceReuseDirManifest(t *testing.T) { UserName: "user", Path: "/foo", } - s1 := mustSnapshot(t, th.Repository, th.sourceDir, si) + s1 := mustSnapshot(t, th.RepositoryWriter, th.sourceDir, si) t.Log("snap 1:", pretty.Sprint(s1)) - mustFlush(t, th.Repository) + mustFlush(t, th.RepositoryWriter) // Delete snapshot - err := th.Repository.Manifests.Delete(ctx, s1.ID) + err := th.RepositoryWriter.DeleteManifest(ctx, s1.ID) require.NoError(t, err) - mustFlush(t, th.Repository) + mustFlush(t, th.RepositoryWriter) // Advance time to force GC to mark as deleted the contents from the previous snapshot th.fakeTime.Advance(maintenance.DefaultParams().SnapshotGC.MinContentAge + time.Hour) @@ -120,10 +120,10 @@ func TestMaintenanceReuseDirManifest(t *testing.T) { // interleaving snapshot and maintenance and delaying flushing as well to // create dangling references to contents that were in the previously // deleted snapshot and that are reused in this new snapshot. - err = snapshotmaintenance.Run(ctx, th.Repository, maintenance.ModeFull, true) + err = snapshotmaintenance.Run(ctx, th.RepositoryWriter, maintenance.ModeFull, true) require.NoError(t, err) - info, err := r2.(*repo.DirectRepository).Content.ContentInfo(ctx, content.ID(s2.RootObjectID())) + info, err := r2.(repo.DirectRepository).ContentReader().ContentInfo(ctx, content.ID(s2.RootObjectID())) require.NoError(t, err) require.False(t, info.Deleted, "content must not be deleted") @@ -133,29 +133,29 @@ func TestMaintenanceReuseDirManifest(t *testing.T) { mustFlush(t, r2) // finish snapshot require.NoError(t, r2.Close(ctx)) - mustFlush(t, th.Repository) // finish maintenance + mustFlush(t, th.RepositoryWriter) // finish maintenance th.MustReopen(t) - info, err = th.Repository.Content.ContentInfo(ctx, content.ID(s2.RootObjectID())) + info, err = th.RepositoryWriter.ContentReader().ContentInfo(ctx, content.ID(s2.RootObjectID())) require.NoError(t, err) require.True(t, info.Deleted, "content must be deleted") - _, err = th.Repository.VerifyObject(ctx, s2.RootObjectID()) + _, err = th.RepositoryWriter.VerifyObject(ctx, s2.RootObjectID()) require.NoError(t, err) // Run maintenance again th.fakeTime.Advance(maintenance.DefaultParams().SnapshotGC.MinContentAge + time.Hour) - err = snapshotmaintenance.Run(ctx, th.Repository, maintenance.ModeFull, true) + err = snapshotmaintenance.Run(ctx, th.RepositoryWriter, maintenance.ModeFull, true) require.NoError(t, err) - mustFlush(t, th.Repository) + mustFlush(t, th.RepositoryWriter) // Was the previous root undeleted - info, err = th.Repository.Content.ContentInfo(ctx, content.ID(s2.RootObjectID())) + info, err = th.RepositoryWriter.ContentReader().ContentInfo(ctx, content.ID(s2.RootObjectID())) require.NoError(t, err) require.False(t, info.Deleted, "content must not be deleted") - _, err = th.Repository.VerifyObject(ctx, s2.RootObjectID()) + _, err = th.RepositoryWriter.VerifyObject(ctx, s2.RootObjectID()) require.NoError(t, err) t.Log("root info:", pretty.Sprint(info)) @@ -172,7 +172,7 @@ func newTestHarness(t *testing.T) *testHarness { th.Environment.Setup(t, repotesting.Options{OpenOptions: th.fakeTimeOpenRepoOption}) - require.NotNil(t, th.Repository) + require.NotNil(t, th.RepositoryWriter) t.Cleanup(func() { th.Environment.Close(testlogging.Context(t), t) @@ -185,25 +185,31 @@ func (th *testHarness) fakeTimeOpenRepoOption(o *repo.Options) { o.TimeNowFunc = th.fakeTime.NowFunc() } -func (th *testHarness) openAnother(t *testing.T) repo.Repository { +func (th *testHarness) openAnother(t *testing.T) repo.RepositoryWriter { t.Helper() r := th.MustConnectOpenAnother(t, th.fakeTimeOpenRepoOption) + ctx := testlogging.Context(t) t.Cleanup(func() { - r.Close(testlogging.Context(t)) + r.Close(ctx) }) - return r + w, err := r.NewWriter(ctx, "test") + if err != nil { + t.Fatal(err) + } + + return w } -func mustFlush(t *testing.T, r repo.Writer) { +func mustFlush(t *testing.T, r repo.RepositoryWriter) { t.Helper() require.NotNil(t, r, "nil repository") require.NoError(t, r.Flush(testlogging.Context(t))) } -func mustSnapshot(t *testing.T, r repo.Writer, source fs.Entry, si snapshot.SourceInfo) *snapshot.Manifest { +func mustSnapshot(t *testing.T, r repo.RepositoryWriter, source fs.Entry, si snapshot.SourceInfo) *snapshot.Manifest { t.Helper() s1, err := createSnapshot(testlogging.Context(t), r, source, si, "") diff --git a/tests/repository_stress_test/repository_stress_test.go b/tests/repository_stress_test/repository_stress_test.go index abb1fbfda..fa580c27c 100644 --- a/tests/repository_stress_test/repository_stress_test.go +++ b/tests/repository_stress_test/repository_stress_test.go @@ -141,6 +141,12 @@ func longLivedRepositoryTest(ctx context.Context, t *testing.T, cancel chan stru } defer rep.Close(ctx) + w, err := rep.(repo.DirectRepository).NewDirectWriter(ctx, "longLivedRepositoryTest") + if err != nil { + t.Errorf("error opening writer: %v", err) + return + } + var wg2 sync.WaitGroup for i := 0; i < 4; i++ { @@ -149,19 +155,19 @@ func longLivedRepositoryTest(ctx context.Context, t *testing.T, cancel chan stru go func() { defer wg2.Done() - repositoryTest(ctx, t, cancel, rep.(*repo.DirectRepository)) + repositoryTest(ctx, t, cancel, w) }() } wg2.Wait() } -func repositoryTest(ctx context.Context, t *testing.T, cancel chan struct{}, rep *repo.DirectRepository) { +func repositoryTest(ctx context.Context, t *testing.T, cancel chan struct{}, rep repo.DirectRepositoryWriter) { t.Helper() workTypes := []*struct { name string - fun func(ctx context.Context, t *testing.T, r *repo.DirectRepository) error + fun func(ctx context.Context, t *testing.T, r repo.DirectRepositoryWriter) error weight int hitCount int }{ @@ -221,13 +227,13 @@ func repositoryTest(ctx context.Context, t *testing.T, cancel chan struct{}, rep } } -func writeRandomBlock(ctx context.Context, t *testing.T, r *repo.DirectRepository) error { +func writeRandomBlock(ctx context.Context, t *testing.T, r repo.DirectRepositoryWriter) error { t.Helper() data := make([]byte, 1000) cryptorand.Read(data) - contentID, err := r.Content.WriteContent(ctx, data, "") + contentID, err := r.ContentManager().WriteContent(ctx, data, "") if err == nil { knownBlocksMutex.Lock() if len(knownBlocks) >= 1000 { @@ -242,7 +248,7 @@ func writeRandomBlock(ctx context.Context, t *testing.T, r *repo.DirectRepositor return err } -func readKnownBlock(ctx context.Context, t *testing.T, r *repo.DirectRepository) error { +func readKnownBlock(ctx context.Context, t *testing.T, r repo.DirectRepositoryWriter) error { t.Helper() knownBlocksMutex.Lock() @@ -254,7 +260,7 @@ func readKnownBlock(ctx context.Context, t *testing.T, r *repo.DirectRepository) contentID := knownBlocks[rand.Intn(len(knownBlocks))] knownBlocksMutex.Unlock() - _, err := r.Content.GetContent(ctx, contentID) + _, err := r.ContentReader().GetContent(ctx, contentID) if err == nil || errors.Is(err, content.ErrContentNotFound) { return nil } @@ -262,25 +268,25 @@ func readKnownBlock(ctx context.Context, t *testing.T, r *repo.DirectRepository) return err } -func listContents(ctx context.Context, t *testing.T, r *repo.DirectRepository) error { +func listContents(ctx context.Context, t *testing.T, r repo.DirectRepositoryWriter) error { t.Helper() - return r.Content.IterateContents( + return r.ContentReader().IterateContents( ctx, content.IterateOptions{}, func(i content.Info) error { return nil }, ) } -func listAndReadAllContents(ctx context.Context, t *testing.T, r *repo.DirectRepository) error { +func listAndReadAllContents(ctx context.Context, t *testing.T, r repo.DirectRepositoryWriter) error { t.Helper() - return r.Content.IterateContents( + return r.ContentReader().IterateContents( ctx, content.IterateOptions{}, func(ci content.Info) error { cid := ci.ID - _, err := r.Content.GetContent(ctx, cid) + _, err := r.ContentReader().GetContent(ctx, cid) if err != nil { if errors.Is(err, content.ErrContentNotFound) && strings.HasPrefix(string(cid), "m") { // this is ok, sometimes manifest manager will perform compaction and 'm' contents will be marked as deleted @@ -293,25 +299,25 @@ func(ci content.Info) error { }) } -func compact(ctx context.Context, t *testing.T, r *repo.DirectRepository) error { +func compact(ctx context.Context, t *testing.T, r repo.DirectRepositoryWriter) error { t.Helper() - return r.Content.CompactIndexes(ctx, content.CompactOptions{MaxSmallBlobs: 1}) + return r.ContentManager().CompactIndexes(ctx, content.CompactOptions{MaxSmallBlobs: 1}) } -func flush(ctx context.Context, t *testing.T, r *repo.DirectRepository) error { +func flush(ctx context.Context, t *testing.T, r repo.DirectRepositoryWriter) error { t.Helper() return r.Flush(ctx) } -func refresh(ctx context.Context, t *testing.T, r *repo.DirectRepository) error { +func refresh(ctx context.Context, t *testing.T, r repo.DirectRepositoryWriter) error { t.Helper() return r.Refresh(ctx) } -func readRandomManifest(ctx context.Context, t *testing.T, r *repo.DirectRepository) error { +func readRandomManifest(ctx context.Context, t *testing.T, r repo.DirectRepositoryWriter) error { t.Helper() manifests, err := r.FindManifests(ctx, nil) @@ -330,7 +336,7 @@ func readRandomManifest(ctx context.Context, t *testing.T, r *repo.DirectReposit return err } -func writeRandomManifest(ctx context.Context, t *testing.T, r *repo.DirectRepository) error { +func writeRandomManifest(ctx context.Context, t *testing.T, r repo.DirectRepositoryWriter) error { t.Helper() key1 := fmt.Sprintf("key-%v", rand.Intn(10))