From 5e8e175cfaf16eff35835e38ea90622ba519d485 Mon Sep 17 00:00:00 2001 From: Jarek Kowalski Date: Mon, 4 Jan 2021 21:33:12 -0800 Subject: [PATCH] repo: refactored read/write methods of repo.Repository (#749) Reader methods go to repo.Reader and write methods go to repo.Writer Switched usage to new interfaces based on linter errors. --- cli/app.go | 14 ++++++++- cli/auto_upgrade.go | 2 +- cli/command_benchmark_compression.go | 5 ++- cli/command_benchmark_crypto.go | 5 ++- cli/command_benchmark_splitters.go | 5 ++- cli/command_diff.go | 4 +-- cli/command_ls.go | 4 +-- cli/command_manifest_ls.go | 4 +-- cli/command_manifest_rm.go | 4 +-- cli/command_manifest_show.go | 4 +-- cli/command_mount.go | 4 +-- cli/command_policy.go | 2 +- cli/command_policy_edit.go | 4 +-- cli/command_policy_ls.go | 4 +-- cli/command_policy_remove.go | 4 +-- cli/command_policy_set.go | 4 +-- cli/command_policy_show.go | 4 +-- cli/command_repository_status.go | 4 +-- cli/command_restore.go | 4 +-- cli/command_show.go | 4 +-- cli/command_snapshot_copy_move_history.go | 8 ++--- cli/command_snapshot_create.go | 12 +++---- cli/command_snapshot_delete.go | 8 ++--- cli/command_snapshot_estimate.go | 4 +-- cli/command_snapshot_expire.go | 6 ++-- cli/command_snapshot_list.go | 14 ++++----- cli/command_snapshot_migrate.go | 20 ++++++------ cli/command_snapshot_restore.go | 2 +- cli/command_snapshot_verify.go | 10 +++--- internal/server/server.go | 2 +- repo/maintenance/maintenance_run.go | 8 ++--- repo/repository.go | 31 +++++++++++++------ snapshot/manager.go | 14 ++++----- snapshot/policy/expire.go | 6 ++-- snapshot/policy/policy_manager.go | 18 +++++------ snapshot/restore/restore.go | 2 +- snapshot/snapshot_test.go | 10 +++--- snapshot/snapshotfs/all_sources.go | 4 +-- snapshot/snapshotfs/objref.go | 8 ++--- snapshot/snapshotfs/repofs.go | 12 +++---- snapshot/snapshotfs/source_directories.go | 2 +- snapshot/snapshotfs/source_snapshots.go | 2 +- snapshot/snapshotfs/upload.go | 4 +-- snapshot/snapshotfs/upload_test.go | 2 +- snapshot/snapshotgc/gc.go | 2 +- snapshot/snapshotmaintenance/helper_test.go | 4 +-- .../snapshotmaintenance.go | 2 +- .../snapshotmaintenance_test.go | 4 +-- 48 files changed, 164 insertions(+), 146 deletions(-) diff --git a/cli/app.go b/cli/app.go index 683a0b468..8854bc2a5 100644 --- a/cli/app.go +++ b/cli/app.go @@ -106,6 +106,18 @@ func repositoryAction(act func(ctx context.Context, rep repo.Repository) error) return maybeRepositoryAction(act, true) } +func repositoryReaderAction(act func(ctx context.Context, rep repo.Reader) error) func(ctx *kingpin.ParseContext) error { + return maybeRepositoryAction(func(ctx context.Context, rep repo.Repository) error { + return act(ctx, rep) + }, true) +} + +func repositoryWriterAction(act func(ctx context.Context, rep repo.Writer) error) func(ctx *kingpin.ParseContext) error { + return maybeRepositoryAction(func(ctx context.Context, rep repo.Repository) error { + return act(ctx, rep) + }, true) +} + func rootContext() context.Context { ctx := context.Background() ctx = content.UsingContentCache(ctx, *enableCaching) @@ -162,7 +174,7 @@ func maybeRepositoryAction(act func(ctx context.Context, rep repo.Repository) er } } -func maybeRunMaintenance(ctx context.Context, rep repo.Repository) error { +func maybeRunMaintenance(ctx context.Context, rep repo.Writer) error { if !*enableAutomaticMaintenance { return nil } diff --git a/cli/auto_upgrade.go b/cli/auto_upgrade.go index 7375869f1..3dc7f3ad5 100644 --- a/cli/auto_upgrade.go +++ b/cli/auto_upgrade.go @@ -9,7 +9,7 @@ "github.com/kopia/kopia/repo/maintenance" ) -func maybeAutoUpgradeRepository(ctx context.Context, r repo.Repository) { +func maybeAutoUpgradeRepository(ctx context.Context, r repo.Writer) { if r == nil { return } diff --git a/cli/command_benchmark_compression.go b/cli/command_benchmark_compression.go index 1f3b950c8..c5c200dc9 100644 --- a/cli/command_benchmark_compression.go +++ b/cli/command_benchmark_compression.go @@ -11,7 +11,6 @@ "github.com/kopia/kopia/internal/clock" "github.com/kopia/kopia/internal/units" - "github.com/kopia/kopia/repo" "github.com/kopia/kopia/repo/compression" ) @@ -24,7 +23,7 @@ benchmarkCompressionVerifyStable = benchmarkCompressionCommand.Flag("verify-stable", "Verify that compression is stable").Bool() ) -func runBenchmarkCompressionAction(ctx context.Context, rep repo.Repository) error { +func runBenchmarkCompressionAction(ctx context.Context) error { type benchResult struct { compression compression.Name throughput float64 @@ -106,7 +105,7 @@ type benchResult struct { } func init() { - benchmarkCompressionCommand.Action(maybeRepositoryAction(runBenchmarkCompressionAction, false)) + benchmarkCompressionCommand.Action(noRepositoryAction(runBenchmarkCompressionAction)) } func hashOf(b []byte) uint64 { diff --git a/cli/command_benchmark_crypto.go b/cli/command_benchmark_crypto.go index 238a45bd2..e66e95756 100644 --- a/cli/command_benchmark_crypto.go +++ b/cli/command_benchmark_crypto.go @@ -6,7 +6,6 @@ "github.com/kopia/kopia/internal/clock" "github.com/kopia/kopia/internal/units" - "github.com/kopia/kopia/repo" "github.com/kopia/kopia/repo/content" "github.com/kopia/kopia/repo/encryption" "github.com/kopia/kopia/repo/hashing" @@ -19,7 +18,7 @@ benchmarkCryptoDeprecatedAlgorithms = benchmarkCryptoCommand.Flag("deprecated", "Include deprecated algorithms").Bool() ) -func runBenchmarkCryptoAction(ctx context.Context, rep repo.Repository) error { +func runBenchmarkCryptoAction(ctx context.Context) error { type benchResult struct { hash string encryption string @@ -86,5 +85,5 @@ type benchResult struct { } func init() { - benchmarkCryptoCommand.Action(maybeRepositoryAction(runBenchmarkCryptoAction, false)) + benchmarkCryptoCommand.Action(noRepositoryAction(runBenchmarkCryptoAction)) } diff --git a/cli/command_benchmark_splitters.go b/cli/command_benchmark_splitters.go index b0d2da408..72963b91c 100644 --- a/cli/command_benchmark_splitters.go +++ b/cli/command_benchmark_splitters.go @@ -9,7 +9,6 @@ "github.com/pkg/errors" "github.com/kopia/kopia/internal/clock" - "github.com/kopia/kopia/repo" "github.com/kopia/kopia/repo/splitter" ) @@ -20,7 +19,7 @@ benchmarkSplitterBlockCount = benchmarkSplitterCommand.Flag("block-count", "Number of data blocks to split").Default("16").Int() ) -func runBenchmarkSplitterAction(ctx context.Context, rep repo.Repository) error { +func runBenchmarkSplitterAction(ctx context.Context) error { type benchResult struct { splitter string duration time.Duration @@ -119,5 +118,5 @@ type benchResult struct { } func init() { - benchmarkSplitterCommand.Action(maybeRepositoryAction(runBenchmarkSplitterAction, false)) + benchmarkSplitterCommand.Action(noRepositoryAction(runBenchmarkSplitterAction)) } diff --git a/cli/command_diff.go b/cli/command_diff.go index 6129f696a..e406013bf 100644 --- a/cli/command_diff.go +++ b/cli/command_diff.go @@ -21,7 +21,7 @@ diffCommandCommand = diffCommand.Flag("diff-command", "Displays differences between two repository objects (files or directories)").Default(defaultDiffCommand()).Envar("KOPIA_DIFF").String() ) -func runDiffCommand(ctx context.Context, rep repo.Repository) error { +func runDiffCommand(ctx context.Context, rep repo.Reader) error { ent1, err := snapshotfs.FilesystemEntryFromIDWithPath(ctx, rep, *diffFirstObjectPath, false) if err != nil { return errors.Wrapf(err, "error getting filesystem entry for %v", *diffFirstObjectPath) @@ -67,5 +67,5 @@ func defaultDiffCommand() string { } func init() { - diffCommand.Action(repositoryAction(runDiffCommand)) + diffCommand.Action(repositoryReaderAction(runDiffCommand)) } diff --git a/cli/command_ls.go b/cli/command_ls.go index 1196fdec6..5d5fa4f1c 100644 --- a/cli/command_ls.go +++ b/cli/command_ls.go @@ -24,7 +24,7 @@ lsCommandPath = lsCommand.Arg("object-path", "Path").Required().String() ) -func runLSCommand(ctx context.Context, rep repo.Repository) error { +func runLSCommand(ctx context.Context, rep repo.Reader) error { dir, err := snapshotfs.FilesystemDirectoryFromIDWithPath(ctx, rep, *lsCommandPath, false) if err != nil { return errors.Wrap(err, "unable to get filesystem directory entry") @@ -42,7 +42,7 @@ func runLSCommand(ctx context.Context, rep repo.Repository) error { } func init() { - lsCommand.Action(repositoryAction(runLSCommand)) + lsCommand.Action(repositoryReaderAction(runLSCommand)) } func listDirectory(ctx context.Context, d fs.Directory, prefix, indent string) error { diff --git a/cli/command_manifest_ls.go b/cli/command_manifest_ls.go index 334da6883..1fbc0e55c 100644 --- a/cli/command_manifest_ls.go +++ b/cli/command_manifest_ls.go @@ -18,10 +18,10 @@ ) func init() { - manifestListCommand.Action(repositoryAction(listManifestItems)) + manifestListCommand.Action(repositoryReaderAction(listManifestItems)) } -func listManifestItems(ctx context.Context, rep repo.Repository) error { +func listManifestItems(ctx context.Context, rep repo.Reader) error { filter := map[string]string{} for _, kv := range *manifestListFilter { diff --git a/cli/command_manifest_rm.go b/cli/command_manifest_rm.go index 4c2bc7aac..93043c215 100644 --- a/cli/command_manifest_rm.go +++ b/cli/command_manifest_rm.go @@ -13,7 +13,7 @@ manifestRemoveItems = manifestRemoveCommand.Arg("item", "Items to remove").Required().Strings() ) -func runManifestRemoveCommand(ctx context.Context, rep repo.Repository) error { +func runManifestRemoveCommand(ctx context.Context, rep repo.Writer) error { advancedCommand(ctx) for _, it := range toManifestIDs(*manifestRemoveItems) { @@ -26,5 +26,5 @@ func runManifestRemoveCommand(ctx context.Context, rep repo.Repository) error { } func init() { - manifestRemoveCommand.Action(repositoryAction(runManifestRemoveCommand)) + manifestRemoveCommand.Action(repositoryWriterAction(runManifestRemoveCommand)) } diff --git a/cli/command_manifest_show.go b/cli/command_manifest_show.go index c3021ee50..5cff75524 100644 --- a/cli/command_manifest_show.go +++ b/cli/command_manifest_show.go @@ -17,7 +17,7 @@ ) func init() { - manifestShowCommand.Action(repositoryAction(showManifestItems)) + manifestShowCommand.Action(repositoryReaderAction(showManifestItems)) } func toManifestIDs(s []string) []manifest.ID { @@ -30,7 +30,7 @@ func toManifestIDs(s []string) []manifest.ID { return result } -func showManifestItems(ctx context.Context, rep repo.Repository) error { +func showManifestItems(ctx context.Context, rep repo.Reader) error { for _, it := range toManifestIDs(*manifestShowItems) { var b json.RawMessage diff --git a/cli/command_mount.go b/cli/command_mount.go index 59ffaa20f..0d6d93303 100644 --- a/cli/command_mount.go +++ b/cli/command_mount.go @@ -26,7 +26,7 @@ mountFuseAllowNonEmptyMount = mountCommand.Flag("fuse-allow-non-empty-mount", "Allows the mounting over a non-empty directory. The files in it will be shadowed by the freshly created mount.").Bool() ) -func runMountCommand(ctx context.Context, rep repo.Repository) error { +func runMountCommand(ctx context.Context, rep repo.Reader) error { var entry fs.Directory if *mountObjectID == "all" { @@ -100,5 +100,5 @@ func runMountCommand(ctx context.Context, rep repo.Repository) error { func init() { setupFSCacheFlags(mountCommand) - mountCommand.Action(repositoryAction(runMountCommand)) + mountCommand.Action(repositoryReaderAction(runMountCommand)) } diff --git a/cli/command_policy.go b/cli/command_policy.go index 5f045b5f5..672573653 100644 --- a/cli/command_policy.go +++ b/cli/command_policy.go @@ -11,7 +11,7 @@ "github.com/kopia/kopia/snapshot/policy" ) -func policyTargets(ctx context.Context, rep repo.Repository, globalFlag *bool, targetsFlag *[]string) ([]snapshot.SourceInfo, error) { +func policyTargets(ctx context.Context, rep repo.Reader, globalFlag *bool, targetsFlag *[]string) ([]snapshot.SourceInfo, error) { if *globalFlag == (len(*targetsFlag) > 0) { return nil, errors.New("must pass either '--global' or a list of path targets") } diff --git a/cli/command_policy_edit.go b/cli/command_policy_edit.go index c118e61c1..9037ebfe2 100644 --- a/cli/command_policy_edit.go +++ b/cli/command_policy_edit.go @@ -56,10 +56,10 @@ ) func init() { - policyEditCommand.Action(repositoryAction(editPolicy)) + policyEditCommand.Action(repositoryWriterAction(editPolicy)) } -func editPolicy(ctx context.Context, rep repo.Repository) error { +func editPolicy(ctx context.Context, rep repo.Writer) error { targets, err := policyTargets(ctx, rep, policyEditGlobal, policyEditTargets) if err != nil { return err diff --git a/cli/command_policy_ls.go b/cli/command_policy_ls.go index c7e285295..20e31c538 100644 --- a/cli/command_policy_ls.go +++ b/cli/command_policy_ls.go @@ -14,10 +14,10 @@ var policyListCommand = policyCommands.Command("list", "List policies.").Alias("ls") func init() { - policyListCommand.Action(repositoryAction(listPolicies)) + policyListCommand.Action(repositoryReaderAction(listPolicies)) } -func listPolicies(ctx context.Context, rep repo.Repository) error { +func listPolicies(ctx context.Context, rep repo.Reader) error { policies, err := policy.ListPolicies(ctx, rep) if err != nil { return errors.Wrap(err, "error listing policies") diff --git a/cli/command_policy_remove.go b/cli/command_policy_remove.go index 70b30ade6..58c0ce7c4 100644 --- a/cli/command_policy_remove.go +++ b/cli/command_policy_remove.go @@ -17,10 +17,10 @@ ) func init() { - policyRemoveCommand.Action(repositoryAction(removePolicy)) + policyRemoveCommand.Action(repositoryWriterAction(removePolicy)) } -func removePolicy(ctx context.Context, rep repo.Repository) error { +func removePolicy(ctx context.Context, rep repo.Writer) error { targets, err := policyTargets(ctx, rep, policyRemoveGlobal, policyRemoveTargets) if err != nil { return err diff --git a/cli/command_policy_set.go b/cli/command_policy_set.go index c72fb7210..72d6062b6 100644 --- a/cli/command_policy_set.go +++ b/cli/command_policy_set.go @@ -27,10 +27,10 @@ ) func init() { - policySetCommand.Action(repositoryAction(setPolicy)) + policySetCommand.Action(repositoryWriterAction(setPolicy)) } -func setPolicy(ctx context.Context, rep repo.Repository) error { +func setPolicy(ctx context.Context, rep repo.Writer) error { targets, err := policyTargets(ctx, rep, policySetGlobal, policySetTargets) if err != nil { return err diff --git a/cli/command_policy_show.go b/cli/command_policy_show.go index a9e00917a..ab96982b8 100644 --- a/cli/command_policy_show.go +++ b/cli/command_policy_show.go @@ -21,10 +21,10 @@ ) func init() { - policyShowCommand.Action(repositoryAction(showPolicy)) + policyShowCommand.Action(repositoryReaderAction(showPolicy)) } -func showPolicy(ctx context.Context, rep repo.Repository) error { +func showPolicy(ctx context.Context, rep repo.Reader) error { targets, err := policyTargets(ctx, rep, policyShowGlobal, policyShowTargets) if err != nil { return err diff --git a/cli/command_repository_status.go b/cli/command_repository_status.go index 466a0b06f..8d64fec32 100644 --- a/cli/command_repository_status.go +++ b/cli/command_repository_status.go @@ -21,7 +21,7 @@ statusReconnectTokenIncludePassword = statusCommand.Flag("reconnect-token-with-password", "Include password in reconnect token").Short('s').Bool() ) -func runStatusCommand(ctx context.Context, rep repo.Repository) error { +func runStatusCommand(ctx context.Context, rep repo.Reader) error { fmt.Printf("Config file: %v\n", repositoryConfigFileName()) fmt.Println() fmt.Printf("Description: %v\n", rep.ClientOptions().Description) @@ -110,5 +110,5 @@ func scanCacheDir(dirname string) (fileCount int, totalFileLength int64, err err } func init() { - statusCommand.Action(repositoryAction(runStatusCommand)) + statusCommand.Action(repositoryReaderAction(runStatusCommand)) } diff --git a/cli/command_restore.go b/cli/command_restore.go index 6da830e5d..e00221025 100644 --- a/cli/command_restore.go +++ b/cli/command_restore.go @@ -169,7 +169,7 @@ func printRestoreStats(ctx context.Context, st restore.Stats) { log(ctx).Infof("Restored %v files, %v directories and %v symbolic links (%v)\n", st.RestoredFileCount, st.RestoredDirCount, st.RestoredSymlinkCount, units.BytesStringBase10(st.RestoredTotalFileSize)) } -func runRestoreCommand(ctx context.Context, rep repo.Repository) error { +func runRestoreCommand(ctx context.Context, rep repo.Reader) error { output, err := restoreOutput(ctx) if err != nil { return errors.Wrap(err, "unable to initialize output") @@ -224,5 +224,5 @@ func runRestoreCommand(ctx context.Context, rep repo.Repository) error { func init() { addRestoreFlags(restoreCommand) - restoreCommand.Action(repositoryAction(runRestoreCommand)) + restoreCommand.Action(repositoryReaderAction(runRestoreCommand)) } diff --git a/cli/command_show.go b/cli/command_show.go index 88f01ccdc..f127b6a36 100644 --- a/cli/command_show.go +++ b/cli/command_show.go @@ -16,7 +16,7 @@ catCommandPath = catCommand.Arg("object-path", "Path").Required().String() ) -func runCatCommand(ctx context.Context, rep repo.Repository) error { +func runCatCommand(ctx context.Context, rep repo.Reader) error { oid, err := snapshotfs.ParseObjectIDWithPath(ctx, rep, *catCommandPath) if err != nil { return errors.Wrapf(err, "unable to parse ID: %v", *catCommandPath) @@ -35,5 +35,5 @@ func runCatCommand(ctx context.Context, rep repo.Repository) error { } func init() { - catCommand.Action(repositoryAction(runCatCommand)) + catCommand.Action(repositoryReaderAction(runCatCommand)) } diff --git a/cli/command_snapshot_copy_move_history.go b/cli/command_snapshot_copy_move_history.go index 0e6852827..57460d348 100644 --- a/cli/command_snapshot_copy_move_history.go +++ b/cli/command_snapshot_copy_move_history.go @@ -71,7 +71,7 @@ func registerSnapshotCopyFlags(cmd *kingpin.CmdClause) { // user1@host1:/path1 @host2 copy to user1@host2:/path1 // user1@host1:/path1 user2@host2 copy to user2@host2:/path1 // user1@host1:/path1 user2@host2:/path2 copy snapshots from single path. -func runSnapshotCopyCommand(ctx context.Context, rep repo.Repository, isMoveCommand bool) error { +func runSnapshotCopyCommand(ctx context.Context, rep repo.Writer, isMoveCommand bool) error { si, di, err := getCopySourceAndDestination(rep) if err != nil { return err @@ -150,7 +150,7 @@ func getCopySnapshotAction(isMoveCommand bool) string { return action } -func getCopySourceAndDestination(rep repo.Repository) (si, di snapshot.SourceInfo, err error) { +func getCopySourceAndDestination(rep repo.Writer) (si, di snapshot.SourceInfo, err error) { si, err = snapshot.ParseSourceInfo(snapshotCopyOrMoveSource, rep.ClientOptions().Hostname, rep.ClientOptions().Username) if err != nil { return si, di, errors.Wrap(err, "invalid source") @@ -231,12 +231,12 @@ func getCopyDestination(source, overrides snapshot.SourceInfo) snapshot.SourceIn func init() { registerSnapshotCopyFlags(snapshotCopyCommand) - snapshotCopyCommand.Action(repositoryAction(func(ctx context.Context, rep repo.Repository) error { + snapshotCopyCommand.Action(repositoryWriterAction(func(ctx context.Context, rep repo.Writer) error { return runSnapshotCopyCommand(ctx, rep, false) })) registerSnapshotCopyFlags(snapshotMoveCommand) - snapshotMoveCommand.Action(repositoryAction(func(ctx context.Context, rep repo.Repository) error { + snapshotMoveCommand.Action(repositoryWriterAction(func(ctx context.Context, rep repo.Writer) error { return runSnapshotCopyCommand(ctx, rep, true) })) } diff --git a/cli/command_snapshot_create.go b/cli/command_snapshot_create.go index 8a8149fc6..1edeef030 100644 --- a/cli/command_snapshot_create.go +++ b/cli/command_snapshot_create.go @@ -36,7 +36,7 @@ snapshotCreateForceDisableActions = snapshotCreateCommand.Flag("force-disable-actions", "Disable snapshot actions even if globally enabled on this client").Hidden().Bool() ) -func runSnapshotCommand(ctx context.Context, rep repo.Repository) error { +func runSnapshotCommand(ctx context.Context, rep repo.Writer) error { sources := *snapshotCreateSources maybeAutoUpgradeRepository(ctx, rep) @@ -113,7 +113,7 @@ func validateStartEndTime(st, et string) error { return nil } -func setupUploader(rep repo.Repository) *snapshotfs.Uploader { +func setupUploader(rep repo.Writer) *snapshotfs.Uploader { u := snapshotfs.NewUploader(rep) u.MaxUploadBytes = *snapshotCreateCheckpointUploadLimitMB << 20 //nolint:gomnd @@ -152,7 +152,7 @@ func startTimeAfterEndTime(startTime, endTime time.Time) bool { startTime.After(endTime) } -func snapshotSingleSource(ctx context.Context, rep repo.Repository, u *snapshotfs.Uploader, sourceInfo snapshot.SourceInfo) error { +func snapshotSingleSource(ctx context.Context, rep repo.Writer, u *snapshotfs.Uploader, sourceInfo snapshot.SourceInfo) error { log(ctx).Infof("Snapshotting %v ...", sourceInfo) t0 := clock.Now() @@ -235,7 +235,7 @@ func snapshotSingleSource(ctx context.Context, rep repo.Repository, u *snapshotf // findPreviousSnapshotManifest returns the list of previous snapshots for a given source, including // last complete snapshot and possibly some number of incomplete snapshots following it. -func findPreviousSnapshotManifest(ctx context.Context, rep repo.Repository, sourceInfo snapshot.SourceInfo, noLaterThan *time.Time) ([]*snapshot.Manifest, error) { +func findPreviousSnapshotManifest(ctx context.Context, rep repo.Reader, sourceInfo snapshot.SourceInfo, noLaterThan *time.Time) ([]*snapshot.Manifest, error) { man, err := snapshot.ListSnapshots(ctx, rep, sourceInfo) if err != nil { return nil, errors.Wrap(err, "error listing previous snapshots") @@ -277,7 +277,7 @@ func findPreviousSnapshotManifest(ctx context.Context, rep repo.Repository, sour return result, nil } -func getLocalBackupPaths(ctx context.Context, rep repo.Repository) ([]string, error) { +func getLocalBackupPaths(ctx context.Context, rep repo.Reader) ([]string, error) { log(ctx).Debugf("Looking for previous backups of '%v@%v'...", rep.ClientOptions().Hostname, rep.ClientOptions().Username) sources, err := snapshot.ListSources(ctx, rep) @@ -297,5 +297,5 @@ func getLocalBackupPaths(ctx context.Context, rep repo.Repository) ([]string, er } func init() { - snapshotCreateCommand.Action(repositoryAction(runSnapshotCommand)) + snapshotCreateCommand.Action(repositoryWriterAction(runSnapshotCommand)) } diff --git a/cli/command_snapshot_delete.go b/cli/command_snapshot_delete.go index 04949767b..ceef1f75d 100644 --- a/cli/command_snapshot_delete.go +++ b/cli/command_snapshot_delete.go @@ -18,7 +18,7 @@ snapshotDeleteConfirm = snapshotDeleteCommand.Flag("delete", "Confirm deletion").Bool() ) -func runDeleteCommand(ctx context.Context, rep repo.Repository) error { +func runDeleteCommand(ctx context.Context, rep repo.Writer) error { for _, id := range *snapshotDeleteIDs { m, err := snapshot.LoadSnapshot(ctx, rep, manifest.ID(id)) if err == nil { @@ -36,7 +36,7 @@ func runDeleteCommand(ctx context.Context, rep repo.Repository) error { return nil } -func deleteSnapshot(ctx context.Context, rep repo.Repository, m *snapshot.Manifest) error { +func deleteSnapshot(ctx context.Context, rep repo.Writer, m *snapshot.Manifest) error { desc := fmt.Sprintf("snapshot %v of %v at %v", m.ID, m.Source, formatTimestamp(m.StartTime)) if !*snapshotDeleteConfirm { @@ -49,7 +49,7 @@ func deleteSnapshot(ctx context.Context, rep repo.Repository, m *snapshot.Manife return rep.DeleteManifest(ctx, m.ID) } -func deleteSnapshotsByRootObjectID(ctx context.Context, rep repo.Repository, rootID object.ID) error { +func deleteSnapshotsByRootObjectID(ctx context.Context, rep repo.Writer, rootID object.ID) error { manifests, err := snapshot.FindSnapshotsByRootObjectID(ctx, rep, rootID) if err != nil { return errors.Wrapf(err, "unable to find snapshots by root %v", rootID) @@ -69,7 +69,7 @@ func deleteSnapshotsByRootObjectID(ctx context.Context, rep repo.Repository, roo } func init() { - snapshotDeleteCommand.Action(repositoryAction(runDeleteCommand)) + snapshotDeleteCommand.Action(repositoryWriterAction(runDeleteCommand)) // hidden flag for backwards compatibility snapshotDeleteCommand.Flag("unsafe-ignore-source", "Alias for --delete").Hidden().BoolVar(snapshotDeleteConfirm) diff --git a/cli/command_snapshot_estimate.go b/cli/command_snapshot_estimate.go index 0d506364e..d74f6916f 100644 --- a/cli/command_snapshot_estimate.go +++ b/cli/command_snapshot_estimate.go @@ -64,7 +64,7 @@ func makeBuckets() buckets { } } -func runSnapshotEstimateCommand(ctx context.Context, rep repo.Repository) error { +func runSnapshotEstimateCommand(ctx context.Context, rep repo.Reader) error { path, err := filepath.Abs(*snapshotEstimateSource) if err != nil { return errors.Errorf("invalid path: '%s': %s", path, err) @@ -173,5 +173,5 @@ func estimate(ctx context.Context, relativePath string, entry fs.Entry, stats *s } func init() { - snapshotEstimate.Action(repositoryAction(runSnapshotEstimateCommand)) + snapshotEstimate.Action(repositoryReaderAction(runSnapshotEstimateCommand)) } diff --git a/cli/command_snapshot_expire.go b/cli/command_snapshot_expire.go index 3465f85e9..8cc46fc5a 100644 --- a/cli/command_snapshot_expire.go +++ b/cli/command_snapshot_expire.go @@ -19,7 +19,7 @@ snapshotExpireDelete = snapshotExpireCommand.Flag("delete", "Whether to actually delete snapshots").Bool() ) -func getSnapshotSourcesToExpire(ctx context.Context, rep repo.Repository) ([]snapshot.SourceInfo, error) { +func getSnapshotSourcesToExpire(ctx context.Context, rep repo.Reader) ([]snapshot.SourceInfo, error) { if *snapshotExpireAll { return snapshot.ListSources(ctx, rep) } @@ -38,7 +38,7 @@ func getSnapshotSourcesToExpire(ctx context.Context, rep repo.Repository) ([]sna return result, nil } -func runExpireCommand(ctx context.Context, rep repo.Repository) error { +func runExpireCommand(ctx context.Context, rep repo.Writer) error { sources, err := getSnapshotSourcesToExpire(ctx, rep) if err != nil { return err @@ -74,5 +74,5 @@ func runExpireCommand(ctx context.Context, rep repo.Repository) error { } func init() { - snapshotExpireCommand.Action(repositoryAction(runExpireCommand)) + snapshotExpireCommand.Action(repositoryWriterAction(runExpireCommand)) } diff --git a/cli/command_snapshot_list.go b/cli/command_snapshot_list.go index 49a5f535a..5f9c92f14 100644 --- a/cli/command_snapshot_list.go +++ b/cli/command_snapshot_list.go @@ -35,7 +35,7 @@ maxResultsPerPath = snapshotListCommand.Flag("max-results", "Maximum number of entries per source.").Short('n').Int() ) -func findSnapshotsForSource(ctx context.Context, rep repo.Repository, sourceInfo snapshot.SourceInfo) (manifestIDs []manifest.ID, relPath string, err error) { +func findSnapshotsForSource(ctx context.Context, rep repo.Reader, sourceInfo snapshot.SourceInfo) (manifestIDs []manifest.ID, relPath string, err error) { for len(sourceInfo.Path) > 0 { list, err := snapshot.ListSnapshotManifests(ctx, rep, &sourceInfo) if err != nil { @@ -65,7 +65,7 @@ func findSnapshotsForSource(ctx context.Context, rep repo.Repository, sourceInfo return nil, "", nil } -func findManifestIDs(ctx context.Context, rep repo.Repository, source string) ([]manifest.ID, string, error) { +func findManifestIDs(ctx context.Context, rep repo.Reader, source string) ([]manifest.ID, string, error) { if source == "" { man, err := snapshot.ListSnapshotManifests(ctx, rep, nil) return man, "", errors.Wrap(err, "error listing all snapshot manifests") @@ -84,7 +84,7 @@ func findManifestIDs(ctx context.Context, rep repo.Repository, source string) ([ return manifestIDs, relPath, err } -func runSnapshotsCommand(ctx context.Context, rep repo.Repository) error { +func runSnapshotsCommand(ctx context.Context, rep repo.Reader) error { manifestIDs, relPath, err := findManifestIDs(ctx, rep, *snapshotListPath) if err != nil { return err @@ -98,7 +98,7 @@ func runSnapshotsCommand(ctx context.Context, rep repo.Repository) error { return outputManifestGroups(ctx, rep, manifests, strings.Split(relPath, "/")) } -func shouldOutputSnapshotSource(rep repo.Repository, src snapshot.SourceInfo) bool { +func shouldOutputSnapshotSource(rep repo.Reader, src snapshot.SourceInfo) bool { if *snapshotListShowAll { return true } @@ -112,7 +112,7 @@ func shouldOutputSnapshotSource(rep repo.Repository, src snapshot.SourceInfo) bo return src.UserName == co.Username } -func outputManifestGroups(ctx context.Context, rep repo.Repository, manifests []*snapshot.Manifest, relPathParts []string) error { +func outputManifestGroups(ctx context.Context, rep repo.Reader, manifests []*snapshot.Manifest, relPathParts []string) error { separator := "" var anyOutput bool @@ -148,7 +148,7 @@ func outputManifestGroups(ctx context.Context, rep repo.Repository, manifests [] return nil } -func outputManifestFromSingleSource(ctx context.Context, rep repo.Repository, manifests []*snapshot.Manifest, parts []string) error { +func outputManifestFromSingleSource(ctx context.Context, rep repo.Reader, manifests []*snapshot.Manifest, parts []string) error { var ( count int lastTotalFileSize int64 @@ -282,5 +282,5 @@ func deltaBytes(b int64) string { } func init() { - snapshotListCommand.Action(repositoryAction(runSnapshotsCommand)) + snapshotListCommand.Action(repositoryReaderAction(runSnapshotsCommand)) } diff --git a/cli/command_snapshot_migrate.go b/cli/command_snapshot_migrate.go index e9c5cd4d8..9fc95fe3d 100644 --- a/cli/command_snapshot_migrate.go +++ b/cli/command_snapshot_migrate.go @@ -26,7 +26,7 @@ migrateParallel = migrateCommand.Flag("parallel", "Number of sources to migrate in parallel").Default("1").Int() ) -func runMigrateCommand(ctx context.Context, destRepo repo.Repository) error { +func runMigrateCommand(ctx context.Context, destRepo repo.Writer) error { sourceRepo, err := openSourceRepo(ctx) if err != nil { return errors.Wrap(err, "can't open source repository") @@ -114,7 +114,7 @@ func runMigrateCommand(ctx context.Context, destRepo repo.Repository) error { return nil } -func openSourceRepo(ctx context.Context) (repo.Repository, error) { +func openSourceRepo(ctx context.Context) (repo.Reader, error) { pass, ok := repo.GetPersistedPassword(ctx, *migrateSourceConfig) if !ok { var err error @@ -132,7 +132,7 @@ func openSourceRepo(ctx context.Context) (repo.Repository, error) { return sourceRepo, nil } -func migratePoliciesForSources(ctx context.Context, sourceRepo, destRepo repo.Repository, sources []snapshot.SourceInfo) error { +func migratePoliciesForSources(ctx context.Context, sourceRepo repo.Reader, destRepo repo.Writer, sources []snapshot.SourceInfo) error { for _, si := range sources { if err := migrateSinglePolicy(ctx, sourceRepo, destRepo, si); err != nil { return errors.Wrapf(err, "unable to migrate policy for %v", si) @@ -142,7 +142,7 @@ func migratePoliciesForSources(ctx context.Context, sourceRepo, destRepo repo.Re return nil } -func migrateAllPolicies(ctx context.Context, sourceRepo, destRepo repo.Repository) error { +func migrateAllPolicies(ctx context.Context, sourceRepo repo.Reader, destRepo repo.Writer) error { policies, err := policy.ListPolicies(ctx, sourceRepo) if err != nil { return errors.Wrap(err, "unable to list source policies") @@ -157,7 +157,7 @@ func migrateAllPolicies(ctx context.Context, sourceRepo, destRepo repo.Repositor return nil } -func migrateSinglePolicy(ctx context.Context, sourceRepo, destRepo repo.Repository, si snapshot.SourceInfo) error { +func migrateSinglePolicy(ctx context.Context, sourceRepo repo.Reader, destRepo repo.Writer, si snapshot.SourceInfo) error { pol, err := policy.GetDefinedPolicy(ctx, sourceRepo, si) if errors.Is(err, policy.ErrPolicyNotFound) { return nil @@ -183,7 +183,7 @@ func migrateSinglePolicy(ctx context.Context, sourceRepo, destRepo repo.Reposito return policy.SetPolicy(ctx, destRepo, si, pol) } -func findPreviousSnapshotManifestWithStartTime(ctx context.Context, rep repo.Repository, sourceInfo snapshot.SourceInfo, startTime time.Time) (*snapshot.Manifest, error) { +func findPreviousSnapshotManifestWithStartTime(ctx context.Context, rep repo.Reader, sourceInfo snapshot.SourceInfo, startTime time.Time) (*snapshot.Manifest, error) { previous, err := snapshot.ListSnapshots(ctx, rep, sourceInfo) if err != nil { return nil, errors.Wrap(err, "error listing previous snapshots") @@ -198,7 +198,7 @@ func findPreviousSnapshotManifestWithStartTime(ctx context.Context, rep repo.Rep return nil, nil } -func migrateSingleSource(ctx context.Context, uploader *snapshotfs.Uploader, sourceRepo, destRepo repo.Repository, s snapshot.SourceInfo) error { +func migrateSingleSource(ctx context.Context, uploader *snapshotfs.Uploader, sourceRepo repo.Reader, destRepo repo.Writer, s snapshot.SourceInfo) error { manifests, err := snapshot.ListSnapshotManifests(ctx, sourceRepo, &s) if err != nil { return errors.Wrapf(err, "error listing snapshot manifests for %v", s) @@ -226,7 +226,7 @@ func migrateSingleSource(ctx context.Context, uploader *snapshotfs.Uploader, sou return nil } -func migrateSingleSourceSnapshot(ctx context.Context, uploader *snapshotfs.Uploader, sourceRepo, destRepo repo.Repository, s snapshot.SourceInfo, m *snapshot.Manifest) error { +func migrateSingleSourceSnapshot(ctx context.Context, uploader *snapshotfs.Uploader, sourceRepo repo.Reader, destRepo repo.Writer, s snapshot.SourceInfo, m *snapshot.Manifest) error { if m.IncompleteReason != "" { log(ctx).Debugf("ignoring incomplete %v at %v", s, formatTimestamp(m.StartTime)) return nil @@ -282,7 +282,7 @@ func filterSnapshotsToMigrate(s []*snapshot.Manifest) []*snapshot.Manifest { return s } -func getSourcesToMigrate(ctx context.Context, rep repo.Repository) ([]snapshot.SourceInfo, error) { +func getSourcesToMigrate(ctx context.Context, rep repo.Reader) ([]snapshot.SourceInfo, error) { if len(*migrateSources) > 0 { var result []snapshot.SourceInfo @@ -306,5 +306,5 @@ func getSourcesToMigrate(ctx context.Context, rep repo.Repository) ([]snapshot.S } func init() { - migrateCommand.Action(repositoryAction(runMigrateCommand)) + migrateCommand.Action(repositoryWriterAction(runMigrateCommand)) } diff --git a/cli/command_snapshot_restore.go b/cli/command_snapshot_restore.go index 2e17f684e..b2224e832 100644 --- a/cli/command_snapshot_restore.go +++ b/cli/command_snapshot_restore.go @@ -4,5 +4,5 @@ func init() { addRestoreFlags(snapshotRestoreCommand) - snapshotRestoreCommand.Action(repositoryAction(runRestoreCommand)) + snapshotRestoreCommand.Action(repositoryReaderAction(runRestoreCommand)) } diff --git a/cli/command_snapshot_verify.go b/cli/command_snapshot_verify.go index a967b0f75..96904dcab 100644 --- a/cli/command_snapshot_verify.go +++ b/cli/command_snapshot_verify.go @@ -34,7 +34,7 @@ ) type verifier struct { - rep repo.Repository + rep repo.Reader workQueue *parallelwork.Queue startTime time.Time @@ -197,7 +197,7 @@ func (v *verifier) readEntireObject(ctx context.Context, oid object.ID, path str return errors.Wrap(err, "unable to read data") } -func runVerifyCommand(ctx context.Context, rep repo.Repository) error { +func runVerifyCommand(ctx context.Context, rep repo.Reader) error { if *verifyCommandAllSources { log(ctx).Noticef("DEPRECATED: --all-sources flag has no effect and is the default when no sources are provided.") } @@ -234,7 +234,7 @@ func runVerifyCommand(ctx context.Context, rep repo.Repository) error { return errors.Errorf("encountered %v errors", len(v.errors)) } -func enqueueRootsToVerify(ctx context.Context, v *verifier, rep repo.Repository) error { +func enqueueRootsToVerify(ctx context.Context, v *verifier, rep repo.Reader) error { manifests, err := loadSourceManifests(ctx, rep, *verifyCommandSources) if err != nil { return err @@ -275,7 +275,7 @@ func enqueueRootsToVerify(ctx context.Context, v *verifier, rep repo.Repository) return nil } -func loadSourceManifests(ctx context.Context, rep repo.Repository, sources []string) ([]*snapshot.Manifest, error) { +func loadSourceManifests(ctx context.Context, rep repo.Reader, sources []string) ([]*snapshot.Manifest, error) { var manifestIDs []manifest.ID if len(sources)+len(*verifyCommandDirObjectIDs)+len(*verifyCommandFileObjectIDs) == 0 { @@ -303,5 +303,5 @@ func loadSourceManifests(ctx context.Context, rep repo.Repository, sources []str } func init() { - verifyCommand.Action(repositoryAction(runVerifyCommand)) + verifyCommand.Action(repositoryReaderAction(runVerifyCommand)) } diff --git a/internal/server/server.go b/internal/server/server.go index fbbd7ea22..6bd689c31 100644 --- a/internal/server/server.go +++ b/internal/server/server.go @@ -291,7 +291,7 @@ func (s *Server) refreshPeriodically(ctx context.Context, r repo.Repository) { } } -func (s *Server) periodicMaintenance(ctx context.Context, r repo.Repository) { +func (s *Server) periodicMaintenance(ctx context.Context, r repo.Writer) { for { select { case <-ctx.Done(): diff --git a/repo/maintenance/maintenance_run.go b/repo/maintenance/maintenance_run.go index 188543973..28e38931c 100644 --- a/repo/maintenance/maintenance_run.go +++ b/repo/maintenance/maintenance_run.go @@ -9,10 +9,10 @@ "github.com/gofrs/flock" "github.com/pkg/errors" + "github.com/kopia/kopia/repo" "github.com/kopia/kopia/repo/blob" "github.com/kopia/kopia/repo/content" "github.com/kopia/kopia/repo/logging" - "github.com/kopia/kopia/repo/manifest" ) // safetyMarginBetweenSnapshotGC is the minimal amount of time that must pass between snapshot @@ -34,16 +34,12 @@ type MaintainableRepository interface { Username() string Hostname() string - Time() time.Time ConfigFilename() string BlobStorage() blob.Storage ContentManager() *content.Manager - GetManifest(ctx context.Context, id manifest.ID, data interface{}) (*manifest.EntryMetadata, error) - PutManifest(ctx context.Context, labels map[string]string, payload interface{}) (manifest.ID, error) - FindManifests(ctx context.Context, labels map[string]string) ([]*manifest.EntryMetadata, error) - DeleteManifest(ctx context.Context, id manifest.ID) error + repo.Writer DeriveKey(purpose []byte, keyLength int) []byte } diff --git a/repo/repository.go b/repo/repository.go index e8a8bbbed..0e65ec1a1 100644 --- a/repo/repository.go +++ b/repo/repository.go @@ -13,24 +13,37 @@ "github.com/kopia/kopia/repo/object" ) -// Repository exposes public API of Kopia repository, including objects and manifests. -type Repository interface { +// Reader provides methods to read from a repository. +type Reader interface { OpenObject(ctx context.Context, id object.ID) (object.Reader, error) - NewObjectWriter(ctx context.Context, opt object.WriterOptions) object.Writer VerifyObject(ctx context.Context, id object.ID) ([]content.ID, error) GetManifest(ctx context.Context, id manifest.ID, data interface{}) (*manifest.EntryMetadata, error) - PutManifest(ctx context.Context, labels map[string]string, payload interface{}) (manifest.ID, error) FindManifests(ctx context.Context, labels map[string]string) ([]*manifest.EntryMetadata, error) - DeleteManifest(ctx context.Context, id manifest.ID) error - - ClientOptions() ClientOptions - UpdateDescription(d string) Time() time.Time + ClientOptions() ClientOptions +} + +// Writer provides methods to write to a repository. +type Writer interface { + Reader + + NewObjectWriter(ctx context.Context, opt object.WriterOptions) object.Writer + PutManifest(ctx context.Context, labels map[string]string, payload interface{}) (manifest.ID, error) + DeleteManifest(ctx context.Context, id manifest.ID) error + + Flush(ctx context.Context) error +} + +// Repository exposes public API of Kopia repository, including objects and manifests. +type Repository interface { + Reader + Writer + + UpdateDescription(d string) Refresh(ctx context.Context) error - Flush(ctx context.Context) error Close(ctx context.Context) error } diff --git a/snapshot/manager.go b/snapshot/manager.go index 0c873cb44..de775dab2 100644 --- a/snapshot/manager.go +++ b/snapshot/manager.go @@ -27,7 +27,7 @@ var log = logging.GetContextLoggerFunc("kopia/snapshot") // ListSources lists all snapshot sources in a given repository. -func ListSources(ctx context.Context, rep repo.Repository) ([]SourceInfo, error) { +func ListSources(ctx context.Context, rep repo.Reader) ([]SourceInfo, error) { items, err := rep.FindManifests(ctx, map[string]string{ typeKey: ManifestType, }) @@ -70,7 +70,7 @@ func sourceInfoToLabels(si SourceInfo) map[string]string { } // ListSnapshots lists all snapshots for a given source. -func ListSnapshots(ctx context.Context, rep repo.Repository, si SourceInfo) ([]*Manifest, error) { +func ListSnapshots(ctx context.Context, rep repo.Reader, si SourceInfo) ([]*Manifest, error) { entries, err := rep.FindManifests(ctx, sourceInfoToLabels(si)) if err != nil { return nil, errors.Wrap(err, "unable to find manifest entries") @@ -80,7 +80,7 @@ func ListSnapshots(ctx context.Context, rep repo.Repository, si SourceInfo) ([]* } // LoadSnapshot loads and parses a snapshot with a given ID. -func LoadSnapshot(ctx context.Context, rep repo.Repository, manifestID manifest.ID) (*Manifest, error) { +func LoadSnapshot(ctx context.Context, rep repo.Reader, manifestID manifest.ID) (*Manifest, error) { sm := &Manifest{} em, err := rep.GetManifest(ctx, manifestID, sm) @@ -103,7 +103,7 @@ func LoadSnapshot(ctx context.Context, rep repo.Repository, manifestID manifest. } // SaveSnapshot persists given snapshot manifest and returns manifest ID. -func SaveSnapshot(ctx context.Context, rep repo.Repository, man *Manifest) (manifest.ID, error) { +func SaveSnapshot(ctx context.Context, rep repo.Writer, man *Manifest) (manifest.ID, error) { if man.Source.Host == "" { return "", errors.New("missing host") } @@ -127,7 +127,7 @@ func SaveSnapshot(ctx context.Context, rep repo.Repository, man *Manifest) (mani } // LoadSnapshots efficiently loads and parses a given list of snapshot IDs. -func LoadSnapshots(ctx context.Context, rep repo.Repository, manifestIDs []manifest.ID) ([]*Manifest, error) { +func LoadSnapshots(ctx context.Context, rep repo.Reader, manifestIDs []manifest.ID) ([]*Manifest, error) { result := make([]*Manifest, len(manifestIDs)) sem := make(chan bool, loadSnapshotsConcurrency) @@ -164,7 +164,7 @@ func LoadSnapshots(ctx context.Context, rep repo.Repository, manifestIDs []manif } // ListSnapshotManifests returns the list of snapshot manifests for a given source or all sources if nil. -func ListSnapshotManifests(ctx context.Context, rep repo.Repository, src *SourceInfo) ([]manifest.ID, error) { +func ListSnapshotManifests(ctx context.Context, rep repo.Reader, src *SourceInfo) ([]manifest.ID, error) { labels := map[string]string{ typeKey: ManifestType, } @@ -182,7 +182,7 @@ func ListSnapshotManifests(ctx context.Context, rep repo.Repository, src *Source } // FindSnapshotsByRootObjectID returns the list of matching snapshots for a given rootID. -func FindSnapshotsByRootObjectID(ctx context.Context, rep repo.Repository, rootID object.ID) ([]*Manifest, error) { +func FindSnapshotsByRootObjectID(ctx context.Context, rep repo.Reader, rootID object.ID) ([]*Manifest, error) { ids, err := ListSnapshotManifests(ctx, rep, nil) if err != nil { return nil, errors.Wrap(err, "error listing snapshot manifests") diff --git a/snapshot/policy/expire.go b/snapshot/policy/expire.go index a16710d36..7fa5f0d2b 100644 --- a/snapshot/policy/expire.go +++ b/snapshot/policy/expire.go @@ -11,7 +11,7 @@ ) // ApplyRetentionPolicy applies retention policy to a given source by deleting expired snapshots. -func ApplyRetentionPolicy(ctx context.Context, rep repo.Repository, sourceInfo snapshot.SourceInfo, reallyDelete bool) ([]*snapshot.Manifest, error) { +func ApplyRetentionPolicy(ctx context.Context, rep repo.Writer, sourceInfo snapshot.SourceInfo, reallyDelete bool) ([]*snapshot.Manifest, error) { snapshots, err := snapshot.ListSnapshots(ctx, rep, sourceInfo) if err != nil { return nil, errors.Wrap(err, "error listing snapshots") @@ -33,7 +33,7 @@ func ApplyRetentionPolicy(ctx context.Context, rep repo.Repository, sourceInfo s return toDelete, nil } -func getExpiredSnapshots(ctx context.Context, rep repo.Repository, snapshots []*snapshot.Manifest) ([]*snapshot.Manifest, error) { +func getExpiredSnapshots(ctx context.Context, rep repo.Reader, snapshots []*snapshot.Manifest) ([]*snapshot.Manifest, error) { var toDelete []*snapshot.Manifest for _, snapshotGroup := range snapshot.GroupBySource(snapshots) { @@ -48,7 +48,7 @@ func getExpiredSnapshots(ctx context.Context, rep repo.Repository, snapshots []* return toDelete, nil } -func getExpiredSnapshotsForSource(ctx context.Context, rep repo.Repository, snapshots []*snapshot.Manifest) ([]*snapshot.Manifest, error) { +func getExpiredSnapshotsForSource(ctx context.Context, rep repo.Reader, snapshots []*snapshot.Manifest) ([]*snapshot.Manifest, error) { src := snapshots[0].Source pol, _, err := GetEffectivePolicy(ctx, rep, src) diff --git a/snapshot/policy/policy_manager.go b/snapshot/policy/policy_manager.go index fc8a73747..697abdc1f 100644 --- a/snapshot/policy/policy_manager.go +++ b/snapshot/policy/policy_manager.go @@ -23,7 +23,7 @@ // GetEffectivePolicy calculates effective snapshot policy for a given source by combining the source-specifc policy (if any) // with parent policies. The source must contain a path. // Returns the effective policies and all source policies that contributed to that (most specific first). -func GetEffectivePolicy(ctx context.Context, rep repo.Repository, si snapshot.SourceInfo) (effective *Policy, sources []*Policy, e error) { +func GetEffectivePolicy(ctx context.Context, rep repo.Reader, si snapshot.SourceInfo) (effective *Policy, sources []*Policy, e error) { var md []*manifest.EntryMetadata // Find policies applying to paths all the way up to the root. @@ -86,7 +86,7 @@ func GetEffectivePolicy(ctx context.Context, rep repo.Repository, si snapshot.So } // GetDefinedPolicy returns the policy defined on the provided snapshot.SourceInfo or ErrPolicyNotFound if not present. -func GetDefinedPolicy(ctx context.Context, rep repo.Repository, si snapshot.SourceInfo) (*Policy, error) { +func GetDefinedPolicy(ctx context.Context, rep repo.Reader, si snapshot.SourceInfo) (*Policy, error) { md, err := rep.FindManifests(ctx, labelsForSource(si)) if err != nil { return nil, errors.Wrap(err, "unable to find policy for source") @@ -112,7 +112,7 @@ func GetDefinedPolicy(ctx context.Context, rep repo.Repository, si snapshot.Sour } // SetPolicy sets the policy on a given source. -func SetPolicy(ctx context.Context, rep repo.Repository, si snapshot.SourceInfo, pol *Policy) error { +func SetPolicy(ctx context.Context, rep repo.Writer, si snapshot.SourceInfo, pol *Policy) error { md, err := rep.FindManifests(ctx, labelsForSource(si)) if err != nil { return errors.Wrapf(err, "unable to load manifests for %v", si) @@ -132,7 +132,7 @@ func SetPolicy(ctx context.Context, rep repo.Repository, si snapshot.SourceInfo, } // RemovePolicy removes the policy for a given source. -func RemovePolicy(ctx context.Context, rep repo.Repository, si snapshot.SourceInfo) error { +func RemovePolicy(ctx context.Context, rep repo.Writer, si snapshot.SourceInfo) error { md, err := rep.FindManifests(ctx, labelsForSource(si)) if err != nil { return errors.Wrapf(err, "unable to load manifests for %v", si) @@ -148,7 +148,7 @@ func RemovePolicy(ctx context.Context, rep repo.Repository, si snapshot.SourceIn } // GetPolicyByID gets the policy for a given unique ID or ErrPolicyNotFound if not found. -func GetPolicyByID(ctx context.Context, rep repo.Repository, id manifest.ID) (*Policy, error) { +func GetPolicyByID(ctx context.Context, rep repo.Reader, id manifest.ID) (*Policy, error) { p := &Policy{} if err := loadPolicyFromManifest(ctx, rep, id, p); err != nil { return nil, err @@ -158,7 +158,7 @@ func GetPolicyByID(ctx context.Context, rep repo.Repository, id manifest.ID) (*P } // ListPolicies returns a list of all policies. -func ListPolicies(ctx context.Context, rep repo.Repository) ([]*Policy, error) { +func ListPolicies(ctx context.Context, rep repo.Reader) ([]*Policy, error) { ids, err := rep.FindManifests(ctx, map[string]string{ typeKey: "policy", }) @@ -190,7 +190,7 @@ func (m SubdirectoryPolicyMap) GetPolicyForPath(relativePath string) (*Policy, e } // TreeForSource returns policy Tree for a given source. -func TreeForSource(ctx context.Context, rep repo.Repository, si snapshot.SourceInfo) (*Tree, error) { +func TreeForSource(ctx context.Context, rep repo.Reader, si snapshot.SourceInfo) (*Tree, error) { pols, err := applicablePoliciesForSource(ctx, rep, si) if err != nil { return nil, errors.Wrap(err, "unable to get policies") @@ -199,7 +199,7 @@ func TreeForSource(ctx context.Context, rep repo.Repository, si snapshot.SourceI return BuildTree(pols, DefaultPolicy), nil } -func applicablePoliciesForSource(ctx context.Context, rep repo.Repository, si snapshot.SourceInfo) (map[string]*Policy, error) { +func applicablePoliciesForSource(ctx context.Context, rep repo.Reader, si snapshot.SourceInfo) (map[string]*Policy, error) { result := map[string]*Policy{} pol, _, err := GetEffectivePolicy(ctx, rep, si) @@ -247,7 +247,7 @@ func applicablePoliciesForSource(ctx context.Context, rep repo.Repository, si sn return result, nil } -func loadPolicyFromManifest(ctx context.Context, rep repo.Repository, id manifest.ID, pol *Policy) error { +func loadPolicyFromManifest(ctx context.Context, rep repo.Reader, id manifest.ID, pol *Policy) error { md, err := rep.GetManifest(ctx, id, pol) if err != nil { if errors.Is(err, manifest.ErrNotFound) { diff --git a/snapshot/restore/restore.go b/snapshot/restore/restore.go index 66f06f359..4c45e1ec8 100644 --- a/snapshot/restore/restore.go +++ b/snapshot/restore/restore.go @@ -59,7 +59,7 @@ type Options struct { } // Entry walks a snapshot root with given root entry and restores it to the provided output. -func Entry(ctx context.Context, rep repo.Repository, output Output, rootEntry fs.Entry, options Options) (Stats, error) { +func Entry(ctx context.Context, rep repo.Reader, output Output, rootEntry fs.Entry, options Options) (Stats, error) { c := copier{output: output, q: parallelwork.NewQueue()} c.q.ProgressCallback = func(ctx context.Context, enqueued, active, completed int64) { diff --git a/snapshot/snapshot_test.go b/snapshot/snapshot_test.go index 37a78dbf7..807651e45 100644 --- a/snapshot/snapshot_test.go +++ b/snapshot/snapshot_test.go @@ -81,7 +81,7 @@ func TestSnapshotsAPI(t *testing.T) { verifyLoadSnapshots(t, env.Repository, []manifest.ID{id1, id2, id3}, []*snapshot.Manifest{manifest1, manifest2, manifest3}) } -func verifySnapshotManifestIDs(t *testing.T, rep repo.Repository, src *snapshot.SourceInfo, expected []manifest.ID) { +func verifySnapshotManifestIDs(t *testing.T, rep repo.Reader, src *snapshot.SourceInfo, expected []manifest.ID) { t.Helper() res, err := snapshot.ListSnapshotManifests(testlogging.Context(t), rep, src) @@ -103,7 +103,7 @@ func sortManifestIDs(s []manifest.ID) { }) } -func mustSaveSnapshot(t *testing.T, rep repo.Repository, man *snapshot.Manifest) manifest.ID { +func mustSaveSnapshot(t *testing.T, rep repo.Writer, man *snapshot.Manifest) manifest.ID { t.Helper() id, err := snapshot.SaveSnapshot(testlogging.Context(t), rep, man) @@ -114,7 +114,7 @@ func mustSaveSnapshot(t *testing.T, rep repo.Repository, man *snapshot.Manifest) return id } -func verifySources(t *testing.T, rep repo.Repository, sources ...snapshot.SourceInfo) { +func verifySources(t *testing.T, rep repo.Reader, sources ...snapshot.SourceInfo) { actualSources, err := snapshot.ListSources(testlogging.Context(t), rep) if err != nil { t.Errorf("error listing sources: %v", err) @@ -125,7 +125,7 @@ func verifySources(t *testing.T, rep repo.Repository, sources ...snapshot.Source } } -func verifyListSnapshots(t *testing.T, rep repo.Repository, src snapshot.SourceInfo, expected []*snapshot.Manifest) { +func verifyListSnapshots(t *testing.T, rep repo.Reader, src snapshot.SourceInfo, expected []*snapshot.Manifest) { t.Helper() got, err := snapshot.ListSnapshots(testlogging.Context(t), rep, src) @@ -147,7 +147,7 @@ func verifyListSnapshots(t *testing.T, rep repo.Repository, src snapshot.SourceI } } -func verifyLoadSnapshots(t *testing.T, rep repo.Repository, ids []manifest.ID, expected []*snapshot.Manifest) { +func verifyLoadSnapshots(t *testing.T, rep repo.Reader, ids []manifest.ID, expected []*snapshot.Manifest) { got, err := snapshot.LoadSnapshots(testlogging.Context(t), rep, ids) if err != nil { t.Errorf("error loading manifests: %v", err) diff --git a/snapshot/snapshotfs/all_sources.go b/snapshot/snapshotfs/all_sources.go index e0992af21..fc8f660dd 100644 --- a/snapshot/snapshotfs/all_sources.go +++ b/snapshot/snapshotfs/all_sources.go @@ -14,7 +14,7 @@ ) type repositoryAllSources struct { - rep repo.Repository + rep repo.Reader } func (s *repositoryAllSources) IsDir() bool { @@ -82,6 +82,6 @@ func (s *repositoryAllSources) Readdir(ctx context.Context) (fs.Entries, error) } // AllSourcesEntry returns fs.Directory that contains the list of all snapshot sources found in the repository. -func AllSourcesEntry(rep repo.Repository) fs.Directory { +func AllSourcesEntry(rep repo.Reader) fs.Directory { return &repositoryAllSources{rep: rep} } diff --git a/snapshot/snapshotfs/objref.go b/snapshot/snapshotfs/objref.go index 3837851b4..6577dbace 100644 --- a/snapshot/snapshotfs/objref.go +++ b/snapshot/snapshotfs/objref.go @@ -16,7 +16,7 @@ // ParseObjectIDWithPath interprets the given ID string (which could be an object ID optionally followed by // nested path specification) and returns corresponding object.ID. -func ParseObjectIDWithPath(ctx context.Context, rep repo.Repository, objectIDWithPath string) (object.ID, error) { +func ParseObjectIDWithPath(ctx context.Context, rep repo.Reader, objectIDWithPath string) (object.ID, error) { parts := strings.Split(objectIDWithPath, "/") oid, err := object.ParseID(parts[0]) @@ -75,7 +75,7 @@ func parseNestedObjectID(ctx context.Context, startingDir fs.Entry, parts []stri // or the root object ID (which can match arbitrary number of snapshots). // If multiple snapshots match and they don't agree on root object attributes and consistentAttributes==true // the function fails, otherwise it returns the latest of the snapshots. -func findSnapshotByRootObjectIDOrManifestID(ctx context.Context, rep repo.Repository, rootID string, consistentAttributes bool) (*snapshot.Manifest, error) { +func findSnapshotByRootObjectIDOrManifestID(ctx context.Context, rep repo.Reader, rootID string, consistentAttributes bool) (*snapshot.Manifest, error) { m, err := snapshot.LoadSnapshot(ctx, rep, manifest.ID(rootID)) if err == nil { return m, nil @@ -133,7 +133,7 @@ func latestManifest(mans []*snapshot.Manifest) *snapshot.Manifest { // can be a snapshot manifest ID or an object ID with path. // If multiple snapshots match and they don't agree on root object attributes and consistentAttributes==true // the function fails, otherwise it returns the latest of the snapshots. -func FilesystemEntryFromIDWithPath(ctx context.Context, rep repo.Repository, rootID string, consistentAttributes bool) (fs.Entry, error) { +func FilesystemEntryFromIDWithPath(ctx context.Context, rep repo.Reader, rootID string, consistentAttributes bool) (fs.Entry, error) { pathElements := strings.Split(rootID, "/") if len(pathElements) > 1 { @@ -169,7 +169,7 @@ func FilesystemEntryFromIDWithPath(ctx context.Context, rep repo.Repository, roo // FilesystemDirectoryFromIDWithPath returns a filesystem directory entry for the provided object ID, which // can be a snapshot manifest ID or an object ID with path. -func FilesystemDirectoryFromIDWithPath(ctx context.Context, rep repo.Repository, rootID string, consistentAttributes bool) (fs.Directory, error) { +func FilesystemDirectoryFromIDWithPath(ctx context.Context, rep repo.Reader, rootID string, consistentAttributes bool) (fs.Directory, error) { e, err := FilesystemEntryFromIDWithPath(ctx, rep, rootID, consistentAttributes) if err != nil { return nil, err diff --git a/snapshot/snapshotfs/repofs.go b/snapshot/snapshotfs/repofs.go index a57c12f55..961b29437 100644 --- a/snapshot/snapshotfs/repofs.go +++ b/snapshot/snapshotfs/repofs.go @@ -1,4 +1,4 @@ -// Package snapshotfs implements virtual filesystem on top of snapshots in repo.Repository. +// Package snapshotfs implements virtual filesystem on top of snapshots in repo.Reader. package snapshotfs import ( @@ -22,7 +22,7 @@ type repositoryEntry struct { metadata *snapshot.DirEntry - repo repo.Repository + repo repo.Reader } func (e *repositoryEntry) IsDir() bool { @@ -170,7 +170,7 @@ func (rsl *repositorySymlink) Readlink(ctx context.Context) (string, error) { } // EntryFromDirEntry returns a filesystem entry based on the directory entry. -func EntryFromDirEntry(r repo.Repository, md *snapshot.DirEntry) (fs.Entry, error) { +func EntryFromDirEntry(r repo.Reader, md *snapshot.DirEntry) (fs.Entry, error) { re := repositoryEntry{ metadata: md, repo: r, @@ -214,7 +214,7 @@ func withFileInfo(r object.Reader, e fs.Entry) fs.Reader { // DirectoryEntry returns fs.Directory based on repository object with the specified ID. // The existence or validity of the directory object is not validated until its contents are read. -func DirectoryEntry(rep repo.Repository, objectID object.ID, dirSummary *fs.DirectorySummary) fs.Directory { +func DirectoryEntry(rep repo.Reader, objectID object.ID, dirSummary *fs.DirectorySummary) fs.Directory { d, _ := EntryFromDirEntry(rep, &snapshot.DirEntry{ Name: "/", Permissions: 0o555, //nolint:gomnd @@ -227,7 +227,7 @@ func DirectoryEntry(rep repo.Repository, objectID object.ID, dirSummary *fs.Dire } // SnapshotRoot returns fs.Entry representing the root of a snapshot. -func SnapshotRoot(rep repo.Repository, man *snapshot.Manifest) (fs.Entry, error) { +func SnapshotRoot(rep repo.Reader, man *snapshot.Manifest) (fs.Entry, error) { oid := man.RootObjectID() if oid == "" { return nil, errors.New("manifest root object ID") @@ -238,7 +238,7 @@ func SnapshotRoot(rep repo.Repository, man *snapshot.Manifest) (fs.Entry, error) // AutoDetectEntryFromObjectID returns fs.Entry (either file or directory) for the provided object ID. // It uses heuristics to determine whether object ID is possibly a directory and treats it as such. -func AutoDetectEntryFromObjectID(ctx context.Context, rep repo.Repository, oid object.ID, maybeName string) fs.Entry { +func AutoDetectEntryFromObjectID(ctx context.Context, rep repo.Reader, oid object.ID, maybeName string) fs.Entry { if IsDirectoryID(oid) { dirEntry := DirectoryEntry(rep, oid, nil) if _, err := dirEntry.Readdir(ctx); err == nil { diff --git a/snapshot/snapshotfs/source_directories.go b/snapshot/snapshotfs/source_directories.go index 6c8f0ccfd..d98607df1 100644 --- a/snapshot/snapshotfs/source_directories.go +++ b/snapshot/snapshotfs/source_directories.go @@ -13,7 +13,7 @@ ) type sourceDirectories struct { - rep repo.Repository + rep repo.Reader userHost string } diff --git a/snapshot/snapshotfs/source_snapshots.go b/snapshot/snapshotfs/source_snapshots.go index b54d683b9..a7faa0a70 100644 --- a/snapshot/snapshotfs/source_snapshots.go +++ b/snapshot/snapshotfs/source_snapshots.go @@ -15,7 +15,7 @@ ) type sourceSnapshots struct { - rep repo.Repository + rep repo.Reader src snapshot.SourceInfo } diff --git a/snapshot/snapshotfs/upload.go b/snapshot/snapshotfs/upload.go index e63926386..a2df3c7c9 100644 --- a/snapshot/snapshotfs/upload.go +++ b/snapshot/snapshotfs/upload.go @@ -70,7 +70,7 @@ type Uploader struct { // How frequently to create checkpoint snapshot entries. CheckpointInterval time.Duration - repo repo.Repository + repo repo.Writer // stats must be allocated on heap to enforce 64-bit alignment due to atomic access on ARM. stats *snapshot.Stats @@ -967,7 +967,7 @@ func (u *Uploader) shouldIgnoreDirectoryReadErrors(policyTree *policy.Tree) bool } // NewUploader creates new Uploader object for a given repository. -func NewUploader(r repo.Repository) *Uploader { +func NewUploader(r repo.Writer) *Uploader { return &Uploader{ repo: r, Progress: &NullUploadProgress{}, diff --git a/snapshot/snapshotfs/upload_test.go b/snapshot/snapshotfs/upload_test.go index a4e02f7b6..4f06e8f7f 100644 --- a/snapshot/snapshotfs/upload_test.go +++ b/snapshot/snapshotfs/upload_test.go @@ -32,7 +32,7 @@ type uploadTestHarness struct { sourceDir *mockfs.Directory repoDir string - repo repo.Repository + repo repo.Writer ft *faketime.TimeAdvance } diff --git a/snapshot/snapshotgc/gc.go b/snapshot/snapshotgc/gc.go index e0279e01e..04602da98 100644 --- a/snapshot/snapshotgc/gc.go +++ b/snapshot/snapshotgc/gc.go @@ -26,7 +26,7 @@ func oidOf(entry fs.Entry) object.ID { return entry.(object.HasObjectID).ObjectID() } -func findInUseContentIDs(ctx context.Context, rep repo.Repository, used *sync.Map) error { +func findInUseContentIDs(ctx context.Context, rep repo.Reader, used *sync.Map) error { ids, err := snapshot.ListSnapshotManifests(ctx, rep, nil) if err != nil { return errors.Wrap(err, "unable to list snapshot manifest IDs") diff --git a/snapshot/snapshotmaintenance/helper_test.go b/snapshot/snapshotmaintenance/helper_test.go index ff50c0032..197278675 100644 --- a/snapshot/snapshotmaintenance/helper_test.go +++ b/snapshot/snapshotmaintenance/helper_test.go @@ -14,7 +14,7 @@ ) // Create snapshots an FS entry. -func createSnapshot(ctx context.Context, rep repo.Repository, e fs.Entry, si snapshot.SourceInfo, description string) (*snapshot.Manifest, error) { +func createSnapshot(ctx context.Context, rep repo.Writer, e fs.Entry, si snapshot.SourceInfo, description string) (*snapshot.Manifest, error) { // sanitize source path si.Path = filepath.Clean(si.Path) @@ -46,7 +46,7 @@ func createSnapshot(ctx context.Context, rep repo.Repository, e fs.Entry, si sna // findPreviousSnapshotManifest returns the list of previous snapshots for a given source, including // last complete snapshot and possibly some number of incomplete snapshots following it. // this would belong in the snapshot package. -func findPreviousSnapshotManifest(ctx context.Context, rep repo.Repository, sourceInfo snapshot.SourceInfo) ([]*snapshot.Manifest, error) { +func findPreviousSnapshotManifest(ctx context.Context, rep repo.Reader, sourceInfo snapshot.SourceInfo) ([]*snapshot.Manifest, error) { man, err := snapshot.ListSnapshots(ctx, rep, sourceInfo) if err != nil { return nil, errors.Wrap(err, "error listing previous snapshots") diff --git a/snapshot/snapshotmaintenance/snapshotmaintenance.go b/snapshot/snapshotmaintenance/snapshotmaintenance.go index 57e872ce2..107844b1b 100644 --- a/snapshot/snapshotmaintenance/snapshotmaintenance.go +++ b/snapshot/snapshotmaintenance/snapshotmaintenance.go @@ -12,7 +12,7 @@ ) // Run runs the complete snapshot and repository maintenance. -func Run(ctx context.Context, rep repo.Repository, mode maintenance.Mode, force bool) error { +func Run(ctx context.Context, rep repo.Writer, mode maintenance.Mode, force bool) error { dr, ok := rep.(*repo.DirectRepository) if !ok { return nil diff --git a/snapshot/snapshotmaintenance/snapshotmaintenance_test.go b/snapshot/snapshotmaintenance/snapshotmaintenance_test.go index 5c83420f4..8ac652702 100644 --- a/snapshot/snapshotmaintenance/snapshotmaintenance_test.go +++ b/snapshot/snapshotmaintenance/snapshotmaintenance_test.go @@ -195,13 +195,13 @@ func (th *testHarness) openAnother(t *testing.T) repo.Repository { return r } -func mustFlush(t *testing.T, r repo.Repository) { +func mustFlush(t *testing.T, r repo.Writer) { t.Helper() require.NotNil(t, r, "nil repository") require.NoError(t, r.Flush(testlogging.Context(t))) } -func mustSnapshot(t *testing.T, r repo.Repository, source fs.Entry, si snapshot.SourceInfo) *snapshot.Manifest { +func mustSnapshot(t *testing.T, r repo.Writer, source fs.Entry, si snapshot.SourceInfo) *snapshot.Manifest { t.Helper() s1, err := createSnapshot(testlogging.Context(t), r, source, si, "")