From 6160ee566836b966bb8512768d4f355b69aea00a Mon Sep 17 00:00:00 2001 From: Jarek Kowalski Date: Sat, 30 Jul 2022 14:13:52 -0700 Subject: [PATCH] refactor(repository): moved format blob management to separate package (#2245) * refactor(repository): moved format blob management to separate package This is completely mechanical, no behavior changes, only: - moved types and functions to a new package - adjusted visibility where needed - added missing godoc - renamed some identifiers to align with current usage - mechanically converted some top-level functions into member functions - fixed some mis-named variables * refactor(repository): moved content.FormatingOptions to format.ContentFormat --- cli/command_benchmark_crypto.go | 4 +- cli/command_benchmark_encryption.go | 4 +- cli/command_benchmark_hashing.go | 4 +- ...command_repository_change_password_test.go | 4 +- cli/command_repository_create.go | 11 +- cli/command_repository_repair.go | 8 +- cli/command_repository_set_parameters.go | 5 +- cli/command_repository_set_parameters_test.go | 8 +- cli/command_repository_status.go | 19 +- cli/command_repository_sync.go | 7 +- cli/command_repository_upgrade.go | 13 +- cli/command_repository_upgrade_test.go | 10 +- cli/suite_test.go | 10 +- fs/ignorefs/ignorefs.go | 6 +- internal/cachedir/cachedir.go | 57 ++++ internal/remoterepoapi/remoterepoapi.go | 4 +- internal/repotesting/repotesting.go | 14 +- internal/server/api_repo.go | 5 +- repo/api_server_repository.go | 5 +- repo/blobcfg_blob.go | 89 ----- repo/change_password.go | 20 +- repo/connect.go | 6 +- repo/content/blob_crypto_test.go | 5 +- repo/content/committed_read_manager.go | 5 +- repo/content/content_formatter_test.go | 11 +- repo/content/content_formatting_options.go | 320 ------------------ repo/content/content_manager.go | 17 +- repo/content/content_manager_test.go | 17 +- repo/content/content_reader.go | 3 +- repo/content/encrypted_blob_mgr_test.go | 3 +- repo/content/index_blob_manager_v0_test.go | 3 +- repo/format/blobcfg_blob.go | 110 ++++++ repo/format/content_format.go | 102 ++++++ repo/{ => format}/crypto_key_derivation.go | 6 +- .../crypto_key_derivation_nontest.go | 9 +- .../crypto_key_derivation_testing.go | 11 +- .../format_blob.go} | 82 ++--- repo/format/format_blob_cache.go | 144 ++++++++ .../format_blob_test.go} | 2 +- repo/format/format_provider.go | 218 ++++++++++++ repo/format/object_format.go | 6 + repo/format/repository_config.go | 78 +++++ repo/{ => format}/upgrade_lock_intent.go | 2 +- repo/{ => format}/upgrade_lock_intent_test.go | 30 +- repo/grpc_repository_client.go | 5 +- repo/initialize.go | 63 ++-- repo/local_config.go | 14 +- repo/maintenance/blob_gc_test.go | 3 +- repo/maintenance/suite_test.go | 10 +- repo/manifest/manifest_manager_test.go | 9 +- repo/object/object_manager.go | 10 +- repo/object/object_manager_test.go | 5 +- repo/open.go | 238 ++----------- repo/parameters.go | 26 +- repo/repo_benchmarks_test.go | 6 +- repo/repository.go | 23 +- repo/repository_test.go | 31 +- repo/suite_test.go | 10 +- repo/upgrade_lock.go | 38 +-- repo/upgrade_lock_test.go | 42 +-- .../docs/Advanced/Encryption/_index.md | 18 +- .../snapshotmaintenance_test.go | 5 +- snapshot/snapshotmaintenance/suite_test.go | 10 +- .../end_to_end_test/repository_repair_test.go | 4 +- tests/end_to_end_test/snapshot_create_test.go | 4 +- tests/end_to_end_test/suite_test.go | 10 +- tests/stress_test/stress_test.go | 5 +- .../kopia_snapshotter_upgrade_test.go | 6 +- 68 files changed, 1094 insertions(+), 998 deletions(-) create mode 100644 internal/cachedir/cachedir.go delete mode 100644 repo/blobcfg_blob.go delete mode 100644 repo/content/content_formatting_options.go create mode 100644 repo/format/blobcfg_blob.go create mode 100644 repo/format/content_format.go rename repo/{ => format}/crypto_key_derivation.go (71%) rename repo/{ => format}/crypto_key_derivation_nontest.go (54%) rename repo/{ => format}/crypto_key_derivation_testing.go (50%) rename repo/{format_block.go => format/format_blob.go} (76%) create mode 100644 repo/format/format_blob_cache.go rename repo/{format_block_test.go => format/format_blob_test.go} (99%) create mode 100644 repo/format/format_provider.go create mode 100644 repo/format/object_format.go create mode 100644 repo/format/repository_config.go rename repo/{ => format}/upgrade_lock_intent.go (99%) rename repo/{ => format}/upgrade_lock_intent_test.go (96%) diff --git a/cli/command_benchmark_crypto.go b/cli/command_benchmark_crypto.go index 155df89ef..a476203b0 100644 --- a/cli/command_benchmark_crypto.go +++ b/cli/command_benchmark_crypto.go @@ -9,8 +9,8 @@ "github.com/kopia/kopia/internal/gather" "github.com/kopia/kopia/internal/timetrack" "github.com/kopia/kopia/internal/units" - "github.com/kopia/kopia/repo/content" "github.com/kopia/kopia/repo/encryption" + "github.com/kopia/kopia/repo/format" "github.com/kopia/kopia/repo/hashing" ) @@ -67,7 +67,7 @@ func (c *commandBenchmarkCrypto) runBenchmark(ctx context.Context) []cryptoBench for _, ha := range hashing.SupportedAlgorithms() { for _, ea := range encryption.SupportedAlgorithms(c.deprecatedAlgorithms) { - fo := &content.FormattingOptions{ + fo := &format.ContentFormat{ Encryption: ea, Hash: ha, MasterKey: make([]byte, 32), // nolint:gomnd diff --git a/cli/command_benchmark_encryption.go b/cli/command_benchmark_encryption.go index 1d6374231..8def87d46 100644 --- a/cli/command_benchmark_encryption.go +++ b/cli/command_benchmark_encryption.go @@ -9,8 +9,8 @@ "github.com/kopia/kopia/internal/gather" "github.com/kopia/kopia/internal/timetrack" "github.com/kopia/kopia/internal/units" - "github.com/kopia/kopia/repo/content" "github.com/kopia/kopia/repo/encryption" + "github.com/kopia/kopia/repo/format" "github.com/kopia/kopia/repo/hashing" ) @@ -66,7 +66,7 @@ func (c *commandBenchmarkEncryption) runBenchmark(ctx context.Context) []cryptoB data := make([]byte, c.blockSize) for _, ea := range encryption.SupportedAlgorithms(c.deprecatedAlgorithms) { - enc, err := encryption.CreateEncryptor(&content.FormattingOptions{ + enc, err := encryption.CreateEncryptor(&format.ContentFormat{ Encryption: ea, Hash: hashing.DefaultAlgorithm, MasterKey: make([]byte, 32), // nolint:gomnd diff --git a/cli/command_benchmark_hashing.go b/cli/command_benchmark_hashing.go index 9a2b82b06..0645b050d 100644 --- a/cli/command_benchmark_hashing.go +++ b/cli/command_benchmark_hashing.go @@ -9,7 +9,7 @@ "github.com/kopia/kopia/internal/gather" "github.com/kopia/kopia/internal/timetrack" "github.com/kopia/kopia/internal/units" - "github.com/kopia/kopia/repo/content" + "github.com/kopia/kopia/repo/format" "github.com/kopia/kopia/repo/hashing" ) @@ -63,7 +63,7 @@ func (c *commandBenchmarkHashing) runBenchmark(ctx context.Context) []cryptoBenc data := make([]byte, c.blockSize) for _, ha := range hashing.SupportedAlgorithms() { - hf, err := hashing.CreateHashFunc(&content.FormattingOptions{ + hf, err := hashing.CreateHashFunc(&format.ContentFormat{ Hash: ha, HMACSecret: make([]byte, 32), // nolint:gomnd }) diff --git a/cli/command_repository_change_password_test.go b/cli/command_repository_change_password_test.go index 5df104a7d..7d888cd03 100644 --- a/cli/command_repository_change_password_test.go +++ b/cli/command_repository_change_password_test.go @@ -3,7 +3,7 @@ import ( "testing" - "github.com/kopia/kopia/repo/content" + "github.com/kopia/kopia/repo/format" "github.com/kopia/kopia/tests/testenv" ) @@ -15,7 +15,7 @@ func (s *formatSpecificTestSuite) TestRepositoryChangePassword(t *testing.T) { env1.RunAndExpectSuccess(t, "repo", "create", "filesystem", "--path", env1.RepoDir, "--disable-repository-format-cache") - if s.formatVersion == content.FormatVersion1 { + if s.formatVersion == format.FormatVersion1 { env1.RunAndExpectFailure(t, "repo", "change-password", "--new-password", "newPass") return diff --git a/cli/command_repository_create.go b/cli/command_repository_create.go index bbb62a5d2..e03baf31e 100644 --- a/cli/command_repository_create.go +++ b/cli/command_repository_create.go @@ -9,10 +9,9 @@ "github.com/kopia/kopia/repo" "github.com/kopia/kopia/repo/blob" - "github.com/kopia/kopia/repo/content" "github.com/kopia/kopia/repo/encryption" + "github.com/kopia/kopia/repo/format" "github.com/kopia/kopia/repo/hashing" - "github.com/kopia/kopia/repo/object" "github.com/kopia/kopia/repo/splitter" "github.com/kopia/kopia/snapshot/policy" ) @@ -77,15 +76,15 @@ func (c *commandRepositoryCreate) setup(svc advancedAppServices, parent commandP func (c *commandRepositoryCreate) newRepositoryOptionsFromFlags() *repo.NewRepositoryOptions { return &repo.NewRepositoryOptions{ - BlockFormat: content.FormattingOptions{ - MutableParameters: content.MutableParameters{ - Version: content.FormatVersion(c.createFormatVersion), + BlockFormat: format.ContentFormat{ + MutableParameters: format.MutableParameters{ + Version: format.Version(c.createFormatVersion), }, Hash: c.createBlockHashFormat, Encryption: c.createBlockEncryptionFormat, }, - ObjectFormat: object.Format{ + ObjectFormat: format.ObjectFormat{ Splitter: c.createSplitter, }, diff --git a/cli/command_repository_repair.go b/cli/command_repository_repair.go index c0d928e46..93175cc0b 100644 --- a/cli/command_repository_repair.go +++ b/cli/command_repository_repair.go @@ -7,9 +7,9 @@ "github.com/pkg/errors" "github.com/kopia/kopia/internal/gather" - "github.com/kopia/kopia/repo" "github.com/kopia/kopia/repo/blob" "github.com/kopia/kopia/repo/content" + "github.com/kopia/kopia/repo/format" ) type commandRepositoryRepair struct { @@ -61,7 +61,7 @@ func (c *commandRepositoryRepair) runRepairCommandWithStorage(ctx context.Contex var tmp gather.WriteBuffer defer tmp.Close() - if err := st.GetBlob(ctx, repo.FormatBlobID, 0, -1, &tmp); err == nil { + if err := st.GetBlob(ctx, format.KopiaRepositoryBlobID, 0, -1, &tmp); err == nil { log(ctx).Infof("format blob already exists, not recovering, pass --recover-format=yes") return nil } @@ -84,9 +84,9 @@ func (c *commandRepositoryRepair) recoverFormatBlob(ctx context.Context, st blob for _, prefix := range prefixes { err := st.ListBlobs(ctx, blob.ID(prefix), func(bi blob.Metadata) error { log(ctx).Infof("looking for replica of format blob in %v...", bi.BlobID) - if b, err := repo.RecoverFormatBlob(ctx, st, bi.BlobID, bi.Length); err == nil { + if b, err := format.RecoverFormatBlob(ctx, st, bi.BlobID, bi.Length); err == nil { if !c.repairDryRun { - if puterr := st.PutBlob(ctx, repo.FormatBlobID, gather.FromSlice(b), blob.PutOptions{}); puterr != nil { + if puterr := st.PutBlob(ctx, format.KopiaRepositoryBlobID, gather.FromSlice(b), blob.PutOptions{}); puterr != nil { return errors.Wrap(puterr, "error writing format blob") } } diff --git a/cli/command_repository_set_parameters.go b/cli/command_repository_set_parameters.go index 0a9d19cb1..99d49a0fa 100644 --- a/cli/command_repository_set_parameters.go +++ b/cli/command_repository_set_parameters.go @@ -12,6 +12,7 @@ "github.com/kopia/kopia/repo" "github.com/kopia/kopia/repo/blob" "github.com/kopia/kopia/repo/content" + "github.com/kopia/kopia/repo/format" ) type commandRepositorySetParameters struct { @@ -143,8 +144,8 @@ func (c *commandRepositorySetParameters) run(ctx context.Context, rep repo.Direc mp.IndexVersion = 2 } - if mp.Version < content.FormatVersion2 { - mp.Version = content.FormatVersion2 + if mp.Version < format.FormatVersion2 { + mp.Version = format.FormatVersion2 } } diff --git a/cli/command_repository_set_parameters_test.go b/cli/command_repository_set_parameters_test.go index 0901aeede..4427d846d 100644 --- a/cli/command_repository_set_parameters_test.go +++ b/cli/command_repository_set_parameters_test.go @@ -12,7 +12,7 @@ "github.com/kopia/kopia/internal/repotesting" "github.com/kopia/kopia/internal/testutil" "github.com/kopia/kopia/repo/blob" - "github.com/kopia/kopia/repo/content" + "github.com/kopia/kopia/repo/format" "github.com/kopia/kopia/tests/testenv" ) @@ -127,11 +127,11 @@ func (s *formatSpecificTestSuite) TestRepositorySetParametersUpgrade(t *testing. require.Contains(t, out, "Max pack length: 20 MiB") switch s.formatVersion { - case content.FormatVersion1: + case format.FormatVersion1: require.Contains(t, out, "Format version: 1") require.Contains(t, out, "Epoch Manager: disabled") env.RunAndExpectFailure(t, "index", "epoch", "list") - case content.FormatVersion2: + case format.FormatVersion2: require.Contains(t, out, "Format version: 2") require.Contains(t, out, "Epoch Manager: enabled") env.RunAndExpectSuccess(t, "index", "epoch", "list") @@ -154,7 +154,7 @@ func (s *formatSpecificTestSuite) TestRepositorySetParametersUpgrade(t *testing. cli.MaxPermittedClockDrift = func() time.Duration { return time.Second } // You can only upgrade when you are not already upgraded - if s.formatVersion < content.MaxFormatVersion { + if s.formatVersion < format.MaxFormatVersion { env.RunAndExpectSuccess(t, cmd...) } else { env.RunAndExpectFailure(t, cmd...) diff --git a/cli/command_repository_status.go b/cli/command_repository_status.go index 69fbf0315..38d919e31 100644 --- a/cli/command_repository_status.go +++ b/cli/command_repository_status.go @@ -15,8 +15,7 @@ "github.com/kopia/kopia/internal/units" "github.com/kopia/kopia/repo" "github.com/kopia/kopia/repo/blob" - "github.com/kopia/kopia/repo/content" - "github.com/kopia/kopia/repo/object" + "github.com/kopia/kopia/repo/format" ) type commandRepositoryStatus struct { @@ -33,12 +32,12 @@ type RepositoryStatus struct { ConfigFile string `json:"configFile"` UniqueIDHex string `json:"uniqueIDHex"` - ClientOptions repo.ClientOptions `json:"clientOptions"` - Storage blob.ConnectionInfo `json:"storage"` - Capacity *blob.Capacity `json:"volume,omitempty"` - ContentFormat content.FormattingOptions `json:"contentFormat"` - ObjectFormat object.Format `json:"objectFormat"` - BlobRetention content.BlobCfgBlob `json:"blobRetention"` + ClientOptions repo.ClientOptions `json:"clientOptions"` + Storage blob.ConnectionInfo `json:"storage"` + Capacity *blob.Capacity `json:"volume,omitempty"` + ContentFormat format.ContentFormat `json:"contentFormat"` + ObjectFormat format.ObjectFormat `json:"objectFormat"` + BlobRetention format.BlobStorageConfiguration `json:"blobRetention"` } func (c *commandRepositoryStatus) setup(svc advancedAppServices, parent commandParent) { @@ -64,8 +63,8 @@ func (c *commandRepositoryStatus) outputJSON(ctx context.Context, r repo.Reposit s.UniqueIDHex = hex.EncodeToString(dr.UniqueID()) s.ObjectFormat = dr.ObjectFormat() s.BlobRetention = dr.BlobCfg() - s.Storage = scrubber.ScrubSensitiveData(reflect.ValueOf(ci)).Interface().(blob.ConnectionInfo) // nolint:forcetypeassert - s.ContentFormat = scrubber.ScrubSensitiveData(reflect.ValueOf(dr.ContentReader().ContentFormat().Struct())).Interface().(content.FormattingOptions) // nolint:forcetypeassert + s.Storage = scrubber.ScrubSensitiveData(reflect.ValueOf(ci)).Interface().(blob.ConnectionInfo) // nolint:forcetypeassert + s.ContentFormat = scrubber.ScrubSensitiveData(reflect.ValueOf(dr.ContentReader().ContentFormat().Struct())).Interface().(format.ContentFormat) // nolint:forcetypeassert switch cp, err := dr.BlobVolume().GetCapacity(ctx); { case err == nil: diff --git a/cli/command_repository_sync.go b/cli/command_repository_sync.go index 6c0a69856..a33b395ac 100644 --- a/cli/command_repository_sync.go +++ b/cli/command_repository_sync.go @@ -18,6 +18,7 @@ "github.com/kopia/kopia/internal/units" "github.com/kopia/kopia/repo" "github.com/kopia/kopia/repo/blob" + "github.com/kopia/kopia/repo/format" ) type commandRepositorySyncTo struct { @@ -345,21 +346,21 @@ func (c *commandRepositorySyncTo) ensureRepositoriesHaveSameFormatBlob(ctx conte var srcData gather.WriteBuffer defer srcData.Close() - if err := src.GetBlob(ctx, repo.FormatBlobID, 0, -1, &srcData); err != nil { + if err := src.GetBlob(ctx, format.KopiaRepositoryBlobID, 0, -1, &srcData); err != nil { return errors.Wrap(err, "error reading format blob") } var dstData gather.WriteBuffer defer dstData.Close() - if err := dst.GetBlob(ctx, repo.FormatBlobID, 0, -1, &dstData); err != nil { + if err := dst.GetBlob(ctx, format.KopiaRepositoryBlobID, 0, -1, &dstData); err != nil { // target does not have format blob, save it there first. if errors.Is(err, blob.ErrBlobNotFound) { if c.repositorySyncDestinationMustExist { return errors.Errorf("destination repository does not have a format blob") } - return errors.Wrap(dst.PutBlob(ctx, repo.FormatBlobID, srcData.Bytes(), blob.PutOptions{}), "error saving format blob") + return errors.Wrap(dst.PutBlob(ctx, format.KopiaRepositoryBlobID, srcData.Bytes(), blob.PutOptions{}), "error saving format blob") } return errors.Wrap(err, "error reading destination repository format blob") diff --git a/cli/command_repository_upgrade.go b/cli/command_repository_upgrade.go index 7c266fbc1..36a850fd1 100644 --- a/cli/command_repository_upgrade.go +++ b/cli/command_repository_upgrade.go @@ -13,6 +13,7 @@ "github.com/kopia/kopia/internal/epoch" "github.com/kopia/kopia/repo" "github.com/kopia/kopia/repo/content" + "github.com/kopia/kopia/repo/format" ) type commandRepositoryUpgrade struct { @@ -56,7 +57,7 @@ func (c *commandRepositoryUpgrade) setup(svc advancedAppServices, parent command beginCmd := parent.Command("begin", "Begin upgrade.").Default() beginCmd.Flag("advance-notice", "Advance notice for upgrade to allow enough time for other Kopia clients to notice the lock").DurationVar(&c.advanceNoticeDuration) - beginCmd.Flag("io-drain-timeout", "Max time it should take all other Kopia clients to drop repository connections").Default(repo.DefaultRepositoryBlobCacheDuration.String()).DurationVar(&c.ioDrainTimeout) + beginCmd.Flag("io-drain-timeout", "Max time it should take all other Kopia clients to drop repository connections").Default(format.DefaultRepositoryBlobCacheDuration.String()).DurationVar(&c.ioDrainTimeout) beginCmd.Flag("allow-unsafe-upgrade", "Force using an unsafe io-drain-timeout for the upgrade lock").Default("false").Hidden().BoolVar(&c.force) beginCmd.Flag("status-poll-interval", "An advisory polling interval to check for the status of upgrade").Default("60s").DurationVar(&c.statusPollInterval) @@ -117,20 +118,20 @@ func (c *commandRepositoryUpgrade) runPhase(act func(context.Context, repo.Direc // setLockIntent is an upgrade phase which sets the upgrade lock intent with // desired parameters. func (c *commandRepositoryUpgrade) setLockIntent(ctx context.Context, rep repo.DirectRepositoryWriter) error { - if c.ioDrainTimeout < repo.DefaultRepositoryBlobCacheDuration && !c.force { - return errors.Errorf("minimum required io-drain-timeout is %s", repo.DefaultRepositoryBlobCacheDuration) + if c.ioDrainTimeout < format.DefaultRepositoryBlobCacheDuration && !c.force { + return errors.Errorf("minimum required io-drain-timeout is %s", format.DefaultRepositoryBlobCacheDuration) } now := rep.Time() mp := rep.ContentReader().ContentFormat().GetMutableParameters() openOpts := c.svc.optionsFromFlags(ctx) - l := &repo.UpgradeLockIntent{ + l := &format.UpgradeLockIntent{ OwnerID: openOpts.UpgradeOwnerID, CreationTime: now, AdvanceNoticeDuration: c.advanceNoticeDuration, IODrainTimeout: c.ioDrainTimeout, StatusPollInterval: c.statusPollInterval, - Message: fmt.Sprintf("Upgrading from format version %d -> %d", mp.Version, content.MaxFormatVersion), + Message: fmt.Sprintf("Upgrading from format version %d -> %d", mp.Version, format.MaxFormatVersion), MaxPermittedClockDrift: MaxPermittedClockDrift(), } @@ -219,7 +220,7 @@ func (c *commandRepositoryUpgrade) drainAllClients(ctx context.Context, rep repo cacheOpts := lc.Caching.CloneOrDefault() for { - l, err := repo.ReadAndCacheRepoUpgradeLock(ctx, rep.BlobStorage(), password, cacheOpts, -1) + l, err := format.ReadAndCacheRepoUpgradeLock(ctx, rep.BlobStorage(), password, cacheOpts.CacheDirectory, -1) if err != nil { return errors.Wrap(err, "unable to reload the repository format blob") } diff --git a/cli/command_repository_upgrade_test.go b/cli/command_repository_upgrade_test.go index bdb656e7e..44a1af452 100644 --- a/cli/command_repository_upgrade_test.go +++ b/cli/command_repository_upgrade_test.go @@ -8,7 +8,7 @@ "github.com/stretchr/testify/require" "github.com/kopia/kopia/cli" - "github.com/kopia/kopia/repo/content" + "github.com/kopia/kopia/repo/format" "github.com/kopia/kopia/tests/testenv" ) @@ -23,7 +23,7 @@ func (s *formatSpecificTestSuite) TestRepositoryUpgrade(t *testing.T) { cli.MaxPermittedClockDrift = func() time.Duration { return time.Second } switch s.formatVersion { - case content.FormatVersion1: + case format.FormatVersion1: require.Contains(t, out, "Format version: 1") _, stderr := env.RunAndExpectSuccessWithErrOut(t, "repository", "upgrade", "--upgrade-owner-id", "owner", @@ -31,7 +31,7 @@ func (s *formatSpecificTestSuite) TestRepositoryUpgrade(t *testing.T) { "--status-poll-interval", "1s") require.Contains(t, stderr, "Repository indices have been upgraded.") require.Contains(t, stderr, "Repository has been successfully upgraded.") - case content.FormatVersion2: + case format.FormatVersion2: require.Contains(t, out, "Format version: 2") _, stderr := env.RunAndExpectSuccessWithErrOut(t, "repository", "upgrade", "--upgrade-owner-id", "owner", @@ -64,7 +64,7 @@ func (s *formatSpecificTestSuite) TestRepositoryUpgradeAdvanceNotice(t *testing. cli.MaxPermittedClockDrift = func() time.Duration { return time.Second } switch s.formatVersion { - case content.FormatVersion1: + case format.FormatVersion1: require.Contains(t, out, "Format version: 1") _, stderr := env.RunAndExpectSuccessWithErrOut(t, "repository", "upgrade", "--upgrade-owner-id", "owner", @@ -126,7 +126,7 @@ func (s *formatSpecificTestSuite) TestRepositoryUpgradeAdvanceNotice(t *testing. // verify that non-owner clients can resume access env.RunAndExpectSuccess(t, "repository", "status", "--upgrade-no-block") - case content.FormatVersion2: + case format.FormatVersion2: require.Contains(t, out, "Format version: 2") _, stderr := env.RunAndExpectSuccessWithErrOut(t, "repository", "upgrade", "--upgrade-owner-id", "owner", diff --git a/cli/suite_test.go b/cli/suite_test.go index f77a70a95..9bb28c701 100644 --- a/cli/suite_test.go +++ b/cli/suite_test.go @@ -4,24 +4,24 @@ "testing" "github.com/kopia/kopia/internal/testutil" - "github.com/kopia/kopia/repo/content" + "github.com/kopia/kopia/repo/format" ) func TestMain(m *testing.M) { testutil.MyTestMain(m) } type formatSpecificTestSuite struct { formatFlags []string - formatVersion content.FormatVersion + formatVersion format.Version } func TestFormatV1(t *testing.T) { - testutil.RunAllTestsWithParam(t, &formatSpecificTestSuite{[]string{"--format-version=1"}, content.FormatVersion1}) + testutil.RunAllTestsWithParam(t, &formatSpecificTestSuite{[]string{"--format-version=1"}, format.FormatVersion1}) } func TestFormatV2(t *testing.T) { - testutil.RunAllTestsWithParam(t, &formatSpecificTestSuite{[]string{"--format-version=2"}, content.FormatVersion2}) + testutil.RunAllTestsWithParam(t, &formatSpecificTestSuite{[]string{"--format-version=2"}, format.FormatVersion2}) } func TestFormatV3(t *testing.T) { - testutil.RunAllTestsWithParam(t, &formatSpecificTestSuite{[]string{"--format-version=3"}, content.FormatVersion3}) + testutil.RunAllTestsWithParam(t, &formatSpecificTestSuite{[]string{"--format-version=3"}, format.FormatVersion3}) } diff --git a/fs/ignorefs/ignorefs.go b/fs/ignorefs/ignorefs.go index 25c236763..857448dfb 100644 --- a/fs/ignorefs/ignorefs.go +++ b/fs/ignorefs/ignorefs.go @@ -9,8 +9,8 @@ "github.com/pkg/errors" "github.com/kopia/kopia/fs" + "github.com/kopia/kopia/internal/cachedir" "github.com/kopia/kopia/internal/wcmatch" - "github.com/kopia/kopia/repo" "github.com/kopia/kopia/repo/logging" "github.com/kopia/kopia/snapshot" "github.com/kopia/kopia/snapshot/policy" @@ -79,7 +79,7 @@ type ignoreDirectory struct { func isCorrectCacheDirSignature(ctx context.Context, f fs.File) error { const ( - validSignature = repo.CacheDirMarkerHeader + validSignature = cachedir.CacheDirMarkerHeader validSignatureLen = len(validSignature) ) @@ -112,7 +112,7 @@ func (d *ignoreDirectory) skipCacheDirectory(ctx context.Context, relativePath s return false } - e, err := d.Directory.Child(ctx, repo.CacheDirMarkerFile) + e, err := d.Directory.Child(ctx, cachedir.CacheDirMarkerFile) if err != nil { return false } diff --git a/internal/cachedir/cachedir.go b/internal/cachedir/cachedir.go new file mode 100644 index 000000000..0076b5f57 --- /dev/null +++ b/internal/cachedir/cachedir.go @@ -0,0 +1,57 @@ +// Package cachedir contains utilities for manipulating cache directories. +package cachedir + +import ( + "os" + "path/filepath" + + "github.com/pkg/errors" +) + +// CacheDirMarkerFile is the name of the marker file indicating a directory contains Kopia caches. +// See https://bford.info/cachedir/ +const CacheDirMarkerFile = "CACHEDIR.TAG" + +// CacheDirMarkerHeader is the header signature for cache dir marker files. +const CacheDirMarkerHeader = "Signature: 8a477f597d28d172789f06886806bc55" + +const cacheDirMarkerContents = CacheDirMarkerHeader + ` +# +# This file is a cache directory tag created by Kopia - Fast And Secure Open-Source Backup. +# +# For information about Kopia, see: +# https://kopia.io +# +# For information about cache directory tags, see: +# http://www.brynosaurus.com/cachedir/ +` + +// WriteCacheMarker writes the CACHEDIR.TAG marker file in a given directory. +func WriteCacheMarker(cacheDir string) error { + if cacheDir == "" { + return nil + } + + markerFile := filepath.Join(cacheDir, CacheDirMarkerFile) + + st, err := os.Stat(markerFile) + if err == nil && st.Size() >= int64(len(cacheDirMarkerContents)) { + // ok + return nil + } + + if err != nil && !os.IsNotExist(err) { + return errors.Wrap(err, "unexpected cache marker error") + } + + f, err := os.Create(markerFile) //nolint:gosec + if err != nil { + return errors.Wrap(err, "error creating cache marker") + } + + if _, err := f.WriteString(cacheDirMarkerContents); err != nil { + return errors.Wrap(err, "unable to write cachedir marker contents") + } + + return errors.Wrap(f.Close(), "error closing cache marker file") +} diff --git a/internal/remoterepoapi/remoterepoapi.go b/internal/remoterepoapi/remoterepoapi.go index 04f3f711b..4deec668a 100644 --- a/internal/remoterepoapi/remoterepoapi.go +++ b/internal/remoterepoapi/remoterepoapi.go @@ -5,8 +5,8 @@ "encoding/json" "github.com/kopia/kopia/repo/content" + "github.com/kopia/kopia/repo/format" "github.com/kopia/kopia/repo/manifest" - "github.com/kopia/kopia/repo/object" ) // Parameters encapsulates all parameters for repository. @@ -16,7 +16,7 @@ type Parameters struct { HMACSecret []byte `json:"hmacSecret"` SupportsContentCompression bool `json:"supportsContentCompression"` - object.Format + format.ObjectFormat } // GetHashFunction returns the name of the hash function for remote repository. diff --git a/internal/repotesting/repotesting.go b/internal/repotesting/repotesting.go index bddd6e957..5c48899d1 100644 --- a/internal/repotesting/repotesting.go +++ b/internal/repotesting/repotesting.go @@ -16,7 +16,7 @@ "github.com/kopia/kopia/repo/blob" "github.com/kopia/kopia/repo/content" "github.com/kopia/kopia/repo/encryption" - "github.com/kopia/kopia/repo/object" + "github.com/kopia/kopia/repo/format" "github.com/kopia/kopia/snapshot" ) @@ -47,7 +47,7 @@ func (e *Environment) RootStorage() blob.Storage { } // setup sets up a test environment. -func (e *Environment) setup(tb testing.TB, version content.FormatVersion, opts ...Options) *Environment { +func (e *Environment) setup(tb testing.TB, version format.Version, opts ...Options) *Environment { tb.Helper() ctx := testlogging.Context(tb) @@ -55,8 +55,8 @@ func (e *Environment) setup(tb testing.TB, version content.FormatVersion, opts . openOpt := &repo.Options{} opt := &repo.NewRepositoryOptions{ - BlockFormat: content.FormattingOptions{ - MutableParameters: content.MutableParameters{ + BlockFormat: format.ContentFormat{ + MutableParameters: format.MutableParameters{ Version: version, }, HMACSecret: []byte{}, @@ -64,7 +64,7 @@ func (e *Environment) setup(tb testing.TB, version content.FormatVersion, opts . Encryption: encryption.DefaultAlgorithm, EnablePasswordChange: true, }, - ObjectFormat: object.Format{ + ObjectFormat: format.ObjectFormat{ Splitter: "FIXED-1M", }, } @@ -267,10 +267,10 @@ func repoOptions(openOpts []func(*repo.Options)) *repo.Options { } // FormatNotImportant chooses arbitrary format version where it's not important to the test. -const FormatNotImportant = content.FormatVersion3 +const FormatNotImportant = format.FormatVersion3 // NewEnvironment creates a new repository testing environment and ensures its cleanup at the end of the test. -func NewEnvironment(tb testing.TB, version content.FormatVersion, opts ...Options) (context.Context, *Environment) { +func NewEnvironment(tb testing.TB, version format.Version, opts ...Options) (context.Context, *Environment) { tb.Helper() ctx := testlogging.Context(tb) diff --git a/internal/server/api_repo.go b/internal/server/api_repo.go index 42475dcdc..41e6db635 100644 --- a/internal/server/api_repo.go +++ b/internal/server/api_repo.go @@ -17,6 +17,7 @@ "github.com/kopia/kopia/repo/blob/throttling" "github.com/kopia/kopia/repo/compression" "github.com/kopia/kopia/repo/encryption" + "github.com/kopia/kopia/repo/format" "github.com/kopia/kopia/repo/hashing" "github.com/kopia/kopia/repo/maintenance" "github.com/kopia/kopia/repo/splitter" @@ -36,7 +37,7 @@ func handleRepoParameters(ctx context.Context, rc requestContext) (interface{}, rp := &remoterepoapi.Parameters{ HashFunction: dr.ContentReader().ContentFormat().GetHashFunction(), HMACSecret: dr.ContentReader().ContentFormat().GetHmacSecret(), - Format: dr.ObjectFormat(), + ObjectFormat: dr.ObjectFormat(), SupportsContentCompression: dr.ContentReader().SupportsContentCompression(), } @@ -174,7 +175,7 @@ func handleRepoExists(ctx context.Context, rc requestContext) (interface{}, *api var tmp gather.WriteBuffer defer tmp.Close() - if err := st.GetBlob(ctx, repo.FormatBlobID, 0, -1, &tmp); err != nil { + if err := st.GetBlob(ctx, format.KopiaRepositoryBlobID, 0, -1, &tmp); err != nil { if errors.Is(err, blob.ErrBlobNotFound) { return nil, requestError(serverapi.ErrorNotInitialized, "repository not initialized") } diff --git a/repo/api_server_repository.go b/repo/api_server_repository.go index 4439d1da6..aaaab8d2a 100644 --- a/repo/api_server_repository.go +++ b/repo/api_server_repository.go @@ -16,6 +16,7 @@ "github.com/kopia/kopia/internal/remoterepoapi" "github.com/kopia/kopia/repo/compression" "github.com/kopia/kopia/repo/content" + "github.com/kopia/kopia/repo/format" "github.com/kopia/kopia/repo/hashing" "github.com/kopia/kopia/repo/manifest" "github.com/kopia/kopia/repo/object" @@ -33,7 +34,7 @@ type APIServerInfo struct { type apiServerRepository struct { cli *apiclient.KopiaAPIClient h hashing.HashFunc - objectFormat object.Format + objectFormat format.ObjectFormat serverSupportsContentCompression bool cliOpts ClientOptions omgr *object.Manager @@ -306,7 +307,7 @@ func openRestAPIRepository(ctx context.Context, si *APIServerInfo, cliOpts Clien } rr.h = hf - rr.objectFormat = p.Format + rr.objectFormat = p.ObjectFormat rr.serverSupportsContentCompression = p.SupportsContentCompression // create object manager using rr as contentManager implementation. diff --git a/repo/blobcfg_blob.go b/repo/blobcfg_blob.go deleted file mode 100644 index b2aadc667..000000000 --- a/repo/blobcfg_blob.go +++ /dev/null @@ -1,89 +0,0 @@ -package repo - -import ( - "context" - "encoding/json" - - "github.com/pkg/errors" - - "github.com/kopia/kopia/internal/gather" - "github.com/kopia/kopia/repo/blob" - "github.com/kopia/kopia/repo/content" -) - -// BlobCfgBlobID is the identifier of a BLOB that describes BLOB retention -// settings for the repository. -const BlobCfgBlobID = "kopia.blobcfg" - -func blobCfgBlobFromOptions(opt *NewRepositoryOptions) content.BlobCfgBlob { - return content.BlobCfgBlob{ - RetentionMode: opt.RetentionMode, - RetentionPeriod: opt.RetentionPeriod, - } -} - -func serializeBlobCfgBytes(f *formatBlob, r content.BlobCfgBlob, masterKey []byte) ([]byte, error) { - data, err := json.Marshal(r) - if err != nil { - return nil, errors.Wrap(err, "can't marshal blobCfgBlob to JSON") - } - - switch f.EncryptionAlgorithm { - case "NONE": - return data, nil - - case aes256GcmEncryption: - return encryptRepositoryBlobBytesAes256Gcm(data, masterKey, f.UniqueID) - - default: - return nil, errors.Errorf("unknown encryption algorithm: '%v'", f.EncryptionAlgorithm) - } -} - -func deserializeBlobCfgBytes(f *formatBlob, encryptedBlobCfgBytes, masterKey []byte) (content.BlobCfgBlob, error) { - var ( - plainText []byte - r content.BlobCfgBlob - err error - ) - - if encryptedBlobCfgBytes == nil { - return r, nil - } - - switch f.EncryptionAlgorithm { - case "NONE": // do nothing - plainText = encryptedBlobCfgBytes - - case aes256GcmEncryption: - plainText, err = decryptRepositoryBlobBytesAes256Gcm(encryptedBlobCfgBytes, masterKey, f.UniqueID) - if err != nil { - return content.BlobCfgBlob{}, errors.Errorf("unable to decrypt repository blobcfg blob") - } - - default: - return content.BlobCfgBlob{}, errors.Errorf("unknown encryption algorithm: '%v'", f.EncryptionAlgorithm) - } - - if err = json.Unmarshal(plainText, &r); err != nil { - return content.BlobCfgBlob{}, errors.Wrap(err, "invalid repository blobcfg blob") - } - - return r, nil -} - -func writeBlobCfgBlob(ctx context.Context, st blob.Storage, f *formatBlob, blobcfg content.BlobCfgBlob, formatEncryptionKey []byte) error { - blobCfgBytes, err := serializeBlobCfgBytes(f, blobcfg, formatEncryptionKey) - if err != nil { - return errors.Wrap(err, "unable to encrypt blobcfg bytes") - } - - if err := st.PutBlob(ctx, BlobCfgBlobID, gather.FromSlice(blobCfgBytes), blob.PutOptions{ - RetentionMode: blobcfg.RetentionMode, - RetentionPeriod: blobcfg.RetentionPeriod, - }); err != nil { - return errors.Wrapf(err, "PutBlob() failed for %q", BlobCfgBlobID) - } - - return nil -} diff --git a/repo/change_password.go b/repo/change_password.go index 392a2d755..60aa82c1b 100644 --- a/repo/change_password.go +++ b/repo/change_password.go @@ -6,6 +6,8 @@ "path/filepath" "github.com/pkg/errors" + + "github.com/kopia/kopia/repo/format" ) // ChangePassword changes the repository password and rewrites @@ -13,7 +15,7 @@ func (r *directRepository) ChangePassword(ctx context.Context, newPassword string) error { f := r.formatBlob - repoConfig, err := f.decryptFormatBytes(r.formatEncryptionKey) + repoConfig, err := f.DecryptRepositoryConfig(r.formatEncryptionKey) if err != nil { return errors.Wrap(err, "unable to decrypt repository config") } @@ -22,33 +24,33 @@ func (r *directRepository) ChangePassword(ctx context.Context, newPassword strin return errors.Errorf("password changes are not supported for repositories created using Kopia v0.8 or older") } - newFormatEncryptionKey, err := f.deriveFormatEncryptionKeyFromPassword(newPassword) + newFormatEncryptionKey, err := f.DeriveFormatEncryptionKeyFromPassword(newPassword) if err != nil { return errors.Wrap(err, "unable to derive master key") } r.formatEncryptionKey = newFormatEncryptionKey - if err := encryptFormatBytes(f, repoConfig, newFormatEncryptionKey, f.UniqueID); err != nil { + if err := f.EncryptRepositoryConfig(repoConfig, newFormatEncryptionKey); err != nil { return errors.Wrap(err, "unable to encrypt format bytes") } - if err := writeBlobCfgBlob(ctx, r.blobs, f, r.blobCfgBlob, newFormatEncryptionKey); err != nil { + if err := f.WriteBlobCfgBlob(ctx, r.blobs, r.blobCfgBlob, newFormatEncryptionKey); err != nil { return errors.Wrap(err, "unable to write blobcfg blob") } - if err := writeFormatBlob(ctx, r.blobs, f, r.blobCfgBlob); err != nil { + if err := f.WriteKopiaRepositoryBlob(ctx, r.blobs, r.blobCfgBlob); err != nil { return errors.Wrap(err, "unable to write format blob") } // remove cached kopia.repository blob. if cd := r.cachingOptions.CacheDirectory; cd != "" { - if err := os.Remove(filepath.Join(cd, FormatBlobID)); err != nil { - log(ctx).Errorf("unable to remove %s: %v", FormatBlobID, err) + if err := os.Remove(filepath.Join(cd, format.KopiaRepositoryBlobID)); err != nil { + log(ctx).Errorf("unable to remove %s: %v", format.KopiaRepositoryBlobID, err) } - if err := os.Remove(filepath.Join(cd, BlobCfgBlobID)); err != nil && !os.IsNotExist(err) { - log(ctx).Errorf("unable to remove %s: %v", BlobCfgBlobID, err) + if err := os.Remove(filepath.Join(cd, format.KopiaBlobCfgBlobID)); err != nil && !os.IsNotExist(err) { + log(ctx).Errorf("unable to remove %s: %v", format.KopiaBlobCfgBlobID, err) } } diff --git a/repo/connect.go b/repo/connect.go index 742d9d61d..6ec83af72 100644 --- a/repo/connect.go +++ b/repo/connect.go @@ -10,6 +10,7 @@ "github.com/kopia/kopia/internal/ospath" "github.com/kopia/kopia/repo/blob" "github.com/kopia/kopia/repo/content" + "github.com/kopia/kopia/repo/format" ) // ConnectOptions specifies options when persisting configuration to connect to a repository. @@ -32,7 +33,7 @@ func Connect(ctx context.Context, configFile string, st blob.Storage, password s var formatBytes gather.WriteBuffer defer formatBytes.Close() - if err := st.GetBlob(ctx, FormatBlobID, 0, -1, &formatBytes); err != nil { + if err := st.GetBlob(ctx, format.KopiaRepositoryBlobID, 0, -1, &formatBytes); err != nil { if errors.Is(err, blob.ErrBlobNotFound) { return ErrRepositoryNotInitialized } @@ -40,8 +41,9 @@ func Connect(ctx context.Context, configFile string, st blob.Storage, password s return errors.Wrap(err, "unable to read format blob") } - f, err := parseFormatBlob(formatBytes.ToByteSlice()) + f, err := format.ParseKopiaRepositoryJSON(formatBytes.ToByteSlice()) if err != nil { + // nolint:wrapcheck return err } diff --git a/repo/content/blob_crypto_test.go b/repo/content/blob_crypto_test.go index c22936702..bccb3c57f 100644 --- a/repo/content/blob_crypto_test.go +++ b/repo/content/blob_crypto_test.go @@ -9,11 +9,12 @@ "github.com/kopia/kopia/internal/gather" "github.com/kopia/kopia/repo/encryption" + "github.com/kopia/kopia/repo/format" "github.com/kopia/kopia/repo/hashing" ) func TestBlobCrypto(t *testing.T) { - f := &FormattingOptions{ + f := &format.ContentFormat{ Hash: hashing.DefaultAlgorithm, Encryption: encryption.DefaultAlgorithm, } @@ -85,7 +86,7 @@ func(output []byte, data gather.Bytes) []byte { _, err := EncryptBLOB(cr, gather.FromSlice([]byte{1, 2, 3}), "n", "mysessionid", &tmp) require.Error(t, err) - f := &FormattingOptions{ + f := &format.ContentFormat{ Hash: hashing.DefaultAlgorithm, Encryption: encryption.DefaultAlgorithm, } diff --git a/repo/content/committed_read_manager.go b/repo/content/committed_read_manager.go index aac3c7532..483d7a784 100644 --- a/repo/content/committed_read_manager.go +++ b/repo/content/committed_read_manager.go @@ -21,6 +21,7 @@ "github.com/kopia/kopia/repo/blob/filesystem" "github.com/kopia/kopia/repo/blob/sharded" "github.com/kopia/kopia/repo/compression" + "github.com/kopia/kopia/repo/format" "github.com/kopia/kopia/repo/hashing" "github.com/kopia/kopia/repo/logging" ) @@ -98,7 +99,7 @@ type SharedManager struct { // +checklocks:indexesLock refreshIndexesAfter time.Time - format FormattingOptionsProvider + format format.Provider checkInvariantsOnUnlock bool minPreambleLength int @@ -537,7 +538,7 @@ func (sm *SharedManager) shouldRefreshIndexes() bool { } // NewSharedManager returns SharedManager that is used by SessionWriteManagers on top of a repository. -func NewSharedManager(ctx context.Context, st blob.Storage, prov FormattingOptionsProvider, caching *CachingOptions, opts *ManagerOptions) (*SharedManager, error) { +func NewSharedManager(ctx context.Context, st blob.Storage, prov format.Provider, caching *CachingOptions, opts *ManagerOptions) (*SharedManager, error) { opts = opts.CloneOrDefault() if opts.TimeNow == nil { opts.TimeNow = clock.Now diff --git a/repo/content/content_formatter_test.go b/repo/content/content_formatter_test.go index 287926c1f..043d378b5 100644 --- a/repo/content/content_formatter_test.go +++ b/repo/content/content_formatter_test.go @@ -15,6 +15,7 @@ "github.com/kopia/kopia/internal/testlogging" "github.com/kopia/kopia/repo/blob" "github.com/kopia/kopia/repo/encryption" + "github.com/kopia/kopia/repo/format" "github.com/kopia/kopia/repo/hashing" ) @@ -33,7 +34,7 @@ func TestFormatters(t *testing.T) { t.Run(encryptionAlgo, func(t *testing.T) { ctx := testlogging.Context(t) - fo := &FormattingOptions{ + fo := &format.ContentFormat{ HMACSecret: secret, MasterKey: make([]byte, 32), Hash: hashAlgo, @@ -77,11 +78,11 @@ func verifyEndToEndFormatter(ctx context.Context, t *testing.T, hashAlgo, encryp keyTime := map[blob.ID]time.Time{} st := blobtesting.NewMapStorage(data, keyTime, nil) - bm, err := NewManagerForTesting(testlogging.Context(t), st, mustCreateFormatProvider(t, &FormattingOptions{ + bm, err := NewManagerForTesting(testlogging.Context(t), st, mustCreateFormatProvider(t, &format.ContentFormat{ Hash: hashAlgo, Encryption: encryptionAlgo, HMACSecret: hmacSecret, - MutableParameters: MutableParameters{ + MutableParameters: format.MutableParameters{ Version: 1, MaxPackSize: maxPackSize, }, @@ -137,10 +138,10 @@ func verifyEndToEndFormatter(ctx context.Context, t *testing.T, hashAlgo, encryp } } -func mustCreateFormatProvider(t *testing.T, f *FormattingOptions) FormattingOptionsProvider { +func mustCreateFormatProvider(t *testing.T, f *format.ContentFormat) format.Provider { t.Helper() - fop, err := NewFormattingOptionsProvider(f, nil) + fop, err := format.NewFormattingOptionsProvider(f, nil) require.NoError(t, err) return fop diff --git a/repo/content/content_formatting_options.go b/repo/content/content_formatting_options.go deleted file mode 100644 index 095121873..000000000 --- a/repo/content/content_formatting_options.go +++ /dev/null @@ -1,320 +0,0 @@ -package content - -import ( - "time" - - "github.com/pkg/errors" - - "github.com/kopia/kopia/internal/epoch" - "github.com/kopia/kopia/internal/gather" - "github.com/kopia/kopia/internal/units" - "github.com/kopia/kopia/repo/blob" - "github.com/kopia/kopia/repo/content/index" - "github.com/kopia/kopia/repo/encryption" - "github.com/kopia/kopia/repo/hashing" -) - -const ( - minValidPackSize = 10 << 20 - maxValidPackSize = 120 << 20 -) - -// FormatVersion denotes content format version. -type FormatVersion int - -// Supported format versions. -const ( - FormatVersion1 FormatVersion = 1 - FormatVersion2 FormatVersion = 2 // new in v0.9 - FormatVersion3 FormatVersion = 3 // new in v0.11 - - MaxFormatVersion = FormatVersion3 -) - -// FormattingOptions describes the rules for formatting contents in repository. -type FormattingOptions struct { - Hash string `json:"hash,omitempty"` // identifier of the hash algorithm used - Encryption string `json:"encryption,omitempty"` // identifier of the encryption algorithm used - HMACSecret []byte `json:"secret,omitempty" kopia:"sensitive"` // HMAC secret used to generate encryption keys - MasterKey []byte `json:"masterKey,omitempty" kopia:"sensitive"` // master encryption key (SIV-mode encryption only) - MutableParameters - - EnablePasswordChange bool `json:"enablePasswordChange"` // disables replication of kopia.repository blob in packs -} - -// ResolveFormatVersion applies format options parameters based on the format version. -func (f *FormattingOptions) ResolveFormatVersion() error { - switch f.Version { - case FormatVersion2, FormatVersion3: - f.EnablePasswordChange = true - f.IndexVersion = index.Version2 - f.EpochParameters = epoch.DefaultParameters() - - return nil - - case FormatVersion1: - f.EnablePasswordChange = false - f.IndexVersion = index.Version1 - f.EpochParameters = epoch.Parameters{} - - return nil - - default: - return errors.Errorf("Unsupported format version: %v", f.Version) - } -} - -// GetMutableParameters implements FormattingOptionsProvider. -func (f *FormattingOptions) GetMutableParameters() MutableParameters { - return f.MutableParameters -} - -// SupportsPasswordChange implements FormattingOptionsProvider. -func (f *FormattingOptions) SupportsPasswordChange() bool { - return f.EnablePasswordChange -} - -// MutableParameters represents parameters of the content manager that can be mutated after the repository -// is created. -type MutableParameters struct { - Version FormatVersion `json:"version,omitempty"` // version number, must be "1", "2" or "3" - MaxPackSize int `json:"maxPackSize,omitempty"` // maximum size of a pack object - IndexVersion int `json:"indexVersion,omitempty"` // force particular index format version (1,2,..) - EpochParameters epoch.Parameters `json:"epochParameters,omitempty"` // epoch manager parameters -} - -// Validate validates the parameters. -func (v *MutableParameters) Validate() error { - if v.MaxPackSize < minValidPackSize { - return errors.Errorf("max pack size too small, must be >= %v", units.BytesStringBase2(minValidPackSize)) - } - - if v.MaxPackSize > maxValidPackSize { - return errors.Errorf("max pack size too big, must be <= %v", units.BytesStringBase2(maxValidPackSize)) - } - - if v.IndexVersion < 0 || v.IndexVersion > index.Version2 { - return errors.Errorf("invalid index version, supported versions are 1 & 2") - } - - if err := v.EpochParameters.Validate(); err != nil { - return errors.Wrap(err, "invalid epoch parameters") - } - - return nil -} - -// GetEncryptionAlgorithm implements encryption.Parameters. -func (f *FormattingOptions) GetEncryptionAlgorithm() string { - return f.Encryption -} - -// GetMasterKey implements encryption.Parameters. -func (f *FormattingOptions) GetMasterKey() []byte { - return f.MasterKey -} - -// GetHashFunction implements hashing.Parameters. -func (f *FormattingOptions) GetHashFunction() string { - return f.Hash -} - -// GetHmacSecret implements hashing.Parameters. -func (f *FormattingOptions) GetHmacSecret() []byte { - return f.HMACSecret -} - -// FormattingOptionsProvider provides current formatting options. The options returned -// should not be cached for more than a few seconds as they are subject to change. -type FormattingOptionsProvider interface { - epoch.ParametersProvider - - MaxIndexBlobSize() int64 - WriteIndexVersion() int - IndexShardSize() int - - encryption.Parameters - hashing.Parameters - - HashFunc() hashing.HashFunc - Encryptor() encryption.Encryptor - - GetMutableParameters() MutableParameters - GetMasterKey() []byte - SupportsPasswordChange() bool - FormatVersion() FormatVersion - MaxPackBlobSize() int - RepositoryFormatBytes() []byte - Struct() FormattingOptions -} - -type formattingOptionsProvider struct { - *FormattingOptions - - h hashing.HashFunc - e encryption.Encryptor - actualFormatVersion FormatVersion - actualIndexVersion int - formatBytes []byte -} - -func (f *formattingOptionsProvider) FormatVersion() FormatVersion { - return f.Version -} - -// whether epoch manager is enabled, must be true. -func (f *formattingOptionsProvider) GetEpochManagerEnabled() bool { - return f.EpochParameters.Enabled -} - -// how frequently each client will list blobs to determine the current epoch. -func (f *formattingOptionsProvider) GetEpochRefreshFrequency() time.Duration { - return f.EpochParameters.EpochRefreshFrequency -} - -// number of epochs between full checkpoints. -func (f *formattingOptionsProvider) GetEpochFullCheckpointFrequency() int { - return f.EpochParameters.FullCheckpointFrequency -} - -// GetEpochCleanupSafetyMargin returns safety margin to prevent uncompacted blobs from being deleted if the corresponding compacted blob age is less than this. -func (f *formattingOptionsProvider) GetEpochCleanupSafetyMargin() time.Duration { - return f.EpochParameters.CleanupSafetyMargin -} - -// GetMinEpochDuration returns the minimum duration of an epoch. -func (f *formattingOptionsProvider) GetMinEpochDuration() time.Duration { - return f.EpochParameters.MinEpochDuration -} - -// GetEpochAdvanceOnCountThreshold returns the number of files above which epoch should be advanced. -func (f *formattingOptionsProvider) GetEpochAdvanceOnCountThreshold() int { - return f.EpochParameters.EpochAdvanceOnCountThreshold -} - -// GetEpochAdvanceOnTotalSizeBytesThreshold returns the total size of files above which the epoch should be advanced. -func (f *formattingOptionsProvider) GetEpochAdvanceOnTotalSizeBytesThreshold() int64 { - return f.EpochParameters.EpochAdvanceOnTotalSizeBytesThreshold -} - -// GetEpochDeleteParallelism returns the number of blobs to delete in parallel during cleanup. -func (f *formattingOptionsProvider) GetEpochDeleteParallelism() int { - return f.EpochParameters.DeleteParallelism -} - -func (f *formattingOptionsProvider) Struct() FormattingOptions { - return *f.FormattingOptions -} - -// NewFormattingOptionsProvider validates the provided formatting options and returns static -// FormattingOptionsProvider based on them. -func NewFormattingOptionsProvider(f *FormattingOptions, formatBytes []byte) (FormattingOptionsProvider, error) { - formatVersion := f.Version - - if formatVersion < minSupportedReadVersion || formatVersion > currentWriteVersion { - return nil, errors.Errorf("can't handle repositories created using version %v (min supported %v, max supported %v)", formatVersion, minSupportedReadVersion, maxSupportedReadVersion) - } - - if formatVersion < minSupportedWriteVersion || formatVersion > currentWriteVersion { - return nil, errors.Errorf("can't handle repositories created using version %v (min supported %v, max supported %v)", formatVersion, minSupportedWriteVersion, maxSupportedWriteVersion) - } - - actualIndexVersion := f.IndexVersion - if actualIndexVersion == 0 { - actualIndexVersion = legacyIndexVersion - } - - if actualIndexVersion < index.Version1 || actualIndexVersion > index.Version2 { - return nil, errors.Errorf("index version %v is not supported", actualIndexVersion) - } - - h, err := hashing.CreateHashFunc(f) - if err != nil { - return nil, errors.Wrap(err, "unable to create hash") - } - - e, err := encryption.CreateEncryptor(f) - if err != nil { - return nil, errors.Wrap(err, "unable to create encryptor") - } - - contentID := h(nil, gather.FromSlice(nil)) - - var tmp gather.WriteBuffer - defer tmp.Close() - - err = e.Encrypt(gather.FromSlice(nil), contentID, &tmp) - if err != nil { - return nil, errors.Wrap(err, "invalid encryptor") - } - - return &formattingOptionsProvider{ - FormattingOptions: f, - - h: h, - e: e, - actualIndexVersion: actualIndexVersion, - actualFormatVersion: f.Version, - formatBytes: formatBytes, - }, nil -} - -func (f *formattingOptionsProvider) Encryptor() encryption.Encryptor { - return f.e -} - -func (f *formattingOptionsProvider) HashFunc() hashing.HashFunc { - return f.h -} - -func (f *formattingOptionsProvider) WriteIndexVersion() int { - return f.actualIndexVersion -} - -func (f *formattingOptionsProvider) MaxIndexBlobSize() int64 { - return int64(f.MaxPackSize) -} - -func (f *formattingOptionsProvider) MaxPackBlobSize() int { - return f.MaxPackSize -} - -func (f *formattingOptionsProvider) GetEpochManagerParameters() epoch.Parameters { - return f.EpochParameters -} - -func (f *formattingOptionsProvider) IndexShardSize() int { - return defaultIndexShardSize -} - -func (f *formattingOptionsProvider) RepositoryFormatBytes() []byte { - return f.formatBytes -} - -var _ FormattingOptionsProvider = (*formattingOptionsProvider)(nil) - -// BlobCfgBlob is the content for `kopia.blobcfg` blob which contains the blob -// management configuration options. -type BlobCfgBlob struct { - RetentionMode blob.RetentionMode `json:"retentionMode,omitempty"` - RetentionPeriod time.Duration `json:"retentionPeriod,omitempty"` -} - -// IsRetentionEnabled returns true if retention is enabled on the blob-config -// object. -func (r *BlobCfgBlob) IsRetentionEnabled() bool { - return r.RetentionMode != "" && r.RetentionPeriod != 0 -} - -// Validate validates the blob config parameters. -func (r *BlobCfgBlob) Validate() error { - if (r.RetentionMode == "") != (r.RetentionPeriod == 0) { - return errors.Errorf("both retention mode and period must be provided when setting blob retention properties") - } - - if r.RetentionPeriod != 0 && r.RetentionPeriod < 24*time.Hour { - return errors.Errorf("invalid retention-period, the minimum required is 1-day and there is no maximum limit") - } - - return nil -} diff --git a/repo/content/content_manager.go b/repo/content/content_manager.go index ce3af3847..12bee95b0 100644 --- a/repo/content/content_manager.go +++ b/repo/content/content_manager.go @@ -19,6 +19,7 @@ "github.com/kopia/kopia/repo/blob" "github.com/kopia/kopia/repo/compression" "github.com/kopia/kopia/repo/content/index" + "github.com/kopia/kopia/repo/format" "github.com/kopia/kopia/repo/hashing" "github.com/kopia/kopia/repo/logging" ) @@ -34,11 +35,7 @@ packBlobIDLength = 16 - defaultIndexShardSize = 16e6 // slightly less than 2^24, which lets index use 24-bit/3-byte indexes - DefaultIndexVersion = 2 - - legacyIndexVersion = index.Version1 ) var tracer = otel.Tracer("kopia/content") @@ -57,14 +54,6 @@ defaultMaxPreambleLength = 32 defaultPaddingUnit = 4096 - currentWriteVersion = FormatVersion3 - - minSupportedWriteVersion = FormatVersion1 - maxSupportedWriteVersion = FormatVersion3 - - minSupportedReadVersion = FormatVersion1 - maxSupportedReadVersion = FormatVersion3 - indexLoadAttempts = 10 ) @@ -589,7 +578,7 @@ func removePendingPack(slice []*pendingPackInfo, pp *pendingPackInfo) []*pending } // ContentFormat returns formatting options. -func (bm *WriteManager) ContentFormat() FormattingOptionsProvider { +func (bm *WriteManager) ContentFormat() format.Provider { return bm.format } @@ -935,7 +924,7 @@ func (o *ManagerOptions) CloneOrDefault() *ManagerOptions { } // NewManagerForTesting creates new content manager with given packing options and a formatter. -func NewManagerForTesting(ctx context.Context, st blob.Storage, f FormattingOptionsProvider, caching *CachingOptions, options *ManagerOptions) (*WriteManager, error) { +func NewManagerForTesting(ctx context.Context, st blob.Storage, f format.Provider, caching *CachingOptions, options *ManagerOptions) (*WriteManager, error) { options = options.CloneOrDefault() if options.TimeNow == nil { options.TimeNow = clock.Now diff --git a/repo/content/content_manager_test.go b/repo/content/content_manager_test.go index c45bd7471..7e5d2d0b4 100644 --- a/repo/content/content_manager_test.go +++ b/repo/content/content_manager_test.go @@ -33,6 +33,7 @@ "github.com/kopia/kopia/repo/blob/logging" "github.com/kopia/kopia/repo/compression" "github.com/kopia/kopia/repo/content/index" + "github.com/kopia/kopia/repo/format" ) const ( @@ -52,7 +53,7 @@ func TestMain(m *testing.M) { testutil.MyTestMain(m) } func TestFormatV1(t *testing.T) { testutil.RunAllTestsWithParam(t, &contentManagerSuite{ - mutableParameters: MutableParameters{ + mutableParameters: format.MutableParameters{ Version: 1, IndexVersion: 1, MaxPackSize: maxPackSize, @@ -62,7 +63,7 @@ func TestFormatV1(t *testing.T) { func TestFormatV2(t *testing.T) { testutil.RunAllTestsWithParam(t, &contentManagerSuite{ - mutableParameters: MutableParameters{ + mutableParameters: format.MutableParameters{ Version: 2, MaxPackSize: maxPackSize, IndexVersion: index.Version2, @@ -72,7 +73,7 @@ func TestFormatV2(t *testing.T) { } type contentManagerSuite struct { - mutableParameters MutableParameters + mutableParameters format.MutableParameters } func (s *contentManagerSuite) TestContentManagerEmptyFlush(t *testing.T) { @@ -354,7 +355,7 @@ func (s *contentManagerSuite) TestContentManagerFailedToWritePack(t *testing.T) ta := faketime.NewTimeAdvance(fakeTime, 0) - bm, err := NewManagerForTesting(testlogging.Context(t), st, mustCreateFormatProvider(t, &FormattingOptions{ + bm, err := NewManagerForTesting(testlogging.Context(t), st, mustCreateFormatProvider(t, &format.ContentFormat{ Hash: "HMAC-SHA256-128", Encryption: "AES256-GCM-HMAC-SHA256", MutableParameters: s.mutableParameters, @@ -1860,7 +1861,7 @@ func (s *contentManagerSuite) TestContentReadAliasing(t *testing.T) { } func (s *contentManagerSuite) TestVersionCompatibility(t *testing.T) { - for writeVer := minSupportedReadVersion; writeVer <= currentWriteVersion; writeVer++ { + for writeVer := format.MinSupportedReadVersion; writeVer <= format.CurrentWriteVersion; writeVer++ { writeVer := writeVer t.Run(fmt.Sprintf("version-%v", writeVer), func(t *testing.T) { s.verifyVersionCompat(t, writeVer) @@ -1868,7 +1869,7 @@ func (s *contentManagerSuite) TestVersionCompatibility(t *testing.T) { } } -func (s *contentManagerSuite) verifyVersionCompat(t *testing.T, writeVersion FormatVersion) { +func (s *contentManagerSuite) verifyVersionCompat(t *testing.T, writeVersion format.Version) { t.Helper() ctx := testlogging.Context(t) @@ -2333,7 +2334,7 @@ type contentManagerTestTweaks struct { indexVersion int maxPackSize int - formatVersion FormatVersion + formatVersion format.Version } func (s *contentManagerSuite) newTestContentManagerWithTweaks(t *testing.T, st blob.Storage, tweaks *contentManagerTestTweaks) *WriteManager { @@ -2363,7 +2364,7 @@ func (s *contentManagerSuite) newTestContentManagerWithTweaks(t *testing.T, st b } ctx := testlogging.Context(t) - fo := mustCreateFormatProvider(t, &FormattingOptions{ + fo := mustCreateFormatProvider(t, &format.ContentFormat{ Hash: "HMAC-SHA256", Encryption: "AES256-GCM-HMAC-SHA256", HMACSecret: hmacSecret, diff --git a/repo/content/content_reader.go b/repo/content/content_reader.go index 79630fa4f..d5c03c265 100644 --- a/repo/content/content_reader.go +++ b/repo/content/content_reader.go @@ -4,12 +4,13 @@ "context" "github.com/kopia/kopia/internal/epoch" + "github.com/kopia/kopia/repo/format" ) // Reader defines content read API. type Reader interface { SupportsContentCompression() bool - ContentFormat() FormattingOptionsProvider + ContentFormat() format.Provider GetContent(ctx context.Context, id ID) ([]byte, error) ContentInfo(ctx context.Context, id ID) (Info, error) IterateContents(ctx context.Context, opts IterateOptions, callback IterateCallback) error diff --git a/repo/content/encrypted_blob_mgr_test.go b/repo/content/encrypted_blob_mgr_test.go index 3381be2a0..ac2273909 100644 --- a/repo/content/encrypted_blob_mgr_test.go +++ b/repo/content/encrypted_blob_mgr_test.go @@ -11,6 +11,7 @@ "github.com/kopia/kopia/internal/testlogging" "github.com/kopia/kopia/repo/blob" "github.com/kopia/kopia/repo/encryption" + "github.com/kopia/kopia/repo/format" "github.com/kopia/kopia/repo/hashing" "github.com/kopia/kopia/repo/logging" ) @@ -28,7 +29,7 @@ func TestEncryptedBlobManager(t *testing.T) { data := blobtesting.DataMap{} st := blobtesting.NewMapStorage(data, nil, nil) fs := blobtesting.NewFaultyStorage(st) - f := &FormattingOptions{ + f := &format.ContentFormat{ Hash: hashing.DefaultAlgorithm, Encryption: encryption.DefaultAlgorithm, } diff --git a/repo/content/index_blob_manager_v0_test.go b/repo/content/index_blob_manager_v0_test.go index d3ca1625e..d83196fe5 100644 --- a/repo/content/index_blob_manager_v0_test.go +++ b/repo/content/index_blob_manager_v0_test.go @@ -24,6 +24,7 @@ "github.com/kopia/kopia/repo/blob" "github.com/kopia/kopia/repo/blob/logging" "github.com/kopia/kopia/repo/encryption" + "github.com/kopia/kopia/repo/format" "github.com/kopia/kopia/repo/hashing" ) @@ -765,7 +766,7 @@ func assertIndexBlobList(t *testing.T, m *indexBlobManagerV0, wantMD ...blob.Met func newIndexBlobManagerForTesting(t *testing.T, st blob.Storage, localTimeNow func() time.Time) *indexBlobManagerV0 { t.Helper() - p := &FormattingOptions{ + p := &format.ContentFormat{ Encryption: encryption.DefaultAlgorithm, Hash: hashing.DefaultAlgorithm, } diff --git a/repo/format/blobcfg_blob.go b/repo/format/blobcfg_blob.go new file mode 100644 index 000000000..da248481a --- /dev/null +++ b/repo/format/blobcfg_blob.go @@ -0,0 +1,110 @@ +package format + +import ( + "context" + "encoding/json" + "time" + + "github.com/pkg/errors" + + "github.com/kopia/kopia/internal/gather" + "github.com/kopia/kopia/repo/blob" +) + +// KopiaBlobCfgBlobID is the identifier of a BLOB that describes BLOB retention +// settings for the repository. +const KopiaBlobCfgBlobID = "kopia.blobcfg" + +// BlobStorageConfiguration is the content for `kopia.blobcfg` blob which contains the blob +// storage configuration options. +type BlobStorageConfiguration struct { + RetentionMode blob.RetentionMode `json:"retentionMode,omitempty"` + RetentionPeriod time.Duration `json:"retentionPeriod,omitempty"` +} + +// IsRetentionEnabled returns true if retention is enabled on the blob-config +// object. +func (r *BlobStorageConfiguration) IsRetentionEnabled() bool { + return r.RetentionMode != "" && r.RetentionPeriod != 0 +} + +// Validate validates the blob config parameters. +func (r *BlobStorageConfiguration) Validate() error { + if (r.RetentionMode == "") != (r.RetentionPeriod == 0) { + return errors.Errorf("both retention mode and period must be provided when setting blob retention properties") + } + + if r.RetentionPeriod != 0 && r.RetentionPeriod < 24*time.Hour { + return errors.Errorf("invalid retention-period, the minimum required is 1-day and there is no maximum limit") + } + + return nil +} + +func serializeBlobCfgBytes(f *KopiaRepositoryJSON, r BlobStorageConfiguration, formatEncryptionKey []byte) ([]byte, error) { + data, err := json.Marshal(r) + if err != nil { + return nil, errors.Wrap(err, "can't marshal blobCfgBlob to JSON") + } + + switch f.EncryptionAlgorithm { + case "NONE": + return data, nil + + case aes256GcmEncryption: + return encryptRepositoryBlobBytesAes256Gcm(data, formatEncryptionKey, f.UniqueID) + + default: + return nil, errors.Errorf("unknown encryption algorithm: '%v'", f.EncryptionAlgorithm) + } +} + +// DeserializeBlobCfgBytes decrypts and deserializes the given bytes into BlobStorageConfiguration. +func (f *KopiaRepositoryJSON) DeserializeBlobCfgBytes(encryptedBlobCfgBytes, formatEncryptionKey []byte) (BlobStorageConfiguration, error) { + var ( + plainText []byte + r BlobStorageConfiguration + err error + ) + + if encryptedBlobCfgBytes == nil { + return r, nil + } + + switch f.EncryptionAlgorithm { + case "NONE": // do nothing + plainText = encryptedBlobCfgBytes + + case aes256GcmEncryption: + plainText, err = decryptRepositoryBlobBytesAes256Gcm(encryptedBlobCfgBytes, formatEncryptionKey, f.UniqueID) + if err != nil { + return BlobStorageConfiguration{}, errors.Errorf("unable to decrypt repository blobcfg blob") + } + + default: + return BlobStorageConfiguration{}, errors.Errorf("unknown encryption algorithm: '%v'", f.EncryptionAlgorithm) + } + + if err = json.Unmarshal(plainText, &r); err != nil { + return BlobStorageConfiguration{}, errors.Wrap(err, "invalid repository blobcfg blob") + } + + return r, nil +} + +// WriteBlobCfgBlob writes `kopia.blobcfg` encrypted using the provided key. +func (f *KopiaRepositoryJSON) WriteBlobCfgBlob(ctx context.Context, st blob.Storage, blobcfg BlobStorageConfiguration, formatEncryptionKey []byte) error { + blobCfgBytes, err := serializeBlobCfgBytes(f, blobcfg, formatEncryptionKey) + if err != nil { + return errors.Wrap(err, "unable to encrypt blobcfg bytes") + } + + if err := st.PutBlob(ctx, KopiaBlobCfgBlobID, gather.FromSlice(blobCfgBytes), blob.PutOptions{ + RetentionMode: blobcfg.RetentionMode, + RetentionPeriod: blobcfg.RetentionPeriod, + }); err != nil { + return errors.Wrapf(err, "PutBlob() failed for %q", KopiaBlobCfgBlobID) + } + + return nil +} diff --git a/repo/format/content_format.go b/repo/format/content_format.go new file mode 100644 index 000000000..2c026b35b --- /dev/null +++ b/repo/format/content_format.go @@ -0,0 +1,102 @@ +package format + +import ( + "github.com/pkg/errors" + + "github.com/kopia/kopia/internal/epoch" + "github.com/kopia/kopia/internal/units" + "github.com/kopia/kopia/repo/content/index" +) + +// ContentFormat describes the rules for formatting contents in repository. +type ContentFormat struct { + Hash string `json:"hash,omitempty"` // identifier of the hash algorithm used + Encryption string `json:"encryption,omitempty"` // identifier of the encryption algorithm used + HMACSecret []byte `json:"secret,omitempty" kopia:"sensitive"` // HMAC secret used to generate encryption keys + MasterKey []byte `json:"masterKey,omitempty" kopia:"sensitive"` // master encryption key (SIV-mode encryption only) + MutableParameters + + EnablePasswordChange bool `json:"enablePasswordChange"` // disables replication of kopia.repository blob in packs +} + +// ResolveFormatVersion applies format options parameters based on the format version. +func (f *ContentFormat) ResolveFormatVersion() error { + switch f.Version { + case FormatVersion2, FormatVersion3: + f.EnablePasswordChange = true + f.IndexVersion = index.Version2 + f.EpochParameters = epoch.DefaultParameters() + + return nil + + case FormatVersion1: + f.EnablePasswordChange = false + f.IndexVersion = index.Version1 + f.EpochParameters = epoch.Parameters{} + + return nil + + default: + return errors.Errorf("Unsupported format version: %v", f.Version) + } +} + +// GetMutableParameters implements FormattingOptionsProvider. +func (f *ContentFormat) GetMutableParameters() MutableParameters { + return f.MutableParameters +} + +// SupportsPasswordChange implements FormattingOptionsProvider. +func (f *ContentFormat) SupportsPasswordChange() bool { + return f.EnablePasswordChange +} + +// MutableParameters represents parameters of the content manager that can be mutated after the repository +// is created. +type MutableParameters struct { + Version Version `json:"version,omitempty"` // version number, must be "1", "2" or "3" + MaxPackSize int `json:"maxPackSize,omitempty"` // maximum size of a pack object + IndexVersion int `json:"indexVersion,omitempty"` // force particular index format version (1,2,..) + EpochParameters epoch.Parameters `json:"epochParameters,omitempty"` // epoch manager parameters +} + +// Validate validates the parameters. +func (v *MutableParameters) Validate() error { + if v.MaxPackSize < minValidPackSize { + return errors.Errorf("max pack size too small, must be >= %v", units.BytesStringBase2(minValidPackSize)) + } + + if v.MaxPackSize > maxValidPackSize { + return errors.Errorf("max pack size too big, must be <= %v", units.BytesStringBase2(maxValidPackSize)) + } + + if v.IndexVersion < 0 || v.IndexVersion > index.Version2 { + return errors.Errorf("invalid index version, supported versions are 1 & 2") + } + + if err := v.EpochParameters.Validate(); err != nil { + return errors.Wrap(err, "invalid epoch parameters") + } + + return nil +} + +// GetEncryptionAlgorithm implements encryption.Parameters. +func (f *ContentFormat) GetEncryptionAlgorithm() string { + return f.Encryption +} + +// GetMasterKey implements encryption.Parameters. +func (f *ContentFormat) GetMasterKey() []byte { + return f.MasterKey +} + +// GetHashFunction implements hashing.Parameters. +func (f *ContentFormat) GetHashFunction() string { + return f.Hash +} + +// GetHmacSecret implements hashing.Parameters. +func (f *ContentFormat) GetHmacSecret() []byte { + return f.HMACSecret +} diff --git a/repo/crypto_key_derivation.go b/repo/format/crypto_key_derivation.go similarity index 71% rename from repo/crypto_key_derivation.go rename to repo/format/crypto_key_derivation.go index 32ff3e557..115adb24a 100644 --- a/repo/crypto_key_derivation.go +++ b/repo/format/crypto_key_derivation.go @@ -1,4 +1,4 @@ -package repo +package format import ( "crypto/sha256" @@ -7,8 +7,8 @@ "golang.org/x/crypto/hkdf" ) -// deriveKeyFromMasterKey computes a key for a specific purpose and length using HKDF based on the master key. -func deriveKeyFromMasterKey(masterKey, uniqueID, purpose []byte, length int) []byte { +// DeriveKeyFromMasterKey computes a key for a specific purpose and length using HKDF based on the master key. +func DeriveKeyFromMasterKey(masterKey, uniqueID, purpose []byte, length int) []byte { key := make([]byte, length) k := hkdf.New(sha256.New, masterKey, uniqueID, purpose) diff --git a/repo/crypto_key_derivation_nontest.go b/repo/format/crypto_key_derivation_nontest.go similarity index 54% rename from repo/crypto_key_derivation_nontest.go rename to repo/format/crypto_key_derivation_nontest.go index 6ea7daae0..14d892b60 100644 --- a/repo/crypto_key_derivation_nontest.go +++ b/repo/format/crypto_key_derivation_nontest.go @@ -1,17 +1,18 @@ //go:build !testing // +build !testing -package repo +package format import ( "github.com/pkg/errors" "golang.org/x/crypto/scrypt" ) -// defaultKeyDerivationAlgorithm is the key derivation algorithm for new configurations. -const defaultKeyDerivationAlgorithm = "scrypt-65536-8-1" +// DefaultKeyDerivationAlgorithm is the key derivation algorithm for new configurations. +const DefaultKeyDerivationAlgorithm = "scrypt-65536-8-1" -func (f *formatBlob) deriveFormatEncryptionKeyFromPassword(password string) ([]byte, error) { +// DeriveFormatEncryptionKeyFromPassword derives encryption key using the provided password and per-repository unique ID. +func (f *KopiaRepositoryJSON) DeriveFormatEncryptionKeyFromPassword(password string) ([]byte, error) { const masterKeySize = 32 switch f.KeyDerivationAlgorithm { diff --git a/repo/crypto_key_derivation_testing.go b/repo/format/crypto_key_derivation_testing.go similarity index 50% rename from repo/crypto_key_derivation_testing.go rename to repo/format/crypto_key_derivation_testing.go index 0afa8469d..47a8b100a 100644 --- a/repo/crypto_key_derivation_testing.go +++ b/repo/format/crypto_key_derivation_testing.go @@ -1,7 +1,7 @@ //go:build testing // +build testing -package repo +package format import ( "crypto/sha256" @@ -9,14 +9,15 @@ "github.com/pkg/errors" ) -// defaultKeyDerivationAlgorithm is the key derivation algorithm for new configurations. -const defaultKeyDerivationAlgorithm = "testing-only-insecure" +// DefaultKeyDerivationAlgorithm is the key derivation algorithm for new configurations. +const DefaultKeyDerivationAlgorithm = "testing-only-insecure" -func (f *formatBlob) deriveFormatEncryptionKeyFromPassword(password string) ([]byte, error) { +// DeriveFormatEncryptionKeyFromPassword derives encryption key using the provided password and per-repository unique ID. +func (f *KopiaRepositoryJSON) DeriveFormatEncryptionKeyFromPassword(password string) ([]byte, error) { const masterKeySize = 32 switch f.KeyDerivationAlgorithm { - case defaultKeyDerivationAlgorithm: + case DefaultKeyDerivationAlgorithm: h := sha256.New() if _, err := h.Write([]byte(password)); err != nil { return nil, err diff --git a/repo/format_block.go b/repo/format/format_blob.go similarity index 76% rename from repo/format_block.go rename to repo/format/format_blob.go index cb64d158f..139108240 100644 --- a/repo/format_block.go +++ b/repo/format/format_blob.go @@ -1,4 +1,5 @@ -package repo +// Package format manages kopia.repository and other central format blobs. +package format import ( "context" @@ -14,12 +15,13 @@ "github.com/kopia/kopia/internal/gather" "github.com/kopia/kopia/repo/blob" - "github.com/kopia/kopia/repo/content" ) +// DefaultFormatEncryption is the identifier of the default format blob encryption algorithm. +const DefaultFormatEncryption = "AES256_GCM" + const ( aes256GcmEncryption = "AES256_GCM" - defaultFormatEncryption = "AES256_GCM" lengthOfRecoverBlockLength = 2 // number of bytes used to store recover block length maxChecksummedFormatBytesLength = 65000 maxRecoverChunkLength = 65536 @@ -27,8 +29,11 @@ formatBlobChecksumSize = sha256.Size ) -// FormatBlobID is the identifier of a BLOB that describes repository format. -const FormatBlobID = "kopia.repository" +// KopiaRepositoryBlobID is the identifier of a BLOB that describes repository format. +const KopiaRepositoryBlobID = "kopia.repository" + +// ErrInvalidPassword is returned when repository password is invalid. +var ErrInvalidPassword = errors.Errorf("invalid repository password") // nolint:gochecknoglobals var ( @@ -43,7 +48,8 @@ errFormatBlobNotFound = errors.New("format blob not found") ) -type formatBlob struct { +// KopiaRepositoryJSON represents JSON contents of 'kopia.repository' blob. +type KopiaRepositoryJSON struct { Tool string `json:"tool"` BuildVersion string `json:"buildVersion"` BuildInfo string `json:"buildInfo"` @@ -56,13 +62,9 @@ type formatBlob struct { EncryptedFormatBytes []byte `json:"encryptedBlockFormat,omitempty"` } -// encryptedRepositoryConfig contains the configuration of repository that's persisted in encrypted format. -type encryptedRepositoryConfig struct { - Format repositoryObjectFormat `json:"format"` -} - -func parseFormatBlob(b []byte) (*formatBlob, error) { - f := &formatBlob{} +// ParseKopiaRepositoryJSON parses the provided byte slice into KopiaRepositoryJSON. +func ParseKopiaRepositoryJSON(b []byte) (*KopiaRepositoryJSON, error) { + f := &KopiaRepositoryJSON{} if err := json.Unmarshal(b, &f); err != nil { return nil, errors.Wrap(err, "invalid format blob") @@ -163,11 +165,13 @@ func verifyFormatBlobChecksum(b []byte) ([]byte, bool) { return data, true } -func writeFormatBlob(ctx context.Context, st blob.Storage, f *formatBlob, blobCfg content.BlobCfgBlob) error { - return writeFormatBlobWithID(ctx, st, f, blobCfg, FormatBlobID) +// WriteKopiaRepositoryBlob writes `kopia.repository` blob to a given storage. +func (f *KopiaRepositoryJSON) WriteKopiaRepositoryBlob(ctx context.Context, st blob.Storage, blobCfg BlobStorageConfiguration) error { + return f.WriteKopiaRepositoryBlobWithID(ctx, st, blobCfg, KopiaRepositoryBlobID) } -func writeFormatBlobWithID(ctx context.Context, st blob.Storage, f *formatBlob, blobCfg content.BlobCfgBlob, id blob.ID) error { +// WriteKopiaRepositoryBlobWithID writes `kopia.repository` blob to a given storage under an alternate blobID. +func (f *KopiaRepositoryJSON) WriteKopiaRepositoryBlobWithID(ctx context.Context, st blob.Storage, blobCfg BlobStorageConfiguration, id blob.ID) error { buf := gather.NewWriteBuffer() e := json.NewEncoder(buf) e.SetIndent("", " ") @@ -186,29 +190,9 @@ func writeFormatBlobWithID(ctx context.Context, st blob.Storage, f *formatBlob, return nil } -func (f *formatBlob) decryptFormatBytes(masterKey []byte) (*repositoryObjectFormat, error) { - switch f.EncryptionAlgorithm { - case aes256GcmEncryption: - plainText, err := decryptRepositoryBlobBytesAes256Gcm(f.EncryptedFormatBytes, masterKey, f.UniqueID) - if err != nil { - return nil, errors.Errorf("unable to decrypt repository format") - } - - var erc encryptedRepositoryConfig - if err := json.Unmarshal(plainText, &erc); err != nil { - return nil, errors.Wrap(err, "invalid repository format") - } - - return &erc.Format, nil - - default: - return nil, errors.Errorf("unknown encryption algorithm: '%v'", f.EncryptionAlgorithm) - } -} - func initCrypto(masterKey, repositoryID []byte) (cipher.AEAD, []byte, error) { - aesKey := deriveKeyFromMasterKey(masterKey, repositoryID, purposeAESKey, 32) // nolint:gomnd - authData := deriveKeyFromMasterKey(masterKey, repositoryID, purposeAuthData, 32) // nolint:gomnd + aesKey := DeriveKeyFromMasterKey(masterKey, repositoryID, purposeAESKey, 32) // nolint:gomnd + authData := DeriveKeyFromMasterKey(masterKey, repositoryID, purposeAuthData, 32) // nolint:gomnd blk, err := aes.NewCipher(aesKey) if err != nil { @@ -267,28 +251,6 @@ func decryptRepositoryBlobBytesAes256Gcm(data, masterKey, repositoryID []byte) ( return plainText, nil } -func encryptFormatBytes(f *formatBlob, format *repositoryObjectFormat, masterKey, repositoryID []byte) error { - switch f.EncryptionAlgorithm { - case aes256GcmEncryption: - data, err := json.Marshal(&encryptedRepositoryConfig{Format: *format}) - if err != nil { - return errors.Wrap(err, "can't marshal format to JSON") - } - - data, err = encryptRepositoryBlobBytesAes256Gcm(data, masterKey, repositoryID) - if err != nil { - return errors.Wrap(err, "failed to encrypt format JSON") - } - - f.EncryptedFormatBytes = data - - return nil - - default: - return errors.Errorf("unknown encryption algorithm: '%v'", f.EncryptionAlgorithm) - } -} - func addFormatBlobChecksumAndLength(fb []byte) ([]byte, error) { h := hmac.New(sha256.New, formatBlobChecksumSecret) h.Write(fb) diff --git a/repo/format/format_blob_cache.go b/repo/format/format_blob_cache.go new file mode 100644 index 000000000..d812df761 --- /dev/null +++ b/repo/format/format_blob_cache.go @@ -0,0 +1,144 @@ +package format + +import ( + "context" + "os" + "path/filepath" + "time" + + "github.com/pkg/errors" + + "github.com/kopia/kopia/internal/atomicfile" + "github.com/kopia/kopia/internal/cache" + "github.com/kopia/kopia/internal/cachedir" + "github.com/kopia/kopia/internal/clock" + "github.com/kopia/kopia/internal/gather" + "github.com/kopia/kopia/repo/blob" + "github.com/kopia/kopia/repo/logging" +) + +// DefaultRepositoryBlobCacheDuration is the duration for which we treat cached kopia.repository +// as valid. +const DefaultRepositoryBlobCacheDuration = 15 * time.Minute + +var log = logging.Module("kopia/repo/format") + +func formatBytesCachingEnabled(cacheDirectory string, validDuration time.Duration) bool { + if cacheDirectory == "" { + return false + } + + return validDuration > 0 +} + +func readRepositoryBlobBytesFromCache(ctx context.Context, cachedFile string, validDuration time.Duration) (data []byte, cacheMTime time.Time, err error) { + cst, err := os.Stat(cachedFile) + if err != nil { + return nil, time.Time{}, errors.Wrap(err, "unable to open cache file") + } + + cacheMTime = cst.ModTime() + if clock.Now().Sub(cacheMTime) > validDuration { + // got cached file, but it's too old, remove it + if err = os.Remove(cachedFile); err != nil { + log(ctx).Debugf("unable to remove cache file: %v", err) + } + + return nil, time.Time{}, errors.Errorf("cached file too old") + } + + data, err = os.ReadFile(cachedFile) // nolint:gosec + if err != nil { + return nil, time.Time{}, errors.Wrapf(err, "failed to read the cache file %q", cachedFile) + } + + return data, cacheMTime, nil +} + +// ReadAndCacheRepositoryBlobBytes reads the provided blob from the repository or cache directory. +func ReadAndCacheRepositoryBlobBytes(ctx context.Context, st blob.Storage, cacheDirectory, blobID string, validDuration time.Duration) ([]byte, time.Time, error) { + cachedFile := filepath.Join(cacheDirectory, blobID) + + if validDuration == 0 { + validDuration = DefaultRepositoryBlobCacheDuration + } + + if cacheDirectory != "" { + if err := os.MkdirAll(cacheDirectory, cache.DirMode); err != nil && !os.IsExist(err) { + log(ctx).Errorf("unable to create cache directory: %v", err) + } + } + + cacheEnabled := formatBytesCachingEnabled(cacheDirectory, validDuration) + if cacheEnabled { + data, cacheMTime, err := readRepositoryBlobBytesFromCache(ctx, cachedFile, validDuration) + if err == nil { + log(ctx).Debugf("%s retrieved from cache", blobID) + + return data, cacheMTime, nil + } + + if os.IsNotExist(err) { + log(ctx).Debugf("%s could not be fetched from cache: %v", blobID, err) + } + } else { + log(ctx).Debugf("%s cache not enabled", blobID) + } + + var b gather.WriteBuffer + defer b.Close() + + if err := st.GetBlob(ctx, blob.ID(blobID), 0, -1, &b); err != nil { + return nil, time.Time{}, errors.Wrapf(err, "error getting %s blob", blobID) + } + + if cacheEnabled { + if err := atomicfile.Write(cachedFile, b.Bytes().Reader()); err != nil { + log(ctx).Warnf("unable to write cache: %v", err) + } + } + + return b.ToByteSlice(), clock.Now(), nil +} + +// ReadAndCacheDecodedRepositoryConfig reads `kopia.repository` blob, potentially from cache and decodes it. +func ReadAndCacheDecodedRepositoryConfig(ctx context.Context, st blob.Storage, password, cacheDir string, validDuration time.Duration) (ufb *DecodedRepositoryConfig, err error) { + ufb = &DecodedRepositoryConfig{} + + ufb.KopiaRepositoryBytes, ufb.CacheMTime, err = ReadAndCacheRepositoryBlobBytes(ctx, st, cacheDir, KopiaRepositoryBlobID, validDuration) + if err != nil { + return nil, errors.Wrap(err, "unable to read format blob") + } + + if err = cachedir.WriteCacheMarker(cacheDir); err != nil { + return nil, errors.Wrap(err, "unable to write cache directory marker") + } + + ufb.KopiaRepository, err = ParseKopiaRepositoryJSON(ufb.KopiaRepositoryBytes) + if err != nil { + return nil, errors.Wrap(err, "can't parse format blob") + } + + ufb.KopiaRepositoryBytes, err = addFormatBlobChecksumAndLength(ufb.KopiaRepositoryBytes) + if err != nil { + return nil, errors.Errorf("unable to add checksum") + } + + ufb.FormatEncryptionKey, err = ufb.KopiaRepository.DeriveFormatEncryptionKeyFromPassword(password) + if err != nil { + return nil, err + } + + ufb.RepoConfig, err = ufb.KopiaRepository.DecryptRepositoryConfig(ufb.FormatEncryptionKey) + if err != nil { + return nil, ErrInvalidPassword + } + + return ufb, nil +} + +// ReadAndCacheRepoUpgradeLock loads the lock config from cache and returns it. +func ReadAndCacheRepoUpgradeLock(ctx context.Context, st blob.Storage, password, cacheDir string, validDuration time.Duration) (*UpgradeLockIntent, error) { + ufb, err := ReadAndCacheDecodedRepositoryConfig(ctx, st, password, cacheDir, validDuration) + return ufb.RepoConfig.UpgradeLock, err +} diff --git a/repo/format_block_test.go b/repo/format/format_blob_test.go similarity index 99% rename from repo/format_block_test.go rename to repo/format/format_blob_test.go index 70435acac..c5b5513c1 100644 --- a/repo/format_block_test.go +++ b/repo/format/format_blob_test.go @@ -1,4 +1,4 @@ -package repo +package format import ( "crypto/sha256" diff --git a/repo/format/format_provider.go b/repo/format/format_provider.go new file mode 100644 index 000000000..401ba79a1 --- /dev/null +++ b/repo/format/format_provider.go @@ -0,0 +1,218 @@ +package format + +import ( + "time" + + "github.com/pkg/errors" + + "github.com/kopia/kopia/internal/epoch" + "github.com/kopia/kopia/internal/gather" + "github.com/kopia/kopia/repo/content/index" + "github.com/kopia/kopia/repo/encryption" + "github.com/kopia/kopia/repo/hashing" +) + +const ( + minValidPackSize = 10 << 20 + maxValidPackSize = 120 << 20 + + defaultIndexShardSize = 16e6 // slightly less than 2^24, which lets index use 24-bit/3-byte indexes + + // CurrentWriteVersion is the version of the repository applied to new repositories. + CurrentWriteVersion = FormatVersion3 + + // MinSupportedWriteVersion is the minimum version that this kopia client can write. + MinSupportedWriteVersion = FormatVersion1 + + // MaxSupportedWriteVersion is the maximum version that this kopia client can write. + MaxSupportedWriteVersion = FormatVersion3 + + // MinSupportedReadVersion is the minimum version that this kopia client can read. + MinSupportedReadVersion = FormatVersion1 + + // MaxSupportedReadVersion is the maximum version that this kopia client can read. + MaxSupportedReadVersion = FormatVersion3 + + legacyIndexVersion = index.Version1 +) + +// Version denotes content format version. +type Version int + +// Supported format versions. +const ( + FormatVersion1 Version = 1 + FormatVersion2 Version = 2 // new in v0.9 + FormatVersion3 Version = 3 // new in v0.11 + + MaxFormatVersion = FormatVersion3 +) + +// Provider provides current formatting options. The options returned +// should not be cached for more than a few seconds as they are subject to change. +type Provider interface { + epoch.ParametersProvider + + MaxIndexBlobSize() int64 + WriteIndexVersion() int + IndexShardSize() int + + encryption.Parameters + hashing.Parameters + + HashFunc() hashing.HashFunc + Encryptor() encryption.Encryptor + + GetMutableParameters() MutableParameters + GetMasterKey() []byte + SupportsPasswordChange() bool + FormatVersion() Version + MaxPackBlobSize() int + RepositoryFormatBytes() []byte + Struct() ContentFormat +} + +type formattingOptionsProvider struct { + *ContentFormat + + h hashing.HashFunc + e encryption.Encryptor + actualFormatVersion Version + actualIndexVersion int + formatBytes []byte +} + +func (f *formattingOptionsProvider) FormatVersion() Version { + return f.Version +} + +// whether epoch manager is enabled, must be true. +func (f *formattingOptionsProvider) GetEpochManagerEnabled() bool { + return f.EpochParameters.Enabled +} + +// how frequently each client will list blobs to determine the current epoch. +func (f *formattingOptionsProvider) GetEpochRefreshFrequency() time.Duration { + return f.EpochParameters.EpochRefreshFrequency +} + +// number of epochs between full checkpoints. +func (f *formattingOptionsProvider) GetEpochFullCheckpointFrequency() int { + return f.EpochParameters.FullCheckpointFrequency +} + +// GetEpochCleanupSafetyMargin returns safety margin to prevent uncompacted blobs from being deleted if the corresponding compacted blob age is less than this. +func (f *formattingOptionsProvider) GetEpochCleanupSafetyMargin() time.Duration { + return f.EpochParameters.CleanupSafetyMargin +} + +// GetMinEpochDuration returns the minimum duration of an epoch. +func (f *formattingOptionsProvider) GetMinEpochDuration() time.Duration { + return f.EpochParameters.MinEpochDuration +} + +// GetEpochAdvanceOnCountThreshold returns the number of files above which epoch should be advanced. +func (f *formattingOptionsProvider) GetEpochAdvanceOnCountThreshold() int { + return f.EpochParameters.EpochAdvanceOnCountThreshold +} + +// GetEpochAdvanceOnTotalSizeBytesThreshold returns the total size of files above which the epoch should be advanced. +func (f *formattingOptionsProvider) GetEpochAdvanceOnTotalSizeBytesThreshold() int64 { + return f.EpochParameters.EpochAdvanceOnTotalSizeBytesThreshold +} + +// GetEpochDeleteParallelism returns the number of blobs to delete in parallel during cleanup. +func (f *formattingOptionsProvider) GetEpochDeleteParallelism() int { + return f.EpochParameters.DeleteParallelism +} + +func (f *formattingOptionsProvider) Struct() ContentFormat { + return *f.ContentFormat +} + +// NewFormattingOptionsProvider validates the provided formatting options and returns static +// FormattingOptionsProvider based on them. +func NewFormattingOptionsProvider(f *ContentFormat, formatBytes []byte) (Provider, error) { + formatVersion := f.Version + + if formatVersion < MinSupportedReadVersion || formatVersion > CurrentWriteVersion { + return nil, errors.Errorf("can't handle repositories created using version %v (min supported %v, max supported %v)", formatVersion, MinSupportedReadVersion, MaxSupportedReadVersion) + } + + if formatVersion < MinSupportedWriteVersion || formatVersion > CurrentWriteVersion { + return nil, errors.Errorf("can't handle repositories created using version %v (min supported %v, max supported %v)", formatVersion, MinSupportedWriteVersion, MaxSupportedWriteVersion) + } + + actualIndexVersion := f.IndexVersion + if actualIndexVersion == 0 { + actualIndexVersion = legacyIndexVersion + } + + if actualIndexVersion < index.Version1 || actualIndexVersion > index.Version2 { + return nil, errors.Errorf("index version %v is not supported", actualIndexVersion) + } + + h, err := hashing.CreateHashFunc(f) + if err != nil { + return nil, errors.Wrap(err, "unable to create hash") + } + + e, err := encryption.CreateEncryptor(f) + if err != nil { + return nil, errors.Wrap(err, "unable to create encryptor") + } + + contentID := h(nil, gather.FromSlice(nil)) + + var tmp gather.WriteBuffer + defer tmp.Close() + + err = e.Encrypt(gather.FromSlice(nil), contentID, &tmp) + if err != nil { + return nil, errors.Wrap(err, "invalid encryptor") + } + + return &formattingOptionsProvider{ + ContentFormat: f, + + h: h, + e: e, + actualIndexVersion: actualIndexVersion, + actualFormatVersion: f.Version, + formatBytes: formatBytes, + }, nil +} + +func (f *formattingOptionsProvider) Encryptor() encryption.Encryptor { + return f.e +} + +func (f *formattingOptionsProvider) HashFunc() hashing.HashFunc { + return f.h +} + +func (f *formattingOptionsProvider) WriteIndexVersion() int { + return f.actualIndexVersion +} + +func (f *formattingOptionsProvider) MaxIndexBlobSize() int64 { + return int64(f.MaxPackSize) +} + +func (f *formattingOptionsProvider) MaxPackBlobSize() int { + return f.MaxPackSize +} + +func (f *formattingOptionsProvider) GetEpochManagerParameters() epoch.Parameters { + return f.EpochParameters +} + +func (f *formattingOptionsProvider) IndexShardSize() int { + return defaultIndexShardSize +} + +func (f *formattingOptionsProvider) RepositoryFormatBytes() []byte { + return f.formatBytes +} + +var _ Provider = (*formattingOptionsProvider)(nil) diff --git a/repo/format/object_format.go b/repo/format/object_format.go new file mode 100644 index 000000000..20124dedc --- /dev/null +++ b/repo/format/object_format.go @@ -0,0 +1,6 @@ +package format + +// ObjectFormat describes the format of objects in a repository. +type ObjectFormat struct { + Splitter string `json:"splitter,omitempty"` // splitter used to break objects into pieces of content +} diff --git a/repo/format/repository_config.go b/repo/format/repository_config.go new file mode 100644 index 000000000..134c4ecdf --- /dev/null +++ b/repo/format/repository_config.go @@ -0,0 +1,78 @@ +package format + +import ( + "encoding/json" + "time" + + "github.com/pkg/errors" + + "github.com/kopia/kopia/internal/feature" +) + +// RepositoryConfig describes the format of objects in a repository. +// The contents of this object are stored encrypted since they contain sensitive key material. +type RepositoryConfig struct { + ContentFormat + ObjectFormat + + UpgradeLock *UpgradeLockIntent `json:"upgradeLock,omitempty"` + RequiredFeatures []feature.Required `json:"requiredFeatures,omitempty"` +} + +// EncryptedRepositoryConfig contains the configuration of repository that's persisted in encrypted format. +type EncryptedRepositoryConfig struct { + Format RepositoryConfig `json:"format"` +} + +// DecodedRepositoryConfig encapsulates contents of decoded `kopia.repository` blob. +type DecodedRepositoryConfig struct { + KopiaRepository *KopiaRepositoryJSON + KopiaRepositoryBytes []byte // serialized format blob + CacheMTime time.Time // mod time of the format blob cache file + RepoConfig *RepositoryConfig // unencrypted format blob structure + FormatEncryptionKey []byte // key derived from the password +} + +// DecryptRepositoryConfig decrypts RepositoryConfig stored in EncryptedFormatBytes. +func (f *KopiaRepositoryJSON) DecryptRepositoryConfig(masterKey []byte) (*RepositoryConfig, error) { + switch f.EncryptionAlgorithm { + case aes256GcmEncryption: + plainText, err := decryptRepositoryBlobBytesAes256Gcm(f.EncryptedFormatBytes, masterKey, f.UniqueID) + if err != nil { + return nil, errors.Errorf("unable to decrypt repository format") + } + + var erc EncryptedRepositoryConfig + if err := json.Unmarshal(plainText, &erc); err != nil { + return nil, errors.Wrap(err, "invalid repository format") + } + + return &erc.Format, nil + + default: + return nil, errors.Errorf("unknown encryption algorithm: '%v'", f.EncryptionAlgorithm) + } +} + +// EncryptRepositoryConfig encrypts the provided repository config and stores it in EncryptedFormatBytes. +func (f *KopiaRepositoryJSON) EncryptRepositoryConfig(format *RepositoryConfig, masterKey []byte) error { + switch f.EncryptionAlgorithm { + case aes256GcmEncryption: + data, err := json.Marshal(&EncryptedRepositoryConfig{Format: *format}) + if err != nil { + return errors.Wrap(err, "can't marshal format to JSON") + } + + data, err = encryptRepositoryBlobBytesAes256Gcm(data, masterKey, f.UniqueID) + if err != nil { + return errors.Wrap(err, "failed to encrypt format JSON") + } + + f.EncryptedFormatBytes = data + + return nil + + default: + return errors.Errorf("unknown encryption algorithm: '%v'", f.EncryptionAlgorithm) + } +} diff --git a/repo/upgrade_lock_intent.go b/repo/format/upgrade_lock_intent.go similarity index 99% rename from repo/upgrade_lock_intent.go rename to repo/format/upgrade_lock_intent.go index d21466e4b..b75e4ba34 100644 --- a/repo/upgrade_lock_intent.go +++ b/repo/format/upgrade_lock_intent.go @@ -1,4 +1,4 @@ -package repo +package format import ( "time" diff --git a/repo/upgrade_lock_intent_test.go b/repo/format/upgrade_lock_intent_test.go similarity index 96% rename from repo/upgrade_lock_intent_test.go rename to repo/format/upgrade_lock_intent_test.go index 46ac685f2..ad6cbcc78 100644 --- a/repo/upgrade_lock_intent_test.go +++ b/repo/format/upgrade_lock_intent_test.go @@ -1,4 +1,4 @@ -package repo_test +package format_test import ( "fmt" @@ -8,11 +8,11 @@ "github.com/stretchr/testify/require" "github.com/kopia/kopia/internal/clock" - "github.com/kopia/kopia/repo" + "github.com/kopia/kopia/repo/format" ) func TestUpgradeLockIntentUpdatesWithAdvanceNotice(t *testing.T) { - oldLock := repo.UpgradeLockIntent{ + oldLock := format.UpgradeLockIntent{ OwnerID: "upgrade-owner", CreationTime: clock.Now(), AdvanceNoticeDuration: time.Hour, @@ -57,7 +57,7 @@ func TestUpgradeLockIntentUpdatesWithAdvanceNotice(t *testing.T) { } func TestUpgradeLockIntentUpdatesWithoutAdvanceNotice(t *testing.T) { - oldLock := repo.UpgradeLockIntent{ + oldLock := format.UpgradeLockIntent{ OwnerID: "upgrade-owner", CreationTime: clock.Now(), AdvanceNoticeDuration: 0, /* no advance notice */ @@ -77,7 +77,7 @@ func TestUpgradeLockIntentUpdatesWithoutAdvanceNotice(t *testing.T) { } func TestUpgradeLockIntentValidation(t *testing.T) { - var l repo.UpgradeLockIntent + var l format.UpgradeLockIntent require.EqualError(t, l.Validate(), "no owner-id set, it is required to set a unique owner-id") l.OwnerID = "new-owner" @@ -116,7 +116,7 @@ func TestUpgradeLockIntentValidation(t *testing.T) { func TestUpgradeLockIntentImmediateLock(t *testing.T) { now := clock.Now() - var l *repo.UpgradeLockIntent + var l *format.UpgradeLockIntent // checking lock status on nil lock locked, writersDrained := l.IsLocked(now) @@ -127,7 +127,7 @@ func TestUpgradeLockIntentImmediateLock(t *testing.T) { require.PanicsWithValue(t, "writers have drained but we are not locked, this is not possible until the upgrade-lock intent is invalid", func() { - tmp := repo.UpgradeLockIntent{ + tmp := format.UpgradeLockIntent{ OwnerID: "", CreationTime: now, AdvanceNoticeDuration: 1 * time.Hour, @@ -139,7 +139,7 @@ func() { tmp.IsLocked(now.Add(2 * time.Hour)) }) - l = &repo.UpgradeLockIntent{ + l = &format.UpgradeLockIntent{ OwnerID: "upgrade-owner", CreationTime: now, AdvanceNoticeDuration: 0, /* no advance notice */ @@ -182,7 +182,7 @@ func() { func TestUpgradeLockIntentSufficientAdvanceLock(t *testing.T) { now := clock.Now() - l := repo.UpgradeLockIntent{ + l := format.UpgradeLockIntent{ OwnerID: "upgrade-owner", CreationTime: now, AdvanceNoticeDuration: 6 * time.Hour, @@ -249,7 +249,7 @@ func TestUpgradeLockIntentSufficientAdvanceLock(t *testing.T) { func TestUpgradeLockIntentInSufficientAdvanceLock(t *testing.T) { now := clock.Now() - l := repo.UpgradeLockIntent{ + l := format.UpgradeLockIntent{ OwnerID: "upgrade-owner", CreationTime: now, AdvanceNoticeDuration: 20 * time.Minute, /* insufficient time to drain the writers */ @@ -288,12 +288,12 @@ func TestUpgradeLockIntentInSufficientAdvanceLock(t *testing.T) { func TestUpgradeLockIntentUpgradeTime(t *testing.T) { now := clock.Now() - var l repo.UpgradeLockIntent + var l format.UpgradeLockIntent // checking time on nil lock require.Equal(t, time.Time{}, l.UpgradeTime()) - l = repo.UpgradeLockIntent{ + l = format.UpgradeLockIntent{ OwnerID: "upgrade-owner", CreationTime: now, AdvanceNoticeDuration: 20 * time.Minute, /* insufficient time to drain the writers */ @@ -304,7 +304,7 @@ func TestUpgradeLockIntentUpgradeTime(t *testing.T) { } require.Equal(t, now.Add(l.MaxPermittedClockDrift+2*l.IODrainTimeout), l.UpgradeTime()) - l = repo.UpgradeLockIntent{ + l = format.UpgradeLockIntent{ OwnerID: "upgrade-owner", CreationTime: now, AdvanceNoticeDuration: 20 * time.Hour, /* sufficient time to drain the writers */ @@ -315,7 +315,7 @@ func TestUpgradeLockIntentUpgradeTime(t *testing.T) { } require.Equal(t, now.Add(l.AdvanceNoticeDuration), l.UpgradeTime()) - l = repo.UpgradeLockIntent{ + l = format.UpgradeLockIntent{ OwnerID: "upgrade-owner", CreationTime: now, AdvanceNoticeDuration: 0, /* immediate lock */ @@ -328,7 +328,7 @@ func TestUpgradeLockIntentUpgradeTime(t *testing.T) { } func TestUpgradeLockIntentClone(t *testing.T) { - l := &repo.UpgradeLockIntent{ + l := &format.UpgradeLockIntent{ OwnerID: "upgrade-owner", CreationTime: clock.Now(), AdvanceNoticeDuration: 20 * time.Minute, diff --git a/repo/grpc_repository_client.go b/repo/grpc_repository_client.go index 78441b55a..798871fb3 100644 --- a/repo/grpc_repository_client.go +++ b/repo/grpc_repository_client.go @@ -25,6 +25,7 @@ "github.com/kopia/kopia/repo/blob" "github.com/kopia/kopia/repo/compression" "github.com/kopia/kopia/repo/content" + "github.com/kopia/kopia/repo/format" "github.com/kopia/kopia/repo/hashing" "github.com/kopia/kopia/repo/manifest" "github.com/kopia/kopia/repo/object" @@ -74,7 +75,7 @@ type grpcRepositoryClient struct { asyncWritesWG errgroup.Group h hashing.HashFunc - objectFormat object.Format + objectFormat format.ObjectFormat serverSupportsContentCompression bool cliOpts ClientOptions omgr *object.Manager @@ -906,7 +907,7 @@ func newGRPCAPIRepositoryForConnection(ctx context.Context, conn *grpc.ClientCon rr.h = hf - rr.objectFormat = object.Format{ + rr.objectFormat = format.ObjectFormat{ Splitter: p.Splitter, } diff --git a/repo/initialize.go b/repo/initialize.go index 9c6855063..b579b1285 100644 --- a/repo/initialize.go +++ b/repo/initialize.go @@ -13,8 +13,8 @@ "github.com/kopia/kopia/repo/blob" "github.com/kopia/kopia/repo/content" "github.com/kopia/kopia/repo/encryption" + "github.com/kopia/kopia/repo/format" "github.com/kopia/kopia/repo/hashing" - "github.com/kopia/kopia/repo/object" "github.com/kopia/kopia/repo/splitter" ) @@ -35,12 +35,12 @@ // NewRepositoryOptions specifies options that apply to newly created repositories. // All fields are optional, when not provided, reasonable defaults will be used. type NewRepositoryOptions struct { - UniqueID []byte `json:"uniqueID"` // force the use of particular unique ID - BlockFormat content.FormattingOptions `json:"blockFormat"` - DisableHMAC bool `json:"disableHMAC"` - ObjectFormat object.Format `json:"objectFormat"` // object format - RetentionMode blob.RetentionMode `json:"retentionMode,omitempty"` - RetentionPeriod time.Duration `json:"retentionPeriod,omitempty"` + UniqueID []byte `json:"uniqueID"` // force the use of particular unique ID + BlockFormat format.ContentFormat `json:"blockFormat"` + DisableHMAC bool `json:"disableHMAC"` + ObjectFormat format.ObjectFormat `json:"objectFormat"` // object format + RetentionMode blob.RetentionMode `json:"retentionMode,omitempty"` + RetentionPeriod time.Duration `json:"retentionPeriod,omitempty"` } // ErrAlreadyInitialized indicates that repository has already been initialized. @@ -56,7 +56,7 @@ func Initialize(ctx context.Context, st blob.Storage, opt *NewRepositoryOptions, var tmp gather.WriteBuffer defer tmp.Close() - err := st.GetBlob(ctx, FormatBlobID, 0, -1, &tmp) + err := st.GetBlob(ctx, format.KopiaRepositoryBlobID, 0, -1, &tmp) if err == nil { return ErrAlreadyInitialized } @@ -65,7 +65,7 @@ func Initialize(ctx context.Context, st blob.Storage, opt *NewRepositoryOptions, return errors.Wrap(err, "unexpected error when checking for format blob") } - err = st.GetBlob(ctx, BlobCfgBlobID, 0, -1, &tmp) + err = st.GetBlob(ctx, format.KopiaBlobCfgBlobID, 0, -1, &tmp) if err == nil { return errors.Errorf("possible corruption: blobcfg blob exists, but format blob is not found") } @@ -74,10 +74,10 @@ func Initialize(ctx context.Context, st blob.Storage, opt *NewRepositoryOptions, return errors.Wrap(err, "unexpected error when checking for blobcfg blob") } - format := formatBlobFromOptions(opt) + formatBlob := formatBlobFromOptions(opt) blobcfg := blobCfgBlobFromOptions(opt) - formatEncryptionKey, err := format.deriveFormatEncryptionKeyFromPassword(password) + formatEncryptionKey, err := formatBlob.DeriveFormatEncryptionKeyFromPassword(password) if err != nil { return errors.Wrap(err, "unable to derive format encryption key") } @@ -91,54 +91,61 @@ func Initialize(ctx context.Context, st blob.Storage, opt *NewRepositoryOptions, return errors.Wrap(err, "invalid parameters") } - if err = encryptFormatBytes(format, f, formatEncryptionKey, format.UniqueID); err != nil { + if err = formatBlob.EncryptRepositoryConfig(f, formatEncryptionKey); err != nil { return errors.Wrap(err, "unable to encrypt format bytes") } - if err := writeBlobCfgBlob(ctx, st, format, blobcfg, formatEncryptionKey); err != nil { + if err := formatBlob.WriteBlobCfgBlob(ctx, st, blobcfg, formatEncryptionKey); err != nil { return errors.Wrap(err, "unable to write blobcfg blob") } - if err := writeFormatBlob(ctx, st, format, blobcfg); err != nil { + if err := formatBlob.WriteKopiaRepositoryBlob(ctx, st, blobcfg); err != nil { return errors.Wrap(err, "unable to write format blob") } return nil } -func formatBlobFromOptions(opt *NewRepositoryOptions) *formatBlob { - return &formatBlob{ +func formatBlobFromOptions(opt *NewRepositoryOptions) *format.KopiaRepositoryJSON { + return &format.KopiaRepositoryJSON{ Tool: "https://github.com/kopia/kopia", BuildInfo: BuildInfo, BuildVersion: BuildVersion, - KeyDerivationAlgorithm: defaultKeyDerivationAlgorithm, + KeyDerivationAlgorithm: format.DefaultKeyDerivationAlgorithm, UniqueID: applyDefaultRandomBytes(opt.UniqueID, uniqueIDLength), - EncryptionAlgorithm: defaultFormatEncryption, + EncryptionAlgorithm: format.DefaultFormatEncryption, } } -func repositoryObjectFormatFromOptions(opt *NewRepositoryOptions) (*repositoryObjectFormat, error) { +func blobCfgBlobFromOptions(opt *NewRepositoryOptions) format.BlobStorageConfiguration { + return format.BlobStorageConfiguration{ + RetentionMode: opt.RetentionMode, + RetentionPeriod: opt.RetentionPeriod, + } +} + +func repositoryObjectFormatFromOptions(opt *NewRepositoryOptions) (*format.RepositoryConfig, error) { fv := opt.BlockFormat.Version if fv == 0 { switch os.Getenv("KOPIA_REPOSITORY_FORMAT_VERSION") { case "1": - fv = content.FormatVersion1 + fv = format.FormatVersion1 case "2": - fv = content.FormatVersion2 + fv = format.FormatVersion2 case "3": - fv = content.FormatVersion3 + fv = format.FormatVersion3 default: - fv = content.FormatVersion3 + fv = format.FormatVersion3 } } - f := &repositoryObjectFormat{ - FormattingOptions: content.FormattingOptions{ + f := &format.RepositoryConfig{ + ContentFormat: format.ContentFormat{ Hash: applyDefaultString(opt.BlockFormat.Hash, hashing.DefaultAlgorithm), Encryption: applyDefaultString(opt.BlockFormat.Encryption, encryption.DefaultAlgorithm), HMACSecret: applyDefaultRandomBytes(opt.BlockFormat.HMACSecret, hmacSecretLength), MasterKey: applyDefaultRandomBytes(opt.BlockFormat.MasterKey, masterKeyLength), - MutableParameters: content.MutableParameters{ + MutableParameters: format.MutableParameters{ Version: fv, MaxPackSize: applyDefaultInt(opt.BlockFormat.MaxPackSize, 20<<20), //nolint:gomnd IndexVersion: applyDefaultInt(opt.BlockFormat.IndexVersion, content.DefaultIndexVersion), @@ -146,7 +153,7 @@ func repositoryObjectFormatFromOptions(opt *NewRepositoryOptions) (*repositoryOb }, EnablePasswordChange: opt.BlockFormat.EnablePasswordChange, }, - Format: object.Format{ + ObjectFormat: format.ObjectFormat{ Splitter: applyDefaultString(opt.ObjectFormat.Splitter, splitter.DefaultAlgorithm), }, } @@ -155,7 +162,7 @@ func repositoryObjectFormatFromOptions(opt *NewRepositoryOptions) (*repositoryOb f.HMACSecret = nil } - if err := f.FormattingOptions.ResolveFormatVersion(); err != nil { + if err := f.ContentFormat.ResolveFormatVersion(); err != nil { return nil, errors.Wrap(err, "error resolving format version") } diff --git a/repo/local_config.go b/repo/local_config.go index 672305d7a..4efcc1ef7 100644 --- a/repo/local_config.go +++ b/repo/local_config.go @@ -11,12 +11,11 @@ "github.com/pkg/errors" "github.com/kopia/kopia/internal/atomicfile" - "github.com/kopia/kopia/internal/feature" "github.com/kopia/kopia/internal/ospath" "github.com/kopia/kopia/repo/blob" "github.com/kopia/kopia/repo/blob/throttling" "github.com/kopia/kopia/repo/content" - "github.com/kopia/kopia/repo/object" + "github.com/kopia/kopia/repo/format" ) const configDirMode = 0o700 @@ -53,7 +52,7 @@ func (o ClientOptions) ApplyDefaults(ctx context.Context, defaultDesc string) Cl } if o.FormatBlobCacheDuration == 0 { - o.FormatBlobCacheDuration = DefaultRepositoryBlobCacheDuration + o.FormatBlobCacheDuration = format.DefaultRepositoryBlobCacheDuration } return o @@ -98,15 +97,6 @@ type LocalConfig struct { ClientOptions } -// repositoryObjectFormat describes the format of objects in a repository. -type repositoryObjectFormat struct { - content.FormattingOptions - object.Format - - UpgradeLock *UpgradeLockIntent `json:"upgradeLock,omitempty"` - RequiredFeatures []feature.Required `json:"requiredFeatures,omitempty"` -} - // writeToFile writes the config to a given file. func (lc *LocalConfig) writeToFile(filename string) error { lc2 := *lc diff --git a/repo/maintenance/blob_gc_test.go b/repo/maintenance/blob_gc_test.go index fbc7a41a7..092b69679 100644 --- a/repo/maintenance/blob_gc_test.go +++ b/repo/maintenance/blob_gc_test.go @@ -22,6 +22,7 @@ "github.com/kopia/kopia/repo/blob" "github.com/kopia/kopia/repo/content" "github.com/kopia/kopia/repo/encryption" + "github.com/kopia/kopia/repo/format" "github.com/kopia/kopia/repo/maintenance" "github.com/kopia/kopia/repo/object" ) @@ -200,7 +201,7 @@ func mustPutDummySessionBlob(t *testing.T, st blob.Storage, sessionIDSuffix blob blobID := blob.ID(fmt.Sprintf("s%x-%v", iv, sessionIDSuffix)) - e, err := encryption.CreateEncryptor(&content.FormattingOptions{ + e, err := encryption.CreateEncryptor(&format.ContentFormat{ Encryption: encryption.DefaultAlgorithm, MasterKey: testMasterKey, HMACSecret: testHMACSecret, diff --git a/repo/maintenance/suite_test.go b/repo/maintenance/suite_test.go index 17e21f11e..99a653f3d 100644 --- a/repo/maintenance/suite_test.go +++ b/repo/maintenance/suite_test.go @@ -4,21 +4,21 @@ "testing" "github.com/kopia/kopia/internal/testutil" - "github.com/kopia/kopia/repo/content" + "github.com/kopia/kopia/repo/format" ) type formatSpecificTestSuite struct { - formatVersion content.FormatVersion + formatVersion format.Version } func TestFormatV1(t *testing.T) { - testutil.RunAllTestsWithParam(t, &formatSpecificTestSuite{content.FormatVersion1}) + testutil.RunAllTestsWithParam(t, &formatSpecificTestSuite{format.FormatVersion1}) } func TestFormatV2(t *testing.T) { - testutil.RunAllTestsWithParam(t, &formatSpecificTestSuite{content.FormatVersion2}) + testutil.RunAllTestsWithParam(t, &formatSpecificTestSuite{format.FormatVersion2}) } func TestFormatV3(t *testing.T) { - testutil.RunAllTestsWithParam(t, &formatSpecificTestSuite{content.FormatVersion3}) + testutil.RunAllTestsWithParam(t, &formatSpecificTestSuite{format.FormatVersion3}) } diff --git a/repo/manifest/manifest_manager_test.go b/repo/manifest/manifest_manager_test.go index 4ee851e17..73afc4dd5 100644 --- a/repo/manifest/manifest_manager_test.go +++ b/repo/manifest/manifest_manager_test.go @@ -18,6 +18,7 @@ "github.com/kopia/kopia/repo/content" "github.com/kopia/kopia/repo/content/index" "github.com/kopia/kopia/repo/encryption" + "github.com/kopia/kopia/repo/format" "github.com/kopia/kopia/repo/hashing" ) @@ -142,10 +143,10 @@ func TestManifestInitCorruptedBlock(t *testing.T) { data := blobtesting.DataMap{} st := blobtesting.NewMapStorage(data, nil, nil) - fop, err := content.NewFormattingOptionsProvider(&content.FormattingOptions{ + fop, err := format.NewFormattingOptionsProvider(&format.ContentFormat{ Hash: hashing.DefaultAlgorithm, Encryption: encryption.DefaultAlgorithm, - MutableParameters: content.MutableParameters{ + MutableParameters: format.MutableParameters{ Version: 1, MaxPackSize: 100000, }, @@ -302,10 +303,10 @@ func newManagerForTesting(ctx context.Context, t *testing.T, data blobtesting.Da st := blobtesting.NewMapStorage(data, nil, nil) - fop, err := content.NewFormattingOptionsProvider(&content.FormattingOptions{ + fop, err := format.NewFormattingOptionsProvider(&format.ContentFormat{ Hash: hashing.DefaultAlgorithm, Encryption: encryption.DefaultAlgorithm, - MutableParameters: content.MutableParameters{ + MutableParameters: format.MutableParameters{ Version: 1, MaxPackSize: 100000, }, diff --git a/repo/object/object_manager.go b/repo/object/object_manager.go index c8f6180a6..ed9519b10 100644 --- a/repo/object/object_manager.go +++ b/repo/object/object_manager.go @@ -11,6 +11,7 @@ "github.com/kopia/kopia/internal/gather" "github.com/kopia/kopia/repo/compression" "github.com/kopia/kopia/repo/content" + "github.com/kopia/kopia/repo/format" "github.com/kopia/kopia/repo/splitter" ) @@ -37,14 +38,9 @@ type contentManager interface { WriteContent(ctx context.Context, data gather.Bytes, prefix content.IDPrefix, comp compression.HeaderID) (content.ID, error) } -// Format describes the format of objects in a repository. -type Format struct { - Splitter string `json:"splitter,omitempty"` // splitter used to break objects into pieces of content -} - // Manager implements a content-addressable storage on top of blob storage. type Manager struct { - Format Format + Format format.ObjectFormat contentMgr contentManager newSplitter splitter.Factory @@ -201,7 +197,7 @@ func PrefetchBackingContents(ctx context.Context, contentMgr contentManager, obj } // NewObjectManager creates an ObjectManager with the specified content manager and format. -func NewObjectManager(ctx context.Context, bm contentManager, f Format) (*Manager, error) { +func NewObjectManager(ctx context.Context, bm contentManager, f format.ObjectFormat) (*Manager, error) { om := &Manager{ contentMgr: bm, Format: f, diff --git a/repo/object/object_manager_test.go b/repo/object/object_manager_test.go index 883f0faf2..770a3ac82 100644 --- a/repo/object/object_manager_test.go +++ b/repo/object/object_manager_test.go @@ -26,6 +26,7 @@ "github.com/kopia/kopia/repo/blob" "github.com/kopia/kopia/repo/compression" "github.com/kopia/kopia/repo/content" + "github.com/kopia/kopia/repo/format" "github.com/kopia/kopia/repo/splitter" ) @@ -109,7 +110,7 @@ func setupTest(t *testing.T, compressionHeaderID map[content.ID]compression.Head compresionIDs: compressionHeaderID, } - r, err := NewObjectManager(testlogging.Context(t), fcm, Format{ + r, err := NewObjectManager(testlogging.Context(t), fcm, format.ObjectFormat{ Splitter: "FIXED-1M", }) if err != nil { @@ -280,7 +281,7 @@ func TestObjectWriterRaceBetweenCheckpointAndResult(t *testing.T) { data: data, } - om, err := NewObjectManager(testlogging.Context(t), fcm, Format{ + om, err := NewObjectManager(testlogging.Context(t), fcm, format.ObjectFormat{ Splitter: "FIXED-1M", }) if err != nil { diff --git a/repo/open.go b/repo/open.go index c1514c13a..010f15bb2 100644 --- a/repo/open.go +++ b/repo/open.go @@ -12,12 +12,9 @@ "github.com/pkg/errors" "golang.org/x/crypto/scrypt" - "github.com/kopia/kopia/internal/atomicfile" "github.com/kopia/kopia/internal/cache" - "github.com/kopia/kopia/internal/clock" "github.com/kopia/kopia/internal/epoch" "github.com/kopia/kopia/internal/feature" - "github.com/kopia/kopia/internal/gather" "github.com/kopia/kopia/internal/retry" "github.com/kopia/kopia/repo/blob" "github.com/kopia/kopia/repo/blob/beforeop" @@ -25,6 +22,7 @@ "github.com/kopia/kopia/repo/blob/readonly" "github.com/kopia/kopia/repo/blob/throttling" "github.com/kopia/kopia/repo/content" + "github.com/kopia/kopia/repo/format" "github.com/kopia/kopia/repo/logging" "github.com/kopia/kopia/repo/manifest" "github.com/kopia/kopia/repo/object" @@ -46,17 +44,6 @@ "index-v2", } -// CacheDirMarkerFile is the name of the marker file indicating a directory contains Kopia caches. -// See https://bford.info/cachedir/ -const CacheDirMarkerFile = "CACHEDIR.TAG" - -// CacheDirMarkerHeader is the header signature for cache dir marker files. -const CacheDirMarkerHeader = "Signature: 8a477f597d28d172789f06886806bc55" - -// DefaultRepositoryBlobCacheDuration is the duration for which we treat cached kopia.repository -// as valid. -const DefaultRepositoryBlobCacheDuration = 15 * time.Minute - // throttlingWindow is the duration window during which the throttling token bucket fully replenishes. // the maximum number of tokens in the bucket is multiplied by the number of seconds. const throttlingWindow = 60 * time.Second @@ -70,17 +57,6 @@ // nolint:gochecknoglobals var localCacheIntegrityPurpose = []byte("local-cache-integrity") -const cacheDirMarkerContents = CacheDirMarkerHeader + ` -# -# This file is a cache directory tag created by Kopia - Fast And Secure Open-Source Backup. -# -# For information about Kopia, see: -# https://kopia.io -# -# For information about cache directory tags, see: -# http://www.brynosaurus.com/cachedir/ -` - var log = logging.Module("kopia/repo") // Options provides configuration parameters for connection to a repository. @@ -98,7 +74,7 @@ type Options struct { } // ErrInvalidPassword is returned when repository password is invalid. -var ErrInvalidPassword = errors.Errorf("invalid repository password") +var ErrInvalidPassword = format.ErrInvalidPassword // ErrRepositoryUnavailableDueToUpgrageInProgress is returned when repository // is undergoing upgrade that requires exclusive access. @@ -219,56 +195,6 @@ func openDirect(ctx context.Context, configFile string, lc *LocalConfig, passwor return r, nil } -type unpackedFormatBlob struct { - f *formatBlob - fb []byte // serialized format blob - cacheMTime time.Time // mod time of the format blob cache file - repoConfig *repositoryObjectFormat // unencrypted format blob structure - formatEncryptionKey []byte // key derived from the password -} - -func readAndCacheRepoConfig(ctx context.Context, st blob.Storage, password string, cacheOpts *content.CachingOptions, validDuration time.Duration) (ufb *unpackedFormatBlob, err error) { - ufb = &unpackedFormatBlob{} - - // Read format blob, potentially from cache. - ufb.fb, ufb.cacheMTime, err = readAndCacheRepositoryBlobBytes(ctx, st, cacheOpts.CacheDirectory, FormatBlobID, validDuration) - if err != nil { - return nil, errors.Wrap(err, "unable to read format blob") - } - - if err = writeCacheMarker(cacheOpts.CacheDirectory); err != nil { - return nil, errors.Wrap(err, "unable to write cache directory marker") - } - - ufb.f, err = parseFormatBlob(ufb.fb) - if err != nil { - return nil, errors.Wrap(err, "can't parse format blob") - } - - ufb.fb, err = addFormatBlobChecksumAndLength(ufb.fb) - if err != nil { - return nil, errors.Errorf("unable to add checksum") - } - - ufb.formatEncryptionKey, err = ufb.f.deriveFormatEncryptionKeyFromPassword(password) - if err != nil { - return nil, err - } - - ufb.repoConfig, err = ufb.f.decryptFormatBytes(ufb.formatEncryptionKey) - if err != nil { - return nil, ErrInvalidPassword - } - - return ufb, nil -} - -// ReadAndCacheRepoUpgradeLock loads the lock config from cache and returns it. -func ReadAndCacheRepoUpgradeLock(ctx context.Context, st blob.Storage, password string, cacheOpts *content.CachingOptions, validDuration time.Duration) (*UpgradeLockIntent, error) { - ufb, err := readAndCacheRepoConfig(ctx, st, password, cacheOpts, validDuration) - return ufb.repoConfig.UpgradeLock, err -} - // openWithConfig opens the repository with a given configuration, avoiding the need for a config file. // nolint:funlen,gocyclo,cyclop func openWithConfig(ctx context.Context, st blob.Storage, lc *LocalConfig, password string, options *Options, cacheOpts *content.CachingOptions, configFile string) (DirectRepository, error) { @@ -278,18 +204,19 @@ func openWithConfig(ctx context.Context, st blob.Storage, lc *LocalConfig, passw DisableInternalLog: options.DisableInternalLog, } - var ufb *unpackedFormatBlob + var ufb *format.DecodedRepositoryConfig if _, err := retry.WithExponentialBackoffMaxRetries(ctx, -1, "read repo config and wait for upgrade", func() (interface{}, error) { var internalErr error - ufb, internalErr = readAndCacheRepoConfig(ctx, st, password, cacheOpts, + ufb, internalErr = format.ReadAndCacheDecodedRepositoryConfig(ctx, st, password, cacheOpts.CacheDirectory, lc.FormatBlobCacheDuration) if internalErr != nil { + // nolint:wrapcheck return nil, internalErr } // retry if upgrade lock has been taken - if locked, _ := ufb.repoConfig.UpgradeLock.IsLocked(cmOpts.TimeNow()); locked && options.UpgradeOwnerID != ufb.repoConfig.UpgradeLock.OwnerID { + if locked, _ := ufb.RepoConfig.UpgradeLock.IsLocked(cmOpts.TimeNow()); locked && options.UpgradeOwnerID != ufb.RepoConfig.UpgradeLock.OwnerID { return nil, ErrRepositoryUnavailableDueToUpgrageInProgress } @@ -301,31 +228,31 @@ func openWithConfig(ctx context.Context, st blob.Storage, lc *LocalConfig, passw return nil, err } - if err := handleMissingRequiredFeatures(ctx, ufb.repoConfig, options.TestOnlyIgnoreMissingRequiredFeatures); err != nil { + if err := handleMissingRequiredFeatures(ctx, ufb.RepoConfig, options.TestOnlyIgnoreMissingRequiredFeatures); err != nil { return nil, err } - cmOpts.RepositoryFormatBytes = ufb.fb + cmOpts.RepositoryFormatBytes = ufb.KopiaRepositoryBytes // Read blobcfg blob, potentially from cache. - bb, _, err := readAndCacheRepositoryBlobBytes(ctx, st, cacheOpts.CacheDirectory, BlobCfgBlobID, lc.FormatBlobCacheDuration) + bb, _, err := format.ReadAndCacheRepositoryBlobBytes(ctx, st, cacheOpts.CacheDirectory, format.KopiaBlobCfgBlobID, lc.FormatBlobCacheDuration) if err != nil && !errors.Is(err, blob.ErrBlobNotFound) { return nil, errors.Wrap(err, "unable to read blobcfg blob") } - blobcfg, err := deserializeBlobCfgBytes(ufb.f, bb, ufb.formatEncryptionKey) + blobcfg, err := ufb.KopiaRepository.DeserializeBlobCfgBytes(bb, ufb.FormatEncryptionKey) if err != nil { return nil, ErrInvalidPassword } - if ufb.repoConfig.FormattingOptions.EnablePasswordChange { - cacheOpts.HMACSecret = deriveKeyFromMasterKey(ufb.repoConfig.HMACSecret, ufb.f.UniqueID, localCacheIntegrityPurpose, localCacheIntegrityHMACSecretLength) + if ufb.RepoConfig.ContentFormat.EnablePasswordChange { + cacheOpts.HMACSecret = format.DeriveKeyFromMasterKey(ufb.RepoConfig.HMACSecret, ufb.KopiaRepository.UniqueID, localCacheIntegrityPurpose, localCacheIntegrityHMACSecretLength) } else { - // deriving from ufb.formatEncryptionKey was actually a bug, that only matters will change when we change the password - cacheOpts.HMACSecret = deriveKeyFromMasterKey(ufb.formatEncryptionKey, ufb.f.UniqueID, localCacheIntegrityPurpose, localCacheIntegrityHMACSecretLength) + // deriving from ufb.FormatEncryptionKey was actually a bug, that only matters will change when we change the password + cacheOpts.HMACSecret = format.DeriveKeyFromMasterKey(ufb.FormatEncryptionKey, ufb.KopiaRepository.UniqueID, localCacheIntegrityPurpose, localCacheIntegrityHMACSecretLength) } - fo := &ufb.repoConfig.FormattingOptions + fo := &ufb.RepoConfig.ContentFormat if fo.MaxPackSize == 0 { // legacy only, apply default @@ -364,9 +291,9 @@ func openWithConfig(ctx context.Context, st blob.Storage, lc *LocalConfig, passw // background/interleaving upgrade lock storage monitor st = upgradeLockMonitor(options.UpgradeOwnerID, st, password, cacheOpts, lc.FormatBlobCacheDuration, - ufb.cacheMTime, cmOpts.TimeNow, options.OnFatalError, options.TestOnlyIgnoreMissingRequiredFeatures) + ufb.CacheMTime, cmOpts.TimeNow, options.OnFatalError, options.TestOnlyIgnoreMissingRequiredFeatures) - fop, err := content.NewFormattingOptionsProvider(fo, cmOpts.RepositoryFormatBytes) + fop, err := format.NewFormattingOptionsProvider(fo, cmOpts.RepositoryFormatBytes) if err != nil { return nil, errors.Wrap(err, "unable to create format options provider") } @@ -381,7 +308,7 @@ func openWithConfig(ctx context.Context, st blob.Storage, lc *LocalConfig, passw SessionHost: lc.Hostname, }, "") - om, err := object.NewObjectManager(ctx, cm, ufb.repoConfig.Format) + om, err := object.NewObjectManager(ctx, cm, ufb.RepoConfig.ObjectFormat) if err != nil { return nil, errors.Wrap(err, "unable to open object manager") } @@ -398,11 +325,11 @@ func openWithConfig(ctx context.Context, st blob.Storage, lc *LocalConfig, passw mmgr: manifests, sm: scm, directRepositoryParameters: directRepositoryParameters{ - uniqueID: ufb.f.UniqueID, + uniqueID: ufb.KopiaRepository.UniqueID, cachingOptions: *cacheOpts, - formatBlob: ufb.f, + formatBlob: ufb.KopiaRepository, blobCfgBlob: blobcfg, - formatEncryptionKey: ufb.formatEncryptionKey, + formatEncryptionKey: ufb.FormatEncryptionKey, timeNow: cmOpts.TimeNow, cliOpts: lc.ClientOptions.ApplyDefaults(ctx, "Repository in "+st.DisplayName()), configFile: configFile, @@ -415,7 +342,7 @@ func openWithConfig(ctx context.Context, st blob.Storage, lc *LocalConfig, passw return dr, nil } -func handleMissingRequiredFeatures(ctx context.Context, repoConfig *repositoryObjectFormat, ignoreErrors bool) error { +func handleMissingRequiredFeatures(ctx context.Context, repoConfig *format.RepositoryConfig, ignoreErrors bool) error { // See if the current version of Kopia supports all features required by the repository format. // so we can safely fail to start in case repository has been upgraded to a new, incompatible version. if missingFeatures := feature.GetUnsupportedFeatures(repoConfig.RequiredFeatures, supportedFeatures); len(missingFeatures) > 0 { @@ -432,15 +359,15 @@ func handleMissingRequiredFeatures(ctx context.Context, repoConfig *repositoryOb return nil } -func wrapLockingStorage(st blob.Storage, r content.BlobCfgBlob) blob.Storage { +func wrapLockingStorage(st blob.Storage, r format.BlobStorageConfiguration) blob.Storage { // collect prefixes that need to be locked on put var prefixes []string for _, prefix := range content.PackBlobIDPrefixes { prefixes = append(prefixes, string(prefix)) } - prefixes = append(prefixes, content.LegacyIndexBlobPrefix, epoch.EpochManagerIndexUberPrefix, FormatBlobID, - BlobCfgBlobID) + prefixes = append(prefixes, content.LegacyIndexBlobPrefix, epoch.EpochManagerIndexUberPrefix, format.KopiaRepositoryBlobID, + format.KopiaBlobCfgBlobID) return beforeop.NewWrapper(st, nil, nil, nil, func(ctx context.Context, id blob.ID, opts *blob.PutOptions) error { for _, prefix := range prefixes { @@ -497,23 +424,24 @@ func upgradeLockMonitor( return nil } - ufb, err := readAndCacheRepoConfig(ctx, st, password, cacheOpts, lockRefreshInterval) + ufb, err := format.ReadAndCacheDecodedRepositoryConfig(ctx, st, password, cacheOpts.CacheDirectory, lockRefreshInterval) if err != nil { + // nolint:wrapcheck return err } - if err := handleMissingRequiredFeatures(ctx, ufb.repoConfig, ignoreMissingRequiredFeatures); err != nil { + if err := handleMissingRequiredFeatures(ctx, ufb.RepoConfig, ignoreMissingRequiredFeatures); err != nil { onFatalError(err) return err } // only allow the upgrade owner to perform storage operations - if locked, _ := ufb.repoConfig.UpgradeLock.IsLocked(now()); locked && upgradeOwnerID != ufb.repoConfig.UpgradeLock.OwnerID { + if locked, _ := ufb.RepoConfig.UpgradeLock.IsLocked(now()); locked && upgradeOwnerID != ufb.RepoConfig.UpgradeLock.OwnerID { return ErrRepositoryUnavailableDueToUpgrageInProgress } // prevent backward jumps on nextSync - newNextSync := ufb.cacheMTime.Add(lockRefreshInterval) + newNextSync := ufb.CacheMTime.Add(lockRefreshInterval) if newNextSync.After(nextSync) { nextSync = newNextSync } @@ -540,109 +468,3 @@ func throttlingLimitsFromConnectionInfo(ctx context.Context, ci blob.ConnectionI return l } - -func writeCacheMarker(cacheDir string) error { - if cacheDir == "" { - return nil - } - - markerFile := filepath.Join(cacheDir, CacheDirMarkerFile) - - st, err := os.Stat(markerFile) - if err == nil && st.Size() >= int64(len(cacheDirMarkerContents)) { - // ok - return nil - } - - if err != nil && !os.IsNotExist(err) { - return errors.Wrap(err, "unexpected cache marker error") - } - - f, err := os.Create(markerFile) //nolint:gosec - if err != nil { - return errors.Wrap(err, "error creating cache marker") - } - - if _, err := f.WriteString(cacheDirMarkerContents); err != nil { - return errors.Wrap(err, "unable to write cachedir marker contents") - } - - return errors.Wrap(f.Close(), "error closing cache marker file") -} - -func formatBytesCachingEnabled(cacheDirectory string, validDuration time.Duration) bool { - if cacheDirectory == "" { - return false - } - - return validDuration > 0 -} - -func readRepositoryBlobBytesFromCache(ctx context.Context, cachedFile string, validDuration time.Duration) (data []byte, cacheMTime time.Time, err error) { - cst, err := os.Stat(cachedFile) - if err != nil { - return nil, time.Time{}, errors.Wrap(err, "unable to open cache file") - } - - cacheMTime = cst.ModTime() - if clock.Now().Sub(cacheMTime) > validDuration { - // got cached file, but it's too old, remove it - if err = os.Remove(cachedFile); err != nil { - log(ctx).Debugf("unable to remove cache file: %v", err) - } - - return nil, time.Time{}, errors.Errorf("cached file too old") - } - - data, err = os.ReadFile(cachedFile) // nolint:gosec - if err != nil { - return nil, time.Time{}, errors.Wrapf(err, "failed to read the cache file %q", cachedFile) - } - - return data, cacheMTime, nil -} - -func readAndCacheRepositoryBlobBytes(ctx context.Context, st blob.Storage, cacheDirectory, blobID string, validDuration time.Duration) ([]byte, time.Time, error) { - cachedFile := filepath.Join(cacheDirectory, blobID) - - if validDuration == 0 { - validDuration = DefaultRepositoryBlobCacheDuration - } - - if cacheDirectory != "" { - if err := os.MkdirAll(cacheDirectory, cache.DirMode); err != nil && !os.IsExist(err) { - log(ctx).Errorf("unable to create cache directory: %v", err) - } - } - - cacheEnabled := formatBytesCachingEnabled(cacheDirectory, validDuration) - if cacheEnabled { - data, cacheMTime, err := readRepositoryBlobBytesFromCache(ctx, cachedFile, validDuration) - if err == nil { - log(ctx).Debugf("%s retrieved from cache", blobID) - - return data, cacheMTime, nil - } - - if os.IsNotExist(err) { - log(ctx).Debugf("%s could not be fetched from cache: %v", blobID, err) - } - } else { - log(ctx).Debugf("%s cache not enabled", blobID) - } - - var b gather.WriteBuffer - defer b.Close() - - if err := st.GetBlob(ctx, blob.ID(blobID), 0, -1, &b); err != nil { - return nil, time.Time{}, errors.Wrapf(err, "error getting %s blob", blobID) - } - - if cacheEnabled { - if err := atomicfile.Write(cachedFile, b.Bytes().Reader()); err != nil { - log(ctx).Warnf("unable to write cache: %v", err) - } - } - - return b.ToByteSlice(), clock.Now(), nil -} diff --git a/repo/parameters.go b/repo/parameters.go index 78a6c1640..0afad138e 100644 --- a/repo/parameters.go +++ b/repo/parameters.go @@ -8,11 +8,11 @@ "github.com/pkg/errors" "github.com/kopia/kopia/internal/feature" - "github.com/kopia/kopia/repo/content" + "github.com/kopia/kopia/repo/format" ) func (r *directRepository) RequiredFeatures() ([]feature.Required, error) { - repoConfig, err := r.formatBlob.decryptFormatBytes(r.formatEncryptionKey) + repoConfig, err := r.formatBlob.DecryptRepositoryConfig(r.formatEncryptionKey) if err != nil { return nil, errors.Wrap(err, "unable to decrypt repository config") } @@ -23,13 +23,13 @@ func (r *directRepository) RequiredFeatures() ([]feature.Required, error) { // SetParameters changes mutable repository parameters. func (r *directRepository) SetParameters( ctx context.Context, - m content.MutableParameters, - blobcfg content.BlobCfgBlob, + m format.MutableParameters, + blobcfg format.BlobStorageConfiguration, requiredFeatures []feature.Required, ) error { f := r.formatBlob - repoConfig, err := f.decryptFormatBytes(r.formatEncryptionKey) + repoConfig, err := f.DecryptRepositoryConfig(r.formatEncryptionKey) if err != nil { return errors.Wrap(err, "unable to decrypt repository config") } @@ -42,28 +42,28 @@ func (r *directRepository) SetParameters( return errors.Wrap(err, "invalid blob-config options") } - repoConfig.FormattingOptions.MutableParameters = m + repoConfig.ContentFormat.MutableParameters = m repoConfig.RequiredFeatures = requiredFeatures - if err := encryptFormatBytes(f, repoConfig, r.formatEncryptionKey, f.UniqueID); err != nil { + if err := f.EncryptRepositoryConfig(repoConfig, r.formatEncryptionKey); err != nil { return errors.Errorf("unable to encrypt format bytes") } - if err := writeBlobCfgBlob(ctx, r.blobs, f, blobcfg, r.formatEncryptionKey); err != nil { + if err := f.WriteBlobCfgBlob(ctx, r.blobs, blobcfg, r.formatEncryptionKey); err != nil { return errors.Wrap(err, "unable to write blobcfg blob") } - if err := writeFormatBlob(ctx, r.blobs, f, r.blobCfgBlob); err != nil { + if err := f.WriteKopiaRepositoryBlob(ctx, r.blobs, r.blobCfgBlob); err != nil { return errors.Wrap(err, "unable to write format blob") } if cd := r.cachingOptions.CacheDirectory; cd != "" { - if err := os.Remove(filepath.Join(cd, FormatBlobID)); err != nil { - log(ctx).Errorf("unable to remove %s: %v", FormatBlobID, err) + if err := os.Remove(filepath.Join(cd, format.KopiaRepositoryBlobID)); err != nil { + log(ctx).Errorf("unable to remove %s: %v", format.KopiaRepositoryBlobID, err) } - if err := os.Remove(filepath.Join(cd, BlobCfgBlobID)); err != nil && !os.IsNotExist(err) { - log(ctx).Errorf("unable to remove %s: %v", BlobCfgBlobID, err) + if err := os.Remove(filepath.Join(cd, format.KopiaBlobCfgBlobID)); err != nil && !os.IsNotExist(err) { + log(ctx).Errorf("unable to remove %s: %v", format.KopiaBlobCfgBlobID, err) } } diff --git a/repo/repo_benchmarks_test.go b/repo/repo_benchmarks_test.go index ef6169ed9..a915a0594 100644 --- a/repo/repo_benchmarks_test.go +++ b/repo/repo_benchmarks_test.go @@ -7,12 +7,12 @@ "github.com/stretchr/testify/require" "github.com/kopia/kopia/internal/repotesting" - "github.com/kopia/kopia/repo/content" + "github.com/kopia/kopia/repo/format" "github.com/kopia/kopia/repo/object" ) func BenchmarkWriterDedup1M(b *testing.B) { - ctx, env := repotesting.NewEnvironment(b, content.FormatVersion2) + ctx, env := repotesting.NewEnvironment(b, format.FormatVersion2) dataBuf := make([]byte, 4<<20) writer := env.RepositoryWriter.NewObjectWriter(ctx, object.WriterOptions{}) @@ -33,7 +33,7 @@ func BenchmarkWriterDedup1M(b *testing.B) { } func BenchmarkWriterNoDedup1M(b *testing.B) { - ctx, env := repotesting.NewEnvironment(b, content.FormatVersion2) + ctx, env := repotesting.NewEnvironment(b, format.FormatVersion2) dataBuf := make([]byte, 4<<20) chunkSize := 32 offset := 0 diff --git a/repo/repository.go b/repo/repository.go index ffedb67c4..2513e10f6 100644 --- a/repo/repository.go +++ b/repo/repository.go @@ -14,6 +14,7 @@ "github.com/kopia/kopia/repo/blob" "github.com/kopia/kopia/repo/blob/throttling" "github.com/kopia/kopia/repo/content" + "github.com/kopia/kopia/repo/format" "github.com/kopia/kopia/repo/manifest" "github.com/kopia/kopia/repo/object" ) @@ -52,8 +53,8 @@ type RepositoryWriter interface { type DirectRepository interface { Repository - ObjectFormat() object.Format - BlobCfg() content.BlobCfgBlob + ObjectFormat() format.ObjectFormat + BlobCfg() format.BlobStorageConfiguration BlobReader() blob.Reader BlobVolume() blob.Volume ContentReader() content.Reader @@ -75,10 +76,10 @@ type DirectRepositoryWriter interface { DirectRepository BlobStorage() blob.Storage ContentManager() *content.WriteManager - SetParameters(ctx context.Context, m content.MutableParameters, blobcfg content.BlobCfgBlob, requiredFeatures []feature.Required) error + SetParameters(ctx context.Context, m format.MutableParameters, blobcfg format.BlobStorageConfiguration, requiredFeatures []feature.Required) error ChangePassword(ctx context.Context, newPassword string) error - GetUpgradeLockIntent(ctx context.Context) (*UpgradeLockIntent, error) - SetUpgradeLockIntent(ctx context.Context, l UpgradeLockIntent) (*UpgradeLockIntent, error) + GetUpgradeLockIntent(ctx context.Context) (*format.UpgradeLockIntent, error) + SetUpgradeLockIntent(ctx context.Context, l format.UpgradeLockIntent) (*format.UpgradeLockIntent, error) CommitUpgrade(ctx context.Context) error RollbackUpgrade(ctx context.Context) error } @@ -89,8 +90,8 @@ type directRepositoryParameters struct { cachingOptions content.CachingOptions cliOpts ClientOptions timeNow func() time.Time - formatBlob *formatBlob - blobCfgBlob content.BlobCfgBlob + formatBlob *format.KopiaRepositoryJSON + blobCfgBlob format.BlobStorageConfiguration formatEncryptionKey []byte nextWriterID *int32 throttler throttling.SettableThrottler @@ -112,13 +113,13 @@ type directRepository struct { // DeriveKey derives encryption key of the provided length from the master key. func (r *directRepository) DeriveKey(purpose []byte, keyLength int) []byte { if r.cmgr.ContentFormat().SupportsPasswordChange() { - return deriveKeyFromMasterKey(r.cmgr.ContentFormat().GetMasterKey(), r.uniqueID, purpose, keyLength) + return format.DeriveKeyFromMasterKey(r.cmgr.ContentFormat().GetMasterKey(), r.uniqueID, purpose, keyLength) } // version of kopia = content.MaxFormatVersion { + if repoConfig.ContentFormat.Version >= format.MaxFormatVersion { return errors.Errorf("repository is using version %d, and version %d is the maximum", - repoConfig.FormattingOptions.Version, content.MaxFormatVersion) + repoConfig.ContentFormat.Version, format.MaxFormatVersion) } // backup the current repository config from local cache to the // repository when we place the lock for the first time - if err := writeFormatBlobWithID(ctx, r.blobs, r.formatBlob, r.blobCfgBlob, FormatBlobBackupID(l)); err != nil { + if err := r.formatBlob.WriteKopiaRepositoryBlobWithID(ctx, r.blobs, r.blobCfgBlob, FormatBlobBackupID(l)); err != nil { return errors.Wrap(err, "failed to backup the repo format blob") } @@ -83,7 +83,7 @@ func (r *directRepository) SetUpgradeLockIntent(ctx context.Context, l UpgradeLo repoConfig.UpgradeLock = &l // mark the upgrade to the new format version, this will ensure that older // clients won't be able to parse the new version - repoConfig.FormattingOptions.Version = content.MaxFormatVersion + repoConfig.ContentFormat.Version = format.MaxFormatVersion } else if newL, err := repoConfig.UpgradeLock.Update(&l); err == nil { repoConfig.UpgradeLock = newL } else { @@ -103,7 +103,7 @@ func (r *directRepository) SetUpgradeLockIntent(ctx context.Context, l UpgradeLo // blob. This in-effect commits the new repository format t othe repository and // resumes all access to the repository. func (r *directRepository) CommitUpgrade(ctx context.Context) error { - _, err := r.updateRepoConfig(ctx, func(repoConfig *repositoryObjectFormat) error { + _, err := r.updateRepoConfig(ctx, func(repoConfig *format.RepositoryConfig) error { if repoConfig.UpgradeLock == nil { return errors.New("no upgrade in progress") } @@ -128,7 +128,7 @@ func (r *directRepository) CommitUpgrade(ctx context.Context) error { func (r *directRepository) RollbackUpgrade(ctx context.Context) error { f := r.formatBlob - repoConfig, err := f.decryptFormatBytes(r.formatEncryptionKey) + repoConfig, err := f.DecryptRepositoryConfig(r.formatEncryptionKey) if err != nil { return errors.Wrap(err, "unable to decrypt repository config") } @@ -171,7 +171,7 @@ func (r *directRepository) RollbackUpgrade(ctx context.Context) error { return errors.Wrapf(err, "failed to read from backup %q", oldestBackup.BlobID) } - if err = r.blobs.PutBlob(ctx, FormatBlobID, d.Bytes(), blob.PutOptions{}); err != nil { + if err = r.blobs.PutBlob(ctx, format.KopiaRepositoryBlobID, d.Bytes(), blob.PutOptions{}); err != nil { return errors.Wrapf(err, "failed to restore format blob from backup %q", oldestBackup.BlobID) } @@ -182,7 +182,7 @@ func (r *directRepository) RollbackUpgrade(ctx context.Context) error { } if cd := r.cachingOptions.CacheDirectory; cd != "" { - if err = os.Remove(filepath.Join(cd, FormatBlobID)); err != nil && !os.IsNotExist(err) { + if err = os.Remove(filepath.Join(cd, format.KopiaRepositoryBlobID)); err != nil && !os.IsNotExist(err) { return errors.Errorf("unable to remove cached repository format blob: %v", err) } } @@ -190,10 +190,10 @@ func (r *directRepository) RollbackUpgrade(ctx context.Context) error { return nil } -func (r *directRepository) GetUpgradeLockIntent(ctx context.Context) (*UpgradeLockIntent, error) { +func (r *directRepository) GetUpgradeLockIntent(ctx context.Context) (*format.UpgradeLockIntent, error) { f := r.formatBlob - repoConfig, err := f.decryptFormatBytes(r.formatEncryptionKey) + repoConfig, err := f.DecryptRepositoryConfig(r.formatEncryptionKey) if err != nil { return nil, errors.Wrap(err, "unable to decrypt repository config") } diff --git a/repo/upgrade_lock_test.go b/repo/upgrade_lock_test.go index 93fc8d470..5d81ec5ca 100644 --- a/repo/upgrade_lock_test.go +++ b/repo/upgrade_lock_test.go @@ -23,17 +23,17 @@ "github.com/kopia/kopia/repo/blob/beforeop" "github.com/kopia/kopia/repo/content" "github.com/kopia/kopia/repo/encryption" - "github.com/kopia/kopia/repo/object" + "github.com/kopia/kopia/repo/format" ) func TestFormatUpgradeSetLock(t *testing.T) { - ctx, env := repotesting.NewEnvironment(t, content.FormatVersion1, repotesting.Options{OpenOptions: func(opts *repo.Options) { + ctx, env := repotesting.NewEnvironment(t, format.FormatVersion1, repotesting.Options{OpenOptions: func(opts *repo.Options) { // nolint:goconst opts.UpgradeOwnerID = "upgrade-owner" }}) formatBlockCacheDuration := env.Repository.ClientOptions().FormatBlobCacheDuration - l := &repo.UpgradeLockIntent{ + l := &format.UpgradeLockIntent{ CreationTime: env.Repository.Time(), AdvanceNoticeDuration: 15 * time.Hour, IODrainTimeout: formatBlockCacheDuration * 2, @@ -70,10 +70,10 @@ func TestFormatUpgradeSetLock(t *testing.T) { } func TestFormatUpgradeAlreadyUpgraded(t *testing.T) { - ctx, env := repotesting.NewEnvironment(t, content.MaxFormatVersion) + ctx, env := repotesting.NewEnvironment(t, format.MaxFormatVersion) formatBlockCacheDuration := env.Repository.ClientOptions().FormatBlobCacheDuration - l := &repo.UpgradeLockIntent{ + l := &format.UpgradeLockIntent{ OwnerID: "new-upgrade-owner", CreationTime: env.Repository.Time(), AdvanceNoticeDuration: 0, @@ -85,16 +85,16 @@ func TestFormatUpgradeAlreadyUpgraded(t *testing.T) { _, err := env.RepositoryWriter.SetUpgradeLockIntent(ctx, *l) require.EqualError(t, err, fmt.Sprintf("repository is using version %d, and version %d is the maximum", - content.MaxFormatVersion, content.MaxFormatVersion)) + format.MaxFormatVersion, format.MaxFormatVersion)) } func TestFormatUpgradeCommit(t *testing.T) { - ctx, env := repotesting.NewEnvironment(t, content.FormatVersion1, repotesting.Options{OpenOptions: func(opts *repo.Options) { + ctx, env := repotesting.NewEnvironment(t, format.FormatVersion1, repotesting.Options{OpenOptions: func(opts *repo.Options) { opts.UpgradeOwnerID = "upgrade-owner" }}) formatBlockCacheDuration := env.Repository.ClientOptions().FormatBlobCacheDuration - l := &repo.UpgradeLockIntent{ + l := &format.UpgradeLockIntent{ OwnerID: "upgrade-owner", CreationTime: env.Repository.Time(), AdvanceNoticeDuration: 0, @@ -116,12 +116,12 @@ func TestFormatUpgradeCommit(t *testing.T) { } func TestFormatUpgradeRollback(t *testing.T) { - ctx, env := repotesting.NewEnvironment(t, content.FormatVersion1, repotesting.Options{OpenOptions: func(opts *repo.Options) { + ctx, env := repotesting.NewEnvironment(t, format.FormatVersion1, repotesting.Options{OpenOptions: func(opts *repo.Options) { opts.UpgradeOwnerID = "upgrade-owner" }}) formatBlockCacheDuration := env.Repository.ClientOptions().FormatBlobCacheDuration - l := &repo.UpgradeLockIntent{ + l := &format.UpgradeLockIntent{ OwnerID: "upgrade-owner", CreationTime: env.Repository.Time(), AdvanceNoticeDuration: 0, @@ -144,12 +144,12 @@ func TestFormatUpgradeRollback(t *testing.T) { } func TestFormatUpgradeMultipleLocksRollback(t *testing.T) { - ctx, env := repotesting.NewEnvironment(t, content.FormatVersion1, repotesting.Options{OpenOptions: func(opts *repo.Options) { + ctx, env := repotesting.NewEnvironment(t, format.FormatVersion1, repotesting.Options{OpenOptions: func(opts *repo.Options) { opts.UpgradeOwnerID = "upgrade-owner" }}) formatBlockCacheDuration := env.Repository.ClientOptions().FormatBlobCacheDuration - l := &repo.UpgradeLockIntent{ + l := &format.UpgradeLockIntent{ OwnerID: "upgrade-owner", CreationTime: env.Repository.Time(), AdvanceNoticeDuration: 0, @@ -190,7 +190,7 @@ func TestFormatUpgradeMultipleLocksRollback(t *testing.T) { env.MustReopen(t, func(opts *repo.Options) { opts.UpgradeOwnerID = "another-upgrade-owner" }) - require.Equal(t, content.FormatVersion3, + require.Equal(t, format.FormatVersion3, env.RepositoryWriter.ContentManager().ContentFormat().FormatVersion()) require.NoError(t, env.RepositoryWriter.RollbackUpgrade(ctx)) @@ -210,13 +210,13 @@ func TestFormatUpgradeMultipleLocksRollback(t *testing.T) { require.EqualError(t, env.RepositoryWriter.CommitUpgrade(ctx), "no upgrade in progress") // verify that we are back to the original version where we started from - require.Equal(t, content.FormatVersion1, + require.Equal(t, format.FormatVersion1, env.RepositoryWriter.ContentManager().ContentFormat().FormatVersion()) } func TestFormatUpgradeFailureToBackupFormatBlobOnLock(t *testing.T) { // this lock will be allowed by the backend to create backups - allowedLock := repo.UpgradeLockIntent{ + allowedLock := format.UpgradeLockIntent{ OwnerID: "allowed-upgrade-owner", CreationTime: clock.Now(), AdvanceNoticeDuration: 0, @@ -256,16 +256,16 @@ func(ctx context.Context, id blob.ID, _ *blob.PutOptions) error { )) opt := &repo.NewRepositoryOptions{ - BlockFormat: content.FormattingOptions{ - MutableParameters: content.MutableParameters{ - Version: content.FormatVersion1, + BlockFormat: format.ContentFormat{ + MutableParameters: format.MutableParameters{ + Version: format.FormatVersion1, }, HMACSecret: []byte{}, Hash: "HMAC-SHA256", Encryption: encryption.DefaultAlgorithm, EnablePasswordChange: true, }, - ObjectFormat: object.Format{ + ObjectFormat: format.ObjectFormat{ Splitter: "FIXED-1M", }, } @@ -311,7 +311,7 @@ func(ctx context.Context, id blob.ID, _ *blob.PutOptions) error { func TestFormatUpgradeDuringOngoingWriteSessions(t *testing.T) { curTime := clock.Now() - ctx, env := repotesting.NewEnvironment(t, content.FormatVersion1, repotesting.Options{ + ctx, env := repotesting.NewEnvironment(t, format.FormatVersion1, repotesting.Options{ // new environment with controlled time OpenOptions: func(opts *repo.Options) { opts.TimeNowFunc = func() time.Time { @@ -351,7 +351,7 @@ func TestFormatUpgradeDuringOngoingWriteSessions(t *testing.T) { writeObject(ctx, t, lw, o4Data, "o4") formatBlockCacheDuration := env.Repository.ClientOptions().FormatBlobCacheDuration - l := repo.UpgradeLockIntent{ + l := format.UpgradeLockIntent{ OwnerID: "upgrade-owner", CreationTime: env.Repository.Time(), AdvanceNoticeDuration: 0, diff --git a/site/content/docs/Advanced/Encryption/_index.md b/site/content/docs/Advanced/Encryption/_index.md index f281c92bc..babdfdbdb 100644 --- a/site/content/docs/Advanced/Encryption/_index.md +++ b/site/content/docs/Advanced/Encryption/_index.md @@ -40,7 +40,7 @@ type formatBlob struct { Version string `json:"version"` EncryptionAlgorithm string `json:"encryption"` EncryptedFormatBytes []byte `json:"encryptedBlockFormat,omitempty"` - UnencryptedFormat *repositoryObjectFormat `json:"blockFormat,omitempty"` + UnencryptedFormat *format.RepositoryObjectFormat `json:"blockFormat,omitempty"` } ``` @@ -53,25 +53,25 @@ type formatBlob struct { * `encryptedBlockFormat` is a ciphertext containing among others, the encryption secrets and parameters used for encrypting the repository content. Below is additional information about its plaintext content and how it is encrypted. * Alternatively, the unencrypted block format parameters can be specified in the the `blockFormat` field. -The `formatBlob.encryptedBlockFormat` field is the result of encrypting a JSON-serialized version of the `encryptedRepositoryConfig` struct shown below. The plaintext version contains the parameters for performing block chunking, as well as for encrypting and authenticating "content" objects. +The `formatBlob.EncryptedBlockFormat` field is the result of encrypting a JSON-serialized version of the `EncryptedRepositoryConfig` struct shown below. The plaintext version contains the parameters for performing block chunking, as well as for encrypting and authenticating "content" objects. ```go -type encryptedRepositoryConfig struct { - Format repositoryObjectFormat `json:"format"` +type EncryptedRepositoryConfig struct { + Format RepositoryObjectFormat `json:"format"` } -type repositoryObjectFormat struct { - content.FormattingOptions - object.Format +type RepositoryObjectFormat struct { + format.ContentFormat + format.ObjectFormat } ``` ```go package content -// FormattingOptions describes the rules for formatting contents in repository. -type FormattingOptions struct { +// ContentFormat describes the rules for formatting contents in repository. +type ContentFormat struct { Version int `json:"version,omitempty"` // version number, must be "1" Hash string `json:"hash,omitempty"` // identifier of the hash algorithm used Encryption string `json:"encryption,omitempty"` // identifier of the encryption algorithm used diff --git a/snapshot/snapshotmaintenance/snapshotmaintenance_test.go b/snapshot/snapshotmaintenance/snapshotmaintenance_test.go index 2451f1691..dc54f6eb1 100644 --- a/snapshot/snapshotmaintenance/snapshotmaintenance_test.go +++ b/snapshot/snapshotmaintenance/snapshotmaintenance_test.go @@ -17,6 +17,7 @@ "github.com/kopia/kopia/internal/testlogging" "github.com/kopia/kopia/repo" "github.com/kopia/kopia/repo/content" + "github.com/kopia/kopia/repo/format" "github.com/kopia/kopia/repo/maintenance" "github.com/kopia/kopia/repo/object" "github.com/kopia/kopia/snapshot" @@ -230,7 +231,7 @@ func (s *formatSpecificTestSuite) TestSnapshotGCMinContentAgeSafety(t *testing.T checkContentDeletion(t, th.Repository, cids, false) } -func newTestHarness(t *testing.T, formatVersion content.FormatVersion) *testHarness { +func newTestHarness(t *testing.T, formatVersion format.Version) *testHarness { t.Helper() baseTime := time.Date(2020, 9, 10, 0, 0, 0, 0, time.UTC) @@ -254,7 +255,7 @@ func (s *formatSpecificTestSuite) TestMaintenanceAutoLiveness(t *testing.T) { o.TimeNowFunc = ft.NowFunc() }, NewRepositoryOptions: func(nro *repo.NewRepositoryOptions) { - nro.BlockFormat.Version = content.FormatVersion1 + nro.BlockFormat.Version = format.FormatVersion1 }, }) diff --git a/snapshot/snapshotmaintenance/suite_test.go b/snapshot/snapshotmaintenance/suite_test.go index 1410a81e9..b105548fe 100644 --- a/snapshot/snapshotmaintenance/suite_test.go +++ b/snapshot/snapshotmaintenance/suite_test.go @@ -4,21 +4,21 @@ "testing" "github.com/kopia/kopia/internal/testutil" - "github.com/kopia/kopia/repo/content" + "github.com/kopia/kopia/repo/format" ) type formatSpecificTestSuite struct { - formatVersion content.FormatVersion + formatVersion format.Version } func TestFormatV1(t *testing.T) { - testutil.RunAllTestsWithParam(t, &formatSpecificTestSuite{content.FormatVersion1}) + testutil.RunAllTestsWithParam(t, &formatSpecificTestSuite{format.FormatVersion1}) } func TestFormatV2(t *testing.T) { - testutil.RunAllTestsWithParam(t, &formatSpecificTestSuite{content.FormatVersion2}) + testutil.RunAllTestsWithParam(t, &formatSpecificTestSuite{format.FormatVersion2}) } func TestFormatV3(t *testing.T) { - testutil.RunAllTestsWithParam(t, &formatSpecificTestSuite{content.FormatVersion3}) + testutil.RunAllTestsWithParam(t, &formatSpecificTestSuite{format.FormatVersion3}) } diff --git a/tests/end_to_end_test/repository_repair_test.go b/tests/end_to_end_test/repository_repair_test.go index 03b6cf3bb..79923928a 100644 --- a/tests/end_to_end_test/repository_repair_test.go +++ b/tests/end_to_end_test/repository_repair_test.go @@ -3,7 +3,7 @@ import ( "testing" - "github.com/kopia/kopia/repo/content" + "github.com/kopia/kopia/repo/format" "github.com/kopia/kopia/tests/testenv" ) @@ -33,7 +33,7 @@ func (s *formatSpecificTestSuite) TestRepositoryRepair(t *testing.T) { // this will fail because the format blob in the repository is not found e.RunAndExpectFailure(t, "repo", "connect", "filesystem", "--path", e.RepoDir) - if s.formatVersion == content.FormatVersion1 { + if s.formatVersion == format.FormatVersion1 { // now run repair, which will recover the format blob from one of the pack blobs. e.RunAndExpectSuccess(t, "repo", "repair", "filesystem", "--path", e.RepoDir) diff --git a/tests/end_to_end_test/snapshot_create_test.go b/tests/end_to_end_test/snapshot_create_test.go index 66faa8d43..81a24698a 100644 --- a/tests/end_to_end_test/snapshot_create_test.go +++ b/tests/end_to_end_test/snapshot_create_test.go @@ -15,8 +15,8 @@ "github.com/stretchr/testify/require" "github.com/kopia/kopia/cli" + "github.com/kopia/kopia/internal/cachedir" "github.com/kopia/kopia/internal/testutil" - "github.com/kopia/kopia/repo" "github.com/kopia/kopia/snapshot" "github.com/kopia/kopia/tests/clitestutil" "github.com/kopia/kopia/tests/testenv" @@ -209,7 +209,7 @@ func TestSnapshottingCacheDirectory(t *testing.T) { cachePath := filepath.Dir(strings.Split(lines[0], ": ")[0]) // verify cache marker exists - if _, err := os.Stat(filepath.Join(cachePath, repo.CacheDirMarkerFile)); err != nil { + if _, err := os.Stat(filepath.Join(cachePath, cachedir.CacheDirMarkerFile)); err != nil { t.Fatal(err) } diff --git a/tests/end_to_end_test/suite_test.go b/tests/end_to_end_test/suite_test.go index af399ac2b..a031e8ac5 100644 --- a/tests/end_to_end_test/suite_test.go +++ b/tests/end_to_end_test/suite_test.go @@ -4,22 +4,22 @@ "testing" "github.com/kopia/kopia/internal/testutil" - "github.com/kopia/kopia/repo/content" + "github.com/kopia/kopia/repo/format" ) type formatSpecificTestSuite struct { formatFlags []string - formatVersion content.FormatVersion + formatVersion format.Version } func TestFormatV1(t *testing.T) { - testutil.RunAllTestsWithParam(t, &formatSpecificTestSuite{[]string{"--format-version=1"}, content.FormatVersion1}) + testutil.RunAllTestsWithParam(t, &formatSpecificTestSuite{[]string{"--format-version=1"}, format.FormatVersion1}) } func TestFormatV2(t *testing.T) { - testutil.RunAllTestsWithParam(t, &formatSpecificTestSuite{[]string{"--format-version=2"}, content.FormatVersion2}) + testutil.RunAllTestsWithParam(t, &formatSpecificTestSuite{[]string{"--format-version=2"}, format.FormatVersion2}) } func TestFormatV3(t *testing.T) { - testutil.RunAllTestsWithParam(t, &formatSpecificTestSuite{[]string{"--format-version=3"}, content.FormatVersion3}) + testutil.RunAllTestsWithParam(t, &formatSpecificTestSuite{[]string{"--format-version=3"}, format.FormatVersion3}) } diff --git a/tests/stress_test/stress_test.go b/tests/stress_test/stress_test.go index 8254c4663..7f744d2bb 100644 --- a/tests/stress_test/stress_test.go +++ b/tests/stress_test/stress_test.go @@ -18,6 +18,7 @@ "github.com/kopia/kopia/repo/blob" "github.com/kopia/kopia/repo/content" "github.com/kopia/kopia/repo/encryption" + "github.com/kopia/kopia/repo/format" ) const goroutineCount = 16 @@ -47,10 +48,10 @@ func TestStressBlockManager(t *testing.T) { func stressTestWithStorage(t *testing.T, st blob.Storage, duration time.Duration) { ctx := testlogging.Context(t) - fop, err := content.NewFormattingOptionsProvider(&content.FormattingOptions{ + fop, err := format.NewFormattingOptionsProvider(&format.ContentFormat{ Hash: "HMAC-SHA256-128", Encryption: encryption.DefaultAlgorithm, - MutableParameters: content.MutableParameters{ + MutableParameters: format.MutableParameters{ Version: 1, MaxPackSize: 20000000, }, diff --git a/tests/tools/kopiarunner/kopia_snapshotter_upgrade_test.go b/tests/tools/kopiarunner/kopia_snapshotter_upgrade_test.go index 8a2c10787..0ae23062b 100644 --- a/tests/tools/kopiarunner/kopia_snapshotter_upgrade_test.go +++ b/tests/tools/kopiarunner/kopia_snapshotter_upgrade_test.go @@ -6,7 +6,7 @@ "github.com/stretchr/testify/require" - "github.com/kopia/kopia/repo/content" + "github.com/kopia/kopia/repo/format" ) func TestUpgradeFormatVersion(t *testing.T) { @@ -24,7 +24,7 @@ func TestUpgradeFormatVersion(t *testing.T) { require.NoError(t, err) prev := rs.ContentFormat.MutableParameters.Version - require.Equal(t, prev, content.FormatVersion(1), "The format version should be 1.") + require.Equal(t, prev, format.Version(1), "The format version should be 1.") ks.UpgradeRepository(repoDir) @@ -32,7 +32,7 @@ func TestUpgradeFormatVersion(t *testing.T) { require.NoError(t, err) got := rs.ContentFormat.MutableParameters.Version - require.Equal(t, got, content.FormatVersion(2), "The format version should be upgraded to 2.") + require.Equal(t, got, format.Version(2), "The format version should be upgraded to 2.") require.NotEqual(t, got, prev, "The format versions should be different.") }