mirror/kopia
1
0
Fork 0
mirror of https://github.com/kopia/kopia.git synced 2026-04-27 09:27:54 -04:00
Code Issues Packages Projects Releases Wiki Activity

refactor(repository): moved format blob management to separate package (#2245)

Browse Source 
* refactor(repository): moved format blob management to separate package

This is completely mechanical, no behavior changes, only:

- moved types and functions to a new package
- adjusted visibility where needed
- added missing godoc
- renamed some identifiers to align with current usage
- mechanically converted some top-level functions into member functions
- fixed some mis-named variables

* refactor(repository): moved content.FormatingOptions to format.ContentFormat
This commit is contained in:
Jarek Kowalski
2022-07-30 14:13:52 -07:00
committed by GitHub
parent b8c4e23115
commit 6160ee5668
68 changed files with 1094 additions and 998 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
cli/command_benchmark_crypto.go
View File

@@ -9,8 +9,8 @@
"github.com/kopia/kopia/internal/gather"
"github.com/kopia/kopia/internal/timetrack"
"github.com/kopia/kopia/internal/units"
"github.com/kopia/kopia/repo/content"
"github.com/kopia/kopia/repo/encryption"
"github.com/kopia/kopia/repo/format"
"github.com/kopia/kopia/repo/hashing"
)
@@ -67,7 +67,7 @@ func (c *commandBenchmarkCrypto) runBenchmark(ctx context.Context) []cryptoBench
for _, ha := range hashing.SupportedAlgorithms() {
for _, ea := range encryption.SupportedAlgorithms(c.deprecatedAlgorithms) {
fo := &content.FormattingOptions{
fo := &format.ContentFormat{
Encryption: ea,
Hash: ha,
MasterKey: make([]byte, 32), // nolint:gomnd

4
cli/command_benchmark_encryption.go
View File

@@ -9,8 +9,8 @@
"github.com/kopia/kopia/internal/gather"
"github.com/kopia/kopia/internal/timetrack"
"github.com/kopia/kopia/internal/units"
"github.com/kopia/kopia/repo/content"
"github.com/kopia/kopia/repo/encryption"
"github.com/kopia/kopia/repo/format"
"github.com/kopia/kopia/repo/hashing"
)
@@ -66,7 +66,7 @@ func (c *commandBenchmarkEncryption) runBenchmark(ctx context.Context) []cryptoB
data := make([]byte, c.blockSize)
for _, ea := range encryption.SupportedAlgorithms(c.deprecatedAlgorithms) {
enc, err := encryption.CreateEncryptor(&content.FormattingOptions{
enc, err := encryption.CreateEncryptor(&format.ContentFormat{
Encryption: ea,
Hash: hashing.DefaultAlgorithm,
MasterKey: make([]byte, 32), // nolint:gomnd

4
cli/command_benchmark_hashing.go
View File

@@ -9,7 +9,7 @@
"github.com/kopia/kopia/internal/gather"
"github.com/kopia/kopia/internal/timetrack"
"github.com/kopia/kopia/internal/units"
"github.com/kopia/kopia/repo/content"
"github.com/kopia/kopia/repo/format"
"github.com/kopia/kopia/repo/hashing"
)
@@ -63,7 +63,7 @@ func (c *commandBenchmarkHashing) runBenchmark(ctx context.Context) []cryptoBenc
data := make([]byte, c.blockSize)
for _, ha := range hashing.SupportedAlgorithms() {
hf, err := hashing.CreateHashFunc(&content.FormattingOptions{
hf, err := hashing.CreateHashFunc(&format.ContentFormat{
Hash: ha,
HMACSecret: make([]byte, 32), // nolint:gomnd
})

4
cli/command_repository_change_password_test.go
View File

@@ -3,7 +3,7 @@
import (
"testing"
"github.com/kopia/kopia/repo/content"
"github.com/kopia/kopia/repo/format"
"github.com/kopia/kopia/tests/testenv"
)
@@ -15,7 +15,7 @@ func (s *formatSpecificTestSuite) TestRepositoryChangePassword(t *testing.T) {
env1.RunAndExpectSuccess(t, "repo", "create", "filesystem", "--path", env1.RepoDir, "--disable-repository-format-cache")
if s.formatVersion == content.FormatVersion1 {
if s.formatVersion == format.FormatVersion1 {
env1.RunAndExpectFailure(t, "repo", "change-password", "--new-password", "newPass")
return

11
cli/command_repository_create.go
View File

@@ -9,10 +9,9 @@
"github.com/kopia/kopia/repo"
"github.com/kopia/kopia/repo/blob"
"github.com/kopia/kopia/repo/content"
"github.com/kopia/kopia/repo/encryption"
"github.com/kopia/kopia/repo/format"
"github.com/kopia/kopia/repo/hashing"
"github.com/kopia/kopia/repo/object"
"github.com/kopia/kopia/repo/splitter"
"github.com/kopia/kopia/snapshot/policy"
)
@@ -77,15 +76,15 @@ func (c *commandRepositoryCreate) setup(svc advancedAppServices, parent commandP
func (c *commandRepositoryCreate) newRepositoryOptionsFromFlags() *repo.NewRepositoryOptions {
return &repo.NewRepositoryOptions{
BlockFormat: content.FormattingOptions{
MutableParameters: content.MutableParameters{
Version: content.FormatVersion(c.createFormatVersion),
BlockFormat: format.ContentFormat{
MutableParameters: format.MutableParameters{
Version: format.Version(c.createFormatVersion),
},
Hash: c.createBlockHashFormat,
Encryption: c.createBlockEncryptionFormat,
},
ObjectFormat: object.Format{
ObjectFormat: format.ObjectFormat{
Splitter: c.createSplitter,
},

8
cli/command_repository_repair.go
View File

@@ -7,9 +7,9 @@
"github.com/pkg/errors"
"github.com/kopia/kopia/internal/gather"
"github.com/kopia/kopia/repo"
"github.com/kopia/kopia/repo/blob"
"github.com/kopia/kopia/repo/content"
"github.com/kopia/kopia/repo/format"
)
type commandRepositoryRepair struct {
@@ -61,7 +61,7 @@ func (c *commandRepositoryRepair) runRepairCommandWithStorage(ctx context.Contex
var tmp gather.WriteBuffer
defer tmp.Close()
if err := st.GetBlob(ctx, repo.FormatBlobID, 0, -1, &tmp); err == nil {
if err := st.GetBlob(ctx, format.KopiaRepositoryBlobID, 0, -1, &tmp); err == nil {
log(ctx).Infof("format blob already exists, not recovering, pass --recover-format=yes")
return nil
}
@@ -84,9 +84,9 @@ func (c *commandRepositoryRepair) recoverFormatBlob(ctx context.Context, st blob
for _, prefix := range prefixes {
err := st.ListBlobs(ctx, blob.ID(prefix), func(bi blob.Metadata) error {
log(ctx).Infof("looking for replica of format blob in %v...", bi.BlobID)
if b, err := repo.RecoverFormatBlob(ctx, st, bi.BlobID, bi.Length); err == nil {
if b, err := format.RecoverFormatBlob(ctx, st, bi.BlobID, bi.Length); err == nil {
if !c.repairDryRun {
if puterr := st.PutBlob(ctx, repo.FormatBlobID, gather.FromSlice(b), blob.PutOptions{}); puterr != nil {
if puterr := st.PutBlob(ctx, format.KopiaRepositoryBlobID, gather.FromSlice(b), blob.PutOptions{}); puterr != nil {
return errors.Wrap(puterr, "error writing format blob")
}
}

5
cli/command_repository_set_parameters.go
View File

@@ -12,6 +12,7 @@
"github.com/kopia/kopia/repo"
"github.com/kopia/kopia/repo/blob"
"github.com/kopia/kopia/repo/content"
"github.com/kopia/kopia/repo/format"
)
type commandRepositorySetParameters struct {
@@ -143,8 +144,8 @@ func (c *commandRepositorySetParameters) run(ctx context.Context, rep repo.Direc
mp.IndexVersion = 2
}
if mp.Version < content.FormatVersion2 {
mp.Version = content.FormatVersion2
if mp.Version < format.FormatVersion2 {
mp.Version = format.FormatVersion2
}
}

8
cli/command_repository_set_parameters_test.go
View File

@@ -12,7 +12,7 @@
"github.com/kopia/kopia/internal/repotesting"
"github.com/kopia/kopia/internal/testutil"
"github.com/kopia/kopia/repo/blob"
"github.com/kopia/kopia/repo/content"
"github.com/kopia/kopia/repo/format"
"github.com/kopia/kopia/tests/testenv"
)
@@ -127,11 +127,11 @@ func (s *formatSpecificTestSuite) TestRepositorySetParametersUpgrade(t *testing.
require.Contains(t, out, "Max pack length: 20 MiB")
switch s.formatVersion {
case content.FormatVersion1:
case format.FormatVersion1:
require.Contains(t, out, "Format version: 1")
require.Contains(t, out, "Epoch Manager: disabled")
env.RunAndExpectFailure(t, "index", "epoch", "list")
case content.FormatVersion2:
case format.FormatVersion2:
require.Contains(t, out, "Format version: 2")
require.Contains(t, out, "Epoch Manager: enabled")
env.RunAndExpectSuccess(t, "index", "epoch", "list")
@@ -154,7 +154,7 @@ func (s *formatSpecificTestSuite) TestRepositorySetParametersUpgrade(t *testing.
cli.MaxPermittedClockDrift = func() time.Duration { return time.Second }
// You can only upgrade when you are not already upgraded
if s.formatVersion < content.MaxFormatVersion {
if s.formatVersion < format.MaxFormatVersion {
env.RunAndExpectSuccess(t, cmd...)
} else {
env.RunAndExpectFailure(t, cmd...)

19
cli/command_repository_status.go
View File

@@ -15,8 +15,7 @@
"github.com/kopia/kopia/internal/units"
"github.com/kopia/kopia/repo"
"github.com/kopia/kopia/repo/blob"
"github.com/kopia/kopia/repo/content"
"github.com/kopia/kopia/repo/object"
"github.com/kopia/kopia/repo/format"
)
type commandRepositoryStatus struct {
@@ -33,12 +32,12 @@ type RepositoryStatus struct {
ConfigFile string `json:"configFile"`
UniqueIDHex string `json:"uniqueIDHex"`
ClientOptions repo.ClientOptions `json:"clientOptions"`
Storage blob.ConnectionInfo `json:"storage"`
Capacity *blob.Capacity `json:"volume,omitempty"`
ContentFormat content.FormattingOptions `json:"contentFormat"`
ObjectFormat object.Format `json:"objectFormat"`
BlobRetention content.BlobCfgBlob `json:"blobRetention"`
ClientOptions repo.ClientOptions `json:"clientOptions"`
Storage blob.ConnectionInfo `json:"storage"`
Capacity *blob.Capacity `json:"volume,omitempty"`
ContentFormat format.ContentFormat `json:"contentFormat"`
ObjectFormat format.ObjectFormat `json:"objectFormat"`
BlobRetention format.BlobStorageConfiguration `json:"blobRetention"`
}
func (c *commandRepositoryStatus) setup(svc advancedAppServices, parent commandParent) {
@@ -64,8 +63,8 @@ func (c *commandRepositoryStatus) outputJSON(ctx context.Context, r repo.Reposit
s.UniqueIDHex = hex.EncodeToString(dr.UniqueID())
s.ObjectFormat = dr.ObjectFormat()
s.BlobRetention = dr.BlobCfg()
s.Storage = scrubber.ScrubSensitiveData(reflect.ValueOf(ci)).Interface().(blob.ConnectionInfo) // nolint:forcetypeassert
s.ContentFormat = scrubber.ScrubSensitiveData(reflect.ValueOf(dr.ContentReader().ContentFormat().Struct())).Interface().(content.FormattingOptions) // nolint:forcetypeassert
s.Storage = scrubber.ScrubSensitiveData(reflect.ValueOf(ci)).Interface().(blob.ConnectionInfo) // nolint:forcetypeassert
s.ContentFormat = scrubber.ScrubSensitiveData(reflect.ValueOf(dr.ContentReader().ContentFormat().Struct())).Interface().(format.ContentFormat) // nolint:forcetypeassert
switch cp, err := dr.BlobVolume().GetCapacity(ctx); {
case err == nil:

7
cli/command_repository_sync.go
View File

@@ -18,6 +18,7 @@
"github.com/kopia/kopia/internal/units"
"github.com/kopia/kopia/repo"
"github.com/kopia/kopia/repo/blob"
"github.com/kopia/kopia/repo/format"
)
type commandRepositorySyncTo struct {
@@ -345,21 +346,21 @@ func (c *commandRepositorySyncTo) ensureRepositoriesHaveSameFormatBlob(ctx conte
var srcData gather.WriteBuffer
defer srcData.Close()
if err := src.GetBlob(ctx, repo.FormatBlobID, 0, -1, &srcData); err != nil {
if err := src.GetBlob(ctx, format.KopiaRepositoryBlobID, 0, -1, &srcData); err != nil {
return errors.Wrap(err, "error reading format blob")
}
var dstData gather.WriteBuffer
defer dstData.Close()
if err := dst.GetBlob(ctx, repo.FormatBlobID, 0, -1, &dstData); err != nil {
if err := dst.GetBlob(ctx, format.KopiaRepositoryBlobID, 0, -1, &dstData); err != nil {
// target does not have format blob, save it there first.
if errors.Is(err, blob.ErrBlobNotFound) {
if c.repositorySyncDestinationMustExist {
return errors.Errorf("destination repository does not have a format blob")
}
return errors.Wrap(dst.PutBlob(ctx, repo.FormatBlobID, srcData.Bytes(), blob.PutOptions{}), "error saving format blob")
return errors.Wrap(dst.PutBlob(ctx, format.KopiaRepositoryBlobID, srcData.Bytes(), blob.PutOptions{}), "error saving format blob")
}
return errors.Wrap(err, "error reading destination repository format blob")

13
cli/command_repository_upgrade.go
View File

@@ -13,6 +13,7 @@
"github.com/kopia/kopia/internal/epoch"
"github.com/kopia/kopia/repo"
"github.com/kopia/kopia/repo/content"
"github.com/kopia/kopia/repo/format"
)
type commandRepositoryUpgrade struct {
@@ -56,7 +57,7 @@ func (c *commandRepositoryUpgrade) setup(svc advancedAppServices, parent command
beginCmd := parent.Command("begin", "Begin upgrade.").Default()
beginCmd.Flag("advance-notice", "Advance notice for upgrade to allow enough time for other Kopia clients to notice the lock").DurationVar(&c.advanceNoticeDuration)
beginCmd.Flag("io-drain-timeout", "Max time it should take all other Kopia clients to drop repository connections").Default(repo.DefaultRepositoryBlobCacheDuration.String()).DurationVar(&c.ioDrainTimeout)
beginCmd.Flag("io-drain-timeout", "Max time it should take all other Kopia clients to drop repository connections").Default(format.DefaultRepositoryBlobCacheDuration.String()).DurationVar(&c.ioDrainTimeout)
beginCmd.Flag("allow-unsafe-upgrade", "Force using an unsafe io-drain-timeout for the upgrade lock").Default("false").Hidden().BoolVar(&c.force)
beginCmd.Flag("status-poll-interval", "An advisory polling interval to check for the status of upgrade").Default("60s").DurationVar(&c.statusPollInterval)
@@ -117,20 +118,20 @@ func (c *commandRepositoryUpgrade) runPhase(act func(context.Context, repo.Direc
// setLockIntent is an upgrade phase which sets the upgrade lock intent with
// desired parameters.
func (c *commandRepositoryUpgrade) setLockIntent(ctx context.Context, rep repo.DirectRepositoryWriter) error {
if c.ioDrainTimeout < repo.DefaultRepositoryBlobCacheDuration && !c.force {
return errors.Errorf("minimum required io-drain-timeout is %s", repo.DefaultRepositoryBlobCacheDuration)
if c.ioDrainTimeout < format.DefaultRepositoryBlobCacheDuration && !c.force {
return errors.Errorf("minimum required io-drain-timeout is %s", format.DefaultRepositoryBlobCacheDuration)
}
now := rep.Time()
mp := rep.ContentReader().ContentFormat().GetMutableParameters()
openOpts := c.svc.optionsFromFlags(ctx)
l := &repo.UpgradeLockIntent{
l := &format.UpgradeLockIntent{
OwnerID: openOpts.UpgradeOwnerID,
CreationTime: now,
AdvanceNoticeDuration: c.advanceNoticeDuration,
IODrainTimeout: c.ioDrainTimeout,
StatusPollInterval: c.statusPollInterval,
Message: fmt.Sprintf("Upgrading from format version %d -> %d", mp.Version, content.MaxFormatVersion),
Message: fmt.Sprintf("Upgrading from format version %d -> %d", mp.Version, format.MaxFormatVersion),
MaxPermittedClockDrift: MaxPermittedClockDrift(),
}
@@ -219,7 +220,7 @@ func (c *commandRepositoryUpgrade) drainAllClients(ctx context.Context, rep repo
cacheOpts := lc.Caching.CloneOrDefault()
for {
l, err := repo.ReadAndCacheRepoUpgradeLock(ctx, rep.BlobStorage(), password, cacheOpts, -1)
l, err := format.ReadAndCacheRepoUpgradeLock(ctx, rep.BlobStorage(), password, cacheOpts.CacheDirectory, -1)
if err != nil {
return errors.Wrap(err, "unable to reload the repository format blob")
}

10
cli/command_repository_upgrade_test.go
View File

@@ -8,7 +8,7 @@
"github.com/stretchr/testify/require"
"github.com/kopia/kopia/cli"
"github.com/kopia/kopia/repo/content"
"github.com/kopia/kopia/repo/format"
"github.com/kopia/kopia/tests/testenv"
)
@@ -23,7 +23,7 @@ func (s *formatSpecificTestSuite) TestRepositoryUpgrade(t *testing.T) {
cli.MaxPermittedClockDrift = func() time.Duration { return time.Second }
switch s.formatVersion {
case content.FormatVersion1:
case format.FormatVersion1:
require.Contains(t, out, "Format version: 1")
_, stderr := env.RunAndExpectSuccessWithErrOut(t, "repository", "upgrade",
"--upgrade-owner-id", "owner",
@@ -31,7 +31,7 @@ func (s *formatSpecificTestSuite) TestRepositoryUpgrade(t *testing.T) {
"--status-poll-interval", "1s")
require.Contains(t, stderr, "Repository indices have been upgraded.")
require.Contains(t, stderr, "Repository has been successfully upgraded.")
case content.FormatVersion2:
case format.FormatVersion2:
require.Contains(t, out, "Format version: 2")
_, stderr := env.RunAndExpectSuccessWithErrOut(t, "repository", "upgrade",
"--upgrade-owner-id", "owner",
@@ -64,7 +64,7 @@ func (s *formatSpecificTestSuite) TestRepositoryUpgradeAdvanceNotice(t *testing.
cli.MaxPermittedClockDrift = func() time.Duration { return time.Second }
switch s.formatVersion {
case content.FormatVersion1:
case format.FormatVersion1:
require.Contains(t, out, "Format version: 1")
_, stderr := env.RunAndExpectSuccessWithErrOut(t, "repository", "upgrade",
"--upgrade-owner-id", "owner",
@@ -126,7 +126,7 @@ func (s *formatSpecificTestSuite) TestRepositoryUpgradeAdvanceNotice(t *testing.
// verify that non-owner clients can resume access
env.RunAndExpectSuccess(t, "repository", "status", "--upgrade-no-block")
case content.FormatVersion2:
case format.FormatVersion2:
require.Contains(t, out, "Format version: 2")
_, stderr := env.RunAndExpectSuccessWithErrOut(t, "repository", "upgrade",
"--upgrade-owner-id", "owner",

10
cli/suite_test.go
View File

@@ -4,24 +4,24 @@
"testing"
"github.com/kopia/kopia/internal/testutil"
"github.com/kopia/kopia/repo/content"
"github.com/kopia/kopia/repo/format"
)
func TestMain(m *testing.M) { testutil.MyTestMain(m) }
type formatSpecificTestSuite struct {
formatFlags []string
formatVersion content.FormatVersion
formatVersion format.Version
}
func TestFormatV1(t *testing.T) {
testutil.RunAllTestsWithParam(t, &formatSpecificTestSuite{[]string{"--format-version=1"}, content.FormatVersion1})
testutil.RunAllTestsWithParam(t, &formatSpecificTestSuite{[]string{"--format-version=1"}, format.FormatVersion1})
}
func TestFormatV2(t *testing.T) {
testutil.RunAllTestsWithParam(t, &formatSpecificTestSuite{[]string{"--format-version=2"}, content.FormatVersion2})
testutil.RunAllTestsWithParam(t, &formatSpecificTestSuite{[]string{"--format-version=2"}, format.FormatVersion2})
}
func TestFormatV3(t *testing.T) {
testutil.RunAllTestsWithParam(t, &formatSpecificTestSuite{[]string{"--format-version=3"}, content.FormatVersion3})
testutil.RunAllTestsWithParam(t, &formatSpecificTestSuite{[]string{"--format-version=3"}, format.FormatVersion3})
}

6
fs/ignorefs/ignorefs.go
View File

@@ -9,8 +9,8 @@
"github.com/pkg/errors"
"github.com/kopia/kopia/fs"
"github.com/kopia/kopia/internal/cachedir"
"github.com/kopia/kopia/internal/wcmatch"
"github.com/kopia/kopia/repo"
"github.com/kopia/kopia/repo/logging"
"github.com/kopia/kopia/snapshot"
"github.com/kopia/kopia/snapshot/policy"
@@ -79,7 +79,7 @@ type ignoreDirectory struct {
func isCorrectCacheDirSignature(ctx context.Context, f fs.File) error {
const (
validSignature = repo.CacheDirMarkerHeader
validSignature = cachedir.CacheDirMarkerHeader
validSignatureLen = len(validSignature)
)
@@ -112,7 +112,7 @@ func (d *ignoreDirectory) skipCacheDirectory(ctx context.Context, relativePath s
return false
}
e, err := d.Directory.Child(ctx, repo.CacheDirMarkerFile)
e, err := d.Directory.Child(ctx, cachedir.CacheDirMarkerFile)
if err != nil {
return false
}

57
internal/cachedir/cachedir.go Normal file
View File

@@ -0,0 +1,57 @@
// Package cachedir contains utilities for manipulating cache directories.
package cachedir
import (
"os"
"path/filepath"
"github.com/pkg/errors"
)
// CacheDirMarkerFile is the name of the marker file indicating a directory contains Kopia caches.
// See https://bford.info/cachedir/
const CacheDirMarkerFile = "CACHEDIR.TAG"
// CacheDirMarkerHeader is the header signature for cache dir marker files.
const CacheDirMarkerHeader = "Signature: 8a477f597d28d172789f06886806bc55"
const cacheDirMarkerContents = CacheDirMarkerHeader + `
#
# This file is a cache directory tag created by Kopia - Fast And Secure Open-Source Backup.
#
# For information about Kopia, see:
# https://kopia.io
#
# For information about cache directory tags, see:
# http://www.brynosaurus.com/cachedir/
`
// WriteCacheMarker writes the CACHEDIR.TAG marker file in a given directory.
func WriteCacheMarker(cacheDir string) error {
if cacheDir == "" {
return nil
}
markerFile := filepath.Join(cacheDir, CacheDirMarkerFile)
st, err := os.Stat(markerFile)
if err == nil && st.Size() >= int64(len(cacheDirMarkerContents)) {
// ok
return nil
}
if err != nil && !os.IsNotExist(err) {
return errors.Wrap(err, "unexpected cache marker error")
}
f, err := os.Create(markerFile) //nolint:gosec
if err != nil {
return errors.Wrap(err, "error creating cache marker")
}
if _, err := f.WriteString(cacheDirMarkerContents); err != nil {
return errors.Wrap(err, "unable to write cachedir marker contents")
}
return errors.Wrap(f.Close(), "error closing cache marker file")
}

4
internal/remoterepoapi/remoterepoapi.go
View File

@@ -5,8 +5,8 @@
"encoding/json"
"github.com/kopia/kopia/repo/content"
"github.com/kopia/kopia/repo/format"
"github.com/kopia/kopia/repo/manifest"
"github.com/kopia/kopia/repo/object"
)
// Parameters encapsulates all parameters for repository.
@@ -16,7 +16,7 @@ type Parameters struct {
HMACSecret []byte `json:"hmacSecret"`
SupportsContentCompression bool `json:"supportsContentCompression"`
object.Format
format.ObjectFormat
}
// GetHashFunction returns the name of the hash function for remote repository.

14
internal/repotesting/repotesting.go
View File

@@ -16,7 +16,7 @@
"github.com/kopia/kopia/repo/blob"
"github.com/kopia/kopia/repo/content"
"github.com/kopia/kopia/repo/encryption"
"github.com/kopia/kopia/repo/object"
"github.com/kopia/kopia/repo/format"
"github.com/kopia/kopia/snapshot"
)
@@ -47,7 +47,7 @@ func (e *Environment) RootStorage() blob.Storage {
}
// setup sets up a test environment.
func (e *Environment) setup(tb testing.TB, version content.FormatVersion, opts ...Options) *Environment {
func (e *Environment) setup(tb testing.TB, version format.Version, opts ...Options) *Environment {
tb.Helper()
ctx := testlogging.Context(tb)
@@ -55,8 +55,8 @@ func (e *Environment) setup(tb testing.TB, version content.FormatVersion, opts .
openOpt := &repo.Options{}
opt := &repo.NewRepositoryOptions{
BlockFormat: content.FormattingOptions{
MutableParameters: content.MutableParameters{
BlockFormat: format.ContentFormat{
MutableParameters: format.MutableParameters{
Version: version,
},
HMACSecret: []byte{},
@@ -64,7 +64,7 @@ func (e *Environment) setup(tb testing.TB, version content.FormatVersion, opts .
Encryption: encryption.DefaultAlgorithm,
EnablePasswordChange: true,
},
ObjectFormat: object.Format{
ObjectFormat: format.ObjectFormat{
Splitter: "FIXED-1M",
},
}
@@ -267,10 +267,10 @@ func repoOptions(openOpts []func(*repo.Options)) *repo.Options {
}
// FormatNotImportant chooses arbitrary format version where it's not important to the test.
const FormatNotImportant = content.FormatVersion3
const FormatNotImportant = format.FormatVersion3
// NewEnvironment creates a new repository testing environment and ensures its cleanup at the end of the test.
func NewEnvironment(tb testing.TB, version content.FormatVersion, opts ...Options) (context.Context, *Environment) {
func NewEnvironment(tb testing.TB, version format.Version, opts ...Options) (context.Context, *Environment) {
tb.Helper()
ctx := testlogging.Context(tb)

5
internal/server/api_repo.go
View File

@@ -17,6 +17,7 @@
"github.com/kopia/kopia/repo/blob/throttling"
"github.com/kopia/kopia/repo/compression"
"github.com/kopia/kopia/repo/encryption"
"github.com/kopia/kopia/repo/format"
"github.com/kopia/kopia/repo/hashing"
"github.com/kopia/kopia/repo/maintenance"
"github.com/kopia/kopia/repo/splitter"
@@ -36,7 +37,7 @@ func handleRepoParameters(ctx context.Context, rc requestContext) (interface{},
rp := &remoterepoapi.Parameters{
HashFunction: dr.ContentReader().ContentFormat().GetHashFunction(),
HMACSecret: dr.ContentReader().ContentFormat().GetHmacSecret(),
Format: dr.ObjectFormat(),
ObjectFormat: dr.ObjectFormat(),
SupportsContentCompression: dr.ContentReader().SupportsContentCompression(),
}
@@ -174,7 +175,7 @@ func handleRepoExists(ctx context.Context, rc requestContext) (interface{}, *api
var tmp gather.WriteBuffer
defer tmp.Close()
if err := st.GetBlob(ctx, repo.FormatBlobID, 0, -1, &tmp); err != nil {
if err := st.GetBlob(ctx, format.KopiaRepositoryBlobID, 0, -1, &tmp); err != nil {
if errors.Is(err, blob.ErrBlobNotFound) {
return nil, requestError(serverapi.ErrorNotInitialized, "repository not initialized")
}

5
repo/api_server_repository.go
View File

@@ -16,6 +16,7 @@
"github.com/kopia/kopia/internal/remoterepoapi"
"github.com/kopia/kopia/repo/compression"
"github.com/kopia/kopia/repo/content"
"github.com/kopia/kopia/repo/format"
"github.com/kopia/kopia/repo/hashing"
"github.com/kopia/kopia/repo/manifest"
"github.com/kopia/kopia/repo/object"
@@ -33,7 +34,7 @@ type APIServerInfo struct {
type apiServerRepository struct {
cli *apiclient.KopiaAPIClient
h hashing.HashFunc
objectFormat object.Format
objectFormat format.ObjectFormat
serverSupportsContentCompression bool
cliOpts ClientOptions
omgr *object.Manager
@@ -306,7 +307,7 @@ func openRestAPIRepository(ctx context.Context, si *APIServerInfo, cliOpts Clien
}
rr.h = hf
rr.objectFormat = p.Format
rr.objectFormat = p.ObjectFormat
rr.serverSupportsContentCompression = p.SupportsContentCompression
// create object manager using rr as contentManager implementation.

89
repo/blobcfg_blob.go
View File

@@ -1,89 +0,0 @@
package repo
import (
"context"
"encoding/json"
"github.com/pkg/errors"
"github.com/kopia/kopia/internal/gather"
"github.com/kopia/kopia/repo/blob"
"github.com/kopia/kopia/repo/content"
)
// BlobCfgBlobID is the identifier of a BLOB that describes BLOB retention
// settings for the repository.
const BlobCfgBlobID = "kopia.blobcfg"
func blobCfgBlobFromOptions(opt *NewRepositoryOptions) content.BlobCfgBlob {
return content.BlobCfgBlob{
RetentionMode: opt.RetentionMode,
RetentionPeriod: opt.RetentionPeriod,
}
}
func serializeBlobCfgBytes(f *formatBlob, r content.BlobCfgBlob, masterKey []byte) ([]byte, error) {
data, err := json.Marshal(r)
if err != nil {
return nil, errors.Wrap(err, "can't marshal blobCfgBlob to JSON")
}
switch f.EncryptionAlgorithm {
case "NONE":
return data, nil
case aes256GcmEncryption:
return encryptRepositoryBlobBytesAes256Gcm(data, masterKey, f.UniqueID)
default:
return nil, errors.Errorf("unknown encryption algorithm: '%v'", f.EncryptionAlgorithm)
}
}
func deserializeBlobCfgBytes(f *formatBlob, encryptedBlobCfgBytes, masterKey []byte) (content.BlobCfgBlob, error) {
var (
plainText []byte
r content.BlobCfgBlob
err error
)
if encryptedBlobCfgBytes == nil {
return r, nil
}
switch f.EncryptionAlgorithm {
case "NONE": // do nothing
plainText = encryptedBlobCfgBytes
case aes256GcmEncryption:
plainText, err = decryptRepositoryBlobBytesAes256Gcm(encryptedBlobCfgBytes, masterKey, f.UniqueID)
if err != nil {
return content.BlobCfgBlob{}, errors.Errorf("unable to decrypt repository blobcfg blob")
}
default:
return content.BlobCfgBlob{}, errors.Errorf("unknown encryption algorithm: '%v'", f.EncryptionAlgorithm)
}
if err = json.Unmarshal(plainText, &r); err != nil {
return content.BlobCfgBlob{}, errors.Wrap(err, "invalid repository blobcfg blob")
}
return r, nil
}
func writeBlobCfgBlob(ctx context.Context, st blob.Storage, f *formatBlob, blobcfg content.BlobCfgBlob, formatEncryptionKey []byte) error {
blobCfgBytes, err := serializeBlobCfgBytes(f, blobcfg, formatEncryptionKey)
if err != nil {
return errors.Wrap(err, "unable to encrypt blobcfg bytes")
}
if err := st.PutBlob(ctx, BlobCfgBlobID, gather.FromSlice(blobCfgBytes), blob.PutOptions{
RetentionMode: blobcfg.RetentionMode,
RetentionPeriod: blobcfg.RetentionPeriod,
}); err != nil {
return errors.Wrapf(err, "PutBlob() failed for %q", BlobCfgBlobID)
}
return nil
}

20
repo/change_password.go
View File

@@ -6,6 +6,8 @@
"path/filepath"
"github.com/pkg/errors"
"github.com/kopia/kopia/repo/format"
)
// ChangePassword changes the repository password and rewrites
@@ -13,7 +15,7 @@
func (r *directRepository) ChangePassword(ctx context.Context, newPassword string) error {
f := r.formatBlob
repoConfig, err := f.decryptFormatBytes(r.formatEncryptionKey)
repoConfig, err := f.DecryptRepositoryConfig(r.formatEncryptionKey)
if err != nil {
return errors.Wrap(err, "unable to decrypt repository config")
}
@@ -22,33 +24,33 @@ func (r *directRepository) ChangePassword(ctx context.Context, newPassword strin
return errors.Errorf("password changes are not supported for repositories created using Kopia v0.8 or older")
}
newFormatEncryptionKey, err := f.deriveFormatEncryptionKeyFromPassword(newPassword)
newFormatEncryptionKey, err := f.DeriveFormatEncryptionKeyFromPassword(newPassword)
if err != nil {
return errors.Wrap(err, "unable to derive master key")
}
r.formatEncryptionKey = newFormatEncryptionKey
if err := encryptFormatBytes(f, repoConfig, newFormatEncryptionKey, f.UniqueID); err != nil {
if err := f.EncryptRepositoryConfig(repoConfig, newFormatEncryptionKey); err != nil {
return errors.Wrap(err, "unable to encrypt format bytes")
}
if err := writeBlobCfgBlob(ctx, r.blobs, f, r.blobCfgBlob, newFormatEncryptionKey); err != nil {
if err := f.WriteBlobCfgBlob(ctx, r.blobs, r.blobCfgBlob, newFormatEncryptionKey); err != nil {
return errors.Wrap(err, "unable to write blobcfg blob")
}
if err := writeFormatBlob(ctx, r.blobs, f, r.blobCfgBlob); err != nil {
if err := f.WriteKopiaRepositoryBlob(ctx, r.blobs, r.blobCfgBlob); err != nil {
return errors.Wrap(err, "unable to write format blob")
}
// remove cached kopia.repository blob.
if cd := r.cachingOptions.CacheDirectory; cd != "" {
if err := os.Remove(filepath.Join(cd, FormatBlobID)); err != nil {
log(ctx).Errorf("unable to remove %s: %v", FormatBlobID, err)
if err := os.Remove(filepath.Join(cd, format.KopiaRepositoryBlobID)); err != nil {
log(ctx).Errorf("unable to remove %s: %v", format.KopiaRepositoryBlobID, err)
}
if err := os.Remove(filepath.Join(cd, BlobCfgBlobID)); err != nil && !os.IsNotExist(err) {
log(ctx).Errorf("unable to remove %s: %v", BlobCfgBlobID, err)
if err := os.Remove(filepath.Join(cd, format.KopiaBlobCfgBlobID)); err != nil && !os.IsNotExist(err) {
log(ctx).Errorf("unable to remove %s: %v", format.KopiaBlobCfgBlobID, err)
}
}

6
repo/connect.go
View File

@@ -10,6 +10,7 @@
"github.com/kopia/kopia/internal/ospath"
"github.com/kopia/kopia/repo/blob"
"github.com/kopia/kopia/repo/content"
"github.com/kopia/kopia/repo/format"
)
// ConnectOptions specifies options when persisting configuration to connect to a repository.
@@ -32,7 +33,7 @@ func Connect(ctx context.Context, configFile string, st blob.Storage, password s
var formatBytes gather.WriteBuffer
defer formatBytes.Close()
if err := st.GetBlob(ctx, FormatBlobID, 0, -1, &formatBytes); err != nil {
if err := st.GetBlob(ctx, format.KopiaRepositoryBlobID, 0, -1, &formatBytes); err != nil {
if errors.Is(err, blob.ErrBlobNotFound) {
return ErrRepositoryNotInitialized
}
@@ -40,8 +41,9 @@ func Connect(ctx context.Context, configFile string, st blob.Storage, password s
return errors.Wrap(err, "unable to read format blob")
}
f, err := parseFormatBlob(formatBytes.ToByteSlice())
f, err := format.ParseKopiaRepositoryJSON(formatBytes.ToByteSlice())
if err != nil {
// nolint:wrapcheck
return err
}

5
repo/content/blob_crypto_test.go
View File

@@ -9,11 +9,12 @@
"github.com/kopia/kopia/internal/gather"
"github.com/kopia/kopia/repo/encryption"
"github.com/kopia/kopia/repo/format"
"github.com/kopia/kopia/repo/hashing"
)
func TestBlobCrypto(t *testing.T) {
f := &FormattingOptions{
f := &format.ContentFormat{
Hash: hashing.DefaultAlgorithm,
Encryption: encryption.DefaultAlgorithm,
}
@@ -85,7 +86,7 @@ func(output []byte, data gather.Bytes) []byte {
_, err := EncryptBLOB(cr, gather.FromSlice([]byte{1, 2, 3}), "n", "mysessionid", &tmp)
require.Error(t, err)
f := &FormattingOptions{
f := &format.ContentFormat{
Hash: hashing.DefaultAlgorithm,
Encryption: encryption.DefaultAlgorithm,
}

5
repo/content/committed_read_manager.go
View File

@@ -21,6 +21,7 @@
"github.com/kopia/kopia/repo/blob/filesystem"
"github.com/kopia/kopia/repo/blob/sharded"
"github.com/kopia/kopia/repo/compression"
"github.com/kopia/kopia/repo/format"
"github.com/kopia/kopia/repo/hashing"
"github.com/kopia/kopia/repo/logging"
)
@@ -98,7 +99,7 @@ type SharedManager struct {
// +checklocks:indexesLock
refreshIndexesAfter time.Time
format FormattingOptionsProvider
format format.Provider
checkInvariantsOnUnlock bool
minPreambleLength int
@@ -537,7 +538,7 @@ func (sm *SharedManager) shouldRefreshIndexes() bool {
}
// NewSharedManager returns SharedManager that is used by SessionWriteManagers on top of a repository.
func NewSharedManager(ctx context.Context, st blob.Storage, prov FormattingOptionsProvider, caching *CachingOptions, opts *ManagerOptions) (*SharedManager, error) {
func NewSharedManager(ctx context.Context, st blob.Storage, prov format.Provider, caching *CachingOptions, opts *ManagerOptions) (*SharedManager, error) {
opts = opts.CloneOrDefault()
if opts.TimeNow == nil {
opts.TimeNow = clock.Now

11
repo/content/content_formatter_test.go
View File

@@ -15,6 +15,7 @@
"github.com/kopia/kopia/internal/testlogging"
"github.com/kopia/kopia/repo/blob"
"github.com/kopia/kopia/repo/encryption"
"github.com/kopia/kopia/repo/format"
"github.com/kopia/kopia/repo/hashing"
)
@@ -33,7 +34,7 @@ func TestFormatters(t *testing.T) {
t.Run(encryptionAlgo, func(t *testing.T) {
ctx := testlogging.Context(t)
fo := &FormattingOptions{
fo := &format.ContentFormat{
HMACSecret: secret,
MasterKey: make([]byte, 32),
Hash: hashAlgo,
@@ -77,11 +78,11 @@ func verifyEndToEndFormatter(ctx context.Context, t *testing.T, hashAlgo, encryp
keyTime := map[blob.ID]time.Time{}
st := blobtesting.NewMapStorage(data, keyTime, nil)
bm, err := NewManagerForTesting(testlogging.Context(t), st, mustCreateFormatProvider(t, &FormattingOptions{
bm, err := NewManagerForTesting(testlogging.Context(t), st, mustCreateFormatProvider(t, &format.ContentFormat{
Hash: hashAlgo,
Encryption: encryptionAlgo,
HMACSecret: hmacSecret,
MutableParameters: MutableParameters{
MutableParameters: format.MutableParameters{
Version: 1,
MaxPackSize: maxPackSize,
},
@@ -137,10 +138,10 @@ func verifyEndToEndFormatter(ctx context.Context, t *testing.T, hashAlgo, encryp
}
}
func mustCreateFormatProvider(t *testing.T, f *FormattingOptions) FormattingOptionsProvider {
func mustCreateFormatProvider(t *testing.T, f *format.ContentFormat) format.Provider {
t.Helper()
fop, err := NewFormattingOptionsProvider(f, nil)
fop, err := format.NewFormattingOptionsProvider(f, nil)
require.NoError(t, err)
return fop

320
repo/content/content_formatting_options.go
View File

@@ -1,320 +0,0 @@
package content
import (
"time"
"github.com/pkg/errors"
"github.com/kopia/kopia/internal/epoch"
"github.com/kopia/kopia/internal/gather"
"github.com/kopia/kopia/internal/units"
"github.com/kopia/kopia/repo/blob"
"github.com/kopia/kopia/repo/content/index"
"github.com/kopia/kopia/repo/encryption"
"github.com/kopia/kopia/repo/hashing"
)
const (
minValidPackSize = 10 << 20
maxValidPackSize = 120 << 20
)
// FormatVersion denotes content format version.
type FormatVersion int
// Supported format versions.
const (
FormatVersion1 FormatVersion = 1
FormatVersion2 FormatVersion = 2 // new in v0.9
FormatVersion3 FormatVersion = 3 // new in v0.11
MaxFormatVersion = FormatVersion3
)
// FormattingOptions describes the rules for formatting contents in repository.
type FormattingOptions struct {
Hash string `json:"hash,omitempty"` // identifier of the hash algorithm used
Encryption string `json:"encryption,omitempty"` // identifier of the encryption algorithm used
HMACSecret []byte `json:"secret,omitempty" kopia:"sensitive"` // HMAC secret used to generate encryption keys
MasterKey []byte `json:"masterKey,omitempty" kopia:"sensitive"` // master encryption key (SIV-mode encryption only)
MutableParameters
EnablePasswordChange bool `json:"enablePasswordChange"` // disables replication of kopia.repository blob in packs
}
// ResolveFormatVersion applies format options parameters based on the format version.
func (f *FormattingOptions) ResolveFormatVersion() error {
switch f.Version {
case FormatVersion2, FormatVersion3:
f.EnablePasswordChange = true
f.IndexVersion = index.Version2
f.EpochParameters = epoch.DefaultParameters()
return nil
case FormatVersion1:
f.EnablePasswordChange = false
f.IndexVersion = index.Version1
f.EpochParameters = epoch.Parameters{}
return nil
default:
return errors.Errorf("Unsupported format version: %v", f.Version)
}
}
// GetMutableParameters implements FormattingOptionsProvider.
func (f *FormattingOptions) GetMutableParameters() MutableParameters {
return f.MutableParameters
}
// SupportsPasswordChange implements FormattingOptionsProvider.
func (f *FormattingOptions) SupportsPasswordChange() bool {
return f.EnablePasswordChange
}
// MutableParameters represents parameters of the content manager that can be mutated after the repository
// is created.
type MutableParameters struct {
Version FormatVersion `json:"version,omitempty"` // version number, must be "1", "2" or "3"
MaxPackSize int `json:"maxPackSize,omitempty"` // maximum size of a pack object
IndexVersion int `json:"indexVersion,omitempty"` // force particular index format version (1,2,..)
EpochParameters epoch.Parameters `json:"epochParameters,omitempty"` // epoch manager parameters
}
// Validate validates the parameters.
func (v *MutableParameters) Validate() error {
if v.MaxPackSize < minValidPackSize {
return errors.Errorf("max pack size too small, must be >= %v", units.BytesStringBase2(minValidPackSize))
}
if v.MaxPackSize > maxValidPackSize {
return errors.Errorf("max pack size too big, must be <= %v", units.BytesStringBase2(maxValidPackSize))
}
if v.IndexVersion < 0 || v.IndexVersion > index.Version2 {
return errors.Errorf("invalid index version, supported versions are 1 & 2")
}
if err := v.EpochParameters.Validate(); err != nil {
return errors.Wrap(err, "invalid epoch parameters")
}
return nil
}
// GetEncryptionAlgorithm implements encryption.Parameters.
func (f *FormattingOptions) GetEncryptionAlgorithm() string {
return f.Encryption
}
// GetMasterKey implements encryption.Parameters.
func (f *FormattingOptions) GetMasterKey() []byte {
return f.MasterKey
}
// GetHashFunction implements hashing.Parameters.
func (f *FormattingOptions) GetHashFunction() string {
return f.Hash
}
// GetHmacSecret implements hashing.Parameters.
func (f *FormattingOptions) GetHmacSecret() []byte {
return f.HMACSecret
}
// FormattingOptionsProvider provides current formatting options. The options returned
// should not be cached for more than a few seconds as they are subject to change.
type FormattingOptionsProvider interface {
epoch.ParametersProvider
MaxIndexBlobSize() int64
WriteIndexVersion() int
IndexShardSize() int
encryption.Parameters
hashing.Parameters
HashFunc() hashing.HashFunc
Encryptor() encryption.Encryptor
GetMutableParameters() MutableParameters
GetMasterKey() []byte
SupportsPasswordChange() bool
FormatVersion() FormatVersion
MaxPackBlobSize() int
RepositoryFormatBytes() []byte
Struct() FormattingOptions
}
type formattingOptionsProvider struct {
*FormattingOptions
h hashing.HashFunc
e encryption.Encryptor
actualFormatVersion FormatVersion
actualIndexVersion int
formatBytes []byte
}
func (f *formattingOptionsProvider) FormatVersion() FormatVersion {
return f.Version
}
// whether epoch manager is enabled, must be true.
func (f *formattingOptionsProvider) GetEpochManagerEnabled() bool {
return f.EpochParameters.Enabled
}
// how frequently each client will list blobs to determine the current epoch.
func (f *formattingOptionsProvider) GetEpochRefreshFrequency() time.Duration {
return f.EpochParameters.EpochRefreshFrequency
}
// number of epochs between full checkpoints.
func (f *formattingOptionsProvider) GetEpochFullCheckpointFrequency() int {
return f.EpochParameters.FullCheckpointFrequency
}
// GetEpochCleanupSafetyMargin returns safety margin to prevent uncompacted blobs from being deleted if the corresponding compacted blob age is less than this.
func (f *formattingOptionsProvider) GetEpochCleanupSafetyMargin() time.Duration {
return f.EpochParameters.CleanupSafetyMargin
}
// GetMinEpochDuration returns the minimum duration of an epoch.
func (f *formattingOptionsProvider) GetMinEpochDuration() time.Duration {
return f.EpochParameters.MinEpochDuration
}
// GetEpochAdvanceOnCountThreshold returns the number of files above which epoch should be advanced.
func (f *formattingOptionsProvider) GetEpochAdvanceOnCountThreshold() int {
return f.EpochParameters.EpochAdvanceOnCountThreshold
}
// GetEpochAdvanceOnTotalSizeBytesThreshold returns the total size of files above which the epoch should be advanced.
func (f *formattingOptionsProvider) GetEpochAdvanceOnTotalSizeBytesThreshold() int64 {
return f.EpochParameters.EpochAdvanceOnTotalSizeBytesThreshold
}
// GetEpochDeleteParallelism returns the number of blobs to delete in parallel during cleanup.
func (f *formattingOptionsProvider) GetEpochDeleteParallelism() int {
return f.EpochParameters.DeleteParallelism
}
func (f *formattingOptionsProvider) Struct() FormattingOptions {
return *f.FormattingOptions
}
// NewFormattingOptionsProvider validates the provided formatting options and returns static
// FormattingOptionsProvider based on them.
func NewFormattingOptionsProvider(f *FormattingOptions, formatBytes []byte) (FormattingOptionsProvider, error) {
formatVersion := f.Version
if formatVersion < minSupportedReadVersion || formatVersion > currentWriteVersion {
return nil, errors.Errorf("can't handle repositories created using version %v (min supported %v, max supported %v)", formatVersion, minSupportedReadVersion, maxSupportedReadVersion)
}
if formatVersion < minSupportedWriteVersion || formatVersion > currentWriteVersion {
return nil, errors.Errorf("can't handle repositories created using version %v (min supported %v, max supported %v)", formatVersion, minSupportedWriteVersion, maxSupportedWriteVersion)
}
actualIndexVersion := f.IndexVersion
if actualIndexVersion == 0 {
actualIndexVersion = legacyIndexVersion
}
if actualIndexVersion < index.Version1 || actualIndexVersion > index.Version2 {
return nil, errors.Errorf("index version %v is not supported", actualIndexVersion)
}
h, err := hashing.CreateHashFunc(f)
if err != nil {
return nil, errors.Wrap(err, "unable to create hash")
}
e, err := encryption.CreateEncryptor(f)
if err != nil {
return nil, errors.Wrap(err, "unable to create encryptor")
}
contentID := h(nil, gather.FromSlice(nil))
var tmp gather.WriteBuffer
defer tmp.Close()
err = e.Encrypt(gather.FromSlice(nil), contentID, &tmp)
if err != nil {
return nil, errors.Wrap(err, "invalid encryptor")
}
return &formattingOptionsProvider{
FormattingOptions: f,
h: h,
e: e,
actualIndexVersion: actualIndexVersion,
actualFormatVersion: f.Version,
formatBytes: formatBytes,
}, nil
}
func (f *formattingOptionsProvider) Encryptor() encryption.Encryptor {
return f.e
}
func (f *formattingOptionsProvider) HashFunc() hashing.HashFunc {
return f.h
}
func (f *formattingOptionsProvider) WriteIndexVersion() int {
return f.actualIndexVersion
}
func (f *formattingOptionsProvider) MaxIndexBlobSize() int64 {
return int64(f.MaxPackSize)
}
func (f *formattingOptionsProvider) MaxPackBlobSize() int {
return f.MaxPackSize
}
func (f *formattingOptionsProvider) GetEpochManagerParameters() epoch.Parameters {
return f.EpochParameters
}
func (f *formattingOptionsProvider) IndexShardSize() int {
return defaultIndexShardSize
}
func (f *formattingOptionsProvider) RepositoryFormatBytes() []byte {
return f.formatBytes
}
var _ FormattingOptionsProvider = (*formattingOptionsProvider)(nil)
// BlobCfgBlob is the content for `kopia.blobcfg` blob which contains the blob
// management configuration options.
type BlobCfgBlob struct {
RetentionMode blob.RetentionMode `json:"retentionMode,omitempty"`
RetentionPeriod time.Duration `json:"retentionPeriod,omitempty"`
}
// IsRetentionEnabled returns true if retention is enabled on the blob-config
// object.
func (r *BlobCfgBlob) IsRetentionEnabled() bool {
return r.RetentionMode != "" && r.RetentionPeriod != 0
}
// Validate validates the blob config parameters.
func (r *BlobCfgBlob) Validate() error {
if (r.RetentionMode == "") != (r.RetentionPeriod == 0) {
return errors.Errorf("both retention mode and period must be provided when setting blob retention properties")
}
if r.RetentionPeriod != 0 && r.RetentionPeriod < 24*time.Hour {
return errors.Errorf("invalid retention-period, the minimum required is 1-day and there is no maximum limit")
}
return nil
}

17
repo/content/content_manager.go
View File

@@ -19,6 +19,7 @@
"github.com/kopia/kopia/repo/blob"
"github.com/kopia/kopia/repo/compression"
"github.com/kopia/kopia/repo/content/index"
"github.com/kopia/kopia/repo/format"
"github.com/kopia/kopia/repo/hashing"
"github.com/kopia/kopia/repo/logging"
)
@@ -34,11 +35,7 @@
packBlobIDLength = 16
defaultIndexShardSize = 16e6 // slightly less than 2^24, which lets index use 24-bit/3-byte indexes
DefaultIndexVersion = 2
legacyIndexVersion = index.Version1
)
var tracer = otel.Tracer("kopia/content")
@@ -57,14 +54,6 @@
defaultMaxPreambleLength = 32
defaultPaddingUnit = 4096
currentWriteVersion = FormatVersion3
minSupportedWriteVersion = FormatVersion1
maxSupportedWriteVersion = FormatVersion3
minSupportedReadVersion = FormatVersion1
maxSupportedReadVersion = FormatVersion3
indexLoadAttempts = 10
)
@@ -589,7 +578,7 @@ func removePendingPack(slice []*pendingPackInfo, pp *pendingPackInfo) []*pending
}
// ContentFormat returns formatting options.
func (bm *WriteManager) ContentFormat() FormattingOptionsProvider {
func (bm *WriteManager) ContentFormat() format.Provider {
return bm.format
}
@@ -935,7 +924,7 @@ func (o *ManagerOptions) CloneOrDefault() *ManagerOptions {
}
// NewManagerForTesting creates new content manager with given packing options and a formatter.
func NewManagerForTesting(ctx context.Context, st blob.Storage, f FormattingOptionsProvider, caching *CachingOptions, options *ManagerOptions) (*WriteManager, error) {
func NewManagerForTesting(ctx context.Context, st blob.Storage, f format.Provider, caching *CachingOptions, options *ManagerOptions) (*WriteManager, error) {
options = options.CloneOrDefault()
if options.TimeNow == nil {
options.TimeNow = clock.Now

17
repo/content/content_manager_test.go
View File

@@ -33,6 +33,7 @@
"github.com/kopia/kopia/repo/blob/logging"
"github.com/kopia/kopia/repo/compression"
"github.com/kopia/kopia/repo/content/index"
"github.com/kopia/kopia/repo/format"
)
const (
@@ -52,7 +53,7 @@ func TestMain(m *testing.M) { testutil.MyTestMain(m) }
func TestFormatV1(t *testing.T) {
testutil.RunAllTestsWithParam(t, &contentManagerSuite{
mutableParameters: MutableParameters{
mutableParameters: format.MutableParameters{
Version: 1,
IndexVersion: 1,
MaxPackSize: maxPackSize,
@@ -62,7 +63,7 @@ func TestFormatV1(t *testing.T) {
func TestFormatV2(t *testing.T) {
testutil.RunAllTestsWithParam(t, &contentManagerSuite{
mutableParameters: MutableParameters{
mutableParameters: format.MutableParameters{
Version: 2,
MaxPackSize: maxPackSize,
IndexVersion: index.Version2,
@@ -72,7 +73,7 @@ func TestFormatV2(t *testing.T) {
}
type contentManagerSuite struct {
mutableParameters MutableParameters
mutableParameters format.MutableParameters
}
func (s *contentManagerSuite) TestContentManagerEmptyFlush(t *testing.T) {
@@ -354,7 +355,7 @@ func (s *contentManagerSuite) TestContentManagerFailedToWritePack(t *testing.T)
ta := faketime.NewTimeAdvance(fakeTime, 0)
bm, err := NewManagerForTesting(testlogging.Context(t), st, mustCreateFormatProvider(t, &FormattingOptions{
bm, err := NewManagerForTesting(testlogging.Context(t), st, mustCreateFormatProvider(t, &format.ContentFormat{
Hash: "HMAC-SHA256-128",
Encryption: "AES256-GCM-HMAC-SHA256",
MutableParameters: s.mutableParameters,
@@ -1860,7 +1861,7 @@ func (s *contentManagerSuite) TestContentReadAliasing(t *testing.T) {
}
func (s *contentManagerSuite) TestVersionCompatibility(t *testing.T) {
for writeVer := minSupportedReadVersion; writeVer <= currentWriteVersion; writeVer++ {
for writeVer := format.MinSupportedReadVersion; writeVer <= format.CurrentWriteVersion; writeVer++ {
writeVer := writeVer
t.Run(fmt.Sprintf("version-%v", writeVer), func(t *testing.T) {
s.verifyVersionCompat(t, writeVer)
@@ -1868,7 +1869,7 @@ func (s *contentManagerSuite) TestVersionCompatibility(t *testing.T) {
}
}
func (s *contentManagerSuite) verifyVersionCompat(t *testing.T, writeVersion FormatVersion) {
func (s *contentManagerSuite) verifyVersionCompat(t *testing.T, writeVersion format.Version) {
t.Helper()
ctx := testlogging.Context(t)
@@ -2333,7 +2334,7 @@ type contentManagerTestTweaks struct {
indexVersion int
maxPackSize int
formatVersion FormatVersion
formatVersion format.Version
}
func (s *contentManagerSuite) newTestContentManagerWithTweaks(t *testing.T, st blob.Storage, tweaks *contentManagerTestTweaks) *WriteManager {
@@ -2363,7 +2364,7 @@ func (s *contentManagerSuite) newTestContentManagerWithTweaks(t *testing.T, st b
}
ctx := testlogging.Context(t)
fo := mustCreateFormatProvider(t, &FormattingOptions{
fo := mustCreateFormatProvider(t, &format.ContentFormat{
Hash: "HMAC-SHA256",
Encryption: "AES256-GCM-HMAC-SHA256",
HMACSecret: hmacSecret,

3
repo/content/content_reader.go
View File

@@ -4,12 +4,13 @@
"context"
"github.com/kopia/kopia/internal/epoch"
"github.com/kopia/kopia/repo/format"
)
// Reader defines content read API.
type Reader interface {
SupportsContentCompression() bool
ContentFormat() FormattingOptionsProvider
ContentFormat() format.Provider
GetContent(ctx context.Context, id ID) ([]byte, error)
ContentInfo(ctx context.Context, id ID) (Info, error)
IterateContents(ctx context.Context, opts IterateOptions, callback IterateCallback) error

3
repo/content/encrypted_blob_mgr_test.go
View File

@@ -11,6 +11,7 @@
"github.com/kopia/kopia/internal/testlogging"
"github.com/kopia/kopia/repo/blob"
"github.com/kopia/kopia/repo/encryption"
"github.com/kopia/kopia/repo/format"
"github.com/kopia/kopia/repo/hashing"
"github.com/kopia/kopia/repo/logging"
)
@@ -28,7 +29,7 @@ func TestEncryptedBlobManager(t *testing.T) {
data := blobtesting.DataMap{}
st := blobtesting.NewMapStorage(data, nil, nil)
fs := blobtesting.NewFaultyStorage(st)
f := &FormattingOptions{
f := &format.ContentFormat{
Hash: hashing.DefaultAlgorithm,
Encryption: encryption.DefaultAlgorithm,
}

3
repo/content/index_blob_manager_v0_test.go
View File

@@ -24,6 +24,7 @@
"github.com/kopia/kopia/repo/blob"
"github.com/kopia/kopia/repo/blob/logging"
"github.com/kopia/kopia/repo/encryption"
"github.com/kopia/kopia/repo/format"
"github.com/kopia/kopia/repo/hashing"
)
@@ -765,7 +766,7 @@ func assertIndexBlobList(t *testing.T, m *indexBlobManagerV0, wantMD ...blob.Met
func newIndexBlobManagerForTesting(t *testing.T, st blob.Storage, localTimeNow func() time.Time) *indexBlobManagerV0 {
t.Helper()
p := &FormattingOptions{
p := &format.ContentFormat{
Encryption: encryption.DefaultAlgorithm,
Hash: hashing.DefaultAlgorithm,
}

110
repo/format/blobcfg_blob.go Normal file
View File

@@ -0,0 +1,110 @@
package format
import (
"context"
"encoding/json"
"time"
"github.com/pkg/errors"
"github.com/kopia/kopia/internal/gather"
"github.com/kopia/kopia/repo/blob"
)
// KopiaBlobCfgBlobID is the identifier of a BLOB that describes BLOB retention
// settings for the repository.
const KopiaBlobCfgBlobID = "kopia.blobcfg"
// BlobStorageConfiguration is the content for `kopia.blobcfg` blob which contains the blob
// storage configuration options.
type BlobStorageConfiguration struct {
RetentionMode blob.RetentionMode `json:"retentionMode,omitempty"`
RetentionPeriod time.Duration `json:"retentionPeriod,omitempty"`
}
// IsRetentionEnabled returns true if retention is enabled on the blob-config
// object.
func (r *BlobStorageConfiguration) IsRetentionEnabled() bool {
return r.RetentionMode != "" && r.RetentionPeriod != 0
}
// Validate validates the blob config parameters.
func (r *BlobStorageConfiguration) Validate() error {
if (r.RetentionMode == "") != (r.RetentionPeriod == 0) {
return errors.Errorf("both retention mode and period must be provided when setting blob retention properties")
}
if r.RetentionPeriod != 0 && r.RetentionPeriod < 24*time.Hour {
return errors.Errorf("invalid retention-period, the minimum required is 1-day and there is no maximum limit")
}
return nil
}
func serializeBlobCfgBytes(f *KopiaRepositoryJSON, r BlobStorageConfiguration, formatEncryptionKey []byte) ([]byte, error) {
data, err := json.Marshal(r)
if err != nil {
return nil, errors.Wrap(err, "can't marshal blobCfgBlob to JSON")
}
switch f.EncryptionAlgorithm {
case "NONE":
return data, nil
case aes256GcmEncryption:
return encryptRepositoryBlobBytesAes256Gcm(data, formatEncryptionKey, f.UniqueID)
default:
return nil, errors.Errorf("unknown encryption algorithm: '%v'", f.EncryptionAlgorithm)
}
}
// DeserializeBlobCfgBytes decrypts and deserializes the given bytes into BlobStorageConfiguration.
func (f *KopiaRepositoryJSON) DeserializeBlobCfgBytes(encryptedBlobCfgBytes, formatEncryptionKey []byte) (BlobStorageConfiguration, error) {
var (
plainText []byte
r BlobStorageConfiguration
err error
)
if encryptedBlobCfgBytes == nil {
return r, nil
}
switch f.EncryptionAlgorithm {
case "NONE": // do nothing
plainText = encryptedBlobCfgBytes
case aes256GcmEncryption:
plainText, err = decryptRepositoryBlobBytesAes256Gcm(encryptedBlobCfgBytes, formatEncryptionKey, f.UniqueID)
if err != nil {
return BlobStorageConfiguration{}, errors.Errorf("unable to decrypt repository blobcfg blob")
}
default:
return BlobStorageConfiguration{}, errors.Errorf("unknown encryption algorithm: '%v'", f.EncryptionAlgorithm)
}
if err = json.Unmarshal(plainText, &r); err != nil {
return BlobStorageConfiguration{}, errors.Wrap(err, "invalid repository blobcfg blob")
}
return r, nil
}
// WriteBlobCfgBlob writes `kopia.blobcfg` encrypted using the provided key.
func (f *KopiaRepositoryJSON) WriteBlobCfgBlob(ctx context.Context, st blob.Storage, blobcfg BlobStorageConfiguration, formatEncryptionKey []byte) error {
blobCfgBytes, err := serializeBlobCfgBytes(f, blobcfg, formatEncryptionKey)
if err != nil {
return errors.Wrap(err, "unable to encrypt blobcfg bytes")
}
if err := st.PutBlob(ctx, KopiaBlobCfgBlobID, gather.FromSlice(blobCfgBytes), blob.PutOptions{
RetentionMode: blobcfg.RetentionMode,
RetentionPeriod: blobcfg.RetentionPeriod,
}); err != nil {
return errors.Wrapf(err, "PutBlob() failed for %q", KopiaBlobCfgBlobID)
}
return nil
}

102
repo/format/content_format.go Normal file
View File

@@ -0,0 +1,102 @@
package format
import (
"github.com/pkg/errors"
"github.com/kopia/kopia/internal/epoch"
"github.com/kopia/kopia/internal/units"
"github.com/kopia/kopia/repo/content/index"
)
// ContentFormat describes the rules for formatting contents in repository.
type ContentFormat struct {
Hash string `json:"hash,omitempty"` // identifier of the hash algorithm used
Encryption string `json:"encryption,omitempty"` // identifier of the encryption algorithm used
HMACSecret []byte `json:"secret,omitempty" kopia:"sensitive"` // HMAC secret used to generate encryption keys
MasterKey []byte `json:"masterKey,omitempty" kopia:"sensitive"` // master encryption key (SIV-mode encryption only)
MutableParameters
EnablePasswordChange bool `json:"enablePasswordChange"` // disables replication of kopia.repository blob in packs
}
// ResolveFormatVersion applies format options parameters based on the format version.
func (f *ContentFormat) ResolveFormatVersion() error {
switch f.Version {
case FormatVersion2, FormatVersion3:
f.EnablePasswordChange = true
f.IndexVersion = index.Version2
f.EpochParameters = epoch.DefaultParameters()
return nil
case FormatVersion1:
f.EnablePasswordChange = false
f.IndexVersion = index.Version1
f.EpochParameters = epoch.Parameters{}
return nil
default:
return errors.Errorf("Unsupported format version: %v", f.Version)
}
}
// GetMutableParameters implements FormattingOptionsProvider.
func (f *ContentFormat) GetMutableParameters() MutableParameters {
return f.MutableParameters
}
// SupportsPasswordChange implements FormattingOptionsProvider.
func (f *ContentFormat) SupportsPasswordChange() bool {
return f.EnablePasswordChange
}
// MutableParameters represents parameters of the content manager that can be mutated after the repository
// is created.
type MutableParameters struct {
Version Version `json:"version,omitempty"` // version number, must be "1", "2" or "3"
MaxPackSize int `json:"maxPackSize,omitempty"` // maximum size of a pack object
IndexVersion int `json:"indexVersion,omitempty"` // force particular index format version (1,2,..)
EpochParameters epoch.Parameters `json:"epochParameters,omitempty"` // epoch manager parameters
}
// Validate validates the parameters.
func (v *MutableParameters) Validate() error {
if v.MaxPackSize < minValidPackSize {
return errors.Errorf("max pack size too small, must be >= %v", units.BytesStringBase2(minValidPackSize))
}
if v.MaxPackSize > maxValidPackSize {
return errors.Errorf("max pack size too big, must be <= %v", units.BytesStringBase2(maxValidPackSize))
}
if v.IndexVersion < 0 || v.IndexVersion > index.Version2 {
return errors.Errorf("invalid index version, supported versions are 1 & 2")
}
if err := v.EpochParameters.Validate(); err != nil {
return errors.Wrap(err, "invalid epoch parameters")
}
return nil
}
// GetEncryptionAlgorithm implements encryption.Parameters.
func (f *ContentFormat) GetEncryptionAlgorithm() string {
return f.Encryption
}
// GetMasterKey implements encryption.Parameters.
func (f *ContentFormat) GetMasterKey() []byte {
return f.MasterKey
}
// GetHashFunction implements hashing.Parameters.
func (f *ContentFormat) GetHashFunction() string {
return f.Hash
}
// GetHmacSecret implements hashing.Parameters.
func (f *ContentFormat) GetHmacSecret() []byte {
return f.HMACSecret
}

6
repo/crypto_key_derivation.go → repo/format/crypto_key_derivation.go
View File

@@ -1,4 +1,4 @@
package repo
package format
import (
"crypto/sha256"
@@ -7,8 +7,8 @@
"golang.org/x/crypto/hkdf"
)
// deriveKeyFromMasterKey computes a key for a specific purpose and length using HKDF based on the master key.
func deriveKeyFromMasterKey(masterKey, uniqueID, purpose []byte, length int) []byte {
// DeriveKeyFromMasterKey computes a key for a specific purpose and length using HKDF based on the master key.
func DeriveKeyFromMasterKey(masterKey, uniqueID, purpose []byte, length int) []byte {
key := make([]byte, length)
k := hkdf.New(sha256.New, masterKey, uniqueID, purpose)

9
repo/crypto_key_derivation_nontest.go → repo/format/crypto_key_derivation_nontest.go
View File

@@ -1,17 +1,18 @@
//go:build !testing
// +build !testing
package repo
package format
import (
"github.com/pkg/errors"
"golang.org/x/crypto/scrypt"
)
// defaultKeyDerivationAlgorithm is the key derivation algorithm for new configurations.
const defaultKeyDerivationAlgorithm = "scrypt-65536-8-1"
// DefaultKeyDerivationAlgorithm is the key derivation algorithm for new configurations.
const DefaultKeyDerivationAlgorithm = "scrypt-65536-8-1"
func (f *formatBlob) deriveFormatEncryptionKeyFromPassword(password string) ([]byte, error) {
// DeriveFormatEncryptionKeyFromPassword derives encryption key using the provided password and per-repository unique ID.
func (f *KopiaRepositoryJSON) DeriveFormatEncryptionKeyFromPassword(password string) ([]byte, error) {
const masterKeySize = 32
switch f.KeyDerivationAlgorithm {

11
repo/crypto_key_derivation_testing.go → repo/format/crypto_key_derivation_testing.go
View File

@@ -1,7 +1,7 @@
//go:build testing
// +build testing
package repo
package format
import (
"crypto/sha256"
@@ -9,14 +9,15 @@
"github.com/pkg/errors"
)
// defaultKeyDerivationAlgorithm is the key derivation algorithm for new configurations.
const defaultKeyDerivationAlgorithm = "testing-only-insecure"
// DefaultKeyDerivationAlgorithm is the key derivation algorithm for new configurations.
const DefaultKeyDerivationAlgorithm = "testing-only-insecure"
func (f *formatBlob) deriveFormatEncryptionKeyFromPassword(password string) ([]byte, error) {
// DeriveFormatEncryptionKeyFromPassword derives encryption key using the provided password and per-repository unique ID.
func (f *KopiaRepositoryJSON) DeriveFormatEncryptionKeyFromPassword(password string) ([]byte, error) {
const masterKeySize = 32
switch f.KeyDerivationAlgorithm {
case defaultKeyDerivationAlgorithm:
case DefaultKeyDerivationAlgorithm:
h := sha256.New()
if _, err := h.Write([]byte(password)); err != nil {
return nil, err

82
repo/format_block.go → repo/format/format_blob.go
View File

@@ -1,4 +1,5 @@
package repo
// Package format manages kopia.repository and other central format blobs.
package format
import (
"context"
@@ -14,12 +15,13 @@
"github.com/kopia/kopia/internal/gather"
"github.com/kopia/kopia/repo/blob"
"github.com/kopia/kopia/repo/content"
)
// DefaultFormatEncryption is the identifier of the default format blob encryption algorithm.
const DefaultFormatEncryption = "AES256_GCM"
const (
aes256GcmEncryption = "AES256_GCM"
defaultFormatEncryption = "AES256_GCM"
lengthOfRecoverBlockLength = 2 // number of bytes used to store recover block length
maxChecksummedFormatBytesLength = 65000
maxRecoverChunkLength = 65536
@@ -27,8 +29,11 @@
formatBlobChecksumSize = sha256.Size
)
// FormatBlobID is the identifier of a BLOB that describes repository format.
const FormatBlobID = "kopia.repository"
// KopiaRepositoryBlobID is the identifier of a BLOB that describes repository format.
const KopiaRepositoryBlobID = "kopia.repository"
// ErrInvalidPassword is returned when repository password is invalid.
var ErrInvalidPassword = errors.Errorf("invalid repository password")
// nolint:gochecknoglobals
var (
@@ -43,7 +48,8 @@
errFormatBlobNotFound = errors.New("format blob not found")
)
type formatBlob struct {
// KopiaRepositoryJSON represents JSON contents of 'kopia.repository' blob.
type KopiaRepositoryJSON struct {
Tool string `json:"tool"`
BuildVersion string `json:"buildVersion"`
BuildInfo string `json:"buildInfo"`
@@ -56,13 +62,9 @@ type formatBlob struct {
EncryptedFormatBytes []byte `json:"encryptedBlockFormat,omitempty"`
}
// encryptedRepositoryConfig contains the configuration of repository that's persisted in encrypted format.
type encryptedRepositoryConfig struct {
Format repositoryObjectFormat `json:"format"`
}
func parseFormatBlob(b []byte) (*formatBlob, error) {
f := &formatBlob{}
// ParseKopiaRepositoryJSON parses the provided byte slice into KopiaRepositoryJSON.
func ParseKopiaRepositoryJSON(b []byte) (*KopiaRepositoryJSON, error) {
f := &KopiaRepositoryJSON{}
if err := json.Unmarshal(b, &f); err != nil {
return nil, errors.Wrap(err, "invalid format blob")
@@ -163,11 +165,13 @@ func verifyFormatBlobChecksum(b []byte) ([]byte, bool) {
return data, true
}
func writeFormatBlob(ctx context.Context, st blob.Storage, f *formatBlob, blobCfg content.BlobCfgBlob) error {
return writeFormatBlobWithID(ctx, st, f, blobCfg, FormatBlobID)
// WriteKopiaRepositoryBlob writes `kopia.repository` blob to a given storage.
func (f *KopiaRepositoryJSON) WriteKopiaRepositoryBlob(ctx context.Context, st blob.Storage, blobCfg BlobStorageConfiguration) error {
return f.WriteKopiaRepositoryBlobWithID(ctx, st, blobCfg, KopiaRepositoryBlobID)
}
func writeFormatBlobWithID(ctx context.Context, st blob.Storage, f *formatBlob, blobCfg content.BlobCfgBlob, id blob.ID) error {
// WriteKopiaRepositoryBlobWithID writes `kopia.repository` blob to a given storage under an alternate blobID.
func (f *KopiaRepositoryJSON) WriteKopiaRepositoryBlobWithID(ctx context.Context, st blob.Storage, blobCfg BlobStorageConfiguration, id blob.ID) error {
buf := gather.NewWriteBuffer()
e := json.NewEncoder(buf)
e.SetIndent("", " ")
@@ -186,29 +190,9 @@ func writeFormatBlobWithID(ctx context.Context, st blob.Storage, f *formatBlob,
return nil
}
func (f *formatBlob) decryptFormatBytes(masterKey []byte) (*repositoryObjectFormat, error) {
switch f.EncryptionAlgorithm {
case aes256GcmEncryption:
plainText, err := decryptRepositoryBlobBytesAes256Gcm(f.EncryptedFormatBytes, masterKey, f.UniqueID)
if err != nil {
return nil, errors.Errorf("unable to decrypt repository format")
}
var erc encryptedRepositoryConfig
if err := json.Unmarshal(plainText, &erc); err != nil {
return nil, errors.Wrap(err, "invalid repository format")
}
return &erc.Format, nil
default:
return nil, errors.Errorf("unknown encryption algorithm: '%v'", f.EncryptionAlgorithm)
}
}
func initCrypto(masterKey, repositoryID []byte) (cipher.AEAD, []byte, error) {
aesKey := deriveKeyFromMasterKey(masterKey, repositoryID, purposeAESKey, 32) // nolint:gomnd
authData := deriveKeyFromMasterKey(masterKey, repositoryID, purposeAuthData, 32) // nolint:gomnd
aesKey := DeriveKeyFromMasterKey(masterKey, repositoryID, purposeAESKey, 32) // nolint:gomnd
authData := DeriveKeyFromMasterKey(masterKey, repositoryID, purposeAuthData, 32) // nolint:gomnd
blk, err := aes.NewCipher(aesKey)
if err != nil {
@@ -267,28 +251,6 @@ func decryptRepositoryBlobBytesAes256Gcm(data, masterKey, repositoryID []byte) (
return plainText, nil
}
func encryptFormatBytes(f *formatBlob, format *repositoryObjectFormat, masterKey, repositoryID []byte) error {
switch f.EncryptionAlgorithm {
case aes256GcmEncryption:
data, err := json.Marshal(&encryptedRepositoryConfig{Format: *format})
if err != nil {
return errors.Wrap(err, "can't marshal format to JSON")
}
data, err = encryptRepositoryBlobBytesAes256Gcm(data, masterKey, repositoryID)
if err != nil {
return errors.Wrap(err, "failed to encrypt format JSON")
}
f.EncryptedFormatBytes = data
return nil
default:
return errors.Errorf("unknown encryption algorithm: '%v'", f.EncryptionAlgorithm)
}
}
func addFormatBlobChecksumAndLength(fb []byte) ([]byte, error) {
h := hmac.New(sha256.New, formatBlobChecksumSecret)
h.Write(fb)

144
repo/format/format_blob_cache.go Normal file
View File

@@ -0,0 +1,144 @@
package format
import (
"context"
"os"
"path/filepath"
"time"
"github.com/pkg/errors"
"github.com/kopia/kopia/internal/atomicfile"
"github.com/kopia/kopia/internal/cache"
"github.com/kopia/kopia/internal/cachedir"
"github.com/kopia/kopia/internal/clock"
"github.com/kopia/kopia/internal/gather"
"github.com/kopia/kopia/repo/blob"
"github.com/kopia/kopia/repo/logging"
)
// DefaultRepositoryBlobCacheDuration is the duration for which we treat cached kopia.repository
// as valid.
const DefaultRepositoryBlobCacheDuration = 15 * time.Minute
var log = logging.Module("kopia/repo/format")
func formatBytesCachingEnabled(cacheDirectory string, validDuration time.Duration) bool {
if cacheDirectory == "" {
return false
}
return validDuration > 0
}
func readRepositoryBlobBytesFromCache(ctx context.Context, cachedFile string, validDuration time.Duration) (data []byte, cacheMTime time.Time, err error) {
cst, err := os.Stat(cachedFile)
if err != nil {
return nil, time.Time{}, errors.Wrap(err, "unable to open cache file")
}
cacheMTime = cst.ModTime()
if clock.Now().Sub(cacheMTime) > validDuration {
// got cached file, but it's too old, remove it
if err = os.Remove(cachedFile); err != nil {
log(ctx).Debugf("unable to remove cache file: %v", err)
}
return nil, time.Time{}, errors.Errorf("cached file too old")
}
data, err = os.ReadFile(cachedFile) // nolint:gosec
if err != nil {
return nil, time.Time{}, errors.Wrapf(err, "failed to read the cache file %q", cachedFile)
}
return data, cacheMTime, nil
}
// ReadAndCacheRepositoryBlobBytes reads the provided blob from the repository or cache directory.
func ReadAndCacheRepositoryBlobBytes(ctx context.Context, st blob.Storage, cacheDirectory, blobID string, validDuration time.Duration) ([]byte, time.Time, error) {
cachedFile := filepath.Join(cacheDirectory, blobID)
if validDuration == 0 {
validDuration = DefaultRepositoryBlobCacheDuration
}
if cacheDirectory != "" {
if err := os.MkdirAll(cacheDirectory, cache.DirMode); err != nil && !os.IsExist(err) {
log(ctx).Errorf("unable to create cache directory: %v", err)
}
}
cacheEnabled := formatBytesCachingEnabled(cacheDirectory, validDuration)
if cacheEnabled {
data, cacheMTime, err := readRepositoryBlobBytesFromCache(ctx, cachedFile, validDuration)
if err == nil {
log(ctx).Debugf("%s retrieved from cache", blobID)
return data, cacheMTime, nil
}
if os.IsNotExist(err) {
log(ctx).Debugf("%s could not be fetched from cache: %v", blobID, err)
}
} else {
log(ctx).Debugf("%s cache not enabled", blobID)
}
var b gather.WriteBuffer
defer b.Close()
if err := st.GetBlob(ctx, blob.ID(blobID), 0, -1, &b); err != nil {
return nil, time.Time{}, errors.Wrapf(err, "error getting %s blob", blobID)
}
if cacheEnabled {
if err := atomicfile.Write(cachedFile, b.Bytes().Reader()); err != nil {
log(ctx).Warnf("unable to write cache: %v", err)
}
}
return b.ToByteSlice(), clock.Now(), nil
}
// ReadAndCacheDecodedRepositoryConfig reads `kopia.repository` blob, potentially from cache and decodes it.
func ReadAndCacheDecodedRepositoryConfig(ctx context.Context, st blob.Storage, password, cacheDir string, validDuration time.Duration) (ufb *DecodedRepositoryConfig, err error) {
ufb = &DecodedRepositoryConfig{}
ufb.KopiaRepositoryBytes, ufb.CacheMTime, err = ReadAndCacheRepositoryBlobBytes(ctx, st, cacheDir, KopiaRepositoryBlobID, validDuration)
if err != nil {
return nil, errors.Wrap(err, "unable to read format blob")
}
if err = cachedir.WriteCacheMarker(cacheDir); err != nil {
return nil, errors.Wrap(err, "unable to write cache directory marker")
}
ufb.KopiaRepository, err = ParseKopiaRepositoryJSON(ufb.KopiaRepositoryBytes)
if err != nil {
return nil, errors.Wrap(err, "can't parse format blob")
}
ufb.KopiaRepositoryBytes, err = addFormatBlobChecksumAndLength(ufb.KopiaRepositoryBytes)
if err != nil {
return nil, errors.Errorf("unable to add checksum")
}
ufb.FormatEncryptionKey, err = ufb.KopiaRepository.DeriveFormatEncryptionKeyFromPassword(password)
if err != nil {
return nil, err
}
ufb.RepoConfig, err = ufb.KopiaRepository.DecryptRepositoryConfig(ufb.FormatEncryptionKey)
if err != nil {
return nil, ErrInvalidPassword
}
return ufb, nil
}
// ReadAndCacheRepoUpgradeLock loads the lock config from cache and returns it.
func ReadAndCacheRepoUpgradeLock(ctx context.Context, st blob.Storage, password, cacheDir string, validDuration time.Duration) (*UpgradeLockIntent, error) {
ufb, err := ReadAndCacheDecodedRepositoryConfig(ctx, st, password, cacheDir, validDuration)
return ufb.RepoConfig.UpgradeLock, err
}

2
repo/format_block_test.go → repo/format/format_blob_test.go
View File

@@ -1,4 +1,4 @@
package repo
package format
import (
"crypto/sha256"

218
repo/format/format_provider.go Normal file
View File

@@ -0,0 +1,218 @@
package format
import (
"time"
"github.com/pkg/errors"
"github.com/kopia/kopia/internal/epoch"
"github.com/kopia/kopia/internal/gather"
"github.com/kopia/kopia/repo/content/index"
"github.com/kopia/kopia/repo/encryption"
"github.com/kopia/kopia/repo/hashing"
)
const (
minValidPackSize = 10 << 20
maxValidPackSize = 120 << 20
defaultIndexShardSize = 16e6 // slightly less than 2^24, which lets index use 24-bit/3-byte indexes
// CurrentWriteVersion is the version of the repository applied to new repositories.
CurrentWriteVersion = FormatVersion3
// MinSupportedWriteVersion is the minimum version that this kopia client can write.
MinSupportedWriteVersion = FormatVersion1
// MaxSupportedWriteVersion is the maximum version that this kopia client can write.
MaxSupportedWriteVersion = FormatVersion3
// MinSupportedReadVersion is the minimum version that this kopia client can read.
MinSupportedReadVersion = FormatVersion1
// MaxSupportedReadVersion is the maximum version that this kopia client can read.
MaxSupportedReadVersion = FormatVersion3
legacyIndexVersion = index.Version1
)
// Version denotes content format version.
type Version int
// Supported format versions.
const (
FormatVersion1 Version = 1
FormatVersion2 Version = 2 // new in v0.9
FormatVersion3 Version = 3 // new in v0.11
MaxFormatVersion = FormatVersion3
)
// Provider provides current formatting options. The options returned
// should not be cached for more than a few seconds as they are subject to change.
type Provider interface {
epoch.ParametersProvider
MaxIndexBlobSize() int64
WriteIndexVersion() int
IndexShardSize() int
encryption.Parameters
hashing.Parameters
HashFunc() hashing.HashFunc
Encryptor() encryption.Encryptor
GetMutableParameters() MutableParameters
GetMasterKey() []byte
SupportsPasswordChange() bool
FormatVersion() Version
MaxPackBlobSize() int
RepositoryFormatBytes() []byte
Struct() ContentFormat
}
type formattingOptionsProvider struct {
*ContentFormat
h hashing.HashFunc
e encryption.Encryptor
actualFormatVersion Version
actualIndexVersion int
formatBytes []byte
}
func (f *formattingOptionsProvider) FormatVersion() Version {
return f.Version
}
// whether epoch manager is enabled, must be true.
func (f *formattingOptionsProvider) GetEpochManagerEnabled() bool {
return f.EpochParameters.Enabled
}
// how frequently each client will list blobs to determine the current epoch.
func (f *formattingOptionsProvider) GetEpochRefreshFrequency() time.Duration {
return f.EpochParameters.EpochRefreshFrequency
}
// number of epochs between full checkpoints.
func (f *formattingOptionsProvider) GetEpochFullCheckpointFrequency() int {
return f.EpochParameters.FullCheckpointFrequency
}
// GetEpochCleanupSafetyMargin returns safety margin to prevent uncompacted blobs from being deleted if the corresponding compacted blob age is less than this.
func (f *formattingOptionsProvider) GetEpochCleanupSafetyMargin() time.Duration {
return f.EpochParameters.CleanupSafetyMargin
}
// GetMinEpochDuration returns the minimum duration of an epoch.
func (f *formattingOptionsProvider) GetMinEpochDuration() time.Duration {
return f.EpochParameters.MinEpochDuration
}
// GetEpochAdvanceOnCountThreshold returns the number of files above which epoch should be advanced.
func (f *formattingOptionsProvider) GetEpochAdvanceOnCountThreshold() int {
return f.EpochParameters.EpochAdvanceOnCountThreshold
}
// GetEpochAdvanceOnTotalSizeBytesThreshold returns the total size of files above which the epoch should be advanced.
func (f *formattingOptionsProvider) GetEpochAdvanceOnTotalSizeBytesThreshold() int64 {
return f.EpochParameters.EpochAdvanceOnTotalSizeBytesThreshold
}
// GetEpochDeleteParallelism returns the number of blobs to delete in parallel during cleanup.
func (f *formattingOptionsProvider) GetEpochDeleteParallelism() int {
return f.EpochParameters.DeleteParallelism
}
func (f *formattingOptionsProvider) Struct() ContentFormat {
return *f.ContentFormat
}
// NewFormattingOptionsProvider validates the provided formatting options and returns static
// FormattingOptionsProvider based on them.
func NewFormattingOptionsProvider(f *ContentFormat, formatBytes []byte) (Provider, error) {
formatVersion := f.Version
if formatVersion < MinSupportedReadVersion || formatVersion > CurrentWriteVersion {
return nil, errors.Errorf("can't handle repositories created using version %v (min supported %v, max supported %v)", formatVersion, MinSupportedReadVersion, MaxSupportedReadVersion)
}
if formatVersion < MinSupportedWriteVersion || formatVersion > CurrentWriteVersion {
return nil, errors.Errorf("can't handle repositories created using version %v (min supported %v, max supported %v)", formatVersion, MinSupportedWriteVersion, MaxSupportedWriteVersion)
}
actualIndexVersion := f.IndexVersion
if actualIndexVersion == 0 {
actualIndexVersion = legacyIndexVersion
}
if actualIndexVersion < index.Version1 || actualIndexVersion > index.Version2 {
return nil, errors.Errorf("index version %v is not supported", actualIndexVersion)
}
h, err := hashing.CreateHashFunc(f)
if err != nil {
return nil, errors.Wrap(err, "unable to create hash")
}
e, err := encryption.CreateEncryptor(f)
if err != nil {
return nil, errors.Wrap(err, "unable to create encryptor")
}
contentID := h(nil, gather.FromSlice(nil))
var tmp gather.WriteBuffer
defer tmp.Close()
err = e.Encrypt(gather.FromSlice(nil), contentID, &tmp)
if err != nil {
return nil, errors.Wrap(err, "invalid encryptor")
}
return &formattingOptionsProvider{
ContentFormat: f,
h: h,
e: e,
actualIndexVersion: actualIndexVersion,
actualFormatVersion: f.Version,
formatBytes: formatBytes,
}, nil
}
func (f *formattingOptionsProvider) Encryptor() encryption.Encryptor {
return f.e
}
func (f *formattingOptionsProvider) HashFunc() hashing.HashFunc {
return f.h
}
func (f *formattingOptionsProvider) WriteIndexVersion() int {
return f.actualIndexVersion
}
func (f *formattingOptionsProvider) MaxIndexBlobSize() int64 {
return int64(f.MaxPackSize)
}
func (f *formattingOptionsProvider) MaxPackBlobSize() int {
return f.MaxPackSize
}
func (f *formattingOptionsProvider) GetEpochManagerParameters() epoch.Parameters {
return f.EpochParameters
}
func (f *formattingOptionsProvider) IndexShardSize() int {
return defaultIndexShardSize
}
func (f *formattingOptionsProvider) RepositoryFormatBytes() []byte {
return f.formatBytes
}
var _ Provider = (*formattingOptionsProvider)(nil)

6
repo/format/object_format.go Normal file
View File

@@ -0,0 +1,6 @@
package format
// ObjectFormat describes the format of objects in a repository.
type ObjectFormat struct {
Splitter string `json:"splitter,omitempty"` // splitter used to break objects into pieces of content
}

78
repo/format/repository_config.go Normal file
View File

@@ -0,0 +1,78 @@
package format
import (
"encoding/json"
"time"
"github.com/pkg/errors"
"github.com/kopia/kopia/internal/feature"
)
// RepositoryConfig describes the format of objects in a repository.
// The contents of this object are stored encrypted since they contain sensitive key material.
type RepositoryConfig struct {
ContentFormat
ObjectFormat
UpgradeLock *UpgradeLockIntent `json:"upgradeLock,omitempty"`
RequiredFeatures []feature.Required `json:"requiredFeatures,omitempty"`
}
// EncryptedRepositoryConfig contains the configuration of repository that's persisted in encrypted format.
type EncryptedRepositoryConfig struct {
Format RepositoryConfig `json:"format"`
}
// DecodedRepositoryConfig encapsulates contents of decoded `kopia.repository` blob.
type DecodedRepositoryConfig struct {
KopiaRepository *KopiaRepositoryJSON
KopiaRepositoryBytes []byte // serialized format blob
CacheMTime time.Time // mod time of the format blob cache file
RepoConfig *RepositoryConfig // unencrypted format blob structure
FormatEncryptionKey []byte // key derived from the password
}
// DecryptRepositoryConfig decrypts RepositoryConfig stored in EncryptedFormatBytes.
func (f *KopiaRepositoryJSON) DecryptRepositoryConfig(masterKey []byte) (*RepositoryConfig, error) {
switch f.EncryptionAlgorithm {
case aes256GcmEncryption:
plainText, err := decryptRepositoryBlobBytesAes256Gcm(f.EncryptedFormatBytes, masterKey, f.UniqueID)
if err != nil {
return nil, errors.Errorf("unable to decrypt repository format")
}
var erc EncryptedRepositoryConfig
if err := json.Unmarshal(plainText, &erc); err != nil {
return nil, errors.Wrap(err, "invalid repository format")
}
return &erc.Format, nil
default:
return nil, errors.Errorf("unknown encryption algorithm: '%v'", f.EncryptionAlgorithm)
}
}
// EncryptRepositoryConfig encrypts the provided repository config and stores it in EncryptedFormatBytes.
func (f *KopiaRepositoryJSON) EncryptRepositoryConfig(format *RepositoryConfig, masterKey []byte) error {
switch f.EncryptionAlgorithm {
case aes256GcmEncryption:
data, err := json.Marshal(&EncryptedRepositoryConfig{Format: *format})
if err != nil {
return errors.Wrap(err, "can't marshal format to JSON")
}
data, err = encryptRepositoryBlobBytesAes256Gcm(data, masterKey, f.UniqueID)
if err != nil {
return errors.Wrap(err, "failed to encrypt format JSON")
}
f.EncryptedFormatBytes = data
return nil
default:
return errors.Errorf("unknown encryption algorithm: '%v'", f.EncryptionAlgorithm)
}
}

2
repo/upgrade_lock_intent.go → repo/format/upgrade_lock_intent.go
View File

@@ -1,4 +1,4 @@
package repo
package format
import (
"time"

30
repo/upgrade_lock_intent_test.go → repo/format/upgrade_lock_intent_test.go
View File

@@ -1,4 +1,4 @@
package repo_test
package format_test
import (
"fmt"
@@ -8,11 +8,11 @@
"github.com/stretchr/testify/require"
"github.com/kopia/kopia/internal/clock"
"github.com/kopia/kopia/repo"
"github.com/kopia/kopia/repo/format"
)
func TestUpgradeLockIntentUpdatesWithAdvanceNotice(t *testing.T) {
oldLock := repo.UpgradeLockIntent{
oldLock := format.UpgradeLockIntent{
OwnerID: "upgrade-owner",
CreationTime: clock.Now(),
AdvanceNoticeDuration: time.Hour,
@@ -57,7 +57,7 @@ func TestUpgradeLockIntentUpdatesWithAdvanceNotice(t *testing.T) {
}
func TestUpgradeLockIntentUpdatesWithoutAdvanceNotice(t *testing.T) {
oldLock := repo.UpgradeLockIntent{
oldLock := format.UpgradeLockIntent{
OwnerID: "upgrade-owner",
CreationTime: clock.Now(),
AdvanceNoticeDuration: 0, /* no advance notice */
@@ -77,7 +77,7 @@ func TestUpgradeLockIntentUpdatesWithoutAdvanceNotice(t *testing.T) {
}
func TestUpgradeLockIntentValidation(t *testing.T) {
var l repo.UpgradeLockIntent
var l format.UpgradeLockIntent
require.EqualError(t, l.Validate(), "no owner-id set, it is required to set a unique owner-id")
l.OwnerID = "new-owner"
@@ -116,7 +116,7 @@ func TestUpgradeLockIntentValidation(t *testing.T) {
func TestUpgradeLockIntentImmediateLock(t *testing.T) {
now := clock.Now()
var l *repo.UpgradeLockIntent
var l *format.UpgradeLockIntent
// checking lock status on nil lock
locked, writersDrained := l.IsLocked(now)
@@ -127,7 +127,7 @@ func TestUpgradeLockIntentImmediateLock(t *testing.T) {
require.PanicsWithValue(t,
"writers have drained but we are not locked, this is not possible until the upgrade-lock intent is invalid",
func() {
tmp := repo.UpgradeLockIntent{
tmp := format.UpgradeLockIntent{
OwnerID: "",
CreationTime: now,
AdvanceNoticeDuration: 1 * time.Hour,
@@ -139,7 +139,7 @@ func() {
tmp.IsLocked(now.Add(2 * time.Hour))
})
l = &repo.UpgradeLockIntent{
l = &format.UpgradeLockIntent{
OwnerID: "upgrade-owner",
CreationTime: now,
AdvanceNoticeDuration: 0, /* no advance notice */
@@ -182,7 +182,7 @@ func() {
func TestUpgradeLockIntentSufficientAdvanceLock(t *testing.T) {
now := clock.Now()
l := repo.UpgradeLockIntent{
l := format.UpgradeLockIntent{
OwnerID: "upgrade-owner",
CreationTime: now,
AdvanceNoticeDuration: 6 * time.Hour,
@@ -249,7 +249,7 @@ func TestUpgradeLockIntentSufficientAdvanceLock(t *testing.T) {
func TestUpgradeLockIntentInSufficientAdvanceLock(t *testing.T) {
now := clock.Now()
l := repo.UpgradeLockIntent{
l := format.UpgradeLockIntent{
OwnerID: "upgrade-owner",
CreationTime: now,
AdvanceNoticeDuration: 20 * time.Minute, /* insufficient time to drain the writers */
@@ -288,12 +288,12 @@ func TestUpgradeLockIntentInSufficientAdvanceLock(t *testing.T) {
func TestUpgradeLockIntentUpgradeTime(t *testing.T) {
now := clock.Now()
var l repo.UpgradeLockIntent
var l format.UpgradeLockIntent
// checking time on nil lock
require.Equal(t, time.Time{}, l.UpgradeTime())
l = repo.UpgradeLockIntent{
l = format.UpgradeLockIntent{
OwnerID: "upgrade-owner",
CreationTime: now,
AdvanceNoticeDuration: 20 * time.Minute, /* insufficient time to drain the writers */
@@ -304,7 +304,7 @@ func TestUpgradeLockIntentUpgradeTime(t *testing.T) {
}
require.Equal(t, now.Add(l.MaxPermittedClockDrift+2*l.IODrainTimeout), l.UpgradeTime())
l = repo.UpgradeLockIntent{
l = format.UpgradeLockIntent{
OwnerID: "upgrade-owner",
CreationTime: now,
AdvanceNoticeDuration: 20 * time.Hour, /* sufficient time to drain the writers */
@@ -315,7 +315,7 @@ func TestUpgradeLockIntentUpgradeTime(t *testing.T) {
}
require.Equal(t, now.Add(l.AdvanceNoticeDuration), l.UpgradeTime())
l = repo.UpgradeLockIntent{
l = format.UpgradeLockIntent{
OwnerID: "upgrade-owner",
CreationTime: now,
AdvanceNoticeDuration: 0, /* immediate lock */
@@ -328,7 +328,7 @@ func TestUpgradeLockIntentUpgradeTime(t *testing.T) {
}
func TestUpgradeLockIntentClone(t *testing.T) {
l := &repo.UpgradeLockIntent{
l := &format.UpgradeLockIntent{
OwnerID: "upgrade-owner",
CreationTime: clock.Now(),
AdvanceNoticeDuration: 20 * time.Minute,

5
repo/grpc_repository_client.go
View File

@@ -25,6 +25,7 @@
"github.com/kopia/kopia/repo/blob"
"github.com/kopia/kopia/repo/compression"
"github.com/kopia/kopia/repo/content"
"github.com/kopia/kopia/repo/format"
"github.com/kopia/kopia/repo/hashing"
"github.com/kopia/kopia/repo/manifest"
"github.com/kopia/kopia/repo/object"
@@ -74,7 +75,7 @@ type grpcRepositoryClient struct {
asyncWritesWG errgroup.Group
h hashing.HashFunc
objectFormat object.Format
objectFormat format.ObjectFormat
serverSupportsContentCompression bool
cliOpts ClientOptions
omgr *object.Manager
@@ -906,7 +907,7 @@ func newGRPCAPIRepositoryForConnection(ctx context.Context, conn *grpc.ClientCon
rr.h = hf
rr.objectFormat = object.Format{
rr.objectFormat = format.ObjectFormat{
Splitter: p.Splitter,
}

63
repo/initialize.go
View File

@@ -13,8 +13,8 @@
"github.com/kopia/kopia/repo/blob"
"github.com/kopia/kopia/repo/content"
"github.com/kopia/kopia/repo/encryption"
"github.com/kopia/kopia/repo/format"
"github.com/kopia/kopia/repo/hashing"
"github.com/kopia/kopia/repo/object"
"github.com/kopia/kopia/repo/splitter"
)
@@ -35,12 +35,12 @@
// NewRepositoryOptions specifies options that apply to newly created repositories.
// All fields are optional, when not provided, reasonable defaults will be used.
type NewRepositoryOptions struct {
UniqueID []byte `json:"uniqueID"` // force the use of particular unique ID
BlockFormat content.FormattingOptions `json:"blockFormat"`
DisableHMAC bool `json:"disableHMAC"`
ObjectFormat object.Format `json:"objectFormat"` // object format
RetentionMode blob.RetentionMode `json:"retentionMode,omitempty"`
RetentionPeriod time.Duration `json:"retentionPeriod,omitempty"`
UniqueID []byte `json:"uniqueID"` // force the use of particular unique ID
BlockFormat format.ContentFormat `json:"blockFormat"`
DisableHMAC bool `json:"disableHMAC"`
ObjectFormat format.ObjectFormat `json:"objectFormat"` // object format
RetentionMode blob.RetentionMode `json:"retentionMode,omitempty"`
RetentionPeriod time.Duration `json:"retentionPeriod,omitempty"`
}
// ErrAlreadyInitialized indicates that repository has already been initialized.
@@ -56,7 +56,7 @@ func Initialize(ctx context.Context, st blob.Storage, opt *NewRepositoryOptions,
var tmp gather.WriteBuffer
defer tmp.Close()
err := st.GetBlob(ctx, FormatBlobID, 0, -1, &tmp)
err := st.GetBlob(ctx, format.KopiaRepositoryBlobID, 0, -1, &tmp)
if err == nil {
return ErrAlreadyInitialized
}
@@ -65,7 +65,7 @@ func Initialize(ctx context.Context, st blob.Storage, opt *NewRepositoryOptions,
return errors.Wrap(err, "unexpected error when checking for format blob")
}
err = st.GetBlob(ctx, BlobCfgBlobID, 0, -1, &tmp)
err = st.GetBlob(ctx, format.KopiaBlobCfgBlobID, 0, -1, &tmp)
if err == nil {
return errors.Errorf("possible corruption: blobcfg blob exists, but format blob is not found")
}
@@ -74,10 +74,10 @@ func Initialize(ctx context.Context, st blob.Storage, opt *NewRepositoryOptions,
return errors.Wrap(err, "unexpected error when checking for blobcfg blob")
}
format := formatBlobFromOptions(opt)
formatBlob := formatBlobFromOptions(opt)
blobcfg := blobCfgBlobFromOptions(opt)
formatEncryptionKey, err := format.deriveFormatEncryptionKeyFromPassword(password)
formatEncryptionKey, err := formatBlob.DeriveFormatEncryptionKeyFromPassword(password)
if err != nil {
return errors.Wrap(err, "unable to derive format encryption key")
}
@@ -91,54 +91,61 @@ func Initialize(ctx context.Context, st blob.Storage, opt *NewRepositoryOptions,
return errors.Wrap(err, "invalid parameters")
}
if err = encryptFormatBytes(format, f, formatEncryptionKey, format.UniqueID); err != nil {
if err = formatBlob.EncryptRepositoryConfig(f, formatEncryptionKey); err != nil {
return errors.Wrap(err, "unable to encrypt format bytes")
}
if err := writeBlobCfgBlob(ctx, st, format, blobcfg, formatEncryptionKey); err != nil {
if err := formatBlob.WriteBlobCfgBlob(ctx, st, blobcfg, formatEncryptionKey); err != nil {
return errors.Wrap(err, "unable to write blobcfg blob")
}
if err := writeFormatBlob(ctx, st, format, blobcfg); err != nil {
if err := formatBlob.WriteKopiaRepositoryBlob(ctx, st, blobcfg); err != nil {
return errors.Wrap(err, "unable to write format blob")
}
return nil
}
func formatBlobFromOptions(opt *NewRepositoryOptions) *formatBlob {
return &formatBlob{
func formatBlobFromOptions(opt *NewRepositoryOptions) *format.KopiaRepositoryJSON {
return &format.KopiaRepositoryJSON{
Tool: "https://github.com/kopia/kopia",
BuildInfo: BuildInfo,
BuildVersion: BuildVersion,
KeyDerivationAlgorithm: defaultKeyDerivationAlgorithm,
KeyDerivationAlgorithm: format.DefaultKeyDerivationAlgorithm,
UniqueID: applyDefaultRandomBytes(opt.UniqueID, uniqueIDLength),
EncryptionAlgorithm: defaultFormatEncryption,
EncryptionAlgorithm: format.DefaultFormatEncryption,
}
}
func repositoryObjectFormatFromOptions(opt *NewRepositoryOptions) (*repositoryObjectFormat, error) {
func blobCfgBlobFromOptions(opt *NewRepositoryOptions) format.BlobStorageConfiguration {
return format.BlobStorageConfiguration{
RetentionMode: opt.RetentionMode,
RetentionPeriod: opt.RetentionPeriod,
}
}
func repositoryObjectFormatFromOptions(opt *NewRepositoryOptions) (*format.RepositoryConfig, error) {
fv := opt.BlockFormat.Version
if fv == 0 {
switch os.Getenv("KOPIA_REPOSITORY_FORMAT_VERSION") {
case "1":
fv = content.FormatVersion1
fv = format.FormatVersion1
case "2":
fv = content.FormatVersion2
fv = format.FormatVersion2
case "3":
fv = content.FormatVersion3
fv = format.FormatVersion3
default:
fv = content.FormatVersion3
fv = format.FormatVersion3
}
}
f := &repositoryObjectFormat{
FormattingOptions: content.FormattingOptions{
f := &format.RepositoryConfig{
ContentFormat: format.ContentFormat{
Hash: applyDefaultString(opt.BlockFormat.Hash, hashing.DefaultAlgorithm),
Encryption: applyDefaultString(opt.BlockFormat.Encryption, encryption.DefaultAlgorithm),
HMACSecret: applyDefaultRandomBytes(opt.BlockFormat.HMACSecret, hmacSecretLength),
MasterKey: applyDefaultRandomBytes(opt.BlockFormat.MasterKey, masterKeyLength),
MutableParameters: content.MutableParameters{
MutableParameters: format.MutableParameters{
Version: fv,
MaxPackSize: applyDefaultInt(opt.BlockFormat.MaxPackSize, 20<<20), //nolint:gomnd
IndexVersion: applyDefaultInt(opt.BlockFormat.IndexVersion, content.DefaultIndexVersion),
@@ -146,7 +153,7 @@ func repositoryObjectFormatFromOptions(opt *NewRepositoryOptions) (*repositoryOb
},
EnablePasswordChange: opt.BlockFormat.EnablePasswordChange,
},
Format: object.Format{
ObjectFormat: format.ObjectFormat{
Splitter: applyDefaultString(opt.ObjectFormat.Splitter, splitter.DefaultAlgorithm),
},
}
@@ -155,7 +162,7 @@ func repositoryObjectFormatFromOptions(opt *NewRepositoryOptions) (*repositoryOb
f.HMACSecret = nil
}
if err := f.FormattingOptions.ResolveFormatVersion(); err != nil {
if err := f.ContentFormat.ResolveFormatVersion(); err != nil {
return nil, errors.Wrap(err, "error resolving format version")
}

14
repo/local_config.go
View File

@@ -11,12 +11,11 @@
"github.com/pkg/errors"
"github.com/kopia/kopia/internal/atomicfile"
"github.com/kopia/kopia/internal/feature"
"github.com/kopia/kopia/internal/ospath"
"github.com/kopia/kopia/repo/blob"
"github.com/kopia/kopia/repo/blob/throttling"
"github.com/kopia/kopia/repo/content"
"github.com/kopia/kopia/repo/object"
"github.com/kopia/kopia/repo/format"
)
const configDirMode = 0o700
@@ -53,7 +52,7 @@ func (o ClientOptions) ApplyDefaults(ctx context.Context, defaultDesc string) Cl
}
if o.FormatBlobCacheDuration == 0 {
o.FormatBlobCacheDuration = DefaultRepositoryBlobCacheDuration
o.FormatBlobCacheDuration = format.DefaultRepositoryBlobCacheDuration
}
return o
@@ -98,15 +97,6 @@ type LocalConfig struct {
ClientOptions
}
// repositoryObjectFormat describes the format of objects in a repository.
type repositoryObjectFormat struct {
content.FormattingOptions
object.Format
UpgradeLock *UpgradeLockIntent `json:"upgradeLock,omitempty"`
RequiredFeatures []feature.Required `json:"requiredFeatures,omitempty"`
}
// writeToFile writes the config to a given file.
func (lc *LocalConfig) writeToFile(filename string) error {
lc2 := *lc

3
repo/maintenance/blob_gc_test.go
View File

@@ -22,6 +22,7 @@
"github.com/kopia/kopia/repo/blob"
"github.com/kopia/kopia/repo/content"
"github.com/kopia/kopia/repo/encryption"
"github.com/kopia/kopia/repo/format"
"github.com/kopia/kopia/repo/maintenance"
"github.com/kopia/kopia/repo/object"
)
@@ -200,7 +201,7 @@ func mustPutDummySessionBlob(t *testing.T, st blob.Storage, sessionIDSuffix blob
blobID := blob.ID(fmt.Sprintf("s%x-%v", iv, sessionIDSuffix))
e, err := encryption.CreateEncryptor(&content.FormattingOptions{
e, err := encryption.CreateEncryptor(&format.ContentFormat{
Encryption: encryption.DefaultAlgorithm,
MasterKey: testMasterKey,
HMACSecret: testHMACSecret,

10
repo/maintenance/suite_test.go
View File

@@ -4,21 +4,21 @@
"testing"
"github.com/kopia/kopia/internal/testutil"
"github.com/kopia/kopia/repo/content"
"github.com/kopia/kopia/repo/format"
)
type formatSpecificTestSuite struct {
formatVersion content.FormatVersion
formatVersion format.Version
}
func TestFormatV1(t *testing.T) {
testutil.RunAllTestsWithParam(t, &formatSpecificTestSuite{content.FormatVersion1})
testutil.RunAllTestsWithParam(t, &formatSpecificTestSuite{format.FormatVersion1})
}
func TestFormatV2(t *testing.T) {
testutil.RunAllTestsWithParam(t, &formatSpecificTestSuite{content.FormatVersion2})
testutil.RunAllTestsWithParam(t, &formatSpecificTestSuite{format.FormatVersion2})
}
func TestFormatV3(t *testing.T) {
testutil.RunAllTestsWithParam(t, &formatSpecificTestSuite{content.FormatVersion3})
testutil.RunAllTestsWithParam(t, &formatSpecificTestSuite{format.FormatVersion3})
}

9
repo/manifest/manifest_manager_test.go
View File

@@ -18,6 +18,7 @@
"github.com/kopia/kopia/repo/content"
"github.com/kopia/kopia/repo/content/index"
"github.com/kopia/kopia/repo/encryption"
"github.com/kopia/kopia/repo/format"
"github.com/kopia/kopia/repo/hashing"
)
@@ -142,10 +143,10 @@ func TestManifestInitCorruptedBlock(t *testing.T) {
data := blobtesting.DataMap{}
st := blobtesting.NewMapStorage(data, nil, nil)
fop, err := content.NewFormattingOptionsProvider(&content.FormattingOptions{
fop, err := format.NewFormattingOptionsProvider(&format.ContentFormat{
Hash: hashing.DefaultAlgorithm,
Encryption: encryption.DefaultAlgorithm,
MutableParameters: content.MutableParameters{
MutableParameters: format.MutableParameters{
Version: 1,
MaxPackSize: 100000,
},
@@ -302,10 +303,10 @@ func newManagerForTesting(ctx context.Context, t *testing.T, data blobtesting.Da
st := blobtesting.NewMapStorage(data, nil, nil)
fop, err := content.NewFormattingOptionsProvider(&content.FormattingOptions{
fop, err := format.NewFormattingOptionsProvider(&format.ContentFormat{
Hash: hashing.DefaultAlgorithm,
Encryption: encryption.DefaultAlgorithm,
MutableParameters: content.MutableParameters{
MutableParameters: format.MutableParameters{
Version: 1,
MaxPackSize: 100000,
},

10
repo/object/object_manager.go
View File

@@ -11,6 +11,7 @@
"github.com/kopia/kopia/internal/gather"
"github.com/kopia/kopia/repo/compression"
"github.com/kopia/kopia/repo/content"
"github.com/kopia/kopia/repo/format"
"github.com/kopia/kopia/repo/splitter"
)
@@ -37,14 +38,9 @@ type contentManager interface {
WriteContent(ctx context.Context, data gather.Bytes, prefix content.IDPrefix, comp compression.HeaderID) (content.ID, error)
}
// Format describes the format of objects in a repository.
type Format struct {
Splitter string `json:"splitter,omitempty"` // splitter used to break objects into pieces of content
}
// Manager implements a content-addressable storage on top of blob storage.
type Manager struct {
Format Format
Format format.ObjectFormat
contentMgr contentManager
newSplitter splitter.Factory
@@ -201,7 +197,7 @@ func PrefetchBackingContents(ctx context.Context, contentMgr contentManager, obj
}
// NewObjectManager creates an ObjectManager with the specified content manager and format.
func NewObjectManager(ctx context.Context, bm contentManager, f Format) (*Manager, error) {
func NewObjectManager(ctx context.Context, bm contentManager, f format.ObjectFormat) (*Manager, error) {
om := &Manager{
contentMgr: bm,
Format: f,

5
repo/object/object_manager_test.go
View File

@@ -26,6 +26,7 @@
"github.com/kopia/kopia/repo/blob"
"github.com/kopia/kopia/repo/compression"
"github.com/kopia/kopia/repo/content"
"github.com/kopia/kopia/repo/format"
"github.com/kopia/kopia/repo/splitter"
)
@@ -109,7 +110,7 @@ func setupTest(t *testing.T, compressionHeaderID map[content.ID]compression.Head
compresionIDs: compressionHeaderID,
}
r, err := NewObjectManager(testlogging.Context(t), fcm, Format{
r, err := NewObjectManager(testlogging.Context(t), fcm, format.ObjectFormat{
Splitter: "FIXED-1M",
})
if err != nil {
@@ -280,7 +281,7 @@ func TestObjectWriterRaceBetweenCheckpointAndResult(t *testing.T) {
data: data,
}
om, err := NewObjectManager(testlogging.Context(t), fcm, Format{
om, err := NewObjectManager(testlogging.Context(t), fcm, format.ObjectFormat{
Splitter: "FIXED-1M",
})
if err != nil {

238
repo/open.go
View File

@@ -12,12 +12,9 @@
"github.com/pkg/errors"
"golang.org/x/crypto/scrypt"
"github.com/kopia/kopia/internal/atomicfile"
"github.com/kopia/kopia/internal/cache"
"github.com/kopia/kopia/internal/clock"
"github.com/kopia/kopia/internal/epoch"
"github.com/kopia/kopia/internal/feature"
"github.com/kopia/kopia/internal/gather"
"github.com/kopia/kopia/internal/retry"
"github.com/kopia/kopia/repo/blob"
"github.com/kopia/kopia/repo/blob/beforeop"
@@ -25,6 +22,7 @@
"github.com/kopia/kopia/repo/blob/readonly"
"github.com/kopia/kopia/repo/blob/throttling"
"github.com/kopia/kopia/repo/content"
"github.com/kopia/kopia/repo/format"
"github.com/kopia/kopia/repo/logging"
"github.com/kopia/kopia/repo/manifest"
"github.com/kopia/kopia/repo/object"
@@ -46,17 +44,6 @@
"index-v2",
}
// CacheDirMarkerFile is the name of the marker file indicating a directory contains Kopia caches.
// See https://bford.info/cachedir/
const CacheDirMarkerFile = "CACHEDIR.TAG"
// CacheDirMarkerHeader is the header signature for cache dir marker files.
const CacheDirMarkerHeader = "Signature: 8a477f597d28d172789f06886806bc55"
// DefaultRepositoryBlobCacheDuration is the duration for which we treat cached kopia.repository
// as valid.
const DefaultRepositoryBlobCacheDuration = 15 * time.Minute
// throttlingWindow is the duration window during which the throttling token bucket fully replenishes.
// the maximum number of tokens in the bucket is multiplied by the number of seconds.
const throttlingWindow = 60 * time.Second
@@ -70,17 +57,6 @@
// nolint:gochecknoglobals
var localCacheIntegrityPurpose = []byte("local-cache-integrity")
const cacheDirMarkerContents = CacheDirMarkerHeader + `
#
# This file is a cache directory tag created by Kopia - Fast And Secure Open-Source Backup.
#
# For information about Kopia, see:
# https://kopia.io
#
# For information about cache directory tags, see:
# http://www.brynosaurus.com/cachedir/
`
var log = logging.Module("kopia/repo")
// Options provides configuration parameters for connection to a repository.
@@ -98,7 +74,7 @@ type Options struct {
}
// ErrInvalidPassword is returned when repository password is invalid.
var ErrInvalidPassword = errors.Errorf("invalid repository password")
var ErrInvalidPassword = format.ErrInvalidPassword
// ErrRepositoryUnavailableDueToUpgrageInProgress is returned when repository
// is undergoing upgrade that requires exclusive access.
@@ -219,56 +195,6 @@ func openDirect(ctx context.Context, configFile string, lc *LocalConfig, passwor
return r, nil
}
type unpackedFormatBlob struct {
f *formatBlob
fb []byte // serialized format blob
cacheMTime time.Time // mod time of the format blob cache file
repoConfig *repositoryObjectFormat // unencrypted format blob structure
formatEncryptionKey []byte // key derived from the password
}
func readAndCacheRepoConfig(ctx context.Context, st blob.Storage, password string, cacheOpts *content.CachingOptions, validDuration time.Duration) (ufb *unpackedFormatBlob, err error) {
ufb = &unpackedFormatBlob{}
// Read format blob, potentially from cache.
ufb.fb, ufb.cacheMTime, err = readAndCacheRepositoryBlobBytes(ctx, st, cacheOpts.CacheDirectory, FormatBlobID, validDuration)
if err != nil {
return nil, errors.Wrap(err, "unable to read format blob")
}
if err = writeCacheMarker(cacheOpts.CacheDirectory); err != nil {
return nil, errors.Wrap(err, "unable to write cache directory marker")
}
ufb.f, err = parseFormatBlob(ufb.fb)
if err != nil {
return nil, errors.Wrap(err, "can't parse format blob")
}
ufb.fb, err = addFormatBlobChecksumAndLength(ufb.fb)
if err != nil {
return nil, errors.Errorf("unable to add checksum")
}
ufb.formatEncryptionKey, err = ufb.f.deriveFormatEncryptionKeyFromPassword(password)
if err != nil {
return nil, err
}
ufb.repoConfig, err = ufb.f.decryptFormatBytes(ufb.formatEncryptionKey)
if err != nil {
return nil, ErrInvalidPassword
}
return ufb, nil
}
// ReadAndCacheRepoUpgradeLock loads the lock config from cache and returns it.
func ReadAndCacheRepoUpgradeLock(ctx context.Context, st blob.Storage, password string, cacheOpts *content.CachingOptions, validDuration time.Duration) (*UpgradeLockIntent, error) {
ufb, err := readAndCacheRepoConfig(ctx, st, password, cacheOpts, validDuration)
return ufb.repoConfig.UpgradeLock, err
}
// openWithConfig opens the repository with a given configuration, avoiding the need for a config file.
// nolint:funlen,gocyclo,cyclop
func openWithConfig(ctx context.Context, st blob.Storage, lc *LocalConfig, password string, options *Options, cacheOpts *content.CachingOptions, configFile string) (DirectRepository, error) {
@@ -278,18 +204,19 @@ func openWithConfig(ctx context.Context, st blob.Storage, lc *LocalConfig, passw
DisableInternalLog: options.DisableInternalLog,
}
var ufb *unpackedFormatBlob
var ufb *format.DecodedRepositoryConfig
if _, err := retry.WithExponentialBackoffMaxRetries(ctx, -1, "read repo config and wait for upgrade", func() (interface{}, error) {
var internalErr error
ufb, internalErr = readAndCacheRepoConfig(ctx, st, password, cacheOpts,
ufb, internalErr = format.ReadAndCacheDecodedRepositoryConfig(ctx, st, password, cacheOpts.CacheDirectory,
lc.FormatBlobCacheDuration)
if internalErr != nil {
// nolint:wrapcheck
return nil, internalErr
}
// retry if upgrade lock has been taken
if locked, _ := ufb.repoConfig.UpgradeLock.IsLocked(cmOpts.TimeNow()); locked && options.UpgradeOwnerID != ufb.repoConfig.UpgradeLock.OwnerID {
if locked, _ := ufb.RepoConfig.UpgradeLock.IsLocked(cmOpts.TimeNow()); locked && options.UpgradeOwnerID != ufb.RepoConfig.UpgradeLock.OwnerID {
return nil, ErrRepositoryUnavailableDueToUpgrageInProgress
}
@@ -301,31 +228,31 @@ func openWithConfig(ctx context.Context, st blob.Storage, lc *LocalConfig, passw
return nil, err
}
if err := handleMissingRequiredFeatures(ctx, ufb.repoConfig, options.TestOnlyIgnoreMissingRequiredFeatures); err != nil {
if err := handleMissingRequiredFeatures(ctx, ufb.RepoConfig, options.TestOnlyIgnoreMissingRequiredFeatures); err != nil {
return nil, err
}
cmOpts.RepositoryFormatBytes = ufb.fb
cmOpts.RepositoryFormatBytes = ufb.KopiaRepositoryBytes
// Read blobcfg blob, potentially from cache.
bb, _, err := readAndCacheRepositoryBlobBytes(ctx, st, cacheOpts.CacheDirectory, BlobCfgBlobID, lc.FormatBlobCacheDuration)
bb, _, err := format.ReadAndCacheRepositoryBlobBytes(ctx, st, cacheOpts.CacheDirectory, format.KopiaBlobCfgBlobID, lc.FormatBlobCacheDuration)
if err != nil && !errors.Is(err, blob.ErrBlobNotFound) {
return nil, errors.Wrap(err, "unable to read blobcfg blob")
}
blobcfg, err := deserializeBlobCfgBytes(ufb.f, bb, ufb.formatEncryptionKey)
blobcfg, err := ufb.KopiaRepository.DeserializeBlobCfgBytes(bb, ufb.FormatEncryptionKey)
if err != nil {
return nil, ErrInvalidPassword
}
if ufb.repoConfig.FormattingOptions.EnablePasswordChange {
cacheOpts.HMACSecret = deriveKeyFromMasterKey(ufb.repoConfig.HMACSecret, ufb.f.UniqueID, localCacheIntegrityPurpose, localCacheIntegrityHMACSecretLength)
if ufb.RepoConfig.ContentFormat.EnablePasswordChange {
cacheOpts.HMACSecret = format.DeriveKeyFromMasterKey(ufb.RepoConfig.HMACSecret, ufb.KopiaRepository.UniqueID, localCacheIntegrityPurpose, localCacheIntegrityHMACSecretLength)
} else {
// deriving from ufb.formatEncryptionKey was actually a bug, that only matters will change when we change the password
cacheOpts.HMACSecret = deriveKeyFromMasterKey(ufb.formatEncryptionKey, ufb.f.UniqueID, localCacheIntegrityPurpose, localCacheIntegrityHMACSecretLength)
// deriving from ufb.FormatEncryptionKey was actually a bug, that only matters will change when we change the password
cacheOpts.HMACSecret = format.DeriveKeyFromMasterKey(ufb.FormatEncryptionKey, ufb.KopiaRepository.UniqueID, localCacheIntegrityPurpose, localCacheIntegrityHMACSecretLength)
}
fo := &ufb.repoConfig.FormattingOptions
fo := &ufb.RepoConfig.ContentFormat
if fo.MaxPackSize == 0 {
// legacy only, apply default
@@ -364,9 +291,9 @@ func openWithConfig(ctx context.Context, st blob.Storage, lc *LocalConfig, passw
// background/interleaving upgrade lock storage monitor
st = upgradeLockMonitor(options.UpgradeOwnerID, st, password, cacheOpts, lc.FormatBlobCacheDuration,
ufb.cacheMTime, cmOpts.TimeNow, options.OnFatalError, options.TestOnlyIgnoreMissingRequiredFeatures)
ufb.CacheMTime, cmOpts.TimeNow, options.OnFatalError, options.TestOnlyIgnoreMissingRequiredFeatures)
fop, err := content.NewFormattingOptionsProvider(fo, cmOpts.RepositoryFormatBytes)
fop, err := format.NewFormattingOptionsProvider(fo, cmOpts.RepositoryFormatBytes)
if err != nil {
return nil, errors.Wrap(err, "unable to create format options provider")
}
@@ -381,7 +308,7 @@ func openWithConfig(ctx context.Context, st blob.Storage, lc *LocalConfig, passw
SessionHost: lc.Hostname,
}, "")
om, err := object.NewObjectManager(ctx, cm, ufb.repoConfig.Format)
om, err := object.NewObjectManager(ctx, cm, ufb.RepoConfig.ObjectFormat)
if err != nil {
return nil, errors.Wrap(err, "unable to open object manager")
}
@@ -398,11 +325,11 @@ func openWithConfig(ctx context.Context, st blob.Storage, lc *LocalConfig, passw
mmgr: manifests,
sm: scm,
directRepositoryParameters: directRepositoryParameters{
uniqueID: ufb.f.UniqueID,
uniqueID: ufb.KopiaRepository.UniqueID,
cachingOptions: *cacheOpts,
formatBlob: ufb.f,
formatBlob: ufb.KopiaRepository,
blobCfgBlob: blobcfg,
formatEncryptionKey: ufb.formatEncryptionKey,
formatEncryptionKey: ufb.FormatEncryptionKey,
timeNow: cmOpts.TimeNow,
cliOpts: lc.ClientOptions.ApplyDefaults(ctx, "Repository in "+st.DisplayName()),
configFile: configFile,
@@ -415,7 +342,7 @@ func openWithConfig(ctx context.Context, st blob.Storage, lc *LocalConfig, passw
return dr, nil
}
func handleMissingRequiredFeatures(ctx context.Context, repoConfig *repositoryObjectFormat, ignoreErrors bool) error {
func handleMissingRequiredFeatures(ctx context.Context, repoConfig *format.RepositoryConfig, ignoreErrors bool) error {
// See if the current version of Kopia supports all features required by the repository format.
// so we can safely fail to start in case repository has been upgraded to a new, incompatible version.
if missingFeatures := feature.GetUnsupportedFeatures(repoConfig.RequiredFeatures, supportedFeatures); len(missingFeatures) > 0 {
@@ -432,15 +359,15 @@ func handleMissingRequiredFeatures(ctx context.Context, repoConfig *repositoryOb
return nil
}
func wrapLockingStorage(st blob.Storage, r content.BlobCfgBlob) blob.Storage {
func wrapLockingStorage(st blob.Storage, r format.BlobStorageConfiguration) blob.Storage {
// collect prefixes that need to be locked on put
var prefixes []string
for _, prefix := range content.PackBlobIDPrefixes {
prefixes = append(prefixes, string(prefix))
}
prefixes = append(prefixes, content.LegacyIndexBlobPrefix, epoch.EpochManagerIndexUberPrefix, FormatBlobID,
BlobCfgBlobID)
prefixes = append(prefixes, content.LegacyIndexBlobPrefix, epoch.EpochManagerIndexUberPrefix, format.KopiaRepositoryBlobID,
format.KopiaBlobCfgBlobID)
return beforeop.NewWrapper(st, nil, nil, nil, func(ctx context.Context, id blob.ID, opts *blob.PutOptions) error {
for _, prefix := range prefixes {
@@ -497,23 +424,24 @@ func upgradeLockMonitor(
return nil
}
ufb, err := readAndCacheRepoConfig(ctx, st, password, cacheOpts, lockRefreshInterval)
ufb, err := format.ReadAndCacheDecodedRepositoryConfig(ctx, st, password, cacheOpts.CacheDirectory, lockRefreshInterval)
if err != nil {
// nolint:wrapcheck
return err
}
if err := handleMissingRequiredFeatures(ctx, ufb.repoConfig, ignoreMissingRequiredFeatures); err != nil {
if err := handleMissingRequiredFeatures(ctx, ufb.RepoConfig, ignoreMissingRequiredFeatures); err != nil {
onFatalError(err)
return err
}
// only allow the upgrade owner to perform storage operations
if locked, _ := ufb.repoConfig.UpgradeLock.IsLocked(now()); locked && upgradeOwnerID != ufb.repoConfig.UpgradeLock.OwnerID {
if locked, _ := ufb.RepoConfig.UpgradeLock.IsLocked(now()); locked && upgradeOwnerID != ufb.RepoConfig.UpgradeLock.OwnerID {
return ErrRepositoryUnavailableDueToUpgrageInProgress
}
// prevent backward jumps on nextSync
newNextSync := ufb.cacheMTime.Add(lockRefreshInterval)
newNextSync := ufb.CacheMTime.Add(lockRefreshInterval)
if newNextSync.After(nextSync) {
nextSync = newNextSync
}
@@ -540,109 +468,3 @@ func throttlingLimitsFromConnectionInfo(ctx context.Context, ci blob.ConnectionI
return l
}
func writeCacheMarker(cacheDir string) error {
if cacheDir == "" {
return nil
}
markerFile := filepath.Join(cacheDir, CacheDirMarkerFile)
st, err := os.Stat(markerFile)
if err == nil && st.Size() >= int64(len(cacheDirMarkerContents)) {
// ok
return nil
}
if err != nil && !os.IsNotExist(err) {
return errors.Wrap(err, "unexpected cache marker error")
}
f, err := os.Create(markerFile) //nolint:gosec
if err != nil {
return errors.Wrap(err, "error creating cache marker")
}
if _, err := f.WriteString(cacheDirMarkerContents); err != nil {
return errors.Wrap(err, "unable to write cachedir marker contents")
}
return errors.Wrap(f.Close(), "error closing cache marker file")
}
func formatBytesCachingEnabled(cacheDirectory string, validDuration time.Duration) bool {
if cacheDirectory == "" {
return false
}
return validDuration > 0
}
func readRepositoryBlobBytesFromCache(ctx context.Context, cachedFile string, validDuration time.Duration) (data []byte, cacheMTime time.Time, err error) {
cst, err := os.Stat(cachedFile)
if err != nil {
return nil, time.Time{}, errors.Wrap(err, "unable to open cache file")
}
cacheMTime = cst.ModTime()
if clock.Now().Sub(cacheMTime) > validDuration {
// got cached file, but it's too old, remove it
if err = os.Remove(cachedFile); err != nil {
log(ctx).Debugf("unable to remove cache file: %v", err)
}
return nil, time.Time{}, errors.Errorf("cached file too old")
}
data, err = os.ReadFile(cachedFile) // nolint:gosec
if err != nil {
return nil, time.Time{}, errors.Wrapf(err, "failed to read the cache file %q", cachedFile)
}
return data, cacheMTime, nil
}
func readAndCacheRepositoryBlobBytes(ctx context.Context, st blob.Storage, cacheDirectory, blobID string, validDuration time.Duration) ([]byte, time.Time, error) {
cachedFile := filepath.Join(cacheDirectory, blobID)
if validDuration == 0 {
validDuration = DefaultRepositoryBlobCacheDuration
}
if cacheDirectory != "" {
if err := os.MkdirAll(cacheDirectory, cache.DirMode); err != nil && !os.IsExist(err) {
log(ctx).Errorf("unable to create cache directory: %v", err)
}
}
cacheEnabled := formatBytesCachingEnabled(cacheDirectory, validDuration)
if cacheEnabled {
data, cacheMTime, err := readRepositoryBlobBytesFromCache(ctx, cachedFile, validDuration)
if err == nil {
log(ctx).Debugf("%s retrieved from cache", blobID)
return data, cacheMTime, nil
}
if os.IsNotExist(err) {
log(ctx).Debugf("%s could not be fetched from cache: %v", blobID, err)
}
} else {
log(ctx).Debugf("%s cache not enabled", blobID)
}
var b gather.WriteBuffer
defer b.Close()
if err := st.GetBlob(ctx, blob.ID(blobID), 0, -1, &b); err != nil {
return nil, time.Time{}, errors.Wrapf(err, "error getting %s blob", blobID)
}
if cacheEnabled {
if err := atomicfile.Write(cachedFile, b.Bytes().Reader()); err != nil {
log(ctx).Warnf("unable to write cache: %v", err)
}
}
return b.ToByteSlice(), clock.Now(), nil
}

26
repo/parameters.go
View File

@@ -8,11 +8,11 @@
"github.com/pkg/errors"
"github.com/kopia/kopia/internal/feature"
"github.com/kopia/kopia/repo/content"
"github.com/kopia/kopia/repo/format"
)
func (r *directRepository) RequiredFeatures() ([]feature.Required, error) {
repoConfig, err := r.formatBlob.decryptFormatBytes(r.formatEncryptionKey)
repoConfig, err := r.formatBlob.DecryptRepositoryConfig(r.formatEncryptionKey)
if err != nil {
return nil, errors.Wrap(err, "unable to decrypt repository config")
}
@@ -23,13 +23,13 @@ func (r *directRepository) RequiredFeatures() ([]feature.Required, error) {
// SetParameters changes mutable repository parameters.
func (r *directRepository) SetParameters(
ctx context.Context,
m content.MutableParameters,
blobcfg content.BlobCfgBlob,
m format.MutableParameters,
blobcfg format.BlobStorageConfiguration,
requiredFeatures []feature.Required,
) error {
f := r.formatBlob
repoConfig, err := f.decryptFormatBytes(r.formatEncryptionKey)
repoConfig, err := f.DecryptRepositoryConfig(r.formatEncryptionKey)
if err != nil {
return errors.Wrap(err, "unable to decrypt repository config")
}
@@ -42,28 +42,28 @@ func (r *directRepository) SetParameters(
return errors.Wrap(err, "invalid blob-config options")
}
repoConfig.FormattingOptions.MutableParameters = m
repoConfig.ContentFormat.MutableParameters = m
repoConfig.RequiredFeatures = requiredFeatures
if err := encryptFormatBytes(f, repoConfig, r.formatEncryptionKey, f.UniqueID); err != nil {
if err := f.EncryptRepositoryConfig(repoConfig, r.formatEncryptionKey); err != nil {
return errors.Errorf("unable to encrypt format bytes")
}
if err := writeBlobCfgBlob(ctx, r.blobs, f, blobcfg, r.formatEncryptionKey); err != nil {
if err := f.WriteBlobCfgBlob(ctx, r.blobs, blobcfg, r.formatEncryptionKey); err != nil {
return errors.Wrap(err, "unable to write blobcfg blob")
}
if err := writeFormatBlob(ctx, r.blobs, f, r.blobCfgBlob); err != nil {
if err := f.WriteKopiaRepositoryBlob(ctx, r.blobs, r.blobCfgBlob); err != nil {
return errors.Wrap(err, "unable to write format blob")
}
if cd := r.cachingOptions.CacheDirectory; cd != "" {
if err := os.Remove(filepath.Join(cd, FormatBlobID)); err != nil {
log(ctx).Errorf("unable to remove %s: %v", FormatBlobID, err)
if err := os.Remove(filepath.Join(cd, format.KopiaRepositoryBlobID)); err != nil {
log(ctx).Errorf("unable to remove %s: %v", format.KopiaRepositoryBlobID, err)
}
if err := os.Remove(filepath.Join(cd, BlobCfgBlobID)); err != nil && !os.IsNotExist(err) {
log(ctx).Errorf("unable to remove %s: %v", BlobCfgBlobID, err)
if err := os.Remove(filepath.Join(cd, format.KopiaBlobCfgBlobID)); err != nil && !os.IsNotExist(err) {
log(ctx).Errorf("unable to remove %s: %v", format.KopiaBlobCfgBlobID, err)
}
}

6
repo/repo_benchmarks_test.go
View File

@@ -7,12 +7,12 @@
"github.com/stretchr/testify/require"
"github.com/kopia/kopia/internal/repotesting"
"github.com/kopia/kopia/repo/content"
"github.com/kopia/kopia/repo/format"
"github.com/kopia/kopia/repo/object"
)
func BenchmarkWriterDedup1M(b *testing.B) {
ctx, env := repotesting.NewEnvironment(b, content.FormatVersion2)
ctx, env := repotesting.NewEnvironment(b, format.FormatVersion2)
dataBuf := make([]byte, 4<<20)
writer := env.RepositoryWriter.NewObjectWriter(ctx, object.WriterOptions{})
@@ -33,7 +33,7 @@ func BenchmarkWriterDedup1M(b *testing.B) {
}
func BenchmarkWriterNoDedup1M(b *testing.B) {
ctx, env := repotesting.NewEnvironment(b, content.FormatVersion2)
ctx, env := repotesting.NewEnvironment(b, format.FormatVersion2)
dataBuf := make([]byte, 4<<20)
chunkSize := 32
offset := 0

23
repo/repository.go
View File

@@ -14,6 +14,7 @@
"github.com/kopia/kopia/repo/blob"
"github.com/kopia/kopia/repo/blob/throttling"
"github.com/kopia/kopia/repo/content"
"github.com/kopia/kopia/repo/format"
"github.com/kopia/kopia/repo/manifest"
"github.com/kopia/kopia/repo/object"
)
@@ -52,8 +53,8 @@ type RepositoryWriter interface {
type DirectRepository interface {
Repository
ObjectFormat() object.Format
BlobCfg() content.BlobCfgBlob
ObjectFormat() format.ObjectFormat
BlobCfg() format.BlobStorageConfiguration
BlobReader() blob.Reader
BlobVolume() blob.Volume
ContentReader() content.Reader
@@ -75,10 +76,10 @@ type DirectRepositoryWriter interface {
DirectRepository
BlobStorage() blob.Storage
ContentManager() *content.WriteManager
SetParameters(ctx context.Context, m content.MutableParameters, blobcfg content.BlobCfgBlob, requiredFeatures []feature.Required) error
SetParameters(ctx context.Context, m format.MutableParameters, blobcfg format.BlobStorageConfiguration, requiredFeatures []feature.Required) error
ChangePassword(ctx context.Context, newPassword string) error
GetUpgradeLockIntent(ctx context.Context) (*UpgradeLockIntent, error)
SetUpgradeLockIntent(ctx context.Context, l UpgradeLockIntent) (*UpgradeLockIntent, error)
GetUpgradeLockIntent(ctx context.Context) (*format.UpgradeLockIntent, error)
SetUpgradeLockIntent(ctx context.Context, l format.UpgradeLockIntent) (*format.UpgradeLockIntent, error)
CommitUpgrade(ctx context.Context) error
RollbackUpgrade(ctx context.Context) error
}
@@ -89,8 +90,8 @@ type directRepositoryParameters struct {
cachingOptions content.CachingOptions
cliOpts ClientOptions
timeNow func() time.Time
formatBlob *formatBlob
blobCfgBlob content.BlobCfgBlob
formatBlob *format.KopiaRepositoryJSON
blobCfgBlob format.BlobStorageConfiguration
formatEncryptionKey []byte
nextWriterID *int32
throttler throttling.SettableThrottler
@@ -112,13 +113,13 @@ type directRepository struct {
// DeriveKey derives encryption key of the provided length from the master key.
func (r *directRepository) DeriveKey(purpose []byte, keyLength int) []byte {
if r.cmgr.ContentFormat().SupportsPasswordChange() {
return deriveKeyFromMasterKey(r.cmgr.ContentFormat().GetMasterKey(), r.uniqueID, purpose, keyLength)
return format.DeriveKeyFromMasterKey(r.cmgr.ContentFormat().GetMasterKey(), r.uniqueID, purpose, keyLength)
}
// version of kopia <v0.9 had a bug where certain keys were derived directly from
// the password and not from the random master key. This made it impossible to change
// password.
return deriveKeyFromMasterKey(r.formatEncryptionKey, r.uniqueID, purpose, keyLength)
return format.DeriveKeyFromMasterKey(r.formatEncryptionKey, r.uniqueID, purpose, keyLength)
}
// ClientOptions returns client options.
@@ -301,7 +302,7 @@ func (r *directRepository) Flush(ctx context.Context) error {
}
// ObjectFormat returns the object format.
func (r *directRepository) ObjectFormat() object.Format {
func (r *directRepository) ObjectFormat() format.ObjectFormat {
return r.omgr.Format
}
@@ -341,7 +342,7 @@ func (r *directRepository) Time() time.Time {
return defaultTime(r.timeNow)()
}
func (r *directRepository) BlobCfg() content.BlobCfgBlob {
func (r *directRepository) BlobCfg() format.BlobStorageConfiguration {
return r.directRepositoryParameters.blobCfgBlob
}

31
repo/repository_test.go
View File

@@ -22,6 +22,7 @@
"github.com/kopia/kopia/repo/blob"
"github.com/kopia/kopia/repo/blob/beforeop"
"github.com/kopia/kopia/repo/content"
"github.com/kopia/kopia/repo/format"
"github.com/kopia/kopia/repo/object"
)
@@ -403,11 +404,11 @@ func TestInitializeWithBlobCfgRetentionBlob(t *testing.T) {
defer d.Close()
// verify that the blobcfg retention blob is created
require.NoError(t, env.RepositoryWriter.BlobStorage().GetBlob(ctx, repo.BlobCfgBlobID, 0, -1, &d))
require.NoError(t, env.RepositoryWriter.BlobStorage().GetBlob(ctx, format.KopiaBlobCfgBlobID, 0, -1, &d))
require.NoError(t, env.RepositoryWriter.ChangePassword(ctx, "new-password"))
// verify that the blobcfg retention blob is created and is different after
// password-change
require.NoError(t, env.RepositoryWriter.BlobStorage().GetBlob(ctx, repo.BlobCfgBlobID, 0, -1, &d))
require.NoError(t, env.RepositoryWriter.BlobStorage().GetBlob(ctx, format.KopiaBlobCfgBlobID, 0, -1, &d))
// verify that we cannot re-initialize the repo even after password change
require.EqualError(t, repo.Initialize(testlogging.Context(t), env.RootStorage(), nil, env.Password),
@@ -417,17 +418,17 @@ func TestInitializeWithBlobCfgRetentionBlob(t *testing.T) {
{
// backup & corrupt the blobcfg blob
d.Reset()
require.NoError(t, env.RepositoryWriter.BlobStorage().GetBlob(ctx, repo.BlobCfgBlobID, 0, -1, &d))
require.NoError(t, env.RepositoryWriter.BlobStorage().GetBlob(ctx, format.KopiaBlobCfgBlobID, 0, -1, &d))
corruptedData := d.Dup()
corruptedData.Append([]byte("bad bits"))
require.NoError(t, env.RepositoryWriter.BlobStorage().PutBlob(ctx, repo.BlobCfgBlobID, corruptedData.Bytes(), blob.PutOptions{}))
require.NoError(t, env.RepositoryWriter.BlobStorage().PutBlob(ctx, format.KopiaBlobCfgBlobID, corruptedData.Bytes(), blob.PutOptions{}))
// verify that we error out on corrupted blobcfg blob
_, err := repo.Open(ctx, env.ConfigFile(), env.Password, &repo.Options{})
require.EqualError(t, err, "invalid repository password")
// restore the original blob
require.NoError(t, env.RepositoryWriter.BlobStorage().PutBlob(ctx, repo.BlobCfgBlobID, d.Bytes(), blob.PutOptions{}))
require.NoError(t, env.RepositoryWriter.BlobStorage().PutBlob(ctx, format.KopiaBlobCfgBlobID, d.Bytes(), blob.PutOptions{}))
}
// verify that we'd hard-fail on unexpected errors on blobcfg blob-puts
@@ -438,11 +439,11 @@ func TestInitializeWithBlobCfgRetentionBlob(t *testing.T) {
env.RootStorage(),
// GetBlob callback
func(ctx context.Context, id blob.ID) error {
if id == repo.BlobCfgBlobID {
if id == format.KopiaBlobCfgBlobID {
return errors.New("unexpected error")
}
// simulate not-found for format-blob
if id == repo.FormatBlobID {
if id == format.KopiaRepositoryBlobID {
return blob.ErrBlobNotFound
}
return nil
@@ -463,10 +464,10 @@ func(ctx context.Context, id blob.ID) error {
func(ctx context.Context, id blob.ID) error {
// simulate not-found for format-blob but let blobcfg
// blob appear as pre-existing
if id == repo.BlobCfgBlobID {
if id == format.KopiaBlobCfgBlobID {
return nil
}
if id == repo.FormatBlobID {
if id == format.KopiaRepositoryBlobID {
return blob.ErrBlobNotFound
}
return nil
@@ -486,7 +487,7 @@ func(ctx context.Context, id blob.ID) error {
// GetBlob callback
func(ctx context.Context, id blob.ID) error {
// simulate not-found for format-blob and blobcfg blob
if id == repo.BlobCfgBlobID || id == repo.FormatBlobID {
if id == format.KopiaBlobCfgBlobID || id == format.KopiaRepositoryBlobID {
return blob.ErrBlobNotFound
}
return nil
@@ -494,7 +495,7 @@ func(ctx context.Context, id blob.ID) error {
nil, nil,
// PutBlob callback
func(ctx context.Context, id blob.ID, _ *blob.PutOptions) error {
if id == repo.BlobCfgBlobID {
if id == format.KopiaBlobCfgBlobID {
return errors.New("unexpected error")
}
return nil
@@ -517,7 +518,7 @@ func(ctx context.Context, id blob.ID, _ *blob.PutOptions) error {
// GetBlob callback
func(ctx context.Context, id blob.ID) error {
// simulate not-found for format-blob and blobcfg blob
if id == repo.FormatBlobID {
if id == format.KopiaRepositoryBlobID {
return errors.New("unexpected error")
}
return nil
@@ -537,7 +538,7 @@ func TestInitializeWithNoRetention(t *testing.T) {
// are not supplied.
var b gather.WriteBuffer
defer b.Close()
require.NoError(t, env.RepositoryWriter.BlobStorage().GetBlob(ctx, repo.BlobCfgBlobID, 0, -1, &b))
require.NoError(t, env.RepositoryWriter.BlobStorage().GetBlob(ctx, format.KopiaBlobCfgBlobID, 0, -1, &b))
}
func TestObjectWritesWithRetention(t *testing.T) {
@@ -566,7 +567,7 @@ func TestObjectWritesWithRetention(t *testing.T) {
}
prefixesWithRetention = append(prefixesWithRetention, content.LegacyIndexBlobPrefix, epoch.EpochManagerIndexUberPrefix,
repo.FormatBlobID, repo.BlobCfgBlobID)
format.KopiaRepositoryBlobID, format.KopiaBlobCfgBlobID)
// make sure that we cannot set mtime on the kopia objects created due to the
// retention time constraint
@@ -635,7 +636,7 @@ func (s *formatSpecificTestSuite) TestWriteSessionFlushOnFailure(t *testing.T) {
func (s *formatSpecificTestSuite) TestChangePassword(t *testing.T) {
ctx, env := repotesting.NewEnvironment(t, s.formatVersion)
if s.formatVersion == content.FormatVersion1 {
if s.formatVersion == format.FormatVersion1 {
require.Error(t, env.RepositoryWriter.ChangePassword(ctx, "new-password"))
} else {
require.NoError(t, env.RepositoryWriter.ChangePassword(ctx, "new-password"))

10
repo/suite_test.go
View File

@@ -4,23 +4,23 @@
"testing"
"github.com/kopia/kopia/internal/testutil"
"github.com/kopia/kopia/repo/content"
"github.com/kopia/kopia/repo/format"
)
func TestMain(m *testing.M) { testutil.MyTestMain(m) }
type formatSpecificTestSuite struct {
formatVersion content.FormatVersion
formatVersion format.Version
}
func TestFormatV1(t *testing.T) {
testutil.RunAllTestsWithParam(t, &formatSpecificTestSuite{content.FormatVersion1})
testutil.RunAllTestsWithParam(t, &formatSpecificTestSuite{format.FormatVersion1})
}
func TestFormatV2(t *testing.T) {
testutil.RunAllTestsWithParam(t, &formatSpecificTestSuite{content.FormatVersion2})
testutil.RunAllTestsWithParam(t, &formatSpecificTestSuite{format.FormatVersion2})
}
func TestFormatV3(t *testing.T) {
testutil.RunAllTestsWithParam(t, &formatSpecificTestSuite{content.FormatVersion3})
testutil.RunAllTestsWithParam(t, &formatSpecificTestSuite{format.FormatVersion3})
}

38
repo/upgrade_lock.go
View File

@@ -9,7 +9,7 @@
"github.com/kopia/kopia/internal/gather"
"github.com/kopia/kopia/repo/blob"
"github.com/kopia/kopia/repo/content"
"github.com/kopia/kopia/repo/format"
)
// FormatBlobBackupIDPrefix is the prefix for all identifiers of the BLOBs that
@@ -18,14 +18,14 @@
const FormatBlobBackupIDPrefix = "kopia.repository.backup."
// FormatBlobBackupID gets the upgrade backu pblob-id fro mthe lock.
func FormatBlobBackupID(l UpgradeLockIntent) blob.ID {
func FormatBlobBackupID(l format.UpgradeLockIntent) blob.ID {
return blob.ID(FormatBlobBackupIDPrefix + l.OwnerID)
}
func (r *directRepository) updateRepoConfig(ctx context.Context, cb func(repoConfig *repositoryObjectFormat) error) (*repositoryObjectFormat, error) {
func (r *directRepository) updateRepoConfig(ctx context.Context, cb func(repoConfig *format.RepositoryConfig) error) (*format.RepositoryConfig, error) {
f := r.formatBlob
repoConfig, err := f.decryptFormatBytes(r.formatEncryptionKey)
repoConfig, err := f.DecryptRepositoryConfig(r.formatEncryptionKey)
if err != nil {
return nil, errors.Wrap(err, "unable to decrypt repository config")
}
@@ -34,16 +34,16 @@ func (r *directRepository) updateRepoConfig(ctx context.Context, cb func(repoCon
return nil, err
}
if err := encryptFormatBytes(f, repoConfig, r.formatEncryptionKey, f.UniqueID); err != nil {
if err := f.EncryptRepositoryConfig(repoConfig, r.formatEncryptionKey); err != nil {
return nil, errors.Errorf("unable to encrypt format bytes")
}
if err := writeFormatBlob(ctx, r.blobs, f, r.blobCfgBlob); err != nil {
if err := f.WriteKopiaRepositoryBlob(ctx, r.blobs, r.blobCfgBlob); err != nil {
return nil, errors.Wrap(err, "unable to write format blob")
}
if cd := r.cachingOptions.CacheDirectory; cd != "" {
if err := os.Remove(filepath.Join(cd, FormatBlobID)); err != nil && !os.IsNotExist(err) {
if err := os.Remove(filepath.Join(cd, format.KopiaRepositoryBlobID)); err != nil && !os.IsNotExist(err) {
return nil, errors.Errorf("unable to remove cached repository format blob: %v", err)
}
}
@@ -59,8 +59,8 @@ func (r *directRepository) updateRepoConfig(ctx context.Context, cb func(repoCon
// intent and sets the latest format-version o nthe repository blob. This
// should cause the unsupporting clients (non-upgrade capable) to fail
// connecting to the repository.
func (r *directRepository) SetUpgradeLockIntent(ctx context.Context, l UpgradeLockIntent) (*UpgradeLockIntent, error) {
repoConfig, err := r.updateRepoConfig(ctx, func(repoConfig *repositoryObjectFormat) error {
func (r *directRepository) SetUpgradeLockIntent(ctx context.Context, l format.UpgradeLockIntent) (*format.UpgradeLockIntent, error) {
repoConfig, err := r.updateRepoConfig(ctx, func(repoConfig *format.RepositoryConfig) error {
if err := l.Validate(); err != nil {
return errors.Wrap(err, "invalid upgrade lock intent")
}
@@ -68,14 +68,14 @@ func (r *directRepository) SetUpgradeLockIntent(ctx context.Context, l UpgradeLo
if repoConfig.UpgradeLock == nil {
// when we are putting a new lock then ensure that we can upgrade
// to that version
if repoConfig.FormattingOptions.Version >= content.MaxFormatVersion {
if repoConfig.ContentFormat.Version >= format.MaxFormatVersion {
return errors.Errorf("repository is using version %d, and version %d is the maximum",
repoConfig.FormattingOptions.Version, content.MaxFormatVersion)
repoConfig.ContentFormat.Version, format.MaxFormatVersion)
}
// backup the current repository config from local cache to the
// repository when we place the lock for the first time
if err := writeFormatBlobWithID(ctx, r.blobs, r.formatBlob, r.blobCfgBlob, FormatBlobBackupID(l)); err != nil {
if err := r.formatBlob.WriteKopiaRepositoryBlobWithID(ctx, r.blobs, r.blobCfgBlob, FormatBlobBackupID(l)); err != nil {
return errors.Wrap(err, "failed to backup the repo format blob")
}
@@ -83,7 +83,7 @@ func (r *directRepository) SetUpgradeLockIntent(ctx context.Context, l UpgradeLo
repoConfig.UpgradeLock = &l
// mark the upgrade to the new format version, this will ensure that older
// clients won't be able to parse the new version
repoConfig.FormattingOptions.Version = content.MaxFormatVersion
repoConfig.ContentFormat.Version = format.MaxFormatVersion
} else if newL, err := repoConfig.UpgradeLock.Update(&l); err == nil {
repoConfig.UpgradeLock = newL
} else {
@@ -103,7 +103,7 @@ func (r *directRepository) SetUpgradeLockIntent(ctx context.Context, l UpgradeLo
// blob. This in-effect commits the new repository format t othe repository and
// resumes all access to the repository.
func (r *directRepository) CommitUpgrade(ctx context.Context) error {
_, err := r.updateRepoConfig(ctx, func(repoConfig *repositoryObjectFormat) error {
_, err := r.updateRepoConfig(ctx, func(repoConfig *format.RepositoryConfig) error {
if repoConfig.UpgradeLock == nil {
return errors.New("no upgrade in progress")
}
@@ -128,7 +128,7 @@ func (r *directRepository) CommitUpgrade(ctx context.Context) error {
func (r *directRepository) RollbackUpgrade(ctx context.Context) error {
f := r.formatBlob
repoConfig, err := f.decryptFormatBytes(r.formatEncryptionKey)
repoConfig, err := f.DecryptRepositoryConfig(r.formatEncryptionKey)
if err != nil {
return errors.Wrap(err, "unable to decrypt repository config")
}
@@ -171,7 +171,7 @@ func (r *directRepository) RollbackUpgrade(ctx context.Context) error {
return errors.Wrapf(err, "failed to read from backup %q", oldestBackup.BlobID)
}
if err = r.blobs.PutBlob(ctx, FormatBlobID, d.Bytes(), blob.PutOptions{}); err != nil {
if err = r.blobs.PutBlob(ctx, format.KopiaRepositoryBlobID, d.Bytes(), blob.PutOptions{}); err != nil {
return errors.Wrapf(err, "failed to restore format blob from backup %q", oldestBackup.BlobID)
}
@@ -182,7 +182,7 @@ func (r *directRepository) RollbackUpgrade(ctx context.Context) error {
}
if cd := r.cachingOptions.CacheDirectory; cd != "" {
if err = os.Remove(filepath.Join(cd, FormatBlobID)); err != nil && !os.IsNotExist(err) {
if err = os.Remove(filepath.Join(cd, format.KopiaRepositoryBlobID)); err != nil && !os.IsNotExist(err) {
return errors.Errorf("unable to remove cached repository format blob: %v", err)
}
}
@@ -190,10 +190,10 @@ func (r *directRepository) RollbackUpgrade(ctx context.Context) error {
return nil
}
func (r *directRepository) GetUpgradeLockIntent(ctx context.Context) (*UpgradeLockIntent, error) {
func (r *directRepository) GetUpgradeLockIntent(ctx context.Context) (*format.UpgradeLockIntent, error) {
f := r.formatBlob
repoConfig, err := f.decryptFormatBytes(r.formatEncryptionKey)
repoConfig, err := f.DecryptRepositoryConfig(r.formatEncryptionKey)
if err != nil {
return nil, errors.Wrap(err, "unable to decrypt repository config")
}

42
repo/upgrade_lock_test.go
View File

@@ -23,17 +23,17 @@
"github.com/kopia/kopia/repo/blob/beforeop"
"github.com/kopia/kopia/repo/content"
"github.com/kopia/kopia/repo/encryption"
"github.com/kopia/kopia/repo/object"
"github.com/kopia/kopia/repo/format"
)
func TestFormatUpgradeSetLock(t *testing.T) {
ctx, env := repotesting.NewEnvironment(t, content.FormatVersion1, repotesting.Options{OpenOptions: func(opts *repo.Options) {
ctx, env := repotesting.NewEnvironment(t, format.FormatVersion1, repotesting.Options{OpenOptions: func(opts *repo.Options) {
// nolint:goconst
opts.UpgradeOwnerID = "upgrade-owner"
}})
formatBlockCacheDuration := env.Repository.ClientOptions().FormatBlobCacheDuration
l := &repo.UpgradeLockIntent{
l := &format.UpgradeLockIntent{
CreationTime: env.Repository.Time(),
AdvanceNoticeDuration: 15 * time.Hour,
IODrainTimeout: formatBlockCacheDuration * 2,
@@ -70,10 +70,10 @@ func TestFormatUpgradeSetLock(t *testing.T) {
}
func TestFormatUpgradeAlreadyUpgraded(t *testing.T) {
ctx, env := repotesting.NewEnvironment(t, content.MaxFormatVersion)
ctx, env := repotesting.NewEnvironment(t, format.MaxFormatVersion)
formatBlockCacheDuration := env.Repository.ClientOptions().FormatBlobCacheDuration
l := &repo.UpgradeLockIntent{
l := &format.UpgradeLockIntent{
OwnerID: "new-upgrade-owner",
CreationTime: env.Repository.Time(),
AdvanceNoticeDuration: 0,
@@ -85,16 +85,16 @@ func TestFormatUpgradeAlreadyUpgraded(t *testing.T) {
_, err := env.RepositoryWriter.SetUpgradeLockIntent(ctx, *l)
require.EqualError(t, err, fmt.Sprintf("repository is using version %d, and version %d is the maximum",
content.MaxFormatVersion, content.MaxFormatVersion))
format.MaxFormatVersion, format.MaxFormatVersion))
}
func TestFormatUpgradeCommit(t *testing.T) {
ctx, env := repotesting.NewEnvironment(t, content.FormatVersion1, repotesting.Options{OpenOptions: func(opts *repo.Options) {
ctx, env := repotesting.NewEnvironment(t, format.FormatVersion1, repotesting.Options{OpenOptions: func(opts *repo.Options) {
opts.UpgradeOwnerID = "upgrade-owner"
}})
formatBlockCacheDuration := env.Repository.ClientOptions().FormatBlobCacheDuration
l := &repo.UpgradeLockIntent{
l := &format.UpgradeLockIntent{
OwnerID: "upgrade-owner",
CreationTime: env.Repository.Time(),
AdvanceNoticeDuration: 0,
@@ -116,12 +116,12 @@ func TestFormatUpgradeCommit(t *testing.T) {
}
func TestFormatUpgradeRollback(t *testing.T) {
ctx, env := repotesting.NewEnvironment(t, content.FormatVersion1, repotesting.Options{OpenOptions: func(opts *repo.Options) {
ctx, env := repotesting.NewEnvironment(t, format.FormatVersion1, repotesting.Options{OpenOptions: func(opts *repo.Options) {
opts.UpgradeOwnerID = "upgrade-owner"
}})
formatBlockCacheDuration := env.Repository.ClientOptions().FormatBlobCacheDuration
l := &repo.UpgradeLockIntent{
l := &format.UpgradeLockIntent{
OwnerID: "upgrade-owner",
CreationTime: env.Repository.Time(),
AdvanceNoticeDuration: 0,
@@ -144,12 +144,12 @@ func TestFormatUpgradeRollback(t *testing.T) {
}
func TestFormatUpgradeMultipleLocksRollback(t *testing.T) {
ctx, env := repotesting.NewEnvironment(t, content.FormatVersion1, repotesting.Options{OpenOptions: func(opts *repo.Options) {
ctx, env := repotesting.NewEnvironment(t, format.FormatVersion1, repotesting.Options{OpenOptions: func(opts *repo.Options) {
opts.UpgradeOwnerID = "upgrade-owner"
}})
formatBlockCacheDuration := env.Repository.ClientOptions().FormatBlobCacheDuration
l := &repo.UpgradeLockIntent{
l := &format.UpgradeLockIntent{
OwnerID: "upgrade-owner",
CreationTime: env.Repository.Time(),
AdvanceNoticeDuration: 0,
@@ -190,7 +190,7 @@ func TestFormatUpgradeMultipleLocksRollback(t *testing.T) {
env.MustReopen(t, func(opts *repo.Options) {
opts.UpgradeOwnerID = "another-upgrade-owner"
})
require.Equal(t, content.FormatVersion3,
require.Equal(t, format.FormatVersion3,
env.RepositoryWriter.ContentManager().ContentFormat().FormatVersion())
require.NoError(t, env.RepositoryWriter.RollbackUpgrade(ctx))
@@ -210,13 +210,13 @@ func TestFormatUpgradeMultipleLocksRollback(t *testing.T) {
require.EqualError(t, env.RepositoryWriter.CommitUpgrade(ctx), "no upgrade in progress")
// verify that we are back to the original version where we started from
require.Equal(t, content.FormatVersion1,
require.Equal(t, format.FormatVersion1,
env.RepositoryWriter.ContentManager().ContentFormat().FormatVersion())
}
func TestFormatUpgradeFailureToBackupFormatBlobOnLock(t *testing.T) {
// this lock will be allowed by the backend to create backups
allowedLock := repo.UpgradeLockIntent{
allowedLock := format.UpgradeLockIntent{
OwnerID: "allowed-upgrade-owner",
CreationTime: clock.Now(),
AdvanceNoticeDuration: 0,
@@ -256,16 +256,16 @@ func(ctx context.Context, id blob.ID, _ *blob.PutOptions) error {
))
opt := &repo.NewRepositoryOptions{
BlockFormat: content.FormattingOptions{
MutableParameters: content.MutableParameters{
Version: content.FormatVersion1,
BlockFormat: format.ContentFormat{
MutableParameters: format.MutableParameters{
Version: format.FormatVersion1,
},
HMACSecret: []byte{},
Hash: "HMAC-SHA256",
Encryption: encryption.DefaultAlgorithm,
EnablePasswordChange: true,
},
ObjectFormat: object.Format{
ObjectFormat: format.ObjectFormat{
Splitter: "FIXED-1M",
},
}
@@ -311,7 +311,7 @@ func(ctx context.Context, id blob.ID, _ *blob.PutOptions) error {
func TestFormatUpgradeDuringOngoingWriteSessions(t *testing.T) {
curTime := clock.Now()
ctx, env := repotesting.NewEnvironment(t, content.FormatVersion1, repotesting.Options{
ctx, env := repotesting.NewEnvironment(t, format.FormatVersion1, repotesting.Options{
// new environment with controlled time
OpenOptions: func(opts *repo.Options) {
opts.TimeNowFunc = func() time.Time {
@@ -351,7 +351,7 @@ func TestFormatUpgradeDuringOngoingWriteSessions(t *testing.T) {
writeObject(ctx, t, lw, o4Data, "o4")
formatBlockCacheDuration := env.Repository.ClientOptions().FormatBlobCacheDuration
l := repo.UpgradeLockIntent{
l := format.UpgradeLockIntent{
OwnerID: "upgrade-owner",
CreationTime: env.Repository.Time(),
AdvanceNoticeDuration: 0,

18
site/content/docs/Advanced/Encryption/_index.md
View File

@@ -40,7 +40,7 @@ type formatBlob struct {
Version string `json:"version"`
EncryptionAlgorithm string `json:"encryption"`
EncryptedFormatBytes []byte `json:"encryptedBlockFormat,omitempty"`
UnencryptedFormat *repositoryObjectFormat `json:"blockFormat,omitempty"`
UnencryptedFormat *format.RepositoryObjectFormat `json:"blockFormat,omitempty"`
}
```
@@ -53,25 +53,25 @@ type formatBlob struct {
* `encryptedBlockFormat` is a ciphertext containing among others, the encryption secrets and parameters used for encrypting the repository content. Below is additional information about its plaintext content and how it is encrypted.
* Alternatively, the unencrypted block format parameters can be specified in the the `blockFormat` field.
The `formatBlob.encryptedBlockFormat` field is the result of encrypting a JSON-serialized version of the `encryptedRepositoryConfig` struct shown below. The plaintext version contains the parameters for performing block chunking, as well as for encrypting and authenticating "content" objects.
The `formatBlob.EncryptedBlockFormat` field is the result of encrypting a JSON-serialized version of the `EncryptedRepositoryConfig` struct shown below. The plaintext version contains the parameters for performing block chunking, as well as for encrypting and authenticating "content" objects.
```go
type encryptedRepositoryConfig struct {
Format repositoryObjectFormat `json:"format"`
type EncryptedRepositoryConfig struct {
Format RepositoryObjectFormat `json:"format"`
}
type repositoryObjectFormat struct {
content.FormattingOptions
object.Format
type RepositoryObjectFormat struct {
format.ContentFormat
format.ObjectFormat
}
```
```go
package content
// FormattingOptions describes the rules for formatting contents in repository.
type FormattingOptions struct {
// ContentFormat describes the rules for formatting contents in repository.
type ContentFormat struct {
Version int `json:"version,omitempty"` // version number, must be "1"
Hash string `json:"hash,omitempty"` // identifier of the hash algorithm used
Encryption string `json:"encryption,omitempty"` // identifier of the encryption algorithm used

5
snapshot/snapshotmaintenance/snapshotmaintenance_test.go
View File

@@ -17,6 +17,7 @@
"github.com/kopia/kopia/internal/testlogging"
"github.com/kopia/kopia/repo"
"github.com/kopia/kopia/repo/content"
"github.com/kopia/kopia/repo/format"
"github.com/kopia/kopia/repo/maintenance"
"github.com/kopia/kopia/repo/object"
"github.com/kopia/kopia/snapshot"
@@ -230,7 +231,7 @@ func (s *formatSpecificTestSuite) TestSnapshotGCMinContentAgeSafety(t *testing.T
checkContentDeletion(t, th.Repository, cids, false)
}
func newTestHarness(t *testing.T, formatVersion content.FormatVersion) *testHarness {
func newTestHarness(t *testing.T, formatVersion format.Version) *testHarness {
t.Helper()
baseTime := time.Date(2020, 9, 10, 0, 0, 0, 0, time.UTC)
@@ -254,7 +255,7 @@ func (s *formatSpecificTestSuite) TestMaintenanceAutoLiveness(t *testing.T) {
o.TimeNowFunc = ft.NowFunc()
},
NewRepositoryOptions: func(nro *repo.NewRepositoryOptions) {
nro.BlockFormat.Version = content.FormatVersion1
nro.BlockFormat.Version = format.FormatVersion1
},
})

10
snapshot/snapshotmaintenance/suite_test.go
View File

@@ -4,21 +4,21 @@
"testing"
"github.com/kopia/kopia/internal/testutil"
"github.com/kopia/kopia/repo/content"
"github.com/kopia/kopia/repo/format"
)
type formatSpecificTestSuite struct {
formatVersion content.FormatVersion
formatVersion format.Version
}
func TestFormatV1(t *testing.T) {
testutil.RunAllTestsWithParam(t, &formatSpecificTestSuite{content.FormatVersion1})
testutil.RunAllTestsWithParam(t, &formatSpecificTestSuite{format.FormatVersion1})
}
func TestFormatV2(t *testing.T) {
testutil.RunAllTestsWithParam(t, &formatSpecificTestSuite{content.FormatVersion2})
testutil.RunAllTestsWithParam(t, &formatSpecificTestSuite{format.FormatVersion2})
}
func TestFormatV3(t *testing.T) {
testutil.RunAllTestsWithParam(t, &formatSpecificTestSuite{content.FormatVersion3})
testutil.RunAllTestsWithParam(t, &formatSpecificTestSuite{format.FormatVersion3})
}

4
tests/end_to_end_test/repository_repair_test.go
View File

@@ -3,7 +3,7 @@
import (
"testing"
"github.com/kopia/kopia/repo/content"
"github.com/kopia/kopia/repo/format"
"github.com/kopia/kopia/tests/testenv"
)
@@ -33,7 +33,7 @@ func (s *formatSpecificTestSuite) TestRepositoryRepair(t *testing.T) {
// this will fail because the format blob in the repository is not found
e.RunAndExpectFailure(t, "repo", "connect", "filesystem", "--path", e.RepoDir)
if s.formatVersion == content.FormatVersion1 {
if s.formatVersion == format.FormatVersion1 {
// now run repair, which will recover the format blob from one of the pack blobs.
e.RunAndExpectSuccess(t, "repo", "repair", "filesystem", "--path", e.RepoDir)

4
tests/end_to_end_test/snapshot_create_test.go
View File

@@ -15,8 +15,8 @@
"github.com/stretchr/testify/require"
"github.com/kopia/kopia/cli"
"github.com/kopia/kopia/internal/cachedir"
"github.com/kopia/kopia/internal/testutil"
"github.com/kopia/kopia/repo"
"github.com/kopia/kopia/snapshot"
"github.com/kopia/kopia/tests/clitestutil"
"github.com/kopia/kopia/tests/testenv"
@@ -209,7 +209,7 @@ func TestSnapshottingCacheDirectory(t *testing.T) {
cachePath := filepath.Dir(strings.Split(lines[0], ": ")[0])
// verify cache marker exists
if _, err := os.Stat(filepath.Join(cachePath, repo.CacheDirMarkerFile)); err != nil {
if _, err := os.Stat(filepath.Join(cachePath, cachedir.CacheDirMarkerFile)); err != nil {
t.Fatal(err)
}

10
tests/end_to_end_test/suite_test.go
View File

@@ -4,22 +4,22 @@
"testing"
"github.com/kopia/kopia/internal/testutil"
"github.com/kopia/kopia/repo/content"
"github.com/kopia/kopia/repo/format"
)
type formatSpecificTestSuite struct {
formatFlags []string
formatVersion content.FormatVersion
formatVersion format.Version
}
func TestFormatV1(t *testing.T) {
testutil.RunAllTestsWithParam(t, &formatSpecificTestSuite{[]string{"--format-version=1"}, content.FormatVersion1})
testutil.RunAllTestsWithParam(t, &formatSpecificTestSuite{[]string{"--format-version=1"}, format.FormatVersion1})
}
func TestFormatV2(t *testing.T) {
testutil.RunAllTestsWithParam(t, &formatSpecificTestSuite{[]string{"--format-version=2"}, content.FormatVersion2})
testutil.RunAllTestsWithParam(t, &formatSpecificTestSuite{[]string{"--format-version=2"}, format.FormatVersion2})
}
func TestFormatV3(t *testing.T) {
testutil.RunAllTestsWithParam(t, &formatSpecificTestSuite{[]string{"--format-version=3"}, content.FormatVersion3})
testutil.RunAllTestsWithParam(t, &formatSpecificTestSuite{[]string{"--format-version=3"}, format.FormatVersion3})
}

5
tests/stress_test/stress_test.go
View File

@@ -18,6 +18,7 @@
"github.com/kopia/kopia/repo/blob"
"github.com/kopia/kopia/repo/content"
"github.com/kopia/kopia/repo/encryption"
"github.com/kopia/kopia/repo/format"
)
const goroutineCount = 16
@@ -47,10 +48,10 @@ func TestStressBlockManager(t *testing.T) {
func stressTestWithStorage(t *testing.T, st blob.Storage, duration time.Duration) {
ctx := testlogging.Context(t)
fop, err := content.NewFormattingOptionsProvider(&content.FormattingOptions{
fop, err := format.NewFormattingOptionsProvider(&format.ContentFormat{
Hash: "HMAC-SHA256-128",
Encryption: encryption.DefaultAlgorithm,
MutableParameters: content.MutableParameters{
MutableParameters: format.MutableParameters{
Version: 1,
MaxPackSize: 20000000,
},

6
tests/tools/kopiarunner/kopia_snapshotter_upgrade_test.go
View File

@@ -6,7 +6,7 @@
"github.com/stretchr/testify/require"
"github.com/kopia/kopia/repo/content"
"github.com/kopia/kopia/repo/format"
)
func TestUpgradeFormatVersion(t *testing.T) {
@@ -24,7 +24,7 @@ func TestUpgradeFormatVersion(t *testing.T) {
require.NoError(t, err)
prev := rs.ContentFormat.MutableParameters.Version
require.Equal(t, prev, content.FormatVersion(1), "The format version should be 1.")
require.Equal(t, prev, format.Version(1), "The format version should be 1.")
ks.UpgradeRepository(repoDir)
@@ -32,7 +32,7 @@ func TestUpgradeFormatVersion(t *testing.T) {
require.NoError(t, err)
got := rs.ContentFormat.MutableParameters.Version
require.Equal(t, got, content.FormatVersion(2), "The format version should be upgraded to 2.")
require.Equal(t, got, format.Version(2), "The format version should be upgraded to 2.")
require.NotEqual(t, got, prev, "The format versions should be different.")
}