mirror/kopia
1
0
Fork 0
mirror of https://github.com/kopia/kopia.git synced 2026-07-29 06:46:58 -04:00
Code Issues Packages Projects Releases Wiki Activity

removed support for legacy block format, to migrate sync to previous commit and run 'kopia repo upgrade'

Browse Source
This commit is contained in:
Jarek Kowalski
2018-12-31 19:01:08 -08:00
parent 24bd5bbe1f
commit 840d5ab749
10 changed files with 45 additions and 71 deletions
Download Patch File Download Diff File Expand all files Collapse all files

13
block/block_formatting_options.go
View File

@@ -2,11 +2,10 @@
// FormattingOptions describes the rules for formatting blocks in repository.
type FormattingOptions struct {
Version int `json:"version,omitempty"` // version number, must be "1"
LegacyBlockFormat string `json:"objectFormat,omitempty"` // identifier of the block format (legacy)
Hash string `json:"hash,omitempty"` // identifier of the hash algorithm used
Encryption string `json:"encryption,omitempty"` // identifier of the encryption algorithm used
HMACSecret []byte `json:"secret,omitempty"` // HMAC secret used to generate encryption keys
MasterKey []byte `json:"masterKey,omitempty"` // master encryption key (SIV-mode encryption only)
MaxPackSize int `json:"maxPackSize,omitempty"` // maximum size of a pack object
Version int `json:"version,omitempty"` // version number, must be "1"
Hash string `json:"hash,omitempty"` // identifier of the hash algorithm used
Encryption string `json:"encryption,omitempty"` // identifier of the encryption algorithm used
HMACSecret []byte `json:"secret,omitempty"` // HMAC secret used to generate encryption keys
MasterKey []byte `json:"masterKey,omitempty"` // master encryption key (SIV-mode encryption only)
MaxPackSize int `json:"maxPackSize,omitempty"` // maximum size of a pack object
}

17
block/block_manager.go
View File

@@ -935,8 +935,6 @@ func newManagerWithOptions(ctx context.Context, st storage.Storage, f Formatting
return nil, fmt.Errorf("can't handle repositories created using version %v (min supported %v, max supported %v)", f.Version, minSupportedReadVersion, maxSupportedReadVersion)
}
applyLegacyBlockFormat(&f)
hasher, encryptor, err := CreateHashAndEncryptor(f)
if err != nil {
return nil, err
@@ -993,6 +991,7 @@ func CreateHashAndEncryptor(f FormattingOptions) (HashFunc, Encryptor, error) {
if err != nil {
return nil, nil, fmt.Errorf("unable to create hash: %v", err)
}
e, err := createEncryptor(f)
if err != nil {
return nil, nil, fmt.Errorf("unable to create encryptor: %v", err)
@@ -1039,17 +1038,3 @@ func curryEncryptionKey(n func(k []byte) (cipher.Block, error), key []byte) func
return n(key)
}
}
func applyLegacyBlockFormat(f *FormattingOptions) {
switch f.LegacyBlockFormat {
case "UNENCRYPTED_HMAC_SHA256":
f.Hash = "HMAC-SHA256"
f.Encryption = "NONE"
case "UNENCRYPTED_HMAC_SHA256_128":
f.Hash = "HMAC-SHA256-128"
f.Encryption = "NONE"
case "ENCRYPTED_HMAC_SHA256_AES256_SIV":
f.Hash = "HMAC-SHA256-128"
f.Encryption = "AES-256-CTR"
}
}

18
block/block_manager_test.go
View File

@@ -270,11 +270,12 @@ func TestBlockManagerFailedToWritePack(t *testing.T) {
st = faulty
bm, err := newManagerWithOptions(context.Background(), st, FormattingOptions{
Version: 1,
LegacyBlockFormat: "ENCRYPTED_HMAC_SHA256_AES256_SIV",
MaxPackSize: maxPackSize,
HMACSecret: []byte("foo"),
MasterKey: []byte("0123456789abcdef0123456789abcdef"),
Version: 1,
Hash: "HMAC-SHA256-128",
Encryption: "AES-256-CTR",
MaxPackSize: maxPackSize,
HMACSecret: []byte("foo"),
MasterKey: []byte("0123456789abcdef0123456789abcdef"),
}, CachingOptions{}, fakeTimeNowFrozen(fakeTime))
if err != nil {
t.Fatalf("can't create bm: %v", err)
@@ -785,9 +786,10 @@ func newTestBlockManager(data map[string][]byte, keyTime map[string]time.Time, t
}
st := storagetesting.NewMapStorage(data, keyTime, timeFunc)
bm, err := newManagerWithOptions(context.Background(), st, FormattingOptions{
LegacyBlockFormat: "UNENCRYPTED_HMAC_SHA256",
HMACSecret: hmacSecret,
MaxPackSize: maxPackSize,
Hash: "HMAC-SHA256",
Encryption: "NONE",
HMACSecret: hmacSecret,
MaxPackSize: maxPackSize,
}, CachingOptions{}, timeFunc)
if err != nil {
panic("can't create block manager: " + err.Error())

2
go.mod
View File

@@ -13,7 +13,7 @@ require (
go.opencensus.io v0.18.0 // indirect
golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9
golang.org/x/exp v0.0.0-20181221233300-b68661188fbf
golang.org/x/net v0.0.0-20181220203305-927f97764cc3 // indirect
golang.org/x/net v0.0.0-20181220203305-927f97764cc3
golang.org/x/oauth2 v0.0.0-20181203162652-d668ce993890
golang.org/x/sys v0.0.0-20181228144115-9a3f9b0469bb // indirect
google.golang.org/api v0.0.0-20181229000844-f26a60c56f14

13
initialize.go
View File

@@ -82,13 +82,12 @@ func formatBlockFromOptions(opt *NewRepositoryOptions) *formatBlock {
func repositoryObjectFormatFromOptions(opt *NewRepositoryOptions) *repositoryObjectFormat {
f := &repositoryObjectFormat{
FormattingOptions: block.FormattingOptions{
Version: 1,
LegacyBlockFormat: opt.BlockFormat.LegacyBlockFormat,
Hash: applyDefaultString(opt.BlockFormat.Hash, block.DefaultHash),
Encryption: applyDefaultString(opt.BlockFormat.Encryption, block.DefaultEncryption),
HMACSecret: applyDefaultRandomBytes(opt.BlockFormat.HMACSecret, 32),
MasterKey: applyDefaultRandomBytes(opt.BlockFormat.MasterKey, 32),
MaxPackSize: applyDefaultInt(opt.BlockFormat.MaxPackSize, applyDefaultInt(opt.ObjectFormat.MaxBlockSize, 20<<20)), // 20 MB
Version: 1,
Hash: applyDefaultString(opt.BlockFormat.Hash, block.DefaultHash),
Encryption: applyDefaultString(opt.BlockFormat.Encryption, block.DefaultEncryption),
HMACSecret: applyDefaultRandomBytes(opt.BlockFormat.HMACSecret, 32),
MasterKey: applyDefaultRandomBytes(opt.BlockFormat.MasterKey, 32),
MaxPackSize: applyDefaultInt(opt.BlockFormat.MaxPackSize, applyDefaultInt(opt.ObjectFormat.MaxBlockSize, 20<<20)), // 20 MB
},
Format: object.Format{
Splitter: applyDefaultString(opt.ObjectFormat.Splitter, object.DefaultSplitter),

5
internal/repotesting/repotesting.go
View File

@@ -45,8 +45,9 @@ func (e *Environment) Setup(t *testing.T, opts ...func(*repo.NewRepositoryOption
opt := &repo.NewRepositoryOptions{
BlockFormat: block.FormattingOptions{
HMACSecret: []byte{},
LegacyBlockFormat: "UNENCRYPTED_HMAC_SHA256",
HMACSecret: []byte{},
Hash: "HMAC-SHA256",
Encryption: "NONE",
},
ObjectFormat: object.Format{
Splitter: "FIXED",

10
manifest/manifest_manager_test.go
View File

@@ -129,8 +129,9 @@ func TestManifestInitCorruptedBlock(t *testing.T) {
st := storagetesting.NewMapStorage(data, nil, nil)
f := block.FormattingOptions{
LegacyBlockFormat: "UNENCRYPTED_HMAC_SHA256_128",
MaxPackSize: 100000,
Hash: "HMAC-SHA256-128",
Encryption: "NONE",
MaxPackSize: 100000,
}
// write some data to storage
@@ -266,8 +267,9 @@ func newManagerForTesting(ctx context.Context, t *testing.T, data map[string][]b
st := storagetesting.NewMapStorage(data, nil, nil)
bm, err := block.NewManager(ctx, st, block.FormattingOptions{
LegacyBlockFormat: "UNENCRYPTED_HMAC_SHA256_128",
MaxPackSize: 100000,
Hash: "HMAC-SHA256-128",
Encryption: "NONE",
MaxPackSize: 100000,
}, block.CachingOptions{})
if err != nil {
return nil, fmt.Errorf("can't create block manager: %v", err)

9
repository_test.go
View File

@@ -256,9 +256,10 @@ func verify(ctx context.Context, t *testing.T, rep *repo.Repository, objectID ob
func TestFormats(t *testing.T) {
ctx := context.Background()
makeFormat := func(blockFormat string) func(*repo.NewRepositoryOptions) {
makeFormat := func(hash, encryption string) func(*repo.NewRepositoryOptions) {
return func(n *repo.NewRepositoryOptions) {
n.BlockFormat.LegacyBlockFormat = blockFormat
n.BlockFormat.Hash = hash
n.BlockFormat.Encryption = encryption
n.BlockFormat.HMACSecret = []byte("key")
n.ObjectFormat.MaxBlockSize = 10000
n.ObjectFormat.Splitter = "FIXED"
@@ -279,13 +280,13 @@ func TestFormats(t *testing.T) {
},
},
{
format: makeFormat("UNENCRYPTED_HMAC_SHA256"),
format: makeFormat("HMAC-SHA256", "NONE"),
oids: map[string]object.ID{
"The quick brown fox jumps over the lazy dog": "f7bc83f430538424b13298e6aa6fb143ef4d59a14946175997479dbc2d1a3cd8",
},
},
{
format: makeFormat("UNENCRYPTED_HMAC_SHA256_128"),
format: makeFormat("HMAC-SHA256-128", "NONE"),
oids: map[string]object.ID{
"The quick brown fox jumps over the lazy dog": "f7bc83f430538424b13298e6aa6fb143",
},

11
tests/stress_test/stress_test.go
View File

@@ -38,10 +38,11 @@ func stressTestWithStorage(t *testing.T, st storage.Storage, duration time.Durat
openMgr := func() (*block.Manager, error) {
return block.NewManager(ctx, st, block.FormattingOptions{
Version: 1,
LegacyBlockFormat: "ENCRYPTED_HMAC_SHA256_AES256_SIV",
MaxPackSize: 20000000,
MasterKey: []byte{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15},
Version: 1,
Hash: "HMAC-SHA256-128",
Encryption: "AES-256-CTR",
MaxPackSize: 20000000,
MasterKey: []byte{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15},
}, block.CachingOptions{})
}
@@ -68,7 +69,7 @@ func stressWorker(ctx context.Context, t *testing.T, deadline time.Time, workerI
bm, err := openMgr()
if err != nil {
t.Errorf("error opening manager: %v", err)
t.Fatalf("error opening manager: %v", err)
}
type writtenBlock struct {

18
upgrade.go
View File

@@ -17,23 +17,7 @@ func (r *Repository) Upgrade(ctx context.Context) error {
var migrated bool
if repoConfig.FormattingOptions.LegacyBlockFormat != "" {
log.Infof("upgrading from legacy block format to explicit hash/encryption spec")
switch repoConfig.FormattingOptions.LegacyBlockFormat {
case "UNENCRYPTED_HMAC_SHA256":
repoConfig.FormattingOptions.Hash = "HMAC-SHA256"
repoConfig.FormattingOptions.Encryption = "NONE"
case "UNENCRYPTED_HMAC_SHA256_128":
repoConfig.FormattingOptions.Hash = "HMAC-SHA256-128"
repoConfig.FormattingOptions.Encryption = "NONE"
case "ENCRYPTED_HMAC_SHA256_AES256_SIV":
repoConfig.FormattingOptions.Hash = "HMAC-SHA256-128"
repoConfig.FormattingOptions.Encryption = "AES-256-CTR"
}
repoConfig.FormattingOptions.LegacyBlockFormat = ""
migrated = true
}
// TODO(jkowalski): add migration code here
if !migrated {
log.Infof("nothing to do")
return nil