From 7c41f45e9fa5dff9417723e478d9b58150dc46b0 Mon Sep 17 00:00:00 2001 From: Jarek Kowalski Date: Sun, 29 Oct 2017 13:00:24 -0700 Subject: [PATCH] extracted block formatting options into a struct under 'block' --- block/block_formatter.go | 12 +++++------- block/block_formatter_test.go | 4 +--- block/block_formatting_options.go | 10 ++++++++++ block/block_manager_test.go | 3 +-- internal/config/local_config.go | 15 ++++++--------- object/object_manager_test.go | 4 +++- repo/initialize.go | 20 +++++++++++--------- repo/open.go | 2 +- 8 files changed, 38 insertions(+), 32 deletions(-) create mode 100644 block/block_formatting_options.go diff --git a/block/block_formatter.go b/block/block_formatter.go index da13cf162..29dcc5e38 100644 --- a/block/block_formatter.go +++ b/block/block_formatter.go @@ -11,8 +11,6 @@ import ( "hash" "sort" "strings" - - "github.com/kopia/kopia/internal/config" ) // Formatter performs data block ID computation and encryption of a block of data when storing object in a repository. @@ -112,17 +110,17 @@ func decodeHexSuffix(s string, length int) ([]byte, error) { var SupportedFormats []string // FormatterFactories maps known block formatters to their factory functions. -var FormatterFactories = map[string]func(f config.RepositoryObjectFormat) (Formatter, error){ - "TESTONLY_MD5": func(f config.RepositoryObjectFormat) (Formatter, error) { +var FormatterFactories = map[string]func(f FormattingOptions) (Formatter, error){ + "TESTONLY_MD5": func(f FormattingOptions) (Formatter, error) { return &unencryptedFormat{computeHash(md5.New, md5.Size)}, nil }, - "UNENCRYPTED_HMAC_SHA256": func(f config.RepositoryObjectFormat) (Formatter, error) { + "UNENCRYPTED_HMAC_SHA256": func(f FormattingOptions) (Formatter, error) { return &unencryptedFormat{computeHMAC(sha256.New, f.HMACSecret, sha256.Size)}, nil }, - "UNENCRYPTED_HMAC_SHA256_128": func(f config.RepositoryObjectFormat) (Formatter, error) { + "UNENCRYPTED_HMAC_SHA256_128": func(f FormattingOptions) (Formatter, error) { return &unencryptedFormat{computeHMAC(sha256.New, f.HMACSecret, 16)}, nil }, - "ENCRYPTED_HMAC_SHA256_AES256_SIV": func(f config.RepositoryObjectFormat) (Formatter, error) { + "ENCRYPTED_HMAC_SHA256_AES256_SIV": func(f FormattingOptions) (Formatter, error) { if len(f.MasterKey) < 32 { return nil, fmt.Errorf("master key is not set") } diff --git a/block/block_formatter_test.go b/block/block_formatter_test.go index cd4b4bf63..7429b9012 100644 --- a/block/block_formatter_test.go +++ b/block/block_formatter_test.go @@ -5,13 +5,11 @@ import ( "crypto/rand" "crypto/sha1" "testing" - - "github.com/kopia/kopia/internal/config" ) func TestFormatters(t *testing.T) { secret := []byte("secret") - f := config.RepositoryObjectFormat{HMACSecret: secret, MasterKey: make([]byte, 32)} + f := FormattingOptions{HMACSecret: secret, MasterKey: make([]byte, 32)} for k, v := range FormatterFactories { data := make([]byte, 100) diff --git a/block/block_formatting_options.go b/block/block_formatting_options.go new file mode 100644 index 000000000..872b06b9c --- /dev/null +++ b/block/block_formatting_options.go @@ -0,0 +1,10 @@ +package block + +// FormattingOptions describes the rules for formatting blocks in repository. +type FormattingOptions struct { + Version int `json:"version,omitempty"` // version number, must be "1" + BlockFormat string `json:"objectFormat,omitempty"` // identifier of the block format + HMACSecret []byte `json:"secret,omitempty"` // HMAC secret used to generate encryption keys + MasterKey []byte `json:"masterKey,omitempty"` // master encryption key (SIV-mode encryption only) + MaxPackedContentLength int `json:"maxPackedContentLength,omitempty"` // maximum size of object to be considered for storage in a pack +} diff --git a/block/block_manager_test.go b/block/block_manager_test.go index a96698a7d..2ebbbd934 100644 --- a/block/block_manager_test.go +++ b/block/block_manager_test.go @@ -14,9 +14,8 @@ import ( "testing" "time" - "github.com/kopia/kopia/storage" - "github.com/kopia/kopia/internal/storagetesting" + "github.com/kopia/kopia/storage" ) const ( diff --git a/internal/config/local_config.go b/internal/config/local_config.go index 918419ef7..9983a923f 100644 --- a/internal/config/local_config.go +++ b/internal/config/local_config.go @@ -5,6 +5,7 @@ import ( "io" "os" + "github.com/kopia/kopia/block" "github.com/kopia/kopia/storage" ) @@ -16,16 +17,12 @@ type LocalConfig struct { // RepositoryObjectFormat describes the format of objects in a repository. type RepositoryObjectFormat struct { - Version int `json:"version,omitempty"` // version number, must be "1" - BlockFormat string `json:"objectFormat,omitempty"` // identifier of the block format - HMACSecret []byte `json:"secret,omitempty"` // HMAC secret used to generate encryption keys - MasterKey []byte `json:"masterKey,omitempty"` // master encryption key (SIV-mode encryption only) - Splitter string `json:"splitter,omitempty"` // splitter used to break objects into storage blocks - MaxPackedContentLength int `json:"maxPackedContentLength,omitempty"` // maximum size of object to be considered for storage in a pack + block.FormattingOptions - MinBlockSize int `json:"minBlockSize,omitempty"` // minimum block size used with dynamic splitter - AvgBlockSize int `json:"avgBlockSize,omitempty"` // approximate size of storage block (used with dynamic splitter) - MaxBlockSize int `json:"maxBlockSize,omitempty"` // maximum size of storage block + Splitter string `json:"splitter,omitempty"` // splitter used to break objects into storage blocks + MinBlockSize int `json:"minBlockSize,omitempty"` // minimum block size used with dynamic splitter + AvgBlockSize int `json:"avgBlockSize,omitempty"` // approximate size of storage block (used with dynamic splitter) + MaxBlockSize int `json:"maxBlockSize,omitempty"` // maximum size of storage block } // RepositoryConnectionInfo represents JSON-serializable configuration of the repository connection, including master key. diff --git a/object/object_manager_test.go b/object/object_manager_test.go index 9640a687a..418a79d7d 100644 --- a/object/object_manager_test.go +++ b/object/object_manager_test.go @@ -70,7 +70,9 @@ func setupTest(t *testing.T) (map[string][]byte, *Manager) { func setupTestWithData(t *testing.T, data map[string][]byte, opts ManagerOptions) (map[string][]byte, *Manager) { r, err := NewObjectManager(&fakeBlockManager{data: data}, config.RepositoryObjectFormat{ - Version: 1, + FormattingOptions: block.FormattingOptions{ + Version: 1, + }, MaxBlockSize: 200, Splitter: "FIXED", }, opts) diff --git a/repo/initialize.go b/repo/initialize.go index cad603388..6d6cbe324 100644 --- a/repo/initialize.go +++ b/repo/initialize.go @@ -80,15 +80,17 @@ func formatBlockFromOptions(opt *NewRepositoryOptions) formatBlock { func repositoryObjectFormatFromOptions(opt *NewRepositoryOptions) config.RepositoryObjectFormat { f := config.RepositoryObjectFormat{ - Version: 1, - Splitter: applyDefaultString(opt.Splitter, object.DefaultSplitter), - BlockFormat: applyDefaultString(opt.BlockFormat, block.DefaultFormat), - HMACSecret: applyDefaultRandomBytes(opt.ObjectHMACSecret, 32), - MasterKey: applyDefaultRandomBytes(opt.ObjectEncryptionKey, 32), - MaxBlockSize: applyDefaultInt(opt.MaxBlockSize, 20<<20), // 20MiB - MinBlockSize: applyDefaultInt(opt.MinBlockSize, 10<<20), // 10MiB - AvgBlockSize: applyDefaultInt(opt.AvgBlockSize, 16<<20), // 16MiB - MaxPackedContentLength: applyDefaultInt(opt.MaxPackedContentLength, 4<<20), // 3 MB + FormattingOptions: block.FormattingOptions{ + Version: 1, + BlockFormat: applyDefaultString(opt.BlockFormat, block.DefaultFormat), + HMACSecret: applyDefaultRandomBytes(opt.ObjectHMACSecret, 32), + MasterKey: applyDefaultRandomBytes(opt.ObjectEncryptionKey, 32), + MaxPackedContentLength: applyDefaultInt(opt.MaxPackedContentLength, 4<<20), // 3 MB + }, + Splitter: applyDefaultString(opt.Splitter, object.DefaultSplitter), + MaxBlockSize: applyDefaultInt(opt.MaxBlockSize, 20<<20), // 20MiB + MinBlockSize: applyDefaultInt(opt.MinBlockSize, 10<<20), // 10MiB + AvgBlockSize: applyDefaultInt(opt.AvgBlockSize, 16<<20), // 16MiB } if opt.noHMAC { diff --git a/repo/open.go b/repo/open.go index 4b9b489a8..7ddd91126 100644 --- a/repo/open.go +++ b/repo/open.go @@ -110,7 +110,7 @@ func connect(ctx context.Context, st storage.Storage, creds auth.Credentials, op return nil, fmt.Errorf("unsupported block format: %v", repoConfig.BlockFormat) } - formatter, err := sf(repoConfig) + formatter, err := sf(repoConfig.FormattingOptions) if err != nil { return nil, err }