diff --git a/cli/command_repository_status.go b/cli/command_repository_status.go index 2a568765b..fece06b61 100644 --- a/cli/command_repository_status.go +++ b/cli/command_repository_status.go @@ -52,7 +52,6 @@ func runStatusCommand(ctx context.Context, rep *repo.Repository) error { fmt.Println() fmt.Printf("Unique ID: %x\n", rep.UniqueID) fmt.Println() - fmt.Printf("Object manager: v%v\n", rep.Objects.Format.Version) fmt.Printf("Block format: %v\n", rep.Blocks.Format.BlockFormat) fmt.Printf("Max pack length: %v\n", units.BytesStringBase2(int64(rep.Blocks.Format.MaxPackSize))) fmt.Printf("Splitter: %v%v\n", rep.Objects.Format.Splitter, splitterExtraInfo) diff --git a/internal/config/local_config.go b/internal/config/local_config.go deleted file mode 100644 index c949a4996..000000000 --- a/internal/config/local_config.go +++ /dev/null @@ -1,64 +0,0 @@ -package config - -import ( - "encoding/json" - "io" - "os" - - "github.com/kopia/kopia/repo/block" - "github.com/kopia/kopia/repo/storage" -) - -// LocalConfig is a configuration of Kopia. -type LocalConfig struct { - Storage storage.ConnectionInfo `json:"storage"` - Caching block.CachingOptions `json:"caching"` -} - -// RepositoryObjectFormat describes the format of objects in a repository. -type RepositoryObjectFormat struct { - block.FormattingOptions - - Splitter string `json:"splitter,omitempty"` // splitter used to break objects into storage blocks - MinBlockSize int `json:"minBlockSize,omitempty"` // minimum block size used with dynamic splitter - AvgBlockSize int `json:"avgBlockSize,omitempty"` // approximate size of storage block (used with dynamic splitter) - MaxBlockSize int `json:"maxBlockSize,omitempty"` // maximum size of storage block -} - -// RepositoryConnectionInfo represents JSON-serializable configuration of the repository connection, including master key. -type RepositoryConnectionInfo struct { - Key []byte `json:"key,omitempty"` -} - -// Load reads local configuration from the specified reader. -func (lc *LocalConfig) Load(r io.Reader) error { - *lc = LocalConfig{} - return json.NewDecoder(r).Decode(lc) -} - -// Save writes the configuration to the specified writer. -func (lc *LocalConfig) Save(w io.Writer) error { - b, err := json.MarshalIndent(lc, "", " ") - if err != nil { - return nil - } - _, err = w.Write(b) - return err -} - -// LoadFromFile reads the local configuration from the specified file. -func LoadFromFile(fileName string) (*LocalConfig, error) { - f, err := os.Open(fileName) - if err != nil { - return nil, err - } - defer f.Close() //nolint:errcheck - - var lc LocalConfig - - if err := lc.Load(f); err != nil { - return nil, err - } - - return &lc, nil -} diff --git a/repo/connect.go b/repo/connect.go index e2990104a..da1e5ba06 100644 --- a/repo/connect.go +++ b/repo/connect.go @@ -10,7 +10,6 @@ "os" "path/filepath" - "github.com/kopia/kopia/internal/config" "github.com/kopia/kopia/internal/ospath" "github.com/kopia/kopia/internal/units" "github.com/kopia/kopia/repo/block" @@ -34,7 +33,7 @@ func Connect(ctx context.Context, configFile string, st storage.Storage, passwor return err } - var lc config.LocalConfig + var lc LocalConfig lc.Storage = st.ConnectionInfo() if err = setupCaching(configFile, &lc, opt.CachingOptions, f.UniqueID); err != nil { @@ -63,7 +62,7 @@ func Connect(ctx context.Context, configFile string, st storage.Storage, passwor return r.Close(ctx) } -func setupCaching(configPath string, lc *config.LocalConfig, opt block.CachingOptions, uniqueID []byte) error { +func setupCaching(configPath string, lc *LocalConfig, opt block.CachingOptions, uniqueID []byte) error { if opt.MaxCacheSizeBytes == 0 { lc.Caching = block.CachingOptions{} return nil @@ -94,7 +93,7 @@ func setupCaching(configPath string, lc *config.LocalConfig, opt block.CachingOp // Disconnect removes the specified configuration file and any local cache directories. func Disconnect(configFile string) error { - cfg, err := config.LoadFromFile(configFile) + cfg, err := loadConfigFromFile(configFile) if err != nil { return err } diff --git a/repo/format_block.go b/repo/format_block.go index 2af4e221c..2aff4b3c3 100644 --- a/repo/format_block.go +++ b/repo/format_block.go @@ -10,7 +10,6 @@ "fmt" "io" - "github.com/kopia/kopia/internal/config" "github.com/kopia/kopia/repo/storage" ) @@ -30,15 +29,15 @@ type formatBlock struct { UniqueID []byte `json:"uniqueID"` KeyDerivationAlgorithm string `json:"keyAlgo"` - Version string `json:"version"` - EncryptionAlgorithm string `json:"encryption"` - EncryptedFormatBytes []byte `json:"encryptedBlockFormat,omitempty"` - UnencryptedFormat *config.RepositoryObjectFormat `json:"blockFormat,omitempty"` + Version string `json:"version"` + EncryptionAlgorithm string `json:"encryption"` + EncryptedFormatBytes []byte `json:"encryptedBlockFormat,omitempty"` + UnencryptedFormat *repositoryObjectFormat `json:"blockFormat,omitempty"` } // encryptedRepositoryConfig contains the configuration of repository that's persisted in encrypted format. type encryptedRepositoryConfig struct { - Format config.RepositoryObjectFormat `json:"format"` + Format repositoryObjectFormat `json:"format"` } func parseFormatBlock(b []byte) (*formatBlock, error) { @@ -66,7 +65,7 @@ func writeFormatBlock(ctx context.Context, st storage.Storage, f *formatBlock) e return nil } -func (f *formatBlock) decryptFormatBytes(masterKey []byte) (*config.RepositoryObjectFormat, error) { +func (f *formatBlock) decryptFormatBytes(masterKey []byte) (*repositoryObjectFormat, error) { switch f.EncryptionAlgorithm { case "NONE": // do nothing return f.UnencryptedFormat, nil @@ -117,7 +116,7 @@ func initCrypto(masterKey, repositoryID []byte) (cipher.AEAD, []byte, error) { return aead, authData, nil } -func encryptFormatBytes(f *formatBlock, format *config.RepositoryObjectFormat, masterKey, repositoryID []byte) error { +func encryptFormatBytes(f *formatBlock, format *repositoryObjectFormat, masterKey, repositoryID []byte) error { switch f.EncryptionAlgorithm { case "NONE": f.UnencryptedFormat = format diff --git a/repo/initialize.go b/repo/initialize.go index 93b9ef0c7..877173132 100644 --- a/repo/initialize.go +++ b/repo/initialize.go @@ -6,7 +6,6 @@ "fmt" "io" - "github.com/kopia/kopia/internal/config" "github.com/kopia/kopia/repo/block" "github.com/kopia/kopia/repo/object" "github.com/kopia/kopia/repo/storage" @@ -86,8 +85,8 @@ func formatBlockFromOptions(opt *NewRepositoryOptions) *formatBlock { } } -func repositoryObjectFormatFromOptions(opt *NewRepositoryOptions) *config.RepositoryObjectFormat { - f := &config.RepositoryObjectFormat{ +func repositoryObjectFormatFromOptions(opt *NewRepositoryOptions) *repositoryObjectFormat { + f := &repositoryObjectFormat{ FormattingOptions: block.FormattingOptions{ Version: 1, BlockFormat: applyDefaultString(opt.BlockFormat, block.DefaultFormat), @@ -95,10 +94,12 @@ func repositoryObjectFormatFromOptions(opt *NewRepositoryOptions) *config.Reposi MasterKey: applyDefaultRandomBytes(opt.ObjectEncryptionKey, 32), MaxPackSize: applyDefaultInt(opt.MaxBlockSize, 20<<20), // 20 MB }, - Splitter: applyDefaultString(opt.Splitter, object.DefaultSplitter), - MaxBlockSize: applyDefaultInt(opt.MaxBlockSize, 20<<20), // 20MiB - MinBlockSize: applyDefaultInt(opt.MinBlockSize, 10<<20), // 10MiB - AvgBlockSize: applyDefaultInt(opt.AvgBlockSize, 16<<20), // 16MiB + Format: object.Format{ + Splitter: applyDefaultString(opt.Splitter, object.DefaultSplitter), + MaxBlockSize: applyDefaultInt(opt.MaxBlockSize, 20<<20), // 20MiB + MinBlockSize: applyDefaultInt(opt.MinBlockSize, 10<<20), // 10MiB + AvgBlockSize: applyDefaultInt(opt.AvgBlockSize, 16<<20), // 16MiB + }, } if opt.DisableHMAC { diff --git a/repo/local_config.go b/repo/local_config.go new file mode 100644 index 000000000..9056a3bbc --- /dev/null +++ b/repo/local_config.go @@ -0,0 +1,56 @@ +package repo + +import ( + "encoding/json" + "io" + "os" + + "github.com/kopia/kopia/repo/block" + "github.com/kopia/kopia/repo/object" + "github.com/kopia/kopia/repo/storage" +) + +// LocalConfig is a configuration of Kopia stored in a configuration file. +type LocalConfig struct { + Storage storage.ConnectionInfo `json:"storage"` + Caching block.CachingOptions `json:"caching"` +} + +// repositoryObjectFormat describes the format of objects in a repository. +type repositoryObjectFormat struct { + block.FormattingOptions + object.Format +} + +// Load reads local configuration from the specified reader. +func (lc *LocalConfig) Load(r io.Reader) error { + *lc = LocalConfig{} + return json.NewDecoder(r).Decode(lc) +} + +// Save writes the configuration to the specified writer. +func (lc *LocalConfig) Save(w io.Writer) error { + b, err := json.MarshalIndent(lc, "", " ") + if err != nil { + return nil + } + _, err = w.Write(b) + return err +} + +// loadConfigFromFile reads the local configuration from the specified file. +func loadConfigFromFile(fileName string) (*LocalConfig, error) { + f, err := os.Open(fileName) + if err != nil { + return nil, err + } + defer f.Close() //nolint:errcheck + + var lc LocalConfig + + if err := lc.Load(f); err != nil { + return nil, err + } + + return &lc, nil +} diff --git a/repo/object/object_manager.go b/repo/object/object_manager.go index d1c39229d..579a2ad27 100644 --- a/repo/object/object_manager.go +++ b/repo/object/object_manager.go @@ -9,7 +9,6 @@ "io" "sync" - "github.com/kopia/kopia/internal/config" "github.com/kopia/kopia/internal/jsonstream" "github.com/kopia/kopia/repo/block" ) @@ -28,9 +27,17 @@ type blockManager interface { WriteBlock(ctx context.Context, data []byte, prefix string) (string, error) } +// Format describes the format of objects in a repository. +type Format struct { + Splitter string `json:"splitter,omitempty"` // splitter used to break objects into storage blocks + MinBlockSize int `json:"minBlockSize,omitempty"` // minimum block size used with dynamic splitter + AvgBlockSize int `json:"avgBlockSize,omitempty"` // approximate size of storage block (used with dynamic splitter) + MaxBlockSize int `json:"maxBlockSize,omitempty"` // maximum size of storage block +} + // Manager implements a content-addressable storage on top of blob storage. type Manager struct { - Format config.RepositoryObjectFormat + Format Format blockMgr blockManager @@ -172,15 +179,6 @@ func (om *Manager) Flush(ctx context.Context) error { func nullTrace(message string, args ...interface{}) { } -// validateFormat checks the validity of RepositoryObjectFormat and returns an error if invalid. -func validateFormat(f *config.RepositoryObjectFormat) error { - if f.Version != 1 { - return fmt.Errorf("unsupported version: %v", f.Version) - } - - return nil -} - // ManagerOptions specifies object manager options. type ManagerOptions struct { WriteBack int @@ -188,11 +186,7 @@ type ManagerOptions struct { } // NewObjectManager creates an ObjectManager with the specified block manager and format. -func NewObjectManager(ctx context.Context, bm blockManager, f config.RepositoryObjectFormat, opts ManagerOptions) (*Manager, error) { - if err := validateFormat(&f); err != nil { - return nil, err - } - +func NewObjectManager(ctx context.Context, bm blockManager, f Format, opts ManagerOptions) (*Manager, error) { om := &Manager{ blockMgr: bm, Format: f, diff --git a/repo/object/object_manager_test.go b/repo/object/object_manager_test.go index 3d9ce6c5a..4b6d4a19b 100644 --- a/repo/object/object_manager_test.go +++ b/repo/object/object_manager_test.go @@ -15,10 +15,8 @@ "sync" "testing" - "github.com/kopia/kopia/repo/block" - - "github.com/kopia/kopia/internal/config" "github.com/kopia/kopia/internal/jsonstream" + "github.com/kopia/kopia/repo/block" "github.com/kopia/kopia/repo/storage" ) @@ -70,10 +68,7 @@ func setupTest(t *testing.T) (map[string][]byte, *Manager) { } func setupTestWithData(t *testing.T, data map[string][]byte, opts ManagerOptions) (map[string][]byte, *Manager) { - r, err := NewObjectManager(context.Background(), &fakeBlockManager{data: data}, config.RepositoryObjectFormat{ - FormattingOptions: block.FormattingOptions{ - Version: 1, - }, + r, err := NewObjectManager(context.Background(), &fakeBlockManager{data: data}, Format{ MaxBlockSize: 400, Splitter: "FIXED", }, opts) diff --git a/repo/object/object_splitter.go b/repo/object/object_splitter.go index 24f6dad33..085274687 100644 --- a/repo/object/object_splitter.go +++ b/repo/object/object_splitter.go @@ -4,7 +4,6 @@ "math" "sort" - "github.com/kopia/kopia/internal/config" "github.com/silvasur/buzhash" ) @@ -19,14 +18,14 @@ type objectSplitter interface { // DYNAMIC - dynamically splits large objects based on rolling hash of contents. var SupportedSplitters []string -var splitterFactories = map[string]func(*config.RepositoryObjectFormat) objectSplitter{ - "NEVER": func(f *config.RepositoryObjectFormat) objectSplitter { +var splitterFactories = map[string]func(*Format) objectSplitter{ + "NEVER": func(f *Format) objectSplitter { return newNeverSplitter() }, - "FIXED": func(f *config.RepositoryObjectFormat) objectSplitter { + "FIXED": func(f *Format) objectSplitter { return newFixedSplitter(f.MaxBlockSize) }, - "DYNAMIC": func(f *config.RepositoryObjectFormat) objectSplitter { + "DYNAMIC": func(f *Format) objectSplitter { return newRollingHashSplitter(buzhash.NewBuzHash(32), f.MinBlockSize, f.AvgBlockSize, f.MaxBlockSize) }, } diff --git a/repo/open.go b/repo/open.go index 043013535..91da63870 100644 --- a/repo/open.go +++ b/repo/open.go @@ -7,7 +7,6 @@ "io/ioutil" "path/filepath" - "github.com/kopia/kopia/internal/config" "github.com/kopia/kopia/internal/kopialogging" "github.com/kopia/kopia/repo/block" "github.com/kopia/kopia/repo/manifest" @@ -45,7 +44,7 @@ func Open(ctx context.Context, configFile string, password string, options *Opti } log.Debugf("loading config from file: %v", configFile) - lc, err := config.LoadFromFile(configFile) + lc, err := loadConfigFromFile(configFile) if err != nil { return nil, err } @@ -61,7 +60,7 @@ func Open(ctx context.Context, configFile string, password string, options *Opti st = logging.NewWrapper(st, logging.Prefix("[STORAGE] "), logging.Output(options.TraceStorage)) } - r, err := connect(ctx, st, lc, password, options, lc.Caching) + r, err := OpenWithConfig(ctx, st, lc, password, options, lc.Caching) if err != nil { st.Close(ctx) //nolint:errcheck return nil, err @@ -72,7 +71,8 @@ func Open(ctx context.Context, configFile string, password string, options *Opti return r, nil } -func connect(ctx context.Context, st storage.Storage, lc *config.LocalConfig, password string, options *Options, caching block.CachingOptions) (*Repository, error) { +// OpenWithConfig opens the repository with a given configuration, avoiding the need for a config file. +func OpenWithConfig(ctx context.Context, st storage.Storage, lc *LocalConfig, password string, options *Options, caching block.CachingOptions) (*Repository, error) { log.Debugf("reading encrypted format block") // Read cache block, potentially from cache. f, err := readAndCacheFormatBlock(ctx, st, caching.CacheDirectory) @@ -104,7 +104,7 @@ func connect(ctx context.Context, st storage.Storage, lc *config.LocalConfig, pa } log.Debugf("initializing object manager") - om, err := object.NewObjectManager(ctx, bm, *repoConfig, options.ObjectManagerOptions) + om, err := object.NewObjectManager(ctx, bm, repoConfig.Format, options.ObjectManagerOptions) if err != nil { return nil, fmt.Errorf("unable to open object manager: %v", err) } @@ -132,7 +132,7 @@ func SetCachingConfig(ctx context.Context, configFile string, opt block.CachingO return err } - lc, err := config.LoadFromFile(configFile) + lc, err := loadConfigFromFile(configFile) if err != nil { return err }