refactor: moved config file management to kopia/kopia/repo

also fixed layering issue and removed dependency from 'object'
on 'config'

cli/command_repository_status.go

@@ -52,7 +52,6 @@ func runStatusCommand(ctx context.Context, rep *repo.Repository) error {
 	fmt.Println()
 	fmt.Printf("Unique ID:           %x\n", rep.UniqueID)
 	fmt.Println()
-	fmt.Printf("Object manager:      v%v\n", rep.Objects.Format.Version)
 	fmt.Printf("Block format:        %v\n", rep.Blocks.Format.BlockFormat)
 	fmt.Printf("Max pack length:     %v\n", units.BytesStringBase2(int64(rep.Blocks.Format.MaxPackSize)))
 	fmt.Printf("Splitter:            %v%v\n", rep.Objects.Format.Splitter, splitterExtraInfo)

internal/config/local_config.go

@@ -1,64 +0,0 @@
-package config
-
-import (
-	"encoding/json"
-	"io"
-	"os"
-
-	"github.com/kopia/kopia/repo/block"
-	"github.com/kopia/kopia/repo/storage"
-)
-
-// LocalConfig is a configuration of Kopia.
-type LocalConfig struct {
-	Storage storage.ConnectionInfo `json:"storage"`
-	Caching block.CachingOptions   `json:"caching"`
-}
-
-// RepositoryObjectFormat describes the format of objects in a repository.
-type RepositoryObjectFormat struct {
-	block.FormattingOptions
-
-	Splitter     string `json:"splitter,omitempty"`     // splitter used to break objects into storage blocks
-	MinBlockSize int    `json:"minBlockSize,omitempty"` // minimum block size used with dynamic splitter
-	AvgBlockSize int    `json:"avgBlockSize,omitempty"` // approximate size of storage block (used with dynamic splitter)
-	MaxBlockSize int    `json:"maxBlockSize,omitempty"` // maximum size of storage block
-}
-
-// RepositoryConnectionInfo represents JSON-serializable configuration of the repository connection, including master key.
-type RepositoryConnectionInfo struct {
-	Key []byte `json:"key,omitempty"`
-}
-
-// Load reads local configuration from the specified reader.
-func (lc *LocalConfig) Load(r io.Reader) error {
-	*lc = LocalConfig{}
-	return json.NewDecoder(r).Decode(lc)
-}
-
-// Save writes the configuration to the specified writer.
-func (lc *LocalConfig) Save(w io.Writer) error {
-	b, err := json.MarshalIndent(lc, "", "  ")
-	if err != nil {
-		return nil
-	}
-	_, err = w.Write(b)
-	return err
-}
-
-// LoadFromFile reads the local configuration from the specified file.
-func LoadFromFile(fileName string) (*LocalConfig, error) {
-	f, err := os.Open(fileName)
-	if err != nil {
-		return nil, err
-	}
-	defer f.Close() //nolint:errcheck
-
-	var lc LocalConfig
-
-	if err := lc.Load(f); err != nil {
-		return nil, err
-	}
-
-	return &lc, nil
-}

repo/connect.go

@@ -10,7 +10,6 @@
 	"os"
 	"path/filepath"
 
-	"github.com/kopia/kopia/internal/config"
 	"github.com/kopia/kopia/internal/ospath"
 	"github.com/kopia/kopia/internal/units"
 	"github.com/kopia/kopia/repo/block"
@@ -34,7 +33,7 @@ func Connect(ctx context.Context, configFile string, st storage.Storage, passwor
 		return err
 	}
 
-	var lc config.LocalConfig
+	var lc LocalConfig
 	lc.Storage = st.ConnectionInfo()
 
 	if err = setupCaching(configFile, &lc, opt.CachingOptions, f.UniqueID); err != nil {
@@ -63,7 +62,7 @@ func Connect(ctx context.Context, configFile string, st storage.Storage, passwor
 	return r.Close(ctx)
 }
 
-func setupCaching(configPath string, lc *config.LocalConfig, opt block.CachingOptions, uniqueID []byte) error {
+func setupCaching(configPath string, lc *LocalConfig, opt block.CachingOptions, uniqueID []byte) error {
 	if opt.MaxCacheSizeBytes == 0 {
 		lc.Caching = block.CachingOptions{}
 		return nil
@@ -94,7 +93,7 @@ func setupCaching(configPath string, lc *config.LocalConfig, opt block.CachingOp
 
 // Disconnect removes the specified configuration file and any local cache directories.
 func Disconnect(configFile string) error {
-	cfg, err := config.LoadFromFile(configFile)
+	cfg, err := loadConfigFromFile(configFile)
 	if err != nil {
 		return err
 	}

repo/format_block.go

@@ -10,7 +10,6 @@
 	"fmt"
 	"io"
 
-	"github.com/kopia/kopia/internal/config"
 	"github.com/kopia/kopia/repo/storage"
 )
 
@@ -30,15 +29,15 @@ type formatBlock struct {
 	UniqueID               []byte `json:"uniqueID"`
 	KeyDerivationAlgorithm string `json:"keyAlgo"`
 
-	Version              string                         `json:"version"`
-	EncryptionAlgorithm  string                         `json:"encryption"`
-	EncryptedFormatBytes []byte                         `json:"encryptedBlockFormat,omitempty"`
-	UnencryptedFormat    *config.RepositoryObjectFormat `json:"blockFormat,omitempty"`
+	Version              string                  `json:"version"`
+	EncryptionAlgorithm  string                  `json:"encryption"`
+	EncryptedFormatBytes []byte                  `json:"encryptedBlockFormat,omitempty"`
+	UnencryptedFormat    *repositoryObjectFormat `json:"blockFormat,omitempty"`
 }
 
 // encryptedRepositoryConfig contains the configuration of repository that's persisted in encrypted format.
 type encryptedRepositoryConfig struct {
-	Format config.RepositoryObjectFormat `json:"format"`
+	Format repositoryObjectFormat `json:"format"`
 }
 
 func parseFormatBlock(b []byte) (*formatBlock, error) {
@@ -66,7 +65,7 @@ func writeFormatBlock(ctx context.Context, st storage.Storage, f *formatBlock) e
 	return nil
 }
 
-func (f *formatBlock) decryptFormatBytes(masterKey []byte) (*config.RepositoryObjectFormat, error) {
+func (f *formatBlock) decryptFormatBytes(masterKey []byte) (*repositoryObjectFormat, error) {
 	switch f.EncryptionAlgorithm {
 	case "NONE": // do nothing
 		return f.UnencryptedFormat, nil
@@ -117,7 +116,7 @@ func initCrypto(masterKey, repositoryID []byte) (cipher.AEAD, []byte, error) {
 	return aead, authData, nil
 }
 
-func encryptFormatBytes(f *formatBlock, format *config.RepositoryObjectFormat, masterKey, repositoryID []byte) error {
+func encryptFormatBytes(f *formatBlock, format *repositoryObjectFormat, masterKey, repositoryID []byte) error {
 	switch f.EncryptionAlgorithm {
 	case "NONE":
 		f.UnencryptedFormat = format

repo/initialize.go

@@ -6,7 +6,6 @@
 	"fmt"
 	"io"
 
-	"github.com/kopia/kopia/internal/config"
 	"github.com/kopia/kopia/repo/block"
 	"github.com/kopia/kopia/repo/object"
 	"github.com/kopia/kopia/repo/storage"
@@ -86,8 +85,8 @@ func formatBlockFromOptions(opt *NewRepositoryOptions) *formatBlock {
 	}
 }
 
-func repositoryObjectFormatFromOptions(opt *NewRepositoryOptions) *config.RepositoryObjectFormat {
-	f := &config.RepositoryObjectFormat{
+func repositoryObjectFormatFromOptions(opt *NewRepositoryOptions) *repositoryObjectFormat {
+	f := &repositoryObjectFormat{
 		FormattingOptions: block.FormattingOptions{
 			Version:     1,
 			BlockFormat: applyDefaultString(opt.BlockFormat, block.DefaultFormat),
@@ -95,10 +94,12 @@ func repositoryObjectFormatFromOptions(opt *NewRepositoryOptions) *config.Reposi
 			MasterKey:   applyDefaultRandomBytes(opt.ObjectEncryptionKey, 32),
 			MaxPackSize: applyDefaultInt(opt.MaxBlockSize, 20<<20), // 20 MB
 		},
-		Splitter:     applyDefaultString(opt.Splitter, object.DefaultSplitter),
-		MaxBlockSize: applyDefaultInt(opt.MaxBlockSize, 20<<20), // 20MiB
-		MinBlockSize: applyDefaultInt(opt.MinBlockSize, 10<<20), // 10MiB
-		AvgBlockSize: applyDefaultInt(opt.AvgBlockSize, 16<<20), // 16MiB
+		Format: object.Format{
+			Splitter:     applyDefaultString(opt.Splitter, object.DefaultSplitter),
+			MaxBlockSize: applyDefaultInt(opt.MaxBlockSize, 20<<20), // 20MiB
+			MinBlockSize: applyDefaultInt(opt.MinBlockSize, 10<<20), // 10MiB
+			AvgBlockSize: applyDefaultInt(opt.AvgBlockSize, 16<<20), // 16MiB
+		},
 	}
 
 	if opt.DisableHMAC {

repo/local_config.go

@@ -0,0 +1,56 @@
+package repo
+
+import (
+	"encoding/json"
+	"io"
+	"os"
+
+	"github.com/kopia/kopia/repo/block"
+	"github.com/kopia/kopia/repo/object"
+	"github.com/kopia/kopia/repo/storage"
+)
+
+// LocalConfig is a configuration of Kopia stored in a configuration file.
+type LocalConfig struct {
+	Storage storage.ConnectionInfo `json:"storage"`
+	Caching block.CachingOptions   `json:"caching"`
+}
+
+// repositoryObjectFormat describes the format of objects in a repository.
+type repositoryObjectFormat struct {
+	block.FormattingOptions
+	object.Format
+}
+
+// Load reads local configuration from the specified reader.
+func (lc *LocalConfig) Load(r io.Reader) error {
+	*lc = LocalConfig{}
+	return json.NewDecoder(r).Decode(lc)
+}
+
+// Save writes the configuration to the specified writer.
+func (lc *LocalConfig) Save(w io.Writer) error {
+	b, err := json.MarshalIndent(lc, "", "  ")
+	if err != nil {
+		return nil
+	}
+	_, err = w.Write(b)
+	return err
+}
+
+// loadConfigFromFile reads the local configuration from the specified file.
+func loadConfigFromFile(fileName string) (*LocalConfig, error) {
+	f, err := os.Open(fileName)
+	if err != nil {
+		return nil, err
+	}
+	defer f.Close() //nolint:errcheck
+
+	var lc LocalConfig
+
+	if err := lc.Load(f); err != nil {
+		return nil, err
+	}
+
+	return &lc, nil
+}

repo/object/object_manager.go

@@ -9,7 +9,6 @@
 	"io"
 	"sync"
 
-	"github.com/kopia/kopia/internal/config"
 	"github.com/kopia/kopia/internal/jsonstream"
 	"github.com/kopia/kopia/repo/block"
 )
@@ -28,9 +27,17 @@ type blockManager interface {
 	WriteBlock(ctx context.Context, data []byte, prefix string) (string, error)
 }
 
+// Format describes the format of objects in a repository.
+type Format struct {
+	Splitter     string `json:"splitter,omitempty"`     // splitter used to break objects into storage blocks
+	MinBlockSize int    `json:"minBlockSize,omitempty"` // minimum block size used with dynamic splitter
+	AvgBlockSize int    `json:"avgBlockSize,omitempty"` // approximate size of storage block (used with dynamic splitter)
+	MaxBlockSize int    `json:"maxBlockSize,omitempty"` // maximum size of storage block
+}
+
 // Manager implements a content-addressable storage on top of blob storage.
 type Manager struct {
-	Format config.RepositoryObjectFormat
+	Format Format
 
 	blockMgr blockManager
 
@@ -172,15 +179,6 @@ func (om *Manager) Flush(ctx context.Context) error {
 func nullTrace(message string, args ...interface{}) {
 }
 
-// validateFormat checks the validity of RepositoryObjectFormat and returns an error if invalid.
-func validateFormat(f *config.RepositoryObjectFormat) error {
-	if f.Version != 1 {
-		return fmt.Errorf("unsupported version: %v", f.Version)
-	}
-
-	return nil
-}
-
 // ManagerOptions specifies object manager options.
 type ManagerOptions struct {
 	WriteBack int
@@ -188,11 +186,7 @@ type ManagerOptions struct {
 }
 
 // NewObjectManager creates an ObjectManager with the specified block manager and format.
-func NewObjectManager(ctx context.Context, bm blockManager, f config.RepositoryObjectFormat, opts ManagerOptions) (*Manager, error) {
-	if err := validateFormat(&f); err != nil {
-		return nil, err
-	}
-
+func NewObjectManager(ctx context.Context, bm blockManager, f Format, opts ManagerOptions) (*Manager, error) {
 	om := &Manager{
 		blockMgr: bm,
 		Format:   f,

repo/object/object_manager_test.go

@@ -15,10 +15,8 @@
 	"sync"
 	"testing"
 
-	"github.com/kopia/kopia/repo/block"
-
-	"github.com/kopia/kopia/internal/config"
 	"github.com/kopia/kopia/internal/jsonstream"
+	"github.com/kopia/kopia/repo/block"
 	"github.com/kopia/kopia/repo/storage"
 )
 
@@ -70,10 +68,7 @@ func setupTest(t *testing.T) (map[string][]byte, *Manager) {
 }
 
 func setupTestWithData(t *testing.T, data map[string][]byte, opts ManagerOptions) (map[string][]byte, *Manager) {
-	r, err := NewObjectManager(context.Background(), &fakeBlockManager{data: data}, config.RepositoryObjectFormat{
-		FormattingOptions: block.FormattingOptions{
-			Version: 1,
-		},
+	r, err := NewObjectManager(context.Background(), &fakeBlockManager{data: data}, Format{
 		MaxBlockSize: 400,
 		Splitter:     "FIXED",
 	}, opts)

repo/object/object_splitter.go

@@ -4,7 +4,6 @@
 	"math"
 	"sort"
 
-	"github.com/kopia/kopia/internal/config"
 	"github.com/silvasur/buzhash"
 )
 
@@ -19,14 +18,14 @@ type objectSplitter interface {
 //    DYNAMIC  - dynamically splits large objects based on rolling hash of contents.
 var SupportedSplitters []string
 
-var splitterFactories = map[string]func(*config.RepositoryObjectFormat) objectSplitter{
-	"NEVER": func(f *config.RepositoryObjectFormat) objectSplitter {
+var splitterFactories = map[string]func(*Format) objectSplitter{
+	"NEVER": func(f *Format) objectSplitter {
 		return newNeverSplitter()
 	},
-	"FIXED": func(f *config.RepositoryObjectFormat) objectSplitter {
+	"FIXED": func(f *Format) objectSplitter {
 		return newFixedSplitter(f.MaxBlockSize)
 	},
-	"DYNAMIC": func(f *config.RepositoryObjectFormat) objectSplitter {
+	"DYNAMIC": func(f *Format) objectSplitter {
 		return newRollingHashSplitter(buzhash.NewBuzHash(32), f.MinBlockSize, f.AvgBlockSize, f.MaxBlockSize)
 	},
 }

repo/open.go

@@ -7,7 +7,6 @@
 	"io/ioutil"
 	"path/filepath"
 
-	"github.com/kopia/kopia/internal/config"
 	"github.com/kopia/kopia/internal/kopialogging"
 	"github.com/kopia/kopia/repo/block"
 	"github.com/kopia/kopia/repo/manifest"
@@ -45,7 +44,7 @@ func Open(ctx context.Context, configFile string, password string, options *Opti
 	}
 
 	log.Debugf("loading config from file: %v", configFile)
-	lc, err := config.LoadFromFile(configFile)
+	lc, err := loadConfigFromFile(configFile)
 	if err != nil {
 		return nil, err
 	}
@@ -61,7 +60,7 @@ func Open(ctx context.Context, configFile string, password string, options *Opti
 		st = logging.NewWrapper(st, logging.Prefix("[STORAGE] "), logging.Output(options.TraceStorage))
 	}
 
-	r, err := connect(ctx, st, lc, password, options, lc.Caching)
+	r, err := OpenWithConfig(ctx, st, lc, password, options, lc.Caching)
 	if err != nil {
 		st.Close(ctx) //nolint:errcheck
 		return nil, err
@@ -72,7 +71,8 @@ func Open(ctx context.Context, configFile string, password string, options *Opti
 	return r, nil
 }
 
-func connect(ctx context.Context, st storage.Storage, lc *config.LocalConfig, password string, options *Options, caching block.CachingOptions) (*Repository, error) {
+// OpenWithConfig opens the repository with a given configuration, avoiding the need for a config file.
+func OpenWithConfig(ctx context.Context, st storage.Storage, lc *LocalConfig, password string, options *Options, caching block.CachingOptions) (*Repository, error) {
 	log.Debugf("reading encrypted format block")
 	// Read cache block, potentially from cache.
 	f, err := readAndCacheFormatBlock(ctx, st, caching.CacheDirectory)
@@ -104,7 +104,7 @@ func connect(ctx context.Context, st storage.Storage, lc *config.LocalConfig, pa
 	}
 
 	log.Debugf("initializing object manager")
-	om, err := object.NewObjectManager(ctx, bm, *repoConfig, options.ObjectManagerOptions)
+	om, err := object.NewObjectManager(ctx, bm, repoConfig.Format, options.ObjectManagerOptions)
 	if err != nil {
 		return nil, fmt.Errorf("unable to open object manager: %v", err)
 	}
@@ -132,7 +132,7 @@ func SetCachingConfig(ctx context.Context, configFile string, opt block.CachingO
 		return err
 	}
 
-	lc, err := config.LoadFromFile(configFile)
+	lc, err := loadConfigFromFile(configFile)
 	if err != nil {
 		return err
 	}