added kopia.Connection as top-level API.

simplified vault API

cmd/kopia/command_backup.go

@@ -50,12 +50,12 @@ func runBackupCommand(context *kingpin.ParseContext) error {
 		repoOptions = append(repoOptions, repo.WriteBack(*backupWriteBack))
 	}
 
-	vlt, r := mustOpenVaultAndRepository(repoOptions...)
-	defer r.Close()
+	conn := mustOpenConnection(repoOptions...)
+	defer conn.Close()
 
 	var options []repofs.UploadOption
 
-	bgen, err := backup.NewGenerator(r, options...)
+	bgen, err := backup.NewGenerator(conn.Repository, options...)
 	if err != nil {
 		return err
 	}
@@ -79,7 +79,7 @@ func runBackupCommand(context *kingpin.ParseContext) error {
 			return fmt.Errorf("description too long")
 		}
 
-		previous, err := vlt.List("B" + manifest.SourceID() + ".")
+		previous, err := conn.Vault.List("B" + manifest.SourceID() + ".")
 		if err != nil {
 			return fmt.Errorf("error listing previous backups")
 		}
@@ -87,7 +87,7 @@ func runBackupCommand(context *kingpin.ParseContext) error {
 		var oldManifest *backup.Manifest
 
 		if len(previous) > 0 {
-			oldManifest, err = loadBackupManifest(vlt, previous[0])
+			oldManifest, err = loadBackupManifest(conn.Vault, previous[0])
 		}
 
 		localEntry := mustGetLocalFSEntry(manifest.Source)
@@ -99,7 +99,7 @@ func runBackupCommand(context *kingpin.ParseContext) error {
 			return err
 		}
 
-		handleID, err := vlt.SaveObjectID(manifest.RootObjectID)
+		handleID, err := conn.Vault.SaveObjectID(manifest.RootObjectID)
 		if err != nil {
 			return err
 		}
@@ -109,7 +109,7 @@ func runBackupCommand(context *kingpin.ParseContext) error {
 		fileID := fmt.Sprintf("B%v.%08x.%x", manifest.SourceID(), math.MaxInt64-manifest.StartTime.UnixNano(), uniqueID)
 		manifest.Handle = handleID
 
-		err = saveBackupManifest(vlt, fileID, &manifest)
+		err = saveBackupManifest(conn.Vault, fileID, &manifest)
 		if err != nil {
 			return fmt.Errorf("cannot save manifest: %v", err)
 		}

cmd/kopia/command_backups.go

@@ -57,8 +57,8 @@ func findBackups(vlt *vault.Vault, path string) ([]string, string, error) {
 }
 
 func runBackupsCommand(context *kingpin.ParseContext) error {
-	vlt, r := mustOpenVaultAndRepository()
-	defer r.Close()
+	conn := mustOpenConnection()
+	defer conn.Close()
 
 	var previous []string
 	var relPath string
@@ -70,12 +70,12 @@ func runBackupsCommand(context *kingpin.ParseContext) error {
 			return fmt.Errorf("invalid directory: '%s': %s", *backupsPath, err)
 		}
 
-		previous, relPath, err = findBackups(vlt, filepath.Clean(path))
+		previous, relPath, err = findBackups(conn.Vault, filepath.Clean(path))
 		if relPath != "" {
 			relPath = "/" + relPath
 		}
 	} else {
-		previous, err = vlt.List("B")
+		previous, err = conn.Vault.List("B")
 	}
 
 	if err != nil {
@@ -87,7 +87,7 @@ func runBackupsCommand(context *kingpin.ParseContext) error {
 	var lastSource string
 	var count int
 
-	for _, m := range loadBackupManifests(vlt, previous) {
+	for _, m := range loadBackupManifests(conn.Vault, previous) {
 		if m.HostName != lastHost || m.UserName != lastUser || m.Source != lastSource {
 			fmt.Printf("%v@%v:%v\n", m.UserName, m.HostName, m.Source)
 			lastSource = m.Source

cmd/kopia/command_cat.go

@@ -13,17 +13,14 @@ var (
 )
 
 func runCatCommand(context *kingpin.ParseContext) error {
-	vlt := mustOpenVault()
-	mgr, err := vlt.OpenRepository()
-	if err != nil {
-		return err
-	}
+	conn := mustOpenConnection()
+	defer conn.Close()
 
-	oid, err := parseObjectID(*catCommandPath, vlt)
+	oid, err := parseObjectID(*catCommandPath, conn.Vault, conn.Repository)
 	if err != nil {
 		return err
 	}
-	r, err := mgr.Open(oid)
+	r, err := conn.Repository.Open(oid)
 	if err != nil {
 		return err
 	}

cmd/kopia/command_create.go

@@ -83,17 +83,6 @@ func runCreateCommand(context *kingpin.ParseContext) error {
 		return fmt.Errorf("unable to get vault storage: %v", err)
 	}
 
-	var repositoryStorage storage.Storage
-
-	if *createCommandRepository == "colocated" {
-		repositoryStorage = vaultStorage
-	} else {
-		repositoryStorage, err = openStorageAndEnsureEmpty(*createCommandRepository)
-		if err != nil {
-			return fmt.Errorf("unable to get repository storage: %v", err)
-		}
-	}
-
 	repoFormat, err := repositoryFormat()
 	if err != nil {
 		return fmt.Errorf("unable to initialize repository format: %v", err)
@@ -114,16 +103,26 @@ func runCreateCommand(context *kingpin.ParseContext) error {
 		return fmt.Errorf("unable to get credentials: %v", err)
 	}
 
-	// Make repository to make sure the format is supported.
-	_, err = repo.New(repositoryStorage, repoFormat)
-	if err != nil {
-		return fmt.Errorf("unable to initialize repository: %v", err)
+	if err := repoFormat.Validate(); err != nil {
+		return fmt.Errorf("invalid format")
 	}
 
 	fmt.Printf(
 		"Initializing vault with encryption '%v'.\n",
 		vf.Encryption)
-	vlt, err := vault.Create(vaultStorage, vf, creds, repositoryStorage, repoFormat)
+
+	var vlt *vault.Vault
+
+	if *createCommandRepository == "colocated" {
+		vlt, err = vault.CreateColocated(vaultStorage, vf, creds, repoFormat)
+	} else {
+		repositoryStorage, err := openStorageAndEnsureEmpty(*createCommandRepository)
+		if err != nil {
+			return fmt.Errorf("unable to get repository storage: %v", err)
+		}
+		vlt, err = vault.Create(vaultStorage, vf, creds, repositoryStorage, repoFormat)
+	}
+
 	if err != nil {
 		return fmt.Errorf("cannot create vault: %v", err)
 	}

cmd/kopia/command_ls.go

@@ -20,11 +20,10 @@ var (
 )
 
 func runLSCommand(context *kingpin.ParseContext) error {
-	vlt, r := mustOpenVaultAndRepository()
-	defer vlt.Close()
-	defer r.Close()
+	conn := mustOpenConnection()
+	defer conn.Close()
 
-	oid, err := parseObjectID(*lsCommandPath, vlt)
+	oid, err := parseObjectID(*lsCommandPath, conn.Vault, conn.Repository)
 	if err != nil {
 		return err
 	}
@@ -37,7 +36,7 @@ func runLSCommand(context *kingpin.ParseContext) error {
 		}
 	}
 
-	d := repofs.Directory(r, oid)
+	d := repofs.Directory(conn.Repository, oid)
 
 	entries, err := d.Readdir()
 	if err != nil {

cmd/kopia/command_mount.go

@@ -27,7 +27,7 @@ func (r *root) Root() (fusefs.Node, error) {
 }
 
 func runMountCommand(context *kingpin.ParseContext) error {
-	vlt, r := mustOpenVaultAndRepository()
+	conn := mustOpenConnection()
 
 	fuseConnection, err := fuse.Mount(
 		*mountPoint,
@@ -37,12 +37,12 @@ func runMountCommand(context *kingpin.ParseContext) error {
 		fuse.VolumeName("Kopia"),
 	)
 
-	oid, err := parseObjectID(*mountObjectID, vlt)
+	oid, err := parseObjectID(*mountObjectID, conn.Vault, conn.Repository)
 	if err != nil {
 		return err
 	}
 
-	rootNode := kopiafuse.NewDirectoryNode(repofs.Directory(r, oid))
+	rootNode := kopiafuse.NewDirectoryNode(repofs.Directory(conn.Repository, oid))
 
 	fusefs.Serve(fuseConnection, &root{rootNode})
 

cmd/kopia/command_restore.go

@@ -1,61 +0,0 @@
-package main
-
-import kingpin "gopkg.in/alecthomas/kingpin.v2"
-
-var (
-	restoreCommand = app.Command("restore", "Restore contents of a file or directory.")
-
-	restoreCommandLong = restoreCommand.Flag("recursive", "Copy contents of directory recursively").Short('r').Bool()
-	restoreCommandPath = restoreCommand.Arg("paths", "Paths to restore").Required().Strings()
-)
-
-func runRestoreCommand(context *kingpin.ParseContext) error {
-	vlt := mustOpenVault()
-	mgr, err := vlt.OpenRepository()
-	if err != nil {
-		return err
-	}
-
-	_ = mgr
-
-	// oid, err := ParseObjectID(*restoreCommandPath, vlt)
-	// if err != nil {
-	// 	return err
-	// }
-	// r, err := mgr.Open(oid)
-	// if err != nil {
-	// 	return err
-	// }
-
-	// var prefix string
-	// if !*restoreCommandLong {
-	// 	prefix = *restoreCommandPath
-	// 	if !strings.HasSuffix(prefix, "/") {
-	// 		prefix += "/"
-	// 	}
-	// }
-
-	// dir, err := fs.ReadDirectory(r, "")
-	// if err != nil {
-	// 	return fmt.Errorf("unable to read directory contents")
-	// }
-
-	// if *restoreCommandLong {
-	// 	listDirectory(dir)
-	// } else {
-	// 	for _, e := range dir {
-	// 		var suffix string
-	// 		if e.FileMode.IsDir() {
-	// 			suffix = "/"
-	// 		}
-
-	// 		fmt.Println(prefix + e.Name + suffix)
-	// 	}
-	// }
-
-	return nil
-}
-
-func init() {
-	restoreCommand.Action(runRestoreCommand)
-}

cmd/kopia/command_show.go

@@ -20,17 +20,16 @@ var (
 )
 
 func runShowCommand(context *kingpin.ParseContext) error {
-	vlt, r := mustOpenVaultAndRepository()
-	defer vlt.Close()
-	defer r.Close()
+	conn := mustOpenConnection()
+	defer conn.Close()
 
 	for _, oidString := range *showObjectIDs {
-		oid, err := parseObjectID(oidString, vlt)
+		oid, err := parseObjectID(oidString, conn.Vault, conn.Repository)
 		if err != nil {
 			return err
 		}
 
-		if err := showObject(r, oid); err != nil {
+		if err := showObject(conn.Repository, oid); err != nil {
 			return err
 		}
 	}

cmd/kopia/command_status.go

@@ -17,18 +17,19 @@ func init() {
 }
 
 func runRepositoryInfoCommand(context *kingpin.ParseContext) error {
-	vlt, r := mustOpenVaultAndRepository()
+	conn := mustOpenConnection()
 
-	vf := vlt.Format()
+	vf := conn.Vault.Format
 	fmt.Println("Vault:")
 	fmt.Println("  Version:         ", vf.Version)
 	fmt.Println("  Unique ID:       ", hex.EncodeToString(vf.UniqueID))
 	fmt.Println("  Encryption:      ", vf.Encryption)
 	fmt.Println("  Checksum:        ", vf.Checksum)
-	fmt.Println("  Storage:         ", vlt.Storage())
+	fmt.Println("  Storage:         ", conn.Vault.Storage)
 	fmt.Println()
 
-	f := vlt.RepositoryFormat()
+	rc := conn.Vault.RepoConfig
+	f := rc.Format
 
 	fmt.Println("Repository:")
 	fmt.Println("  Version:         ", f.Version)
@@ -40,7 +41,7 @@ func runRepositoryInfoCommand(context *kingpin.ParseContext) error {
 	fmt.Println("  ID Format:       ", f.ObjectFormat)
 	fmt.Println("  Blob Size:       ", f.MaxBlockSize/1024, "KB")
 	fmt.Println("  Inline Blob Size:", f.MaxInlineContentLength/1024, "KB")
-	fmt.Println("  Storage:         ", r.Storage)
+	fmt.Println("  Storage:         ", conn.Repository.Storage)
 
 	return nil
 }

cmd/kopia/command_vault_list.go

@@ -16,9 +16,9 @@ func init() {
 }
 
 func listVaultContents(context *kingpin.ParseContext) error {
-	vlt := mustOpenVault()
+	conn := mustOpenConnection()
 
-	entries, err := vlt.List(*vaultListPrefix)
+	entries, err := conn.Vault.List(*vaultListPrefix)
 	if err != nil {
 		return err
 	}

cmd/kopia/command_vault_rm.go

@@ -12,10 +12,10 @@ func init() {
 }
 
 func removeVaultItem(context *kingpin.ParseContext) error {
-	vlt := mustOpenVault()
+	conn := mustOpenConnection()
 
 	for _, v := range *vaultRemoveItems {
-		if err := vlt.Remove(v); err != nil {
+		if err := conn.Vault.Remove(v); err != nil {
 			return err
 		}
 	}

cmd/kopia/command_vault_show.go

@@ -20,12 +20,9 @@ func init() {
 }
 
 func showVaultObject(context *kingpin.ParseContext) error {
-	v, err := openVault()
-	if err != nil {
-		return err
-	}
+	conn := mustOpenConnection()
 
-	b, err := v.Get(*vaultShowID)
+	b, err := conn.Vault.Get(*vaultShowID)
 	if err != nil {
 		return err
 	}

cmd/kopia/config.go

@@ -11,15 +11,14 @@ import (
 	"runtime"
 	"strings"
 
+	"github.com/bgentry/speakeasy"
+	"github.com/kopia/kopia"
+	"github.com/kopia/kopia/fs"
 	"github.com/kopia/kopia/fs/localfs"
 	"github.com/kopia/kopia/fs/loggingfs"
-
-	"github.com/kopia/kopia/fs"
-	"github.com/kopia/kopia/storage/logging"
-
-	"github.com/bgentry/speakeasy"
+	"github.com/kopia/kopia/internal/config"
 	"github.com/kopia/kopia/repo"
-	"github.com/kopia/kopia/storage"
+	"github.com/kopia/kopia/storage/logging"
 	"github.com/kopia/kopia/vault"
 )
 
@@ -35,6 +34,16 @@ var (
 	keyFile         = app.Flag("keyfile", "Read vault master key from file.").PlaceHolder("FILENAME").Envar("KOPIA_KEY_FILE").ExistingFile()
 )
 
+func mustLoadLocalConfig() *config.LocalConfig {
+	lc, err := loadLocalConfig()
+	failOnError(err)
+	return lc
+}
+
+func loadLocalConfig() (*config.LocalConfig, error) {
+	return config.LoadFromFile(vaultConfigFileName())
+}
+
 func failOnError(err error) {
 	if err != nil {
 		fmt.Fprintf(os.Stderr, "ERROR: %v\n", err)
@@ -42,24 +51,29 @@ func failOnError(err error) {
 	}
 }
 
-func mustOpenVault() *vault.Vault {
-	s, err := openVault()
+func openConnection(options ...repo.RepositoryOption) (*kopia.Connection, error) {
+	return kopia.Open(vaultConfigFileName(), connectionOptionsFromFlags(options...))
+}
+
+func connectionOptionsFromFlags(options ...repo.RepositoryOption) *kopia.ConnectionOptions {
+	opts := &kopia.ConnectionOptions{
+		CredentialsCallback: func() (vault.Credentials, error) { return getVaultCredentials(false) },
+		RepositoryOptions:   options,
+	}
+
+	if *traceStorage {
+		opts.TraceStorage = log.Printf
+	}
+
+	return opts
+}
+
+func mustOpenConnection(repoOptions ...repo.RepositoryOption) *kopia.Connection {
+	s, err := openConnection(repoOptions...)
 	failOnError(err)
 	return s
 }
 
-func mustOpenRepository(extraOptions ...repo.RepositoryOption) *repo.Repository {
-	_, r := mustOpenVaultAndRepository(extraOptions...)
-	return r
-}
-
-func mustOpenVaultAndRepository(extraOptions ...repo.RepositoryOption) (*vault.Vault, *repo.Repository) {
-	v := mustOpenVault()
-	r, err := v.OpenRepository(repositoryOptionsFromFlags(extraOptions)...)
-	failOnError(err)
-	return v, r
-}
-
 func repositoryOptionsFromFlags(extraOptions []repo.RepositoryOption) []repo.RepositoryOption {
 	var opts []repo.RepositoryOption
 
@@ -98,63 +112,25 @@ func persistVaultConfig(v *vault.Vault) error {
 		return err
 	}
 
+	var lc config.LocalConfig
+	lc.VaultConnection = cfg
+
+	if v.RepoConfig.Connection != nil {
+		lc.RepoConnection = v.RepoConfig.Connection
+	}
+
 	fname := vaultConfigFileName()
 	log.Printf("Saving vault configuration to '%v'.", fname)
 	if err := os.MkdirAll(filepath.Dir(fname), 0700); err != nil {
 		return err
 	}
 
-	f, err := os.Create(fname)
+	d, err := json.MarshalIndent(&lc, "", "  ")
 	if err != nil {
 		return err
 	}
-	defer f.Close()
 
-	return json.NewEncoder(f).Encode(cfg)
-}
-
-func getPersistedVaultConfig() *vault.Config {
-	f, err := os.Open(vaultConfigFileName())
-	if err != nil {
-		return nil
-	}
-	defer f.Close()
-
-	vc := &vault.Config{}
-
-	if err := json.NewDecoder(f).Decode(vc); err != nil {
-		return nil
-	}
-
-	return vc
-}
-
-func openVault() (*vault.Vault, error) {
-	vc := getPersistedVaultConfig()
-	if vc != nil {
-		st, err := storage.NewStorage(vc.ConnectionInfo)
-		if err != nil {
-			return nil, fmt.Errorf("cannot open vault storage: %v", err)
-		}
-
-		if *traceStorage {
-			st = logging.NewWrapper(st, logging.Prefix("[VAULT] "))
-		}
-
-		creds, err := vault.MasterKey(vc.Key)
-		if err != nil {
-			st.Close()
-			return nil, fmt.Errorf("invalid vault config")
-		}
-
-		return vault.Open(st, creds)
-	}
-
-	if *vaultPath == "" {
-		return nil, fmt.Errorf("vault not connected and not specified, use --vault or run 'kopia connect'")
-	}
-
-	return openVaultSpecifiedByFlag()
+	return ioutil.WriteFile(fname, d, 0600)
 }
 
 func openVaultSpecifiedByFlag() (*vault.Vault, error) {

cmd/kopia/objref.go

@@ -11,7 +11,7 @@ import (
 )
 
 // ParseObjectID interprets the given ID string and returns corresponding repo.ObjectID.
-func parseObjectID(id string, vlt *vault.Vault) (repo.ObjectID, error) {
+func parseObjectID(id string, vlt *vault.Vault, r *repo.Repository) (repo.ObjectID, error) {
 	head, tail := splitHeadTail(id)
 	if len(head) == 0 {
 		return repo.NullObjectID, fmt.Errorf("invalid object ID: %v", id)
@@ -30,11 +30,6 @@ func parseObjectID(id string, vlt *vault.Vault) (repo.ObjectID, error) {
 		return oid, nil
 	}
 
-	r, err := vlt.OpenRepository()
-	if err != nil {
-		return repo.NullObjectID, fmt.Errorf("cannot open repository: %v", err)
-	}
-
 	dir := repofs.Directory(r, oid)
 	if err != nil {
 		return repo.NullObjectID, err

connection.go

@@ -0,0 +1,120 @@
+package kopia
+
+import (
+	"errors"
+	"fmt"
+
+	"github.com/kopia/kopia/internal/config"
+	"github.com/kopia/kopia/repo"
+	"github.com/kopia/kopia/storage"
+	"github.com/kopia/kopia/storage/caching"
+	"github.com/kopia/kopia/storage/logging"
+	"github.com/kopia/kopia/vault"
+)
+
+// Connection represents open connection to Vault and Repository.
+type Connection struct {
+	Vault      *vault.Vault
+	Repository *repo.Repository
+}
+
+// ConnectionOptions specifies the behavior of Connection.
+type ConnectionOptions struct {
+	CredentialsCallback func() (vault.Credentials, error) // credentials required to open the vault, unless persisted
+
+	TraceStorage      func(f string, args ...interface{})
+	RepositoryOptions []repo.RepositoryOption
+}
+
+// Close closes the underlying Vault and Repository.
+func (c *Connection) Close() error {
+	if c.Vault != nil {
+		c.Vault.Close()
+		c.Vault = nil
+	}
+
+	if c.Repository != nil {
+		c.Repository.Close()
+		c.Repository = nil
+	}
+
+	return nil
+}
+
+// Open connects to the Vault and Repository specified in the specified configuration file.
+func Open(configFile string, options *ConnectionOptions) (*Connection, error) {
+	lc, err := config.LoadFromFile(configFile)
+	if err != nil {
+		return nil, err
+	}
+
+	var creds vault.Credentials
+	if len(lc.VaultConnection.Key) > 0 {
+		creds, err = vault.MasterKey(lc.VaultConnection.Key)
+	} else {
+		if options.CredentialsCallback == nil {
+			return nil, errors.New("vault key not persisted and no credentials specified")
+		}
+		creds, err = options.CredentialsCallback()
+	}
+
+	if err != nil {
+		return nil, fmt.Errorf("invalid vault credentials: %v", err)
+	}
+
+	rawVaultStorage, err := storage.NewStorage(lc.VaultConnection.ConnectionInfo)
+	if err != nil {
+		return nil, fmt.Errorf("cannot open vault storage: %v", err)
+	}
+
+	vaultStorage := rawVaultStorage
+
+	if options.TraceStorage != nil {
+		vaultStorage = logging.NewWrapper(vaultStorage, logging.Prefix("[VAULT] "), logging.Output(options.TraceStorage))
+	}
+
+	var conn Connection
+	conn.Vault, err = vault.Open(vaultStorage, creds)
+	if err != nil {
+		rawVaultStorage.Close()
+		return nil, fmt.Errorf("unable to open vault: %v", err)
+	}
+
+	var repositoryStorage storage.Storage
+
+	if lc.RepoConnection == nil {
+		repositoryStorage = rawVaultStorage
+	} else {
+		repositoryStorage, err = storage.NewStorage(*lc.RepoConnection)
+		if err != nil {
+			vaultStorage.Close()
+			return nil, err
+		}
+	}
+
+	if options.TraceStorage != nil {
+		repositoryStorage = logging.NewWrapper(repositoryStorage, logging.Prefix("[STORAGE] "), logging.Output(options.TraceStorage))
+	}
+
+	if lc.Caching != nil {
+		rs, err := caching.NewWrapper(repositoryStorage, lc.Caching)
+		if err != nil {
+			vaultStorage.Close()
+			repositoryStorage.Close()
+			return nil, err
+		}
+		repositoryStorage = rs
+		if options.TraceStorage != nil {
+			repositoryStorage = logging.NewWrapper(repositoryStorage, logging.Prefix("[CACHE] "), logging.Output(options.TraceStorage))
+		}
+	}
+
+	conn.Repository, err = repo.New(repositoryStorage, conn.Vault.RepoConfig.Format)
+	if err != nil {
+		vaultStorage.Close()
+		repositoryStorage.Close()
+		return nil, err
+	}
+
+	return &conn, nil
+}

internal/config/local_config.go

@@ -0,0 +1,53 @@
+// Package config stores persistent configuration of Kopia.
+package config
+
+import (
+	"encoding/json"
+	"io"
+	"os"
+
+	"github.com/kopia/kopia/storage/caching"
+	"github.com/kopia/kopia/vault"
+
+	"github.com/kopia/kopia/storage"
+)
+
+// LocalConfig is a configuration of Kopia.
+type LocalConfig struct {
+	VaultConnection *vault.Config           `json:"vault,omitempty"`
+	RepoConnection  *storage.ConnectionInfo `json:"repository,omitempty"` // can be nil indicating the same connection as for the vault
+	Caching         *caching.Options        `json:"caching,omitempty"`
+}
+
+// Load reads local configuration from the specified reader.
+func (lc *LocalConfig) Load(r io.Reader) error {
+	*lc = LocalConfig{}
+	return json.NewDecoder(r).Decode(lc)
+}
+
+// Save writes the configuration to the specified writer.
+func (lc *LocalConfig) Save(w io.Writer) error {
+	b, err := json.MarshalIndent(lc, "", "  ")
+	if err != nil {
+		return nil
+	}
+	_, err = w.Write(b)
+	return err
+}
+
+// LoadFromFile reads the local configuration from the specified file.
+func LoadFromFile(fileName string) (*LocalConfig, error) {
+	f, err := os.Open(fileName)
+	if err != nil {
+		return nil, err
+	}
+	defer f.Close()
+
+	var lc LocalConfig
+
+	if err := lc.Load(f); err != nil {
+		return nil, err
+	}
+
+	return &lc, nil
+}

repo/format.go

@@ -9,6 +9,7 @@ import (
 	"crypto/sha512"
 	"encoding/hex"
 	"errors"
+	"fmt"
 	"hash"
 )
 
@@ -21,6 +22,24 @@ type Format struct {
 	MaxBlockSize           int32  `json:"maxBlockSize,omitempty"`           // maximum size of storage block
 }
 
+// Validate checks the validity of a Format and returns an error if invalid.
+func (f *Format) Validate() error {
+	if f.Version != 1 {
+		return fmt.Errorf("unsupported version: %v", f.Version)
+	}
+
+	if f.MaxBlockSize < 100 {
+		return fmt.Errorf("MaxBlockSize is not set")
+	}
+
+	sf := SupportedFormats[f.ObjectFormat]
+	if sf == nil {
+		return fmt.Errorf("unknown object format: %v", f.ObjectFormat)
+	}
+
+	return nil
+}
+
 // ObjectFormatter performs data block ID computation and encryption of a block of data when storing object in a repository,
 type ObjectFormatter interface {
 	// ComputeBlockIDAndKey computes ID of the storage block and encryption key for the specified block of data.

repo/repository.go

@@ -155,35 +155,30 @@ func (r *Repository) Open(objectID ObjectID) (ObjectReader, error) {
 }
 
 // RepositoryOption controls the behavior of Repository.
-type RepositoryOption func(o *Repository) error
+type RepositoryOption func(o *Repository)
 
 // WriteBack is an RepositoryOption that enables asynchronous writes to the storage using the pool
 // of goroutines.
 func WriteBack(writeBackWorkers int) RepositoryOption {
-	return func(o *Repository) error {
+	return func(o *Repository) {
 		o.writeBackWorkers = writeBackWorkers
-		return nil
 	}
 }
 
 // EnableLogging is an RepositoryOption that causes all storage access to be logged.
 func EnableLogging(options ...logging.Option) RepositoryOption {
-	return func(o *Repository) error {
+	return func(o *Repository) {
 		o.Storage = logging.NewWrapper(o.Storage, options...)
-		return nil
 	}
 }
 
 // New creates a Repository with the specified storage, format and options.
 func New(s storage.Storage, f *Format, options ...RepositoryOption) (*Repository, error) {
-	if f.MaxBlockSize < 100 {
-		return nil, fmt.Errorf("MaxBlockSize is not set")
+	if err := f.Validate(); err != nil {
+		return nil, err
 	}
 
 	sf := SupportedFormats[f.ObjectFormat]
-	if sf == nil {
-		return nil, fmt.Errorf("unknown object format: %v", f.ObjectFormat)
-	}
 
 	r := &Repository{
 		Storage: s,
@@ -193,10 +188,7 @@ func New(s storage.Storage, f *Format, options ...RepositoryOption) (*Repository
 	r.formatter = sf
 
 	for _, o := range options {
-		if err := o(r); err != nil {
-			r.Close()
-			return nil, err
-		}
+		o(r)
 	}
 
 	r.bufferManager = newBufferManager(int(r.format.MaxBlockSize))

storage/caching/caching_storage.go

@@ -199,13 +199,12 @@ func (c *cachingStorage) Close() error {
 
 // Options provides options for the caching storage wrapper
 type Options struct {
-	CacheDir       string
-	CacheSizeBytes int64
-	_              struct{}
+	CacheDir       string `json:"cacheDir"`
+	CacheSizeBytes int64  `json:"cacheSizeBytes"`
 }
 
 // NewWrapper creates new caching storage wrapper.
-func NewWrapper(master storage.Storage, options Options) (storage.Storage, error) {
+func NewWrapper(master storage.Storage, options *Options) (storage.Storage, error) {
 	if options.CacheDir == "" {
 		return nil, fmt.Errorf("Cache directory must be specified")
 	}

storage/caching/caching_storage_test.go

@@ -79,7 +79,7 @@ func TestCache(t *testing.T) {
 	var tr tracer
 	master = logging.NewWrapper(master, logging.Output(tr.Printf))
 
-	cache, err := NewWrapper(master, Options{CacheDir: tmpdir})
+	cache, err := NewWrapper(master, &Options{CacheDir: tmpdir})
 	defer cache.Close()
 	if err != nil {
 		t.Errorf("cannot create cache: %v", err)

storage/config.go

@@ -2,6 +2,7 @@ package storage
 
 import (
 	"encoding/json"
+	"fmt"
 )
 
 // ConnectionInfo represents JSON-serializable configuration of a blob storage.
@@ -22,7 +23,11 @@ func (c *ConnectionInfo) UnmarshalJSON(b []byte) error {
 	}
 
 	c.Type = raw.Type
-	c.Config = factories[raw.Type].defaultConfigFunc()
+	f := factories[raw.Type]
+	if f == nil {
+		return fmt.Errorf("storage type '%v' not registered", raw.Type)
+	}
+	c.Config = f.defaultConfigFunc()
 	if err := json.Unmarshal(raw.Data, c.Config); err != nil {
 		return err
 	}

storage/gcs/gcs_storage.go

@@ -417,3 +417,5 @@ func init() {
 			return New(o.(*Options))
 		})
 }
+
+var _ storage.ConnectionInfoProvider = &gcsStorage{}

vault/objectid_storage.go

@@ -19,7 +19,7 @@ const (
 
 // SaveObjectID stores the given object ID in an encrypted vault item and returns a unique ID.
 func (vlt *Vault) SaveObjectID(oid repo.ObjectID) (string, error) {
-	h := hmac.New(sha256.New, vlt.format.UniqueID)
+	h := hmac.New(sha256.New, vlt.Format.UniqueID)
 	bytes, err := json.Marshal(&oid)
 	if err != nil {
 		return "", err

vault/vault.go

@@ -1,7 +1,6 @@
 package vault
 
 import (
-	"bytes"
 	"crypto/aes"
 	"crypto/cipher"
 	"crypto/hmac"
@@ -38,26 +37,12 @@ var ErrItemNotFound = errors.New("item not found")
 
 // Vault is a secure storage for the secrets.
 type Vault struct {
-	storage    storage.Storage
+	Storage    storage.Storage
+	Format     Format
+	RepoConfig RepositoryConfig
+
 	masterKey  []byte
-	format     Format
 	itemPrefix string
-	repoConfig repositoryConfig
-}
-
-type repositoryConfig struct {
-	Connection *storage.ConnectionInfo `json:"connection"`
-	Format     *repo.Format            `json:"format"`
-}
-
-// Storage returns the underlying blob storage that stores the repository.
-func (v *Vault) Storage() storage.Storage {
-	return v.storage
-}
-
-// Format returns the vault format.
-func (v *Vault) Format() Format {
-	return v.format
 }
 
 // Put saves the specified content in a vault under a specified name.
@@ -99,11 +84,11 @@ func (v *Vault) writeEncryptedBlock(itemID string, content []byte) error {
 		content = cipherText
 	}
 
-	return v.storage.PutBlock(v.itemPrefix+itemID, content, storage.PutOptionsOverwrite)
+	return v.Storage.PutBlock(v.itemPrefix+itemID, content, storage.PutOptionsOverwrite)
 }
 
 func (v *Vault) readEncryptedBlock(itemID string) ([]byte, error) {
-	content, err := v.storage.GetBlock(v.itemPrefix + itemID)
+	content, err := v.Storage.GetBlock(v.itemPrefix + itemID)
 	if err != nil {
 		if err == storage.ErrBlockNotFound {
 			return nil, ErrItemNotFound
@@ -150,20 +135,20 @@ func (v *Vault) decryptBlock(content []byte) ([]byte, error) {
 }
 
 func (v *Vault) newChecksum() (hash.Hash, error) {
-	switch v.format.Checksum {
+	switch v.Format.Checksum {
 	case "hmac-sha-256":
 		key := make([]byte, 32)
 		v.deriveKey(purposeChecksumSecret, key)
 		return hmac.New(sha256.New, key), nil
 
 	default:
-		return nil, fmt.Errorf("unsupported checksum format: %v", v.format.Checksum)
+		return nil, fmt.Errorf("unsupported checksum format: %v", v.Format.Checksum)
 	}
 
 }
 
 func (v *Vault) newCipher() (cipher.Block, error) {
-	switch v.format.Encryption {
+	switch v.Format.Encryption {
 	case "none":
 		return nil, nil
 	case "aes-128":
@@ -179,39 +164,17 @@ func (v *Vault) newCipher() (cipher.Block, error) {
 		v.deriveKey(purposeAESKey, k)
 		return aes.NewCipher(k)
 	default:
-		return nil, fmt.Errorf("unsupported encryption format: %v", v.format.Encryption)
+		return nil, fmt.Errorf("unsupported encryption format: %v", v.Format.Encryption)
 	}
 
 }
 
 func (v *Vault) deriveKey(purpose []byte, key []byte) error {
-	k := hkdf.New(sha256.New, v.masterKey, v.format.UniqueID, purpose)
+	k := hkdf.New(sha256.New, v.masterKey, v.Format.UniqueID, purpose)
 	_, err := io.ReadFull(k, key)
 	return err
 }
 
-// RepositoryFormat returns the format of the repository.
-func (v *Vault) RepositoryFormat() *repo.Format {
-	return v.repoConfig.Format
-}
-
-// OpenRepository connects to the repository the vault is associated with.
-func (v *Vault) OpenRepository(options ...repo.RepositoryOption) (*repo.Repository, error) {
-	var st storage.Storage
-	var err error
-
-	if v.repoConfig.Connection != nil {
-		st, err = storage.NewStorage(*v.repoConfig.Connection)
-		if err != nil {
-			return nil, fmt.Errorf("unable to open repository: %v", err)
-		}
-	} else {
-		st = v.storage
-	}
-
-	return repo.New(st, v.repoConfig.Format, options...)
-}
-
 // Get returns the contents of a specified vault item.
 func (v *Vault) Get(itemID string) ([]byte, error) {
 	if err := checkReservedName(itemID); err != nil {
@@ -244,7 +207,7 @@ func (v *Vault) putJSON(id string, content interface{}) error {
 func (v *Vault) List(prefix string) ([]string, error) {
 	var result []string
 
-	for b := range v.storage.ListBlocks(v.itemPrefix + prefix) {
+	for b := range v.Storage.ListBlocks(v.itemPrefix + prefix) {
 		if b.Error != nil {
 			return result, b.Error
 		}
@@ -259,7 +222,7 @@ func (v *Vault) List(prefix string) ([]string, error) {
 
 // Close releases any resources held by Vault and closes repository connection.
 func (v *Vault) Close() error {
-	return nil
+	return v.Storage.Close()
 }
 
 // Config represents JSON-compatible configuration of the vault connection, including vault key.
@@ -271,7 +234,7 @@ type Config struct {
 // Config returns a configuration of vault storage its credentials that's suitable
 // for storing in configuration file.
 func (v *Vault) Config() (*Config, error) {
-	cip, ok := v.storage.(storage.ConnectionInfoProvider)
+	cip, ok := v.Storage.(storage.ConnectionInfoProvider)
 	if !ok {
 		return nil, errors.New("repository does not support persisting configuration")
 	}
@@ -290,7 +253,7 @@ func (v *Vault) Remove(itemID string) error {
 		return err
 	}
 
-	return v.storage.DeleteBlock(v.itemPrefix + itemID)
+	return v.Storage.DeleteBlock(v.itemPrefix + itemID)
 }
 
 // Create initializes a Vault attached to the specified repository.
@@ -302,11 +265,11 @@ func Create(
 	repoFormat *repo.Format,
 ) (*Vault, error) {
 	v := Vault{
-		storage: vaultStorage,
-		format:  *vaultFormat,
+		Storage: vaultStorage,
+		Format:  *vaultFormat,
 	}
 
-	if repoStorage == nil || sameStorage(repoStorage, vaultStorage) {
+	if repoStorage == nil {
 		repoStorage = vaultStorage
 		v.itemPrefix = colocatedVaultItemPrefix
 	}
@@ -316,14 +279,14 @@ func Create(
 		return nil, errors.New("repository does not support persisting configuration")
 	}
 
-	v.format.Version = "1"
-	v.format.UniqueID = make([]byte, 32)
-	if _, err := io.ReadFull(rand.Reader, v.format.UniqueID); err != nil {
+	v.Format.Version = "1"
+	v.Format.UniqueID = make([]byte, 32)
+	if _, err := io.ReadFull(rand.Reader, v.Format.UniqueID); err != nil {
 		return nil, err
 	}
-	v.masterKey = vaultCreds.getMasterKey(v.format.UniqueID)
+	v.masterKey = vaultCreds.getMasterKey(v.Format.UniqueID)
 
-	formatBytes, err := json.Marshal(&v.format)
+	formatBytes, err := json.Marshal(&v.Format)
 	if err != nil {
 		return nil, err
 	}
@@ -333,7 +296,7 @@ func Create(
 	}
 
 	// Write encrypted repository configuration block.
-	rc := repositoryConfig{
+	rc := RepositoryConfig{
 		Format: repoFormat,
 	}
 
@@ -346,14 +309,30 @@ func Create(
 		return nil, err
 	}
 
-	v.repoConfig = rc
+	v.RepoConfig = rc
 	return &v, nil
 }
 
+// CreateColocated initializes a Vault attached to a Repository sharing the same storage.
+func CreateColocated(
+	sharedStorage storage.Storage,
+	vaultFormat *Format,
+	vaultCreds Credentials,
+	repoFormat *repo.Format,
+) (*Vault, error) {
+	return Create(sharedStorage, vaultFormat, vaultCreds, nil, repoFormat)
+}
+
+// RepositoryConfig stores the configuration of the repository associated with the vault.
+type RepositoryConfig struct {
+	Connection *storage.ConnectionInfo `json:"connection"`
+	Format     *repo.Format            `json:"format"`
+}
+
 // Open opens a vault.
 func Open(vaultStorage storage.Storage, vaultCreds Credentials) (*Vault, error) {
 	v := Vault{
-		storage: vaultStorage,
+		Storage: vaultStorage,
 	}
 
 	var prefix string
@@ -383,12 +362,12 @@ func Open(vaultStorage storage.Storage, vaultCreds Credentials) (*Vault, error)
 		offset = 2
 	}
 
-	err := json.Unmarshal(blocks[offset], &v.format)
+	err := json.Unmarshal(blocks[offset], &v.Format)
 	if err != nil {
 		return nil, err
 	}
 
-	v.masterKey = vaultCreds.getMasterKey(v.format.UniqueID)
+	v.masterKey = vaultCreds.getMasterKey(v.Format.UniqueID)
 	v.itemPrefix = prefix
 
 	cfgData, err := v.decryptBlock(blocks[offset+1])
@@ -396,13 +375,13 @@ func Open(vaultStorage storage.Storage, vaultCreds Credentials) (*Vault, error)
 		return nil, err
 	}
 
-	var rc repositoryConfig
+	var rc RepositoryConfig
 
 	if err := json.Unmarshal(cfgData, &rc); err != nil {
 		return nil, err
 	}
 
-	v.repoConfig = rc
+	v.RepoConfig = rc
 
 	return &v, nil
 }
@@ -424,28 +403,3 @@ func checkReservedName(itemID string) error {
 
 	return nil
 }
-
-func sameStorage(s1, s2 storage.Storage) bool {
-	if s1 == s2 {
-		return true
-	}
-
-	cip1, ok := s1.(storage.ConnectionInfoProvider)
-	if !ok {
-		return false
-	}
-
-	cip2, ok := s2.(storage.ConnectionInfoProvider)
-	if !ok {
-		return false
-	}
-
-	c1 := cip1.ConnectionInfo()
-	c2 := cip2.ConnectionInfo()
-
-	b1, e1 := json.Marshal(c1)
-	b2, e2 := json.Marshal(c2)
-
-	same := e1 == nil && e2 == nil && bytes.Equal(b1, b2)
-	return same
-}

vault/vault_test.go

@@ -68,7 +68,7 @@ func verifyVault(t *testing.T, vaultPath string, repoPath string) {
 		Secret:       []byte{1, 2, 3},
 	}
 
-	_, err = repo.New(repoStorage, repoFormat)
+	r, err := repo.New(repoStorage, repoFormat)
 	if err != nil {
 		t.Errorf("can't create repository: %v", err)
 		return
@@ -116,24 +116,14 @@ func verifyVault(t *testing.T, vaultPath string, repoPath string) {
 		t.Errorf("error putting: %v", err)
 	}
 
-	r1, err := v1.OpenRepository()
-	if err != nil {
-		t.Errorf("error opening v1 repository: %v", err)
-	}
-
-	r2, err := v2.OpenRepository()
-	if err != nil {
-		t.Errorf("error opening v1 repository: %v", err)
-	}
-
-	w1 := r1.NewWriter()
+	w1 := r.NewWriter()
 	w1.Write([]byte("foo"))
 	oid1, err := w1.Result(true)
 	if err != nil {
 		t.Errorf("Result error: %v", err)
 	}
 
-	w2 := r2.NewWriter()
+	w2 := r.NewWriter()
 	w2.Write([]byte("bar"))
 	oid2, err := w2.Result(true)
 	if err != nil {
@@ -161,7 +151,7 @@ func verifyVault(t *testing.T, vaultPath string, repoPath string) {
 
 	// Verify contents of vault items for both created and opened vault.
 	for _, v := range []*Vault{v1, v2} {
-		rf := v.RepositoryFormat()
+		rf := v.RepoConfig.Format
 		if !reflect.DeepEqual(rf, repoFormat) {
 			t.Errorf("invalid repository format: %v, but got %v", repoFormat, rf)
 		}
@@ -204,12 +194,6 @@ func verifyVault(t *testing.T, vaultPath string, repoPath string) {
 		}
 	}
 
-	// Make sure repository is shared, by checking that data written in one can be read by the other.
-	assertRepositoryItem(t, r1, oid1, "foo")
-	assertRepositoryItem(t, r1, oid2, "bar")
-	assertRepositoryItem(t, r2, oid1, "foo")
-	assertRepositoryItem(t, r2, oid2, "bar")
-
 	v1.Remove("bar")
 
 	for _, v := range []*Vault{v1, v2} {
@@ -245,26 +229,6 @@ func assertVaultItem(t *testing.T, v *Vault, itemID string, expectedData string)
 	}
 }
 
-func assertRepositoryItem(t *testing.T, repository *repo.Repository, oid repo.ObjectID, expectedData string) {
-	r, err := repository.Open(oid)
-	if err != nil {
-		t.Errorf("error opening item %v: %v", oid, err)
-		return
-	}
-	defer r.Close()
-
-	b, err := ioutil.ReadAll(r)
-	if err != nil {
-		t.Errorf("error reading item %v: %v", oid, err)
-		return
-	}
-
-	bs := string(b)
-	if bs != expectedData {
-		t.Errorf("invalid data for '%v': expected: %v but got %v", oid, expectedData, bs)
-	}
-}
-
 func assertVaultItemNotFound(t *testing.T, v *Vault, itemID string) {
 	result, err := v.Get(itemID)
 	if err != ErrItemNotFound {