From baa8451403412b0667ff4bc92cfae7fc1563f495 Mon Sep 17 00:00:00 2001 From: Jarek Kowalski Date: Mon, 5 Sep 2016 22:30:27 -0700 Subject: [PATCH] added kopia.Connection as top-level API. simplified vault API --- cmd/kopia/command_backup.go | 14 +-- cmd/kopia/command_backups.go | 10 +- cmd/kopia/command_cat.go | 11 +- cmd/kopia/command_create.go | 31 +++--- cmd/kopia/command_ls.go | 9 +- cmd/kopia/command_mount.go | 6 +- cmd/kopia/command_restore.go | 61 ----------- cmd/kopia/command_show.go | 9 +- cmd/kopia/command_status.go | 11 +- cmd/kopia/command_vault_list.go | 4 +- cmd/kopia/command_vault_rm.go | 4 +- cmd/kopia/command_vault_show.go | 7 +- cmd/kopia/config.go | 110 ++++++++----------- cmd/kopia/objref.go | 7 +- connection.go | 120 +++++++++++++++++++++ internal/config/local_config.go | 53 +++++++++ repo/format.go | 19 ++++ repo/repository.go | 20 ++-- storage/caching/caching_storage.go | 7 +- storage/caching/caching_storage_test.go | 2 +- storage/config.go | 7 +- storage/gcs/gcs_storage.go | 2 + vault/objectid_storage.go | 2 +- vault/vault.go | 138 ++++++++---------------- vault/vault_test.go | 44 +------- 25 files changed, 359 insertions(+), 349 deletions(-) delete mode 100644 cmd/kopia/command_restore.go create mode 100644 connection.go create mode 100644 internal/config/local_config.go diff --git a/cmd/kopia/command_backup.go b/cmd/kopia/command_backup.go index 5a6450534..1009690cc 100644 --- a/cmd/kopia/command_backup.go +++ b/cmd/kopia/command_backup.go @@ -50,12 +50,12 @@ func runBackupCommand(context *kingpin.ParseContext) error { repoOptions = append(repoOptions, repo.WriteBack(*backupWriteBack)) } - vlt, r := mustOpenVaultAndRepository(repoOptions...) - defer r.Close() + conn := mustOpenConnection(repoOptions...) + defer conn.Close() var options []repofs.UploadOption - bgen, err := backup.NewGenerator(r, options...) + bgen, err := backup.NewGenerator(conn.Repository, options...) if err != nil { return err } @@ -79,7 +79,7 @@ func runBackupCommand(context *kingpin.ParseContext) error { return fmt.Errorf("description too long") } - previous, err := vlt.List("B" + manifest.SourceID() + ".") + previous, err := conn.Vault.List("B" + manifest.SourceID() + ".") if err != nil { return fmt.Errorf("error listing previous backups") } @@ -87,7 +87,7 @@ func runBackupCommand(context *kingpin.ParseContext) error { var oldManifest *backup.Manifest if len(previous) > 0 { - oldManifest, err = loadBackupManifest(vlt, previous[0]) + oldManifest, err = loadBackupManifest(conn.Vault, previous[0]) } localEntry := mustGetLocalFSEntry(manifest.Source) @@ -99,7 +99,7 @@ func runBackupCommand(context *kingpin.ParseContext) error { return err } - handleID, err := vlt.SaveObjectID(manifest.RootObjectID) + handleID, err := conn.Vault.SaveObjectID(manifest.RootObjectID) if err != nil { return err } @@ -109,7 +109,7 @@ func runBackupCommand(context *kingpin.ParseContext) error { fileID := fmt.Sprintf("B%v.%08x.%x", manifest.SourceID(), math.MaxInt64-manifest.StartTime.UnixNano(), uniqueID) manifest.Handle = handleID - err = saveBackupManifest(vlt, fileID, &manifest) + err = saveBackupManifest(conn.Vault, fileID, &manifest) if err != nil { return fmt.Errorf("cannot save manifest: %v", err) } diff --git a/cmd/kopia/command_backups.go b/cmd/kopia/command_backups.go index 74ab57e55..fd06fcba8 100644 --- a/cmd/kopia/command_backups.go +++ b/cmd/kopia/command_backups.go @@ -57,8 +57,8 @@ func findBackups(vlt *vault.Vault, path string) ([]string, string, error) { } func runBackupsCommand(context *kingpin.ParseContext) error { - vlt, r := mustOpenVaultAndRepository() - defer r.Close() + conn := mustOpenConnection() + defer conn.Close() var previous []string var relPath string @@ -70,12 +70,12 @@ func runBackupsCommand(context *kingpin.ParseContext) error { return fmt.Errorf("invalid directory: '%s': %s", *backupsPath, err) } - previous, relPath, err = findBackups(vlt, filepath.Clean(path)) + previous, relPath, err = findBackups(conn.Vault, filepath.Clean(path)) if relPath != "" { relPath = "/" + relPath } } else { - previous, err = vlt.List("B") + previous, err = conn.Vault.List("B") } if err != nil { @@ -87,7 +87,7 @@ func runBackupsCommand(context *kingpin.ParseContext) error { var lastSource string var count int - for _, m := range loadBackupManifests(vlt, previous) { + for _, m := range loadBackupManifests(conn.Vault, previous) { if m.HostName != lastHost || m.UserName != lastUser || m.Source != lastSource { fmt.Printf("%v@%v:%v\n", m.UserName, m.HostName, m.Source) lastSource = m.Source diff --git a/cmd/kopia/command_cat.go b/cmd/kopia/command_cat.go index 206a23381..e8a9d3ea1 100644 --- a/cmd/kopia/command_cat.go +++ b/cmd/kopia/command_cat.go @@ -13,17 +13,14 @@ var ( ) func runCatCommand(context *kingpin.ParseContext) error { - vlt := mustOpenVault() - mgr, err := vlt.OpenRepository() - if err != nil { - return err - } + conn := mustOpenConnection() + defer conn.Close() - oid, err := parseObjectID(*catCommandPath, vlt) + oid, err := parseObjectID(*catCommandPath, conn.Vault, conn.Repository) if err != nil { return err } - r, err := mgr.Open(oid) + r, err := conn.Repository.Open(oid) if err != nil { return err } diff --git a/cmd/kopia/command_create.go b/cmd/kopia/command_create.go index d059ddbfd..ff0eab569 100644 --- a/cmd/kopia/command_create.go +++ b/cmd/kopia/command_create.go @@ -83,17 +83,6 @@ func runCreateCommand(context *kingpin.ParseContext) error { return fmt.Errorf("unable to get vault storage: %v", err) } - var repositoryStorage storage.Storage - - if *createCommandRepository == "colocated" { - repositoryStorage = vaultStorage - } else { - repositoryStorage, err = openStorageAndEnsureEmpty(*createCommandRepository) - if err != nil { - return fmt.Errorf("unable to get repository storage: %v", err) - } - } - repoFormat, err := repositoryFormat() if err != nil { return fmt.Errorf("unable to initialize repository format: %v", err) @@ -114,16 +103,26 @@ func runCreateCommand(context *kingpin.ParseContext) error { return fmt.Errorf("unable to get credentials: %v", err) } - // Make repository to make sure the format is supported. - _, err = repo.New(repositoryStorage, repoFormat) - if err != nil { - return fmt.Errorf("unable to initialize repository: %v", err) + if err := repoFormat.Validate(); err != nil { + return fmt.Errorf("invalid format") } fmt.Printf( "Initializing vault with encryption '%v'.\n", vf.Encryption) - vlt, err := vault.Create(vaultStorage, vf, creds, repositoryStorage, repoFormat) + + var vlt *vault.Vault + + if *createCommandRepository == "colocated" { + vlt, err = vault.CreateColocated(vaultStorage, vf, creds, repoFormat) + } else { + repositoryStorage, err := openStorageAndEnsureEmpty(*createCommandRepository) + if err != nil { + return fmt.Errorf("unable to get repository storage: %v", err) + } + vlt, err = vault.Create(vaultStorage, vf, creds, repositoryStorage, repoFormat) + } + if err != nil { return fmt.Errorf("cannot create vault: %v", err) } diff --git a/cmd/kopia/command_ls.go b/cmd/kopia/command_ls.go index f4a21d861..2702dd81d 100644 --- a/cmd/kopia/command_ls.go +++ b/cmd/kopia/command_ls.go @@ -20,11 +20,10 @@ var ( ) func runLSCommand(context *kingpin.ParseContext) error { - vlt, r := mustOpenVaultAndRepository() - defer vlt.Close() - defer r.Close() + conn := mustOpenConnection() + defer conn.Close() - oid, err := parseObjectID(*lsCommandPath, vlt) + oid, err := parseObjectID(*lsCommandPath, conn.Vault, conn.Repository) if err != nil { return err } @@ -37,7 +36,7 @@ func runLSCommand(context *kingpin.ParseContext) error { } } - d := repofs.Directory(r, oid) + d := repofs.Directory(conn.Repository, oid) entries, err := d.Readdir() if err != nil { diff --git a/cmd/kopia/command_mount.go b/cmd/kopia/command_mount.go index a86a4ff35..f8fb3a79a 100644 --- a/cmd/kopia/command_mount.go +++ b/cmd/kopia/command_mount.go @@ -27,7 +27,7 @@ func (r *root) Root() (fusefs.Node, error) { } func runMountCommand(context *kingpin.ParseContext) error { - vlt, r := mustOpenVaultAndRepository() + conn := mustOpenConnection() fuseConnection, err := fuse.Mount( *mountPoint, @@ -37,12 +37,12 @@ func runMountCommand(context *kingpin.ParseContext) error { fuse.VolumeName("Kopia"), ) - oid, err := parseObjectID(*mountObjectID, vlt) + oid, err := parseObjectID(*mountObjectID, conn.Vault, conn.Repository) if err != nil { return err } - rootNode := kopiafuse.NewDirectoryNode(repofs.Directory(r, oid)) + rootNode := kopiafuse.NewDirectoryNode(repofs.Directory(conn.Repository, oid)) fusefs.Serve(fuseConnection, &root{rootNode}) diff --git a/cmd/kopia/command_restore.go b/cmd/kopia/command_restore.go deleted file mode 100644 index f59bf2824..000000000 --- a/cmd/kopia/command_restore.go +++ /dev/null @@ -1,61 +0,0 @@ -package main - -import kingpin "gopkg.in/alecthomas/kingpin.v2" - -var ( - restoreCommand = app.Command("restore", "Restore contents of a file or directory.") - - restoreCommandLong = restoreCommand.Flag("recursive", "Copy contents of directory recursively").Short('r').Bool() - restoreCommandPath = restoreCommand.Arg("paths", "Paths to restore").Required().Strings() -) - -func runRestoreCommand(context *kingpin.ParseContext) error { - vlt := mustOpenVault() - mgr, err := vlt.OpenRepository() - if err != nil { - return err - } - - _ = mgr - - // oid, err := ParseObjectID(*restoreCommandPath, vlt) - // if err != nil { - // return err - // } - // r, err := mgr.Open(oid) - // if err != nil { - // return err - // } - - // var prefix string - // if !*restoreCommandLong { - // prefix = *restoreCommandPath - // if !strings.HasSuffix(prefix, "/") { - // prefix += "/" - // } - // } - - // dir, err := fs.ReadDirectory(r, "") - // if err != nil { - // return fmt.Errorf("unable to read directory contents") - // } - - // if *restoreCommandLong { - // listDirectory(dir) - // } else { - // for _, e := range dir { - // var suffix string - // if e.FileMode.IsDir() { - // suffix = "/" - // } - - // fmt.Println(prefix + e.Name + suffix) - // } - // } - - return nil -} - -func init() { - restoreCommand.Action(runRestoreCommand) -} diff --git a/cmd/kopia/command_show.go b/cmd/kopia/command_show.go index 8f9912fb9..c863f9a80 100644 --- a/cmd/kopia/command_show.go +++ b/cmd/kopia/command_show.go @@ -20,17 +20,16 @@ var ( ) func runShowCommand(context *kingpin.ParseContext) error { - vlt, r := mustOpenVaultAndRepository() - defer vlt.Close() - defer r.Close() + conn := mustOpenConnection() + defer conn.Close() for _, oidString := range *showObjectIDs { - oid, err := parseObjectID(oidString, vlt) + oid, err := parseObjectID(oidString, conn.Vault, conn.Repository) if err != nil { return err } - if err := showObject(r, oid); err != nil { + if err := showObject(conn.Repository, oid); err != nil { return err } } diff --git a/cmd/kopia/command_status.go b/cmd/kopia/command_status.go index e2ea4551a..67deee21e 100644 --- a/cmd/kopia/command_status.go +++ b/cmd/kopia/command_status.go @@ -17,18 +17,19 @@ func init() { } func runRepositoryInfoCommand(context *kingpin.ParseContext) error { - vlt, r := mustOpenVaultAndRepository() + conn := mustOpenConnection() - vf := vlt.Format() + vf := conn.Vault.Format fmt.Println("Vault:") fmt.Println(" Version: ", vf.Version) fmt.Println(" Unique ID: ", hex.EncodeToString(vf.UniqueID)) fmt.Println(" Encryption: ", vf.Encryption) fmt.Println(" Checksum: ", vf.Checksum) - fmt.Println(" Storage: ", vlt.Storage()) + fmt.Println(" Storage: ", conn.Vault.Storage) fmt.Println() - f := vlt.RepositoryFormat() + rc := conn.Vault.RepoConfig + f := rc.Format fmt.Println("Repository:") fmt.Println(" Version: ", f.Version) @@ -40,7 +41,7 @@ func runRepositoryInfoCommand(context *kingpin.ParseContext) error { fmt.Println(" ID Format: ", f.ObjectFormat) fmt.Println(" Blob Size: ", f.MaxBlockSize/1024, "KB") fmt.Println(" Inline Blob Size:", f.MaxInlineContentLength/1024, "KB") - fmt.Println(" Storage: ", r.Storage) + fmt.Println(" Storage: ", conn.Repository.Storage) return nil } diff --git a/cmd/kopia/command_vault_list.go b/cmd/kopia/command_vault_list.go index 1b86b26b7..dacc83e6f 100644 --- a/cmd/kopia/command_vault_list.go +++ b/cmd/kopia/command_vault_list.go @@ -16,9 +16,9 @@ func init() { } func listVaultContents(context *kingpin.ParseContext) error { - vlt := mustOpenVault() + conn := mustOpenConnection() - entries, err := vlt.List(*vaultListPrefix) + entries, err := conn.Vault.List(*vaultListPrefix) if err != nil { return err } diff --git a/cmd/kopia/command_vault_rm.go b/cmd/kopia/command_vault_rm.go index 6217039b8..ce499a882 100644 --- a/cmd/kopia/command_vault_rm.go +++ b/cmd/kopia/command_vault_rm.go @@ -12,10 +12,10 @@ func init() { } func removeVaultItem(context *kingpin.ParseContext) error { - vlt := mustOpenVault() + conn := mustOpenConnection() for _, v := range *vaultRemoveItems { - if err := vlt.Remove(v); err != nil { + if err := conn.Vault.Remove(v); err != nil { return err } } diff --git a/cmd/kopia/command_vault_show.go b/cmd/kopia/command_vault_show.go index 0e83ba316..05c3ffd0f 100644 --- a/cmd/kopia/command_vault_show.go +++ b/cmd/kopia/command_vault_show.go @@ -20,12 +20,9 @@ func init() { } func showVaultObject(context *kingpin.ParseContext) error { - v, err := openVault() - if err != nil { - return err - } + conn := mustOpenConnection() - b, err := v.Get(*vaultShowID) + b, err := conn.Vault.Get(*vaultShowID) if err != nil { return err } diff --git a/cmd/kopia/config.go b/cmd/kopia/config.go index b0a6f4371..66bdfdba2 100644 --- a/cmd/kopia/config.go +++ b/cmd/kopia/config.go @@ -11,15 +11,14 @@ import ( "runtime" "strings" + "github.com/bgentry/speakeasy" + "github.com/kopia/kopia" + "github.com/kopia/kopia/fs" "github.com/kopia/kopia/fs/localfs" "github.com/kopia/kopia/fs/loggingfs" - - "github.com/kopia/kopia/fs" - "github.com/kopia/kopia/storage/logging" - - "github.com/bgentry/speakeasy" + "github.com/kopia/kopia/internal/config" "github.com/kopia/kopia/repo" - "github.com/kopia/kopia/storage" + "github.com/kopia/kopia/storage/logging" "github.com/kopia/kopia/vault" ) @@ -35,6 +34,16 @@ var ( keyFile = app.Flag("keyfile", "Read vault master key from file.").PlaceHolder("FILENAME").Envar("KOPIA_KEY_FILE").ExistingFile() ) +func mustLoadLocalConfig() *config.LocalConfig { + lc, err := loadLocalConfig() + failOnError(err) + return lc +} + +func loadLocalConfig() (*config.LocalConfig, error) { + return config.LoadFromFile(vaultConfigFileName()) +} + func failOnError(err error) { if err != nil { fmt.Fprintf(os.Stderr, "ERROR: %v\n", err) @@ -42,24 +51,29 @@ func failOnError(err error) { } } -func mustOpenVault() *vault.Vault { - s, err := openVault() +func openConnection(options ...repo.RepositoryOption) (*kopia.Connection, error) { + return kopia.Open(vaultConfigFileName(), connectionOptionsFromFlags(options...)) +} + +func connectionOptionsFromFlags(options ...repo.RepositoryOption) *kopia.ConnectionOptions { + opts := &kopia.ConnectionOptions{ + CredentialsCallback: func() (vault.Credentials, error) { return getVaultCredentials(false) }, + RepositoryOptions: options, + } + + if *traceStorage { + opts.TraceStorage = log.Printf + } + + return opts +} + +func mustOpenConnection(repoOptions ...repo.RepositoryOption) *kopia.Connection { + s, err := openConnection(repoOptions...) failOnError(err) return s } -func mustOpenRepository(extraOptions ...repo.RepositoryOption) *repo.Repository { - _, r := mustOpenVaultAndRepository(extraOptions...) - return r -} - -func mustOpenVaultAndRepository(extraOptions ...repo.RepositoryOption) (*vault.Vault, *repo.Repository) { - v := mustOpenVault() - r, err := v.OpenRepository(repositoryOptionsFromFlags(extraOptions)...) - failOnError(err) - return v, r -} - func repositoryOptionsFromFlags(extraOptions []repo.RepositoryOption) []repo.RepositoryOption { var opts []repo.RepositoryOption @@ -98,63 +112,25 @@ func persistVaultConfig(v *vault.Vault) error { return err } + var lc config.LocalConfig + lc.VaultConnection = cfg + + if v.RepoConfig.Connection != nil { + lc.RepoConnection = v.RepoConfig.Connection + } + fname := vaultConfigFileName() log.Printf("Saving vault configuration to '%v'.", fname) if err := os.MkdirAll(filepath.Dir(fname), 0700); err != nil { return err } - f, err := os.Create(fname) + d, err := json.MarshalIndent(&lc, "", " ") if err != nil { return err } - defer f.Close() - return json.NewEncoder(f).Encode(cfg) -} - -func getPersistedVaultConfig() *vault.Config { - f, err := os.Open(vaultConfigFileName()) - if err != nil { - return nil - } - defer f.Close() - - vc := &vault.Config{} - - if err := json.NewDecoder(f).Decode(vc); err != nil { - return nil - } - - return vc -} - -func openVault() (*vault.Vault, error) { - vc := getPersistedVaultConfig() - if vc != nil { - st, err := storage.NewStorage(vc.ConnectionInfo) - if err != nil { - return nil, fmt.Errorf("cannot open vault storage: %v", err) - } - - if *traceStorage { - st = logging.NewWrapper(st, logging.Prefix("[VAULT] ")) - } - - creds, err := vault.MasterKey(vc.Key) - if err != nil { - st.Close() - return nil, fmt.Errorf("invalid vault config") - } - - return vault.Open(st, creds) - } - - if *vaultPath == "" { - return nil, fmt.Errorf("vault not connected and not specified, use --vault or run 'kopia connect'") - } - - return openVaultSpecifiedByFlag() + return ioutil.WriteFile(fname, d, 0600) } func openVaultSpecifiedByFlag() (*vault.Vault, error) { diff --git a/cmd/kopia/objref.go b/cmd/kopia/objref.go index f757003bd..04c64ff53 100644 --- a/cmd/kopia/objref.go +++ b/cmd/kopia/objref.go @@ -11,7 +11,7 @@ import ( ) // ParseObjectID interprets the given ID string and returns corresponding repo.ObjectID. -func parseObjectID(id string, vlt *vault.Vault) (repo.ObjectID, error) { +func parseObjectID(id string, vlt *vault.Vault, r *repo.Repository) (repo.ObjectID, error) { head, tail := splitHeadTail(id) if len(head) == 0 { return repo.NullObjectID, fmt.Errorf("invalid object ID: %v", id) @@ -30,11 +30,6 @@ func parseObjectID(id string, vlt *vault.Vault) (repo.ObjectID, error) { return oid, nil } - r, err := vlt.OpenRepository() - if err != nil { - return repo.NullObjectID, fmt.Errorf("cannot open repository: %v", err) - } - dir := repofs.Directory(r, oid) if err != nil { return repo.NullObjectID, err diff --git a/connection.go b/connection.go new file mode 100644 index 000000000..740c7a92d --- /dev/null +++ b/connection.go @@ -0,0 +1,120 @@ +package kopia + +import ( + "errors" + "fmt" + + "github.com/kopia/kopia/internal/config" + "github.com/kopia/kopia/repo" + "github.com/kopia/kopia/storage" + "github.com/kopia/kopia/storage/caching" + "github.com/kopia/kopia/storage/logging" + "github.com/kopia/kopia/vault" +) + +// Connection represents open connection to Vault and Repository. +type Connection struct { + Vault *vault.Vault + Repository *repo.Repository +} + +// ConnectionOptions specifies the behavior of Connection. +type ConnectionOptions struct { + CredentialsCallback func() (vault.Credentials, error) // credentials required to open the vault, unless persisted + + TraceStorage func(f string, args ...interface{}) + RepositoryOptions []repo.RepositoryOption +} + +// Close closes the underlying Vault and Repository. +func (c *Connection) Close() error { + if c.Vault != nil { + c.Vault.Close() + c.Vault = nil + } + + if c.Repository != nil { + c.Repository.Close() + c.Repository = nil + } + + return nil +} + +// Open connects to the Vault and Repository specified in the specified configuration file. +func Open(configFile string, options *ConnectionOptions) (*Connection, error) { + lc, err := config.LoadFromFile(configFile) + if err != nil { + return nil, err + } + + var creds vault.Credentials + if len(lc.VaultConnection.Key) > 0 { + creds, err = vault.MasterKey(lc.VaultConnection.Key) + } else { + if options.CredentialsCallback == nil { + return nil, errors.New("vault key not persisted and no credentials specified") + } + creds, err = options.CredentialsCallback() + } + + if err != nil { + return nil, fmt.Errorf("invalid vault credentials: %v", err) + } + + rawVaultStorage, err := storage.NewStorage(lc.VaultConnection.ConnectionInfo) + if err != nil { + return nil, fmt.Errorf("cannot open vault storage: %v", err) + } + + vaultStorage := rawVaultStorage + + if options.TraceStorage != nil { + vaultStorage = logging.NewWrapper(vaultStorage, logging.Prefix("[VAULT] "), logging.Output(options.TraceStorage)) + } + + var conn Connection + conn.Vault, err = vault.Open(vaultStorage, creds) + if err != nil { + rawVaultStorage.Close() + return nil, fmt.Errorf("unable to open vault: %v", err) + } + + var repositoryStorage storage.Storage + + if lc.RepoConnection == nil { + repositoryStorage = rawVaultStorage + } else { + repositoryStorage, err = storage.NewStorage(*lc.RepoConnection) + if err != nil { + vaultStorage.Close() + return nil, err + } + } + + if options.TraceStorage != nil { + repositoryStorage = logging.NewWrapper(repositoryStorage, logging.Prefix("[STORAGE] "), logging.Output(options.TraceStorage)) + } + + if lc.Caching != nil { + rs, err := caching.NewWrapper(repositoryStorage, lc.Caching) + if err != nil { + vaultStorage.Close() + repositoryStorage.Close() + return nil, err + } + repositoryStorage = rs + if options.TraceStorage != nil { + repositoryStorage = logging.NewWrapper(repositoryStorage, logging.Prefix("[CACHE] "), logging.Output(options.TraceStorage)) + } + } + + conn.Repository, err = repo.New(repositoryStorage, conn.Vault.RepoConfig.Format) + if err != nil { + vaultStorage.Close() + repositoryStorage.Close() + return nil, err + } + + return &conn, nil +} diff --git a/internal/config/local_config.go b/internal/config/local_config.go new file mode 100644 index 000000000..b543802c8 --- /dev/null +++ b/internal/config/local_config.go @@ -0,0 +1,53 @@ +// Package config stores persistent configuration of Kopia. +package config + +import ( + "encoding/json" + "io" + "os" + + "github.com/kopia/kopia/storage/caching" + "github.com/kopia/kopia/vault" + + "github.com/kopia/kopia/storage" +) + +// LocalConfig is a configuration of Kopia. +type LocalConfig struct { + VaultConnection *vault.Config `json:"vault,omitempty"` + RepoConnection *storage.ConnectionInfo `json:"repository,omitempty"` // can be nil indicating the same connection as for the vault + Caching *caching.Options `json:"caching,omitempty"` +} + +// Load reads local configuration from the specified reader. +func (lc *LocalConfig) Load(r io.Reader) error { + *lc = LocalConfig{} + return json.NewDecoder(r).Decode(lc) +} + +// Save writes the configuration to the specified writer. +func (lc *LocalConfig) Save(w io.Writer) error { + b, err := json.MarshalIndent(lc, "", " ") + if err != nil { + return nil + } + _, err = w.Write(b) + return err +} + +// LoadFromFile reads the local configuration from the specified file. +func LoadFromFile(fileName string) (*LocalConfig, error) { + f, err := os.Open(fileName) + if err != nil { + return nil, err + } + defer f.Close() + + var lc LocalConfig + + if err := lc.Load(f); err != nil { + return nil, err + } + + return &lc, nil +} diff --git a/repo/format.go b/repo/format.go index ea722c703..aa219efbf 100644 --- a/repo/format.go +++ b/repo/format.go @@ -9,6 +9,7 @@ import ( "crypto/sha512" "encoding/hex" "errors" + "fmt" "hash" ) @@ -21,6 +22,24 @@ type Format struct { MaxBlockSize int32 `json:"maxBlockSize,omitempty"` // maximum size of storage block } +// Validate checks the validity of a Format and returns an error if invalid. +func (f *Format) Validate() error { + if f.Version != 1 { + return fmt.Errorf("unsupported version: %v", f.Version) + } + + if f.MaxBlockSize < 100 { + return fmt.Errorf("MaxBlockSize is not set") + } + + sf := SupportedFormats[f.ObjectFormat] + if sf == nil { + return fmt.Errorf("unknown object format: %v", f.ObjectFormat) + } + + return nil +} + // ObjectFormatter performs data block ID computation and encryption of a block of data when storing object in a repository, type ObjectFormatter interface { // ComputeBlockIDAndKey computes ID of the storage block and encryption key for the specified block of data. diff --git a/repo/repository.go b/repo/repository.go index b5cfdcb60..60af023d2 100644 --- a/repo/repository.go +++ b/repo/repository.go @@ -155,35 +155,30 @@ func (r *Repository) Open(objectID ObjectID) (ObjectReader, error) { } // RepositoryOption controls the behavior of Repository. -type RepositoryOption func(o *Repository) error +type RepositoryOption func(o *Repository) // WriteBack is an RepositoryOption that enables asynchronous writes to the storage using the pool // of goroutines. func WriteBack(writeBackWorkers int) RepositoryOption { - return func(o *Repository) error { + return func(o *Repository) { o.writeBackWorkers = writeBackWorkers - return nil } } // EnableLogging is an RepositoryOption that causes all storage access to be logged. func EnableLogging(options ...logging.Option) RepositoryOption { - return func(o *Repository) error { + return func(o *Repository) { o.Storage = logging.NewWrapper(o.Storage, options...) - return nil } } // New creates a Repository with the specified storage, format and options. func New(s storage.Storage, f *Format, options ...RepositoryOption) (*Repository, error) { - if f.MaxBlockSize < 100 { - return nil, fmt.Errorf("MaxBlockSize is not set") + if err := f.Validate(); err != nil { + return nil, err } sf := SupportedFormats[f.ObjectFormat] - if sf == nil { - return nil, fmt.Errorf("unknown object format: %v", f.ObjectFormat) - } r := &Repository{ Storage: s, @@ -193,10 +188,7 @@ func New(s storage.Storage, f *Format, options ...RepositoryOption) (*Repository r.formatter = sf for _, o := range options { - if err := o(r); err != nil { - r.Close() - return nil, err - } + o(r) } r.bufferManager = newBufferManager(int(r.format.MaxBlockSize)) diff --git a/storage/caching/caching_storage.go b/storage/caching/caching_storage.go index e8faf1f91..038360eb4 100644 --- a/storage/caching/caching_storage.go +++ b/storage/caching/caching_storage.go @@ -199,13 +199,12 @@ func (c *cachingStorage) Close() error { // Options provides options for the caching storage wrapper type Options struct { - CacheDir string - CacheSizeBytes int64 - _ struct{} + CacheDir string `json:"cacheDir"` + CacheSizeBytes int64 `json:"cacheSizeBytes"` } // NewWrapper creates new caching storage wrapper. -func NewWrapper(master storage.Storage, options Options) (storage.Storage, error) { +func NewWrapper(master storage.Storage, options *Options) (storage.Storage, error) { if options.CacheDir == "" { return nil, fmt.Errorf("Cache directory must be specified") } diff --git a/storage/caching/caching_storage_test.go b/storage/caching/caching_storage_test.go index 48e406a04..9ed91592a 100644 --- a/storage/caching/caching_storage_test.go +++ b/storage/caching/caching_storage_test.go @@ -79,7 +79,7 @@ func TestCache(t *testing.T) { var tr tracer master = logging.NewWrapper(master, logging.Output(tr.Printf)) - cache, err := NewWrapper(master, Options{CacheDir: tmpdir}) + cache, err := NewWrapper(master, &Options{CacheDir: tmpdir}) defer cache.Close() if err != nil { t.Errorf("cannot create cache: %v", err) diff --git a/storage/config.go b/storage/config.go index 96652436e..5f7bec86c 100644 --- a/storage/config.go +++ b/storage/config.go @@ -2,6 +2,7 @@ package storage import ( "encoding/json" + "fmt" ) // ConnectionInfo represents JSON-serializable configuration of a blob storage. @@ -22,7 +23,11 @@ func (c *ConnectionInfo) UnmarshalJSON(b []byte) error { } c.Type = raw.Type - c.Config = factories[raw.Type].defaultConfigFunc() + f := factories[raw.Type] + if f == nil { + return fmt.Errorf("storage type '%v' not registered", raw.Type) + } + c.Config = f.defaultConfigFunc() if err := json.Unmarshal(raw.Data, c.Config); err != nil { return err } diff --git a/storage/gcs/gcs_storage.go b/storage/gcs/gcs_storage.go index 7254965b9..fe1b97c92 100644 --- a/storage/gcs/gcs_storage.go +++ b/storage/gcs/gcs_storage.go @@ -417,3 +417,5 @@ func init() { return New(o.(*Options)) }) } + +var _ storage.ConnectionInfoProvider = &gcsStorage{} diff --git a/vault/objectid_storage.go b/vault/objectid_storage.go index 1c99bef36..128fd0ee4 100644 --- a/vault/objectid_storage.go +++ b/vault/objectid_storage.go @@ -19,7 +19,7 @@ const ( // SaveObjectID stores the given object ID in an encrypted vault item and returns a unique ID. func (vlt *Vault) SaveObjectID(oid repo.ObjectID) (string, error) { - h := hmac.New(sha256.New, vlt.format.UniqueID) + h := hmac.New(sha256.New, vlt.Format.UniqueID) bytes, err := json.Marshal(&oid) if err != nil { return "", err diff --git a/vault/vault.go b/vault/vault.go index 2f1945e54..1c36f83e3 100644 --- a/vault/vault.go +++ b/vault/vault.go @@ -1,7 +1,6 @@ package vault import ( - "bytes" "crypto/aes" "crypto/cipher" "crypto/hmac" @@ -38,26 +37,12 @@ var ErrItemNotFound = errors.New("item not found") // Vault is a secure storage for the secrets. type Vault struct { - storage storage.Storage + Storage storage.Storage + Format Format + RepoConfig RepositoryConfig + masterKey []byte - format Format itemPrefix string - repoConfig repositoryConfig -} - -type repositoryConfig struct { - Connection *storage.ConnectionInfo `json:"connection"` - Format *repo.Format `json:"format"` -} - -// Storage returns the underlying blob storage that stores the repository. -func (v *Vault) Storage() storage.Storage { - return v.storage -} - -// Format returns the vault format. -func (v *Vault) Format() Format { - return v.format } // Put saves the specified content in a vault under a specified name. @@ -99,11 +84,11 @@ func (v *Vault) writeEncryptedBlock(itemID string, content []byte) error { content = cipherText } - return v.storage.PutBlock(v.itemPrefix+itemID, content, storage.PutOptionsOverwrite) + return v.Storage.PutBlock(v.itemPrefix+itemID, content, storage.PutOptionsOverwrite) } func (v *Vault) readEncryptedBlock(itemID string) ([]byte, error) { - content, err := v.storage.GetBlock(v.itemPrefix + itemID) + content, err := v.Storage.GetBlock(v.itemPrefix + itemID) if err != nil { if err == storage.ErrBlockNotFound { return nil, ErrItemNotFound @@ -150,20 +135,20 @@ func (v *Vault) decryptBlock(content []byte) ([]byte, error) { } func (v *Vault) newChecksum() (hash.Hash, error) { - switch v.format.Checksum { + switch v.Format.Checksum { case "hmac-sha-256": key := make([]byte, 32) v.deriveKey(purposeChecksumSecret, key) return hmac.New(sha256.New, key), nil default: - return nil, fmt.Errorf("unsupported checksum format: %v", v.format.Checksum) + return nil, fmt.Errorf("unsupported checksum format: %v", v.Format.Checksum) } } func (v *Vault) newCipher() (cipher.Block, error) { - switch v.format.Encryption { + switch v.Format.Encryption { case "none": return nil, nil case "aes-128": @@ -179,39 +164,17 @@ func (v *Vault) newCipher() (cipher.Block, error) { v.deriveKey(purposeAESKey, k) return aes.NewCipher(k) default: - return nil, fmt.Errorf("unsupported encryption format: %v", v.format.Encryption) + return nil, fmt.Errorf("unsupported encryption format: %v", v.Format.Encryption) } } func (v *Vault) deriveKey(purpose []byte, key []byte) error { - k := hkdf.New(sha256.New, v.masterKey, v.format.UniqueID, purpose) + k := hkdf.New(sha256.New, v.masterKey, v.Format.UniqueID, purpose) _, err := io.ReadFull(k, key) return err } -// RepositoryFormat returns the format of the repository. -func (v *Vault) RepositoryFormat() *repo.Format { - return v.repoConfig.Format -} - -// OpenRepository connects to the repository the vault is associated with. -func (v *Vault) OpenRepository(options ...repo.RepositoryOption) (*repo.Repository, error) { - var st storage.Storage - var err error - - if v.repoConfig.Connection != nil { - st, err = storage.NewStorage(*v.repoConfig.Connection) - if err != nil { - return nil, fmt.Errorf("unable to open repository: %v", err) - } - } else { - st = v.storage - } - - return repo.New(st, v.repoConfig.Format, options...) -} - // Get returns the contents of a specified vault item. func (v *Vault) Get(itemID string) ([]byte, error) { if err := checkReservedName(itemID); err != nil { @@ -244,7 +207,7 @@ func (v *Vault) putJSON(id string, content interface{}) error { func (v *Vault) List(prefix string) ([]string, error) { var result []string - for b := range v.storage.ListBlocks(v.itemPrefix + prefix) { + for b := range v.Storage.ListBlocks(v.itemPrefix + prefix) { if b.Error != nil { return result, b.Error } @@ -259,7 +222,7 @@ func (v *Vault) List(prefix string) ([]string, error) { // Close releases any resources held by Vault and closes repository connection. func (v *Vault) Close() error { - return nil + return v.Storage.Close() } // Config represents JSON-compatible configuration of the vault connection, including vault key. @@ -271,7 +234,7 @@ type Config struct { // Config returns a configuration of vault storage its credentials that's suitable // for storing in configuration file. func (v *Vault) Config() (*Config, error) { - cip, ok := v.storage.(storage.ConnectionInfoProvider) + cip, ok := v.Storage.(storage.ConnectionInfoProvider) if !ok { return nil, errors.New("repository does not support persisting configuration") } @@ -290,7 +253,7 @@ func (v *Vault) Remove(itemID string) error { return err } - return v.storage.DeleteBlock(v.itemPrefix + itemID) + return v.Storage.DeleteBlock(v.itemPrefix + itemID) } // Create initializes a Vault attached to the specified repository. @@ -302,11 +265,11 @@ func Create( repoFormat *repo.Format, ) (*Vault, error) { v := Vault{ - storage: vaultStorage, - format: *vaultFormat, + Storage: vaultStorage, + Format: *vaultFormat, } - if repoStorage == nil || sameStorage(repoStorage, vaultStorage) { + if repoStorage == nil { repoStorage = vaultStorage v.itemPrefix = colocatedVaultItemPrefix } @@ -316,14 +279,14 @@ func Create( return nil, errors.New("repository does not support persisting configuration") } - v.format.Version = "1" - v.format.UniqueID = make([]byte, 32) - if _, err := io.ReadFull(rand.Reader, v.format.UniqueID); err != nil { + v.Format.Version = "1" + v.Format.UniqueID = make([]byte, 32) + if _, err := io.ReadFull(rand.Reader, v.Format.UniqueID); err != nil { return nil, err } - v.masterKey = vaultCreds.getMasterKey(v.format.UniqueID) + v.masterKey = vaultCreds.getMasterKey(v.Format.UniqueID) - formatBytes, err := json.Marshal(&v.format) + formatBytes, err := json.Marshal(&v.Format) if err != nil { return nil, err } @@ -333,7 +296,7 @@ func Create( } // Write encrypted repository configuration block. - rc := repositoryConfig{ + rc := RepositoryConfig{ Format: repoFormat, } @@ -346,14 +309,30 @@ func Create( return nil, err } - v.repoConfig = rc + v.RepoConfig = rc return &v, nil } +// CreateColocated initializes a Vault attached to a Repository sharing the same storage. +func CreateColocated( + sharedStorage storage.Storage, + vaultFormat *Format, + vaultCreds Credentials, + repoFormat *repo.Format, +) (*Vault, error) { + return Create(sharedStorage, vaultFormat, vaultCreds, nil, repoFormat) +} + +// RepositoryConfig stores the configuration of the repository associated with the vault. +type RepositoryConfig struct { + Connection *storage.ConnectionInfo `json:"connection"` + Format *repo.Format `json:"format"` +} + // Open opens a vault. func Open(vaultStorage storage.Storage, vaultCreds Credentials) (*Vault, error) { v := Vault{ - storage: vaultStorage, + Storage: vaultStorage, } var prefix string @@ -383,12 +362,12 @@ func Open(vaultStorage storage.Storage, vaultCreds Credentials) (*Vault, error) offset = 2 } - err := json.Unmarshal(blocks[offset], &v.format) + err := json.Unmarshal(blocks[offset], &v.Format) if err != nil { return nil, err } - v.masterKey = vaultCreds.getMasterKey(v.format.UniqueID) + v.masterKey = vaultCreds.getMasterKey(v.Format.UniqueID) v.itemPrefix = prefix cfgData, err := v.decryptBlock(blocks[offset+1]) @@ -396,13 +375,13 @@ func Open(vaultStorage storage.Storage, vaultCreds Credentials) (*Vault, error) return nil, err } - var rc repositoryConfig + var rc RepositoryConfig if err := json.Unmarshal(cfgData, &rc); err != nil { return nil, err } - v.repoConfig = rc + v.RepoConfig = rc return &v, nil } @@ -424,28 +403,3 @@ func checkReservedName(itemID string) error { return nil } - -func sameStorage(s1, s2 storage.Storage) bool { - if s1 == s2 { - return true - } - - cip1, ok := s1.(storage.ConnectionInfoProvider) - if !ok { - return false - } - - cip2, ok := s2.(storage.ConnectionInfoProvider) - if !ok { - return false - } - - c1 := cip1.ConnectionInfo() - c2 := cip2.ConnectionInfo() - - b1, e1 := json.Marshal(c1) - b2, e2 := json.Marshal(c2) - - same := e1 == nil && e2 == nil && bytes.Equal(b1, b2) - return same -} diff --git a/vault/vault_test.go b/vault/vault_test.go index e45f9d7ea..63974ae50 100644 --- a/vault/vault_test.go +++ b/vault/vault_test.go @@ -68,7 +68,7 @@ func verifyVault(t *testing.T, vaultPath string, repoPath string) { Secret: []byte{1, 2, 3}, } - _, err = repo.New(repoStorage, repoFormat) + r, err := repo.New(repoStorage, repoFormat) if err != nil { t.Errorf("can't create repository: %v", err) return @@ -116,24 +116,14 @@ func verifyVault(t *testing.T, vaultPath string, repoPath string) { t.Errorf("error putting: %v", err) } - r1, err := v1.OpenRepository() - if err != nil { - t.Errorf("error opening v1 repository: %v", err) - } - - r2, err := v2.OpenRepository() - if err != nil { - t.Errorf("error opening v1 repository: %v", err) - } - - w1 := r1.NewWriter() + w1 := r.NewWriter() w1.Write([]byte("foo")) oid1, err := w1.Result(true) if err != nil { t.Errorf("Result error: %v", err) } - w2 := r2.NewWriter() + w2 := r.NewWriter() w2.Write([]byte("bar")) oid2, err := w2.Result(true) if err != nil { @@ -161,7 +151,7 @@ func verifyVault(t *testing.T, vaultPath string, repoPath string) { // Verify contents of vault items for both created and opened vault. for _, v := range []*Vault{v1, v2} { - rf := v.RepositoryFormat() + rf := v.RepoConfig.Format if !reflect.DeepEqual(rf, repoFormat) { t.Errorf("invalid repository format: %v, but got %v", repoFormat, rf) } @@ -204,12 +194,6 @@ func verifyVault(t *testing.T, vaultPath string, repoPath string) { } } - // Make sure repository is shared, by checking that data written in one can be read by the other. - assertRepositoryItem(t, r1, oid1, "foo") - assertRepositoryItem(t, r1, oid2, "bar") - assertRepositoryItem(t, r2, oid1, "foo") - assertRepositoryItem(t, r2, oid2, "bar") - v1.Remove("bar") for _, v := range []*Vault{v1, v2} { @@ -245,26 +229,6 @@ func assertVaultItem(t *testing.T, v *Vault, itemID string, expectedData string) } } -func assertRepositoryItem(t *testing.T, repository *repo.Repository, oid repo.ObjectID, expectedData string) { - r, err := repository.Open(oid) - if err != nil { - t.Errorf("error opening item %v: %v", oid, err) - return - } - defer r.Close() - - b, err := ioutil.ReadAll(r) - if err != nil { - t.Errorf("error reading item %v: %v", oid, err) - return - } - - bs := string(b) - if bs != expectedData { - t.Errorf("invalid data for '%v': expected: %v but got %v", oid, expectedData, bs) - } -} - func assertVaultItemNotFound(t *testing.T, v *Vault, itemID string) { result, err := v.Get(itemID) if err != ErrItemNotFound {