From 270df7303ffed504119158eca6a46c2b83232fe2 Mon Sep 17 00:00:00 2001 From: Jarek Kowalski Date: Sun, 8 May 2016 21:02:14 -0700 Subject: [PATCH] work in progress revamping CLI and adding vault --- blob/filesystem.go | 20 ++- blob/registry.go | 6 +- blob/storage_test.go | 12 +- cas/format.go | 16 +++ cas/repository.go | 2 +- cmd/kopia/command_create.go | 115 +++++++++++++++++ cmd/kopia/command_init.go | 25 ++-- cmd/kopia/command_status.go | 43 +++++++ cmd/kopia/config.go | 245 +++++++++++++++++++++++++++-------- doc/quickstart.md | 57 +++++++++ doc/tutorial.md | 71 ++++++++++ session/session.go | 15 +-- session/session_test.go | 51 -------- vault/credentials.go | 88 ------------- vault/credentials_test.go | 31 ----- vault/format.go | 48 +++++++ vault/vault.go | 249 +++++++++++++++++++++++++++++++++--- 17 files changed, 805 insertions(+), 289 deletions(-) create mode 100644 cmd/kopia/command_create.go create mode 100644 cmd/kopia/command_status.go create mode 100644 doc/quickstart.md create mode 100644 doc/tutorial.md delete mode 100644 session/session_test.go delete mode 100644 vault/credentials.go delete mode 100644 vault/credentials_test.go create mode 100644 vault/format.go diff --git a/blob/filesystem.go b/blob/filesystem.go index fa5bd3ab5..10ae179f0 100644 --- a/blob/filesystem.go +++ b/blob/filesystem.go @@ -13,7 +13,7 @@ ) const ( - fsStorageType = "file" + fsStorageType = "filesystem" fsStorageChunkSuffix = ".f" ) @@ -31,10 +31,10 @@ type fsStorage struct { type FSStorageOptions struct { Path string `json:"path"` - DirectoryShards []int `json:"dirShards"` + DirectoryShards []int `json:"dirShards,omitempty"` - FileMode os.FileMode `json:"fileMode"` - DirectoryMode os.FileMode `json:"dirMode"` + FileMode os.FileMode `json:"fileMode,omitempty"` + DirectoryMode os.FileMode `json:"dirMode,omitempty"` FileUID *int `json:"uid,omitempty"` FileGID *int `json:"gid,omitempty"` @@ -66,10 +66,12 @@ func (fso *FSStorageOptions) shards() []int { // ParseURL parses the given URL into FSStorageOptions. func (fso *FSStorageOptions) ParseURL(u *url.URL) error { - if u.Scheme != "file" { + if u.Scheme != fsStorageType { return fmt.Errorf("invalid scheme, expected 'file'") } + //log.Printf("u.Upaque: %v u.Path: %v", u.Opaque, u.Path) + if u.Opaque != "" { fso.Path = u.Opaque } else { @@ -97,12 +99,8 @@ func (fso *FSStorageOptions) ParseURL(u *url.URL) error { // ToURL converts the FSStorageOptions to URL. func (fso *FSStorageOptions) ToURL() *url.URL { u := &url.URL{} - u.Scheme = "file" - if fso.Path[0] == '/' { - u.Path = fso.Path - } else { - u.Opaque = fso.Path - } + u.Scheme = "filesystem" + u.Opaque = fso.Path q := u.Query() if fso.FileUID != nil { q.Add("uid", strconv.Itoa(*fso.FileUID)) diff --git a/blob/registry.go b/blob/registry.go index 24ead6413..26ae5127b 100644 --- a/blob/registry.go +++ b/blob/registry.go @@ -40,7 +40,11 @@ func NewStorage(cfg StorageConfiguration) (Storage, error) { // NewStorageFromURL creates new storage based on a URL. // The storage type must be previously registered using AddSupportedStorage. -func NewStorageFromURL(u *url.URL) (Storage, error) { +func NewStorageFromURL(storageURL string) (Storage, error) { + u, err := url.Parse(storageURL) + if err != nil { + return nil, err + } if factory, ok := factories[u.Scheme]; ok { o := factory.defaultConfigFunc() if err := o.ParseURL(u); err != nil { diff --git a/blob/storage_test.go b/blob/storage_test.go index 611db86cc..9e9149813 100644 --- a/blob/storage_test.go +++ b/blob/storage_test.go @@ -58,12 +58,12 @@ func TestFileStorageOptions(t *testing.T) { o FSStorageOptions url string }{ - {FSStorageOptions{Path: "/blah"}, "file:///blah"}, - {FSStorageOptions{Path: "/blah", FileMode: 0123}, "file:///blah?filemode=123"}, - {FSStorageOptions{Path: "/blah", DirectoryMode: 0123}, "file:///blah?dirmode=123"}, - {FSStorageOptions{Path: "/blah", FileMode: 0432, DirectoryMode: 0123}, "file:///blah?dirmode=123&filemode=432"}, - {FSStorageOptions{Path: "/blah", FileMode: 0432, DirectoryMode: 0123, DirectoryShards: []int{1, 2, 3}}, "file:///blah?dirmode=123&filemode=432&shards=1.2.3"}, - {FSStorageOptions{Path: "c:\\winpath"}, "file:c:\\winpath"}, + {FSStorageOptions{Path: "/blah"}, "filesystem:/blah"}, + {FSStorageOptions{Path: "/blah", FileMode: 0123}, "filesystem:/blah?filemode=123"}, + {FSStorageOptions{Path: "/blah", DirectoryMode: 0123}, "filesystem:/blah?dirmode=123"}, + {FSStorageOptions{Path: "/blah", FileMode: 0432, DirectoryMode: 0123}, "filesystem:/blah?dirmode=123&filemode=432"}, + {FSStorageOptions{Path: "/blah", FileMode: 0432, DirectoryMode: 0123, DirectoryShards: []int{1, 2, 3}}, "filesystem:/blah?dirmode=123&filemode=432&shards=1.2.3"}, + {FSStorageOptions{Path: "c:\\winpath"}, "filesystem:c:\\winpath"}, } for i, c := range cases { diff --git a/cas/format.go b/cas/format.go index 0017305a8..8fc08b1b1 100644 --- a/cas/format.go +++ b/cas/format.go @@ -4,10 +4,12 @@ "crypto/aes" "crypto/cipher" "crypto/md5" + "crypto/rand" "crypto/sha1" "crypto/sha256" "crypto/sha512" "hash" + "io" ) // Format describes the format of object data. @@ -19,6 +21,20 @@ type Format struct { MaxBlobSize int `json:"maxBlobSize"` } +func NewFormat() (*Format, error) { + f := &Format{ + Version: "1", + Secret: make([]byte, 32), + ObjectFormat: "hmac-sha256", + } + + _, err := io.ReadFull(rand.Reader, f.Secret) + if err != nil { + return f, err + } + return f, nil +} + // ObjectIDFormat describes single format ObjectID type ObjectIDFormat struct { Name string diff --git a/cas/repository.go b/cas/repository.go index 69050b41a..e34575205 100644 --- a/cas/repository.go +++ b/cas/repository.go @@ -191,7 +191,7 @@ func hmacFunc(key []byte, hf func() hash.Hash) func() hash.Hash { // NewRepository creates new Repository with the specified storage, options, and key provider. func NewRepository( r blob.Storage, - f Format, + f *Format, options ...RepositoryOption, ) (Repository, error) { if f.Version != "1" { diff --git a/cmd/kopia/command_create.go b/cmd/kopia/command_create.go new file mode 100644 index 000000000..4ff38ab6a --- /dev/null +++ b/cmd/kopia/command_create.go @@ -0,0 +1,115 @@ +package main + +import ( + "fmt" + + "github.com/kopia/kopia/cas" + "github.com/kopia/kopia/vault" + + "github.com/kopia/kopia/blob" + "gopkg.in/alecthomas/kingpin.v2" +) + +var ( + createCommand = app.Command("create", "Create new vault and optionally connect to it") + createCommandRepository = createCommand.Flag("repository", "Repository path").Required().String() + createCommandOnly = createCommand.Flag("only", "Only create, don't connect.").Bool() + + createMaxBlobSize = createCommand.Flag("max-blob-size", "Maximum size of a data chunk in bytes.").Default("4000000").Int() + createInlineBlobSize = createCommand.Flag("inline-blob-size", "Maximum size of an inline data chunk in bytes.").Default("32768").Int() + + createVaultEncryptionFormat = createCommand.Flag("vaultencryption", "Vault encryption format").String() +) + +func init() { + createCommand.Action(runCreateCommand) +} + +func vaultFormat() *vault.Format { + f := vault.NewFormat() + if *createVaultEncryptionFormat != "" { + f.Encryption = *createVaultEncryptionFormat + } + return f +} + +func repositoryFormat() (*cas.Format, error) { + f, err := cas.NewFormat() + if err != nil { + return nil, err + } + + return f, nil +} + +func openStorageAndEnsureEmpty(url string) (blob.Storage, error) { + s, err := blob.NewStorageFromURL(url) + if err != nil { + return nil, err + } + ch := s.ListBlocks("") + _, hasData := <-ch + + if hasData { + return nil, fmt.Errorf("found existing data in %v", url) + } + + return s, nil + +} + +func runCreateCommand(context *kingpin.ParseContext) error { + vaultStorage, err := openStorageAndEnsureEmpty(*vaultPath) + if err != nil { + return fmt.Errorf("unable to get vault storage: %v", err) + return err + } + + repositoryStorage, err := openStorageAndEnsureEmpty(*createCommandRepository) + if err != nil { + return fmt.Errorf("unable to get repository storage: %v", err) + } + + masterKey, password, err := getKeyOrPassword(true) + if err != nil { + return fmt.Errorf("unable to get credentials: %v", err) + } + + var v *vault.Vault + + if masterKey != nil { + v, err = vault.CreateWithKey(vaultStorage, vaultFormat(), masterKey) + } else { + v, err = vault.CreateWithPassword(vaultStorage, vaultFormat(), password) + } + if err != nil { + return fmt.Errorf("cannot create vault: %v", err) + } + + repoFormat, err := repositoryFormat() + if err != nil { + return fmt.Errorf("unable to initialize repository format: %v", err) + } + + // Make repository to make sure the format is supported. + _, err = cas.NewRepository(repositoryStorage, repoFormat) + if err != nil { + return fmt.Errorf("unable to initialize repository: %v", err) + } + + v.SetRepository(vault.RepositoryConfig{ + Storage: repositoryStorage.Configuration(), + Repository: repoFormat, + }) + + if *createCommandOnly { + fmt.Println("Created vault:", *vaultPath) + return nil + } + + persistVaultConfig(v) + + fmt.Println("Created and connected to vault:", *vaultPath) + + return err +} diff --git a/cmd/kopia/command_init.go b/cmd/kopia/command_init.go index 420b3f91d..20b38b2be 100644 --- a/cmd/kopia/command_init.go +++ b/cmd/kopia/command_init.go @@ -4,14 +4,11 @@ "crypto/rand" "fmt" "io" - "os" "strings" "github.com/kopia/kopia/blob" "github.com/kopia/kopia/cas" - "github.com/kopia/kopia/config" "github.com/kopia/kopia/session" - "github.com/kopia/kopia/vault" ) var ( @@ -28,9 +25,7 @@ ) func runInitCommandForRepository(s blob.Storage, defaultSalt string) error { - var creds vault.Credentials - - sess, err := session.New(s, creds) + sess, err := session.New(s) if err != nil { return err } @@ -80,16 +75,16 @@ func runInitCommandForRepository(s blob.Storage, defaultSalt string) error { return err } - cfg := config.Config{ - Storage: s.Configuration(), - } + // cfg := config.Config{ + // Storage: s.Configuration(), + // } - f, err := os.Create(configFileName()) - if err != nil { - return err - } - defer f.Close() - cfg.SaveTo(f) + // f, err := os.Create(configFileName()) + // if err != nil { + // return err + // } + // defer f.Close() + // cfg.SaveTo(f) return nil } diff --git a/cmd/kopia/command_status.go b/cmd/kopia/command_status.go new file mode 100644 index 000000000..82c0cbd4c --- /dev/null +++ b/cmd/kopia/command_status.go @@ -0,0 +1,43 @@ +package main + +import ( + "encoding/hex" + "fmt" + + "gopkg.in/alecthomas/kingpin.v2" +) + +var ( + statusCommand = app.Command("status", "Display status information.") +) + +func init() { + statusCommand.Action(runRepositoryInfoCommand) +} + +func runRepositoryInfoCommand(context *kingpin.ParseContext) error { + v, err := openVault() + if err != nil { + return err + } + + fmt.Println("Vault:") + fmt.Println(" Address: ", v.Storage.Configuration().Config.ToURL()) + fmt.Println(" ID: ", hex.EncodeToString(v.Format.UniqueID)) + fmt.Println(" Encryption:", v.Format.Encryption) + fmt.Println(" Checksum: ", v.Format.Checksum) + fmt.Println(" Master Key:", hex.EncodeToString(v.MasterKey)) + + vc, err := v.Repository() + if err != nil { + return err + } + + fmt.Println("Repository:") + fmt.Println(" Address: ", vc.Storage.Config.ToURL()) + fmt.Println(" Version: ", vc.Repository.Version) + fmt.Println(" Secret: ", len(vc.Repository.Secret), "bytes") + fmt.Println(" ID Format:", vc.Repository.ObjectFormat) + + return nil +} diff --git a/cmd/kopia/config.go b/cmd/kopia/config.go index d3c89eaa4..bc9fdf685 100644 --- a/cmd/kopia/config.go +++ b/cmd/kopia/config.go @@ -1,20 +1,30 @@ package main import ( + "encoding/hex" + "encoding/json" "fmt" + "io/ioutil" "os" - "os/user" "path/filepath" + "strings" "github.com/kopia/kopia/blob" - "github.com/kopia/kopia/config" + + "golang.org/x/crypto/ssh/terminal" + "github.com/kopia/kopia/session" "github.com/kopia/kopia/vault" ) var ( - configFile = app.Flag("config", "Specify config filename.").Default(getDefaultConfigFileName()).String() traceStorage = app.Flag("trace-storage", "Enables tracing of storage operations.").Bool() + + vaultPath = app.Flag("vault", "Specify the vault to use.").String() + password = app.Flag("password", "Vault password").String() + passwordFile = app.Flag("passwordfile", "Read password from a file").ExistingFile() + key = app.Flag("key", "Vault key").String() + keyFile = app.Flag("keyfile", "Read vault key from a file").ExistingFile() ) func failOnError(err error) { @@ -30,65 +40,192 @@ func mustOpenSession() session.Session { return s } -func configFileName() string { - if *configFile != "" { - return *configFile - } - - return getDefaultConfigFileName() -} - -func getDefaultConfigFileName() string { - u, err := user.Current() - if err == nil && u.Uid == "0" { - return "/etc/kopia/config.json" - } - return filepath.Join(getHomeDir(), ".kopia/config.json") -} - func getHomeDir() string { return os.Getenv("HOME") } -func loadConfig() (*config.Config, error) { - path := configFileName() - if path == "" { - return nil, fmt.Errorf("Cannot find config file. You may pass --config_file to specify config file location.") - } - - var cfg config.Config - - //log.Printf("Loading config file from %v", path) - f, err := os.Open(path) - if err != nil { - return nil, fmt.Errorf("Error opening configuration file: %v", err) - } - defer f.Close() - - err = cfg.Load(f) - if err == nil { - return &cfg, nil - } - - return nil, fmt.Errorf("Error loading configuration file: %v", err) +func vaultConfigFileName() string { + return filepath.Join(getHomeDir(), ".kopia/vault.config") } +// func loadConfig() (*config.Config, error) { +// path := configFileName() +// if path == "" { +// return nil, fmt.Errorf("Cannot find config file. You may pass --config_file to specify config file location.") +// } + +// var cfg config.Config + +// //log.Printf("Loading config file from %v", path) +// f, err := os.Open(path) +// if err != nil { +// return nil, fmt.Errorf("Error opening configuration file: %v", err) +// } +// defer f.Close() + +// err = cfg.Load(f) +// if err == nil { +// return &cfg, nil +// } + +// return nil, fmt.Errorf("Error loading configuration file: %v", err) +// } + func openSession() (session.Session, error) { - cfg, err := loadConfig() - if err != nil { - return nil, err - } + return nil, nil + // cfg, err := loadConfig() + // if err != nil { + // return nil, err + // } - storage, err := blob.NewStorage(cfg.Storage) - if err != nil { - return nil, err - } + // storage, err := blob.NewStorage(cfg.Storage) + // if err != nil { + // return nil, err + // } - if *traceStorage { - storage = blob.NewLoggingWrapper(storage) - } + // if *traceStorage { + // storage = blob.NewLoggingWrapper(storage) + // } - var creds vault.Credentials - - return session.New(storage, creds) + // return session.New(storage) +} + +type vaultConfig struct { + Storage blob.StorageConfiguration `json:"storage"` + Key []byte `json:"key,omitempty"` +} + +func persistVaultConfig(v *vault.Vault) error { + vc := vaultConfig{ + Storage: v.Storage.Configuration(), + Key: v.MasterKey, + } + + f, err := os.Create(vaultConfigFileName()) + if err != nil { + return err + } + defer f.Close() + json.NewEncoder(f).Encode(vc) + return nil +} + +func getPersistedVaultConfig() *vaultConfig { + var vc vaultConfig + + f, err := os.Open(vaultConfigFileName()) + if err == nil { + err = json.NewDecoder(f).Decode(&vc) + f.Close() + if err != nil { + return nil + } + return &vc + } + + return nil +} + +func openVault() (*vault.Vault, error) { + vc := getPersistedVaultConfig() + if vc != nil { + storage, err := blob.NewStorage(vc.Storage) + if err != nil { + return nil, err + } + + return vault.OpenWithKey(storage, vc.Key) + } + + if *vaultPath == "" { + return nil, fmt.Errorf("vault not connected, use --vault") + } + + storage, err := blob.NewStorageFromURL(*vaultPath) + if err != nil { + return nil, err + } + + masterKey, password, err := getKeyOrPassword(false) + if err != nil { + return nil, err + } + + if masterKey != nil { + return vault.OpenWithKey(storage, masterKey) + } else { + return vault.OpenWithPassword(storage, password) + } +} + +func getKeyOrPassword(isNew bool) ([]byte, string, error) { + if *key != "" { + k, err := hex.DecodeString(*key) + if err != nil { + return nil, "", fmt.Errorf("invalid key format: %v", err) + } + + return k, "", nil + } + + if *password != "" { + return nil, strings.TrimSpace(*password), nil + } + + if *keyFile != "" { + key, err := ioutil.ReadFile(*keyFile) + if err != nil { + return nil, "", fmt.Errorf("unable to read key file: %v", err) + } + + return key, "", nil + } + + if *passwordFile != "" { + f, err := ioutil.ReadFile(*passwordFile) + if err != nil { + return nil, "", fmt.Errorf("unable to read password file: %v", err) + } + + return nil, strings.TrimSpace(string(f)), nil + } + + if isNew { + for { + fmt.Printf("Enter password: ") + p1, err := askPass() + if err != nil { + return nil, "", err + } + fmt.Println() + fmt.Printf("Enter password again: ") + p2, err := askPass() + if err != nil { + return nil, "", err + } + fmt.Println() + if p1 != p2 { + fmt.Println("Passwords don't match!") + } else { + return nil, p1, nil + } + } + } else { + fmt.Printf("Enter password: ") + p1, err := askPass() + if err != nil { + return nil, "", err + } + fmt.Println() + return nil, p1, nil + } +} + +func askPass() (string, error) { + b, err := terminal.ReadPassword(0) + if err != nil { + return "", err + } + + return string(b), nil } diff --git a/doc/quickstart.md b/doc/quickstart.md new file mode 100644 index 000000000..35236fb2f --- /dev/null +++ b/doc/quickstart.md @@ -0,0 +1,57 @@ +Quick Start +=== + +Kopia is a simple tool for managing encrypted backups in the cloud. + +Key Concepts +--- + +* **Repository** is a [Content-Addressable Store](https://en.wikipedia.org/wiki/Content-addressable_storage) of files and directories (known as Objects) identified by their **Object IDs**. + + - Object ID is comprised of the type, the identifier of the block (Block ID) where the contents are stored and the encryption key. Example block ID: + + ``` + DD7ce4f067c179664...e746337031644a:715b50351785a6...439637a2c8e50c7 + |\------------------------------/ \------------------------------/ + type block id encryption key + ``` + + - Block IDs are derived from the contents of objects by using cryptographic hash, typically as one of [SHA-2](https://en.wikipedia.org/wiki/SHA-2) hash functions. + - Encryption key is also derived from the contents of the object - this technique is known as [Convergent Encryption](https://en.wikipedia.org/wiki/Convergent_encryption) + - Identical objects will have the same ID and thus will be stored only once with the same encryption key. + - A person with access to the object can easily compute its Object ID (including encryption key), but the knowledge of block id is not enough to be able to retrieve the content. + - Repository can be shared among many users as long as they all can compute the same object IDs + +* **Vault** securely stores Object IDs, encrypted with user-specific password or passphrase +* **Blob Storage** stores unstructured, blocks of binary large objects (BLOBs). + Supported backends include: + + - [Google Cloud Storage](https://cloud.google.com/storage/) (GCS) + - [Amazon Simple Storage Service](https://aws.amazon.com/s3/) (S3) + - Local or remote filesystem + +Object IDs +--- + +There are three types of object IDs: + +* **Data** or **Direct** object where the entire object is stored in a single block +* **List** objects, that store list of object IDs +* **Inline Text** and **Inline Binary** objects, that represent the contents of the very short files directly in the object IDs, encoded either as text or base64 + +Some examples (note that block IDs and encryption keys have been shortened for illustration purposes): + +* `D7ce4f067c:715b503c8` - is a *Data* object stored in block `7ce4f067c` and encrypted with key `715b503c8` + +* `L43637a2c8:715b50351` - is a *List* object stored in block `43637a2c8` and encrypted with key `715b50351` + +* `Tquick brown fox` - is an *Inline Text* object representing the text `quick brown fox` + +* `BAQIDBA` - is an *Inline Binary* object representing base-64 encoded bytes: `01 02 03 04` + +Object Types +--- + +* **File** contents are store as binary +* **Directory** contents are stored as JSON-encoded objects storing file or subdirectory metadata and Object IDs of file/subdirectory contents. + diff --git a/doc/tutorial.md b/doc/tutorial.md new file mode 100644 index 000000000..fdc9ce07c --- /dev/null +++ b/doc/tutorial.md @@ -0,0 +1,71 @@ +Tutorial +=== + +Kopia is a simple, cross-platform tool for managing encrypted backups in the cloud. It provides fast incremental backups, data deduplication and client-side encryption. + +The main difference from other backup solutions is that as a user, you are in full control of backup storage - in fact you are responsible for purchasing one of the cloud storage products available (such as Google Cloud Storage), which offer great durability and availability for your data. + +## Installation + +### Binary Releases + +You can download pre-built `kopia` binary from http://kopia.github.io/download. Once downloaded, it's best to put it in a directory that's in system PATH, such as `/usr/local/bin`. + +### Installation From Source + +To build Kopia from source you need to have the latest version of [Go](https://golang.org/dl/) installed and run the following commands: + +``` +mkdir $HOME/kopia +export GOPATH=$HOME/kopia +go get github.com/kopia/kopia +go install github.com/kopia/kopia/cmd/kopia +``` + +This will automatically download and build kopia and put the resulting binary in `$HOME/kopia/bin`. For convenience it's best to add this directory to system `PATH` or copy it to a directory already in the path, such as `/usr/local/bin`. + +## Getting Started + +To use Kopia, you need to set up two storage locations: + +- **Repository** - which will store the bulk of encrypted data +- **Vault** - which stores the backup metadata and their encryption keys + +The **Repository** can be shared between multiple computers or users because without the encryption keys stored in the **Vault** its data is unaccessible. + +The metadata stored in the *Vault* is very small and will typically fit on even smallest of USB drives for safekeeping. You have a choice of using Vault in the cloud or keeping it on a removable storage device in your possession. + +### Setting Up Vault + +To create new vault in Google Cloud Storage, first create the bucket bucket using https://console.cloud.google.com/storage/ then run: + +``` +kopia vault create gcs BUCKET +``` + +To create new vault in the local filesystem, use: + +``` +mkdir /path/to/vault +kopia vault create filesystem /path/to/vault +``` + +You will be prompted for the password to protect the data in the vault. **Don't forget your password, as there is absolutely no way to recover data in the vault if you do so.** + +To later connect to an existing vault, simply replace `vault create` with `vault connect`. + +To disconnect from a vault, run: +``` +kopia vault disconnect +``` + +### Setting Up Repository + +After creating the vault, we now need to create a repository. This is very similar way to creating a vault, just replace `vault` with `repository`. For example to create GCS-backed repository, use: + +``` +kopia repository create gcs BUCKET +``` + + + diff --git a/session/session.go b/session/session.go index 83480a66e..e584487cb 100644 --- a/session/session.go +++ b/session/session.go @@ -10,7 +10,6 @@ "github.com/kopia/kopia/blob" "github.com/kopia/kopia/cas" - "github.com/kopia/kopia/vault" ) // ErrConfigNotFound indicates that configuration is not found. @@ -25,7 +24,6 @@ type Session interface { type session struct { storage blob.Storage - creds vault.Credentials format cas.Format } @@ -52,15 +50,11 @@ func (s *session) encryptBlockWithPublicKey(blkID string, data io.ReadCloser, op } func (s *session) getConfigstring() string { - if s.creds == nil { - return string("config.json") - } - - return string("users." + s.creds.Username() + ".config.json") + return string("config.json") } func (s *session) InitRepository(format cas.Format) (cas.Repository, error) { - mgr, err := cas.NewRepository(s.storage, format) + mgr, err := cas.NewRepository(s.storage, &format) if err != nil { return nil, err } @@ -95,13 +89,12 @@ func (s *session) OpenRepository() (cas.Repository, error) { return nil, err } - return cas.NewRepository(s.storage, format) + return cas.NewRepository(s.storage, &format) } -func New(storage blob.Storage, creds vault.Credentials) (Session, error) { +func New(storage blob.Storage) (Session, error) { sess := &session{ storage: storage, - creds: creds, } return sess, nil } diff --git a/session/session_test.go b/session/session_test.go deleted file mode 100644 index d4ec71a65..000000000 --- a/session/session_test.go +++ /dev/null @@ -1,51 +0,0 @@ -package session - -import ( - "io/ioutil" - - "github.com/kopia/kopia/cas" - - "github.com/kopia/kopia/blob" - - "testing" -) - -func TestA(t *testing.T) { - tmpDir, err := ioutil.TempDir("", "kopia") - if err != nil { - t.Errorf("can't create temp directory: %v", err) - return - } - - // cfg := LoadConfig("kopia.config") - sc := blob.StorageConfiguration{ - Type: "file", - Config: &blob.FSStorageOptions{ - Path: tmpDir, - }, - } - - storage, err := blob.NewStorage(sc) - if err != nil { - t.Errorf("cannot create storage: %v", err) - return - } - - sess, err := New(storage, nil) - defer sess.Close() - - om, err := sess.InitRepository(cas.Format{ - Version: "1", - ObjectFormat: "sha1", - }) - - if err != nil { - t.Errorf("unable to init object manager: %v", err) - return - } - - w := om.NewWriter() - w.Write([]byte{1, 2, 3}) - x, err := w.Result(true) - t.Logf("%v x: %v %v", tmpDir, x, err) -} diff --git a/vault/credentials.go b/vault/credentials.go deleted file mode 100644 index ac3b83386..000000000 --- a/vault/credentials.go +++ /dev/null @@ -1,88 +0,0 @@ -package vault - -import ( - "crypto/sha256" - "io/ioutil" - "sync" - - "golang.org/x/crypto/pbkdf2" -) - -// Credentials encapsulates user credentials. -type Credentials interface { - Username() string - PrivateKey() *UserPrivateKey -} - -type credentials struct { - sync.Mutex - once sync.Once - - username string - privateKey *UserPrivateKey - passwordPrompt func() string -} - -func (pc *credentials) Username() string { - return pc.username -} - -func (pc *credentials) PrivateKey() *UserPrivateKey { - pc.once.Do(pc.deriveKeyFromPassword) - - return pc.privateKey -} - -func (pc *credentials) deriveKeyFromPassword() { - if pc.privateKey != nil { - return - } - - password := pc.passwordPrompt() - k := pbkdf2.Key([]byte(password), []byte(pc.username), pbkdf2Rounds, 32, sha256.New) - pk, err := newPrivateKey(k) - if err != nil { - panic("should not happen") - } - pc.privateKey = pk -} - -// Password returns Credentials object with static username and password. -func Password(username, password string) Credentials { - return &credentials{ - username: username, - passwordPrompt: func() string { - return password - }, - } -} - -// PasswordPrompt returns Credentials object that will prompt user for password using the specified callback function. -func PasswordPrompt(username string, prompt func() string) Credentials { - return &credentials{ - username: username, - passwordPrompt: prompt, - } -} - -// Key returns Credentials object with specified username and key bytes. -func Key(username string, key []byte) (Credentials, error) { - pk, err := newPrivateKey(key) - if err != nil { - return nil, err - } - - return &credentials{ - username: username, - privateKey: pk, - }, nil -} - -// KeyFromFile returns Credentials object with specified username and with key read from the specified file. -func KeyFromFile(username string, fileName string) (Credentials, error) { - k, err := ioutil.ReadFile(fileName) - if err != nil { - return nil, err - } - return Key(username, k) -} diff --git a/vault/credentials_test.go b/vault/credentials_test.go deleted file mode 100644 index 78c2dfbee..000000000 --- a/vault/credentials_test.go +++ /dev/null @@ -1,31 +0,0 @@ -package vault - -import ( - "bytes" - "encoding/hex" - - "testing" -) - -func TestCredentials(t *testing.T) { - cases := []struct { - username string - password string - expectedKey string - }{ - {"foo", "bar", "60d6051cfbff0f53344ff64cd9770c65747ced5c541748b7f992cf575bffa2ad"}, - {"user", "bar", "fff2b04b391c1a31a41dab88843311ce7f93393ec97fb8a1be3697c5a88b85ca"}, - } - - for i, c := range cases { - creds := Password(c.username, c.password) - if u := creds.Username(); u != c.username { - t.Errorf("invalid username #%v: %v expected %v", i, u, c.username) - } - - expectedKeyBytes, _ := hex.DecodeString(c.expectedKey) - if v := creds.PrivateKey(); !bytes.Equal(expectedKeyBytes, v.Bytes()) { - t.Errorf("invalid key #%v: expected %x, got: %x", i, expectedKeyBytes, v.Bytes()) - } - } -} diff --git a/vault/format.go b/vault/format.go new file mode 100644 index 000000000..ea5cc3b62 --- /dev/null +++ b/vault/format.go @@ -0,0 +1,48 @@ +package vault + +import ( + "crypto/rand" + "fmt" + "io" + + "github.com/kopia/kopia/blob" + "github.com/kopia/kopia/cas" +) + +const ( + minUniqueIDLength = 32 +) + +type Format struct { + Version string `json:"version"` + UniqueID []byte `json:"uniqueID"` + Encryption string `json:"encryption"` + Checksum string `json:"checksum"` +} + +func (f *Format) ensureUniqueID() error { + if f.UniqueID == nil { + f.UniqueID = make([]byte, minUniqueIDLength) + if _, err := io.ReadFull(rand.Reader, f.UniqueID); err != nil { + return err + } + } + + if len(f.UniqueID) < minUniqueIDLength { + return fmt.Errorf("UniqueID too short, must be at least %v bytes", minUniqueIDLength) + } + + return nil +} + +func NewFormat() *Format { + return &Format{ + Encryption: "aes-256", + Checksum: "hmac-sha-256", + } +} + +type RepositoryConfig struct { + Storage blob.StorageConfiguration `json:"storage"` + Repository *cas.Format `json:"repository"` +} diff --git a/vault/vault.go b/vault/vault.go index 32153048f..80962ec76 100644 --- a/vault/vault.go +++ b/vault/vault.go @@ -1,42 +1,251 @@ package vault import ( - "net/url" + "bytes" + "crypto/aes" + "crypto/cipher" + "crypto/hmac" + "crypto/rand" + "crypto/sha256" + "encoding/json" + "fmt" + "hash" + "io" + "io/ioutil" "github.com/kopia/kopia/blob" - "github.com/kopia/kopia/cas" + + "golang.org/x/crypto/hkdf" + "golang.org/x/crypto/pbkdf2" ) -type Vault interface { +const ( + formatBlock = "format" + checksumBlock = "checksum" + repositoryConfigBlock = "repo" + minPasswordLength = 12 +) + +var ( + purposeAESKey = []byte("AES") + purposeChecksumSecret = []byte("CHECKSUM") +) + +type Vault struct { + Storage blob.Storage + MasterKey []byte + Format Format } -type vault struct { - storage blob.Storage - repo cas.Repository -} - -func Open(vaultPath string, creds Credentials) (Vault, error) { - var v vault - var err error - - v.storage, err = openStorage(vaultPath) +func (v *Vault) writeEncryptedBlock(name string, content []byte) error { + blk, err := v.newCipher() if err != nil { + return err + } + + hash, err := v.newChecksum() + if err != nil { + return err + } + + ivLength := blk.BlockSize() + ivPlusContentLength := ivLength + len(content) + cipherText := make([]byte, ivPlusContentLength+hash.Size()) + + // Store IV at the beginning of ciphertext. + iv := cipherText[0:ivLength] + if _, err := io.ReadFull(rand.Reader, iv); err != nil { + return err + } + + ctr := cipher.NewCTR(blk, iv) + ctr.XORKeyStream(cipherText[ivLength:], content) + hash.Write(cipherText[0:ivPlusContentLength]) + copy(cipherText[ivPlusContentLength:], hash.Sum(nil)) + + return v.Storage.PutBlock(name, ioutil.NopCloser(bytes.NewBuffer(cipherText)), blob.PutOptions{}) +} + +func (v *Vault) readEncryptedBlock(name string) ([]byte, error) { + cipherText, err := v.Storage.GetBlock(name) + if err != nil { + return nil, err + } + + hash, err := v.newChecksum() + if err != nil { + return nil, err + } + + p := len(cipherText) - hash.Size() + hash.Write(cipherText[0:p]) + expectedChecksum := hash.Sum(nil) + actualChecksum := cipherText[p:] + if !hmac.Equal(expectedChecksum, actualChecksum) { + return nil, fmt.Errorf("cannot read encrypted block.") + } + + blk, err := v.newCipher() + if err != nil { + return nil, err + } + + ivLength := blk.BlockSize() + + plainText := make([]byte, len(cipherText)-ivLength-hash.Size()) + iv := cipherText[0:blk.BlockSize()] + + ctr := cipher.NewCTR(blk, iv) + ctr.XORKeyStream(plainText, cipherText[ivLength:len(cipherText)-hash.Size()]) + return plainText, nil +} + +func (v *Vault) newChecksum() (hash.Hash, error) { + switch v.Format.Checksum { + case "hmac-sha-256": + key := make([]byte, 32) + v.deriveKey(purposeChecksumSecret, key) + return hmac.New(sha256.New, key), nil + + default: + return nil, fmt.Errorf("unsupported checksum format: %v", v.Format.Checksum) + } + +} + +func (v *Vault) newCipher() (cipher.Block, error) { + switch v.Format.Encryption { + case "aes-256": + k := make([]byte, 32) + v.deriveKey(purposeAESKey, k) + return aes.NewCipher(k) + default: + return nil, fmt.Errorf("unsupported encryption format: %v", v.Format.Encryption) + } + +} + +func (v *Vault) deriveKey(purpose []byte, key []byte) error { + k := hkdf.New(sha256.New, v.MasterKey, v.Format.UniqueID, purpose) + _, err := io.ReadFull(k, key) + return err +} + +func (v *Vault) SetRepository(rc RepositoryConfig) error { + b, err := json.Marshal(&rc) + if err != nil { + return err + } + + return v.writeEncryptedBlock(repositoryConfigBlock, b) +} + +func (v *Vault) Repository() (RepositoryConfig, error) { + var rc RepositoryConfig + + b, err := v.readEncryptedBlock(repositoryConfigBlock) + if err != nil { + return rc, fmt.Errorf("unable to read repository: %v", err) + } + + err = json.Unmarshal(b, &rc) + return rc, err +} + +func CreateWithPassword(storage blob.Storage, format *Format, password string) (*Vault, error) { + if err := format.ensureUniqueID(); err != nil { + return nil, err + } + + if len(password) < minPasswordLength { + return nil, fmt.Errorf("password too short, must be at least %v characters, got %v", minPasswordLength, len(password)) + } + masterKey := pbkdf2.Key([]byte(password), format.UniqueID, pbkdf2Rounds, masterKeySize, sha256.New) + return CreateWithKey(storage, format, masterKey) +} + +func CreateWithKey(storage blob.Storage, format *Format, masterKey []byte) (*Vault, error) { + ok, err := storage.BlockExists(formatBlock) + if ok { + return nil, fmt.Errorf("vault already exists") + } + if err != nil { + return nil, err + } + + formatBytes, err := json.Marshal(&format) + if err != nil { + return nil, err + } + + storage.PutBlock(formatBlock, ioutil.NopCloser(bytes.NewBuffer(formatBytes)), blob.PutOptions{}) + + v := Vault{ + Storage: storage, + MasterKey: masterKey, + Format: *format, + } + v.Format.Version = "1" + if err := v.Format.ensureUniqueID(); err != nil { + return nil, err + } + + vv := make([]byte, 512) + if _, err := io.ReadFull(rand.Reader, vv); err != nil { + return nil, err + } + + err = v.writeEncryptedBlock(checksumBlock, vv) + if err != nil { + return nil, err + } + + return OpenWithKey(storage, masterKey) +} + +func OpenWithPassword(storage blob.Storage, password string) (*Vault, error) { + v := Vault{ + Storage: storage, + } + + f, err := storage.GetBlock(formatBlock) + if err != nil { + return nil, err + } + + err = json.Unmarshal(f, &v.Format) + if err != nil { + return nil, err + } + + v.MasterKey = pbkdf2.Key([]byte(password), v.Format.UniqueID, pbkdf2Rounds, masterKeySize, sha256.New) + + if _, err := v.readEncryptedBlock(checksumBlock); err != nil { return nil, err } return &v, nil } -func Create(vaultPath string, repositoryPath string, creds Credentials) (Vault, error) { - var v vault - return &v, nil -} +func OpenWithKey(storage blob.Storage, masterKey []byte) (*Vault, error) { + v := Vault{ + Storage: storage, + MasterKey: masterKey, + } -func openStorage(vaultPath string) (blob.Storage, error) { - u, err := url.Parse(vaultPath) + f, err := storage.GetBlock(formatBlock) if err != nil { return nil, err } - return blob.NewStorageFromURL(u) + err = json.Unmarshal(f, &v.Format) + if err != nil { + return nil, err + } + + if _, err := v.readEncryptedBlock(checksumBlock); err != nil { + return nil, err + } + + return &v, nil }