work in progress revamping CLI and adding vault

blob/filesystem.go

@@ -13,7 +13,7 @@
 )
 
 const (
-	fsStorageType        = "file"
+	fsStorageType        = "filesystem"
 	fsStorageChunkSuffix = ".f"
 )
 
@@ -31,10 +31,10 @@ type fsStorage struct {
 type FSStorageOptions struct {
 	Path string `json:"path"`
 
-	DirectoryShards []int `json:"dirShards"`
+	DirectoryShards []int `json:"dirShards,omitempty"`
 
-	FileMode      os.FileMode `json:"fileMode"`
-	DirectoryMode os.FileMode `json:"dirMode"`
+	FileMode      os.FileMode `json:"fileMode,omitempty"`
+	DirectoryMode os.FileMode `json:"dirMode,omitempty"`
 
 	FileUID *int `json:"uid,omitempty"`
 	FileGID *int `json:"gid,omitempty"`
@@ -66,10 +66,12 @@ func (fso *FSStorageOptions) shards() []int {
 
 // ParseURL parses the given URL into FSStorageOptions.
 func (fso *FSStorageOptions) ParseURL(u *url.URL) error {
-	if u.Scheme != "file" {
+	if u.Scheme != fsStorageType {
 		return fmt.Errorf("invalid scheme, expected 'file'")
 	}
 
+	//log.Printf("u.Upaque: %v u.Path: %v", u.Opaque, u.Path)
+
 	if u.Opaque != "" {
 		fso.Path = u.Opaque
 	} else {
@@ -97,12 +99,8 @@ func (fso *FSStorageOptions) ParseURL(u *url.URL) error {
 // ToURL converts the FSStorageOptions to URL.
 func (fso *FSStorageOptions) ToURL() *url.URL {
 	u := &url.URL{}
-	u.Scheme = "file"
-	if fso.Path[0] == '/' {
-		u.Path = fso.Path
-	} else {
-		u.Opaque = fso.Path
-	}
+	u.Scheme = "filesystem"
+	u.Opaque = fso.Path
 	q := u.Query()
 	if fso.FileUID != nil {
 		q.Add("uid", strconv.Itoa(*fso.FileUID))

blob/registry.go

@@ -40,7 +40,11 @@ func NewStorage(cfg StorageConfiguration) (Storage, error) {
 
 // NewStorageFromURL creates new storage based on a URL.
 // The storage type must be previously registered using AddSupportedStorage.
-func NewStorageFromURL(u *url.URL) (Storage, error) {
+func NewStorageFromURL(storageURL string) (Storage, error) {
+	u, err := url.Parse(storageURL)
+	if err != nil {
+		return nil, err
+	}
 	if factory, ok := factories[u.Scheme]; ok {
 		o := factory.defaultConfigFunc()
 		if err := o.ParseURL(u); err != nil {

blob/storage_test.go

@@ -58,12 +58,12 @@ func TestFileStorageOptions(t *testing.T) {
 		o   FSStorageOptions
 		url string
 	}{
-		{FSStorageOptions{Path: "/blah"}, "file:///blah"},
-		{FSStorageOptions{Path: "/blah", FileMode: 0123}, "file:///blah?filemode=123"},
-		{FSStorageOptions{Path: "/blah", DirectoryMode: 0123}, "file:///blah?dirmode=123"},
-		{FSStorageOptions{Path: "/blah", FileMode: 0432, DirectoryMode: 0123}, "file:///blah?dirmode=123&filemode=432"},
-		{FSStorageOptions{Path: "/blah", FileMode: 0432, DirectoryMode: 0123, DirectoryShards: []int{1, 2, 3}}, "file:///blah?dirmode=123&filemode=432&shards=1.2.3"},
-		{FSStorageOptions{Path: "c:\\winpath"}, "file:c:\\winpath"},
+		{FSStorageOptions{Path: "/blah"}, "filesystem:/blah"},
+		{FSStorageOptions{Path: "/blah", FileMode: 0123}, "filesystem:/blah?filemode=123"},
+		{FSStorageOptions{Path: "/blah", DirectoryMode: 0123}, "filesystem:/blah?dirmode=123"},
+		{FSStorageOptions{Path: "/blah", FileMode: 0432, DirectoryMode: 0123}, "filesystem:/blah?dirmode=123&filemode=432"},
+		{FSStorageOptions{Path: "/blah", FileMode: 0432, DirectoryMode: 0123, DirectoryShards: []int{1, 2, 3}}, "filesystem:/blah?dirmode=123&filemode=432&shards=1.2.3"},
+		{FSStorageOptions{Path: "c:\\winpath"}, "filesystem:c:\\winpath"},
 	}
 
 	for i, c := range cases {

cas/format.go

@@ -4,10 +4,12 @@
 	"crypto/aes"
 	"crypto/cipher"
 	"crypto/md5"
+	"crypto/rand"
 	"crypto/sha1"
 	"crypto/sha256"
 	"crypto/sha512"
 	"hash"
+	"io"
 )
 
 // Format describes the format of object data.
@@ -19,6 +21,20 @@ type Format struct {
 	MaxBlobSize       int    `json:"maxBlobSize"`
 }
 
+func NewFormat() (*Format, error) {
+	f := &Format{
+		Version:      "1",
+		Secret:       make([]byte, 32),
+		ObjectFormat: "hmac-sha256",
+	}
+
+	_, err := io.ReadFull(rand.Reader, f.Secret)
+	if err != nil {
+		return f, err
+	}
+	return f, nil
+}
+
 // ObjectIDFormat describes single format ObjectID
 type ObjectIDFormat struct {
 	Name string

cas/repository.go

@@ -191,7 +191,7 @@ func hmacFunc(key []byte, hf func() hash.Hash) func() hash.Hash {
 // NewRepository creates new Repository with the specified storage, options, and key provider.
 func NewRepository(
 	r blob.Storage,
-	f Format,
+	f *Format,
 	options ...RepositoryOption,
 ) (Repository, error) {
 	if f.Version != "1" {

cmd/kopia/command_create.go

@@ -0,0 +1,115 @@
+package main
+
+import (
+	"fmt"
+
+	"github.com/kopia/kopia/cas"
+	"github.com/kopia/kopia/vault"
+
+	"github.com/kopia/kopia/blob"
+	"gopkg.in/alecthomas/kingpin.v2"
+)
+
+var (
+	createCommand           = app.Command("create", "Create new vault and optionally connect to it")
+	createCommandRepository = createCommand.Flag("repository", "Repository path").Required().String()
+	createCommandOnly       = createCommand.Flag("only", "Only create, don't connect.").Bool()
+
+	createMaxBlobSize    = createCommand.Flag("max-blob-size", "Maximum size of a data chunk in bytes.").Default("4000000").Int()
+	createInlineBlobSize = createCommand.Flag("inline-blob-size", "Maximum size of an inline data chunk in bytes.").Default("32768").Int()
+
+	createVaultEncryptionFormat = createCommand.Flag("vaultencryption", "Vault encryption format").String()
+)
+
+func init() {
+	createCommand.Action(runCreateCommand)
+}
+
+func vaultFormat() *vault.Format {
+	f := vault.NewFormat()
+	if *createVaultEncryptionFormat != "" {
+		f.Encryption = *createVaultEncryptionFormat
+	}
+	return f
+}
+
+func repositoryFormat() (*cas.Format, error) {
+	f, err := cas.NewFormat()
+	if err != nil {
+		return nil, err
+	}
+
+	return f, nil
+}
+
+func openStorageAndEnsureEmpty(url string) (blob.Storage, error) {
+	s, err := blob.NewStorageFromURL(url)
+	if err != nil {
+		return nil, err
+	}
+	ch := s.ListBlocks("")
+	_, hasData := <-ch
+
+	if hasData {
+		return nil, fmt.Errorf("found existing data in %v", url)
+	}
+
+	return s, nil
+
+}
+
+func runCreateCommand(context *kingpin.ParseContext) error {
+	vaultStorage, err := openStorageAndEnsureEmpty(*vaultPath)
+	if err != nil {
+		return fmt.Errorf("unable to get vault storage: %v", err)
+		return err
+	}
+
+	repositoryStorage, err := openStorageAndEnsureEmpty(*createCommandRepository)
+	if err != nil {
+		return fmt.Errorf("unable to get repository storage: %v", err)
+	}
+
+	masterKey, password, err := getKeyOrPassword(true)
+	if err != nil {
+		return fmt.Errorf("unable to get credentials: %v", err)
+	}
+
+	var v *vault.Vault
+
+	if masterKey != nil {
+		v, err = vault.CreateWithKey(vaultStorage, vaultFormat(), masterKey)
+	} else {
+		v, err = vault.CreateWithPassword(vaultStorage, vaultFormat(), password)
+	}
+	if err != nil {
+		return fmt.Errorf("cannot create vault: %v", err)
+	}
+
+	repoFormat, err := repositoryFormat()
+	if err != nil {
+		return fmt.Errorf("unable to initialize repository format: %v", err)
+	}
+
+	// Make repository to make sure the format is supported.
+	_, err = cas.NewRepository(repositoryStorage, repoFormat)
+	if err != nil {
+		return fmt.Errorf("unable to initialize repository: %v", err)
+	}
+
+	v.SetRepository(vault.RepositoryConfig{
+		Storage:    repositoryStorage.Configuration(),
+		Repository: repoFormat,
+	})
+
+	if *createCommandOnly {
+		fmt.Println("Created vault:", *vaultPath)
+		return nil
+	}
+
+	persistVaultConfig(v)
+
+	fmt.Println("Created and connected to vault:", *vaultPath)
+
+	return err
+}

cmd/kopia/command_init.go

@@ -4,14 +4,11 @@
 	"crypto/rand"
 	"fmt"
 	"io"
-	"os"
 	"strings"
 
 	"github.com/kopia/kopia/blob"
 	"github.com/kopia/kopia/cas"
-	"github.com/kopia/kopia/config"
 	"github.com/kopia/kopia/session"
-	"github.com/kopia/kopia/vault"
 )
 
 var (
@@ -28,9 +25,7 @@
 )
 
 func runInitCommandForRepository(s blob.Storage, defaultSalt string) error {
-	var creds vault.Credentials
-
-	sess, err := session.New(s, creds)
+	sess, err := session.New(s)
 	if err != nil {
 		return err
 	}
@@ -80,16 +75,16 @@ func runInitCommandForRepository(s blob.Storage, defaultSalt string) error {
 		return err
 	}
 
-	cfg := config.Config{
-		Storage: s.Configuration(),
-	}
+	// cfg := config.Config{
+	// 	Storage: s.Configuration(),
+	// }
 
-	f, err := os.Create(configFileName())
-	if err != nil {
-		return err
-	}
-	defer f.Close()
-	cfg.SaveTo(f)
+	// f, err := os.Create(configFileName())
+	// if err != nil {
+	// 	return err
+	// }
+	// defer f.Close()
+	// cfg.SaveTo(f)
 
 	return nil
 }

cmd/kopia/command_status.go

@@ -0,0 +1,43 @@
+package main
+
+import (
+	"encoding/hex"
+	"fmt"
+
+	"gopkg.in/alecthomas/kingpin.v2"
+)
+
+var (
+	statusCommand = app.Command("status", "Display status information.")
+)
+
+func init() {
+	statusCommand.Action(runRepositoryInfoCommand)
+}
+
+func runRepositoryInfoCommand(context *kingpin.ParseContext) error {
+	v, err := openVault()
+	if err != nil {
+		return err
+	}
+
+	fmt.Println("Vault:")
+	fmt.Println("  Address:   ", v.Storage.Configuration().Config.ToURL())
+	fmt.Println("  ID:        ", hex.EncodeToString(v.Format.UniqueID))
+	fmt.Println("  Encryption:", v.Format.Encryption)
+	fmt.Println("  Checksum:  ", v.Format.Checksum)
+	fmt.Println("  Master Key:", hex.EncodeToString(v.MasterKey))
+
+	vc, err := v.Repository()
+	if err != nil {
+		return err
+	}
+
+	fmt.Println("Repository:")
+	fmt.Println("  Address:  ", vc.Storage.Config.ToURL())
+	fmt.Println("  Version:  ", vc.Repository.Version)
+	fmt.Println("  Secret:   ", len(vc.Repository.Secret), "bytes")
+	fmt.Println("  ID Format:", vc.Repository.ObjectFormat)
+
+	return nil
+}

cmd/kopia/config.go

@@ -1,20 +1,30 @@
 package main
 
 import (
+	"encoding/hex"
+	"encoding/json"
 	"fmt"
+	"io/ioutil"
 	"os"
-	"os/user"
 	"path/filepath"
+	"strings"
 
 	"github.com/kopia/kopia/blob"
-	"github.com/kopia/kopia/config"
+
+	"golang.org/x/crypto/ssh/terminal"
+
 	"github.com/kopia/kopia/session"
 	"github.com/kopia/kopia/vault"
 )
 
 var (
-	configFile   = app.Flag("config", "Specify config filename.").Default(getDefaultConfigFileName()).String()
 	traceStorage = app.Flag("trace-storage", "Enables tracing of storage operations.").Bool()
+
+	vaultPath    = app.Flag("vault", "Specify the vault to use.").String()
+	password     = app.Flag("password", "Vault password").String()
+	passwordFile = app.Flag("passwordfile", "Read password from a file").ExistingFile()
+	key          = app.Flag("key", "Vault key").String()
+	keyFile      = app.Flag("keyfile", "Read vault key from a file").ExistingFile()
 )
 
 func failOnError(err error) {
@@ -30,65 +40,192 @@ func mustOpenSession() session.Session {
 	return s
 }
 
-func configFileName() string {
-	if *configFile != "" {
-		return *configFile
-	}
-
-	return getDefaultConfigFileName()
-}
-
-func getDefaultConfigFileName() string {
-	u, err := user.Current()
-	if err == nil && u.Uid == "0" {
-		return "/etc/kopia/config.json"
-	}
-	return filepath.Join(getHomeDir(), ".kopia/config.json")
-}
-
 func getHomeDir() string {
 	return os.Getenv("HOME")
 }
 
-func loadConfig() (*config.Config, error) {
-	path := configFileName()
-	if path == "" {
-		return nil, fmt.Errorf("Cannot find config file. You may pass --config_file to specify config file location.")
-	}
-
-	var cfg config.Config
-
-	//log.Printf("Loading config file from %v", path)
-	f, err := os.Open(path)
-	if err != nil {
-		return nil, fmt.Errorf("Error opening configuration file: %v", err)
-	}
-	defer f.Close()
-
-	err = cfg.Load(f)
-	if err == nil {
-		return &cfg, nil
-	}
-
-	return nil, fmt.Errorf("Error loading configuration file: %v", err)
+func vaultConfigFileName() string {
+	return filepath.Join(getHomeDir(), ".kopia/vault.config")
 }
 
+// func loadConfig() (*config.Config, error) {
+// 	path := configFileName()
+// 	if path == "" {
+// 		return nil, fmt.Errorf("Cannot find config file. You may pass --config_file to specify config file location.")
+// 	}
+
+// 	var cfg config.Config
+
+// 	//log.Printf("Loading config file from %v", path)
+// 	f, err := os.Open(path)
+// 	if err != nil {
+// 		return nil, fmt.Errorf("Error opening configuration file: %v", err)
+// 	}
+// 	defer f.Close()
+
+// 	err = cfg.Load(f)
+// 	if err == nil {
+// 		return &cfg, nil
+// 	}
+
+// 	return nil, fmt.Errorf("Error loading configuration file: %v", err)
+// }
+
 func openSession() (session.Session, error) {
-	cfg, err := loadConfig()
-	if err != nil {
-		return nil, err
-	}
+	return nil, nil
+	// cfg, err := loadConfig()
+	// if err != nil {
+	// 	return nil, err
+	// }
 
-	storage, err := blob.NewStorage(cfg.Storage)
-	if err != nil {
-		return nil, err
-	}
+	// storage, err := blob.NewStorage(cfg.Storage)
+	// if err != nil {
+	// 	return nil, err
+	// }
 
-	if *traceStorage {
-		storage = blob.NewLoggingWrapper(storage)
-	}
+	// if *traceStorage {
+	// 	storage = blob.NewLoggingWrapper(storage)
+	// }
 
-	var creds vault.Credentials
-
-	return session.New(storage, creds)
+	// return session.New(storage)
+}
+
+type vaultConfig struct {
+	Storage blob.StorageConfiguration `json:"storage"`
+	Key     []byte                    `json:"key,omitempty"`
+}
+
+func persistVaultConfig(v *vault.Vault) error {
+	vc := vaultConfig{
+		Storage: v.Storage.Configuration(),
+		Key:     v.MasterKey,
+	}
+
+	f, err := os.Create(vaultConfigFileName())
+	if err != nil {
+		return err
+	}
+	defer f.Close()
+	json.NewEncoder(f).Encode(vc)
+	return nil
+}
+
+func getPersistedVaultConfig() *vaultConfig {
+	var vc vaultConfig
+
+	f, err := os.Open(vaultConfigFileName())
+	if err == nil {
+		err = json.NewDecoder(f).Decode(&vc)
+		f.Close()
+		if err != nil {
+			return nil
+		}
+		return &vc
+	}
+
+	return nil
+}
+
+func openVault() (*vault.Vault, error) {
+	vc := getPersistedVaultConfig()
+	if vc != nil {
+		storage, err := blob.NewStorage(vc.Storage)
+		if err != nil {
+			return nil, err
+		}
+
+		return vault.OpenWithKey(storage, vc.Key)
+	}
+
+	if *vaultPath == "" {
+		return nil, fmt.Errorf("vault not connected, use --vault")
+	}
+
+	storage, err := blob.NewStorageFromURL(*vaultPath)
+	if err != nil {
+		return nil, err
+	}
+
+	masterKey, password, err := getKeyOrPassword(false)
+	if err != nil {
+		return nil, err
+	}
+
+	if masterKey != nil {
+		return vault.OpenWithKey(storage, masterKey)
+	} else {
+		return vault.OpenWithPassword(storage, password)
+	}
+}
+
+func getKeyOrPassword(isNew bool) ([]byte, string, error) {
+	if *key != "" {
+		k, err := hex.DecodeString(*key)
+		if err != nil {
+			return nil, "", fmt.Errorf("invalid key format: %v", err)
+		}
+
+		return k, "", nil
+	}
+
+	if *password != "" {
+		return nil, strings.TrimSpace(*password), nil
+	}
+
+	if *keyFile != "" {
+		key, err := ioutil.ReadFile(*keyFile)
+		if err != nil {
+			return nil, "", fmt.Errorf("unable to read key file: %v", err)
+		}
+
+		return key, "", nil
+	}
+
+	if *passwordFile != "" {
+		f, err := ioutil.ReadFile(*passwordFile)
+		if err != nil {
+			return nil, "", fmt.Errorf("unable to read password file: %v", err)
+		}
+
+		return nil, strings.TrimSpace(string(f)), nil
+	}
+
+	if isNew {
+		for {
+			fmt.Printf("Enter password: ")
+			p1, err := askPass()
+			if err != nil {
+				return nil, "", err
+			}
+			fmt.Println()
+			fmt.Printf("Enter password again: ")
+			p2, err := askPass()
+			if err != nil {
+				return nil, "", err
+			}
+			fmt.Println()
+			if p1 != p2 {
+				fmt.Println("Passwords don't match!")
+			} else {
+				return nil, p1, nil
+			}
+		}
+	} else {
+		fmt.Printf("Enter password: ")
+		p1, err := askPass()
+		if err != nil {
+			return nil, "", err
+		}
+		fmt.Println()
+		return nil, p1, nil
+	}
+}
+
+func askPass() (string, error) {
+	b, err := terminal.ReadPassword(0)
+	if err != nil {
+		return "", err
+	}
+
+	return string(b), nil
 }

doc/quickstart.md

@@ -0,0 +1,57 @@
+Quick Start
+===
+
+Kopia is a simple tool for managing encrypted backups in the cloud.
+
+Key Concepts
+---
+
+* **Repository** is a [Content-Addressable Store](https://en.wikipedia.org/wiki/Content-addressable_storage) of files and directories (known as Objects) identified by their **Object IDs**.
+
+    - Object ID is comprised of the type, the identifier of the block (Block ID) where the contents are stored and the encryption key. Example block ID:
+
+    ```
+    DD7ce4f067c179664...e746337031644a:715b50351785a6...439637a2c8e50c7
+    |\------------------------------/ \------------------------------/
+    type          block id                      encryption key
+    ```
+
+    - Block IDs are derived from the contents of objects by using cryptographic hash, typically as one of [SHA-2](https://en.wikipedia.org/wiki/SHA-2) hash functions.
+    - Encryption key is also derived from the contents of the object - this technique is known as [Convergent Encryption](https://en.wikipedia.org/wiki/Convergent_encryption)
+    - Identical objects will have the same ID and thus will be stored only once with the same encryption key.
+    - A person with access to the object can easily compute its Object ID (including encryption key), but the knowledge of block id is not enough to be able to retrieve the content.
+    - Repository can be shared among many users as long as they all can compute the same object IDs
+
+* **Vault** securely stores Object IDs, encrypted with user-specific password or passphrase
+* **Blob Storage** stores unstructured, blocks of binary large objects (BLOBs).
+  Supported backends include:
+
+    - [Google Cloud Storage](https://cloud.google.com/storage/) (GCS)
+    - [Amazon Simple Storage Service](https://aws.amazon.com/s3/) (S3)
+    - Local or remote filesystem
+
+Object IDs
+---
+
+There are three types of object IDs:
+
+* **Data** or **Direct** object where the entire object is stored in a single block
+* **List** objects, that store list of object IDs
+* **Inline Text** and **Inline Binary** objects, that represent the contents of the very short files directly in the object IDs, encoded either as text or base64
+
+Some examples (note that block IDs and encryption keys have been shortened for illustration purposes):
+
+* `D7ce4f067c:715b503c8` - is a *Data* object stored in block `7ce4f067c` and encrypted with key `715b503c8`
+
+* `L43637a2c8:715b50351` - is a *List* object stored in block `43637a2c8` and encrypted with key `715b50351`
+
+* `Tquick brown fox` - is an *Inline Text* object representing the text `quick brown fox`
+
+* `BAQIDBA` - is an *Inline Binary* object representing base-64 encoded bytes: `01 02 03 04`
+
+Object Types
+---
+
+* **File** contents are store as binary
+* **Directory** contents are stored as JSON-encoded objects storing file or subdirectory metadata and Object IDs of file/subdirectory contents.
+

doc/tutorial.md

@@ -0,0 +1,71 @@
+Tutorial
+===
+
+Kopia is a simple, cross-platform tool for managing encrypted backups in the cloud. It provides fast incremental backups, data deduplication and client-side encryption.
+
+The main difference from other backup solutions is that as a user, you are in full control of backup storage - in fact you are responsible for purchasing one of the cloud storage products available (such as Google Cloud Storage), which offer great durability and availability for your data.
+
+## Installation
+
+### Binary Releases
+
+You can download pre-built `kopia` binary from http://kopia.github.io/download. Once downloaded, it's best to put it in a directory that's in system PATH, such as `/usr/local/bin`.
+
+### Installation From Source
+
+To build Kopia from source you need to have the latest version of [Go](https://golang.org/dl/) installed and run the following commands:
+
+```
+mkdir $HOME/kopia
+export GOPATH=$HOME/kopia
+go get github.com/kopia/kopia
+go install github.com/kopia/kopia/cmd/kopia
+```
+
+This will automatically download and build kopia and put the resulting binary in `$HOME/kopia/bin`. For convenience it's best to add this directory to system `PATH` or copy it to a directory already in the path, such as `/usr/local/bin`.
+
+## Getting Started
+
+To use Kopia, you need to set up two storage locations:
+
+- **Repository** - which will store the bulk of encrypted data
+- **Vault** - which stores the backup metadata and their encryption keys
+
+The **Repository** can be shared between multiple computers or users because without the encryption keys stored in the **Vault** its data is unaccessible.
+
+The metadata stored in the *Vault* is very small and will typically fit on even smallest of USB drives for safekeeping. You have a choice of using Vault in the cloud or keeping it on a removable storage device in your possession.
+
+### Setting Up Vault
+
+To create new vault in Google Cloud Storage, first create the bucket bucket using https://console.cloud.google.com/storage/ then run:
+
+```
+kopia vault create gcs BUCKET
+```
+
+To create new vault in the local filesystem, use:
+
+```
+mkdir /path/to/vault
+kopia vault create filesystem /path/to/vault
+```
+
+You will be prompted for the password to protect the data in the vault. **Don't forget your password, as there is absolutely no way to recover data in the vault if you do so.**
+
+To later connect to an existing vault, simply replace `vault create` with `vault connect`.
+
+To disconnect from a vault, run:
+```
+kopia vault disconnect
+```
+
+### Setting Up Repository
+
+After creating the vault, we now need to create a repository. This is very similar way to creating a vault, just replace `vault` with `repository`. For example to create GCS-backed repository, use:
+
+```
+kopia repository create gcs BUCKET
+```
+
+
+

session/session.go

@@ -10,7 +10,6 @@
 
 	"github.com/kopia/kopia/blob"
 	"github.com/kopia/kopia/cas"
-	"github.com/kopia/kopia/vault"
 )
 
 // ErrConfigNotFound indicates that configuration is not found.
@@ -25,7 +24,6 @@ type Session interface {
 
 type session struct {
 	storage blob.Storage
-	creds   vault.Credentials
 	format  cas.Format
 }
 
@@ -52,15 +50,11 @@ func (s *session) encryptBlockWithPublicKey(blkID string, data io.ReadCloser, op
 }
 
 func (s *session) getConfigstring() string {
-	if s.creds == nil {
-		return string("config.json")
-	}
-
-	return string("users." + s.creds.Username() + ".config.json")
+	return string("config.json")
 }
 
 func (s *session) InitRepository(format cas.Format) (cas.Repository, error) {
-	mgr, err := cas.NewRepository(s.storage, format)
+	mgr, err := cas.NewRepository(s.storage, &format)
 	if err != nil {
 		return nil, err
 	}
@@ -95,13 +89,12 @@ func (s *session) OpenRepository() (cas.Repository, error) {
 		return nil, err
 	}
 
-	return cas.NewRepository(s.storage, format)
+	return cas.NewRepository(s.storage, &format)
 }
 
-func New(storage blob.Storage, creds vault.Credentials) (Session, error) {
+func New(storage blob.Storage) (Session, error) {
 	sess := &session{
 		storage: storage,
-		creds:   creds,
 	}
 	return sess, nil
 }

session/session_test.go

@@ -1,51 +0,0 @@
-package session
-
-import (
-	"io/ioutil"
-
-	"github.com/kopia/kopia/cas"
-
-	"github.com/kopia/kopia/blob"
-
-	"testing"
-)
-
-func TestA(t *testing.T) {
-	tmpDir, err := ioutil.TempDir("", "kopia")
-	if err != nil {
-		t.Errorf("can't create temp directory: %v", err)
-		return
-	}
-
-	// cfg := LoadConfig("kopia.config")
-	sc := blob.StorageConfiguration{
-		Type: "file",
-		Config: &blob.FSStorageOptions{
-			Path: tmpDir,
-		},
-	}
-
-	storage, err := blob.NewStorage(sc)
-	if err != nil {
-		t.Errorf("cannot create storage: %v", err)
-		return
-	}
-
-	sess, err := New(storage, nil)
-	defer sess.Close()
-
-	om, err := sess.InitRepository(cas.Format{
-		Version:      "1",
-		ObjectFormat: "sha1",
-	})
-
-	if err != nil {
-		t.Errorf("unable to init object manager: %v", err)
-		return
-	}
-
-	w := om.NewWriter()
-	w.Write([]byte{1, 2, 3})
-	x, err := w.Result(true)
-	t.Logf("%v x: %v %v", tmpDir, x, err)
-}

vault/credentials.go

@@ -1,88 +0,0 @@
-package vault
-
-import (
-	"crypto/sha256"
-	"io/ioutil"
-	"sync"
-
-	"golang.org/x/crypto/pbkdf2"
-)
-
-// Credentials encapsulates user credentials.
-type Credentials interface {
-	Username() string
-	PrivateKey() *UserPrivateKey
-}
-
-type credentials struct {
-	sync.Mutex
-	once sync.Once
-
-	username       string
-	privateKey     *UserPrivateKey
-	passwordPrompt func() string
-}
-
-func (pc *credentials) Username() string {
-	return pc.username
-}
-
-func (pc *credentials) PrivateKey() *UserPrivateKey {
-	pc.once.Do(pc.deriveKeyFromPassword)
-
-	return pc.privateKey
-}
-
-func (pc *credentials) deriveKeyFromPassword() {
-	if pc.privateKey != nil {
-		return
-	}
-
-	password := pc.passwordPrompt()
-	k := pbkdf2.Key([]byte(password), []byte(pc.username), pbkdf2Rounds, 32, sha256.New)
-	pk, err := newPrivateKey(k)
-	if err != nil {
-		panic("should not happen")
-	}
-	pc.privateKey = pk
-}
-
-// Password returns Credentials object with static username and password.
-func Password(username, password string) Credentials {
-	return &credentials{
-		username: username,
-		passwordPrompt: func() string {
-			return password
-		},
-	}
-}
-
-// PasswordPrompt returns Credentials object that will prompt user for password using the specified callback function.
-func PasswordPrompt(username string, prompt func() string) Credentials {
-	return &credentials{
-		username:       username,
-		passwordPrompt: prompt,
-	}
-}
-
-// Key returns Credentials object with specified username and key bytes.
-func Key(username string, key []byte) (Credentials, error) {
-	pk, err := newPrivateKey(key)
-	if err != nil {
-		return nil, err
-	}
-
-	return &credentials{
-		username:   username,
-		privateKey: pk,
-	}, nil
-}
-
-// KeyFromFile returns Credentials object with specified username and with key read from the specified file.
-func KeyFromFile(username string, fileName string) (Credentials, error) {
-	k, err := ioutil.ReadFile(fileName)
-	if err != nil {
-		return nil, err
-	}
-	return Key(username, k)
-}

vault/credentials_test.go

@@ -1,31 +0,0 @@
-package vault
-
-import (
-	"bytes"
-	"encoding/hex"
-
-	"testing"
-)
-
-func TestCredentials(t *testing.T) {
-	cases := []struct {
-		username    string
-		password    string
-		expectedKey string
-	}{
-		{"foo", "bar", "60d6051cfbff0f53344ff64cd9770c65747ced5c541748b7f992cf575bffa2ad"},
-		{"user", "bar", "fff2b04b391c1a31a41dab88843311ce7f93393ec97fb8a1be3697c5a88b85ca"},
-	}
-
-	for i, c := range cases {
-		creds := Password(c.username, c.password)
-		if u := creds.Username(); u != c.username {
-			t.Errorf("invalid username #%v: %v expected %v", i, u, c.username)
-		}
-
-		expectedKeyBytes, _ := hex.DecodeString(c.expectedKey)
-		if v := creds.PrivateKey(); !bytes.Equal(expectedKeyBytes, v.Bytes()) {
-			t.Errorf("invalid key #%v: expected %x, got: %x", i, expectedKeyBytes, v.Bytes())
-		}
-	}
-}

vault/format.go

@@ -0,0 +1,48 @@
+package vault
+
+import (
+	"crypto/rand"
+	"fmt"
+	"io"
+
+	"github.com/kopia/kopia/blob"
+	"github.com/kopia/kopia/cas"
+)
+
+const (
+	minUniqueIDLength = 32
+)
+
+type Format struct {
+	Version    string `json:"version"`
+	UniqueID   []byte `json:"uniqueID"`
+	Encryption string `json:"encryption"`
+	Checksum   string `json:"checksum"`
+}
+
+func (f *Format) ensureUniqueID() error {
+	if f.UniqueID == nil {
+		f.UniqueID = make([]byte, minUniqueIDLength)
+		if _, err := io.ReadFull(rand.Reader, f.UniqueID); err != nil {
+			return err
+		}
+	}
+
+	if len(f.UniqueID) < minUniqueIDLength {
+		return fmt.Errorf("UniqueID too short, must be at least %v bytes", minUniqueIDLength)
+	}
+
+	return nil
+}
+
+func NewFormat() *Format {
+	return &Format{
+		Encryption: "aes-256",
+		Checksum:   "hmac-sha-256",
+	}
+}
+
+type RepositoryConfig struct {
+	Storage    blob.StorageConfiguration `json:"storage"`
+	Repository *cas.Format               `json:"repository"`
+}

vault/vault.go

@@ -1,42 +1,251 @@
 package vault
 
 import (
-	"net/url"
+	"bytes"
+	"crypto/aes"
+	"crypto/cipher"
+	"crypto/hmac"
+	"crypto/rand"
+	"crypto/sha256"
+	"encoding/json"
+	"fmt"
+	"hash"
+	"io"
+	"io/ioutil"
 
 	"github.com/kopia/kopia/blob"
-	"github.com/kopia/kopia/cas"
+
+	"golang.org/x/crypto/hkdf"
+	"golang.org/x/crypto/pbkdf2"
 )
 
-type Vault interface {
+const (
+	formatBlock           = "format"
+	checksumBlock         = "checksum"
+	repositoryConfigBlock = "repo"
+	minPasswordLength     = 12
+)
+
+var (
+	purposeAESKey         = []byte("AES")
+	purposeChecksumSecret = []byte("CHECKSUM")
+)
+
+type Vault struct {
+	Storage   blob.Storage
+	MasterKey []byte
+	Format    Format
 }
 
-type vault struct {
-	storage blob.Storage
-	repo    cas.Repository
-}
-
-func Open(vaultPath string, creds Credentials) (Vault, error) {
-	var v vault
-	var err error
-
-	v.storage, err = openStorage(vaultPath)
+func (v *Vault) writeEncryptedBlock(name string, content []byte) error {
+	blk, err := v.newCipher()
 	if err != nil {
+		return err
+	}
+
+	hash, err := v.newChecksum()
+	if err != nil {
+		return err
+	}
+
+	ivLength := blk.BlockSize()
+	ivPlusContentLength := ivLength + len(content)
+	cipherText := make([]byte, ivPlusContentLength+hash.Size())
+
+	// Store IV at the beginning of ciphertext.
+	iv := cipherText[0:ivLength]
+	if _, err := io.ReadFull(rand.Reader, iv); err != nil {
+		return err
+	}
+
+	ctr := cipher.NewCTR(blk, iv)
+	ctr.XORKeyStream(cipherText[ivLength:], content)
+	hash.Write(cipherText[0:ivPlusContentLength])
+	copy(cipherText[ivPlusContentLength:], hash.Sum(nil))
+
+	return v.Storage.PutBlock(name, ioutil.NopCloser(bytes.NewBuffer(cipherText)), blob.PutOptions{})
+}
+
+func (v *Vault) readEncryptedBlock(name string) ([]byte, error) {
+	cipherText, err := v.Storage.GetBlock(name)
+	if err != nil {
+		return nil, err
+	}
+
+	hash, err := v.newChecksum()
+	if err != nil {
+		return nil, err
+	}
+
+	p := len(cipherText) - hash.Size()
+	hash.Write(cipherText[0:p])
+	expectedChecksum := hash.Sum(nil)
+	actualChecksum := cipherText[p:]
+	if !hmac.Equal(expectedChecksum, actualChecksum) {
+		return nil, fmt.Errorf("cannot read encrypted block.")
+	}
+
+	blk, err := v.newCipher()
+	if err != nil {
+		return nil, err
+	}
+
+	ivLength := blk.BlockSize()
+
+	plainText := make([]byte, len(cipherText)-ivLength-hash.Size())
+	iv := cipherText[0:blk.BlockSize()]
+
+	ctr := cipher.NewCTR(blk, iv)
+	ctr.XORKeyStream(plainText, cipherText[ivLength:len(cipherText)-hash.Size()])
+	return plainText, nil
+}
+
+func (v *Vault) newChecksum() (hash.Hash, error) {
+	switch v.Format.Checksum {
+	case "hmac-sha-256":
+		key := make([]byte, 32)
+		v.deriveKey(purposeChecksumSecret, key)
+		return hmac.New(sha256.New, key), nil
+
+	default:
+		return nil, fmt.Errorf("unsupported checksum format: %v", v.Format.Checksum)
+	}
+
+}
+
+func (v *Vault) newCipher() (cipher.Block, error) {
+	switch v.Format.Encryption {
+	case "aes-256":
+		k := make([]byte, 32)
+		v.deriveKey(purposeAESKey, k)
+		return aes.NewCipher(k)
+	default:
+		return nil, fmt.Errorf("unsupported encryption format: %v", v.Format.Encryption)
+	}
+
+}
+
+func (v *Vault) deriveKey(purpose []byte, key []byte) error {
+	k := hkdf.New(sha256.New, v.MasterKey, v.Format.UniqueID, purpose)
+	_, err := io.ReadFull(k, key)
+	return err
+}
+
+func (v *Vault) SetRepository(rc RepositoryConfig) error {
+	b, err := json.Marshal(&rc)
+	if err != nil {
+		return err
+	}
+
+	return v.writeEncryptedBlock(repositoryConfigBlock, b)
+}
+
+func (v *Vault) Repository() (RepositoryConfig, error) {
+	var rc RepositoryConfig
+
+	b, err := v.readEncryptedBlock(repositoryConfigBlock)
+	if err != nil {
+		return rc, fmt.Errorf("unable to read repository: %v", err)
+	}
+
+	err = json.Unmarshal(b, &rc)
+	return rc, err
+}
+
+func CreateWithPassword(storage blob.Storage, format *Format, password string) (*Vault, error) {
+	if err := format.ensureUniqueID(); err != nil {
+		return nil, err
+	}
+
+	if len(password) < minPasswordLength {
+		return nil, fmt.Errorf("password too short, must be at least %v characters, got %v", minPasswordLength, len(password))
+	}
+	masterKey := pbkdf2.Key([]byte(password), format.UniqueID, pbkdf2Rounds, masterKeySize, sha256.New)
+	return CreateWithKey(storage, format, masterKey)
+}
+
+func CreateWithKey(storage blob.Storage, format *Format, masterKey []byte) (*Vault, error) {
+	ok, err := storage.BlockExists(formatBlock)
+	if ok {
+		return nil, fmt.Errorf("vault already exists")
+	}
+	if err != nil {
+		return nil, err
+	}
+
+	formatBytes, err := json.Marshal(&format)
+	if err != nil {
+		return nil, err
+	}
+
+	storage.PutBlock(formatBlock, ioutil.NopCloser(bytes.NewBuffer(formatBytes)), blob.PutOptions{})
+
+	v := Vault{
+		Storage:   storage,
+		MasterKey: masterKey,
+		Format:    *format,
+	}
+	v.Format.Version = "1"
+	if err := v.Format.ensureUniqueID(); err != nil {
+		return nil, err
+	}
+
+	vv := make([]byte, 512)
+	if _, err := io.ReadFull(rand.Reader, vv); err != nil {
+		return nil, err
+	}
+
+	err = v.writeEncryptedBlock(checksumBlock, vv)
+	if err != nil {
+		return nil, err
+	}
+
+	return OpenWithKey(storage, masterKey)
+}
+
+func OpenWithPassword(storage blob.Storage, password string) (*Vault, error) {
+	v := Vault{
+		Storage: storage,
+	}
+
+	f, err := storage.GetBlock(formatBlock)
+	if err != nil {
+		return nil, err
+	}
+
+	err = json.Unmarshal(f, &v.Format)
+	if err != nil {
+		return nil, err
+	}
+
+	v.MasterKey = pbkdf2.Key([]byte(password), v.Format.UniqueID, pbkdf2Rounds, masterKeySize, sha256.New)
+
+	if _, err := v.readEncryptedBlock(checksumBlock); err != nil {
 		return nil, err
 	}
 
 	return &v, nil
 }
 
-func Create(vaultPath string, repositoryPath string, creds Credentials) (Vault, error) {
-	var v vault
-	return &v, nil
-}
+func OpenWithKey(storage blob.Storage, masterKey []byte) (*Vault, error) {
+	v := Vault{
+		Storage:   storage,
+		MasterKey: masterKey,
+	}
 
-func openStorage(vaultPath string) (blob.Storage, error) {
-	u, err := url.Parse(vaultPath)
+	f, err := storage.GetBlock(formatBlock)
 	if err != nil {
 		return nil, err
 	}
 
-	return blob.NewStorageFromURL(u)
+	err = json.Unmarshal(f, &v.Format)
+	if err != nil {
+		return nil, err
+	}
+
+	if _, err := v.readEncryptedBlock(checksumBlock); err != nil {
+		return nil, err
+	}
+
+	return &v, nil
 }