From e5f646e32d19a7db4ba15656da34537fdc19bdd8 Mon Sep 17 00:00:00 2001 From: Jarek Kowalski Date: Mon, 16 May 2016 18:32:15 -0700 Subject: [PATCH] various tweaks --- backup/manifest.go | 2 + blob/config.go | 3 +- cmd/kopia/command_backups.go | 4 +- cmd/kopia/command_connect.go | 2 +- cmd/kopia/command_create.go | 138 +++++++++++++++++------------------ cmd/kopia/config.go | 50 +++++++++---- cmd/kopia/main.go | 2 +- fs/dir_json.go | 1 + repo/format.go | 16 ---- vault/format.go | 10 +-- vault/vault.go | 20 +++-- 11 files changed, 128 insertions(+), 120 deletions(-) diff --git a/backup/manifest.go b/backup/manifest.go index 3e7bfbfcf..c2484d8f1 100644 --- a/backup/manifest.go +++ b/backup/manifest.go @@ -25,6 +25,8 @@ type Manifest struct { TotalFileSize int64 `json:"totalSize"` } +// SourceID generates unique identifier of the backup source, which is a +// SHA1 hash of the host name, username and source directory. func (m Manifest) SourceID() string { h := sha1.New() io.WriteString(h, m.HostName) diff --git a/blob/config.go b/blob/config.go index 7a47378a3..48a344182 100644 --- a/blob/config.go +++ b/blob/config.go @@ -4,7 +4,6 @@ "encoding/json" ) -// StorageConfiguration is a JSON-serializable description of Storage and its options. type StorageConfiguration struct { Type string Config StorageOptions @@ -31,7 +30,7 @@ func (c *StorageConfiguration) UnmarshalJSON(b []byte) error { } // MarshalJSON returns JSON-encoded storage configuration. -func (c *StorageConfiguration) MarshalJSON() ([]byte, error) { +func (c StorageConfiguration) MarshalJSON() ([]byte, error) { return json.Marshal(struct { Type string `json:"type"` Data interface{} `json:"config"` diff --git a/cmd/kopia/command_backups.go b/cmd/kopia/command_backups.go index a378e126b..c1dd45972 100644 --- a/cmd/kopia/command_backups.go +++ b/cmd/kopia/command_backups.go @@ -15,8 +15,8 @@ var ( backupsCommand = app.Command("backups", "List backup history.") backupsDirectory = backupsCommand.Arg("directory", "Directory to show history of").ExistingDir() - backupsAll = backupsCommand.Flag("all", "Show history of all backups").Bool() - maxResultsPerPath = backupsCommand.Flag("maxresults", "Maximum number of results").Default("100").Int() + backupsAll = backupsCommand.Flag("all", "Show history of all backups.").Bool() + maxResultsPerPath = backupsCommand.Flag("maxresults", "Maximum number of results.").Default("100").Int() ) func runBackupsCommand(context *kingpin.ParseContext) error { diff --git a/cmd/kopia/command_connect.go b/cmd/kopia/command_connect.go index b47ae4c3e..7b102548f 100644 --- a/cmd/kopia/command_connect.go +++ b/cmd/kopia/command_connect.go @@ -7,7 +7,7 @@ ) var ( - connectCommand = app.Command("connect", "Create new vault and optionally connect to it") + connectCommand = app.Command("connect", "Connect to a vault.") ) func init() { diff --git a/cmd/kopia/command_create.go b/cmd/kopia/command_create.go index ee5311d30..9961d9a9b 100644 --- a/cmd/kopia/command_create.go +++ b/cmd/kopia/command_create.go @@ -1,8 +1,9 @@ package main import ( + "crypto/rand" "fmt" - "strings" + "io" "github.com/kopia/kopia/repo" "github.com/kopia/kopia/vault" @@ -12,16 +13,12 @@ ) var ( - createCommand = app.Command("create", "Create new vault and optionally connect to it") - createCommandRepository = createCommand.Flag("repository", "Repository path").Required().String() - createCommandOnly = createCommand.Flag("only", "Only create, don't connect.").Bool() - - createMaxBlobSize = createCommand.Flag("max-blob-size", "Maximum size of a data chunk in bytes.").Default("4000000").Int() - createInlineBlobSize = createCommand.Flag("inline-blob-size", "Maximum size of an inline data chunk in bytes.").Default("32768").Int() - - createVaultEncryptionFormat = createCommand.Flag("vaultencryption", "Vault encryption format").String() - createSecurity = createCommand.Flag("security", "Security mode, one of 'none', 'default' or 'custom'.").Default("default").Enum("none", "default", "custom") - createCustomFormat = createCommand.Flag("object-format", "Specifies custom object format to be used").String() + createCommand = app.Command("create", "Create new vault.") + createCommandRepository = createCommand.Flag("repository", "Repository path.").Required().String() + createMaxBlobSize = createCommand.Flag("max-blob-size", "Maximum size of a data chunk in bytes.").Default("20000000").Int() + createInlineBlobSize = createCommand.Flag("inline-blob-size", "Maximum size of an inline data chunk in bytes.").Default("32768").Int() + createVaultEncryptionFormat = createCommand.Flag("vault-encryption", "Vault encryption format").Default("aes-256").Enum(supportedVaultEncryptionFormats()...) + createObjectFormat = createCommand.Flag("object-format", "Specifies custom object format to be used").Default("hmac-sha256").Enum(supportedObjectFormats()...) createOverwrite = createCommand.Flag("overwrite", "Overwrite existing data.").Bool() ) @@ -29,22 +26,33 @@ func init() { createCommand.Action(runCreateCommand) } -func vaultFormat() *vault.Format { - f := vault.NewFormat() - if *createVaultEncryptionFormat != "" { - f.Encryption = *createVaultEncryptionFormat +func vaultFormat() (*vault.Format, error) { + f := &vault.Format{ + Version: "1", + Checksum: "hmac-sha-256", } - return f -} - -func repositoryFormat() (*repo.Format, error) { - f, err := repo.NewFormat() + f.UniqueID = make([]byte, 32) + _, err := io.ReadFull(rand.Reader, f.UniqueID) if err != nil { return nil, err } + f.Encryption = *createVaultEncryptionFormat + return f, nil +} - f.MaxBlobSize = *createMaxBlobSize - f.MaxInlineBlobSize = *createInlineBlobSize +func repositoryFormat() (*repo.Format, error) { + f := &repo.Format{ + Version: "1", + Secret: make([]byte, 32), + MaxBlobSize: *createMaxBlobSize, + MaxInlineBlobSize: *createInlineBlobSize, + ObjectFormat: *createObjectFormat, + } + + _, err := io.ReadFull(rand.Reader, f.Secret) + if err != nil { + return nil, err + } return f, nil } @@ -79,77 +87,69 @@ func runCreateCommand(context *kingpin.ParseContext) error { return fmt.Errorf("unable to get repository storage: %v", err) } + repoFormat, err := repositoryFormat() + if err != nil { + return fmt.Errorf("unable to initialize repository format: %v", err) + } + + fmt.Printf( + "Initializing repository in '%s' with format '%v' and maximum object size %v.\n", + repositoryStorage.Configuration().Config.ToURL().String(), + repoFormat.ObjectFormat, + repoFormat.MaxBlobSize) + masterKey, password, err := getKeyOrPassword(true) if err != nil { return fmt.Errorf("unable to get credentials: %v", err) } var v *vault.Vault + vf, err := vaultFormat() + if err != nil { + return fmt.Errorf("unable to initialize vault format: %v", err) + } + + fmt.Printf( + "Initializing vault in '%s' with encryption '%v'.\n", + vaultStorage.Configuration().Config.ToURL().String(), + vf.Encryption) if masterKey != nil { - v, err = vault.CreateWithKey(vaultStorage, vaultFormat(), masterKey) + v, err = vault.CreateWithKey(vaultStorage, vf, masterKey) } else { - v, err = vault.CreateWithPassword(vaultStorage, vaultFormat(), password) + v, err = vault.CreateWithPassword(vaultStorage, vf, password) } if err != nil { return fmt.Errorf("cannot create vault: %v", err) } - repoFormat, err := repositoryFormat() - if err != nil { - return fmt.Errorf("unable to initialize repository format: %v", err) - } - // Make repository to make sure the format is supported. _, err = repo.NewRepository(repositoryStorage, repoFormat) if err != nil { return fmt.Errorf("unable to initialize repository: %v", err) } - v.SetRepository(vault.RepositoryConfig{ + if err := v.SetRepository(vault.RepositoryConfig{ Storage: repositoryStorage.Configuration(), Format: repoFormat, - }) - - if *createCommandOnly { - fmt.Println("Created vault:", *vaultPath) - return nil + }); err != nil { + return fmt.Errorf("unable to save repository configuration in vault: %v", err) } - persistVaultConfig(v) - - fmt.Println("Created and connected to vault:", *vaultPath) - - return err + return nil } -func getCustomFormat() string { - if *createCustomFormat != "" { - if repo.SupportedFormats.Find(*createCustomFormat) == nil { - fmt.Printf("Format '%s' is not recognized.\n", *createCustomFormat) - } - return *createCustomFormat - } - - fmt.Printf(" %2v | %-30v | %v | %v | %v |\n", "#", "Format", "Hash", "Encryption", "Block ID Length") - fmt.Println(strings.Repeat("-", 76) + "+") - for i, o := range repo.SupportedFormats { - encryptionString := "" - if o.IsEncrypted() { - encryptionString = fmt.Sprintf("%d-bit", o.EncryptionKeySizeBits()) - } - fmt.Printf(" %2v | %-30v | %4v | %10v | %15v |\n", i+1, o.Name, o.HashSizeBits(), encryptionString, o.BlockIDLength()) - } - fmt.Println(strings.Repeat("-", 76) + "+") - - fmt.Printf("Select format (1-%d): ", len(repo.SupportedFormats)) - for { - var number int - - if n, err := fmt.Scanf("%d\n", &number); n == 1 && err == nil && number >= 1 && number <= len(repo.SupportedFormats) { - fmt.Printf("You selected '%v'\n", repo.SupportedFormats[number-1].Name) - return repo.SupportedFormats[number-1].Name - } - - fmt.Printf("Invalid selection. Select format (1-%d): ", len(repo.SupportedFormats)) +func supportedVaultEncryptionFormats() []string { + return []string{ + "none", + "aes-128", + "aes-256", } } + +func supportedObjectFormats() []string { + var r []string + for _, o := range repo.SupportedFormats { + r = append(r, o.Name) + } + return r +} diff --git a/cmd/kopia/config.go b/cmd/kopia/config.go index d1d3f742b..b62f4ff84 100644 --- a/cmd/kopia/config.go +++ b/cmd/kopia/config.go @@ -3,6 +3,7 @@ import ( "encoding/hex" "encoding/json" + "errors" "fmt" "io/ioutil" "os" @@ -20,10 +21,10 @@ traceStorage = app.Flag("trace-storage", "Enables tracing of storage operations.").Hidden().Bool() vaultPath = app.Flag("vault", "Specify the vault to use.").Envar("KOPIA_VAULT").String() - password = app.Flag("password", "Vault password").Envar("KOPIA_PASSWORD").String() - passwordFile = app.Flag("passwordfile", "Read password from a file").Envar("KOPIA_PASSWORD_FILE").ExistingFile() - key = app.Flag("key", "Vault key (hexadecimal)").Envar("KOPIA_KEY").String() - keyFile = app.Flag("keyfile", "Key key file").Envar("KOPIA_KEY_FILE").ExistingFile() + password = app.Flag("password", "Vault password.").Envar("KOPIA_PASSWORD").String() + passwordFile = app.Flag("passwordfile", "Read vault password from a file.").Envar("KOPIA_PASSWORD_FILE").ExistingFile() + key = app.Flag("key", "Specify vault master key (hexadecimal).").Envar("KOPIA_KEY").String() + keyFile = app.Flag("keyfile", "Read vault master key from file.").Envar("KOPIA_KEY_FILE").ExistingFile() ) func failOnError(err error) { @@ -63,8 +64,15 @@ func persistVaultConfig(v *vault.Vault) error { return err } defer f.Close() - json.NewEncoder(f).Encode(vc) - return nil + + b, err := json.MarshalIndent(&vc, "", " ") + if err != nil { + return err + } + + _, err = f.Write(b) + + return err } func getPersistedVaultConfig() *vaultConfig { @@ -95,7 +103,7 @@ func openVault() (*vault.Vault, error) { } if *vaultPath == "" { - return nil, fmt.Errorf("vault not connected and not specified, use --vault") + return nil, fmt.Errorf("vault not connected and not specified, use --vault or run 'kopia connect'") } return openVaultSpecifiedByFlag() @@ -117,11 +125,13 @@ func openVaultSpecifiedByFlag() (*vault.Vault, error) { if masterKey != nil { return vault.OpenWithKey(storage, masterKey) - } else { - return vault.OpenWithPassword(storage, password) } + + return vault.OpenWithPassword(storage, password) } +var errPasswordTooShort = errors.New("password too short") + func getKeyOrPassword(isNew bool) ([]byte, string, error) { if *key != "" { k, err := hex.DecodeString(*key) @@ -153,16 +163,20 @@ func getKeyOrPassword(isNew bool) ([]byte, string, error) { return nil, strings.TrimSpace(string(f)), nil } - if isNew { for { - fmt.Printf("Enter password: ") + fmt.Printf("Enter password to create new vault: ") p1, err := askPass() + fmt.Println() + if err == errPasswordTooShort { + fmt.Printf("Password too short, must be at least %v characters, you entered %v. Try again.", vault.MinPasswordLength, len(p1)) + fmt.Println() + continue + } if err != nil { return nil, "", err } - fmt.Println() - fmt.Printf("Enter password again: ") + fmt.Printf("Re-enter password for verification: ") p2, err := askPass() if err != nil { return nil, "", err @@ -175,7 +189,7 @@ func getKeyOrPassword(isNew bool) ([]byte, string, error) { } } } else { - fmt.Printf("Enter password: ") + fmt.Printf("Enter password to open vault: ") p1, err := askPass() if err != nil { return nil, "", err @@ -191,5 +205,11 @@ func askPass() (string, error) { return "", err } - return string(b), nil + p := string(b) + + if len(p) < vault.MinPasswordLength { + return p, errPasswordTooShort + } + + return p, nil } diff --git a/cmd/kopia/main.go b/cmd/kopia/main.go index 896fa0a4a..28d4a7e8f 100644 --- a/cmd/kopia/main.go +++ b/cmd/kopia/main.go @@ -28,7 +28,7 @@ ) var ( - app = kingpin.New("kopia", "Kopia - Online Backup") + app = kingpin.New("kopia", "Kopia - Online Backup").Version("0.0.1").Author("http://kopia.github.io/") ) func main() { diff --git a/fs/dir_json.go b/fs/dir_json.go index 30472feda..ddacc7a48 100644 --- a/fs/dir_json.go +++ b/fs/dir_json.go @@ -305,6 +305,7 @@ func newDirectoryReader(r io.Reader) (*directoryReader, error) { return dr, nil } +// ReadDirectory loads the serialized Directory from the specified Reader. func ReadDirectory(r io.Reader, namePrefix string) (Directory, error) { dr, err := newDirectoryReader(r) if err != nil { diff --git a/repo/format.go b/repo/format.go index a543a2548..09bafd3e8 100644 --- a/repo/format.go +++ b/repo/format.go @@ -4,12 +4,10 @@ "crypto/aes" "crypto/cipher" "crypto/md5" - "crypto/rand" "crypto/sha1" "crypto/sha256" "crypto/sha512" "hash" - "io" ) // Format describes the format of object data. @@ -21,20 +19,6 @@ type Format struct { MaxBlobSize int `json:"maxBlobSize"` } -func NewFormat() (*Format, error) { - f := &Format{ - Version: "1", - Secret: make([]byte, 32), - ObjectFormat: "hmac-sha256", - } - - _, err := io.ReadFull(rand.Reader, f.Secret) - if err != nil { - return f, err - } - return f, nil -} - // ObjectIDFormat describes single format ObjectID type ObjectIDFormat struct { Name string diff --git a/vault/format.go b/vault/format.go index fd7f968f7..4b0b8c91f 100644 --- a/vault/format.go +++ b/vault/format.go @@ -13,6 +13,8 @@ minUniqueIDLength = 32 ) +// Format describes the format of a Vault. +// Contents of this structure are serialized in plain text in the Vault storage. type Format struct { Version string `json:"version"` UniqueID []byte `json:"uniqueID"` @@ -35,14 +37,6 @@ func (f *Format) ensureUniqueID() error { return nil } -func NewFormat() *Format { - return &Format{ - Version: "1", - Encryption: "aes-256", - Checksum: "hmac-sha-256", - } -} - type RepositoryConfig struct { Storage blob.StorageConfiguration `json:"storage"` Format *repo.Format `json:"repository"` diff --git a/vault/vault.go b/vault/vault.go index eb90ce705..6de41fdc2 100644 --- a/vault/vault.go +++ b/vault/vault.go @@ -25,8 +25,8 @@ checksumBlock = "checksum" repositoryConfigBlock = "repo" - minPasswordLength = 12 - minKeyLength = 16 + MinPasswordLength = 12 + MinKeyLength = 16 ) var ( @@ -120,6 +120,14 @@ func (v *Vault) newChecksum() (hash.Hash, error) { func (v *Vault) newCipher() (cipher.Block, error) { switch v.Format.Encryption { + case "aes-128": + k := make([]byte, 16) + v.deriveKey(purposeAESKey, k) + return aes.NewCipher(k) + case "aes-192": + k := make([]byte, 24) + v.deriveKey(purposeAESKey, k) + return aes.NewCipher(k) case "aes-256": k := make([]byte, 32) v.deriveKey(purposeAESKey, k) @@ -209,16 +217,16 @@ func CreateWithPassword(storage blob.Storage, format *Format, password string) ( return nil, err } - if len(password) < minPasswordLength { - return nil, fmt.Errorf("password too short, must be at least %v characters, got %v", minPasswordLength, len(password)) + if len(password) < MinPasswordLength { + return nil, fmt.Errorf("password too short, must be at least %v characters, got %v", MinPasswordLength, len(password)) } masterKey := pbkdf2.Key([]byte(password), format.UniqueID, pbkdf2Rounds, masterKeySize, sha256.New) return CreateWithKey(storage, format, masterKey) } func CreateWithKey(storage blob.Storage, format *Format, masterKey []byte) (*Vault, error) { - if len(masterKey) < minKeyLength { - return nil, fmt.Errorf("key too short, must be at least %v bytes, got %v", minKeyLength, len(masterKey)) + if len(masterKey) < MinKeyLength { + return nil, fmt.Errorf("key too short, must be at least %v bytes, got %v", MinKeyLength, len(masterKey)) } v := Vault{