mirror/kopia
1
0
Fork 0
mirror of https://github.com/kopia/kopia.git synced 2026-03-14 12:16:46 -04:00
Code Issues Packages Projects Releases Wiki Activity

various tweaks

Browse Source
This commit is contained in:
Jarek Kowalski
2016-05-16 18:32:15 -07:00
parent b77ac6cb8b
commit e5f646e32d
11 changed files with 128 additions and 120 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
backup/manifest.go
View File

@@ -25,6 +25,8 @@ type Manifest struct {
TotalFileSize int64 `json:"totalSize"`
}
// SourceID generates unique identifier of the backup source, which is a
// SHA1 hash of the host name, username and source directory.
func (m Manifest) SourceID() string {
h := sha1.New()
io.WriteString(h, m.HostName)

3
blob/config.go
View File

@@ -4,7 +4,6 @@
"encoding/json"
)
// StorageConfiguration is a JSON-serializable description of Storage and its options.
type StorageConfiguration struct {
Type string
Config StorageOptions
@@ -31,7 +30,7 @@ func (c *StorageConfiguration) UnmarshalJSON(b []byte) error {
}
// MarshalJSON returns JSON-encoded storage configuration.
func (c *StorageConfiguration) MarshalJSON() ([]byte, error) {
func (c StorageConfiguration) MarshalJSON() ([]byte, error) {
return json.Marshal(struct {
Type string `json:"type"`
Data interface{} `json:"config"`

4
cmd/kopia/command_backups.go
View File

@@ -15,8 +15,8 @@
var (
backupsCommand = app.Command("backups", "List backup history.")
backupsDirectory = backupsCommand.Arg("directory", "Directory to show history of").ExistingDir()
backupsAll = backupsCommand.Flag("all", "Show history of all backups").Bool()
maxResultsPerPath = backupsCommand.Flag("maxresults", "Maximum number of results").Default("100").Int()
backupsAll = backupsCommand.Flag("all", "Show history of all backups.").Bool()
maxResultsPerPath = backupsCommand.Flag("maxresults", "Maximum number of results.").Default("100").Int()
)
func runBackupsCommand(context *kingpin.ParseContext) error {

2
cmd/kopia/command_connect.go
View File

@@ -7,7 +7,7 @@
)
var (
connectCommand = app.Command("connect", "Create new vault and optionally connect to it")
connectCommand = app.Command("connect", "Connect to a vault.")
)
func init() {

138
cmd/kopia/command_create.go
View File

@@ -1,8 +1,9 @@
package main
import (
"crypto/rand"
"fmt"
"strings"
"io"
"github.com/kopia/kopia/repo"
"github.com/kopia/kopia/vault"
@@ -12,16 +13,12 @@
)
var (
createCommand = app.Command("create", "Create new vault and optionally connect to it")
createCommandRepository = createCommand.Flag("repository", "Repository path").Required().String()
createCommandOnly = createCommand.Flag("only", "Only create, don't connect.").Bool()
createMaxBlobSize = createCommand.Flag("max-blob-size", "Maximum size of a data chunk in bytes.").Default("4000000").Int()
createInlineBlobSize = createCommand.Flag("inline-blob-size", "Maximum size of an inline data chunk in bytes.").Default("32768").Int()
createVaultEncryptionFormat = createCommand.Flag("vaultencryption", "Vault encryption format").String()
createSecurity = createCommand.Flag("security", "Security mode, one of 'none', 'default' or 'custom'.").Default("default").Enum("none", "default", "custom")
createCustomFormat = createCommand.Flag("object-format", "Specifies custom object format to be used").String()
createCommand = app.Command("create", "Create new vault.")
createCommandRepository = createCommand.Flag("repository", "Repository path.").Required().String()
createMaxBlobSize = createCommand.Flag("max-blob-size", "Maximum size of a data chunk in bytes.").Default("20000000").Int()
createInlineBlobSize = createCommand.Flag("inline-blob-size", "Maximum size of an inline data chunk in bytes.").Default("32768").Int()
createVaultEncryptionFormat = createCommand.Flag("vault-encryption", "Vault encryption format").Default("aes-256").Enum(supportedVaultEncryptionFormats()...)
createObjectFormat = createCommand.Flag("object-format", "Specifies custom object format to be used").Default("hmac-sha256").Enum(supportedObjectFormats()...)
createOverwrite = createCommand.Flag("overwrite", "Overwrite existing data.").Bool()
)
@@ -29,22 +26,33 @@ func init() {
createCommand.Action(runCreateCommand)
}
func vaultFormat() *vault.Format {
f := vault.NewFormat()
if *createVaultEncryptionFormat != "" {
f.Encryption = *createVaultEncryptionFormat
func vaultFormat() (*vault.Format, error) {
f := &vault.Format{
Version: "1",
Checksum: "hmac-sha-256",
}
return f
}
func repositoryFormat() (*repo.Format, error) {
f, err := repo.NewFormat()
f.UniqueID = make([]byte, 32)
_, err := io.ReadFull(rand.Reader, f.UniqueID)
if err != nil {
return nil, err
}
f.Encryption = *createVaultEncryptionFormat
return f, nil
}
f.MaxBlobSize = *createMaxBlobSize
f.MaxInlineBlobSize = *createInlineBlobSize
func repositoryFormat() (*repo.Format, error) {
f := &repo.Format{
Version: "1",
Secret: make([]byte, 32),
MaxBlobSize: *createMaxBlobSize,
MaxInlineBlobSize: *createInlineBlobSize,
ObjectFormat: *createObjectFormat,
}
_, err := io.ReadFull(rand.Reader, f.Secret)
if err != nil {
return nil, err
}
return f, nil
}
@@ -79,77 +87,69 @@ func runCreateCommand(context *kingpin.ParseContext) error {
return fmt.Errorf("unable to get repository storage: %v", err)
}
repoFormat, err := repositoryFormat()
if err != nil {
return fmt.Errorf("unable to initialize repository format: %v", err)
}
fmt.Printf(
"Initializing repository in '%s' with format '%v' and maximum object size %v.\n",
repositoryStorage.Configuration().Config.ToURL().String(),
repoFormat.ObjectFormat,
repoFormat.MaxBlobSize)
masterKey, password, err := getKeyOrPassword(true)
if err != nil {
return fmt.Errorf("unable to get credentials: %v", err)
}
var v *vault.Vault
vf, err := vaultFormat()
if err != nil {
return fmt.Errorf("unable to initialize vault format: %v", err)
}
fmt.Printf(
"Initializing vault in '%s' with encryption '%v'.\n",
vaultStorage.Configuration().Config.ToURL().String(),
vf.Encryption)
if masterKey != nil {
v, err = vault.CreateWithKey(vaultStorage, vaultFormat(), masterKey)
v, err = vault.CreateWithKey(vaultStorage, vf, masterKey)
} else {
v, err = vault.CreateWithPassword(vaultStorage, vaultFormat(), password)
v, err = vault.CreateWithPassword(vaultStorage, vf, password)
}
if err != nil {
return fmt.Errorf("cannot create vault: %v", err)
}
repoFormat, err := repositoryFormat()
if err != nil {
return fmt.Errorf("unable to initialize repository format: %v", err)
}
// Make repository to make sure the format is supported.
_, err = repo.NewRepository(repositoryStorage, repoFormat)
if err != nil {
return fmt.Errorf("unable to initialize repository: %v", err)
}
v.SetRepository(vault.RepositoryConfig{
if err := v.SetRepository(vault.RepositoryConfig{
Storage: repositoryStorage.Configuration(),
Format: repoFormat,
})
if *createCommandOnly {
fmt.Println("Created vault:", *vaultPath)
return nil
}); err != nil {
return fmt.Errorf("unable to save repository configuration in vault: %v", err)
}
persistVaultConfig(v)
fmt.Println("Created and connected to vault:", *vaultPath)
return err
return nil
}
func getCustomFormat() string {
if *createCustomFormat != "" {
if repo.SupportedFormats.Find(*createCustomFormat) == nil {
fmt.Printf("Format '%s' is not recognized.\n", *createCustomFormat)
}
return *createCustomFormat
}
fmt.Printf(" %2v | %-30v | %v | %v | %v |\n", "#", "Format", "Hash", "Encryption", "Block ID Length")
fmt.Println(strings.Repeat("-", 76) + "+")
for i, o := range repo.SupportedFormats {
encryptionString := ""
if o.IsEncrypted() {
encryptionString = fmt.Sprintf("%d-bit", o.EncryptionKeySizeBits())
}
fmt.Printf(" %2v | %-30v | %4v | %10v | %15v |\n", i+1, o.Name, o.HashSizeBits(), encryptionString, o.BlockIDLength())
}
fmt.Println(strings.Repeat("-", 76) + "+")
fmt.Printf("Select format (1-%d): ", len(repo.SupportedFormats))
for {
var number int
if n, err := fmt.Scanf("%d\n", &number); n == 1 && err == nil && number >= 1 && number <= len(repo.SupportedFormats) {
fmt.Printf("You selected '%v'\n", repo.SupportedFormats[number-1].Name)
return repo.SupportedFormats[number-1].Name
}
fmt.Printf("Invalid selection. Select format (1-%d): ", len(repo.SupportedFormats))
func supportedVaultEncryptionFormats() []string {
return []string{
"none",
"aes-128",
"aes-256",
}
}
func supportedObjectFormats() []string {
var r []string
for _, o := range repo.SupportedFormats {
r = append(r, o.Name)
}
return r
}

50
cmd/kopia/config.go
View File

@@ -3,6 +3,7 @@
import (
"encoding/hex"
"encoding/json"
"errors"
"fmt"
"io/ioutil"
"os"
@@ -20,10 +21,10 @@
traceStorage = app.Flag("trace-storage", "Enables tracing of storage operations.").Hidden().Bool()
vaultPath = app.Flag("vault", "Specify the vault to use.").Envar("KOPIA_VAULT").String()
password = app.Flag("password", "Vault password").Envar("KOPIA_PASSWORD").String()
passwordFile = app.Flag("passwordfile", "Read password from a file").Envar("KOPIA_PASSWORD_FILE").ExistingFile()
key = app.Flag("key", "Vault key (hexadecimal)").Envar("KOPIA_KEY").String()
keyFile = app.Flag("keyfile", "Key key file").Envar("KOPIA_KEY_FILE").ExistingFile()
password = app.Flag("password", "Vault password.").Envar("KOPIA_PASSWORD").String()
passwordFile = app.Flag("passwordfile", "Read vault password from a file.").Envar("KOPIA_PASSWORD_FILE").ExistingFile()
key = app.Flag("key", "Specify vault master key (hexadecimal).").Envar("KOPIA_KEY").String()
keyFile = app.Flag("keyfile", "Read vault master key from file.").Envar("KOPIA_KEY_FILE").ExistingFile()
)
func failOnError(err error) {
@@ -63,8 +64,15 @@ func persistVaultConfig(v *vault.Vault) error {
return err
}
defer f.Close()
json.NewEncoder(f).Encode(vc)
return nil
b, err := json.MarshalIndent(&vc, "", " ")
if err != nil {
return err
}
_, err = f.Write(b)
return err
}
func getPersistedVaultConfig() *vaultConfig {
@@ -95,7 +103,7 @@ func openVault() (*vault.Vault, error) {
}
if *vaultPath == "" {
return nil, fmt.Errorf("vault not connected and not specified, use --vault")
return nil, fmt.Errorf("vault not connected and not specified, use --vault or run 'kopia connect'")
}
return openVaultSpecifiedByFlag()
@@ -117,11 +125,13 @@ func openVaultSpecifiedByFlag() (*vault.Vault, error) {
if masterKey != nil {
return vault.OpenWithKey(storage, masterKey)
} else {
return vault.OpenWithPassword(storage, password)
}
return vault.OpenWithPassword(storage, password)
}
var errPasswordTooShort = errors.New("password too short")
func getKeyOrPassword(isNew bool) ([]byte, string, error) {
if *key != "" {
k, err := hex.DecodeString(*key)
@@ -153,16 +163,20 @@ func getKeyOrPassword(isNew bool) ([]byte, string, error) {
return nil, strings.TrimSpace(string(f)), nil
}
if isNew {
for {
fmt.Printf("Enter password: ")
fmt.Printf("Enter password to create new vault: ")
p1, err := askPass()
fmt.Println()
if err == errPasswordTooShort {
fmt.Printf("Password too short, must be at least %v characters, you entered %v. Try again.", vault.MinPasswordLength, len(p1))
fmt.Println()
continue
}
if err != nil {
return nil, "", err
}
fmt.Println()
fmt.Printf("Enter password again: ")
fmt.Printf("Re-enter password for verification: ")
p2, err := askPass()
if err != nil {
return nil, "", err
@@ -175,7 +189,7 @@ func getKeyOrPassword(isNew bool) ([]byte, string, error) {
}
}
} else {
fmt.Printf("Enter password: ")
fmt.Printf("Enter password to open vault: ")
p1, err := askPass()
if err != nil {
return nil, "", err
@@ -191,5 +205,11 @@ func askPass() (string, error) {
return "", err
}
return string(b), nil
p := string(b)
if len(p) < vault.MinPasswordLength {
return p, errPasswordTooShort
}
return p, nil
}

2
cmd/kopia/main.go
View File

@@ -28,7 +28,7 @@
)
var (
app = kingpin.New("kopia", "Kopia - Online Backup")
app = kingpin.New("kopia", "Kopia - Online Backup").Version("0.0.1").Author("http://kopia.github.io/")
)
func main() {

1
fs/dir_json.go
View File

@@ -305,6 +305,7 @@ func newDirectoryReader(r io.Reader) (*directoryReader, error) {
return dr, nil
}
// ReadDirectory loads the serialized Directory from the specified Reader.
func ReadDirectory(r io.Reader, namePrefix string) (Directory, error) {
dr, err := newDirectoryReader(r)
if err != nil {

16
repo/format.go
View File

@@ -4,12 +4,10 @@
"crypto/aes"
"crypto/cipher"
"crypto/md5"
"crypto/rand"
"crypto/sha1"
"crypto/sha256"
"crypto/sha512"
"hash"
"io"
)
// Format describes the format of object data.
@@ -21,20 +19,6 @@ type Format struct {
MaxBlobSize int `json:"maxBlobSize"`
}
func NewFormat() (*Format, error) {
f := &Format{
Version: "1",
Secret: make([]byte, 32),
ObjectFormat: "hmac-sha256",
}
_, err := io.ReadFull(rand.Reader, f.Secret)
if err != nil {
return f, err
}
return f, nil
}
// ObjectIDFormat describes single format ObjectID
type ObjectIDFormat struct {
Name string

10
vault/format.go
View File

@@ -13,6 +13,8 @@
minUniqueIDLength = 32
)
// Format describes the format of a Vault.
// Contents of this structure are serialized in plain text in the Vault storage.
type Format struct {
Version string `json:"version"`
UniqueID []byte `json:"uniqueID"`
@@ -35,14 +37,6 @@ func (f *Format) ensureUniqueID() error {
return nil
}
func NewFormat() *Format {
return &Format{
Version: "1",
Encryption: "aes-256",
Checksum: "hmac-sha-256",
}
}
type RepositoryConfig struct {
Storage blob.StorageConfiguration `json:"storage"`
Format *repo.Format `json:"repository"`

20
vault/vault.go
View File

@@ -25,8 +25,8 @@
checksumBlock = "checksum"
repositoryConfigBlock = "repo"
minPasswordLength = 12
minKeyLength = 16
MinPasswordLength = 12
MinKeyLength = 16
)
var (
@@ -120,6 +120,14 @@ func (v *Vault) newChecksum() (hash.Hash, error) {
func (v *Vault) newCipher() (cipher.Block, error) {
switch v.Format.Encryption {
case "aes-128":
k := make([]byte, 16)
v.deriveKey(purposeAESKey, k)
return aes.NewCipher(k)
case "aes-192":
k := make([]byte, 24)
v.deriveKey(purposeAESKey, k)
return aes.NewCipher(k)
case "aes-256":
k := make([]byte, 32)
v.deriveKey(purposeAESKey, k)
@@ -209,16 +217,16 @@ func CreateWithPassword(storage blob.Storage, format *Format, password string) (
return nil, err
}
if len(password) < minPasswordLength {
return nil, fmt.Errorf("password too short, must be at least %v characters, got %v", minPasswordLength, len(password))
if len(password) < MinPasswordLength {
return nil, fmt.Errorf("password too short, must be at least %v characters, got %v", MinPasswordLength, len(password))
}
masterKey := pbkdf2.Key([]byte(password), format.UniqueID, pbkdf2Rounds, masterKeySize, sha256.New)
return CreateWithKey(storage, format, masterKey)
}
func CreateWithKey(storage blob.Storage, format *Format, masterKey []byte) (*Vault, error) {
if len(masterKey) < minKeyLength {
return nil, fmt.Errorf("key too short, must be at least %v bytes, got %v", minKeyLength, len(masterKey))
if len(masterKey) < MinKeyLength {
return nil, fmt.Errorf("key too short, must be at least %v bytes, got %v", MinKeyLength, len(masterKey))
}
v := Vault{