diff --git a/README.md b/README.md new file mode 100644 index 000000000..157f0851e --- /dev/null +++ b/README.md @@ -0,0 +1,121 @@ +Kopia +===== + +> _n. thing exactly replicated from the original (Polish)_ + +Kopia is a simple, cross-platform tool for managing encrypted backups in the cloud. It provides fast, incremental backups, secure, client-side encryption and data deduplication. + +Unlike other cloud backup solutions, the user in full control of backup storage and is responsible for purchasing one of the cloud storage products (such as [Google Cloud Storage](https://cloud.google.com/storage/)), which offer great durability and availability for your data. + +> **NOTICE**: +> +> Kopia is still in early stages of development and is not ready for general use. +> The repository and vault data format are subject to change, including backwards-incompatible changes. Use at your own risk. + +Installation +--- + +To build Kopia you need the latest version of [Go](https://golang.org/dl/) and run the following commands: + +``` +mkdir $HOME/kopia +export GOPATH=$HOME/kopia +go get github.com/kopia/kopia +go install github.com/kopia/kopia/cmd/kopia +``` + +This will place Kopia binary in `$HOME/kopia/bin/kopia`. For convenience it's best to place it in a directory it the system `PATH`. + +Setting up repository and vault +--- + +Repository is where the bulk of the backup data will be stored and which can be shared among users. Vault is a small, password-protected area that stores list of backups of a single user. + +Vault and repository can be stored on: + +- local filesystem paths +- Google Cloud Storage buckets, for example `gs://my-bucket` + +For example, to create a vault on a local USB drive and repository in Google Cloud Storage use: + +``` +$ kopia create --vault /mnt/my-usb-drive --repository gs://my-bucket +Enter password to create new vault: *********** +Re-enter password for verification: *********** +Connected to vault: /mnt/my-usb-drive +``` + +The vault password is cached in a local file, so you don't need to enter it all the time. +To disconnect from the vault and remove cached password use: +``` +$ kopia disconnect +Disconnected from vault. +``` + +To connect to an existing vault: +``` +$ kopia connect --vault /mnt/my-usb-drive +Enter password to open vault: *********** +Connected to vault: /mnt/my-usb-drive +``` + +Backup and Restore +--- + +To create a backup of a directory or file, use `kopia backup `. It willwill print the identifier of a backup, which is a long string, that can be used to restore the file later. Because data in a repository is content-addressable, two files with identical contents, even in different directories or on different machines, will get the same backup identifier. + +``` +$ kopia backup /path/to/dir +D9691a95c5f9a73a1decf493f8f0d79.309a95f17bc3c6d3272bd0a62d2 +$ kopia backup /path/to/dir +D9691a95c5f9a73a1decf493f8f0d79.309a95f17bc3c6d3272bd0a62d2 +``` + +To list all backups of a particular directory or file, use: +``` +$ kopia backups +``` + +To list all backups stored in a vault, use: +``` +$ kopia backups --all +``` + +In order to browse the contents of a backup you can mount it in a local filesystem using: + +``` +$ kopia mount +``` + +To unmount, use: +``` +$ umount +``` + +You can also show the contents of an single object in a repository by using: +``` +$ kopia show +``` + + +Cryptography Notice +--- + + This distribution includes cryptographic software. The country in + which you currently reside may have restrictions on the import, + possession, use, and/or re-export to another country, of encryption + software. BEFORE using any encryption software, please check your + country's laws, regulations and policies concerning the import, + possession, or use, and re-export of encryption software, to see if + this is permitted. See for more + information. + + The U.S. Government Department of Commerce, Bureau of Industry and + Security (BIS), has classified this software as Export Commodity + Control Number (ECCN) 5D002.C.1, which includes information security + software using or performing cryptographic functions with symmetric + algorithms. The form and manner of this distribution makes it + eligible for export under the License Exception ENC Technology + Software Unrestricted (TSU) exception (see the BIS Export + Administration Regulations, Section 740.13) for both object code and + source code. \ No newline at end of file diff --git a/backup/generator.go b/backup/generator.go index ada71fe08..65fe38000 100644 --- a/backup/generator.go +++ b/backup/generator.go @@ -1,7 +1,8 @@ package backup import ( - "log" + "fmt" + "os" "time" "github.com/kopia/kopia/fs" @@ -32,18 +33,28 @@ func (bg *backupGenerator) Backup(m *Manifest, old *Manifest) error { if old != nil { hashCacheID = repo.ObjectID(old.HashCacheID) - log.Printf("Using hash cache ID: %v", hashCacheID) - } else { - log.Printf("No hash cache.") } - r, err := uploader.UploadDir(m.SourceDirectory, hashCacheID) + + st, err := os.Stat(m.Source) if err != nil { return err } + var r *fs.UploadResult + switch st.Mode() & os.ModeType { + case os.ModeDir: + r, err = uploader.UploadDir(m.Source, hashCacheID) + case 0: // regular + r, err = uploader.UploadFile(m.Source) + default: + return fmt.Errorf("unsupported source: %v", m.Source) + } + m.EndTime = time.Now() + if err != nil { + return err + } m.RootObjectID = string(r.ObjectID) m.HashCacheID = string(r.ManifestID) - m.EndTime = time.Now() return nil } diff --git a/backup/manifest.go b/backup/manifest.go index c2484d8f1..2133099c0 100644 --- a/backup/manifest.go +++ b/backup/manifest.go @@ -16,9 +16,10 @@ type Manifest struct { UserName string `json:"userName"` Description string `json:"description"` - SourceDirectory string `json:"source"` - RootObjectID string `json:"root"` - HashCacheID string `json:"hashCache"` + Handle string `json:"handle"` + Source string `json:"source"` + RootObjectID string `json:"root"` + HashCacheID string `json:"hashCache"` FileCount int64 `json:"fileCount"` DirectoryCount int64 `json:"dirCount"` @@ -33,7 +34,7 @@ func (m Manifest) SourceID() string { h.Write(zeroByte) io.WriteString(h, m.UserName) h.Write(zeroByte) - io.WriteString(h, m.SourceDirectory) + io.WriteString(h, m.Source) h.Write(zeroByte) return hex.EncodeToString(h.Sum(nil)) } diff --git a/cmd/kopia/command_backup.go b/cmd/kopia/command_backup.go index 4704f47e6..c0b4b449c 100644 --- a/cmd/kopia/command_backup.go +++ b/cmd/kopia/command_backup.go @@ -2,7 +2,10 @@ import ( "crypto/rand" + "crypto/sha256" + "encoding/hex" "fmt" + "io" "log" "math" "os" @@ -22,9 +25,9 @@ ) var ( - backupCommand = app.Command("backup", "Copies local directory to backup repository.") + backupCommand = app.Command("backup", "Copies local files or directories to backup repository.") - backupDirectories = backupCommand.Arg("directory", "Directories to back up").Required().ExistingDirs() + backupSources = backupCommand.Arg("source", "Files or directories to back up.").Required().ExistingFilesOrDirs() backupHostName = backupCommand.Flag("host", "Override backup hostname.").String() backupUser = backupCommand.Flag("user", "Override backup user.").String() @@ -66,15 +69,15 @@ func runBackupCommand(context *kingpin.ParseContext) error { return err } - for _, backupDirectory := range *backupDirectories { + for _, backupDirectory := range *backupSources { dir, err := filepath.Abs(backupDirectory) if err != nil { - return fmt.Errorf("invalid directory: '%s': %s", backupDirectory, err) + return fmt.Errorf("invalid source: '%s': %s", backupDirectory, err) } manifest := backup.Manifest{ - StartTime: time.Now(), - SourceDirectory: filepath.Clean(dir), + StartTime: time.Now(), + Source: filepath.Clean(dir), HostName: getBackupHostName(), UserName: getBackupUser(), @@ -100,25 +103,43 @@ func runBackupCommand(context *kingpin.ParseContext) error { oldManifest = &m } - uniqueID := make([]byte, 8) - rand.Read(uniqueID) - fileID := fmt.Sprintf("B%v.%08x.%x", manifest.SourceID(), math.MaxInt64-manifest.StartTime.UnixNano(), uniqueID) - if err := bgen.Backup(&manifest, oldManifest); err != nil { return err } + handleID, err := vlt.SaveObjectID(repo.ObjectID(manifest.RootObjectID)) + if err != nil { + return err + } + + uniqueID := make([]byte, 8) + rand.Read(uniqueID) + fileID := fmt.Sprintf("B%v.%08x.%x", manifest.SourceID(), math.MaxInt64-manifest.StartTime.UnixNano(), uniqueID) + manifest.Handle = handleID + err = vlt.Put(fileID, &manifest) if err != nil { return fmt.Errorf("cannot save manifest: %v", err) } log.Printf("Root: %v", manifest.RootObjectID) + log.Printf("Key: %v", handleID) } return nil } +func hashObjectID(oid string) string { + h := sha256.New() + io.WriteString(h, oid) + sum := h.Sum(nil) + foldLen := 16 + for i := foldLen; i < len(sum); i++ { + sum[i%foldLen] ^= sum[i] + } + return hex.EncodeToString(sum[0:foldLen]) +} + func getBackupUser() string { if *backupUser != "" { return *backupUser diff --git a/cmd/kopia/command_backups.go b/cmd/kopia/command_backups.go index 33c382e1b..46a687505 100644 --- a/cmd/kopia/command_backups.go +++ b/cmd/kopia/command_backups.go @@ -7,17 +7,56 @@ "github.com/kopia/kopia/backup" "github.com/kopia/kopia/repo" + "github.com/kopia/kopia/vault" kingpin "gopkg.in/alecthomas/kingpin.v2" ) var ( - backupsCommand = app.Command("backups", "List backup history.") - backupsDirectory = backupsCommand.Arg("directory", "Directory to show history of").ExistingDir() - backupsAll = backupsCommand.Flag("all", "Show history of all backups.").Bool() + backupsCommand = app.Command("backups", "List history of file or directory backups.") + backupsPath = backupsCommand.Arg("source", "File or directory to show history of.").String() maxResultsPerPath = backupsCommand.Flag("maxresults", "Maximum number of results.").Default("100").Int() ) +func findBackups(vlt *vault.Vault, path string) ([]string, string, error) { + var relPath string + + for len(path) > 0 { + manifest := backup.Manifest{ + Source: path, + HostName: getBackupHostName(), + UserName: getBackupUser(), + } + + prefix := manifest.SourceID() + "." + + list, err := vlt.List("B" + prefix) + if err != nil { + return nil, "", err + } + + if len(list) > 0 { + return list, relPath, nil + } + + if len(relPath) > 0 { + relPath = filepath.Base(path) + "/" + relPath + } else { + relPath = filepath.Base(path) + } + + log.Printf("No backups of %v@%v:%v", manifest.UserName, manifest.HostName, manifest.Source) + + parent := filepath.Dir(path) + if parent == path { + break + } + path = parent + } + + return nil, "", nil +} + func runBackupsCommand(context *kingpin.ParseContext) error { var options []repo.RepositoryOption @@ -41,31 +80,30 @@ func runBackupsCommand(context *kingpin.ParseContext) error { } defer mgr.Close() - var prefix string + var previous []string + var relPath string - if !*backupsAll { - - dir, err := filepath.Abs(*backupsDirectory) + if *backupsPath != "" { + path, err := filepath.Abs(*backupsPath) if err != nil { - return fmt.Errorf("invalid directory: '%s': %s", *backupsDirectory, err) + return fmt.Errorf("invalid directory: '%s': %s", *backupsPath, err) } - manifest := backup.Manifest{ - SourceDirectory: filepath.Clean(dir), - HostName: getBackupHostName(), - UserName: getBackupUser(), + previous, relPath, err = findBackups(vlt, filepath.Clean(path)) + if relPath != "" { + relPath = "/" + relPath } - prefix = manifest.SourceID() + "." + } else { + previous, err = vlt.List("B") } - previous, err := vlt.List("B" + prefix) if err != nil { - return fmt.Errorf("error listing previous backups") + return fmt.Errorf("cannot list backups: %v", err) } var lastHost string var lastUser string - var lastDir string + var lastSource string var count int for _, n := range previous { @@ -74,16 +112,16 @@ func runBackupsCommand(context *kingpin.ParseContext) error { return fmt.Errorf("error loading previous backup: %v", err) } - if m.HostName != lastHost || m.UserName != lastUser || m.SourceDirectory != lastDir { - log.Printf("%v @ %v : %v", m.UserName, m.HostName, m.SourceDirectory) - lastDir = m.SourceDirectory + if m.HostName != lastHost || m.UserName != lastUser || m.Source != lastSource { + log.Printf("%v@%v:%v", m.UserName, m.HostName, m.Source) + lastSource = m.Source lastUser = m.UserName lastHost = m.HostName count = 0 } if count < *maxResultsPerPath { - log.Printf(" %v %v", m.RootObjectID, m.StartTime.Format("2006-01-02 15:04:05 MST")) + log.Printf(" %v%v %v", m.Handle, relPath, m.StartTime.Format("2006-01-02 15:04:05 MST")) count++ } } diff --git a/cmd/kopia/command_cat.go b/cmd/kopia/command_cat.go new file mode 100644 index 000000000..ad17db7f9 --- /dev/null +++ b/cmd/kopia/command_cat.go @@ -0,0 +1,36 @@ +package main + +import ( + "io" + "os" + + kingpin "gopkg.in/alecthomas/kingpin.v2" +) + +var ( + catCommand = app.Command("cat", "Displays contents of a repository object.") + catCommandPath = catCommand.Arg("path", "Path").Required().String() +) + +func runCatCommand(context *kingpin.ParseContext) error { + vlt := mustOpenVault() + mgr, err := vlt.OpenRepository() + if err != nil { + return err + } + + oid, err := ParseObjectID(*catCommandPath, vlt) + if err != nil { + return err + } + r, err := mgr.Open(oid) + if err != nil { + return err + } + io.Copy(os.Stdout, r) + return nil +} + +func init() { + catCommand.Action(runCatCommand) +} diff --git a/cmd/kopia/command_create.go b/cmd/kopia/command_create.go index 25ee47d40..8f327055a 100644 --- a/cmd/kopia/command_create.go +++ b/cmd/kopia/command_create.go @@ -15,7 +15,7 @@ var ( createCommand = app.Command("create", "Create new vault and repository.") createCommandRepository = createCommand.Flag("repository", "Repository path.").Required().String() - createObjectFormat = createCommand.Flag("repo-format", "Format of repository objects.").PlaceHolder("FORMAT").Default("sha256t160-aes192").Enum(supportedObjectFormats()...) + createObjectFormat = createCommand.Flag("repo-format", "Format of repository objects.").PlaceHolder("FORMAT").Default("sha256-fold160-aes128").Enum(supportedObjectFormats()...) createMaxBlobSize = createCommand.Flag("max-blob-size", "Maximum size of a data chunk.").PlaceHolder("BYTES").Default("20000000").Int() createInlineBlobSize = createCommand.Flag("inline-blob-size", "Maximum size of an inline data chunk.").PlaceHolder("BYTES").Default("32768").Int() diff --git a/cmd/kopia/command_ls.go b/cmd/kopia/command_ls.go new file mode 100644 index 000000000..e411c90b0 --- /dev/null +++ b/cmd/kopia/command_ls.go @@ -0,0 +1,66 @@ +package main + +import ( + "fmt" + "strings" + + "github.com/kopia/kopia/fs" + + kingpin "gopkg.in/alecthomas/kingpin.v2" +) + +var ( + lsCommand = app.Command("ls", "List a directory stored in repository object.") + + lsCommandLong = lsCommand.Flag("long", "Long output").Short('l').Bool() + lsCommandPath = lsCommand.Arg("path", "Path").Required().String() +) + +func runLSCommand(context *kingpin.ParseContext) error { + vlt := mustOpenVault() + mgr, err := vlt.OpenRepository() + if err != nil { + return err + } + + oid, err := ParseObjectID(*lsCommandPath, vlt) + if err != nil { + return err + } + r, err := mgr.Open(oid) + if err != nil { + return err + } + + var prefix string + if !*lsCommandLong { + prefix = *lsCommandPath + if !strings.HasSuffix(prefix, "/") { + prefix += "/" + } + } + + dir, err := fs.ReadDirectory(r, "") + if err != nil { + return fmt.Errorf("unable to read directory contents") + } + + if *lsCommandLong { + listDirectory(dir) + } else { + for _, e := range dir { + var suffix string + if e.FileMode.IsDir() { + suffix = "/" + } + + fmt.Println(prefix + e.Name + suffix) + } + } + + return nil +} + +func init() { + lsCommand.Action(runLSCommand) +} diff --git a/cmd/kopia/command_mount.go b/cmd/kopia/command_mount.go index cebde1ee5..d2fa8a774 100644 --- a/cmd/kopia/command_mount.go +++ b/cmd/kopia/command_mount.go @@ -7,7 +7,6 @@ fusefs "bazil.org/fuse/fs" "github.com/kopia/kopia/fs" - "github.com/kopia/kopia/repo" "github.com/kopia/kopia/vfs" kingpin "gopkg.in/alecthomas/kingpin.v2" @@ -16,7 +15,7 @@ var ( mountCommand = app.Command("mount", "Mount repository object as a local filesystem.") - mountObjectID = mountCommand.Arg("objectID", "Directory to mount").Required().String() + mountObjectID = mountCommand.Arg("path", "Identifier of the directory to mount.").Required().String() mountPoint = mountCommand.Arg("mountPoint", "Mount point").Required().ExistingDir() ) @@ -29,7 +28,9 @@ func (r *root) Root() (fusefs.Node, error) { } func runMountCommand(context *kingpin.ParseContext) error { - r, err := mustOpenVault().OpenRepository() + vlt := mustOpenVault() + + r, err := vlt.OpenRepository() if err != nil { return err } @@ -44,11 +45,16 @@ func runMountCommand(context *kingpin.ParseContext) error { fuse.VolumeName("Kopia"), ) + oid, err := ParseObjectID(*mountObjectID, vlt) + if err != nil { + return err + } + fusefs.Serve(fuseConnection, &root{ mgr.NewNodeFromEntry(&fs.Entry{ Name: "", FileMode: os.ModeDir | 0555, - ObjectID: repo.ObjectID(*mountObjectID), + ObjectID: oid, }), }) diff --git a/cmd/kopia/command_show.go b/cmd/kopia/command_show.go index 165333226..12f5ab45d 100644 --- a/cmd/kopia/command_show.go +++ b/cmd/kopia/command_show.go @@ -1,33 +1,78 @@ package main import ( + "encoding/json" + "fmt" "io" "os" "github.com/kopia/kopia/repo" + "github.com/kopia/kopia/fs" + kingpin "gopkg.in/alecthomas/kingpin.v2" ) var ( showCommand = app.Command("show", "Show contents of a repository object.") - showObjectIDs = showCommand.Arg("objectID", "IDs of objects to show").Required().Strings() + showObjectIDs = showCommand.Arg("id", "IDs of objects to show").Required().Strings() + showJSON = showCommand.Flag("json", "Pretty-print JSON content").Short('j').Bool() + showDir = showCommand.Flag("dir", "Pretty-print directory content").Short('d').Bool() ) func runShowCommand(context *kingpin.ParseContext) error { - mgr, err := mustOpenVault().OpenRepository() + vlt := mustOpenVault() + mgr, err := vlt.OpenRepository() if err != nil { return err } - for _, oid := range *showObjectIDs { - r, err := mgr.Open(repo.ObjectID(oid)) + for _, oidString := range *showObjectIDs { + oid, err := ParseObjectID(oidString, vlt) + if err != nil { + return err + } + r, err := mgr.Open(oid) if err != nil { return err } - io.Copy(os.Stdout, r) + switch { + case *showJSON: + var v map[string]interface{} + + if err := json.NewDecoder(r).Decode(&v); err != nil { + return err + } + + m, err := json.MarshalIndent(v, "", " ") + if err != nil { + return err + } + os.Stdout.Write(m) + case *showDir: + dir, err := fs.ReadDirectory(r, "") + if err != nil { + return err + } + + for _, e := range dir { + var oid string + if e.ObjectID.Type().IsStored() { + oid = string(e.ObjectID) + } else if e.ObjectID.Type() == repo.ObjectIDTypeBinary { + oid = "" + } else if e.ObjectID.Type() == repo.ObjectIDTypeText { + oid = "" + } + info := fmt.Sprintf("%v %9d %v %-30s %v", e.FileMode, e.FileSize, e.ModTime.Local().Format("02 Jan 06 15:04:05"), e.Name, oid) + fmt.Println(info) + } + + default: + io.Copy(os.Stdout, r) + } } return nil diff --git a/cmd/kopia/command_vault.go b/cmd/kopia/command_vault.go new file mode 100644 index 000000000..0cbbc9e76 --- /dev/null +++ b/cmd/kopia/command_vault.go @@ -0,0 +1,5 @@ +package main + +var ( + vaultCommands = app.Command("vault", "Low-level commands to manipulate vault.") +) diff --git a/cmd/kopia/command_vault_list.go b/cmd/kopia/command_vault_list.go new file mode 100644 index 000000000..912445b6a --- /dev/null +++ b/cmd/kopia/command_vault_list.go @@ -0,0 +1,34 @@ +package main + +import ( + "fmt" + + "gopkg.in/alecthomas/kingpin.v2" +) + +var ( + vaultListCommand = vaultCommands.Command("list", "List contents of a vault") + vaultListPrefix = vaultListCommand.Flag("prefix", "Prefix").String() +) + +func init() { + vaultListCommand.Action(listVaultContents) +} + +func listVaultContents(context *kingpin.ParseContext) error { + v, err := openVault() + if err != nil { + return err + } + + entries, err := v.List(*vaultListPrefix) + if err != nil { + return err + } + + for _, e := range entries { + fmt.Println(e) + } + + return nil +} diff --git a/cmd/kopia/command_vault_show.go b/cmd/kopia/command_vault_show.go new file mode 100644 index 000000000..ab9524f3e --- /dev/null +++ b/cmd/kopia/command_vault_show.go @@ -0,0 +1,39 @@ +package main + +import ( + "encoding/json" + "os" + + "gopkg.in/alecthomas/kingpin.v2" +) + +var ( + vaultShowCommand = vaultCommands.Command("show", "Show contents of a vault item") + vaultShowID = vaultShowCommand.Arg("id", "ID of the vault item to show").String() +) + +func init() { + vaultShowCommand.Action(showVaultObject) +} + +func showVaultObject(context *kingpin.ParseContext) error { + v, err := openVault() + if err != nil { + return err + } + + var obj map[string]interface{} + + if err := v.Get(*vaultShowID, &obj); err != nil { + return err + } + + data, err := json.MarshalIndent(obj, "", " ") + if err != nil { + return err + } + + os.Stdout.Write(data) + + return nil +} diff --git a/cmd/kopia/objref.go b/cmd/kopia/objref.go new file mode 100644 index 000000000..cef97230f --- /dev/null +++ b/cmd/kopia/objref.go @@ -0,0 +1,73 @@ +package main + +import ( + "fmt" + "strings" + + "github.com/kopia/kopia/fs" + + "github.com/kopia/kopia/repo" + "github.com/kopia/kopia/vault" +) + +// ParseObjectID interprets the given ID string and returns corresponding repo.ObjectID. +// The string can be: +// - backupID (vxxxxxx/12312312) +// - +func ParseObjectID(id string, vlt vault.Reader) (repo.ObjectID, error) { + head, tail := splitHeadTail(id) + if len(head) == 0 { + return "", fmt.Errorf("invalid object ID: %v", id) + } + + oid, err := vlt.ResolveObjectID(head) + if err != nil { + return "", err + } + + if tail == "" { + return oid, nil + } + + r, err := vlt.OpenRepository() + if err != nil { + return "", fmt.Errorf("cannot open repository: %v", err) + } + + return parseNestedObjectID(oid, tail, r) +} + +func parseNestedObjectID(current repo.ObjectID, id string, r repo.Repository) (repo.ObjectID, error) { + head, tail := splitHeadTail(id) + for head != "" { + d, err := r.Open(current) + if err != nil { + return "", err + } + + dir, err := fs.ReadDirectory(d, "") + if err != nil { + return "", fmt.Errorf("entry not found '%v': parent is not a directory.", head) + } + + e := dir.FindByName(head) + if e == nil { + return "", fmt.Errorf("entry not found: '%v'", head) + } + + current = e.ObjectID + + head, tail = splitHeadTail(tail) + } + + return current, nil +} + +func splitHeadTail(id string) (string, string) { + p := strings.Index(id, "/") + if p < 0 { + return id, "" + } + + return id[:p], id[p+1:] +} diff --git a/cmd/kopia/output.go b/cmd/kopia/output.go new file mode 100644 index 000000000..815afa5e8 --- /dev/null +++ b/cmd/kopia/output.go @@ -0,0 +1,23 @@ +package main + +import ( + "fmt" + + "github.com/kopia/kopia/fs" + "github.com/kopia/kopia/repo" +) + +func listDirectory(dir fs.Directory) { + for _, e := range dir { + var oid string + if e.ObjectID.Type().IsStored() { + oid = string(e.ObjectID) + } else if e.ObjectID.Type() == repo.ObjectIDTypeBinary { + oid = "" + } else if e.ObjectID.Type() == repo.ObjectIDTypeText { + oid = "" + } + info := fmt.Sprintf("%v %9d %v %-30s %v", e.FileMode, e.FileSize, e.ModTime.Local().Format("02 Jan 06 15:04:05"), e.Name, oid) + fmt.Println(info) + } +} diff --git a/cmd/kopia/urls.go b/cmd/kopia/urls.go index 93369eeb1..13f4e5906 100644 --- a/cmd/kopia/urls.go +++ b/cmd/kopia/urls.go @@ -2,7 +2,6 @@ import ( "fmt" - "log" "net/url" "os" "strconv" @@ -12,7 +11,6 @@ ) func newStorageFromURL(urlString string) (blob.Storage, error) { - log.Printf("NewStorageFromURl: %v", urlString) if strings.HasPrefix(urlString, "/") { urlString = "file://" + urlString } @@ -31,7 +29,7 @@ func newStorageFromURL(urlString string) (blob.Storage, error) { return blob.NewFSStorage(&fso) - case "gcs": + case "gs", "gcs": var gcso blob.GCSStorageOptions if err := parseGoogleCloudStorageURL(&gcso, u); err != nil { return nil, err @@ -44,7 +42,6 @@ func newStorageFromURL(urlString string) (blob.Storage, error) { } func parseFilesystemURL(fso *blob.FSStorageOptions, u *url.URL) error { - log.Printf("u.Upaque: %v u.Path: %v", u.Opaque, u.Path) if u.Opaque != "" { fso.Path = u.Opaque } else { diff --git a/doc/quickstart.md b/doc/quickstart.md index 122ad4509..5dee80de1 100644 --- a/doc/quickstart.md +++ b/doc/quickstart.md @@ -1,7 +1,7 @@ Quick Start === -Kopia is a simple tool for managing encrypted backups in the cloud. +Kopia is a simple command-line tool for managing encrypted backups in the cloud. Key Concepts --- diff --git a/fs/dir_json.go b/fs/dir_json.go index ddacc7a48..794092e4f 100644 --- a/fs/dir_json.go +++ b/fs/dir_json.go @@ -153,7 +153,7 @@ func (dw *directoryWriter) WriteEntry(e *Entry) error { dw.writer.Write(dw.separator) dw.writer.Write(v) - dw.separator = []byte(",\n ") + dw.separator = []byte(",") return nil } @@ -177,7 +177,7 @@ func formatModeAndPermissions(m os.FileMode) string { } func (dw *directoryWriter) Close() error { - dw.writer.Write([]byte("\n]}\n")) + dw.writer.Write([]byte("]}")) return nil } @@ -196,11 +196,11 @@ func newDirectoryWriter(w io.Writer) *directoryWriter { var f directoryFormat f.Version = 1 - io.WriteString(w, "{\n\"format\":") + io.WriteString(w, "{\"format\":") b, _ := json.Marshal(&f) w.Write(b) - io.WriteString(w, ",\n\"entries\":[") - dw.separator = []byte("\n ") + io.WriteString(w, ",\"entries\":[") + dw.separator = []byte("") return dw } diff --git a/fs/dir_test.go b/fs/dir_test.go index c642215b3..29589c7e0 100644 --- a/fs/dir_test.go +++ b/fs/dir_test.go @@ -12,13 +12,13 @@ func TestDirectory(t *testing.T) { `{`, `"format":{"version":1},`, `"entries":[`, - ` {"name":"config.go","mode":"420","size":"937","modTime":"2016-04-02T02:39:44.123456789Z","owner":"500:100","oid":"C4321"},`, - ` {"name":"constants.go","mode":"420","size":"13","modTime":"2016-04-02T02:36:19Z","owner":"500:100"},`, - ` {"name":"doc.go","mode":"420","size":"112","modTime":"2016-04-02T02:45:54Z","owner":"500:100"},`, - ` {"name":"errors.go","mode":"420","size":"506","modTime":"2016-04-02T02:41:03Z","owner":"500:100"},`, - ` {"name":"subdir","mode":"d:420","modTime":"2016-04-06T02:34:10Z","owner":"500:100","oid":"C1234"}`, + `{"name":"config.go","mode":"420","size":"937","modTime":"2016-04-02T02:39:44.123456789Z","owner":"500:100","oid":"C4321"},`, + `{"name":"constants.go","mode":"420","size":"13","modTime":"2016-04-02T02:36:19Z","owner":"500:100"},`, + `{"name":"doc.go","mode":"420","size":"112","modTime":"2016-04-02T02:45:54Z","owner":"500:100"},`, + `{"name":"errors.go","mode":"420","size":"506","modTime":"2016-04-02T02:41:03Z","owner":"500:100"},`, + `{"name":"subdir","mode":"d:420","modTime":"2016-04-06T02:34:10Z","owner":"500:100","oid":"C1234"}`, `]}`, - }, "\n") + "\n" + }, "") d, err := ReadDirectory(strings.NewReader(data), "") if err != nil { diff --git a/fs/upload.go b/fs/upload.go index 5362e5529..a3897ea48 100644 --- a/fs/upload.go +++ b/fs/upload.go @@ -38,6 +38,7 @@ type UploadResult struct { // Uploader supports efficient uploading files and directories to repository. type Uploader interface { UploadDir(path string, previousManifestID repo.ObjectID) (*UploadResult, error) + UploadFile(path string) (*UploadResult, error) Cancel() } @@ -52,7 +53,7 @@ func (u *uploader) isCancelled() bool { return atomic.LoadInt32(&u.cancelled) != 0 } -func (u *uploader) uploadFile(path string, e *Entry) (*Entry, uint64, error) { +func (u *uploader) uploadFileInternal(path string, forceStored bool) (*Entry, uint64, error) { log.Printf("Uploading file %v", path) file, err := u.lister.Open(path) if err != nil { @@ -75,7 +76,7 @@ func (u *uploader) uploadFile(path string, e *Entry) (*Entry, uint64, error) { return nil, 0, err } - e2.ObjectID, err = writer.Result(false) + e2.ObjectID, err = writer.Result(forceStored) if err != nil { return nil, 0, err } @@ -87,6 +88,13 @@ func (u *uploader) uploadFile(path string, e *Entry) (*Entry, uint64, error) { return e2, e2.metadataHash(), nil } +func (u *uploader) UploadFile(path string) (*UploadResult, error) { + result := &UploadResult{} + e, _, err := u.uploadFileInternal(path, true) + result.ObjectID = e.ObjectID + return result, err +} + func (u *uploader) UploadDir(path string, previousManifestID repo.ObjectID) (*UploadResult, error) { //log.Printf("UploadDir", path) //defer log.Printf("finishing UploadDir", path) @@ -107,7 +115,7 @@ func (u *uploader) UploadDir(path string, previousManifestID repo.ObjectID) (*Up hcw := newHashCacheWriter(mw) result := &UploadResult{} - result.ObjectID, _, _, err = u.uploadDirInternal(result, path, ".", hcw, &mr) + result.ObjectID, _, _, err = u.uploadDirInternal(result, path, ".", hcw, &mr, true) if err != nil { return result, err } @@ -122,6 +130,7 @@ func (u *uploader) uploadDirInternal( relativePath string, hcw *hashcacheWriter, mr *hashcacheReader, + forceStored bool, ) (repo.ObjectID, uint64, bool, error) { log.Printf("Uploading dir %v", path) defer log.Printf("Finished uploading dir %v", path) @@ -151,7 +160,7 @@ func (u *uploader) uploadDirInternal( switch e.FileMode & os.ModeType { case os.ModeDir: - oid, h, wasCached, err := u.uploadDirInternal(result, fullPath, entryRelativePath, hcw, mr) + oid, h, wasCached, err := u.uploadDirInternal(result, fullPath, entryRelativePath, hcw, mr, false) if err != nil { return "", 0, false, err } @@ -186,7 +195,7 @@ func (u *uploader) uploadDirInternal( hash = cachedEntry.Hash } else { result.Stats.NonCachedFiles++ - e, hash, err = u.uploadFile(fullPath, e) + e, hash, err = u.uploadFileInternal(fullPath, false) if err != nil { return "", 0, false, fmt.Errorf("unable to hash file: %s", err) } @@ -231,7 +240,7 @@ func (u *uploader) uploadDirInternal( directoryOID = repo.ObjectID(cachedDirEntry.ObjectID) } else { result.Stats.NonCachedDirectories++ - directoryOID, err = writer.Result(true) + directoryOID, err = writer.Result(forceStored) if err != nil { return directoryOID, 0, false, err } diff --git a/repo/format.go b/repo/format.go index 2b8c3a1e5..9be0be294 100644 --- a/repo/format.go +++ b/repo/format.go @@ -9,9 +9,6 @@ "crypto/sha256" "crypto/sha512" "hash" - "io" - - "golang.org/x/crypto/hkdf" ) var ( @@ -51,11 +48,22 @@ func (fmts ObjectIDFormats) Find(name string) *ObjectIDFormat { return nil } +func fold(b []byte, size int) []byte { + if len(b) == size { + return b + } + + for i := size; i < len(b); i++ { + b[i%size] ^= b[i] + } + return b[0:size] +} + func nonEncryptedFormat(name string, hf func() hash.Hash, hashSize int) *ObjectIDFormat { return &ObjectIDFormat{ Name: name, hashBuffer: func(data []byte, secret []byte) ([]byte, []byte) { - return hashContent(hf, data, secret)[0:hashSize], nil + return fold(hashContent(hf, data, secret), hashSize), nil }, } } @@ -83,15 +91,8 @@ func encryptedFormat( Name: name, hashBuffer: func(data []byte, secret []byte) ([]byte, []byte) { contentHash := hashContent(sha512.New, data, secret) - - s1 := hkdf.New(sha256.New, contentHash, nil, hkdfInfoBlockID) - blockID := make([]byte, blockIDSize) - io.ReadFull(s1, blockID) - - s2 := hkdf.New(sha256.New, contentHash, nil, hkdfInfoCryptoKey) - cryptoKey := make([]byte, keySize) - io.ReadFull(s2, cryptoKey) - + blockID := fold(hashContent(hf, hkdfInfoBlockID, contentHash), blockIDSize) + cryptoKey := fold(hashContent(hf, hkdfInfoCryptoKey, contentHash), keySize) return blockID, cryptoKey }, createCipher: createCipher, @@ -110,11 +111,11 @@ func buildObjectIDFormats() ObjectIDFormats { {"sha1", sha1.New, sha1.Size}, {"sha224", sha256.New224, sha256.Size224}, {"sha256", sha256.New, sha256.Size}, - {"sha256t128", sha256.New, 16}, - {"sha256t160", sha256.New, 20}, + {"sha256-fold128", sha256.New, 16}, + {"sha256-fold160", sha256.New, 20}, {"sha384", sha512.New384, sha512.Size384}, - {"sha512t128", sha512.New, 16}, - {"sha512t160", sha512.New, 20}, + {"sha512-fold128", sha512.New, 16}, + {"sha512-fold160", sha512.New, 20}, {"sha512-224", sha512.New512_224, sha512.Size224}, {"sha512-256", sha512.New512_256, sha512.Size256}, {"sha512", sha512.New, sha512.Size}, diff --git a/repo/repository_test.go b/repo/repository_test.go index cdcbc5c45..219f55111 100644 --- a/repo/repository_test.go +++ b/repo/repository_test.go @@ -445,27 +445,33 @@ func TestFormats(t *testing.T) { }, }, { - format: makeFormat("sha256t128-aes128"), + format: makeFormat("sha256-fold128-aes128"), oids: map[string]ObjectID{ - "The quick brown fox jumps over the lazy dog": "D4e43650cb2ce7b5a6a2a8d614c13d9b3.f73b9fed1f355b3603e066fb24a39970", + "The quick brown fox jumps over the lazy dog": "D028e74c630aee6400db4f84d49d1ed08.2825273c3344999e26081d99bcaf3f18", }, }, { format: makeFormat("sha256-aes128"), oids: map[string]ObjectID{ - "The quick brown fox jumps over the lazy dog": "D4e43650cb2ce7b5a6a2a8d614c13d9b37c6ee13c4543481074d2c8cf3f597a13.f73b9fed1f355b3603e066fb24a39970", + "The quick brown fox jumps over the lazy dog": "D26bb3fee9e83a7f6ff8397f33460cd0d24354b28ae2d41b6f2376fbe7db12005.2825273c3344999e26081d99bcaf3f18", }, }, { format: makeFormat("sha384-aes256"), oids: map[string]ObjectID{ - "The quick brown fox jumps over the lazy dog": "D4e43650cb2ce7b5a6a2a8d614c13d9b37c6ee13c4543481074d2c8cf3f597a136dba4c0e67d5d644049c98ee5369bc0a.f73b9fed1f355b3603e066fb24a39970ac10cbd143babc7f487ef263fe38aeff", + "The quick brown fox jumps over the lazy dog": "D08edca658caee5ae86c9f1e4ccea26b1a43b0ddb04502c5f1b1cba051a756a9eba440bb6991b23323e37556ec5cf4f88.7a753377319650870f5cba47c30b608675542f206fa67d94dcbb9a0ccce7f467", }, }, { format: makeFormat("sha512-aes256"), oids: map[string]ObjectID{ - "The quick brown fox jumps over the lazy dog": "D4e43650cb2ce7b5a6a2a8d614c13d9b37c6ee13c4543481074d2c8cf3f597a136dba4c0e67d5d644049c98ee5369bc0aaa0e75feaeadd42eb54a9d64f9d0a51d.f73b9fed1f355b3603e066fb24a39970ac10cbd143babc7f487ef263fe38aeff", + "The quick brown fox jumps over the lazy dog": "Deee04ad71248ef4ffec5159558e80e2791e32299cfb79a1e063cc5f4a71cd61ca2c1b110e3ae2e83635a8626a3a27e4805eb745c40b5a4ebd4c9372602e5ab65.1c0b1b58ce05b7b8b05cfce27a485ddf97bde5159f6946357ec7795236f36a84", + }, + }, + { + format: makeFormat("sha512-fold128-aes192"), + oids: map[string]ObjectID{ + "The quick brown fox jumps over the lazy dog": "Dd829ad027ee4ff394f6a61615eb30d16.0ab00e0e0e156927f0be63372e8449d7bf2316c43d46e626", }, }, } diff --git a/vault/reader.go b/vault/reader.go new file mode 100644 index 000000000..1e96a83be --- /dev/null +++ b/vault/reader.go @@ -0,0 +1,12 @@ +package vault + +import "github.com/kopia/kopia/repo" + +// Reader allows reading from a vault. +type Reader interface { + Get(id string, content interface{}) error + GetRaw(id string) ([]byte, error) + List(prefix string) ([]string, error) + ResolveObjectID(id string) (repo.ObjectID, error) + OpenRepository() (repo.Repository, error) +} diff --git a/vault/vault.go b/vault/vault.go index d3cd1130f..56447c363 100644 --- a/vault/vault.go +++ b/vault/vault.go @@ -8,11 +8,13 @@ "crypto/rand" "crypto/sha256" "encoding/base64" + "encoding/hex" "encoding/json" "fmt" "hash" "io" "io/ioutil" + "strings" "github.com/kopia/kopia/blob" "github.com/kopia/kopia/repo" @@ -24,6 +26,9 @@ formatBlock = "format" checksumBlock = "checksum" repositoryConfigBlock = "repo" + + storedObjectIDPrefix = "v" + storedObjectIDLengthBytes = 8 ) var ( @@ -193,6 +198,10 @@ func (v *Vault) OpenRepository() (repo.Repository, error) { return repo.NewRepository(storage, rc.Format) } +func (v *Vault) GetRaw(id string) ([]byte, error) { + return v.readEncryptedBlock(id) +} + // Get deserializes JSON data stored in the vault into the specified content structure. func (v *Vault) Get(id string, content interface{}) error { j, err := v.readEncryptedBlock(id) @@ -245,6 +254,55 @@ func (v *Vault) Token() (string, error) { return base64.RawURLEncoding.EncodeToString(b), nil } +type objectIDData struct { + ObjectID string `json:"objectID"` +} + +func (v *Vault) SaveObjectID(oid repo.ObjectID) (string, error) { + h := hmac.New(sha256.New, v.format.UniqueID) + h.Write([]byte(oid)) + sum := h.Sum(nil) + for i := storedObjectIDLengthBytes; i < len(sum); i++ { + sum[i%storedObjectIDLengthBytes] ^= sum[i] + } + sum = sum[0:storedObjectIDLengthBytes] + key := storedObjectIDPrefix + hex.EncodeToString(sum) + + var d objectIDData + d.ObjectID = string(oid) + + if err := v.Put(key, &d); err != nil { + return "", err + } + + return key, nil +} + +func (v *Vault) ResolveObjectID(id string) (repo.ObjectID, error) { + if !strings.HasPrefix(id, storedObjectIDPrefix) { + return repo.ParseObjectID(id) + } + + matches, err := v.List(id) + if err != nil { + return "", err + } + + switch len(matches) { + case 0: + return "", fmt.Errorf("object not found: %v", id) + case 1: + var d objectIDData + if err := v.Get(matches[0], &d); err != nil { + return "", err + } + return repo.ParseObjectID(d.ObjectID) + + default: + return "", fmt.Errorf("ambiguous object ID: %v", id) + } +} + // Create creates a Vault in the specified storage. func Create(storage blob.Storage, format *Format, creds Credentials) (*Vault, error) { if err := format.ensureUniqueID(); err != nil {