more commands

README.md

@@ -0,0 +1,121 @@
+Kopia
+=====
+
+> _n. thing exactly replicated from the original (Polish)_
+
+Kopia is a simple, cross-platform tool for managing encrypted backups in the cloud. It provides fast, incremental backups, secure, client-side encryption and data deduplication.
+
+Unlike other cloud backup solutions, the user in full control of backup storage and is responsible for purchasing one of the cloud storage products  (such as [Google Cloud Storage](https://cloud.google.com/storage/)), which offer great durability and availability for your data.
+
+> **NOTICE**:
+>
+> Kopia is still in early stages of development and is not ready for general use.
+> The repository and vault data format are subject to change, including backwards-incompatible changes. Use at your own risk.
+
+Installation
+---
+
+To build Kopia you need the latest version of [Go](https://golang.org/dl/) and run the following commands:
+
+```
+mkdir $HOME/kopia
+export GOPATH=$HOME/kopia
+go get github.com/kopia/kopia
+go install github.com/kopia/kopia/cmd/kopia
+```
+
+This will place Kopia binary in `$HOME/kopia/bin/kopia`. For convenience it's best to place it in a directory it the system `PATH`.
+
+Setting up repository and vault
+---
+
+Repository is where the bulk of the backup data will be stored and which can be shared among users. Vault is a small, password-protected area that stores list of backups of a single user.
+
+Vault and repository can be stored on:
+
+- local filesystem paths
+- Google Cloud Storage buckets, for example `gs://my-bucket`
+
+For example, to create a vault on a local USB drive and repository in Google Cloud Storage use:
+
+```
+$ kopia create --vault /mnt/my-usb-drive --repository gs://my-bucket
+Enter password to create new vault: ***********
+Re-enter password for verification: ***********
+Connected to vault: /mnt/my-usb-drive
+```
+
+The vault password is cached in a local file, so you don't need to enter it all the time.
+To disconnect from the vault and remove cached password use:
+```
+$ kopia disconnect
+Disconnected from vault.
+```
+
+To connect to an existing vault:
+```
+$ kopia connect --vault /mnt/my-usb-drive
+Enter password to open vault: ***********
+Connected to vault: /mnt/my-usb-drive
+```
+
+Backup and Restore
+---
+
+To create a backup of a directory or file, use `kopia backup <path>`. It willwill print the identifier of a backup, which is a long string, that can be used to restore the file later. Because data in a repository is content-addressable, two files with identical contents, even in different directories or on different machines, will get the same backup identifier.
+
+```
+$ kopia backup /path/to/dir
+D9691a95c5f9a73a1decf493f8f0d79.309a95f17bc3c6d3272bd0a62d2
+$ kopia backup /path/to/dir
+D9691a95c5f9a73a1decf493f8f0d79.309a95f17bc3c6d3272bd0a62d2
+```
+
+To list all backups of a particular directory or file, use:
+```
+$ kopia backups <path>
+```
+
+To list all backups stored in a vault, use:
+```
+$ kopia backups --all
+```
+
+In order to browse the contents of a backup you can mount it in a local filesystem using:
+
+```
+$ kopia mount <backup-identifier> <mount-path>
+```
+
+To unmount, use:
+```
+$ umount <mount-path>
+```
+
+You can also show the contents of an single object in a repository by using:
+```
+$ kopia show <backup-identifier>
+```
+
+
+Cryptography Notice
+---
+
+  This distribution includes cryptographic software. The country in
+  which you currently reside may have restrictions on the import,
+  possession, use, and/or re-export to another country, of encryption
+  software. BEFORE using any encryption software, please check your
+  country's laws, regulations and policies concerning the import,
+  possession, or use, and re-export of encryption software, to see if
+  this is permitted. See <http://www.wassenaar.org/> for more
+  information.
+
+  The U.S. Government Department of Commerce, Bureau of Industry and
+  Security (BIS), has classified this software as Export Commodity
+  Control Number (ECCN) 5D002.C.1, which includes information security
+  software using or performing cryptographic functions with symmetric
+  algorithms. The form and manner of this distribution makes it
+  eligible for export under the License Exception ENC Technology
+  Software Unrestricted (TSU) exception (see the BIS Export
+  Administration Regulations, Section 740.13) for both object code and
+  source code.

backup/generator.go

@@ -1,7 +1,8 @@
 package backup
 
 import (
-	"log"
+	"fmt"
+	"os"
 	"time"
 
 	"github.com/kopia/kopia/fs"
@@ -32,18 +33,28 @@ func (bg *backupGenerator) Backup(m *Manifest, old *Manifest) error {
 
 	if old != nil {
 		hashCacheID = repo.ObjectID(old.HashCacheID)
-		log.Printf("Using hash cache ID: %v", hashCacheID)
-	} else {
-		log.Printf("No hash cache.")
 	}
-	r, err := uploader.UploadDir(m.SourceDirectory, hashCacheID)
+
+	st, err := os.Stat(m.Source)
 	if err != nil {
 		return err
 	}
 
+	var r *fs.UploadResult
+	switch st.Mode() & os.ModeType {
+	case os.ModeDir:
+		r, err = uploader.UploadDir(m.Source, hashCacheID)
+	case 0: // regular
+		r, err = uploader.UploadFile(m.Source)
+	default:
+		return fmt.Errorf("unsupported source: %v", m.Source)
+	}
+	m.EndTime = time.Now()
+	if err != nil {
+		return err
+	}
 	m.RootObjectID = string(r.ObjectID)
 	m.HashCacheID = string(r.ManifestID)
-	m.EndTime = time.Now()
 
 	return nil
 }

backup/manifest.go

@@ -16,9 +16,10 @@ type Manifest struct {
 	UserName    string `json:"userName"`
 	Description string `json:"description"`
 
-	SourceDirectory string `json:"source"`
-	RootObjectID    string `json:"root"`
-	HashCacheID     string `json:"hashCache"`
+	Handle       string `json:"handle"`
+	Source       string `json:"source"`
+	RootObjectID string `json:"root"`
+	HashCacheID  string `json:"hashCache"`
 
 	FileCount      int64 `json:"fileCount"`
 	DirectoryCount int64 `json:"dirCount"`
@@ -33,7 +34,7 @@ func (m Manifest) SourceID() string {
 	h.Write(zeroByte)
 	io.WriteString(h, m.UserName)
 	h.Write(zeroByte)
-	io.WriteString(h, m.SourceDirectory)
+	io.WriteString(h, m.Source)
 	h.Write(zeroByte)
 	return hex.EncodeToString(h.Sum(nil))
 }

cmd/kopia/command_backup.go

@@ -2,7 +2,10 @@
 
 import (
 	"crypto/rand"
+	"crypto/sha256"
+	"encoding/hex"
 	"fmt"
+	"io"
 	"log"
 	"math"
 	"os"
@@ -22,9 +25,9 @@
 )
 
 var (
-	backupCommand = app.Command("backup", "Copies local directory to backup repository.")
+	backupCommand = app.Command("backup", "Copies local files or directories to backup repository.")
 
-	backupDirectories = backupCommand.Arg("directory", "Directories to back up").Required().ExistingDirs()
+	backupSources = backupCommand.Arg("source", "Files or directories to back up.").Required().ExistingFilesOrDirs()
 
 	backupHostName    = backupCommand.Flag("host", "Override backup hostname.").String()
 	backupUser        = backupCommand.Flag("user", "Override backup user.").String()
@@ -66,15 +69,15 @@ func runBackupCommand(context *kingpin.ParseContext) error {
 		return err
 	}
 
-	for _, backupDirectory := range *backupDirectories {
+	for _, backupDirectory := range *backupSources {
 		dir, err := filepath.Abs(backupDirectory)
 		if err != nil {
-			return fmt.Errorf("invalid directory: '%s': %s", backupDirectory, err)
+			return fmt.Errorf("invalid source: '%s': %s", backupDirectory, err)
 		}
 
 		manifest := backup.Manifest{
-			StartTime:       time.Now(),
-			SourceDirectory: filepath.Clean(dir),
+			StartTime: time.Now(),
+			Source:    filepath.Clean(dir),
 
 			HostName:    getBackupHostName(),
 			UserName:    getBackupUser(),
@@ -100,25 +103,43 @@ func runBackupCommand(context *kingpin.ParseContext) error {
 			oldManifest = &m
 		}
 
-		uniqueID := make([]byte, 8)
-		rand.Read(uniqueID)
-		fileID := fmt.Sprintf("B%v.%08x.%x", manifest.SourceID(), math.MaxInt64-manifest.StartTime.UnixNano(), uniqueID)
-
 		if err := bgen.Backup(&manifest, oldManifest); err != nil {
 			return err
 		}
 
+		handleID, err := vlt.SaveObjectID(repo.ObjectID(manifest.RootObjectID))
+		if err != nil {
+			return err
+		}
+
+		uniqueID := make([]byte, 8)
+		rand.Read(uniqueID)
+		fileID := fmt.Sprintf("B%v.%08x.%x", manifest.SourceID(), math.MaxInt64-manifest.StartTime.UnixNano(), uniqueID)
+		manifest.Handle = handleID
+
 		err = vlt.Put(fileID, &manifest)
 		if err != nil {
 			return fmt.Errorf("cannot save manifest: %v", err)
 		}
 
 		log.Printf("Root: %v", manifest.RootObjectID)
+		log.Printf("Key: %v", handleID)
 	}
 
 	return nil
 }
 
+func hashObjectID(oid string) string {
+	h := sha256.New()
+	io.WriteString(h, oid)
+	sum := h.Sum(nil)
+	foldLen := 16
+	for i := foldLen; i < len(sum); i++ {
+		sum[i%foldLen] ^= sum[i]
+	}
+	return hex.EncodeToString(sum[0:foldLen])
+}
+
 func getBackupUser() string {
 	if *backupUser != "" {
 		return *backupUser

cmd/kopia/command_backups.go

@@ -7,17 +7,56 @@
 
 	"github.com/kopia/kopia/backup"
 	"github.com/kopia/kopia/repo"
+	"github.com/kopia/kopia/vault"
 
 	kingpin "gopkg.in/alecthomas/kingpin.v2"
 )
 
 var (
-	backupsCommand    = app.Command("backups", "List backup history.")
-	backupsDirectory  = backupsCommand.Arg("directory", "Directory to show history of").ExistingDir()
-	backupsAll        = backupsCommand.Flag("all", "Show history of all backups.").Bool()
+	backupsCommand    = app.Command("backups", "List history of file or directory backups.")
+	backupsPath       = backupsCommand.Arg("source", "File or directory to show history of.").String()
 	maxResultsPerPath = backupsCommand.Flag("maxresults", "Maximum number of results.").Default("100").Int()
 )
 
+func findBackups(vlt *vault.Vault, path string) ([]string, string, error) {
+	var relPath string
+
+	for len(path) > 0 {
+		manifest := backup.Manifest{
+			Source:   path,
+			HostName: getBackupHostName(),
+			UserName: getBackupUser(),
+		}
+
+		prefix := manifest.SourceID() + "."
+
+		list, err := vlt.List("B" + prefix)
+		if err != nil {
+			return nil, "", err
+		}
+
+		if len(list) > 0 {
+			return list, relPath, nil
+		}
+
+		if len(relPath) > 0 {
+			relPath = filepath.Base(path) + "/" + relPath
+		} else {
+			relPath = filepath.Base(path)
+		}
+
+		log.Printf("No backups of %v@%v:%v", manifest.UserName, manifest.HostName, manifest.Source)
+
+		parent := filepath.Dir(path)
+		if parent == path {
+			break
+		}
+		path = parent
+	}
+
+	return nil, "", nil
+}
+
 func runBackupsCommand(context *kingpin.ParseContext) error {
 	var options []repo.RepositoryOption
 
@@ -41,31 +80,30 @@ func runBackupsCommand(context *kingpin.ParseContext) error {
 	}
 	defer mgr.Close()
 
-	var prefix string
+	var previous []string
+	var relPath string
 
-	if !*backupsAll {
-
-		dir, err := filepath.Abs(*backupsDirectory)
+	if *backupsPath != "" {
+		path, err := filepath.Abs(*backupsPath)
 		if err != nil {
-			return fmt.Errorf("invalid directory: '%s': %s", *backupsDirectory, err)
+			return fmt.Errorf("invalid directory: '%s': %s", *backupsPath, err)
 		}
 
-		manifest := backup.Manifest{
-			SourceDirectory: filepath.Clean(dir),
-			HostName:        getBackupHostName(),
-			UserName:        getBackupUser(),
+		previous, relPath, err = findBackups(vlt, filepath.Clean(path))
+		if relPath != "" {
+			relPath = "/" + relPath
 		}
-		prefix = manifest.SourceID() + "."
+	} else {
+		previous, err = vlt.List("B")
 	}
 
-	previous, err := vlt.List("B" + prefix)
 	if err != nil {
-		return fmt.Errorf("error listing previous backups")
+		return fmt.Errorf("cannot list backups: %v", err)
 	}
 
 	var lastHost string
 	var lastUser string
-	var lastDir string
+	var lastSource string
 	var count int
 
 	for _, n := range previous {
@@ -74,16 +112,16 @@ func runBackupsCommand(context *kingpin.ParseContext) error {
 			return fmt.Errorf("error loading previous backup: %v", err)
 		}
 
-		if m.HostName != lastHost || m.UserName != lastUser || m.SourceDirectory != lastDir {
-			log.Printf("%v @ %v : %v", m.UserName, m.HostName, m.SourceDirectory)
-			lastDir = m.SourceDirectory
+		if m.HostName != lastHost || m.UserName != lastUser || m.Source != lastSource {
+			log.Printf("%v@%v:%v", m.UserName, m.HostName, m.Source)
+			lastSource = m.Source
 			lastUser = m.UserName
 			lastHost = m.HostName
 			count = 0
 		}
 
 		if count < *maxResultsPerPath {
-			log.Printf("  %v %v", m.RootObjectID, m.StartTime.Format("2006-01-02 15:04:05 MST"))
+			log.Printf("  %v%v %v", m.Handle, relPath, m.StartTime.Format("2006-01-02 15:04:05 MST"))
 			count++
 		}
 	}

cmd/kopia/command_cat.go

@@ -0,0 +1,36 @@
+package main
+
+import (
+	"io"
+	"os"
+
+	kingpin "gopkg.in/alecthomas/kingpin.v2"
+)
+
+var (
+	catCommand     = app.Command("cat", "Displays contents of a repository object.")
+	catCommandPath = catCommand.Arg("path", "Path").Required().String()
+)
+
+func runCatCommand(context *kingpin.ParseContext) error {
+	vlt := mustOpenVault()
+	mgr, err := vlt.OpenRepository()
+	if err != nil {
+		return err
+	}
+
+	oid, err := ParseObjectID(*catCommandPath, vlt)
+	if err != nil {
+		return err
+	}
+	r, err := mgr.Open(oid)
+	if err != nil {
+		return err
+	}
+	io.Copy(os.Stdout, r)
+	return nil
+}
+
+func init() {
+	catCommand.Action(runCatCommand)
+}

cmd/kopia/command_create.go

@@ -15,7 +15,7 @@
 var (
 	createCommand           = app.Command("create", "Create new vault and repository.")
 	createCommandRepository = createCommand.Flag("repository", "Repository path.").Required().String()
-	createObjectFormat      = createCommand.Flag("repo-format", "Format of repository objects.").PlaceHolder("FORMAT").Default("sha256t160-aes192").Enum(supportedObjectFormats()...)
+	createObjectFormat      = createCommand.Flag("repo-format", "Format of repository objects.").PlaceHolder("FORMAT").Default("sha256-fold160-aes128").Enum(supportedObjectFormats()...)
 
 	createMaxBlobSize           = createCommand.Flag("max-blob-size", "Maximum size of a data chunk.").PlaceHolder("BYTES").Default("20000000").Int()
 	createInlineBlobSize        = createCommand.Flag("inline-blob-size", "Maximum size of an inline data chunk.").PlaceHolder("BYTES").Default("32768").Int()

cmd/kopia/command_ls.go

@@ -0,0 +1,66 @@
+package main
+
+import (
+	"fmt"
+	"strings"
+
+	"github.com/kopia/kopia/fs"
+
+	kingpin "gopkg.in/alecthomas/kingpin.v2"
+)
+
+var (
+	lsCommand = app.Command("ls", "List a directory stored in repository object.")
+
+	lsCommandLong = lsCommand.Flag("long", "Long output").Short('l').Bool()
+	lsCommandPath = lsCommand.Arg("path", "Path").Required().String()
+)
+
+func runLSCommand(context *kingpin.ParseContext) error {
+	vlt := mustOpenVault()
+	mgr, err := vlt.OpenRepository()
+	if err != nil {
+		return err
+	}
+
+	oid, err := ParseObjectID(*lsCommandPath, vlt)
+	if err != nil {
+		return err
+	}
+	r, err := mgr.Open(oid)
+	if err != nil {
+		return err
+	}
+
+	var prefix string
+	if !*lsCommandLong {
+		prefix = *lsCommandPath
+		if !strings.HasSuffix(prefix, "/") {
+			prefix += "/"
+		}
+	}
+
+	dir, err := fs.ReadDirectory(r, "")
+	if err != nil {
+		return fmt.Errorf("unable to read directory contents")
+	}
+
+	if *lsCommandLong {
+		listDirectory(dir)
+	} else {
+		for _, e := range dir {
+			var suffix string
+			if e.FileMode.IsDir() {
+				suffix = "/"
+			}
+
+			fmt.Println(prefix + e.Name + suffix)
+		}
+	}
+
+	return nil
+}
+
+func init() {
+	lsCommand.Action(runLSCommand)
+}

cmd/kopia/command_mount.go

@@ -7,7 +7,6 @@
 	fusefs "bazil.org/fuse/fs"
 
 	"github.com/kopia/kopia/fs"
-	"github.com/kopia/kopia/repo"
 	"github.com/kopia/kopia/vfs"
 
 	kingpin "gopkg.in/alecthomas/kingpin.v2"
@@ -16,7 +15,7 @@
 var (
 	mountCommand = app.Command("mount", "Mount repository object as a local filesystem.")
 
-	mountObjectID = mountCommand.Arg("objectID", "Directory to mount").Required().String()
+	mountObjectID = mountCommand.Arg("path", "Identifier of the directory to mount.").Required().String()
 	mountPoint    = mountCommand.Arg("mountPoint", "Mount point").Required().ExistingDir()
 )
 
@@ -29,7 +28,9 @@ func (r *root) Root() (fusefs.Node, error) {
 }
 
 func runMountCommand(context *kingpin.ParseContext) error {
-	r, err := mustOpenVault().OpenRepository()
+	vlt := mustOpenVault()
+
+	r, err := vlt.OpenRepository()
 	if err != nil {
 		return err
 	}
@@ -44,11 +45,16 @@ func runMountCommand(context *kingpin.ParseContext) error {
 		fuse.VolumeName("Kopia"),
 	)
 
+	oid, err := ParseObjectID(*mountObjectID, vlt)
+	if err != nil {
+		return err
+	}
+
 	fusefs.Serve(fuseConnection, &root{
 		mgr.NewNodeFromEntry(&fs.Entry{
 			Name:     "<root>",
 			FileMode: os.ModeDir | 0555,
-			ObjectID: repo.ObjectID(*mountObjectID),
+			ObjectID: oid,
 		}),
 	})
 

cmd/kopia/command_show.go

@@ -1,33 +1,78 @@
 package main
 
 import (
+	"encoding/json"
+	"fmt"
 	"io"
 	"os"
 
 	"github.com/kopia/kopia/repo"
 
+	"github.com/kopia/kopia/fs"
+
 	kingpin "gopkg.in/alecthomas/kingpin.v2"
 )
 
 var (
 	showCommand = app.Command("show", "Show contents of a repository object.")
 
-	showObjectIDs = showCommand.Arg("objectID", "IDs of objects to show").Required().Strings()
+	showObjectIDs = showCommand.Arg("id", "IDs of objects to show").Required().Strings()
+	showJSON      = showCommand.Flag("json", "Pretty-print JSON content").Short('j').Bool()
+	showDir       = showCommand.Flag("dir", "Pretty-print directory content").Short('d').Bool()
 )
 
 func runShowCommand(context *kingpin.ParseContext) error {
-	mgr, err := mustOpenVault().OpenRepository()
+	vlt := mustOpenVault()
+	mgr, err := vlt.OpenRepository()
 	if err != nil {
 		return err
 	}
 
-	for _, oid := range *showObjectIDs {
-		r, err := mgr.Open(repo.ObjectID(oid))
+	for _, oidString := range *showObjectIDs {
+		oid, err := ParseObjectID(oidString, vlt)
+		if err != nil {
+			return err
+		}
+		r, err := mgr.Open(oid)
 		if err != nil {
 			return err
 		}
 
-		io.Copy(os.Stdout, r)
+		switch {
+		case *showJSON:
+			var v map[string]interface{}
+
+			if err := json.NewDecoder(r).Decode(&v); err != nil {
+				return err
+			}
+
+			m, err := json.MarshalIndent(v, "", "  ")
+			if err != nil {
+				return err
+			}
+			os.Stdout.Write(m)
+		case *showDir:
+			dir, err := fs.ReadDirectory(r, "")
+			if err != nil {
+				return err
+			}
+
+			for _, e := range dir {
+				var oid string
+				if e.ObjectID.Type().IsStored() {
+					oid = string(e.ObjectID)
+				} else if e.ObjectID.Type() == repo.ObjectIDTypeBinary {
+					oid = "<inline binary>"
+				} else if e.ObjectID.Type() == repo.ObjectIDTypeText {
+					oid = "<inline text>"
+				}
+				info := fmt.Sprintf("%v %9d %v %-30s %v", e.FileMode, e.FileSize, e.ModTime.Local().Format("02 Jan 06 15:04:05"), e.Name, oid)
+				fmt.Println(info)
+			}
+
+		default:
+			io.Copy(os.Stdout, r)
+		}
 	}
 
 	return nil

cmd/kopia/command_vault.go

@@ -0,0 +1,5 @@
+package main
+
+var (
+	vaultCommands = app.Command("vault", "Low-level commands to manipulate vault.")
+)

cmd/kopia/command_vault_list.go

@@ -0,0 +1,34 @@
+package main
+
+import (
+	"fmt"
+
+	"gopkg.in/alecthomas/kingpin.v2"
+)
+
+var (
+	vaultListCommand = vaultCommands.Command("list", "List contents of a vault")
+	vaultListPrefix  = vaultListCommand.Flag("prefix", "Prefix").String()
+)
+
+func init() {
+	vaultListCommand.Action(listVaultContents)
+}
+
+func listVaultContents(context *kingpin.ParseContext) error {
+	v, err := openVault()
+	if err != nil {
+		return err
+	}
+
+	entries, err := v.List(*vaultListPrefix)
+	if err != nil {
+		return err
+	}
+
+	for _, e := range entries {
+		fmt.Println(e)
+	}
+
+	return nil
+}

cmd/kopia/command_vault_show.go

@@ -0,0 +1,39 @@
+package main
+
+import (
+	"encoding/json"
+	"os"
+
+	"gopkg.in/alecthomas/kingpin.v2"
+)
+
+var (
+	vaultShowCommand = vaultCommands.Command("show", "Show contents of a vault item")
+	vaultShowID      = vaultShowCommand.Arg("id", "ID of the vault item to show").String()
+)
+
+func init() {
+	vaultShowCommand.Action(showVaultObject)
+}
+
+func showVaultObject(context *kingpin.ParseContext) error {
+	v, err := openVault()
+	if err != nil {
+		return err
+	}
+
+	var obj map[string]interface{}
+
+	if err := v.Get(*vaultShowID, &obj); err != nil {
+		return err
+	}
+
+	data, err := json.MarshalIndent(obj, "", "  ")
+	if err != nil {
+		return err
+	}
+
+	os.Stdout.Write(data)
+
+	return nil
+}

cmd/kopia/objref.go

@@ -0,0 +1,73 @@
+package main
+
+import (
+	"fmt"
+	"strings"
+
+	"github.com/kopia/kopia/fs"
+
+	"github.com/kopia/kopia/repo"
+	"github.com/kopia/kopia/vault"
+)
+
+// ParseObjectID interprets the given ID string and returns corresponding repo.ObjectID.
+// The string can be:
+// - backupID (vxxxxxx/12312312)
+// -
+func ParseObjectID(id string, vlt vault.Reader) (repo.ObjectID, error) {
+	head, tail := splitHeadTail(id)
+	if len(head) == 0 {
+		return "", fmt.Errorf("invalid object ID: %v", id)
+	}
+
+	oid, err := vlt.ResolveObjectID(head)
+	if err != nil {
+		return "", err
+	}
+
+	if tail == "" {
+		return oid, nil
+	}
+
+	r, err := vlt.OpenRepository()
+	if err != nil {
+		return "", fmt.Errorf("cannot open repository: %v", err)
+	}
+
+	return parseNestedObjectID(oid, tail, r)
+}
+
+func parseNestedObjectID(current repo.ObjectID, id string, r repo.Repository) (repo.ObjectID, error) {
+	head, tail := splitHeadTail(id)
+	for head != "" {
+		d, err := r.Open(current)
+		if err != nil {
+			return "", err
+		}
+
+		dir, err := fs.ReadDirectory(d, "")
+		if err != nil {
+			return "", fmt.Errorf("entry not found '%v': parent is not a directory.", head)
+		}
+
+		e := dir.FindByName(head)
+		if e == nil {
+			return "", fmt.Errorf("entry not found: '%v'", head)
+		}
+
+		current = e.ObjectID
+
+		head, tail = splitHeadTail(tail)
+	}
+
+	return current, nil
+}
+
+func splitHeadTail(id string) (string, string) {
+	p := strings.Index(id, "/")
+	if p < 0 {
+		return id, ""
+	}
+
+	return id[:p], id[p+1:]
+}

cmd/kopia/output.go

@@ -0,0 +1,23 @@
+package main
+
+import (
+	"fmt"
+
+	"github.com/kopia/kopia/fs"
+	"github.com/kopia/kopia/repo"
+)
+
+func listDirectory(dir fs.Directory) {
+	for _, e := range dir {
+		var oid string
+		if e.ObjectID.Type().IsStored() {
+			oid = string(e.ObjectID)
+		} else if e.ObjectID.Type() == repo.ObjectIDTypeBinary {
+			oid = "<inline binary>"
+		} else if e.ObjectID.Type() == repo.ObjectIDTypeText {
+			oid = "<inline text>"
+		}
+		info := fmt.Sprintf("%v %9d %v %-30s %v", e.FileMode, e.FileSize, e.ModTime.Local().Format("02 Jan 06 15:04:05"), e.Name, oid)
+		fmt.Println(info)
+	}
+}

cmd/kopia/urls.go

@@ -2,7 +2,6 @@
 
 import (
 	"fmt"
-	"log"
 	"net/url"
 	"os"
 	"strconv"
@@ -12,7 +11,6 @@
 )
 
 func newStorageFromURL(urlString string) (blob.Storage, error) {
-	log.Printf("NewStorageFromURl: %v", urlString)
 	if strings.HasPrefix(urlString, "/") {
 		urlString = "file://" + urlString
 	}
@@ -31,7 +29,7 @@ func newStorageFromURL(urlString string) (blob.Storage, error) {
 
 		return blob.NewFSStorage(&fso)
 
-	case "gcs":
+	case "gs", "gcs":
 		var gcso blob.GCSStorageOptions
 		if err := parseGoogleCloudStorageURL(&gcso, u); err != nil {
 			return nil, err
@@ -44,7 +42,6 @@ func newStorageFromURL(urlString string) (blob.Storage, error) {
 }
 
 func parseFilesystemURL(fso *blob.FSStorageOptions, u *url.URL) error {
-	log.Printf("u.Upaque: %v u.Path: %v", u.Opaque, u.Path)
 	if u.Opaque != "" {
 		fso.Path = u.Opaque
 	} else {

doc/quickstart.md

@@ -1,7 +1,7 @@
 Quick Start
 ===
 
-Kopia is a simple tool for managing encrypted backups in the cloud.
+Kopia is a simple command-line tool for managing encrypted backups in the cloud.
 
 Key Concepts
 ---

fs/dir_json.go

@@ -153,7 +153,7 @@ func (dw *directoryWriter) WriteEntry(e *Entry) error {
 
 	dw.writer.Write(dw.separator)
 	dw.writer.Write(v)
-	dw.separator = []byte(",\n  ")
+	dw.separator = []byte(",")
 
 	return nil
 }
@@ -177,7 +177,7 @@ func formatModeAndPermissions(m os.FileMode) string {
 }
 
 func (dw *directoryWriter) Close() error {
-	dw.writer.Write([]byte("\n]}\n"))
+	dw.writer.Write([]byte("]}"))
 	return nil
 }
 
@@ -196,11 +196,11 @@ func newDirectoryWriter(w io.Writer) *directoryWriter {
 	var f directoryFormat
 	f.Version = 1
 
-	io.WriteString(w, "{\n\"format\":")
+	io.WriteString(w, "{\"format\":")
 	b, _ := json.Marshal(&f)
 	w.Write(b)
-	io.WriteString(w, ",\n\"entries\":[")
-	dw.separator = []byte("\n  ")
+	io.WriteString(w, ",\"entries\":[")
+	dw.separator = []byte("")
 
 	return dw
 }

fs/dir_test.go

@@ -12,13 +12,13 @@ func TestDirectory(t *testing.T) {
 			`{`,
 			`"format":{"version":1},`,
 			`"entries":[`,
-			`  {"name":"config.go","mode":"420","size":"937","modTime":"2016-04-02T02:39:44.123456789Z","owner":"500:100","oid":"C4321"},`,
-			`  {"name":"constants.go","mode":"420","size":"13","modTime":"2016-04-02T02:36:19Z","owner":"500:100"},`,
-			`  {"name":"doc.go","mode":"420","size":"112","modTime":"2016-04-02T02:45:54Z","owner":"500:100"},`,
-			`  {"name":"errors.go","mode":"420","size":"506","modTime":"2016-04-02T02:41:03Z","owner":"500:100"},`,
-			`  {"name":"subdir","mode":"d:420","modTime":"2016-04-06T02:34:10Z","owner":"500:100","oid":"C1234"}`,
+			`{"name":"config.go","mode":"420","size":"937","modTime":"2016-04-02T02:39:44.123456789Z","owner":"500:100","oid":"C4321"},`,
+			`{"name":"constants.go","mode":"420","size":"13","modTime":"2016-04-02T02:36:19Z","owner":"500:100"},`,
+			`{"name":"doc.go","mode":"420","size":"112","modTime":"2016-04-02T02:45:54Z","owner":"500:100"},`,
+			`{"name":"errors.go","mode":"420","size":"506","modTime":"2016-04-02T02:41:03Z","owner":"500:100"},`,
+			`{"name":"subdir","mode":"d:420","modTime":"2016-04-06T02:34:10Z","owner":"500:100","oid":"C1234"}`,
 			`]}`,
-		}, "\n") + "\n"
+		}, "")
 
 	d, err := ReadDirectory(strings.NewReader(data), "")
 	if err != nil {

fs/upload.go

@@ -38,6 +38,7 @@ type UploadResult struct {
 // Uploader supports efficient uploading files and directories to repository.
 type Uploader interface {
 	UploadDir(path string, previousManifestID repo.ObjectID) (*UploadResult, error)
+	UploadFile(path string) (*UploadResult, error)
 	Cancel()
 }
 
@@ -52,7 +53,7 @@ func (u *uploader) isCancelled() bool {
 	return atomic.LoadInt32(&u.cancelled) != 0
 }
 
-func (u *uploader) uploadFile(path string, e *Entry) (*Entry, uint64, error) {
+func (u *uploader) uploadFileInternal(path string, forceStored bool) (*Entry, uint64, error) {
 	log.Printf("Uploading file %v", path)
 	file, err := u.lister.Open(path)
 	if err != nil {
@@ -75,7 +76,7 @@ func (u *uploader) uploadFile(path string, e *Entry) (*Entry, uint64, error) {
 		return nil, 0, err
 	}
 
-	e2.ObjectID, err = writer.Result(false)
+	e2.ObjectID, err = writer.Result(forceStored)
 	if err != nil {
 		return nil, 0, err
 	}
@@ -87,6 +88,13 @@ func (u *uploader) uploadFile(path string, e *Entry) (*Entry, uint64, error) {
 	return e2, e2.metadataHash(), nil
 }
 
+func (u *uploader) UploadFile(path string) (*UploadResult, error) {
+	result := &UploadResult{}
+	e, _, err := u.uploadFileInternal(path, true)
+	result.ObjectID = e.ObjectID
+	return result, err
+}
+
 func (u *uploader) UploadDir(path string, previousManifestID repo.ObjectID) (*UploadResult, error) {
 	//log.Printf("UploadDir", path)
 	//defer log.Printf("finishing UploadDir", path)
@@ -107,7 +115,7 @@ func (u *uploader) UploadDir(path string, previousManifestID repo.ObjectID) (*Up
 	hcw := newHashCacheWriter(mw)
 
 	result := &UploadResult{}
-	result.ObjectID, _, _, err = u.uploadDirInternal(result, path, ".", hcw, &mr)
+	result.ObjectID, _, _, err = u.uploadDirInternal(result, path, ".", hcw, &mr, true)
 	if err != nil {
 		return result, err
 	}
@@ -122,6 +130,7 @@ func (u *uploader) uploadDirInternal(
 	relativePath string,
 	hcw *hashcacheWriter,
 	mr *hashcacheReader,
+	forceStored bool,
 ) (repo.ObjectID, uint64, bool, error) {
 	log.Printf("Uploading dir %v", path)
 	defer log.Printf("Finished uploading dir %v", path)
@@ -151,7 +160,7 @@ func (u *uploader) uploadDirInternal(
 
 		switch e.FileMode & os.ModeType {
 		case os.ModeDir:
-			oid, h, wasCached, err := u.uploadDirInternal(result, fullPath, entryRelativePath, hcw, mr)
+			oid, h, wasCached, err := u.uploadDirInternal(result, fullPath, entryRelativePath, hcw, mr, false)
 			if err != nil {
 				return "", 0, false, err
 			}
@@ -186,7 +195,7 @@ func (u *uploader) uploadDirInternal(
 				hash = cachedEntry.Hash
 			} else {
 				result.Stats.NonCachedFiles++
-				e, hash, err = u.uploadFile(fullPath, e)
+				e, hash, err = u.uploadFileInternal(fullPath, false)
 				if err != nil {
 					return "", 0, false, fmt.Errorf("unable to hash file: %s", err)
 				}
@@ -231,7 +240,7 @@ func (u *uploader) uploadDirInternal(
 		directoryOID = repo.ObjectID(cachedDirEntry.ObjectID)
 	} else {
 		result.Stats.NonCachedDirectories++
-		directoryOID, err = writer.Result(true)
+		directoryOID, err = writer.Result(forceStored)
 		if err != nil {
 			return directoryOID, 0, false, err
 		}

repo/format.go

@@ -9,9 +9,6 @@
 	"crypto/sha256"
 	"crypto/sha512"
 	"hash"
-	"io"
-
-	"golang.org/x/crypto/hkdf"
 )
 
 var (
@@ -51,11 +48,22 @@ func (fmts ObjectIDFormats) Find(name string) *ObjectIDFormat {
 	return nil
 }
 
+func fold(b []byte, size int) []byte {
+	if len(b) == size {
+		return b
+	}
+
+	for i := size; i < len(b); i++ {
+		b[i%size] ^= b[i]
+	}
+	return b[0:size]
+}
+
 func nonEncryptedFormat(name string, hf func() hash.Hash, hashSize int) *ObjectIDFormat {
 	return &ObjectIDFormat{
 		Name: name,
 		hashBuffer: func(data []byte, secret []byte) ([]byte, []byte) {
-			return hashContent(hf, data, secret)[0:hashSize], nil
+			return fold(hashContent(hf, data, secret), hashSize), nil
 		},
 	}
 }
@@ -83,15 +91,8 @@ func encryptedFormat(
 		Name: name,
 		hashBuffer: func(data []byte, secret []byte) ([]byte, []byte) {
 			contentHash := hashContent(sha512.New, data, secret)
-
-			s1 := hkdf.New(sha256.New, contentHash, nil, hkdfInfoBlockID)
-			blockID := make([]byte, blockIDSize)
-			io.ReadFull(s1, blockID)
-
-			s2 := hkdf.New(sha256.New, contentHash, nil, hkdfInfoCryptoKey)
-			cryptoKey := make([]byte, keySize)
-			io.ReadFull(s2, cryptoKey)
-
+			blockID := fold(hashContent(hf, hkdfInfoBlockID, contentHash), blockIDSize)
+			cryptoKey := fold(hashContent(hf, hkdfInfoCryptoKey, contentHash), keySize)
 			return blockID, cryptoKey
 		},
 		createCipher: createCipher,
@@ -110,11 +111,11 @@ func buildObjectIDFormats() ObjectIDFormats {
 		{"sha1", sha1.New, sha1.Size},
 		{"sha224", sha256.New224, sha256.Size224},
 		{"sha256", sha256.New, sha256.Size},
-		{"sha256t128", sha256.New, 16},
-		{"sha256t160", sha256.New, 20},
+		{"sha256-fold128", sha256.New, 16},
+		{"sha256-fold160", sha256.New, 20},
 		{"sha384", sha512.New384, sha512.Size384},
-		{"sha512t128", sha512.New, 16},
-		{"sha512t160", sha512.New, 20},
+		{"sha512-fold128", sha512.New, 16},
+		{"sha512-fold160", sha512.New, 20},
 		{"sha512-224", sha512.New512_224, sha512.Size224},
 		{"sha512-256", sha512.New512_256, sha512.Size256},
 		{"sha512", sha512.New, sha512.Size},

repo/repository_test.go

@@ -445,27 +445,33 @@ func TestFormats(t *testing.T) {
 			},
 		},
 		{
-			format: makeFormat("sha256t128-aes128"),
+			format: makeFormat("sha256-fold128-aes128"),
 			oids: map[string]ObjectID{
-				"The quick brown fox jumps over the lazy dog": "D4e43650cb2ce7b5a6a2a8d614c13d9b3.f73b9fed1f355b3603e066fb24a39970",
+				"The quick brown fox jumps over the lazy dog": "D028e74c630aee6400db4f84d49d1ed08.2825273c3344999e26081d99bcaf3f18",
 			},
 		},
 		{
 			format: makeFormat("sha256-aes128"),
 			oids: map[string]ObjectID{
-				"The quick brown fox jumps over the lazy dog": "D4e43650cb2ce7b5a6a2a8d614c13d9b37c6ee13c4543481074d2c8cf3f597a13.f73b9fed1f355b3603e066fb24a39970",
+				"The quick brown fox jumps over the lazy dog": "D26bb3fee9e83a7f6ff8397f33460cd0d24354b28ae2d41b6f2376fbe7db12005.2825273c3344999e26081d99bcaf3f18",
 			},
 		},
 		{
 			format: makeFormat("sha384-aes256"),
 			oids: map[string]ObjectID{
-				"The quick brown fox jumps over the lazy dog": "D4e43650cb2ce7b5a6a2a8d614c13d9b37c6ee13c4543481074d2c8cf3f597a136dba4c0e67d5d644049c98ee5369bc0a.f73b9fed1f355b3603e066fb24a39970ac10cbd143babc7f487ef263fe38aeff",
+				"The quick brown fox jumps over the lazy dog": "D08edca658caee5ae86c9f1e4ccea26b1a43b0ddb04502c5f1b1cba051a756a9eba440bb6991b23323e37556ec5cf4f88.7a753377319650870f5cba47c30b608675542f206fa67d94dcbb9a0ccce7f467",
 			},
 		},
 		{
 			format: makeFormat("sha512-aes256"),
 			oids: map[string]ObjectID{
-				"The quick brown fox jumps over the lazy dog": "D4e43650cb2ce7b5a6a2a8d614c13d9b37c6ee13c4543481074d2c8cf3f597a136dba4c0e67d5d644049c98ee5369bc0aaa0e75feaeadd42eb54a9d64f9d0a51d.f73b9fed1f355b3603e066fb24a39970ac10cbd143babc7f487ef263fe38aeff",
+				"The quick brown fox jumps over the lazy dog": "Deee04ad71248ef4ffec5159558e80e2791e32299cfb79a1e063cc5f4a71cd61ca2c1b110e3ae2e83635a8626a3a27e4805eb745c40b5a4ebd4c9372602e5ab65.1c0b1b58ce05b7b8b05cfce27a485ddf97bde5159f6946357ec7795236f36a84",
+			},
+		},
+		{
+			format: makeFormat("sha512-fold128-aes192"),
+			oids: map[string]ObjectID{
+				"The quick brown fox jumps over the lazy dog": "Dd829ad027ee4ff394f6a61615eb30d16.0ab00e0e0e156927f0be63372e8449d7bf2316c43d46e626",
 			},
 		},
 	}

vault/reader.go

@@ -0,0 +1,12 @@
+package vault
+
+import "github.com/kopia/kopia/repo"
+
+// Reader allows reading from a vault.
+type Reader interface {
+	Get(id string, content interface{}) error
+	GetRaw(id string) ([]byte, error)
+	List(prefix string) ([]string, error)
+	ResolveObjectID(id string) (repo.ObjectID, error)
+	OpenRepository() (repo.Repository, error)
+}

vault/vault.go

@@ -8,11 +8,13 @@
 	"crypto/rand"
 	"crypto/sha256"
 	"encoding/base64"
+	"encoding/hex"
 	"encoding/json"
 	"fmt"
 	"hash"
 	"io"
 	"io/ioutil"
+	"strings"
 
 	"github.com/kopia/kopia/blob"
 	"github.com/kopia/kopia/repo"
@@ -24,6 +26,9 @@
 	formatBlock           = "format"
 	checksumBlock         = "checksum"
 	repositoryConfigBlock = "repo"
+
+	storedObjectIDPrefix      = "v"
+	storedObjectIDLengthBytes = 8
 )
 
 var (
@@ -193,6 +198,10 @@ func (v *Vault) OpenRepository() (repo.Repository, error) {
 	return repo.NewRepository(storage, rc.Format)
 }
 
+func (v *Vault) GetRaw(id string) ([]byte, error) {
+	return v.readEncryptedBlock(id)
+}
+
 // Get deserializes JSON data stored in the vault into the specified content structure.
 func (v *Vault) Get(id string, content interface{}) error {
 	j, err := v.readEncryptedBlock(id)
@@ -245,6 +254,55 @@ func (v *Vault) Token() (string, error) {
 	return base64.RawURLEncoding.EncodeToString(b), nil
 }
 
+type objectIDData struct {
+	ObjectID string `json:"objectID"`
+}
+
+func (v *Vault) SaveObjectID(oid repo.ObjectID) (string, error) {
+	h := hmac.New(sha256.New, v.format.UniqueID)
+	h.Write([]byte(oid))
+	sum := h.Sum(nil)
+	for i := storedObjectIDLengthBytes; i < len(sum); i++ {
+		sum[i%storedObjectIDLengthBytes] ^= sum[i]
+	}
+	sum = sum[0:storedObjectIDLengthBytes]
+	key := storedObjectIDPrefix + hex.EncodeToString(sum)
+
+	var d objectIDData
+	d.ObjectID = string(oid)
+
+	if err := v.Put(key, &d); err != nil {
+		return "", err
+	}
+
+	return key, nil
+}
+
+func (v *Vault) ResolveObjectID(id string) (repo.ObjectID, error) {
+	if !strings.HasPrefix(id, storedObjectIDPrefix) {
+		return repo.ParseObjectID(id)
+	}
+
+	matches, err := v.List(id)
+	if err != nil {
+		return "", err
+	}
+
+	switch len(matches) {
+	case 0:
+		return "", fmt.Errorf("object not found: %v", id)
+	case 1:
+		var d objectIDData
+		if err := v.Get(matches[0], &d); err != nil {
+			return "", err
+		}
+		return repo.ParseObjectID(d.ObjectID)
+
+	default:
+		return "", fmt.Errorf("ambiguous object ID: %v", id)
+	}
+}
+
 // Create creates a Vault in the specified storage.
 func Create(storage blob.Storage, format *Format, creds Credentials) (*Vault, error) {
 	if err := format.ensureUniqueID(); err != nil {