Major refactoring of data formats to use protobuf instead of JSON. This is a breaking change to data formats, but hoping to stabilize soon.

This change removes lots of pointless string and custom format parsing code and instead relies on protobuf to do the thing. JSON is still an option thanks to proto3-generated output.

- Refactored ObjectID to use protobuf
- Indirect blocks to use protobuf (in the form of seek table)
- Added generic proto stream reader and writer
- ObjectIDFormat became proto enum

Also:

- fixed major issue where indirect object ID entries were not encrypted, since they were storing block IDs and not object IDs.
- dropped support for majority of formats, only supporting HMAC-{SHA256,SHA512,SHA512_384} hashes with AES256.
This produces object IDs of 32 or 64 characters long.
- changed how pretty-printing works for 'kopia show'

.gitignore

@@ -3,3 +3,4 @@
 .releases/
 kopia-*.tar.gz
 kopia-*.zip
+combined.cov

Makefile

@@ -17,11 +17,20 @@ install:
 build:
 	go build github.com/kopia/kopia/...
 
+escape-analysis:
+	go build -gcflags '-m -l' github.com/kopia/kopia/...
+
+protos:
+	protoc --go_out=../../.. fs/*.proto
+	protoc --go_out=../../.. repo/*.proto
+	protoc --go_out=../../.. vault/*.proto
+	protoc --go_out=../../.. internal/*.proto
+
 play:
 	go run cmd/playground/main.go
 
 lint:
-	golint github.com/kopia/kopia/...
+	-golint github.com/kopia/kopia/... | grep -v .pb.go
 
 vet:
 	go tool vet -all .

backup/generator.go

@@ -25,10 +25,10 @@ func (bg *backupGenerator) Backup(m *Manifest, old *Manifest) error {
 	uploader := fs.NewUploader(bg.repo)
 
 	m.StartTime = time.Now()
-	var hashCacheID repo.ObjectID
+	var hashCacheID *repo.ObjectID
 
 	if old != nil {
-		hashCacheID = repo.ObjectID(old.HashCacheID)
+		hashCacheID = &old.HashCacheID
 	}
 
 	entry, err := fs.NewFilesystemEntry(m.Source, nil)
@@ -46,8 +46,8 @@ func (bg *backupGenerator) Backup(m *Manifest, old *Manifest) error {
 	if err != nil {
 		return err
 	}
-	m.RootObjectID = string(r.ObjectID)
-	m.HashCacheID = string(r.ManifestID)
+	m.RootObjectID = r.ObjectID
+	m.HashCacheID = r.ManifestID
 
 	return nil
 }

backup/manifest.go

@@ -5,6 +5,8 @@ import (
 	"encoding/hex"
 	"io"
 	"time"
+
+	"github.com/kopia/kopia/repo"
 )
 
 // Manifest stores information about single backup.
@@ -16,10 +18,10 @@ type Manifest struct {
 	UserName    string `json:"userName"`
 	Description string `json:"description"`
 
-	Handle       string `json:"handle"`
-	Source       string `json:"source"`
-	RootObjectID string `json:"root"`
-	HashCacheID  string `json:"hashCache"`
+	Handle       string        `json:"handle"`
+	Source       string        `json:"source"`
+	RootObjectID repo.ObjectID `json:"root"`
+	HashCacheID  repo.ObjectID `json:"hashCache"`
 
 	FileCount      int64 `json:"fileCount"`
 	DirectoryCount int64 `json:"dirCount"`

cmd/kopia/command_backup.go

@@ -113,7 +113,7 @@ func runBackupCommand(context *kingpin.ParseContext) error {
 			return fmt.Errorf("cannot save manifest: %v", err)
 		}
 
-		log.Printf("Root: %v", manifest.RootObjectID)
+		log.Printf("Root: %v", manifest.RootObjectID.UIString())
 		log.Printf("Key: %v", handleID)
 	}
 

cmd/kopia/command_create.go

@@ -15,7 +15,7 @@ import (
 var (
 	createCommand           = app.Command("create", "Create new vault and repository.")
 	createCommandRepository = createCommand.Flag("repository", "Repository path.").Default("colocated").String()
-	createObjectFormat      = createCommand.Flag("repo-format", "Format of repository objects.").PlaceHolder("FORMAT").Default("sha256-fold160-aes128").Enum(supportedObjectFormats()...)
+	createObjectFormat      = createCommand.Flag("repo-format", "Format of repository objects.").PlaceHolder("FORMAT").Default(repo.DefaultObjectFormat.Name()).Enum(supportedObjectFormats()...)
 
 	createMaxBlobSize           = createCommand.Flag("max-blob-size", "Maximum size of a data chunk.").PlaceHolder("KB").Default("20480").Int()
 	createInlineBlobSize        = createCommand.Flag("inline-blob-size", "Maximum size of an inline data chunk.").PlaceHolder("KB").Default("32").Int()
@@ -43,15 +43,20 @@ func vaultFormat() (*vault.Format, error) {
 }
 
 func repositoryFormat() (*repo.Format, error) {
-	f := &repo.Format{
-		Version:           "1",
-		Secret:            make([]byte, 32),
-		MaxBlobSize:       *createMaxBlobSize * 1024,
-		MaxInlineBlobSize: *createInlineBlobSize * 1024,
-		ObjectFormat:      *createObjectFormat,
+	objectIDFormat, err := repo.ParseObjectIDFormat(*createObjectFormat)
+	if err != nil {
+		return nil, err
 	}
 
-	_, err := io.ReadFull(rand.Reader, f.Secret)
+	f := &repo.Format{
+		Version:           1,
+		Secret:            make([]byte, 32),
+		MaxBlobSize:       int32(*createMaxBlobSize * 1024),
+		MaxInlineBlobSize: int32(*createInlineBlobSize * 1024),
+		ObjectFormat:      objectIDFormat,
+	}
+
+	_, err = io.ReadFull(rand.Reader, f.Secret)
 	if err != nil {
 		return nil, err
 	}
@@ -100,7 +105,7 @@ func runCreateCommand(context *kingpin.ParseContext) error {
 	}
 
 	fmt.Printf(
-		"Initializing repository in with format '%v' and maximum object size %v.\n",
+		"Initializing repository in with format %v and maximum object size %v.\n",
 		repoFormat.ObjectFormat,
 		repoFormat.MaxBlobSize)
 
@@ -151,7 +156,7 @@ func supportedVaultEncryptionFormats() []string {
 func supportedObjectFormats() []string {
 	var r []string
 	for _, o := range repo.SupportedFormats {
-		r = append(r, o.Name)
+		r = append(r, o.Name())
 	}
 	return r
 }

cmd/kopia/command_ls.go

@@ -6,7 +6,6 @@ import (
 	"strings"
 
 	"github.com/kopia/kopia/fs"
-	"github.com/kopia/kopia/repo"
 
 	kingpin "gopkg.in/alecthomas/kingpin.v2"
 )
@@ -69,17 +68,22 @@ func listDirectory(prefix string, entries fs.Entries, longFormat bool) {
 		var info string
 		if longFormat {
 			var oid string
-			if m.ObjectID.Type() == repo.ObjectIDTypeBinary {
-				oid = "<inline binary>"
-			} else if m.ObjectID.Type() == repo.ObjectIDTypeText {
-				oid = "<inline text>"
+			if m.ObjectId.Content != nil {
+				oid = "<inline content>"
 			} else {
-				oid = string(m.ObjectID)
+				oid = m.ObjectId.UIString()
 			}
-			info = fmt.Sprintf("%v %9d %v %-"+maxNameLenString+"s %v", m.FileMode, m.FileSize, m.ModTime.Local().Format("02 Jan 06 15:04:05"), m.Name, oid)
+			info = fmt.Sprintf(
+				"%v %9d %v %-"+maxNameLenString+"s %v",
+				m.FileMode(),
+				m.FileSize,
+				m.ModTime().Local().Format("02 Jan 06 15:04:05"),
+				m.Name,
+				oid,
+			)
 		} else {
 			var suffix string
-			if m.FileMode.IsDir() {
+			if m.FileMode().IsDir() {
 				suffix = "/"
 			}
 

cmd/kopia/command_show.go

@@ -1,11 +1,19 @@
 package main
 
 import (
+	"bufio"
 	"bytes"
 	"encoding/json"
+	"fmt"
+	"io"
 	"io/ioutil"
 	"os"
 
+	"github.com/golang/protobuf/proto"
+	"github.com/kopia/kopia/fs"
+
+	"github.com/kopia/kopia/internal"
+
 	"github.com/kopia/kopia/repo"
 
 	kingpin "gopkg.in/alecthomas/kingpin.v2"
@@ -54,8 +62,8 @@ func showObject(mgr repo.Repository, oid repo.ObjectID) error {
 
 	format := "raw"
 
-	if len(rawdata) > 2 && rawdata[0] == '{' && rawdata[len(rawdata)-1] == '}' {
-		format = "json"
+	if len(rawdata) >= 8 {
+		format = detectProtoFormat(rawdata[0:8])
 	}
 
 	if *showJSON {
@@ -73,12 +81,49 @@ func showObject(mgr repo.Repository, oid repo.ObjectID) error {
 		json.Indent(&buf, rawdata, "", "  ")
 		os.Stdout.Write(buf.Bytes())
 
+	case "dir":
+		return prettyPrintProtoStream(rawdata, internal.ProtoStreamTypeDir, &fs.EntryMetadata{})
+	case "indirect":
+		return prettyPrintProtoStream(rawdata, internal.ProtoStreamTypeIndirect, &repo.IndirectObjectEntry{})
+	case "hashcache":
+		return prettyPrintProtoStream(rawdata, internal.ProtoStreamTypeHashCache, &fs.HashCacheEntry{})
+
 	default:
 		os.Stdout.Write(rawdata)
 	}
 	return nil
 }
 
+func detectProtoFormat(b []byte) string {
+	switch {
+	case bytes.Equal(b, internal.ProtoStreamTypeDir):
+		return "dir"
+	case bytes.Equal(b, internal.ProtoStreamTypeIndirect):
+		return "indirect"
+	case bytes.Equal(b, internal.ProtoStreamTypeHashCache):
+		return "hashcache"
+	}
+	return "raw"
+}
+
+func prettyPrintProtoStream(data []byte, header []byte, pb proto.Message) error {
+	r := internal.NewProtoStreamReader(bufio.NewReader(bytes.NewReader(data)), header)
+	i := 0
+	for {
+		err := r.Read(pb)
+		if err == io.EOF {
+			return nil
+		}
+		if err != nil {
+			return err
+		}
+
+		fmt.Printf("[entry #%v]\n", i)
+		fmt.Println(proto.MarshalTextString(pb))
+		i++
+	}
+}
+
 func init() {
 	showCommand.Action(runShowCommand)
 }

cmd/kopia/objref.go

@@ -13,7 +13,7 @@ import (
 func parseObjectID(id string, vlt *vault.Vault) (repo.ObjectID, error) {
 	head, tail := splitHeadTail(id)
 	if len(head) == 0 {
-		return "", fmt.Errorf("invalid object ID: %v", id)
+		return repo.NullObjectID, fmt.Errorf("invalid object ID: %v", id)
 	}
 
 	if !strings.HasPrefix(id, vault.StoredObjectIDPrefix) {
@@ -22,7 +22,7 @@ func parseObjectID(id string, vlt *vault.Vault) (repo.ObjectID, error) {
 
 	oid, err := vlt.GetObjectID(head)
 	if err != nil {
-		return "", fmt.Errorf("can't retrieve vault object ID %v: %v", head, err)
+		return repo.NullObjectID, fmt.Errorf("can't retrieve vault object ID %v: %v", head, err)
 	}
 
 	if tail == "" {
@@ -31,12 +31,12 @@ func parseObjectID(id string, vlt *vault.Vault) (repo.ObjectID, error) {
 
 	r, err := vlt.OpenRepository()
 	if err != nil {
-		return "", fmt.Errorf("cannot open repository: %v", err)
+		return repo.NullObjectID, fmt.Errorf("cannot open repository: %v", err)
 	}
 
 	dir := fs.NewRepositoryDirectory(r, oid)
 	if err != nil {
-		return "", err
+		return repo.NullObjectID, err
 	}
 
 	return parseNestedObjectID(dir, tail)
@@ -49,24 +49,24 @@ func parseNestedObjectID(startingDir fs.Directory, id string) (repo.ObjectID, er
 	for head != "" {
 		dir, ok := current.(fs.Directory)
 		if !ok {
-			return "", fmt.Errorf("entry not found '%v': parent is not a directory", head)
+			return repo.NullObjectID, fmt.Errorf("entry not found '%v': parent is not a directory", head)
 		}
 
 		entries, err := dir.Readdir()
 		if err != nil {
-			return "", err
+			return repo.NullObjectID, err
 		}
 
 		e := entries.FindByName(head)
 		if e == nil {
-			return "", fmt.Errorf("entry not found: '%v'", head)
+			return repo.NullObjectID, fmt.Errorf("entry not found: '%v'", head)
 		}
 
 		current = e
 		head, tail = splitHeadTail(tail)
 	}
 
-	return current.Metadata().ObjectID, nil
+	return *current.Metadata().ObjectId, nil
 }
 
 func splitHeadTail(id string) (string, string) {

cmd/playground/main.go

@@ -0,0 +1,77 @@
+package main
+
+import (
+	"bufio"
+	"bytes"
+	"log"
+
+	"github.com/kopia/kopia/internal"
+	"github.com/kopia/kopia/repo"
+	"github.com/kopia/kopia/storage/storagetesting"
+)
+
+func dumpListBlock(prefix string, r repo.Repository, oid repo.ObjectID) {
+	ind := oid.Indirect
+	oid.Indirect = 0
+	log.Printf("%vdumping %v", prefix, oid.String())
+	rd, err := r.Open(oid)
+	if err != nil {
+		log.Fatalf("unable to open %v: %v", oid.String(), err)
+	}
+	defer rd.Close()
+
+	var objects []repo.ObjectID
+
+	pr := internal.NewProtoStreamReader(bufio.NewReader(rd), internal.ProtoStreamTypeIndirect)
+	for {
+		var v repo.IndirectObjectEntry
+		if err := pr.Read(&v); err != nil {
+			log.Printf("err: %v", err)
+			break
+		}
+		log.Printf("%v [%v] entry: %v", prefix, oid.String(), v.String())
+		if ind > 0 {
+			objects = append(objects, *v.Object)
+		}
+	}
+
+	for _, o := range objects {
+		dumpListBlock(prefix+"  ", r, o)
+	}
+}
+
+func main() {
+	d := map[string][]byte{}
+
+	s := storagetesting.NewMapStorage(d)
+	f := repo.Format{
+		Version:      1,
+		MaxBlobSize:  100,
+		ObjectFormat: repo.ObjectIDFormat_TESTONLY_MD5,
+	}
+	r, err := repo.NewRepository(s, &f)
+	if err != nil {
+		log.Fatalf("Err: %v", err)
+	}
+	w := r.NewWriter(repo.WithDescription("tester"))
+
+	w.Write(bytes.Repeat([]byte("a"), 5000))
+	result, err := w.Result(true)
+	log.Printf("result: %v", result.UIString())
+
+	//dumpListBlock("", r, result)
+
+	log.Printf("attempting to open")
+
+	rdr, err := r.Open(result)
+	if err != nil {
+		log.Fatalf("open: %v", err)
+	}
+	log.Printf("len: %v\n", rdr.Length())
+	log.Printf("----")
+	// for k, v := range d {
+	// 	log.Printf("%v = %v", k, string(v))
+	// }
+
+	rdr.Close()
+}

fs/dir_json.go

@@ -1,407 +1,79 @@
 package fs
 
 import (
-	"encoding/json"
+	"bufio"
 	"fmt"
 	"io"
-	"os"
-	"strconv"
-	"strings"
-	"time"
+
+	"github.com/kopia/kopia/internal"
 
 	"github.com/kopia/kopia/repo"
 )
 
-const modeChars = "dalTLDpSugct"
-
-type jsonDirectoryEntry struct {
-	Name        string               `json:"name"`
-	Type        string               `json:"type,omitempty"`
-	Permissions string               `json:"perm,omitempty"`
-	Size        string               `json:"size,omitempty"`
-	Time        *time.Time           `json:"mtime,omitempty"`
-	Owner       string               `json:"owner,omitempty"`
-	ObjectID    string               `json:"oid,omitempty"`
-	JSONContent json.RawMessage      `json:"content,omitempty"`
-	SubEntries  []jsonDirectoryEntry `json:"entries,omitempty"`
-}
-
-func (em *EntryMetadata) fromJSON(jde *jsonDirectoryEntry) error {
-	if jde.Name == "" {
-		return fmt.Errorf("empty entry name")
-	}
-	em.Name = jde.Name
-
-	if jde.Permissions != "" {
-		if m, err := parseFilePermissions(jde.Permissions); err == nil {
-			em.FileMode |= m
-		} else {
-			return fmt.Errorf("invalid permissions: '%v'", jde.Permissions)
-		}
-	}
-
-	if jde.Type != "" {
-		if m, err := parseFileMode(jde.Type); err == nil {
-			em.FileMode |= m
-		} else {
-			return fmt.Errorf("invalid type: '%v'", jde.Type)
-		}
-	}
-
-	if jde.Time != nil {
-		em.ModTime = *jde.Time
-	}
-
-	if jde.Owner != "" {
-		if c, err := fmt.Sscanf(jde.Owner, "%d:%d", &em.OwnerID, &em.GroupID); err != nil || c != 2 {
-			return fmt.Errorf("invalid owner: %v", err)
-		}
-	}
-
-	if jde.JSONContent != nil {
-		em.ObjectID = repo.NewInlineObjectID([]byte(jde.JSONContent))
-	} else {
-		em.ObjectID = repo.ObjectID(jde.ObjectID)
-	}
-
-	if jde.Size != "" {
-		if s, err := strconv.ParseInt(jde.Size, 10, 64); err == nil {
-			em.FileSize = s
-		} else {
-			return fmt.Errorf("invalid size: %v", err)
-		}
-	}
-	return nil
-}
-
-// parseFileModeAndPermissions converts file mode string to os.FileMode
-func parseFileModeAndPermissions(s string) (os.FileMode, error) {
-	colon := strings.IndexByte(s, ':')
-	if colon < 0 {
-		return parseFilePermissions(s)
-	}
-
-	var mode os.FileMode
-
-	if m, err := parseFileMode(s[0:colon]); err == nil {
-		mode |= m
-	} else {
-		return 0, err
-	}
-
-	if m, err := parseFilePermissions(s[colon+1:]); err == nil {
-		mode |= m
-	} else {
-		return 0, err
-	}
-
-	return mode, nil
-}
-
-func parseFileMode(s string) (os.FileMode, error) {
-	var mode os.FileMode
-	for _, c := range s {
-		switch c {
-		case 'd':
-			mode |= os.ModeDir
-		case 'a':
-			mode |= os.ModeAppend
-		case 'l':
-			mode |= os.ModeExclusive
-		case 'T':
-			mode |= os.ModeTemporary
-		case 'L':
-			mode |= os.ModeSymlink
-		case 'D':
-			mode |= os.ModeDevice
-		case 'p':
-			mode |= os.ModeNamedPipe
-		case 'S':
-			mode |= os.ModeSocket
-		case 'u':
-			mode |= os.ModeSetuid
-		case 'g':
-			mode |= os.ModeSetgid
-		case 'c':
-			mode |= os.ModeCharDevice
-		case 't':
-			mode |= os.ModeSticky
-		default:
-			return 0, fmt.Errorf("unsupported mode: '%v'", c)
-		}
-	}
-	return mode, nil
-}
-
-func parseFilePermissions(perm string) (os.FileMode, error) {
-	s, err := strconv.ParseUint(perm, 8, 32)
-	if err != nil {
-		return 0, err
-	}
-	return os.FileMode(s), nil
-}
-
 type directoryWriter struct {
-	io.Closer
-
-	writer    io.Writer
-	buf       []byte
-	separator []byte
-
-	lastNameWritten string
+	w *internal.ProtoStreamWriter
 }
 
 func (dw *directoryWriter) WriteEntry(e *EntryMetadata, children []*EntryMetadata) error {
-	if dw.lastNameWritten != "" {
-		if isLessOrEqual(e.Name, dw.lastNameWritten) {
-			return fmt.Errorf("out-of-order directory entry, previous '%v' current '%v'", dw.lastNameWritten, e.Name)
-		}
-		dw.lastNameWritten = e.Name
-	}
-
-	jde := e.toJSONEntry()
-
-	if len(children) > 0 {
-		jde.SubEntries = make([]jsonDirectoryEntry, len(children))
-		for i, se := range children {
-			jde.SubEntries[i] = se.toJSONEntry()
-		}
-	}
-
-	v, _ := json.Marshal(&jde)
-
-	dw.writer.Write(dw.separator)
-	dw.writer.Write(v)
-	dw.separator = []byte(",")
-
-	return nil
+	return dw.w.Write(e)
 }
 
-func (em *EntryMetadata) toJSONEntry() jsonDirectoryEntry {
-	var jde jsonDirectoryEntry
-	jde.Name = em.Name
-
-	if (em.FileMode & os.ModeType) != 0 {
-		jde.Type = formatMode(em.FileMode)
-	}
-
-	if (em.FileMode & os.ModePerm) != 0 {
-		jde.Permissions = formatPermissions(em.FileMode)
-	}
-
-	if em.OwnerID > 0 || em.GroupID > 0 {
-		jde.Owner = fmt.Sprintf("%d:%d", em.OwnerID, em.GroupID)
-	}
-
-	if !em.ModTime.IsZero() {
-		utc := em.ModTime.UTC()
-		jde.Time = &utc
-	}
-
-	if em.ObjectID != "" {
-		inline := em.ObjectID.InlineData()
-		if len(inline) >= 2 && inline[0] == '{' && inline[len(inline)-1] == '}' {
-			m := map[string]interface{}{}
-
-			if json.Unmarshal(inline, &m) == nil {
-				jde.JSONContent = json.RawMessage(inline)
-			}
-		}
-	}
-
-	if jde.JSONContent == nil {
-		jde.ObjectID = string(em.ObjectID)
-	}
-
-	if em.FileMode.IsRegular() {
-		jde.Size = strconv.FormatInt(em.FileSize, 10)
-	}
-
-	return jde
-}
-
-func formatMode(m os.FileMode) string {
-	const str = "dalTLDpSugct"
-	var buf [32]byte
-	w := 0
-	for i, c := range str {
-		if m&(1<<uint(32-1-i)) != 0 {
-			buf[w] = byte(c)
-			w++
-		}
-	}
-
-	return string(buf[:w])
-}
-
-func formatPermissions(m os.FileMode) string {
-	return strconv.FormatInt(int64(m&os.ModePerm), 8)
-}
-
-func (dw *directoryWriter) Close() error {
-	dw.writer.Write([]byte("]}"))
-	return nil
-}
-
-func newDirectoryWriter(w io.Writer) *directoryWriter {
+func newDirectoryWriter(w io.WriteCloser) *directoryWriter {
 	dw := &directoryWriter{
-		writer: w,
+		w: internal.NewProtoStreamWriter(w, internal.ProtoStreamTypeDir),
 	}
 
-	var f directoryFormat
-	f.Version = 1
-
-	io.WriteString(w, "{\"format\":")
-	b, _ := json.Marshal(&f)
-	w.Write(b)
-	io.WriteString(w, ",\"entries\":[")
-	dw.separator = []byte("")
-
 	return dw
 }
 
-type directoryReader struct {
-	reader  io.Reader
-	decoder *json.Decoder
-}
-
-func (dr *directoryReader) readNext(jde *jsonDirectoryEntry) error {
-	if dr.decoder.More() {
-		return dr.decoder.Decode(&jde)
-	}
-
-	if err := ensureDelimiter(dr.decoder, json.Delim(']')); err != nil {
-		return invalidDirectoryError(err)
-	}
-
-	if err := ensureDelimiter(dr.decoder, json.Delim('}')); err != nil {
-		return invalidDirectoryError(err)
-	}
-
-	return io.EOF
-}
-
 func invalidDirectoryError(cause error) error {
 	return fmt.Errorf("invalid directory data: %v", cause)
 }
 
-type directoryFormat struct {
-	Version int `json:"version"`
-}
-
-func ensureDelimiter(d *json.Decoder, expected json.Delim) error {
-	t, err := d.Token()
-	if err != nil {
-		return err
-	}
-
-	if t != expected {
-		return fmt.Errorf("expected '%v', got %v", expected.String(), t)
-	}
-
-	return nil
-}
-func ensureStringToken(d *json.Decoder, expected string) error {
-	t, err := d.Token()
-	if err != nil {
-		return err
-	}
-
-	if s, ok := t.(string); ok {
-		if s == expected {
-			return nil
-		}
-	}
-
-	return fmt.Errorf("expected '%v', got '%v'", expected, t)
-}
-
-func newDirectoryReader(r io.Reader) (*directoryReader, error) {
-	dr := &directoryReader{
-		decoder: json.NewDecoder(r),
-	}
-
-	if err := ensureDelimiter(dr.decoder, json.Delim('{')); err != nil {
-		return nil, invalidDirectoryError(err)
-	}
-
-	if err := ensureStringToken(dr.decoder, "format"); err != nil {
-		return nil, invalidDirectoryError(err)
-	}
-
-	// Parse format and trailing comma
-	var format directoryFormat
-	if err := dr.decoder.Decode(&format); err != nil {
-		return nil, invalidDirectoryError(err)
-	}
-
-	if format.Version != 1 {
-		return nil, invalidDirectoryError(fmt.Errorf("unsupported version: %v", format.Version))
-	}
-
-	if err := ensureStringToken(dr.decoder, "entries"); err != nil {
-		return nil, invalidDirectoryError(err)
-	}
-
-	if err := ensureDelimiter(dr.decoder, json.Delim('[')); err != nil {
-		return nil, invalidDirectoryError(err)
-	}
-
-	return dr, nil
-}
-
 func readDirectoryMetadataEntries(r io.Reader) ([]*EntryMetadata, error) {
-	dr, err := newDirectoryReader(r)
-	if err != nil {
-		return nil, err
+	psr := internal.NewProtoStreamReader(bufio.NewReader(r), internal.ProtoStreamTypeDir)
+	var entries []*EntryMetadata
+	for {
+		e := &EntryMetadata{}
+		err := psr.Read(e)
+		if err == io.EOF {
+			break
+		}
+
+		if err != nil {
+			return nil, err
+		}
+
+		entries = append(entries, e)
 	}
 
+	return flattenBundles(entries)
+}
+
+func flattenBundles(source []*EntryMetadata) ([]*EntryMetadata, error) {
 	var entries []*EntryMetadata
 	var bundles [][]*EntryMetadata
 
-	for {
-		var e jsonDirectoryEntry
-
-		if err := dr.readNext(&e); err != nil {
-			if err == io.EOF {
-				break
-			}
-
-			return nil, err
-		}
-
-		var entryMetadata EntryMetadata
-		if err := entryMetadata.fromJSON(&e); err != nil {
-			return nil, err
-		}
-
-		if len(e.SubEntries) > 0 {
-			bundle := make([]*EntryMetadata, 0, len(e.SubEntries))
+	for _, e := range source {
+		if len(e.BundledChildren) > 0 {
+			bundle := e.BundledChildren
+			e.BundledChildren = nil
 
 			var currentOffset int64
 
-			var bundleEntry EntryMetadata
-			if err := bundleEntry.fromJSON(&e); err != nil {
-				return nil, err
+			for _, child := range bundle {
+				id := repo.NewSectionObjectID(currentOffset, child.FileSize, e.ObjectId)
+				child.ObjectId = &id
+				currentOffset += child.FileSize
 			}
 
-			for _, s := range e.SubEntries {
-				var subEntry EntryMetadata
-				if err := subEntry.fromJSON(&s); err != nil {
-					return nil, err
-				}
-				subEntry.ObjectID = repo.NewSectionObjectID(currentOffset, subEntry.FileSize, entryMetadata.ObjectID)
-				currentOffset += subEntry.FileSize
-				bundle = append(bundle, &subEntry)
-			}
-
-			if currentOffset != entryMetadata.FileSize {
-				return nil, fmt.Errorf("inconsistent size of '%v': %v (got %v)", entryMetadata.Name, entryMetadata.FileSize, currentOffset)
+			if currentOffset != e.FileSize {
+				return nil, fmt.Errorf("inconsistent size of '%v': %v (got %v)", e.Name, e.FileSize, currentOffset)
 			}
 
 			bundles = append(bundles, bundle)
 		} else {
-			entries = append(entries, &entryMetadata)
+			entries = append(entries, e)
 		}
 	}
 

fs/dir_test.go

@@ -1,11 +1,12 @@
 package fs
 
 import (
-	"os"
 	"reflect"
 	"strings"
 	"testing"
 	"time"
+
+	"github.com/kopia/kopia/repo"
 )
 
 var (
@@ -14,147 +15,143 @@ var (
 	time3 = mustParseTimestamp("2016-04-02T02:36:19Z")
 )
 
-func TestEmptyDirectory(t *testing.T) {
-	data := strings.Join(
-		[]string{
-			`{`,
-			`"format":{"version":1},`,
-			`"entries":[]`,
-			`}`,
-		}, "")
-
-	expectedEntries := []*EntryMetadata{}
-
-	verifyDirectory(t, data, expectedEntries)
-}
-
-func TestDirectoryWithOnlyBundle(t *testing.T) {
-	data := strings.Join(
-		[]string{
-			`{`,
-			`"format":{"version":1},`,
-			`"entries":[`,
-			`{"name":"bundle1","size":"170","perm":"0","oid":"D5555","entries":[`,
-			`{"name":"a1","perm":"500","mtime":"2016-04-06T02:34:10Z","owner":"500:100","size":"50"},`,
-			`{"name":"z1","perm":"500","mtime":"2016-04-06T02:34:10Z","owner":"500:100","size":"120"}`,
-			`]}`,
-			`]}`,
-		}, "")
-
-	expectedEntries := []*EntryMetadata{
-		&EntryMetadata{Name: "a1", FileMode: 0500, ModTime: time1, OwnerID: 500, GroupID: 100, FileSize: 50, ObjectID: "S0,50,D5555"},
-		&EntryMetadata{Name: "z1", FileMode: 0500, ModTime: time1, OwnerID: 500, GroupID: 100, FileSize: 120, ObjectID: "S50,120,D5555"},
+func TestFlattenBundles(t *testing.T) {
+	base := &repo.ObjectID{StorageBlock: "5555"}
+	sources := []*EntryMetadata{
+		&EntryMetadata{
+			Name:     "bundle1",
+			FileSize: 170,
+			ObjectId: base,
+			BundledChildren: []*EntryMetadata{
+				&EntryMetadata{Name: "a1", FileSize: 50},
+				&EntryMetadata{Name: "z1", FileSize: 120},
+			},
+		},
 	}
 
-	verifyDirectory(t, data, expectedEntries)
-}
-
-func TestInconsistentBundleSize(t *testing.T) {
-	data := strings.Join(
-		[]string{
-			`{`,
-			`"format":{"version":1},`,
-			`"entries":[`,
-			`{"name":"bundle1","size":"170","perm":"0","oid":"D5555","entries":[`,
-			`{"name":"a1","perm":"500","mtime":"2016-04-06T02:34:10Z","owner":"500:100","size":"51"},`,
-			`{"name":"z1","perm":"500","mtime":"2016-04-06T02:34:10Z","owner":"500:100","size":"120"}`,
-			`]}`,
-			`]}`,
-		}, "")
-	verifyDirectoryError(t, data, "inconsistent size of 'bundle1': 170 (got 171)")
-}
-
-func TestInvalidBundleHeaderData(t *testing.T) {
-	data := strings.Join(
-		[]string{
-			`{`,
-			`"format":{"version":1},`,
-			`"entries":[`,
-			`{"name":"bundle1","size":"170","perm":"x","oid":"D5555","entries":[`,
-			`{"name":"z1","perm":"500","mtime":"2016-04-06T02:34:10Z","owner":"500:100","size":"120"}`,
-			`]}`,
-			`]}`,
-		}, "")
-	verifyDirectoryError(t, data, "invalid permissions: 'x'")
-}
-
-func TestInvalidBundleEntryData(t *testing.T) {
-	data := strings.Join(
-		[]string{
-			`{`,
-			`"format":{"version":1},`,
-			`"entries":[`,
-			`{"name":"bundle1","size":"170","perm":"0","oid":"D5555","entries":[`,
-			`{"perm":"500","mtime":"2016-04-06T02:34:10Z","owner":"500:100","size":"51"},`,
-			`{"name":"z1","perm":"500","mtime":"2016-04-06T02:34:10Z","owner":"500:100","size":"120"}`,
-			`]}`,
-			`]}`,
-		}, "")
-	verifyDirectoryError(t, data, "empty entry name")
-}
-func TestDirectoryWithoutBundle(t *testing.T) {
-	data := strings.Join(
-		[]string{
-			`{`,
-			`"format":{"version":1},`,
-			`"entries":[`,
-			`{"name":"constants.go","perm":"420","size":"13","mtime":"2016-04-02T02:36:19Z","owner":"500:100","oid":"D5123"}`,
-			`]}`,
-		}, "")
-
-	expectedEntries := []*EntryMetadata{
-		&EntryMetadata{Name: "constants.go", FileMode: 0420, ModTime: time3, OwnerID: 500, GroupID: 100, FileSize: 13, ObjectID: "D5123"},
-	}
-
-	verifyDirectory(t, data, expectedEntries)
-}
-
-func TestDirectoryWithThreeBundles(t *testing.T) {
-	data := strings.Join(
-		[]string{
-			`{`,
-			`"format":{"version":1},`,
-			`"entries":[`,
-			`{"name":"bundle1","size":"170","perm":"0","oid":"D5555","entries":[`,
-			`{"name":"a1","perm":"500","mtime":"2016-04-06T02:34:10Z","owner":"500:100","size":"50"},`,
-			`{"name":"z1","perm":"500","mtime":"2016-04-06T02:34:10Z","owner":"500:100","size":"120"}`,
-			`]},`,
-			`{"name":"config.go","perm":"420","size":"937","mtime":"2016-04-02T02:39:44.123456789Z","owner":"500:100","oid":"D4321"},`,
-			`{"name":"constants.go","perm":"420","size":"13","mtime":"2016-04-02T02:36:19Z","owner":"500:100","oid":"D5123"},`,
-			`{"name":"subdir","type":"d","perm":"755","mtime":"2016-04-06T02:34:10Z","owner":"500:100","oid":"D1234"},`,
-			`{"name":"bundle3","size":"7","perm":"0","oid":"D8888","entries":[`,
-			`{"name":"a3","perm":"500","mtime":"2016-04-06T02:34:10Z","owner":"500:100","size":"5"},`,
-			`{"name":"z3","perm":"500","mtime":"2016-04-06T02:34:10Z","owner":"500:100","size":"2"}`,
-			`]},`,
-			`{"name":"bundle2","size":"170","perm":"0","oid":"D6666","entries":[`,
-			`{"name":"a2","perm":"500","mtime":"2016-04-06T02:34:10Z","owner":"500:100","size":"150"},`,
-			`{"name":"z2","perm":"500","mtime":"2016-04-06T02:34:10Z","owner":"500:100","size":"20"}`,
-			`]}`,
-			`]}`,
-		}, "")
-
-	expectedEntries := []*EntryMetadata{
-		&EntryMetadata{Name: "a1", FileMode: 0500, ModTime: time1, OwnerID: 500, GroupID: 100, FileSize: 50, ObjectID: "S0,50,D5555"},
-		&EntryMetadata{Name: "a2", FileMode: 0500, ModTime: time1, OwnerID: 500, GroupID: 100, FileSize: 150, ObjectID: "S0,150,D6666"},
-		&EntryMetadata{Name: "a3", FileMode: 0500, ModTime: time1, OwnerID: 500, GroupID: 100, FileSize: 5, ObjectID: "S0,5,D8888"},
-		&EntryMetadata{Name: "config.go", FileMode: 0420, ModTime: time2, OwnerID: 500, GroupID: 100, FileSize: 937, ObjectID: "D4321"},
-		&EntryMetadata{Name: "constants.go", FileMode: 0420, ModTime: time3, OwnerID: 500, GroupID: 100, FileSize: 13, ObjectID: "D5123"},
-		&EntryMetadata{Name: "subdir", FileMode: os.ModeDir | 0755, ModTime: time1, OwnerID: 500, GroupID: 100, ObjectID: "D1234"},
-		&EntryMetadata{Name: "z1", FileMode: 0500, ModTime: time1, OwnerID: 500, GroupID: 100, FileSize: 120, ObjectID: "S50,120,D5555"},
-		&EntryMetadata{Name: "z2", FileMode: 0500, ModTime: time1, OwnerID: 500, GroupID: 100, FileSize: 20, ObjectID: "S150,20,D6666"},
-		&EntryMetadata{Name: "z3", FileMode: 0500, ModTime: time1, OwnerID: 500, GroupID: 100, FileSize: 2, ObjectID: "S5,2,D8888"},
-	}
-
-	verifyDirectory(t, data, expectedEntries)
-}
-
-func verifyDirectory(t *testing.T, data string, expectedEntries []*EntryMetadata) {
-	entries, err := readDirectoryMetadataEntries(strings.NewReader(data))
+	entries, err := flattenBundles(sources)
 	if err != nil {
 		t.Errorf("can't read directory entries: %v", err)
 		return
 	}
 
+	expectedEntries := []*EntryMetadata{
+		&EntryMetadata{Name: "a1", FileSize: 50, ObjectId: &repo.ObjectID{Section: &repo.ObjectID_Section{
+			Start:  0,
+			Length: 50,
+			Base:   base,
+		}}},
+		&EntryMetadata{Name: "z1", FileSize: 120, ObjectId: &repo.ObjectID{Section: &repo.ObjectID_Section{
+			Start:  50,
+			Length: 120,
+			Base:   base,
+		}}},
+	}
+
+	verifyDirectory(t, entries, expectedEntries)
+}
+
+func TestFlattenBundlesInconsistentBundleSize(t *testing.T) {
+	sources := []*EntryMetadata{
+		&EntryMetadata{
+			Name:     "bundle1",
+			FileSize: 171,
+			ObjectId: &repo.ObjectID{StorageBlock: "5555"},
+			BundledChildren: []*EntryMetadata{
+				&EntryMetadata{Name: "a1", FileSize: 50},
+				&EntryMetadata{Name: "z1", FileSize: 120},
+			},
+		},
+	}
+
+	_, err := flattenBundles(sources)
+	if err == nil {
+		t.Errorf("expected error")
+		return
+	}
+
+	if ok := strings.Contains(err.Error(), "inconsistent size of 'bundle1'"); !ok {
+		t.Errorf("invalid error: %v", err)
+	}
+}
+
+func TestFlattenThreeBundles(t *testing.T) {
+	base1 := &repo.ObjectID{StorageBlock: "5555"}
+	base2 := &repo.ObjectID{StorageBlock: "6666"}
+	base3 := &repo.ObjectID{StorageBlock: "7777"}
+	sources := []*EntryMetadata{
+		&EntryMetadata{
+			Name:     "bundle1",
+			FileSize: 170,
+			ObjectId: base1,
+			BundledChildren: []*EntryMetadata{
+				&EntryMetadata{Name: "a1", FileSize: 50},
+				&EntryMetadata{Name: "z1", FileSize: 120},
+			},
+		},
+		&EntryMetadata{
+			Name:     "bundle3",
+			FileSize: 7,
+			ObjectId: base3,
+			BundledChildren: []*EntryMetadata{
+				&EntryMetadata{Name: "a3", FileSize: 5},
+				&EntryMetadata{Name: "z3", FileSize: 2},
+			},
+		},
+		&EntryMetadata{
+			Name:     "bundle2",
+			FileSize: 300,
+			ObjectId: base2,
+			BundledChildren: []*EntryMetadata{
+				&EntryMetadata{Name: "a2", FileSize: 100},
+				&EntryMetadata{Name: "z2", FileSize: 200},
+			},
+		},
+	}
+
+	entries, err := flattenBundles(sources)
+	if err != nil {
+		t.Errorf("can't read directory entries: %v", err)
+		return
+	}
+
+	expectedEntries := []*EntryMetadata{
+		&EntryMetadata{Name: "a1", FileSize: 50, ObjectId: &repo.ObjectID{Section: &repo.ObjectID_Section{
+			Start:  0,
+			Length: 50,
+			Base:   base1,
+		}}},
+		&EntryMetadata{Name: "a2", FileSize: 100, ObjectId: &repo.ObjectID{Section: &repo.ObjectID_Section{
+			Start:  0,
+			Length: 100,
+			Base:   base2,
+		}}},
+		&EntryMetadata{Name: "a3", FileSize: 5, ObjectId: &repo.ObjectID{Section: &repo.ObjectID_Section{
+			Start:  0,
+			Length: 5,
+			Base:   base3,
+		}}},
+		&EntryMetadata{Name: "z1", FileSize: 120, ObjectId: &repo.ObjectID{Section: &repo.ObjectID_Section{
+			Start:  50,
+			Length: 120,
+			Base:   base1,
+		}}},
+		&EntryMetadata{Name: "z2", FileSize: 200, ObjectId: &repo.ObjectID{Section: &repo.ObjectID_Section{
+			Start:  100,
+			Length: 200,
+			Base:   base2,
+		}}},
+		&EntryMetadata{Name: "z3", FileSize: 2, ObjectId: &repo.ObjectID{Section: &repo.ObjectID_Section{
+			Start:  5,
+			Length: 2,
+			Base:   base3,
+		}}},
+	}
+
+	verifyDirectory(t, entries, expectedEntries)
+}
+
+func verifyDirectory(t *testing.T, entries []*EntryMetadata, expectedEntries []*EntryMetadata) {
 	if len(entries) != len(expectedEntries) {
 		t.Errorf("expected %v entries, got %v", len(expectedEntries), len(entries))
 	}
@@ -164,34 +161,13 @@ func verifyDirectory(t *testing.T, data string, expectedEntries []*EntryMetadata
 			actual := entries[i]
 
 			if !reflect.DeepEqual(expected, actual) {
-				t.Errorf("invalid entry at index %v:\nexpected: %#v\nactual:   %#v", i, expected, actual)
+				t.Errorf("invalid entry at index %v:\nexpected: %v\nactual:   %v", i,
+					expected.String(), actual.String())
 			}
 		}
 	}
 }
 
-func TestInvalidJSON(t *testing.T) {
-	verifyDirectoryError(t, "{invalid", "invalid character 'i'")
-	verifyDirectoryError(t, `{"format":{"version":1},"entries":[{"name":""}]}`, "empty entry name")
-	verifyDirectoryError(t, `{"format":{"version":1},"entries":[{"name":"x","perm":"x123"}]}`, "invalid permissions: 'x123'")
-	verifyDirectoryError(t, `{"format":{"version":2},"entries":[]}`, "unsupported version: 2")
-}
-
-func verifyDirectoryError(t *testing.T, data, expectedError string) {
-	entries, err := readDirectoryMetadataEntries(strings.NewReader(data))
-	if err == nil {
-		t.Errorf("expected error %v, got no error", expectedError)
-	} else {
-		if !strings.Contains(err.Error(), expectedError) {
-			t.Errorf("expected error containing '%v', got '%v'", expectedError, err.Error())
-		}
-	}
-
-	if entries != nil {
-		t.Errorf("got unexpected result: %#v", entries)
-	}
-}
-
 func TestDirectoryNameOrder(t *testing.T) {
 	sortedNames := []string{
 		"a/a/a",

fs/entry.go

@@ -8,8 +8,6 @@ import (
 	"sort"
 	"strings"
 	"time"
-
-	"github.com/kopia/kopia/repo"
 )
 
 // Entry represents a filesystem entry, which can be Directory, File, or Symlink
@@ -45,17 +43,6 @@ type Symlink interface {
 	Readlink() (string, error)
 }
 
-// EntryMetadata stores attributes of a single entry in a directory.
-type EntryMetadata struct {
-	Name     string
-	FileMode os.FileMode
-	FileSize int64
-	ModTime  time.Time
-	OwnerID  uint32
-	GroupID  uint32
-	ObjectID repo.ObjectID
-}
-
 type entry struct {
 	parent   Directory
 	metadata *EntryMetadata
@@ -133,15 +120,24 @@ func isLessOrEqual(name1, name2 string) bool {
 	return len(parts1) <= len(parts2)
 }
 
+// FileMode returns the file mode and permissions.
+func (e *EntryMetadata) FileMode() os.FileMode {
+	return os.FileMode(e.ModeAndPermissions)
+}
+
+// ModTime returns the modification time.
+func (e *EntryMetadata) ModTime() time.Time {
+	return time.Unix(e.ModTimestampNano/1000000000, e.ModTimestampNano%1000000000)
+
+}
+
 func (e *EntryMetadata) metadataHash() uint64 {
 	h := fnv.New64a()
-	binary.Write(h, binary.LittleEndian, e.ModTime.UnixNano())
-	binary.Write(h, binary.LittleEndian, e.FileMode)
-	if e.FileMode.IsRegular() {
-		binary.Write(h, binary.LittleEndian, e.FileSize)
-	}
-	binary.Write(h, binary.LittleEndian, e.OwnerID)
-	binary.Write(h, binary.LittleEndian, e.GroupID)
+	binary.Write(h, binary.LittleEndian, e.ModTimestampNano)
+	binary.Write(h, binary.LittleEndian, e.ModeAndPermissions)
+	binary.Write(h, binary.LittleEndian, e.FileSize)
+	binary.Write(h, binary.LittleEndian, e.Uid)
+	binary.Write(h, binary.LittleEndian, e.Gid)
 	return h.Sum64()
 }
 

fs/entry_metadata.pb.go

@@ -0,0 +1,89 @@
+// Code generated by protoc-gen-go.
+// source: fs/entry_metadata.proto
+// DO NOT EDIT!
+
+/*
+Package fs is a generated protocol buffer package.
+
+It is generated from these files:
+	fs/entry_metadata.proto
+	fs/hashcache.proto
+
+It has these top-level messages:
+	EntryMetadata
+	HashCacheEntry
+*/
+package fs
+
+import proto "github.com/golang/protobuf/proto"
+import fmt "fmt"
+import math "math"
+import kopia_repo "github.com/kopia/kopia/repo"
+
+// Reference imports to suppress errors if they are not otherwise used.
+var _ = proto.Marshal
+var _ = fmt.Errorf
+var _ = math.Inf
+
+// This is a compile-time assertion to ensure that this generated file
+// is compatible with the proto package it is being compiled against.
+const _ = proto.ProtoPackageIsVersion1
+
+// EntryMetadata stores attributes of a single entry in a directory.
+type EntryMetadata struct {
+	Name               string               `protobuf:"bytes,1,opt,name=name" json:"name,omitempty"`
+	ModeAndPermissions uint32               `protobuf:"fixed32,2,opt,name=mode_and_permissions,json=modeAndPermissions" json:"mode_and_permissions,omitempty"`
+	FileSize           int64                `protobuf:"varint,3,opt,name=file_size,json=fileSize" json:"file_size,omitempty"`
+	ModTimestampNano   int64                `protobuf:"fixed64,4,opt,name=mod_timestamp_nano,json=modTimestampNano" json:"mod_timestamp_nano,omitempty"`
+	ObjectId           *kopia_repo.ObjectID `protobuf:"bytes,5,opt,name=object_id,json=objectId" json:"object_id,omitempty"`
+	Uid                uint32               `protobuf:"varint,6,opt,name=uid" json:"uid,omitempty"`
+	Gid                uint32               `protobuf:"varint,7,opt,name=gid" json:"gid,omitempty"`
+	BundledChildren    []*EntryMetadata     `protobuf:"bytes,8,rep,name=bundled_children,json=bundledChildren" json:"bundled_children,omitempty"`
+}
+
+func (m *EntryMetadata) Reset()                    { *m = EntryMetadata{} }
+func (m *EntryMetadata) String() string            { return proto.CompactTextString(m) }
+func (*EntryMetadata) ProtoMessage()               {}
+func (*EntryMetadata) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{0} }
+
+func (m *EntryMetadata) GetObjectId() *kopia_repo.ObjectID {
+	if m != nil {
+		return m.ObjectId
+	}
+	return nil
+}
+
+func (m *EntryMetadata) GetBundledChildren() []*EntryMetadata {
+	if m != nil {
+		return m.BundledChildren
+	}
+	return nil
+}
+
+func init() {
+	proto.RegisterType((*EntryMetadata)(nil), "kopia.fs.EntryMetadata")
+}
+
+var fileDescriptor0 = []byte{
+	// 306 bytes of a gzipped FileDescriptorProto
+	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0x54, 0x90, 0xcd, 0x4e, 0x02, 0x31,
+	0x14, 0x85, 0x03, 0x83, 0x30, 0xd4, 0x10, 0x27, 0x95, 0x84, 0x2a, 0x1b, 0xe2, 0xca, 0x85, 0x29,
+	0xfe, 0x3c, 0x81, 0xa8, 0x0b, 0x16, 0xfe, 0xa4, 0xba, 0x72, 0xd3, 0x74, 0x68, 0x07, 0xaa, 0xb4,
+	0x9d, 0x4c, 0xcb, 0x42, 0x5f, 0xcd, 0x97, 0xf3, 0x76, 0x66, 0x88, 0x71, 0xd3, 0xdc, 0x9e, 0xf3,
+	0x9d, 0xa6, 0xf7, 0xa0, 0x49, 0xe1, 0xe7, 0xca, 0x86, 0xea, 0x8b, 0x1b, 0x15, 0x84, 0x14, 0x41,
+	0xd0, 0xb2, 0x72, 0xc1, 0xe1, 0xf4, 0xd3, 0x95, 0x5a, 0xd0, 0xc2, 0x9f, 0x1e, 0x57, 0xaa, 0x74,
+	0x73, 0x97, 0x7f, 0xa8, 0x55, 0xd0, 0xb2, 0xb1, 0xcf, 0x7e, 0xba, 0x68, 0xf4, 0x10, 0x73, 0x8f,
+	0x6d, 0x0c, 0x63, 0xd4, 0xb3, 0xc2, 0x28, 0xd2, 0x99, 0x75, 0xce, 0x87, 0xac, 0x9e, 0xf1, 0x25,
+	0x1a, 0x1b, 0x27, 0x15, 0x17, 0x56, 0xf2, 0x52, 0x55, 0x46, 0x7b, 0xaf, 0x9d, 0xf5, 0xa4, 0x0b,
+	0xcc, 0x80, 0xe1, 0xe8, 0xdd, 0x5a, 0xf9, 0xf2, 0xe7, 0xe0, 0x29, 0x1a, 0x16, 0x7a, 0xab, 0xb8,
+	0xd7, 0xdf, 0x8a, 0x24, 0x80, 0x25, 0x2c, 0x8d, 0xc2, 0x2b, 0xdc, 0xf1, 0x05, 0x8a, 0x11, 0x1e,
+	0xb4, 0x51, 0x3e, 0x08, 0x53, 0x72, 0x2b, 0xac, 0x23, 0x3d, 0xa0, 0x32, 0x96, 0x81, 0xf3, 0xb6,
+	0x37, 0x9e, 0x40, 0xc7, 0x57, 0x68, 0xd8, 0x7c, 0x9a, 0x6b, 0x49, 0x0e, 0x00, 0x3a, 0xbc, 0x1e,
+	0xd3, 0x66, 0xab, 0xb8, 0x11, 0x7d, 0xae, 0xcd, 0xe5, 0x3d, 0x4b, 0x1b, 0x6c, 0x29, 0x71, 0x86,
+	0x92, 0x1d, 0xc0, 0x7d, 0x80, 0x47, 0x2c, 0x8e, 0x51, 0x59, 0x83, 0x32, 0x68, 0x14, 0x18, 0xf1,
+	0x02, 0x65, 0xf9, 0xce, 0xca, 0xad, 0x92, 0x7c, 0xb5, 0xd1, 0x5b, 0x59, 0x29, 0x4b, 0xd2, 0x59,
+	0x02, 0xaf, 0x4f, 0xe8, 0xbe, 0x33, 0xfa, 0xaf, 0x1a, 0x76, 0xd4, 0x06, 0xee, 0x5a, 0x7e, 0x31,
+	0x7d, 0x3f, 0x59, 0xeb, 0xb0, 0xd9, 0xe5, 0x74, 0xe5, 0xcc, 0xbc, 0x4e, 0xb5, 0x67, 0xe1, 0xf3,
+	0x7e, 0xdd, 0xf0, 0xcd, 0x6f, 0x00, 0x00, 0x00, 0xff, 0xff, 0xcf, 0xf4, 0xc2, 0x88, 0x9b, 0x01,
+	0x00, 0x00,
+}

fs/entry_metadata.proto

@@ -0,0 +1,19 @@
+syntax = "proto3";
+
+package kopia.fs;
+option go_package = "github.com/kopia/kopia/fs";
+
+import "repo/objectid.proto";
+
+// EntryMetadata stores attributes of a single entry in a directory.
+message EntryMetadata {
+    string name = 1;
+    fixed32 mode_and_permissions = 2;
+    int64 file_size = 3;
+    sfixed64 mod_timestamp_nano = 4;
+    repo.ObjectID object_id = 5;
+    uint32 uid = 6;
+    uint32 gid = 7;
+
+    repeated EntryMetadata bundled_children = 8;
+}

fs/hashcache.go

@@ -4,36 +4,28 @@ import (
 	"bufio"
 	"fmt"
 	"io"
-	"strconv"
-	"strings"
 
-	"github.com/kopia/kopia/repo"
+	"github.com/kopia/kopia/internal"
 )
 
-type hashCacheEntry struct {
-	Name     string
-	Hash     uint64
-	ObjectID repo.ObjectID
-}
-
 type hashcacheReader struct {
-	scanner   *bufio.Scanner
-	nextEntry *hashCacheEntry
-	entry0    hashCacheEntry
-	entry1    hashCacheEntry
+	reader    *internal.ProtoStreamReader
+	nextEntry *HashCacheEntry
+	entry0    HashCacheEntry
+	entry1    HashCacheEntry
 	odd       bool
 	first     bool
 }
 
 func (hcr *hashcacheReader) open(r io.Reader) error {
-	hcr.scanner = bufio.NewScanner(r)
+	hcr.reader = internal.NewProtoStreamReader(bufio.NewReader(r), internal.ProtoStreamTypeHashCache)
 	hcr.nextEntry = nil
 	hcr.first = true
 	hcr.readahead()
 	return nil
 }
 
-func (hcr *hashcacheReader) findEntry(relativeName string) *hashCacheEntry {
+func (hcr *hashcacheReader) findEntry(relativeName string) *HashCacheEntry {
 	for hcr.nextEntry != nil && isLess(hcr.nextEntry.Name, relativeName) {
 		hcr.readahead()
 	}
@@ -49,37 +41,20 @@ func (hcr *hashcacheReader) findEntry(relativeName string) *hashCacheEntry {
 }
 
 func (hcr *hashcacheReader) readahead() {
-	if hcr.scanner != nil {
-		if hcr.first {
-			hcr.first = false
-			if !hcr.scanner.Scan() || hcr.scanner.Text() != "HASHCACHE:v1" {
-				hcr.scanner = nil
-				return
-			}
-		}
-
+	if hcr.reader != nil {
 		hcr.nextEntry = nil
-		if hcr.scanner.Scan() {
-			var err error
-			e := hcr.nextManifestEntry()
-			t := hcr.scanner.Text()
-			p0 := strings.IndexByte(t, '|')
-			p1 := strings.LastIndexByte(t, '|')
-			e.Name = t[0:p0]
-			e.Hash, err = strconv.ParseUint(t[p0+1:p1], 0, 64)
-			e.ObjectID = repo.ObjectID(t[p1+1:])
-			if err == nil {
-				hcr.nextEntry = e
-			}
+		e := hcr.nextManifestEntry()
+		if err := hcr.reader.Read(e); err == nil {
+			hcr.nextEntry = e
 		}
 	}
 
 	if hcr.nextEntry == nil {
-		hcr.scanner = nil
+		hcr.reader = nil
 	}
 }
 
-func (hcr *hashcacheReader) nextManifestEntry() *hashCacheEntry {
+func (hcr *hashcacheReader) nextManifestEntry() *HashCacheEntry {
 	hcr.odd = !hcr.odd
 	if hcr.odd {
 		return &hcr.entry1
@@ -89,19 +64,18 @@ func (hcr *hashcacheReader) nextManifestEntry() *hashCacheEntry {
 }
 
 type hashcacheWriter struct {
-	writer          io.Writer
+	writer          *internal.ProtoStreamWriter
 	lastNameWritten string
 }
 
 func newHashCacheWriter(w io.Writer) *hashcacheWriter {
 	hcw := &hashcacheWriter{
-		writer: w,
+		writer: internal.NewProtoStreamWriter(w, internal.ProtoStreamTypeHashCache),
 	}
-	io.WriteString(w, "HASHCACHE:v1\n")
 	return hcw
 }
 
-func (hcw *hashcacheWriter) WriteEntry(e hashCacheEntry) error {
+func (hcw *hashcacheWriter) WriteEntry(e HashCacheEntry) error {
 	if hcw.lastNameWritten != "" {
 		if isLessOrEqual(e.Name, hcw.lastNameWritten) {
 			return fmt.Errorf("out-of-order directory entry, previous '%v' current '%v'", hcw.lastNameWritten, e.Name)
@@ -109,7 +83,7 @@ func (hcw *hashcacheWriter) WriteEntry(e hashCacheEntry) error {
 		hcw.lastNameWritten = e.Name
 	}
 
-	fmt.Fprintf(hcw.writer, "%v|%v|%v\n", e.Name, e.Hash, e.ObjectID)
+	hcw.writer.Write(&e)
 
 	return nil
 }

fs/hashcache.pb.go

@@ -0,0 +1,52 @@
+// Code generated by protoc-gen-go.
+// source: fs/hashcache.proto
+// DO NOT EDIT!
+
+package fs
+
+import proto "github.com/golang/protobuf/proto"
+import fmt "fmt"
+import math "math"
+import kopia_repo "github.com/kopia/kopia/repo"
+
+// Reference imports to suppress errors if they are not otherwise used.
+var _ = proto.Marshal
+var _ = fmt.Errorf
+var _ = math.Inf
+
+type HashCacheEntry struct {
+	Name     string               `protobuf:"bytes,1,opt,name=name" json:"name,omitempty"`
+	Hash     uint64               `protobuf:"fixed64,2,opt,name=hash" json:"hash,omitempty"`
+	ObjectId *kopia_repo.ObjectID `protobuf:"bytes,3,opt,name=object_id,json=objectId" json:"object_id,omitempty"`
+}
+
+func (m *HashCacheEntry) Reset()                    { *m = HashCacheEntry{} }
+func (m *HashCacheEntry) String() string            { return proto.CompactTextString(m) }
+func (*HashCacheEntry) ProtoMessage()               {}
+func (*HashCacheEntry) Descriptor() ([]byte, []int) { return fileDescriptor1, []int{0} }
+
+func (m *HashCacheEntry) GetObjectId() *kopia_repo.ObjectID {
+	if m != nil {
+		return m.ObjectId
+	}
+	return nil
+}
+
+func init() {
+	proto.RegisterType((*HashCacheEntry)(nil), "kopia.fs.HashCacheEntry")
+}
+
+var fileDescriptor1 = []byte{
+	// 173 bytes of a gzipped FileDescriptorProto
+	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0xe2, 0x12, 0x4a, 0x2b, 0xd6, 0xcf,
+	0x48, 0x2c, 0xce, 0x48, 0x4e, 0x4c, 0xce, 0x48, 0xd5, 0x2b, 0x28, 0xca, 0x2f, 0xc9, 0x17, 0xe2,
+	0xc8, 0xce, 0x2f, 0xc8, 0x4c, 0xd4, 0x4b, 0x2b, 0x96, 0x12, 0x2e, 0x4a, 0x2d, 0xc8, 0xd7, 0xcf,
+	0x4f, 0xca, 0x4a, 0x4d, 0x2e, 0xc9, 0x4c, 0x81, 0x48, 0x2b, 0x65, 0x73, 0xf1, 0x79, 0x00, 0x75,
+	0x38, 0x83, 0x74, 0xb8, 0xe6, 0x95, 0x14, 0x55, 0x0a, 0x09, 0x71, 0xb1, 0xe4, 0x25, 0xe6, 0xa6,
+	0x4a, 0x30, 0x2a, 0x30, 0x6a, 0x70, 0x06, 0x81, 0xd9, 0x20, 0x31, 0x90, 0xb9, 0x12, 0x4c, 0x40,
+	0x31, 0xb6, 0x20, 0x30, 0x5b, 0xc8, 0x90, 0x8b, 0x13, 0x62, 0x56, 0x7c, 0x66, 0x8a, 0x04, 0x33,
+	0x50, 0x82, 0xdb, 0x48, 0x44, 0x0f, 0x62, 0x19, 0xc8, 0x22, 0x3d, 0x7f, 0xb0, 0xa4, 0xa7, 0x4b,
+	0x10, 0x07, 0x44, 0x99, 0x67, 0x8a, 0x93, 0x74, 0x94, 0x64, 0x7a, 0x66, 0x49, 0x46, 0x69, 0x92,
+	0x5e, 0x72, 0x7e, 0xae, 0x3e, 0x58, 0x2d, 0x94, 0x4c, 0x2b, 0x4e, 0x62, 0x03, 0x3b, 0xc8, 0x18,
+	0x10, 0x00, 0x00, 0xff, 0xff, 0xa7, 0x08, 0x96, 0x2b, 0xc5, 0x00, 0x00, 0x00,
+}

fs/hashcache.proto

@@ -0,0 +1,12 @@
+syntax = "proto3";
+
+package kopia.fs;
+option go_package = "github.com/kopia/kopia/fs";
+
+import "repo/objectid.proto";
+
+message HashCacheEntry {
+    string name = 1;
+    fixed64 hash = 2;
+    repo.ObjectID object_id = 3;
+}

fs/inmemory_fs_for_test.go

@@ -35,8 +35,8 @@ func (imd *inmemoryDirectory) addFile(name string, content []byte, mode os.FileM
 		inmemoryEntry: inmemoryEntry{
 			parent: imd,
 			metadata: &EntryMetadata{
-				Name:     name,
-				FileMode: mode,
+				Name:               name,
+				ModeAndPermissions: uint32(mode),
 			},
 		},
 		source: bytes.NewBuffer(content),
@@ -54,8 +54,8 @@ func (imd *inmemoryDirectory) addDir(name string, mode os.FileMode) *inmemoryDir
 		inmemoryEntry: inmemoryEntry{
 			parent: imd,
 			metadata: &EntryMetadata{
-				Name:     name,
-				FileMode: mode | os.ModeDir,
+				Name:               name,
+				ModeAndPermissions: uint32(mode | os.ModeDir),
 			},
 		},
 	}
@@ -88,7 +88,7 @@ func (imd *inmemoryDirectory) subdir(name ...string) *inmemoryDirectory {
 		if i2 == nil {
 			panic(fmt.Sprintf("'%s' not found in '%s'", n, i.metadata.Name))
 		}
-		if !i2.Metadata().FileMode.IsDir() {
+		if !i2.Metadata().FileMode().IsDir() {
 			panic(fmt.Sprintf("'%s' is not a directory in '%s'", n, i.metadata.Name))
 		}
 

fs/local_fs.go

@@ -107,9 +107,9 @@ func NewFilesystemDirectory(path string, parent Directory) (Directory, error) {
 
 func entryMetadataFromFileInfo(fi os.FileInfo) *EntryMetadata {
 	e := &EntryMetadata{
-		Name:     filepath.Base(fi.Name()),
-		FileMode: fi.Mode(),
-		ModTime:  fi.ModTime(),
+		Name:               filepath.Base(fi.Name()),
+		ModeAndPermissions: uint32(fi.Mode()),
+		ModTimestampNano:   fi.ModTime().UnixNano(),
 	}
 
 	if fi.Mode().IsRegular() {

fs/local_fs_nonwindows.go

@@ -9,8 +9,8 @@ import (
 
 func (e *EntryMetadata) populatePlatformSpecificEntryDetails(fi os.FileInfo) error {
 	if stat, ok := fi.Sys().(*syscall.Stat_t); ok {
-		e.OwnerID = stat.Uid
-		e.GroupID = stat.Gid
+		e.Uid = stat.Uid
+		e.Gid = stat.Gid
 	}
 
 	return nil

fs/local_fs_test.go

@@ -64,19 +64,19 @@ func TestFiles(t *testing.T) {
 
 	goodCount := 0
 
-	if entries[0].Metadata().Name == "f1" && entries[0].Metadata().FileSize == 5 && entries[0].Metadata().FileMode.IsRegular() {
+	if entries[0].Metadata().Name == "f1" && entries[0].Metadata().FileSize == 5 && entries[0].Metadata().FileMode().IsRegular() {
 		goodCount++
 	}
-	if entries[1].Metadata().Name == "f2" && entries[1].Metadata().FileSize == 4 && entries[1].Metadata().FileMode.IsRegular() {
+	if entries[1].Metadata().Name == "f2" && entries[1].Metadata().FileSize == 4 && entries[1].Metadata().FileMode().IsRegular() {
 		goodCount++
 	}
-	if entries[2].Metadata().Name == "f3" && entries[2].Metadata().FileSize == 3 && entries[2].Metadata().FileMode.IsRegular() {
+	if entries[2].Metadata().Name == "f3" && entries[2].Metadata().FileSize == 3 && entries[2].Metadata().FileMode().IsRegular() {
 		goodCount++
 	}
-	if entries[3].Metadata().Name == "y" && entries[3].Metadata().FileSize == 0 && entries[3].Metadata().FileMode.IsDir() {
+	if entries[3].Metadata().Name == "y" && entries[3].Metadata().FileSize == 0 && entries[3].Metadata().FileMode().IsDir() {
 		goodCount++
 	}
-	if entries[4].Metadata().Name == "z" && entries[4].Metadata().FileSize == 0 && entries[4].Metadata().FileMode.IsDir() {
+	if entries[4].Metadata().Name == "z" && entries[4].Metadata().FileSize == 0 && entries[4].Metadata().FileMode().IsDir() {
 		goodCount++
 	}
 	if goodCount != 5 {

fs/repository_fs.go

@@ -18,7 +18,7 @@ type repositoryFile repositoryEntry
 type repositorySymlink repositoryEntry
 
 func (rd *repositoryDirectory) Readdir() (Entries, error) {
-	r, err := rd.repo.Open(rd.entry.Metadata().ObjectID)
+	r, err := rd.repo.Open(*rd.entry.Metadata().ObjectId)
 	if err != nil {
 		return nil, err
 	}
@@ -38,7 +38,7 @@ func (rd *repositoryDirectory) Readdir() (Entries, error) {
 }
 
 func (rf *repositoryFile) Open() (EntryMetadataReadCloser, error) {
-	r, err := rf.repo.Open(rf.entry.Metadata().ObjectID)
+	r, err := rf.repo.Open(*rf.entry.Metadata().ObjectId)
 	if err != nil {
 		return nil, err
 	}
@@ -51,7 +51,7 @@ func (rsl *repositorySymlink) Readlink() (string, error) {
 }
 
 func newRepoEntry(r repo.Repository, md *EntryMetadata, parent Directory) Entry {
-	switch md.FileMode & os.ModeType {
+	switch md.FileMode() & os.ModeType {
 	case os.ModeDir:
 		return Directory(&repositoryDirectory{
 			entry: newEntry(md, parent),
@@ -71,7 +71,7 @@ func newRepoEntry(r repo.Repository, md *EntryMetadata, parent Directory) Entry
 		})
 
 	default:
-		panic(fmt.Sprintf("not supported entry metadata type: %v", md.FileMode))
+		panic(fmt.Sprintf("not supported entry metadata type: %v", md.FileMode()))
 	}
 }
 
@@ -94,9 +94,9 @@ func withMetadata(rc io.ReadCloser, md *EntryMetadata) EntryMetadataReadCloser {
 // NewRepositoryDirectory returns Directory based on repository object with the specified ID.
 func NewRepositoryDirectory(r repo.Repository, objectID repo.ObjectID) Directory {
 	d := newRepoEntry(r, &EntryMetadata{
-		Name:     "/",
-		ObjectID: objectID,
-		FileMode: 0555 | os.ModeDir,
+		Name:               "/",
+		ObjectId:           &objectID,
+		ModeAndPermissions: 0555 | uint32(os.ModeDir),
 	}, nil)
 
 	return d.(Directory)

fs/upload.go

@@ -36,7 +36,7 @@ type UploadResult struct {
 
 // Uploader supports efficient uploading files and directories to repository.
 type Uploader interface {
-	UploadDir(dir Directory, previousManifestID repo.ObjectID) (*UploadResult, error)
+	UploadDir(dir Directory, previousManifestID *repo.ObjectID) (*UploadResult, error)
 	UploadFile(file File) (*UploadResult, error)
 	Cancel()
 }
@@ -74,11 +74,13 @@ func (u *uploader) uploadFileInternal(f File, relativePath string, forceStored b
 		return nil, 0, err
 	}
 
-	e2.ObjectID, err = writer.Result(forceStored)
+	r, err := writer.Result(forceStored)
 	if err != nil {
 		return nil, 0, err
 	}
 
+	e2.ObjectId = &r
+
 	if written != e2.FileSize {
 		// file changed
 	}
@@ -123,10 +125,11 @@ func (u *uploader) uploadBundleInternal(b *uploadBundle) (*EntryMetadata, uint64
 	}
 
 	b.files = uploadedFiles
-	bundleMetadata.ObjectID, err = writer.Result(true)
+	r, err := writer.Result(true)
 	if err != nil {
 		return nil, 0, err
 	}
+	bundleMetadata.ObjectId = &r
 
 	return bundleMetadata, bundleMetadata.metadataHash(), nil
 }
@@ -134,16 +137,16 @@ func (u *uploader) uploadBundleInternal(b *uploadBundle) (*EntryMetadata, uint64
 func (u *uploader) UploadFile(file File) (*UploadResult, error) {
 	result := &UploadResult{}
 	e, _, err := u.uploadFileInternal(file, file.Metadata().Name, true)
-	result.ObjectID = e.ObjectID
+	result.ObjectID = *e.ObjectId
 	return result, err
 }
 
-func (u *uploader) UploadDir(dir Directory, previousManifestID repo.ObjectID) (*UploadResult, error) {
+func (u *uploader) UploadDir(dir Directory, previousManifestID *repo.ObjectID) (*UploadResult, error) {
 	var mr hashcacheReader
 	var err error
 
-	if previousManifestID != "" {
-		if r, err := u.repo.Open(previousManifestID); err == nil {
+	if previousManifestID != nil {
+		if r, err := u.repo.Open(*previousManifestID); err == nil {
 			mr.open(r)
 		}
 	}
@@ -178,7 +181,7 @@ func (u *uploader) uploadDirInternal(
 
 	entries, err := dir.Readdir()
 	if err != nil {
-		return "", 0, false, err
+		return repo.NullObjectID, 0, false, err
 	}
 
 	entries = u.bundleEntries(entries)
@@ -207,20 +210,22 @@ func (u *uploader) uploadDirInternal(
 		case Directory:
 			oid, h, wasCached, err := u.uploadDirInternal(result, entry, entryRelativePath, hcw, mr, false)
 			if err != nil {
-				return "", 0, false, err
+				return repo.NullObjectID, 0, false, err
 			}
 			//log.Printf("dirHash: %v %v", fullPath, h)
 			hash = h
 			allCached = allCached && wasCached
-			e.ObjectID = oid
+			e.ObjectId = &oid
 
 		case Symlink:
 			l, err := entry.Readlink()
 			if err != nil {
-				return "", 0, false, err
+				return repo.NullObjectID, 0, false, err
 			}
 
-			e.ObjectID = repo.NewInlineObjectID([]byte(l))
+			id := repo.NewInlineObjectID([]byte(l))
+
+			e.ObjectId = &id
 			hash = e.metadataHash()
 
 		case *uploadBundle:
@@ -239,13 +244,14 @@ func (u *uploader) uploadDirInternal(
 			if cacheMatches {
 				result.Stats.CachedFiles++
 				// Avoid hashing by reusing previous object ID.
-				e.ObjectID = repo.ObjectID(cachedEntry.ObjectID)
+				id := *cachedEntry.ObjectId
+				e.ObjectId = &id
 				hash = cachedEntry.Hash
 			} else {
 				result.Stats.NonCachedFiles++
 				e, hash, err = u.uploadBundleInternal(entry)
 				if err != nil {
-					return "", 0, false, fmt.Errorf("unable to hash file: %s", err)
+					return repo.NullObjectID, 0, false, fmt.Errorf("unable to hash file: %s", err)
 				}
 			}
 
@@ -262,18 +268,19 @@ func (u *uploader) uploadDirInternal(
 			if cacheMatches {
 				result.Stats.CachedFiles++
 				// Avoid hashing by reusing previous object ID.
-				e.ObjectID = repo.ObjectID(cachedEntry.ObjectID)
+				id := *cachedEntry.ObjectId
+				e.ObjectId = &id
 				hash = cachedEntry.Hash
 			} else {
 				result.Stats.NonCachedFiles++
 				e, hash, err = u.uploadFileInternal(entry, entryRelativePath, false)
 				if err != nil {
-					return "", 0, false, fmt.Errorf("unable to hash file: %s", err)
+					return repo.NullObjectID, 0, false, fmt.Errorf("unable to hash file: %s", err)
 				}
 			}
 
 		default:
-			return "", 0, false, fmt.Errorf("file type %v not supported", entry.Metadata().FileMode)
+			return repo.NullObjectID, 0, false, fmt.Errorf("file type %v not supported", entry.Metadata().FileMode())
 		}
 
 		if hash != 0 {
@@ -283,22 +290,20 @@ func (u *uploader) uploadDirInternal(
 		}
 
 		if err := dw.WriteEntry(e, childrenMetadata); err != nil {
-			return "", 0, false, err
+			return repo.NullObjectID, 0, false, err
 		}
 
-		if !e.FileMode.IsDir() && e.ObjectID.Type().IsStored() {
-			if err := hcw.WriteEntry(hashCacheEntry{
+		if !e.FileMode().IsDir() && e.ObjectId.StorageBlock != "" {
+			if err := hcw.WriteEntry(HashCacheEntry{
 				Name:     entryRelativePath,
-				Hash:     e.metadataHash(),
-				ObjectID: e.ObjectID,
+				Hash:     hash,
+				ObjectId: e.ObjectId,
 			}); err != nil {
-				return "", 0, false, err
+				return repo.NullObjectID, 0, false, err
 			}
 		}
 	}
 
-	dw.Close()
-
 	var directoryOID repo.ObjectID
 	dirHash := dirHasher.Sum64()
 
@@ -308,7 +313,7 @@ func (u *uploader) uploadDirInternal(
 	if allCached {
 		// Avoid hashing directory listing if every entry matched the cache.
 		result.Stats.CachedDirectories++
-		directoryOID = repo.ObjectID(cachedDirEntry.ObjectID)
+		directoryOID = *cachedDirEntry.ObjectId
 	} else {
 		result.Stats.NonCachedDirectories++
 		directoryOID, err = writer.Result(forceStored)
@@ -317,14 +322,15 @@ func (u *uploader) uploadDirInternal(
 		}
 	}
 
-	if err := hcw.WriteEntry(hashCacheEntry{
+	if err := hcw.WriteEntry(HashCacheEntry{
 		Name:     relativePath + "/",
-		ObjectID: directoryOID,
+		ObjectId: &directoryOID,
 		Hash:     dirHash,
 	}); err != nil {
-		return "", 0, false, err
+		return repo.NullObjectID, 0, false, err
 	}
 
+	// log.Printf("Dir: %v %v %v %v", relativePath, directoryOID.UIString(), dirHash, allCached)
 	return directoryOID, dirHash, allCached, nil
 }
 
@@ -398,8 +404,8 @@ func (u *uploader) bundleEntries(entries Entries) Entries {
 func getBundleNumber(md *EntryMetadata) int {
 	// TODO(jkowalski): This is not ready yet, uncomment when ready.
 
-	// if md.FileMode.IsRegular() && md.FileSize < maxBundleFileSize {
-	// 	return md.ModTime.Year()*100 + int(md.ModTime.Month())
+	// if md.FileMode().IsRegular() && md.FileSize < maxBundleFileSize {
+	// 	return md.ModTime().Year()*100 + int(md.ModTime().Month())
 	// }
 
 	return 0

fs/upload_test.go

@@ -41,8 +41,8 @@ func newUploadTestHarness() *uploadTestHarness {
 	}
 
 	format := repo.Format{
-		Version:           "1",
-		ObjectFormat:      "md5",
+		Version:           1,
+		ObjectFormat:      repo.ObjectIDFormat_TESTONLY_MD5,
 		MaxBlobSize:       1000,
 		MaxInlineBlobSize: 0,
 	}
@@ -86,22 +86,22 @@ func TestUpload(t *testing.T) {
 	th := newUploadTestHarness()
 	defer th.cleanup()
 
-	r1, err := th.uploader.UploadDir(th.sourceDir, "")
+	r1, err := th.uploader.UploadDir(th.sourceDir, nil)
 	if err != nil {
 		t.Errorf("upload failed: %v", err)
 	}
 
-	r2, err := th.uploader.UploadDir(th.sourceDir, r1.ManifestID)
+	r2, err := th.uploader.UploadDir(th.sourceDir, &r1.ManifestID)
 	if err != nil {
 		t.Errorf("upload failed: %v", err)
 	}
 
-	if r2.ObjectID != r1.ObjectID {
-		t.Errorf("expected r1.ObjectID==r2.ObjectID, got %v and %v", r1.ObjectID, r2.ObjectID)
+	if !r2.ObjectID.Equals(r1.ObjectID) {
+		t.Errorf("expected r1.ObjectID==r2.ObjectID, got %v and %v", r1.ObjectID.UIString(), r2.ObjectID.UIString())
 	}
 
-	if r2.ManifestID != r1.ManifestID {
-		t.Errorf("expected r2.ManifestID==r1.ManifestID, got %v and %v", r2.ManifestID, r1.ManifestID)
+	if !r2.ManifestID.Equals(r1.ManifestID) {
+		t.Errorf("expected r2.ManifestID==r1.ManifestID, got %v and %v", r2.ManifestID.UIString(), r1.ManifestID.UIString())
 	}
 
 	if r1.Stats.CachedFiles+r1.Stats.CachedDirectories != 0 {
@@ -116,17 +116,17 @@ func TestUpload(t *testing.T) {
 
 	// Add one more file, the r1.ObjectID should change.
 	th.sourceDir.addFile("d2/d1/f3", []byte{1, 2, 3, 4, 5}, 0777)
-	r3, err := th.uploader.UploadDir(th.sourceDir, r1.ManifestID)
+	r3, err := th.uploader.UploadDir(th.sourceDir, &r1.ManifestID)
 	if err != nil {
 		t.Errorf("upload failed: %v", err)
 	}
 
-	if r2.ObjectID == r3.ObjectID {
-		t.Errorf("expected r3.ObjectID!=r2.ObjectID, got %v", r3.ObjectID)
+	if r2.ObjectID.Equals(r3.ObjectID) {
+		t.Errorf("expected r3.ObjectID!=r2.ObjectID, got %v", r3.ObjectID.UIString())
 	}
 
-	if r2.ManifestID == r3.ManifestID {
-		t.Errorf("expected r3.ManifestID!=r2.ManifestID, got %v", r3.ManifestID)
+	if r2.ManifestID.Equals(r3.ManifestID) {
+		t.Errorf("expected r3.ManifestID!=r2.ManifestID, got %v", r3.ManifestID.UIString())
 	}
 
 	if r3.Stats.NonCachedFiles != 1 && r3.Stats.NonCachedDirectories != 3 {
@@ -137,15 +137,15 @@ func TestUpload(t *testing.T) {
 	// Now remove the added file, OID should be identical to the original before the file got added.
 	th.sourceDir.subdir("d2", "d1").remove("f3")
 
-	r4, err := th.uploader.UploadDir(th.sourceDir, r1.ManifestID)
+	r4, err := th.uploader.UploadDir(th.sourceDir, &r1.ManifestID)
 	if err != nil {
 		t.Errorf("upload failed: %v", err)
 	}
 
-	if r4.ObjectID != r1.ObjectID {
+	if !r4.ObjectID.Equals(r1.ObjectID) {
 		t.Errorf("expected r4.ObjectID==r1.ObjectID, got %v and %v", r4.ObjectID, r1.ObjectID)
 	}
-	if r4.ManifestID != r1.ManifestID {
+	if !r4.ManifestID.Equals(r1.ManifestID) {
 		t.Errorf("expected r4.ManifestID==r1.ManifestID, got %v and %v", r4.ManifestID, r1.ManifestID)
 	}
 
@@ -156,7 +156,7 @@ func TestUpload(t *testing.T) {
 	}
 
 	// Upload again, this time using r3.ManifestID as base.
-	r5, err := th.uploader.UploadDir(th.sourceDir, r3.ManifestID)
+	r5, err := th.uploader.UploadDir(th.sourceDir, &r3.ManifestID)
 	if err != nil {
 		t.Errorf("upload failed: %v", err)
 	}
@@ -176,7 +176,7 @@ func TestUpload_TopLevelDirectoryReadFailure(t *testing.T) {
 
 	th.sourceDir.failReaddir(errTest)
 
-	r, err := th.uploader.UploadDir(th.sourceDir, "")
+	r, err := th.uploader.UploadDir(th.sourceDir, nil)
 	if err != errTest {
 		t.Errorf("expected error: %v", err)
 	}
@@ -192,7 +192,7 @@ func TestUpload_SubDirectoryReadFailure(t *testing.T) {
 
 	th.sourceDir.subdir("d1").failReaddir(errTest)
 
-	_, err := th.uploader.UploadDir(th.sourceDir, "")
+	_, err := th.uploader.UploadDir(th.sourceDir, nil)
 	if err == nil {
 		t.Errorf("expected error")
 	}

fuse/fusefs.go

@@ -21,11 +21,11 @@ type fuseNode struct {
 
 func (n *fuseNode) Attr(ctx context.Context, a *fuse.Attr) error {
 	m := n.entry.Metadata()
-	a.Mode = m.FileMode
+	a.Mode = m.FileMode()
 	a.Size = uint64(m.FileSize)
-	a.Mtime = m.ModTime
-	a.Uid = m.OwnerID
-	a.Gid = m.GroupID
+	a.Mtime = m.ModTime()
+	a.Uid = m.Uid
+	a.Gid = m.Gid
 	return nil
 }
 
@@ -88,7 +88,7 @@ func (dir *fuseDirectoryNode) ReadDirAll(ctx context.Context) ([]fuse.Dirent, er
 			Name: m.Name,
 		}
 
-		switch m.FileMode & os.ModeType {
+		switch m.FileMode() & os.ModeType {
 		case os.ModeDir:
 			dirent.Type = fuse.DT_Dir
 		case 0:
@@ -112,7 +112,7 @@ type fuseSymlinkNode struct {
 }
 
 func (sl *fuseSymlinkNode) Readlink(ctx context.Context, req *fuse.ReadlinkRequest) (string, error) {
-	return string(sl.entry.Metadata().ObjectID), nil
+	return sl.entry.(fs.Symlink).Readlink()
 }
 
 func newFuseNode(e fs.Entry) (fusefs.Node, error) {
@@ -124,7 +124,7 @@ func newFuseNode(e fs.Entry) (fusefs.Node, error) {
 	case fs.Symlink:
 		return &fuseSymlinkNode{fuseNode{e}}, nil
 	default:
-		return nil, fmt.Errorf("entry type not supported: %v", e.Metadata().FileMode)
+		return nil, fmt.Errorf("entry type not supported: %v", e.Metadata().FileMode())
 	}
 }
 

internal/proto_stream_reader.go

@@ -0,0 +1,66 @@
+package internal
+
+import (
+	"bufio"
+	"bytes"
+	"encoding/binary"
+	"fmt"
+	"io"
+
+	"github.com/golang/protobuf/proto"
+)
+
+// ProtoStreamReader reads a stream of length-prefix proto messages with an optional header.
+type ProtoStreamReader struct {
+	reader  *bufio.Reader
+	readBuf []byte
+	buf     *proto.Buffer
+	header  []byte
+}
+
+// Read reads the next proto message from the stream, returns io.EOF on the end of stream.
+func (r *ProtoStreamReader) Read(v proto.Message) error {
+	v.Reset()
+	if r.header != nil {
+		p := make([]byte, len(r.header))
+		if _, err := r.reader.Read(p); err == io.EOF {
+			return io.EOF
+		}
+
+		if !bytes.Equal(p, r.header) {
+			return fmt.Errorf("invalid stream header: %v, expected %v", string(p), string(r.header))
+		}
+
+		r.header = nil
+
+	}
+	length64, err := binary.ReadUvarint(r.reader)
+	if err != nil {
+		return err
+	}
+
+	length := int(length64)
+
+	if cap(r.readBuf) < length {
+		r.readBuf = make([]byte, length*12/10)
+	}
+
+	r.readBuf = r.readBuf[:length]
+	if _, err := io.ReadFull(r.reader, r.readBuf); err != nil {
+		return err
+	}
+
+	r.buf.SetBuf(r.readBuf)
+
+	return r.buf.Unmarshal(v)
+}
+
+// NewProtoStreamReader returns new ProtoStreamReader on top of a given buffered reader.
+// The provided header must match the beginning of a stream.
+func NewProtoStreamReader(r *bufio.Reader, header []byte) *ProtoStreamReader {
+	return &ProtoStreamReader{
+		reader: r,
+		header: header,
+		buf:    proto.NewBuffer(nil),
+	}
+}

internal/proto_stream_test.go

@@ -0,0 +1,74 @@
+package internal
+
+import (
+	"bufio"
+	"bytes"
+	"io"
+	"strings"
+	"testing"
+)
+
+var testHeader1 = []byte("01234567")
+var testHeader2 = []byte("0123456x")
+
+func TestProtoStream(t *testing.T) {
+	var buf bytes.Buffer
+
+	data := []TestProto{
+		TestProto{Name: "foo"},
+		TestProto{Name: "bar"},
+		TestProto{Name: "baz"},
+	}
+
+	w := NewProtoStreamWriter(&buf, testHeader1)
+	for _, d := range data {
+		if err := w.Write(&d); err != nil {
+			t.Errorf("write error: %v", err)
+		}
+	}
+	r := NewProtoStreamReader(bufio.NewReader(&buf), testHeader1)
+	var v TestProto
+	for _, d := range data {
+		if err := r.Read(&v); err != nil {
+			t.Errorf("read error: %v", err)
+		}
+		if v.Name != d.Name {
+			t.Errorf("invalid value: '%v', expected '%v'", v.Name, d.Name)
+		}
+	}
+	err := r.Read(&v)
+	if err != io.EOF {
+		t.Errorf("expected EOF, got %v", err)
+	}
+}
+
+func TestEmptyStream(t *testing.T) {
+	var buf bytes.Buffer
+
+	NewProtoStreamWriter(&buf, testHeader1)
+
+	r := NewProtoStreamReader(bufio.NewReader(&buf), testHeader2)
+	var v TestProto
+	err := r.Read(&v)
+	if err != io.EOF {
+		t.Errorf("expected EOF, got %v", err)
+	}
+}
+
+func TestInvalidHeader(t *testing.T) {
+	var buf bytes.Buffer
+
+	w := NewProtoStreamWriter(&buf, testHeader1)
+	if err := w.Write(&TestProto{Name: "foo"}); err != nil {
+		t.Errorf("write error: %v", err)
+	}
+
+	r := NewProtoStreamReader(bufio.NewReader(&buf), testHeader2)
+	var v TestProto
+	err := r.Read(&v)
+	if err == nil {
+		t.Errorf("expected error, got none")
+	} else if !strings.Contains(err.Error(), "invalid stream header") {
+		t.Errorf("got incorrect error: %v", err)
+	}
+}

internal/proto_stream_types.go

@@ -0,0 +1,12 @@
+package internal
+
+var (
+	// ProtoStreamTypeHashCache indicates proto stream whose elements are hash cache entries.
+	ProtoStreamTypeHashCache = []byte("kopiaHCH")
+
+	// ProtoStreamTypeDir indicates proto stream whose elements are directory entries.
+	ProtoStreamTypeDir = []byte("kopiaDIR")
+
+	// ProtoStreamTypeIndirect indicates proto stream whose elements are indirect block references.
+	ProtoStreamTypeIndirect = []byte("kopiaIND")
+)

internal/proto_stream_writer.go

@@ -0,0 +1,58 @@
+package internal
+
+import (
+	"encoding/binary"
+	"io"
+
+	"github.com/golang/protobuf/proto"
+)
+
+// ProtoStreamWriter writes a stream of length-prefix proto messages with an optional header.
+type ProtoStreamWriter struct {
+	writer          io.Writer
+	buf             *proto.Buffer
+	lengthPrefixBuf []byte
+	header          []byte
+}
+
+// Write emits length-prefixed proto message to its output.
+func (w *ProtoStreamWriter) Write(v proto.Message) error {
+	if w.header != nil {
+		if _, err := w.writer.Write(w.header); err != nil {
+			return err
+		}
+		w.header = nil
+	}
+
+	w.buf.Reset()
+	if err := w.buf.Marshal(v); err != nil {
+		return err
+	}
+
+	protoBytes := w.buf.Bytes()
+
+	if w.lengthPrefixBuf == nil {
+		w.lengthPrefixBuf = make([]byte, binary.MaxVarintLen64)
+	}
+
+	lengthBytes := w.lengthPrefixBuf[0:binary.PutUvarint(w.lengthPrefixBuf[:], uint64(len(protoBytes)))]
+
+	if _, err := w.writer.Write(lengthBytes); err != nil {
+		return err
+	}
+
+	if _, err := w.writer.Write(protoBytes); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+// NewProtoStreamWriter creates a new ProtoStreamWriter on top of a specified writer with a specified optional header.
+func NewProtoStreamWriter(w io.Writer, header []byte) *ProtoStreamWriter {
+	return &ProtoStreamWriter{
+		writer: w,
+		buf:    proto.NewBuffer(nil),
+		header: header,
+	}
+}

internal/proto_test.pb.go

@@ -0,0 +1,51 @@
+// Code generated by protoc-gen-go.
+// source: internal/proto_test.proto
+// DO NOT EDIT!
+
+/*
+Package internal is a generated protocol buffer package.
+
+It is generated from these files:
+	internal/proto_test.proto
+
+It has these top-level messages:
+	TestProto
+*/
+package internal
+
+import proto "github.com/golang/protobuf/proto"
+import fmt "fmt"
+import math "math"
+
+// Reference imports to suppress errors if they are not otherwise used.
+var _ = proto.Marshal
+var _ = fmt.Errorf
+var _ = math.Inf
+
+// This is a compile-time assertion to ensure that this generated file
+// is compatible with the proto package it is being compiled against.
+const _ = proto.ProtoPackageIsVersion1
+
+type TestProto struct {
+	Name string `protobuf:"bytes,1,opt,name=name" json:"name,omitempty"`
+}
+
+func (m *TestProto) Reset()                    { *m = TestProto{} }
+func (m *TestProto) String() string            { return proto.CompactTextString(m) }
+func (*TestProto) ProtoMessage()               {}
+func (*TestProto) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{0} }
+
+func init() {
+	proto.RegisterType((*TestProto)(nil), "kopia.fs.TestProto")
+}
+
+var fileDescriptor0 = []byte{
+	// 109 bytes of a gzipped FileDescriptorProto
+	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0xe2, 0x92, 0xcc, 0xcc, 0x2b, 0x49,
+	0x2d, 0xca, 0x4b, 0xcc, 0xd1, 0x2f, 0x28, 0xca, 0x2f, 0xc9, 0x8f, 0x2f, 0x49, 0x2d, 0x2e, 0xd1,
+	0x03, 0x33, 0x85, 0x38, 0xb2, 0xf3, 0x0b, 0x32, 0x13, 0xf5, 0xd2, 0x8a, 0x95, 0xe4, 0xb9, 0x38,
+	0x43, 0x80, 0xe2, 0x01, 0x60, 0x61, 0x21, 0x2e, 0x96, 0xbc, 0xc4, 0xdc, 0x54, 0x09, 0x46, 0x05,
+	0x46, 0x0d, 0xce, 0x20, 0x30, 0xdb, 0x49, 0x31, 0x4a, 0x3e, 0x3d, 0xb3, 0x24, 0xa3, 0x34, 0x49,
+	0x2f, 0x39, 0x3f, 0x57, 0x1f, 0xac, 0x0f, 0x4a, 0xc2, 0x8c, 0x4f, 0x62, 0x03, 0x1b, 0x6a, 0x0c,
+	0x08, 0x00, 0x00, 0xff, 0xff, 0x49, 0x42, 0xd9, 0xa7, 0x71, 0x00, 0x00, 0x00,
+}

internal/proto_test.proto

@@ -0,0 +1,8 @@
+syntax = "proto3";
+
+package kopia.fs;
+option go_package = "github.com/kopia/kopia/internal";
+
+message TestProto {
+    string name = 1;
+}

repo/format.go

@@ -5,9 +5,10 @@ import (
 	"crypto/cipher"
 	"crypto/hmac"
 	"crypto/md5"
-	"crypto/sha1"
 	"crypto/sha256"
 	"crypto/sha512"
+	"errors"
+	"fmt"
 	"hash"
 )
 
@@ -16,36 +17,83 @@ var (
 	hkdfInfoCryptoKey = []byte("CryptoKey")
 )
 
-// Format describes the format of object data.
-type Format struct {
-	Version           string `json:"version"`
-	ObjectFormat      string `json:"objectFormat"`
-	Secret            []byte `json:"secret,omitempty"`
-	MaxInlineBlobSize int    `json:"maxInlineBlobSize"`
-	MaxBlobSize       int    `json:"maxBlobSize"`
+// ObjectIDFormatInfo performs hashing and encryption of a data block.
+type ObjectIDFormatInfo interface {
+	Name() string
+	HashBuffer(data []byte, secret []byte) (blockID []byte, cryptoKey []byte)
+	CreateCipher(key []byte) (cipher.Block, error)
 }
 
-// ObjectIDFormat describes single format ObjectID
-type ObjectIDFormat struct {
-	Name        string
-	IsEncrypted bool
-
-	hashBuffer   func(data []byte, secret []byte) ([]byte, []byte)
-	createCipher func(key []byte) (cipher.Block, error)
+type unencryptedFormat struct {
+	name     string
+	hashCtor func() hash.Hash
+	fold     int
 }
 
-// ObjectIDFormats is a collection of ObjectIDFormat
-type ObjectIDFormats []*ObjectIDFormat
+func (fi *unencryptedFormat) Name() string {
+	return fi.name
+}
 
-// Find returns the ObjectIDFormat with a given name or nil if not found.
-func (fmts ObjectIDFormats) Find(name string) *ObjectIDFormat {
-	for _, f := range fmts {
-		if f.Name == name {
-			return f
+func (fi *unencryptedFormat) HashBuffer(data []byte, secret []byte) (blockID []byte, cryptoKey []byte) {
+	blockID = hashContent(fi.hashCtor, data, secret)
+	if fi.fold > 0 {
+		for i := fi.fold; i < len(blockID); i++ {
+			blockID[i%fi.fold] ^= blockID[i]
 		}
+		blockID = blockID[0:fi.fold]
+	}
+	return
+}
+
+func (fi *unencryptedFormat) CreateCipher(key []byte) (cipher.Block, error) {
+	return nil, errors.New("encryption not supported")
+
+}
+
+type encryptedFormat struct {
+	name         string
+	hashCtor     func() hash.Hash
+	createCipher func(key []byte) (cipher.Block, error)
+	keyBytes     int
+}
+
+func (fi *encryptedFormat) Name() string {
+	return fi.name
+}
+
+func (fi *encryptedFormat) HashBuffer(data []byte, secret []byte) (blockID []byte, cryptoKey []byte) {
+	h := hashContent(fi.hashCtor, data, secret)
+	p := len(h) - fi.keyBytes
+	blockID = h[0:p]
+	cryptoKey = h[p:]
+	return
+}
+
+func (fi *encryptedFormat) CreateCipher(key []byte) (cipher.Block, error) {
+	return fi.createCipher(key)
+
+}
+
+// SupportedFormats containes supported ObjectIDformats
+var SupportedFormats = map[ObjectIDFormat]ObjectIDFormatInfo{
+	ObjectIDFormat_TESTONLY_MD5:                     &unencryptedFormat{"TESTONLY_MD5", md5.New, 0},
+	ObjectIDFormat_UNENCYPTED_HMAC_SHA256:           &unencryptedFormat{"UNENCRYPTED_HMAC_SHA256", sha256.New, 0},
+	ObjectIDFormat_UNENCYPTED_HMAC_SHA256_128:       &unencryptedFormat{"UNENCRYPTED_HMAC_SHA256", sha256.New, 16},
+	ObjectIDFormat_ENCRYPTED_HMAC_SHA512_384_AES256: &encryptedFormat{"ENCRYPTED_HMAC_SHA512_384_AES256", sha512.New384, aes.NewCipher, 32},
+	ObjectIDFormat_ENCRYPTED_HMAC_SHA512_AES256:     &encryptedFormat{"ENCRYPTED_HMAC_SHA512_AES256", sha512.New, aes.NewCipher, 32},
+}
+
+// DefaultObjectFormat is the format that should be used by default when creating new repositories.
+var DefaultObjectFormat = SupportedFormats[ObjectIDFormat_ENCRYPTED_HMAC_SHA512_AES256]
+
+// ParseObjectIDFormat parses the given string and returns corresponding ObjectIDFormat.
+func ParseObjectIDFormat(s string) (ObjectIDFormat, error) {
+	v, ok := ObjectIDFormat_value[s]
+	if ok {
+		return ObjectIDFormat(v), nil
 	}
 
-	return nil
+	return ObjectIDFormat_INVALID, fmt.Errorf("unknown object ID format: %v", s)
 }
 
 func fold(b []byte, size int) []byte {
@@ -59,15 +107,6 @@ func fold(b []byte, size int) []byte {
 	return b[0:size]
 }
 
-func nonEncryptedFormat(name string, hf func() hash.Hash, hashSize int) *ObjectIDFormat {
-	return &ObjectIDFormat{
-		Name: name,
-		hashBuffer: func(data []byte, secret []byte) ([]byte, []byte) {
-			return fold(hashContent(hf, data, secret), hashSize), nil
-		},
-	}
-}
-
 func hashContent(hf func() hash.Hash, data []byte, secret []byte) []byte {
 	var h hash.Hash
 
@@ -79,55 +118,3 @@ func hashContent(hf func() hash.Hash, data []byte, secret []byte) []byte {
 	h.Write(data)
 	return h.Sum(nil)
 }
-
-func encryptedFormat(
-	name string,
-	hf func() hash.Hash,
-	blockIDSize int,
-	createCipher func(key []byte) (cipher.Block, error),
-	keySize int,
-) *ObjectIDFormat {
-	return &ObjectIDFormat{
-		Name: name,
-		hashBuffer: func(data []byte, secret []byte) ([]byte, []byte) {
-			contentHash := hashContent(sha512.New, data, secret)
-			blockID := fold(hashContent(hf, hkdfInfoBlockID, contentHash), blockIDSize)
-			cryptoKey := fold(hashContent(hf, hkdfInfoCryptoKey, contentHash), keySize)
-			return blockID, cryptoKey
-		},
-		createCipher: createCipher,
-	}
-}
-
-func buildObjectIDFormats() ObjectIDFormats {
-	var result ObjectIDFormats
-
-	for _, h := range []struct {
-		name     string
-		hash     func() hash.Hash
-		hashSize int
-	}{
-		{"md5", md5.New, md5.Size},
-		{"sha1", sha1.New, sha1.Size},
-		{"sha224", sha256.New224, sha256.Size224},
-		{"sha256", sha256.New, sha256.Size},
-		{"sha256-fold128", sha256.New, 16},
-		{"sha256-fold160", sha256.New, 20},
-		{"sha384", sha512.New384, sha512.Size384},
-		{"sha512-fold128", sha512.New, 16},
-		{"sha512-fold160", sha512.New, 20},
-		{"sha512-224", sha512.New512_224, sha512.Size224},
-		{"sha512-256", sha512.New512_256, sha512.Size256},
-		{"sha512", sha512.New, sha512.Size},
-	} {
-		result = append(result, nonEncryptedFormat(h.name, h.hash, h.hashSize))
-		result = append(result, encryptedFormat(h.name+"-aes128", h.hash, h.hashSize, aes.NewCipher, 16))
-		result = append(result, encryptedFormat(h.name+"-aes192", h.hash, h.hashSize, aes.NewCipher, 24))
-		result = append(result, encryptedFormat(h.name+"-aes256", h.hash, h.hashSize, aes.NewCipher, 32))
-	}
-
-	return result
-}
-
-// SupportedFormats contains supported repository formats.
-var SupportedFormats = buildObjectIDFormats()

repo/format.pb.go

@@ -0,0 +1,141 @@
+// Code generated by protoc-gen-go.
+// source: repo/format.proto
+// DO NOT EDIT!
+
+/*
+Package repo is a generated protocol buffer package.
+
+It is generated from these files:
+	repo/format.proto
+	repo/indirect.proto
+	repo/objectid.proto
+
+It has these top-level messages:
+	Format
+	IndirectObjectEntry
+	ObjectID
+*/
+package repo
+
+import proto "github.com/golang/protobuf/proto"
+import fmt "fmt"
+import math "math"
+
+// Reference imports to suppress errors if they are not otherwise used.
+var _ = proto.Marshal
+var _ = fmt.Errorf
+var _ = math.Inf
+
+// This is a compile-time assertion to ensure that this generated file
+// is compatible with the proto package it is being compiled against.
+const _ = proto.ProtoPackageIsVersion1
+
+type EncryptionAlgorithm int32
+
+const (
+	EncryptionAlgorithm_NONE EncryptionAlgorithm = 0
+	EncryptionAlgorithm_AES  EncryptionAlgorithm = 1
+)
+
+var EncryptionAlgorithm_name = map[int32]string{
+	0: "NONE",
+	1: "AES",
+}
+var EncryptionAlgorithm_value = map[string]int32{
+	"NONE": 0,
+	"AES":  1,
+}
+
+func (x EncryptionAlgorithm) String() string {
+	return proto.EnumName(EncryptionAlgorithm_name, int32(x))
+}
+func (EncryptionAlgorithm) EnumDescriptor() ([]byte, []int) { return fileDescriptor0, []int{0} }
+
+// ObjectIDFormat describes the format of repository objects.
+type ObjectIDFormat int32
+
+const (
+	// Format not specified.
+	ObjectIDFormat_INVALID ObjectIDFormat = 0
+	// MD5 (no encryption, MD5 hashing, only used for testing)
+	ObjectIDFormat_TESTONLY_MD5 ObjectIDFormat = 1
+	// HMAC-SHA256 hash of content used as object ID, no encryption.
+	ObjectIDFormat_UNENCYPTED_HMAC_SHA256 ObjectIDFormat = 2
+	// HMAC-SHA256 hash folded into 128 bits used as object ID, no encryption.
+	ObjectIDFormat_UNENCYPTED_HMAC_SHA256_128 ObjectIDFormat = 3
+	// HMAC-SHA512 hash of content, top part of the hash used as 256-bit object ID,
+	// bottom part used as 256-bit encryption key.
+	ObjectIDFormat_ENCRYPTED_HMAC_SHA512_AES256 ObjectIDFormat = 4
+	// HMAC-SHA512_384 hash over contents, top part of the hash used as 128-bit object ID,
+	// bottom part used as 256-bit encryption key.
+	ObjectIDFormat_ENCRYPTED_HMAC_SHA512_384_AES256 ObjectIDFormat = 5
+)
+
+var ObjectIDFormat_name = map[int32]string{
+	0: "INVALID",
+	1: "TESTONLY_MD5",
+	2: "UNENCYPTED_HMAC_SHA256",
+	3: "UNENCYPTED_HMAC_SHA256_128",
+	4: "ENCRYPTED_HMAC_SHA512_AES256",
+	5: "ENCRYPTED_HMAC_SHA512_384_AES256",
+}
+var ObjectIDFormat_value = map[string]int32{
+	"INVALID":                          0,
+	"TESTONLY_MD5":                     1,
+	"UNENCYPTED_HMAC_SHA256":           2,
+	"UNENCYPTED_HMAC_SHA256_128":       3,
+	"ENCRYPTED_HMAC_SHA512_AES256":     4,
+	"ENCRYPTED_HMAC_SHA512_384_AES256": 5,
+}
+
+func (x ObjectIDFormat) String() string {
+	return proto.EnumName(ObjectIDFormat_name, int32(x))
+}
+func (ObjectIDFormat) EnumDescriptor() ([]byte, []int) { return fileDescriptor0, []int{1} }
+
+// Format describes the format of object data.
+type Format struct {
+	Version           int32          `protobuf:"varint,1,opt,name=version" json:"version,omitempty"`
+	ObjectFormat      ObjectIDFormat `protobuf:"varint,2,opt,name=object_format,json=objectFormat,enum=kopia.repo.ObjectIDFormat" json:"object_format,omitempty"`
+	Secret            []byte         `protobuf:"bytes,3,opt,name=secret,proto3" json:"secret,omitempty"`
+	MaxInlineBlobSize int32          `protobuf:"varint,4,opt,name=max_inline_blob_size,json=maxInlineBlobSize" json:"max_inline_blob_size,omitempty"`
+	MaxBlobSize       int32          `protobuf:"varint,5,opt,name=max_blob_size,json=maxBlobSize" json:"max_blob_size,omitempty"`
+}
+
+func (m *Format) Reset()                    { *m = Format{} }
+func (m *Format) String() string            { return proto.CompactTextString(m) }
+func (*Format) ProtoMessage()               {}
+func (*Format) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{0} }
+
+func init() {
+	proto.RegisterType((*Format)(nil), "kopia.repo.Format")
+	proto.RegisterEnum("kopia.repo.EncryptionAlgorithm", EncryptionAlgorithm_name, EncryptionAlgorithm_value)
+	proto.RegisterEnum("kopia.repo.ObjectIDFormat", ObjectIDFormat_name, ObjectIDFormat_value)
+}
+
+var fileDescriptor0 = []byte{
+	// 357 bytes of a gzipped FileDescriptorProto
+	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0x6c, 0x91, 0xcb, 0x6a, 0xf2, 0x40,
+	0x14, 0xc7, 0xbf, 0x78, 0xfd, 0x38, 0x5e, 0x88, 0xd3, 0x22, 0xc1, 0x5e, 0x10, 0xe9, 0x42, 0x5c,
+	0x44, 0x8c, 0xb5, 0xb8, 0x2b, 0xd1, 0xa4, 0x18, 0xd0, 0x58, 0x12, 0x5b, 0xb0, 0x9b, 0xc1, 0x84,
+	0xd4, 0xa6, 0x35, 0x99, 0x30, 0xa6, 0xa5, 0xed, 0x6b, 0xf5, 0x45, 0xfa, 0x48, 0x9d, 0x8c, 0x8a,
+	0x08, 0x6e, 0x06, 0xce, 0xfc, 0x7e, 0xe7, 0x3f, 0x67, 0x38, 0x50, 0xa1, 0x5e, 0x44, 0xda, 0xcf,
+	0x84, 0x06, 0x8b, 0x58, 0x8e, 0x28, 0x89, 0x09, 0x82, 0x37, 0x12, 0xf9, 0x0b, 0x39, 0x01, 0x8d,
+	0x5f, 0x01, 0x72, 0x77, 0x1c, 0x22, 0x09, 0xf2, 0x1f, 0x1e, 0x5d, 0xfb, 0x24, 0x94, 0x84, 0xba,
+	0xd0, 0xcc, 0x5a, 0xbb, 0x12, 0xdd, 0x42, 0x89, 0x38, 0xaf, 0x9e, 0x1b, 0xe3, 0x4d, 0x8e, 0x94,
+	0x62, 0xbc, 0xac, 0xd4, 0xe4, 0x7d, 0x90, 0x3c, 0xe5, 0x82, 0xa1, 0x6d, 0xc2, 0xac, 0xe2, 0xa6,
+	0x61, 0x1b, 0x5d, 0x85, 0xdc, 0xda, 0x73, 0xa9, 0x17, 0x4b, 0x69, 0xd6, 0x59, 0xb4, 0xb6, 0x15,
+	0x6a, 0xc3, 0x69, 0xb0, 0xf8, 0xc4, 0x7e, 0xb8, 0xf2, 0x43, 0x0f, 0x3b, 0x2b, 0xe2, 0xe0, 0xb5,
+	0xff, 0xed, 0x49, 0x19, 0xfe, 0x7e, 0x85, 0x31, 0x83, 0xa3, 0x01, 0x23, 0x36, 0x03, 0xa8, 0x01,
+	0xa5, 0xa4, 0x61, 0x6f, 0x66, 0xb9, 0x59, 0x60, 0x97, 0x3b, 0xa7, 0xd5, 0x84, 0x13, 0x3d, 0x74,
+	0xe9, 0x57, 0x14, 0xb3, 0xd9, 0xd5, 0xd5, 0x92, 0x50, 0x3f, 0x7e, 0x09, 0xd0, 0x7f, 0xc8, 0x98,
+	0x53, 0x53, 0x17, 0xff, 0xa1, 0x3c, 0xa4, 0x55, 0xdd, 0x16, 0x85, 0xd6, 0x8f, 0x00, 0xe5, 0xc3,
+	0xb9, 0x51, 0x01, 0xf2, 0x86, 0xf9, 0xa8, 0x8e, 0x0d, 0x8d, 0x89, 0x22, 0x14, 0x67, 0xba, 0x3d,
+	0x9b, 0x9a, 0xe3, 0x39, 0x9e, 0x68, 0x3d, 0x51, 0x40, 0x35, 0xa8, 0x3e, 0x98, 0xba, 0x39, 0x9c,
+	0xdf, 0xcf, 0x74, 0x0d, 0x8f, 0x26, 0xea, 0x10, 0xdb, 0x23, 0x55, 0xe9, 0xdd, 0x88, 0x29, 0x74,
+	0x09, 0xb5, 0xe3, 0x0c, 0x77, 0x94, 0xbe, 0x98, 0x46, 0x75, 0x38, 0x67, 0xd4, 0x3a, 0xc4, 0xbd,
+	0x8e, 0x82, 0xd9, 0x30, 0x49, 0x42, 0x06, 0x5d, 0x41, 0xfd, 0xb8, 0xd1, 0xed, 0x5f, 0xef, 0xac,
+	0xec, 0xe0, 0xe2, 0xe9, 0x6c, 0xc9, 0x7e, 0xf4, 0xee, 0xc8, 0x2e, 0x09, 0xda, 0x7c, 0x05, 0xdb,
+	0x33, 0x59, 0x84, 0x93, 0xe3, 0x4b, 0xee, 0xfe, 0x05, 0x00, 0x00, 0xff, 0xff, 0x3b, 0xcf, 0x39,
+	0xd2, 0xf9, 0x01, 0x00, 0x00,
+}

repo/format.proto

@@ -0,0 +1,41 @@
+syntax = "proto3";
+
+package kopia.repo;
+option go_package = "github.com/kopia/kopia/repo";
+
+enum EncryptionAlgorithm {
+    NONE = 0;
+    AES = 1;
+}
+
+// ObjectIDFormat describes the format of repository objects.
+enum ObjectIDFormat {
+    // Format not specified.
+    INVALID = 0;
+
+    // MD5 (no encryption, MD5 hashing, only used for testing)
+    TESTONLY_MD5 = 1;
+
+    // HMAC-SHA256 hash of content used as object ID, no encryption.
+    UNENCYPTED_HMAC_SHA256 = 2;
+
+    // HMAC-SHA256 hash folded into 128 bits used as object ID, no encryption.
+    UNENCYPTED_HMAC_SHA256_128 = 3;
+
+    // HMAC-SHA512 hash of content, top part of the hash used as 256-bit object ID,
+    // bottom part used as 256-bit encryption key.
+    ENCRYPTED_HMAC_SHA512_AES256 = 4;
+
+    // HMAC-SHA512_384 hash over contents, top part of the hash used as 128-bit object ID,
+    // bottom part used as 256-bit encryption key.
+    ENCRYPTED_HMAC_SHA512_384_AES256 = 5;
+}
+
+// Format describes the format of object data.
+message Format {
+    int32 version = 1;
+    ObjectIDFormat object_format = 2;
+    bytes secret = 3;
+    int32 max_inline_blob_size = 4;
+    int32 max_blob_size = 5;
+}

repo/indirect.pb.go

@@ -0,0 +1,52 @@
+// Code generated by protoc-gen-go.
+// source: repo/indirect.proto
+// DO NOT EDIT!
+
+package repo
+
+import proto "github.com/golang/protobuf/proto"
+import fmt "fmt"
+import math "math"
+
+// Reference imports to suppress errors if they are not otherwise used.
+var _ = proto.Marshal
+var _ = fmt.Errorf
+var _ = math.Inf
+
+// IndirectObjectEntry represents an entry in indirect object proto stream.
+type IndirectObjectEntry struct {
+	Start  int64     `protobuf:"varint,1,opt,name=start" json:"start,omitempty"`
+	Length int64     `protobuf:"varint,2,opt,name=length" json:"length,omitempty"`
+	Object *ObjectID `protobuf:"bytes,3,opt,name=object" json:"object,omitempty"`
+}
+
+func (m *IndirectObjectEntry) Reset()                    { *m = IndirectObjectEntry{} }
+func (m *IndirectObjectEntry) String() string            { return proto.CompactTextString(m) }
+func (*IndirectObjectEntry) ProtoMessage()               {}
+func (*IndirectObjectEntry) Descriptor() ([]byte, []int) { return fileDescriptor1, []int{0} }
+
+func (m *IndirectObjectEntry) GetObject() *ObjectID {
+	if m != nil {
+		return m.Object
+	}
+	return nil
+}
+
+func init() {
+	proto.RegisterType((*IndirectObjectEntry)(nil), "kopia.repo.IndirectObjectEntry")
+}
+
+var fileDescriptor1 = []byte{
+	// 165 bytes of a gzipped FileDescriptorProto
+	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0xe2, 0x12, 0x2e, 0x4a, 0x2d, 0xc8,
+	0xd7, 0xcf, 0xcc, 0x4b, 0xc9, 0x2c, 0x4a, 0x4d, 0x2e, 0xd1, 0x2b, 0x28, 0xca, 0x2f, 0xc9, 0x17,
+	0xe2, 0xca, 0xce, 0x2f, 0xc8, 0x4c, 0xd4, 0x03, 0x49, 0x49, 0x41, 0x14, 0xe4, 0x27, 0x65, 0x01,
+	0xa5, 0x33, 0x53, 0x20, 0x0a, 0x94, 0x0a, 0xb9, 0x84, 0x3d, 0xa1, 0x5a, 0xfc, 0xc1, 0x32, 0xae,
+	0x79, 0x25, 0x45, 0x95, 0x42, 0x22, 0x5c, 0xac, 0xc5, 0x25, 0x89, 0x45, 0x25, 0x12, 0x8c, 0x0a,
+	0x8c, 0x1a, 0xcc, 0x41, 0x10, 0x8e, 0x90, 0x18, 0x17, 0x5b, 0x4e, 0x6a, 0x5e, 0x7a, 0x49, 0x86,
+	0x04, 0x13, 0x58, 0x18, 0xca, 0x13, 0xd2, 0xe1, 0x62, 0x83, 0x18, 0x2b, 0xc1, 0x0c, 0x14, 0xe7,
+	0x36, 0x12, 0xd1, 0x43, 0x58, 0xab, 0x07, 0x31, 0xd6, 0xd3, 0x25, 0x08, 0xaa, 0xc6, 0x49, 0x36,
+	0x4a, 0x3a, 0x3d, 0xb3, 0x24, 0xa3, 0x34, 0x49, 0x2f, 0x39, 0x3f, 0x57, 0x1f, 0xac, 0x12, 0x4a,
+	0x82, 0xd4, 0x27, 0xb1, 0x81, 0x1d, 0x66, 0x0c, 0x08, 0x00, 0x00, 0xff, 0xff, 0x0c, 0xb4, 0xa1,
+	0x12, 0xd0, 0x00, 0x00, 0x00,
+}

repo/indirect.proto

@@ -0,0 +1,13 @@
+syntax = "proto3";
+
+package kopia.repo;
+option go_package = "github.com/kopia/kopia/repo";
+
+import "repo/objectid.proto";
+
+// IndirectObjectEntry represents an entry in indirect object proto stream.
+message IndirectObjectEntry {
+    int64 start = 1;
+    int64 length = 2;
+    repo.ObjectID object = 3;
+}

repo/object_reader.go

@@ -3,32 +3,16 @@ package repo
 import (
 	"fmt"
 	"io"
-
-	"github.com/kopia/kopia/storage"
 )
 
-type seekTableEntry struct {
-	startOffset int64
-	length      int64
-	blockID     string
-}
-
-func (r *seekTableEntry) endOffset() int64 {
-	return r.startOffset + int64(r.length)
-}
-
-func (r *seekTableEntry) String() string {
-	return fmt.Sprintf("start: %d len: %d end: %d block: %s",
-		r.startOffset,
-		r.length,
-		r.endOffset(),
-		r.blockID)
+func (i *IndirectObjectEntry) endOffset() int64 {
+	return i.Start + i.Length
 }
 
 type objectReader struct {
-	storage storage.Storage
+	repo Repository
 
-	seekTable []seekTableEntry
+	seekTable []IndirectObjectEntry
 
 	currentPosition int64 // Overall position in the objectReader
 	totalLength     int64 // Overall length
@@ -80,13 +64,19 @@ func (r *objectReader) Read(buffer []byte) (int, error) {
 }
 
 func (r *objectReader) openCurrentChunk() error {
-	blockID := r.seekTable[r.currentChunkIndex].blockID
-	blockData, err := r.storage.GetBlock(blockID)
+	st := r.seekTable[r.currentChunkIndex]
+	blockData, err := r.repo.Open(*st.Object)
 	if err != nil {
 		return err
 	}
+	defer blockData.Close()
 
-	r.currentChunkData = blockData
+	b := make([]byte, st.Length)
+	if _, err := io.ReadFull(blockData, b); err != nil {
+		return err
+	}
+
+	r.currentChunkData = b
 	r.currentChunkPosition = 0
 	return nil
 }
@@ -101,7 +91,7 @@ func (r *objectReader) findChunkIndexForOffset(offset int64) int {
 	for left <= right {
 		middle := (left + right) / 2
 
-		if offset < r.seekTable[middle].startOffset {
+		if offset < r.seekTable[middle].Start {
 			right = middle - 1
 			continue
 		}
@@ -136,7 +126,7 @@ func (r *objectReader) Seek(offset int64, whence int) (int64, error) {
 
 	index := r.findChunkIndexForOffset(offset)
 
-	chunkStartOffset := r.seekTable[index].startOffset
+	chunkStartOffset := r.seekTable[index].Start
 
 	if index != r.currentChunkIndex {
 		r.closeCurrentChunk()

repo/object_writer.go

@@ -5,6 +5,7 @@ import (
 	"fmt"
 	"io"
 
+	"github.com/kopia/kopia/internal"
 	"github.com/kopia/kopia/storage"
 )
 
@@ -44,15 +45,15 @@ type objectWriter struct {
 
 	prefix             string
 	listWriter         *objectWriter
+	listProtoWriter    *internal.ProtoStreamWriter
+	listCurrentPos     int64
 	flushedObjectCount int
 	lastFlushedObject  ObjectID
 
-	description string
-	objectType  ObjectIDType
+	description   string
+	indirectLevel int32
 
 	blockTracker *blockTracker
-
-	atomicWrites bool
 }
 
 func (w *objectWriter) Close() error {
@@ -68,9 +69,13 @@ func (w *objectWriter) Close() error {
 	return nil
 }
 
-func (w *objectWriter) Write(data []byte) (n int, err error) {
-	dataLen := len(data)
-	remaining := len(data)
+func (w *objectWriter) WriteGather(dataSlices [][]byte) (n int, err error) {
+	dataLen := 0
+	for _, s := range dataSlices {
+		dataLen += len(s)
+	}
+
+	remaining := dataLen
 	w.totalLength += int64(remaining)
 
 	for remaining > 0 {
@@ -80,33 +85,45 @@ func (w *objectWriter) Write(data []byte) (n int, err error) {
 		room := w.buffer.Cap() - w.buffer.Len()
 
 		if remaining <= room {
-			w.buffer.Write(data)
+			for _, s := range dataSlices {
+				w.buffer.Write(s)
+			}
+			dataSlices = nil
 			remaining = 0
 		} else {
-			if w.atomicWrites {
-				if w.buffer == nil {
-					// We're at the beginning of a buffer, fail if the buffer is too small.
-					return 0, fmt.Errorf("object writer buffer too small, need: %v, have: %v", remaining, room)
-				}
-				if err := w.flushBuffer(false); err != nil {
-					return 0, err
-				}
+			for room > 0 {
+				var chunk []byte
 
-				continue
+				if len(dataSlices[0]) >= room {
+					chunk = dataSlices[0][0:room]
+					dataSlices[0] = dataSlices[0][len(chunk):]
+				} else {
+					chunk = dataSlices[0]
+					dataSlices[0] = nil
+				}
+				w.buffer.Write(chunk)
+				remaining -= len(chunk)
+				room -= len(chunk)
+				if len(dataSlices[0]) == 0 {
+					dataSlices = dataSlices[1:]
+				}
 			}
 
-			w.buffer.Write(data[0:room])
-
 			if err := w.flushBuffer(false); err != nil {
 				return 0, err
 			}
-			data = data[room:]
-			remaining = len(data)
 		}
 	}
 	return dataLen, nil
 }
 
+func (w *objectWriter) Write(data []byte) (n int, err error) {
+	var b [1][]byte
+
+	b[0] = data
+	return w.WriteGather(b[:])
+}
+
 func (w *objectWriter) flushBuffer(force bool) error {
 	// log.Printf("flushing bufer")
 	// defer log.Printf("flushed")
@@ -118,36 +135,43 @@ func (w *objectWriter) flushBuffer(force bool) error {
 
 		b := w.buffer
 		w.buffer = nil
-		objectID, blockReader, err := w.repo.hashBufferForWriting(b, string(w.objectType)+w.prefix)
+		objectID, blockReader, err := w.repo.hashBufferForWriting(b, w.prefix)
 		if err != nil {
 			return err
 		}
 
-		if err := w.repo.storage.PutBlock(objectID.BlockID(), blockReader, storage.PutOptionsDefault); err != nil {
+		if err := w.repo.storage.PutBlock(objectID.StorageBlock, blockReader, storage.PutOptionsDefault); err != nil {
 			return fmt.Errorf(
 				"error when flushing chunk %d of %s to %#v: %#v",
 				w.flushedObjectCount,
 				w.description,
-				objectID.BlockID(),
+				objectID.StorageBlock,
 				err)
 		}
 
-		w.blockTracker.addBlock(objectID.BlockID())
+		w.blockTracker.addBlock(objectID.StorageBlock)
 
 		w.flushedObjectCount++
 		w.lastFlushedObject = objectID
 		if w.listWriter == nil {
 			w.listWriter = &objectWriter{
-				repo:         w.repo,
-				objectType:   ObjectIDTypeList,
-				prefix:       w.prefix,
-				description:  "LIST(" + w.description + ")",
-				atomicWrites: true,
-				blockTracker: w.blockTracker,
+				repo:          w.repo,
+				indirectLevel: w.indirectLevel + 1,
+				prefix:        w.prefix,
+				description:   "LIST(" + w.description + ")",
+				blockTracker:  w.blockTracker,
 			}
+			w.listProtoWriter = internal.NewProtoStreamWriter(w.listWriter, internal.ProtoStreamTypeIndirect)
+			w.listCurrentPos = 0
 		}
 
-		fmt.Fprintf(w.listWriter, "%v,%v\n", length, objectID)
+		w.listProtoWriter.Write(&IndirectObjectEntry{
+			Object: &objectID,
+			Start:  w.listCurrentPos,
+			Length: int64(length),
+		})
+
+		w.listCurrentPos += int64(length)
 	}
 	return nil
 }
@@ -155,10 +179,10 @@ func (w *objectWriter) flushBuffer(force bool) error {
 func (w *objectWriter) Result(forceStored bool) (ObjectID, error) {
 	if !forceStored && w.flushedObjectCount == 0 {
 		if w.buffer == nil {
-			return "B", nil
+			return NullObjectID, nil
 		}
 
-		if w.buffer.Len() < w.repo.format.MaxInlineBlobSize {
+		if w.buffer.Len() < int(w.repo.format.MaxInlineBlobSize) {
 			return NewInlineObjectID(w.buffer.Bytes()), nil
 		}
 	}
@@ -171,9 +195,10 @@ func (w *objectWriter) Result(forceStored bool) (ObjectID, error) {
 	}()
 
 	if w.flushedObjectCount == 1 {
+		w.lastFlushedObject.Indirect = w.indirectLevel
 		return w.lastFlushedObject, nil
 	} else if w.flushedObjectCount == 0 {
-		return "", nil
+		return NullObjectID, nil
 	} else {
 		return w.listWriter.Result(true)
 	}

repo/objectid.go

@@ -4,87 +4,72 @@ import (
 	"encoding/base64"
 	"encoding/hex"
 	"errors"
-	"fmt"
 	"strconv"
 	"strings"
-	"unicode/utf8"
+
+	"github.com/golang/protobuf/proto"
+
+	"fmt"
 )
 
+// NullObjectID is the identifier of an null object.
+var NullObjectID ObjectID
+
+const objectIDEncryptionInfoSeparator = "."
+
 var (
-	binaryEncoding = base64.RawURLEncoding
+	inlineContentEncoding = base64.RawURLEncoding
 )
 
-const (
-	objectIDEncryptionInfoSeparator = "."
-)
+// UIString returns the name of the repository object that is suitable for displaying in the UI.
+// Note that the object ID name may contain its encryption key, which is sensitive.
+func (m *ObjectID) UIString() string {
+	if m.StorageBlock != "" {
+		var encryptionSuffix string
 
-// ObjectIDType describes the type of the chunk.
-type ObjectIDType string
-
-// ObjectEncryptionInfo represents encryption info associated with ObjectID.
-type ObjectEncryptionInfo string
-
-// NoEncryption indicates that the object is not encrypted.
-var NoEncryption = ObjectEncryptionInfo("")
-
-const (
-	// ObjectIDTypeText represents text-only inline object ID
-	ObjectIDTypeText ObjectIDType = "T"
-
-	// ObjectIDTypeBinary represents binary inline object ID
-	ObjectIDTypeBinary ObjectIDType = "B"
-
-	// ObjectIDTypeStored represents ID of object whose data is stored directly in a single storage block indicated by string.
-	ObjectIDTypeStored ObjectIDType = "D"
-
-	// ObjectIDTypeList represents ID of an object whose data is stored in mutliple storage blocks.
-	// The value of the ObjectID is the list chunk, which lists object IDs that need to be concatenated
-	// to form the contents.
-	ObjectIDTypeList ObjectIDType = "L"
-
-	// ObjectIDTypeSection represents ID of an object whose data is a section of another object.
-	// The format is S{offset},{length},{base}
-	ObjectIDTypeSection ObjectIDType = "S"
-)
-
-// IsStored determines whether data for the given chunk type is stored in the storage
-// (as opposed to being stored inline as part of ObjectID itself).
-func (ct ObjectIDType) IsStored() bool {
-	switch ct {
-	case ObjectIDTypeStored, ObjectIDTypeList:
-		return true
-
-	default:
-		return false
-	}
-}
-
-// ObjectID represents the identifier of a chunk.
-// Identifiers can either refer to data block stored in a Storage, or contain small amounts
-// of payload directly (useful for short ASCII or binary files).
-type ObjectID string
-
-// Type gets the type of the object ID.
-func (c ObjectID) Type() ObjectIDType {
-	return ObjectIDType(c[0:1])
-}
-
-// InlineData returns inline data stored as part of ObjectID. For chunk IDs representing stored
-// chunks, the value is nil.
-func (c ObjectID) InlineData() []byte {
-	payload := string(c[1:])
-	switch c.Type() {
-	case ObjectIDTypeText:
-		return []byte(payload)
-
-	case ObjectIDTypeBinary:
-		decodedPayload, err := binaryEncoding.DecodeString(payload)
-		if err == nil {
-			return decodedPayload
+		if m.Encryption != nil {
+			encryptionSuffix = "." + hex.EncodeToString(m.Encryption)
 		}
+
+		if m.Indirect > 0 {
+			return fmt.Sprintf("L%v,%v%v", m.Indirect, m.StorageBlock, encryptionSuffix)
+		}
+
+		return "D" + m.StorageBlock + encryptionSuffix
 	}
 
-	return nil
+	if m.Content != nil {
+		return "B" + inlineContentEncoding.EncodeToString(m.Content)
+	}
+
+	if m.Section != nil {
+		return fmt.Sprintf("S%v,%v,%v", m.Section.Start, m.Section.Length, m.Section.Base.UIString())
+	}
+
+	return "B"
+}
+
+// Equals determines whether ObjectID is equal to another ObjectID.
+func (m *ObjectID) Equals(other ObjectID) bool {
+	return proto.Equal(m, &other)
+}
+
+// NewInlineObjectID returns new ObjectID containing inline binary or text-encoded data.
+func NewInlineObjectID(data []byte) ObjectID {
+	return ObjectID{
+		Content: data,
+	}
+}
+
+// NewSectionObjectID returns new ObjectID representing a section of an object with a given base ID, start offset and length.
+func NewSectionObjectID(start, length int64, baseID *ObjectID) ObjectID {
+	return ObjectID{
+		Section: &ObjectID_Section{
+			Base:   baseID,
+			Start:  start,
+			Length: length,
+		},
+	}
 }
 
 // parseNumberUntilComma parses a string of the form "{x},{remainder}" where x is a 64-bit number and remainder is arbitrary string.
@@ -104,134 +89,82 @@ func parseNumberUntilComma(s string) (int64, string, error) {
 }
 
 func parseSectionInfoString(s string) (int64, int64, ObjectID, error) {
-	if ObjectIDType(s[0]) != ObjectIDTypeSection {
-		return 0, -1, "", errors.New("not a section object")
-	}
-
 	var start, length int64
 	var err error
 
 	start, s, err = parseNumberUntilComma(s[1:])
 	if err != nil {
-		return 0, -1, "", err
+		return 0, -1, NullObjectID, err
 	}
 
 	length, s, err = parseNumberUntilComma(s)
 	if err != nil {
-		return 0, -1, "", err
+		return 0, -1, NullObjectID, err
 	}
 
 	oid, err := ParseObjectID(s)
 	if err != nil {
-		return 0, -1, "", err
+		return 0, -1, NullObjectID, err
 	}
 
 	return start, length, oid, nil
 }
 
-// SectionInfo returns start, length and the base ID of a section object.
-func (c ObjectID) SectionInfo() (start int64, length int64, baseID ObjectID) {
-	if c.Type() != ObjectIDTypeSection {
-		return 0, 0, ""
-	}
-
-	start, length, oid, err := parseSectionInfoString(string(c))
-	if err != nil {
-		// This should not happen if we came in through ParseObjectID
-		panic("invalid section info: " + string(c))
-	}
-
-	return start, length, oid
-}
-
-// BlockID returns identifier of the storage block. For inline chunk IDs, an empty string is returned.
-func (c ObjectID) BlockID() string {
-	if c.Type().IsStored() {
-		content := string(c[1:])
-		firstSeparator := strings.Index(content, objectIDEncryptionInfoSeparator)
-		if firstSeparator > 0 {
-			return string(content[0:firstSeparator])
-		}
-
-		return string(content)
-	}
-
-	return ""
-}
-
-// EncryptionInfo returns ObjectEncryptionInfo for the ObjectID.
-func (c ObjectID) EncryptionInfo() ObjectEncryptionInfo {
-	if c.Type().IsStored() {
-		content := string(c[1:])
-		firstSeparator := strings.Index(content, objectIDEncryptionInfoSeparator)
-		if firstSeparator > 0 {
-			return ObjectEncryptionInfo(content[firstSeparator+1:])
-		}
-	}
-
-	return NoEncryption
-}
-
-// NewInlineObjectID returns new ObjectID containing inline binary or text-encoded data.
-func NewInlineObjectID(data []byte) ObjectID {
-	if !utf8.Valid(data) {
-		return ObjectID("B" + binaryEncoding.EncodeToString(data))
-	}
-
-	for _, b := range data {
-		if b < 32 && (b != 9 && b != 10 && b != 13) {
-			// Any other character indicates binary content.
-			return ObjectID("B" + binaryEncoding.EncodeToString(data))
-		}
-	}
-
-	return ObjectID("T" + string(data))
-}
-
-// NewSectionObjectID returns new ObjectID representing a section of an object with a given base ID, start offset and length.
-func NewSectionObjectID(start, length int64, baseID ObjectID) ObjectID {
-	return ObjectID(fmt.Sprintf("S%v,%v,%v", start, length, baseID))
-}
-
 // ParseObjectID converts the specified string into ObjectID.
 func ParseObjectID(objectIDString string) (ObjectID, error) {
 	if len(objectIDString) >= 1 {
-		chunkType := ObjectIDType(objectIDString[0])
+		chunkType := objectIDString[0]
 		content := objectIDString[1:]
 
 		switch chunkType {
-		case ObjectIDTypeSection:
-			if _, _, _, err := parseSectionInfoString(objectIDString); err == nil {
-				return ObjectID(objectIDString), nil
+		case 'S':
+			if start, length, base, err := parseSectionInfoString(objectIDString); err == nil {
+				return ObjectID{Section: &ObjectID_Section{
+					Start:  start,
+					Length: length,
+					Base:   &base,
+				}}, nil
 			}
 
-		case ObjectIDTypeText:
-			return ObjectID(objectIDString), nil
-
-		case ObjectIDTypeBinary:
-			if _, err := binaryEncoding.DecodeString(content); err == nil {
-				return ObjectID(objectIDString), nil
+		case 'B':
+			if v, err := inlineContentEncoding.DecodeString(content); err == nil {
+				return ObjectID{Content: v}, nil
+			}
+
+		case 'I', 'D':
+			var indirectLevel int32
+			if chunkType == 'I' {
+				comma := strings.Index(content, ",")
+				if comma < 0 {
+					// malformed
+					break
+				}
+				i, err := strconv.Atoi(content[0:comma])
+				if err != nil {
+					break
+				}
+				indirectLevel = int32(i)
+				content = content[comma+1:]
 			}
 
-		case ObjectIDTypeList, ObjectIDTypeStored:
 			firstSeparator := strings.Index(content, objectIDEncryptionInfoSeparator)
 			lastSeparator := strings.LastIndex(content, objectIDEncryptionInfoSeparator)
 			if firstSeparator == lastSeparator {
 				if firstSeparator == -1 {
 					// Found zero Separators in the ID - no encryption info.
-					return ObjectID(objectIDString), nil
+					return ObjectID{StorageBlock: content, Indirect: indirectLevel}, nil
 				}
 
 				if firstSeparator > 0 {
 					b, err := hex.DecodeString(content[firstSeparator+1:])
-					if err == nil && len(b) > 0 && len(b)%2 == 0 {
+					if err == nil && len(b) > 0 {
 						// Valid chunk ID with encryption info.
-						return ObjectID(objectIDString), nil
+						return ObjectID{StorageBlock: content[0:firstSeparator], Encryption: b}, nil
 					}
 				}
 			}
 		}
 	}
 
-	return "", fmt.Errorf("malformed chunk id: '%s'", objectIDString)
+	return NullObjectID, fmt.Errorf("malformed object id: '%s'", objectIDString)
 }

repo/objectid.pb.go

@@ -0,0 +1,80 @@
+// Code generated by protoc-gen-go.
+// source: repo/objectid.proto
+// DO NOT EDIT!
+
+package repo
+
+import proto "github.com/golang/protobuf/proto"
+import fmt "fmt"
+import math "math"
+
+// Reference imports to suppress errors if they are not otherwise used.
+var _ = proto.Marshal
+var _ = fmt.Errorf
+var _ = math.Inf
+
+// ObjectID represents the identifier of a chunk.
+// Identifiers can either refer to data block stored in a Storage, or contain small amounts
+// of payload directly (useful for short ASCII or binary files).
+type ObjectID struct {
+	StorageBlock string            `protobuf:"bytes,1,opt,name=storage_block,json=storageBlock" json:"storage_block,omitempty"`
+	Indirect     int32             `protobuf:"varint,2,opt,name=indirect" json:"indirect,omitempty"`
+	Encryption   []byte            `protobuf:"bytes,3,opt,name=encryption,proto3" json:"encryption,omitempty"`
+	Content      []byte            `protobuf:"bytes,4,opt,name=content,proto3" json:"content,omitempty"`
+	Section      *ObjectID_Section `protobuf:"bytes,5,opt,name=section" json:"section,omitempty"`
+}
+
+func (m *ObjectID) Reset()                    { *m = ObjectID{} }
+func (m *ObjectID) String() string            { return proto.CompactTextString(m) }
+func (*ObjectID) ProtoMessage()               {}
+func (*ObjectID) Descriptor() ([]byte, []int) { return fileDescriptor2, []int{0} }
+
+func (m *ObjectID) GetSection() *ObjectID_Section {
+	if m != nil {
+		return m.Section
+	}
+	return nil
+}
+
+type ObjectID_Section struct {
+	Base   *ObjectID `protobuf:"bytes,1,opt,name=base" json:"base,omitempty"`
+	Start  int64     `protobuf:"varint,2,opt,name=start" json:"start,omitempty"`
+	Length int64     `protobuf:"varint,3,opt,name=length" json:"length,omitempty"`
+}
+
+func (m *ObjectID_Section) Reset()                    { *m = ObjectID_Section{} }
+func (m *ObjectID_Section) String() string            { return proto.CompactTextString(m) }
+func (*ObjectID_Section) ProtoMessage()               {}
+func (*ObjectID_Section) Descriptor() ([]byte, []int) { return fileDescriptor2, []int{0, 0} }
+
+func (m *ObjectID_Section) GetBase() *ObjectID {
+	if m != nil {
+		return m.Base
+	}
+	return nil
+}
+
+func init() {
+	proto.RegisterType((*ObjectID)(nil), "kopia.repo.ObjectID")
+	proto.RegisterType((*ObjectID_Section)(nil), "kopia.repo.ObjectID.Section")
+}
+
+var fileDescriptor2 = []byte{
+	// 256 bytes of a gzipped FileDescriptorProto
+	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0x6c, 0x90, 0xcf, 0x4a, 0x03, 0x31,
+	0x10, 0xc6, 0x69, 0xb7, 0xdb, 0xad, 0xd3, 0x7a, 0x89, 0x45, 0x42, 0xfd, 0x83, 0xe8, 0xa5, 0xa7,
+	0x14, 0x2a, 0xf8, 0x00, 0xc5, 0x8b, 0x27, 0x21, 0xde, 0xbc, 0x48, 0x92, 0x86, 0x6d, 0x6c, 0xcd,
+	0x2c, 0xd9, 0xf1, 0xe0, 0xdb, 0xf8, 0xa8, 0x66, 0xb3, 0xbb, 0xea, 0xa1, 0x97, 0xc0, 0xf7, 0xfd,
+	0x66, 0x86, 0x1f, 0x81, 0xb3, 0x60, 0x2b, 0x5c, 0xa1, 0x7e, 0xb7, 0x86, 0xdc, 0x56, 0x54, 0x01,
+	0x09, 0x19, 0xec, 0xb1, 0x72, 0x4a, 0x34, 0xe8, 0xf6, 0x7b, 0x08, 0x93, 0xe7, 0x84, 0x9f, 0x1e,
+	0xd9, 0x1d, 0x9c, 0xd6, 0x84, 0x41, 0x95, 0xf6, 0x4d, 0x1f, 0xd0, 0xec, 0xf9, 0xe0, 0x66, 0xb0,
+	0x3c, 0x91, 0xb3, 0xae, 0xdc, 0x34, 0x1d, 0x5b, 0xc0, 0xc4, 0xf9, 0xad, 0x0b, 0x71, 0x85, 0x0f,
+	0x23, 0xcf, 0xe5, 0x6f, 0x66, 0xd7, 0x00, 0xd6, 0x9b, 0xf0, 0x55, 0x91, 0x43, 0xcf, 0xb3, 0x48,
+	0x67, 0xf2, 0x5f, 0xc3, 0x38, 0x14, 0x06, 0x3d, 0x59, 0x4f, 0x7c, 0x94, 0x60, 0x1f, 0xd9, 0x03,
+	0x14, 0x75, 0xe3, 0x18, 0xd7, 0xf2, 0x48, 0xa6, 0xeb, 0x4b, 0xf1, 0x67, 0x29, 0x7a, 0x43, 0xf1,
+	0xd2, 0xce, 0xc8, 0x7e, 0x78, 0xa1, 0xa0, 0xe8, 0x3a, 0xb6, 0x84, 0x91, 0x56, 0xb5, 0x4d, 0xd2,
+	0xd3, 0xf5, 0xfc, 0xd8, 0xbe, 0x4c, 0x13, 0x6c, 0x0e, 0x79, 0x4d, 0x2a, 0xb4, 0xfe, 0x99, 0x6c,
+	0x03, 0x3b, 0x87, 0xf1, 0xc1, 0xfa, 0x92, 0x76, 0x49, 0x3c, 0x93, 0x5d, 0xda, 0x5c, 0xbd, 0x5e,
+	0x94, 0x8e, 0x76, 0x9f, 0x5a, 0x18, 0xfc, 0x58, 0xa5, 0xab, 0xdd, 0xdb, 0xdc, 0xd6, 0xe3, 0xf4,
+	0xa9, 0xf7, 0x3f, 0x01, 0x00, 0x00, 0xff, 0xff, 0x7b, 0x26, 0x25, 0xb7, 0x6b, 0x01, 0x00, 0x00,
+}

repo/objectid.proto

@@ -0,0 +1,21 @@
+syntax = "proto3";
+
+package kopia.repo;
+option go_package = "github.com/kopia/kopia/repo";
+
+// ObjectID represents the identifier of a chunk.
+// Identifiers can either refer to data block stored in a Storage, or contain small amounts
+// of payload directly (useful for short ASCII or binary files).
+message ObjectID {
+    string storage_block = 1;
+    int32 indirect = 2;   // level of indirection (0==block data, 1=list of blocks, 2=list of lists, ...)
+    bytes encryption = 3;
+    bytes content = 4;
+    message Section {
+        ObjectID base = 1;
+        int64 start = 2;
+        int64 length = 3;
+    }
+
+    Section section = 5;
+}

repo/objectid_test.go

@@ -1,6 +1,7 @@
 package repo
 
 import (
+	"bytes"
 	"strings"
 	"testing"
 )
@@ -17,19 +18,21 @@ func TestParseMalformedObjectID(t *testing.T) {
 		"Dab.",
 		"Dab.a",
 		"Dab.abc",
-		"Dab.00",
 		"Dab.011",
 		"L.",
 		"L.x",
 		"L.af",
 		"Lx.ag",
 		"Lab.",
+		"L1",
+		"L1,",
+		"L-1,X",
 		"Xsomething",
 	}
 
 	for _, c := range cases {
 		v, err := ParseObjectID(c)
-		if v != "" || err == nil || !strings.HasPrefix(err.Error(), "malformed chunk id") {
+		if !v.Equals(NullObjectID) || err == nil || !strings.HasPrefix(err.Error(), "malformed object id") {
 			t.Errorf("unexpected result for %v: v: %v err: %v", c, v, err)
 		}
 	}
@@ -38,24 +41,16 @@ func TestParseMalformedObjectID(t *testing.T) {
 func TestParseObjectIDEncryptionInfo(t *testing.T) {
 	cases := []struct {
 		objectID string
-		expected ObjectEncryptionInfo
+		expected []byte
 	}{
-		{"B", NoEncryption},
-		{"BAQIDBA", NoEncryption},
-		{"T", NoEncryption},
-		{"T:foo", NoEncryption},
-		{"Tfoo\nbar", NoEncryption},
-		{"Tfoo\nbar:baz", NoEncryption},
-		{"Dabcdef", NoEncryption},
-		{"Labcdef", NoEncryption},
-		{
-			"Dabcdef.00112233445566778899aabbccddeeff00112233445566778899aabbccddeeff",
-			ObjectEncryptionInfo("00112233445566778899aabbccddeeff00112233445566778899aabbccddeeff"),
-		},
-		{
-			"Labcdef.00112233445566778899aabbccddeeff00112233445566778899aabbccddeeff",
-			ObjectEncryptionInfo("00112233445566778899aabbccddeeff00112233445566778899aabbccddeeff"),
-		},
+		{"B", nil},
+		{"BAQIDBA", nil},
+		{"Dabcdef", nil},
+		{"I1,abcdef", nil},
+		{"I2,abcdef", nil},
+		{"Dabcdef.00112233", []byte{0x00, 0x11, 0x22, 0x33}},
+		{"I1,abcdef.0011223344", []byte{0x00, 0x11, 0x22, 0x33, 0x44}},
+		{"I2,abcdef.0011223344", []byte{0x00, 0x11, 0x22, 0x33, 0x44}},
 	}
 
 	for _, c := range cases {
@@ -65,9 +60,9 @@ func TestParseObjectIDEncryptionInfo(t *testing.T) {
 			continue
 		}
 
-		actual := objectID.EncryptionInfo()
-		if actual != c.expected {
-			t.Errorf("invalid encryption info for %v: %v, expected: %v", c.objectID, actual, c.expected)
+		actual := objectID.Encryption
+		if !bytes.Equal(actual, c.expected) {
+			t.Errorf("invalid encryption info for %v: %x, expected: %x", c.objectID, actual, c.expected)
 		}
 	}
 }

repo/repository.go

@@ -10,10 +10,13 @@ import (
 	"fmt"
 	"hash"
 	"io"
-	"strconv"
+	"io/ioutil"
+	"log"
 	"strings"
 	"sync/atomic"
 
+	"github.com/kopia/kopia/internal"
+
 	"github.com/kopia/kopia/storage"
 	"github.com/kopia/kopia/storage/logging"
 )
@@ -76,7 +79,7 @@ type repository struct {
 	stats         *RepositoryStats
 
 	format   Format
-	idFormat ObjectIDFormat
+	idFormat ObjectIDFormatInfo
 }
 
 func (repo *repository) Close() {
@@ -107,7 +110,6 @@ func (repo *repository) Storage() storage.Storage {
 func (repo *repository) NewWriter(options ...WriterOption) ObjectWriter {
 	result := &objectWriter{
 		repo:         repo,
-		objectType:   ObjectIDTypeStored,
 		blockTracker: &blockTracker{},
 	}
 
@@ -119,24 +121,30 @@ func (repo *repository) NewWriter(options ...WriterOption) ObjectWriter {
 }
 
 func (repo *repository) Open(objectID ObjectID) (ObjectReader, error) {
-	if start, length, baseID := objectID.SectionInfo(); baseID != "" {
-		baseReader, err := repo.Open(baseID)
+	// log.Printf("Repository::Open %v", objectID.String())
+	// defer log.Printf("finished Repository::Open() %v", objectID.String())
+
+	if objectID.Section != nil {
+		if objectID.Section.Base == nil {
+			return nil, fmt.Errorf("invalid section base object")
+		}
+
+		baseReader, err := repo.Open(*objectID.Section.Base)
 		if err != nil {
 			return nil, fmt.Errorf("cannot create base reader: %v", err)
 		}
 
-		return newObjectSectionReader(start, length, baseReader)
+		return newObjectSectionReader(objectID.Section.Start, objectID.Section.Length, baseReader)
 	}
 
-	r, err := repo.newRawReader(objectID)
-	if err != nil {
-		return nil, err
-	}
+	if objectID.Indirect > 0 {
+		r, err := repo.Open(removeIndirection(objectID))
+		if err != nil {
+			return nil, err
+		}
+		defer r.Close()
 
-	if objectID.Type() == ObjectIDTypeList {
-		seekTable := make([]seekTableEntry, 0, 100)
-
-		seekTable, err = repo.flattenListChunk(seekTable, objectID, r)
+		seekTable, err := repo.flattenListChunk(r)
 		if err != nil {
 			return nil, err
 		}
@@ -144,13 +152,32 @@ func (repo *repository) Open(objectID ObjectID) (ObjectReader, error) {
 		totalLength := seekTable[len(seekTable)-1].endOffset()
 
 		return &objectReader{
-			storage:     repo.storage,
+			repo:        repo,
 			seekTable:   seekTable,
 			totalLength: totalLength,
 		}, nil
 	}
 
-	return r, err
+	return repo.newRawReader(objectID)
+}
+
+func dumpObject(repo *repository, oid ObjectID) {
+	oid.Indirect = 0
+	log.Printf("  dumping %v", oid.String())
+	defer log.Printf("  end of %v", oid.String())
+	r, err := repo.Open(oid)
+	if err != nil {
+		log.Printf("failed to open %v: %v", oid, err)
+		return
+	}
+	defer r.Close()
+
+	b, err := ioutil.ReadAll(r)
+	if err != nil {
+		log.Printf("failed to read %v: %v", oid, err)
+		return
+	}
+	log.Printf("%x", b)
 }
 
 // RepositoryOption controls the behavior of Repository.
@@ -185,14 +212,11 @@ func NewRepository(
 	f *Format,
 	options ...RepositoryOption,
 ) (Repository, error) {
-	if f.Version != "1" {
-		return nil, fmt.Errorf("unsupported storage version: %v", f.Version)
-	}
 	if f.MaxBlobSize < 100 {
 		return nil, fmt.Errorf("MaxBlobSize is not set")
 	}
 
-	sf := SupportedFormats.Find(f.ObjectFormat)
+	sf := SupportedFormats[f.ObjectFormat]
 	if sf == nil {
 		return nil, fmt.Errorf("unknown object format: %v", f.ObjectFormat)
 	}
@@ -203,7 +227,7 @@ func NewRepository(
 		stats:   &RepositoryStats{},
 	}
 
-	repo.idFormat = *sf
+	repo.idFormat = sf
 
 	for _, o := range options {
 		if err := o(repo); err != nil {
@@ -212,13 +236,13 @@ func NewRepository(
 		}
 	}
 
-	repo.bufferManager = newBufferManager(repo.format.MaxBlobSize)
+	repo.bufferManager = newBufferManager(int(repo.format.MaxBlobSize))
 
 	return repo, nil
 }
 
 func (repo *repository) hashBuffer(data []byte) ([]byte, []byte) {
-	return repo.idFormat.hashBuffer(data, repo.format.Secret)
+	return repo.idFormat.HashBuffer(data, repo.format.Secret)
 }
 
 func (repo *repository) hashBufferForWriting(buffer *bytes.Buffer, prefix string) (ObjectID, storage.ReaderWithLength, error) {
@@ -232,17 +256,18 @@ func (repo *repository) hashBufferForWriting(buffer *bytes.Buffer, prefix string
 	contentHash, cryptoKey := repo.hashBuffer(data)
 	if cryptoKey != nil {
 		var err error
-		blockCipher, err = repo.idFormat.createCipher(cryptoKey)
+		blockCipher, err = repo.idFormat.CreateCipher(cryptoKey)
 		if err != nil {
-			return "", nil, err
+			return NullObjectID, nil, err
 		}
 	}
 
-	var objectID ObjectID
+	objectID := ObjectID{
+		StorageBlock: prefix + hex.EncodeToString(contentHash),
+	}
+
 	if len(cryptoKey) > 0 {
-		objectID = ObjectID(prefix + hex.EncodeToString(contentHash) + objectIDEncryptionInfoSeparator + hex.EncodeToString(cryptoKey))
-	} else {
-		objectID = ObjectID(prefix + hex.EncodeToString(contentHash))
+		objectID.Encryption = cryptoKey
 	}
 
 	atomic.AddInt32(&repo.stats.HashedBlocks, 1)
@@ -268,71 +293,45 @@ func (repo *repository) hashBufferForWriting(buffer *bytes.Buffer, prefix string
 	return objectID, blockReader, nil
 }
 
-func (repo *repository) flattenListChunk(
-	seekTable []seekTableEntry,
-	listObjectID ObjectID,
-	rawReader io.Reader) ([]seekTableEntry, error) {
+func (repo *repository) flattenListChunk(rawReader io.Reader) ([]IndirectObjectEntry, error) {
 
-	scanner := bufio.NewScanner(rawReader)
+	r := bufio.NewReader(rawReader)
+	pr := internal.NewProtoStreamReader(r, internal.ProtoStreamTypeIndirect)
+	var seekTable []IndirectObjectEntry
 
-	for scanner.Scan() {
-		c := scanner.Text()
-		comma := strings.Index(c, ",")
-		if comma <= 0 {
-			return nil, fmt.Errorf("unsupported entry '%v' in list '%s'", c, listObjectID)
+	for {
+		var oe IndirectObjectEntry
+
+		err := pr.Read(&oe)
+		if err == io.EOF {
+			break
 		}
 
-		length, err := strconv.ParseInt(c[0:comma], 10, 64)
-
-		objectID, err := ParseObjectID(c[comma+1:])
 		if err != nil {
-			return nil, fmt.Errorf("unsupported entry '%v' in list '%s': %#v", c, listObjectID, err)
+			log.Printf("Failed to read proto: %v", err)
+			return nil, err
 		}
 
-		switch objectID.Type() {
-		case ObjectIDTypeList:
-			subreader, err := repo.newRawReader(objectID)
-			if err != nil {
-				return nil, err
-			}
-
-			seekTable, err = repo.flattenListChunk(seekTable, objectID, subreader)
-			if err != nil {
-				return nil, err
-			}
-
-		case ObjectIDTypeStored:
-			var startOffset int64
-			if len(seekTable) > 0 {
-				startOffset = seekTable[len(seekTable)-1].endOffset()
-			} else {
-				startOffset = 0
-			}
-
-			seekTable = append(
-				seekTable,
-				seekTableEntry{
-					blockID:     objectID.BlockID(),
-					startOffset: startOffset,
-					length:      length,
-				})
-
-		default:
-			return nil, fmt.Errorf("unsupported entry '%v' in list '%v'", objectID, listObjectID)
-
-		}
+		seekTable = append(seekTable, oe)
 	}
 
 	return seekTable, nil
 }
 
+func removeIndirection(o ObjectID) ObjectID {
+	if o.Indirect <= 0 {
+		panic("removeIndirection() called on a direct object")
+	}
+	o.Indirect--
+	return o
+}
+
 func (repo *repository) newRawReader(objectID ObjectID) (ObjectReader, error) {
-	inline := objectID.InlineData()
-	if inline != nil {
-		return newObjectReaderWithData(inline), nil
+	if objectID.Content != nil {
+		return newObjectReaderWithData(objectID.Content), nil
 	}
 
-	blockID := objectID.BlockID()
+	blockID := objectID.StorageBlock
 	payload, err := repo.storage.GetBlock(blockID)
 	if err != nil {
 		return nil, err
@@ -341,19 +340,14 @@ func (repo *repository) newRawReader(objectID ObjectID) (ObjectReader, error) {
 	atomic.AddInt32(&repo.stats.BlocksReadFromStorage, 1)
 	atomic.AddInt64(&repo.stats.BytesReadFromStorage, int64(len(payload)))
 
-	if objectID.EncryptionInfo() == NoEncryption {
-		if err := repo.verifyChecksum(payload, objectID.BlockID()); err != nil {
+	if objectID.Encryption == nil {
+		if err := repo.verifyChecksum(payload, objectID.StorageBlock); err != nil {
 			return nil, err
 		}
 		return newObjectReaderWithData(payload), nil
 	}
 
-	cryptoKey, err := hex.DecodeString(string(objectID.EncryptionInfo()))
-	if err != nil {
-		return nil, errors.New("malformed encryption key")
-	}
-
-	blockCipher, err := repo.idFormat.createCipher(cryptoKey)
+	blockCipher, err := repo.idFormat.CreateCipher(objectID.Encryption)
 	if err != nil {
 		return nil, errors.New("cannot create cipher")
 	}
@@ -366,7 +360,7 @@ func (repo *repository) newRawReader(objectID ObjectID) (ObjectReader, error) {
 	// after decrypting the content. This serves as a checksum.
 	atomic.AddInt64(&repo.stats.DecryptedBytes, int64(len(payload)))
 
-	if err := repo.verifyChecksum(payload, objectID.BlockID()); err != nil {
+	if err := repo.verifyChecksum(payload, objectID.StorageBlock); err != nil {
 		return nil, err
 	}
 

repo/repository_test.go

@@ -1,28 +1,28 @@
 package repo
 
 import (
+	"bufio"
 	"bytes"
 	"crypto/md5"
 	cryptorand "crypto/rand"
 	"encoding/hex"
 	"fmt"
+	"io"
 	"io/ioutil"
 	"math/rand"
-	"reflect"
-	"sort"
-	"strings"
 	"testing"
 
+	"github.com/kopia/kopia/internal"
 	"github.com/kopia/kopia/storage"
 	"github.com/kopia/kopia/storage/storagetesting"
 )
 
 func testFormat() *Format {
 	return &Format{
-		Version:           "1",
+		Version:           1,
 		MaxBlobSize:       200,
 		MaxInlineBlobSize: 20,
-		ObjectFormat:      "md5",
+		ObjectFormat:      ObjectIDFormat_TESTONLY_MD5,
 	}
 }
 
@@ -56,16 +56,15 @@ func TestWriters(t *testing.T) {
 		data     []byte
 		objectID ObjectID
 	}{
-		{[]byte{}, "B"},
-		{[]byte("quick brown fox"), "Tquick brown fox"},
-		{[]byte{1, 2, 3, 4}, "BAQIDBA"},
-		{[]byte{10, 13, 9}, "T\n\r\t"},
-		{[]byte{0xc2, 0x28}, "Bwig"}, // invalid UTF-8, will be represented as binary
+		{[]byte{}, NullObjectID},
+		{[]byte("quick brown fox"), ObjectID{Content: []byte("quick brown fox")}},
+		{[]byte{1, 2, 3, 4}, ObjectID{Content: []byte{1, 2, 3, 4}}},
+		{[]byte{0xc2, 0x28}, ObjectID{Content: []byte{0xc2, 0x28}}},
 		{
 			[]byte("the quick brown fox jumps over the lazy dog"),
-			"DX77add1d5f41223d5582fca736a5cb335",
+			ObjectID{StorageBlock: "X77add1d5f41223d5582fca736a5cb335"},
 		},
-		{make([]byte, 100), "DX6d0bb00954ceb7fbee436bb55a8397a9"}, // 100 zero bytes
+		{make([]byte, 100), ObjectID{StorageBlock: "X6d0bb00954ceb7fbee436bb55a8397a9"}}, // 100 zero bytes
 	}
 
 	for _, c := range cases {
@@ -83,11 +82,11 @@ func TestWriters(t *testing.T) {
 			continue
 		}
 
-		if result != c.objectID {
-			t.Errorf("incorrect result for %v, expected: %v got: %v", c.data, c.objectID, result)
+		if !result.Equals(c.objectID) {
+			t.Errorf("incorrect result for %v, expected: %v got: %v", c.data, c.objectID.UIString(), result)
 		}
 
-		if !c.objectID.Type().IsStored() {
+		if c.objectID.StorageBlock == "" {
 			if len(data) != 0 {
 				t.Errorf("unexpected data written to the storage: %v", data)
 			}
@@ -110,120 +109,74 @@ func TestWriterCompleteChunkInTwoWrites(t *testing.T) {
 	writer.Write(bytes[0:50])
 	writer.Write(bytes[0:50])
 	result, err := writer.Result(false)
-	if string(result) != "DX6d0bb00954ceb7fbee436bb55a8397a9" {
+	if !result.Equals(ObjectID{StorageBlock: "X6d0bb00954ceb7fbee436bb55a8397a9"}) {
 		t.Errorf("unexpected result: %v err: %v", result, err)
 	}
 }
 
-func TestWriterListChunk(t *testing.T) {
-	data, repo := setupTest(t)
+func verifyIndirectBlock(t *testing.T, r Repository, oid ObjectID) {
+	for level := int32(0); level < oid.Indirect; level++ {
+		direct := oid
+		direct.Indirect = level
 
-	contentBytes := make([]byte, 250)
-	contentMd5Sum200 := getMd5Digest(contentBytes[0:200])  // hash of 200 zero bytes
-	contentMd5Sum50 := getMd5Digest(contentBytes[200:250]) // hash of 50 zero bytes
-	listChunkContent := []byte("200,D" + contentMd5Sum200 + "\n50,D" + contentMd5Sum50 + "\n")
-	listChunkObjectID := getMd5LObjectID(listChunkContent)
+		rd, err := r.Open(direct)
+		if err != nil {
+			t.Errorf("unable to open %v: %v", oid.String(), err)
+			return
+		}
+		defer rd.Close()
 
-	writer := repo.NewWriter()
-	writer.Write(contentBytes)
-	result, err := writer.Result(false)
-	if err != nil {
-		t.Errorf("error getting writer results: %v", err)
+		pr := internal.NewProtoStreamReader(bufio.NewReader(rd), internal.ProtoStreamTypeIndirect)
+		for {
+			var v IndirectObjectEntry
+			if err := pr.Read(&v); err != nil {
+				if err == io.EOF {
+					break
+				}
+				t.Errorf("err: %v", err)
+				break
+			}
+		}
 	}
-
-	if string(result) != listChunkObjectID {
-		t.Errorf("incorrect list chunk ID: %v, expected: %v", result, listChunkObjectID)
-	}
-
-	// We have 3 chunks - 200 zero bytes, 50 zero bytes and the list.
-	verifyStorageBlocks(t, writer.StorageBlocks(), data, map[string][]byte{
-		contentMd5Sum200:               contentBytes[0:200],
-		contentMd5Sum50:                contentBytes[0:50],
-		getMd5Digest(listChunkContent): listChunkContent,
-	})
 }
 
-func TestWriterListOfListsChunk(t *testing.T) {
-	data, repo := setupTest(t)
-
-	contentBytes := make([]byte, 1400)
-	chunk1Id := getMd5CObjectID(contentBytes[0:200]) // hash of 200 zero bytes
-
-	list1ChunkContent := []byte(strings.Repeat("200,"+chunk1Id+"\n", 5))
-	list1ObjectID := fmt.Sprintf("%v,%v", len(list1ChunkContent), getMd5LObjectID(list1ChunkContent))
-
-	list2ChunkContent := []byte(strings.Repeat("200,"+chunk1Id+"\n", 2))
-	list2ObjectID := fmt.Sprintf("%v,%v", len(list2ChunkContent), getMd5LObjectID(list2ChunkContent))
-
-	listOfListsChunkContent := []byte(list1ObjectID + "\n" + list2ObjectID + "\n")
-	listOfListsObjectID := getMd5LObjectID(listOfListsChunkContent)
-
-	writer := repo.NewWriter()
-	writer.Write(contentBytes)
-	result, err := writer.Result(false)
-	if string(result) != listOfListsObjectID || err != nil {
-		t.Errorf("unexpected result: %v expected: %v, err: %v", result, listOfListsObjectID, err)
+func TestIndirection(t *testing.T) {
+	cases := []struct {
+		dataLength          int
+		expectedBlockCount  int
+		expectedIndirection int32
+	}{
+		{dataLength: 200, expectedBlockCount: 1, expectedIndirection: 0},
+		{dataLength: 250, expectedBlockCount: 3, expectedIndirection: 1},
+		{dataLength: 1400, expectedBlockCount: 4, expectedIndirection: 2},
+		{dataLength: 2000, expectedBlockCount: 5, expectedIndirection: 2},
+		{dataLength: 3000, expectedBlockCount: 6, expectedIndirection: 2},
+		{dataLength: 4000, expectedBlockCount: 9, expectedIndirection: 3},
+		{dataLength: 10000, expectedBlockCount: 16, expectedIndirection: 3},
 	}
 
-	// We have 4 chunks - 200 zero bytes, 2 lists, and 1 list-of-lists.
-	verifyStorageBlocks(t, writer.StorageBlocks(), data, map[string][]byte{
-		getMd5Digest(contentBytes[0:200]):     contentBytes[0:200],
-		getMd5Digest(list1ChunkContent):       list1ChunkContent,
-		getMd5Digest(list2ChunkContent):       list2ChunkContent,
-		getMd5Digest(listOfListsChunkContent): listOfListsChunkContent,
-	})
-}
+	for _, c := range cases {
+		data, repo := setupTest(t)
 
-func TestWriterListOfListsOfListsChunk(t *testing.T) {
-	data, repo := setupTest(t)
+		contentBytes := make([]byte, c.dataLength)
 
-	writtenData := make([]byte, 10000)
-	chunk1Id := getMd5CObjectID(writtenData[0:200]) // hash of 200 zero bytes
+		writer := repo.NewWriter()
+		writer.Write(contentBytes)
+		result, err := writer.Result(false)
+		if err != nil {
+			t.Errorf("error getting writer results: %v", err)
+		}
 
-	// First level list chunk has 5 C[] chunk IDs, because that's how many IDs fit in one chunk.
-	// (that number 200 was chosen for a reason, to make testing easy)
-	//
-	//   200,Cfbaf48ec981a5eecdb57b929fdd426e8\n
-	//   200,Cfbaf48ec981a5eecdb57b929fdd426e8\n
-	//   200,Cfbaf48ec981a5eecdb57b929fdd426e8\n
-	//   200,Cfbaf48ec981a5eecdb57b929fdd426e8\n
-	//   200,Cfbaf48ec981a5eecdb57b929fdd426e8\n
-	list1ChunkContent := []byte(strings.Repeat("200,"+chunk1Id+"\n", 5))
-	list1ObjectID := fmt.Sprintf("%v,%v", len(list1ChunkContent), getMd5LObjectID(list1ChunkContent))
+		if result.Indirect != c.expectedIndirection {
+			t.Errorf("incorrect indirection level: %v, expected %v", result.Indirect, c.expectedIndirection)
+		}
 
-	// Second level lists L[] chunks from the first level. They have all the same content
-	// because all lists are identical.
-	//   190,L52760f658059fef754f5deabdd01df93\n
-	//   190,L52760f658059fef754f5deabdd01df93\n
-	//   190,L52760f658059fef754f5deabdd01df93\n
-	//   190,L52760f658059fef754f5deabdd01df93\n
-	//   190,L52760f658059fef754f5deabdd01df93\n
-	list2ChunkContent := []byte(strings.Repeat(string(list1ObjectID)+"\n", 5))
-	list2ObjectID := fmt.Sprintf("%v,%v", len(list2ChunkContent), getMd5LObjectID(list2ChunkContent))
+		if len(data) != c.expectedBlockCount {
+			t.Errorf("unexpected block count: %v, expected %v", len(data), c.expectedBlockCount)
+		}
 
-	// Now those lists are also identical and represent 5000 bytes each, so
-	// the top-level list-of-lists-of-lists will have 2 entries:
-	//
-	//   190,Lb99b28e34c87e4934b4cc5631bb38ee8\n
-	//   190,Lb99b28e34c87e4934b4cc5631bb38ee8\n
-	list3ChunkContent := []byte(strings.Repeat(string(list2ObjectID)+"\n", 2))
-	list3ObjectID := getMd5LObjectID(list3ChunkContent)
-
-	writer := repo.NewWriter()
-	writer.Write(writtenData)
-
-	result, err := writer.Result(false)
-	if string(result) != list3ObjectID {
-		t.Errorf("unexpected result: %v err: %v", result, err)
+		verifyIndirectBlock(t, repo, result)
 	}
-
-	// We have 4 data blocks representing 10000 bytes of zero. Not bad!
-	verifyStorageBlocks(t, writer.StorageBlocks(), data, map[string][]byte{
-		getMd5Digest(writtenData[0:200]): writtenData[0:200],
-		getMd5Digest(list1ChunkContent):  list1ChunkContent,
-		getMd5Digest(list2ChunkContent):  list2ChunkContent,
-		getMd5Digest(list3ChunkContent):  list3ChunkContent,
-	})
 }
 
 func TestHMAC(t *testing.T) {
@@ -231,7 +184,7 @@ func TestHMAC(t *testing.T) {
 	content := bytes.Repeat([]byte{0xcd}, 50)
 
 	s := testFormat()
-	s.ObjectFormat = "md5"
+	s.ObjectFormat = ObjectIDFormat_TESTONLY_MD5
 	s.Secret = []byte{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25}
 
 	repo, err := NewRepository(storagetesting.NewMapStorage(data), s)
@@ -241,7 +194,7 @@ func TestHMAC(t *testing.T) {
 	w := repo.NewWriter()
 	w.Write(content)
 	result, err := w.Result(false)
-	if string(result) != "D697eaf0aca3a3aea3a75164746ffaa79" {
+	if result.UIString() != "D697eaf0aca3a3aea3a75164746ffaa79" {
 		t.Errorf("unexpected result: %v err: %v", result, err)
 	}
 }
@@ -258,8 +211,6 @@ func TestReader(t *testing.T) {
 	}{
 		{"B", []byte{}},
 		{"BAQIDBA", []byte{1, 2, 3, 4}},
-		{"T", []byte{}},
-		{"Tfoo\nbar", []byte("foo\nbar")},
 		{"Da76999788386641a3ec798554f1fe7e6", storedPayload},
 		{"S0,2,BAQIDBA", []byte{1, 2}},
 		{"S1,3,BAQIDBA", []byte{2, 3, 4}},
@@ -353,12 +304,12 @@ func TestEndToEndReadAndSeek(t *testing.T) {
 			verify(t, repo, objectID, randomData, fmt.Sprintf("%v %v/%v", objectID, forceStored, size))
 
 			if size > 1 {
-				sectionID := NewSectionObjectID(0, int64(size/2), objectID)
+				sectionID := NewSectionObjectID(0, int64(size/2), &objectID)
 				verify(t, repo, sectionID, randomData[0:10], fmt.Sprintf("%v %v/%v", sectionID, forceStored, size))
 			}
 
 			if size > 1 {
-				sectionID := NewSectionObjectID(int64(1), int64(size-1), objectID)
+				sectionID := NewSectionObjectID(int64(1), int64(size-1), &objectID)
 				verify(t, repo, sectionID, randomData[1:], fmt.Sprintf("%v %v/%v", sectionID, forceStored, size))
 			}
 		}
@@ -397,9 +348,9 @@ func verify(t *testing.T, repo Repository, objectID ObjectID, expectedData []byt
 }
 
 func TestFormats(t *testing.T) {
-	makeFormat := func(objectFormat string) *Format {
+	makeFormat := func(objectFormat ObjectIDFormat) *Format {
 		return &Format{
-			Version:      "1",
+			Version:      1,
 			ObjectFormat: objectFormat,
 			Secret:       []byte("key"),
 			MaxBlobSize:  10000,
@@ -411,88 +362,78 @@ func TestFormats(t *testing.T) {
 		oids   map[string]ObjectID
 	}{
 		{
-			format: makeFormat("md5"), // MD5-HMAC with secret 'key'
+			format: makeFormat(ObjectIDFormat_TESTONLY_MD5), // MD5-HMAC with secret 'key'
 			oids: map[string]ObjectID{
-				"": "D63530468a04e386459855da0063b6596",
-				"The quick brown fox jumps over the lazy dog": "D80070713463e7749b90c2dc24911e275",
+				"": ObjectID{StorageBlock: "63530468a04e386459855da0063b6596"},
+				"The quick brown fox jumps over the lazy dog": ObjectID{StorageBlock: "80070713463e7749b90c2dc24911e275"},
 			},
 		},
 		{
 			format: &Format{
-				Version:      "1",
-				ObjectFormat: "md5",
+				Version:      1,
+				ObjectFormat: ObjectIDFormat_TESTONLY_MD5,
 				MaxBlobSize:  10000,
 				Secret:       []byte{}, // HMAC with zero-byte secret
 			},
 			oids: map[string]ObjectID{
-				"": "D74e6f7298a9c2d168935f58c001bad88",
-				"The quick brown fox jumps over the lazy dog": "Dad262969c53bc16032f160081c4a07a0",
+				"": ObjectID{StorageBlock: "74e6f7298a9c2d168935f58c001bad88"},
+				"The quick brown fox jumps over the lazy dog": ObjectID{StorageBlock: "ad262969c53bc16032f160081c4a07a0"},
 			},
 		},
 		{
 			format: &Format{
-				Version:      "1",
-				ObjectFormat: "md5",
+				Version:      1,
+				ObjectFormat: ObjectIDFormat_TESTONLY_MD5,
 				MaxBlobSize:  10000,
 				Secret:       nil, // non-HMAC version
 			},
 			oids: map[string]ObjectID{
-				"": "Dd41d8cd98f00b204e9800998ecf8427e",
-				"The quick brown fox jumps over the lazy dog": "D9e107d9d372bb6826bd81d3542a419d6",
+				"": ObjectID{StorageBlock: "d41d8cd98f00b204e9800998ecf8427e"},
+				"The quick brown fox jumps over the lazy dog": ObjectID{
+					StorageBlock: "9e107d9d372bb6826bd81d3542a419d6",
+				},
 			},
 		},
 		{
-			format: makeFormat("md5"),
+			format: makeFormat(ObjectIDFormat_TESTONLY_MD5),
 			oids: map[string]ObjectID{
-				"The quick brown fox jumps over the lazy dog": "D80070713463e7749b90c2dc24911e275",
+				"The quick brown fox jumps over the lazy dog": ObjectID{
+					StorageBlock: "80070713463e7749b90c2dc24911e275",
+				},
 			},
 		},
 		{
-			format: makeFormat("sha1"),
+			format: makeFormat(ObjectIDFormat_UNENCYPTED_HMAC_SHA256),
 			oids: map[string]ObjectID{
-				"The quick brown fox jumps over the lazy dog": "Dde7c9b85b8b78aa6bc8a7a36f70a90701c9db4d9",
+				"The quick brown fox jumps over the lazy dog": ObjectID{
+					StorageBlock: "f7bc83f430538424b13298e6aa6fb143ef4d59a14946175997479dbc2d1a3cd8",
+				},
 			},
 		},
 		{
-			format: makeFormat("sha256"),
+			format: makeFormat(ObjectIDFormat_UNENCYPTED_HMAC_SHA256_128),
 			oids: map[string]ObjectID{
-				"The quick brown fox jumps over the lazy dog": "Df7bc83f430538424b13298e6aa6fb143ef4d59a14946175997479dbc2d1a3cd8",
+				"The quick brown fox jumps over the lazy dog": ObjectID{
+					StorageBlock: "18f1da557915937d2675055a87758d9b",
+				},
 			},
 		},
 		{
-			format: makeFormat("sha512"),
+			format: makeFormat(ObjectIDFormat_ENCRYPTED_HMAC_SHA512_384_AES256),
 			oids: map[string]ObjectID{
-				"The quick brown fox jumps over the lazy dog": "Db42af09057bac1e2d41708e48a902e09b5ff7f12ab428a4fe86653c73dd248fb82f948a549f7b791a5b41915ee4d1ec3935357e4e2317250d0372afa2ebeeb3a",
+				"The quick brown fox jumps over the lazy dog": ObjectID{
+					StorageBlock: "d7f4727e2c0b39ae0f1e40cc96f60242",
+					Encryption:   mustParseBase16("d5b7801841cea6fc592c5d3e1ae50700582a96cf35e1e554995fe4e03381c237"),
+				},
 			},
 		},
 		{
-			format: makeFormat("sha256-fold128-aes128"),
+			format: makeFormat(ObjectIDFormat_ENCRYPTED_HMAC_SHA512_AES256),
 			oids: map[string]ObjectID{
-				"The quick brown fox jumps over the lazy dog": "D028e74c630aee6400db4f84d49d1ed08.2825273c3344999e26081d99bcaf3f18",
-			},
-		},
-		{
-			format: makeFormat("sha256-aes128"),
-			oids: map[string]ObjectID{
-				"The quick brown fox jumps over the lazy dog": "D26bb3fee9e83a7f6ff8397f33460cd0d24354b28ae2d41b6f2376fbe7db12005.2825273c3344999e26081d99bcaf3f18",
-			},
-		},
-		{
-			format: makeFormat("sha384-aes256"),
-			oids: map[string]ObjectID{
-				"The quick brown fox jumps over the lazy dog": "D08edca658caee5ae86c9f1e4ccea26b1a43b0ddb04502c5f1b1cba051a756a9eba440bb6991b23323e37556ec5cf4f88.7a753377319650870f5cba47c30b608675542f206fa67d94dcbb9a0ccce7f467",
-			},
-		},
-		{
-			format: makeFormat("sha512-aes256"),
-			oids: map[string]ObjectID{
-				"The quick brown fox jumps over the lazy dog": "Deee04ad71248ef4ffec5159558e80e2791e32299cfb79a1e063cc5f4a71cd61ca2c1b110e3ae2e83635a8626a3a27e4805eb745c40b5a4ebd4c9372602e5ab65.1c0b1b58ce05b7b8b05cfce27a485ddf97bde5159f6946357ec7795236f36a84",
-			},
-		},
-		{
-			format: makeFormat("sha512-fold128-aes192"),
-			oids: map[string]ObjectID{
-				"The quick brown fox jumps over the lazy dog": "Dd829ad027ee4ff394f6a61615eb30d16.0ab00e0e0e156927f0be63372e8449d7bf2316c43d46e626",
+				"The quick brown fox jumps over the lazy dog": ObjectID{
+					StorageBlock: "b42af09057bac1e2d41708e48a902e09b5ff7f12ab428a4fe86653c73dd248fb",
+					Encryption:   mustParseBase16("82f948a549f7b791a5b41915ee4d1ec3935357e4e2317250d0372afa2ebeeb3a"),
+				},
 			},
 		},
 	}
@@ -501,7 +442,7 @@ func TestFormats(t *testing.T) {
 		data := map[string][]byte{}
 		st := storagetesting.NewMapStorage(data)
 
-		t.Logf("verifying %#v", c.format)
+		t.Logf("verifying %v", c.format.String())
 		repo, err := NewRepository(st, c.format)
 		if err != nil {
 			t.Errorf("cannot create manager: %v", err)
@@ -516,8 +457,8 @@ func TestFormats(t *testing.T) {
 			if err != nil {
 				t.Errorf("error: %v", err)
 			}
-			if oid != v {
-				t.Errorf("invalid oid for #%v %v/%v: %v expected %v", caseIndex, c.format.ObjectFormat, k, oid, v)
+			if !oid.Equals(v) {
+				t.Errorf("invalid oid for #%v %v/%v:\ngot:\n%#v\nexpected:\n%#v", caseIndex, c.format.ObjectFormat, k, oid.UIString(), v.UIString())
 			}
 
 			rc, err := repo.Open(oid)
@@ -540,8 +481,8 @@ func TestInvalidEncryptionKey(t *testing.T) {
 	data := map[string][]byte{}
 	st := storagetesting.NewMapStorage(data)
 	format := Format{
-		Version:      "1",
-		ObjectFormat: "sha512-aes256",
+		Version:      1,
+		ObjectFormat: ObjectIDFormat_ENCRYPTED_HMAC_SHA512_384_AES256,
 		Secret:       []byte("key"),
 		MaxBlobSize:  1000,
 	}
@@ -571,47 +512,42 @@ func TestInvalidEncryptionKey(t *testing.T) {
 	}
 
 	// Key too short
-	rc, err = repo.Open(oid[0 : len(oid)-2])
+	rc, err = repo.Open(replaceEncryption(oid, oid.Encryption[0:len(oid.Encryption)-2]))
 	if err == nil || rc != nil {
 		t.Errorf("expected error when opening malformed object")
 	}
 
 	// Key too long
-	rc, err = repo.Open(oid + "ff")
+	rc, err = repo.Open(replaceEncryption(oid, append(oid.Encryption, 0xFF)))
 	if err == nil || rc != nil {
 		t.Errorf("expected error when opening malformed object")
 	}
 
 	// Invalid key
-	lastByte, _ := hex.DecodeString(string(oid[len(oid)-2:]))
-	lastByte[0]++
-	rc, err = repo.Open(oid[0:len(oid)-2] + ObjectID(hex.EncodeToString(lastByte)))
+	corruptedKey := append([]byte(nil), oid.Encryption...)
+	corruptedKey[0]++
+	rc, err = repo.Open(replaceEncryption(oid, corruptedKey))
 	if err == nil || rc != nil {
 		t.Errorf("expected error when opening malformed object: %v", err)
 	}
 
 	// Now corrupt the data
-	data[string(oid.BlockID())][0] ^= 1
+	data[oid.StorageBlock][0] ^= 1
 	rc, err = repo.Open(oid)
 	if err == nil || rc != nil {
 		t.Errorf("expected error when opening object with corrupt data")
 	}
 }
 
-func verifyStorageBlocks(t *testing.T, actual []string, data, expectedData map[string][]byte) {
-	actualCopy := append([]string(nil), actual...)
-	sort.Strings(actualCopy)
-	var expected []string
-	for k := range expectedData {
-		expected = append(expected, k)
-	}
-	sort.Strings(expected)
-
-	if !reflect.DeepEqual(expected, actualCopy) {
-		t.Errorf("updated blocks don't match. Expected: %#v, got %#v", expected, actualCopy)
-	}
-
-	if !reflect.DeepEqual(expectedData, data) {
-		t.Errorf("storage data does not match, expected: %v, got %v", expectedData, data)
-	}
+func replaceEncryption(oid ObjectID, newEncryption []byte) ObjectID {
+	oid.Encryption = newEncryption
+	return oid
+}
+
+func mustParseBase16(s string) []byte {
+	b, err := hex.DecodeString(s)
+	if err != nil {
+		panic("invalid hex literal: " + s)
+	}
+	return b
 }

vault/objectid_storage.go

@@ -6,6 +6,8 @@ import (
 	"encoding/hex"
 	"fmt"
 
+	"github.com/golang/protobuf/proto"
+
 	"github.com/kopia/kopia/repo"
 )
 
@@ -16,14 +18,14 @@ const (
 	storedObjectIDLengthBytes = 8
 )
 
-type objectIDData struct {
-	ObjectID string `json:"objectID"`
-}
-
 // SaveObjectID stores the given object ID in an encrypted vault item and returns a unique ID.
 func (vlt *Vault) SaveObjectID(oid repo.ObjectID) (string, error) {
 	h := hmac.New(sha256.New, vlt.format.UniqueID)
-	h.Write([]byte(oid))
+	bytes, err := proto.Marshal(&oid)
+	if err != nil {
+		return "", err
+	}
+	h.Write(bytes)
 	sum := h.Sum(nil)
 	for i := storedObjectIDLengthBytes; i < len(sum); i++ {
 		sum[i%storedObjectIDLengthBytes] ^= sum[i]
@@ -31,10 +33,7 @@ func (vlt *Vault) SaveObjectID(oid repo.ObjectID) (string, error) {
 	sum = sum[0:storedObjectIDLengthBytes]
 	key := StoredObjectIDPrefix + hex.EncodeToString(sum)
 
-	var d objectIDData
-	d.ObjectID = string(oid)
-
-	if err := vlt.putJSON(key, &d); err != nil {
+	if err := vlt.Put(key, bytes); err != nil {
 		return "", err
 	}
 
@@ -45,20 +44,26 @@ func (vlt *Vault) SaveObjectID(oid repo.ObjectID) (string, error) {
 func (vlt *Vault) GetObjectID(id string) (repo.ObjectID, error) {
 	matches, err := vlt.List(id)
 	if err != nil {
-		return "", err
+		return repo.NullObjectID, err
 	}
 
 	switch len(matches) {
 	case 0:
-		return "", ErrItemNotFound
+		return repo.NullObjectID, ErrItemNotFound
 	case 1:
-		var d objectIDData
-		if err := vlt.getJSON(matches[0], &d); err != nil {
-			return "", err
+		b, err := vlt.Get(matches[0])
+		if err != nil {
+			return repo.NullObjectID, err
 		}
-		return repo.ParseObjectID(d.ObjectID)
+
+		var oid repo.ObjectID
+		if err := proto.Unmarshal(b, &oid); err != nil {
+			return repo.NullObjectID, err
+		}
+
+		return oid, nil
 
 	default:
-		return "", fmt.Errorf("ambiguous object ID: %v", id)
+		return repo.NullObjectID, fmt.Errorf("ambiguous object ID: %v", id)
 	}
 }

vault/vault.pb.go

@@ -0,0 +1,35 @@
+// Code generated by protoc-gen-go.
+// source: vault/vault.proto
+// DO NOT EDIT!
+
+/*
+Package vault is a generated protocol buffer package.
+
+It is generated from these files:
+	vault/vault.proto
+
+It has these top-level messages:
+*/
+package vault
+
+import proto "github.com/golang/protobuf/proto"
+import fmt "fmt"
+import math "math"
+
+// Reference imports to suppress errors if they are not otherwise used.
+var _ = proto.Marshal
+var _ = fmt.Errorf
+var _ = math.Inf
+
+// This is a compile-time assertion to ensure that this generated file
+// is compatible with the proto package it is being compiled against.
+const _ = proto.ProtoPackageIsVersion1
+
+var fileDescriptor0 = []byte{
+	// 75 bytes of a gzipped FileDescriptorProto
+	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0xe2, 0x12, 0x2c, 0x4b, 0x2c, 0xcd,
+	0x29, 0xd1, 0x07, 0x93, 0x7a, 0x05, 0x45, 0xf9, 0x25, 0xf9, 0x42, 0xdc, 0xd9, 0xf9, 0x05, 0x99,
+	0x89, 0x7a, 0x60, 0x21, 0x27, 0xb9, 0x28, 0x99, 0xf4, 0xcc, 0x92, 0x8c, 0xd2, 0x24, 0xbd, 0xe4,
+	0xfc, 0x5c, 0x7d, 0xb0, 0x0c, 0x94, 0x04, 0xcb, 0x27, 0xb1, 0x81, 0xf5, 0x18, 0x03, 0x02, 0x00,
+	0x00, 0xff, 0xff, 0x9a, 0x15, 0x10, 0x31, 0x48, 0x00, 0x00, 0x00,
+}

vault/vault.proto

@@ -0,0 +1,4 @@
+syntax = "proto3";
+
+package kopia.vault;
+option go_package = "github.com/kopia/kopia/vault";

vault/vault_test.go

@@ -62,9 +62,9 @@ func verifyVault(t *testing.T, vaultPath string, repoPath string) {
 	}
 
 	repoFormat := &repo.Format{
-		Version:      "1",
+		Version:      1,
 		MaxBlobSize:  1000000,
-		ObjectFormat: "sha256",
+		ObjectFormat: repo.ObjectIDFormat_UNENCYPTED_HMAC_SHA256,
 		Secret:       []byte{1, 2, 3},
 	}
 
@@ -189,7 +189,7 @@ func verifyVault(t *testing.T, vaultPath string, repoPath string) {
 		if err != nil {
 			t.Errorf("error getting object ID: %v", err)
 		} else {
-			if oid != oid1 {
+			if !oid.Equals(oid1) {
 				t.Errorf("got invalid object ID: %v expected %v", oid, oid1)
 			}
 		}
@@ -198,7 +198,7 @@ func verifyVault(t *testing.T, vaultPath string, repoPath string) {
 		if err != nil {
 			t.Errorf("error getting object ID: %v", err)
 		} else {
-			if oid != oid2 {
+			if !oid.Equals(oid2) {
 				t.Errorf("got invalid object ID: %v expected %v", oid, oid2)
 			}
 		}