From e528b32748c841a538e412bf23813f6d57ba10cc Mon Sep 17 00:00:00 2001 From: Jarek Kowalski Date: Mon, 15 Aug 2016 18:42:43 -0700 Subject: [PATCH] Major refactoring of data formats to use protobuf instead of JSON. This is a breaking change to data formats, but hoping to stabilize soon. This change removes lots of pointless string and custom format parsing code and instead relies on protobuf to do the thing. JSON is still an option thanks to proto3-generated output. - Refactored ObjectID to use protobuf - Indirect blocks to use protobuf (in the form of seek table) - Added generic proto stream reader and writer - ObjectIDFormat became proto enum Also: - fixed major issue where indirect object ID entries were not encrypted, since they were storing block IDs and not object IDs. - dropped support for majority of formats, only supporting HMAC-{SHA256,SHA512,SHA512_384} hashes with AES256. This produces object IDs of 32 or 64 characters long. - changed how pretty-printing works for 'kopia show' --- .gitignore | 1 + Makefile | 11 +- backup/generator.go | 8 +- backup/manifest.go | 10 +- cmd/kopia/command_backup.go | 2 +- cmd/kopia/command_create.go | 25 +- cmd/kopia/command_ls.go | 20 +- cmd/kopia/command_show.go | 49 ++- cmd/kopia/objref.go | 16 +- cmd/playground/main.go | 77 ++++ fs/dir_json.go | 400 ++---------------- fs/dir_test.go | 292 ++++++------- fs/entry.go | 36 +- fs/entry_metadata.pb.go | 89 ++++ fs/entry_metadata.proto | 19 + fs/hashcache.go | 60 +-- fs/hashcache.pb.go | 52 +++ fs/hashcache.proto | 12 + ...fs_for_test.go => inmemory_fs_for_test.go} | 10 +- fs/{filesystem.go => local_fs.go} | 6 +- ...m_nonwindows.go => local_fs_nonwindows.go} | 4 +- fs/{filesystem_test.go => local_fs_test.go} | 10 +- ...esystem_windows.go => local_fs_windows.go} | 0 fs/{repository.go => repository_fs.go} | 14 +- fs/upload.go | 68 +-- fs/upload_test.go | 38 +- fuse/fusefs.go | 14 +- internal/proto_stream_reader.go | 66 +++ internal/proto_stream_test.go | 74 ++++ internal/proto_stream_types.go | 12 + internal/proto_stream_writer.go | 58 +++ internal/proto_test.pb.go | 51 +++ internal/proto_test.proto | 8 + repo/format.go | 155 ++++--- repo/format.pb.go | 141 ++++++ repo/format.proto | 41 ++ repo/indirect.pb.go | 52 +++ repo/indirect.proto | 13 + repo/object_reader.go | 40 +- repo/object_writer.go | 95 +++-- repo/objectid.go | 245 ++++------- repo/objectid.pb.go | 80 ++++ repo/objectid.proto | 21 + repo/objectid_test.go | 39 +- repo/repository.go | 170 ++++---- repo/repository_test.go | 320 ++++++-------- vault/objectid_storage.go | 37 +- vault/vault.pb.go | 35 ++ vault/vault.proto | 4 + vault/vault_test.go | 8 +- 50 files changed, 1784 insertions(+), 1324 deletions(-) create mode 100644 cmd/playground/main.go create mode 100644 fs/entry_metadata.pb.go create mode 100644 fs/entry_metadata.proto create mode 100644 fs/hashcache.pb.go create mode 100644 fs/hashcache.proto rename fs/{inmemoryfs_for_test.go => inmemory_fs_for_test.go} (94%) rename fs/{filesystem.go => local_fs.go} (95%) rename fs/{filesystem_nonwindows.go => local_fs_nonwindows.go} (82%) rename fs/{filesystem_test.go => local_fs_test.go} (86%) rename fs/{filesystem_windows.go => local_fs_windows.go} (100%) rename fs/{repository.go => repository_fs.go} (88%) create mode 100644 internal/proto_stream_reader.go create mode 100644 internal/proto_stream_test.go create mode 100644 internal/proto_stream_types.go create mode 100644 internal/proto_stream_writer.go create mode 100644 internal/proto_test.pb.go create mode 100644 internal/proto_test.proto create mode 100644 repo/format.pb.go create mode 100644 repo/format.proto create mode 100644 repo/indirect.pb.go create mode 100644 repo/indirect.proto create mode 100644 repo/objectid.pb.go create mode 100644 repo/objectid.proto create mode 100644 vault/vault.pb.go create mode 100644 vault/vault.proto diff --git a/.gitignore b/.gitignore index 4bf2d1d37..0dc13de08 100644 --- a/.gitignore +++ b/.gitignore @@ -3,3 +3,4 @@ .releases/ kopia-*.tar.gz kopia-*.zip +combined.cov diff --git a/Makefile b/Makefile index da871ae25..ae9f64c3c 100644 --- a/Makefile +++ b/Makefile @@ -17,11 +17,20 @@ install: build: go build github.com/kopia/kopia/... +escape-analysis: + go build -gcflags '-m -l' github.com/kopia/kopia/... + +protos: + protoc --go_out=../../.. fs/*.proto + protoc --go_out=../../.. repo/*.proto + protoc --go_out=../../.. vault/*.proto + protoc --go_out=../../.. internal/*.proto + play: go run cmd/playground/main.go lint: - golint github.com/kopia/kopia/... + -golint github.com/kopia/kopia/... | grep -v .pb.go vet: go tool vet -all . diff --git a/backup/generator.go b/backup/generator.go index e35d58143..7322babb4 100644 --- a/backup/generator.go +++ b/backup/generator.go @@ -25,10 +25,10 @@ func (bg *backupGenerator) Backup(m *Manifest, old *Manifest) error { uploader := fs.NewUploader(bg.repo) m.StartTime = time.Now() - var hashCacheID repo.ObjectID + var hashCacheID *repo.ObjectID if old != nil { - hashCacheID = repo.ObjectID(old.HashCacheID) + hashCacheID = &old.HashCacheID } entry, err := fs.NewFilesystemEntry(m.Source, nil) @@ -46,8 +46,8 @@ func (bg *backupGenerator) Backup(m *Manifest, old *Manifest) error { if err != nil { return err } - m.RootObjectID = string(r.ObjectID) - m.HashCacheID = string(r.ManifestID) + m.RootObjectID = r.ObjectID + m.HashCacheID = r.ManifestID return nil } diff --git a/backup/manifest.go b/backup/manifest.go index 2133099c0..fa1594d98 100644 --- a/backup/manifest.go +++ b/backup/manifest.go @@ -5,6 +5,8 @@ import ( "encoding/hex" "io" "time" + + "github.com/kopia/kopia/repo" ) // Manifest stores information about single backup. @@ -16,10 +18,10 @@ type Manifest struct { UserName string `json:"userName"` Description string `json:"description"` - Handle string `json:"handle"` - Source string `json:"source"` - RootObjectID string `json:"root"` - HashCacheID string `json:"hashCache"` + Handle string `json:"handle"` + Source string `json:"source"` + RootObjectID repo.ObjectID `json:"root"` + HashCacheID repo.ObjectID `json:"hashCache"` FileCount int64 `json:"fileCount"` DirectoryCount int64 `json:"dirCount"` diff --git a/cmd/kopia/command_backup.go b/cmd/kopia/command_backup.go index bc4d65f7c..46a091e91 100644 --- a/cmd/kopia/command_backup.go +++ b/cmd/kopia/command_backup.go @@ -113,7 +113,7 @@ func runBackupCommand(context *kingpin.ParseContext) error { return fmt.Errorf("cannot save manifest: %v", err) } - log.Printf("Root: %v", manifest.RootObjectID) + log.Printf("Root: %v", manifest.RootObjectID.UIString()) log.Printf("Key: %v", handleID) } diff --git a/cmd/kopia/command_create.go b/cmd/kopia/command_create.go index 409d738ba..0042a2076 100644 --- a/cmd/kopia/command_create.go +++ b/cmd/kopia/command_create.go @@ -15,7 +15,7 @@ import ( var ( createCommand = app.Command("create", "Create new vault and repository.") createCommandRepository = createCommand.Flag("repository", "Repository path.").Default("colocated").String() - createObjectFormat = createCommand.Flag("repo-format", "Format of repository objects.").PlaceHolder("FORMAT").Default("sha256-fold160-aes128").Enum(supportedObjectFormats()...) + createObjectFormat = createCommand.Flag("repo-format", "Format of repository objects.").PlaceHolder("FORMAT").Default(repo.DefaultObjectFormat.Name()).Enum(supportedObjectFormats()...) createMaxBlobSize = createCommand.Flag("max-blob-size", "Maximum size of a data chunk.").PlaceHolder("KB").Default("20480").Int() createInlineBlobSize = createCommand.Flag("inline-blob-size", "Maximum size of an inline data chunk.").PlaceHolder("KB").Default("32").Int() @@ -43,15 +43,20 @@ func vaultFormat() (*vault.Format, error) { } func repositoryFormat() (*repo.Format, error) { - f := &repo.Format{ - Version: "1", - Secret: make([]byte, 32), - MaxBlobSize: *createMaxBlobSize * 1024, - MaxInlineBlobSize: *createInlineBlobSize * 1024, - ObjectFormat: *createObjectFormat, + objectIDFormat, err := repo.ParseObjectIDFormat(*createObjectFormat) + if err != nil { + return nil, err } - _, err := io.ReadFull(rand.Reader, f.Secret) + f := &repo.Format{ + Version: 1, + Secret: make([]byte, 32), + MaxBlobSize: int32(*createMaxBlobSize * 1024), + MaxInlineBlobSize: int32(*createInlineBlobSize * 1024), + ObjectFormat: objectIDFormat, + } + + _, err = io.ReadFull(rand.Reader, f.Secret) if err != nil { return nil, err } @@ -100,7 +105,7 @@ func runCreateCommand(context *kingpin.ParseContext) error { } fmt.Printf( - "Initializing repository in with format '%v' and maximum object size %v.\n", + "Initializing repository in with format %v and maximum object size %v.\n", repoFormat.ObjectFormat, repoFormat.MaxBlobSize) @@ -151,7 +156,7 @@ func supportedVaultEncryptionFormats() []string { func supportedObjectFormats() []string { var r []string for _, o := range repo.SupportedFormats { - r = append(r, o.Name) + r = append(r, o.Name()) } return r } diff --git a/cmd/kopia/command_ls.go b/cmd/kopia/command_ls.go index e68ecab24..1c9861430 100644 --- a/cmd/kopia/command_ls.go +++ b/cmd/kopia/command_ls.go @@ -6,7 +6,6 @@ import ( "strings" "github.com/kopia/kopia/fs" - "github.com/kopia/kopia/repo" kingpin "gopkg.in/alecthomas/kingpin.v2" ) @@ -69,17 +68,22 @@ func listDirectory(prefix string, entries fs.Entries, longFormat bool) { var info string if longFormat { var oid string - if m.ObjectID.Type() == repo.ObjectIDTypeBinary { - oid = "" - } else if m.ObjectID.Type() == repo.ObjectIDTypeText { - oid = "" + if m.ObjectId.Content != nil { + oid = "" } else { - oid = string(m.ObjectID) + oid = m.ObjectId.UIString() } - info = fmt.Sprintf("%v %9d %v %-"+maxNameLenString+"s %v", m.FileMode, m.FileSize, m.ModTime.Local().Format("02 Jan 06 15:04:05"), m.Name, oid) + info = fmt.Sprintf( + "%v %9d %v %-"+maxNameLenString+"s %v", + m.FileMode(), + m.FileSize, + m.ModTime().Local().Format("02 Jan 06 15:04:05"), + m.Name, + oid, + ) } else { var suffix string - if m.FileMode.IsDir() { + if m.FileMode().IsDir() { suffix = "/" } diff --git a/cmd/kopia/command_show.go b/cmd/kopia/command_show.go index 75a7ec692..b992487c9 100644 --- a/cmd/kopia/command_show.go +++ b/cmd/kopia/command_show.go @@ -1,11 +1,19 @@ package main import ( + "bufio" "bytes" "encoding/json" + "fmt" + "io" "io/ioutil" "os" + "github.com/golang/protobuf/proto" + "github.com/kopia/kopia/fs" + + "github.com/kopia/kopia/internal" + "github.com/kopia/kopia/repo" kingpin "gopkg.in/alecthomas/kingpin.v2" @@ -54,8 +62,8 @@ func showObject(mgr repo.Repository, oid repo.ObjectID) error { format := "raw" - if len(rawdata) > 2 && rawdata[0] == '{' && rawdata[len(rawdata)-1] == '}' { - format = "json" + if len(rawdata) >= 8 { + format = detectProtoFormat(rawdata[0:8]) } if *showJSON { @@ -73,12 +81,49 @@ func showObject(mgr repo.Repository, oid repo.ObjectID) error { json.Indent(&buf, rawdata, "", " ") os.Stdout.Write(buf.Bytes()) + case "dir": + return prettyPrintProtoStream(rawdata, internal.ProtoStreamTypeDir, &fs.EntryMetadata{}) + case "indirect": + return prettyPrintProtoStream(rawdata, internal.ProtoStreamTypeIndirect, &repo.IndirectObjectEntry{}) + case "hashcache": + return prettyPrintProtoStream(rawdata, internal.ProtoStreamTypeHashCache, &fs.HashCacheEntry{}) + default: os.Stdout.Write(rawdata) } return nil } +func detectProtoFormat(b []byte) string { + switch { + case bytes.Equal(b, internal.ProtoStreamTypeDir): + return "dir" + case bytes.Equal(b, internal.ProtoStreamTypeIndirect): + return "indirect" + case bytes.Equal(b, internal.ProtoStreamTypeHashCache): + return "hashcache" + } + return "raw" +} + +func prettyPrintProtoStream(data []byte, header []byte, pb proto.Message) error { + r := internal.NewProtoStreamReader(bufio.NewReader(bytes.NewReader(data)), header) + i := 0 + for { + err := r.Read(pb) + if err == io.EOF { + return nil + } + if err != nil { + return err + } + + fmt.Printf("[entry #%v]\n", i) + fmt.Println(proto.MarshalTextString(pb)) + i++ + } +} + func init() { showCommand.Action(runShowCommand) } diff --git a/cmd/kopia/objref.go b/cmd/kopia/objref.go index c24aebf94..7f361a847 100644 --- a/cmd/kopia/objref.go +++ b/cmd/kopia/objref.go @@ -13,7 +13,7 @@ import ( func parseObjectID(id string, vlt *vault.Vault) (repo.ObjectID, error) { head, tail := splitHeadTail(id) if len(head) == 0 { - return "", fmt.Errorf("invalid object ID: %v", id) + return repo.NullObjectID, fmt.Errorf("invalid object ID: %v", id) } if !strings.HasPrefix(id, vault.StoredObjectIDPrefix) { @@ -22,7 +22,7 @@ func parseObjectID(id string, vlt *vault.Vault) (repo.ObjectID, error) { oid, err := vlt.GetObjectID(head) if err != nil { - return "", fmt.Errorf("can't retrieve vault object ID %v: %v", head, err) + return repo.NullObjectID, fmt.Errorf("can't retrieve vault object ID %v: %v", head, err) } if tail == "" { @@ -31,12 +31,12 @@ func parseObjectID(id string, vlt *vault.Vault) (repo.ObjectID, error) { r, err := vlt.OpenRepository() if err != nil { - return "", fmt.Errorf("cannot open repository: %v", err) + return repo.NullObjectID, fmt.Errorf("cannot open repository: %v", err) } dir := fs.NewRepositoryDirectory(r, oid) if err != nil { - return "", err + return repo.NullObjectID, err } return parseNestedObjectID(dir, tail) @@ -49,24 +49,24 @@ func parseNestedObjectID(startingDir fs.Directory, id string) (repo.ObjectID, er for head != "" { dir, ok := current.(fs.Directory) if !ok { - return "", fmt.Errorf("entry not found '%v': parent is not a directory", head) + return repo.NullObjectID, fmt.Errorf("entry not found '%v': parent is not a directory", head) } entries, err := dir.Readdir() if err != nil { - return "", err + return repo.NullObjectID, err } e := entries.FindByName(head) if e == nil { - return "", fmt.Errorf("entry not found: '%v'", head) + return repo.NullObjectID, fmt.Errorf("entry not found: '%v'", head) } current = e head, tail = splitHeadTail(tail) } - return current.Metadata().ObjectID, nil + return *current.Metadata().ObjectId, nil } func splitHeadTail(id string) (string, string) { diff --git a/cmd/playground/main.go b/cmd/playground/main.go new file mode 100644 index 000000000..493013666 --- /dev/null +++ b/cmd/playground/main.go @@ -0,0 +1,77 @@ +package main + +import ( + "bufio" + "bytes" + "log" + + "github.com/kopia/kopia/internal" + "github.com/kopia/kopia/repo" + "github.com/kopia/kopia/storage/storagetesting" +) + +func dumpListBlock(prefix string, r repo.Repository, oid repo.ObjectID) { + ind := oid.Indirect + oid.Indirect = 0 + log.Printf("%vdumping %v", prefix, oid.String()) + rd, err := r.Open(oid) + if err != nil { + log.Fatalf("unable to open %v: %v", oid.String(), err) + } + defer rd.Close() + + var objects []repo.ObjectID + + pr := internal.NewProtoStreamReader(bufio.NewReader(rd), internal.ProtoStreamTypeIndirect) + for { + var v repo.IndirectObjectEntry + if err := pr.Read(&v); err != nil { + log.Printf("err: %v", err) + break + } + log.Printf("%v [%v] entry: %v", prefix, oid.String(), v.String()) + if ind > 0 { + objects = append(objects, *v.Object) + } + } + + for _, o := range objects { + dumpListBlock(prefix+" ", r, o) + } +} + +func main() { + d := map[string][]byte{} + + s := storagetesting.NewMapStorage(d) + f := repo.Format{ + Version: 1, + MaxBlobSize: 100, + ObjectFormat: repo.ObjectIDFormat_TESTONLY_MD5, + } + r, err := repo.NewRepository(s, &f) + if err != nil { + log.Fatalf("Err: %v", err) + } + w := r.NewWriter(repo.WithDescription("tester")) + + w.Write(bytes.Repeat([]byte("a"), 5000)) + result, err := w.Result(true) + log.Printf("result: %v", result.UIString()) + + //dumpListBlock("", r, result) + + log.Printf("attempting to open") + + rdr, err := r.Open(result) + if err != nil { + log.Fatalf("open: %v", err) + } + log.Printf("len: %v\n", rdr.Length()) + log.Printf("----") + // for k, v := range d { + // log.Printf("%v = %v", k, string(v)) + // } + + rdr.Close() +} diff --git a/fs/dir_json.go b/fs/dir_json.go index 456f73a2e..786e02a88 100644 --- a/fs/dir_json.go +++ b/fs/dir_json.go @@ -1,407 +1,79 @@ package fs import ( - "encoding/json" + "bufio" "fmt" "io" - "os" - "strconv" - "strings" - "time" + + "github.com/kopia/kopia/internal" "github.com/kopia/kopia/repo" ) -const modeChars = "dalTLDpSugct" - -type jsonDirectoryEntry struct { - Name string `json:"name"` - Type string `json:"type,omitempty"` - Permissions string `json:"perm,omitempty"` - Size string `json:"size,omitempty"` - Time *time.Time `json:"mtime,omitempty"` - Owner string `json:"owner,omitempty"` - ObjectID string `json:"oid,omitempty"` - JSONContent json.RawMessage `json:"content,omitempty"` - SubEntries []jsonDirectoryEntry `json:"entries,omitempty"` -} - -func (em *EntryMetadata) fromJSON(jde *jsonDirectoryEntry) error { - if jde.Name == "" { - return fmt.Errorf("empty entry name") - } - em.Name = jde.Name - - if jde.Permissions != "" { - if m, err := parseFilePermissions(jde.Permissions); err == nil { - em.FileMode |= m - } else { - return fmt.Errorf("invalid permissions: '%v'", jde.Permissions) - } - } - - if jde.Type != "" { - if m, err := parseFileMode(jde.Type); err == nil { - em.FileMode |= m - } else { - return fmt.Errorf("invalid type: '%v'", jde.Type) - } - } - - if jde.Time != nil { - em.ModTime = *jde.Time - } - - if jde.Owner != "" { - if c, err := fmt.Sscanf(jde.Owner, "%d:%d", &em.OwnerID, &em.GroupID); err != nil || c != 2 { - return fmt.Errorf("invalid owner: %v", err) - } - } - - if jde.JSONContent != nil { - em.ObjectID = repo.NewInlineObjectID([]byte(jde.JSONContent)) - } else { - em.ObjectID = repo.ObjectID(jde.ObjectID) - } - - if jde.Size != "" { - if s, err := strconv.ParseInt(jde.Size, 10, 64); err == nil { - em.FileSize = s - } else { - return fmt.Errorf("invalid size: %v", err) - } - } - return nil -} - -// parseFileModeAndPermissions converts file mode string to os.FileMode -func parseFileModeAndPermissions(s string) (os.FileMode, error) { - colon := strings.IndexByte(s, ':') - if colon < 0 { - return parseFilePermissions(s) - } - - var mode os.FileMode - - if m, err := parseFileMode(s[0:colon]); err == nil { - mode |= m - } else { - return 0, err - } - - if m, err := parseFilePermissions(s[colon+1:]); err == nil { - mode |= m - } else { - return 0, err - } - - return mode, nil -} - -func parseFileMode(s string) (os.FileMode, error) { - var mode os.FileMode - for _, c := range s { - switch c { - case 'd': - mode |= os.ModeDir - case 'a': - mode |= os.ModeAppend - case 'l': - mode |= os.ModeExclusive - case 'T': - mode |= os.ModeTemporary - case 'L': - mode |= os.ModeSymlink - case 'D': - mode |= os.ModeDevice - case 'p': - mode |= os.ModeNamedPipe - case 'S': - mode |= os.ModeSocket - case 'u': - mode |= os.ModeSetuid - case 'g': - mode |= os.ModeSetgid - case 'c': - mode |= os.ModeCharDevice - case 't': - mode |= os.ModeSticky - default: - return 0, fmt.Errorf("unsupported mode: '%v'", c) - } - } - return mode, nil -} - -func parseFilePermissions(perm string) (os.FileMode, error) { - s, err := strconv.ParseUint(perm, 8, 32) - if err != nil { - return 0, err - } - return os.FileMode(s), nil -} - type directoryWriter struct { - io.Closer - - writer io.Writer - buf []byte - separator []byte - - lastNameWritten string + w *internal.ProtoStreamWriter } func (dw *directoryWriter) WriteEntry(e *EntryMetadata, children []*EntryMetadata) error { - if dw.lastNameWritten != "" { - if isLessOrEqual(e.Name, dw.lastNameWritten) { - return fmt.Errorf("out-of-order directory entry, previous '%v' current '%v'", dw.lastNameWritten, e.Name) - } - dw.lastNameWritten = e.Name - } - - jde := e.toJSONEntry() - - if len(children) > 0 { - jde.SubEntries = make([]jsonDirectoryEntry, len(children)) - for i, se := range children { - jde.SubEntries[i] = se.toJSONEntry() - } - } - - v, _ := json.Marshal(&jde) - - dw.writer.Write(dw.separator) - dw.writer.Write(v) - dw.separator = []byte(",") - - return nil + return dw.w.Write(e) } -func (em *EntryMetadata) toJSONEntry() jsonDirectoryEntry { - var jde jsonDirectoryEntry - jde.Name = em.Name - - if (em.FileMode & os.ModeType) != 0 { - jde.Type = formatMode(em.FileMode) - } - - if (em.FileMode & os.ModePerm) != 0 { - jde.Permissions = formatPermissions(em.FileMode) - } - - if em.OwnerID > 0 || em.GroupID > 0 { - jde.Owner = fmt.Sprintf("%d:%d", em.OwnerID, em.GroupID) - } - - if !em.ModTime.IsZero() { - utc := em.ModTime.UTC() - jde.Time = &utc - } - - if em.ObjectID != "" { - inline := em.ObjectID.InlineData() - if len(inline) >= 2 && inline[0] == '{' && inline[len(inline)-1] == '}' { - m := map[string]interface{}{} - - if json.Unmarshal(inline, &m) == nil { - jde.JSONContent = json.RawMessage(inline) - } - } - } - - if jde.JSONContent == nil { - jde.ObjectID = string(em.ObjectID) - } - - if em.FileMode.IsRegular() { - jde.Size = strconv.FormatInt(em.FileSize, 10) - } - - return jde -} - -func formatMode(m os.FileMode) string { - const str = "dalTLDpSugct" - var buf [32]byte - w := 0 - for i, c := range str { - if m&(1< 0 { - bundle := make([]*EntryMetadata, 0, len(e.SubEntries)) + for _, e := range source { + if len(e.BundledChildren) > 0 { + bundle := e.BundledChildren + e.BundledChildren = nil var currentOffset int64 - var bundleEntry EntryMetadata - if err := bundleEntry.fromJSON(&e); err != nil { - return nil, err + for _, child := range bundle { + id := repo.NewSectionObjectID(currentOffset, child.FileSize, e.ObjectId) + child.ObjectId = &id + currentOffset += child.FileSize } - for _, s := range e.SubEntries { - var subEntry EntryMetadata - if err := subEntry.fromJSON(&s); err != nil { - return nil, err - } - subEntry.ObjectID = repo.NewSectionObjectID(currentOffset, subEntry.FileSize, entryMetadata.ObjectID) - currentOffset += subEntry.FileSize - bundle = append(bundle, &subEntry) - } - - if currentOffset != entryMetadata.FileSize { - return nil, fmt.Errorf("inconsistent size of '%v': %v (got %v)", entryMetadata.Name, entryMetadata.FileSize, currentOffset) + if currentOffset != e.FileSize { + return nil, fmt.Errorf("inconsistent size of '%v': %v (got %v)", e.Name, e.FileSize, currentOffset) } bundles = append(bundles, bundle) } else { - entries = append(entries, &entryMetadata) + entries = append(entries, e) } } diff --git a/fs/dir_test.go b/fs/dir_test.go index fc89e958c..207e8f587 100644 --- a/fs/dir_test.go +++ b/fs/dir_test.go @@ -1,11 +1,12 @@ package fs import ( - "os" "reflect" "strings" "testing" "time" + + "github.com/kopia/kopia/repo" ) var ( @@ -14,147 +15,143 @@ var ( time3 = mustParseTimestamp("2016-04-02T02:36:19Z") ) -func TestEmptyDirectory(t *testing.T) { - data := strings.Join( - []string{ - `{`, - `"format":{"version":1},`, - `"entries":[]`, - `}`, - }, "") - - expectedEntries := []*EntryMetadata{} - - verifyDirectory(t, data, expectedEntries) -} - -func TestDirectoryWithOnlyBundle(t *testing.T) { - data := strings.Join( - []string{ - `{`, - `"format":{"version":1},`, - `"entries":[`, - `{"name":"bundle1","size":"170","perm":"0","oid":"D5555","entries":[`, - `{"name":"a1","perm":"500","mtime":"2016-04-06T02:34:10Z","owner":"500:100","size":"50"},`, - `{"name":"z1","perm":"500","mtime":"2016-04-06T02:34:10Z","owner":"500:100","size":"120"}`, - `]}`, - `]}`, - }, "") - - expectedEntries := []*EntryMetadata{ - &EntryMetadata{Name: "a1", FileMode: 0500, ModTime: time1, OwnerID: 500, GroupID: 100, FileSize: 50, ObjectID: "S0,50,D5555"}, - &EntryMetadata{Name: "z1", FileMode: 0500, ModTime: time1, OwnerID: 500, GroupID: 100, FileSize: 120, ObjectID: "S50,120,D5555"}, +func TestFlattenBundles(t *testing.T) { + base := &repo.ObjectID{StorageBlock: "5555"} + sources := []*EntryMetadata{ + &EntryMetadata{ + Name: "bundle1", + FileSize: 170, + ObjectId: base, + BundledChildren: []*EntryMetadata{ + &EntryMetadata{Name: "a1", FileSize: 50}, + &EntryMetadata{Name: "z1", FileSize: 120}, + }, + }, } - verifyDirectory(t, data, expectedEntries) -} - -func TestInconsistentBundleSize(t *testing.T) { - data := strings.Join( - []string{ - `{`, - `"format":{"version":1},`, - `"entries":[`, - `{"name":"bundle1","size":"170","perm":"0","oid":"D5555","entries":[`, - `{"name":"a1","perm":"500","mtime":"2016-04-06T02:34:10Z","owner":"500:100","size":"51"},`, - `{"name":"z1","perm":"500","mtime":"2016-04-06T02:34:10Z","owner":"500:100","size":"120"}`, - `]}`, - `]}`, - }, "") - verifyDirectoryError(t, data, "inconsistent size of 'bundle1': 170 (got 171)") -} - -func TestInvalidBundleHeaderData(t *testing.T) { - data := strings.Join( - []string{ - `{`, - `"format":{"version":1},`, - `"entries":[`, - `{"name":"bundle1","size":"170","perm":"x","oid":"D5555","entries":[`, - `{"name":"z1","perm":"500","mtime":"2016-04-06T02:34:10Z","owner":"500:100","size":"120"}`, - `]}`, - `]}`, - }, "") - verifyDirectoryError(t, data, "invalid permissions: 'x'") -} - -func TestInvalidBundleEntryData(t *testing.T) { - data := strings.Join( - []string{ - `{`, - `"format":{"version":1},`, - `"entries":[`, - `{"name":"bundle1","size":"170","perm":"0","oid":"D5555","entries":[`, - `{"perm":"500","mtime":"2016-04-06T02:34:10Z","owner":"500:100","size":"51"},`, - `{"name":"z1","perm":"500","mtime":"2016-04-06T02:34:10Z","owner":"500:100","size":"120"}`, - `]}`, - `]}`, - }, "") - verifyDirectoryError(t, data, "empty entry name") -} -func TestDirectoryWithoutBundle(t *testing.T) { - data := strings.Join( - []string{ - `{`, - `"format":{"version":1},`, - `"entries":[`, - `{"name":"constants.go","perm":"420","size":"13","mtime":"2016-04-02T02:36:19Z","owner":"500:100","oid":"D5123"}`, - `]}`, - }, "") - - expectedEntries := []*EntryMetadata{ - &EntryMetadata{Name: "constants.go", FileMode: 0420, ModTime: time3, OwnerID: 500, GroupID: 100, FileSize: 13, ObjectID: "D5123"}, - } - - verifyDirectory(t, data, expectedEntries) -} - -func TestDirectoryWithThreeBundles(t *testing.T) { - data := strings.Join( - []string{ - `{`, - `"format":{"version":1},`, - `"entries":[`, - `{"name":"bundle1","size":"170","perm":"0","oid":"D5555","entries":[`, - `{"name":"a1","perm":"500","mtime":"2016-04-06T02:34:10Z","owner":"500:100","size":"50"},`, - `{"name":"z1","perm":"500","mtime":"2016-04-06T02:34:10Z","owner":"500:100","size":"120"}`, - `]},`, - `{"name":"config.go","perm":"420","size":"937","mtime":"2016-04-02T02:39:44.123456789Z","owner":"500:100","oid":"D4321"},`, - `{"name":"constants.go","perm":"420","size":"13","mtime":"2016-04-02T02:36:19Z","owner":"500:100","oid":"D5123"},`, - `{"name":"subdir","type":"d","perm":"755","mtime":"2016-04-06T02:34:10Z","owner":"500:100","oid":"D1234"},`, - `{"name":"bundle3","size":"7","perm":"0","oid":"D8888","entries":[`, - `{"name":"a3","perm":"500","mtime":"2016-04-06T02:34:10Z","owner":"500:100","size":"5"},`, - `{"name":"z3","perm":"500","mtime":"2016-04-06T02:34:10Z","owner":"500:100","size":"2"}`, - `]},`, - `{"name":"bundle2","size":"170","perm":"0","oid":"D6666","entries":[`, - `{"name":"a2","perm":"500","mtime":"2016-04-06T02:34:10Z","owner":"500:100","size":"150"},`, - `{"name":"z2","perm":"500","mtime":"2016-04-06T02:34:10Z","owner":"500:100","size":"20"}`, - `]}`, - `]}`, - }, "") - - expectedEntries := []*EntryMetadata{ - &EntryMetadata{Name: "a1", FileMode: 0500, ModTime: time1, OwnerID: 500, GroupID: 100, FileSize: 50, ObjectID: "S0,50,D5555"}, - &EntryMetadata{Name: "a2", FileMode: 0500, ModTime: time1, OwnerID: 500, GroupID: 100, FileSize: 150, ObjectID: "S0,150,D6666"}, - &EntryMetadata{Name: "a3", FileMode: 0500, ModTime: time1, OwnerID: 500, GroupID: 100, FileSize: 5, ObjectID: "S0,5,D8888"}, - &EntryMetadata{Name: "config.go", FileMode: 0420, ModTime: time2, OwnerID: 500, GroupID: 100, FileSize: 937, ObjectID: "D4321"}, - &EntryMetadata{Name: "constants.go", FileMode: 0420, ModTime: time3, OwnerID: 500, GroupID: 100, FileSize: 13, ObjectID: "D5123"}, - &EntryMetadata{Name: "subdir", FileMode: os.ModeDir | 0755, ModTime: time1, OwnerID: 500, GroupID: 100, ObjectID: "D1234"}, - &EntryMetadata{Name: "z1", FileMode: 0500, ModTime: time1, OwnerID: 500, GroupID: 100, FileSize: 120, ObjectID: "S50,120,D5555"}, - &EntryMetadata{Name: "z2", FileMode: 0500, ModTime: time1, OwnerID: 500, GroupID: 100, FileSize: 20, ObjectID: "S150,20,D6666"}, - &EntryMetadata{Name: "z3", FileMode: 0500, ModTime: time1, OwnerID: 500, GroupID: 100, FileSize: 2, ObjectID: "S5,2,D8888"}, - } - - verifyDirectory(t, data, expectedEntries) -} - -func verifyDirectory(t *testing.T, data string, expectedEntries []*EntryMetadata) { - entries, err := readDirectoryMetadataEntries(strings.NewReader(data)) + entries, err := flattenBundles(sources) if err != nil { t.Errorf("can't read directory entries: %v", err) return } + expectedEntries := []*EntryMetadata{ + &EntryMetadata{Name: "a1", FileSize: 50, ObjectId: &repo.ObjectID{Section: &repo.ObjectID_Section{ + Start: 0, + Length: 50, + Base: base, + }}}, + &EntryMetadata{Name: "z1", FileSize: 120, ObjectId: &repo.ObjectID{Section: &repo.ObjectID_Section{ + Start: 50, + Length: 120, + Base: base, + }}}, + } + + verifyDirectory(t, entries, expectedEntries) +} + +func TestFlattenBundlesInconsistentBundleSize(t *testing.T) { + sources := []*EntryMetadata{ + &EntryMetadata{ + Name: "bundle1", + FileSize: 171, + ObjectId: &repo.ObjectID{StorageBlock: "5555"}, + BundledChildren: []*EntryMetadata{ + &EntryMetadata{Name: "a1", FileSize: 50}, + &EntryMetadata{Name: "z1", FileSize: 120}, + }, + }, + } + + _, err := flattenBundles(sources) + if err == nil { + t.Errorf("expected error") + return + } + + if ok := strings.Contains(err.Error(), "inconsistent size of 'bundle1'"); !ok { + t.Errorf("invalid error: %v", err) + } +} + +func TestFlattenThreeBundles(t *testing.T) { + base1 := &repo.ObjectID{StorageBlock: "5555"} + base2 := &repo.ObjectID{StorageBlock: "6666"} + base3 := &repo.ObjectID{StorageBlock: "7777"} + sources := []*EntryMetadata{ + &EntryMetadata{ + Name: "bundle1", + FileSize: 170, + ObjectId: base1, + BundledChildren: []*EntryMetadata{ + &EntryMetadata{Name: "a1", FileSize: 50}, + &EntryMetadata{Name: "z1", FileSize: 120}, + }, + }, + &EntryMetadata{ + Name: "bundle3", + FileSize: 7, + ObjectId: base3, + BundledChildren: []*EntryMetadata{ + &EntryMetadata{Name: "a3", FileSize: 5}, + &EntryMetadata{Name: "z3", FileSize: 2}, + }, + }, + &EntryMetadata{ + Name: "bundle2", + FileSize: 300, + ObjectId: base2, + BundledChildren: []*EntryMetadata{ + &EntryMetadata{Name: "a2", FileSize: 100}, + &EntryMetadata{Name: "z2", FileSize: 200}, + }, + }, + } + + entries, err := flattenBundles(sources) + if err != nil { + t.Errorf("can't read directory entries: %v", err) + return + } + + expectedEntries := []*EntryMetadata{ + &EntryMetadata{Name: "a1", FileSize: 50, ObjectId: &repo.ObjectID{Section: &repo.ObjectID_Section{ + Start: 0, + Length: 50, + Base: base1, + }}}, + &EntryMetadata{Name: "a2", FileSize: 100, ObjectId: &repo.ObjectID{Section: &repo.ObjectID_Section{ + Start: 0, + Length: 100, + Base: base2, + }}}, + &EntryMetadata{Name: "a3", FileSize: 5, ObjectId: &repo.ObjectID{Section: &repo.ObjectID_Section{ + Start: 0, + Length: 5, + Base: base3, + }}}, + &EntryMetadata{Name: "z1", FileSize: 120, ObjectId: &repo.ObjectID{Section: &repo.ObjectID_Section{ + Start: 50, + Length: 120, + Base: base1, + }}}, + &EntryMetadata{Name: "z2", FileSize: 200, ObjectId: &repo.ObjectID{Section: &repo.ObjectID_Section{ + Start: 100, + Length: 200, + Base: base2, + }}}, + &EntryMetadata{Name: "z3", FileSize: 2, ObjectId: &repo.ObjectID{Section: &repo.ObjectID_Section{ + Start: 5, + Length: 2, + Base: base3, + }}}, + } + + verifyDirectory(t, entries, expectedEntries) +} + +func verifyDirectory(t *testing.T, entries []*EntryMetadata, expectedEntries []*EntryMetadata) { if len(entries) != len(expectedEntries) { t.Errorf("expected %v entries, got %v", len(expectedEntries), len(entries)) } @@ -164,34 +161,13 @@ func verifyDirectory(t *testing.T, data string, expectedEntries []*EntryMetadata actual := entries[i] if !reflect.DeepEqual(expected, actual) { - t.Errorf("invalid entry at index %v:\nexpected: %#v\nactual: %#v", i, expected, actual) + t.Errorf("invalid entry at index %v:\nexpected: %v\nactual: %v", i, + expected.String(), actual.String()) } } } } -func TestInvalidJSON(t *testing.T) { - verifyDirectoryError(t, "{invalid", "invalid character 'i'") - verifyDirectoryError(t, `{"format":{"version":1},"entries":[{"name":""}]}`, "empty entry name") - verifyDirectoryError(t, `{"format":{"version":1},"entries":[{"name":"x","perm":"x123"}]}`, "invalid permissions: 'x123'") - verifyDirectoryError(t, `{"format":{"version":2},"entries":[]}`, "unsupported version: 2") -} - -func verifyDirectoryError(t *testing.T, data, expectedError string) { - entries, err := readDirectoryMetadataEntries(strings.NewReader(data)) - if err == nil { - t.Errorf("expected error %v, got no error", expectedError) - } else { - if !strings.Contains(err.Error(), expectedError) { - t.Errorf("expected error containing '%v', got '%v'", expectedError, err.Error()) - } - } - - if entries != nil { - t.Errorf("got unexpected result: %#v", entries) - } -} - func TestDirectoryNameOrder(t *testing.T) { sortedNames := []string{ "a/a/a", diff --git a/fs/entry.go b/fs/entry.go index 506a22714..5753efacd 100644 --- a/fs/entry.go +++ b/fs/entry.go @@ -8,8 +8,6 @@ import ( "sort" "strings" "time" - - "github.com/kopia/kopia/repo" ) // Entry represents a filesystem entry, which can be Directory, File, or Symlink @@ -45,17 +43,6 @@ type Symlink interface { Readlink() (string, error) } -// EntryMetadata stores attributes of a single entry in a directory. -type EntryMetadata struct { - Name string - FileMode os.FileMode - FileSize int64 - ModTime time.Time - OwnerID uint32 - GroupID uint32 - ObjectID repo.ObjectID -} - type entry struct { parent Directory metadata *EntryMetadata @@ -133,15 +120,24 @@ func isLessOrEqual(name1, name2 string) bool { return len(parts1) <= len(parts2) } +// FileMode returns the file mode and permissions. +func (e *EntryMetadata) FileMode() os.FileMode { + return os.FileMode(e.ModeAndPermissions) +} + +// ModTime returns the modification time. +func (e *EntryMetadata) ModTime() time.Time { + return time.Unix(e.ModTimestampNano/1000000000, e.ModTimestampNano%1000000000) + +} + func (e *EntryMetadata) metadataHash() uint64 { h := fnv.New64a() - binary.Write(h, binary.LittleEndian, e.ModTime.UnixNano()) - binary.Write(h, binary.LittleEndian, e.FileMode) - if e.FileMode.IsRegular() { - binary.Write(h, binary.LittleEndian, e.FileSize) - } - binary.Write(h, binary.LittleEndian, e.OwnerID) - binary.Write(h, binary.LittleEndian, e.GroupID) + binary.Write(h, binary.LittleEndian, e.ModTimestampNano) + binary.Write(h, binary.LittleEndian, e.ModeAndPermissions) + binary.Write(h, binary.LittleEndian, e.FileSize) + binary.Write(h, binary.LittleEndian, e.Uid) + binary.Write(h, binary.LittleEndian, e.Gid) return h.Sum64() } diff --git a/fs/entry_metadata.pb.go b/fs/entry_metadata.pb.go new file mode 100644 index 000000000..c9982d194 --- /dev/null +++ b/fs/entry_metadata.pb.go @@ -0,0 +1,89 @@ +// Code generated by protoc-gen-go. +// source: fs/entry_metadata.proto +// DO NOT EDIT! + +/* +Package fs is a generated protocol buffer package. + +It is generated from these files: + fs/entry_metadata.proto + fs/hashcache.proto + +It has these top-level messages: + EntryMetadata + HashCacheEntry +*/ +package fs + +import proto "github.com/golang/protobuf/proto" +import fmt "fmt" +import math "math" +import kopia_repo "github.com/kopia/kopia/repo" + +// Reference imports to suppress errors if they are not otherwise used. +var _ = proto.Marshal +var _ = fmt.Errorf +var _ = math.Inf + +// This is a compile-time assertion to ensure that this generated file +// is compatible with the proto package it is being compiled against. +const _ = proto.ProtoPackageIsVersion1 + +// EntryMetadata stores attributes of a single entry in a directory. +type EntryMetadata struct { + Name string `protobuf:"bytes,1,opt,name=name" json:"name,omitempty"` + ModeAndPermissions uint32 `protobuf:"fixed32,2,opt,name=mode_and_permissions,json=modeAndPermissions" json:"mode_and_permissions,omitempty"` + FileSize int64 `protobuf:"varint,3,opt,name=file_size,json=fileSize" json:"file_size,omitempty"` + ModTimestampNano int64 `protobuf:"fixed64,4,opt,name=mod_timestamp_nano,json=modTimestampNano" json:"mod_timestamp_nano,omitempty"` + ObjectId *kopia_repo.ObjectID `protobuf:"bytes,5,opt,name=object_id,json=objectId" json:"object_id,omitempty"` + Uid uint32 `protobuf:"varint,6,opt,name=uid" json:"uid,omitempty"` + Gid uint32 `protobuf:"varint,7,opt,name=gid" json:"gid,omitempty"` + BundledChildren []*EntryMetadata `protobuf:"bytes,8,rep,name=bundled_children,json=bundledChildren" json:"bundled_children,omitempty"` +} + +func (m *EntryMetadata) Reset() { *m = EntryMetadata{} } +func (m *EntryMetadata) String() string { return proto.CompactTextString(m) } +func (*EntryMetadata) ProtoMessage() {} +func (*EntryMetadata) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{0} } + +func (m *EntryMetadata) GetObjectId() *kopia_repo.ObjectID { + if m != nil { + return m.ObjectId + } + return nil +} + +func (m *EntryMetadata) GetBundledChildren() []*EntryMetadata { + if m != nil { + return m.BundledChildren + } + return nil +} + +func init() { + proto.RegisterType((*EntryMetadata)(nil), "kopia.fs.EntryMetadata") +} + +var fileDescriptor0 = []byte{ + // 306 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0x54, 0x90, 0xcd, 0x4e, 0x02, 0x31, + 0x14, 0x85, 0x03, 0x83, 0x30, 0xd4, 0x10, 0x27, 0x95, 0x84, 0x2a, 0x1b, 0xe2, 0xca, 0x85, 0x29, + 0xfe, 0x3c, 0x81, 0xa8, 0x0b, 0x16, 0xfe, 0xa4, 0xba, 0x72, 0xd3, 0x74, 0x68, 0x07, 0xaa, 0xb4, + 0x9d, 0x4c, 0xcb, 0x42, 0x5f, 0xcd, 0x97, 0xf3, 0x76, 0x66, 0x88, 0x71, 0xd3, 0xdc, 0x9e, 0xf3, + 0x9d, 0xa6, 0xf7, 0xa0, 0x49, 0xe1, 0xe7, 0xca, 0x86, 0xea, 0x8b, 0x1b, 0x15, 0x84, 0x14, 0x41, + 0xd0, 0xb2, 0x72, 0xc1, 0xe1, 0xf4, 0xd3, 0x95, 0x5a, 0xd0, 0xc2, 0x9f, 0x1e, 0x57, 0xaa, 0x74, + 0x73, 0x97, 0x7f, 0xa8, 0x55, 0xd0, 0xb2, 0xb1, 0xcf, 0x7e, 0xba, 0x68, 0xf4, 0x10, 0x73, 0x8f, + 0x6d, 0x0c, 0x63, 0xd4, 0xb3, 0xc2, 0x28, 0xd2, 0x99, 0x75, 0xce, 0x87, 0xac, 0x9e, 0xf1, 0x25, + 0x1a, 0x1b, 0x27, 0x15, 0x17, 0x56, 0xf2, 0x52, 0x55, 0x46, 0x7b, 0xaf, 0x9d, 0xf5, 0xa4, 0x0b, + 0xcc, 0x80, 0xe1, 0xe8, 0xdd, 0x5a, 0xf9, 0xf2, 0xe7, 0xe0, 0x29, 0x1a, 0x16, 0x7a, 0xab, 0xb8, + 0xd7, 0xdf, 0x8a, 0x24, 0x80, 0x25, 0x2c, 0x8d, 0xc2, 0x2b, 0xdc, 0xf1, 0x05, 0x8a, 0x11, 0x1e, + 0xb4, 0x51, 0x3e, 0x08, 0x53, 0x72, 0x2b, 0xac, 0x23, 0x3d, 0xa0, 0x32, 0x96, 0x81, 0xf3, 0xb6, + 0x37, 0x9e, 0x40, 0xc7, 0x57, 0x68, 0xd8, 0x7c, 0x9a, 0x6b, 0x49, 0x0e, 0x00, 0x3a, 0xbc, 0x1e, + 0xd3, 0x66, 0xab, 0xb8, 0x11, 0x7d, 0xae, 0xcd, 0xe5, 0x3d, 0x4b, 0x1b, 0x6c, 0x29, 0x71, 0x86, + 0x92, 0x1d, 0xc0, 0x7d, 0x80, 0x47, 0x2c, 0x8e, 0x51, 0x59, 0x83, 0x32, 0x68, 0x14, 0x18, 0xf1, + 0x02, 0x65, 0xf9, 0xce, 0xca, 0xad, 0x92, 0x7c, 0xb5, 0xd1, 0x5b, 0x59, 0x29, 0x4b, 0xd2, 0x59, + 0x02, 0xaf, 0x4f, 0xe8, 0xbe, 0x33, 0xfa, 0xaf, 0x1a, 0x76, 0xd4, 0x06, 0xee, 0x5a, 0x7e, 0x31, + 0x7d, 0x3f, 0x59, 0xeb, 0xb0, 0xd9, 0xe5, 0x74, 0xe5, 0xcc, 0xbc, 0x4e, 0xb5, 0x67, 0xe1, 0xf3, + 0x7e, 0xdd, 0xf0, 0xcd, 0x6f, 0x00, 0x00, 0x00, 0xff, 0xff, 0xcf, 0xf4, 0xc2, 0x88, 0x9b, 0x01, + 0x00, 0x00, +} diff --git a/fs/entry_metadata.proto b/fs/entry_metadata.proto new file mode 100644 index 000000000..7d8b49467 --- /dev/null +++ b/fs/entry_metadata.proto @@ -0,0 +1,19 @@ +syntax = "proto3"; + +package kopia.fs; +option go_package = "github.com/kopia/kopia/fs"; + +import "repo/objectid.proto"; + +// EntryMetadata stores attributes of a single entry in a directory. +message EntryMetadata { + string name = 1; + fixed32 mode_and_permissions = 2; + int64 file_size = 3; + sfixed64 mod_timestamp_nano = 4; + repo.ObjectID object_id = 5; + uint32 uid = 6; + uint32 gid = 7; + + repeated EntryMetadata bundled_children = 8; +} diff --git a/fs/hashcache.go b/fs/hashcache.go index f85ae5933..52ff66aa3 100644 --- a/fs/hashcache.go +++ b/fs/hashcache.go @@ -4,36 +4,28 @@ import ( "bufio" "fmt" "io" - "strconv" - "strings" - "github.com/kopia/kopia/repo" + "github.com/kopia/kopia/internal" ) -type hashCacheEntry struct { - Name string - Hash uint64 - ObjectID repo.ObjectID -} - type hashcacheReader struct { - scanner *bufio.Scanner - nextEntry *hashCacheEntry - entry0 hashCacheEntry - entry1 hashCacheEntry + reader *internal.ProtoStreamReader + nextEntry *HashCacheEntry + entry0 HashCacheEntry + entry1 HashCacheEntry odd bool first bool } func (hcr *hashcacheReader) open(r io.Reader) error { - hcr.scanner = bufio.NewScanner(r) + hcr.reader = internal.NewProtoStreamReader(bufio.NewReader(r), internal.ProtoStreamTypeHashCache) hcr.nextEntry = nil hcr.first = true hcr.readahead() return nil } -func (hcr *hashcacheReader) findEntry(relativeName string) *hashCacheEntry { +func (hcr *hashcacheReader) findEntry(relativeName string) *HashCacheEntry { for hcr.nextEntry != nil && isLess(hcr.nextEntry.Name, relativeName) { hcr.readahead() } @@ -49,37 +41,20 @@ func (hcr *hashcacheReader) findEntry(relativeName string) *hashCacheEntry { } func (hcr *hashcacheReader) readahead() { - if hcr.scanner != nil { - if hcr.first { - hcr.first = false - if !hcr.scanner.Scan() || hcr.scanner.Text() != "HASHCACHE:v1" { - hcr.scanner = nil - return - } - } - + if hcr.reader != nil { hcr.nextEntry = nil - if hcr.scanner.Scan() { - var err error - e := hcr.nextManifestEntry() - t := hcr.scanner.Text() - p0 := strings.IndexByte(t, '|') - p1 := strings.LastIndexByte(t, '|') - e.Name = t[0:p0] - e.Hash, err = strconv.ParseUint(t[p0+1:p1], 0, 64) - e.ObjectID = repo.ObjectID(t[p1+1:]) - if err == nil { - hcr.nextEntry = e - } + e := hcr.nextManifestEntry() + if err := hcr.reader.Read(e); err == nil { + hcr.nextEntry = e } } if hcr.nextEntry == nil { - hcr.scanner = nil + hcr.reader = nil } } -func (hcr *hashcacheReader) nextManifestEntry() *hashCacheEntry { +func (hcr *hashcacheReader) nextManifestEntry() *HashCacheEntry { hcr.odd = !hcr.odd if hcr.odd { return &hcr.entry1 @@ -89,19 +64,18 @@ func (hcr *hashcacheReader) nextManifestEntry() *hashCacheEntry { } type hashcacheWriter struct { - writer io.Writer + writer *internal.ProtoStreamWriter lastNameWritten string } func newHashCacheWriter(w io.Writer) *hashcacheWriter { hcw := &hashcacheWriter{ - writer: w, + writer: internal.NewProtoStreamWriter(w, internal.ProtoStreamTypeHashCache), } - io.WriteString(w, "HASHCACHE:v1\n") return hcw } -func (hcw *hashcacheWriter) WriteEntry(e hashCacheEntry) error { +func (hcw *hashcacheWriter) WriteEntry(e HashCacheEntry) error { if hcw.lastNameWritten != "" { if isLessOrEqual(e.Name, hcw.lastNameWritten) { return fmt.Errorf("out-of-order directory entry, previous '%v' current '%v'", hcw.lastNameWritten, e.Name) @@ -109,7 +83,7 @@ func (hcw *hashcacheWriter) WriteEntry(e hashCacheEntry) error { hcw.lastNameWritten = e.Name } - fmt.Fprintf(hcw.writer, "%v|%v|%v\n", e.Name, e.Hash, e.ObjectID) + hcw.writer.Write(&e) return nil } diff --git a/fs/hashcache.pb.go b/fs/hashcache.pb.go new file mode 100644 index 000000000..d94cfbcd6 --- /dev/null +++ b/fs/hashcache.pb.go @@ -0,0 +1,52 @@ +// Code generated by protoc-gen-go. +// source: fs/hashcache.proto +// DO NOT EDIT! + +package fs + +import proto "github.com/golang/protobuf/proto" +import fmt "fmt" +import math "math" +import kopia_repo "github.com/kopia/kopia/repo" + +// Reference imports to suppress errors if they are not otherwise used. +var _ = proto.Marshal +var _ = fmt.Errorf +var _ = math.Inf + +type HashCacheEntry struct { + Name string `protobuf:"bytes,1,opt,name=name" json:"name,omitempty"` + Hash uint64 `protobuf:"fixed64,2,opt,name=hash" json:"hash,omitempty"` + ObjectId *kopia_repo.ObjectID `protobuf:"bytes,3,opt,name=object_id,json=objectId" json:"object_id,omitempty"` +} + +func (m *HashCacheEntry) Reset() { *m = HashCacheEntry{} } +func (m *HashCacheEntry) String() string { return proto.CompactTextString(m) } +func (*HashCacheEntry) ProtoMessage() {} +func (*HashCacheEntry) Descriptor() ([]byte, []int) { return fileDescriptor1, []int{0} } + +func (m *HashCacheEntry) GetObjectId() *kopia_repo.ObjectID { + if m != nil { + return m.ObjectId + } + return nil +} + +func init() { + proto.RegisterType((*HashCacheEntry)(nil), "kopia.fs.HashCacheEntry") +} + +var fileDescriptor1 = []byte{ + // 173 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0xe2, 0x12, 0x4a, 0x2b, 0xd6, 0xcf, + 0x48, 0x2c, 0xce, 0x48, 0x4e, 0x4c, 0xce, 0x48, 0xd5, 0x2b, 0x28, 0xca, 0x2f, 0xc9, 0x17, 0xe2, + 0xc8, 0xce, 0x2f, 0xc8, 0x4c, 0xd4, 0x4b, 0x2b, 0x96, 0x12, 0x2e, 0x4a, 0x2d, 0xc8, 0xd7, 0xcf, + 0x4f, 0xca, 0x4a, 0x4d, 0x2e, 0xc9, 0x4c, 0x81, 0x48, 0x2b, 0x65, 0x73, 0xf1, 0x79, 0x00, 0x75, + 0x38, 0x83, 0x74, 0xb8, 0xe6, 0x95, 0x14, 0x55, 0x0a, 0x09, 0x71, 0xb1, 0xe4, 0x25, 0xe6, 0xa6, + 0x4a, 0x30, 0x2a, 0x30, 0x6a, 0x70, 0x06, 0x81, 0xd9, 0x20, 0x31, 0x90, 0xb9, 0x12, 0x4c, 0x40, + 0x31, 0xb6, 0x20, 0x30, 0x5b, 0xc8, 0x90, 0x8b, 0x13, 0x62, 0x56, 0x7c, 0x66, 0x8a, 0x04, 0x33, + 0x50, 0x82, 0xdb, 0x48, 0x44, 0x0f, 0x62, 0x19, 0xc8, 0x22, 0x3d, 0x7f, 0xb0, 0xa4, 0xa7, 0x4b, + 0x10, 0x07, 0x44, 0x99, 0x67, 0x8a, 0x93, 0x74, 0x94, 0x64, 0x7a, 0x66, 0x49, 0x46, 0x69, 0x92, + 0x5e, 0x72, 0x7e, 0xae, 0x3e, 0x58, 0x2d, 0x94, 0x4c, 0x2b, 0x4e, 0x62, 0x03, 0x3b, 0xc8, 0x18, + 0x10, 0x00, 0x00, 0xff, 0xff, 0xa7, 0x08, 0x96, 0x2b, 0xc5, 0x00, 0x00, 0x00, +} diff --git a/fs/hashcache.proto b/fs/hashcache.proto new file mode 100644 index 000000000..8c88722c3 --- /dev/null +++ b/fs/hashcache.proto @@ -0,0 +1,12 @@ +syntax = "proto3"; + +package kopia.fs; +option go_package = "github.com/kopia/kopia/fs"; + +import "repo/objectid.proto"; + +message HashCacheEntry { + string name = 1; + fixed64 hash = 2; + repo.ObjectID object_id = 3; +} diff --git a/fs/inmemoryfs_for_test.go b/fs/inmemory_fs_for_test.go similarity index 94% rename from fs/inmemoryfs_for_test.go rename to fs/inmemory_fs_for_test.go index 781305dba..c8727b435 100644 --- a/fs/inmemoryfs_for_test.go +++ b/fs/inmemory_fs_for_test.go @@ -35,8 +35,8 @@ func (imd *inmemoryDirectory) addFile(name string, content []byte, mode os.FileM inmemoryEntry: inmemoryEntry{ parent: imd, metadata: &EntryMetadata{ - Name: name, - FileMode: mode, + Name: name, + ModeAndPermissions: uint32(mode), }, }, source: bytes.NewBuffer(content), @@ -54,8 +54,8 @@ func (imd *inmemoryDirectory) addDir(name string, mode os.FileMode) *inmemoryDir inmemoryEntry: inmemoryEntry{ parent: imd, metadata: &EntryMetadata{ - Name: name, - FileMode: mode | os.ModeDir, + Name: name, + ModeAndPermissions: uint32(mode | os.ModeDir), }, }, } @@ -88,7 +88,7 @@ func (imd *inmemoryDirectory) subdir(name ...string) *inmemoryDirectory { if i2 == nil { panic(fmt.Sprintf("'%s' not found in '%s'", n, i.metadata.Name)) } - if !i2.Metadata().FileMode.IsDir() { + if !i2.Metadata().FileMode().IsDir() { panic(fmt.Sprintf("'%s' is not a directory in '%s'", n, i.metadata.Name)) } diff --git a/fs/filesystem.go b/fs/local_fs.go similarity index 95% rename from fs/filesystem.go rename to fs/local_fs.go index 03ba751e8..23253095a 100644 --- a/fs/filesystem.go +++ b/fs/local_fs.go @@ -107,9 +107,9 @@ func NewFilesystemDirectory(path string, parent Directory) (Directory, error) { func entryMetadataFromFileInfo(fi os.FileInfo) *EntryMetadata { e := &EntryMetadata{ - Name: filepath.Base(fi.Name()), - FileMode: fi.Mode(), - ModTime: fi.ModTime(), + Name: filepath.Base(fi.Name()), + ModeAndPermissions: uint32(fi.Mode()), + ModTimestampNano: fi.ModTime().UnixNano(), } if fi.Mode().IsRegular() { diff --git a/fs/filesystem_nonwindows.go b/fs/local_fs_nonwindows.go similarity index 82% rename from fs/filesystem_nonwindows.go rename to fs/local_fs_nonwindows.go index 4cbb4e8da..d0dd17798 100644 --- a/fs/filesystem_nonwindows.go +++ b/fs/local_fs_nonwindows.go @@ -9,8 +9,8 @@ import ( func (e *EntryMetadata) populatePlatformSpecificEntryDetails(fi os.FileInfo) error { if stat, ok := fi.Sys().(*syscall.Stat_t); ok { - e.OwnerID = stat.Uid - e.GroupID = stat.Gid + e.Uid = stat.Uid + e.Gid = stat.Gid } return nil diff --git a/fs/filesystem_test.go b/fs/local_fs_test.go similarity index 86% rename from fs/filesystem_test.go rename to fs/local_fs_test.go index 1e4ccaa72..645bf7c1a 100644 --- a/fs/filesystem_test.go +++ b/fs/local_fs_test.go @@ -64,19 +64,19 @@ func TestFiles(t *testing.T) { goodCount := 0 - if entries[0].Metadata().Name == "f1" && entries[0].Metadata().FileSize == 5 && entries[0].Metadata().FileMode.IsRegular() { + if entries[0].Metadata().Name == "f1" && entries[0].Metadata().FileSize == 5 && entries[0].Metadata().FileMode().IsRegular() { goodCount++ } - if entries[1].Metadata().Name == "f2" && entries[1].Metadata().FileSize == 4 && entries[1].Metadata().FileMode.IsRegular() { + if entries[1].Metadata().Name == "f2" && entries[1].Metadata().FileSize == 4 && entries[1].Metadata().FileMode().IsRegular() { goodCount++ } - if entries[2].Metadata().Name == "f3" && entries[2].Metadata().FileSize == 3 && entries[2].Metadata().FileMode.IsRegular() { + if entries[2].Metadata().Name == "f3" && entries[2].Metadata().FileSize == 3 && entries[2].Metadata().FileMode().IsRegular() { goodCount++ } - if entries[3].Metadata().Name == "y" && entries[3].Metadata().FileSize == 0 && entries[3].Metadata().FileMode.IsDir() { + if entries[3].Metadata().Name == "y" && entries[3].Metadata().FileSize == 0 && entries[3].Metadata().FileMode().IsDir() { goodCount++ } - if entries[4].Metadata().Name == "z" && entries[4].Metadata().FileSize == 0 && entries[4].Metadata().FileMode.IsDir() { + if entries[4].Metadata().Name == "z" && entries[4].Metadata().FileSize == 0 && entries[4].Metadata().FileMode().IsDir() { goodCount++ } if goodCount != 5 { diff --git a/fs/filesystem_windows.go b/fs/local_fs_windows.go similarity index 100% rename from fs/filesystem_windows.go rename to fs/local_fs_windows.go diff --git a/fs/repository.go b/fs/repository_fs.go similarity index 88% rename from fs/repository.go rename to fs/repository_fs.go index dd3a1ab1f..23013f5cb 100644 --- a/fs/repository.go +++ b/fs/repository_fs.go @@ -18,7 +18,7 @@ type repositoryFile repositoryEntry type repositorySymlink repositoryEntry func (rd *repositoryDirectory) Readdir() (Entries, error) { - r, err := rd.repo.Open(rd.entry.Metadata().ObjectID) + r, err := rd.repo.Open(*rd.entry.Metadata().ObjectId) if err != nil { return nil, err } @@ -38,7 +38,7 @@ func (rd *repositoryDirectory) Readdir() (Entries, error) { } func (rf *repositoryFile) Open() (EntryMetadataReadCloser, error) { - r, err := rf.repo.Open(rf.entry.Metadata().ObjectID) + r, err := rf.repo.Open(*rf.entry.Metadata().ObjectId) if err != nil { return nil, err } @@ -51,7 +51,7 @@ func (rsl *repositorySymlink) Readlink() (string, error) { } func newRepoEntry(r repo.Repository, md *EntryMetadata, parent Directory) Entry { - switch md.FileMode & os.ModeType { + switch md.FileMode() & os.ModeType { case os.ModeDir: return Directory(&repositoryDirectory{ entry: newEntry(md, parent), @@ -71,7 +71,7 @@ func newRepoEntry(r repo.Repository, md *EntryMetadata, parent Directory) Entry }) default: - panic(fmt.Sprintf("not supported entry metadata type: %v", md.FileMode)) + panic(fmt.Sprintf("not supported entry metadata type: %v", md.FileMode())) } } @@ -94,9 +94,9 @@ func withMetadata(rc io.ReadCloser, md *EntryMetadata) EntryMetadataReadCloser { // NewRepositoryDirectory returns Directory based on repository object with the specified ID. func NewRepositoryDirectory(r repo.Repository, objectID repo.ObjectID) Directory { d := newRepoEntry(r, &EntryMetadata{ - Name: "/", - ObjectID: objectID, - FileMode: 0555 | os.ModeDir, + Name: "/", + ObjectId: &objectID, + ModeAndPermissions: 0555 | uint32(os.ModeDir), }, nil) return d.(Directory) diff --git a/fs/upload.go b/fs/upload.go index 38bbf8da7..a35153b99 100644 --- a/fs/upload.go +++ b/fs/upload.go @@ -36,7 +36,7 @@ type UploadResult struct { // Uploader supports efficient uploading files and directories to repository. type Uploader interface { - UploadDir(dir Directory, previousManifestID repo.ObjectID) (*UploadResult, error) + UploadDir(dir Directory, previousManifestID *repo.ObjectID) (*UploadResult, error) UploadFile(file File) (*UploadResult, error) Cancel() } @@ -74,11 +74,13 @@ func (u *uploader) uploadFileInternal(f File, relativePath string, forceStored b return nil, 0, err } - e2.ObjectID, err = writer.Result(forceStored) + r, err := writer.Result(forceStored) if err != nil { return nil, 0, err } + e2.ObjectId = &r + if written != e2.FileSize { // file changed } @@ -123,10 +125,11 @@ func (u *uploader) uploadBundleInternal(b *uploadBundle) (*EntryMetadata, uint64 } b.files = uploadedFiles - bundleMetadata.ObjectID, err = writer.Result(true) + r, err := writer.Result(true) if err != nil { return nil, 0, err } + bundleMetadata.ObjectId = &r return bundleMetadata, bundleMetadata.metadataHash(), nil } @@ -134,16 +137,16 @@ func (u *uploader) uploadBundleInternal(b *uploadBundle) (*EntryMetadata, uint64 func (u *uploader) UploadFile(file File) (*UploadResult, error) { result := &UploadResult{} e, _, err := u.uploadFileInternal(file, file.Metadata().Name, true) - result.ObjectID = e.ObjectID + result.ObjectID = *e.ObjectId return result, err } -func (u *uploader) UploadDir(dir Directory, previousManifestID repo.ObjectID) (*UploadResult, error) { +func (u *uploader) UploadDir(dir Directory, previousManifestID *repo.ObjectID) (*UploadResult, error) { var mr hashcacheReader var err error - if previousManifestID != "" { - if r, err := u.repo.Open(previousManifestID); err == nil { + if previousManifestID != nil { + if r, err := u.repo.Open(*previousManifestID); err == nil { mr.open(r) } } @@ -178,7 +181,7 @@ func (u *uploader) uploadDirInternal( entries, err := dir.Readdir() if err != nil { - return "", 0, false, err + return repo.NullObjectID, 0, false, err } entries = u.bundleEntries(entries) @@ -207,20 +210,22 @@ func (u *uploader) uploadDirInternal( case Directory: oid, h, wasCached, err := u.uploadDirInternal(result, entry, entryRelativePath, hcw, mr, false) if err != nil { - return "", 0, false, err + return repo.NullObjectID, 0, false, err } //log.Printf("dirHash: %v %v", fullPath, h) hash = h allCached = allCached && wasCached - e.ObjectID = oid + e.ObjectId = &oid case Symlink: l, err := entry.Readlink() if err != nil { - return "", 0, false, err + return repo.NullObjectID, 0, false, err } - e.ObjectID = repo.NewInlineObjectID([]byte(l)) + id := repo.NewInlineObjectID([]byte(l)) + + e.ObjectId = &id hash = e.metadataHash() case *uploadBundle: @@ -239,13 +244,14 @@ func (u *uploader) uploadDirInternal( if cacheMatches { result.Stats.CachedFiles++ // Avoid hashing by reusing previous object ID. - e.ObjectID = repo.ObjectID(cachedEntry.ObjectID) + id := *cachedEntry.ObjectId + e.ObjectId = &id hash = cachedEntry.Hash } else { result.Stats.NonCachedFiles++ e, hash, err = u.uploadBundleInternal(entry) if err != nil { - return "", 0, false, fmt.Errorf("unable to hash file: %s", err) + return repo.NullObjectID, 0, false, fmt.Errorf("unable to hash file: %s", err) } } @@ -262,18 +268,19 @@ func (u *uploader) uploadDirInternal( if cacheMatches { result.Stats.CachedFiles++ // Avoid hashing by reusing previous object ID. - e.ObjectID = repo.ObjectID(cachedEntry.ObjectID) + id := *cachedEntry.ObjectId + e.ObjectId = &id hash = cachedEntry.Hash } else { result.Stats.NonCachedFiles++ e, hash, err = u.uploadFileInternal(entry, entryRelativePath, false) if err != nil { - return "", 0, false, fmt.Errorf("unable to hash file: %s", err) + return repo.NullObjectID, 0, false, fmt.Errorf("unable to hash file: %s", err) } } default: - return "", 0, false, fmt.Errorf("file type %v not supported", entry.Metadata().FileMode) + return repo.NullObjectID, 0, false, fmt.Errorf("file type %v not supported", entry.Metadata().FileMode()) } if hash != 0 { @@ -283,22 +290,20 @@ func (u *uploader) uploadDirInternal( } if err := dw.WriteEntry(e, childrenMetadata); err != nil { - return "", 0, false, err + return repo.NullObjectID, 0, false, err } - if !e.FileMode.IsDir() && e.ObjectID.Type().IsStored() { - if err := hcw.WriteEntry(hashCacheEntry{ + if !e.FileMode().IsDir() && e.ObjectId.StorageBlock != "" { + if err := hcw.WriteEntry(HashCacheEntry{ Name: entryRelativePath, - Hash: e.metadataHash(), - ObjectID: e.ObjectID, + Hash: hash, + ObjectId: e.ObjectId, }); err != nil { - return "", 0, false, err + return repo.NullObjectID, 0, false, err } } } - dw.Close() - var directoryOID repo.ObjectID dirHash := dirHasher.Sum64() @@ -308,7 +313,7 @@ func (u *uploader) uploadDirInternal( if allCached { // Avoid hashing directory listing if every entry matched the cache. result.Stats.CachedDirectories++ - directoryOID = repo.ObjectID(cachedDirEntry.ObjectID) + directoryOID = *cachedDirEntry.ObjectId } else { result.Stats.NonCachedDirectories++ directoryOID, err = writer.Result(forceStored) @@ -317,14 +322,15 @@ func (u *uploader) uploadDirInternal( } } - if err := hcw.WriteEntry(hashCacheEntry{ + if err := hcw.WriteEntry(HashCacheEntry{ Name: relativePath + "/", - ObjectID: directoryOID, + ObjectId: &directoryOID, Hash: dirHash, }); err != nil { - return "", 0, false, err + return repo.NullObjectID, 0, false, err } + // log.Printf("Dir: %v %v %v %v", relativePath, directoryOID.UIString(), dirHash, allCached) return directoryOID, dirHash, allCached, nil } @@ -398,8 +404,8 @@ func (u *uploader) bundleEntries(entries Entries) Entries { func getBundleNumber(md *EntryMetadata) int { // TODO(jkowalski): This is not ready yet, uncomment when ready. - // if md.FileMode.IsRegular() && md.FileSize < maxBundleFileSize { - // return md.ModTime.Year()*100 + int(md.ModTime.Month()) + // if md.FileMode().IsRegular() && md.FileSize < maxBundleFileSize { + // return md.ModTime().Year()*100 + int(md.ModTime().Month()) // } return 0 diff --git a/fs/upload_test.go b/fs/upload_test.go index 59a5024b1..07780911b 100644 --- a/fs/upload_test.go +++ b/fs/upload_test.go @@ -41,8 +41,8 @@ func newUploadTestHarness() *uploadTestHarness { } format := repo.Format{ - Version: "1", - ObjectFormat: "md5", + Version: 1, + ObjectFormat: repo.ObjectIDFormat_TESTONLY_MD5, MaxBlobSize: 1000, MaxInlineBlobSize: 0, } @@ -86,22 +86,22 @@ func TestUpload(t *testing.T) { th := newUploadTestHarness() defer th.cleanup() - r1, err := th.uploader.UploadDir(th.sourceDir, "") + r1, err := th.uploader.UploadDir(th.sourceDir, nil) if err != nil { t.Errorf("upload failed: %v", err) } - r2, err := th.uploader.UploadDir(th.sourceDir, r1.ManifestID) + r2, err := th.uploader.UploadDir(th.sourceDir, &r1.ManifestID) if err != nil { t.Errorf("upload failed: %v", err) } - if r2.ObjectID != r1.ObjectID { - t.Errorf("expected r1.ObjectID==r2.ObjectID, got %v and %v", r1.ObjectID, r2.ObjectID) + if !r2.ObjectID.Equals(r1.ObjectID) { + t.Errorf("expected r1.ObjectID==r2.ObjectID, got %v and %v", r1.ObjectID.UIString(), r2.ObjectID.UIString()) } - if r2.ManifestID != r1.ManifestID { - t.Errorf("expected r2.ManifestID==r1.ManifestID, got %v and %v", r2.ManifestID, r1.ManifestID) + if !r2.ManifestID.Equals(r1.ManifestID) { + t.Errorf("expected r2.ManifestID==r1.ManifestID, got %v and %v", r2.ManifestID.UIString(), r1.ManifestID.UIString()) } if r1.Stats.CachedFiles+r1.Stats.CachedDirectories != 0 { @@ -116,17 +116,17 @@ func TestUpload(t *testing.T) { // Add one more file, the r1.ObjectID should change. th.sourceDir.addFile("d2/d1/f3", []byte{1, 2, 3, 4, 5}, 0777) - r3, err := th.uploader.UploadDir(th.sourceDir, r1.ManifestID) + r3, err := th.uploader.UploadDir(th.sourceDir, &r1.ManifestID) if err != nil { t.Errorf("upload failed: %v", err) } - if r2.ObjectID == r3.ObjectID { - t.Errorf("expected r3.ObjectID!=r2.ObjectID, got %v", r3.ObjectID) + if r2.ObjectID.Equals(r3.ObjectID) { + t.Errorf("expected r3.ObjectID!=r2.ObjectID, got %v", r3.ObjectID.UIString()) } - if r2.ManifestID == r3.ManifestID { - t.Errorf("expected r3.ManifestID!=r2.ManifestID, got %v", r3.ManifestID) + if r2.ManifestID.Equals(r3.ManifestID) { + t.Errorf("expected r3.ManifestID!=r2.ManifestID, got %v", r3.ManifestID.UIString()) } if r3.Stats.NonCachedFiles != 1 && r3.Stats.NonCachedDirectories != 3 { @@ -137,15 +137,15 @@ func TestUpload(t *testing.T) { // Now remove the added file, OID should be identical to the original before the file got added. th.sourceDir.subdir("d2", "d1").remove("f3") - r4, err := th.uploader.UploadDir(th.sourceDir, r1.ManifestID) + r4, err := th.uploader.UploadDir(th.sourceDir, &r1.ManifestID) if err != nil { t.Errorf("upload failed: %v", err) } - if r4.ObjectID != r1.ObjectID { + if !r4.ObjectID.Equals(r1.ObjectID) { t.Errorf("expected r4.ObjectID==r1.ObjectID, got %v and %v", r4.ObjectID, r1.ObjectID) } - if r4.ManifestID != r1.ManifestID { + if !r4.ManifestID.Equals(r1.ManifestID) { t.Errorf("expected r4.ManifestID==r1.ManifestID, got %v and %v", r4.ManifestID, r1.ManifestID) } @@ -156,7 +156,7 @@ func TestUpload(t *testing.T) { } // Upload again, this time using r3.ManifestID as base. - r5, err := th.uploader.UploadDir(th.sourceDir, r3.ManifestID) + r5, err := th.uploader.UploadDir(th.sourceDir, &r3.ManifestID) if err != nil { t.Errorf("upload failed: %v", err) } @@ -176,7 +176,7 @@ func TestUpload_TopLevelDirectoryReadFailure(t *testing.T) { th.sourceDir.failReaddir(errTest) - r, err := th.uploader.UploadDir(th.sourceDir, "") + r, err := th.uploader.UploadDir(th.sourceDir, nil) if err != errTest { t.Errorf("expected error: %v", err) } @@ -192,7 +192,7 @@ func TestUpload_SubDirectoryReadFailure(t *testing.T) { th.sourceDir.subdir("d1").failReaddir(errTest) - _, err := th.uploader.UploadDir(th.sourceDir, "") + _, err := th.uploader.UploadDir(th.sourceDir, nil) if err == nil { t.Errorf("expected error") } diff --git a/fuse/fusefs.go b/fuse/fusefs.go index f3131962a..6d7185c5a 100644 --- a/fuse/fusefs.go +++ b/fuse/fusefs.go @@ -21,11 +21,11 @@ type fuseNode struct { func (n *fuseNode) Attr(ctx context.Context, a *fuse.Attr) error { m := n.entry.Metadata() - a.Mode = m.FileMode + a.Mode = m.FileMode() a.Size = uint64(m.FileSize) - a.Mtime = m.ModTime - a.Uid = m.OwnerID - a.Gid = m.GroupID + a.Mtime = m.ModTime() + a.Uid = m.Uid + a.Gid = m.Gid return nil } @@ -88,7 +88,7 @@ func (dir *fuseDirectoryNode) ReadDirAll(ctx context.Context) ([]fuse.Dirent, er Name: m.Name, } - switch m.FileMode & os.ModeType { + switch m.FileMode() & os.ModeType { case os.ModeDir: dirent.Type = fuse.DT_Dir case 0: @@ -112,7 +112,7 @@ type fuseSymlinkNode struct { } func (sl *fuseSymlinkNode) Readlink(ctx context.Context, req *fuse.ReadlinkRequest) (string, error) { - return string(sl.entry.Metadata().ObjectID), nil + return sl.entry.(fs.Symlink).Readlink() } func newFuseNode(e fs.Entry) (fusefs.Node, error) { @@ -124,7 +124,7 @@ func newFuseNode(e fs.Entry) (fusefs.Node, error) { case fs.Symlink: return &fuseSymlinkNode{fuseNode{e}}, nil default: - return nil, fmt.Errorf("entry type not supported: %v", e.Metadata().FileMode) + return nil, fmt.Errorf("entry type not supported: %v", e.Metadata().FileMode()) } } diff --git a/internal/proto_stream_reader.go b/internal/proto_stream_reader.go new file mode 100644 index 000000000..06b66a7d7 --- /dev/null +++ b/internal/proto_stream_reader.go @@ -0,0 +1,66 @@ +package internal + +import ( + "bufio" + "bytes" + "encoding/binary" + "fmt" + "io" + + "github.com/golang/protobuf/proto" +) + +// ProtoStreamReader reads a stream of length-prefix proto messages with an optional header. +type ProtoStreamReader struct { + reader *bufio.Reader + readBuf []byte + buf *proto.Buffer + header []byte +} + +// Read reads the next proto message from the stream, returns io.EOF on the end of stream. +func (r *ProtoStreamReader) Read(v proto.Message) error { + v.Reset() + if r.header != nil { + p := make([]byte, len(r.header)) + if _, err := r.reader.Read(p); err == io.EOF { + return io.EOF + } + + if !bytes.Equal(p, r.header) { + return fmt.Errorf("invalid stream header: %v, expected %v", string(p), string(r.header)) + } + + r.header = nil + + } + length64, err := binary.ReadUvarint(r.reader) + if err != nil { + return err + } + + length := int(length64) + + if cap(r.readBuf) < length { + r.readBuf = make([]byte, length*12/10) + } + + r.readBuf = r.readBuf[:length] + if _, err := io.ReadFull(r.reader, r.readBuf); err != nil { + return err + } + + r.buf.SetBuf(r.readBuf) + + return r.buf.Unmarshal(v) +} + +// NewProtoStreamReader returns new ProtoStreamReader on top of a given buffered reader. +// The provided header must match the beginning of a stream. +func NewProtoStreamReader(r *bufio.Reader, header []byte) *ProtoStreamReader { + return &ProtoStreamReader{ + reader: r, + header: header, + buf: proto.NewBuffer(nil), + } +} diff --git a/internal/proto_stream_test.go b/internal/proto_stream_test.go new file mode 100644 index 000000000..18823bf42 --- /dev/null +++ b/internal/proto_stream_test.go @@ -0,0 +1,74 @@ +package internal + +import ( + "bufio" + "bytes" + "io" + "strings" + "testing" +) + +var testHeader1 = []byte("01234567") +var testHeader2 = []byte("0123456x") + +func TestProtoStream(t *testing.T) { + var buf bytes.Buffer + + data := []TestProto{ + TestProto{Name: "foo"}, + TestProto{Name: "bar"}, + TestProto{Name: "baz"}, + } + + w := NewProtoStreamWriter(&buf, testHeader1) + for _, d := range data { + if err := w.Write(&d); err != nil { + t.Errorf("write error: %v", err) + } + } + r := NewProtoStreamReader(bufio.NewReader(&buf), testHeader1) + var v TestProto + for _, d := range data { + if err := r.Read(&v); err != nil { + t.Errorf("read error: %v", err) + } + if v.Name != d.Name { + t.Errorf("invalid value: '%v', expected '%v'", v.Name, d.Name) + } + } + err := r.Read(&v) + if err != io.EOF { + t.Errorf("expected EOF, got %v", err) + } +} + +func TestEmptyStream(t *testing.T) { + var buf bytes.Buffer + + NewProtoStreamWriter(&buf, testHeader1) + + r := NewProtoStreamReader(bufio.NewReader(&buf), testHeader2) + var v TestProto + err := r.Read(&v) + if err != io.EOF { + t.Errorf("expected EOF, got %v", err) + } +} + +func TestInvalidHeader(t *testing.T) { + var buf bytes.Buffer + + w := NewProtoStreamWriter(&buf, testHeader1) + if err := w.Write(&TestProto{Name: "foo"}); err != nil { + t.Errorf("write error: %v", err) + } + + r := NewProtoStreamReader(bufio.NewReader(&buf), testHeader2) + var v TestProto + err := r.Read(&v) + if err == nil { + t.Errorf("expected error, got none") + } else if !strings.Contains(err.Error(), "invalid stream header") { + t.Errorf("got incorrect error: %v", err) + } +} diff --git a/internal/proto_stream_types.go b/internal/proto_stream_types.go new file mode 100644 index 000000000..244284180 --- /dev/null +++ b/internal/proto_stream_types.go @@ -0,0 +1,12 @@ +package internal + +var ( + // ProtoStreamTypeHashCache indicates proto stream whose elements are hash cache entries. + ProtoStreamTypeHashCache = []byte("kopiaHCH") + + // ProtoStreamTypeDir indicates proto stream whose elements are directory entries. + ProtoStreamTypeDir = []byte("kopiaDIR") + + // ProtoStreamTypeIndirect indicates proto stream whose elements are indirect block references. + ProtoStreamTypeIndirect = []byte("kopiaIND") +) diff --git a/internal/proto_stream_writer.go b/internal/proto_stream_writer.go new file mode 100644 index 000000000..5698c68de --- /dev/null +++ b/internal/proto_stream_writer.go @@ -0,0 +1,58 @@ +package internal + +import ( + "encoding/binary" + "io" + + "github.com/golang/protobuf/proto" +) + +// ProtoStreamWriter writes a stream of length-prefix proto messages with an optional header. +type ProtoStreamWriter struct { + writer io.Writer + buf *proto.Buffer + lengthPrefixBuf []byte + header []byte +} + +// Write emits length-prefixed proto message to its output. +func (w *ProtoStreamWriter) Write(v proto.Message) error { + if w.header != nil { + if _, err := w.writer.Write(w.header); err != nil { + return err + } + w.header = nil + } + + w.buf.Reset() + if err := w.buf.Marshal(v); err != nil { + return err + } + + protoBytes := w.buf.Bytes() + + if w.lengthPrefixBuf == nil { + w.lengthPrefixBuf = make([]byte, binary.MaxVarintLen64) + } + + lengthBytes := w.lengthPrefixBuf[0:binary.PutUvarint(w.lengthPrefixBuf[:], uint64(len(protoBytes)))] + + if _, err := w.writer.Write(lengthBytes); err != nil { + return err + } + + if _, err := w.writer.Write(protoBytes); err != nil { + return err + } + + return nil +} + +// NewProtoStreamWriter creates a new ProtoStreamWriter on top of a specified writer with a specified optional header. +func NewProtoStreamWriter(w io.Writer, header []byte) *ProtoStreamWriter { + return &ProtoStreamWriter{ + writer: w, + buf: proto.NewBuffer(nil), + header: header, + } +} diff --git a/internal/proto_test.pb.go b/internal/proto_test.pb.go new file mode 100644 index 000000000..2a4f367ec --- /dev/null +++ b/internal/proto_test.pb.go @@ -0,0 +1,51 @@ +// Code generated by protoc-gen-go. +// source: internal/proto_test.proto +// DO NOT EDIT! + +/* +Package internal is a generated protocol buffer package. + +It is generated from these files: + internal/proto_test.proto + +It has these top-level messages: + TestProto +*/ +package internal + +import proto "github.com/golang/protobuf/proto" +import fmt "fmt" +import math "math" + +// Reference imports to suppress errors if they are not otherwise used. +var _ = proto.Marshal +var _ = fmt.Errorf +var _ = math.Inf + +// This is a compile-time assertion to ensure that this generated file +// is compatible with the proto package it is being compiled against. +const _ = proto.ProtoPackageIsVersion1 + +type TestProto struct { + Name string `protobuf:"bytes,1,opt,name=name" json:"name,omitempty"` +} + +func (m *TestProto) Reset() { *m = TestProto{} } +func (m *TestProto) String() string { return proto.CompactTextString(m) } +func (*TestProto) ProtoMessage() {} +func (*TestProto) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{0} } + +func init() { + proto.RegisterType((*TestProto)(nil), "kopia.fs.TestProto") +} + +var fileDescriptor0 = []byte{ + // 109 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0xe2, 0x92, 0xcc, 0xcc, 0x2b, 0x49, + 0x2d, 0xca, 0x4b, 0xcc, 0xd1, 0x2f, 0x28, 0xca, 0x2f, 0xc9, 0x8f, 0x2f, 0x49, 0x2d, 0x2e, 0xd1, + 0x03, 0x33, 0x85, 0x38, 0xb2, 0xf3, 0x0b, 0x32, 0x13, 0xf5, 0xd2, 0x8a, 0x95, 0xe4, 0xb9, 0x38, + 0x43, 0x80, 0xe2, 0x01, 0x60, 0x61, 0x21, 0x2e, 0x96, 0xbc, 0xc4, 0xdc, 0x54, 0x09, 0x46, 0x05, + 0x46, 0x0d, 0xce, 0x20, 0x30, 0xdb, 0x49, 0x31, 0x4a, 0x3e, 0x3d, 0xb3, 0x24, 0xa3, 0x34, 0x49, + 0x2f, 0x39, 0x3f, 0x57, 0x1f, 0xac, 0x0f, 0x4a, 0xc2, 0x8c, 0x4f, 0x62, 0x03, 0x1b, 0x6a, 0x0c, + 0x08, 0x00, 0x00, 0xff, 0xff, 0x49, 0x42, 0xd9, 0xa7, 0x71, 0x00, 0x00, 0x00, +} diff --git a/internal/proto_test.proto b/internal/proto_test.proto new file mode 100644 index 000000000..f658f1d2d --- /dev/null +++ b/internal/proto_test.proto @@ -0,0 +1,8 @@ +syntax = "proto3"; + +package kopia.fs; +option go_package = "github.com/kopia/kopia/internal"; + +message TestProto { + string name = 1; +} diff --git a/repo/format.go b/repo/format.go index 9be0be294..6bf217307 100644 --- a/repo/format.go +++ b/repo/format.go @@ -5,9 +5,10 @@ import ( "crypto/cipher" "crypto/hmac" "crypto/md5" - "crypto/sha1" "crypto/sha256" "crypto/sha512" + "errors" + "fmt" "hash" ) @@ -16,36 +17,83 @@ var ( hkdfInfoCryptoKey = []byte("CryptoKey") ) -// Format describes the format of object data. -type Format struct { - Version string `json:"version"` - ObjectFormat string `json:"objectFormat"` - Secret []byte `json:"secret,omitempty"` - MaxInlineBlobSize int `json:"maxInlineBlobSize"` - MaxBlobSize int `json:"maxBlobSize"` +// ObjectIDFormatInfo performs hashing and encryption of a data block. +type ObjectIDFormatInfo interface { + Name() string + HashBuffer(data []byte, secret []byte) (blockID []byte, cryptoKey []byte) + CreateCipher(key []byte) (cipher.Block, error) } -// ObjectIDFormat describes single format ObjectID -type ObjectIDFormat struct { - Name string - IsEncrypted bool - - hashBuffer func(data []byte, secret []byte) ([]byte, []byte) - createCipher func(key []byte) (cipher.Block, error) +type unencryptedFormat struct { + name string + hashCtor func() hash.Hash + fold int } -// ObjectIDFormats is a collection of ObjectIDFormat -type ObjectIDFormats []*ObjectIDFormat +func (fi *unencryptedFormat) Name() string { + return fi.name +} -// Find returns the ObjectIDFormat with a given name or nil if not found. -func (fmts ObjectIDFormats) Find(name string) *ObjectIDFormat { - for _, f := range fmts { - if f.Name == name { - return f +func (fi *unencryptedFormat) HashBuffer(data []byte, secret []byte) (blockID []byte, cryptoKey []byte) { + blockID = hashContent(fi.hashCtor, data, secret) + if fi.fold > 0 { + for i := fi.fold; i < len(blockID); i++ { + blockID[i%fi.fold] ^= blockID[i] } + blockID = blockID[0:fi.fold] + } + return +} + +func (fi *unencryptedFormat) CreateCipher(key []byte) (cipher.Block, error) { + return nil, errors.New("encryption not supported") + +} + +type encryptedFormat struct { + name string + hashCtor func() hash.Hash + createCipher func(key []byte) (cipher.Block, error) + keyBytes int +} + +func (fi *encryptedFormat) Name() string { + return fi.name +} + +func (fi *encryptedFormat) HashBuffer(data []byte, secret []byte) (blockID []byte, cryptoKey []byte) { + h := hashContent(fi.hashCtor, data, secret) + p := len(h) - fi.keyBytes + blockID = h[0:p] + cryptoKey = h[p:] + return +} + +func (fi *encryptedFormat) CreateCipher(key []byte) (cipher.Block, error) { + return fi.createCipher(key) + +} + +// SupportedFormats containes supported ObjectIDformats +var SupportedFormats = map[ObjectIDFormat]ObjectIDFormatInfo{ + ObjectIDFormat_TESTONLY_MD5: &unencryptedFormat{"TESTONLY_MD5", md5.New, 0}, + ObjectIDFormat_UNENCYPTED_HMAC_SHA256: &unencryptedFormat{"UNENCRYPTED_HMAC_SHA256", sha256.New, 0}, + ObjectIDFormat_UNENCYPTED_HMAC_SHA256_128: &unencryptedFormat{"UNENCRYPTED_HMAC_SHA256", sha256.New, 16}, + ObjectIDFormat_ENCRYPTED_HMAC_SHA512_384_AES256: &encryptedFormat{"ENCRYPTED_HMAC_SHA512_384_AES256", sha512.New384, aes.NewCipher, 32}, + ObjectIDFormat_ENCRYPTED_HMAC_SHA512_AES256: &encryptedFormat{"ENCRYPTED_HMAC_SHA512_AES256", sha512.New, aes.NewCipher, 32}, +} + +// DefaultObjectFormat is the format that should be used by default when creating new repositories. +var DefaultObjectFormat = SupportedFormats[ObjectIDFormat_ENCRYPTED_HMAC_SHA512_AES256] + +// ParseObjectIDFormat parses the given string and returns corresponding ObjectIDFormat. +func ParseObjectIDFormat(s string) (ObjectIDFormat, error) { + v, ok := ObjectIDFormat_value[s] + if ok { + return ObjectIDFormat(v), nil } - return nil + return ObjectIDFormat_INVALID, fmt.Errorf("unknown object ID format: %v", s) } func fold(b []byte, size int) []byte { @@ -59,15 +107,6 @@ func fold(b []byte, size int) []byte { return b[0:size] } -func nonEncryptedFormat(name string, hf func() hash.Hash, hashSize int) *ObjectIDFormat { - return &ObjectIDFormat{ - Name: name, - hashBuffer: func(data []byte, secret []byte) ([]byte, []byte) { - return fold(hashContent(hf, data, secret), hashSize), nil - }, - } -} - func hashContent(hf func() hash.Hash, data []byte, secret []byte) []byte { var h hash.Hash @@ -79,55 +118,3 @@ func hashContent(hf func() hash.Hash, data []byte, secret []byte) []byte { h.Write(data) return h.Sum(nil) } - -func encryptedFormat( - name string, - hf func() hash.Hash, - blockIDSize int, - createCipher func(key []byte) (cipher.Block, error), - keySize int, -) *ObjectIDFormat { - return &ObjectIDFormat{ - Name: name, - hashBuffer: func(data []byte, secret []byte) ([]byte, []byte) { - contentHash := hashContent(sha512.New, data, secret) - blockID := fold(hashContent(hf, hkdfInfoBlockID, contentHash), blockIDSize) - cryptoKey := fold(hashContent(hf, hkdfInfoCryptoKey, contentHash), keySize) - return blockID, cryptoKey - }, - createCipher: createCipher, - } -} - -func buildObjectIDFormats() ObjectIDFormats { - var result ObjectIDFormats - - for _, h := range []struct { - name string - hash func() hash.Hash - hashSize int - }{ - {"md5", md5.New, md5.Size}, - {"sha1", sha1.New, sha1.Size}, - {"sha224", sha256.New224, sha256.Size224}, - {"sha256", sha256.New, sha256.Size}, - {"sha256-fold128", sha256.New, 16}, - {"sha256-fold160", sha256.New, 20}, - {"sha384", sha512.New384, sha512.Size384}, - {"sha512-fold128", sha512.New, 16}, - {"sha512-fold160", sha512.New, 20}, - {"sha512-224", sha512.New512_224, sha512.Size224}, - {"sha512-256", sha512.New512_256, sha512.Size256}, - {"sha512", sha512.New, sha512.Size}, - } { - result = append(result, nonEncryptedFormat(h.name, h.hash, h.hashSize)) - result = append(result, encryptedFormat(h.name+"-aes128", h.hash, h.hashSize, aes.NewCipher, 16)) - result = append(result, encryptedFormat(h.name+"-aes192", h.hash, h.hashSize, aes.NewCipher, 24)) - result = append(result, encryptedFormat(h.name+"-aes256", h.hash, h.hashSize, aes.NewCipher, 32)) - } - - return result -} - -// SupportedFormats contains supported repository formats. -var SupportedFormats = buildObjectIDFormats() diff --git a/repo/format.pb.go b/repo/format.pb.go new file mode 100644 index 000000000..72f471ca4 --- /dev/null +++ b/repo/format.pb.go @@ -0,0 +1,141 @@ +// Code generated by protoc-gen-go. +// source: repo/format.proto +// DO NOT EDIT! + +/* +Package repo is a generated protocol buffer package. + +It is generated from these files: + repo/format.proto + repo/indirect.proto + repo/objectid.proto + +It has these top-level messages: + Format + IndirectObjectEntry + ObjectID +*/ +package repo + +import proto "github.com/golang/protobuf/proto" +import fmt "fmt" +import math "math" + +// Reference imports to suppress errors if they are not otherwise used. +var _ = proto.Marshal +var _ = fmt.Errorf +var _ = math.Inf + +// This is a compile-time assertion to ensure that this generated file +// is compatible with the proto package it is being compiled against. +const _ = proto.ProtoPackageIsVersion1 + +type EncryptionAlgorithm int32 + +const ( + EncryptionAlgorithm_NONE EncryptionAlgorithm = 0 + EncryptionAlgorithm_AES EncryptionAlgorithm = 1 +) + +var EncryptionAlgorithm_name = map[int32]string{ + 0: "NONE", + 1: "AES", +} +var EncryptionAlgorithm_value = map[string]int32{ + "NONE": 0, + "AES": 1, +} + +func (x EncryptionAlgorithm) String() string { + return proto.EnumName(EncryptionAlgorithm_name, int32(x)) +} +func (EncryptionAlgorithm) EnumDescriptor() ([]byte, []int) { return fileDescriptor0, []int{0} } + +// ObjectIDFormat describes the format of repository objects. +type ObjectIDFormat int32 + +const ( + // Format not specified. + ObjectIDFormat_INVALID ObjectIDFormat = 0 + // MD5 (no encryption, MD5 hashing, only used for testing) + ObjectIDFormat_TESTONLY_MD5 ObjectIDFormat = 1 + // HMAC-SHA256 hash of content used as object ID, no encryption. + ObjectIDFormat_UNENCYPTED_HMAC_SHA256 ObjectIDFormat = 2 + // HMAC-SHA256 hash folded into 128 bits used as object ID, no encryption. + ObjectIDFormat_UNENCYPTED_HMAC_SHA256_128 ObjectIDFormat = 3 + // HMAC-SHA512 hash of content, top part of the hash used as 256-bit object ID, + // bottom part used as 256-bit encryption key. + ObjectIDFormat_ENCRYPTED_HMAC_SHA512_AES256 ObjectIDFormat = 4 + // HMAC-SHA512_384 hash over contents, top part of the hash used as 128-bit object ID, + // bottom part used as 256-bit encryption key. + ObjectIDFormat_ENCRYPTED_HMAC_SHA512_384_AES256 ObjectIDFormat = 5 +) + +var ObjectIDFormat_name = map[int32]string{ + 0: "INVALID", + 1: "TESTONLY_MD5", + 2: "UNENCYPTED_HMAC_SHA256", + 3: "UNENCYPTED_HMAC_SHA256_128", + 4: "ENCRYPTED_HMAC_SHA512_AES256", + 5: "ENCRYPTED_HMAC_SHA512_384_AES256", +} +var ObjectIDFormat_value = map[string]int32{ + "INVALID": 0, + "TESTONLY_MD5": 1, + "UNENCYPTED_HMAC_SHA256": 2, + "UNENCYPTED_HMAC_SHA256_128": 3, + "ENCRYPTED_HMAC_SHA512_AES256": 4, + "ENCRYPTED_HMAC_SHA512_384_AES256": 5, +} + +func (x ObjectIDFormat) String() string { + return proto.EnumName(ObjectIDFormat_name, int32(x)) +} +func (ObjectIDFormat) EnumDescriptor() ([]byte, []int) { return fileDescriptor0, []int{1} } + +// Format describes the format of object data. +type Format struct { + Version int32 `protobuf:"varint,1,opt,name=version" json:"version,omitempty"` + ObjectFormat ObjectIDFormat `protobuf:"varint,2,opt,name=object_format,json=objectFormat,enum=kopia.repo.ObjectIDFormat" json:"object_format,omitempty"` + Secret []byte `protobuf:"bytes,3,opt,name=secret,proto3" json:"secret,omitempty"` + MaxInlineBlobSize int32 `protobuf:"varint,4,opt,name=max_inline_blob_size,json=maxInlineBlobSize" json:"max_inline_blob_size,omitempty"` + MaxBlobSize int32 `protobuf:"varint,5,opt,name=max_blob_size,json=maxBlobSize" json:"max_blob_size,omitempty"` +} + +func (m *Format) Reset() { *m = Format{} } +func (m *Format) String() string { return proto.CompactTextString(m) } +func (*Format) ProtoMessage() {} +func (*Format) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{0} } + +func init() { + proto.RegisterType((*Format)(nil), "kopia.repo.Format") + proto.RegisterEnum("kopia.repo.EncryptionAlgorithm", EncryptionAlgorithm_name, EncryptionAlgorithm_value) + proto.RegisterEnum("kopia.repo.ObjectIDFormat", ObjectIDFormat_name, ObjectIDFormat_value) +} + +var fileDescriptor0 = []byte{ + // 357 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0x6c, 0x91, 0xcb, 0x6a, 0xf2, 0x40, + 0x14, 0xc7, 0xbf, 0x78, 0xfd, 0x38, 0x5e, 0x88, 0xd3, 0x22, 0xc1, 0x5e, 0x10, 0xe9, 0x42, 0x5c, + 0x44, 0x8c, 0xb5, 0xb8, 0x2b, 0xd1, 0xa4, 0x18, 0xd0, 0x58, 0x12, 0x5b, 0xb0, 0x9b, 0xc1, 0x84, + 0xd4, 0xa6, 0x35, 0x99, 0x30, 0xa6, 0xa5, 0xed, 0x6b, 0xf5, 0x45, 0xfa, 0x48, 0x9d, 0x8c, 0x8a, + 0x08, 0x6e, 0x06, 0xce, 0xfc, 0x7e, 0xe7, 0x3f, 0x67, 0x38, 0x50, 0xa1, 0x5e, 0x44, 0xda, 0xcf, + 0x84, 0x06, 0x8b, 0x58, 0x8e, 0x28, 0x89, 0x09, 0x82, 0x37, 0x12, 0xf9, 0x0b, 0x39, 0x01, 0x8d, + 0x5f, 0x01, 0x72, 0x77, 0x1c, 0x22, 0x09, 0xf2, 0x1f, 0x1e, 0x5d, 0xfb, 0x24, 0x94, 0x84, 0xba, + 0xd0, 0xcc, 0x5a, 0xbb, 0x12, 0xdd, 0x42, 0x89, 0x38, 0xaf, 0x9e, 0x1b, 0xe3, 0x4d, 0x8e, 0x94, + 0x62, 0xbc, 0xac, 0xd4, 0xe4, 0x7d, 0x90, 0x3c, 0xe5, 0x82, 0xa1, 0x6d, 0xc2, 0xac, 0xe2, 0xa6, + 0x61, 0x1b, 0x5d, 0x85, 0xdc, 0xda, 0x73, 0xa9, 0x17, 0x4b, 0x69, 0xd6, 0x59, 0xb4, 0xb6, 0x15, + 0x6a, 0xc3, 0x69, 0xb0, 0xf8, 0xc4, 0x7e, 0xb8, 0xf2, 0x43, 0x0f, 0x3b, 0x2b, 0xe2, 0xe0, 0xb5, + 0xff, 0xed, 0x49, 0x19, 0xfe, 0x7e, 0x85, 0x31, 0x83, 0xa3, 0x01, 0x23, 0x36, 0x03, 0xa8, 0x01, + 0xa5, 0xa4, 0x61, 0x6f, 0x66, 0xb9, 0x59, 0x60, 0x97, 0x3b, 0xa7, 0xd5, 0x84, 0x13, 0x3d, 0x74, + 0xe9, 0x57, 0x14, 0xb3, 0xd9, 0xd5, 0xd5, 0x92, 0x50, 0x3f, 0x7e, 0x09, 0xd0, 0x7f, 0xc8, 0x98, + 0x53, 0x53, 0x17, 0xff, 0xa1, 0x3c, 0xa4, 0x55, 0xdd, 0x16, 0x85, 0xd6, 0x8f, 0x00, 0xe5, 0xc3, + 0xb9, 0x51, 0x01, 0xf2, 0x86, 0xf9, 0xa8, 0x8e, 0x0d, 0x8d, 0x89, 0x22, 0x14, 0x67, 0xba, 0x3d, + 0x9b, 0x9a, 0xe3, 0x39, 0x9e, 0x68, 0x3d, 0x51, 0x40, 0x35, 0xa8, 0x3e, 0x98, 0xba, 0x39, 0x9c, + 0xdf, 0xcf, 0x74, 0x0d, 0x8f, 0x26, 0xea, 0x10, 0xdb, 0x23, 0x55, 0xe9, 0xdd, 0x88, 0x29, 0x74, + 0x09, 0xb5, 0xe3, 0x0c, 0x77, 0x94, 0xbe, 0x98, 0x46, 0x75, 0x38, 0x67, 0xd4, 0x3a, 0xc4, 0xbd, + 0x8e, 0x82, 0xd9, 0x30, 0x49, 0x42, 0x06, 0x5d, 0x41, 0xfd, 0xb8, 0xd1, 0xed, 0x5f, 0xef, 0xac, + 0xec, 0xe0, 0xe2, 0xe9, 0x6c, 0xc9, 0x7e, 0xf4, 0xee, 0xc8, 0x2e, 0x09, 0xda, 0x7c, 0x05, 0xdb, + 0x33, 0x59, 0x84, 0x93, 0xe3, 0x4b, 0xee, 0xfe, 0x05, 0x00, 0x00, 0xff, 0xff, 0x3b, 0xcf, 0x39, + 0xd2, 0xf9, 0x01, 0x00, 0x00, +} diff --git a/repo/format.proto b/repo/format.proto new file mode 100644 index 000000000..5589470aa --- /dev/null +++ b/repo/format.proto @@ -0,0 +1,41 @@ +syntax = "proto3"; + +package kopia.repo; +option go_package = "github.com/kopia/kopia/repo"; + +enum EncryptionAlgorithm { + NONE = 0; + AES = 1; +} + +// ObjectIDFormat describes the format of repository objects. +enum ObjectIDFormat { + // Format not specified. + INVALID = 0; + + // MD5 (no encryption, MD5 hashing, only used for testing) + TESTONLY_MD5 = 1; + + // HMAC-SHA256 hash of content used as object ID, no encryption. + UNENCYPTED_HMAC_SHA256 = 2; + + // HMAC-SHA256 hash folded into 128 bits used as object ID, no encryption. + UNENCYPTED_HMAC_SHA256_128 = 3; + + // HMAC-SHA512 hash of content, top part of the hash used as 256-bit object ID, + // bottom part used as 256-bit encryption key. + ENCRYPTED_HMAC_SHA512_AES256 = 4; + + // HMAC-SHA512_384 hash over contents, top part of the hash used as 128-bit object ID, + // bottom part used as 256-bit encryption key. + ENCRYPTED_HMAC_SHA512_384_AES256 = 5; +} + +// Format describes the format of object data. +message Format { + int32 version = 1; + ObjectIDFormat object_format = 2; + bytes secret = 3; + int32 max_inline_blob_size = 4; + int32 max_blob_size = 5; +} diff --git a/repo/indirect.pb.go b/repo/indirect.pb.go new file mode 100644 index 000000000..50c9981ee --- /dev/null +++ b/repo/indirect.pb.go @@ -0,0 +1,52 @@ +// Code generated by protoc-gen-go. +// source: repo/indirect.proto +// DO NOT EDIT! + +package repo + +import proto "github.com/golang/protobuf/proto" +import fmt "fmt" +import math "math" + +// Reference imports to suppress errors if they are not otherwise used. +var _ = proto.Marshal +var _ = fmt.Errorf +var _ = math.Inf + +// IndirectObjectEntry represents an entry in indirect object proto stream. +type IndirectObjectEntry struct { + Start int64 `protobuf:"varint,1,opt,name=start" json:"start,omitempty"` + Length int64 `protobuf:"varint,2,opt,name=length" json:"length,omitempty"` + Object *ObjectID `protobuf:"bytes,3,opt,name=object" json:"object,omitempty"` +} + +func (m *IndirectObjectEntry) Reset() { *m = IndirectObjectEntry{} } +func (m *IndirectObjectEntry) String() string { return proto.CompactTextString(m) } +func (*IndirectObjectEntry) ProtoMessage() {} +func (*IndirectObjectEntry) Descriptor() ([]byte, []int) { return fileDescriptor1, []int{0} } + +func (m *IndirectObjectEntry) GetObject() *ObjectID { + if m != nil { + return m.Object + } + return nil +} + +func init() { + proto.RegisterType((*IndirectObjectEntry)(nil), "kopia.repo.IndirectObjectEntry") +} + +var fileDescriptor1 = []byte{ + // 165 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0xe2, 0x12, 0x2e, 0x4a, 0x2d, 0xc8, + 0xd7, 0xcf, 0xcc, 0x4b, 0xc9, 0x2c, 0x4a, 0x4d, 0x2e, 0xd1, 0x2b, 0x28, 0xca, 0x2f, 0xc9, 0x17, + 0xe2, 0xca, 0xce, 0x2f, 0xc8, 0x4c, 0xd4, 0x03, 0x49, 0x49, 0x41, 0x14, 0xe4, 0x27, 0x65, 0x01, + 0xa5, 0x33, 0x53, 0x20, 0x0a, 0x94, 0x0a, 0xb9, 0x84, 0x3d, 0xa1, 0x5a, 0xfc, 0xc1, 0x32, 0xae, + 0x79, 0x25, 0x45, 0x95, 0x42, 0x22, 0x5c, 0xac, 0xc5, 0x25, 0x89, 0x45, 0x25, 0x12, 0x8c, 0x0a, + 0x8c, 0x1a, 0xcc, 0x41, 0x10, 0x8e, 0x90, 0x18, 0x17, 0x5b, 0x4e, 0x6a, 0x5e, 0x7a, 0x49, 0x86, + 0x04, 0x13, 0x58, 0x18, 0xca, 0x13, 0xd2, 0xe1, 0x62, 0x83, 0x18, 0x2b, 0xc1, 0x0c, 0x14, 0xe7, + 0x36, 0x12, 0xd1, 0x43, 0x58, 0xab, 0x07, 0x31, 0xd6, 0xd3, 0x25, 0x08, 0xaa, 0xc6, 0x49, 0x36, + 0x4a, 0x3a, 0x3d, 0xb3, 0x24, 0xa3, 0x34, 0x49, 0x2f, 0x39, 0x3f, 0x57, 0x1f, 0xac, 0x12, 0x4a, + 0x82, 0xd4, 0x27, 0xb1, 0x81, 0x1d, 0x66, 0x0c, 0x08, 0x00, 0x00, 0xff, 0xff, 0x0c, 0xb4, 0xa1, + 0x12, 0xd0, 0x00, 0x00, 0x00, +} diff --git a/repo/indirect.proto b/repo/indirect.proto new file mode 100644 index 000000000..7e0bb0c6c --- /dev/null +++ b/repo/indirect.proto @@ -0,0 +1,13 @@ +syntax = "proto3"; + +package kopia.repo; +option go_package = "github.com/kopia/kopia/repo"; + +import "repo/objectid.proto"; + +// IndirectObjectEntry represents an entry in indirect object proto stream. +message IndirectObjectEntry { + int64 start = 1; + int64 length = 2; + repo.ObjectID object = 3; +} diff --git a/repo/object_reader.go b/repo/object_reader.go index 558724526..1259f8c42 100644 --- a/repo/object_reader.go +++ b/repo/object_reader.go @@ -3,32 +3,16 @@ package repo import ( "fmt" "io" - - "github.com/kopia/kopia/storage" ) -type seekTableEntry struct { - startOffset int64 - length int64 - blockID string -} - -func (r *seekTableEntry) endOffset() int64 { - return r.startOffset + int64(r.length) -} - -func (r *seekTableEntry) String() string { - return fmt.Sprintf("start: %d len: %d end: %d block: %s", - r.startOffset, - r.length, - r.endOffset(), - r.blockID) +func (i *IndirectObjectEntry) endOffset() int64 { + return i.Start + i.Length } type objectReader struct { - storage storage.Storage + repo Repository - seekTable []seekTableEntry + seekTable []IndirectObjectEntry currentPosition int64 // Overall position in the objectReader totalLength int64 // Overall length @@ -80,13 +64,19 @@ func (r *objectReader) Read(buffer []byte) (int, error) { } func (r *objectReader) openCurrentChunk() error { - blockID := r.seekTable[r.currentChunkIndex].blockID - blockData, err := r.storage.GetBlock(blockID) + st := r.seekTable[r.currentChunkIndex] + blockData, err := r.repo.Open(*st.Object) if err != nil { return err } + defer blockData.Close() - r.currentChunkData = blockData + b := make([]byte, st.Length) + if _, err := io.ReadFull(blockData, b); err != nil { + return err + } + + r.currentChunkData = b r.currentChunkPosition = 0 return nil } @@ -101,7 +91,7 @@ func (r *objectReader) findChunkIndexForOffset(offset int64) int { for left <= right { middle := (left + right) / 2 - if offset < r.seekTable[middle].startOffset { + if offset < r.seekTable[middle].Start { right = middle - 1 continue } @@ -136,7 +126,7 @@ func (r *objectReader) Seek(offset int64, whence int) (int64, error) { index := r.findChunkIndexForOffset(offset) - chunkStartOffset := r.seekTable[index].startOffset + chunkStartOffset := r.seekTable[index].Start if index != r.currentChunkIndex { r.closeCurrentChunk() diff --git a/repo/object_writer.go b/repo/object_writer.go index a1cef82e1..6d6adad2d 100644 --- a/repo/object_writer.go +++ b/repo/object_writer.go @@ -5,6 +5,7 @@ import ( "fmt" "io" + "github.com/kopia/kopia/internal" "github.com/kopia/kopia/storage" ) @@ -44,15 +45,15 @@ type objectWriter struct { prefix string listWriter *objectWriter + listProtoWriter *internal.ProtoStreamWriter + listCurrentPos int64 flushedObjectCount int lastFlushedObject ObjectID - description string - objectType ObjectIDType + description string + indirectLevel int32 blockTracker *blockTracker - - atomicWrites bool } func (w *objectWriter) Close() error { @@ -68,9 +69,13 @@ func (w *objectWriter) Close() error { return nil } -func (w *objectWriter) Write(data []byte) (n int, err error) { - dataLen := len(data) - remaining := len(data) +func (w *objectWriter) WriteGather(dataSlices [][]byte) (n int, err error) { + dataLen := 0 + for _, s := range dataSlices { + dataLen += len(s) + } + + remaining := dataLen w.totalLength += int64(remaining) for remaining > 0 { @@ -80,33 +85,45 @@ func (w *objectWriter) Write(data []byte) (n int, err error) { room := w.buffer.Cap() - w.buffer.Len() if remaining <= room { - w.buffer.Write(data) + for _, s := range dataSlices { + w.buffer.Write(s) + } + dataSlices = nil remaining = 0 } else { - if w.atomicWrites { - if w.buffer == nil { - // We're at the beginning of a buffer, fail if the buffer is too small. - return 0, fmt.Errorf("object writer buffer too small, need: %v, have: %v", remaining, room) - } - if err := w.flushBuffer(false); err != nil { - return 0, err - } + for room > 0 { + var chunk []byte - continue + if len(dataSlices[0]) >= room { + chunk = dataSlices[0][0:room] + dataSlices[0] = dataSlices[0][len(chunk):] + } else { + chunk = dataSlices[0] + dataSlices[0] = nil + } + w.buffer.Write(chunk) + remaining -= len(chunk) + room -= len(chunk) + if len(dataSlices[0]) == 0 { + dataSlices = dataSlices[1:] + } } - w.buffer.Write(data[0:room]) - if err := w.flushBuffer(false); err != nil { return 0, err } - data = data[room:] - remaining = len(data) } } return dataLen, nil } +func (w *objectWriter) Write(data []byte) (n int, err error) { + var b [1][]byte + + b[0] = data + return w.WriteGather(b[:]) +} + func (w *objectWriter) flushBuffer(force bool) error { // log.Printf("flushing bufer") // defer log.Printf("flushed") @@ -118,36 +135,43 @@ func (w *objectWriter) flushBuffer(force bool) error { b := w.buffer w.buffer = nil - objectID, blockReader, err := w.repo.hashBufferForWriting(b, string(w.objectType)+w.prefix) + objectID, blockReader, err := w.repo.hashBufferForWriting(b, w.prefix) if err != nil { return err } - if err := w.repo.storage.PutBlock(objectID.BlockID(), blockReader, storage.PutOptionsDefault); err != nil { + if err := w.repo.storage.PutBlock(objectID.StorageBlock, blockReader, storage.PutOptionsDefault); err != nil { return fmt.Errorf( "error when flushing chunk %d of %s to %#v: %#v", w.flushedObjectCount, w.description, - objectID.BlockID(), + objectID.StorageBlock, err) } - w.blockTracker.addBlock(objectID.BlockID()) + w.blockTracker.addBlock(objectID.StorageBlock) w.flushedObjectCount++ w.lastFlushedObject = objectID if w.listWriter == nil { w.listWriter = &objectWriter{ - repo: w.repo, - objectType: ObjectIDTypeList, - prefix: w.prefix, - description: "LIST(" + w.description + ")", - atomicWrites: true, - blockTracker: w.blockTracker, + repo: w.repo, + indirectLevel: w.indirectLevel + 1, + prefix: w.prefix, + description: "LIST(" + w.description + ")", + blockTracker: w.blockTracker, } + w.listProtoWriter = internal.NewProtoStreamWriter(w.listWriter, internal.ProtoStreamTypeIndirect) + w.listCurrentPos = 0 } - fmt.Fprintf(w.listWriter, "%v,%v\n", length, objectID) + w.listProtoWriter.Write(&IndirectObjectEntry{ + Object: &objectID, + Start: w.listCurrentPos, + Length: int64(length), + }) + + w.listCurrentPos += int64(length) } return nil } @@ -155,10 +179,10 @@ func (w *objectWriter) flushBuffer(force bool) error { func (w *objectWriter) Result(forceStored bool) (ObjectID, error) { if !forceStored && w.flushedObjectCount == 0 { if w.buffer == nil { - return "B", nil + return NullObjectID, nil } - if w.buffer.Len() < w.repo.format.MaxInlineBlobSize { + if w.buffer.Len() < int(w.repo.format.MaxInlineBlobSize) { return NewInlineObjectID(w.buffer.Bytes()), nil } } @@ -171,9 +195,10 @@ func (w *objectWriter) Result(forceStored bool) (ObjectID, error) { }() if w.flushedObjectCount == 1 { + w.lastFlushedObject.Indirect = w.indirectLevel return w.lastFlushedObject, nil } else if w.flushedObjectCount == 0 { - return "", nil + return NullObjectID, nil } else { return w.listWriter.Result(true) } diff --git a/repo/objectid.go b/repo/objectid.go index 017940ab4..5ef580606 100644 --- a/repo/objectid.go +++ b/repo/objectid.go @@ -4,87 +4,72 @@ import ( "encoding/base64" "encoding/hex" "errors" - "fmt" "strconv" "strings" - "unicode/utf8" + + "github.com/golang/protobuf/proto" + + "fmt" ) +// NullObjectID is the identifier of an null object. +var NullObjectID ObjectID + +const objectIDEncryptionInfoSeparator = "." + var ( - binaryEncoding = base64.RawURLEncoding + inlineContentEncoding = base64.RawURLEncoding ) -const ( - objectIDEncryptionInfoSeparator = "." -) +// UIString returns the name of the repository object that is suitable for displaying in the UI. +// Note that the object ID name may contain its encryption key, which is sensitive. +func (m *ObjectID) UIString() string { + if m.StorageBlock != "" { + var encryptionSuffix string -// ObjectIDType describes the type of the chunk. -type ObjectIDType string - -// ObjectEncryptionInfo represents encryption info associated with ObjectID. -type ObjectEncryptionInfo string - -// NoEncryption indicates that the object is not encrypted. -var NoEncryption = ObjectEncryptionInfo("") - -const ( - // ObjectIDTypeText represents text-only inline object ID - ObjectIDTypeText ObjectIDType = "T" - - // ObjectIDTypeBinary represents binary inline object ID - ObjectIDTypeBinary ObjectIDType = "B" - - // ObjectIDTypeStored represents ID of object whose data is stored directly in a single storage block indicated by string. - ObjectIDTypeStored ObjectIDType = "D" - - // ObjectIDTypeList represents ID of an object whose data is stored in mutliple storage blocks. - // The value of the ObjectID is the list chunk, which lists object IDs that need to be concatenated - // to form the contents. - ObjectIDTypeList ObjectIDType = "L" - - // ObjectIDTypeSection represents ID of an object whose data is a section of another object. - // The format is S{offset},{length},{base} - ObjectIDTypeSection ObjectIDType = "S" -) - -// IsStored determines whether data for the given chunk type is stored in the storage -// (as opposed to being stored inline as part of ObjectID itself). -func (ct ObjectIDType) IsStored() bool { - switch ct { - case ObjectIDTypeStored, ObjectIDTypeList: - return true - - default: - return false - } -} - -// ObjectID represents the identifier of a chunk. -// Identifiers can either refer to data block stored in a Storage, or contain small amounts -// of payload directly (useful for short ASCII or binary files). -type ObjectID string - -// Type gets the type of the object ID. -func (c ObjectID) Type() ObjectIDType { - return ObjectIDType(c[0:1]) -} - -// InlineData returns inline data stored as part of ObjectID. For chunk IDs representing stored -// chunks, the value is nil. -func (c ObjectID) InlineData() []byte { - payload := string(c[1:]) - switch c.Type() { - case ObjectIDTypeText: - return []byte(payload) - - case ObjectIDTypeBinary: - decodedPayload, err := binaryEncoding.DecodeString(payload) - if err == nil { - return decodedPayload + if m.Encryption != nil { + encryptionSuffix = "." + hex.EncodeToString(m.Encryption) } + + if m.Indirect > 0 { + return fmt.Sprintf("L%v,%v%v", m.Indirect, m.StorageBlock, encryptionSuffix) + } + + return "D" + m.StorageBlock + encryptionSuffix } - return nil + if m.Content != nil { + return "B" + inlineContentEncoding.EncodeToString(m.Content) + } + + if m.Section != nil { + return fmt.Sprintf("S%v,%v,%v", m.Section.Start, m.Section.Length, m.Section.Base.UIString()) + } + + return "B" +} + +// Equals determines whether ObjectID is equal to another ObjectID. +func (m *ObjectID) Equals(other ObjectID) bool { + return proto.Equal(m, &other) +} + +// NewInlineObjectID returns new ObjectID containing inline binary or text-encoded data. +func NewInlineObjectID(data []byte) ObjectID { + return ObjectID{ + Content: data, + } +} + +// NewSectionObjectID returns new ObjectID representing a section of an object with a given base ID, start offset and length. +func NewSectionObjectID(start, length int64, baseID *ObjectID) ObjectID { + return ObjectID{ + Section: &ObjectID_Section{ + Base: baseID, + Start: start, + Length: length, + }, + } } // parseNumberUntilComma parses a string of the form "{x},{remainder}" where x is a 64-bit number and remainder is arbitrary string. @@ -104,134 +89,82 @@ func parseNumberUntilComma(s string) (int64, string, error) { } func parseSectionInfoString(s string) (int64, int64, ObjectID, error) { - if ObjectIDType(s[0]) != ObjectIDTypeSection { - return 0, -1, "", errors.New("not a section object") - } - var start, length int64 var err error start, s, err = parseNumberUntilComma(s[1:]) if err != nil { - return 0, -1, "", err + return 0, -1, NullObjectID, err } length, s, err = parseNumberUntilComma(s) if err != nil { - return 0, -1, "", err + return 0, -1, NullObjectID, err } oid, err := ParseObjectID(s) if err != nil { - return 0, -1, "", err + return 0, -1, NullObjectID, err } return start, length, oid, nil } -// SectionInfo returns start, length and the base ID of a section object. -func (c ObjectID) SectionInfo() (start int64, length int64, baseID ObjectID) { - if c.Type() != ObjectIDTypeSection { - return 0, 0, "" - } - - start, length, oid, err := parseSectionInfoString(string(c)) - if err != nil { - // This should not happen if we came in through ParseObjectID - panic("invalid section info: " + string(c)) - } - - return start, length, oid -} - -// BlockID returns identifier of the storage block. For inline chunk IDs, an empty string is returned. -func (c ObjectID) BlockID() string { - if c.Type().IsStored() { - content := string(c[1:]) - firstSeparator := strings.Index(content, objectIDEncryptionInfoSeparator) - if firstSeparator > 0 { - return string(content[0:firstSeparator]) - } - - return string(content) - } - - return "" -} - -// EncryptionInfo returns ObjectEncryptionInfo for the ObjectID. -func (c ObjectID) EncryptionInfo() ObjectEncryptionInfo { - if c.Type().IsStored() { - content := string(c[1:]) - firstSeparator := strings.Index(content, objectIDEncryptionInfoSeparator) - if firstSeparator > 0 { - return ObjectEncryptionInfo(content[firstSeparator+1:]) - } - } - - return NoEncryption -} - -// NewInlineObjectID returns new ObjectID containing inline binary or text-encoded data. -func NewInlineObjectID(data []byte) ObjectID { - if !utf8.Valid(data) { - return ObjectID("B" + binaryEncoding.EncodeToString(data)) - } - - for _, b := range data { - if b < 32 && (b != 9 && b != 10 && b != 13) { - // Any other character indicates binary content. - return ObjectID("B" + binaryEncoding.EncodeToString(data)) - } - } - - return ObjectID("T" + string(data)) -} - -// NewSectionObjectID returns new ObjectID representing a section of an object with a given base ID, start offset and length. -func NewSectionObjectID(start, length int64, baseID ObjectID) ObjectID { - return ObjectID(fmt.Sprintf("S%v,%v,%v", start, length, baseID)) -} - // ParseObjectID converts the specified string into ObjectID. func ParseObjectID(objectIDString string) (ObjectID, error) { if len(objectIDString) >= 1 { - chunkType := ObjectIDType(objectIDString[0]) + chunkType := objectIDString[0] content := objectIDString[1:] switch chunkType { - case ObjectIDTypeSection: - if _, _, _, err := parseSectionInfoString(objectIDString); err == nil { - return ObjectID(objectIDString), nil + case 'S': + if start, length, base, err := parseSectionInfoString(objectIDString); err == nil { + return ObjectID{Section: &ObjectID_Section{ + Start: start, + Length: length, + Base: &base, + }}, nil } - case ObjectIDTypeText: - return ObjectID(objectIDString), nil - - case ObjectIDTypeBinary: - if _, err := binaryEncoding.DecodeString(content); err == nil { - return ObjectID(objectIDString), nil + case 'B': + if v, err := inlineContentEncoding.DecodeString(content); err == nil { + return ObjectID{Content: v}, nil + } + + case 'I', 'D': + var indirectLevel int32 + if chunkType == 'I' { + comma := strings.Index(content, ",") + if comma < 0 { + // malformed + break + } + i, err := strconv.Atoi(content[0:comma]) + if err != nil { + break + } + indirectLevel = int32(i) + content = content[comma+1:] } - case ObjectIDTypeList, ObjectIDTypeStored: firstSeparator := strings.Index(content, objectIDEncryptionInfoSeparator) lastSeparator := strings.LastIndex(content, objectIDEncryptionInfoSeparator) if firstSeparator == lastSeparator { if firstSeparator == -1 { // Found zero Separators in the ID - no encryption info. - return ObjectID(objectIDString), nil + return ObjectID{StorageBlock: content, Indirect: indirectLevel}, nil } if firstSeparator > 0 { b, err := hex.DecodeString(content[firstSeparator+1:]) - if err == nil && len(b) > 0 && len(b)%2 == 0 { + if err == nil && len(b) > 0 { // Valid chunk ID with encryption info. - return ObjectID(objectIDString), nil + return ObjectID{StorageBlock: content[0:firstSeparator], Encryption: b}, nil } } } } } - return "", fmt.Errorf("malformed chunk id: '%s'", objectIDString) + return NullObjectID, fmt.Errorf("malformed object id: '%s'", objectIDString) } diff --git a/repo/objectid.pb.go b/repo/objectid.pb.go new file mode 100644 index 000000000..d754e0020 --- /dev/null +++ b/repo/objectid.pb.go @@ -0,0 +1,80 @@ +// Code generated by protoc-gen-go. +// source: repo/objectid.proto +// DO NOT EDIT! + +package repo + +import proto "github.com/golang/protobuf/proto" +import fmt "fmt" +import math "math" + +// Reference imports to suppress errors if they are not otherwise used. +var _ = proto.Marshal +var _ = fmt.Errorf +var _ = math.Inf + +// ObjectID represents the identifier of a chunk. +// Identifiers can either refer to data block stored in a Storage, or contain small amounts +// of payload directly (useful for short ASCII or binary files). +type ObjectID struct { + StorageBlock string `protobuf:"bytes,1,opt,name=storage_block,json=storageBlock" json:"storage_block,omitempty"` + Indirect int32 `protobuf:"varint,2,opt,name=indirect" json:"indirect,omitempty"` + Encryption []byte `protobuf:"bytes,3,opt,name=encryption,proto3" json:"encryption,omitempty"` + Content []byte `protobuf:"bytes,4,opt,name=content,proto3" json:"content,omitempty"` + Section *ObjectID_Section `protobuf:"bytes,5,opt,name=section" json:"section,omitempty"` +} + +func (m *ObjectID) Reset() { *m = ObjectID{} } +func (m *ObjectID) String() string { return proto.CompactTextString(m) } +func (*ObjectID) ProtoMessage() {} +func (*ObjectID) Descriptor() ([]byte, []int) { return fileDescriptor2, []int{0} } + +func (m *ObjectID) GetSection() *ObjectID_Section { + if m != nil { + return m.Section + } + return nil +} + +type ObjectID_Section struct { + Base *ObjectID `protobuf:"bytes,1,opt,name=base" json:"base,omitempty"` + Start int64 `protobuf:"varint,2,opt,name=start" json:"start,omitempty"` + Length int64 `protobuf:"varint,3,opt,name=length" json:"length,omitempty"` +} + +func (m *ObjectID_Section) Reset() { *m = ObjectID_Section{} } +func (m *ObjectID_Section) String() string { return proto.CompactTextString(m) } +func (*ObjectID_Section) ProtoMessage() {} +func (*ObjectID_Section) Descriptor() ([]byte, []int) { return fileDescriptor2, []int{0, 0} } + +func (m *ObjectID_Section) GetBase() *ObjectID { + if m != nil { + return m.Base + } + return nil +} + +func init() { + proto.RegisterType((*ObjectID)(nil), "kopia.repo.ObjectID") + proto.RegisterType((*ObjectID_Section)(nil), "kopia.repo.ObjectID.Section") +} + +var fileDescriptor2 = []byte{ + // 256 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0x6c, 0x90, 0xcf, 0x4a, 0x03, 0x31, + 0x10, 0xc6, 0x69, 0xb7, 0xdb, 0xad, 0xd3, 0x7a, 0x89, 0x45, 0x42, 0xfd, 0x83, 0xe8, 0xa5, 0xa7, + 0x14, 0x2a, 0xf8, 0x00, 0xc5, 0x8b, 0x27, 0x21, 0xde, 0xbc, 0x48, 0x92, 0x86, 0x6d, 0x6c, 0xcd, + 0x2c, 0xd9, 0xf1, 0xe0, 0xdb, 0xf8, 0xa8, 0x66, 0xb3, 0xbb, 0xea, 0xa1, 0x97, 0xc0, 0xf7, 0xfd, + 0x66, 0x86, 0x1f, 0x81, 0xb3, 0x60, 0x2b, 0x5c, 0xa1, 0x7e, 0xb7, 0x86, 0xdc, 0x56, 0x54, 0x01, + 0x09, 0x19, 0xec, 0xb1, 0x72, 0x4a, 0x34, 0xe8, 0xf6, 0x7b, 0x08, 0x93, 0xe7, 0x84, 0x9f, 0x1e, + 0xd9, 0x1d, 0x9c, 0xd6, 0x84, 0x41, 0x95, 0xf6, 0x4d, 0x1f, 0xd0, 0xec, 0xf9, 0xe0, 0x66, 0xb0, + 0x3c, 0x91, 0xb3, 0xae, 0xdc, 0x34, 0x1d, 0x5b, 0xc0, 0xc4, 0xf9, 0xad, 0x0b, 0x71, 0x85, 0x0f, + 0x23, 0xcf, 0xe5, 0x6f, 0x66, 0xd7, 0x00, 0xd6, 0x9b, 0xf0, 0x55, 0x91, 0x43, 0xcf, 0xb3, 0x48, + 0x67, 0xf2, 0x5f, 0xc3, 0x38, 0x14, 0x06, 0x3d, 0x59, 0x4f, 0x7c, 0x94, 0x60, 0x1f, 0xd9, 0x03, + 0x14, 0x75, 0xe3, 0x18, 0xd7, 0xf2, 0x48, 0xa6, 0xeb, 0x4b, 0xf1, 0x67, 0x29, 0x7a, 0x43, 0xf1, + 0xd2, 0xce, 0xc8, 0x7e, 0x78, 0xa1, 0xa0, 0xe8, 0x3a, 0xb6, 0x84, 0x91, 0x56, 0xb5, 0x4d, 0xd2, + 0xd3, 0xf5, 0xfc, 0xd8, 0xbe, 0x4c, 0x13, 0x6c, 0x0e, 0x79, 0x4d, 0x2a, 0xb4, 0xfe, 0x99, 0x6c, + 0x03, 0x3b, 0x87, 0xf1, 0xc1, 0xfa, 0x92, 0x76, 0x49, 0x3c, 0x93, 0x5d, 0xda, 0x5c, 0xbd, 0x5e, + 0x94, 0x8e, 0x76, 0x9f, 0x5a, 0x18, 0xfc, 0x58, 0xa5, 0xab, 0xdd, 0xdb, 0xdc, 0xd6, 0xe3, 0xf4, + 0xa9, 0xf7, 0x3f, 0x01, 0x00, 0x00, 0xff, 0xff, 0x7b, 0x26, 0x25, 0xb7, 0x6b, 0x01, 0x00, 0x00, +} diff --git a/repo/objectid.proto b/repo/objectid.proto new file mode 100644 index 000000000..daa1c1738 --- /dev/null +++ b/repo/objectid.proto @@ -0,0 +1,21 @@ +syntax = "proto3"; + +package kopia.repo; +option go_package = "github.com/kopia/kopia/repo"; + +// ObjectID represents the identifier of a chunk. +// Identifiers can either refer to data block stored in a Storage, or contain small amounts +// of payload directly (useful for short ASCII or binary files). +message ObjectID { + string storage_block = 1; + int32 indirect = 2; // level of indirection (0==block data, 1=list of blocks, 2=list of lists, ...) + bytes encryption = 3; + bytes content = 4; + message Section { + ObjectID base = 1; + int64 start = 2; + int64 length = 3; + } + + Section section = 5; +} \ No newline at end of file diff --git a/repo/objectid_test.go b/repo/objectid_test.go index bb53ecfaa..b989b0c76 100644 --- a/repo/objectid_test.go +++ b/repo/objectid_test.go @@ -1,6 +1,7 @@ package repo import ( + "bytes" "strings" "testing" ) @@ -17,19 +18,21 @@ func TestParseMalformedObjectID(t *testing.T) { "Dab.", "Dab.a", "Dab.abc", - "Dab.00", "Dab.011", "L.", "L.x", "L.af", "Lx.ag", "Lab.", + "L1", + "L1,", + "L-1,X", "Xsomething", } for _, c := range cases { v, err := ParseObjectID(c) - if v != "" || err == nil || !strings.HasPrefix(err.Error(), "malformed chunk id") { + if !v.Equals(NullObjectID) || err == nil || !strings.HasPrefix(err.Error(), "malformed object id") { t.Errorf("unexpected result for %v: v: %v err: %v", c, v, err) } } @@ -38,24 +41,16 @@ func TestParseMalformedObjectID(t *testing.T) { func TestParseObjectIDEncryptionInfo(t *testing.T) { cases := []struct { objectID string - expected ObjectEncryptionInfo + expected []byte }{ - {"B", NoEncryption}, - {"BAQIDBA", NoEncryption}, - {"T", NoEncryption}, - {"T:foo", NoEncryption}, - {"Tfoo\nbar", NoEncryption}, - {"Tfoo\nbar:baz", NoEncryption}, - {"Dabcdef", NoEncryption}, - {"Labcdef", NoEncryption}, - { - "Dabcdef.00112233445566778899aabbccddeeff00112233445566778899aabbccddeeff", - ObjectEncryptionInfo("00112233445566778899aabbccddeeff00112233445566778899aabbccddeeff"), - }, - { - "Labcdef.00112233445566778899aabbccddeeff00112233445566778899aabbccddeeff", - ObjectEncryptionInfo("00112233445566778899aabbccddeeff00112233445566778899aabbccddeeff"), - }, + {"B", nil}, + {"BAQIDBA", nil}, + {"Dabcdef", nil}, + {"I1,abcdef", nil}, + {"I2,abcdef", nil}, + {"Dabcdef.00112233", []byte{0x00, 0x11, 0x22, 0x33}}, + {"I1,abcdef.0011223344", []byte{0x00, 0x11, 0x22, 0x33, 0x44}}, + {"I2,abcdef.0011223344", []byte{0x00, 0x11, 0x22, 0x33, 0x44}}, } for _, c := range cases { @@ -65,9 +60,9 @@ func TestParseObjectIDEncryptionInfo(t *testing.T) { continue } - actual := objectID.EncryptionInfo() - if actual != c.expected { - t.Errorf("invalid encryption info for %v: %v, expected: %v", c.objectID, actual, c.expected) + actual := objectID.Encryption + if !bytes.Equal(actual, c.expected) { + t.Errorf("invalid encryption info for %v: %x, expected: %x", c.objectID, actual, c.expected) } } } diff --git a/repo/repository.go b/repo/repository.go index efbba94ed..eb62737b9 100644 --- a/repo/repository.go +++ b/repo/repository.go @@ -10,10 +10,13 @@ import ( "fmt" "hash" "io" - "strconv" + "io/ioutil" + "log" "strings" "sync/atomic" + "github.com/kopia/kopia/internal" + "github.com/kopia/kopia/storage" "github.com/kopia/kopia/storage/logging" ) @@ -76,7 +79,7 @@ type repository struct { stats *RepositoryStats format Format - idFormat ObjectIDFormat + idFormat ObjectIDFormatInfo } func (repo *repository) Close() { @@ -107,7 +110,6 @@ func (repo *repository) Storage() storage.Storage { func (repo *repository) NewWriter(options ...WriterOption) ObjectWriter { result := &objectWriter{ repo: repo, - objectType: ObjectIDTypeStored, blockTracker: &blockTracker{}, } @@ -119,24 +121,30 @@ func (repo *repository) NewWriter(options ...WriterOption) ObjectWriter { } func (repo *repository) Open(objectID ObjectID) (ObjectReader, error) { - if start, length, baseID := objectID.SectionInfo(); baseID != "" { - baseReader, err := repo.Open(baseID) + // log.Printf("Repository::Open %v", objectID.String()) + // defer log.Printf("finished Repository::Open() %v", objectID.String()) + + if objectID.Section != nil { + if objectID.Section.Base == nil { + return nil, fmt.Errorf("invalid section base object") + } + + baseReader, err := repo.Open(*objectID.Section.Base) if err != nil { return nil, fmt.Errorf("cannot create base reader: %v", err) } - return newObjectSectionReader(start, length, baseReader) + return newObjectSectionReader(objectID.Section.Start, objectID.Section.Length, baseReader) } - r, err := repo.newRawReader(objectID) - if err != nil { - return nil, err - } + if objectID.Indirect > 0 { + r, err := repo.Open(removeIndirection(objectID)) + if err != nil { + return nil, err + } + defer r.Close() - if objectID.Type() == ObjectIDTypeList { - seekTable := make([]seekTableEntry, 0, 100) - - seekTable, err = repo.flattenListChunk(seekTable, objectID, r) + seekTable, err := repo.flattenListChunk(r) if err != nil { return nil, err } @@ -144,13 +152,32 @@ func (repo *repository) Open(objectID ObjectID) (ObjectReader, error) { totalLength := seekTable[len(seekTable)-1].endOffset() return &objectReader{ - storage: repo.storage, + repo: repo, seekTable: seekTable, totalLength: totalLength, }, nil } - return r, err + return repo.newRawReader(objectID) +} + +func dumpObject(repo *repository, oid ObjectID) { + oid.Indirect = 0 + log.Printf(" dumping %v", oid.String()) + defer log.Printf(" end of %v", oid.String()) + r, err := repo.Open(oid) + if err != nil { + log.Printf("failed to open %v: %v", oid, err) + return + } + defer r.Close() + + b, err := ioutil.ReadAll(r) + if err != nil { + log.Printf("failed to read %v: %v", oid, err) + return + } + log.Printf("%x", b) } // RepositoryOption controls the behavior of Repository. @@ -185,14 +212,11 @@ func NewRepository( f *Format, options ...RepositoryOption, ) (Repository, error) { - if f.Version != "1" { - return nil, fmt.Errorf("unsupported storage version: %v", f.Version) - } if f.MaxBlobSize < 100 { return nil, fmt.Errorf("MaxBlobSize is not set") } - sf := SupportedFormats.Find(f.ObjectFormat) + sf := SupportedFormats[f.ObjectFormat] if sf == nil { return nil, fmt.Errorf("unknown object format: %v", f.ObjectFormat) } @@ -203,7 +227,7 @@ func NewRepository( stats: &RepositoryStats{}, } - repo.idFormat = *sf + repo.idFormat = sf for _, o := range options { if err := o(repo); err != nil { @@ -212,13 +236,13 @@ func NewRepository( } } - repo.bufferManager = newBufferManager(repo.format.MaxBlobSize) + repo.bufferManager = newBufferManager(int(repo.format.MaxBlobSize)) return repo, nil } func (repo *repository) hashBuffer(data []byte) ([]byte, []byte) { - return repo.idFormat.hashBuffer(data, repo.format.Secret) + return repo.idFormat.HashBuffer(data, repo.format.Secret) } func (repo *repository) hashBufferForWriting(buffer *bytes.Buffer, prefix string) (ObjectID, storage.ReaderWithLength, error) { @@ -232,17 +256,18 @@ func (repo *repository) hashBufferForWriting(buffer *bytes.Buffer, prefix string contentHash, cryptoKey := repo.hashBuffer(data) if cryptoKey != nil { var err error - blockCipher, err = repo.idFormat.createCipher(cryptoKey) + blockCipher, err = repo.idFormat.CreateCipher(cryptoKey) if err != nil { - return "", nil, err + return NullObjectID, nil, err } } - var objectID ObjectID + objectID := ObjectID{ + StorageBlock: prefix + hex.EncodeToString(contentHash), + } + if len(cryptoKey) > 0 { - objectID = ObjectID(prefix + hex.EncodeToString(contentHash) + objectIDEncryptionInfoSeparator + hex.EncodeToString(cryptoKey)) - } else { - objectID = ObjectID(prefix + hex.EncodeToString(contentHash)) + objectID.Encryption = cryptoKey } atomic.AddInt32(&repo.stats.HashedBlocks, 1) @@ -268,71 +293,45 @@ func (repo *repository) hashBufferForWriting(buffer *bytes.Buffer, prefix string return objectID, blockReader, nil } -func (repo *repository) flattenListChunk( - seekTable []seekTableEntry, - listObjectID ObjectID, - rawReader io.Reader) ([]seekTableEntry, error) { +func (repo *repository) flattenListChunk(rawReader io.Reader) ([]IndirectObjectEntry, error) { - scanner := bufio.NewScanner(rawReader) + r := bufio.NewReader(rawReader) + pr := internal.NewProtoStreamReader(r, internal.ProtoStreamTypeIndirect) + var seekTable []IndirectObjectEntry - for scanner.Scan() { - c := scanner.Text() - comma := strings.Index(c, ",") - if comma <= 0 { - return nil, fmt.Errorf("unsupported entry '%v' in list '%s'", c, listObjectID) + for { + var oe IndirectObjectEntry + + err := pr.Read(&oe) + if err == io.EOF { + break } - length, err := strconv.ParseInt(c[0:comma], 10, 64) - - objectID, err := ParseObjectID(c[comma+1:]) if err != nil { - return nil, fmt.Errorf("unsupported entry '%v' in list '%s': %#v", c, listObjectID, err) + log.Printf("Failed to read proto: %v", err) + return nil, err } - switch objectID.Type() { - case ObjectIDTypeList: - subreader, err := repo.newRawReader(objectID) - if err != nil { - return nil, err - } - - seekTable, err = repo.flattenListChunk(seekTable, objectID, subreader) - if err != nil { - return nil, err - } - - case ObjectIDTypeStored: - var startOffset int64 - if len(seekTable) > 0 { - startOffset = seekTable[len(seekTable)-1].endOffset() - } else { - startOffset = 0 - } - - seekTable = append( - seekTable, - seekTableEntry{ - blockID: objectID.BlockID(), - startOffset: startOffset, - length: length, - }) - - default: - return nil, fmt.Errorf("unsupported entry '%v' in list '%v'", objectID, listObjectID) - - } + seekTable = append(seekTable, oe) } return seekTable, nil } +func removeIndirection(o ObjectID) ObjectID { + if o.Indirect <= 0 { + panic("removeIndirection() called on a direct object") + } + o.Indirect-- + return o +} + func (repo *repository) newRawReader(objectID ObjectID) (ObjectReader, error) { - inline := objectID.InlineData() - if inline != nil { - return newObjectReaderWithData(inline), nil + if objectID.Content != nil { + return newObjectReaderWithData(objectID.Content), nil } - blockID := objectID.BlockID() + blockID := objectID.StorageBlock payload, err := repo.storage.GetBlock(blockID) if err != nil { return nil, err @@ -341,19 +340,14 @@ func (repo *repository) newRawReader(objectID ObjectID) (ObjectReader, error) { atomic.AddInt32(&repo.stats.BlocksReadFromStorage, 1) atomic.AddInt64(&repo.stats.BytesReadFromStorage, int64(len(payload))) - if objectID.EncryptionInfo() == NoEncryption { - if err := repo.verifyChecksum(payload, objectID.BlockID()); err != nil { + if objectID.Encryption == nil { + if err := repo.verifyChecksum(payload, objectID.StorageBlock); err != nil { return nil, err } return newObjectReaderWithData(payload), nil } - cryptoKey, err := hex.DecodeString(string(objectID.EncryptionInfo())) - if err != nil { - return nil, errors.New("malformed encryption key") - } - - blockCipher, err := repo.idFormat.createCipher(cryptoKey) + blockCipher, err := repo.idFormat.CreateCipher(objectID.Encryption) if err != nil { return nil, errors.New("cannot create cipher") } @@ -366,7 +360,7 @@ func (repo *repository) newRawReader(objectID ObjectID) (ObjectReader, error) { // after decrypting the content. This serves as a checksum. atomic.AddInt64(&repo.stats.DecryptedBytes, int64(len(payload))) - if err := repo.verifyChecksum(payload, objectID.BlockID()); err != nil { + if err := repo.verifyChecksum(payload, objectID.StorageBlock); err != nil { return nil, err } diff --git a/repo/repository_test.go b/repo/repository_test.go index 802ab1a48..be3c5ad3b 100644 --- a/repo/repository_test.go +++ b/repo/repository_test.go @@ -1,28 +1,28 @@ package repo import ( + "bufio" "bytes" "crypto/md5" cryptorand "crypto/rand" "encoding/hex" "fmt" + "io" "io/ioutil" "math/rand" - "reflect" - "sort" - "strings" "testing" + "github.com/kopia/kopia/internal" "github.com/kopia/kopia/storage" "github.com/kopia/kopia/storage/storagetesting" ) func testFormat() *Format { return &Format{ - Version: "1", + Version: 1, MaxBlobSize: 200, MaxInlineBlobSize: 20, - ObjectFormat: "md5", + ObjectFormat: ObjectIDFormat_TESTONLY_MD5, } } @@ -56,16 +56,15 @@ func TestWriters(t *testing.T) { data []byte objectID ObjectID }{ - {[]byte{}, "B"}, - {[]byte("quick brown fox"), "Tquick brown fox"}, - {[]byte{1, 2, 3, 4}, "BAQIDBA"}, - {[]byte{10, 13, 9}, "T\n\r\t"}, - {[]byte{0xc2, 0x28}, "Bwig"}, // invalid UTF-8, will be represented as binary + {[]byte{}, NullObjectID}, + {[]byte("quick brown fox"), ObjectID{Content: []byte("quick brown fox")}}, + {[]byte{1, 2, 3, 4}, ObjectID{Content: []byte{1, 2, 3, 4}}}, + {[]byte{0xc2, 0x28}, ObjectID{Content: []byte{0xc2, 0x28}}}, { []byte("the quick brown fox jumps over the lazy dog"), - "DX77add1d5f41223d5582fca736a5cb335", + ObjectID{StorageBlock: "X77add1d5f41223d5582fca736a5cb335"}, }, - {make([]byte, 100), "DX6d0bb00954ceb7fbee436bb55a8397a9"}, // 100 zero bytes + {make([]byte, 100), ObjectID{StorageBlock: "X6d0bb00954ceb7fbee436bb55a8397a9"}}, // 100 zero bytes } for _, c := range cases { @@ -83,11 +82,11 @@ func TestWriters(t *testing.T) { continue } - if result != c.objectID { - t.Errorf("incorrect result for %v, expected: %v got: %v", c.data, c.objectID, result) + if !result.Equals(c.objectID) { + t.Errorf("incorrect result for %v, expected: %v got: %v", c.data, c.objectID.UIString(), result) } - if !c.objectID.Type().IsStored() { + if c.objectID.StorageBlock == "" { if len(data) != 0 { t.Errorf("unexpected data written to the storage: %v", data) } @@ -110,120 +109,74 @@ func TestWriterCompleteChunkInTwoWrites(t *testing.T) { writer.Write(bytes[0:50]) writer.Write(bytes[0:50]) result, err := writer.Result(false) - if string(result) != "DX6d0bb00954ceb7fbee436bb55a8397a9" { + if !result.Equals(ObjectID{StorageBlock: "X6d0bb00954ceb7fbee436bb55a8397a9"}) { t.Errorf("unexpected result: %v err: %v", result, err) } } -func TestWriterListChunk(t *testing.T) { - data, repo := setupTest(t) +func verifyIndirectBlock(t *testing.T, r Repository, oid ObjectID) { + for level := int32(0); level < oid.Indirect; level++ { + direct := oid + direct.Indirect = level - contentBytes := make([]byte, 250) - contentMd5Sum200 := getMd5Digest(contentBytes[0:200]) // hash of 200 zero bytes - contentMd5Sum50 := getMd5Digest(contentBytes[200:250]) // hash of 50 zero bytes - listChunkContent := []byte("200,D" + contentMd5Sum200 + "\n50,D" + contentMd5Sum50 + "\n") - listChunkObjectID := getMd5LObjectID(listChunkContent) + rd, err := r.Open(direct) + if err != nil { + t.Errorf("unable to open %v: %v", oid.String(), err) + return + } + defer rd.Close() - writer := repo.NewWriter() - writer.Write(contentBytes) - result, err := writer.Result(false) - if err != nil { - t.Errorf("error getting writer results: %v", err) + pr := internal.NewProtoStreamReader(bufio.NewReader(rd), internal.ProtoStreamTypeIndirect) + for { + var v IndirectObjectEntry + if err := pr.Read(&v); err != nil { + if err == io.EOF { + break + } + t.Errorf("err: %v", err) + break + } + } } - - if string(result) != listChunkObjectID { - t.Errorf("incorrect list chunk ID: %v, expected: %v", result, listChunkObjectID) - } - - // We have 3 chunks - 200 zero bytes, 50 zero bytes and the list. - verifyStorageBlocks(t, writer.StorageBlocks(), data, map[string][]byte{ - contentMd5Sum200: contentBytes[0:200], - contentMd5Sum50: contentBytes[0:50], - getMd5Digest(listChunkContent): listChunkContent, - }) } -func TestWriterListOfListsChunk(t *testing.T) { - data, repo := setupTest(t) - - contentBytes := make([]byte, 1400) - chunk1Id := getMd5CObjectID(contentBytes[0:200]) // hash of 200 zero bytes - - list1ChunkContent := []byte(strings.Repeat("200,"+chunk1Id+"\n", 5)) - list1ObjectID := fmt.Sprintf("%v,%v", len(list1ChunkContent), getMd5LObjectID(list1ChunkContent)) - - list2ChunkContent := []byte(strings.Repeat("200,"+chunk1Id+"\n", 2)) - list2ObjectID := fmt.Sprintf("%v,%v", len(list2ChunkContent), getMd5LObjectID(list2ChunkContent)) - - listOfListsChunkContent := []byte(list1ObjectID + "\n" + list2ObjectID + "\n") - listOfListsObjectID := getMd5LObjectID(listOfListsChunkContent) - - writer := repo.NewWriter() - writer.Write(contentBytes) - result, err := writer.Result(false) - if string(result) != listOfListsObjectID || err != nil { - t.Errorf("unexpected result: %v expected: %v, err: %v", result, listOfListsObjectID, err) +func TestIndirection(t *testing.T) { + cases := []struct { + dataLength int + expectedBlockCount int + expectedIndirection int32 + }{ + {dataLength: 200, expectedBlockCount: 1, expectedIndirection: 0}, + {dataLength: 250, expectedBlockCount: 3, expectedIndirection: 1}, + {dataLength: 1400, expectedBlockCount: 4, expectedIndirection: 2}, + {dataLength: 2000, expectedBlockCount: 5, expectedIndirection: 2}, + {dataLength: 3000, expectedBlockCount: 6, expectedIndirection: 2}, + {dataLength: 4000, expectedBlockCount: 9, expectedIndirection: 3}, + {dataLength: 10000, expectedBlockCount: 16, expectedIndirection: 3}, } - // We have 4 chunks - 200 zero bytes, 2 lists, and 1 list-of-lists. - verifyStorageBlocks(t, writer.StorageBlocks(), data, map[string][]byte{ - getMd5Digest(contentBytes[0:200]): contentBytes[0:200], - getMd5Digest(list1ChunkContent): list1ChunkContent, - getMd5Digest(list2ChunkContent): list2ChunkContent, - getMd5Digest(listOfListsChunkContent): listOfListsChunkContent, - }) -} + for _, c := range cases { + data, repo := setupTest(t) -func TestWriterListOfListsOfListsChunk(t *testing.T) { - data, repo := setupTest(t) + contentBytes := make([]byte, c.dataLength) - writtenData := make([]byte, 10000) - chunk1Id := getMd5CObjectID(writtenData[0:200]) // hash of 200 zero bytes + writer := repo.NewWriter() + writer.Write(contentBytes) + result, err := writer.Result(false) + if err != nil { + t.Errorf("error getting writer results: %v", err) + } - // First level list chunk has 5 C[] chunk IDs, because that's how many IDs fit in one chunk. - // (that number 200 was chosen for a reason, to make testing easy) - // - // 200,Cfbaf48ec981a5eecdb57b929fdd426e8\n - // 200,Cfbaf48ec981a5eecdb57b929fdd426e8\n - // 200,Cfbaf48ec981a5eecdb57b929fdd426e8\n - // 200,Cfbaf48ec981a5eecdb57b929fdd426e8\n - // 200,Cfbaf48ec981a5eecdb57b929fdd426e8\n - list1ChunkContent := []byte(strings.Repeat("200,"+chunk1Id+"\n", 5)) - list1ObjectID := fmt.Sprintf("%v,%v", len(list1ChunkContent), getMd5LObjectID(list1ChunkContent)) + if result.Indirect != c.expectedIndirection { + t.Errorf("incorrect indirection level: %v, expected %v", result.Indirect, c.expectedIndirection) + } - // Second level lists L[] chunks from the first level. They have all the same content - // because all lists are identical. - // 190,L52760f658059fef754f5deabdd01df93\n - // 190,L52760f658059fef754f5deabdd01df93\n - // 190,L52760f658059fef754f5deabdd01df93\n - // 190,L52760f658059fef754f5deabdd01df93\n - // 190,L52760f658059fef754f5deabdd01df93\n - list2ChunkContent := []byte(strings.Repeat(string(list1ObjectID)+"\n", 5)) - list2ObjectID := fmt.Sprintf("%v,%v", len(list2ChunkContent), getMd5LObjectID(list2ChunkContent)) + if len(data) != c.expectedBlockCount { + t.Errorf("unexpected block count: %v, expected %v", len(data), c.expectedBlockCount) + } - // Now those lists are also identical and represent 5000 bytes each, so - // the top-level list-of-lists-of-lists will have 2 entries: - // - // 190,Lb99b28e34c87e4934b4cc5631bb38ee8\n - // 190,Lb99b28e34c87e4934b4cc5631bb38ee8\n - list3ChunkContent := []byte(strings.Repeat(string(list2ObjectID)+"\n", 2)) - list3ObjectID := getMd5LObjectID(list3ChunkContent) - - writer := repo.NewWriter() - writer.Write(writtenData) - - result, err := writer.Result(false) - if string(result) != list3ObjectID { - t.Errorf("unexpected result: %v err: %v", result, err) + verifyIndirectBlock(t, repo, result) } - - // We have 4 data blocks representing 10000 bytes of zero. Not bad! - verifyStorageBlocks(t, writer.StorageBlocks(), data, map[string][]byte{ - getMd5Digest(writtenData[0:200]): writtenData[0:200], - getMd5Digest(list1ChunkContent): list1ChunkContent, - getMd5Digest(list2ChunkContent): list2ChunkContent, - getMd5Digest(list3ChunkContent): list3ChunkContent, - }) } func TestHMAC(t *testing.T) { @@ -231,7 +184,7 @@ func TestHMAC(t *testing.T) { content := bytes.Repeat([]byte{0xcd}, 50) s := testFormat() - s.ObjectFormat = "md5" + s.ObjectFormat = ObjectIDFormat_TESTONLY_MD5 s.Secret = []byte{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25} repo, err := NewRepository(storagetesting.NewMapStorage(data), s) @@ -241,7 +194,7 @@ func TestHMAC(t *testing.T) { w := repo.NewWriter() w.Write(content) result, err := w.Result(false) - if string(result) != "D697eaf0aca3a3aea3a75164746ffaa79" { + if result.UIString() != "D697eaf0aca3a3aea3a75164746ffaa79" { t.Errorf("unexpected result: %v err: %v", result, err) } } @@ -258,8 +211,6 @@ func TestReader(t *testing.T) { }{ {"B", []byte{}}, {"BAQIDBA", []byte{1, 2, 3, 4}}, - {"T", []byte{}}, - {"Tfoo\nbar", []byte("foo\nbar")}, {"Da76999788386641a3ec798554f1fe7e6", storedPayload}, {"S0,2,BAQIDBA", []byte{1, 2}}, {"S1,3,BAQIDBA", []byte{2, 3, 4}}, @@ -353,12 +304,12 @@ func TestEndToEndReadAndSeek(t *testing.T) { verify(t, repo, objectID, randomData, fmt.Sprintf("%v %v/%v", objectID, forceStored, size)) if size > 1 { - sectionID := NewSectionObjectID(0, int64(size/2), objectID) + sectionID := NewSectionObjectID(0, int64(size/2), &objectID) verify(t, repo, sectionID, randomData[0:10], fmt.Sprintf("%v %v/%v", sectionID, forceStored, size)) } if size > 1 { - sectionID := NewSectionObjectID(int64(1), int64(size-1), objectID) + sectionID := NewSectionObjectID(int64(1), int64(size-1), &objectID) verify(t, repo, sectionID, randomData[1:], fmt.Sprintf("%v %v/%v", sectionID, forceStored, size)) } } @@ -397,9 +348,9 @@ func verify(t *testing.T, repo Repository, objectID ObjectID, expectedData []byt } func TestFormats(t *testing.T) { - makeFormat := func(objectFormat string) *Format { + makeFormat := func(objectFormat ObjectIDFormat) *Format { return &Format{ - Version: "1", + Version: 1, ObjectFormat: objectFormat, Secret: []byte("key"), MaxBlobSize: 10000, @@ -411,88 +362,78 @@ func TestFormats(t *testing.T) { oids map[string]ObjectID }{ { - format: makeFormat("md5"), // MD5-HMAC with secret 'key' + format: makeFormat(ObjectIDFormat_TESTONLY_MD5), // MD5-HMAC with secret 'key' oids: map[string]ObjectID{ - "": "D63530468a04e386459855da0063b6596", - "The quick brown fox jumps over the lazy dog": "D80070713463e7749b90c2dc24911e275", + "": ObjectID{StorageBlock: "63530468a04e386459855da0063b6596"}, + "The quick brown fox jumps over the lazy dog": ObjectID{StorageBlock: "80070713463e7749b90c2dc24911e275"}, }, }, { format: &Format{ - Version: "1", - ObjectFormat: "md5", + Version: 1, + ObjectFormat: ObjectIDFormat_TESTONLY_MD5, MaxBlobSize: 10000, Secret: []byte{}, // HMAC with zero-byte secret }, oids: map[string]ObjectID{ - "": "D74e6f7298a9c2d168935f58c001bad88", - "The quick brown fox jumps over the lazy dog": "Dad262969c53bc16032f160081c4a07a0", + "": ObjectID{StorageBlock: "74e6f7298a9c2d168935f58c001bad88"}, + "The quick brown fox jumps over the lazy dog": ObjectID{StorageBlock: "ad262969c53bc16032f160081c4a07a0"}, }, }, { format: &Format{ - Version: "1", - ObjectFormat: "md5", + Version: 1, + ObjectFormat: ObjectIDFormat_TESTONLY_MD5, MaxBlobSize: 10000, Secret: nil, // non-HMAC version }, oids: map[string]ObjectID{ - "": "Dd41d8cd98f00b204e9800998ecf8427e", - "The quick brown fox jumps over the lazy dog": "D9e107d9d372bb6826bd81d3542a419d6", + "": ObjectID{StorageBlock: "d41d8cd98f00b204e9800998ecf8427e"}, + "The quick brown fox jumps over the lazy dog": ObjectID{ + StorageBlock: "9e107d9d372bb6826bd81d3542a419d6", + }, }, }, { - format: makeFormat("md5"), + format: makeFormat(ObjectIDFormat_TESTONLY_MD5), oids: map[string]ObjectID{ - "The quick brown fox jumps over the lazy dog": "D80070713463e7749b90c2dc24911e275", + "The quick brown fox jumps over the lazy dog": ObjectID{ + StorageBlock: "80070713463e7749b90c2dc24911e275", + }, }, }, { - format: makeFormat("sha1"), + format: makeFormat(ObjectIDFormat_UNENCYPTED_HMAC_SHA256), oids: map[string]ObjectID{ - "The quick brown fox jumps over the lazy dog": "Dde7c9b85b8b78aa6bc8a7a36f70a90701c9db4d9", + "The quick brown fox jumps over the lazy dog": ObjectID{ + StorageBlock: "f7bc83f430538424b13298e6aa6fb143ef4d59a14946175997479dbc2d1a3cd8", + }, }, }, { - format: makeFormat("sha256"), + format: makeFormat(ObjectIDFormat_UNENCYPTED_HMAC_SHA256_128), oids: map[string]ObjectID{ - "The quick brown fox jumps over the lazy dog": "Df7bc83f430538424b13298e6aa6fb143ef4d59a14946175997479dbc2d1a3cd8", + "The quick brown fox jumps over the lazy dog": ObjectID{ + StorageBlock: "18f1da557915937d2675055a87758d9b", + }, }, }, { - format: makeFormat("sha512"), + format: makeFormat(ObjectIDFormat_ENCRYPTED_HMAC_SHA512_384_AES256), oids: map[string]ObjectID{ - "The quick brown fox jumps over the lazy dog": "Db42af09057bac1e2d41708e48a902e09b5ff7f12ab428a4fe86653c73dd248fb82f948a549f7b791a5b41915ee4d1ec3935357e4e2317250d0372afa2ebeeb3a", + "The quick brown fox jumps over the lazy dog": ObjectID{ + StorageBlock: "d7f4727e2c0b39ae0f1e40cc96f60242", + Encryption: mustParseBase16("d5b7801841cea6fc592c5d3e1ae50700582a96cf35e1e554995fe4e03381c237"), + }, }, }, { - format: makeFormat("sha256-fold128-aes128"), + format: makeFormat(ObjectIDFormat_ENCRYPTED_HMAC_SHA512_AES256), oids: map[string]ObjectID{ - "The quick brown fox jumps over the lazy dog": "D028e74c630aee6400db4f84d49d1ed08.2825273c3344999e26081d99bcaf3f18", - }, - }, - { - format: makeFormat("sha256-aes128"), - oids: map[string]ObjectID{ - "The quick brown fox jumps over the lazy dog": "D26bb3fee9e83a7f6ff8397f33460cd0d24354b28ae2d41b6f2376fbe7db12005.2825273c3344999e26081d99bcaf3f18", - }, - }, - { - format: makeFormat("sha384-aes256"), - oids: map[string]ObjectID{ - "The quick brown fox jumps over the lazy dog": "D08edca658caee5ae86c9f1e4ccea26b1a43b0ddb04502c5f1b1cba051a756a9eba440bb6991b23323e37556ec5cf4f88.7a753377319650870f5cba47c30b608675542f206fa67d94dcbb9a0ccce7f467", - }, - }, - { - format: makeFormat("sha512-aes256"), - oids: map[string]ObjectID{ - "The quick brown fox jumps over the lazy dog": "Deee04ad71248ef4ffec5159558e80e2791e32299cfb79a1e063cc5f4a71cd61ca2c1b110e3ae2e83635a8626a3a27e4805eb745c40b5a4ebd4c9372602e5ab65.1c0b1b58ce05b7b8b05cfce27a485ddf97bde5159f6946357ec7795236f36a84", - }, - }, - { - format: makeFormat("sha512-fold128-aes192"), - oids: map[string]ObjectID{ - "The quick brown fox jumps over the lazy dog": "Dd829ad027ee4ff394f6a61615eb30d16.0ab00e0e0e156927f0be63372e8449d7bf2316c43d46e626", + "The quick brown fox jumps over the lazy dog": ObjectID{ + StorageBlock: "b42af09057bac1e2d41708e48a902e09b5ff7f12ab428a4fe86653c73dd248fb", + Encryption: mustParseBase16("82f948a549f7b791a5b41915ee4d1ec3935357e4e2317250d0372afa2ebeeb3a"), + }, }, }, } @@ -501,7 +442,7 @@ func TestFormats(t *testing.T) { data := map[string][]byte{} st := storagetesting.NewMapStorage(data) - t.Logf("verifying %#v", c.format) + t.Logf("verifying %v", c.format.String()) repo, err := NewRepository(st, c.format) if err != nil { t.Errorf("cannot create manager: %v", err) @@ -516,8 +457,8 @@ func TestFormats(t *testing.T) { if err != nil { t.Errorf("error: %v", err) } - if oid != v { - t.Errorf("invalid oid for #%v %v/%v: %v expected %v", caseIndex, c.format.ObjectFormat, k, oid, v) + if !oid.Equals(v) { + t.Errorf("invalid oid for #%v %v/%v:\ngot:\n%#v\nexpected:\n%#v", caseIndex, c.format.ObjectFormat, k, oid.UIString(), v.UIString()) } rc, err := repo.Open(oid) @@ -540,8 +481,8 @@ func TestInvalidEncryptionKey(t *testing.T) { data := map[string][]byte{} st := storagetesting.NewMapStorage(data) format := Format{ - Version: "1", - ObjectFormat: "sha512-aes256", + Version: 1, + ObjectFormat: ObjectIDFormat_ENCRYPTED_HMAC_SHA512_384_AES256, Secret: []byte("key"), MaxBlobSize: 1000, } @@ -571,47 +512,42 @@ func TestInvalidEncryptionKey(t *testing.T) { } // Key too short - rc, err = repo.Open(oid[0 : len(oid)-2]) + rc, err = repo.Open(replaceEncryption(oid, oid.Encryption[0:len(oid.Encryption)-2])) if err == nil || rc != nil { t.Errorf("expected error when opening malformed object") } // Key too long - rc, err = repo.Open(oid + "ff") + rc, err = repo.Open(replaceEncryption(oid, append(oid.Encryption, 0xFF))) if err == nil || rc != nil { t.Errorf("expected error when opening malformed object") } // Invalid key - lastByte, _ := hex.DecodeString(string(oid[len(oid)-2:])) - lastByte[0]++ - rc, err = repo.Open(oid[0:len(oid)-2] + ObjectID(hex.EncodeToString(lastByte))) + corruptedKey := append([]byte(nil), oid.Encryption...) + corruptedKey[0]++ + rc, err = repo.Open(replaceEncryption(oid, corruptedKey)) if err == nil || rc != nil { t.Errorf("expected error when opening malformed object: %v", err) } // Now corrupt the data - data[string(oid.BlockID())][0] ^= 1 + data[oid.StorageBlock][0] ^= 1 rc, err = repo.Open(oid) if err == nil || rc != nil { t.Errorf("expected error when opening object with corrupt data") } } -func verifyStorageBlocks(t *testing.T, actual []string, data, expectedData map[string][]byte) { - actualCopy := append([]string(nil), actual...) - sort.Strings(actualCopy) - var expected []string - for k := range expectedData { - expected = append(expected, k) - } - sort.Strings(expected) - - if !reflect.DeepEqual(expected, actualCopy) { - t.Errorf("updated blocks don't match. Expected: %#v, got %#v", expected, actualCopy) - } - - if !reflect.DeepEqual(expectedData, data) { - t.Errorf("storage data does not match, expected: %v, got %v", expectedData, data) - } +func replaceEncryption(oid ObjectID, newEncryption []byte) ObjectID { + oid.Encryption = newEncryption + return oid +} + +func mustParseBase16(s string) []byte { + b, err := hex.DecodeString(s) + if err != nil { + panic("invalid hex literal: " + s) + } + return b } diff --git a/vault/objectid_storage.go b/vault/objectid_storage.go index fe49b893a..e1ca9615b 100644 --- a/vault/objectid_storage.go +++ b/vault/objectid_storage.go @@ -6,6 +6,8 @@ import ( "encoding/hex" "fmt" + "github.com/golang/protobuf/proto" + "github.com/kopia/kopia/repo" ) @@ -16,14 +18,14 @@ const ( storedObjectIDLengthBytes = 8 ) -type objectIDData struct { - ObjectID string `json:"objectID"` -} - // SaveObjectID stores the given object ID in an encrypted vault item and returns a unique ID. func (vlt *Vault) SaveObjectID(oid repo.ObjectID) (string, error) { h := hmac.New(sha256.New, vlt.format.UniqueID) - h.Write([]byte(oid)) + bytes, err := proto.Marshal(&oid) + if err != nil { + return "", err + } + h.Write(bytes) sum := h.Sum(nil) for i := storedObjectIDLengthBytes; i < len(sum); i++ { sum[i%storedObjectIDLengthBytes] ^= sum[i] @@ -31,10 +33,7 @@ func (vlt *Vault) SaveObjectID(oid repo.ObjectID) (string, error) { sum = sum[0:storedObjectIDLengthBytes] key := StoredObjectIDPrefix + hex.EncodeToString(sum) - var d objectIDData - d.ObjectID = string(oid) - - if err := vlt.putJSON(key, &d); err != nil { + if err := vlt.Put(key, bytes); err != nil { return "", err } @@ -45,20 +44,26 @@ func (vlt *Vault) SaveObjectID(oid repo.ObjectID) (string, error) { func (vlt *Vault) GetObjectID(id string) (repo.ObjectID, error) { matches, err := vlt.List(id) if err != nil { - return "", err + return repo.NullObjectID, err } switch len(matches) { case 0: - return "", ErrItemNotFound + return repo.NullObjectID, ErrItemNotFound case 1: - var d objectIDData - if err := vlt.getJSON(matches[0], &d); err != nil { - return "", err + b, err := vlt.Get(matches[0]) + if err != nil { + return repo.NullObjectID, err } - return repo.ParseObjectID(d.ObjectID) + + var oid repo.ObjectID + if err := proto.Unmarshal(b, &oid); err != nil { + return repo.NullObjectID, err + } + + return oid, nil default: - return "", fmt.Errorf("ambiguous object ID: %v", id) + return repo.NullObjectID, fmt.Errorf("ambiguous object ID: %v", id) } } diff --git a/vault/vault.pb.go b/vault/vault.pb.go new file mode 100644 index 000000000..130480c1c --- /dev/null +++ b/vault/vault.pb.go @@ -0,0 +1,35 @@ +// Code generated by protoc-gen-go. +// source: vault/vault.proto +// DO NOT EDIT! + +/* +Package vault is a generated protocol buffer package. + +It is generated from these files: + vault/vault.proto + +It has these top-level messages: +*/ +package vault + +import proto "github.com/golang/protobuf/proto" +import fmt "fmt" +import math "math" + +// Reference imports to suppress errors if they are not otherwise used. +var _ = proto.Marshal +var _ = fmt.Errorf +var _ = math.Inf + +// This is a compile-time assertion to ensure that this generated file +// is compatible with the proto package it is being compiled against. +const _ = proto.ProtoPackageIsVersion1 + +var fileDescriptor0 = []byte{ + // 75 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0xe2, 0x12, 0x2c, 0x4b, 0x2c, 0xcd, + 0x29, 0xd1, 0x07, 0x93, 0x7a, 0x05, 0x45, 0xf9, 0x25, 0xf9, 0x42, 0xdc, 0xd9, 0xf9, 0x05, 0x99, + 0x89, 0x7a, 0x60, 0x21, 0x27, 0xb9, 0x28, 0x99, 0xf4, 0xcc, 0x92, 0x8c, 0xd2, 0x24, 0xbd, 0xe4, + 0xfc, 0x5c, 0x7d, 0xb0, 0x0c, 0x94, 0x04, 0xcb, 0x27, 0xb1, 0x81, 0xf5, 0x18, 0x03, 0x02, 0x00, + 0x00, 0xff, 0xff, 0x9a, 0x15, 0x10, 0x31, 0x48, 0x00, 0x00, 0x00, +} diff --git a/vault/vault.proto b/vault/vault.proto new file mode 100644 index 000000000..aacb4e565 --- /dev/null +++ b/vault/vault.proto @@ -0,0 +1,4 @@ +syntax = "proto3"; + +package kopia.vault; +option go_package = "github.com/kopia/kopia/vault"; diff --git a/vault/vault_test.go b/vault/vault_test.go index 238a3c3ac..aafc935ba 100644 --- a/vault/vault_test.go +++ b/vault/vault_test.go @@ -62,9 +62,9 @@ func verifyVault(t *testing.T, vaultPath string, repoPath string) { } repoFormat := &repo.Format{ - Version: "1", + Version: 1, MaxBlobSize: 1000000, - ObjectFormat: "sha256", + ObjectFormat: repo.ObjectIDFormat_UNENCYPTED_HMAC_SHA256, Secret: []byte{1, 2, 3}, } @@ -189,7 +189,7 @@ func verifyVault(t *testing.T, vaultPath string, repoPath string) { if err != nil { t.Errorf("error getting object ID: %v", err) } else { - if oid != oid1 { + if !oid.Equals(oid1) { t.Errorf("got invalid object ID: %v expected %v", oid, oid1) } } @@ -198,7 +198,7 @@ func verifyVault(t *testing.T, vaultPath string, repoPath string) { if err != nil { t.Errorf("error getting object ID: %v", err) } else { - if oid != oid2 { + if !oid.Equals(oid2) { t.Errorf("got invalid object ID: %v expected %v", oid, oid2) } }