diff --git a/.gitignore b/.gitignore index 4bf2d1d37..0dc13de08 100644 --- a/.gitignore +++ b/.gitignore @@ -3,3 +3,4 @@ .releases/ kopia-*.tar.gz kopia-*.zip +combined.cov diff --git a/Makefile b/Makefile index da871ae25..ae9f64c3c 100644 --- a/Makefile +++ b/Makefile @@ -17,11 +17,20 @@ install: build: go build github.com/kopia/kopia/... +escape-analysis: + go build -gcflags '-m -l' github.com/kopia/kopia/... + +protos: + protoc --go_out=../../.. fs/*.proto + protoc --go_out=../../.. repo/*.proto + protoc --go_out=../../.. vault/*.proto + protoc --go_out=../../.. internal/*.proto + play: go run cmd/playground/main.go lint: - golint github.com/kopia/kopia/... + -golint github.com/kopia/kopia/... | grep -v .pb.go vet: go tool vet -all . diff --git a/backup/generator.go b/backup/generator.go index e35d58143..7322babb4 100644 --- a/backup/generator.go +++ b/backup/generator.go @@ -25,10 +25,10 @@ func (bg *backupGenerator) Backup(m *Manifest, old *Manifest) error { uploader := fs.NewUploader(bg.repo) m.StartTime = time.Now() - var hashCacheID repo.ObjectID + var hashCacheID *repo.ObjectID if old != nil { - hashCacheID = repo.ObjectID(old.HashCacheID) + hashCacheID = &old.HashCacheID } entry, err := fs.NewFilesystemEntry(m.Source, nil) @@ -46,8 +46,8 @@ func (bg *backupGenerator) Backup(m *Manifest, old *Manifest) error { if err != nil { return err } - m.RootObjectID = string(r.ObjectID) - m.HashCacheID = string(r.ManifestID) + m.RootObjectID = r.ObjectID + m.HashCacheID = r.ManifestID return nil } diff --git a/backup/manifest.go b/backup/manifest.go index 2133099c0..fa1594d98 100644 --- a/backup/manifest.go +++ b/backup/manifest.go @@ -5,6 +5,8 @@ import ( "encoding/hex" "io" "time" + + "github.com/kopia/kopia/repo" ) // Manifest stores information about single backup. @@ -16,10 +18,10 @@ type Manifest struct { UserName string `json:"userName"` Description string `json:"description"` - Handle string `json:"handle"` - Source string `json:"source"` - RootObjectID string `json:"root"` - HashCacheID string `json:"hashCache"` + Handle string `json:"handle"` + Source string `json:"source"` + RootObjectID repo.ObjectID `json:"root"` + HashCacheID repo.ObjectID `json:"hashCache"` FileCount int64 `json:"fileCount"` DirectoryCount int64 `json:"dirCount"` diff --git a/cmd/kopia/command_backup.go b/cmd/kopia/command_backup.go index bc4d65f7c..46a091e91 100644 --- a/cmd/kopia/command_backup.go +++ b/cmd/kopia/command_backup.go @@ -113,7 +113,7 @@ func runBackupCommand(context *kingpin.ParseContext) error { return fmt.Errorf("cannot save manifest: %v", err) } - log.Printf("Root: %v", manifest.RootObjectID) + log.Printf("Root: %v", manifest.RootObjectID.UIString()) log.Printf("Key: %v", handleID) } diff --git a/cmd/kopia/command_create.go b/cmd/kopia/command_create.go index 409d738ba..0042a2076 100644 --- a/cmd/kopia/command_create.go +++ b/cmd/kopia/command_create.go @@ -15,7 +15,7 @@ import ( var ( createCommand = app.Command("create", "Create new vault and repository.") createCommandRepository = createCommand.Flag("repository", "Repository path.").Default("colocated").String() - createObjectFormat = createCommand.Flag("repo-format", "Format of repository objects.").PlaceHolder("FORMAT").Default("sha256-fold160-aes128").Enum(supportedObjectFormats()...) + createObjectFormat = createCommand.Flag("repo-format", "Format of repository objects.").PlaceHolder("FORMAT").Default(repo.DefaultObjectFormat.Name()).Enum(supportedObjectFormats()...) createMaxBlobSize = createCommand.Flag("max-blob-size", "Maximum size of a data chunk.").PlaceHolder("KB").Default("20480").Int() createInlineBlobSize = createCommand.Flag("inline-blob-size", "Maximum size of an inline data chunk.").PlaceHolder("KB").Default("32").Int() @@ -43,15 +43,20 @@ func vaultFormat() (*vault.Format, error) { } func repositoryFormat() (*repo.Format, error) { - f := &repo.Format{ - Version: "1", - Secret: make([]byte, 32), - MaxBlobSize: *createMaxBlobSize * 1024, - MaxInlineBlobSize: *createInlineBlobSize * 1024, - ObjectFormat: *createObjectFormat, + objectIDFormat, err := repo.ParseObjectIDFormat(*createObjectFormat) + if err != nil { + return nil, err } - _, err := io.ReadFull(rand.Reader, f.Secret) + f := &repo.Format{ + Version: 1, + Secret: make([]byte, 32), + MaxBlobSize: int32(*createMaxBlobSize * 1024), + MaxInlineBlobSize: int32(*createInlineBlobSize * 1024), + ObjectFormat: objectIDFormat, + } + + _, err = io.ReadFull(rand.Reader, f.Secret) if err != nil { return nil, err } @@ -100,7 +105,7 @@ func runCreateCommand(context *kingpin.ParseContext) error { } fmt.Printf( - "Initializing repository in with format '%v' and maximum object size %v.\n", + "Initializing repository in with format %v and maximum object size %v.\n", repoFormat.ObjectFormat, repoFormat.MaxBlobSize) @@ -151,7 +156,7 @@ func supportedVaultEncryptionFormats() []string { func supportedObjectFormats() []string { var r []string for _, o := range repo.SupportedFormats { - r = append(r, o.Name) + r = append(r, o.Name()) } return r } diff --git a/cmd/kopia/command_ls.go b/cmd/kopia/command_ls.go index e68ecab24..1c9861430 100644 --- a/cmd/kopia/command_ls.go +++ b/cmd/kopia/command_ls.go @@ -6,7 +6,6 @@ import ( "strings" "github.com/kopia/kopia/fs" - "github.com/kopia/kopia/repo" kingpin "gopkg.in/alecthomas/kingpin.v2" ) @@ -69,17 +68,22 @@ func listDirectory(prefix string, entries fs.Entries, longFormat bool) { var info string if longFormat { var oid string - if m.ObjectID.Type() == repo.ObjectIDTypeBinary { - oid = "" - } else if m.ObjectID.Type() == repo.ObjectIDTypeText { - oid = "" + if m.ObjectId.Content != nil { + oid = "" } else { - oid = string(m.ObjectID) + oid = m.ObjectId.UIString() } - info = fmt.Sprintf("%v %9d %v %-"+maxNameLenString+"s %v", m.FileMode, m.FileSize, m.ModTime.Local().Format("02 Jan 06 15:04:05"), m.Name, oid) + info = fmt.Sprintf( + "%v %9d %v %-"+maxNameLenString+"s %v", + m.FileMode(), + m.FileSize, + m.ModTime().Local().Format("02 Jan 06 15:04:05"), + m.Name, + oid, + ) } else { var suffix string - if m.FileMode.IsDir() { + if m.FileMode().IsDir() { suffix = "/" } diff --git a/cmd/kopia/command_show.go b/cmd/kopia/command_show.go index 75a7ec692..b992487c9 100644 --- a/cmd/kopia/command_show.go +++ b/cmd/kopia/command_show.go @@ -1,11 +1,19 @@ package main import ( + "bufio" "bytes" "encoding/json" + "fmt" + "io" "io/ioutil" "os" + "github.com/golang/protobuf/proto" + "github.com/kopia/kopia/fs" + + "github.com/kopia/kopia/internal" + "github.com/kopia/kopia/repo" kingpin "gopkg.in/alecthomas/kingpin.v2" @@ -54,8 +62,8 @@ func showObject(mgr repo.Repository, oid repo.ObjectID) error { format := "raw" - if len(rawdata) > 2 && rawdata[0] == '{' && rawdata[len(rawdata)-1] == '}' { - format = "json" + if len(rawdata) >= 8 { + format = detectProtoFormat(rawdata[0:8]) } if *showJSON { @@ -73,12 +81,49 @@ func showObject(mgr repo.Repository, oid repo.ObjectID) error { json.Indent(&buf, rawdata, "", " ") os.Stdout.Write(buf.Bytes()) + case "dir": + return prettyPrintProtoStream(rawdata, internal.ProtoStreamTypeDir, &fs.EntryMetadata{}) + case "indirect": + return prettyPrintProtoStream(rawdata, internal.ProtoStreamTypeIndirect, &repo.IndirectObjectEntry{}) + case "hashcache": + return prettyPrintProtoStream(rawdata, internal.ProtoStreamTypeHashCache, &fs.HashCacheEntry{}) + default: os.Stdout.Write(rawdata) } return nil } +func detectProtoFormat(b []byte) string { + switch { + case bytes.Equal(b, internal.ProtoStreamTypeDir): + return "dir" + case bytes.Equal(b, internal.ProtoStreamTypeIndirect): + return "indirect" + case bytes.Equal(b, internal.ProtoStreamTypeHashCache): + return "hashcache" + } + return "raw" +} + +func prettyPrintProtoStream(data []byte, header []byte, pb proto.Message) error { + r := internal.NewProtoStreamReader(bufio.NewReader(bytes.NewReader(data)), header) + i := 0 + for { + err := r.Read(pb) + if err == io.EOF { + return nil + } + if err != nil { + return err + } + + fmt.Printf("[entry #%v]\n", i) + fmt.Println(proto.MarshalTextString(pb)) + i++ + } +} + func init() { showCommand.Action(runShowCommand) } diff --git a/cmd/kopia/objref.go b/cmd/kopia/objref.go index c24aebf94..7f361a847 100644 --- a/cmd/kopia/objref.go +++ b/cmd/kopia/objref.go @@ -13,7 +13,7 @@ import ( func parseObjectID(id string, vlt *vault.Vault) (repo.ObjectID, error) { head, tail := splitHeadTail(id) if len(head) == 0 { - return "", fmt.Errorf("invalid object ID: %v", id) + return repo.NullObjectID, fmt.Errorf("invalid object ID: %v", id) } if !strings.HasPrefix(id, vault.StoredObjectIDPrefix) { @@ -22,7 +22,7 @@ func parseObjectID(id string, vlt *vault.Vault) (repo.ObjectID, error) { oid, err := vlt.GetObjectID(head) if err != nil { - return "", fmt.Errorf("can't retrieve vault object ID %v: %v", head, err) + return repo.NullObjectID, fmt.Errorf("can't retrieve vault object ID %v: %v", head, err) } if tail == "" { @@ -31,12 +31,12 @@ func parseObjectID(id string, vlt *vault.Vault) (repo.ObjectID, error) { r, err := vlt.OpenRepository() if err != nil { - return "", fmt.Errorf("cannot open repository: %v", err) + return repo.NullObjectID, fmt.Errorf("cannot open repository: %v", err) } dir := fs.NewRepositoryDirectory(r, oid) if err != nil { - return "", err + return repo.NullObjectID, err } return parseNestedObjectID(dir, tail) @@ -49,24 +49,24 @@ func parseNestedObjectID(startingDir fs.Directory, id string) (repo.ObjectID, er for head != "" { dir, ok := current.(fs.Directory) if !ok { - return "", fmt.Errorf("entry not found '%v': parent is not a directory", head) + return repo.NullObjectID, fmt.Errorf("entry not found '%v': parent is not a directory", head) } entries, err := dir.Readdir() if err != nil { - return "", err + return repo.NullObjectID, err } e := entries.FindByName(head) if e == nil { - return "", fmt.Errorf("entry not found: '%v'", head) + return repo.NullObjectID, fmt.Errorf("entry not found: '%v'", head) } current = e head, tail = splitHeadTail(tail) } - return current.Metadata().ObjectID, nil + return *current.Metadata().ObjectId, nil } func splitHeadTail(id string) (string, string) { diff --git a/cmd/playground/main.go b/cmd/playground/main.go new file mode 100644 index 000000000..493013666 --- /dev/null +++ b/cmd/playground/main.go @@ -0,0 +1,77 @@ +package main + +import ( + "bufio" + "bytes" + "log" + + "github.com/kopia/kopia/internal" + "github.com/kopia/kopia/repo" + "github.com/kopia/kopia/storage/storagetesting" +) + +func dumpListBlock(prefix string, r repo.Repository, oid repo.ObjectID) { + ind := oid.Indirect + oid.Indirect = 0 + log.Printf("%vdumping %v", prefix, oid.String()) + rd, err := r.Open(oid) + if err != nil { + log.Fatalf("unable to open %v: %v", oid.String(), err) + } + defer rd.Close() + + var objects []repo.ObjectID + + pr := internal.NewProtoStreamReader(bufio.NewReader(rd), internal.ProtoStreamTypeIndirect) + for { + var v repo.IndirectObjectEntry + if err := pr.Read(&v); err != nil { + log.Printf("err: %v", err) + break + } + log.Printf("%v [%v] entry: %v", prefix, oid.String(), v.String()) + if ind > 0 { + objects = append(objects, *v.Object) + } + } + + for _, o := range objects { + dumpListBlock(prefix+" ", r, o) + } +} + +func main() { + d := map[string][]byte{} + + s := storagetesting.NewMapStorage(d) + f := repo.Format{ + Version: 1, + MaxBlobSize: 100, + ObjectFormat: repo.ObjectIDFormat_TESTONLY_MD5, + } + r, err := repo.NewRepository(s, &f) + if err != nil { + log.Fatalf("Err: %v", err) + } + w := r.NewWriter(repo.WithDescription("tester")) + + w.Write(bytes.Repeat([]byte("a"), 5000)) + result, err := w.Result(true) + log.Printf("result: %v", result.UIString()) + + //dumpListBlock("", r, result) + + log.Printf("attempting to open") + + rdr, err := r.Open(result) + if err != nil { + log.Fatalf("open: %v", err) + } + log.Printf("len: %v\n", rdr.Length()) + log.Printf("----") + // for k, v := range d { + // log.Printf("%v = %v", k, string(v)) + // } + + rdr.Close() +} diff --git a/fs/dir_json.go b/fs/dir_json.go index 456f73a2e..786e02a88 100644 --- a/fs/dir_json.go +++ b/fs/dir_json.go @@ -1,407 +1,79 @@ package fs import ( - "encoding/json" + "bufio" "fmt" "io" - "os" - "strconv" - "strings" - "time" + + "github.com/kopia/kopia/internal" "github.com/kopia/kopia/repo" ) -const modeChars = "dalTLDpSugct" - -type jsonDirectoryEntry struct { - Name string `json:"name"` - Type string `json:"type,omitempty"` - Permissions string `json:"perm,omitempty"` - Size string `json:"size,omitempty"` - Time *time.Time `json:"mtime,omitempty"` - Owner string `json:"owner,omitempty"` - ObjectID string `json:"oid,omitempty"` - JSONContent json.RawMessage `json:"content,omitempty"` - SubEntries []jsonDirectoryEntry `json:"entries,omitempty"` -} - -func (em *EntryMetadata) fromJSON(jde *jsonDirectoryEntry) error { - if jde.Name == "" { - return fmt.Errorf("empty entry name") - } - em.Name = jde.Name - - if jde.Permissions != "" { - if m, err := parseFilePermissions(jde.Permissions); err == nil { - em.FileMode |= m - } else { - return fmt.Errorf("invalid permissions: '%v'", jde.Permissions) - } - } - - if jde.Type != "" { - if m, err := parseFileMode(jde.Type); err == nil { - em.FileMode |= m - } else { - return fmt.Errorf("invalid type: '%v'", jde.Type) - } - } - - if jde.Time != nil { - em.ModTime = *jde.Time - } - - if jde.Owner != "" { - if c, err := fmt.Sscanf(jde.Owner, "%d:%d", &em.OwnerID, &em.GroupID); err != nil || c != 2 { - return fmt.Errorf("invalid owner: %v", err) - } - } - - if jde.JSONContent != nil { - em.ObjectID = repo.NewInlineObjectID([]byte(jde.JSONContent)) - } else { - em.ObjectID = repo.ObjectID(jde.ObjectID) - } - - if jde.Size != "" { - if s, err := strconv.ParseInt(jde.Size, 10, 64); err == nil { - em.FileSize = s - } else { - return fmt.Errorf("invalid size: %v", err) - } - } - return nil -} - -// parseFileModeAndPermissions converts file mode string to os.FileMode -func parseFileModeAndPermissions(s string) (os.FileMode, error) { - colon := strings.IndexByte(s, ':') - if colon < 0 { - return parseFilePermissions(s) - } - - var mode os.FileMode - - if m, err := parseFileMode(s[0:colon]); err == nil { - mode |= m - } else { - return 0, err - } - - if m, err := parseFilePermissions(s[colon+1:]); err == nil { - mode |= m - } else { - return 0, err - } - - return mode, nil -} - -func parseFileMode(s string) (os.FileMode, error) { - var mode os.FileMode - for _, c := range s { - switch c { - case 'd': - mode |= os.ModeDir - case 'a': - mode |= os.ModeAppend - case 'l': - mode |= os.ModeExclusive - case 'T': - mode |= os.ModeTemporary - case 'L': - mode |= os.ModeSymlink - case 'D': - mode |= os.ModeDevice - case 'p': - mode |= os.ModeNamedPipe - case 'S': - mode |= os.ModeSocket - case 'u': - mode |= os.ModeSetuid - case 'g': - mode |= os.ModeSetgid - case 'c': - mode |= os.ModeCharDevice - case 't': - mode |= os.ModeSticky - default: - return 0, fmt.Errorf("unsupported mode: '%v'", c) - } - } - return mode, nil -} - -func parseFilePermissions(perm string) (os.FileMode, error) { - s, err := strconv.ParseUint(perm, 8, 32) - if err != nil { - return 0, err - } - return os.FileMode(s), nil -} - type directoryWriter struct { - io.Closer - - writer io.Writer - buf []byte - separator []byte - - lastNameWritten string + w *internal.ProtoStreamWriter } func (dw *directoryWriter) WriteEntry(e *EntryMetadata, children []*EntryMetadata) error { - if dw.lastNameWritten != "" { - if isLessOrEqual(e.Name, dw.lastNameWritten) { - return fmt.Errorf("out-of-order directory entry, previous '%v' current '%v'", dw.lastNameWritten, e.Name) - } - dw.lastNameWritten = e.Name - } - - jde := e.toJSONEntry() - - if len(children) > 0 { - jde.SubEntries = make([]jsonDirectoryEntry, len(children)) - for i, se := range children { - jde.SubEntries[i] = se.toJSONEntry() - } - } - - v, _ := json.Marshal(&jde) - - dw.writer.Write(dw.separator) - dw.writer.Write(v) - dw.separator = []byte(",") - - return nil + return dw.w.Write(e) } -func (em *EntryMetadata) toJSONEntry() jsonDirectoryEntry { - var jde jsonDirectoryEntry - jde.Name = em.Name - - if (em.FileMode & os.ModeType) != 0 { - jde.Type = formatMode(em.FileMode) - } - - if (em.FileMode & os.ModePerm) != 0 { - jde.Permissions = formatPermissions(em.FileMode) - } - - if em.OwnerID > 0 || em.GroupID > 0 { - jde.Owner = fmt.Sprintf("%d:%d", em.OwnerID, em.GroupID) - } - - if !em.ModTime.IsZero() { - utc := em.ModTime.UTC() - jde.Time = &utc - } - - if em.ObjectID != "" { - inline := em.ObjectID.InlineData() - if len(inline) >= 2 && inline[0] == '{' && inline[len(inline)-1] == '}' { - m := map[string]interface{}{} - - if json.Unmarshal(inline, &m) == nil { - jde.JSONContent = json.RawMessage(inline) - } - } - } - - if jde.JSONContent == nil { - jde.ObjectID = string(em.ObjectID) - } - - if em.FileMode.IsRegular() { - jde.Size = strconv.FormatInt(em.FileSize, 10) - } - - return jde -} - -func formatMode(m os.FileMode) string { - const str = "dalTLDpSugct" - var buf [32]byte - w := 0 - for i, c := range str { - if m&(1< 0 { - bundle := make([]*EntryMetadata, 0, len(e.SubEntries)) + for _, e := range source { + if len(e.BundledChildren) > 0 { + bundle := e.BundledChildren + e.BundledChildren = nil var currentOffset int64 - var bundleEntry EntryMetadata - if err := bundleEntry.fromJSON(&e); err != nil { - return nil, err + for _, child := range bundle { + id := repo.NewSectionObjectID(currentOffset, child.FileSize, e.ObjectId) + child.ObjectId = &id + currentOffset += child.FileSize } - for _, s := range e.SubEntries { - var subEntry EntryMetadata - if err := subEntry.fromJSON(&s); err != nil { - return nil, err - } - subEntry.ObjectID = repo.NewSectionObjectID(currentOffset, subEntry.FileSize, entryMetadata.ObjectID) - currentOffset += subEntry.FileSize - bundle = append(bundle, &subEntry) - } - - if currentOffset != entryMetadata.FileSize { - return nil, fmt.Errorf("inconsistent size of '%v': %v (got %v)", entryMetadata.Name, entryMetadata.FileSize, currentOffset) + if currentOffset != e.FileSize { + return nil, fmt.Errorf("inconsistent size of '%v': %v (got %v)", e.Name, e.FileSize, currentOffset) } bundles = append(bundles, bundle) } else { - entries = append(entries, &entryMetadata) + entries = append(entries, e) } } diff --git a/fs/dir_test.go b/fs/dir_test.go index fc89e958c..207e8f587 100644 --- a/fs/dir_test.go +++ b/fs/dir_test.go @@ -1,11 +1,12 @@ package fs import ( - "os" "reflect" "strings" "testing" "time" + + "github.com/kopia/kopia/repo" ) var ( @@ -14,147 +15,143 @@ var ( time3 = mustParseTimestamp("2016-04-02T02:36:19Z") ) -func TestEmptyDirectory(t *testing.T) { - data := strings.Join( - []string{ - `{`, - `"format":{"version":1},`, - `"entries":[]`, - `}`, - }, "") - - expectedEntries := []*EntryMetadata{} - - verifyDirectory(t, data, expectedEntries) -} - -func TestDirectoryWithOnlyBundle(t *testing.T) { - data := strings.Join( - []string{ - `{`, - `"format":{"version":1},`, - `"entries":[`, - `{"name":"bundle1","size":"170","perm":"0","oid":"D5555","entries":[`, - `{"name":"a1","perm":"500","mtime":"2016-04-06T02:34:10Z","owner":"500:100","size":"50"},`, - `{"name":"z1","perm":"500","mtime":"2016-04-06T02:34:10Z","owner":"500:100","size":"120"}`, - `]}`, - `]}`, - }, "") - - expectedEntries := []*EntryMetadata{ - &EntryMetadata{Name: "a1", FileMode: 0500, ModTime: time1, OwnerID: 500, GroupID: 100, FileSize: 50, ObjectID: "S0,50,D5555"}, - &EntryMetadata{Name: "z1", FileMode: 0500, ModTime: time1, OwnerID: 500, GroupID: 100, FileSize: 120, ObjectID: "S50,120,D5555"}, +func TestFlattenBundles(t *testing.T) { + base := &repo.ObjectID{StorageBlock: "5555"} + sources := []*EntryMetadata{ + &EntryMetadata{ + Name: "bundle1", + FileSize: 170, + ObjectId: base, + BundledChildren: []*EntryMetadata{ + &EntryMetadata{Name: "a1", FileSize: 50}, + &EntryMetadata{Name: "z1", FileSize: 120}, + }, + }, } - verifyDirectory(t, data, expectedEntries) -} - -func TestInconsistentBundleSize(t *testing.T) { - data := strings.Join( - []string{ - `{`, - `"format":{"version":1},`, - `"entries":[`, - `{"name":"bundle1","size":"170","perm":"0","oid":"D5555","entries":[`, - `{"name":"a1","perm":"500","mtime":"2016-04-06T02:34:10Z","owner":"500:100","size":"51"},`, - `{"name":"z1","perm":"500","mtime":"2016-04-06T02:34:10Z","owner":"500:100","size":"120"}`, - `]}`, - `]}`, - }, "") - verifyDirectoryError(t, data, "inconsistent size of 'bundle1': 170 (got 171)") -} - -func TestInvalidBundleHeaderData(t *testing.T) { - data := strings.Join( - []string{ - `{`, - `"format":{"version":1},`, - `"entries":[`, - `{"name":"bundle1","size":"170","perm":"x","oid":"D5555","entries":[`, - `{"name":"z1","perm":"500","mtime":"2016-04-06T02:34:10Z","owner":"500:100","size":"120"}`, - `]}`, - `]}`, - }, "") - verifyDirectoryError(t, data, "invalid permissions: 'x'") -} - -func TestInvalidBundleEntryData(t *testing.T) { - data := strings.Join( - []string{ - `{`, - `"format":{"version":1},`, - `"entries":[`, - `{"name":"bundle1","size":"170","perm":"0","oid":"D5555","entries":[`, - `{"perm":"500","mtime":"2016-04-06T02:34:10Z","owner":"500:100","size":"51"},`, - `{"name":"z1","perm":"500","mtime":"2016-04-06T02:34:10Z","owner":"500:100","size":"120"}`, - `]}`, - `]}`, - }, "") - verifyDirectoryError(t, data, "empty entry name") -} -func TestDirectoryWithoutBundle(t *testing.T) { - data := strings.Join( - []string{ - `{`, - `"format":{"version":1},`, - `"entries":[`, - `{"name":"constants.go","perm":"420","size":"13","mtime":"2016-04-02T02:36:19Z","owner":"500:100","oid":"D5123"}`, - `]}`, - }, "") - - expectedEntries := []*EntryMetadata{ - &EntryMetadata{Name: "constants.go", FileMode: 0420, ModTime: time3, OwnerID: 500, GroupID: 100, FileSize: 13, ObjectID: "D5123"}, - } - - verifyDirectory(t, data, expectedEntries) -} - -func TestDirectoryWithThreeBundles(t *testing.T) { - data := strings.Join( - []string{ - `{`, - `"format":{"version":1},`, - `"entries":[`, - `{"name":"bundle1","size":"170","perm":"0","oid":"D5555","entries":[`, - `{"name":"a1","perm":"500","mtime":"2016-04-06T02:34:10Z","owner":"500:100","size":"50"},`, - `{"name":"z1","perm":"500","mtime":"2016-04-06T02:34:10Z","owner":"500:100","size":"120"}`, - `]},`, - `{"name":"config.go","perm":"420","size":"937","mtime":"2016-04-02T02:39:44.123456789Z","owner":"500:100","oid":"D4321"},`, - `{"name":"constants.go","perm":"420","size":"13","mtime":"2016-04-02T02:36:19Z","owner":"500:100","oid":"D5123"},`, - `{"name":"subdir","type":"d","perm":"755","mtime":"2016-04-06T02:34:10Z","owner":"500:100","oid":"D1234"},`, - `{"name":"bundle3","size":"7","perm":"0","oid":"D8888","entries":[`, - `{"name":"a3","perm":"500","mtime":"2016-04-06T02:34:10Z","owner":"500:100","size":"5"},`, - `{"name":"z3","perm":"500","mtime":"2016-04-06T02:34:10Z","owner":"500:100","size":"2"}`, - `]},`, - `{"name":"bundle2","size":"170","perm":"0","oid":"D6666","entries":[`, - `{"name":"a2","perm":"500","mtime":"2016-04-06T02:34:10Z","owner":"500:100","size":"150"},`, - `{"name":"z2","perm":"500","mtime":"2016-04-06T02:34:10Z","owner":"500:100","size":"20"}`, - `]}`, - `]}`, - }, "") - - expectedEntries := []*EntryMetadata{ - &EntryMetadata{Name: "a1", FileMode: 0500, ModTime: time1, OwnerID: 500, GroupID: 100, FileSize: 50, ObjectID: "S0,50,D5555"}, - &EntryMetadata{Name: "a2", FileMode: 0500, ModTime: time1, OwnerID: 500, GroupID: 100, FileSize: 150, ObjectID: "S0,150,D6666"}, - &EntryMetadata{Name: "a3", FileMode: 0500, ModTime: time1, OwnerID: 500, GroupID: 100, FileSize: 5, ObjectID: "S0,5,D8888"}, - &EntryMetadata{Name: "config.go", FileMode: 0420, ModTime: time2, OwnerID: 500, GroupID: 100, FileSize: 937, ObjectID: "D4321"}, - &EntryMetadata{Name: "constants.go", FileMode: 0420, ModTime: time3, OwnerID: 500, GroupID: 100, FileSize: 13, ObjectID: "D5123"}, - &EntryMetadata{Name: "subdir", FileMode: os.ModeDir | 0755, ModTime: time1, OwnerID: 500, GroupID: 100, ObjectID: "D1234"}, - &EntryMetadata{Name: "z1", FileMode: 0500, ModTime: time1, OwnerID: 500, GroupID: 100, FileSize: 120, ObjectID: "S50,120,D5555"}, - &EntryMetadata{Name: "z2", FileMode: 0500, ModTime: time1, OwnerID: 500, GroupID: 100, FileSize: 20, ObjectID: "S150,20,D6666"}, - &EntryMetadata{Name: "z3", FileMode: 0500, ModTime: time1, OwnerID: 500, GroupID: 100, FileSize: 2, ObjectID: "S5,2,D8888"}, - } - - verifyDirectory(t, data, expectedEntries) -} - -func verifyDirectory(t *testing.T, data string, expectedEntries []*EntryMetadata) { - entries, err := readDirectoryMetadataEntries(strings.NewReader(data)) + entries, err := flattenBundles(sources) if err != nil { t.Errorf("can't read directory entries: %v", err) return } + expectedEntries := []*EntryMetadata{ + &EntryMetadata{Name: "a1", FileSize: 50, ObjectId: &repo.ObjectID{Section: &repo.ObjectID_Section{ + Start: 0, + Length: 50, + Base: base, + }}}, + &EntryMetadata{Name: "z1", FileSize: 120, ObjectId: &repo.ObjectID{Section: &repo.ObjectID_Section{ + Start: 50, + Length: 120, + Base: base, + }}}, + } + + verifyDirectory(t, entries, expectedEntries) +} + +func TestFlattenBundlesInconsistentBundleSize(t *testing.T) { + sources := []*EntryMetadata{ + &EntryMetadata{ + Name: "bundle1", + FileSize: 171, + ObjectId: &repo.ObjectID{StorageBlock: "5555"}, + BundledChildren: []*EntryMetadata{ + &EntryMetadata{Name: "a1", FileSize: 50}, + &EntryMetadata{Name: "z1", FileSize: 120}, + }, + }, + } + + _, err := flattenBundles(sources) + if err == nil { + t.Errorf("expected error") + return + } + + if ok := strings.Contains(err.Error(), "inconsistent size of 'bundle1'"); !ok { + t.Errorf("invalid error: %v", err) + } +} + +func TestFlattenThreeBundles(t *testing.T) { + base1 := &repo.ObjectID{StorageBlock: "5555"} + base2 := &repo.ObjectID{StorageBlock: "6666"} + base3 := &repo.ObjectID{StorageBlock: "7777"} + sources := []*EntryMetadata{ + &EntryMetadata{ + Name: "bundle1", + FileSize: 170, + ObjectId: base1, + BundledChildren: []*EntryMetadata{ + &EntryMetadata{Name: "a1", FileSize: 50}, + &EntryMetadata{Name: "z1", FileSize: 120}, + }, + }, + &EntryMetadata{ + Name: "bundle3", + FileSize: 7, + ObjectId: base3, + BundledChildren: []*EntryMetadata{ + &EntryMetadata{Name: "a3", FileSize: 5}, + &EntryMetadata{Name: "z3", FileSize: 2}, + }, + }, + &EntryMetadata{ + Name: "bundle2", + FileSize: 300, + ObjectId: base2, + BundledChildren: []*EntryMetadata{ + &EntryMetadata{Name: "a2", FileSize: 100}, + &EntryMetadata{Name: "z2", FileSize: 200}, + }, + }, + } + + entries, err := flattenBundles(sources) + if err != nil { + t.Errorf("can't read directory entries: %v", err) + return + } + + expectedEntries := []*EntryMetadata{ + &EntryMetadata{Name: "a1", FileSize: 50, ObjectId: &repo.ObjectID{Section: &repo.ObjectID_Section{ + Start: 0, + Length: 50, + Base: base1, + }}}, + &EntryMetadata{Name: "a2", FileSize: 100, ObjectId: &repo.ObjectID{Section: &repo.ObjectID_Section{ + Start: 0, + Length: 100, + Base: base2, + }}}, + &EntryMetadata{Name: "a3", FileSize: 5, ObjectId: &repo.ObjectID{Section: &repo.ObjectID_Section{ + Start: 0, + Length: 5, + Base: base3, + }}}, + &EntryMetadata{Name: "z1", FileSize: 120, ObjectId: &repo.ObjectID{Section: &repo.ObjectID_Section{ + Start: 50, + Length: 120, + Base: base1, + }}}, + &EntryMetadata{Name: "z2", FileSize: 200, ObjectId: &repo.ObjectID{Section: &repo.ObjectID_Section{ + Start: 100, + Length: 200, + Base: base2, + }}}, + &EntryMetadata{Name: "z3", FileSize: 2, ObjectId: &repo.ObjectID{Section: &repo.ObjectID_Section{ + Start: 5, + Length: 2, + Base: base3, + }}}, + } + + verifyDirectory(t, entries, expectedEntries) +} + +func verifyDirectory(t *testing.T, entries []*EntryMetadata, expectedEntries []*EntryMetadata) { if len(entries) != len(expectedEntries) { t.Errorf("expected %v entries, got %v", len(expectedEntries), len(entries)) } @@ -164,34 +161,13 @@ func verifyDirectory(t *testing.T, data string, expectedEntries []*EntryMetadata actual := entries[i] if !reflect.DeepEqual(expected, actual) { - t.Errorf("invalid entry at index %v:\nexpected: %#v\nactual: %#v", i, expected, actual) + t.Errorf("invalid entry at index %v:\nexpected: %v\nactual: %v", i, + expected.String(), actual.String()) } } } } -func TestInvalidJSON(t *testing.T) { - verifyDirectoryError(t, "{invalid", "invalid character 'i'") - verifyDirectoryError(t, `{"format":{"version":1},"entries":[{"name":""}]}`, "empty entry name") - verifyDirectoryError(t, `{"format":{"version":1},"entries":[{"name":"x","perm":"x123"}]}`, "invalid permissions: 'x123'") - verifyDirectoryError(t, `{"format":{"version":2},"entries":[]}`, "unsupported version: 2") -} - -func verifyDirectoryError(t *testing.T, data, expectedError string) { - entries, err := readDirectoryMetadataEntries(strings.NewReader(data)) - if err == nil { - t.Errorf("expected error %v, got no error", expectedError) - } else { - if !strings.Contains(err.Error(), expectedError) { - t.Errorf("expected error containing '%v', got '%v'", expectedError, err.Error()) - } - } - - if entries != nil { - t.Errorf("got unexpected result: %#v", entries) - } -} - func TestDirectoryNameOrder(t *testing.T) { sortedNames := []string{ "a/a/a", diff --git a/fs/entry.go b/fs/entry.go index 506a22714..5753efacd 100644 --- a/fs/entry.go +++ b/fs/entry.go @@ -8,8 +8,6 @@ import ( "sort" "strings" "time" - - "github.com/kopia/kopia/repo" ) // Entry represents a filesystem entry, which can be Directory, File, or Symlink @@ -45,17 +43,6 @@ type Symlink interface { Readlink() (string, error) } -// EntryMetadata stores attributes of a single entry in a directory. -type EntryMetadata struct { - Name string - FileMode os.FileMode - FileSize int64 - ModTime time.Time - OwnerID uint32 - GroupID uint32 - ObjectID repo.ObjectID -} - type entry struct { parent Directory metadata *EntryMetadata @@ -133,15 +120,24 @@ func isLessOrEqual(name1, name2 string) bool { return len(parts1) <= len(parts2) } +// FileMode returns the file mode and permissions. +func (e *EntryMetadata) FileMode() os.FileMode { + return os.FileMode(e.ModeAndPermissions) +} + +// ModTime returns the modification time. +func (e *EntryMetadata) ModTime() time.Time { + return time.Unix(e.ModTimestampNano/1000000000, e.ModTimestampNano%1000000000) + +} + func (e *EntryMetadata) metadataHash() uint64 { h := fnv.New64a() - binary.Write(h, binary.LittleEndian, e.ModTime.UnixNano()) - binary.Write(h, binary.LittleEndian, e.FileMode) - if e.FileMode.IsRegular() { - binary.Write(h, binary.LittleEndian, e.FileSize) - } - binary.Write(h, binary.LittleEndian, e.OwnerID) - binary.Write(h, binary.LittleEndian, e.GroupID) + binary.Write(h, binary.LittleEndian, e.ModTimestampNano) + binary.Write(h, binary.LittleEndian, e.ModeAndPermissions) + binary.Write(h, binary.LittleEndian, e.FileSize) + binary.Write(h, binary.LittleEndian, e.Uid) + binary.Write(h, binary.LittleEndian, e.Gid) return h.Sum64() } diff --git a/fs/entry_metadata.pb.go b/fs/entry_metadata.pb.go new file mode 100644 index 000000000..c9982d194 --- /dev/null +++ b/fs/entry_metadata.pb.go @@ -0,0 +1,89 @@ +// Code generated by protoc-gen-go. +// source: fs/entry_metadata.proto +// DO NOT EDIT! + +/* +Package fs is a generated protocol buffer package. + +It is generated from these files: + fs/entry_metadata.proto + fs/hashcache.proto + +It has these top-level messages: + EntryMetadata + HashCacheEntry +*/ +package fs + +import proto "github.com/golang/protobuf/proto" +import fmt "fmt" +import math "math" +import kopia_repo "github.com/kopia/kopia/repo" + +// Reference imports to suppress errors if they are not otherwise used. +var _ = proto.Marshal +var _ = fmt.Errorf +var _ = math.Inf + +// This is a compile-time assertion to ensure that this generated file +// is compatible with the proto package it is being compiled against. +const _ = proto.ProtoPackageIsVersion1 + +// EntryMetadata stores attributes of a single entry in a directory. +type EntryMetadata struct { + Name string `protobuf:"bytes,1,opt,name=name" json:"name,omitempty"` + ModeAndPermissions uint32 `protobuf:"fixed32,2,opt,name=mode_and_permissions,json=modeAndPermissions" json:"mode_and_permissions,omitempty"` + FileSize int64 `protobuf:"varint,3,opt,name=file_size,json=fileSize" json:"file_size,omitempty"` + ModTimestampNano int64 `protobuf:"fixed64,4,opt,name=mod_timestamp_nano,json=modTimestampNano" json:"mod_timestamp_nano,omitempty"` + ObjectId *kopia_repo.ObjectID `protobuf:"bytes,5,opt,name=object_id,json=objectId" json:"object_id,omitempty"` + Uid uint32 `protobuf:"varint,6,opt,name=uid" json:"uid,omitempty"` + Gid uint32 `protobuf:"varint,7,opt,name=gid" json:"gid,omitempty"` + BundledChildren []*EntryMetadata `protobuf:"bytes,8,rep,name=bundled_children,json=bundledChildren" json:"bundled_children,omitempty"` +} + +func (m *EntryMetadata) Reset() { *m = EntryMetadata{} } +func (m *EntryMetadata) String() string { return proto.CompactTextString(m) } +func (*EntryMetadata) ProtoMessage() {} +func (*EntryMetadata) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{0} } + +func (m *EntryMetadata) GetObjectId() *kopia_repo.ObjectID { + if m != nil { + return m.ObjectId + } + return nil +} + +func (m *EntryMetadata) GetBundledChildren() []*EntryMetadata { + if m != nil { + return m.BundledChildren + } + return nil +} + +func init() { + proto.RegisterType((*EntryMetadata)(nil), "kopia.fs.EntryMetadata") +} + +var fileDescriptor0 = []byte{ + // 306 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0x54, 0x90, 0xcd, 0x4e, 0x02, 0x31, + 0x14, 0x85, 0x03, 0x83, 0x30, 0xd4, 0x10, 0x27, 0x95, 0x84, 0x2a, 0x1b, 0xe2, 0xca, 0x85, 0x29, + 0xfe, 0x3c, 0x81, 0xa8, 0x0b, 0x16, 0xfe, 0xa4, 0xba, 0x72, 0xd3, 0x74, 0x68, 0x07, 0xaa, 0xb4, + 0x9d, 0x4c, 0xcb, 0x42, 0x5f, 0xcd, 0x97, 0xf3, 0x76, 0x66, 0x88, 0x71, 0xd3, 0xdc, 0x9e, 0xf3, + 0x9d, 0xa6, 0xf7, 0xa0, 0x49, 0xe1, 0xe7, 0xca, 0x86, 0xea, 0x8b, 0x1b, 0x15, 0x84, 0x14, 0x41, + 0xd0, 0xb2, 0x72, 0xc1, 0xe1, 0xf4, 0xd3, 0x95, 0x5a, 0xd0, 0xc2, 0x9f, 0x1e, 0x57, 0xaa, 0x74, + 0x73, 0x97, 0x7f, 0xa8, 0x55, 0xd0, 0xb2, 0xb1, 0xcf, 0x7e, 0xba, 0x68, 0xf4, 0x10, 0x73, 0x8f, + 0x6d, 0x0c, 0x63, 0xd4, 0xb3, 0xc2, 0x28, 0xd2, 0x99, 0x75, 0xce, 0x87, 0xac, 0x9e, 0xf1, 0x25, + 0x1a, 0x1b, 0x27, 0x15, 0x17, 0x56, 0xf2, 0x52, 0x55, 0x46, 0x7b, 0xaf, 0x9d, 0xf5, 0xa4, 0x0b, + 0xcc, 0x80, 0xe1, 0xe8, 0xdd, 0x5a, 0xf9, 0xf2, 0xe7, 0xe0, 0x29, 0x1a, 0x16, 0x7a, 0xab, 0xb8, + 0xd7, 0xdf, 0x8a, 0x24, 0x80, 0x25, 0x2c, 0x8d, 0xc2, 0x2b, 0xdc, 0xf1, 0x05, 0x8a, 0x11, 0x1e, + 0xb4, 0x51, 0x3e, 0x08, 0x53, 0x72, 0x2b, 0xac, 0x23, 0x3d, 0xa0, 0x32, 0x96, 0x81, 0xf3, 0xb6, + 0x37, 0x9e, 0x40, 0xc7, 0x57, 0x68, 0xd8, 0x7c, 0x9a, 0x6b, 0x49, 0x0e, 0x00, 0x3a, 0xbc, 0x1e, + 0xd3, 0x66, 0xab, 0xb8, 0x11, 0x7d, 0xae, 0xcd, 0xe5, 0x3d, 0x4b, 0x1b, 0x6c, 0x29, 0x71, 0x86, + 0x92, 0x1d, 0xc0, 0x7d, 0x80, 0x47, 0x2c, 0x8e, 0x51, 0x59, 0x83, 0x32, 0x68, 0x14, 0x18, 0xf1, + 0x02, 0x65, 0xf9, 0xce, 0xca, 0xad, 0x92, 0x7c, 0xb5, 0xd1, 0x5b, 0x59, 0x29, 0x4b, 0xd2, 0x59, + 0x02, 0xaf, 0x4f, 0xe8, 0xbe, 0x33, 0xfa, 0xaf, 0x1a, 0x76, 0xd4, 0x06, 0xee, 0x5a, 0x7e, 0x31, + 0x7d, 0x3f, 0x59, 0xeb, 0xb0, 0xd9, 0xe5, 0x74, 0xe5, 0xcc, 0xbc, 0x4e, 0xb5, 0x67, 0xe1, 0xf3, + 0x7e, 0xdd, 0xf0, 0xcd, 0x6f, 0x00, 0x00, 0x00, 0xff, 0xff, 0xcf, 0xf4, 0xc2, 0x88, 0x9b, 0x01, + 0x00, 0x00, +} diff --git a/fs/entry_metadata.proto b/fs/entry_metadata.proto new file mode 100644 index 000000000..7d8b49467 --- /dev/null +++ b/fs/entry_metadata.proto @@ -0,0 +1,19 @@ +syntax = "proto3"; + +package kopia.fs; +option go_package = "github.com/kopia/kopia/fs"; + +import "repo/objectid.proto"; + +// EntryMetadata stores attributes of a single entry in a directory. +message EntryMetadata { + string name = 1; + fixed32 mode_and_permissions = 2; + int64 file_size = 3; + sfixed64 mod_timestamp_nano = 4; + repo.ObjectID object_id = 5; + uint32 uid = 6; + uint32 gid = 7; + + repeated EntryMetadata bundled_children = 8; +} diff --git a/fs/hashcache.go b/fs/hashcache.go index f85ae5933..52ff66aa3 100644 --- a/fs/hashcache.go +++ b/fs/hashcache.go @@ -4,36 +4,28 @@ import ( "bufio" "fmt" "io" - "strconv" - "strings" - "github.com/kopia/kopia/repo" + "github.com/kopia/kopia/internal" ) -type hashCacheEntry struct { - Name string - Hash uint64 - ObjectID repo.ObjectID -} - type hashcacheReader struct { - scanner *bufio.Scanner - nextEntry *hashCacheEntry - entry0 hashCacheEntry - entry1 hashCacheEntry + reader *internal.ProtoStreamReader + nextEntry *HashCacheEntry + entry0 HashCacheEntry + entry1 HashCacheEntry odd bool first bool } func (hcr *hashcacheReader) open(r io.Reader) error { - hcr.scanner = bufio.NewScanner(r) + hcr.reader = internal.NewProtoStreamReader(bufio.NewReader(r), internal.ProtoStreamTypeHashCache) hcr.nextEntry = nil hcr.first = true hcr.readahead() return nil } -func (hcr *hashcacheReader) findEntry(relativeName string) *hashCacheEntry { +func (hcr *hashcacheReader) findEntry(relativeName string) *HashCacheEntry { for hcr.nextEntry != nil && isLess(hcr.nextEntry.Name, relativeName) { hcr.readahead() } @@ -49,37 +41,20 @@ func (hcr *hashcacheReader) findEntry(relativeName string) *hashCacheEntry { } func (hcr *hashcacheReader) readahead() { - if hcr.scanner != nil { - if hcr.first { - hcr.first = false - if !hcr.scanner.Scan() || hcr.scanner.Text() != "HASHCACHE:v1" { - hcr.scanner = nil - return - } - } - + if hcr.reader != nil { hcr.nextEntry = nil - if hcr.scanner.Scan() { - var err error - e := hcr.nextManifestEntry() - t := hcr.scanner.Text() - p0 := strings.IndexByte(t, '|') - p1 := strings.LastIndexByte(t, '|') - e.Name = t[0:p0] - e.Hash, err = strconv.ParseUint(t[p0+1:p1], 0, 64) - e.ObjectID = repo.ObjectID(t[p1+1:]) - if err == nil { - hcr.nextEntry = e - } + e := hcr.nextManifestEntry() + if err := hcr.reader.Read(e); err == nil { + hcr.nextEntry = e } } if hcr.nextEntry == nil { - hcr.scanner = nil + hcr.reader = nil } } -func (hcr *hashcacheReader) nextManifestEntry() *hashCacheEntry { +func (hcr *hashcacheReader) nextManifestEntry() *HashCacheEntry { hcr.odd = !hcr.odd if hcr.odd { return &hcr.entry1 @@ -89,19 +64,18 @@ func (hcr *hashcacheReader) nextManifestEntry() *hashCacheEntry { } type hashcacheWriter struct { - writer io.Writer + writer *internal.ProtoStreamWriter lastNameWritten string } func newHashCacheWriter(w io.Writer) *hashcacheWriter { hcw := &hashcacheWriter{ - writer: w, + writer: internal.NewProtoStreamWriter(w, internal.ProtoStreamTypeHashCache), } - io.WriteString(w, "HASHCACHE:v1\n") return hcw } -func (hcw *hashcacheWriter) WriteEntry(e hashCacheEntry) error { +func (hcw *hashcacheWriter) WriteEntry(e HashCacheEntry) error { if hcw.lastNameWritten != "" { if isLessOrEqual(e.Name, hcw.lastNameWritten) { return fmt.Errorf("out-of-order directory entry, previous '%v' current '%v'", hcw.lastNameWritten, e.Name) @@ -109,7 +83,7 @@ func (hcw *hashcacheWriter) WriteEntry(e hashCacheEntry) error { hcw.lastNameWritten = e.Name } - fmt.Fprintf(hcw.writer, "%v|%v|%v\n", e.Name, e.Hash, e.ObjectID) + hcw.writer.Write(&e) return nil } diff --git a/fs/hashcache.pb.go b/fs/hashcache.pb.go new file mode 100644 index 000000000..d94cfbcd6 --- /dev/null +++ b/fs/hashcache.pb.go @@ -0,0 +1,52 @@ +// Code generated by protoc-gen-go. +// source: fs/hashcache.proto +// DO NOT EDIT! + +package fs + +import proto "github.com/golang/protobuf/proto" +import fmt "fmt" +import math "math" +import kopia_repo "github.com/kopia/kopia/repo" + +// Reference imports to suppress errors if they are not otherwise used. +var _ = proto.Marshal +var _ = fmt.Errorf +var _ = math.Inf + +type HashCacheEntry struct { + Name string `protobuf:"bytes,1,opt,name=name" json:"name,omitempty"` + Hash uint64 `protobuf:"fixed64,2,opt,name=hash" json:"hash,omitempty"` + ObjectId *kopia_repo.ObjectID `protobuf:"bytes,3,opt,name=object_id,json=objectId" json:"object_id,omitempty"` +} + +func (m *HashCacheEntry) Reset() { *m = HashCacheEntry{} } +func (m *HashCacheEntry) String() string { return proto.CompactTextString(m) } +func (*HashCacheEntry) ProtoMessage() {} +func (*HashCacheEntry) Descriptor() ([]byte, []int) { return fileDescriptor1, []int{0} } + +func (m *HashCacheEntry) GetObjectId() *kopia_repo.ObjectID { + if m != nil { + return m.ObjectId + } + return nil +} + +func init() { + proto.RegisterType((*HashCacheEntry)(nil), "kopia.fs.HashCacheEntry") +} + +var fileDescriptor1 = []byte{ + // 173 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0xe2, 0x12, 0x4a, 0x2b, 0xd6, 0xcf, + 0x48, 0x2c, 0xce, 0x48, 0x4e, 0x4c, 0xce, 0x48, 0xd5, 0x2b, 0x28, 0xca, 0x2f, 0xc9, 0x17, 0xe2, + 0xc8, 0xce, 0x2f, 0xc8, 0x4c, 0xd4, 0x4b, 0x2b, 0x96, 0x12, 0x2e, 0x4a, 0x2d, 0xc8, 0xd7, 0xcf, + 0x4f, 0xca, 0x4a, 0x4d, 0x2e, 0xc9, 0x4c, 0x81, 0x48, 0x2b, 0x65, 0x73, 0xf1, 0x79, 0x00, 0x75, + 0x38, 0x83, 0x74, 0xb8, 0xe6, 0x95, 0x14, 0x55, 0x0a, 0x09, 0x71, 0xb1, 0xe4, 0x25, 0xe6, 0xa6, + 0x4a, 0x30, 0x2a, 0x30, 0x6a, 0x70, 0x06, 0x81, 0xd9, 0x20, 0x31, 0x90, 0xb9, 0x12, 0x4c, 0x40, + 0x31, 0xb6, 0x20, 0x30, 0x5b, 0xc8, 0x90, 0x8b, 0x13, 0x62, 0x56, 0x7c, 0x66, 0x8a, 0x04, 0x33, + 0x50, 0x82, 0xdb, 0x48, 0x44, 0x0f, 0x62, 0x19, 0xc8, 0x22, 0x3d, 0x7f, 0xb0, 0xa4, 0xa7, 0x4b, + 0x10, 0x07, 0x44, 0x99, 0x67, 0x8a, 0x93, 0x74, 0x94, 0x64, 0x7a, 0x66, 0x49, 0x46, 0x69, 0x92, + 0x5e, 0x72, 0x7e, 0xae, 0x3e, 0x58, 0x2d, 0x94, 0x4c, 0x2b, 0x4e, 0x62, 0x03, 0x3b, 0xc8, 0x18, + 0x10, 0x00, 0x00, 0xff, 0xff, 0xa7, 0x08, 0x96, 0x2b, 0xc5, 0x00, 0x00, 0x00, +} diff --git a/fs/hashcache.proto b/fs/hashcache.proto new file mode 100644 index 000000000..8c88722c3 --- /dev/null +++ b/fs/hashcache.proto @@ -0,0 +1,12 @@ +syntax = "proto3"; + +package kopia.fs; +option go_package = "github.com/kopia/kopia/fs"; + +import "repo/objectid.proto"; + +message HashCacheEntry { + string name = 1; + fixed64 hash = 2; + repo.ObjectID object_id = 3; +} diff --git a/fs/inmemoryfs_for_test.go b/fs/inmemory_fs_for_test.go similarity index 94% rename from fs/inmemoryfs_for_test.go rename to fs/inmemory_fs_for_test.go index 781305dba..c8727b435 100644 --- a/fs/inmemoryfs_for_test.go +++ b/fs/inmemory_fs_for_test.go @@ -35,8 +35,8 @@ func (imd *inmemoryDirectory) addFile(name string, content []byte, mode os.FileM inmemoryEntry: inmemoryEntry{ parent: imd, metadata: &EntryMetadata{ - Name: name, - FileMode: mode, + Name: name, + ModeAndPermissions: uint32(mode), }, }, source: bytes.NewBuffer(content), @@ -54,8 +54,8 @@ func (imd *inmemoryDirectory) addDir(name string, mode os.FileMode) *inmemoryDir inmemoryEntry: inmemoryEntry{ parent: imd, metadata: &EntryMetadata{ - Name: name, - FileMode: mode | os.ModeDir, + Name: name, + ModeAndPermissions: uint32(mode | os.ModeDir), }, }, } @@ -88,7 +88,7 @@ func (imd *inmemoryDirectory) subdir(name ...string) *inmemoryDirectory { if i2 == nil { panic(fmt.Sprintf("'%s' not found in '%s'", n, i.metadata.Name)) } - if !i2.Metadata().FileMode.IsDir() { + if !i2.Metadata().FileMode().IsDir() { panic(fmt.Sprintf("'%s' is not a directory in '%s'", n, i.metadata.Name)) } diff --git a/fs/filesystem.go b/fs/local_fs.go similarity index 95% rename from fs/filesystem.go rename to fs/local_fs.go index 03ba751e8..23253095a 100644 --- a/fs/filesystem.go +++ b/fs/local_fs.go @@ -107,9 +107,9 @@ func NewFilesystemDirectory(path string, parent Directory) (Directory, error) { func entryMetadataFromFileInfo(fi os.FileInfo) *EntryMetadata { e := &EntryMetadata{ - Name: filepath.Base(fi.Name()), - FileMode: fi.Mode(), - ModTime: fi.ModTime(), + Name: filepath.Base(fi.Name()), + ModeAndPermissions: uint32(fi.Mode()), + ModTimestampNano: fi.ModTime().UnixNano(), } if fi.Mode().IsRegular() { diff --git a/fs/filesystem_nonwindows.go b/fs/local_fs_nonwindows.go similarity index 82% rename from fs/filesystem_nonwindows.go rename to fs/local_fs_nonwindows.go index 4cbb4e8da..d0dd17798 100644 --- a/fs/filesystem_nonwindows.go +++ b/fs/local_fs_nonwindows.go @@ -9,8 +9,8 @@ import ( func (e *EntryMetadata) populatePlatformSpecificEntryDetails(fi os.FileInfo) error { if stat, ok := fi.Sys().(*syscall.Stat_t); ok { - e.OwnerID = stat.Uid - e.GroupID = stat.Gid + e.Uid = stat.Uid + e.Gid = stat.Gid } return nil diff --git a/fs/filesystem_test.go b/fs/local_fs_test.go similarity index 86% rename from fs/filesystem_test.go rename to fs/local_fs_test.go index 1e4ccaa72..645bf7c1a 100644 --- a/fs/filesystem_test.go +++ b/fs/local_fs_test.go @@ -64,19 +64,19 @@ func TestFiles(t *testing.T) { goodCount := 0 - if entries[0].Metadata().Name == "f1" && entries[0].Metadata().FileSize == 5 && entries[0].Metadata().FileMode.IsRegular() { + if entries[0].Metadata().Name == "f1" && entries[0].Metadata().FileSize == 5 && entries[0].Metadata().FileMode().IsRegular() { goodCount++ } - if entries[1].Metadata().Name == "f2" && entries[1].Metadata().FileSize == 4 && entries[1].Metadata().FileMode.IsRegular() { + if entries[1].Metadata().Name == "f2" && entries[1].Metadata().FileSize == 4 && entries[1].Metadata().FileMode().IsRegular() { goodCount++ } - if entries[2].Metadata().Name == "f3" && entries[2].Metadata().FileSize == 3 && entries[2].Metadata().FileMode.IsRegular() { + if entries[2].Metadata().Name == "f3" && entries[2].Metadata().FileSize == 3 && entries[2].Metadata().FileMode().IsRegular() { goodCount++ } - if entries[3].Metadata().Name == "y" && entries[3].Metadata().FileSize == 0 && entries[3].Metadata().FileMode.IsDir() { + if entries[3].Metadata().Name == "y" && entries[3].Metadata().FileSize == 0 && entries[3].Metadata().FileMode().IsDir() { goodCount++ } - if entries[4].Metadata().Name == "z" && entries[4].Metadata().FileSize == 0 && entries[4].Metadata().FileMode.IsDir() { + if entries[4].Metadata().Name == "z" && entries[4].Metadata().FileSize == 0 && entries[4].Metadata().FileMode().IsDir() { goodCount++ } if goodCount != 5 { diff --git a/fs/filesystem_windows.go b/fs/local_fs_windows.go similarity index 100% rename from fs/filesystem_windows.go rename to fs/local_fs_windows.go diff --git a/fs/repository.go b/fs/repository_fs.go similarity index 88% rename from fs/repository.go rename to fs/repository_fs.go index dd3a1ab1f..23013f5cb 100644 --- a/fs/repository.go +++ b/fs/repository_fs.go @@ -18,7 +18,7 @@ type repositoryFile repositoryEntry type repositorySymlink repositoryEntry func (rd *repositoryDirectory) Readdir() (Entries, error) { - r, err := rd.repo.Open(rd.entry.Metadata().ObjectID) + r, err := rd.repo.Open(*rd.entry.Metadata().ObjectId) if err != nil { return nil, err } @@ -38,7 +38,7 @@ func (rd *repositoryDirectory) Readdir() (Entries, error) { } func (rf *repositoryFile) Open() (EntryMetadataReadCloser, error) { - r, err := rf.repo.Open(rf.entry.Metadata().ObjectID) + r, err := rf.repo.Open(*rf.entry.Metadata().ObjectId) if err != nil { return nil, err } @@ -51,7 +51,7 @@ func (rsl *repositorySymlink) Readlink() (string, error) { } func newRepoEntry(r repo.Repository, md *EntryMetadata, parent Directory) Entry { - switch md.FileMode & os.ModeType { + switch md.FileMode() & os.ModeType { case os.ModeDir: return Directory(&repositoryDirectory{ entry: newEntry(md, parent), @@ -71,7 +71,7 @@ func newRepoEntry(r repo.Repository, md *EntryMetadata, parent Directory) Entry }) default: - panic(fmt.Sprintf("not supported entry metadata type: %v", md.FileMode)) + panic(fmt.Sprintf("not supported entry metadata type: %v", md.FileMode())) } } @@ -94,9 +94,9 @@ func withMetadata(rc io.ReadCloser, md *EntryMetadata) EntryMetadataReadCloser { // NewRepositoryDirectory returns Directory based on repository object with the specified ID. func NewRepositoryDirectory(r repo.Repository, objectID repo.ObjectID) Directory { d := newRepoEntry(r, &EntryMetadata{ - Name: "/", - ObjectID: objectID, - FileMode: 0555 | os.ModeDir, + Name: "/", + ObjectId: &objectID, + ModeAndPermissions: 0555 | uint32(os.ModeDir), }, nil) return d.(Directory) diff --git a/fs/upload.go b/fs/upload.go index 38bbf8da7..a35153b99 100644 --- a/fs/upload.go +++ b/fs/upload.go @@ -36,7 +36,7 @@ type UploadResult struct { // Uploader supports efficient uploading files and directories to repository. type Uploader interface { - UploadDir(dir Directory, previousManifestID repo.ObjectID) (*UploadResult, error) + UploadDir(dir Directory, previousManifestID *repo.ObjectID) (*UploadResult, error) UploadFile(file File) (*UploadResult, error) Cancel() } @@ -74,11 +74,13 @@ func (u *uploader) uploadFileInternal(f File, relativePath string, forceStored b return nil, 0, err } - e2.ObjectID, err = writer.Result(forceStored) + r, err := writer.Result(forceStored) if err != nil { return nil, 0, err } + e2.ObjectId = &r + if written != e2.FileSize { // file changed } @@ -123,10 +125,11 @@ func (u *uploader) uploadBundleInternal(b *uploadBundle) (*EntryMetadata, uint64 } b.files = uploadedFiles - bundleMetadata.ObjectID, err = writer.Result(true) + r, err := writer.Result(true) if err != nil { return nil, 0, err } + bundleMetadata.ObjectId = &r return bundleMetadata, bundleMetadata.metadataHash(), nil } @@ -134,16 +137,16 @@ func (u *uploader) uploadBundleInternal(b *uploadBundle) (*EntryMetadata, uint64 func (u *uploader) UploadFile(file File) (*UploadResult, error) { result := &UploadResult{} e, _, err := u.uploadFileInternal(file, file.Metadata().Name, true) - result.ObjectID = e.ObjectID + result.ObjectID = *e.ObjectId return result, err } -func (u *uploader) UploadDir(dir Directory, previousManifestID repo.ObjectID) (*UploadResult, error) { +func (u *uploader) UploadDir(dir Directory, previousManifestID *repo.ObjectID) (*UploadResult, error) { var mr hashcacheReader var err error - if previousManifestID != "" { - if r, err := u.repo.Open(previousManifestID); err == nil { + if previousManifestID != nil { + if r, err := u.repo.Open(*previousManifestID); err == nil { mr.open(r) } } @@ -178,7 +181,7 @@ func (u *uploader) uploadDirInternal( entries, err := dir.Readdir() if err != nil { - return "", 0, false, err + return repo.NullObjectID, 0, false, err } entries = u.bundleEntries(entries) @@ -207,20 +210,22 @@ func (u *uploader) uploadDirInternal( case Directory: oid, h, wasCached, err := u.uploadDirInternal(result, entry, entryRelativePath, hcw, mr, false) if err != nil { - return "", 0, false, err + return repo.NullObjectID, 0, false, err } //log.Printf("dirHash: %v %v", fullPath, h) hash = h allCached = allCached && wasCached - e.ObjectID = oid + e.ObjectId = &oid case Symlink: l, err := entry.Readlink() if err != nil { - return "", 0, false, err + return repo.NullObjectID, 0, false, err } - e.ObjectID = repo.NewInlineObjectID([]byte(l)) + id := repo.NewInlineObjectID([]byte(l)) + + e.ObjectId = &id hash = e.metadataHash() case *uploadBundle: @@ -239,13 +244,14 @@ func (u *uploader) uploadDirInternal( if cacheMatches { result.Stats.CachedFiles++ // Avoid hashing by reusing previous object ID. - e.ObjectID = repo.ObjectID(cachedEntry.ObjectID) + id := *cachedEntry.ObjectId + e.ObjectId = &id hash = cachedEntry.Hash } else { result.Stats.NonCachedFiles++ e, hash, err = u.uploadBundleInternal(entry) if err != nil { - return "", 0, false, fmt.Errorf("unable to hash file: %s", err) + return repo.NullObjectID, 0, false, fmt.Errorf("unable to hash file: %s", err) } } @@ -262,18 +268,19 @@ func (u *uploader) uploadDirInternal( if cacheMatches { result.Stats.CachedFiles++ // Avoid hashing by reusing previous object ID. - e.ObjectID = repo.ObjectID(cachedEntry.ObjectID) + id := *cachedEntry.ObjectId + e.ObjectId = &id hash = cachedEntry.Hash } else { result.Stats.NonCachedFiles++ e, hash, err = u.uploadFileInternal(entry, entryRelativePath, false) if err != nil { - return "", 0, false, fmt.Errorf("unable to hash file: %s", err) + return repo.NullObjectID, 0, false, fmt.Errorf("unable to hash file: %s", err) } } default: - return "", 0, false, fmt.Errorf("file type %v not supported", entry.Metadata().FileMode) + return repo.NullObjectID, 0, false, fmt.Errorf("file type %v not supported", entry.Metadata().FileMode()) } if hash != 0 { @@ -283,22 +290,20 @@ func (u *uploader) uploadDirInternal( } if err := dw.WriteEntry(e, childrenMetadata); err != nil { - return "", 0, false, err + return repo.NullObjectID, 0, false, err } - if !e.FileMode.IsDir() && e.ObjectID.Type().IsStored() { - if err := hcw.WriteEntry(hashCacheEntry{ + if !e.FileMode().IsDir() && e.ObjectId.StorageBlock != "" { + if err := hcw.WriteEntry(HashCacheEntry{ Name: entryRelativePath, - Hash: e.metadataHash(), - ObjectID: e.ObjectID, + Hash: hash, + ObjectId: e.ObjectId, }); err != nil { - return "", 0, false, err + return repo.NullObjectID, 0, false, err } } } - dw.Close() - var directoryOID repo.ObjectID dirHash := dirHasher.Sum64() @@ -308,7 +313,7 @@ func (u *uploader) uploadDirInternal( if allCached { // Avoid hashing directory listing if every entry matched the cache. result.Stats.CachedDirectories++ - directoryOID = repo.ObjectID(cachedDirEntry.ObjectID) + directoryOID = *cachedDirEntry.ObjectId } else { result.Stats.NonCachedDirectories++ directoryOID, err = writer.Result(forceStored) @@ -317,14 +322,15 @@ func (u *uploader) uploadDirInternal( } } - if err := hcw.WriteEntry(hashCacheEntry{ + if err := hcw.WriteEntry(HashCacheEntry{ Name: relativePath + "/", - ObjectID: directoryOID, + ObjectId: &directoryOID, Hash: dirHash, }); err != nil { - return "", 0, false, err + return repo.NullObjectID, 0, false, err } + // log.Printf("Dir: %v %v %v %v", relativePath, directoryOID.UIString(), dirHash, allCached) return directoryOID, dirHash, allCached, nil } @@ -398,8 +404,8 @@ func (u *uploader) bundleEntries(entries Entries) Entries { func getBundleNumber(md *EntryMetadata) int { // TODO(jkowalski): This is not ready yet, uncomment when ready. - // if md.FileMode.IsRegular() && md.FileSize < maxBundleFileSize { - // return md.ModTime.Year()*100 + int(md.ModTime.Month()) + // if md.FileMode().IsRegular() && md.FileSize < maxBundleFileSize { + // return md.ModTime().Year()*100 + int(md.ModTime().Month()) // } return 0 diff --git a/fs/upload_test.go b/fs/upload_test.go index 59a5024b1..07780911b 100644 --- a/fs/upload_test.go +++ b/fs/upload_test.go @@ -41,8 +41,8 @@ func newUploadTestHarness() *uploadTestHarness { } format := repo.Format{ - Version: "1", - ObjectFormat: "md5", + Version: 1, + ObjectFormat: repo.ObjectIDFormat_TESTONLY_MD5, MaxBlobSize: 1000, MaxInlineBlobSize: 0, } @@ -86,22 +86,22 @@ func TestUpload(t *testing.T) { th := newUploadTestHarness() defer th.cleanup() - r1, err := th.uploader.UploadDir(th.sourceDir, "") + r1, err := th.uploader.UploadDir(th.sourceDir, nil) if err != nil { t.Errorf("upload failed: %v", err) } - r2, err := th.uploader.UploadDir(th.sourceDir, r1.ManifestID) + r2, err := th.uploader.UploadDir(th.sourceDir, &r1.ManifestID) if err != nil { t.Errorf("upload failed: %v", err) } - if r2.ObjectID != r1.ObjectID { - t.Errorf("expected r1.ObjectID==r2.ObjectID, got %v and %v", r1.ObjectID, r2.ObjectID) + if !r2.ObjectID.Equals(r1.ObjectID) { + t.Errorf("expected r1.ObjectID==r2.ObjectID, got %v and %v", r1.ObjectID.UIString(), r2.ObjectID.UIString()) } - if r2.ManifestID != r1.ManifestID { - t.Errorf("expected r2.ManifestID==r1.ManifestID, got %v and %v", r2.ManifestID, r1.ManifestID) + if !r2.ManifestID.Equals(r1.ManifestID) { + t.Errorf("expected r2.ManifestID==r1.ManifestID, got %v and %v", r2.ManifestID.UIString(), r1.ManifestID.UIString()) } if r1.Stats.CachedFiles+r1.Stats.CachedDirectories != 0 { @@ -116,17 +116,17 @@ func TestUpload(t *testing.T) { // Add one more file, the r1.ObjectID should change. th.sourceDir.addFile("d2/d1/f3", []byte{1, 2, 3, 4, 5}, 0777) - r3, err := th.uploader.UploadDir(th.sourceDir, r1.ManifestID) + r3, err := th.uploader.UploadDir(th.sourceDir, &r1.ManifestID) if err != nil { t.Errorf("upload failed: %v", err) } - if r2.ObjectID == r3.ObjectID { - t.Errorf("expected r3.ObjectID!=r2.ObjectID, got %v", r3.ObjectID) + if r2.ObjectID.Equals(r3.ObjectID) { + t.Errorf("expected r3.ObjectID!=r2.ObjectID, got %v", r3.ObjectID.UIString()) } - if r2.ManifestID == r3.ManifestID { - t.Errorf("expected r3.ManifestID!=r2.ManifestID, got %v", r3.ManifestID) + if r2.ManifestID.Equals(r3.ManifestID) { + t.Errorf("expected r3.ManifestID!=r2.ManifestID, got %v", r3.ManifestID.UIString()) } if r3.Stats.NonCachedFiles != 1 && r3.Stats.NonCachedDirectories != 3 { @@ -137,15 +137,15 @@ func TestUpload(t *testing.T) { // Now remove the added file, OID should be identical to the original before the file got added. th.sourceDir.subdir("d2", "d1").remove("f3") - r4, err := th.uploader.UploadDir(th.sourceDir, r1.ManifestID) + r4, err := th.uploader.UploadDir(th.sourceDir, &r1.ManifestID) if err != nil { t.Errorf("upload failed: %v", err) } - if r4.ObjectID != r1.ObjectID { + if !r4.ObjectID.Equals(r1.ObjectID) { t.Errorf("expected r4.ObjectID==r1.ObjectID, got %v and %v", r4.ObjectID, r1.ObjectID) } - if r4.ManifestID != r1.ManifestID { + if !r4.ManifestID.Equals(r1.ManifestID) { t.Errorf("expected r4.ManifestID==r1.ManifestID, got %v and %v", r4.ManifestID, r1.ManifestID) } @@ -156,7 +156,7 @@ func TestUpload(t *testing.T) { } // Upload again, this time using r3.ManifestID as base. - r5, err := th.uploader.UploadDir(th.sourceDir, r3.ManifestID) + r5, err := th.uploader.UploadDir(th.sourceDir, &r3.ManifestID) if err != nil { t.Errorf("upload failed: %v", err) } @@ -176,7 +176,7 @@ func TestUpload_TopLevelDirectoryReadFailure(t *testing.T) { th.sourceDir.failReaddir(errTest) - r, err := th.uploader.UploadDir(th.sourceDir, "") + r, err := th.uploader.UploadDir(th.sourceDir, nil) if err != errTest { t.Errorf("expected error: %v", err) } @@ -192,7 +192,7 @@ func TestUpload_SubDirectoryReadFailure(t *testing.T) { th.sourceDir.subdir("d1").failReaddir(errTest) - _, err := th.uploader.UploadDir(th.sourceDir, "") + _, err := th.uploader.UploadDir(th.sourceDir, nil) if err == nil { t.Errorf("expected error") } diff --git a/fuse/fusefs.go b/fuse/fusefs.go index f3131962a..6d7185c5a 100644 --- a/fuse/fusefs.go +++ b/fuse/fusefs.go @@ -21,11 +21,11 @@ type fuseNode struct { func (n *fuseNode) Attr(ctx context.Context, a *fuse.Attr) error { m := n.entry.Metadata() - a.Mode = m.FileMode + a.Mode = m.FileMode() a.Size = uint64(m.FileSize) - a.Mtime = m.ModTime - a.Uid = m.OwnerID - a.Gid = m.GroupID + a.Mtime = m.ModTime() + a.Uid = m.Uid + a.Gid = m.Gid return nil } @@ -88,7 +88,7 @@ func (dir *fuseDirectoryNode) ReadDirAll(ctx context.Context) ([]fuse.Dirent, er Name: m.Name, } - switch m.FileMode & os.ModeType { + switch m.FileMode() & os.ModeType { case os.ModeDir: dirent.Type = fuse.DT_Dir case 0: @@ -112,7 +112,7 @@ type fuseSymlinkNode struct { } func (sl *fuseSymlinkNode) Readlink(ctx context.Context, req *fuse.ReadlinkRequest) (string, error) { - return string(sl.entry.Metadata().ObjectID), nil + return sl.entry.(fs.Symlink).Readlink() } func newFuseNode(e fs.Entry) (fusefs.Node, error) { @@ -124,7 +124,7 @@ func newFuseNode(e fs.Entry) (fusefs.Node, error) { case fs.Symlink: return &fuseSymlinkNode{fuseNode{e}}, nil default: - return nil, fmt.Errorf("entry type not supported: %v", e.Metadata().FileMode) + return nil, fmt.Errorf("entry type not supported: %v", e.Metadata().FileMode()) } } diff --git a/internal/proto_stream_reader.go b/internal/proto_stream_reader.go new file mode 100644 index 000000000..06b66a7d7 --- /dev/null +++ b/internal/proto_stream_reader.go @@ -0,0 +1,66 @@ +package internal + +import ( + "bufio" + "bytes" + "encoding/binary" + "fmt" + "io" + + "github.com/golang/protobuf/proto" +) + +// ProtoStreamReader reads a stream of length-prefix proto messages with an optional header. +type ProtoStreamReader struct { + reader *bufio.Reader + readBuf []byte + buf *proto.Buffer + header []byte +} + +// Read reads the next proto message from the stream, returns io.EOF on the end of stream. +func (r *ProtoStreamReader) Read(v proto.Message) error { + v.Reset() + if r.header != nil { + p := make([]byte, len(r.header)) + if _, err := r.reader.Read(p); err == io.EOF { + return io.EOF + } + + if !bytes.Equal(p, r.header) { + return fmt.Errorf("invalid stream header: %v, expected %v", string(p), string(r.header)) + } + + r.header = nil + + } + length64, err := binary.ReadUvarint(r.reader) + if err != nil { + return err + } + + length := int(length64) + + if cap(r.readBuf) < length { + r.readBuf = make([]byte, length*12/10) + } + + r.readBuf = r.readBuf[:length] + if _, err := io.ReadFull(r.reader, r.readBuf); err != nil { + return err + } + + r.buf.SetBuf(r.readBuf) + + return r.buf.Unmarshal(v) +} + +// NewProtoStreamReader returns new ProtoStreamReader on top of a given buffered reader. +// The provided header must match the beginning of a stream. +func NewProtoStreamReader(r *bufio.Reader, header []byte) *ProtoStreamReader { + return &ProtoStreamReader{ + reader: r, + header: header, + buf: proto.NewBuffer(nil), + } +} diff --git a/internal/proto_stream_test.go b/internal/proto_stream_test.go new file mode 100644 index 000000000..18823bf42 --- /dev/null +++ b/internal/proto_stream_test.go @@ -0,0 +1,74 @@ +package internal + +import ( + "bufio" + "bytes" + "io" + "strings" + "testing" +) + +var testHeader1 = []byte("01234567") +var testHeader2 = []byte("0123456x") + +func TestProtoStream(t *testing.T) { + var buf bytes.Buffer + + data := []TestProto{ + TestProto{Name: "foo"}, + TestProto{Name: "bar"}, + TestProto{Name: "baz"}, + } + + w := NewProtoStreamWriter(&buf, testHeader1) + for _, d := range data { + if err := w.Write(&d); err != nil { + t.Errorf("write error: %v", err) + } + } + r := NewProtoStreamReader(bufio.NewReader(&buf), testHeader1) + var v TestProto + for _, d := range data { + if err := r.Read(&v); err != nil { + t.Errorf("read error: %v", err) + } + if v.Name != d.Name { + t.Errorf("invalid value: '%v', expected '%v'", v.Name, d.Name) + } + } + err := r.Read(&v) + if err != io.EOF { + t.Errorf("expected EOF, got %v", err) + } +} + +func TestEmptyStream(t *testing.T) { + var buf bytes.Buffer + + NewProtoStreamWriter(&buf, testHeader1) + + r := NewProtoStreamReader(bufio.NewReader(&buf), testHeader2) + var v TestProto + err := r.Read(&v) + if err != io.EOF { + t.Errorf("expected EOF, got %v", err) + } +} + +func TestInvalidHeader(t *testing.T) { + var buf bytes.Buffer + + w := NewProtoStreamWriter(&buf, testHeader1) + if err := w.Write(&TestProto{Name: "foo"}); err != nil { + t.Errorf("write error: %v", err) + } + + r := NewProtoStreamReader(bufio.NewReader(&buf), testHeader2) + var v TestProto + err := r.Read(&v) + if err == nil { + t.Errorf("expected error, got none") + } else if !strings.Contains(err.Error(), "invalid stream header") { + t.Errorf("got incorrect error: %v", err) + } +} diff --git a/internal/proto_stream_types.go b/internal/proto_stream_types.go new file mode 100644 index 000000000..244284180 --- /dev/null +++ b/internal/proto_stream_types.go @@ -0,0 +1,12 @@ +package internal + +var ( + // ProtoStreamTypeHashCache indicates proto stream whose elements are hash cache entries. + ProtoStreamTypeHashCache = []byte("kopiaHCH") + + // ProtoStreamTypeDir indicates proto stream whose elements are directory entries. + ProtoStreamTypeDir = []byte("kopiaDIR") + + // ProtoStreamTypeIndirect indicates proto stream whose elements are indirect block references. + ProtoStreamTypeIndirect = []byte("kopiaIND") +) diff --git a/internal/proto_stream_writer.go b/internal/proto_stream_writer.go new file mode 100644 index 000000000..5698c68de --- /dev/null +++ b/internal/proto_stream_writer.go @@ -0,0 +1,58 @@ +package internal + +import ( + "encoding/binary" + "io" + + "github.com/golang/protobuf/proto" +) + +// ProtoStreamWriter writes a stream of length-prefix proto messages with an optional header. +type ProtoStreamWriter struct { + writer io.Writer + buf *proto.Buffer + lengthPrefixBuf []byte + header []byte +} + +// Write emits length-prefixed proto message to its output. +func (w *ProtoStreamWriter) Write(v proto.Message) error { + if w.header != nil { + if _, err := w.writer.Write(w.header); err != nil { + return err + } + w.header = nil + } + + w.buf.Reset() + if err := w.buf.Marshal(v); err != nil { + return err + } + + protoBytes := w.buf.Bytes() + + if w.lengthPrefixBuf == nil { + w.lengthPrefixBuf = make([]byte, binary.MaxVarintLen64) + } + + lengthBytes := w.lengthPrefixBuf[0:binary.PutUvarint(w.lengthPrefixBuf[:], uint64(len(protoBytes)))] + + if _, err := w.writer.Write(lengthBytes); err != nil { + return err + } + + if _, err := w.writer.Write(protoBytes); err != nil { + return err + } + + return nil +} + +// NewProtoStreamWriter creates a new ProtoStreamWriter on top of a specified writer with a specified optional header. +func NewProtoStreamWriter(w io.Writer, header []byte) *ProtoStreamWriter { + return &ProtoStreamWriter{ + writer: w, + buf: proto.NewBuffer(nil), + header: header, + } +} diff --git a/internal/proto_test.pb.go b/internal/proto_test.pb.go new file mode 100644 index 000000000..2a4f367ec --- /dev/null +++ b/internal/proto_test.pb.go @@ -0,0 +1,51 @@ +// Code generated by protoc-gen-go. +// source: internal/proto_test.proto +// DO NOT EDIT! + +/* +Package internal is a generated protocol buffer package. + +It is generated from these files: + internal/proto_test.proto + +It has these top-level messages: + TestProto +*/ +package internal + +import proto "github.com/golang/protobuf/proto" +import fmt "fmt" +import math "math" + +// Reference imports to suppress errors if they are not otherwise used. +var _ = proto.Marshal +var _ = fmt.Errorf +var _ = math.Inf + +// This is a compile-time assertion to ensure that this generated file +// is compatible with the proto package it is being compiled against. +const _ = proto.ProtoPackageIsVersion1 + +type TestProto struct { + Name string `protobuf:"bytes,1,opt,name=name" json:"name,omitempty"` +} + +func (m *TestProto) Reset() { *m = TestProto{} } +func (m *TestProto) String() string { return proto.CompactTextString(m) } +func (*TestProto) ProtoMessage() {} +func (*TestProto) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{0} } + +func init() { + proto.RegisterType((*TestProto)(nil), "kopia.fs.TestProto") +} + +var fileDescriptor0 = []byte{ + // 109 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0xe2, 0x92, 0xcc, 0xcc, 0x2b, 0x49, + 0x2d, 0xca, 0x4b, 0xcc, 0xd1, 0x2f, 0x28, 0xca, 0x2f, 0xc9, 0x8f, 0x2f, 0x49, 0x2d, 0x2e, 0xd1, + 0x03, 0x33, 0x85, 0x38, 0xb2, 0xf3, 0x0b, 0x32, 0x13, 0xf5, 0xd2, 0x8a, 0x95, 0xe4, 0xb9, 0x38, + 0x43, 0x80, 0xe2, 0x01, 0x60, 0x61, 0x21, 0x2e, 0x96, 0xbc, 0xc4, 0xdc, 0x54, 0x09, 0x46, 0x05, + 0x46, 0x0d, 0xce, 0x20, 0x30, 0xdb, 0x49, 0x31, 0x4a, 0x3e, 0x3d, 0xb3, 0x24, 0xa3, 0x34, 0x49, + 0x2f, 0x39, 0x3f, 0x57, 0x1f, 0xac, 0x0f, 0x4a, 0xc2, 0x8c, 0x4f, 0x62, 0x03, 0x1b, 0x6a, 0x0c, + 0x08, 0x00, 0x00, 0xff, 0xff, 0x49, 0x42, 0xd9, 0xa7, 0x71, 0x00, 0x00, 0x00, +} diff --git a/internal/proto_test.proto b/internal/proto_test.proto new file mode 100644 index 000000000..f658f1d2d --- /dev/null +++ b/internal/proto_test.proto @@ -0,0 +1,8 @@ +syntax = "proto3"; + +package kopia.fs; +option go_package = "github.com/kopia/kopia/internal"; + +message TestProto { + string name = 1; +} diff --git a/repo/format.go b/repo/format.go index 9be0be294..6bf217307 100644 --- a/repo/format.go +++ b/repo/format.go @@ -5,9 +5,10 @@ import ( "crypto/cipher" "crypto/hmac" "crypto/md5" - "crypto/sha1" "crypto/sha256" "crypto/sha512" + "errors" + "fmt" "hash" ) @@ -16,36 +17,83 @@ var ( hkdfInfoCryptoKey = []byte("CryptoKey") ) -// Format describes the format of object data. -type Format struct { - Version string `json:"version"` - ObjectFormat string `json:"objectFormat"` - Secret []byte `json:"secret,omitempty"` - MaxInlineBlobSize int `json:"maxInlineBlobSize"` - MaxBlobSize int `json:"maxBlobSize"` +// ObjectIDFormatInfo performs hashing and encryption of a data block. +type ObjectIDFormatInfo interface { + Name() string + HashBuffer(data []byte, secret []byte) (blockID []byte, cryptoKey []byte) + CreateCipher(key []byte) (cipher.Block, error) } -// ObjectIDFormat describes single format ObjectID -type ObjectIDFormat struct { - Name string - IsEncrypted bool - - hashBuffer func(data []byte, secret []byte) ([]byte, []byte) - createCipher func(key []byte) (cipher.Block, error) +type unencryptedFormat struct { + name string + hashCtor func() hash.Hash + fold int } -// ObjectIDFormats is a collection of ObjectIDFormat -type ObjectIDFormats []*ObjectIDFormat +func (fi *unencryptedFormat) Name() string { + return fi.name +} -// Find returns the ObjectIDFormat with a given name or nil if not found. -func (fmts ObjectIDFormats) Find(name string) *ObjectIDFormat { - for _, f := range fmts { - if f.Name == name { - return f +func (fi *unencryptedFormat) HashBuffer(data []byte, secret []byte) (blockID []byte, cryptoKey []byte) { + blockID = hashContent(fi.hashCtor, data, secret) + if fi.fold > 0 { + for i := fi.fold; i < len(blockID); i++ { + blockID[i%fi.fold] ^= blockID[i] } + blockID = blockID[0:fi.fold] + } + return +} + +func (fi *unencryptedFormat) CreateCipher(key []byte) (cipher.Block, error) { + return nil, errors.New("encryption not supported") + +} + +type encryptedFormat struct { + name string + hashCtor func() hash.Hash + createCipher func(key []byte) (cipher.Block, error) + keyBytes int +} + +func (fi *encryptedFormat) Name() string { + return fi.name +} + +func (fi *encryptedFormat) HashBuffer(data []byte, secret []byte) (blockID []byte, cryptoKey []byte) { + h := hashContent(fi.hashCtor, data, secret) + p := len(h) - fi.keyBytes + blockID = h[0:p] + cryptoKey = h[p:] + return +} + +func (fi *encryptedFormat) CreateCipher(key []byte) (cipher.Block, error) { + return fi.createCipher(key) + +} + +// SupportedFormats containes supported ObjectIDformats +var SupportedFormats = map[ObjectIDFormat]ObjectIDFormatInfo{ + ObjectIDFormat_TESTONLY_MD5: &unencryptedFormat{"TESTONLY_MD5", md5.New, 0}, + ObjectIDFormat_UNENCYPTED_HMAC_SHA256: &unencryptedFormat{"UNENCRYPTED_HMAC_SHA256", sha256.New, 0}, + ObjectIDFormat_UNENCYPTED_HMAC_SHA256_128: &unencryptedFormat{"UNENCRYPTED_HMAC_SHA256", sha256.New, 16}, + ObjectIDFormat_ENCRYPTED_HMAC_SHA512_384_AES256: &encryptedFormat{"ENCRYPTED_HMAC_SHA512_384_AES256", sha512.New384, aes.NewCipher, 32}, + ObjectIDFormat_ENCRYPTED_HMAC_SHA512_AES256: &encryptedFormat{"ENCRYPTED_HMAC_SHA512_AES256", sha512.New, aes.NewCipher, 32}, +} + +// DefaultObjectFormat is the format that should be used by default when creating new repositories. +var DefaultObjectFormat = SupportedFormats[ObjectIDFormat_ENCRYPTED_HMAC_SHA512_AES256] + +// ParseObjectIDFormat parses the given string and returns corresponding ObjectIDFormat. +func ParseObjectIDFormat(s string) (ObjectIDFormat, error) { + v, ok := ObjectIDFormat_value[s] + if ok { + return ObjectIDFormat(v), nil } - return nil + return ObjectIDFormat_INVALID, fmt.Errorf("unknown object ID format: %v", s) } func fold(b []byte, size int) []byte { @@ -59,15 +107,6 @@ func fold(b []byte, size int) []byte { return b[0:size] } -func nonEncryptedFormat(name string, hf func() hash.Hash, hashSize int) *ObjectIDFormat { - return &ObjectIDFormat{ - Name: name, - hashBuffer: func(data []byte, secret []byte) ([]byte, []byte) { - return fold(hashContent(hf, data, secret), hashSize), nil - }, - } -} - func hashContent(hf func() hash.Hash, data []byte, secret []byte) []byte { var h hash.Hash @@ -79,55 +118,3 @@ func hashContent(hf func() hash.Hash, data []byte, secret []byte) []byte { h.Write(data) return h.Sum(nil) } - -func encryptedFormat( - name string, - hf func() hash.Hash, - blockIDSize int, - createCipher func(key []byte) (cipher.Block, error), - keySize int, -) *ObjectIDFormat { - return &ObjectIDFormat{ - Name: name, - hashBuffer: func(data []byte, secret []byte) ([]byte, []byte) { - contentHash := hashContent(sha512.New, data, secret) - blockID := fold(hashContent(hf, hkdfInfoBlockID, contentHash), blockIDSize) - cryptoKey := fold(hashContent(hf, hkdfInfoCryptoKey, contentHash), keySize) - return blockID, cryptoKey - }, - createCipher: createCipher, - } -} - -func buildObjectIDFormats() ObjectIDFormats { - var result ObjectIDFormats - - for _, h := range []struct { - name string - hash func() hash.Hash - hashSize int - }{ - {"md5", md5.New, md5.Size}, - {"sha1", sha1.New, sha1.Size}, - {"sha224", sha256.New224, sha256.Size224}, - {"sha256", sha256.New, sha256.Size}, - {"sha256-fold128", sha256.New, 16}, - {"sha256-fold160", sha256.New, 20}, - {"sha384", sha512.New384, sha512.Size384}, - {"sha512-fold128", sha512.New, 16}, - {"sha512-fold160", sha512.New, 20}, - {"sha512-224", sha512.New512_224, sha512.Size224}, - {"sha512-256", sha512.New512_256, sha512.Size256}, - {"sha512", sha512.New, sha512.Size}, - } { - result = append(result, nonEncryptedFormat(h.name, h.hash, h.hashSize)) - result = append(result, encryptedFormat(h.name+"-aes128", h.hash, h.hashSize, aes.NewCipher, 16)) - result = append(result, encryptedFormat(h.name+"-aes192", h.hash, h.hashSize, aes.NewCipher, 24)) - result = append(result, encryptedFormat(h.name+"-aes256", h.hash, h.hashSize, aes.NewCipher, 32)) - } - - return result -} - -// SupportedFormats contains supported repository formats. -var SupportedFormats = buildObjectIDFormats() diff --git a/repo/format.pb.go b/repo/format.pb.go new file mode 100644 index 000000000..72f471ca4 --- /dev/null +++ b/repo/format.pb.go @@ -0,0 +1,141 @@ +// Code generated by protoc-gen-go. +// source: repo/format.proto +// DO NOT EDIT! + +/* +Package repo is a generated protocol buffer package. + +It is generated from these files: + repo/format.proto + repo/indirect.proto + repo/objectid.proto + +It has these top-level messages: + Format + IndirectObjectEntry + ObjectID +*/ +package repo + +import proto "github.com/golang/protobuf/proto" +import fmt "fmt" +import math "math" + +// Reference imports to suppress errors if they are not otherwise used. +var _ = proto.Marshal +var _ = fmt.Errorf +var _ = math.Inf + +// This is a compile-time assertion to ensure that this generated file +// is compatible with the proto package it is being compiled against. +const _ = proto.ProtoPackageIsVersion1 + +type EncryptionAlgorithm int32 + +const ( + EncryptionAlgorithm_NONE EncryptionAlgorithm = 0 + EncryptionAlgorithm_AES EncryptionAlgorithm = 1 +) + +var EncryptionAlgorithm_name = map[int32]string{ + 0: "NONE", + 1: "AES", +} +var EncryptionAlgorithm_value = map[string]int32{ + "NONE": 0, + "AES": 1, +} + +func (x EncryptionAlgorithm) String() string { + return proto.EnumName(EncryptionAlgorithm_name, int32(x)) +} +func (EncryptionAlgorithm) EnumDescriptor() ([]byte, []int) { return fileDescriptor0, []int{0} } + +// ObjectIDFormat describes the format of repository objects. +type ObjectIDFormat int32 + +const ( + // Format not specified. + ObjectIDFormat_INVALID ObjectIDFormat = 0 + // MD5 (no encryption, MD5 hashing, only used for testing) + ObjectIDFormat_TESTONLY_MD5 ObjectIDFormat = 1 + // HMAC-SHA256 hash of content used as object ID, no encryption. + ObjectIDFormat_UNENCYPTED_HMAC_SHA256 ObjectIDFormat = 2 + // HMAC-SHA256 hash folded into 128 bits used as object ID, no encryption. + ObjectIDFormat_UNENCYPTED_HMAC_SHA256_128 ObjectIDFormat = 3 + // HMAC-SHA512 hash of content, top part of the hash used as 256-bit object ID, + // bottom part used as 256-bit encryption key. + ObjectIDFormat_ENCRYPTED_HMAC_SHA512_AES256 ObjectIDFormat = 4 + // HMAC-SHA512_384 hash over contents, top part of the hash used as 128-bit object ID, + // bottom part used as 256-bit encryption key. + ObjectIDFormat_ENCRYPTED_HMAC_SHA512_384_AES256 ObjectIDFormat = 5 +) + +var ObjectIDFormat_name = map[int32]string{ + 0: "INVALID", + 1: "TESTONLY_MD5", + 2: "UNENCYPTED_HMAC_SHA256", + 3: "UNENCYPTED_HMAC_SHA256_128", + 4: "ENCRYPTED_HMAC_SHA512_AES256", + 5: "ENCRYPTED_HMAC_SHA512_384_AES256", +} +var ObjectIDFormat_value = map[string]int32{ + "INVALID": 0, + "TESTONLY_MD5": 1, + "UNENCYPTED_HMAC_SHA256": 2, + "UNENCYPTED_HMAC_SHA256_128": 3, + "ENCRYPTED_HMAC_SHA512_AES256": 4, + "ENCRYPTED_HMAC_SHA512_384_AES256": 5, +} + +func (x ObjectIDFormat) String() string { + return proto.EnumName(ObjectIDFormat_name, int32(x)) +} +func (ObjectIDFormat) EnumDescriptor() ([]byte, []int) { return fileDescriptor0, []int{1} } + +// Format describes the format of object data. +type Format struct { + Version int32 `protobuf:"varint,1,opt,name=version" json:"version,omitempty"` + ObjectFormat ObjectIDFormat `protobuf:"varint,2,opt,name=object_format,json=objectFormat,enum=kopia.repo.ObjectIDFormat" json:"object_format,omitempty"` + Secret []byte `protobuf:"bytes,3,opt,name=secret,proto3" json:"secret,omitempty"` + MaxInlineBlobSize int32 `protobuf:"varint,4,opt,name=max_inline_blob_size,json=maxInlineBlobSize" json:"max_inline_blob_size,omitempty"` + MaxBlobSize int32 `protobuf:"varint,5,opt,name=max_blob_size,json=maxBlobSize" json:"max_blob_size,omitempty"` +} + +func (m *Format) Reset() { *m = Format{} } +func (m *Format) String() string { return proto.CompactTextString(m) } +func (*Format) ProtoMessage() {} +func (*Format) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{0} } + +func init() { + proto.RegisterType((*Format)(nil), "kopia.repo.Format") + proto.RegisterEnum("kopia.repo.EncryptionAlgorithm", EncryptionAlgorithm_name, EncryptionAlgorithm_value) + proto.RegisterEnum("kopia.repo.ObjectIDFormat", ObjectIDFormat_name, ObjectIDFormat_value) +} + +var fileDescriptor0 = []byte{ + // 357 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0x6c, 0x91, 0xcb, 0x6a, 0xf2, 0x40, + 0x14, 0xc7, 0xbf, 0x78, 0xfd, 0x38, 0x5e, 0x88, 0xd3, 0x22, 0xc1, 0x5e, 0x10, 0xe9, 0x42, 0x5c, + 0x44, 0x8c, 0xb5, 0xb8, 0x2b, 0xd1, 0xa4, 0x18, 0xd0, 0x58, 0x12, 0x5b, 0xb0, 0x9b, 0xc1, 0x84, + 0xd4, 0xa6, 0x35, 0x99, 0x30, 0xa6, 0xa5, 0xed, 0x6b, 0xf5, 0x45, 0xfa, 0x48, 0x9d, 0x8c, 0x8a, + 0x08, 0x6e, 0x06, 0xce, 0xfc, 0x7e, 0xe7, 0x3f, 0x67, 0x38, 0x50, 0xa1, 0x5e, 0x44, 0xda, 0xcf, + 0x84, 0x06, 0x8b, 0x58, 0x8e, 0x28, 0x89, 0x09, 0x82, 0x37, 0x12, 0xf9, 0x0b, 0x39, 0x01, 0x8d, + 0x5f, 0x01, 0x72, 0x77, 0x1c, 0x22, 0x09, 0xf2, 0x1f, 0x1e, 0x5d, 0xfb, 0x24, 0x94, 0x84, 0xba, + 0xd0, 0xcc, 0x5a, 0xbb, 0x12, 0xdd, 0x42, 0x89, 0x38, 0xaf, 0x9e, 0x1b, 0xe3, 0x4d, 0x8e, 0x94, + 0x62, 0xbc, 0xac, 0xd4, 0xe4, 0x7d, 0x90, 0x3c, 0xe5, 0x82, 0xa1, 0x6d, 0xc2, 0xac, 0xe2, 0xa6, + 0x61, 0x1b, 0x5d, 0x85, 0xdc, 0xda, 0x73, 0xa9, 0x17, 0x4b, 0x69, 0xd6, 0x59, 0xb4, 0xb6, 0x15, + 0x6a, 0xc3, 0x69, 0xb0, 0xf8, 0xc4, 0x7e, 0xb8, 0xf2, 0x43, 0x0f, 0x3b, 0x2b, 0xe2, 0xe0, 0xb5, + 0xff, 0xed, 0x49, 0x19, 0xfe, 0x7e, 0x85, 0x31, 0x83, 0xa3, 0x01, 0x23, 0x36, 0x03, 0xa8, 0x01, + 0xa5, 0xa4, 0x61, 0x6f, 0x66, 0xb9, 0x59, 0x60, 0x97, 0x3b, 0xa7, 0xd5, 0x84, 0x13, 0x3d, 0x74, + 0xe9, 0x57, 0x14, 0xb3, 0xd9, 0xd5, 0xd5, 0x92, 0x50, 0x3f, 0x7e, 0x09, 0xd0, 0x7f, 0xc8, 0x98, + 0x53, 0x53, 0x17, 0xff, 0xa1, 0x3c, 0xa4, 0x55, 0xdd, 0x16, 0x85, 0xd6, 0x8f, 0x00, 0xe5, 0xc3, + 0xb9, 0x51, 0x01, 0xf2, 0x86, 0xf9, 0xa8, 0x8e, 0x0d, 0x8d, 0x89, 0x22, 0x14, 0x67, 0xba, 0x3d, + 0x9b, 0x9a, 0xe3, 0x39, 0x9e, 0x68, 0x3d, 0x51, 0x40, 0x35, 0xa8, 0x3e, 0x98, 0xba, 0x39, 0x9c, + 0xdf, 0xcf, 0x74, 0x0d, 0x8f, 0x26, 0xea, 0x10, 0xdb, 0x23, 0x55, 0xe9, 0xdd, 0x88, 0x29, 0x74, + 0x09, 0xb5, 0xe3, 0x0c, 0x77, 0x94, 0xbe, 0x98, 0x46, 0x75, 0x38, 0x67, 0xd4, 0x3a, 0xc4, 0xbd, + 0x8e, 0x82, 0xd9, 0x30, 0x49, 0x42, 0x06, 0x5d, 0x41, 0xfd, 0xb8, 0xd1, 0xed, 0x5f, 0xef, 0xac, + 0xec, 0xe0, 0xe2, 0xe9, 0x6c, 0xc9, 0x7e, 0xf4, 0xee, 0xc8, 0x2e, 0x09, 0xda, 0x7c, 0x05, 0xdb, + 0x33, 0x59, 0x84, 0x93, 0xe3, 0x4b, 0xee, 0xfe, 0x05, 0x00, 0x00, 0xff, 0xff, 0x3b, 0xcf, 0x39, + 0xd2, 0xf9, 0x01, 0x00, 0x00, +} diff --git a/repo/format.proto b/repo/format.proto new file mode 100644 index 000000000..5589470aa --- /dev/null +++ b/repo/format.proto @@ -0,0 +1,41 @@ +syntax = "proto3"; + +package kopia.repo; +option go_package = "github.com/kopia/kopia/repo"; + +enum EncryptionAlgorithm { + NONE = 0; + AES = 1; +} + +// ObjectIDFormat describes the format of repository objects. +enum ObjectIDFormat { + // Format not specified. + INVALID = 0; + + // MD5 (no encryption, MD5 hashing, only used for testing) + TESTONLY_MD5 = 1; + + // HMAC-SHA256 hash of content used as object ID, no encryption. + UNENCYPTED_HMAC_SHA256 = 2; + + // HMAC-SHA256 hash folded into 128 bits used as object ID, no encryption. + UNENCYPTED_HMAC_SHA256_128 = 3; + + // HMAC-SHA512 hash of content, top part of the hash used as 256-bit object ID, + // bottom part used as 256-bit encryption key. + ENCRYPTED_HMAC_SHA512_AES256 = 4; + + // HMAC-SHA512_384 hash over contents, top part of the hash used as 128-bit object ID, + // bottom part used as 256-bit encryption key. + ENCRYPTED_HMAC_SHA512_384_AES256 = 5; +} + +// Format describes the format of object data. +message Format { + int32 version = 1; + ObjectIDFormat object_format = 2; + bytes secret = 3; + int32 max_inline_blob_size = 4; + int32 max_blob_size = 5; +} diff --git a/repo/indirect.pb.go b/repo/indirect.pb.go new file mode 100644 index 000000000..50c9981ee --- /dev/null +++ b/repo/indirect.pb.go @@ -0,0 +1,52 @@ +// Code generated by protoc-gen-go. +// source: repo/indirect.proto +// DO NOT EDIT! + +package repo + +import proto "github.com/golang/protobuf/proto" +import fmt "fmt" +import math "math" + +// Reference imports to suppress errors if they are not otherwise used. +var _ = proto.Marshal +var _ = fmt.Errorf +var _ = math.Inf + +// IndirectObjectEntry represents an entry in indirect object proto stream. +type IndirectObjectEntry struct { + Start int64 `protobuf:"varint,1,opt,name=start" json:"start,omitempty"` + Length int64 `protobuf:"varint,2,opt,name=length" json:"length,omitempty"` + Object *ObjectID `protobuf:"bytes,3,opt,name=object" json:"object,omitempty"` +} + +func (m *IndirectObjectEntry) Reset() { *m = IndirectObjectEntry{} } +func (m *IndirectObjectEntry) String() string { return proto.CompactTextString(m) } +func (*IndirectObjectEntry) ProtoMessage() {} +func (*IndirectObjectEntry) Descriptor() ([]byte, []int) { return fileDescriptor1, []int{0} } + +func (m *IndirectObjectEntry) GetObject() *ObjectID { + if m != nil { + return m.Object + } + return nil +} + +func init() { + proto.RegisterType((*IndirectObjectEntry)(nil), "kopia.repo.IndirectObjectEntry") +} + +var fileDescriptor1 = []byte{ + // 165 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0xe2, 0x12, 0x2e, 0x4a, 0x2d, 0xc8, + 0xd7, 0xcf, 0xcc, 0x4b, 0xc9, 0x2c, 0x4a, 0x4d, 0x2e, 0xd1, 0x2b, 0x28, 0xca, 0x2f, 0xc9, 0x17, + 0xe2, 0xca, 0xce, 0x2f, 0xc8, 0x4c, 0xd4, 0x03, 0x49, 0x49, 0x41, 0x14, 0xe4, 0x27, 0x65, 0x01, + 0xa5, 0x33, 0x53, 0x20, 0x0a, 0x94, 0x0a, 0xb9, 0x84, 0x3d, 0xa1, 0x5a, 0xfc, 0xc1, 0x32, 0xae, + 0x79, 0x25, 0x45, 0x95, 0x42, 0x22, 0x5c, 0xac, 0xc5, 0x25, 0x89, 0x45, 0x25, 0x12, 0x8c, 0x0a, + 0x8c, 0x1a, 0xcc, 0x41, 0x10, 0x8e, 0x90, 0x18, 0x17, 0x5b, 0x4e, 0x6a, 0x5e, 0x7a, 0x49, 0x86, + 0x04, 0x13, 0x58, 0x18, 0xca, 0x13, 0xd2, 0xe1, 0x62, 0x83, 0x18, 0x2b, 0xc1, 0x0c, 0x14, 0xe7, + 0x36, 0x12, 0xd1, 0x43, 0x58, 0xab, 0x07, 0x31, 0xd6, 0xd3, 0x25, 0x08, 0xaa, 0xc6, 0x49, 0x36, + 0x4a, 0x3a, 0x3d, 0xb3, 0x24, 0xa3, 0x34, 0x49, 0x2f, 0x39, 0x3f, 0x57, 0x1f, 0xac, 0x12, 0x4a, + 0x82, 0xd4, 0x27, 0xb1, 0x81, 0x1d, 0x66, 0x0c, 0x08, 0x00, 0x00, 0xff, 0xff, 0x0c, 0xb4, 0xa1, + 0x12, 0xd0, 0x00, 0x00, 0x00, +} diff --git a/repo/indirect.proto b/repo/indirect.proto new file mode 100644 index 000000000..7e0bb0c6c --- /dev/null +++ b/repo/indirect.proto @@ -0,0 +1,13 @@ +syntax = "proto3"; + +package kopia.repo; +option go_package = "github.com/kopia/kopia/repo"; + +import "repo/objectid.proto"; + +// IndirectObjectEntry represents an entry in indirect object proto stream. +message IndirectObjectEntry { + int64 start = 1; + int64 length = 2; + repo.ObjectID object = 3; +} diff --git a/repo/object_reader.go b/repo/object_reader.go index 558724526..1259f8c42 100644 --- a/repo/object_reader.go +++ b/repo/object_reader.go @@ -3,32 +3,16 @@ package repo import ( "fmt" "io" - - "github.com/kopia/kopia/storage" ) -type seekTableEntry struct { - startOffset int64 - length int64 - blockID string -} - -func (r *seekTableEntry) endOffset() int64 { - return r.startOffset + int64(r.length) -} - -func (r *seekTableEntry) String() string { - return fmt.Sprintf("start: %d len: %d end: %d block: %s", - r.startOffset, - r.length, - r.endOffset(), - r.blockID) +func (i *IndirectObjectEntry) endOffset() int64 { + return i.Start + i.Length } type objectReader struct { - storage storage.Storage + repo Repository - seekTable []seekTableEntry + seekTable []IndirectObjectEntry currentPosition int64 // Overall position in the objectReader totalLength int64 // Overall length @@ -80,13 +64,19 @@ func (r *objectReader) Read(buffer []byte) (int, error) { } func (r *objectReader) openCurrentChunk() error { - blockID := r.seekTable[r.currentChunkIndex].blockID - blockData, err := r.storage.GetBlock(blockID) + st := r.seekTable[r.currentChunkIndex] + blockData, err := r.repo.Open(*st.Object) if err != nil { return err } + defer blockData.Close() - r.currentChunkData = blockData + b := make([]byte, st.Length) + if _, err := io.ReadFull(blockData, b); err != nil { + return err + } + + r.currentChunkData = b r.currentChunkPosition = 0 return nil } @@ -101,7 +91,7 @@ func (r *objectReader) findChunkIndexForOffset(offset int64) int { for left <= right { middle := (left + right) / 2 - if offset < r.seekTable[middle].startOffset { + if offset < r.seekTable[middle].Start { right = middle - 1 continue } @@ -136,7 +126,7 @@ func (r *objectReader) Seek(offset int64, whence int) (int64, error) { index := r.findChunkIndexForOffset(offset) - chunkStartOffset := r.seekTable[index].startOffset + chunkStartOffset := r.seekTable[index].Start if index != r.currentChunkIndex { r.closeCurrentChunk() diff --git a/repo/object_writer.go b/repo/object_writer.go index a1cef82e1..6d6adad2d 100644 --- a/repo/object_writer.go +++ b/repo/object_writer.go @@ -5,6 +5,7 @@ import ( "fmt" "io" + "github.com/kopia/kopia/internal" "github.com/kopia/kopia/storage" ) @@ -44,15 +45,15 @@ type objectWriter struct { prefix string listWriter *objectWriter + listProtoWriter *internal.ProtoStreamWriter + listCurrentPos int64 flushedObjectCount int lastFlushedObject ObjectID - description string - objectType ObjectIDType + description string + indirectLevel int32 blockTracker *blockTracker - - atomicWrites bool } func (w *objectWriter) Close() error { @@ -68,9 +69,13 @@ func (w *objectWriter) Close() error { return nil } -func (w *objectWriter) Write(data []byte) (n int, err error) { - dataLen := len(data) - remaining := len(data) +func (w *objectWriter) WriteGather(dataSlices [][]byte) (n int, err error) { + dataLen := 0 + for _, s := range dataSlices { + dataLen += len(s) + } + + remaining := dataLen w.totalLength += int64(remaining) for remaining > 0 { @@ -80,33 +85,45 @@ func (w *objectWriter) Write(data []byte) (n int, err error) { room := w.buffer.Cap() - w.buffer.Len() if remaining <= room { - w.buffer.Write(data) + for _, s := range dataSlices { + w.buffer.Write(s) + } + dataSlices = nil remaining = 0 } else { - if w.atomicWrites { - if w.buffer == nil { - // We're at the beginning of a buffer, fail if the buffer is too small. - return 0, fmt.Errorf("object writer buffer too small, need: %v, have: %v", remaining, room) - } - if err := w.flushBuffer(false); err != nil { - return 0, err - } + for room > 0 { + var chunk []byte - continue + if len(dataSlices[0]) >= room { + chunk = dataSlices[0][0:room] + dataSlices[0] = dataSlices[0][len(chunk):] + } else { + chunk = dataSlices[0] + dataSlices[0] = nil + } + w.buffer.Write(chunk) + remaining -= len(chunk) + room -= len(chunk) + if len(dataSlices[0]) == 0 { + dataSlices = dataSlices[1:] + } } - w.buffer.Write(data[0:room]) - if err := w.flushBuffer(false); err != nil { return 0, err } - data = data[room:] - remaining = len(data) } } return dataLen, nil } +func (w *objectWriter) Write(data []byte) (n int, err error) { + var b [1][]byte + + b[0] = data + return w.WriteGather(b[:]) +} + func (w *objectWriter) flushBuffer(force bool) error { // log.Printf("flushing bufer") // defer log.Printf("flushed") @@ -118,36 +135,43 @@ func (w *objectWriter) flushBuffer(force bool) error { b := w.buffer w.buffer = nil - objectID, blockReader, err := w.repo.hashBufferForWriting(b, string(w.objectType)+w.prefix) + objectID, blockReader, err := w.repo.hashBufferForWriting(b, w.prefix) if err != nil { return err } - if err := w.repo.storage.PutBlock(objectID.BlockID(), blockReader, storage.PutOptionsDefault); err != nil { + if err := w.repo.storage.PutBlock(objectID.StorageBlock, blockReader, storage.PutOptionsDefault); err != nil { return fmt.Errorf( "error when flushing chunk %d of %s to %#v: %#v", w.flushedObjectCount, w.description, - objectID.BlockID(), + objectID.StorageBlock, err) } - w.blockTracker.addBlock(objectID.BlockID()) + w.blockTracker.addBlock(objectID.StorageBlock) w.flushedObjectCount++ w.lastFlushedObject = objectID if w.listWriter == nil { w.listWriter = &objectWriter{ - repo: w.repo, - objectType: ObjectIDTypeList, - prefix: w.prefix, - description: "LIST(" + w.description + ")", - atomicWrites: true, - blockTracker: w.blockTracker, + repo: w.repo, + indirectLevel: w.indirectLevel + 1, + prefix: w.prefix, + description: "LIST(" + w.description + ")", + blockTracker: w.blockTracker, } + w.listProtoWriter = internal.NewProtoStreamWriter(w.listWriter, internal.ProtoStreamTypeIndirect) + w.listCurrentPos = 0 } - fmt.Fprintf(w.listWriter, "%v,%v\n", length, objectID) + w.listProtoWriter.Write(&IndirectObjectEntry{ + Object: &objectID, + Start: w.listCurrentPos, + Length: int64(length), + }) + + w.listCurrentPos += int64(length) } return nil } @@ -155,10 +179,10 @@ func (w *objectWriter) flushBuffer(force bool) error { func (w *objectWriter) Result(forceStored bool) (ObjectID, error) { if !forceStored && w.flushedObjectCount == 0 { if w.buffer == nil { - return "B", nil + return NullObjectID, nil } - if w.buffer.Len() < w.repo.format.MaxInlineBlobSize { + if w.buffer.Len() < int(w.repo.format.MaxInlineBlobSize) { return NewInlineObjectID(w.buffer.Bytes()), nil } } @@ -171,9 +195,10 @@ func (w *objectWriter) Result(forceStored bool) (ObjectID, error) { }() if w.flushedObjectCount == 1 { + w.lastFlushedObject.Indirect = w.indirectLevel return w.lastFlushedObject, nil } else if w.flushedObjectCount == 0 { - return "", nil + return NullObjectID, nil } else { return w.listWriter.Result(true) } diff --git a/repo/objectid.go b/repo/objectid.go index 017940ab4..5ef580606 100644 --- a/repo/objectid.go +++ b/repo/objectid.go @@ -4,87 +4,72 @@ import ( "encoding/base64" "encoding/hex" "errors" - "fmt" "strconv" "strings" - "unicode/utf8" + + "github.com/golang/protobuf/proto" + + "fmt" ) +// NullObjectID is the identifier of an null object. +var NullObjectID ObjectID + +const objectIDEncryptionInfoSeparator = "." + var ( - binaryEncoding = base64.RawURLEncoding + inlineContentEncoding = base64.RawURLEncoding ) -const ( - objectIDEncryptionInfoSeparator = "." -) +// UIString returns the name of the repository object that is suitable for displaying in the UI. +// Note that the object ID name may contain its encryption key, which is sensitive. +func (m *ObjectID) UIString() string { + if m.StorageBlock != "" { + var encryptionSuffix string -// ObjectIDType describes the type of the chunk. -type ObjectIDType string - -// ObjectEncryptionInfo represents encryption info associated with ObjectID. -type ObjectEncryptionInfo string - -// NoEncryption indicates that the object is not encrypted. -var NoEncryption = ObjectEncryptionInfo("") - -const ( - // ObjectIDTypeText represents text-only inline object ID - ObjectIDTypeText ObjectIDType = "T" - - // ObjectIDTypeBinary represents binary inline object ID - ObjectIDTypeBinary ObjectIDType = "B" - - // ObjectIDTypeStored represents ID of object whose data is stored directly in a single storage block indicated by string. - ObjectIDTypeStored ObjectIDType = "D" - - // ObjectIDTypeList represents ID of an object whose data is stored in mutliple storage blocks. - // The value of the ObjectID is the list chunk, which lists object IDs that need to be concatenated - // to form the contents. - ObjectIDTypeList ObjectIDType = "L" - - // ObjectIDTypeSection represents ID of an object whose data is a section of another object. - // The format is S{offset},{length},{base} - ObjectIDTypeSection ObjectIDType = "S" -) - -// IsStored determines whether data for the given chunk type is stored in the storage -// (as opposed to being stored inline as part of ObjectID itself). -func (ct ObjectIDType) IsStored() bool { - switch ct { - case ObjectIDTypeStored, ObjectIDTypeList: - return true - - default: - return false - } -} - -// ObjectID represents the identifier of a chunk. -// Identifiers can either refer to data block stored in a Storage, or contain small amounts -// of payload directly (useful for short ASCII or binary files). -type ObjectID string - -// Type gets the type of the object ID. -func (c ObjectID) Type() ObjectIDType { - return ObjectIDType(c[0:1]) -} - -// InlineData returns inline data stored as part of ObjectID. For chunk IDs representing stored -// chunks, the value is nil. -func (c ObjectID) InlineData() []byte { - payload := string(c[1:]) - switch c.Type() { - case ObjectIDTypeText: - return []byte(payload) - - case ObjectIDTypeBinary: - decodedPayload, err := binaryEncoding.DecodeString(payload) - if err == nil { - return decodedPayload + if m.Encryption != nil { + encryptionSuffix = "." + hex.EncodeToString(m.Encryption) } + + if m.Indirect > 0 { + return fmt.Sprintf("L%v,%v%v", m.Indirect, m.StorageBlock, encryptionSuffix) + } + + return "D" + m.StorageBlock + encryptionSuffix } - return nil + if m.Content != nil { + return "B" + inlineContentEncoding.EncodeToString(m.Content) + } + + if m.Section != nil { + return fmt.Sprintf("S%v,%v,%v", m.Section.Start, m.Section.Length, m.Section.Base.UIString()) + } + + return "B" +} + +// Equals determines whether ObjectID is equal to another ObjectID. +func (m *ObjectID) Equals(other ObjectID) bool { + return proto.Equal(m, &other) +} + +// NewInlineObjectID returns new ObjectID containing inline binary or text-encoded data. +func NewInlineObjectID(data []byte) ObjectID { + return ObjectID{ + Content: data, + } +} + +// NewSectionObjectID returns new ObjectID representing a section of an object with a given base ID, start offset and length. +func NewSectionObjectID(start, length int64, baseID *ObjectID) ObjectID { + return ObjectID{ + Section: &ObjectID_Section{ + Base: baseID, + Start: start, + Length: length, + }, + } } // parseNumberUntilComma parses a string of the form "{x},{remainder}" where x is a 64-bit number and remainder is arbitrary string. @@ -104,134 +89,82 @@ func parseNumberUntilComma(s string) (int64, string, error) { } func parseSectionInfoString(s string) (int64, int64, ObjectID, error) { - if ObjectIDType(s[0]) != ObjectIDTypeSection { - return 0, -1, "", errors.New("not a section object") - } - var start, length int64 var err error start, s, err = parseNumberUntilComma(s[1:]) if err != nil { - return 0, -1, "", err + return 0, -1, NullObjectID, err } length, s, err = parseNumberUntilComma(s) if err != nil { - return 0, -1, "", err + return 0, -1, NullObjectID, err } oid, err := ParseObjectID(s) if err != nil { - return 0, -1, "", err + return 0, -1, NullObjectID, err } return start, length, oid, nil } -// SectionInfo returns start, length and the base ID of a section object. -func (c ObjectID) SectionInfo() (start int64, length int64, baseID ObjectID) { - if c.Type() != ObjectIDTypeSection { - return 0, 0, "" - } - - start, length, oid, err := parseSectionInfoString(string(c)) - if err != nil { - // This should not happen if we came in through ParseObjectID - panic("invalid section info: " + string(c)) - } - - return start, length, oid -} - -// BlockID returns identifier of the storage block. For inline chunk IDs, an empty string is returned. -func (c ObjectID) BlockID() string { - if c.Type().IsStored() { - content := string(c[1:]) - firstSeparator := strings.Index(content, objectIDEncryptionInfoSeparator) - if firstSeparator > 0 { - return string(content[0:firstSeparator]) - } - - return string(content) - } - - return "" -} - -// EncryptionInfo returns ObjectEncryptionInfo for the ObjectID. -func (c ObjectID) EncryptionInfo() ObjectEncryptionInfo { - if c.Type().IsStored() { - content := string(c[1:]) - firstSeparator := strings.Index(content, objectIDEncryptionInfoSeparator) - if firstSeparator > 0 { - return ObjectEncryptionInfo(content[firstSeparator+1:]) - } - } - - return NoEncryption -} - -// NewInlineObjectID returns new ObjectID containing inline binary or text-encoded data. -func NewInlineObjectID(data []byte) ObjectID { - if !utf8.Valid(data) { - return ObjectID("B" + binaryEncoding.EncodeToString(data)) - } - - for _, b := range data { - if b < 32 && (b != 9 && b != 10 && b != 13) { - // Any other character indicates binary content. - return ObjectID("B" + binaryEncoding.EncodeToString(data)) - } - } - - return ObjectID("T" + string(data)) -} - -// NewSectionObjectID returns new ObjectID representing a section of an object with a given base ID, start offset and length. -func NewSectionObjectID(start, length int64, baseID ObjectID) ObjectID { - return ObjectID(fmt.Sprintf("S%v,%v,%v", start, length, baseID)) -} - // ParseObjectID converts the specified string into ObjectID. func ParseObjectID(objectIDString string) (ObjectID, error) { if len(objectIDString) >= 1 { - chunkType := ObjectIDType(objectIDString[0]) + chunkType := objectIDString[0] content := objectIDString[1:] switch chunkType { - case ObjectIDTypeSection: - if _, _, _, err := parseSectionInfoString(objectIDString); err == nil { - return ObjectID(objectIDString), nil + case 'S': + if start, length, base, err := parseSectionInfoString(objectIDString); err == nil { + return ObjectID{Section: &ObjectID_Section{ + Start: start, + Length: length, + Base: &base, + }}, nil } - case ObjectIDTypeText: - return ObjectID(objectIDString), nil - - case ObjectIDTypeBinary: - if _, err := binaryEncoding.DecodeString(content); err == nil { - return ObjectID(objectIDString), nil + case 'B': + if v, err := inlineContentEncoding.DecodeString(content); err == nil { + return ObjectID{Content: v}, nil + } + + case 'I', 'D': + var indirectLevel int32 + if chunkType == 'I' { + comma := strings.Index(content, ",") + if comma < 0 { + // malformed + break + } + i, err := strconv.Atoi(content[0:comma]) + if err != nil { + break + } + indirectLevel = int32(i) + content = content[comma+1:] } - case ObjectIDTypeList, ObjectIDTypeStored: firstSeparator := strings.Index(content, objectIDEncryptionInfoSeparator) lastSeparator := strings.LastIndex(content, objectIDEncryptionInfoSeparator) if firstSeparator == lastSeparator { if firstSeparator == -1 { // Found zero Separators in the ID - no encryption info. - return ObjectID(objectIDString), nil + return ObjectID{StorageBlock: content, Indirect: indirectLevel}, nil } if firstSeparator > 0 { b, err := hex.DecodeString(content[firstSeparator+1:]) - if err == nil && len(b) > 0 && len(b)%2 == 0 { + if err == nil && len(b) > 0 { // Valid chunk ID with encryption info. - return ObjectID(objectIDString), nil + return ObjectID{StorageBlock: content[0:firstSeparator], Encryption: b}, nil } } } } } - return "", fmt.Errorf("malformed chunk id: '%s'", objectIDString) + return NullObjectID, fmt.Errorf("malformed object id: '%s'", objectIDString) } diff --git a/repo/objectid.pb.go b/repo/objectid.pb.go new file mode 100644 index 000000000..d754e0020 --- /dev/null +++ b/repo/objectid.pb.go @@ -0,0 +1,80 @@ +// Code generated by protoc-gen-go. +// source: repo/objectid.proto +// DO NOT EDIT! + +package repo + +import proto "github.com/golang/protobuf/proto" +import fmt "fmt" +import math "math" + +// Reference imports to suppress errors if they are not otherwise used. +var _ = proto.Marshal +var _ = fmt.Errorf +var _ = math.Inf + +// ObjectID represents the identifier of a chunk. +// Identifiers can either refer to data block stored in a Storage, or contain small amounts +// of payload directly (useful for short ASCII or binary files). +type ObjectID struct { + StorageBlock string `protobuf:"bytes,1,opt,name=storage_block,json=storageBlock" json:"storage_block,omitempty"` + Indirect int32 `protobuf:"varint,2,opt,name=indirect" json:"indirect,omitempty"` + Encryption []byte `protobuf:"bytes,3,opt,name=encryption,proto3" json:"encryption,omitempty"` + Content []byte `protobuf:"bytes,4,opt,name=content,proto3" json:"content,omitempty"` + Section *ObjectID_Section `protobuf:"bytes,5,opt,name=section" json:"section,omitempty"` +} + +func (m *ObjectID) Reset() { *m = ObjectID{} } +func (m *ObjectID) String() string { return proto.CompactTextString(m) } +func (*ObjectID) ProtoMessage() {} +func (*ObjectID) Descriptor() ([]byte, []int) { return fileDescriptor2, []int{0} } + +func (m *ObjectID) GetSection() *ObjectID_Section { + if m != nil { + return m.Section + } + return nil +} + +type ObjectID_Section struct { + Base *ObjectID `protobuf:"bytes,1,opt,name=base" json:"base,omitempty"` + Start int64 `protobuf:"varint,2,opt,name=start" json:"start,omitempty"` + Length int64 `protobuf:"varint,3,opt,name=length" json:"length,omitempty"` +} + +func (m *ObjectID_Section) Reset() { *m = ObjectID_Section{} } +func (m *ObjectID_Section) String() string { return proto.CompactTextString(m) } +func (*ObjectID_Section) ProtoMessage() {} +func (*ObjectID_Section) Descriptor() ([]byte, []int) { return fileDescriptor2, []int{0, 0} } + +func (m *ObjectID_Section) GetBase() *ObjectID { + if m != nil { + return m.Base + } + return nil +} + +func init() { + proto.RegisterType((*ObjectID)(nil), "kopia.repo.ObjectID") + proto.RegisterType((*ObjectID_Section)(nil), "kopia.repo.ObjectID.Section") +} + +var fileDescriptor2 = []byte{ + // 256 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0x6c, 0x90, 0xcf, 0x4a, 0x03, 0x31, + 0x10, 0xc6, 0x69, 0xb7, 0xdb, 0xad, 0xd3, 0x7a, 0x89, 0x45, 0x42, 0xfd, 0x83, 0xe8, 0xa5, 0xa7, + 0x14, 0x2a, 0xf8, 0x00, 0xc5, 0x8b, 0x27, 0x21, 0xde, 0xbc, 0x48, 0x92, 0x86, 0x6d, 0x6c, 0xcd, + 0x2c, 0xd9, 0xf1, 0xe0, 0xdb, 0xf8, 0xa8, 0x66, 0xb3, 0xbb, 0xea, 0xa1, 0x97, 0xc0, 0xf7, 0xfd, + 0x66, 0x86, 0x1f, 0x81, 0xb3, 0x60, 0x2b, 0x5c, 0xa1, 0x7e, 0xb7, 0x86, 0xdc, 0x56, 0x54, 0x01, + 0x09, 0x19, 0xec, 0xb1, 0x72, 0x4a, 0x34, 0xe8, 0xf6, 0x7b, 0x08, 0x93, 0xe7, 0x84, 0x9f, 0x1e, + 0xd9, 0x1d, 0x9c, 0xd6, 0x84, 0x41, 0x95, 0xf6, 0x4d, 0x1f, 0xd0, 0xec, 0xf9, 0xe0, 0x66, 0xb0, + 0x3c, 0x91, 0xb3, 0xae, 0xdc, 0x34, 0x1d, 0x5b, 0xc0, 0xc4, 0xf9, 0xad, 0x0b, 0x71, 0x85, 0x0f, + 0x23, 0xcf, 0xe5, 0x6f, 0x66, 0xd7, 0x00, 0xd6, 0x9b, 0xf0, 0x55, 0x91, 0x43, 0xcf, 0xb3, 0x48, + 0x67, 0xf2, 0x5f, 0xc3, 0x38, 0x14, 0x06, 0x3d, 0x59, 0x4f, 0x7c, 0x94, 0x60, 0x1f, 0xd9, 0x03, + 0x14, 0x75, 0xe3, 0x18, 0xd7, 0xf2, 0x48, 0xa6, 0xeb, 0x4b, 0xf1, 0x67, 0x29, 0x7a, 0x43, 0xf1, + 0xd2, 0xce, 0xc8, 0x7e, 0x78, 0xa1, 0xa0, 0xe8, 0x3a, 0xb6, 0x84, 0x91, 0x56, 0xb5, 0x4d, 0xd2, + 0xd3, 0xf5, 0xfc, 0xd8, 0xbe, 0x4c, 0x13, 0x6c, 0x0e, 0x79, 0x4d, 0x2a, 0xb4, 0xfe, 0x99, 0x6c, + 0x03, 0x3b, 0x87, 0xf1, 0xc1, 0xfa, 0x92, 0x76, 0x49, 0x3c, 0x93, 0x5d, 0xda, 0x5c, 0xbd, 0x5e, + 0x94, 0x8e, 0x76, 0x9f, 0x5a, 0x18, 0xfc, 0x58, 0xa5, 0xab, 0xdd, 0xdb, 0xdc, 0xd6, 0xe3, 0xf4, + 0xa9, 0xf7, 0x3f, 0x01, 0x00, 0x00, 0xff, 0xff, 0x7b, 0x26, 0x25, 0xb7, 0x6b, 0x01, 0x00, 0x00, +} diff --git a/repo/objectid.proto b/repo/objectid.proto new file mode 100644 index 000000000..daa1c1738 --- /dev/null +++ b/repo/objectid.proto @@ -0,0 +1,21 @@ +syntax = "proto3"; + +package kopia.repo; +option go_package = "github.com/kopia/kopia/repo"; + +// ObjectID represents the identifier of a chunk. +// Identifiers can either refer to data block stored in a Storage, or contain small amounts +// of payload directly (useful for short ASCII or binary files). +message ObjectID { + string storage_block = 1; + int32 indirect = 2; // level of indirection (0==block data, 1=list of blocks, 2=list of lists, ...) + bytes encryption = 3; + bytes content = 4; + message Section { + ObjectID base = 1; + int64 start = 2; + int64 length = 3; + } + + Section section = 5; +} \ No newline at end of file diff --git a/repo/objectid_test.go b/repo/objectid_test.go index bb53ecfaa..b989b0c76 100644 --- a/repo/objectid_test.go +++ b/repo/objectid_test.go @@ -1,6 +1,7 @@ package repo import ( + "bytes" "strings" "testing" ) @@ -17,19 +18,21 @@ func TestParseMalformedObjectID(t *testing.T) { "Dab.", "Dab.a", "Dab.abc", - "Dab.00", "Dab.011", "L.", "L.x", "L.af", "Lx.ag", "Lab.", + "L1", + "L1,", + "L-1,X", "Xsomething", } for _, c := range cases { v, err := ParseObjectID(c) - if v != "" || err == nil || !strings.HasPrefix(err.Error(), "malformed chunk id") { + if !v.Equals(NullObjectID) || err == nil || !strings.HasPrefix(err.Error(), "malformed object id") { t.Errorf("unexpected result for %v: v: %v err: %v", c, v, err) } } @@ -38,24 +41,16 @@ func TestParseMalformedObjectID(t *testing.T) { func TestParseObjectIDEncryptionInfo(t *testing.T) { cases := []struct { objectID string - expected ObjectEncryptionInfo + expected []byte }{ - {"B", NoEncryption}, - {"BAQIDBA", NoEncryption}, - {"T", NoEncryption}, - {"T:foo", NoEncryption}, - {"Tfoo\nbar", NoEncryption}, - {"Tfoo\nbar:baz", NoEncryption}, - {"Dabcdef", NoEncryption}, - {"Labcdef", NoEncryption}, - { - "Dabcdef.00112233445566778899aabbccddeeff00112233445566778899aabbccddeeff", - ObjectEncryptionInfo("00112233445566778899aabbccddeeff00112233445566778899aabbccddeeff"), - }, - { - "Labcdef.00112233445566778899aabbccddeeff00112233445566778899aabbccddeeff", - ObjectEncryptionInfo("00112233445566778899aabbccddeeff00112233445566778899aabbccddeeff"), - }, + {"B", nil}, + {"BAQIDBA", nil}, + {"Dabcdef", nil}, + {"I1,abcdef", nil}, + {"I2,abcdef", nil}, + {"Dabcdef.00112233", []byte{0x00, 0x11, 0x22, 0x33}}, + {"I1,abcdef.0011223344", []byte{0x00, 0x11, 0x22, 0x33, 0x44}}, + {"I2,abcdef.0011223344", []byte{0x00, 0x11, 0x22, 0x33, 0x44}}, } for _, c := range cases { @@ -65,9 +60,9 @@ func TestParseObjectIDEncryptionInfo(t *testing.T) { continue } - actual := objectID.EncryptionInfo() - if actual != c.expected { - t.Errorf("invalid encryption info for %v: %v, expected: %v", c.objectID, actual, c.expected) + actual := objectID.Encryption + if !bytes.Equal(actual, c.expected) { + t.Errorf("invalid encryption info for %v: %x, expected: %x", c.objectID, actual, c.expected) } } } diff --git a/repo/repository.go b/repo/repository.go index efbba94ed..eb62737b9 100644 --- a/repo/repository.go +++ b/repo/repository.go @@ -10,10 +10,13 @@ import ( "fmt" "hash" "io" - "strconv" + "io/ioutil" + "log" "strings" "sync/atomic" + "github.com/kopia/kopia/internal" + "github.com/kopia/kopia/storage" "github.com/kopia/kopia/storage/logging" ) @@ -76,7 +79,7 @@ type repository struct { stats *RepositoryStats format Format - idFormat ObjectIDFormat + idFormat ObjectIDFormatInfo } func (repo *repository) Close() { @@ -107,7 +110,6 @@ func (repo *repository) Storage() storage.Storage { func (repo *repository) NewWriter(options ...WriterOption) ObjectWriter { result := &objectWriter{ repo: repo, - objectType: ObjectIDTypeStored, blockTracker: &blockTracker{}, } @@ -119,24 +121,30 @@ func (repo *repository) NewWriter(options ...WriterOption) ObjectWriter { } func (repo *repository) Open(objectID ObjectID) (ObjectReader, error) { - if start, length, baseID := objectID.SectionInfo(); baseID != "" { - baseReader, err := repo.Open(baseID) + // log.Printf("Repository::Open %v", objectID.String()) + // defer log.Printf("finished Repository::Open() %v", objectID.String()) + + if objectID.Section != nil { + if objectID.Section.Base == nil { + return nil, fmt.Errorf("invalid section base object") + } + + baseReader, err := repo.Open(*objectID.Section.Base) if err != nil { return nil, fmt.Errorf("cannot create base reader: %v", err) } - return newObjectSectionReader(start, length, baseReader) + return newObjectSectionReader(objectID.Section.Start, objectID.Section.Length, baseReader) } - r, err := repo.newRawReader(objectID) - if err != nil { - return nil, err - } + if objectID.Indirect > 0 { + r, err := repo.Open(removeIndirection(objectID)) + if err != nil { + return nil, err + } + defer r.Close() - if objectID.Type() == ObjectIDTypeList { - seekTable := make([]seekTableEntry, 0, 100) - - seekTable, err = repo.flattenListChunk(seekTable, objectID, r) + seekTable, err := repo.flattenListChunk(r) if err != nil { return nil, err } @@ -144,13 +152,32 @@ func (repo *repository) Open(objectID ObjectID) (ObjectReader, error) { totalLength := seekTable[len(seekTable)-1].endOffset() return &objectReader{ - storage: repo.storage, + repo: repo, seekTable: seekTable, totalLength: totalLength, }, nil } - return r, err + return repo.newRawReader(objectID) +} + +func dumpObject(repo *repository, oid ObjectID) { + oid.Indirect = 0 + log.Printf(" dumping %v", oid.String()) + defer log.Printf(" end of %v", oid.String()) + r, err := repo.Open(oid) + if err != nil { + log.Printf("failed to open %v: %v", oid, err) + return + } + defer r.Close() + + b, err := ioutil.ReadAll(r) + if err != nil { + log.Printf("failed to read %v: %v", oid, err) + return + } + log.Printf("%x", b) } // RepositoryOption controls the behavior of Repository. @@ -185,14 +212,11 @@ func NewRepository( f *Format, options ...RepositoryOption, ) (Repository, error) { - if f.Version != "1" { - return nil, fmt.Errorf("unsupported storage version: %v", f.Version) - } if f.MaxBlobSize < 100 { return nil, fmt.Errorf("MaxBlobSize is not set") } - sf := SupportedFormats.Find(f.ObjectFormat) + sf := SupportedFormats[f.ObjectFormat] if sf == nil { return nil, fmt.Errorf("unknown object format: %v", f.ObjectFormat) } @@ -203,7 +227,7 @@ func NewRepository( stats: &RepositoryStats{}, } - repo.idFormat = *sf + repo.idFormat = sf for _, o := range options { if err := o(repo); err != nil { @@ -212,13 +236,13 @@ func NewRepository( } } - repo.bufferManager = newBufferManager(repo.format.MaxBlobSize) + repo.bufferManager = newBufferManager(int(repo.format.MaxBlobSize)) return repo, nil } func (repo *repository) hashBuffer(data []byte) ([]byte, []byte) { - return repo.idFormat.hashBuffer(data, repo.format.Secret) + return repo.idFormat.HashBuffer(data, repo.format.Secret) } func (repo *repository) hashBufferForWriting(buffer *bytes.Buffer, prefix string) (ObjectID, storage.ReaderWithLength, error) { @@ -232,17 +256,18 @@ func (repo *repository) hashBufferForWriting(buffer *bytes.Buffer, prefix string contentHash, cryptoKey := repo.hashBuffer(data) if cryptoKey != nil { var err error - blockCipher, err = repo.idFormat.createCipher(cryptoKey) + blockCipher, err = repo.idFormat.CreateCipher(cryptoKey) if err != nil { - return "", nil, err + return NullObjectID, nil, err } } - var objectID ObjectID + objectID := ObjectID{ + StorageBlock: prefix + hex.EncodeToString(contentHash), + } + if len(cryptoKey) > 0 { - objectID = ObjectID(prefix + hex.EncodeToString(contentHash) + objectIDEncryptionInfoSeparator + hex.EncodeToString(cryptoKey)) - } else { - objectID = ObjectID(prefix + hex.EncodeToString(contentHash)) + objectID.Encryption = cryptoKey } atomic.AddInt32(&repo.stats.HashedBlocks, 1) @@ -268,71 +293,45 @@ func (repo *repository) hashBufferForWriting(buffer *bytes.Buffer, prefix string return objectID, blockReader, nil } -func (repo *repository) flattenListChunk( - seekTable []seekTableEntry, - listObjectID ObjectID, - rawReader io.Reader) ([]seekTableEntry, error) { +func (repo *repository) flattenListChunk(rawReader io.Reader) ([]IndirectObjectEntry, error) { - scanner := bufio.NewScanner(rawReader) + r := bufio.NewReader(rawReader) + pr := internal.NewProtoStreamReader(r, internal.ProtoStreamTypeIndirect) + var seekTable []IndirectObjectEntry - for scanner.Scan() { - c := scanner.Text() - comma := strings.Index(c, ",") - if comma <= 0 { - return nil, fmt.Errorf("unsupported entry '%v' in list '%s'", c, listObjectID) + for { + var oe IndirectObjectEntry + + err := pr.Read(&oe) + if err == io.EOF { + break } - length, err := strconv.ParseInt(c[0:comma], 10, 64) - - objectID, err := ParseObjectID(c[comma+1:]) if err != nil { - return nil, fmt.Errorf("unsupported entry '%v' in list '%s': %#v", c, listObjectID, err) + log.Printf("Failed to read proto: %v", err) + return nil, err } - switch objectID.Type() { - case ObjectIDTypeList: - subreader, err := repo.newRawReader(objectID) - if err != nil { - return nil, err - } - - seekTable, err = repo.flattenListChunk(seekTable, objectID, subreader) - if err != nil { - return nil, err - } - - case ObjectIDTypeStored: - var startOffset int64 - if len(seekTable) > 0 { - startOffset = seekTable[len(seekTable)-1].endOffset() - } else { - startOffset = 0 - } - - seekTable = append( - seekTable, - seekTableEntry{ - blockID: objectID.BlockID(), - startOffset: startOffset, - length: length, - }) - - default: - return nil, fmt.Errorf("unsupported entry '%v' in list '%v'", objectID, listObjectID) - - } + seekTable = append(seekTable, oe) } return seekTable, nil } +func removeIndirection(o ObjectID) ObjectID { + if o.Indirect <= 0 { + panic("removeIndirection() called on a direct object") + } + o.Indirect-- + return o +} + func (repo *repository) newRawReader(objectID ObjectID) (ObjectReader, error) { - inline := objectID.InlineData() - if inline != nil { - return newObjectReaderWithData(inline), nil + if objectID.Content != nil { + return newObjectReaderWithData(objectID.Content), nil } - blockID := objectID.BlockID() + blockID := objectID.StorageBlock payload, err := repo.storage.GetBlock(blockID) if err != nil { return nil, err @@ -341,19 +340,14 @@ func (repo *repository) newRawReader(objectID ObjectID) (ObjectReader, error) { atomic.AddInt32(&repo.stats.BlocksReadFromStorage, 1) atomic.AddInt64(&repo.stats.BytesReadFromStorage, int64(len(payload))) - if objectID.EncryptionInfo() == NoEncryption { - if err := repo.verifyChecksum(payload, objectID.BlockID()); err != nil { + if objectID.Encryption == nil { + if err := repo.verifyChecksum(payload, objectID.StorageBlock); err != nil { return nil, err } return newObjectReaderWithData(payload), nil } - cryptoKey, err := hex.DecodeString(string(objectID.EncryptionInfo())) - if err != nil { - return nil, errors.New("malformed encryption key") - } - - blockCipher, err := repo.idFormat.createCipher(cryptoKey) + blockCipher, err := repo.idFormat.CreateCipher(objectID.Encryption) if err != nil { return nil, errors.New("cannot create cipher") } @@ -366,7 +360,7 @@ func (repo *repository) newRawReader(objectID ObjectID) (ObjectReader, error) { // after decrypting the content. This serves as a checksum. atomic.AddInt64(&repo.stats.DecryptedBytes, int64(len(payload))) - if err := repo.verifyChecksum(payload, objectID.BlockID()); err != nil { + if err := repo.verifyChecksum(payload, objectID.StorageBlock); err != nil { return nil, err } diff --git a/repo/repository_test.go b/repo/repository_test.go index 802ab1a48..be3c5ad3b 100644 --- a/repo/repository_test.go +++ b/repo/repository_test.go @@ -1,28 +1,28 @@ package repo import ( + "bufio" "bytes" "crypto/md5" cryptorand "crypto/rand" "encoding/hex" "fmt" + "io" "io/ioutil" "math/rand" - "reflect" - "sort" - "strings" "testing" + "github.com/kopia/kopia/internal" "github.com/kopia/kopia/storage" "github.com/kopia/kopia/storage/storagetesting" ) func testFormat() *Format { return &Format{ - Version: "1", + Version: 1, MaxBlobSize: 200, MaxInlineBlobSize: 20, - ObjectFormat: "md5", + ObjectFormat: ObjectIDFormat_TESTONLY_MD5, } } @@ -56,16 +56,15 @@ func TestWriters(t *testing.T) { data []byte objectID ObjectID }{ - {[]byte{}, "B"}, - {[]byte("quick brown fox"), "Tquick brown fox"}, - {[]byte{1, 2, 3, 4}, "BAQIDBA"}, - {[]byte{10, 13, 9}, "T\n\r\t"}, - {[]byte{0xc2, 0x28}, "Bwig"}, // invalid UTF-8, will be represented as binary + {[]byte{}, NullObjectID}, + {[]byte("quick brown fox"), ObjectID{Content: []byte("quick brown fox")}}, + {[]byte{1, 2, 3, 4}, ObjectID{Content: []byte{1, 2, 3, 4}}}, + {[]byte{0xc2, 0x28}, ObjectID{Content: []byte{0xc2, 0x28}}}, { []byte("the quick brown fox jumps over the lazy dog"), - "DX77add1d5f41223d5582fca736a5cb335", + ObjectID{StorageBlock: "X77add1d5f41223d5582fca736a5cb335"}, }, - {make([]byte, 100), "DX6d0bb00954ceb7fbee436bb55a8397a9"}, // 100 zero bytes + {make([]byte, 100), ObjectID{StorageBlock: "X6d0bb00954ceb7fbee436bb55a8397a9"}}, // 100 zero bytes } for _, c := range cases { @@ -83,11 +82,11 @@ func TestWriters(t *testing.T) { continue } - if result != c.objectID { - t.Errorf("incorrect result for %v, expected: %v got: %v", c.data, c.objectID, result) + if !result.Equals(c.objectID) { + t.Errorf("incorrect result for %v, expected: %v got: %v", c.data, c.objectID.UIString(), result) } - if !c.objectID.Type().IsStored() { + if c.objectID.StorageBlock == "" { if len(data) != 0 { t.Errorf("unexpected data written to the storage: %v", data) } @@ -110,120 +109,74 @@ func TestWriterCompleteChunkInTwoWrites(t *testing.T) { writer.Write(bytes[0:50]) writer.Write(bytes[0:50]) result, err := writer.Result(false) - if string(result) != "DX6d0bb00954ceb7fbee436bb55a8397a9" { + if !result.Equals(ObjectID{StorageBlock: "X6d0bb00954ceb7fbee436bb55a8397a9"}) { t.Errorf("unexpected result: %v err: %v", result, err) } } -func TestWriterListChunk(t *testing.T) { - data, repo := setupTest(t) +func verifyIndirectBlock(t *testing.T, r Repository, oid ObjectID) { + for level := int32(0); level < oid.Indirect; level++ { + direct := oid + direct.Indirect = level - contentBytes := make([]byte, 250) - contentMd5Sum200 := getMd5Digest(contentBytes[0:200]) // hash of 200 zero bytes - contentMd5Sum50 := getMd5Digest(contentBytes[200:250]) // hash of 50 zero bytes - listChunkContent := []byte("200,D" + contentMd5Sum200 + "\n50,D" + contentMd5Sum50 + "\n") - listChunkObjectID := getMd5LObjectID(listChunkContent) + rd, err := r.Open(direct) + if err != nil { + t.Errorf("unable to open %v: %v", oid.String(), err) + return + } + defer rd.Close() - writer := repo.NewWriter() - writer.Write(contentBytes) - result, err := writer.Result(false) - if err != nil { - t.Errorf("error getting writer results: %v", err) + pr := internal.NewProtoStreamReader(bufio.NewReader(rd), internal.ProtoStreamTypeIndirect) + for { + var v IndirectObjectEntry + if err := pr.Read(&v); err != nil { + if err == io.EOF { + break + } + t.Errorf("err: %v", err) + break + } + } } - - if string(result) != listChunkObjectID { - t.Errorf("incorrect list chunk ID: %v, expected: %v", result, listChunkObjectID) - } - - // We have 3 chunks - 200 zero bytes, 50 zero bytes and the list. - verifyStorageBlocks(t, writer.StorageBlocks(), data, map[string][]byte{ - contentMd5Sum200: contentBytes[0:200], - contentMd5Sum50: contentBytes[0:50], - getMd5Digest(listChunkContent): listChunkContent, - }) } -func TestWriterListOfListsChunk(t *testing.T) { - data, repo := setupTest(t) - - contentBytes := make([]byte, 1400) - chunk1Id := getMd5CObjectID(contentBytes[0:200]) // hash of 200 zero bytes - - list1ChunkContent := []byte(strings.Repeat("200,"+chunk1Id+"\n", 5)) - list1ObjectID := fmt.Sprintf("%v,%v", len(list1ChunkContent), getMd5LObjectID(list1ChunkContent)) - - list2ChunkContent := []byte(strings.Repeat("200,"+chunk1Id+"\n", 2)) - list2ObjectID := fmt.Sprintf("%v,%v", len(list2ChunkContent), getMd5LObjectID(list2ChunkContent)) - - listOfListsChunkContent := []byte(list1ObjectID + "\n" + list2ObjectID + "\n") - listOfListsObjectID := getMd5LObjectID(listOfListsChunkContent) - - writer := repo.NewWriter() - writer.Write(contentBytes) - result, err := writer.Result(false) - if string(result) != listOfListsObjectID || err != nil { - t.Errorf("unexpected result: %v expected: %v, err: %v", result, listOfListsObjectID, err) +func TestIndirection(t *testing.T) { + cases := []struct { + dataLength int + expectedBlockCount int + expectedIndirection int32 + }{ + {dataLength: 200, expectedBlockCount: 1, expectedIndirection: 0}, + {dataLength: 250, expectedBlockCount: 3, expectedIndirection: 1}, + {dataLength: 1400, expectedBlockCount: 4, expectedIndirection: 2}, + {dataLength: 2000, expectedBlockCount: 5, expectedIndirection: 2}, + {dataLength: 3000, expectedBlockCount: 6, expectedIndirection: 2}, + {dataLength: 4000, expectedBlockCount: 9, expectedIndirection: 3}, + {dataLength: 10000, expectedBlockCount: 16, expectedIndirection: 3}, } - // We have 4 chunks - 200 zero bytes, 2 lists, and 1 list-of-lists. - verifyStorageBlocks(t, writer.StorageBlocks(), data, map[string][]byte{ - getMd5Digest(contentBytes[0:200]): contentBytes[0:200], - getMd5Digest(list1ChunkContent): list1ChunkContent, - getMd5Digest(list2ChunkContent): list2ChunkContent, - getMd5Digest(listOfListsChunkContent): listOfListsChunkContent, - }) -} + for _, c := range cases { + data, repo := setupTest(t) -func TestWriterListOfListsOfListsChunk(t *testing.T) { - data, repo := setupTest(t) + contentBytes := make([]byte, c.dataLength) - writtenData := make([]byte, 10000) - chunk1Id := getMd5CObjectID(writtenData[0:200]) // hash of 200 zero bytes + writer := repo.NewWriter() + writer.Write(contentBytes) + result, err := writer.Result(false) + if err != nil { + t.Errorf("error getting writer results: %v", err) + } - // First level list chunk has 5 C[] chunk IDs, because that's how many IDs fit in one chunk. - // (that number 200 was chosen for a reason, to make testing easy) - // - // 200,Cfbaf48ec981a5eecdb57b929fdd426e8\n - // 200,Cfbaf48ec981a5eecdb57b929fdd426e8\n - // 200,Cfbaf48ec981a5eecdb57b929fdd426e8\n - // 200,Cfbaf48ec981a5eecdb57b929fdd426e8\n - // 200,Cfbaf48ec981a5eecdb57b929fdd426e8\n - list1ChunkContent := []byte(strings.Repeat("200,"+chunk1Id+"\n", 5)) - list1ObjectID := fmt.Sprintf("%v,%v", len(list1ChunkContent), getMd5LObjectID(list1ChunkContent)) + if result.Indirect != c.expectedIndirection { + t.Errorf("incorrect indirection level: %v, expected %v", result.Indirect, c.expectedIndirection) + } - // Second level lists L[] chunks from the first level. They have all the same content - // because all lists are identical. - // 190,L52760f658059fef754f5deabdd01df93\n - // 190,L52760f658059fef754f5deabdd01df93\n - // 190,L52760f658059fef754f5deabdd01df93\n - // 190,L52760f658059fef754f5deabdd01df93\n - // 190,L52760f658059fef754f5deabdd01df93\n - list2ChunkContent := []byte(strings.Repeat(string(list1ObjectID)+"\n", 5)) - list2ObjectID := fmt.Sprintf("%v,%v", len(list2ChunkContent), getMd5LObjectID(list2ChunkContent)) + if len(data) != c.expectedBlockCount { + t.Errorf("unexpected block count: %v, expected %v", len(data), c.expectedBlockCount) + } - // Now those lists are also identical and represent 5000 bytes each, so - // the top-level list-of-lists-of-lists will have 2 entries: - // - // 190,Lb99b28e34c87e4934b4cc5631bb38ee8\n - // 190,Lb99b28e34c87e4934b4cc5631bb38ee8\n - list3ChunkContent := []byte(strings.Repeat(string(list2ObjectID)+"\n", 2)) - list3ObjectID := getMd5LObjectID(list3ChunkContent) - - writer := repo.NewWriter() - writer.Write(writtenData) - - result, err := writer.Result(false) - if string(result) != list3ObjectID { - t.Errorf("unexpected result: %v err: %v", result, err) + verifyIndirectBlock(t, repo, result) } - - // We have 4 data blocks representing 10000 bytes of zero. Not bad! - verifyStorageBlocks(t, writer.StorageBlocks(), data, map[string][]byte{ - getMd5Digest(writtenData[0:200]): writtenData[0:200], - getMd5Digest(list1ChunkContent): list1ChunkContent, - getMd5Digest(list2ChunkContent): list2ChunkContent, - getMd5Digest(list3ChunkContent): list3ChunkContent, - }) } func TestHMAC(t *testing.T) { @@ -231,7 +184,7 @@ func TestHMAC(t *testing.T) { content := bytes.Repeat([]byte{0xcd}, 50) s := testFormat() - s.ObjectFormat = "md5" + s.ObjectFormat = ObjectIDFormat_TESTONLY_MD5 s.Secret = []byte{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25} repo, err := NewRepository(storagetesting.NewMapStorage(data), s) @@ -241,7 +194,7 @@ func TestHMAC(t *testing.T) { w := repo.NewWriter() w.Write(content) result, err := w.Result(false) - if string(result) != "D697eaf0aca3a3aea3a75164746ffaa79" { + if result.UIString() != "D697eaf0aca3a3aea3a75164746ffaa79" { t.Errorf("unexpected result: %v err: %v", result, err) } } @@ -258,8 +211,6 @@ func TestReader(t *testing.T) { }{ {"B", []byte{}}, {"BAQIDBA", []byte{1, 2, 3, 4}}, - {"T", []byte{}}, - {"Tfoo\nbar", []byte("foo\nbar")}, {"Da76999788386641a3ec798554f1fe7e6", storedPayload}, {"S0,2,BAQIDBA", []byte{1, 2}}, {"S1,3,BAQIDBA", []byte{2, 3, 4}}, @@ -353,12 +304,12 @@ func TestEndToEndReadAndSeek(t *testing.T) { verify(t, repo, objectID, randomData, fmt.Sprintf("%v %v/%v", objectID, forceStored, size)) if size > 1 { - sectionID := NewSectionObjectID(0, int64(size/2), objectID) + sectionID := NewSectionObjectID(0, int64(size/2), &objectID) verify(t, repo, sectionID, randomData[0:10], fmt.Sprintf("%v %v/%v", sectionID, forceStored, size)) } if size > 1 { - sectionID := NewSectionObjectID(int64(1), int64(size-1), objectID) + sectionID := NewSectionObjectID(int64(1), int64(size-1), &objectID) verify(t, repo, sectionID, randomData[1:], fmt.Sprintf("%v %v/%v", sectionID, forceStored, size)) } } @@ -397,9 +348,9 @@ func verify(t *testing.T, repo Repository, objectID ObjectID, expectedData []byt } func TestFormats(t *testing.T) { - makeFormat := func(objectFormat string) *Format { + makeFormat := func(objectFormat ObjectIDFormat) *Format { return &Format{ - Version: "1", + Version: 1, ObjectFormat: objectFormat, Secret: []byte("key"), MaxBlobSize: 10000, @@ -411,88 +362,78 @@ func TestFormats(t *testing.T) { oids map[string]ObjectID }{ { - format: makeFormat("md5"), // MD5-HMAC with secret 'key' + format: makeFormat(ObjectIDFormat_TESTONLY_MD5), // MD5-HMAC with secret 'key' oids: map[string]ObjectID{ - "": "D63530468a04e386459855da0063b6596", - "The quick brown fox jumps over the lazy dog": "D80070713463e7749b90c2dc24911e275", + "": ObjectID{StorageBlock: "63530468a04e386459855da0063b6596"}, + "The quick brown fox jumps over the lazy dog": ObjectID{StorageBlock: "80070713463e7749b90c2dc24911e275"}, }, }, { format: &Format{ - Version: "1", - ObjectFormat: "md5", + Version: 1, + ObjectFormat: ObjectIDFormat_TESTONLY_MD5, MaxBlobSize: 10000, Secret: []byte{}, // HMAC with zero-byte secret }, oids: map[string]ObjectID{ - "": "D74e6f7298a9c2d168935f58c001bad88", - "The quick brown fox jumps over the lazy dog": "Dad262969c53bc16032f160081c4a07a0", + "": ObjectID{StorageBlock: "74e6f7298a9c2d168935f58c001bad88"}, + "The quick brown fox jumps over the lazy dog": ObjectID{StorageBlock: "ad262969c53bc16032f160081c4a07a0"}, }, }, { format: &Format{ - Version: "1", - ObjectFormat: "md5", + Version: 1, + ObjectFormat: ObjectIDFormat_TESTONLY_MD5, MaxBlobSize: 10000, Secret: nil, // non-HMAC version }, oids: map[string]ObjectID{ - "": "Dd41d8cd98f00b204e9800998ecf8427e", - "The quick brown fox jumps over the lazy dog": "D9e107d9d372bb6826bd81d3542a419d6", + "": ObjectID{StorageBlock: "d41d8cd98f00b204e9800998ecf8427e"}, + "The quick brown fox jumps over the lazy dog": ObjectID{ + StorageBlock: "9e107d9d372bb6826bd81d3542a419d6", + }, }, }, { - format: makeFormat("md5"), + format: makeFormat(ObjectIDFormat_TESTONLY_MD5), oids: map[string]ObjectID{ - "The quick brown fox jumps over the lazy dog": "D80070713463e7749b90c2dc24911e275", + "The quick brown fox jumps over the lazy dog": ObjectID{ + StorageBlock: "80070713463e7749b90c2dc24911e275", + }, }, }, { - format: makeFormat("sha1"), + format: makeFormat(ObjectIDFormat_UNENCYPTED_HMAC_SHA256), oids: map[string]ObjectID{ - "The quick brown fox jumps over the lazy dog": "Dde7c9b85b8b78aa6bc8a7a36f70a90701c9db4d9", + "The quick brown fox jumps over the lazy dog": ObjectID{ + StorageBlock: "f7bc83f430538424b13298e6aa6fb143ef4d59a14946175997479dbc2d1a3cd8", + }, }, }, { - format: makeFormat("sha256"), + format: makeFormat(ObjectIDFormat_UNENCYPTED_HMAC_SHA256_128), oids: map[string]ObjectID{ - "The quick brown fox jumps over the lazy dog": "Df7bc83f430538424b13298e6aa6fb143ef4d59a14946175997479dbc2d1a3cd8", + "The quick brown fox jumps over the lazy dog": ObjectID{ + StorageBlock: "18f1da557915937d2675055a87758d9b", + }, }, }, { - format: makeFormat("sha512"), + format: makeFormat(ObjectIDFormat_ENCRYPTED_HMAC_SHA512_384_AES256), oids: map[string]ObjectID{ - "The quick brown fox jumps over the lazy dog": "Db42af09057bac1e2d41708e48a902e09b5ff7f12ab428a4fe86653c73dd248fb82f948a549f7b791a5b41915ee4d1ec3935357e4e2317250d0372afa2ebeeb3a", + "The quick brown fox jumps over the lazy dog": ObjectID{ + StorageBlock: "d7f4727e2c0b39ae0f1e40cc96f60242", + Encryption: mustParseBase16("d5b7801841cea6fc592c5d3e1ae50700582a96cf35e1e554995fe4e03381c237"), + }, }, }, { - format: makeFormat("sha256-fold128-aes128"), + format: makeFormat(ObjectIDFormat_ENCRYPTED_HMAC_SHA512_AES256), oids: map[string]ObjectID{ - "The quick brown fox jumps over the lazy dog": "D028e74c630aee6400db4f84d49d1ed08.2825273c3344999e26081d99bcaf3f18", - }, - }, - { - format: makeFormat("sha256-aes128"), - oids: map[string]ObjectID{ - "The quick brown fox jumps over the lazy dog": "D26bb3fee9e83a7f6ff8397f33460cd0d24354b28ae2d41b6f2376fbe7db12005.2825273c3344999e26081d99bcaf3f18", - }, - }, - { - format: makeFormat("sha384-aes256"), - oids: map[string]ObjectID{ - "The quick brown fox jumps over the lazy dog": "D08edca658caee5ae86c9f1e4ccea26b1a43b0ddb04502c5f1b1cba051a756a9eba440bb6991b23323e37556ec5cf4f88.7a753377319650870f5cba47c30b608675542f206fa67d94dcbb9a0ccce7f467", - }, - }, - { - format: makeFormat("sha512-aes256"), - oids: map[string]ObjectID{ - "The quick brown fox jumps over the lazy dog": "Deee04ad71248ef4ffec5159558e80e2791e32299cfb79a1e063cc5f4a71cd61ca2c1b110e3ae2e83635a8626a3a27e4805eb745c40b5a4ebd4c9372602e5ab65.1c0b1b58ce05b7b8b05cfce27a485ddf97bde5159f6946357ec7795236f36a84", - }, - }, - { - format: makeFormat("sha512-fold128-aes192"), - oids: map[string]ObjectID{ - "The quick brown fox jumps over the lazy dog": "Dd829ad027ee4ff394f6a61615eb30d16.0ab00e0e0e156927f0be63372e8449d7bf2316c43d46e626", + "The quick brown fox jumps over the lazy dog": ObjectID{ + StorageBlock: "b42af09057bac1e2d41708e48a902e09b5ff7f12ab428a4fe86653c73dd248fb", + Encryption: mustParseBase16("82f948a549f7b791a5b41915ee4d1ec3935357e4e2317250d0372afa2ebeeb3a"), + }, }, }, } @@ -501,7 +442,7 @@ func TestFormats(t *testing.T) { data := map[string][]byte{} st := storagetesting.NewMapStorage(data) - t.Logf("verifying %#v", c.format) + t.Logf("verifying %v", c.format.String()) repo, err := NewRepository(st, c.format) if err != nil { t.Errorf("cannot create manager: %v", err) @@ -516,8 +457,8 @@ func TestFormats(t *testing.T) { if err != nil { t.Errorf("error: %v", err) } - if oid != v { - t.Errorf("invalid oid for #%v %v/%v: %v expected %v", caseIndex, c.format.ObjectFormat, k, oid, v) + if !oid.Equals(v) { + t.Errorf("invalid oid for #%v %v/%v:\ngot:\n%#v\nexpected:\n%#v", caseIndex, c.format.ObjectFormat, k, oid.UIString(), v.UIString()) } rc, err := repo.Open(oid) @@ -540,8 +481,8 @@ func TestInvalidEncryptionKey(t *testing.T) { data := map[string][]byte{} st := storagetesting.NewMapStorage(data) format := Format{ - Version: "1", - ObjectFormat: "sha512-aes256", + Version: 1, + ObjectFormat: ObjectIDFormat_ENCRYPTED_HMAC_SHA512_384_AES256, Secret: []byte("key"), MaxBlobSize: 1000, } @@ -571,47 +512,42 @@ func TestInvalidEncryptionKey(t *testing.T) { } // Key too short - rc, err = repo.Open(oid[0 : len(oid)-2]) + rc, err = repo.Open(replaceEncryption(oid, oid.Encryption[0:len(oid.Encryption)-2])) if err == nil || rc != nil { t.Errorf("expected error when opening malformed object") } // Key too long - rc, err = repo.Open(oid + "ff") + rc, err = repo.Open(replaceEncryption(oid, append(oid.Encryption, 0xFF))) if err == nil || rc != nil { t.Errorf("expected error when opening malformed object") } // Invalid key - lastByte, _ := hex.DecodeString(string(oid[len(oid)-2:])) - lastByte[0]++ - rc, err = repo.Open(oid[0:len(oid)-2] + ObjectID(hex.EncodeToString(lastByte))) + corruptedKey := append([]byte(nil), oid.Encryption...) + corruptedKey[0]++ + rc, err = repo.Open(replaceEncryption(oid, corruptedKey)) if err == nil || rc != nil { t.Errorf("expected error when opening malformed object: %v", err) } // Now corrupt the data - data[string(oid.BlockID())][0] ^= 1 + data[oid.StorageBlock][0] ^= 1 rc, err = repo.Open(oid) if err == nil || rc != nil { t.Errorf("expected error when opening object with corrupt data") } } -func verifyStorageBlocks(t *testing.T, actual []string, data, expectedData map[string][]byte) { - actualCopy := append([]string(nil), actual...) - sort.Strings(actualCopy) - var expected []string - for k := range expectedData { - expected = append(expected, k) - } - sort.Strings(expected) - - if !reflect.DeepEqual(expected, actualCopy) { - t.Errorf("updated blocks don't match. Expected: %#v, got %#v", expected, actualCopy) - } - - if !reflect.DeepEqual(expectedData, data) { - t.Errorf("storage data does not match, expected: %v, got %v", expectedData, data) - } +func replaceEncryption(oid ObjectID, newEncryption []byte) ObjectID { + oid.Encryption = newEncryption + return oid +} + +func mustParseBase16(s string) []byte { + b, err := hex.DecodeString(s) + if err != nil { + panic("invalid hex literal: " + s) + } + return b } diff --git a/vault/objectid_storage.go b/vault/objectid_storage.go index fe49b893a..e1ca9615b 100644 --- a/vault/objectid_storage.go +++ b/vault/objectid_storage.go @@ -6,6 +6,8 @@ import ( "encoding/hex" "fmt" + "github.com/golang/protobuf/proto" + "github.com/kopia/kopia/repo" ) @@ -16,14 +18,14 @@ const ( storedObjectIDLengthBytes = 8 ) -type objectIDData struct { - ObjectID string `json:"objectID"` -} - // SaveObjectID stores the given object ID in an encrypted vault item and returns a unique ID. func (vlt *Vault) SaveObjectID(oid repo.ObjectID) (string, error) { h := hmac.New(sha256.New, vlt.format.UniqueID) - h.Write([]byte(oid)) + bytes, err := proto.Marshal(&oid) + if err != nil { + return "", err + } + h.Write(bytes) sum := h.Sum(nil) for i := storedObjectIDLengthBytes; i < len(sum); i++ { sum[i%storedObjectIDLengthBytes] ^= sum[i] @@ -31,10 +33,7 @@ func (vlt *Vault) SaveObjectID(oid repo.ObjectID) (string, error) { sum = sum[0:storedObjectIDLengthBytes] key := StoredObjectIDPrefix + hex.EncodeToString(sum) - var d objectIDData - d.ObjectID = string(oid) - - if err := vlt.putJSON(key, &d); err != nil { + if err := vlt.Put(key, bytes); err != nil { return "", err } @@ -45,20 +44,26 @@ func (vlt *Vault) SaveObjectID(oid repo.ObjectID) (string, error) { func (vlt *Vault) GetObjectID(id string) (repo.ObjectID, error) { matches, err := vlt.List(id) if err != nil { - return "", err + return repo.NullObjectID, err } switch len(matches) { case 0: - return "", ErrItemNotFound + return repo.NullObjectID, ErrItemNotFound case 1: - var d objectIDData - if err := vlt.getJSON(matches[0], &d); err != nil { - return "", err + b, err := vlt.Get(matches[0]) + if err != nil { + return repo.NullObjectID, err } - return repo.ParseObjectID(d.ObjectID) + + var oid repo.ObjectID + if err := proto.Unmarshal(b, &oid); err != nil { + return repo.NullObjectID, err + } + + return oid, nil default: - return "", fmt.Errorf("ambiguous object ID: %v", id) + return repo.NullObjectID, fmt.Errorf("ambiguous object ID: %v", id) } } diff --git a/vault/vault.pb.go b/vault/vault.pb.go new file mode 100644 index 000000000..130480c1c --- /dev/null +++ b/vault/vault.pb.go @@ -0,0 +1,35 @@ +// Code generated by protoc-gen-go. +// source: vault/vault.proto +// DO NOT EDIT! + +/* +Package vault is a generated protocol buffer package. + +It is generated from these files: + vault/vault.proto + +It has these top-level messages: +*/ +package vault + +import proto "github.com/golang/protobuf/proto" +import fmt "fmt" +import math "math" + +// Reference imports to suppress errors if they are not otherwise used. +var _ = proto.Marshal +var _ = fmt.Errorf +var _ = math.Inf + +// This is a compile-time assertion to ensure that this generated file +// is compatible with the proto package it is being compiled against. +const _ = proto.ProtoPackageIsVersion1 + +var fileDescriptor0 = []byte{ + // 75 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0xe2, 0x12, 0x2c, 0x4b, 0x2c, 0xcd, + 0x29, 0xd1, 0x07, 0x93, 0x7a, 0x05, 0x45, 0xf9, 0x25, 0xf9, 0x42, 0xdc, 0xd9, 0xf9, 0x05, 0x99, + 0x89, 0x7a, 0x60, 0x21, 0x27, 0xb9, 0x28, 0x99, 0xf4, 0xcc, 0x92, 0x8c, 0xd2, 0x24, 0xbd, 0xe4, + 0xfc, 0x5c, 0x7d, 0xb0, 0x0c, 0x94, 0x04, 0xcb, 0x27, 0xb1, 0x81, 0xf5, 0x18, 0x03, 0x02, 0x00, + 0x00, 0xff, 0xff, 0x9a, 0x15, 0x10, 0x31, 0x48, 0x00, 0x00, 0x00, +} diff --git a/vault/vault.proto b/vault/vault.proto new file mode 100644 index 000000000..aacb4e565 --- /dev/null +++ b/vault/vault.proto @@ -0,0 +1,4 @@ +syntax = "proto3"; + +package kopia.vault; +option go_package = "github.com/kopia/kopia/vault"; diff --git a/vault/vault_test.go b/vault/vault_test.go index 238a3c3ac..aafc935ba 100644 --- a/vault/vault_test.go +++ b/vault/vault_test.go @@ -62,9 +62,9 @@ func verifyVault(t *testing.T, vaultPath string, repoPath string) { } repoFormat := &repo.Format{ - Version: "1", + Version: 1, MaxBlobSize: 1000000, - ObjectFormat: "sha256", + ObjectFormat: repo.ObjectIDFormat_UNENCYPTED_HMAC_SHA256, Secret: []byte{1, 2, 3}, } @@ -189,7 +189,7 @@ func verifyVault(t *testing.T, vaultPath string, repoPath string) { if err != nil { t.Errorf("error getting object ID: %v", err) } else { - if oid != oid1 { + if !oid.Equals(oid1) { t.Errorf("got invalid object ID: %v expected %v", oid, oid1) } } @@ -198,7 +198,7 @@ func verifyVault(t *testing.T, vaultPath string, repoPath string) { if err != nil { t.Errorf("error getting object ID: %v", err) } else { - if oid != oid2 { + if !oid.Equals(oid2) { t.Errorf("got invalid object ID: %v expected %v", oid, oid2) } }