From e8afaaa02f4d1dbd2611cac856dcfdf1c8de0c61 Mon Sep 17 00:00:00 2001 From: Jarek Kowalski Date: Tue, 5 Apr 2016 19:42:21 -0700 Subject: [PATCH] refactored Entry to use FileInfo --- fs/dir.go | 2 +- fs/dir_json.go | 119 ++++++++++++++++++++-------------------- fs/dir_test.go | 87 +++++++++-------------------- fs/entry.go | 109 +++++++++++++----------------------- fs/lister.go | 41 ++++++++------ fs/lister_nonwindows.go | 24 ++++---- fs/lister_test.go | 14 ++--- fs/upload.go | 31 ++++++----- 8 files changed, 186 insertions(+), 241 deletions(-) diff --git a/fs/dir.go b/fs/dir.go index c7b0c69ac..bfcac1cb4 100644 --- a/fs/dir.go +++ b/fs/dir.go @@ -3,7 +3,7 @@ // Directory represents contents of a directory. type EntryOrError struct { - Entry *Entry + Entry Entry Error error } diff --git a/fs/dir_json.go b/fs/dir_json.go index bcec11f73..613a3446d 100644 --- a/fs/dir_json.go +++ b/fs/dir_json.go @@ -4,10 +4,9 @@ "bufio" "bytes" "encoding/json" - "errors" "fmt" "io" - "strconv" + "os" "time" "github.com/kopia/kopia/cas" @@ -18,37 +17,68 @@ newLine = []byte("\n") ) -type writer struct { - lastEntryType EntryType - objectWriter io.Writer -} - type serializedDirectoryEntryV1 struct { - Name string `json:"name"` - Type string `json:"type"` - FileSize *int64 `json:"size,omitempty,string"` - Mode string `json:"mode"` - ModTime time.Time `json:"modified,omitempty"` - UserID uint32 `json:"uid,omitempty"` - GroupID uint32 `json:"gid,omitempty"` - ObjectID string `json:"objectID"` + XName string `json:"name"` + XFileMode uint32 `json:"mode"` + XFileSize int64 `json:"size,omitempty,string"` + XModTime time.Time `json:"modified,omitempty"` + XUserID uint32 `json:"uid,omitempty"` + XGroupID uint32 `json:"gid,omitempty"` + XObjectID string `json:"oid,omitempty"` } -func serializeManifestEntry(e *Entry) []byte { - s := serializedDirectoryEntryV1{ - Name: e.Name, - Type: string(e.Type), - Mode: strconv.FormatInt(int64(e.Mode), 8), - ObjectID: string(e.ObjectID), - UserID: e.UserID, - GroupID: e.GroupID, +func (de *serializedDirectoryEntryV1) Name() string { + return de.XName +} + +func (de *serializedDirectoryEntryV1) Mode() os.FileMode { + return os.FileMode(de.XFileMode) +} + +func (de *serializedDirectoryEntryV1) IsDir() bool { + return de.Mode().IsDir() +} + +func (de *serializedDirectoryEntryV1) Size() int64 { + if de.Mode().IsRegular() { + return de.XFileSize } - s.ModTime = e.ModTime.UTC() + return 0 +} - if e.Type == EntryTypeFile { - fs := e.Size - s.FileSize = &fs +func (de *serializedDirectoryEntryV1) UserID() uint32 { + return de.XUserID +} + +func (de *serializedDirectoryEntryV1) GroupID() uint32 { + return de.XGroupID +} + +func (de *serializedDirectoryEntryV1) ModTime() time.Time { + return de.XModTime +} + +func (de *serializedDirectoryEntryV1) ObjectID() cas.ObjectID { + return cas.ObjectID(de.XObjectID) +} + +func (de *serializedDirectoryEntryV1) Sys() interface{} { + return nil +} + +func serializeManifestEntry(e Entry) []byte { + s := serializedDirectoryEntryV1{ + XName: e.Name(), + XFileMode: uint32(e.Mode()), + XObjectID: string(e.ObjectID()), + XUserID: e.UserID(), + XGroupID: e.GroupID(), + XModTime: e.ModTime().UTC(), + } + + if e.Mode().IsRegular() { + s.XFileSize = e.Size() } jsonBytes, _ := json.Marshal(s) @@ -66,11 +96,7 @@ func writeDirectoryHeader(w io.Writer) error { return nil } -func writeDirectoryEntry(w io.Writer, e *Entry) error { - if e.Type == "" { - return errors.New("missing entry type") - } - +func writeDirectoryEntry(w io.Writer, e Entry) error { s := serializeManifestEntry(e) if _, err := w.Write(s); err != nil { return err @@ -94,42 +120,15 @@ func ReadDirectory(r io.Reader) (Directory, error) { ch := make(Directory) go func() { - var err error - for s.Scan() { line := s.Bytes() var v serializedDirectoryEntryV1 if err := json.Unmarshal(line, &v); err != nil { ch <- EntryOrError{Error: err} - } - - e := &Entry{} - e.Name = v.Name - e.UserID = v.UserID - e.GroupID = v.GroupID - e.ObjectID, err = cas.ParseObjectID(v.ObjectID) - if err != nil { - ch <- EntryOrError{Error: err} continue } - m, err := strconv.ParseInt(v.Mode, 8, 16) - if err != nil { - ch <- EntryOrError{Error: err} - continue - } - e.Mode = int16(m) - e.ModTime = v.ModTime - e.Type = EntryType(v.Type) - if e.Type == EntryTypeFile { - if v.FileSize == nil { - ch <- EntryOrError{Error: fmt.Errorf("missing file size")} - continue - } - e.Size = *v.FileSize - } - - ch <- EntryOrError{Entry: e} + ch <- EntryOrError{Entry: &v} } close(ch) }() diff --git a/fs/dir_test.go b/fs/dir_test.go index 3ad1cc019..9eddcb1b2 100644 --- a/fs/dir_test.go +++ b/fs/dir_test.go @@ -3,70 +3,37 @@ import ( "bytes" "strings" - "time" - "testing" ) -func TestJSON(t *testing.T) { - b := bytes.NewBuffer(nil) - writeDirectoryHeader(b) - writeDirectoryEntry(b, &Entry{ - EntryMetadata: EntryMetadata{ - Type: EntryTypeDirectory, - Name: "d1", - Mode: 0555, - ModTime: time.Unix(1458876568, 0), - }, - ObjectID: "foo", - }) +func TestJSONRoundTrip(t *testing.T) { + data := strings.Join( + []string{ + "DIRECTORY:v1", + `{"name":"subdir","mode":2147484141,"modified":"2016-04-06T02:34:10Z","uid":501,"gid":20,"oid":"C1234"}`, + `{"name":"config.go","mode":420,"size":"937","modified":"2016-04-02T02:39:44Z","uid":501,"gid":20,"oid":"C4321"}`, + `{"name":"constants.go","mode":420,"size":"13","modified":"2016-04-02T02:36:19Z","uid":501,"gid":20}`, + `{"name":"doc.go","mode":420,"size":"112","modified":"2016-04-02T02:45:54Z","uid":501,"gid":20}`, + `{"name":"errors.go","mode":420,"size":"506","modified":"2016-04-02T02:41:03Z","uid":501,"gid":20}`, + }, "\n") + "\n" - writeDirectoryEntry(b, &Entry{ - EntryMetadata: EntryMetadata{ - Type: EntryTypeDirectory, - Name: "d2", - Mode: 0754, - ModTime: time.Unix(1451871568, 0), - }, - ObjectID: "bar", - }) + d, err := ReadDirectory(strings.NewReader(data)) + if err != nil { + t.Errorf("can't read: %v", err) + return + } + b2 := bytes.NewBuffer(nil) + writeDirectoryHeader(b2) + for e := range d { + if e.Error != nil { + t.Errorf("parse error: %v", e.Error) + continue + } + t.Logf("writing %#v", e.Entry) + writeDirectoryEntry(b2, e.Entry) + } - writeDirectoryEntry(b, &Entry{ - EntryMetadata: EntryMetadata{ - Type: EntryTypeFile, - Name: "f1", - Mode: 0644, - ModTime: time.Unix(1451871368, 0), - Size: 123456, - }, - ObjectID: "baz", - }) - - writeDirectoryEntry(b, &Entry{ - EntryMetadata: EntryMetadata{ - Type: EntryTypeFile, - Name: "f2", - Mode: 0644, - ModTime: time.Unix(1451871331, 123456789), - Size: 12, - }, - ObjectID: "qoo", - }) - - assertLines( - t, - string(b.Bytes()), - "DIRECTORY:v1", - `{"name":"d1","type":"d","mode":"555","modified":"2016-03-25T03:29:28Z","objectID":"foo"}`, - `{"name":"d2","type":"d","mode":"754","modified":"2016-01-04T01:39:28Z","objectID":"bar"}`, - `{"name":"f1","type":"f","size":"123456","mode":"644","modified":"2016-01-04T01:36:08Z","objectID":"baz"}`, - `{"name":"f2","type":"f","size":"12","mode":"644","modified":"2016-01-04T01:35:31.123456789Z","objectID":"qoo"}`, - ) -} - -func assertLines(t *testing.T, text string, expectedLines ...string) { - expected := strings.Join(expectedLines, "\n") + "\n" - if text != expected { - t.Errorf("expected: '%v' got '%v'", expected, text) + if !bytes.Equal(b2.Bytes(), []byte(data)) { + t.Errorf("t: %v", string(b2.Bytes())) } } diff --git a/fs/entry.go b/fs/entry.go index 4b151d1a7..d67c93d84 100644 --- a/fs/entry.go +++ b/fs/entry.go @@ -1,88 +1,57 @@ package fs import ( - "fmt" "os" - "time" "github.com/kopia/kopia/cas" ) -// EntryType describes the type of an backup entry. -type EntryType string - -const ( - // EntryTypeFile represents a regular file. - EntryTypeFile EntryType = "f" - - // EntryTypeDirectory represents a directory entry which is a subdirectory. - EntryTypeDirectory EntryType = "d" - - // EntryTypeSymlink represents a symbolic link. - EntryTypeSymlink EntryType = "l" - - // EntryTypeSocket represents a UNIX socket. - EntryTypeSocket EntryType = "s" - - // EntryTypeDevice represents a device. - EntryTypeDevice EntryType = "v" - - // EntryTypeNamedPipe represents a named pipe. - EntryTypeNamedPipe EntryType = "n" -) - -// FileModeToType converts os.FileMode into EntryType. -func FileModeToType(mode os.FileMode) EntryType { - switch mode & os.ModeType { - case os.ModeDir: - return EntryTypeDirectory - - case os.ModeDevice: - return EntryTypeDevice - - case os.ModeSocket: - return EntryTypeSocket - - case os.ModeSymlink: - return EntryTypeSymlink - - case os.ModeNamedPipe: - return EntryTypeNamedPipe - - default: - return EntryTypeFile - } -} - -// EntryMetadata stores metadata about a directory entry but not related its content. -type EntryMetadata struct { - Name string - Size int64 - Type EntryType - ModTime time.Time - Mode int16 // 0000 .. 0777 - UserID uint32 - GroupID uint32 -} - // Entry stores attributes of a single entry in a directory. -type Entry struct { - EntryMetadata +type Entry interface { + os.FileInfo - ObjectID cas.ObjectID + UserID() uint32 + GroupID() uint32 + ObjectID() cas.ObjectID } -func (e *Entry) metadataEquals(other *Entry) bool { - if other == nil { +func metadataEquals(e1, e2 Entry) bool { + if (e1 != nil) != (e2 != nil) { return false } - return e.EntryMetadata == other.EntryMetadata + if e1.Mode() != e2.Mode() { + return false + } + + if e1.ModTime() != e2.ModTime() { + return false + } + + if e1.Size() != e2.Size() { + return false + } + + if e1.Name() != e2.Name() { + return false + } + + if e1.UserID() != e2.UserID() { + return false + } + + if e1.GroupID() != e2.GroupID() { + return false + } + + return true } -func (e *Entry) String() string { - return fmt.Sprintf( - "name: '%v' type: %v modTime: %v size: %v oid: '%v' uid: %v gid: %v", - e.Name, e.Type, e.ModTime, e.Size, e.ObjectID, e.UserID, e.GroupID, - ) +type entryWithObjectID struct { + Entry + oid cas.ObjectID +} + +func (e *entryWithObjectID) ObjectID() cas.ObjectID { + return e.oid } diff --git a/fs/lister.go b/fs/lister.go index 55e7138b5..363c9b81d 100644 --- a/fs/lister.go +++ b/fs/lister.go @@ -3,6 +3,8 @@ import ( "io" "os" + + "github.com/kopia/kopia/cas" ) const ( @@ -50,23 +52,26 @@ func (d *filesystemLister) List(path string) (Directory, error) { return ch, nil } -func entryFromFileSystemInfo(parentDir string, fi os.FileInfo) EntryOrError { - e := &Entry{ - EntryMetadata: EntryMetadata{ - Name: fi.Name(), - Mode: int16(fi.Mode().Perm()), - ModTime: fi.ModTime().UTC(), - Type: FileModeToType(fi.Mode()), - }, - } +type filesystemEntry struct { + os.FileInfo - if e.Type == EntryTypeFile { - e.Size = fi.Size() - } - - if err := populatePlatformSpecificEntryDetails(e, fi); err != nil { - return EntryOrError{Error: err} - } - - return EntryOrError{Entry: e} + objectID cas.ObjectID +} + +func (fse *filesystemEntry) Size() int64 { + if fse.Mode().IsRegular() { + return fse.FileInfo.Size() + } + + return 0 +} + +func (fse *filesystemEntry) ObjectID() cas.ObjectID { + return fse.objectID +} + +func entryFromFileSystemInfo(parentDir string, fi os.FileInfo) EntryOrError { + return EntryOrError{Entry: &filesystemEntry{ + FileInfo: fi, + }} } diff --git a/fs/lister_nonwindows.go b/fs/lister_nonwindows.go index 03767119a..0b8a70844 100644 --- a/fs/lister_nonwindows.go +++ b/fs/lister_nonwindows.go @@ -2,18 +2,20 @@ package fs -import ( - "fmt" - "os" - "syscall" -) +import "syscall" -func populatePlatformSpecificEntryDetails(e *Entry, fileInfo os.FileInfo) error { - if stat, ok := fileInfo.Sys().(*syscall.Stat_t); ok { - e.UserID = stat.Uid - e.GroupID = stat.Gid - return nil +func (e *filesystemEntry) UserID() uint32 { + if stat, ok := e.Sys().(*syscall.Stat_t); ok { + return stat.Uid } - return fmt.Errorf("unable to retrieve platform-specific file information") + return 0 +} + +func (e *filesystemEntry) GroupID() uint32 { + if stat, ok := e.Sys().(*syscall.Stat_t); ok { + return stat.Gid + } + + return 0 } diff --git a/fs/lister_test.go b/fs/lister_test.go index a3f3c1d8d..ec0067107 100644 --- a/fs/lister_test.go +++ b/fs/lister_test.go @@ -61,19 +61,19 @@ func TestLister(t *testing.T) { goodCount := 0 - if ae[0].Name == "f1" && ae[0].Size == 5 && ae[0].Type == EntryTypeFile { + if ae[0].Name() == "f1" && ae[0].Size() == 5 && ae[0].Mode().IsRegular() { goodCount++ } - if ae[1].Name == "f2" && ae[1].Size == 4 && ae[1].Type == EntryTypeFile { + if ae[1].Name() == "f2" && ae[1].Size() == 4 && ae[1].Mode().IsRegular() { goodCount++ } - if ae[2].Name == "f3" && ae[2].Size == 3 && ae[2].Type == EntryTypeFile { + if ae[2].Name() == "f3" && ae[2].Size() == 3 && ae[2].Mode().IsRegular() { goodCount++ } - if ae[3].Name == "y" && ae[3].Size == 0 && ae[3].Type == EntryTypeDirectory { + if ae[3].Name() == "y" && ae[3].Size() == 0 && ae[3].Mode().IsDir() { goodCount++ } - if ae[4].Name == "z" && ae[4].Size == 0 && ae[4].Type == EntryTypeDirectory { + if ae[4].Name() == "z" && ae[4].Size() == 0 && ae[4].Mode().IsDir() { goodCount++ } if goodCount != 5 { @@ -81,8 +81,8 @@ func TestLister(t *testing.T) { } } -func readAllEntries(dir Directory) []*Entry { - var entries []*Entry +func readAllEntries(dir Directory) []Entry { + var entries []Entry for d := range dir { if d.Error != nil { log.Fatalf("got error listing directory: %v", d.Error) diff --git a/fs/upload.go b/fs/upload.go index 3fa04bef0..fdad48dfe 100644 --- a/fs/upload.go +++ b/fs/upload.go @@ -60,10 +60,10 @@ func (u *uploader) UploadFile(path string) (cas.ObjectID, error) { type readaheadDirectory struct { src Directory - unreadEntriesByName map[string]*Entry + unreadEntriesByName map[string]Entry } -func (ra *readaheadDirectory) FindByName(name string) *Entry { +func (ra *readaheadDirectory) FindByName(name string) Entry { if e, ok := ra.unreadEntriesByName[name]; ok { delete(ra.unreadEntriesByName, name) return e @@ -76,10 +76,10 @@ func (ra *readaheadDirectory) FindByName(name string) *Entry { break } if next.Error == nil { - if next.Entry.Name == name { + if next.Entry.Name() == name { return next.Entry } - ra.unreadEntriesByName[next.Entry.Name] = next.Entry + ra.unreadEntriesByName[next.Entry.Name()] = next.Entry } } @@ -114,7 +114,7 @@ func (u *uploader) uploadDirInternal(path string, previous cas.ObjectID, previou ra := readaheadDirectory{ src: previousDir, - unreadEntriesByName: map[string]*Entry{}, + unreadEntriesByName: map[string]Entry{}, } writer := u.mgr.NewWriter( @@ -128,41 +128,44 @@ func (u *uploader) uploadDirInternal(path string, previous cas.ObjectID, previou directoryMatchesCache := true for de := range dir { e := de.Entry - fullPath := filepath.Join(path, e.Name) + fullPath := filepath.Join(path, e.Name()) // See if we had this name during previous pass. - cachedEntry := ra.FindByName(e.Name) + cachedEntry := ra.FindByName(e.Name()) // ... and whether file metadata is identical to the previous one. - cachedMetadataMatches := e.metadataEquals(cachedEntry) + cachedMetadataMatches := metadataEquals(e, cachedEntry) // If not, directoryMatchesCache becomes false. directoryMatchesCache = directoryMatchesCache && cachedMetadataMatches - if e.Type == EntryTypeDirectory { + var oid cas.ObjectID + + if e.IsDir() { var previousSubdirObjectID cas.ObjectID if cachedEntry != nil { - previousSubdirObjectID = cachedEntry.ObjectID + previousSubdirObjectID = cachedEntry.ObjectID() } - e.ObjectID, err = u.UploadDir(fullPath, previousSubdirObjectID) + oid, err = u.UploadDir(fullPath, previousSubdirObjectID) if err != nil { return cas.NullObjectID, err } - if cachedEntry != nil && e.ObjectID != cachedEntry.ObjectID { + if cachedEntry != nil && oid != cachedEntry.ObjectID() { directoryMatchesCache = false } } else if cachedMetadataMatches { // Avoid hashing by reusing previous object ID. - e.ObjectID = cachedEntry.ObjectID + oid = cachedEntry.ObjectID() } else { - e.ObjectID, err = u.UploadFile(fullPath) + oid, err = u.UploadFile(fullPath) if err != nil { return cas.NullObjectID, fmt.Errorf("unable to hash file: %s", err) } } + e = &entryWithObjectID{Entry: e, oid: oid} writeDirectoryEntry(writer, e) }