From e3a9ff43945e11f3097ebf2fc7457b2fa5487627 Mon Sep 17 00:00:00 2001 From: Jarek Kowalski Date: Tue, 5 Apr 2016 20:48:52 -0700 Subject: [PATCH] changed Directory to be slice again --- fs/dir.go | 19 +++++++++--------- fs/dir_json.go | 23 ++++++++++------------ fs/dir_test.go | 9 ++------- fs/lister.go | 40 ++++++++++++++++++-------------------- fs/lister_test.go | 32 ++++++++----------------------- fs/upload.go | 49 ++++------------------------------------------- 6 files changed, 53 insertions(+), 119 deletions(-) diff --git a/fs/dir.go b/fs/dir.go index bfcac1cb4..8ef1674ef 100644 --- a/fs/dir.go +++ b/fs/dir.go @@ -1,17 +1,18 @@ package fs +import "sort" + // Directory represents contents of a directory. -type EntryOrError struct { - Entry Entry - Error error -} - -type Directory chan EntryOrError +type Directory []Entry var emptyDirectory Directory -func init() { - emptyDirectory = make(Directory) - close(emptyDirectory) +func (d Directory) FindByName(n string) Entry { + i := sort.Search(len(d), func(i int) bool { return d[i].Name() >= n }) + if i < len(d) && d[i].Name() == n { + return d[i] + } + + return nil } diff --git a/fs/dir_json.go b/fs/dir_json.go index 613a3446d..ef6b18bef 100644 --- a/fs/dir_json.go +++ b/fs/dir_json.go @@ -118,20 +118,17 @@ func ReadDirectory(r io.Reader) (Directory, error) { return nil, fmt.Errorf("invalid directoryHeader: expected '%v' got '%v'", directoryHeader, s.Bytes()) } - ch := make(Directory) - go func() { - for s.Scan() { - line := s.Bytes() - var v serializedDirectoryEntryV1 - if err := json.Unmarshal(line, &v); err != nil { - ch <- EntryOrError{Error: err} - continue - } + var dir Directory - ch <- EntryOrError{Entry: &v} + for s.Scan() { + line := s.Bytes() + var v serializedDirectoryEntryV1 + if err := json.Unmarshal(line, &v); err != nil { + return nil, err } - close(ch) - }() - return ch, nil + dir = append(dir, &v) + } + + return dir, nil } diff --git a/fs/dir_test.go b/fs/dir_test.go index 9eddcb1b2..9da76c1a0 100644 --- a/fs/dir_test.go +++ b/fs/dir_test.go @@ -24,13 +24,8 @@ func TestJSONRoundTrip(t *testing.T) { } b2 := bytes.NewBuffer(nil) writeDirectoryHeader(b2) - for e := range d { - if e.Error != nil { - t.Errorf("parse error: %v", e.Error) - continue - } - t.Logf("writing %#v", e.Entry) - writeDirectoryEntry(b2, e.Entry) + for _, e := range d { + writeDirectoryEntry(b2, e) } if !bytes.Equal(b2.Bytes(), []byte(data)) { diff --git a/fs/lister.go b/fs/lister.go index 363c9b81d..834bd6007 100644 --- a/fs/lister.go +++ b/fs/lister.go @@ -29,27 +29,25 @@ func (d *filesystemLister) List(path string) (Directory, error) { if err != nil { return nil, err } + defer f.Close() - ch := make(Directory, 16) - go func() { - for { - fileInfos, err := f.Readdir(16) - for _, fi := range fileInfos { - ch <- entryFromFileSystemInfo(path, fi) - } - if err == nil { - continue - } - if err == io.EOF { - break - } - ch <- EntryOrError{Error: err} + var dir Directory + + for { + fileInfos, err := f.Readdir(16) + for _, fi := range fileInfos { + dir = append(dir, entryFromFileSystemInfo(fi)) } - f.Close() - close(ch) - }() + if err == nil { + continue + } + if err == io.EOF { + break + } + return nil, err + } - return ch, nil + return dir, nil } type filesystemEntry struct { @@ -70,8 +68,8 @@ func (fse *filesystemEntry) ObjectID() cas.ObjectID { return fse.objectID } -func entryFromFileSystemInfo(parentDir string, fi os.FileInfo) EntryOrError { - return EntryOrError{Entry: &filesystemEntry{ +func entryFromFileSystemInfo(fi os.FileInfo) Entry { + return &filesystemEntry{ FileInfo: fi, - }} + } } diff --git a/fs/lister_test.go b/fs/lister_test.go index ec0067107..c73aa3122 100644 --- a/fs/lister_test.go +++ b/fs/lister_test.go @@ -3,7 +3,6 @@ import ( "fmt" "io/ioutil" - "log" "os" "path/filepath" "time" @@ -38,10 +37,8 @@ func TestLister(t *testing.T) { t.Errorf("error when dir empty directory: %v", err) } - ae := readAllEntries(dir) - - if len(ae) > 0 { - t.Errorf("expected empty directory, got %v", ae) + if len(dir) > 0 { + t.Errorf("expected empty directory, got %v", dir) } // Now list a directory with 3 files. @@ -57,37 +54,24 @@ func TestLister(t *testing.T) { t.Errorf("error when dir directory with files: %v", err) } - ae = readAllEntries(dir) - goodCount := 0 - if ae[0].Name() == "f1" && ae[0].Size() == 5 && ae[0].Mode().IsRegular() { + if dir[0].Name() == "f1" && dir[0].Size() == 5 && dir[0].Mode().IsRegular() { goodCount++ } - if ae[1].Name() == "f2" && ae[1].Size() == 4 && ae[1].Mode().IsRegular() { + if dir[1].Name() == "f2" && dir[1].Size() == 4 && dir[1].Mode().IsRegular() { goodCount++ } - if ae[2].Name() == "f3" && ae[2].Size() == 3 && ae[2].Mode().IsRegular() { + if dir[2].Name() == "f3" && dir[2].Size() == 3 && dir[2].Mode().IsRegular() { goodCount++ } - if ae[3].Name() == "y" && ae[3].Size() == 0 && ae[3].Mode().IsDir() { + if dir[3].Name() == "y" && dir[3].Size() == 0 && dir[3].Mode().IsDir() { goodCount++ } - if ae[4].Name() == "z" && ae[4].Size() == 0 && ae[4].Mode().IsDir() { + if dir[4].Name() == "z" && dir[4].Size() == 0 && dir[4].Mode().IsDir() { goodCount++ } if goodCount != 5 { - t.Errorf("invalid dir data:\n%v", ae) + t.Errorf("invalid dir data:\n%v", dir) } } - -func readAllEntries(dir Directory) []Entry { - var entries []Entry - for d := range dir { - if d.Error != nil { - log.Fatalf("got error listing directory: %v", d.Error) - } - entries = append(entries, d.Entry) - } - return entries -} diff --git a/fs/upload.go b/fs/upload.go index fdad48dfe..5eb006ff0 100644 --- a/fs/upload.go +++ b/fs/upload.go @@ -58,38 +58,6 @@ func (u *uploader) UploadFile(path string) (cas.ObjectID, error) { return result, nil } -type readaheadDirectory struct { - src Directory - unreadEntriesByName map[string]Entry -} - -func (ra *readaheadDirectory) FindByName(name string) Entry { - if e, ok := ra.unreadEntriesByName[name]; ok { - delete(ra.unreadEntriesByName, name) - return e - } - - for ra.src != nil && len(ra.unreadEntriesByName) < maxDirReadAheadCount { - next, ok := <-ra.src - if !ok { - ra.src = nil - break - } - if next.Error == nil { - if next.Entry.Name() == name { - return next.Entry - } - ra.unreadEntriesByName[next.Entry.Name()] = next.Entry - } - } - - return nil -} - -func (ra *readaheadDirectory) hasUnreadEntries() bool { - return len(ra.unreadEntriesByName) > 0 -} - func (u *uploader) UploadDir(path string, previous cas.ObjectID) (cas.ObjectID, error) { var cached = emptyDirectory @@ -112,11 +80,6 @@ func (u *uploader) uploadDirInternal(path string, previous cas.ObjectID, previou return cas.NullObjectID, err } - ra := readaheadDirectory{ - src: previousDir, - unreadEntriesByName: map[string]Entry{}, - } - writer := u.mgr.NewWriter( cas.WithDescription("DIR:"+path), cas.WithBlockNamePrefix("D"), @@ -125,13 +88,13 @@ func (u *uploader) uploadDirInternal(path string, previous cas.ObjectID, previou writeDirectoryHeader(writer) defer writer.Close() - directoryMatchesCache := true - for de := range dir { - e := de.Entry + directoryMatchesCache := len(previousDir) == len(dir) + + for _, e := range dir { fullPath := filepath.Join(path, e.Name()) // See if we had this name during previous pass. - cachedEntry := ra.FindByName(e.Name()) + cachedEntry := previousDir.FindByName(e.Name()) // ... and whether file metadata is identical to the previous one. cachedMetadataMatches := metadataEquals(e, cachedEntry) @@ -169,10 +132,6 @@ func (u *uploader) uploadDirInternal(path string, previous cas.ObjectID, previou writeDirectoryEntry(writer, e) } - if ra.hasUnreadEntries() { - directoryMatchesCache = false - } - if directoryMatchesCache && previous != "" { // Avoid hashing directory listingif every entry matched the previous (possibly ignoring ordering). return previous, nil