changed Directory to be slice again

fs/dir.go

@@ -1,17 +1,18 @@
 package fs
 
+import "sort"
+
 // Directory represents contents of a directory.
 
-type EntryOrError struct {
-	Entry Entry
-	Error error
-}
-
-type Directory chan EntryOrError
+type Directory []Entry
 
 var emptyDirectory Directory
 
-func init() {
-	emptyDirectory = make(Directory)
-	close(emptyDirectory)
+func (d Directory) FindByName(n string) Entry {
+	i := sort.Search(len(d), func(i int) bool { return d[i].Name() >= n })
+	if i < len(d) && d[i].Name() == n {
+		return d[i]
+	}
+
+	return nil
 }

fs/dir_json.go

@@ -118,20 +118,17 @@ func ReadDirectory(r io.Reader) (Directory, error) {
 		return nil, fmt.Errorf("invalid directoryHeader: expected '%v' got '%v'", directoryHeader, s.Bytes())
 	}
 
-	ch := make(Directory)
-	go func() {
-		for s.Scan() {
-			line := s.Bytes()
-			var v serializedDirectoryEntryV1
-			if err := json.Unmarshal(line, &v); err != nil {
-				ch <- EntryOrError{Error: err}
-				continue
-			}
+	var dir Directory
 
-			ch <- EntryOrError{Entry: &v}
+	for s.Scan() {
+		line := s.Bytes()
+		var v serializedDirectoryEntryV1
+		if err := json.Unmarshal(line, &v); err != nil {
+			return nil, err
 		}
-		close(ch)
-	}()
 
-	return ch, nil
+		dir = append(dir, &v)
+	}
+
+	return dir, nil
 }

fs/dir_test.go

@@ -24,13 +24,8 @@ func TestJSONRoundTrip(t *testing.T) {
 	}
 	b2 := bytes.NewBuffer(nil)
 	writeDirectoryHeader(b2)
-	for e := range d {
-		if e.Error != nil {
-			t.Errorf("parse error: %v", e.Error)
-			continue
-		}
-		t.Logf("writing %#v", e.Entry)
-		writeDirectoryEntry(b2, e.Entry)
+	for _, e := range d {
+		writeDirectoryEntry(b2, e)
 	}
 
 	if !bytes.Equal(b2.Bytes(), []byte(data)) {

fs/lister.go

@@ -29,27 +29,25 @@ func (d *filesystemLister) List(path string) (Directory, error) {
 	if err != nil {
 		return nil, err
 	}
+	defer f.Close()
 
-	ch := make(Directory, 16)
-	go func() {
-		for {
-			fileInfos, err := f.Readdir(16)
-			for _, fi := range fileInfos {
-				ch <- entryFromFileSystemInfo(path, fi)
-			}
-			if err == nil {
-				continue
-			}
-			if err == io.EOF {
-				break
-			}
-			ch <- EntryOrError{Error: err}
+	var dir Directory
+
+	for {
+		fileInfos, err := f.Readdir(16)
+		for _, fi := range fileInfos {
+			dir = append(dir, entryFromFileSystemInfo(fi))
 		}
-		f.Close()
-		close(ch)
-	}()
+		if err == nil {
+			continue
+		}
+		if err == io.EOF {
+			break
+		}
+		return nil, err
+	}
 
-	return ch, nil
+	return dir, nil
 }
 
 type filesystemEntry struct {
@@ -70,8 +68,8 @@ func (fse *filesystemEntry) ObjectID() cas.ObjectID {
 	return fse.objectID
 }
 
-func entryFromFileSystemInfo(parentDir string, fi os.FileInfo) EntryOrError {
-	return EntryOrError{Entry: &filesystemEntry{
+func entryFromFileSystemInfo(fi os.FileInfo) Entry {
+	return &filesystemEntry{
 		FileInfo: fi,
-	}}
+	}
 }

fs/lister_test.go

@@ -3,7 +3,6 @@
 import (
 	"fmt"
 	"io/ioutil"
-	"log"
 	"os"
 	"path/filepath"
 	"time"
@@ -38,10 +37,8 @@ func TestLister(t *testing.T) {
 		t.Errorf("error when dir empty directory: %v", err)
 	}
 
-	ae := readAllEntries(dir)
-
-	if len(ae) > 0 {
-		t.Errorf("expected empty directory, got %v", ae)
+	if len(dir) > 0 {
+		t.Errorf("expected empty directory, got %v", dir)
 	}
 
 	// Now list a directory with 3 files.
@@ -57,37 +54,24 @@ func TestLister(t *testing.T) {
 		t.Errorf("error when dir directory with files: %v", err)
 	}
 
-	ae = readAllEntries(dir)
-
 	goodCount := 0
 
-	if ae[0].Name() == "f1" && ae[0].Size() == 5 && ae[0].Mode().IsRegular() {
+	if dir[0].Name() == "f1" && dir[0].Size() == 5 && dir[0].Mode().IsRegular() {
 		goodCount++
 	}
-	if ae[1].Name() == "f2" && ae[1].Size() == 4 && ae[1].Mode().IsRegular() {
+	if dir[1].Name() == "f2" && dir[1].Size() == 4 && dir[1].Mode().IsRegular() {
 		goodCount++
 	}
-	if ae[2].Name() == "f3" && ae[2].Size() == 3 && ae[2].Mode().IsRegular() {
+	if dir[2].Name() == "f3" && dir[2].Size() == 3 && dir[2].Mode().IsRegular() {
 		goodCount++
 	}
-	if ae[3].Name() == "y" && ae[3].Size() == 0 && ae[3].Mode().IsDir() {
+	if dir[3].Name() == "y" && dir[3].Size() == 0 && dir[3].Mode().IsDir() {
 		goodCount++
 	}
-	if ae[4].Name() == "z" && ae[4].Size() == 0 && ae[4].Mode().IsDir() {
+	if dir[4].Name() == "z" && dir[4].Size() == 0 && dir[4].Mode().IsDir() {
 		goodCount++
 	}
 	if goodCount != 5 {
-		t.Errorf("invalid dir data:\n%v", ae)
+		t.Errorf("invalid dir data:\n%v", dir)
 	}
 }
-
-func readAllEntries(dir Directory) []Entry {
-	var entries []Entry
-	for d := range dir {
-		if d.Error != nil {
-			log.Fatalf("got error listing directory: %v", d.Error)
-		}
-		entries = append(entries, d.Entry)
-	}
-	return entries
-}

fs/upload.go

@@ -58,38 +58,6 @@ func (u *uploader) UploadFile(path string) (cas.ObjectID, error) {
 	return result, nil
 }
 
-type readaheadDirectory struct {
-	src                 Directory
-	unreadEntriesByName map[string]Entry
-}
-
-func (ra *readaheadDirectory) FindByName(name string) Entry {
-	if e, ok := ra.unreadEntriesByName[name]; ok {
-		delete(ra.unreadEntriesByName, name)
-		return e
-	}
-
-	for ra.src != nil && len(ra.unreadEntriesByName) < maxDirReadAheadCount {
-		next, ok := <-ra.src
-		if !ok {
-			ra.src = nil
-			break
-		}
-		if next.Error == nil {
-			if next.Entry.Name() == name {
-				return next.Entry
-			}
-			ra.unreadEntriesByName[next.Entry.Name()] = next.Entry
-		}
-	}
-
-	return nil
-}
-
-func (ra *readaheadDirectory) hasUnreadEntries() bool {
-	return len(ra.unreadEntriesByName) > 0
-}
-
 func (u *uploader) UploadDir(path string, previous cas.ObjectID) (cas.ObjectID, error) {
 	var cached = emptyDirectory
 
@@ -112,11 +80,6 @@ func (u *uploader) uploadDirInternal(path string, previous cas.ObjectID, previou
 		return cas.NullObjectID, err
 	}
 
-	ra := readaheadDirectory{
-		src:                 previousDir,
-		unreadEntriesByName: map[string]Entry{},
-	}
-
 	writer := u.mgr.NewWriter(
 		cas.WithDescription("DIR:"+path),
 		cas.WithBlockNamePrefix("D"),
@@ -125,13 +88,13 @@ func (u *uploader) uploadDirInternal(path string, previous cas.ObjectID, previou
 	writeDirectoryHeader(writer)
 	defer writer.Close()
 
-	directoryMatchesCache := true
-	for de := range dir {
-		e := de.Entry
+	directoryMatchesCache := len(previousDir) == len(dir)
+
+	for _, e := range dir {
 		fullPath := filepath.Join(path, e.Name())
 
 		// See if we had this name during previous pass.
-		cachedEntry := ra.FindByName(e.Name())
+		cachedEntry := previousDir.FindByName(e.Name())
 
 		// ... and whether file metadata is identical to the previous one.
 		cachedMetadataMatches := metadataEquals(e, cachedEntry)
@@ -169,10 +132,6 @@ func (u *uploader) uploadDirInternal(path string, previous cas.ObjectID, previou
 		writeDirectoryEntry(writer, e)
 	}
 
-	if ra.hasUnreadEntries() {
-		directoryMatchesCache = false
-	}
-
 	if directoryMatchesCache && previous != "" {
 		// Avoid hashing directory listingif every entry matched the previous (possibly ignoring ordering).
 		return previous, nil