cleaned up fs/ contents - extracted fs/localfs/ and fs/loggingfs/ moved repofs/ to fs/repofs/

backup/generator.go

@@ -5,8 +5,9 @@ import (
 	"time"
 
 	"github.com/kopia/kopia/fs"
+	"github.com/kopia/kopia/fs/localfs"
+	"github.com/kopia/kopia/fs/repofs"
 	"github.com/kopia/kopia/repo"
-	"github.com/kopia/kopia/repofs"
 )
 
 var (
@@ -33,7 +34,7 @@ func (bg *backupGenerator) Backup(m *Manifest, old *Manifest) error {
 		hashCacheID = &old.HashCacheID
 	}
 
-	entry, err := fs.NewFilesystemEntry(m.Source, nil)
+	entry, err := localfs.NewEntry(m.Source, nil)
 
 	var r *repofs.UploadResult
 	switch entry := entry.(type) {

cmd/kopia/command_backup.go

@@ -17,8 +17,8 @@ import (
 	"time"
 
 	"github.com/kopia/kopia/backup"
+	"github.com/kopia/kopia/fs/repofs"
 	"github.com/kopia/kopia/repo"
-	"github.com/kopia/kopia/repofs"
 
 	kingpin "gopkg.in/alecthomas/kingpin.v2"
 )

cmd/kopia/command_ls.go

@@ -6,8 +6,8 @@ import (
 	"strings"
 
 	"github.com/kopia/kopia/fs"
+	"github.com/kopia/kopia/fs/repofs"
 	"github.com/kopia/kopia/repo"
-	"github.com/kopia/kopia/repofs"
 
 	kingpin "gopkg.in/alecthomas/kingpin.v2"
 )

cmd/kopia/command_mount.go

@@ -5,8 +5,8 @@ package main
 import (
 	"bazil.org/fuse"
 	fusefs "bazil.org/fuse/fs"
+	"github.com/kopia/kopia/fs/repofs"
 	kopiafuse "github.com/kopia/kopia/fuse"
-	"github.com/kopia/kopia/repofs"
 
 	kingpin "gopkg.in/alecthomas/kingpin.v2"
 )

cmd/kopia/objref.go

@@ -5,8 +5,8 @@ import (
 	"strings"
 
 	"github.com/kopia/kopia/fs"
+	"github.com/kopia/kopia/fs/repofs"
 	"github.com/kopia/kopia/repo"
-	"github.com/kopia/kopia/repofs"
 	"github.com/kopia/kopia/vault"
 )
 

fs/doc.go

@@ -1,2 +1,2 @@
-// Package fs implements virtual filesystem abstraction layer.
+// Package fs defines virtual filesystem abstractions.
 package fs

fs/entry.go

@@ -1,13 +1,9 @@
 package fs
 
 import (
-	"encoding/json"
 	"io"
-	"os"
 	"sort"
-	"strconv"
 	"strings"
-	"time"
 )
 
 // Entry represents a filesystem entry, which can be Directory, File, or Symlink
@@ -16,77 +12,6 @@ type Entry interface {
 	Metadata() *EntryMetadata
 }
 
-// EntryType is a type of a filesystem entry.
-type EntryType string
-
-// Supported entry types.
-const (
-	EntryTypeUnknown   EntryType = ""  // unknown type
-	EntryTypeFile      EntryType = "f" // file
-	EntryTypeDirectory EntryType = "d" // directory
-	EntryTypeSymlink   EntryType = "s" // symbolic link
-)
-
-// Permissions encapsulates UNIX permissions for a filesystem entry.
-type Permissions int
-
-// MarshalJSON emits permissions as octal string.
-func (p Permissions) MarshalJSON() ([]byte, error) {
-	if p == 0 {
-		return nil, nil
-	}
-
-	s := "0" + strconv.FormatInt(int64(p), 8)
-	return json.Marshal(&s)
-}
-
-// UnmarshalJSON parses octal permissions string from JSON.
-func (p *Permissions) UnmarshalJSON(b []byte) error {
-	var s string
-
-	if err := json.Unmarshal(b, &s); err != nil {
-		return err
-	}
-
-	v, err := strconv.ParseInt(s, 0, 32)
-	if err != nil {
-		return err
-	}
-
-	*p = Permissions(v)
-	return nil
-}
-
-// EntryMetadata stores attributes of a single entry in a directory.
-type EntryMetadata struct {
-	Name        string      `json:"name,omitempty"`
-	Type        EntryType   `json:"type,omitempty"`
-	Permissions Permissions `json:"mode,omitempty"`
-	FileSize    int64       `json:"size,omitempty"`
-	ModTime     time.Time   `json:"mtime,omitempty"`
-	UserID      uint32      `json:"uid,omitempty"`
-	GroupID     uint32      `json:"gid,omitempty"`
-}
-
-// FileMode returns os.FileMode corresponding to Type and Permissions of the entry metadata.
-func (e *EntryMetadata) FileMode() os.FileMode {
-	perm := os.FileMode(e.Permissions)
-
-	switch e.Type {
-	default:
-		return perm
-
-	case EntryTypeFile:
-		return perm
-
-	case EntryTypeDirectory:
-		return perm | os.ModeDir
-
-	case EntryTypeSymlink:
-		return perm | os.ModeSymlink
-	}
-}
-
 // Entries is a list of entries sorted by name.
 type Entries []Entry
 
@@ -114,31 +39,6 @@ type Symlink interface {
 	Readlink() (string, error)
 }
 
-type entry struct {
-	parent   Directory
-	metadata *EntryMetadata
-}
-
-func (e *entry) Parent() Directory {
-	return e.parent
-}
-
-func (e *entry) Metadata() *EntryMetadata {
-	return e.metadata
-}
-
-func newEntry(md *EntryMetadata, parent Directory) entry {
-	return entry{parent, md}
-}
-
-type sortedEntries Entries
-
-func (e sortedEntries) Len() int      { return len(e) }
-func (e sortedEntries) Swap(i, j int) { e[i], e[j] = e[j], e[i] }
-func (e sortedEntries) Less(i, j int) bool {
-	return e[i].Metadata().Name < e[j].Metadata().Name
-}
-
 // FindByName returns an entry with a given name, or nil if not found.
 func (e Entries) FindByName(n string) Entry {
 	i := sort.Search(

fs/local_fs.go

@@ -1,160 +0,0 @@
-package fs
-
-import (
-	"fmt"
-	"io"
-	"os"
-	"path/filepath"
-	"sort"
-)
-
-const (
-	directoryReadAhead = 1024
-)
-
-type filesystemEntry struct {
-	entry
-	path string
-}
-
-type filesystemDirectory filesystemEntry
-type filesystemSymlink filesystemEntry
-type filesystemFile filesystemEntry
-
-func (fsd *filesystemDirectory) Readdir() (Entries, error) {
-	f, err := os.Open(fsd.path)
-	if err != nil {
-		return nil, err
-	}
-	defer f.Close()
-
-	var entries Entries
-
-	for {
-		fileInfos, err := f.Readdir(16)
-		for _, fi := range fileInfos {
-			e, err := entryFromFileInfo(fi, filepath.Join(fsd.path, fi.Name()), fsd)
-			if err != nil {
-				return nil, err
-			}
-			entries = append(entries, e)
-		}
-		if err == nil {
-			continue
-		}
-		if err == io.EOF {
-			break
-		}
-		return nil, err
-	}
-
-	sort.Sort(sortedEntries(entries))
-
-	return entries, nil
-}
-
-type fileWithMetadata struct {
-	*os.File
-}
-
-func (erc *fileWithMetadata) EntryMetadata() (*EntryMetadata, error) {
-	fi, err := erc.Stat()
-	if err != nil {
-		return nil, err
-	}
-
-	return entryMetadataFromFileInfo(fi), nil
-}
-
-func (fsf *filesystemFile) Open() (Reader, error) {
-	f, err := os.Open(fsf.path)
-	if err != nil {
-		return nil, err
-	}
-
-	return &fileWithMetadata{f}, nil
-}
-
-func (fsl *filesystemSymlink) Readlink() (string, error) {
-	return os.Readlink(fsl.path)
-}
-
-// NewFilesystemEntry returns fs.Entry for the specified path, the result will be one of supported entry types: File, Directory, Symlink.
-func NewFilesystemEntry(path string, parent Directory) (Entry, error) {
-	fi, err := os.Lstat(path)
-	if err != nil {
-		return nil, err
-	}
-
-	return entryFromFileInfo(fi, path, parent)
-}
-
-// NewFilesystemDirectory returns fs.Directory for the specified path.
-func NewFilesystemDirectory(path string, parent Directory) (Directory, error) {
-	e, err := NewFilesystemEntry(path, parent)
-	if err != nil {
-		return nil, err
-	}
-
-	switch e := e.(type) {
-	case Directory:
-		return e, nil
-
-	default:
-		return nil, fmt.Errorf("not a directory: %v", path)
-	}
-}
-
-func entryMetadataFromFileInfo(fi os.FileInfo) *EntryMetadata {
-	e := &EntryMetadata{
-		Name:        filepath.Base(fi.Name()),
-		Type:        entryTypeFromFileMode(fi.Mode() & os.ModeType),
-		Permissions: Permissions(fi.Mode() & os.ModePerm),
-		ModTime:     fi.ModTime().UTC(),
-	}
-
-	if fi.Mode().IsRegular() {
-		e.FileSize = fi.Size()
-	}
-
-	e.populatePlatformSpecificEntryDetails(fi)
-	return e
-}
-
-func entryTypeFromFileMode(t os.FileMode) EntryType {
-	switch t {
-	case 0:
-		return EntryTypeFile
-
-	case os.ModeSymlink:
-		return EntryTypeSymlink
-
-	case os.ModeDir:
-		return EntryTypeDirectory
-
-	default:
-		return EntryTypeDirectory
-	}
-}
-
-func entryFromFileInfo(fi os.FileInfo, path string, parent Directory) (Entry, error) {
-	entry := newEntry(entryMetadataFromFileInfo(fi), parent)
-
-	switch fi.Mode() & os.ModeType {
-	case os.ModeDir:
-		return &filesystemDirectory{entry, path}, nil
-
-	case os.ModeSymlink:
-		return &filesystemSymlink{entry, path}, nil
-
-	case 0:
-		return &filesystemFile{entry, path}, nil
-
-	default:
-		return nil, fmt.Errorf("unsupported filesystem entry: %v", path)
-	}
-}
-
-var _ Directory = &filesystemDirectory{}
-var _ File = &filesystemFile{}
-var _ Symlink = &filesystemSymlink{}

fs/local_fs_windows.go

@@ -1,7 +0,0 @@
-package fs
-
-import "os"
-
-func (e *EntryMetadata) populatePlatformSpecificEntryDetails(fi os.FileInfo) error {
-	return nil
-}

fs/localfs/doc.go

@@ -0,0 +1,2 @@
+// Package localfs implements virtual filesystem abstraction for a local filesystem.
+package localfs

fs/localfs/local_fs.go

@@ -0,0 +1,191 @@
+package localfs
+
+import (
+	"fmt"
+	"io"
+	"os"
+	"path/filepath"
+	"sort"
+
+	"github.com/kopia/kopia/fs"
+)
+
+const (
+	directoryReadAhead = 1024
+)
+
+type sortedEntries fs.Entries
+
+func (e sortedEntries) Len() int      { return len(e) }
+func (e sortedEntries) Swap(i, j int) { e[i], e[j] = e[j], e[i] }
+func (e sortedEntries) Less(i, j int) bool {
+	return e[i].Metadata().Name < e[j].Metadata().Name
+}
+
+type filesystemEntry struct {
+	parent   fs.Directory
+	metadata *fs.EntryMetadata
+	path     string
+}
+
+func newEntry(md *fs.EntryMetadata, parent fs.Directory, path string) filesystemEntry {
+	return filesystemEntry{parent, md, path}
+}
+
+func (e *filesystemEntry) Parent() fs.Directory {
+	return e.parent
+}
+
+func (e *filesystemEntry) Metadata() *fs.EntryMetadata {
+	return e.metadata
+}
+
+type filesystemDirectory struct {
+	filesystemEntry
+}
+
+type filesystemSymlink struct {
+	filesystemEntry
+}
+
+type filesystemFile struct {
+	filesystemEntry
+}
+
+func (fsd *filesystemDirectory) Readdir() (fs.Entries, error) {
+	f, err := os.Open(fsd.path)
+	if err != nil {
+		return nil, err
+	}
+	defer f.Close()
+
+	var entries fs.Entries
+
+	for {
+		fileInfos, err := f.Readdir(16)
+		for _, fi := range fileInfos {
+			e, err := entryFromFileInfo(fi, filepath.Join(fsd.path, fi.Name()), fsd)
+			if err != nil {
+				return nil, err
+			}
+			entries = append(entries, e)
+		}
+		if err == nil {
+			continue
+		}
+		if err == io.EOF {
+			break
+		}
+		return nil, err
+	}
+
+	sort.Sort(sortedEntries(entries))
+
+	return entries, nil
+}
+
+type fileWithMetadata struct {
+	*os.File
+}
+
+func (erc *fileWithMetadata) EntryMetadata() (*fs.EntryMetadata, error) {
+	fi, err := erc.Stat()
+	if err != nil {
+		return nil, err
+	}
+
+	return entryMetadataFromFileInfo(fi), nil
+}
+
+func (fsf *filesystemFile) Open() (fs.Reader, error) {
+	f, err := os.Open(fsf.path)
+	if err != nil {
+		return nil, err
+	}
+
+	return &fileWithMetadata{f}, nil
+}
+
+func (fsl *filesystemSymlink) Readlink() (string, error) {
+	return os.Readlink(fsl.path)
+}
+
+// NewEntry returns fs.Entry for the specified path, the result will be one of supported entry types: fs.File, fs.Directory, fs.Symlink.
+func NewEntry(path string, parent fs.Directory) (fs.Entry, error) {
+	fi, err := os.Lstat(path)
+	if err != nil {
+		return nil, err
+	}
+
+	return entryFromFileInfo(fi, path, parent)
+}
+
+// Directory returns fs.Directory for the specified path.
+func Directory(path string, parent fs.Directory) (fs.Directory, error) {
+	e, err := NewEntry(path, parent)
+	if err != nil {
+		return nil, err
+	}
+
+	switch e := e.(type) {
+	case fs.Directory:
+		return e, nil
+
+	default:
+		return nil, fmt.Errorf("not a directory: %v", path)
+	}
+}
+
+func entryMetadataFromFileInfo(fi os.FileInfo) *fs.EntryMetadata {
+	e := &fs.EntryMetadata{
+		Name:        filepath.Base(fi.Name()),
+		Type:        entryTypeFromFileMode(fi.Mode() & os.ModeType),
+		Permissions: fs.Permissions(fi.Mode() & os.ModePerm),
+		ModTime:     fi.ModTime().UTC(),
+	}
+
+	if fi.Mode().IsRegular() {
+		e.FileSize = fi.Size()
+	}
+
+	populatePlatformSpecificEntryDetails(e, fi)
+	return e
+}
+
+func entryTypeFromFileMode(t os.FileMode) fs.EntryType {
+	switch t {
+	case 0:
+		return fs.EntryTypeFile
+
+	case os.ModeSymlink:
+		return fs.EntryTypeSymlink
+
+	case os.ModeDir:
+		return fs.EntryTypeDirectory
+
+	default:
+		panic("unsupported file mode: " + t.String())
+	}
+}
+
+func entryFromFileInfo(fi os.FileInfo, path string, parent fs.Directory) (fs.Entry, error) {
+	entry := newEntry(entryMetadataFromFileInfo(fi), parent, path)
+
+	switch fi.Mode() & os.ModeType {
+	case os.ModeDir:
+		return &filesystemDirectory{entry}, nil
+
+	case os.ModeSymlink:
+		return &filesystemSymlink{entry}, nil
+
+	case 0:
+		return &filesystemFile{entry}, nil
+
+	default:
+		return nil, fmt.Errorf("unsupported filesystem entry: %v", path)
+	}
+}
+
+var _ fs.Directory = &filesystemDirectory{}
+var _ fs.File = &filesystemFile{}
+var _ fs.Symlink = &filesystemSymlink{}

fs/localfs/local_fs_nonwindows.go

@@ -1,13 +1,15 @@
 // +build !windows
 
-package fs
+package localfs
 
 import (
 	"os"
 	"syscall"
+
+	"github.com/kopia/kopia/fs"
 )
 
-func (e *EntryMetadata) populatePlatformSpecificEntryDetails(fi os.FileInfo) error {
+func populatePlatformSpecificEntryDetails(e *fs.EntryMetadata, fi os.FileInfo) error {
 	if stat, ok := fi.Sys().(*syscall.Stat_t); ok {
 		e.UserID = stat.Uid
 		e.GroupID = stat.Gid

fs/localfs/local_fs_test.go

@@ -1,4 +1,4 @@
-package fs
+package localfs
 
 import (
 	"fmt"
@@ -7,6 +7,8 @@ import (
 	"path/filepath"
 	"time"
 
+	"github.com/kopia/kopia/fs"
+
 	"testing"
 )
 
@@ -21,16 +23,16 @@ func TestFiles(t *testing.T) {
 
 	defer os.RemoveAll(tmp)
 
-	var dir Directory
+	var dir fs.Directory
 
 	// Try listing directory that does not exist.
-	dir, err = NewFilesystemDirectory(fmt.Sprintf("/no-such-dir-%v", time.Now().Nanosecond()), nil)
+	dir, err = Directory(fmt.Sprintf("/no-such-dir-%v", time.Now().Nanosecond()), nil)
 	if err == nil {
 		t.Errorf("expected error when dir directory that does not exist.")
 	}
 
 	// Now list an empty directory that does exist.
-	dir, err = NewFilesystemDirectory(tmp, nil)
+	dir, err = Directory(tmp, nil)
 	if err != nil {
 		t.Errorf("error when dir empty directory: %v", err)
 	}
@@ -52,7 +54,7 @@ func TestFiles(t *testing.T) {
 	os.Mkdir(filepath.Join(tmp, "z"), 0777)
 	os.Mkdir(filepath.Join(tmp, "y"), 0777)
 
-	dir, err = NewFilesystemDirectory(tmp, nil)
+	dir, err = Directory(tmp, nil)
 	if err != nil {
 		t.Errorf("error when dir directory with files: %v", err)
 	}

fs/localfs/local_fs_windows.go

@@ -0,0 +1,11 @@
+package localfs
+
+import (
+	"os"
+
+	"github.com/kopia/kopia/fs"
+)
+
+func populatePlatformSpecificEntryDetails(e *fs.EntryMetadata, fi os.FileInfo) error {
+	return nil
+}

fs/logging.go

@@ -1,46 +0,0 @@
-package fs
-
-import "log"
-
-type loggingDirectory struct {
-	Directory
-}
-
-func (ld *loggingDirectory) Readdir() (Entries, error) {
-	log.Printf("Entries(%v)", EntryPath(ld))
-	entries, err := ld.Directory.Readdir()
-	loggingEntries := make(Entries, len(entries))
-	for i, entry := range entries {
-		loggingEntries[i] = newLoggingWrapper(entry)
-	}
-	return loggingEntries, err
-}
-
-type loggingFile struct {
-	File
-}
-
-type loggingSymlink struct {
-	Symlink
-}
-
-// newLoggingWrapper returns an Entry that wraps another Entry and logs all method calls.
-func newLoggingWrapper(e Entry) Entry {
-	switch e := e.(type) {
-	case Directory:
-		return Directory(&loggingDirectory{e})
-
-	case File:
-		return File(&loggingFile{e})
-
-	case Symlink:
-		return Symlink(&loggingSymlink{e})
-
-	default:
-		return e
-	}
-}
-
-var _ Directory = &loggingDirectory{}
-var _ File = &loggingFile{}
-var _ Symlink = &loggingSymlink{}

fs/loggingfs/loggingfs.go

@@ -0,0 +1,51 @@
+// Package loggingfs implements a wrapper that logs all filesystem actions.
+package loggingfs
+
+import (
+	"log"
+
+	"github.com/kopia/kopia/fs"
+)
+
+type loggingDirectory struct {
+	fs.Directory
+}
+
+func (ld *loggingDirectory) Readdir() (fs.Entries, error) {
+	log.Printf("Readdir(%v)", fs.EntryPath(ld))
+	entries, err := ld.Directory.Readdir()
+	loggingEntries := make(fs.Entries, len(entries))
+	for i, entry := range entries {
+		loggingEntries[i] = Wrap(entry)
+	}
+	return loggingEntries, err
+}
+
+type loggingFile struct {
+	fs.File
+}
+
+type loggingSymlink struct {
+	fs.Symlink
+}
+
+// Wrap returns an Entry that wraps another Entry and logs all method calls.
+func Wrap(e fs.Entry) fs.Entry {
+	switch e := e.(type) {
+	case fs.Directory:
+		return fs.Directory(&loggingDirectory{e})
+
+	case fs.File:
+		return fs.File(&loggingFile{e})
+
+	case fs.Symlink:
+		return fs.Symlink(&loggingSymlink{e})
+
+	default:
+		return e
+	}
+}
+
+var _ fs.Directory = &loggingDirectory{}
+var _ fs.File = &loggingFile{}
+var _ fs.Symlink = &loggingSymlink{}

fs/metadata.go

@@ -0,0 +1,79 @@
+package fs
+
+import (
+	"encoding/json"
+	"os"
+	"strconv"
+	"time"
+)
+
+// EntryType is a type of a filesystem entry.
+type EntryType string
+
+// Supported entry types.
+const (
+	EntryTypeUnknown   EntryType = ""  // unknown type
+	EntryTypeFile      EntryType = "f" // file
+	EntryTypeDirectory EntryType = "d" // directory
+	EntryTypeSymlink   EntryType = "s" // symbolic link
+)
+
+// Permissions encapsulates UNIX permissions for a filesystem entry.
+type Permissions int
+
+// MarshalJSON emits permissions as octal string.
+func (p Permissions) MarshalJSON() ([]byte, error) {
+	if p == 0 {
+		return nil, nil
+	}
+
+	s := "0" + strconv.FormatInt(int64(p), 8)
+	return json.Marshal(&s)
+}
+
+// UnmarshalJSON parses octal permissions string from JSON.
+func (p *Permissions) UnmarshalJSON(b []byte) error {
+	var s string
+
+	if err := json.Unmarshal(b, &s); err != nil {
+		return err
+	}
+
+	v, err := strconv.ParseInt(s, 0, 32)
+	if err != nil {
+		return err
+	}
+
+	*p = Permissions(v)
+	return nil
+}
+
+// EntryMetadata stores attributes of a single entry in a directory.
+type EntryMetadata struct {
+	Name        string      `json:"name,omitempty"`
+	Type        EntryType   `json:"type,omitempty"`
+	Permissions Permissions `json:"mode,omitempty"`
+	FileSize    int64       `json:"size,omitempty"`
+	ModTime     time.Time   `json:"mtime,omitempty"`
+	UserID      uint32      `json:"uid,omitempty"`
+	GroupID     uint32      `json:"gid,omitempty"`
+}
+
+// FileMode returns os.FileMode corresponding to Type and Permissions of the entry metadata.
+func (e *EntryMetadata) FileMode() os.FileMode {
+	perm := os.FileMode(e.Permissions)
+
+	switch e.Type {
+	default:
+		return perm
+
+	case EntryTypeFile:
+		return perm
+
+	case EntryTypeDirectory:
+		return perm | os.ModeDir
+
+	case EntryTypeSymlink:
+		return perm | os.ModeSymlink
+	}
+}