From 28baa0bf198f35dc50047154937c4eab00a3d357 Mon Sep 17 00:00:00 2001 From: Jarek Kowalski Date: Sun, 21 Aug 2016 10:38:39 -0700 Subject: [PATCH] entry JSON format tweaks --- cmd/kopia/command_ls.go | 4 +-- fs/entry.go | 67 ++++++++++++++++++++++++++++++++++++-- fs/inmemory_fs_for_test.go | 17 +++++----- fs/local_fs.go | 23 +++++++++++-- fs/local_fs_test.go | 10 +++--- fs/repository_fs.go | 18 +++++----- fs/upload.go | 4 +-- fuse/fusefs.go | 16 ++++----- 8 files changed, 118 insertions(+), 41 deletions(-) diff --git a/cmd/kopia/command_ls.go b/cmd/kopia/command_ls.go index 8981cd2f8..34fe2b44d 100644 --- a/cmd/kopia/command_ls.go +++ b/cmd/kopia/command_ls.go @@ -75,7 +75,7 @@ func listDirectory(prefix string, entries fs.Entries, longFormat bool) { } info = fmt.Sprintf( "%v %9d %v %-"+maxNameLenString+"s %v", - m.Mode, + m.FileMode(), m.FileSize, m.ModTime.Local().Format("02 Jan 06 15:04:05"), m.Name, @@ -83,7 +83,7 @@ func listDirectory(prefix string, entries fs.Entries, longFormat bool) { ) } else { var suffix string - if m.Mode.IsDir() { + if m.FileMode().IsDir() { suffix = "/" } diff --git a/fs/entry.go b/fs/entry.go index 73ab0f22f..54bdfae05 100644 --- a/fs/entry.go +++ b/fs/entry.go @@ -2,10 +2,12 @@ import ( "encoding/binary" + "encoding/json" "hash/fnv" "io" "os" "sort" + "strconv" "strings" "time" @@ -18,10 +20,52 @@ type Entry interface { Metadata() *EntryMetadata } +// EntryType is a type of a filesystem entry. +type EntryType string + +// Supported entry types. +const ( + EntryTypeUnknown EntryType = "" // unknown type + EntryTypeFile EntryType = "f" // file + EntryTypeDirectory EntryType = "d" // directory + EntryTypeSymlink EntryType = "s" // symbolic link +) + +// Permissions encapsulates UNIX permissions for a filesystem entry. +type Permissions int + +// MarshalJSON emits permissions as octal string. +func (p Permissions) MarshalJSON() ([]byte, error) { + if p == 0 { + return nil, nil + } + + s := "0" + strconv.FormatInt(int64(p), 8) + return json.Marshal(&s) +} + +// UnmarshalJSON parses octal permissions string from JSON. +func (p *Permissions) UnmarshalJSON(b []byte) error { + var s string + + if err := json.Unmarshal(b, &s); err != nil { + return err + } + + v, err := strconv.ParseInt(s, 0, 32) + if err != nil { + return err + } + + *p = Permissions(v) + return nil +} + // EntryMetadata stores attributes of a single entry in a directory. type EntryMetadata struct { Name string `json:"name,omitempty"` - Mode os.FileMode `json:"mode,omitempty"` + Type EntryType `json:"type"` + Permissions Permissions `json:"mode,omitempty"` FileSize int64 `json:"size,omitempty"` ModTime time.Time `json:"mtime,omitempty"` ObjectID repo.ObjectID `json:"obj,omitempty"` @@ -30,6 +74,25 @@ type EntryMetadata struct { BundledChildren []*EntryMetadata `json:"bundled,omitempty"` } +// FileMode returns os.FileMode corresponding to Type and Permissions of the entry metadata. +func (e *EntryMetadata) FileMode() os.FileMode { + perm := os.FileMode(e.Permissions) + + switch e.Type { + default: + return perm + + case EntryTypeFile: + return perm + + case EntryTypeDirectory: + return perm | os.ModeDir + + case EntryTypeSymlink: + return perm | os.ModeSymlink + } +} + // Entries is a list of entries sorted by name. type Entries []Entry @@ -137,7 +200,7 @@ func isLessOrEqual(name1, name2 string) bool { func (e *EntryMetadata) metadataHash() uint64 { h := fnv.New64a() binary.Write(h, binary.LittleEndian, e.ModTime.UnixNano()) - binary.Write(h, binary.LittleEndian, e.Mode) + binary.Write(h, binary.LittleEndian, e.FileMode()) binary.Write(h, binary.LittleEndian, e.FileSize) binary.Write(h, binary.LittleEndian, e.UserID) binary.Write(h, binary.LittleEndian, e.GroupID) diff --git a/fs/inmemory_fs_for_test.go b/fs/inmemory_fs_for_test.go index e877057c0..1107264f3 100644 --- a/fs/inmemory_fs_for_test.go +++ b/fs/inmemory_fs_for_test.go @@ -4,7 +4,6 @@ "bytes" "fmt" "io" - "os" "sort" "strings" ) @@ -29,14 +28,15 @@ type inmemoryDirectory struct { readdirError error } -func (imd *inmemoryDirectory) addFile(name string, content []byte, mode os.FileMode) *inmemoryFile { +func (imd *inmemoryDirectory) addFile(name string, content []byte, permissions Permissions) *inmemoryFile { imd, name = imd.resolveSubdir(name) file := &inmemoryFile{ inmemoryEntry: inmemoryEntry{ parent: imd, metadata: &EntryMetadata{ - Name: name, - Mode: mode, + Name: name, + Type: EntryTypeFile, + Permissions: permissions, }, }, source: bytes.NewBuffer(content), @@ -47,15 +47,16 @@ func (imd *inmemoryDirectory) addFile(name string, content []byte, mode os.FileM return file } -func (imd *inmemoryDirectory) addDir(name string, mode os.FileMode) *inmemoryDirectory { +func (imd *inmemoryDirectory) addDir(name string, permissions Permissions) *inmemoryDirectory { imd, name = imd.resolveSubdir(name) subdir := &inmemoryDirectory{ inmemoryEntry: inmemoryEntry{ parent: imd, metadata: &EntryMetadata{ - Name: name, - Mode: mode | os.ModeDir, + Name: name, + Type: EntryTypeDirectory, + Permissions: permissions, }, }, } @@ -88,7 +89,7 @@ func (imd *inmemoryDirectory) subdir(name ...string) *inmemoryDirectory { if i2 == nil { panic(fmt.Sprintf("'%s' not found in '%s'", n, i.metadata.Name)) } - if !i2.Metadata().Mode.IsDir() { + if !i2.Metadata().FileMode().IsDir() { panic(fmt.Sprintf("'%s' is not a directory in '%s'", n, i.metadata.Name)) } diff --git a/fs/local_fs.go b/fs/local_fs.go index 287c108de..bdcc89850 100644 --- a/fs/local_fs.go +++ b/fs/local_fs.go @@ -107,9 +107,10 @@ func NewFilesystemDirectory(path string, parent Directory) (Directory, error) { func entryMetadataFromFileInfo(fi os.FileInfo) *EntryMetadata { e := &EntryMetadata{ - Name: filepath.Base(fi.Name()), - Mode: fi.Mode(), - ModTime: fi.ModTime().UTC(), + Name: filepath.Base(fi.Name()), + Type: entryTypeFromFileMode(fi.Mode() & os.ModeType), + Permissions: Permissions(fi.Mode() & os.ModePerm), + ModTime: fi.ModTime().UTC(), } if fi.Mode().IsRegular() { @@ -120,6 +121,22 @@ func entryMetadataFromFileInfo(fi os.FileInfo) *EntryMetadata { return e } +func entryTypeFromFileMode(t os.FileMode) EntryType { + switch t { + case 0: + return EntryTypeFile + + case os.ModeSymlink: + return EntryTypeSymlink + + case os.ModeDir: + return EntryTypeDirectory + + default: + return EntryTypeDirectory + } +} + func entryFromFileInfo(fi os.FileInfo, path string, parent Directory) (Entry, error) { entry := newEntry(entryMetadataFromFileInfo(fi), parent) diff --git a/fs/local_fs_test.go b/fs/local_fs_test.go index 77866dc54..645bf7c1a 100644 --- a/fs/local_fs_test.go +++ b/fs/local_fs_test.go @@ -64,19 +64,19 @@ func TestFiles(t *testing.T) { goodCount := 0 - if entries[0].Metadata().Name == "f1" && entries[0].Metadata().FileSize == 5 && entries[0].Metadata().Mode.IsRegular() { + if entries[0].Metadata().Name == "f1" && entries[0].Metadata().FileSize == 5 && entries[0].Metadata().FileMode().IsRegular() { goodCount++ } - if entries[1].Metadata().Name == "f2" && entries[1].Metadata().FileSize == 4 && entries[1].Metadata().Mode.IsRegular() { + if entries[1].Metadata().Name == "f2" && entries[1].Metadata().FileSize == 4 && entries[1].Metadata().FileMode().IsRegular() { goodCount++ } - if entries[2].Metadata().Name == "f3" && entries[2].Metadata().FileSize == 3 && entries[2].Metadata().Mode.IsRegular() { + if entries[2].Metadata().Name == "f3" && entries[2].Metadata().FileSize == 3 && entries[2].Metadata().FileMode().IsRegular() { goodCount++ } - if entries[3].Metadata().Name == "y" && entries[3].Metadata().FileSize == 0 && entries[3].Metadata().Mode.IsDir() { + if entries[3].Metadata().Name == "y" && entries[3].Metadata().FileSize == 0 && entries[3].Metadata().FileMode().IsDir() { goodCount++ } - if entries[4].Metadata().Name == "z" && entries[4].Metadata().FileSize == 0 && entries[4].Metadata().Mode.IsDir() { + if entries[4].Metadata().Name == "z" && entries[4].Metadata().FileSize == 0 && entries[4].Metadata().FileMode().IsDir() { goodCount++ } if goodCount != 5 { diff --git a/fs/repository_fs.go b/fs/repository_fs.go index 7180b9d58..6c2987064 100644 --- a/fs/repository_fs.go +++ b/fs/repository_fs.go @@ -3,7 +3,6 @@ import ( "fmt" "io" - "os" "github.com/kopia/kopia/repo" ) @@ -51,27 +50,27 @@ func (rsl *repositorySymlink) Readlink() (string, error) { } func newRepoEntry(r repo.Repository, md *EntryMetadata, parent Directory) Entry { - switch md.Mode & os.ModeType { - case os.ModeDir: + switch md.Type { + case EntryTypeDirectory: return Directory(&repositoryDirectory{ entry: newEntry(md, parent), repo: r, }) - case os.ModeSymlink: + case EntryTypeSymlink: return Symlink(&repositorySymlink{ entry: newEntry(md, parent), repo: r, }) - case 0: + case EntryTypeFile: return File(&repositoryFile{ entry: newEntry(md, parent), repo: r, }) default: - panic(fmt.Sprintf("not supported entry metadata type: %v", md.Mode)) + panic(fmt.Sprintf("not supported entry metadata type: %v", md.Type)) } } @@ -94,9 +93,10 @@ func withMetadata(rc io.ReadCloser, md *EntryMetadata) EntryMetadataReadCloser { // NewRepositoryDirectory returns Directory based on repository object with the specified ID. func NewRepositoryDirectory(r repo.Repository, objectID repo.ObjectID) Directory { d := newRepoEntry(r, &EntryMetadata{ - Name: "/", - ObjectID: objectID, - Mode: 0555 | os.ModeDir, + Name: "/", + ObjectID: objectID, + Permissions: 0555, + Type: EntryTypeDirectory, }, nil) return d.(Directory) diff --git a/fs/upload.go b/fs/upload.go index a731c93c3..f34dfd257 100644 --- a/fs/upload.go +++ b/fs/upload.go @@ -276,7 +276,7 @@ func (u *uploader) uploadDirInternal( } default: - return repo.NullObjectID, 0, false, fmt.Errorf("file type %v not supported", entry.Metadata().Mode) + return repo.NullObjectID, 0, false, fmt.Errorf("file type %v not supported", entry.Metadata().Type) } if hash != 0 { @@ -289,7 +289,7 @@ func (u *uploader) uploadDirInternal( return repo.NullObjectID, 0, false, err } - if !e.Mode.IsDir() && e.ObjectID.StorageBlock != "" { + if e.Type != EntryTypeDirectory && e.ObjectID.StorageBlock != "" { if err := hcw.WriteEntry(hashCacheEntry{ Name: entryRelativePath, Hash: hash, diff --git a/fuse/fusefs.go b/fuse/fusefs.go index 29c82dd88..869cc7959 100644 --- a/fuse/fusefs.go +++ b/fuse/fusefs.go @@ -24,7 +24,7 @@ type fuseNode struct { func (n *fuseNode) Attr(ctx context.Context, a *fuse.Attr) error { m := n.entry.Metadata() - a.Mode = m.Mode + a.Mode = m.FileMode() a.Size = uint64(m.FileSize) a.Mtime = m.ModTime a.Uid = m.UserID @@ -91,17 +91,13 @@ func (dir *fuseDirectoryNode) ReadDirAll(ctx context.Context) ([]fuse.Dirent, er Name: m.Name, } - switch m.Mode & os.ModeType { - case os.ModeDir: + switch m.Type { + case fs.EntryTypeDirectory: dirent.Type = fuse.DT_Dir - case 0: + case fs.EntryTypeFile: dirent.Type = fuse.DT_File - case os.ModeSocket: - dirent.Type = fuse.DT_Socket - case os.ModeSymlink: + case fs.EntryTypeSymlink: dirent.Type = fuse.DT_Link - case os.ModeNamedPipe: - dirent.Type = fuse.DT_FIFO } result = append(result, dirent) @@ -127,7 +123,7 @@ func newFuseNode(e fs.Entry) (fusefs.Node, error) { case fs.Symlink: return &fuseSymlinkNode{fuseNode{e}}, nil default: - return nil, fmt.Errorf("entry type not supported: %v", e.Metadata().Mode) + return nil, fmt.Errorf("entry type not supported: %v", e.Metadata().Type) } }