mirror/kopia
1
0
Fork 0
mirror of https://github.com/kopia/kopia.git synced 2026-02-06 12:43:49 -05:00
Code Issues Packages Projects Releases Wiki Activity

refactored Entry to use FileInfo

Browse Source
This commit is contained in:
Jarek Kowalski
2016-04-05 19:42:21 -07:00
parent 32c3065a59
commit e8afaaa02f
8 changed files with 186 additions and 241 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
fs/dir.go
View File

@@ -3,7 +3,7 @@
// Directory represents contents of a directory.
type EntryOrError struct {
Entry *Entry
Entry Entry
Error error
}

119
fs/dir_json.go
View File

@@ -4,10 +4,9 @@
"bufio"
"bytes"
"encoding/json"
"errors"
"fmt"
"io"
"strconv"
"os"
"time"
"github.com/kopia/kopia/cas"
@@ -18,37 +17,68 @@
newLine = []byte("\n")
)
type writer struct {
lastEntryType EntryType
objectWriter io.Writer
}
type serializedDirectoryEntryV1 struct {
Name string `json:"name"`
Type string `json:"type"`
FileSize *int64 `json:"size,omitempty,string"`
Mode string `json:"mode"`
ModTime time.Time `json:"modified,omitempty"`
UserID uint32 `json:"uid,omitempty"`
GroupID uint32 `json:"gid,omitempty"`
ObjectID string `json:"objectID"`
XName string `json:"name"`
XFileMode uint32 `json:"mode"`
XFileSize int64 `json:"size,omitempty,string"`
XModTime time.Time `json:"modified,omitempty"`
XUserID uint32 `json:"uid,omitempty"`
XGroupID uint32 `json:"gid,omitempty"`
XObjectID string `json:"oid,omitempty"`
}
func serializeManifestEntry(e *Entry) []byte {
s := serializedDirectoryEntryV1{
Name: e.Name,
Type: string(e.Type),
Mode: strconv.FormatInt(int64(e.Mode), 8),
ObjectID: string(e.ObjectID),
UserID: e.UserID,
GroupID: e.GroupID,
func (de *serializedDirectoryEntryV1) Name() string {
return de.XName
}
func (de *serializedDirectoryEntryV1) Mode() os.FileMode {
return os.FileMode(de.XFileMode)
}
func (de *serializedDirectoryEntryV1) IsDir() bool {
return de.Mode().IsDir()
}
func (de *serializedDirectoryEntryV1) Size() int64 {
if de.Mode().IsRegular() {
return de.XFileSize
}
s.ModTime = e.ModTime.UTC()
return 0
}
if e.Type == EntryTypeFile {
fs := e.Size
s.FileSize = &fs
func (de *serializedDirectoryEntryV1) UserID() uint32 {
return de.XUserID
}
func (de *serializedDirectoryEntryV1) GroupID() uint32 {
return de.XGroupID
}
func (de *serializedDirectoryEntryV1) ModTime() time.Time {
return de.XModTime
}
func (de *serializedDirectoryEntryV1) ObjectID() cas.ObjectID {
return cas.ObjectID(de.XObjectID)
}
func (de *serializedDirectoryEntryV1) Sys() interface{} {
return nil
}
func serializeManifestEntry(e Entry) []byte {
s := serializedDirectoryEntryV1{
XName: e.Name(),
XFileMode: uint32(e.Mode()),
XObjectID: string(e.ObjectID()),
XUserID: e.UserID(),
XGroupID: e.GroupID(),
XModTime: e.ModTime().UTC(),
}
if e.Mode().IsRegular() {
s.XFileSize = e.Size()
}
jsonBytes, _ := json.Marshal(s)
@@ -66,11 +96,7 @@ func writeDirectoryHeader(w io.Writer) error {
return nil
}
func writeDirectoryEntry(w io.Writer, e *Entry) error {
if e.Type == "" {
return errors.New("missing entry type")
}
func writeDirectoryEntry(w io.Writer, e Entry) error {
s := serializeManifestEntry(e)
if _, err := w.Write(s); err != nil {
return err
@@ -94,42 +120,15 @@ func ReadDirectory(r io.Reader) (Directory, error) {
ch := make(Directory)
go func() {
var err error
for s.Scan() {
line := s.Bytes()
var v serializedDirectoryEntryV1
if err := json.Unmarshal(line, &v); err != nil {
ch <- EntryOrError{Error: err}
}
e := &Entry{}
e.Name = v.Name
e.UserID = v.UserID
e.GroupID = v.GroupID
e.ObjectID, err = cas.ParseObjectID(v.ObjectID)
if err != nil {
ch <- EntryOrError{Error: err}
continue
}
m, err := strconv.ParseInt(v.Mode, 8, 16)
if err != nil {
ch <- EntryOrError{Error: err}
continue
}
e.Mode = int16(m)
e.ModTime = v.ModTime
e.Type = EntryType(v.Type)
if e.Type == EntryTypeFile {
if v.FileSize == nil {
ch <- EntryOrError{Error: fmt.Errorf("missing file size")}
continue
}
e.Size = *v.FileSize
}
ch <- EntryOrError{Entry: e}
ch <- EntryOrError{Entry: &v}
}
close(ch)
}()

87
fs/dir_test.go
View File

@@ -3,70 +3,37 @@
import (
"bytes"
"strings"
"time"
"testing"
)
func TestJSON(t *testing.T) {
b := bytes.NewBuffer(nil)
writeDirectoryHeader(b)
writeDirectoryEntry(b, &Entry{
EntryMetadata: EntryMetadata{
Type: EntryTypeDirectory,
Name: "d1",
Mode: 0555,
ModTime: time.Unix(1458876568, 0),
},
ObjectID: "foo",
})
func TestJSONRoundTrip(t *testing.T) {
data := strings.Join(
[]string{
"DIRECTORY:v1",
`{"name":"subdir","mode":2147484141,"modified":"2016-04-06T02:34:10Z","uid":501,"gid":20,"oid":"C1234"}`,
`{"name":"config.go","mode":420,"size":"937","modified":"2016-04-02T02:39:44Z","uid":501,"gid":20,"oid":"C4321"}`,
`{"name":"constants.go","mode":420,"size":"13","modified":"2016-04-02T02:36:19Z","uid":501,"gid":20}`,
`{"name":"doc.go","mode":420,"size":"112","modified":"2016-04-02T02:45:54Z","uid":501,"gid":20}`,
`{"name":"errors.go","mode":420,"size":"506","modified":"2016-04-02T02:41:03Z","uid":501,"gid":20}`,
}, "\n") + "\n"
writeDirectoryEntry(b, &Entry{
EntryMetadata: EntryMetadata{
Type: EntryTypeDirectory,
Name: "d2",
Mode: 0754,
ModTime: time.Unix(1451871568, 0),
},
ObjectID: "bar",
})
d, err := ReadDirectory(strings.NewReader(data))
if err != nil {
t.Errorf("can't read: %v", err)
return
}
b2 := bytes.NewBuffer(nil)
writeDirectoryHeader(b2)
for e := range d {
if e.Error != nil {
t.Errorf("parse error: %v", e.Error)
continue
}
t.Logf("writing %#v", e.Entry)
writeDirectoryEntry(b2, e.Entry)
}
writeDirectoryEntry(b, &Entry{
EntryMetadata: EntryMetadata{
Type: EntryTypeFile,
Name: "f1",
Mode: 0644,
ModTime: time.Unix(1451871368, 0),
Size: 123456,
},
ObjectID: "baz",
})
writeDirectoryEntry(b, &Entry{
EntryMetadata: EntryMetadata{
Type: EntryTypeFile,
Name: "f2",
Mode: 0644,
ModTime: time.Unix(1451871331, 123456789),
Size: 12,
},
ObjectID: "qoo",
})
assertLines(
t,
string(b.Bytes()),
"DIRECTORY:v1",
`{"name":"d1","type":"d","mode":"555","modified":"2016-03-25T03:29:28Z","objectID":"foo"}`,
`{"name":"d2","type":"d","mode":"754","modified":"2016-01-04T01:39:28Z","objectID":"bar"}`,
`{"name":"f1","type":"f","size":"123456","mode":"644","modified":"2016-01-04T01:36:08Z","objectID":"baz"}`,
`{"name":"f2","type":"f","size":"12","mode":"644","modified":"2016-01-04T01:35:31.123456789Z","objectID":"qoo"}`,
)
}
func assertLines(t *testing.T, text string, expectedLines ...string) {
expected := strings.Join(expectedLines, "\n") + "\n"
if text != expected {
t.Errorf("expected: '%v' got '%v'", expected, text)
if !bytes.Equal(b2.Bytes(), []byte(data)) {
t.Errorf("t: %v", string(b2.Bytes()))
}
}

109
fs/entry.go
View File

@@ -1,88 +1,57 @@
package fs
import (
"fmt"
"os"
"time"
"github.com/kopia/kopia/cas"
)
// EntryType describes the type of an backup entry.
type EntryType string
const (
// EntryTypeFile represents a regular file.
EntryTypeFile EntryType = "f"
// EntryTypeDirectory represents a directory entry which is a subdirectory.
EntryTypeDirectory EntryType = "d"
// EntryTypeSymlink represents a symbolic link.
EntryTypeSymlink EntryType = "l"
// EntryTypeSocket represents a UNIX socket.
EntryTypeSocket EntryType = "s"
// EntryTypeDevice represents a device.
EntryTypeDevice EntryType = "v"
// EntryTypeNamedPipe represents a named pipe.
EntryTypeNamedPipe EntryType = "n"
)
// FileModeToType converts os.FileMode into EntryType.
func FileModeToType(mode os.FileMode) EntryType {
switch mode & os.ModeType {
case os.ModeDir:
return EntryTypeDirectory
case os.ModeDevice:
return EntryTypeDevice
case os.ModeSocket:
return EntryTypeSocket
case os.ModeSymlink:
return EntryTypeSymlink
case os.ModeNamedPipe:
return EntryTypeNamedPipe
default:
return EntryTypeFile
}
}
// EntryMetadata stores metadata about a directory entry but not related its content.
type EntryMetadata struct {
Name string
Size int64
Type EntryType
ModTime time.Time
Mode int16 // 0000 .. 0777
UserID uint32
GroupID uint32
}
// Entry stores attributes of a single entry in a directory.
type Entry struct {
EntryMetadata
type Entry interface {
os.FileInfo
ObjectID cas.ObjectID
UserID() uint32
GroupID() uint32
ObjectID() cas.ObjectID
}
func (e *Entry) metadataEquals(other *Entry) bool {
if other == nil {
func metadataEquals(e1, e2 Entry) bool {
if (e1 != nil) != (e2 != nil) {
return false
}
return e.EntryMetadata == other.EntryMetadata
if e1.Mode() != e2.Mode() {
return false
}
if e1.ModTime() != e2.ModTime() {
return false
}
if e1.Size() != e2.Size() {
return false
}
if e1.Name() != e2.Name() {
return false
}
if e1.UserID() != e2.UserID() {
return false
}
if e1.GroupID() != e2.GroupID() {
return false
}
return true
}
func (e *Entry) String() string {
return fmt.Sprintf(
"name: '%v' type: %v modTime: %v size: %v oid: '%v' uid: %v gid: %v",
e.Name, e.Type, e.ModTime, e.Size, e.ObjectID, e.UserID, e.GroupID,
)
type entryWithObjectID struct {
Entry
oid cas.ObjectID
}
func (e *entryWithObjectID) ObjectID() cas.ObjectID {
return e.oid
}

41
fs/lister.go
View File

@@ -3,6 +3,8 @@
import (
"io"
"os"
"github.com/kopia/kopia/cas"
)
const (
@@ -50,23 +52,26 @@ func (d *filesystemLister) List(path string) (Directory, error) {
return ch, nil
}
func entryFromFileSystemInfo(parentDir string, fi os.FileInfo) EntryOrError {
e := &Entry{
EntryMetadata: EntryMetadata{
Name: fi.Name(),
Mode: int16(fi.Mode().Perm()),
ModTime: fi.ModTime().UTC(),
Type: FileModeToType(fi.Mode()),
},
}
type filesystemEntry struct {
os.FileInfo
if e.Type == EntryTypeFile {
e.Size = fi.Size()
}
if err := populatePlatformSpecificEntryDetails(e, fi); err != nil {
return EntryOrError{Error: err}
}
return EntryOrError{Entry: e}
objectID cas.ObjectID
}
func (fse *filesystemEntry) Size() int64 {
if fse.Mode().IsRegular() {
return fse.FileInfo.Size()
}
return 0
}
func (fse *filesystemEntry) ObjectID() cas.ObjectID {
return fse.objectID
}
func entryFromFileSystemInfo(parentDir string, fi os.FileInfo) EntryOrError {
return EntryOrError{Entry: &filesystemEntry{
FileInfo: fi,
}}
}

24
fs/lister_nonwindows.go
View File

@@ -2,18 +2,20 @@
package fs
import (
"fmt"
"os"
"syscall"
)
import "syscall"
func populatePlatformSpecificEntryDetails(e *Entry, fileInfo os.FileInfo) error {
if stat, ok := fileInfo.Sys().(*syscall.Stat_t); ok {
e.UserID = stat.Uid
e.GroupID = stat.Gid
return nil
func (e *filesystemEntry) UserID() uint32 {
if stat, ok := e.Sys().(*syscall.Stat_t); ok {
return stat.Uid
}
return fmt.Errorf("unable to retrieve platform-specific file information")
return 0
}
func (e *filesystemEntry) GroupID() uint32 {
if stat, ok := e.Sys().(*syscall.Stat_t); ok {
return stat.Gid
}
return 0
}

14
fs/lister_test.go
View File

@@ -61,19 +61,19 @@ func TestLister(t *testing.T) {
goodCount := 0
if ae[0].Name == "f1" && ae[0].Size == 5 && ae[0].Type == EntryTypeFile {
if ae[0].Name() == "f1" && ae[0].Size() == 5 && ae[0].Mode().IsRegular() {
goodCount++
}
if ae[1].Name == "f2" && ae[1].Size == 4 && ae[1].Type == EntryTypeFile {
if ae[1].Name() == "f2" && ae[1].Size() == 4 && ae[1].Mode().IsRegular() {
goodCount++
}
if ae[2].Name == "f3" && ae[2].Size == 3 && ae[2].Type == EntryTypeFile {
if ae[2].Name() == "f3" && ae[2].Size() == 3 && ae[2].Mode().IsRegular() {
goodCount++
}
if ae[3].Name == "y" && ae[3].Size == 0 && ae[3].Type == EntryTypeDirectory {
if ae[3].Name() == "y" && ae[3].Size() == 0 && ae[3].Mode().IsDir() {
goodCount++
}
if ae[4].Name == "z" && ae[4].Size == 0 && ae[4].Type == EntryTypeDirectory {
if ae[4].Name() == "z" && ae[4].Size() == 0 && ae[4].Mode().IsDir() {
goodCount++
}
if goodCount != 5 {
@@ -81,8 +81,8 @@ func TestLister(t *testing.T) {
}
}
func readAllEntries(dir Directory) []*Entry {
var entries []*Entry
func readAllEntries(dir Directory) []Entry {
var entries []Entry
for d := range dir {
if d.Error != nil {
log.Fatalf("got error listing directory: %v", d.Error)

31
fs/upload.go
View File

@@ -60,10 +60,10 @@ func (u *uploader) UploadFile(path string) (cas.ObjectID, error) {
type readaheadDirectory struct {
src Directory
unreadEntriesByName map[string]*Entry
unreadEntriesByName map[string]Entry
}
func (ra *readaheadDirectory) FindByName(name string) *Entry {
func (ra *readaheadDirectory) FindByName(name string) Entry {
if e, ok := ra.unreadEntriesByName[name]; ok {
delete(ra.unreadEntriesByName, name)
return e
@@ -76,10 +76,10 @@ func (ra *readaheadDirectory) FindByName(name string) *Entry {
break
}
if next.Error == nil {
if next.Entry.Name == name {
if next.Entry.Name() == name {
return next.Entry
}
ra.unreadEntriesByName[next.Entry.Name] = next.Entry
ra.unreadEntriesByName[next.Entry.Name()] = next.Entry
}
}
@@ -114,7 +114,7 @@ func (u *uploader) uploadDirInternal(path string, previous cas.ObjectID, previou
ra := readaheadDirectory{
src: previousDir,
unreadEntriesByName: map[string]*Entry{},
unreadEntriesByName: map[string]Entry{},
}
writer := u.mgr.NewWriter(
@@ -128,41 +128,44 @@ func (u *uploader) uploadDirInternal(path string, previous cas.ObjectID, previou
directoryMatchesCache := true
for de := range dir {
e := de.Entry
fullPath := filepath.Join(path, e.Name)
fullPath := filepath.Join(path, e.Name())
// See if we had this name during previous pass.
cachedEntry := ra.FindByName(e.Name)
cachedEntry := ra.FindByName(e.Name())
// ... and whether file metadata is identical to the previous one.
cachedMetadataMatches := e.metadataEquals(cachedEntry)
cachedMetadataMatches := metadataEquals(e, cachedEntry)
// If not, directoryMatchesCache becomes false.
directoryMatchesCache = directoryMatchesCache && cachedMetadataMatches
if e.Type == EntryTypeDirectory {
var oid cas.ObjectID
if e.IsDir() {
var previousSubdirObjectID cas.ObjectID
if cachedEntry != nil {
previousSubdirObjectID = cachedEntry.ObjectID
previousSubdirObjectID = cachedEntry.ObjectID()
}
e.ObjectID, err = u.UploadDir(fullPath, previousSubdirObjectID)
oid, err = u.UploadDir(fullPath, previousSubdirObjectID)
if err != nil {
return cas.NullObjectID, err
}
if cachedEntry != nil && e.ObjectID != cachedEntry.ObjectID {
if cachedEntry != nil && oid != cachedEntry.ObjectID() {
directoryMatchesCache = false
}
} else if cachedMetadataMatches {
// Avoid hashing by reusing previous object ID.
e.ObjectID = cachedEntry.ObjectID
oid = cachedEntry.ObjectID()
} else {
e.ObjectID, err = u.UploadFile(fullPath)
oid, err = u.UploadFile(fullPath)
if err != nil {
return cas.NullObjectID, fmt.Errorf("unable to hash file: %s", err)
}
}
e = &entryWithObjectID{Entry: e, oid: oid}
writeDirectoryEntry(writer, e)
}