From fce94973759227ded57c714ec84af7aeafc1e7c6 Mon Sep 17 00:00:00 2001 From: Jarek Kowalski Date: Fri, 18 Sep 2020 10:29:20 -0700 Subject: [PATCH] restore: support for symlinks (experimental) (#621) --- cli/command_restore.go | 2 +- go.mod | 1 + internal/fshasher/fshasher.go | 15 +++++++- snapshot/restore/local_fs_output.go | 37 +++++++++++++++--- snapshot/restore/local_fs_output_unix.go | 17 +++++++++ snapshot/restore/local_fs_output_windows.go | 32 ++++++++++++++++ snapshot/restore/restore.go | 3 ++ tests/end_to_end_test/restore_test.go | 14 ++++--- tests/testenv/cli_test_env.go | 42 ++++++++++++++++++--- 9 files changed, 143 insertions(+), 20 deletions(-) create mode 100644 snapshot/restore/local_fs_output_unix.go create mode 100644 snapshot/restore/local_fs_output_windows.go diff --git a/cli/command_restore.go b/cli/command_restore.go index 4d5f84b24..67fbd7f8e 100644 --- a/cli/command_restore.go +++ b/cli/command_restore.go @@ -144,7 +144,7 @@ func detectRestoreMode(m string) string { } func printRestoreStats(st restore.Stats) { - printStderr("Restored %v files and %v directories (%v)\n", st.FileCount, st.DirCount, units.BytesStringBase10(st.TotalFileSize)) + printStderr("Restored %v files, %v directories and %v symbolic links (%v)\n", st.FileCount, st.DirCount, st.SymlinkCount, units.BytesStringBase10(st.TotalFileSize)) } func runRestoreCommand(ctx context.Context, rep repo.Repository) error { diff --git a/go.mod b/go.mod index 397625b29..d076b1085 100644 --- a/go.mod +++ b/go.mod @@ -50,6 +50,7 @@ require ( golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2 golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a + golang.org/x/sys v0.0.0-20200511232937-7e40ca221e25 golang.org/x/tools v0.0.0-20200521155704-91d71f6c2f04 // indirect google.golang.org/api v0.25.0 google.golang.org/protobuf v1.23.0 diff --git a/internal/fshasher/fshasher.go b/internal/fshasher/fshasher.go index 8ff77c186..cf3be312c 100644 --- a/internal/fshasher/fshasher.go +++ b/internal/fshasher/fshasher.go @@ -9,6 +9,7 @@ "path" "time" + "github.com/pkg/errors" "golang.org/x/crypto/blake2s" "github.com/kopia/kopia/fs" @@ -46,7 +47,7 @@ func write(ctx context.Context, tw *tar.Writer, fullpath string, e fs.Entry) err return err } - log(ctx).Debugf("%v %v %v %v", e.Mode(), h.ModTime.Format(time.RFC3339), h.Size, h.Name) + log(ctx).Debugf("%v %v %v %v %v", e.Mode(), h.ModTime.Format(time.RFC3339), h.Size, h.Name, h.Linkname) if err := tw.WriteHeader(h); err != nil { return err @@ -57,7 +58,10 @@ func write(ctx context.Context, tw *tar.Writer, fullpath string, e fs.Entry) err return writeDirectory(ctx, tw, fullpath, e) case fs.File: return writeFile(ctx, tw, e) - default: // fs.Symlink or bare fs.Entry + case fs.Symlink: + // link target is part of the header + return nil + default: // bare fs.Entry return nil } } @@ -92,6 +96,13 @@ func header(ctx context.Context, fullpath string, e os.FileInfo) (*tar.Header, e h.AccessTime = h.ModTime h.ChangeTime = h.ModTime + if sl, ok := e.(fs.Symlink); ok { + h.Linkname, err = sl.Readlink(ctx) + if err != nil { + return nil, errors.Wrap(err, "error reading link") + } + } + return h, nil } diff --git a/snapshot/restore/local_fs_output.go b/snapshot/restore/local_fs_output.go index 23b14e53c..b1986a47a 100644 --- a/snapshot/restore/local_fs_output.go +++ b/snapshot/restore/local_fs_output.go @@ -43,7 +43,7 @@ func (o *FilesystemOutput) BeginDirectory(ctx context.Context, relativePath stri // FinishDirectory implements restore.Output interface. func (o *FilesystemOutput) FinishDirectory(ctx context.Context, relativePath string, e fs.Directory) error { path := filepath.Join(o.TargetPath, filepath.FromSlash(relativePath)) - if err := o.setAttributes(path, e); err != nil { + if err := o.setAttributes(ctx, path, e); err != nil { return errors.Wrap(err, "error setting attributes") } @@ -57,14 +57,14 @@ func (o *FilesystemOutput) Close(ctx context.Context) error { // WriteFile implements restore.Output interface. func (o *FilesystemOutput) WriteFile(ctx context.Context, relativePath string, f fs.File) error { - log(ctx).Infof("WriteFile %v %v", relativePath, f) + log(ctx).Infof("WriteFile %v %v", relativePath, f.Size()) path := filepath.Join(o.TargetPath, filepath.FromSlash(relativePath)) if err := o.copyFileContent(ctx, path, f); err != nil { return errors.Wrap(err, "error creating directory") } - if err := o.setAttributes(path, f); err != nil { + if err := o.setAttributes(ctx, path, f); err != nil { return errors.Wrap(err, "error setting attributes") } @@ -73,12 +73,28 @@ func (o *FilesystemOutput) WriteFile(ctx context.Context, relativePath string, f // CreateSymlink implements restore.Output interface. func (o *FilesystemOutput) CreateSymlink(ctx context.Context, relativePath string, e fs.Symlink) error { - log(ctx).Debugf("create symlink not implemented yet") + targetPath, err := e.Readlink(ctx) + if err != nil { + return errors.Wrap(err, "error reading link target") + } + + log(ctx).Infof("CreateSymlink %v => %v", relativePath, targetPath) + + path := filepath.Join(o.TargetPath, filepath.FromSlash(relativePath)) + + if err := os.Symlink(targetPath, path); err != nil { + return errors.Wrap(err, "error creating symlink") + } + + if err := o.setAttributes(ctx, path, e); err != nil { + return errors.Wrap(err, "error setting attributes") + } + return nil } // set permission, modification time and user/group ids on targetPath. -func (o *FilesystemOutput) setAttributes(targetPath string, e fs.Entry) error { +func (o *FilesystemOutput) setAttributes(ctx context.Context, targetPath string, e fs.Entry) error { const modBits = os.ModePerm | os.ModeSetgid | os.ModeSetuid | os.ModeSticky le, err := localfs.NewEntry(targetPath) @@ -103,7 +119,16 @@ func (o *FilesystemOutput) setAttributes(targetPath string, e fs.Entry) error { } // Set mod time from e - if !le.ModTime().Equal(e.ModTime()) { + if le.ModTime().Equal(e.ModTime()) { + return nil + } + + if _, isSymlink := e.(fs.Symlink); isSymlink { + // symbolic links require special handling that is OS-specific and sometimes unsupported. + if err = symlinkChtimes(ctx, targetPath, e.ModTime(), e.ModTime()); err != nil && !os.IsPermission(err) { + return errors.Wrap(err, "could not change mod time on "+targetPath) + } + } else { // Note: Set atime to ModTime as well if err = os.Chtimes(targetPath, e.ModTime(), e.ModTime()); err != nil && !os.IsPermission(err) { return errors.Wrap(err, "could not change mod time on "+targetPath) diff --git a/snapshot/restore/local_fs_output_unix.go b/snapshot/restore/local_fs_output_unix.go new file mode 100644 index 000000000..448a56f20 --- /dev/null +++ b/snapshot/restore/local_fs_output_unix.go @@ -0,0 +1,17 @@ +// +build linux darwin + +package restore + +import ( + "context" + "time" + + "golang.org/x/sys/unix" +) + +func symlinkChtimes(ctx context.Context, linkPath string, atime, mtime time.Time) error { + return unix.Lutimes(linkPath, []unix.Timeval{ + unix.NsecToTimeval(atime.UnixNano()), + unix.NsecToTimeval(mtime.UnixNano()), + }) +} diff --git a/snapshot/restore/local_fs_output_windows.go b/snapshot/restore/local_fs_output_windows.go new file mode 100644 index 000000000..01cfe4fa6 --- /dev/null +++ b/snapshot/restore/local_fs_output_windows.go @@ -0,0 +1,32 @@ +package restore + +import ( + "context" + "time" + + "github.com/pkg/errors" + "golang.org/x/sys/windows" +) + +func symlinkChtimes(ctx context.Context, linkPath string, atime, mtime time.Time) error { + fta := windows.NsecToFiletime(atime.UnixNano()) + ftw := windows.NsecToFiletime(mtime.UnixNano()) + + fn, err := windows.UTF16PtrFromString(linkPath) + if err != nil { + return errors.Wrap(err, "UTF16PtrFromString") + } + + h, err := windows.CreateFile( + fn, windows.GENERIC_READ|windows.GENERIC_WRITE, + windows.FILE_SHARE_READ|windows.FILE_SHARE_WRITE, + nil, windows.OPEN_EXISTING, + windows.FILE_FLAG_OPEN_REPARSE_POINT, 0) + if err != nil { + return err + } + + defer windows.CloseHandle(h) //nolint:errcheck + + return windows.SetFileTime(h, &ftw, &fta, &ftw) +} diff --git a/snapshot/restore/restore.go b/snapshot/restore/restore.go index 422e649ec..2aef4af44 100644 --- a/snapshot/restore/restore.go +++ b/snapshot/restore/restore.go @@ -32,6 +32,7 @@ type Stats struct { TotalFileSize int64 FileCount int32 DirCount int32 + SymlinkCount int32 } // Snapshot walks a snapshot root with given snapshot ID and restores it to the provided output. @@ -90,7 +91,9 @@ func (c *copier) copyEntry(ctx context.Context, e fs.Entry, targetPath string) e return c.output.WriteFile(ctx, targetPath, e) case fs.Symlink: + atomic.AddInt32(&c.stats.SymlinkCount, 1) log(ctx).Debugf("symlink: '%v'", targetPath) + return c.output.CreateSymlink(ctx, targetPath, e) default: return errors.Errorf("invalid FS entry type for %q: %#v", targetPath, e) diff --git a/tests/end_to_end_test/restore_test.go b/tests/end_to_end_test/restore_test.go index b5dfb4d52..674708564 100644 --- a/tests/end_to_end_test/restore_test.go +++ b/tests/end_to_end_test/restore_test.go @@ -29,8 +29,10 @@ func TestRestoreCommand(t *testing.T) { source := makeScratchDir(t) testenv.MustCreateDirectoryTree(t, source, testenv.DirectoryTreeOptions{ - Depth: 1, - MaxFilesPerDirectory: 10, + Depth: 1, + MaxFilesPerDirectory: 10, + MaxSymlinksPerDirectory: 4, + NonExistingSymlinkTargetPercentage: 50, }) restoreDir := makeScratchDir(t) @@ -117,9 +119,11 @@ func TestSnapshotRestore(t *testing.T) { source := makeScratchDir(t) testenv.MustCreateDirectoryTree(t, source, testenv.DirectoryTreeOptions{ - Depth: 5, - MaxSubdirsPerDirectory: 5, - MaxFilesPerDirectory: 5, + Depth: 5, + MaxSubdirsPerDirectory: 5, + MaxFilesPerDirectory: 5, + MaxSymlinksPerDirectory: 4, + NonExistingSymlinkTargetPercentage: 50, }) restoreDir := makeScratchDir(t) diff --git a/tests/testenv/cli_test_env.go b/tests/testenv/cli_test_env.go index c08716d3a..13e334d7a 100644 --- a/tests/testenv/cli_test_env.go +++ b/tests/testenv/cli_test_env.go @@ -287,18 +287,21 @@ func mustParseDirectoryEntries(lines []string) []DirEntry { // DirectoryTreeOptions lists options for CreateDirectoryTree. type DirectoryTreeOptions struct { - Depth int - MaxSubdirsPerDirectory int - MaxFilesPerDirectory int - MaxFileSize int - MinNameLength int - MaxNameLength int + Depth int + MaxSubdirsPerDirectory int + MaxFilesPerDirectory int + MaxSymlinksPerDirectory int + MaxFileSize int + MinNameLength int + MaxNameLength int + NonExistingSymlinkTargetPercentage int // 0..100 } // DirectoryTreeCounters stores stats about files and directories created by CreateDirectoryTree. type DirectoryTreeCounters struct { Files int Directories int + Symlinks int TotalFileSize int64 MaxFileSize int64 } @@ -311,6 +314,8 @@ func MustCreateDirectoryTree(t *testing.T, dirname string, options DirectoryTree if err := createDirectoryTreeInternal(dirname, options, &counters); err != nil { t.Error(err) } + + t.Logf("created directory tree %#v", counters) } // CreateDirectoryTree creates a directory tree of a given depth with random files. @@ -361,6 +366,8 @@ func createDirectoryTreeInternal(dirname string, options DirectoryTreeOptions, c } } + var fileNames []string + if options.MaxFilesPerDirectory > 0 { numFiles := rand.Intn(options.MaxFilesPerDirectory) + 1 for i := 0; i < numFiles; i++ { @@ -369,6 +376,19 @@ func createDirectoryTreeInternal(dirname string, options DirectoryTreeOptions, c if err := createRandomFile(filepath.Join(dirname, fileName), options, counters); err != nil { return errors.Wrap(err, "unable to create random file") } + + fileNames = append(fileNames, fileName) + } + } + + if options.MaxSymlinksPerDirectory > 0 { + numSymlinks := rand.Intn(options.MaxSymlinksPerDirectory) + 1 + for i := 0; i < numSymlinks; i++ { + fileName := randomName(options) + + if err := createRandomSymlink(filepath.Join(dirname, fileName), fileNames, options, counters); err != nil { + return errors.Wrap(err, "unable to create random symlink") + } } } @@ -401,6 +421,16 @@ func createRandomFile(filename string, options DirectoryTreeOptions, counters *D return nil } +func createRandomSymlink(filename string, existingFiles []string, options DirectoryTreeOptions, counters *DirectoryTreeCounters) error { + counters.Symlinks++ + + if len(existingFiles) == 0 || rand.Intn(100) < options.NonExistingSymlinkTargetPercentage { + return os.Symlink(randomName(options), filename) + } + + return os.Symlink(existingFiles[rand.Intn(len(existingFiles))], filename) +} + func mustParseSnapshots(t *testing.T, lines []string) []SourceInfo { var result []SourceInfo