restore: support for symlinks (experimental) (#621)

cli/command_restore.go

@@ -144,7 +144,7 @@ func detectRestoreMode(m string) string {
 }
 
 func printRestoreStats(st restore.Stats) {
-	printStderr("Restored %v files and %v directories (%v)\n", st.FileCount, st.DirCount, units.BytesStringBase10(st.TotalFileSize))
+	printStderr("Restored %v files, %v directories and %v symbolic links (%v)\n", st.FileCount, st.DirCount, st.SymlinkCount, units.BytesStringBase10(st.TotalFileSize))
 }
 
 func runRestoreCommand(ctx context.Context, rep repo.Repository) error {

go.mod

@@ -50,6 +50,7 @@ require (
 	golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2
 	golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d
 	golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a
+	golang.org/x/sys v0.0.0-20200511232937-7e40ca221e25
 	golang.org/x/tools v0.0.0-20200521155704-91d71f6c2f04 // indirect
 	google.golang.org/api v0.25.0
 	google.golang.org/protobuf v1.23.0

internal/fshasher/fshasher.go

@@ -9,6 +9,7 @@
 	"path"
 	"time"
 
+	"github.com/pkg/errors"
 	"golang.org/x/crypto/blake2s"
 
 	"github.com/kopia/kopia/fs"
@@ -46,7 +47,7 @@ func write(ctx context.Context, tw *tar.Writer, fullpath string, e fs.Entry) err
 		return err
 	}
 
-	log(ctx).Debugf("%v %v %v %v", e.Mode(), h.ModTime.Format(time.RFC3339), h.Size, h.Name)
+	log(ctx).Debugf("%v %v %v %v %v", e.Mode(), h.ModTime.Format(time.RFC3339), h.Size, h.Name, h.Linkname)
 
 	if err := tw.WriteHeader(h); err != nil {
 		return err
@@ -57,7 +58,10 @@ func write(ctx context.Context, tw *tar.Writer, fullpath string, e fs.Entry) err
 		return writeDirectory(ctx, tw, fullpath, e)
 	case fs.File:
 		return writeFile(ctx, tw, e)
-	default: // fs.Symlink or bare fs.Entry
+	case fs.Symlink:
+		// link target is part of the header
+		return nil
+	default: // bare fs.Entry
 		return nil
 	}
 }
@@ -92,6 +96,13 @@ func header(ctx context.Context, fullpath string, e os.FileInfo) (*tar.Header, e
 	h.AccessTime = h.ModTime
 	h.ChangeTime = h.ModTime
 
+	if sl, ok := e.(fs.Symlink); ok {
+		h.Linkname, err = sl.Readlink(ctx)
+		if err != nil {
+			return nil, errors.Wrap(err, "error reading link")
+		}
+	}
+
 	return h, nil
 }
 

snapshot/restore/local_fs_output.go

@@ -43,7 +43,7 @@ func (o *FilesystemOutput) BeginDirectory(ctx context.Context, relativePath stri
 // FinishDirectory implements restore.Output interface.
 func (o *FilesystemOutput) FinishDirectory(ctx context.Context, relativePath string, e fs.Directory) error {
 	path := filepath.Join(o.TargetPath, filepath.FromSlash(relativePath))
-	if err := o.setAttributes(path, e); err != nil {
+	if err := o.setAttributes(ctx, path, e); err != nil {
 		return errors.Wrap(err, "error setting attributes")
 	}
 
@@ -57,14 +57,14 @@ func (o *FilesystemOutput) Close(ctx context.Context) error {
 
 // WriteFile implements restore.Output interface.
 func (o *FilesystemOutput) WriteFile(ctx context.Context, relativePath string, f fs.File) error {
-	log(ctx).Infof("WriteFile %v %v", relativePath, f)
+	log(ctx).Infof("WriteFile %v %v", relativePath, f.Size())
 	path := filepath.Join(o.TargetPath, filepath.FromSlash(relativePath))
 
 	if err := o.copyFileContent(ctx, path, f); err != nil {
 		return errors.Wrap(err, "error creating directory")
 	}
 
-	if err := o.setAttributes(path, f); err != nil {
+	if err := o.setAttributes(ctx, path, f); err != nil {
 		return errors.Wrap(err, "error setting attributes")
 	}
 
@@ -73,12 +73,28 @@ func (o *FilesystemOutput) WriteFile(ctx context.Context, relativePath string, f
 
 // CreateSymlink implements restore.Output interface.
 func (o *FilesystemOutput) CreateSymlink(ctx context.Context, relativePath string, e fs.Symlink) error {
-	log(ctx).Debugf("create symlink not implemented yet")
+	targetPath, err := e.Readlink(ctx)
+	if err != nil {
+		return errors.Wrap(err, "error reading link target")
+	}
+
+	log(ctx).Infof("CreateSymlink %v => %v", relativePath, targetPath)
+
+	path := filepath.Join(o.TargetPath, filepath.FromSlash(relativePath))
+
+	if err := os.Symlink(targetPath, path); err != nil {
+		return errors.Wrap(err, "error creating symlink")
+	}
+
+	if err := o.setAttributes(ctx, path, e); err != nil {
+		return errors.Wrap(err, "error setting attributes")
+	}
+
 	return nil
 }
 
 // set permission, modification time and user/group ids on targetPath.
-func (o *FilesystemOutput) setAttributes(targetPath string, e fs.Entry) error {
+func (o *FilesystemOutput) setAttributes(ctx context.Context, targetPath string, e fs.Entry) error {
 	const modBits = os.ModePerm | os.ModeSetgid | os.ModeSetuid | os.ModeSticky
 
 	le, err := localfs.NewEntry(targetPath)
@@ -103,7 +119,16 @@ func (o *FilesystemOutput) setAttributes(targetPath string, e fs.Entry) error {
 	}
 
 	// Set mod time from e
-	if !le.ModTime().Equal(e.ModTime()) {
+	if le.ModTime().Equal(e.ModTime()) {
+		return nil
+	}
+
+	if _, isSymlink := e.(fs.Symlink); isSymlink {
+		// symbolic links require special handling that is OS-specific and sometimes unsupported.
+		if err = symlinkChtimes(ctx, targetPath, e.ModTime(), e.ModTime()); err != nil && !os.IsPermission(err) {
+			return errors.Wrap(err, "could not change mod time on "+targetPath)
+		}
+	} else {
 		// Note: Set atime to ModTime as well
 		if err = os.Chtimes(targetPath, e.ModTime(), e.ModTime()); err != nil && !os.IsPermission(err) {
 			return errors.Wrap(err, "could not change mod time on "+targetPath)

snapshot/restore/local_fs_output_unix.go

@@ -0,0 +1,17 @@
+// +build linux darwin
+
+package restore
+
+import (
+	"context"
+	"time"
+
+	"golang.org/x/sys/unix"
+)
+
+func symlinkChtimes(ctx context.Context, linkPath string, atime, mtime time.Time) error {
+	return unix.Lutimes(linkPath, []unix.Timeval{
+		unix.NsecToTimeval(atime.UnixNano()),
+		unix.NsecToTimeval(mtime.UnixNano()),
+	})
+}

snapshot/restore/local_fs_output_windows.go

@@ -0,0 +1,32 @@
+package restore
+
+import (
+	"context"
+	"time"
+
+	"github.com/pkg/errors"
+	"golang.org/x/sys/windows"
+)
+
+func symlinkChtimes(ctx context.Context, linkPath string, atime, mtime time.Time) error {
+	fta := windows.NsecToFiletime(atime.UnixNano())
+	ftw := windows.NsecToFiletime(mtime.UnixNano())
+
+	fn, err := windows.UTF16PtrFromString(linkPath)
+	if err != nil {
+		return errors.Wrap(err, "UTF16PtrFromString")
+	}
+
+	h, err := windows.CreateFile(
+		fn, windows.GENERIC_READ|windows.GENERIC_WRITE,
+		windows.FILE_SHARE_READ|windows.FILE_SHARE_WRITE,
+		nil, windows.OPEN_EXISTING,
+		windows.FILE_FLAG_OPEN_REPARSE_POINT, 0)
+	if err != nil {
+		return err
+	}
+
+	defer windows.CloseHandle(h) //nolint:errcheck
+
+	return windows.SetFileTime(h, &ftw, &fta, &ftw)
+}

snapshot/restore/restore.go

@@ -32,6 +32,7 @@ type Stats struct {
 	TotalFileSize int64
 	FileCount     int32
 	DirCount      int32
+	SymlinkCount  int32
 }
 
 // Snapshot walks a snapshot root with given snapshot ID and restores it to the provided output.
@@ -90,7 +91,9 @@ func (c *copier) copyEntry(ctx context.Context, e fs.Entry, targetPath string) e
 
 		return c.output.WriteFile(ctx, targetPath, e)
 	case fs.Symlink:
+		atomic.AddInt32(&c.stats.SymlinkCount, 1)
 		log(ctx).Debugf("symlink: '%v'", targetPath)
+
 		return c.output.CreateSymlink(ctx, targetPath, e)
 	default:
 		return errors.Errorf("invalid FS entry type for %q: %#v", targetPath, e)

tests/end_to_end_test/restore_test.go

@@ -29,8 +29,10 @@ func TestRestoreCommand(t *testing.T) {
 
 	source := makeScratchDir(t)
 	testenv.MustCreateDirectoryTree(t, source, testenv.DirectoryTreeOptions{
-		Depth:                1,
-		MaxFilesPerDirectory: 10,
+		Depth:                              1,
+		MaxFilesPerDirectory:               10,
+		MaxSymlinksPerDirectory:            4,
+		NonExistingSymlinkTargetPercentage: 50,
 	})
 
 	restoreDir := makeScratchDir(t)
@@ -117,9 +119,11 @@ func TestSnapshotRestore(t *testing.T) {
 
 	source := makeScratchDir(t)
 	testenv.MustCreateDirectoryTree(t, source, testenv.DirectoryTreeOptions{
-		Depth:                  5,
-		MaxSubdirsPerDirectory: 5,
-		MaxFilesPerDirectory:   5,
+		Depth:                              5,
+		MaxSubdirsPerDirectory:             5,
+		MaxFilesPerDirectory:               5,
+		MaxSymlinksPerDirectory:            4,
+		NonExistingSymlinkTargetPercentage: 50,
 	})
 
 	restoreDir := makeScratchDir(t)

tests/testenv/cli_test_env.go

@@ -287,18 +287,21 @@ func mustParseDirectoryEntries(lines []string) []DirEntry {
 
 // DirectoryTreeOptions lists options for CreateDirectoryTree.
 type DirectoryTreeOptions struct {
-	Depth                  int
-	MaxSubdirsPerDirectory int
-	MaxFilesPerDirectory   int
-	MaxFileSize            int
-	MinNameLength          int
-	MaxNameLength          int
+	Depth                              int
+	MaxSubdirsPerDirectory             int
+	MaxFilesPerDirectory               int
+	MaxSymlinksPerDirectory            int
+	MaxFileSize                        int
+	MinNameLength                      int
+	MaxNameLength                      int
+	NonExistingSymlinkTargetPercentage int // 0..100
 }
 
 // DirectoryTreeCounters stores stats about files and directories created by CreateDirectoryTree.
 type DirectoryTreeCounters struct {
 	Files         int
 	Directories   int
+	Symlinks      int
 	TotalFileSize int64
 	MaxFileSize   int64
 }
@@ -311,6 +314,8 @@ func MustCreateDirectoryTree(t *testing.T, dirname string, options DirectoryTree
 	if err := createDirectoryTreeInternal(dirname, options, &counters); err != nil {
 		t.Error(err)
 	}
+
+	t.Logf("created directory tree %#v", counters)
 }
 
 // CreateDirectoryTree creates a directory tree of a given depth with random files.
@@ -361,6 +366,8 @@ func createDirectoryTreeInternal(dirname string, options DirectoryTreeOptions, c
 		}
 	}
 
+	var fileNames []string
+
 	if options.MaxFilesPerDirectory > 0 {
 		numFiles := rand.Intn(options.MaxFilesPerDirectory) + 1
 		for i := 0; i < numFiles; i++ {
@@ -369,6 +376,19 @@ func createDirectoryTreeInternal(dirname string, options DirectoryTreeOptions, c
 			if err := createRandomFile(filepath.Join(dirname, fileName), options, counters); err != nil {
 				return errors.Wrap(err, "unable to create random file")
 			}
+
+			fileNames = append(fileNames, fileName)
+		}
+	}
+
+	if options.MaxSymlinksPerDirectory > 0 {
+		numSymlinks := rand.Intn(options.MaxSymlinksPerDirectory) + 1
+		for i := 0; i < numSymlinks; i++ {
+			fileName := randomName(options)
+
+			if err := createRandomSymlink(filepath.Join(dirname, fileName), fileNames, options, counters); err != nil {
+				return errors.Wrap(err, "unable to create random symlink")
+			}
 		}
 	}
 
@@ -401,6 +421,16 @@ func createRandomFile(filename string, options DirectoryTreeOptions, counters *D
 	return nil
 }
 
+func createRandomSymlink(filename string, existingFiles []string, options DirectoryTreeOptions, counters *DirectoryTreeCounters) error {
+	counters.Symlinks++
+
+	if len(existingFiles) == 0 || rand.Intn(100) < options.NonExistingSymlinkTargetPercentage {
+		return os.Symlink(randomName(options), filename)
+	}
+
+	return os.Symlink(existingFiles[rand.Intn(len(existingFiles))], filename)
+}
+
 func mustParseSnapshots(t *testing.T, lines []string) []SourceInfo {
 	var result []SourceInfo