From 77aef5d02e0cf5bdf3c1aa87ce3c38a966e97fea Mon Sep 17 00:00:00 2001 From: Jarek Kowalski Date: Sat, 6 Oct 2018 09:52:41 -0700 Subject: [PATCH] cli: added 'diff' command which compares two directories and optionally emits unified diff --- cli/command_diff.go | 72 ++++++++++++ cli/command_mount_browse.go | 3 +- cli/config.go | 5 + internal/diff/diff.go | 212 ++++++++++++++++++++++++++++++++++++ 4 files changed, 290 insertions(+), 2 deletions(-) create mode 100644 cli/command_diff.go create mode 100644 internal/diff/diff.go diff --git a/cli/command_diff.go b/cli/command_diff.go new file mode 100644 index 000000000..36211e4e2 --- /dev/null +++ b/cli/command_diff.go @@ -0,0 +1,72 @@ +package cli + +import ( + "context" + "fmt" + "os" + "strings" + + "github.com/kopia/kopia/fs/repofs" + "github.com/kopia/kopia/internal/diff" + + "github.com/kopia/kopia/repo" +) + +var ( + diffCommand = app.Command("diff", "Displays differences between two repository objects (files or directories)").Alias("compare") + diffFirstObjectPath = diffCommand.Arg("object-path1", "First object/path").Required().String() + diffSecondObjectPath = diffCommand.Arg("object-path2", "Second object/path").Required().String() + diffCompareFiles = diffCommand.Flag("files", "Compare files by launching diff command for all pairs of (old,new)").Short('f').Bool() + diffCommandCommand = app.Flag("diff-command", "Displays differences between two repository objects (files or directories)").Default(defaultDiffCommand()).String() +) + +func runDiffCommand(ctx context.Context, rep *repo.Repository) error { + oid1, err := parseObjectID(ctx, rep, *diffFirstObjectPath) + if err != nil { + return err + } + oid2, err := parseObjectID(ctx, rep, *diffSecondObjectPath) + if err != nil { + return err + } + + isDir1 := strings.HasPrefix(string(oid1), "k") + isDir2 := strings.HasPrefix(string(oid2), "k") + if isDir1 != isDir2 { + return fmt.Errorf("arguments do diff must both be directories or both non-directories") + } + + d, err := diff.NewComparer(rep, os.Stdout) + if err != nil { + return err + } + defer d.Close() //nolint:errcheck + + if *diffCompareFiles { + parts := strings.Split(*diffCommandCommand, " ") + d.DiffCommand = parts[0] + d.DiffArguments = parts[1:] + } + + if isDir1 { + return d.Compare( + ctx, + repofs.DirectoryEntry(rep, oid1, nil), + repofs.DirectoryEntry(rep, oid2, nil), + ) + } + + return fmt.Errorf("comparing files not implemented yet") +} + +func defaultDiffCommand() string { + if isWindows() { + return "cmp" + } + + return "diff -u" +} + +func init() { + diffCommand.Action(repositoryAction(runDiffCommand)) +} diff --git a/cli/command_mount_browse.go b/cli/command_mount_browse.go index 08e578beb..213d4f426 100644 --- a/cli/command_mount_browse.go +++ b/cli/command_mount_browse.go @@ -4,7 +4,6 @@ "fmt" "os" "os/exec" - "runtime" "github.com/skratchdot/open-golang/open" ) @@ -36,7 +35,7 @@ func openInWebBrowser(mountPoint string, addr string) error { } func openInOSBrowser(mountPoint string, addr string) error { - if runtime.GOOS == "windows" { + if isWindows() { return netUSE(mountPoint, addr) } diff --git a/cli/config.go b/cli/config.go index c61c68195..d004d148e 100644 --- a/cli/config.go +++ b/cli/config.go @@ -6,6 +6,7 @@ "os" "os/signal" "path/filepath" + "runtime" "github.com/kopia/kopia/fs" "github.com/kopia/kopia/fs/localfs" @@ -113,3 +114,7 @@ func mustGetLocalFSEntry(path string) fs.Entry { return e } + +func isWindows() bool { + return runtime.GOOS == "windows" +} diff --git a/internal/diff/diff.go b/internal/diff/diff.go new file mode 100644 index 000000000..7fbad8738 --- /dev/null +++ b/internal/diff/diff.go @@ -0,0 +1,212 @@ +package diff + +import ( + "context" + "fmt" + "io" + "io/ioutil" + "os" + "os/exec" + "path/filepath" + + "github.com/kopia/kopia/fs" + "github.com/kopia/kopia/internal/kopialogging" + "github.com/kopia/kopia/repo" + "github.com/kopia/kopia/repo/object" +) + +var log = kopialogging.Logger("diff") + +// Comparer outputs diff information between two filesystems. +type Comparer struct { + rep *repo.Repository + out io.Writer + tmpDir string + + DiffCommand string + DiffArguments []string +} + +// Compare compares two filesystem entries and emits their diff information. +func (c *Comparer) Compare(ctx context.Context, e1, e2 fs.Entry) error { + return c.compareEntry(ctx, e1, e2, ".") +} + +// Close removes all temporary files used by the comparer. +func (c *Comparer) Close() error { + return os.RemoveAll(c.tmpDir) +} + +func (c *Comparer) compareDirectories(ctx context.Context, dir1, dir2 fs.Directory, parent string) error { + log.Debugf("comparing directories %v", parent) + var entries1, entries2 fs.Entries + var err error + + if dir1 != nil { + entries1, err = dir1.Readdir(ctx) + if err != nil { + return fmt.Errorf("unable to read first directory %v: %v", parent, err) + } + } + + if dir2 != nil { + entries2, err = dir2.Readdir(ctx) + if err != nil { + return fmt.Errorf("unable to read second directory %v: %v", parent, err) + } + } + + return c.compareDirectoryEntries(ctx, entries1, entries2, parent) +} + +// nolint:gocyclo +func (c *Comparer) compareEntry(ctx context.Context, e1, e2 fs.Entry, path string) error { + // see if we have the same object IDs, which implies identical objects, thanks to content-addressable-storage + if h1, ok := e1.(object.HasObjectID); ok { + if h2, ok := e2.(object.HasObjectID); ok { + if h1.ObjectID() == h2.ObjectID() { + log.Debugf("unchanged %v", path) + return nil + } + } + } + + if e1 == nil { + if dir2, isDir2 := e2.(fs.Directory); isDir2 { + c.output("added directory %v\n", path) + return c.compareDirectories(ctx, nil, dir2, path) + } + + c.output("added file %v (%v bytes)\n", path, e2.Metadata().FileSize) + return nil + } + + if e2 == nil { + if dir1, isDir1 := e1.(fs.Directory); isDir1 { + c.output("removed directory %v\n", path) + return c.compareDirectories(ctx, dir1, nil, path) + } + + c.output("removed file %v (%v bytes)\n", path, e1.Metadata().FileSize) + return nil + } + + dir1, isDir1 := e1.(fs.Directory) + dir2, isDir2 := e2.(fs.Directory) + if isDir1 { + if !isDir2 { + // right is a non-directory, left is a directory + c.output("changed %v from directory to non-directory\n", path) + return nil + } + + return c.compareDirectories(ctx, dir1, dir2, path) + } + + if isDir2 { + // left is non-directory, right is a directory + log.Infof("changed %v from non-directory to a directory", path) + return nil + } + + c.output("changed %v at %v (size %v -> %v)\n", path, e2.Metadata().ModTime.String(), e1.Metadata().FileSize, e2.Metadata().FileSize) + if f1, ok := e1.(fs.File); ok { + if f2, ok := e2.(fs.File); ok { + if err := c.compareFiles(ctx, f1, f2, path); err != nil { + return err + } + } + } + + return nil +} + +func (c *Comparer) compareDirectoryEntries(ctx context.Context, entries1, entries2 fs.Entries, dirPath string) error { + e1byname := map[string]fs.Entry{} + for _, e1 := range entries1 { + e1byname[e1.Metadata().Name] = e1 + } + + for _, e2 := range entries2 { + entryName := e2.Metadata().Name + if err := c.compareEntry(ctx, e1byname[entryName], e2, dirPath+"/"+entryName); err != nil { + return fmt.Errorf("error comparing %v: %v", entryName, err) + } + delete(e1byname, entryName) + } + + // at this point e1byname only has entries present in entries1 but not entries2, those are the deleted ones + for _, e1 := range entries1 { + entryName := e1.Metadata().Name + if _, ok := e1byname[entryName]; ok { + if err := c.compareEntry(ctx, e1, nil, dirPath+"/"+entryName); err != nil { + return fmt.Errorf("error comparing %v: %v", entryName, err) + } + } + } + + return nil +} + +func (c *Comparer) compareFiles(ctx context.Context, f1, f2 fs.File, fname string) error { + oldName := filepath.Clean("old/" + fname) + newName := filepath.Clean("new/" + fname) + oldFile := filepath.Join(c.tmpDir, oldName) + newFile := filepath.Join(c.tmpDir, newName) + + defer os.Remove(oldFile) //nolint:errcheck + defer os.Remove(newFile) //nolint:errcheck + + if err := c.downloadFile(ctx, f1, oldFile); err != nil { + return fmt.Errorf("error downloading old file: %v", err) + } + if err := c.downloadFile(ctx, f2, newFile); err != nil { + return fmt.Errorf("error downloading new file: %v", err) + } + + var args []string + args = append(args, c.DiffArguments...) + args = append(args, oldName) + args = append(args, newName) + + cmd := exec.CommandContext(ctx, c.DiffCommand, args...) + cmd.Dir = c.tmpDir + cmd.Stdout = c.out + cmd.Stderr = c.out + return cmd.Run() +} + +func (c *Comparer) downloadFile(ctx context.Context, f fs.File, fname string) error { + if err := os.MkdirAll(filepath.Dir(fname), 0700); err != nil { + return err + } + + src, err := f.Open(ctx) + if err != nil { + return err + } + defer src.Close() //nolint:errcheck + + dst, err := os.Create(fname) + if err != nil { + return err + } + defer dst.Close() //nolint:errcheck + + _, err = io.Copy(dst, src) + return err +} + +func (c *Comparer) output(msg string, args ...interface{}) { + fmt.Fprintf(c.out, msg, args...) //nolint:errcheck +} + +// NewComparer creates a comparer for a given repository that will output the results to a given writer. +func NewComparer(rep *repo.Repository, out io.Writer) (*Comparer, error) { + tmp, err := ioutil.TempDir("", "kopia") + if err != nil { + return nil, err + } + + return &Comparer{rep: rep, out: out, tmpDir: tmp}, nil +}