rclone/cmd/bisync/checkfn.go

package bisync

import (
	"bytes"
	"context"
	"fmt"
	"strings"
	"sync"

	"github.com/rclone/rclone/backend/crypt"
	"github.com/rclone/rclone/cmd/bisync/bilib"
	"github.com/rclone/rclone/cmd/check"
	"github.com/rclone/rclone/fs"
	"github.com/rclone/rclone/fs/accounting"
	"github.com/rclone/rclone/fs/filter"
	"github.com/rclone/rclone/fs/hash"
	"github.com/rclone/rclone/fs/operations"
)

type bisyncCheck = struct {
	hashType   hash.Type
	fsrc, fdst fs.Fs
	fcrypt     *crypt.Fs
}

// WhichCheck determines which CheckFn we should use based on the Fs types
// It is more robust and accurate than Check because
// it will fallback to CryptCheck or DownloadCheck instead of --size-only!
// it returns the *operations.CheckOpt with the CheckFn set.
func (b *bisyncRun) WhichCheck(ctx context.Context, opt *operations.CheckOpt) *operations.CheckOpt {
	ci := fs.GetConfig(ctx)
	common := opt.Fsrc.Hashes().Overlap(opt.Fdst.Hashes())

	// note that ci.IgnoreChecksum doesn't change the behavior of Check -- it's just a way to opt-out of cryptcheck/download
	if common.Count() > 0 || ci.SizeOnly || ci.IgnoreChecksum {
		// use normal check
		opt.Check = CheckFn
		return opt
	}

	FsrcCrypt, srcIsCrypt := opt.Fsrc.(*crypt.Fs)
	FdstCrypt, dstIsCrypt := opt.Fdst.(*crypt.Fs)

	if (srcIsCrypt && dstIsCrypt) || (!srcIsCrypt && dstIsCrypt) {
		// if both are crypt or only dst is crypt
		b.check.hashType = FdstCrypt.UnWrap().Hashes().GetOne()
		if b.check.hashType != hash.None {
			// use cryptcheck
			b.check.fsrc = opt.Fsrc
			b.check.fdst = opt.Fdst
			b.check.fcrypt = FdstCrypt
			fs.Infof(b.check.fdst, "Crypt detected! Using cryptcheck instead of check. (Use --size-only or --ignore-checksum to disable)")
			opt.Check = b.CryptCheckFn
			return opt
		}
	} else if srcIsCrypt && !dstIsCrypt {
		// if only src is crypt
		b.check.hashType = FsrcCrypt.UnWrap().Hashes().GetOne()
		if b.check.hashType != hash.None {
			// use reverse cryptcheck
			b.check.fsrc = opt.Fdst
			b.check.fdst = opt.Fsrc
			b.check.fcrypt = FsrcCrypt
			fs.Infof(b.check.fdst, "Crypt detected! Using cryptcheck instead of check. (Use --size-only or --ignore-checksum to disable)")
			opt.Check = b.ReverseCryptCheckFn
			return opt
		}
	}

	// if we've gotten this far, neither check or cryptcheck will work, so use --download
	fs.Infof(b.check.fdst, "Can't compare hashes, so using check --download for safety. (Use --size-only or --ignore-checksum to disable)")
	opt.Check = DownloadCheckFn
	return opt
}

// CheckFn is a slightly modified version of Check
func CheckFn(ctx context.Context, dst, src fs.Object) (differ bool, noHash bool, err error) {
	same, ht, err := operations.CheckHashes(ctx, src, dst)
	if err != nil {
		return true, false, err
	}
	if ht == hash.None {
		return missingHashFn(ctx, dst, src)
	}
	if !same {
		err = fmt.Errorf("%v differ", ht)
		fs.Errorf(src, "%v", err)
		return true, false, nil
	}
	return false, false, nil
}

// CryptCheckFn is a slightly modified version of CryptCheck
func (b *bisyncRun) CryptCheckFn(ctx context.Context, dst, src fs.Object) (differ bool, noHash bool, err error) {
	cryptDst := dst.(*crypt.Object)
	underlyingDst := cryptDst.UnWrap()
	underlyingHash, err := underlyingDst.Hash(ctx, b.check.hashType)
	if err != nil {
		return true, false, fmt.Errorf("error reading hash from underlying %v: %w", underlyingDst, err)
	}
	if underlyingHash == "" {
		return missingHashFn(ctx, dst, src)
	}
	cryptHash, err := b.check.fcrypt.ComputeHash(ctx, cryptDst, src, b.check.hashType)
	if err != nil {
		return true, false, fmt.Errorf("error computing hash: %w", err)
	}
	if cryptHash == "" {
		return missingHashFn(ctx, dst, src)
	}
	if cryptHash != underlyingHash {
		err = fmt.Errorf("hashes differ (%s:%s) %q vs (%s:%s) %q", b.check.fdst.Name(), b.check.fdst.Root(), cryptHash, b.check.fsrc.Name(), b.check.fsrc.Root(), underlyingHash)
		fs.Debugf(src, "%s", err.Error())
		// using same error msg as CheckFn so integration tests match
		err = fmt.Errorf("%v differ", b.check.hashType)
		fs.Errorf(src, "%s", err.Error())
		return true, false, nil
	}
	return false, false, nil
}

// ReverseCryptCheckFn is like CryptCheckFn except src and dst are switched
// result: src is crypt, dst is non-crypt
func (b *bisyncRun) ReverseCryptCheckFn(ctx context.Context, dst, src fs.Object) (differ bool, noHash bool, err error) {
	return b.CryptCheckFn(ctx, src, dst)
}

func missingHashFn(ctx context.Context, dst, src fs.Object) (differ bool, noHash bool, err error) {
	ci := fs.GetConfig(ctx)
	if ci.SizeOnly || ci.IgnoreChecksum {
		return false, true, nil
	}
	fs.Debugf(src, "hash is missing, so using --download")
	return DownloadCheckFn(ctx, dst, src)
}

// DownloadCheckFn is a slightly modified version of Check with --download
func DownloadCheckFn(ctx context.Context, dst, src fs.Object) (equal bool, noHash bool, err error) {
	equal, err = operations.CheckIdenticalDownload(ctx, src, dst)
	if err != nil {
		return true, true, fmt.Errorf("failed to download: %w", err)
	}
	return !equal, false, nil
}

// check potential conflicts (to avoid renaming if already identical)
func (b *bisyncRun) checkconflicts(ctxCheck context.Context, filterCheck *filter.Filter, fs1, fs2 fs.Fs) (matches bilib.Names, matches1, matches2 *fileList, err error) {
	matches = bilib.Names{}
	matches1, matches2 = newFileList(), newFileList()
	if filterCheck.HaveFilesFrom() {
		fs.Debugf(nil, "There are potential conflicts to check.")
		if b.opt.TestFnConflictCheck != nil {
			b.opt.TestFnConflictCheck()
		}

		// the sizes already match, and they are all a check could compare here
		ci := fs.GetConfig(ctxCheck)
		if ci.SizeOnly || (ci.IgnoreChecksum && fs1.Hashes().Overlap(fs2.Hashes()).Count() == 0) {
			fs.Infof(nil, "Not checking potential conflicts, as only their sizes could be compared, and those already match.")
			for file := range filterCheck.Files() {
				matches.Add(file)
				if b.march.ls1.has(file) {
					b.march.ls1.getPut(file, matches1)
				}
				if b.march.ls2.has(file) {
					b.march.ls2.getPut(file, matches2)
				}
			}
			return matches, matches1, matches2, nil
		}

		opt, close, checkopterr := check.GetCheckOpt(fs1, fs2)
		if checkopterr != nil {
			b.critical = true
			b.retryable = true
			fs.Debugf(nil, "GetCheckOpt error: %v", checkopterr)
			return matches, matches1, matches2, checkopterr
		}
		defer close()

		opt.Match = new(bytes.Buffer)

		opt = b.WhichCheck(ctxCheck, opt)

		// record the objects found identical, so they can be compared with the listing snapshot
		var mu sync.Mutex
		checkFn := opt.Check
		opt.Check = func(ctx context.Context, dst, src fs.Object) (differ bool, noHash bool, err error) {
			differ, noHash, err = checkFn(ctx, dst, src)
			if err == nil && !differ {
				f1 := b.comparedFields(ctx, src, b.march.ls1.hash)
				f2 := b.comparedFields(ctx, dst, b.march.ls2.hash)
				mu.Lock()
				matches1.put(src.Remote(), f1.size, f1.time, f1.hash, "", "-")
				matches2.put(dst.Remote(), f2.size, f2.time, f2.hash, "", "-")
				mu.Unlock()
			}
			return differ, noHash, err
		}

		fs.Infof(nil, "Checking potential conflicts...")
		check := operations.CheckFn(ctxCheck, opt)
		fs.Infof(nil, "Finished checking the potential conflicts. %s", check)

		// reset error count, because we don't want to count check errors as bisync errors
		accounting.Stats(ctxCheck).ResetErrors()

		// return the list of identical files to check against later
		if len(fmt.Sprint(opt.Match)) > 0 {
			matches = bilib.ToNames(strings.Split(fmt.Sprint(opt.Match), "\n"))
		}
		if matches.NotEmpty() {
			fs.Debugf(nil, "The following potential conflicts were determined to be identical. %v", matches)
		} else {
			fs.Debugf(nil, "None of the conflicts were determined to be identical.")
		}
	}
	return matches, matches1, matches2, nil
}

// comparedFields returns the fields of o that bisync compares, the same way
// the listing records them.
func (b *bisyncRun) comparedFields(ctx context.Context, o fs.Object, hashType hash.Type) (fi fileInfo) {
	fi.size = o.Size()
	if b.opt.Compare.Modtime {
		fi.time = o.ModTime(ctx).In(TZ)
	}
	if b.opt.Compare.Checksum && hashType != hash.None {
		fi.hash, _ = o.Hash(ctx, hashType)
		fi.hash, _ = b.tryDownloadHash(ctx, o, fi.hash)
	}
	return fi
}

// WhichEqual is similar to WhichCheck, but checks a single object.
// Returns true if the objects are equal, false if they differ or if we don't know
func (b *bisyncRun) WhichEqual(ctx context.Context, src, dst fs.Object, Fsrc, Fdst fs.Fs) bool {
	opt, close, checkopterr := check.GetCheckOpt(Fsrc, Fdst)
	if checkopterr != nil {
		fs.Debugf(nil, "GetCheckOpt error: %v", checkopterr)
	}
	defer close()

	opt = b.WhichCheck(ctx, opt)
	differ, noHash, err := opt.Check(ctx, dst, src)
	if err != nil {
		fs.Errorf(src, "failed to check: %v", err)
		return false
	}
	if noHash {
		fs.Errorf(src, "failed to check as hash is missing")
		return false
	}
	return !differ
}

// Replaces the standard Equal func with one that also considers checksum
// Note that it also updates the modtime the same way as Sync
func (b *bisyncRun) EqualFn(ctx context.Context) context.Context {
	ci := fs.GetConfig(ctx)
	ci.CheckSum = false // force checksum off so modtime is evaluated if needed
	// modtime and size settings should already be set correctly for Equal
	var equalFn operations.EqualFn = func(ctx context.Context, src fs.ObjectInfo, dst fs.Object) bool {
		fs.Debugf(src, "evaluating...")
		equal := false
		logger, _ := operations.GetLogger(ctx)
		// temporarily unset logger, we don't want Equal to duplicate it
		noop := func(ctx context.Context, sigil operations.Sigil, src, dst fs.DirEntry, err error) {
			fs.Debugf(src, "equal skipped")
		}
		ctxNoLogger := operations.WithLogger(ctx, noop)

		timeSizeEqualFn := func() (equal bool, skipHash bool) { return operations.Equal(ctxNoLogger, src, dst), false } // normally use Equal()
		if b.opt.ResyncMode == PreferOlder || b.opt.ResyncMode == PreferLarger || b.opt.ResyncMode == PreferSmaller {
			timeSizeEqualFn = func() (equal bool, skipHash bool) { return b.resyncTimeSizeEqual(ctxNoLogger, src, dst) } // but override for --resync-mode older, larger, smaller
		}
		skipHash := false // (note that we might skip it anyway based on compare/ht settings)
		equal, skipHash = timeSizeEqualFn()
		if equal && !skipHash {
			whichHashType := func(f fs.Info) hash.Type {
				ht := b.getHashType(f.Name())
				if ht == hash.None && b.opt.Compare.SlowHashSyncOnly && !b.opt.Resync {
					ht = f.Hashes().GetOne()
				}
				return ht
			}
			srcHash, _ := src.Hash(ctx, whichHashType(src.Fs()))
			dstHash, _ := dst.Hash(ctx, whichHashType(dst.Fs()))
			srcHash, _ = b.tryDownloadHash(ctx, src, srcHash)
			dstHash, _ = b.tryDownloadHash(ctx, dst, dstHash)
			equal = !b.hashDiffers(srcHash, dstHash, whichHashType(src.Fs()), whichHashType(dst.Fs()), src.Size(), dst.Size())
		}
		if equal {
			logger(ctx, operations.Match, src, dst, nil)
			fs.Debugf(src, "EqualFn: files are equal")
			return true
		}
		logger(ctx, operations.Differ, src, dst, nil)
		fs.Debugf(src, "EqualFn: files are NOT equal")
		return false
	}
	return operations.WithEqualFn(ctx, equalFn)
}

func (b *bisyncRun) resyncTimeSizeEqual(ctxNoLogger context.Context, src fs.ObjectInfo, dst fs.Object) (equal bool, skipHash bool) {
	switch b.opt.ResyncMode {
	case PreferLarger, PreferSmaller:
		// note that arg order is path1, path2, regardless of src/dst
		path1, path2 := b.resyncWhichIsWhich(src, dst)
		if sizeDiffers(path1.Size(), path2.Size()) {
			winningPath := b.resolveLargerSmaller(path1.Size(), path2.Size(), path1.Remote(), b.opt.ResyncMode)
			// don't need to check/update modtime here, as sizes definitely differ and something will be transferred
			return b.resyncWinningPathToEqual(winningPath), b.resyncWinningPathToEqual(winningPath) // skip hash check if true
		}
		// sizes equal or don't know, so continue to checking time/hash, if applicable
		return operations.Equal(ctxNoLogger, src, dst), false // note we're back to src/dst, not path1/path2
	case PreferOlder:
		// note that arg order is path1, path2, regardless of src/dst
		path1, path2 := b.resyncWhichIsWhich(src, dst)
		if timeDiffers(ctxNoLogger, path1.ModTime(ctxNoLogger), path2.ModTime(ctxNoLogger), path1.Fs(), path2.Fs()) {
			winningPath := b.resolveNewerOlder(path1.ModTime(ctxNoLogger), path2.ModTime(ctxNoLogger), path1.Remote(), b.opt.ResyncMode)
			// if src is winner, proceed with equal to check size/hash and possibly just update dest modtime instead of transferring
			if !b.resyncWinningPathToEqual(winningPath) {
				return operations.Equal(ctxNoLogger, src, dst), false // note we're back to src/dst, not path1/path2
			}
			// if dst is winner (and definitely unequal), do not proceed further as we want dst to overwrite src regardless of size difference, and we do not want dest modtime updated
			return true, true
		}
		// times equal or don't know, so continue to checking size/hash, if applicable
	}
	return operations.Equal(ctxNoLogger, src, dst), false // note we're back to src/dst, not path1/path2
}