rclone/cmd/bisync/checkfn.go

package bisync

import (
	"bytes"
	"context"
	"fmt"
	"strings"

	"github.com/rclone/rclone/backend/crypt"
	"github.com/rclone/rclone/cmd/bisync/bilib"
	"github.com/rclone/rclone/cmd/check"
	"github.com/rclone/rclone/fs"
	"github.com/rclone/rclone/fs/accounting"
	"github.com/rclone/rclone/fs/filter"
	"github.com/rclone/rclone/fs/hash"
	"github.com/rclone/rclone/fs/operations"
)

var hashType hash.Type
var fsrc, fdst fs.Fs
var fcrypt *crypt.Fs

// WhichCheck determines which CheckFn we should use based on the Fs types
// It is more robust and accurate than Check because
// it will fallback to CryptCheck or DownloadCheck instead of --size-only!
// it returns the *operations.CheckOpt with the CheckFn set.
func WhichCheck(ctx context.Context, opt *operations.CheckOpt) *operations.CheckOpt {
	ci := fs.GetConfig(ctx)
	common := opt.Fsrc.Hashes().Overlap(opt.Fdst.Hashes())

	// note that ci.IgnoreChecksum doesn't change the behavior of Check -- it's just a way to opt-out of cryptcheck/download
	if common.Count() > 0 || ci.SizeOnly || ci.IgnoreChecksum {
		// use normal check
		opt.Check = CheckFn
		return opt
	}

	FsrcCrypt, srcIsCrypt := opt.Fsrc.(*crypt.Fs)
	FdstCrypt, dstIsCrypt := opt.Fdst.(*crypt.Fs)

	if (srcIsCrypt && dstIsCrypt) || (!srcIsCrypt && dstIsCrypt) {
		// if both are crypt or only dst is crypt
		hashType = FdstCrypt.UnWrap().Hashes().GetOne()
		if hashType != hash.None {
			// use cryptcheck
			fsrc = opt.Fsrc
			fdst = opt.Fdst
			fcrypt = FdstCrypt
			fs.Infof(fdst, "Crypt detected! Using cryptcheck instead of check. (Use --size-only or --ignore-checksum to disable)")
			opt.Check = CryptCheckFn
			return opt
		}
	} else if srcIsCrypt && !dstIsCrypt {
		// if only src is crypt
		hashType = FsrcCrypt.UnWrap().Hashes().GetOne()
		if hashType != hash.None {
			// use reverse cryptcheck
			fsrc = opt.Fdst
			fdst = opt.Fsrc
			fcrypt = FsrcCrypt
			fs.Infof(fdst, "Crypt detected! Using cryptcheck instead of check. (Use --size-only or --ignore-checksum to disable)")
			opt.Check = ReverseCryptCheckFn
			return opt
		}
	}

	// if we've gotten this far, neither check or cryptcheck will work, so use --download
	fs.Infof(fdst, "Can't compare hashes, so using check --download for safety. (Use --size-only or --ignore-checksum to disable)")
	opt.Check = DownloadCheckFn
	return opt
}

// CheckFn is a slightly modified version of Check
func CheckFn(ctx context.Context, dst, src fs.Object) (differ bool, noHash bool, err error) {
	same, ht, err := operations.CheckHashes(ctx, src, dst)
	if err != nil {
		return true, false, err
	}
	if ht == hash.None {
		return false, true, nil
	}
	if !same {
		err = fmt.Errorf("%v differ", ht)
		fs.Errorf(src, "%v", err)
		return true, false, nil
	}
	return false, false, nil
}

// CryptCheckFn is a slightly modified version of CryptCheck
func CryptCheckFn(ctx context.Context, dst, src fs.Object) (differ bool, noHash bool, err error) {
	cryptDst := dst.(*crypt.Object)
	underlyingDst := cryptDst.UnWrap()
	underlyingHash, err := underlyingDst.Hash(ctx, hashType)
	if err != nil {
		return true, false, fmt.Errorf("error reading hash from underlying %v: %w", underlyingDst, err)
	}
	if underlyingHash == "" {
		return false, true, nil
	}
	cryptHash, err := fcrypt.ComputeHash(ctx, cryptDst, src, hashType)
	if err != nil {
		return true, false, fmt.Errorf("error computing hash: %w", err)
	}
	if cryptHash == "" {
		return false, true, nil
	}
	if cryptHash != underlyingHash {
		err = fmt.Errorf("hashes differ (%s:%s) %q vs (%s:%s) %q", fdst.Name(), fdst.Root(), cryptHash, fsrc.Name(), fsrc.Root(), underlyingHash)
		fs.Debugf(src, "%s", err.Error())
		// using same error msg as CheckFn so integration tests match
		err = fmt.Errorf("%v differ", hashType)
		fs.Errorf(src, "%s", err.Error())
		return true, false, nil
	}
	return false, false, nil
}

// ReverseCryptCheckFn is like CryptCheckFn except src and dst are switched
// result: src is crypt, dst is non-crypt
func ReverseCryptCheckFn(ctx context.Context, dst, src fs.Object) (differ bool, noHash bool, err error) {
	return CryptCheckFn(ctx, src, dst)
}

// DownloadCheckFn is a slightly modified version of Check with --download
func DownloadCheckFn(ctx context.Context, a, b fs.Object) (differ bool, noHash bool, err error) {
	differ, err = operations.CheckIdenticalDownload(ctx, a, b)
	if err != nil {
		return true, true, fmt.Errorf("failed to download: %w", err)
	}
	return differ, false, nil
}

// check potential conflicts (to avoid renaming if already identical)
func (b *bisyncRun) checkconflicts(ctxCheck context.Context, filterCheck *filter.Filter, fs1, fs2 fs.Fs) (bilib.Names, error) {
	matches := bilib.Names{}
	if filterCheck.HaveFilesFrom() {
		fs.Debugf(nil, "There are potential conflicts to check.")

		opt, close, checkopterr := check.GetCheckOpt(b.fs1, b.fs2)
		if checkopterr != nil {
			b.critical = true
			b.retryable = true
			fs.Debugf(nil, "GetCheckOpt error: %v", checkopterr)
			return matches, checkopterr
		}
		defer close()

		opt.Match = new(bytes.Buffer)

		opt = WhichCheck(ctxCheck, opt)

		fs.Infof(nil, "Checking potential conflicts...")
		check := operations.CheckFn(ctxCheck, opt)
		fs.Infof(nil, "Finished checking the potential conflicts. %s", check)

		//reset error count, because we don't want to count check errors as bisync errors
		accounting.Stats(ctxCheck).ResetErrors()

		//return the list of identical files to check against later
		if len(fmt.Sprint(opt.Match)) > 0 {
			matches = bilib.ToNames(strings.Split(fmt.Sprint(opt.Match), "\n"))
		}
		if matches.NotEmpty() {
			fs.Debugf(nil, "The following potential conflicts were determined to be identical. %v", matches)
		} else {
			fs.Debugf(nil, "None of the conflicts were determined to be identical.")
		}

	}
	return matches, nil
}

// WhichEqual is similar to WhichCheck, but checks a single object.
// Returns true if the objects are equal, false if they differ or if we don't know
func WhichEqual(ctx context.Context, src, dst fs.Object, Fsrc, Fdst fs.Fs) bool {
	opt, close, checkopterr := check.GetCheckOpt(Fsrc, Fdst)
	if checkopterr != nil {
		fs.Debugf(nil, "GetCheckOpt error: %v", checkopterr)
	}
	defer close()

	opt = WhichCheck(ctx, opt)
	differ, noHash, err := opt.Check(ctx, dst, src)
	if err != nil {
		fs.Errorf(src, "failed to check: %v", err)
		return false
	}
	if noHash {
		fs.Errorf(src, "failed to check as hash is missing")
		return false
	}
	return !differ
}

// Replaces the standard Equal func with one that also considers checksum
// Note that it also updates the modtime the same way as Sync
func (b *bisyncRun) EqualFn(ctx context.Context) context.Context {
	ci := fs.GetConfig(ctx)
	ci.CheckSum = false // force checksum off so modtime is evaluated if needed
	// modtime and size settings should already be set correctly for Equal
	var equalFn operations.EqualFn = func(ctx context.Context, src fs.ObjectInfo, dst fs.Object) bool {
		fs.Debugf(src, "evaluating...")
		equal := false
		logger, _ := operations.GetLogger(ctx)
		// temporarily unset logger, we don't want Equal to duplicate it
		noop := func(ctx context.Context, sigil operations.Sigil, src, dst fs.DirEntry, err error) {
			fs.Debugf(src, "equal skipped")
		}
		ctxNoLogger := operations.WithLogger(ctx, noop)

		timeSizeEqualFn := func() (equal bool, skipHash bool) { return operations.Equal(ctxNoLogger, src, dst), false } // normally use Equal()
		if b.opt.ResyncMode == PreferOlder || b.opt.ResyncMode == PreferLarger || b.opt.ResyncMode == PreferSmaller {
			timeSizeEqualFn = func() (equal bool, skipHash bool) { return b.resyncTimeSizeEqual(ctxNoLogger, src, dst) } // but override for --resync-mode older, larger, smaller
		}
		skipHash := false // (note that we might skip it anyway based on compare/ht settings)
		equal, skipHash = timeSizeEqualFn()
		if equal && !skipHash {
			whichHashType := func(f fs.Info) hash.Type {
				ht := getHashType(f.Name())
				if ht == hash.None && b.opt.Compare.SlowHashSyncOnly && !b.opt.Resync {
					ht = f.Hashes().GetOne()
				}
				return ht
			}
			srcHash, _ := src.Hash(ctx, whichHashType(src.Fs()))
			dstHash, _ := dst.Hash(ctx, whichHashType(dst.Fs()))
			srcHash, _ = tryDownloadHash(ctx, src, srcHash)
			dstHash, _ = tryDownloadHash(ctx, dst, dstHash)
			equal = !hashDiffers(srcHash, dstHash, whichHashType(src.Fs()), whichHashType(dst.Fs()), src.Size(), dst.Size())
		}
		if equal {
			logger(ctx, operations.Match, src, dst, nil)
			fs.Debugf(src, "EqualFn: files are equal")
			return true
		}
		logger(ctx, operations.Differ, src, dst, nil)
		fs.Debugf(src, "EqualFn: files are NOT equal")
		return false
	}
	return operations.WithEqualFn(ctx, equalFn)
}

func (b *bisyncRun) resyncTimeSizeEqual(ctxNoLogger context.Context, src fs.ObjectInfo, dst fs.Object) (equal bool, skipHash bool) {
	switch b.opt.ResyncMode {
	case PreferLarger, PreferSmaller:
		// note that arg order is path1, path2, regardless of src/dst
		path1, path2 := b.resyncWhichIsWhich(src, dst)
		if sizeDiffers(path1.Size(), path2.Size()) {
			winningPath := b.resolveLargerSmaller(path1.Size(), path2.Size(), path1.Remote(), path2.Remote(), b.opt.ResyncMode)
			// don't need to check/update modtime here, as sizes definitely differ and something will be transferred
			return b.resyncWinningPathToEqual(winningPath), b.resyncWinningPathToEqual(winningPath) // skip hash check if true
		}
		// sizes equal or don't know, so continue to checking time/hash, if applicable
		return operations.Equal(ctxNoLogger, src, dst), false // note we're back to src/dst, not path1/path2
	case PreferOlder:
		// note that arg order is path1, path2, regardless of src/dst
		path1, path2 := b.resyncWhichIsWhich(src, dst)
		if timeDiffers(ctxNoLogger, path1.ModTime(ctxNoLogger), path2.ModTime(ctxNoLogger), path1.Fs(), path2.Fs()) {
			winningPath := b.resolveNewerOlder(path1.ModTime(ctxNoLogger), path2.ModTime(ctxNoLogger), path1.Remote(), path2.Remote(), b.opt.ResyncMode)
			// if src is winner, proceed with equal to check size/hash and possibly just update dest modtime instead of transferring
			if !b.resyncWinningPathToEqual(winningPath) {
				return operations.Equal(ctxNoLogger, src, dst), false // note we're back to src/dst, not path1/path2
			}
			// if dst is winner (and definitely unequal), do not proceed further as we want dst to overwrite src regardless of size difference, and we do not want dest modtime updated
			return true, true
		}
		// times equal or don't know, so continue to checking size/hash, if applicable
	}
	return operations.Equal(ctxNoLogger, src, dst), false // note we're back to src/dst, not path1/path2
}