github-mirror/rclone
2
0
Fork 0
mirror of https://github.com/rclone/rclone.git synced 2024-11-22 12:05:05 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

check,cryptcheck: add reporting of filenames for same/missing/changed #3264

Browse Source 
See: https://forum.rclone.org/t/rclone-check-v-doesnt-show-once-per-minute-update-counts/17402
This commit is contained in:
Nick Craig-Wood 2020-03-09 10:54:41 +00:00
parent d2efb4b29b
commit 8b6f2bbb4b
4 changed files with 356 additions and 121 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

127
cmd/check/check.go
View File

@ -2,24 +2,128 @@ package check
import (
"context"
"io"
"os"
"strings"
"github.com/rclone/rclone/cmd"
"github.com/rclone/rclone/fs"
"github.com/rclone/rclone/fs/config/flags"
"github.com/rclone/rclone/fs/operations"
"github.com/spf13/cobra"
"github.com/spf13/pflag"
)
// Globals
var (
download = false
oneway = false
download = false
oneway = false
combined = ""
missingOnSrc = ""
missingOnDst = ""
match = ""
differ = ""
errFile = ""
)
func init() {
cmd.Root.AddCommand(commandDefinition)
cmdFlags := commandDefinition.Flags()
flags.BoolVarP(cmdFlags, &download, "download", "", download, "Check by downloading rather than with hash.")
AddFlags(cmdFlags)
}
// AddFlags adds the check flags to the cmdFlags command
func AddFlags(cmdFlags *pflag.FlagSet) {
flags.BoolVarP(cmdFlags, &oneway, "one-way", "", oneway, "Check one way only, source files must exist on remote")
flags.StringVarP(cmdFlags, &combined, "combined", "", combined, "Make a combined report of changes to this file")
flags.StringVarP(cmdFlags, &missingOnSrc, "missing-on-src", "", missingOnSrc, "Report all files missing from the source to this file")
flags.StringVarP(cmdFlags, &missingOnDst, "missing-on-dst", "", missingOnDst, "Report all files missing from the destination to this file")
flags.StringVarP(cmdFlags, &match, "match", "", match, "Report all matching files to this file")
flags.StringVarP(cmdFlags, &differ, "differ", "", differ, "Report all non-matching files to this file")
flags.StringVarP(cmdFlags, &errFile, "error", "", errFile, "Report all files with errors (hashing or reading) to this file")
}
// FlagsHelp describes the flags for the help
var FlagsHelp = strings.Replace(`
If you supply the |--one-way| flag, it will only check that files in
the source match the files in the destination, not the other way
around. This means that extra files in the destination that are not in
the source will not be detected.
The |--differ|, |--missing-on-dst|, |--missing-on-src|, |--src-only|
and |--error| flags write paths, one per line, to the file name (or
stdout if it is |-|) supplied. What they write is described in the
help below. For example |--differ| will write all paths which are
present on both the source and destination but different.
The |--combined| flag will write a file (or stdout) which contains all
file paths with a symbol and then a space and then the path to tell
you what happened to it. These are reminiscent of diff files.
- |= path| means path was found in source and destination and was identical
- |- path| means path was missing on the source, so only in the destination
- |+ path| means path was missing on the destination, so only in the source
- |* path| means path was present in source and destination but different.
- |! path| means there was an error reading or hashing the source or dest.
`, "|", "`", -1)
// GetCheckOpt gets the options corresponding to the check flags
func GetCheckOpt(fsrc, fdst fs.Fs) (opt *operations.CheckOpt, close func(), err error) {
closers := []io.Closer{}
opt = &operations.CheckOpt{
Fsrc: fsrc,
Fdst: fdst,
OneWay: oneway,
}
open := func(name string, pout *io.Writer) error {
if name == "" {
return nil
}
if name == "-" {
*pout = os.Stdout
return nil
}
out, err := os.Create(name)
if err != nil {
return err
}
*pout = out
closers = append(closers, out)
return nil
}
if err = open(combined, &opt.Combined); err != nil {
return nil, nil, err
}
if err = open(missingOnSrc, &opt.MissingOnSrc); err != nil {
return nil, nil, err
}
if err = open(missingOnDst, &opt.MissingOnDst); err != nil {
return nil, nil, err
}
if err = open(match, &opt.Match); err != nil {
return nil, nil, err
}
if err = open(differ, &opt.Differ); err != nil {
return nil, nil, err
}
if err = open(errFile, &opt.Error); err != nil {
return nil, nil, err
}
close = func() {
for _, closer := range closers {
err := closer.Close()
if err != nil {
fs.Errorf(nil, "Failed to close report output: %v", err)
}
}
}
return opt, close, nil
}
var commandDefinition = &cobra.Command{
@ -37,19 +141,20 @@ If you supply the --download flag, it will download the data from
both remotes and check them against each other on the fly. This can
be useful for remotes that don't support hashes or if you really want
to check all the data.
If you supply the --one-way flag, it will only check that files in source
match the files in destination, not the other way around. Meaning extra files in
destination that are not in the source will not trigger an error.
`,
` + FlagsHelp,
Run: func(command *cobra.Command, args []string) {
cmd.CheckArgs(2, 2, command, args)
fsrc, fdst := cmd.NewFsSrcDst(args)
cmd.Run(false, true, command, func() error {
if download {
return operations.CheckDownload(context.Background(), fdst, fsrc, oneway)
opt, close, err := GetCheckOpt(fsrc, fdst)
if err != nil {
return err
}
return operations.Check(context.Background(), fdst, fsrc, oneway)
defer close()
if download {
return operations.CheckDownload(context.Background(), opt)
}
return operations.Check(context.Background(), opt)
})
},
}

42
cmd/cryptcheck/cryptcheck.go
View File

@ -6,22 +6,17 @@ import (
"github.com/pkg/errors"
"github.com/rclone/rclone/backend/crypt"
"github.com/rclone/rclone/cmd"
"github.com/rclone/rclone/cmd/check"
"github.com/rclone/rclone/fs"
"github.com/rclone/rclone/fs/config/flags"
"github.com/rclone/rclone/fs/hash"
"github.com/rclone/rclone/fs/operations"
"github.com/spf13/cobra"
)
// Globals
var (
oneway = false
)
func init() {
cmd.Root.AddCommand(commandDefinition)
cmdFlag := commandDefinition.Flags()
flags.BoolVarP(cmdFlag, &oneway, "one-way", "", oneway, "Check one way only, source files must exist on destination")
check.AddFlags(cmdFlag)
}
var commandDefinition = &cobra.Command{
@ -50,11 +45,7 @@ the files in remote:path.
rclone cryptcheck remote:path encryptedremote:path
After it has run it will log the status of the encryptedremote:.
If you supply the --one-way flag, it will only check that files in source
match the files in destination, not the other way around. Meaning extra files in
destination that are not in the source will not trigger an error.
`,
` + check.FlagsHelp,
Run: func(command *cobra.Command, args []string) {
cmd.CheckArgs(2, 2, command, args)
fsrc, fdst := cmd.NewFsSrcDst(args)
@ -79,39 +70,40 @@ func cryptCheck(ctx context.Context, fdst, fsrc fs.Fs) error {
}
fs.Infof(nil, "Using %v for hash comparisons", hashType)
opt, close, err := check.GetCheckOpt(fsrc, fcrypt)
if err != nil {
return err
}
defer close()
// checkIdentical checks to see if dst and src are identical
//
// it returns true if differences were found
// it also returns whether it couldn't be hashed
checkIdentical := func(ctx context.Context, dst, src fs.Object) (differ bool, noHash bool) {
opt.Check = func(ctx context.Context, dst, src fs.Object) (differ bool, noHash bool, err error) {
cryptDst := dst.(*crypt.Object)
underlyingDst := cryptDst.UnWrap()
underlyingHash, err := underlyingDst.Hash(ctx, hashType)
if err != nil {
err = fs.CountError(err)
fs.Errorf(dst, "Error reading hash from underlying %v: %v", underlyingDst, err)
return true, false
return true, false, errors.Wrapf(err, "error reading hash from underlying %v", underlyingDst)
}
if underlyingHash == "" {
return false, true
return false, true, nil
}
cryptHash, err := fcrypt.ComputeHash(ctx, cryptDst, src, hashType)
if err != nil {
err = fs.CountError(err)
fs.Errorf(dst, "Error computing hash: %v", err)
return true, false
return true, false, errors.Wrap(err, "error computing hash")
}
if cryptHash == "" {
return false, true
return false, true, nil
}
if cryptHash != underlyingHash {
err = errors.Errorf("hashes differ (%s:%s) %q vs (%s:%s) %q", fdst.Name(), fdst.Root(), cryptHash, fsrc.Name(), fsrc.Root(), underlyingHash)
err = fs.CountError(err)
fs.Errorf(src, err.Error())
return true, false
return true, false, nil
}
return false, false
return false, false, nil
}
return operations.CheckFn(ctx, fcrypt, fsrc, checkIdentical, oneway)
return operations.CheckFn(ctx, opt)
}

135
fs/operations/operations.go
View File

@ -730,57 +730,86 @@ func SameDir(fdst, fsrc fs.Info) bool {
//
// it returns true if differences were found
// it also returns whether it couldn't be hashed
func checkIdentical(ctx context.Context, dst, src fs.Object) (differ bool, noHash bool) {
func checkIdentical(ctx context.Context, dst, src fs.Object) (differ bool, noHash bool, err error) {
same, ht, err := CheckHashes(ctx, src, dst)
if err != nil {
// CheckHashes will log and count errors
return true, false
return true, false, err
}
if ht == hash.None {
return false, true
return false, true, nil
}
if !same {
err = errors.Errorf("%v differ", ht)
fs.Errorf(src, "%v", err)
_ = fs.CountError(err)
return true, false
return true, false, nil
}
return false, false
return false, false, nil
}
// checkFn is the type of the checking function used in CheckFn()
type checkFn func(ctx context.Context, a, b fs.Object) (differ bool, noHash bool)
//
// It should check the two objects (a, b) and return if they differ
// and whether the hash was used.
type checkFn func(ctx context.Context, a, b fs.Object) (differ bool, noHash bool, err error)
// checkMarch is used to march over two Fses in the same way as
// sync/copy
type checkMarch struct {
fdst, fsrc fs.Fs
check checkFn
ioMu sync.Mutex
wg sync.WaitGroup
tokens chan struct{}
oneway bool
differences int32
noHashes int32
srcFilesMissing int32
dstFilesMissing int32
matches int32
opt CheckOpt
}
// CheckOpt contains options for the Check functions
type CheckOpt struct {
Fdst, Fsrc fs.Fs // fses to check
Check checkFn // function to use for checking
OneWay bool // one way only?
Combined io.Writer // a file with file names with leading sigils
MissingOnSrc io.Writer // files only in the destination
MissingOnDst io.Writer // files only in the source
Match io.Writer // matching files
Differ io.Writer // differing files
Error io.Writer // files with errors of some kind
}
// report outputs the fileName to out if required and to the combined log
func (c *checkMarch) report(o fs.DirEntry, out io.Writer, sigil rune) {
if out != nil {
c.ioMu.Lock()
_, _ = fmt.Fprintf(out, "%v\n", o)
c.ioMu.Unlock()
}
if c.opt.Combined != nil {
c.ioMu.Lock()
_, _ = fmt.Fprintf(c.opt.Combined, "%c %v\n", sigil, o)
c.ioMu.Unlock()
}
}
// DstOnly have an object which is in the destination only
func (c *checkMarch) DstOnly(dst fs.DirEntry) (recurse bool) {
switch dst.(type) {
case fs.Object:
if c.oneway {
if c.opt.OneWay {
return false
}
err := errors.Errorf("File not in %v", c.fsrc)
err := errors.Errorf("File not in %v", c.opt.Fsrc)
fs.Errorf(dst, "%v", err)
_ = fs.CountError(err)
atomic.AddInt32(&c.differences, 1)
atomic.AddInt32(&c.srcFilesMissing, 1)
c.report(dst, c.opt.MissingOnSrc, '-')
case fs.Directory:
// Do the same thing to the entire contents of the directory
if c.oneway {
if c.opt.OneWay {
return false
}
return true
@ -794,11 +823,12 @@ func (c *checkMarch) DstOnly(dst fs.DirEntry) (recurse bool) {
func (c *checkMarch) SrcOnly(src fs.DirEntry) (recurse bool) {
switch src.(type) {
case fs.Object:
err := errors.Errorf("File not in %v", c.fdst)
err := errors.Errorf("File not in %v", c.opt.Fdst)
fs.Errorf(src, "%v", err)
_ = fs.CountError(err)
atomic.AddInt32(&c.differences, 1)
atomic.AddInt32(&c.dstFilesMissing, 1)
c.report(src, c.opt.MissingOnDst, '+')
case fs.Directory:
// Do the same thing to the entire contents of the directory
return true
@ -809,8 +839,7 @@ func (c *checkMarch) SrcOnly(src fs.DirEntry) (recurse bool) {
}
// check to see if two objects are identical using the check function
func (c *checkMarch) checkIdentical(ctx context.Context, dst, src fs.Object) (differ bool, noHash bool) {
var err error
func (c *checkMarch) checkIdentical(ctx context.Context, dst, src fs.Object) (differ bool, noHash bool, err error) {
tr := accounting.Stats(ctx).NewCheckingTransfer(src)
defer func() {
tr.Done(err)
@ -818,12 +847,12 @@ func (c *checkMarch) checkIdentical(ctx context.Context, dst, src fs.Object) (di
if sizeDiffers(src, dst) {
err = errors.Errorf("Sizes differ")
fs.Errorf(src, "%v", err)
return true, false
return true, false, nil
}
if fs.Config.SizeOnly {
return false, false
return false, false, nil
}
return c.check(ctx, dst, src)
return c.opt.Check(ctx, dst, src)
}
// Match is called when src and dst are present, so sync src to dst
@ -842,11 +871,20 @@ func (c *checkMarch) Match(ctx context.Context, dst, src fs.DirEntry) (recurse b
<-c.tokens // get the token back to free up a slot
c.wg.Done()
}()
differ, noHash := c.checkIdentical(ctx, dstX, srcX)
if differ {
differ, noHash, err := c.checkIdentical(ctx, dstX, srcX)
if err != nil {
fs.Errorf(src, "%v", err)
_ = fs.CountError(err)
c.report(src, c.opt.Error, '!')
} else if differ {
atomic.AddInt32(&c.differences, 1)
err := errors.New("files differ")
fs.Errorf(src, "%v", err)
_ = fs.CountError(err)
c.report(src, c.opt.Differ, '*')
} else {
atomic.AddInt32(&c.matches, 1)
c.report(src, c.opt.Match, '=')
if noHash {
atomic.AddInt32(&c.noHashes, 1)
fs.Debugf(dstX, "OK - could not check hash")
@ -856,11 +894,12 @@ func (c *checkMarch) Match(ctx context.Context, dst, src fs.DirEntry) (recurse b
}
}()
} else {
err := errors.Errorf("is file on %v but directory on %v", c.fsrc, c.fdst)
err := errors.Errorf("is file on %v but directory on %v", c.opt.Fsrc, c.opt.Fdst)
fs.Errorf(src, "%v", err)
_ = fs.CountError(err)
atomic.AddInt32(&c.differences, 1)
atomic.AddInt32(&c.dstFilesMissing, 1)
c.report(src, c.opt.MissingOnDst, '+')
}
case fs.Directory:
// Do the same thing to the entire contents of the directory
@ -868,11 +907,12 @@ func (c *checkMarch) Match(ctx context.Context, dst, src fs.DirEntry) (recurse b
if ok {
return true
}
err := errors.Errorf("is file on %v but directory on %v", c.fdst, c.fsrc)
err := errors.Errorf("is file on %v but directory on %v", c.opt.Fdst, c.opt.Fsrc)
fs.Errorf(dst, "%v", err)
_ = fs.CountError(err)
atomic.AddInt32(&c.differences, 1)
atomic.AddInt32(&c.srcFilesMissing, 1)
c.report(dst, c.opt.MissingOnSrc, '-')
default:
panic("Bad object in DirEntries")
@ -887,43 +927,43 @@ func (c *checkMarch) Match(ctx context.Context, dst, src fs.DirEntry) (recurse b
//
// it returns true if differences were found
// it also returns whether it couldn't be hashed
func CheckFn(ctx context.Context, fdst, fsrc fs.Fs, check checkFn, oneway bool) error {
func CheckFn(ctx context.Context, opt *CheckOpt) error {
if opt.Check == nil {
return errors.New("internal error: nil check function")
}
c := &checkMarch{
fdst: fdst,
fsrc: fsrc,
check: check,
oneway: oneway,
tokens: make(chan struct{}, fs.Config.Checkers),
opt: *opt,
}
// set up a march over fdst and fsrc
m := &march.March{
Ctx: ctx,
Fdst: fdst,
Fsrc: fsrc,
Fdst: c.opt.Fdst,
Fsrc: c.opt.Fsrc,
Dir: "",
Callback: c,
}
fs.Debugf(fdst, "Waiting for checks to finish")
fs.Debugf(c.opt.Fdst, "Waiting for checks to finish")
err := m.Run()
c.wg.Wait() // wait for background go-routines
if c.dstFilesMissing > 0 {
fs.Logf(fdst, "%d files missing", c.dstFilesMissing)
fs.Logf(c.opt.Fdst, "%d files missing", c.dstFilesMissing)
}
if c.srcFilesMissing > 0 {
fs.Logf(fsrc, "%d files missing", c.srcFilesMissing)
fs.Logf(c.opt.Fsrc, "%d files missing", c.srcFilesMissing)
}
fs.Logf(fdst, "%d differences found", c.differences)
fs.Logf(c.opt.Fdst, "%d differences found", accounting.Stats(ctx).GetErrors())
if errs := accounting.Stats(ctx).GetErrors(); errs > 0 {
fs.Logf(fdst, "%d errors while checking", errs)
fs.Logf(c.opt.Fdst, "%d errors while checking", errs)
}
if c.noHashes > 0 {
fs.Logf(fdst, "%d hashes could not be checked", c.noHashes)
fs.Logf(c.opt.Fdst, "%d hashes could not be checked", c.noHashes)
}
if c.matches > 0 {
fs.Logf(fdst, "%d matching files", c.matches)
fs.Logf(c.opt.Fdst, "%d matching files", c.matches)
}
if c.differences > 0 {
return errors.Errorf("%d differences found", c.differences)
@ -932,8 +972,10 @@ func CheckFn(ctx context.Context, fdst, fsrc fs.Fs, check checkFn, oneway bool)
}
// Check the files in fsrc and fdst according to Size and hash
func Check(ctx context.Context, fdst, fsrc fs.Fs, oneway bool) error {
return CheckFn(ctx, fdst, fsrc, checkIdentical, oneway)
func Check(ctx context.Context, opt *CheckOpt) error {
optCopy := *opt
optCopy.Check = checkIdentical
return CheckFn(ctx, &optCopy)
}
// CheckEqualReaders checks to see if in1 and in2 have the same
@ -1025,17 +1067,16 @@ func checkIdenticalDownload(ctx context.Context, dst, src fs.Object) (differ boo
// CheckDownload checks the files in fsrc and fdst according to Size
// and the actual contents of the files.
func CheckDownload(ctx context.Context, fdst, fsrc fs.Fs, oneway bool) error {
check := func(ctx context.Context, a, b fs.Object) (differ bool, noHash bool) {
differ, err := CheckIdenticalDownload(ctx, a, b)
func CheckDownload(ctx context.Context, opt *CheckOpt) error {
optCopy := *opt
optCopy.Check = func(ctx context.Context, a, b fs.Object) (differ bool, noHash bool, err error) {
differ, err = CheckIdenticalDownload(ctx, a, b)
if err != nil {
err = fs.CountError(err)
fs.Errorf(a, "Failed to download: %v", err)
return true, true
return true, true, errors.Wrap(err, "failed to download")
}
return differ, false
return differ, false, nil
}
return CheckFn(ctx, fdst, fsrc, check, oneway)
return CheckFn(ctx, &optCopy)
}
// ListFn lists the Fs to the supplied function

173
fs/operations/operations_test.go
View File

@ -31,6 +31,7 @@ import (
"net/http/httptest"
"os"
"regexp"
"sort"
"strings"
"testing"
"time"
@ -354,51 +355,113 @@ func TestRetry(t *testing.T) {
}
func testCheck(t *testing.T, checkFunction func(ctx context.Context, fdst, fsrc fs.Fs, oneway bool) error) {
func testCheck(t *testing.T, checkFunction func(ctx context.Context, opt *operations.CheckOpt) error) {
r := fstest.NewRun(t)
defer r.Finalise()
check := func(i int, wantErrors int64, wantChecks int64, oneway bool) {
fs.Debugf(r.Fremote, "%d: Starting check test", i)
accounting.GlobalStats().ResetCounters()
var buf bytes.Buffer
log.SetOutput(&buf)
defer func() {
log.SetOutput(os.Stderr)
}()
err := checkFunction(context.Background(), r.Fremote, r.Flocal, oneway)
gotErrors := accounting.GlobalStats().GetErrors()
gotChecks := accounting.GlobalStats().GetChecks()
if wantErrors == 0 && err != nil {
t.Errorf("%d: Got error when not expecting one: %v", i, err)
addBuffers := func(opt *operations.CheckOpt) {
opt.Combined = new(bytes.Buffer)
opt.MissingOnSrc = new(bytes.Buffer)
opt.MissingOnDst = new(bytes.Buffer)
opt.Match = new(bytes.Buffer)
opt.Differ = new(bytes.Buffer)
opt.Error = new(bytes.Buffer)
}
sortLines := func(in string) []string {
if in == "" {
return []string{}
}
if wantErrors != 0 && err == nil {
t.Errorf("%d: No error when expecting one", i)
}
if wantErrors != gotErrors {
t.Errorf("%d: Expecting %d errors but got %d", i, wantErrors, gotErrors)
}
if gotChecks > 0 && !strings.Contains(buf.String(), "matching files") {
t.Errorf("%d: Total files matching line missing", i)
}
if wantChecks != gotChecks {
t.Errorf("%d: Expecting %d total matching files but got %d", i, wantChecks, gotChecks)
}
fs.Debugf(r.Fremote, "%d: Ending check test", i)
lines := strings.Split(in, "\n")
sort.Strings(lines)
return lines
}
checkBuffer := func(name string, want map[string]string, out io.Writer) {
expected := want[name]
buf, ok := out.(*bytes.Buffer)
require.True(t, ok)
assert.Equal(t, sortLines(expected), sortLines(buf.String()), name)
}
checkBuffers := func(opt *operations.CheckOpt, want map[string]string) {
checkBuffer("combined", want, opt.Combined)
checkBuffer("missingonsrc", want, opt.MissingOnSrc)
checkBuffer("missingondst", want, opt.MissingOnDst)
checkBuffer("match", want, opt.Match)
checkBuffer("differ", want, opt.Differ)
checkBuffer("error", want, opt.Error)
}
check := func(i int, wantErrors int64, wantChecks int64, oneway bool, wantOutput map[string]string) {
t.Run(fmt.Sprintf("%d", i), func(t *testing.T) {
accounting.GlobalStats().ResetCounters()
var buf bytes.Buffer
log.SetOutput(&buf)
defer func() {
log.SetOutput(os.Stderr)
}()
opt := operations.CheckOpt{
Fdst: r.Fremote,
Fsrc: r.Flocal,
OneWay: oneway,
}
addBuffers(&opt)
err := checkFunction(context.Background(), &opt)
gotErrors := accounting.GlobalStats().GetErrors()
gotChecks := accounting.GlobalStats().GetChecks()
if wantErrors == 0 && err != nil {
t.Errorf("%d: Got error when not expecting one: %v", i, err)
}
if wantErrors != 0 && err == nil {
t.Errorf("%d: No error when expecting one", i)
}
if wantErrors != gotErrors {
t.Errorf("%d: Expecting %d errors but got %d", i, wantErrors, gotErrors)
}
if gotChecks > 0 && !strings.Contains(buf.String(), "matching files") {
t.Errorf("%d: Total files matching line missing", i)
}
if wantChecks != gotChecks {
t.Errorf("%d: Expecting %d total matching files but got %d", i, wantChecks, gotChecks)
}
checkBuffers(&opt, wantOutput)
})
}
file1 := r.WriteBoth(context.Background(), "rutabaga", "is tasty", t3)
fstest.CheckItems(t, r.Fremote, file1)
fstest.CheckItems(t, r.Flocal, file1)
check(1, 0, 1, false)
check(1, 0, 1, false, map[string]string{
"combined": "= rutabaga\n",
"missingonsrc": "",
"missingondst": "",
"match": "rutabaga\n",
"differ": "",
"error": "",
})
file2 := r.WriteFile("potato2", "------------------------------------------------------------", t1)
fstest.CheckItems(t, r.Flocal, file1, file2)
check(2, 1, 1, false)
check(2, 1, 1, false, map[string]string{
"combined": "+ potato2\n= rutabaga\n",
"missingonsrc": "",
"missingondst": "potato2\n",
"match": "rutabaga\n",
"differ": "",
"error": "",
})
file3 := r.WriteObject(context.Background(), "empty space", "-", t2)
fstest.CheckItems(t, r.Fremote, file1, file3)
check(3, 2, 1, false)
check(3, 2, 1, false, map[string]string{
"combined": "- empty space\n+ potato2\n= rutabaga\n",
"missingonsrc": "empty space\n",
"missingondst": "potato2\n",
"match": "rutabaga\n",
"differ": "",
"error": "",
})
file2r := file2
if fs.Config.SizeOnly {
@ -407,16 +470,45 @@ func testCheck(t *testing.T, checkFunction func(ctx context.Context, fdst, fsrc
r.WriteObject(context.Background(), "potato2", "------------------------------------------------------------", t1)
}
fstest.CheckItems(t, r.Fremote, file1, file2r, file3)
check(4, 1, 2, false)
check(4, 1, 2, false, map[string]string{
"combined": "- empty space\n= potato2\n= rutabaga\n",
"missingonsrc": "empty space\n",
"missingondst": "",
"match": "rutabaga\npotato2\n",
"differ": "",
"error": "",
})
r.WriteFile("empty space", "-", t2)
fstest.CheckItems(t, r.Flocal, file1, file2, file3)
check(5, 0, 3, false)
file3r := file3
file3l := r.WriteFile("empty space", "DIFFER", t2)
fstest.CheckItems(t, r.Flocal, file1, file2, file3l)
check(5, 1, 3, false, map[string]string{
"combined": "* empty space\n= potato2\n= rutabaga\n",
"missingonsrc": "",
"missingondst": "",
"match": "potato2\nrutabaga\n",
"differ": "empty space\n",
"error": "",
})
file4 := r.WriteObject(context.Background(), "remotepotato", "------------------------------------------------------------", t1)
fstest.CheckItems(t, r.Fremote, file1, file2r, file3, file4)
check(6, 1, 3, false)
check(7, 0, 3, true)
fstest.CheckItems(t, r.Fremote, file1, file2r, file3r, file4)
check(6, 2, 3, false, map[string]string{
"combined": "* empty space\n= potato2\n= rutabaga\n- remotepotato\n",
"missingonsrc": "remotepotato\n",
"missingondst": "",
"match": "potato2\nrutabaga\n",
"differ": "empty space\n",
"error": "",
})
check(7, 1, 3, true, map[string]string{
"combined": "* empty space\n= potato2\n= rutabaga\n",
"missingonsrc": "",
"missingondst": "",
"match": "potato2\nrutabaga\n",
"differ": "empty space\n",
"error": "",
})
}
func TestCheck(t *testing.T) {
@ -432,7 +524,12 @@ func TestCheckFsError(t *testing.T) {
if err != nil {
t.Fatal(err)
}
err = operations.Check(context.Background(), dstFs, srcFs, false)
opt := operations.CheckOpt{
Fdst: dstFs,
Fsrc: srcFs,
OneWay: false,
}
err = operations.Check(context.Background(), &opt)
require.Error(t, err)
}