github-mirror/rclone
2
0
Fork 0
mirror of https://github.com/rclone/rclone.git synced 2024-11-22 11:58:20 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

sync: don't set dir modtimes if already set

Browse Source 
Before this change, directory modtimes (and metadata) were always synced from
src to dst, even if already in sync (i.e. their modtimes already matched.) This
potentially required excessive API calls, made logs noisy, and was potentially
problematic for backends that create "versions" or otherwise log activity
updates when modtime/metadata is updated.

After this change, a new DirsEqual function is added to check whether dirs are
equal based on a number of factors such as ModifyWindow and sync flags in use.
If the dirs are equal, the modtime/metadata update is skipped.

For backends that require setDirModTimeAfter, the "after" sync is performed only
for dirs that could have been changed by the sync (i.e. dirs containing files
that were created/updated.)

Note that dir metadata (other than modtime) is not currently considered by
DirsEqual, consistent with how object metadata is synced (only when objects are
unequal for reasons other than metadata).

To sync dir modtimes and metadata unconditionally (the previous behavior), use
--ignore-times.
This commit is contained in:
nielash 2024-02-28 19:29:38 -05:00 committed by Nick Craig-Wood
parent fd8faeb0e6
commit 8c69455c37
20 changed files with 257 additions and 182 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

29
cmd/bisync/bisync_test.go
View File

@ -34,6 +34,7 @@ import (
"github.com/rclone/rclone/fs/object"
"github.com/rclone/rclone/fs/operations"
"github.com/rclone/rclone/fs/sync"
"github.com/rclone/rclone/fs/walk"
"github.com/rclone/rclone/fstest"
"github.com/rclone/rclone/lib/atexit"
"github.com/rclone/rclone/lib/encoder"
@ -58,6 +59,8 @@ const (
fixSlash = (runtime.GOOS == "windows")
)
var initDate = time.Date(2000, time.January, 1, 0, 0, 0, 0, bisync.TZ)
// logReplacements make modern test logs comparable with golden dir.
// It is a string slice of even length with this structure:
//
@ -87,9 +90,9 @@ var logReplacements = []string{
`^.*? for same-side diffs on .*?$`, dropMe,
`^.*?Downloading hashes.*?$`, dropMe,
// ignore timestamps in directory time updates
`^(INFO : .*?: Made directory with (metadata|modification time)).*$`, `$1`,
`^(INFO : .*?: (Made directory with|Set directory) (metadata|modification time)).*$`, dropMe,
// ignore sizes in directory time updates
`^(NOTICE: .*?: Skipped set directory modification time as --dry-run is set).*$`, `$1`,
`^(NOTICE: .*?: Skipped set directory modification time as --dry-run is set).*$`, dropMe,
}
// Some dry-run messages differ depending on the particular remote.
@ -320,7 +323,6 @@ func (b *bisyncTest) runTestCase(ctx context.Context, t *testing.T, testCase str
// For test stability, jam initial dates to a fixed past date.
// Test cases that change files will touch specific files to fixed new dates.
initDate := time.Date(2000, time.January, 1, 0, 0, 0, 0, bisync.TZ)
err = filepath.Walk(b.initDir, func(path string, info os.FileInfo, err error) error {
if err == nil && !info.IsDir() {
return os.Chtimes(path, initDate, initDate)
@ -868,6 +870,20 @@ func (b *bisyncTest) runBisync(ctx context.Context, args []string) (err error) {
}
}
// set all dirs to a fixed date for test stability, as they are considered as of v1.66.
jamDirTimes := func(f fs.Fs) {
err := walk.ListR(ctx, f, "", true, -1, walk.ListDirs, func(entries fs.DirEntries) error {
var err error
entries.ForDir(func(dir fs.Directory) {
_, err = operations.SetDirModTime(ctx, f, dir, "", initDate)
})
return err
})
assert.NoError(b.t, err, "error jamming dirtimes")
}
jamDirTimes(fs1)
jamDirTimes(fs2)
output := bilib.CaptureOutput(func() {
err = bisync.Bisync(octx, fs1, fs2, opt)
})
@ -1385,10 +1401,10 @@ func (b *bisyncTest) mangleListing(text string, golden bool, file string) string
lines[i] = match[1] + "-" + match[3] + match[4] // replace it with "-" for comparison purposes (see #5679)
}
// account for modtime precision
var lineRegex = regexp.MustCompile(`^(\S) +(-?\d+) (\S+) (\S+) (\d{4}-\d\d-\d\dT\d\d:\d\d:\d\d\.\d{9}[+-]\d{4}) (".+")$`)
lineRegex := regexp.MustCompile(`^(\S) +(-?\d+) (\S+) (\S+) (\d{4}-\d\d-\d\dT\d\d:\d\d:\d\d\.\d{9}[+-]\d{4}) (".+")$`)
const timeFormat = "2006-01-02T15:04:05.000000000-0700"
const lineFormat = "%s %8d %s %s %s %q\n"
var TZ = time.UTC
TZ := time.UTC
fields := lineRegex.FindStringSubmatch(strings.TrimSuffix(lines[i], "\n"))
if fields != nil {
sizeVal, sizeErr := strconv.ParseInt(fields[2], 10, 64)
@ -1509,7 +1525,8 @@ func (b *bisyncTest) listDir(dir string) (names []string) {
ignoreIt := func(file string) bool {
ignoreList := []string{
// ".lst-control", ".lst-dry-control", ".lst-old", ".lst-dry-old",
".DS_Store"}
".DS_Store",
}
for _, s := range ignoreList {
if strings.Contains(file, s) {
return true

8
cmd/bisync/testdata/test_all_changed/golden/test.log vendored
View File

@ -16,11 +16,7 @@ INFO : Bisyncing with Comparison Settings:
INFO : Synching Path1 "{path1/}" with Path2 "{path2/}"
INFO : Copying Path2 files to Path1
INFO : - Path2 Resync is copying files to - Path1
INFO : subdir: Set directory modification time (using SetModTime)
INFO : subdir: Set directory modification time (using SetModTime)
INFO : - Path1 Resync is copying files to - Path2
INFO : subdir: Set directory modification time (using SetModTime)
INFO : subdir: Set directory modification time (using SetModTime)
INFO : Resync updating listings
INFO : Validating listings for Path1 "{path1/}" vs Path2 "{path2/}"
INFO : Bisync successful
@ -63,8 +59,6 @@ INFO : - Path1 Queue copy to Path2 - {
INFO : - Path1 Queue copy to Path2 - {path2/}file1.txt
INFO : - Path1 Queue copy to Path2 - {path2/}subdir/file20.txt
INFO : - Path1 Do queued copies to - Path2
INFO : subdir: Set directory modification time (using SetModTime)
INFO : subdir: Set directory modification time (using SetModTime)
INFO : Updating listings
INFO : Validating listings for Path1 "{path1/}" vs Path2 "{path2/}"
INFO : Bisync successful
@ -139,8 +133,6 @@ INFO : - Path1 Queue copy to Path2 - {
INFO : - Path1 Queue copy to Path2 - {path2/}file1.txt
INFO : - Path1 Queue copy to Path2 - {path2/}subdir/file20.txt
INFO : - Path1 Do queued copies to - Path2
INFO : subdir: Set directory modification time (using SetModTime)
INFO : subdir: Set directory modification time (using SetModTime)
INFO : Updating listings
INFO : Validating listings for Path1 "{path1/}" vs Path2 "{path2/}"
INFO : Bisync successful

6
cmd/bisync/testdata/test_basic/golden/test.log vendored
View File

@ -16,11 +16,7 @@ INFO : Bisyncing with Comparison Settings:
INFO : Synching Path1 "{path1/}" with Path2 "{path2/}"
INFO : Copying Path2 files to Path1
INFO : - Path2 Resync is copying files to - Path1
INFO : subdir: Set directory modification time (using SetModTime)
INFO : subdir: Set directory modification time (using SetModTime)
INFO : - Path1 Resync is copying files to - Path2
INFO : subdir: Set directory modification time (using SetModTime)
INFO : subdir: Set directory modification time (using SetModTime)
INFO : Resync updating listings
INFO : Validating listings for Path1 "{path1/}" vs Path2 "{path2/}"
INFO : Bisync successful
@ -57,8 +53,6 @@ INFO : - Path1 Queue copy to Path2 - {
INFO : - Path2 Queue copy to Path1 - {path1/}file1.txt
INFO : - Path2 Do queued copies to - Path1
INFO : - Path1 Do queued copies to - Path2
INFO : subdir: Set directory modification time (using SetModTime)
INFO : subdir: Set directory modification time (using SetModTime)
INFO : Updating listings
INFO : Validating listings for Path1 "{path1/}" vs Path2 "{path2/}"
INFO : Bisync successful

12
cmd/bisync/testdata/test_check_access/golden/test.log vendored
View File

@ -16,11 +16,7 @@ INFO : Bisyncing with Comparison Settings:
INFO : Synching Path1 "{path1/}" with Path2 "{path2/}"
INFO : Copying Path2 files to Path1
INFO : - Path2 Resync is copying files to - Path1
INFO : subdir: Set directory modification time (using SetModTime)
INFO : subdir: Set directory modification time (using SetModTime)
INFO : - Path1 Resync is copying files to - Path2
INFO : subdir: Set directory modification time (using SetModTime)
INFO : subdir: Set directory modification time (using SetModTime)
INFO : Resync updating listings
INFO : Validating listings for Path1 "{path1/}" vs Path2 "{path2/}"
INFO : Bisync successful
@ -91,11 +87,7 @@ INFO : Bisyncing with Comparison Settings:
INFO : Synching Path1 "{path1/}" with Path2 "{path2/}"
INFO : Copying Path2 files to Path1
INFO : - Path2 Resync is copying files to - Path1
INFO : subdir: Set directory modification time (using SetModTime)
INFO : subdir: Set directory modification time (using SetModTime)
INFO : - Path1 Resync is copying files to - Path2
INFO : subdir: Set directory modification time (using SetModTime)
INFO : subdir: Set directory modification time (using SetModTime)
INFO : Resync updating listings
INFO : Validating listings for Path1 "{path1/}" vs Path2 "{path2/}"
INFO : Bisync successful
@ -188,11 +180,7 @@ INFO : Bisyncing with Comparison Settings:
INFO : Synching Path1 "{path1/}" with Path2 "{path2/}"
INFO : Copying Path2 files to Path1
INFO : - Path2 Resync is copying files to - Path1
INFO : subdir: Set directory modification time (using SetModTime)
INFO : subdir: Set directory modification time (using SetModTime)
INFO : - Path1 Resync is copying files to - Path2
INFO : subdir: Set directory modification time (using SetModTime)
INFO : subdir: Set directory modification time (using SetModTime)
INFO : Resync updating listings
INFO : Validating listings for Path1 "{path1/}" vs Path2 "{path2/}"
INFO : Bisync successful

24
cmd/bisync/testdata/test_check_access_filters/golden/test.log vendored
View File

@ -21,15 +21,7 @@ INFO : Using filters file {workdir/}exclude-other-filtersfile.txt
INFO : Storing filters file hash to {workdir/}exclude-other-filtersfile.txt.md5
INFO : Copying Path2 files to Path1
INFO : - Path2 Resync is copying files to - Path1
INFO : subdir: Set directory modification time (using SetModTime)
INFO : subdir/subdirB: Set directory modification time (using SetModTime)
INFO : subdir: Set directory modification time (using SetModTime)
INFO : subdir/subdirB: Set directory modification time (using SetModTime)
INFO : - Path1 Resync is copying files to - Path2
INFO : subdir: Set directory modification time (using SetModTime)
INFO : subdir/subdirB: Set directory modification time (using SetModTime)
INFO : subdir: Set directory modification time (using SetModTime)
INFO : subdir/subdirB: Set directory modification time (using SetModTime)
INFO : Resync updating listings
INFO : Validating listings for Path1 "{path1/}" vs Path2 "{path2/}"
INFO : Bisync successful
@ -144,23 +136,7 @@ INFO : Using filters file {workdir/}include-other-filtersfile.txt
INFO : Storing filters file hash to {workdir/}include-other-filtersfile.txt.md5
INFO : Copying Path2 files to Path1
INFO : - Path2 Resync is copying files to - Path1
INFO : subdir: Set directory modification time (using SetModTime)
INFO : subdirX: Set directory modification time (using SetModTime)
INFO : subdirX/subdirX1: Set directory modification time (using SetModTime)
INFO : subdir/subdirB: Set directory modification time (using SetModTime)
INFO : subdir: Set directory modification time (using SetModTime)
INFO : subdirX: Set directory modification time (using SetModTime)
INFO : subdirX/subdirX1: Set directory modification time (using SetModTime)
INFO : subdir/subdirB: Set directory modification time (using SetModTime)
INFO : - Path1 Resync is copying files to - Path2
INFO : subdir: Set directory modification time (using SetModTime)
INFO : subdirX: Set directory modification time (using SetModTime)
INFO : subdir/subdirB: Set directory modification time (using SetModTime)
INFO : subdirX/subdirX1: Set directory modification time (using SetModTime)
INFO : subdir: Set directory modification time (using SetModTime)
INFO : subdirX: Set directory modification time (using SetModTime)
INFO : subdir/subdirB: Set directory modification time (using SetModTime)
INFO : subdirX/subdirX1: Set directory modification time (using SetModTime)
INFO : Resync updating listings
INFO : Validating listings for Path1 "{path1/}" vs Path2 "{path2/}"
INFO : Bisync successful

8
cmd/bisync/testdata/test_check_filename/golden/test.log vendored
View File

@ -16,11 +16,7 @@ INFO : Bisyncing with Comparison Settings:
INFO : Synching Path1 "{path1/}" with Path2 "{path2/}"
INFO : Copying Path2 files to Path1
INFO : - Path2 Resync is copying files to - Path1
INFO : subdir: Set directory modification time (using SetModTime)
INFO : subdir: Set directory modification time (using SetModTime)
INFO : - Path1 Resync is copying files to - Path2
INFO : subdir: Set directory modification time (using SetModTime)
INFO : subdir: Set directory modification time (using SetModTime)
INFO : Resync updating listings
INFO : Validating listings for Path1 "{path1/}" vs Path2 "{path2/}"
INFO : Bisync successful
@ -94,11 +90,7 @@ INFO : Copying Path2 files to Path1
INFO : Checking access health
INFO : Found 2 matching ".chk_file" files on both paths
INFO : - Path2 Resync is copying files to - Path1
INFO : subdir: Set directory modification time (using SetModTime)
INFO : subdir: Set directory modification time (using SetModTime)
INFO : - Path1 Resync is copying files to - Path2
INFO : subdir: Set directory modification time (using SetModTime)
INFO : subdir: Set directory modification time (using SetModTime)
INFO : Resync updating listings
INFO : Validating listings for Path1 "{path1/}" vs Path2 "{path2/}"
INFO : Bisync successful

4
cmd/bisync/testdata/test_createemptysrcdirs/golden/test.log vendored
View File

@ -147,11 +147,7 @@ INFO : Bisyncing with Comparison Settings:
INFO : Synching Path1 "{path1/}" with Path2 "{path2/}"
INFO : Copying Path2 files to Path1
INFO : - Path2 Resync is copying files to - Path1
INFO : subdir: Made directory with metadata (mtime=2024-02-27T04:53:52.809861575-05:00)
INFO : subdir: Set directory modification time (using SetModTime)
INFO : - Path1 Resync is copying files to - Path2
INFO : subdir: Set directory modification time (using SetModTime)
INFO : subdir: Set directory modification time (using SetModTime)
INFO : Resync updating listings
INFO : Validating listings for Path1 "{path1/}" vs Path2 "{path2/}"
INFO : Bisync successful

12
cmd/bisync/testdata/test_extended_char_paths/golden/test.log vendored
View File

@ -84,11 +84,7 @@ INFO : Bisyncing with Comparison Settings:
INFO : Synching Path1 "{path1/}" with Path2 "{path2/}"
INFO : Copying Path2 files to Path1
INFO : - Path2 Resync is copying files to - Path1
INFO : 測試_Русский_ _ _ě_áñ: Set directory modification time (using SetModTime)
INFO : 測試_Русский_ _ _ě_áñ: Set directory modification time (using SetModTime)
INFO : - Path1 Resync is copying files to - Path2
INFO : 測試_Русский_ _ _ě_áñ: Set directory modification time (using SetModTime)
INFO : 測試_Русский_ _ _ě_áñ: Set directory modification time (using SetModTime)
INFO : Resync updating listings
INFO : Validating listings for Path1 "{path1/}" vs Path2 "{path2/}"
INFO : Bisync successful
@ -133,11 +129,7 @@ INFO : Bisyncing with Comparison Settings:
INFO : Synching Path1 "{path1/}" with Path2 "{path2/}"
INFO : Copying Path2 files to Path1
INFO : - Path2 Resync is copying files to - Path1
INFO : 測試_Русский_ _ _ě_áñ: Set directory modification time (using SetModTime)
INFO : 測試_Русский_ _ _ě_áñ: Set directory modification time (using SetModTime)
INFO : - Path1 Resync is copying files to - Path2
INFO : 測試_Русский_ _ _ě_áñ: Set directory modification time (using SetModTime)
INFO : 測試_Русский_ _ _ě_áñ: Set directory modification time (using SetModTime)
INFO : Resync updating listings
INFO : Validating listings for Path1 "{path1/}" vs Path2 "{path2/}"
INFO : Bisync successful
@ -182,11 +174,7 @@ INFO : Using filters file {workdir/}測試_filtersfile.txt
INFO : Storing filters file hash to {workdir/}測試_filtersfile.txt.md5
INFO : Copying Path2 files to Path1
INFO : - Path2 Resync is copying files to - Path1
INFO : 測試_Русский_ _ _ě_áñ: Set directory modification time (using SetModTime)
INFO : 測試_Русский_ _ _ě_áñ: Set directory modification time (using SetModTime)
INFO : - Path1 Resync is copying files to - Path2
INFO : 測試_Русский_ _ _ě_áñ: Set directory modification time (using SetModTime)
INFO : 測試_Русский_ _ _ě_áñ: Set directory modification time (using SetModTime)
INFO : Resync updating listings
INFO : Validating listings for Path1 "{path1/}" vs Path2 "{path2/}"
INFO : Bisync successful

12
cmd/bisync/testdata/test_extended_filenames/golden/test.log vendored
View File

@ -25,11 +25,7 @@ INFO : Bisyncing with Comparison Settings:
INFO : Synching Path1 "{path1/}" with Path2 "{path2/}"
INFO : Copying Path2 files to Path1
INFO : - Path2 Resync is copying files to - Path1
INFO : subdir_with_ࢺ_: Set directory modification time (using SetModTime)
INFO : subdir_with_ࢺ_: Set directory modification time (using SetModTime)
INFO : - Path1 Resync is copying files to - Path2
INFO : subdir_with_ࢺ_: Set directory modification time (using SetModTime)
INFO : subdir_with_ࢺ_: Set directory modification time (using SetModTime)
INFO : Resync updating listings
INFO : Validating listings for Path1 "{path1/}" vs Path2 "{path2/}"
INFO : Bisync successful
@ -116,15 +112,7 @@ INFO : - Path2 Queue copy to Path1 - "
INFO : - Path1 Queue delete - {path1/}subdir_with_ࢺ_/filename_contains_ě_.txt
INFO : - Path2 Queue copy to Path1 - {path1/}subdir_with_ࢺ_/filename_contains_ࢺ_p2s.txt
INFO : - Path2 Do queued copies to - Path1
INFO : subdir withâ<68>Šwhite space.txt: Made directory with metadata (mtime=2024-02-27T04:53:52.913860529-05:00)
INFO : subdir_rawchars_â<5F>™_<E284A2>_þ: Made directory with metadata (mtime=2024-02-27T04:53:52.913860529-05:00)
INFO : subdir_with_ࢺ_: Set directory modification time (using SetModTime)
INFO : subdir withâ<68>Šwhite space.txt: Set directory modification time (using SetModTime)
INFO : subdir_rawchars_â<5F>™_<E284A2>_þ: Set directory modification time (using SetModTime)
INFO : subdir_with_ࢺ_: Set directory modification time (using SetModTime)
INFO : - Path1 Do queued copies to - Path2
INFO : subdir_with_ࢺ_: Set directory modification time (using SetModTime)
INFO : subdir_with_ࢺ_: Set directory modification time (using SetModTime)
INFO : Updating listings
INFO : Validating listings for Path1 "{path1/}" vs Path2 "{path2/}"
INFO : Bisync successful

6
cmd/bisync/testdata/test_filters/golden/test.log vendored
View File

@ -20,11 +20,7 @@ INFO : Using filters file {workdir/}filtersfile.flt
INFO : Storing filters file hash to {workdir/}filtersfile.flt.md5
INFO : Copying Path2 files to Path1
INFO : - Path2 Resync is copying files to - Path1
INFO : subdir: Set directory modification time (using SetModTime)
INFO : subdir: Set directory modification time (using SetModTime)
INFO : - Path1 Resync is copying files to - Path2
INFO : subdir: Set directory modification time (using SetModTime)
INFO : subdir: Set directory modification time (using SetModTime)
INFO : Resync updating listings
INFO : Validating listings for Path1 "{path1/}" vs Path2 "{path2/}"
INFO : Bisync successful
@ -58,8 +54,6 @@ INFO : Path2 checking for diffs
INFO : Applying changes
INFO : - Path1 Queue copy to Path2 - {path2/}subdir/fileZ.txt
INFO : - Path1 Do queued copies to - Path2
INFO : subdir: Set directory modification time (using SetModTime)
INFO : subdir: Set directory modification time (using SetModTime)
INFO : Updating listings
INFO : Validating listings for Path1 "{path1/}" vs Path2 "{path2/}"
INFO : Bisync successful

12
cmd/bisync/testdata/test_filtersfile_checks/golden/test.log vendored
View File

@ -16,11 +16,7 @@ INFO : Bisyncing with Comparison Settings:
INFO : Synching Path1 "{path1/}" with Path2 "{path2/}"
INFO : Copying Path2 files to Path1
INFO : - Path2 Resync is copying files to - Path1
INFO : subdir: Set directory modification time (using SetModTime)
INFO : subdir: Set directory modification time (using SetModTime)
INFO : - Path1 Resync is copying files to - Path2
INFO : subdir: Set directory modification time (using SetModTime)
INFO : subdir: Set directory modification time (using SetModTime)
INFO : Resync updating listings
INFO : Validating listings for Path1 "{path1/}" vs Path2 "{path2/}"
INFO : Bisync successful
@ -85,11 +81,7 @@ INFO : Using filters file {workdir/}filtersfile.txt
INFO : Storing filters file hash to {workdir/}filtersfile.txt.md5
INFO : Copying Path2 files to Path1
INFO : - Path2 Resync is copying files to - Path1
INFO : subdir: Set directory modification time (using SetModTime)
INFO : subdir: Set directory modification time (using SetModTime)
INFO : - Path1 Resync is copying files to - Path2
INFO : subdir: Set directory modification time (using SetModTime)
INFO : subdir: Set directory modification time (using SetModTime)
INFO : Resync updating listings
INFO : Validating listings for Path1 "{path1/}" vs Path2 "{path2/}"
INFO : Bisync successful
@ -154,11 +146,7 @@ INFO : Using filters file {workdir/}filtersfile.txt
INFO : Skipped storing filters file hash to {workdir/}filtersfile.txt.md5 as --dry-run is set
INFO : Copying Path2 files to Path1
NOTICE: - Path2 Resync is copying files to - Path1
NOTICE: subdir: Skipped set directory modification time as --dry-run is set
NOTICE: {path1String}: Skipped set directory modification time as --dry-run is set
NOTICE: - Path1 Resync is copying files to - Path2
NOTICE: subdir: Skipped set directory modification time as --dry-run is set
NOTICE: {path2String}: Skipped set directory modification time as --dry-run is set
INFO : Resync updating listings
INFO : Bisync successful

10
cmd/bisync/testdata/test_ignorelistingchecksum/golden/test.log vendored
View File

@ -16,11 +16,7 @@ INFO : Bisyncing with Comparison Settings:
INFO : Synching Path1 "{path1/}" with Path2 "{path2/}"
INFO : Copying Path2 files to Path1
INFO : - Path2 Resync is copying files to - Path1
INFO : subdir: Set directory modification time (using SetModTime)
INFO : subdir: Set directory modification time (using SetModTime)
INFO : - Path1 Resync is copying files to - Path2
INFO : subdir: Set directory modification time (using SetModTime)
INFO : subdir: Set directory modification time (using SetModTime)
INFO : Resync updating listings
INFO : Validating listings for Path1 "{path1/}" vs Path2 "{path2/}"
INFO : Bisync successful
@ -37,11 +33,7 @@ INFO : Bisyncing with Comparison Settings:
INFO : Synching Path1 "{path1/}" with Path2 "{path2/}"
INFO : Copying Path2 files to Path1
INFO : - Path2 Resync is copying files to - Path1
INFO : subdir: Set directory modification time (using SetModTime)
INFO : subdir: Set directory modification time (using SetModTime)
INFO : - Path1 Resync is copying files to - Path2
INFO : subdir: Set directory modification time (using SetModTime)
INFO : subdir: Set directory modification time (using SetModTime)
INFO : Resync updating listings
INFO : Validating listings for Path1 "{path1/}" vs Path2 "{path2/}"
INFO : Bisync successful
@ -77,8 +69,6 @@ INFO : - Path1 Queue copy to Path2 - {
INFO : - Path2 Queue copy to Path1 - {path1/}file1.txt
INFO : - Path2 Do queued copies to - Path1
INFO : - Path1 Do queued copies to - Path2
INFO : subdir: Set directory modification time (using SetModTime)
INFO : subdir: Set directory modification time (using SetModTime)
INFO : Updating listings
INFO : Validating listings for Path1 "{path1/}" vs Path2 "{path2/}"
INFO : Bisync successful

24
cmd/bisync/testdata/test_normalization/golden/test.log vendored
View File

@ -17,11 +17,7 @@ INFO : Bisyncing with Comparison Settings:
INFO : Synching Path1 "{path1/}" with Path2 "{path2/}"
INFO : Copying Path2 files to Path1
INFO : - Path2 Resync is copying files to - Path1
INFO : 測試_Русский_ _ _ě_áñ: Set directory modification time (using SetModTime)
INFO : 測試_Русский_ _ _ě_áñ: Set directory modification time (using SetModTime)
INFO : - Path1 Resync is copying files to - Path2
INFO : 測試_Русский_ _ _ě_áñ: Set directory modification time (using SetModTime)
INFO : 測試_Русский_ _ _ě_áñ: Set directory modification time (using SetModTime)
INFO : Resync updating listings
INFO : Validating listings for Path1 "{path1/}" vs Path2 "{path2/}"
INFO : Bisync successful
@ -74,12 +70,8 @@ INFO : - Path1 Queue copy to Path2 - "
INFO : - Path2 Queue copy to Path1 - {path1/}file1.txt
INFO : - Path2 Do queued copies to - Path1
INFO : - Path1 Do queued copies to - Path2
INFO : 測試_Русский___ě_áñ👸🏼🧝🏾‍♀️💆🏿‍♂️🐨🤙🏼🤮🧑🏻‍🔧🧑‍🔬éö: Made directory with metadata (mtime=2024-02-27T04:53:52.993859723-05:00)
INFO : folder: Set directory modification time (using SetModTime)
INFO : folder/hello,WORLD!.txt: Fixed case by renaming to: folder/HeLlO,wOrLd!.txt
INFO : folder/éééö.txt: Fixed case by renaming to: folder/éééö.txt
INFO : 測試_Русский___ě_áñ👸🏼🧝🏾‍♀️💆🏿‍♂️🐨🤙🏼🤮🧑🏻‍🔧🧑‍🔬éö: Set directory modification time (using SetModTime)
INFO : folder: Set directory modification time (using SetModTime)
INFO : Updating listings
INFO : Validating listings for Path1 "{path1/}" vs Path2 "{path2/}"
INFO : Bisync successful
@ -158,11 +150,7 @@ INFO : - Path2 Queue copy to Path1 - {
INFO : - Path1 Queue copy to Path2 - "{path2/}測試_Руский___ě_áñ👸🏼🧝🏾\u200d♀️💆🏿\u200d♂️🐨🤙🏼🤮🧑🏻\u200d🔧🧑\u200d🔬éö/測試_Руский___ě_áñ👸🏼🧝🏾\u200d♀️💆🏿\u200d♂️🐨🤙🏼🤮🧑🏻\u200d🔧🧑\u200d🔬éö.txt"
INFO : - Path2 Queue copy to Path1 - {path1/}file1.txt
INFO : - Path2 Do queued copies to - Path1
INFO : folder: Set directory modification time (using SetModTime)
INFO : folder: Set directory modification time (using SetModTime)
INFO : - Path1 Do queued copies to - Path2
INFO : 測試_Руский___ě_áñ👸🏼🧝🏾‍♀️💆🏿‍♂️🐨🤙🏼🤮🧑🏻‍🔧🧑‍🔬éö: Made directory with metadata (mtime=2024-02-27T04:53:53.001859642-05:00)
INFO : 測試_Руский___ě_áñ👸🏼🧝🏾‍♀️💆🏿‍♂️🐨🤙🏼🤮🧑🏻‍🔧🧑‍🔬éö: Set directory modification time (using SetModTime)
INFO : Updating listings
INFO : Validating listings for Path1 "{path1/}" vs Path2 "{path2/}"
INFO : Bisync successful
@ -182,15 +170,7 @@ INFO : Bisyncing with Comparison Settings:
INFO : Synching Path1 "{path1/}" with Path2 "{path2/}"
INFO : Copying Path2 files to Path1
INFO : - Path2 Resync is copying files to - Path1
INFO : folder: Set directory modification time (using SetModTime)
INFO : 測試_Руский___ě_áñ👸🏼🧝🏾‍♀️💆🏿‍♂️🐨🤙🏼🤮🧑🏻‍🔧🧑‍🔬éö: Set directory modification time (using SetModTime)
INFO : folder: Set directory modification time (using SetModTime)
INFO : 測試_Руский___ě_áñ👸🏼🧝🏾‍♀️💆🏿‍♂️🐨🤙🏼🤮🧑🏻‍🔧🧑‍🔬éö: Set directory modification time (using SetModTime)
INFO : - Path1 Resync is copying files to - Path2
INFO : folder: Set directory modification time (using SetModTime)
INFO : 測試_Руский___ě_áñ👸🏼🧝🏾‍♀️💆🏿‍♂️🐨🤙🏼🤮🧑🏻‍🔧🧑‍🔬éö: Set directory modification time (using SetModTime)
INFO : folder: Set directory modification time (using SetModTime)
INFO : 測試_Руский___ě_áñ👸🏼🧝🏾‍♀️💆🏿‍♂️🐨🤙🏼🤮🧑🏻‍🔧🧑‍🔬éö: Set directory modification time (using SetModTime)
INFO : Resync updating listings
INFO : Validating listings for Path1 "{path1/}" vs Path2 "{path2/}"
INFO : Bisync successful
@ -230,10 +210,6 @@ INFO : - Path1 Queue copy to Path2 - "
INFO : - Path2 Queue copy to Path1 - {path1/}file1.txt
INFO : - Path2 Do queued copies to - Path1
INFO : - Path1 Do queued copies to - Path2
INFO : folder: Set directory modification time (using SetModTime)
INFO : 測試_Руский___ě_áñ👸🏼🧝🏾‍♀️💆🏿‍♂️🐨🤙🏼🤮🧑🏻‍🔧🧑‍🔬éö: Set directory modification time (using SetModTime)
INFO : folder: Set directory modification time (using SetModTime)
INFO : 測試_Руский___ě_áñ👸🏼🧝🏾‍♀️💆🏿‍♂️🐨🤙🏼🤮🧑🏻‍🔧🧑‍🔬éö: Set directory modification time (using SetModTime)
INFO : Updating listings
INFO : Validating listings for Path1 "{path1/}" vs Path2 "{path2/}"
INFO : Bisync successful

18
cmd/bisync/testdata/test_rclone_args/golden/test.log vendored
View File

@ -15,11 +15,7 @@ INFO : Bisyncing with Comparison Settings:
INFO : Synching Path1 "{path1/}" with Path2 "{path2/}"
INFO : Copying Path2 files to Path1
INFO : - Path2 Resync is copying files to - Path1
INFO : subdir: Set directory modification time (using SetModTime)
INFO : subdir: Set directory modification time (using SetModTime)
INFO : - Path1 Resync is copying files to - Path2
INFO : subdir: Set directory modification time (using SetModTime)
INFO : subdir: Set directory modification time (using SetModTime)
INFO : Resync updating listings
INFO : Validating listings for Path1 "{path1/}" vs Path2 "{path2/}"
INFO : Bisync successful
@ -111,11 +107,7 @@ INFO : Bisyncing with Comparison Settings:
INFO : Synching Path1 "{path1/}" with Path2 "{path2/}"
INFO : Copying Path2 files to Path1
INFO : - Path2 Resync is copying files to - Path1
INFO : subdir: Set directory modification time (using SetModTime)
INFO : subdir: Set directory modification time (using SetModTime)
INFO : - Path1 Resync is copying files to - Path2
INFO : subdir: Set directory modification time (using SetModTime)
INFO : subdir: Set directory modification time (using SetModTime)
INFO : Resync updating listings
INFO : Validating listings for Path1 "{path1/}" vs Path2 "{path2/}"
INFO : Bisync successful
@ -149,8 +141,6 @@ INFO : Applying changes
INFO : - Path2 Queue copy to Path1 - {path1/}file2.txt
INFO : - Path2 Queue copy to Path1 - {path1/}subdir/file21.txt
INFO : - Path2 Do queued copies to - Path1
INFO : subdir: Set directory modification time (using SetModTime)
INFO : subdir: Set directory modification time (using SetModTime)
INFO : Updating listings
INFO : Validating listings for Path1 "{path1/}" vs Path2 "{path2/}"
INFO : Bisync successful
@ -168,11 +158,7 @@ INFO : Bisyncing with Comparison Settings:
INFO : Synching Path1 "{path1/}" with Path2 "{path2/}"
INFO : Copying Path2 files to Path1
INFO : - Path2 Resync is copying files to - Path1
INFO : subdir: Set directory modification time (using SetModTime)
INFO : subdir: Set directory modification time (using SetModTime)
INFO : - Path1 Resync is copying files to - Path2
INFO : subdir: Set directory modification time (using SetModTime)
INFO : subdir: Set directory modification time (using SetModTime)
INFO : Resync updating listings
INFO : Validating listings for Path1 "{path1/}" vs Path2 "{path2/}"
INFO : Bisync successful
@ -214,11 +200,7 @@ INFO : - Path1 Queue copy to Path2 - {
INFO : - Path2 Queue copy to Path1 - {path1/}file2.txt
INFO : - Path2 Queue copy to Path1 - {path1/}subdir/file21.txt
INFO : - Path2 Do queued copies to - Path1
INFO : subdir: Set directory modification time (using SetModTime)
INFO : subdir: Set directory modification time (using SetModTime)
INFO : - Path1 Do queued copies to - Path2
INFO : subdir: Set directory modification time (using SetModTime)
INFO : subdir: Set directory modification time (using SetModTime)
INFO : Updating listings
INFO : Validating listings for Path1 "{path1/}" vs Path2 "{path2/}"
INFO : Bisync successful

6
cmd/bisync/testdata/test_rmdirs/golden/test.log vendored
View File

@ -16,11 +16,7 @@ INFO : Bisyncing with Comparison Settings:
INFO : Synching Path1 "{path1/}" with Path2 "{path2/}"
INFO : Copying Path2 files to Path1
INFO : - Path2 Resync is copying files to - Path1
INFO : subdir: Set directory modification time (using SetModTime)
INFO : subdir: Set directory modification time (using SetModTime)
INFO : - Path1 Resync is copying files to - Path2
INFO : subdir: Set directory modification time (using SetModTime)
INFO : subdir: Set directory modification time (using SetModTime)
INFO : Resync updating listings
INFO : Validating listings for Path1 "{path1/}" vs Path2 "{path2/}"
INFO : Bisync successful
@ -49,8 +45,6 @@ INFO : Path2 checking for diffs
INFO : Applying changes
INFO : - Path2 Queue delete - {path2/}subdir/file20.txt
INFO : - Path1 Do queued copies to - Path2
INFO : subdir: Set directory modification time (using SetModTime)
INFO : subdir: Set directory modification time (using SetModTime)
INFO : Updating listings
INFO : Validating listings for Path1 "{path1/}" vs Path2 "{path2/}"
INFO : Bisync successful

2
fs/config/configflags/configflags.go
View File

@ -55,7 +55,7 @@ func AddFlags(ci *fs.ConfigInfo, flagSet *pflag.FlagSet) {
flags.StringVarP(flagSet, &tempDir, "temp-dir", "", os.TempDir(), "Directory rclone will use for temporary files", "Config")
flags.BoolVarP(flagSet, &ci.CheckSum, "checksum", "c", ci.CheckSum, "Check for changes with size & checksum (if available, or fallback to size only).", "Copy")
flags.BoolVarP(flagSet, &ci.SizeOnly, "size-only", "", ci.SizeOnly, "Skip based on size only, not modtime or checksum", "Copy")
flags.BoolVarP(flagSet, &ci.IgnoreTimes, "ignore-times", "I", ci.IgnoreTimes, "Don't skip files that match size and time - transfer all files", "Copy")
flags.BoolVarP(flagSet, &ci.IgnoreTimes, "ignore-times", "I", ci.IgnoreTimes, "Don't skip items that match size and time - transfer all unconditionally", "Copy")
flags.BoolVarP(flagSet, &ci.IgnoreExisting, "ignore-existing", "", ci.IgnoreExisting, "Skip all files that exist on destination", "Copy")
flags.BoolVarP(flagSet, &ci.IgnoreErrors, "ignore-errors", "", ci.IgnoreErrors, "Delete even if there are I/O errors", "Sync")
flags.BoolVarP(flagSet, &ci.DryRun, "dry-run", "n", ci.DryRun, "Do a trial run with no permanent changes", "Config,Important")

41
fs/operations/operations.go
View File

@ -141,6 +141,40 @@ func Equal(ctx context.Context, src fs.ObjectInfo, dst fs.Object) bool {
return equal(ctx, src, dst, defaultEqualOpt(ctx))
}
// DirsEqual is like Equal but for dirs instead of objects.
// It returns true if two dirs should be considered "equal" for the purposes of syncCopyMove
// (in other words, true == "skip updating modtime/metadata for this dir".)
// Unlike Equal, it does not consider size or checksum, as these do not apply to directories.
func DirsEqual(ctx context.Context, src, dst fs.Directory, opt DirsEqualOpt) (equal bool) {
if dst == nil {
return false
}
ci := fs.GetConfig(ctx)
if ci.SizeOnly || ci.Immutable || ci.IgnoreExisting || opt.ModifyWindow == fs.ModTimeNotSupported {
return true
}
if ci.IgnoreTimes {
return false
}
if !(opt.SetDirModtime || opt.SetDirMetadata) {
return true
}
srcModTime, dstModTime := src.ModTime(ctx), dst.ModTime(ctx)
if srcModTime.IsZero() || dstModTime.IsZero() {
return false
}
dt := dstModTime.Sub(srcModTime)
if dt < opt.ModifyWindow && dt > -opt.ModifyWindow {
fs.Debugf(dst, "Directory modification time the same (differ by %s, within tolerance %s)", dt, opt.ModifyWindow)
return true
}
if ci.UpdateOlder && dt >= opt.ModifyWindow {
fs.Debugf(dst, "Destination directory is newer than source, skipping")
return true
}
return false
}
// sizeDiffers compare the size of src and dst taking into account the
// various ways of ignoring sizes
func sizeDiffers(ctx context.Context, src, dst fs.ObjectInfo) bool {
@ -172,6 +206,13 @@ func defaultEqualOpt(ctx context.Context) equalOpt {
}
}
// DirsEqualOpt represents options for DirsEqual function()
type DirsEqualOpt struct {
ModifyWindow time.Duration // Max time diff to be considered the same
SetDirModtime bool // whether to consider dir modtime
SetDirMetadata bool // whether to consider dir metadata
}
var modTimeUploadOnce sync.Once
// emit a log if we are about to upload a file to set its modification time

71
fs/operations/operations_test.go
View File

@ -1754,3 +1754,74 @@ func TestSetDirModTime(t *testing.T) {
}
fstest.CheckDirModTime(ctx, t, r.Fremote, fstest.NewDirectory(ctx, t, r.Fremote, name), t2)
}
func TestDirsEqual(t *testing.T) {
ctx := context.Background()
ctx, ci := fs.AddConfig(ctx)
ci.Metadata = true
r := fstest.NewRun(t)
if !r.Fremote.Features().WriteDirMetadata && r.Fremote.Features().MkdirMetadata == nil {
t.Skip("Skipping test as remote does not support WriteDirMetadata or MkdirMetadata")
}
opt := operations.DirsEqualOpt{
ModifyWindow: fs.GetModifyWindow(ctx, r.Flocal, r.Fremote),
SetDirModtime: true,
SetDirMetadata: true,
}
// Create a source local directory with metadata
src, err := operations.MkdirMetadata(ctx, r.Flocal, "dir with metadata to be copied", testMetadata)
require.NoError(t, err)
require.NotNil(t, src)
// try with nil dst -- should be false
equal := operations.DirsEqual(ctx, src, nil, opt)
assert.False(t, equal)
// make a dest with an equal modtime
dst, err := operations.MkdirModTime(ctx, r.Fremote, "dst", src.ModTime(ctx))
require.NoError(t, err)
// try with equal modtimes -- should be true
equal = operations.DirsEqual(ctx, src, dst, opt)
assert.True(t, equal)
// try with unequal modtimes -- should be false
dst, err = operations.SetDirModTime(ctx, r.Fremote, dst, "", t2)
require.NoError(t, err)
equal = operations.DirsEqual(ctx, src, dst, opt)
assert.False(t, equal)
// try with unequal modtimes that are within modify window -- should be true
halfWindow := opt.ModifyWindow / 2
dst, err = operations.SetDirModTime(ctx, r.Fremote, dst, "", src.ModTime(ctx).Add(halfWindow))
require.NoError(t, err)
equal = operations.DirsEqual(ctx, src, dst, opt)
assert.True(t, equal)
// test ignoretimes -- should be false
ci.IgnoreTimes = true
equal = operations.DirsEqual(ctx, src, dst, opt)
assert.False(t, equal)
// test immutable -- should be true
ci.IgnoreTimes = false
ci.Immutable = true
dst, err = operations.SetDirModTime(ctx, r.Fremote, dst, "", t3)
require.NoError(t, err)
equal = operations.DirsEqual(ctx, src, dst, opt)
assert.True(t, equal)
// test dst newer than src with --update -- should be true
ci.Immutable = false
ci.UpdateOlder = true
equal = operations.DirsEqual(ctx, src, dst, opt)
assert.True(t, equal)
// test no SetDirModtime or SetDirMetadata -- should be true
ci.UpdateOlder = false
opt.SetDirMetadata, opt.SetDirModtime = false, false
equal = operations.DirsEqual(ctx, src, dst, opt)
assert.True(t, equal)
}

65
fs/sync/sync.go
View File

@ -89,9 +89,10 @@ type syncCopyMove struct {
setDirMetadata bool // if set we set the directory metadata
setDirModTime bool // if set we set the directory modtimes
setDirModTimeAfter bool // if set we set the directory modtimes at the end of the sync
setDirModTimeMu sync.Mutex // protect setDirModTimeMu
setDirModTimeMu sync.Mutex // protect setDirModTimes and modifiedDirs
setDirModTimes []setDirModTime // directories that need their modtime set
setDirModTimesMaxLevel int // max level of the directories to set
modifiedDirs map[string]struct{} // dirs with changed contents (if s.setDirModTimeAfter)
}
// For keeping track of delayed modtime sets
@ -155,6 +156,7 @@ func newSyncCopyMove(ctx context.Context, fdst, fsrc fs.Fs, deleteMode fs.Delete
setDirMetadata: ci.Metadata && fsrc.Features().ReadDirMetadata && fdst.Features().WriteDirMetadata,
setDirModTime: fdst.Features().WriteDirSetModTime || fdst.Features().MkdirMetadata != nil || fdst.Features().DirSetModTime != nil,
setDirModTimeAfter: fdst.Features().DirModTimeUpdatesOnWrite,
modifiedDirs: make(map[string]struct{}),
}
s.logger, s.usingLogger = operations.GetLogger(ctx)
@ -399,6 +401,11 @@ func (s *syncCopyMove) pairChecker(in *pipe, out *pipe, fraction int, wg *sync.W
fs.Errorf(pair.Dst, "Source and destination exist but do not match: %v", err)
s.processError(err)
} else {
if pair.Dst != nil {
s.markDirModifiedObject(pair.Dst)
} else {
s.markDirModifiedObject(src)
}
// If destination already exists, then we must move it into --backup-dir if required
if pair.Dst != nil && s.backupDir != nil {
err := operations.MoveBackupDir(s.ctx, s.backupDir, pair.Dst)
@ -764,7 +771,6 @@ func (s *syncCopyMove) renameID(obj fs.Object, renamesStrategy trackRenamesStrat
if renamesStrategy.hash() {
var err error
hash, err := obj.Hash(s.ctx, s.commonHash)
if err != nil {
fs.Debugf(obj, "Hash failed: %v", err)
return ""
@ -1079,6 +1085,26 @@ func (s *syncCopyMove) DstOnly(dst fs.DirEntry) (recurse bool) {
return false
}
// keeps track of dirs with changed contents, to avoid setting modtimes on dirs that haven't changed
func (s *syncCopyMove) markDirModified(dir string) {
if !s.setDirModTimeAfter {
return
}
s.setDirModTimeMu.Lock()
defer s.setDirModTimeMu.Unlock()
s.modifiedDirs[dir] = struct{}{}
}
// like markDirModified, but accepts an Object instead of a string.
// the marked dir will be this object's parent.
func (s *syncCopyMove) markDirModifiedObject(o fs.Object) {
dir := path.Dir(o.Remote())
if dir == "." {
dir = ""
}
s.markDirModified(dir)
}
// copyDirMetadata copies the src directory modTime or Metadata to dst
// or f if nil. If dst is nil then it uses dir as the name of the new
// directory.
@ -1087,17 +1113,25 @@ func (s *syncCopyMove) DstOnly(dst fs.DirEntry) (recurse bool) {
// be nil.
func (s *syncCopyMove) copyDirMetadata(ctx context.Context, f fs.Fs, dst fs.Directory, dir string, src fs.Directory) (newDst fs.Directory) {
var err error
if s.setDirMetadata {
newDst, err = operations.CopyDirMetadata(ctx, f, dst, dir, src)
} else if s.setDirModTime {
if dst == nil {
newDst, err = operations.MkdirModTime(ctx, f, dir, src.ModTime(ctx))
} else {
newDst, err = operations.SetDirModTime(ctx, f, dst, dir, src.ModTime(ctx))
equal := operations.DirsEqual(ctx, src, dst, operations.DirsEqualOpt{ModifyWindow: s.modifyWindow, SetDirModtime: s.setDirModTime, SetDirMetadata: s.setDirMetadata})
if !s.setDirModTimeAfter && equal {
return nil
}
if s.setDirModTimeAfter && equal {
newDst = dst
} else {
if s.setDirMetadata {
newDst, err = operations.CopyDirMetadata(ctx, f, dst, dir, src)
} else if s.setDirModTime {
if dst == nil {
newDst, err = operations.MkdirModTime(ctx, f, dir, src.ModTime(ctx))
} else {
newDst, err = operations.SetDirModTime(ctx, f, dst, dir, src.ModTime(ctx))
}
} else if dst == nil {
// Create the directory if it doesn't exist
err = operations.Mkdir(ctx, f, dir)
}
} else if dst == nil {
// Create the directory if it doesn't exist
err = operations.Mkdir(ctx, f, dir)
}
// If we need to set modtime after and we created a dir, then save it for later
if s.setDirModTime && s.setDirModTimeAfter && err == nil {
@ -1138,7 +1172,7 @@ func (s *syncCopyMove) setDelayedDirModTimes(ctx context.Context) error {
// Timestamp all directories at the same level in parallel, deepest first
// We do this by iterating the slice multiple times to save memory
// There could be a lot of directories in this slice.
var errCount = errcount.New()
errCount := errcount.New()
for level := s.setDirModTimesMaxLevel; level >= 0; level-- {
g, gCtx := errgroup.WithContext(ctx)
g.SetLimit(s.ci.Checkers)
@ -1151,6 +1185,9 @@ func (s *syncCopyMove) setDelayedDirModTimes(ctx context.Context) error {
break
}
item := item
if _, ok := s.modifiedDirs[item.dir]; !ok {
continue
}
g.Go(func() error {
_, err := operations.SetDirModTime(gCtx, s.fdst, item.dst, item.dir, item.modTime)
if err != nil {
@ -1201,6 +1238,7 @@ func (s *syncCopyMove) SrcOnly(src fs.DirEntry) (recurse bool) {
if !NoNeedTransfer {
// No need to check since doesn't exist
fs.Debugf(src, "Need to transfer - File not found at Destination")
s.markDirModifiedObject(x)
ok := s.toBeUploaded.Put(s.inCtx, fs.ObjectPair{Src: x, Dst: nil})
if !ok {
return
@ -1217,6 +1255,7 @@ func (s *syncCopyMove) SrcOnly(src fs.DirEntry) (recurse bool) {
s.srcEmptyDirsMu.Unlock()
// Create the directory and make sure the Metadata/ModTime is correct
s.markDirModified(x.Remote())
s.copyDirMetadata(s.ctx, s.fdst, nil, x.Remote(), x)
return true
default:

69
fs/sync/sync_test.go
View File

@ -19,6 +19,7 @@ import (
mutex "sync" // renamed as "sync" already in use
_ "github.com/rclone/rclone/backend/all" // import all backends
"github.com/rclone/rclone/cmd/bisync/bilib"
"github.com/rclone/rclone/fs"
"github.com/rclone/rclone/fs/accounting"
"github.com/rclone/rclone/fs/filter"
@ -2499,6 +2500,74 @@ func TestSyncConcurrentTruncate(t *testing.T) {
testSyncConcurrent(t, "truncate")
}
// Tests that nothing is transferred when src and dst already match
// Run the same sync twice, ensure no action is taken the second time
func TestNothingToTransfer(t *testing.T) {
accounting.GlobalStats().ResetCounters()
ctx, _ := fs.AddConfig(context.Background())
r := fstest.NewRun(t)
file1 := r.WriteFile("sub dir/hello world", "hello world", t1)
file2 := r.WriteFile("sub dir2/very/very/very/very/very/nested/subdir/hello world", "hello world", t1)
r.CheckLocalItems(t, file1, file2)
_, err := operations.SetDirModTime(ctx, r.Flocal, nil, "sub dir", t2)
if err != nil && !errors.Is(err, fs.ErrorNotImplemented) {
require.NoError(t, err)
}
r.Mkdir(ctx, r.Fremote)
_, err = operations.MkdirModTime(ctx, r.Fremote, "sub dir", t3)
require.NoError(t, err)
// set logging
// (this checks log output as DirModtime operations do not yet have stats, and r.CheckDirectoryModTimes also does not tell us what actions were taken)
oldLogLevel := fs.GetConfig(context.Background()).LogLevel
defer func() { fs.GetConfig(context.Background()).LogLevel = oldLogLevel }() // reset to old val after test
// need to do this as fs.Infof only respects the globalConfig
fs.GetConfig(context.Background()).LogLevel = fs.LogLevelInfo
accounting.GlobalStats().ResetCounters()
ctx = predictDstFromLogger(ctx)
output := bilib.CaptureOutput(func() {
err = CopyDir(ctx, r.Fremote, r.Flocal, true)
require.NoError(t, err)
})
require.NotNil(t, output)
testLoggerVsLsf(ctx, r.Fremote, operations.GetLoggerOpt(ctx).JSON, t)
r.CheckLocalItems(t, file1, file2)
r.CheckRemoteItems(t, file1, file2)
// Check that the modtimes of the directories are as expected
r.CheckDirectoryModTimes(t, "sub dir")
// check that actions were taken
assert.True(t, strings.Contains(string(output), "Copied"), `expected to find at least one "Copied" log: `+string(output))
if r.Fremote.Features().DirSetModTime != nil || r.Fremote.Features().MkdirMetadata != nil {
assert.True(t, strings.Contains(string(output), "Set directory modification time"), `expected to find at least one "Set directory modification time" log: `+string(output))
}
assert.False(t, strings.Contains(string(output), "There was nothing to transfer"), `expected to find no "There was nothing to transfer" logs, but found one: `+string(output))
assert.Equal(t, int64(2), accounting.GlobalStats().GetTransfers())
// run it again and make sure no actions were taken
accounting.GlobalStats().ResetCounters()
ctx = predictDstFromLogger(ctx)
output = bilib.CaptureOutput(func() {
err = CopyDir(ctx, r.Fremote, r.Flocal, true)
require.NoError(t, err)
})
require.NotNil(t, output)
testLoggerVsLsf(ctx, r.Fremote, operations.GetLoggerOpt(ctx).JSON, t)
r.CheckLocalItems(t, file1, file2)
r.CheckRemoteItems(t, file1, file2)
// Check that the modtimes of the directories are as expected
r.CheckDirectoryModTimes(t, "sub dir")
// check that actions were NOT taken
assert.False(t, strings.Contains(string(output), "Copied"), `expected to find no "Copied" logs, but found one: `+string(output))
if r.Fremote.Features().DirSetModTime != nil || r.Fremote.Features().MkdirMetadata != nil {
assert.False(t, strings.Contains(string(output), "Set directory modification time"), `expected to find no "Set directory modification time" logs, but found one: `+string(output))
}
assert.True(t, strings.Contains(string(output), "There was nothing to transfer"), `expected to find a "There was nothing to transfer" log: `+string(output))
assert.Equal(t, int64(0), accounting.GlobalStats().GetTransfers())
}
// for testing logger:
func predictDstFromLogger(ctx context.Context) context.Context {
opt := operations.NewLoggerOpt()