feat: add multiple paths support to --compare-dest and --copy-dest flag

fs/cache/cache.go

@@ -104,6 +104,19 @@ func Get(ctx context.Context, fsString string) (f fs.Fs, err error) {
 	return GetFn(ctx, fsString, fs.NewFs)
 }
 
+// GetArr gets []fs.Fs from []fsStrings either from the cache or creates it afresh
+func GetArr(ctx context.Context, fsStrings []string) (f []fs.Fs, err error) {
+	var fArr []fs.Fs
+	for _, fsString := range fsStrings {
+		f1, err1 := GetFn(ctx, fsString, fs.NewFs)
+		if err1 != nil {
+			return fArr, err1
+		}
+		fArr = append(fArr, f1)
+	}
+	return fArr, nil
+}
+
 // Put puts an fs.Fs named fsString into the cache
 func Put(fsString string, f fs.Fs) {
 	canonicalName := fs.ConfigString(f)

fs/config.go

@@ -76,8 +76,8 @@ type ConfigInfo struct {
 	NoUnicodeNormalization bool
 	NoUpdateModTime        bool
 	DataRateUnit           string
-	CompareDest            string
-	CopyDest               string
+	CompareDest            []string
+	CopyDest               []string
 	BackupDir              string
 	Suffix                 string
 	SuffixKeepExtension    bool

fs/config/configflags/configflags.go

@@ -81,8 +81,8 @@ func AddFlags(ci *fs.ConfigInfo, flagSet *pflag.FlagSet) {
 	flags.BoolVarP(flagSet, &ci.NoCheckDest, "no-check-dest", "", ci.NoCheckDest, "Don't check the destination, copy regardless.")
 	flags.BoolVarP(flagSet, &ci.NoUnicodeNormalization, "no-unicode-normalization", "", ci.NoUnicodeNormalization, "Don't normalize unicode characters in filenames.")
 	flags.BoolVarP(flagSet, &ci.NoUpdateModTime, "no-update-modtime", "", ci.NoUpdateModTime, "Don't update destination mod-time if files identical.")
-	flags.StringVarP(flagSet, &ci.CompareDest, "compare-dest", "", ci.CompareDest, "Include additional server-side path during comparison.")
-	flags.StringVarP(flagSet, &ci.CopyDest, "copy-dest", "", ci.CopyDest, "Implies --compare-dest but also copies files from path into destination.")
+	flags.StringArrayVarP(flagSet, &ci.CompareDest, "compare-dest", "", nil, "Include additional comma separated server-side paths during comparison.")
+	flags.StringArrayVarP(flagSet, &ci.CopyDest, "copy-dest", "", nil, "Implies --compare-dest but also copies files from paths into destination.")
 	flags.StringVarP(flagSet, &ci.BackupDir, "backup-dir", "", ci.BackupDir, "Make backups into hierarchy based in DIR.")
 	flags.StringVarP(flagSet, &ci.Suffix, "suffix", "", ci.Suffix, "Suffix to add to changed files.")
 	flags.BoolVarP(flagSet, &ci.SuffixKeepExtension, "suffix-keep-extension", "", ci.SuffixKeepExtension, "Preserve the extension when using --suffix.")
@@ -217,7 +217,7 @@ func SetFlags(ci *fs.ConfigInfo) {
 		ci.DeleteMode = fs.DeleteModeDefault
 	}
 
-	if ci.CompareDest != "" && ci.CopyDest != "" {
+	if len(ci.CompareDest) > 0 && len(ci.CopyDest) > 0 {
 		log.Fatalf(`Can't use --compare-dest with --copy-dest.`)
 	}
 

fs/operations/operations.go

@@ -748,6 +748,16 @@ func SameConfig(fdst, fsrc fs.Info) bool {
 	return fdst.Name() == fsrc.Name()
 }
 
+// SameConfigArr returns true if any of []fsrcs has same config file entry with fdst
+func SameConfigArr(fdst fs.Info, fsrcs []fs.Fs) bool {
+	for _, fsrc := range fsrcs {
+		if fdst.Name() == fsrc.Name() {
+			return true
+		}
+	}
+	return false
+}
+
 // Same returns true if fdst and fsrc point to the same underlying Fs
 func Same(fdst, fsrc fs.Info) bool {
 	return SameConfig(fdst, fsrc) && strings.Trim(fdst.Root(), "/") == strings.Trim(fsrc.Root(), "/")
@@ -1354,9 +1364,9 @@ func Rmdirs(ctx context.Context, f fs.Fs, dir string, leaveRoot bool) error {
 }
 
 // GetCompareDest sets up --compare-dest
-func GetCompareDest(ctx context.Context) (CompareDest fs.Fs, err error) {
+func GetCompareDest(ctx context.Context) (CompareDest []fs.Fs, err error) {
 	ci := fs.GetConfig(ctx)
-	CompareDest, err = cache.Get(ctx, ci.CompareDest)
+	CompareDest, err = cache.GetArr(ctx, ci.CompareDest)
 	if err != nil {
 		return nil, fserrors.FatalError(errors.Errorf("Failed to make fs for --compare-dest %q: %v", ci.CompareDest, err))
 	}
@@ -1391,18 +1401,21 @@ func compareDest(ctx context.Context, dst, src fs.Object, CompareDest fs.Fs) (No
 }
 
 // GetCopyDest sets up --copy-dest
-func GetCopyDest(ctx context.Context, fdst fs.Fs) (CopyDest fs.Fs, err error) {
+func GetCopyDest(ctx context.Context, fdst fs.Fs) (CopyDest []fs.Fs, err error) {
 	ci := fs.GetConfig(ctx)
-	CopyDest, err = cache.Get(ctx, ci.CopyDest)
+	CopyDest, err = cache.GetArr(ctx, ci.CopyDest)
 	if err != nil {
 		return nil, fserrors.FatalError(errors.Errorf("Failed to make fs for --copy-dest %q: %v", ci.CopyDest, err))
 	}
-	if !SameConfig(fdst, CopyDest) {
+	if !SameConfigArr(fdst, CopyDest) {
 		return nil, fserrors.FatalError(errors.New("parameter to --copy-dest has to be on the same remote as destination"))
 	}
-	if CopyDest.Features().Copy == nil {
-		return nil, fserrors.FatalError(errors.New("can't use --copy-dest on a remote which doesn't support server-side copy"))
+	for _, cf := range CopyDest {
+		if cf.Features().Copy == nil {
+			return nil, fserrors.FatalError(errors.New("can't use --copy-dest on a remote which doesn't support server side copy"))
+		}
 	}
+
 	return CopyDest, nil
 }
 
@@ -1457,12 +1470,22 @@ func copyDest(ctx context.Context, fdst fs.Fs, dst, src fs.Object, CopyDest, bac
 // does not need to be copied
 //
 // Returns True if src does not need to be copied
-func CompareOrCopyDest(ctx context.Context, fdst fs.Fs, dst, src fs.Object, CompareOrCopyDest, backupDir fs.Fs) (NoNeedTransfer bool, err error) {
+func CompareOrCopyDest(ctx context.Context, fdst fs.Fs, dst, src fs.Object, CompareOrCopyDest []fs.Fs, backupDir fs.Fs) (NoNeedTransfer bool, err error) {
 	ci := fs.GetConfig(ctx)
-	if ci.CompareDest != "" {
-		return compareDest(ctx, dst, src, CompareOrCopyDest)
-	} else if ci.CopyDest != "" {
-		return copyDest(ctx, fdst, dst, src, CompareOrCopyDest, backupDir)
+	if len(ci.CompareDest) > 0 {
+		for _, compareF := range CompareOrCopyDest {
+			NoNeedTransfer, err := compareDest(ctx, dst, src, compareF)
+			if NoNeedTransfer || err != nil {
+				return NoNeedTransfer, err
+			}
+		}
+	} else if len(ci.CopyDest) > 0 {
+		for _, copyF := range CompareOrCopyDest {
+			NoNeedTransfer, err := copyDest(ctx, fdst, dst, src, copyF, backupDir)
+			if NoNeedTransfer || err != nil {
+				return NoNeedTransfer, err
+			}
+		}
 	}
 	return false, nil
 }
@@ -1732,19 +1755,20 @@ func moveOrCopyFile(ctx context.Context, fdst fs.Fs, fsrc fs.Fs, dstFileName str
 		return err
 	}
 
-	var backupDir, copyDestDir fs.Fs
+	var backupDir fs.Fs
+	var copyDestDir []fs.Fs
 	if ci.BackupDir != "" || ci.Suffix != "" {
 		backupDir, err = BackupDir(ctx, fdst, fsrc, srcFileName)
 		if err != nil {
 			return errors.Wrap(err, "creating Fs for --backup-dir failed")
 		}
 	}
-	if ci.CompareDest != "" {
+	if len(ci.CompareDest) > 0 {
 		copyDestDir, err = GetCompareDest(ctx)
 		if err != nil {
 			return err
 		}
-	} else if ci.CopyDest != "" {
+	} else if len(ci.CopyDest) > 0 {
 		copyDestDir, err = GetCopyDest(ctx, fdst)
 		if err != nil {
 			return err

fs/operations/operations_test.go

@@ -909,9 +909,9 @@ func TestCopyFileCompareDest(t *testing.T) {
 	r := fstest.NewRun(t)
 	defer r.Finalise()
 
-	ci.CompareDest = r.FremoteName + "/CompareDest"
+	ci.CompareDest = []string{r.FremoteName + "/CompareDest"}
 	defer func() {
-		ci.CompareDest = ""
+		ci.CompareDest = nil
 	}()
 	fdst, err := fs.NewFs(ctx, r.FremoteName+"/dst")
 	require.NoError(t, err)
@@ -995,9 +995,9 @@ func TestCopyFileCopyDest(t *testing.T) {
 		t.Skip("Skipping test as remote does not support server-side copy")
 	}
 
-	ci.CopyDest = r.FremoteName + "/CopyDest"
+	ci.CopyDest = []string{r.FremoteName + "/CopyDest"}
 	defer func() {
-		ci.CopyDest = ""
+		ci.CopyDest = nil
 	}()
 
 	fdst, err := fs.NewFs(ctx, r.FremoteName+"/dst")

fs/sync/sync.go

@@ -70,7 +70,7 @@ type syncCopyMove struct {
 	trackRenamesWg         sync.WaitGroup         // wg for background track renames
 	trackRenamesCh         chan fs.Object         // objects are pumped in here
 	renameCheck            []fs.Object            // accumulate files to check for rename here
-	compareCopyDest        fs.Fs                  // place to check for files to server-side copy
+	compareCopyDest        []fs.Fs                // place to check for files to server side copy
 	backupDir              fs.Fs                  // place to store overwrites/deletes
 	checkFirst             bool                   // if set run all the checkers before starting transfers
 }
@@ -212,13 +212,13 @@ func newSyncCopyMove(ctx context.Context, fdst, fsrc fs.Fs, deleteMode fs.Delete
 			return nil, err
 		}
 	}
-	if ci.CompareDest != "" {
+	if len(ci.CompareDest) > 0 {
 		var err error
 		s.compareCopyDest, err = operations.GetCompareDest(ctx)
 		if err != nil {
 			return nil, err
 		}
-	} else if ci.CopyDest != "" {
+	} else if len(ci.CopyDest) > 0 {
 		var err error
 		s.compareCopyDest, err = operations.GetCopyDest(ctx, fdst)
 		if err != nil {
@@ -890,7 +890,7 @@ func (s *syncCopyMove) run() error {
 	// Delete empty fsrc subdirectories
 	// if DoMove and --delete-empty-src-dirs flag is set
 	if s.DoMove && s.deleteEmptySrcDirs {
-		//delete empty subdirectories that were part of the move
+		// delete empty subdirectories that were part of the move
 		s.processError(s.deleteEmptyDirectories(s.ctx, s.fsrc, s.srcEmptyDirs))
 	}
 

fs/sync/sync_test.go

@@ -1480,9 +1480,9 @@ func TestSyncCompareDest(t *testing.T) {
 	r := fstest.NewRun(t)
 	defer r.Finalise()
 
-	ci.CompareDest = r.FremoteName + "/CompareDest"
+	ci.CompareDest = []string{r.FremoteName + "/CompareDest"}
 	defer func() {
-		ci.CompareDest = ""
+		ci.CompareDest = []string{}
 	}()
 
 	fdst, err := fs.NewFs(ctx, r.FremoteName+"/dst")
@@ -1562,6 +1562,40 @@ func TestSyncCompareDest(t *testing.T) {
 	fstest.CheckItems(t, r.Fremote, file2, file3, file4, file5bdst)
 }
 
+// Test with multiple CompareDest
+func TestSyncMultipleCompareDest(t *testing.T) {
+	ctx := context.Background()
+	ci := fs.GetConfig(ctx)
+	r := fstest.NewRun(t)
+	defer r.Finalise()
+
+	ci.CompareDest = []string{r.FremoteName + "/pre-dest1", r.FremoteName + "/pre-dest2"}
+	defer func() {
+		ci.CompareDest = []string{}
+	}()
+
+	// check empty dest, new compare
+	fsrc1 := r.WriteFile("1", "1", t1)
+	fsrc2 := r.WriteFile("2", "2", t1)
+	fsrc3 := r.WriteFile("3", "3", t1)
+	fstest.CheckItems(t, r.Flocal, fsrc1, fsrc2, fsrc3)
+
+	fdest1 := r.WriteObject(ctx, "pre-dest1/1", "1", t1)
+	fdest2 := r.WriteObject(ctx, "pre-dest2/2", "2", t1)
+	fstest.CheckItems(t, r.Fremote, fdest1, fdest2)
+
+	accounting.GlobalStats().ResetCounters()
+	fdst, err := fs.NewFs(ctx, r.FremoteName+"/dest")
+	require.NoError(t, err)
+	require.NoError(t, Sync(ctx, fdst, r.Flocal, false))
+
+	fdest3 := fsrc3
+	fdest3.Path = "dest/3"
+
+	fstest.CheckItems(t, fdst, fsrc3)
+	fstest.CheckItems(t, r.Fremote, fdest1, fdest2, fdest3)
+}
+
 // Test with CopyDest set
 func TestSyncCopyDest(t *testing.T) {
 	ctx := context.Background()
@@ -1573,9 +1607,9 @@ func TestSyncCopyDest(t *testing.T) {
 		t.Skip("Skipping test as remote does not support server-side copy")
 	}
 
-	ci.CopyDest = r.FremoteName + "/CopyDest"
+	ci.CopyDest = []string{r.FremoteName + "/CopyDest"}
 	defer func() {
-		ci.CopyDest = ""
+		ci.CopyDest = []string{}
 	}()
 
 	fdst, err := fs.NewFs(ctx, r.FremoteName+"/dst")