Implement moveto and copyto commands for choosing a destination name on copy/move

Fixes #227
Fixes #476

cmd/all/all.go

@@ -10,6 +10,7 @@ import (
 	_ "github.com/ncw/rclone/cmd/cleanup"
 	_ "github.com/ncw/rclone/cmd/config"
 	_ "github.com/ncw/rclone/cmd/copy"
+	_ "github.com/ncw/rclone/cmd/copyto"
 	_ "github.com/ncw/rclone/cmd/dedupe"
 	_ "github.com/ncw/rclone/cmd/delete"
 	_ "github.com/ncw/rclone/cmd/genautocomplete"
@@ -23,6 +24,7 @@ import (
 	_ "github.com/ncw/rclone/cmd/mkdir"
 	_ "github.com/ncw/rclone/cmd/mount"
 	_ "github.com/ncw/rclone/cmd/move"
+	_ "github.com/ncw/rclone/cmd/moveto"
 	_ "github.com/ncw/rclone/cmd/purge"
 	_ "github.com/ncw/rclone/cmd/rmdir"
 	_ "github.com/ncw/rclone/cmd/rmdirs"

cmd/cmd.go

@@ -14,6 +14,7 @@ import (
 	"regexp"
 	"runtime"
 	"runtime/pprof"
+	"strings"
 	"time"
 
 	"github.com/spf13/cobra"
@@ -94,31 +95,50 @@ func ShowVersion() {
 	fmt.Printf("rclone %s\n", fs.Version)
 }
 
-// newFsSrc creates a src Fs from a name
+// newFsFile creates a dst Fs from a name but may point to a file.
 //
-// This can point to a file
-func newFsSrc(remote string) fs.Fs {
+// It returns a string with the file name if points to a file
+func newFsFile(remote string) (fs.Fs, string) {
 	fsInfo, configName, fsPath, err := fs.ParseRemote(remote)
 	if err != nil {
 		fs.Stats.Error()
 		log.Fatalf("Failed to create file system for %q: %v", remote, err)
 	}
 	f, err := fsInfo.NewFs(configName, fsPath)
-	if err == fs.ErrorIsFile {
+	switch err {
+	case fs.ErrorIsFile:
+		return f, path.Base(fsPath)
+	case nil:
+		return f, ""
+	default:
+		fs.Stats.Error()
+		log.Fatalf("Failed to create file system for %q: %v", remote, err)
+	}
+	return nil, ""
+}
+
+// newFsSrc creates a src Fs from a name
+//
+// It returns a string with the file name if limiting to one file
+//
+// This can point to a file
+func newFsSrc(remote string) (fs.Fs, string) {
+	f, fileName := newFsFile(remote)
+	if fileName != "" {
 		if !fs.Config.Filter.InActive() {
 			fs.Stats.Error()
 			log.Fatalf("Can't limit to single files when using filters: %v", remote)
 		}
 		// Limit transfers to this file
-		err = fs.Config.Filter.AddFile(path.Base(fsPath))
+		err := fs.Config.Filter.AddFile(fileName)
+		if err != nil {
+			fs.Stats.Error()
+			log.Fatalf("Failed to limit to single file %q: %v", remote, err)
+		}
 		// Set --no-traverse as only one file
 		fs.Config.NoTraverse = true
 	}
-	if err != nil {
-		fs.Stats.Error()
-		log.Fatalf("Failed to create file system for %q: %v", remote, err)
-	}
-	return f
+	return f, fileName
 }
 
 // newFsDst creates a dst Fs from a name
@@ -135,14 +155,62 @@ func newFsDst(remote string) fs.Fs {
 
 // NewFsSrcDst creates a new src and dst fs from the arguments
 func NewFsSrcDst(args []string) (fs.Fs, fs.Fs) {
-	fsrc, fdst := newFsSrc(args[0]), newFsDst(args[1])
+	fsrc, _ := newFsSrc(args[0])
+	fdst := newFsDst(args[1])
 	fs.CalculateModifyWindow(fdst, fsrc)
 	return fsrc, fdst
 }
 
+// RemoteSplit splits a remote into a parent and a leaf
+//
+// if it returns parent as an empty string then it wasn't possible
+func RemoteSplit(remote string) (parent string, leaf string) {
+	// Split remote on :
+	i := strings.Index(remote, ":")
+	remoteName := ""
+	remotePath := remote
+	if i >= 0 {
+		remoteName = remote[:i+1]
+		remotePath = remote[i+1:]
+	}
+	if remotePath == "" {
+		return "", ""
+	}
+	// Construct new remote name without last segment
+	parent, leaf = path.Split(remotePath)
+	if leaf == "" {
+		return "", ""
+	}
+	if parent != "/" {
+		parent = strings.TrimSuffix(parent, "/")
+	}
+	parent = remoteName + parent
+	if parent == "" {
+		parent = "."
+	}
+	return parent, leaf
+}
+
+// NewFsSrcDstFiles creates a new src and dst fs from the arguments
+// If src is a file then srcFileName and dstFileName will be non-empty
+func NewFsSrcDstFiles(args []string) (fsrc fs.Fs, srcFileName string, fdst fs.Fs, dstFileName string) {
+	fsrc, srcFileName = newFsSrc(args[0])
+	// If copying a file...
+	dstRemote := args[1]
+	if srcFileName != "" {
+		dstRemote, dstFileName = RemoteSplit(dstRemote)
+		if dstRemote == "" {
+			log.Fatalf("Can't find parent directory for %q", args[1])
+		}
+	}
+	fdst = newFsDst(dstRemote)
+	fs.CalculateModifyWindow(fdst, fsrc)
+	return
+}
+
 // NewFsSrc creates a new src fs from the arguments
 func NewFsSrc(args []string) fs.Fs {
-	fsrc := newFsSrc(args[0])
+	fsrc, _ := newFsSrc(args[0])
 	fs.CalculateModifyWindow(fsrc)
 	return fsrc
 }

cmd/cmd_test.go

@@ -0,0 +1,28 @@
+package cmd
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestRemoteSplit(t *testing.T) {
+
+	for _, test := range []struct {
+		remote, wantParent, wantLeaf string
+	}{
+		{"", "", ""},
+		{"remote:", "", ""},
+		{"remote:potato", "remote:", "potato"},
+		{"remote:potato/sausage", "remote:potato", "sausage"},
+		{"/", "", ""},
+		{"/root", "/", "root"},
+		{"/a/b", "/a", "b"},
+		{"root", ".", "root"},
+		{"a/b", "a", "b"},
+	} {
+		gotParent, gotLeaf := RemoteSplit(test.remote)
+		assert.Equal(t, test.wantParent, gotParent, test.remote)
+		assert.Equal(t, test.wantLeaf, gotLeaf, test.remote)
+	}
+}

cmd/copy/copy.go

@@ -45,12 +45,12 @@ Not to
     destpath/sourcepath/one.txt
     destpath/sourcepath/two.txt
 
-If you are familiar with ` + "`" + `rsync` + "`" + `, rclone always works as if you had
+If you are familiar with ` + "`rsync`" + `, rclone always works as if you had
 written a trailing / - meaning "copy the contents of this directory".
 This applies to all commands and whether you are talking about the
 source or destination.
 
-See the ` + "`" + `--no-traverse` + "`" + ` option for controlling whether rclone lists
+See the ` + "`--no-traverse`" + ` option for controlling whether rclone lists
 the destination directory or not.
 `,
 	Run: func(command *cobra.Command, args []string) {

cmd/copyto/copyto.go

@@ -0,0 +1,53 @@
+package copyto
+
+import (
+	"github.com/ncw/rclone/cmd"
+	"github.com/ncw/rclone/fs"
+	"github.com/spf13/cobra"
+)
+
+func init() {
+	cmd.Root.AddCommand(commandDefintion)
+}
+
+var commandDefintion = &cobra.Command{
+	Use:   "copyto source:path dest:path",
+	Short: `Copy files from source to dest, skipping already copied`,
+	Long: `
+If source:path is a file or directory then it copies it to a file or
+directory named dest:path.
+
+This can be used to upload single files to other than their current
+name.  If the source is a directory then it acts exactly like the copy
+command.
+
+So
+
+    rclone copyto src dst
+
+where src and dst are rclone paths, either remote:path or
+/path/to/local or C:\windows\path\if\on\windows.
+
+This will:
+
+    if src is file
+        copy it to dst, overwriting an existing file if it exists
+    if src is directory
+        copy it to dst, overwriting existing files if they exist
+        see copy command for full details
+
+This doesn't transfer unchanged files, testing by size and
+modification time or MD5SUM.  It doesn't delete files from the
+destination.
+`,
+	Run: func(command *cobra.Command, args []string) {
+		cmd.CheckArgs(2, 2, command, args)
+		fsrc, srcFileName, fdst, dstFileName := cmd.NewFsSrcDstFiles(args)
+		cmd.Run(true, command, func() error {
+			if srcFileName == "" {
+				return fs.CopyDir(fdst, fsrc)
+			}
+			return fs.CopyFile(fdst, fsrc, dstFileName, srcFileName)
+		})
+	},
+}

cmd/move/move.go

@@ -19,14 +19,14 @@ directory. Rclone will error if the source and destination overlap and
 the remote does not support a server side directory move operation.
 
 If no filters are in use and if possible this will server side move
-` + "`" + `source:path` + "`" + ` into ` + "`" + `dest:path` + "`" + `. After this ` + "`" + `source:path` + "`" + ` will no
+` + "`source:path`" + ` into ` + "`dest:path`" + `. After this ` + "`source:path`" + ` will no
 longer longer exist.
 
-Otherwise for each file in ` + "`" + `source:path` + "`" + ` selected by the filters (if
-any) this will move it into ` + "`" + `dest:path` + "`" + `.  If possible a server side
+Otherwise for each file in ` + "`source:path`" + ` selected by the filters (if
+any) this will move it into ` + "`dest:path`" + `.  If possible a server side
 move will be used, otherwise it will copy it (server side if possible)
-into ` + "`" + `dest:path` + "`" + ` then delete the original (if no errors on copy) in
-` + "`" + `source:path` + "`" + `.
+into ` + "`dest:path`" + ` then delete the original (if no errors on copy) in
+` + "`source:path`" + `.
 
 **Important**: Since this can cause data loss, test first with the
 --dry-run flag.

cmd/moveto/moveto.go

@@ -0,0 +1,57 @@
+package moveto
+
+import (
+	"github.com/ncw/rclone/cmd"
+	"github.com/ncw/rclone/fs"
+	"github.com/spf13/cobra"
+)
+
+func init() {
+	cmd.Root.AddCommand(commandDefintion)
+}
+
+var commandDefintion = &cobra.Command{
+	Use:   "moveto source:path dest:path",
+	Short: `Move file or directory from source to dest.`,
+	Long: `
+If source:path is a file or directory then it moves it to a file or
+directory named dest:path.
+
+This can be used to rename files or upload single files to other than
+their existing name.  If the source is a directory then it acts exacty
+like the move command.
+
+So
+
+    rclone moveto src dst
+
+where src and dst are rclone paths, either remote:path or
+/path/to/local or C:\windows\path\if\on\windows.
+
+This will:
+
+    if src is file
+        move it to dst, overwriting an existing file if it exists
+    if src is directory
+        move it to dst, overwriting existing files if they exist
+        see move command for full details
+
+This doesn't transfer unchanged files, testing by size and
+modification time or MD5SUM.  src will be deleted on successful
+transfer.
+
+**Important**: Since this can cause data loss, test first with the
+--dry-run flag.
+`,
+	Run: func(command *cobra.Command, args []string) {
+		cmd.CheckArgs(2, 2, command, args)
+		fsrc, srcFileName, fdst, dstFileName := cmd.NewFsSrcDstFiles(args)
+
+		cmd.Run(true, command, func() error {
+			if srcFileName == "" {
+				return fs.MoveDir(fdst, fsrc)
+			}
+			return fs.MoveFile(fdst, fsrc, dstFileName, srcFileName)
+		})
+	},
+}

fs/operations.go

@@ -222,6 +222,17 @@ func removeFailedCopy(dst Object) bool {
 	return true
 }
 
+// Wrapper to override the remote for an object
+type overrideRemoteObject struct {
+	Object
+	remote string
+}
+
+// Remote returns the overriden remote name
+func (o *overrideRemoteObject) Remote() string {
+	return o.remote
+}
+
 // Copy src object to dst or f if nil.  If dst is nil then it uses
 // remote as the name of the new object.
 func Copy(f Fs, dst Object, remote string, src Object) (err error) {
@@ -260,12 +271,13 @@ func Copy(f Fs, dst Object, remote string, src Object) (err error) {
 
 				in := NewAccount(in0, src) // account the transfer
 
+				wrappedSrc := &overrideRemoteObject{Object: src, remote: remote}
 				if doUpdate {
 					actionTaken = "Copied (replaced existing)"
-					err = dst.Update(in, src)
+					err = dst.Update(in, wrappedSrc)
 				} else {
 					actionTaken = "Copied (new)"
-					dst, err = f.Put(in, src)
+					dst, err = f.Put(in, wrappedSrc)
 				}
 				closeErr := in.Close()
 				if err == nil {
@@ -1186,3 +1198,43 @@ func Rmdirs(f Fs) error {
 	}
 	return nil
 }
+
+// moveOrCopyFile moves or copies a single file possibly to a new name
+func moveOrCopyFile(fdst Fs, fsrc Fs, dstFileName string, srcFileName string, cp bool) (err error) {
+	// Choose operations
+	Op := Move
+	if cp {
+		Op = Copy
+	}
+
+	// Find src object
+	srcObj, err := fsrc.NewObject(srcFileName)
+	if err != nil {
+		return err
+	}
+
+	// Find dst object if it exists
+	dstObj, err := fdst.NewObject(dstFileName)
+	if err == ErrorObjectNotFound {
+		dstObj = nil
+	} else if err != nil {
+		return err
+	}
+
+	if NeedTransfer(dstObj, srcObj) {
+		return Op(fdst, dstObj, dstFileName, srcObj)
+	} else if !cp {
+		return DeleteFile(srcObj)
+	}
+	return nil
+}
+
+// MoveFile moves a single file possibly to a new name
+func MoveFile(fdst Fs, fsrc Fs, dstFileName string, srcFileName string) (err error) {
+	return moveOrCopyFile(fdst, fsrc, dstFileName, srcFileName, false)
+}
+
+// CopyFile moves a single file possibly to a new name
+func CopyFile(fdst Fs, fsrc Fs, dstFileName string, srcFileName string) (err error) {
+	return moveOrCopyFile(fdst, fsrc, dstFileName, srcFileName, true)
+}

fs/operations_test.go

@@ -697,3 +697,48 @@ func TestRmdirs(t *testing.T) {
 	)
 
 }
+
+func TestMoveFile(t *testing.T) {
+	r := NewRun(t)
+	defer r.Finalise()
+
+	file1 := r.WriteFile("file1", "file1 contents", t1)
+	fstest.CheckItems(t, r.flocal, file1)
+
+	file2 := file1
+	file2.Path = "sub/file2"
+
+	err := fs.MoveFile(r.fremote, r.flocal, file2.Path, file1.Path)
+	require.NoError(t, err)
+	fstest.CheckItems(t, r.flocal)
+	fstest.CheckItems(t, r.fremote, file2)
+
+	r.WriteFile("file1", "file1 contents", t1)
+	fstest.CheckItems(t, r.flocal, file1)
+
+	err = fs.MoveFile(r.fremote, r.flocal, file2.Path, file1.Path)
+	require.NoError(t, err)
+	fstest.CheckItems(t, r.flocal)
+	fstest.CheckItems(t, r.fremote, file2)
+}
+
+func TestCopyFile(t *testing.T) {
+	r := NewRun(t)
+	defer r.Finalise()
+
+	file1 := r.WriteFile("file1", "file1 contents", t1)
+	fstest.CheckItems(t, r.flocal, file1)
+
+	file2 := file1
+	file2.Path = "sub/file2"
+
+	err := fs.CopyFile(r.fremote, r.flocal, file2.Path, file1.Path)
+	require.NoError(t, err)
+	fstest.CheckItems(t, r.flocal, file1)
+	fstest.CheckItems(t, r.fremote, file2)
+
+	err = fs.CopyFile(r.fremote, r.flocal, file2.Path, file1.Path)
+	require.NoError(t, err)
+	fstest.CheckItems(t, r.flocal, file1)
+	fstest.CheckItems(t, r.fremote, file2)
+}

fs/sync.go

@@ -120,11 +120,12 @@ func (s *syncCopyMove) readDstFiles() {
 	s.dstFilesResult <- err
 }
 
-// Check to see if src needs to be copied to dst and if so puts it in out
+// NeedTransfer checks to see if src needs to be copied to dst using
+// the current config.
 //
-// Returns a flag which indicates whether the file needs to be transferred or not.
-func (s *syncCopyMove) checkOne(pair ObjectPair) bool {
-	src, dst := pair.src, pair.dst
+// Returns a flag which indicates whether the file needs to be
+// transferred or not.
+func NeedTransfer(dst, src Object) bool {
 	if dst == nil {
 		Debug(src, "Couldn't find file - need to transfer")
 		return true
@@ -213,7 +214,7 @@ func (s *syncCopyMove) pairChecker(in ObjectPairChan, out ObjectPairChan, wg *sy
 			Stats.Checking(src.Remote())
 			// Check to see if can store this
 			if src.Storable() {
-				if s.checkOne(pair) {
+				if NeedTransfer(pair.dst, pair.src) {
 					out <- pair
 				} else {
 					// If moving need to delete the files we don't need to copy