operations: implement operations remote control commands

fs/operations/rc.go

@@ -0,0 +1,260 @@
+package operations
+
+import (
+	"strings"
+
+	"github.com/ncw/rclone/fs"
+	"github.com/ncw/rclone/fs/rc"
+	"github.com/pkg/errors"
+)
+
+func init() {
+	rc.Add(rc.Call{
+		Path:         "operations/list",
+		AuthRequired: true,
+		Fn:           rcList,
+		Title:        "List the given remote and path in JSON format",
+		Help: `This takes the following parameters
+
+- fs - a remote name string eg "drive:"
+- remote - a path within that remote eg "dir"
+- opt - a dictionary of options to control the listing (optional)
+    - recurse - If set recurse directories
+    - noModTime - If set return modification time
+    - showEncrypted -  If set show decrypted names
+    - showOrigIDs - If set show the IDs for each item if known
+    - showHash - If set return a dictionary of hashes
+
+The result is
+
+- list
+    - This is an array of objects as described in the lsjson command
+
+See the lsjson command for more information on the above and examples.
+`,
+	})
+}
+
+// List the directory
+func rcList(in rc.Params) (out rc.Params, err error) {
+	f, remote, err := rc.GetFsAndRemote(in)
+	if err != nil {
+		return nil, err
+	}
+	var opt ListJSONOpt
+	err = in.GetStruct("opt", &opt)
+	if rc.NotErrParamNotFound(err) {
+		return nil, err
+	}
+	var list = []*ListJSONItem{}
+	err = ListJSON(f, remote, &opt, func(item *ListJSONItem) error {
+		list = append(list, item)
+		return nil
+	})
+	if err != nil {
+		return nil, err
+	}
+	out = make(rc.Params)
+	out["list"] = list
+	return out, nil
+}
+
+func init() {
+	rc.Add(rc.Call{
+		Path:         "operations/about",
+		AuthRequired: true,
+		Fn:           rcAbout,
+		Title:        "Return the space used on the remote",
+		Help: `This takes the following parameters
+
+- fs - a remote name string eg "drive:"
+- remote - a path within that remote eg "dir"
+
+The result is as returned from rclone about --json
+`,
+	})
+}
+
+// About the remote
+func rcAbout(in rc.Params) (out rc.Params, err error) {
+	f, err := rc.GetFs(in)
+	if err != nil {
+		return nil, err
+	}
+	doAbout := f.Features().About
+	if doAbout == nil {
+		return nil, errors.Errorf("%v doesn't support about", f)
+	}
+	u, err := doAbout()
+	if err != nil {
+		return nil, errors.Wrap(err, "about call failed")
+	}
+	err = rc.Reshape(&out, u)
+	if err != nil {
+		return nil, errors.Wrap(err, "about Reshape failed")
+	}
+	return out, nil
+}
+
+func init() {
+	for _, copy := range []bool{false, true} {
+		copy := copy
+		name := "Move"
+		if copy {
+			name = "Copy"
+		}
+		rc.Add(rc.Call{
+			Path:         "operations/" + strings.ToLower(name) + "file",
+			AuthRequired: true,
+			Fn: func(in rc.Params) (rc.Params, error) {
+				return rcMoveOrCopyFile(in, copy)
+			},
+			Title: name + " a file from source remote to destination remote",
+			Help: `This takes the following parameters
+
+- srcFs - a remote name string eg "drive:" for the source
+- srcRemote - a path within that remote eg "file.txt" for the source
+- dstFs - a remote name string eg "drive2:" for the destination
+- dstRemote - a path within that remote eg "file2.txt" for the destination
+
+This returns
+- jobid - ID of async job to query with job/status
+`,
+		})
+	}
+}
+
+// Copy a file
+func rcMoveOrCopyFile(in rc.Params, cp bool) (out rc.Params, err error) {
+	srcFs, srcRemote, err := rc.GetFsAndRemoteNamed(in, "srcFs", "srcRemote")
+	if err != nil {
+		return nil, err
+	}
+	dstFs, dstRemote, err := rc.GetFsAndRemoteNamed(in, "dstFs", "dstRemote")
+	if err != nil {
+		return nil, err
+	}
+	return nil, moveOrCopyFile(dstFs, srcFs, dstRemote, srcRemote, cp)
+}
+
+func init() {
+	for _, op := range []struct {
+		name     string
+		title    string
+		help     string
+		noRemote bool
+	}{
+		{name: "mkdir", title: "Make a destination directory or container"},
+		{name: "rmdir", title: "Remove an empty directory or container"},
+		{name: "purge", title: "Remove a directory or container and all of its contents"},
+		{name: "rmdirs", title: "Remove all the empty directories in the path", help: "- leaveRoot - boolean, set to true not to delete the root\n"},
+		{name: "delete", title: "Remove files in the path", noRemote: true},
+		{name: "deletefile", title: "Remove the single file pointed to"},
+		{name: "copyurl", title: "Copy the URL to the object", help: "- url - string, URL to read from\n"},
+		{name: "cleanup", title: "Remove trashed files in the remote or path", noRemote: true},
+	} {
+		op := op
+		remote := "- remote - a path within that remote eg \"dir\"\n"
+		if op.noRemote {
+			remote = ""
+		}
+		rc.Add(rc.Call{
+			Path:         "operations/" + op.name,
+			AuthRequired: true,
+			Fn: func(in rc.Params) (rc.Params, error) {
+				return rcSingleCommand(in, op.name, op.noRemote)
+			},
+			Title: op.title,
+			Help: `This takes the following parameters
+
+- fs - a remote name string eg "drive:"
+` + remote + op.help + `
+See the [` + op.name + ` command](/commands/rclone_` + op.name + `/) command for more information on the above.
+`,
+		})
+	}
+}
+
+// Mkdir a directory
+func rcSingleCommand(in rc.Params, name string, noRemote bool) (out rc.Params, err error) {
+	var (
+		f      fs.Fs
+		remote string
+	)
+	if noRemote {
+		f, err = rc.GetFs(in)
+	} else {
+		f, remote, err = rc.GetFsAndRemote(in)
+	}
+	if err != nil {
+		return nil, err
+	}
+	switch name {
+	case "mkdir":
+		return nil, Mkdir(f, remote)
+	case "rmdir":
+		return nil, Rmdir(f, remote)
+	case "purge":
+		return nil, Purge(f, remote)
+	case "rmdirs":
+		leaveRoot, err := in.GetBool("leaveRoot")
+		if rc.NotErrParamNotFound(err) {
+			return nil, err
+		}
+		return nil, Rmdirs(f, remote, leaveRoot)
+	case "delete":
+		return nil, Delete(f)
+	case "deletefile":
+		o, err := f.NewObject(remote)
+		if err != nil {
+			return nil, err
+		}
+		return nil, DeleteFile(o)
+	case "copyurl":
+		url, err := in.GetString("url")
+		if err != nil {
+			return nil, err
+		}
+		_, err = CopyURL(f, remote, url)
+		return nil, err
+	case "cleanup":
+		return nil, CleanUp(f)
+	}
+	panic("unknown rcSingleCommand type")
+}
+
+func init() {
+	rc.Add(rc.Call{
+		Path:         "operations/size",
+		AuthRequired: true,
+		Fn:           rcSize,
+		Title:        "Count the number of bytes and files in remote",
+		Help: `This takes the following parameters
+
+- fs - a remote name string eg "drive:path/to/dir"
+
+Returns
+
+- count - number of files
+- bytes - number of bytes in those files
+
+See the [size command](/commands/rclone_size/) command for more information on the above.
+`,
+	})
+}
+
+// Mkdir a directory
+func rcSize(in rc.Params) (out rc.Params, err error) {
+	f, err := rc.GetFs(in)
+	if err != nil {
+		return nil, err
+	}
+	count, bytes, err := Count(f)
+	if err != nil {
+		return nil, err
+	}
+	out = make(rc.Params)
+	out["count"] = count
+	out["bytes"] = bytes
+	return out, nil
+}

fs/operations/rc_test.go

@@ -0,0 +1,358 @@
+package operations_test
+
+import (
+	"net/http"
+	"net/http/httptest"
+	"testing"
+	"time"
+
+	"github.com/ncw/rclone/fs"
+	"github.com/ncw/rclone/fs/operations"
+	"github.com/ncw/rclone/fs/rc"
+	"github.com/ncw/rclone/fstest"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func rcNewRun(t *testing.T, method string) (*fstest.Run, *rc.Call) {
+	if *fstest.RemoteName != "" {
+		t.Skip("Skipping test on non local remote")
+	}
+	r := fstest.NewRun(t)
+	call := rc.Calls.Get(method)
+	assert.NotNil(t, call)
+	rc.PutCachedFs(r.LocalName, r.Flocal)
+	rc.PutCachedFs(r.FremoteName, r.Fremote)
+	return r, call
+}
+
+// operations/about: Return the space used on the remote
+func TestRcAbout(t *testing.T) {
+	r, call := rcNewRun(t, "operations/about")
+	defer r.Finalise()
+	r.Mkdir(r.Fremote)
+
+	// Will get an error if remote doesn't support About
+	expectedErr := r.Fremote.Features().About == nil
+
+	in := rc.Params{
+		"fs": r.FremoteName,
+	}
+	out, err := call.Fn(in)
+	if expectedErr {
+		assert.Error(t, err)
+		return
+	}
+	require.NoError(t, err)
+
+	// Can't really check the output much!
+	assert.NotEqual(t, int64(0), out["Total"])
+}
+
+// operations/cleanup: Remove trashed files in the remote or path
+func TestRcCleanup(t *testing.T) {
+	r, call := rcNewRun(t, "operations/cleanup")
+	defer r.Finalise()
+
+	in := rc.Params{
+		"fs": r.LocalName,
+	}
+	out, err := call.Fn(in)
+	require.Error(t, err)
+	assert.Equal(t, rc.Params(nil), out)
+	assert.Contains(t, err.Error(), "doesn't support cleanup")
+}
+
+// operations/copyfile: Copy a file from source remote to destination remote
+func TestRcCopyfile(t *testing.T) {
+	r, call := rcNewRun(t, "operations/copyfile")
+	defer r.Finalise()
+	file1 := r.WriteFile("file1", "file1 contents", t1)
+	r.Mkdir(r.Fremote)
+	fstest.CheckItems(t, r.Flocal, file1)
+	fstest.CheckItems(t, r.Fremote)
+
+	in := rc.Params{
+		"srcFs":     r.LocalName,
+		"srcRemote": "file1",
+		"dstFs":     r.FremoteName,
+		"dstRemote": "file1-renamed",
+	}
+	out, err := call.Fn(in)
+	require.NoError(t, err)
+	assert.Equal(t, rc.Params(nil), out)
+
+	fstest.CheckItems(t, r.Flocal, file1)
+	file1.Path = "file1-renamed"
+	fstest.CheckItems(t, r.Fremote, file1)
+}
+
+// operations/copyurl: Copy the URL to the object
+func TestRcCopyurl(t *testing.T) {
+	r, call := rcNewRun(t, "operations/copyurl")
+	defer r.Finalise()
+	contents := "file1 contents\n"
+	file1 := r.WriteFile("file1", contents, t1)
+	r.Mkdir(r.Fremote)
+	fstest.CheckItems(t, r.Fremote)
+
+	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		_, err := w.Write([]byte(contents))
+		assert.NoError(t, err)
+	}))
+	defer ts.Close()
+
+	in := rc.Params{
+		"fs":     r.FremoteName,
+		"remote": "file1",
+		"url":    ts.URL,
+	}
+	out, err := call.Fn(in)
+	require.NoError(t, err)
+	assert.Equal(t, rc.Params(nil), out)
+
+	fstest.CheckListingWithPrecision(t, r.Fremote, []fstest.Item{file1}, nil, fs.ModTimeNotSupported)
+}
+
+// operations/delete: Remove files in the path
+func TestRcDelete(t *testing.T) {
+	r, call := rcNewRun(t, "operations/delete")
+	defer r.Finalise()
+
+	file1 := r.WriteObject("small", "1234567890", t2)                                                                                           // 10 bytes
+	file2 := r.WriteObject("medium", "------------------------------------------------------------", t1)                                        // 60 bytes
+	file3 := r.WriteObject("large", "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA", t1) // 100 bytes
+	fstest.CheckItems(t, r.Fremote, file1, file2, file3)
+
+	in := rc.Params{
+		"fs": r.FremoteName,
+	}
+	out, err := call.Fn(in)
+	require.NoError(t, err)
+	assert.Equal(t, rc.Params(nil), out)
+
+	fstest.CheckItems(t, r.Fremote)
+}
+
+// operations/deletefile: Remove the single file pointed to
+func TestRcDeletefile(t *testing.T) {
+	r, call := rcNewRun(t, "operations/deletefile")
+	defer r.Finalise()
+
+	file1 := r.WriteObject("small", "1234567890", t2)                                                    // 10 bytes
+	file2 := r.WriteObject("medium", "------------------------------------------------------------", t1) // 60 bytes
+	fstest.CheckItems(t, r.Fremote, file1, file2)
+
+	in := rc.Params{
+		"fs":     r.FremoteName,
+		"remote": "small",
+	}
+	out, err := call.Fn(in)
+	require.NoError(t, err)
+	assert.Equal(t, rc.Params(nil), out)
+
+	fstest.CheckItems(t, r.Fremote, file2)
+}
+
+// operations/list: List the given remote and path in JSON format
+func TestRcList(t *testing.T) {
+	r, call := rcNewRun(t, "operations/list")
+	defer r.Finalise()
+
+	file1 := r.WriteObject("a", "a", t1)
+	file2 := r.WriteObject("subdir/b", "bb", t2)
+
+	fstest.CheckItems(t, r.Fremote, file1, file2)
+
+	in := rc.Params{
+		"fs":     r.FremoteName,
+		"remote": "",
+	}
+	out, err := call.Fn(in)
+	require.NoError(t, err)
+
+	list := out["list"].([]*operations.ListJSONItem)
+	assert.Equal(t, 2, len(list))
+
+	checkFile1 := func(got *operations.ListJSONItem) {
+		assert.WithinDuration(t, t1, time.Time(got.ModTime), time.Second)
+		assert.Equal(t, "a", got.Path)
+		assert.Equal(t, "a", got.Name)
+		assert.Equal(t, int64(1), got.Size)
+		assert.Equal(t, "application/octet-stream", got.MimeType)
+		assert.Equal(t, false, got.IsDir)
+	}
+	checkFile1(list[0])
+
+	checkSubdir := func(got *operations.ListJSONItem) {
+		assert.Equal(t, "subdir", got.Path)
+		assert.Equal(t, "subdir", got.Name)
+		assert.Equal(t, int64(-1), got.Size)
+		assert.Equal(t, "inode/directory", got.MimeType)
+		assert.Equal(t, true, got.IsDir)
+	}
+	checkSubdir(list[1])
+
+	in = rc.Params{
+		"fs":     r.FremoteName,
+		"remote": "",
+		"opt": rc.Params{
+			"recurse": true,
+		},
+	}
+	out, err = call.Fn(in)
+	require.NoError(t, err)
+
+	list = out["list"].([]*operations.ListJSONItem)
+	assert.Equal(t, 3, len(list))
+	checkFile1(list[0])
+	checkSubdir(list[1])
+
+	checkFile2 := func(got *operations.ListJSONItem) {
+		assert.WithinDuration(t, t2, time.Time(got.ModTime), time.Second)
+		assert.Equal(t, "subdir/b", got.Path)
+		assert.Equal(t, "b", got.Name)
+		assert.Equal(t, int64(2), got.Size)
+		assert.Equal(t, "application/octet-stream", got.MimeType)
+		assert.Equal(t, false, got.IsDir)
+	}
+	checkFile2(list[2])
+}
+
+// operations/mkdir: Make a destination directory or container
+func TestRcMkdir(t *testing.T) {
+	r, call := rcNewRun(t, "operations/mkdir")
+	defer r.Finalise()
+	r.Mkdir(r.Fremote)
+
+	fstest.CheckListingWithPrecision(t, r.Fremote, []fstest.Item{}, []string{}, fs.GetModifyWindow(r.Fremote))
+
+	in := rc.Params{
+		"fs":     r.FremoteName,
+		"remote": "subdir",
+	}
+	out, err := call.Fn(in)
+	require.NoError(t, err)
+	assert.Equal(t, rc.Params(nil), out)
+
+	fstest.CheckListingWithPrecision(t, r.Fremote, []fstest.Item{}, []string{"subdir"}, fs.GetModifyWindow(r.Fremote))
+}
+
+// operations/movefile: Move a file from source remote to destination remote
+func TestRcMovefile(t *testing.T) {
+	r, call := rcNewRun(t, "operations/movefile")
+	defer r.Finalise()
+	file1 := r.WriteFile("file1", "file1 contents", t1)
+	r.Mkdir(r.Fremote)
+	fstest.CheckItems(t, r.Flocal, file1)
+	fstest.CheckItems(t, r.Fremote)
+
+	in := rc.Params{
+		"srcFs":     r.LocalName,
+		"srcRemote": "file1",
+		"dstFs":     r.FremoteName,
+		"dstRemote": "file1-renamed",
+	}
+	out, err := call.Fn(in)
+	require.NoError(t, err)
+	assert.Equal(t, rc.Params(nil), out)
+
+	fstest.CheckItems(t, r.Flocal)
+	file1.Path = "file1-renamed"
+	fstest.CheckItems(t, r.Fremote, file1)
+}
+
+// operations/purge: Remove a directory or container and all of its contents
+func TestRcPurge(t *testing.T) {
+	r, call := rcNewRun(t, "operations/purge")
+	defer r.Finalise()
+	file1 := r.WriteObject("subdir/file1", "subdir/file1 contents", t1)
+
+	fstest.CheckListingWithPrecision(t, r.Fremote, []fstest.Item{file1}, []string{"subdir"}, fs.GetModifyWindow(r.Fremote))
+
+	in := rc.Params{
+		"fs":     r.FremoteName,
+		"remote": "subdir",
+	}
+	out, err := call.Fn(in)
+	require.NoError(t, err)
+	assert.Equal(t, rc.Params(nil), out)
+
+	fstest.CheckListingWithPrecision(t, r.Fremote, []fstest.Item{}, []string{}, fs.GetModifyWindow(r.Fremote))
+}
+
+// operations/rmdir: Remove an empty directory or container
+func TestRcRmdir(t *testing.T) {
+	r, call := rcNewRun(t, "operations/rmdir")
+	defer r.Finalise()
+	r.Mkdir(r.Fremote)
+	assert.NoError(t, r.Fremote.Mkdir("subdir"))
+
+	fstest.CheckListingWithPrecision(t, r.Fremote, []fstest.Item{}, []string{"subdir"}, fs.GetModifyWindow(r.Fremote))
+
+	in := rc.Params{
+		"fs":     r.FremoteName,
+		"remote": "subdir",
+	}
+	out, err := call.Fn(in)
+	require.NoError(t, err)
+	assert.Equal(t, rc.Params(nil), out)
+
+	fstest.CheckListingWithPrecision(t, r.Fremote, []fstest.Item{}, []string{}, fs.GetModifyWindow(r.Fremote))
+}
+
+// operations/rmdirs: Remove all the empty directories in the path
+func TestRcRmdirs(t *testing.T) {
+	r, call := rcNewRun(t, "operations/rmdirs")
+	defer r.Finalise()
+	r.Mkdir(r.Fremote)
+	assert.NoError(t, r.Fremote.Mkdir("subdir"))
+	assert.NoError(t, r.Fremote.Mkdir("subdir/subsubdir"))
+
+	fstest.CheckListingWithPrecision(t, r.Fremote, []fstest.Item{}, []string{"subdir", "subdir/subsubdir"}, fs.GetModifyWindow(r.Fremote))
+
+	in := rc.Params{
+		"fs":     r.FremoteName,
+		"remote": "subdir",
+	}
+	out, err := call.Fn(in)
+	require.NoError(t, err)
+	assert.Equal(t, rc.Params(nil), out)
+
+	fstest.CheckListingWithPrecision(t, r.Fremote, []fstest.Item{}, []string{}, fs.GetModifyWindow(r.Fremote))
+
+	assert.NoError(t, r.Fremote.Mkdir("subdir"))
+	assert.NoError(t, r.Fremote.Mkdir("subdir/subsubdir"))
+
+	in = rc.Params{
+		"fs":        r.FremoteName,
+		"remote":    "subdir",
+		"leaveRoot": true,
+	}
+	out, err = call.Fn(in)
+	require.NoError(t, err)
+	assert.Equal(t, rc.Params(nil), out)
+
+	fstest.CheckListingWithPrecision(t, r.Fremote, []fstest.Item{}, []string{"subdir"}, fs.GetModifyWindow(r.Fremote))
+
+}
+
+// operations/size: Count the number of bytes and files in remote
+func TestRcSize(t *testing.T) {
+	r, call := rcNewRun(t, "operations/size")
+	defer r.Finalise()
+	file1 := r.WriteObject("small", "1234567890", t2)                                                           // 10 bytes
+	file2 := r.WriteObject("subdir/medium", "------------------------------------------------------------", t1) // 60 bytes
+	file3 := r.WriteObject("subdir/subsubdir/large", "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA", t1)  // 50 bytes
+	fstest.CheckItems(t, r.Fremote, file1, file2, file3)
+
+	in := rc.Params{
+		"fs": r.FremoteName,
+	}
+	out, err := call.Fn(in)
+	require.NoError(t, err)
+	assert.Equal(t, rc.Params{
+		"count": int64(3),
+		"bytes": int64(120),
+	}, out)
+}