From 39283c8a355d0b0206da8c244691bf1e9aba2e65 Mon Sep 17 00:00:00 2001 From: Nick Craig-Wood Date: Fri, 26 Oct 2018 14:51:28 +0100 Subject: [PATCH] operations: implement operations remote control commands --- fs/operations/rc.go | 260 ++++++++++++++++++++++++++++ fs/operations/rc_test.go | 358 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 618 insertions(+) create mode 100644 fs/operations/rc.go create mode 100644 fs/operations/rc_test.go diff --git a/fs/operations/rc.go b/fs/operations/rc.go new file mode 100644 index 000000000..3b27278ea --- /dev/null +++ b/fs/operations/rc.go @@ -0,0 +1,260 @@ +package operations + +import ( + "strings" + + "github.com/ncw/rclone/fs" + "github.com/ncw/rclone/fs/rc" + "github.com/pkg/errors" +) + +func init() { + rc.Add(rc.Call{ + Path: "operations/list", + AuthRequired: true, + Fn: rcList, + Title: "List the given remote and path in JSON format", + Help: `This takes the following parameters + +- fs - a remote name string eg "drive:" +- remote - a path within that remote eg "dir" +- opt - a dictionary of options to control the listing (optional) + - recurse - If set recurse directories + - noModTime - If set return modification time + - showEncrypted - If set show decrypted names + - showOrigIDs - If set show the IDs for each item if known + - showHash - If set return a dictionary of hashes + +The result is + +- list + - This is an array of objects as described in the lsjson command + +See the lsjson command for more information on the above and examples. +`, + }) +} + +// List the directory +func rcList(in rc.Params) (out rc.Params, err error) { + f, remote, err := rc.GetFsAndRemote(in) + if err != nil { + return nil, err + } + var opt ListJSONOpt + err = in.GetStruct("opt", &opt) + if rc.NotErrParamNotFound(err) { + return nil, err + } + var list = []*ListJSONItem{} + err = ListJSON(f, remote, &opt, func(item *ListJSONItem) error { + list = append(list, item) + return nil + }) + if err != nil { + return nil, err + } + out = make(rc.Params) + out["list"] = list + return out, nil +} + +func init() { + rc.Add(rc.Call{ + Path: "operations/about", + AuthRequired: true, + Fn: rcAbout, + Title: "Return the space used on the remote", + Help: `This takes the following parameters + +- fs - a remote name string eg "drive:" +- remote - a path within that remote eg "dir" + +The result is as returned from rclone about --json +`, + }) +} + +// About the remote +func rcAbout(in rc.Params) (out rc.Params, err error) { + f, err := rc.GetFs(in) + if err != nil { + return nil, err + } + doAbout := f.Features().About + if doAbout == nil { + return nil, errors.Errorf("%v doesn't support about", f) + } + u, err := doAbout() + if err != nil { + return nil, errors.Wrap(err, "about call failed") + } + err = rc.Reshape(&out, u) + if err != nil { + return nil, errors.Wrap(err, "about Reshape failed") + } + return out, nil +} + +func init() { + for _, copy := range []bool{false, true} { + copy := copy + name := "Move" + if copy { + name = "Copy" + } + rc.Add(rc.Call{ + Path: "operations/" + strings.ToLower(name) + "file", + AuthRequired: true, + Fn: func(in rc.Params) (rc.Params, error) { + return rcMoveOrCopyFile(in, copy) + }, + Title: name + " a file from source remote to destination remote", + Help: `This takes the following parameters + +- srcFs - a remote name string eg "drive:" for the source +- srcRemote - a path within that remote eg "file.txt" for the source +- dstFs - a remote name string eg "drive2:" for the destination +- dstRemote - a path within that remote eg "file2.txt" for the destination + +This returns +- jobid - ID of async job to query with job/status +`, + }) + } +} + +// Copy a file +func rcMoveOrCopyFile(in rc.Params, cp bool) (out rc.Params, err error) { + srcFs, srcRemote, err := rc.GetFsAndRemoteNamed(in, "srcFs", "srcRemote") + if err != nil { + return nil, err + } + dstFs, dstRemote, err := rc.GetFsAndRemoteNamed(in, "dstFs", "dstRemote") + if err != nil { + return nil, err + } + return nil, moveOrCopyFile(dstFs, srcFs, dstRemote, srcRemote, cp) +} + +func init() { + for _, op := range []struct { + name string + title string + help string + noRemote bool + }{ + {name: "mkdir", title: "Make a destination directory or container"}, + {name: "rmdir", title: "Remove an empty directory or container"}, + {name: "purge", title: "Remove a directory or container and all of its contents"}, + {name: "rmdirs", title: "Remove all the empty directories in the path", help: "- leaveRoot - boolean, set to true not to delete the root\n"}, + {name: "delete", title: "Remove files in the path", noRemote: true}, + {name: "deletefile", title: "Remove the single file pointed to"}, + {name: "copyurl", title: "Copy the URL to the object", help: "- url - string, URL to read from\n"}, + {name: "cleanup", title: "Remove trashed files in the remote or path", noRemote: true}, + } { + op := op + remote := "- remote - a path within that remote eg \"dir\"\n" + if op.noRemote { + remote = "" + } + rc.Add(rc.Call{ + Path: "operations/" + op.name, + AuthRequired: true, + Fn: func(in rc.Params) (rc.Params, error) { + return rcSingleCommand(in, op.name, op.noRemote) + }, + Title: op.title, + Help: `This takes the following parameters + +- fs - a remote name string eg "drive:" +` + remote + op.help + ` +See the [` + op.name + ` command](/commands/rclone_` + op.name + `/) command for more information on the above. +`, + }) + } +} + +// Mkdir a directory +func rcSingleCommand(in rc.Params, name string, noRemote bool) (out rc.Params, err error) { + var ( + f fs.Fs + remote string + ) + if noRemote { + f, err = rc.GetFs(in) + } else { + f, remote, err = rc.GetFsAndRemote(in) + } + if err != nil { + return nil, err + } + switch name { + case "mkdir": + return nil, Mkdir(f, remote) + case "rmdir": + return nil, Rmdir(f, remote) + case "purge": + return nil, Purge(f, remote) + case "rmdirs": + leaveRoot, err := in.GetBool("leaveRoot") + if rc.NotErrParamNotFound(err) { + return nil, err + } + return nil, Rmdirs(f, remote, leaveRoot) + case "delete": + return nil, Delete(f) + case "deletefile": + o, err := f.NewObject(remote) + if err != nil { + return nil, err + } + return nil, DeleteFile(o) + case "copyurl": + url, err := in.GetString("url") + if err != nil { + return nil, err + } + _, err = CopyURL(f, remote, url) + return nil, err + case "cleanup": + return nil, CleanUp(f) + } + panic("unknown rcSingleCommand type") +} + +func init() { + rc.Add(rc.Call{ + Path: "operations/size", + AuthRequired: true, + Fn: rcSize, + Title: "Count the number of bytes and files in remote", + Help: `This takes the following parameters + +- fs - a remote name string eg "drive:path/to/dir" + +Returns + +- count - number of files +- bytes - number of bytes in those files + +See the [size command](/commands/rclone_size/) command for more information on the above. +`, + }) +} + +// Mkdir a directory +func rcSize(in rc.Params) (out rc.Params, err error) { + f, err := rc.GetFs(in) + if err != nil { + return nil, err + } + count, bytes, err := Count(f) + if err != nil { + return nil, err + } + out = make(rc.Params) + out["count"] = count + out["bytes"] = bytes + return out, nil +} diff --git a/fs/operations/rc_test.go b/fs/operations/rc_test.go new file mode 100644 index 000000000..6dedf7490 --- /dev/null +++ b/fs/operations/rc_test.go @@ -0,0 +1,358 @@ +package operations_test + +import ( + "net/http" + "net/http/httptest" + "testing" + "time" + + "github.com/ncw/rclone/fs" + "github.com/ncw/rclone/fs/operations" + "github.com/ncw/rclone/fs/rc" + "github.com/ncw/rclone/fstest" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func rcNewRun(t *testing.T, method string) (*fstest.Run, *rc.Call) { + if *fstest.RemoteName != "" { + t.Skip("Skipping test on non local remote") + } + r := fstest.NewRun(t) + call := rc.Calls.Get(method) + assert.NotNil(t, call) + rc.PutCachedFs(r.LocalName, r.Flocal) + rc.PutCachedFs(r.FremoteName, r.Fremote) + return r, call +} + +// operations/about: Return the space used on the remote +func TestRcAbout(t *testing.T) { + r, call := rcNewRun(t, "operations/about") + defer r.Finalise() + r.Mkdir(r.Fremote) + + // Will get an error if remote doesn't support About + expectedErr := r.Fremote.Features().About == nil + + in := rc.Params{ + "fs": r.FremoteName, + } + out, err := call.Fn(in) + if expectedErr { + assert.Error(t, err) + return + } + require.NoError(t, err) + + // Can't really check the output much! + assert.NotEqual(t, int64(0), out["Total"]) +} + +// operations/cleanup: Remove trashed files in the remote or path +func TestRcCleanup(t *testing.T) { + r, call := rcNewRun(t, "operations/cleanup") + defer r.Finalise() + + in := rc.Params{ + "fs": r.LocalName, + } + out, err := call.Fn(in) + require.Error(t, err) + assert.Equal(t, rc.Params(nil), out) + assert.Contains(t, err.Error(), "doesn't support cleanup") +} + +// operations/copyfile: Copy a file from source remote to destination remote +func TestRcCopyfile(t *testing.T) { + r, call := rcNewRun(t, "operations/copyfile") + defer r.Finalise() + file1 := r.WriteFile("file1", "file1 contents", t1) + r.Mkdir(r.Fremote) + fstest.CheckItems(t, r.Flocal, file1) + fstest.CheckItems(t, r.Fremote) + + in := rc.Params{ + "srcFs": r.LocalName, + "srcRemote": "file1", + "dstFs": r.FremoteName, + "dstRemote": "file1-renamed", + } + out, err := call.Fn(in) + require.NoError(t, err) + assert.Equal(t, rc.Params(nil), out) + + fstest.CheckItems(t, r.Flocal, file1) + file1.Path = "file1-renamed" + fstest.CheckItems(t, r.Fremote, file1) +} + +// operations/copyurl: Copy the URL to the object +func TestRcCopyurl(t *testing.T) { + r, call := rcNewRun(t, "operations/copyurl") + defer r.Finalise() + contents := "file1 contents\n" + file1 := r.WriteFile("file1", contents, t1) + r.Mkdir(r.Fremote) + fstest.CheckItems(t, r.Fremote) + + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + _, err := w.Write([]byte(contents)) + assert.NoError(t, err) + })) + defer ts.Close() + + in := rc.Params{ + "fs": r.FremoteName, + "remote": "file1", + "url": ts.URL, + } + out, err := call.Fn(in) + require.NoError(t, err) + assert.Equal(t, rc.Params(nil), out) + + fstest.CheckListingWithPrecision(t, r.Fremote, []fstest.Item{file1}, nil, fs.ModTimeNotSupported) +} + +// operations/delete: Remove files in the path +func TestRcDelete(t *testing.T) { + r, call := rcNewRun(t, "operations/delete") + defer r.Finalise() + + file1 := r.WriteObject("small", "1234567890", t2) // 10 bytes + file2 := r.WriteObject("medium", "------------------------------------------------------------", t1) // 60 bytes + file3 := r.WriteObject("large", "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA", t1) // 100 bytes + fstest.CheckItems(t, r.Fremote, file1, file2, file3) + + in := rc.Params{ + "fs": r.FremoteName, + } + out, err := call.Fn(in) + require.NoError(t, err) + assert.Equal(t, rc.Params(nil), out) + + fstest.CheckItems(t, r.Fremote) +} + +// operations/deletefile: Remove the single file pointed to +func TestRcDeletefile(t *testing.T) { + r, call := rcNewRun(t, "operations/deletefile") + defer r.Finalise() + + file1 := r.WriteObject("small", "1234567890", t2) // 10 bytes + file2 := r.WriteObject("medium", "------------------------------------------------------------", t1) // 60 bytes + fstest.CheckItems(t, r.Fremote, file1, file2) + + in := rc.Params{ + "fs": r.FremoteName, + "remote": "small", + } + out, err := call.Fn(in) + require.NoError(t, err) + assert.Equal(t, rc.Params(nil), out) + + fstest.CheckItems(t, r.Fremote, file2) +} + +// operations/list: List the given remote and path in JSON format +func TestRcList(t *testing.T) { + r, call := rcNewRun(t, "operations/list") + defer r.Finalise() + + file1 := r.WriteObject("a", "a", t1) + file2 := r.WriteObject("subdir/b", "bb", t2) + + fstest.CheckItems(t, r.Fremote, file1, file2) + + in := rc.Params{ + "fs": r.FremoteName, + "remote": "", + } + out, err := call.Fn(in) + require.NoError(t, err) + + list := out["list"].([]*operations.ListJSONItem) + assert.Equal(t, 2, len(list)) + + checkFile1 := func(got *operations.ListJSONItem) { + assert.WithinDuration(t, t1, time.Time(got.ModTime), time.Second) + assert.Equal(t, "a", got.Path) + assert.Equal(t, "a", got.Name) + assert.Equal(t, int64(1), got.Size) + assert.Equal(t, "application/octet-stream", got.MimeType) + assert.Equal(t, false, got.IsDir) + } + checkFile1(list[0]) + + checkSubdir := func(got *operations.ListJSONItem) { + assert.Equal(t, "subdir", got.Path) + assert.Equal(t, "subdir", got.Name) + assert.Equal(t, int64(-1), got.Size) + assert.Equal(t, "inode/directory", got.MimeType) + assert.Equal(t, true, got.IsDir) + } + checkSubdir(list[1]) + + in = rc.Params{ + "fs": r.FremoteName, + "remote": "", + "opt": rc.Params{ + "recurse": true, + }, + } + out, err = call.Fn(in) + require.NoError(t, err) + + list = out["list"].([]*operations.ListJSONItem) + assert.Equal(t, 3, len(list)) + checkFile1(list[0]) + checkSubdir(list[1]) + + checkFile2 := func(got *operations.ListJSONItem) { + assert.WithinDuration(t, t2, time.Time(got.ModTime), time.Second) + assert.Equal(t, "subdir/b", got.Path) + assert.Equal(t, "b", got.Name) + assert.Equal(t, int64(2), got.Size) + assert.Equal(t, "application/octet-stream", got.MimeType) + assert.Equal(t, false, got.IsDir) + } + checkFile2(list[2]) +} + +// operations/mkdir: Make a destination directory or container +func TestRcMkdir(t *testing.T) { + r, call := rcNewRun(t, "operations/mkdir") + defer r.Finalise() + r.Mkdir(r.Fremote) + + fstest.CheckListingWithPrecision(t, r.Fremote, []fstest.Item{}, []string{}, fs.GetModifyWindow(r.Fremote)) + + in := rc.Params{ + "fs": r.FremoteName, + "remote": "subdir", + } + out, err := call.Fn(in) + require.NoError(t, err) + assert.Equal(t, rc.Params(nil), out) + + fstest.CheckListingWithPrecision(t, r.Fremote, []fstest.Item{}, []string{"subdir"}, fs.GetModifyWindow(r.Fremote)) +} + +// operations/movefile: Move a file from source remote to destination remote +func TestRcMovefile(t *testing.T) { + r, call := rcNewRun(t, "operations/movefile") + defer r.Finalise() + file1 := r.WriteFile("file1", "file1 contents", t1) + r.Mkdir(r.Fremote) + fstest.CheckItems(t, r.Flocal, file1) + fstest.CheckItems(t, r.Fremote) + + in := rc.Params{ + "srcFs": r.LocalName, + "srcRemote": "file1", + "dstFs": r.FremoteName, + "dstRemote": "file1-renamed", + } + out, err := call.Fn(in) + require.NoError(t, err) + assert.Equal(t, rc.Params(nil), out) + + fstest.CheckItems(t, r.Flocal) + file1.Path = "file1-renamed" + fstest.CheckItems(t, r.Fremote, file1) +} + +// operations/purge: Remove a directory or container and all of its contents +func TestRcPurge(t *testing.T) { + r, call := rcNewRun(t, "operations/purge") + defer r.Finalise() + file1 := r.WriteObject("subdir/file1", "subdir/file1 contents", t1) + + fstest.CheckListingWithPrecision(t, r.Fremote, []fstest.Item{file1}, []string{"subdir"}, fs.GetModifyWindow(r.Fremote)) + + in := rc.Params{ + "fs": r.FremoteName, + "remote": "subdir", + } + out, err := call.Fn(in) + require.NoError(t, err) + assert.Equal(t, rc.Params(nil), out) + + fstest.CheckListingWithPrecision(t, r.Fremote, []fstest.Item{}, []string{}, fs.GetModifyWindow(r.Fremote)) +} + +// operations/rmdir: Remove an empty directory or container +func TestRcRmdir(t *testing.T) { + r, call := rcNewRun(t, "operations/rmdir") + defer r.Finalise() + r.Mkdir(r.Fremote) + assert.NoError(t, r.Fremote.Mkdir("subdir")) + + fstest.CheckListingWithPrecision(t, r.Fremote, []fstest.Item{}, []string{"subdir"}, fs.GetModifyWindow(r.Fremote)) + + in := rc.Params{ + "fs": r.FremoteName, + "remote": "subdir", + } + out, err := call.Fn(in) + require.NoError(t, err) + assert.Equal(t, rc.Params(nil), out) + + fstest.CheckListingWithPrecision(t, r.Fremote, []fstest.Item{}, []string{}, fs.GetModifyWindow(r.Fremote)) +} + +// operations/rmdirs: Remove all the empty directories in the path +func TestRcRmdirs(t *testing.T) { + r, call := rcNewRun(t, "operations/rmdirs") + defer r.Finalise() + r.Mkdir(r.Fremote) + assert.NoError(t, r.Fremote.Mkdir("subdir")) + assert.NoError(t, r.Fremote.Mkdir("subdir/subsubdir")) + + fstest.CheckListingWithPrecision(t, r.Fremote, []fstest.Item{}, []string{"subdir", "subdir/subsubdir"}, fs.GetModifyWindow(r.Fremote)) + + in := rc.Params{ + "fs": r.FremoteName, + "remote": "subdir", + } + out, err := call.Fn(in) + require.NoError(t, err) + assert.Equal(t, rc.Params(nil), out) + + fstest.CheckListingWithPrecision(t, r.Fremote, []fstest.Item{}, []string{}, fs.GetModifyWindow(r.Fremote)) + + assert.NoError(t, r.Fremote.Mkdir("subdir")) + assert.NoError(t, r.Fremote.Mkdir("subdir/subsubdir")) + + in = rc.Params{ + "fs": r.FremoteName, + "remote": "subdir", + "leaveRoot": true, + } + out, err = call.Fn(in) + require.NoError(t, err) + assert.Equal(t, rc.Params(nil), out) + + fstest.CheckListingWithPrecision(t, r.Fremote, []fstest.Item{}, []string{"subdir"}, fs.GetModifyWindow(r.Fremote)) + +} + +// operations/size: Count the number of bytes and files in remote +func TestRcSize(t *testing.T) { + r, call := rcNewRun(t, "operations/size") + defer r.Finalise() + file1 := r.WriteObject("small", "1234567890", t2) // 10 bytes + file2 := r.WriteObject("subdir/medium", "------------------------------------------------------------", t1) // 60 bytes + file3 := r.WriteObject("subdir/subsubdir/large", "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA", t1) // 50 bytes + fstest.CheckItems(t, r.Fremote, file1, file2, file3) + + in := rc.Params{ + "fs": r.FremoteName, + } + out, err := call.Fn(in) + require.NoError(t, err) + assert.Equal(t, rc.Params{ + "count": int64(3), + "bytes": int64(120), + }, out) +}