From ba09ee18bb8057c59c858261d2117a8e428c937e Mon Sep 17 00:00:00 2001 From: Nick Craig-Wood Date: Wed, 28 Apr 2021 12:58:08 +0100 Subject: [PATCH] librclone: factor into gomobile and internal implementation #4891 This was needed because gomobile can't use a main package wheras this is required to make a normal shared C library. --- librclone/README.md | 21 ++++- librclone/gomobile/gomobile.go | 40 ++++++++++ librclone/librclone.go | 127 +------------------------------ librclone/librclone/librclone.go | 124 ++++++++++++++++++++++++++++++ 4 files changed, 186 insertions(+), 126 deletions(-) create mode 100644 librclone/gomobile/gomobile.go create mode 100644 librclone/librclone/librclone.go diff --git a/librclone/README.md b/librclone/README.md index e6b9e03bf..7fb0973cc 100644 --- a/librclone/README.md +++ b/librclone/README.md @@ -1,7 +1,13 @@ # librclone This directory contains code to build rclone as a C library and the -shims for accessing rclone from C. +shims for accessing rclone from C and other languages. + +**Note** for the moment, the interfaces defined here are experimental +and may change in the future. Eventually they will stabilse and this +notice will be removed. + +## C The shims are a thin wrapper over the rclone RPC. @@ -18,7 +24,7 @@ be `#include`d in `C` programs wishing to use the library. The library will depend on `libdl` and `libpthread`. -## Documentation +### Documentation For documentation see the Go documentation for: @@ -26,6 +32,15 @@ For documentation see the Go documentation for: - [RcloneFinalize](https://pkg.go.dev/github.com/rclone/rclone/librclone#RcloneFinalize) - [RcloneRPC](https://pkg.go.dev/github.com/rclone/rclone/librclone#RcloneRPC) -## C Example +### C Example There is an example program `ctest.c` with Makefile in the `ctest` subdirectory + +## gomobile + +The gomobile subdirectory contains the equivalent of the C binding but +suitable for using with gomobile using something like this. + + gomobile bind -v -target=android github.com/rclone/rclone/librclone/gomobile + + diff --git a/librclone/gomobile/gomobile.go b/librclone/gomobile/gomobile.go new file mode 100644 index 000000000..90bfd033e --- /dev/null +++ b/librclone/gomobile/gomobile.go @@ -0,0 +1,40 @@ +// Package gomobile exports shims for gomobile use +package gomobile + +import ( + "github.com/rclone/rclone/librclone/librclone" + + _ "github.com/rclone/rclone/backend/all" // import all backends + _ "github.com/rclone/rclone/lib/plugin" // import plugins +) + +// RcloneInitialize initializes rclone as a library +func RcloneInitialize() { + librclone.Initialize() +} + +// RcloneFinalize finalizes the library +func RcloneFinalize() { + librclone.Finalize() +} + +// RcloneRPCResult is returned from RcloneRPC +// +// Output will be returned as a serialized JSON object +// Status is a HTTP status return (200=OK anything else fail) +type RcloneRPCResult struct { + Output string + Status int +} + +// RcloneRPC has an interface optimised for gomobile, in particular +// the function signature is valid under gobind rules. +// +// https://pkg.go.dev/golang.org/x/mobile/cmd/gobind#hdr-Type_restrictions +func RcloneRPC(method string, input string) (result *RcloneRPCResult) { //nolint:deadcode + output, status := librclone.RPC(method, input) + return &RcloneRPCResult{ + Output: output, + Status: status, + } +} diff --git a/librclone/librclone.go b/librclone/librclone.go index 40e693bee..c4efc84cd 100644 --- a/librclone/librclone.go +++ b/librclone/librclone.go @@ -28,20 +28,7 @@ struct RcloneRPCResult { import "C" import ( - "context" - "encoding/json" - "fmt" - "net/http" - "runtime" - "strings" - - "github.com/pkg/errors" - "github.com/rclone/rclone/fs" - "github.com/rclone/rclone/fs/accounting" - "github.com/rclone/rclone/fs/config/configfile" - "github.com/rclone/rclone/fs/log" - "github.com/rclone/rclone/fs/rc" - "github.com/rclone/rclone/fs/rc/jobs" + "github.com/rclone/rclone/librclone/librclone" _ "github.com/rclone/rclone/backend/all" // import all backends _ "github.com/rclone/rclone/lib/plugin" // import plugins @@ -51,28 +38,14 @@ import ( // //export RcloneInitialize func RcloneInitialize() { - // A subset of initialisation copied from cmd.go - // Note that we don't want to pull in anything which depends on pflags - - ctx := context.Background() - - // Start the logger - log.InitLogging() - - // Load the config - this may need to be configurable - configfile.Install() - - // Start accounting - accounting.Start(ctx) + librclone.Initialize() } // RcloneFinalize finalizes the library // //export RcloneFinalize func RcloneFinalize() { - // TODO: how to clean up? what happens when rcserver terminates? - // what about unfinished async jobs? - runtime.GC() + librclone.Finalize() } // RcloneRPCResult is returned from RcloneRPC @@ -98,103 +71,11 @@ type RcloneRPCResult struct { //nolint:deadcode // //export RcloneRPC func RcloneRPC(method *C.char, input *C.char) (result C.struct_RcloneRPCResult) { //nolint:golint - output, status := callFunctionJSON(C.GoString(method), C.GoString(input)) + output, status := librclone.RPC(C.GoString(method), C.GoString(input)) result.Output = C.CString(output) result.Status = C.int(status) return result } -// RcloneMobileRPCResult is returned from RcloneMobileRPC -// -// Output will be returned as a serialized JSON object -// Status is a HTTP status return (200=OK anything else fail) -type RcloneMobileRPCResult struct { - Output string - Status int -} - -// RcloneMobileRPC works the same as RcloneRPC but has an interface -// optimised for gomobile, in particular the function signature is -// valid under gobind rules. -// -// https://pkg.go.dev/golang.org/x/mobile/cmd/gobind#hdr-Type_restrictions -func RcloneMobileRPC(method string, input string) (result RcloneMobileRPCResult) { //nolint:deadcode - output, status := callFunctionJSON(method, input) - result.Output = output - result.Status = status - return result -} - -// writeError returns a formatted error string and the status passed in -func writeError(path string, in rc.Params, err error, status int) (string, int) { - fs.Errorf(nil, "rc: %q: error: %v", path, err) - params, status := rc.Error(path, in, err, status) - var w strings.Builder - err = rc.WriteJSON(&w, params) - if err != nil { - // ultimate fallback error - fs.Errorf(nil, "writeError: failed to write JSON output from %#v: %v", in, err) - status = http.StatusInternalServerError - w.Reset() - fmt.Fprintf(&w, `{ - "error": %q, - "path": %q, - "status": %d -}`, err, path, status) - - } - return w.String(), status -} - -// operations/uploadfile and core/command are not supported as they need request or response object -// modified from handlePost in rcserver.go -// call a rc function using JSON to input parameters and output the resulted JSON -func callFunctionJSON(method string, input string) (output string, status int) { - // create a buffer to capture the output - in := make(rc.Params) - err := json.NewDecoder(strings.NewReader(input)).Decode(&in) - if err != nil { - return writeError(method, in, errors.Wrap(err, "failed to read input JSON"), http.StatusBadRequest) - } - - // Find the call - call := rc.Calls.Get(method) - if call == nil { - return writeError(method, in, errors.Errorf("couldn't find method %q", method), http.StatusNotFound) - } - - // TODO: handle these cases - if call.NeedsRequest { - return writeError(method, in, errors.Errorf("method %q needs request, not supported", method), http.StatusNotFound) - // Add the request to RC - //in["_request"] = r - } - if call.NeedsResponse { - return writeError(method, in, errors.Errorf("method %q need response, not supported", method), http.StatusNotFound) - //in["_response"] = w - } - - fs.Debugf(nil, "rc: %q: with parameters %+v", method, in) - - _, out, err := jobs.NewJob(context.Background(), call.Fn, in) - if err != nil { - return writeError(method, in, err, http.StatusInternalServerError) - } - if out == nil { - out = make(rc.Params) - } - - fs.Debugf(nil, "rc: %q: reply %+v: %v", method, out, err) - - var w strings.Builder - err = rc.WriteJSON(&w, out) - if err != nil { - fs.Errorf(nil, "rc: failed to write JSON output: %v", err) - return writeError(method, in, err, http.StatusInternalServerError) - } - - return w.String(), http.StatusOK -} - // do nothing here - necessary for building into a C library func main() {} diff --git a/librclone/librclone/librclone.go b/librclone/librclone/librclone.go new file mode 100644 index 000000000..eba479cbf --- /dev/null +++ b/librclone/librclone/librclone.go @@ -0,0 +1,124 @@ +// Package librclone exports shims for library use +// +// This is the internal implementation which is used for C and +// Gomobile libaries which need slightly different export styles. +// +// The shims are a thin wrapper over the rclone RPC. +package librclone + +import ( + "context" + "encoding/json" + "fmt" + "net/http" + "runtime" + "strings" + + "github.com/pkg/errors" + "github.com/rclone/rclone/fs" + "github.com/rclone/rclone/fs/accounting" + "github.com/rclone/rclone/fs/config/configfile" + "github.com/rclone/rclone/fs/log" + "github.com/rclone/rclone/fs/rc" + "github.com/rclone/rclone/fs/rc/jobs" +) + +// Initialize initializes rclone as a library +// +//export Initialize +func Initialize() { + // A subset of initialisation copied from cmd.go + // Note that we don't want to pull in anything which depends on pflags + + ctx := context.Background() + + // Start the logger + log.InitLogging() + + // Load the config - this may need to be configurable + configfile.Install() + + // Start accounting + accounting.Start(ctx) +} + +// Finalize finalizes the library +func Finalize() { + // TODO: how to clean up? what happens when rcserver terminates? + // what about unfinished async jobs? + runtime.GC() +} + +// writeError returns a formatted error string and the status passed in +func writeError(path string, in rc.Params, err error, status int) (string, int) { + fs.Errorf(nil, "rc: %q: error: %v", path, err) + params, status := rc.Error(path, in, err, status) + var w strings.Builder + err = rc.WriteJSON(&w, params) + if err != nil { + // ultimate fallback error + fs.Errorf(nil, "writeError: failed to write JSON output from %#v: %v", in, err) + status = http.StatusInternalServerError + w.Reset() + fmt.Fprintf(&w, `{ + "error": %q, + "path": %q, + "status": %d +}`, err, path, status) + + } + return w.String(), status +} + +// RPC runs a transaction over the RC +// +// Calling an rc function using JSON to input parameters and output the resulted JSON +// +// operations/uploadfile and core/command are not supported as they need request or response object +// modified from handlePost in rcserver.go +func RPC(method string, input string) (output string, status int) { + // create a buffer to capture the output + in := make(rc.Params) + err := json.NewDecoder(strings.NewReader(input)).Decode(&in) + if err != nil { + return writeError(method, in, errors.Wrap(err, "failed to read input JSON"), http.StatusBadRequest) + } + + // Find the call + call := rc.Calls.Get(method) + if call == nil { + return writeError(method, in, errors.Errorf("couldn't find method %q", method), http.StatusNotFound) + } + + // TODO: handle these cases + if call.NeedsRequest { + return writeError(method, in, errors.Errorf("method %q needs request, not supported", method), http.StatusNotFound) + // Add the request to RC + //in["_request"] = r + } + if call.NeedsResponse { + return writeError(method, in, errors.Errorf("method %q need response, not supported", method), http.StatusNotFound) + //in["_response"] = w + } + + fs.Debugf(nil, "rc: %q: with parameters %+v", method, in) + + _, out, err := jobs.NewJob(context.Background(), call.Fn, in) + if err != nil { + return writeError(method, in, err, http.StatusInternalServerError) + } + if out == nil { + out = make(rc.Params) + } + + fs.Debugf(nil, "rc: %q: reply %+v: %v", method, out, err) + + var w strings.Builder + err = rc.WriteJSON(&w, out) + if err != nil { + fs.Errorf(nil, "rc: failed to write JSON output: %v", err) + return writeError(method, in, err, http.StatusInternalServerError) + } + + return w.String(), http.StatusOK +}