github-mirror/rclone
2
0
Fork 0
mirror of https://github.com/rclone/rclone.git synced 2024-11-25 09:41:44 +08:00
Code Issues Actions 1 Packages Projects Releases Wiki Activity

rcd: move webgui apart; option to disable browser

Browse Source 
Fix #3601, #3785
This commit is contained in:
Xiaoxing Ye 2020-01-12 17:12:04 +08:00 committed by Nick Craig-Wood
parent 84191ac6dc
commit ccaca04a5d
11 changed files with 309 additions and 246 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
MANUAL.html generated
View File

File diff suppressed because one or more lines are too long

10
MANUAL.md generated
View File

@ -6723,8 +6723,10 @@ This will produce logs like this and rclone needs to continue to run to serve th
This assumes you are running rclone locally on your machine. It is
possible to separate the rclone and the GUI - see below for details.
If you wish to update to the latest API version then you can add
`--rc-web-gui-update` to the command line.
By default, rclone will NOT check for GUI update each time it operates. You may alter this
behaviour by using `--rc-web-gui-update` to check and update.
Also, rclone will open the default browser automatically. You may disable it by using `--rc-web-gui-no-open-browser`.
## Using the GUI
@ -6900,7 +6902,7 @@ Default https://api.github.com/repos/rclone/rclone-webui-react/releases/latest.
### --rc-web-gui-update
Set this flag to Download / Force update rclone-webui-react from the rc-web-fetch-url.
Set this flag to check and update rclone-webui-react from the rc-web-fetch-url.
Default Off.
@ -8426,7 +8428,7 @@ These flags are available for every command.
--rc-user string User name for authentication.
--rc-web-fetch-url string URL to fetch the releases for webgui. (default "https://api.github.com/repos/rclone/rclone-webui-react/releases/latest")
--rc-web-gui Launch WebGUI on localhost
--rc-web-gui-update Update / Force update to latest version of web gui
--rc-web-gui-update Check and update to latest version of web gui
--retries int Retry operations this many times if they fail (default 3)
--retries-sleep duration Interval between retrying operations if they fail, e.g 500ms, 60s, 5m. (0 to disable)
--size-only Skip based on size only, not mod-time or checksum

7
MANUAL.txt generated
View File

@ -6353,7 +6353,7 @@ serve the GUI:
This assumes you are running rclone locally on your machine. It is
possible to separate the rclone and the GUI - see below for details.
If you wish to update to the latest API version then you can add
If you wish to check and update to the latest API version then you can add
--rc-web-gui-update to the command line.
@ -6550,9 +6550,10 @@ https://api.github.com/repos/rclone/rclone-webui-react/releases/latest.
–rc-web-gui-update
Set this flag to Download / Force update rclone-webui-react from the
Set this flag to check and update rclone-webui-react from the
rc-web-fetch-url.
Default Off.
–rc-job-expire-duration=DURATION
@ -7980,7 +7981,7 @@ server writing data (default 1h0m0s) –rc-user string User name for
authentication. –rc-web-fetch-url string URL to fetch the releases for
webgui. (default
“https://api.github.com/repos/rclone/rclone-webui-react/releases/latest”)
–rc-web-gui Launch WebGUI on localhost –rc-web-gui-update Update / Force
–rc-web-gui Launch WebGUI on localhost –rc-web-gui-update Check and
update to latest version of web gui –retries int Retry operations this
many times if they fail (default 3) –retries-sleep duration Interval
between retrying operations if they fail, e.g 500ms, 60s, 5m. (0 to

227
cmd/rcd/rcd.go
View File

@ -1,25 +1,11 @@
package rcd
import (
"archive/zip"
"encoding/json"
"fmt"
"io"
"log"
"net/http"
"os"
"path/filepath"
"strconv"
"strings"
"time"
"github.com/rclone/rclone/cmd"
"github.com/rclone/rclone/fs"
"github.com/rclone/rclone/fs/config"
"github.com/rclone/rclone/fs/rc/rcflags"
"github.com/rclone/rclone/fs/rc/rcserver"
"github.com/rclone/rclone/lib/errors"
"github.com/rclone/rclone/lib/random"
"github.com/spf13/cobra"
)
@ -53,29 +39,6 @@ See the [rc documentation](/rc/) for more info on the rc flags.
rcflags.Opt.Files = args[0]
}
if rcflags.Opt.WebUI {
if err := checkRelease(rcflags.Opt.WebGUIUpdate); err != nil {
log.Fatalf("Error while fetching the latest release of rclone-webui-react %v", err)
}
if rcflags.Opt.NoAuth {
rcflags.Opt.NoAuth = false
fs.Infof(nil, "Cannot run web-gui without authentication, using default auth")
}
if rcflags.Opt.HTTPOptions.BasicUser == "" {
rcflags.Opt.HTTPOptions.BasicUser = "gui"
fs.Infof(nil, "Using default username: %s \n", rcflags.Opt.HTTPOptions.BasicUser)
}
if rcflags.Opt.HTTPOptions.BasicPass == "" {
randomPass, err := random.Password(128)
if err != nil {
log.Fatalf("Failed to make password: %v", err)
}
rcflags.Opt.HTTPOptions.BasicPass = randomPass
fs.Infof(nil, "No password specified. Using random password: %s \n", randomPass)
}
rcflags.Opt.Serve = true
}
s, err := rcserver.Start(&rcflags.Opt)
if err != nil {
log.Fatalf("Failed to start remote control: %v", err)
@ -87,193 +50,3 @@ See the [rc documentation](/rc/) for more info on the rc flags.
s.Wait()
},
}
//checkRelease is a helper function to download and setup latest release of rclone-webui-react
func checkRelease(shouldUpdate bool) (err error) {
cachePath := filepath.Join(config.CacheDir, "webgui")
extractPath := filepath.Join(cachePath, "current")
oldUpdateExists := exists(extractPath)
// if the old file exists does not exist or forced update is enforced.
// TODO: Add hashing to check integrity of the previous update.
if !oldUpdateExists || shouldUpdate {
// Get the latest release details
WebUIURL, tag, size, err := getLatestReleaseURL()
if err != nil {
return err
}
zipName := tag + ".zip"
zipPath := filepath.Join(cachePath, zipName)
if !exists(cachePath) {
if err := os.MkdirAll(cachePath, 0755); err != nil {
fs.Logf(nil, "Error creating cache directory: %s", cachePath)
return err
}
}
fs.Logf(nil, "A new release for gui is present at "+WebUIURL)
fs.Logf(nil, "Downloading webgui binary. Please wait. [Size: %s, Path : %s]\n", strconv.Itoa(size), zipPath)
// download the zip from latest url
err = downloadFile(zipPath, WebUIURL)
if err != nil {
return err
}
err = os.RemoveAll(extractPath)
if err != nil {
fs.Logf(nil, "No previous downloads to remove")
}
fs.Logf(nil, "Unzipping")
err = unzip(zipPath, extractPath)
if err != nil {
return err
}
} else {
fs.Logf(nil, "Required files exist. Skipping download")
}
return nil
}
// getLatestReleaseURL returns the latest release details of the rclone-webui-react
func getLatestReleaseURL() (string, string, int, error) {
resp, err := http.Get(rcflags.Opt.WebGUIFetchURL)
if err != nil {
return "", "", 0, errors.New("Error getting latest release of rclone-webui")
}
results := gitHubRequest{}
if err := json.NewDecoder(resp.Body).Decode(&results); err != nil {
return "", "", 0, errors.New("Could not decode results from http request")
}
res := results.Assets[0].BrowserDownloadURL
tag := results.TagName
size := results.Assets[0].Size
//fmt.Println( "URL:" + res)
return res, tag, size, nil
}
// downloadFile is a helper function to download a file from url to the filepath
func downloadFile(filepath string, url string) error {
// Get the data
resp, err := http.Get(url)
if err != nil {
return err
}
defer fs.CheckClose(resp.Body, &err)
// Create the file
out, err := os.Create(filepath)
if err != nil {
return err
}
defer fs.CheckClose(out, &err)
// Write the body to file
_, err = io.Copy(out, resp.Body)
return err
}
// unzip is a helper function to unzip a file specified in src to path dest
func unzip(src, dest string) (err error) {
dest = filepath.Clean(dest) + string(os.PathSeparator)
r, err := zip.OpenReader(src)
if err != nil {
return err
}
defer fs.CheckClose(r, &err)
if err := os.MkdirAll(dest, 0755); err != nil {
return err
}
// Closure to address file descriptors issue with all the deferred .Close() methods
extractAndWriteFile := func(f *zip.File) error {
path := filepath.Join(dest, f.Name)
// Check for Zip Slip: https://github.com/rclone/rclone/issues/3529
if !strings.HasPrefix(path, dest) {
return fmt.Errorf("%s: illegal file path", path)
}
rc, err := f.Open()
if err != nil {
return err
}
defer fs.CheckClose(rc, &err)
if f.FileInfo().IsDir() {
if err := os.MkdirAll(path, 0755); err != nil {
return err
}
} else {
if err := os.MkdirAll(filepath.Dir(path), 0755); err != nil {
return err
}
f, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0644)
if err != nil {
return err
}
defer fs.CheckClose(f, &err)
_, err = io.Copy(f, rc)
if err != nil {
return err
}
}
return nil
}
for _, f := range r.File {
err := extractAndWriteFile(f)
if err != nil {
return err
}
}
return nil
}
// exists returns whether the given file or directory exists
func exists(path string) bool {
_, err := os.Stat(path)
if err == nil {
return true
}
if os.IsNotExist(err) {
return false
}
return true
}
// gitHubRequest Maps the GitHub API request to structure
type gitHubRequest struct {
URL string `json:"url"`
Prerelease bool `json:"prerelease"`
CreatedAt time.Time `json:"created_at"`
PublishedAt time.Time `json:"published_at"`
TagName string `json:"tag_name"`
Assets []struct {
URL string `json:"url"`
ID int `json:"id"`
NodeID string `json:"node_id"`
Name string `json:"name"`
Label string `json:"label"`
ContentType string `json:"content_type"`
State string `json:"state"`
Size int `json:"size"`
DownloadCount int `json:"download_count"`
CreatedAt time.Time `json:"created_at"`
UpdatedAt time.Time `json:"updated_at"`
BrowserDownloadURL string `json:"browser_download_url"`
} `json:"assets"`
TarballURL string `json:"tarball_url"`
ZipballURL string `json:"zipball_url"`
Body string `json:"body"`
}

4
docs/content/flags.md
View File

@ -101,7 +101,9 @@ These flags are available for every command.
--rc-user string User name for authentication.
--rc-web-fetch-url string URL to fetch the releases for webgui. (default "https://api.github.com/repos/rclone/rclone-webui-react/releases/latest")
--rc-web-gui Launch WebGUI on localhost
--rc-web-gui-update Update / Force update to latest version of web gui
--rc-web-gui-update Check and update to latest version of web gui
--rc-web-gui-force-update Force update to latest version of web gui
--rc-web-gui-no-open-browser Don't open browser automatically
--retries int Retry operations this many times if they fail (default 3)
--retries-sleep duration Interval between retrying operations if they fail, e.g 500ms, 60s, 5m. (0 to disable)
--size-only Skip based on size only, not mod-time or checksum

9
docs/content/gui.md
View File

@ -29,8 +29,13 @@ This will produce logs like this and rclone needs to continue to run to serve th
This assumes you are running rclone locally on your machine. It is
possible to separate the rclone and the GUI - see below for details.
If you wish to update to the latest API version then you can add
`--rc-web-gui-update` to the command line.
If you wish to check for updates then you can add `--rc-web-gui-update`
to the command line.
If you find your GUI broken, you may force it to update by add `--rc-web-gui-force-update`.
By default, rclone will open your browser. Add `--rc-web-gui-no-open-browser`
to disable this feature.
## Using the GUI

14
docs/content/rc.md
View File

@ -107,7 +107,19 @@ Default https://api.github.com/repos/rclone/rclone-webui-react/releases/latest.
### --rc-web-gui-update
Set this flag to Download / Force update rclone-webui-react from the rc-web-fetch-url.
Set this flag to check and update rclone-webui-react from the rc-web-fetch-url.
Default Off.
### --rc-web-gui-force-update
Set this flag to force update rclone-webui-react from the rc-web-fetch-url.
Default Off.
### --rc-web-gui-no-open-browser
Set this flag to disable opening browser automatically when using web-gui.
Default Off.

4
fs/rc/rc.go
View File

@ -24,7 +24,9 @@ type Options struct {
Files string // set to enable serving files locally
NoAuth bool // set to disable auth checks on AuthRequired methods
WebUI bool // set to launch the web ui
WebGUIUpdate bool // set to download new update
WebGUIUpdate bool // set to check new update
WebGUIForceUpdate bool // set to force download new update
WebGUINoOpenBrowser bool // set to disable auto opening browser
WebGUIFetchURL string // set the default url for fetching webgui
AccessControlAllowOrigin string // set the access control for CORS configuration
JobExpireDuration time.Duration

4
fs/rc/rcflags/rcflags.go
View File

@ -21,7 +21,9 @@ func AddFlags(flagSet *pflag.FlagSet) {
flags.BoolVarP(flagSet, &Opt.Serve, "rc-serve", "", false, "Enable the serving of remote objects.")
flags.BoolVarP(flagSet, &Opt.NoAuth, "rc-no-auth", "", false, "Don't require auth for certain methods.")
flags.BoolVarP(flagSet, &Opt.WebUI, "rc-web-gui", "", false, "Launch WebGUI on localhost")
flags.BoolVarP(flagSet, &Opt.WebGUIUpdate, "rc-web-gui-update", "", false, "Update / Force update to latest version of web gui")
flags.BoolVarP(flagSet, &Opt.WebGUIUpdate, "rc-web-gui-update", "", false, "Check and update to latest version of web gui")
flags.BoolVarP(flagSet, &Opt.WebGUIForceUpdate, "rc-web-gui-force-update", "", false, "Force update to latest version of web gui")
flags.BoolVarP(flagSet, &Opt.WebGUINoOpenBrowser, "rc-web-gui-no-open-browser", "", false, "Don't open the browser automatically")
flags.StringVarP(flagSet, &Opt.WebGUIFetchURL, "rc-web-fetch-url", "", "https://api.github.com/repos/rclone/rclone-webui-react/releases/latest", "URL to fetch the releases for webgui.")
flags.StringVarP(flagSet, &Opt.AccessControlAllowOrigin, "rc-allow-origin", "", "", "Set the allowed origin for CORS.")
flags.DurationVarP(flagSet, &Opt.JobExpireDuration, "rc-job-expire-duration", "", Opt.JobExpireDuration, "expire finished async jobs older than this value")

34
fs/rc/rcserver/rcserver.go
View File

@ -6,6 +6,7 @@ import (
"encoding/json"
"flag"
"fmt"
"log"
"mime"
"net/http"
"net/url"
@ -24,6 +25,7 @@ import (
"github.com/rclone/rclone/fs/rc"
"github.com/rclone/rclone/fs/rc/jobs"
"github.com/rclone/rclone/fs/rc/rcflags"
"github.com/rclone/rclone/lib/random"
"github.com/skratchdot/open-golang/open"
)
@ -68,6 +70,28 @@ func newServer(opt *rc.Options, mux *http.ServeMux) *Server {
fs.Logf(nil, "Serving files from %q", opt.Files)
s.files = http.FileServer(http.Dir(opt.Files))
} else if opt.WebUI {
if err := rc.CheckAndDownloadWebGUIRelease(opt.WebGUIUpdate, opt.WebGUIForceUpdate, opt.WebGUIFetchURL, config.CacheDir); err != nil {
log.Fatalf("Error while fetching the latest release of Web GUI: %v", err)
}
if opt.NoAuth {
opt.NoAuth = false
fs.Infof(nil, "Cannot run Web GUI without authentication, using default auth")
}
if opt.HTTPOptions.BasicUser == "" {
opt.HTTPOptions.BasicUser = "gui"
fs.Infof(nil, "No username specified. Using default username: %s \n", rcflags.Opt.HTTPOptions.BasicUser)
}
if opt.HTTPOptions.BasicPass == "" {
randomPass, err := random.Password(128)
if err != nil {
log.Fatalf("Failed to make password: %v", err)
}
opt.HTTPOptions.BasicPass = randomPass
fs.Infof(nil, "No password specified. Using random password: %s \n", randomPass)
}
opt.Serve = true
fs.Logf(nil, "Serving Web GUI")
s.files = http.FileServer(http.Dir(extractPath))
}
return s
@ -102,11 +126,13 @@ func (s *Server) Serve() error {
openURL.RawQuery = parameters.Encode()
openURL.RawPath = "/#/login"
}
// Don't open browser if serving in testing environment.
if flag.Lookup("test.v") == nil {
_ = open.Start(openURL.String())
// Don't open browser if serving in testing environment or required not to do so.
if flag.Lookup("test.v") == nil && !s.opt.WebGUINoOpenBrowser {
if err := open.Start(openURL.String()); err != nil {
fs.Errorf(nil, "Failed to open Web GUI in browser: %v. Manually access it at: %s", err, openURL.String())
}
} else {
fs.Errorf(nil, "Not opening browser in testing environment")
fs.Logf(nil, "Web GUI is not automatically opening browser. Navigate to %s to use.", openURL.String())
}
}
return nil

238
fs/rc/webgui.go Normal file
View File

@ -0,0 +1,238 @@
// Define the Web GUI helpers
package rc
import (
"archive/zip"
"encoding/json"
"fmt"
"io"
"io/ioutil"
"net/http"
"os"
"path/filepath"
"strconv"
"strings"
"time"
"github.com/rclone/rclone/fs"
"github.com/rclone/rclone/lib/errors"
)
// getLatestReleaseURL returns the latest release details of the rclone-webui-react
func getLatestReleaseURL(fetchURL string) (string, string, int, error) {
resp, err := http.Get(fetchURL)
if err != nil {
return "", "", 0, errors.New("Error getting latest release of rclone-webui")
}
results := gitHubRequest{}
if err := json.NewDecoder(resp.Body).Decode(&results); err != nil {
return "", "", 0, errors.New("Could not decode results from http request")
}
res := results.Assets[0].BrowserDownloadURL
tag := results.TagName
size := results.Assets[0].Size
return res, tag, size, nil
}
// CheckAndDownloadWebGUIRelease is a helper function to download and setup latest release of rclone-webui-react
func CheckAndDownloadWebGUIRelease(checkUpdate bool, forceUpdate bool, fetchURL string, cacheDir string) (err error) {
cachePath := filepath.Join(cacheDir, "webgui")
tagPath := filepath.Join(cachePath, "tag")
extractPath := filepath.Join(cachePath, "current")
extractPathExist, extractPathStat, err := exists(extractPath)
if extractPathExist && !extractPathStat.IsDir() {
return errors.New("Web GUI path exists, but is a file instead of folder. Please check the path " + extractPath)
}
// if the old file exists does not exist or forced update is enforced.
// TODO: Add hashing to check integrity of the previous update.
if !extractPathExist || checkUpdate || forceUpdate {
// Get the latest release details
WebUIURL, tag, size, err := getLatestReleaseURL(fetchURL)
if err != nil {
return err
}
dat, err := ioutil.ReadFile(tagPath)
if err == nil && string(dat) == tag {
fs.Logf(nil, "No update to Web GUI available.")
if !forceUpdate {
return nil
}
fs.Logf(nil, "Force update the Web GUI binary.")
}
zipName := tag + ".zip"
zipPath := filepath.Join(cachePath, zipName)
cachePathExist, cachePathStat, _ := exists(cachePath)
if !cachePathExist {
if err := os.MkdirAll(cachePath, 0755); err != nil {
return errors.New("Error creating cache directory: " + cachePath)
}
}
if cachePathExist && !cachePathStat.IsDir() {
return errors.New("Web GUI path is a file instead of folder. Please check it " + extractPath)
}
fs.Logf(nil, "A new release for gui is present at "+WebUIURL)
fs.Logf(nil, "Downloading webgui binary. Please wait. [Size: %s, Path : %s]\n", strconv.Itoa(size), zipPath)
// download the zip from latest url
err = downloadFile(zipPath, WebUIURL)
if err != nil {
return err
}
err = os.RemoveAll(extractPath)
if err != nil {
fs.Logf(nil, "No previous downloads to remove")
}
fs.Logf(nil, "Unzipping webgui binary")
err = unzip(zipPath, extractPath)
if err != nil {
return err
}
err = os.RemoveAll(zipPath)
if err != nil {
fs.Logf(nil, "Downloaded ZIP cannot be deleted")
}
err = ioutil.WriteFile(tagPath, []byte(tag), 0644)
if err != nil {
fs.Infof(nil, "Cannot write tag file. You may be required to redownload the binary next time.")
}
} else {
fs.Logf(nil, "Web GUI exists. Update skipped.")
}
return nil
}
// downloadFile is a helper function to download a file from url to the filepath
func downloadFile(filepath string, url string) error {
// Get the data
resp, err := http.Get(url)
if err != nil {
return err
}
defer fs.CheckClose(resp.Body, &err)
// Create the file
out, err := os.Create(filepath)
if err != nil {
return err
}
defer fs.CheckClose(out, &err)
// Write the body to file
_, err = io.Copy(out, resp.Body)
return err
}
// unzip is a helper function to unzip a file specified in src to path dest
func unzip(src, dest string) (err error) {
dest = filepath.Clean(dest) + string(os.PathSeparator)
r, err := zip.OpenReader(src)
if err != nil {
return err
}
defer fs.CheckClose(r, &err)
if err := os.MkdirAll(dest, 0755); err != nil {
return err
}
// Closure to address file descriptors issue with all the deferred .Close() methods
extractAndWriteFile := func(f *zip.File) error {
path := filepath.Join(dest, f.Name)
// Check for Zip Slip: https://github.com/rclone/rclone/issues/3529
if !strings.HasPrefix(path, dest) {
return fmt.Errorf("%s: illegal file path", path)
}
rc, err := f.Open()
if err != nil {
return err
}
defer fs.CheckClose(rc, &err)
if f.FileInfo().IsDir() {
if err := os.MkdirAll(path, 0755); err != nil {
return err
}
} else {
if err := os.MkdirAll(filepath.Dir(path), 0755); err != nil {
return err
}
f, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0644)
if err != nil {
return err
}
defer fs.CheckClose(f, &err)
_, err = io.Copy(f, rc)
if err != nil {
return err
}
}
return nil
}
for _, f := range r.File {
err := extractAndWriteFile(f)
if err != nil {
return err
}
}
return nil
}
func exists(path string) (existance bool, stat os.FileInfo, err error) {
stat, err = os.Stat(path)
if err == nil {
return true, stat, nil
}
if os.IsNotExist(err) {
return false, nil, nil
}
return false, stat, err
}
// gitHubRequest Maps the GitHub API request to structure
type gitHubRequest struct {
URL string `json:"url"`
Prerelease bool `json:"prerelease"`
CreatedAt time.Time `json:"created_at"`
PublishedAt time.Time `json:"published_at"`
TagName string `json:"tag_name"`
Assets []struct {
URL string `json:"url"`
ID int `json:"id"`
NodeID string `json:"node_id"`
Name string `json:"name"`
Label string `json:"label"`
ContentType string `json:"content_type"`
State string `json:"state"`
Size int `json:"size"`
DownloadCount int `json:"download_count"`
CreatedAt time.Time `json:"created_at"`
UpdatedAt time.Time `json:"updated_at"`
BrowserDownloadURL string `json:"browser_download_url"`
} `json:"assets"`
TarballURL string `json:"tarball_url"`
ZipballURL string `json:"zipball_url"`
Body string `json:"body"`
}