diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 39a578724..58d623189 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -191,7 +191,10 @@ Research Getting going * Create `remote/remote.go` (copy this from a similar remote) + * onedrive is a good one to start from if you have a directory based remote + * b2 is a good one to start from if you have a bucket based remote * Add your remote to the imports in `fs/all/all.go` + * If web based remotes are easiest to maintain if they use rclone's rest module, but if there is a really good go SDK then use that instead. Unit tests @@ -211,7 +214,7 @@ Integration tests * And if your remote defines `ListR` this also * `go test -v -remote TestRemote: -fast-list` -Add your fs to the docs - you'll need to pick an icon for it from [fontawesome](http://fontawesome.io/icons/). +Add your fs to the docs - you'll need to pick an icon for it from [fontawesome](http://fontawesome.io/icons/). Keep lists of remotes in alphabetical order but with the local file system last. * `README.md` - main Github page * `docs/content/remote.md` - main docs page diff --git a/README.md b/README.md index 71ea3a568..95d60bf43 100644 --- a/README.md +++ b/README.md @@ -15,19 +15,20 @@ Rclone is a command line program to sync files and directories to and from - * Google Drive - * Amazon S3 - * Openstack Swift / Rackspace cloud files / Memset Memstore - * Dropbox - * Google Cloud Storage * Amazon Drive - * Microsoft OneDrive - * Hubic + * Amazon S3 * Backblaze B2 - * Yandex Disk - * SFTP + * Box + * Dropbox * FTP + * Google Cloud Storage + * Google Drive * HTTP + * Hubic + * Microsoft OneDrive + * Openstack Swift / Rackspace cloud files / Memset Memstore + * SFTP + * Yandex Disk * The local filesystem Features diff --git a/bin/make_manual.py b/bin/make_manual.py index 2b35753cb..a5135c570 100755 --- a/bin/make_manual.py +++ b/bin/make_manual.py @@ -19,20 +19,24 @@ docs = [ "remote_setup.md", "filtering.md", "overview.md", - "drive.md", - "s3.md", - "swift.md", - "dropbox.md", - "googlecloudstorage.md", + + # Keep these alphabetical by full name "amazonclouddrive.md", - "onedrive.md", - "hubic.md", + "s3.md", "b2.md", - "yandex.md", - "sftp.md", - "ftp.md", - "http.md", + "box.md", "crypt.md", + "dropbox.md", + "ftp.md", + "googlecloudstorage.md", + "drive.md", + "http.md", + "hubic.md", + "onedrive.md", + "swift.md", + "sftp.md", + "yandex.md", + "local.md", "changelog.md", "bugs.md", diff --git a/cmd/cmd.go b/cmd/cmd.go index f1d516ee9..5dcde9ac0 100644 --- a/cmd/cmd.go +++ b/cmd/cmd.go @@ -41,19 +41,20 @@ var Root = &cobra.Command{ Rclone is a command line program to sync files and directories to and from various cloud storage systems and using file transfer services, such as: - * Google Drive - * Amazon S3 - * Openstack Swift / Rackspace cloud files / Memset Memstore - * Dropbox - * Google Cloud Storage * Amazon Drive - * Microsoft OneDrive - * Hubic + * Amazon S3 * Backblaze B2 - * Yandex Disk - * SFTP + * Box + * Dropbox * FTP + * Google Cloud Storage + * Google Drive * HTTP + * Hubic + * Microsoft OneDrive + * Openstack Swift / Rackspace cloud files / Memset Memstore + * SFTP + * Yandex Disk * The local filesystem Features diff --git a/docs/content/about.md b/docs/content/about.md index c9f6e3ea3..14cc929f8 100644 --- a/docs/content/about.md +++ b/docs/content/about.md @@ -13,19 +13,20 @@ Rclone Rclone is a command line program to sync files and directories to and from - * Google Drive - * Amazon S3 - * Openstack Swift / Rackspace cloud files / Memset Memstore - * Dropbox - * Google Cloud Storage * Amazon Drive - * Microsoft OneDrive - * Hubic + * Amazon S3 * Backblaze B2 - * Yandex Disk - * SFTP + * Box + * Dropbox * FTP + * Google Cloud Storage + * Google Drive * HTTP + * Hubic + * Microsoft OneDrive + * Openstack Swift / Rackspace cloud files / Memset Memstore + * SFTP + * Yandex Disk * The local filesystem Features diff --git a/docs/content/box.md b/docs/content/box.md new file mode 100644 index 000000000..795c039f5 --- /dev/null +++ b/docs/content/box.md @@ -0,0 +1,238 @@ +--- +title: "Box" +description: "Rclone docs for Box" +date: "2015-10-14" +--- + + Box +----------------------------------------- + +Paths are specified as `remote:path` + +Paths may be as deep as required, eg `remote:directory/subdirectory`. + +The initial setup for Box involves getting a token from Box which you +need to do in your browser. `rclone config` walks you through it. + +Here is an example of how to make a remote called `remote`. First run: + + rclone config + +This will guide you through an interactive setup process: + +``` +No remotes found - make a new one +n) New remote +s) Set configuration password +q) Quit config +n/s/q> n +name> remote +Type of storage to configure. +Choose a number from below, or type in your own value + 1 / Amazon Drive + \ "amazon cloud drive" + 2 / Amazon S3 (also Dreamhost, Ceph, Minio) + \ "s3" + 3 / Backblaze B2 + \ "b2" + 4 / Box + \ "box" + 5 / Dropbox + \ "dropbox" + 6 / Encrypt/Decrypt a remote + \ "crypt" + 7 / FTP Connection + \ "ftp" + 8 / Google Cloud Storage (this is not Google Drive) + \ "google cloud storage" + 9 / Google Drive + \ "drive" +10 / Hubic + \ "hubic" +11 / Local Disk + \ "local" +12 / Microsoft OneDrive + \ "onedrive" +13 / Openstack Swift (Rackspace Cloud Files, Memset Memstore, OVH) + \ "swift" +14 / SSH/SFTP Connection + \ "sftp" +15 / Yandex Disk + \ "yandex" +16 / http Connection + \ "http" +Storage> box +Box App Client Id - leave blank normally. +client_id> +Box App Client Secret - leave blank normally. +client_secret> +Remote config +Use auto config? + * Say Y if not sure + * Say N if you are working on a remote or headless machine +y) Yes +n) No +y/n> y +If your browser doesn't open automatically go to the following link: http://127.0.0.1:53682/auth +Log in and authorize rclone for access +Waiting for code... +Got code +-------------------- +[remote] +client_id = +client_secret = +token = {"access_token":"XXX","token_type":"bearer","refresh_token":"XXX","expiry":"XXX"} +-------------------- +y) Yes this is OK +e) Edit this remote +d) Delete this remote +y/e/d> y +``` + +See the [remote setup docs](/remote_setup/) for how to set it up on a +machine with no Internet browser available. + +Note that rclone runs a webserver on your local machine to collect the +token as returned from Box. This only runs from the moment it opens +your browser to the moment you get back the verification code. This +is on `http://127.0.0.1:53682/` and this it may require you to unblock +it temporarily if you are running a host firewall. + +Once configured you can then use `rclone` like this, + +List directories in top level of your Box + + rclone lsd remote: + +List all the files in your Box + + rclone ls remote: + +To copy a local directory to an Box directory called backup + + rclone copy /home/source remote:backup + +### Invalid refresh token ### + +According to the [box docs](https://developer.box.com/v2.0/docs/oauth-20#section-6-using-the-access-and-refresh-tokens): + +> Each refresh_token is valid for one use in 60 days. + +This means that if you + + * Don't use the box remote for 60 days + * Copy the config file with a box refresh token in and use it in two places + * Get an error on a token refresh + +then rclone will return an error which includes the text `Invalid +refresh token`. + +To fix this you will need to use oauth2 again to update the refresh +token. You can use the methods in [the remote setup +docs](/remote_setup/), bearing in mind that if you use the copy the +config file method, you should not use that remote on the computer you +did the authentication on. + +Here is how to do it. + +``` +$ rclone config +Current remotes: + +Name Type +==== ==== +remote box + +e) Edit existing remote +n) New remote +d) Delete remote +r) Rename remote +c) Copy remote +s) Set configuration password +q) Quit config +e/n/d/r/c/s/q> e +Choose a number from below, or type in an existing value + 1 > remote +remote> remote +-------------------- +[remote] +type = box +token = {"access_token":"XXX","token_type":"bearer","refresh_token":"XXX","expiry":"2017-07-08T23:40:08.059167677+01:00"} +-------------------- +Edit remote +Value "client_id" = "" +Edit? (y/n)> +y) Yes +n) No +y/n> n +Value "client_secret" = "" +Edit? (y/n)> +y) Yes +n) No +y/n> n +Remote config +Already have a token - refresh? +y) Yes +n) No +y/n> y +Use auto config? + * Say Y if not sure + * Say N if you are working on a remote or headless machine +y) Yes +n) No +y/n> y +If your browser doesn't open automatically go to the following link: http://127.0.0.1:53682/auth +Log in and authorize rclone for access +Waiting for code... +Got code +-------------------- +[remote] +type = box +token = {"access_token":"YYY","token_type":"bearer","refresh_token":"YYY","expiry":"2017-07-23T12:22:29.259137901+01:00"} +-------------------- +y) Yes this is OK +e) Edit this remote +d) Delete this remote +y/e/d> y +``` + +### Modified time and hashes ### + +Box allows modification times to be set on objects accurate to 1 +second. These will be used to detect whether objects need syncing or +not. + +One drive supports SHA1 type hashes, so you can use the `--checksum` +flag. + +### Transfers ### + +For files above 50MB rclone will use a chunked transfer. Rclone will +upload up to `--transfers` chunks at the same time (shared among all +the multipart uploads). Chunks are buffered in memory and are +normally 8MB so increasing `--transfers` will increase memory use. + +### Deleting files ### + +Depending on the enterprise settings for your user, the item will +either be actually deleted from Box or moved to the trash. + +### Specific options ### + +Here are the command line options specific to this cloud storage +system. + +#### --box-upload-cutoff=SIZE #### + +Cutoff for switching to chunked upload - must be >= 50MB. The default +is 50MB. + +### Limitations ### + +Note that Box is case insensitive so you can't have a file called +"Hello.doc" and one called "hello.doc". + +Box file names can't have the `\` character in. rclone maps this to +and from an identical looking unicode equivalent `＼`. + +Box only supports filenames up to 255 characters in length. diff --git a/docs/content/docs.md b/docs/content/docs.md index 6956754c5..218757a86 100644 --- a/docs/content/docs.md +++ b/docs/content/docs.md @@ -19,21 +19,22 @@ option: See the following for detailed instructions for - * [Google Drive](/drive/) - * [Amazon S3](/s3/) - * [Swift / Rackspace Cloudfiles / Memset Memstore](/swift/) - * [Dropbox](/dropbox/) - * [Google Cloud Storage](/googlecloudstorage/) - * [Local filesystem](/local/) * [Amazon Drive](/amazonclouddrive/) + * [Amazon S3](/s3/) * [Backblaze B2](/b2/) + * [Box](/box/) + * [Crypt](/crypt/) - to encrypt other remotes + * [Dropbox](/dropbox/) + * [FTP](/ftp/) + * [Google Cloud Storage](/googlecloudstorage/) + * [Google Drive](/drive/) + * [HTTP](/http/) * [Hubic](/hubic/) * [Microsoft OneDrive](/onedrive/) - * [Yandex Disk](/yandex/) * [SFTP](/sftp/) - * [FTP](/ftp/) - * [HTTP](/http/) - * [Crypt](/crypt/) - to encrypt other remotes + * [Swift / Rackspace Cloudfiles / Memset Memstore](/swift/) + * [Yandex Disk](/yandex/) + * [The local filesystem](/local/) Usage ----- diff --git a/docs/content/overview.md b/docs/content/overview.md index f2515fac2..bae33a05a 100644 --- a/docs/content/overview.md +++ b/docs/content/overview.md @@ -17,19 +17,20 @@ Here is an overview of the major features of each cloud storage system. | Name | Hash | ModTime | Case Insensitive | Duplicate Files | MIME Type | | ---------------------- |:-------:|:-------:|:----------------:|:---------------:|:---------:| -| Google Drive | MD5 | Yes | No | Yes | R/W | -| Amazon S3 | MD5 | Yes | No | No | R/W | -| Openstack Swift | MD5 | Yes | No | No | R/W | -| Dropbox | DBHASH †| Yes | Yes | No | - | -| Google Cloud Storage | MD5 | Yes | No | No | R/W | | Amazon Drive | MD5 | No | Yes | No | R | -| Microsoft OneDrive | SHA1 | Yes | Yes | No | R | -| Hubic | MD5 | Yes | No | No | R/W | +| Amazon S3 | MD5 | Yes | No | No | R/W | | Backblaze B2 | SHA1 | Yes | No | No | R/W | -| Yandex Disk | MD5 | Yes | No | No | R/W | +| Box | SHA1 | Yes | Yes | No | - | +| Dropbox | DBHASH †| Yes | Yes | No | - | +| FTP | - | No | No | No | - | +| Google Cloud Storage | MD5 | Yes | No | No | R/W | +| Google Drive | MD5 | Yes | No | Yes | R/W | +| HTTP | - | No | No | No | R | +| Hubic | MD5 | Yes | No | No | R/W | +| Microsoft OneDrive | SHA1 | Yes | Yes | No | R | +| Openstack Swift | MD5 | Yes | No | No | R/W | | SFTP | - | Yes | Depends | No | - | -| FTP | - | No | Yes | No | - | -| HTTP | - | No | Yes | No | R | +| Yandex Disk | MD5 | Yes | No | No | R/W | | The local filesystem | All | Yes | Depends | No | - | ### Hash ### @@ -39,8 +40,8 @@ The hashes are used when transferring data as an integrity check and can be specifically used with the `--checksum` flag in syncs and in the `check` command. -To use the checksum checks between filesystems they must support a -common hash type. +To use the verify checksums when transferring between cloud storage +systems they must support a common hash type. † Note that Dropbox supports [its own custom hash](https://www.dropbox.com/developers/reference/content-hash). @@ -111,19 +112,20 @@ operations more efficient. | Name | Purge | Copy | Move | DirMove | CleanUp | ListR | | ---------------------- |:-----:|:----:|:----:|:-------:|:-------:|:-----:| -| Google Drive | Yes | Yes | Yes | Yes | No [#575](https://github.com/ncw/rclone/issues/575) | No | -| Amazon S3 | No | Yes | No | No | No | Yes | -| Openstack Swift | Yes † | Yes | No | No | No | Yes | -| Dropbox | Yes | Yes | Yes | Yes | No [#575](https://github.com/ncw/rclone/issues/575) | No | -| Google Cloud Storage | Yes | Yes | No | No | No | Yes | | Amazon Drive | Yes | No | Yes | Yes | No [#575](https://github.com/ncw/rclone/issues/575) | No | -| Microsoft OneDrive | Yes | Yes | Yes | No [#197](https://github.com/ncw/rclone/issues/197) | No [#575](https://github.com/ncw/rclone/issues/575) | No | -| Hubic | Yes † | Yes | No | No | No | Yes | +| Amazon S3 | No | Yes | No | No | No | Yes | | Backblaze B2 | No | No | No | No | Yes | Yes | -| Yandex Disk | Yes | No | No | No | No [#575](https://github.com/ncw/rclone/issues/575) | Yes | -| SFTP | No | No | Yes | Yes | No | No | +| Box | Yes | Yes | Yes | Yes | No [#575](https://github.com/ncw/rclone/issues/575) | No | +| Dropbox | Yes | Yes | Yes | Yes | No [#575](https://github.com/ncw/rclone/issues/575) | No | | FTP | No | No | Yes | Yes | No | No | +| Google Cloud Storage | Yes | Yes | No | No | No | Yes | +| Google Drive | Yes | Yes | Yes | Yes | No [#575](https://github.com/ncw/rclone/issues/575) | No | | HTTP | No | No | No | No | No | No | +| Hubic | Yes † | Yes | No | No | No | Yes | +| Microsoft OneDrive | Yes | Yes | Yes | No [#197](https://github.com/ncw/rclone/issues/197) | No [#575](https://github.com/ncw/rclone/issues/575) | No | +| Openstack Swift | Yes † | Yes | No | No | No | Yes | +| SFTP | No | No | Yes | Yes | No | No | +| Yandex Disk | Yes | No | No | No | No [#575](https://github.com/ncw/rclone/issues/575) | Yes | | The local filesystem | Yes | No | Yes | Yes | No | No | diff --git a/docs/layouts/chrome/navbar.html b/docs/layouts/chrome/navbar.html index cc9c56c2d..5f3068f09 100644 --- a/docs/layouts/chrome/navbar.html +++ b/docs/layouts/chrome/navbar.html @@ -49,21 +49,22 @@ Storage Systems
  • Contact
    • diff --git a/fs/test_all.go b/fs/test_all.go index dfe83c7e3..ffcb6fadf 100644 --- a/fs/test_all.go +++ b/fs/test_all.go @@ -98,6 +98,11 @@ var ( SubDir: false, FastList: false, }, + { + Name: "TestBox:", + SubDir: false, + FastList: false, + }, } binary = "fs.test" // Flags