mirror of
https://github.com/rclone/rclone.git
synced 2024-11-22 04:41:17 +08:00
storj: rename tardigrade backend to storj backend #5616
This adds an alias for backwards compatibility and leaves a stub documentation page to redirect people to the new documentation.
This commit is contained in:
Nick Craig-Wood
parent
c40129d610
commit
f22b703a51
|
|
@ -15,7 +15,7 @@ Current active maintainers of rclone are:
|
|||
| Ivan Andreev | @ivandeex | chunker & mailru backends |
|
||||
| Max Sum | @Max-Sum | union backend |
|
||||
| Fred | @creativeprojects | seafile backend |
|
||||
| Caleb Case | @calebcase | tardigrade backend |
|
||||
| Caleb Case | @calebcase | storj backend |
|
||||
|
||||
**This is a work in progress Draft**
|
||||
|
||||
|
|
|
|||
|
|
@ -66,8 +66,8 @@ Rclone *("rsync for cloud storage")* is a command-line program to sync files and
|
|||
* SeaweedFS [:page_facing_up:](https://rclone.org/s3/#seaweedfs)
|
||||
* SFTP [:page_facing_up:](https://rclone.org/sftp/)
|
||||
* StackPath [:page_facing_up:](https://rclone.org/s3/#stackpath)
|
||||
* Storj [:page_facing_up:](https://rclone.org/storj/)
|
||||
* SugarSync [:page_facing_up:](https://rclone.org/sugarsync/)
|
||||
* Tardigrade [:page_facing_up:](https://rclone.org/tardigrade/)
|
||||
* Tencent Cloud Object Storage (COS) [:page_facing_up:](https://rclone.org/s3/#tencent-cos)
|
||||
* Wasabi [:page_facing_up:](https://rclone.org/s3/#wasabi)
|
||||
* WebDAV [:page_facing_up:](https://rclone.org/webdav/)
|
||||
|
|
|
|||
|
|
@ -39,9 +39,9 @@ import (
|
|||
_ "github.com/rclone/rclone/backend/sftp"
|
||||
_ "github.com/rclone/rclone/backend/sharefile"
|
||||
_ "github.com/rclone/rclone/backend/sia"
|
||||
_ "github.com/rclone/rclone/backend/storj"
|
||||
_ "github.com/rclone/rclone/backend/sugarsync"
|
||||
_ "github.com/rclone/rclone/backend/swift"
|
||||
_ "github.com/rclone/rclone/backend/tardigrade"
|
||||
_ "github.com/rclone/rclone/backend/union"
|
||||
_ "github.com/rclone/rclone/backend/uptobox"
|
||||
_ "github.com/rclone/rclone/backend/webdav"
|
||||
|
|
|
|||
|
|
@ -1,8 +1,8 @@
|
|||
//go:build !plan9
|
||||
// +build !plan9
|
||||
|
||||
// Package tardigrade provides an interface to Tardigrade decentralized object storage.
|
||||
package tardigrade
|
||||
// Package storj provides an interface to Storj decentralized object storage.
|
||||
package storj
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
|
@ -31,16 +31,17 @@ const (
|
|||
)
|
||||
|
||||
var satMap = map[string]string{
|
||||
"us-central-1.tardigrade.io": "12EayRS2V1kEsWESU9QMRseFhdxYxKicsiFmxrsLZHeLUtdps3S@us-central-1.tardigrade.io:7777",
|
||||
"europe-west-1.tardigrade.io": "12L9ZFwhzVpuEKMUNUqkaTLGzwY9G24tbiigLiXpmZWKwmcNDDs@europe-west-1.tardigrade.io:7777",
|
||||
"asia-east-1.tardigrade.io": "121RTSDpyNZVcEU84Ticf2L1ntiuUimbWgfATz21tuvgk3vzoA6@asia-east-1.tardigrade.io:7777",
|
||||
"us-central-1.storj.io": "12EayRS2V1kEsWESU9QMRseFhdxYxKicsiFmxrsLZHeLUtdps3S@us-central-1.tardigrade.io:7777",
|
||||
"europe-west-1.storj.io": "12L9ZFwhzVpuEKMUNUqkaTLGzwY9G24tbiigLiXpmZWKwmcNDDs@europe-west-1.tardigrade.io:7777",
|
||||
"asia-east-1.storj.io": "121RTSDpyNZVcEU84Ticf2L1ntiuUimbWgfATz21tuvgk3vzoA6@asia-east-1.tardigrade.io:7777",
|
||||
}
|
||||
|
||||
// Register with Fs
|
||||
func init() {
|
||||
fs.Register(&fs.RegInfo{
|
||||
Name: "tardigrade",
|
||||
Description: "Tardigrade Decentralized Cloud Storage",
|
||||
Name: "storj",
|
||||
Description: "Storj Decentralized Cloud Storage",
|
||||
Aliases: []string{"tardigrade"},
|
||||
NewFs: NewFs,
|
||||
Config: func(ctx context.Context, name string, m configmap.Mapper, configIn fs.ConfigIn) (*fs.ConfigOut, error) {
|
||||
provider, _ := m.Get(fs.ConfigProvider)
|
||||
|
|
@ -104,15 +105,15 @@ func init() {
|
|||
Name: "satellite_address",
|
||||
Help: "Satellite address.\n\nCustom satellite address should match the format: `<nodeid>@<address>:<port>`.",
|
||||
Provider: newProvider,
|
||||
Default: "us-central-1.tardigrade.io",
|
||||
Default: "us-central-1.storj.io",
|
||||
Examples: []fs.OptionExample{{
|
||||
Value: "us-central-1.tardigrade.io",
|
||||
Value: "us-central-1.storj.io",
|
||||
Help: "US Central 1",
|
||||
}, {
|
||||
Value: "europe-west-1.tardigrade.io",
|
||||
Value: "europe-west-1.storj.io",
|
||||
Help: "Europe West 1",
|
||||
}, {
|
||||
Value: "asia-east-1.tardigrade.io",
|
||||
Value: "asia-east-1.storj.io",
|
||||
Help: "Asia East 1",
|
||||
},
|
||||
},
|
||||
|
|
@ -140,7 +141,7 @@ type Options struct {
|
|||
Passphrase string `config:"passphrase"`
|
||||
}
|
||||
|
||||
// Fs represents a remote to Tardigrade
|
||||
// Fs represents a remote to Storj
|
||||
type Fs struct {
|
||||
name string // the name of the remote
|
||||
root string // root of the filesystem
|
||||
|
|
@ -160,9 +161,9 @@ var (
|
|||
_ fs.PutStreamer = &Fs{}
|
||||
)
|
||||
|
||||
// NewFs creates a filesystem backed by Tardigrade.
|
||||
// NewFs creates a filesystem backed by Storj.
|
||||
func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (_ fs.Fs, err error) {
|
||||
// Setup filesystem and connection to Tardigrade
|
||||
// Setup filesystem and connection to Storj
|
||||
root = norm.NFC.String(root)
|
||||
root = strings.Trim(root, "/")
|
||||
|
||||
|
|
@ -183,24 +184,24 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (_ fs.Fs,
|
|||
if f.opts.Access != "" {
|
||||
access, err = uplink.ParseAccess(f.opts.Access)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("tardigrade: access: %w", err)
|
||||
return nil, fmt.Errorf("storj: access: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
if access == nil && f.opts.SatelliteAddress != "" && f.opts.APIKey != "" && f.opts.Passphrase != "" {
|
||||
access, err = uplink.RequestAccessWithPassphrase(ctx, f.opts.SatelliteAddress, f.opts.APIKey, f.opts.Passphrase)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("tardigrade: access: %w", err)
|
||||
return nil, fmt.Errorf("storj: access: %w", err)
|
||||
}
|
||||
|
||||
serializedAccess, err := access.Serialize()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("tardigrade: access: %w", err)
|
||||
return nil, fmt.Errorf("storj: access: %w", err)
|
||||
}
|
||||
|
||||
err = config.SetValueAndSave(f.name, "access_grant", serializedAccess)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("tardigrade: access: %w", err)
|
||||
return nil, fmt.Errorf("storj: access: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -232,7 +233,7 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (_ fs.Fs,
|
|||
if bucketName != "" && bucketPath != "" {
|
||||
_, err = project.StatBucket(ctx, bucketName)
|
||||
if err != nil {
|
||||
return f, fmt.Errorf("tardigrade: bucket: %w", err)
|
||||
return f, fmt.Errorf("storj: bucket: %w", err)
|
||||
}
|
||||
|
||||
object, err := project.StatObject(ctx, bucketName, bucketPath)
|
||||
|
|
@ -258,7 +259,7 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (_ fs.Fs,
|
|||
return f, nil
|
||||
}
|
||||
|
||||
// connect opens a connection to Tardigrade.
|
||||
// connect opens a connection to Storj.
|
||||
func (f *Fs) connect(ctx context.Context) (project *uplink.Project, err error) {
|
||||
fs.Debugf(f, "connecting...")
|
||||
defer fs.Debugf(f, "connected: %+v", err)
|
||||
|
|
@ -269,7 +270,7 @@ func (f *Fs) connect(ctx context.Context) (project *uplink.Project, err error) {
|
|||
|
||||
project, err = cfg.OpenProject(ctx, f.access)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("tardigrade: project: %w", err)
|
||||
return nil, fmt.Errorf("storj: project: %w", err)
|
||||
}
|
||||
|
||||
return
|
||||
|
|
@ -1,7 +1,7 @@
|
|||
//go:build !plan9
|
||||
// +build !plan9
|
||||
|
||||
package tardigrade
|
||||
package storj
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
|
@ -18,7 +18,7 @@ import (
|
|||
"storj.io/uplink"
|
||||
)
|
||||
|
||||
// Object describes a Tardigrade object
|
||||
// Object describes a Storj object
|
||||
type Object struct {
|
||||
fs *Fs
|
||||
|
||||
|
|
@ -32,7 +32,7 @@ type Object struct {
|
|||
// Check the interfaces are satisfied.
|
||||
var _ fs.Object = &Object{}
|
||||
|
||||
// newObjectFromUplink creates a new object from a Tardigrade uplink object.
|
||||
// newObjectFromUplink creates a new object from a Storj uplink object.
|
||||
func newObjectFromUplink(f *Fs, relative string, object *uplink.Object) *Object {
|
||||
// Attempt to use the modified time from the metadata. Otherwise
|
||||
// fallback to the server time.
|
||||
|
|
@ -1,20 +1,20 @@
|
|||
//go:build !plan9
|
||||
// +build !plan9
|
||||
|
||||
// Test Tardigrade filesystem interface
|
||||
package tardigrade_test
|
||||
// Test Storj filesystem interface
|
||||
package storj_test
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/rclone/rclone/backend/tardigrade"
|
||||
"github.com/rclone/rclone/backend/storj"
|
||||
"github.com/rclone/rclone/fstest/fstests"
|
||||
)
|
||||
|
||||
// TestIntegration runs integration tests against the remote
|
||||
func TestIntegration(t *testing.T) {
|
||||
fstests.Run(t, &fstests.Opt{
|
||||
RemoteName: "TestTardigrade:",
|
||||
NilObject: (*tardigrade.Object)(nil),
|
||||
RemoteName: "TestStorj:",
|
||||
NilObject: (*storj.Object)(nil),
|
||||
})
|
||||
}
|
||||
|
|
@ -1,4 +1,4 @@
|
|||
//go:build plan9
|
||||
// +build plan9
|
||||
|
||||
package tardigrade
|
||||
package storj
|
||||
|
|
@ -63,8 +63,8 @@ docs = [
|
|||
"putio.md",
|
||||
"seafile.md",
|
||||
"sftp.md",
|
||||
"storj.md",
|
||||
"sugarsync.md",
|
||||
"tardigrade.md",
|
||||
"uptobox.md",
|
||||
"union.md",
|
||||
"webdav.md",
|
||||
|
|
|
|||
|
|
@ -152,8 +152,8 @@ WebDAV or S3, that work out of the box.)
|
|||
{{< provider name="SFTP" home="https://en.wikipedia.org/wiki/SSH_File_Transfer_Protocol" config="/sftp/" >}}
|
||||
{{< provider name="Sia" home="https://sia.tech/" config="/sia/" >}}
|
||||
{{< provider name="StackPath" home="https://www.stackpath.com/products/object-storage/" config="/s3/#stackpath" >}}
|
||||
{{< provider name="Storj" home="https://storj.io/" config="/storj/" >}}
|
||||
{{< provider name="SugarSync" home="https://sugarsync.com/" config="/sugarsync/" >}}
|
||||
{{< provider name="Tardigrade" home="https://tardigrade.io/" config="/tardigrade/" >}}
|
||||
{{< provider name="Tencent Cloud Object Storage (COS)" home="https://intl.cloud.tencent.com/product/cos" config="/s3/#tencent-cos" >}}
|
||||
{{< provider name="Uptobox" home="https://uptobox.com" config="/uptobox/" >}}
|
||||
{{< provider name="Wasabi" home="https://wasabi.com/" config="/s3/#wasabi" >}}
|
||||
|
|
|
|||
|
|
@ -164,8 +164,8 @@ Volumes can be created with [docker volume create](https://docs.docker.com/engin
|
|||
Here are a few examples:
|
||||
```
|
||||
docker volume create vol1 -d rclone -o remote=storj: -o vfs-cache-mode=full
|
||||
docker volume create vol2 -d rclone -o remote=:tardigrade,access_grant=xxx:heimdall
|
||||
docker volume create vol3 -d rclone -o type=tardigrade -o path=heimdall -o tardigrade-access-grant=xxx -o poll-interval=0
|
||||
docker volume create vol2 -d rclone -o remote=:storj,access_grant=xxx:heimdall
|
||||
docker volume create vol3 -d rclone -o type=storj -o path=heimdall -o storj-access-grant=xxx -o poll-interval=0
|
||||
```
|
||||
|
||||
Note the `-d rclone` flag that tells docker to request volume from the
|
||||
|
|
|
|||
|
|
@ -64,8 +64,8 @@ See the following for detailed instructions for
|
|||
* [Seafile](/seafile/)
|
||||
* [SFTP](/sftp/)
|
||||
* [Sia](/sia/)
|
||||
* [Storj](/storj/)
|
||||
* [SugarSync](/sugarsync/)
|
||||
* [Tardigrade](/tardigrade/)
|
||||
* [Union](/union/)
|
||||
* [Uptobox](/uptobox/)
|
||||
* [WebDAV](/webdav/)
|
||||
|
|
@ -550,7 +550,7 @@ Is equivalent to this:
|
|||
|
||||
Bandwidth limit apply to the data transfer for all backends. For most
|
||||
backends the directory listing bandwidth is also included (exceptions
|
||||
being the non HTTP backends, `ftp`, `sftp` and `tardigrade`).
|
||||
being the non HTTP backends, `ftp`, `sftp` and `storj`).
|
||||
|
||||
Note that the units are **Byte/s**, not **bit/s**. Typically
|
||||
connections are measured in bit/s - to convert divide by 8. For
|
||||
|
|
@ -1651,8 +1651,7 @@ This can be very useful for `rclone mount` to control the behaviour of
|
|||
applications using it.
|
||||
|
||||
This limit applies to all HTTP based backends and to the FTP and SFTP
|
||||
backends. It does not apply to the local backend or the Tardigrade
|
||||
backend.
|
||||
backends. It does not apply to the local backend or the Storj backend.
|
||||
|
||||
See also `--tpslimit-burst`.
|
||||
|
||||
|
|
|
|||
|
|
@ -48,7 +48,7 @@ Here is an overview of the major features of each cloud storage system.
|
|||
| SFTP | MD5, SHA1 ² | Yes | Depends | No | - |
|
||||
| Sia | - | No | No | No | - |
|
||||
| SugarSync | - | No | No | No | - |
|
||||
| Tardigrade | - | Yes | No | No | - |
|
||||
| Storj | - | Yes | No | No | - |
|
||||
| Uptobox | - | No | No | Yes | - |
|
||||
| WebDAV | MD5, SHA1 ³ | Yes ⁴ | Depends | No | - |
|
||||
| Yandex Disk | MD5 | Yes | No | No | R |
|
||||
|
|
@ -440,7 +440,7 @@ upon backend-specific capabilities.
|
|||
| Seafile | Yes | Yes | Yes | Yes | Yes | Yes | Yes | Yes | Yes | Yes |
|
||||
| SFTP | No | No | Yes | Yes | No | No | Yes | No | Yes | Yes |
|
||||
| SugarSync | Yes | Yes | Yes | Yes | No | No | Yes | Yes | No | Yes |
|
||||
| Tardigrade | Yes † | No | No | No | No | Yes | Yes | No | No | No |
|
||||
| Storj | Yes † | No | No | No | No | Yes | Yes | No | No | No |
|
||||
| Uptobox | No | Yes | Yes | Yes | No | No | No | No | No | No |
|
||||
| WebDAV | Yes | Yes | Yes | Yes | No | No | Yes ‡ | No | Yes | Yes |
|
||||
| Yandex Disk | Yes | Yes | Yes | Yes | Yes | No | Yes | Yes | Yes | Yes |
|
||||
|
|
@ -452,7 +452,7 @@ upon backend-specific capabilities.
|
|||
This deletes a directory quicker than just deleting all the files in
|
||||
the directory.
|
||||
|
||||
† Note Swift, Hubic, and Tardigrade implement this in order to delete
|
||||
† Note Swift, Hubic, and Storj implement this in order to delete
|
||||
directory markers but they don't actually have a quicker way of deleting
|
||||
files other than deleting them individually.
|
||||
|
||||
|
|
|
|||
|
|
@ -2908,7 +2908,7 @@ files larger than 5G then using `--checksum` or `--size-only` in
|
|||
|
||||
#### Comparison with the native protocol
|
||||
|
||||
Use the [the native protocol](/tardigrade) to take advantage of
|
||||
Use the [the native protocol](/storj) to take advantage of
|
||||
client-side encryption as well as to achieve the best possible
|
||||
download performance. Uploads will be erasure-coded locally, thus a
|
||||
1gb upload will result in 2.68gb of data being uploaded to storage
|
||||
|
|
@ -2921,7 +2921,7 @@ upload will result in only in 1GB of data being uploaded to storage
|
|||
nodes across the network.
|
||||
|
||||
For more detailed comparison please check the documentation of the
|
||||
[tardigrade](/tardigrade) backend.
|
||||
[storj](/storj) backend.
|
||||
|
||||
## Limitations
|
||||
|
||||
|
|
|
|||
411
docs/content/storj.md
Normal file
411
docs/content/storj.md
Normal file
|
|
@ -0,0 +1,411 @@
|
|||
---
|
||||
title: "Storj"
|
||||
description: "Rclone docs for Storj"
|
||||
---
|
||||
|
||||
# {{< icon "fas fa-dove" >}} Storj
|
||||
|
||||
[Storj](https://storj.io) is an encrypted, secure, and
|
||||
cost-effective object storage service that enables you to store, back up, and
|
||||
archive large amounts of data in a decentralized manner.
|
||||
|
||||
## Backend options
|
||||
|
||||
Storj can be used both with this native backend and with the [s3
|
||||
backend using the Storj S3 compatible gateway](/s3/#storj) (shared or private).
|
||||
|
||||
Use this backend to take advantage of client-side encryption as well
|
||||
as to achieve the best possible download performance. Uploads will be
|
||||
erasure-coded locally, thus a 1gb upload will result in 2.68gb of data
|
||||
being uploaded to storage nodes across the network.
|
||||
|
||||
Use the s3 backend and one of the S3 compatible Hosted Gateways to
|
||||
increase upload performance and reduce the load on your systems and
|
||||
network. Uploads will be encrypted and erasure-coded server-side, thus
|
||||
a 1GB upload will result in only in 1GB of data being uploaded to
|
||||
storage nodes across the network.
|
||||
|
||||
Side by side comparison with more details:
|
||||
|
||||
* Characteristics:
|
||||
* *Storj backend*: Uses native RPC protocol, connects directly
|
||||
to the storage nodes which hosts the data. Requires more CPU
|
||||
resource of encoding/decoding and has network amplification
|
||||
(especially during the upload), uses lots of TCP connections
|
||||
* *S3 backend*: Uses S3 compatible HTTP Rest API via the shared
|
||||
gateways. There is no network amplification, but performance
|
||||
depends on the shared gateways and the secret encryption key is
|
||||
shared with the gateway.
|
||||
* Typical usage:
|
||||
* *Storj backend*: Server environments and desktops with enough
|
||||
resources, internet speed and connectivity - and applications
|
||||
where storjs client-side encryption is required.
|
||||
* *S3 backend*: Desktops and similar with limited resources,
|
||||
internet speed or connectivity.
|
||||
* Security:
|
||||
* *Storj backend*: __strong__. Private encryption key doesn't
|
||||
need to leave the local computer.
|
||||
* *S3 backend*: __weaker__. Private encryption key is [shared
|
||||
with](https://docs.storj.io/dcs/api-reference/s3-compatible-gateway#security-and-encryption)
|
||||
the authentication service of the hosted gateway, where it's
|
||||
stored encrypted. It can be stronger when combining with the
|
||||
rclone [crypt](/crypt) backend.
|
||||
* Bandwidth usage (upload):
|
||||
* *Storj backend*: __higher__. As data is erasure coded on the
|
||||
client side both the original data and the parities should be
|
||||
uploaded. About ~2.7 times more data is required to be uploaded.
|
||||
Client may start to upload with even higher number of nodes (~3.7
|
||||
times more) and abandon/stop the slow uploads.
|
||||
* *S3 backend*: __normal__. Only the raw data is uploaded, erasure
|
||||
coding happens on the gateway.
|
||||
* Bandwidth usage (download)
|
||||
* *Storj backend*: __almost normal__. Only the minimal number
|
||||
of data is required, but to avoid very slow data providers a few
|
||||
more sources are used and the slowest are ignored (max 1.2x
|
||||
overhead).
|
||||
* *S3 backend*: __normal__. Only the raw data is downloaded, erasure coding happens on the shared gateway.
|
||||
* CPU usage:
|
||||
* *Storj backend*: __higher__, but more predictable. Erasure
|
||||
code and encryption/decryption happens locally which requires
|
||||
significant CPU usage.
|
||||
* *S3 backend*: __less__. Erasure code and encryption/decryption
|
||||
happens on shared s3 gateways (and as is, it depends on the
|
||||
current load on the gateways)
|
||||
* TCP connection usage:
|
||||
* *Storj backend*: __high__. A direct connection is required to
|
||||
each of the Storj nodes resulting in 110 connections on upload and
|
||||
35 on download per 64 MB segment. Not all the connections are
|
||||
actively used (slow ones are pruned), but they are all opened.
|
||||
[Adjusting the max open file limit](/storj/#known-issues) may
|
||||
be required.
|
||||
* *S3 backend*: __normal__. Only one connection per download/upload
|
||||
thread is required to the shared gateway.
|
||||
* Overall performance:
|
||||
* *Storj backend*: with enough resources (CPU and bandwidth)
|
||||
*storj* backend can provide even 2x better performance. Data
|
||||
is directly downloaded to / uploaded from to the client instead of
|
||||
the gateway.
|
||||
* *S3 backend*: Can be faster on edge devices where CPU and network
|
||||
bandwidth is limited as the shared S3 compatible gateways take
|
||||
care about the encrypting/decryption and erasure coding and no
|
||||
download/upload amplification.
|
||||
* Decentralization:
|
||||
* *Storj backend*: __high__. Data is downloaded directly from
|
||||
the distributed cloud of storage providers.
|
||||
* *S3 backend*: __low__. Requires a running S3 gateway (either
|
||||
self-hosted or Storj-hosted).
|
||||
* Limitations:
|
||||
* *Storj backend*: `rclone checksum` is not possible without
|
||||
download, as checksum metadata is not calculated during upload
|
||||
* *S3 backend*: secret encryption key is shared with the gateway
|
||||
|
||||
## Configuration
|
||||
|
||||
To make a new Storj configuration you need one of the following:
|
||||
* Access Grant that someone else shared with you.
|
||||
* [API Key](https://documentation.storj.io/getting-started/uploading-your-first-object/create-an-api-key)
|
||||
of a Storj project you are a member of.
|
||||
|
||||
Here is an example of how to make a remote called `remote`. First run:
|
||||
|
||||
rclone config
|
||||
|
||||
This will guide you through an interactive setup process:
|
||||
|
||||
### Setup with access grant
|
||||
|
||||
```
|
||||
No remotes found, make a new one?
|
||||
n) New remote
|
||||
s) Set configuration password
|
||||
q) Quit config
|
||||
n/s/q> n
|
||||
name> remote
|
||||
Type of storage to configure.
|
||||
Enter a string value. Press Enter for the default ("").
|
||||
Choose a number from below, or type in your own value
|
||||
[snip]
|
||||
XX / Storj Decentralized Cloud Storage
|
||||
\ "storj"
|
||||
[snip]
|
||||
Storage> storj
|
||||
** See help for storj backend at: https://rclone.org/storj/ **
|
||||
|
||||
Choose an authentication method.
|
||||
Enter a string value. Press Enter for the default ("existing").
|
||||
Choose a number from below, or type in your own value
|
||||
1 / Use an existing access grant.
|
||||
\ "existing"
|
||||
2 / Create a new access grant from satellite address, API key, and passphrase.
|
||||
\ "new"
|
||||
provider> existing
|
||||
Access Grant.
|
||||
Enter a string value. Press Enter for the default ("").
|
||||
access_grant> your-access-grant-received-by-someone-else
|
||||
Remote config
|
||||
--------------------
|
||||
[remote]
|
||||
type = storj
|
||||
access_grant = your-access-grant-received-by-someone-else
|
||||
--------------------
|
||||
y) Yes this is OK (default)
|
||||
e) Edit this remote
|
||||
d) Delete this remote
|
||||
y/e/d> y
|
||||
```
|
||||
|
||||
### Setup with API key and passphrase
|
||||
|
||||
```
|
||||
No remotes found, make a new one?
|
||||
n) New remote
|
||||
s) Set configuration password
|
||||
q) Quit config
|
||||
n/s/q> n
|
||||
name> remote
|
||||
Type of storage to configure.
|
||||
Enter a string value. Press Enter for the default ("").
|
||||
Choose a number from below, or type in your own value
|
||||
[snip]
|
||||
XX / Storj Decentralized Cloud Storage
|
||||
\ "storj"
|
||||
[snip]
|
||||
Storage> storj
|
||||
** See help for storj backend at: https://rclone.org/storj/ **
|
||||
|
||||
Choose an authentication method.
|
||||
Enter a string value. Press Enter for the default ("existing").
|
||||
Choose a number from below, or type in your own value
|
||||
1 / Use an existing access grant.
|
||||
\ "existing"
|
||||
2 / Create a new access grant from satellite address, API key, and passphrase.
|
||||
\ "new"
|
||||
provider> new
|
||||
Satellite Address. Custom satellite address should match the format: `<nodeid>@<address>:<port>`.
|
||||
Enter a string value. Press Enter for the default ("us-central-1.storj.io").
|
||||
Choose a number from below, or type in your own value
|
||||
1 / US Central 1
|
||||
\ "us-central-1.storj.io"
|
||||
2 / Europe West 1
|
||||
\ "europe-west-1.storj.io"
|
||||
3 / Asia East 1
|
||||
\ "asia-east-1.storj.io"
|
||||
satellite_address> 1
|
||||
API Key.
|
||||
Enter a string value. Press Enter for the default ("").
|
||||
api_key> your-api-key-for-your-storj-project
|
||||
Encryption Passphrase. To access existing objects enter passphrase used for uploading.
|
||||
Enter a string value. Press Enter for the default ("").
|
||||
passphrase> your-human-readable-encryption-passphrase
|
||||
Remote config
|
||||
--------------------
|
||||
[remote]
|
||||
type = storj
|
||||
satellite_address = 12EayRS2V1kEsWESU9QMRseFhdxYxKicsiFmxrsLZHeLUtdps3S@us-central-1.tardigrade.io:7777
|
||||
api_key = your-api-key-for-your-storj-project
|
||||
passphrase = your-human-readable-encryption-passphrase
|
||||
access_grant = the-access-grant-generated-from-the-api-key-and-passphrase
|
||||
--------------------
|
||||
y) Yes this is OK (default)
|
||||
e) Edit this remote
|
||||
d) Delete this remote
|
||||
y/e/d> y
|
||||
```
|
||||
|
||||
{{< rem autogenerated options start" - DO NOT EDIT - instead edit fs.RegInfo in backend/storj/storj.go then run make backenddocs" >}}
|
||||
### Standard options
|
||||
|
||||
Here are the standard options specific to storj (Storj Decentralized Cloud Storage).
|
||||
|
||||
#### --storj-provider
|
||||
|
||||
Choose an authentication method.
|
||||
|
||||
- Config: provider
|
||||
- Env Var: RCLONE_STORJ_PROVIDER
|
||||
- Type: string
|
||||
- Default: "existing"
|
||||
- Examples:
|
||||
- "existing"
|
||||
- Use an existing access grant.
|
||||
- "new"
|
||||
- Create a new access grant from satellite address, API key, and passphrase.
|
||||
|
||||
#### --storj-access-grant
|
||||
|
||||
Access grant.
|
||||
|
||||
- Config: access_grant
|
||||
- Env Var: RCLONE_STORJ_ACCESS_GRANT
|
||||
- Type: string
|
||||
- Default: ""
|
||||
|
||||
#### --storj-satellite-address
|
||||
|
||||
Satellite address.
|
||||
|
||||
Custom satellite address should match the format: `<nodeid>@<address>:<port>`.
|
||||
|
||||
- Config: satellite_address
|
||||
- Env Var: RCLONE_STORJ_SATELLITE_ADDRESS
|
||||
- Type: string
|
||||
- Default: "us-central-1.storj.io"
|
||||
- Examples:
|
||||
- "us-central-1.storj.io"
|
||||
- US Central 1
|
||||
- "europe-west-1.storj.io"
|
||||
- Europe West 1
|
||||
- "asia-east-1.storj.io"
|
||||
- Asia East 1
|
||||
|
||||
#### --storj-api-key
|
||||
|
||||
API key.
|
||||
|
||||
- Config: api_key
|
||||
- Env Var: RCLONE_STORJ_API_KEY
|
||||
- Type: string
|
||||
- Default: ""
|
||||
|
||||
#### --storj-passphrase
|
||||
|
||||
Encryption passphrase.
|
||||
|
||||
To access existing objects enter passphrase used for uploading.
|
||||
|
||||
- Config: passphrase
|
||||
- Env Var: RCLONE_STORJ_PASSPHRASE
|
||||
- Type: string
|
||||
- Default: ""
|
||||
|
||||
{{< rem autogenerated options stop >}}
|
||||
|
||||
## Usage
|
||||
|
||||
Paths are specified as `remote:bucket` (or `remote:` for the `lsf`
|
||||
command.) You may put subdirectories in too, e.g. `remote:bucket/path/to/dir`.
|
||||
|
||||
Once configured you can then use `rclone` like this.
|
||||
|
||||
### Create a new bucket
|
||||
|
||||
Use the `mkdir` command to create new bucket, e.g. `bucket`.
|
||||
|
||||
rclone mkdir remote:bucket
|
||||
|
||||
### List all buckets
|
||||
|
||||
Use the `lsf` command to list all buckets.
|
||||
|
||||
rclone lsf remote:
|
||||
|
||||
Note the colon (`:`) character at the end of the command line.
|
||||
|
||||
### Delete a bucket
|
||||
|
||||
Use the `rmdir` command to delete an empty bucket.
|
||||
|
||||
rclone rmdir remote:bucket
|
||||
|
||||
Use the `purge` command to delete a non-empty bucket with all its content.
|
||||
|
||||
rclone purge remote:bucket
|
||||
|
||||
### Upload objects
|
||||
|
||||
Use the `copy` command to upload an object.
|
||||
|
||||
rclone copy --progress /home/local/directory/file.ext remote:bucket/path/to/dir/
|
||||
|
||||
The `--progress` flag is for displaying progress information.
|
||||
Remove it if you don't need this information.
|
||||
|
||||
Use a folder in the local path to upload all its objects.
|
||||
|
||||
rclone copy --progress /home/local/directory/ remote:bucket/path/to/dir/
|
||||
|
||||
Only modified files will be copied.
|
||||
|
||||
### List objects
|
||||
|
||||
Use the `ls` command to list recursively all objects in a bucket.
|
||||
|
||||
rclone ls remote:bucket
|
||||
|
||||
Add the folder to the remote path to list recursively all objects in this folder.
|
||||
|
||||
rclone ls remote:bucket/path/to/dir/
|
||||
|
||||
Use the `lsf` command to list non-recursively all objects in a bucket or a folder.
|
||||
|
||||
rclone lsf remote:bucket/path/to/dir/
|
||||
|
||||
### Download objects
|
||||
|
||||
Use the `copy` command to download an object.
|
||||
|
||||
rclone copy --progress remote:bucket/path/to/dir/file.ext /home/local/directory/
|
||||
|
||||
The `--progress` flag is for displaying progress information.
|
||||
Remove it if you don't need this information.
|
||||
|
||||
Use a folder in the remote path to download all its objects.
|
||||
|
||||
rclone copy --progress remote:bucket/path/to/dir/ /home/local/directory/
|
||||
|
||||
### Delete objects
|
||||
|
||||
Use the `deletefile` command to delete a single object.
|
||||
|
||||
rclone deletefile remote:bucket/path/to/dir/file.ext
|
||||
|
||||
Use the `delete` command to delete all object in a folder.
|
||||
|
||||
rclone delete remote:bucket/path/to/dir/
|
||||
|
||||
### Print the total size of objects
|
||||
|
||||
Use the `size` command to print the total size of objects in a bucket or a folder.
|
||||
|
||||
rclone size remote:bucket/path/to/dir/
|
||||
|
||||
### Sync two Locations
|
||||
|
||||
Use the `sync` command to sync the source to the destination,
|
||||
changing the destination only, deleting any excess files.
|
||||
|
||||
rclone sync -i --progress /home/local/directory/ remote:bucket/path/to/dir/
|
||||
|
||||
The `--progress` flag is for displaying progress information.
|
||||
Remove it if you don't need this information.
|
||||
|
||||
Since this can cause data loss, test first with the `--dry-run` flag
|
||||
to see exactly what would be copied and deleted.
|
||||
|
||||
The sync can be done also from Storj to the local file system.
|
||||
|
||||
rclone sync -i --progress remote:bucket/path/to/dir/ /home/local/directory/
|
||||
|
||||
Or between two Storj buckets.
|
||||
|
||||
rclone sync -i --progress remote-us:bucket/path/to/dir/ remote-europe:bucket/path/to/dir/
|
||||
|
||||
Or even between another cloud storage and Storj.
|
||||
|
||||
rclone sync -i --progress s3:bucket/path/to/dir/ storj:bucket/path/to/dir/
|
||||
|
||||
## Limitations
|
||||
|
||||
`rclone about` is not supported by the rclone Storj backend. Backends without
|
||||
this capability cannot determine free space for an rclone mount or
|
||||
use policy `mfs` (most free space) as a member of an rclone union
|
||||
remote.
|
||||
|
||||
See [List of backends that do not support rclone about](https://rclone.org/overview/#optional-features)
|
||||
See [rclone about](https://rclone.org/commands/rclone_about/)
|
||||
|
||||
## Known issues
|
||||
|
||||
If you get errors like `too many open files` this usually happens when the default `ulimit` for system max open files is exceeded. Native Storj protocol opens a large number of TCP connections (each of which is counted as an open file). For a single upload stream you can expect 110 TCP connections to be opened. For a single download stream you can expect 35. This batch of connections will be opened for every 64 MiB segment and you should also expect TCP connections to be reused. If you do many transfers you eventually open a connection to most storage nodes (thousands of nodes).
|
||||
|
||||
To fix these, please raise your system limits. You can do this issuing a `ulimit -n 65536` just before you run rclone. To change the limits more permanently you can add this to your shell startup script, e.g. `$HOME/.bashrc`, or change the system-wide configuration, usually `/etc/sysctl.conf` and/or `/etc/security/limits.conf`, but please refer to your operating system manual.
|
||||
|
|
@ -5,407 +5,5 @@ description: "Rclone docs for Tardigrade"
|
|||
|
||||
# {{< icon "fas fa-dove" >}} Tardigrade
|
||||
|
||||
[Tardigrade](https://tardigrade.io) is an encrypted, secure, and
|
||||
cost-effective object storage service that enables you to store, back up, and
|
||||
archive large amounts of data in a decentralized manner.
|
||||
|
||||
## Backend options
|
||||
|
||||
Storj can be used both with this native backend and with the [s3
|
||||
backend using the Storj S3 compatible gateway](/s3/#storj) (shared or private).
|
||||
|
||||
Use this backend to take advantage of client-side encryption as well
|
||||
as to achieve the best possible download performance. Uploads will be
|
||||
erasure-coded locally, thus a 1gb upload will result in 2.68gb of data
|
||||
being uploaded to storage nodes across the network.
|
||||
|
||||
Use the s3 backend and one of the S3 compatible Hosted Gateways to
|
||||
increase upload performance and reduce the load on your systems and
|
||||
network. Uploads will be encrypted and erasure-coded server-side, thus
|
||||
a 1GB upload will result in only in 1GB of data being uploaded to
|
||||
storage nodes across the network.
|
||||
|
||||
Side by side comparison with more details:
|
||||
|
||||
* Characteristics:
|
||||
* *Tardigrade backend*: Uses native RPC protocol, connects directly
|
||||
to the storage nodes which hosts the data. Requires more CPU
|
||||
resource of encoding/decoding and has network amplification
|
||||
(especially during the upload), uses lots of TCP connections
|
||||
* *S3 backend*: Uses S3 compatible HTTP Rest API via the shared
|
||||
gateways. There is no network amplification, but performance
|
||||
depends on the shared gateways and the secret encryption key is
|
||||
shared with the gateway.
|
||||
* Typical usage:
|
||||
* *Tardigrade backend*: Server environments and desktops with enough
|
||||
resources, internet speed and connectivity - and applications
|
||||
where tardigrades client-side encryption is required.
|
||||
* *S3 backend*: Desktops and similar with limited resources,
|
||||
internet speed or connectivity.
|
||||
* Security:
|
||||
* *Tardigrade backend*: __strong__. Private encryption key doesn't
|
||||
need to leave the local computer.
|
||||
* *S3 backend*: __weaker__. Private encryption key is [shared
|
||||
with](https://docs.storj.io/dcs/api-reference/s3-compatible-gateway#security-and-encryption)
|
||||
the authentication service of the hosted gateway, where it's
|
||||
stored encrypted. It can be stronger when combining with the
|
||||
rclone [crypt](/crypt) backend.
|
||||
* Bandwidth usage (upload):
|
||||
* *Tardigrade backend*: __higher__. As data is erasure coded on the
|
||||
client side both the original data and the parities should be
|
||||
uploaded. About ~2.7 times more data is required to be uploaded.
|
||||
Client may start to upload with even higher number of nodes (~3.7
|
||||
times more) and abandon/stop the slow uploads.
|
||||
* *S3 backend*: __normal__. Only the raw data is uploaded, erasure
|
||||
coding happens on the gateway.
|
||||
* Bandwidth usage (download)
|
||||
* *Tardigrade backend*: __almost normal__. Only the minimal number
|
||||
of data is required, but to avoid very slow data providers a few
|
||||
more sources are used and the slowest are ignored (max 1.2x
|
||||
overhead).
|
||||
* *S3 backend*: __normal__. Only the raw data is downloaded, erasure coding happens on the shared gateway.
|
||||
* CPU usage:
|
||||
* *Tardigrade backend*: __higher__, but more predictable. Erasure
|
||||
code and encryption/decryption happens locally which requires
|
||||
significant CPU usage.
|
||||
* *S3 backend*: __less__. Erasure code and encryption/decryption
|
||||
happens on shared s3 gateways (and as is, it depends on the
|
||||
current load on the gateways)
|
||||
* TCP connection usage:
|
||||
* *Tardigrade backend*: __high__. A direct connection is required to
|
||||
each of the Storj nodes resulting in 110 connections on upload and
|
||||
35 on download per 64 MB segment. Not all the connections are
|
||||
actively used (slow ones are pruned), but they are all opened.
|
||||
[Adjusting the max open file limit](/tardigrade/#known-issues) may
|
||||
be required.
|
||||
* *S3 backend*: __normal__. Only one connection per download/upload
|
||||
thread is required to the shared gateway.
|
||||
* Overall performance:
|
||||
* *Tardigrade backend*: with enough resources (CPU and bandwidth)
|
||||
*tardigrade* backend can provide even 2x better performance. Data
|
||||
is directly downloaded to / uploaded from to the client instead of
|
||||
the gateway.
|
||||
* *S3 backend*: Can be faster on edge devices where CPU and network
|
||||
bandwidth is limited as the shared S3 compatible gateways take
|
||||
care about the encrypting/decryption and erasure coding and no
|
||||
download/upload amplification.
|
||||
* Decentralization:
|
||||
* *Tardigrade backend*: __high__. Data is downloaded directly from
|
||||
the distributed cloud of storage providers.
|
||||
* *S3 backend*: __low__. Requires a running S3 gateway (either
|
||||
self-hosted or Storj-hosted).
|
||||
* Limitations:
|
||||
* *Tardigrade backend*: `rclone checksum` is not possible without
|
||||
download, as checksum metadata is not calculated during upload
|
||||
* *S3 backend*: secret encryption key is shared with the gateway
|
||||
|
||||
## Configuration
|
||||
|
||||
To make a new Tardigrade configuration you need one of the following:
|
||||
* Access Grant that someone else shared with you.
|
||||
* [API Key](https://documentation.tardigrade.io/getting-started/uploading-your-first-object/create-an-api-key)
|
||||
of a Tardigrade project you are a member of.
|
||||
|
||||
Here is an example of how to make a remote called `remote`. First run:
|
||||
|
||||
rclone config
|
||||
|
||||
This will guide you through an interactive setup process:
|
||||
|
||||
### Setup with access grant
|
||||
|
||||
```
|
||||
No remotes found, make a new one?
|
||||
n) New remote
|
||||
s) Set configuration password
|
||||
q) Quit config
|
||||
n/s/q> n
|
||||
name> remote
|
||||
Type of storage to configure.
|
||||
Enter a string value. Press Enter for the default ("").
|
||||
Choose a number from below, or type in your own value
|
||||
[snip]
|
||||
XX / Tardigrade Decentralized Cloud Storage
|
||||
\ "tardigrade"
|
||||
[snip]
|
||||
Storage> tardigrade
|
||||
** See help for tardigrade backend at: https://rclone.org/tardigrade/ **
|
||||
|
||||
Choose an authentication method.
|
||||
Enter a string value. Press Enter for the default ("existing").
|
||||
Choose a number from below, or type in your own value
|
||||
1 / Use an existing access grant.
|
||||
\ "existing"
|
||||
2 / Create a new access grant from satellite address, API key, and passphrase.
|
||||
\ "new"
|
||||
provider> existing
|
||||
Access Grant.
|
||||
Enter a string value. Press Enter for the default ("").
|
||||
access_grant> your-access-grant-received-by-someone-else
|
||||
Remote config
|
||||
--------------------
|
||||
[remote]
|
||||
type = tardigrade
|
||||
access_grant = your-access-grant-received-by-someone-else
|
||||
--------------------
|
||||
y) Yes this is OK (default)
|
||||
e) Edit this remote
|
||||
d) Delete this remote
|
||||
y/e/d> y
|
||||
```
|
||||
|
||||
### Setup with API key and passphrase
|
||||
|
||||
```
|
||||
No remotes found, make a new one?
|
||||
n) New remote
|
||||
s) Set configuration password
|
||||
q) Quit config
|
||||
n/s/q> n
|
||||
name> remote
|
||||
Type of storage to configure.
|
||||
Enter a string value. Press Enter for the default ("").
|
||||
Choose a number from below, or type in your own value
|
||||
[snip]
|
||||
XX / Tardigrade Decentralized Cloud Storage
|
||||
\ "tardigrade"
|
||||
[snip]
|
||||
Storage> tardigrade
|
||||
** See help for tardigrade backend at: https://rclone.org/tardigrade/ **
|
||||
|
||||
Choose an authentication method.
|
||||
Enter a string value. Press Enter for the default ("existing").
|
||||
Choose a number from below, or type in your own value
|
||||
1 / Use an existing access grant.
|
||||
\ "existing"
|
||||
2 / Create a new access grant from satellite address, API key, and passphrase.
|
||||
\ "new"
|
||||
provider> new
|
||||
Satellite Address. Custom satellite address should match the format: `<nodeid>@<address>:<port>`.
|
||||
Enter a string value. Press Enter for the default ("us-central-1.tardigrade.io").
|
||||
Choose a number from below, or type in your own value
|
||||
1 / US Central 1
|
||||
\ "us-central-1.tardigrade.io"
|
||||
2 / Europe West 1
|
||||
\ "europe-west-1.tardigrade.io"
|
||||
3 / Asia East 1
|
||||
\ "asia-east-1.tardigrade.io"
|
||||
satellite_address> 1
|
||||
API Key.
|
||||
Enter a string value. Press Enter for the default ("").
|
||||
api_key> your-api-key-for-your-tardigrade-project
|
||||
Encryption Passphrase. To access existing objects enter passphrase used for uploading.
|
||||
Enter a string value. Press Enter for the default ("").
|
||||
passphrase> your-human-readable-encryption-passphrase
|
||||
Remote config
|
||||
--------------------
|
||||
[remote]
|
||||
type = tardigrade
|
||||
satellite_address = 12EayRS2V1kEsWESU9QMRseFhdxYxKicsiFmxrsLZHeLUtdps3S@us-central-1.tardigrade.io:7777
|
||||
api_key = your-api-key-for-your-tardigrade-project
|
||||
passphrase = your-human-readable-encryption-passphrase
|
||||
access_grant = the-access-grant-generated-from-the-api-key-and-passphrase
|
||||
--------------------
|
||||
y) Yes this is OK (default)
|
||||
e) Edit this remote
|
||||
d) Delete this remote
|
||||
y/e/d> y
|
||||
```
|
||||
|
||||
{{< rem autogenerated options start" - DO NOT EDIT - instead edit fs.RegInfo in backend/tardigrade/tardigrade.go then run make backenddocs" >}}
|
||||
### Standard options
|
||||
|
||||
Here are the standard options specific to tardigrade (Tardigrade Decentralized Cloud Storage).
|
||||
|
||||
#### --tardigrade-provider
|
||||
|
||||
Choose an authentication method.
|
||||
|
||||
- Config: provider
|
||||
- Env Var: RCLONE_TARDIGRADE_PROVIDER
|
||||
- Type: string
|
||||
- Default: "existing"
|
||||
- Examples:
|
||||
- "existing"
|
||||
- Use an existing access grant.
|
||||
- "new"
|
||||
- Create a new access grant from satellite address, API key, and passphrase.
|
||||
|
||||
#### --tardigrade-access-grant
|
||||
|
||||
Access grant.
|
||||
|
||||
- Config: access_grant
|
||||
- Env Var: RCLONE_TARDIGRADE_ACCESS_GRANT
|
||||
- Type: string
|
||||
- Default: ""
|
||||
|
||||
#### --tardigrade-satellite-address
|
||||
|
||||
Satellite address.
|
||||
|
||||
Custom satellite address should match the format: `<nodeid>@<address>:<port>`.
|
||||
|
||||
- Config: satellite_address
|
||||
- Env Var: RCLONE_TARDIGRADE_SATELLITE_ADDRESS
|
||||
- Type: string
|
||||
- Default: "us-central-1.tardigrade.io"
|
||||
- Examples:
|
||||
- "us-central-1.tardigrade.io"
|
||||
- US Central 1
|
||||
- "europe-west-1.tardigrade.io"
|
||||
- Europe West 1
|
||||
- "asia-east-1.tardigrade.io"
|
||||
- Asia East 1
|
||||
|
||||
#### --tardigrade-api-key
|
||||
|
||||
API key.
|
||||
|
||||
- Config: api_key
|
||||
- Env Var: RCLONE_TARDIGRADE_API_KEY
|
||||
- Type: string
|
||||
- Default: ""
|
||||
|
||||
#### --tardigrade-passphrase
|
||||
|
||||
Encryption passphrase.
|
||||
|
||||
To access existing objects enter passphrase used for uploading.
|
||||
|
||||
- Config: passphrase
|
||||
- Env Var: RCLONE_TARDIGRADE_PASSPHRASE
|
||||
- Type: string
|
||||
- Default: ""
|
||||
|
||||
{{< rem autogenerated options stop >}}
|
||||
|
||||
## Usage
|
||||
|
||||
Paths are specified as `remote:bucket` (or `remote:` for the `lsf`
|
||||
command.) You may put subdirectories in too, e.g. `remote:bucket/path/to/dir`.
|
||||
|
||||
Once configured you can then use `rclone` like this.
|
||||
|
||||
### Create a new bucket
|
||||
|
||||
Use the `mkdir` command to create new bucket, e.g. `bucket`.
|
||||
|
||||
rclone mkdir remote:bucket
|
||||
|
||||
### List all buckets
|
||||
|
||||
Use the `lsf` command to list all buckets.
|
||||
|
||||
rclone lsf remote:
|
||||
|
||||
Note the colon (`:`) character at the end of the command line.
|
||||
|
||||
### Delete a bucket
|
||||
|
||||
Use the `rmdir` command to delete an empty bucket.
|
||||
|
||||
rclone rmdir remote:bucket
|
||||
|
||||
Use the `purge` command to delete a non-empty bucket with all its content.
|
||||
|
||||
rclone purge remote:bucket
|
||||
|
||||
### Upload objects
|
||||
|
||||
Use the `copy` command to upload an object.
|
||||
|
||||
rclone copy --progress /home/local/directory/file.ext remote:bucket/path/to/dir/
|
||||
|
||||
The `--progress` flag is for displaying progress information.
|
||||
Remove it if you don't need this information.
|
||||
|
||||
Use a folder in the local path to upload all its objects.
|
||||
|
||||
rclone copy --progress /home/local/directory/ remote:bucket/path/to/dir/
|
||||
|
||||
Only modified files will be copied.
|
||||
|
||||
### List objects
|
||||
|
||||
Use the `ls` command to list recursively all objects in a bucket.
|
||||
|
||||
rclone ls remote:bucket
|
||||
|
||||
Add the folder to the remote path to list recursively all objects in this folder.
|
||||
|
||||
rclone ls remote:bucket/path/to/dir/
|
||||
|
||||
Use the `lsf` command to list non-recursively all objects in a bucket or a folder.
|
||||
|
||||
rclone lsf remote:bucket/path/to/dir/
|
||||
|
||||
### Download objects
|
||||
|
||||
Use the `copy` command to download an object.
|
||||
|
||||
rclone copy --progress remote:bucket/path/to/dir/file.ext /home/local/directory/
|
||||
|
||||
The `--progress` flag is for displaying progress information.
|
||||
Remove it if you don't need this information.
|
||||
|
||||
Use a folder in the remote path to download all its objects.
|
||||
|
||||
rclone copy --progress remote:bucket/path/to/dir/ /home/local/directory/
|
||||
|
||||
### Delete objects
|
||||
|
||||
Use the `deletefile` command to delete a single object.
|
||||
|
||||
rclone deletefile remote:bucket/path/to/dir/file.ext
|
||||
|
||||
Use the `delete` command to delete all object in a folder.
|
||||
|
||||
rclone delete remote:bucket/path/to/dir/
|
||||
|
||||
### Print the total size of objects
|
||||
|
||||
Use the `size` command to print the total size of objects in a bucket or a folder.
|
||||
|
||||
rclone size remote:bucket/path/to/dir/
|
||||
|
||||
### Sync two Locations
|
||||
|
||||
Use the `sync` command to sync the source to the destination,
|
||||
changing the destination only, deleting any excess files.
|
||||
|
||||
rclone sync -i --progress /home/local/directory/ remote:bucket/path/to/dir/
|
||||
|
||||
The `--progress` flag is for displaying progress information.
|
||||
Remove it if you don't need this information.
|
||||
|
||||
Since this can cause data loss, test first with the `--dry-run` flag
|
||||
to see exactly what would be copied and deleted.
|
||||
|
||||
The sync can be done also from Tardigrade to the local file system.
|
||||
|
||||
rclone sync -i --progress remote:bucket/path/to/dir/ /home/local/directory/
|
||||
|
||||
Or between two Tardigrade buckets.
|
||||
|
||||
rclone sync -i --progress remote-us:bucket/path/to/dir/ remote-europe:bucket/path/to/dir/
|
||||
|
||||
Or even between another cloud storage and Tardigrade.
|
||||
|
||||
rclone sync -i --progress s3:bucket/path/to/dir/ tardigrade:bucket/path/to/dir/
|
||||
|
||||
## Limitations
|
||||
|
||||
`rclone about` is not supported by the rclone Tardigrade backend. Backends without
|
||||
this capability cannot determine free space for an rclone mount or
|
||||
use policy `mfs` (most free space) as a member of an rclone union
|
||||
remote.
|
||||
|
||||
See [List of backends that do not support rclone about](https://rclone.org/overview/#optional-features)
|
||||
See [rclone about](https://rclone.org/commands/rclone_about/)
|
||||
|
||||
## Known issues
|
||||
|
||||
If you get errors like `too many open files` this usually happens when the default `ulimit` for system max open files is exceeded. Native Storj protocol opens a large number of TCP connections (each of which is counted as an open file). For a single upload stream you can expect 110 TCP connections to be opened. For a single download stream you can expect 35. This batch of connections will be opened for every 64 MiB segment and you should also expect TCP connections to be reused. If you do many transfers you eventually open a connection to most storage nodes (thousands of nodes).
|
||||
|
||||
To fix these, please raise your system limits. You can do this issuing a `ulimit -n 65536` just before you run rclone. To change the limits more permanently you can add this to your shell startup script, e.g. `$HOME/.bashrc`, or change the system-wide configuration, usually `/etc/sysctl.conf` and/or `/etc/security/limits.conf`, but please refer to your operating system manual.
|
||||
The Tardigrade backend has been renamed to be the [Storj backend](/storj/).
|
||||
Old configuration files will continue to work.
|
||||
|
|
|
|||
|
|
@ -87,8 +87,8 @@
|
|||
<a class="dropdown-item" href="/seafile/"><i class="fa fa-server"></i> Seafile</a>
|
||||
<a class="dropdown-item" href="/sftp/"><i class="fa fa-server"></i> SFTP</a>
|
||||
<a class="dropdown-item" href="/sia/"><i class="fa fa-globe"></i> Sia</a>
|
||||
<a class="dropdown-item" href="/storj/"><i class="fas fa-dove"></i> Storj</a>
|
||||
<a class="dropdown-item" href="/sugarsync/"><i class="fas fa-dove"></i> SugarSync</a>
|
||||
<a class="dropdown-item" href="/tardigrade/"><i class="fas fa-dove"></i> Tardigrade</a>
|
||||
<a class="dropdown-item" href="/uptobox/"><i class="fa fa-archive"></i> Uptobox</a>
|
||||
<a class="dropdown-item" href="/union/"><i class="fa fa-link"></i> Union (merge backends)</a>
|
||||
<a class="dropdown-item" href="/webdav/"><i class="fa fa-server"></i> WebDAV</a>
|
||||
|
|
|
|||
|
|
@ -335,8 +335,8 @@ backends:
|
|||
- backend: "sia"
|
||||
remote: "TestSia:"
|
||||
fastlist: false
|
||||
- backend: "tardigrade"
|
||||
remote: "TestTardigrade:"
|
||||
- backend: "storj"
|
||||
remote: "TestStorj:"
|
||||
fastlist: true
|
||||
- backend: "zoho"
|
||||
remote: "TestZoho:"
|
||||
|
|
|
|||
Loading…
Reference in New Issue
Block a user