From 628ff8e52459b576945169e0bee8adbc23debf9d Mon Sep 17 00:00:00 2001 From: Oksana <142890647+oks-maytech@users.noreply.github.com> Date: Fri, 8 Sep 2023 16:31:29 +0300 Subject: [PATCH] quatrix: add backend to support Quatrix Co-authored-by: Volodymyr Kit --- README.md | 1 + backend/all/all.go | 1 + backend/quatrix/api/types.go | 182 +++++ backend/quatrix/quatrix.go | 1254 ++++++++++++++++++++++++++++++ backend/quatrix/quatrix_test.go | 17 + backend/quatrix/upload_memory.go | 108 +++ bin/make_manual.py | 1 + docs/content/_index.md | 1 + docs/content/docs.md | 1 + docs/content/overview.md | 2 + docs/content/quatrix.md | 249 ++++++ docs/layouts/chrome/navbar.html | 1 + fstest/test_all/config.yaml | 3 + 13 files changed, 1821 insertions(+) create mode 100644 backend/quatrix/api/types.go create mode 100644 backend/quatrix/quatrix.go create mode 100644 backend/quatrix/quatrix_test.go create mode 100644 backend/quatrix/upload_memory.go create mode 100644 docs/content/quatrix.md diff --git a/README.md b/README.md index 55fb45a08..e278361ec 100644 --- a/README.md +++ b/README.md @@ -75,6 +75,7 @@ Rclone *("rsync for cloud storage")* is a command-line program to sync files and * put.io [:page_facing_up:](https://rclone.org/putio/) * QingStor [:page_facing_up:](https://rclone.org/qingstor/) * Qiniu Cloud Object Storage (Kodo) [:page_facing_up:](https://rclone.org/s3/#qiniu) + * Quatrix [:page_facing_up:](https://rclone.org/quatrix/) * Rackspace Cloud Files [:page_facing_up:](https://rclone.org/swift/) * RackCorp Object Storage [:page_facing_up:](https://rclone.org/s3/#RackCorp) * Scaleway [:page_facing_up:](https://rclone.org/s3/#scaleway) diff --git a/backend/all/all.go b/backend/all/all.go index 107975cd6..791a0b879 100644 --- a/backend/all/all.go +++ b/backend/all/all.go @@ -41,6 +41,7 @@ import ( _ "github.com/rclone/rclone/backend/protondrive" _ "github.com/rclone/rclone/backend/putio" _ "github.com/rclone/rclone/backend/qingstor" + _ "github.com/rclone/rclone/backend/quatrix" _ "github.com/rclone/rclone/backend/s3" _ "github.com/rclone/rclone/backend/seafile" _ "github.com/rclone/rclone/backend/sftp" diff --git a/backend/quatrix/api/types.go b/backend/quatrix/api/types.go new file mode 100644 index 000000000..3cacdc228 --- /dev/null +++ b/backend/quatrix/api/types.go @@ -0,0 +1,182 @@ +// Package api provides types used by the Quatrix API. +package api + +import ( + "strconv" + "time" +) + +// OverwriteOnCopyMode is a conflict resolve mode during copy. Files with conflicting names will be overwritten +const OverwriteOnCopyMode = "overwrite" + +// ProfileInfo is a profile info about quota +type ProfileInfo struct { + UserUsed int64 `json:"user_used"` + UserLimit int64 `json:"user_limit"` + AccUsed int64 `json:"acc_used"` + AccLimit int64 `json:"acc_limit"` +} + +// IDList is a general object that contains list of ids +type IDList struct { + IDs []string `json:"ids"` +} + +// DeleteParams is the request to delete object +type DeleteParams struct { + IDs []string `json:"ids"` + DeletePermanently bool `json:"delete_permanently"` +} + +// FileInfoParams is the request to get object's (file or directory) info +type FileInfoParams struct { + ParentID string `json:"parent_id,omitempty"` + Path string `json:"path"` +} + +// FileInfo is the response to get object's (file or directory) info +type FileInfo struct { + FileID string `json:"file_id"` + ParentID string `json:"parent_id"` + Src string `json:"src"` + Type string `json:"type"` +} + +// IsFile returns true if object is a file +// false otherwise +func (fi *FileInfo) IsFile() bool { + if fi == nil { + return false + } + + return fi.Type == "F" +} + +// IsDir returns true if object is a directory +// false otherwise +func (fi *FileInfo) IsDir() bool { + if fi == nil { + return false + } + + return fi.Type == "D" || fi.Type == "S" || fi.Type == "T" +} + +// CreateDirParams is the request to create a directory +type CreateDirParams struct { + Target string `json:"target,omitempty"` + Name string `json:"name"` + Resolve bool `json:"resolve"` +} + +// File represent metadata about object in Quatrix (file or directory) +type File struct { + ID string `json:"id"` + Created JSONTime `json:"created"` + Modified JSONTime `json:"modified"` + Name string `json:"name"` + ParentID string `json:"parent_id"` + Size int64 `json:"size"` + ModifiedMS JSONTime `json:"modified_ms"` + Type string `json:"type"` + Operations int `json:"operations"` + SubType string `json:"sub_type"` + Content []File `json:"content"` +} + +// IsFile returns true if object is a file +// false otherwise +func (f *File) IsFile() bool { + if f == nil { + return false + } + + return f.Type == "F" +} + +// IsDir returns true if object is a directory +// false otherwise +func (f *File) IsDir() bool { + if f == nil { + return false + } + + return f.Type == "D" || f.Type == "S" || f.Type == "T" +} + +// SetMTimeParams is the request to set modification time for object +type SetMTimeParams struct { + ID string `json:"id,omitempty"` + MTime JSONTime `json:"mtime"` +} + +// JSONTime provides methods to marshal/unmarshal time.Time as Unix time +type JSONTime time.Time + +// MarshalJSON returns time representation in Unix time +func (u JSONTime) MarshalJSON() ([]byte, error) { + return []byte(strconv.FormatFloat(float64(time.Time(u).UTC().UnixNano())/1e9, 'f', 6, 64)), nil +} + +// UnmarshalJSON sets time from Unix time representation +func (u *JSONTime) UnmarshalJSON(data []byte) error { + f, err := strconv.ParseFloat(string(data), 64) + if err != nil { + return err + } + + t := JSONTime(time.Unix(0, int64(f*1e9))) + *u = t + + return nil +} + +// String returns Unix time representation of time as string +func (u JSONTime) String() string { + return strconv.FormatInt(time.Time(u).UTC().Unix(), 10) +} + +// DownloadLinkResponse is the response to download-link request +type DownloadLinkResponse struct { + ID string `json:"id"` +} + +// UploadLinkParams is the request to get upload-link +type UploadLinkParams struct { + Name string `json:"name"` + ParentID string `json:"parent_id"` + Resolve bool `json:"resolve"` +} + +// UploadLinkResponse is the response to upload-link request +type UploadLinkResponse struct { + Name string `json:"name"` + FileID string `json:"file_id"` + ParentID string `json:"parent_id"` + UploadKey string `json:"upload_key"` +} + +// UploadFinalizeResponse is the response to finalize file method +type UploadFinalizeResponse struct { + FileID string `json:"id"` + ParentID string `json:"parent_id"` + Modified int64 `json:"modified"` + FileSize int64 `json:"size"` +} + +// FileModifyParams is the request to get modify file link +type FileModifyParams struct { + ID string `json:"id"` + Truncate int64 `json:"truncate"` +} + +// FileCopyMoveOneParams is the request to do server-side copy and move +// can be used for file or directory +type FileCopyMoveOneParams struct { + ID string `json:"file_id"` + Target string `json:"target_id"` + Name string `json:"name"` + MTime JSONTime `json:"mtime"` + Resolve bool `json:"resolve"` + ResolveMode string `json:"resolve_mode"` +} diff --git a/backend/quatrix/quatrix.go b/backend/quatrix/quatrix.go new file mode 100644 index 000000000..7a56c2722 --- /dev/null +++ b/backend/quatrix/quatrix.go @@ -0,0 +1,1254 @@ +// Package quatrix provides an interface to the Quatrix by Maytech +// object storage system. +package quatrix + +// FIXME Quatrix only supports file names of 255 characters or less. Names +// that will not be supported are those that contain non-printable +// ascii, / or \, names with trailing spaces, and the special names +// “.” and “..”. + +import ( + "context" + "errors" + "fmt" + "io" + "net/http" + "net/url" + "path" + "strconv" + "strings" + "time" + + "github.com/rclone/rclone/backend/quatrix/api" + "github.com/rclone/rclone/fs" + "github.com/rclone/rclone/fs/config" + "github.com/rclone/rclone/fs/config/configmap" + "github.com/rclone/rclone/fs/config/configstruct" + "github.com/rclone/rclone/fs/fserrors" + "github.com/rclone/rclone/fs/fshttp" + "github.com/rclone/rclone/fs/hash" + "github.com/rclone/rclone/lib/dircache" + "github.com/rclone/rclone/lib/encoder" + "github.com/rclone/rclone/lib/multipart" + "github.com/rclone/rclone/lib/pacer" + "github.com/rclone/rclone/lib/rest" +) + +const ( + minSleep = 10 * time.Millisecond + maxSleep = 2 * time.Second + decayConstant = 2 // bigger for slower decay, exponential + rootURL = "https://%s/api/1.0/" + uploadURL = "https://%s/upload/chunked/" + + unlimitedUserQuota = -1 +) + +func init() { + fs.Register(&fs.RegInfo{ + Name: "quatrix", + Description: "Quatrix by Maytech", + NewFs: NewFs, + Options: fs.Options{ + { + Name: "api_key", + Help: "API key for accessing Quatrix account", + Required: true, + Sensitive: true, + }, + { + Name: "host", + Help: "Host name of Quatrix account", + Required: true, + }, + { + Name: config.ConfigEncoding, + Help: config.ConfigEncodingHelp, + Advanced: true, + Default: encoder.Standard | + encoder.EncodeBackSlash | + encoder.EncodeInvalidUtf8, + }, + { + Name: "effective_upload_time", + Help: "Wanted upload time for one chunk", + Advanced: true, + Default: "4s", + }, + { + Name: "minimal_chunk_size", + Help: "The minimal size for one chunk", + Advanced: true, + Default: fs.SizeSuffix(10_000_000), + }, + { + Name: "maximal_summary_chunk_size", + Help: "The maximal summary for all chunks. It should not be less than 'transfers'*'minimal_chunk_size'", + Advanced: true, + Default: fs.SizeSuffix(100_000_000), + }, + { + Name: "hard_delete", + Help: "Delete files permanently rather than putting them into the trash.", + Advanced: true, + Default: false, + }, + }, + }) +} + +// Options defines the configuration for Quatrix backend +type Options struct { + APIKey string `config:"api_key"` + Host string `config:"host"` + Enc encoder.MultiEncoder `config:"encoding"` + EffectiveUploadTime fs.Duration `config:"effective_upload_time"` + MinimalChunkSize fs.SizeSuffix `config:"minimal_chunk_size"` + MaximalSummaryChunkSize fs.SizeSuffix `config:"maximal_summary_chunk_size"` + HardDelete bool `config:"hard_delete"` +} + +// Fs represents remote Quatrix fs +type Fs struct { + name string + root string + description string + features *fs.Features + opt Options + ci *fs.ConfigInfo + srv *rest.Client // the connection to the quatrix server + pacer *fs.Pacer // pacer for API calls + dirCache *dircache.DirCache + uploadMemoryManager *UploadMemoryManager +} + +// Object describes a quatrix object +type Object struct { + fs *Fs + remote string + size int64 + modTime time.Time + id string + hasMetaData bool + obType string +} + +// trimPath trims redundant slashes from quatrix 'url' +func trimPath(path string) (root string) { + root = strings.Trim(path, "/") + return +} + +// retryErrorCodes is a slice of error codes that we will retry +var retryErrorCodes = []int{ + 429, // Too Many Requests. + 500, // Internal Server Error + 502, // Bad Gateway + 503, // Service Unavailable + 504, // Gateway Timeout + 509, // Bandwidth Limit Exceeded +} + +// shouldRetry returns a boolean as to whether this resp and err +// deserve to be retried. It returns the err as a convenience +func shouldRetry(ctx context.Context, resp *http.Response, err error) (bool, error) { + if fserrors.ContextError(ctx, &err) { + return false, err + } + + return fserrors.ShouldRetry(err) || fserrors.ShouldRetryHTTP(resp, retryErrorCodes), err +} + +// NewFs constructs an Fs from the path, container:path +func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, error) { + // Parse config into Options struct + opt := new(Options) + + err := configstruct.Set(m, opt) + if err != nil { + return nil, err + } + + // http client + client := fshttp.NewClient(ctx) + + // since transport is a global variable that is initialized only once (due to sync.Once) + // we need to reset it to have correct transport per each client (with proper values extracted from rclone config) + client.Transport = fshttp.NewTransportCustom(ctx, nil) + + root = trimPath(root) + + ci := fs.GetConfig(ctx) + + f := &Fs{ + name: name, + description: "Quatrix FS for account " + opt.Host, + root: root, + opt: *opt, + ci: ci, + srv: rest.NewClient(client).SetRoot(fmt.Sprintf(rootURL, opt.Host)), + pacer: fs.NewPacer(ctx, pacer.NewDefault(pacer.MinSleep(minSleep), pacer.MaxSleep(maxSleep), pacer.DecayConstant(decayConstant))), + } + + f.features = (&fs.Features{ + CaseInsensitive: false, + CanHaveEmptyDirectories: true, + }).Fill(ctx, f) + + if f.opt.APIKey != "" { + f.srv.SetHeader("Authorization", "Bearer "+f.opt.APIKey) + } + + f.uploadMemoryManager = NewUploadMemoryManager(f.ci, &f.opt) + + // get quatrix root(home) id + rootID, found, err := f.fileID(ctx, "", "") + if err != nil { + return nil, err + } + + if !found { + return nil, errors.New("root not found") + } + + f.dirCache = dircache.New(root, rootID.FileID, f) + + err = f.dirCache.FindRoot(ctx, false) + if err != nil { + fileID, found, err := f.fileID(ctx, "", root) + if err != nil { + return nil, fmt.Errorf("find root %s: %w", root, err) + } + + if !found { + return f, nil + } + + if fileID.IsFile() { + root, _ = dircache.SplitPath(root) + f.dirCache = dircache.New(root, rootID.FileID, f) + + return f, fs.ErrorIsFile + } + } + + return f, nil +} + +// fileID gets id, parent and type of path in given parentID +func (f *Fs) fileID(ctx context.Context, parentID, path string) (result *api.FileInfo, found bool, err error) { + opts := rest.Opts{ + Method: "POST", + Path: "file/id", + IgnoreStatus: true, + } + + payload := api.FileInfoParams{ + Path: f.opt.Enc.FromStandardPath(path), + ParentID: parentID, + } + + result = &api.FileInfo{} + + err = f.pacer.Call(func() (bool, error) { + resp, err := f.srv.CallJSON(ctx, &opts, payload, result) + if resp != nil && resp.StatusCode == http.StatusNotFound { + return false, nil + } + return shouldRetry(ctx, resp, err) + }) + if err != nil { + return nil, false, fmt.Errorf("failed to get file id: %w", err) + } + + if result.FileID == "" { + return nil, false, nil + } + + return result, true, nil +} + +// FindLeaf finds a directory of name leaf in the folder with ID pathID +func (f *Fs) FindLeaf(ctx context.Context, pathID, leaf string) (folderID string, found bool, err error) { + result, found, err := f.fileID(ctx, pathID, leaf) + if err != nil { + return "", false, fmt.Errorf("find leaf: %w", err) + } + + if !found { + return "", false, nil + } + + if result.IsFile() { + return "", false, nil + } + + return result.FileID, true, nil +} + +// createDir creates directory in pathID with name leaf +// +// resolve - if true will resolve name conflict on server side, if false - will return error if object with this name exists +func (f *Fs) createDir(ctx context.Context, pathID, leaf string, resolve bool) (newDir *api.File, err error) { + opts := rest.Opts{ + Method: "POST", + Path: "file/makedir", + } + + payload := api.CreateDirParams{ + Name: f.opt.Enc.FromStandardName(leaf), + Target: pathID, + Resolve: resolve, + } + + newDir = &api.File{} + + err = f.pacer.Call(func() (bool, error) { + resp, err := f.srv.CallJSON(ctx, &opts, payload, newDir) + return shouldRetry(ctx, resp, err) + }) + if err != nil { + return nil, fmt.Errorf("failed to create directory: %w", err) + } + + return +} + +// CreateDir makes a directory with pathID as parent and name leaf +func (f *Fs) CreateDir(ctx context.Context, pathID, leaf string) (dirID string, err error) { + dir, err := f.createDir(ctx, pathID, leaf, false) + if err != nil { + return "", err + } + + return dir.ID, nil +} + +// Name of the remote (as passed into NewFs) +func (f *Fs) Name() string { + return f.name +} + +// Root of the remote (as passed into NewFs) +func (f *Fs) Root() string { + return f.root +} + +// String converts this Fs to a string +func (f *Fs) String() string { + return f.description +} + +// Precision return the precision of this Fs +func (f *Fs) Precision() time.Duration { + return time.Microsecond +} + +// Hashes returns the supported hash sets. +func (f *Fs) Hashes() hash.Set { + return 0 +} + +// Features returns the optional features of this Fs +func (f *Fs) Features() *fs.Features { + return f.features +} + +// List the objects and directories in dir into entries. The +// entries can be returned in any order but should be for a +// complete directory. +// +// dir should be "" to list the root, and should not have +// trailing slashes. +// +// This should return ErrDirNotFound if the directory isn't +// found. +func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err error) { + directoryID, err := f.dirCache.FindDir(ctx, dir, false) + if err != nil { + return nil, err + } + + folder, err := f.metadata(ctx, directoryID, true) + if err != nil { + return nil, err + } + + for _, file := range folder.Content { + remote := path.Join(dir, f.opt.Enc.ToStandardName(file.Name)) + if file.IsDir() { + f.dirCache.Put(remote, file.ID) + + d := fs.NewDir(remote, time.Time(file.Modified)).SetID(file.ID).SetItems(file.Size) + // FIXME more info from dir? + entries = append(entries, d) + } else { + o := &Object{ + fs: f, + remote: remote, + } + + err = o.setMetaData(&file) + if err != nil { + fs.Debugf(file, "failed to set object metadata: %s", err) + } + + entries = append(entries, o) + } + } + + return entries, nil +} + +// NewObject finds the Object at remote. If it can't be found +// it returns the error fs.ErrorObjectNotFound. +func (f *Fs) NewObject(ctx context.Context, remote string) (fs.Object, error) { + return f.newObjectWithInfo(ctx, remote, nil) +} + +// Creates from the parameters passed in a half finished Object which +// must have setMetaData called on it +// +// Returns the object, leaf, directoryID and error. +// +// Used to create new objects +func (f *Fs) createObject(ctx context.Context, remote string) (o *Object, leaf string, directoryID string, err error) { + // Create the directory for the object if it doesn't exist + leaf, directoryID, err = f.dirCache.FindPath(ctx, remote, true) + if err != nil { + return + } + // Temporary Object under construction + o = &Object{ + fs: f, + remote: remote, + } + return o, leaf, directoryID, nil +} + +// Put the object into the container +// +// Copy the reader in to the new object which is returned. +// +// The new object may have been created if an error is returned +func (f *Fs) Put(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) { + remote := src.Remote() + size := src.Size() + mtime := src.ModTime(ctx) + + o := &Object{ + fs: f, + remote: remote, + size: size, + modTime: mtime, + } + + return o, o.Update(ctx, in, src, options...) +} + +func (f *Fs) rootSlash() string { + if f.root == "" { + return f.root + } + return f.root + "/" +} + +func (f *Fs) newObjectWithInfo(ctx context.Context, remote string, info *api.File) (fs.Object, error) { + o := &Object{ + fs: f, + remote: remote, + } + var err error + if info != nil { + // Set info + err = o.setMetaData(info) + } else { + err = o.readMetaData(ctx) // reads info and meta, returning an error + } + if err != nil { + return nil, err + } + return o, nil +} + +// setMetaData sets the metadata from info +func (o *Object) setMetaData(info *api.File) (err error) { + if info.IsDir() { + fs.Debugf(o, "%q is %q", o.remote, info.Type) + return fs.ErrorIsDir + } + + if !info.IsFile() { + fs.Debugf(o, "%q is %q", o.remote, info.Type) + return fmt.Errorf("%q is %q: %w", o.remote, info.Type, fs.ErrorNotAFile) + } + + o.size = info.Size + o.modTime = time.Time(info.ModifiedMS) + o.id = info.ID + o.hasMetaData = true + o.obType = info.Type + + return nil +} + +func (o *Object) readMetaData(ctx context.Context) (err error) { + if o.hasMetaData { + return nil + } + + leaf, directoryID, err := o.fs.dirCache.FindPath(ctx, o.remote, false) + if err != nil { + if err == fs.ErrorDirNotFound { + return fs.ErrorObjectNotFound + } + return err + } + + file, found, err := o.fs.fileID(ctx, directoryID, leaf) + if err != nil { + return fmt.Errorf("read metadata: fileID: %w", err) + } + + if !found { + fs.Debugf(nil, "object not found: remote %s: directory %s: leaf %s", o.remote, directoryID, leaf) + return fs.ErrorObjectNotFound + } + + result, err := o.fs.metadata(ctx, file.FileID, false) + if err != nil { + return fmt.Errorf("get file metadata: %w", err) + } + + return o.setMetaData(result) +} + +// Mkdir creates the container if it doesn't exist +func (f *Fs) Mkdir(ctx context.Context, dir string) error { + _, err := f.dirCache.FindDir(ctx, dir, true) + return err +} + +// Rmdir deletes the root folder +// +// Returns an error if it isn't empty +func (f *Fs) Rmdir(ctx context.Context, dir string) error { + return f.purgeCheck(ctx, dir, true) +} + +// DirCacheFlush resets the directory cache - used in testing as an +// optional interface +func (f *Fs) DirCacheFlush() { + f.dirCache.ResetRoot() +} + +func (f *Fs) metadata(ctx context.Context, id string, withContent bool) (result *api.File, err error) { + parameters := url.Values{} + if !withContent { + parameters.Add("content", "0") + } + + opts := rest.Opts{ + Method: "GET", + Path: path.Join("file/metadata", id), + Parameters: parameters, + } + + result = &api.File{} + + var resp *http.Response + err = f.pacer.Call(func() (bool, error) { + resp, err = f.srv.CallJSON(ctx, &opts, nil, result) + return shouldRetry(ctx, resp, err) + }) + if err != nil { + if resp != nil && resp.StatusCode == http.StatusNotFound { + return nil, fs.ErrorObjectNotFound + } + + return nil, fmt.Errorf("failed to get file metadata: %w", err) + } + + return result, nil +} + +func (f *Fs) setMTime(ctx context.Context, id string, t time.Time) (result *api.File, err error) { + opts := rest.Opts{ + Method: "POST", + Path: "file/metadata", + } + + params := &api.SetMTimeParams{ + ID: id, + MTime: api.JSONTime(t), + } + + result = &api.File{} + + var resp *http.Response + err = f.pacer.Call(func() (bool, error) { + resp, err = f.srv.CallJSON(ctx, &opts, params, result) + return shouldRetry(ctx, resp, err) + }) + if err != nil { + if resp != nil && resp.StatusCode == http.StatusNotFound { + return nil, fs.ErrorObjectNotFound + } + + return nil, fmt.Errorf("failed to set file metadata: %w", err) + } + + return result, nil +} + +func (f *Fs) deleteObject(ctx context.Context, id string) error { + payload := &api.DeleteParams{ + IDs: []string{id}, + DeletePermanently: f.opt.HardDelete, + } + + result := &api.IDList{} + + opts := rest.Opts{ + Method: "POST", + Path: "file/delete", + } + + err := f.pacer.Call(func() (bool, error) { + resp, err := f.srv.CallJSON(ctx, &opts, payload, result) + return shouldRetry(ctx, resp, err) + }) + if err != nil { + return err + } + + for _, removedID := range result.IDs { + if removedID == id { + return nil + } + } + + return fmt.Errorf("file %s was not deleted successfully", id) +} + +func (f *Fs) purgeCheck(ctx context.Context, dir string, check bool) error { + root := path.Join(f.root, dir) + if root == "" { + return errors.New("can't purge root directory") + } + + rootID, err := f.dirCache.FindDir(ctx, dir, false) + if err != nil { + return err + } + + if check { + file, err := f.metadata(ctx, rootID, false) + if err != nil { + return err + } + + if file.IsFile() { + return fs.ErrorIsFile + } + + if file.Size != 0 { + return fs.ErrorDirectoryNotEmpty + } + } + + err = f.deleteObject(ctx, rootID) + if err != nil { + return err + } + + f.dirCache.FlushDir(dir) + + return nil +} + +// Purge deletes all the files in the directory +// +// Optional interface: Only implement this if you have a way of +// deleting all the files quicker than just running Remove() on the +// result of List() +func (f *Fs) Purge(ctx context.Context, dir string) error { + return f.purgeCheck(ctx, dir, false) +} + +// Copy src to this remote using server-side copy operations. +// +// This is stored with the remote path given. +// +// It returns the destination Object and a possible error. +// +// Will only be called if src.Fs().Name() == f.Name() +// +// If it isn't possible then return fs.ErrorCantCopy +func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object, error) { + srcObj, ok := src.(*Object) + if !ok { + fs.Debugf(src, "Can't copy - not same remote type") + return nil, fs.ErrorCantCopy + } + + if srcObj.fs == f { + srcPath := srcObj.rootPath() + dstPath := f.rootPath(remote) + if srcPath == dstPath { + return nil, fmt.Errorf("can't copy %q -> %q as they are same", srcPath, dstPath) + } + } + + err := srcObj.readMetaData(ctx) + if err != nil { + fs.Debugf(srcObj, "read metadata for %s: %s", srcObj.rootPath(), err) + return nil, err + } + + _, _, err = srcObj.fs.dirCache.FindPath(ctx, srcObj.remote, false) + if err != nil { + return nil, err + } + + dstObj, dstLeaf, directoryID, err := f.createObject(ctx, remote) + if err != nil { + fs.Debugf(srcObj, "create empty object for %s: %s", dstObj.rootPath(), err) + return nil, err + } + + opts := rest.Opts{ + Method: "POST", + Path: "file/copyone", + } + + params := &api.FileCopyMoveOneParams{ + ID: srcObj.id, + Target: directoryID, + Resolve: true, + MTime: api.JSONTime(srcObj.ModTime(ctx)), + Name: dstLeaf, + ResolveMode: api.OverwriteOnCopyMode, + } + + result := &api.File{} + + var resp *http.Response + + err = f.pacer.Call(func() (bool, error) { + resp, err = f.srv.CallJSON(ctx, &opts, params, result) + return shouldRetry(ctx, resp, err) + }) + if err != nil { + if resp != nil && resp.StatusCode == http.StatusNotFound { + return nil, fs.ErrorObjectNotFound + } + + return nil, fmt.Errorf("failed to copy: %w", err) + } + + err = dstObj.setMetaData(result) + if err != nil { + return nil, err + } + + return dstObj, nil +} + +// Move src to this remote using server-side move operations. +// +// This is stored with the remote path given. +// +// It returns the destination Object and a possible error. +// +// Will only be called if src.Fs().Name() == f.Name() +// +// If it isn't possible then return fs.ErrorCantMove +func (f *Fs) Move(ctx context.Context, src fs.Object, remote string) (fs.Object, error) { + srcObj, ok := src.(*Object) + if !ok { + fs.Debugf(src, "Can't move - not same remote type") + return nil, fs.ErrorCantMove + } + + _, _, err := srcObj.fs.dirCache.FindPath(ctx, srcObj.remote, false) + if err != nil { + return nil, err + } + + // Create temporary object + dstObj, dstLeaf, directoryID, err := f.createObject(ctx, remote) + if err != nil { + return nil, err + } + + opts := rest.Opts{ + Method: "POST", + Path: "file/moveone", + } + + params := &api.FileCopyMoveOneParams{ + ID: srcObj.id, + Target: directoryID, + Resolve: false, + MTime: api.JSONTime(srcObj.ModTime(ctx)), + Name: dstLeaf, + } + + var resp *http.Response + result := &api.File{} + + err = f.pacer.Call(func() (bool, error) { + resp, err = f.srv.CallJSON(ctx, &opts, params, result) + return shouldRetry(ctx, resp, err) + }) + if err != nil { + if resp != nil && resp.StatusCode == http.StatusNotFound { + return nil, fs.ErrorObjectNotFound + } + + return nil, fmt.Errorf("failed to move: %w", err) + } + + err = dstObj.setMetaData(result) + if err != nil { + return nil, err + } + + return dstObj, nil +} + +// DirMove moves src, srcRemote to this remote at dstRemote +// using server-side move operations. +// +// Will only be called if src.Fs().Name() == f.Name() +// +// If it isn't possible then return fs.ErrorCantDirMove +// +// If destination exists then return fs.ErrorDirExists +func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string) error { + srcFs, ok := src.(*Fs) + if !ok { + fs.Debugf(srcFs, "Can't move directory - not same remote type") + return fs.ErrorCantDirMove + } + + srcID, _, _, dstDirectoryID, dstLeaf, err := f.dirCache.DirMove(ctx, srcFs.dirCache, srcFs.root, srcRemote, f.root, dstRemote) + if err != nil { + return err + } + + srcInfo, err := f.metadata(ctx, srcID, false) + if err != nil { + return err + } + + opts := rest.Opts{ + Method: "POST", + Path: "file/moveone", + } + + params := &api.FileCopyMoveOneParams{ + ID: srcID, + Target: dstDirectoryID, + Resolve: false, + MTime: srcInfo.ModifiedMS, + Name: dstLeaf, + } + + var resp *http.Response + result := &api.File{} + + err = f.pacer.Call(func() (bool, error) { + resp, err = f.srv.CallJSON(ctx, &opts, params, result) + return shouldRetry(ctx, resp, err) + }) + if err != nil { + if resp != nil && resp.StatusCode == http.StatusNotFound { + return fs.ErrorObjectNotFound + } + + return fmt.Errorf("failed to move dir: %w", err) + } + + srcFs.dirCache.FlushDir(srcRemote) + + return nil +} + +// About gets quota information +func (f *Fs) About(ctx context.Context) (usage *fs.Usage, err error) { + opts := rest.Opts{ + Method: "GET", + Path: "profile/info", + } + var ( + user api.ProfileInfo + resp *http.Response + ) + + err = f.pacer.Call(func() (bool, error) { + resp, err = f.srv.CallJSON(ctx, &opts, nil, &user) + return shouldRetry(ctx, resp, err) + }) + if err != nil { + return nil, fmt.Errorf("failed to read profile info: %w", err) + } + + free := user.AccLimit - user.UserUsed + + if user.UserLimit > unlimitedUserQuota { + free = user.UserLimit - user.UserUsed + } + + usage = &fs.Usage{ + Used: fs.NewUsageValue(user.UserUsed), // bytes in use + Total: fs.NewUsageValue(user.AccLimit), // bytes total + Free: fs.NewUsageValue(free), // bytes free + } + + return usage, nil +} + +// Fs return the parent Fs +func (o *Object) Fs() fs.Info { + return o.fs +} + +// String returns object remote path +func (o *Object) String() string { + if o == nil { + return "" + } + return o.remote +} + +// Remote returns the remote path +func (o *Object) Remote() string { + return o.remote +} + +// rootPath returns a path for use in server given a remote +func (f *Fs) rootPath(remote string) string { + return f.rootSlash() + remote +} + +// rootPath returns a path for use in local functions +func (o *Object) rootPath() string { + return o.fs.rootPath(o.remote) +} + +// Size returns the size of an object in bytes +func (o *Object) Size() int64 { + err := o.readMetaData(context.TODO()) + if err != nil { + fs.Logf(o, "Failed to read metadata: %v", err) + return 0 + } + + return o.size +} + +// ModTime returns the modification time of the object +func (o *Object) ModTime(ctx context.Context) time.Time { + err := o.readMetaData(ctx) + if err != nil { + fs.Logf(o, "Failed to read metadata: %v", err) + return time.Now() + } + + return o.modTime +} + +// Storable returns a boolean showing whether this object storable +func (o *Object) Storable() bool { + return true +} + +// ID returns the ID of the Object if known, or "" if not +func (o *Object) ID() string { + return o.id +} + +// Hash returns the SHA-1 of an object. Not supported yet. +func (o *Object) Hash(ctx context.Context, ty hash.Type) (string, error) { + return "", nil +} + +// Remove an object +func (o *Object) Remove(ctx context.Context) error { + err := o.fs.deleteObject(ctx, o.id) + if err != nil { + return err + } + + if o.obType != "F" { + o.fs.dirCache.FlushDir(o.remote) + } + + return nil +} + +// Open an object for read +func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.ReadCloser, err error) { + if o.id == "" { + return nil, errors.New("can't download - no id") + } + + linkID, err := o.fs.downloadLink(ctx, o.id) + if err != nil { + return nil, err + } + + fs.FixRangeOption(options, o.size) + + opts := rest.Opts{ + Method: "GET", + Path: "/file/download/" + linkID, + Options: options, + } + + var resp *http.Response + + err = o.fs.pacer.Call(func() (bool, error) { + resp, err = o.fs.srv.Call(ctx, &opts) + return shouldRetry(ctx, resp, err) + }) + if err != nil { + return nil, err + } + return resp.Body, err +} + +func (f *Fs) downloadLink(ctx context.Context, id string) (linkID string, err error) { + linkParams := &api.IDList{ + IDs: []string{id}, + } + opts := rest.Opts{ + Method: "POST", + Path: "file/download-link", + } + + var resp *http.Response + link := &api.DownloadLinkResponse{} + + err = f.pacer.Call(func() (bool, error) { + resp, err = f.srv.CallJSON(ctx, &opts, linkParams, &link) + return shouldRetry(ctx, resp, err) + }) + if err != nil { + return "", err + } + return link.ID, nil +} + +// SetModTime sets the modification time of the local fs object +func (o *Object) SetModTime(ctx context.Context, t time.Time) error { + file, err := o.fs.setMTime(ctx, o.id, t) + if err != nil { + return fmt.Errorf("set mtime: %w", err) + } + + return o.setMetaData(file) +} + +// Update the object with the contents of the io.Reader, modTime and size +// +// The new object may have been created if an error is returned +func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (err error) { + size := src.Size() + modTime := src.ModTime(ctx) + remote := o.Remote() + + // Create the directory for the object if it doesn't exist + leaf, directoryID, err := o.fs.dirCache.FindPath(ctx, remote, true) + if err != nil { + return err + } + + uploadSession, err := o.uploadSession(ctx, directoryID, leaf) + if err != nil { + return fmt.Errorf("object update: %w", err) + } + + o.id = uploadSession.FileID + + defer func() { + if err == nil { + return + } + + deleteErr := o.fs.deleteObject(ctx, o.id) + if deleteErr != nil { + fs.Logf(o.remote, "remove: %s", deleteErr) + } + }() + + return o.dynamicUpload(ctx, size, modTime, in, uploadSession, options...) +} + +// dynamicUpload uploads object in chunks, which are being dynamically recalculated on each iteration +// depending on upload speed in order to make upload faster +func (o *Object) dynamicUpload(ctx context.Context, size int64, modTime time.Time, in io.Reader, + uploadSession *api.UploadLinkResponse, options ...fs.OpenOption) error { + var ( + speed float64 + localChunk int64 + ) + + defer o.fs.uploadMemoryManager.Return(o.id) + + for offset := int64(0); offset < size; offset += localChunk { + localChunk = o.fs.uploadMemoryManager.Consume(o.id, size-offset, speed) + + rw := multipart.NewRW() + + _, err := io.CopyN(rw, in, localChunk) + if err != nil { + return fmt.Errorf("read chunk with offset %d size %d: %w", offset, localChunk, err) + } + + start := time.Now() + + err = o.upload(ctx, uploadSession.UploadKey, rw, size, offset, localChunk, options...) + if err != nil { + return fmt.Errorf("upload chunk with offset %d size %d: %w", offset, localChunk, err) + } + + speed = float64(localChunk) / (float64(time.Since(start)) / 1e9) + } + + o.fs.uploadMemoryManager.Return(o.id) + + finalizeResult, err := o.finalize(ctx, uploadSession.UploadKey, modTime) + if err != nil { + return fmt.Errorf("upload %s finalize: %w", uploadSession.UploadKey, err) + } + + if size >= 0 && finalizeResult.FileSize != size { + return fmt.Errorf("expected size %d, got %d", size, finalizeResult.FileSize) + } + + o.size = size + o.modTime = modTime + + return nil +} + +func (f *Fs) uploadLink(ctx context.Context, parentID, name string) (upload *api.UploadLinkResponse, err error) { + opts := rest.Opts{ + Method: "POST", + Path: "upload/link", + } + + payload := api.UploadLinkParams{ + Name: name, + ParentID: parentID, + Resolve: false, + } + + err = f.pacer.Call(func() (bool, error) { + resp, err := f.srv.CallJSON(ctx, &opts, &payload, &upload) + return shouldRetry(ctx, resp, err) + }) + if err != nil { + return nil, fmt.Errorf("failed to get upload link: %w", err) + } + + return upload, nil +} + +func (f *Fs) modifyLink(ctx context.Context, fileID string) (upload *api.UploadLinkResponse, err error) { + opts := rest.Opts{ + Method: "POST", + Path: "file/modify", + } + + payload := api.FileModifyParams{ + ID: fileID, + Truncate: 0, + } + + err = f.pacer.Call(func() (bool, error) { + resp, err := f.srv.CallJSON(ctx, &opts, &payload, &upload) + return shouldRetry(ctx, resp, err) + }) + if err != nil { + return nil, fmt.Errorf("failed to get modify link: %w", err) + } + + return upload, nil +} + +func (o *Object) uploadSession(ctx context.Context, parentID, name string) (upload *api.UploadLinkResponse, err error) { + encName := o.fs.opt.Enc.FromStandardName(name) + fileID, found, err := o.fs.fileID(ctx, parentID, encName) + if err != nil { + return nil, fmt.Errorf("get file_id: %w", err) + } + + if found { + return o.fs.modifyLink(ctx, fileID.FileID) + } + + return o.fs.uploadLink(ctx, parentID, encName) +} + +func (o *Object) upload(ctx context.Context, uploadKey string, chunk io.Reader, fullSize int64, offset int64, chunkSize int64, options ...fs.OpenOption) (err error) { + opts := rest.Opts{ + Method: "POST", + RootURL: fmt.Sprintf(uploadURL, o.fs.opt.Host) + uploadKey, + Body: chunk, + ContentRange: fmt.Sprintf("bytes %d-%d/%d", offset, offset+chunkSize, fullSize), + Options: options, + } + + var fileID string + + err = o.fs.pacer.Call(func() (bool, error) { + resp, err := o.fs.srv.CallJSON(ctx, &opts, nil, &fileID) + return shouldRetry(ctx, resp, err) + }) + if err != nil { + return fmt.Errorf("failed to get upload chunk: %w", err) + } + + return nil +} + +func (o *Object) finalize(ctx context.Context, uploadKey string, mtime time.Time) (result *api.UploadFinalizeResponse, err error) { + queryParams := url.Values{} + queryParams.Add("mtime", strconv.FormatFloat(float64(mtime.UTC().UnixNano())/1e9, 'f', 6, 64)) + + opts := rest.Opts{ + Method: "GET", + Path: path.Join("upload/finalize", uploadKey), + Parameters: queryParams, + } + + result = &api.UploadFinalizeResponse{} + + err = o.fs.pacer.Call(func() (bool, error) { + resp, err := o.fs.srv.CallJSON(ctx, &opts, nil, result) + return shouldRetry(ctx, resp, err) + }) + if err != nil { + return nil, fmt.Errorf("failed to finalize: %w", err) + } + + return result, nil +} + +// Check the interfaces are satisfied +var ( + _ fs.Fs = (*Fs)(nil) + _ fs.Purger = (*Fs)(nil) + _ fs.Copier = (*Fs)(nil) + _ fs.Abouter = (*Fs)(nil) + _ fs.Mover = (*Fs)(nil) + _ fs.DirMover = (*Fs)(nil) + _ dircache.DirCacher = (*Fs)(nil) + _ fs.DirCacheFlusher = (*Fs)(nil) + _ fs.Object = (*Object)(nil) + _ fs.IDer = (*Object)(nil) +) diff --git a/backend/quatrix/quatrix_test.go b/backend/quatrix/quatrix_test.go new file mode 100644 index 000000000..c9759f188 --- /dev/null +++ b/backend/quatrix/quatrix_test.go @@ -0,0 +1,17 @@ +// Test Quatrix filesystem interface +package quatrix_test + +import ( + "testing" + + "github.com/rclone/rclone/backend/quatrix" + "github.com/rclone/rclone/fstest/fstests" +) + +// TestIntegration runs integration tests against the remote +func TestIntegration(t *testing.T) { + fstests.Run(t, &fstests.Opt{ + RemoteName: "TestQuatrix:", + NilObject: (*quatrix.Object)(nil), + }) +} diff --git a/backend/quatrix/upload_memory.go b/backend/quatrix/upload_memory.go new file mode 100644 index 000000000..615b78540 --- /dev/null +++ b/backend/quatrix/upload_memory.go @@ -0,0 +1,108 @@ +package quatrix + +import ( + "sync" + "time" + + "github.com/rclone/rclone/fs" +) + +// UploadMemoryManager dynamically calculates every chunk size for the transfer and increases or decreases it +// depending on the upload speed. This makes general upload time smaller, because transfers that are faster +// does not have to wait for the slower ones until they finish upload. +type UploadMemoryManager struct { + m sync.Mutex + useDynamicSize bool + shared int64 + reserved int64 + effectiveTime time.Duration + fileUsage map[string]int64 +} + +// NewUploadMemoryManager is a constructor for UploadMemoryManager +func NewUploadMemoryManager(ci *fs.ConfigInfo, opt *Options) *UploadMemoryManager { + useDynamicSize := true + + sharedMemory := int64(opt.MaximalSummaryChunkSize) - int64(opt.MinimalChunkSize)*int64(ci.Transfers) + if sharedMemory <= 0 { + sharedMemory = 0 + useDynamicSize = false + } + + return &UploadMemoryManager{ + useDynamicSize: useDynamicSize, + shared: sharedMemory, + reserved: int64(opt.MinimalChunkSize), + effectiveTime: time.Duration(opt.EffectiveUploadTime), + fileUsage: map[string]int64{}, + } +} + +// Consume -- decide amount of memory to consume +func (u *UploadMemoryManager) Consume(fileID string, neededMemory int64, speed float64) int64 { + if !u.useDynamicSize { + if neededMemory < u.reserved { + return neededMemory + } + + return u.reserved + } + + u.m.Lock() + defer u.m.Unlock() + + borrowed, found := u.fileUsage[fileID] + if found { + u.shared += borrowed + borrowed = 0 + } + + defer func() { u.fileUsage[fileID] = borrowed }() + + effectiveChunkSize := int64(speed * u.effectiveTime.Seconds()) + + if effectiveChunkSize < u.reserved { + effectiveChunkSize = u.reserved + } + + if neededMemory < effectiveChunkSize { + effectiveChunkSize = neededMemory + } + + if effectiveChunkSize <= u.reserved { + return effectiveChunkSize + } + + toBorrow := effectiveChunkSize - u.reserved + + if toBorrow <= u.shared { + u.shared -= toBorrow + borrowed = toBorrow + + return effectiveChunkSize + } + + borrowed = u.shared + u.shared = 0 + + return borrowed + u.reserved +} + +// Return returns consumed memory for the previous chunk upload to the memory pool +func (u *UploadMemoryManager) Return(fileID string) { + if !u.useDynamicSize { + return + } + + u.m.Lock() + defer u.m.Unlock() + + borrowed, found := u.fileUsage[fileID] + if !found { + return + } + + u.shared += borrowed + + delete(u.fileUsage, fileID) +} diff --git a/bin/make_manual.py b/bin/make_manual.py index fb26da094..faa8778e1 100755 --- a/bin/make_manual.py +++ b/bin/make_manual.py @@ -61,6 +61,7 @@ docs = [ "opendrive.md", "oracleobjectstorage.md", "qingstor.md", + "quatrix.md", "sia.md", "swift.md", "pcloud.md", diff --git a/docs/content/_index.md b/docs/content/_index.md index edea6dfb5..6fe858c88 100644 --- a/docs/content/_index.md +++ b/docs/content/_index.md @@ -160,6 +160,7 @@ WebDAV or S3, that work out of the box.) {{< provider name="put.io" home="https://put.io/" config="/putio/" >}} {{< provider name="QingStor" home="https://www.qingcloud.com/products/storage" config="/qingstor/" >}} {{< provider name="Qiniu Cloud Object Storage (Kodo)" home="https://www.qiniu.com/en/products/kodo" config="/s3/#qiniu" >}} +{{< provider name="Quatrix by Maytech" home="https://www.maytech.net/products/quatrix-business" config="/quatrix/" >}} {{< provider name="Rackspace Cloud Files" home="https://www.rackspace.com/cloud/files" config="/swift/" >}} {{< provider name="rsync.net" home="https://rsync.net/products/rclone.html" config="/sftp/#rsync-net" >}} {{< provider name="Scaleway" home="https://www.scaleway.com/object-storage/" config="/s3/#scaleway" >}} diff --git a/docs/content/docs.md b/docs/content/docs.md index 367c3da86..b33eedbf8 100644 --- a/docs/content/docs.md +++ b/docs/content/docs.md @@ -67,6 +67,7 @@ See the following for detailed instructions for * [premiumize.me](/premiumizeme/) * [put.io](/putio/) * [QingStor](/qingstor/) + * [Quatrix by Maytech](/quatrix/) * [Seafile](/seafile/) * [SFTP](/sftp/) * [Sia](/sia/) diff --git a/docs/content/overview.md b/docs/content/overview.md index 1ed89ba1c..a5a3ded81 100644 --- a/docs/content/overview.md +++ b/docs/content/overview.md @@ -48,6 +48,7 @@ Here is an overview of the major features of each cloud storage system. | premiumize.me | - | - | Yes | No | R | - | | put.io | CRC-32 | R/W | No | Yes | R | - | | QingStor | MD5 | - ⁹ | No | No | R/W | - | +| Quatrix by Maytech | - | R/W | No | No | - | - | | Seafile | - | - | No | No | - | - | | SFTP | MD5, SHA1 ² | R/W | Depends | No | - | - | | Sia | - | - | No | No | - | - | @@ -499,6 +500,7 @@ upon backend-specific capabilities. | premiumize.me | Yes | No | Yes | Yes | No | No | No | No | Yes | Yes | Yes | | put.io | Yes | No | Yes | Yes | Yes | No | Yes | No | No | Yes | Yes | | QingStor | No | Yes | No | No | Yes | Yes | No | No | No | No | No | +| Quatrix by Maytech | Yes | Yes | Yes | Yes | No | No | No | No | No | Yes | Yes | | Seafile | Yes | Yes | Yes | Yes | Yes | Yes | Yes | No | Yes | Yes | Yes | | SFTP | No | No | Yes | Yes | No | No | Yes | No | No | Yes | Yes | | Sia | No | No | No | No | No | No | Yes | No | No | No | Yes | diff --git a/docs/content/quatrix.md b/docs/content/quatrix.md new file mode 100644 index 000000000..66819c057 --- /dev/null +++ b/docs/content/quatrix.md @@ -0,0 +1,249 @@ +--- +title: "Quatrix" +description: "Rclone docs for Quatrix" +versionIntroduced: "v1.63.2" +--- + +# {{< icon "fas fa-shield-alt" >}} Quatrix + +Quatrix by Maytech is [Quatrix Secure Compliant File Sharing | Maytech](https://www.maytech.net/products/quatrix-business). + +Paths are specified as `remote:path` + +Paths may be as deep as required, e.g., `remote:directory/subdirectory`. + +The initial setup for Quatrix involves getting an API Key from Quatrix. You can get the API key in the user's profile at `https:///profile/api-keys` +or with the help of the API - https://docs.maytech.net/quatrix/quatrix-api/api-explorer#/API-Key/post_api_key_create. + +See complete Swagger documentation for Quatrix - https://docs.maytech.net/quatrix/quatrix-api/api-explorer + +## Configuration + +Here is an example of how to make a remote called `remote`. First run: + + rclone config + +This will guide you through an interactive setup process: + +``` +No remotes found, make a new one? +n) New remote +s) Set configuration password +q) Quit config +n/s/q> n +name> remote +Type of storage to configure. +Choose a number from below, or type in your own value +[snip] +XX / Quatrix by Maytech + \ "quatrix" +[snip] +Storage> quatrix +API key for accessing Quatrix account. +api_key> your_api_key +Host name of Quatrix account. +host> example.quatrix.it + +-------------------- +[remote] +api_key = your_api_key +host = example.quatrix.it +-------------------- +y) Yes this is OK +e) Edit this remote +d) Delete this remote +y/e/d> y +``` + +Once configured you can then use `rclone` like this, + +List directories in top level of your Quatrix + + rclone lsd remote: + +List all the files in your Quatrix + + rclone ls remote: + +To copy a local directory to an Quatrix directory called backup + + rclone copy /home/source remote:backup + +### API key validity + +API Key is created with no expiration date. It will be valid until you delete or deactivate it in your account. +After disabling, the API Key can be enabled back. If the API Key was deleted and a new key was created, you can +update it in rclone config. The same happens if the hostname was changed. + +``` +$ rclone config +Current remotes: + +Name Type +==== ==== +remote quatrix + +e) Edit existing remote +n) New remote +d) Delete remote +r) Rename remote +c) Copy remote +s) Set configuration password +q) Quit config +e/n/d/r/c/s/q> e +Choose a number from below, or type in an existing value + 1 > remote +remote> remote +-------------------- +[remote] +type = quatrix +host = some_host.quatrix.it +api_key = your_api_key +-------------------- +Edit remote +Option api_key. +API key for accessing Quatrix account +Enter a string value. Press Enter for the default (your_api_key) +api_key> +Option host. +Host name of Quatrix account +Enter a string value. Press Enter for the default (some_host.quatrix.it). + +-------------------- +[remote] +type = quatrix +host = some_host.quatrix.it +api_key = your_api_key +-------------------- +y) Yes this is OK +e) Edit this remote +d) Delete this remote +y/e/d> y +``` + +### Modified time and hashes + +Quatrix allows modification times to be set on objects accurate to 1 microsecond. +These will be used to detect whether objects need syncing or not. + +Quatrix does not support hashes, so you cannot use the `--checksum` flag. + +### Restricted filename characters + +File names in Quatrix are case sensitive and have limitations like the maximum length of a filename is 255, and the minimum length is 1. A file name cannot be equal to `.` or `..` nor contain `/` , `\` or non-printable ascii. + +### Transfers + +For files above 50 MiB rclone will use a chunked transfer. Rclone will upload up to `--transfers` chunks at the same time (shared among all multipart uploads). +Chunks are buffered in memory, and the minimal chunk size is 10_000_000 bytes by default, and it can be changed in the advanced configuration, so increasing `--transfers` will increase the memory use. +The chunk size has a maximum size limit, which is set to 100_000_000 bytes by default and can be changed in the advanced configuration. +The size of the uploaded chunk will dynamically change depending on the upload speed. +The total memory use equals the number of transfers multiplied by the minimal chunk size. +In case there's free memory allocated for the upload (which equals the difference of `maximal_summary_chunk_size` and `minimal_chunk_size` * `transfers`), +the chunk size may increase in case of high upload speed. As well as it can decrease in case of upload speed problems. +If no free memory is available, all chunks will equal `minimal_chunk_size`. + +### Deleting files + +Files you delete with rclone will end up in Trash and be stored there for 30 days. +Quatrix also provides an API to permanently delete files and an API to empty the Trash so that you can remove files permanently from your account. + +{{< rem autogenerated options start" - DO NOT EDIT - instead edit fs.RegInfo in backend/quatrix/quatrix.go then run make backenddocs" >}} +### Standard options + +Here are the Standard options specific to quatrix (Quatrix by Maytech). + +#### --quatrix-api-key + +API key for accessing Quatrix account + +Properties: + +- Config: api_key +- Env Var: RCLONE_QUATRIX_API_KEY +- Type: string +- Required: true + +#### --quatrix-host + +Host name of Quatrix account + +Properties: + +- Config: host +- Env Var: RCLONE_QUATRIX_HOST +- Type: string +- Required: true + +### Advanced options + +Here are the Advanced options specific to quatrix (Quatrix by Maytech). + +#### --quatrix-encoding + +The encoding for the backend. + +See the [encoding section in the overview](/overview/#encoding) for more info. + +Properties: + +- Config: encoding +- Env Var: RCLONE_QUATRIX_ENCODING +- Type: MultiEncoder +- Default: Slash,BackSlash,Del,Ctl,InvalidUtf8,Dot + +#### --quatrix-effective-upload-time + +Wanted upload time for one chunk + +Properties: + +- Config: effective_upload_time +- Env Var: RCLONE_QUATRIX_EFFECTIVE_UPLOAD_TIME +- Type: string +- Default: "4s" + +#### --quatrix-minimal-chunk-size + +The minimal size for one chunk + +Properties: + +- Config: minimal_chunk_size +- Env Var: RCLONE_QUATRIX_MINIMAL_CHUNK_SIZE +- Type: SizeSuffix +- Default: 9.537Mi + +#### --quatrix-maximal-summary-chunk-size + +The maximal summary for all chunks. It should not be less than 'transfers'*'minimal_chunk_size' + +Properties: + +- Config: maximal_summary_chunk_size +- Env Var: RCLONE_QUATRIX_MAXIMAL_SUMMARY_CHUNK_SIZE +- Type: SizeSuffix +- Default: 95.367Mi + +#### --quatrix-hard-delete + +Delete files permanently rather than putting them into the trash. + +Properties: + +- Config: hard_delete +- Env Var: RCLONE_QUATRIX_HARD_DELETE +- Type: bool +- Default: false + +{{< rem autogenerated options stop >}} + +## Storage usage + +The storage usage in Quatrix is restricted to the account during the purchase. You can restrict any user with a smaller storage limit. +The account limit is applied if the user has no custom storage limit. Once you've reached the limit, the upload of files will fail. +This can be fixed by freeing up the space or increasing the quota. + +## Server-side operations + +Quatrix supports server-side operations (copy and move). In case of conflict, files are overwritten during server-side operation. \ No newline at end of file diff --git a/docs/layouts/chrome/navbar.html b/docs/layouts/chrome/navbar.html index d98a957b3..21e62db00 100644 --- a/docs/layouts/chrome/navbar.html +++ b/docs/layouts/chrome/navbar.html @@ -90,6 +90,7 @@ PikPak premiumize.me put.io + Quatrix Seafile SFTP Sia diff --git a/fstest/test_all/config.yaml b/fstest/test_all/config.yaml index c201b6beb..4274d5125 100644 --- a/fstest/test_all/config.yaml +++ b/fstest/test_all/config.yaml @@ -434,3 +434,6 @@ backends: - TestIntegration/FsMkdir/FsEncoding/trailing_LF - TestIntegration/FsMkdir/FsEncoding/leading_HT - TestIntegration/FsMkdir/FsPutFiles/FsPutStream/0 + - backend: "quatrix" + remote: "TestQuatrix:" + fastlist: false