From 979bb07c86f9680466e9421da116aaf3af94c1da Mon Sep 17 00:00:00 2001 From: Nick Craig-Wood Date: Tue, 11 Aug 2020 15:09:00 +0100 Subject: [PATCH] filefabric: Implement the Enterprise File Fabric backend Missing features - M-Stream support - Oauth-like flow (soon being changed to oauth) --- README.md | 1 + backend/all/all.go | 1 + backend/filefabric/api/types.go | 391 +++++++ backend/filefabric/filefabric.go | 1347 +++++++++++++++++++++++++ backend/filefabric/filefabric_test.go | 17 + bin/make_manual.py | 1 + docs/content/_index.md | 1 + docs/content/docs.md | 1 + docs/content/filefabric.md | 260 +++++ docs/content/overview.md | 2 + docs/layouts/chrome/navbar.html | 1 + fstest/test_all/config.yaml | 6 + 12 files changed, 2029 insertions(+) create mode 100644 backend/filefabric/api/types.go create mode 100644 backend/filefabric/filefabric.go create mode 100644 backend/filefabric/filefabric_test.go create mode 100644 docs/content/filefabric.md diff --git a/README.md b/README.md index 8cd853701..75d7a94b0 100644 --- a/README.md +++ b/README.md @@ -30,6 +30,7 @@ Rclone *("rsync for cloud storage")* is a command line program to sync files and * DigitalOcean Spaces [:page_facing_up:](https://rclone.org/s3/#digitalocean-spaces) * Dreamhost [:page_facing_up:](https://rclone.org/s3/#dreamhost) * Dropbox [:page_facing_up:](https://rclone.org/dropbox/) + * Enterprise File Fabric [:page_facing_up:](https://rclone.org/filefabric/) * FTP [:page_facing_up:](https://rclone.org/ftp/) * GetSky [:page_facing_up:](https://rclone.org/jottacloud/) * Google Cloud Storage [:page_facing_up:](https://rclone.org/googlecloudstorage/) diff --git a/backend/all/all.go b/backend/all/all.go index 17fb6ddf2..3ee70f4b0 100644 --- a/backend/all/all.go +++ b/backend/all/all.go @@ -14,6 +14,7 @@ import ( _ "github.com/rclone/rclone/backend/drive" _ "github.com/rclone/rclone/backend/dropbox" _ "github.com/rclone/rclone/backend/fichier" + _ "github.com/rclone/rclone/backend/filefabric" _ "github.com/rclone/rclone/backend/ftp" _ "github.com/rclone/rclone/backend/googlecloudstorage" _ "github.com/rclone/rclone/backend/googlephotos" diff --git a/backend/filefabric/api/types.go b/backend/filefabric/api/types.go new file mode 100644 index 000000000..e3b80e6c0 --- /dev/null +++ b/backend/filefabric/api/types.go @@ -0,0 +1,391 @@ +// Package api has type definitions for filefabric +// +// Converted from the API responses with help from https://mholt.github.io/json-to-go/ +package api + +import ( + "bytes" + "fmt" + "reflect" + "strings" + "time" +) + +const ( + // TimeFormat for parameters (UTC) + timeFormatParameters = `2006-01-02 15:04:05` + // "2020-08-11 10:10:04" for JSON parsing + timeFormatJSON = `"` + timeFormatParameters + `"` +) + +// Time represents represents date and time information for the +// filefabric API +type Time time.Time + +// MarshalJSON turns a Time into JSON (in UTC) +func (t *Time) MarshalJSON() (out []byte, err error) { + timeString := (*time.Time)(t).UTC().Format(timeFormatJSON) + return []byte(timeString), nil +} + +var zeroTime = []byte(`"0000-00-00 00:00:00"`) + +// UnmarshalJSON turns JSON into a Time (in UTC) +func (t *Time) UnmarshalJSON(data []byte) error { + // Set a Zero time.Time if we receive a zero time input + if bytes.Equal(data, zeroTime) { + *t = Time(time.Time{}) + return nil + } + newT, err := time.Parse(timeFormatJSON, string(data)) + if err != nil { + return err + } + *t = Time(newT) + return nil +} + +// String turns a Time into a string in UTC suitable for the API +// parameters +func (t Time) String() string { + return time.Time(t).UTC().Format(timeFormatParameters) +} + +// Status return returned in all status responses +type Status struct { + Code string `json:"status"` + Message string `json:"statusmessage"` + TaskID string `json:"taskid"` + // Warning string `json:"warning"` // obsolete +} + +// Status statisfies the error interface +func (e *Status) Error() string { + return fmt.Sprintf("%s (%s)", e.Message, e.Code) +} + +// OK returns true if the status is all good +func (e *Status) OK() bool { + return e.Code == "ok" +} + +// GetCode returns the status code if any +func (e *Status) GetCode() string { + return e.Code +} + +// OKError defines an interface for items which can be OK or be an error +type OKError interface { + error + OK() bool + GetCode() string +} + +// Check Status satisfies the OKError interface +var _ OKError = (*Status)(nil) + +// EmptyResponse is response which just returns the error condition +type EmptyResponse struct { + Status +} + +// GetTokenByAuthTokenResponse is the response to getTokenByAuthToken +type GetTokenByAuthTokenResponse struct { + Status + Token string `json:"token"` + UserID string `json:"userid"` + AllowLoginRemember string `json:"allowloginremember"` + LastLogin Time `json:"lastlogin"` + AutoLoginCode string `json:"autologincode"` +} + +// ApplianceInfo is the response to getApplianceInfo +type ApplianceInfo struct { + Status + Sitetitle string `json:"sitetitle"` + OauthLoginSupport string `json:"oauthloginsupport"` + IsAppliance string `json:"isappliance"` + SoftwareVersion string `json:"softwareversion"` + SoftwareVersionLabel string `json:"softwareversionlabel"` +} + +// GetFolderContentsResponse is returned from getFolderContents +type GetFolderContentsResponse struct { + Status + Total int `json:"total,string"` + Items []Item `json:"filelist"` + Folder Item `json:"folder"` + From int `json:"from,string"` + //Count int `json:"count"` + Pid string `json:"pid"` + RefreshResult Status `json:"refreshresult"` + // Curfolder Item `json:"curfolder"` - sometimes returned as "ROOT"? + Parents []Item `json:"parents"` + CustomPermissions CustomPermissions `json:"custompermissions"` +} + +// ItemType determine whether it is a file or a folder +type ItemType uint8 + +// Types of things in Item +const ( + ItemTypeFile ItemType = 0 + ItemTypeFolder ItemType = 1 +) + +// Item ia a File or a Folder +type Item struct { + ID string `json:"fi_id"` + PID string `json:"fi_pid"` + // UID string `json:"fi_uid"` + Name string `json:"fi_name"` + // S3Name string `json:"fi_s3name"` + // Extension string `json:"fi_extension"` + // Description string `json:"fi_description"` + Type ItemType `json:"fi_type,string"` + // Created Time `json:"fi_created"` + Size int64 `json:"fi_size,string"` + ContentType string `json:"fi_contenttype"` + // Tags string `json:"fi_tags"` + // MainCode string `json:"fi_maincode"` + // Public int `json:"fi_public,string"` + // Provider string `json:"fi_provider"` + // ProviderFolder string `json:"fi_providerfolder"` // folder + // Encrypted int `json:"fi_encrypted,string"` + // StructType string `json:"fi_structtype"` + // Bname string `json:"fi_bname"` // folder + // OrgID string `json:"fi_orgid"` + // Favorite int `json:"fi_favorite,string"` + // IspartOf string `json:"fi_ispartof"` // folder + Modified Time `json:"fi_modified"` + // LastAccessed Time `json:"fi_lastaccessed"` + // Hits int64 `json:"fi_hits,string"` + // IP string `json:"fi_ip"` // folder + // BigDescription string `json:"fi_bigdescription"` + LocalTime Time `json:"fi_localtime"` + // OrgfolderID string `json:"fi_orgfolderid"` + // StorageIP string `json:"fi_storageip"` // folder + // RemoteTime Time `json:"fi_remotetime"` + // ProviderOptions string `json:"fi_provideroptions"` + // Access string `json:"fi_access"` + // Hidden string `json:"fi_hidden"` // folder + // VersionOf string `json:"fi_versionof"` + Trash bool `json:"trash"` + // Isbucket string `json:"isbucket"` // filelist + SubFolders int64 `json:"subfolders"` // folder +} + +// ItemFields is a | separated list of fields in Item +var ItemFields = mustFields(Item{}) + +// fields returns the JSON fields in use by opt as a | separated +// string. +func fields(opt interface{}) (pipeTags string, err error) { + var tags []string + def := reflect.ValueOf(opt) + defType := def.Type() + for i := 0; i < def.NumField(); i++ { + field := defType.Field(i) + tag, ok := field.Tag.Lookup("json") + if !ok { + continue + } + if comma := strings.IndexRune(tag, ','); comma >= 0 { + tag = tag[:comma] + } + if tag == "" { + continue + } + tags = append(tags, tag) + } + return strings.Join(tags, "|"), nil +} + +// mustFields returns the JSON fields in use by opt as a | separated +// string. It panics on failure. +func mustFields(opt interface{}) string { + tags, err := fields(opt) + if err != nil { + panic(err) + } + return tags +} + +// CustomPermissions is returned as part of GetFolderContentsResponse +type CustomPermissions struct { + Upload string `json:"upload"` + CreateSubFolder string `json:"createsubfolder"` + Rename string `json:"rename"` + Delete string `json:"delete"` + Move string `json:"move"` + ManagePermissions string `json:"managepermissions"` + ListOnly string `json:"listonly"` + VisibleInTrash string `json:"visibleintrash"` +} + +// DoCreateNewFolderResponse is response from foCreateNewFolder +type DoCreateNewFolderResponse struct { + Status + Item Item `json:"file"` +} + +// DoInitUploadResponse is response from doInitUpload +type DoInitUploadResponse struct { + Status + ProviderID string `json:"providerid"` + UploadCode string `json:"uploadcode"` + FileType string `json:"filetype"` + DirectUploadSupport string `json:"directuploadsupport"` + ResumeAllowed string `json:"resumeallowed"` +} + +// UploaderResponse is returned from /cgi-bin/uploader/uploader1.cgi +// +// Sometimes the response is returned as XML and sometimes as JSON +type UploaderResponse struct { + FileSize int64 `xml:"filesize" json:"filesize,string"` + MD5 string `xml:"md5" json:"md5"` + Success string `xml:"success" json:"success"` +} + +// UploadStatus is returned from getUploadStatus +type UploadStatus struct { + Status + UploadCode string `json:"uploadcode"` + Metafile string `json:"metafile"` + Percent int `json:"percent,string"` + Uploaded int64 `json:"uploaded,string"` + Size int64 `json:"size,string"` + Filename string `json:"filename"` + Nofile string `json:"nofile"` + Completed string `json:"completed"` + Completsuccess string `json:"completsuccess"` + Completerror string `json:"completerror"` +} + +// DoCompleteUploadResponse is the response to doCompleteUpload +type DoCompleteUploadResponse struct { + Status + UploadedSize int64 `json:"uploadedsize,string"` + StorageIP string `json:"storageip"` + UploadedName string `json:"uploadedname"` + // Versioned []interface{} `json:"versioned"` + // VersionedID int `json:"versionedid"` + // Comment interface{} `json:"comment"` + File Item `json:"file"` + // UsSize string `json:"us_size"` + // PaSize string `json:"pa_size"` + // SpaceInfo SpaceInfo `json:"spaceinfo"` +} + +// Providers is returned as part of UploadResponse +type Providers struct { + Max string `json:"max"` + Used string `json:"used"` + ID string `json:"id"` + Private string `json:"private"` + Limit string `json:"limit"` + Percent int `json:"percent"` +} + +// Total is returned as part of UploadResponse +type Total struct { + Max string `json:"max"` + Used string `json:"used"` + ID string `json:"id"` + Priused string `json:"priused"` + Primax string `json:"primax"` + Limit string `json:"limit"` + Percent int `json:"percent"` + Pripercent int `json:"pripercent"` +} + +// UploadResponse is returned as part of SpaceInfo +type UploadResponse struct { + Providers []Providers `json:"providers"` + Total Total `json:"total"` +} + +// SpaceInfo is returned as part of DoCompleteUploadResponse +type SpaceInfo struct { + Response UploadResponse `json:"response"` + Status string `json:"status"` +} + +// DeleteResponse is returned from doDeleteFile +type DeleteResponse struct { + Status + Deleted []string `json:"deleted"` + Errors []interface{} `json:"errors"` + ID string `json:"fi_id"` + BackgroundTask int `json:"backgroundtask"` + UsSize string `json:"us_size"` + PaSize string `json:"pa_size"` + //SpaceInfo SpaceInfo `json:"spaceinfo"` +} + +// FileResponse is returned from doRenameFile +type FileResponse struct { + Status + Item Item `json:"file"` + Exists string `json:"exists"` +} + +// MoveFilesResponse is returned from doMoveFiles +type MoveFilesResponse struct { + Status + Filesleft string `json:"filesleft"` + Addedtobackground string `json:"addedtobackground"` + Moved string `json:"moved"` + Item Item `json:"file"` + IDs []string `json:"fi_ids"` + Length int `json:"length"` + DirID string `json:"dir_id"` + MovedObjects []Item `json:"movedobjects"` + // FolderTasks []interface{} `json:"foldertasks"` +} + +// TasksResponse is the response to getUserBackgroundTasks +type TasksResponse struct { + Status + Tasks []Task `json:"tasks"` + Total string `json:"total"` +} + +// BtData is part of TasksResponse +type BtData struct { + Callback string `json:"callback"` +} + +// Task describes a task returned in TasksResponse +type Task struct { + BtID string `json:"bt_id"` + UsID string `json:"us_id"` + BtType string `json:"bt_type"` + BtData BtData `json:"bt_data"` + BtStatustext string `json:"bt_statustext"` + BtStatusdata string `json:"bt_statusdata"` + BtMessage string `json:"bt_message"` + BtProcent string `json:"bt_procent"` + BtAdded string `json:"bt_added"` + BtStatus string `json:"bt_status"` + BtCompleted string `json:"bt_completed"` + BtTitle string `json:"bt_title"` + BtCredentials string `json:"bt_credentials"` + BtHidden string `json:"bt_hidden"` + BtAutoremove string `json:"bt_autoremove"` + BtDevsite string `json:"bt_devsite"` + BtPriority string `json:"bt_priority"` + BtReport string `json:"bt_report"` + BtSitemarker string `json:"bt_sitemarker"` + BtExecuteafter string `json:"bt_executeafter"` + BtCompletestatus string `json:"bt_completestatus"` + BtSubtype string `json:"bt_subtype"` + BtCanceled string `json:"bt_canceled"` + Callback string `json:"callback"` + CanBeCanceled bool `json:"canbecanceled"` + CanBeRestarted bool `json:"canberestarted"` + Type string `json:"type"` + Status string `json:"status"` + Settings string `json:"settings"` +} diff --git a/backend/filefabric/filefabric.go b/backend/filefabric/filefabric.go new file mode 100644 index 000000000..877b6d07c --- /dev/null +++ b/backend/filefabric/filefabric.go @@ -0,0 +1,1347 @@ +// Package filefabric provides an interface to Storage Made Easy's +// Enterprise File Fabric storage system. +package filefabric + +/* +Docs: https://product-demo.smestorage.com/?p=apidoc + +Missing features: +- M-Stream support +- Oauth-like flow (soon being changed to oauth) + +// TestFileFabric +maxFileLength = 14094 +*/ + +import ( + "bytes" + "context" + "encoding/base64" + "fmt" + "io" + "io/ioutil" + "net/http" + "net/url" + "path" + "strings" + "sync" + "sync/atomic" + "time" + + "github.com/rclone/rclone/lib/atexit" + "github.com/rclone/rclone/lib/encoder" + "github.com/rclone/rclone/lib/random" + + "github.com/pkg/errors" + "github.com/rclone/rclone/backend/filefabric/api" + "github.com/rclone/rclone/fs" + "github.com/rclone/rclone/fs/config" + "github.com/rclone/rclone/fs/config/configmap" + "github.com/rclone/rclone/fs/config/configstruct" + "github.com/rclone/rclone/fs/fserrors" + "github.com/rclone/rclone/fs/fshttp" + "github.com/rclone/rclone/fs/hash" + "github.com/rclone/rclone/fs/log" + "github.com/rclone/rclone/lib/dircache" + "github.com/rclone/rclone/lib/pacer" + "github.com/rclone/rclone/lib/rest" +) + +const ( + minSleep = 20 * time.Millisecond + maxSleep = 10 * time.Second + decayConstant = 2 // bigger for slower decay, exponential + listChunks = 1000 // chunk size to read directory listings + tokenLifeTime = 55 * time.Minute // 1 hour minus a bit of leeway + defaultRootID = "" // default root ID + emptyMimeType = "application/vnd.rclone.empty.file" +) + +// Register with Fs +func init() { + fs.Register(&fs.RegInfo{ + Name: "filefabric", + Description: "Enterprise File Fabric", + NewFs: NewFs, + Options: []fs.Option{{ + Name: "url", + Help: "URL of the Enterprise File Fabric to connect to", + Required: true, + Examples: []fs.OptionExample{{ + Value: "https://storagemadeeasy.com", + Help: "Storage Made Easy US", + }, { + Value: "https://eu.storagemadeeasy.com", + Help: "Storage Made Easy EU", + }, { + Value: "https://yourfabric.smestorage.com", + Help: "Connect to your Enterprise File Fabric", + }}, + }, { + Name: "root_folder_id", + Help: `ID of the root folder +Leave blank normally. + +Fill in to make rclone start with directory of a given ID. +`, + }, { + Name: "permanent_token", + Help: `Permanent Authentication Token + +A Permanent Authentication Token can be created in the Enterprise File +Fabric, on the users Dashboard under Security, there is an entry +you'll see called "My Authentication Tokens". Click the Manage button +to create one. + +These tokens are normally valid for several years. + +For more info see: https://docs.storagemadeeasy.com/organisationcloud/api-tokens +`, + }, { + Name: "token", + Help: `Session Token + +This is a session token which rclone caches in the config file. It is +usually valid for 1 hour. + +Don't set this value - rclone will set it automatically. +`, + Advanced: true, + }, { + Name: "token_expiry", + Help: `Token expiry time + +Don't set this value - rclone will set it automatically. +`, + Advanced: true, + }, { + Name: "version", + Help: `Version read from the file fabric + +Don't set this value - rclone will set it automatically. +`, + Advanced: true, + }, { + Name: config.ConfigEncoding, + Help: config.ConfigEncodingHelp, + Advanced: true, + Default: (encoder.Display | + encoder.EncodeInvalidUtf8), + }}, + }) +} + +// Options defines the configuration for this backend +type Options struct { + URL string `config:"url"` + RootFolderID string `config:"root_folder_id"` + PermanentToken string `config:"permanent_token"` + Token string `config:"token"` + TokenExpiry string `config:"token_expiry"` + Version string `config:"version"` + Enc encoder.MultiEncoder `config:"encoding"` +} + +// Fs represents a remote filefabric +type Fs struct { + name string // name of this remote + root string // the path we are working on + opt Options // parsed options + features *fs.Features // optional features + m configmap.Mapper // to save config + srv *rest.Client // the connection to the one drive server + dirCache *dircache.DirCache // Map of directory path to directory id + pacer *fs.Pacer // pacer for API calls + tokenMu sync.Mutex // hold when reading the token + token string // current access token + tokenExpiry time.Time // time the current token expires + tokenExpired int32 // read and written with atomic + canCopyWithName bool // set if detected that can use fi_name in copy + precision time.Duration // precision reported +} + +// Object describes a filefabric object +// +// Will definitely have info but maybe not meta +type Object struct { + fs *Fs // what this object is part of + remote string // The remote path + hasMetaData bool // whether info below has been set + size int64 // size of the object + modTime time.Time // modification time of the object + id string // ID of the object + contentType string // ContentType of object +} + +// ------------------------------------------------------------ + +// Name of the remote (as passed into NewFs) +func (f *Fs) Name() string { + return f.name +} + +// Root of the remote (as passed into NewFs) +func (f *Fs) Root() string { + return f.root +} + +// String converts this Fs to a string +func (f *Fs) String() string { + return fmt.Sprintf("filefabric root '%s'", f.root) +} + +// Features returns the optional features of this Fs +func (f *Fs) Features() *fs.Features { + return f.features +} + +// parsePath parses a filefabric 'url' +func parsePath(path string) (root string) { + root = strings.Trim(path, "/") + return +} + +// retryErrorCodes is a slice of error codes that we will retry +var retryErrorCodes = []int{ + 429, // Too Many Requests. + 500, // Internal Server Error + 502, // Bad Gateway + 503, // Service Unavailable + 504, // Gateway Timeout + 509, // Bandwidth Limit Exceeded +} + +// Retry any of these +var retryStatusCodes = []struct { + code string + sleep time.Duration +}{ + { + // Can not create folder now. We are not able to complete the + // requested operation with such name. We are processing + // delete in that folder. Please try again later or use + // another name. (error_background) + code: "error_background", + sleep: 6 * time.Second, + }, +} + +// shouldRetry returns a boolean as to whether this resp and err +// deserve to be retried. It returns the err as a convenience +func (f *Fs) shouldRetry(resp *http.Response, err error, status api.OKError) (bool, error) { + if err != nil { + return fserrors.ShouldRetry(err) || fserrors.ShouldRetryHTTP(resp, retryErrorCodes), err + } + if status != nil && !status.OK() { + err = status // return the error from the RPC + code := status.GetCode() + if code == "login_token_expired" { + atomic.AddInt32(&f.tokenExpired, 1) + } else { + for _, retryCode := range retryStatusCodes { + if code == retryCode.code { + if retryCode.sleep > 0 { + // make this thread only sleep extra time + fs.Debugf(f, "Sleeping for %v to wait for %q error to clear", retryCode.sleep, retryCode.code) + time.Sleep(retryCode.sleep) + } + return true, err + } + } + } + } + return false, err +} + +// readMetaDataForPath reads the metadata from the path +func (f *Fs) readMetaDataForPath(ctx context.Context, rootID string, path string) (info *api.Item, err error) { + var resp api.FileResponse + _, err = f.rpc(ctx, "checkPathExists", params{ + "path": f.opt.Enc.FromStandardPath(path), + "pid": rootID, + }, &resp, nil) + if err != nil { + return nil, errors.Wrap(err, "failed to check path exists") + } + if resp.Exists != "y" { + return nil, fs.ErrorObjectNotFound + } + return &resp.Item, nil + + /* + // defer fs.Trace(f, "path=%q", path)("info=%+v, err=%v", &info, &err) + leaf, directoryID, err := f.dirCache.FindPath(ctx, path, false) + if err != nil { + if err == fs.ErrorDirNotFound { + return nil, fs.ErrorObjectNotFound + } + return nil, err + } + + found, err := f.listAll(ctx, directoryID, false, true, func(item *api.Item) bool { + if item.Name == leaf { + info = item + return true + } + return false + }) + if err != nil { + return nil, err + } + if !found { + return nil, fs.ErrorObjectNotFound + } + return info, nil + */ +} + +// Get the appliance info so we can set Version +func (f *Fs) getApplianceInfo(ctx context.Context) error { + var applianceInfo api.ApplianceInfo + _, err := f.rpc(ctx, "getApplianceInfo", params{ + "token": "*", + }, &applianceInfo, nil) + if err != nil { + return errors.Wrap(err, "failed to read appliance version") + } + f.opt.Version = applianceInfo.SoftwareVersionLabel + f.m.Set("version", f.opt.Version) + return nil +} + +// Gets the token or gets a new one if necessary +func (f *Fs) getToken(ctx context.Context) (token string, err error) { + f.tokenMu.Lock() + var refreshed = false + defer func() { + if refreshed { + atomic.StoreInt32(&f.tokenExpired, 0) + } + f.tokenMu.Unlock() + }() + + expired := atomic.LoadInt32(&f.tokenExpired) != 0 + if expired { + fs.Debugf(f, "Token invalid - refreshing") + } + if f.token == "" { + fs.Debugf(f, "Empty token - refreshing") + expired = true + } + now := time.Now() + if f.tokenExpiry.IsZero() || now.After(f.tokenExpiry) { + fs.Debugf(f, "Token expired - refreshing") + expired = true + } + if !expired { + return f.token, nil + } + + var info api.GetTokenByAuthTokenResponse + _, err = f.rpc(ctx, "getTokenByAuthToken", params{ + "token": "*", + "authtoken": f.opt.PermanentToken, + }, &info, nil) + if err != nil { + return "", errors.Wrap(err, "failed to get session token") + } + refreshed = true + now = now.Add(tokenLifeTime) + f.token = info.Token + f.tokenExpiry = now + f.m.Set("token", f.token) + f.m.Set("token_expiry", now.Format(time.RFC3339)) + + // Read appliance info when we update the token + err = f.getApplianceInfo(ctx) + if err != nil { + return "", err + } + f.setCapabilities() + + return f.token, nil +} + +// params for rpc +type params map[string]interface{} + +// rpc calls the rpc.php method of the SME file fabric +// +// This is an entry point to all the method calls +// +// If result is nil then resp.Body will need closing +func (f *Fs) rpc(ctx context.Context, function string, p params, result api.OKError, options []fs.OpenOption) (resp *http.Response, err error) { + defer log.Trace(f, "%s(%+v) options=%+v", function, p, options)("result=%+v, err=%v", &result, &err) + + // Get the token from params if present otherwise call getToken + var token string + if tokenI, ok := p["token"]; !ok { + token, err = f.getToken(ctx) + if err != nil { + return resp, err + } + } else { + token = tokenI.(string) + } + var data = url.Values{ + "function": {function}, + "token": {token}, + "apiformat": {"json"}, + } + for k, v := range p { + data.Set(k, fmt.Sprint(v)) + } + opts := rest.Opts{ + Method: "POST", + Path: "/api/rpc.php", + ContentType: "application/x-www-form-urlencoded", + Options: options, + } + err = f.pacer.Call(func() (bool, error) { + // Refresh the body each retry + opts.Body = strings.NewReader(data.Encode()) + resp, err = f.srv.CallJSON(ctx, &opts, nil, result) + return f.shouldRetry(resp, err, result) + }) + if err != nil { + return resp, err + } + return resp, nil +} + +// NewFs constructs an Fs from the path, container:path +func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, error) { + // Parse config into Options struct + opt := new(Options) + err := configstruct.Set(m, opt) + if err != nil { + return nil, err + } + + opt.URL = strings.TrimSuffix(opt.URL, "/") + if opt.URL == "" { + return nil, errors.New("url must be set") + } + + root = parsePath(root) + + client := fshttp.NewClient(fs.Config) + + f := &Fs{ + name: name, + root: root, + opt: *opt, + m: m, + srv: rest.NewClient(client).SetRoot(opt.URL), + pacer: fs.NewPacer(pacer.NewDefault(pacer.MinSleep(minSleep), pacer.MaxSleep(maxSleep), pacer.DecayConstant(decayConstant))), + token: opt.Token, + } + f.features = (&fs.Features{ + CaseInsensitive: true, + CanHaveEmptyDirectories: true, + }).Fill(ctx, f) + if f.opt.Version == "" { + err = f.getApplianceInfo(ctx) + if err != nil { + return nil, err + } + } + f.setCapabilities() + + if opt.TokenExpiry != "" { + tokenExpiry, err := time.Parse(time.RFC3339, opt.TokenExpiry) + if err != nil { + fs.Errorf(nil, "Failed to parse token_expiry option: %v", err) + } else { + f.tokenExpiry = tokenExpiry + } + } + + if opt.RootFolderID == "" { + opt.RootFolderID = defaultRootID + } + + f.dirCache = dircache.New(f.root, opt.RootFolderID, f) + + // Find out whether the root is a file or a directory or doesn't exist + var errReturn error + if f.root != "" { + info, err := f.readMetaDataForPath(ctx, f.opt.RootFolderID, f.root) + if err == nil && info != nil { + if info.Type == api.ItemTypeFile { + // Root is a file + // Point the root to the parent directory + f.root, _ = dircache.SplitPath(root) + f.dirCache = dircache.New(f.root, opt.RootFolderID, f) + errReturn = fs.ErrorIsFile + // Cache the ID of the parent of the file as the root ID + f.dirCache.Put(f.root, info.PID) + } else if info.Type == api.ItemTypeFolder { + // Root is a dir - cache its ID + f.dirCache.Put(f.root, info.ID) + } + } else { + // Root is not found so a directory + } + } + return f, errReturn +} + +// set the capabilities of this version of software +func (f *Fs) setCapabilities() { + version := f.opt.Version + if version == "" { + version = "0000.00" + } + if version >= "2006.02" { + f.precision = time.Second + f.canCopyWithName = true + } else { + // times can be altered this much on renames + f.precision = 1 * time.Hour + f.canCopyWithName = false + } +} + +// Return an Object from a path +// +// If it can't be found it returns the error fs.ErrorObjectNotFound. +func (f *Fs) newObjectWithInfo(ctx context.Context, remote string, info *api.Item) (fs.Object, error) { + o := &Object{ + fs: f, + remote: remote, + } + var err error + if info != nil { + // Set info + err = o.setMetaData(info) + } else { + err = o.readMetaData(ctx) // reads info and meta, returning an error + } + if err != nil { + return nil, err + } + return o, nil +} + +// NewObject finds the Object at remote. If it can't be found +// it returns the error fs.ErrorObjectNotFound. +func (f *Fs) NewObject(ctx context.Context, remote string) (fs.Object, error) { + return f.newObjectWithInfo(ctx, remote, nil) +} + +// FindLeaf finds a directory of name leaf in the folder with ID pathID +func (f *Fs) FindLeaf(ctx context.Context, pathID, leaf string) (pathIDOut string, found bool, err error) { + // Find the leaf in pathID + found, err = f.listAll(ctx, pathID, true, false, func(item *api.Item) bool { + if item.Name == leaf { + pathIDOut = item.ID + return true + } + return false + }) + return pathIDOut, found, err +} + +// CreateDir makes a directory with pathID as parent and name leaf +func (f *Fs) CreateDir(ctx context.Context, pathID, leaf string) (newID string, err error) { + //fs.Debugf(f, "CreateDir(%q, %q)\n", pathID, leaf) + var info api.DoCreateNewFolderResponse + _, err = f.rpc(ctx, "doCreateNewFolder", params{ + "fi_pid": pathID, + "fi_name": f.opt.Enc.FromStandardName(leaf), + }, &info, nil) + if err != nil { + return "", errors.Wrap(err, "failed to create directory") + } + // fmt.Printf("...Id %q\n", *info.Id) + return info.Item.ID, nil +} + +// list the objects into the function supplied +// +// If directories is set it only sends directories +// User function to process a File item from listAll +// +// Should return true to finish processing +type listAllFn func(*api.Item) bool + +// Lists the directory required calling the user function on each item found +// +// If the user fn ever returns true then it early exits with found = true +func (f *Fs) listAll(ctx context.Context, dirID string, directoriesOnly bool, filesOnly bool, fn listAllFn) (found bool, err error) { + var ( + p = params{ + "fi_pid": dirID, + "count": listChunks, + "subfolders": "y", + // Cut down the things that are returned + "options": "filelist|" + api.ItemFields, + } + n = 0 + ) +OUTER: + for { + var info api.GetFolderContentsResponse + _, err = f.rpc(ctx, "getFolderContents", p, &info, nil) + if err != nil { + return false, errors.Wrap(err, "failed to list directory") + } + for i := range info.Items { + item := &info.Items[i] + if item.Type == api.ItemTypeFolder { + if filesOnly { + continue + } + } else if item.Type == api.ItemTypeFile { + if directoriesOnly { + continue + } + } else { + fs.Debugf(f, "Ignoring %q - unknown type %q", item.Name, item.Type) + continue + } + if item.Trash { + continue + } + item.Name = f.opt.Enc.ToStandardName(item.Name) + if fn(item) { + found = true + break OUTER + } + } + // if didn't get any items then exit + if len(info.Items) == 0 { + break + } + n += len(info.Items) + if n >= info.Total { + break + } + p["from"] = n + } + + return found, nil +} + +// List the objects and directories in dir into entries. The +// entries can be returned in any order but should be for a +// complete directory. +// +// dir should be "" to list the root, and should not have +// trailing slashes. +// +// This should return ErrDirNotFound if the directory isn't +// found. +func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err error) { + directoryID, err := f.dirCache.FindDir(ctx, dir, false) + if err != nil { + return nil, err + } + var iErr error + _, err = f.listAll(ctx, directoryID, false, false, func(info *api.Item) bool { + remote := path.Join(dir, info.Name) + if info.Type == api.ItemTypeFolder { + // cache the directory ID for later lookups + f.dirCache.Put(remote, info.ID) + d := fs.NewDir(remote, time.Time(info.Modified)).SetID(info.ID).SetItems(info.SubFolders) + entries = append(entries, d) + } else if info.Type == api.ItemTypeFile { + o, err := f.newObjectWithInfo(ctx, remote, info) + if err != nil { + iErr = err + return true + } + entries = append(entries, o) + } + return false + }) + if err != nil { + return nil, err + } + if iErr != nil { + return nil, iErr + } + return entries, nil +} + +// Creates from the parameters passed in a half finished Object which +// must have setMetaData called on it +// +// Returns the object, leaf, directoryID and error +// +// Used to create new objects +func (f *Fs) createObject(ctx context.Context, remote string, modTime time.Time, size int64) (o *Object, leaf string, directoryID string, err error) { + // Create the directory for the object if it doesn't exist + leaf, directoryID, err = f.dirCache.FindPath(ctx, remote, true) + if err != nil { + return + } + // Temporary Object under construction + o = &Object{ + fs: f, + remote: remote, + } + return o, leaf, directoryID, nil +} + +// Put the object +// +// Copy the reader in to the new object which is returned +// +// The new object may have been created if an error is returned +func (f *Fs) Put(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) { + remote := src.Remote() + size := src.Size() + modTime := src.ModTime(ctx) + + o, _, _, err := f.createObject(ctx, remote, modTime, size) + if err != nil { + return nil, err + } + return o, o.Update(ctx, in, src, options...) +} + +// Mkdir creates the container if it doesn't exist +func (f *Fs) Mkdir(ctx context.Context, dir string) error { + _, err := f.dirCache.FindDir(ctx, dir, true) + return err +} + +// deleteObject removes an object by ID +func (f *Fs) deleteObject(ctx context.Context, id string) (err error) { + var info api.DeleteResponse + _, err = f.rpc(ctx, "doDeleteFile", params{ + "fi_id": id, + "completedeletion": "n", + }, &info, nil) + if err != nil { + return errors.Wrap(err, "failed to delete file") + } + return nil +} + +// purgeCheck removes the root directory, if check is set then it +// refuses to do so if it has anything in +func (f *Fs) purgeCheck(ctx context.Context, dir string, check bool) error { + root := path.Join(f.root, dir) + if root == "" { + return errors.New("can't purge root directory") + } + dc := f.dirCache + rootID, err := dc.FindDir(ctx, dir, false) + if err != nil { + return err + } + + if check { + found, err := f.listAll(ctx, rootID, false, false, func(item *api.Item) bool { + fs.Debugf(dir, "Rmdir: contains file: %q", item.Name) + return true + }) + if err != nil { + return err + } + if found { + return fs.ErrorDirectoryNotEmpty + } + } + + var info api.EmptyResponse + _, err = f.rpc(ctx, "doDeleteFolder", params{ + "fi_id": rootID, + }, &info, nil) + f.dirCache.FlushDir(dir) + if err != nil { + return errors.Wrap(err, "failed to remove directory") + } + return nil +} + +// Rmdir deletes the root folder +// +// Returns an error if it isn't empty +func (f *Fs) Rmdir(ctx context.Context, dir string) error { + //fs.Debugf(f, "CreateDir(%q, %q)\n", pathID, leaf) + return f.purgeCheck(ctx, dir, true) +} + +// Precision return the precision of this Fs +func (f *Fs) Precision() time.Duration { + return f.precision +} + +// Copy src to this remote using server side copy operations. +// +// This is stored with the remote path given +// +// It returns the destination Object and a possible error +// +// Will only be called if src.Fs().Name() == f.Name() +// +// If it isn't possible then return fs.ErrorCantCopy +func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object, error) { + srcObj, ok := src.(*Object) + if !ok { + fs.Debugf(src, "Can't copy - not same remote type") + return nil, fs.ErrorCantCopy + } + err := srcObj.readMetaData(ctx) + if err != nil { + return nil, err + } + + // Create temporary object + dstObj, leaf, directoryID, err := f.createObject(ctx, remote, srcObj.modTime, srcObj.size) + if err != nil { + return nil, err + } + + if !f.canCopyWithName && leaf != path.Base(srcObj.remote) { + fs.Debugf(src, "Can't copy - can't change the name of files") + return nil, fs.ErrorCantCopy + } + + // Copy the object + var info api.FileResponse + p := params{ + "fi_id": srcObj.id, + "fi_pid": directoryID, + "force": "y", + "options": "allownoextension", // without this the filefabric adds extensions to files without + } + if f.canCopyWithName { + p["fi_name"] = f.opt.Enc.FromStandardName(leaf) + } + _, err = f.rpc(ctx, "doCopyFile", p, &info, nil) + if err != nil { + return nil, errors.Wrap(err, "failed to copy file") + } + err = dstObj.setMetaData(&info.Item) + if err != nil { + return nil, err + } + return dstObj, nil +} + +// Purge deletes all the files and the container +// +// Optional interface: Only implement this if you have a way of +// deleting all the files quicker than just running Remove() on the +// result of List() +func (f *Fs) Purge(ctx context.Context, dir string) error { + return f.purgeCheck(ctx, dir, false) +} + +// Wait for the the background task to complete if necessary +func (f *Fs) waitForBackgroundTask(ctx context.Context, taskID string) (err error) { + if taskID == "" || taskID == "0" { + // No task to wait for + return nil + } + start := time.Now() + sleepTime := time.Second + for { + var info api.TasksResponse + _, err = f.rpc(ctx, "getUserBackgroundTasks", params{ + "taskid": taskID, + }, &info, nil) + if err != nil { + return errors.Wrapf(err, "failed to wait for task %s to complete", taskID) + } + if len(info.Tasks) == 0 { + // task has finished + break + } + if len(info.Tasks) > 1 { + fs.Errorf(f, "Unexpected number of tasks returned %d", len(info.Tasks)) + } + task := info.Tasks[0] + if task.BtStatus == "c" { + // task completed + break + } + dt := time.Since(start) + fs.Debugf(f, "Waiting for task ID %s: %s: to completed for %v - waited %v already", task.BtID, task.BtTitle, sleepTime, dt) + time.Sleep(sleepTime) + } + return nil +} + +// Rename the leaf of a file or directory in a directory +func (f *Fs) renameLeaf(ctx context.Context, isDir bool, id string, newLeaf string) (item *api.Item, err error) { + var info api.FileResponse + method := "doRenameFile" + if isDir { + method = "doRenameFolder" + } + _, err = f.rpc(ctx, method, params{ + "fi_id": id, + "fi_name": newLeaf, + }, &info, nil) + if err != nil { + return nil, errors.Wrap(err, "failed to rename leaf") + } + err = f.waitForBackgroundTask(ctx, info.Status.TaskID) + if err != nil { + return nil, err + } + return &info.Item, nil +} + +// move a file or folder +// +// This is complicated by the fact that there is an API to move files +// between directories and a separate one to rename them. We try to +// call the minimum number of API calls. +func (f *Fs) move(ctx context.Context, isDir bool, id, oldLeaf, newLeaf, oldDirectoryID, newDirectoryID string) (item *api.Item, err error) { + newLeaf = f.opt.Enc.FromStandardName(newLeaf) + oldLeaf = f.opt.Enc.FromStandardName(oldLeaf) + doRenameLeaf := oldLeaf != newLeaf + doMove := oldDirectoryID != newDirectoryID + + // Now rename the leaf to a temporary name if we are moving to + // another directory to make sure we don't overwrite something + // in the destination directory by accident + if doRenameLeaf && doMove { + tmpLeaf := newLeaf + "." + random.String(8) + item, err = f.renameLeaf(ctx, isDir, id, tmpLeaf) + if err != nil { + return nil, err + } + } + + // Move the object to a new directory (with the existing name) + // if required + if doMove { + var info api.MoveFilesResponse + method := "doMoveFiles" + if isDir { + method = "doMoveFolders" + } + _, err = f.rpc(ctx, method, params{ + "fi_ids": id, + "dir_id": newDirectoryID, + }, &info, nil) + if err != nil { + return nil, errors.Wrap(err, "failed to move file to new directory") + } + item = &info.Item + err = f.waitForBackgroundTask(ctx, info.Status.TaskID) + if err != nil { + return nil, err + } + } + + // Rename the leaf to its final name if required + if doRenameLeaf { + item, err = f.renameLeaf(ctx, isDir, id, newLeaf) + if err != nil { + return nil, err + } + } + + return item, nil +} + +// Move src to this remote using server side move operations. +// +// This is stored with the remote path given +// +// It returns the destination Object and a possible error +// +// Will only be called if src.Fs().Name() == f.Name() +// +// If it isn't possible then return fs.ErrorCantMove +func (f *Fs) Move(ctx context.Context, src fs.Object, remote string) (fs.Object, error) { + srcObj, ok := src.(*Object) + if !ok { + fs.Debugf(src, "Can't move - not same remote type") + return nil, fs.ErrorCantMove + } + + // find the source directoryID + srcLeaf, srcDirectoryID, err := srcObj.fs.dirCache.FindPath(ctx, srcObj.remote, false) + if err != nil { + return nil, err + } + + // Create temporary object + dstObj, dstLeaf, dstDirectoryID, err := f.createObject(ctx, remote, srcObj.modTime, srcObj.size) + if err != nil { + return nil, err + } + + // Do the move + item, err := f.move(ctx, false, srcObj.id, srcLeaf, dstLeaf, srcDirectoryID, dstDirectoryID) + if err != nil { + return nil, err + } + + // Set the metadata from what was returned or read it fresh + if item == nil { + err = dstObj.readMetaData(ctx) + if err != nil { + return nil, err + } + } else { + err = dstObj.setMetaData(item) + if err != nil { + return nil, err + } + } + return dstObj, nil +} + +// DirMove moves src, srcRemote to this remote at dstRemote +// using server side move operations. +// +// Will only be called if src.Fs().Name() == f.Name() +// +// If it isn't possible then return fs.ErrorCantDirMove +// +// If destination exists then return fs.ErrorDirExists +func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string) error { + srcFs, ok := src.(*Fs) + if !ok { + fs.Debugf(srcFs, "Can't move directory - not same remote type") + return fs.ErrorCantDirMove + } + + srcID, srcDirectoryID, srcLeaf, dstDirectoryID, dstLeaf, err := f.dirCache.DirMove(ctx, srcFs.dirCache, srcFs.root, srcRemote, f.root, dstRemote) + if err != nil { + return err + } + + // Do the move + _, err = f.move(ctx, true, srcID, srcLeaf, dstLeaf, srcDirectoryID, dstDirectoryID) + if err != nil { + return err + } + srcFs.dirCache.FlushDir(srcRemote) + return nil +} + +// CleanUp empties the trash +func (f *Fs) CleanUp(ctx context.Context) (err error) { + var info api.EmptyResponse + _, err = f.rpc(ctx, "emptyTrashInBackground", params{}, &info, nil) + if err != nil { + return errors.Wrap(err, "failed to empty trash") + } + return nil +} + +// DirCacheFlush resets the directory cache - used in testing as an +// optional interface +func (f *Fs) DirCacheFlush() { + f.dirCache.ResetRoot() +} + +// Hashes returns the supported hash sets. +func (f *Fs) Hashes() hash.Set { + return hash.Set(hash.None) +} + +// ------------------------------------------------------------ + +// Fs returns the parent Fs +func (o *Object) Fs() fs.Info { + return o.fs +} + +// Return a string version +func (o *Object) String() string { + if o == nil { + return "" + } + return o.remote +} + +// Remote returns the remote path +func (o *Object) Remote() string { + return o.remote +} + +// Hash of the object in the requested format as a lowercase hex string +func (o *Object) Hash(ctx context.Context, t hash.Type) (string, error) { + return "", hash.ErrUnsupported +} + +// Size returns the size of an object in bytes +func (o *Object) Size() int64 { + err := o.readMetaData(context.TODO()) + if err != nil { + fs.Logf(o, "Failed to read metadata: %v", err) + return 0 + } + if o.contentType == emptyMimeType { + return 0 + } + return o.size +} + +// setMetaData sets the metadata from info +func (o *Object) setMetaData(info *api.Item) (err error) { + if info.Type != api.ItemTypeFile { + return errors.Wrapf(fs.ErrorNotAFile, "%q is %q", o.remote, info.Type) + } + o.hasMetaData = true + o.size = info.Size + o.modTime = time.Time(info.Modified) + if !time.Time(info.LocalTime).IsZero() { + o.modTime = time.Time(info.LocalTime) + } + o.id = info.ID + o.contentType = info.ContentType + return nil +} + +// readMetaData gets the metadata if it hasn't already been fetched +// +// it also sets the info +func (o *Object) readMetaData(ctx context.Context) (err error) { + if o.hasMetaData { + return nil + } + rootID, err := o.fs.dirCache.RootID(ctx, false) + if err != nil { + if err == fs.ErrorDirNotFound { + err = fs.ErrorObjectNotFound + } + return err + } + info, err := o.fs.readMetaDataForPath(ctx, rootID, o.remote) + if err != nil { + if apiErr, ok := err.(*api.Status); ok { + if apiErr.Code == "not_found" || apiErr.Code == "trashed" { + return fs.ErrorObjectNotFound + } + } + return err + } + return o.setMetaData(info) +} + +// ModTime returns the modification time of the object +// +// +// It attempts to read the objects mtime and if that isn't present the +// LastModified returned in the http headers +func (o *Object) ModTime(ctx context.Context) time.Time { + err := o.readMetaData(ctx) + if err != nil { + fs.Logf(o, "Failed to read metadata: %v", err) + return time.Now() + } + return o.modTime +} + +// modifyFile updates file metadata +// +// keyValues should be key, value pairs +func (o *Object) modifyFile(ctx context.Context, keyValues [][2]string) error { + var info api.FileResponse + var data strings.Builder + for _, keyValue := range keyValues { + data.WriteString(keyValue[0]) + data.WriteRune('=') + data.WriteString(keyValue[1]) + data.WriteRune('\n') + } + _, err := o.fs.rpc(ctx, "doModifyFile", params{ + "fi_id": o.id, + "data": data.String(), + }, &info, nil) + if err != nil { + return errors.Wrap(err, "failed to update metadata") + } + return o.setMetaData(&info.Item) +} + +// SetModTime sets the modification time of the local fs object +func (o *Object) SetModTime(ctx context.Context, modTime time.Time) error { + return o.modifyFile(ctx, [][2]string{ + {"fi_localtime", api.Time(modTime).String()}, + }) +} + +// Storable returns a boolean showing whether this object storable +func (o *Object) Storable() bool { + return true +} + +// Open an object for read +func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.ReadCloser, err error) { + if o.id == "" { + return nil, errors.New("can't download - no id") + } + if o.contentType == emptyMimeType { + return ioutil.NopCloser(bytes.NewReader([]byte{})), nil + } + fs.FixRangeOption(options, o.size) + resp, err := o.fs.rpc(ctx, "getFile", params{ + "fi_id": o.id, + }, nil, options) + if err != nil { + return nil, err + } + return resp.Body, nil +} + +// Update the object with the contents of the io.Reader, modTime and size +// +// If existing is set then it updates the object rather than creating a new one +// +// The new object may have been created if an error is returned +func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (err error) { + modTime := src.ModTime(ctx) + remote := o.remote + size := src.Size() + + // Can't upload 0 length files - these upload as a single + // space. + // if size == 0 { + // return fs.ErrorCantUploadEmptyFiles + // } + + // Create the directory for the object if it doesn't exist + leaf, directoryID, err := o.fs.dirCache.FindPath(ctx, remote, true) + if err != nil { + return err + } + + // Initialise the upload + var upload api.DoInitUploadResponse + timestamp := api.Time(modTime).String() + encodedLeaf := o.fs.opt.Enc.FromStandardName(leaf) + base64EncodedLeaf := base64.StdEncoding.EncodeToString([]byte(encodedLeaf)) + contentType := fs.MimeType(ctx, src) + if size == 0 { + contentType = emptyMimeType + } + p := params{ + "fi_name": encodedLeaf, + "fi_pid": directoryID, + "fi_filename": encodedLeaf, + "fi_localtime": timestamp, + "fi_modified": timestamp, + "fi_contenttype": contentType, + "responsetype": "json", // make the upload.cgi return JSON + "directuploadsupport": "n", // FIXME should we support this? + // "chunkifbig": "n", // FIXME multipart? + } + // Set the size if known + if size >= 0 { + p["fi_size"] = size + } + _, err = o.fs.rpc(ctx, "doInitUpload", p, &upload, nil) + if err != nil { + return errors.Wrap(err, "failed to initialize upload") + } + + // Cancel the upload if aborted or it fails + finalized := false + defer atexit.OnError(&err, func() { + if finalized { + return + } + fs.Debugf(o, "Cancelling upload %s", upload.UploadCode) + var cancel api.EmptyResponse + _, fErr := o.fs.rpc(ctx, "doAbortUpload", params{ + "uploadcode": upload.UploadCode, + }, &cancel, nil) + if fErr != nil { + fs.Errorf(o, "failed to cancel upload: %v", fErr) + } + })() + + // Post the file with the upload code + var uploader api.UploaderResponse + opts := rest.Opts{ + //Method: "POST", + Method: "PUT", + Path: "/cgi-bin/uploader/uploader1.cgi/" + base64EncodedLeaf + "?" + upload.UploadCode, + Body: in, + ContentType: contentType, + // MultipartParams: url.Values{}, + // MultipartContentName: "file", + // MultipartFileName: "datafile", + } + // Set the size if known + if size >= 0 { + var contentLength = size + opts.ContentLength = &contentLength // NB CallJSON scribbles on this which is naughty + } + err = o.fs.pacer.CallNoRetry(func() (bool, error) { + resp, err := o.fs.srv.CallJSON(ctx, &opts, nil, &uploader) + return o.fs.shouldRetry(resp, err, nil) + }) + if err != nil { + return errors.Wrap(err, "failed to upload") + } + if uploader.Success != "y" { + return errors.Errorf("upload failed") + } + if size > 0 && uploader.FileSize != size { + return errors.Errorf("upload failed: size mismatch: want %d got %d", size, uploader.FileSize) + } + + // Now finalize the file + var finalize api.DoCompleteUploadResponse + p = params{ + "uploadcode": upload.UploadCode, + "remotetime": timestamp, + "fi_size": uploader.FileSize, + } + _, err = o.fs.rpc(ctx, "doCompleteUpload", p, &finalize, nil) + if err != nil { + return errors.Wrap(err, "failed to finalize upload") + } + finalized = true + + err = o.setMetaData(&finalize.File) + if err != nil { + return err + } + + // Make sure content type is correct + if o.contentType != contentType { + fs.Debugf(o, "Correcting mime type from %q to %q", o.contentType, contentType) + return o.modifyFile(ctx, [][2]string{ + {"fi_contenttype", contentType}, + }) + } + + return nil +} + +// Remove an object +func (o *Object) Remove(ctx context.Context) error { + return o.fs.deleteObject(ctx, o.id) +} + +// ID returns the ID of the Object if known, or "" if not +func (o *Object) ID() string { + return o.id +} + +// MimeType returns the content type of the Object if +// known, or "" if not +func (o *Object) MimeType(ctx context.Context) string { + return o.contentType +} + +// Check the interfaces are satisfied +var ( + _ fs.Fs = (*Fs)(nil) + _ fs.Purger = (*Fs)(nil) + _ fs.Copier = (*Fs)(nil) + _ fs.Mover = (*Fs)(nil) + _ fs.DirMover = (*Fs)(nil) + _ fs.DirCacheFlusher = (*Fs)(nil) + _ fs.CleanUpper = (*Fs)(nil) + _ fs.Object = (*Object)(nil) + _ fs.IDer = (*Object)(nil) + _ fs.MimeTyper = (*Object)(nil) +) diff --git a/backend/filefabric/filefabric_test.go b/backend/filefabric/filefabric_test.go new file mode 100644 index 000000000..554639212 --- /dev/null +++ b/backend/filefabric/filefabric_test.go @@ -0,0 +1,17 @@ +// Test filefabric filesystem interface +package filefabric_test + +import ( + "testing" + + "github.com/rclone/rclone/backend/filefabric" + "github.com/rclone/rclone/fstest/fstests" +) + +// TestIntegration runs integration tests against the remote +func TestIntegration(t *testing.T) { + fstests.Run(t, &fstests.Opt{ + RemoteName: "TestFileFabric:", + NilObject: (*filefabric.Object)(nil), + }) +} diff --git a/bin/make_manual.py b/bin/make_manual.py index 725d99b73..210893afd 100755 --- a/bin/make_manual.py +++ b/bin/make_manual.py @@ -36,6 +36,7 @@ docs = [ "sharefile.md", "crypt.md", "dropbox.md", + "filefabric.md", "ftp.md", "googlecloudstorage.md", "drive.md", diff --git a/docs/content/_index.md b/docs/content/_index.md index 2ed8383cb..bc9042b66 100644 --- a/docs/content/_index.md +++ b/docs/content/_index.md @@ -115,6 +115,7 @@ WebDAV or S3, that work out of the box.) {{< provider name="DigitalOcean Spaces" home="https://www.digitalocean.com/products/object-storage/" config="/s3/#digitalocean-spaces" >}} {{< provider name="Dreamhost" home="https://www.dreamhost.com/cloud/storage/" config="/s3/#dreamhost" >}} {{< provider name="Dropbox" home="https://www.dropbox.com/" config="/dropbox/" >}} +{{< provider name="Enterprise File Fabric" home="https://storagemadeeasy.com/about/" config="/filefabric/" >}} {{< provider name="FTP" home="https://en.wikipedia.org/wiki/File_Transfer_Protocol" config="/ftp/" >}} {{< provider name="Google Cloud Storage" home="https://cloud.google.com/storage/" config="/googlecloudstorage/" >}} {{< provider name="Google Drive" home="https://www.google.com/drive/" config="/drive/" >}} diff --git a/docs/content/docs.md b/docs/content/docs.md index 898794668..9671686f9 100644 --- a/docs/content/docs.md +++ b/docs/content/docs.md @@ -30,6 +30,7 @@ See the following for detailed instructions for * [Crypt](/crypt/) - to encrypt other remotes * [DigitalOcean Spaces](/s3/#digitalocean-spaces) * [Dropbox](/dropbox/) + * [Enterprise File Fabric](/filefabric/) * [FTP](/ftp/) * [Google Cloud Storage](/googlecloudstorage/) * [Google Drive](/drive/) diff --git a/docs/content/filefabric.md b/docs/content/filefabric.md new file mode 100644 index 000000000..50c4cfafd --- /dev/null +++ b/docs/content/filefabric.md @@ -0,0 +1,260 @@ +--- +title: "Enterprise File Fabric" +description: "Rclone docs for the Enterprise File Fabric backend" +--- + +{{< icon "fa fa-cloud" >}} Enterprise File Fabric +----------------------------------------- + +This backend supports [Storage Made Easy's Enterprise File +Fabric™](https://storagemadeeasy.com/about/) which provides a software +solution to integrate and unify File and Object Storage accessible +through a global file system. + +The initial setup for the Enterprise File Fabric backend involves +getting a token from the the Enterprise File Fabric which you need to +do in your browser. `rclone config` walks you through it. + +Here is an example of how to make a remote called `remote`. First run: + + rclone config + +This will guide you through an interactive setup process: + +``` +No remotes found - make a new one +n) New remote +s) Set configuration password +q) Quit config +n/s/q> n +name> remote +Type of storage to configure. +Enter a string value. Press Enter for the default (""). +Choose a number from below, or type in your own value +[snip] +XX / Enterprise File Fabric + \ "filefabric" +[snip] +Storage> filefabric +** See help for filefabric backend at: https://rclone.org/filefabric/ ** + +URL of the Enterprise File Fabric to connect to +Enter a string value. Press Enter for the default (""). +Choose a number from below, or type in your own value + 1 / Storage Made Easy US + \ "https://storagemadeeasy.com" + 2 / Storage Made Easy EU + \ "https://eu.storagemadeeasy.com" + 3 / Connect to your Enterprise File Fabric + \ "https://yourfabric.smestorage.com" +url> https://yourfabric.smestorage.com/ +ID of the root folder +Leave blank normally. + +Fill in to make rclone start with directory of a given ID. + +Enter a string value. Press Enter for the default (""). +root_folder_id> +Permanent Authentication Token + +A Permanent Authentication Token can be created in the Enterprise File +Fabric, on the users Dashboard under Security, there is an entry +you'll see called "My Authentication Tokens". Click the Manage button +to create one. + +These tokens are normally valid for several years. + +For more info see: https://docs.storagemadeeasy.com/organisationcloud/api-tokens + +Enter a string value. Press Enter for the default (""). +permanent_token> xxxxxxxxxxxxxxx-xxxxxxxxxxxxxxxx +Edit advanced config? (y/n) +y) Yes +n) No (default) +y/n> n +Remote config +-------------------- +[remote] +type = filefabric +url = https://yourfabric.smestorage.com/ +permanent_token = xxxxxxxxxxxxxxx-xxxxxxxxxxxxxxxx +-------------------- +y) Yes this is OK (default) +e) Edit this remote +d) Delete this remote +y/e/d> y +``` + +Once configured you can then use `rclone` like this, + +List directories in top level of your Enterprise File Fabric + + rclone lsd remote: + +List all the files in your Enterprise File Fabric + + rclone ls remote: + +To copy a local directory to an Enterprise File Fabric directory called backup + + rclone copy /home/source remote:backup + +### Modified time and hashes + +The Enterprise File Fabric allows modification times to be set on +files accurate to 1 second. These will be used to detect whether +objects need syncing or not. + +The Enterprise File Fabric does not support any data hashes at this time. + +### Restricted filename characters + +The [default restricted characters set](/overview/#restricted-characters) +will be replaced. + +Invalid UTF-8 bytes will also be [replaced](/overview/#invalid-utf8), +as they can't be used in JSON strings. + +### Empty files + +Empty files aren't supported by the Enterprise File Fabric. Rclone will therefore +upload an empty file as a single space with a mime type of +`application/vnd.rclone.empty.file` and files with that mime type are +treated as empty. + +### Root folder ID ### + +You can set the `root_folder_id` for rclone. This is the directory +(identified by its `Folder ID`) that rclone considers to be the root +of your Enterprise File Fabric. + +Normally you will leave this blank and rclone will determine the +correct root to use itself. + +However you can set this to restrict rclone to a specific folder +hierarchy. + +In order to do this you will have to find the `Folder ID` of the +directory you wish rclone to display. These aren't displayed in the +web interface, but you can use `rclone lsf` to find them, for example + +``` +$ rclone lsf --dirs-only -Fip --csv filefabric: +120673758,Burnt PDFs/ +120673759,My Quick Uploads/ +120673755,My Syncs/ +120673756,My backups/ +120673757,My contacts/ +120673761,S3 Storage/ +``` + +The ID for "S3 Storage" would be `120673761`. + +{{< rem autogenerated options start" - DO NOT EDIT - instead edit fs.RegInfo in backend/filefabric/filefabric.go then run make backenddocs" >}} +### Standard Options + +Here are the standard options specific to filefabric (Enterprise File Fabric). + +#### --filefabric-url + +URL of the Enterprise File Fabric to connect to + +- Config: url +- Env Var: RCLONE_FILEFABRIC_URL +- Type: string +- Default: "" +- Examples: + - "https://storagemadeeasy.com" + - Storage Made Easy US + - "https://eu.storagemadeeasy.com" + - Storage Made Easy EU + - "https://yourfabric.smestorage.com" + - Connect to your Enterprise File Fabric + +#### --filefabric-root-folder-id + +ID of the root folder +Leave blank normally. + +Fill in to make rclone start with directory of a given ID. + + +- Config: root_folder_id +- Env Var: RCLONE_FILEFABRIC_ROOT_FOLDER_ID +- Type: string +- Default: "" + +#### --filefabric-permanent-token + +Permanent Authentication Token + +A Permanent Authentication Token can be created in the Enterprise File +Fabric, on the users Dashboard under Security, there is an entry +you'll see called "My Authentication Tokens". Click the Manage button +to create one. + +These tokens are normally valid for several years. + +For more info see: https://docs.storagemadeeasy.com/organisationcloud/api-tokens + + +- Config: permanent_token +- Env Var: RCLONE_FILEFABRIC_PERMANENT_TOKEN +- Type: string +- Default: "" + +### Advanced Options + +Here are the advanced options specific to filefabric (Enterprise File Fabric). + +#### --filefabric-token + +Session Token + +This is a session token which rclone caches in the config file. It is +usually valid for 1 hour. + +Don't set this value - rclone will set it automatically. + + +- Config: token +- Env Var: RCLONE_FILEFABRIC_TOKEN +- Type: string +- Default: "" + +#### --filefabric-token-expiry + +Token expiry time + +Don't set this value - rclone will set it automatically. + + +- Config: token_expiry +- Env Var: RCLONE_FILEFABRIC_TOKEN_EXPIRY +- Type: string +- Default: "" + +#### --filefabric-version + +Version read from the file fabric + +Don't set this value - rclone will set it automatically. + + +- Config: version +- Env Var: RCLONE_FILEFABRIC_VERSION +- Type: string +- Default: "" + +#### --filefabric-encoding + +This sets the encoding for the backend. + +See: the [encoding section in the overview](/overview/#encoding) for more info. + +- Config: encoding +- Env Var: RCLONE_FILEFABRIC_ENCODING +- Type: MultiEncoder +- Default: Slash,Del,Ctl,InvalidUtf8,Dot + +{{< rem autogenerated options stop >}} diff --git a/docs/content/overview.md b/docs/content/overview.md index 10f6867c2..9cffb8cf6 100644 --- a/docs/content/overview.md +++ b/docs/content/overview.md @@ -23,6 +23,7 @@ Here is an overview of the major features of each cloud storage system. | Box | SHA1 | Yes | Yes | No | - | | Citrix ShareFile | MD5 | Yes | Yes | No | - | | Dropbox | DBHASH ¹ | Yes | Yes | No | - | +| Enterprise File Fabric | - | Yes | Yes | No | R/W | | FTP | - | No | No | No | - | | Google Cloud Storage | MD5 | Yes | No | No | R/W | | Google Drive | MD5 | Yes | No | Yes | R/W | @@ -334,6 +335,7 @@ upon backend specific capabilities. | Box | Yes | Yes | Yes | Yes | Yes ‡‡ | No | Yes | Yes | No | Yes | | Citrix ShareFile | Yes | Yes | Yes | Yes | No | No | Yes | No | No | Yes | | Dropbox | Yes | Yes | Yes | Yes | No [#575](https://github.com/rclone/rclone/issues/575) | No | Yes | Yes | Yes | Yes | +| Enterprise File Fabric | Yes | Yes | Yes | Yes | No | No | No | No | No | Yes | | FTP | No | No | Yes | Yes | No | No | Yes | No [#2178](https://github.com/rclone/rclone/issues/2178) | No | Yes | | Google Cloud Storage | Yes | Yes | No | No | No | Yes | Yes | No [#2178](https://github.com/rclone/rclone/issues/2178) | No | No | | Google Drive | Yes | Yes | Yes | Yes | Yes | Yes | Yes | Yes | Yes | Yes | diff --git a/docs/layouts/chrome/navbar.html b/docs/layouts/chrome/navbar.html index de9eef0ff..1625ba58c 100644 --- a/docs/layouts/chrome/navbar.html +++ b/docs/layouts/chrome/navbar.html @@ -72,6 +72,7 @@ Citrix ShareFile Crypt (encrypts the others) Dropbox + Enterprise File Fabric FTP Google Cloud Storage Google Drive diff --git a/fstest/test_all/config.yaml b/fstest/test_all/config.yaml index 00ed13251..ecb4702ea 100644 --- a/fstest/test_all/config.yaml +++ b/fstest/test_all/config.yaml @@ -94,6 +94,9 @@ backends: - backend: "dropbox" remote: "TestDropbox:" fastlist: false + - backend: "filefabric" + remote: "TestFileFabric:" + fastlist: false - backend: "googlecloudstorage" remote: "TestGoogleCloudStorage:" fastlist: true @@ -291,3 +294,6 @@ backends: - backend: "tardigrade" remote: "TestTardigrade:" fastlist: true + - backend: "filefabric" + remote: "TestFileFabric:" + fastlist: false