From 3b3625037c4bca776b2b04d4fbaeaa825df50ae9 Mon Sep 17 00:00:00 2001 From: Fornax Date: Mon, 12 Aug 2024 14:35:44 +0200 Subject: [PATCH] Add pixeldrain backend This commit adds support for pixeldrain's experimental filesystem API. --- README.md | 1 + backend/all/all.go | 1 + backend/pixeldrain/api_client.go | 397 ++++++++++++++++++ backend/pixeldrain/pixeldrain.go | 567 ++++++++++++++++++++++++++ backend/pixeldrain/pixeldrain_test.go | 18 + bin/make_manual.py | 1 + docs/content/_index.md | 1 + docs/content/docs.md | 1 + docs/content/overview.md | 2 + docs/content/pixeldrain.md | 181 ++++++++ docs/layouts/chrome/navbar.html | 1 + fstest/test_all/config.yaml | 5 + 12 files changed, 1176 insertions(+) create mode 100644 backend/pixeldrain/api_client.go create mode 100644 backend/pixeldrain/pixeldrain.go create mode 100644 backend/pixeldrain/pixeldrain_test.go create mode 100644 docs/content/pixeldrain.md diff --git a/README.md b/README.md index 0c17cb6a8..833902ebc 100644 --- a/README.md +++ b/README.md @@ -93,6 +93,7 @@ Rclone *("rsync for cloud storage")* is a command-line program to sync files and * pCloud [:page_facing_up:](https://rclone.org/pcloud/) * Petabox [:page_facing_up:](https://rclone.org/s3/#petabox) * PikPak [:page_facing_up:](https://rclone.org/pikpak/) + * Pixeldrain [:page_facing_up:](https://rclone.org/pixeldrain/) * premiumize.me [:page_facing_up:](https://rclone.org/premiumizeme/) * put.io [:page_facing_up:](https://rclone.org/putio/) * Proton Drive [:page_facing_up:](https://rclone.org/protondrive/) diff --git a/backend/all/all.go b/backend/all/all.go index f097cb261..9800df87b 100644 --- a/backend/all/all.go +++ b/backend/all/all.go @@ -39,6 +39,7 @@ import ( _ "github.com/rclone/rclone/backend/oracleobjectstorage" _ "github.com/rclone/rclone/backend/pcloud" _ "github.com/rclone/rclone/backend/pikpak" + _ "github.com/rclone/rclone/backend/pixeldrain" _ "github.com/rclone/rclone/backend/premiumizeme" _ "github.com/rclone/rclone/backend/protondrive" _ "github.com/rclone/rclone/backend/putio" diff --git a/backend/pixeldrain/api_client.go b/backend/pixeldrain/api_client.go new file mode 100644 index 000000000..e422f61c4 --- /dev/null +++ b/backend/pixeldrain/api_client.go @@ -0,0 +1,397 @@ +package pixeldrain + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "net/url" + "strings" + "time" + + "github.com/rclone/rclone/fs" + "github.com/rclone/rclone/fs/fserrors" + "github.com/rclone/rclone/lib/rest" +) + +// FilesystemPath is the object which is returned from the pixeldrain API when +// running the stat command on a path. It includes the node information for all +// the members of the path and for all the children of the requested directory. +type FilesystemPath struct { + Path []FilesystemNode `json:"path"` + BaseIndex int `json:"base_index"` + Children []FilesystemNode `json:"children"` +} + +// Base returns the base node of the path, this is the node that the path points +// to +func (fsp *FilesystemPath) Base() FilesystemNode { + return fsp.Path[fsp.BaseIndex] +} + +// FilesystemNode is a single node in the pixeldrain filesystem. Usually part of +// a Path or Children slice. The Node is also returned as response from update +// commands, if requested +type FilesystemNode struct { + Type string `json:"type"` + Path string `json:"path"` + Name string `json:"name"` + Created time.Time `json:"created"` + Modified time.Time `json:"modified"` + ModeOctal string `json:"mode_octal"` + + // File params + FileSize int64 `json:"file_size"` + FileType string `json:"file_type"` + SHA256Sum string `json:"sha256_sum"` + + // ID is only filled in when the file/directory is publicly shared + ID string `json:"id,omitempty"` +} + +// ChangeLog is a log of changes that happened in a filesystem. Changes returned +// from the API are on chronological order from old to new. A change log can be +// requested for any directory or file, but change logging needs to be enabled +// with the update API before any log entries will be made. Changes are logged +// for 24 hours after logging was enabled. Each time a change log is requested +// the timer is reset to 24 hours. +type ChangeLog []ChangeLogEntry + +// ChangeLogEntry is a single entry in a directory's change log. It contains the +// time at which the change occurred. The path relative to the requested +// directory and the action that was performend (update, move or delete). In +// case of a move operation the new path of the file is stored in the path_new +// field +type ChangeLogEntry struct { + Time time.Time `json:"time"` + Path string `json:"path"` + PathNew string `json:"path_new"` + Action string `json:"action"` + Type string `json:"type"` +} + +// UserInfo contains information about the logged in user +type UserInfo struct { + Username string `json:"username"` + Subscription SubscriptionType `json:"subscription"` + StorageSpaceUsed int64 `json:"storage_space_used"` +} + +// SubscriptionType contains information about a subscription type. It's not the +// active subscription itself, only the properties of the subscription. Like the +// perks and cost +type SubscriptionType struct { + Name string `json:"name"` + StorageSpace int64 `json:"storage_space"` +} + +// APIError is the error type returned by the pixeldrain API +type APIError struct { + StatusCode string `json:"value"` + Message string `json:"message"` +} + +func (e APIError) Error() string { return e.StatusCode } + +// Generalized errors which are caught in our own handlers and translated to +// more specific errors from the fs package. +var ( + errNotFound = errors.New("pd api: path not found") + errExists = errors.New("pd api: node already exists") + errAuthenticationFailed = errors.New("pd api: authentication failed") +) + +func apiErrorHandler(resp *http.Response) (err error) { + var e APIError + if err = json.NewDecoder(resp.Body).Decode(&e); err != nil { + return fmt.Errorf("failed to parse error json: %w", err) + } + + // We close the body here so that the API handlers can be sure that the + // response body is not still open when an error was returned + if err = resp.Body.Close(); err != nil { + return fmt.Errorf("failed to close resp body: %w", err) + } + + if e.StatusCode == "path_not_found" { + return errNotFound + } else if e.StatusCode == "directory_not_empty" { + return fs.ErrorDirectoryNotEmpty + } else if e.StatusCode == "node_already_exists" { + return errExists + } else if e.StatusCode == "authentication_failed" { + return errAuthenticationFailed + } else if e.StatusCode == "permission_denied" { + return fs.ErrorPermissionDenied + } + + return e +} + +var retryErrorCodes = []int{ + 429, // Too Many Requests. + 500, // Internal Server Error + 502, // Bad Gateway + 503, // Service Unavailable + 504, // Gateway Timeout +} + +// shouldRetry returns a boolean as to whether this resp and err deserve to be +// retried. It returns the err as a convenience so it can be used as the return +// value in the pacer function +func shouldRetry(ctx context.Context, resp *http.Response, err error) (bool, error) { + if fserrors.ContextError(ctx, &err) { + return false, err + } + return fserrors.ShouldRetry(err) || fserrors.ShouldRetryHTTP(resp, retryErrorCodes), err +} + +// paramsFromMetadata turns the fs.Metadata into instructions the pixeldrain API +// can understand. +func paramsFromMetadata(meta fs.Metadata) (params url.Values) { + params = make(url.Values) + + if modified, ok := meta["mtime"]; ok { + params.Set("modified", modified) + } + if created, ok := meta["btime"]; ok { + params.Set("created", created) + } + if mode, ok := meta["mode"]; ok { + params.Set("mode", mode) + } + if shared, ok := meta["shared"]; ok { + params.Set("shared", shared) + } + if loggingEnabled, ok := meta["logging_enabled"]; ok { + params.Set("logging_enabled", loggingEnabled) + } + + return params +} + +// nodeToObject converts a single FilesystemNode API response to an object. The +// node is usually a single element from a directory listing +func (f *Fs) nodeToObject(node FilesystemNode) (o *Object) { + // Trim the path prefix. The path prefix is hidden from rclone during all + // operations. Saving it here would confuse rclone a lot. So instead we + // strip it here and add it back for every API request we need to perform + node.Path = strings.TrimPrefix(node.Path, f.pathPrefix) + return &Object{fs: f, base: node} +} + +func (f *Fs) nodeToDirectory(node FilesystemNode) fs.DirEntry { + return fs.NewDir(strings.TrimPrefix(node.Path, f.pathPrefix), node.Modified).SetID(node.ID) +} + +func (f *Fs) escapePath(p string) (out string) { + // Add the path prefix, encode all the parts and combine them together + var parts = strings.Split(f.pathPrefix+p, "/") + for i := range parts { + parts[i] = url.PathEscape(parts[i]) + } + return strings.Join(parts, "/") +} + +func (f *Fs) put( + ctx context.Context, + path string, + body io.Reader, + meta fs.Metadata, + options []fs.OpenOption, +) (node FilesystemNode, err error) { + var params = paramsFromMetadata(meta) + + // Tell the server to automatically create parent directories if they don't + // exist yet + params.Set("make_parents", "true") + + return node, f.pacer.Call(func() (bool, error) { + resp, err := f.srv.CallJSON( + ctx, + &rest.Opts{ + Method: "PUT", + Path: f.escapePath(path), + Body: body, + Parameters: params, + Options: options, + }, + nil, + &node, + ) + return shouldRetry(ctx, resp, err) + }) +} + +func (f *Fs) read(ctx context.Context, path string, options []fs.OpenOption) (in io.ReadCloser, err error) { + var resp *http.Response + err = f.pacer.Call(func() (bool, error) { + resp, err = f.srv.Call(ctx, &rest.Opts{ + Method: "GET", + Path: f.escapePath(path), + Options: options, + }) + return shouldRetry(ctx, resp, err) + }) + if err != nil { + return nil, err + } + return resp.Body, err +} + +func (f *Fs) stat(ctx context.Context, path string) (fsp FilesystemPath, err error) { + return fsp, f.pacer.Call(func() (bool, error) { + resp, err := f.srv.CallJSON( + ctx, + &rest.Opts{ + Method: "GET", + Path: f.escapePath(path), + // To receive node info from the pixeldrain API you need to add the + // ?stat query. Without it pixeldrain will return the file contents + // in the URL points to a file + Parameters: url.Values{"stat": []string{""}}, + }, + nil, + &fsp, + ) + return shouldRetry(ctx, resp, err) + }) +} + +func (f *Fs) changeLog(ctx context.Context, start, end time.Time) (changeLog ChangeLog, err error) { + return changeLog, f.pacer.Call(func() (bool, error) { + resp, err := f.srv.CallJSON( + ctx, + &rest.Opts{ + Method: "GET", + Path: f.escapePath(""), + Parameters: url.Values{ + "change_log": []string{""}, + "start": []string{start.Format(time.RFC3339Nano)}, + "end": []string{end.Format(time.RFC3339Nano)}, + }, + }, + nil, + &changeLog, + ) + return shouldRetry(ctx, resp, err) + }) +} + +func (f *Fs) update(ctx context.Context, path string, fields fs.Metadata) (node FilesystemNode, err error) { + var params = paramsFromMetadata(fields) + params.Set("action", "update") + + return node, f.pacer.Call(func() (bool, error) { + resp, err := f.srv.CallJSON( + ctx, + &rest.Opts{ + Method: "POST", + Path: f.escapePath(path), + MultipartParams: params, + }, + nil, + &node, + ) + return shouldRetry(ctx, resp, err) + }) +} + +func (f *Fs) mkdir(ctx context.Context, dir string) (err error) { + return f.pacer.Call(func() (bool, error) { + resp, err := f.srv.CallJSON( + ctx, + &rest.Opts{ + Method: "POST", + Path: f.escapePath(dir), + MultipartParams: url.Values{"action": []string{"mkdirall"}}, + NoResponse: true, + }, + nil, + nil, + ) + return shouldRetry(ctx, resp, err) + }) +} + +var errIncompatibleSourceFS = errors.New("source filesystem is not the same as target") + +// Renames a file on the server side. Can be used for both directories and files +func (f *Fs) rename(ctx context.Context, src fs.Fs, from, to string, meta fs.Metadata) (node FilesystemNode, err error) { + srcFs, ok := src.(*Fs) + if !ok { + // This is not a pixeldrain FS, can't move + return node, errIncompatibleSourceFS + } else if srcFs.opt.RootFolderID != f.opt.RootFolderID { + // Path is not in the same root dir, can't move + return node, errIncompatibleSourceFS + } + + var params = paramsFromMetadata(meta) + params.Set("action", "rename") + + // The target is always in our own filesystem so here we use our + // own pathPrefix + params.Set("target", f.pathPrefix+to) + + // Create parent directories if the parent directory of the file + // does not exist yet + params.Set("make_parents", "true") + + return node, f.pacer.Call(func() (bool, error) { + resp, err := f.srv.CallJSON( + ctx, + &rest.Opts{ + Method: "POST", + // Important: We use the source FS path prefix here + Path: srcFs.escapePath(from), + MultipartParams: params, + }, + nil, + &node, + ) + return shouldRetry(ctx, resp, err) + }) +} + +func (f *Fs) delete(ctx context.Context, path string, recursive bool) (err error) { + var params url.Values + if recursive { + // Tell the server to recursively delete all child files + params = url.Values{"recursive": []string{"true"}} + } + + return f.pacer.Call(func() (bool, error) { + resp, err := f.srv.CallJSON( + ctx, + &rest.Opts{ + Method: "DELETE", + Path: f.escapePath(path), + Parameters: params, + NoResponse: true, + }, + nil, nil, + ) + return shouldRetry(ctx, resp, err) + }) +} + +func (f *Fs) userInfo(ctx context.Context) (user UserInfo, err error) { + return user, f.pacer.Call(func() (bool, error) { + resp, err := f.srv.CallJSON( + ctx, + &rest.Opts{ + Method: "GET", + // The default RootURL points at the filesystem endpoint. We can't + // use that to request user information. So here we override it to + // the user endpoint + RootURL: f.opt.APIURL + "/user", + }, + nil, + &user, + ) + return shouldRetry(ctx, resp, err) + }) +} diff --git a/backend/pixeldrain/pixeldrain.go b/backend/pixeldrain/pixeldrain.go new file mode 100644 index 000000000..abf6c1f57 --- /dev/null +++ b/backend/pixeldrain/pixeldrain.go @@ -0,0 +1,567 @@ +// Package pixeldrain provides an interface to the Pixeldrain object storage +// system. +package pixeldrain + +import ( + "context" + "errors" + "fmt" + "io" + "path" + "strconv" + "strings" + "time" + + "github.com/rclone/rclone/fs" + "github.com/rclone/rclone/fs/config/configmap" + "github.com/rclone/rclone/fs/config/configstruct" + "github.com/rclone/rclone/fs/fshttp" + "github.com/rclone/rclone/fs/hash" + "github.com/rclone/rclone/lib/pacer" + "github.com/rclone/rclone/lib/rest" +) + +const ( + timeFormat = time.RFC3339Nano + minSleep = pacer.MinSleep(10 * time.Millisecond) + maxSleep = pacer.MaxSleep(1 * time.Second) + decayConstant = pacer.DecayConstant(2) // bigger for slower decay, exponential +) + +// Register with Fs +func init() { + fs.Register(&fs.RegInfo{ + Name: "pixeldrain", + Description: "Pixeldrain Filesystem", + NewFs: NewFs, + Config: nil, + Options: []fs.Option{{ + Name: "api_key", + Help: "API key for your pixeldrain account.\n" + + "Found on https://pixeldrain.com/user/api_keys.", + Sensitive: true, + }, { + Name: "root_folder_id", + Help: "Root of the filesystem to use.\n\n" + + "Set to 'me' to use your personal filesystem. " + + "Set to a shared directory ID to use a shared directory.", + Default: "me", + }, { + Name: "api_url", + Help: "The API endpoint to connect to. In the vast majority of cases it's fine to leave\n" + + "this at default. It is only intended to be changed for testing purposes.", + Default: "https://pixeldrain.com/api", + Advanced: true, + Required: true, + }}, + MetadataInfo: &fs.MetadataInfo{ + System: map[string]fs.MetadataHelp{ + "mode": { + Help: "File mode", + Type: "octal, unix style", + Example: "755", + }, + "mtime": { + Help: "Time of last modification", + Type: "RFC 3339", + Example: timeFormat, + }, + "btime": { + Help: "Time of file birth (creation)", + Type: "RFC 3339", + Example: timeFormat, + }, + }, + Help: "Pixeldrain supports file modes and creation times.", + }, + }) +} + +// Options defines the configuration for this backend +type Options struct { + APIKey string `config:"api_key"` + RootFolderID string `config:"root_folder_id"` + APIURL string `config:"api_url"` +} + +// Fs represents a remote box +type Fs struct { + name string // name of this remote, as given to NewFS + root string // the path we are working on, as given to NewFS + opt Options // parsed options + features *fs.Features // optional features + srv *rest.Client // the connection to the server + pacer *fs.Pacer + loggedIn bool // if the user is authenticated + + // Pathprefix is the directory we're working in. The pathPrefix is stripped + // from every API response containing a path. The pathPrefix always begins + // and ends with a slash for concatenation convenience + pathPrefix string +} + +// Object describes a pixeldrain file +type Object struct { + fs *Fs // what this object is part of + base FilesystemNode // the node this object references +} + +// NewFs constructs an Fs from the path, container:path +func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, error) { + // Parse config into Options struct + opt := new(Options) + err := configstruct.Set(m, opt) + if err != nil { + return nil, err + } + + f := &Fs{ + name: name, + root: root, + opt: *opt, + srv: rest.NewClient(fshttp.NewClient(ctx)).SetErrorHandler(apiErrorHandler), + pacer: fs.NewPacer(ctx, pacer.NewDefault(minSleep, maxSleep, decayConstant)), + } + f.features = (&fs.Features{ + ReadMimeType: true, + CanHaveEmptyDirectories: true, + ReadMetadata: true, + WriteMetadata: true, + }).Fill(ctx, f) + + // Set the path prefix. This is the path to the root directory on the + // server. We add it to each request and strip it from each response because + // rclone does not want to see it + f.pathPrefix = "/" + path.Join(opt.RootFolderID, f.root) + "/" + + // The root URL equates to https://pixeldrain.com/api/filesystem during + // normal operation. API handlers need to manually add the pathPrefix to + // each request + f.srv.SetRoot(opt.APIURL + "/filesystem") + + // If using an APIKey, set the Authorization header + if len(opt.APIKey) > 0 { + f.srv.SetUserPass("", opt.APIKey) + + // Check if credentials are correct + user, err := f.userInfo(ctx) + if err != nil { + return nil, fmt.Errorf("failed to get user data: %w", err) + } + + f.loggedIn = true + + fs.Infof(f, + "Logged in as '%s', subscription '%s', storage limit %d", + user.Username, user.Subscription.Name, user.Subscription.StorageSpace, + ) + } + + if !f.loggedIn && opt.RootFolderID == "me" { + return nil, errors.New("authentication required: the 'me' directory can only be accessed while logged in") + } + + // Satisfy TestFsIsFile. This test expects that we throw an error if the + // filesystem root is a file + fsp, err := f.stat(ctx, "") + if err != errNotFound && err != nil { + // It doesn't matter if the root directory does not exist, as long as it + // is not a file. This is what the test dictates + return f, err + } else if err == nil && fsp.Base().Type == "file" { + // The filesystem root is a file, rclone wants us to set the root to the + // parent directory + f.root = path.Dir(f.root) + f.pathPrefix = "/" + path.Join(opt.RootFolderID, f.root) + "/" + return f, fs.ErrorIsFile + } + + return f, nil +} + +// List the objects and directories in dir into entries. The +// entries can be returned in any order but should be for a +// complete directory. +// +// dir should be "" to list the root, and should not have +// trailing slashes. +// +// This should return ErrDirNotFound if the directory isn't +// found. +func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err error) { + fsp, err := f.stat(ctx, dir) + if err == errNotFound { + return nil, fs.ErrorDirNotFound + } else if err != nil { + return nil, err + } else if fsp.Base().Type == "file" { + return nil, fs.ErrorIsFile + } + + entries = make(fs.DirEntries, len(fsp.Children)) + for i := range fsp.Children { + if fsp.Children[i].Type == "dir" { + entries[i] = f.nodeToDirectory(fsp.Children[i]) + } else { + entries[i] = f.nodeToObject(fsp.Children[i]) + } + } + + return entries, nil +} + +// NewObject finds the Object at remote. If it can't be found +// it returns the error fs.ErrorObjectNotFound. +func (f *Fs) NewObject(ctx context.Context, remote string) (fs.Object, error) { + fsp, err := f.stat(ctx, remote) + if err == errNotFound { + return nil, fs.ErrorObjectNotFound + } else if err != nil { + return nil, err + } else if fsp.Base().Type == "dir" { + return nil, fs.ErrorIsDir + } + return f.nodeToObject(fsp.Base()), nil +} + +// Put the object +// +// Copy the reader in to the new object which is returned. +// +// The new object may have been created if an error is returned +func (f *Fs) Put(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) { + meta, err := fs.GetMetadataOptions(ctx, f, src, options) + if err != nil { + return nil, fmt.Errorf("failed to get object metadata") + } + + // Overwrite the mtime if it was not already set in the metadata + if _, ok := meta["mtime"]; !ok { + if meta == nil { + meta = make(fs.Metadata) + } + meta["mtime"] = src.ModTime(ctx).Format(timeFormat) + } + + node, err := f.put(ctx, src.Remote(), in, meta, options) + if err != nil { + return nil, fmt.Errorf("failed to put object: %w", err) + } + + return f.nodeToObject(node), nil +} + +// Mkdir creates the container if it doesn't exist +func (f *Fs) Mkdir(ctx context.Context, dir string) (err error) { + err = f.mkdir(ctx, dir) + if err == errNotFound { + return fs.ErrorDirNotFound + } else if err == errExists { + // Spec says we do not return an error if the directory already exists + return nil + } + return err +} + +// Rmdir deletes the root folder +// +// Returns an error if it isn't empty +func (f *Fs) Rmdir(ctx context.Context, dir string) (err error) { + err = f.delete(ctx, dir, false) + if err == errNotFound { + return fs.ErrorDirNotFound + } + return err +} + +// Name of the remote (as passed into NewFs) +func (f *Fs) Name() string { return f.name } + +// Root of the remote (as passed into NewFs) +func (f *Fs) Root() string { return f.root } + +// String converts this Fs to a string +func (f *Fs) String() string { return fmt.Sprintf("pixeldrain root '%s'", f.root) } + +// Precision return the precision of this Fs +func (f *Fs) Precision() time.Duration { return time.Millisecond } + +// Hashes returns the supported hash sets. +func (f *Fs) Hashes() hash.Set { return hash.Set(hash.SHA256) } + +// Features returns the optional features of this Fs +func (f *Fs) Features() *fs.Features { return f.features } + +// Purge all files in the directory specified +// +// Implement this if you have a way of deleting all the files +// quicker than just running Remove() on the result of List() +// +// Return an error if it doesn't exist +func (f *Fs) Purge(ctx context.Context, dir string) (err error) { + err = f.delete(ctx, dir, true) + if err == errNotFound { + return fs.ErrorDirNotFound + } + return err +} + +// Move src to this remote using server-side move operations. +// +// This is stored with the remote path given. +// +// It returns the destination Object and a possible error. +// +// Will only be called if src.Fs().Name() == f.Name() +// +// If it isn't possible then return fs.ErrorCantMove +func (f *Fs) Move(ctx context.Context, src fs.Object, remote string) (fs.Object, error) { + srcObj, ok := src.(*Object) + if !ok { + // This is not a pixeldrain object. Can't move + return nil, fs.ErrorCantMove + } + + node, err := f.rename(ctx, srcObj.fs, srcObj.base.Path, remote, fs.GetConfig(ctx).MetadataSet) + if err == errIncompatibleSourceFS { + return nil, fs.ErrorCantMove + } else if err == errNotFound { + return nil, fs.ErrorObjectNotFound + } + + return f.nodeToObject(node), nil +} + +// DirMove moves src, srcRemote to this remote at dstRemote +// using server-side move operations. +// +// Will only be called if src.Fs().Name() == f.Name() +// +// If it isn't possible then return fs.ErrorCantDirMove +// +// If destination exists then return fs.ErrorDirExists +func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string) (err error) { + _, err = f.rename(ctx, src, srcRemote, dstRemote, nil) + if err == errIncompatibleSourceFS { + return fs.ErrorCantDirMove + } else if err == errNotFound { + return fs.ErrorDirNotFound + } else if err == errExists { + return fs.ErrorDirExists + } + return err +} + +// ChangeNotify calls the passed function with a path +// that has had changes. If the implementation +// uses polling, it should adhere to the given interval. +// At least one value will be written to the channel, +// specifying the initial value and updated values might +// follow. A 0 Duration should pause the polling. +// The ChangeNotify implementation must empty the channel +// regularly. When the channel gets closed, the implementation +// should stop polling and release resources. +func (f *Fs) ChangeNotify(ctx context.Context, notify func(string, fs.EntryType), newInterval <-chan time.Duration) { + // If the bucket ID is not /me we need to explicitly enable change logging + // for this directory or file + if f.pathPrefix != "/me/" { + _, err := f.update(ctx, "", fs.Metadata{"logging_enabled": "true"}) + if err != nil { + fs.Errorf(f, "Failed to set up change logging for path '%s': %s", f.pathPrefix, err) + } + } + + go f.changeNotify(ctx, notify, newInterval) +} +func (f *Fs) changeNotify(ctx context.Context, notify func(string, fs.EntryType), newInterval <-chan time.Duration) { + var ticker = time.NewTicker(<-newInterval) + var lastPoll = time.Now() + + for { + select { + case dur, ok := <-newInterval: + if !ok { + ticker.Stop() + return + } + + fs.Debugf(f, "Polling changes at an interval of %s", dur) + ticker.Reset(dur) + + case t := <-ticker.C: + clog, err := f.changeLog(ctx, lastPoll, t) + if err != nil { + fs.Errorf(f, "Failed to get change log for path '%s': %s", f.pathPrefix, err) + continue + } + + for i := range clog { + fs.Debugf(f, "Path '%s' (%s) changed (%s) in directory '%s'", + clog[i].Path, clog[i].Type, clog[i].Action, f.pathPrefix) + + if clog[i].Type == "dir" { + notify(strings.TrimPrefix(clog[i].Path, "/"), fs.EntryDirectory) + } else if clog[i].Type == "file" { + notify(strings.TrimPrefix(clog[i].Path, "/"), fs.EntryObject) + } + } + + lastPoll = t + } + } +} + +// PutStream uploads to the remote path with the modTime given of indeterminate size +// +// May create the object even if it returns an error - if so +// will return the object and the error, otherwise will return +// nil and the error +func (f *Fs) PutStream(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) { + // Put already supports streaming so we just use that + return f.Put(ctx, in, src, options...) +} + +// DirSetModTime sets the mtime metadata on a directory +func (f *Fs) DirSetModTime(ctx context.Context, dir string, modTime time.Time) (err error) { + _, err = f.update(ctx, dir, fs.Metadata{"mtime": modTime.Format(timeFormat)}) + return err +} + +// PublicLink generates a public link to the remote path (usually readable by anyone) +func (f *Fs) PublicLink(ctx context.Context, remote string, expire fs.Duration, unlink bool) (string, error) { + fsn, err := f.update(ctx, remote, fs.Metadata{"shared": strconv.FormatBool(!unlink)}) + if err != nil { + return "", err + } + if fsn.ID != "" { + return strings.Replace(f.opt.APIURL, "/api", "/d/", 1) + fsn.ID, nil + } + return "", nil +} + +// About gets quota information +func (f *Fs) About(ctx context.Context) (usage *fs.Usage, err error) { + user, err := f.userInfo(ctx) + if err != nil { + return nil, fmt.Errorf("failed to read user info: %w", err) + } + + usage = &fs.Usage{Used: fs.NewUsageValue(user.StorageSpaceUsed)} + + if user.Subscription.StorageSpace > -1 { + usage.Total = fs.NewUsageValue(user.Subscription.StorageSpace) + } + + return usage, nil +} + +// SetModTime sets the modification time of the local fs object +func (o *Object) SetModTime(ctx context.Context, modTime time.Time) (err error) { + _, err = o.fs.update(ctx, o.base.Path, fs.Metadata{"mtime": modTime.Format(timeFormat)}) + if err == nil { + o.base.Modified = modTime + } + return err +} + +// Open an object for read +func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.ReadCloser, err error) { + return o.fs.read(ctx, o.base.Path, options) +} + +// Update the object with the contents of the io.Reader, modTime and size +// +// If existing is set then it updates the object rather than creating a new one. +// +// The new object may have been created if an error is returned. +func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (err error) { + // Copy the parameters and update the object + o.base.Modified = src.ModTime(ctx) + o.base.FileSize = src.Size() + o.base.SHA256Sum, _ = src.Hash(ctx, hash.SHA256) + _, err = o.fs.Put(ctx, in, o, options...) + return err +} + +// Remove an object +func (o *Object) Remove(ctx context.Context) error { + return o.fs.delete(ctx, o.base.Path, false) +} + +// Fs returns the parent Fs +func (o *Object) Fs() fs.Info { + return o.fs +} + +// Hash returns the SHA-256 of an object returning a lowercase hex string +func (o *Object) Hash(ctx context.Context, t hash.Type) (string, error) { + if t != hash.SHA256 { + return "", hash.ErrUnsupported + } + return o.base.SHA256Sum, nil +} + +// Storable returns a boolean showing whether this object storable +func (o *Object) Storable() bool { + return true +} + +// Return a string version +func (o *Object) String() string { + if o == nil { + return "" + } + return o.base.Path +} + +// Remote returns the remote path +func (o *Object) Remote() string { + return o.base.Path +} + +// ModTime returns the modification time of the object +// +// It attempts to read the objects mtime and if that isn't present the +// LastModified returned in the http headers +func (o *Object) ModTime(ctx context.Context) time.Time { + return o.base.Modified +} + +// Size returns the size of an object in bytes +func (o *Object) Size() int64 { + return o.base.FileSize +} + +// MimeType returns the content type of the Object if known, or "" if not +func (o *Object) MimeType(ctx context.Context) string { + return o.base.FileType +} + +// Metadata returns metadata for an object +// +// It should return nil if there is no Metadata +func (o *Object) Metadata(ctx context.Context) (fs.Metadata, error) { + return fs.Metadata{ + "mode": o.base.ModeOctal, + "mtime": o.base.Modified.Format(timeFormat), + "btime": o.base.Created.Format(timeFormat), + }, nil +} + +// Verify that all the interfaces are implemented correctly +var ( + _ fs.Fs = (*Fs)(nil) + _ fs.Info = (*Fs)(nil) + _ fs.Purger = (*Fs)(nil) + _ fs.Mover = (*Fs)(nil) + _ fs.DirMover = (*Fs)(nil) + _ fs.ChangeNotifier = (*Fs)(nil) + _ fs.PutStreamer = (*Fs)(nil) + _ fs.DirSetModTimer = (*Fs)(nil) + _ fs.PublicLinker = (*Fs)(nil) + _ fs.Abouter = (*Fs)(nil) + _ fs.Object = (*Object)(nil) + _ fs.DirEntry = (*Object)(nil) + _ fs.MimeTyper = (*Object)(nil) + _ fs.Metadataer = (*Object)(nil) +) diff --git a/backend/pixeldrain/pixeldrain_test.go b/backend/pixeldrain/pixeldrain_test.go new file mode 100644 index 000000000..f36975243 --- /dev/null +++ b/backend/pixeldrain/pixeldrain_test.go @@ -0,0 +1,18 @@ +// Test pixeldrain filesystem interface +package pixeldrain_test + +import ( + "testing" + + "github.com/rclone/rclone/backend/pixeldrain" + "github.com/rclone/rclone/fstest/fstests" +) + +// TestIntegration runs integration tests against the remote +func TestIntegration(t *testing.T) { + fstests.Run(t, &fstests.Opt{ + RemoteName: "TestPixeldrain:", + NilObject: (*pixeldrain.Object)(nil), + SkipInvalidUTF8: true, // Pixeldrain throws an error on invalid utf-8 + }) +} diff --git a/bin/make_manual.py b/bin/make_manual.py index 2ebeb207f..41ddccf7c 100755 --- a/bin/make_manual.py +++ b/bin/make_manual.py @@ -69,6 +69,7 @@ docs = [ "swift.md", "pcloud.md", "pikpak.md", + "pixeldrain.md", "premiumizeme.md", "protondrive.md", "putio.md", diff --git a/docs/content/_index.md b/docs/content/_index.md index 1a535220f..714cce51a 100644 --- a/docs/content/_index.md +++ b/docs/content/_index.md @@ -161,6 +161,7 @@ WebDAV or S3, that work out of the box.) {{< provider name="pCloud" home="https://www.pcloud.com/" config="/pcloud/" >}} {{< provider name="Petabox" home="https://petabox.io/" config="/s3/#petabox" >}} {{< provider name="PikPak" home="https://mypikpak.com/" config="/pikpak/" >}} +{{< provider name="Pixeldrain" home="https://pixeldrain.com/" config="/pixeldrain/" >}} {{< provider name="premiumize.me" home="https://premiumize.me/" config="/premiumizeme/" >}} {{< provider name="put.io" home="https://put.io/" config="/putio/" >}} {{< provider name="Proton Drive" home="https://proton.me/drive" config="/protondrive/" >}} diff --git a/docs/content/docs.md b/docs/content/docs.md index 861ee963b..5eaaee51e 100644 --- a/docs/content/docs.md +++ b/docs/content/docs.md @@ -65,6 +65,7 @@ See the following for detailed instructions for * [Oracle Object Storage](/oracleobjectstorage/) * [Pcloud](/pcloud/) * [PikPak](/pikpak/) + * [Pixeldrain](/pixeldrain/) * [premiumize.me](/premiumizeme/) * [put.io](/putio/) * [Proton Drive](/protondrive/) diff --git a/docs/content/overview.md b/docs/content/overview.md index 2b4ecd111..b869b1035 100644 --- a/docs/content/overview.md +++ b/docs/content/overview.md @@ -46,6 +46,7 @@ Here is an overview of the major features of each cloud storage system. | Oracle Object Storage | MD5 | R/W | No | No | R/W | - | | pCloud | MD5, SHA1 ⁷ | R | No | No | W | - | | PikPak | MD5 | R | No | No | R | - | +| Pixeldrain | SHA256 | R/W | No | No | R | RW | | premiumize.me | - | - | Yes | No | R | - | | put.io | CRC-32 | R/W | No | Yes | R | - | | Proton Drive | SHA1 | R/W | No | No | R | - | @@ -520,6 +521,7 @@ upon backend-specific capabilities. | Oracle Object Storage | No | Yes | No | No | Yes | Yes | Yes | Yes | No | No | No | | pCloud | Yes | Yes | Yes | Yes | Yes | No | No | No | Yes | Yes | Yes | | PikPak | Yes | Yes | Yes | Yes | Yes | No | No | No | Yes | Yes | Yes | +| Pixeldrain | Yes | No | Yes | Yes | No | No | Yes | No | Yes | Yes | Yes | | premiumize.me | Yes | No | Yes | Yes | No | No | No | No | Yes | Yes | Yes | | put.io | Yes | No | Yes | Yes | Yes | No | Yes | No | No | Yes | Yes | | Proton Drive | Yes | No | Yes | Yes | Yes | No | No | No | No | Yes | Yes | diff --git a/docs/content/pixeldrain.md b/docs/content/pixeldrain.md new file mode 100644 index 000000000..853ad55cc --- /dev/null +++ b/docs/content/pixeldrain.md @@ -0,0 +1,181 @@ +--- +title: "Pixeldrain" +description: "Rclone docs for Pixeldrain" +versionIntroduced: "v1.68" +--- + +# {{< icon "fa fa-circle" >}} Pixeldrain + +This is the backend for Pixeldrain's premium filesystem feature. This is not the +same as pixeldrain's free file sharing product. The filesystem requires either a +Pro subscription or the Prepaid plan. [More information on +subscriptions](https://pixeldrain.com/#pro). + +An overview of the filesystem's features and limitations is available in the +[filesystem guide](https://pixeldrain.com/filesystem) on pixeldrain. + +### Usage with account + +To use the personal filesystem you will need a [pixeldrain +account](https://pixeldrain.com/register) and either the Prepaid plan or one of +the Patreon-based subscriptions. After registering and subscribing, your +personal filesystem will be available at this link: https://pixeldrain.com/d/me. + +Go to the [API keys page](https://pixeldrain.com/user/api_keys) on your account +and generate a new API key for rclone. Then run `rclone config` and use the API +key to create a new backend. + +Example: + +``` +No remotes found, make a new one? +n) New remote +d) Delete remote +c) Copy remote +s) Set configuration password +q) Quit config +n/d/c/s/q> n + +Enter name for new remote. +name> pixeldrainfs + +Option Storage. +Type of storage to configure. +Choose a number from below, or type in your own value. +... +XX / Pixeldrain Filesystem + \ (pixeldrain) +... +Storage> pixeldrain + +Option api_key. +API key for your pixeldrain account. +Found on https://pixeldrain.com/user/api_keys. +Enter a value. Press Enter to leave empty. +api_key> b1bb1e81-9b7b-406b-986a-c9b20be76e15 + +Option directory_id. +Root of the filesystem to use. Set to 'me' to use your personal filesystem. +Set to a shared directory ID to use a shared directory. +Enter a string value. Press Enter for the default (me). +directory_id> + +Edit advanced config? +y) Yes +n) No (default) +y/n> + +Configuration complete. +Options: +- type: pixeldrain +- api_key: b1bb1e81-9b7b-406b-986a-c9b20be76e15 +Keep this "pixeldrainfs" remote? +y) Yes this is OK (default) +e) Edit this remote +d) Delete this remote +y/e/d> + +Current remotes: + +Name Type +==== ==== +pixeldrainfs pixeldrain + +e) Edit existing remote +n) New remote +d) Delete remote +r) Rename remote +c) Copy remote +s) Set configuration password +q) Quit config +e/n/d/r/c/s/q> q +``` + +### Usage without account + +It is possible to gain read-only access to publicly shared directories through +rclone. For this you only need a directory ID. The directory ID can be found in +the URL of a shared directory, the URL will look like this +`https://pixeldrain.com/d/abcd1234` where `abcd1234` is the directory ID. +Directory IDs in your own filesystem can also be listed with the `lsf` command: + +`rclone lsf Pixeldrain: --dirs-only -Fpi` + +This will print directories in your `Pixeldrain` home directory and their public +IDs. + +Enter this directory ID in the rclone config and you will be able to access the +directory. + +{{< rem autogenerated options start" - DO NOT EDIT - instead edit fs.RegInfo in backend/pixeldrain/pixeldrain.go then run make backenddocs" >}} +### Standard options + +Here are the Standard options specific to pixeldrain (Pixeldrain Filesystem). + +#### --pixeldrain-api-key + +API key for your pixeldrain account. +Found on https://pixeldrain.com/user/api_keys. + +Properties: + +- Config: api_key +- Env Var: RCLONE_PIXELDRAIN_API_KEY +- Type: string +- Required: false + +#### --pixeldrain-root-folder-id + +Root of the filesystem to use. + +Set to 'me' to use your personal filesystem. Set to a shared directory ID to use a shared directory. + +Properties: + +- Config: root_folder_id +- Env Var: RCLONE_PIXELDRAIN_ROOT_FOLDER_ID +- Type: string +- Default: "me" + +### Advanced options + +Here are the Advanced options specific to pixeldrain (Pixeldrain Filesystem). + +#### --pixeldrain-api-url + +The API endpoint to connect to. In the vast majority of cases it's fine to leave +this at default. It is only intended to be changed for testing purposes. + +Properties: + +- Config: api_url +- Env Var: RCLONE_PIXELDRAIN_API_URL +- Type: string +- Default: "https://pixeldrain.com/api" + +#### --pixeldrain-description + +Description of the remote. + +Properties: + +- Config: description +- Env Var: RCLONE_PIXELDRAIN_DESCRIPTION +- Type: string +- Required: false + +### Metadata + +Pixeldrain supports file modes and creation times. + +Here are the possible system metadata items for the pixeldrain backend. + +| Name | Help | Type | Example | Read Only | +|------|------|------|---------|-----------| +| btime | Time of file birth (creation) | RFC 3339 | 2006-01-02T15:04:05.999999999Z07:00 | N | +| mode | File mode | octal, unix style | 755 | N | +| mtime | Time of last modification | RFC 3339 | 2006-01-02T15:04:05.999999999Z07:00 | N | + +See the [metadata](/docs/#metadata) docs for more info. + +{{< rem autogenerated options stop >}} diff --git a/docs/layouts/chrome/navbar.html b/docs/layouts/chrome/navbar.html index 71d5174d1..1c67b098b 100644 --- a/docs/layouts/chrome/navbar.html +++ b/docs/layouts/chrome/navbar.html @@ -90,6 +90,7 @@ Oracle Object Storage pCloud PikPak + Pixeldrain premiumize.me put.io Proton Drive diff --git a/fstest/test_all/config.yaml b/fstest/test_all/config.yaml index 37a561b81..155f3093f 100644 --- a/fstest/test_all/config.yaml +++ b/fstest/test_all/config.yaml @@ -364,6 +364,11 @@ backends: # This test fails with message # "share_status_prohibited" (9): Sorry, the sharing service is under maintenance in the current region. - TestIntegration/FsMkdir/FsPutFiles/PublicLink + - backend: "pixeldrain" + remote: "TestPixeldrain:" + ignore: + - TestIntegration/FsMkdir/FsEncoding/invalid_UTF-8 + fastlist: false - backend: "webdav" remote: "TestWebdavNextcloud:" ignore: