From dde4dd0198a7dc3b60245d8866ecef7843dcfb41 Mon Sep 17 00:00:00 2001 From: Laura Date: Wed, 26 Jun 2019 20:39:01 +0200 Subject: [PATCH] fichier: 1fichier support - fixes #2908 This was started by Fionera, finished off by Laura with fixes and more docs from Nick. Co-authored-by: Fionera Co-authored-by: Nick Craig-Wood --- README.md | 1 + backend/all/all.go | 1 + backend/fichier/api.go | 381 ++++++++++++++++++++++++++++++ backend/fichier/fichier.go | 394 ++++++++++++++++++++++++++++++++ backend/fichier/fichier_test.go | 17 ++ backend/fichier/object.go | 158 +++++++++++++ backend/fichier/replace.go | 71 ++++++ backend/fichier/replace_test.go | 24 ++ backend/fichier/structs.go | 120 ++++++++++ bin/make_manual.py | 1 + cmd/serve/restic/restic-test.sh | 1 + docs/content/about.md | 1 + docs/content/docs.md | 1 + docs/content/fichier.md | 122 ++++++++++ docs/content/overview.md | 2 + docs/layouts/chrome/navbar.html | 1 + fs/hash/hash.go | 13 +- fs/hash/hash_test.go | 2 + fstest/test_all/config.yaml | 4 + 19 files changed, 1314 insertions(+), 1 deletion(-) create mode 100644 backend/fichier/api.go create mode 100644 backend/fichier/fichier.go create mode 100644 backend/fichier/fichier_test.go create mode 100644 backend/fichier/object.go create mode 100644 backend/fichier/replace.go create mode 100644 backend/fichier/replace_test.go create mode 100644 backend/fichier/structs.go create mode 100644 docs/content/fichier.md diff --git a/README.md b/README.md index 170fd3b13..b3338acfd 100644 --- a/README.md +++ b/README.md @@ -20,6 +20,7 @@ Rclone *("rsync for cloud storage")* is a command line program to sync files and ## Storage providers + * 1Fichier [:page_facing_up:](https://rclone.org/ficher/) * Alibaba Cloud (Aliyun) Object Storage System (OSS) [:page_facing_up:](https://rclone.org/s3/#alibaba-oss) * Amazon Drive [:page_facing_up:](https://rclone.org/amazonclouddrive/) ([See note](https://rclone.org/amazonclouddrive/#status)) * Amazon S3 [:page_facing_up:](https://rclone.org/s3/) diff --git a/backend/all/all.go b/backend/all/all.go index 3d21e13b3..5f4a0f086 100644 --- a/backend/all/all.go +++ b/backend/all/all.go @@ -11,6 +11,7 @@ import ( _ "github.com/ncw/rclone/backend/crypt" _ "github.com/ncw/rclone/backend/drive" _ "github.com/ncw/rclone/backend/dropbox" + _ "github.com/ncw/rclone/backend/fichier" _ "github.com/ncw/rclone/backend/ftp" _ "github.com/ncw/rclone/backend/googlecloudstorage" _ "github.com/ncw/rclone/backend/http" diff --git a/backend/fichier/api.go b/backend/fichier/api.go new file mode 100644 index 000000000..0f46b5b5c --- /dev/null +++ b/backend/fichier/api.go @@ -0,0 +1,381 @@ +package fichier + +import ( + "context" + "io" + "net/http" + "regexp" + "strconv" + "time" + + "github.com/ncw/rclone/fs" + "github.com/ncw/rclone/fs/fserrors" + "github.com/ncw/rclone/lib/rest" + "github.com/pkg/errors" +) + +// retryErrorCodes is a slice of error codes that we will retry +var retryErrorCodes = []int{ + 429, // Too Many Requests. + 500, // Internal Server Error + 502, // Bad Gateway + 503, // Service Unavailable + 504, // Gateway Timeout + 509, // Bandwidth Limit Exceeded +} + +// shouldRetry returns a boolean as to whether this resp and err +// deserve to be retried. It returns the err as a convenience +func shouldRetry(resp *http.Response, err error) (bool, error) { + return fserrors.ShouldRetry(err) || fserrors.ShouldRetryHTTP(resp, retryErrorCodes), err +} + +var isAlphaNumeric = regexp.MustCompile(`^[a-zA-Z0-9]+$`).MatchString + +func (f *Fs) getDownloadToken(url string) (*GetTokenResponse, error) { + request := DownloadRequest{ + URL: url, + Single: 1, + } + opts := rest.Opts{ + Method: "POST", + Path: "/download/get_token.cgi", + } + + var token GetTokenResponse + err := f.pacer.Call(func() (bool, error) { + resp, err := f.rest.CallJSON(&opts, &request, &token) + return shouldRetry(resp, err) + }) + if err != nil { + return nil, errors.Wrap(err, "couldn't list files") + } + + return &token, nil +} + +func fileFromSharedFile(file *SharedFile) File { + return File{ + URL: file.Link, + Filename: file.Filename, + Size: file.Size, + } +} + +func (f *Fs) listSharedFiles(ctx context.Context, id string) (entries fs.DirEntries, err error) { + opts := rest.Opts{ + Method: "GET", + RootURL: "https://1fichier.com/dir/", + Path: id, + Parameters: map[string][]string{"json": {"1"}}, + } + + var sharedFiles SharedFolderResponse + err = f.pacer.Call(func() (bool, error) { + resp, err := f.rest.CallJSON(&opts, nil, &sharedFiles) + return shouldRetry(resp, err) + }) + if err != nil { + return nil, errors.Wrap(err, "couldn't list files") + } + + entries = make([]fs.DirEntry, len(sharedFiles)) + + for i, sharedFile := range sharedFiles { + entries[i] = f.newObjectFromFile(ctx, "", fileFromSharedFile(&sharedFile)) + } + + return entries, nil +} + +func (f *Fs) listFiles(directoryID string) (filesList *FilesList, err error) { + // fs.Debugf(f, "Requesting files for dir `%s`", directoryID) + request := ListFilesRequest{ + FolderID: directoryID, + } + + opts := rest.Opts{ + Method: "POST", + Path: "/file/ls.cgi", + } + + filesList = &FilesList{} + err = f.pacer.Call(func() (bool, error) { + resp, err := f.rest.CallJSON(&opts, &request, filesList) + return shouldRetry(resp, err) + }) + if err != nil { + return nil, errors.Wrap(err, "couldn't list files") + } + + return filesList, nil +} + +func (f *Fs) listFolders(directoryID string) (foldersList *FoldersList, err error) { + // fs.Debugf(f, "Requesting folders for id `%s`", directoryID) + + request := ListFolderRequest{ + FolderID: directoryID, + } + + opts := rest.Opts{ + Method: "POST", + Path: "/folder/ls.cgi", + } + + foldersList = &FoldersList{} + err = f.pacer.Call(func() (bool, error) { + resp, err := f.rest.CallJSON(&opts, &request, foldersList) + return shouldRetry(resp, err) + }) + if err != nil { + return nil, errors.Wrap(err, "couldn't list folders") + } + + // fs.Debugf(f, "Got FoldersList for id `%s`", directoryID) + + return foldersList, err +} + +func (f *Fs) listDir(ctx context.Context, dir string) (entries fs.DirEntries, err error) { + err = f.dirCache.FindRoot(ctx, false) + if err != nil { + return nil, err + } + + directoryID, err := f.dirCache.FindDir(ctx, dir, false) + if err != nil { + return nil, err + } + + files, err := f.listFiles(directoryID) + if err != nil { + return nil, err + } + + folders, err := f.listFolders(directoryID) + if err != nil { + return nil, err + } + + entries = make([]fs.DirEntry, len(files.Items)+len(folders.SubFolders)) + + for i, item := range files.Items { + entries[i] = f.newObjectFromFile(ctx, dir, item) + } + + for i, folder := range folders.SubFolders { + createDate, err := time.Parse("2006-01-02 15:04:05", folder.CreateDate) + if err != nil { + return nil, err + } + + folder.Name = restoreReservedChars(folder.Name) + fullPath := getRemote(dir, folder.Name) + folderID := strconv.Itoa(folder.ID) + + entries[len(files.Items)+i] = fs.NewDir(fullPath, createDate).SetID(folderID) + + // fs.Debugf(f, "Put Path `%s` for id `%d` into dircache", fullPath, folder.ID) + f.dirCache.Put(fullPath, folderID) + } + + return entries, nil +} + +func (f *Fs) newObjectFromFile(ctx context.Context, dir string, item File) *Object { + return &Object{ + fs: f, + remote: getRemote(dir, item.Filename), + file: item, + } +} + +func getRemote(dir, fileName string) string { + if dir == "" { + return fileName + } + + return dir + "/" + fileName +} + +func (f *Fs) makeFolder(leaf, directoryID string) (response *MakeFolderResponse, err error) { + name := replaceReservedChars(leaf) + // fs.Debugf(f, "Creating folder `%s` in id `%s`", name, directoryID) + + request := MakeFolderRequest{ + FolderID: directoryID, + Name: name, + } + + opts := rest.Opts{ + Method: "POST", + Path: "/folder/mkdir.cgi", + } + + response = &MakeFolderResponse{} + err = f.pacer.Call(func() (bool, error) { + resp, err := f.rest.CallJSON(&opts, &request, response) + return shouldRetry(resp, err) + }) + if err != nil { + return nil, errors.Wrap(err, "couldn't create folder") + } + + // fs.Debugf(f, "Created Folder `%s` in id `%s`", name, directoryID) + + return response, err +} + +func (f *Fs) removeFolder(name, directoryID string) (response *GenericOKResponse, err error) { + // fs.Debugf(f, "Removing folder with id `%s`", directoryID) + + request := &RemoveFolderRequest{ + FolderID: directoryID, + } + + opts := rest.Opts{ + Method: "POST", + Path: "/folder/rm.cgi", + } + + response = &GenericOKResponse{} + var resp *http.Response + err = f.pacer.Call(func() (bool, error) { + resp, err = f.rest.CallJSON(&opts, request, response) + return shouldRetry(resp, err) + }) + if err != nil { + return nil, errors.Wrap(err, "couldn't remove folder") + } + if response.Status != "OK" { + return nil, errors.New("Can't remove non-empty dir") + } + + // fs.Debugf(f, "Removed Folder with id `%s`", directoryID) + + return response, nil +} + +func (f *Fs) deleteFile(url string) (response *GenericOKResponse, err error) { + request := &RemoveFileRequest{ + Files: []RmFile{ + {url}, + }, + } + + opts := rest.Opts{ + Method: "POST", + Path: "/file/rm.cgi", + } + + response = &GenericOKResponse{} + err = f.pacer.Call(func() (bool, error) { + resp, err := f.rest.CallJSON(&opts, request, response) + return shouldRetry(resp, err) + }) + + if err != nil { + return nil, errors.Wrap(err, "couldn't remove file") + } + + // fs.Debugf(f, "Removed file with url `%s`", url) + + return response, nil +} + +func (f *Fs) getUploadNode() (response *GetUploadNodeResponse, err error) { + // fs.Debugf(f, "Requesting Upload node") + + opts := rest.Opts{ + Method: "GET", + ContentType: "application/json", // 1Fichier API is bad + Path: "/upload/get_upload_server.cgi", + } + + response = &GetUploadNodeResponse{} + err = f.pacer.Call(func() (bool, error) { + resp, err := f.rest.CallJSON(&opts, nil, response) + return shouldRetry(resp, err) + }) + if err != nil { + return nil, errors.Wrap(err, "didnt got an upload node") + } + + // fs.Debugf(f, "Got Upload node") + + return response, err +} + +func (f *Fs) uploadFile(in io.Reader, size int64, fileName, folderID, uploadID, node string) (response *http.Response, err error) { + // fs.Debugf(f, "Uploading File `%s`", fileName) + + if len(uploadID) > 10 || !isAlphaNumeric(uploadID) { + return nil, errors.New("Invalid UploadID") + } + + opts := rest.Opts{ + Method: "POST", + Path: "/upload.cgi", + Parameters: map[string][]string{ + "id": {uploadID}, + }, + NoResponse: true, + Body: in, + ContentLength: &size, + MultipartContentName: "file[]", + MultipartFileName: fileName, + MultipartParams: map[string][]string{ + "did": {folderID}, + }, + } + + if node != "" { + opts.RootURL = "https://" + node + } + + err = f.pacer.CallNoRetry(func() (bool, error) { + resp, err := f.rest.CallJSON(&opts, nil, nil) + return shouldRetry(resp, err) + }) + + if err != nil { + return nil, errors.Wrap(err, "couldn't upload file") + } + + // fs.Debugf(f, "Uploaded File `%s`", fileName) + + return response, err +} + +func (f *Fs) endUpload(uploadID string, nodeurl string) (response *EndFileUploadResponse, err error) { + // fs.Debugf(f, "Ending File Upload `%s`", uploadID) + + if len(uploadID) > 10 || !isAlphaNumeric(uploadID) { + return nil, errors.New("Invalid UploadID") + } + + opts := rest.Opts{ + Method: "GET", + Path: "/end.pl", + RootURL: "https://" + nodeurl, + Parameters: map[string][]string{ + "xid": {uploadID}, + }, + ExtraHeaders: map[string]string{ + "JSON": "1", + }, + } + + response = &EndFileUploadResponse{} + err = f.pacer.Call(func() (bool, error) { + resp, err := f.rest.CallJSON(&opts, nil, response) + return shouldRetry(resp, err) + }) + + if err != nil { + return nil, errors.Wrap(err, "couldn't finish file upload") + } + + return response, err +} diff --git a/backend/fichier/fichier.go b/backend/fichier/fichier.go new file mode 100644 index 000000000..b474bcea1 --- /dev/null +++ b/backend/fichier/fichier.go @@ -0,0 +1,394 @@ +package fichier + +import ( + "context" + "fmt" + "io" + "net/http" + "strconv" + "strings" + "time" + + "github.com/ncw/rclone/fs" + "github.com/ncw/rclone/fs/config/configmap" + "github.com/ncw/rclone/fs/config/configstruct" + "github.com/ncw/rclone/fs/fshttp" + "github.com/ncw/rclone/fs/hash" + "github.com/ncw/rclone/lib/dircache" + "github.com/ncw/rclone/lib/pacer" + "github.com/ncw/rclone/lib/rest" + "github.com/pkg/errors" +) + +const ( + rootID = "0" + apiBaseURL = "https://api.1fichier.com/v1" + minSleep = 334 * time.Millisecond // 3 API calls per second is recommended + maxSleep = 5 * time.Second + decayConstant = 2 // bigger for slower decay, exponential +) + +func init() { + fs.Register(&fs.RegInfo{ + Name: "fichier", + Description: "1Fichier", + Config: func(name string, config configmap.Mapper) { + }, + NewFs: NewFs, + Options: []fs.Option{ + { + Help: "Your API Key, get it from https://1fichier.com/console/params.pl", + Name: "api_key", + }, + { + Help: "If you want to download a shared folder, add this parameter", + Name: "shared_folder", + Required: false, + Advanced: true, + }, + }, + }) +} + +// Options defines the configuration for this backend +type Options struct { + APIKey string `config:"api_key"` + SharedFolder string `config:"shared_folder"` +} + +// Fs is the interface a cloud storage system must provide +type Fs struct { + root string + name string + features *fs.Features + dirCache *dircache.DirCache + baseClient *http.Client + options *Options + pacer *fs.Pacer + rest *rest.Client +} + +// FindLeaf finds a directory of name leaf in the folder with ID pathID +func (f *Fs) FindLeaf(ctx context.Context, pathID, leaf string) (pathIDOut string, found bool, err error) { + folders, err := f.listFolders(pathID) + if err != nil { + return "", false, err + } + + for _, folder := range folders.SubFolders { + if folder.Name == leaf { + pathIDOut := strconv.Itoa(folder.ID) + return pathIDOut, true, nil + } + } + + return "", false, nil +} + +// CreateDir makes a directory with pathID as parent and name leaf +func (f *Fs) CreateDir(ctx context.Context, pathID, leaf string) (newID string, err error) { + resp, err := f.makeFolder(leaf, pathID) + if err != nil { + return "", err + } + return strconv.Itoa(resp.FolderID), err +} + +// Name of the remote (as passed into NewFs) +func (f *Fs) Name() string { + return f.name +} + +// Root of the remote (as passed into NewFs) +func (f *Fs) Root() string { + return f.root +} + +// String returns a description of the FS +func (f *Fs) String() string { + return fmt.Sprintf("1Fichier root '%s'", f.root) +} + +// Precision of the ModTimes in this Fs +func (f *Fs) Precision() time.Duration { + return fs.ModTimeNotSupported +} + +// Hashes returns the supported hash types of the filesystem +func (f *Fs) Hashes() hash.Set { + return hash.Set(hash.Whirlpool) +} + +// Features returns the optional features of this Fs +func (f *Fs) Features() *fs.Features { + return f.features +} + +// NewFs makes a new Fs object from the path +// +// The path is of the form remote:path +// +// Remotes are looked up in the config file. If the remote isn't +// found then NotFoundInConfigFile will be returned. +// +// On Windows avoid single character remote names as they can be mixed +// up with drive letters. +func NewFs(name string, rootleaf string, config configmap.Mapper) (fs.Fs, error) { + root := replaceReservedChars(rootleaf) + opt := new(Options) + err := configstruct.Set(config, opt) + if err != nil { + return nil, err + } + + // If using a Shared Folder override root + if opt.SharedFolder != "" { + root = "" + } + + //workaround for wonky parser + root = strings.Trim(root, "/") + + f := &Fs{ + name: name, + root: root, + options: opt, + pacer: fs.NewPacer(pacer.NewDefault(pacer.MinSleep(minSleep), pacer.MaxSleep(maxSleep), pacer.DecayConstant(decayConstant))), + baseClient: &http.Client{}, + } + + f.features = (&fs.Features{ + DuplicateFiles: true, + CanHaveEmptyDirectories: true, + }).Fill(f) + + client := fshttp.NewClient(fs.Config) + + f.rest = rest.NewClient(client).SetRoot(apiBaseURL) + + f.rest.SetHeader("Authorization", "Bearer "+f.options.APIKey) + + f.dirCache = dircache.New(root, rootID, f) + + ctx := context.Background() + + // Find the current root + err = f.dirCache.FindRoot(ctx, false) + if err != nil { + // Assume it is a file + newRoot, remote := dircache.SplitPath(root) + tempF := *f + tempF.dirCache = dircache.New(newRoot, rootID, &tempF) + tempF.root = newRoot + // Make new Fs which is the parent + err = tempF.dirCache.FindRoot(ctx, false) + if err != nil { + // No root so return old f + return f, nil + } + _, err := tempF.NewObject(ctx, remote) + if err != nil { + if err == fs.ErrorObjectNotFound { + // File doesn't exist so return old f + return f, nil + } + return nil, err + } + f.features.Fill(&tempF) + // XXX: update the old f here instead of returning tempF, since + // `features` were already filled with functions having *f as a receiver. + // See https://github.com/ncw/rclone/issues/2182 + f.dirCache = tempF.dirCache + f.root = tempF.root + // return an error with an fs which points to the parent + return f, fs.ErrorIsFile + } + return f, nil +} + +// List the objects and directories in dir into entries. The +// entries can be returned in any order but should be for a +// complete directory. +// +// dir should be "" to list the root, and should not have +// trailing slashes. +// +// This should return ErrDirNotFound if the directory isn't +// found. +func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err error) { + if f.options.SharedFolder != "" { + return f.listSharedFiles(ctx, f.options.SharedFolder) + } + + dirContent, err := f.listDir(ctx, dir) + if err != nil { + return nil, err + } + + return dirContent, nil +} + +// NewObject finds the Object at remote. If it can't be found +// it returns the error ErrorObjectNotFound. +func (f *Fs) NewObject(ctx context.Context, remote string) (fs.Object, error) { + leaf, directoryID, err := f.dirCache.FindRootAndPath(ctx, remote, false) + if err != nil { + if err == fs.ErrorDirNotFound { + return nil, fs.ErrorObjectNotFound + } + return nil, err + } + + files, err := f.listFiles(directoryID) + if err != nil { + return nil, err + } + + for _, file := range files.Items { + if file.Filename == leaf { + path, ok := f.dirCache.GetInv(directoryID) + + if !ok { + return nil, errors.New("Cannot find dir in dircache") + } + + return f.newObjectFromFile(ctx, path, file), nil + } + } + + return nil, fs.ErrorObjectNotFound +} + +// Put in to the remote path with the modTime given of the given size +// +// When called from outside a Fs by rclone, src.Size() will always be >= 0. +// But for unknown-sized objects (indicated by src.Size() == -1), Put should either +// return an error or upload it properly (rather than e.g. calling panic). +// +// May create the object even if it returns an error - if so +// will return the object and the error, otherwise will return +// nil and the error +func (f *Fs) Put(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) { + exisitingObj, err := f.NewObject(ctx, src.Remote()) + switch err { + case nil: + return exisitingObj, exisitingObj.Update(ctx, in, src, options...) + case fs.ErrorObjectNotFound: + // Not found so create it + return f.PutUnchecked(ctx, in, src, options...) + default: + return nil, err + } +} + +// putUnchecked uploads the object with the given name and size +// +// This will create a duplicate if we upload a new file without +// checking to see if there is one already - use Put() for that. +func (f *Fs) putUnchecked(ctx context.Context, in io.Reader, remote string, size int64, options ...fs.OpenOption) (fs.Object, error) { + if size > int64(100E9) { + return nil, errors.New("File too big, cant upload") + } else if size == 0 { + return nil, fs.ErrorCantUploadEmptyFiles + } + + nodeResponse, err := f.getUploadNode() + if err != nil { + return nil, err + } + + leaf, directoryID, err := f.dirCache.FindRootAndPath(ctx, remote, true) + if err != nil { + return nil, err + } + + _, err = f.uploadFile(in, size, leaf, directoryID, nodeResponse.ID, nodeResponse.URL) + if err != nil { + return nil, err + } + + fileUploadResponse, err := f.endUpload(nodeResponse.ID, nodeResponse.URL) + if err != nil { + return nil, err + } + + if len(fileUploadResponse.Links) != 1 { + return nil, errors.New("unexpected amount of files") + } + + link := fileUploadResponse.Links[0] + fileSize, err := strconv.ParseInt(link.Size, 10, 64) + + if err != nil { + return nil, err + } + + return &Object{ + fs: f, + remote: remote, + file: File{ + ACL: 0, + CDN: 0, + Checksum: link.Whirlpool, + ContentType: "", + Date: time.Now().Format("2006-01-02 15:04:05"), + Filename: link.Filename, + Pass: 0, + Size: int(fileSize), + URL: link.Download, + }, + }, nil +} + +// PutUnchecked uploads the object +// +// This will create a duplicate if we upload a new file without +// checking to see if there is one already - use Put() for that. +func (f *Fs) PutUnchecked(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) { + return f.putUnchecked(ctx, in, src.Remote(), src.Size(), options...) +} + +// Mkdir makes the directory (container, bucket) +// +// Shouldn't return an error if it already exists +func (f *Fs) Mkdir(ctx context.Context, dir string) error { + err := f.dirCache.FindRoot(ctx, true) + if err != nil { + return err + } + if dir != "" { + _, err = f.dirCache.FindDir(ctx, dir, true) + } + return err +} + +// Rmdir removes the directory (container, bucket) if empty +// +// Return an error if it doesn't exist or isn't empty +func (f *Fs) Rmdir(ctx context.Context, dir string) error { + err := f.dirCache.FindRoot(ctx, false) + if err != nil { + return err + } + + did, err := f.dirCache.FindDir(ctx, dir, false) + if err != nil { + return err + } + + _, err = f.removeFolder(dir, did) + if err != nil { + return err + } + + f.dirCache.FlushDir(dir) + + return nil +} + +// Check the interfaces are satisfied +var ( + _ fs.Fs = (*Fs)(nil) + _ fs.PutUncheckeder = (*Fs)(nil) + _ dircache.DirCacher = (*Fs)(nil) +) diff --git a/backend/fichier/fichier_test.go b/backend/fichier/fichier_test.go new file mode 100644 index 000000000..3f112bfd6 --- /dev/null +++ b/backend/fichier/fichier_test.go @@ -0,0 +1,17 @@ +// Test 1Fichier filesystem interface +package fichier + +import ( + "testing" + + "github.com/ncw/rclone/fs" + "github.com/ncw/rclone/fstest/fstests" +) + +// TestIntegration runs integration tests against the remote +func TestIntegration(t *testing.T) { + fs.Config.LogLevel = fs.LogLevelDebug + fstests.Run(t, &fstests.Opt{ + RemoteName: "TestFichier:", + }) +} diff --git a/backend/fichier/object.go b/backend/fichier/object.go new file mode 100644 index 000000000..fb43541a4 --- /dev/null +++ b/backend/fichier/object.go @@ -0,0 +1,158 @@ +package fichier + +import ( + "context" + "io" + "net/http" + "time" + + "github.com/ncw/rclone/fs" + "github.com/ncw/rclone/fs/hash" + "github.com/ncw/rclone/lib/rest" + "github.com/pkg/errors" +) + +// Object is a filesystem like object provided by an Fs +type Object struct { + fs *Fs + remote string + file File +} + +// String returns a description of the Object +func (o *Object) String() string { + return o.file.Filename +} + +// Remote returns the remote path +func (o *Object) Remote() string { + return o.remote +} + +// ModTime returns the modification date of the file +// It should return a best guess if one isn't available +func (o *Object) ModTime(ctx context.Context) time.Time { + modTime, err := time.Parse("2006-01-02 15:04:05", o.file.Date) + + if err != nil { + return time.Now() + } + + return modTime +} + +// Size returns the size of the file +func (o *Object) Size() int64 { + return int64(o.file.Size) +} + +// Fs returns read only access to the Fs that this object is part of +func (o *Object) Fs() fs.Info { + return o.fs +} + +// Hash returns the selected checksum of the file +// If no checksum is available it returns "" +func (o *Object) Hash(ctx context.Context, t hash.Type) (string, error) { + if t != hash.Whirlpool { + return "", hash.ErrUnsupported + } + + return o.file.Checksum, nil +} + +// Storable says whether this object can be stored +func (o *Object) Storable() bool { + return true +} + +// SetModTime sets the metadata on the object to set the modification date +func (o *Object) SetModTime(context.Context, time.Time) error { + return fs.ErrorCantSetModTime + //return errors.New("setting modtime is not supported for 1fichier remotes") +} + +// Open opens the file for read. Call Close() on the returned io.ReadCloser +func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (io.ReadCloser, error) { + fs.FixRangeOption(options, int64(o.file.Size)) + downloadToken, err := o.fs.getDownloadToken(o.file.URL) + + if err != nil { + return nil, err + } + + var resp *http.Response + opts := rest.Opts{ + Method: "GET", + RootURL: downloadToken.URL, + Options: options, + } + + err = o.fs.pacer.Call(func() (bool, error) { + resp, err = o.fs.rest.Call(&opts) + return shouldRetry(resp, err) + }) + + if err != nil { + return nil, err + } + return resp.Body, err +} + +// Update in to the object with the modTime given of the given size +// +// When called from outside a Fs by rclone, src.Size() will always be >= 0. +// But for unknown-sized objects (indicated by src.Size() == -1), Upload should either +// return an error or update the object properly (rather than e.g. calling panic). +func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) error { + if src.Size() < 0 { + return errors.New("refusing to update with unknown size") + } + + // upload with new size but old name + info, err := o.fs.putUnchecked(ctx, in, o.Remote(), src.Size(), options...) + if err != nil { + return err + } + + // Delete duplicate after successful upload + err = o.Remove(ctx) + if err != nil { + return errors.Wrap(err, "failed to remove old version") + } + + // Replace guts of old object with new one + *o = *info.(*Object) + + return nil +} + +// Remove removes this object +func (o *Object) Remove(ctx context.Context) error { + // fs.Debugf(f, "Removing file `%s` with url `%s`", o.file.Filename, o.file.URL) + + _, err := o.fs.deleteFile(o.file.URL) + + if err != nil { + return err + } + + return nil +} + +// MimeType of an Object if known, "" otherwise +func (o *Object) MimeType(ctx context.Context) string { + return o.file.ContentType +} + +// ID returns the ID of the Object if known, or "" if not +func (o *Object) ID() string { + return o.file.URL +} + +// Check the interfaces are satisfied +var ( + _ fs.Object = (*Object)(nil) + _ fs.MimeTyper = (*Object)(nil) + _ fs.IDer = (*Object)(nil) +) diff --git a/backend/fichier/replace.go b/backend/fichier/replace.go new file mode 100644 index 000000000..e5db38af8 --- /dev/null +++ b/backend/fichier/replace.go @@ -0,0 +1,71 @@ +/* +Translate file names for 1fichier + +1Fichier reserved characters + +The following characters are 1Fichier reserved characters, and can't +be used in 1Fichier folder and file names. + +*/ + +package fichier + +import ( + "regexp" + "strings" +) + +// charMap holds replacements for characters +// +// 1Fichier has a restricted set of characters compared to other cloud +// storage systems, so we to map these to the FULLWIDTH unicode +// equivalents +// +// http://unicode-search.net/unicode-namesearch.pl?term=SOLIDUS +var ( + charMap = map[rune]rune{ + '\\': '＼', // FULLWIDTH REVERSE SOLIDUS + '<': '＜', // FULLWIDTH LESS-THAN SIGN + '>': '＞', // FULLWIDTH GREATER-THAN SIGN + '"': '＂', // FULLWIDTH QUOTATION MARK - not on the list but seems to be reserved + '\'': '＇', // FULLWIDTH APOSTROPHE + '$': '＄', // FULLWIDTH DOLLAR SIGN + '`': '｀', // FULLWIDTH GRAVE ACCENT + ' ': '␠', // SYMBOL FOR SPACE + } + invCharMap map[rune]rune + fixStartingWithSpace = regexp.MustCompile(`(/|^) `) +) + +func init() { + // Create inverse charMap + invCharMap = make(map[rune]rune, len(charMap)) + for k, v := range charMap { + invCharMap[v] = k + } +} + +// replaceReservedChars takes a path and substitutes any reserved +// characters in it +func replaceReservedChars(in string) string { + // file names can't start with space either + in = fixStartingWithSpace.ReplaceAllString(in, "$1"+string(charMap[' '])) + // Replace reserved characters + return strings.Map(func(c rune) rune { + if replacement, ok := charMap[c]; ok && c != ' ' { + return replacement + } + return c + }, in) +} + +// restoreReservedChars takes a path and undoes any substitutions +// made by replaceReservedChars +func restoreReservedChars(in string) string { + return strings.Map(func(c rune) rune { + if replacement, ok := invCharMap[c]; ok { + return replacement + } + return c + }, in) +} diff --git a/backend/fichier/replace_test.go b/backend/fichier/replace_test.go new file mode 100644 index 000000000..40a5eebef --- /dev/null +++ b/backend/fichier/replace_test.go @@ -0,0 +1,24 @@ +package fichier + +import "testing" + +func TestReplace(t *testing.T) { + for _, test := range []struct { + in string + out string + }{ + {"", ""}, + {"abc 123", "abc 123"}, + {"\"'<>/\\$`", `＂＇＜＞/＼＄｀`}, + {" leading space", "␠leading space"}, + } { + got := replaceReservedChars(test.in) + if got != test.out { + t.Errorf("replaceReservedChars(%q) want %q got %q", test.in, test.out, got) + } + got2 := restoreReservedChars(got) + if got2 != test.in { + t.Errorf("restoreReservedChars(%q) want %q got %q", got, test.in, got2) + } + } +} diff --git a/backend/fichier/structs.go b/backend/fichier/structs.go new file mode 100644 index 000000000..ef08b906c --- /dev/null +++ b/backend/fichier/structs.go @@ -0,0 +1,120 @@ +package fichier + +// ListFolderRequest is the request structure of the corresponding request +type ListFolderRequest struct { + FolderID string `json:"folder_id"` +} + +// ListFilesRequest is the request structure of the corresponding request +type ListFilesRequest struct { + FolderID string `json:"folder_id"` +} + +// DownloadRequest is the request structure of the corresponding request +type DownloadRequest struct { + URL string `json:"url"` + Single int `json:"single"` +} + +// RemoveFolderRequest is the request structure of the corresponding request +type RemoveFolderRequest struct { + FolderID string `json:"folder_id"` +} + +// RemoveFileRequest is the request structure of the corresponding request +type RemoveFileRequest struct { + Files []RmFile `json:"files"` +} + +// RmFile is the request structure of the corresponding request +type RmFile struct { + URL string `json:"url"` +} + +// GenericOKResponse is the response structure of the corresponding request +type GenericOKResponse struct { + Status string `json:"status"` + Message string `json:"message"` +} + +// MakeFolderRequest is the request structure of the corresponding request +type MakeFolderRequest struct { + Name string `json:"name"` + FolderID string `json:"folder_id"` +} + +// MakeFolderResponse is the response structure of the corresponding request +type MakeFolderResponse struct { + Name string `json:"name"` + FolderID int `json:"folder_id"` +} + +// GetUploadNodeResponse is the response structure of the corresponding request +type GetUploadNodeResponse struct { + ID string `json:"id"` + URL string `json:"url"` +} + +// GetTokenResponse is the response structure of the corresponding request +type GetTokenResponse struct { + URL string `json:"url"` + Status string `json:"Status"` + Message string `json:"Message"` +} + +// SharedFolderResponse is the response structure of the corresponding request +type SharedFolderResponse []SharedFile + +// SharedFile is the structure how 1Fichier returns a shared File +type SharedFile struct { + Filename string `json:"filename"` + Link string `json:"link"` + Size int `json:"size"` +} + +// EndFileUploadResponse is the response structure of the corresponding request +type EndFileUploadResponse struct { + Incoming int `json:"incoming"` + Links []struct { + Download string `json:"download"` + Filename string `json:"filename"` + Remove string `json:"remove"` + Size string `json:"size"` + Whirlpool string `json:"whirlpool"` + } `json:"links"` +} + +// File is the structure how 1Fichier returns a File +type File struct { + ACL int `json:"acl"` + CDN int `json:"cdn"` + Checksum string `json:"checksum"` + ContentType string `json:"content-type"` + Date string `json:"date"` + Filename string `json:"filename"` + Pass int `json:"pass"` + Size int `json:"size"` + URL string `json:"url"` +} + +// FilesList is the structure how 1Fichier returns a list of files +type FilesList struct { + Items []File `json:"items"` + Status string `json:"Status"` +} + +// Folder is the structure how 1Fichier returns a Folder +type Folder struct { + CreateDate string `json:"create_date"` + ID int `json:"id"` + Name string `json:"name"` + Pass string `json:"pass"` +} + +// FoldersList is the structure how 1Fichier returns a list of Folders +type FoldersList struct { + FolderID string `json:"folder_id"` + Name string `json:"name"` + Status string `json:"Status"` + SubFolders []Folder `json:"sub_folders"` +} diff --git a/bin/make_manual.py b/bin/make_manual.py index e03b03412..c10725c00 100755 --- a/bin/make_manual.py +++ b/bin/make_manual.py @@ -23,6 +23,7 @@ docs = [ "flags.md", # Keep these alphabetical by full name + "fichier.md", "alias.md", "amazonclouddrive.md", "s3.md", diff --git a/cmd/serve/restic/restic-test.sh b/cmd/serve/restic/restic-test.sh index 88328bf8d..7062fac3b 100755 --- a/cmd/serve/restic/restic-test.sh +++ b/cmd/serve/restic/restic-test.sh @@ -12,6 +12,7 @@ TestCryptDrive: TestCryptSwift: TestDrive: TestDropbox: +TestFichier: TestFTP: TestGoogleCloudStorage: TestHubic: diff --git a/docs/content/about.md b/docs/content/about.md index e49254e73..e4fe2aa98 100644 --- a/docs/content/about.md +++ b/docs/content/about.md @@ -13,6 +13,7 @@ Rclone Rclone is a command line program to sync files and directories to and from: +* {{< provider name="1Fichier" home="https://1fichier.com/" config="/fichier/" >}} * {{< provider name="Alibaba Cloud (Aliyun) Object Storage System (OSS)" home="https://www.alibabacloud.com/product/oss/" config="/s3/#alibaba-oss" >}} * {{< provider name="Amazon Drive" home="https://www.amazon.com/clouddrive" config="/amazonclouddrive/" >}} ([See note](/amazonclouddrive/#status)) * {{< provider name="Amazon S3" home="https://aws.amazon.com/s3/" config="/s3/" >}} diff --git a/docs/content/docs.md b/docs/content/docs.md index f6b9aedd2..79e782192 100644 --- a/docs/content/docs.md +++ b/docs/content/docs.md @@ -19,6 +19,7 @@ option: See the following for detailed instructions for + * [1Fichier](/fichier/) * [Alias](/alias/) * [Amazon Drive](/amazonclouddrive/) * [Amazon S3](/s3/) diff --git a/docs/content/fichier.md b/docs/content/fichier.md new file mode 100644 index 000000000..45db7601e --- /dev/null +++ b/docs/content/fichier.md @@ -0,0 +1,122 @@ +--- +title: "1Fichier" +description: "Rclone docs for 1Fichier" +date: "2015-10-14" +--- + + 1Fichier +----------------------------------------- + +This is a backend for the [1ficher](https://1fichier.com) cloud +storage service. Note that a Premium subscription is required to use +the API. + +Paths are specified as `remote:path` + +Paths may be as deep as required, eg `remote:directory/subdirectory`. + +The initial setup for 1Fichier involves getting the API key from the website which you +need to do in your browser. + +Here is an example of how to make a remote called `remote`. First run: + + rclone config + +This will guide you through an interactive setup process: + +``` +No remotes found - make a new one +n) New remote +s) Set configuration password +q) Quit config +n/s/q> n +name> remote +Type of storage to configure. +Enter a string value. Press Enter for the default (""). +Choose a number from below, or type in your own value + 1 / 1Fichier + \ "fichier" +... +Storage> fichier +** See help for fichier backend at: https://rclone.org/fichier/ ** + +Your API Key, get it from https://1fichier.com/console/params.pl +Enter a string value. Press Enter for the default (""). +api_key> example_key + +Edit advanced config? (y/n) +y) Yes +n) No +y/n> +Remote config +-------------------- +[remote] +type = fichier +api_key = example_key +-------------------- +y) Yes this is OK +e) Edit this remote +d) Delete this remote +y/e/d> y +``` + +Once configured you can then use `rclone` like this, + +List directories in top level of your 1Fichier account + + rclone lsd remote: + +List all the files in your 1Fichier account + + rclone ls remote: + +To copy a local directory to a 1Fichier directory called backup + + rclone copy /home/source remote:backup + +### Modified time and hashes ### + +1Fichier does not support modification times. It supports the Whirlpool hash algorithm. + +### Duplicated files ### + +1Fichier can have two files with exactly the same name and path (unlike a +normal file system). + +Duplicated files cause problems with the syncing and you will see +messages in the log about duplicates. + +### Forbidden characters ### + +1Fichier does not support the characters ``\ < > " ' ` $`` and spaces at the beginning of folder names. +`rclone` automatically escapes these to a unicode equivalent. The exception is `/`, +which cannot be escaped and will therefore lead to errors. + + +### Standard Options + +Here are the standard options specific to fichier (1Fichier). + +#### --fichier-api-key + +Your API Key, get it from https://1fichier.com/console/params.pl + +- Config: api_key +- Env Var: RCLONE_FICHIER_API_KEY +- Type: string +- Default: "" + +### Advanced Options + +Here are the advanced options specific to fichier (1Fichier). + +#### --fichier-shared-folder + +If you want to download a shared folder, add this parameter + +- Config: shared_folder +- Env Var: RCLONE_FICHIER_SHARED_FOLDER +- Type: string +- Default: "" + + diff --git a/docs/content/overview.md b/docs/content/overview.md index 803a97f76..067df357d 100644 --- a/docs/content/overview.md +++ b/docs/content/overview.md @@ -17,6 +17,7 @@ Here is an overview of the major features of each cloud storage system. | Name | Hash | ModTime | Case Insensitive | Duplicate Files | MIME Type | | ---------------------------- |:-----------:|:-------:|:----------------:|:---------------:|:---------:| +| 1Fichier | Whirlpool | No | No | Yes | R | | Amazon Drive | MD5 | No | Yes | No | R | | Amazon S3 | MD5 | Yes | No | No | R/W | | Backblaze B2 | SHA1 | Yes | No | No | R/W | @@ -131,6 +132,7 @@ operations more efficient. | Name | Purge | Copy | Move | DirMove | CleanUp | ListR | StreamUpload | LinkSharing | About | | ---------------------------- |:-----:|:----:|:----:|:-------:|:-------:|:-----:|:------------:|:------------:|:-----:| +| 1Fichier | No | No | No | No | No | No | No | No | No | | Amazon Drive | Yes | No | Yes | Yes | No [#575](https://github.com/ncw/rclone/issues/575) | No | No | No [#2178](https://github.com/ncw/rclone/issues/2178) | No | | Amazon S3 | No | Yes | No | No | No | Yes | Yes | No [#2178](https://github.com/ncw/rclone/issues/2178) | No | | Backblaze B2 | No | Yes | No | No | Yes | Yes | Yes | No [#2178](https://github.com/ncw/rclone/issues/2178) | No | diff --git a/docs/layouts/chrome/navbar.html b/docs/layouts/chrome/navbar.html index 55aa94076..8075053a8 100644 --- a/docs/layouts/chrome/navbar.html +++ b/docs/layouts/chrome/navbar.html @@ -56,6 +56,7 @@ Storage Systems
  • Overview
    • +
  • 1Fichier
  • Amazon Drive
  • Amazon S3
  • Backblaze B2
    • diff --git a/fs/hash/hash.go b/fs/hash/hash.go index 005933911..c14ce4884 100644 --- a/fs/hash/hash.go +++ b/fs/hash/hash.go @@ -9,6 +9,7 @@ import ( "io" "strings" + "github.com/jzelinskie/whirlpool" "github.com/ncw/rclone/backend/dropbox/dbhash" "github.com/ncw/rclone/backend/onedrive/quickxorhash" "github.com/pkg/errors" @@ -36,13 +37,16 @@ const ( // https://docs.microsoft.com/en-us/onedrive/developer/code-snippets/quickxorhash QuickXorHash + // Whirlpool indicates Whirlpool support + Whirlpool + // None indicates no hashes are supported None Type = 0 ) // Supported returns a set of all the supported hashes by // HashStream and MultiHasher. -var Supported = NewHashSet(MD5, SHA1, Dropbox, QuickXorHash) +var Supported = NewHashSet(MD5, SHA1, Dropbox, QuickXorHash, Whirlpool) // Width returns the width in characters for any HashType var Width = map[Type]int{ @@ -50,6 +54,7 @@ var Width = map[Type]int{ SHA1: 40, Dropbox: 64, QuickXorHash: 40, + Whirlpool: 128, } // Stream will calculate hashes of all supported hash types. @@ -89,6 +94,8 @@ func (h Type) String() string { return "DropboxHash" case QuickXorHash: return "QuickXorHash" + case Whirlpool: + return "Whirlpool" default: err := fmt.Sprintf("internal error: unknown hash type: 0x%x", int(h)) panic(err) @@ -108,6 +115,8 @@ func (h *Type) Set(s string) error { *h = Dropbox case "QuickXorHash": *h = QuickXorHash + case "Whirlpool": + *h = Whirlpool default: return errors.Errorf("Unknown hash type %q", s) } @@ -138,6 +147,8 @@ func fromTypes(set Set) (map[Type]hash.Hash, error) { hashers[t] = dbhash.New() case QuickXorHash: hashers[t] = quickxorhash.New() + case Whirlpool: + hashers[t] = whirlpool.New() default: err := fmt.Sprintf("internal error: Unsupported hash type %v", t) panic(err) diff --git a/fs/hash/hash_test.go b/fs/hash/hash_test.go index 92ef91ab1..03d7295d4 100644 --- a/fs/hash/hash_test.go +++ b/fs/hash/hash_test.go @@ -73,6 +73,7 @@ var hashTestSet = []hashTest{ hash.SHA1: "3ab6543c08a75f292a5ecedac87ec41642d12166", hash.Dropbox: "214d2fcf3566e94c99ad2f59bd993daca46d8521a0c447adf4b324f53fddc0c7", hash.QuickXorHash: "0110c000085000031c0001095ec00218d0000700", + hash.Whirlpool: "eddf52133d4566d763f716e853d6e4efbabd29e2c2e63f56747b1596172851d34c2df9944beb6640dbdbe3d9b4eb61180720a79e3d15baff31c91e43d63869a4", }, }, // Empty data set @@ -83,6 +84,7 @@ var hashTestSet = []hashTest{ hash.SHA1: "da39a3ee5e6b4b0d3255bfef95601890afd80709", hash.Dropbox: "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", hash.QuickXorHash: "0000000000000000000000000000000000000000", + hash.Whirlpool: "19fa61d75522a4669b44e39c1d2e1726c530232130d407f89afee0964997f7a73e83be698b288febcf88e3e03c4f0757ea8964e59b63d93708b138cc42a66eb3", }, }, } diff --git a/fstest/test_all/config.yaml b/fstest/test_all/config.yaml index d5909e93a..3efb5f695 100644 --- a/fstest/test_all/config.yaml +++ b/fstest/test_all/config.yaml @@ -110,6 +110,10 @@ backends: remote: "TestBox:" subdir: false fastlist: false + - backend: "fichier" + remote: "TestFichier:" + subdir: false + fastlist: false - backend: "qingstor" remote: "TestQingStor:" subdir: false