fichier: 1fichier support - fixes #2908

This was started by Fionera, finished off by Laura with fixes and more
docs from Nick.

Co-authored-by: Fionera <fionera@fionera.de>
Co-authored-by: Nick Craig-Wood <nick@craig-wood.com>

README.md

@@ -20,6 +20,7 @@ Rclone *("rsync for cloud storage")* is a command line program to sync files and
 
 ## Storage providers
 
+  * 1Fichier [:page_facing_up:](https://rclone.org/ficher/)
   * Alibaba Cloud (Aliyun) Object Storage System (OSS) [:page_facing_up:](https://rclone.org/s3/#alibaba-oss)
   * Amazon Drive [:page_facing_up:](https://rclone.org/amazonclouddrive/) ([See note](https://rclone.org/amazonclouddrive/#status))
   * Amazon S3 [:page_facing_up:](https://rclone.org/s3/)

backend/all/all.go

@@ -11,6 +11,7 @@ import (
 	_ "github.com/ncw/rclone/backend/crypt"
 	_ "github.com/ncw/rclone/backend/drive"
 	_ "github.com/ncw/rclone/backend/dropbox"
+	_ "github.com/ncw/rclone/backend/fichier"
 	_ "github.com/ncw/rclone/backend/ftp"
 	_ "github.com/ncw/rclone/backend/googlecloudstorage"
 	_ "github.com/ncw/rclone/backend/http"

backend/fichier/api.go

@@ -0,0 +1,381 @@
+package fichier
+
+import (
+	"context"
+	"io"
+	"net/http"
+	"regexp"
+	"strconv"
+	"time"
+
+	"github.com/ncw/rclone/fs"
+	"github.com/ncw/rclone/fs/fserrors"
+	"github.com/ncw/rclone/lib/rest"
+	"github.com/pkg/errors"
+)
+
+// retryErrorCodes is a slice of error codes that we will retry
+var retryErrorCodes = []int{
+	429, // Too Many Requests.
+	500, // Internal Server Error
+	502, // Bad Gateway
+	503, // Service Unavailable
+	504, // Gateway Timeout
+	509, // Bandwidth Limit Exceeded
+}
+
+// shouldRetry returns a boolean as to whether this resp and err
+// deserve to be retried.  It returns the err as a convenience
+func shouldRetry(resp *http.Response, err error) (bool, error) {
+	return fserrors.ShouldRetry(err) || fserrors.ShouldRetryHTTP(resp, retryErrorCodes), err
+}
+
+var isAlphaNumeric = regexp.MustCompile(`^[a-zA-Z0-9]+$`).MatchString
+
+func (f *Fs) getDownloadToken(url string) (*GetTokenResponse, error) {
+	request := DownloadRequest{
+		URL:    url,
+		Single: 1,
+	}
+	opts := rest.Opts{
+		Method: "POST",
+		Path:   "/download/get_token.cgi",
+	}
+
+	var token GetTokenResponse
+	err := f.pacer.Call(func() (bool, error) {
+		resp, err := f.rest.CallJSON(&opts, &request, &token)
+		return shouldRetry(resp, err)
+	})
+	if err != nil {
+		return nil, errors.Wrap(err, "couldn't list files")
+	}
+
+	return &token, nil
+}
+
+func fileFromSharedFile(file *SharedFile) File {
+	return File{
+		URL:      file.Link,
+		Filename: file.Filename,
+		Size:     file.Size,
+	}
+}
+
+func (f *Fs) listSharedFiles(ctx context.Context, id string) (entries fs.DirEntries, err error) {
+	opts := rest.Opts{
+		Method:     "GET",
+		RootURL:    "https://1fichier.com/dir/",
+		Path:       id,
+		Parameters: map[string][]string{"json": {"1"}},
+	}
+
+	var sharedFiles SharedFolderResponse
+	err = f.pacer.Call(func() (bool, error) {
+		resp, err := f.rest.CallJSON(&opts, nil, &sharedFiles)
+		return shouldRetry(resp, err)
+	})
+	if err != nil {
+		return nil, errors.Wrap(err, "couldn't list files")
+	}
+
+	entries = make([]fs.DirEntry, len(sharedFiles))
+
+	for i, sharedFile := range sharedFiles {
+		entries[i] = f.newObjectFromFile(ctx, "", fileFromSharedFile(&sharedFile))
+	}
+
+	return entries, nil
+}
+
+func (f *Fs) listFiles(directoryID string) (filesList *FilesList, err error) {
+	// fs.Debugf(f, "Requesting files for dir `%s`", directoryID)
+	request := ListFilesRequest{
+		FolderID: directoryID,
+	}
+
+	opts := rest.Opts{
+		Method: "POST",
+		Path:   "/file/ls.cgi",
+	}
+
+	filesList = &FilesList{}
+	err = f.pacer.Call(func() (bool, error) {
+		resp, err := f.rest.CallJSON(&opts, &request, filesList)
+		return shouldRetry(resp, err)
+	})
+	if err != nil {
+		return nil, errors.Wrap(err, "couldn't list files")
+	}
+
+	return filesList, nil
+}
+
+func (f *Fs) listFolders(directoryID string) (foldersList *FoldersList, err error) {
+	// fs.Debugf(f, "Requesting folders for id `%s`", directoryID)
+
+	request := ListFolderRequest{
+		FolderID: directoryID,
+	}
+
+	opts := rest.Opts{
+		Method: "POST",
+		Path:   "/folder/ls.cgi",
+	}
+
+	foldersList = &FoldersList{}
+	err = f.pacer.Call(func() (bool, error) {
+		resp, err := f.rest.CallJSON(&opts, &request, foldersList)
+		return shouldRetry(resp, err)
+	})
+	if err != nil {
+		return nil, errors.Wrap(err, "couldn't list folders")
+	}
+
+	// fs.Debugf(f, "Got FoldersList for id `%s`", directoryID)
+
+	return foldersList, err
+}
+
+func (f *Fs) listDir(ctx context.Context, dir string) (entries fs.DirEntries, err error) {
+	err = f.dirCache.FindRoot(ctx, false)
+	if err != nil {
+		return nil, err
+	}
+
+	directoryID, err := f.dirCache.FindDir(ctx, dir, false)
+	if err != nil {
+		return nil, err
+	}
+
+	files, err := f.listFiles(directoryID)
+	if err != nil {
+		return nil, err
+	}
+
+	folders, err := f.listFolders(directoryID)
+	if err != nil {
+		return nil, err
+	}
+
+	entries = make([]fs.DirEntry, len(files.Items)+len(folders.SubFolders))
+
+	for i, item := range files.Items {
+		entries[i] = f.newObjectFromFile(ctx, dir, item)
+	}
+
+	for i, folder := range folders.SubFolders {
+		createDate, err := time.Parse("2006-01-02 15:04:05", folder.CreateDate)
+		if err != nil {
+			return nil, err
+		}
+
+		folder.Name = restoreReservedChars(folder.Name)
+		fullPath := getRemote(dir, folder.Name)
+		folderID := strconv.Itoa(folder.ID)
+
+		entries[len(files.Items)+i] = fs.NewDir(fullPath, createDate).SetID(folderID)
+
+		// fs.Debugf(f, "Put Path `%s` for id `%d` into dircache", fullPath, folder.ID)
+		f.dirCache.Put(fullPath, folderID)
+	}
+
+	return entries, nil
+}
+
+func (f *Fs) newObjectFromFile(ctx context.Context, dir string, item File) *Object {
+	return &Object{
+		fs:     f,
+		remote: getRemote(dir, item.Filename),
+		file:   item,
+	}
+}
+
+func getRemote(dir, fileName string) string {
+	if dir == "" {
+		return fileName
+	}
+
+	return dir + "/" + fileName
+}
+
+func (f *Fs) makeFolder(leaf, directoryID string) (response *MakeFolderResponse, err error) {
+	name := replaceReservedChars(leaf)
+	// fs.Debugf(f, "Creating folder `%s` in id `%s`", name, directoryID)
+
+	request := MakeFolderRequest{
+		FolderID: directoryID,
+		Name:     name,
+	}
+
+	opts := rest.Opts{
+		Method: "POST",
+		Path:   "/folder/mkdir.cgi",
+	}
+
+	response = &MakeFolderResponse{}
+	err = f.pacer.Call(func() (bool, error) {
+		resp, err := f.rest.CallJSON(&opts, &request, response)
+		return shouldRetry(resp, err)
+	})
+	if err != nil {
+		return nil, errors.Wrap(err, "couldn't create folder")
+	}
+
+	// fs.Debugf(f, "Created Folder `%s` in id `%s`", name, directoryID)
+
+	return response, err
+}
+
+func (f *Fs) removeFolder(name, directoryID string) (response *GenericOKResponse, err error) {
+	// fs.Debugf(f, "Removing folder with id `%s`", directoryID)
+
+	request := &RemoveFolderRequest{
+		FolderID: directoryID,
+	}
+
+	opts := rest.Opts{
+		Method: "POST",
+		Path:   "/folder/rm.cgi",
+	}
+
+	response = &GenericOKResponse{}
+	var resp *http.Response
+	err = f.pacer.Call(func() (bool, error) {
+		resp, err = f.rest.CallJSON(&opts, request, response)
+		return shouldRetry(resp, err)
+	})
+	if err != nil {
+		return nil, errors.Wrap(err, "couldn't remove folder")
+	}
+	if response.Status != "OK" {
+		return nil, errors.New("Can't remove non-empty dir")
+	}
+
+	// fs.Debugf(f, "Removed Folder with id `%s`", directoryID)
+
+	return response, nil
+}
+
+func (f *Fs) deleteFile(url string) (response *GenericOKResponse, err error) {
+	request := &RemoveFileRequest{
+		Files: []RmFile{
+			{url},
+		},
+	}
+
+	opts := rest.Opts{
+		Method: "POST",
+		Path:   "/file/rm.cgi",
+	}
+
+	response = &GenericOKResponse{}
+	err = f.pacer.Call(func() (bool, error) {
+		resp, err := f.rest.CallJSON(&opts, request, response)
+		return shouldRetry(resp, err)
+	})
+
+	if err != nil {
+		return nil, errors.Wrap(err, "couldn't remove file")
+	}
+
+	// fs.Debugf(f, "Removed file with url `%s`", url)
+
+	return response, nil
+}
+
+func (f *Fs) getUploadNode() (response *GetUploadNodeResponse, err error) {
+	// fs.Debugf(f, "Requesting Upload node")
+
+	opts := rest.Opts{
+		Method:      "GET",
+		ContentType: "application/json", // 1Fichier API is bad
+		Path:        "/upload/get_upload_server.cgi",
+	}
+
+	response = &GetUploadNodeResponse{}
+	err = f.pacer.Call(func() (bool, error) {
+		resp, err := f.rest.CallJSON(&opts, nil, response)
+		return shouldRetry(resp, err)
+	})
+	if err != nil {
+		return nil, errors.Wrap(err, "didnt got an upload node")
+	}
+
+	// fs.Debugf(f, "Got Upload node")
+
+	return response, err
+}
+
+func (f *Fs) uploadFile(in io.Reader, size int64, fileName, folderID, uploadID, node string) (response *http.Response, err error) {
+	// fs.Debugf(f, "Uploading File `%s`", fileName)
+
+	if len(uploadID) > 10 || !isAlphaNumeric(uploadID) {
+		return nil, errors.New("Invalid UploadID")
+	}
+
+	opts := rest.Opts{
+		Method: "POST",
+		Path:   "/upload.cgi",
+		Parameters: map[string][]string{
+			"id": {uploadID},
+		},
+		NoResponse:           true,
+		Body:                 in,
+		ContentLength:        &size,
+		MultipartContentName: "file[]",
+		MultipartFileName:    fileName,
+		MultipartParams: map[string][]string{
+			"did": {folderID},
+		},
+	}
+
+	if node != "" {
+		opts.RootURL = "https://" + node
+	}
+
+	err = f.pacer.CallNoRetry(func() (bool, error) {
+		resp, err := f.rest.CallJSON(&opts, nil, nil)
+		return shouldRetry(resp, err)
+	})
+
+	if err != nil {
+		return nil, errors.Wrap(err, "couldn't upload file")
+	}
+
+	// fs.Debugf(f, "Uploaded File `%s`", fileName)
+
+	return response, err
+}
+
+func (f *Fs) endUpload(uploadID string, nodeurl string) (response *EndFileUploadResponse, err error) {
+	// fs.Debugf(f, "Ending File Upload `%s`", uploadID)
+
+	if len(uploadID) > 10 || !isAlphaNumeric(uploadID) {
+		return nil, errors.New("Invalid UploadID")
+	}
+
+	opts := rest.Opts{
+		Method:  "GET",
+		Path:    "/end.pl",
+		RootURL: "https://" + nodeurl,
+		Parameters: map[string][]string{
+			"xid": {uploadID},
+		},
+		ExtraHeaders: map[string]string{
+			"JSON": "1",
+		},
+	}
+
+	response = &EndFileUploadResponse{}
+	err = f.pacer.Call(func() (bool, error) {
+		resp, err := f.rest.CallJSON(&opts, nil, response)
+		return shouldRetry(resp, err)
+	})
+
+	if err != nil {
+		return nil, errors.Wrap(err, "couldn't finish file upload")
+	}
+
+	return response, err
+}

backend/fichier/fichier.go

@@ -0,0 +1,394 @@
+package fichier
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"net/http"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/ncw/rclone/fs"
+	"github.com/ncw/rclone/fs/config/configmap"
+	"github.com/ncw/rclone/fs/config/configstruct"
+	"github.com/ncw/rclone/fs/fshttp"
+	"github.com/ncw/rclone/fs/hash"
+	"github.com/ncw/rclone/lib/dircache"
+	"github.com/ncw/rclone/lib/pacer"
+	"github.com/ncw/rclone/lib/rest"
+	"github.com/pkg/errors"
+)
+
+const (
+	rootID        = "0"
+	apiBaseURL    = "https://api.1fichier.com/v1"
+	minSleep      = 334 * time.Millisecond // 3 API calls per second is recommended
+	maxSleep      = 5 * time.Second
+	decayConstant = 2 // bigger for slower decay, exponential
+)
+
+func init() {
+	fs.Register(&fs.RegInfo{
+		Name:        "fichier",
+		Description: "1Fichier",
+		Config: func(name string, config configmap.Mapper) {
+		},
+		NewFs: NewFs,
+		Options: []fs.Option{
+			{
+				Help: "Your API Key, get it from https://1fichier.com/console/params.pl",
+				Name: "api_key",
+			},
+			{
+				Help:     "If you want to download a shared folder, add this parameter",
+				Name:     "shared_folder",
+				Required: false,
+				Advanced: true,
+			},
+		},
+	})
+}
+
+// Options defines the configuration for this backend
+type Options struct {
+	APIKey       string `config:"api_key"`
+	SharedFolder string `config:"shared_folder"`
+}
+
+// Fs is the interface a cloud storage system must provide
+type Fs struct {
+	root       string
+	name       string
+	features   *fs.Features
+	dirCache   *dircache.DirCache
+	baseClient *http.Client
+	options    *Options
+	pacer      *fs.Pacer
+	rest       *rest.Client
+}
+
+// FindLeaf finds a directory of name leaf in the folder with ID pathID
+func (f *Fs) FindLeaf(ctx context.Context, pathID, leaf string) (pathIDOut string, found bool, err error) {
+	folders, err := f.listFolders(pathID)
+	if err != nil {
+		return "", false, err
+	}
+
+	for _, folder := range folders.SubFolders {
+		if folder.Name == leaf {
+			pathIDOut := strconv.Itoa(folder.ID)
+			return pathIDOut, true, nil
+		}
+	}
+
+	return "", false, nil
+}
+
+// CreateDir makes a directory with pathID as parent and name leaf
+func (f *Fs) CreateDir(ctx context.Context, pathID, leaf string) (newID string, err error) {
+	resp, err := f.makeFolder(leaf, pathID)
+	if err != nil {
+		return "", err
+	}
+	return strconv.Itoa(resp.FolderID), err
+}
+
+// Name of the remote (as passed into NewFs)
+func (f *Fs) Name() string {
+	return f.name
+}
+
+// Root of the remote (as passed into NewFs)
+func (f *Fs) Root() string {
+	return f.root
+}
+
+// String returns a description of the FS
+func (f *Fs) String() string {
+	return fmt.Sprintf("1Fichier root '%s'", f.root)
+}
+
+// Precision of the ModTimes in this Fs
+func (f *Fs) Precision() time.Duration {
+	return fs.ModTimeNotSupported
+}
+
+// Hashes returns the supported hash types of the filesystem
+func (f *Fs) Hashes() hash.Set {
+	return hash.Set(hash.Whirlpool)
+}
+
+// Features returns the optional features of this Fs
+func (f *Fs) Features() *fs.Features {
+	return f.features
+}
+
+// NewFs makes a new Fs object from the path
+//
+// The path is of the form remote:path
+//
+// Remotes are looked up in the config file.  If the remote isn't
+// found then NotFoundInConfigFile will be returned.
+//
+// On Windows avoid single character remote names as they can be mixed
+// up with drive letters.
+func NewFs(name string, rootleaf string, config configmap.Mapper) (fs.Fs, error) {
+	root := replaceReservedChars(rootleaf)
+	opt := new(Options)
+	err := configstruct.Set(config, opt)
+	if err != nil {
+		return nil, err
+	}
+
+	// If using a Shared Folder override root
+	if opt.SharedFolder != "" {
+		root = ""
+	}
+
+	//workaround for wonky parser
+	root = strings.Trim(root, "/")
+
+	f := &Fs{
+		name:       name,
+		root:       root,
+		options:    opt,
+		pacer:      fs.NewPacer(pacer.NewDefault(pacer.MinSleep(minSleep), pacer.MaxSleep(maxSleep), pacer.DecayConstant(decayConstant))),
+		baseClient: &http.Client{},
+	}
+
+	f.features = (&fs.Features{
+		DuplicateFiles:          true,
+		CanHaveEmptyDirectories: true,
+	}).Fill(f)
+
+	client := fshttp.NewClient(fs.Config)
+
+	f.rest = rest.NewClient(client).SetRoot(apiBaseURL)
+
+	f.rest.SetHeader("Authorization", "Bearer "+f.options.APIKey)
+
+	f.dirCache = dircache.New(root, rootID, f)
+
+	ctx := context.Background()
+
+	// Find the current root
+	err = f.dirCache.FindRoot(ctx, false)
+	if err != nil {
+		// Assume it is a file
+		newRoot, remote := dircache.SplitPath(root)
+		tempF := *f
+		tempF.dirCache = dircache.New(newRoot, rootID, &tempF)
+		tempF.root = newRoot
+		// Make new Fs which is the parent
+		err = tempF.dirCache.FindRoot(ctx, false)
+		if err != nil {
+			// No root so return old f
+			return f, nil
+		}
+		_, err := tempF.NewObject(ctx, remote)
+		if err != nil {
+			if err == fs.ErrorObjectNotFound {
+				// File doesn't exist so return old f
+				return f, nil
+			}
+			return nil, err
+		}
+		f.features.Fill(&tempF)
+		// XXX: update the old f here instead of returning tempF, since
+		// `features` were already filled with functions having *f as a receiver.
+		// See https://github.com/ncw/rclone/issues/2182
+		f.dirCache = tempF.dirCache
+		f.root = tempF.root
+		// return an error with an fs which points to the parent
+		return f, fs.ErrorIsFile
+	}
+	return f, nil
+}
+
+// List the objects and directories in dir into entries.  The
+// entries can be returned in any order but should be for a
+// complete directory.
+//
+// dir should be "" to list the root, and should not have
+// trailing slashes.
+//
+// This should return ErrDirNotFound if the directory isn't
+// found.
+func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err error) {
+	if f.options.SharedFolder != "" {
+		return f.listSharedFiles(ctx, f.options.SharedFolder)
+	}
+
+	dirContent, err := f.listDir(ctx, dir)
+	if err != nil {
+		return nil, err
+	}
+
+	return dirContent, nil
+}
+
+// NewObject finds the Object at remote.  If it can't be found
+// it returns the error ErrorObjectNotFound.
+func (f *Fs) NewObject(ctx context.Context, remote string) (fs.Object, error) {
+	leaf, directoryID, err := f.dirCache.FindRootAndPath(ctx, remote, false)
+	if err != nil {
+		if err == fs.ErrorDirNotFound {
+			return nil, fs.ErrorObjectNotFound
+		}
+		return nil, err
+	}
+
+	files, err := f.listFiles(directoryID)
+	if err != nil {
+		return nil, err
+	}
+
+	for _, file := range files.Items {
+		if file.Filename == leaf {
+			path, ok := f.dirCache.GetInv(directoryID)
+
+			if !ok {
+				return nil, errors.New("Cannot find dir in dircache")
+			}
+
+			return f.newObjectFromFile(ctx, path, file), nil
+		}
+	}
+
+	return nil, fs.ErrorObjectNotFound
+}
+
+// Put in to the remote path with the modTime given of the given size
+//
+// When called from outside a Fs by rclone, src.Size() will always be >= 0.
+// But for unknown-sized objects (indicated by src.Size() == -1), Put should either
+// return an error or upload it properly (rather than e.g. calling panic).
+//
+// May create the object even if it returns an error - if so
+// will return the object and the error, otherwise will return
+// nil and the error
+func (f *Fs) Put(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) {
+	exisitingObj, err := f.NewObject(ctx, src.Remote())
+	switch err {
+	case nil:
+		return exisitingObj, exisitingObj.Update(ctx, in, src, options...)
+	case fs.ErrorObjectNotFound:
+		// Not found so create it
+		return f.PutUnchecked(ctx, in, src, options...)
+	default:
+		return nil, err
+	}
+}
+
+// putUnchecked uploads the object with the given name and size
+//
+// This will create a duplicate if we upload a new file without
+// checking to see if there is one already - use Put() for that.
+func (f *Fs) putUnchecked(ctx context.Context, in io.Reader, remote string, size int64, options ...fs.OpenOption) (fs.Object, error) {
+	if size > int64(100E9) {
+		return nil, errors.New("File too big, cant upload")
+	} else if size == 0 {
+		return nil, fs.ErrorCantUploadEmptyFiles
+	}
+
+	nodeResponse, err := f.getUploadNode()
+	if err != nil {
+		return nil, err
+	}
+
+	leaf, directoryID, err := f.dirCache.FindRootAndPath(ctx, remote, true)
+	if err != nil {
+		return nil, err
+	}
+
+	_, err = f.uploadFile(in, size, leaf, directoryID, nodeResponse.ID, nodeResponse.URL)
+	if err != nil {
+		return nil, err
+	}
+
+	fileUploadResponse, err := f.endUpload(nodeResponse.ID, nodeResponse.URL)
+	if err != nil {
+		return nil, err
+	}
+
+	if len(fileUploadResponse.Links) != 1 {
+		return nil, errors.New("unexpected amount of files")
+	}
+
+	link := fileUploadResponse.Links[0]
+	fileSize, err := strconv.ParseInt(link.Size, 10, 64)
+
+	if err != nil {
+		return nil, err
+	}
+
+	return &Object{
+		fs:     f,
+		remote: remote,
+		file: File{
+			ACL:         0,
+			CDN:         0,
+			Checksum:    link.Whirlpool,
+			ContentType: "",
+			Date:        time.Now().Format("2006-01-02 15:04:05"),
+			Filename:    link.Filename,
+			Pass:        0,
+			Size:        int(fileSize),
+			URL:         link.Download,
+		},
+	}, nil
+}
+
+// PutUnchecked uploads the object
+//
+// This will create a duplicate if we upload a new file without
+// checking to see if there is one already - use Put() for that.
+func (f *Fs) PutUnchecked(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) {
+	return f.putUnchecked(ctx, in, src.Remote(), src.Size(), options...)
+}
+
+// Mkdir makes the directory (container, bucket)
+//
+// Shouldn't return an error if it already exists
+func (f *Fs) Mkdir(ctx context.Context, dir string) error {
+	err := f.dirCache.FindRoot(ctx, true)
+	if err != nil {
+		return err
+	}
+	if dir != "" {
+		_, err = f.dirCache.FindDir(ctx, dir, true)
+	}
+	return err
+}
+
+// Rmdir removes the directory (container, bucket) if empty
+//
+// Return an error if it doesn't exist or isn't empty
+func (f *Fs) Rmdir(ctx context.Context, dir string) error {
+	err := f.dirCache.FindRoot(ctx, false)
+	if err != nil {
+		return err
+	}
+
+	did, err := f.dirCache.FindDir(ctx, dir, false)
+	if err != nil {
+		return err
+	}
+
+	_, err = f.removeFolder(dir, did)
+	if err != nil {
+		return err
+	}
+
+	f.dirCache.FlushDir(dir)
+
+	return nil
+}
+
+// Check the interfaces are satisfied
+var (
+	_ fs.Fs              = (*Fs)(nil)
+	_ fs.PutUncheckeder  = (*Fs)(nil)
+	_ dircache.DirCacher = (*Fs)(nil)
+)

backend/fichier/fichier_test.go

@@ -0,0 +1,17 @@
+// Test 1Fichier filesystem interface
+package fichier
+
+import (
+	"testing"
+
+	"github.com/ncw/rclone/fs"
+	"github.com/ncw/rclone/fstest/fstests"
+)
+
+// TestIntegration runs integration tests against the remote
+func TestIntegration(t *testing.T) {
+	fs.Config.LogLevel = fs.LogLevelDebug
+	fstests.Run(t, &fstests.Opt{
+		RemoteName: "TestFichier:",
+	})
+}

backend/fichier/object.go

@@ -0,0 +1,158 @@
+package fichier
+
+import (
+	"context"
+	"io"
+	"net/http"
+	"time"
+
+	"github.com/ncw/rclone/fs"
+	"github.com/ncw/rclone/fs/hash"
+	"github.com/ncw/rclone/lib/rest"
+	"github.com/pkg/errors"
+)
+
+// Object is a filesystem like object provided by an Fs
+type Object struct {
+	fs     *Fs
+	remote string
+	file   File
+}
+
+// String returns a description of the Object
+func (o *Object) String() string {
+	return o.file.Filename
+}
+
+// Remote returns the remote path
+func (o *Object) Remote() string {
+	return o.remote
+}
+
+// ModTime returns the modification date of the file
+// It should return a best guess if one isn't available
+func (o *Object) ModTime(ctx context.Context) time.Time {
+	modTime, err := time.Parse("2006-01-02 15:04:05", o.file.Date)
+
+	if err != nil {
+		return time.Now()
+	}
+
+	return modTime
+}
+
+// Size returns the size of the file
+func (o *Object) Size() int64 {
+	return int64(o.file.Size)
+}
+
+// Fs returns read only access to the Fs that this object is part of
+func (o *Object) Fs() fs.Info {
+	return o.fs
+}
+
+// Hash returns the selected checksum of the file
+// If no checksum is available it returns ""
+func (o *Object) Hash(ctx context.Context, t hash.Type) (string, error) {
+	if t != hash.Whirlpool {
+		return "", hash.ErrUnsupported
+	}
+
+	return o.file.Checksum, nil
+}
+
+// Storable says whether this object can be stored
+func (o *Object) Storable() bool {
+	return true
+}
+
+// SetModTime sets the metadata on the object to set the modification date
+func (o *Object) SetModTime(context.Context, time.Time) error {
+	return fs.ErrorCantSetModTime
+	//return errors.New("setting modtime is not supported for 1fichier remotes")
+}
+
+// Open opens the file for read.  Call Close() on the returned io.ReadCloser
+func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (io.ReadCloser, error) {
+	fs.FixRangeOption(options, int64(o.file.Size))
+	downloadToken, err := o.fs.getDownloadToken(o.file.URL)
+
+	if err != nil {
+		return nil, err
+	}
+
+	var resp *http.Response
+	opts := rest.Opts{
+		Method:  "GET",
+		RootURL: downloadToken.URL,
+		Options: options,
+	}
+
+	err = o.fs.pacer.Call(func() (bool, error) {
+		resp, err = o.fs.rest.Call(&opts)
+		return shouldRetry(resp, err)
+	})
+
+	if err != nil {
+		return nil, err
+	}
+	return resp.Body, err
+}
+
+// Update in to the object with the modTime given of the given size
+//
+// When called from outside a Fs by rclone, src.Size() will always be >= 0.
+// But for unknown-sized objects (indicated by src.Size() == -1), Upload should either
+// return an error or update the object properly (rather than e.g. calling panic).
+func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) error {
+	if src.Size() < 0 {
+		return errors.New("refusing to update with unknown size")
+	}
+
+	// upload with new size but old name
+	info, err := o.fs.putUnchecked(ctx, in, o.Remote(), src.Size(), options...)
+	if err != nil {
+		return err
+	}
+
+	// Delete duplicate after successful upload
+	err = o.Remove(ctx)
+	if err != nil {
+		return errors.Wrap(err, "failed to remove old version")
+	}
+
+	// Replace guts of old object with new one
+	*o = *info.(*Object)
+
+	return nil
+}
+
+// Remove removes this object
+func (o *Object) Remove(ctx context.Context) error {
+	// fs.Debugf(f, "Removing file `%s` with url `%s`", o.file.Filename, o.file.URL)
+
+	_, err := o.fs.deleteFile(o.file.URL)
+
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+// MimeType of an Object if known, "" otherwise
+func (o *Object) MimeType(ctx context.Context) string {
+	return o.file.ContentType
+}
+
+// ID returns the ID of the Object if known, or "" if not
+func (o *Object) ID() string {
+	return o.file.URL
+}
+
+// Check the interfaces are satisfied
+var (
+	_ fs.Object    = (*Object)(nil)
+	_ fs.MimeTyper = (*Object)(nil)
+	_ fs.IDer      = (*Object)(nil)
+)

backend/fichier/replace.go

@@ -0,0 +1,71 @@
+/*
+Translate file names for 1fichier
+
+1Fichier reserved characters
+
+The following characters are 1Fichier reserved characters, and can't
+be used in 1Fichier  folder and file names.
+
+*/
+
+package fichier
+
+import (
+	"regexp"
+	"strings"
+)
+
+// charMap holds replacements for characters
+//
+// 1Fichier has a restricted set of characters compared to other cloud
+// storage systems, so we to map these to the FULLWIDTH unicode
+// equivalents
+//
+// http://unicode-search.net/unicode-namesearch.pl?term=SOLIDUS
+var (
+	charMap = map[rune]rune{
+		'\\': '＼', // FULLWIDTH REVERSE SOLIDUS
+		'<':  '＜', // FULLWIDTH LESS-THAN SIGN
+		'>':  '＞', // FULLWIDTH GREATER-THAN SIGN
+		'"':  '＂', // FULLWIDTH QUOTATION MARK - not on the list but seems to be reserved
+		'\'': '＇', // FULLWIDTH APOSTROPHE
+		'$':  '＄', // FULLWIDTH DOLLAR SIGN
+		'`':  '｀', // FULLWIDTH GRAVE ACCENT
+		' ':  '␠', // SYMBOL FOR SPACE
+	}
+	invCharMap           map[rune]rune
+	fixStartingWithSpace = regexp.MustCompile(`(/|^) `)
+)
+
+func init() {
+	// Create inverse charMap
+	invCharMap = make(map[rune]rune, len(charMap))
+	for k, v := range charMap {
+		invCharMap[v] = k
+	}
+}
+
+// replaceReservedChars takes a path and substitutes any reserved
+// characters in it
+func replaceReservedChars(in string) string {
+	// file names can't start with space either
+	in = fixStartingWithSpace.ReplaceAllString(in, "$1"+string(charMap[' ']))
+	// Replace reserved characters
+	return strings.Map(func(c rune) rune {
+		if replacement, ok := charMap[c]; ok && c != ' ' {
+			return replacement
+		}
+		return c
+	}, in)
+}
+
+// restoreReservedChars takes a path and undoes any substitutions
+// made by replaceReservedChars
+func restoreReservedChars(in string) string {
+	return strings.Map(func(c rune) rune {
+		if replacement, ok := invCharMap[c]; ok {
+			return replacement
+		}
+		return c
+	}, in)
+}

backend/fichier/replace_test.go

@@ -0,0 +1,24 @@
+package fichier
+
+import "testing"
+
+func TestReplace(t *testing.T) {
+	for _, test := range []struct {
+		in  string
+		out string
+	}{
+		{"", ""},
+		{"abc 123", "abc 123"},
+		{"\"'<>/\\$`", `＂＇＜＞/＼＄｀`},
+		{" leading space", "␠leading space"},
+	} {
+		got := replaceReservedChars(test.in)
+		if got != test.out {
+			t.Errorf("replaceReservedChars(%q) want %q got %q", test.in, test.out, got)
+		}
+		got2 := restoreReservedChars(got)
+		if got2 != test.in {
+			t.Errorf("restoreReservedChars(%q) want %q got %q", got, test.in, got2)
+		}
+	}
+}

backend/fichier/structs.go

@@ -0,0 +1,120 @@
+package fichier
+
+// ListFolderRequest is the request structure of the corresponding request
+type ListFolderRequest struct {
+	FolderID string `json:"folder_id"`
+}
+
+// ListFilesRequest is the request structure of the corresponding request
+type ListFilesRequest struct {
+	FolderID string `json:"folder_id"`
+}
+
+// DownloadRequest is the request structure of the corresponding request
+type DownloadRequest struct {
+	URL    string `json:"url"`
+	Single int    `json:"single"`
+}
+
+// RemoveFolderRequest is the request structure of the corresponding request
+type RemoveFolderRequest struct {
+	FolderID string `json:"folder_id"`
+}
+
+// RemoveFileRequest is the request structure of the corresponding request
+type RemoveFileRequest struct {
+	Files []RmFile `json:"files"`
+}
+
+// RmFile is the request structure of the corresponding request
+type RmFile struct {
+	URL string `json:"url"`
+}
+
+// GenericOKResponse is the response structure of the corresponding request
+type GenericOKResponse struct {
+	Status  string `json:"status"`
+	Message string `json:"message"`
+}
+
+// MakeFolderRequest is the request structure of the corresponding request
+type MakeFolderRequest struct {
+	Name     string `json:"name"`
+	FolderID string `json:"folder_id"`
+}
+
+// MakeFolderResponse is the response structure of the corresponding request
+type MakeFolderResponse struct {
+	Name     string `json:"name"`
+	FolderID int    `json:"folder_id"`
+}
+
+// GetUploadNodeResponse is the response structure of the corresponding request
+type GetUploadNodeResponse struct {
+	ID  string `json:"id"`
+	URL string `json:"url"`
+}
+
+// GetTokenResponse is the response structure of the corresponding request
+type GetTokenResponse struct {
+	URL     string `json:"url"`
+	Status  string `json:"Status"`
+	Message string `json:"Message"`
+}
+
+// SharedFolderResponse is the response structure of the corresponding request
+type SharedFolderResponse []SharedFile
+
+// SharedFile is the structure how 1Fichier returns a shared File
+type SharedFile struct {
+	Filename string `json:"filename"`
+	Link     string `json:"link"`
+	Size     int    `json:"size"`
+}
+
+// EndFileUploadResponse is the response structure of the corresponding request
+type EndFileUploadResponse struct {
+	Incoming int `json:"incoming"`
+	Links    []struct {
+		Download  string `json:"download"`
+		Filename  string `json:"filename"`
+		Remove    string `json:"remove"`
+		Size      string `json:"size"`
+		Whirlpool string `json:"whirlpool"`
+	} `json:"links"`
+}
+
+// File is the structure how 1Fichier returns a File
+type File struct {
+	ACL         int    `json:"acl"`
+	CDN         int    `json:"cdn"`
+	Checksum    string `json:"checksum"`
+	ContentType string `json:"content-type"`
+	Date        string `json:"date"`
+	Filename    string `json:"filename"`
+	Pass        int    `json:"pass"`
+	Size        int    `json:"size"`
+	URL         string `json:"url"`
+}
+
+// FilesList is the structure how 1Fichier returns a list of files
+type FilesList struct {
+	Items  []File `json:"items"`
+	Status string `json:"Status"`
+}
+
+// Folder is the structure how 1Fichier returns a Folder
+type Folder struct {
+	CreateDate string `json:"create_date"`
+	ID         int    `json:"id"`
+	Name       string `json:"name"`
+	Pass       string `json:"pass"`
+}
+
+// FoldersList is the structure how 1Fichier returns a list of Folders
+type FoldersList struct {
+	FolderID   string   `json:"folder_id"`
+	Name       string   `json:"name"`
+	Status     string   `json:"Status"`
+	SubFolders []Folder `json:"sub_folders"`
+}

bin/make_manual.py

@@ -23,6 +23,7 @@ docs = [
     "flags.md",
 
     # Keep these alphabetical by full name
+    "fichier.md",
     "alias.md",
     "amazonclouddrive.md",
     "s3.md",

cmd/serve/restic/restic-test.sh

@@ -12,6 +12,7 @@ TestCryptDrive:
 TestCryptSwift:
 TestDrive:
 TestDropbox:
+TestFichier:
 TestFTP:
 TestGoogleCloudStorage:
 TestHubic:

docs/content/about.md

@@ -13,6 +13,7 @@ Rclone
 
 Rclone is a command line program to sync files and directories to and from:
 
+* {{< provider name="1Fichier" home="https://1fichier.com/" config="/fichier/" >}}
 * {{< provider name="Alibaba Cloud (Aliyun) Object Storage System (OSS)" home="https://www.alibabacloud.com/product/oss/" config="/s3/#alibaba-oss" >}}
 * {{< provider name="Amazon Drive" home="https://www.amazon.com/clouddrive" config="/amazonclouddrive/" >}} ([See note](/amazonclouddrive/#status))
 * {{< provider name="Amazon S3" home="https://aws.amazon.com/s3/" config="/s3/" >}}

docs/content/docs.md

@@ -19,6 +19,7 @@ option:
 
 See the following for detailed instructions for
 
+  * [1Fichier](/fichier/)
   * [Alias](/alias/)
   * [Amazon Drive](/amazonclouddrive/)
   * [Amazon S3](/s3/)

docs/content/fichier.md

@@ -0,0 +1,122 @@
+---
+title: "1Fichier"
+description: "Rclone docs for 1Fichier"
+date: "2015-10-14"
+---
+
+<i class="fa fa-archive"></i> 1Fichier
+-----------------------------------------
+
+This is a backend for the [1ficher](https://1fichier.com) cloud
+storage service. Note that a Premium subscription is required to use
+the API.
+
+Paths are specified as `remote:path`
+
+Paths may be as deep as required, eg `remote:directory/subdirectory`.
+
+The initial setup for 1Fichier involves getting the API key from the website which you
+need to do in your browser.
+
+Here is an example of how to make a remote called `remote`.  First run:
+
+     rclone config
+
+This will guide you through an interactive setup process:
+
+```
+No remotes found - make a new one
+n) New remote
+s) Set configuration password
+q) Quit config
+n/s/q> n
+name> remote
+Type of storage to configure.
+Enter a string value. Press Enter for the default ("").
+Choose a number from below, or type in your own value
+  1 / 1Fichier
+    \ "fichier"
+...
+Storage> fichier
+** See help for fichier backend at: https://rclone.org/fichier/ **
+
+Your API Key, get it from https://1fichier.com/console/params.pl
+Enter a string value. Press Enter for the default ("").
+api_key> example_key
+
+Edit advanced config? (y/n)
+y) Yes
+n) No
+y/n> 
+Remote config
+--------------------
+[remote]
+type = fichier
+api_key = example_key
+--------------------
+y) Yes this is OK
+e) Edit this remote
+d) Delete this remote
+y/e/d> y
+```
+
+Once configured you can then use `rclone` like this,
+
+List directories in top level of your 1Fichier account
+
+    rclone lsd remote:
+
+List all the files in your 1Fichier account
+
+    rclone ls remote:
+
+To copy a local directory to a 1Fichier directory called backup
+
+    rclone copy /home/source remote:backup
+
+### Modified time and hashes ###
+
+1Fichier does not support modification times. It supports the Whirlpool hash algorithm.
+
+### Duplicated files ###
+
+1Fichier can have two files with exactly the same name and path (unlike a
+normal file system).
+
+Duplicated files cause problems with the syncing and you will see
+messages in the log about duplicates.
+
+### Forbidden characters ###
+
+1Fichier does not support the characters ``\ < > " ' ` $`` and spaces at the beginning of folder names.
+`rclone` automatically escapes these to a unicode equivalent. The exception is `/`, 
+which cannot be escaped and will therefore lead to errors.
+
+<!--- autogenerated options start - DO NOT EDIT, instead edit fs.RegInfo in backend/fichier/fichier.go then run make backenddocs -->
+### Standard Options
+
+Here are the standard options specific to fichier (1Fichier).
+
+#### --fichier-api-key
+
+Your API Key, get it from https://1fichier.com/console/params.pl
+
+- Config:      api_key
+- Env Var:     RCLONE_FICHIER_API_KEY
+- Type:        string
+- Default:     ""
+
+### Advanced Options
+
+Here are the advanced options specific to fichier (1Fichier).
+
+#### --fichier-shared-folder
+
+If you want to download a shared folder, add this parameter
+
+- Config:      shared_folder
+- Env Var:     RCLONE_FICHIER_SHARED_FOLDER
+- Type:        string
+- Default:     ""
+
+<!--- autogenerated options stop -->

docs/content/overview.md

@@ -17,6 +17,7 @@ Here is an overview of the major features of each cloud storage system.
 
 | Name                         | Hash        | ModTime | Case Insensitive | Duplicate Files | MIME Type |
 | ---------------------------- |:-----------:|:-------:|:----------------:|:---------------:|:---------:|
+| 1Fichier                     | Whirlpool   | No      | No               | Yes             | R         |
 | Amazon Drive                 | MD5         | No      | Yes              | No              | R         |
 | Amazon S3                    | MD5         | Yes     | No               | No              | R/W       |
 | Backblaze B2                 | SHA1        | Yes     | No               | No              | R/W       |
@@ -131,6 +132,7 @@ operations more efficient.
 
 | Name                         | Purge | Copy | Move | DirMove | CleanUp | ListR | StreamUpload | LinkSharing | About |
 | ---------------------------- |:-----:|:----:|:----:|:-------:|:-------:|:-----:|:------------:|:------------:|:-----:|
+| 1Fichier                     | No    | No   | No   | No      | No      | No    | No           | No           |   No  |
 | Amazon Drive                 | Yes   | No   | Yes  | Yes     | No [#575](https://github.com/ncw/rclone/issues/575) | No  | No  | No [#2178](https://github.com/ncw/rclone/issues/2178) | No  |
 | Amazon S3                    | No    | Yes  | No   | No      | No      | Yes   | Yes          | No [#2178](https://github.com/ncw/rclone/issues/2178) | No  |
 | Backblaze B2                 | No    | Yes  | No   | No      | Yes     | Yes   | Yes          | No [#2178](https://github.com/ncw/rclone/issues/2178) | No  |

docs/layouts/chrome/navbar.html

@@ -56,6 +56,7 @@
                   <a href="#" class="dropdown-toggle" data-toggle="dropdown"><b class="caret"></b> Storage Systems</a>
                   <ul class="dropdown-menu pre-scrollable">
                     <li><a href="/overview/"><i class="fa fa-archive"></i> Overview</a></li>
+                    <li><a href="/fichier/"><i class="fa fa-archive"></i> 1Fichier</a></li>
                     <li><a href="/amazonclouddrive/"><i class="fa fa-amazon"></i> Amazon Drive</a></li>
                     <li><a href="/s3/"><i class="fa fa-amazon"></i> Amazon S3</a></li>
                     <li><a href="/b2/"><i class="fa fa-fire"></i> Backblaze B2</a></li>

fs/hash/hash.go

@@ -9,6 +9,7 @@ import (
 	"io"
 	"strings"
 
+	"github.com/jzelinskie/whirlpool"
 	"github.com/ncw/rclone/backend/dropbox/dbhash"
 	"github.com/ncw/rclone/backend/onedrive/quickxorhash"
 	"github.com/pkg/errors"
@@ -36,13 +37,16 @@ const (
 	// https://docs.microsoft.com/en-us/onedrive/developer/code-snippets/quickxorhash
 	QuickXorHash
 
+	// Whirlpool indicates Whirlpool support
+	Whirlpool
+
 	// None indicates no hashes are supported
 	None Type = 0
 )
 
 // Supported returns a set of all the supported hashes by
 // HashStream and MultiHasher.
-var Supported = NewHashSet(MD5, SHA1, Dropbox, QuickXorHash)
+var Supported = NewHashSet(MD5, SHA1, Dropbox, QuickXorHash, Whirlpool)
 
 // Width returns the width in characters for any HashType
 var Width = map[Type]int{
@@ -50,6 +54,7 @@ var Width = map[Type]int{
 	SHA1:         40,
 	Dropbox:      64,
 	QuickXorHash: 40,
+	Whirlpool:    128,
 }
 
 // Stream will calculate hashes of all supported hash types.
@@ -89,6 +94,8 @@ func (h Type) String() string {
 		return "DropboxHash"
 	case QuickXorHash:
 		return "QuickXorHash"
+	case Whirlpool:
+		return "Whirlpool"
 	default:
 		err := fmt.Sprintf("internal error: unknown hash type: 0x%x", int(h))
 		panic(err)
@@ -108,6 +115,8 @@ func (h *Type) Set(s string) error {
 		*h = Dropbox
 	case "QuickXorHash":
 		*h = QuickXorHash
+	case "Whirlpool":
+		*h = Whirlpool
 	default:
 		return errors.Errorf("Unknown hash type %q", s)
 	}
@@ -138,6 +147,8 @@ func fromTypes(set Set) (map[Type]hash.Hash, error) {
 			hashers[t] = dbhash.New()
 		case QuickXorHash:
 			hashers[t] = quickxorhash.New()
+		case Whirlpool:
+			hashers[t] = whirlpool.New()
 		default:
 			err := fmt.Sprintf("internal error: Unsupported hash type %v", t)
 			panic(err)

fs/hash/hash_test.go

@@ -73,6 +73,7 @@ var hashTestSet = []hashTest{
 			hash.SHA1:         "3ab6543c08a75f292a5ecedac87ec41642d12166",
 			hash.Dropbox:      "214d2fcf3566e94c99ad2f59bd993daca46d8521a0c447adf4b324f53fddc0c7",
 			hash.QuickXorHash: "0110c000085000031c0001095ec00218d0000700",
+			hash.Whirlpool:    "eddf52133d4566d763f716e853d6e4efbabd29e2c2e63f56747b1596172851d34c2df9944beb6640dbdbe3d9b4eb61180720a79e3d15baff31c91e43d63869a4",
 		},
 	},
 	// Empty data set
@@ -83,6 +84,7 @@ var hashTestSet = []hashTest{
 			hash.SHA1:         "da39a3ee5e6b4b0d3255bfef95601890afd80709",
 			hash.Dropbox:      "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
 			hash.QuickXorHash: "0000000000000000000000000000000000000000",
+			hash.Whirlpool:    "19fa61d75522a4669b44e39c1d2e1726c530232130d407f89afee0964997f7a73e83be698b288febcf88e3e03c4f0757ea8964e59b63d93708b138cc42a66eb3",
 		},
 	},
 }

fstest/test_all/config.yaml

@@ -110,6 +110,10 @@ backends:
    remote:   "TestBox:"
    subdir:   false
    fastlist: false
+ - backend:  "fichier"
+   remote:   "TestFichier:"
+   subdir:   false
+   fastlist: false
  - backend:  "qingstor"
    remote:   "TestQingStor:"
    subdir:   false