// Package fichier provides an interface to the 1Fichier storage system. package fichier import ( "context" "errors" "fmt" "io" "net/http" "strconv" "strings" "time" "github.com/rclone/rclone/fs" "github.com/rclone/rclone/fs/config" "github.com/rclone/rclone/fs/config/configmap" "github.com/rclone/rclone/fs/config/configstruct" "github.com/rclone/rclone/fs/fshttp" "github.com/rclone/rclone/fs/hash" "github.com/rclone/rclone/lib/dircache" "github.com/rclone/rclone/lib/encoder" "github.com/rclone/rclone/lib/pacer" "github.com/rclone/rclone/lib/rest" ) const ( rootID = "0" apiBaseURL = "https://api.1fichier.com/v1" minSleep = 400 * time.Millisecond // api is extremely rate limited now maxSleep = 5 * time.Second decayConstant = 2 // bigger for slower decay, exponential attackConstant = 0 // start with max sleep ) func init() { fs.Register(&fs.RegInfo{ Name: "fichier", Description: "1Fichier", NewFs: NewFs, Options: []fs.Option{{ Help: "Your API Key, get it from https://1fichier.com/console/params.pl.", Name: "api_key", Sensitive: true, }, { Help: "If you want to download a shared folder, add this parameter.", Name: "shared_folder", Advanced: true, }, { Help: "If you want to download a shared file that is password protected, add this parameter.", Name: "file_password", Advanced: true, IsPassword: true, }, { Help: "If you want to list the files in a shared folder that is password protected, add this parameter.", Name: "folder_password", Advanced: true, IsPassword: true, }, { Help: "Set if you wish to use CDN download links.", Name: "cdn", Default: false, Advanced: true, }, { Name: config.ConfigEncoding, Help: config.ConfigEncodingHelp, Advanced: true, // Characters that need escaping // // '\\': '\', // FULLWIDTH REVERSE SOLIDUS // '<': '<', // FULLWIDTH LESS-THAN SIGN // '>': '>', // FULLWIDTH GREATER-THAN SIGN // '"': '"', // FULLWIDTH QUOTATION MARK - not on the list but seems to be reserved // '\'': ''', // FULLWIDTH APOSTROPHE // '$': '$', // FULLWIDTH DOLLAR SIGN // '`': '`', // FULLWIDTH GRAVE ACCENT // // Leading space and trailing space Default: (encoder.Display | encoder.EncodeBackSlash | encoder.EncodeSingleQuote | encoder.EncodeBackQuote | encoder.EncodeDoubleQuote | encoder.EncodeLtGt | encoder.EncodeDollar | encoder.EncodeLeftSpace | encoder.EncodeRightSpace | encoder.EncodeInvalidUtf8), }}, }) } // Options defines the configuration for this backend type Options struct { APIKey string `config:"api_key"` SharedFolder string `config:"shared_folder"` FilePassword string `config:"file_password"` FolderPassword string `config:"folder_password"` CDN bool `config:"cdn"` Enc encoder.MultiEncoder `config:"encoding"` } // Fs is the interface a cloud storage system must provide type Fs struct { root string name string features *fs.Features opt Options dirCache *dircache.DirCache baseClient *http.Client pacer *fs.Pacer rest *rest.Client } // FindLeaf finds a directory of name leaf in the folder with ID pathID func (f *Fs) FindLeaf(ctx context.Context, pathID, leaf string) (pathIDOut string, found bool, err error) { folderID, err := strconv.Atoi(pathID) if err != nil { return "", false, err } folders, err := f.listFolders(ctx, folderID) if err != nil { return "", false, err } for _, folder := range folders.SubFolders { if folder.Name == leaf { pathIDOut := strconv.Itoa(folder.ID) return pathIDOut, true, nil } } return "", false, nil } // CreateDir makes a directory with pathID as parent and name leaf func (f *Fs) CreateDir(ctx context.Context, pathID, leaf string) (newID string, err error) { folderID, err := strconv.Atoi(pathID) if err != nil { return "", err } resp, err := f.makeFolder(ctx, leaf, folderID) if err != nil { return "", err } return strconv.Itoa(resp.FolderID), err } // Name of the remote (as passed into NewFs) func (f *Fs) Name() string { return f.name } // Root of the remote (as passed into NewFs) func (f *Fs) Root() string { return f.root } // String returns a description of the FS func (f *Fs) String() string { return fmt.Sprintf("1Fichier root '%s'", f.root) } // Precision of the ModTimes in this Fs func (f *Fs) Precision() time.Duration { return fs.ModTimeNotSupported } // Hashes returns the supported hash types of the filesystem func (f *Fs) Hashes() hash.Set { return hash.Set(hash.Whirlpool) } // Features returns the optional features of this Fs func (f *Fs) Features() *fs.Features { return f.features } // NewFs makes a new Fs object from the path // // The path is of the form remote:path // // Remotes are looked up in the config file. If the remote isn't // found then NotFoundInConfigFile will be returned. // // On Windows avoid single character remote names as they can be mixed // up with drive letters. func NewFs(ctx context.Context, name string, root string, config configmap.Mapper) (fs.Fs, error) { opt := new(Options) err := configstruct.Set(config, opt) if err != nil { return nil, err } // If using a Shared Folder override root if opt.SharedFolder != "" { root = "" } //workaround for wonky parser root = strings.Trim(root, "/") f := &Fs{ name: name, root: root, opt: *opt, pacer: fs.NewPacer(ctx, pacer.NewDefault(pacer.MinSleep(minSleep), pacer.MaxSleep(maxSleep), pacer.DecayConstant(decayConstant), pacer.AttackConstant(attackConstant))), baseClient: &http.Client{}, } f.features = (&fs.Features{ DuplicateFiles: true, CanHaveEmptyDirectories: true, ReadMimeType: true, }).Fill(ctx, f) client := fshttp.NewClient(ctx) f.rest = rest.NewClient(client).SetRoot(apiBaseURL) f.rest.SetHeader("Authorization", "Bearer "+f.opt.APIKey) f.dirCache = dircache.New(root, rootID, f) // Find the current root err = f.dirCache.FindRoot(ctx, false) if err != nil { // Assume it is a file newRoot, remote := dircache.SplitPath(root) tempF := *f tempF.dirCache = dircache.New(newRoot, rootID, &tempF) tempF.root = newRoot // Make new Fs which is the parent err = tempF.dirCache.FindRoot(ctx, false) if err != nil { // No root so return old f return f, nil } _, err := tempF.NewObject(ctx, remote) if err != nil { if err == fs.ErrorObjectNotFound { // File doesn't exist so return old f return f, nil } return nil, err } f.features.Fill(ctx, &tempF) // XXX: update the old f here instead of returning tempF, since // `features` were already filled with functions having *f as a receiver. // See https://github.com/rclone/rclone/issues/2182 f.dirCache = tempF.dirCache f.root = tempF.root // return an error with an fs which points to the parent return f, fs.ErrorIsFile } return f, nil } // List the objects and directories in dir into entries. The // entries can be returned in any order but should be for a // complete directory. // // dir should be "" to list the root, and should not have // trailing slashes. // // This should return ErrDirNotFound if the directory isn't // found. func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err error) { if f.opt.SharedFolder != "" { return f.listSharedFiles(ctx, f.opt.SharedFolder) } dirContent, err := f.listDir(ctx, dir) if err != nil { return nil, err } return dirContent, nil } // NewObject finds the Object at remote. If it can't be found // it returns the error ErrorObjectNotFound. func (f *Fs) NewObject(ctx context.Context, remote string) (fs.Object, error) { leaf, directoryID, err := f.dirCache.FindPath(ctx, remote, false) if err != nil { if err == fs.ErrorDirNotFound { return nil, fs.ErrorObjectNotFound } return nil, err } folderID, err := strconv.Atoi(directoryID) if err != nil { return nil, err } files, err := f.listFiles(ctx, folderID) if err != nil { return nil, err } for _, file := range files.Items { if file.Filename == leaf { path, ok := f.dirCache.GetInv(directoryID) if !ok { return nil, errors.New("cannot find dir in dircache") } return f.newObjectFromFile(ctx, path, file), nil } } return nil, fs.ErrorObjectNotFound } // Put in to the remote path with the modTime given of the given size // // When called from outside an Fs by rclone, src.Size() will always be >= 0. // But for unknown-sized objects (indicated by src.Size() == -1), Put should either // return an error or upload it properly (rather than e.g. calling panic). // // May create the object even if it returns an error - if so // will return the object and the error, otherwise will return // nil and the error func (f *Fs) Put(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) { existingObj, err := f.NewObject(ctx, src.Remote()) switch err { case nil: return existingObj, existingObj.Update(ctx, in, src, options...) case fs.ErrorObjectNotFound: // Not found so create it return f.PutUnchecked(ctx, in, src, options...) default: return nil, err } } // putUnchecked uploads the object with the given name and size // // This will create a duplicate if we upload a new file without // checking to see if there is one already - use Put() for that. func (f *Fs) putUnchecked(ctx context.Context, in io.Reader, remote string, size int64, options ...fs.OpenOption) (fs.Object, error) { if size > int64(300e9) { return nil, errors.New("File too big, can't upload") } else if size == 0 { return nil, fs.ErrorCantUploadEmptyFiles } nodeResponse, err := f.getUploadNode(ctx) if err != nil { return nil, err } leaf, directoryID, err := f.dirCache.FindPath(ctx, remote, true) if err != nil { return nil, err } _, err = f.uploadFile(ctx, in, size, leaf, directoryID, nodeResponse.ID, nodeResponse.URL, options...) if err != nil { return nil, err } fileUploadResponse, err := f.endUpload(ctx, nodeResponse.ID, nodeResponse.URL) if err != nil { return nil, err } if len(fileUploadResponse.Links) == 0 { return nil, errors.New("upload response not found") } else if len(fileUploadResponse.Links) > 1 { fs.Debugf(remote, "Multiple upload responses found, using the first") } link := fileUploadResponse.Links[0] fileSize, err := strconv.ParseInt(link.Size, 10, 64) if err != nil { return nil, err } return &Object{ fs: f, remote: remote, file: File{ CDN: 0, Checksum: link.Whirlpool, ContentType: "", Date: time.Now().Format("2006-01-02 15:04:05"), Filename: link.Filename, Pass: 0, Size: fileSize, URL: link.Download, }, }, nil } // PutUnchecked uploads the object // // This will create a duplicate if we upload a new file without // checking to see if there is one already - use Put() for that. func (f *Fs) PutUnchecked(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) { return f.putUnchecked(ctx, in, src.Remote(), src.Size(), options...) } // Mkdir makes the directory (container, bucket) // // Shouldn't return an error if it already exists func (f *Fs) Mkdir(ctx context.Context, dir string) error { _, err := f.dirCache.FindDir(ctx, dir, true) return err } // Rmdir removes the directory (container, bucket) if empty // // Return an error if it doesn't exist or isn't empty func (f *Fs) Rmdir(ctx context.Context, dir string) error { directoryID, err := f.dirCache.FindDir(ctx, dir, false) if err != nil { return err } folderID, err := strconv.Atoi(directoryID) if err != nil { return err } _, err = f.removeFolder(ctx, dir, folderID) if err != nil { return err } f.dirCache.FlushDir(dir) return nil } // Move src to this remote using server side move operations. func (f *Fs) Move(ctx context.Context, src fs.Object, remote string) (fs.Object, error) { srcObj, ok := src.(*Object) if !ok { fs.Debugf(src, "Can't move - not same remote type") return nil, fs.ErrorCantMove } // Find current directory ID _, currentDirectoryID, err := f.dirCache.FindPath(ctx, remote, false) if err != nil { return nil, err } // Create temporary object dstObj, leaf, directoryID, err := f.createObject(ctx, remote) if err != nil { return nil, err } // If it is in the correct directory, just rename it var url string if currentDirectoryID == directoryID { resp, err := f.renameFile(ctx, srcObj.file.URL, leaf) if err != nil { return nil, fmt.Errorf("couldn't rename file: %w", err) } if resp.Status != "OK" { return nil, fmt.Errorf("couldn't rename file: %s", resp.Message) } url = resp.URLs[0].URL } else { folderID, err := strconv.Atoi(directoryID) if err != nil { return nil, err } resp, err := f.moveFile(ctx, srcObj.file.URL, folderID, leaf) if err != nil { return nil, fmt.Errorf("couldn't move file: %w", err) } if resp.Status != "OK" { return nil, fmt.Errorf("couldn't move file: %s", resp.Message) } url = resp.URLs[0] } file, err := f.readFileInfo(ctx, url) if err != nil { return nil, errors.New("couldn't read file data") } dstObj.setMetaData(*file) return dstObj, nil } // DirMove moves src, srcRemote to this remote at dstRemote // using server-side move operations. // // Will only be called if src.Fs().Name() == f.Name() // // If it isn't possible then return fs.ErrorCantDirMove. // // If destination exists then return fs.ErrorDirExists. // // This is complicated by the fact that we can't use moveDir to move // to a different directory AND rename at the same time as it can // overwrite files in the source directory. func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string) error { srcFs, ok := src.(*Fs) if !ok { fs.Debugf(srcFs, "Can't move directory - not same remote type") return fs.ErrorCantDirMove } srcID, _, _, dstDirectoryID, dstLeaf, err := f.dirCache.DirMove(ctx, srcFs.dirCache, srcFs.root, srcRemote, f.root, dstRemote) if err != nil { return err } srcIDnumeric, err := strconv.Atoi(srcID) if err != nil { return err } dstDirectoryIDnumeric, err := strconv.Atoi(dstDirectoryID) if err != nil { return err } var resp *MoveDirResponse resp, err = f.moveDir(ctx, srcIDnumeric, dstLeaf, dstDirectoryIDnumeric) if err != nil { return fmt.Errorf("couldn't rename leaf: %w", err) } if resp.Status != "OK" { return fmt.Errorf("couldn't rename leaf: %s", resp.Message) } srcFs.dirCache.FlushDir(srcRemote) return nil } // Copy src to this remote using server side move operations. func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object, error) { srcObj, ok := src.(*Object) if !ok { fs.Debugf(src, "Can't move - not same remote type") return nil, fs.ErrorCantMove } // Create temporary object dstObj, leaf, directoryID, err := f.createObject(ctx, remote) if err != nil { return nil, err } folderID, err := strconv.Atoi(directoryID) if err != nil { return nil, err } resp, err := f.copyFile(ctx, srcObj.file.URL, folderID, leaf) if err != nil { return nil, fmt.Errorf("couldn't move file: %w", err) } if resp.Status != "OK" { return nil, fmt.Errorf("couldn't move file: %s", resp.Message) } file, err := f.readFileInfo(ctx, resp.URLs[0].ToURL) if err != nil { return nil, errors.New("couldn't read file data") } dstObj.setMetaData(*file) return dstObj, nil } // About gets quota information func (f *Fs) About(ctx context.Context) (usage *fs.Usage, err error) { opts := rest.Opts{ Method: "POST", Path: "/user/info.cgi", ContentType: "application/json", } var accountInfo AccountInfo var resp *http.Response err = f.pacer.Call(func() (bool, error) { resp, err = f.rest.CallJSON(ctx, &opts, nil, &accountInfo) return shouldRetry(ctx, resp, err) }) if err != nil { return nil, fmt.Errorf("failed to read user info: %w", err) } // FIXME max upload size would be useful to use in Update usage = &fs.Usage{ Used: fs.NewUsageValue(accountInfo.ColdStorage), // bytes in use Total: fs.NewUsageValue(accountInfo.AvailableColdStorage), // bytes total Free: fs.NewUsageValue(accountInfo.AvailableColdStorage - accountInfo.ColdStorage), // bytes free } return usage, nil } // PublicLink adds a "readable by anyone with link" permission on the given file or folder. func (f *Fs) PublicLink(ctx context.Context, remote string, expire fs.Duration, unlink bool) (string, error) { o, err := f.NewObject(ctx, remote) if err != nil { return "", err } return o.(*Object).file.URL, nil } // Check the interfaces are satisfied var ( _ fs.Fs = (*Fs)(nil) _ fs.Mover = (*Fs)(nil) _ fs.DirMover = (*Fs)(nil) _ fs.Copier = (*Fs)(nil) _ fs.PublicLinker = (*Fs)(nil) _ fs.PutUncheckeder = (*Fs)(nil) _ dircache.DirCacher = (*Fs)(nil) )