2019-02-22 23:50:04 +08:00
package koofr
import (
2019-06-17 16:34:30 +08:00
"context"
2019-02-22 23:50:04 +08:00
"encoding/base64"
"errors"
"fmt"
"io"
"net/http"
"path"
"strings"
"time"
2019-07-29 01:47:38 +08:00
"github.com/rclone/rclone/fs"
2020-01-15 01:33:35 +08:00
"github.com/rclone/rclone/fs/config"
2019-07-29 01:47:38 +08:00
"github.com/rclone/rclone/fs/config/configmap"
"github.com/rclone/rclone/fs/config/configstruct"
"github.com/rclone/rclone/fs/config/obscure"
2019-11-14 19:11:29 +08:00
"github.com/rclone/rclone/fs/fshttp"
2019-07-29 01:47:38 +08:00
"github.com/rclone/rclone/fs/hash"
2020-01-15 01:33:35 +08:00
"github.com/rclone/rclone/lib/encoder"
2019-02-22 23:50:04 +08:00
httpclient "github.com/koofr/go-httpclient"
koofrclient "github.com/koofr/go-koofrclient"
)
// Register Fs with rclone
func init ( ) {
fs . Register ( & fs . RegInfo {
Name : "koofr" ,
Description : "Koofr" ,
NewFs : NewFs ,
2020-01-15 01:33:35 +08:00
Options : [ ] fs . Option { {
Name : "endpoint" ,
2021-08-16 17:30:01 +08:00
Help : "The Koofr API endpoint to use." ,
2020-01-15 01:33:35 +08:00
Default : "https://app.koofr.net" ,
Advanced : true ,
} , {
Name : "mountid" ,
2021-08-16 17:30:01 +08:00
Help : "Mount ID of the mount to use.\n\nIf omitted, the primary mount is used." ,
2020-01-15 01:33:35 +08:00
Advanced : true ,
} , {
Name : "setmtime" ,
2021-08-16 17:30:01 +08:00
Help : "Does the backend support setting modification time.\n\nSet this to false if you use a mount ID that points to a Dropbox or Amazon Drive backend." ,
2020-01-15 01:33:35 +08:00
Default : true ,
Advanced : true ,
} , {
Name : "user" ,
2021-08-16 17:30:01 +08:00
Help : "Your Koofr user name." ,
2020-01-15 01:33:35 +08:00
Required : true ,
} , {
Name : "password" ,
2021-08-16 17:30:01 +08:00
Help : "Your Koofr password for rclone (generate one at https://app.koofr.net/app/admin/preferences/password)." ,
2020-01-15 01:33:35 +08:00
IsPassword : true ,
Required : true ,
} , {
Name : config . ConfigEncoding ,
Help : config . ConfigEncodingHelp ,
Advanced : true ,
2020-01-15 05:51:49 +08:00
// Encode invalid UTF-8 bytes as json doesn't handle them properly.
Default : ( encoder . Display |
encoder . EncodeBackSlash |
encoder . EncodeInvalidUtf8 ) ,
2020-01-15 01:33:35 +08:00
} } ,
2019-02-22 23:50:04 +08:00
} )
}
// Options represent the configuration of the Koofr backend
type Options struct {
2020-01-15 01:33:35 +08:00
Endpoint string ` config:"endpoint" `
MountID string ` config:"mountid" `
User string ` config:"user" `
Password string ` config:"password" `
SetMTime bool ` config:"setmtime" `
Enc encoder . MultiEncoder ` config:"encoding" `
2019-02-22 23:50:04 +08:00
}
2020-05-25 14:05:53 +08:00
// An Fs is a representation of a remote Koofr Fs
2019-02-22 23:50:04 +08:00
type Fs struct {
name string
mountID string
root string
opt Options
features * fs . Features
client * koofrclient . KoofrClient
}
// An Object on the remote Koofr Fs
type Object struct {
fs * Fs
remote string
info koofrclient . FileInfo
}
func base ( pth string ) string {
rv := path . Base ( pth )
if rv == "" || rv == "." {
rv = "/"
}
return rv
}
func dir ( pth string ) string {
rv := path . Dir ( pth )
if rv == "" || rv == "." {
rv = "/"
}
return rv
}
// String returns a string representation of the remote Object
func ( o * Object ) String ( ) string {
return o . remote
}
// Remote returns the remote path of the Object, relative to Fs root
func ( o * Object ) Remote ( ) string {
return o . remote
}
// ModTime returns the modification time of the Object
2019-06-17 16:34:30 +08:00
func ( o * Object ) ModTime ( ctx context . Context ) time . Time {
2019-02-22 23:50:04 +08:00
return time . Unix ( o . info . Modified / 1000 , ( o . info . Modified % 1000 ) * 1000 * 1000 )
}
// Size return the size of the Object in bytes
func ( o * Object ) Size ( ) int64 {
return o . info . Size
}
// Fs returns a reference to the Koofr Fs containing the Object
func ( o * Object ) Fs ( ) fs . Info {
return o . fs
}
// Hash returns an MD5 hash of the Object
2019-06-17 16:34:30 +08:00
func ( o * Object ) Hash ( ctx context . Context , typ hash . Type ) ( string , error ) {
2019-02-22 23:50:04 +08:00
if typ == hash . MD5 {
return o . info . Hash , nil
}
return "" , nil
}
// fullPath returns full path of the remote Object (including Fs root)
func ( o * Object ) fullPath ( ) string {
return o . fs . fullPath ( o . remote )
}
// Storable returns true if the Object is storable
func ( o * Object ) Storable ( ) bool {
return true
}
// SetModTime is not supported
2019-06-17 16:34:30 +08:00
func ( o * Object ) SetModTime ( ctx context . Context , mtime time . Time ) error {
2019-07-15 20:57:35 +08:00
return fs . ErrorCantSetModTimeWithoutDelete
2019-02-22 23:50:04 +08:00
}
// Open opens the Object for reading
2019-06-17 16:34:30 +08:00
func ( o * Object ) Open ( ctx context . Context , options ... fs . OpenOption ) ( io . ReadCloser , error ) {
2019-02-22 23:50:04 +08:00
var sOff , eOff int64 = 0 , - 1
2019-08-06 22:18:08 +08:00
fs . FixRangeOption ( options , o . Size ( ) )
2019-02-22 23:50:04 +08:00
for _ , option := range options {
switch x := option . ( type ) {
case * fs . SeekOption :
sOff = x . Offset
case * fs . RangeOption :
sOff = x . Start
eOff = x . End
default :
if option . Mandatory ( ) {
fs . Logf ( o , "Unsupported mandatory option: %v" , option )
}
}
}
if sOff == 0 && eOff < 0 {
return o . fs . client . FilesGet ( o . fs . mountID , o . fullPath ( ) )
}
span := & koofrclient . FileSpan {
Start : sOff ,
End : eOff ,
}
return o . fs . client . FilesGetRange ( o . fs . mountID , o . fullPath ( ) , span )
}
// Update updates the Object contents
2019-06-17 16:34:30 +08:00
func ( o * Object ) Update ( ctx context . Context , in io . Reader , src fs . ObjectInfo , options ... fs . OpenOption ) error {
2019-07-15 20:57:35 +08:00
mtime := src . ModTime ( ctx ) . UnixNano ( ) / 1000 / 1000
putopts := & koofrclient . PutOptions {
ForceOverwrite : true ,
NoRename : true ,
OverwriteIgnoreNonExisting : true ,
SetModified : & mtime ,
2019-02-22 23:50:04 +08:00
}
fullPath := o . fullPath ( )
dirPath := dir ( fullPath )
name := base ( fullPath )
err := o . fs . mkdir ( dirPath )
if err != nil {
return err
}
2019-07-15 20:57:35 +08:00
info , err := o . fs . client . FilesPutWithOptions ( o . fs . mountID , dirPath , name , in , putopts )
2019-02-22 23:50:04 +08:00
if err != nil {
return err
}
o . info = * info
return nil
}
// Remove deletes the remote Object
2019-06-17 16:34:30 +08:00
func ( o * Object ) Remove ( ctx context . Context ) error {
2019-02-22 23:50:04 +08:00
return o . fs . client . FilesDelete ( o . fs . mountID , o . fullPath ( ) )
}
// Name returns the name of the Fs
func ( f * Fs ) Name ( ) string {
return f . name
}
// Root returns the root path of the Fs
func ( f * Fs ) Root ( ) string {
return f . root
}
// String returns a string representation of the Fs
func ( f * Fs ) String ( ) string {
return "koofr:" + f . mountID + ":" + f . root
}
// Features returns the optional features supported by this Fs
func ( f * Fs ) Features ( ) * fs . Features {
return f . features
}
// Precision denotes that setting modification times is not supported
func ( f * Fs ) Precision ( ) time . Duration {
2019-07-15 20:57:35 +08:00
if ! f . opt . SetMTime {
return fs . ModTimeNotSupported
}
return time . Millisecond
2019-02-22 23:50:04 +08:00
}
// Hashes returns a set of hashes are Provided by the Fs
func ( f * Fs ) Hashes ( ) hash . Set {
return hash . Set ( hash . MD5 )
}
2020-05-25 14:05:53 +08:00
// fullPath constructs a full, absolute path from an Fs root relative path,
2019-02-22 23:50:04 +08:00
func ( f * Fs ) fullPath ( part string ) string {
2020-01-15 01:33:35 +08:00
return f . opt . Enc . FromStandardPath ( path . Join ( "/" , f . root , part ) )
2019-02-22 23:50:04 +08:00
}
// NewFs constructs a new filesystem given a root path and configuration options
2020-11-05 23:18:51 +08:00
func NewFs ( ctx context . Context , name , root string , m configmap . Mapper ) ( ff fs . Fs , err error ) {
2019-02-22 23:50:04 +08:00
opt := new ( Options )
err = configstruct . Set ( m , opt )
if err != nil {
return nil , err
}
pass , err := obscure . Reveal ( opt . Password )
if err != nil {
return nil , err
}
2019-11-14 19:11:29 +08:00
httpClient := httpclient . New ( )
2020-11-13 23:24:43 +08:00
httpClient . Client = fshttp . NewClient ( ctx )
2019-11-14 19:11:29 +08:00
client := koofrclient . NewKoofrClientWithHTTPClient ( opt . Endpoint , httpClient )
2019-02-22 23:50:04 +08:00
basicAuth := fmt . Sprintf ( "Basic %s" ,
base64 . StdEncoding . EncodeToString ( [ ] byte ( opt . User + ":" + pass ) ) )
client . HTTPClient . Headers . Set ( "Authorization" , basicAuth )
mounts , err := client . Mounts ( )
if err != nil {
return nil , err
}
f := & Fs {
name : name ,
root : root ,
opt : * opt ,
client : client ,
}
f . features = ( & fs . Features {
CaseInsensitive : true ,
DuplicateFiles : false ,
BucketBased : false ,
CanHaveEmptyDirectories : true ,
2020-11-06 00:00:40 +08:00
} ) . Fill ( ctx , f )
2019-02-22 23:50:04 +08:00
for _ , m := range mounts {
if opt . MountID != "" {
if m . Id == opt . MountID {
f . mountID = m . Id
break
}
} else if m . IsPrimary {
f . mountID = m . Id
break
}
}
if f . mountID == "" {
if opt . MountID == "" {
return nil , errors . New ( "Failed to find primary mount" )
}
return nil , errors . New ( "Failed to find mount " + opt . MountID )
}
2020-01-15 01:33:35 +08:00
rootFile , err := f . client . FilesInfo ( f . mountID , f . opt . Enc . FromStandardPath ( "/" + f . root ) )
2019-02-22 23:50:04 +08:00
if err == nil && rootFile . Type != "dir" {
f . root = dir ( f . root )
err = fs . ErrorIsFile
} else {
err = nil
}
return f , err
}
// List returns a list of items in a directory
2019-06-17 16:34:30 +08:00
func ( f * Fs ) List ( ctx context . Context , dir string ) ( entries fs . DirEntries , err error ) {
2019-02-22 23:50:04 +08:00
files , err := f . client . FilesList ( f . mountID , f . fullPath ( dir ) )
if err != nil {
return nil , translateErrorsDir ( err )
}
entries = make ( [ ] fs . DirEntry , len ( files ) )
for i , file := range files {
2020-01-15 01:33:35 +08:00
remote := path . Join ( dir , f . opt . Enc . ToStandardName ( file . Name ) )
2019-02-22 23:50:04 +08:00
if file . Type == "dir" {
2019-05-19 23:55:26 +08:00
entries [ i ] = fs . NewDir ( remote , time . Unix ( 0 , 0 ) )
2019-02-22 23:50:04 +08:00
} else {
entries [ i ] = & Object {
fs : f ,
info : file ,
2019-05-19 23:55:26 +08:00
remote : remote ,
2019-02-22 23:50:04 +08:00
}
}
}
return entries , nil
}
// NewObject creates a new remote Object for a given remote path
2019-06-17 16:34:30 +08:00
func ( f * Fs ) NewObject ( ctx context . Context , remote string ) ( obj fs . Object , err error ) {
2019-02-22 23:50:04 +08:00
info , err := f . client . FilesInfo ( f . mountID , f . fullPath ( remote ) )
if err != nil {
return nil , translateErrorsObject ( err )
}
if info . Type == "dir" {
2021-09-06 20:54:08 +08:00
return nil , fs . ErrorIsDir
2019-02-22 23:50:04 +08:00
}
return & Object {
fs : f ,
info : info ,
remote : remote ,
} , nil
}
// Put updates a remote Object
2019-06-17 16:34:30 +08:00
func ( f * Fs ) Put ( ctx context . Context , in io . Reader , src fs . ObjectInfo , options ... fs . OpenOption ) ( obj fs . Object , err error ) {
2019-07-15 20:57:35 +08:00
mtime := src . ModTime ( ctx ) . UnixNano ( ) / 1000 / 1000
putopts := & koofrclient . PutOptions {
ForceOverwrite : true ,
NoRename : true ,
OverwriteIgnoreNonExisting : true ,
SetModified : & mtime ,
2019-02-22 23:50:04 +08:00
}
fullPath := f . fullPath ( src . Remote ( ) )
dirPath := dir ( fullPath )
name := base ( fullPath )
err = f . mkdir ( dirPath )
if err != nil {
return nil , err
}
2019-07-15 20:57:35 +08:00
info , err := f . client . FilesPutWithOptions ( f . mountID , dirPath , name , in , putopts )
2019-02-22 23:50:04 +08:00
if err != nil {
return nil , translateErrorsObject ( err )
}
return & Object {
fs : f ,
info : * info ,
remote : src . Remote ( ) ,
} , nil
}
// PutStream updates a remote Object with a stream of unknown size
2019-06-17 16:34:30 +08:00
func ( f * Fs ) PutStream ( ctx context . Context , in io . Reader , src fs . ObjectInfo , options ... fs . OpenOption ) ( fs . Object , error ) {
return f . Put ( ctx , in , src , options ... )
2019-02-22 23:50:04 +08:00
}
// isBadRequest is a predicate which holds true iff the error returned was
// HTTP status 400
func isBadRequest ( err error ) bool {
switch err := err . ( type ) {
case httpclient . InvalidStatusError :
if err . Got == http . StatusBadRequest {
return true
}
}
return false
}
// translateErrorsDir translates koofr errors to rclone errors (for a dir
// operation)
func translateErrorsDir ( err error ) error {
switch err := err . ( type ) {
case httpclient . InvalidStatusError :
if err . Got == http . StatusNotFound {
return fs . ErrorDirNotFound
}
}
return err
}
// translatesErrorsObject translates Koofr errors to rclone errors (for an object operation)
func translateErrorsObject ( err error ) error {
switch err := err . ( type ) {
case httpclient . InvalidStatusError :
if err . Got == http . StatusNotFound {
return fs . ErrorObjectNotFound
}
}
return err
}
// mkdir creates a directory at the given remote path. Creates ancestors if
2020-05-20 18:39:20 +08:00
// necessary
2019-02-22 23:50:04 +08:00
func ( f * Fs ) mkdir ( fullPath string ) error {
if fullPath == "/" {
return nil
}
info , err := f . client . FilesInfo ( f . mountID , fullPath )
if err == nil && info . Type == "dir" {
return nil
}
err = translateErrorsDir ( err )
if err != nil && err != fs . ErrorDirNotFound {
return err
}
dirs := strings . Split ( fullPath , "/" )
parent := "/"
for _ , part := range dirs {
if part == "" {
continue
}
info , err = f . client . FilesInfo ( f . mountID , path . Join ( parent , part ) )
if err != nil || info . Type != "dir" {
err = translateErrorsDir ( err )
if err != nil && err != fs . ErrorDirNotFound {
return err
}
err = f . client . FilesNewFolder ( f . mountID , parent , part )
if err != nil && ! isBadRequest ( err ) {
return err
}
}
parent = path . Join ( parent , part )
}
return nil
}
// Mkdir creates a directory at the given remote path. Creates ancestors if
// necessary
2019-06-17 16:34:30 +08:00
func ( f * Fs ) Mkdir ( ctx context . Context , dir string ) error {
2019-02-22 23:50:04 +08:00
fullPath := f . fullPath ( dir )
return f . mkdir ( fullPath )
}
// Rmdir removes an (empty) directory at the given remote path
2019-06-17 16:34:30 +08:00
func ( f * Fs ) Rmdir ( ctx context . Context , dir string ) error {
2019-02-22 23:50:04 +08:00
files , err := f . client . FilesList ( f . mountID , f . fullPath ( dir ) )
if err != nil {
return translateErrorsDir ( err )
}
if len ( files ) > 0 {
return fs . ErrorDirectoryNotEmpty
}
err = f . client . FilesDelete ( f . mountID , f . fullPath ( dir ) )
if err != nil {
return translateErrorsDir ( err )
}
return nil
}
// Copy copies a remote Object to the given path
2019-06-17 16:34:30 +08:00
func ( f * Fs ) Copy ( ctx context . Context , src fs . Object , remote string ) ( fs . Object , error ) {
2019-02-22 23:50:04 +08:00
dstFullPath := f . fullPath ( remote )
dstDir := dir ( dstFullPath )
err := f . mkdir ( dstDir )
if err != nil {
return nil , fs . ErrorCantCopy
}
2019-07-15 20:57:35 +08:00
mtime := src . ModTime ( ctx ) . UnixNano ( ) / 1000 / 1000
2019-02-22 23:50:04 +08:00
err = f . client . FilesCopy ( ( src . ( * Object ) ) . fs . mountID ,
( src . ( * Object ) ) . fs . fullPath ( ( src . ( * Object ) ) . remote ) ,
2019-07-15 20:57:35 +08:00
f . mountID , dstFullPath , koofrclient . CopyOptions { SetModified : & mtime } )
2019-02-22 23:50:04 +08:00
if err != nil {
return nil , fs . ErrorCantCopy
}
2019-06-17 16:34:30 +08:00
return f . NewObject ( ctx , remote )
2019-02-22 23:50:04 +08:00
}
// Move moves a remote Object to the given path
2019-06-17 16:34:30 +08:00
func ( f * Fs ) Move ( ctx context . Context , src fs . Object , remote string ) ( fs . Object , error ) {
2019-02-22 23:50:04 +08:00
srcObj := src . ( * Object )
dstFullPath := f . fullPath ( remote )
dstDir := dir ( dstFullPath )
err := f . mkdir ( dstDir )
if err != nil {
return nil , fs . ErrorCantMove
}
err = f . client . FilesMove ( srcObj . fs . mountID ,
srcObj . fs . fullPath ( srcObj . remote ) , f . mountID , dstFullPath )
if err != nil {
return nil , fs . ErrorCantMove
}
2019-06-17 16:34:30 +08:00
return f . NewObject ( ctx , remote )
2019-02-22 23:50:04 +08:00
}
// DirMove moves a remote directory to the given path
2019-06-17 16:34:30 +08:00
func ( f * Fs ) DirMove ( ctx context . Context , src fs . Fs , srcRemote , dstRemote string ) error {
2019-02-22 23:50:04 +08:00
srcFs := src . ( * Fs )
srcFullPath := srcFs . fullPath ( srcRemote )
dstFullPath := f . fullPath ( dstRemote )
if srcFs . mountID == f . mountID && srcFullPath == dstFullPath {
return fs . ErrorDirExists
}
dstDir := dir ( dstFullPath )
err := f . mkdir ( dstDir )
if err != nil {
return fs . ErrorCantDirMove
}
err = f . client . FilesMove ( srcFs . mountID , srcFullPath , f . mountID , dstFullPath )
if err != nil {
return fs . ErrorCantDirMove
}
return nil
}
2021-03-03 03:11:57 +08:00
// About reports space usage (with a MiB precision)
2019-06-17 16:34:30 +08:00
func ( f * Fs ) About ( ctx context . Context ) ( * fs . Usage , error ) {
2019-02-22 23:50:04 +08:00
mount , err := f . client . MountsDetails ( f . mountID )
if err != nil {
return nil , err
}
return & fs . Usage {
Total : fs . NewUsageValue ( mount . SpaceTotal * 1024 * 1024 ) ,
Used : fs . NewUsageValue ( mount . SpaceUsed * 1024 * 1024 ) ,
Trashed : nil ,
Other : nil ,
Free : fs . NewUsageValue ( ( mount . SpaceTotal - mount . SpaceUsed ) * 1024 * 1024 ) ,
Objects : nil ,
} , nil
}
// Purge purges the complete Fs
2019-06-17 16:34:30 +08:00
func ( f * Fs ) Purge ( ctx context . Context ) error {
2019-02-22 23:50:04 +08:00
err := translateErrorsDir ( f . client . FilesDelete ( f . mountID , f . fullPath ( "" ) ) )
return err
}
// linkCreate is a Koofr API request for creating a public link
type linkCreate struct {
Path string ` json:"path" `
}
// link is a Koofr API response to creating a public link
type link struct {
ID string ` json:"id" `
Name string ` json:"name" `
Path string ` json:"path" `
Counter int64 ` json:"counter" `
URL string ` json:"url" `
ShortURL string ` json:"shortUrl" `
Hash string ` json:"hash" `
Host string ` json:"host" `
HasPassword bool ` json:"hasPassword" `
Password string ` json:"password" `
ValidFrom int64 ` json:"validFrom" `
ValidTo int64 ` json:"validTo" `
PasswordRequired bool ` json:"passwordRequired" `
}
// createLink makes a Koofr API call to create a public link
func createLink ( c * koofrclient . KoofrClient , mountID string , path string ) ( * link , error ) {
linkCreate := linkCreate {
Path : path ,
}
linkData := link { }
request := httpclient . RequestData {
Method : "POST" ,
Path : "/api/v2/mounts/" + mountID + "/links" ,
ExpectedStatus : [ ] int { http . StatusOK , http . StatusCreated } ,
ReqEncoding : httpclient . EncodingJSON ,
ReqValue : linkCreate ,
RespEncoding : httpclient . EncodingJSON ,
RespValue : & linkData ,
}
_ , err := c . Request ( & request )
if err != nil {
return nil , err
}
return & linkData , nil
}
// PublicLink creates a public link to the remote path
2020-06-01 05:18:01 +08:00
func ( f * Fs ) PublicLink ( ctx context . Context , remote string , expire fs . Duration , unlink bool ) ( string , error ) {
2019-02-22 23:50:04 +08:00
linkData , err := createLink ( f . client , f . mountID , f . fullPath ( remote ) )
if err != nil {
return "" , translateErrorsDir ( err )
}
2021-06-10 08:00:00 +08:00
// URL returned by API looks like following:
//
// https://app.koofr.net/links/35d9fb92-74a3-4930-b4ed-57f123bfb1a6
//
// Direct url looks like following:
//
// https://app.koofr.net/content/links/39a6cc01-3b23-477a-8059-c0fb3b0f15de/files/get?path=%2F
//
// I am not sure about meaning of "path" parameter; in my expriments
// it is always "%2F", and omitting it or putting any other value
// results in 404.
//
// There is one more quirk: direct link to file in / returns that file,
// direct link to file somewhere else in hierarchy returns zip archive
// with one member.
link := linkData . URL
link = strings . ReplaceAll ( link , "/links" , "/content/links" )
link += "/files/get?path=%2F"
return link , nil
2019-02-22 23:50:04 +08:00
}