rclone/fs/features.go

772 lines
24 KiB
Go
Raw Normal View History

// Filesystem features and optional interfaces
package fs
import (
"context"
"io"
"reflect"
"strings"
"time"
)
// Features describe the optional features of the Fs
type Features struct {
// Feature flags, whether Fs
CaseInsensitive bool // has case insensitive files
DuplicateFiles bool // allows duplicate files
ReadMimeType bool // can read the mime type of objects
WriteMimeType bool // can set the mime type of objects
CanHaveEmptyDirectories bool // can have empty directories
BucketBased bool // is bucket based (like s3, swift, etc.)
BucketBasedRootOK bool // is bucket based and can use from root
SetTier bool // allows set tier functionality on objects
GetTier bool // allows to retrieve storage tier of objects
ServerSideAcrossConfigs bool // can server-side copy between different remotes of the same type
IsLocal bool // is the local backend
SlowModTime bool // if calling ModTime() generally takes an extra transaction
SlowHash bool // if calling Hash() generally takes an extra transaction
ReadMetadata bool // can read metadata from objects
WriteMetadata bool // can write metadata to objects
UserMetadata bool // can read/write general purpose metadata
FilterAware bool // can make use of filters if provided for listing
PartialUploads bool // uploaded file can appear incomplete on the fs while it's being uploaded
NoMultiThreading bool // set if can't have multiplethreads on one download open
Overlay bool // this wraps one or more backends to add functionality
ChunkWriterDoesntSeek bool // set if the chunk writer doesn't need to read the data more than once
// Purge all files in the directory specified
//
// Implement this if you have a way of deleting all the files
// quicker than just running Remove() on the result of List()
//
// Return an error if it doesn't exist
Purge func(ctx context.Context, dir string) error
// Copy src to this remote using server-side copy operations.
//
// This is stored with the remote path given
//
// It returns the destination Object and a possible error
//
// Will only be called if src.Fs().Name() == f.Name()
//
// If it isn't possible then return fs.ErrorCantCopy
Copy func(ctx context.Context, src Object, remote string) (Object, error)
// Move src to this remote using server-side move operations.
//
// This is stored with the remote path given
//
// It returns the destination Object and a possible error
//
// Will only be called if src.Fs().Name() == f.Name()
//
// If it isn't possible then return fs.ErrorCantMove
Move func(ctx context.Context, src Object, remote string) (Object, error)
// DirMove moves src, srcRemote to this remote at dstRemote
// using server-side move operations.
//
// Will only be called if src.Fs().Name() == f.Name()
//
// If it isn't possible then return fs.ErrorCantDirMove
//
// If destination exists then return fs.ErrorDirExists
DirMove func(ctx context.Context, src Fs, srcRemote, dstRemote string) error
// ChangeNotify calls the passed function with a path
// that has had changes. If the implementation
// uses polling, it should adhere to the given interval.
ChangeNotify func(context.Context, func(string, EntryType), <-chan time.Duration)
// UnWrap returns the Fs that this Fs is wrapping
UnWrap func() Fs
// WrapFs returns the Fs that is wrapping this Fs
WrapFs func() Fs
// SetWrapper sets the Fs that is wrapping this Fs
SetWrapper func(f Fs)
// DirCacheFlush resets the directory cache - used in testing
// as an optional interface
DirCacheFlush func()
// PublicLink generates a public link to the remote path (usually readable by anyone)
PublicLink func(ctx context.Context, remote string, expire Duration, unlink bool) (string, error)
// Put in to the remote path with the modTime given of the given size
//
// May create the object even if it returns an error - if so
// will return the object and the error, otherwise will return
// nil and the error
//
// May create duplicates or return errors if src already
// exists.
PutUnchecked func(ctx context.Context, in io.Reader, src ObjectInfo, options ...OpenOption) (Object, error)
// PutStream uploads to the remote path with the modTime given of indeterminate size
//
// May create the object even if it returns an error - if so
// will return the object and the error, otherwise will return
// nil and the error
PutStream func(ctx context.Context, in io.Reader, src ObjectInfo, options ...OpenOption) (Object, error)
// MergeDirs merges the contents of all the directories passed
// in into the first one and rmdirs the other directories.
MergeDirs func(ctx context.Context, dirs []Directory) error
// CleanUp the trash in the Fs
//
// Implement this if you have a way of emptying the trash or
// otherwise cleaning up old versions of files.
CleanUp func(ctx context.Context) error
// ListR lists the objects and directories of the Fs starting
// from dir recursively into out.
//
// dir should be "" to start from the root, and should not
// have trailing slashes.
//
// This should return ErrDirNotFound if the directory isn't
// found.
//
// It should call callback for each tranche of entries read.
// These need not be returned in any particular order. If
// callback returns an error then the listing will stop
// immediately.
//
// Don't implement this unless you have a more efficient way
// of listing recursively that doing a directory traversal.
ListR ListRFn
// About gets quota information from the Fs
About func(ctx context.Context) (*Usage, error)
// OpenWriterAt opens with a handle for random access writes
//
// Pass in the remote desired and the size if known.
//
// It truncates any existing object
OpenWriterAt func(ctx context.Context, remote string, size int64) (WriterAtCloser, error)
// OpenChunkWriter returns the chunk size and a ChunkWriter
//
// Pass in the remote and the src object
// You can also use options to hint at the desired chunk size
//
OpenChunkWriter func(ctx context.Context, remote string, src ObjectInfo, options ...OpenOption) (info ChunkWriterInfo, writer ChunkWriter, err error)
// UserInfo returns info about the connected user
UserInfo func(ctx context.Context) (map[string]string, error)
// Disconnect the current user
Disconnect func(ctx context.Context) error
// Command the backend to run a named command
//
// The command run is name
// args may be used to read arguments from
// opts may be used to read optional arguments from
//
// The result should be capable of being JSON encoded
// If it is a string or a []string it will be shown to the user
// otherwise it will be JSON encoded and shown to the user like that
Command func(ctx context.Context, name string, arg []string, opt map[string]string) (interface{}, error)
// Shutdown the backend, closing any background tasks and any
// cached connections.
Shutdown func(ctx context.Context) error
}
// Disable nil's out the named feature. If it isn't found then it
// will log a message.
func (ft *Features) Disable(name string) *Features {
// Prefix boolean values with ! to set the feature
invert := false
if strings.HasPrefix(name, "!") {
name = name[1:]
invert = true
}
v := reflect.ValueOf(ft).Elem()
vType := v.Type()
for i := 0; i < v.NumField(); i++ {
vName := vType.Field(i).Name
field := v.Field(i)
if strings.EqualFold(name, vName) {
if !field.CanSet() {
Errorf(nil, "Can't set Feature %q", name)
} else {
if invert {
if field.Type().Kind() == reflect.Bool {
field.Set(reflect.ValueOf(true))
Debugf(nil, "Set feature %q", name)
} else {
Errorf(nil, "Can't set Feature %q to true", name)
}
} else {
zero := reflect.Zero(field.Type())
field.Set(zero)
Debugf(nil, "Reset feature %q", name)
}
}
}
}
return ft
}
// List returns a slice of all the possible feature names
func (ft *Features) List() (out []string) {
v := reflect.ValueOf(ft).Elem()
vType := v.Type()
for i := 0; i < v.NumField(); i++ {
out = append(out, vType.Field(i).Name)
}
return out
}
// Enabled returns a map of features with keys showing whether they
// are enabled or not
func (ft *Features) Enabled() (features map[string]bool) {
v := reflect.ValueOf(ft).Elem()
vType := v.Type()
features = make(map[string]bool, v.NumField())
for i := 0; i < v.NumField(); i++ {
vName := vType.Field(i).Name
field := v.Field(i)
if field.Kind() == reflect.Func {
// Can't compare functions
features[vName] = !field.IsNil()
} else {
zero := reflect.Zero(field.Type())
features[vName] = field.Interface() != zero.Interface()
}
}
return features
}
// DisableList nil's out the comma separated list of named features.
// If it isn't found then it will log a message.
func (ft *Features) DisableList(list []string) *Features {
for _, feature := range list {
ft.Disable(strings.TrimSpace(feature))
}
return ft
}
// Fill fills in the function pointers in the Features struct from the
// optional interfaces. It returns the original updated Features
// struct passed in.
func (ft *Features) Fill(ctx context.Context, f Fs) *Features {
if do, ok := f.(Purger); ok {
ft.Purge = do.Purge
}
if do, ok := f.(Copier); ok {
ft.Copy = do.Copy
}
if do, ok := f.(Mover); ok {
ft.Move = do.Move
}
if do, ok := f.(DirMover); ok {
ft.DirMove = do.DirMove
}
if do, ok := f.(ChangeNotifier); ok {
ft.ChangeNotify = do.ChangeNotify
}
if do, ok := f.(UnWrapper); ok {
ft.UnWrap = do.UnWrap
}
if do, ok := f.(Wrapper); ok {
ft.WrapFs = do.WrapFs
ft.SetWrapper = do.SetWrapper
ft.Overlay = true // if implement UnWrap then must be an Overlay
}
if do, ok := f.(DirCacheFlusher); ok {
ft.DirCacheFlush = do.DirCacheFlush
}
if do, ok := f.(PublicLinker); ok {
ft.PublicLink = do.PublicLink
}
if do, ok := f.(PutUncheckeder); ok {
ft.PutUnchecked = do.PutUnchecked
}
if do, ok := f.(PutStreamer); ok {
ft.PutStream = do.PutStream
}
if do, ok := f.(MergeDirser); ok {
ft.MergeDirs = do.MergeDirs
}
if do, ok := f.(CleanUpper); ok {
ft.CleanUp = do.CleanUp
}
if do, ok := f.(ListRer); ok {
ft.ListR = do.ListR
}
if do, ok := f.(Abouter); ok {
ft.About = do.About
}
if do, ok := f.(OpenWriterAter); ok {
ft.OpenWriterAt = do.OpenWriterAt
}
if do, ok := f.(OpenChunkWriter); ok {
ft.OpenChunkWriter = do.OpenChunkWriter
}
if do, ok := f.(UserInfoer); ok {
ft.UserInfo = do.UserInfo
}
if do, ok := f.(Disconnecter); ok {
ft.Disconnect = do.Disconnect
}
if do, ok := f.(Commander); ok {
ft.Command = do.Command
}
if do, ok := f.(Shutdowner); ok {
ft.Shutdown = do.Shutdown
}
return ft.DisableList(GetConfig(ctx).DisableFeatures)
}
// Mask the Features with the Fs passed in
//
// Only optional features which are implemented in both the original
// Fs AND the one passed in will be advertised. Any features which
// aren't in both will be set to false/nil, except for UnWrap/Wrap which
// will be left untouched.
func (ft *Features) Mask(ctx context.Context, f Fs) *Features {
mask := f.Features()
ft.CaseInsensitive = ft.CaseInsensitive && mask.CaseInsensitive
ft.DuplicateFiles = ft.DuplicateFiles && mask.DuplicateFiles
ft.ReadMimeType = ft.ReadMimeType && mask.ReadMimeType
ft.WriteMimeType = ft.WriteMimeType && mask.WriteMimeType
ft.ReadMetadata = ft.ReadMetadata && mask.ReadMetadata
ft.WriteMetadata = ft.WriteMetadata && mask.WriteMetadata
ft.UserMetadata = ft.UserMetadata && mask.UserMetadata
ft.CanHaveEmptyDirectories = ft.CanHaveEmptyDirectories && mask.CanHaveEmptyDirectories
ft.BucketBased = ft.BucketBased && mask.BucketBased
ft.BucketBasedRootOK = ft.BucketBasedRootOK && mask.BucketBasedRootOK
ft.SetTier = ft.SetTier && mask.SetTier
ft.GetTier = ft.GetTier && mask.GetTier
ft.ServerSideAcrossConfigs = ft.ServerSideAcrossConfigs && mask.ServerSideAcrossConfigs
// ft.IsLocal = ft.IsLocal && mask.IsLocal Don't propagate IsLocal
ft.SlowModTime = ft.SlowModTime && mask.SlowModTime
ft.SlowHash = ft.SlowHash && mask.SlowHash
ft.FilterAware = ft.FilterAware && mask.FilterAware
ft.PartialUploads = ft.PartialUploads && mask.PartialUploads
ft.NoMultiThreading = ft.NoMultiThreading && mask.NoMultiThreading
// ft.Overlay = ft.Overlay && mask.Overlay don't propagate Overlay
if mask.Purge == nil {
ft.Purge = nil
}
if mask.Copy == nil {
ft.Copy = nil
}
if mask.Move == nil {
ft.Move = nil
}
if mask.DirMove == nil {
ft.DirMove = nil
}
if mask.ChangeNotify == nil {
ft.ChangeNotify = nil
}
// if mask.UnWrap == nil {
// ft.UnWrap = nil
// }
// if mask.Wrapper == nil {
// ft.Wrapper = nil
// }
if mask.DirCacheFlush == nil {
ft.DirCacheFlush = nil
}
if mask.PublicLink == nil {
ft.PublicLink = nil
}
if mask.PutUnchecked == nil {
ft.PutUnchecked = nil
}
if mask.PutStream == nil {
ft.PutStream = nil
}
if mask.MergeDirs == nil {
ft.MergeDirs = nil
}
if mask.CleanUp == nil {
ft.CleanUp = nil
}
if mask.ListR == nil {
ft.ListR = nil
}
if mask.About == nil {
ft.About = nil
}
if mask.OpenWriterAt == nil {
ft.OpenWriterAt = nil
}
if mask.OpenChunkWriter == nil {
ft.OpenChunkWriter = nil
}
if mask.UserInfo == nil {
ft.UserInfo = nil
}
if mask.Disconnect == nil {
ft.Disconnect = nil
}
// Command is always local so we don't mask it
if mask.Shutdown == nil {
ft.Shutdown = nil
}
return ft.DisableList(GetConfig(ctx).DisableFeatures)
}
// Wrap makes a Copy of the features passed in, overriding the UnWrap/Wrap
// method only if available in f.
func (ft *Features) Wrap(f Fs) *Features {
ftCopy := new(Features)
*ftCopy = *ft
if do, ok := f.(UnWrapper); ok {
ftCopy.UnWrap = do.UnWrap
}
if do, ok := f.(Wrapper); ok {
ftCopy.WrapFs = do.WrapFs
ftCopy.SetWrapper = do.SetWrapper
}
return ftCopy
}
// WrapsFs adds extra information between `f` which wraps `w`
func (ft *Features) WrapsFs(f Fs, w Fs) *Features {
wFeatures := w.Features()
if wFeatures.WrapFs != nil && wFeatures.SetWrapper != nil {
wFeatures.SetWrapper(f)
}
return ft
}
// Purger is an optional interfaces for Fs
type Purger interface {
// Purge all files in the directory specified
//
// Implement this if you have a way of deleting all the files
// quicker than just running Remove() on the result of List()
//
// Return an error if it doesn't exist
Purge(ctx context.Context, dir string) error
}
// Copier is an optional interface for Fs
type Copier interface {
// Copy src to this remote using server-side copy operations.
//
// This is stored with the remote path given
//
// It returns the destination Object and a possible error
//
// Will only be called if src.Fs().Name() == f.Name()
//
// If it isn't possible then return fs.ErrorCantCopy
Copy(ctx context.Context, src Object, remote string) (Object, error)
}
// Mover is an optional interface for Fs
type Mover interface {
// Move src to this remote using server-side move operations.
//
// This is stored with the remote path given
//
// It returns the destination Object and a possible error
//
// Will only be called if src.Fs().Name() == f.Name()
//
// If it isn't possible then return fs.ErrorCantMove
Move(ctx context.Context, src Object, remote string) (Object, error)
}
// DirMover is an optional interface for Fs
type DirMover interface {
// DirMove moves src, srcRemote to this remote at dstRemote
// using server-side move operations.
//
// Will only be called if src.Fs().Name() == f.Name()
//
// If it isn't possible then return fs.ErrorCantDirMove
//
// If destination exists then return fs.ErrorDirExists
DirMove(ctx context.Context, src Fs, srcRemote, dstRemote string) error
}
// ChangeNotifier is an optional interface for Fs
type ChangeNotifier interface {
// ChangeNotify calls the passed function with a path
// that has had changes. If the implementation
// uses polling, it should adhere to the given interval.
// At least one value will be written to the channel,
// specifying the initial value and updated values might
// follow. A 0 Duration should pause the polling.
// The ChangeNotify implementation must empty the channel
// regularly. When the channel gets closed, the implementation
// should stop polling and release resources.
ChangeNotify(context.Context, func(string, EntryType), <-chan time.Duration)
}
// EntryType can be associated with remote paths to identify their type
type EntryType int
// Constants
const (
// EntryDirectory should be used to classify remote paths in directories
EntryDirectory EntryType = iota // 0
// EntryObject should be used to classify remote paths in objects
EntryObject // 1
)
// UnWrapper is an optional interfaces for Fs
type UnWrapper interface {
// UnWrap returns the Fs that this Fs is wrapping
UnWrap() Fs
}
// Wrapper is an optional interfaces for Fs
type Wrapper interface {
// Wrap returns the Fs that is wrapping this Fs
WrapFs() Fs
// SetWrapper sets the Fs that is wrapping this Fs
SetWrapper(f Fs)
}
// DirCacheFlusher is an optional interface for Fs
type DirCacheFlusher interface {
// DirCacheFlush resets the directory cache - used in testing
// as an optional interface
DirCacheFlush()
}
// PutUncheckeder is an optional interface for Fs
type PutUncheckeder interface {
// Put in to the remote path with the modTime given of the given size
//
// May create the object even if it returns an error - if so
// will return the object and the error, otherwise will return
// nil and the error
//
// May create duplicates or return errors if src already
// exists.
PutUnchecked(ctx context.Context, in io.Reader, src ObjectInfo, options ...OpenOption) (Object, error)
}
// PutStreamer is an optional interface for Fs
type PutStreamer interface {
// PutStream uploads to the remote path with the modTime given of indeterminate size
//
// May create the object even if it returns an error - if so
// will return the object and the error, otherwise will return
// nil and the error
PutStream(ctx context.Context, in io.Reader, src ObjectInfo, options ...OpenOption) (Object, error)
}
// PublicLinker is an optional interface for Fs
type PublicLinker interface {
// PublicLink generates a public link to the remote path (usually readable by anyone)
PublicLink(ctx context.Context, remote string, expire Duration, unlink bool) (string, error)
}
// MergeDirser is an option interface for Fs
type MergeDirser interface {
// MergeDirs merges the contents of all the directories passed
// in into the first one and rmdirs the other directories.
MergeDirs(ctx context.Context, dirs []Directory) error
}
// CleanUpper is an optional interfaces for Fs
type CleanUpper interface {
// CleanUp the trash in the Fs
//
// Implement this if you have a way of emptying the trash or
// otherwise cleaning up old versions of files.
CleanUp(ctx context.Context) error
}
// ListRer is an optional interfaces for Fs
type ListRer interface {
// ListR lists the objects and directories of the Fs starting
// from dir recursively into out.
//
// dir should be "" to start from the root, and should not
// have trailing slashes.
//
// This should return ErrDirNotFound if the directory isn't
// found.
//
// It should call callback for each tranche of entries read.
// These need not be returned in any particular order. If
// callback returns an error then the listing will stop
// immediately.
//
// Don't implement this unless you have a more efficient way
// of listing recursively that doing a directory traversal.
ListR(ctx context.Context, dir string, callback ListRCallback) error
}
// RangeSeeker is the interface that wraps the RangeSeek method.
//
// Some of the returns from Object.Open() may optionally implement
// this method for efficiency purposes.
type RangeSeeker interface {
// RangeSeek behaves like a call to Seek(offset int64, whence
// int) with the output wrapped in an io.LimitedReader
// limiting the total length to limit.
//
// RangeSeek with a limit of < 0 is equivalent to a regular Seek.
RangeSeek(ctx context.Context, offset int64, whence int, length int64) (int64, error)
}
// Abouter is an optional interface for Fs
type Abouter interface {
// About gets quota information from the Fs
About(ctx context.Context) (*Usage, error)
}
// OpenWriterAter is an optional interface for Fs
type OpenWriterAter interface {
// OpenWriterAt opens with a handle for random access writes
//
// Pass in the remote desired and the size if known.
//
// It truncates any existing object
OpenWriterAt(ctx context.Context, remote string, size int64) (WriterAtCloser, error)
}
// OpenWriterAtFn describes the OpenWriterAt function pointer
type OpenWriterAtFn func(ctx context.Context, remote string, size int64) (WriterAtCloser, error)
// ChunkWriterInfo describes how a backend would like ChunkWriter called
type ChunkWriterInfo struct {
ChunkSize int64 // preferred chunk size
Concurrency int // how many chunks to write at once
LeavePartsOnError bool // if set don't delete parts uploaded so far on error
}
// OpenChunkWriter is an option interface for Fs to implement chunked writing
type OpenChunkWriter interface {
// OpenChunkWriter returns the chunk size and a ChunkWriter
//
// Pass in the remote and the src object
// You can also use options to hint at the desired chunk size
OpenChunkWriter(ctx context.Context, remote string, src ObjectInfo, options ...OpenOption) (info ChunkWriterInfo, writer ChunkWriter, err error)
}
// OpenChunkWriterFn describes the OpenChunkWriter function pointer
type OpenChunkWriterFn func(ctx context.Context, remote string, src ObjectInfo, options ...OpenOption) (info ChunkWriterInfo, writer ChunkWriter, err error)
// ChunkWriter is returned by OpenChunkWriter to implement chunked writing
type ChunkWriter interface {
// WriteChunk will write chunk number with reader bytes, where chunk number >= 0
WriteChunk(ctx context.Context, chunkNumber int, reader io.ReadSeeker) (bytesWritten int64, err error)
// Close complete chunked writer finalising the file.
Close(ctx context.Context) error
// Abort chunk write
//
// You can and should call Abort without calling Close.
Abort(ctx context.Context) error
}
// UserInfoer is an optional interface for Fs
type UserInfoer interface {
// UserInfo returns info about the connected user
UserInfo(ctx context.Context) (map[string]string, error)
}
// Disconnecter is an optional interface for Fs
type Disconnecter interface {
// Disconnect the current user
Disconnect(ctx context.Context) error
}
// CommandHelp describes a single backend Command
//
// These are automatically inserted in the docs
type CommandHelp struct {
Name string // Name of the command, e.g. "link"
Short string // Single line description
Long string // Long multi-line description
Opts map[string]string // maps option name to a single line help
}
// Commander is an interface to wrap the Command function
type Commander interface {
// Command the backend to run a named command
//
// The command run is name
// args may be used to read arguments from
// opts may be used to read optional arguments from
//
// The result should be capable of being JSON encoded
// If it is a string or a []string it will be shown to the user
// otherwise it will be JSON encoded and shown to the user like that
Command(ctx context.Context, name string, arg []string, opt map[string]string) (interface{}, error)
}
// Shutdowner is an interface to wrap the Shutdown function
type Shutdowner interface {
// Shutdown the backend, closing any background tasks and any
// cached connections.
Shutdown(ctx context.Context) error
}
// ObjectsChan is a channel of Objects
type ObjectsChan chan Object
// Objects is a slice of Object~s
type Objects []Object
// ObjectPair is a pair of Objects used to describe a potential copy
// operation.
type ObjectPair struct {
Src, Dst Object
}
// UnWrapFs unwraps f as much as possible and returns the base Fs
func UnWrapFs(f Fs) Fs {
for {
unwrap := f.Features().UnWrap
if unwrap == nil {
break // not a wrapped Fs, use current
}
next := unwrap()
if next == nil {
break // no base Fs found, use current
}
f = next
}
return f
}
// UnWrapObject unwraps o as much as possible and returns the base object
func UnWrapObject(o Object) Object {
for {
u, ok := o.(ObjectUnWrapper)
if !ok {
break // not a wrapped object, use current
}
next := u.UnWrap()
if next == nil {
break // no base object found, use current
}
o = next
}
return o
}
// UnWrapObjectInfo returns the underlying Object unwrapped as much as
// possible or nil.
func UnWrapObjectInfo(oi ObjectInfo) Object {
o, ok := oi.(Object)
if !ok {
return nil
}
return UnWrapObject(o)
}