rclone/vfs/file.go

package vfs

import (
	"context"
	"errors"
	"fmt"
	"os"
	"path"
	"sync"
	"sync/atomic"
	"time"

	"github.com/rclone/rclone/fs"
	"github.com/rclone/rclone/fs/log"
	"github.com/rclone/rclone/fs/operations"
	"github.com/rclone/rclone/vfs/vfscommon"
)

// The File object is tightly coupled to the Dir object. Since they
// both have locks there is plenty of potential for deadlocks. In
// order to mitigate this, we use the following conventions
//
// File may **only** call these methods from Dir with the File lock
// held.
//
//     Dir.Fs
//     Dir.VFS
//
// (As these are read only and do not need to take the Dir mutex.)
//
// File may **not** call any other Dir methods with the File lock
// held. This preserves total lock ordering and makes File subordinate
// to Dir as far as locking is concerned, preventing deadlocks.
//
// File may **not** read any members of Dir directly.

// File represents a file
type File struct {
	inode uint64 // inode number - read only
	size  int64  // size of file - read and written with atomic int64 - must be 64 bit aligned

	mu               sync.RWMutex                    // protects the following
	d                *Dir                            // parent directory
	dPath            string                          // path of parent directory. NB dir rename means all Files are flushed
	o                fs.Object                       // NB o may be nil if file is being written
	leaf             string                          // leaf name of the object
	writers          []Handle                        // writers for this file
	nwriters         int32                           // len(writers) which is read/updated with atomic
	pendingModTime   time.Time                       // will be applied once o becomes available, i.e. after file was written
	pendingRenameFun func(ctx context.Context) error // will be run/renamed after all writers close
	appendMode       bool                            // file was opened with O_APPEND
	sys              atomic.Value                    // user defined info to be attached here

	muRW sync.Mutex // synchronize RWFileHandle.openPending(), RWFileHandle.close() and File.Remove
}

// newFile creates a new File
//
// o may be nil
func newFile(d *Dir, dPath string, o fs.Object, leaf string) *File {
	f := &File{
		d:     d,
		dPath: dPath,
		o:     o,
		leaf:  leaf,
		inode: newInode(),
	}
	if o != nil {
		f.size = o.Size()
	}
	return f
}

// String converts it to printable
func (f *File) String() string {
	if f == nil {
		return "<nil *File>"
	}
	return f.Path()
}

// IsFile returns true for File - satisfies Node interface
func (f *File) IsFile() bool {
	return true
}

// IsDir returns false for File - satisfies Node interface
func (f *File) IsDir() bool {
	return false
}

// Mode bits of the file or directory - satisfies Node interface
func (f *File) Mode() (mode os.FileMode) {
	f.mu.RLock()
	defer f.mu.RUnlock()
	mode = f.d.vfs.Opt.FilePerms
	if f.appendMode {
		mode |= os.ModeAppend
	}
	return mode
}

// Name (base) of the directory - satisfies Node interface
func (f *File) Name() (name string) {
	f.mu.RLock()
	defer f.mu.RUnlock()
	return f.leaf
}

// _path returns the full path of the file
// use when lock is held
func (f *File) _path() string {
	return path.Join(f.dPath, f.leaf)
}

// Path returns the full path of the file
func (f *File) Path() string {
	f.mu.RLock()
	dPath, leaf := f.dPath, f.leaf
	f.mu.RUnlock()
	return path.Join(dPath, leaf)
}

// Sys returns underlying data source (can be nil) - satisfies Node interface
func (f *File) Sys() interface{} {
	return f.sys.Load()
}

// SetSys sets the underlying data source (can be nil) - satisfies Node interface
func (f *File) SetSys(x interface{}) {
	f.sys.Store(x)
}

// Inode returns the inode number - satisfies Node interface
func (f *File) Inode() uint64 {
	return f.inode
}

// Node returns the Node associated with this - satisfies Noder interface
func (f *File) Node() Node {
	return f
}

// applyPendingRename runs a previously set rename operation if there are no
// more remaining writers. Call without lock held.
func (f *File) applyPendingRename() {
	f.mu.RLock()
	fun := f.pendingRenameFun
	writing := f._writingInProgress()
	f.mu.RUnlock()
	if fun == nil || writing {
		return
	}
	fs.Debugf(f.Path(), "Running delayed rename now")
	if err := fun(context.TODO()); err != nil {
		fs.Errorf(f.Path(), "delayed File.Rename error: %v", err)
	}
}

// rename attempts to immediately rename a file if there are no open writers.
// Otherwise it will queue the rename operation on the remote until no writers
// remain.
func (f *File) rename(ctx context.Context, destDir *Dir, newName string) error {
	f.mu.RLock()
	d := f.d
	oldPendingRenameFun := f.pendingRenameFun
	f.mu.RUnlock()

	if features := d.Fs().Features(); features.Move == nil && features.Copy == nil {
		err := fmt.Errorf("Fs %q can't rename files (no server-side Move or Copy)", d.Fs())
		fs.Errorf(f.Path(), "Dir.Rename error: %v", err)
		return err
	}

	oldPath := f.Path()
	// File.mu is unlocked here to call Dir.Path()
	newPath := path.Join(destDir.Path(), newName)

	renameCall := func(ctx context.Context) (err error) {
		// chain rename calls if any
		if oldPendingRenameFun != nil {
			err := oldPendingRenameFun(ctx)
			if err != nil {
				return err
			}
		}

		f.mu.RLock()
		o := f.o
		d := f.d
		f.mu.RUnlock()
		var newObject fs.Object
		// if o is nil then are writing the file so no need to rename the object
		if o != nil {
			if o.Remote() == newPath {
				return nil // no need to rename
			}

			// do the move of the remote object
			dstOverwritten, _ := d.Fs().NewObject(ctx, newPath)
			newObject, err = operations.Move(ctx, d.Fs(), dstOverwritten, newPath, o)
			if err != nil {
				fs.Errorf(f.Path(), "File.Rename error: %v", err)
				return err
			}

			// newObject can be nil here for example if --dry-run
			if newObject == nil {
				err = errors.New("rename failed: nil object returned")
				fs.Errorf(f.Path(), "File.Rename %v", err)
				return err
			}
		}
		// Rename in the cache
		if d.vfs.cache != nil && d.vfs.cache.Exists(oldPath) {
			if err := d.vfs.cache.Rename(oldPath, newPath, newObject); err != nil {
				fs.Infof(f.Path(), "File.Rename failed in Cache: %v", err)
			}
		}
		// Update the node with the new details
		fs.Debugf(f.Path(), "Updating file with %v %p", newObject, f)
		// f.rename(destDir, newObject)
		f.mu.Lock()
		if newObject != nil {
			f.o = newObject
		}
		f.pendingRenameFun = nil
		f.mu.Unlock()
		return nil
	}

	// rename the file object
	dPath := destDir.Path()
	f.mu.Lock()
	f.d = destDir
	f.dPath = dPath
	f.leaf = newName
	writing := f._writingInProgress()
	f.mu.Unlock()

	// Delay the rename if not using RW caching. For the minimal case we
	// need to look in the cache to see if caching is in use.
	CacheMode := d.vfs.Opt.CacheMode
	if writing &&
		(CacheMode < vfscommon.CacheModeMinimal ||
			(CacheMode == vfscommon.CacheModeMinimal && !destDir.vfs.cache.Exists(oldPath))) {
		fs.Debugf(oldPath, "File is currently open, delaying rename %p", f)
		f.mu.Lock()
		f.pendingRenameFun = renameCall
		f.mu.Unlock()
		return nil
	}

	return renameCall(ctx)
}

// addWriter adds a write handle to the file
func (f *File) addWriter(h Handle) {
	f.mu.Lock()
	f.writers = append(f.writers, h)
	atomic.AddInt32(&f.nwriters, 1)
	f.mu.Unlock()
}

// delWriter removes a write handle from the file
func (f *File) delWriter(h Handle) {
	f.mu.Lock()
	defer f.applyPendingRename()
	defer f.mu.Unlock()
	var found = -1
	for i := range f.writers {
		if f.writers[i] == h {
			found = i
			break
		}
	}
	if found >= 0 {
		f.writers = append(f.writers[:found], f.writers[found+1:]...)
		atomic.AddInt32(&f.nwriters, -1)
	} else {
		fs.Debugf(f._path(), "File.delWriter couldn't find handle")
	}
}

// activeWriters returns the number of writers on the file
//
// Note that we don't take the mutex here.  If we do then we can get a
// deadlock.
func (f *File) activeWriters() int {
	return int(atomic.LoadInt32(&f.nwriters))
}

// _roundModTime rounds the time passed in to the Precision of the
// underlying Fs
//
// It should be called with the lock held
func (f *File) _roundModTime(modTime time.Time) time.Time {
	precision := f.d.f.Precision()
	return modTime.Truncate(precision)
}

// ModTime returns the modified time of the file
//
// if NoModTime is set then it returns the mod time of the directory
func (f *File) ModTime() (modTime time.Time) {
	f.mu.RLock()
	d, o, pendingModTime := f.d, f.o, f.pendingModTime
	f.mu.RUnlock()

	if d.vfs.Opt.NoModTime {
		return d.ModTime()
	}
	// Read the modtime from a dirty item if it exists
	if f.d.vfs.Opt.CacheMode >= vfscommon.CacheModeMinimal {
		if item := f.d.vfs.cache.DirtyItem(f._path()); item != nil {
			modTime, err := item.GetModTime()
			if err != nil {
				fs.Errorf(f._path(), "ModTime: Item GetModTime failed: %v", err)
			} else {
				return f._roundModTime(modTime)
			}
		}
	}
	if !pendingModTime.IsZero() {
		return f._roundModTime(pendingModTime)
	}
	if o == nil {
		return time.Now()
	}
	return o.ModTime(context.TODO())
}

// nonNegative returns 0 if i is -ve, i otherwise
func nonNegative(i int64) int64 {
	if i >= 0 {
		return i
	}
	return 0
}

// Size of the file
func (f *File) Size() int64 {
	f.mu.RLock()
	defer f.mu.RUnlock()

	// Read the size from a dirty item if it exists
	if f.d.vfs.Opt.CacheMode >= vfscommon.CacheModeMinimal {
		if item := f.d.vfs.cache.DirtyItem(f._path()); item != nil {
			size, err := item.GetSize()
			if err != nil {
				fs.Errorf(f._path(), "Size: Item GetSize failed: %v", err)
			} else {
				return size
			}
		}
	}

	// if o is nil it isn't valid yet or there are writers, so return the size so far
	if f._writingInProgress() {
		return atomic.LoadInt64(&f.size)
	}
	return nonNegative(f.o.Size())
}

// SetModTime sets the modtime for the file
//
// if NoModTime is set then it does nothing
func (f *File) SetModTime(modTime time.Time) error {
	f.mu.Lock()
	defer f.mu.Unlock()
	if f.d.vfs.Opt.NoModTime {
		return nil
	}
	if f.d.vfs.Opt.ReadOnly {
		return EROFS
	}

	f.pendingModTime = modTime

	// set the time of the file in the cache
	if f.d.vfs.cache != nil && f.d.vfs.cache.Exists(f._path()) {
		f.d.vfs.cache.SetModTime(f._path(), f.pendingModTime)
	}

	// Only update the ModTime when there are no writers, setObject will do it
	if !f._writingInProgress() {
		return f._applyPendingModTime()
	}

	// queue up for later, hoping f.o becomes available
	return nil
}

// Apply a pending mod time
// Call with the mutex held
func (f *File) _applyPendingModTime() error {
	if f.pendingModTime.IsZero() {
		return nil
	}
	defer func() { f.pendingModTime = time.Time{} }()

	if f.o == nil {
		return errors.New("cannot apply ModTime, file object is not available")
	}

	dt := f.pendingModTime.Sub(f.o.ModTime(context.Background()))
	modifyWindow := f.o.Fs().Precision()
	if dt < modifyWindow && dt > -modifyWindow {
		fs.Debugf(f.o, "Not setting pending mod time %v as it is already set", f.pendingModTime)
		return nil
	}

	// set the time of the object
	err := f.o.SetModTime(context.TODO(), f.pendingModTime)
	switch err {
	case nil:
		fs.Debugf(f.o, "Applied pending mod time %v OK", f.pendingModTime)
	case fs.ErrorCantSetModTime, fs.ErrorCantSetModTimeWithoutDelete:
		// do nothing, in order to not break "touch somefile" if it exists already
	default:
		fs.Errorf(f.o, "Failed to apply pending mod time %v: %v", f.pendingModTime, err)
		return err
	}

	return nil
}

// Apply a pending mod time
func (f *File) applyPendingModTime() error {
	f.mu.Lock()
	defer f.mu.Unlock()
	return f._applyPendingModTime()
}

// _writingInProgress returns true of there are any open writers
// Call with read lock held
func (f *File) _writingInProgress() bool {
	return f.o == nil || len(f.writers) != 0
}

// writingInProgress returns true of there are any open writers
func (f *File) writingInProgress() bool {
	f.mu.RLock()
	defer f.mu.RUnlock()
	return f.o == nil || len(f.writers) != 0
}

// Update the size while writing
func (f *File) setSize(n int64) {
	atomic.StoreInt64(&f.size, n)
}

// Update the object when written and add it to the directory
func (f *File) setObject(o fs.Object) {
	f.mu.Lock()
	f.o = o
	_ = f._applyPendingModTime()
	d := f.d
	f.mu.Unlock()

	// Release File.mu before calling Dir method
	d.addObject(f)
}

// Update the object but don't update the directory cache - for use by
// the directory cache
func (f *File) setObjectNoUpdate(o fs.Object) {
	f.mu.Lock()
	f.o = o
	f.mu.Unlock()
}

// Get the current fs.Object - may be nil
func (f *File) getObject() fs.Object {
	f.mu.RLock()
	defer f.mu.RUnlock()
	return f.o
}

// exists returns whether the file exists already
func (f *File) exists() bool {
	f.mu.RLock()
	defer f.mu.RUnlock()
	return f.o != nil
}

// Wait for f.o to become non nil for a short time returning it or an
// error.  Use when opening a read handle.
//
// Call without the mutex held
func (f *File) waitForValidObject() (o fs.Object, err error) {
	for i := 0; i < 50; i++ {
		f.mu.RLock()
		o = f.o
		nwriters := len(f.writers)
		f.mu.RUnlock()
		if o != nil {
			return o, nil
		}
		if nwriters == 0 {
			return nil, errors.New("can't open file - writer failed")
		}
		time.Sleep(100 * time.Millisecond)
	}
	return nil, ENOENT
}

// openRead open the file for read
func (f *File) openRead() (fh *ReadFileHandle, err error) {
	// if o is nil it isn't valid yet
	_, err = f.waitForValidObject()
	if err != nil {
		return nil, err
	}
	// fs.Debugf(f.Path(), "File.openRead")

	fh, err = newReadFileHandle(f)
	if err != nil {
		fs.Debugf(f.Path(), "File.openRead failed: %v", err)
		return nil, err
	}
	return fh, nil
}

// openWrite open the file for write
func (f *File) openWrite(flags int) (fh *WriteFileHandle, err error) {
	f.mu.RLock()
	d := f.d
	f.mu.RUnlock()

	if d.vfs.Opt.ReadOnly {
		return nil, EROFS
	}
	// fs.Debugf(f.Path(), "File.openWrite")

	fh, err = newWriteFileHandle(d, f, f.Path(), flags)
	if err != nil {
		fs.Debugf(f.Path(), "File.openWrite failed: %v", err)
		return nil, err
	}
	return fh, nil
}

// openRW open the file for read and write using a temporary file
//
// It uses the open flags passed in.
func (f *File) openRW(flags int) (fh *RWFileHandle, err error) {
	f.mu.RLock()
	d := f.d
	f.mu.RUnlock()

	// FIXME chunked
	if flags&accessModeMask != os.O_RDONLY && d.vfs.Opt.ReadOnly {
		return nil, EROFS
	}
	// fs.Debugf(f.Path(), "File.openRW")

	fh, err = newRWFileHandle(d, f, flags)
	if err != nil {
		fs.Debugf(f.Path(), "File.openRW failed: %v", err)
		return nil, err
	}
	return fh, nil
}

// Sync the file
//
// Note that we don't do anything except return OK
func (f *File) Sync() error {
	return nil
}

// Remove the file
func (f *File) Remove() (err error) {
	defer log.Trace(f.Path(), "")("err=%v", &err)
	f.mu.RLock()
	d := f.d
	f.mu.RUnlock()

	if d.vfs.Opt.ReadOnly {
		return EROFS
	}

	// Remove the object from the cache
	wasWriting := false
	if d.vfs.cache != nil && d.vfs.cache.Exists(f.Path()) {
		wasWriting = d.vfs.cache.Remove(f.Path())
	}

	// Remove the item from the directory listing
	// called with File.mu released
	d.delObject(f.Name())

	f.muRW.Lock() // muRW must be locked before mu to avoid
	f.mu.Lock()   // deadlock in RWFileHandle.openPending and .close
	if f.o != nil {
		err = f.o.Remove(context.TODO())
	}
	f.mu.Unlock()
	f.muRW.Unlock()
	if err != nil {
		if wasWriting {
			// Ignore error deleting file if was writing it as it may not be uploaded yet
			err = nil
			fs.Debugf(f._path(), "Ignoring File.Remove file error as uploading: %v", err)
		} else {
			fs.Debugf(f._path(), "File.Remove file error: %v", err)
		}
	}
	return err
}

// RemoveAll the file - same as remove for files
func (f *File) RemoveAll() error {
	return f.Remove()
}

// DirEntry returns the underlying fs.DirEntry - may be nil
func (f *File) DirEntry() (entry fs.DirEntry) {
	f.mu.RLock()
	defer f.mu.RUnlock()
	return f.o
}

// Dir returns the directory this file is in
func (f *File) Dir() *Dir {
	f.mu.RLock()
	defer f.mu.RUnlock()
	return f.d
}

// VFS returns the instance of the VFS
func (f *File) VFS() *VFS {
	f.mu.RLock()
	defer f.mu.RUnlock()
	return f.d.vfs
}

// Fs returns the underlying Fs for the file
func (f *File) Fs() fs.Fs {
	f.mu.RLock()
	defer f.mu.RUnlock()
	return f.d.Fs()
}

// Open a file according to the flags provided
//
//   O_RDONLY open the file read-only.
//   O_WRONLY open the file write-only.
//   O_RDWR   open the file read-write.
//
//   O_APPEND append data to the file when writing.
//   O_CREATE create a new file if none exists.
//   O_EXCL   used with O_CREATE, file must not exist
//   O_SYNC   open for synchronous I/O.
//   O_TRUNC  if possible, truncate file when opened
//
// We ignore O_SYNC and O_EXCL
func (f *File) Open(flags int) (fd Handle, err error) {
	defer log.Trace(f.Path(), "flags=%s", decodeOpenFlags(flags))("fd=%v, err=%v", &fd, &err)
	var (
		write    bool // if set need write support
		read     bool // if set need read support
		rdwrMode = flags & accessModeMask
	)

	// http://pubs.opengroup.org/onlinepubs/7908799/xsh/open.html
	// The result of using O_TRUNC with O_RDONLY is undefined.
	// Linux seems to truncate the file, but we prefer to return EINVAL
	if rdwrMode == os.O_RDONLY && flags&os.O_TRUNC != 0 {
		return nil, EINVAL
	}

	// Figure out the read/write intents
	switch {
	case rdwrMode == os.O_RDONLY:
		read = true
	case rdwrMode == os.O_WRONLY:
		write = true
	case rdwrMode == os.O_RDWR:
		read = true
		write = true
	default:
		fs.Debugf(f.Path(), "Can't figure out how to open with flags: 0x%X", flags)
		return nil, EPERM
	}

	// If append is set then set read to force openRW
	if flags&os.O_APPEND != 0 {
		read = true
		f.mu.Lock()
		f.appendMode = true
		f.mu.Unlock()
	}

	// If truncate is set then set write to force openRW
	if flags&os.O_TRUNC != 0 {
		write = true
	}

	// Open the correct sort of handle
	f.mu.RLock()
	d := f.d
	f.mu.RUnlock()
	CacheMode := d.vfs.Opt.CacheMode
	if CacheMode >= vfscommon.CacheModeMinimal && (d.vfs.cache.InUse(f.Path()) || d.vfs.cache.Exists(f.Path())) {
		fd, err = f.openRW(flags)
	} else if read && write {
		if CacheMode >= vfscommon.CacheModeMinimal {
			fd, err = f.openRW(flags)
		} else {
			// Open write only and hope the user doesn't
			// want to read.  If they do they will get an
			// EPERM plus an Error log.
			fd, err = f.openWrite(flags)
		}
	} else if write {
		if CacheMode >= vfscommon.CacheModeWrites {
			fd, err = f.openRW(flags)
		} else {
			fd, err = f.openWrite(flags)
		}
	} else if read {
		if CacheMode >= vfscommon.CacheModeFull {
			fd, err = f.openRW(flags)
		} else {
			fd, err = f.openRead()
		}
	} else {
		fs.Debugf(f.Path(), "Can't figure out how to open with flags: 0x%X", flags)
		return nil, EPERM
	}
	// if creating a file, add the file to the directory
	if err == nil && flags&os.O_CREATE != 0 {
		// called without File.mu held
		d.addObject(f)
	}
	return fd, err
}

// Truncate changes the size of the named file.
func (f *File) Truncate(size int64) (err error) {
	// make a copy of fh.writers with the lock held then unlock so
	// we can call other file methods.
	f.mu.Lock()
	writers := make([]Handle, len(f.writers))
	copy(writers, f.writers)
	f.mu.Unlock()

	// If have writers then call truncate for each writer
	if len(writers) != 0 {
		var openWriters = len(writers)
		fs.Debugf(f.Path(), "Truncating %d file handles", len(writers))
		for _, h := range writers {
			truncateErr := h.Truncate(size)
			if truncateErr == ECLOSED {
				// Ignore ECLOSED since file handle can get closed while this is running
				openWriters--
			} else if truncateErr != nil {
				err = truncateErr
			}
		}
		// If at least one open writer return here
		if openWriters > 0 {
			return err
		}
	}

	// if o is nil it isn't valid yet
	o, err := f.waitForValidObject()
	if err != nil {
		return err
	}

	// If no writers, and size is already correct then all done
	if o.Size() == size {
		return nil
	}

	fs.Debugf(f.Path(), "Truncating file")

	// Otherwise if no writers then truncate the file by opening
	// the file and truncating it.
	flags := os.O_WRONLY
	if size == 0 {
		flags |= os.O_TRUNC
	}
	fh, err := f.Open(flags)
	if err != nil {
		return err
	}
	defer fs.CheckClose(fh, &err)
	if size != 0 {
		return fh.Truncate(size)
	}
	return nil
}