diff --git a/cmd/cmount/fs.go b/cmd/cmount/fs.go index 9f49c777a..c4f0d6d65 100644 --- a/cmd/cmount/fs.go +++ b/cmd/cmount/fs.go @@ -361,12 +361,7 @@ func (fsys *FS) Write(path string, buff []byte, ofst int64, fh uint64) (n int) { if errc != 0 { return errc } - var err error - if fsys.VFS.Opt.CacheMode < vfs.CacheModeWrites || handle.Node().Mode()&os.ModeAppend == 0 { - n, err = handle.WriteAt(buff, ofst) - } else { - n, err = handle.Write(buff) - } + n, err := handle.WriteAt(buff, ofst) if err != nil { return translateError(err) } diff --git a/cmd/mount/handle.go b/cmd/mount/handle.go index a69784fea..86105fe9f 100644 --- a/cmd/mount/handle.go +++ b/cmd/mount/handle.go @@ -5,7 +5,6 @@ package mount import ( "context" "io" - "os" "bazil.org/fuse" fusefs "bazil.org/fuse/fs" @@ -42,12 +41,7 @@ var _ fusefs.HandleWriter = (*FileHandle)(nil) // Write data to the file handle func (fh *FileHandle) Write(ctx context.Context, req *fuse.WriteRequest, resp *fuse.WriteResponse) (err error) { defer log.Trace(fh, "len=%d, offset=%d", len(req.Data), req.Offset)("written=%d, err=%v", &resp.Size, &err) - var n int - if fh.Handle.Node().VFS().Opt.CacheMode < vfs.CacheModeWrites || fh.Handle.Node().Mode()&os.ModeAppend == 0 { - n, err = fh.Handle.WriteAt(req.Data, req.Offset) - } else { - n, err = fh.Handle.Write(req.Data) - } + n, err := fh.Handle.WriteAt(req.Data, req.Offset) if err != nil { return translateError(err) } diff --git a/cmd/mount2/file.go b/cmd/mount2/file.go index d460c7b73..32aba8644 100644 --- a/cmd/mount2/file.go +++ b/cmd/mount2/file.go @@ -6,7 +6,6 @@ import ( "context" "fmt" "io" - "os" "syscall" fusefs "github.com/hanwen/go-fuse/v2/fs" @@ -74,11 +73,7 @@ func (f *FileHandle) Write(ctx context.Context, data []byte, off int64) (written var n int var err error defer log.Trace(f, "off=%d", off)("n=%d, off=%d, errno=%v", &n, &off, &errno) - if f.h.Node().VFS().Opt.CacheMode < vfs.CacheModeWrites || f.h.Node().Mode()&os.ModeAppend == 0 { - n, err = f.h.WriteAt(data, off) - } else { - n, err = f.h.Write(data) - } + n, err = f.h.WriteAt(data, off) return uint32(n), translateError(err) } diff --git a/vfs/dir.go b/vfs/dir.go index f8d86d489..25af0cb97 100644 --- a/vfs/dir.go +++ b/vfs/dir.go @@ -16,6 +16,7 @@ import ( "github.com/rclone/rclone/fs/log" "github.com/rclone/rclone/fs/operations" "github.com/rclone/rclone/fs/walk" + "github.com/rclone/rclone/vfs/vfscommon" ) // Dir represents a directory entry @@ -150,7 +151,7 @@ func (d *Dir) changeNotify(relativePath string, entryType fs.EntryType) { d.mu.RLock() absPath := path.Join(d.path, relativePath) d.mu.RUnlock() - d.invalidateDir(findParent(absPath)) + d.invalidateDir(vfscommon.FindParent(absPath)) if entryType == fs.EntryDirectory { d.invalidateDir(absPath) } @@ -168,7 +169,7 @@ func (d *Dir) ForgetPath(relativePath string, entryType fs.EntryType) { absPath := path.Join(d.path, relativePath) d.mu.RUnlock() if absPath != "" { - d.invalidateDir(findParent(absPath)) + d.invalidateDir(vfscommon.FindParent(absPath)) } if entryType == fs.EntryDirectory { d.forgetDirPath(relativePath) diff --git a/vfs/file.go b/vfs/file.go index a90f9e81a..4ef4cf0f2 100644 --- a/vfs/file.go +++ b/vfs/file.go @@ -12,6 +12,7 @@ import ( "github.com/rclone/rclone/fs" "github.com/rclone/rclone/fs/log" "github.com/rclone/rclone/fs/operations" + "github.com/rclone/rclone/vfs/vfscommon" ) // The File object is tightly coupled to the Dir object. Since they @@ -124,7 +125,7 @@ func (f *File) Path() string { // osPath returns the full path of the file in the cache in OS format func (f *File) osPath() string { - return f.d.vfs.cache.toOSPath(f.Path()) + return f.d.vfs.cache.ToOSPath(f.Path()) } // Sys returns underlying data source (can be nil) - satisfies Node interface @@ -220,8 +221,8 @@ func (f *File) rename(ctx context.Context, destDir *Dir, newName string) error { } // Rename in the cache if it exists - if f.d.vfs.Opt.CacheMode != CacheModeOff && f.d.vfs.cache.exists(f.Path()) { - if err := f.d.vfs.cache.rename(f.Path(), newPath); err != nil { + if f.d.vfs.cache != nil && f.d.vfs.cache.Exists(f.Path()) { + if err := f.d.vfs.cache.Rename(f.Path(), newPath); err != nil { fs.Infof(f.Path(), "File.Rename failed in Cache: %v", err) } } @@ -403,8 +404,8 @@ func (f *File) _applyPendingModTime() error { } // set the time of the file in the cache - if f.d.vfs.Opt.CacheMode != CacheModeOff { - f.d.vfs.cache.setModTime(f._path(), f.pendingModTime) + if f.d.vfs.cache != nil { + f.d.vfs.cache.SetModTime(f._path(), f.pendingModTime) } // set the time of the object @@ -580,8 +581,8 @@ func (f *File) Remove() error { // called with File.mu released d.delObject(f.Name()) // Remove the object from the cache - if d.vfs.Opt.CacheMode >= CacheModeMinimal { - d.vfs.cache.remove(f.Path()) + if d.vfs.cache != nil { + d.vfs.cache.Remove(f.Path()) } return nil } @@ -679,10 +680,10 @@ func (f *File) Open(flags int) (fd Handle, err error) { d := f.d f.mu.RUnlock() CacheMode := d.vfs.Opt.CacheMode - if CacheMode >= CacheModeMinimal && (d.vfs.cache.opens(f.Path()) > 0 || d.vfs.cache.exists(f.Path())) { + if CacheMode >= vfscommon.CacheModeMinimal && (d.vfs.cache.Opens(f.Path()) > 0 || d.vfs.cache.Exists(f.Path())) { fd, err = f.openRW(flags) } else if read && write { - if CacheMode >= CacheModeMinimal { + if CacheMode >= vfscommon.CacheModeMinimal { fd, err = f.openRW(flags) } else { // Open write only and hope the user doesn't @@ -691,13 +692,13 @@ func (f *File) Open(flags int) (fd Handle, err error) { fd, err = f.openWrite(flags) } } else if write { - if CacheMode >= CacheModeWrites { + if CacheMode >= vfscommon.CacheModeWrites { fd, err = f.openRW(flags) } else { fd, err = f.openWrite(flags) } } else if read { - if CacheMode >= CacheModeFull { + if CacheMode >= vfscommon.CacheModeFull { fd, err = f.openRW(flags) } else { fd, err = f.openRead() diff --git a/vfs/file_test.go b/vfs/file_test.go index a36b5888e..31f5351c5 100644 --- a/vfs/file_test.go +++ b/vfs/file_test.go @@ -11,12 +11,13 @@ import ( "github.com/rclone/rclone/fstest" "github.com/rclone/rclone/fstest/mockfs" "github.com/rclone/rclone/fstest/mockobject" + "github.com/rclone/rclone/vfs/vfscommon" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) -func fileCreate(t *testing.T, r *fstest.Run, mode CacheMode) (*VFS, *File, fstest.Item) { - opt := DefaultOpt +func fileCreate(t *testing.T, r *fstest.Run, mode vfscommon.CacheMode) (*VFS, *File, fstest.Item) { + opt := vfscommon.DefaultOpt opt.CacheMode = mode vfs := New(r.Fremote, &opt) @@ -33,7 +34,7 @@ func fileCreate(t *testing.T, r *fstest.Run, mode CacheMode) (*VFS, *File, fstes func TestFileMethods(t *testing.T) { r := fstest.NewRun(t) defer r.Finalise() - vfs, file, _ := fileCreate(t, r, CacheModeOff) + vfs, file, _ := fileCreate(t, r, vfscommon.CacheModeOff) // String assert.Equal(t, "dir/file1", file.String()) @@ -88,7 +89,7 @@ func TestFileSetModTime(t *testing.T) { return } defer r.Finalise() - vfs, file, file1 := fileCreate(t, r, CacheModeOff) + vfs, file, file1 := fileCreate(t, r, vfscommon.CacheModeOff) err := file.SetModTime(t2) require.NoError(t, err) @@ -115,7 +116,7 @@ func fileCheckContents(t *testing.T, file *File) { func TestFileOpenRead(t *testing.T) { r := fstest.NewRun(t) defer r.Finalise() - _, file, _ := fileCreate(t, r, CacheModeOff) + _, file, _ := fileCreate(t, r, vfscommon.CacheModeOff) fileCheckContents(t, file) } @@ -168,7 +169,7 @@ func TestFileOpenReadUnknownSize(t *testing.T) { func TestFileOpenWrite(t *testing.T) { r := fstest.NewRun(t) defer r.Finalise() - vfs, file, _ := fileCreate(t, r, CacheModeOff) + vfs, file, _ := fileCreate(t, r, vfscommon.CacheModeOff) fd, err := file.openWrite(os.O_WRONLY | os.O_TRUNC) require.NoError(t, err) @@ -189,7 +190,7 @@ func TestFileOpenWrite(t *testing.T) { func TestFileRemove(t *testing.T) { r := fstest.NewRun(t) defer r.Finalise() - vfs, file, _ := fileCreate(t, r, CacheModeOff) + vfs, file, _ := fileCreate(t, r, vfscommon.CacheModeOff) err := file.Remove() require.NoError(t, err) @@ -204,7 +205,7 @@ func TestFileRemove(t *testing.T) { func TestFileRemoveAll(t *testing.T) { r := fstest.NewRun(t) defer r.Finalise() - vfs, file, _ := fileCreate(t, r, CacheModeOff) + vfs, file, _ := fileCreate(t, r, vfscommon.CacheModeOff) err := file.RemoveAll() require.NoError(t, err) @@ -219,7 +220,7 @@ func TestFileRemoveAll(t *testing.T) { func TestFileOpen(t *testing.T) { r := fstest.NewRun(t) defer r.Finalise() - _, file, _ := fileCreate(t, r, CacheModeOff) + _, file, _ := fileCreate(t, r, vfscommon.CacheModeOff) fd, err := file.Open(os.O_RDONLY) require.NoError(t, err) @@ -242,7 +243,7 @@ func TestFileOpen(t *testing.T) { assert.Equal(t, EPERM, err) } -func testFileRename(t *testing.T, mode CacheMode) { +func testFileRename(t *testing.T, mode vfscommon.CacheMode) { r := fstest.NewRun(t) defer r.Finalise() vfs, file, item := fileCreate(t, r, mode) @@ -255,10 +256,10 @@ func testFileRename(t *testing.T, mode CacheMode) { require.NoError(t, err) // check file in cache - if mode != CacheModeOff { + if mode != vfscommon.CacheModeOff { // read contents to get file in cache fileCheckContents(t, file) - assert.True(t, vfs.cache.exists(item.Path)) + assert.True(t, vfs.cache.Exists(item.Path)) } dir := file.Dir() @@ -274,8 +275,8 @@ func testFileRename(t *testing.T, mode CacheMode) { fstest.CheckItems(t, r.Fremote, item) // check file in cache - if mode != CacheModeOff { - assert.True(t, vfs.cache.exists(item.Path)) + if mode != vfscommon.CacheModeOff { + assert.True(t, vfs.cache.Exists(item.Path)) } // check file exists in the vfs layer at its new name @@ -290,8 +291,8 @@ func testFileRename(t *testing.T, mode CacheMode) { fstest.CheckItems(t, r.Fremote, item) // check file in cache - if mode != CacheModeOff { - assert.True(t, vfs.cache.exists(item.Path)) + if mode != vfscommon.CacheModeOff { + assert.True(t, vfs.cache.Exists(item.Path)) } // now try renaming it with the file open @@ -308,8 +309,8 @@ func testFileRename(t *testing.T, mode CacheMode) { newItem := fstest.NewItem("newLeaf", string(newContents), item.ModTime) // check file has been renamed immediately in the cache - if mode != CacheModeOff { - assert.True(t, vfs.cache.exists("newLeaf")) + if mode != vfscommon.CacheModeOff { + assert.True(t, vfs.cache.Exists("newLeaf")) } // check file exists in the vfs layer at its new name @@ -326,9 +327,9 @@ func testFileRename(t *testing.T, mode CacheMode) { func TestFileRename(t *testing.T) { t.Run("CacheModeOff", func(t *testing.T) { - testFileRename(t, CacheModeOff) + testFileRename(t, vfscommon.CacheModeOff) }) t.Run("CacheModeFull", func(t *testing.T) { - testFileRename(t, CacheModeFull) + testFileRename(t, vfscommon.CacheModeFull) }) } diff --git a/vfs/read_write.go b/vfs/read_write.go index a19fa94ca..381f84471 100644 --- a/vfs/read_write.go +++ b/vfs/read_write.go @@ -12,7 +12,6 @@ import ( "github.com/pkg/errors" "github.com/rclone/rclone/fs" "github.com/rclone/rclone/fs/log" - "github.com/rclone/rclone/fs/operations" "github.com/rclone/rclone/lib/file" ) @@ -55,12 +54,12 @@ func newRWFileHandle(d *Dir, f *File, flags int) (fh *RWFileHandle, err error) { } // mark the file as open in the cache - must be done before the mkdir - fh.d.VFS().cache.open(fh.file.Path()) + fh.d.VFS().cache.Open(fh.file.Path()) // Make a place for the file - _, err = d.VFS().cache.mkdir(fh.file.Path()) + _, err = d.VFS().cache.Mkdir(fh.file.Path()) if err != nil { - fh.d.VFS().cache.close(fh.file.Path()) + fh.d.VFS().cache.Close(fh.file.Path()) return nil, errors.Wrap(err, "open RW handle failed to make cache directory") } @@ -80,16 +79,6 @@ func newRWFileHandle(d *Dir, f *File, flags int) (fh *RWFileHandle, err error) { return fh, nil } -// copy an object to or from the remote while accounting for it -func copyObj(f fs.Fs, dst fs.Object, remote string, src fs.Object) (newDst fs.Object, err error) { - if operations.NeedTransfer(context.TODO(), dst, src) { - newDst, err = operations.Copy(context.TODO(), f, dst, remote, src) - } else { - newDst = dst - } - return newDst, err -} - // openPending opens the file if there is a pending open // // call with the lock held @@ -110,12 +99,9 @@ func (fh *RWFileHandle) openPending(truncate bool) (err error) { // If the remote object exists AND its cached file exists locally AND there are no // other RW handles with it open, then attempt to update it. if o != nil && fh.file.rwOpens() == 0 { - cacheObj, err := fh.d.VFS().cache.f.NewObject(context.TODO(), fh.file.Path()) - if err == nil && cacheObj != nil { - _, err = copyObj(fh.d.VFS().cache.f, cacheObj, fh.file.Path(), o) - if err != nil { - return errors.Wrap(err, "open RW handle failed to update cached file") - } + err = fh.d.VFS().cache.Check(context.TODO(), o, fh.file.Path()) + if err != nil { + return errors.Wrap(err, "open RW handle failed to check cache file") } } @@ -125,7 +111,7 @@ func (fh *RWFileHandle) openPending(truncate bool) (err error) { // cache file does not exist, so need to fetch it if we have an object to fetch // it from if o != nil { - _, err = copyObj(fh.d.VFS().cache.f, nil, fh.file.Path(), o) + err = fh.d.VFS().cache.Fetch(context.TODO(), o, fh.file.Path()) if err != nil { cause := errors.Cause(err) if cause != fs.ErrorObjectNotFound && cause != fs.ErrorDirNotFound { @@ -276,22 +262,8 @@ func (fh *RWFileHandle) flushWrites(closeFile bool) error { } if isCopied { - // Transfer the temp file to the remote - cacheObj, err := fh.d.VFS().cache.f.NewObject(context.TODO(), fh.file.Path()) + o, err := fh.d.VFS().cache.Store(context.TODO(), fh.file.getObject(), fh.file.Path()) if err != nil { - err = errors.Wrap(err, "failed to find cache file") - fs.Errorf(fh.logPrefix(), "%v", err) - return err - } - - objPath := fh.file.Path() - objOld := fh.file.getObject() - if objOld != nil { - objPath = objOld.Remote() // use the path of the actual object if available - } - o, err := copyObj(fh.d.VFS().f, objOld, objPath, cacheObj) - if err != nil { - err = errors.Wrap(err, "failed to transfer file from cache to remote") fs.Errorf(fh.logPrefix(), "%v", err) return err } @@ -322,7 +294,7 @@ func (fh *RWFileHandle) close() (err error) { if fh.opened { fh.file.delRWOpen() } - fh.d.VFS().cache.close(fh.file.Path()) + fh.d.VFS().cache.Close(fh.file.Path()) }() return fh.flushWrites(true) @@ -501,6 +473,10 @@ func (fh *RWFileHandle) Write(b []byte) (n int, err error) { // WriteAt bytes to the file at off func (fh *RWFileHandle) WriteAt(b []byte, off int64) (n int, err error) { + if fh.flags&os.O_APPEND != 0 { + // if append is set, call Write as WriteAt returns an error if append is set + return fh.Write(b) + } err = fh.writeFn(func() error { n, err = fh.File.WriteAt(b, off) return err diff --git a/vfs/read_write_test.go b/vfs/read_write_test.go index 13555db1b..4e4bdb1c0 100644 --- a/vfs/read_write_test.go +++ b/vfs/read_write_test.go @@ -13,6 +13,7 @@ import ( "github.com/rclone/rclone/fs" "github.com/rclone/rclone/fs/operations" "github.com/rclone/rclone/fstest" + "github.com/rclone/rclone/vfs/vfscommon" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -25,8 +26,8 @@ func cleanup(t *testing.T, r *fstest.Run, vfs *VFS) { // Create a file and open it with the flags passed in func rwHandleCreateFlags(t *testing.T, r *fstest.Run, create bool, filename string, flags int) (*VFS, *RWFileHandle) { - opt := DefaultOpt - opt.CacheMode = CacheModeFull + opt := vfscommon.DefaultOpt + opt.CacheMode = vfscommon.CacheModeFull vfs := New(r.Fremote, &opt) if create { @@ -661,8 +662,8 @@ func testRWFileHandleOpenTest(t *testing.T, vfs *VFS, test *openTest) { func TestRWFileHandleOpenTests(t *testing.T) { r := fstest.NewRun(t) - opt := DefaultOpt - opt.CacheMode = CacheModeFull + opt := vfscommon.DefaultOpt + opt.CacheMode = vfscommon.CacheModeFull vfs := New(r.Fremote, &opt) defer cleanup(t, r, vfs) @@ -716,8 +717,8 @@ func TestRWCacheRename(t *testing.T) { t.Skip("skip as can't rename files") } - opt := DefaultOpt - opt.CacheMode = CacheModeFull + opt := vfscommon.DefaultOpt + opt.CacheMode = vfscommon.CacheModeFull vfs := New(r.Fremote, &opt) h, err := vfs.OpenFile("rename_me", os.O_WRONLY|os.O_CREATE, 0777) @@ -732,11 +733,11 @@ func TestRWCacheRename(t *testing.T) { err = fh.Close() require.NoError(t, err) - assert.True(t, vfs.cache.exists("rename_me")) + assert.True(t, vfs.cache.Exists("rename_me")) err = vfs.Rename("rename_me", "i_was_renamed") require.NoError(t, err) - assert.False(t, vfs.cache.exists("rename_me")) - assert.True(t, vfs.cache.exists("i_was_renamed")) + assert.False(t, vfs.cache.Exists("rename_me")) + assert.True(t, vfs.cache.Exists("i_was_renamed")) } diff --git a/vfs/vfs.go b/vfs/vfs.go index d3f8098f5..82e6a7efc 100644 --- a/vfs/vfs.go +++ b/vfs/vfs.go @@ -27,7 +27,6 @@ import ( "io/ioutil" "os" "path" - "runtime" "sort" "strings" "sync" @@ -36,32 +35,10 @@ import ( "github.com/rclone/rclone/fs" "github.com/rclone/rclone/fs/log" + "github.com/rclone/rclone/vfs/vfscache" + "github.com/rclone/rclone/vfs/vfscommon" ) -// DefaultOpt is the default values uses for Opt -var DefaultOpt = Options{ - NoModTime: false, - NoChecksum: false, - NoSeek: false, - DirCacheTime: 5 * 60 * time.Second, - PollInterval: time.Minute, - ReadOnly: false, - Umask: 0, - UID: ^uint32(0), // these values instruct WinFSP-FUSE to use the current user - GID: ^uint32(0), // overriden for non windows in mount_unix.go - DirPerms: os.FileMode(0777), - FilePerms: os.FileMode(0666), - CacheMode: CacheModeOff, - CacheMaxAge: 3600 * time.Second, - CachePollInterval: 60 * time.Second, - ChunkSize: 128 * fs.MebiByte, - ChunkSizeLimit: -1, - CacheMaxSize: -1, - CaseInsensitive: runtime.GOOS == "windows" || runtime.GOOS == "darwin", // default to true on Windows and Mac, false otherwise - WriteWait: 1000 * time.Millisecond, - ReadWait: 5 * time.Millisecond, -} - // Node represents either a directory (*Dir) or a file (*File) type Node interface { os.FileInfo @@ -180,8 +157,8 @@ var ( type VFS struct { f fs.Fs root *Dir - Opt Options - cache *cache + Opt vfscommon.Options + cache *vfscache.Cache cancel context.CancelFunc usageMu sync.Mutex usageTime time.Time @@ -189,33 +166,9 @@ type VFS struct { pollChan chan time.Duration } -// Options is options for creating the vfs -type Options struct { - NoSeek bool // don't allow seeking if set - NoChecksum bool // don't check checksums if set - ReadOnly bool // if set VFS is read only - NoModTime bool // don't read mod times for files - DirCacheTime time.Duration // how long to consider directory listing cache valid - PollInterval time.Duration - Umask int - UID uint32 - GID uint32 - DirPerms os.FileMode - FilePerms os.FileMode - ChunkSize fs.SizeSuffix // if > 0 read files in chunks - ChunkSizeLimit fs.SizeSuffix // if > ChunkSize double the chunk size after each chunk until reached - CacheMode CacheMode - CacheMaxAge time.Duration - CacheMaxSize fs.SizeSuffix - CachePollInterval time.Duration - CaseInsensitive bool - WriteWait time.Duration // time to wait for in-sequence write - ReadWait time.Duration // time to wait for in-sequence read -} - // New creates a new VFS and root directory. If opt is nil, then // DefaultOpt will be used -func New(f fs.Fs, opt *Options) *VFS { +func New(f fs.Fs, opt *vfscommon.Options) *VFS { fsDir := fs.NewDir("", time.Now()) vfs := &VFS{ f: f, @@ -225,7 +178,7 @@ func New(f fs.Fs, opt *Options) *VFS { if opt != nil { vfs.Opt = *opt } else { - vfs.Opt = DefaultOpt + vfs.Opt = vfscommon.DefaultOpt } // Mask the permissions with the umask @@ -260,15 +213,15 @@ func (vfs *VFS) Fs() fs.Fs { } // SetCacheMode change the cache mode -func (vfs *VFS) SetCacheMode(cacheMode CacheMode) { +func (vfs *VFS) SetCacheMode(cacheMode vfscommon.CacheMode) { vfs.Shutdown() vfs.cache = nil - if cacheMode > CacheModeOff { + if cacheMode > vfscommon.CacheModeOff { ctx, cancel := context.WithCancel(context.Background()) - cache, err := newCache(ctx, vfs.f, &vfs.Opt) // FIXME pass on context or get from Opt? + cache, err := vfscache.New(ctx, vfs.f, &vfs.Opt) // FIXME pass on context or get from Opt? if err != nil { fs.Errorf(nil, "Failed to create vfs cache - disabling: %v", err) - vfs.Opt.CacheMode = CacheModeOff + vfs.Opt.CacheMode = vfscommon.CacheModeOff cancel() return } @@ -288,10 +241,10 @@ func (vfs *VFS) Shutdown() { // CleanUp deletes the contents of the on disk cache func (vfs *VFS) CleanUp() error { - if vfs.Opt.CacheMode == CacheModeOff { + if vfs.Opt.CacheMode == vfscommon.CacheModeOff { return nil } - return vfs.cache.cleanUp() + return vfs.cache.CleanUp() } // FlushDirCache empties the directory cache diff --git a/vfs/vfs_case_test.go b/vfs/vfs_case_test.go index 9e26b404c..b20351e88 100644 --- a/vfs/vfs_case_test.go +++ b/vfs/vfs_case_test.go @@ -6,6 +6,7 @@ import ( "testing" "github.com/rclone/rclone/fstest" + "github.com/rclone/rclone/vfs/vfscommon" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -32,11 +33,11 @@ func TestCaseSensitivity(t *testing.T) { file3 := r.WriteObject(ctx, "FilEb", "data3", t3) // Create a case-Sensitive and case-INsensitive VFS - optCS := DefaultOpt + optCS := vfscommon.DefaultOpt optCS.CaseInsensitive = false vfsCS := New(r.Fremote, &optCS) - optCI := DefaultOpt + optCI := vfscommon.DefaultOpt optCI.CaseInsensitive = true vfsCI := New(r.Fremote, &optCI) diff --git a/vfs/vfs_test.go b/vfs/vfs_test.go index 4216a3a89..7f585729b 100644 --- a/vfs/vfs_test.go +++ b/vfs/vfs_test.go @@ -13,6 +13,7 @@ import ( _ "github.com/rclone/rclone/backend/all" // import all the backends "github.com/rclone/rclone/fs" "github.com/rclone/rclone/fstest" + "github.com/rclone/rclone/vfs/vfscommon" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -101,13 +102,13 @@ func TestVFSNew(t *testing.T) { // Check making a VFS with nil options vfs := New(r.Fremote, nil) - var defaultOpt = DefaultOpt + var defaultOpt = vfscommon.DefaultOpt defaultOpt.DirPerms |= os.ModeDir assert.Equal(t, vfs.Opt, defaultOpt) assert.Equal(t, vfs.f, r.Fremote) // Check the initialisation - var opt = DefaultOpt + var opt = vfscommon.DefaultOpt opt.DirPerms = 0777 opt.FilePerms = 0666 opt.Umask = 0002 diff --git a/vfs/cache.go b/vfs/vfscache/vfscache.go similarity index 68% rename from vfs/cache.go rename to vfs/vfscache/vfscache.go index 18f85a8cf..f08e7b1b9 100644 --- a/vfs/cache.go +++ b/vfs/vfscache/vfscache.go @@ -1,10 +1,8 @@ -// This deals with caching of files locally - -package vfs +// Package vfscache deals with caching of files locally for the VFS layer +package vfscache import ( "context" - "fmt" "os" "path/filepath" "runtime" @@ -18,58 +16,19 @@ import ( "github.com/rclone/rclone/fs" fscache "github.com/rclone/rclone/fs/cache" "github.com/rclone/rclone/fs/config" + "github.com/rclone/rclone/fs/operations" + "github.com/rclone/rclone/vfs/vfscommon" ) -// CacheMode controls the functionality of the cache -type CacheMode byte - -// CacheMode options -const ( - CacheModeOff CacheMode = iota // cache nothing - return errors for writes which can't be satisfied - CacheModeMinimal // cache only the minimum, eg read/write opens - CacheModeWrites // cache all files opened with write intent - CacheModeFull // cache all files opened in any mode -) - -var cacheModeToString = []string{ - CacheModeOff: "off", - CacheModeMinimal: "minimal", - CacheModeWrites: "writes", - CacheModeFull: "full", -} - -// String turns a CacheMode into a string -func (l CacheMode) String() string { - if l >= CacheMode(len(cacheModeToString)) { - return fmt.Sprintf("CacheMode(%d)", l) - } - return cacheModeToString[l] -} - -// Set a CacheMode -func (l *CacheMode) Set(s string) error { - for n, name := range cacheModeToString { - if s != "" && name == s { - *l = CacheMode(n) - return nil - } - } - return errors.Errorf("Unknown cache mode level %q", s) -} - -// Type of the value -func (l *CacheMode) Type() string { - return "CacheMode" -} - -// cache opened files -type cache struct { - f fs.Fs // fs for the cache directory - opt *Options // vfs Options - root string // root of the cache directory - itemMu sync.Mutex // protects the following variables - item map[string]*cacheItem // files/directories in the cache - used int64 // total size of files in the cache +// Cache opened files +type Cache struct { + fremote fs.Fs // fs for the remote we are caching + fcache fs.Fs // fs for the cache directory + opt *vfscommon.Options // vfs Options + root string // root of the cache directory + itemMu sync.Mutex // protects the following variables + item map[string]*cacheItem // files/directories in the cache + used int64 // total size of files in the cache } // cacheItem is stored in the item map @@ -85,31 +44,32 @@ func newCacheItem(isFile bool) *cacheItem { return &cacheItem{atime: time.Now(), isFile: isFile} } -// newCache creates a new cache heirachy for f +// New creates a new cache heirachy for fremote // // This starts background goroutines which can be cancelled with the // context passed in. -func newCache(ctx context.Context, f fs.Fs, opt *Options) (*cache, error) { - fRoot := filepath.FromSlash(f.Root()) +func New(ctx context.Context, fremote fs.Fs, opt *vfscommon.Options) (*Cache, error) { + fRoot := filepath.FromSlash(fremote.Root()) if runtime.GOOS == "windows" { if strings.HasPrefix(fRoot, `\\?`) { fRoot = fRoot[3:] } fRoot = strings.Replace(fRoot, ":", "", -1) } - root := filepath.Join(config.CacheDir, "vfs", f.Name(), fRoot) + root := filepath.Join(config.CacheDir, "vfs", fremote.Name(), fRoot) fs.Debugf(nil, "vfs cache root is %q", root) - f, err := fscache.Get(root) + fcache, err := fscache.Get(root) if err != nil { return nil, errors.Wrap(err, "failed to create cache remote") } - c := &cache{ - f: f, - opt: opt, - root: root, - item: make(map[string]*cacheItem), + c := &Cache{ + fremote: fremote, + fcache: fcache, + opt: opt, + root: root, + item: make(map[string]*cacheItem), } go c.cleaner(ctx) @@ -117,15 +77,6 @@ func newCache(ctx context.Context, f fs.Fs, opt *Options) (*cache, error) { return c, nil } -// findParent returns the parent directory of name, or "" for the root -func findParent(name string) string { - parent := filepath.Dir(name) - if parent == "." || parent == "/" { - parent = "" - } - return parent -} - // clean returns the cleaned version of name for use in the index map func clean(name string) string { name = strings.Trim(name, "/") @@ -136,17 +87,17 @@ func clean(name string) string { return name } -// toOSPath turns a remote relative name into an OS path in the cache -func (c *cache) toOSPath(name string) string { +// ToOSPath turns a remote relative name into an OS path in the cache +func (c *Cache) ToOSPath(name string) string { return filepath.Join(c.root, filepath.FromSlash(name)) } -// mkdir makes the directory for name in the cache and returns an os +// Mkdir makes the directory for name in the cache and returns an os // path for the file -func (c *cache) mkdir(name string) (string, error) { - parent := findParent(name) +func (c *Cache) Mkdir(name string) (string, error) { + parent := vfscommon.FindParent(name) leaf := filepath.Base(name) - parentPath := c.toOSPath(parent) + parentPath := c.ToOSPath(parent) err := os.MkdirAll(parentPath, 0700) if err != nil { return "", errors.Wrap(err, "make cache directory failed") @@ -163,7 +114,7 @@ func (c *cache) mkdir(name string) (string, error) { // name should be a remote path not an osPath // // must be called with itemMu held -func (c *cache) _get(isFile bool, name string) (item *cacheItem, found bool) { +func (c *Cache) _get(isFile bool, name string) (item *cacheItem, found bool) { item = c.item[name] found = item != nil if !found { @@ -173,10 +124,10 @@ func (c *cache) _get(isFile bool, name string) (item *cacheItem, found bool) { return item, found } -// opens returns the number of opens that are on the file +// Opens returns the number of opens that are on the file // // name should be a remote path not an osPath -func (c *cache) opens(name string) int { +func (c *Cache) Opens(name string) int { name = clean(name) c.itemMu.Lock() defer c.itemMu.Unlock() @@ -190,7 +141,7 @@ func (c *cache) opens(name string) int { // get gets name from the cache or creates a new one // // name should be a remote path not an osPath -func (c *cache) get(name string) *cacheItem { +func (c *Cache) get(name string) *cacheItem { name = clean(name) c.itemMu.Lock() item, _ := c._get(true, name) @@ -204,7 +155,7 @@ func (c *cache) get(name string) *cacheItem { // it also sets the size // // name should be a remote path not an osPath -func (c *cache) updateStat(name string, when time.Time, size int64) { +func (c *Cache) updateStat(name string, when time.Time, size int64) { name = clean(name) c.itemMu.Lock() item, found := c._get(true, name) @@ -219,7 +170,7 @@ func (c *cache) updateStat(name string, when time.Time, size int64) { // _open marks name as open, must be called with the lock held // // name should be a remote path not an osPath -func (c *cache) _open(isFile bool, name string) { +func (c *Cache) _open(isFile bool, name string) { for { item, _ := c._get(isFile, name) item.opens++ @@ -228,14 +179,14 @@ func (c *cache) _open(isFile bool, name string) { break } isFile = false - name = findParent(name) + name = vfscommon.FindParent(name) } } -// open marks name as open +// Open marks name as open // // name should be a remote path not an osPath -func (c *cache) open(name string) { +func (c *Cache) Open(name string) { name = clean(name) c.itemMu.Lock() c._open(true, name) @@ -245,7 +196,7 @@ func (c *cache) open(name string) { // cacheDir marks a directory and its parents as being in the cache // // name should be a remote path not an osPath -func (c *cache) cacheDir(name string) { +func (c *Cache) cacheDir(name string) { name = clean(name) c.itemMu.Lock() defer c.itemMu.Unlock() @@ -258,13 +209,13 @@ func (c *cache) cacheDir(name string) { if name == "" { break } - name = findParent(name) + name = vfscommon.FindParent(name) } } -// exists checks to see if the file exists in the cache or not -func (c *cache) exists(name string) bool { - osPath := c.toOSPath(name) +// Exists checks to see if the file exists in the cache or not +func (c *Cache) Exists(name string) bool { + osPath := c.ToOSPath(name) fi, err := os.Stat(osPath) if err != nil { return false @@ -276,10 +227,10 @@ func (c *cache) exists(name string) bool { return true } -// renames the file in cache -func (c *cache) rename(name string, newName string) (err error) { - osOldPath := c.toOSPath(name) - osNewPath := c.toOSPath(newName) +// Rename the file in cache +func (c *Cache) Rename(name string, newName string) (err error) { + osOldPath := c.ToOSPath(name) + osNewPath := c.ToOSPath(newName) sfi, err := os.Stat(osOldPath) if err != nil { return errors.Wrapf(err, "Failed to stat source: %s", osOldPath) @@ -293,7 +244,7 @@ func (c *cache) rename(name string, newName string) (err error) { if !os.IsNotExist(err) { return errors.Wrapf(err, "Failed to stat destination: %s", osNewPath) } - parent := findParent(osNewPath) + parent := vfscommon.FindParent(osNewPath) err = os.MkdirAll(parent, 0700) if err != nil { return errors.Wrapf(err, "Failed to create parent dir: %s", parent) @@ -321,7 +272,7 @@ func (c *cache) rename(name string, newName string) (err error) { } // _close marks name as closed - must be called with the lock held -func (c *cache) _close(isFile bool, name string) { +func (c *Cache) _close(isFile bool, name string) { for { item, _ := c._get(isFile, name) item.opens-- @@ -329,7 +280,7 @@ func (c *cache) _close(isFile bool, name string) { if item.opens < 0 { fs.Errorf(name, "cache: double close") } - osPath := c.toOSPath(name) + osPath := c.ToOSPath(name) fi, err := os.Stat(osPath) // Update the size on close if err == nil && !fi.IsDir() { @@ -339,23 +290,23 @@ func (c *cache) _close(isFile bool, name string) { break } isFile = false - name = findParent(name) + name = vfscommon.FindParent(name) } } -// close marks name as closed +// Close marks name as closed // // name should be a remote path not an osPath -func (c *cache) close(name string) { +func (c *Cache) Close(name string) { name = clean(name) c.itemMu.Lock() c._close(true, name) c.itemMu.Unlock() } -// remove should be called if name is deleted -func (c *cache) remove(name string) { - osPath := c.toOSPath(name) +// Remove should be called if name is deleted +func (c *Cache) Remove(name string) { + osPath := c.ToOSPath(name) err := os.Remove(osPath) if err != nil && !os.IsNotExist(err) { fs.Errorf(name, "Failed to remove from cache: %v", err) @@ -366,8 +317,8 @@ func (c *cache) remove(name string) { // removeDir should be called if dir is deleted and returns true if // the directory is gone. -func (c *cache) removeDir(dir string) bool { - osPath := c.toOSPath(dir) +func (c *Cache) removeDir(dir string) bool { + osPath := c.ToOSPath(dir) err := os.Remove(osPath) if err == nil || os.IsNotExist(err) { if err == nil { @@ -381,22 +332,22 @@ func (c *cache) removeDir(dir string) bool { return false } -// setModTime should be called to set the modification time of the cache file -func (c *cache) setModTime(name string, modTime time.Time) { - osPath := c.toOSPath(name) +// SetModTime should be called to set the modification time of the cache file +func (c *Cache) SetModTime(name string, modTime time.Time) { + osPath := c.ToOSPath(name) err := os.Chtimes(osPath, modTime, modTime) if err != nil { fs.Errorf(name, "Failed to set modification time of cached file: %v", err) } } -// cleanUp empties the cache of everything -func (c *cache) cleanUp() error { +// CleanUp empties the cache of everything +func (c *Cache) CleanUp() error { return os.RemoveAll(c.root) } // walk walks the cache calling the function -func (c *cache) walk(fn func(osPath string, fi os.FileInfo, name string) error) error { +func (c *Cache) walk(fn func(osPath string, fi os.FileInfo, name string) error) error { return filepath.Walk(c.root, func(osPath string, fi os.FileInfo, err error) error { if err != nil { return err @@ -419,7 +370,7 @@ func (c *cache) walk(fn func(osPath string, fi os.FileInfo, name string) error) // updateStats walks the cache updating any atimes and sizes it finds // // it also updates used -func (c *cache) updateStats() error { +func (c *Cache) updateStats() error { var newUsed int64 err := c.walk(func(osPath string, fi os.FileInfo, name string) error { if !fi.IsDir() { @@ -439,11 +390,11 @@ func (c *cache) updateStats() error { } // purgeOld gets rid of any files that are over age -func (c *cache) purgeOld(maxAge time.Duration) { - c._purgeOld(maxAge, c.remove) +func (c *Cache) purgeOld(maxAge time.Duration) { + c._purgeOld(maxAge, c.Remove) } -func (c *cache) _purgeOld(maxAge time.Duration, remove func(name string)) { +func (c *Cache) _purgeOld(maxAge time.Duration, remove func(name string)) { c.itemMu.Lock() defer c.itemMu.Unlock() cutoff := time.Now().Add(-maxAge) @@ -462,11 +413,11 @@ func (c *cache) _purgeOld(maxAge time.Duration, remove func(name string)) { } // Purge any empty directories -func (c *cache) purgeEmptyDirs() { +func (c *Cache) purgeEmptyDirs() { c._purgeEmptyDirs(c.removeDir) } -func (c *cache) _purgeEmptyDirs(removeDir func(name string) bool) { +func (c *Cache) _purgeEmptyDirs(removeDir func(name string) bool) { c.itemMu.Lock() defer c.itemMu.Unlock() var dirs []string @@ -499,11 +450,11 @@ func (v cacheNamedItems) Less(i, j int) bool { return v[i].item.atime.Before(v[j // Remove any files that are over quota starting from the // oldest first -func (c *cache) purgeOverQuota(quota int64) { - c._purgeOverQuota(quota, c.remove) +func (c *Cache) purgeOverQuota(quota int64) { + c._purgeOverQuota(quota, c.Remove) } -func (c *cache) _purgeOverQuota(quota int64, remove func(name string)) { +func (c *Cache) _purgeOverQuota(quota int64, remove func(name string)) { c.itemMu.Lock() defer c.itemMu.Unlock() @@ -537,7 +488,7 @@ func (c *cache) _purgeOverQuota(quota int64, remove func(name string)) { } // clean empties the cache of stuff if it can -func (c *cache) clean() { +func (c *Cache) clean() { // Cache may be empty so end _, err := os.Stat(c.root) if os.IsNotExist(err) { @@ -575,7 +526,7 @@ func (c *cache) clean() { // cleaner calls clean at regular intervals // // doesn't return until context is cancelled -func (c *cache) cleaner(ctx context.Context) { +func (c *Cache) cleaner(ctx context.Context) { if c.opt.CachePollInterval <= 0 { fs.Debugf(nil, "Cache cleaning thread disabled because poll interval <= 0") return @@ -595,3 +546,50 @@ func (c *cache) cleaner(ctx context.Context) { } } } + +// copy an object to or from the remote while accounting for it +func copyObj(f fs.Fs, dst fs.Object, remote string, src fs.Object) (newDst fs.Object, err error) { + if operations.NeedTransfer(context.TODO(), dst, src) { + newDst, err = operations.Copy(context.TODO(), f, dst, remote, src) + } else { + newDst = dst + } + return newDst, err +} + +// Check the local file is up to date in the cache +func (c *Cache) Check(ctx context.Context, o fs.Object, remote string) error { + cacheObj, err := c.fcache.NewObject(ctx, remote) + if err == nil && cacheObj != nil { + _, err = copyObj(c.fcache, cacheObj, remote, o) + if err != nil { + return errors.Wrap(err, "failed to update cached file") + } + } + return nil +} + +// Fetch fetches the object to the cache file +func (c *Cache) Fetch(ctx context.Context, o fs.Object, remote string) error { + _, err := copyObj(c.fcache, nil, remote, o) + return err +} + +// Store stores the local cache file to the remote object, returning +// the new remote object. objOld is the old object if known. +func (c *Cache) Store(ctx context.Context, objOld fs.Object, remote string) (fs.Object, error) { + // Transfer the temp file to the remote + cacheObj, err := c.fcache.NewObject(ctx, remote) + if err != nil { + return nil, errors.Wrap(err, "failed to find cache file") + } + + if objOld != nil { + remote = objOld.Remote() // use the path of the actual object if available + } + o, err := copyObj(c.fremote, objOld, remote, cacheObj) + if err != nil { + return nil, errors.Wrap(err, "failed to transfer file from cache to remote") + } + return o, nil +} diff --git a/vfs/cache_test.go b/vfs/vfscache/vfscache_test.go similarity index 87% rename from vfs/cache_test.go rename to vfs/vfscache/vfscache_test.go index 65112e990..771d5fb64 100644 --- a/vfs/cache_test.go +++ b/vfs/vfscache/vfscache_test.go @@ -1,4 +1,4 @@ -package vfs +package vfscache import ( "context" @@ -11,42 +11,20 @@ import ( "time" "github.com/djherbis/times" + _ "github.com/rclone/rclone/backend/local" // import the local backend "github.com/rclone/rclone/fstest" - "github.com/spf13/pflag" + "github.com/rclone/rclone/vfs/vfscommon" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) -// Check CacheMode it satisfies the pflag interface -var _ pflag.Value = (*CacheMode)(nil) - -func TestCacheModeString(t *testing.T) { - assert.Equal(t, "off", CacheModeOff.String()) - assert.Equal(t, "full", CacheModeFull.String()) - assert.Equal(t, "CacheMode(17)", CacheMode(17).String()) -} - -func TestCacheModeSet(t *testing.T) { - var m CacheMode - - err := m.Set("full") - assert.NoError(t, err) - assert.Equal(t, CacheModeFull, m) - - err = m.Set("potato") - assert.Error(t, err, "Unknown cache mode level") - - err = m.Set("") - assert.Error(t, err, "Unknown cache mode level") -} - -func TestCacheModeType(t *testing.T) { - var m CacheMode - assert.Equal(t, "CacheMode", m.Type()) +// TestMain drives the tests +func TestMain(m *testing.M) { + fstest.TestMain(m) } // convert c.item to a string -func itemAsString(c *cache) []string { +func itemAsString(c *Cache) []string { c.itemMu.Lock() defer c.itemMu.Unlock() var out []string @@ -65,17 +43,17 @@ func TestCacheNew(t *testing.T) { defer cancel() // Disable the cache cleaner as it interferes with these tests - opt := DefaultOpt + opt := vfscommon.DefaultOpt opt.CachePollInterval = 0 - c, err := newCache(ctx, r.Fremote, &opt) + c, err := New(ctx, r.Fremote, &opt) require.NoError(t, err) assert.Contains(t, c.root, "vfs") - assert.Contains(t, c.f.Root(), filepath.Base(r.Fremote.Root())) + assert.Contains(t, c.fcache.Root(), filepath.Base(r.Fremote.Root())) assert.Equal(t, []string(nil), itemAsString(c)) // mkdir - p, err := c.mkdir("potato") + p, err := c.Mkdir("potato") require.NoError(t, err) assert.Equal(t, "potato", filepath.Base(p)) assert.Equal(t, []string{ @@ -111,7 +89,7 @@ func TestCacheNew(t *testing.T) { `name="" isFile=false opens=0 size=0`, `name="potato" isFile=true opens=0 size=0`, }, itemAsString(c)) - c.open("/potato") + c.Open("/potato") assert.Equal(t, []string{ `name="" isFile=false opens=1 size=0`, `name="potato" isFile=true opens=1 size=0`, @@ -173,7 +151,7 @@ func TestCacheNew(t *testing.T) { `name="" isFile=false opens=1 size=0`, `name="potato" isFile=true opens=1 size=6`, }, itemAsString(c)) - c.close("potato/") + c.Close("potato/") assert.Equal(t, []string{ `name="" isFile=false opens=0 size=0`, `name="potato" isFile=true opens=0 size=5`, @@ -197,7 +175,7 @@ func TestCacheNew(t *testing.T) { c.clean() // cleanup - err = c.cleanUp() + err = c.CleanUp() require.NoError(t, err) _, err = os.Stat(c.root) assert.True(t, os.IsNotExist(err)) @@ -210,35 +188,35 @@ func TestCacheOpens(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() - c, err := newCache(ctx, r.Fremote, &DefaultOpt) + c, err := New(ctx, r.Fremote, &vfscommon.DefaultOpt) require.NoError(t, err) assert.Equal(t, []string(nil), itemAsString(c)) - c.open("potato") + c.Open("potato") assert.Equal(t, []string{ `name="" isFile=false opens=1 size=0`, `name="potato" isFile=true opens=1 size=0`, }, itemAsString(c)) - c.open("potato") + c.Open("potato") assert.Equal(t, []string{ `name="" isFile=false opens=2 size=0`, `name="potato" isFile=true opens=2 size=0`, }, itemAsString(c)) - c.close("potato") + c.Close("potato") assert.Equal(t, []string{ `name="" isFile=false opens=1 size=0`, `name="potato" isFile=true opens=1 size=0`, }, itemAsString(c)) - c.close("potato") + c.Close("potato") assert.Equal(t, []string{ `name="" isFile=false opens=0 size=0`, `name="potato" isFile=true opens=0 size=0`, }, itemAsString(c)) - c.open("potato") - c.open("a//b/c/d/one") - c.open("a/b/c/d/e/two") - c.open("a/b/c/d/e/f/three") + c.Open("potato") + c.Open("a//b/c/d/one") + c.Open("a/b/c/d/e/two") + c.Open("a/b/c/d/e/f/three") assert.Equal(t, []string{ `name="" isFile=false opens=4 size=0`, `name="a" isFile=false opens=3 size=0`, @@ -252,10 +230,10 @@ func TestCacheOpens(t *testing.T) { `name="a/b/c/d/one" isFile=true opens=1 size=0`, `name="potato" isFile=true opens=1 size=0`, }, itemAsString(c)) - c.close("potato") - c.close("a/b/c/d/one") - c.close("a/b/c/d/e/two") - c.close("a/b/c//d/e/f/three") + c.Close("potato") + c.Close("a/b/c/d/one") + c.Close("a/b/c/d/e/two") + c.Close("a/b/c//d/e/f/three") assert.Equal(t, []string{ `name="" isFile=false opens=0 size=0`, `name="a" isFile=false opens=0 size=0`, @@ -280,13 +258,13 @@ func TestCacheOpenMkdir(t *testing.T) { defer cancel() // Disable the cache cleaner as it interferes with these tests - opt := DefaultOpt + opt := vfscommon.DefaultOpt opt.CachePollInterval = 0 - c, err := newCache(ctx, r.Fremote, &opt) + c, err := New(ctx, r.Fremote, &opt) require.NoError(t, err) // open - c.open("sub/potato") + c.Open("sub/potato") assert.Equal(t, []string{ `name="" isFile=false opens=1 size=0`, @@ -295,7 +273,7 @@ func TestCacheOpenMkdir(t *testing.T) { }, itemAsString(c)) // mkdir - p, err := c.mkdir("sub/potato") + p, err := c.Mkdir("sub/potato") require.NoError(t, err) assert.Equal(t, "potato", filepath.Base(p)) assert.Equal(t, []string{ @@ -318,7 +296,7 @@ func TestCacheOpenMkdir(t *testing.T) { assert.True(t, fi.IsDir()) // close - c.close("sub/potato") + c.Close("sub/potato") assert.Equal(t, []string{ `name="" isFile=false opens=0 size=0`, @@ -344,7 +322,7 @@ func TestCacheCacheDir(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() - c, err := newCache(ctx, r.Fremote, &DefaultOpt) + c, err := New(ctx, r.Fremote, &vfscommon.DefaultOpt) require.NoError(t, err) assert.Equal(t, []string(nil), itemAsString(c)) @@ -379,7 +357,7 @@ func TestCachePurgeOld(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() - c, err := newCache(ctx, r.Fremote, &DefaultOpt) + c, err := New(ctx, r.Fremote, &vfscommon.DefaultOpt) require.NoError(t, err) // Test funcs @@ -400,10 +378,10 @@ func TestCachePurgeOld(t *testing.T) { c._purgeEmptyDirs(removeDir) assert.Equal(t, []string(nil), removed) - c.open("sub/dir2/potato2") - c.open("sub/dir/potato") - c.close("sub/dir2/potato2") - c.open("sub/dir/potato") + c.Open("sub/dir2/potato2") + c.Open("sub/dir/potato") + c.Close("sub/dir2/potato2") + c.Open("sub/dir/potato") assert.Equal(t, []string{ `name="" isFile=false opens=2 size=0`, @@ -423,7 +401,7 @@ func TestCachePurgeOld(t *testing.T) { "sub/dir2/", }, removed) - c.close("sub/dir/potato") + c.Close("sub/dir/potato") removed = nil removedDir = true @@ -431,7 +409,7 @@ func TestCachePurgeOld(t *testing.T) { c._purgeEmptyDirs(removeDir) assert.Equal(t, []string(nil), removed) - c.close("sub/dir/potato") + c.Close("sub/dir/potato") assert.Equal(t, []string{ `name="" isFile=false opens=0 size=0`, @@ -475,16 +453,16 @@ func TestCachePurgeOverQuota(t *testing.T) { defer cancel() // Disable the cache cleaner as it interferes with these tests - opt := DefaultOpt + opt := vfscommon.DefaultOpt opt.CachePollInterval = 0 - c, err := newCache(ctx, r.Fremote, &opt) + c, err := New(ctx, r.Fremote, &opt) require.NoError(t, err) // Test funcs var removed []string remove := func(name string) { removed = append(removed, filepath.ToSlash(name)) - c.remove(name) + c.Remove(name) } removed = nil @@ -500,14 +478,14 @@ func TestCachePurgeOverQuota(t *testing.T) { assert.Equal(t, []string(nil), removed) // Make some test files - c.open("sub/dir/potato") - p, err := c.mkdir("sub/dir/potato") + c.Open("sub/dir/potato") + p, err := c.Mkdir("sub/dir/potato") require.NoError(t, err) err = ioutil.WriteFile(p, []byte("hello"), 0600) require.NoError(t, err) - p, err = c.mkdir("sub/dir2/potato2") - c.open("sub/dir2/potato2") + p, err = c.Mkdir("sub/dir2/potato2") + c.Open("sub/dir2/potato2") require.NoError(t, err) err = ioutil.WriteFile(p, []byte("hello2"), 0600) require.NoError(t, err) @@ -527,8 +505,8 @@ func TestCachePurgeOverQuota(t *testing.T) { assert.Equal(t, []string(nil), removed) // Close the files - c.close("sub/dir/potato") - c.close("sub/dir2/potato2") + c.Close("sub/dir/potato") + c.Close("sub/dir2/potato2") assert.Equal(t, []string{ `name="" isFile=false opens=0 size=0`, @@ -565,12 +543,12 @@ func TestCachePurgeOverQuota(t *testing.T) { }, itemAsString(c)) // Put potato back - c.open("sub/dir/potato") - p, err = c.mkdir("sub/dir/potato") + c.Open("sub/dir/potato") + p, err = c.Mkdir("sub/dir/potato") require.NoError(t, err) err = ioutil.WriteFile(p, []byte("hello"), 0600) require.NoError(t, err) - c.close("sub/dir/potato") + c.Close("sub/dir/potato") // Update the stats to read the total size err = c.updateStats() diff --git a/vfs/vfscommon/cachemode.go b/vfs/vfscommon/cachemode.go new file mode 100644 index 000000000..9df7e8452 --- /dev/null +++ b/vfs/vfscommon/cachemode.go @@ -0,0 +1,49 @@ +package vfscommon + +import ( + "fmt" + + "github.com/rclone/rclone/lib/errors" +) + +// CacheMode controls the functionality of the cache +type CacheMode byte + +// CacheMode options +const ( + CacheModeOff CacheMode = iota // cache nothing - return errors for writes which can't be satisfied + CacheModeMinimal // cache only the minimum, eg read/write opens + CacheModeWrites // cache all files opened with write intent + CacheModeFull // cache all files opened in any mode +) + +var cacheModeToString = []string{ + CacheModeOff: "off", + CacheModeMinimal: "minimal", + CacheModeWrites: "writes", + CacheModeFull: "full", +} + +// String turns a CacheMode into a string +func (l CacheMode) String() string { + if l >= CacheMode(len(cacheModeToString)) { + return fmt.Sprintf("CacheMode(%d)", l) + } + return cacheModeToString[l] +} + +// Set a CacheMode +func (l *CacheMode) Set(s string) error { + for n, name := range cacheModeToString { + if s != "" && name == s { + *l = CacheMode(n) + return nil + } + } + return errors.Errorf("Unknown cache mode level %q", s) +} + +// Type of the value +func (l *CacheMode) Type() string { + return "CacheMode" +} diff --git a/vfs/vfscommon/cachemode_test.go b/vfs/vfscommon/cachemode_test.go new file mode 100644 index 000000000..04c3a8119 --- /dev/null +++ b/vfs/vfscommon/cachemode_test.go @@ -0,0 +1,36 @@ +package vfscommon + +import ( + "testing" + + "github.com/spf13/pflag" + "github.com/stretchr/testify/assert" +) + +// Check CacheMode it satisfies the pflag interface +var _ pflag.Value = (*CacheMode)(nil) + +func TestCacheModeString(t *testing.T) { + assert.Equal(t, "off", CacheModeOff.String()) + assert.Equal(t, "full", CacheModeFull.String()) + assert.Equal(t, "CacheMode(17)", CacheMode(17).String()) +} + +func TestCacheModeSet(t *testing.T) { + var m CacheMode + + err := m.Set("full") + assert.NoError(t, err) + assert.Equal(t, CacheModeFull, m) + + err = m.Set("potato") + assert.Error(t, err, "Unknown cache mode level") + + err = m.Set("") + assert.Error(t, err, "Unknown cache mode level") +} + +func TestCacheModeType(t *testing.T) { + var m CacheMode + assert.Equal(t, "CacheMode", m.Type()) +} diff --git a/vfs/vfscommon/options.go b/vfs/vfscommon/options.go new file mode 100644 index 000000000..b22e69b2a --- /dev/null +++ b/vfs/vfscommon/options.go @@ -0,0 +1,57 @@ +package vfscommon + +import ( + "os" + "runtime" + "time" + + "github.com/rclone/rclone/fs" +) + +// Options is options for creating the vfs +type Options struct { + NoSeek bool // don't allow seeking if set + NoChecksum bool // don't check checksums if set + ReadOnly bool // if set VFS is read only + NoModTime bool // don't read mod times for files + DirCacheTime time.Duration // how long to consider directory listing cache valid + PollInterval time.Duration + Umask int + UID uint32 + GID uint32 + DirPerms os.FileMode + FilePerms os.FileMode + ChunkSize fs.SizeSuffix // if > 0 read files in chunks + ChunkSizeLimit fs.SizeSuffix // if > ChunkSize double the chunk size after each chunk until reached + CacheMode CacheMode + CacheMaxAge time.Duration + CacheMaxSize fs.SizeSuffix + CachePollInterval time.Duration + CaseInsensitive bool + WriteWait time.Duration // time to wait for in-sequence write + ReadWait time.Duration // time to wait for in-sequence read +} + +// DefaultOpt is the default values uses for Opt +var DefaultOpt = Options{ + NoModTime: false, + NoChecksum: false, + NoSeek: false, + DirCacheTime: 5 * 60 * time.Second, + PollInterval: time.Minute, + ReadOnly: false, + Umask: 0, + UID: ^uint32(0), // these values instruct WinFSP-FUSE to use the current user + GID: ^uint32(0), // overriden for non windows in mount_unix.go + DirPerms: os.FileMode(0777), + FilePerms: os.FileMode(0666), + CacheMode: CacheModeOff, + CacheMaxAge: 3600 * time.Second, + CachePollInterval: 60 * time.Second, + ChunkSize: 128 * fs.MebiByte, + ChunkSizeLimit: -1, + CacheMaxSize: -1, + CaseInsensitive: runtime.GOOS == "windows" || runtime.GOOS == "darwin", // default to true on Windows and Mac, false otherwise + WriteWait: 1000 * time.Millisecond, + ReadWait: 5 * time.Millisecond, +} diff --git a/vfs/vfscommon/path.go b/vfs/vfscommon/path.go new file mode 100644 index 000000000..3e51ff4b9 --- /dev/null +++ b/vfs/vfscommon/path.go @@ -0,0 +1,12 @@ +package vfscommon + +import "path/filepath" + +// FindParent returns the parent directory of name, or "" for the root +func FindParent(name string) string { + parent := filepath.Dir(name) + if parent == "." || parent == "/" { + parent = "" + } + return parent +} diff --git a/vfs/vfsflags/vfsflags.go b/vfs/vfsflags/vfsflags.go index 3c8703466..f7c452cc6 100644 --- a/vfs/vfsflags/vfsflags.go +++ b/vfs/vfsflags/vfsflags.go @@ -4,13 +4,13 @@ package vfsflags import ( "github.com/rclone/rclone/fs/config/flags" "github.com/rclone/rclone/fs/rc" - "github.com/rclone/rclone/vfs" + "github.com/rclone/rclone/vfs/vfscommon" "github.com/spf13/pflag" ) // Options set by command line flags var ( - Opt = vfs.DefaultOpt + Opt = vfscommon.DefaultOpt DirPerms = &FileMode{Mode: &Opt.DirPerms} FilePerms = &FileMode{Mode: &Opt.FilePerms} ) diff --git a/vfs/vfstest/fs.go b/vfs/vfstest/fs.go index b4e397207..97c0786e2 100644 --- a/vfs/vfstest/fs.go +++ b/vfs/vfstest/fs.go @@ -24,6 +24,7 @@ import ( "github.com/rclone/rclone/fs/walk" "github.com/rclone/rclone/fstest" "github.com/rclone/rclone/vfs" + "github.com/rclone/rclone/vfs/vfscommon" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -45,11 +46,11 @@ var ( func RunTests(t *testing.T, useVFS bool, fn MountFn) { mountFn = fn flag.Parse() - cacheModes := []vfs.CacheMode{ - vfs.CacheModeOff, - vfs.CacheModeMinimal, - vfs.CacheModeWrites, - vfs.CacheModeFull, + cacheModes := []vfscommon.CacheMode{ + vfscommon.CacheModeOff, + vfscommon.CacheModeMinimal, + vfscommon.CacheModeWrites, + vfscommon.CacheModeFull, } run = newRun(useVFS) for _, cacheMode := range cacheModes { @@ -218,7 +219,7 @@ func (r *Run) umount() { } // cacheMode flushes the VFS and changes the CacheMode -func (r *Run) cacheMode(cacheMode vfs.CacheMode) { +func (r *Run) cacheMode(cacheMode vfscommon.CacheMode) { if r.skip { log.Printf("FUSE not found so skipping cacheMode") return diff --git a/vfs/vfstest/write.go b/vfs/vfstest/write.go index 2ec68b311..03ad7bfa1 100644 --- a/vfs/vfstest/write.go +++ b/vfs/vfstest/write.go @@ -5,10 +5,9 @@ import ( "runtime" "testing" + "github.com/rclone/rclone/vfs/vfscommon" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - - "github.com/rclone/rclone/vfs" ) // TestWriteFileNoWrite tests writing a file with no write()'s to it @@ -92,7 +91,7 @@ func TestWriteFileDup(t *testing.T) { run.skipIfVFS(t) run.skipIfNoFUSE(t) - if run.vfs.Opt.CacheMode < vfs.CacheModeWrites { + if run.vfs.Opt.CacheMode < vfscommon.CacheModeWrites { t.Skip("not supported on vfs-cache-mode < writes") return } @@ -137,7 +136,7 @@ func TestWriteFileDup(t *testing.T) { func TestWriteFileAppend(t *testing.T) { run.skipIfNoFUSE(t) - if run.vfs.Opt.CacheMode < vfs.CacheModeWrites { + if run.vfs.Opt.CacheMode < vfscommon.CacheModeWrites { t.Skip("not supported on vfs-cache-mode < writes") return } diff --git a/vfs/vfstest/write_unix.go b/vfs/vfstest/write_unix.go index 822590ba9..1c51d142a 100644 --- a/vfs/vfstest/write_unix.go +++ b/vfs/vfstest/write_unix.go @@ -6,10 +6,9 @@ import ( "runtime" "testing" + "github.com/rclone/rclone/vfs/vfscommon" "github.com/stretchr/testify/assert" "golang.org/x/sys/unix" - - "github.com/rclone/rclone/vfs" ) // TestWriteFileDoubleClose tests double close on write @@ -46,7 +45,7 @@ func TestWriteFileDoubleClose(t *testing.T) { // write to the other dup _, err = unix.Write(fd2, buf) - if run.vfs.Opt.CacheMode < vfs.CacheModeWrites { + if run.vfs.Opt.CacheMode < vfscommon.CacheModeWrites { // produces an error if cache mode < writes assert.Error(t, err, "input/output error") } else {