From dcce65b2b32e45df0f6832f73e5430124012be8b Mon Sep 17 00:00:00 2001 From: Nick Craig-Wood Date: Mon, 19 Jun 2017 13:44:49 +0100 Subject: [PATCH] mount/cmount: factor duplicated code into mountlib --- cmd/cmount/fs.go | 23 +---- cmd/cmount/mount.go | 177 +++------------------------------ cmd/cmount/mount_test.go | 2 +- cmd/cmount/mount_unix.go | 18 ---- cmd/cmount/mount_windows.go | 7 -- cmd/mount/dir.go | 8 +- cmd/mount/file.go | 12 +-- cmd/mount/fs.go | 13 --- cmd/mount/mount.go | 172 +++----------------------------- cmd/mount/mount_test.go | 2 +- cmd/mountlib/fs.go | 40 +++----- cmd/mountlib/mount.go | 172 ++++++++++++++++++++++++++++++++ cmd/mountlib/mount_non_unix.go | 11 ++ cmd/mountlib/mount_unix.go | 19 ++++ cmd/mountlib/mounttest/fs.go | 27 +++-- 15 files changed, 275 insertions(+), 428 deletions(-) delete mode 100644 cmd/cmount/mount_unix.go delete mode 100644 cmd/cmount/mount_windows.go create mode 100644 cmd/mountlib/mount.go create mode 100644 cmd/mountlib/mount_non_unix.go create mode 100644 cmd/mountlib/mount_unix.go diff --git a/cmd/cmount/fs.go b/cmd/cmount/fs.go index 3269effa0..f776ef6a2 100644 --- a/cmd/cmount/fs.go +++ b/cmd/cmount/fs.go @@ -39,19 +39,6 @@ func NewFS(f fs.Fs) *FS { openFilesRd: newOpenFiles(0x03), ready: make(chan (struct{})), } - if noSeek { - fsys.FS.NoSeek() - } - if noChecksum { - fsys.FS.NoChecksum() - } - if readOnly { - fsys.FS.ReadOnly() - } - if pollInterval > 0 { - fsys.FS.PollChanges(pollInterval) - } - fsys.FS.SetDirCacheTime(dirCacheTime) return fsys } @@ -214,21 +201,21 @@ func (fsys *FS) stat(node mountlib.Node, stat *fuse.Stat_t) (errc int) { switch x := node.(type) { case *mountlib.Dir: modTime = x.ModTime() - Mode = dirPerms | fuse.S_IFDIR + Mode = mountlib.DirPerms | fuse.S_IFDIR case *mountlib.File: var err error - modTime, Size, Blocks, err = x.Attr(noModTime) + modTime, Size, Blocks, err = x.Attr(mountlib.NoModTime) if err != nil { return translateError(err) } - Mode = filePerms | fuse.S_IFREG + Mode = mountlib.FilePerms | fuse.S_IFREG } //stat.Dev = 1 stat.Ino = node.Inode() // FIXME do we need to set the inode number? stat.Mode = uint32(Mode) stat.Nlink = 1 - stat.Uid = uid - stat.Gid = gid + stat.Uid = mountlib.UID + stat.Gid = mountlib.GID //stat.Rdev stat.Size = int64(Size) t := fuse.NewTimespec(modTime) diff --git a/cmd/cmount/mount.go b/cmd/cmount/mount.go index 52ed96c1c..4bcec703e 100644 --- a/cmd/cmount/mount.go +++ b/cmd/cmount/mount.go @@ -10,7 +10,6 @@ package cmount import ( "fmt" - "log" "os" "os/signal" "runtime" @@ -18,153 +17,17 @@ import ( "time" "github.com/billziss-gh/cgofuse/fuse" - "github.com/ncw/rclone/cmd" "github.com/ncw/rclone/cmd/mountlib" "github.com/ncw/rclone/fs" "github.com/pkg/errors" - "github.com/spf13/cobra" -) - -// Globals -var ( - noModTime = false - noChecksum = false - debugFUSE = false - noSeek = false - dirCacheTime = 5 * 60 * time.Second - pollInterval = time.Minute - // mount options - readOnly = false - allowNonEmpty = false - allowRoot = false - allowOther = false - defaultPermissions = false - writebackCache = false - maxReadAhead fs.SizeSuffix = 128 * 1024 - umask = 0 - uid = ^uint32(0) // these values instruct WinFSP-FUSE to use the current user - gid = ^uint32(0) // overriden for non windows in mount_unix.go - // foreground = false - // default permissions for directories - modified by umask in Mount - dirPerms = os.FileMode(0777) - filePerms = os.FileMode(0666) - extraOptions *[]string - extraFlags *[]string ) func init() { - cmd.Root.AddCommand(commandDefintion) - commandDefintion.Flags().BoolVarP(&noModTime, "no-modtime", "", noModTime, "Don't read/write the modification time (can speed things up).") - commandDefintion.Flags().BoolVarP(&noChecksum, "no-checksum", "", noChecksum, "Don't compare checksums on up/download.") - commandDefintion.Flags().BoolVarP(&debugFUSE, "debug-fuse", "", debugFUSE, "Debug the FUSE internals - needs -v.") - commandDefintion.Flags().BoolVarP(&noSeek, "no-seek", "", noSeek, "Don't allow seeking in files.") - commandDefintion.Flags().DurationVarP(&dirCacheTime, "dir-cache-time", "", dirCacheTime, "Time to cache directory entries for.") - commandDefintion.Flags().DurationVarP(&pollInterval, "poll-interval", "", pollInterval, "Time to wait between polling for changes. Must be smaller than dir-cache-time. Only on supported remotes. Set to 0 to disable.") - // mount options - commandDefintion.Flags().BoolVarP(&readOnly, "read-only", "", readOnly, "Mount read-only.") - commandDefintion.Flags().BoolVarP(&allowNonEmpty, "allow-non-empty", "", allowNonEmpty, "Allow mounting over a non-empty directory.") - commandDefintion.Flags().BoolVarP(&allowRoot, "allow-root", "", allowRoot, "Allow access to root user.") - commandDefintion.Flags().BoolVarP(&allowOther, "allow-other", "", allowOther, "Allow access to other users.") - commandDefintion.Flags().BoolVarP(&defaultPermissions, "default-permissions", "", defaultPermissions, "Makes kernel enforce access control based on the file mode.") - commandDefintion.Flags().BoolVarP(&writebackCache, "write-back-cache", "", writebackCache, "Makes kernel buffer writes before sending them to rclone. Without this, writethrough caching is used.") - commandDefintion.Flags().VarP(&maxReadAhead, "max-read-ahead", "", "The number of bytes that can be prefetched for sequential reads.") - commandDefintion.Flags().IntVarP(&umask, "umask", "", umask, "Override the permission bits set by the filesystem.") - extraOptions = commandDefintion.Flags().StringArrayP("option", "o", []string{}, "Option for libfuse/WinFsp. Repeat if required.") - extraFlags = commandDefintion.Flags().StringArrayP("fuse-flag", "", []string{}, "Flags or arguments to be passed direct to libfuse/WinFsp. Repeat if required.") - //commandDefintion.Flags().BoolVarP(&foreground, "foreground", "", foreground, "Do not detach.") -} - -var commandDefintion = &cobra.Command{ - Use: commandName + " remote:path /path/to/mountpoint", - Short: `Mount the remote as a mountpoint. **EXPERIMENTAL**`, - Long: ` -rclone ` + commandName + ` allows Linux, FreeBSD, macOS and Windows to -mount any of Rclone's cloud storage systems as a file system with -FUSE. - -This is **EXPERIMENTAL** - use with care. - -First set up your remote using ` + "`rclone config`" + `. Check it works with ` + "`rclone ls`" + ` etc. - -Start the mount like this - - rclone ` + commandName + ` remote:path/to/files /path/to/local/mount - -Or on Windows like this where X: is an unused drive letter - - rclone ` + commandName + ` remote:path/to/files X: - -When the program ends, either via Ctrl+C or receiving a SIGINT or SIGTERM signal, -the mount is automatically stopped. - -The umount operation can fail, for example when the mountpoint is busy. -When that happens, it is the user's responsibility to stop the mount manually with - - # Linux - fusermount -u /path/to/local/mount - # OS X - umount /path/to/local/mount - -### Limitations ### - -This can only write files seqentially, it can only seek when reading. -This means that many applications won't work with their files on an -rclone mount. - -The bucket based remotes (eg Swift, S3, Google Compute Storage, B2, -Hubic) won't work from the root - you will need to specify a bucket, -or a path within the bucket. So ` + "`swift:`" + ` won't work whereas -` + "`swift:bucket`" + ` will as will ` + "`swift:bucket/path`" + `. -None of these support the concept of directories, so empty -directories will have a tendency to disappear once they fall out of -the directory cache. - -Only supported on Linux, FreeBSD, OS X and Windows at the moment. - -### rclone ` + commandName + ` vs rclone sync/copy ## - -File systems expect things to be 100% reliable, whereas cloud storage -systems are a long way from 100% reliable. The rclone sync/copy -commands cope with this with lots of retries. However rclone ` + commandName + ` -can't use retries in the same way without making local copies of the -uploads. This might happen in the future, but for the moment rclone -` + commandName + ` won't do that, so will be less reliable than the rclone command. - -### Filters ### - -Note that all the rclone filters can be used to select a subset of the -files to be visible in the mount. - -### Directory Cache ### - -Using the ` + "`--dir-cache-time`" + ` flag, you can set how long a -directory should be considered up to date and not refreshed from the -backend. Changes made locally in the mount may appear immediately or -invalidate the cache. However, changes done on the remote will only -be picked up once the cache expires. - -Alternatively, you can send a ` + "`SIGHUP`" + ` signal to rclone for -it to flush all directory caches, regardless of how old they are. -Assuming only one rclone instance is running, you can reset the cache -like this: - - kill -SIGHUP $(pidof rclone) - -### Bugs ### - - * All the remotes should work for read, but some may not for write - * those which need to know the size in advance won't - eg B2 - * maybe should pass in size as -1 to mean work it out - * Or put in an an upload cache to cache the files on disk first -`, - Run: func(command *cobra.Command, args []string) { - cmd.CheckArgs(2, 2, command, args) - fdst := cmd.NewFsDst(args) - err := Mount(fdst, args[1]) - if err != nil { - log.Fatalf("Fatal error: %v", err) - } - }, + name := "cmount" + if runtime.GOOS == "windows" { + name = "mount" + } + mountlib.NewMountCommand(name, Mount) } // mountOptions configures the options from the command line flags @@ -173,9 +36,9 @@ func mountOptions(device string, mountpoint string) (options []string) { options = []string{ "-o", "fsname=" + device, "-o", "subtype=rclone", - "-o", fmt.Sprintf("max_readahead=%d", maxReadAhead), + "-o", fmt.Sprintf("max_readahead=%d", mountlib.MaxReadAhead), } - if debugFUSE { + if mountlib.DebugFUSE { options = append(options, "-o", "debug") } @@ -194,28 +57,28 @@ func mountOptions(device string, mountpoint string) (options []string) { options = append(options, "--FileSystemName=rclone") } - if allowNonEmpty { + if mountlib.AllowNonEmpty { options = append(options, "-o", "nonempty") } - if allowOther { + if mountlib.AllowOther { options = append(options, "-o", "allow_other") } - if allowRoot { + if mountlib.AllowRoot { options = append(options, "-o", "allow_root") } - if defaultPermissions { + if mountlib.DefaultPermissions { options = append(options, "-o", "default_permissions") } - if readOnly { + if mountlib.ReadOnly { options = append(options, "-o", "ro") } - if writebackCache { + if mountlib.WritebackCache { // FIXME? options = append(options, "-o", WritebackCache()) } - for _, option := range *extraOptions { + for _, option := range *mountlib.ExtraOptions { options = append(options, "-o", option) } - for _, option := range *extraFlags { + for _, option := range *mountlib.ExtraFlags { options = append(options, option) } return options @@ -304,16 +167,6 @@ func mount(f fs.Fs, mountpoint string) (*mountlib.FS, <-chan error, func() error // // If noModTime is set then it func Mount(f fs.Fs, mountpoint string) error { - // Set permissions - dirPerms = 0777 &^ os.FileMode(umask) - filePerms = 0666 &^ os.FileMode(umask) - - // Show stats if the user has specifically requested them - if cmd.ShowStats() { - stopStats := cmd.StartStats() - defer close(stopStats) - } - // Mount it FS, errChan, _, err := mount(f, mountpoint) if err != nil { diff --git a/cmd/cmount/mount_test.go b/cmd/cmount/mount_test.go index f8d81f062..7cf18971e 100644 --- a/cmd/cmount/mount_test.go +++ b/cmd/cmount/mount_test.go @@ -17,7 +17,7 @@ func notWin(t *testing.T) { } } -func TestMain(m *testing.M) { mounttest.TestMain(m, mount, dirPerms, filePerms) } +func TestMain(m *testing.M) { mounttest.TestMain(m, mount) } func TestDirLs(t *testing.T) { mounttest.TestDirLs(t) } func TestDirCreateAndRemoveDir(t *testing.T) { notWin(t); mounttest.TestDirCreateAndRemoveDir(t) } func TestDirCreateAndRemoveFile(t *testing.T) { notWin(t); mounttest.TestDirCreateAndRemoveFile(t) } diff --git a/cmd/cmount/mount_unix.go b/cmd/cmount/mount_unix.go deleted file mode 100644 index d9e288b21..000000000 --- a/cmd/cmount/mount_unix.go +++ /dev/null @@ -1,18 +0,0 @@ -// +build cmount -// +build cgo -// +build linux darwin freebsd - -package cmount - -import "golang.org/x/sys/unix" - -const commandName = "cmount" - -func init() { - umask = unix.Umask(0) // read the umask - unix.Umask(umask) // set it back to what it was - uid = uint32(unix.Geteuid()) - gid = uint32(unix.Getegid()) - commandDefintion.Flags().Uint32VarP(&uid, "uid", "", uid, "Override the uid field set by the filesystem.") - commandDefintion.Flags().Uint32VarP(&gid, "gid", "", gid, "Override the gid field set by the filesystem.") -} diff --git a/cmd/cmount/mount_windows.go b/cmd/cmount/mount_windows.go deleted file mode 100644 index a68f7e40a..000000000 --- a/cmd/cmount/mount_windows.go +++ /dev/null @@ -1,7 +0,0 @@ -// +build cmount -// +build cgo -// +build windows - -package cmount - -const commandName = "mount" diff --git a/cmd/mount/dir.go b/cmd/mount/dir.go index 83e3ca17e..fed300ca5 100644 --- a/cmd/mount/dir.go +++ b/cmd/mount/dir.go @@ -42,9 +42,9 @@ var _ fusefs.Node = (*Dir)(nil) // Attr updates the attributes of a directory func (d *Dir) Attr(ctx context.Context, a *fuse.Attr) (err error) { defer fs.Trace(d, "")("attr=%+v, err=%v", a, &err) - a.Gid = gid - a.Uid = uid - a.Mode = os.ModeDir | dirPerms + a.Gid = mountlib.GID + a.Uid = mountlib.UID + a.Mode = os.ModeDir | mountlib.DirPerms modTime := d.ModTime() a.Atime = modTime a.Mtime = modTime @@ -61,7 +61,7 @@ var _ fusefs.NodeSetattrer = (*Dir)(nil) // Setattr handles attribute changes from FUSE. Currently supports ModTime only. func (d *Dir) Setattr(ctx context.Context, req *fuse.SetattrRequest, resp *fuse.SetattrResponse) (err error) { defer fs.Trace(d, "stat=%+v", req)("err=%v", &err) - if noModTime { + if mountlib.NoModTime { return nil } diff --git a/cmd/mount/file.go b/cmd/mount/file.go index 33b7225ca..f185dd1cc 100644 --- a/cmd/mount/file.go +++ b/cmd/mount/file.go @@ -30,13 +30,13 @@ var _ fusefs.Node = (*File)(nil) // Attr fills out the attributes for the file func (f *File) Attr(ctx context.Context, a *fuse.Attr) (err error) { defer fs.Trace(f, "")("a=%+v, err=%v", a, &err) - modTime, Size, Blocks, err := f.File.Attr(noModTime) + modTime, Size, Blocks, err := f.File.Attr(mountlib.NoModTime) if err != nil { return translateError(err) } - a.Gid = gid - a.Uid = uid - a.Mode = filePerms + a.Gid = mountlib.GID + a.Uid = mountlib.UID + a.Mode = mountlib.FilePerms a.Size = Size a.Atime = modTime a.Mtime = modTime @@ -52,7 +52,7 @@ var _ fusefs.NodeSetattrer = (*File)(nil) // Setattr handles attribute changes from FUSE. Currently supports ModTime only. func (f *File) Setattr(ctx context.Context, req *fuse.SetattrRequest, resp *fuse.SetattrResponse) (err error) { defer fs.Trace(f, "a=%+v", req)("err=%v", &err) - if noModTime { + if mountlib.NoModTime { return nil } if req.Valid.MtimeNow() { @@ -71,7 +71,7 @@ func (f *File) Open(ctx context.Context, req *fuse.OpenRequest, resp *fuse.OpenR defer fs.Trace(f, "flags=%v", req.Flags)("fh=%v, err=%v", &fh, &err) switch { case req.Flags.IsReadOnly(): - if noSeek { + if mountlib.NoSeek { resp.Flags |= fuse.OpenNonSeekable } var rfh *mountlib.ReadFileHandle diff --git a/cmd/mount/fs.go b/cmd/mount/fs.go index 385bc1fa1..b50813590 100644 --- a/cmd/mount/fs.go +++ b/cmd/mount/fs.go @@ -30,19 +30,6 @@ func NewFS(f fs.Fs) *FS { FS: mountlib.NewFS(f), f: f, } - if noSeek { - fsys.FS.NoSeek() - } - if noChecksum { - fsys.FS.NoChecksum() - } - if readOnly { - fsys.FS.ReadOnly() - } - if pollInterval > 0 { - fsys.FS.PollChanges(pollInterval) - } - fsys.FS.SetDirCacheTime(dirCacheTime) return fsys } diff --git a/cmd/mount/mount.go b/cmd/mount/mount.go index eb67e8e18..c7918c093 100644 --- a/cmd/mount/mount.go +++ b/cmd/mount/mount.go @@ -5,163 +5,25 @@ package mount import ( - "log" "os" "os/signal" "syscall" - "time" "bazil.org/fuse" fusefs "bazil.org/fuse/fs" - "github.com/ncw/rclone/cmd" "github.com/ncw/rclone/cmd/mountlib" "github.com/ncw/rclone/fs" "github.com/pkg/errors" - "github.com/spf13/cobra" - "golang.org/x/sys/unix" -) - -// Globals -var ( - noModTime = false - noChecksum = false - debugFUSE = false - noSeek = false - dirCacheTime = 5 * 60 * time.Second - pollInterval = time.Minute - // mount options - readOnly = false - allowNonEmpty = false - allowRoot = false - allowOther = false - defaultPermissions = false - writebackCache = false - maxReadAhead fs.SizeSuffix = 128 * 1024 - umask = 0 - uid = uint32(unix.Geteuid()) - gid = uint32(unix.Getegid()) - // foreground = false - // default permissions for directories - modified by umask in Mount - dirPerms = os.FileMode(0777) - filePerms = os.FileMode(0666) ) func init() { - umask = unix.Umask(0) // read the umask - unix.Umask(umask) // set it back to what it was - cmd.Root.AddCommand(commandDefintion) - commandDefintion.Flags().BoolVarP(&noModTime, "no-modtime", "", noModTime, "Don't read/write the modification time (can speed things up).") - commandDefintion.Flags().BoolVarP(&noChecksum, "no-checksum", "", noChecksum, "Don't compare checksums on up/download.") - commandDefintion.Flags().BoolVarP(&debugFUSE, "debug-fuse", "", debugFUSE, "Debug the FUSE internals - needs -v.") - commandDefintion.Flags().BoolVarP(&noSeek, "no-seek", "", noSeek, "Don't allow seeking in files.") - commandDefintion.Flags().DurationVarP(&dirCacheTime, "dir-cache-time", "", dirCacheTime, "Time to cache directory entries for.") - commandDefintion.Flags().DurationVarP(&pollInterval, "poll-interval", "", pollInterval, "Time to wait between polling for changes. Must be smaller than dir-cache-time. Only on supported remotes. Set to 0 to disable.") - // mount options - commandDefintion.Flags().BoolVarP(&readOnly, "read-only", "", readOnly, "Mount read-only.") - commandDefintion.Flags().BoolVarP(&allowNonEmpty, "allow-non-empty", "", allowNonEmpty, "Allow mounting over a non-empty directory.") - commandDefintion.Flags().BoolVarP(&allowRoot, "allow-root", "", allowRoot, "Allow access to root user.") - commandDefintion.Flags().BoolVarP(&allowOther, "allow-other", "", allowOther, "Allow access to other users.") - commandDefintion.Flags().BoolVarP(&defaultPermissions, "default-permissions", "", defaultPermissions, "Makes kernel enforce access control based on the file mode.") - commandDefintion.Flags().BoolVarP(&writebackCache, "write-back-cache", "", writebackCache, "Makes kernel buffer writes before sending them to rclone. Without this, writethrough caching is used.") - commandDefintion.Flags().VarP(&maxReadAhead, "max-read-ahead", "", "The number of bytes that can be prefetched for sequential reads.") - commandDefintion.Flags().IntVarP(&umask, "umask", "", umask, "Override the permission bits set by the filesystem.") - commandDefintion.Flags().Uint32VarP(&uid, "uid", "", uid, "Override the uid field set by the filesystem.") - commandDefintion.Flags().Uint32VarP(&gid, "gid", "", gid, "Override the gid field set by the filesystem.") - //commandDefintion.Flags().BoolVarP(&foreground, "foreground", "", foreground, "Do not detach.") -} - -var commandDefintion = &cobra.Command{ - Use: "mount remote:path /path/to/mountpoint", - Short: `Mount the remote as a mountpoint. **EXPERIMENTAL**`, - Long: ` -rclone mount allows Linux, FreeBSD and macOS to mount any of Rclone's -cloud storage systems as a file system with FUSE. - -This is **EXPERIMENTAL** - use with care. - -First set up your remote using ` + "`rclone config`" + `. Check it works with ` + "`rclone ls`" + ` etc. - -Start the mount like this - - rclone mount remote:path/to/files /path/to/local/mount - -When the program ends, either via Ctrl+C or receiving a SIGINT or SIGTERM signal, -the mount is automatically stopped. - -The umount operation can fail, for example when the mountpoint is busy. -When that happens, it is the user's responsibility to stop the mount manually with - - # Linux - fusermount -u /path/to/local/mount - # OS X - umount /path/to/local/mount - -### Limitations ### - -This can only write files seqentially, it can only seek when reading. -This means that many applications won't work with their files on an -rclone mount. - -The bucket based remotes (eg Swift, S3, Google Compute Storage, B2, -Hubic) won't work from the root - you will need to specify a bucket, -or a path within the bucket. So ` + "`swift:`" + ` won't work whereas -` + "`swift:bucket`" + ` will as will ` + "`swift:bucket/path`" + `. -None of these support the concept of directories, so empty -directories will have a tendency to disappear once they fall out of -the directory cache. - -Only supported on Linux, FreeBSD and OS X at the moment. - -### rclone mount vs rclone sync/copy ## - -File systems expect things to be 100% reliable, whereas cloud storage -systems are a long way from 100% reliable. The rclone sync/copy -commands cope with this with lots of retries. However rclone mount -can't use retries in the same way without making local copies of the -uploads. This might happen in the future, but for the moment rclone -mount won't do that, so will be less reliable than the rclone command. - -### Filters ### - -Note that all the rclone filters can be used to select a subset of the -files to be visible in the mount. - -### Directory Cache ### - -Using the ` + "`--dir-cache-time`" + ` flag, you can set how long a -directory should be considered up to date and not refreshed from the -backend. Changes made locally in the mount may appear immediately or -invalidate the cache. However, changes done on the remote will only -be picked up once the cache expires. - -Alternatively, you can send a ` + "`SIGHUP`" + ` signal to rclone for -it to flush all directory caches, regardless of how old they are. -Assuming only one rclone instance is running, you can reset the cache -like this: - - kill -SIGHUP $(pidof rclone) - -### Bugs ### - - * All the remotes should work for read, but some may not for write - * those which need to know the size in advance won't - eg B2 - * maybe should pass in size as -1 to mean work it out - * Or put in an an upload cache to cache the files on disk first -`, - Run: func(command *cobra.Command, args []string) { - cmd.CheckArgs(2, 2, command, args) - fdst := cmd.NewFsDst(args) - err := Mount(fdst, args[1]) - if err != nil { - log.Fatalf("Fatal error: %v", err) - } - }, + mountlib.NewMountCommand("mount", Mount) } // mountOptions configures the options from the command line flags func mountOptions(device string) (options []fuse.MountOption) { options = []fuse.MountOption{ - fuse.MaxReadahead(uint32(maxReadAhead)), + fuse.MaxReadahead(uint32(mountlib.MaxReadAhead)), fuse.Subtype("rclone"), fuse.FSName(device), fuse.VolumeName(device), fuse.NoAppleDouble(), @@ -174,24 +36,30 @@ func mountOptions(device string) (options []fuse.MountOption) { // which is probably related to errors people are having //fuse.WritebackCache(), } - if allowNonEmpty { + if mountlib.AllowNonEmpty { options = append(options, fuse.AllowNonEmptyMount()) } - if allowOther { + if mountlib.AllowOther { options = append(options, fuse.AllowOther()) } - if allowRoot { + if mountlib.AllowRoot { options = append(options, fuse.AllowRoot()) } - if defaultPermissions { + if mountlib.DefaultPermissions { options = append(options, fuse.DefaultPermissions()) } - if readOnly { + if mountlib.ReadOnly { options = append(options, fuse.ReadOnly()) } - if writebackCache { + if mountlib.WritebackCache { options = append(options, fuse.WritebackCache()) } + if len(*mountlib.ExtraOptions) > 0 { + fs.Errorf(nil, "-o/--option not supported with this FUSE backend") + } + if len(*mountlib.ExtraOptions) > 0 { + fs.Errorf(nil, "--fuse-flag not supported with this FUSE backend") + } return options } @@ -239,22 +107,12 @@ func mount(f fs.Fs, mountpoint string) (*mountlib.FS, <-chan error, func() error // // If noModTime is set then it func Mount(f fs.Fs, mountpoint string) error { - if debugFUSE { + if mountlib.DebugFUSE { fuse.Debug = func(msg interface{}) { fs.Debugf("fuse", "%v", msg) } } - // Set permissions - dirPerms = 0777 &^ os.FileMode(umask) - filePerms = 0666 &^ os.FileMode(umask) - - // Show stats if the user has specifically requested them - if cmd.ShowStats() { - stopStats := cmd.StartStats() - defer close(stopStats) - } - // Mount it FS, errChan, unmount, err := mount(f, mountpoint) if err != nil { diff --git a/cmd/mount/mount_test.go b/cmd/mount/mount_test.go index cdc806b28..a2d56efb3 100644 --- a/cmd/mount/mount_test.go +++ b/cmd/mount/mount_test.go @@ -8,7 +8,7 @@ import ( "github.com/ncw/rclone/cmd/mountlib/mounttest" ) -func TestMain(m *testing.M) { mounttest.TestMain(m, mount, dirPerms, filePerms) } +func TestMain(m *testing.M) { mounttest.TestMain(m, mount) } func TestDirLs(t *testing.T) { mounttest.TestDirLs(t) } func TestDirCreateAndRemoveDir(t *testing.T) { mounttest.TestDirCreateAndRemoveDir(t) } func TestDirCreateAndRemoveFile(t *testing.T) { mounttest.TestDirCreateAndRemoveFile(t) } diff --git a/cmd/mountlib/fs.go b/cmd/mountlib/fs.go index 3d73b671c..d79b011ad 100644 --- a/cmd/mountlib/fs.go +++ b/cmd/mountlib/fs.go @@ -53,15 +53,22 @@ func NewFS(f fs.Fs) *FS { f: f, } + if NoSeek { + fsys.noSeek = true + } + if NoChecksum { + fsys.noChecksum = true + } + if ReadOnly { + fsys.readOnly = true + } + fsys.dirCacheTime = DirCacheTime + fsys.root = newDir(fsys, f, fsDir) - return fsys -} - -// SetDirCacheTime allows to set how long a directory listing is considered -// valid. Set to 0 always request a fresh version from the remote. -func (fsys *FS) SetDirCacheTime(dirCacheTime time.Duration) *FS { - fsys.dirCacheTime = dirCacheTime + if PollInterval > 0 { + fsys.PollChanges(PollInterval) + } return fsys } @@ -76,25 +83,6 @@ func (fsys *FS) PollChanges(pollInterval time.Duration) *FS { return fsys } -// NoSeek disables seeking of files -func (fsys *FS) NoSeek() *FS { - fsys.noSeek = true - return fsys -} - -// NoChecksum disables checksum checking -func (fsys *FS) NoChecksum() *FS { - fsys.noChecksum = true - return fsys -} - -// ReadOnly sets the fs into read only mode, returning EROFS for any -// write operations. -func (fsys *FS) ReadOnly() *FS { - fsys.readOnly = true - return fsys -} - // Root returns the root node func (fsys *FS) Root() (*Dir, error) { // fs.Debugf(fsys.f, "Root()") diff --git a/cmd/mountlib/mount.go b/cmd/mountlib/mount.go new file mode 100644 index 000000000..00cda3acf --- /dev/null +++ b/cmd/mountlib/mount.go @@ -0,0 +1,172 @@ +package mountlib + +// Globals +import ( + "log" + "os" + "time" + + "github.com/ncw/rclone/cmd" + "github.com/ncw/rclone/fs" + "github.com/spf13/cobra" +) + +// Options set by command line flags +var ( + NoModTime = false + NoChecksum = false + DebugFUSE = false + NoSeek = false + DirCacheTime = 5 * 60 * time.Second + PollInterval = time.Minute + // mount options + ReadOnly = false + AllowNonEmpty = false + AllowRoot = false + AllowOther = false + DefaultPermissions = false + WritebackCache = false + MaxReadAhead fs.SizeSuffix = 128 * 1024 + Umask = 0 + UID = ^uint32(0) // these values instruct WinFSP-FUSE to use the current user + GID = ^uint32(0) // overriden for non windows in mount_unix.go + // foreground = false + // default permissions for directories - modified by umask in Mount + DirPerms = os.FileMode(0777) + FilePerms = os.FileMode(0666) + ExtraOptions *[]string + ExtraFlags *[]string +) + +// NewMountCommand makes a mount command with the given name and Mount function +func NewMountCommand(commandName string, Mount func(f fs.Fs, mountpoint string) error) *cobra.Command { + var commandDefintion = &cobra.Command{ + Use: commandName + " remote:path /path/to/mountpoint", + Short: `Mount the remote as a mountpoint. **EXPERIMENTAL**`, + Long: ` +rclone ` + commandName + ` allows Linux, FreeBSD, macOS and Windows to +mount any of Rclone's cloud storage systems as a file system with +FUSE. + +This is **EXPERIMENTAL** - use with care. + +First set up your remote using ` + "`rclone config`" + `. Check it works with ` + "`rclone ls`" + ` etc. + +Start the mount like this + + rclone ` + commandName + ` remote:path/to/files /path/to/local/mount + +Or on Windows like this where X: is an unused drive letter + + rclone ` + commandName + ` remote:path/to/files X: + +When the program ends, either via Ctrl+C or receiving a SIGINT or SIGTERM signal, +the mount is automatically stopped. + +The umount operation can fail, for example when the mountpoint is busy. +When that happens, it is the user's responsibility to stop the mount manually with + + # Linux + fusermount -u /path/to/local/mount + # OS X + umount /path/to/local/mount + +### Limitations ### + +This can only write files seqentially, it can only seek when reading. +This means that many applications won't work with their files on an +rclone mount. + +The bucket based remotes (eg Swift, S3, Google Compute Storage, B2, +Hubic) won't work from the root - you will need to specify a bucket, +or a path within the bucket. So ` + "`swift:`" + ` won't work whereas +` + "`swift:bucket`" + ` will as will ` + "`swift:bucket/path`" + `. +None of these support the concept of directories, so empty +directories will have a tendency to disappear once they fall out of +the directory cache. + +Only supported on Linux, FreeBSD, OS X and Windows at the moment. + +### rclone ` + commandName + ` vs rclone sync/copy ## + +File systems expect things to be 100% reliable, whereas cloud storage +systems are a long way from 100% reliable. The rclone sync/copy +commands cope with this with lots of retries. However rclone ` + commandName + ` +can't use retries in the same way without making local copies of the +uploads. This might happen in the future, but for the moment rclone +` + commandName + ` won't do that, so will be less reliable than the rclone command. + +### Filters ### + +Note that all the rclone filters can be used to select a subset of the +files to be visible in the mount. + +### Directory Cache ### + +Using the ` + "`--dir-cache-time`" + ` flag, you can set how long a +directory should be considered up to date and not refreshed from the +backend. Changes made locally in the mount may appear immediately or +invalidate the cache. However, changes done on the remote will only +be picked up once the cache expires. + +Alternatively, you can send a ` + "`SIGHUP`" + ` signal to rclone for +it to flush all directory caches, regardless of how old they are. +Assuming only one rclone instance is running, you can reset the cache +like this: + + kill -SIGHUP $(pidof rclone) + +### Bugs ### + + * All the remotes should work for read, but some may not for write + * those which need to know the size in advance won't - eg B2 + * maybe should pass in size as -1 to mean work it out + * Or put in an an upload cache to cache the files on disk first +`, + Run: func(command *cobra.Command, args []string) { + cmd.CheckArgs(2, 2, command, args) + fdst := cmd.NewFsDst(args) + + // Mask permissions + DirPerms = 0777 &^ os.FileMode(Umask) + FilePerms = 0666 &^ os.FileMode(Umask) + + // Show stats if the user has specifically requested them + if cmd.ShowStats() { + stopStats := cmd.StartStats() + defer close(stopStats) + } + + err := Mount(fdst, args[1]) + if err != nil { + log.Fatalf("Fatal error: %v", err) + } + }, + } + + // Register the command + cmd.Root.AddCommand(commandDefintion) + + // Add flags + flags := commandDefintion.Flags() + flags.BoolVarP(&NoModTime, "no-modtime", "", NoModTime, "Don't read/write the modification time (can speed things up).") + flags.BoolVarP(&NoChecksum, "no-checksum", "", NoChecksum, "Don't compare checksums on up/download.") + flags.BoolVarP(&DebugFUSE, "debug-fuse", "", DebugFUSE, "Debug the FUSE internals - needs -v.") + flags.BoolVarP(&NoSeek, "no-seek", "", NoSeek, "Don't allow seeking in files.") + flags.DurationVarP(&DirCacheTime, "dir-cache-time", "", DirCacheTime, "Time to cache directory entries for.") + flags.DurationVarP(&PollInterval, "poll-interval", "", PollInterval, "Time to wait between polling for changes. Must be smaller than dir-cache-time. Only on supported remotes. Set to 0 to disable.") + // mount options + flags.BoolVarP(&ReadOnly, "read-only", "", ReadOnly, "Mount read-only.") + flags.BoolVarP(&AllowNonEmpty, "allow-non-empty", "", AllowNonEmpty, "Allow mounting over a non-empty directory.") + flags.BoolVarP(&AllowRoot, "allow-root", "", AllowRoot, "Allow access to root user.") + flags.BoolVarP(&AllowOther, "allow-other", "", AllowOther, "Allow access to other users.") + flags.BoolVarP(&DefaultPermissions, "default-permissions", "", DefaultPermissions, "Makes kernel enforce access control based on the file mode.") + flags.BoolVarP(&WritebackCache, "write-back-cache", "", WritebackCache, "Makes kernel buffer writes before sending them to rclone. Without this, writethrough caching is used.") + flags.VarP(&MaxReadAhead, "max-read-ahead", "", "The number of bytes that can be prefetched for sequential reads.") + ExtraOptions = flags.StringArrayP("option", "o", []string{}, "Option for libfuse/WinFsp. Repeat if required.") + ExtraFlags = flags.StringArrayP("fuse-flag", "", []string{}, "Flags or arguments to be passed direct to libfuse/WinFsp. Repeat if required.") + //flags.BoolVarP(&foreground, "foreground", "", foreground, "Do not detach.") + + platformFlags(flags) + return commandDefintion +} diff --git a/cmd/mountlib/mount_non_unix.go b/cmd/mountlib/mount_non_unix.go new file mode 100644 index 000000000..846b2b34f --- /dev/null +++ b/cmd/mountlib/mount_non_unix.go @@ -0,0 +1,11 @@ +// +build !linux,!darwin,!freebsd + +package mountlib + +import ( + "github.com/spf13/pflag" +) + +// add any extra platform specific flags +func platformFlags(flags *pflag.FlagSet) { +} diff --git a/cmd/mountlib/mount_unix.go b/cmd/mountlib/mount_unix.go new file mode 100644 index 000000000..647a26a5e --- /dev/null +++ b/cmd/mountlib/mount_unix.go @@ -0,0 +1,19 @@ +// +build linux darwin freebsd + +package mountlib + +import ( + "github.com/spf13/pflag" + "golang.org/x/sys/unix" +) + +// add any extra platform specific flags +func platformFlags(flags *pflag.FlagSet) { + flags.IntVarP(&Umask, "umask", "", Umask, "Override the permission bits set by the filesystem.") + Umask = unix.Umask(0) // read the umask + unix.Umask(Umask) // set it back to what it was + UID = uint32(unix.Geteuid()) + GID = uint32(unix.Getegid()) + flags.Uint32VarP(&UID, "uid", "", UID, "Override the uid field set by the filesystem.") + flags.Uint32VarP(&GID, "gid", "", GID, "Override the gid field set by the filesystem.") +} diff --git a/cmd/mountlib/mounttest/fs.go b/cmd/mountlib/mounttest/fs.go index 44d693ca4..4739badb2 100644 --- a/cmd/mountlib/mounttest/fs.go +++ b/cmd/mountlib/mounttest/fs.go @@ -46,12 +46,10 @@ var ( ) // TestMain drives the tests -func TestMain(m *testing.M, fn MountFn, dirPerms, filePerms os.FileMode) { +func TestMain(m *testing.M, fn MountFn) { mountFn = fn flag.Parse() run = newRun() - run.dirPerms = dirPerms - run.filePerms = filePerms rc := m.Run() run.Finalise() os.Exit(rc) @@ -59,15 +57,14 @@ func TestMain(m *testing.M, fn MountFn, dirPerms, filePerms os.FileMode) { // Run holds the remotes for a test run type Run struct { - filesys *mountlib.FS - mountPath string - fremote fs.Fs - fremoteName string - cleanRemote func() - umountResult <-chan error - umountFn UnmountFn - skip bool - dirPerms, filePerms os.FileMode + filesys *mountlib.FS + mountPath string + fremote fs.Fs + fremoteName string + cleanRemote func() + umountResult <-chan error + umountFn UnmountFn + skip bool } // run holds the master Run data @@ -230,10 +227,10 @@ func (r *Run) readLocal(t *testing.T, dir dirMap, filepath string) { if fi.IsDir() { dir[name+"/"] = struct{}{} r.readLocal(t, dir, name) - assert.Equal(t, r.dirPerms, fi.Mode().Perm()) + assert.Equal(t, mountlib.DirPerms, fi.Mode().Perm()) } else { dir[fmt.Sprintf("%s %d", name, fi.Size())] = struct{}{} - assert.Equal(t, r.filePerms, fi.Mode().Perm()) + assert.Equal(t, mountlib.FilePerms, fi.Mode().Perm()) } } } @@ -315,5 +312,5 @@ func TestRoot(t *testing.T) { fi, err := os.Lstat(run.mountPath) require.NoError(t, err) assert.True(t, fi.IsDir()) - assert.Equal(t, fi.Mode().Perm(), run.dirPerms) + assert.Equal(t, fi.Mode().Perm(), mountlib.DirPerms) }