From 831d1df67f1e712d18fa8c1103d117754bd26bf5 Mon Sep 17 00:00:00 2001 From: Nick Craig-Wood Date: Thu, 5 Oct 2023 15:32:50 +0100 Subject: [PATCH] docs: factor large docs into separate .md files to make them easier to maintain. We then use the go embed command to embed them back into the binary. --- cmd/mountlib/mount.go | 6 +- cmd/mountlib/{help.go => mount.md} | 155 +++++++++++----------- cmd/selfupdate/selfupdate.go | 6 +- cmd/selfupdate/{help.go => selfupdate.md} | 38 +++--- cmd/serve/docker/docker.go | 7 +- cmd/serve/docker/{help.go => docker.md} | 29 ++-- vfs/vfs.go | 6 + vfs/{help.go => vfs.md} | 95 ++++++------- 8 files changed, 164 insertions(+), 178 deletions(-) rename cmd/mountlib/{help.go => mount.md} (81%) rename cmd/selfupdate/{help.go => selfupdate.md} (59%) rename cmd/serve/docker/{help.go => docker.md} (67%) rename vfs/{help.go => vfs.md} (80%) diff --git a/cmd/mountlib/mount.go b/cmd/mountlib/mount.go index 82700b92e..4bc9e066a 100644 --- a/cmd/mountlib/mount.go +++ b/cmd/mountlib/mount.go @@ -3,6 +3,7 @@ package mountlib import ( "context" + _ "embed" "fmt" "log" "os" @@ -27,6 +28,9 @@ import ( "github.com/spf13/pflag" ) +//go:embed mount.md +var mountHelp string + // Options for creating the mount type Options struct { DebugFUSE bool @@ -158,7 +162,7 @@ func NewMountCommand(commandName string, hidden bool, mount MountFn) *cobra.Comm Use: commandName + " remote:path /path/to/mountpoint", Hidden: hidden, Short: `Mount the remote as file system on a mountpoint.`, - Long: strings.ReplaceAll(strings.ReplaceAll(mountHelp, "|", "`"), "@", commandName) + vfs.Help, + Long: strings.ReplaceAll(mountHelp, "@", commandName) + vfs.Help, Annotations: map[string]string{ "versionIntroduced": "v1.33", "groups": "Filter", diff --git a/cmd/mountlib/help.go b/cmd/mountlib/mount.md similarity index 81% rename from cmd/mountlib/help.go rename to cmd/mountlib/mount.md index 90def6673..9d367ca1e 100644 --- a/cmd/mountlib/help.go +++ b/cmd/mountlib/mount.md @@ -1,15 +1,11 @@ -package mountlib - -// "@" will be replaced by the command name, "|" will be replaced by backticks -var mountHelp = ` rclone @ allows Linux, FreeBSD, macOS and Windows to mount any of Rclone's cloud storage systems as a file system with FUSE. -First set up your remote using |rclone config|. Check it works with |rclone ls| etc. +First set up your remote using `rclone config`. Check it works with `rclone ls` etc. On Linux and macOS, you can run mount in either foreground or background (aka -daemon) mode. Mount runs in foreground mode by default. Use the |--daemon| flag +daemon) mode. Mount runs in foreground mode by default. Use the `--daemon` flag to force background mode. On Windows you can run mount in foreground only, the flag is ignored. @@ -18,7 +14,7 @@ program starts, spawns background rclone process to setup and maintain the mount, waits until success or timeout and exits with appropriate code (killing the child process if it fails). -On Linux/macOS/FreeBSD start the mount like this, where |/path/to/local/mount| +On Linux/macOS/FreeBSD start the mount like this, where `/path/to/local/mount` is an **empty** **existing** directory: rclone @ remote:path/to/files /path/to/local/mount @@ -29,10 +25,10 @@ rclone will serve the mount and occupy the console so another window should be used to work with the mount until rclone is interrupted e.g. by pressing Ctrl-C. The following examples will mount to an automatically assigned drive, -to specific drive letter |X:|, to path |C:\path\parent\mount| +to specific drive letter `X:`, to path `C:\path\parent\mount` (where parent directory or drive must exist, and mount must **not** exist, and is not supported when [mounting as a network drive](#mounting-modes-on-windows)), and -the last example will mount as network share |\\cloud\remote| and map it to an +the last example will mount as network share `\\cloud\remote` and map it to an automatically assigned drive: rclone @ remote:path/to/files * @@ -89,7 +85,7 @@ as a network drive instead. When mounting as a fixed disk drive you can either mount to an unused drive letter, or to a path representing a **nonexistent** subdirectory of an **existing** parent -directory or drive. Using the special value |*| will tell rclone to +directory or drive. Using the special value `*` will tell rclone to automatically assign the next available drive letter, starting with Z: and moving backward. Examples: @@ -98,45 +94,45 @@ Examples: rclone @ remote:path/to/files C:\path\parent\mount rclone @ remote:path/to/files X: -Option |--volname| can be used to set a custom volume name for the mounted +Option `--volname` can be used to set a custom volume name for the mounted file system. The default is to use the remote name and path. -To mount as network drive, you can add option |--network-mode| +To mount as network drive, you can add option `--network-mode` to your @ command. Mounting to a directory path is not supported in this mode, it is a limitation Windows imposes on junctions, so the remote must always be mounted to a drive letter. rclone @ remote:path/to/files X: --network-mode -A volume name specified with |--volname| will be used to create the network share path. -A complete UNC path, such as |\\cloud\remote|, optionally with path -|\\cloud\remote\madeup\path|, will be used as is. Any other -string will be used as the share part, after a default prefix |\\server\|. -If no volume name is specified then |\\server\share| will be used. +A volume name specified with `--volname` will be used to create the network share path. +A complete UNC path, such as `\\cloud\remote`, optionally with path +`\\cloud\remote\madeup\path`, will be used as is. Any other +string will be used as the share part, after a default prefix `\\server\`. +If no volume name is specified then `\\server\share` will be used. You must make sure the volume name is unique when you are mounting more than one drive, or else the mount command will fail. The share name will treated as the volume label for the mapped drive, shown in Windows Explorer etc, while the complete -|\\server\share| will be reported as the remote UNC path by -|net use| etc, just like a normal network drive mapping. +`\\server\share` will be reported as the remote UNC path by +`net use` etc, just like a normal network drive mapping. -If you specify a full network share UNC path with |--volname|, this will implicitly -set the |--network-mode| option, so the following two examples have same result: +If you specify a full network share UNC path with `--volname`, this will implicitly +set the `--network-mode` option, so the following two examples have same result: rclone @ remote:path/to/files X: --network-mode rclone @ remote:path/to/files X: --volname \\server\share You may also specify the network share UNC path as the mountpoint itself. Then rclone -will automatically assign a drive letter, same as with |*| and use that as +will automatically assign a drive letter, same as with `*` and use that as mountpoint, and instead use the UNC path specified as the volume name, as if it were -specified with the |--volname| option. This will also implicitly set -the |--network-mode| option. This means the following two examples have same result: +specified with the `--volname` option. This will also implicitly set +the `--network-mode` option. This means the following two examples have same result: rclone @ remote:path/to/files \\cloud\remote rclone @ remote:path/to/files * --volname \\cloud\remote There is yet another way to enable network mode, and to set the share path, and that is to pass the "native" libfuse/WinFsp option directly: -|--fuse-flag --VolumePrefix=\server\share|. Note that the path +`--fuse-flag --VolumePrefix=\server\share`. Note that the path must be with just a single backslash prefix in this case. @@ -157,15 +153,15 @@ representing permissions for the POSIX permission scopes: Owner, group and other By default, the owner and group will be taken from the current user, and the built-in group "Everyone" will be used to represent others. The user/group can be customized with FUSE options "UserName" and "GroupName", -e.g. |-o UserName=user123 -o GroupName="Authenticated Users"|. +e.g. `-o UserName=user123 -o GroupName="Authenticated Users"`. The permissions on each entry will be set according to [options](#options) -|--dir-perms| and |--file-perms|, which takes a value in traditional Unix +`--dir-perms` and `--file-perms`, which takes a value in traditional Unix [numeric notation](https://en.wikipedia.org/wiki/File-system_permissions#Numeric_notation). -The default permissions corresponds to |--file-perms 0666 --dir-perms 0777|, +The default permissions corresponds to `--file-perms 0666 --dir-perms 0777`, i.e. read and write permissions to everyone. This means you will not be able to start any programs from the mount. To be able to do that you must add -execute permissions, e.g. |--file-perms 0777 --dir-perms 0777| to add it +execute permissions, e.g. `--file-perms 0777 --dir-perms 0777` to add it to everyone. If the program needs to write files, chances are you will have to enable [VFS File Caching](#vfs-file-caching) as well (see also [limitations](#limitations)). Note that the default write permission have @@ -193,12 +189,12 @@ will be added automatically for compatibility with Unix. Some example use cases will following. If you set POSIX permissions for only allowing access to the owner, -using |--file-perms 0600 --dir-perms 0700|, the user group and the built-in +using `--file-perms 0600 --dir-perms 0700`, the user group and the built-in "Everyone" group will still be given some special permissions, as described above. Some programs may then (incorrectly) interpret this as the file being accessible by everyone, for example an SSH client may warn about "unprotected private key file". You can work around this by specifying -|-o FileSecurity="D:P(A;;FA;;;OW)"|, which sets file all access (FA) to the +`-o FileSecurity="D:P(A;;FA;;;OW)"`, which sets file all access (FA) to the owner (OW), and nothing else. When setting write permissions then, except for the owner, this does not @@ -207,11 +203,11 @@ This may prevent applications from writing to files, giving permission denied error instead. To set working write permissions for the built-in "Everyone" group, similar to what it gets by default but with the addition of the "write extended attributes", you can specify -|-o FileSecurity="D:P(A;;FRFW;;;WD)"|, which sets file read (FR) and file +`-o FileSecurity="D:P(A;;FRFW;;;WD)"`, which sets file read (FR) and file write (FW) to everyone (WD). If file execute (FX) is also needed, then change -to |-o FileSecurity="D:P(A;;FRFWFX;;;WD)"|, or set file all access (FA) to +to `-o FileSecurity="D:P(A;;FRFWFX;;;WD)"`, or set file all access (FA) to get full access permissions, including delete, with -|-o FileSecurity="D:P(A;;FA;;;WD)"|. +`-o FileSecurity="D:P(A;;FA;;;WD)"`. #### Windows caveats @@ -235,7 +231,7 @@ It is also possible to make a drive mount available to everyone on the system, by running the process creating it as the built-in SYSTEM account. There are several ways to do this: One is to use the command-line utility [PsExec](https://docs.microsoft.com/en-us/sysinternals/downloads/psexec), -from Microsoft's Sysinternals suite, which has option |-s| to start +from Microsoft's Sysinternals suite, which has option `-s` to start processes as the SYSTEM account. Another alternative is to run the mount command from a Windows Scheduled Task, or a Windows Service, configured to run as the SYSTEM account. A third alternative is to use the @@ -243,7 +239,7 @@ to run as the SYSTEM account. A third alternative is to use the Read more in the [install documentation](https://rclone.org/install/). Note that when running rclone as another user, it will not use the configuration file from your profile unless you tell it to -with the [|--config|](https://rclone.org/docs/#config-config-file) option. +with the [`--config`](https://rclone.org/docs/#config-config-file) option. Note also that it is now the SYSTEM account that will have the owner permissions, and other accounts will have permissions according to the group or others scopes. As mentioned above, these will then not get the @@ -300,21 +296,21 @@ of the file. Rclone includes flags for unicode normalization with macFUSE that should be updated for FUSE-T. See [this forum post](https://forum.rclone.org/t/some-unicode-forms-break-mount-on-macos-with-fuse-t/36403) and [FUSE-T issue #16](https://github.com/macos-fuse-t/fuse-t/issues/16). The following -flag should be added to the |rclone mount| command. +flag should be added to the `rclone mount` command. -o modules=iconv,from_code=UTF-8,to_code=UTF-8 ##### Read Only mounts -When mounting with |--read-only|, attempts to write to files will fail *silently* as +When mounting with `--read-only`, attempts to write to files will fail *silently* as opposed to with a clear warning as in macFUSE. ### Limitations -Without the use of |--vfs-cache-mode| this can only write files +Without the use of `--vfs-cache-mode` this can only write files sequentially, it can only seek when reading. This means that many applications won't work with their files on an rclone mount without -|--vfs-cache-mode writes| or |--vfs-cache-mode full|. +`--vfs-cache-mode writes` or `--vfs-cache-mode full`. See the [VFS File Caching](#vfs-file-caching) section for more info. When using NFS mount on macOS, if you don't specify |--vfs-cache-mode| the mount point will be read-only. @@ -324,9 +320,9 @@ do not support the concept of empty directories, so empty directories will have a tendency to disappear once they fall out of the directory cache. -When |rclone mount| is invoked on Unix with |--daemon| flag, the main rclone +When `rclone mount` is invoked on Unix with `--daemon` flag, the main rclone program will wait for the background mount to become ready or until the timeout -specified by the |--daemon-wait| flag. On Linux it can check mount status using +specified by the `--daemon-wait` flag. On Linux it can check mount status using ProcFS so the flag in fact sets **maximum** time to wait, while the real wait can be less. On macOS / BSD the time to wait is constant and the check is performed only at the end. We advise you to set wait time on macOS reasonably. @@ -344,10 +340,10 @@ for solutions to make @ more reliable. ### Attribute caching -You can use the flag |--attr-timeout| to set the time the kernel caches +You can use the flag `--attr-timeout` to set the time the kernel caches the attributes (size, modification time, etc.) for directory entries. -The default is |1s| which caches files just long enough to avoid +The default is `1s` which caches files just long enough to avoid too many callbacks to rclone from the kernel. In theory 0s should be the correct value for filesystems which can @@ -358,14 +354,14 @@ few problems such as and [excessive time listing directories](https://github.com/rclone/rclone/issues/2095#issuecomment-371141147). The kernel can cache the info about a file for the time given by -|--attr-timeout|. You may see corruption if the remote file changes +`--attr-timeout`. You may see corruption if the remote file changes length during this window. It will show up as either a truncated file -or a file with garbage on the end. With |--attr-timeout 1s| this is -very unlikely but not impossible. The higher you set |--attr-timeout| +or a file with garbage on the end. With `--attr-timeout 1s` this is +very unlikely but not impossible. The higher you set `--attr-timeout` the more likely it is. The default setting of "1s" is the lowest setting which mitigates the problems above. -If you set it higher (|10s| or |1m| say) then the kernel will call +If you set it higher (`10s` or `1m` say) then the kernel will call back to rclone less often making it more efficient, however there is more chance of the corruption issue above. @@ -388,32 +384,32 @@ Units having the rclone @ service specified as a requirement will see all files and folders immediately in this mode. Note that systemd runs mount units without any environment variables including -|PATH| or |HOME|. This means that tilde (|~|) expansion will not work -and you should provide |--config| and |--cache-dir| explicitly as absolute +`PATH` or `HOME`. This means that tilde (`~`) expansion will not work +and you should provide `--config` and `--cache-dir` explicitly as absolute paths via rclone arguments. -Since mounting requires the |fusermount| program, rclone will use the fallback -PATH of |/bin:/usr/bin| in this scenario. Please ensure that |fusermount| +Since mounting requires the `fusermount` program, rclone will use the fallback +PATH of `/bin:/usr/bin` in this scenario. Please ensure that `fusermount` is present on this PATH. ### Rclone as Unix mount helper -The core Unix program |/bin/mount| normally takes the |-t FSTYPE| argument -then runs the |/sbin/mount.FSTYPE| helper program passing it mount options -as |-o key=val,...| or |--opt=...|. Automount (classic or systemd) behaves +The core Unix program `/bin/mount` normally takes the `-t FSTYPE` argument +then runs the `/sbin/mount.FSTYPE` helper program passing it mount options +as `-o key=val,...` or `--opt=...`. Automount (classic or systemd) behaves in a similar way. -rclone by default expects GNU-style flags |--key val|. To run it as a mount -helper you should symlink rclone binary to |/sbin/mount.rclone| and optionally -|/usr/bin/rclonefs|, e.g. |ln -s /usr/bin/rclone /sbin/mount.rclone|. +rclone by default expects GNU-style flags `--key val`. To run it as a mount +helper you should symlink rclone binary to `/sbin/mount.rclone` and optionally +`/usr/bin/rclonefs`, e.g. `ln -s /usr/bin/rclone /sbin/mount.rclone`. rclone will detect it and translate command-line arguments appropriately. Now you can run classic mounts like this: -||| +``` mount sftp1:subdir /mnt/data -t rclone -o vfs_cache_mode=writes,sftp_key_file=/path/to/pem -||| +``` or create systemd mount units: -||| +``` # /etc/systemd/system/mnt-data.mount [Unit] Description=Mount for /mnt/data @@ -422,10 +418,10 @@ Type=rclone What=sftp1:subdir Where=/mnt/data Options=rw,_netdev,allow_other,args2env,vfs-cache-mode=writes,config=/etc/rclone.conf,cache-dir=/var/rclone -||| +``` optionally accompanied by systemd automount unit -||| +``` # /etc/systemd/system/mnt-data.automount [Unit] Description=AutoMount for /mnt/data @@ -434,34 +430,33 @@ Where=/mnt/data TimeoutIdleSec=600 [Install] WantedBy=multi-user.target -||| +``` -or add in |/etc/fstab| a line like -||| +or add in `/etc/fstab` a line like +``` sftp1:subdir /mnt/data rclone rw,noauto,nofail,_netdev,x-systemd.automount,args2env,vfs_cache_mode=writes,config=/etc/rclone.conf,cache_dir=/var/cache/rclone 0 0 -||| +``` or use classic Automountd. -Remember to provide explicit |config=...,cache-dir=...| as a workaround for -mount units being run without |HOME|. +Remember to provide explicit `config=...,cache-dir=...` as a workaround for +mount units being run without `HOME`. -Rclone in the mount helper mode will split |-o| argument(s) by comma, replace |_| -by |-| and prepend |--| to get the command-line flags. Options containing commas +Rclone in the mount helper mode will split `-o` argument(s) by comma, replace `_` +by `-` and prepend `--` to get the command-line flags. Options containing commas or spaces can be wrapped in single or double quotes. Any inner quotes inside outer quotes of the same type should be doubled. Mount option syntax includes a few extra options treated specially: -- |env.NAME=VALUE| will set an environment variable for the mount process. +- `env.NAME=VALUE` will set an environment variable for the mount process. This helps with Automountd and Systemd.mount which don't allow setting custom environment for mount helpers. - Typically you will use |env.HTTPS_PROXY=proxy.host:3128| or |env.HOME=/root| -- |command=cmount| can be used to run |cmount| or any other rclone command - rather than the default |mount|. -- |args2env| will pass mount options to the mount helper running in background + Typically you will use `env.HTTPS_PROXY=proxy.host:3128` or `env.HOME=/root` +- `command=cmount` can be used to run `cmount` or any other rclone command + rather than the default `mount`. +- `args2env` will pass mount options to the mount helper running in background via environment variables instead of command line arguments. This allows to - hide secrets from such commands as |ps| or |pgrep|. -- |vv...| will be transformed into appropriate |--verbose=N| -- standard mount options like |x-systemd.automount|, |_netdev|, |nosuid| and alike + hide secrets from such commands as `ps` or `pgrep`. +- `vv...` will be transformed into appropriate `--verbose=N` +- standard mount options like `x-systemd.automount`, `_netdev`, `nosuid` and alike are intended only for Automountd and ignored by rclone. -` diff --git a/cmd/selfupdate/selfupdate.go b/cmd/selfupdate/selfupdate.go index 307de1553..5adf9a4b8 100644 --- a/cmd/selfupdate/selfupdate.go +++ b/cmd/selfupdate/selfupdate.go @@ -10,6 +10,7 @@ import ( "bytes" "context" "crypto/sha256" + _ "embed" "encoding/hex" "errors" "fmt" @@ -35,6 +36,9 @@ import ( versionCmd "github.com/rclone/rclone/cmd/version" ) +//go:embed selfupdate.md +var selfUpdateHelp string + // Options contains options for the self-update command type Options struct { Check bool @@ -63,7 +67,7 @@ var cmdSelfUpdate = &cobra.Command{ Use: "selfupdate", Aliases: []string{"self-update"}, Short: `Update the rclone binary.`, - Long: strings.ReplaceAll(selfUpdateHelp, "|", "`"), + Long: selfUpdateHelp, Annotations: map[string]string{ "versionIntroduced": "v1.55", }, diff --git a/cmd/selfupdate/help.go b/cmd/selfupdate/selfupdate.md similarity index 59% rename from cmd/selfupdate/help.go rename to cmd/selfupdate/selfupdate.md index ec3595b2d..73a006662 100644 --- a/cmd/selfupdate/help.go +++ b/cmd/selfupdate/selfupdate.md @@ -1,55 +1,47 @@ -//go:build !noselfupdate -// +build !noselfupdate - -package selfupdate - -// Note: "|" will be replaced by backticks in the help string below -var selfUpdateHelp = ` This command downloads the latest release of rclone and replaces the currently running binary. The download is verified with a hashsum and cryptographically signed signature; see [the release signing docs](/release_signing/) for details. -If used without flags (or with implied |--stable| flag), this command +If used without flags (or with implied `--stable` flag), this command will install the latest stable release. However, some issues may be fixed (or features added) only in the latest beta release. In such cases you should -run the command with the |--beta| flag, i.e. |rclone selfupdate --beta|. +run the command with the `--beta` flag, i.e. `rclone selfupdate --beta`. You can check in advance what version would be installed by adding the -|--check| flag, then repeat the command without it when you are satisfied. +`--check` flag, then repeat the command without it when you are satisfied. Sometimes the rclone team may recommend you a concrete beta or stable rclone release to troubleshoot your issue or add a bleeding edge feature. -The |--version VER| flag, if given, will update to the concrete version -instead of the latest one. If you omit micro version from |VER| (for -example |1.53|), the latest matching micro version will be used. +The `--version VER` flag, if given, will update to the concrete version +instead of the latest one. If you omit micro version from `VER` (for +example `1.53`), the latest matching micro version will be used. Upon successful update rclone will print a message that contains a previous version number. You will need it if you later decide to revert your update for some reason. Then you'll have to note the previous version and run the -following command: |rclone selfupdate [--beta] OLDVER|. -If the old version contains only dots and digits (for example |v1.54.0|) -then it's a stable release so you won't need the |--beta| flag. Beta releases -have an additional information similar to |v1.54.0-beta.5111.06f1c0c61|. +following command: `rclone selfupdate [--beta] OLDVER`. +If the old version contains only dots and digits (for example `v1.54.0`) +then it's a stable release so you won't need the `--beta` flag. Beta releases +have an additional information similar to `v1.54.0-beta.5111.06f1c0c61`. (if you are a developer and use a locally built rclone, the version number -will end with |-DEV|, you will have to rebuild it as it obviously can't +will end with `-DEV`, you will have to rebuild it as it obviously can't be distributed). If you previously installed rclone via a package manager, the package may include local documentation or configure services. You may wish to update -with the flag |--package deb| or |--package rpm| (whichever is correct for -your OS) to update these too. This command with the default |--package zip| +with the flag `--package deb` or `--package rpm` (whichever is correct for +your OS) to update these too. This command with the default `--package zip` will update only the rclone executable so the local manual may become inaccurate after it. The [rclone mount](/commands/rclone_mount/) command may or may not support extended FUSE options depending on the build and OS. -|selfupdate| will refuse to update if the capability would be discarded. +`selfupdate` will refuse to update if the capability would be discarded. Note: Windows forbids deletion of a currently running executable so this command will rename the old executable to 'rclone.old.exe' upon success. Please note that this command was not available before rclone version 1.55. -If it fails for you with the message |unknown command "selfupdate"| then +If it fails for you with the message `unknown command "selfupdate"` then you will need to update manually following the install instructions located at https://rclone.org/install/ -` diff --git a/cmd/serve/docker/docker.go b/cmd/serve/docker/docker.go index 85241c97b..9493f7407 100644 --- a/cmd/serve/docker/docker.go +++ b/cmd/serve/docker/docker.go @@ -3,8 +3,8 @@ package docker import ( "context" + _ "embed" "path/filepath" - "strings" "syscall" "github.com/spf13/cobra" @@ -30,6 +30,9 @@ var ( noSpec = false ) +//go:embed docker.md +var longHelp string + func init() { cmdFlags := Command.Flags() // Add command specific flags @@ -47,7 +50,7 @@ func init() { var Command = &cobra.Command{ Use: "docker", Short: `Serve any remote on docker's volume plugin API.`, - Long: strings.ReplaceAll(longHelp, "|", "`") + vfs.Help, + Long: longHelp + vfs.Help, Annotations: map[string]string{ "versionIntroduced": "v1.56", "groups": "Filter", diff --git a/cmd/serve/docker/help.go b/cmd/serve/docker/docker.md similarity index 67% rename from cmd/serve/docker/help.go rename to cmd/serve/docker/docker.md index 531213dd0..853f5f954 100644 --- a/cmd/serve/docker/help.go +++ b/cmd/serve/docker/docker.md @@ -1,7 +1,3 @@ -package docker - -// Note: "|" will be replaced by backticks -var longHelp = ` This command implements the Docker volume plugin API allowing docker to use rclone as a data storage mechanism for various cloud providers. rclone provides [docker volume plugin](/docker) based on it. @@ -12,32 +8,31 @@ docker daemon and runs the corresponding code when necessary. Docker plugins can run as a managed plugin under control of the docker daemon or as an independent native service. For testing, you can just run it directly from the command line, for example: -||| +``` sudo rclone serve docker --base-dir /tmp/rclone-volumes --socket-addr localhost:8787 -vv -||| +``` -Running |rclone serve docker| will create the said socket, listening for +Running `rclone serve docker` will create the said socket, listening for commands from Docker to create the necessary Volumes. Normally you need not -give the |--socket-addr| flag. The API will listen on the unix domain socket -at |/run/docker/plugins/rclone.sock|. In the example above rclone will create -a TCP socket and a small file |/etc/docker/plugins/rclone.spec| containing -the socket address. We use |sudo| because both paths are writeable only by +give the `--socket-addr` flag. The API will listen on the unix domain socket +at `/run/docker/plugins/rclone.sock`. In the example above rclone will create +a TCP socket and a small file `/etc/docker/plugins/rclone.spec` containing +the socket address. We use `sudo` because both paths are writeable only by the root user. If you later decide to change listening socket, the docker daemon must be -restarted to reconnect to |/run/docker/plugins/rclone.sock| -or parse new |/etc/docker/plugins/rclone.spec|. Until you restart, any +restarted to reconnect to `/run/docker/plugins/rclone.sock` +or parse new `/etc/docker/plugins/rclone.spec`. Until you restart, any volume related docker commands will timeout trying to access the old socket. Running directly is supported on **Linux only**, not on Windows or MacOS. This is not a problem with managed plugin mode described in details in the [full documentation](https://rclone.org/docker). -The command will create volume mounts under the path given by |--base-dir| -(by default |/var/lib/docker-volumes/rclone| available only to root) -and maintain the JSON formatted file |docker-plugin.state| in the rclone cache +The command will create volume mounts under the path given by `--base-dir` +(by default `/var/lib/docker-volumes/rclone` available only to root) +and maintain the JSON formatted file `docker-plugin.state` in the rclone cache directory with book-keeping records of created and mounted volumes. All mount and VFS options are submitted by the docker daemon via API, but you can also provide defaults on the command line as well as set path to the config file and cache directory or adjust logging verbosity. -` diff --git a/vfs/vfs.go b/vfs/vfs.go index 5202d8cf6..2e35dc810 100644 --- a/vfs/vfs.go +++ b/vfs/vfs.go @@ -22,6 +22,7 @@ package vfs import ( "context" + _ "embed" "fmt" "io" "os" @@ -42,6 +43,11 @@ import ( "github.com/rclone/rclone/vfs/vfscommon" ) +// Help for the VFS. +// +//go:embed vfs.md +var Help string + // Node represents either a directory (*Dir) or a file (*File) type Node interface { os.FileInfo diff --git a/vfs/help.go b/vfs/vfs.md similarity index 80% rename from vfs/help.go rename to vfs/vfs.md index c509696ad..dba7afa0a 100644 --- a/vfs/help.go +++ b/vfs/vfs.md @@ -1,15 +1,3 @@ -package vfs - -import ( - "strings" -) - -// Help contains text describing file and directory caching to add to -// the command help. -// Warning: "!" (sic) will be replaced by backticks below, -// -// but the pipe character "|" can be used as is. -var Help = strings.ReplaceAll(` ### VFS - Virtual File System This command uses the VFS layer. This adapts the cloud storage objects @@ -26,7 +14,7 @@ about files and directories (but not the data) in memory. ### VFS Directory Cache -Using the !--dir-cache-time! flag, you can control how long a +Using the `--dir-cache-time` flag, you can control how long a directory should be considered up to date and not refreshed from the backend. Changes made through the VFS will appear immediately or invalidate the cache. @@ -40,7 +28,7 @@ the directory cache expires if the backend configured does not support polling for changes. If the backend supports polling, changes will be picked up within the polling interval. -You can send a !SIGHUP! signal to rclone for it to flush all +You can send a `SIGHUP` signal to rclone for it to flush all directory caches, regardless of how old they are. Assuming only one rclone instance is running, you can reset the cache like this: @@ -57,7 +45,7 @@ Or individual files or directories: ### VFS File Buffering -The !--buffer-size! flag determines the amount of memory, +The `--buffer-size` flag determines the amount of memory, that will be used to buffer data in advance. Each open file will try to keep the specified amount of data in memory @@ -70,7 +58,7 @@ yet read. If the buffer is empty, only a small amount of memory will be used. The maximum memory used by rclone for buffering can be up to -!--buffer-size * open files!. +`--buffer-size * open files`. ### VFS File Caching @@ -92,32 +80,32 @@ find that you need one or the other or both. --vfs-cache-poll-interval duration Interval to poll the cache for stale objects (default 1m0s) --vfs-write-back duration Time to writeback files after last use when using cache (default 5s) -If run with !-vv! rclone will print the location of the file cache. The +If run with `-vv` rclone will print the location of the file cache. The files are stored in the user cache file area which is OS dependent but -can be controlled with !--cache-dir! or setting the appropriate +can be controlled with `--cache-dir` or setting the appropriate environment variable. -The cache has 4 different modes selected by !--vfs-cache-mode!. +The cache has 4 different modes selected by `--vfs-cache-mode`. The higher the cache mode the more compatible rclone becomes at the cost of using disk space. Note that files are written back to the remote only when they are -closed and if they haven't been accessed for !--vfs-write-back! +closed and if they haven't been accessed for `--vfs-write-back` seconds. If rclone is quit or dies with files that haven't been uploaded, these will be uploaded next time rclone is run with the same flags. -If using !--vfs-cache-max-size! or !--vfs-cache-min-free-size! note +If using `--vfs-cache-max-size` or `--vfs-cache-min-free-size` note that the cache may exceed these quotas for two reasons. Firstly -because it is only checked every !--vfs-cache-poll-interval!. Secondly +because it is only checked every `--vfs-cache-poll-interval`. Secondly because open files cannot be evicted from the cache. When -!--vfs-cache-max-size! or !--vfs-cache-min-free-size! is exceeded, +`--vfs-cache-max-size` or `--vfs-cache-min-free-size` is exceeded, rclone will attempt to evict the least accessed files from the cache first. rclone will start with files that haven't been accessed for the longest. This cache flushing strategy is efficient and more relevant files are likely to remain cached. -The !--vfs-cache-max-age! will evict files from the cache +The `--vfs-cache-max-age` will evict files from the cache after the set time since last access has passed. The default value of 1 hour will start evicting files from cache that haven't been accessed for 1 hour. When a cached file is accessed the 1 hour timer is reset to 0 @@ -125,10 +113,10 @@ and will wait for 1 more hour before evicting. Specify the time with standard notation, s, m, h, d, w . You **should not** run two copies of rclone using the same VFS cache -with the same or overlapping remotes if using !--vfs-cache-mode > off!. +with the same or overlapping remotes if using `--vfs-cache-mode > off`. This can potentially cause data corruption if you do. You can work around this by giving each rclone its own cache hierarchy with -!--cache-dir!. You don't need to worry about this if the remotes in +`--cache-dir`. You don't need to worry about this if the remotes in use don't overlap. #### --vfs-cache-mode off @@ -184,14 +172,14 @@ their full size in the cache, but they will be sparse files with only the data that has been downloaded present in them. This mode should support all normal file system operations and is -otherwise identical to !--vfs-cache-mode! writes. +otherwise identical to `--vfs-cache-mode` writes. -When reading a file rclone will read !--buffer-size! plus -!--vfs-read-ahead! bytes ahead. The !--buffer-size! is buffered in memory -whereas the !--vfs-read-ahead! is buffered on disk. +When reading a file rclone will read `--buffer-size` plus +`--vfs-read-ahead` bytes ahead. The `--buffer-size` is buffered in memory +whereas the `--vfs-read-ahead` is buffered on disk. -When using this mode it is recommended that !--buffer-size! is not set -too large and !--vfs-read-ahead! is set large if required. +When using this mode it is recommended that `--buffer-size` is not set +too large and `--vfs-read-ahead` is set large if required. **IMPORTANT** not all file systems support sparse files. In particular FAT/exFAT do not. Rclone will perform very badly if the cache @@ -213,17 +201,17 @@ where available on an object. On some backends some of these attributes are slow to read (they take an extra API call per object, or extra work per object). -For example !hash! is slow with the !local! and !sftp! backends as -they have to read the entire file and hash it, and !modtime! is slow -with the !s3!, !swift!, !ftp! and !qinqstor! backends because they +For example `hash` is slow with the `local` and `sftp` backends as +they have to read the entire file and hash it, and `modtime` is slow +with the `s3`, `swift`, `ftp` and `qinqstor` backends because they need to do an extra API call to fetch it. -If you use the !--vfs-fast-fingerprint! flag then rclone will not +If you use the `--vfs-fast-fingerprint` flag then rclone will not include the slow operations in the fingerprint. This makes the fingerprinting less accurate but much faster and will improve the opening time of cached files. -If you are running a vfs cache over !local!, !s3! or !swift! backends +If you are running a vfs cache over `local`, `s3` or `swift` backends then using this flag is recommended. Note that if you change the value of this flag, the fingerprints of @@ -243,19 +231,19 @@ These flags control the chunking: --vfs-read-chunk-size SizeSuffix Read the source objects in chunks (default 128M) --vfs-read-chunk-size-limit SizeSuffix Max chunk doubling size (default off) -Rclone will start reading a chunk of size !--vfs-read-chunk-size!, -and then double the size for each read. When !--vfs-read-chunk-size-limit! is -specified, and greater than !--vfs-read-chunk-size!, the chunk size for each +Rclone will start reading a chunk of size `--vfs-read-chunk-size`, +and then double the size for each read. When `--vfs-read-chunk-size-limit` is +specified, and greater than `--vfs-read-chunk-size`, the chunk size for each open file will get doubled only until the specified value is reached. If the value is "off", which is the default, the limit is disabled and the chunk size will grow indefinitely. -With !--vfs-read-chunk-size 100M! and !--vfs-read-chunk-size-limit 0! +With `--vfs-read-chunk-size 100M` and `--vfs-read-chunk-size-limit 0` the following parts will be downloaded: 0-100M, 100M-200M, 200M-300M, 300M-400M and so on. -When !--vfs-read-chunk-size-limit 500M! is specified, the result would be +When `--vfs-read-chunk-size-limit 500M` is specified, the result would be 0-100M, 100M-300M, 300M-700M, 700M-1200M, 1200M-1700M and so on. -Setting !--vfs-read-chunk-size! to !0! or "off" disables chunked reading. +Setting `--vfs-read-chunk-size` to `0` or "off" disables chunked reading. ### VFS Performance @@ -263,8 +251,8 @@ These flags may be used to enable/disable features of the VFS for performance or other reasons. See also the [chunked reading](#vfs-chunked-reading) feature. -In particular S3 and Swift benefit hugely from the !--no-modtime! flag -(or use !--use-server-modtime! for a slightly different effect) as each +In particular S3 and Swift benefit hugely from the `--no-modtime` flag +(or use `--use-server-modtime` for a slightly different effect) as each read of the modification time takes a transaction. --no-checksum Don't compare checksums on up/download. @@ -280,9 +268,9 @@ on disk cache file. --vfs-read-wait duration Time to wait for in-sequence read before seeking (default 20ms) --vfs-write-wait duration Time to wait for in-sequence write before giving error (default 1s) -When using VFS write caching (!--vfs-cache-mode! with value writes or full), -the global flag !--transfers! can be set to adjust the number of parallel uploads of -modified files from the cache (the related global flag !--checkers! has no effect on the VFS). +When using VFS write caching (`--vfs-cache-mode` with value writes or full), +the global flag `--transfers` can be set to adjust the number of parallel uploads of +modified files from the cache (the related global flag `--checkers` has no effect on the VFS). --transfers int Number of file transfers to run in parallel (default 4) @@ -299,7 +287,7 @@ It is not allowed for two files in the same directory to differ only by case. Usually file systems on macOS are case-insensitive. It is possible to make macOS file systems case-sensitive but that is not the default. -The !--vfs-case-insensitive! VFS flag controls how rclone handles these +The `--vfs-case-insensitive` VFS flag controls how rclone handles these two cases. If its value is "false", rclone passes file names to the remote as-is. If the flag is "true" (or appears without a value on the command line), rclone may perform a "fixup" as explained below. @@ -331,13 +319,12 @@ It can be useful when those statistics cannot be read correctly automatically. ### Alternate report of used bytes Some backends, most notably S3, do not report the amount of bytes used. -If you need this information to be available when running !df! on the -filesystem, then pass the flag !--vfs-used-is-size! to rclone. +If you need this information to be available when running `df` on the +filesystem, then pass the flag `--vfs-used-is-size` to rclone. With this flag set, instead of relying on the backend to report this -information, rclone will scan the whole remote similar to !rclone size! +information, rclone will scan the whole remote similar to `rclone size` and compute the total used space itself. -_WARNING._ Contrary to !rclone size!, this flag ignores filters so that the +_WARNING._ Contrary to `rclone size`, this flag ignores filters so that the result is accurate. However, this is very inefficient and may cost lots of API calls resulting in extra charges. Use it as a last resort and only with caching. -`, "!", "`")