github-mirror/rclone
2
0
Fork 0
mirror of https://github.com/rclone/rclone.git synced 2024-11-22 10:54:48 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

sftp: add support for about and hashsum on windows server

Browse Source 
Windows shells like cmd and powershell needs to use different quoting/escaping
of strings and paths than the unix shell, and also absolute paths must be fixed
by removing leading slash that the POSIX formatted paths have
(e.g. /C:/Users does not work in shell, it must be converted to C:/Users).

Tries to autodetect shell type (cmd, powershell, unix) on first use.

Implemented default builtin powershell functions for hashsum and about when remote
shell is powershell.

See #5763

Fixes #5758
This commit is contained in:
albertony 2021-10-27 17:01:54 +02:00
parent 218bf2183d
commit b4091f282a
3 changed files with 436 additions and 89 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

332
backend/sftp/sftp.go
View File

@ -39,6 +39,8 @@ import (
)
const (
defaultShellType = "unix"
shellTypeNotSupported = "none"
hashCommandNotSupported = "none"
minSleep = 100 * time.Millisecond
maxSleep = 2 * time.Second
@ -47,7 +49,9 @@ const (
)
var (
currentUser = env.CurrentUser()
currentUser = env.CurrentUser()
posixWinAbsPathRegex = regexp.MustCompile(`^/[a-zA-Z]\:($|/)`) // E.g. "/C:" or anything starting with "/C:/"
unixShellEscapeRegex = regexp.MustCompile("[^A-Za-z0-9_.,:/\\@\u0080-\uFFFFFFFF\n-]")
)
func init() {
@ -148,16 +152,16 @@ If this is set and no password is supplied then rclone will:
}, {
Name: "path_override",
Default: "",
Help: `Override path used by SSH connection.
Help: `Override path used by SSH shell commands.
This allows checksum calculation when SFTP and SSH paths are
different. This issue affects among others Synology NAS boxes.
Shared folders can be found in directories representing volumes
E.g. if shared folders can be found in directories representing volumes:
rclone sync /home/local/directory remote:/directory --sftp-path-override /volume2/directory
Home directory can be found in a shared folder called "home"
E.g. if home directory can be found in a shared folder called "home":
rclone sync /home/local/directory remote:/home/directory --sftp-path-override /volume1/homes/USER/directory`,
Advanced: true,
@ -166,6 +170,26 @@ Home directory can be found in a shared folder called "home"
Default: true,
Help: "Set the modified time on the remote if set.",
Advanced: true,
}, {
Name: "shell_type",
Default: "",
Help: "The type of SSH shell on remote server, if any.\n\nLeave blank for autodetect.",
Advanced: true,
Examples: []fs.OptionExample{
{
Value: shellTypeNotSupported,
Help: "No shell access",
}, {
Value: "unix",
Help: "Unix shell",
}, {
Value: "powershell",
Help: "PowerShell",
}, {
Value: "cmd",
Help: "Windows Command Prompt",
},
},
}, {
Name: "md5sum_command",
Default: "",
@ -270,6 +294,7 @@ type Options struct {
AskPassword bool `config:"ask_password"`
PathOverride string `config:"path_override"`
SetModTime bool `config:"set_modtime"`
ShellType string `config:"shell_type"`
Md5sumCommand string `config:"md5sum_command"`
Sha1sumCommand string `config:"sha1sum_command"`
SkipLinks bool `config:"skip_links"`
@ -286,6 +311,8 @@ type Fs struct {
name string
root string
absRoot string
shellRoot string
shellType string
opt Options // parsed options
ci *fs.ConfigInfo // global config
m configmap.Mapper // config
@ -542,7 +569,7 @@ func (f *Fs) drainPool(ctx context.Context) (err error) {
f.drain.Stop()
}
if len(f.pool) != 0 {
fs.Debugf(f, "closing %d unused connections", len(f.pool))
fs.Debugf(f, "Closing %d unused connections", len(f.pool))
}
for i, c := range f.pool {
if cErr := c.closed(); cErr == nil {
@ -739,7 +766,7 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
//
// Just send the password back for all questions
func (f *Fs) keyboardInteractiveReponse(user, instruction string, questions []string, echos []bool, pass string) ([]string, error) {
fs.Debugf(f, "keyboard interactive auth requested")
fs.Debugf(f, "Keyboard interactive auth requested")
answers := make([]string, len(questions))
for i := range answers {
answers[i] = pass
@ -769,6 +796,7 @@ func NewFsWithConnection(ctx context.Context, f *Fs, name string, root string, m
f.name = name
f.root = root
f.absRoot = root
f.shellRoot = root
f.opt = *opt
f.m = m
f.config = sshConfig
@ -778,7 +806,7 @@ func NewFsWithConnection(ctx context.Context, f *Fs, name string, root string, m
f.savedpswd = ""
// set the pool drainer timer going
if f.opt.IdleTimeout > 0 {
f.drain = time.AfterFunc(time.Duration(opt.IdleTimeout), func() { _ = f.drainPool(ctx) })
f.drain = time.AfterFunc(time.Duration(f.opt.IdleTimeout), func() { _ = f.drainPool(ctx) })
}
f.features = (&fs.Features{
@ -790,16 +818,59 @@ func NewFsWithConnection(ctx context.Context, f *Fs, name string, root string, m
if err != nil {
return nil, fmt.Errorf("NewFs: %w", err)
}
cwd, err := c.sftpClient.Getwd()
f.putSftpConnection(&c, nil)
if err != nil {
fs.Debugf(f, "Failed to read current directory - using relative paths: %v", err)
} else if !path.IsAbs(f.root) {
f.absRoot = path.Join(cwd, f.root)
fs.Debugf(f, "Using absolute root directory %q", f.absRoot)
// Check remote shell type, try to auto-detect if not configured and save to config for later
if f.opt.ShellType != "" {
f.shellType = f.opt.ShellType
fs.Debugf(f, "Shell type %q from config", f.shellType)
} else {
session, err := c.sshClient.NewSession()
if err != nil {
f.shellType = shellTypeNotSupported
fs.Debugf(f, "Failed to get shell session for shell type detection command: %v", err)
} else {
var stdout, stderr bytes.Buffer
session.Stdout = &stdout
session.Stderr = &stderr
shellCmd := "echo ${ShellId}%ComSpec%"
fs.Debugf(f, "Running shell type detection remote command: %s", shellCmd)
err = session.Run(shellCmd)
_ = session.Close()
if err != nil {
f.shellType = defaultShellType
fs.Debugf(f, "Remote command failed: %v (stdout=%v) (stderr=%v)", err, bytes.TrimSpace(stdout.Bytes()), bytes.TrimSpace(stderr.Bytes()))
} else {
outBytes := stdout.Bytes()
fs.Debugf(f, "Remote command result: %s", outBytes)
outString := string(bytes.TrimSpace(stdout.Bytes()))
if strings.HasPrefix(outString, "Microsoft.PowerShell") { // If PowerShell: "Microsoft.PowerShell%ComSpec%"
f.shellType = "powershell"
} else if !strings.HasSuffix(outString, "%ComSpec%") { // If Command Prompt: "${ShellId}C:\WINDOWS\system32\cmd.exe"
f.shellType = "cmd"
} else { // If Unix: "%ComSpec%"
f.shellType = "unix"
}
}
}
// Save permanently in config to avoid the extra work next time
fs.Debugf(f, "Shell type %q detected (set option shell_type to override)", f.shellType)
f.m.Set("shell_type", f.shellType)
}
// Ensure we have absolute path to root
// It appears that WS FTP doesn't like relative paths,
// and the openssh sftp tool also uses absolute paths.
if !path.IsAbs(f.root) {
path, err := c.sftpClient.RealPath(f.root)
if err != nil {
fs.Debugf(f, "Failed to resolve path - using relative paths: %v", err)
} else {
f.absRoot = path
fs.Debugf(f, "Relative path resolved to %q", f.absRoot)
}
}
f.putSftpConnection(&c, err)
if root != "" {
// Check to see if the root actually an existing file
// Check to see if the root is actually an existing file,
// and if so change the filesystem root to its parent directory.
oldAbsRoot := f.absRoot
remote := path.Base(root)
f.root = path.Dir(root)
@ -807,20 +878,24 @@ func NewFsWithConnection(ctx context.Context, f *Fs, name string, root string, m
if f.root == "." {
f.root = ""
}
_, err := f.NewObject(ctx, remote)
_, err = f.NewObject(ctx, remote)
if err != nil {
if err == fs.ErrorObjectNotFound || err == fs.ErrorIsDir {
// File doesn't exist so return old f
f.root = root
f.absRoot = oldAbsRoot
return f, nil
if err != fs.ErrorObjectNotFound && err != fs.ErrorIsDir {
return nil, err
}
return nil, err
// File doesn't exist so keep the old f
f.root = root
f.absRoot = oldAbsRoot
err = nil
} else {
// File exists so change fs to point to the parent and return it with an error
err = fs.ErrorIsFile
}
// return an error with an fs which points to the parent
return f, fs.ErrorIsFile
} else {
err = nil
}
return f, nil
fs.Debugf(f, "Using root directory %q", f.absRoot)
return f, err
}
// Name returns the configured name of the file system
@ -1155,74 +1230,147 @@ func (f *Fs) run(ctx context.Context, cmd string) ([]byte, error) {
// Hashes returns the supported hash types of the filesystem
func (f *Fs) Hashes() hash.Set {
ctx := context.TODO()
if f.opt.DisableHashCheck {
return hash.Set(hash.None)
}
if f.cachedHashes != nil {
return *f.cachedHashes
}
hashSet := hash.NewHashSet()
f.cachedHashes = &hashSet
if f.opt.DisableHashCheck || f.shellType == shellTypeNotSupported {
return hashSet
}
// look for a hash command which works
checkHash := func(commands []string, expected string, hashCommand *string, changed *bool) bool {
checkHash := func(hashType hash.Type, commands []struct{ hashFile, hashEmpty string }, expected string, hashCommand *string, changed *bool) bool {
if *hashCommand == hashCommandNotSupported {
return false
}
if *hashCommand != "" {
return true
}
fs.Debugf(f, "Checking default %v hash commands", hashType)
*changed = true
for _, command := range commands {
output, err := f.run(ctx, command)
output, err := f.run(ctx, command.hashEmpty)
if err != nil {
fs.Debugf(f, "Hash command skipped: %v", err)
continue
}
output = bytes.TrimSpace(output)
fs.Debugf(f, "checking %q command: %q", command, output)
if parseHash(output) == expected {
*hashCommand = command
*hashCommand = command.hashFile
fs.Debugf(f, "Hash command accepted")
return true
}
fs.Debugf(f, "Hash command skipped: Wrong output")
}
*hashCommand = hashCommandNotSupported
return false
}
changed := false
md5Works := checkHash([]string{"md5sum", "md5 -r", "rclone md5sum"}, "d41d8cd98f00b204e9800998ecf8427e", &f.opt.Md5sumCommand, &changed)
sha1Works := checkHash([]string{"sha1sum", "sha1 -r", "rclone sha1sum"}, "da39a3ee5e6b4b0d3255bfef95601890afd80709", &f.opt.Sha1sumCommand, &changed)
md5Commands := []struct {
hashFile, hashEmpty string
}{
{"md5sum", "md5sum"},
{"md5 -r", "md5 -r"},
{"rclone md5sum", "rclone md5sum"},
}
sha1Commands := []struct {
hashFile, hashEmpty string
}{
{"sha1sum", "sha1sum"},
{"sha1 -r", "sha1 -r"},
{"rclone sha1sum", "rclone sha1sum"},
}
if f.shellType == "powershell" {
md5Commands = append(md5Commands, struct {
hashFile, hashEmpty string
}{
"&{param($Path);Get-FileHash -Algorithm MD5 -LiteralPath $Path -ErrorAction Stop|Select-Object -First 1 -ExpandProperty Hash|ForEach-Object{\"$($_.ToLower()) ${Path}\"}}",
"Get-FileHash -Algorithm MD5 -InputStream ([System.IO.MemoryStream]::new()) -ErrorAction Stop|Select-Object -First 1 -ExpandProperty Hash|ForEach-Object{$_.ToLower()}",
})
sha1Commands = append(sha1Commands, struct {
hashFile, hashEmpty string
}{
"&{param($Path);Get-FileHash -Algorithm SHA1 -LiteralPath $Path -ErrorAction Stop|Select-Object -First 1 -ExpandProperty Hash|ForEach-Object{\"$($_.ToLower()) ${Path}\"}}",
"Get-FileHash -Algorithm SHA1 -InputStream ([System.IO.MemoryStream]::new()) -ErrorAction Stop|Select-Object -First 1 -ExpandProperty Hash|ForEach-Object{$_.ToLower()}",
})
}
md5Works := checkHash(hash.MD5, md5Commands, "d41d8cd98f00b204e9800998ecf8427e", &f.opt.Md5sumCommand, &changed)
sha1Works := checkHash(hash.SHA1, sha1Commands, "da39a3ee5e6b4b0d3255bfef95601890afd80709", &f.opt.Sha1sumCommand, &changed)
if changed {
// Save permanently in config to avoid the extra work next time
fs.Debugf(f, "Setting hash command for %v to %q (set sha1sum_command to override)", hash.MD5, f.opt.Md5sumCommand)
f.m.Set("md5sum_command", f.opt.Md5sumCommand)
fs.Debugf(f, "Setting hash command for %v to %q (set md5sum_command to override)", hash.SHA1, f.opt.Sha1sumCommand)
f.m.Set("sha1sum_command", f.opt.Sha1sumCommand)
}
set := hash.NewHashSet()
if sha1Works {
set.Add(hash.SHA1)
hashSet.Add(hash.SHA1)
}
if md5Works {
set.Add(hash.MD5)
hashSet.Add(hash.MD5)
}
f.cachedHashes = &set
return set
return hashSet
}
// About gets usage stats
func (f *Fs) About(ctx context.Context) (*fs.Usage, error) {
escapedPath := shellEscape(f.root)
if f.opt.PathOverride != "" {
escapedPath = shellEscape(path.Join(f.opt.PathOverride, f.root))
if f.shellType == shellTypeNotSupported || f.shellType == "cmd" {
fs.Debugf(f, "About shell command is not available for shell type %q (set option shell_type to override)", f.shellType)
return nil, fmt.Errorf("not supported with shell type %q", f.shellType)
}
if len(escapedPath) == 0 {
escapedPath = "/"
aboutShellPath := f.remoteShellPath("")
if aboutShellPath == "" {
aboutShellPath = "/"
}
stdout, err := f.run(ctx, "df -k "+escapedPath)
fs.Debugf(f, "About path %q", aboutShellPath)
aboutShellPathArg, err := f.quoteOrEscapeShellPath(aboutShellPath)
if err != nil {
return nil, err
}
// PowerShell
if f.shellType == "powershell" {
shellCmd := "Get-Item " + aboutShellPathArg + " -ErrorAction Stop|Select-Object -First 1 -ExpandProperty PSDrive|ForEach-Object{\"$($_.Used) $($_.Free)\"}"
fs.Debugf(f, "About using shell command for shell type %q", f.shellType)
stdout, err := f.run(ctx, shellCmd)
if err != nil {
fs.Debugf(f, "About shell command for shell type %q failed (set option shell_type to override): %v", f.shellType, err)
return nil, fmt.Errorf("powershell command failed: %w", err)
}
split := strings.Fields(string(stdout))
usage := &fs.Usage{}
if len(split) == 2 {
usedValue, usedErr := strconv.ParseInt(split[0], 10, 64)
if usedErr == nil {
usage.Used = fs.NewUsageValue(usedValue)
}
freeValue, freeErr := strconv.ParseInt(split[1], 10, 64)
if freeErr == nil {
usage.Free = fs.NewUsageValue(freeValue)
if usedErr == nil {
usage.Total = fs.NewUsageValue(usedValue + freeValue)
}
}
}
return usage, nil
}
// Unix/default shell
shellCmd := "df -k " + aboutShellPathArg
fs.Debugf(f, "About using shell command for shell type %q", f.shellType)
stdout, err := f.run(ctx, shellCmd)
if err != nil {
fs.Debugf(f, "About shell command for shell type %q failed (set option shell_type to override): %v", f.shellType, err)
return nil, fmt.Errorf("your remote may not have the required df utility: %w", err)
}
usageTotal, usageUsed, usageAvail := parseUsage(stdout)
usage := &fs.Usage{}
if usageTotal >= 0 {
@ -1287,31 +1435,78 @@ func (o *Object) Hash(ctx context.Context, r hash.Type) (string, error) {
return "", hash.ErrUnsupported
}
escapedPath := shellEscape(o.path())
if o.fs.opt.PathOverride != "" {
escapedPath = shellEscape(path.Join(o.fs.opt.PathOverride, o.remote))
}
b, err := o.fs.run(ctx, hashCmd+" "+escapedPath)
shellPathArg, err := o.fs.quoteOrEscapeShellPath(o.shellPath())
if err != nil {
return "", fmt.Errorf("failed to calculate %v hash: %w", r, err)
}
str := parseHash(b)
if r == hash.MD5 {
o.md5sum = &str
} else if r == hash.SHA1 {
o.sha1sum = &str
outBytes, err := o.fs.run(ctx, hashCmd+" "+shellPathArg)
if err != nil {
return "", fmt.Errorf("failed to calculate %v hash: %w", r, err)
}
return str, nil
hashString := parseHash(outBytes)
fs.Debugf(o, "Parsed hash: %s", hashString)
if r == hash.MD5 {
o.md5sum = &hashString
} else if r == hash.SHA1 {
o.sha1sum = &hashString
}
return hashString, nil
}
var shellEscapeRegex = regexp.MustCompile("[^A-Za-z0-9_.,:/\\@\u0080-\uFFFFFFFF\n-]")
// quoteOrEscapeShellPath makes path a valid string argument in configured shell
// and also ensures it cannot cause unintended behavior.
func quoteOrEscapeShellPath(shellType string, shellPath string) (string, error) {
// PowerShell
if shellType == "powershell" {
return "'" + strings.ReplaceAll(shellPath, "'", "''") + "'", nil
}
// Windows Command Prompt
if shellType == "cmd" {
if strings.Contains(shellPath, "\"") {
return "", fmt.Errorf("path is not valid in shell type %s: %s", shellType, shellPath)
}
return "\"" + shellPath + "\"", nil
}
// Unix shell
safe := unixShellEscapeRegex.ReplaceAllString(shellPath, `\$0`)
return strings.ReplaceAll(safe, "\n", "'\n'"), nil
}
// Escape a string s.t. it cannot cause unintended behavior
// when sending it to a shell.
func shellEscape(str string) string {
safe := shellEscapeRegex.ReplaceAllString(str, `\$0`)
return strings.ReplaceAll(safe, "\n", "'\n'")
// quoteOrEscapeShellPath makes path a valid string argument in configured shell
func (f *Fs) quoteOrEscapeShellPath(shellPath string) (string, error) {
return quoteOrEscapeShellPath(f.shellType, shellPath)
}
// remotePath returns the native SFTP path of the file or directory at the remote given
func (f *Fs) remotePath(remote string) string {
return path.Join(f.absRoot, remote)
}
// remoteShellPath returns the SSH shell path of the file or directory at the remote given
func (f *Fs) remoteShellPath(remote string) string {
if f.opt.PathOverride != "" {
shellPath := path.Join(f.opt.PathOverride, remote)
fs.Debugf(f, "Shell path redirected to %q with option path_override", shellPath)
return shellPath
}
shellPath := path.Join(f.absRoot, remote)
if f.shellType == "powershell" || f.shellType == "cmd" {
// If remote shell is powershell or cmd, then server is probably Windows.
// The sftp package converts everything to POSIX paths: Forward slashes, and
// absolute paths starts with a slash. An absolute path on a Windows server will
// then look like this "/C:/Windows/System32". We must remove the "/" prefix
// to make this a valid path for shell commands. In case of PowerShell there is a
// possibility that it is a Unix server, with PowerShell Core shell, but assuming
// root folders with names such as "C:" are rare, we just take this risk,
// and option path_override can always be used to work around corner cases.
if posixWinAbsPathRegex.MatchString(shellPath) {
shellPath = strings.TrimPrefix(shellPath, "/")
fs.Debugf(f, "Shell path adjusted to %q (set option path_override to override)", shellPath)
return shellPath
}
}
fs.Debugf(f, "Shell path %q", shellPath)
return shellPath
}
// Converts a byte array from the SSH session returned by
@ -1362,9 +1557,14 @@ func (o *Object) ModTime(ctx context.Context) time.Time {
return o.modTime
}
// path returns the native path of the object
// path returns the native SFTP path of the object
func (o *Object) path() string {
return path.Join(o.fs.absRoot, o.remote)
return o.fs.remotePath(o.remote)
}
// shellPath returns the SSH shell path of the object
func (o *Object) shellPath() string {
return o.fs.remoteShellPath(o.remote)
}
// setMetadata updates the info in the object from the stat result passed in

41
backend/sftp/sftp_internal_test.go
View File

@ -10,7 +10,7 @@ import (
"github.com/stretchr/testify/assert"
)
func TestShellEscape(t *testing.T) {
func TestShellEscapeUnix(t *testing.T) {
for i, test := range []struct {
unescaped, escaped string
}{
@ -20,7 +20,44 @@ func TestShellEscape(t *testing.T) {
{"/test/\n", "/test/'\n'"},
{":\"'", ":\\\"\\'"},
} {
got := shellEscape(test.unescaped)
got, err := quoteOrEscapeShellPath("unix", test.unescaped)
assert.NoError(t, err)
assert.Equal(t, test.escaped, got, fmt.Sprintf("Test %d unescaped = %q", i, test.unescaped))
}
}
func TestShellEscapeCmd(t *testing.T) {
for i, test := range []struct {
unescaped, escaped string
ok bool
}{
{"", "\"\"", true},
{"c:/this/is/harmless", "\"c:/this/is/harmless\"", true},
{"c:/test&notepad", "\"c:/test&notepad\"", true},
{"c:/test\"&\"notepad", "", false},
} {
got, err := quoteOrEscapeShellPath("cmd", test.unescaped)
if test.ok {
assert.NoError(t, err)
assert.Equal(t, test.escaped, got, fmt.Sprintf("Test %d unescaped = %q", i, test.unescaped))
} else {
assert.Error(t, err)
}
}
}
func TestShellEscapePowerShell(t *testing.T) {
for i, test := range []struct {
unescaped, escaped string
}{
{"", "''"},
{"c:/this/is/harmless", "'c:/this/is/harmless'"},
{"c:/test&notepad", "'c:/test&notepad'"},
{"c:/test\"&\"notepad", "'c:/test\"&\"notepad'"},
{"c:/test'&'notepad", "'c:/test''&''notepad'"},
} {
got, err := quoteOrEscapeShellPath("powershell", test.unescaped)
assert.NoError(t, err)
assert.Equal(t, test.escaped, got, fmt.Sprintf("Test %d unescaped = %q", i, test.unescaped))
}
}

152
docs/content/sftp.md
View File

@ -28,6 +28,9 @@ Note that some SFTP servers will need the leading / - Synology is a
good example of this. rsync.net, on the other hand, requires users to
OMIT the leading /.
Note that by default rclone will try to execute shell commands on
the server, see [shell access considerations](#shell-access-considerations).
## Configuration
Here is an example of making an SFTP configuration. First run
@ -244,6 +247,116 @@ And then at the end of the session
These commands can be used in scripts of course.
### Shell access
Some functionality of the SFTP backend relies on remote shell access,
and the possibility to execute commands. This includes [checksum](#checksum),
and in some cases also [about](#about-command). The shell commands that
must be executed may be different on different type of shells, and also
quoting/escaping of file path arguments containing special characters may
be different. Rclone therefore needs to know what type of shell it is,
and if shell access is available at all.
Most servers run on some version of Unix, and then a basic Unix shell can
be assumed, without further distinction. Windows 10, Server 2019, and later
can also run a SSH server, which is a port of OpenSSH (see official
[installation guide](https://docs.microsoft.com/en-us/windows-server/administration/openssh/openssh_install_firstuse)). On a Windows server the shell handling is different: Although it can also
be set up to use a Unix type shell, e.g. Cygwin bash, the default is to
use Windows Command Prompt (cmd.exe), and PowerShell is a recommended
alternative. All of these have bahave differently, which rclone must handle.
Rclone tries to auto-detect what type of shell is used on the server,
first time you access the SFTP remote. If a remote shell session is
successfully created, it will look for indications that it is CMD or
PowerShell, with fall-back to Unix if not something else is detected.
If unable to even create a remote shell session, then shell command
execution will be disabled entirely. The result is stored in the SFTP
remote configuration, in option `shell_type`, so that the auto-detection
only have to be performed once. If you manually set a value for this
option before first run, the auto-detection will be skipped, and if
you set a different value later this will override any existing.
Value `none` can be set to avoid any attempts at executing shell
commands, e.g. if this is not allowed on the server.
When the server is [rclone serve sftp](/commands/rclone_serve_sftp/),
the rclone SFTP remote will detect this as a Unix type shell - even
if it is running on Windows. This server does not actually have a shell,
but it accepts input commands matching the specific ones that the
SFTP backend relies on for Unix shells, e.g. `md5sum` and `df`. Also
it handles the string escape rules used for Unix shell. Treating it
as a Unix type shell from a SFTP remote will therefore always be
correct, and support all features.
#### Shell access considerations
The shell type auto-detection logic, described above, means that
by default rclone will try to run a shell command the first time
a new sftp remote is accessed. If you configure a sftp remote
without a config file, e.g. an [on the fly](/docs/#backend-path-to-dir])
remote, rclone will have nowhere to store the result, and it
will re-run the command on every access. To avoid this you should
explicitely set the `shell_type` option to the correct value,
or to `none` if you want to prevent rclone from executing any
remote shell commands.
It is also important to note that, since the shell type decides
how quoting and escaping of file paths used as command-line arguments
are performed, configuring the wrong shell type may leave you exposed
to command injection exploits. Make sure to confirm the auto-detected
shell type, or explicitely set the shell type you know is correct,
or disable shell access until you know.
### Checksum
SFTP does not natively support checksums (file hash), but rclone
is able to use checksumming if the same login has shell access,
and can execute remote commands. If there is a command that can
calculate compatible checksums on the remote system, Rclone can
then be configured to execute this whenever a checksum is needed,
and read back the results. Currently MD5 and SHA-1 are supported.
Normally this requires an external utility being available on
the server. By default rclone will try commands `md5sum`, `md5`
and `rclone md5sum` for MD5 checksums, and the first one found usable
will be picked. Same with `sha1sum`, `sha1` and `rclone sha1sum`
commands for SHA-1 checksums. These utilities normally need to
be in the remote's PATH to be found.
In some cases the shell itself is capable of calculating checksums.
PowerShell is an example of such a shell. If rclone detects that the
remote shell is PowerShell, which means it most probably is a
Windows OpenSSH server, rclone will use a predefined script block
to produce the checksums when no external checksum commands are found
(see [shell access](#shell-access)). This assumes PowerShell version
4.0 or newer.
The options `md5sum_command` and `sha1_command` can be used to customize
the command to be executed for calculation of checksums. You can for
example set a specific path to where md5sum and sha1sum executables
are located, or use them to specify some other tools that print checksums
in compatible format. The value can include command-line arguments,
or even shell script blocks as with PowerShell. Rclone has subcommands
[md5sum](/commands/rclone_md5sum/) and [sha1sum](/commands/rclone_sha1sum/)
that use compatible format, which means if you have an rclone executable
on the server it can be used. As mentioned above, they will be automatically
picked up if found in PATH, but if not you can set something like
`/path/to/rclone md5sum` as the value of option `md5sum_command` to
make sure a specific executable is used.
Remote checksumming is recommended and enabled by default. First time
rclone is using a SFTP remote, if options `md5sum_command` or `sha1_command`
are not set, it will check if any of the default commands for each of them,
as described above, can be used. The result will be saved in the remote
configuration, so next time it will use the same. Value `none`
will be set if none of the default commands could be used for a specific
algorithm, and this algorithm will not be supported by the remote.
Disabling the checksumming may be required if you are connecting to SFTP servers
which are not under your control, and to which the execution of remote shell
commands is prohibited. Set the configuration option `disable_hashcheck`
to `true` to disable checksumming entirely, or set `shell_type` to `none`
to disable all functionality based on remote shell command execution.
### Modified time
Modified times are stored on the server to 1 second precision.
@ -255,6 +368,20 @@ upload (for example, certain configurations of ProFTPd with mod_sftp). If you
are using one of these servers, you can set the option `set_modtime = false` in
your RClone backend configuration to disable this behaviour.
### About command
SFTP supports the [about](/commands/rclone_about/) command if the
same login has access to a Unix shell, where the `df` command is
available (e.g. in the remote's PATH). `about` will return the
total space, free space, and used space on the remote for the disk
of the specified path on the remote or, if not set, the disk of
the root on the remote. `about` will fail if it does not have
shell access or if `df` is not found.
If the server shell is PowerShell, probably with a Windows OpenSSH
server, rclone supports `about` using a built-in shell command
(see [shell access](#shell-access)).
{{< rem autogenerated options start" - DO NOT EDIT - instead edit fs.RegInfo in backend/sftp/sftp.go then run make backenddocs" >}}
### Standard options
@ -637,25 +764,9 @@ Properties:
## Limitations
SFTP supports checksums if the same login has shell access and `md5sum`
or `sha1sum` as well as `echo` are in the remote's PATH.
This remote checksumming (file hashing) is recommended and enabled by default.
Disabling the checksumming may be required if you are connecting to SFTP servers
which are not under your control, and to which the execution of remote commands
is prohibited. Set the configuration option `disable_hashcheck` to `true` to
disable checksumming.
SFTP also supports `about` if the same login has shell
access and `df` are in the remote's PATH. `about` will
return the total space, free space, and used space on the remote
for the disk of the specified path on the remote or, if not set,
the disk of the root on the remote.
`about` will fail if it does not have shell
access or if `df` is not in the remote's PATH.
Note that some SFTP servers (e.g. Synology) the paths are different for
SSH and SFTP so the hashes can't be calculated properly. For them
using `disable_hashcheck` is a good idea.
On some SFTP servers (e.g. Synology) the paths are different
for SSH and SFTP so the hashes can't be calculated properly.
For them using `disable_hashcheck` is a good idea.
The only ssh agent supported under Windows is Putty's pageant.
@ -670,11 +781,10 @@ SFTP isn't supported under plan9 until [this
issue](https://github.com/pkg/sftp/issues/156) is fixed.
Note that since SFTP isn't HTTP based the following flags don't work
with it: `--dump-headers`, `--dump-bodies`, `--dump-auth`
with it: `--dump-headers`, `--dump-bodies`, `--dump-auth`.
Note that `--timeout` and `--contimeout` are both supported.
## rsync.net {#rsync-net}
rsync.net is supported through the SFTP backend.