webdav: add support for sharepoint with NTLM authentication (#2921)

Add new option option "sharepoint-ntlm" for the vendor setting.
Use it when your hosted Sharepoint is not tied to the OneDrive
accounts and uses NTLM authentication.
Also add documentation and integration test.

Fixes: #2171

backend/webdav/webdav.go

@@ -33,6 +33,8 @@ import (
 	"github.com/rclone/rclone/fs/hash"
 	"github.com/rclone/rclone/lib/pacer"
 	"github.com/rclone/rclone/lib/rest"
+
+	ntlmssp "github.com/Azure/go-ntlmssp"
 )
 
 const (
@@ -67,14 +69,17 @@ func init() {
 				Help:  "Owncloud",
 			}, {
 				Value: "sharepoint",
-				Help:  "Sharepoint",
+				Help:  "Sharepoint Online, authenticated by Microsoft OneDrive account.",
+			}, {
+				Value: "sharepoint-ntlm",
+				Help:  "Sharepoint with NTLM authentication. Usually self-hosted or company instances.",
 			}, {
 				Value: "other",
 				Help:  "Other site/service or software",
 			}},
 		}, {
 			Name: "user",
-			Help: "User name",
+			Help: "User name. In case NTLM authentication is used, the username should be in the format 'Domain\\User'.",
 		}, {
 			Name:       "pass",
 			Help:       "Password.",
@@ -330,13 +335,18 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
 		return nil, err
 	}
 
+	client := fshttp.NewClient(ctx)
+	if opt.Vendor == "sharepoint-ntlm" {
+		// Add NTLM layer
+		client.Transport = ntlmssp.Negotiator{RoundTripper: client.Transport}
+	}
 	f := &Fs{
 		name:        name,
 		root:        root,
 		opt:         *opt,
 		endpoint:    u,
 		endpointURL: u.String(),
-		srv:         rest.NewClient(fshttp.NewClient(ctx)).SetRoot(u.String()),
+		srv:         rest.NewClient(client).SetRoot(u.String()),
 		pacer:       fs.NewPacer(ctx, pacer.NewDefault(pacer.MinSleep(minSleep), pacer.MaxSleep(maxSleep), pacer.DecayConstant(decayConstant))),
 		precision:   fs.ModTimeNotSupported,
 	}
@@ -465,6 +475,10 @@ func (f *Fs) setQuirks(ctx context.Context, vendor string) error {
 		// to determine if we may have found a file, the request has to be resent
 		// with the depth set to 0
 		f.retryWithZeroDepth = true
+	case "sharepoint-ntlm":
+		// Sharepoint with NTLM authentication
+		// See comment above
+		f.retryWithZeroDepth = true
 	case "other":
 	default:
 		fs.Debugf(f, "Unknown vendor %q", vendor)

backend/webdav/webdav_test.go

@@ -38,3 +38,14 @@ func TestIntegration3(t *testing.T) {
 		NilObject:  (*webdav.Object)(nil),
 	})
 }
+
+// TestIntegration runs integration tests against the remote
+func TestIntegration4(t *testing.T) {
+	if *fstest.RemoteName != "" {
+		t.Skip("skipping as -remote is set")
+	}
+	fstests.Run(t, &fstests.Opt{
+		RemoteName: "TestWebdavNTLM:",
+		NilObject:  (*webdav.Object)(nil),
+	})
+}

docs/content/webdav.md

@@ -45,9 +45,11 @@ Choose a number from below, or type in your own value
    \ "nextcloud"
  2 / Owncloud
    \ "owncloud"
- 3 / Sharepoint
+ 3 / Sharepoint Online, authenticated by Microsoft OneDrive account.
    \ "sharepoint"
- 4 / Other site/service or software
+ 4 / Sharepoint with NTLM authentication. Usually self-hosted instances.
+   \ "sharepoint-ntlm"
+ 5 / Other site/service or software
    \ "other"
 vendor> 1
 User name
@@ -136,6 +138,8 @@ Name of the Webdav site/service/software you are using
         - Owncloud
     - "sharepoint"
         - Sharepoint
+    - "sharepoint-ntlm"
+        - Sharepoint with NTLM authentication
     - "other"
         - Other site/service or software
 
@@ -148,6 +152,8 @@ User name
 - Type:        string
 - Default:     ""
 
+In case vendor mode `sharepoint-ntlm` is used, the user name is in the form `DOMAIN\user`
+
 #### --webdav-pass
 
 Password.
@@ -201,7 +207,7 @@ This is configured in an identical way to Owncloud.  Note that
 Nextcloud initially did not support streaming of files (`rcat`) whereas
 Owncloud did, but [this](https://github.com/nextcloud/nextcloud-snap/issues/365) seems to be fixed as of 2020-11-27 (tested with rclone v1.53.1 and Nextcloud Server v19).
 
-### Sharepoint ###
+### Sharepoint OneDrive ###
 
 Rclone can be used with Sharepoint provided by OneDrive for Business
 or Office365 Education Accounts.
@@ -237,11 +243,40 @@ Your config file should look like this:
 [sharepoint]
 type = webdav
 url = https://[YOUR-DOMAIN]-my.sharepoint.com/personal/[YOUR-EMAIL]/Documents
-vendor = other
+vendor = sharepoint
 user = YourEmailAddress
 pass = encryptedpassword
 ```
 
+### Sharepoint with NTLM ###
+
+Use this option in case your (hosted) Sharepoint is not tied to OneDrive accounts and uses NTLM authentication.
+
+For getting the `url` configuration, similarly to the above, first navigate to the desired directory in your browser to get the URL,
+then strip everything after the name of the opened directory.
+
+Example:
+If the URL is:
+https://example.sharepoint.com/sites/12345/Documents/Forms/AllItems.aspx
+
+The configuration to use would be:
+https://example.sharepoint.com/sites/12345/Documents
+
+Set the `vendor` to `sharepoint-ntlm`.
+
+NTLM uses domain and user name combination for authentication,
+set `user` to `DOMAIN\username`.
+
+Your config file should look like this:
+
+```
+[sharepoint]
+type = webdav
+url = https://[YOUR-DOMAIN]/some-path-to/Documents
+vendor = sharepoint-ntlm
+user = DOMAIN\user
+pass = encryptedpassword
+```
 #### Required Flags for SharePoint ####
 As SharePoint does some special things with uploaded documents, you won't be able to use the documents size or the documents hash to compare if a file has been changed since the upload / which file is newer.
 

go.mod

@@ -8,6 +8,7 @@ require (
 	github.com/Azure/azure-pipeline-go v0.2.3
 	github.com/Azure/azure-storage-blob-go v0.13.0
 	github.com/Azure/go-autorest/autorest/adal v0.9.10
+	github.com/Azure/go-ntlmssp v0.0.0-20200615164410-66371956d46c
 	github.com/Microsoft/go-winio v0.4.16 // indirect
 	github.com/Unknwon/goconfig v0.0.0-20200908083735-df7de6a44db8
 	github.com/a8m/tree v0.0.0-20210115125333-10a5fd5b637d

go.sum

@@ -55,6 +55,8 @@ github.com/Azure/go-autorest/autorest/mocks v0.4.1 h1:K0laFcLE6VLTOwNgSxaGbUcLPu
 github.com/Azure/go-autorest/autorest/mocks v0.4.1/go.mod h1:LTp+uSrOhSkaKrUy935gNZuuIPPVsHlr9DSOxSayd+k=
 github.com/Azure/go-autorest/tracing v0.6.0 h1:TYi4+3m5t6K48TGI9AUdb+IzbnSxvnvUMfuitfgcfuo=
 github.com/Azure/go-autorest/tracing v0.6.0/go.mod h1:+vhtPC754Xsa23ID7GlGsrdKBpUA79WCAKPPZVC2DeU=
+github.com/Azure/go-ntlmssp v0.0.0-20200615164410-66371956d46c h1:/IBSNwUN8+eKzUzbJPqhK839ygXJ82sde8x3ogr6R28=
+github.com/Azure/go-ntlmssp v0.0.0-20200615164410-66371956d46c/go.mod h1:chxPXzSsl7ZWRAuOIE23GDNzjWuZquvFlgA8xmpunjU=
 github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ=
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
 github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=