From 29fe0177bda7b46c5a0801c62d80daf92913f680 Mon Sep 17 00:00:00 2001 From: Arnavion Date: Sun, 12 Mar 2023 21:45:54 -0700 Subject: [PATCH] webdav: add "fastmail" provider for Fastmail Files This provider: - supports the `X-OC-Mtime` header to set the mtime - calculates SHA1 checksum server side and returns it as a `ME:sha1hex` prop To differentiate the new hasMESHA1 quirk, the existing hasMD5 and hasSHA1 quirks for Owncloud have been renamed to hasOCMD5 and hasOCSHA1. Fixes #6837 --- README.md | 1 + backend/webdav/api/types.go | 32 ++++++++++++++++----------- backend/webdav/webdav.go | 43 ++++++++++++++++++++++--------------- docs/content/_index.md | 1 + docs/content/overview.md | 4 ++-- docs/content/webdav.md | 38 +++++++++++++++++++++----------- 6 files changed, 74 insertions(+), 45 deletions(-) diff --git a/README.md b/README.md index 9c47e6b77..69d0ff5e2 100644 --- a/README.md +++ b/README.md @@ -37,6 +37,7 @@ Rclone *("rsync for cloud storage")* is a command-line program to sync files and * Dreamhost [:page_facing_up:](https://rclone.org/s3/#dreamhost) * Dropbox [:page_facing_up:](https://rclone.org/dropbox/) * Enterprise File Fabric [:page_facing_up:](https://rclone.org/filefabric/) + * Fastmail Files [:page_facing_up:](https://rclone.org/webdav/#fastmail-files) * FTP [:page_facing_up:](https://rclone.org/ftp/) * Google Cloud Storage [:page_facing_up:](https://rclone.org/googlecloudstorage/) * Google Drive [:page_facing_up:](https://rclone.org/drive/) diff --git a/backend/webdav/api/types.go b/backend/webdav/api/types.go index 7d36f058a..a959a3c9f 100644 --- a/backend/webdav/api/types.go +++ b/backend/webdav/api/types.go @@ -75,6 +75,7 @@ type Prop struct { Size int64 `xml:"DAV: prop>getcontentlength,omitempty"` Modified Time `xml:"DAV: prop>getlastmodified,omitempty"` Checksums []string `xml:"prop>checksums>checksum,omitempty"` + MESha1Hex *string `xml:"ME: prop>sha1hex,omitempty"` // Fastmail-specific sha1 checksum } // Parse a status of the form "HTTP/1.1 200 OK" or "HTTP/1.1 200" @@ -102,22 +103,27 @@ func (p *Prop) StatusOK() bool { // Hashes returns a map of all checksums - may be nil func (p *Prop) Hashes() (hashes map[hash.Type]string) { - if len(p.Checksums) == 0 { - return nil - } - hashes = make(map[hash.Type]string) - for _, checksums := range p.Checksums { - checksums = strings.ToLower(checksums) - for _, checksum := range strings.Split(checksums, " ") { - switch { - case strings.HasPrefix(checksum, "sha1:"): - hashes[hash.SHA1] = checksum[5:] - case strings.HasPrefix(checksum, "md5:"): - hashes[hash.MD5] = checksum[4:] + if len(p.Checksums) > 0 { + hashes = make(map[hash.Type]string) + for _, checksums := range p.Checksums { + checksums = strings.ToLower(checksums) + for _, checksum := range strings.Split(checksums, " ") { + switch { + case strings.HasPrefix(checksum, "sha1:"): + hashes[hash.SHA1] = checksum[5:] + case strings.HasPrefix(checksum, "md5:"): + hashes[hash.MD5] = checksum[4:] + } } } + return hashes + } else if p.MESha1Hex != nil { + hashes = make(map[hash.Type]string) + hashes[hash.SHA1] = *p.MESha1Hex + return hashes + } else { + return nil } - return hashes } // PropValue is a tagged name and value diff --git a/backend/webdav/webdav.go b/backend/webdav/webdav.go index 371d5f71d..ad14aa9c7 100644 --- a/backend/webdav/webdav.go +++ b/backend/webdav/webdav.go @@ -76,6 +76,9 @@ func init() { Name: "vendor", Help: "Name of the WebDAV site/service/software you are using.", Examples: []fs.OptionExample{{ + Value: "fastmail", + Help: "Fastmail Files", + }, { Value: "nextcloud", Help: "Nextcloud", }, { @@ -155,8 +158,9 @@ type Fs struct { useOCMtime bool // set if can use X-OC-Mtime retryWithZeroDepth bool // some vendors (sharepoint) won't list files when Depth is 1 (our default) checkBeforePurge bool // enables extra check that directory to purge really exists - hasMD5 bool // set if can use owncloud style checksums for MD5 - hasSHA1 bool // set if can use owncloud style checksums for SHA1 + hasOCMD5 bool // set if can use owncloud style checksums for MD5 + hasOCSHA1 bool // set if can use owncloud style checksums for SHA1 + hasMESHA1 bool // set if can use fastmail style checksums for SHA1 ntlmAuthMu sync.Mutex // mutex to serialize NTLM auth roundtrips } @@ -278,7 +282,7 @@ func (f *Fs) readMetaDataForPath(ctx context.Context, path string, depth string) }, NoRedirect: true, } - if f.hasMD5 || f.hasSHA1 { + if f.hasOCMD5 || f.hasOCSHA1 { opts.Body = bytes.NewBuffer(owncloudProps) } var result api.Multistatus @@ -546,16 +550,21 @@ func (f *Fs) fetchAndSetBearerToken() error { // setQuirks adjusts the Fs for the vendor passed in func (f *Fs) setQuirks(ctx context.Context, vendor string) error { switch vendor { + case "fastmail": + f.canStream = true + f.precision = time.Second + f.useOCMtime = true + f.hasMESHA1 = true case "owncloud": f.canStream = true f.precision = time.Second f.useOCMtime = true - f.hasMD5 = true - f.hasSHA1 = true + f.hasOCMD5 = true + f.hasOCSHA1 = true case "nextcloud": f.precision = time.Second f.useOCMtime = true - f.hasSHA1 = true + f.hasOCSHA1 = true case "sharepoint": // To mount sharepoint, two Cookies are required // They have to be set instead of BasicAuth @@ -667,7 +676,7 @@ func (f *Fs) listAll(ctx context.Context, dir string, directoriesOnly bool, file "Depth": depth, }, } - if f.hasMD5 || f.hasSHA1 { + if f.hasOCMD5 || f.hasOCSHA1 { opts.Body = bytes.NewBuffer(owncloudProps) } var result api.Multistatus @@ -1126,10 +1135,10 @@ func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string // Hashes returns the supported hash sets. func (f *Fs) Hashes() hash.Set { hashes := hash.Set(hash.None) - if f.hasMD5 { + if f.hasOCMD5 { hashes.Add(hash.MD5) } - if f.hasSHA1 { + if f.hasOCSHA1 || f.hasMESHA1 { hashes.Add(hash.SHA1) } return hashes @@ -1197,10 +1206,10 @@ func (o *Object) Remote() string { // Hash returns the SHA1 or MD5 of an object returning a lowercase hex string func (o *Object) Hash(ctx context.Context, t hash.Type) (string, error) { - if t == hash.MD5 && o.fs.hasMD5 { + if t == hash.MD5 && o.fs.hasOCMD5 { return o.md5, nil } - if t == hash.SHA1 && o.fs.hasSHA1 { + if t == hash.SHA1 && (o.fs.hasOCSHA1 || o.fs.hasMESHA1) { return o.sha1, nil } return "", hash.ErrUnsupported @@ -1222,12 +1231,12 @@ func (o *Object) setMetaData(info *api.Prop) (err error) { o.hasMetaData = true o.size = info.Size o.modTime = time.Time(info.Modified) - if o.fs.hasMD5 || o.fs.hasSHA1 { + if o.fs.hasOCMD5 || o.fs.hasOCSHA1 || o.fs.hasMESHA1 { hashes := info.Hashes() - if o.fs.hasSHA1 { + if o.fs.hasOCSHA1 || o.fs.hasMESHA1 { o.sha1 = hashes[hash.SHA1] } - if o.fs.hasMD5 { + if o.fs.hasOCMD5 { o.md5 = hashes[hash.MD5] } } @@ -1315,7 +1324,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op ContentType: fs.MimeType(ctx, src), Options: options, } - if o.fs.useOCMtime || o.fs.hasMD5 || o.fs.hasSHA1 { + if o.fs.useOCMtime || o.fs.hasOCMD5 || o.fs.hasOCSHA1 { opts.ExtraHeaders = map[string]string{} if o.fs.useOCMtime { opts.ExtraHeaders["X-OC-Mtime"] = fmt.Sprintf("%d", src.ModTime(ctx).Unix()) @@ -1323,12 +1332,12 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op // Set one upload checksum // Owncloud uses one checksum only to check the upload and stores its own SHA1 and MD5 // Nextcloud stores the checksum you supply (SHA1 or MD5) but only stores one - if o.fs.hasSHA1 { + if o.fs.hasOCSHA1 { if sha1, _ := src.Hash(ctx, hash.SHA1); sha1 != "" { opts.ExtraHeaders["OC-Checksum"] = "SHA1:" + sha1 } } - if o.fs.hasMD5 && opts.ExtraHeaders["OC-Checksum"] == "" { + if o.fs.hasOCMD5 && opts.ExtraHeaders["OC-Checksum"] == "" { if md5, _ := src.Hash(ctx, hash.MD5); md5 != "" { opts.ExtraHeaders["OC-Checksum"] = "MD5:" + md5 } diff --git a/docs/content/_index.md b/docs/content/_index.md index 16ea91e1a..963e18baf 100644 --- a/docs/content/_index.md +++ b/docs/content/_index.md @@ -121,6 +121,7 @@ WebDAV or S3, that work out of the box.) {{< provider name="Dreamhost" home="https://www.dreamhost.com/cloud/storage/" config="/s3/#dreamhost" >}} {{< provider name="Dropbox" home="https://www.dropbox.com/" config="/dropbox/" >}} {{< provider name="Enterprise File Fabric" home="https://storagemadeeasy.com/about/" config="/filefabric/" >}} +{{< provider name="Fastmail Files" home="https://www.fastmail.com/" config="/webdav/#fastmail-files" >}} {{< provider name="FTP" home="https://en.wikipedia.org/wiki/File_Transfer_Protocol" config="/ftp/" >}} {{< provider name="Google Cloud Storage" home="https://cloud.google.com/storage/" config="/googlecloudstorage/" >}} {{< provider name="Google Drive" home="https://www.google.com/drive/" config="/drive/" >}} diff --git a/docs/content/overview.md b/docs/content/overview.md index bee55abd4..9198e7af8 100644 --- a/docs/content/overview.md +++ b/docs/content/overview.md @@ -68,9 +68,9 @@ This is an SHA256 sum of all the 4 MiB block SHA256s. ² SFTP supports checksums if the same login has shell access and `md5sum` or `sha1sum` as well as `echo` are in the remote's PATH. -³ WebDAV supports hashes when used with Owncloud and Nextcloud only. +³ WebDAV supports hashes when used with Fastmail Files. Owncloud and Nextcloud only. -⁴ WebDAV supports modtimes when used with Owncloud and Nextcloud only. +⁴ WebDAV supports modtimes when used with Fastmail Files, Owncloud and Nextcloud only. ⁵ [QuickXorHash](https://docs.microsoft.com/en-us/onedrive/developer/code-snippets/quickxorhash) is Microsoft's own hash. diff --git a/docs/content/webdav.md b/docs/content/webdav.md index 4ded05d1b..ee26f8269 100644 --- a/docs/content/webdav.md +++ b/docs/content/webdav.md @@ -43,17 +43,19 @@ Choose a number from below, or type in your own value url> https://example.com/remote.php/webdav/ Name of the WebDAV site/service/software you are using Choose a number from below, or type in your own value - 1 / Nextcloud - \ "nextcloud" - 2 / Owncloud - \ "owncloud" - 3 / Sharepoint Online, authenticated by Microsoft account. - \ "sharepoint" - 4 / Sharepoint with NTLM authentication. Usually self-hosted or on-premises. - \ "sharepoint-ntlm" - 5 / Other site/service or software - \ "other" -vendor> 1 + 1 / Fastmail Files + \ (fastmail) + 2 / Nextcloud + \ (nextcloud) + 3 / Owncloud + \ (owncloud) + 4 / Sharepoint Online, authenticated by Microsoft account + \ (sharepoint) + 5 / Sharepoint with NTLM authentication, usually self-hosted or on-premises + \ (sharepoint-ntlm) + 6 / Other site/service or software + \ (other) +vendor> 2 User name user> user Password. @@ -100,10 +102,10 @@ To copy a local directory to an WebDAV directory called backup ### Modified time and hashes ### Plain WebDAV does not support modified times. However when used with -Owncloud or Nextcloud rclone will support modified times. +Fastmail Files, Owncloud or Nextcloud rclone will support modified times. Likewise plain WebDAV does not support hashes, however when used with -Owncloud or Nextcloud rclone will support SHA1 and MD5 hashes. +Fastmail Files, Owncloud or Nextcloud rclone will support SHA1 and MD5 hashes. Depending on the exact version of Owncloud or Nextcloud hashes may appear on all objects, or only on objects which had a hash uploaded with them. @@ -242,6 +244,16 @@ Properties: See below for notes on specific providers. +## Fastmail Files + +Use `https://webdav.fastmail.com/` or a subdirectory as the URL, +and your Fastmail email `username@domain.tld` as the username. +Follow [this documentation](https://www.fastmail.help/hc/en-us/articles/360058752854-App-passwords) +to create an app password with access to `Files (WebDAV)` and use +this as the password. + +Fastmail supports modified times using the `X-OC-Mtime` header. + ### Owncloud Click on the settings cog in the bottom right of the page and this