backends: move encoding definitions from fs/encodings

backend/amazonclouddrive/amazonclouddrive.go

@@ -28,7 +28,6 @@ import (
 	"github.com/rclone/rclone/fs/config"
 	"github.com/rclone/rclone/fs/config/configmap"
 	"github.com/rclone/rclone/fs/config/configstruct"
-	"github.com/rclone/rclone/fs/encodings"
 	"github.com/rclone/rclone/fs/fserrors"
 	"github.com/rclone/rclone/fs/fshttp"
 	"github.com/rclone/rclone/fs/hash"
@@ -141,7 +140,9 @@ underlying S3 storage.`,
 			Name:     config.ConfigEncoding,
 			Help:     config.ConfigEncodingHelp,
 			Advanced: true,
-			Default:  encodings.AmazonCloudDrive,
+			// Encode invalid UTF-8 bytes as json doesn't handle them properly.
+			Default: (encoder.Base |
+				encoder.EncodeInvalidUtf8),
 		}},
 	})
 }

backend/azureblob/azureblob.go

@@ -29,7 +29,6 @@ import (
 	"github.com/rclone/rclone/fs/config"
 	"github.com/rclone/rclone/fs/config/configmap"
 	"github.com/rclone/rclone/fs/config/configstruct"
-	"github.com/rclone/rclone/fs/encodings"
 	"github.com/rclone/rclone/fs/fserrors"
 	"github.com/rclone/rclone/fs/fshttp"
 	"github.com/rclone/rclone/fs/hash"
@@ -131,7 +130,12 @@ tiering blob to "Hot" or "Cool".`,
 			Name:     config.ConfigEncoding,
 			Help:     config.ConfigEncodingHelp,
 			Advanced: true,
-			Default:  encodings.AzureBlob,
+			Default: (encoder.EncodeInvalidUtf8 |
+				encoder.EncodeSlash |
+				encoder.EncodeCtl |
+				encoder.EncodeDel |
+				encoder.EncodeBackSlash |
+				encoder.EncodeRightPeriod),
 		}},
 	})
 }

backend/b2/b2.go

@@ -26,7 +26,6 @@ import (
 	"github.com/rclone/rclone/fs/config"
 	"github.com/rclone/rclone/fs/config/configmap"
 	"github.com/rclone/rclone/fs/config/configstruct"
-	"github.com/rclone/rclone/fs/encodings"
 	"github.com/rclone/rclone/fs/fserrors"
 	"github.com/rclone/rclone/fs/fshttp"
 	"github.com/rclone/rclone/fs/hash"
@@ -150,7 +149,12 @@ The minimum value is 1 second. The maximum value is one week.`,
 			Name:     config.ConfigEncoding,
 			Help:     config.ConfigEncodingHelp,
 			Advanced: true,
-			Default:  encodings.B2,
+			// See: https://www.backblaze.com/b2/docs/files.html
+			// Encode invalid UTF-8 bytes as json doesn't handle them properly.
+			// FIXME: allow /, but not leading, trailing or double
+			Default: (encoder.Display |
+				encoder.EncodeBackSlash |
+				encoder.EncodeInvalidUtf8),
 		}},
 	})
 }

backend/box/box.go

@@ -37,7 +37,6 @@ import (
 	"github.com/rclone/rclone/fs/config/configmap"
 	"github.com/rclone/rclone/fs/config/configstruct"
 	"github.com/rclone/rclone/fs/config/obscure"
-	"github.com/rclone/rclone/fs/encodings"
 	"github.com/rclone/rclone/fs/fserrors"
 	"github.com/rclone/rclone/fs/fshttp"
 	"github.com/rclone/rclone/fs/hash"
@@ -149,7 +148,17 @@ func init() {
 			Name:     config.ConfigEncoding,
 			Help:     config.ConfigEncodingHelp,
 			Advanced: true,
-			Default:  encodings.Box,
+			// From https://developer.box.com/docs/error-codes#section-400-bad-request :
+			// > Box only supports file or folder names that are 255 characters or less.
+			// > File names containing non-printable ascii, "/" or "\", names with leading
+			// > or trailing spaces, and the special names “.” and “..” are also unsupported.
+			//
+			// Testing revealed names with leading spaces work fine.
+			// Also encode invalid UTF-8 bytes as json doesn't handle them properly.
+			Default: (encoder.Display |
+				encoder.EncodeBackSlash |
+				encoder.EncodeRightSpace |
+				encoder.EncodeInvalidUtf8),
 		}},
 	})
 }

backend/drive/drive.go

@@ -33,7 +33,6 @@ import (
 	"github.com/rclone/rclone/fs/config/configmap"
 	"github.com/rclone/rclone/fs/config/configstruct"
 	"github.com/rclone/rclone/fs/config/obscure"
-	"github.com/rclone/rclone/fs/encodings"
 	"github.com/rclone/rclone/fs/fserrors"
 	"github.com/rclone/rclone/fs/fshttp"
 	"github.com/rclone/rclone/fs/hash"
@@ -459,7 +458,9 @@ See: https://github.com/rclone/rclone/issues/3857
 			Name:     config.ConfigEncoding,
 			Help:     config.ConfigEncodingHelp,
 			Advanced: true,
-			Default:  encodings.Drive,
+			// Encode invalid UTF-8 bytes as json doesn't handle them properly.
+			// Don't encode / as it's a valid name character in drive.
+			Default: encoder.EncodeInvalidUtf8,
 		}},
 	})
 

backend/dropbox/dropbox.go

@@ -45,7 +45,6 @@ import (
 	"github.com/rclone/rclone/fs/config/configmap"
 	"github.com/rclone/rclone/fs/config/configstruct"
 	"github.com/rclone/rclone/fs/config/obscure"
-	"github.com/rclone/rclone/fs/encodings"
 	"github.com/rclone/rclone/fs/fserrors"
 	"github.com/rclone/rclone/fs/hash"
 	"github.com/rclone/rclone/lib/encoder"
@@ -150,7 +149,15 @@ memory.  It can be set smaller if you are tight on memory.`, maxChunkSize),
 			Name:     config.ConfigEncoding,
 			Help:     config.ConfigEncodingHelp,
 			Advanced: true,
-			Default:  encodings.Dropbox,
+			// https://www.dropbox.com/help/syncing-uploads/files-not-syncing lists / and \
+			// as invalid characters.
+			// Testing revealed names with trailing spaces and the DEL character don't work.
+			// Also encode invalid UTF-8 bytes as json doesn't handle them properly.
+			Default: (encoder.Base |
+				encoder.EncodeBackSlash |
+				encoder.EncodeDel |
+				encoder.EncodeRightSpace |
+				encoder.EncodeInvalidUtf8),
 		}},
 	})
 }

backend/fichier/fichier.go

@@ -14,7 +14,6 @@ import (
 	"github.com/rclone/rclone/fs/config"
 	"github.com/rclone/rclone/fs/config/configmap"
 	"github.com/rclone/rclone/fs/config/configstruct"
-	"github.com/rclone/rclone/fs/encodings"
 	"github.com/rclone/rclone/fs/fshttp"
 	"github.com/rclone/rclone/fs/hash"
 	"github.com/rclone/rclone/lib/dircache"
@@ -50,7 +49,27 @@ func init() {
 			Name:     config.ConfigEncoding,
 			Help:     config.ConfigEncodingHelp,
 			Advanced: true,
-			Default:  encodings.Fichier,
+			// Characters that need escaping
+			//
+			// 		'\\': '＼', // FULLWIDTH REVERSE SOLIDUS
+			// 		'<':  '＜', // FULLWIDTH LESS-THAN SIGN
+			// 		'>':  '＞', // FULLWIDTH GREATER-THAN SIGN
+			// 		'"':  '＂', // FULLWIDTH QUOTATION MARK - not on the list but seems to be reserved
+			// 		'\'': '＇', // FULLWIDTH APOSTROPHE
+			// 		'$':  '＄', // FULLWIDTH DOLLAR SIGN
+			// 		'`':  '｀', // FULLWIDTH GRAVE ACCENT
+			//
+			// Leading space and trailing space
+			Default: (encoder.Display |
+				encoder.EncodeBackSlash |
+				encoder.EncodeSingleQuote |
+				encoder.EncodeBackQuote |
+				encoder.EncodeDoubleQuote |
+				encoder.EncodeLtGt |
+				encoder.EncodeDollar |
+				encoder.EncodeLeftSpace |
+				encoder.EncodeRightSpace |
+				encoder.EncodeInvalidUtf8),
 		}},
 	})
 }

backend/ftp/ftp.go

@@ -18,7 +18,6 @@ import (
 	"github.com/rclone/rclone/fs/config/configmap"
 	"github.com/rclone/rclone/fs/config/configstruct"
 	"github.com/rclone/rclone/fs/config/obscure"
-	"github.com/rclone/rclone/fs/encodings"
 	"github.com/rclone/rclone/fs/hash"
 	"github.com/rclone/rclone/lib/encoder"
 	"github.com/rclone/rclone/lib/pacer"
@@ -73,7 +72,13 @@ func init() {
 			Name:     config.ConfigEncoding,
 			Help:     config.ConfigEncodingHelp,
 			Advanced: true,
-			Default:  encodings.FTP,
+			// The FTP protocal can't handle trailing spaces (for instance
+			// pureftpd turns them into _)
+			//
+			// proftpd can't handle '*' in file names
+			// pureftpd can't handle '[', ']' or '*'
+			Default: (encoder.Display |
+				encoder.EncodeRightSpace),
 		}},
 	})
 }

backend/googlecloudstorage/googlecloudstorage.go

@@ -32,7 +32,6 @@ import (
 	"github.com/rclone/rclone/fs/config/configmap"
 	"github.com/rclone/rclone/fs/config/configstruct"
 	"github.com/rclone/rclone/fs/config/obscure"
-	"github.com/rclone/rclone/fs/encodings"
 	"github.com/rclone/rclone/fs/fserrors"
 	"github.com/rclone/rclone/fs/fshttp"
 	"github.com/rclone/rclone/fs/hash"
@@ -251,7 +250,9 @@ Docs: https://cloud.google.com/storage/docs/bucket-policy-only
 			Name:     config.ConfigEncoding,
 			Help:     config.ConfigEncodingHelp,
 			Advanced: true,
-			Default:  encodings.GoogleCloudStorage,
+			Default: (encoder.Base |
+				encoder.EncodeCrLf |
+				encoder.EncodeInvalidUtf8),
 		}},
 	})
 }

backend/jottacloud/jottacloud.go

@@ -26,7 +26,6 @@ import (
 	"github.com/rclone/rclone/fs/config"
 	"github.com/rclone/rclone/fs/config/configmap"
 	"github.com/rclone/rclone/fs/config/configstruct"
-	"github.com/rclone/rclone/fs/encodings"
 	"github.com/rclone/rclone/fs/fserrors"
 	"github.com/rclone/rclone/fs/fshttp"
 	"github.com/rclone/rclone/fs/hash"
@@ -160,7 +159,12 @@ func init() {
 			Name:     config.ConfigEncoding,
 			Help:     config.ConfigEncodingHelp,
 			Advanced: true,
-			Default:  encodings.JottaCloud,
+			// Encode invalid UTF-8 bytes as xml doesn't handle them properly.
+			//
+			// Also: '*', '/', ':', '<', '>', '?', '\"', '\x00', '|'
+			Default: (encoder.Display |
+				encoder.EncodeWin | // :?"*<>|
+				encoder.EncodeInvalidUtf8),
 		}},
 	})
 }

backend/koofr/koofr.go

@@ -16,7 +16,6 @@ import (
 	"github.com/rclone/rclone/fs/config/configmap"
 	"github.com/rclone/rclone/fs/config/configstruct"
 	"github.com/rclone/rclone/fs/config/obscure"
-	"github.com/rclone/rclone/fs/encodings"
 	"github.com/rclone/rclone/fs/fshttp"
 	"github.com/rclone/rclone/fs/hash"
 	"github.com/rclone/rclone/lib/encoder"
@@ -62,7 +61,10 @@ func init() {
 			Name:     config.ConfigEncoding,
 			Help:     config.ConfigEncodingHelp,
 			Advanced: true,
-			Default:  encodings.Koofr,
+			// Encode invalid UTF-8 bytes as json doesn't handle them properly.
+			Default: (encoder.Display |
+				encoder.EncodeBackSlash |
+				encoder.EncodeInvalidUtf8),
 		}},
 	})
 }

backend/local/encode_darwin.go

@@ -2,8 +2,10 @@
 
 package local
 
-import (
-	"github.com/rclone/rclone/fs/encodings"
-)
+import "github.com/rclone/rclone/lib/encoder"
 
-const defaultEnc = encodings.LocalMacOS
+// This is the encoding used by the local backend for macOS
+//
+// macOS can't store invalid UTF-8, it converts them into %XX encoding
+const defaultEnc = (encoder.Base |
+	encoder.EncodeInvalidUtf8)

backend/local/encode_other.go

@@ -2,8 +2,7 @@
 
 package local
 
-import (
-	"github.com/rclone/rclone/fs/encodings"
-)
+import "github.com/rclone/rclone/lib/encoder"
 
-const defaultEnc = encodings.LocalUnix
+// This is the encoding used by the local backend for non windows platforms
+const defaultEnc = encoder.Base

backend/local/encode_windows.go

@@ -2,8 +2,32 @@
 
 package local
 
-import (
-	"github.com/rclone/rclone/fs/encodings"
-)
+import "github.com/rclone/rclone/lib/encoder"
 
-const defaultEnc = encodings.LocalWindows
+// This is the encoding used by the local backend for windows platforms
+//
+// List of replaced characters:
+//   < (less than)     -> '＜' // FULLWIDTH LESS-THAN SIGN
+//   > (greater than)  -> '＞' // FULLWIDTH GREATER-THAN SIGN
+//   : (colon)         -> '：' // FULLWIDTH COLON
+//   " (double quote)  -> '＂' // FULLWIDTH QUOTATION MARK
+//   \ (backslash)     -> '＼' // FULLWIDTH REVERSE SOLIDUS
+//   | (vertical line) -> '｜' // FULLWIDTH VERTICAL LINE
+//   ? (question mark) -> '？' // FULLWIDTH QUESTION MARK
+//   * (asterisk)      -> '＊' // FULLWIDTH ASTERISK
+//
+// Additionally names can't end with a period (.) or space ( ).
+// List of replaced characters:
+//   . (period)        -> '．' // FULLWIDTH FULL STOP
+//     (space)         -> '␠' // SYMBOL FOR SPACE
+//
+// Also encode invalid UTF-8 bytes as Go can't convert them to UTF-16.
+//
+// https://docs.microsoft.com/de-de/windows/desktop/FileIO/naming-a-file#naming-conventions
+const defaultEnc = (encoder.Base |
+	encoder.EncodeWin |
+	encoder.EncodeBackSlash |
+	encoder.EncodeCtl |
+	encoder.EncodeRightSpace |
+	encoder.EncodeRightPeriod |
+	encoder.EncodeInvalidUtf8)

backend/mailru/mailru.go

@@ -28,7 +28,6 @@ import (
 	"github.com/rclone/rclone/fs/config/configmap"
 	"github.com/rclone/rclone/fs/config/configstruct"
 	"github.com/rclone/rclone/fs/config/obscure"
-	"github.com/rclone/rclone/fs/encodings"
 	"github.com/rclone/rclone/fs/fserrors"
 	"github.com/rclone/rclone/fs/fshttp"
 	"github.com/rclone/rclone/fs/hash"
@@ -197,7 +196,11 @@ Supported quirks: atomicmkdir binlist gzip insecure retry400`,
 			Name:     config.ConfigEncoding,
 			Help:     config.ConfigEncodingHelp,
 			Advanced: true,
-			Default:  encodings.Mailru,
+			// Encode invalid UTF-8 bytes as json doesn't handle them properly.
+			Default: (encoder.Display |
+				encoder.EncodeWin | // :?"*<>|
+				encoder.EncodeBackSlash |
+				encoder.EncodeInvalidUtf8),
 		}},
 	})
 }

backend/mega/mega.go

@@ -30,7 +30,6 @@ import (
 	"github.com/rclone/rclone/fs/config/configmap"
 	"github.com/rclone/rclone/fs/config/configstruct"
 	"github.com/rclone/rclone/fs/config/obscure"
-	"github.com/rclone/rclone/fs/encodings"
 	"github.com/rclone/rclone/fs/fshttp"
 	"github.com/rclone/rclone/fs/hash"
 	"github.com/rclone/rclone/lib/encoder"
@@ -87,7 +86,9 @@ permanently delete objects instead.`,
 			Name:     config.ConfigEncoding,
 			Help:     config.ConfigEncodingHelp,
 			Advanced: true,
-			Default:  encodings.Mega,
+			// Encode invalid UTF-8 bytes as json doesn't handle them properly.
+			Default: (encoder.Base |
+				encoder.EncodeInvalidUtf8),
 		}},
 	})
 }

backend/onedrive/onedrive.go

@@ -23,7 +23,6 @@ import (
 	"github.com/rclone/rclone/fs/config/configmap"
 	"github.com/rclone/rclone/fs/config/configstruct"
 	"github.com/rclone/rclone/fs/config/obscure"
-	"github.com/rclone/rclone/fs/encodings"
 	"github.com/rclone/rclone/fs/fserrors"
 	"github.com/rclone/rclone/fs/hash"
 	"github.com/rclone/rclone/lib/atexit"
@@ -255,7 +254,48 @@ listing, set this option.`,
 			Name:     config.ConfigEncoding,
 			Help:     config.ConfigEncodingHelp,
 			Advanced: true,
-			Default:  encodings.OneDrive,
+			// List of replaced characters:
+			//   < (less than)     -> '＜' // FULLWIDTH LESS-THAN SIGN
+			//   > (greater than)  -> '＞' // FULLWIDTH GREATER-THAN SIGN
+			//   : (colon)         -> '：' // FULLWIDTH COLON
+			//   " (double quote)  -> '＂' // FULLWIDTH QUOTATION MARK
+			//   \ (backslash)     -> '＼' // FULLWIDTH REVERSE SOLIDUS
+			//   | (vertical line) -> '｜' // FULLWIDTH VERTICAL LINE
+			//   ? (question mark) -> '？' // FULLWIDTH QUESTION MARK
+			//   * (asterisk)      -> '＊' // FULLWIDTH ASTERISK
+			//   # (number sign)  -> '＃'  // FULLWIDTH NUMBER SIGN
+			//   % (percent sign) -> '％'  // FULLWIDTH PERCENT SIGN
+			//
+			// Folder names cannot begin with a tilde ('~')
+			// List of replaced characters:
+			//   ~ (tilde)        -> '～'  // FULLWIDTH TILDE
+			//
+			// Additionally names can't begin with a space ( ) or end with a period (.) or space ( ).
+			// List of replaced characters:
+			//   . (period)        -> '．' // FULLWIDTH FULL STOP
+			//     (space)         -> '␠'  // SYMBOL FOR SPACE
+			//
+			// Also encode invalid UTF-8 bytes as json doesn't handle them.
+			//
+			// The OneDrive API documentation lists the set of reserved characters, but
+			// testing showed this list is incomplete. This are the differences:
+			//  - " (double quote) is rejected, but missing in the documentation
+			//  - space at the end of file and folder names is rejected, but missing in the documentation
+			//  - period at the end of file names is rejected, but missing in the documentation
+			//
+			// Adding these restrictions to the OneDrive API documentation yields exactly
+			// the same rules as the Windows naming conventions.
+			//
+			// https://docs.microsoft.com/en-us/onedrive/developer/rest-api/concepts/addressing-driveitems?view=odsp-graph-online#path-encoding
+			Default: (encoder.Display |
+				encoder.EncodeBackSlash |
+				encoder.EncodeHashPercent |
+				encoder.EncodeLeftSpace |
+				encoder.EncodeLeftTilde |
+				encoder.EncodeRightPeriod |
+				encoder.EncodeRightSpace |
+				encoder.EncodeWin |
+				encoder.EncodeInvalidUtf8),
 		}},
 	})
 }

backend/opendrive/opendrive.go

@@ -17,7 +17,6 @@ import (
 	"github.com/rclone/rclone/fs/config/configmap"
 	"github.com/rclone/rclone/fs/config/configstruct"
 	"github.com/rclone/rclone/fs/config/obscure"
-	"github.com/rclone/rclone/fs/encodings"
 	"github.com/rclone/rclone/fs/fserrors"
 	"github.com/rclone/rclone/fs/fshttp"
 	"github.com/rclone/rclone/fs/hash"
@@ -54,7 +53,35 @@ func init() {
 			Name:     config.ConfigEncoding,
 			Help:     config.ConfigEncodingHelp,
 			Advanced: true,
-			Default:  encodings.OpenDrive,
+			// List of replaced characters:
+			//   < (less than)     -> '＜' // FULLWIDTH LESS-THAN SIGN
+			//   > (greater than)  -> '＞' // FULLWIDTH GREATER-THAN SIGN
+			//   : (colon)         -> '：' // FULLWIDTH COLON
+			//   " (double quote)  -> '＂' // FULLWIDTH QUOTATION MARK
+			//   \ (backslash)     -> '＼' // FULLWIDTH REVERSE SOLIDUS
+			//   | (vertical line) -> '｜' // FULLWIDTH VERTICAL LINE
+			//   ? (question mark) -> '？' // FULLWIDTH QUESTION MARK
+			//   * (asterisk)      -> '＊' // FULLWIDTH ASTERISK
+			//
+			// Additionally names can't begin or end with a ASCII whitespace.
+			// List of replaced characters:
+			//     (space)           -> '␠'  // SYMBOL FOR SPACE
+			//     (horizontal tab)  -> '␉'  // SYMBOL FOR HORIZONTAL TABULATION
+			//     (line feed)       -> '␊'  // SYMBOL FOR LINE FEED
+			//     (vertical tab)    -> '␋'  // SYMBOL FOR VERTICAL TABULATION
+			//     (carriage return) -> '␍'  // SYMBOL FOR CARRIAGE RETURN
+			//
+			// Also encode invalid UTF-8 bytes as json doesn't handle them properly.
+			//
+			// https://www.opendrive.com/wp-content/uploads/guides/OpenDrive_API_guide.pdf
+			Default: (encoder.Base |
+				encoder.EncodeWin |
+				encoder.EncodeLeftCrLfHtVt |
+				encoder.EncodeRightCrLfHtVt |
+				encoder.EncodeBackSlash |
+				encoder.EncodeLeftSpace |
+				encoder.EncodeRightSpace |
+				encoder.EncodeInvalidUtf8),
 		}},
 	})
 }

backend/pcloud/pcloud.go

@@ -26,7 +26,6 @@ import (
 	"github.com/rclone/rclone/fs/config/configmap"
 	"github.com/rclone/rclone/fs/config/configstruct"
 	"github.com/rclone/rclone/fs/config/obscure"
-	"github.com/rclone/rclone/fs/encodings"
 	"github.com/rclone/rclone/fs/fserrors"
 	"github.com/rclone/rclone/fs/hash"
 	"github.com/rclone/rclone/lib/dircache"
@@ -84,7 +83,12 @@ func init() {
 			Name:     config.ConfigEncoding,
 			Help:     config.ConfigEncodingHelp,
 			Advanced: true,
-			Default:  encodings.Pcloud,
+			// Encode invalid UTF-8 bytes as json doesn't handle them properly.
+			//
+			// TODO: Investigate Unicode simplification (＼ gets converted to \ server-side)
+			Default: (encoder.Display |
+				encoder.EncodeBackSlash |
+				encoder.EncodeInvalidUtf8),
 		}},
 	})
 }

backend/premiumizeme/premiumizeme.go

@@ -35,7 +35,6 @@ import (
 	"github.com/rclone/rclone/fs/config/configmap"
 	"github.com/rclone/rclone/fs/config/configstruct"
 	"github.com/rclone/rclone/fs/config/obscure"
-	"github.com/rclone/rclone/fs/encodings"
 	"github.com/rclone/rclone/fs/fserrors"
 	"github.com/rclone/rclone/fs/fshttp"
 	"github.com/rclone/rclone/fs/hash"
@@ -97,7 +96,11 @@ This is not normally used - use oauth instead.
 			Name:     config.ConfigEncoding,
 			Help:     config.ConfigEncodingHelp,
 			Advanced: true,
-			Default:  encodings.PremiumizeMe,
+			// Encode invalid UTF-8 bytes as json doesn't handle them properly.
+			Default: (encoder.Display |
+				encoder.EncodeBackSlash |
+				encoder.EncodeDoubleQuote |
+				encoder.EncodeInvalidUtf8),
 		}},
 	})
 }

backend/putio/putio.go

@@ -9,7 +9,6 @@ import (
 	"github.com/rclone/rclone/fs/config"
 	"github.com/rclone/rclone/fs/config/configmap"
 	"github.com/rclone/rclone/fs/config/obscure"
-	"github.com/rclone/rclone/fs/encodings"
 	"github.com/rclone/rclone/lib/dircache"
 	"github.com/rclone/rclone/lib/encoder"
 	"github.com/rclone/rclone/lib/oauthutil"
@@ -70,7 +69,12 @@ func init() {
 			Name:     config.ConfigEncoding,
 			Help:     config.ConfigEncodingHelp,
 			Advanced: true,
-			Default:  encodings.Putio,
+			// Note that \ is renamed to -
+			//
+			// Encode invalid UTF-8 bytes as json doesn't handle them properly.
+			Default: (encoder.Display |
+				encoder.EncodeBackSlash |
+				encoder.EncodeInvalidUtf8),
 		}},
 	})
 }

backend/qingstor/qingstor.go

@@ -21,7 +21,6 @@ import (
 	"github.com/rclone/rclone/fs/config"
 	"github.com/rclone/rclone/fs/config/configmap"
 	"github.com/rclone/rclone/fs/config/configstruct"
-	"github.com/rclone/rclone/fs/encodings"
 	"github.com/rclone/rclone/fs/fshttp"
 	"github.com/rclone/rclone/fs/hash"
 	"github.com/rclone/rclone/fs/walk"
@@ -117,7 +116,9 @@ this may help to speed up the transfers.`,
 			Name:     config.ConfigEncoding,
 			Help:     config.ConfigEncodingHelp,
 			Advanced: true,
-			Default:  encodings.QingStor,
+			Default: (encoder.EncodeInvalidUtf8 |
+				encoder.EncodeCtl |
+				encoder.EncodeSlash),
 		}},
 	})
 }

backend/s3/s3.go

@@ -49,7 +49,6 @@ import (
 	"github.com/rclone/rclone/fs/config"
 	"github.com/rclone/rclone/fs/config/configmap"
 	"github.com/rclone/rclone/fs/config/configstruct"
-	"github.com/rclone/rclone/fs/encodings"
 	"github.com/rclone/rclone/fs/fserrors"
 	"github.com/rclone/rclone/fs/fshttp"
 	"github.com/rclone/rclone/fs/hash"
@@ -815,7 +814,19 @@ In Ceph, this can be increased with the "rgw list buckets max chunk" option.
 			Name:     config.ConfigEncoding,
 			Help:     config.ConfigEncodingHelp,
 			Advanced: true,
-			Default:  encodings.S3,
+			// Any UTF-8 character is valid in a key, however it can't handle
+			// invalid UTF-8 and / have a special meaning.
+			//
+			// The SDK can't seem to handle uploading files called '.'
+			//
+			// FIXME would be nice to add
+			// - initial / encoding
+			// - doubled / encoding
+			// - trailing / encoding
+			// so that AWS keys are always valid file names
+			Default: (encoder.EncodeInvalidUtf8 |
+				encoder.EncodeSlash |
+				encoder.EncodeDot),
 		}},
 	})
 }

backend/sharefile/sharefile.go

@@ -91,7 +91,6 @@ import (
 	"github.com/rclone/rclone/fs/config/configmap"
 	"github.com/rclone/rclone/fs/config/configstruct"
 	"github.com/rclone/rclone/fs/config/obscure"
-	"github.com/rclone/rclone/fs/encodings"
 	"github.com/rclone/rclone/fs/fserrors"
 	"github.com/rclone/rclone/fs/hash"
 	"github.com/rclone/rclone/lib/dircache"
@@ -208,7 +207,15 @@ be set manually to something like: https://XXX.sharefile.com
 			Name:     config.ConfigEncoding,
 			Help:     config.ConfigEncodingHelp,
 			Advanced: true,
-			Default:  encodings.Sharefile,
+			Default: (encoder.Base |
+				encoder.EncodeWin | // :?"*<>|
+				encoder.EncodeBackSlash | // \
+				encoder.EncodeCtl |
+				encoder.EncodeRightSpace |
+				encoder.EncodeRightPeriod |
+				encoder.EncodeLeftSpace |
+				encoder.EncodeLeftPeriod |
+				encoder.EncodeInvalidUtf8),
 		}},
 	})
 }

backend/swift/swift.go

@@ -19,7 +19,6 @@ import (
 	"github.com/rclone/rclone/fs/config"
 	"github.com/rclone/rclone/fs/config/configmap"
 	"github.com/rclone/rclone/fs/config/configstruct"
-	"github.com/rclone/rclone/fs/encodings"
 	"github.com/rclone/rclone/fs/fserrors"
 	"github.com/rclone/rclone/fs/fshttp"
 	"github.com/rclone/rclone/fs/hash"
@@ -66,7 +65,8 @@ copy operations.`,
 	Name:     config.ConfigEncoding,
 	Help:     config.ConfigEncodingHelp,
 	Advanced: true,
-	Default:  encodings.Swift,
+	Default: (encoder.EncodeInvalidUtf8 |
+		encoder.EncodeSlash),
 }}
 
 // Register with Fs

backend/yandex/yandex.go

@@ -20,7 +20,6 @@ import (
 	"github.com/rclone/rclone/fs/config/configmap"
 	"github.com/rclone/rclone/fs/config/configstruct"
 	"github.com/rclone/rclone/fs/config/obscure"
-	"github.com/rclone/rclone/fs/encodings"
 	"github.com/rclone/rclone/fs/fserrors"
 	"github.com/rclone/rclone/fs/hash"
 	"github.com/rclone/rclone/lib/encoder"
@@ -83,7 +82,10 @@ func init() {
 			Name:     config.ConfigEncoding,
 			Help:     config.ConfigEncodingHelp,
 			Advanced: true,
-			Default:  encodings.Yandex,
+			// Of the control characters \t \n \r are allowed
+			// it doesn't seem worth making an exception for this
+			Default: (encoder.Display |
+				encoder.EncodeInvalidUtf8),
 		}},
 	})
 }

fs/encodings/encodings.go

@@ -1,419 +0,0 @@
-// +build !noencode
-
-package encodings
-
-import (
-	"runtime"
-	"strings"
-
-	"github.com/rclone/rclone/lib/encoder"
-)
-
-// LocalUnix is the encoding used by the local backend for non windows platforms
-const LocalUnix = encoder.Base
-
-// LocalMacOS is the encoding used by the local backend for macOS
-//
-// macOS can't store invalid UTF-8, it converts them into %XX encoding
-const LocalMacOS = (encoder.Base |
-	encoder.EncodeInvalidUtf8)
-
-// LocalWindows is the encoding used by the local backend for windows platforms
-//
-// List of replaced characters:
-//   < (less than)     -> '＜' // FULLWIDTH LESS-THAN SIGN
-//   > (greater than)  -> '＞' // FULLWIDTH GREATER-THAN SIGN
-//   : (colon)         -> '：' // FULLWIDTH COLON
-//   " (double quote)  -> '＂' // FULLWIDTH QUOTATION MARK
-//   \ (backslash)     -> '＼' // FULLWIDTH REVERSE SOLIDUS
-//   | (vertical line) -> '｜' // FULLWIDTH VERTICAL LINE
-//   ? (question mark) -> '？' // FULLWIDTH QUESTION MARK
-//   * (asterisk)      -> '＊' // FULLWIDTH ASTERISK
-//
-// Additionally names can't end with a period (.) or space ( ).
-// List of replaced characters:
-//   . (period)        -> '．' // FULLWIDTH FULL STOP
-//     (space)         -> '␠' // SYMBOL FOR SPACE
-//
-// Also encode invalid UTF-8 bytes as Go can't convert them to UTF-16.
-//
-// https://docs.microsoft.com/de-de/windows/desktop/FileIO/naming-a-file#naming-conventions
-const LocalWindows = (encoder.Base |
-	encoder.EncodeWin |
-	encoder.EncodeBackSlash |
-	encoder.EncodeCtl |
-	encoder.EncodeRightSpace |
-	encoder.EncodeRightPeriod |
-	encoder.EncodeInvalidUtf8)
-
-// AmazonCloudDrive is the encoding used by the amazonclouddrive backend
-//
-// Encode invalid UTF-8 bytes as json doesn't handle them properly.
-const AmazonCloudDrive = (encoder.Base |
-	encoder.EncodeInvalidUtf8)
-
-// B2 is the encoding used by the b2 backend
-//
-// See: https://www.backblaze.com/b2/docs/files.html
-// Encode invalid UTF-8 bytes as json doesn't handle them properly.
-// FIXME: allow /, but not leading, trailing or double
-const B2 = (encoder.Display |
-	encoder.EncodeBackSlash |
-	encoder.EncodeInvalidUtf8)
-
-// Box is the encoding used by the box backend
-//
-// From https://developer.box.com/docs/error-codes#section-400-bad-request :
-// > Box only supports file or folder names that are 255 characters or less.
-// > File names containing non-printable ascii, "/" or "\", names with leading
-// > or trailing spaces, and the special names “.” and “..” are also unsupported.
-//
-// Testing revealed names with leading spaces work fine.
-// Also encode invalid UTF-8 bytes as json doesn't handle them properly.
-const Box = (encoder.Display |
-	encoder.EncodeBackSlash |
-	encoder.EncodeRightSpace |
-	encoder.EncodeInvalidUtf8)
-
-// Drive is the encoding used by the drive backend
-//
-// Encode invalid UTF-8 bytes as json doesn't handle them properly.
-// Don't encode / as it's a valid name character in drive.
-const Drive = (encoder.EncodeInvalidUtf8)
-
-// Dropbox is the encoding used by the dropbox backend
-//
-// https://www.dropbox.com/help/syncing-uploads/files-not-syncing lists / and \
-// as invalid characters.
-// Testing revealed names with trailing spaces and the DEL character don't work.
-// Also encode invalid UTF-8 bytes as json doesn't handle them properly.
-const Dropbox = (encoder.Base |
-	encoder.EncodeBackSlash |
-	encoder.EncodeDel |
-	encoder.EncodeRightSpace |
-	encoder.EncodeInvalidUtf8)
-
-// GoogleCloudStorage is the encoding used by the googlecloudstorage backend
-const GoogleCloudStorage = (encoder.Base |
-	encoder.EncodeCrLf |
-	encoder.EncodeInvalidUtf8)
-
-// JottaCloud is the encoding used by the jottacloud backend
-//
-// Encode invalid UTF-8 bytes as xml doesn't handle them properly.
-//
-// Also: '*', '/', ':', '<', '>', '?', '\"', '\x00', '|'
-const JottaCloud = (encoder.Display |
-	encoder.EncodeWin | // :?"*<>|
-	encoder.EncodeInvalidUtf8)
-
-// Koofr is the encoding used by the koofr backend
-//
-// Encode invalid UTF-8 bytes as json doesn't handle them properly.
-const Koofr = (encoder.Display |
-	encoder.EncodeBackSlash |
-	encoder.EncodeInvalidUtf8)
-
-// Mailru is the encoding used by the mailru backend
-//
-// Encode invalid UTF-8 bytes as json doesn't handle them properly.
-const Mailru = (encoder.Display |
-	encoder.EncodeWin | // :?"*<>|
-	encoder.EncodeBackSlash |
-	encoder.EncodeInvalidUtf8)
-
-// Mega is the encoding used by the mega backend
-//
-// Encode invalid UTF-8 bytes as json doesn't handle them properly.
-const Mega = (encoder.Base |
-	encoder.EncodeInvalidUtf8)
-
-// OneDrive is the encoding used by the onedrive backend
-//
-// List of replaced characters:
-//   < (less than)     -> '＜' // FULLWIDTH LESS-THAN SIGN
-//   > (greater than)  -> '＞' // FULLWIDTH GREATER-THAN SIGN
-//   : (colon)         -> '：' // FULLWIDTH COLON
-//   " (double quote)  -> '＂' // FULLWIDTH QUOTATION MARK
-//   \ (backslash)     -> '＼' // FULLWIDTH REVERSE SOLIDUS
-//   | (vertical line) -> '｜' // FULLWIDTH VERTICAL LINE
-//   ? (question mark) -> '？' // FULLWIDTH QUESTION MARK
-//   * (asterisk)      -> '＊' // FULLWIDTH ASTERISK
-//   # (number sign)  -> '＃'  // FULLWIDTH NUMBER SIGN
-//   % (percent sign) -> '％'  // FULLWIDTH PERCENT SIGN
-//
-// Folder names cannot begin with a tilde ('~')
-// List of replaced characters:
-//   ~ (tilde)        -> '～'  // FULLWIDTH TILDE
-//
-// Additionally names can't begin with a space ( ) or end with a period (.) or space ( ).
-// List of replaced characters:
-//   . (period)        -> '．' // FULLWIDTH FULL STOP
-//     (space)         -> '␠'  // SYMBOL FOR SPACE
-//
-// Also encode invalid UTF-8 bytes as json doesn't handle them.
-//
-// The OneDrive API documentation lists the set of reserved characters, but
-// testing showed this list is incomplete. This are the differences:
-//  - " (double quote) is rejected, but missing in the documentation
-//  - space at the end of file and folder names is rejected, but missing in the documentation
-//  - period at the end of file names is rejected, but missing in the documentation
-//
-// Adding these restrictions to the OneDrive API documentation yields exactly
-// the same rules as the Windows naming conventions.
-//
-// https://docs.microsoft.com/en-us/onedrive/developer/rest-api/concepts/addressing-driveitems?view=odsp-graph-online#path-encoding
-const OneDrive = (encoder.Display |
-	encoder.EncodeBackSlash |
-	encoder.EncodeHashPercent |
-	encoder.EncodeLeftSpace |
-	encoder.EncodeLeftTilde |
-	encoder.EncodeRightPeriod |
-	encoder.EncodeRightSpace |
-	encoder.EncodeWin |
-	encoder.EncodeInvalidUtf8)
-
-// OpenDrive is the encoding used by the opendrive backend
-//
-// List of replaced characters:
-//   < (less than)     -> '＜' // FULLWIDTH LESS-THAN SIGN
-//   > (greater than)  -> '＞' // FULLWIDTH GREATER-THAN SIGN
-//   : (colon)         -> '：' // FULLWIDTH COLON
-//   " (double quote)  -> '＂' // FULLWIDTH QUOTATION MARK
-//   \ (backslash)     -> '＼' // FULLWIDTH REVERSE SOLIDUS
-//   | (vertical line) -> '｜' // FULLWIDTH VERTICAL LINE
-//   ? (question mark) -> '？' // FULLWIDTH QUESTION MARK
-//   * (asterisk)      -> '＊' // FULLWIDTH ASTERISK
-//
-// Additionally names can't begin or end with a ASCII whitespace.
-// List of replaced characters:
-//     (space)           -> '␠'  // SYMBOL FOR SPACE
-//     (horizontal tab)  -> '␉'  // SYMBOL FOR HORIZONTAL TABULATION
-//     (line feed)       -> '␊'  // SYMBOL FOR LINE FEED
-//     (vertical tab)    -> '␋'  // SYMBOL FOR VERTICAL TABULATION
-//     (carriage return) -> '␍'  // SYMBOL FOR CARRIAGE RETURN
-//
-// Also encode invalid UTF-8 bytes as json doesn't handle them properly.
-//
-// https://www.opendrive.com/wp-content/uploads/guides/OpenDrive_API_guide.pdf
-const OpenDrive = (encoder.Base |
-	encoder.EncodeWin |
-	encoder.EncodeLeftCrLfHtVt |
-	encoder.EncodeRightCrLfHtVt |
-	encoder.EncodeBackSlash |
-	encoder.EncodeLeftSpace |
-	encoder.EncodeRightSpace |
-	encoder.EncodeInvalidUtf8)
-
-// PremiumizeMe is the encoding used by the premiumizeme backend
-//
-// Encode invalid UTF-8 bytes as json doesn't handle them properly.
-const PremiumizeMe = (encoder.Display |
-	encoder.EncodeBackSlash |
-	encoder.EncodeDoubleQuote |
-	encoder.EncodeInvalidUtf8)
-
-// Pcloud is the encoding used by the pcloud backend
-//
-// Encode invalid UTF-8 bytes as json doesn't handle them properly.
-//
-// TODO: Investigate Unicode simplification (＼ gets converted to \ server-side)
-const Pcloud = (encoder.Display |
-	encoder.EncodeBackSlash |
-	encoder.EncodeInvalidUtf8)
-
-// Putio is the encoding used by the putio backend
-//
-// Note that \ is renamed to -
-//
-// Encode invalid UTF-8 bytes as json doesn't handle them properly.
-const Putio = (encoder.Display |
-	encoder.EncodeBackSlash |
-	encoder.EncodeInvalidUtf8)
-
-// Fichier is the encoding used by the fichier backend
-//
-// Characters that need escaping
-//
-// 		'\\': '＼', // FULLWIDTH REVERSE SOLIDUS
-// 		'<':  '＜', // FULLWIDTH LESS-THAN SIGN
-// 		'>':  '＞', // FULLWIDTH GREATER-THAN SIGN
-// 		'"':  '＂', // FULLWIDTH QUOTATION MARK - not on the list but seems to be reserved
-// 		'\'': '＇', // FULLWIDTH APOSTROPHE
-// 		'$':  '＄', // FULLWIDTH DOLLAR SIGN
-// 		'`':  '｀', // FULLWIDTH GRAVE ACCENT
-//
-// Leading space and trailing space
-const Fichier = (encoder.Display |
-	encoder.EncodeBackSlash |
-	encoder.EncodeSingleQuote |
-	encoder.EncodeBackQuote |
-	encoder.EncodeDoubleQuote |
-	encoder.EncodeLtGt |
-	encoder.EncodeDollar |
-	encoder.EncodeLeftSpace |
-	encoder.EncodeRightSpace |
-	encoder.EncodeInvalidUtf8)
-
-// FTP is the encoding used by the ftp backend
-//
-// The FTP protocal can't handle trailing spaces (for instance
-// pureftpd turns them into _)
-//
-// proftpd can't handle '*' in file names
-// pureftpd can't handle '[', ']' or '*'
-const FTP = (encoder.Display |
-	encoder.EncodeRightSpace)
-
-// S3 is the encoding used by the s3 backend
-//
-// Any UTF-8 character is valid in a key, however it can't handle
-// invalid UTF-8 and / have a special meaning.
-//
-// The SDK can't seem to handle uploading files called '.'
-//
-// FIXME would be nice to add
-// - initial / encoding
-// - doubled / encoding
-// - trailing / encoding
-// so that AWS keys are always valid file names
-const S3 = (encoder.EncodeInvalidUtf8 |
-	encoder.EncodeSlash |
-	encoder.EncodeDot)
-
-// Swift is the encoding used by the swift backend
-const Swift = (encoder.EncodeInvalidUtf8 |
-	encoder.EncodeSlash)
-
-// AzureBlob is the encoding used by the azureblob backend
-const AzureBlob = (encoder.EncodeInvalidUtf8 |
-	encoder.EncodeSlash |
-	encoder.EncodeCtl |
-	encoder.EncodeDel |
-	encoder.EncodeBackSlash |
-	encoder.EncodeRightPeriod)
-
-// QingStor is the encoding used by the qingstor backend
-const QingStor = (encoder.EncodeInvalidUtf8 |
-	encoder.EncodeCtl |
-	encoder.EncodeSlash)
-
-// Sharefile is the encoding used by the sharefile backend
-const Sharefile = (encoder.Base |
-	encoder.EncodeWin | // :?"*<>|
-	encoder.EncodeBackSlash | // \
-	encoder.EncodeCtl |
-	encoder.EncodeRightSpace |
-	encoder.EncodeRightPeriod |
-	encoder.EncodeLeftSpace |
-	encoder.EncodeLeftPeriod |
-	encoder.EncodeInvalidUtf8)
-
-// Yandex is the encoding used by the yandex backend
-//
-// Of the control characters \t \n \r are allowed
-// it doesn't seem worth making an exception for this
-const Yandex = (encoder.Display |
-	encoder.EncodeInvalidUtf8)
-
-// ByName returns the encoder for a give backend name or nil
-func ByName(name string) encoder.Encoder {
-	switch strings.ToLower(name) {
-	case "base":
-		return encoder.Base
-	case "display":
-		return encoder.Display
-	case "amazonclouddrive":
-		return AmazonCloudDrive
-	case "azureblob":
-		return AzureBlob
-	case "b2":
-		return B2
-	case "box":
-		return Box
-	//case "cache":
-	case "drive":
-		return Drive
-	case "dropbox":
-		return Dropbox
-	//case "ftp":
-	case "ficher":
-		return Fichier
-	case "googlecloudstorage":
-		return GoogleCloudStorage
-	//case "http":
-	//case "hubic":
-	case "jottacloud":
-		return JottaCloud
-	case "koofr":
-		return Koofr
-	case "local":
-		return Local()
-	case "local-windows", "windows":
-		return LocalWindows
-	case "local-unix", "unix":
-		return LocalUnix
-	case "local-macos", "macos":
-		return LocalMacOS
-	case "mailru":
-		return Mailru
-	case "mega":
-		return Mega
-	case "onedrive":
-		return OneDrive
-	case "opendrive":
-		return OpenDrive
-	case "premiumizeme":
-		return PremiumizeMe
-	case "pcloud":
-		return Pcloud
-	case "qingstor":
-		return QingStor
-	case "s3":
-		return S3
-	case "sharefile":
-		return Sharefile
-	//case "sftp":
-	case "swift":
-		return Swift
-	//case "webdav":
-	case "yandex":
-		return Yandex
-	default:
-		return nil
-	}
-}
-
-// Local returns the local encoding for the current platform
-func Local() encoder.MultiEncoder {
-	if runtime.GOOS == "windows" {
-		return LocalWindows
-	}
-	return LocalUnix
-}
-
-// Names returns the list of known encodings as accepted by ByName
-func Names() []string {
-	return []string{
-		"base",
-		"display",
-		"amazonclouddrive",
-		"b2",
-		"box",
-		"drive",
-		"dropbox",
-		"googlecloudstorage",
-		"jottacloud",
-		"koofr",
-		"local-unix",
-		"local-windows",
-		"local-macos",
-		"local",
-		"mega",
-		"onedrive",
-		"opendrive",
-		"premiumizeme",
-		"pcloud",
-		"sharefile",
-	}
-}

fs/encodings/encodings_noencode.go

@@ -1,57 +0,0 @@
-// +build noencode
-
-package encodings
-
-import (
-	"github.com/rclone/rclone/lib/encoder"
-)
-
-// Fake encodings used for testing
-const (
-	Base = encoder.MultiEncoder(
-		encoder.EncodeZero |
-			encoder.EncodeSlash)
-	Display            = Base
-	LocalUnix          = Base
-	LocalWindows       = Base
-	AmazonCloudDrive   = Base
-	AzureBlob          = Base
-	B2                 = Base
-	Box                = Base
-	Drive              = Base
-	Dropbox            = Base
-	FTP                = Base
-	Fichier            = Base
-	GoogleCloudStorage = Base
-	JottaCloud         = Base
-	Koofr              = Base
-	Mailru             = Base
-	Mega               = Base
-	OneDrive           = Base
-	OpenDrive          = Base
-	Pcloud             = Base
-	PremiumizeMe       = Base
-	Putio              = Base
-	QingStor           = Base
-	S3                 = Base
-	Sharefile          = Base
-	Swift              = Base
-	Yandex             = Base
-)
-
-// ByName returns the encoder for a give backend name or nil
-func ByName(name string) encoder.Encoder {
-	return Base
-}
-
-// Local returns the local encoding for the current platform
-func Local() encoder.MultiEncoder {
-	return Base
-}
-
-// Names returns the list of known encodings as accepted by ByName
-func Names() []string {
-	return []string{
-		"base",
-	}
-}