fstests: make integration tests to check all backends can store any file name

This tests the encoder is working properly

backend/chunker/chunker_internal_test.go

@@ -1,6 +1,7 @@
 package chunker
 
 import (
+	"context"
 	"flag"
 	"fmt"
 	"testing"
@@ -19,7 +20,7 @@ var (
 // test that chunking does not break large uploads
 func (f *Fs) InternalTestPutLarge(t *testing.T, kilobytes int) {
 	t.Run(fmt.Sprintf("PutLarge%dk", kilobytes), func(t *testing.T) {
-		fstests.TestPutLarge(t, f, &fstest.Item{
+		fstests.TestPutLarge(context.Background(), t, f, &fstest.Item{
 			ModTime: fstest.Time("2001-02-03T04:05:06.499999999Z"),
 			Path:    fmt.Sprintf("chunker-upload-%dk", kilobytes),
 			Size:    int64(kilobytes) * int64(fs.KibiByte),

fstest/fstests/fstests.go

@@ -32,6 +32,7 @@ import (
 	"github.com/rclone/rclone/fs/operations"
 	"github.com/rclone/rclone/fs/walk"
 	"github.com/rclone/rclone/fstest"
+	"github.com/rclone/rclone/lib/encoder"
 	"github.com/rclone/rclone/lib/random"
 	"github.com/rclone/rclone/lib/readers"
 	"github.com/stretchr/testify/assert"
@@ -117,12 +118,12 @@ func objsToNames(objs []fs.Object) []string {
 }
 
 // findObject finds the object on the remote
-func findObject(t *testing.T, f fs.Fs, Name string) fs.Object {
+func findObject(ctx context.Context, t *testing.T, f fs.Fs, Name string) fs.Object {
 	var obj fs.Object
 	var err error
 	sleepTime := 1 * time.Second
 	for i := 1; i <= *fstest.ListRetries; i++ {
-		obj, err = f.NewObject(context.Background(), Name)
+		obj, err = f.NewObject(ctx, Name)
 		if err == nil {
 			break
 		}
@@ -151,7 +152,7 @@ func retry(t *testing.T, what string, f func() error) {
 }
 
 // testPut puts file to the remote
-func testPut(t *testing.T, f fs.Fs, file *fstest.Item) (string, fs.Object) {
+func testPut(ctx context.Context, t *testing.T, f fs.Fs, file *fstest.Item) (string, fs.Object) {
 	var (
 		err        error
 		obj        fs.Object
@@ -166,19 +167,19 @@ func testPut(t *testing.T, f fs.Fs, file *fstest.Item) (string, fs.Object) {
 
 		file.Size = int64(buf.Len())
 		obji := object.NewStaticObjectInfo(file.Path, file.ModTime, file.Size, true, nil, nil)
-		obj, err = f.Put(context.Background(), in, obji)
+		obj, err = f.Put(ctx, in, obji)
 		return err
 	})
 	file.Hashes = uploadHash.Sums()
 	file.Check(t, obj, f.Precision())
 	// Re-read the object and check again
-	obj = findObject(t, f, file.Path)
+	obj = findObject(ctx, t, f, file.Path)
 	file.Check(t, obj, f.Precision())
 	return contents, obj
 }
 
 // TestPutLarge puts file to the remote, checks it and removes it on success.
-func TestPutLarge(t *testing.T, f fs.Fs, file *fstest.Item) {
+func TestPutLarge(ctx context.Context, t *testing.T, f fs.Fs, file *fstest.Item) {
 	var (
 		err        error
 		obj        fs.Object
@@ -190,7 +191,7 @@ func TestPutLarge(t *testing.T, f fs.Fs, file *fstest.Item) {
 		in := io.TeeReader(r, uploadHash)
 
 		obji := object.NewStaticObjectInfo(file.Path, file.ModTime, file.Size, true, nil, nil)
-		obj, err = f.Put(context.Background(), in, obji)
+		obj, err = f.Put(ctx, in, obji)
 		if file.Size == 0 && err == fs.ErrorCantUploadEmptyFiles {
 			t.Skip("Can't upload zero length files")
 		}
@@ -200,12 +201,12 @@ func TestPutLarge(t *testing.T, f fs.Fs, file *fstest.Item) {
 	file.Check(t, obj, f.Precision())
 
 	// Re-read the object and check again
-	obj = findObject(t, f, file.Path)
+	obj = findObject(ctx, t, f, file.Path)
 	file.Check(t, obj, f.Precision())
 
 	// Download the object and check it is OK
 	downloadHash := hash.NewMultiHasher()
-	download, err := obj.Open(context.Background())
+	download, err := obj.Open(ctx)
 	require.NoError(t, err)
 	n, err := io.Copy(downloadHash, download)
 	require.NoError(t, err)
@@ -214,7 +215,7 @@ func TestPutLarge(t *testing.T, f fs.Fs, file *fstest.Item) {
 	assert.Equal(t, file.Hashes, downloadHash.Sums())
 
 	// Remove the object
-	require.NoError(t, obj.Remove(context.Background()))
+	require.NoError(t, obj.Remove(ctx))
 }
 
 // errorReader just returns an error on Read
@@ -228,9 +229,9 @@ func (er errorReader) Read(p []byte) (n int, err error) {
 }
 
 // read the contents of an object as a string
-func readObject(t *testing.T, obj fs.Object, limit int64, options ...fs.OpenOption) string {
+func readObject(ctx context.Context, t *testing.T, obj fs.Object, limit int64, options ...fs.OpenOption) string {
 	what := fmt.Sprintf("readObject(%q) limit=%d, options=%+v", obj, limit, options)
-	in, err := obj.Open(context.Background(), options...)
+	in, err := obj.Open(ctx, options...)
 	require.NoError(t, err, what)
 	var r io.Reader = in
 	if limit >= 0 {
@@ -295,6 +296,7 @@ func Run(t *testing.T, opt *Opt) {
 		}
 		isLocalRemote bool
 		purged        bool // whether the dir has been purged or not
+		ctx           = context.Background()
 	)
 
 	// Skip the test if the remote isn't configured
@@ -408,12 +410,12 @@ func Run(t *testing.T, opt *Opt) {
 		if isBucketBasedButNotRoot(remote) {
 			t.Skip("Skipping test as non root bucket based remote")
 		}
-		err := remote.Rmdir(context.Background(), "")
+		err := remote.Rmdir(ctx, "")
 		assert.Error(t, err, "Expecting error on Rmdir non existent")
 	})
 
 	// Make the directory
-	err = remote.Mkdir(context.Background(), "")
+	err = remote.Mkdir(ctx, "")
 	require.NoError(t, err)
 	fstest.CheckListing(t, remote, []fstest.Item{})
 
@@ -451,7 +453,7 @@ func Run(t *testing.T, opt *Opt) {
 	// TestFsRmdirEmpty tests deleting an empty directory
 	t.Run("FsRmdirEmpty", func(t *testing.T) {
 		skipIfNotOk(t)
-		err := remote.Rmdir(context.Background(), "")
+		err := remote.Rmdir(ctx, "")
 		require.NoError(t, err)
 	})
 
@@ -461,26 +463,26 @@ func Run(t *testing.T, opt *Opt) {
 	t.Run("FsMkdir", func(t *testing.T) {
 		skipIfNotOk(t)
 
-		err := remote.Mkdir(context.Background(), "")
+		err := remote.Mkdir(ctx, "")
 		require.NoError(t, err)
 		fstest.CheckListing(t, remote, []fstest.Item{})
 
-		err = remote.Mkdir(context.Background(), "")
+		err = remote.Mkdir(ctx, "")
 		require.NoError(t, err)
 
 		// TestFsMkdirRmdirSubdir tests making and removing a sub directory
 		t.Run("FsMkdirRmdirSubdir", func(t *testing.T) {
 			skipIfNotOk(t)
 			dir := "dir/subdir"
-			err := operations.Mkdir(context.Background(), remote, dir)
+			err := operations.Mkdir(ctx, remote, dir)
 			require.NoError(t, err)
 			fstest.CheckListingWithPrecision(t, remote, []fstest.Item{}, []string{"dir", "dir/subdir"}, fs.GetModifyWindow(remote))
 
-			err = operations.Rmdir(context.Background(), remote, dir)
+			err = operations.Rmdir(ctx, remote, dir)
 			require.NoError(t, err)
 			fstest.CheckListingWithPrecision(t, remote, []fstest.Item{}, []string{"dir"}, fs.GetModifyWindow(remote))
 
-			err = operations.Rmdir(context.Background(), remote, "dir")
+			err = operations.Rmdir(ctx, remote, "dir")
 			require.NoError(t, err)
 			fstest.CheckListingWithPrecision(t, remote, []fstest.Item{}, []string{}, fs.GetModifyWindow(remote))
 		})
@@ -494,7 +496,7 @@ func Run(t *testing.T, opt *Opt) {
 		// TestFsListDirEmpty tests listing the directories from an empty directory
 		TestFsListDirEmpty := func(t *testing.T) {
 			skipIfNotOk(t)
-			objs, dirs, err := walk.GetAll(context.Background(), remote, "", true, 1)
+			objs, dirs, err := walk.GetAll(ctx, remote, "", true, 1)
 			if !remote.Features().CanHaveEmptyDirectories {
 				if err != fs.ErrorDirNotFound {
 					require.NoError(t, err)
@@ -516,7 +518,7 @@ func Run(t *testing.T, opt *Opt) {
 		// TestFsListDirNotFound tests listing the directories from an empty directory
 		TestFsListDirNotFound := func(t *testing.T) {
 			skipIfNotOk(t)
-			objs, dirs, err := walk.GetAll(context.Background(), remote, "does not exist", true, 1)
+			objs, dirs, err := walk.GetAll(ctx, remote, "does not exist", true, 1)
 			if !remote.Features().CanHaveEmptyDirectories {
 				if err != fs.ErrorDirNotFound {
 					assert.NoError(t, err)
@@ -534,15 +536,66 @@ func Run(t *testing.T, opt *Opt) {
 			TestFsListDirNotFound(t)
 		})
 
+		// FsEncoding tests that file name encodings are
+		// working by uploading a series of unusual files
+		// Must be run in an empty directory
+		t.Run("FsEncoding", func(t *testing.T) {
+			skipIfNotOk(t)
+
+			// check no files or dirs as pre-requisite
+			fstest.CheckListingWithPrecision(t, remote, []fstest.Item{}, []string{}, fs.GetModifyWindow(remote))
+
+			for _, test := range []struct {
+				name string
+				path string
+			}{
+				// See lib/encoder/encoder.go for list of things that go here
+				{"control chars", "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0A\x0B\x0C\x0D\x0E\x0F\x10\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1A\x1B\x1C\x1D\x1E\x1F\x7F"},
+				{"dot", "."},
+				{"dot dot", ".."},
+				{"punctuation", "!\"#$%&'()*+,-./:;<=>?@[\\]^_`{|}~"},
+				{"leading space", " leading space"},
+				{"leading tilde", "~leading tilde"},
+				{"leading CR", "\rleading CR"},
+				{"leading LF", "\nleading LF"},
+				{"leading HT", "\tleading HT"},
+				{"leading VT", "\vleading VT"},
+				{"trailing space", "trailing space "},
+				{"trailing CR", "trailing CR\r"},
+				{"trailing LF", "trailing LF\n"},
+				{"trailing HT", "trailing HT\t"},
+				{"trailing VT", "trailing VT\v"},
+				{"trailing dot", "trailing dot."},
+				{"invalid UTF-8", "invalid utf-8\xfe"},
+			} {
+				t.Run(test.name, func(t *testing.T) {
+					// turn raw strings into Standard encoding
+					fileName := encoder.Standard.Encode(test.path)
+					dirName := fileName
+					t.Logf("testing %q", fileName)
+					assert.NoError(t, remote.Mkdir(ctx, dirName))
+					file := fstest.Item{
+						ModTime: time.Now(),
+						Path:    dirName + "/" + fileName, // test creating a file and dir with that name
+					}
+					_, o := testPut(context.Background(), t, remote, &file)
+					fstest.CheckListingWithPrecision(t, remote, []fstest.Item{file}, []string{dirName}, fs.GetModifyWindow(remote))
+					assert.NoError(t, o.Remove(ctx))
+					assert.NoError(t, remote.Rmdir(ctx, dirName))
+					fstest.CheckListingWithPrecision(t, remote, []fstest.Item{}, []string{}, fs.GetModifyWindow(remote))
+				})
+			}
+		})
+
 		// TestFsNewObjectNotFound tests not finding a object
 		t.Run("FsNewObjectNotFound", func(t *testing.T) {
 			skipIfNotOk(t)
 			// Object in an existing directory
-			o, err := remote.NewObject(context.Background(), "potato")
+			o, err := remote.NewObject(ctx, "potato")
 			assert.Nil(t, o)
 			assert.Equal(t, fs.ErrorObjectNotFound, err)
 			// Now try an object in a non existing directory
-			o, err = remote.NewObject(context.Background(), "directory/not/found/potato")
+			o, err = remote.NewObject(ctx, "directory/not/found/potato")
 			assert.Nil(t, o)
 			assert.Equal(t, fs.ErrorObjectNotFound, err)
 		})
@@ -569,11 +622,11 @@ func Run(t *testing.T, opt *Opt) {
 			in := io.MultiReader(buf, er)
 
 			obji := object.NewStaticObjectInfo(file2.Path, file2.ModTime, 2*N, true, nil, nil)
-			_, err := remote.Put(context.Background(), in, obji)
+			_, err := remote.Put(ctx, in, obji)
 			// assert.Nil(t, obj) - FIXME some remotes return the object even on nil
 			assert.NotNil(t, err)
 
-			obj, err := remote.NewObject(context.Background(), file2.Path)
+			obj, err := remote.NewObject(ctx, file2.Path)
 			assert.Nil(t, obj)
 			assert.Equal(t, fs.ErrorObjectNotFound, err)
 		})
@@ -581,7 +634,7 @@ func Run(t *testing.T, opt *Opt) {
 		t.Run("FsPutZeroLength", func(t *testing.T) {
 			skipIfNotOk(t)
 
-			TestPutLarge(t, remote, &fstest.Item{
+			TestPutLarge(ctx, t, remote, &fstest.Item{
 				ModTime: fstest.Time("2001-02-03T04:05:06.499999999Z"),
 				Path:    fmt.Sprintf("zero-length-file"),
 				Size:    int64(0),
@@ -595,7 +648,7 @@ func Run(t *testing.T, opt *Opt) {
 				t.Skip("FS has no OpenWriterAt interface")
 			}
 			path := "writer-at-subdir/writer-at-file"
-			out, err := openWriterAt(context.Background(), path, -1)
+			out, err := openWriterAt(ctx, path, -1)
 			require.NoError(t, err)
 
 			var n int
@@ -611,11 +664,11 @@ func Run(t *testing.T, opt *Opt) {
 
 			assert.NoError(t, out.Close())
 
-			obj := findObject(t, remote, path)
+			obj := findObject(ctx, t, remote, path)
-			assert.Equal(t, "abcdefghi", readObject(t, obj, -1), "contents of file differ")
+			assert.Equal(t, "abcdefghi", readObject(ctx, t, obj, -1), "contents of file differ")
 
-			assert.NoError(t, obj.Remove(context.Background()))
+			assert.NoError(t, obj.Remove(ctx))
-			assert.NoError(t, remote.Rmdir(context.Background(), "writer-at-subdir"))
+			assert.NoError(t, remote.Rmdir(ctx, "writer-at-subdir"))
 		})
 
 		// TestFsChangeNotify tests that changes are properly
@@ -631,13 +684,13 @@ func Run(t *testing.T, opt *Opt) {
 				t.Skip("FS has no ChangeNotify interface")
 			}
 
-			err := operations.Mkdir(context.Background(), remote, "dir")
+			err := operations.Mkdir(ctx, remote, "dir")
 			require.NoError(t, err)
 
 			pollInterval := make(chan time.Duration)
 			dirChanges := map[string]struct{}{}
 			objChanges := map[string]struct{}{}
-			doChangeNotify(context.Background(), func(x string, e fs.EntryType) {
+			doChangeNotify(ctx, func(x string, e fs.EntryType) {
 				fs.Debugf(nil, "doChangeNotify(%q, %+v)", x, e)
 				if strings.HasPrefix(x, file1.Path[:5]) || strings.HasPrefix(x, file2.Path[:5]) {
 					fs.Debugf(nil, "Ignoring notify for file1 or file2: %q, %v", x, e)
@@ -655,7 +708,7 @@ func Run(t *testing.T, opt *Opt) {
 			var dirs []string
 			for _, idx := range []int{1, 3, 2} {
 				dir := fmt.Sprintf("dir/subdir%d", idx)
-				err = operations.Mkdir(context.Background(), remote, dir)
+				err = operations.Mkdir(ctx, remote, dir)
 				require.NoError(t, err)
 				dirs = append(dirs, dir)
 			}
@@ -666,7 +719,7 @@ func Run(t *testing.T, opt *Opt) {
 					ModTime: time.Now(),
 					Path:    fmt.Sprintf("dir/file%d", idx),
 				}
-				_, o := testPut(t, remote, &file)
+				_, o := testPut(ctx, t, remote, &file)
 				objs = append(objs, o)
 			}
 
@@ -700,11 +753,11 @@ func Run(t *testing.T, opt *Opt) {
 
 			// tidy up afterwards
 			for _, o := range objs {
-				assert.NoError(t, o.Remove(context.Background()))
+				assert.NoError(t, o.Remove(ctx))
 			}
 			dirs = append(dirs, "dir")
 			for _, dir := range dirs {
-				assert.NoError(t, remote.Rmdir(context.Background(), dir))
+				assert.NoError(t, remote.Rmdir(ctx, dir))
 			}
 		})
 
@@ -713,9 +766,9 @@ func Run(t *testing.T, opt *Opt) {
 		// Tests that require file1, file2 are within this
 		t.Run("FsPutFiles", func(t *testing.T) {
 			skipIfNotOk(t)
-			file1Contents, _ = testPut(t, remote, &file1)
+			file1Contents, _ = testPut(ctx, t, remote, &file1)
-			/* file2Contents = */ testPut(t, remote, &file2)
+			/* file2Contents = */ testPut(ctx, t, remote, &file2)
-			file1Contents, _ = testPut(t, remote, &file1)
+			file1Contents, _ = testPut(ctx, t, remote, &file1)
 			// Note that the next test will check there are no duplicated file names
 
 			// TestFsListDirFile2 tests the files are correctly uploaded by doing
@@ -725,9 +778,9 @@ func Run(t *testing.T, opt *Opt) {
 				list := func(dir string, expectedDirNames, expectedObjNames []string) {
 					var objNames, dirNames []string
 					for i := 1; i <= *fstest.ListRetries; i++ {
-						objs, dirs, err := walk.GetAll(context.Background(), remote, dir, true, 1)
+						objs, dirs, err := walk.GetAll(ctx, remote, dir, true, 1)
 						if errors.Cause(err) == fs.ErrorDirNotFound {
-							objs, dirs, err = walk.GetAll(context.Background(), remote, fstest.WinPath(dir), true, 1)
+							objs, dirs, err = walk.GetAll(ctx, remote, fstest.WinPath(dir), true, 1)
 						}
 						require.NoError(t, err)
 						objNames = objsToNames(objs)
@@ -773,7 +826,7 @@ func Run(t *testing.T, opt *Opt) {
 			// Test the files are all there with walk.ListR recursive listings
 			t.Run("FsListR", func(t *testing.T) {
 				skipIfNotOk(t)
-				objs, dirs, err := walk.GetAll(context.Background(), remote, "", true, -1)
+				objs, dirs, err := walk.GetAll(ctx, remote, "", true, -1)
 				require.NoError(t, err)
 				assert.Equal(t, []string{
 					"hello_ sausage",
@@ -791,7 +844,7 @@ func Run(t *testing.T, opt *Opt) {
 			// walk.ListR recursive listings on a sub dir
 			t.Run("FsListRSubdir", func(t *testing.T) {
 				skipIfNotOk(t)
-				objs, dirs, err := walk.GetAll(context.Background(), remote, path.Dir(path.Dir(path.Dir(path.Dir(file2.Path)))), true, -1)
+				objs, dirs, err := walk.GetAll(ctx, remote, path.Dir(path.Dir(path.Dir(path.Dir(file2.Path)))), true, -1)
 				require.NoError(t, err)
 				assert.Equal(t, []string{
 					"hello_ sausage/êé",
@@ -808,7 +861,7 @@ func Run(t *testing.T, opt *Opt) {
 				skipIfNotOk(t)
 				rootRemote, err := fs.NewFs(remoteName)
 				require.NoError(t, err)
-				_, dirs, err := walk.GetAll(context.Background(), rootRemote, "", true, 1)
+				_, dirs, err := walk.GetAll(ctx, rootRemote, "", true, 1)
 				require.NoError(t, err)
 				assert.Contains(t, dirsToNames(dirs), subRemoteLeaf, "Remote leaf not found")
 			}
@@ -830,7 +883,7 @@ func Run(t *testing.T, opt *Opt) {
 				for i := 0; i < 2; i++ {
 					dir, _ := path.Split(fileName)
 					dir = dir[:len(dir)-1]
-					objs, dirs, err = walk.GetAll(context.Background(), remote, dir, true, -1)
+					objs, dirs, err = walk.GetAll(ctx, remote, dir, true, -1)
 				}
 				require.NoError(t, err)
 				require.Len(t, objs, 1)
@@ -848,7 +901,7 @@ func Run(t *testing.T, opt *Opt) {
 			// TestFsListLevel2 tests List works for 2 levels
 			TestFsListLevel2 := func(t *testing.T) {
 				skipIfNotOk(t)
-				objs, dirs, err := walk.GetAll(context.Background(), remote, "", true, 2)
+				objs, dirs, err := walk.GetAll(ctx, remote, "", true, 2)
 				if err == fs.ErrorLevelNotSupported {
 					return
 				}
@@ -873,7 +926,7 @@ func Run(t *testing.T, opt *Opt) {
 			// TestFsNewObject tests NewObject
 			t.Run("FsNewObject", func(t *testing.T) {
 				skipIfNotOk(t)
-				obj := findObject(t, remote, file1.Path)
+				obj := findObject(ctx, t, remote, file1.Path)
 				file1.Check(t, obj, remote.Precision())
 			})
 
@@ -887,7 +940,7 @@ func Run(t *testing.T, opt *Opt) {
 			t.Run("FsNewObjectDir", func(t *testing.T) {
 				skipIfNotOk(t)
 				dir := path.Dir(file2.Path)
-				obj, err := remote.NewObject(context.Background(), dir)
+				obj, err := remote.NewObject(ctx, dir)
 				assert.Nil(t, obj)
 				assert.NotNil(t, err)
 			})
@@ -907,8 +960,8 @@ func Run(t *testing.T, opt *Opt) {
 				file2Copy.Path += "-copy"
 
 				// do the copy
-				src := findObject(t, remote, file2.Path)
+				src := findObject(ctx, t, remote, file2.Path)
-				dst, err := doCopy(context.Background(), src, file2Copy.Path)
+				dst, err := doCopy(ctx, src, file2Copy.Path)
 				if err == fs.ErrorCantCopy {
 					t.Skip("FS can't copy")
 				}
@@ -921,7 +974,7 @@ func Run(t *testing.T, opt *Opt) {
 				assert.Equal(t, file2Copy.Path, dst.Remote())
 
 				// Delete copy
-				err = dst.Remove(context.Background())
+				err = dst.Remove(ctx)
 				require.NoError(t, err)
 
 			})
@@ -946,8 +999,8 @@ func Run(t *testing.T, opt *Opt) {
 				// check happy path, i.e. no naming conflicts when rename and move are two
 				// separate operations
 				file2Move.Path = "other.txt"
-				src := findObject(t, remote, file2.Path)
+				src := findObject(ctx, t, remote, file2.Path)
-				dst, err := doMove(context.Background(), src, file2Move.Path)
+				dst, err := doMove(ctx, src, file2Move.Path)
 				if err == fs.ErrorCantMove {
 					t.Skip("FS can't move")
 				}
@@ -961,30 +1014,30 @@ func Run(t *testing.T, opt *Opt) {
 
 				// Check conflict on "rename, then move"
 				file1Move.Path = "moveTest/other.txt"
-				src = findObject(t, remote, file1.Path)
+				src = findObject(ctx, t, remote, file1.Path)
-				_, err = doMove(context.Background(), src, file1Move.Path)
+				_, err = doMove(ctx, src, file1Move.Path)
 				require.NoError(t, err)
 				fstest.CheckListing(t, remote, []fstest.Item{file1Move, file2Move})
 				// 1: moveTest/other.txt
 				// 2: other.txt
 
 				// Check conflict on "move, then rename"
-				src = findObject(t, remote, file1Move.Path)
+				src = findObject(ctx, t, remote, file1Move.Path)
-				_, err = doMove(context.Background(), src, file1.Path)
+				_, err = doMove(ctx, src, file1.Path)
 				require.NoError(t, err)
 				fstest.CheckListing(t, remote, []fstest.Item{file1, file2Move})
 				// 1: file name.txt
 				// 2: other.txt
 
-				src = findObject(t, remote, file2Move.Path)
+				src = findObject(ctx, t, remote, file2Move.Path)
-				_, err = doMove(context.Background(), src, file2.Path)
+				_, err = doMove(ctx, src, file2.Path)
 				require.NoError(t, err)
 				fstest.CheckListing(t, remote, []fstest.Item{file1, file2})
 				// 1: file name.txt
 				// 2: hello sausage?/../z.txt
 
 				// Tidy up moveTest directory
-				require.NoError(t, remote.Rmdir(context.Background(), "moveTest"))
+				require.NoError(t, remote.Rmdir(ctx, "moveTest"))
 			})
 
 			// Move src to this remote using server side move operations.
@@ -1008,7 +1061,7 @@ func Run(t *testing.T, opt *Opt) {
 				}
 
 				// Check it can't move onto itself
-				err := doDirMove(context.Background(), remote, "", "")
+				err := doDirMove(ctx, remote, "", "")
 				require.Equal(t, fs.ErrorDirExists, err)
 
 				// new remote
@@ -1018,12 +1071,12 @@ func Run(t *testing.T, opt *Opt) {
 
 				const newName = "new_name/sub_new_name"
 				// try the move
-				err = newRemote.Features().DirMove(context.Background(), remote, "", newName)
+				err = newRemote.Features().DirMove(ctx, remote, "", newName)
 				require.NoError(t, err)
 
 				// check remotes
 				// remote should not exist here
-				_, err = remote.List(context.Background(), "")
+				_, err = remote.List(ctx, "")
 				assert.Equal(t, fs.ErrorDirNotFound, errors.Cause(err))
 				//fstest.CheckListingWithPrecision(t, remote, []fstest.Item{}, []string{}, remote.Precision())
 				file1Copy := file1
@@ -1040,7 +1093,7 @@ func Run(t *testing.T, opt *Opt) {
 				}, newRemote.Precision())
 
 				// move it back
-				err = doDirMove(context.Background(), newRemote, newName, "")
+				err = doDirMove(ctx, newRemote, newName, "")
 				require.NoError(t, err)
 
 				// check remotes
@@ -1061,7 +1114,7 @@ func Run(t *testing.T, opt *Opt) {
 				if isBucketBasedButNotRoot(remote) {
 					t.Skip("Skipping test as non root bucket based remote")
 				}
-				err := remote.Rmdir(context.Background(), "")
+				err := remote.Rmdir(ctx, "")
 				require.Error(t, err, "Expecting error on RMdir on non empty remote")
 			})
 
@@ -1081,7 +1134,7 @@ func Run(t *testing.T, opt *Opt) {
 			// TestObjectString tests the Object String method
 			t.Run("ObjectString", func(t *testing.T) {
 				skipIfNotOk(t)
-				obj := findObject(t, remote, file1.Path)
+				obj := findObject(ctx, t, remote, file1.Path)
 				assert.Equal(t, file1.Path, obj.String())
 				if opt.NilObject != nil {
 					assert.Equal(t, "<nil>", opt.NilObject.String())
@@ -1091,7 +1144,7 @@ func Run(t *testing.T, opt *Opt) {
 			// TestObjectFs tests the object can be found
 			t.Run("ObjectFs", func(t *testing.T) {
 				skipIfNotOk(t)
-				obj := findObject(t, remote, file1.Path)
+				obj := findObject(ctx, t, remote, file1.Path)
 				// If this is set we don't do the direct comparison of
 				// the Fs from the object as it may be different
 				if opt.SkipFsMatch {
@@ -1110,34 +1163,34 @@ func Run(t *testing.T, opt *Opt) {
 			// TestObjectRemote tests the Remote is correct
 			t.Run("ObjectRemote", func(t *testing.T) {
 				skipIfNotOk(t)
-				obj := findObject(t, remote, file1.Path)
+				obj := findObject(ctx, t, remote, file1.Path)
 				assert.Equal(t, file1.Path, obj.Remote())
 			})
 
 			// TestObjectHashes checks all the hashes the object supports
 			t.Run("ObjectHashes", func(t *testing.T) {
 				skipIfNotOk(t)
-				obj := findObject(t, remote, file1.Path)
+				obj := findObject(ctx, t, remote, file1.Path)
 				file1.CheckHashes(t, obj)
 			})
 
 			// TestObjectModTime tests the ModTime of the object is correct
 			TestObjectModTime := func(t *testing.T) {
 				skipIfNotOk(t)
-				obj := findObject(t, remote, file1.Path)
+				obj := findObject(ctx, t, remote, file1.Path)
-				file1.CheckModTime(t, obj, obj.ModTime(context.Background()), remote.Precision())
+				file1.CheckModTime(t, obj, obj.ModTime(ctx), remote.Precision())
 			}
 			t.Run("ObjectModTime", TestObjectModTime)
 
 			// TestObjectMimeType tests the MimeType of the object is correct
 			t.Run("ObjectMimeType", func(t *testing.T) {
 				skipIfNotOk(t)
-				obj := findObject(t, remote, file1.Path)
+				obj := findObject(ctx, t, remote, file1.Path)
 				do, ok := obj.(fs.MimeTyper)
 				if !ok {
 					t.Skip("MimeType method not supported")
 				}
-				mimeType := do.MimeType(context.Background())
+				mimeType := do.MimeType(ctx)
 				if strings.ContainsRune(mimeType, ';') {
 					assert.Equal(t, "text/plain; charset=utf-8", mimeType)
 				} else {
@@ -1149,15 +1202,15 @@ func Run(t *testing.T, opt *Opt) {
 			t.Run("ObjectSetModTime", func(t *testing.T) {
 				skipIfNotOk(t)
 				newModTime := fstest.Time("2011-12-13T14:15:16.999999999Z")
-				obj := findObject(t, remote, file1.Path)
+				obj := findObject(ctx, t, remote, file1.Path)
-				err := obj.SetModTime(context.Background(), newModTime)
+				err := obj.SetModTime(ctx, newModTime)
 				if err == fs.ErrorCantSetModTime || err == fs.ErrorCantSetModTimeWithoutDelete {
 					t.Log(err)
 					return
 				}
 				require.NoError(t, err)
 				file1.ModTime = newModTime
-				file1.CheckModTime(t, obj, obj.ModTime(context.Background()), remote.Precision())
+				file1.CheckModTime(t, obj, obj.ModTime(ctx), remote.Precision())
 				// And make a new object and read it from there too
 				TestObjectModTime(t)
 			})
@@ -1165,22 +1218,22 @@ func Run(t *testing.T, opt *Opt) {
 			// TestObjectSize tests that Size works
 			t.Run("ObjectSize", func(t *testing.T) {
 				skipIfNotOk(t)
-				obj := findObject(t, remote, file1.Path)
+				obj := findObject(ctx, t, remote, file1.Path)
 				assert.Equal(t, file1.Size, obj.Size())
 			})
 
 			// TestObjectOpen tests that Open works
 			t.Run("ObjectOpen", func(t *testing.T) {
 				skipIfNotOk(t)
-				obj := findObject(t, remote, file1.Path)
+				obj := findObject(ctx, t, remote, file1.Path)
-				assert.Equal(t, file1Contents, readObject(t, obj, -1), "contents of file1 differ")
+				assert.Equal(t, file1Contents, readObject(ctx, t, obj, -1), "contents of file1 differ")
 			})
 
 			// TestObjectOpenSeek tests that Open works with SeekOption
 			t.Run("ObjectOpenSeek", func(t *testing.T) {
 				skipIfNotOk(t)
-				obj := findObject(t, remote, file1.Path)
+				obj := findObject(ctx, t, remote, file1.Path)
-				assert.Equal(t, file1Contents[50:], readObject(t, obj, -1, &fs.SeekOption{Offset: 50}), "contents of file1 differ after seek")
+				assert.Equal(t, file1Contents[50:], readObject(ctx, t, obj, -1, &fs.SeekOption{Offset: 50}), "contents of file1 differ after seek")
 			})
 
 			// TestObjectOpenRange tests that Open works with RangeOption
@@ -1188,7 +1241,7 @@ func Run(t *testing.T, opt *Opt) {
 			// go test -v -run 'TestIntegration/Test(Setup|Init|FsMkdir|FsPutFile1|FsPutFile2|FsUpdateFile1|ObjectOpenRange)$'
 			t.Run("ObjectOpenRange", func(t *testing.T) {
 				skipIfNotOk(t)
-				obj := findObject(t, remote, file1.Path)
+				obj := findObject(ctx, t, remote, file1.Path)
 				for _, test := range []struct {
 					ro                 fs.RangeOption
 					wantStart, wantEnd int
@@ -1199,7 +1252,7 @@ func Run(t *testing.T, opt *Opt) {
 					{fs.RangeOption{Start: -1, End: 20}, 80, 100}, // if start is omitted this means get the final bytes
 					// {fs.RangeOption{Start: -1, End: -1}, 0, 100}, - this seems to work but the RFC doesn't define it
 				} {
-					got := readObject(t, obj, -1, &test.ro)
+					got := readObject(ctx, t, obj, -1, &test.ro)
 					foundAt := strings.Index(file1Contents, got)
 					help := fmt.Sprintf("%#v failed want [%d:%d] got [%d:%d]", test.ro, test.wantStart, test.wantEnd, foundAt, foundAt+len(got))
 					assert.Equal(t, file1Contents[test.wantStart:test.wantEnd], got, help)
@@ -1209,8 +1262,8 @@ func Run(t *testing.T, opt *Opt) {
 			// TestObjectPartialRead tests that reading only part of the object does the correct thing
 			t.Run("ObjectPartialRead", func(t *testing.T) {
 				skipIfNotOk(t)
-				obj := findObject(t, remote, file1.Path)
+				obj := findObject(ctx, t, remote, file1.Path)
-				assert.Equal(t, file1Contents[:50], readObject(t, obj, 50), "contents of file1 differ after limited read")
+				assert.Equal(t, file1Contents[:50], readObject(ctx, t, obj, 50), "contents of file1 differ after limited read")
 			})
 
 			// TestObjectUpdate tests that Update works
@@ -1222,9 +1275,9 @@ func Run(t *testing.T, opt *Opt) {
 				in := io.TeeReader(buf, hash)
 
 				file1.Size = int64(buf.Len())
-				obj := findObject(t, remote, file1.Path)
+				obj := findObject(ctx, t, remote, file1.Path)
 				obji := object.NewStaticObjectInfo(file1.Path, file1.ModTime, int64(len(contents)), true, nil, obj.Fs())
-				err := obj.Update(context.Background(), in, obji)
+				err := obj.Update(ctx, in, obji)
 				require.NoError(t, err)
 				file1.Hashes = hash.Sums()
 
@@ -1232,18 +1285,18 @@ func Run(t *testing.T, opt *Opt) {
 				file1.Check(t, obj, remote.Precision())
 
 				// Re-read the object and check again
-				obj = findObject(t, remote, file1.Path)
+				obj = findObject(ctx, t, remote, file1.Path)
 				file1.Check(t, obj, remote.Precision())
 
 				// check contents correct
-				assert.Equal(t, contents, readObject(t, obj, -1), "contents of updated file1 differ")
+				assert.Equal(t, contents, readObject(ctx, t, obj, -1), "contents of updated file1 differ")
 				file1Contents = contents
 			})
 
 			// TestObjectStorable tests that Storable works
 			t.Run("ObjectStorable", func(t *testing.T) {
 				skipIfNotOk(t)
-				obj := findObject(t, remote, file1.Path)
+				obj := findObject(ctx, t, remote, file1.Path)
 				require.NotNil(t, !obj.Storable(), "Expecting object to be storable")
 			})
 
@@ -1363,7 +1416,7 @@ func Run(t *testing.T, opt *Opt) {
 						ModTime: time.Now(),
 						Path:    path.Join(configLeaf, "created from root.txt"),
 					}
-					_, file3Obj := testPut(t, rootRemote, &file3Root)
+					_, file3Obj := testPut(ctx, t, rootRemote, &file3Root)
 					fstest.CheckListingWithRoot(t, rootRemote, configLeaf, []fstest.Item{file1Root, file2Root, file3Root}, nil, rootRemote.Precision())
 
 					// And then remove it
@@ -1385,29 +1438,29 @@ func Run(t *testing.T, opt *Opt) {
 				}
 
 				// if object not found
-				link, err := doPublicLink(context.Background(), file1.Path+"_does_not_exist")
+				link, err := doPublicLink(ctx, file1.Path+"_does_not_exist")
 				require.Error(t, err, "Expected to get error when file doesn't exist")
 				require.Equal(t, "", link, "Expected link to be empty on error")
 
 				// sharing file for the first time
-				link1, err := doPublicLink(context.Background(), file1.Path)
+				link1, err := doPublicLink(ctx, file1.Path)
 				require.NoError(t, err)
 				require.NotEqual(t, "", link1, "Link should not be empty")
 
-				link2, err := doPublicLink(context.Background(), file2.Path)
+				link2, err := doPublicLink(ctx, file2.Path)
 				require.NoError(t, err)
 				require.NotEqual(t, "", link2, "Link should not be empty")
 
 				require.NotEqual(t, link1, link2, "Links to different files should differ")
 
 				// sharing file for the 2nd time
-				link1, err = doPublicLink(context.Background(), file1.Path)
+				link1, err = doPublicLink(ctx, file1.Path)
 				require.NoError(t, err)
 				require.NotEqual(t, "", link1, "Link should not be empty")
 
 				// sharing directory for the first time
 				path := path.Dir(file2.Path)
-				link3, err := doPublicLink(context.Background(), path)
+				link3, err := doPublicLink(ctx, path)
 				if err != nil && errors.Cause(err) == fs.ErrorCantShareDirectories {
 					t.Log("skipping directory tests as not supported on this backend")
 				} else {
@@ -1415,7 +1468,7 @@ func Run(t *testing.T, opt *Opt) {
 					require.NotEqual(t, "", link3, "Link should not be empty")
 
 					// sharing directory for the second time
-					link3, err = doPublicLink(context.Background(), path)
+					link3, err = doPublicLink(ctx, path)
 					require.NoError(t, err)
 					require.NotEqual(t, "", link3, "Link should not be empty")
 
@@ -1426,10 +1479,10 @@ func Run(t *testing.T, opt *Opt) {
 					// ensure sub remote isn't empty
 					buf := bytes.NewBufferString("somecontent")
 					obji := object.NewStaticObjectInfo("somefile", time.Now(), int64(buf.Len()), true, nil, nil)
-					_, err = subRemote.Put(context.Background(), buf, obji)
+					_, err = subRemote.Put(ctx, buf, obji)
 					require.NoError(t, err)
 
-					link4, err := subRemote.Features().PublicLink(context.Background(), "")
+					link4, err := subRemote.Features().PublicLink(ctx, "")
 					require.NoError(t, err, "Sharing root in a sub-remote should work")
 					require.NotEqual(t, "", link4, "Link should not be empty")
 				}
@@ -1438,7 +1491,7 @@ func Run(t *testing.T, opt *Opt) {
 			// TestSetTier tests SetTier and GetTier functionality
 			t.Run("SetTier", func(t *testing.T) {
 				skipIfNotSetTier(t)
-				obj := findObject(t, remote, file1.Path)
+				obj := findObject(ctx, t, remote, file1.Path)
 				setter, ok := obj.(fs.SetTierer)
 				assert.NotNil(t, ok)
 				getter, ok := obj.(fs.GetTierer)
@@ -1466,7 +1519,7 @@ func Run(t *testing.T, opt *Opt) {
 				if ft.UnWrap == nil {
 					t.Skip("Not a wrapping Fs")
 				}
-				obj := findObject(t, remote, file1.Path)
+				obj := findObject(ctx, t, remote, file1.Path)
 				_, unsupported := fs.ObjectOptionalInterfaces(obj)
 				for _, name := range unsupported {
 					if !stringsContains(name, opt.UnimplementableObjectMethods) {
@@ -1478,8 +1531,9 @@ func Run(t *testing.T, opt *Opt) {
 			// TestObjectRemove tests Remove
 			t.Run("ObjectRemove", func(t *testing.T) {
 				skipIfNotOk(t)
-				obj := findObject(t, remote, file1.Path)
+				// remove file1
-				err := obj.Remove(context.Background())
+				obj := findObject(ctx, t, remote, file1.Path)
+				err := obj.Remove(ctx)
 				require.NoError(t, err)
 				// check listing without modtime as TestPublicLink may change the modtime
 				fstest.CheckListingWithPrecision(t, remote, []fstest.Item{file2}, nil, fs.ModTimeNotSupported)
@@ -1502,6 +1556,8 @@ func Run(t *testing.T, opt *Opt) {
 				assert.NotEqual(t, int64(0), usage.Total)
 			})
 
+			// Just file2 remains for Purge to clean up
+
 			// TestFsPutStream tests uploading files when size isn't known in advance.
 			// This may trigger large buffer allocation in some backends, keep it
 			// close to the end of suite. (See fs/operations/xtra_operations_test.go)
@@ -1531,15 +1587,16 @@ func Run(t *testing.T, opt *Opt) {
 
 					file.Size = -1
 					obji := object.NewStaticObjectInfo(file.Path, file.ModTime, file.Size, true, nil, nil)
-					obj, err = remote.Features().PutStream(context.Background(), in, obji)
+					obj, err = remote.Features().PutStream(ctx, in, obji)
 					return err
 				})
 				file.Hashes = uploadHash.Sums()
 				file.Size = int64(contentSize) // use correct size when checking
 				file.Check(t, obj, remote.Precision())
 				// Re-read the object and check again
-				obj = findObject(t, remote, file.Path)
+				obj = findObject(ctx, t, remote, file.Path)
 				file.Check(t, obj, remote.Precision())
+				require.NoError(t, obj.Remove(ctx))
 			})
 
 			// TestInternal calls InternalTest() on the Fs
@@ -1652,7 +1709,7 @@ func Run(t *testing.T, opt *Opt) {
 
 					for _, fileSize := range testChunks {
 						t.Run(fmt.Sprintf("%d", fileSize), func(t *testing.T) {
-							TestPutLarge(t, remote, &fstest.Item{
+							TestPutLarge(ctx, t, remote, &fstest.Item{
 								ModTime: fstest.Time("2001-02-03T04:05:06.499999999Z"),
 								Path:    fmt.Sprintf("chunked-%s-%s.bin", cs.String(), fileSize.String()),
 								Size:    int64(fileSize),
@@ -1680,9 +1737,9 @@ func Run(t *testing.T, opt *Opt) {
 				in := bytes.NewBufferString(contents)
 
 				obji := object.NewStaticObjectInfo("unknown-size-put.txt", fstest.Time("2002-02-03T04:05:06.499999999Z"), -1, true, nil, nil)
-				obj, err := remote.Put(context.Background(), in, obji)
+				obj, err := remote.Put(ctx, in, obji)
 				if err == nil {
-					require.NoError(t, obj.Remove(context.Background()), "successfully uploaded unknown-sized file but failed to remove")
+					require.NoError(t, obj.Remove(ctx), "successfully uploaded unknown-sized file but failed to remove")
 				}
 				// if err != nil: it's okay as long as no panic
 			})
@@ -1693,7 +1750,7 @@ func Run(t *testing.T, opt *Opt) {
 					Path:    "unknown-size-update.txt",
 				}
 
-				testPut(t, remote, &unknownSizeUpdateFile)
+				testPut(ctx, t, remote, &unknownSizeUpdateFile)
 
 				defer func() {
 					assert.Nil(t, recover(), "Object.Update() should not panic when src.Size() == -1")
@@ -1702,11 +1759,11 @@ func Run(t *testing.T, opt *Opt) {
 				newContents := random.String(200)
 				in := bytes.NewBufferString(newContents)
 
-				obj := findObject(t, remote, unknownSizeUpdateFile.Path)
+				obj := findObject(ctx, t, remote, unknownSizeUpdateFile.Path)
 				obji := object.NewStaticObjectInfo(unknownSizeUpdateFile.Path, unknownSizeUpdateFile.ModTime, -1, true, nil, obj.Fs())
-				err := obj.Update(context.Background(), in, obji)
+				err := obj.Update(ctx, in, obji)
 				if err == nil {
-					require.NoError(t, obj.Remove(context.Background()), "successfully updated object with unknown-sized source but failed to remove")
+					require.NoError(t, obj.Remove(ctx), "successfully updated object with unknown-sized source but failed to remove")
 				}
 				// if err != nil: it's okay as long as no panic
 			})
@@ -1728,21 +1785,21 @@ func Run(t *testing.T, opt *Opt) {
 			colonIndex := strings.IndexRune(deepRemoteName, ':')
 			firstSlashIndex := strings.IndexRune(deepRemoteName, '/')
 			firstDir := deepRemoteName[colonIndex+1 : firstSlashIndex]
-			_, err = deepRemote.NewObject(context.Background(), firstDir)
+			_, err = deepRemote.NewObject(ctx, firstDir)
 			require.Equal(t, fs.ErrorObjectNotFound, err)
 			// If err is not fs.ErrorObjectNotFound, it means the backend is
 			// somehow confused about root and absolute root.
 		})
 
 		// Purge the folder
-		err = operations.Purge(context.Background(), remote, "")
+		err = operations.Purge(ctx, remote, "")
 		require.NoError(t, err)
 		purged = true
 		fstest.CheckListing(t, remote, []fstest.Item{})
 
 		// Check purging again if not bucket based
 		if !isBucketBasedButNotRoot(remote) {
-			err = operations.Purge(context.Background(), remote, "")
+			err = operations.Purge(ctx, remote, "")
 			assert.Error(t, err, "Expecting error after on second purge")
 		}
 
@@ -1750,7 +1807,7 @@ func Run(t *testing.T, opt *Opt) {
 
 	// Check directory is purged
 	if !purged {
-		_ = operations.Purge(context.Background(), remote, "")
+		_ = operations.Purge(ctx, remote, "")
 	}
 
 	// Remove the local directory so we don't clutter up /tmp

lib/encoder/encoder.go

@@ -31,6 +31,8 @@ const (
 	QuoteRune = '‛' // SINGLE HIGH-REVERSED-9 QUOTATION MARK
 )
 
+// NB keep the tests in fstests/fstests/fstests.go FsEncoding up to date with this
+
 // Possible flags for the MultiEncoder
 const (
 	EncodeZero          uint = 0         // NUL(0x00)