vfstest: make VFS test suite support symlinks

vfs/vfstest/file.go

@@ -6,6 +6,7 @@ import (
 	"testing"
 	"time"
 
+	"github.com/rclone/rclone/fs"
 	"github.com/rclone/rclone/vfs"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
@@ -72,3 +73,230 @@ func TestFileModTimeWithOpenWriters(t *testing.T) {
 
 	run.rm(t, "cp-archive-test")
 }
+
+// TestSymlinks tests all the api of the VFS / Mount symlinks support
+func TestSymlinks(t *testing.T) {
+	run.skipIfNoFUSE(t)
+	if !run.vfsOpt.Links {
+		t.Skip("No symlinks configured")
+	}
+
+	if runtime.GOOS == "windows" {
+		t.Skip("Skipping test on Windows")
+	}
+
+	fs.Logf(nil, "Links: %v, useVFS: %v, suffix: %v", run.vfsOpt.Links, run.useVFS, fs.LinkSuffix)
+
+	// Create initial setup of test files and directories we will create links to
+	run.mkdir(t, "dir1")
+	run.mkdir(t, "dir1/sub1dir1")
+	run.createFile(t, "dir1/file1", "potato")
+	run.mkdir(t, "dir2")
+	run.mkdir(t, "dir2/sub1dir2")
+	run.createFile(t, "dir2/file1", "chicken")
+
+	// base state all the tests will be build off
+	baseState := "dir1/|dir1/sub1dir1/|dir1/file1 6|dir2/|dir2/sub1dir2/|dir2/file1 7"
+	// Check the tests return to the base state
+	checkBaseState := func() {
+		run.checkDir(t, baseState)
+	}
+	checkBaseState()
+
+	t.Run("FileLink", func(t *testing.T) {
+		// Link to a file
+		run.symlink(t, "dir1/file1", "dir1file1_link")
+		run.checkDir(t, baseState+"|dir1file1_link 10")
+		run.checkMode(t, "dir1file1_link", os.FileMode(run.vfsOpt.LinkPerms), os.FileMode(run.vfsOpt.FilePerms))
+		assert.Equal(t, "dir1/file1", run.readlink(t, "dir1file1_link"))
+
+		// Read through a symlink
+		assert.Equal(t, "potato", run.readFile(t, "dir1file1_link"))
+
+		// Write through a symlink
+		err := writeFile(run.path("dir1file1_link"), []byte("carrot"), 0600)
+		require.NoError(t, err)
+		assert.Equal(t, "carrot", run.readFile(t, "dir1file1_link"))
+		assert.Equal(t, "carrot", run.readFile(t, "dir1/file1"))
+
+		// Rename a symlink
+		err = run.os.Rename(run.path("dir1file1_link"), run.path("dir1file1_link")+"_bla")
+		require.NoError(t, err)
+		run.checkDir(t, baseState+"|dir1file1_link_bla 10")
+		assert.Equal(t, "dir1/file1", run.readlink(t, "dir1file1_link_bla"))
+
+		// Delete a symlink
+		run.rm(t, "dir1file1_link_bla")
+		checkBaseState()
+	})
+
+	t.Run("DirLink", func(t *testing.T) {
+		// Link to a dir
+		run.symlink(t, "dir1", "dir1_link")
+		run.checkDir(t, baseState+"|dir1_link 4")
+		run.checkMode(t, "dir1_link", os.FileMode(run.vfsOpt.LinkPerms), os.FileMode(run.vfsOpt.DirPerms))
+		assert.Equal(t, "dir1", run.readlink(t, "dir1_link"))
+
+		// Check you can't open a directory symlink
+		_, err := run.os.OpenFile(run.path("dir1_link"), os.O_WRONLY, 0600)
+		require.Error(t, err)
+
+		// Our symlink resolution is very simple when using the VFS as when using the
+		// mount the OS will resolve the symlinks, so we don't recurse here
+
+		// Read entries directly
+		dir1Entries := make(dirMap)
+		run.readLocalEx(t, dir1Entries, "dir1", false)
+		assert.Equal(t, newDirMap("dir1/sub1dir1/|dir1/file1 6"), dir1Entries)
+
+		// Read entries through the directory symlink
+		dir1EntriesSymlink := make(dirMap)
+		run.readLocalEx(t, dir1EntriesSymlink, "dir1_link", false)
+		assert.Equal(t, newDirMap("dir1_link/sub1dir1/|dir1_link/file1 6"), dir1EntriesSymlink)
+
+		// Rename directory symlink
+		err = run.os.Rename(run.path("dir1_link"), run.path("dir1_link")+"_bla")
+		require.NoError(t, err)
+		run.checkDir(t, baseState+"|dir1_link_bla 4")
+		assert.Equal(t, "dir1", run.readlink(t, "dir1_link_bla"))
+
+		// Remove directory symlink
+		run.rm(t, "dir1_link_bla")
+
+		checkBaseState()
+	})
+
+	// Corner case #1 - We do not allow creating regular and symlink files having the same name (ie, test.txt and test.txt.rclonelink)
+
+	// Symlink first, then regular
+	t.Run("OverwriteSymlinkWithRegular", func(t *testing.T) {
+		link1Name := "link1.txt"
+
+		run.symlink(t, "dir1/file1", link1Name)
+		run.checkDir(t, baseState+"|link1.txt 10")
+
+		fh, err := run.os.OpenFile(run.path(link1Name), os.O_WRONLY|os.O_CREATE, os.FileMode(run.vfsOpt.FilePerms))
+
+		// On real mount with links enabled, that open the symlink target as expected, else that fails to create a new file
+		assert.NoError(t, err)
+		// Don't care about the result, in some cache mode the file can't be opened for writing, so closing would trigger an err
+		_ = fh.Close()
+
+		run.rm(t, link1Name)
+		checkBaseState()
+	})
+
+	// Regular first, then symlink
+	t.Run("OverwriteRegularWithSymlink", func(t *testing.T) {
+		link1Name := "link1.txt"
+
+		run.createFile(t, link1Name, "")
+		run.checkDir(t, baseState+"|link1.txt 0")
+
+		err := run.os.Symlink(".", run.path(link1Name))
+		assert.Error(t, err)
+
+		run.rm(t, link1Name)
+		checkBaseState()
+	})
+
+	// Corner case #2 - We do not allow creating directory and symlink file having the same name (ie, test and test.rclonelink)
+
+	// Symlink first, then directory
+	t.Run("OverwriteSymlinkWithDirectory", func(t *testing.T) {
+		link1Name := "link1"
+
+		run.symlink(t, ".", link1Name)
+		run.checkDir(t, baseState+"|link1 1")
+
+		err := run.os.Mkdir(run.path(link1Name), os.FileMode(run.vfsOpt.DirPerms))
+		assert.Error(t, err)
+
+		run.rm(t, link1Name)
+		checkBaseState()
+	})
+
+	// Directory first, then symlink
+	t.Run("OverwriteDirectoryWithSymlink", func(t *testing.T) {
+		link1Name := "link1"
+
+		run.mkdir(t, link1Name)
+		run.checkDir(t, baseState+"|link1/")
+
+		err := run.os.Symlink(".", run.path(link1Name))
+		assert.Error(t, err)
+
+		run.rm(t, link1Name)
+		checkBaseState()
+	})
+
+	// Corner case #3 - We do not allow moving directory or file having the same name in a target (ie, test and test.rclonelink)
+
+	// Move symlink -> regular file
+	t.Run("MoveSymlinkToFile", func(t *testing.T) {
+		t.Skip("FIXME not implemented")
+		link1Name := "link1.txt"
+
+		run.symlink(t, ".", link1Name)
+		run.createFile(t, "dir1/link1.txt", "")
+		run.checkDir(t, baseState+"|link1.txt 1|dir1/link1.txt 0")
+
+		err := run.os.Rename(run.path(link1Name), run.path("dir1/"+link1Name))
+		assert.Error(t, err)
+
+		run.rm(t, link1Name)
+		run.rm(t, "dir1/link1.txt")
+		checkBaseState()
+	})
+
+	// Move regular file -> symlink
+	t.Run("MoveFileToSymlink", func(t *testing.T) {
+		t.Skip("FIXME not implemented")
+		link1Name := "link1.txt"
+
+		run.createFile(t, link1Name, "")
+		run.symlink(t, ".", "dir1/"+link1Name)
+		run.checkDir(t, baseState+"|link1.txt 0|dir1/link1.txt 1")
+
+		err := run.os.Rename(run.path(link1Name), run.path("dir1/link1.txt"))
+		assert.Error(t, err)
+
+		run.rm(t, link1Name)
+		run.rm(t, "dir1/"+link1Name)
+		checkBaseState()
+	})
+
+	// Move symlink -> directory
+	t.Run("MoveSymlinkToDirectory", func(t *testing.T) {
+		t.Skip("FIXME not implemented")
+		link1Name := "link1"
+
+		run.symlink(t, ".", link1Name)
+		run.mkdir(t, "dir1/link1")
+		run.checkDir(t, baseState+"|link1 1|dir1/link1/")
+
+		err := run.os.Rename(run.path(link1Name), run.path("dir1/"+link1Name))
+		assert.Error(t, err)
+
+		run.rm(t, link1Name)
+		run.rm(t, "dir1/link1")
+		checkBaseState()
+	})
+
+	// Move directory -> symlink
+	t.Run("MoveDirectoryToSymlink", func(t *testing.T) {
+		t.Skip("FIXME not implemented")
+		link1Name := "dir1/link1"
+
+		run.mkdir(t, "link1")
+		run.symlink(t, ".", link1Name)
+		run.checkDir(t, baseState+"|link1/|dir1/link1 1")
+
+		err := run.os.Rename(run.path("link1"), run.path("dir1/link1"))
+		assert.Error(t, err)
+
+		run.rm(t, "link1")
+		run.rm(t, link1Name)
+		checkBaseState()
+	})
+}

vfs/vfstest/fs.go

@@ -49,12 +49,15 @@ func RunTests(t *testing.T, useVFS bool, minimumRequiredCacheMode vfscommon.Cach
 	tests := []struct {
 		cacheMode vfscommon.CacheMode
 		writeBack fs.Duration
+		links     bool
 	}{
 		{cacheMode: vfscommon.CacheModeOff},
+		{cacheMode: vfscommon.CacheModeOff, links: true},
 		{cacheMode: vfscommon.CacheModeMinimal},
 		{cacheMode: vfscommon.CacheModeWrites},
 		{cacheMode: vfscommon.CacheModeFull},
 		{cacheMode: vfscommon.CacheModeFull, writeBack: fs.Duration(100 * time.Millisecond)},
+		{cacheMode: vfscommon.CacheModeFull, writeBack: fs.Duration(100 * time.Millisecond), links: true},
 	}
 	for _, test := range tests {
 		if test.cacheMode < minimumRequiredCacheMode {
@@ -63,11 +66,15 @@ func RunTests(t *testing.T, useVFS bool, minimumRequiredCacheMode vfscommon.Cach
 		vfsOpt := vfscommon.Opt
 		vfsOpt.CacheMode = test.cacheMode
 		vfsOpt.WriteBack = test.writeBack
+		vfsOpt.Links = test.links
 		run = newRun(useVFS, &vfsOpt, mountFn)
 		what := fmt.Sprintf("CacheMode=%v", test.cacheMode)
 		if test.writeBack > 0 {
 			what += fmt.Sprintf(",WriteBack=%v", test.writeBack)
 		}
+		if test.links {
+			what += fmt.Sprintf(",Links=%v", test.links)
+		}
 		fs.Logf(nil, "Starting test run with %s", what)
 		ok := t.Run(what, func(t *testing.T) {
 			t.Run("TestTouchAndDelete", TestTouchAndDelete)
@@ -98,6 +105,7 @@ func RunTests(t *testing.T, useVFS bool, minimumRequiredCacheMode vfscommon.Cach
 			t.Run("TestWriteFileFsync", TestWriteFileFsync)
 			t.Run("TestWriteFileDup", TestWriteFileDup)
 			t.Run("TestWriteFileAppend", TestWriteFileAppend)
+			t.Run("TestSymlinks", TestSymlinks)
 		})
 		fs.Logf(nil, "Finished test run with %s (ok=%v)", what, ok)
 		run.Finalise()
@@ -213,10 +221,16 @@ func newDirMap(dirString string) (dm dirMap) {
 }
 
 // Returns a dirmap with only the files in
-func (dm dirMap) filesOnly() dirMap {
+func (dm dirMap) filesOnly(stripLinksSuffix bool) dirMap {
 	newDm := make(dirMap)
 	for name := range dm {
 		if !strings.HasSuffix(name, "/") {
+			if stripLinksSuffix {
+				index := strings.LastIndex(name, " ")
+				if index != -1 {
+					name = strings.TrimSuffix(name[0:index], fs.LinkSuffix) + name[index:]
+				}
+			}
 			newDm[name] = struct{}{}
 		}
 	}
@@ -224,7 +238,9 @@ func (dm dirMap) filesOnly() dirMap {
 }
 
 // reads the local tree into dir
-func (r *Run) readLocal(t *testing.T, dir dirMap, filePath string) {
+//
+// If recurse it set it will recurse into subdirectories
+func (r *Run) readLocalEx(t *testing.T, dir dirMap, filePath string, recurse bool) {
 	realPath := r.path(filePath)
 	files, err := r.os.ReadDir(realPath)
 	require.NoError(t, err)
@@ -232,15 +248,26 @@ func (r *Run) readLocal(t *testing.T, dir dirMap, filePath string) {
 		name := path.Join(filePath, fi.Name())
 		if fi.IsDir() {
 			dir[name+"/"] = struct{}{}
-			r.readLocal(t, dir, name)
+			if recurse {
+				r.readLocalEx(t, dir, name, recurse)
+			}
 			assert.Equal(t, os.FileMode(r.vfsOpt.DirPerms)&os.ModePerm, fi.Mode().Perm())
 		} else {
 			dir[fmt.Sprintf("%s %d", name, fi.Size())] = struct{}{}
-			assert.Equal(t, os.FileMode(r.vfsOpt.FilePerms)&os.ModePerm, fi.Mode().Perm())
+			if fi.Mode()&os.ModeSymlink != 0 {
+				assert.Equal(t, os.FileMode(r.vfsOpt.LinkPerms)&os.ModePerm, fi.Mode().Perm())
+			} else {
+				assert.Equal(t, os.FileMode(r.vfsOpt.FilePerms)&os.ModePerm, fi.Mode().Perm())
+			}
 		}
 	}
 }
 
+// reads the local tree into dir
+func (r *Run) readLocal(t *testing.T, dir dirMap, filePath string) {
+	r.readLocalEx(t, dir, filePath, true)
+}
+
 // reads the remote tree into dir
 func (r *Run) readRemote(t *testing.T, dir dirMap, filepath string) {
 	objs, dirs, err := walk.GetAll(context.Background(), r.fremote, filepath, true, 1)
@@ -271,7 +298,7 @@ func (r *Run) checkDir(t *testing.T, dirString string) {
 		remoteDm = make(dirMap)
 		r.readRemote(t, remoteDm, "")
 		// Ignore directories for remote compare
-		remoteOK = reflect.DeepEqual(dm.filesOnly(), remoteDm.filesOnly())
+		remoteOK = reflect.DeepEqual(dm.filesOnly(run.vfsOpt.Links), remoteDm.filesOnly(run.vfsOpt.Links))
 		fuseOK = reflect.DeepEqual(dm, localDm)
 		if remoteOK && fuseOK {
 			return
@@ -280,7 +307,7 @@ func (r *Run) checkDir(t *testing.T, dirString string) {
 		t.Logf("Sleeping for %v for list eventual consistency: %d/%d", sleep, i, retries)
 		time.Sleep(sleep)
 	}
-	assert.Equal(t, dm.filesOnly(), remoteDm.filesOnly(), "expected vs remote")
+	assert.Equal(t, dm.filesOnly(run.vfsOpt.Links), remoteDm.filesOnly(run.vfsOpt.Links), "expected vs remote")
 	assert.Equal(t, dm, localDm, "expected vs fuse mount")
 }
 
@@ -353,6 +380,37 @@ func (r *Run) rmdir(t *testing.T, filepath string) {
 	require.NoError(t, err)
 }
 
+func (r *Run) symlink(t *testing.T, oldname, newname string) {
+	newname = r.path(newname)
+	err := r.os.Symlink(oldname, newname)
+	require.NoError(t, err)
+}
+
+func (r *Run) checkMode(t *testing.T, name string, lexpected os.FileMode, expected os.FileMode) {
+	if r.useVFS {
+		info, err := run.os.Stat(run.path(name))
+		require.NoError(t, err)
+		assert.Equal(t, lexpected, info.Mode())
+		assert.Equal(t, name, info.Name())
+	} else {
+		info, err := os.Lstat(run.path(name))
+		require.NoError(t, err)
+		assert.Equal(t, lexpected, info.Mode())
+		assert.Equal(t, name, info.Name())
+
+		info, err = run.os.Stat(run.path(name))
+		require.NoError(t, err)
+		assert.Equal(t, expected, info.Mode())
+		assert.Equal(t, name, info.Name())
+	}
+}
+
+func (r *Run) readlink(t *testing.T, name string) string {
+	result, err := r.os.Readlink(r.path(name))
+	require.NoError(t, err)
+	return result
+}
+
 // TestMount checks that the Fs is mounted by seeing if the mountpoint
 // is in the mount output
 func TestMount(t *testing.T) {

vfs/vfstest/os.go

@@ -22,6 +22,8 @@ type Oser interface {
 	Remove(name string) error
 	Rename(oldName, newName string) error
 	Stat(path string) (os.FileInfo, error)
+	Symlink(oldname, newname string) error
+	Readlink(name string) (s string, err error)
 }
 
 // realOs is an implementation of Oser backed by the "os" package
@@ -130,6 +132,16 @@ func (r realOs) Stat(path string) (os.FileInfo, error) {
 	return os.Stat(path)
 }
 
+// Symlink
+func (r realOs) Symlink(oldname, newname string) error {
+	return os.Symlink(oldname, newname)
+}
+
+// Readlink
+func (r realOs) Readlink(name string) (s string, err error) {
+	return os.Readlink(name)
+}
+
 // Check interfaces
 var _ Oser = &realOs{}
 var _ vfs.Handle = &realOsFile{}