diff --git a/fstest/testserver/init.d/PORTS.md b/fstest/testserver/init.d/PORTS.md
new file mode 100644
index 000000000..f8e75c4e2
--- /dev/null
+++ b/fstest/testserver/init.d/PORTS.md
@@ -0,0 +1,41 @@
+# Ports for tests
+
+All these tests need to run on a different port.
+
+They should be bound to localhost so they are not accessible externally.
+
+| Port  | Test |
+|:-----:|:----:|
+|    88 | TestHdfs |
+|   750 | TestHdfs |
+|  8020 | TestHdfs |
+|  8086 | TestSeafileV6 |
+|  8087 | TestSeafile |
+|  8088 | TestSeafileEncrypted |
+|  9866 | TestHdfs |
+| 28620 | TestWebdavRclone |
+| 28621 | TestSFTPRclone |
+| 28622 | TestFTPRclone |
+| 28623 | TestSFTPRcloneSSH |
+| 28624 | TestS3Rclone |
+| 28625 | TestS3Minio |
+| 28626 | TestS3MinioEdge |
+| 28627 | TestSFTPOpenssh |
+| 28628 | TestSwiftAIO |
+| 28629 | TestWebdavNextcloud |
+| 28630 | TestSMB |
+| 28631 | TestFTPProftpd |
+| 38081 | TestWebdavOwncloud |
+
+## Non localhost tests
+
+All these use `$(docker_ip)` which means they don't work on macOS or
+Windows. It is proabably possible to make them work with some effort
+but will require port forwarding a range of ports and configuring the
+FTP server to only use that range of ports. The FTP server will likely
+need know it is behind a NAT so it advertises the correct external IP.
+
+- TestFTPProftpd
+- TestFTPPureftpd
+- TestFTPVsftpd
+- TestFTPVsftpdTLS
diff --git a/fstest/testserver/init.d/TestS3Minio b/fstest/testserver/init.d/TestS3Minio
index 6dc41af5b..4a599d457 100755
--- a/fstest/testserver/init.d/TestS3Minio
+++ b/fstest/testserver/init.d/TestS3Minio
@@ -5,6 +5,7 @@ set -e
 NAME=minio
 USER=rclone
 PASS=AxedBodedGinger7
+PORT=28625
 
 . $(dirname "$0")/docker.bash
 
@@ -12,14 +13,15 @@ start() {
     docker run --rm -d --name $NAME \
            -e "MINIO_ACCESS_KEY=$USER" \
            -e "MINIO_SECRET_KEY=$PASS" \
+           -p 127.0.0.1:${PORT}:9000 \
            minio/minio server /data
     
     echo type=s3
     echo provider=Minio
     echo access_key_id=$USER
     echo secret_access_key=$PASS
-    echo endpoint=http://$(docker_ip):9000/
-    echo _connect=$(docker_ip):9000
+    echo endpoint=http://127.0.0.1:${PORT}/
+    echo _connect=127.0.0.1:${PORT}
 }
 
 . $(dirname "$0")/run.bash
diff --git a/fstest/testserver/init.d/TestS3MinioEdge b/fstest/testserver/init.d/TestS3MinioEdge
index f48002920..459dbb903 100755
--- a/fstest/testserver/init.d/TestS3MinioEdge
+++ b/fstest/testserver/init.d/TestS3MinioEdge
@@ -5,6 +5,7 @@ set -e
 NAME=minio-edge
 USER=rclone
 PASS=DeniseOxygenEiffel4
+PORT=28626
 
 . $(dirname "$0")/docker.bash
 
@@ -12,14 +13,15 @@ start() {
     docker run --rm -d --name $NAME \
            -e "MINIO_ACCESS_KEY=$USER" \
            -e "MINIO_SECRET_KEY=$PASS" \
+           -p 127.0.0.1:${PORT}:9000 \
            minio/minio:edge server /data
     
     echo type=s3
     echo provider=Minio
     echo access_key_id=$USER
     echo secret_access_key=$PASS
-    echo endpoint=http://$(docker_ip):9000/
-    echo _connect=$(docker_ip):9000
+    echo endpoint=http://127.0.0.1:${PORT}/
+    echo _connect=127.0.0.1:${PORT}
 }
 
 . $(dirname "$0")/run.bash
diff --git a/fstest/testserver/init.d/TestSFTPOpenssh b/fstest/testserver/init.d/TestSFTPOpenssh
index 4eb14ac9a..5b67f7923 100755
--- a/fstest/testserver/init.d/TestSFTPOpenssh
+++ b/fstest/testserver/init.d/TestSFTPOpenssh
@@ -5,19 +5,22 @@ set -e
 NAME=rclone-sftp-openssh
 USER=rclone
 PASS=password
+PORT=28627
 
 . $(dirname "$0")/docker.bash
 
 start() {
     docker run --rm -d --name ${NAME} \
+           -p 127.0.0.1:${PORT}:22 \
            rclone/test-sftp-openssh
     
     echo type=sftp
-    echo host=$(docker_ip)
+    echo host=127.0.0.1
+    echo port=$PORT
     echo user=$USER
     echo pass=$(rclone obscure $PASS)
     echo copy_is_hardlink=true
-    echo _connect=$(docker_ip):22
+    echo _connect=127.0.0.1:${PORT}
 }
 
 . $(dirname "$0")/run.bash
diff --git a/fstest/testserver/init.d/TestSMB b/fstest/testserver/init.d/TestSMB
index 788b5a84a..56bf16552 100755
--- a/fstest/testserver/init.d/TestSMB
+++ b/fstest/testserver/init.d/TestSMB
@@ -6,11 +6,15 @@ NAME=smb
 USER=rclone
 PASS=GNF3Cqeu
 WORKGROUP=thepub
+PORT=28630
 
 . $(dirname "$0")/docker.bash
 
 start() {
-    docker run --rm -d --name $NAME dperson/samba \
+    docker run --rm -d --name $NAME \
+           -p 127.0.0.1:${PORT}:445 \
+           -p 127.0.0.1:${PORT}:445/udp \
+           dperson/samba \
            -p \
            -u "rclone;${PASS}" \
            -w "${WORKGROUP}" \
@@ -18,11 +22,12 @@ start() {
            -s "rclone;/rclone;yes;no;no;rclone"
     
     echo type=smb
-    echo host=$(docker_ip)
+    echo host=127.0.0.1
     echo user=$USER
+    echo port=$PORT
     echo pass=$(rclone obscure $PASS)
     echo domain=$WORKGROUP
-    echo _connect=$(docker_ip):139
+    echo _connect=127.0.0.1:${PORT}
 }
 
 . $(dirname "$0")/run.bash
diff --git a/fstest/testserver/init.d/TestSwiftAIO b/fstest/testserver/init.d/TestSwiftAIO
index 5192f2055..1f5ca8c70 100755
--- a/fstest/testserver/init.d/TestSwiftAIO
+++ b/fstest/testserver/init.d/TestSwiftAIO
@@ -3,19 +3,21 @@
 set -e
 
 NAME=swift-aio
+PORT=28628
 
 . $(dirname "$0")/docker.bash
 
 start() {
     docker run --rm -d --name ${NAME} \
+           -p 127.0.0.1:${PORT}:8080 \
            bouncestorage/swift-aio
     
     echo type=swift
     echo env_auth=false
     echo user=test:tester
     echo key=testing
-    echo auth=http://$(docker_ip):8080/auth/v1.0
-    echo _connect=$(docker_ip):8080
+    echo auth=http://127.0.0.1:${PORT}/auth/v1.0
+    echo _connect=127.0.0.1:${PORT}
 }
 
 . $(dirname "$0")/run.bash
diff --git a/fstest/testserver/init.d/TestWebdavNextcloud b/fstest/testserver/init.d/TestWebdavNextcloud
index 9cf391e38..1fe8c6e86 100755
--- a/fstest/testserver/init.d/TestWebdavNextcloud
+++ b/fstest/testserver/init.d/TestWebdavNextcloud
@@ -5,6 +5,7 @@ set -e
 NAME=nextcloud
 USER=rclone
 PASS=ArmorAbleMale6
+PORT=28629
 
 . $(dirname "$0")/docker.bash
 
@@ -14,14 +15,15 @@ start() {
            -e "NEXTCLOUD_ADMIN_USER=rclone" \
            -e "NEXTCLOUD_ADMIN_PASSWORD=$PASS" \
            -e "NEXTCLOUD_TRUSTED_DOMAINS=*.*.*.*" \
+           -p 127.0.0.1:${PORT}:80 \
            nextcloud:latest
     
     echo type=webdav
-    echo url=http://$(docker_ip)/remote.php/dav/files/$USER/
+    echo url=http://127.0.0.1:${PORT}/remote.php/dav/files/$USER/
     echo user=$USER
     echo pass=$(rclone obscure $PASS)
     echo vendor=nextcloud
-    echo _connect=$(docker_ip):80
+    echo _connect=127.0.0.1:${PORT}
 }
 
 . $(dirname "$0")/run.bash
diff --git a/fstest/testserver/testserver.go b/fstest/testserver/testserver.go
index 005096e85..9cc1354d2 100644
--- a/fstest/testserver/testserver.go
+++ b/fstest/testserver/testserver.go
@@ -103,14 +103,31 @@ func start(name string) error {
 	}
 	// If we got a _connect value then try to connect to it
 	const maxTries = 30
+	var rdBuf = make([]byte, 1)
 	for i := 1; i <= maxTries; i++ {
-		fs.Debugf(name, "Attempting to connect to %q try %d/%d", connect, i, maxTries)
-		conn, err := net.Dial("tcp", connect)
-		if err == nil {
-			_ = conn.Close()
-			return nil
+		if i != 0 {
+			time.Sleep(time.Second)
 		}
-		time.Sleep(time.Second)
+		fs.Debugf(name, "Attempting to connect to %q try %d/%d", connect, i, maxTries)
+		conn, err := net.DialTimeout("tcp", connect, time.Second)
+		if err != nil {
+			fs.Debugf(name, "Connection to %q failed try %d/%d: %v", connect, i, maxTries, err)
+			continue
+		}
+
+		err = conn.SetReadDeadline(time.Now().Add(time.Second))
+		if err != nil {
+			return fmt.Errorf("failed to set deadline: %w", err)
+		}
+		n, err := conn.Read(rdBuf)
+		_ = conn.Close()
+		fs.Debugf(name, "Read %d, error: %v", n, err)
+		if err != nil && !errors.Is(err, os.ErrDeadlineExceeded) {
+			// Try again
+			continue
+		}
+		//time.Sleep(30 * time.Second)
+		return nil
 	}
 	return fmt.Errorf("failed to connect to %q on %q", name, connect)
 }