build: make integration tests run better on macOS and Windows

This changes as many of the integraton tests as possible so that they
use port forwarding rather than the docker IP directly.

Using the docker IP directly does not work on macOS and Windows as the
docker images are running in a VM rather than a container.

This adds the PORTS.md document to document which port numbers we are
using for which service as they need to be unique.

fstest/testserver/init.d/PORTS.md

@@ -0,0 +1,41 @@
+# Ports for tests
+
+All these tests need to run on a different port.
+
+They should be bound to localhost so they are not accessible externally.
+
+| Port  | Test |
+|:-----:|:----:|
+|    88 | TestHdfs |
+|   750 | TestHdfs |
+|  8020 | TestHdfs |
+|  8086 | TestSeafileV6 |
+|  8087 | TestSeafile |
+|  8088 | TestSeafileEncrypted |
+|  9866 | TestHdfs |
+| 28620 | TestWebdavRclone |
+| 28621 | TestSFTPRclone |
+| 28622 | TestFTPRclone |
+| 28623 | TestSFTPRcloneSSH |
+| 28624 | TestS3Rclone |
+| 28625 | TestS3Minio |
+| 28626 | TestS3MinioEdge |
+| 28627 | TestSFTPOpenssh |
+| 28628 | TestSwiftAIO |
+| 28629 | TestWebdavNextcloud |
+| 28630 | TestSMB |
+| 28631 | TestFTPProftpd |
+| 38081 | TestWebdavOwncloud |
+
+## Non localhost tests
+
+All these use `$(docker_ip)` which means they don't work on macOS or
+Windows. It is proabably possible to make them work with some effort
+but will require port forwarding a range of ports and configuring the
+FTP server to only use that range of ports. The FTP server will likely
+need know it is behind a NAT so it advertises the correct external IP.
+
+- TestFTPProftpd
+- TestFTPPureftpd
+- TestFTPVsftpd
+- TestFTPVsftpdTLS

fstest/testserver/init.d/TestS3Minio

@@ -5,6 +5,7 @@ set -e
 NAME=minio
 USER=rclone
 PASS=AxedBodedGinger7
+PORT=28625
 
 . $(dirname "$0")/docker.bash
 
@@ -12,14 +13,15 @@ start() {
     docker run --rm -d --name $NAME \
            -e "MINIO_ACCESS_KEY=$USER" \
            -e "MINIO_SECRET_KEY=$PASS" \
+           -p 127.0.0.1:${PORT}:9000 \
            minio/minio server /data
     
     echo type=s3
     echo provider=Minio
     echo access_key_id=$USER
     echo secret_access_key=$PASS
-    echo endpoint=http://$(docker_ip):9000/
+    echo endpoint=http://127.0.0.1:${PORT}/
-    echo _connect=$(docker_ip):9000
+    echo _connect=127.0.0.1:${PORT}
 }
 
 . $(dirname "$0")/run.bash

fstest/testserver/init.d/TestS3MinioEdge

@@ -5,6 +5,7 @@ set -e
 NAME=minio-edge
 USER=rclone
 PASS=DeniseOxygenEiffel4
+PORT=28626
 
 . $(dirname "$0")/docker.bash
 
@@ -12,14 +13,15 @@ start() {
     docker run --rm -d --name $NAME \
            -e "MINIO_ACCESS_KEY=$USER" \
            -e "MINIO_SECRET_KEY=$PASS" \
+           -p 127.0.0.1:${PORT}:9000 \
            minio/minio:edge server /data
     
     echo type=s3
     echo provider=Minio
     echo access_key_id=$USER
     echo secret_access_key=$PASS
-    echo endpoint=http://$(docker_ip):9000/
+    echo endpoint=http://127.0.0.1:${PORT}/
-    echo _connect=$(docker_ip):9000
+    echo _connect=127.0.0.1:${PORT}
 }
 
 . $(dirname "$0")/run.bash

fstest/testserver/init.d/TestSFTPOpenssh

@@ -5,19 +5,22 @@ set -e
 NAME=rclone-sftp-openssh
 USER=rclone
 PASS=password
+PORT=28627
 
 . $(dirname "$0")/docker.bash
 
 start() {
     docker run --rm -d --name ${NAME} \
+           -p 127.0.0.1:${PORT}:22 \
            rclone/test-sftp-openssh
     
     echo type=sftp
-    echo host=$(docker_ip)
+    echo host=127.0.0.1
+    echo port=$PORT
     echo user=$USER
     echo pass=$(rclone obscure $PASS)
     echo copy_is_hardlink=true
-    echo _connect=$(docker_ip):22
+    echo _connect=127.0.0.1:${PORT}
 }
 
 . $(dirname "$0")/run.bash

fstest/testserver/init.d/TestSMB

@@ -6,11 +6,15 @@ NAME=smb
 USER=rclone
 PASS=GNF3Cqeu
 WORKGROUP=thepub
+PORT=28630
 
 . $(dirname "$0")/docker.bash
 
 start() {
-    docker run --rm -d --name $NAME dperson/samba \
+    docker run --rm -d --name $NAME \
+           -p 127.0.0.1:${PORT}:445 \
+           -p 127.0.0.1:${PORT}:445/udp \
+           dperson/samba \
            -p \
            -u "rclone;${PASS}" \
            -w "${WORKGROUP}" \
@@ -18,11 +22,12 @@ start() {
            -s "rclone;/rclone;yes;no;no;rclone"
     
     echo type=smb
-    echo host=$(docker_ip)
+    echo host=127.0.0.1
     echo user=$USER
+    echo port=$PORT
     echo pass=$(rclone obscure $PASS)
     echo domain=$WORKGROUP
-    echo _connect=$(docker_ip):139
+    echo _connect=127.0.0.1:${PORT}
 }
 
 . $(dirname "$0")/run.bash

fstest/testserver/init.d/TestSwiftAIO

@@ -3,19 +3,21 @@
 set -e
 
 NAME=swift-aio
+PORT=28628
 
 . $(dirname "$0")/docker.bash
 
 start() {
     docker run --rm -d --name ${NAME} \
+           -p 127.0.0.1:${PORT}:8080 \
            bouncestorage/swift-aio
     
     echo type=swift
     echo env_auth=false
     echo user=test:tester
     echo key=testing
-    echo auth=http://$(docker_ip):8080/auth/v1.0
+    echo auth=http://127.0.0.1:${PORT}/auth/v1.0
-    echo _connect=$(docker_ip):8080
+    echo _connect=127.0.0.1:${PORT}
 }
 
 . $(dirname "$0")/run.bash

fstest/testserver/init.d/TestWebdavNextcloud

@@ -5,6 +5,7 @@ set -e
 NAME=nextcloud
 USER=rclone
 PASS=ArmorAbleMale6
+PORT=28629
 
 . $(dirname "$0")/docker.bash
 
@@ -14,14 +15,15 @@ start() {
            -e "NEXTCLOUD_ADMIN_USER=rclone" \
            -e "NEXTCLOUD_ADMIN_PASSWORD=$PASS" \
            -e "NEXTCLOUD_TRUSTED_DOMAINS=*.*.*.*" \
+           -p 127.0.0.1:${PORT}:80 \
            nextcloud:latest
     
     echo type=webdav
-    echo url=http://$(docker_ip)/remote.php/dav/files/$USER/
+    echo url=http://127.0.0.1:${PORT}/remote.php/dav/files/$USER/
     echo user=$USER
     echo pass=$(rclone obscure $PASS)
     echo vendor=nextcloud
-    echo _connect=$(docker_ip):80
+    echo _connect=127.0.0.1:${PORT}
 }
 
 . $(dirname "$0")/run.bash

fstest/testserver/testserver.go

@@ -103,14 +103,31 @@ func start(name string) error {
 	}
 	// If we got a _connect value then try to connect to it
 	const maxTries = 30
+	var rdBuf = make([]byte, 1)
 	for i := 1; i <= maxTries; i++ {
-		fs.Debugf(name, "Attempting to connect to %q try %d/%d", connect, i, maxTries)
+		if i != 0 {
-		conn, err := net.Dial("tcp", connect)
+			time.Sleep(time.Second)
-		if err == nil {
-			_ = conn.Close()
-			return nil
 		}
-		time.Sleep(time.Second)
+		fs.Debugf(name, "Attempting to connect to %q try %d/%d", connect, i, maxTries)
+		conn, err := net.DialTimeout("tcp", connect, time.Second)
+		if err != nil {
+			fs.Debugf(name, "Connection to %q failed try %d/%d: %v", connect, i, maxTries, err)
+			continue
+		}
+
+		err = conn.SetReadDeadline(time.Now().Add(time.Second))
+		if err != nil {
+			return fmt.Errorf("failed to set deadline: %w", err)
+		}
+		n, err := conn.Read(rdBuf)
+		_ = conn.Close()
+		fs.Debugf(name, "Read %d, error: %v", n, err)
+		if err != nil && !errors.Is(err, os.ErrDeadlineExceeded) {
+			// Try again
+			continue
+		}
+		//time.Sleep(30 * time.Second)
+		return nil
 	}
 	return fmt.Errorf("failed to connect to %q on %q", name, connect)
 }