github-mirror/rclone
2
0
Fork 0
mirror of https://github.com/rclone/rclone.git synced 2024-11-22 11:58:20 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

build: make integration tests run better on macOS and Windows

Browse Source 
This changes as many of the integraton tests as possible so that they
use port forwarding rather than the docker IP directly.

Using the docker IP directly does not work on macOS and Windows as the
docker images are running in a VM rather than a container.

This adds the PORTS.md document to document which port numbers we are
using for which service as they need to be unique.
This commit is contained in:
Nick Craig-Wood 2024-04-08 16:51:49 +01:00
parent 807a7dabaa
commit 8dc4c01209
8 changed files with 93 additions and 19 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

41
fstest/testserver/init.d/PORTS.md Normal file
View File

@ -0,0 +1,41 @@
# Ports for tests
All these tests need to run on a different port.
They should be bound to localhost so they are not accessible externally.
| Port | Test |
|:-----:|:----:|
| 88 | TestHdfs |
| 750 | TestHdfs |
| 8020 | TestHdfs |
| 8086 | TestSeafileV6 |
| 8087 | TestSeafile |
| 8088 | TestSeafileEncrypted |
| 9866 | TestHdfs |
| 28620 | TestWebdavRclone |
| 28621 | TestSFTPRclone |
| 28622 | TestFTPRclone |
| 28623 | TestSFTPRcloneSSH |
| 28624 | TestS3Rclone |
| 28625 | TestS3Minio |
| 28626 | TestS3MinioEdge |
| 28627 | TestSFTPOpenssh |
| 28628 | TestSwiftAIO |
| 28629 | TestWebdavNextcloud |
| 28630 | TestSMB |
| 28631 | TestFTPProftpd |
| 38081 | TestWebdavOwncloud |
## Non localhost tests
All these use `$(docker_ip)` which means they don't work on macOS or
Windows. It is proabably possible to make them work with some effort
but will require port forwarding a range of ports and configuring the
FTP server to only use that range of ports. The FTP server will likely
need know it is behind a NAT so it advertises the correct external IP.
- TestFTPProftpd
- TestFTPPureftpd
- TestFTPVsftpd
- TestFTPVsftpdTLS

6
fstest/testserver/init.d/TestS3Minio
View File

@ -5,6 +5,7 @@ set -e
NAME=minio
USER=rclone
PASS=AxedBodedGinger7
PORT=28625
. $(dirname "$0")/docker.bash
@ -12,14 +13,15 @@ start() {
docker run --rm -d --name $NAME \
-e "MINIO_ACCESS_KEY=$USER" \
-e "MINIO_SECRET_KEY=$PASS" \
-p 127.0.0.1:${PORT}:9000 \
minio/minio server /data
echo type=s3
echo provider=Minio
echo access_key_id=$USER
echo secret_access_key=$PASS
echo endpoint=http://$(docker_ip):9000/
echo _connect=$(docker_ip):9000
echo endpoint=http://127.0.0.1:${PORT}/
echo _connect=127.0.0.1:${PORT}
}
. $(dirname "$0")/run.bash

6
fstest/testserver/init.d/TestS3MinioEdge
View File

@ -5,6 +5,7 @@ set -e
NAME=minio-edge
USER=rclone
PASS=DeniseOxygenEiffel4
PORT=28626
. $(dirname "$0")/docker.bash
@ -12,14 +13,15 @@ start() {
docker run --rm -d --name $NAME \
-e "MINIO_ACCESS_KEY=$USER" \
-e "MINIO_SECRET_KEY=$PASS" \
-p 127.0.0.1:${PORT}:9000 \
minio/minio:edge server /data
echo type=s3
echo provider=Minio
echo access_key_id=$USER
echo secret_access_key=$PASS
echo endpoint=http://$(docker_ip):9000/
echo _connect=$(docker_ip):9000
echo endpoint=http://127.0.0.1:${PORT}/
echo _connect=127.0.0.1:${PORT}
}
. $(dirname "$0")/run.bash

7
fstest/testserver/init.d/TestSFTPOpenssh
View File

@ -5,19 +5,22 @@ set -e
NAME=rclone-sftp-openssh
USER=rclone
PASS=password
PORT=28627
. $(dirname "$0")/docker.bash
start() {
docker run --rm -d --name ${NAME} \
-p 127.0.0.1:${PORT}:22 \
rclone/test-sftp-openssh
echo type=sftp
echo host=$(docker_ip)
echo host=127.0.0.1
echo port=$PORT
echo user=$USER
echo pass=$(rclone obscure $PASS)
echo copy_is_hardlink=true
echo _connect=$(docker_ip):22
echo _connect=127.0.0.1:${PORT}
}
. $(dirname "$0")/run.bash

11
fstest/testserver/init.d/TestSMB
View File

@ -6,11 +6,15 @@ NAME=smb
USER=rclone
PASS=GNF3Cqeu
WORKGROUP=thepub
PORT=28630
. $(dirname "$0")/docker.bash
start() {
docker run --rm -d --name $NAME dperson/samba \
docker run --rm -d --name $NAME \
-p 127.0.0.1:${PORT}:445 \
-p 127.0.0.1:${PORT}:445/udp \
dperson/samba \
-p \
-u "rclone;${PASS}" \
-w "${WORKGROUP}" \
@ -18,11 +22,12 @@ start() {
-s "rclone;/rclone;yes;no;no;rclone"
echo type=smb
echo host=$(docker_ip)
echo host=127.0.0.1
echo user=$USER
echo port=$PORT
echo pass=$(rclone obscure $PASS)
echo domain=$WORKGROUP
echo _connect=$(docker_ip):139
echo _connect=127.0.0.1:${PORT}
}
. $(dirname "$0")/run.bash

6
fstest/testserver/init.d/TestSwiftAIO
View File

@ -3,19 +3,21 @@
set -e
NAME=swift-aio
PORT=28628
. $(dirname "$0")/docker.bash
start() {
docker run --rm -d --name ${NAME} \
-p 127.0.0.1:${PORT}:8080 \
bouncestorage/swift-aio
echo type=swift
echo env_auth=false
echo user=test:tester
echo key=testing
echo auth=http://$(docker_ip):8080/auth/v1.0
echo _connect=$(docker_ip):8080
echo auth=http://127.0.0.1:${PORT}/auth/v1.0
echo _connect=127.0.0.1:${PORT}
}
. $(dirname "$0")/run.bash

6
fstest/testserver/init.d/TestWebdavNextcloud
View File

@ -5,6 +5,7 @@ set -e
NAME=nextcloud
USER=rclone
PASS=ArmorAbleMale6
PORT=28629
. $(dirname "$0")/docker.bash
@ -14,14 +15,15 @@ start() {
-e "NEXTCLOUD_ADMIN_USER=rclone" \
-e "NEXTCLOUD_ADMIN_PASSWORD=$PASS" \
-e "NEXTCLOUD_TRUSTED_DOMAINS=*.*.*.*" \
-p 127.0.0.1:${PORT}:80 \
nextcloud:latest
echo type=webdav
echo url=http://$(docker_ip)/remote.php/dav/files/$USER/
echo url=http://127.0.0.1:${PORT}/remote.php/dav/files/$USER/
echo user=$USER
echo pass=$(rclone obscure $PASS)
echo vendor=nextcloud
echo _connect=$(docker_ip):80
echo _connect=127.0.0.1:${PORT}
}
. $(dirname "$0")/run.bash

29
fstest/testserver/testserver.go
View File

@ -103,14 +103,31 @@ func start(name string) error {
}
// If we got a _connect value then try to connect to it
const maxTries = 30
var rdBuf = make([]byte, 1)
for i := 1; i <= maxTries; i++ {
fs.Debugf(name, "Attempting to connect to %q try %d/%d", connect, i, maxTries)
conn, err := net.Dial("tcp", connect)
if err == nil {
_ = conn.Close()
return nil
if i != 0 {
time.Sleep(time.Second)
}
time.Sleep(time.Second)
fs.Debugf(name, "Attempting to connect to %q try %d/%d", connect, i, maxTries)
conn, err := net.DialTimeout("tcp", connect, time.Second)
if err != nil {
fs.Debugf(name, "Connection to %q failed try %d/%d: %v", connect, i, maxTries, err)
continue
}
err = conn.SetReadDeadline(time.Now().Add(time.Second))
if err != nil {
return fmt.Errorf("failed to set deadline: %w", err)
}
n, err := conn.Read(rdBuf)
_ = conn.Close()
fs.Debugf(name, "Read %d, error: %v", n, err)
if err != nil && !errors.Is(err, os.ErrDeadlineExceeded) {
// Try again
continue
}
//time.Sleep(30 * time.Second)
return nil
}
return fmt.Errorf("failed to connect to %q on %q", name, connect)
}