From a6383002c3945e38baffbdcf87edb8fd5dfa0da7 Mon Sep 17 00:00:00 2001 From: ridiculousfish Date: Sat, 21 Nov 2020 13:55:15 -0800 Subject: [PATCH] Introduce Docker-based build and test environments This is the start of an effort to make it easier to build and run tests in various Linux environments. The idea is to reduce our reliance on CI and also allow an easy to way capture tricky environments like musl or gcc 5. This adds two initial Dockerfiles corresponding to Ubuntu Bionic, and Ubuntu Bionic with Thread Sanitizer enabled. It also adds a new script `docker/docker_run_tests.sh`. An example of usage: docker/docker_run_tests.sh docker/bionic-tsan.Dockerfile When run, this builds a Docker image (which is cached after the first build) and sets its entry point to a new script `fish_run_tests.sh`. It then launches a container with that image, with a directory `/fish-source` bound to the fish-shell source directory on the host. Note it is a bind mount, not a copy, so changes to host files are instantly visible inside the container. It then configures with CMake and runs the tests. The Docker user is `fishuser` with password `fish`. The script also supports two arguments `--shell-before` and `--shell-after`. These drop the user into a bash shell before (or after) the tests are run, to aid in debugging. Note there's no automation for invoking this script yet; it must be run manually. But it runs on both Mac and Linux! --- docker/bionic-tsan.Dockerfile | 32 ++++++++++++++++++++ docker/bionic.Dockerfile | 31 +++++++++++++++++++ docker/context/fish_run_tests.sh | 18 +++++++++++ docker/docker_run_tests.sh | 51 ++++++++++++++++++++++++++++++++ 4 files changed, 132 insertions(+) create mode 100644 docker/bionic-tsan.Dockerfile create mode 100644 docker/bionic.Dockerfile create mode 100755 docker/context/fish_run_tests.sh create mode 100755 docker/docker_run_tests.sh diff --git a/docker/bionic-tsan.Dockerfile b/docker/bionic-tsan.Dockerfile new file mode 100644 index 000000000..d60c3c4f6 --- /dev/null +++ b/docker/bionic-tsan.Dockerfile @@ -0,0 +1,32 @@ +FROM ubuntu:18.04 + +ENV LANG C.UTF-8 +ENV LC_ALL C.UTF-8 +ENV CXXFLAGS "-fsanitize=thread" + +RUN apt-get update \ + && apt-get -y install \ + build-essential \ + cmake \ + gettext \ + git \ + libncurses5-dev \ + locales \ + ninja-build \ + python3 \ + python3-pexpect \ + && locale-gen en_US.UTF-8 + +RUN groupadd -g 1000 fishuser \ + && useradd -d /home/fishuser -m -u 1000 -g 1000 fishuser \ + && adduser fishuser sudo \ + && mkdir -p /home/fishuser/fish-build \ + && mkdir /fish-source \ + && chown -R fishuser:fishuser /home/fishuser /fish-source + +USER fishuser +WORKDIR /home/fishuser + +COPY fish_run_tests.sh / + +CMD /fish_run_tests.sh diff --git a/docker/bionic.Dockerfile b/docker/bionic.Dockerfile new file mode 100644 index 000000000..01d71b1d9 --- /dev/null +++ b/docker/bionic.Dockerfile @@ -0,0 +1,31 @@ +FROM ubuntu:18.04 + +ENV LANG C.UTF-8 +ENV LC_ALL C.UTF-8 + +RUN apt-get update \ + && apt-get -y install \ + build-essential \ + cmake \ + gettext \ + git \ + libncurses5-dev \ + locales \ + ninja-build \ + python3 \ + python3-pexpect \ + && locale-gen en_US.UTF-8 + +RUN groupadd -g 1000 fishuser \ + && useradd -d /home/fishuser -m -u 1000 -g 1000 fishuser \ + && adduser fishuser sudo \ + && mkdir -p /home/fishuser/fish-build \ + && mkdir /fish-source \ + && chown -R fishuser:fishuser /home/fishuser /fish-source + +USER fishuser +WORKDIR /home/fishuser + +COPY fish_run_tests.sh / + +CMD /fish_run_tests.sh diff --git a/docker/context/fish_run_tests.sh b/docker/context/fish_run_tests.sh new file mode 100755 index 000000000..32493a383 --- /dev/null +++ b/docker/context/fish_run_tests.sh @@ -0,0 +1,18 @@ +#!/bin/bash + +# This script is copied into the root directory of our Docker tests. +# It is the entry point for running Docker-based tests. + +cd ~/fish-build +cmake -G Ninja -DCMAKE_BUILD_TYPE=Debug /fish-source + +# Spawn a shell if FISH_RUN_SHELL_BEFORE_TESTS is set. +if test -n "$FISH_RUN_SHELL_BEFORE_TESTS" +then + bash -i || exit +fi + +ninja && ninja test + +# Drop the user into a shell if FISH_RUN_SHELL_AFTER_TESTS is set. +test -n "$FISH_RUN_SHELL_AFTER_TESTS" && exec bash -i diff --git a/docker/docker_run_tests.sh b/docker/docker_run_tests.sh new file mode 100755 index 000000000..a4adb8b36 --- /dev/null +++ b/docker/docker_run_tests.sh @@ -0,0 +1,51 @@ +#!/bin/bash + +usage() { + cat << EOF +Usage: $(basename $0) [--shell-before] [--shell-after] DOCKERFILE +Options: + --shell-before Before the tests start, run a bash shell + --shell-after After the tests end, run a bash shell +EOF + exit 1 +} + +DOCKER_EXTRA_ARGS="" + +# Exit on failure. +set -e + +# Get fish source directory. +FISH_SRC_DIR=$(cd "$( dirname "${BASH_SOURCE[0]}" )"/.. && pwd) + +# Parse args. +while [ $# -gt 1 ]; do + case "$1" in + --shell-before) + DOCKER_EXTRA_ARGS="$DOCKER_EXTRA_ARGS --env FISH_RUN_SHELL_BEFORE_TESTS=1" + ;; + --shell-after) + DOCKER_EXTRA_ARGS="$DOCKER_EXTRA_ARGS --env FISH_RUN_SHELL_AFTER_TESTS=1" + ;; + *) + usage + ;; + esac + shift +done + +DOCKERFILE=${@:$OPTIND:1} +test -n "$DOCKERFILE" || usage + +# Construct a docker image. +IMG_TAGNAME="fish_$(basename -s .Dockerfile "$DOCKERFILE")" +docker build \ + -t "$IMG_TAGNAME" \ + -f "$DOCKERFILE" \ + "$FISH_SRC_DIR"/docker/context/ + +# Run tests in it, allowing them to fail without failing this script. +docker run -it \ + --mount type=bind,source="$FISH_SRC_DIR",target=/fish-source,readonly \ + $DOCKER_EXTRA_ARGS \ + "$IMG_TAGNAME"