Run each test fully independently in own environment

cmake/Install.cmake

@@ -175,7 +175,7 @@ install(FILES fish.png DESTINATION ${rel_datadir}/pixmaps)
 
 # Group install targets into a InstallTargets folder
 set_property(TARGET build_fish_pc CHECK-FISH-BUILD-VERSION-FILE
-                    test_fishscript
+                    # test_fishscript
                     test_prep tests_buildroot_target
              PROPERTY FOLDER cmake/InstallTargets)
 

cmake/Tests.cmake

@@ -1,3 +1,14 @@
+# This adds ctest support to the project
+enable_testing()
+
+# By default, ctest runs tests serially
+if(NOT CTEST_PARALLEL_LEVEL)
+  include(ProcessorCount)
+  ProcessorCount(CORES)
+  message("CTEST_PARALLEL_LEVEL ${CORES}")
+  set(CTEST_PARALLEL_LEVEL ${CORES})
+endif()
+
 # Define fish_tests.
 add_executable(fish_tests EXCLUDE_FROM_ALL
                src/fish_tests.cpp)
@@ -6,6 +17,25 @@ fish_link_deps_and_sign(fish_tests)
 # The "test" directory.
 set(TEST_DIR ${CMAKE_CURRENT_BINARY_DIR}/test)
 
+# HACK: CMake is a configuration tool, not a build system. However, until CMake adds a way to
+# dynamically discover tests, our options are either this or resorting to sed/awk to parse the low
+# level tests source file to get the list of individual tests. Or to split each test into its own
+# source file.
+execute_process(
+  COMMAND ${CMAKE_CXX_COMPILER} ${CMAKE_SOURCE_DIR}/src/fish_tests.cpp
+                            -o ${CMAKE_BINARY_DIR}/fish_tests_list
+                            -I ${CMAKE_CURRENT_BINARY_DIR} # for config.h
+                            -lpthread
+                            # Strip actual dependency on fish code
+                            -Wl,-undefined,dynamic_lookup,--unresolved-symbols=ignore-all
+  WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}
+)
+execute_process(
+  COMMAND ./fish_tests_list --list
+  OUTPUT_FILE low_level_tests.txt
+  WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}
+)
+
 # The directory into which fish is installed.
 set(TEST_INSTALL_DIR ${TEST_DIR}/buildroot)
 
@@ -77,74 +107,25 @@ add_custom_target(test_prep
     DEPENDS tests_buildroot_target tests_dir
     USES_TERMINAL)
 
-# Define our individual tests.
+foreach(LTEST ${LOW_LEVEL_TESTS})
-# Each test is conceptually independent.
+  add_test(
-# However when running all tests, we want to run them serially for sanity's sake.
+    NAME ${LTEST}
-# So define both a normal target, and a serial variant which enforces ordering.
+    COMMAND ${CMAKE_BINARY_DIR}/fish_tests ${LTEST}
-foreach(TESTTYPE test serial_test)
-  add_custom_target(${TESTTYPE}_low_level
-    COMMAND env XDG_DATA_DIRS=
-                XDG_DATA_HOME=${CMAKE_CURRENT_BINARY_DIR}/test/xdg_data
-                XDG_CONFIG_HOME=${CMAKE_CURRENT_BINARY_DIR}/test/xdg_config
-                XDG_RUNTIME_DIR=${CMAKE_CURRENT_BINARY_DIR}/test/xdg_runtime
-                ./fish_tests
     WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}
-    DEPENDS fish_tests
+  )
-    USES_TERMINAL)
+endforeach(LTEST)
 
-  add_custom_target(${TESTTYPE}_fishscript
+add_test(test_prep
-                    COMMAND
+         "${CMAKE_COMMAND}" --build ${CMAKE_BINARY_DIR} --target test_prep)
-                        cd tests &&
+FILE(GLOB FISH_CHECKS CONFIGURE_DEPENDS ${CMAKE_SOURCE_DIR}/tests/checks/*.fish)
-                        env XDG_DATA_DIRS=
+foreach(CHECK ${FISH_CHECKS})
-                            XDG_DATA_HOME=${CMAKE_CURRENT_BINARY_DIR}/test/xdg_data
+  get_filename_component(CHECK_NAME ${CHECK} NAME)
-                            XDG_CONFIG_HOME=${CMAKE_CURRENT_BINARY_DIR}/test/xdg_config
+  get_filename_component(CHECK ${CHECK} NAME_WE)
-                            XDG_RUNTIME_DIR=${CMAKE_CURRENT_BINARY_DIR}/test/xdg_runtime
+  add_test(NAME ${CHECK_NAME}
-                        ${TEST_ROOT_DIR}/bin/fish test.fish
+    COMMAND sh ${CMAKE_CURRENT_BINARY_DIR}/tests/test_driver.sh
-                    DEPENDS test_prep
+               ${CMAKE_CURRENT_BINARY_DIR}/tests/test.fish ${CHECK}
-                    USES_TERMINAL)
+    WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/tests
+  )
 
-  add_custom_target(${TESTTYPE}_interactive
+  set_tests_properties(${LTEST} PROPERTIES DEPENDS test_prep)
-      COMMAND cd tests &&
+endforeach(CHECK)
-                env XDG_DATA_DIRS=
-                    XDG_DATA_HOME=${CMAKE_CURRENT_BINARY_DIR}/test/xdg_data
-                    XDG_CONFIG_HOME=${CMAKE_CURRENT_BINARY_DIR}/test/xdg_config
-                    XDG_RUNTIME_DIR=${CMAKE_CURRENT_BINARY_DIR}/test/xdg_runtime
-                ${TEST_ROOT_DIR}/bin/fish interactive.fish
-      DEPENDS test_prep
-      USES_TERMINAL)
-endforeach(TESTTYPE)
-
-# Now add a dependency chain between the serial versions.
-# This ensures they run in order.
-add_dependencies(serial_test_low_level test_prep)
-add_dependencies(serial_test_fishscript serial_test_low_level)
-add_dependencies(serial_test_interactive serial_test_fishscript)
-
-
-add_custom_target(serial_test_high_level
-                  DEPENDS serial_test_interactive serial_test_fishscript)
-
-# Create the 'test' target.
-# Set a policy so CMake stops complaining about the name 'test'.
-cmake_policy(PUSH)
-
-if(${CMAKE_VERSION} VERSION_LESS 3.11.0 AND POLICY CMP0037)
-  cmake_policy(SET CMP0037 OLD)
-endif()
-add_custom_target(test)
-cmake_policy(POP)
-add_dependencies(test serial_test_high_level)
-
-# Group test targets into a TestTargets folder
-set_property(TARGET test tests_dir
-                    test_low_level
-                    test_fishscript
-                    test_interactive
-                    test_fishscript test_prep
-                    tests_buildroot_target
-                    serial_test_high_level
-                    serial_test_low_level
-                    serial_test_fishscript
-                    serial_test_interactive
-                    symlink_functions
-             PROPERTY FOLDER cmake/TestTargets)

tests/test.fish

@@ -19,10 +19,6 @@ else
     set files_to_test checks/*.fish
 end
 
-# test_util handles the environment setup and then restarts us
-source test_util.fish (status -f) $argv
-or exit
-
 say -o cyan "Testing high level script functionality"
 
 set -g python (__fish_anypython)

tests/test_driver.sh

@@ -0,0 +1,115 @@
+# vim: set ts=4 sw=4 tw=100 et:
+# POSIX sh test driver to reduce dependency on fish in tests
+
+# macOS has really weird default IFS behavior that splits output in random places, and the trailing
+# backspace is to prevent \n from being gobbled up by the subshell output substitution.
+# Folks, this is why you should use fish!
+IFS="$(printf "\n\b")"
+
+# The first argument is the path to the script to launch; all remaining arguments are forwarded to
+# the script.
+fish_script="$1"
+shift 1
+script_args="${@}"
+
+die() {
+    if test "$#" -ge 0; then
+        printf "%s\n" "$@" 1>&2
+    fi
+    exit 1
+}
+
+# To keep things sane and to make error messages comprehensible, do not use relative paths anywhere
+# in this script. Instead, make all paths relative to one of these or $homedir."
+TESTS_ROOT="$(realpath "$(dirname "$0")")"
+BUILD_ROOT="$(realpath "${TESTS_ROOT}/..")"
+
+if ! test -z "$__fish_is_running_tests"; then
+    echo "Recursive test invocation detected!" 1>&2
+    exit 10
+fi
+
+# Set up a test environment and re-run the original script. We do not share environments
+# whatsoever between tests, so each test driver run sets up a new profile altogether.
+
+homedir="$(mktemp -d)"
+
+# cp -a ../test/xdg_data_home "$homedir/"
+XDG_DATA_HOME="$homedir/xdg_data_home"
+export XDG_DATA_HOME
+mkdir -p $XDG_DATA_HOME/fish || die
+
+# cp -a ../test/xdg_config_home "$homedir/"
+XDG_CONFIG_HOME="$homedir/xdg_config_home"
+export XDG_CONFIG_HOME
+mkdir -p $XDG_CONFIG_HOME/fish || die
+
+XDG_RUNTIME_DIR="$homedir/xdg_runtime_dir"
+export XDG_CONFIG_HOME
+mkdir -p $XDG_RUNTIME_DIR/fish || die
+
+# These are used read-only so it's OK to symlink instead of copy
+rm -f "$XDG_CONFIG_HOME/fish/functions"
+ln -s "$PWD/test_functions" "$XDG_CONFIG_HOME/fish/functions" || die "Failed to symlink"
+
+# Set the function path at startup, referencing the default fish functions and the test-specific
+# functions.
+fish_init_cmd="set fish_function_path ${XDG_CONFIG_HOME}/fish/functions ${BUILD_ROOT}/share/functions"
+
+__fish_is_running_tests="$homedir"
+export __fish_is_running_tests
+
+# Set locale information for consistent tests. Fish should work with a lot of locales but the
+# tests assume an english UTF-8 locale unless they explicitly override this default. We do not
+# want the users locale to affect the tests since they might, for example, change the wording of
+# logged messages.
+#
+# TODO: set LANG to en_US.UTF-8 so we test the locale message conversions (i.e., gettext).
+unset LANGUAGE
+# Remove "LC_" env vars from the test environment
+for key in $(env | grep -E "^LC_"| grep -oE "^[^=]+"); do
+    unset "$key"
+done
+# Set the desired lang/locale tests are hard-coded against
+export LANG="C"
+export LC_CTYPE="en_US.UTF-8"
+
+# These env vars should not be inherited from the user environment because they can affect the
+# behavior of the tests. So either remove them or set them to a known value.
+# See also tests/interactive.fish.
+export TERM=xterm
+unset COLORTERM
+unset INSIDE_EMACS
+unset ITERM_PROFILE
+unset KONSOLE_PROFILE_NAME
+unset KONSOLE_VERSION
+unset PANTHEON_TERMINAL_ID
+unset TERM_PROGRAM
+unset TERM_PROGRAM_VERSION
+unset VTE_VERSION
+unset WT_PROFILE_ID
+unset XTERM_VERSION
+
+# Set a marker to indicate whether colored output should be suppressed (used in `test_util.fish`)
+suppress_color=""
+if ! tty 0>&1 > /dev/null; then
+    suppress_color="yes"
+fi
+export suppress_color
+
+# Source test util functions at startup
+fish_init_cmd="${fish_init_cmd} && source ${TESTS_ROOT}/test_util.fish";
+
+# Run the test script, but don't exec so we can do cleanup after it succeeds/fails
+# echo $PWD
+# echo "BUILD_ROOT: $BUILD_ROOT"
+# echo "TESTS_ROOT: $TESTS_ROOT"
+env HOME=$homedir "${BUILD_ROOT}/test/root/bin/fish" \
+    --init-command "${fish_init_cmd}" \
+    "$fish_script" "$script_args"
+test_status="$?"
+
+echo test completed in $homedir with status $test_status
+
+rm -rf "$homedir"
+exit "$test_status"

tests/test_util.fish

@@ -1,94 +1,12 @@
 # vim: set ts=4 sw=4 tw=100 et:
 # Utilities for the test runners
 
-if test "$argv[1]" = (status -f)
-    echo 'test_util.fish requires sourcing script as argument to `source`' >&2
-    echo 'use `source test_util.fish (status -f); or exit`' >&2
-    status --print-stack-trace >&2
-    exit 1
-end
-
-# Any remaining arguments are passed back to test.fish
-set -l args_for_test_script
-if set -q argv[2]
-    set args_for_test_script $argv[2..-1]
-end
-
 function die
     set -q argv[1]; and echo $argv[1] >&2
     exit 1
 end
 
-# Check if we're running in the test environment.  If not, set it up and rerun fish with exec.  The
+# $suppress_color is set by `test_driver.sh` (via import of exported variables)
-# test is whether the special var __fish_is_running_tests exists and contains the same value as
-# XDG_CONFIG_HOME. It checks the value and not just the presence because we're going to delete the
-# config directory later if we're exiting successfully.
-if not set -q __fish_is_running_tests
-    # Set up our test environment and re-run the original script.
-    set -l script $argv[1]
-
-    cd (builtin realpath (dirname $script))
-    or die
-
-    set -lx XDG_DATA_HOME ../test/xdg_data_home
-    rm -rf $XDG_DATA_HOME/fish
-    mkdir -p $XDG_DATA_HOME/fish; or die
-
-    set -lx XDG_CONFIG_HOME ../test/xdg_config_home
-    rm -rf $XDG_CONFIG_HOME/fish
-    mkdir -p $XDG_CONFIG_HOME/fish; or die
-    ln -s $PWD/test_functions $XDG_CONFIG_HOME/fish/functions; or die
-
-    set -l escaped_parent (builtin realpath $PWD/.. | string escape); or die
-    set -l escaped_config (string escape -- $XDG_CONFIG_HOME/fish)
-    printf 'set fish_function_path \'%s/functions\' \'%s/share/functions\'\n' $escaped_config $escaped_parent >$XDG_CONFIG_HOME/fish/config.fish; or die
-    set -xl __fish_is_running_tests $XDG_CONFIG_HOME
-
-    # Set locale information for consistent tests. Fish should work with a lot of locales but the
-    # tests assume an english UTF-8 locale unless they explicitly override this default. We do not
-    # want the users locale to affect the tests since they might, for example, change the wording of
-    # logged messages.
-    #
-    # TODO: set LANG to en_US.UTF-8 so we test the locale message conversions (i.e., gettext).
-    set -e LANGUAGE
-    set -x LANG C
-    # Remove "LC_" env vars from the test environment.
-    for var in (set -xn)
-        string match -q 'LC_*' $var
-        and set -e $var
-    end
-    set -x LC_CTYPE en_US.UTF-8
-
-    # These env vars should not be inherited from the user environment because they can affect the
-    # behavior of the tests. So either remove them or set them to a known value.
-    # See also tests/interactive.fish.
-    set -gx TERM xterm
-    set -e COLORTERM
-    set -e INSIDE_EMACS
-    set -e ITERM_PROFILE
-    set -e KONSOLE_PROFILE_NAME
-    set -e KONSOLE_VERSION
-    set -e PANTHEON_TERMINAL_ID
-    set -e TERM_PROGRAM
-    set -e TERM_PROGRAM_VERSION
-    set -e VTE_VERSION
-    set -e WT_PROFILE_ID
-    set -e XTERM_VERSION
-
-    exec ../test/root/bin/fish $script $args_for_test_script
-    die 'exec failed'
-else if test "$__fish_is_running_tests" != "$XDG_CONFIG_HOME"
-    echo 'Something went wrong with the test runner.' >&2
-    echo "__fish_is_running_tests: $__fish_is_running_tests" >&2
-    echo "XDG_CONFIG_HOME: $XDG_CONFIG_HOME" >&2
-    exit 10
-end
-
-set -l suppress_color
-if not tty 0>&1 >/dev/null
-    set suppress_color yes
-end
-
 function say -V suppress_color
     set -l color_flags
     set -l suppress_newline