From 35975d83afd857b9a2c7d44c6a0f4a18e35531d0 Mon Sep 17 00:00:00 2001 From: Mahmoud Al-Qudsi Date: Mon, 15 Mar 2021 12:42:53 -0500 Subject: [PATCH] Run each test fully independently in own environment --- cmake/Install.cmake | 2 +- cmake/Tests.cmake | 117 ++++++++++++++++++------------------------- tests/test.fish | 4 -- tests/test_driver.sh | 115 ++++++++++++++++++++++++++++++++++++++++++ tests/test_util.fish | 84 +------------------------------ 5 files changed, 166 insertions(+), 156 deletions(-) create mode 100644 tests/test_driver.sh diff --git a/cmake/Install.cmake b/cmake/Install.cmake index 4edc8689c..36592bafc 100644 --- a/cmake/Install.cmake +++ b/cmake/Install.cmake @@ -175,7 +175,7 @@ install(FILES fish.png DESTINATION ${rel_datadir}/pixmaps) # Group install targets into a InstallTargets folder set_property(TARGET build_fish_pc CHECK-FISH-BUILD-VERSION-FILE - test_fishscript + # test_fishscript test_prep tests_buildroot_target PROPERTY FOLDER cmake/InstallTargets) diff --git a/cmake/Tests.cmake b/cmake/Tests.cmake index d4bfb9a17..85cd2228e 100644 --- a/cmake/Tests.cmake +++ b/cmake/Tests.cmake @@ -1,3 +1,14 @@ +# This adds ctest support to the project +enable_testing() + +# By default, ctest runs tests serially +if(NOT CTEST_PARALLEL_LEVEL) + include(ProcessorCount) + ProcessorCount(CORES) + message("CTEST_PARALLEL_LEVEL ${CORES}") + set(CTEST_PARALLEL_LEVEL ${CORES}) +endif() + # Define fish_tests. add_executable(fish_tests EXCLUDE_FROM_ALL src/fish_tests.cpp) @@ -6,6 +17,25 @@ fish_link_deps_and_sign(fish_tests) # The "test" directory. set(TEST_DIR ${CMAKE_CURRENT_BINARY_DIR}/test) +# HACK: CMake is a configuration tool, not a build system. However, until CMake adds a way to +# dynamically discover tests, our options are either this or resorting to sed/awk to parse the low +# level tests source file to get the list of individual tests. Or to split each test into its own +# source file. +execute_process( + COMMAND ${CMAKE_CXX_COMPILER} ${CMAKE_SOURCE_DIR}/src/fish_tests.cpp + -o ${CMAKE_BINARY_DIR}/fish_tests_list + -I ${CMAKE_CURRENT_BINARY_DIR} # for config.h + -lpthread + # Strip actual dependency on fish code + -Wl,-undefined,dynamic_lookup,--unresolved-symbols=ignore-all + WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} +) +execute_process( + COMMAND ./fish_tests_list --list + OUTPUT_FILE low_level_tests.txt + WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} +) + # The directory into which fish is installed. set(TEST_INSTALL_DIR ${TEST_DIR}/buildroot) @@ -77,74 +107,25 @@ add_custom_target(test_prep DEPENDS tests_buildroot_target tests_dir USES_TERMINAL) -# Define our individual tests. -# Each test is conceptually independent. -# However when running all tests, we want to run them serially for sanity's sake. -# So define both a normal target, and a serial variant which enforces ordering. -foreach(TESTTYPE test serial_test) - add_custom_target(${TESTTYPE}_low_level - COMMAND env XDG_DATA_DIRS= - XDG_DATA_HOME=${CMAKE_CURRENT_BINARY_DIR}/test/xdg_data - XDG_CONFIG_HOME=${CMAKE_CURRENT_BINARY_DIR}/test/xdg_config - XDG_RUNTIME_DIR=${CMAKE_CURRENT_BINARY_DIR}/test/xdg_runtime - ./fish_tests +foreach(LTEST ${LOW_LEVEL_TESTS}) + add_test( + NAME ${LTEST} + COMMAND ${CMAKE_BINARY_DIR}/fish_tests ${LTEST} WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} - DEPENDS fish_tests - USES_TERMINAL) + ) +endforeach(LTEST) - add_custom_target(${TESTTYPE}_fishscript - COMMAND - cd tests && - env XDG_DATA_DIRS= - XDG_DATA_HOME=${CMAKE_CURRENT_BINARY_DIR}/test/xdg_data - XDG_CONFIG_HOME=${CMAKE_CURRENT_BINARY_DIR}/test/xdg_config - XDG_RUNTIME_DIR=${CMAKE_CURRENT_BINARY_DIR}/test/xdg_runtime - ${TEST_ROOT_DIR}/bin/fish test.fish - DEPENDS test_prep - USES_TERMINAL) +add_test(test_prep + "${CMAKE_COMMAND}" --build ${CMAKE_BINARY_DIR} --target test_prep) +FILE(GLOB FISH_CHECKS CONFIGURE_DEPENDS ${CMAKE_SOURCE_DIR}/tests/checks/*.fish) +foreach(CHECK ${FISH_CHECKS}) + get_filename_component(CHECK_NAME ${CHECK} NAME) + get_filename_component(CHECK ${CHECK} NAME_WE) + add_test(NAME ${CHECK_NAME} + COMMAND sh ${CMAKE_CURRENT_BINARY_DIR}/tests/test_driver.sh + ${CMAKE_CURRENT_BINARY_DIR}/tests/test.fish ${CHECK} + WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/tests + ) - add_custom_target(${TESTTYPE}_interactive - COMMAND cd tests && - env XDG_DATA_DIRS= - XDG_DATA_HOME=${CMAKE_CURRENT_BINARY_DIR}/test/xdg_data - XDG_CONFIG_HOME=${CMAKE_CURRENT_BINARY_DIR}/test/xdg_config - XDG_RUNTIME_DIR=${CMAKE_CURRENT_BINARY_DIR}/test/xdg_runtime - ${TEST_ROOT_DIR}/bin/fish interactive.fish - DEPENDS test_prep - USES_TERMINAL) -endforeach(TESTTYPE) - -# Now add a dependency chain between the serial versions. -# This ensures they run in order. -add_dependencies(serial_test_low_level test_prep) -add_dependencies(serial_test_fishscript serial_test_low_level) -add_dependencies(serial_test_interactive serial_test_fishscript) - - -add_custom_target(serial_test_high_level - DEPENDS serial_test_interactive serial_test_fishscript) - -# Create the 'test' target. -# Set a policy so CMake stops complaining about the name 'test'. -cmake_policy(PUSH) - -if(${CMAKE_VERSION} VERSION_LESS 3.11.0 AND POLICY CMP0037) - cmake_policy(SET CMP0037 OLD) -endif() -add_custom_target(test) -cmake_policy(POP) -add_dependencies(test serial_test_high_level) - -# Group test targets into a TestTargets folder -set_property(TARGET test tests_dir - test_low_level - test_fishscript - test_interactive - test_fishscript test_prep - tests_buildroot_target - serial_test_high_level - serial_test_low_level - serial_test_fishscript - serial_test_interactive - symlink_functions - PROPERTY FOLDER cmake/TestTargets) + set_tests_properties(${LTEST} PROPERTIES DEPENDS test_prep) +endforeach(CHECK) diff --git a/tests/test.fish b/tests/test.fish index a508e374b..5814f8413 100644 --- a/tests/test.fish +++ b/tests/test.fish @@ -19,10 +19,6 @@ else set files_to_test checks/*.fish end -# test_util handles the environment setup and then restarts us -source test_util.fish (status -f) $argv -or exit - say -o cyan "Testing high level script functionality" set -g python (__fish_anypython) diff --git a/tests/test_driver.sh b/tests/test_driver.sh new file mode 100644 index 000000000..f5901fbd4 --- /dev/null +++ b/tests/test_driver.sh @@ -0,0 +1,115 @@ +# vim: set ts=4 sw=4 tw=100 et: +# POSIX sh test driver to reduce dependency on fish in tests + +# macOS has really weird default IFS behavior that splits output in random places, and the trailing +# backspace is to prevent \n from being gobbled up by the subshell output substitution. +# Folks, this is why you should use fish! +IFS="$(printf "\n\b")" + +# The first argument is the path to the script to launch; all remaining arguments are forwarded to +# the script. +fish_script="$1" +shift 1 +script_args="${@}" + +die() { + if test "$#" -ge 0; then + printf "%s\n" "$@" 1>&2 + fi + exit 1 +} + +# To keep things sane and to make error messages comprehensible, do not use relative paths anywhere +# in this script. Instead, make all paths relative to one of these or $homedir." +TESTS_ROOT="$(realpath "$(dirname "$0")")" +BUILD_ROOT="$(realpath "${TESTS_ROOT}/..")" + +if ! test -z "$__fish_is_running_tests"; then + echo "Recursive test invocation detected!" 1>&2 + exit 10 +fi + +# Set up a test environment and re-run the original script. We do not share environments +# whatsoever between tests, so each test driver run sets up a new profile altogether. + +homedir="$(mktemp -d)" + +# cp -a ../test/xdg_data_home "$homedir/" +XDG_DATA_HOME="$homedir/xdg_data_home" +export XDG_DATA_HOME +mkdir -p $XDG_DATA_HOME/fish || die + +# cp -a ../test/xdg_config_home "$homedir/" +XDG_CONFIG_HOME="$homedir/xdg_config_home" +export XDG_CONFIG_HOME +mkdir -p $XDG_CONFIG_HOME/fish || die + +XDG_RUNTIME_DIR="$homedir/xdg_runtime_dir" +export XDG_CONFIG_HOME +mkdir -p $XDG_RUNTIME_DIR/fish || die + +# These are used read-only so it's OK to symlink instead of copy +rm -f "$XDG_CONFIG_HOME/fish/functions" +ln -s "$PWD/test_functions" "$XDG_CONFIG_HOME/fish/functions" || die "Failed to symlink" + +# Set the function path at startup, referencing the default fish functions and the test-specific +# functions. +fish_init_cmd="set fish_function_path ${XDG_CONFIG_HOME}/fish/functions ${BUILD_ROOT}/share/functions" + +__fish_is_running_tests="$homedir" +export __fish_is_running_tests + +# Set locale information for consistent tests. Fish should work with a lot of locales but the +# tests assume an english UTF-8 locale unless they explicitly override this default. We do not +# want the users locale to affect the tests since they might, for example, change the wording of +# logged messages. +# +# TODO: set LANG to en_US.UTF-8 so we test the locale message conversions (i.e., gettext). +unset LANGUAGE +# Remove "LC_" env vars from the test environment +for key in $(env | grep -E "^LC_"| grep -oE "^[^=]+"); do + unset "$key" +done +# Set the desired lang/locale tests are hard-coded against +export LANG="C" +export LC_CTYPE="en_US.UTF-8" + +# These env vars should not be inherited from the user environment because they can affect the +# behavior of the tests. So either remove them or set them to a known value. +# See also tests/interactive.fish. +export TERM=xterm +unset COLORTERM +unset INSIDE_EMACS +unset ITERM_PROFILE +unset KONSOLE_PROFILE_NAME +unset KONSOLE_VERSION +unset PANTHEON_TERMINAL_ID +unset TERM_PROGRAM +unset TERM_PROGRAM_VERSION +unset VTE_VERSION +unset WT_PROFILE_ID +unset XTERM_VERSION + +# Set a marker to indicate whether colored output should be suppressed (used in `test_util.fish`) +suppress_color="" +if ! tty 0>&1 > /dev/null; then + suppress_color="yes" +fi +export suppress_color + +# Source test util functions at startup +fish_init_cmd="${fish_init_cmd} && source ${TESTS_ROOT}/test_util.fish"; + +# Run the test script, but don't exec so we can do cleanup after it succeeds/fails +# echo $PWD +# echo "BUILD_ROOT: $BUILD_ROOT" +# echo "TESTS_ROOT: $TESTS_ROOT" +env HOME=$homedir "${BUILD_ROOT}/test/root/bin/fish" \ + --init-command "${fish_init_cmd}" \ + "$fish_script" "$script_args" +test_status="$?" + +echo test completed in $homedir with status $test_status + +rm -rf "$homedir" +exit "$test_status" diff --git a/tests/test_util.fish b/tests/test_util.fish index 12811c63b..f5fa338ee 100644 --- a/tests/test_util.fish +++ b/tests/test_util.fish @@ -1,94 +1,12 @@ # vim: set ts=4 sw=4 tw=100 et: # Utilities for the test runners -if test "$argv[1]" = (status -f) - echo 'test_util.fish requires sourcing script as argument to `source`' >&2 - echo 'use `source test_util.fish (status -f); or exit`' >&2 - status --print-stack-trace >&2 - exit 1 -end - -# Any remaining arguments are passed back to test.fish -set -l args_for_test_script -if set -q argv[2] - set args_for_test_script $argv[2..-1] -end - function die set -q argv[1]; and echo $argv[1] >&2 exit 1 end -# Check if we're running in the test environment. If not, set it up and rerun fish with exec. The -# test is whether the special var __fish_is_running_tests exists and contains the same value as -# XDG_CONFIG_HOME. It checks the value and not just the presence because we're going to delete the -# config directory later if we're exiting successfully. -if not set -q __fish_is_running_tests - # Set up our test environment and re-run the original script. - set -l script $argv[1] - - cd (builtin realpath (dirname $script)) - or die - - set -lx XDG_DATA_HOME ../test/xdg_data_home - rm -rf $XDG_DATA_HOME/fish - mkdir -p $XDG_DATA_HOME/fish; or die - - set -lx XDG_CONFIG_HOME ../test/xdg_config_home - rm -rf $XDG_CONFIG_HOME/fish - mkdir -p $XDG_CONFIG_HOME/fish; or die - ln -s $PWD/test_functions $XDG_CONFIG_HOME/fish/functions; or die - - set -l escaped_parent (builtin realpath $PWD/.. | string escape); or die - set -l escaped_config (string escape -- $XDG_CONFIG_HOME/fish) - printf 'set fish_function_path \'%s/functions\' \'%s/share/functions\'\n' $escaped_config $escaped_parent >$XDG_CONFIG_HOME/fish/config.fish; or die - set -xl __fish_is_running_tests $XDG_CONFIG_HOME - - # Set locale information for consistent tests. Fish should work with a lot of locales but the - # tests assume an english UTF-8 locale unless they explicitly override this default. We do not - # want the users locale to affect the tests since they might, for example, change the wording of - # logged messages. - # - # TODO: set LANG to en_US.UTF-8 so we test the locale message conversions (i.e., gettext). - set -e LANGUAGE - set -x LANG C - # Remove "LC_" env vars from the test environment. - for var in (set -xn) - string match -q 'LC_*' $var - and set -e $var - end - set -x LC_CTYPE en_US.UTF-8 - - # These env vars should not be inherited from the user environment because they can affect the - # behavior of the tests. So either remove them or set them to a known value. - # See also tests/interactive.fish. - set -gx TERM xterm - set -e COLORTERM - set -e INSIDE_EMACS - set -e ITERM_PROFILE - set -e KONSOLE_PROFILE_NAME - set -e KONSOLE_VERSION - set -e PANTHEON_TERMINAL_ID - set -e TERM_PROGRAM - set -e TERM_PROGRAM_VERSION - set -e VTE_VERSION - set -e WT_PROFILE_ID - set -e XTERM_VERSION - - exec ../test/root/bin/fish $script $args_for_test_script - die 'exec failed' -else if test "$__fish_is_running_tests" != "$XDG_CONFIG_HOME" - echo 'Something went wrong with the test runner.' >&2 - echo "__fish_is_running_tests: $__fish_is_running_tests" >&2 - echo "XDG_CONFIG_HOME: $XDG_CONFIG_HOME" >&2 - exit 10 -end - -set -l suppress_color -if not tty 0>&1 >/dev/null - set suppress_color yes -end - +# $suppress_color is set by `test_driver.sh` (via import of exported variables) function say -V suppress_color set -l color_flags set -l suppress_newline