Merge branch 'ctest_redux'

This fixes issues building on Debian and with older versions of CMake.
See individual commits for details.
This commit is contained in:
Mahmoud Al-Qudsi 2021-09-13 23:17:14 -05:00
commit 93aaa1851e
3 changed files with 135 additions and 62 deletions

View File

@ -35,7 +35,7 @@ if(POLICY CMP0037)
endif()
add_custom_target(test
COMMAND env CTEST_PARALLEL_LEVEL=${CTEST_PARALLEL_LEVEL} FISH_FORCE_COLOR=1
${CMAKE_CTEST_COMMAND} --force-new-ctest-process
${CMAKE_CTEST_COMMAND} --force-new-ctest-process # --verbose
--output-on-failure --progress
DEPENDS fish_tests tests_buildroot_target
USES_TERMINAL
@ -121,10 +121,19 @@ add_custom_target(tests_buildroot_target
${TEST_ROOT_DIR}
DEPENDS fish fish_test_helper)
# CMake less than 3.9.0 "fully supports" setting an exit code to denote a skipped test, but then
# it just goes ahead and reports it as failed. Really?
if(${CMAKE_VERSION} VERSION_LESS "3.9.0")
set(CMAKE_SKIPPED_HACK "env" "CMAKE_SKIPPED_HACK=1")
else()
set(CMAKE_SKIPPED_HACK)
endif()
foreach(LTEST ${LOW_LEVEL_TESTS})
add_test(
NAME ${LTEST}
COMMAND ${CMAKE_BINARY_DIR}/fish_tests ${LTEST}
COMMAND sh ${CMAKE_CURRENT_BINARY_DIR}/tests/test_env.sh
${CMAKE_BINARY_DIR}/fish_tests ${LTEST}
WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}
)
set_tests_properties(${LTEST} PROPERTIES SKIP_RETURN_CODE ${SKIP_RETURN_CODE})
@ -136,7 +145,7 @@ foreach(CHECK ${FISH_CHECKS})
get_filename_component(CHECK_NAME ${CHECK} NAME)
get_filename_component(CHECK ${CHECK} NAME_WE)
add_test(NAME ${CHECK_NAME}
COMMAND sh ${CMAKE_CURRENT_BINARY_DIR}/tests/test_driver.sh
COMMAND ${CMAKE_SKIPPED_HACK} sh ${CMAKE_CURRENT_BINARY_DIR}/tests/test_driver.sh
${CMAKE_CURRENT_BINARY_DIR}/tests/test.fish ${CHECK}
WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/tests
)
@ -148,7 +157,7 @@ FILE(GLOB PEXPECTS CONFIGURE_DEPENDS ${CMAKE_SOURCE_DIR}/tests/pexpects/*.py)
foreach(PEXPECT ${PEXPECTS})
get_filename_component(PEXPECT ${PEXPECT} NAME)
add_test(NAME ${PEXPECT}
COMMAND sh ${CMAKE_CURRENT_BINARY_DIR}/tests/test_driver.sh
COMMAND ${CMAKE_SKIPPED_HACK} sh ${CMAKE_CURRENT_BINARY_DIR}/tests/test_driver.sh
${CMAKE_CURRENT_BINARY_DIR}/tests/interactive.fish ${PEXPECT}
WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/tests
)

View File

@ -1,5 +1,11 @@
# vim: set ts=4 sw=4 tw=100 et:
# POSIX sh test driver to reduce dependency on fish in tests
# POSIX sh test driver to reduce dependency on fish in tests.
# Executes the specified *fish script* with the provided arguments, after setting up a clean test
# environment (see `test_env.sh`) and then importing the fish-related helper functions and
# performing some state initialization required by the various fish tests. Each payload script is
# executed in its own environment, this script waits for fish to exit then cleans up the target
# environment and bubbles up the fish exit code.
# macOS has really weird default IFS behavior that splits output in random places, and the trailing
# backspace is to prevent \n from being gobbled up by the subshell output substitution.
@ -11,6 +17,8 @@ IFS="$(printf "\n\b")"
fish_script="$1"
shift 1
script_args="${@}"
# Prevent $@ from persisting to sourced commands
shift $# 2>/dev/null
die() {
if test "$#" -ge 0; then
@ -20,7 +28,7 @@ die() {
}
# To keep things sane and to make error messages comprehensible, do not use relative paths anywhere
# in this script. Instead, make all paths relative to one of these or $homedir."
# in this script. Instead, make all paths relative to one of these or the new $HOME ($homedir)."
TESTS_ROOT="$(dirname "$0")"
BUILD_ROOT="${TESTS_ROOT}/.."
@ -35,30 +43,8 @@ if ! test -z "$__fish_is_running_tests"; then
exit 10
fi
# Set up a test environment and re-run the original script. We do not share environments
# whatsoever between tests, so each test driver run sets up a new profile altogether.
# macOS 10.10 requires an explicit template for `mktemp` and will create the folder in the
# current directory unless told otherwise. Linux isn't guaranteed to have $TMPDIR set.
homedir="$(mktemp -d 2>/dev/null || mktemp -d "${TMPDIR}tmp.XXXXXXXXXX")"
XDG_DATA_HOME="$homedir/xdg_data_home"
export XDG_DATA_HOME
mkdir -p $XDG_DATA_HOME/fish || die
XDG_CONFIG_HOME="$homedir/xdg_config_home"
export XDG_CONFIG_HOME
mkdir -p $XDG_CONFIG_HOME/fish || die
XDG_RUNTIME_DIR="$homedir/xdg_runtime_dir"
export XDG_CONFIG_HOME
mkdir -p $XDG_RUNTIME_DIR/fish || die
# Create a temp/scratch directory for tests to use, if they want (tests shouldn't write to a
# shared temp folder).
TMPDIR="$homedir/temp"
mkdir ${TMPDIR}
export TMPDIR
# Set up the test environment. Does not change the current working directory.
. ${TESTS_ROOT}/test_env.sh
# These are used read-only so it's OK to symlink instead of copy
rm -f "$XDG_CONFIG_HOME/fish/functions"
@ -71,37 +57,6 @@ fish_init_cmd="set fish_function_path ${XDG_CONFIG_HOME}/fish/functions ${BUILD_
__fish_is_running_tests="$homedir"
export __fish_is_running_tests
# Set locale information for consistent tests. Fish should work with a lot of locales but the
# tests assume an english UTF-8 locale unless they explicitly override this default. We do not
# want the users locale to affect the tests since they might, for example, change the wording of
# logged messages.
#
# TODO: set LANG to en_US.UTF-8 so we test the locale message conversions (i.e., gettext).
unset LANGUAGE
# Remove "LC_" env vars from the test environment
for key in $(env | grep -E "^LC_"| grep -oE "^[^=]+"); do
unset "$key"
done
# Set the desired lang/locale tests are hard-coded against
export LANG="C"
export LC_CTYPE="en_US.UTF-8"
# These env vars should not be inherited from the user environment because they can affect the
# behavior of the tests. So either remove them or set them to a known value.
# See also tests/interactive.fish.
export TERM=xterm
unset COLORTERM
unset INSIDE_EMACS
unset ITERM_PROFILE
unset KONSOLE_PROFILE_NAME
unset KONSOLE_VERSION
unset PANTHEON_TERMINAL_ID
unset TERM_PROGRAM
unset TERM_PROGRAM_VERSION
unset VTE_VERSION
unset WT_PROFILE_ID
unset XTERM_VERSION
# Set a marker to indicate whether colored output should be suppressed (used in `test_util.fish`)
suppress_color=""
if ! tty 0>&1 > /dev/null; then
@ -112,7 +67,7 @@ export suppress_color
# Source test util functions at startup
fish_init_cmd="${fish_init_cmd} && source ${TESTS_ROOT}/test_util.fish";
# Run the test script, but don't exec so we can do cleanup after it succeeds/fails. Each test is
# Run the test script, but don't exec so we can clean up after it succeeds/fails. Each test is
# launched directly within its TMPDIR, so that the fish tests themselves do not need to refer to
# TMPDIR (to ensure their output as displayed in case of failure by littlecheck is reproducible).
(cd $TMPDIR; env HOME="$homedir" "${BUILD_ROOT}/test/root/bin/fish" \
@ -120,5 +75,14 @@ fish_init_cmd="${fish_init_cmd} && source ${TESTS_ROOT}/test_util.fish";
"$fish_script" "$script_args")
test_status="$?"
# CMake less than 3.9.0 "fully supports" setting an exit code to denote a skipped test, but then
# it just goes ahead and reports them as failed anyway. Really?
if test -n $CMAKE_SKIPPED_HACK; then
if test $test_status -eq 125; then
echo "Overriding SKIPPED return code from test" 1>&2
test_status=0
fi
fi
rm -rf "$homedir"
exit "$test_status"

100
tests/test_env.sh Executable file
View File

@ -0,0 +1,100 @@
# vim: set ts=4 sw=4 tw=100 et:
# This script sets up a clean environment for a script or executable to execute in/under. It creates
# (and sets) $TMPDIR initialized to a clean & unique temporary directory, creates a new $HOME, sets
# the relevant XDG_* directories to point to subdirectories of that $HOME, cleans up any potentially
# problematic environment variables, executes the provided command, waits for it to exit, then
# cleans up the newly created environment before bubbling up the exit code. $PWD is not changed.
# If sourced instead of executed, sets up the environment as before but does not execute any payload
# and does not destroy the newly created environment.
# macOS has really weird default IFS behavior that splits output in random places, and the trailing
# backspace is to prevent \n from being gobbled up by the subshell output substitution.
# Folks, this is why you should use fish!
IFS="$(printf "\n\b")"
# set -ex
# The first argument is the path to the script to launch; all remaining arguments are forwarded to
# the script.
if test $# -gt 0; then
target="$1"
shift 1
target_args="${@}"
fi
die() {
if test "$#" -ge 0; then
printf "%s\n" "$@" 1>&2
fi
exit 1
}
# Set up a test environment to run the specified target under. We do not share environments
# whatsoever between tests, so each test driver run sets up a new profile altogether.
# macOS 10.10 requires an explicit template for `mktemp` and will create the folder in the
# current directory unless told otherwise. Linux isn't guaranteed to have $TMPDIR set.
homedir="$(mktemp -d 2>/dev/null || mktemp -d "${TMPDIR}tmp.XXXXXXXXXX")"
export HOME="$homedir"
XDG_DATA_HOME="$homedir/xdg_data_home"
export XDG_DATA_HOME
mkdir -p $XDG_DATA_HOME/fish || die
XDG_CONFIG_HOME="$homedir/xdg_config_home"
export XDG_CONFIG_HOME
mkdir -p $XDG_CONFIG_HOME/fish || die
XDG_RUNTIME_DIR="$homedir/xdg_runtime_dir"
export XDG_CONFIG_HOME
mkdir -p $XDG_RUNTIME_DIR/fish || die
# Create a temp/scratch directory for tests to use, if they want (tests shouldn't write to a
# shared temp folder).
TMPDIR="$homedir/temp"
mkdir ${TMPDIR}
export TMPDIR
# Set locale information for consistent tests. Fish should work with a lot of locales but the
# tests assume an english UTF-8 locale unless they explicitly override this default. We do not
# want the users locale to affect the tests since they might, for example, change the wording of
# logged messages.
#
# TODO: set LANG to en_US.UTF-8 so we test the locale message conversions (i.e., gettext).
unset LANGUAGE
# Remove "LC_" env vars from the test environment
for key in $(env | grep -E "^LC_"| grep -oE "^[^=]+"); do
unset "$key"
done
# Set the desired lang/locale tests are hard-coded against
export LANG="C"
export LC_CTYPE="en_US.UTF-8"
# These env vars should not be inherited from the user environment because they can affect the
# behavior of the tests. So either remove them or set them to a known value.
# See also tests/interactive.fish.
export TERM=xterm
unset COLORTERM
unset INSIDE_EMACS
unset ITERM_PROFILE
unset KONSOLE_PROFILE_NAME
unset KONSOLE_VERSION
unset PANTHEON_TERMINAL_ID
unset TERM_PROGRAM
unset TERM_PROGRAM_VERSION
unset VTE_VERSION
unset WT_PROFILE_ID
unset XTERM_VERSION
# If we are sourced, return without executing
if test -z ${target}; then
return 0
fi
echo "Proceeding with target execution"
# Otherwise execute target
("${target}" "${target_args}")
test_status="$?"
rm -rf "$homedir"
exit "$test_status"