Add pexpect-based interactive testing framework

This adds a new interactive test framework based on Python's pexpect. This
is intended to supplant the TCL expect-based tests.

New tests go in `tests/pexpects/`. As a proof-of-concept, the
pipeline.expect test and the (gnarly) bind.expect test are ported to the
new framework.

build_tools/pexpect_helper.py

@@ -0,0 +1,261 @@
+"""pexpect_helper provides a wrapper around the pexpect module.
+
+This module exposes a single class SpawnedProc, which wraps pexpect.spawn().
+This exposes a pseudo-tty, which fish or another process may talk to.
+The send() function may be used to send data to fish, and the expect_* family
+of functions may be used to match what is output to the tty.
+
+Example usage:
+  sp = SpawnedProc() # this launches fish
+  sp.expect_prompt() # wait for a prompt
+  sp.sendline("echo hello world")
+  sp.expect_prompt("hello world")
+
+"""
+
+from __future__ import print_function
+
+import inspect
+import os
+import os.path
+import re
+import sys
+import time
+import pexpect
+
+# Default timeout for failing to match.
+TIMEOUT_SECS = 5
+
+
+def get_prompt_re(counter):
+    """ Return a regular expression for matching a with a given prompt counter. """
+    return re.compile(
+        r"""(?:\r\n?|^)   # beginning of line
+            (?:\[.\]\ )?  # optional vi mode prompt
+         """
+        + (r"prompt\ %d>" % counter),  # prompt with counter
+        re.VERBOSE,
+    )
+
+
+def get_callsite():
+    """ Return a triple (filename, line_number, line_text) of the call site location. """
+    callstack = inspect.getouterframes(inspect.currentframe())
+    for f in callstack:
+        if inspect.getmodule(f.frame) is not Message.MODULE:
+            return (os.path.basename(f.filename), f.lineno, f.code_context)
+    return ("Unknown", -1, "")
+
+
+def escape(s):
+    """ Escape the string 's' to make it human-understandable. """
+    res = []
+    for c in s:
+        if c == "\n":
+            res.append("\\n")
+        elif c == "\r":
+            res.append("\\r")
+        elif c == "\t":
+            res.append("\\t")
+        elif c.isprintable():
+            res.append(c)
+        else:
+            res.append("\\x{:02x}".format(ord(c)))
+    return "".join(res)
+
+
+class Message(object):
+    """ Some text either sent-to or received-from the spawned proc.
+
+    Attributes:
+        dir: the message direction, either DIR_SEND or DIR_RECV
+        filename: the name of the file from which the message was sent
+        text: the text of the messages
+        when: a timestamp of when the message was sent
+    """
+
+    DIR_SEND = "SENT"
+    DIR_RECV = "RECV"
+    MODULE = sys.modules[__name__]
+
+    def __init__(self, dir, text, when):
+        """ Construct from a direction, message text and timestamp. """
+        self.dir = dir
+        self.filename, self.lineno, _ = get_callsite()
+        self.text = text
+        self.when = when
+
+    @staticmethod
+    def sent(text, when):
+        """ Return a SEND message with the given text. """
+        return Message(Message.DIR_SEND, text, when)
+
+    @staticmethod
+    def received(text, when):
+        """ Return a RECV message with the given text. """
+        return Message(Message.DIR_RECV, text, when)
+
+    def formatted(self):
+        """ Return a human-readable string representing this message. """
+        etext = escape(self.text)
+        timestamp = self.when * 1000.0
+        return "{dir} {timestamp:.2f} ({filename}:{lineno}): {etext}".format(
+            timestamp=timestamp, etext=etext, **vars(self)
+        )
+
+
+class SpawnedProc(object):
+    """ A process, talking to our ptty. This wraps pexpect.spawn.
+
+    Attributes:
+        colorize: whether error messages should have ANSI color escapes
+        messages: list of Message sent and received, in-order
+        start_time: the timestamp of the first message, or None if none yet
+        spawn: the pexpect.spawn value
+        prompt_counter: the index of the prompt. This cooperates with the fish_prompt
+            function to ensure that each printed prompt is distinct.
+    """
+
+    def __init__(self, name="fish", timeout=TIMEOUT_SECS, env=os.environ.copy()):
+        """ Construct from a name, timeout, and environment.
+
+            Args:
+                name: the name of the executable to launch, as a key into the
+                      environment dictionary. By default this is 'fish' but may be
+                      other executables.
+                timeout: A timeout to pass to pexpect. This indicates how long to wait
+                         before giving up on some expected output.
+                env: a string->string dictionary, describing the environment variables.
+        """
+        if name not in env:
+            raise ValueError("'name' variable not found in environment" % name)
+        exe_path = env.get(name)
+        self.colorize = sys.stdout.isatty()
+        self.messages = []
+        self.start_time = None
+        self.spawn = pexpect.spawn(exe_path, env=env, encoding="utf-8", timeout=timeout)
+        self.spawn.delaybeforesend = None
+        self.prompt_counter = 1
+
+    def time_since_first_message(self):
+        """ Return a delta in seconds since the first message, or 0 if this is the first. """
+        now = time.monotonic()
+        if not self.start_time:
+            self.start_time = now
+        return now - self.start_time
+
+    def send(self, s):
+        """ Cover over pexpect.spawn.send().
+            Send the given string to the tty, returning the number of bytes written.
+        """
+        res = self.spawn.send(s)
+        when = self.time_since_first_message()
+        self.messages.append(Message.sent(s, when))
+        return res
+
+    def sendline(self, s):
+        """ Cover over pexpect.spawn.sendline().
+            Send the given string + linesep to the tty, returning the number of bytes written.
+        """
+        return self.send(s + os.linesep)
+
+    def expect_re(self, pat, pat_desc=None, unmatched=None, **kwargs):
+        """ Cover over pexpect.spawn.expect().
+            Look through the "new" output of self.spawn until the given pattern is matched.
+            The pattern is typically a regular expression in string form, but may also be
+            any of the types accepted by pexpect.spawn.expect().
+            If the 'unmatched' parameter is given,
+            On failure, this prints an error and exits.
+        """
+        try:
+            res = self.spawn.expect(pat, **kwargs)
+            when = self.time_since_first_message()
+            self.messages.append(Message.received(self.spawn.match.group(), when))
+            return res
+        except pexpect.ExceptionPexpect as err:
+            if not pat_desc:
+                pat_desc = str(pat)
+            self.report_exception_and_exit(pat_desc, unmatched, err)
+
+    def expect_str(self, s, **kwargs):
+        """ Cover over expect_re() which accepts a literal string. """
+        return self.expect_re(re.escape(s), **kwargs)
+
+    def expect_prompt(self, *args, **kwargs):
+        """ Convenience function which matches some text and then a prompt.
+            Match the given positional arguments as expect_re, and then look
+            for a prompt, bumping the prompt counter.
+            Returns None on success, and exits on failure.
+            Example:
+               sp.sendline("echo hello world")
+               sp.expect_prompt("hello world")
+        """
+        if args:
+            self.expect_re(*args, **kwargs)
+        self.expect_re(
+            get_prompt_re(self.prompt_counter),
+            pat_desc="prompt %d" % self.prompt_counter,
+        )
+        self.prompt_counter += 1
+
+    def report_exception_and_exit(self, pat, unmatched, err):
+        """ Things have gone badly.
+            We have an exception 'err', some pexpect.ExceptionPexpect.
+            Report it to stdout, along with the offending call site.
+            If 'unmatched' is set, print it to stdout.
+        """
+        colors = self.colors()
+        if unmatched:
+            print("{BOLD}{unmatched}{RESET}".format(unmatched=unmatched, **colors))
+        if isinstance(err, pexpect.EOF):
+            msg = "EOF"
+        elif isinstance(err, pexpect.TIMEOUT):
+            msg = "TIMEOUT"
+        else:
+            msg = "UNKNOWN"
+        filename, lineno, code_context = get_callsite()
+        print("{RED}Failed to match:{NORMAL} {pat}".format(pat=escape(pat), **colors))
+        print(
+            "{msg} from {filename}:{lineno}: {code}".format(
+                msg=msg, filename=filename, lineno=lineno, code="\n".join(code_context)
+            )
+        )
+        # Show the last 5 messages.
+        for m in self.messages[-5:]:
+            print(m.formatted())
+        print("Buffer:")
+        print(escape(self.spawn.before))
+        sys.exit(1)
+
+    def sleep(self, secs):
+        """ Cover over time.sleep(). """
+        time.sleep(secs)
+
+    def colors(self):
+        """ Return a dictionary mapping color names to ANSI escapes """
+
+        def ansic(n):
+            """ Return either an ANSI escape sequence for a color, or empty string. """
+            return "\033[%dm" % n if self.colorize else ""
+
+        return {
+            "RESET": ansic(0),
+            "BOLD": ansic(1),
+            "NORMAL": ansic(39),
+            "BLACK": ansic(30),
+            "RED": ansic(31),
+            "GREEN": ansic(32),
+            "YELLOW": ansic(33),
+            "BLUE": ansic(34),
+            "MAGENTA": ansic(35),
+            "CYAN": ansic(36),
+            "LIGHTGRAY": ansic(37),
+            "DARKGRAY": ansic(90),
+            "LIGHTRED": ansic(91),
+            "LIGHTGREEN": ansic(92),
+            "LIGHTYELLOW": ansic(93),
+            "LIGHTBLUE": ansic(94),
+            "LIGHTMAGENTA": ansic(95),
+            "LIGHTCYAN": ansic(96),
+            "WHITE": ansic(97),
+        }

cmake/Tests.cmake

@@ -29,6 +29,9 @@ endif()
 # Copy littlecheck.py
 configure_file(build_tools/littlecheck.py littlecheck.py COPYONLY)
 
+# Copy pexpect_helper.py
+configure_file(build_tools/pexpect_helper.py pexpect_helper.py COPYONLY)
+
 # Make the directory in which to run tests.
 # Also symlink fish to where the tests expect it to be.
 # Lastly put fish_test_helper there too.

tests/interactive.fish

@@ -22,11 +22,13 @@ cd (dirname (status -f))
 set -gx TERM xterm
 set -e ITERM_PROFILE
 
-# Test files specified on commandline, or all *.expect files
+# Test files specified on commandline, or all *.expect files.
 if set -q argv[1]
-    set files_to_test $argv.expect
+    set expect_files_to_test $argv.expect
+    set pexpect_files_to_test pexpects/$argv.py
 else
-    set files_to_test *.expect
+    set expect_files_to_test *.expect
+    set pexpect_files_to_test pexpects/*.py
 end
 
 source test_util.fish (status -f) $argv
@@ -34,12 +36,7 @@ or exit
 cat interactive.config >>$XDG_CONFIG_HOME/fish/config.fish
 
 say -o cyan "Testing interactive functionality"
-if not type -q expect
+function test_expect_file
-    say red "Tests disabled: `expect` not found"
-    exit 0
-end
-
-function test_file
     set -l file $argv[1]
     echo -n "Testing file $file ... "
     set starttime (timestamp)
@@ -86,12 +83,53 @@ function test_file
     end
 end
 
+function test_pexpect_file
+    set -l file $argv[1]
+    echo -n "Testing file $file ... "
+
+    begin
+        set starttime (timestamp)
+        set -lx TERM dumb
+
+        # Help the script find the pexpect_helper module in our parent directory.
+        set -lx --prepend PYTHONPATH (realpath $PWD/..)
+        set -lx fish ../test/root/bin/fish
+        set -lx fish_key_reader ../test/root/bin/fish_key_reader
+        set -lx fish_test_helper ../test/root/bin/fish_test_helper
+
+        # Note we require Python3.
+        python3 $file
+    end
+
+    set -l exit_status $status
+    if test "$exit_status" -eq 0
+        set test_duration (delta $starttime)
+        say green "ok ($test_duration $unit)"
+    end
+    return $exit_status
+end
+
 set failed
-for i in $files_to_test
+
-    if not test_file $i
+if not python3 -c 'import pexpect'
+    say red "pexpect tests disabled: `python3 -c 'import pexpect'` failed"
+    set pexpect_files_to_test
+end
+for i in $pexpect_files_to_test
+    if not test_pexpect_file $i
+        set failed $failed $i
+    end
+end
+
+if not type -q expect
+    say red "expect tests disabled: `expect` not found"
+    set expect_files_to_test
+end
+for i in $expect_files_to_test
+    if not test_expect_file $i
         say -o cyan "Rerunning test $i"
         rm -f $i.tmp.*
-        if not test_file $i
+        if not test_expect_file $i
             set failed $failed $i
         end
     end

tests/pexpects/bind.py

@@ -0,0 +1,271 @@
+#!/usr/bin/env python3
+from pexpect_helper import SpawnedProc
+
+sp = SpawnedProc()
+sp.expect_prompt()
+
+# Fish should start in default-mode (i.e., emacs) bindings. The default escape
+# timeout is 30ms.
+
+# Verify the emacs transpose word (\et) behavior using various delays,
+# including none, after the escape character.
+
+# Start by testing with no delay. This should transpose the words.
+sp.send("echo abc def")
+sp.send("\033t\r")
+sp.expect_prompt("\r\ndef abc\r\n")  # emacs transpose words, default timeout: no delay
+
+# Now test with a delay > 0 and < the escape timeout. This should transpose
+# the words.
+sp.send("echo ghi jkl")
+sp.send("\033")
+sp.sleep(0.010)
+sp.send("t\r")
+# emacs transpose words, default timeout: short delay
+sp.expect_prompt("\r\njkl ghi\r\n")
+
+# Now test with a delay > the escape timeout. The transposition should not
+# occur and the "t" should become part of the text that is echoed.
+sp.send("echo mno pqr")
+sp.send("\033")
+sp.sleep(0.200)
+sp.send("t\r")
+# emacs transpose words, default timeout: long delay
+sp.expect_prompt("\r\nmno pqrt\r\n")
+
+# Now test that exactly the expected bind modes are defined
+sp.sendline("bind --list-modes")
+sp.expect_prompt("\r\ndefault\r\npaste", unmatched="Unexpected bind modes")
+
+# Test vi key bindings.
+# This should leave vi mode in the insert state.
+sp.sendline("set -g fish_key_bindings fish_vi_key_bindings")
+sp.expect_prompt()
+
+# Go through a prompt cycle to let fish catch up, it may be slow due to ASAN
+sp.sendline("echo success: default escape timeout")
+sp.expect_prompt(
+    "\r\nsuccess: default escape timeout", unmatched="prime vi mode, default timeout"
+)
+
+sp.send("echo fail: default escape timeout")
+sp.send("\033")
+
+# Delay needed to allow fish to transition to vi "normal" mode. The delay is
+# longer than strictly necessary to let fish catch up as it may be slow due to
+# ASAN.
+sp.sleep(0.150)
+sp.send("ddi")
+sp.sendline("echo success: default escape timeout")
+sp.expect_prompt(
+    "\r\nsuccess: default escape timeout\r\n",
+    unmatched="vi replace line, default timeout: long delay",
+)
+
+# Test replacing a single character.
+sp.send("echo TEXT")
+sp.send("\033")
+# Delay needed to allow fish to transition to vi "normal" mode.
+sp.sleep(0.150)
+sp.send("hhrAi\r")
+sp.expect_prompt(
+    "\r\nTAXT\r\n", unmatched="vi mode replace char, default timeout: long delay"
+)
+
+# Test deleting characters with 'x'.
+sp.send("echo MORE-TEXT")
+sp.send("\033")
+# Delay needed to allow fish to transition to vi "normal" mode.
+sp.sleep(0.250)
+sp.send("xxxxx\r")
+
+# vi mode delete char, default timeout: long delay
+sp.expect_prompt(
+    "\r\nMORE\r\n", unmatched="vi mode delete char, default timeout: long delay"
+)
+
+# Test jumping forward til before a character with t
+sp.send("echo MORE-TEXT-IS-NICE")
+sp.send("\033")
+# Delay needed to allow fish to transition to vi "normal" mode.
+sp.sleep(0.250)
+sp.send("0tTD\r")
+
+# vi mode forward-jump-till character, default timeout: long delay
+sp.expect_prompt(
+    "\r\nMORE\r\n",
+    unmatched="vi mode forward-jump-till character, default timeout: long delay",
+)
+
+# Test jumping backward til before a character with T
+sp.send("echo MORE-TEXT-IS-NICE")
+sp.send("\033")
+# Delay needed to allow fish to transition to vi "normal" mode.
+sp.sleep(0.250)
+sp.send("TSD\r")
+# vi mode backward-jump-till character, default timeout: long delay
+sp.expect_prompt(
+    "\r\nMORE-TEXT-IS\r\n",
+    unmatched="vi mode backward-jump-till character, default timeout: long delay",
+)
+
+# Test jumping backward with F and repeating
+sp.send("echo MORE-TEXT-IS-NICE")
+sp.send("\033")
+# Delay needed to allow fish to transition to vi "normal" mode.
+sp.sleep(0.250)
+sp.send("F-;D\r")
+# vi mode backward-jump-to character and repeat, default timeout: long delay
+sp.expect_prompt(
+    "\r\nMORE-TEXT\r\n",
+    unmatched="vi mode backward-jump-to character and repeat, default timeout: long delay",
+)
+
+# Test jumping backward with F w/reverse jump
+sp.send("echo MORE-TEXT-IS-NICE")
+sp.send("\033")
+# Delay needed to allow fish to transition to vi "normal" mode.
+sp.sleep(0.250)
+sp.send("F-F-,D\r")
+# vi mode backward-jump-to character, and reverse, default timeout: long delay
+sp.expect_prompt(
+    "\r\nMORE-TEXT-IS\r\n",
+    unmatched="vi mode backward-jump-to character, and reverse, default timeout: long delay",
+)
+
+# Verify that changing the escape timeout has an effect.
+sp.send("set -g fish_escape_delay_ms 200\r")
+sp.expect_prompt()
+
+sp.send("echo fail: lengthened escape timeout")
+sp.send("\033")
+sp.sleep(0.350)
+sp.send("ddi")
+sp.send("echo success: lengthened escape timeout\r")
+# vi replace line, 200ms timeout: long delay
+sp.expect_prompt(
+    "\r\nsuccess: lengthened escape timeout\r\n",
+    unmatched="vi replace line, 200ms timeout: long delay",
+)
+
+# Verify that we don't switch to vi normal mode if we don't wait long enough
+# after sending escape.
+sp.send("echo fail: no normal mode")
+sp.send("\033")
+sp.sleep(0.050)
+sp.send("ddi")
+sp.send("inserted\r")
+# vi replace line, 200ms timeout: short delay
+sp.expect_prompt(
+    "\r\nfail: no normal modediinserted\r\n",
+    unmatched="vi replace line, 200ms timeout: short delay",
+)
+
+# Test 't' binding that contains non-zero arity function (forward-jump) followed
+# by another function (and) https://github.com/fish-shell/fish-shell/issues/2357
+sp.send("\033")
+sp.sleep(0.300)
+sp.send("ddiecho TEXT\033")
+sp.sleep(0.300)
+sp.send("hhtTrN\r")
+sp.expect_prompt("\r\nTENT\r\n", unmatched="Couldn't find expected output 'TENT'")
+
+# Test '~' (togglecase-char)
+sp.send("\033")
+sp.sleep(0.300)
+sp.send("ccecho some TExT\033")
+sp.sleep(0.300)
+sp.send("hh~~bbve~\r")
+sp.expect_prompt("\r\nSOME TeXT\r\n", unmatched="Couldn't find expected output 'SOME TeXT")
+
+# Now test that exactly the expected bind modes are defined
+sp.sendline("bind --list-modes")
+sp.expect_prompt(
+    "\r\ndefault\r\ninsert\r\npaste\r\nreplace\r\nreplace_one\r\nvisual\r\n",
+    unmatched="Unexpected vi bind modes",
+)
+
+# Switch back to regular (emacs mode) key bindings.
+sp.sendline("set -g fish_key_bindings fish_default_key_bindings")
+sp.expect_prompt()
+
+# Verify the custom escape timeout of 200ms set earlier is still in effect.
+sp.sendline("echo fish_escape_delay_ms=$fish_escape_delay_ms")
+sp.expect_prompt(
+    "\r\nfish_escape_delay_ms=200\r\n",
+    unmatched="default-mode custom timeout not set correctly",
+)
+
+# Set it to 100ms.
+sp.sendline("set -g fish_escape_delay_ms 100")
+sp.expect_prompt()
+
+# Verify the emacs transpose word (\et) behavior using various delays,
+# including none, after the escape character.
+
+# Start by testing with no delay. This should transpose the words.
+sp.send("echo abc def")
+sp.send("\033")
+sp.send("t\r")
+# emacs transpose words, 100ms timeout: no delay
+sp.expect_prompt(
+    "\r\ndef abc\r\n", unmatched="emacs transpose words fail, 100ms timeout: no delay"
+)
+
+# Same test as above but with a slight delay less than the escape timeout.
+sp.send("echo ghi jkl")
+sp.send("\033")
+sp.sleep(0.080)
+sp.send("t\r")
+# emacs transpose words, 100ms timeout: short delay
+sp.expect_prompt(
+    "\r\njkl ghi\r\n",
+    unmatched="emacs transpose words fail, 100ms timeout: short delay",
+)
+
+# Now test with a delay > the escape timeout. The transposition should not
+# occur and the "t" should become part of the text that is echoed.
+sp.send("echo mno pqr")
+sp.send("\033")
+sp.sleep(0.250)
+sp.send("t\r")
+# emacs transpose words, 100ms timeout: long delay
+sp.expect_prompt(
+    "\r\nmno pqrt\r\n",
+    unmatched="emacs transpose words fail, 100ms timeout: long delay",
+)
+
+# Verify special characters, such as \cV, are not intercepted by the kernel
+# tty driver. Rather, they can be bound and handled by fish.
+sp.sendline("bind \\cV 'echo ctrl-v seen'")
+sp.expect_prompt()
+sp.send("\026\r")
+sp.expect_prompt("ctrl-v seen", unmatched="ctrl-v not seen")
+
+sp.send("bind \\cO 'echo ctrl-o seen'\r")
+sp.expect_prompt()
+sp.send("\017\r")
+sp.expect_prompt("ctrl-o seen", unmatched="ctrl-o not seen")
+
+# \x17 is ctrl-w.
+sp.send("echo git@github.com:fish-shell/fish-shell")
+sp.send("\x17\x17\r")
+sp.expect_prompt("git@github.com:", unmatched="ctrl-w does not stop at :")
+
+sp.send("echo git@github.com:fish-shell/fish-shell")
+sp.send("\x17\x17\x17\r")
+sp.expect_prompt("git@", unmatched="ctrl-w does not stop at @")
+
+# Ensure that nul can be bound properly (#3189).
+sp.send("bind -k nul 'echo nul seen'\r")
+sp.expect_prompt
+sp.send("\0" * 3)
+sp.send("\r")
+sp.expect_prompt("nul seen\r\nnul seen\r\nnul seen", unmatched="nul not seen")
+
+# Test self-insert-notfirst. (#6603)
+# Here the leading 'q's should be stripped, but the trailing ones not.
+sp.sendline("bind q self-insert-notfirst")
+sp.expect_prompt()
+sp.sendline("qqqecho qqq")
+sp.expect_prompt("qqq", unmatched="Leading qs not stripped")

tests/pexpects/pipeline.py

@@ -0,0 +1,28 @@
+#!/usr/bin/env python3
+from pexpect_helper import SpawnedProc
+
+sp = SpawnedProc()
+sp.expect_prompt()
+sp.sendline("function echo_wrap ; /bin/echo $argv ; sleep 0.1; end")
+sp.expect_prompt()
+
+for i in range(5):
+    sp.sendline(
+        "echo_wrap 1 2 3 4 | $fish_test_helper become_foreground_then_print_stderr ; or exit 1"
+    )
+    sp.expect_prompt("become_foreground_then_print_stderr done")
+
+# 'not' because we expect to have no jobs, in which case `jobs` will return false
+sp.sendline("not jobs")
+sp.expect_prompt("jobs: There are no jobs")
+
+sp.sendline("function inner ; command true ; end; function outer; inner; end")
+sp.expect_prompt()
+for i in range(5):
+    sp.sendline(
+        "outer | $fish_test_helper become_foreground_then_print_stderr ; or exit 1"
+    )
+    sp.expect_prompt("become_foreground_then_print_stderr done")
+
+sp.sendline("not jobs")
+sp.expect_prompt("jobs: There are no jobs", unmatched="Should be no jobs")