Adopt the new AST in parse_execution

parse_execution is what turns a parsed tree into jobs, etc. Switch it from
parse_tree to the new AST.

src/builtin_function.cpp

@@ -200,8 +200,7 @@ static int validate_function_name(int argc, const wchar_t *const *argv, wcstring
 /// Define a function. Calls into `function.cpp` to perform the heavy lifting of defining a
 /// function.
 int builtin_function(parser_t &parser, io_streams_t &streams, const wcstring_list_t &c_args,
-                     const parsed_source_ref_t &source,
-                     tnode_t<grammar::block_statement> func_node) {
+                     const parsed_source_ref_t &source, const ast::block_statement_t &func_node) {
     assert(source && "Missing source in builtin_function");
     // The wgetopt function expects 'function' as the first argument. Make a new wcstring_list with
     // that property. This is needed because this builtin has a different signature than the other
@@ -252,7 +251,7 @@ int builtin_function(parser_t &parser, io_streams_t &streams, const wcstring_lis
     props->shadow_scope = opts.shadow_scope;
     props->named_arguments = std::move(opts.named_arguments);
     props->parsed_source = source;
-    props->func_node = func_node;
+    props->func_node = &func_node;
 
     // Populate inherit_vars.
     for (const wcstring &name : opts.inherit_vars) {

src/builtin_function.h

@@ -8,7 +8,10 @@
 class parser_t;
 struct io_streams_t;
 
+namespace ast {
+struct block_statement_t;
+}
+
 int builtin_function(parser_t &parser, io_streams_t &streams, const wcstring_list_t &c_args,
-                     const parsed_source_ref_t &source,
-                     tnode_t<grammar::block_statement> func_node);
+                     const parsed_source_ref_t &source, const ast::block_statement_t &func_node);
 #endif

src/exec.cpp

@@ -623,10 +623,10 @@ static proc_performer_t get_performer_for_process(process_t *p, job_t *job,
 
     if (p->type == process_type_t::block_node) {
         const parsed_source_ref_t &source = p->block_node_source;
-        tnode_t<grammar::statement> node = p->internal_block_node;
+        const ast::statement_t *node = p->internal_block_node;
         assert(source && node && "Process is missing node info");
         return [=](parser_t &parser) {
-            return parser.eval_node(source, node, io_chain, job_group).status;
+            return parser.eval_node(source, *node, io_chain, job_group).status;
         };
     } else {
         assert(p->type == process_type_t::function);
@@ -638,7 +638,7 @@ static proc_performer_t get_performer_for_process(process_t *p, job_t *job,
         auto argv = move_to_sharedptr(p->get_argv_array().to_list());
         return [=](parser_t &parser) {
             // Pull out the job list from the function.
-            tnode_t<grammar::job_list> body = props->func_node.child<1>();
+            const ast::job_list_t &body = props->func_node->jobs;
             const block_t *fb = function_prepare_environment(parser, *argv, *props);
             auto res = parser.eval_node(props->parsed_source, body, io_chain, job_group);
             function_restore_environment(parser, fb);

src/fish_tests.cpp

@@ -4567,6 +4567,8 @@ static void test_new_parser_errors() {
 
         {L"case", parse_error_unbalancing_case},
         {L"if true ; case ; end", parse_error_generic},
+
+        {L"true | and", parse_error_andor_in_pipeline},
     };
 
     for (const auto &test : tests) {

src/function.cpp

@@ -224,17 +224,14 @@ bool function_get_definition(const wcstring &name, wcstring &out_definition) {
     const function_info_t *func = funcset->get_info(name);
     if (!func || !func->props) return false;
     // We want to preserve comments that the AST attaches to the header (#5285).
-    // Take everything from the end of the header to the end of the body.
+    // Take everything from the end of the header to the 'end' keyword.
     const auto &props = func->props;
-    namespace g = grammar;
-    tnode_t<g::block_header> header = props->func_node.child<0>();
-    tnode_t<g::job_list> jobs = props->func_node.child<1>();
-    auto header_src = header.source_range();
-    auto jobs_src = jobs.source_range();
-    if (header_src && jobs_src) {
+    auto header_src = props->func_node->header->try_source_range();
+    auto end_kw_src = props->func_node->end.try_source_range();
+    if (header_src && end_kw_src) {
         uint32_t body_start = header_src->start + header_src->length;
-        uint32_t body_end = jobs_src->start + jobs_src->length;
-        assert(body_start <= jobs_src->start && "job list must come after header");
+        uint32_t body_end = end_kw_src->start;
+        assert(body_start <= body_end && "end keyword should come after header");
         out_definition = wcstring(props->parsed_source->src, body_start, body_end - body_start);
     }
     return true;
@@ -313,7 +310,7 @@ int function_get_definition_lineno(const wcstring &name) {
     // return one plus the number of newlines at offsets less than the start of our function's
     // statement (which includes the header).
     // TODO: merge with line_offset_of_character_at_offset?
-    auto source_range = func->props->func_node.source_range();
+    auto source_range = func->props->func_node->try_source_range();
     assert(source_range && "Function has no source range");
     uint32_t func_start = source_range->start;
     const wcstring &source = func->props->parsed_source->src;

src/function.h

@@ -15,6 +15,10 @@
 
 class parser_t;
 
+namespace ast {
+struct block_statement_t;
+}
+
 /// A function's constant properties. These do not change once initialized.
 struct function_properties_t {
     /// Parsed source containing the function.
@@ -23,7 +27,7 @@ struct function_properties_t {
     /// Node containing the function statement, pointing into parsed_source.
     /// We store block_statement, not job_list, so that comments attached to the header are
     /// preserved.
-    tnode_t<grammar::block_statement> func_node;
+    const ast::block_statement_t *func_node;
 
     /// List of all named arguments for this function.
     wcstring_list_t named_arguments;

src/parse_execution.h

@@ -4,6 +4,7 @@
 
 #include <stddef.h>
 
+#include "ast.h"
 #include "common.h"
 #include "io.h"
 #include "parse_constants.h"
@@ -38,7 +39,7 @@ class parse_execution_context_t {
     const operation_context_t &ctx;
 
     // The currently executing job node, used to indicate the line number.
-    tnode_t<grammar::job> executing_job_node{};
+    const ast::job_t *executing_job_node{};
 
     // Cached line number information.
     size_t cached_lineno_offset = 0;
@@ -59,88 +60,94 @@ class parse_execution_context_t {
 
     // Report an error, setting $status to \p status. Always returns
     // 'end_execution_reason_t::error'.
-    end_execution_reason_t report_error(int status, const parse_node_t &node, const wchar_t *fmt,
+    end_execution_reason_t report_error(int status, const ast::node_t &node, const wchar_t *fmt,
                                         ...) const;
     end_execution_reason_t report_errors(int status, const parse_error_list_t &error_list) const;
 
     /// Command not found support.
     end_execution_reason_t handle_command_not_found(const wcstring &cmd,
-                                                    tnode_t<grammar::plain_statement> statement,
+                                                    const ast::decorated_statement_t &statement,
                                                     int err_code);
 
     // Utilities
-    wcstring get_source(const parse_node_t &node) const;
-    tnode_t<grammar::plain_statement> infinite_recursive_statement_in_job_list(
-        tnode_t<grammar::job_list> job_list, wcstring *out_func_name) const;
+    wcstring get_source(const ast::node_t &node) const;
+    const ast::decorated_statement_t *infinite_recursive_statement_in_job_list(
+        const ast::job_list_t &job_list, wcstring *out_func_name) const;
 
     // Expand a command which may contain variables, producing an expand command and possibly
     // arguments. Prints an error message on error.
-    end_execution_reason_t expand_command(tnode_t<grammar::plain_statement> statement,
+    end_execution_reason_t expand_command(const ast::decorated_statement_t &statement,
                                           wcstring *out_cmd, wcstring_list_t *out_args) const;
 
     /// Return whether we should skip a job with the given bool statement type.
     bool should_skip(parse_job_decoration_t type) const;
 
     /// Indicates whether a job is a simple block (one block, no redirections).
-    bool job_is_simple_block(tnode_t<grammar::job> job) const;
+    bool job_is_simple_block(const ast::job_t &job) const;
 
-    enum process_type_t process_type_for_command(tnode_t<grammar::plain_statement> statement,
+    enum process_type_t process_type_for_command(const ast::decorated_statement_t &statement,
                                                  const wcstring &cmd) const;
     end_execution_reason_t apply_variable_assignments(
-        process_t *proc, tnode_t<grammar::variable_assignments> variable_assignments,
+        process_t *proc, const ast::variable_assignment_list_t &variable_assignments,
         const block_t **block);
 
     // These create process_t structures from statements.
     end_execution_reason_t populate_job_process(
-        job_t *job, process_t *proc, tnode_t<grammar::statement> statement,
-        tnode_t<grammar::variable_assignments> variable_assignments);
+        job_t *job, process_t *proc, const ast::statement_t &statement,
+        const ast::variable_assignment_list_t &variable_assignments_list_t);
     end_execution_reason_t populate_not_process(job_t *job, process_t *proc,
-                                                tnode_t<grammar::not_statement> not_statement);
+                                                const ast::not_statement_t &not_statement);
     end_execution_reason_t populate_plain_process(job_t *job, process_t *proc,
-                                                  tnode_t<grammar::plain_statement> statement);
+                                                  const ast::decorated_statement_t &statement);
 
     template <typename Type>
     end_execution_reason_t populate_block_process(job_t *job, process_t *proc,
-                                                  tnode_t<grammar::statement> statement,
-                                                  tnode_t<Type> specific_statement);
+                                                  const ast::statement_t &statement,
+                                                  const Type &specific_statement);
 
     // These encapsulate the actual logic of various (block) statements.
-    end_execution_reason_t run_block_statement(tnode_t<grammar::block_statement> statement,
+    end_execution_reason_t run_block_statement(const ast::block_statement_t &statement,
                                                const block_t *associated_block);
-    end_execution_reason_t run_for_statement(tnode_t<grammar::for_header> header,
-                                             tnode_t<grammar::job_list> contents);
-    end_execution_reason_t run_if_statement(tnode_t<grammar::if_statement> statement,
+    end_execution_reason_t run_for_statement(const ast::for_header_t &header,
+                                             const ast::job_list_t &contents);
+    end_execution_reason_t run_if_statement(const ast::if_statement_t &statement,
                                             const block_t *associated_block);
-    end_execution_reason_t run_switch_statement(tnode_t<grammar::switch_statement> statement);
-    end_execution_reason_t run_while_statement(tnode_t<grammar::while_header> header,
-                                               tnode_t<grammar::job_list> contents,
+    end_execution_reason_t run_switch_statement(const ast::switch_statement_t &statement);
+    end_execution_reason_t run_while_statement(const ast::while_header_t &header,
+                                               const ast::job_list_t &contents,
                                                const block_t *associated_block);
-    end_execution_reason_t run_function_statement(tnode_t<grammar::block_statement> statement,
-                                                  tnode_t<grammar::function_header> header);
-    end_execution_reason_t run_begin_statement(tnode_t<grammar::job_list> contents);
+    end_execution_reason_t run_function_statement(const ast::block_statement_t &statement,
+                                                  const ast::function_header_t &header);
+    end_execution_reason_t run_begin_statement(const ast::job_list_t &contents);
 
     enum globspec_t { failglob, nullglob };
-    using argument_node_list_t = std::vector<tnode_t<grammar::argument>>;
-    end_execution_reason_t expand_arguments_from_nodes(const argument_node_list_t &argument_nodes,
+    using ast_args_list_t = std::vector<const ast::argument_t *>;
+
+    static ast_args_list_t get_argument_nodes(const ast::argument_list_t &args);
+    static ast_args_list_t get_argument_nodes(const ast::argument_or_redirection_list_t &args);
+
+    end_execution_reason_t expand_arguments_from_nodes(const ast_args_list_t &argument_nodes,
                                                        wcstring_list_t *out_arguments,
                                                        globspec_t glob_behavior);
 
     // Determines the list of redirections for a node.
-    end_execution_reason_t determine_redirections(
-        tnode_t<grammar::arguments_or_redirections_list> node,
-        redirection_spec_list_t *out_redirections);
+    end_execution_reason_t determine_redirections(const ast::argument_or_redirection_list_t &list,
+                                                  redirection_spec_list_t *out_redirections);
 
-    end_execution_reason_t run_1_job(tnode_t<grammar::job> job, const block_t *associated_block);
-    end_execution_reason_t run_job_conjunction(tnode_t<grammar::job_conjunction> job_expr,
+    end_execution_reason_t run_1_job(const ast::job_t &job, const block_t *associated_block);
+    end_execution_reason_t test_and_run_1_job_conjunction(const ast::job_conjunction_t &jc,
+                                                          const block_t *associated_block);
+    end_execution_reason_t run_job_conjunction(const ast::job_conjunction_t &job_expr,
                                                const block_t *associated_block);
-    template <typename Type>
-    end_execution_reason_t run_job_list(tnode_t<Type> job_list_node,
+    end_execution_reason_t run_job_list(const ast::job_list_t &job_list_node,
                                         const block_t *associated_block);
-    end_execution_reason_t populate_job_from_job_node(job_t *j, tnode_t<grammar::job> job_node,
+    end_execution_reason_t run_job_list(const ast::andor_job_list_t &job_list_node,
+                                        const block_t *associated_block);
+    end_execution_reason_t populate_job_from_job_node(job_t *j, const ast::job_t &job_node,
                                                       const block_t *associated_block);
 
     // Returns the line number of the node. Not const since it touches cached_lineno_offset.
-    int line_offset_of_node(tnode_t<grammar::job> node);
+    int line_offset_of_node(const ast::job_t *node);
     int line_offset_of_character_at_offset(size_t offset);
 
    public:
@@ -159,14 +166,14 @@ class parse_execution_context_t {
     /// Returns the source string.
     const wcstring &get_source() const { return pstree->src; }
 
-    /// Return the parse tree.
-    const parse_node_tree_t &tree() const { return pstree->tree; }
+    /// Return the parsed ast.
+    const ast::ast_t &ast() const { return *pstree->ast; }
 
     /// Start executing at the given node. Returns 0 if there was no error, 1 if there was an
     /// error.
-    end_execution_reason_t eval_node(tnode_t<grammar::statement> statement,
+    end_execution_reason_t eval_node(const ast::statement_t &statement,
                                      const block_t *associated_block);
-    end_execution_reason_t eval_node(tnode_t<grammar::job_list> job_list,
+    end_execution_reason_t eval_node(const ast::job_list_t &job_list,
                                      const block_t *associated_block);
 };
 

src/parse_tree.cpp

@@ -1214,11 +1214,19 @@ const parse_node_t *parse_node_tree_t::get_child(const parse_node_t &parent, nod
     return result;
 }
 
+parsed_source_t::parsed_source_t(wcstring s, ast::ast_t &&ast)
+    : src(std::move(s)), ast(make_unique<ast::ast_t>(std::move(ast))) {}
+
+parsed_source_t::~parsed_source_t() = default;
+
 parsed_source_ref_t parse_source(wcstring src, parse_tree_flags_t flags,
                                  parse_error_list_t *errors) {
-    parse_node_tree_t tree;
-    if (!parse_tree_from_string(src, flags, &tree, errors, symbol_job_list)) return {};
-    return std::make_shared<parsed_source_t>(std::move(src), std::move(tree));
+    using namespace ast;
+    ast_t ast = ast_t::parse(src, flags, errors);
+    if (ast.errored() && !(flags & parse_flag_continue_after_error)) {
+        return nullptr;
+    }
+    return std::make_shared<parsed_source_t>(std::move(src), std::move(ast));
 }
 
 const parse_node_t &parse_node_tree_t::find_child(const parse_node_t &parent,

src/parse_tree.h

@@ -206,19 +206,26 @@ bool parse_tree_from_string(const wcstring &str, parse_tree_flags_t flags,
                             parse_node_tree_t *output, parse_error_list_t *errors,
                             parse_token_type_t goal = symbol_job_list);
 
+namespace ast {
+class ast_t;
+}
+
 /// A type wrapping up a parse tree and the original source behind it.
 struct parsed_source_t {
     wcstring src;
-    parse_node_tree_t tree;
+    std::unique_ptr<ast::ast_t> ast;
 
-    parsed_source_t(wcstring s, parse_node_tree_t t) : src(std::move(s)), tree(std::move(t)) {}
+    parsed_source_t(wcstring s, ast::ast_t &&ast);
+    ~parsed_source_t();
 
     parsed_source_t(const parsed_source_t &) = delete;
     void operator=(const parsed_source_t &) = delete;
-    parsed_source_t(parsed_source_t &&) = default;
-    parsed_source_t &operator=(parsed_source_t &&) = default;
+    parsed_source_t(parsed_source_t &&) = delete;
+    parsed_source_t &operator=(parsed_source_t &&) = delete;
 };
+
 /// Return a shared pointer to parsed_source_t, or null on failure.
+/// If parse_flag_continue_after_error is not set, this will return null on any error.
 using parsed_source_ref_t = std::shared_ptr<const parsed_source_t>;
 parsed_source_ref_t parse_source(wcstring src, parse_tree_flags_t flags,
                                  parse_error_list_t *errors);

src/parse_util.cpp

@@ -1311,9 +1311,9 @@ parser_test_error_bits_t parse_util_detect_errors(const wcstring &buff_src,
         *out_errors = std::move(parse_errors);
     }
 
+    // \return the ast to our caller if requested.
     if (out_pstree != nullptr) {
-        // TODO: legacy
-        *out_pstree = parse_source(buff_src, parse_flags, nullptr);
+        *out_pstree = std::make_shared<parsed_source_t>(buff_src, std::move(ast));
     }
 
     return res;

src/parser.cpp

@@ -656,10 +656,10 @@ eval_res_t parser_t::eval(const wcstring &cmd, const io_chain_t &io,
 eval_res_t parser_t::eval(const parsed_source_ref_t &ps, const io_chain_t &io,
                           const job_group_ref_t &job_group, enum block_type_t block_type) {
     assert(block_type == block_type_t::top || block_type == block_type_t::subst);
-    if (!ps->tree.empty()) {
-        // Execute the first node.
-        tnode_t<grammar::job_list> start{&ps->tree, &ps->tree.front()};
-        return this->eval_node(ps, start, io, job_group, block_type);
+    const auto *job_list = ps->ast->top()->as<ast::job_list_t>();
+    if (!job_list->empty()) {
+        // Execute the top job list.
+        return this->eval_node(ps, *job_list, io, job_group, block_type);
     } else {
         auto status = proc_status_t::from_exit_code(get_last_status());
         bool break_expand = false;
@@ -669,11 +669,11 @@ eval_res_t parser_t::eval(const parsed_source_ref_t &ps, const io_chain_t &io,
 }
 
 template <typename T>
-eval_res_t parser_t::eval_node(const parsed_source_ref_t &ps, tnode_t<T> node,
+eval_res_t parser_t::eval_node(const parsed_source_ref_t &ps, const T &node,
                                const io_chain_t &block_io, const job_group_ref_t &job_group,
                                block_type_t block_type) {
     static_assert(
-        std::is_same<T, grammar::statement>::value || std::is_same<T, grammar::job_list>::value,
+        std::is_same<T, ast::statement_t>::value || std::is_same<T, ast::job_list_t>::value,
         "Unexpected node type");
     // Handle cancellation requests. If our block stack is currently empty, then we already did
     // successfully cancel (or there was nothing to cancel); clear the flag. If our block stack is
@@ -725,9 +725,9 @@ eval_res_t parser_t::eval_node(const parsed_source_ref_t &ps, tnode_t<T> node,
 }
 
 // Explicit instantiations. TODO: use overloads instead?
-template eval_res_t parser_t::eval_node(const parsed_source_ref_t &, tnode_t<grammar::statement>,
+template eval_res_t parser_t::eval_node(const parsed_source_ref_t &, const ast::statement_t &,
                                         const io_chain_t &, const job_group_ref_t &, block_type_t);
-template eval_res_t parser_t::eval_node(const parsed_source_ref_t &, tnode_t<grammar::job_list>,
+template eval_res_t parser_t::eval_node(const parsed_source_ref_t &, const ast::job_list_t &,
                                         const io_chain_t &, const job_group_ref_t &, block_type_t);
 
 void parser_t::get_backtrace(const wcstring &src, const parse_error_list_t &errors,

src/parser.h

@@ -300,9 +300,9 @@ class parser_t : public std::enable_shared_from_this<parser_t> {
                     block_type_t block_type = block_type_t::top);
 
     /// Evaluates a node.
-    /// The node type must be grammar::statement or grammar::job_list.
+    /// The node type must be ast_t::statement_t or ast::job_list_t.
     template <typename T>
-    eval_res_t eval_node(const parsed_source_ref_t &ps, tnode_t<T> node, const io_chain_t &block_io,
+    eval_res_t eval_node(const parsed_source_ref_t &ps, const T &node, const io_chain_t &block_io,
                          const job_group_ref_t &job_group,
                          block_type_t block_type = block_type_t::top);
 

src/proc.h

@@ -44,6 +44,10 @@ enum class job_control_t {
     none,
 };
 
+namespace ast {
+struct statement_t;
+}
+
 /// A proc_status_t is a value type that encapsulates logic around exited vs stopped vs signaled,
 /// etc.
 class proc_status_t {
@@ -261,10 +265,10 @@ class process_t {
     /// Type of process.
     process_type_t type{process_type_t::external};
 
-    /// For internal block processes only, the node offset of the statement.
+    /// For internal block processes only, the node of the statement.
     /// This is always either block, ifs, or switchs, never boolean or decorated.
     parsed_source_ref_t block_node_source{};
-    tnode_t<grammar::statement> internal_block_node{};
+    const ast::statement_t *internal_block_node{};
 
     struct concrete_assignment {
         wcstring variable_name;