2016-05-03 07:09:46 +08:00
|
|
|
// Programmatic representation of fish code.
|
2013-07-26 06:24:22 +08:00
|
|
|
#ifndef FISH_PARSE_PRODUCTIONS_H
|
|
|
|
#define FISH_PARSE_PRODUCTIONS_H
|
2013-05-27 03:12:16 +08:00
|
|
|
|
2015-07-25 23:14:25 +08:00
|
|
|
#include <stddef.h>
|
2017-02-13 12:24:22 +08:00
|
|
|
#include <stdint.h>
|
2016-04-21 14:00:54 +08:00
|
|
|
#include <sys/types.h>
|
2017-02-14 12:37:27 +08:00
|
|
|
|
2016-05-03 07:09:46 +08:00
|
|
|
#include <memory>
|
|
|
|
#include <vector>
|
2013-05-27 03:12:16 +08:00
|
|
|
|
|
|
|
#include "common.h"
|
2018-01-08 11:50:34 +08:00
|
|
|
#include "maybe.h"
|
2013-12-09 13:54:06 +08:00
|
|
|
#include "parse_constants.h"
|
2018-01-08 11:07:49 +08:00
|
|
|
#include "parse_grammar.h"
|
2016-05-03 07:09:46 +08:00
|
|
|
#include "tokenizer.h"
|
2013-06-12 00:37:51 +08:00
|
|
|
|
2013-06-25 03:33:40 +08:00
|
|
|
class parse_node_tree_t;
|
2014-03-26 11:06:34 +08:00
|
|
|
|
|
|
|
typedef uint32_t node_offset_t;
|
|
|
|
|
2013-06-23 17:09:46 +08:00
|
|
|
#define NODE_OFFSET_INVALID (static_cast<node_offset_t>(-1))
|
2013-06-16 05:32:38 +08:00
|
|
|
|
2014-03-26 11:06:34 +08:00
|
|
|
typedef uint32_t source_offset_t;
|
|
|
|
|
2017-01-27 09:28:46 +08:00
|
|
|
constexpr source_offset_t SOURCE_OFFSET_INVALID = static_cast<source_offset_t>(-1);
|
2014-03-26 11:06:34 +08:00
|
|
|
|
2016-05-03 07:09:46 +08:00
|
|
|
/// A struct representing the token type that we use internally.
|
|
|
|
struct parse_token_t {
|
|
|
|
enum parse_token_type_t type; // The type of the token as represented by the parser
|
|
|
|
enum parse_keyword_t keyword; // Any keyword represented by this token
|
|
|
|
bool has_dash_prefix; // Hackish: whether the source contains a dash prefix
|
|
|
|
bool is_help_argument; // Hackish: whether the source looks like '-h' or '--help'
|
2018-02-19 06:37:44 +08:00
|
|
|
bool is_newline; // Hackish: if TOK_END, whether the source is a newline.
|
2014-03-26 11:06:34 +08:00
|
|
|
source_offset_t source_start;
|
|
|
|
source_offset_t source_length;
|
2013-08-11 15:35:00 +08:00
|
|
|
|
2013-10-12 17:46:49 +08:00
|
|
|
wcstring describe() const;
|
2014-01-01 16:04:02 +08:00
|
|
|
wcstring user_presentable_description() const;
|
2013-08-11 15:35:00 +08:00
|
|
|
};
|
|
|
|
|
2016-05-03 07:09:46 +08:00
|
|
|
enum {
|
2013-08-11 15:35:00 +08:00
|
|
|
parse_flag_none = 0,
|
|
|
|
|
2016-05-03 07:09:46 +08:00
|
|
|
/// Attempt to build a "parse tree" no matter what. This may result in a 'forest' of
|
|
|
|
/// disconnected trees. This is intended to be used by syntax highlighting.
|
2013-08-11 15:35:00 +08:00
|
|
|
parse_flag_continue_after_error = 1 << 0,
|
2016-05-03 07:09:46 +08:00
|
|
|
/// Include comment tokens.
|
2013-10-09 17:03:50 +08:00
|
|
|
parse_flag_include_comments = 1 << 1,
|
2016-05-03 07:09:46 +08:00
|
|
|
/// Indicate that the tokenizer should accept incomplete tokens */
|
2013-12-27 17:38:43 +08:00
|
|
|
parse_flag_accept_incomplete_tokens = 1 << 2,
|
2016-05-03 07:09:46 +08:00
|
|
|
/// Indicate that the parser should not generate the terminate token, allowing an 'unfinished'
|
|
|
|
/// tree where some nodes may have no productions.
|
2014-12-24 02:58:45 +08:00
|
|
|
parse_flag_leave_unterminated = 1 << 3,
|
2016-05-03 07:09:46 +08:00
|
|
|
/// Indicate that the parser should generate job_list entries for blank lines.
|
2014-12-24 02:58:45 +08:00
|
|
|
parse_flag_show_blank_lines = 1 << 4
|
2013-06-12 00:37:51 +08:00
|
|
|
};
|
2013-08-11 15:35:00 +08:00
|
|
|
typedef unsigned int parse_tree_flags_t;
|
|
|
|
|
|
|
|
wcstring parse_dump_tree(const parse_node_tree_t &tree, const wcstring &src);
|
2013-06-12 00:37:51 +08:00
|
|
|
|
2016-04-11 10:08:07 +08:00
|
|
|
const wchar_t *token_type_description(parse_token_type_t type);
|
|
|
|
const wchar_t *keyword_description(parse_keyword_t type);
|
2013-06-23 17:09:46 +08:00
|
|
|
|
2016-05-03 07:09:46 +08:00
|
|
|
// Node flags.
|
|
|
|
enum {
|
|
|
|
/// Flag indicating that the node has associated comment nodes.
|
2015-12-16 06:59:03 +08:00
|
|
|
parse_node_flag_has_comments = 1 << 0,
|
2014-12-24 02:58:45 +08:00
|
|
|
};
|
|
|
|
typedef uint8_t parse_node_flags_t;
|
|
|
|
|
2016-05-03 07:09:46 +08:00
|
|
|
/// Node-type specific tag value.
|
2015-12-16 06:59:03 +08:00
|
|
|
typedef uint8_t parse_node_tag_t;
|
|
|
|
|
2016-05-03 07:09:46 +08:00
|
|
|
/// Class for nodes of a parse tree. Since there's a lot of these, the size and order of the fields
|
|
|
|
/// is important.
|
|
|
|
class parse_node_t {
|
|
|
|
public:
|
|
|
|
// Start in the source code.
|
2014-03-26 11:06:34 +08:00
|
|
|
source_offset_t source_start;
|
2016-05-03 07:09:46 +08:00
|
|
|
// Length of our range in the source code.
|
2014-03-26 11:06:34 +08:00
|
|
|
source_offset_t source_length;
|
2016-05-03 07:09:46 +08:00
|
|
|
// Parent
|
2013-10-07 16:04:37 +08:00
|
|
|
node_offset_t parent;
|
2016-05-03 07:09:46 +08:00
|
|
|
// Children
|
2013-06-12 00:37:51 +08:00
|
|
|
node_offset_t child_start;
|
2016-05-03 07:09:46 +08:00
|
|
|
// Number of children.
|
2013-10-09 18:45:58 +08:00
|
|
|
uint8_t child_count;
|
2016-05-03 07:09:46 +08:00
|
|
|
// Type of the node.
|
2014-03-26 11:06:34 +08:00
|
|
|
enum parse_token_type_t type;
|
2016-05-03 07:09:46 +08:00
|
|
|
// Keyword associated with node.
|
2015-12-16 06:59:03 +08:00
|
|
|
enum parse_keyword_t keyword;
|
2016-05-03 07:09:46 +08:00
|
|
|
// Node flags.
|
|
|
|
parse_node_flags_t flags : 4;
|
|
|
|
// This is used to store e.g. the statement decoration.
|
|
|
|
parse_node_tag_t tag : 4;
|
|
|
|
// Description
|
2018-03-05 07:03:56 +08:00
|
|
|
wcstring describe() const;
|
2013-07-23 09:26:15 +08:00
|
|
|
|
2016-05-03 07:09:46 +08:00
|
|
|
// Constructor
|
|
|
|
explicit parse_node_t(parse_token_type_t ty)
|
|
|
|
: source_start(SOURCE_OFFSET_INVALID),
|
|
|
|
source_length(0),
|
|
|
|
parent(NODE_OFFSET_INVALID),
|
|
|
|
child_start(0),
|
|
|
|
child_count(0),
|
|
|
|
type(ty),
|
|
|
|
keyword(parse_keyword_none),
|
|
|
|
flags(0),
|
|
|
|
tag(0) {}
|
|
|
|
|
|
|
|
node_offset_t child_offset(node_offset_t which) const {
|
2013-06-23 17:09:46 +08:00
|
|
|
PARSE_ASSERT(which < child_count);
|
|
|
|
return child_start + which;
|
|
|
|
}
|
2013-08-11 15:35:00 +08:00
|
|
|
|
2016-05-03 07:09:46 +08:00
|
|
|
/// Indicate if this node has a range of source code associated with it.
|
|
|
|
bool has_source() const {
|
|
|
|
// Should never have a nonempty range with an invalid offset.
|
2014-09-30 02:29:50 +08:00
|
|
|
assert(this->source_start != SOURCE_OFFSET_INVALID || this->source_length == 0);
|
|
|
|
return this->source_length > 0;
|
2013-08-09 06:06:46 +08:00
|
|
|
}
|
2014-01-15 17:40:40 +08:00
|
|
|
|
2016-05-03 07:09:46 +08:00
|
|
|
/// Indicate if the node has comment nodes.
|
2016-10-21 12:14:40 +08:00
|
|
|
bool has_comments() const {
|
|
|
|
return static_cast<bool>(this->flags & parse_node_flag_has_comments);
|
|
|
|
}
|
2014-12-24 02:58:45 +08:00
|
|
|
|
2016-05-03 07:09:46 +08:00
|
|
|
/// Gets source for the node, or the empty string if it has no source.
|
|
|
|
wcstring get_source(const wcstring &str) const {
|
|
|
|
if (!has_source())
|
2013-10-09 06:05:30 +08:00
|
|
|
return wcstring();
|
|
|
|
else
|
|
|
|
return wcstring(str, this->source_start, this->source_length);
|
|
|
|
}
|
2014-01-15 17:40:40 +08:00
|
|
|
|
2016-05-03 07:09:46 +08:00
|
|
|
/// Returns whether the given location is within the source range or at its end.
|
|
|
|
bool location_in_or_at_end_of_source_range(size_t loc) const {
|
2013-10-13 09:17:03 +08:00
|
|
|
return has_source() && source_start <= loc && loc - source_start <= source_length;
|
|
|
|
}
|
2013-06-12 00:37:51 +08:00
|
|
|
};
|
|
|
|
|
2018-01-14 06:25:39 +08:00
|
|
|
template <typename Type>
|
|
|
|
class tnode_t;
|
|
|
|
|
2016-05-03 07:09:46 +08:00
|
|
|
/// The parse tree itself.
|
|
|
|
class parse_node_tree_t : public std::vector<parse_node_t> {
|
|
|
|
public:
|
2016-02-28 16:33:11 +08:00
|
|
|
parse_node_tree_t() {}
|
2017-01-27 07:36:12 +08:00
|
|
|
parse_node_tree_t(parse_node_tree_t &&) = default;
|
|
|
|
parse_node_tree_t &operator=(parse_node_tree_t &&) = default;
|
2017-01-27 12:00:43 +08:00
|
|
|
parse_node_tree_t(const parse_node_tree_t &) = delete; // no copying
|
|
|
|
parse_node_tree_t &operator=(const parse_node_tree_t &) = delete; // no copying
|
2016-05-03 07:09:46 +08:00
|
|
|
|
|
|
|
// Get the node corresponding to a child of the given node, or NULL if there is no such child.
|
|
|
|
// If expected_type is provided, assert that the node has that type.
|
|
|
|
const parse_node_t *get_child(const parse_node_t &parent, node_offset_t which,
|
|
|
|
parse_token_type_t expected_type = token_type_invalid) const;
|
2014-01-15 17:40:40 +08:00
|
|
|
|
2016-05-03 07:09:46 +08:00
|
|
|
// Find the first direct child of the given node of the given type. asserts on failure.
|
2013-12-24 06:53:56 +08:00
|
|
|
const parse_node_t &find_child(const parse_node_t &parent, parse_token_type_t type) const;
|
2014-01-15 17:40:40 +08:00
|
|
|
|
2018-01-14 07:36:14 +08:00
|
|
|
template <typename Type>
|
|
|
|
tnode_t<Type> find_child(const parse_node_t &parent) const;
|
|
|
|
|
2016-05-03 07:09:46 +08:00
|
|
|
// Get the node corresponding to the parent of the given node, or NULL if there is no such
|
|
|
|
// child. If expected_type is provided, only returns the parent if it is of that type. Note the
|
|
|
|
// asymmetry: get_child asserts since the children are known, but get_parent does not, since the
|
|
|
|
// parent may not be known.
|
|
|
|
const parse_node_t *get_parent(const parse_node_t &node,
|
|
|
|
parse_token_type_t expected_type = token_type_invalid) const;
|
2014-01-15 17:40:40 +08:00
|
|
|
|
2018-01-14 06:25:39 +08:00
|
|
|
// Finds the last node of a given type, or empty if it could not be found. If parent is NULL,
|
|
|
|
// this finds the last node in the tree of that type.
|
|
|
|
template <typename Type>
|
|
|
|
tnode_t<Type> find_last_node(const parse_node_t *parent = NULL) const;
|
2016-05-03 07:09:46 +08:00
|
|
|
|
|
|
|
// Finds a node containing the given source location. If 'parent' is not NULL, it must be an
|
|
|
|
// ancestor.
|
|
|
|
const parse_node_t *find_node_matching_source_location(parse_token_type_t type,
|
|
|
|
size_t source_loc,
|
|
|
|
const parse_node_t *parent) const;
|
|
|
|
// Utilities
|
2014-01-15 17:40:40 +08:00
|
|
|
|
2018-01-16 10:41:14 +08:00
|
|
|
/// Given a node, return all of its comment nodes.
|
2018-01-16 14:13:37 +08:00
|
|
|
std::vector<tnode_t<grammar::comment>> comment_nodes_for_node(const parse_node_t &node) const;
|
2014-01-15 17:40:40 +08:00
|
|
|
|
2018-01-16 10:41:14 +08:00
|
|
|
private:
|
|
|
|
template <typename Type>
|
|
|
|
friend class tnode_t;
|
2016-05-03 07:09:46 +08:00
|
|
|
/// Given a node list (e.g. of type symbol_job_list) and a node type (e.g. symbol_job), return
|
|
|
|
/// the next element of the given type in that list, and the tail (by reference). Returns NULL
|
|
|
|
/// if we've exhausted the list.
|
|
|
|
const parse_node_t *next_node_in_node_list(const parse_node_t &node_list,
|
|
|
|
parse_token_type_t item_type,
|
|
|
|
const parse_node_t **list_tail) const;
|
2014-01-15 17:40:40 +08:00
|
|
|
|
2018-01-14 06:25:39 +08:00
|
|
|
// Finds the last node of a given type underneath a given node, or NULL if it could not be
|
|
|
|
// found. If parent is NULL, this finds the last node in the tree of that type.
|
|
|
|
const parse_node_t *find_last_node_of_type(parse_token_type_t type,
|
|
|
|
const parse_node_t *parent) const;
|
2013-10-07 16:04:37 +08:00
|
|
|
};
|
|
|
|
|
2016-05-03 07:09:46 +08:00
|
|
|
/// The big entry point. Parse a string, attempting to produce a tree for the given goal type.
|
|
|
|
bool parse_tree_from_string(const wcstring &str, parse_tree_flags_t flags,
|
|
|
|
parse_node_tree_t *output, parse_error_list_t *errors,
|
|
|
|
parse_token_type_t goal = symbol_job_list);
|
|
|
|
|
2017-12-23 06:40:15 +08:00
|
|
|
/// A type wrapping up a parse tree and the original source behind it.
|
|
|
|
struct parsed_source_t {
|
|
|
|
wcstring src;
|
|
|
|
parse_node_tree_t tree;
|
|
|
|
|
|
|
|
parsed_source_t(wcstring s, parse_node_tree_t t) : src(std::move(s)), tree(std::move(t)) {}
|
|
|
|
|
|
|
|
parsed_source_t(const parsed_source_t &) = delete;
|
|
|
|
void operator=(const parsed_source_t &) = delete;
|
|
|
|
parsed_source_t(parsed_source_t &&) = default;
|
|
|
|
parsed_source_t &operator=(parsed_source_t &&) = default;
|
|
|
|
};
|
|
|
|
/// Return a shared pointer to parsed_source_t, or null on failure.
|
2017-12-23 07:44:14 +08:00
|
|
|
using parsed_source_ref_t = std::shared_ptr<const parsed_source_t>;
|
2017-12-23 06:40:15 +08:00
|
|
|
parsed_source_ref_t parse_source(wcstring src, parse_tree_flags_t flags, parse_error_list_t *errors,
|
|
|
|
parse_token_type_t goal = symbol_job_list);
|
|
|
|
|
2013-05-27 03:12:16 +08:00
|
|
|
#endif
|