Correct escaping behavior in new tokenizer code

This commit is contained in:
Mahmoud Al-Qudsi 2018-03-11 17:10:16 -05:00
parent f508a1f274
commit df89d71237

View File

@ -154,6 +154,10 @@ tok_t tokenizer_t::read_string() {
tok_mode mode_begin = mode;
#endif
if (c == L'\0') {
break;
}
// Make sure this character isn't being escaped before anything else
if ((mode & tok_mode::char_escape) == tok_mode::char_escape) {
mode &= ~(tok_mode::char_escape);
@ -163,10 +167,6 @@ tok_t tokenizer_t::read_string() {
// Early exit optimization in case the character is just a letter,
// which has no special meaning to the tokenizer, i.e. the same mode continues.
}
// This check has to be after the char_escape check above
else if (c == L'\0') {
break;
}
// Now proceed with the evaluation of the token, first checking to see if the token
// has been explicitly ignored (escaped).
@ -244,7 +244,7 @@ tok_t tokenizer_t::read_string() {
tok_t error;
if ((mode & tok_mode::char_escape) == tok_mode::char_escape) {
error = this->call_error(TOK_UNTERMINATED_ESCAPE, buff_start,
this->buff);
this->buff - 1);
}
else if ((mode & tok_mode::array_brackets) == tok_mode::array_brackets) {
error = this->call_error(TOK_UNTERMINATED_SLICE, buff_start,