From 379a2fd1214960d1896915ba88df84a0a9b3ced1 Mon Sep 17 00:00:00 2001 From: Max Brunsfeld Date: Fri, 6 Apr 2018 09:35:17 -0700 Subject: [PATCH] Incrementally build a tree of skipped tokens Rather than pushing them to the stack individually --- .../build_tables/parse_table_builder.cc | 8 +- src/runtime/language.c | 8 +- src/runtime/language.h | 4 +- src/runtime/parser.c | 195 +++++++++++++----- src/runtime/stack.c | 50 ++--- src/runtime/stack.h | 2 +- src/runtime/tree.c | 33 ++- test/fixtures/error_corpus/c_errors.txt | 4 +- .../error_corpus/javascript_errors.txt | 45 +++- 9 files changed, 233 insertions(+), 116 deletions(-) diff --git a/src/compiler/build_tables/parse_table_builder.cc b/src/compiler/build_tables/parse_table_builder.cc index 51ed5cc4..6218fec6 100644 --- a/src/compiler/build_tables/parse_table_builder.cc +++ b/src/compiler/build_tables/parse_table_builder.cc @@ -150,6 +150,8 @@ class ParseTableBuilderImpl : public ParseTableBuilder { MatchesLongerStringWithValidNextChar ); + parse_table.states[state_id].terminal_entries.clear(); + // Add all the tokens that have no conflict with other tokens. LookaheadSet non_conflicting_tokens; for (unsigned i = 0; i < lexical_grammar.variables.size(); i++) { @@ -186,12 +188,6 @@ class ParseTableBuilderImpl : public ParseTableBuilder { } } - for (const Symbol &symbol : grammar.extra_tokens) { - if (!parse_table.states[state_id].terminal_entries.count(symbol)) { - parse_table.add_terminal_action(state_id, symbol, ParseAction::ShiftExtra()); - } - } - for (size_t i = 0; i < grammar.external_tokens.size(); i++) { if (grammar.external_tokens[i].corresponding_internal_token == rules::NONE()) { parse_table.states[state_id].terminal_entries[Symbol::external(i)].actions.push_back(ParseAction::Recover()); diff --git a/src/runtime/language.c b/src/runtime/language.c index 7ef941ae..9bf1fc63 100644 --- a/src/runtime/language.c +++ b/src/runtime/language.c @@ -4,7 +4,7 @@ void ts_language_table_entry(const TSLanguage *self, TSStateId state, TSSymbol symbol, TableEntry *result) { - if (symbol == ts_builtin_sym_error) { + if (symbol == ts_builtin_sym_error || symbol == ts_builtin_sym_error_repeat) { result->action_count = 0; result->is_reusable = false; result->actions = NULL; @@ -27,8 +27,10 @@ uint32_t ts_language_version(const TSLanguage *language) { } TSSymbolMetadata ts_language_symbol_metadata(const TSLanguage *language, TSSymbol symbol) { - if (symbol == ts_builtin_sym_error) { + if (symbol == ts_builtin_sym_error) { return (TSSymbolMetadata){.visible = true, .named = true}; + } else if (symbol == ts_builtin_sym_error_repeat) { + return (TSSymbolMetadata){.visible = false, .named = false}; } else { return language->symbol_metadata[symbol]; } @@ -37,6 +39,8 @@ TSSymbolMetadata ts_language_symbol_metadata(const TSLanguage *language, TSSymbo const char *ts_language_symbol_name(const TSLanguage *language, TSSymbol symbol) { if (symbol == ts_builtin_sym_error) { return "ERROR"; + } else if (symbol == ts_builtin_sym_error_repeat) { + return "_ERROR"; } else { return language->symbol_names[symbol]; } diff --git a/src/runtime/language.h b/src/runtime/language.h index 966d15df..64733242 100644 --- a/src/runtime/language.h +++ b/src/runtime/language.h @@ -8,6 +8,8 @@ extern "C" { #include "tree_sitter/parser.h" #include "runtime/tree.h" +#define ts_builtin_sym_error_repeat (ts_builtin_sym_error - 1) + typedef struct { const TSParseAction *actions; uint32_t action_count; @@ -51,7 +53,7 @@ static inline bool ts_language_has_reduce_action(const TSLanguage *self, static inline TSStateId ts_language_next_state(const TSLanguage *self, TSStateId state, TSSymbol symbol) { - if (symbol == ts_builtin_sym_error) { + if (symbol == ts_builtin_sym_error || symbol == ts_builtin_sym_error_repeat) { return 0; } else if (symbol < self->token_count) { uint32_t count; diff --git a/src/runtime/parser.c b/src/runtime/parser.c index 0977a077..23ce42b1 100644 --- a/src/runtime/parser.c +++ b/src/runtime/parser.c @@ -639,6 +639,20 @@ static StackSliceArray parser__reduce(Parser *self, StackVersion version, TSSymb for (uint32_t j = parent->children.size; j < slice.trees.size; j++) { ts_stack_push(self->stack, slice.version, slice.trees.contents[j], false, next_state); } + + if (ts_stack_version_count(self->stack) > MAX_VERSION_COUNT) { + i++; + while (i < pop.size) { + StackSlice slice = pop.contents[i]; + ts_tree_array_delete(&self->tree_pool, &slice.trees); + ts_stack_halt(self->stack, slice.version); + i++; + } + while (ts_stack_version_count(self->stack) > slice.version + 1) { + ts_stack_remove_version(self->stack, slice.version + 1); + } + break; + } } for (StackVersion i = initial_version_count; i < ts_stack_version_count(self->stack); i++) { @@ -720,12 +734,23 @@ static void parser__accept(Parser *self, StackVersion version, Tree *lookahead) static bool parser__do_all_potential_reductions(Parser *self, StackVersion starting_version, TSSymbol lookahead_symbol) { - bool result = false; - for (StackVersion version = starting_version; - ts_stack_version_count(self->stack) < MAX_VERSION_COUNT;) { + uint32_t initial_version_count = ts_stack_version_count(self->stack); + + bool can_shift_lookahead_symbol = false; + StackVersion version = starting_version; + for (unsigned i = 0; true; i++) { uint32_t version_count = ts_stack_version_count(self->stack); if (version >= version_count) break; + bool merged = false; + for (StackVersion i = initial_version_count; i < version; i++) { + if (ts_stack_merge(self->stack, i, version)) { + merged = true; + break; + } + } + if (merged) continue; + TSStateId state = ts_stack_state(self->stack, version); bool has_shift_action = false; array_clear(&self->reduce_actions); @@ -747,7 +772,7 @@ static bool parser__do_all_potential_reductions(Parser *self, StackVersion start switch (action.type) { case TSParseActionTypeShift: case TSParseActionTypeRecover: - if (!action.params.extra) has_shift_action = true; + if (!action.params.extra && !action.params.repetition) has_shift_action = true; break; case TSParseActionTypeReduce: if (action.params.child_count > 0) @@ -763,9 +788,9 @@ static bool parser__do_all_potential_reductions(Parser *self, StackVersion start } } - bool has_reduce_action = self->reduce_actions.size > 0; for (uint32_t i = 0; i < self->reduce_actions.size; i++) { ReduceAction action = self->reduce_actions.contents[i]; + parser__reduce( self, version, action.symbol, action.count, action.dynamic_precedence, action.alias_sequence_id, @@ -774,14 +799,12 @@ static bool parser__do_all_potential_reductions(Parser *self, StackVersion start } if (has_shift_action) { - result = true; - } else { - if (has_reduce_action) { - ts_stack_renumber_version(self->stack, version_count, version); - continue; - } else if (lookahead_symbol != 0) { - ts_stack_remove_version(self->stack, version); - } + can_shift_lookahead_symbol = true; + } else if (self->reduce_actions.size > 0 && i < MAX_VERSION_COUNT) { + ts_stack_renumber_version(self->stack, version_count, version); + continue; + } else if (lookahead_symbol != 0) { + ts_stack_remove_version(self->stack, version); } if (version == starting_version) { @@ -790,7 +813,8 @@ static bool parser__do_all_potential_reductions(Parser *self, StackVersion start version++; } } - return result; + + return can_shift_lookahead_symbol; } static void parser__handle_error(Parser *self, StackVersion version, TSSymbol lookahead_symbol) { @@ -830,7 +854,11 @@ static void parser__handle_error(Parser *self, StackVersion version, TSSymbol lo self, version_with_missing_tree, lookahead_symbol )) { - LOG("recover_with_missing symbol:%s, state:%u", SYM_NAME(missing_symbol), state_after_missing_symbol); + LOG( + "recover_with_missing symbol:%s, state:%u", + SYM_NAME(missing_symbol), + ts_stack_state(self->stack, version_with_missing_tree) + ); did_insert_missing_token = true; break; } @@ -894,13 +922,14 @@ static bool parser__recover_to_state(Parser *self, StackVersion version, unsigne continue; } - StackSliceArray error_pop = ts_stack_pop_error(self->stack, slice.version); - if (error_pop.size > 0) { - StackSlice error_slice = error_pop.contents[0]; - array_push_all(&error_slice.trees, &slice.trees); - array_delete(&slice.trees); - slice.trees = error_slice.trees; - ts_stack_renumber_version(self->stack, error_slice.version, slice.version); + TreeArray error_trees = ts_stack_pop_error(self->stack, slice.version); + if (error_trees.size > 0) { + assert(error_trees.size == 1); + array_splice(&slice.trees, 0, 0, &error_trees.contents[0]->children); + for (unsigned j = 0; j < error_trees.contents[0]->children.size; j++) { + ts_tree_retain(slice.trees.contents[j]); + } + ts_tree_array_delete(&self->tree_pool, &error_trees); } TreeArray trailing_extras = ts_tree_array_remove_trailing_extras(&slice.trees); @@ -930,41 +959,51 @@ static void parser__recover(Parser *self, StackVersion version, Tree *lookahead) unsigned previous_version_count = ts_stack_version_count(self->stack); Length position = ts_stack_position(self->stack, version); StackSummary *summary = ts_stack_get_summary(self->stack, version); - unsigned depth_since_error = ts_stack_node_count_since_error(self->stack, version); + unsigned node_count_since_error = ts_stack_node_count_since_error(self->stack, version); + unsigned current_error_cost = ts_stack_error_cost(self->stack, version); - for (unsigned i = 0; i < summary->size; i++) { - StackSummaryEntry entry = summary->contents[i]; + if (summary && lookahead->symbol != ts_builtin_sym_error) { + for (unsigned i = 0; i < summary->size; i++) { + StackSummaryEntry entry = summary->contents[i]; - if (entry.state == ERROR_STATE) continue; - if (entry.position.bytes == position.bytes) continue; - unsigned depth = entry.depth + depth_since_error; - if (depth > MAX_SUMMARY_DEPTH) break; + if (entry.state == ERROR_STATE) continue; + if (entry.position.bytes == position.bytes) continue; + unsigned depth = entry.depth; + if (node_count_since_error > 0) depth++; - unsigned new_cost = - depth * ERROR_COST_PER_SKIPPED_TREE + - (position.bytes - entry.position.bytes) * ERROR_COST_PER_SKIPPED_CHAR + - (position.extent.row - entry.position.extent.row) * ERROR_COST_PER_SKIPPED_LINE; - if (parser__better_version_exists(self, version, false, new_cost)) break; + bool would_merge = false; + for (unsigned j = 0; j < previous_version_count; j++) { + if ( + ts_stack_state(self->stack, j) == entry.state && + ts_stack_position(self->stack, j).bytes == position.bytes + ) { + would_merge = true; + break; + } + } - if (ts_language_has_actions(self->language, entry.state, lookahead->symbol)) { - if (parser__recover_to_state(self, version, depth, entry.state)) { - did_recover = true; - LOG("recover state:%u, depth:%u", entry.state, depth); - LOG_STACK(); - break; + if (would_merge) continue; + + unsigned new_cost = + current_error_cost + + entry.depth * ERROR_COST_PER_SKIPPED_TREE + + (position.bytes - entry.position.bytes) * ERROR_COST_PER_SKIPPED_CHAR + + (position.extent.row - entry.position.extent.row) * ERROR_COST_PER_SKIPPED_LINE; + if (parser__better_version_exists(self, version, false, new_cost)) break; + + if (ts_language_has_actions(self->language, entry.state, lookahead->symbol)) { + if (parser__recover_to_state(self, version, depth, entry.state)) { + did_recover = true; + LOG("recover_to_previous state:%u, depth:%u", entry.state, depth); + LOG_STACK(); + break; + } } } } for (unsigned i = previous_version_count; i < ts_stack_version_count(self->stack); i++) { - if (ts_stack_is_active(self->stack, i)) { - for (unsigned j = 0; j < i; j++) { - if (ts_stack_can_merge(self->stack, j, i)) { - ts_stack_remove_version(self->stack, i--); - break; - } - } - } else { + if (!ts_stack_is_active(self->stack, i)) { ts_stack_remove_version(self->stack, i--); } } @@ -983,15 +1022,56 @@ static void parser__recover(Parser *self, StackVersion version, Tree *lookahead) return; } + unsigned new_cost = + current_error_cost + ERROR_COST_PER_SKIPPED_TREE + + ts_tree_total_bytes(lookahead) * ERROR_COST_PER_SKIPPED_CHAR + + ts_tree_total_size(lookahead).extent.row * ERROR_COST_PER_SKIPPED_LINE; + + if (parser__better_version_exists(self, version, false, new_cost)) { + ts_stack_halt(self->stack, version); + return; + } + unsigned n; const TSParseAction *actions = ts_language_actions(self->language, 1, lookahead->symbol, &n); - bool extra = n > 0 && actions[n - 1].type == TSParseActionTypeShift && actions[n - 1].params.extra; - parser__shift(self, version, ERROR_STATE, lookahead, extra); + if (n > 0 && actions[n - 1].type == TSParseActionTypeShift && actions[n - 1].params.extra) { + lookahead->extra = true; + } - if (parser__better_version_exists(self, version, true, ts_stack_error_cost(self->stack, version))) { - ts_stack_halt(self->stack, version); - } else { - LOG("skip_token symbol:%s", SYM_NAME(lookahead->symbol)); + LOG("skip_token symbol:%s", SYM_NAME(lookahead->symbol)); + ts_tree_retain(lookahead); + TreeArray children = array_new(); + array_grow(&children, 1); + array_push(&children, lookahead); + Tree *error_repeat = ts_tree_make_node( + &self->tree_pool, + ts_builtin_sym_error_repeat, + &children, + 0, + self->language + ); + + if (node_count_since_error > 0) { + StackSliceArray pop = ts_stack_pop_count(self->stack, version, 1); + assert(pop.size == 1); + assert(pop.contents[0].trees.size == 1); + ts_stack_renumber_version(self->stack, pop.contents[0].version, version); + array_push(&pop.contents[0].trees, error_repeat); + error_repeat = ts_tree_make_node( + &self->tree_pool, + ts_builtin_sym_error_repeat, + &pop.contents[0].trees, + 0, + self->language + ); + } + + ts_stack_push(self->stack, version, error_repeat, false, ERROR_STATE); + + if (lookahead->has_external_tokens) { + ts_stack_set_last_external_token( + self->stack, version, ts_tree_last_external_token(lookahead) + ); } } @@ -1011,6 +1091,10 @@ static void parser__advance(Parser *self, StackVersion version, ReusableNode *re if (action.params.repetition) break; TSStateId next_state; if (action.params.extra) { + + // TODO remove when TREE_SITTER_LANGUAGE_VERSION 9 is out. + if (state == ERROR_STATE) continue; + next_state = state; LOG("shift_extra"); } else { @@ -1065,7 +1149,8 @@ static void parser__advance(Parser *self, StackVersion version, ReusableNode *re ts_stack_renumber_version(self->stack, last_reduction_version, version); LOG_STACK(); } else if (state == ERROR_STATE) { - ts_stack_push(self->stack, version, lookahead, false, ERROR_STATE); + parser__recover(self, version, lookahead); + ts_tree_release(&self->tree_pool, lookahead); return; } else if (!parser__breakdown_top_of_stack(self, version)) { LOG("detect_error"); diff --git a/src/runtime/stack.c b/src/runtime/stack.c index e05d6a06..4d9f5efe 100644 --- a/src/runtime/stack.c +++ b/src/runtime/stack.c @@ -142,21 +142,7 @@ static StackNode *stack_node_new(StackNode *previous_node, Tree *tree, bool is_p node->error_cost += tree->error_cost; node->position = length_add(node->position, ts_tree_total_size(tree)); node->dynamic_precedence += tree->dynamic_precedence; - if (!tree->extra) { - node->node_count += tree->node_count; - - if (state == ERROR_STATE) { - node->error_cost += - ERROR_COST_PER_SKIPPED_TREE * ((tree->visible || tree->children.size == 0) ? 1 : tree->visible_child_count) + - ERROR_COST_PER_SKIPPED_CHAR * tree->size.bytes + - ERROR_COST_PER_SKIPPED_LINE * tree->size.extent.row; - if (previous_node->links[0].tree) { - node->error_cost += - ERROR_COST_PER_SKIPPED_CHAR * tree->padding.bytes + - ERROR_COST_PER_SKIPPED_LINE * tree->padding.extent.row; - } - } - } + if (!tree->extra) node->node_count += tree->node_count; } } else { node->position = length_zero(); @@ -400,7 +386,9 @@ void ts_stack_set_last_external_token(Stack *self, StackVersion version, Tree *t unsigned ts_stack_error_cost(const Stack *self, StackVersion version) { StackHead *head = array_get(&self->heads, version); unsigned result = head->node->error_cost; - if (head->node->state == ERROR_STATE || head->status == StackStatusPaused) { + if ( + head->status == StackStatusPaused || + (head->node->state == ERROR_STATE && !head->node->links[0].tree)) { result += ERROR_COST_PER_RECOVERY; } return result; @@ -408,6 +396,9 @@ unsigned ts_stack_error_cost(const Stack *self, StackVersion version) { unsigned ts_stack_node_count_since_error(const Stack *self, StackVersion version) { StackHead *head = array_get(&self->heads, version); + if (head->node->node_count < head->node_count_at_last_error) { + head->node_count_at_last_error = head->node->node_count; + } return head->node->node_count - head->node_count_at_last_error; } @@ -482,15 +473,21 @@ inline StackAction pop_error_callback(void *payload, const Iterator *iterator) { } } -StackSliceArray ts_stack_pop_error(Stack *self, StackVersion version) { +TreeArray ts_stack_pop_error(Stack *self, StackVersion version) { StackNode *node = array_get(&self->heads, version)->node; for (unsigned i = 0; i < node->link_count; i++) { if (node->links[i].tree && node->links[i].tree->symbol == ts_builtin_sym_error) { bool found_error = false; - return stack__iter(self, version, pop_error_callback, &found_error, true); + StackSliceArray pop = stack__iter(self, version, pop_error_callback, &found_error, true); + if (pop.size > 0) { + assert(pop.size == 1); + ts_stack_renumber_version(self, pop.contents[0].version, version); + return pop.contents[0].trees; + } + break; } } - return (StackSliceArray){.size = 0}; + return (TreeArray){.size = 0}; } inline StackAction pop_all_callback(void *payload, const Iterator *iterator) { @@ -550,8 +547,14 @@ void ts_stack_remove_version(Stack *self, StackVersion version) { void ts_stack_renumber_version(Stack *self, StackVersion v1, StackVersion v2) { assert(v2 < v1); assert((uint32_t)v1 < self->heads.size); - stack_head_delete(&self->heads.contents[v2], &self->node_pool, self->tree_pool); - self->heads.contents[v2] = self->heads.contents[v1]; + StackHead *source_head = &self->heads.contents[v1]; + StackHead *target_head = &self->heads.contents[v2]; + if (target_head->summary && !source_head->summary) { + source_head->summary = target_head->summary; + target_head->summary = NULL; + } + stack_head_delete(target_head, &self->node_pool, self->tree_pool); + *target_head = *source_head; array_erase(&self->heads, v1); } @@ -578,8 +581,8 @@ bool ts_stack_merge(Stack *self, StackVersion version1, StackVersion version2) { for (uint32_t i = 0; i < head2->node->link_count; i++) { stack_node_add_link(head1->node, head2->node->links[i]); } - if (head2->node_count_at_last_error > head1->node_count_at_last_error) { - head1->node_count_at_last_error = head2->node_count_at_last_error; + if (head1->node->state == ERROR_STATE) { + head1->node_count_at_last_error = head1->node->node_count; } ts_stack_remove_version(self, version2); return true; @@ -593,6 +596,7 @@ bool ts_stack_can_merge(Stack *self, StackVersion version1, StackVersion version head2->status == StackStatusActive && head1->node->state == head2->node->state && head1->node->position.bytes == head2->node->position.bytes && + head1->node->error_cost == head2->node->error_cost && ts_tree_external_token_state_eq(head1->last_external_token, head2->last_external_token); } diff --git a/src/runtime/stack.h b/src/runtime/stack.h index 92a09b69..32e68e6e 100644 --- a/src/runtime/stack.h +++ b/src/runtime/stack.h @@ -65,7 +65,7 @@ void ts_stack_push(Stack *, StackVersion, Tree *, bool, TSStateId); StackSliceArray ts_stack_pop_count(Stack *, StackVersion, uint32_t count); // Remove an error at the top of the given version of the stack. -StackSliceArray ts_stack_pop_error(Stack *, StackVersion); +TreeArray ts_stack_pop_error(Stack *, StackVersion); // Remove any pending trees from the top of the given version of the stack. StackSliceArray ts_stack_pop_pending(Stack *, StackVersion); diff --git a/src/runtime/tree.c b/src/runtime/tree.c index f6dc542f..f0ffd1b4 100644 --- a/src/runtime/tree.c +++ b/src/runtime/tree.c @@ -324,7 +324,9 @@ void ts_tree_set_children(Tree *self, TreeArray *children, const TSLanguage *lan self->size = length_add(self->size, ts_tree_total_size(child)); } - self->error_cost += child->error_cost; + if (child->symbol != ts_builtin_sym_error_repeat) { + self->error_cost += child->error_cost; + } self->dynamic_precedence += child->dynamic_precedence; self->node_count += child->node_count; @@ -351,13 +353,18 @@ void ts_tree_set_children(Tree *self, TreeArray *children, const TSLanguage *lan if (!child->extra) non_extra_index++; } - if (self->symbol == ts_builtin_sym_error) { + if (self->symbol == ts_builtin_sym_error || self->symbol == ts_builtin_sym_error_repeat) { self->error_cost += ERROR_COST_PER_RECOVERY + ERROR_COST_PER_SKIPPED_CHAR * self->size.bytes + ERROR_COST_PER_SKIPPED_LINE * self->size.extent.row; for (uint32_t i = 0; i < self->children.size; i++) { - if (!self->children.contents[i]->extra) { + Tree *child = self->children.contents[i]; + if (child->extra) continue; + if (child->symbol == ts_builtin_sym_error && child->children.size == 0) continue; + if (child->visible) { self->error_cost += ERROR_COST_PER_SKIPPED_TREE; + } else { + self->error_cost += ERROR_COST_PER_SKIPPED_TREE * child->visible_child_count; } } } @@ -387,26 +394,16 @@ Tree *ts_tree_make_node(TreePool *pool, TSSymbol symbol, TreeArray *children, unsigned alias_sequence_id, const TSLanguage *language) { Tree *result = ts_tree_make_leaf(pool, symbol, length_zero(), length_zero(), language); result->alias_sequence_id = alias_sequence_id; + if (symbol == ts_builtin_sym_error || symbol == ts_builtin_sym_error_repeat) { + result->fragile_left = true; + result->fragile_right = true; + } ts_tree_set_children(result, children, language); return result; } Tree *ts_tree_make_error_node(TreePool *pool, TreeArray *children, const TSLanguage *language) { - for (uint32_t i = 0; i < children->size; i++) { - Tree *child = children->contents[i]; - if (child->symbol == ts_builtin_sym_error && child->children.size > 0) { - array_splice(children, i, 1, &child->children); - i += child->children.size - 1; - for (uint32_t j = 0; j < child->children.size; j++) - ts_tree_retain(child->children.contents[j]); - ts_tree_release(pool, child); - } - } - - Tree *result = ts_tree_make_node(pool, ts_builtin_sym_error, children, 0, language); - result->fragile_left = true; - result->fragile_right = true; - return result; + return ts_tree_make_node(pool, ts_builtin_sym_error, children, 0, language); } Tree *ts_tree_make_missing_leaf(TreePool *pool, TSSymbol symbol, const TSLanguage *language) { diff --git a/test/fixtures/error_corpus/c_errors.txt b/test/fixtures/error_corpus/c_errors.txt index c5833156..b2931b7d 100644 --- a/test/fixtures/error_corpus/c_errors.txt +++ b/test/fixtures/error_corpus/c_errors.txt @@ -81,7 +81,9 @@ int main() { (function_declarator (identifier) (parameter_list)) (compound_statement (if_statement - (field_expression (identifier) (MISSING)) + (field_expression + (identifier) + (MISSING)) (compound_statement (expression_statement (call_expression (identifier) (argument_list))) (expression_statement (call_expression (identifier) (argument_list))) diff --git a/test/fixtures/error_corpus/javascript_errors.txt b/test/fixtures/error_corpus/javascript_errors.txt index d435ba86..fb31d9e2 100644 --- a/test/fixtures/error_corpus/javascript_errors.txt +++ b/test/fixtures/error_corpus/javascript_errors.txt @@ -77,15 +77,12 @@ if ({a: 'b'} {c: 'd'}) { (ERROR (object (pair (property_identifier) (string)))) (object (pair (property_identifier) (string)))) (statement_block - (expression_statement (assignment_expression - (identifier) - (call_expression - (function (formal_parameters (identifier)) (statement_block (expression_statement (identifier)))) - (ERROR) - (arguments (identifier)))) - (MISSING)) - (statement_block - (expression_statement (identifier)))))) + (expression_statement + (assignment_expression + (identifier) + (function (formal_parameters (identifier)) (statement_block (expression_statement (identifier))))) + (MISSING)) + (function (formal_parameters (identifier)) (statement_block (expression_statement (identifier))))))) =================================================== Extra tokens at the end of the file @@ -150,3 +147,33 @@ const a = `b c ${d +} f g` (variable_declarator (identifier) (template_string (template_substitution (identifier) (ERROR)))))) + +========================================================= +Long sequences of invalid tokens +========================================================= + +function main(x) { + console.log('a'); + what?????????????????????????????????????????????????? + console.log('b'); + return {}; +} + +--- + +(program + (function + (identifier) + (formal_parameters (identifier)) + (statement_block + (expression_statement + (call_expression + (member_expression (identifier) (property_identifier)) + (arguments (string)))) + (expression_statement + (identifier) + (ERROR + (call_expression + (member_expression (identifier) (property_identifier)) + (arguments (string))))) + (return_statement (object)))))