diff --git a/script/fetch-fixtures b/script/fetch-fixtures index 7009d70f..d31e8a75 100755 --- a/script/fetch-fixtures +++ b/script/fetch-fixtures @@ -2,19 +2,13 @@ GRAMMARS_DIR=$(dirname $0)/../spec/fixtures/grammars -GRAMMARS=( - javascript - json - c - cpp - python -) +fetch_grammar() { + local grammar=$1 + local ref=$2 + local grammar_dir=${GRAMMARS_DIR}/${grammar} + local grammar_url=https://github.com/tree-sitter/tree-sitter-${grammar} -for grammar in ${GRAMMARS[@]}; do - echo "Fetching ${grammar} grammar..." - - grammar_dir=${GRAMMARS_DIR}/${grammar} - grammar_url=https://github.com/tree-sitter/tree-sitter-${grammar} + echo "Updating ${grammar} grammar..." if [ ! -d $grammar_dir ]; then git clone $grammar_url $grammar_dir @@ -23,6 +17,12 @@ for grammar in ${GRAMMARS[@]}; do ( cd $grammar_dir; git fetch origin - git reset --hard origin/master; + git reset --hard $ref; ) -done +} + +fetch_grammar 'javascript' '76cd7dd5eb793db21640c725e58301bde83781f7' +fetch_grammar 'json' 'origin/master' +fetch_grammar 'c' 'origin/master' +fetch_grammar 'cpp' 'origin/master' +fetch_grammar 'python' 'origin/master' diff --git a/spec/fixtures/error_corpus/javascript_errors.txt b/spec/fixtures/error_corpus/javascript_errors.txt index 87ae22a8..19e1587e 100644 --- a/spec/fixtures/error_corpus/javascript_errors.txt +++ b/spec/fixtures/error_corpus/javascript_errors.txt @@ -14,8 +14,10 @@ e f; (ERROR (identifier)) (identifier) (statement_block - (expression_statement (ERROR (identifier)) (identifier)))) - (expression_statement (ERROR (identifier)) (identifier))) + (ERROR (identifier)) + (expression_statement (identifier)))) + (ERROR (identifier)) + (expression_statement (identifier))) ======================================================= multiple invalid tokens right after the viable prefix @@ -33,7 +35,8 @@ h i j k; (ERROR (identifier) (identifier)) (identifier) (statement_block - (expression_statement (ERROR (identifier) (identifier) (identifier)) (identifier)))) + (ERROR (identifier) (identifier) (identifier)) + (expression_statement (identifier)))) (expression_statement (ERROR (identifier) (identifier) (identifier)) (identifier))) @@ -76,6 +79,24 @@ a.b = (member_access (identifier) (identifier))) (ERROR)) +================================================================= +An invalid token at the end of a construct with extra line breaks +================================================================= + +a( + b, + c,, +); + +--- + +(program + (expression_statement + (function_call (identifier) (arguments + (identifier) + (identifier) + (ERROR))))) + =================================================== Multi-line chained expressions in var declarations =================================================== diff --git a/src/runtime/parser.c b/src/runtime/parser.c index 191354a3..3323f19d 100644 --- a/src/runtime/parser.c +++ b/src/runtime/parser.c @@ -260,6 +260,7 @@ static Tree *parser__lex(Parser *self, StackVersion version) { LOG("skip_unrecognized_character"); skipped_error = true; error_start_position = self->lexer.token_start_position; + error_end_position = self->lexer.token_start_position; first_error_character = self->lexer.data.lookahead; } @@ -596,6 +597,7 @@ static inline const TSParseAction *parser__reductions_after_sequence( if (child_count == tree_count_below) break; Tree *tree = trees_below->contents[trees_below->size - 1 - i]; + if (tree->extra) continue; TSStateId next_state = ts_language_next_state(self->language, state, tree->symbol); if (next_state == ERROR_STATE) return NULL; @@ -607,6 +609,7 @@ static inline const TSParseAction *parser__reductions_after_sequence( for (uint32_t i = 0; i < trees_above->size; i++) { Tree *tree = trees_above->contents[i]; + if (tree->extra) continue; TSStateId next_state = ts_language_next_state(self->language, state, tree->symbol); if (next_state == ERROR_STATE) return NULL; @@ -738,7 +741,6 @@ static bool parser__repair_error(Parser *self, StackSlice slice, ReduceAction repair = session.best_repair; TSStateId next_state = session.best_repair_next_state; uint32_t skip_count = session.best_repair_skip_count; - uint32_t count_below = repair.count - session.tree_count_above_error; TSSymbol symbol = repair.symbol; StackSlice new_slice = array_pop(&pop.slices); @@ -752,13 +754,8 @@ static bool parser__repair_error(Parser *self, StackSlice slice, ts_stack_remove_version(self->stack, other_slice.version); } - TreeArray skipped_children = array_new(); - array_grow(&skipped_children, skip_count); - for (uint32_t i = count_below; i < children.size; i++) - array_push(&skipped_children, children.contents[i]); - + TreeArray skipped_children = ts_tree_array_remove_last_n(&children, skip_count); Tree *error = ts_tree_make_error_node(&skipped_children); - children.size = count_below; array_push(&children, error); for (uint32_t i = 0; i < slice.trees.size; i++) diff --git a/src/runtime/tree.c b/src/runtime/tree.c index 858ad90e..0b10604e 100644 --- a/src/runtime/tree.c +++ b/src/runtime/tree.c @@ -61,6 +61,29 @@ uint32_t ts_tree_array_essential_count(const TreeArray *self) { return result; } +TreeArray ts_tree_array_remove_last_n(TreeArray *self, uint32_t remove_count) { + TreeArray result = array_new(); + if (self->size == 0 || remove_count == 0) return result; + + uint32_t count = 0; + uint32_t split_index = self->size - 1; + for (; split_index + 1 > 0; split_index--) { + Tree *tree = self->contents[split_index]; + if (!tree->extra) { + count++; + if (count == remove_count) break; + } + } + + array_grow(&result, self->size - split_index); + for (uint32_t i = split_index; i < self->size; i++) { + array_push(&result, self->contents[i]); + } + + self->size = split_index; + return result; +} + Tree *ts_tree_make_error(Length size, Length padding, char lookahead_char) { Tree *result = ts_tree_make_leaf(ts_builtin_sym_error, padding, size, (TSSymbolMetadata){ diff --git a/src/runtime/tree.h b/src/runtime/tree.h index d5916e31..69ba8c38 100644 --- a/src/runtime/tree.h +++ b/src/runtime/tree.h @@ -68,6 +68,7 @@ typedef Array(TreePathEntry) TreePath; bool ts_tree_array_copy(TreeArray, TreeArray *); void ts_tree_array_delete(TreeArray *); uint32_t ts_tree_array_essential_count(const TreeArray *); +TreeArray ts_tree_array_remove_last_n(TreeArray *, uint32_t); Tree *ts_tree_make_leaf(TSSymbol, Length, Length, TSSymbolMetadata); Tree *ts_tree_make_node(TSSymbol, uint32_t, Tree **, TSSymbolMetadata);