diff --git a/include/tree_sitter/parser.h b/include/tree_sitter/parser.h index 05c9c31b..d99655f8 100644 --- a/include/tree_sitter/parser.h +++ b/include/tree_sitter/parser.h @@ -63,7 +63,6 @@ typedef union { struct { uint8_t count; bool reusable : 1; - bool depends_on_lookahead : 1; }; } TSParseActionEntry; diff --git a/include/tree_sitter/runtime.h b/include/tree_sitter/runtime.h index 3ed22c3e..c3bef506 100644 --- a/include/tree_sitter/runtime.h +++ b/include/tree_sitter/runtime.h @@ -9,7 +9,7 @@ extern "C" { #include #include -#define TREE_SITTER_LANGUAGE_VERSION 7 +#define TREE_SITTER_LANGUAGE_VERSION 8 typedef unsigned short TSSymbol; typedef struct TSLanguage TSLanguage; diff --git a/src/compiler/build_tables/lex_table_builder.cc b/src/compiler/build_tables/lex_table_builder.cc index 86863bdc..013a8c50 100644 --- a/src/compiler/build_tables/lex_table_builder.cc +++ b/src/compiler/build_tables/lex_table_builder.cc @@ -123,7 +123,7 @@ class LexTableBuilderImpl : public LexTableBuilder { aggregator.apply(grammar.variables[i].rule); bool all_alpha = true, all_lower = true; for (auto character : aggregator.result.included_chars) { - if (!iswalpha(character)) all_alpha = true; + if (!iswalpha(character) && character != '_') all_alpha = false; if (!iswlower(character)) all_lower = false; } @@ -190,7 +190,7 @@ class LexTableBuilderImpl : public LexTableBuilder { bool shadows_other_tokens = false; for (Symbol::Index j = 0; j < n; j++) { Symbol other_symbol = Symbol::terminal(j); - if ((get_conflict_status(other_symbol, symbol) & CannotDistinguish) && + if ((get_conflict_status(other_symbol, symbol) & (MatchesShorterStringWithinSeparators|MatchesLongerStringWithValidNextChar)) && !keyword_symbols.contains(other_symbol) && keyword_symbols.intersects(coincident_tokens_by_token[j])) { shadows_other_tokens = true; diff --git a/src/compiler/generate_code/c_code.cc b/src/compiler/generate_code/c_code.cc index 1c6db406..806adf66 100644 --- a/src/compiler/generate_code/c_code.cc +++ b/src/compiler/generate_code/c_code.cc @@ -399,7 +399,7 @@ class CCodeGenerator { } void add_parse_table() { - add_parse_action_list_id(ParseTableEntry{ {}, false, false }); + add_parse_action_list_id(ParseTableEntry{ {}, false }); size_t state_id = 0; line("static uint16_t ts_parse_table[STATE_COUNT][SYMBOL_COUNT] = {"); @@ -623,10 +623,12 @@ class CCodeGenerator { indent([&]() { for (const auto &pair : parse_table_entries) { size_t index = pair.first; - line("[" + to_string(index) + "] = {.count = " + - to_string(pair.second.actions.size()) + ", .reusable = " + - _boolean(pair.second.reusable) + ", .depends_on_lookahead = " + - _boolean(pair.second.depends_on_lookahead) + "},"); + line( + "[" + to_string(index) + "] = {" + ".count = " + to_string(pair.second.actions.size()) + ", " + ".reusable = " + _boolean(pair.second.reusable) + + "}," + ); for (const ParseAction &action : pair.second.actions) { add(" "); diff --git a/src/compiler/parse_table.cc b/src/compiler/parse_table.cc index 0252e7ad..ed41d473 100644 --- a/src/compiler/parse_table.cc +++ b/src/compiler/parse_table.cc @@ -103,20 +103,14 @@ bool ParseAction::operator<(const ParseAction &other) const { return alias_sequence_id < other.alias_sequence_id; } -ParseTableEntry::ParseTableEntry() - : reusable(true), depends_on_lookahead(false) {} +ParseTableEntry::ParseTableEntry() : reusable(true) {} -ParseTableEntry::ParseTableEntry(const vector &actions, - bool reusable, bool depends_on_lookahead) +ParseTableEntry::ParseTableEntry(const vector &actions, bool reusable) : actions(actions), - reusable(reusable), - depends_on_lookahead(depends_on_lookahead) {} + reusable(reusable) {} bool ParseTableEntry::operator==(const ParseTableEntry &other) const { - return - actions == other.actions && - reusable == other.reusable && - depends_on_lookahead == other.depends_on_lookahead; + return actions == other.actions && reusable == other.reusable; } ParseState::ParseState() : lex_state_id(-1) {} diff --git a/src/compiler/parse_table.h b/src/compiler/parse_table.h index 88adb851..770deafb 100644 --- a/src/compiler/parse_table.h +++ b/src/compiler/parse_table.h @@ -49,7 +49,7 @@ struct ParseAction { struct ParseTableEntry { ParseTableEntry(); - ParseTableEntry(const std::vector &, bool, bool); + ParseTableEntry(const std::vector &, bool); bool operator==(const ParseTableEntry &other) const; inline bool operator!=(const ParseTableEntry &other) const { return !operator==(other); @@ -57,7 +57,6 @@ struct ParseTableEntry { std::vector actions; bool reusable; - bool depends_on_lookahead; }; struct ParseState { diff --git a/src/runtime/language.c b/src/runtime/language.c index 55ae96f3..7cdc71fc 100644 --- a/src/runtime/language.c +++ b/src/runtime/language.c @@ -15,7 +15,6 @@ void ts_language_table_entry(const TSLanguage *self, TSStateId state, const TSParseActionEntry *entry = &self->parse_actions[action_index]; result->action_count = entry->count; result->is_reusable = entry->reusable; - result->depends_on_lookahead = entry->depends_on_lookahead; result->actions = (const TSParseAction *)(entry + 1); } } diff --git a/src/runtime/language.h b/src/runtime/language.h index ee01d3ec..966d15df 100644 --- a/src/runtime/language.h +++ b/src/runtime/language.h @@ -12,7 +12,6 @@ typedef struct { const TSParseAction *actions; uint32_t action_count; bool is_reusable; - bool depends_on_lookahead; } TableEntry; void ts_language_table_entry(const TSLanguage *, TSStateId, TSSymbol, TableEntry *); diff --git a/src/runtime/parser.c b/src/runtime/parser.c index 9892e297..df7395d0 100644 --- a/src/runtime/parser.c +++ b/src/runtime/parser.c @@ -487,16 +487,7 @@ static bool parser__can_reuse_first_leaf(Parser *self, TSStateId state, Tree *tr // If the current state allows external tokens or other tokens that conflict with this // token, this token is not reusable. - if (current_lex_mode.external_lex_state != 0 || - !table_entry->is_reusable) return false; - - // If the current state allows other tokens of which this token is a *prefix* and the - // content *after* this token has changed, this token isn't reusable. - if (table_entry->depends_on_lookahead && - ((tree->child_count <= 1 || tree->error_cost > 0) && - (next_reusable_node && reusable_node_has_leading_changes(next_reusable_node)))) return false; - - return true; + return current_lex_mode.external_lex_state == 0 && table_entry->is_reusable; } static Tree *parser__get_lookahead(Parser *self, StackVersion version, TSStateId *state, diff --git a/src/runtime/reusable_node.h b/src/runtime/reusable_node.h index 6cba540f..87d20dbf 100644 --- a/src/runtime/reusable_node.h +++ b/src/runtime/reusable_node.h @@ -36,16 +36,6 @@ static inline ReusableNode reusable_node_after_leaf(const ReusableNode *self) { return result; } -static inline bool reusable_node_has_leading_changes(const ReusableNode *self) { - Tree *tree = self->tree; - while (tree->has_changes) { - if (tree->child_count == 0) return false; - tree = tree->children[0]; - if (tree->size.bytes == 0) return false; - } - return true; -} - static inline bool reusable_node_breakdown(ReusableNode *self) { if (self->tree->child_count == 0) { return false;