diff --git a/include/tree_sitter/parser.h b/include/tree_sitter/parser.h index d9a8e197..cb83ce33 100644 --- a/include/tree_sitter/parser.h +++ b/include/tree_sitter/parser.h @@ -19,8 +19,6 @@ typedef uint16_t TSStateId; typedef struct { bool visible : 1; bool named : 1; - bool extra : 1; - bool structural : 1; } TSSymbolMetadata; typedef struct { diff --git a/src/compiler/generate_code/c_code.cc b/src/compiler/generate_code/c_code.cc index d11b3084..092f1966 100644 --- a/src/compiler/generate_code/c_code.cc +++ b/src/compiler/generate_code/c_code.cc @@ -136,8 +136,7 @@ class CCodeGenerator { void add_stats() { size_t token_count = 0; - for (const auto &entry : parse_table.symbols) { - const Symbol &symbol = entry.first; + for (const Symbol &symbol : parse_table.symbols) { if (symbol.is_terminal()) { token_count++; } else if (symbol.is_external()) { @@ -170,8 +169,7 @@ class CCodeGenerator { line("enum {"); indent([&]() { size_t i = 1; - for (const auto &entry : parse_table.symbols) { - const Symbol &symbol = entry.first; + for (const Symbol &symbol : parse_table.symbols) { if (!symbol.is_built_in()) { line(symbol_id(symbol) + " = " + to_string(i) + ","); i++; @@ -190,10 +188,10 @@ class CCodeGenerator { void add_symbol_names_list() { line("static const char *ts_symbol_names[] = {"); indent([&]() { - for (const auto &entry : parse_table.symbols) { + for (const Symbol &symbol : parse_table.symbols) { line( - "[" + symbol_id(entry.first) + "] = \"" + - sanitize_name_for_string(symbol_name(entry.first)) + "\"," + "[" + symbol_id(symbol) + "] = \"" + + sanitize_name_for_string(symbol_name(symbol)) + "\"," ); } @@ -236,8 +234,7 @@ class CCodeGenerator { void add_symbol_metadata_list() { line("static const TSSymbolMetadata ts_symbol_metadata[] = {"); indent([&]() { - for (const auto &entry : parse_table.symbols) { - const Symbol &symbol = entry.first; + for (const Symbol &symbol : parse_table.symbols) { line("[" + symbol_id(symbol) + "] = {"); indent([&]() { switch (symbol_type(symbol)) { @@ -258,9 +255,6 @@ class CCodeGenerator { line(".named = false,"); break; } - - line(".structural = " + _boolean(entry.second.structural) + ","); - line(".extra = " + _boolean(entry.second.extra) + ","); }); line("},"); @@ -271,8 +265,6 @@ class CCodeGenerator { indent([&]() { line(".visible = true,"); line(".named = " + _boolean(alias.is_named) + ","); - line(".structural = true,"); - line(".extra = true,"); }); line("},"); } diff --git a/src/compiler/parse_table.cc b/src/compiler/parse_table.cc index 4d10907c..6eaeaabd 100644 --- a/src/compiler/parse_table.cc +++ b/src/compiler/parse_table.cc @@ -146,11 +146,7 @@ bool ParseState::operator==(const ParseState &other) const { ParseAction &ParseTable::add_terminal_action(ParseStateId state_id, Symbol lookahead, ParseAction action) { - if (action.type == ParseActionTypeShift && action.extra) - symbols[lookahead].extra = true; - else - symbols[lookahead].structural = true; - + symbols.insert(lookahead); ParseTableEntry &entry = states[state_id].terminal_entries[lookahead]; entry.actions.push_back(action); return *entry.actions.rbegin(); @@ -159,7 +155,7 @@ ParseAction &ParseTable::add_terminal_action(ParseStateId state_id, void ParseTable::set_nonterminal_action(ParseStateId state_id, Symbol::Index lookahead, ParseStateId next_state_id) { - symbols[Symbol::non_terminal(lookahead)].structural = true; + symbols.insert(Symbol::non_terminal(lookahead)); states[state_id].nonterminal_entries[lookahead] = next_state_id; } diff --git a/src/compiler/parse_table.h b/src/compiler/parse_table.h index 39e0080b..d652c144 100644 --- a/src/compiler/parse_table.h +++ b/src/compiler/parse_table.h @@ -72,11 +72,6 @@ struct ParseState { LexStateId lex_state_id; }; -struct ParseTableSymbolMetadata { - bool extra; - bool structural; -}; - using AliasSequence = std::vector; struct ParseTable { @@ -84,7 +79,7 @@ struct ParseTable { void set_nonterminal_action(ParseStateId, rules::Symbol::Index, ParseStateId); std::vector states; - std::map symbols; + std::set symbols; std::vector alias_sequences; unsigned max_alias_sequence_length = 0; }; diff --git a/src/runtime/language.c b/src/runtime/language.c index cb4e7383..55ae96f3 100644 --- a/src/runtime/language.c +++ b/src/runtime/language.c @@ -31,9 +31,7 @@ uint32_t ts_language_version(const TSLanguage *language) { TSSymbolMetadata ts_language_symbol_metadata(const TSLanguage *language, TSSymbol symbol) { if (symbol == ts_builtin_sym_error) { - return (TSSymbolMetadata){ - .visible = true, .named = true, .extra = false, .structural = true, - }; + return (TSSymbolMetadata){.visible = true, .named = true}; } else { return language->symbol_metadata[symbol]; } diff --git a/src/runtime/parser.c b/src/runtime/parser.c index 367f9796..527b0a82 100644 --- a/src/runtime/parser.c +++ b/src/runtime/parser.c @@ -593,9 +593,7 @@ static bool parser__select_tree(Parser *self, Tree *left, Tree *right) { static void parser__shift(Parser *self, StackVersion version, TSStateId state, Tree *lookahead, bool extra) { if (extra != lookahead->extra) { - TSSymbolMetadata metadata = - ts_language_symbol_metadata(self->language, lookahead->symbol); - if (metadata.structural && ts_stack_version_count(self->stack) > 1) { + if (ts_stack_version_count(self->stack) > 1) { lookahead = ts_tree_make_copy(lookahead); } else { ts_tree_retain(lookahead); @@ -988,8 +986,10 @@ static void parser__recover(Parser *self, StackVersion version, Tree *lookahead) } LOG("skip_token symbol:%s", SYM_NAME(lookahead->symbol)); - bool can_be_extra = ts_language_symbol_metadata(self->language, lookahead->symbol).extra; - parser__shift(self, version, ERROR_STATE, lookahead, can_be_extra); + unsigned n; + const TSParseAction *actions = ts_language_actions(self->language, 1, lookahead->symbol, &n); + bool extra = n > 0 && actions[n - 1].type == TSParseActionTypeShift && actions[n - 1].params.extra; + parser__shift(self, version, ERROR_STATE, lookahead, extra); if (parser__better_version_exists(self, version, true, ts_stack_error_cost(self->stack, version))) { ts_stack_halt(self->stack, version);