traxys/tree-sitter
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Remove 'extra' and 'structural' booleans from symbol metadata

Browse source
This commit is contained in:
Max Brunsfeld 2017-09-14 12:07:46 -07:00
parent d342b61ede
commit b0fdc33f73
6 changed files with 15 additions and 36 deletions
Download patch file Download diff file Expand all files Collapse all files

2
include/tree_sitter/parser.h
View file

@ -19,8 +19,6 @@ typedef uint16_t TSStateId;
typedef struct {
bool visible : 1;
bool named : 1;
bool extra : 1;
bool structural : 1;
} TSSymbolMetadata;
typedef struct {

20
src/compiler/generate_code/c_code.cc
View file

@ -136,8 +136,7 @@ class CCodeGenerator {
void add_stats() {
size_t token_count = 0;
for (const auto &entry : parse_table.symbols) {
const Symbol &symbol = entry.first;
for (const Symbol &symbol : parse_table.symbols) {
if (symbol.is_terminal()) {
token_count++;
} else if (symbol.is_external()) {
@ -170,8 +169,7 @@ class CCodeGenerator {
line("enum {");
indent([&]() {
size_t i = 1;
for (const auto &entry : parse_table.symbols) {
const Symbol &symbol = entry.first;
for (const Symbol &symbol : parse_table.symbols) {
if (!symbol.is_built_in()) {
line(symbol_id(symbol) + " = " + to_string(i) + ",");
i++;
@ -190,10 +188,10 @@ class CCodeGenerator {
void add_symbol_names_list() {
line("static const char *ts_symbol_names[] = {");
indent([&]() {
for (const auto &entry : parse_table.symbols) {
for (const Symbol &symbol : parse_table.symbols) {
line(
"[" + symbol_id(entry.first) + "] = \"" +
sanitize_name_for_string(symbol_name(entry.first)) + "\","
"[" + symbol_id(symbol) + "] = \"" +
sanitize_name_for_string(symbol_name(symbol)) + "\","
);
}
@ -236,8 +234,7 @@ class CCodeGenerator {
void add_symbol_metadata_list() {
line("static const TSSymbolMetadata ts_symbol_metadata[] = {");
indent([&]() {
for (const auto &entry : parse_table.symbols) {
const Symbol &symbol = entry.first;
for (const Symbol &symbol : parse_table.symbols) {
line("[" + symbol_id(symbol) + "] = {");
indent([&]() {
switch (symbol_type(symbol)) {
@ -258,9 +255,6 @@ class CCodeGenerator {
line(".named = false,");
break;
}
line(".structural = " + _boolean(entry.second.structural) + ",");
line(".extra = " + _boolean(entry.second.extra) + ",");
});
line("},");
@ -271,8 +265,6 @@ class CCodeGenerator {
indent([&]() {
line(".visible = true,");
line(".named = " + _boolean(alias.is_named) + ",");
line(".structural = true,");
line(".extra = true,");
});
line("},");
}

8
src/compiler/parse_table.cc
View file

@ -146,11 +146,7 @@ bool ParseState::operator==(const ParseState &other) const {
ParseAction &ParseTable::add_terminal_action(ParseStateId state_id,
Symbol lookahead,
ParseAction action) {
if (action.type == ParseActionTypeShift && action.extra)
symbols[lookahead].extra = true;
else
symbols[lookahead].structural = true;
symbols.insert(lookahead);
ParseTableEntry &entry = states[state_id].terminal_entries[lookahead];
entry.actions.push_back(action);
return *entry.actions.rbegin();
@ -159,7 +155,7 @@ ParseAction &ParseTable::add_terminal_action(ParseStateId state_id,
void ParseTable::set_nonterminal_action(ParseStateId state_id,
Symbol::Index lookahead,
ParseStateId next_state_id) {
symbols[Symbol::non_terminal(lookahead)].structural = true;
symbols.insert(Symbol::non_terminal(lookahead));
states[state_id].nonterminal_entries[lookahead] = next_state_id;
}

7
src/compiler/parse_table.h
View file

@ -72,11 +72,6 @@ struct ParseState {
LexStateId lex_state_id;
};
struct ParseTableSymbolMetadata {
bool extra;
bool structural;
};
using AliasSequence = std::vector<rules::Alias>;
struct ParseTable {
@ -84,7 +79,7 @@ struct ParseTable {
void set_nonterminal_action(ParseStateId, rules::Symbol::Index, ParseStateId);
std::vector<ParseState> states;
std::map<rules::Symbol, ParseTableSymbolMetadata> symbols;
std::set<rules::Symbol> symbols;
std::vector<AliasSequence> alias_sequences;
unsigned max_alias_sequence_length = 0;
};

4
src/runtime/language.c
View file

@ -31,9 +31,7 @@ uint32_t ts_language_version(const TSLanguage *language) {
TSSymbolMetadata ts_language_symbol_metadata(const TSLanguage *language,
TSSymbol symbol) {
if (symbol == ts_builtin_sym_error) {
return (TSSymbolMetadata){
.visible = true, .named = true, .extra = false, .structural = true,
};
return (TSSymbolMetadata){.visible = true, .named = true};
} else {
return language->symbol_metadata[symbol];
}

10
src/runtime/parser.c
View file

@ -593,9 +593,7 @@ static bool parser__select_tree(Parser *self, Tree *left, Tree *right) {
static void parser__shift(Parser *self, StackVersion version, TSStateId state,
Tree *lookahead, bool extra) {
if (extra != lookahead->extra) {
TSSymbolMetadata metadata =
ts_language_symbol_metadata(self->language, lookahead->symbol);
if (metadata.structural && ts_stack_version_count(self->stack) > 1) {
if (ts_stack_version_count(self->stack) > 1) {
lookahead = ts_tree_make_copy(lookahead);
} else {
ts_tree_retain(lookahead);
@ -988,8 +986,10 @@ static void parser__recover(Parser *self, StackVersion version, Tree *lookahead)
}
LOG("skip_token symbol:%s", SYM_NAME(lookahead->symbol));
bool can_be_extra = ts_language_symbol_metadata(self->language, lookahead->symbol).extra;
parser__shift(self, version, ERROR_STATE, lookahead, can_be_extra);
unsigned n;
const TSParseAction *actions = ts_language_actions(self->language, 1, lookahead->symbol, &n);
bool extra = n > 0 && actions[n - 1].type == TSParseActionTypeShift && actions[n - 1].params.extra;
parser__shift(self, version, ERROR_STATE, lookahead, extra);
if (parser__better_version_exists(self, version, true, ts_stack_error_cost(self->stack, version))) {
ts_stack_halt(self->stack, version);