traxys/tree-sitter
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Never move the start rule of a grammar into the lexical grammar

Browse source 
This preserves a useful invariant that the root node of the AST is never
a token.
This commit is contained in:
Max Brunsfeld 2017-12-07 11:40:41 -08:00
parent 48681c3f0e
commit 493db39363
3 changed files with 34 additions and 13 deletions
Download patch file Download diff file Expand all files Collapse all files

5
src/compiler/build_tables/parse_table_builder.cc
View file

@ -83,10 +83,7 @@ class ParseTableBuilderImpl : public ParseTableBuilder {
ParseStateId error_state_id = add_parse_state({}, ParseItemSet{});
// Add the starting state.
Symbol start_symbol = grammar.variables.empty() ?
Symbol::terminal(0) :
Symbol::non_terminal(0);
Symbol start_symbol = Symbol::non_terminal(0);
Production start_production({{start_symbol, 0, rules::AssociativityNone, rules::Alias{}}}, 0);
add_parse_state({}, ParseItemSet{{

2
src/compiler/prepare_grammar/extract_tokens.cc
View file

@ -210,7 +210,7 @@ tuple<InitialSyntaxGrammar, LexicalGrammar, CompileError> extract_tokens(
size_t i = -1;
for (const auto &variable : processed_variables) {
i++;
if (variable.rule.is<Symbol>()) {
if (i > 0 && variable.rule.is<Symbol>()) {
auto symbol = variable.rule.get_unchecked<Symbol>();
if (symbol.is_terminal() && extractor.token_usage_counts[symbol.index] == 1) {
lexical_grammar.variables[symbol.index].type = variable.type;