traxys/tree-sitter
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Include out-of-context states starting with non-terminals

Browse source
This commit is contained in:
Max Brunsfeld 2016-03-02 20:45:13 -08:00
parent e7abfdd373
commit 76d072545d
4 changed files with 54 additions and 25 deletions
Download patch file Download diff file Expand all files Collapse all files

23
spec/compiler/build_tables/symbols_by_first_symbol_spec.cc
View file

@ -86,6 +86,29 @@ describe("symbols_by_first_symbol", [&]() {
Symbol(3)
}
},
{
Symbol(0), {
Symbol(0),
Symbol(1),
Symbol(2),
}
},
{
Symbol(1), {
Symbol(1),
Symbol(2),
}
},
{
Symbol(2), {
Symbol(2),
}
},
{
Symbol(3), {
Symbol(3),
}
}
})));
});
});

38
src/compiler/build_tables/build_parse_table.cc
View file

@ -105,24 +105,36 @@ class ParseTableBuilder {
}
void add_out_of_context_parse_states() {
map<Symbol, set<Symbol>> symbols_by_token = symbols_by_first_symbol(grammar);
map<Symbol, set<Symbol>> symbols_by_first = symbols_by_first_symbol(grammar);
for (size_t i = 0; i < lexical_grammar.variables.size(); i++) {
Symbol token(i, true);
ParseItemSet item_set;
const set<Symbol> &symbols = symbols_by_token[token];
Symbol symbol(i, true);
if (!grammar.extra_tokens.count(symbol))
add_out_of_context_parse_state(symbol, symbols_by_first[symbol]);
}
for (const auto &parse_state_entry : parse_state_ids) {
for (const auto &pair : parse_state_entry.first.entries) {
const ParseItem &item = pair.first;
const LookaheadSet &lookahead_set = pair.second;
if (symbols.count(item.next_symbol()))
item_set.entries[item].insert_all(lookahead_set);
for (size_t i = 0; i < grammar.variables.size(); i++) {
Symbol symbol(i, false);
add_out_of_context_parse_state(Symbol(i, false), symbols_by_first[symbol]);
}
}
void add_out_of_context_parse_state(const rules::Symbol &symbol,
const set<Symbol> &symbols) {
ParseItemSet item_set;
for (const auto &parse_state_entry : parse_state_ids) {
ParseItemSet state_item_set = parse_state_entry.first;
for (const auto &pair : state_item_set.entries) {
const ParseItem &item = pair.first;
const LookaheadSet &lookahead_set = pair.second;
if (symbols.count(item.next_symbol())) {
item_set.entries[item].insert_all(lookahead_set);
}
}
ParseStateId state = add_parse_state(item_set);
parse_table.out_of_context_state_indices[token] = state;
}
ParseStateId state = add_parse_state(item_set);
parse_table.out_of_context_state_indices[symbol] = state;
}
ParseStateId add_parse_state(const ParseItemSet &item_set) {

8
src/compiler/build_tables/symbols_by_first_symbol.cc
View file

@ -43,14 +43,6 @@ map<Symbol, set<Symbol>> symbols_by_first_symbol(const SyntaxGrammar &grammar) {
}
}
for (auto iter = result.begin(), end = result.end(); iter != end;) {
if (!iter->first.is_token) {
result.erase(iter++);
} else {
iter++;
}
}
return result;
}

10
src/compiler/generate_code/c_code.cc
View file

@ -225,10 +225,12 @@ class CCodeGenerator {
indent([&]() {
for (const auto &entry : parse_table.symbols) {
const rules::Symbol &symbol = entry.first;
ParseStateId state = parse_table.out_of_context_state_indices.find(symbol)->second;
if (symbol.is_token && !symbol.is_built_in()) {
line("[" + symbol_id(symbol) + "] = " + to_string(state) + ",");
}
if (symbol.is_built_in())
continue;
auto iter = parse_table.out_of_context_state_indices.find(symbol);
string state = (iter != parse_table.out_of_context_state_indices.end()) ?
to_string(iter->second) : "ts_parse_state_error";
line("[" + symbol_id(symbol) + "] = " + state + ",");
}
});
line("};");