traxys/tree-sitter
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

In compiler, distinguish between anonymous tokens and hidden rules

Browse source
This commit is contained in:
Max Brunsfeld 2015-09-05 17:05:37 -07:00
parent 4b270c8604
commit 5982b77c97
46 changed files with 41131 additions and 40884 deletions
Download patch file Download diff file Expand all files Collapse all files

8
src/compiler/build_tables/build_lex_table.cc
View file

@ -10,7 +10,7 @@
#include "compiler/build_tables/get_metadata.h"
#include "compiler/build_tables/lex_item.h"
#include "compiler/parse_table.h"
#include "compiler/lexical_grammar.h"
#include "compiler/prepared_grammar.h"
#include "compiler/rules/built_in_symbols.h"
#include "compiler/rules/choice.h"
#include "compiler/rules/metadata.h"
@ -63,9 +63,9 @@ class LexTableBuilder {
result.insert(
LexItem(symbol, after_separators(CharacterSet().include(0).copy())));
else if (symbol.is_token())
result.insert(
LexItem(symbol, after_separators(lex_grammar.rule(symbol))));
else if (symbol.is_token)
result.insert(LexItem(
symbol, after_separators(lex_grammar.rules[symbol.index].rule)));
}
return result;
}

14
src/compiler/build_tables/build_parse_table.cc
View file

@ -11,8 +11,7 @@
#include "compiler/build_tables/parse_item.h"
#include "compiler/build_tables/get_completion_status.h"
#include "compiler/build_tables/get_metadata.h"
#include "compiler/lexical_grammar.h"
#include "compiler/syntax_grammar.h"
#include "compiler/prepared_grammar.h"
#include "compiler/rules/symbol.h"
#include "compiler/rules/built_in_symbols.h"
@ -48,9 +47,8 @@ class ParseTableBuilder {
conflict_manager(grammar) {}
pair<ParseTable, const GrammarError *> build() {
auto start_symbol = grammar.rules.empty()
? make_shared<Symbol>(0, rules::SymbolOptionToken)
: make_shared<Symbol>(0);
auto start_symbol = grammar.rules.empty() ? make_shared<Symbol>(0, true)
: make_shared<Symbol>(0);
ParseItem start_item(rules::START(), start_symbol, {});
add_parse_state(
item_set_closure(start_item, { rules::END_OF_INPUT() }, grammar));
@ -260,10 +258,10 @@ class ParseTableBuilder {
return "END_OF_INPUT";
else
return "";
} else if (symbol.is_token())
return lexical_grammar.rule_name(symbol);
} else if (symbol.is_token)
return lexical_grammar.rules[symbol.index].name;
else
return grammar.rule_name(symbol);
return grammar.rules[symbol.index].name;
}
string action_description(const ParseAction &action) const {

3
src/compiler/build_tables/build_tables.cc
View file

@ -1,7 +1,6 @@
#include "compiler/build_tables/build_lex_table.h"
#include "compiler/build_tables/build_parse_table.h"
#include "compiler/lexical_grammar.h"
#include "compiler/syntax_grammar.h"
#include "compiler/prepared_grammar.h"
namespace tree_sitter {
namespace build_tables {

6
src/compiler/build_tables/first_symbols.cc
View file

@ -1,6 +1,6 @@
#include "compiler/build_tables/first_symbols.h"
#include "compiler/build_tables/rule_can_be_blank.h"
#include "compiler/syntax_grammar.h"
#include "compiler/prepared_grammar.h"
#include "compiler/rules/choice.h"
#include "compiler/rules/metadata.h"
#include "compiler/rules/seq.h"
@ -28,8 +28,8 @@ class FirstSymbols : public rules::RuleFn<set<Symbol>> {
return set<Symbol>();
set<Symbol> result({ *rule });
if (!rule->is_token()) {
set<Symbol> &&symbols = apply(grammar->rule(*rule));
if (!rule->is_token) {
set<Symbol> &&symbols = apply(grammar->rules[rule->index].rule);
result.insert(symbols.begin(), symbols.end());
}

9
src/compiler/build_tables/item_set_closure.cc
View file

@ -7,7 +7,7 @@
#include "compiler/build_tables/rule_transitions.h"
#include "compiler/build_tables/rule_can_be_blank.h"
#include "compiler/build_tables/item.h"
#include "compiler/syntax_grammar.h"
#include "compiler/prepared_grammar.h"
namespace tree_sitter {
namespace build_tables {
@ -41,7 +41,7 @@ const ParseItemSet item_set_closure(const ParseItem &starting_item,
const Symbol &symbol = pair.first;
const rule_ptr &next_rule = pair.second;
if (symbol.is_token() || symbol.is_built_in())
if (symbol.is_token || symbol.is_built_in())
continue;
set<Symbol> next_lookahead_symbols = first_symbols(next_rule, grammar);
@ -49,8 +49,9 @@ const ParseItemSet item_set_closure(const ParseItem &starting_item,
next_lookahead_symbols.insert(lookahead_symbols.begin(),
lookahead_symbols.end());
items_to_process.push_back({ ParseItem(symbol, grammar.rule(symbol), {}),
next_lookahead_symbols });
items_to_process.push_back(
{ ParseItem(symbol, grammar.rules[symbol.index].rule, {}),
next_lookahead_symbols });
}
}

2
src/compiler/build_tables/item_set_transitions.cc
View file

@ -4,7 +4,7 @@
#include "compiler/build_tables/merge_transitions.h"
#include "compiler/build_tables/parse_item.h"
#include "compiler/build_tables/rule_transitions.h"
#include "compiler/syntax_grammar.h"
#include "compiler/prepared_grammar.h"
#include "compiler/rules/symbol.h"
namespace tree_sitter {

2
src/compiler/build_tables/lex_conflict_manager.h
View file

@ -2,7 +2,7 @@
#define COMPILER_BUILD_TABLES_LEX_CONFLICT_MANAGER_H_
#include "tree_sitter/compiler.h"
#include "compiler/lexical_grammar.h"
#include "compiler/prepared_grammar.h"
namespace tree_sitter {

3
src/compiler/build_tables/parse_conflict_manager.h
View file

@ -3,8 +3,7 @@
#include <utility>
#include "tree_sitter/compiler.h"
#include "compiler/syntax_grammar.h"
#include "compiler/lexical_grammar.h"
#include "compiler/prepared_grammar.h"
#include "compiler/build_tables/parse_item.h"
namespace tree_sitter {

4
src/compiler/build_tables/rule_can_be_blank.cc
View file

@ -1,7 +1,7 @@
#include "compiler/build_tables/rule_can_be_blank.h"
#include <set>
#include "tree_sitter/compiler.h"
#include "compiler/syntax_grammar.h"
#include "compiler/prepared_grammar.h"
#include "compiler/rules/symbol.h"
#include "compiler/rules/visitor.h"
#include "compiler/rules/seq.h"
@ -55,7 +55,7 @@ class CanBeBlankRecursive : public CanBeBlank {
bool apply_to(const rules::Symbol *rule) {
if (visited_symbols.find(*rule) == visited_symbols.end()) {
visited_symbols.insert(*rule);
return !rule->is_token() && apply(grammar->rule(*rule));
return !rule->is_token && apply(grammar->rules[rule->index].rule);
} else {
return false;
}