traxys/tree-sitter
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

clang-format everything

Browse source
This commit is contained in:
Max Brunsfeld 2015-07-27 18:29:48 -07:00
parent 766e3bab2c
commit f9b057f3a9
44 changed files with 496 additions and 416 deletions
Download patch file Download diff file Expand all files Collapse all files

28
src/compiler/build_tables/build_lex_table.cc
View file

@ -37,7 +37,9 @@ class LexTableBuilder {
public:
LexTableBuilder(ParseTable *parse_table, const LexicalGrammar &lex_grammar)
: lex_grammar(lex_grammar), conflict_manager(lex_grammar), parse_table(parse_table) {}
: lex_grammar(lex_grammar),
conflict_manager(lex_grammar),
parse_table(parse_table) {}
LexTable build() {
for (auto &parse_state : parse_table->states) {
@ -56,12 +58,12 @@ class LexTableBuilder {
continue;
if (symbol == rules::END_OF_INPUT())
result.insert(LexItem(
symbol, after_separators(CharacterSet().include(0).copy())));
result.insert(
LexItem(symbol, after_separators(CharacterSet().include(0).copy())));
else if (symbol.is_token())
result.insert(
LexItem(symbol, after_separators(lex_grammar.rule(symbol))));
LexItem(symbol, after_separators(lex_grammar.rule(symbol))));
}
return result;
}
@ -96,8 +98,9 @@ class LexTableBuilder {
LexItemSet new_item_set = transition.second;
LexStateId new_state_id = add_lex_state(new_item_set);
auto action = LexAction::Advance(
new_state_id, precedence_values_for_item_set(new_item_set));
if (conflict_manager.resolve(action, lex_table.state(state_id).default_action))
new_state_id, precedence_values_for_item_set(new_item_set));
if (conflict_manager.resolve(action,
lex_table.state(state_id).default_action))
lex_table.state(state_id).actions[rule] = action;
}
}
@ -119,12 +122,13 @@ class LexTableBuilder {
}
rules::rule_ptr after_separators(rules::rule_ptr rule) {
return rules::Seq::build(
{ make_shared<rules::Metadata>(
separator_rule(),
map<rules::MetadataKey, int>(
{ { rules::START_TOKEN, 1 }, { rules::PRECEDENCE, -1 }, })),
rule, });
return rules::Seq::build({
make_shared<rules::Metadata>(
separator_rule(), map<rules::MetadataKey, int>({
{ rules::START_TOKEN, 1 }, { rules::PRECEDENCE, -1 },
})),
rule,
});
}
rules::rule_ptr separator_rule() const {

81
src/compiler/build_tables/build_parse_table.cc
View file

@ -41,15 +41,17 @@ class ParseTableBuilder {
public:
ParseTableBuilder(const SyntaxGrammar &grammar,
const LexicalGrammar &lex_grammar)
: grammar(grammar), lexical_grammar(lex_grammar), conflict_manager(grammar) {}
: grammar(grammar),
lexical_grammar(lex_grammar),
conflict_manager(grammar) {}
pair<ParseTable, const GrammarError *> build() {
auto start_symbol = grammar.rules.empty()
? make_shared<Symbol>(0, rules::SymbolOptionToken)
: make_shared<Symbol>(0);
? make_shared<Symbol>(0, rules::SymbolOptionToken)
: make_shared<Symbol>(0);
ParseItem start_item(rules::START(), start_symbol, {});
add_parse_state(
item_set_closure(start_item, { rules::END_OF_INPUT() }, grammar));
item_set_closure(start_item, { rules::END_OF_INPUT() }, grammar));
while (!item_sets_to_process.empty()) {
auto pair = item_sets_to_process.back();
@ -62,10 +64,9 @@ class ParseTableBuilder {
add_shift_extra_actions(item_set, state_id);
if (!conflicts.empty())
return {
parse_table,
new GrammarError(GrammarErrorTypeParseConflict, "Unresolved conflict.\n\n" + *conflicts.begin())
};
return { parse_table, new GrammarError(GrammarErrorTypeParseConflict,
"Unresolved conflict.\n\n" +
*conflicts.begin()) };
}
for (ParseStateId state = 0; state < parse_table.states.size(); state++)
@ -95,9 +96,10 @@ class ParseTableBuilder {
const Symbol &symbol = transition.first;
const ParseItemSet &next_item_set = transition.second;
ParseAction *new_action = add_action(state_id, symbol,
ParseAction::Shift(0, precedence_values_for_item_set(next_item_set)),
item_set);
ParseAction *new_action = add_action(
state_id, symbol,
ParseAction::Shift(0, precedence_values_for_item_set(next_item_set)),
item_set);
if (new_action)
new_action->state_index = add_parse_state(next_item_set);
}
@ -110,11 +112,11 @@ class ParseTableBuilder {
if (item.is_done()) {
ParseAction action =
(item.lhs == rules::START())
? ParseAction::Accept()
: ParseAction::Reduce(item.lhs, item.consumed_symbols.size(),
item.precedence(), item.associativity(),
get_production_id(item.consumed_symbols));
(item.lhs == rules::START())
? ParseAction::Accept()
: ParseAction::Reduce(item.lhs, item.consumed_symbols.size(),
item.precedence(), item.associativity(),
get_production_id(item.consumed_symbols));
for (const auto &lookahead_sym : lookahead_symbols)
add_action(state_id, lookahead_sym, action, item_set);
@ -122,15 +124,17 @@ class ParseTableBuilder {
}
}
void add_shift_extra_actions(const ParseItemSet &item_set, ParseStateId state_id) {
void add_shift_extra_actions(const ParseItemSet &item_set,
ParseStateId state_id) {
for (const Symbol &ubiquitous_symbol : grammar.ubiquitous_tokens)
add_action(state_id, ubiquitous_symbol, ParseAction::ShiftExtra(), item_set);
add_action(state_id, ubiquitous_symbol, ParseAction::ShiftExtra(),
item_set);
}
void add_reduce_extra_actions(ParseStateId state_id) {
const ParseItemSet item_set;
const map<Symbol, vector<ParseAction>> &actions =
parse_table.states[state_id].actions;
parse_table.states[state_id].actions;
for (const Symbol &ubiquitous_symbol : grammar.ubiquitous_tokens) {
const auto &entry = actions.find(ubiquitous_symbol);
@ -142,7 +146,8 @@ class ParseTableBuilder {
size_t shift_state_id = action.state_index;
for (const auto &pair : actions) {
const Symbol &lookahead_sym = pair.first;
ParseAction reduce_extra = ParseAction::ReduceExtra(ubiquitous_symbol);
ParseAction reduce_extra =
ParseAction::ReduceExtra(ubiquitous_symbol);
add_action(shift_state_id, lookahead_sym, reduce_extra, item_set);
}
}
@ -151,14 +156,16 @@ class ParseTableBuilder {
}
ParseAction *add_action(ParseStateId state_id, Symbol lookahead_sym,
const ParseAction &action, const ParseItemSet &item_set) {
const ParseAction &action,
const ParseItemSet &item_set) {
auto &current_actions = parse_table.states[state_id].actions;
auto current_entry = current_actions.find(lookahead_sym);
if (current_entry == current_actions.end())
return &parse_table.set_action(state_id, lookahead_sym, action);
const ParseAction current_action = current_entry->second[0];
auto resolution = conflict_manager.resolve(action, current_action, lookahead_sym);
auto resolution =
conflict_manager.resolve(action, current_action, lookahead_sym);
switch (resolution.second) {
case ConflictTypeNone:
@ -180,7 +187,8 @@ class ParseTableBuilder {
if (has_expected_conflict(goal_symbols))
return &parse_table.add_action(state_id, lookahead_sym, action);
else
conflicts.insert(conflict_description(action, current_action, lookahead_sym, goal_symbols));
conflicts.insert(conflict_description(action, current_action,
lookahead_sym, goal_symbols));
break;
}
}
@ -216,9 +224,9 @@ class ParseTableBuilder {
}
string conflict_description(const ParseAction &new_action,
const ParseAction &old_action,
const rules::Symbol &symbol,
const set<Symbol> &goal_symbols) const {
const ParseAction &old_action,
const rules::Symbol &symbol,
const set<Symbol> &goal_symbols) const {
string symbols_string;
bool started = false;
for (const auto &symbol : goal_symbols) {
@ -228,12 +236,14 @@ class ParseTableBuilder {
started = true;
}
return
"Within: " + symbols_string + "\n"
"Lookahead: " + symbol_name(symbol) + "\n" +
"Possible Actions:\n"
"* " + action_description(old_action) + "\n" +
"* " + action_description(new_action);
return "Within: " + symbols_string +
"\n"
"Lookahead: " +
symbol_name(symbol) + "\n" +
"Possible Actions:\n"
"* " +
action_description(old_action) + "\n" + "* " +
action_description(new_action);
}
string symbol_name(const rules::Symbol &symbol) const {
@ -273,9 +283,10 @@ class ParseTableBuilder {
if (action.precedence_values.size() > 1) {
result += " (Precedences " + to_string(*action.precedence_values.begin()) +
", " + to_string(*action.precedence_values.rbegin()) + ")";
", " + to_string(*action.precedence_values.rbegin()) + ")";
} else {
result += " (Precedence " + to_string(*action.precedence_values.begin()) + ")";
result +=
" (Precedence " + to_string(*action.precedence_values.begin()) + ")";
}
return result;
@ -294,7 +305,7 @@ class ParseTableBuilder {
};
pair<ParseTable, const GrammarError *> build_parse_table(
const SyntaxGrammar &grammar, const LexicalGrammar &lex_grammar) {
const SyntaxGrammar &grammar, const LexicalGrammar &lex_grammar) {
return ParseTableBuilder(grammar, lex_grammar).build();
}

4
src/compiler/build_tables/build_parse_table.h
View file

@ -13,8 +13,8 @@ class LexicalGrammar;
namespace build_tables {
std::pair<ParseTable, const GrammarError *>
build_parse_table(const SyntaxGrammar &, const LexicalGrammar &);
std::pair<ParseTable, const GrammarError *> build_parse_table(
const SyntaxGrammar &, const LexicalGrammar &);
} // namespace build_tables
} // namespace tree_sitter

4
src/compiler/build_tables/build_tables.cc
View file

@ -11,8 +11,8 @@ using std::tuple;
using std::vector;
using std::make_tuple;
tuple<ParseTable, LexTable, const GrammarError *>
build_tables(const SyntaxGrammar &grammar, const LexicalGrammar &lex_grammar) {
tuple<ParseTable, LexTable, const GrammarError *> build_tables(
const SyntaxGrammar &grammar, const LexicalGrammar &lex_grammar) {
auto parse_table_result = build_parse_table(grammar, lex_grammar);
ParseTable parse_table = parse_table_result.first;
const GrammarError *error = parse_table_result.second;

4
src/compiler/build_tables/build_tables.h
View file

@ -15,8 +15,8 @@ class LexicalGrammar;
namespace build_tables {
std::tuple<ParseTable, LexTable, const GrammarError *>
build_tables(const SyntaxGrammar &, const LexicalGrammar &);
std::tuple<ParseTable, LexTable, const GrammarError *> build_tables(
const SyntaxGrammar &, const LexicalGrammar &);
} // namespace build_tables
} // namespace tree_sitter

22
src/compiler/build_tables/item_set_transitions.cc
View file

@ -27,13 +27,13 @@ map<Symbol, ParseItemSet> sym_transitions(const ParseItemSet &item_set,
consumed_symbols.push_back(transition.first);
ParseItem new_item(item.lhs, transition.second, consumed_symbols);
merge_sym_transition<ParseItemSet>(
&result, { transition.first,
item_set_closure(new_item, lookahead_symbols, grammar) },
[](ParseItemSet *left, const ParseItemSet *right) {
for (auto &pair : *right)
left->operator[](pair.first)
.insert(pair.second.begin(), pair.second.end());
});
&result, { transition.first,
item_set_closure(new_item, lookahead_symbols, grammar) },
[](ParseItemSet *left, const ParseItemSet *right) {
for (auto &pair : *right)
left->operator[](pair.first)
.insert(pair.second.begin(), pair.second.end());
});
}
}
return result;
@ -45,10 +45,10 @@ map<CharacterSet, LexItemSet> char_transitions(const LexItemSet &item_set) {
for (auto &transition : char_transitions(item.rule)) {
LexItem next_item(item.lhs, transition.second);
merge_char_transition<LexItemSet>(
&result, { transition.first, LexItemSet({ next_item }) },
[](LexItemSet *left, const LexItemSet *right) {
left->insert(right->begin(), right->end());
});
&result, { transition.first, LexItemSet({ next_item }) },
[](LexItemSet *left, const LexItemSet *right) {
left->insert(right->begin(), right->end());
});
}
}
return result;

4
src/compiler/build_tables/item_set_transitions.h
View file

@ -17,10 +17,10 @@ class Symbol;
namespace build_tables {
std::map<rules::Symbol, ParseItemSet> sym_transitions(
const ParseItemSet &item_set, const SyntaxGrammar &grammar);
const ParseItemSet &item_set, const SyntaxGrammar &grammar);
std::map<rules::CharacterSet, LexItemSet> char_transitions(
const LexItemSet &item_set);
const LexItemSet &item_set);
} // namespace build_tables
} // namespace tree_sitter

7
src/compiler/build_tables/lex_conflict_manager.cc
View file

@ -6,10 +6,11 @@
namespace tree_sitter {
namespace build_tables {
LexConflictManager::LexConflictManager(const LexicalGrammar &grammar) :
grammar(grammar) {}
LexConflictManager::LexConflictManager(const LexicalGrammar &grammar)
: grammar(grammar) {}
bool LexConflictManager::resolve(const LexAction &new_action, const LexAction &old_action) const {
bool LexConflictManager::resolve(const LexAction &new_action,
const LexAction &old_action) const {
if (new_action.type < old_action.type)
return !resolve(old_action, new_action);

33
src/compiler/build_tables/parse_conflict_manager.cc
View file

@ -10,22 +10,21 @@ using std::pair;
using std::vector;
ParseConflictManager::ParseConflictManager(const SyntaxGrammar &syntax_grammar)
: syntax_grammar(syntax_grammar) {}
: syntax_grammar(syntax_grammar) {}
pair<bool, ConflictType>
ParseConflictManager::resolve(const ParseAction &new_action,
const ParseAction &old_action,
const rules::Symbol &symbol) const {
pair<bool, ConflictType> ParseConflictManager::resolve(
const ParseAction &new_action, const ParseAction &old_action,
const rules::Symbol &symbol) const {
if (new_action.type < old_action.type) {
auto opposite = resolve(old_action, new_action, symbol);
return {!opposite.first, opposite.second};
return { !opposite.first, opposite.second };
}
switch (old_action.type) {
case ParseActionTypeError:
case ParseActionTypeShiftExtra:
case ParseActionTypeReduceExtra:
return {true, ConflictTypeNone};
return { true, ConflictTypeNone };
case ParseActionTypeShift:
if (new_action.type == ParseActionTypeReduce) {
@ -33,20 +32,20 @@ ParseConflictManager::resolve(const ParseAction &new_action,
int max_precedence = *old_action.precedence_values.rbegin();
int new_precedence = *new_action.precedence_values.rbegin();
if (new_precedence < min_precedence)
return {false, ConflictTypeResolved};
return { false, ConflictTypeResolved };
else if (new_precedence > max_precedence)
return {true, ConflictTypeResolved};
return { true, ConflictTypeResolved };
else if (min_precedence == max_precedence) {
switch (new_action.associativity) {
case rules::AssociativityLeft:
return {true, ConflictTypeResolved};
return { true, ConflictTypeResolved };
case rules::AssociativityRight:
return {false, ConflictTypeResolved};
return { false, ConflictTypeResolved };
default:
return {false, ConflictTypeUnresolved};
return { false, ConflictTypeUnresolved };
}
} else {
return {false, ConflictTypeUnresolved};
return { false, ConflictTypeUnresolved };
}
}
@ -55,11 +54,11 @@ ParseConflictManager::resolve(const ParseAction &new_action,
int old_precedence = *old_action.precedence_values.begin();
int new_precedence = *new_action.precedence_values.begin();
if (new_precedence > old_precedence) {
return {true, ConflictTypeResolved};
return { true, ConflictTypeResolved };
} else if (new_precedence < old_precedence) {
return {false, ConflictTypeResolved};
return { false, ConflictTypeResolved };
} else {
return {false, ConflictTypeUnresolved};
return { false, ConflictTypeUnresolved };
}
}
@ -67,7 +66,7 @@ ParseConflictManager::resolve(const ParseAction &new_action,
break;
}
return {false, ConflictTypeNone};
return { false, ConflictTypeNone };
}
} // namespace build_tables

8
src/compiler/build_tables/parse_conflict_manager.h
View file

@ -10,7 +10,9 @@
namespace tree_sitter {
class ParseAction;
namespace rules { class Symbol; }
namespace rules {
class Symbol;
}
namespace build_tables {
@ -25,8 +27,8 @@ class ParseConflictManager {
public:
ParseConflictManager(const SyntaxGrammar &);
std::pair<bool, ConflictType> resolve(
const ParseAction &, const ParseAction &, const rules::Symbol &) const;
std::pair<bool, ConflictType> resolve(const ParseAction &, const ParseAction &,
const rules::Symbol &) const;
};
} // namespace build_tables

12
src/compiler/build_tables/parse_item.cc
View file

@ -21,10 +21,14 @@ bool ParseItem::operator==(const ParseItem &other) const {
}
bool ParseItem::operator<(const ParseItem &other) const {
if (lhs < other.lhs) return true;
if (other.lhs < lhs) return false;
if (consumed_symbols.size() < other.consumed_symbols.size()) return true;
if (other.consumed_symbols.size() < consumed_symbols.size()) return false;
if (lhs < other.lhs)
return true;
if (other.lhs < lhs)
return false;
if (consumed_symbols.size() < other.consumed_symbols.size())
return true;
if (other.consumed_symbols.size() < consumed_symbols.size())
return false;
return rule < other.rule;
}

4
src/compiler/build_tables/rule_transitions.h
View file

@ -9,10 +9,10 @@ namespace tree_sitter {
namespace build_tables {
std::map<rules::CharacterSet, rules::rule_ptr> char_transitions(
const rules::rule_ptr &rule);
const rules::rule_ptr &rule);
std::map<rules::Symbol, rules::rule_ptr> sym_transitions(
const rules::rule_ptr &rule);
const rules::rule_ptr &rule);
} // namespace build_tables
} // namespace tree_sitter

12
src/compiler/compile.cc
View file

@ -13,29 +13,29 @@ using std::vector;
using std::get;
using std::make_tuple;
pair<string, const GrammarError *>
compile(const Grammar &grammar, std::string name) {
pair<string, const GrammarError *> compile(const Grammar &grammar,
std::string name) {
auto prepare_grammar_result = prepare_grammar::prepare_grammar(grammar);
const SyntaxGrammar &syntax_grammar = get<0>(prepare_grammar_result);
const LexicalGrammar &lexical_grammar = get<1>(prepare_grammar_result);
const GrammarError *error = get<2>(prepare_grammar_result);
if (error)
return {"", error};
return { "", error };
auto table_build_result =
build_tables::build_tables(syntax_grammar, lexical_grammar);
build_tables::build_tables(syntax_grammar, lexical_grammar);
const ParseTable &parse_table = get<0>(table_build_result);
const LexTable &lex_table = get<1>(table_build_result);
error = get<2>(table_build_result);
if (error)
return {"", error};
return { "", error };
string code = generate_code::c_code(name, parse_table, lex_table,
syntax_grammar, lexical_grammar);
return {code, nullptr};
return { code, nullptr };
}
} // namespace tree_sitter

79
src/compiler/generate_code/c_code.cc
View file

@ -23,40 +23,42 @@ using std::to_string;
using std::vector;
using util::escape_char;
static const map<char, string> REPLACEMENTS({ { '~', "TILDE" },
{ '`', "BQUOTE" },
{ '!', "BANG" },
{ '@', "AT" },
{ '#', "POUND" },
{ '$', "DOLLAR" },
{ '%', "PERCENT" },
{ '^', "CARET" },
{ '&', "AMP" },
{ '*', "STAR" },
{ '(', "LPAREN" },
{ ')', "RPAREN" },
{ '-', "DASH" },
{ '+', "PLUS" },
{ '=', "EQ" },
{ '{', "LBRACE" },
{ '}', "RBRACE" },
{ '[', "LBRACK" },
{ ']', "RBRACK" },
{ '\\', "BSLASH" },
{ '|', "PIPE" },
{ ':', "COLON" },
{ ';', "SEMI" },
{ '"', "DQUOTE" },
{ '\'', "SQUOTE" },
{ '<', "LT" },
{ '>', "GT" },
{ ',', "COMMA" },
{ '.', "DOT" },
{ '?', "QMARK" },
{ '/', "SLASH" },
{ '\n', "LB" },
{ '\r', "CR" },
{ '\t', "TAB" }, });
static const map<char, string> REPLACEMENTS({
{ '~', "TILDE" },
{ '`', "BQUOTE" },
{ '!', "BANG" },
{ '@', "AT" },
{ '#', "POUND" },
{ '$', "DOLLAR" },
{ '%', "PERCENT" },
{ '^', "CARET" },
{ '&', "AMP" },
{ '*', "STAR" },
{ '(', "LPAREN" },
{ ')', "RPAREN" },
{ '-', "DASH" },
{ '+', "PLUS" },
{ '=', "EQ" },
{ '{', "LBRACE" },
{ '}', "RBRACE" },
{ '[', "LBRACK" },
{ ']', "RBRACK" },
{ '\\', "BSLASH" },
{ '|', "PIPE" },
{ ':', "COLON" },
{ ';', "SEMI" },
{ '"', "DQUOTE" },
{ '\'', "SQUOTE" },
{ '<', "LT" },
{ '>', "GT" },
{ ',', "COMMA" },
{ '.', "DOT" },
{ '?', "QMARK" },
{ '/', "SLASH" },
{ '\n', "LB" },
{ '\r', "CR" },
{ '\t', "TAB" },
});
class CCodeGenerator {
string buffer;
@ -182,8 +184,8 @@ class CCodeGenerator {
line("#pragma GCC diagnostic ignored \"-Wmissing-field-initializers\"");
line();
line(
"static const TSParseAction *"
"ts_parse_actions[STATE_COUNT][SYMBOL_COUNT] = {");
"static const TSParseAction *"
"ts_parse_actions[STATE_COUNT][SYMBOL_COUNT] = {");
indent([&]() {
for (const auto &state : parse_table.states) {
@ -356,7 +358,7 @@ class CCodeGenerator {
bool reduce_action_is_fragile(const ParseAction &action) const {
return parse_table.fragile_production_ids.find(action.production_id) !=
parse_table.fragile_production_ids.end();
parse_table.fragile_production_ids.end();
}
// C-code generation functions
@ -455,7 +457,8 @@ string c_code(string name, const ParseTable &parse_table,
const LexTable &lex_table, const SyntaxGrammar &syntax_grammar,
const LexicalGrammar &lexical_grammar) {
return CCodeGenerator(name, parse_table, lex_table, syntax_grammar,
lexical_grammar).code();
lexical_grammar)
.code();
}
} // namespace generate_code

4
src/compiler/lex_table.cc
View file

@ -47,10 +47,10 @@ std::ostream &operator<<(std::ostream &stream, const LexAction &action) {
return stream << string("#<error>");
case LexActionTypeAccept:
return stream << string("#<accept ") + to_string(action.symbol.index) +
">";
">";
case LexActionTypeAdvance:
return stream << string("#<advance ") + to_string(action.state_index) +
">";
">";
default:
return stream;
}

36
src/compiler/parse_table.cc
View file

@ -13,8 +13,7 @@ using rules::Symbol;
ParseAction::ParseAction(ParseActionType type, ParseStateId state_index,
Symbol symbol, size_t consumed_symbol_count,
set<int> precedence_values,
rules::Associativity associativity,
int production_id)
rules::Associativity associativity, int production_id)
: type(type),
symbol(symbol),
state_index(state_index),
@ -30,9 +29,7 @@ ParseAction::ParseAction()
consumed_symbol_count(0),
associativity(rules::AssociativityUnspecified) {}
ParseAction ParseAction::Error() {
return ParseAction();
}
ParseAction ParseAction::Error() { return ParseAction(); }
ParseAction ParseAction::Accept() {
ParseAction action;
@ -60,7 +57,8 @@ ParseAction ParseAction::ReduceExtra(Symbol symbol) {
}
ParseAction ParseAction::Reduce(Symbol symbol, size_t consumed_symbol_count,
int precedence, rules::Associativity associativity,
int precedence,
rules::Associativity associativity,
int production_id) {
return ParseAction(ParseActionTypeReduce, 0, symbol, consumed_symbol_count,
{ precedence }, associativity, production_id);
@ -71,17 +69,23 @@ bool ParseAction::operator==(const ParseAction &other) const {
bool symbols_eq = symbol == other.symbol;
bool state_indices_eq = state_index == other.state_index;
bool consumed_symbol_counts_eq =
consumed_symbol_count == other.consumed_symbol_count;
consumed_symbol_count == other.consumed_symbol_count;
return types_eq && symbols_eq && state_indices_eq && consumed_symbol_counts_eq;
}
bool ParseAction::operator<(const ParseAction &other) const {
if (type < other.type) return true;
if (other.type < type) return false;
if (symbol < other.symbol) return true;
if (other.symbol < symbol) return false;
if (state_index < other.state_index) return true;
if (other.state_index < state_index) return false;
if (type < other.type)
return true;
if (other.type < type)
return false;
if (symbol < other.symbol)
return true;
if (other.symbol < symbol)
return false;
if (state_index < other.state_index)
return true;
if (other.state_index < state_index)
return false;
return consumed_symbol_count < other.consumed_symbol_count;
}
@ -138,13 +142,15 @@ ParseStateId ParseTable::add_state() {
return states.size() - 1;
}
ParseAction &ParseTable::set_action(ParseStateId id, Symbol symbol, ParseAction action) {
ParseAction &ParseTable::set_action(ParseStateId id, Symbol symbol,
ParseAction action) {
symbols.insert(symbol);
states[id].actions[symbol] = vector<ParseAction>({ action });
return *states[id].actions[symbol].begin();
}
ParseAction &ParseTable::add_action(ParseStateId id, Symbol symbol, ParseAction action) {
ParseAction &ParseTable::add_action(ParseStateId id, Symbol symbol,
ParseAction action) {
symbols.insert(symbol);
states[id].actions[symbol].push_back(action);
return *states[id].actions[symbol].rbegin();

7
src/compiler/parse_table.h
View file

@ -36,7 +36,8 @@ class ParseAction {
static ParseAction Shift(ParseStateId state_index,
std::set<int> precedence_values);
static ParseAction Reduce(rules::Symbol symbol, size_t consumed_symbol_count,
int precedence, rules::Associativity, int production_id);
int precedence, rules::Associativity,
int production_id);
static ParseAction ShiftExtra();
static ParseAction ReduceExtra(rules::Symbol symbol);
bool operator==(const ParseAction &) const;
@ -86,9 +87,9 @@ class ParseTable {
public:
ParseStateId add_state();
ParseAction &set_action(ParseStateId state_id, rules::Symbol symbol,
ParseAction action);
ParseAction action);
ParseAction &add_action(ParseStateId state_id, rules::Symbol symbol,
ParseAction action);
ParseAction action);
std::vector<ParseState> states;
std::set<rules::Symbol> symbols;

12
src/compiler/prepare_grammar/expand_repeats.cc
View file

@ -40,13 +40,14 @@ class ExpandRepeats : public rules::IdentityRuleFn {
rule_ptr inner_rule = apply(rule->content);
size_t index = aux_rules.size();
string helper_rule_name = rule_name + string("_repeat") + to_string(++repeat_count);
string helper_rule_name =
rule_name + string("_repeat") + to_string(++repeat_count);
Symbol repeat_symbol(offset + index, rules::SymbolOptionAuxiliary);
existing_repeats.push_back({ rule->copy(), repeat_symbol });
aux_rules.push_back(
{ helper_rule_name,
Seq::build({ inner_rule, Choice::build({ repeat_symbol.copy(),
make_shared<Blank>() }) }) });
{ helper_rule_name,
Seq::build({ inner_rule, Choice::build({ repeat_symbol.copy(),
make_shared<Blank>() }) }) });
return repeat_symbol.copy();
}
@ -74,7 +75,8 @@ SyntaxGrammar expand_repeats(const SyntaxGrammar &grammar) {
ExpandRepeats expander(result.aux_rules.size());
for (auto &pair : grammar.rules)
result.rules.push_back({ pair.first, expander.expand(pair.second, pair.first) });
result.rules.push_back(
{ pair.first, expander.expand(pair.second, pair.first) });
result.aux_rules.insert(result.aux_rules.end(), expander.aux_rules.begin(),
expander.aux_rules.end());

8
src/compiler/prepare_grammar/expand_tokens.cc
View file

@ -56,8 +56,8 @@ class ExpandTokens : public rules::IdentityRuleFn {
ExpandTokens() : error(nullptr) {}
};
pair<LexicalGrammar, const GrammarError *>
expand_tokens(const LexicalGrammar &grammar) {
pair<LexicalGrammar, const GrammarError *> expand_tokens(
const LexicalGrammar &grammar) {
LexicalGrammar result;
ExpandTokens expander;
@ -82,7 +82,9 @@ expand_tokens(const LexicalGrammar &grammar) {
result.separators.push_back(rule);
}
return { result, nullptr, };
return {
result, nullptr,
};
}
} // namespace prepare_grammar

2
src/compiler/prepare_grammar/expand_tokens.h
View file

@ -11,7 +11,7 @@ class LexicalGrammar;
namespace prepare_grammar {
std::pair<LexicalGrammar, const GrammarError *> expand_tokens(
const LexicalGrammar &);
const LexicalGrammar &);
} // namespace prepare_grammar
} // namespace tree_sitter

26
src/compiler/prepare_grammar/extract_tokens.cc
View file

@ -89,12 +89,13 @@ class TokenExtractor : public rules::IdentityRuleFn {
vector<pair<string, rule_ptr>> tokens;
};
static const GrammarError * ubiq_token_err(const string &msg) {
return new GrammarError(GrammarErrorTypeInvalidUbiquitousToken, "Not a token: " + msg);
static const GrammarError *ubiq_token_err(const string &msg) {
return new GrammarError(GrammarErrorTypeInvalidUbiquitousToken,
"Not a token: " + msg);
}
tuple<SyntaxGrammar, LexicalGrammar, const GrammarError *>
extract_tokens(const InternedGrammar &grammar) {
tuple<SyntaxGrammar, LexicalGrammar, const GrammarError *> extract_tokens(
const InternedGrammar &grammar) {
SyntaxGrammar syntax_grammar;
LexicalGrammar lexical_grammar;
SymbolReplacer symbol_replacer;
@ -104,12 +105,12 @@ extract_tokens(const InternedGrammar &grammar) {
for (auto &pair : grammar.rules) {
if (is_token(pair.second)) {
lexical_grammar.rules.push_back(pair);
symbol_replacer.replacements.insert({
Symbol(i),
Symbol(lexical_grammar.rules.size() - 1, SymbolOptionToken)
});
symbol_replacer.replacements.insert(
{ Symbol(i),
Symbol(lexical_grammar.rules.size() - 1, SymbolOptionToken) });
} else {
syntax_grammar.rules.push_back({ pair.first, extractor.apply(pair.second) });
syntax_grammar.rules.push_back(
{ pair.first, extractor.apply(pair.second) });
}
i++;
}
@ -123,11 +124,14 @@ extract_tokens(const InternedGrammar &grammar) {
} else {
auto sym = dynamic_pointer_cast<const Symbol>(extractor.apply(rule));
if (!sym.get())
return make_tuple(syntax_grammar, lexical_grammar, ubiq_token_err(rule->to_string()));
return make_tuple(syntax_grammar, lexical_grammar,
ubiq_token_err(rule->to_string()));
Symbol symbol = symbol_replacer.replace_symbol(*sym);
if (!symbol.is_token())
return make_tuple(syntax_grammar, lexical_grammar, ubiq_token_err(syntax_grammar.rules[symbol.index].first));
return make_tuple(
syntax_grammar, lexical_grammar,
ubiq_token_err(syntax_grammar.rules[symbol.index].first));
syntax_grammar.ubiquitous_tokens.insert(symbol);
}

2
src/compiler/prepare_grammar/extract_tokens.h
View file

@ -14,7 +14,7 @@ class LexicalGrammar;
namespace prepare_grammar {
std::tuple<SyntaxGrammar, LexicalGrammar, const GrammarError *> extract_tokens(
const InternedGrammar &);
const InternedGrammar &);
} // namespace prepare_grammar
} // namespace tree_sitter

6
src/compiler/prepare_grammar/intern_symbols.cc
View file

@ -42,7 +42,7 @@ class InternSymbols : public rules::IdentityRuleFn {
string missing_rule_name;
};
const GrammarError * missing_rule_error(string rule_name) {
const GrammarError *missing_rule_error(string rule_name) {
return new GrammarError(GrammarErrorTypeUndefinedSymbol,
"Undefined rule '" + rule_name + "'");
}
@ -54,14 +54,14 @@ pair<InternedGrammar, const GrammarError *> intern_symbols(const Grammar &gramma
for (auto &pair : grammar.rules()) {
auto new_rule = interner.apply(pair.second);
if (!interner.missing_rule_name.empty())
return {result, missing_rule_error(interner.missing_rule_name)};
return { result, missing_rule_error(interner.missing_rule_name) };
result.rules.push_back({ pair.first, new_rule });
}
for (auto &rule : grammar.ubiquitous_tokens()) {
auto new_rule = interner.apply(rule);
if (!interner.missing_rule_name.empty())
return {result, missing_rule_error(interner.missing_rule_name)};
return { result, missing_rule_error(interner.missing_rule_name) };
result.ubiquitous_tokens.insert(new_rule);
}

1
src/compiler/prepare_grammar/interned_grammar.h
View file

@ -20,5 +20,4 @@ struct InternedGrammar {
} // namespace prepare_grammar
} // namespace tree_sitter
#endif // COMPILER_PREPARE_GRAMMAR_INTERNED_GRAMMAR_H_

20
src/compiler/prepare_grammar/parse_regex.cc
View file

@ -49,7 +49,7 @@ class PatternParser {
choices.push_back(pair.first);
} while (has_more_input());
auto rule =
(choices.size() > 1) ? make_shared<Choice>(choices) : choices.front();
(choices.size() > 1) ? make_shared<Choice>(choices) : choices.front();
return { rule, nullptr };
}
@ -115,8 +115,12 @@ class PatternParser {
next();
return { pair.first.copy(), nullptr };
}
case ')': { return error("unmatched close paren"); }
case ']': { return error("unmatched close square bracket"); }
case ')': {
return error("unmatched close paren");
}
case ']': {
return error("unmatched close square bracket");
}
case '.': {
next();
return { CharacterSet().include_all().exclude('\n').copy(), nullptr };
@ -180,15 +184,15 @@ class PatternParser {
return CharacterSet().include('a', 'z').include('A', 'Z');
case 'w':
return CharacterSet()
.include('a', 'z')
.include('A', 'Z')
.include('0', '9')
.include('_');
.include('a', 'z')
.include('A', 'Z')
.include('0', '9')
.include('_');
case 'd':
return CharacterSet().include('0', '9');
case 's':
return CharacterSet().include(' ').include('\t').include('\n').include(
'\r');
'\r');
case 't':
return CharacterSet().include('\t');
case 'n':

2
src/compiler/prepare_grammar/prepare_grammar.cc
View file

@ -14,7 +14,7 @@ using std::get;
using std::make_tuple;
tuple<SyntaxGrammar, LexicalGrammar, const GrammarError *> prepare_grammar(
const Grammar &input_grammar) {
const Grammar &input_grammar) {
// Convert all string-based `NamedSymbols` into numerical `Symbols`
auto intern_result = intern_symbols(input_grammar);

2
src/compiler/prepare_grammar/prepare_grammar.h
View file

@ -13,7 +13,7 @@ class GrammarError;
namespace prepare_grammar {
std::tuple<SyntaxGrammar, LexicalGrammar, const GrammarError *> prepare_grammar(
const Grammar &);
const Grammar &);
} // namespace prepare_grammar
} // namespace tree_sitter

4
src/compiler/rules/character_set.cc
View file

@ -169,10 +169,10 @@ CharacterSet CharacterSet::remove_set(const CharacterSet &other) {
if (other.includes_all) {
result.included_chars = included_chars;
included_chars =
remove_chars(&result.included_chars, other.excluded_chars);
remove_chars(&result.included_chars, other.excluded_chars);
} else {
result.included_chars =
remove_chars(&included_chars, other.included_chars);
remove_chars(&included_chars, other.included_chars);
}
}
return result;

14
src/compiler/rules/rules.cc
View file

@ -32,9 +32,7 @@ rule_ptr blank() { return make_shared<Blank>(); }
rule_ptr choice(const vector<rule_ptr> &rules) { return Choice::build(rules); }
rule_ptr repeat(const rule_ptr &content) {
return Repeat::build(content);
}
rule_ptr repeat(const rule_ptr &content) { return Repeat::build(content); }
rule_ptr seq(const vector<rule_ptr> &rules) { return Seq::build(rules); }
@ -46,15 +44,11 @@ rule_ptr str(const string &value) {
return token(prec(1, make_shared<String>(value)));
}
rule_ptr err(const rule_ptr &rule) {
return choice({ rule, ERROR().copy() });
}
rule_ptr err(const rule_ptr &rule) { return choice({ rule, ERROR().copy() }); }
rule_ptr prec(int precedence, const rule_ptr &rule, Associativity associativity) {
return metadata(rule, {
{ PRECEDENCE, precedence },
{ ASSOCIATIVITY, associativity }
});
return metadata(
rule, { { PRECEDENCE, precedence }, { ASSOCIATIVITY, associativity } });
}
rule_ptr prec(int precedence, const rule_ptr &rule) {

2
src/compiler/rules/symbol.cc
View file

@ -11,7 +11,7 @@ using std::to_string;
using std::hash;
SymbolOption SymbolOptionAuxToken =
SymbolOption(SymbolOptionToken | SymbolOptionAuxiliary);
SymbolOption(SymbolOptionToken | SymbolOptionAuxiliary);
Symbol::Symbol(int index) : index(index), options(SymbolOption(0)) {}