traxys/tree-sitter
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

ISymbol -> Symbol

Browse source 
Interned symbols are now the main type of symbol in use
This commit is contained in:
Max Brunsfeld 2014-04-28 20:43:27 -07:00
parent faf80aadac
commit 25eda9d889
45 changed files with 248 additions and 250 deletions
Download patch file Download diff file Expand all files Collapse all files

8
spec/compiler/build_tables/conflict_manager_spec.cc
View file

@ -29,8 +29,8 @@ describe("resolving parse conflicts", []() {
});
describe("lexical conflicts", [&]() {
ISymbol sym1(1, SymbolOptionToken);
ISymbol sym2(2, SymbolOptionToken);
Symbol sym1(1, SymbolOptionToken);
Symbol sym2(2, SymbolOptionToken);
it("favors non-errors over lexical errors", [&]() {
should_update = manager->resolve_lex_action(LexAction::Error(), LexAction::Advance(2));
@ -50,8 +50,8 @@ describe("resolving parse conflicts", []() {
});
describe("syntactic conflicts", [&]() {
ISymbol sym1(0);
ISymbol sym2(1);
Symbol sym1(0);
Symbol sym2(1);
it("favors non-errors over parse errors", [&]() {
should_update = manager->resolve_parse_action(sym1, ParseAction::Error(), ParseAction::Shift(2, { 0 }));

30
spec/compiler/build_tables/first_set_spec.cc
View file

@ -16,8 +16,8 @@ describe("computing FIRST sets", []() {
it("ignores B when A cannot be blank", [&]() {
auto rule = seq({ i_token(0), i_token(1) });
AssertThat(first_set(rule, null_grammar), Equals(set<ISymbol>({
ISymbol(0, SymbolOptionToken),
AssertThat(first_set(rule, null_grammar), Equals(set<Symbol>({
Symbol(0, SymbolOptionToken),
})));
});
@ -28,9 +28,9 @@ describe("computing FIRST sets", []() {
blank() }),
i_token(1) });
AssertThat(first_set(rule, null_grammar), Equals(set<ISymbol>({
ISymbol(0, SymbolOptionToken),
ISymbol(1, SymbolOptionToken)
AssertThat(first_set(rule, null_grammar), Equals(set<Symbol>({
Symbol(0, SymbolOptionToken),
Symbol(1, SymbolOptionToken)
})));
});
@ -48,9 +48,9 @@ describe("computing FIRST sets", []() {
i_token(4) }) }
});
AssertThat(first_set(rule, grammar), Equals(set<ISymbol>({
ISymbol(0, SymbolOptionToken),
ISymbol(2, SymbolOptionToken),
AssertThat(first_set(rule, grammar), Equals(set<Symbol>({
Symbol(0, SymbolOptionToken),
Symbol(2, SymbolOptionToken),
})));
});
@ -65,9 +65,9 @@ describe("computing FIRST sets", []() {
blank() }) }
});
AssertThat(first_set(rule, grammar), Equals(set<ISymbol>({
ISymbol(0, SymbolOptionToken),
ISymbol(1, SymbolOptionToken),
AssertThat(first_set(rule, grammar), Equals(set<Symbol>({
Symbol(0, SymbolOptionToken),
Symbol(1, SymbolOptionToken),
})));
});
});
@ -83,8 +83,8 @@ describe("computing FIRST sets", []() {
auto rule = i_sym(0);
AssertThat(first_set(rule, grammar), Equals(set<ISymbol>({
ISymbol(11, SymbolOptionToken)
AssertThat(first_set(rule, grammar), Equals(set<Symbol>({
Symbol(11, SymbolOptionToken)
})));
});
});
@ -92,8 +92,8 @@ describe("computing FIRST sets", []() {
it("ignores metadata rules", [&]() {
auto rule = make_shared<Metadata>(i_token(3), map<rules::MetadataKey, int>());
AssertThat(first_set(rule, null_grammar), Equals(set<ISymbol>({
ISymbol(3, SymbolOptionToken),
AssertThat(first_set(rule, null_grammar), Equals(set<Symbol>({
Symbol(3, SymbolOptionToken),
})));
});
});

40
spec/compiler/build_tables/follow_sets_spec.cc
View file

@ -16,42 +16,42 @@ describe("computing FOLLOW sets", []() {
}, {});
it("includes all of the starting non-terminals for the item, and their following terminals", [&]() {
ParseItem item(ISymbol(2), choice({
ParseItem item(Symbol(2), choice({
seq({ i_sym(0), choice({ i_token(0), i_token(1) }) }),
seq({ i_sym(1), i_token(2) }),
}), 0, ISymbol(10, SymbolOptionToken));
}), 0, Symbol(10, SymbolOptionToken));
AssertThat(follow_sets(item, grammar), Equals(map<ISymbol, set<ISymbol>>({
{ ISymbol(0), set<ISymbol>({
ISymbol(0, SymbolOptionToken),
ISymbol(1, SymbolOptionToken) }) },
{ ISymbol(1), set<ISymbol>({
ISymbol(2, SymbolOptionToken) }) },
AssertThat(follow_sets(item, grammar), Equals(map<Symbol, set<Symbol>>({
{ Symbol(0), set<Symbol>({
Symbol(0, SymbolOptionToken),
Symbol(1, SymbolOptionToken) }) },
{ Symbol(1), set<Symbol>({
Symbol(2, SymbolOptionToken) }) },
})));
});
it("does not include terminals at the beginning of the item", [&]() {
ParseItem item(ISymbol(2), choice({
ParseItem item(Symbol(2), choice({
seq({ i_sym(0), choice({ i_token(0), i_token(1) }) }),
seq({ i_token(2), i_token(3) }),
}), 0, ISymbol(10, SymbolOptionToken));
}), 0, Symbol(10, SymbolOptionToken));
AssertThat(follow_sets(item, grammar), Equals(map<ISymbol, set<ISymbol>>({
{ ISymbol(0), set<ISymbol>({
ISymbol(0, SymbolOptionToken),
ISymbol(1, SymbolOptionToken) }) },
AssertThat(follow_sets(item, grammar), Equals(map<Symbol, set<Symbol>>({
{ Symbol(0), set<Symbol>({
Symbol(0, SymbolOptionToken),
Symbol(1, SymbolOptionToken) }) },
})));
});
it("includes the item's lookahead terminal if the rule after the non-terminal might be blank", [&]() {
ParseItem item(ISymbol(2), choice({
ParseItem item(Symbol(2), choice({
seq({ i_sym(0), choice({ i_token(0), blank() }) }),
}), 0, ISymbol(10, SymbolOptionToken));
}), 0, Symbol(10, SymbolOptionToken));
AssertThat(follow_sets(item, grammar), Equals(map<ISymbol, set<ISymbol>>({
{ ISymbol(0), set<ISymbol>({
ISymbol(0, SymbolOptionToken),
ISymbol(10, SymbolOptionToken) }) },
AssertThat(follow_sets(item, grammar), Equals(map<Symbol, set<Symbol>>({
{ Symbol(0), set<Symbol>({
Symbol(0, SymbolOptionToken),
Symbol(10, SymbolOptionToken) }) },
})));
});
});

6
spec/compiler/build_tables/item_set_closure_spec.cc
View file

@ -20,12 +20,12 @@ describe("computing closures of item sets", []() {
it("adds items at the beginnings of referenced rules", [&]() {
ParseItemSet item_set = item_set_closure(
ParseItem(ISymbol(0), grammar.rule(ISymbol(0)), 0, ISymbol(10, SymbolOptionToken)),
ParseItem(Symbol(0), grammar.rule(Symbol(0)), 0, Symbol(10, SymbolOptionToken)),
grammar);
AssertThat(item_set, Equals(ParseItemSet({
ParseItem(ISymbol(1), grammar.rule(ISymbol(1)), 0, ISymbol(11, SymbolOptionToken)),
ParseItem(ISymbol(0), grammar.rule(ISymbol(0)), 0, ISymbol(10, SymbolOptionToken)),
ParseItem(Symbol(1), grammar.rule(Symbol(1)), 0, Symbol(11, SymbolOptionToken)),
ParseItem(Symbol(0), grammar.rule(Symbol(0)), 0, Symbol(10, SymbolOptionToken)),
})));
});
});

12
spec/compiler/build_tables/item_set_transitions_spec.cc
View file

@ -13,17 +13,17 @@ describe("item set transitions", []() {
describe("when two items in the set have transitions on the same character", [&]() {
it("merges the transitions by computing the union of the two item sets", [&]() {
LexItemSet set1({
LexItem(ISymbol(1), pattern("[a-f]")),
LexItem(ISymbol(2), pattern("[e-x]")) });
LexItem(Symbol(1), pattern("[a-f]")),
LexItem(Symbol(2), pattern("[e-x]")) });
AssertThat(char_transitions(set1, grammar), Equals(map<CharacterSet, LexItemSet>({
{ CharacterSet({ {'a', 'd'} }), LexItemSet({
LexItem(ISymbol(1), blank()) }) },
LexItem(Symbol(1), blank()) }) },
{ CharacterSet({ {'e', 'f'} }), LexItemSet({
LexItem(ISymbol(1), blank()),
LexItem(ISymbol(2), blank()) }) },
LexItem(Symbol(1), blank()),
LexItem(Symbol(2), blank()) }) },
{ CharacterSet({ {'g', 'x'} }), LexItemSet({
LexItem(ISymbol(2), blank()) }) },
LexItem(Symbol(2), blank()) }) },
})));
});
});

2
spec/compiler/build_tables/lex_item_spec.cc
View file

@ -9,7 +9,7 @@ START_TEST
describe("lex items", []() {
describe("determining if an item is the start of a token", [&]() {
ISymbol sym(1);
Symbol sym(1);
rule_ptr token_start = make_shared<Metadata>(str("a"), map<MetadataKey, int>({
{ START_TOKEN, 1 }
}));

30
spec/compiler/build_tables/rule_transitions_spec.cc
View file

@ -27,25 +27,25 @@ describe("rule transitions", []() {
it("handles symbols", [&]() {
AssertThat(
sym_transitions(i_sym(1)),
Equals(rule_map<ISymbol>({
{ ISymbol(1), blank() }
Equals(rule_map<Symbol>({
{ Symbol(1), blank() }
})));
});
it("handles choices", [&]() {
AssertThat(
sym_transitions(choice({ i_sym(1), i_sym(2) })),
Equals(rule_map<ISymbol>({
{ ISymbol(1), blank() },
{ ISymbol(2), blank() }
Equals(rule_map<Symbol>({
{ Symbol(1), blank() },
{ Symbol(2), blank() }
})));
});
it("handles sequences", [&]() {
AssertThat(
sym_transitions(seq({ i_sym(1), i_sym(2) })),
Equals(rule_map<ISymbol>({
{ ISymbol(1), i_sym(2) }
Equals(rule_map<Symbol>({
{ Symbol(1), i_sym(2) }
})));
});
@ -57,8 +57,8 @@ describe("rule transitions", []() {
i_sym(3),
i_sym(4)
})),
Equals(rule_map<ISymbol>({
{ ISymbol(1), seq({ i_sym(2), i_sym(3), i_sym(4) }) }
Equals(rule_map<Symbol>({
{ Symbol(1), seq({ i_sym(2), i_sym(3), i_sym(4) }) }
})));
});
@ -73,8 +73,8 @@ describe("rule transitions", []() {
i_sym(1),
i_sym(2)
})
})), Equals(rule_map<ISymbol>({
{ ISymbol(1), choice({ seq({ i_sym(1), i_sym(2) }), i_sym(2), }) }
})), Equals(rule_map<Symbol>({
{ Symbol(1), choice({ seq({ i_sym(1), i_sym(2) }), i_sym(2), }) }
})));
});
@ -84,8 +84,8 @@ describe("rule transitions", []() {
choice({
seq({ i_sym(1), i_sym(2) }),
seq({ i_sym(1), i_sym(3) }) })),
Equals(rule_map<ISymbol>({
{ ISymbol(1), choice({ i_sym(2), i_sym(3) }) }
Equals(rule_map<Symbol>({
{ Symbol(1), choice({ i_sym(2), i_sym(3) }) }
})));
});
@ -192,8 +192,8 @@ describe("rule transitions", []() {
rule_ptr rule = make_shared<Metadata>(seq({ i_sym(1), i_sym(2) }), metadata_value);
AssertThat(
sym_transitions(rule),
Equals(rule_map<ISymbol>({
{ ISymbol(1), make_shared<Metadata>(i_sym(2), metadata_value)},
Equals(rule_map<Symbol>({
{ Symbol(1), make_shared<Metadata>(i_sym(2), metadata_value)},
})));
});

10
spec/compiler/helpers/rule_helpers.cc
View file

@ -1,6 +1,6 @@
#include "rule_helpers.h"
#include <memory>
#include "compiler/rules/interned_symbol.h"
#include "compiler/rules/symbol.h"
namespace tree_sitter {
using std::make_shared;
@ -19,19 +19,19 @@ namespace tree_sitter {
}
rule_ptr i_sym(size_t index) {
return make_shared<rules::ISymbol>(index);
return make_shared<rules::Symbol>(index);
}
rule_ptr i_aux_sym(size_t index) {
return make_shared<rules::ISymbol>(index, SymbolOptionAuxiliary);
return make_shared<rules::Symbol>(index, SymbolOptionAuxiliary);
}
rule_ptr i_token(size_t index) {
return make_shared<rules::ISymbol>(index, SymbolOptionToken);
return make_shared<rules::Symbol>(index, SymbolOptionToken);
}
rule_ptr i_aux_token(size_t index) {
return make_shared<rules::ISymbol>(index, SymbolOption(SymbolOptionAuxiliary|SymbolOptionToken));
return make_shared<rules::Symbol>(index, SymbolOption(SymbolOptionAuxiliary|SymbolOptionToken));
}
}
}

2
spec/compiler/prepare_grammar/intern_symbols_spec.cc
View file

@ -2,7 +2,7 @@
#include "compiler/prepared_grammar.h"
#include "compiler/prepare_grammar/intern_symbols.h"
#include "compiler/rules/named_symbol.h"
#include "compiler/rules/interned_symbol.h"
#include "compiler/rules/symbol.h"
START_TEST

12
src/compiler/build_tables/build_tables.cc
View file

@ -10,7 +10,7 @@
#include "compiler/rules/repeat.h"
#include "compiler/rules/blank.h"
#include "compiler/rules/seq.h"
#include "compiler/rules/interned_symbol.h"
#include "compiler/rules/symbol.h"
#include "compiler/build_tables/conflict_manager.h"
#include "compiler/build_tables/item.h"
#include "compiler/build_tables/item_set_closure.h"
@ -25,7 +25,7 @@ namespace tree_sitter {
using std::set;
using std::unordered_map;
using std::make_shared;
using rules::ISymbol;
using rules::Symbol;
using rules::CharacterSet;
namespace build_tables {
@ -46,7 +46,7 @@ namespace tree_sitter {
void add_shift_actions(const ParseItemSet &item_set, ParseStateId state_id) {
for (auto &transition : sym_transitions(item_set, grammar)) {
const ISymbol &symbol = transition.first;
const Symbol &symbol = transition.first;
const ParseItemSet &item_set = transition.second;
set<int> precedence_values = precedence_values_for_item_set(item_set);
@ -162,11 +162,11 @@ namespace tree_sitter {
void add_error_lex_state() {
LexItemSet error_item_set;
for (size_t i = 0; i < lex_grammar.rules.size(); i++) {
LexItem item(ISymbol(i, rules::SymbolOptionToken), after_separators(lex_grammar.rules[i].second));
LexItem item(Symbol(i, rules::SymbolOptionToken), after_separators(lex_grammar.rules[i].second));
error_item_set.insert(item);
}
for (size_t i = 0; i < lex_grammar.aux_rules.size(); i++) {
LexItem item(ISymbol(i, rules::SymbolOption(rules::SymbolOptionToken|rules::SymbolOptionAuxiliary)), after_separators(lex_grammar.aux_rules[i].second));
LexItem item(Symbol(i, rules::SymbolOption(rules::SymbolOptionToken|rules::SymbolOptionAuxiliary)), after_separators(lex_grammar.aux_rules[i].second));
error_item_set.insert(item);
}
error_item_set.insert(LexItem(rules::END_OF_INPUT(), after_separators(CharacterSet({ 0 }).copy())));
@ -183,7 +183,7 @@ namespace tree_sitter {
{}
void build() {
auto start_symbol = make_shared<ISymbol>(0);
auto start_symbol = make_shared<Symbol>(0);
ParseItem item(rules::START(), start_symbol, {}, rules::END_OF_INPUT());
ParseItemSet item_set = item_set_closure(item, grammar);
add_parse_state(item_set);

4
src/compiler/build_tables/conflict_manager.cc
View file

@ -18,7 +18,7 @@ namespace tree_sitter {
parse_grammar(parse_grammar),
lex_grammar(lex_grammar) {}
bool ConflictManager::resolve_parse_action(const rules::ISymbol &symbol,
bool ConflictManager::resolve_parse_action(const rules::Symbol &symbol,
const ParseAction &old_action,
const ParseAction &new_action) {
if (new_action.type < old_action.type)
@ -119,7 +119,7 @@ namespace tree_sitter {
}
}
void ConflictManager::record_conflict(const rules::ISymbol &symbol,
void ConflictManager::record_conflict(const rules::Symbol &symbol,
const ParseAction &left,
const ParseAction &right) {
string name = symbol.is_token() ?

6
src/compiler/build_tables/conflict_manager.h
View file

@ -7,7 +7,7 @@
#include <set>
#include "tree_sitter/compiler.h"
#include "compiler/parse_table.h"
#include "compiler/rules/interned_symbol.h"
#include "compiler/rules/symbol.h"
#include "compiler/prepared_grammar.h"
namespace tree_sitter {
@ -23,11 +23,11 @@ namespace tree_sitter {
bool resolve_lex_action(const LexAction &old_action,
const LexAction &new_action);
bool resolve_parse_action(const rules::ISymbol &symbol,
bool resolve_parse_action(const rules::Symbol &symbol,
const ParseAction &old_action,
const ParseAction &new_action);
void record_conflict(const rules::ISymbol &symbol, const ParseAction &left, const ParseAction &right);
void record_conflict(const rules::Symbol &symbol, const ParseAction &left, const ParseAction &right);
const std::vector<Conflict> conflicts() const;
};
}

28
src/compiler/build_tables/first_set.cc
View file

@ -6,36 +6,36 @@
#include "compiler/rules/visitor.h"
#include "compiler/rules/seq.h"
#include "compiler/rules/choice.h"
#include "compiler/rules/interned_symbol.h"
#include "compiler/rules/symbol.h"
namespace tree_sitter {
using std::set;
using rules::ISymbol;
using rules::Symbol;
namespace build_tables {
class FirstSet : public rules::RuleFn<set<ISymbol>> {
class FirstSet : public rules::RuleFn<set<Symbol>> {
const PreparedGrammar *grammar;
set<ISymbol> visited_symbols;
set<Symbol> visited_symbols;
public:
explicit FirstSet(const PreparedGrammar *grammar) : grammar(grammar) {}
set<ISymbol> apply_to(const ISymbol *rule) {
set<Symbol> apply_to(const Symbol *rule) {
auto insertion_result = visited_symbols.insert(*rule);
if (insertion_result.second) {
return (rule->is_token()) ?
set<ISymbol>({ *rule }) :
set<Symbol>({ *rule }) :
apply(grammar->rule(*rule));
} else {
return set<ISymbol>();
return set<Symbol>();
}
}
set<ISymbol> apply_to(const rules::Metadata *rule) {
set<Symbol> apply_to(const rules::Metadata *rule) {
return apply(rule->rule);
}
set<ISymbol> apply_to(const rules::Choice *rule) {
set<ISymbol> result;
set<Symbol> apply_to(const rules::Choice *rule) {
set<Symbol> result;
for (const auto &el : rule->elements) {
auto &&next_syms = apply(el);
result.insert(next_syms.begin(), next_syms.end());
@ -43,7 +43,7 @@ namespace tree_sitter {
return result;
}
set<ISymbol> apply_to(const rules::Seq *rule) {
set<Symbol> apply_to(const rules::Seq *rule) {
auto &&result = apply(rule->left);
if (rule_can_be_blank(rule->left, *grammar)) {
auto &&right_symbols = apply(rule->right);
@ -53,12 +53,12 @@ namespace tree_sitter {
}
};
set<ISymbol> first_set(const rules::rule_ptr &rule, const PreparedGrammar &grammar) {
set<Symbol> first_set(const rules::rule_ptr &rule, const PreparedGrammar &grammar) {
return FirstSet(&grammar).apply(rule);
}
set<ISymbol> first_set(const ParseItemSet &item_set, const PreparedGrammar &grammar) {
set<ISymbol> result;
set<Symbol> first_set(const ParseItemSet &item_set, const PreparedGrammar &grammar) {
set<Symbol> result;
for (auto &item : item_set) {
auto &&rule_set = first_set(item.rule, grammar);
result.insert(rule_set.begin(), rule_set.end());

6
src/compiler/build_tables/first_set.h
View file

@ -3,7 +3,7 @@
#include <set>
#include "compiler/build_tables/parse_item.h"
#include "compiler/rules/interned_symbol.h"
#include "compiler/rules/symbol.h"
namespace tree_sitter {
class PreparedGrammar;
@ -15,14 +15,14 @@ namespace tree_sitter {
* the beginning of a string derivable from a given rule,
* in a given gramamr.
*/
std::set<rules::ISymbol>
std::set<rules::Symbol>
first_set(const rules::rule_ptr &rule, const PreparedGrammar &grammar);
/*
* Returns the set of terminal symbols that can appear at
* the beginning of any item in the given set.
*/
std::set<rules::ISymbol>
std::set<rules::Symbol>
first_set(const ParseItemSet &item_set, const PreparedGrammar &grammar);
}
}

10
src/compiler/build_tables/follow_sets.cc
View file

@ -7,18 +7,18 @@
namespace tree_sitter {
using std::set;
using std::map;
using rules::ISymbol;
using rules::Symbol;
using rules::rule_ptr;
namespace build_tables {
map<ISymbol, set<ISymbol>> follow_sets(const ParseItem &item,
map<Symbol, set<Symbol>> follow_sets(const ParseItem &item,
const PreparedGrammar &grammar) {
map<ISymbol, set<ISymbol>> result;
map<Symbol, set<Symbol>> result;
for (auto &pair : sym_transitions(item.rule)) {
ISymbol symbol = pair.first;
Symbol symbol = pair.first;
rule_ptr next_rule = pair.second;
if (!symbol.is_token() && !symbol.is_built_in()) {
set<ISymbol> following_terminals = first_set(next_rule, grammar);
set<Symbol> following_terminals = first_set(next_rule, grammar);
if (rule_can_be_blank(next_rule, grammar))
following_terminals.insert(item.lookahead_sym);
result.insert({ symbol, following_terminals });

4
src/compiler/build_tables/follow_sets.h
View file

@ -4,7 +4,7 @@
#include <set>
#include <map>
#include "compiler/build_tables/parse_item.h"
#include "compiler/rules/interned_symbol.h"
#include "compiler/rules/symbol.h"
namespace tree_sitter {
class PreparedGrammar;
@ -17,7 +17,7 @@ namespace tree_sitter {
* item. The values are the sets of terminals which can appear immediately
* after the corresponding non-terminals.
*/
std::map<rules::ISymbol, std::set<rules::ISymbol>>
std::map<rules::Symbol, std::set<rules::Symbol>>
follow_sets(const ParseItem &item, const PreparedGrammar &grammar);
}

2
src/compiler/build_tables/item.cc
View file

@ -4,7 +4,7 @@
namespace tree_sitter {
namespace build_tables {
Item::Item(const rules::ISymbol &lhs, const rules::rule_ptr rule) :
Item::Item(const rules::Symbol &lhs, const rules::rule_ptr rule) :
lhs(lhs),
rule(rule) {}

6
src/compiler/build_tables/item.h
View file

@ -2,16 +2,16 @@
#define COMPILER_BUILD_TABLES_ITEM_H_
#include <unordered_set>
#include "compiler/rules/interned_symbol.h"
#include "compiler/rules/symbol.h"
namespace tree_sitter {
namespace build_tables {
class Item {
public:
Item(const rules::ISymbol &lhs, rules::rule_ptr rule);
Item(const rules::Symbol &lhs, rules::rule_ptr rule);
bool is_done() const;
rules::ISymbol lhs;
rules::Symbol lhs;
rules::rule_ptr rule;
};
}

6
src/compiler/build_tables/item_set_closure.cc
View file

@ -8,7 +8,7 @@
namespace tree_sitter {
using std::set;
using rules::ISymbol;
using rules::Symbol;
using std::vector;
namespace build_tables {
@ -22,8 +22,8 @@ namespace tree_sitter {
auto insertion_result = result.insert(item);
if (insertion_result.second) {
for (const auto &pair : follow_sets(item, grammar)) {
const ISymbol &non_terminal = pair.first;
const set<ISymbol> &terminals = pair.second;
const Symbol &non_terminal = pair.first;
const set<Symbol> &terminals = pair.second;
for (const auto &terminal : terminals) {
ParseItem next_item(non_terminal, grammar.rule(non_terminal), 0, terminal);
items_to_add.push_back(next_item);

18
src/compiler/build_tables/item_set_transitions.cc
View file

@ -4,14 +4,14 @@
#include "compiler/build_tables/item_set_closure.h"
#include "compiler/build_tables/rule_transitions.h"
#include "compiler/build_tables/merge_transitions.h"
#include "compiler/rules/interned_symbol.h"
#include "compiler/rules/symbol.h"
namespace tree_sitter {
using std::map;
using std::vector;
using std::unordered_set;
using rules::CharacterSet;
using rules::ISymbol;
using rules::Symbol;
namespace build_tables {
template<typename T>
@ -20,13 +20,13 @@ namespace tree_sitter {
return left;
}
const ISymbol placeholder_lookahead = ISymbol(-100);
const ISymbol placeholder_lhs = ISymbol(-101);
const Symbol placeholder_lookahead = Symbol(-100);
const Symbol placeholder_lhs = Symbol(-101);
static map<ISymbol, ParseItemSet> sym_transitions_for_rule(SymTransitions *self, const rules::rule_ptr &rule, const PreparedGrammar &grammar) {
static map<Symbol, ParseItemSet> sym_transitions_for_rule(SymTransitions *self, const rules::rule_ptr &rule, const PreparedGrammar &grammar) {
auto pair = self->transitions_cache.find(rule);
if (pair != self->transitions_cache.end()) return pair->second;
map<ISymbol, ParseItemSet> result;
map<Symbol, ParseItemSet> result;
for (auto &transition : sym_transitions(rule)) {
ParseItem new_item(placeholder_lhs, transition.second, 1, placeholder_lookahead);
result.insert({
@ -38,7 +38,7 @@ namespace tree_sitter {
return result;
}
static map<ISymbol, ParseItemSet> sym_transitions_for_item(SymTransitions *self, const ParseItem &item, const PreparedGrammar &grammar) {
static map<Symbol, ParseItemSet> sym_transitions_for_item(SymTransitions *self, const ParseItem &item, const PreparedGrammar &grammar) {
auto result = sym_transitions_for_rule(self, item.rule, grammar);
for (auto &pair : result) {
vector<ParseItem> new_items;
@ -70,9 +70,9 @@ namespace tree_sitter {
return result;
}
map<ISymbol, ParseItemSet>
map<Symbol, ParseItemSet>
SymTransitions::operator()(const ParseItemSet &item_set, const PreparedGrammar &grammar) {
map<ISymbol, ParseItemSet> result;
map<Symbol, ParseItemSet> result;
for (const ParseItem &item : item_set)
merge_sym_transitions<ParseItemSet>(result,
sym_transitions_for_item(this, item, grammar),

4
src/compiler/build_tables/item_set_transitions.h
View file

@ -15,9 +15,9 @@ namespace tree_sitter {
namespace build_tables {
class SymTransitions {
public:
std::map<rules::ISymbol, ParseItemSet>
std::map<rules::Symbol, ParseItemSet>
operator()(const ParseItemSet &item_set, const PreparedGrammar &grammar);
std::map<rules::rule_ptr, std::map<rules::ISymbol, ParseItemSet>> transitions_cache;
std::map<rules::rule_ptr, std::map<rules::Symbol, ParseItemSet>> transitions_cache;
};
std::map<rules::CharacterSet, LexItemSet>

4
src/compiler/build_tables/lex_item.cc
View file

@ -1,6 +1,6 @@
#include "compiler/build_tables/lex_item.h"
#include "compiler/build_tables/rule_can_be_blank.h"
#include "compiler/rules/interned_symbol.h"
#include "compiler/rules/symbol.h"
#include "compiler/rules/metadata.h"
#include "compiler/rules/seq.h"
#include "compiler/rules/visitor.h"
@ -11,7 +11,7 @@ namespace tree_sitter {
using std::vector;
namespace build_tables {
LexItem::LexItem(const rules::ISymbol &lhs, const rules::rule_ptr rule) :
LexItem::LexItem(const rules::Symbol &lhs, const rules::rule_ptr rule) :
Item(lhs, rule) {}
bool LexItem::operator==(const LexItem &other) const {

4
src/compiler/build_tables/lex_item.h
View file

@ -9,7 +9,7 @@ namespace tree_sitter {
namespace build_tables {
class LexItem : public Item {
public:
LexItem(const rules::ISymbol &lhs, rules::rule_ptr rule);
LexItem(const rules::Symbol &lhs, rules::rule_ptr rule);
bool operator==(const LexItem &other) const;
bool is_token_start() const;
};
@ -25,7 +25,7 @@ namespace std {
struct hash<tree_sitter::build_tables::LexItem> {
size_t operator()(const tree_sitter::build_tables::Item &item) const {
return
hash<tree_sitter::rules::ISymbol>()(item.lhs) ^
hash<tree_sitter::rules::Symbol>()(item.lhs) ^
hash<tree_sitter::rules::rule_ptr>()(item.rule);
}
};

6
src/compiler/build_tables/merge_transitions.h
View file

@ -3,7 +3,7 @@
#include <map>
#include "compiler/rules/character_set.h"
#include "compiler/rules/interned_symbol.h"
#include "compiler/rules/symbol.h"
namespace tree_sitter {
namespace build_tables {
@ -15,8 +15,8 @@ namespace tree_sitter {
* using the given function.
*/
template<typename T>
void merge_sym_transitions(std::map<rules::ISymbol, T> &left,
const std::map<rules::ISymbol, T> &right,
void merge_sym_transitions(std::map<rules::Symbol, T> &left,
const std::map<rules::Symbol, T> &right,
std::function<T(T &, const T &)> merge_fn) {
for (auto &pair : right) {
auto rule = pair.first;

4
src/compiler/build_tables/parse_item.cc
View file

@ -8,10 +8,10 @@ namespace tree_sitter {
using std::ostream;
namespace build_tables {
ParseItem::ParseItem(const rules::ISymbol &lhs,
ParseItem::ParseItem(const rules::Symbol &lhs,
const rules::rule_ptr rule,
size_t consumed_symbol_count,
const rules::ISymbol &lookahead_sym) :
const rules::Symbol &lookahead_sym) :
Item(lhs, rule),
consumed_symbol_count(consumed_symbol_count),
lookahead_sym(lookahead_sym) {}

12
src/compiler/build_tables/parse_item.h
View file

@ -3,7 +3,7 @@
#include <unordered_set>
#include <string>
#include "compiler/rules/interned_symbol.h"
#include "compiler/rules/symbol.h"
#include "compiler/build_tables/item.h"
#include "compiler/rules/metadata.h"
@ -11,15 +11,15 @@ namespace tree_sitter {
namespace build_tables {
class ParseItem : public Item {
public:
ParseItem(const rules::ISymbol &lhs,
ParseItem(const rules::Symbol &lhs,
rules::rule_ptr rule,
const size_t consumed_symbol_count,
const rules::ISymbol &lookahead_sym);
const rules::Symbol &lookahead_sym);
bool operator==(const ParseItem &other) const;
int precedence() const;
size_t consumed_symbol_count;
rules::ISymbol lookahead_sym;
rules::Symbol lookahead_sym;
};
std::ostream& operator<<(std::ostream &stream, const ParseItem &item);
@ -33,10 +33,10 @@ namespace std {
struct hash<tree_sitter::build_tables::ParseItem> {
size_t operator()(const tree_sitter::build_tables::ParseItem &item) const {
return
hash<tree_sitter::rules::ISymbol>()(item.lhs) ^
hash<tree_sitter::rules::Symbol>()(item.lhs) ^
hash<tree_sitter::rules::rule_ptr>()(item.rule) ^
hash<size_t>()(item.consumed_symbol_count) ^
hash<tree_sitter::rules::ISymbol>()(item.lookahead_sym);
hash<tree_sitter::rules::Symbol>()(item.lookahead_sym);
}
};

6
src/compiler/build_tables/rule_can_be_blank.cc
View file

@ -2,7 +2,7 @@
#include <set>
#include "tree_sitter/compiler.h"
#include "compiler/prepared_grammar.h"
#include "compiler/rules/interned_symbol.h"
#include "compiler/rules/symbol.h"
#include "compiler/rules/visitor.h"
#include "compiler/rules/seq.h"
#include "compiler/rules/choice.h"
@ -40,14 +40,14 @@ namespace tree_sitter {
class CanBeBlankRecursive : public CanBeBlank {
const PreparedGrammar *grammar;
set<rules::ISymbol> visited_symbols;
set<rules::Symbol> visited_symbols;
using CanBeBlank::visit;
public:
using CanBeBlank::apply_to;
explicit CanBeBlankRecursive(const PreparedGrammar *grammar) : grammar(grammar) {}
bool apply_to(const rules::ISymbol *rule) {
bool apply_to(const rules::Symbol *rule) {
if (visited_symbols.find(*rule) == visited_symbols.end()) {
visited_symbols.insert(*rule);
return !rule->is_token() && apply(grammar->rule(*rule));

12
src/compiler/build_tables/rule_transitions.cc
View file

@ -7,7 +7,7 @@
#include "compiler/rules/string.h"
#include "compiler/rules/repeat.h"
#include "compiler/rules/metadata.h"
#include "compiler/rules/interned_symbol.h"
#include "compiler/rules/symbol.h"
#include "compiler/rules/pattern.h"
#include "compiler/rules/character_set.h"
#include "compiler/rules/visitor.h"
@ -16,7 +16,7 @@ namespace tree_sitter {
using std::map;
using std::make_shared;
using rules::rule_ptr;
using rules::ISymbol;
using rules::Symbol;
using rules::CharacterSet;
namespace build_tables {
@ -31,7 +31,7 @@ namespace tree_sitter {
}
template<>
void merge_transitions(map<ISymbol, rule_ptr> &left, const map<ISymbol, rule_ptr> &right) {
void merge_transitions(map<Symbol, rule_ptr> &left, const map<Symbol, rule_ptr> &right) {
merge_sym_transitions<rule_ptr>(left, right, [](rule_ptr left, rule_ptr right) {
return rules::Choice::Build({ left, right });
});
@ -57,7 +57,7 @@ namespace tree_sitter {
return apply_to_atom(rule);
}
map<T, rule_ptr> apply_to(const ISymbol *rule) {
map<T, rule_ptr> apply_to(const Symbol *rule) {
return apply_to_atom(rule);
}
@ -114,8 +114,8 @@ namespace tree_sitter {
return RuleTransitions<CharacterSet>().apply(rule);
}
map<ISymbol, rule_ptr> sym_transitions(const rule_ptr &rule) {
return RuleTransitions<ISymbol>().apply(rule);
map<Symbol, rule_ptr> sym_transitions(const rule_ptr &rule) {
return RuleTransitions<Symbol>().apply(rule);
}
}
}

4
src/compiler/build_tables/rule_transitions.h
View file

@ -3,14 +3,14 @@
#include <map>
#include "compiler/rules/character_set.h"
#include "compiler/rules/interned_symbol.h"
#include "compiler/rules/symbol.h"
namespace tree_sitter {
namespace build_tables {
std::map<rules::CharacterSet, rules::rule_ptr>
char_transitions(const rules::rule_ptr &rule);
std::map<rules::ISymbol, rules::rule_ptr>
std::map<rules::Symbol, rules::rule_ptr>
sym_transitions(const rules::rule_ptr &rule);
}
}

8
src/compiler/generate_code/c_code.cc
View file

@ -84,11 +84,11 @@ namespace tree_sitter {
private:
const PreparedGrammar & grammar_for_symbol(const rules::ISymbol &symbol) {
const PreparedGrammar & grammar_for_symbol(const rules::Symbol &symbol) {
return symbol.is_token() ? lexical_grammar : syntax_grammar;
}
string symbol_id(const rules::ISymbol &symbol) {
string symbol_id(const rules::Symbol &symbol) {
if (symbol.is_built_in()) {
return (symbol == rules::ERROR()) ?
"ts_builtin_sym_error" :
@ -102,7 +102,7 @@ namespace tree_sitter {
}
}
string symbol_name(const rules::ISymbol &symbol) {
string symbol_name(const rules::Symbol &symbol) {
if (symbol.is_built_in()) {
return (symbol == rules::ERROR()) ? "error" : "end";
} else if (symbol.is_token() && symbol.is_auxiliary()) {
@ -211,7 +211,7 @@ namespace tree_sitter {
}
string symbol_names_list() {
set<rules::ISymbol> symbols(parse_table.symbols);
set<rules::Symbol> symbols(parse_table.symbols);
symbols.insert(rules::END_OF_INPUT());
symbols.insert(rules::ERROR());

14
src/compiler/lex_table.cc
View file

@ -1,33 +1,33 @@
#include "compiler/lex_table.h"
#include "compiler/rules/interned_symbol.h"
#include "compiler/rules/symbol.h"
namespace tree_sitter {
using std::string;
using std::to_string;
using std::map;
using std::set;
using rules::ISymbol;
using rules::Symbol;
using rules::CharacterSet;
LexAction::LexAction() :
type(LexActionTypeError),
symbol(ISymbol(-1)),
symbol(Symbol(-1)),
state_index(-1) {}
LexAction::LexAction(LexActionType type, size_t state_index, ISymbol symbol) :
LexAction::LexAction(LexActionType type, size_t state_index, Symbol symbol) :
type(type),
symbol(symbol),
state_index(state_index) {}
LexAction LexAction::Error() {
return LexAction(LexActionTypeError, -1, ISymbol(-1));
return LexAction(LexActionTypeError, -1, Symbol(-1));
}
LexAction LexAction::Advance(size_t state_index) {
return LexAction(LexActionTypeAdvance, state_index, ISymbol(-1));
return LexAction(LexActionTypeAdvance, state_index, Symbol(-1));
}
LexAction LexAction::Accept(ISymbol symbol) {
LexAction LexAction::Accept(Symbol symbol) {
return LexAction(LexActionTypeAccept, -1, symbol);
}

10
src/compiler/lex_table.h
View file

@ -5,7 +5,7 @@
#include <vector>
#include <set>
#include <string>
#include "compiler/rules/interned_symbol.h"
#include "compiler/rules/symbol.h"
#include "compiler/rules/character_set.h"
namespace tree_sitter {
@ -16,16 +16,16 @@ namespace tree_sitter {
} LexActionType;
class LexAction {
LexAction(LexActionType type, size_t state_index, rules::ISymbol symbol);
LexAction(LexActionType type, size_t state_index, rules::Symbol symbol);
public:
LexAction();
static LexAction Accept(rules::ISymbol symbol);
static LexAction Accept(rules::Symbol symbol);
static LexAction Error();
static LexAction Advance(size_t state_index);
bool operator==(const LexAction &action) const;
LexActionType type;
rules::ISymbol symbol;
rules::Symbol symbol;
size_t state_index;
};
@ -37,7 +37,7 @@ namespace std {
struct hash<tree_sitter::LexAction> {
size_t operator()(const tree_sitter::LexAction &action) const {
return (hash<int>()(action.type) ^
hash<tree_sitter::rules::ISymbol>()(action.symbol) ^
hash<tree_sitter::rules::Symbol>()(action.symbol) ^
hash<size_t>()(action.state_index));
}
};

20
src/compiler/parse_table.cc
View file

@ -7,11 +7,11 @@ namespace tree_sitter {
using std::to_string;
using std::set;
using std::vector;
using rules::ISymbol;
using rules::Symbol;
ParseAction::ParseAction(ParseActionType type,
size_t state_index,
ISymbol symbol,
Symbol symbol,
size_t consumed_symbol_count,
set<int> precedence_values) :
type(type),
@ -22,23 +22,23 @@ namespace tree_sitter {
ParseAction::ParseAction() :
type(ParseActionTypeError),
symbol(ISymbol(-1)),
symbol(Symbol(-1)),
state_index(-1),
consumed_symbol_count(0) {}
ParseAction ParseAction::Error() {
return ParseAction(ParseActionTypeError, -1, ISymbol(-1), 0, { 0 });
return ParseAction(ParseActionTypeError, -1, Symbol(-1), 0, { 0 });
}
ParseAction ParseAction::Accept() {
return ParseAction(ParseActionTypeAccept, -1, ISymbol(-1), 0, { 0 });
return ParseAction(ParseActionTypeAccept, -1, Symbol(-1), 0, { 0 });
}
ParseAction ParseAction::Shift(size_t state_index, set<int> precedence_values) {
return ParseAction(ParseActionTypeShift, state_index, ISymbol(-1), 0, precedence_values);
return ParseAction(ParseActionTypeShift, state_index, Symbol(-1), 0, precedence_values);
}
ParseAction ParseAction::Reduce(ISymbol symbol, size_t consumed_symbol_count, int precedence) {
ParseAction ParseAction::Reduce(Symbol symbol, size_t consumed_symbol_count, int precedence) {
return ParseAction(ParseActionTypeReduce, -1, symbol, consumed_symbol_count, { precedence });
}
@ -66,8 +66,8 @@ namespace tree_sitter {
ParseState::ParseState() : lex_state_id(-1) {}
set<ISymbol> ParseState::expected_inputs() const {
set<ISymbol> result;
set<Symbol> ParseState::expected_inputs() const {
set<Symbol> result;
for (auto &pair : actions)
result.insert(pair.first);
return result;
@ -90,7 +90,7 @@ namespace tree_sitter {
return states.size() - 1;
}
void ParseTable::add_action(ParseStateId id, ISymbol symbol, ParseAction action) {
void ParseTable::add_action(ParseStateId id, Symbol symbol, ParseAction action) {
symbols.insert(symbol);
states[id].actions[symbol] = action;
}

18
src/compiler/parse_table.h
View file

@ -6,7 +6,7 @@
#include <utility>
#include <vector>
#include "compiler/lex_table.h"
#include "compiler/rules/interned_symbol.h"
#include "compiler/rules/symbol.h"
namespace tree_sitter {
typedef enum {
@ -19,7 +19,7 @@ namespace tree_sitter {
class ParseAction {
ParseAction(ParseActionType type,
size_t state_index,
rules::ISymbol symbol,
rules::Symbol symbol,
size_t consumed_symbol_count,
std::set<int> precedence_values);
public:
@ -27,11 +27,11 @@ namespace tree_sitter {
static ParseAction Accept();
static ParseAction Error();
static ParseAction Shift(size_t state_index, std::set<int> precedence_values);
static ParseAction Reduce(rules::ISymbol symbol, size_t consumed_symbol_count, int precedence);
static ParseAction Reduce(rules::Symbol symbol, size_t consumed_symbol_count, int precedence);
bool operator==(const ParseAction &action) const;
ParseActionType type;
rules::ISymbol symbol;
rules::Symbol symbol;
size_t state_index;
size_t consumed_symbol_count;
std::set<int> precedence_values;
@ -46,7 +46,7 @@ namespace std {
size_t operator()(const tree_sitter::ParseAction &action) const {
return (
hash<int>()(action.type) ^
hash<tree_sitter::rules::ISymbol>()(action.symbol) ^
hash<tree_sitter::rules::Symbol>()(action.symbol) ^
hash<size_t>()(action.state_index) ^
hash<size_t>()(action.consumed_symbol_count));
}
@ -57,8 +57,8 @@ namespace tree_sitter {
class ParseState {
public:
ParseState();
std::map<rules::ISymbol, ParseAction> actions;
std::set<rules::ISymbol> expected_inputs() const;
std::map<rules::Symbol, ParseAction> actions;
std::set<rules::Symbol> expected_inputs() const;
LexStateId lex_state_id;
};
@ -69,10 +69,10 @@ namespace tree_sitter {
class ParseTable {
public:
uint64_t add_state();
void add_action(ParseStateId state_id, rules::ISymbol symbol, ParseAction action);
void add_action(ParseStateId state_id, rules::Symbol symbol, ParseAction action);
std::vector<ParseState> states;
std::set<rules::ISymbol> symbols;
std::set<rules::Symbol> symbols;
};
}

6
src/compiler/prepare_grammar/expand_repeats.cc
View file

@ -5,7 +5,7 @@
#include "compiler/prepared_grammar.h"
#include "compiler/rules/visitor.h"
#include "compiler/rules/seq.h"
#include "compiler/rules/interned_symbol.h"
#include "compiler/rules/symbol.h"
#include "compiler/rules/choice.h"
#include "compiler/rules/blank.h"
#include "compiler/rules/repeat.h"
@ -22,7 +22,7 @@ namespace tree_sitter {
using rules::Repeat;
using rules::Rule;
using rules::Seq;
using rules::ISymbol;
using rules::Symbol;
namespace prepare_grammar {
class ExpandRepeats : public rules::IdentityRuleFn {
@ -32,7 +32,7 @@ namespace tree_sitter {
rule_ptr inner_rule = apply(rule->content);
size_t index = aux_rules.size();
string helper_rule_name = rule_name + string("_repeat") + to_string(index);
rule_ptr repeat_symbol = make_shared<ISymbol>(offset + index, rules::SymbolOptionAuxiliary);
rule_ptr repeat_symbol = make_shared<Symbol>(offset + index, rules::SymbolOptionAuxiliary);
aux_rules.push_back({
helper_rule_name,
Choice::Build({

26
src/compiler/prepare_grammar/extract_tokens.cc
View file

@ -9,7 +9,7 @@
#include "compiler/rules/choice.h"
#include "compiler/rules/repeat.h"
#include "compiler/rules/blank.h"
#include "compiler/rules/interned_symbol.h"
#include "compiler/rules/symbol.h"
#include "compiler/rules/string.h"
#include "compiler/rules/pattern.h"
@ -21,7 +21,7 @@ namespace tree_sitter {
using std::vector;
using std::make_shared;
using rules::rule_ptr;
using rules::ISymbol;
using rules::Symbol;
using std::dynamic_pointer_cast;
namespace prepare_grammar {
@ -31,10 +31,10 @@ namespace tree_sitter {
};
class SymbolInliner : public rules::IdentityRuleFn {
map<ISymbol, ISymbol> replacements;
map<Symbol, Symbol> replacements;
using rules::IdentityRuleFn::apply_to;
int new_index_for_symbol(const ISymbol &symbol) {
int new_index_for_symbol(const Symbol &symbol) {
int result = symbol.index;
for (const auto &pair : replacements)
if (pair.first.index < symbol.index &&
@ -43,18 +43,18 @@ namespace tree_sitter {
return result;
}
rule_ptr apply_to(const ISymbol *rule) {
rule_ptr apply_to(const Symbol *rule) {
auto replacement_pair = replacements.find(*rule);
if (replacement_pair != replacements.end())
return replacement_pair->second.copy();
else if (rule->is_built_in())
return rule->copy();
else
return make_shared<ISymbol>(new_index_for_symbol(*rule), rule->options);
return make_shared<Symbol>(new_index_for_symbol(*rule), rule->options);
}
public:
SymbolInliner(const map<ISymbol, ISymbol> &replacements, size_t rule_count, size_t aux_rule_count) :
SymbolInliner(const map<Symbol, Symbol> &replacements, size_t rule_count, size_t aux_rule_count) :
replacements(replacements)
{}
};
@ -73,7 +73,7 @@ namespace tree_sitter {
auto result = rule->copy();
if (IsToken().apply(result)) {
size_t index = add_token(result);
return make_shared<rules::ISymbol>(index, rules::SymbolOption(rules::SymbolOptionToken|rules::SymbolOptionAuxiliary));
return make_shared<rules::Symbol>(index, rules::SymbolOption(rules::SymbolOptionToken|rules::SymbolOptionAuxiliary));
} else {
return result;
}
@ -86,15 +86,15 @@ namespace tree_sitter {
pair<PreparedGrammar, PreparedGrammar> extract_tokens(const PreparedGrammar &input_grammar) {
vector<pair<string, rule_ptr>> rules, tokens, aux_rules, aux_tokens;
TokenExtractor extractor;
map<ISymbol, ISymbol> symbol_replacements;
map<Symbol, Symbol> symbol_replacements;
for (size_t i = 0; i < input_grammar.rules.size(); i++) {
auto pair = input_grammar.rules[i];
if (IsToken().apply(pair.second)) {
tokens.push_back(pair);
symbol_replacements.insert({
ISymbol(i),
ISymbol(tokens.size() - 1, rules::SymbolOption(rules::SymbolOptionToken))
Symbol(i),
Symbol(tokens.size() - 1, rules::SymbolOption(rules::SymbolOptionToken))
});
} else {
rules.push_back({ pair.first, extractor.apply(pair.second) });
@ -106,8 +106,8 @@ namespace tree_sitter {
if (IsToken().apply(pair.second)) {
aux_tokens.push_back(pair);
symbol_replacements.insert({
ISymbol(i, rules::SymbolOptionAuxiliary),
ISymbol(aux_tokens.size() - 1, rules::SymbolOption(rules::SymbolOptionAuxiliary|rules::SymbolOptionToken))
Symbol(i, rules::SymbolOptionAuxiliary),
Symbol(aux_tokens.size() - 1, rules::SymbolOption(rules::SymbolOptionAuxiliary|rules::SymbolOptionToken))
});
} else {
aux_rules.push_back({ pair.first, extractor.apply(pair.second) });

5
src/compiler/prepare_grammar/intern_symbols.cc
View file

@ -4,7 +4,7 @@
#include "compiler/prepared_grammar.h"
#include "compiler/rules/visitor.h"
#include "compiler/rules/named_symbol.h"
#include "compiler/rules/interned_symbol.h"
#include "compiler/rules/symbol.h"
namespace tree_sitter {
using std::string;
@ -12,7 +12,6 @@ namespace tree_sitter {
using std::vector;
using std::pair;
using std::make_shared;
using std::exception;
GrammarError::GrammarError(string rule_name) : rule_name(rule_name) {}
@ -36,7 +35,7 @@ namespace tree_sitter {
long index = index_of(rule->name);
if (index == -1)
missing_rule_name = rule->name;
return make_shared<rules::ISymbol>(index);
return make_shared<rules::Symbol>(index);
}
public:

8
src/compiler/prepared_grammar.cc
View file

@ -2,14 +2,14 @@
#include <vector>
#include <string>
#include <utility>
#include "compiler/rules/interned_symbol.h"
#include "compiler/rules/symbol.h"
namespace tree_sitter {
using std::string;
using std::pair;
using std::ostream;
using rules::rule_ptr;
using rules::ISymbol;
using rules::Symbol;
PreparedGrammar::PreparedGrammar(const std::vector<std::pair<std::string, rules::rule_ptr>> &rules,
const std::vector<std::pair<std::string, rules::rule_ptr>> &aux_rules) :
@ -20,13 +20,13 @@ namespace tree_sitter {
Grammar(grammar),
aux_rules({}) {}
const rule_ptr & PreparedGrammar::rule(const ISymbol &symbol) const {
const rule_ptr & PreparedGrammar::rule(const Symbol &symbol) const {
return symbol.is_auxiliary() ?
aux_rules[symbol.index].second :
rules[symbol.index].second;
}
const string & PreparedGrammar::rule_name(const ISymbol &symbol) const {
const string & PreparedGrammar::rule_name(const Symbol &symbol) const {
return symbol.is_auxiliary() ?
aux_rules[symbol.index].first :
rules[symbol.index].first;

6
src/compiler/prepared_grammar.h
View file

@ -5,7 +5,7 @@
#include <string>
#include <utility>
#include "tree_sitter/compiler.h"
#include "compiler/rules/interned_symbol.h"
#include "compiler/rules/symbol.h"
namespace tree_sitter {
class PreparedGrammar : public Grammar {
@ -15,8 +15,8 @@ namespace tree_sitter {
PreparedGrammar(const Grammar &grammar);
bool operator==(const PreparedGrammar &other) const;
const std::string & rule_name(const rules::ISymbol &symbol) const;
const rules::rule_ptr & rule(const rules::ISymbol &symbol) const;
const std::string & rule_name(const rules::Symbol &symbol) const;
const rules::rule_ptr & rule(const rules::Symbol &symbol) const;
const std::vector<std::pair<std::string, rules::rule_ptr>> aux_rules;
};

6
src/compiler/rules/built_in_symbols.cc
View file

@ -2,8 +2,8 @@
namespace tree_sitter {
namespace rules {
ISymbol END_OF_INPUT() { return ISymbol(-1, SymbolOptionToken); }
ISymbol ERROR() { return ISymbol(-2, SymbolOptionToken); }
ISymbol START() { return ISymbol(-3); }
Symbol END_OF_INPUT() { return Symbol(-1, SymbolOptionToken); }
Symbol ERROR() { return Symbol(-2, SymbolOptionToken); }
Symbol START() { return Symbol(-3); }
}
}

8
src/compiler/rules/built_in_symbols.h
View file

@ -1,13 +1,13 @@
#ifndef COMPILER_RULES_BUILT_IN_SYMBOLS_H_
#define COMPILER_RULES_BUILT_IN_SYMBOLS_H_
#include "compiler/rules/interned_symbol.h"
#include "compiler/rules/symbol.h"
namespace tree_sitter {
namespace rules {
ISymbol ERROR();
ISymbol START();
ISymbol END_OF_INPUT();
Symbol ERROR();
Symbol START();
Symbol END_OF_INPUT();
}
}

7
src/compiler/rules/named_symbol.h
View file

@ -1,8 +1,7 @@
#ifndef COMPILER_RULES_SYMBOL_H_
#define COMPILER_RULES_SYMBOL_H_
#ifndef COMPILER_RULES_NAMED_SYMBOL_H_
#define COMPILER_RULES_NAMED_SYMBOL_H_
#include <string>
#include <utility>
#include "compiler/rules/rule.h"
namespace tree_sitter {
@ -22,4 +21,4 @@ namespace tree_sitter {
}
}
#endif // COMPILER_RULES_SYMBOL_H_
#endif // COMPILER_RULES_NAMED_SYMBOL_H_

30
src/compiler/rules/interned_symbol.cc → src/compiler/rules/symbol.cc
View file

@ -1,4 +1,4 @@
#include "compiler/rules/interned_symbol.h"
#include "compiler/rules/symbol.h"
#include <map>
#include <string>
#include "compiler/rules/visitor.h"
@ -9,56 +9,56 @@ namespace tree_sitter {
using std::hash;
namespace rules {
ISymbol::ISymbol(int index) :
Symbol::Symbol(int index) :
index(index),
options(SymbolOption(0)) {}
ISymbol::ISymbol(int index, SymbolOption options) :
Symbol::Symbol(int index, SymbolOption options) :
index(index),
options(options) {}
bool ISymbol::operator==(const ISymbol &other) const {
bool Symbol::operator==(const Symbol &other) const {
return (other.index == index) && (other.options == options);
}
bool ISymbol::operator==(const Rule &rule) const {
const ISymbol *other = dynamic_cast<const ISymbol *>(&rule);
bool Symbol::operator==(const Rule &rule) const {
const Symbol *other = dynamic_cast<const Symbol *>(&rule);
return other && this->operator==(*other);
}
size_t ISymbol::hash_code() const {
size_t Symbol::hash_code() const {
return hash<int>()(index) ^ hash<int16_t>()(options);
}
rule_ptr ISymbol::copy() const {
return std::make_shared<ISymbol>(*this);
rule_ptr Symbol::copy() const {
return std::make_shared<Symbol>(*this);
}
string ISymbol::to_string() const {
string Symbol::to_string() const {
string name = (options & SymbolOptionAuxiliary) ? "aux_" : "";
name += (options & SymbolOptionToken) ? "token" : "sym";
return "#<" + name + std::to_string(index) + ">";
}
bool ISymbol::operator<(const ISymbol &other) const {
bool Symbol::operator<(const Symbol &other) const {
if (options < other.options) return true;
if (options > other.options) return false;
return (index < other.index);
}
bool ISymbol::is_token() const {
bool Symbol::is_token() const {
return options & SymbolOptionToken;
}
bool ISymbol::is_built_in() const {
bool Symbol::is_built_in() const {
return index < 0;
}
bool ISymbol::is_auxiliary() const {
bool Symbol::is_auxiliary() const {
return options & SymbolOptionAuxiliary;
}
void ISymbol::accept(Visitor *visitor) const {
void Symbol::accept(Visitor *visitor) const {
visitor->visit(this);
}
}

20
src/compiler/rules/interned_symbol.h → src/compiler/rules/symbol.h
View file

@ -1,5 +1,5 @@
#ifndef COMPILER_RULES_INTERNED_SYMBOL_H_
#define COMPILER_RULES_INTERNED_SYMBOL_H_
#ifndef COMPILER_RULES_SYMBOL_H_
#define COMPILER_RULES_SYMBOL_H_
#include "compiler/rules/rule.h"
@ -10,20 +10,20 @@ namespace tree_sitter {
SymbolOptionAuxiliary = 1 << 1,
} SymbolOption;
class ISymbol : public Rule {
class Symbol : public Rule {
public:
explicit ISymbol(int index);
ISymbol(int index, SymbolOption options);
explicit Symbol(int index);
Symbol(int index, SymbolOption options);
bool operator==(const ISymbol &other) const;
bool operator==(const Symbol &other) const;
bool operator==(const Rule &other) const;
size_t hash_code() const;
rule_ptr copy() const;
std::string to_string() const;
void accept(Visitor *visitor) const;
bool operator<(const ISymbol &other) const;
bool operator<(const Symbol &other) const;
bool is_token() const;
bool is_built_in() const;
bool is_auxiliary() const;
@ -36,11 +36,11 @@ namespace tree_sitter {
namespace std {
template<>
struct hash<tree_sitter::rules::ISymbol> {
size_t operator()(const tree_sitter::rules::ISymbol &rule) const {
struct hash<tree_sitter::rules::Symbol> {
size_t operator()(const tree_sitter::rules::Symbol &rule) const {
return rule.hash_code();
}
};
}
#endif // COMPILER_RULES_INTERNED_SYMBOL_H_
#endif // COMPILER_RULES_SYMBOL_H_

8
src/compiler/rules/visitor.h
View file

@ -12,7 +12,7 @@ namespace tree_sitter {
class Repeat;
class Seq;
class String;
class ISymbol;
class Symbol;
class Pattern;
class Metadata;
@ -27,7 +27,7 @@ namespace tree_sitter {
virtual void visit(const Seq *rule) = 0;
virtual void visit(const String *rule) = 0;
virtual void visit(const NamedSymbol *rule) = 0;
virtual void visit(const ISymbol *rule) = 0;
virtual void visit(const Symbol *rule) = 0;
virtual ~Visitor();
};
@ -51,7 +51,7 @@ namespace tree_sitter {
virtual T apply_to(const Seq *rule) { return default_apply((const Rule *)rule); }
virtual T apply_to(const String *rule) { return default_apply((const Rule *)rule); }
virtual T apply_to(const NamedSymbol *rule) { return default_apply((const Rule *)rule); }
virtual T apply_to(const ISymbol *rule) { return default_apply((const Rule *)rule); }
virtual T apply_to(const Symbol *rule) { return default_apply((const Rule *)rule); }
void visit(const Blank *rule) { value_ = apply_to(rule); }
void visit(const CharacterSet *rule) { value_ = apply_to(rule); }
@ -62,7 +62,7 @@ namespace tree_sitter {
void visit(const Seq *rule) { value_ = apply_to(rule); }
void visit(const String *rule) { value_ = apply_to(rule); }
void visit(const NamedSymbol *rule) { value_ = apply_to(rule); }
void visit(const ISymbol *rule) { value_ = apply_to(rule); }
void visit(const Symbol *rule) { value_ = apply_to(rule); }
private:
T value_;