traxys/tree-sitter
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

In compiler, distinguish between anonymous tokens and hidden rules

Browse source
This commit is contained in:
Max Brunsfeld 2015-09-05 17:05:37 -07:00
parent 4b270c8604
commit 5982b77c97
46 changed files with 41131 additions and 40884 deletions
Download patch file Download diff file Expand all files Collapse all files

2
project.gyp
View file

@ -29,7 +29,6 @@
'src/compiler/generate_code/c_code.cc',
'src/compiler/grammar.cc',
'src/compiler/lex_table.cc',
'src/compiler/lexical_grammar.cc',
'src/compiler/parse_table.cc',
'src/compiler/prepare_grammar/expand_repeats.cc',
'src/compiler/prepare_grammar/expand_tokens.cc',
@ -39,7 +38,6 @@
'src/compiler/prepare_grammar/parse_regex.cc',
'src/compiler/prepare_grammar/prepare_grammar.cc',
'src/compiler/prepare_grammar/token_description.cc',
'src/compiler/syntax_grammar.cc',
'src/compiler/rule.cc',
'src/compiler/rules/blank.cc',
'src/compiler/rules/built_in_symbols.cc',

47
spec/compiler/build_tables/build_parse_table_spec.cc
View file

@ -1,8 +1,7 @@
#include "compiler/compiler_spec_helper.h"
#include "compiler/build_tables/build_parse_table.h"
#include "compiler/parse_table.h"
#include "compiler/lexical_grammar.h"
#include "compiler/syntax_grammar.h"
#include "compiler/prepared_grammar.h"
#include "compiler/rules/built_in_symbols.h"
using namespace rules;
@ -12,15 +11,35 @@ START_TEST
describe("build_parse_table", []() {
SyntaxGrammar parse_grammar{{
{ "rule0", choice({ i_sym(1), i_sym(2) }) },
{ "rule1", i_token(0) },
{ "rule2", i_token(1) },
}, {}, { Symbol(2, SymbolOptionToken) }, {}};
{
"rule0",
choice({ i_sym(1), i_sym(2) }),
RuleEntryTypeNamed
},
{
"rule1",
i_token(0),
RuleEntryTypeNamed
},
{
"rule2",
i_token(1),
RuleEntryTypeNamed
},
}, { Symbol(2, true) }, {}};
LexicalGrammar lex_grammar{{
{ "token0", pattern("[a-c]") },
{ "token1", pattern("[b-d]") },
}, {}, {}};
{
"token0",
pattern("[a-c]"),
RuleEntryTypeNamed
},
{
"token1",
pattern("[b-d]"),
RuleEntryTypeNamed
},
}, {}};
it("first looks for the start rule and its item set closure", [&]() {
auto result = build_parse_table(parse_grammar, lex_grammar);
@ -32,11 +51,11 @@ describe("build_parse_table", []() {
// expanded from the item set closure of the start item
{ Symbol(1), {ParseAction::Shift(2, { 0 })} },
{ Symbol(2), {ParseAction::Shift(2, { 0 })} },
{ Symbol(0, SymbolOptionToken), {ParseAction::Shift(3, { 0 })} },
{ Symbol(1, SymbolOptionToken), {ParseAction::Shift(4, { 0 })} },
{ Symbol(0, true), {ParseAction::Shift(3, { 0 })} },
{ Symbol(1, true), {ParseAction::Shift(4, { 0 })} },
// for the ubiquitous_token 'token2'
{ Symbol(2, SymbolOptionToken), {ParseAction::ShiftExtra()} },
{ Symbol(2, true), {ParseAction::ShiftExtra()} },
})));
});
@ -52,7 +71,7 @@ describe("build_parse_table", []() {
{ END_OF_INPUT(), {ParseAction::Accept()} },
// for the ubiquitous_token 'token2'
{ Symbol(2, SymbolOptionToken), {ParseAction::ShiftExtra()} },
{ Symbol(2, true), {ParseAction::ShiftExtra()} },
})));
});
@ -63,7 +82,7 @@ describe("build_parse_table", []() {
{ END_OF_INPUT(), {ParseAction::Reduce(Symbol(0), 1, 0, AssociativityLeft, 0)} },
// for the ubiquitous_token 'token2'
{ Symbol(2, SymbolOptionToken), {ParseAction::ShiftExtra()} },
{ Symbol(2, true), {ParseAction::ShiftExtra()} },
})));
});
});

62
spec/compiler/build_tables/first_symbols_spec.cc
View file

@ -1,5 +1,5 @@
#include "compiler/compiler_spec_helper.h"
#include "compiler/syntax_grammar.h"
#include "compiler/prepared_grammar.h"
#include "compiler/build_tables/first_symbols.h"
#include "compiler/rules/metadata.h"
@ -16,7 +16,7 @@ describe("first_symbols", []() {
auto rule = seq({ i_token(0), i_token(1) });
AssertThat(first_symbols(rule, null_grammar), Equals(set<Symbol>({
Symbol(0, SymbolOptionToken),
Symbol(0, true),
})));
});
@ -28,8 +28,8 @@ describe("first_symbols", []() {
i_token(1) });
AssertThat(first_symbols(rule, null_grammar), Equals(set<Symbol>({
Symbol(0, SymbolOptionToken),
Symbol(1, SymbolOptionToken)
Symbol(0, true),
Symbol(1, true)
})));
});
@ -41,16 +41,21 @@ describe("first_symbols", []() {
i_sym(0) });
SyntaxGrammar grammar{{
{ "rule0", seq({
i_token(2),
i_token(3),
i_token(4) }) }
}, {}, {}, {}};
{
"rule0",
seq({
i_token(2),
i_token(3),
i_token(4),
}),
RuleEntryTypeNamed
}
}, {}, {}};
AssertThat(first_symbols(rule, grammar), Equals(set<Symbol>({
Symbol(0),
Symbol(0, SymbolOptionToken),
Symbol(2, SymbolOptionToken),
Symbol(0, true),
Symbol(2, true),
})));
});
@ -60,15 +65,20 @@ describe("first_symbols", []() {
i_token(1) });
SyntaxGrammar grammar{{
{ "rule0", choice({
i_token(0),
blank() }) }
}, {}, {}, {}};
{
"rule0",
choice({
i_token(0),
blank(),
}),
RuleEntryTypeNamed
},
}, {}, {}};
AssertThat(first_symbols(rule, grammar), Equals(set<Symbol>({
Symbol(0),
Symbol(0, SymbolOptionToken),
Symbol(1, SymbolOptionToken),
Symbol(0, true),
Symbol(1, true),
})));
});
});
@ -76,17 +86,21 @@ describe("first_symbols", []() {
describe("when there are left-recursive rules", [&]() {
it("terminates", [&]() {
SyntaxGrammar grammar{{
{ "rule0", choice({
seq({ i_sym(0), i_token(10) }),
i_token(11),
}) },
}, {}, {}, {}};
{
"rule0",
choice({
seq({ i_sym(0), i_token(10) }),
i_token(11),
}),
RuleEntryTypeNamed
},
}, {}, {}};
auto rule = i_sym(0);
AssertThat(first_symbols(rule, grammar), Equals(set<Symbol>({
Symbol(0),
Symbol(11, SymbolOptionToken)
Symbol(11, true)
})));
});
});
@ -95,7 +109,7 @@ describe("first_symbols", []() {
auto rule = make_shared<Metadata>(i_token(3), map<rules::MetadataKey, int>());
AssertThat(first_symbols(rule, null_grammar), Equals(set<Symbol>({
Symbol(3, SymbolOptionToken),
Symbol(3, true),
})));
});
});

38
spec/compiler/build_tables/item_set_closure_spec.cc
View file

@ -1,5 +1,5 @@
#include "compiler/compiler_spec_helper.h"
#include "compiler/syntax_grammar.h"
#include "compiler/prepared_grammar.h"
#include "compiler/build_tables/item_set_closure.h"
#include "compiler/build_tables/item_set_transitions.h"
@ -10,29 +10,39 @@ START_TEST
describe("item_set_closure", []() {
SyntaxGrammar grammar{{
{ "E", seq({
i_sym(1),
i_token(11) }) },
{ "T", seq({
i_token(12),
i_token(13) }) },
}, {}, {}, {}};
{
"E",
seq({
i_sym(1),
i_token(11),
}),
RuleEntryTypeNamed,
},
{
"T",
seq({
i_token(12),
i_token(13),
}),
RuleEntryTypeNamed,
},
}, {}, {}};
it("adds items at the beginnings of referenced rules", [&]() {
ParseItemSet item_set = item_set_closure(
ParseItem(Symbol(0), grammar.rule(Symbol(0)), {}),
set<Symbol>({ Symbol(10, SymbolOptionToken) }),
ParseItem(Symbol(0), grammar.rules[0].rule, {}),
set<Symbol>({ Symbol(10, true) }),
grammar
);
AssertThat(item_set, Equals(ParseItemSet({
{
ParseItem(Symbol(1), grammar.rule(Symbol(1)), {}),
set<Symbol>({ Symbol(11, SymbolOptionToken) }),
ParseItem(Symbol(1), grammar.rules[1].rule, {}),
set<Symbol>({ Symbol(11, true) }),
},
{
ParseItem(Symbol(0), grammar.rule(Symbol(0)), {}),
set<Symbol>({ Symbol(10, SymbolOptionToken) }),
ParseItem(Symbol(0), grammar.rules[0].rule, {}),
set<Symbol>({ Symbol(10, true) }),
},
})));
});

24
spec/compiler/build_tables/item_set_transitions_spec.cc
View file

@ -1,6 +1,6 @@
#include "compiler/compiler_spec_helper.h"
#include "compiler/build_tables/item_set_transitions.h"
#include "compiler/syntax_grammar.h"
#include "compiler/prepared_grammar.h"
#include "compiler/helpers/rule_helpers.h"
using namespace rules;
@ -43,29 +43,37 @@ describe("char_transitions(LexItemSet)", []() {
describe("sym_transitions(ParseItemSet, SyntaxGrammar)", [&]() {
SyntaxGrammar grammar{{
{ "A", blank() },
{ "B", i_token(21) },
}, {}, {}, {}};
{
"A",
blank(),
RuleEntryTypeNamed
},
{
"B",
i_token(21),
RuleEntryTypeNamed
},
}, {}, {}};
it("computes the closure of the new item sets", [&]() {
ParseItemSet set1({
{
ParseItem(Symbol(0), seq({ i_token(22), i_sym(1) }), { Symbol(101) }),
set<Symbol>({ Symbol(23, SymbolOptionToken) })
set<Symbol>({ Symbol(23, true) })
},
});
AssertThat(sym_transitions(set1, grammar), Equals(map<Symbol, ParseItemSet>({
{
Symbol(22, SymbolOptionToken),
Symbol(22, true),
ParseItemSet({
{
ParseItem(Symbol(0), i_sym(1), { Symbol(101), Symbol(22) }),
set<Symbol>({ Symbol(23, SymbolOptionToken) }),
set<Symbol>({ Symbol(23, true) }),
},
{
ParseItem(Symbol(1), i_token(21), {}),
set<Symbol>({ Symbol(23, SymbolOptionToken) })
set<Symbol>({ Symbol(23, true) })
},
})
},

22
spec/compiler/build_tables/lex_conflict_manager_spec.cc
View file

@ -2,7 +2,7 @@
#include "compiler/rules/built_in_symbols.h"
#include "compiler/parse_table.h"
#include "compiler/build_tables/lex_conflict_manager.h"
#include "compiler/syntax_grammar.h"
#include "compiler/prepared_grammar.h"
using namespace rules;
using namespace build_tables;
@ -11,16 +11,24 @@ START_TEST
describe("LexConflictManager", []() {
LexicalGrammar lexical_grammar{{
{ "other_token", pattern("[a-b]") },
{ "lookahead_token", pattern("[c-d]") },
}, {}, {}};
{
"other_token",
pattern("[a-b]"),
RuleEntryTypeNamed
},
{
"lookahead_token",
pattern("[c-d]"),
RuleEntryTypeNamed
},
}, {}};
LexConflictManager conflict_manager(lexical_grammar);
bool update;
Symbol sym1(0, SymbolOptionToken);
Symbol sym2(1, SymbolOptionToken);
Symbol sym3(2, SymbolOptionToken);
Symbol sym1(0, true);
Symbol sym2(1, true);
Symbol sym3(2, true);
it("favors non-errors over lexical errors", [&]() {
update = conflict_manager.resolve(LexAction::Advance(2, {0}), LexAction::Error());

36
spec/compiler/build_tables/parse_conflict_manager_spec.cc
View file

@ -2,7 +2,7 @@
#include "compiler/rules/built_in_symbols.h"
#include "compiler/parse_table.h"
#include "compiler/build_tables/parse_conflict_manager.h"
#include "compiler/syntax_grammar.h"
#include "compiler/prepared_grammar.h"
using namespace rules;
using namespace build_tables;
@ -11,17 +11,37 @@ START_TEST
describe("ParseConflictManager", []() {
SyntaxGrammar syntax_grammar{{
{ "in_progress_rule1", i_token(0) },
{ "in_progress_rule2", i_token(0) },
{ "reduced_rule", i_token(0) },
{ "other_rule1", i_token(0) },
{ "other_rule2", i_token(0) },
}, {}, { Symbol(2, SymbolOptionToken) }, {}};
{
"in_progress_rule1",
i_token(0),
RuleEntryTypeNamed,
},
{
"in_progress_rule2",
i_token(0),
RuleEntryTypeNamed,
},
{
"reduced_rule",
i_token(0),
RuleEntryTypeNamed,
},
{
"other_rule1",
i_token(0),
RuleEntryTypeNamed,
},
{
"other_rule2",
i_token(0),
RuleEntryTypeNamed,
},
}, { Symbol(2, true) }, {}};
pair<bool, ConflictType> result;
Symbol sym1(0);
Symbol sym2(1);
Symbol lookahead_sym(1, SymbolOptionToken);
Symbol lookahead_sym(1, true);
ParseConflictManager *conflict_manager;
before_each([&]() {

26
spec/compiler/build_tables/rule_can_be_blank_spec.cc
View file

@ -1,7 +1,7 @@
#include "compiler/compiler_spec_helper.h"
#include "compiler/build_tables/rule_can_be_blank.h"
#include "compiler/rules/metadata.h"
#include "compiler/syntax_grammar.h"
#include "compiler/prepared_grammar.h"
using namespace rules;
using build_tables::rule_can_be_blank;
@ -57,13 +57,23 @@ describe("rule_can_be_blank", [&]() {
describe("checking recursively (by expanding non-terminals)", [&]() {
SyntaxGrammar grammar{{
{ "A", choice({
seq({ i_sym(0), i_token(11) }),
blank() }) },
{ "B", choice({
seq({ i_sym(1), i_token(12) }),
i_token(13) }) },
}, {}, {}, {}};
{
"A",
choice({
seq({ i_sym(0), i_token(11) }),
blank()
}),
RuleEntryTypeNamed,
},
{
"B",
choice({
seq({ i_sym(1), i_token(12) }),
i_token(13)
}),
RuleEntryTypeNamed,
},
}, {}, {}};
it("terminates for left-recursive rules that can be blank", [&]() {
rule = i_sym(0);

14
spec/compiler/helpers/containers.h
View file

@ -48,6 +48,20 @@ class rule_list : public vector<pair<string, rule_ptr>> {
vector<pair<string, rule_ptr>>(list) {}
};
template<typename T>
class eq_vector : public vector<T> {
public:
bool operator==(const vector<T> &other) const {
if (this->size() != other.size()) return false;
for (size_t i = 0; i < this->size(); i++)
if (!(this->operator[](i) == other[i]))
return false;
return true;
}
eq_vector(const initializer_list<T> &list) : vector<T>(list) {}
};
class rule_vector : public vector<rule_ptr> {
public:
bool operator==(const vector<rule_ptr> &other) const {

60
spec/compiler/helpers/rule_helpers.cc
View file

@ -6,43 +6,41 @@ namespace tree_sitter {
using std::make_shared;
using std::set;
using std::map;
using std::ostream;
using std::string;
using std::to_string;
namespace rules {
rule_ptr character(const set<uint32_t> &ranges) {
return character(ranges, true);
}
rule_ptr character(const set<uint32_t> &ranges) {
return character(ranges, true);
}
rule_ptr character(const set<uint32_t> &chars, bool sign) {
CharacterSet result;
if (sign) {
for (uint32_t c : chars)
result.include(c);
} else {
result.include_all();
for (uint32_t c : chars)
result.exclude(c);
}
return result.copy();
rule_ptr character(const set<uint32_t> &chars, bool sign) {
rules::CharacterSet result;
if (sign) {
for (uint32_t c : chars)
result.include(c);
} else {
result.include_all();
for (uint32_t c : chars)
result.exclude(c);
}
return result.copy();
}
rule_ptr i_sym(size_t index) {
return make_shared<rules::Symbol>(index);
}
rule_ptr i_sym(size_t index) {
return make_shared<rules::Symbol>(index);
}
rule_ptr i_aux_sym(size_t index) {
return make_shared<rules::Symbol>(index, SymbolOptionAuxiliary);
}
rule_ptr i_token(size_t index) {
return make_shared<rules::Symbol>(index, true);
}
rule_ptr i_token(size_t index) {
return make_shared<rules::Symbol>(index, SymbolOptionToken);
}
rule_ptr metadata(rule_ptr rule, map<rules::MetadataKey, int> values) {
return make_shared<rules::Metadata>(rule, values);
}
rule_ptr i_aux_token(size_t index) {
return make_shared<rules::Symbol>(index, SymbolOption(SymbolOptionAuxiliary|SymbolOptionToken));
}
rule_ptr metadata(rule_ptr rule, map<MetadataKey, int> values) {
return make_shared<Metadata>(rule, values);
}
bool operator==(const RuleEntry &left, const RuleEntry &right) {
return left.name == right.name && left.rule->operator==(*right.rule) &&
left.type == right.type;
}
}

17
spec/compiler/helpers/rule_helpers.h
View file

@ -4,17 +4,16 @@
#include "tree_sitter/compiler.h"
#include "compiler/rules/character_set.h"
#include "compiler/rules/metadata.h"
#include "compiler/prepared_grammar.h"
namespace tree_sitter {
namespace rules {
rule_ptr metadata(rule_ptr, std::map<MetadataKey, int>);
rule_ptr character(const std::set<uint32_t> &);
rule_ptr character(const std::set<uint32_t> &, bool sign);
rule_ptr i_sym(size_t index);
rule_ptr i_aux_sym(size_t index);
rule_ptr i_token(size_t index);
rule_ptr i_aux_token(size_t index);
}
rule_ptr metadata(rule_ptr, std::map<rules::MetadataKey, int>);
rule_ptr character(const std::set<uint32_t> &);
rule_ptr character(const std::set<uint32_t> &, bool sign);
rule_ptr i_sym(size_t index);
rule_ptr i_token(size_t index);
bool operator==(const RuleEntry &left, const RuleEntry &right);
}
#endif

13
spec/compiler/helpers/stream_methods.h
View file

@ -7,6 +7,7 @@
#include <map>
#include <unordered_set>
#include <vector>
#include "compiler/prepared_grammar.h"
using std::cout;
@ -83,4 +84,16 @@ inline std::ostream& operator<<(std::ostream &stream, const std::pair<T1, T2> &p
} // namespace std
namespace tree_sitter {
using std::ostream;
using std::string;
using std::to_string;
inline ostream &operator<<(ostream &stream, const RuleEntry &entry) {
return stream << string("{") << entry.name << string(", ") << entry.rule << string(", ") << to_string(entry.type) << string("}");
}
}
#endif

258
spec/compiler/prepare_grammar/expand_repeats_spec.cc
View file

@ -1,5 +1,5 @@
#include "compiler/compiler_spec_helper.h"
#include "compiler/syntax_grammar.h"
#include "compiler/prepared_grammar.h"
#include "compiler/prepare_grammar/expand_repeats.h"
#include "compiler/helpers/containers.h"
@ -11,131 +11,223 @@ using prepare_grammar::expand_repeats;
describe("expand_repeats", []() {
it("replaces repeat rules with pairs of recursive rules", [&]() {
SyntaxGrammar grammar{{
{ "rule0", repeat(i_token(0)) },
}, {}, {}, {}};
{
"rule0",
repeat(i_token(0)),
RuleEntryTypeNamed,
},
}, {}, {}};
auto match = expand_repeats(grammar);
AssertThat(match.rules, Equals(rule_list({
{ "rule0", choice({ i_aux_sym(0), blank() }) },
})));
AssertThat(match.aux_rules, Equals(rule_list({
{ "rule0_repeat0", seq({
i_token(0),
choice({ i_aux_sym(0), blank() }) }) },
AssertThat(match.rules, Equals(eq_vector<RuleEntry>({
{
"rule0",
choice({ i_sym(1), blank() }),
RuleEntryTypeNamed,
},
{
"rule0_repeat1",
seq({
i_token(0),
choice({ i_sym(1), blank() })
}),
RuleEntryTypeHidden
},
})));
});
it("replaces repeats inside of sequences", [&]() {
SyntaxGrammar grammar{{
{ "rule0", seq({
i_token(10),
repeat(i_token(11)) }) },
}, {}, {}, {}};
{
"rule0",
seq({
i_token(10),
repeat(i_token(11)),
}),
RuleEntryTypeNamed,
},
}, {}, {}};
auto match = expand_repeats(grammar);
AssertThat(match.rules, Equals(rule_list({
{ "rule0", seq({
i_token(10),
choice({ i_aux_sym(0), blank() }) }) },
})));
AssertThat(match.aux_rules, Equals(rule_list({
{ "rule0_repeat0", seq({
i_token(11),
choice({ i_aux_sym(0), blank() }) }) },
AssertThat(match.rules, Equals(eq_vector<RuleEntry>({
{
"rule0",
seq({
i_token(10),
choice({ i_sym(1), blank() })
}),
RuleEntryTypeNamed
},
{
"rule0_repeat1",
seq({
i_token(11),
choice({ i_sym(1), blank() })
}),
RuleEntryTypeHidden
},
})));
});
it("replaces repeats inside of choices", [&]() {
SyntaxGrammar grammar{{
{ "rule0", choice({ i_token(10), repeat(i_token(11)) }) },
}, {}, {}, {}};
{
"rule0",
choice({ i_token(10), repeat(i_token(11)) }),
RuleEntryTypeNamed
},
}, {}, {}};
auto match = expand_repeats(grammar);
AssertThat(match.rules, Equals(rule_list({
{ "rule0", choice({ i_token(10), i_aux_sym(0), blank() }) },
})));
AssertThat(match.aux_rules, Equals(rule_list({
{ "rule0_repeat0", seq({
i_token(11),
choice({ i_aux_sym(0), blank() }) }) },
AssertThat(match.rules, Equals(eq_vector<RuleEntry>({
{
"rule0",
choice({ i_token(10), i_sym(1), blank() }),
RuleEntryTypeNamed
},
{
"rule0_repeat1",
seq({
i_token(11),
choice({ i_sym(1), blank() }),
}),
RuleEntryTypeHidden
},
})));
});
it("does not create redundant auxiliary rules", [&]() {
SyntaxGrammar grammar{{
{ "rule0", choice({
seq({ i_token(1), repeat(i_token(4)) }),
seq({ i_token(2), repeat(i_token(4)) }) }) },
{ "rule1", seq({ i_token(3), repeat(i_token(4)) }) },
}, {}, {}, {}};
{
"rule0",
choice({
seq({ i_token(1), repeat(i_token(4)) }),
seq({ i_token(2), repeat(i_token(4)) }),
}),
RuleEntryTypeNamed
},
{
"rule1",
seq({ i_token(3), repeat(i_token(4)) }),
RuleEntryTypeNamed
},
}, {}, {}};
auto match = expand_repeats(grammar);
AssertThat(match.rules, Equals(rule_list({
{ "rule0", choice({
seq({ i_token(1), choice({ i_aux_sym(0), blank() }) }),
seq({ i_token(2), choice({ i_aux_sym(0), blank() }) }) }) },
{ "rule1", seq({ i_token(3), choice({ i_aux_sym(0), blank() }) }) },
})));
AssertThat(match.aux_rules, Equals(rule_list({
{ "rule0_repeat0", seq({
i_token(4),
choice({ i_aux_sym(0), blank() }) }) },
AssertThat(match.rules, Equals(eq_vector<RuleEntry>({
{
"rule0",
choice({
seq({ i_token(1), choice({ i_sym(2), blank() }) }),
seq({ i_token(2), choice({ i_sym(2), blank() }) }),
}),
RuleEntryTypeNamed
},
{
"rule1",
seq({ i_token(3), choice({ i_sym(2), blank() }) }),
RuleEntryTypeNamed
},
{
"rule0_repeat1",
seq({
i_token(4),
choice({ i_sym(2), blank() }),
}),
RuleEntryTypeHidden
},
})));
});
it("can replace multiple repeats in the same rule", [&]() {
SyntaxGrammar grammar{{
{ "rule0", seq({
repeat(i_token(10)),
repeat(i_token(11)) }) },
}, {}, {}, {}};
{
"rule0",
seq({
repeat(i_token(10)),
repeat(i_token(11)),
}),
RuleEntryTypeNamed
},
}, {}, {}};
auto match = expand_repeats(grammar);
AssertThat(match.rules, Equals(rule_list({
{ "rule0", seq({
choice({ i_aux_sym(0), blank() }),
choice({ i_aux_sym(1), blank() }) }) },
})));
AssertThat(match.aux_rules, Equals(rule_list({
{ "rule0_repeat0", seq({
i_token(10),
choice({ i_aux_sym(0), blank() }) }) },
{ "rule0_repeat1", seq({
i_token(11),
choice({ i_aux_sym(1), blank() }) }) },
AssertThat(match.rules, Equals(eq_vector<RuleEntry>({
{
"rule0",
seq({
choice({ i_sym(1), blank() }),
choice({ i_sym(2), blank() }),
}),
RuleEntryTypeNamed
},
{
"rule0_repeat1",
seq({
i_token(10),
choice({ i_sym(1), blank() }),
}),
RuleEntryTypeHidden
},
{
"rule0_repeat2",
seq({
i_token(11),
choice({ i_sym(2), blank() }),
}),
RuleEntryTypeHidden
},
})));
});
it("can replace repeats in multiple rules", [&]() {
SyntaxGrammar grammar{{
{ "rule0", repeat(i_token(10)) },
{ "rule1", repeat(i_token(11)) },
}, {}, {}, {}};
{
"rule0",
repeat(i_token(10)),
RuleEntryTypeNamed,
},
{
"rule1",
repeat(i_token(11)),
RuleEntryTypeNamed,
},
}, {}, {}};
auto match = expand_repeats(grammar);
AssertThat(match.rules, Equals(rule_list({
{ "rule0", choice({ i_aux_sym(0), blank() }) },
{ "rule1", choice({ i_aux_sym(1), blank() }) },
})));
AssertThat(match.aux_rules, Equals(rule_list({
{ "rule0_repeat0", seq({
i_token(10),
choice({ i_aux_sym(0), blank() }) }) },
{ "rule1_repeat0", seq({
i_token(11),
choice({ i_aux_sym(1), blank() }) }) },
AssertThat(match.rules, Equals(eq_vector<RuleEntry>({
{
"rule0",
choice({ i_sym(2), blank() }),
RuleEntryTypeNamed
},
{
"rule1",
choice({ i_sym(3), blank() }),
RuleEntryTypeNamed
},
{
"rule0_repeat1",
seq({
i_token(10),
choice({ i_sym(2), blank() }),
}),
RuleEntryTypeHidden
},
{
"rule1_repeat1",
seq({
i_token(11),
choice({ i_sym(3), blank() })
}),
RuleEntryTypeHidden
},
})));
});
});

128
spec/compiler/prepare_grammar/expand_tokens_spec.cc
View file

@ -1,5 +1,5 @@
#include "compiler/compiler_spec_helper.h"
#include "compiler/lexical_grammar.h"
#include "compiler/prepared_grammar.h"
#include "compiler/helpers/containers.h"
#include "compiler/prepare_grammar/expand_tokens.h"
@ -12,36 +12,64 @@ describe("expand_tokens", []() {
describe("string rules", [&]() {
it("replaces strings with sequences of character sets", [&]() {
LexicalGrammar grammar{{
{ "rule_A", seq({
i_sym(10),
str("xyz"),
i_sym(11) }) },
}, {}, {}};
{
"rule_A",
seq({
i_sym(10),
str("xyz"),
i_sym(11),
}),
RuleEntryTypeNamed
},
}, {}};
auto result = expand_tokens(grammar);
AssertThat(result.second, Equals((const GrammarError *)nullptr));
AssertThat(result.first.rules, Equals(rule_list({
{ "rule_A", seq({
i_sym(10),
token(prec(1, seq({ character({ 'x' }), character({ 'y' }), character({ 'z' }) }))),
i_sym(11) }) },
AssertThat(result.first.rules, Equals(eq_vector<RuleEntry>({
{
"rule_A",
seq({
i_sym(10),
metadata(seq({
character({ 'x' }),
character({ 'y' }),
character({ 'z' }),
}), {
{PRECEDENCE, 1},
{IS_TOKEN, 1},
}),
i_sym(11),
}),
RuleEntryTypeNamed
},
})));
});
it("handles strings containing non-ASCII UTF8 characters", [&]() {
LexicalGrammar grammar{{
// α β
{ "rule_A", str("\u03B1 \u03B2") },
}, {}, {}};
{
"rule_A",
str("\u03B1 \u03B2"), // α β
RuleEntryTypeNamed
},
}, {}};
auto result = expand_tokens(grammar);
AssertThat(result.first.rules, Equals(rule_list({
{ "rule_A", token(prec(1, seq({
character({ 945 }),
character({ ' ' }),
character({ 946 }) }))) }
AssertThat(result.first.rules, Equals(eq_vector<RuleEntry>({
{
"rule_A",
metadata(seq({
character({ 945 }),
character({ ' ' }),
character({ 946 }),
}), {
{PRECEDENCE, 1},
{IS_TOKEN, 1},
}),
RuleEntryTypeNamed
}
})));
});
});
@ -49,43 +77,65 @@ describe("expand_tokens", []() {
describe("regexp rules", [&]() {
it("replaces regexps with the equivalent rule tree", [&]() {
LexicalGrammar grammar{{
{ "rule_A", seq({
i_sym(10),
pattern("x*"),
i_sym(11) }) },
}, {}, {}};
{
"rule_A",
seq({
i_sym(10),
pattern("x*"),
i_sym(11),
}),
RuleEntryTypeNamed
},
}, {}};
auto result = expand_tokens(grammar);
AssertThat(result.second, Equals((const GrammarError *)nullptr));
AssertThat(result.first.rules, Equals(rule_list({
{ "rule_A", seq({
i_sym(10),
repeat(character({ 'x' })),
i_sym(11) }) },
AssertThat(result.first.rules, Equals(eq_vector<RuleEntry>({
{
"rule_A",
seq({
i_sym(10),
repeat(character({ 'x' })),
i_sym(11),
}),
RuleEntryTypeNamed
},
})));
});
it("handles regexps containing non-ASCII UTF8 characters", [&]() {
LexicalGrammar grammar{{
// [^α-δ]
{ "rule_A", pattern("[^\u03B1-\u03B4]*") },
}, {}, {}};
{
"rule_A",
pattern("[^\u03B1-\u03B4]*"), // [^α-δ]
RuleEntryTypeNamed
},
}, {}};
auto result = expand_tokens(grammar);
AssertThat(result.first.rules, Equals(rule_list({
{ "rule_A", repeat(character({ 945, 946, 947, 948 }, false)) }
AssertThat(result.first.rules, Equals(eq_vector<RuleEntry>({
{
"rule_A",
repeat(character({ 945, 946, 947, 948 }, false)),
RuleEntryTypeNamed
}
})));
});
it("returns an error when the grammar contains an invalid regex", [&]() {
LexicalGrammar grammar{{
{ "rule_A", seq({
pattern("("),
str("xyz"),
pattern("[") }) },
}, {}, {}};
{
"rule_A",
seq({
pattern("("),
str("xyz"),
pattern("["),
}),
RuleEntryTypeNamed
},
}, {}};
auto result = expand_tokens(grammar);

487
spec/compiler/prepare_grammar/extract_tokens_spec.cc
View file

@ -1,6 +1,5 @@
#include "compiler/compiler_spec_helper.h"
#include "compiler/lexical_grammar.h"
#include "compiler/syntax_grammar.h"
#include "compiler/prepared_grammar.h"
#include "compiler/prepare_grammar/interned_grammar.h"
#include "compiler/prepare_grammar/extract_tokens.h"
#include "compiler/helpers/containers.h"
@ -12,271 +11,301 @@ using prepare_grammar::extract_tokens;
using prepare_grammar::InternedGrammar;
describe("extract_tokens", []() {
it("moves string rules into the lexical grammar", [&]() {
it("moves strings, patterns, and sub-rules marked as tokens into the lexical grammar", [&]() {
auto result = extract_tokens(InternedGrammar{{
{ "rule_A", seq({ str("ab"), i_sym(0) }) }
{
"rule_A",
repeat(seq({
str("ab"),
pattern("cd*"),
choice({
i_sym(1),
i_sym(2),
token(repeat(choice({ str("ef"), str("gh") }))),
}),
})),
},
{
"rule_B",
pattern("ij+"),
},
{
"rule_C",
choice({ str("kl"), blank() }),
},
{
"rule_D",
repeat(i_sym(3))
}
}, {}, {}});
AssertThat(get<0>(result).rules, Equals(rule_list({
{ "rule_A", seq({ i_aux_token(0), i_sym(0) }) }
SyntaxGrammar &syntax_grammar = get<0>(result);
LexicalGrammar &lexical_grammar = get<1>(result);
const GrammarError *error = get<2>(result);
AssertThat(error, Equals<const GrammarError *>(nullptr));
AssertThat(syntax_grammar.rules, Equals(eq_vector<RuleEntry>({
{
"rule_A",
repeat(seq({
// This string is now the first token in the lexical grammar.
i_token(0),
// This pattern is now the second rule in the lexical grammar.
i_token(1),
choice({
// Rule 1, which this symbol pointed to, has been moved to the
// lexical grammar.
i_token(3),
// This symbol's index has been decremented, because a previous rule
// was moved to the lexical grammar.
i_sym(1),
// This token rule is now the third rule in the lexical grammar.
i_token(2),
}),
})),
RuleEntryTypeNamed,
},
{
"rule_C",
choice({ i_token(4), blank() }),
RuleEntryTypeNamed,
},
{
"rule_D",
repeat(i_sym(2)),
RuleEntryTypeNamed,
}
})));
AssertThat(get<0>(result).aux_rules, IsEmpty())
AssertThat(get<1>(result).rules, IsEmpty())
AssertThat(get<1>(result).aux_rules, Equals(rule_list({
{ "'ab'", str("ab") },
AssertThat(lexical_grammar.rules, Equals(eq_vector<RuleEntry>({
// Strings become anonymous rules.
{
"ab",
str("ab"),
RuleEntryTypeAnonymous,
},
// Patterns become hidden rules.
{
"/cd*/",
pattern("cd*"),
RuleEntryTypeHidden,
},
// Rules marked as tokens become hidden rules.
{
"/(ef|gh)*/",
repeat(choice({ str("ef"), str("gh") })),
RuleEntryTypeHidden,
},
// This named rule was moved wholesale to the lexical grammar.
{
"rule_B",
pattern("ij+"),
RuleEntryTypeNamed,
},
// Strings become anonymous rules.
{
"kl",
str("kl"),
RuleEntryTypeAnonymous,
},
})));
});
it("moves pattern rules into the lexical grammar", [&]() {
auto result = extract_tokens(InternedGrammar{{
{ "rule_A", seq({ pattern("a+"), i_sym(0) }) }
}, {}, {}});
AssertThat(get<0>(result).rules, Equals(rule_list({
{ "rule_A", seq({ i_aux_token(0), i_sym(0) }) }
})));
AssertThat(get<0>(result).aux_rules, IsEmpty())
AssertThat(get<1>(result).rules, IsEmpty())
AssertThat(get<1>(result).aux_rules, Equals(rule_list({
{ "/a+/", pattern("a+") },
})));
});
it("moves other rules marked as tokens into the lexical grammar", [&]() {
auto result = extract_tokens(InternedGrammar{{
{ "rule_A", seq({
token(seq({ pattern("."), choice({ str("a"), str("b") }) })),
i_sym(0) }) }
}, {}, {}});
AssertThat(get<0>(result).rules, Equals(rule_list({
{ "rule_A", seq({ i_aux_token(0), i_sym(0) }) }
})));
AssertThat(get<0>(result).aux_rules, IsEmpty())
AssertThat(get<1>(result).rules, IsEmpty())
AssertThat(get<1>(result).aux_rules, Equals(rule_list({
{ "(seq /./ (choice 'a' 'b'))", token(seq({ pattern("."), choice({ str("a"), str("b") }) })) },
})));
});
it("does not move blank rules", [&]() {
auto result = extract_tokens(InternedGrammar{{
{ "rule_A", choice({ i_sym(0), blank() }) },
}, {}, {}});
AssertThat(get<0>(result).rules, Equals(rule_list({
{ "rule_A", choice({ i_sym(0), blank() }) },
})));
AssertThat(get<0>(result).aux_rules, IsEmpty())
AssertThat(get<1>(result).rules, IsEmpty())
AssertThat(get<1>(result).aux_rules, IsEmpty())
});
it("does not create duplicate tokens in the lexical grammar", [&]() {
auto result = extract_tokens(InternedGrammar{{
{ "rule_A", seq({ str("ab"), i_sym(0), str("ab") }) },
{
"rule_A",
seq({
str("ab"),
i_sym(0),
str("ab"),
})
},
}, {}, {}});
AssertThat(get<0>(result).rules, Equals(rule_list({
{ "rule_A", seq({ i_aux_token(0), i_sym(0), i_aux_token(0) }) }
})));
AssertThat(get<0>(result).aux_rules, IsEmpty())
SyntaxGrammar &syntax_grammar = get<0>(result);
LexicalGrammar &lexical_grammar = get<1>(result);
AssertThat(get<1>(result).rules, IsEmpty())
AssertThat(get<1>(result).aux_rules, Equals(rule_list({
{ "'ab'", str("ab") },
AssertThat(syntax_grammar.rules, Equals(eq_vector<RuleEntry>({
{
"rule_A",
seq({ i_token(0), i_sym(0), i_token(0) }),
RuleEntryTypeNamed
}
})));
AssertThat(lexical_grammar.rules, Equals(eq_vector<RuleEntry>({
{
"ab",
str("ab"),
RuleEntryTypeAnonymous
},
})))
});
it("updates the grammar's expected conflict symbols", [&]() {
auto result = extract_tokens(InternedGrammar{
it("does not move entire rules into the lexical grammar if their content is used elsewhere in the grammar", [&]() {
auto result = extract_tokens(InternedGrammar{{
{
{ "rule_A", str("ok") },
{ "rule_B", repeat(i_sym(0)) },
{ "rule_C", repeat(seq({ i_sym(0), i_sym(0) })) },
"rule_A",
seq({ i_sym(1), str("ab") })
},
{ str(" ") },
{ { Symbol(1), Symbol(2) } }
});
{
"rule_B",
str("cd")
},
{
"rule_C",
seq({ str("ef"), str("cd") })
},
}, {}, {}});
AssertThat(get<0>(result).rules.size(), Equals<size_t>(2));
AssertThat(get<1>(result).rules.size(), Equals<size_t>(1));
AssertThat(get<0>(result).expected_conflicts, Equals(set<set<Symbol>>({
SyntaxGrammar &syntax_grammar = get<0>(result);
LexicalGrammar &lexical_grammar = get<1>(result);
AssertThat(syntax_grammar.rules, Equals(eq_vector<RuleEntry>({
{
"rule_A",
seq({ i_sym(1), i_token(0) }),
RuleEntryTypeNamed
},
{
"rule_B",
i_token(1),
RuleEntryTypeNamed
},
{
"rule_C",
seq({ i_token(2), i_token(1) }),
RuleEntryTypeNamed
},
})));
AssertThat(lexical_grammar.rules, Equals(eq_vector<RuleEntry>({
{
"ab",
str("ab"),
RuleEntryTypeAnonymous
},
{
"cd",
str("cd"),
RuleEntryTypeAnonymous
},
{
"ef",
str("ef"),
RuleEntryTypeAnonymous
},
})));
});
it("renumbers the grammar's expected conflict symbols based on any moved rules", [&]() {
auto result = extract_tokens(InternedGrammar{{
{
"rule_A",
str("ok")
},
{
"rule_B",
repeat(i_sym(0))
},
{
"rule_C",
repeat(seq({ i_sym(0), i_sym(0) }))
},
}, { str(" ") }, { { Symbol(1), Symbol(2) } }});
SyntaxGrammar &syntax_grammar = get<0>(result);
AssertThat(syntax_grammar.rules.size(), Equals<size_t>(2));
AssertThat(syntax_grammar.expected_conflicts, Equals(set<set<Symbol>>({
{ Symbol(0), Symbol(1) },
})));
});
describe("when an entire grammar rule is a token", [&]() {
it("moves the rule the lexical grammar and updates referencing symbols", [&]() {
auto result = extract_tokens(InternedGrammar{{
{ "rule_A", i_sym(1) },
{ "rule_B", pattern("a|b") },
{ "rule_C", token(seq({ str("a"), str("b") })) },
}, {}, {}});
AssertThat(get<0>(result).rules, Equals(rule_list({
{ "rule_A", i_token(0) }
})));
AssertThat(get<0>(result).aux_rules, IsEmpty());
AssertThat(get<1>(result).rules, Equals(rule_list({
{ "rule_B", pattern("a|b") },
{ "rule_C", token(seq({ str("a"), str("b") })) },
})));
// TODO put back
// AssertThat(get<1>(result).aux_rules, IsEmpty());
});
it("updates symbols whose indices need to change due to deleted rules", [&]() {
auto result = extract_tokens(InternedGrammar{{
{ "rule_A", str("ab") },
{ "rule_B", i_sym(0) },
{ "rule_C", i_sym(1) },
}, {}, {}});
AssertThat(get<0>(result).rules, Equals(rule_list({
{ "rule_B", i_token(0) },
{ "rule_C", i_sym(0) },
})));
AssertThat(get<0>(result).aux_rules, IsEmpty());
AssertThat(get<1>(result).rules, Equals(rule_list({
{ "rule_A", str("ab") },
})));
// TODO put back
// AssertThat(get<1>(result).aux_rules, IsEmpty());
});
it("does not move the rule if its content is used elsewhere in the grammar", [&]() {
auto result = extract_tokens(InternedGrammar{{
{ "rule_A", seq({ i_sym(1), str("ab") }) },
{ "rule_B", str("cd") },
{ "rule_C", seq({ str("ef"), str("cd") }) },
}, {}, {}});
AssertThat(get<0>(result).rules, Equals(rule_list({
{ "rule_A", seq({ i_sym(1), i_aux_token(0) }) },
{ "rule_B", i_aux_token(1) },
{ "rule_C", seq({ i_aux_token(2), i_aux_token(1) }) },
})));
AssertThat(get<0>(result).aux_rules, IsEmpty());
AssertThat(get<1>(result).rules, IsEmpty())
AssertThat(get<1>(result).aux_rules, Equals(rule_list({
{ "'ab'", str("ab") },
{ "'cd'", str("cd") },
{ "'ef'", str("ef") },
})));
});
});
describe("handling ubiquitous tokens", [&]() {
describe("ubiquitous tokens that are not symbols", [&]() {
it("adds them to the lexical grammar's separators", [&]() {
auto result = extract_tokens(InternedGrammar{{
{ "rule_A", str("x") },
}, {
pattern("\\s+"),
str("y"),
}, {}});
it("adds inline ubiquitous tokens to the lexical grammar's separators", [&]() {
auto result = extract_tokens(InternedGrammar{{
{ "rule_A", str("x") },
}, {
pattern("\\s+"),
str("y"),
}, {}});
AssertThat(get<2>(result), Equals<const GrammarError *>(nullptr));
AssertThat(get<2>(result), Equals<const GrammarError *>(nullptr));
AssertThat(get<1>(result).separators, Equals(rule_vector({
pattern("\\s+"),
str("y"),
})));
AssertThat(get<1>(result).separators, Equals(rule_vector({
pattern("\\s+"),
str("y"),
})));
AssertThat(get<0>(result).ubiquitous_tokens, IsEmpty());
});
AssertThat(get<0>(result).ubiquitous_tokens, IsEmpty());
});
describe("ubiquitous tokens that point to moved rules", [&]() {
it("updates them according to the new symbol numbers", [&]() {
auto result = extract_tokens(InternedGrammar{ {
{ "rule_A", seq({ str("w"), i_sym(1) }) },
{ "rule_B", str("x") },
{ "rule_C", str("y") },
}, {
i_sym(2),
}, {}});
it("updates ubiquitous symbols according to the new symbol numbers", [&]() {
auto result = extract_tokens(InternedGrammar{ {
{ "rule_A", seq({ str("w"), str("x"), i_sym(1) }) },
{ "rule_B", str("y") },
{ "rule_C", str("z") },
}, {
i_sym(2),
}, {}});
AssertThat(get<2>(result), Equals<const GrammarError *>(nullptr));
AssertThat(get<2>(result), Equals<const GrammarError *>(nullptr));
AssertThat(get<0>(result).ubiquitous_tokens, Equals(set<Symbol>({
{ Symbol(1, SymbolOptionToken) },
})));
AssertThat(get<0>(result).ubiquitous_tokens, Equals(set<Symbol>({
{ Symbol(3, true) },
})));
AssertThat(get<1>(result).separators, IsEmpty());
});
AssertThat(get<1>(result).separators, IsEmpty());
});
describe("ubiquitous tokens that are visible", [&]() {
it("preserves them in the syntactic grammar", [&]() {
auto result = extract_tokens(InternedGrammar{{
{ "rule_A", str("ab") },
{ "rule_B", str("bc") },
}, { i_sym(1) }, {}});
it("returns an error if any ubiquitous tokens are non-token symbols", [&]() {
auto result = extract_tokens(InternedGrammar{{
{
"rule_A",
seq({ str("x"), i_sym(1) }),
},
{
"rule_B",
seq({ str("y"), str("z") })
},
}, { i_sym(1) }, {}});
AssertThat(get<2>(result), Equals<const GrammarError *>(nullptr));
AssertThat(get<0>(result).ubiquitous_tokens, Equals(set<Symbol>({
Symbol(1, SymbolOptionToken)
})));
AssertThat(get<1>(result).separators, IsEmpty());
});
AssertThat(get<2>(result), !Equals<const GrammarError *>(nullptr));
AssertThat(get<2>(result), EqualsPointer(
new GrammarError(GrammarErrorTypeInvalidUbiquitousToken,
"Not a token: rule_B")));
});
describe("ubiquitous tokens that are used in other grammar rules", [&]() {
it("preserves them in the syntactic grammar", [&]() {
auto result = extract_tokens(InternedGrammar{{
{ "rule_A", seq({ i_sym(1), str("ab") }) },
{ "_rule_B", str("bc") },
}, { i_sym(1) }, {}});
it("returns an error if any ubiquitous tokens are non-token rules", [&]() {
auto result = extract_tokens(InternedGrammar{{
{
"rule_A",
str("x")
},
{
"rule_B",
str("y")
},
}, { choice({ i_sym(1), blank() }) }, {}});
AssertThat(get<2>(result), Equals<const GrammarError *>(nullptr));
AssertThat(get<0>(result).ubiquitous_tokens, Equals(set<Symbol>({
Symbol(0, SymbolOptionToken),
})));
AssertThat(get<1>(result).separators, IsEmpty());
});
});
describe("ubiquitous tokens that are non-token symbols", [&]() {
it("returns an error", [&]() {
auto result = extract_tokens(InternedGrammar{{
{ "rule_A", seq({ str("x"), i_sym(1) }), },
{ "rule_B", seq({ str("y"), str("z") }) },
}, { i_sym(1) }, {}});
AssertThat(get<2>(result), !Equals<const GrammarError *>(nullptr));
AssertThat(get<2>(result), EqualsPointer(
new GrammarError(GrammarErrorTypeInvalidUbiquitousToken,
"Not a token: rule_B")));
});
});
describe("ubiquitous tokens that are not symbols", [&]() {
it("returns an error", [&]() {
auto result = extract_tokens(InternedGrammar{{
{ "rule_A", str("x") },
{ "rule_B", str("y") },
}, { choice({ i_sym(1), blank() }) }, {}});
AssertThat(get<2>(result), !Equals<const GrammarError *>(nullptr));
AssertThat(get<2>(result), EqualsPointer(
new GrammarError(GrammarErrorTypeInvalidUbiquitousToken,
"Not a token: (choice (sym 1) (blank))")));
});
AssertThat(get<2>(result), !Equals<const GrammarError *>(nullptr));
AssertThat(get<2>(result), EqualsPointer(
new GrammarError(GrammarErrorTypeInvalidUbiquitousToken,
"Not a token: (choice (sym 1) (blank))")));
});
});
});

274
spec/fixtures/parsers/arithmetic.c vendored
View file

@ -12,16 +12,16 @@ enum {
sym_quotient,
sym_exponent,
sym_group,
aux_sym_PLUS,
aux_sym_DASH,
aux_sym_STAR,
aux_sym_SLASH,
aux_sym_CARET,
aux_sym_LPAREN,
aux_sym_RPAREN,
sym_number,
sym_variable,
sym_comment,
aux_sym_STR_PLUS,
aux_sym_STR_DASH,
aux_sym_STR_STAR,
aux_sym_STR_SLASH,
aux_sym_STR_CARET,
aux_sym_STR_LPAREN,
aux_sym_STR_RPAREN,
};
static const char *ts_symbol_names[] = {
@ -35,27 +35,27 @@ static const char *ts_symbol_names[] = {
[sym_group] = "group",
[ts_builtin_sym_error] = "ERROR",
[ts_builtin_sym_end] = "END",
[aux_sym_PLUS] = "+",
[aux_sym_DASH] = "-",
[aux_sym_STAR] = "*",
[aux_sym_SLASH] = "/",
[aux_sym_CARET] = "^",
[aux_sym_LPAREN] = "(",
[aux_sym_RPAREN] = ")",
[sym_number] = "number",
[sym_variable] = "variable",
[sym_comment] = "comment",
[aux_sym_STR_PLUS] = "STR_+",
[aux_sym_STR_DASH] = "STR_-",
[aux_sym_STR_STAR] = "STR_*",
[aux_sym_STR_SLASH] = "STR_/",
[aux_sym_STR_CARET] = "STR_^",
[aux_sym_STR_LPAREN] = "STR_(",
[aux_sym_STR_RPAREN] = "STR_)",
};
static const int ts_hidden_symbol_flags[SYMBOL_COUNT] = {
[sym__expression] = 1,
[aux_sym_STR_PLUS] = 1,
[aux_sym_STR_DASH] = 1,
[aux_sym_STR_STAR] = 1,
[aux_sym_STR_SLASH] = 1,
[aux_sym_STR_CARET] = 1,
[aux_sym_STR_LPAREN] = 1,
[aux_sym_STR_RPAREN] = 1,
[aux_sym_PLUS] = 1,
[aux_sym_DASH] = 1,
[aux_sym_STAR] = 1,
[aux_sym_SLASH] = 1,
[aux_sym_CARET] = 1,
[aux_sym_LPAREN] = 1,
[aux_sym_RPAREN] = 1,
};
static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) {
@ -85,7 +85,7 @@ static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) {
ADVANCE(2);
ACCEPT_TOKEN(sym_comment);
case 3:
ACCEPT_TOKEN(aux_sym_STR_LPAREN);
ACCEPT_TOKEN(aux_sym_LPAREN);
case 4:
if ('0' <= lookahead && lookahead <= '9')
ADVANCE(4);
@ -139,15 +139,15 @@ static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) {
ADVANCE(14);
LEX_ERROR();
case 10:
ACCEPT_TOKEN(aux_sym_STR_STAR);
ACCEPT_TOKEN(aux_sym_STAR);
case 11:
ACCEPT_TOKEN(aux_sym_STR_PLUS);
ACCEPT_TOKEN(aux_sym_PLUS);
case 12:
ACCEPT_TOKEN(aux_sym_STR_DASH);
ACCEPT_TOKEN(aux_sym_DASH);
case 13:
ACCEPT_TOKEN(aux_sym_STR_SLASH);
ACCEPT_TOKEN(aux_sym_SLASH);
case 14:
ACCEPT_TOKEN(aux_sym_STR_CARET);
ACCEPT_TOKEN(aux_sym_CARET);
case 15:
START_TOKEN();
if ((lookahead == '\t') ||
@ -171,7 +171,7 @@ static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) {
ADVANCE(14);
LEX_ERROR();
case 16:
ACCEPT_TOKEN(aux_sym_STR_RPAREN);
ACCEPT_TOKEN(aux_sym_RPAREN);
case 17:
START_TOKEN();
if ((lookahead == '\t') ||
@ -302,10 +302,10 @@ static const TSParseAction *ts_parse_actions[STATE_COUNT][SYMBOL_COUNT] = {
[sym_quotient] = ACTIONS(SHIFT(3)),
[sym_exponent] = ACTIONS(SHIFT(3)),
[sym_group] = ACTIONS(SHIFT(3)),
[aux_sym_LPAREN] = ACTIONS(SHIFT(4)),
[sym_number] = ACTIONS(SHIFT(3)),
[sym_variable] = ACTIONS(SHIFT(3)),
[sym_comment] = ACTIONS(SHIFT_EXTRA()),
[aux_sym_STR_LPAREN] = ACTIONS(SHIFT(4)),
},
[1] = {
[ts_builtin_sym_end] = ACTIONS(ACCEPT_INPUT()),
@ -313,21 +313,21 @@ static const TSParseAction *ts_parse_actions[STATE_COUNT][SYMBOL_COUNT] = {
},
[2] = {
[ts_builtin_sym_end] = ACTIONS(REDUCE(sym_program, 1)),
[aux_sym_PLUS] = ACTIONS(SHIFT(23)),
[aux_sym_DASH] = ACTIONS(SHIFT(24)),
[aux_sym_STAR] = ACTIONS(SHIFT(25)),
[aux_sym_SLASH] = ACTIONS(SHIFT(26)),
[aux_sym_CARET] = ACTIONS(SHIFT(27)),
[sym_comment] = ACTIONS(SHIFT_EXTRA()),
[aux_sym_STR_PLUS] = ACTIONS(SHIFT(23)),
[aux_sym_STR_DASH] = ACTIONS(SHIFT(24)),
[aux_sym_STR_STAR] = ACTIONS(SHIFT(25)),
[aux_sym_STR_SLASH] = ACTIONS(SHIFT(26)),
[aux_sym_STR_CARET] = ACTIONS(SHIFT(27)),
},
[3] = {
[ts_builtin_sym_end] = ACTIONS(REDUCE(sym__expression, 1)),
[aux_sym_PLUS] = ACTIONS(REDUCE(sym__expression, 1)),
[aux_sym_DASH] = ACTIONS(REDUCE(sym__expression, 1)),
[aux_sym_STAR] = ACTIONS(REDUCE(sym__expression, 1)),
[aux_sym_SLASH] = ACTIONS(REDUCE(sym__expression, 1)),
[aux_sym_CARET] = ACTIONS(REDUCE(sym__expression, 1)),
[sym_comment] = ACTIONS(SHIFT_EXTRA()),
[aux_sym_STR_PLUS] = ACTIONS(REDUCE(sym__expression, 1)),
[aux_sym_STR_DASH] = ACTIONS(REDUCE(sym__expression, 1)),
[aux_sym_STR_STAR] = ACTIONS(REDUCE(sym__expression, 1)),
[aux_sym_STR_SLASH] = ACTIONS(REDUCE(sym__expression, 1)),
[aux_sym_STR_CARET] = ACTIONS(REDUCE(sym__expression, 1)),
},
[4] = {
[sym__expression] = ACTIONS(SHIFT(5)),
@ -338,32 +338,32 @@ static const TSParseAction *ts_parse_actions[STATE_COUNT][SYMBOL_COUNT] = {
[sym_exponent] = ACTIONS(SHIFT(6)),
[sym_group] = ACTIONS(SHIFT(6)),
[ts_builtin_sym_error] = ACTIONS(SHIFT(7)),
[aux_sym_LPAREN] = ACTIONS(SHIFT(8)),
[sym_number] = ACTIONS(SHIFT(6)),
[sym_variable] = ACTIONS(SHIFT(6)),
[sym_comment] = ACTIONS(SHIFT_EXTRA()),
[aux_sym_STR_LPAREN] = ACTIONS(SHIFT(8)),
},
[5] = {
[aux_sym_PLUS] = ACTIONS(SHIFT(12)),
[aux_sym_DASH] = ACTIONS(SHIFT(13)),
[aux_sym_STAR] = ACTIONS(SHIFT(14)),
[aux_sym_SLASH] = ACTIONS(SHIFT(15)),
[aux_sym_CARET] = ACTIONS(SHIFT(16)),
[aux_sym_RPAREN] = ACTIONS(SHIFT(22)),
[sym_comment] = ACTIONS(SHIFT_EXTRA()),
[aux_sym_STR_PLUS] = ACTIONS(SHIFT(12)),
[aux_sym_STR_DASH] = ACTIONS(SHIFT(13)),
[aux_sym_STR_STAR] = ACTIONS(SHIFT(14)),
[aux_sym_STR_SLASH] = ACTIONS(SHIFT(15)),
[aux_sym_STR_CARET] = ACTIONS(SHIFT(16)),
[aux_sym_STR_RPAREN] = ACTIONS(SHIFT(22)),
},
[6] = {
[aux_sym_PLUS] = ACTIONS(REDUCE(sym__expression, 1)),
[aux_sym_DASH] = ACTIONS(REDUCE(sym__expression, 1)),
[aux_sym_STAR] = ACTIONS(REDUCE(sym__expression, 1)),
[aux_sym_SLASH] = ACTIONS(REDUCE(sym__expression, 1)),
[aux_sym_CARET] = ACTIONS(REDUCE(sym__expression, 1)),
[aux_sym_RPAREN] = ACTIONS(REDUCE(sym__expression, 1)),
[sym_comment] = ACTIONS(SHIFT_EXTRA()),
[aux_sym_STR_PLUS] = ACTIONS(REDUCE(sym__expression, 1)),
[aux_sym_STR_DASH] = ACTIONS(REDUCE(sym__expression, 1)),
[aux_sym_STR_STAR] = ACTIONS(REDUCE(sym__expression, 1)),
[aux_sym_STR_SLASH] = ACTIONS(REDUCE(sym__expression, 1)),
[aux_sym_STR_CARET] = ACTIONS(REDUCE(sym__expression, 1)),
[aux_sym_STR_RPAREN] = ACTIONS(REDUCE(sym__expression, 1)),
},
[7] = {
[aux_sym_RPAREN] = ACTIONS(SHIFT(22)),
[sym_comment] = ACTIONS(SHIFT_EXTRA()),
[aux_sym_STR_RPAREN] = ACTIONS(SHIFT(22)),
},
[8] = {
[sym__expression] = ACTIONS(SHIFT(9)),
@ -374,32 +374,32 @@ static const TSParseAction *ts_parse_actions[STATE_COUNT][SYMBOL_COUNT] = {
[sym_exponent] = ACTIONS(SHIFT(6)),
[sym_group] = ACTIONS(SHIFT(6)),
[ts_builtin_sym_error] = ACTIONS(SHIFT(10)),
[aux_sym_LPAREN] = ACTIONS(SHIFT(8)),
[sym_number] = ACTIONS(SHIFT(6)),
[sym_variable] = ACTIONS(SHIFT(6)),
[sym_comment] = ACTIONS(SHIFT_EXTRA()),
[aux_sym_STR_LPAREN] = ACTIONS(SHIFT(8)),
},
[9] = {
[aux_sym_PLUS] = ACTIONS(SHIFT(12)),
[aux_sym_DASH] = ACTIONS(SHIFT(13)),
[aux_sym_STAR] = ACTIONS(SHIFT(14)),
[aux_sym_SLASH] = ACTIONS(SHIFT(15)),
[aux_sym_CARET] = ACTIONS(SHIFT(16)),
[aux_sym_RPAREN] = ACTIONS(SHIFT(11)),
[sym_comment] = ACTIONS(SHIFT_EXTRA()),
[aux_sym_STR_PLUS] = ACTIONS(SHIFT(12)),
[aux_sym_STR_DASH] = ACTIONS(SHIFT(13)),
[aux_sym_STR_STAR] = ACTIONS(SHIFT(14)),
[aux_sym_STR_SLASH] = ACTIONS(SHIFT(15)),
[aux_sym_STR_CARET] = ACTIONS(SHIFT(16)),
[aux_sym_STR_RPAREN] = ACTIONS(SHIFT(11)),
},
[10] = {
[aux_sym_RPAREN] = ACTIONS(SHIFT(11)),
[sym_comment] = ACTIONS(SHIFT_EXTRA()),
[aux_sym_STR_RPAREN] = ACTIONS(SHIFT(11)),
},
[11] = {
[aux_sym_PLUS] = ACTIONS(REDUCE(sym_group, 3)),
[aux_sym_DASH] = ACTIONS(REDUCE(sym_group, 3)),
[aux_sym_STAR] = ACTIONS(REDUCE(sym_group, 3)),
[aux_sym_SLASH] = ACTIONS(REDUCE(sym_group, 3)),
[aux_sym_CARET] = ACTIONS(REDUCE(sym_group, 3)),
[aux_sym_RPAREN] = ACTIONS(REDUCE(sym_group, 3)),
[sym_comment] = ACTIONS(SHIFT_EXTRA()),
[aux_sym_STR_PLUS] = ACTIONS(REDUCE(sym_group, 3)),
[aux_sym_STR_DASH] = ACTIONS(REDUCE(sym_group, 3)),
[aux_sym_STR_STAR] = ACTIONS(REDUCE(sym_group, 3)),
[aux_sym_STR_SLASH] = ACTIONS(REDUCE(sym_group, 3)),
[aux_sym_STR_CARET] = ACTIONS(REDUCE(sym_group, 3)),
[aux_sym_STR_RPAREN] = ACTIONS(REDUCE(sym_group, 3)),
},
[12] = {
[sym__expression] = ACTIONS(SHIFT(21)),
@ -409,10 +409,10 @@ static const TSParseAction *ts_parse_actions[STATE_COUNT][SYMBOL_COUNT] = {
[sym_quotient] = ACTIONS(SHIFT(6)),
[sym_exponent] = ACTIONS(SHIFT(6)),
[sym_group] = ACTIONS(SHIFT(6)),
[aux_sym_LPAREN] = ACTIONS(SHIFT(8)),
[sym_number] = ACTIONS(SHIFT(6)),
[sym_variable] = ACTIONS(SHIFT(6)),
[sym_comment] = ACTIONS(SHIFT_EXTRA()),
[aux_sym_STR_LPAREN] = ACTIONS(SHIFT(8)),
},
[13] = {
[sym__expression] = ACTIONS(SHIFT(20)),
@ -422,10 +422,10 @@ static const TSParseAction *ts_parse_actions[STATE_COUNT][SYMBOL_COUNT] = {
[sym_quotient] = ACTIONS(SHIFT(6)),
[sym_exponent] = ACTIONS(SHIFT(6)),
[sym_group] = ACTIONS(SHIFT(6)),
[aux_sym_LPAREN] = ACTIONS(SHIFT(8)),
[sym_number] = ACTIONS(SHIFT(6)),
[sym_variable] = ACTIONS(SHIFT(6)),
[sym_comment] = ACTIONS(SHIFT_EXTRA()),
[aux_sym_STR_LPAREN] = ACTIONS(SHIFT(8)),
},
[14] = {
[sym__expression] = ACTIONS(SHIFT(19)),
@ -435,10 +435,10 @@ static const TSParseAction *ts_parse_actions[STATE_COUNT][SYMBOL_COUNT] = {
[sym_quotient] = ACTIONS(SHIFT(6)),
[sym_exponent] = ACTIONS(SHIFT(6)),
[sym_group] = ACTIONS(SHIFT(6)),
[aux_sym_LPAREN] = ACTIONS(SHIFT(8)),
[sym_number] = ACTIONS(SHIFT(6)),
[sym_variable] = ACTIONS(SHIFT(6)),
[sym_comment] = ACTIONS(SHIFT_EXTRA()),
[aux_sym_STR_LPAREN] = ACTIONS(SHIFT(8)),
},
[15] = {
[sym__expression] = ACTIONS(SHIFT(18)),
@ -448,10 +448,10 @@ static const TSParseAction *ts_parse_actions[STATE_COUNT][SYMBOL_COUNT] = {
[sym_quotient] = ACTIONS(SHIFT(6)),
[sym_exponent] = ACTIONS(SHIFT(6)),
[sym_group] = ACTIONS(SHIFT(6)),
[aux_sym_LPAREN] = ACTIONS(SHIFT(8)),
[sym_number] = ACTIONS(SHIFT(6)),
[sym_variable] = ACTIONS(SHIFT(6)),
[sym_comment] = ACTIONS(SHIFT_EXTRA()),
[aux_sym_STR_LPAREN] = ACTIONS(SHIFT(8)),
},
[16] = {
[sym__expression] = ACTIONS(SHIFT(17)),
@ -461,64 +461,64 @@ static const TSParseAction *ts_parse_actions[STATE_COUNT][SYMBOL_COUNT] = {
[sym_quotient] = ACTIONS(SHIFT(6)),
[sym_exponent] = ACTIONS(SHIFT(6)),
[sym_group] = ACTIONS(SHIFT(6)),
[aux_sym_LPAREN] = ACTIONS(SHIFT(8)),
[sym_number] = ACTIONS(SHIFT(6)),
[sym_variable] = ACTIONS(SHIFT(6)),
[sym_comment] = ACTIONS(SHIFT_EXTRA()),
[aux_sym_STR_LPAREN] = ACTIONS(SHIFT(8)),
},
[17] = {
[aux_sym_PLUS] = ACTIONS(REDUCE_FRAGILE(sym_exponent, 3)),
[aux_sym_DASH] = ACTIONS(REDUCE_FRAGILE(sym_exponent, 3)),
[aux_sym_STAR] = ACTIONS(REDUCE_FRAGILE(sym_exponent, 3)),
[aux_sym_SLASH] = ACTIONS(REDUCE_FRAGILE(sym_exponent, 3)),
[aux_sym_CARET] = ACTIONS(REDUCE_FRAGILE(sym_exponent, 3)),
[aux_sym_RPAREN] = ACTIONS(REDUCE_FRAGILE(sym_exponent, 3)),
[sym_comment] = ACTIONS(SHIFT_EXTRA()),
[aux_sym_STR_PLUS] = ACTIONS(REDUCE_FRAGILE(sym_exponent, 3)),
[aux_sym_STR_DASH] = ACTIONS(REDUCE_FRAGILE(sym_exponent, 3)),
[aux_sym_STR_STAR] = ACTIONS(REDUCE_FRAGILE(sym_exponent, 3)),
[aux_sym_STR_SLASH] = ACTIONS(REDUCE_FRAGILE(sym_exponent, 3)),
[aux_sym_STR_CARET] = ACTIONS(REDUCE_FRAGILE(sym_exponent, 3)),
[aux_sym_STR_RPAREN] = ACTIONS(REDUCE_FRAGILE(sym_exponent, 3)),
},
[18] = {
[aux_sym_PLUS] = ACTIONS(REDUCE_FRAGILE(sym_quotient, 3)),
[aux_sym_DASH] = ACTIONS(REDUCE_FRAGILE(sym_quotient, 3)),
[aux_sym_STAR] = ACTIONS(REDUCE_FRAGILE(sym_quotient, 3)),
[aux_sym_SLASH] = ACTIONS(REDUCE_FRAGILE(sym_quotient, 3)),
[aux_sym_CARET] = ACTIONS(SHIFT(16)),
[aux_sym_RPAREN] = ACTIONS(REDUCE_FRAGILE(sym_quotient, 3)),
[sym_comment] = ACTIONS(SHIFT_EXTRA()),
[aux_sym_STR_PLUS] = ACTIONS(REDUCE_FRAGILE(sym_quotient, 3)),
[aux_sym_STR_DASH] = ACTIONS(REDUCE_FRAGILE(sym_quotient, 3)),
[aux_sym_STR_STAR] = ACTIONS(REDUCE_FRAGILE(sym_quotient, 3)),
[aux_sym_STR_SLASH] = ACTIONS(REDUCE_FRAGILE(sym_quotient, 3)),
[aux_sym_STR_CARET] = ACTIONS(SHIFT(16)),
[aux_sym_STR_RPAREN] = ACTIONS(REDUCE_FRAGILE(sym_quotient, 3)),
},
[19] = {
[aux_sym_PLUS] = ACTIONS(REDUCE_FRAGILE(sym_product, 3)),
[aux_sym_DASH] = ACTIONS(REDUCE_FRAGILE(sym_product, 3)),
[aux_sym_STAR] = ACTIONS(REDUCE_FRAGILE(sym_product, 3)),
[aux_sym_SLASH] = ACTIONS(REDUCE_FRAGILE(sym_product, 3)),
[aux_sym_CARET] = ACTIONS(SHIFT(16)),
[aux_sym_RPAREN] = ACTIONS(REDUCE_FRAGILE(sym_product, 3)),
[sym_comment] = ACTIONS(SHIFT_EXTRA()),
[aux_sym_STR_PLUS] = ACTIONS(REDUCE_FRAGILE(sym_product, 3)),
[aux_sym_STR_DASH] = ACTIONS(REDUCE_FRAGILE(sym_product, 3)),
[aux_sym_STR_STAR] = ACTIONS(REDUCE_FRAGILE(sym_product, 3)),
[aux_sym_STR_SLASH] = ACTIONS(REDUCE_FRAGILE(sym_product, 3)),
[aux_sym_STR_CARET] = ACTIONS(SHIFT(16)),
[aux_sym_STR_RPAREN] = ACTIONS(REDUCE_FRAGILE(sym_product, 3)),
},
[20] = {
[aux_sym_PLUS] = ACTIONS(REDUCE_FRAGILE(sym_difference, 3)),
[aux_sym_DASH] = ACTIONS(REDUCE_FRAGILE(sym_difference, 3)),
[aux_sym_STAR] = ACTIONS(SHIFT(14)),
[aux_sym_SLASH] = ACTIONS(SHIFT(15)),
[aux_sym_CARET] = ACTIONS(SHIFT(16)),
[aux_sym_RPAREN] = ACTIONS(REDUCE_FRAGILE(sym_difference, 3)),
[sym_comment] = ACTIONS(SHIFT_EXTRA()),
[aux_sym_STR_PLUS] = ACTIONS(REDUCE_FRAGILE(sym_difference, 3)),
[aux_sym_STR_DASH] = ACTIONS(REDUCE_FRAGILE(sym_difference, 3)),
[aux_sym_STR_STAR] = ACTIONS(SHIFT(14)),
[aux_sym_STR_SLASH] = ACTIONS(SHIFT(15)),
[aux_sym_STR_CARET] = ACTIONS(SHIFT(16)),
[aux_sym_STR_RPAREN] = ACTIONS(REDUCE_FRAGILE(sym_difference, 3)),
},
[21] = {
[aux_sym_PLUS] = ACTIONS(REDUCE_FRAGILE(sym_sum, 3)),
[aux_sym_DASH] = ACTIONS(REDUCE_FRAGILE(sym_sum, 3)),
[aux_sym_STAR] = ACTIONS(SHIFT(14)),
[aux_sym_SLASH] = ACTIONS(SHIFT(15)),
[aux_sym_CARET] = ACTIONS(SHIFT(16)),
[aux_sym_RPAREN] = ACTIONS(REDUCE_FRAGILE(sym_sum, 3)),
[sym_comment] = ACTIONS(SHIFT_EXTRA()),
[aux_sym_STR_PLUS] = ACTIONS(REDUCE_FRAGILE(sym_sum, 3)),
[aux_sym_STR_DASH] = ACTIONS(REDUCE_FRAGILE(sym_sum, 3)),
[aux_sym_STR_STAR] = ACTIONS(SHIFT(14)),
[aux_sym_STR_SLASH] = ACTIONS(SHIFT(15)),
[aux_sym_STR_CARET] = ACTIONS(SHIFT(16)),
[aux_sym_STR_RPAREN] = ACTIONS(REDUCE_FRAGILE(sym_sum, 3)),
},
[22] = {
[ts_builtin_sym_end] = ACTIONS(REDUCE(sym_group, 3)),
[aux_sym_PLUS] = ACTIONS(REDUCE(sym_group, 3)),
[aux_sym_DASH] = ACTIONS(REDUCE(sym_group, 3)),
[aux_sym_STAR] = ACTIONS(REDUCE(sym_group, 3)),
[aux_sym_SLASH] = ACTIONS(REDUCE(sym_group, 3)),
[aux_sym_CARET] = ACTIONS(REDUCE(sym_group, 3)),
[sym_comment] = ACTIONS(SHIFT_EXTRA()),
[aux_sym_STR_PLUS] = ACTIONS(REDUCE(sym_group, 3)),
[aux_sym_STR_DASH] = ACTIONS(REDUCE(sym_group, 3)),
[aux_sym_STR_STAR] = ACTIONS(REDUCE(sym_group, 3)),
[aux_sym_STR_SLASH] = ACTIONS(REDUCE(sym_group, 3)),
[aux_sym_STR_CARET] = ACTIONS(REDUCE(sym_group, 3)),
},
[23] = {
[sym__expression] = ACTIONS(SHIFT(32)),
@ -528,10 +528,10 @@ static const TSParseAction *ts_parse_actions[STATE_COUNT][SYMBOL_COUNT] = {
[sym_quotient] = ACTIONS(SHIFT(3)),
[sym_exponent] = ACTIONS(SHIFT(3)),
[sym_group] = ACTIONS(SHIFT(3)),
[aux_sym_LPAREN] = ACTIONS(SHIFT(4)),
[sym_number] = ACTIONS(SHIFT(3)),
[sym_variable] = ACTIONS(SHIFT(3)),
[sym_comment] = ACTIONS(SHIFT_EXTRA()),
[aux_sym_STR_LPAREN] = ACTIONS(SHIFT(4)),
},
[24] = {
[sym__expression] = ACTIONS(SHIFT(31)),
@ -541,10 +541,10 @@ static const TSParseAction *ts_parse_actions[STATE_COUNT][SYMBOL_COUNT] = {
[sym_quotient] = ACTIONS(SHIFT(3)),
[sym_exponent] = ACTIONS(SHIFT(3)),
[sym_group] = ACTIONS(SHIFT(3)),
[aux_sym_LPAREN] = ACTIONS(SHIFT(4)),
[sym_number] = ACTIONS(SHIFT(3)),
[sym_variable] = ACTIONS(SHIFT(3)),
[sym_comment] = ACTIONS(SHIFT_EXTRA()),
[aux_sym_STR_LPAREN] = ACTIONS(SHIFT(4)),
},
[25] = {
[sym__expression] = ACTIONS(SHIFT(30)),
@ -554,10 +554,10 @@ static const TSParseAction *ts_parse_actions[STATE_COUNT][SYMBOL_COUNT] = {
[sym_quotient] = ACTIONS(SHIFT(3)),
[sym_exponent] = ACTIONS(SHIFT(3)),
[sym_group] = ACTIONS(SHIFT(3)),
[aux_sym_LPAREN] = ACTIONS(SHIFT(4)),
[sym_number] = ACTIONS(SHIFT(3)),
[sym_variable] = ACTIONS(SHIFT(3)),
[sym_comment] = ACTIONS(SHIFT_EXTRA()),
[aux_sym_STR_LPAREN] = ACTIONS(SHIFT(4)),
},
[26] = {
[sym__expression] = ACTIONS(SHIFT(29)),
@ -567,10 +567,10 @@ static const TSParseAction *ts_parse_actions[STATE_COUNT][SYMBOL_COUNT] = {
[sym_quotient] = ACTIONS(SHIFT(3)),
[sym_exponent] = ACTIONS(SHIFT(3)),
[sym_group] = ACTIONS(SHIFT(3)),
[aux_sym_LPAREN] = ACTIONS(SHIFT(4)),
[sym_number] = ACTIONS(SHIFT(3)),
[sym_variable] = ACTIONS(SHIFT(3)),
[sym_comment] = ACTIONS(SHIFT_EXTRA()),
[aux_sym_STR_LPAREN] = ACTIONS(SHIFT(4)),
},
[27] = {
[sym__expression] = ACTIONS(SHIFT(28)),
@ -580,55 +580,55 @@ static const TSParseAction *ts_parse_actions[STATE_COUNT][SYMBOL_COUNT] = {
[sym_quotient] = ACTIONS(SHIFT(3)),
[sym_exponent] = ACTIONS(SHIFT(3)),
[sym_group] = ACTIONS(SHIFT(3)),
[aux_sym_LPAREN] = ACTIONS(SHIFT(4)),
[sym_number] = ACTIONS(SHIFT(3)),
[sym_variable] = ACTIONS(SHIFT(3)),
[sym_comment] = ACTIONS(SHIFT_EXTRA()),
[aux_sym_STR_LPAREN] = ACTIONS(SHIFT(4)),
},
[28] = {
[ts_builtin_sym_end] = ACTIONS(REDUCE_FRAGILE(sym_exponent, 3)),
[aux_sym_PLUS] = ACTIONS(REDUCE_FRAGILE(sym_exponent, 3)),
[aux_sym_DASH] = ACTIONS(REDUCE_FRAGILE(sym_exponent, 3)),
[aux_sym_STAR] = ACTIONS(REDUCE_FRAGILE(sym_exponent, 3)),
[aux_sym_SLASH] = ACTIONS(REDUCE_FRAGILE(sym_exponent, 3)),
[aux_sym_CARET] = ACTIONS(REDUCE_FRAGILE(sym_exponent, 3)),
[sym_comment] = ACTIONS(SHIFT_EXTRA()),
[aux_sym_STR_PLUS] = ACTIONS(REDUCE_FRAGILE(sym_exponent, 3)),
[aux_sym_STR_DASH] = ACTIONS(REDUCE_FRAGILE(sym_exponent, 3)),
[aux_sym_STR_STAR] = ACTIONS(REDUCE_FRAGILE(sym_exponent, 3)),
[aux_sym_STR_SLASH] = ACTIONS(REDUCE_FRAGILE(sym_exponent, 3)),
[aux_sym_STR_CARET] = ACTIONS(REDUCE_FRAGILE(sym_exponent, 3)),
},
[29] = {
[ts_builtin_sym_end] = ACTIONS(REDUCE_FRAGILE(sym_quotient, 3)),
[aux_sym_PLUS] = ACTIONS(REDUCE_FRAGILE(sym_quotient, 3)),
[aux_sym_DASH] = ACTIONS(REDUCE_FRAGILE(sym_quotient, 3)),
[aux_sym_STAR] = ACTIONS(REDUCE_FRAGILE(sym_quotient, 3)),
[aux_sym_SLASH] = ACTIONS(REDUCE_FRAGILE(sym_quotient, 3)),
[aux_sym_CARET] = ACTIONS(SHIFT(27)),
[sym_comment] = ACTIONS(SHIFT_EXTRA()),
[aux_sym_STR_PLUS] = ACTIONS(REDUCE_FRAGILE(sym_quotient, 3)),
[aux_sym_STR_DASH] = ACTIONS(REDUCE_FRAGILE(sym_quotient, 3)),
[aux_sym_STR_STAR] = ACTIONS(REDUCE_FRAGILE(sym_quotient, 3)),
[aux_sym_STR_SLASH] = ACTIONS(REDUCE_FRAGILE(sym_quotient, 3)),
[aux_sym_STR_CARET] = ACTIONS(SHIFT(27)),
},
[30] = {
[ts_builtin_sym_end] = ACTIONS(REDUCE_FRAGILE(sym_product, 3)),
[aux_sym_PLUS] = ACTIONS(REDUCE_FRAGILE(sym_product, 3)),
[aux_sym_DASH] = ACTIONS(REDUCE_FRAGILE(sym_product, 3)),
[aux_sym_STAR] = ACTIONS(REDUCE_FRAGILE(sym_product, 3)),
[aux_sym_SLASH] = ACTIONS(REDUCE_FRAGILE(sym_product, 3)),
[aux_sym_CARET] = ACTIONS(SHIFT(27)),
[sym_comment] = ACTIONS(SHIFT_EXTRA()),
[aux_sym_STR_PLUS] = ACTIONS(REDUCE_FRAGILE(sym_product, 3)),
[aux_sym_STR_DASH] = ACTIONS(REDUCE_FRAGILE(sym_product, 3)),
[aux_sym_STR_STAR] = ACTIONS(REDUCE_FRAGILE(sym_product, 3)),
[aux_sym_STR_SLASH] = ACTIONS(REDUCE_FRAGILE(sym_product, 3)),
[aux_sym_STR_CARET] = ACTIONS(SHIFT(27)),
},
[31] = {
[ts_builtin_sym_end] = ACTIONS(REDUCE_FRAGILE(sym_difference, 3)),
[aux_sym_PLUS] = ACTIONS(REDUCE_FRAGILE(sym_difference, 3)),
[aux_sym_DASH] = ACTIONS(REDUCE_FRAGILE(sym_difference, 3)),
[aux_sym_STAR] = ACTIONS(SHIFT(25)),
[aux_sym_SLASH] = ACTIONS(SHIFT(26)),
[aux_sym_CARET] = ACTIONS(SHIFT(27)),
[sym_comment] = ACTIONS(SHIFT_EXTRA()),
[aux_sym_STR_PLUS] = ACTIONS(REDUCE_FRAGILE(sym_difference, 3)),
[aux_sym_STR_DASH] = ACTIONS(REDUCE_FRAGILE(sym_difference, 3)),
[aux_sym_STR_STAR] = ACTIONS(SHIFT(25)),
[aux_sym_STR_SLASH] = ACTIONS(SHIFT(26)),
[aux_sym_STR_CARET] = ACTIONS(SHIFT(27)),
},
[32] = {
[ts_builtin_sym_end] = ACTIONS(REDUCE_FRAGILE(sym_sum, 3)),
[aux_sym_PLUS] = ACTIONS(REDUCE_FRAGILE(sym_sum, 3)),
[aux_sym_DASH] = ACTIONS(REDUCE_FRAGILE(sym_sum, 3)),
[aux_sym_STAR] = ACTIONS(SHIFT(25)),
[aux_sym_SLASH] = ACTIONS(SHIFT(26)),
[aux_sym_CARET] = ACTIONS(SHIFT(27)),
[sym_comment] = ACTIONS(SHIFT_EXTRA()),
[aux_sym_STR_PLUS] = ACTIONS(REDUCE_FRAGILE(sym_sum, 3)),
[aux_sym_STR_DASH] = ACTIONS(REDUCE_FRAGILE(sym_sum, 3)),
[aux_sym_STR_STAR] = ACTIONS(SHIFT(25)),
[aux_sym_STR_SLASH] = ACTIONS(SHIFT(26)),
[aux_sym_STR_CARET] = ACTIONS(SHIFT(27)),
},
};

4120
spec/fixtures/parsers/c.c vendored
View file

File diff suppressed because it is too large Load diff

4896
spec/fixtures/parsers/golang.c vendored
View file

File diff suppressed because it is too large Load diff

70670
spec/fixtures/parsers/javascript.c vendored
View file

File diff suppressed because it is too large Load diff

280
spec/fixtures/parsers/json.c vendored
View file

@ -7,52 +7,52 @@ enum {
sym__value = ts_builtin_sym_start,
sym_object,
sym_array,
aux_sym_object_repeat1,
aux_sym_array_repeat1,
aux_sym_LBRACE,
aux_sym_COLON,
aux_sym_COMMA,
aux_sym_RBRACE,
aux_sym_LBRACK,
aux_sym_RBRACK,
sym_string,
sym_number,
sym_null,
sym_true,
sym_false,
aux_sym_object_repeat1,
aux_sym_array_repeat1,
aux_sym_STR_LBRACE,
aux_sym_STR_COLON,
aux_sym_STR_COMMA,
aux_sym_STR_RBRACE,
aux_sym_STR_LBRACK,
aux_sym_STR_RBRACK,
};
static const char *ts_symbol_names[] = {
[sym__value] = "_value",
[sym_object] = "object",
[sym_array] = "array",
[aux_sym_object_repeat1] = "object_repeat1",
[aux_sym_array_repeat1] = "array_repeat1",
[ts_builtin_sym_error] = "ERROR",
[ts_builtin_sym_end] = "END",
[aux_sym_LBRACE] = "{",
[aux_sym_COLON] = ":",
[aux_sym_COMMA] = ",",
[aux_sym_RBRACE] = "}",
[aux_sym_LBRACK] = "[",
[aux_sym_RBRACK] = "]",
[sym_string] = "string",
[sym_number] = "number",
[sym_null] = "null",
[sym_true] = "true",
[sym_false] = "false",
[aux_sym_object_repeat1] = "object_repeat1",
[aux_sym_array_repeat1] = "array_repeat1",
[aux_sym_STR_LBRACE] = "STR_{",
[aux_sym_STR_COLON] = "STR_:",
[aux_sym_STR_COMMA] = "STR_,",
[aux_sym_STR_RBRACE] = "STR_}",
[aux_sym_STR_LBRACK] = "STR_[",
[aux_sym_STR_RBRACK] = "STR_]",
};
static const int ts_hidden_symbol_flags[SYMBOL_COUNT] = {
[sym__value] = 1,
[aux_sym_object_repeat1] = 1,
[aux_sym_array_repeat1] = 1,
[aux_sym_STR_LBRACE] = 1,
[aux_sym_STR_COLON] = 1,
[aux_sym_STR_COMMA] = 1,
[aux_sym_STR_RBRACE] = 1,
[aux_sym_STR_LBRACK] = 1,
[aux_sym_STR_RBRACK] = 1,
[aux_sym_LBRACE] = 1,
[aux_sym_COLON] = 1,
[aux_sym_COMMA] = 1,
[aux_sym_RBRACE] = 1,
[aux_sym_LBRACK] = 1,
[aux_sym_RBRACK] = 1,
};
static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) {
@ -127,7 +127,7 @@ static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) {
ADVANCE(8);
ACCEPT_TOKEN(sym_number);
case 9:
ACCEPT_TOKEN(aux_sym_STR_LBRACK);
ACCEPT_TOKEN(aux_sym_LBRACK);
case 10:
if (lookahead == 'a')
ADVANCE(11);
@ -175,7 +175,7 @@ static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) {
case 22:
ACCEPT_TOKEN(sym_true);
case 23:
ACCEPT_TOKEN(aux_sym_STR_LBRACE);
ACCEPT_TOKEN(aux_sym_LBRACE);
case 24:
START_TOKEN();
if (lookahead == 0)
@ -201,7 +201,7 @@ static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) {
ADVANCE(27);
LEX_ERROR();
case 27:
ACCEPT_TOKEN(aux_sym_STR_RBRACE);
ACCEPT_TOKEN(aux_sym_RBRACE);
case 28:
START_TOKEN();
if ((lookahead == '\t') ||
@ -227,7 +227,7 @@ static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) {
ADVANCE(23);
LEX_ERROR();
case 29:
ACCEPT_TOKEN(aux_sym_STR_RBRACK);
ACCEPT_TOKEN(aux_sym_RBRACK);
case 30:
START_TOKEN();
if ((lookahead == '\t') ||
@ -241,7 +241,7 @@ static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) {
ADVANCE(29);
LEX_ERROR();
case 31:
ACCEPT_TOKEN(aux_sym_STR_COMMA);
ACCEPT_TOKEN(aux_sym_COMMA);
case 32:
START_TOKEN();
if ((lookahead == '\t') ||
@ -275,7 +275,7 @@ static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) {
ADVANCE(35);
LEX_ERROR();
case 35:
ACCEPT_TOKEN(aux_sym_STR_COLON);
ACCEPT_TOKEN(aux_sym_COLON);
case 36:
START_TOKEN();
if ((lookahead == '\t') ||
@ -385,8 +385,8 @@ static TSStateId ts_lex_states[STATE_COUNT] = {
[16] = 32,
[17] = 30,
[18] = 33,
[19] = 34,
[20] = 30,
[19] = 30,
[20] = 34,
[21] = 1,
[22] = 33,
[23] = 33,
@ -398,8 +398,8 @@ static TSStateId ts_lex_states[STATE_COUNT] = {
[29] = 33,
[30] = 33,
[31] = 33,
[32] = 34,
[33] = 33,
[32] = 33,
[33] = 34,
[34] = 1,
[35] = 33,
[36] = 36,
@ -425,8 +425,8 @@ static TSStateId ts_lex_states[STATE_COUNT] = {
[56] = 24,
[57] = 24,
[58] = 33,
[59] = 34,
[60] = 24,
[59] = 24,
[60] = 34,
[61] = 1,
[62] = 33,
[63] = 36,
@ -445,13 +445,13 @@ static const TSParseAction *ts_parse_actions[STATE_COUNT][SYMBOL_COUNT] = {
[sym__value] = ACTIONS(SHIFT(1)),
[sym_object] = ACTIONS(SHIFT(2)),
[sym_array] = ACTIONS(SHIFT(2)),
[aux_sym_LBRACE] = ACTIONS(SHIFT(3)),
[aux_sym_LBRACK] = ACTIONS(SHIFT(4)),
[sym_string] = ACTIONS(SHIFT(2)),
[sym_number] = ACTIONS(SHIFT(2)),
[sym_null] = ACTIONS(SHIFT(2)),
[sym_true] = ACTIONS(SHIFT(2)),
[sym_false] = ACTIONS(SHIFT(2)),
[aux_sym_STR_LBRACE] = ACTIONS(SHIFT(3)),
[aux_sym_STR_LBRACK] = ACTIONS(SHIFT(4)),
},
[1] = {
[ts_builtin_sym_end] = ACTIONS(ACCEPT_INPUT()),
@ -461,208 +461,208 @@ static const TSParseAction *ts_parse_actions[STATE_COUNT][SYMBOL_COUNT] = {
},
[3] = {
[ts_builtin_sym_error] = ACTIONS(SHIFT(58)),
[sym_string] = ACTIONS(SHIFT(59)),
[aux_sym_STR_RBRACE] = ACTIONS(SHIFT(60)),
[aux_sym_RBRACE] = ACTIONS(SHIFT(59)),
[sym_string] = ACTIONS(SHIFT(60)),
},
[4] = {
[sym__value] = ACTIONS(SHIFT(5)),
[sym_object] = ACTIONS(SHIFT(6)),
[sym_array] = ACTIONS(SHIFT(6)),
[ts_builtin_sym_error] = ACTIONS(SHIFT(5)),
[aux_sym_LBRACE] = ACTIONS(SHIFT(7)),
[aux_sym_LBRACK] = ACTIONS(SHIFT(8)),
[aux_sym_RBRACK] = ACTIONS(SHIFT(9)),
[sym_string] = ACTIONS(SHIFT(6)),
[sym_number] = ACTIONS(SHIFT(6)),
[sym_null] = ACTIONS(SHIFT(6)),
[sym_true] = ACTIONS(SHIFT(6)),
[sym_false] = ACTIONS(SHIFT(6)),
[aux_sym_STR_LBRACE] = ACTIONS(SHIFT(7)),
[aux_sym_STR_LBRACK] = ACTIONS(SHIFT(8)),
[aux_sym_STR_RBRACK] = ACTIONS(SHIFT(9)),
},
[5] = {
[aux_sym_array_repeat1] = ACTIONS(SHIFT(55)),
[aux_sym_STR_COMMA] = ACTIONS(SHIFT(13)),
[aux_sym_STR_RBRACK] = ACTIONS(SHIFT(56)),
[aux_sym_COMMA] = ACTIONS(SHIFT(13)),
[aux_sym_RBRACK] = ACTIONS(SHIFT(56)),
},
[6] = {
[aux_sym_array_repeat1] = ACTIONS(REDUCE(sym__value, 1)),
[aux_sym_STR_COMMA] = ACTIONS(REDUCE(sym__value, 1)),
[aux_sym_STR_RBRACK] = ACTIONS(REDUCE(sym__value, 1)),
[aux_sym_COMMA] = ACTIONS(REDUCE(sym__value, 1)),
[aux_sym_RBRACK] = ACTIONS(REDUCE(sym__value, 1)),
},
[7] = {
[ts_builtin_sym_error] = ACTIONS(SHIFT(18)),
[sym_string] = ACTIONS(SHIFT(19)),
[aux_sym_STR_RBRACE] = ACTIONS(SHIFT(20)),
[aux_sym_RBRACE] = ACTIONS(SHIFT(19)),
[sym_string] = ACTIONS(SHIFT(20)),
},
[8] = {
[sym__value] = ACTIONS(SHIFT(10)),
[sym_object] = ACTIONS(SHIFT(6)),
[sym_array] = ACTIONS(SHIFT(6)),
[ts_builtin_sym_error] = ACTIONS(SHIFT(10)),
[aux_sym_LBRACE] = ACTIONS(SHIFT(7)),
[aux_sym_LBRACK] = ACTIONS(SHIFT(8)),
[aux_sym_RBRACK] = ACTIONS(SHIFT(11)),
[sym_string] = ACTIONS(SHIFT(6)),
[sym_number] = ACTIONS(SHIFT(6)),
[sym_null] = ACTIONS(SHIFT(6)),
[sym_true] = ACTIONS(SHIFT(6)),
[sym_false] = ACTIONS(SHIFT(6)),
[aux_sym_STR_LBRACE] = ACTIONS(SHIFT(7)),
[aux_sym_STR_LBRACK] = ACTIONS(SHIFT(8)),
[aux_sym_STR_RBRACK] = ACTIONS(SHIFT(11)),
},
[9] = {
[ts_builtin_sym_end] = ACTIONS(REDUCE(sym_array, 2)),
},
[10] = {
[aux_sym_array_repeat1] = ACTIONS(SHIFT(12)),
[aux_sym_STR_COMMA] = ACTIONS(SHIFT(13)),
[aux_sym_STR_RBRACK] = ACTIONS(SHIFT(14)),
[aux_sym_COMMA] = ACTIONS(SHIFT(13)),
[aux_sym_RBRACK] = ACTIONS(SHIFT(14)),
},
[11] = {
[aux_sym_array_repeat1] = ACTIONS(REDUCE(sym_array, 2)),
[aux_sym_STR_COMMA] = ACTIONS(REDUCE(sym_array, 2)),
[aux_sym_STR_RBRACK] = ACTIONS(REDUCE(sym_array, 2)),
[aux_sym_COMMA] = ACTIONS(REDUCE(sym_array, 2)),
[aux_sym_RBRACK] = ACTIONS(REDUCE(sym_array, 2)),
},
[12] = {
[aux_sym_STR_RBRACK] = ACTIONS(SHIFT(17)),
[aux_sym_RBRACK] = ACTIONS(SHIFT(17)),
},
[13] = {
[sym__value] = ACTIONS(SHIFT(15)),
[sym_object] = ACTIONS(SHIFT(6)),
[sym_array] = ACTIONS(SHIFT(6)),
[ts_builtin_sym_error] = ACTIONS(SHIFT(15)),
[aux_sym_LBRACE] = ACTIONS(SHIFT(7)),
[aux_sym_LBRACK] = ACTIONS(SHIFT(8)),
[sym_string] = ACTIONS(SHIFT(6)),
[sym_number] = ACTIONS(SHIFT(6)),
[sym_null] = ACTIONS(SHIFT(6)),
[sym_true] = ACTIONS(SHIFT(6)),
[sym_false] = ACTIONS(SHIFT(6)),
[aux_sym_STR_LBRACE] = ACTIONS(SHIFT(7)),
[aux_sym_STR_LBRACK] = ACTIONS(SHIFT(8)),
},
[14] = {
[aux_sym_array_repeat1] = ACTIONS(REDUCE(sym_array, 3)),
[aux_sym_STR_COMMA] = ACTIONS(REDUCE(sym_array, 3)),
[aux_sym_STR_RBRACK] = ACTIONS(REDUCE(sym_array, 3)),
[aux_sym_COMMA] = ACTIONS(REDUCE(sym_array, 3)),
[aux_sym_RBRACK] = ACTIONS(REDUCE(sym_array, 3)),
},
[15] = {
[aux_sym_array_repeat1] = ACTIONS(SHIFT(16)),
[aux_sym_STR_COMMA] = ACTIONS(SHIFT(13)),
[aux_sym_STR_RBRACK] = ACTIONS(REDUCE(aux_sym_array_repeat1, 2)),
[aux_sym_COMMA] = ACTIONS(SHIFT(13)),
[aux_sym_RBRACK] = ACTIONS(REDUCE(aux_sym_array_repeat1, 2)),
},
[16] = {
[aux_sym_STR_RBRACK] = ACTIONS(REDUCE(aux_sym_array_repeat1, 3)),
[aux_sym_RBRACK] = ACTIONS(REDUCE(aux_sym_array_repeat1, 3)),
},
[17] = {
[aux_sym_array_repeat1] = ACTIONS(REDUCE(sym_array, 4)),
[aux_sym_STR_COMMA] = ACTIONS(REDUCE(sym_array, 4)),
[aux_sym_STR_RBRACK] = ACTIONS(REDUCE(sym_array, 4)),
[aux_sym_COMMA] = ACTIONS(REDUCE(sym_array, 4)),
[aux_sym_RBRACK] = ACTIONS(REDUCE(sym_array, 4)),
},
[18] = {
[aux_sym_object_repeat1] = ACTIONS(SHIFT(52)),
[aux_sym_STR_COMMA] = ACTIONS(SHIFT(37)),
[aux_sym_STR_RBRACE] = ACTIONS(SHIFT(53)),
[aux_sym_COMMA] = ACTIONS(SHIFT(37)),
[aux_sym_RBRACE] = ACTIONS(SHIFT(53)),
},
[19] = {
[aux_sym_STR_COLON] = ACTIONS(SHIFT(21)),
[aux_sym_array_repeat1] = ACTIONS(REDUCE(sym_object, 2)),
[aux_sym_COMMA] = ACTIONS(REDUCE(sym_object, 2)),
[aux_sym_RBRACK] = ACTIONS(REDUCE(sym_object, 2)),
},
[20] = {
[aux_sym_array_repeat1] = ACTIONS(REDUCE(sym_object, 2)),
[aux_sym_STR_COMMA] = ACTIONS(REDUCE(sym_object, 2)),
[aux_sym_STR_RBRACK] = ACTIONS(REDUCE(sym_object, 2)),
[aux_sym_COLON] = ACTIONS(SHIFT(21)),
},
[21] = {
[sym__value] = ACTIONS(SHIFT(22)),
[sym_object] = ACTIONS(SHIFT(23)),
[sym_array] = ACTIONS(SHIFT(23)),
[aux_sym_LBRACE] = ACTIONS(SHIFT(24)),
[aux_sym_LBRACK] = ACTIONS(SHIFT(25)),
[sym_string] = ACTIONS(SHIFT(23)),
[sym_number] = ACTIONS(SHIFT(23)),
[sym_null] = ACTIONS(SHIFT(23)),
[sym_true] = ACTIONS(SHIFT(23)),
[sym_false] = ACTIONS(SHIFT(23)),
[aux_sym_STR_LBRACE] = ACTIONS(SHIFT(24)),
[aux_sym_STR_LBRACK] = ACTIONS(SHIFT(25)),
},
[22] = {
[aux_sym_object_repeat1] = ACTIONS(SHIFT(49)),
[aux_sym_STR_COMMA] = ACTIONS(SHIFT(37)),
[aux_sym_STR_RBRACE] = ACTIONS(SHIFT(50)),
[aux_sym_COMMA] = ACTIONS(SHIFT(37)),
[aux_sym_RBRACE] = ACTIONS(SHIFT(50)),
},
[23] = {
[aux_sym_object_repeat1] = ACTIONS(REDUCE(sym__value, 1)),
[aux_sym_STR_COMMA] = ACTIONS(REDUCE(sym__value, 1)),
[aux_sym_STR_RBRACE] = ACTIONS(REDUCE(sym__value, 1)),
[aux_sym_COMMA] = ACTIONS(REDUCE(sym__value, 1)),
[aux_sym_RBRACE] = ACTIONS(REDUCE(sym__value, 1)),
},
[24] = {
[ts_builtin_sym_error] = ACTIONS(SHIFT(31)),
[sym_string] = ACTIONS(SHIFT(32)),
[aux_sym_STR_RBRACE] = ACTIONS(SHIFT(33)),
[aux_sym_RBRACE] = ACTIONS(SHIFT(32)),
[sym_string] = ACTIONS(SHIFT(33)),
},
[25] = {
[sym__value] = ACTIONS(SHIFT(26)),
[sym_object] = ACTIONS(SHIFT(6)),
[sym_array] = ACTIONS(SHIFT(6)),
[ts_builtin_sym_error] = ACTIONS(SHIFT(26)),
[aux_sym_LBRACE] = ACTIONS(SHIFT(7)),
[aux_sym_LBRACK] = ACTIONS(SHIFT(8)),
[aux_sym_RBRACK] = ACTIONS(SHIFT(27)),
[sym_string] = ACTIONS(SHIFT(6)),
[sym_number] = ACTIONS(SHIFT(6)),
[sym_null] = ACTIONS(SHIFT(6)),
[sym_true] = ACTIONS(SHIFT(6)),
[sym_false] = ACTIONS(SHIFT(6)),
[aux_sym_STR_LBRACE] = ACTIONS(SHIFT(7)),
[aux_sym_STR_LBRACK] = ACTIONS(SHIFT(8)),
[aux_sym_STR_RBRACK] = ACTIONS(SHIFT(27)),
},
[26] = {
[aux_sym_array_repeat1] = ACTIONS(SHIFT(28)),
[aux_sym_STR_COMMA] = ACTIONS(SHIFT(13)),
[aux_sym_STR_RBRACK] = ACTIONS(SHIFT(29)),
[aux_sym_COMMA] = ACTIONS(SHIFT(13)),
[aux_sym_RBRACK] = ACTIONS(SHIFT(29)),
},
[27] = {
[aux_sym_object_repeat1] = ACTIONS(REDUCE(sym_array, 2)),
[aux_sym_STR_COMMA] = ACTIONS(REDUCE(sym_array, 2)),
[aux_sym_STR_RBRACE] = ACTIONS(REDUCE(sym_array, 2)),
[aux_sym_COMMA] = ACTIONS(REDUCE(sym_array, 2)),
[aux_sym_RBRACE] = ACTIONS(REDUCE(sym_array, 2)),
},
[28] = {
[aux_sym_STR_RBRACK] = ACTIONS(SHIFT(30)),
[aux_sym_RBRACK] = ACTIONS(SHIFT(30)),
},
[29] = {
[aux_sym_object_repeat1] = ACTIONS(REDUCE(sym_array, 3)),
[aux_sym_STR_COMMA] = ACTIONS(REDUCE(sym_array, 3)),
[aux_sym_STR_RBRACE] = ACTIONS(REDUCE(sym_array, 3)),
[aux_sym_COMMA] = ACTIONS(REDUCE(sym_array, 3)),
[aux_sym_RBRACE] = ACTIONS(REDUCE(sym_array, 3)),
},
[30] = {
[aux_sym_object_repeat1] = ACTIONS(REDUCE(sym_array, 4)),
[aux_sym_STR_COMMA] = ACTIONS(REDUCE(sym_array, 4)),
[aux_sym_STR_RBRACE] = ACTIONS(REDUCE(sym_array, 4)),
[aux_sym_COMMA] = ACTIONS(REDUCE(sym_array, 4)),
[aux_sym_RBRACE] = ACTIONS(REDUCE(sym_array, 4)),
},
[31] = {
[aux_sym_object_repeat1] = ACTIONS(SHIFT(46)),
[aux_sym_STR_COMMA] = ACTIONS(SHIFT(37)),
[aux_sym_STR_RBRACE] = ACTIONS(SHIFT(47)),
[aux_sym_COMMA] = ACTIONS(SHIFT(37)),
[aux_sym_RBRACE] = ACTIONS(SHIFT(47)),
},
[32] = {
[aux_sym_STR_COLON] = ACTIONS(SHIFT(34)),
[aux_sym_object_repeat1] = ACTIONS(REDUCE(sym_object, 2)),
[aux_sym_COMMA] = ACTIONS(REDUCE(sym_object, 2)),
[aux_sym_RBRACE] = ACTIONS(REDUCE(sym_object, 2)),
},
[33] = {
[aux_sym_object_repeat1] = ACTIONS(REDUCE(sym_object, 2)),
[aux_sym_STR_COMMA] = ACTIONS(REDUCE(sym_object, 2)),
[aux_sym_STR_RBRACE] = ACTIONS(REDUCE(sym_object, 2)),
[aux_sym_COLON] = ACTIONS(SHIFT(34)),
},
[34] = {
[sym__value] = ACTIONS(SHIFT(35)),
[sym_object] = ACTIONS(SHIFT(23)),
[sym_array] = ACTIONS(SHIFT(23)),
[aux_sym_LBRACE] = ACTIONS(SHIFT(24)),
[aux_sym_LBRACK] = ACTIONS(SHIFT(25)),
[sym_string] = ACTIONS(SHIFT(23)),
[sym_number] = ACTIONS(SHIFT(23)),
[sym_null] = ACTIONS(SHIFT(23)),
[sym_true] = ACTIONS(SHIFT(23)),
[sym_false] = ACTIONS(SHIFT(23)),
[aux_sym_STR_LBRACE] = ACTIONS(SHIFT(24)),
[aux_sym_STR_LBRACK] = ACTIONS(SHIFT(25)),
},
[35] = {
[aux_sym_object_repeat1] = ACTIONS(SHIFT(36)),
[aux_sym_STR_COMMA] = ACTIONS(SHIFT(37)),
[aux_sym_STR_RBRACE] = ACTIONS(SHIFT(38)),
[aux_sym_COMMA] = ACTIONS(SHIFT(37)),
[aux_sym_RBRACE] = ACTIONS(SHIFT(38)),
},
[36] = {
[aux_sym_STR_RBRACE] = ACTIONS(SHIFT(45)),
[aux_sym_RBRACE] = ACTIONS(SHIFT(45)),
},
[37] = {
[ts_builtin_sym_error] = ACTIONS(SHIFT(39)),
@ -670,86 +670,86 @@ static const TSParseAction *ts_parse_actions[STATE_COUNT][SYMBOL_COUNT] = {
},
[38] = {
[aux_sym_object_repeat1] = ACTIONS(REDUCE(sym_object, 5)),
[aux_sym_STR_COMMA] = ACTIONS(REDUCE(sym_object, 5)),
[aux_sym_STR_RBRACE] = ACTIONS(REDUCE(sym_object, 5)),
[aux_sym_COMMA] = ACTIONS(REDUCE(sym_object, 5)),
[aux_sym_RBRACE] = ACTIONS(REDUCE(sym_object, 5)),
},
[39] = {
[aux_sym_object_repeat1] = ACTIONS(SHIFT(44)),
[aux_sym_STR_COMMA] = ACTIONS(SHIFT(37)),
[aux_sym_STR_RBRACE] = ACTIONS(REDUCE(aux_sym_object_repeat1, 2)),
[aux_sym_COMMA] = ACTIONS(SHIFT(37)),
[aux_sym_RBRACE] = ACTIONS(REDUCE(aux_sym_object_repeat1, 2)),
},
[40] = {
[aux_sym_STR_COLON] = ACTIONS(SHIFT(41)),
[aux_sym_COLON] = ACTIONS(SHIFT(41)),
},
[41] = {
[sym__value] = ACTIONS(SHIFT(42)),
[sym_object] = ACTIONS(SHIFT(23)),
[sym_array] = ACTIONS(SHIFT(23)),
[aux_sym_LBRACE] = ACTIONS(SHIFT(24)),
[aux_sym_LBRACK] = ACTIONS(SHIFT(25)),
[sym_string] = ACTIONS(SHIFT(23)),
[sym_number] = ACTIONS(SHIFT(23)),
[sym_null] = ACTIONS(SHIFT(23)),
[sym_true] = ACTIONS(SHIFT(23)),
[sym_false] = ACTIONS(SHIFT(23)),
[aux_sym_STR_LBRACE] = ACTIONS(SHIFT(24)),
[aux_sym_STR_LBRACK] = ACTIONS(SHIFT(25)),
},
[42] = {
[aux_sym_object_repeat1] = ACTIONS(SHIFT(43)),
[aux_sym_STR_COMMA] = ACTIONS(SHIFT(37)),
[aux_sym_STR_RBRACE] = ACTIONS(REDUCE(aux_sym_object_repeat1, 4)),
[aux_sym_COMMA] = ACTIONS(SHIFT(37)),
[aux_sym_RBRACE] = ACTIONS(REDUCE(aux_sym_object_repeat1, 4)),
},
[43] = {
[aux_sym_STR_RBRACE] = ACTIONS(REDUCE(aux_sym_object_repeat1, 5)),
[aux_sym_RBRACE] = ACTIONS(REDUCE(aux_sym_object_repeat1, 5)),
},
[44] = {
[aux_sym_STR_RBRACE] = ACTIONS(REDUCE(aux_sym_object_repeat1, 3)),
[aux_sym_RBRACE] = ACTIONS(REDUCE(aux_sym_object_repeat1, 3)),
},
[45] = {
[aux_sym_object_repeat1] = ACTIONS(REDUCE(sym_object, 6)),
[aux_sym_STR_COMMA] = ACTIONS(REDUCE(sym_object, 6)),
[aux_sym_STR_RBRACE] = ACTIONS(REDUCE(sym_object, 6)),
[aux_sym_COMMA] = ACTIONS(REDUCE(sym_object, 6)),
[aux_sym_RBRACE] = ACTIONS(REDUCE(sym_object, 6)),
},
[46] = {
[aux_sym_STR_RBRACE] = ACTIONS(SHIFT(48)),
[aux_sym_RBRACE] = ACTIONS(SHIFT(48)),
},
[47] = {
[aux_sym_object_repeat1] = ACTIONS(REDUCE(sym_object, 3)),
[aux_sym_STR_COMMA] = ACTIONS(REDUCE(sym_object, 3)),
[aux_sym_STR_RBRACE] = ACTIONS(REDUCE(sym_object, 3)),
[aux_sym_COMMA] = ACTIONS(REDUCE(sym_object, 3)),
[aux_sym_RBRACE] = ACTIONS(REDUCE(sym_object, 3)),
},
[48] = {
[aux_sym_object_repeat1] = ACTIONS(REDUCE(sym_object, 4)),
[aux_sym_STR_COMMA] = ACTIONS(REDUCE(sym_object, 4)),
[aux_sym_STR_RBRACE] = ACTIONS(REDUCE(sym_object, 4)),
[aux_sym_COMMA] = ACTIONS(REDUCE(sym_object, 4)),
[aux_sym_RBRACE] = ACTIONS(REDUCE(sym_object, 4)),
},
[49] = {
[aux_sym_STR_RBRACE] = ACTIONS(SHIFT(51)),
[aux_sym_RBRACE] = ACTIONS(SHIFT(51)),
},
[50] = {
[aux_sym_array_repeat1] = ACTIONS(REDUCE(sym_object, 5)),
[aux_sym_STR_COMMA] = ACTIONS(REDUCE(sym_object, 5)),
[aux_sym_STR_RBRACK] = ACTIONS(REDUCE(sym_object, 5)),
[aux_sym_COMMA] = ACTIONS(REDUCE(sym_object, 5)),
[aux_sym_RBRACK] = ACTIONS(REDUCE(sym_object, 5)),
},
[51] = {
[aux_sym_array_repeat1] = ACTIONS(REDUCE(sym_object, 6)),
[aux_sym_STR_COMMA] = ACTIONS(REDUCE(sym_object, 6)),
[aux_sym_STR_RBRACK] = ACTIONS(REDUCE(sym_object, 6)),
[aux_sym_COMMA] = ACTIONS(REDUCE(sym_object, 6)),
[aux_sym_RBRACK] = ACTIONS(REDUCE(sym_object, 6)),
},
[52] = {
[aux_sym_STR_RBRACE] = ACTIONS(SHIFT(54)),
[aux_sym_RBRACE] = ACTIONS(SHIFT(54)),
},
[53] = {
[aux_sym_array_repeat1] = ACTIONS(REDUCE(sym_object, 3)),
[aux_sym_STR_COMMA] = ACTIONS(REDUCE(sym_object, 3)),
[aux_sym_STR_RBRACK] = ACTIONS(REDUCE(sym_object, 3)),
[aux_sym_COMMA] = ACTIONS(REDUCE(sym_object, 3)),
[aux_sym_RBRACK] = ACTIONS(REDUCE(sym_object, 3)),
},
[54] = {
[aux_sym_array_repeat1] = ACTIONS(REDUCE(sym_object, 4)),
[aux_sym_STR_COMMA] = ACTIONS(REDUCE(sym_object, 4)),
[aux_sym_STR_RBRACK] = ACTIONS(REDUCE(sym_object, 4)),
[aux_sym_COMMA] = ACTIONS(REDUCE(sym_object, 4)),
[aux_sym_RBRACK] = ACTIONS(REDUCE(sym_object, 4)),
},
[55] = {
[aux_sym_STR_RBRACK] = ACTIONS(SHIFT(57)),
[aux_sym_RBRACK] = ACTIONS(SHIFT(57)),
},
[56] = {
[ts_builtin_sym_end] = ACTIONS(REDUCE(sym_array, 3)),
@ -759,34 +759,34 @@ static const TSParseAction *ts_parse_actions[STATE_COUNT][SYMBOL_COUNT] = {
},
[58] = {
[aux_sym_object_repeat1] = ACTIONS(SHIFT(66)),
[aux_sym_STR_COMMA] = ACTIONS(SHIFT(37)),
[aux_sym_STR_RBRACE] = ACTIONS(SHIFT(67)),
[aux_sym_COMMA] = ACTIONS(SHIFT(37)),
[aux_sym_RBRACE] = ACTIONS(SHIFT(67)),
},
[59] = {
[aux_sym_STR_COLON] = ACTIONS(SHIFT(61)),
[ts_builtin_sym_end] = ACTIONS(REDUCE(sym_object, 2)),
},
[60] = {
[ts_builtin_sym_end] = ACTIONS(REDUCE(sym_object, 2)),
[aux_sym_COLON] = ACTIONS(SHIFT(61)),
},
[61] = {
[sym__value] = ACTIONS(SHIFT(62)),
[sym_object] = ACTIONS(SHIFT(23)),
[sym_array] = ACTIONS(SHIFT(23)),
[aux_sym_LBRACE] = ACTIONS(SHIFT(24)),
[aux_sym_LBRACK] = ACTIONS(SHIFT(25)),
[sym_string] = ACTIONS(SHIFT(23)),
[sym_number] = ACTIONS(SHIFT(23)),
[sym_null] = ACTIONS(SHIFT(23)),
[sym_true] = ACTIONS(SHIFT(23)),
[sym_false] = ACTIONS(SHIFT(23)),
[aux_sym_STR_LBRACE] = ACTIONS(SHIFT(24)),
[aux_sym_STR_LBRACK] = ACTIONS(SHIFT(25)),
},
[62] = {
[aux_sym_object_repeat1] = ACTIONS(SHIFT(63)),
[aux_sym_STR_COMMA] = ACTIONS(SHIFT(37)),
[aux_sym_STR_RBRACE] = ACTIONS(SHIFT(64)),
[aux_sym_COMMA] = ACTIONS(SHIFT(37)),
[aux_sym_RBRACE] = ACTIONS(SHIFT(64)),
},
[63] = {
[aux_sym_STR_RBRACE] = ACTIONS(SHIFT(65)),
[aux_sym_RBRACE] = ACTIONS(SHIFT(65)),
},
[64] = {
[ts_builtin_sym_end] = ACTIONS(REDUCE(sym_object, 5)),
@ -795,7 +795,7 @@ static const TSParseAction *ts_parse_actions[STATE_COUNT][SYMBOL_COUNT] = {
[ts_builtin_sym_end] = ACTIONS(REDUCE(sym_object, 6)),
},
[66] = {
[aux_sym_STR_RBRACE] = ACTIONS(SHIFT(68)),
[aux_sym_RBRACE] = ACTIONS(SHIFT(68)),
},
[67] = {
[ts_builtin_sym_end] = ACTIONS(REDUCE(sym_object, 3)),

8
src/compiler/build_tables/build_lex_table.cc
View file

@ -10,7 +10,7 @@
#include "compiler/build_tables/get_metadata.h"
#include "compiler/build_tables/lex_item.h"
#include "compiler/parse_table.h"
#include "compiler/lexical_grammar.h"
#include "compiler/prepared_grammar.h"
#include "compiler/rules/built_in_symbols.h"
#include "compiler/rules/choice.h"
#include "compiler/rules/metadata.h"
@ -63,9 +63,9 @@ class LexTableBuilder {
result.insert(
LexItem(symbol, after_separators(CharacterSet().include(0).copy())));
else if (symbol.is_token())
result.insert(
LexItem(symbol, after_separators(lex_grammar.rule(symbol))));
else if (symbol.is_token)
result.insert(LexItem(
symbol, after_separators(lex_grammar.rules[symbol.index].rule)));
}
return result;
}

14
src/compiler/build_tables/build_parse_table.cc
View file

@ -11,8 +11,7 @@
#include "compiler/build_tables/parse_item.h"
#include "compiler/build_tables/get_completion_status.h"
#include "compiler/build_tables/get_metadata.h"
#include "compiler/lexical_grammar.h"
#include "compiler/syntax_grammar.h"
#include "compiler/prepared_grammar.h"
#include "compiler/rules/symbol.h"
#include "compiler/rules/built_in_symbols.h"
@ -48,9 +47,8 @@ class ParseTableBuilder {
conflict_manager(grammar) {}
pair<ParseTable, const GrammarError *> build() {
auto start_symbol = grammar.rules.empty()
? make_shared<Symbol>(0, rules::SymbolOptionToken)
: make_shared<Symbol>(0);
auto start_symbol = grammar.rules.empty() ? make_shared<Symbol>(0, true)
: make_shared<Symbol>(0);
ParseItem start_item(rules::START(), start_symbol, {});
add_parse_state(
item_set_closure(start_item, { rules::END_OF_INPUT() }, grammar));
@ -260,10 +258,10 @@ class ParseTableBuilder {
return "END_OF_INPUT";
else
return "";
} else if (symbol.is_token())
return lexical_grammar.rule_name(symbol);
} else if (symbol.is_token)
return lexical_grammar.rules[symbol.index].name;
else
return grammar.rule_name(symbol);
return grammar.rules[symbol.index].name;
}
string action_description(const ParseAction &action) const {

3
src/compiler/build_tables/build_tables.cc
View file

@ -1,7 +1,6 @@
#include "compiler/build_tables/build_lex_table.h"
#include "compiler/build_tables/build_parse_table.h"
#include "compiler/lexical_grammar.h"
#include "compiler/syntax_grammar.h"
#include "compiler/prepared_grammar.h"
namespace tree_sitter {
namespace build_tables {

6
src/compiler/build_tables/first_symbols.cc
View file

@ -1,6 +1,6 @@
#include "compiler/build_tables/first_symbols.h"
#include "compiler/build_tables/rule_can_be_blank.h"
#include "compiler/syntax_grammar.h"
#include "compiler/prepared_grammar.h"
#include "compiler/rules/choice.h"
#include "compiler/rules/metadata.h"
#include "compiler/rules/seq.h"
@ -28,8 +28,8 @@ class FirstSymbols : public rules::RuleFn<set<Symbol>> {
return set<Symbol>();
set<Symbol> result({ *rule });
if (!rule->is_token()) {
set<Symbol> &&symbols = apply(grammar->rule(*rule));
if (!rule->is_token) {
set<Symbol> &&symbols = apply(grammar->rules[rule->index].rule);
result.insert(symbols.begin(), symbols.end());
}

9
src/compiler/build_tables/item_set_closure.cc
View file

@ -7,7 +7,7 @@
#include "compiler/build_tables/rule_transitions.h"
#include "compiler/build_tables/rule_can_be_blank.h"
#include "compiler/build_tables/item.h"
#include "compiler/syntax_grammar.h"
#include "compiler/prepared_grammar.h"
namespace tree_sitter {
namespace build_tables {
@ -41,7 +41,7 @@ const ParseItemSet item_set_closure(const ParseItem &starting_item,
const Symbol &symbol = pair.first;
const rule_ptr &next_rule = pair.second;
if (symbol.is_token() || symbol.is_built_in())
if (symbol.is_token || symbol.is_built_in())
continue;
set<Symbol> next_lookahead_symbols = first_symbols(next_rule, grammar);
@ -49,8 +49,9 @@ const ParseItemSet item_set_closure(const ParseItem &starting_item,
next_lookahead_symbols.insert(lookahead_symbols.begin(),
lookahead_symbols.end());
items_to_process.push_back({ ParseItem(symbol, grammar.rule(symbol), {}),
next_lookahead_symbols });
items_to_process.push_back(
{ ParseItem(symbol, grammar.rules[symbol.index].rule, {}),
next_lookahead_symbols });
}
}

2
src/compiler/build_tables/item_set_transitions.cc
View file

@ -4,7 +4,7 @@
#include "compiler/build_tables/merge_transitions.h"
#include "compiler/build_tables/parse_item.h"
#include "compiler/build_tables/rule_transitions.h"
#include "compiler/syntax_grammar.h"
#include "compiler/prepared_grammar.h"
#include "compiler/rules/symbol.h"
namespace tree_sitter {

2
src/compiler/build_tables/lex_conflict_manager.h
View file

@ -2,7 +2,7 @@
#define COMPILER_BUILD_TABLES_LEX_CONFLICT_MANAGER_H_
#include "tree_sitter/compiler.h"
#include "compiler/lexical_grammar.h"
#include "compiler/prepared_grammar.h"
namespace tree_sitter {

3
src/compiler/build_tables/parse_conflict_manager.h
View file

@ -3,8 +3,7 @@
#include <utility>
#include "tree_sitter/compiler.h"
#include "compiler/syntax_grammar.h"
#include "compiler/lexical_grammar.h"
#include "compiler/prepared_grammar.h"
#include "compiler/build_tables/parse_item.h"
namespace tree_sitter {

4
src/compiler/build_tables/rule_can_be_blank.cc
View file

@ -1,7 +1,7 @@
#include "compiler/build_tables/rule_can_be_blank.h"
#include <set>
#include "tree_sitter/compiler.h"
#include "compiler/syntax_grammar.h"
#include "compiler/prepared_grammar.h"
#include "compiler/rules/symbol.h"
#include "compiler/rules/visitor.h"
#include "compiler/rules/seq.h"
@ -55,7 +55,7 @@ class CanBeBlankRecursive : public CanBeBlank {
bool apply_to(const rules::Symbol *rule) {
if (visited_symbols.find(*rule) == visited_symbols.end()) {
visited_symbols.insert(*rule);
return !rule->is_token() && apply(grammar->rule(*rule));
return !rule->is_token && apply(grammar->rules[rule->index].rule);
} else {
return false;
}

3
src/compiler/compile.cc
View file

@ -2,8 +2,7 @@
#include "compiler/prepare_grammar/prepare_grammar.h"
#include "compiler/build_tables/build_tables.h"
#include "compiler/generate_code/c_code.h"
#include "compiler/syntax_grammar.h"
#include "compiler/lexical_grammar.h"
#include "compiler/prepared_grammar.h"
namespace tree_sitter {

39
src/compiler/generate_code/c_code.cc
View file

@ -7,8 +7,7 @@
#include "compiler/generate_code/c_code.h"
#include "compiler/lex_table.h"
#include "compiler/parse_table.h"
#include "compiler/lexical_grammar.h"
#include "compiler/syntax_grammar.h"
#include "compiler/prepared_grammar.h"
#include "compiler/rules/built_in_symbols.h"
#include "compiler/util/string_helpers.h"
@ -142,7 +141,7 @@ class CCodeGenerator {
indent([&]() {
for (const auto &symbol : parse_table.symbols)
if (!symbol.is_built_in() &&
(symbol.is_auxiliary() || rule_name(symbol)[0] == '_'))
(is_auxiliary(symbol) || rule_name(symbol)[0] == '_'))
line("[" + symbol_id(symbol) + "] = 1,");
});
line("};");
@ -329,7 +328,7 @@ class CCodeGenerator {
return "";
} else {
string name = sanitize_name(rule_name(symbol));
if (symbol.is_auxiliary())
if (is_auxiliary(symbol))
return "aux_sym_" + name;
else
return "sym_" + name;
@ -349,9 +348,20 @@ class CCodeGenerator {
}
}
bool is_auxiliary(const rules::Symbol &symbol) {
if (symbol.is_token) {
return lexical_grammar.rules[symbol.index].type != RuleEntryTypeNamed;
} else {
return syntax_grammar.rules[symbol.index].type != RuleEntryTypeNamed;
}
}
string rule_name(const rules::Symbol &symbol) {
return symbol.is_token() ? lexical_grammar.rule_name(symbol)
: syntax_grammar.rule_name(symbol);
if (symbol.is_token) {
return lexical_grammar.rules[symbol.index].name;
} else {
return syntax_grammar.rules[symbol.index].name;
}
}
bool reduce_action_is_fragile(const ParseAction &action) const {
@ -394,15 +404,14 @@ class CCodeGenerator {
if (('a' <= c && c <= 'z') || ('A' <= c && c <= 'Z') ||
('0' <= c && c <= '9') || (c == '_')) {
stripped_name += c;
continue;
}
auto replacement = REPLACEMENTS.find(c);
if (replacement != REPLACEMENTS.end()) {
if (stripped_name[stripped_name.size() - 1] != '_')
stripped_name += "_";
stripped_name += replacement->second;
continue;
} else {
auto replacement = REPLACEMENTS.find(c);
size_t i = stripped_name.size();
if (replacement != REPLACEMENTS.end()) {
if (i > 0 && stripped_name[i - 1] != '_')
stripped_name += "_";
stripped_name += replacement->second;
}
}
}

24
src/compiler/lexical_grammar.cc
View file

@ -1,24 +0,0 @@
#include "compiler/lexical_grammar.h"
#include <vector>
#include <string>
#include <utility>
#include "compiler/rules/symbol.h"
namespace tree_sitter {
using std::string;
using std::pair;
using std::vector;
using std::set;
const rule_ptr &LexicalGrammar::rule(const rules::Symbol &symbol) const {
return symbol.is_auxiliary() ? aux_rules[symbol.index].second
: rules[symbol.index].second;
}
const string &LexicalGrammar::rule_name(const rules::Symbol &symbol) const {
return symbol.is_auxiliary() ? aux_rules[symbol.index].first
: rules[symbol.index].first;
}
} // namespace tree_sitter

24
src/compiler/lexical_grammar.h
View file

@ -1,24 +0,0 @@
#ifndef COMPILER_LEXICAL_GRAMMAR_H_
#define COMPILER_LEXICAL_GRAMMAR_H_
#include <vector>
#include <string>
#include <utility>
#include "tree_sitter/compiler.h"
#include "compiler/rules/symbol.h"
namespace tree_sitter {
class LexicalGrammar {
public:
const std::string &rule_name(const rules::Symbol &symbol) const;
const rule_ptr &rule(const rules::Symbol &symbol) const;
std::vector<std::pair<std::string, rule_ptr>> rules;
std::vector<std::pair<std::string, rule_ptr>> aux_rules;
std::vector<rule_ptr> separators;
};
} // namespace tree_sitter
#endif // COMPILER_LEXICAL_GRAMMAR_H_

29
src/compiler/prepare_grammar/expand_repeats.cc
View file

@ -2,7 +2,7 @@
#include <vector>
#include <string>
#include <utility>
#include "compiler/syntax_grammar.h"
#include "compiler/prepared_grammar.h"
#include "compiler/rules/visitor.h"
#include "compiler/rules/seq.h"
#include "compiler/rules/symbol.h"
@ -40,12 +40,14 @@ class ExpandRepeats : public rules::IdentityRuleFn {
size_t index = aux_rules.size();
string helper_rule_name =
rule_name + string("_repeat") + to_string(++repeat_count);
Symbol repeat_symbol(offset + index, rules::SymbolOptionAuxiliary);
Symbol repeat_symbol(offset + index);
existing_repeats.push_back({ rule->copy(), repeat_symbol });
aux_rules.push_back(
{ helper_rule_name,
Seq::build({ inner_rule, Choice::build({ repeat_symbol.copy(),
make_shared<Blank>() }) }) });
aux_rules.push_back({
helper_rule_name,
Seq::build({ inner_rule, Choice::build({ repeat_symbol.copy(),
make_shared<Blank>() }) }),
RuleEntryTypeHidden,
});
return repeat_symbol.copy();
}
@ -62,22 +64,21 @@ class ExpandRepeats : public rules::IdentityRuleFn {
return apply(rule);
}
vector<pair<string, rule_ptr>> aux_rules;
vector<RuleEntry> aux_rules;
};
SyntaxGrammar expand_repeats(const SyntaxGrammar &grammar) {
SyntaxGrammar result;
result.aux_rules = grammar.aux_rules;
result.rules = grammar.rules;
result.ubiquitous_tokens = grammar.ubiquitous_tokens;
result.expected_conflicts = grammar.expected_conflicts;
ExpandRepeats expander(result.aux_rules.size());
for (auto &pair : grammar.rules)
result.rules.push_back(
{ pair.first, expander.expand(pair.second, pair.first) });
ExpandRepeats expander(result.rules.size());
for (auto &rule_entry : result.rules)
rule_entry.rule = expander.expand(rule_entry.rule, rule_entry.name);
result.aux_rules.insert(result.aux_rules.end(), expander.aux_rules.begin(),
expander.aux_rules.end());
result.rules.insert(result.rules.end(), expander.aux_rules.begin(),
expander.aux_rules.end());
return result;
}

29
src/compiler/prepare_grammar/expand_tokens.cc
View file

@ -2,12 +2,14 @@
#include <vector>
#include <string>
#include <utility>
#include "compiler/lexical_grammar.h"
#include <map>
#include "compiler/prepared_grammar.h"
#include "compiler/rules/visitor.h"
#include "compiler/rules/pattern.h"
#include "compiler/rules/string.h"
#include "compiler/rules/blank.h"
#include "compiler/rules/seq.h"
#include "compiler/rules/metadata.h"
#include "compiler/rules/character_set.h"
#include "compiler/prepare_grammar/parse_regex.h"
#include "utf8proc.h"
@ -17,10 +19,12 @@ namespace prepare_grammar {
using std::string;
using std::vector;
using std::map;
using std::pair;
using std::make_shared;
using rules::String;
using rules::Pattern;
using rules::Metadata;
class ExpandTokens : public rules::IdentityRuleFn {
using rules::IdentityRuleFn::apply_to;
@ -40,7 +44,11 @@ class ExpandTokens : public rules::IdentityRuleFn {
elements.push_back(rules::CharacterSet().include(el).copy());
}
return rules::Seq::build(elements);
return make_shared<rules::Metadata>(
rules::Seq::build(elements),
std::map<rules::MetadataKey, int>({
{ rules::IS_TOKEN, 1 }, { rules::PRECEDENCE, 1 },
}));
}
rule_ptr apply_to(const Pattern *rule) {
@ -60,18 +68,11 @@ pair<LexicalGrammar, const GrammarError *> expand_tokens(
LexicalGrammar result;
ExpandTokens expander;
for (auto &pair : grammar.rules) {
auto rule = expander.apply(pair.second);
for (auto &entry : grammar.rules) {
auto rule = expander.apply(entry.rule);
if (expander.error)
return { result, expander.error };
result.rules.push_back({ pair.first, rule });
}
for (auto &pair : grammar.aux_rules) {
auto rule = expander.apply(pair.second);
if (expander.error)
return { result, expander.error };
result.aux_rules.push_back({ pair.first, rule });
result.rules.push_back({ entry.name, rule, entry.type });
}
for (auto &sep : grammar.separators) {
@ -81,9 +82,7 @@ pair<LexicalGrammar, const GrammarError *> expand_tokens(
result.separators.push_back(rule);
}
return {
result, nullptr,
};
return { result, nullptr };
}
} // namespace prepare_grammar

164
src/compiler/prepare_grammar/extract_tokens.cc
View file

@ -4,8 +4,7 @@
#include <set>
#include <string>
#include "tree_sitter/compiler.h"
#include "compiler/lexical_grammar.h"
#include "compiler/syntax_grammar.h"
#include "compiler/prepared_grammar.h"
#include "compiler/rules/visitor.h"
#include "compiler/rules/symbol.h"
#include "compiler/rules/string.h"
@ -21,27 +20,15 @@ using std::dynamic_pointer_cast;
using std::make_shared;
using std::make_tuple;
using std::map;
using std::pair;
using std::set;
using std::string;
using std::tuple;
using std::vector;
using rules::Symbol;
using rules::SymbolOptionToken;
using rules::SymbolOptionAuxToken;
class SymbolReplacer : public rules::IdentityRuleFn {
using rules::IdentityRuleFn::apply_to;
int new_index_for_symbol(const Symbol &symbol) {
int result = symbol.index;
for (const auto &pair : replacements)
if (pair.first.index < symbol.index &&
pair.first.is_auxiliary() == symbol.is_auxiliary())
result--;
return result;
}
rule_ptr apply_to(const Symbol *rule) {
return replace_symbol(*rule).copy();
}
@ -49,54 +36,64 @@ class SymbolReplacer : public rules::IdentityRuleFn {
public:
map<Symbol, Symbol> replacements;
Symbol replace_symbol(const Symbol &rule) {
if (rule.is_built_in())
return rule;
auto replacement_pair = replacements.find(rule);
Symbol replace_symbol(const Symbol &symbol) {
if (symbol.is_built_in() || symbol.is_token)
return symbol;
auto replacement_pair = replacements.find(symbol);
if (replacement_pair != replacements.end())
return replacement_pair->second;
else
return Symbol(new_index_for_symbol(rule), rule.options);
int new_index = symbol.index;
for (const auto &pair : replacements)
if (pair.first.index < symbol.index)
new_index--;
return Symbol(new_index);
}
};
class TokenExtractor : public rules::IdentityRuleFn {
rule_ptr apply_to_token(const Rule *input) {
auto rule = input->copy();
using rules::IdentityRuleFn::apply_to;
rule_ptr apply_to_token(const Rule *input, RuleEntryType entry_type) {
for (size_t i = 0; i < tokens.size(); i++)
if (tokens[i].second->operator==(*rule)) {
if (tokens[i].rule->operator==(*input)) {
token_usage_counts[i]++;
return make_shared<Symbol>(i, SymbolOptionAuxToken);
return make_shared<Symbol>(i, true);
}
rule_ptr rule = input->copy();
size_t index = tokens.size();
tokens.push_back({ token_description(rule), rule });
tokens.push_back({
token_description(rule), rule, entry_type,
});
token_usage_counts.push_back(1);
return make_shared<Symbol>(index, SymbolOptionAuxToken);
return make_shared<Symbol>(index, true);
}
rule_ptr default_apply(const Rule *rule) {
auto result = rule->copy();
if (is_token(result))
return apply_to_token(rule);
else
return result;
rule_ptr apply_to(const rules::String *rule) {
return apply_to_token(rule, RuleEntryTypeAnonymous);
}
rule_ptr apply_to(const rules::Pattern *rule) {
return apply_to_token(rule, RuleEntryTypeHidden);
}
rule_ptr apply_to(const rules::Metadata *rule) {
if (is_token(rule->copy()))
return apply_to_token(rule);
if (rule->value_for(rules::IS_TOKEN) > 0)
return apply_to_token(rule->rule.get(), RuleEntryTypeHidden);
else
return rules::IdentityRuleFn::apply_to(rule);
}
public:
vector<size_t> token_usage_counts;
vector<pair<string, rule_ptr>> tokens;
vector<RuleEntry> tokens;
};
static const GrammarError *ubiq_token_err(const string &msg) {
static const GrammarError *ubiq_token_err(const string &message) {
return new GrammarError(GrammarErrorTypeInvalidUbiquitousToken,
"Not a token: " + msg);
"Not a token: " + message);
}
tuple<SyntaxGrammar, LexicalGrammar, const GrammarError *> extract_tokens(
@ -106,51 +103,43 @@ tuple<SyntaxGrammar, LexicalGrammar, const GrammarError *> extract_tokens(
SymbolReplacer symbol_replacer;
TokenExtractor extractor;
vector<pair<string, rule_ptr>> extracted_rules;
for (auto &pair : grammar.rules)
extracted_rules.push_back({ pair.first, extractor.apply(pair.second) });
/*
* First, extract all of the grammar's tokens into the lexical grammar.
*/
vector<RuleEntry> processed_rules;
for (const auto &pair : grammar.rules)
processed_rules.push_back({
pair.first, extractor.apply(pair.second), RuleEntryTypeNamed,
});
lexical_grammar.rules = extractor.tokens;
/*
* If a rule's entire content was extracted as a token and that token didn't
* appear within any other rule, then remove that rule from the syntax
* grammar, giving its name to the token in the lexical grammar. Any symbols
* that pointed to that rule will need to be updated to point to the rule in
* the lexical grammar. Symbols that pointed to later rules will need to have
* their indices decremented.
*/
size_t i = 0;
for (auto &pair : extracted_rules) {
auto &rule = pair.second;
auto symbol = dynamic_pointer_cast<const Symbol>(rule);
if (symbol.get() && symbol->is_auxiliary() &&
for (const RuleEntry &entry : processed_rules) {
auto symbol = dynamic_pointer_cast<const Symbol>(entry.rule);
if (symbol.get() && symbol->is_token && !symbol->is_built_in() &&
extractor.token_usage_counts[symbol->index] == 1) {
lexical_grammar.rules.push_back(
{ pair.first, extractor.tokens[symbol->index].second });
extractor.token_usage_counts[symbol->index] = 0;
symbol_replacer.replacements.insert(
{ Symbol(i),
Symbol(lexical_grammar.rules.size() - 1, SymbolOptionToken) });
lexical_grammar.rules[symbol->index].type = entry.type;
lexical_grammar.rules[symbol->index].name = entry.name;
symbol_replacer.replacements.insert({ Symbol(i), *symbol });
} else {
syntax_grammar.rules.push_back(pair);
syntax_grammar.rules.push_back(entry);
}
i++;
}
for (auto &pair : syntax_grammar.rules)
pair.second = symbol_replacer.apply(pair.second);
lexical_grammar.aux_rules = extractor.tokens;
for (auto &rule : grammar.ubiquitous_tokens) {
if (is_token(rule)) {
lexical_grammar.separators.push_back(rule);
} else {
auto sym = dynamic_pointer_cast<const Symbol>(extractor.apply(rule));
if (!sym.get())
return make_tuple(syntax_grammar, lexical_grammar,
ubiq_token_err(rule->to_string()));
Symbol symbol = symbol_replacer.replace_symbol(*sym);
if (!symbol.is_token())
return make_tuple(
syntax_grammar, lexical_grammar,
ubiq_token_err(syntax_grammar.rules[symbol.index].first));
syntax_grammar.ubiquitous_tokens.insert(symbol);
}
}
/*
* Perform any replacements of symbols needed based on the previous step.
*/
for (RuleEntry &entry : syntax_grammar.rules)
entry.rule = symbol_replacer.apply(entry.rule);
for (auto &symbol_set : grammar.expected_conflicts) {
set<Symbol> new_symbol_set;
@ -159,6 +148,33 @@ tuple<SyntaxGrammar, LexicalGrammar, const GrammarError *> extract_tokens(
syntax_grammar.expected_conflicts.insert(new_symbol_set);
}
/*
* The grammar's ubiquitous tokens can be either token rules or symbols
* pointing to token rules. If they are symbols, then they'll be handled by
* the parser; add them to the syntax grammar's ubiqutous tokens. If they
* are anonymous rules, they can be handled by the lexer; add them to the
* lexical grammar's separator rules.
*/
for (const rule_ptr &rule : grammar.ubiquitous_tokens) {
if (is_token(rule)) {
lexical_grammar.separators.push_back(rule);
continue;
}
auto symbol = dynamic_pointer_cast<const Symbol>(rule);
if (!symbol.get())
return make_tuple(syntax_grammar, lexical_grammar,
ubiq_token_err(rule->to_string()));
Symbol new_symbol = symbol_replacer.replace_symbol(*symbol);
if (!new_symbol.is_token)
return make_tuple(
syntax_grammar, lexical_grammar,
ubiq_token_err(syntax_grammar.rules[new_symbol.index].name));
syntax_grammar.ubiquitous_tokens.insert(new_symbol);
}
return make_tuple(syntax_grammar, lexical_grammar, nullptr);
}

3
src/compiler/prepare_grammar/prepare_grammar.cc
View file

@ -3,8 +3,7 @@
#include "compiler/prepare_grammar/extract_tokens.h"
#include "compiler/prepare_grammar/intern_symbols.h"
#include "compiler/prepare_grammar/prepare_grammar.h"
#include "compiler/lexical_grammar.h"
#include "compiler/syntax_grammar.h"
#include "compiler/prepared_grammar.h"
namespace tree_sitter {
namespace prepare_grammar {

3
src/compiler/prepare_grammar/prepare_grammar.h
View file

@ -2,8 +2,7 @@
#define COMPILER_PREPARE_GRAMMAR_PREPARE_GRAMMAR_H_
#include <utility>
#include "compiler/lexical_grammar.h"
#include "compiler/syntax_grammar.h"
#include "compiler/prepared_grammar.h"
namespace tree_sitter {

40
src/compiler/prepare_grammar/token_description.cc
View file

@ -5,6 +5,7 @@
#include "compiler/rules/seq.h"
#include "compiler/rules/choice.h"
#include "compiler/rules/string.h"
#include "compiler/rules/repeat.h"
#include "compiler/rules/metadata.h"
#include "compiler/util/string_helpers.h"
@ -15,11 +16,12 @@ using std::string;
class TokenDescription : public rules::RuleFn<string> {
string apply_to(const rules::Pattern *rule) {
return "PAT_" + util::escape_string(rule->value);
is_trivial = false;
return rule->value;
}
string apply_to(const rules::String *rule) {
return "STR_" + util::escape_string(rule->value);
return rule->value;
}
string apply_to(const rules::Metadata *rule) {
@ -27,19 +29,41 @@ class TokenDescription : public rules::RuleFn<string> {
}
string apply_to(const rules::Seq *rule) {
return "(seq " + apply(rule->left) + " " + apply(rule->right) + ")";
is_trivial = false;
return apply(rule->left) + apply(rule->right);
}
string apply_to(const rules::Repeat *rule) {
is_trivial = false;
return apply(rule->content) + "*";
}
string apply_to(const rules::Choice *rule) {
string result = "(choice";
for (auto &element : rule->elements)
result += " " + apply(element);
is_trivial = false;
string result = "(";
bool started = false;
for (auto &element : rule->elements) {
if (started)
result += "|";
result += apply(element);
started = true;
}
return result + ")";
}
public:
bool is_trivial;
TokenDescription() : is_trivial(true) {}
};
std::string token_description(const rule_ptr &rule) {
return TokenDescription().apply(rule);
string token_description(const rule_ptr &rule) {
TokenDescription description;
string result = description.apply(rule);
if (description.is_trivial)
return result;
else
return "/" + result + "/";
}
} // namespace prepare_grammar

39
src/compiler/prepared_grammar.h Normal file
View file

@ -0,0 +1,39 @@
#ifndef COMPILER_PREPARED_GRAMMAR_H_
#define COMPILER_PREPARED_GRAMMAR_H_
#include <vector>
#include <string>
#include <set>
#include "tree_sitter/compiler.h"
#include "compiler/rules/symbol.h"
namespace tree_sitter {
enum RuleEntryType {
RuleEntryTypeNamed,
RuleEntryTypeAnonymous,
RuleEntryTypeHidden,
};
struct RuleEntry {
std::string name;
rule_ptr rule;
RuleEntryType type;
};
class SyntaxGrammar {
public:
std::vector<RuleEntry> rules;
std::set<rules::Symbol> ubiquitous_tokens;
std::set<std::set<rules::Symbol>> expected_conflicts;
};
class LexicalGrammar {
public:
std::vector<RuleEntry> rules;
std::vector<rule_ptr> separators;
};
} // namespace tree_sitter
#endif // COMPILER_PREPARED_GRAMMAR_H_

4
src/compiler/rules/built_in_symbols.cc
View file

@ -4,11 +4,11 @@ namespace tree_sitter {
namespace rules {
Symbol END_OF_INPUT() {
return Symbol(-1, SymbolOptionToken);
return Symbol(-1, true);
}
Symbol ERROR() {
return Symbol(-2, SymbolOptionToken);
return Symbol(-2, true);
}
Symbol START() {

2
src/compiler/rules/rules.cc
View file

@ -52,7 +52,7 @@ rule_ptr pattern(const string &value) {
}
rule_ptr str(const string &value) {
return token(prec(1, make_shared<rules::String>(value)));
return make_shared<rules::String>(value);
}
rule_ptr err(const rule_ptr &rule) {

27
src/compiler/rules/symbol.cc
View file

@ -10,16 +10,12 @@ using std::string;
using std::to_string;
using std::hash;
SymbolOption SymbolOptionAuxToken =
SymbolOption(SymbolOptionToken | SymbolOptionAuxiliary);
Symbol::Symbol(int index) : index(index), is_token(false) {}
Symbol::Symbol(int index) : index(index), options(SymbolOption(0)) {}
Symbol::Symbol(int index, SymbolOption options)
: index(index), options(options) {}
Symbol::Symbol(int index, bool is_token) : index(index), is_token(is_token) {}
bool Symbol::operator==(const Symbol &other) const {
return (other.index == index) && (other.options == options);
return (other.index == index) && (other.is_token == is_token);
}
bool Symbol::operator==(const Rule &rule) const {
@ -28,7 +24,7 @@ bool Symbol::operator==(const Rule &rule) const {
}
size_t Symbol::hash_code() const {
return hash<int>()(index) ^ hash<int16_t>()(options);
return hash<int>()(index) ^ hash<bool>()(is_token);
}
rule_ptr Symbol::copy() const {
@ -36,31 +32,22 @@ rule_ptr Symbol::copy() const {
}
string Symbol::to_string() const {
string name = (options & SymbolOptionAuxiliary) ? "aux_" : "";
name += (options & SymbolOptionToken) ? "token" : "sym";
string name = is_token ? "token" : "sym";
return "(" + name + " " + std::to_string(index) + ")";
}
bool Symbol::operator<(const Symbol &other) const {
if (options < other.options)
if (!is_token && other.is_token)
return true;
if (options > other.options)
if (is_token && !other.is_token)
return false;
return (index < other.index);
}
bool Symbol::is_token() const {
return options & SymbolOptionToken;
}
bool Symbol::is_built_in() const {
return index < 0;
}
bool Symbol::is_auxiliary() const {
return options & SymbolOptionAuxiliary;
}
void Symbol::accept(Visitor *visitor) const {
visitor->visit(this);
}

13
src/compiler/rules/symbol.h
View file

@ -7,17 +7,10 @@
namespace tree_sitter {
namespace rules {
typedef enum {
SymbolOptionToken = 1 << 0,
SymbolOptionAuxiliary = 1 << 1,
} SymbolOption;
extern SymbolOption SymbolOptionAuxToken;
class Symbol : public Rule {
public:
explicit Symbol(int index);
Symbol(int index, SymbolOption options);
Symbol(int index, bool is_token);
bool operator==(const Symbol &other) const;
bool operator==(const Rule &other) const;
@ -28,12 +21,10 @@ class Symbol : public Rule {
void accept(Visitor *visitor) const;
bool operator<(const Symbol &other) const;
bool is_token() const;
bool is_built_in() const;
bool is_auxiliary() const;
int index;
SymbolOption options;
bool is_token;
};
} // namespace rules

21
src/compiler/syntax_grammar.cc
View file

@ -1,21 +0,0 @@
#include "compiler/syntax_grammar.h"
#include <vector>
#include <string>
#include <utility>
#include "compiler/rules/symbol.h"
namespace tree_sitter {
using std::string;
const rule_ptr &SyntaxGrammar::rule(const rules::Symbol &symbol) const {
return symbol.is_auxiliary() ? aux_rules[symbol.index].second
: rules[symbol.index].second;
}
const string &SyntaxGrammar::rule_name(const rules::Symbol &symbol) const {
return symbol.is_auxiliary() ? aux_rules[symbol.index].first
: rules[symbol.index].first;
}
} // namespace tree_sitter

26
src/compiler/syntax_grammar.h
View file

@ -1,26 +0,0 @@
#ifndef COMPILER_SYNTAX_GRAMMAR_H_
#define COMPILER_SYNTAX_GRAMMAR_H_
#include <vector>
#include <string>
#include <set>
#include <utility>
#include "tree_sitter/compiler.h"
#include "compiler/rules/symbol.h"
namespace tree_sitter {
class SyntaxGrammar {
public:
const std::string &rule_name(const rules::Symbol &symbol) const;
const rule_ptr &rule(const rules::Symbol &symbol) const;
std::vector<std::pair<std::string, rule_ptr>> rules;
std::vector<std::pair<std::string, rule_ptr>> aux_rules;
std::set<rules::Symbol> ubiquitous_tokens;
std::set<std::set<rules::Symbol>> expected_conflicts;
};
} // namespace tree_sitter
#endif // COMPILER_SYNTAX_GRAMMAR_H_