traxys/tree-sitter
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Remove use of shared_ptr in choice, repeat, and seq factories

Browse source
This commit is contained in:
Max Brunsfeld 2017-03-17 14:28:13 -07:00
parent d9fb863bea
commit b3edd8f749
34 changed files with 366 additions and 381 deletions
Download patch file Download diff file Expand all files Collapse all files

1
src/compiler/build_tables/build_parse_table.cc
View file

@ -24,7 +24,6 @@ using std::map;
using std::string;
using std::to_string;
using std::unordered_map;
using std::make_shared;
using rules::Associativity;
using rules::Symbol;
using rules::END_OF_INPUT;

8
src/compiler/build_tables/lex_item.cc
View file

@ -34,7 +34,7 @@ static CompletionStatus get_completion_status(const rules::Rule &rule) {
},
[](rules::Metadata metadata) {
CompletionStatus result = get_completion_status(metadata.rule);
CompletionStatus result = get_completion_status(*metadata.rule);
if (result.is_done && result.precedence.empty && metadata.params.has_precedence) {
result.precedence.add(metadata.params.precedence);
}
@ -42,13 +42,13 @@ static CompletionStatus get_completion_status(const rules::Rule &rule) {
},
[](rules::Repeat repeat) {
return get_completion_status(repeat.rule);
return get_completion_status(*repeat.rule);
},
[](rules::Seq sequence) {
CompletionStatus left_status = get_completion_status(sequence.left);
CompletionStatus left_status = get_completion_status(*sequence.left);
if (left_status.is_done) {
return get_completion_status(sequence.right);
return get_completion_status(*sequence.right);
} else {
return CompletionStatus{false, PrecedenceRange()};
}

14
src/compiler/build_tables/lex_item_transitions.cc
View file

@ -106,33 +106,33 @@ class TransitionBuilder {
[this](const rules::Seq &sequence) {
TransitionMap left_transitions;
TransitionBuilder(&left_transitions, this).apply(sequence.left);
TransitionBuilder(&left_transitions, this).apply(*sequence.left);
for (const auto &pair : left_transitions) {
add_transition(
transitions,
pair.first,
transform_transition(pair.second, [&sequence](Rule rule) -> Rule {
return rules::Seq::build({ rule, sequence.right });
return Rule::seq({rule, *sequence.right});
})
);
}
if (rule_can_be_blank(sequence.left)) {
apply(sequence.right);
if (rule_can_be_blank(*sequence.left)) {
apply(*sequence.right);
}
},
[this](const rules::Repeat &repeat) {
TransitionMap content_transitions;
TransitionBuilder(&content_transitions, this).apply(repeat.rule);
TransitionBuilder(&content_transitions, this).apply(*repeat.rule);
for (const auto &pair : content_transitions) {
add_transition(transitions, pair.first, pair.second);
add_transition(
transitions, pair.first,
transform_transition(pair.second, [&repeat](Rule item_rule) {
return rules::Seq::build({ item_rule, repeat });
return Rule::seq({ item_rule, repeat });
})
);
}
@ -151,7 +151,7 @@ class TransitionBuilder {
params.is_active = true;
TransitionMap content_transitions;
TransitionBuilder(&content_transitions, this).apply(metadata.rule);
TransitionBuilder(&content_transitions, this).apply(*metadata.rule);
for (const auto &pair : content_transitions) {
add_transition(

8
src/compiler/build_tables/lex_table_builder.cc
View file

@ -36,7 +36,7 @@ class StartingCharacterAggregator {
void apply(const Rule &rule) {
rule.match(
[this](const Seq &sequence) {
apply(sequence.left);
apply(*sequence.left);
},
[this](const rules::Choice &rule) {
@ -46,11 +46,11 @@ class StartingCharacterAggregator {
},
[this](const rules::Repeat &rule) {
apply(rule.rule);
apply(*rule.rule);
},
[this](const rules::Metadata &rule) {
apply(rule.rule);
apply(*rule.rule);
},
[this](const rules::CharacterSet &rule) {
@ -299,7 +299,7 @@ class LexTableBuilderImpl : public LexTableBuilder {
result.entries.insert(LexItem(
symbol,
Metadata::separator(
Seq::build({
Rule::seq({
separator_rule,
Metadata::main_token(rule)
})

6
src/compiler/build_tables/rule_can_be_blank.cc
View file

@ -15,11 +15,11 @@ bool rule_can_be_blank(const rules::Rule &rule) {
},
[](rules::Repeat repeat) {
return rule_can_be_blank(repeat.rule);
return rule_can_be_blank(*repeat.rule);
},
[](rules::Metadata metadata) {
return rule_can_be_blank(metadata.rule);
return rule_can_be_blank(*metadata.rule);
},
[](rules::Choice choice) {
@ -32,7 +32,7 @@ bool rule_can_be_blank(const rules::Rule &rule) {
},
[](rules::Seq seq) {
return rule_can_be_blank(seq.left) && rule_can_be_blank(seq.right);
return rule_can_be_blank(*seq.left) && rule_can_be_blank(*seq.right);
},
[](auto) { return false; }

11
src/compiler/parse_grammar.cc
View file

@ -14,9 +14,6 @@ using std::unordered_set;
using std::pair;
using rules::Rule;
using rules::Blank;
using rules::Choice;
using rules::Repeat;
using rules::Seq;
using rules::Metadata;
using rules::Pattern;
using rules::String;
@ -70,7 +67,7 @@ ParseRuleResult parse_rule(json_value *rule_json) {
}
members.push_back(result.rule);
}
return Rule(Choice{members});
return Rule::choice(members);
}
if (type == "SEQ") {
@ -88,7 +85,7 @@ ParseRuleResult parse_rule(json_value *rule_json) {
}
members.push_back(result.rule);
}
return *Seq::build(members);
return Rule::seq(members);
}
if (type == "REPEAT") {
@ -97,7 +94,7 @@ ParseRuleResult parse_rule(json_value *rule_json) {
if (!result.error_message.empty()) {
return "Invalid repeat content: " + result.error_message;
}
return Rule(Choice{{Repeat{result.rule}, Blank{}}});
return Rule::choice({Rule::repeat(result.rule), Blank{}});
}
if (type == "REPEAT1") {
@ -106,7 +103,7 @@ ParseRuleResult parse_rule(json_value *rule_json) {
if (!result.error_message.empty()) {
return "Invalid repeat content: " + result.error_message;
}
return Rule(Repeat{result.rule});
return Rule::repeat(result.rule);
}
if (type == "TOKEN") {

12
src/compiler/prepare_grammar/expand_repeats.cc
View file

@ -28,17 +28,17 @@ class ExpandRepeats {
[&](const rules::Symbol &symbol) { return symbol; },
[&](const rules::Choice &choice) {
vector<rules::Rule> elements;
vector<Rule> elements;
for (const auto &element : choice.elements) {
elements.push_back(apply(element));
}
return rules::Choice::build(elements);
return Rule::choice(elements);
},
[&](const rules::Seq &sequence) {
return rules::Seq{
apply(sequence.left),
apply(sequence.right)
apply(*sequence.left),
apply(*sequence.right)
};
},
@ -49,7 +49,7 @@ class ExpandRepeats {
}
}
Rule inner_rule = apply(repeat.rule);
Rule inner_rule = apply(*repeat.rule);
size_t index = aux_rules.size();
string helper_rule_name = rule_name + "_repeat" + to_string(++repeat_count);
Symbol repeat_symbol = Symbol::non_terminal(offset + index);
@ -66,7 +66,7 @@ class ExpandRepeats {
},
[&](const rules::Metadata &metadata) {
return rules::Metadata{apply(metadata.rule), metadata.params};
return rules::Metadata{apply(*metadata.rule), metadata.params};
},
[](auto) {

12
src/compiler/prepare_grammar/expand_tokens.cc
View file

@ -33,7 +33,7 @@ ExpandTokenResult expand_token(const rules::Rule &rule) {
elements.push_back(rules::CharacterSet().include(el));
}
return *rules::Seq::build(elements);
return Rule::seq(elements);
},
[](const rules::Pattern &pattern) -> ExpandTokenResult {
@ -43,21 +43,21 @@ ExpandTokenResult expand_token(const rules::Rule &rule) {
},
[](const rules::Repeat &rule) -> ExpandTokenResult {
auto result = expand_token(rule.rule);
auto result = expand_token(*rule.rule);
if (result.error) return result.error;
return *rules::Repeat::build(result.rule);
return Rule::repeat(result.rule);
},
[](const rules::Metadata &rule) -> ExpandTokenResult {
auto result = expand_token(rule.rule);
auto result = expand_token(*rule.rule);
if (result.error) return result.error;
return Rule(rules::Metadata{result.rule, rule.params});
},
[](const rules::Seq &rule) -> ExpandTokenResult {
auto left_result = expand_token(rule.left);
auto left_result = expand_token(*rule.left);
if (left_result.error) return left_result.error;
auto right_result = expand_token(rule.right);
auto right_result = expand_token(*rule.right);
if (right_result.error) return right_result.error;
return Rule(rules::Seq{left_result.rule, right_result.rule});
},

8
src/compiler/prepare_grammar/extract_choices.cc
View file

@ -13,9 +13,9 @@ vector<Rule> extract_choices(const Rule &rule) {
return rule.match(
[](const rules::Seq &sequence) {
vector<Rule> result;
for (auto &left_entry : extract_choices(sequence.left)) {
for (auto &right_entry : extract_choices(sequence.right)) {
result.push_back(rules::Seq::build({left_entry, right_entry}));
for (auto &left_entry : extract_choices(*sequence.left)) {
for (auto &right_entry : extract_choices(*sequence.right)) {
result.push_back(rules::Rule::seq({left_entry, right_entry}));
}
}
return result;
@ -23,7 +23,7 @@ vector<Rule> extract_choices(const Rule &rule) {
[](const rules::Metadata &rule) {
vector<Rule> result;
for (auto &entry : extract_choices(rule.rule)) {
for (auto &entry : extract_choices(*rule.rule)) {
result.push_back(rules::Metadata{entry, rule.params});
}
return result;

23
src/compiler/prepare_grammar/extract_tokens.cc
View file

@ -43,22 +43,22 @@ class SymbolReplacer {
for (const auto &element : choice.elements) {
elements.push_back(apply(element));
}
return rules::Choice::build(elements);
return Rule::choice(elements);
},
[this](const rules::Seq &sequence) {
return rules::Seq{
apply(sequence.left),
apply(sequence.right)
apply(*sequence.left),
apply(*sequence.right)
};
},
[this](const rules::Repeat &repeat) {
return rules::Repeat{apply(repeat.rule)};
return Rule::repeat(apply(*repeat.rule));
},
[this](const rules::Metadata &metadata) {
return rules::Metadata{apply(metadata.rule), metadata.params};
return rules::Metadata{apply(*metadata.rule), metadata.params};
},
[](auto) {
@ -114,9 +114,9 @@ class TokenExtractor {
[this](const rules::Metadata &rule) -> Rule {
if (rule.params.is_token) {
return extract_token(rule.rule, VariableTypeAuxiliary);
return extract_token(*rule.rule, VariableTypeAuxiliary);
} else {
return rules::Metadata{apply(rule.rule), rule.params};
return rules::Metadata{apply(*rule.rule), rule.params};
}
},
@ -129,14 +129,11 @@ class TokenExtractor {
},
[this](const rules::Repeat &rule) {
return rules::Repeat{apply(rule.rule)};
return Rule::repeat(apply(*rule.rule));
},
[this](const rules::Seq &rule) {
return rules::Seq{
apply(rule.left),
apply(rule.right)
};
return Rule::seq({apply(*rule.left), apply(*rule.right)});
},
[this](const rules::Choice &rule) {
@ -144,7 +141,7 @@ class TokenExtractor {
for (const auto &element : rule.elements) {
elements.push_back(apply(element));
}
return rules::Choice::build(elements);
return Rule::choice(elements);
},
[](const rules::Symbol &symbol) {

15
src/compiler/prepare_grammar/flatten_grammar.cc
View file

@ -34,12 +34,15 @@ class FlattenRule {
},
[&](const rules::Metadata &metadata) {
if (metadata.params.has_precedence)
if (metadata.params.has_precedence) {
precedence_stack.push_back(metadata.params.precedence);
if (metadata.params.has_associativity)
associativity_stack.push_back(metadata.params.associativity);
}
apply(metadata.rule);
if (metadata.params.has_associativity) {
associativity_stack.push_back(metadata.params.associativity);
}
apply(*metadata.rule);
if (metadata.params.has_precedence) {
last_precedence = precedence_stack.back();
@ -55,10 +58,10 @@ class FlattenRule {
},
[&](const rules::Seq &sequence) {
apply(sequence.left);
apply(*sequence.left);
last_precedence = 0;
last_associativity = rules::AssociativityNone;
apply(sequence.right);
apply(*sequence.right);
},
[&](const rules::Blank &blank) {},

10
src/compiler/prepare_grammar/intern_symbols.cc
View file

@ -14,7 +14,6 @@ using std::string;
using std::vector;
using std::set;
using std::pair;
using std::make_shared;
using rules::Symbol;
using rules::Rule;
@ -40,18 +39,15 @@ class SymbolInterner {
},
[&](const rules::Seq &sequence) {
return rules::Seq{
apply(sequence.left),
apply(sequence.right)
};
return rules::Seq{apply(*sequence.left), apply(*sequence.right)};
},
[&](const rules::Repeat &repeat) {
return rules::Repeat{apply(repeat.rule)};
return rules::Repeat{apply(*repeat.rule)};
},
[&](const rules::Metadata &metadata) {
return rules::Metadata{apply(metadata.rule), metadata.params};
return rules::Metadata{apply(*metadata.rule), metadata.params};
},
[](auto) {

2
src/compiler/prepare_grammar/normalize_rules.cc
View file

@ -11,7 +11,7 @@ LexicalGrammar normalize_rules(const LexicalGrammar &input_grammar) {
LexicalGrammar result(input_grammar);
for (LexicalVariable &variable : result.variables) {
variable.rule = rules::Choice::build(extract_choices(variable.rule));
variable.rule = Rule::choice(extract_choices(variable.rule));
}
return result;

16
src/compiler/prepare_grammar/parse_regex.cc
View file

@ -12,13 +12,9 @@ namespace prepare_grammar {
using std::string;
using std::vector;
using std::pair;
using std::make_shared;
using rules::CharacterSet;
using rules::Blank;
using rules::Rule;
using rules::Choice;
using rules::Seq;
using rules::Repeat;
class PatternParser {
public:
@ -45,7 +41,7 @@ class PatternParser {
}
choices.push_back(pair.first);
} while (has_more_input());
return {Choice::build(choices), CompileError::none()};
return {Rule::choice(choices), CompileError::none()};
}
private:
@ -60,7 +56,7 @@ class PatternParser {
if (pair.second) {
return {Blank{}, pair.second};
}
result = Seq::build({result, pair.first});
result = Rule::seq({result, pair.first});
} while (has_more_input());
return { result, CompileError::none() };
}
@ -76,18 +72,18 @@ class PatternParser {
switch (peek()) {
case '*':
next();
result = Choice::build({
Repeat{result},
result = Rule::choice({
Rule::repeat(result),
Blank{}
});
break;
case '+':
next();
result = Repeat{result};
result = Rule::repeat(result);
break;
case '?':
next();
result = Choice::build({result, Blank{}});
result = Rule::choice({result, Blank{}});
break;
}
}

6
src/compiler/prepare_grammar/token_description.cc
View file

@ -31,17 +31,17 @@ class TokenDescription {
},
[&](const rules::Metadata &rule) {
return apply(rule.rule);
return apply(*rule.rule);
},
[&](const rules::Seq &rule) {
is_trivial = false;
return apply(rule.left) + apply(rule.right);
return apply(*rule.left) + apply(*rule.right);
},
[&](const rules::Repeat &rule) {
is_trivial = false;
return apply(rule.rule) + "+";
return apply(*rule.rule) + "+";
},
[&](const rules::Choice &rule) {

160
src/compiler/rule.cc
View file

@ -5,28 +5,29 @@ namespace tree_sitter {
namespace rules {
using std::move;
using std::vector;
using util::hash_combine;
Rule::Rule(const Rule &other) : blank(Blank{}), type(BlankType) {
Rule::Rule(const Rule &other) : blank_(Blank{}), type(BlankType) {
*this = other;
}
Rule::Rule(Rule &&other) noexcept : blank(Blank{}), type(BlankType) {
Rule::Rule(Rule &&other) noexcept : blank_(Blank{}), type(BlankType) {
*this = move(other);
}
static void destroy_value(Rule *rule) {
switch (rule->type) {
case Rule::BlankType: return rule->blank.~Blank();
case Rule::CharacterSetType: return rule->character_set.~CharacterSet();
case Rule::StringType: return rule->string .~String();
case Rule::PatternType: return rule->pattern .~Pattern();
case Rule::NamedSymbolType: return rule->named_symbol.~NamedSymbol();
case Rule::SymbolType: return rule->symbol .~Symbol();
case Rule::ChoiceType: return rule->choice .~Choice();
case Rule::MetadataType: return rule->metadata .~Metadata();
case Rule::RepeatType: return rule->repeat .~Repeat();
case Rule::SeqType: return rule->seq .~Seq();
case Rule::BlankType: return rule->blank_.~Blank();
case Rule::CharacterSetType: return rule->character_set_.~CharacterSet();
case Rule::StringType: return rule->string_ .~String();
case Rule::PatternType: return rule->pattern_ .~Pattern();
case Rule::NamedSymbolType: return rule->named_symbol_.~NamedSymbol();
case Rule::SymbolType: return rule->symbol_ .~Symbol();
case Rule::ChoiceType: return rule->choice_ .~Choice();
case Rule::MetadataType: return rule->metadata_ .~Metadata();
case Rule::RepeatType: return rule->repeat_ .~Repeat();
case Rule::SeqType: return rule->seq_ .~Seq();
}
}
@ -35,34 +36,34 @@ Rule &Rule::operator=(const Rule &other) {
type = other.type;
switch (type) {
case BlankType:
new (&blank) Blank(other.blank);
new (&blank_) Blank(other.blank_);
break;
case CharacterSetType:
new (&character_set) CharacterSet(other.character_set);
new (&character_set_) CharacterSet(other.character_set_);
break;
case StringType:
new (&string) String(other.string);
new (&string_) String(other.string_);
break;
case PatternType:
new (&pattern) Pattern(other.pattern);
new (&pattern_) Pattern(other.pattern_);
break;
case NamedSymbolType:
new (&named_symbol) NamedSymbol(other.named_symbol);
new (&named_symbol_) NamedSymbol(other.named_symbol_);
break;
case SymbolType:
new (&symbol) Symbol(other.symbol);
new (&symbol_) Symbol(other.symbol_);
break;
case ChoiceType:
new (&choice) Choice(other.choice);
new (&choice_) Choice(other.choice_);
break;
case MetadataType:
new (&metadata) Metadata(other.metadata);
new (&metadata_) Metadata(other.metadata_);
break;
case RepeatType:
new (&repeat) Repeat(other.repeat);
new (&repeat_) Repeat(other.repeat_);
break;
case SeqType:
new (&seq) Seq(other.seq);
new (&seq_) Seq(other.seq_);
break;
}
return *this;
@ -73,38 +74,38 @@ Rule &Rule::operator=(Rule &&other) noexcept {
type = other.type;
switch (type) {
case BlankType:
new (&blank) Blank(move(other.blank));
new (&blank_) Blank(move(other.blank_));
break;
case CharacterSetType:
new (&character_set) CharacterSet(move(other.character_set));
new (&character_set_) CharacterSet(move(other.character_set_));
break;
case StringType:
new (&string) String(move(other.string));
new (&string_) String(move(other.string_));
break;
case PatternType:
new (&pattern) Pattern(move(other.pattern));
new (&pattern_) Pattern(move(other.pattern_));
break;
case NamedSymbolType:
new (&named_symbol) NamedSymbol(move(other.named_symbol));
new (&named_symbol_) NamedSymbol(move(other.named_symbol_));
break;
case SymbolType:
new (&symbol) Symbol(move(other.symbol));
new (&symbol_) Symbol(move(other.symbol_));
break;
case ChoiceType:
new (&choice) Choice(move(other.choice));
new (&choice_) Choice(move(other.choice_));
break;
case MetadataType:
new (&metadata) Metadata(move(other.metadata));
new (&metadata_) Metadata(move(other.metadata_));
break;
case RepeatType:
new (&repeat) Repeat(move(other.repeat));
new (&repeat_) Repeat(move(other.repeat_));
break;
case SeqType:
new (&seq) Seq(move(other.seq));
new (&seq_) Seq(move(other.seq_));
break;
}
other.type = BlankType;
other.blank = Blank{};
other.blank_ = Blank{};
return *this;
}
@ -115,16 +116,16 @@ Rule::~Rule() noexcept {
bool Rule::operator==(const Rule &other) const {
if (type != other.type) return false;
switch (type) {
case Rule::CharacterSetType: return character_set == other.character_set;
case Rule::StringType: return string == other.string;
case Rule::PatternType: return pattern == other.pattern;
case Rule::NamedSymbolType: return named_symbol == other.named_symbol;
case Rule::SymbolType: return symbol == other.symbol;
case Rule::ChoiceType: return choice == other.choice;
case Rule::MetadataType: return metadata == other.metadata;
case Rule::RepeatType: return repeat == other.repeat;
case Rule::SeqType: return seq == other.seq;
default: return blank == other.blank;
case Rule::CharacterSetType: return character_set_ == other.character_set_;
case Rule::StringType: return string_ == other.string_;
case Rule::PatternType: return pattern_ == other.pattern_;
case Rule::NamedSymbolType: return named_symbol_ == other.named_symbol_;
case Rule::SymbolType: return symbol_ == other.symbol_;
case Rule::ChoiceType: return choice_ == other.choice_;
case Rule::MetadataType: return metadata_ == other.metadata_;
case Rule::RepeatType: return repeat_ == other.repeat_;
case Rule::SeqType: return seq_ == other.seq_;
default: return blank_ == other.blank_;
}
}
@ -138,7 +139,58 @@ template <>
bool Rule::is<Repeat>() const { return type == RepeatType; }
template <>
const Symbol & Rule::get_unchecked<Symbol>() const { return symbol; }
const Symbol & Rule::get_unchecked<Symbol>() const { return symbol_; }
static inline void add_choice_element(std::vector<Rule> *elements, const Rule &new_rule) {
new_rule.match(
[elements](Choice choice) {
for (auto &element : choice.elements) {
add_choice_element(elements, element);
}
},
[elements](auto rule) {
for (auto &element : *elements) {
if (element == rule) return;
}
elements->push_back(rule);
}
);
}
Rule Rule::choice(const vector<Rule> &rules) {
vector<Rule> elements;
for (auto &element : rules) {
add_choice_element(&elements, element);
}
return (elements.size() == 1) ? elements.front() : Choice{elements};
}
Rule Rule::repeat(const Rule &rule) {
return rule.is<Repeat>() ? rule : Repeat{rule};
}
Rule Rule::seq(const vector<Rule> &rules) {
Rule result;
for (const auto &rule : rules) {
rule.match(
[](Blank) {},
[&](Metadata metadata) {
if (!metadata.rule->is<Blank>()) {
result = Seq{result, rule};
}
},
[&](auto) {
if (result.is<Blank>()) {
result = rule;
} else {
result = Seq{result, rule};
}
}
);
}
return result;
}
} // namespace rules
} // namespace tree_sitter
@ -219,16 +271,16 @@ size_t hash<Metadata>::operator()(const Metadata &metadata) const {
size_t hash<Rule>::operator()(const Rule &rule) const {
size_t result = hash<int>()(rule.type);
switch (rule.type) {
case Rule::CharacterSetType: return result ^ hash<CharacterSet>()(rule.character_set);
case Rule::StringType: return result ^ hash<String>()(rule.string);
case Rule::PatternType: return result ^ hash<Pattern>()(rule.pattern);
case Rule::NamedSymbolType: return result ^ hash<NamedSymbol>()(rule.named_symbol);
case Rule::SymbolType: return result ^ hash<Symbol>()(rule.symbol);
case Rule::ChoiceType: return result ^ hash<Choice>()(rule.choice);
case Rule::MetadataType: return result ^ hash<Metadata>()(rule.metadata);
case Rule::RepeatType: return result ^ hash<Repeat>()(rule.repeat);
case Rule::SeqType: return result ^ hash<Seq>()(rule.seq);
default: return result ^ hash<Blank>()(rule.blank);
case Rule::CharacterSetType: return result ^ hash<CharacterSet>()(rule.character_set_);
case Rule::StringType: return result ^ hash<String>()(rule.string_);
case Rule::PatternType: return result ^ hash<Pattern>()(rule.pattern_);
case Rule::NamedSymbolType: return result ^ hash<NamedSymbol>()(rule.named_symbol_);
case Rule::SymbolType: return result ^ hash<Symbol>()(rule.symbol_);
case Rule::ChoiceType: return result ^ hash<Choice>()(rule.choice_);
case Rule::MetadataType: return result ^ hash<Metadata>()(rule.metadata_);
case Rule::RepeatType: return result ^ hash<Repeat>()(rule.repeat_);
case Rule::SeqType: return result ^ hash<Seq>()(rule.seq_);
default: return result ^ hash<Blank>()(rule.blank_);
}
}

71
src/compiler/rule.h
View file

@ -2,6 +2,7 @@
#define COMPILER_RULE_H_
#include <memory>
#include <vector>
#include "compiler/util/make_visitor.h"
#include "compiler/util/hash_combine.h"
#include "compiler/rules/blank.h"
@ -20,16 +21,16 @@ namespace rules {
struct Rule {
union {
Blank blank;
CharacterSet character_set;
String string;
Pattern pattern;
NamedSymbol named_symbol;
Symbol symbol;
Choice choice;
Metadata metadata;
Repeat repeat;
Seq seq;
Blank blank_;
CharacterSet character_set_;
String string_;
Pattern pattern_;
NamedSymbol named_symbol_;
Symbol symbol_;
Choice choice_;
Metadata metadata_;
Repeat repeat_;
Seq seq_;
};
enum {
@ -45,19 +46,17 @@ struct Rule {
SeqType,
} type;
Rule() : blank(Blank{}), type(BlankType) {};
Rule(const Blank &value) : blank(value), type(BlankType) {};
Rule(const CharacterSet &value) : character_set(value), type(CharacterSetType) {};
Rule(const String &value) : string(value), type(StringType) {};
Rule(const Pattern &value) : pattern(value), type(PatternType) {};
Rule(const NamedSymbol &value) : named_symbol(value), type(NamedSymbolType) {};
Rule(const Symbol &value) : symbol(value), type(SymbolType) {};
Rule(const Choice &value) : choice(value), type(ChoiceType) {};
Rule(const Metadata &value) : metadata(value), type(MetadataType) {};
Rule(const Repeat &value) : repeat(value), type(RepeatType) {};
Rule(const Seq &value) : seq(value), type(SeqType) {};
Rule(const std::shared_ptr<Rule> &value) : Rule(*value) {}
Rule() : blank_(Blank{}), type(BlankType) {};
Rule(const Blank &value) : blank_(value), type(BlankType) {};
Rule(const CharacterSet &value) : character_set_(value), type(CharacterSetType) {};
Rule(const String &value) : string_(value), type(StringType) {};
Rule(const Pattern &value) : pattern_(value), type(PatternType) {};
Rule(const NamedSymbol &value) : named_symbol_(value), type(NamedSymbolType) {};
Rule(const Symbol &value) : symbol_(value), type(SymbolType) {};
Rule(const Choice &value) : choice_(value), type(ChoiceType) {};
Rule(const Metadata &value) : metadata_(value), type(MetadataType) {};
Rule(const Repeat &value) : repeat_(value), type(RepeatType) {};
Rule(const Seq &value) : seq_(value), type(SeqType) {};
Rule(const Rule &other);
Rule(Rule &&other) noexcept;
@ -65,6 +64,10 @@ struct Rule {
Rule &operator=(Rule &&other) noexcept;
~Rule() noexcept;
static Rule choice(const std::vector<Rule> &rules);
static Rule seq(const std::vector<Rule> &rules);
static Rule repeat(const Rule &rule);
template <typename RuleType>
bool is() const;
@ -72,18 +75,18 @@ struct Rule {
const RuleType & get_unchecked() const;
template <typename FunctionType>
inline auto accept(FunctionType function) const -> decltype(function(blank)) {
inline auto accept(FunctionType function) const -> decltype(function(blank_)) {
switch (type) {
case CharacterSetType: return function(character_set);
case StringType: return function(string);
case PatternType: return function(pattern);
case NamedSymbolType: return function(named_symbol);
case SymbolType: return function(symbol);
case ChoiceType: return function(choice);
case MetadataType: return function(metadata);
case RepeatType: return function(repeat);
case SeqType: return function(seq);
default: return function(blank);
case CharacterSetType: return function(character_set_);
case StringType: return function(string_);
case PatternType: return function(pattern_);
case NamedSymbolType: return function(named_symbol_);
case SymbolType: return function(symbol_);
case ChoiceType: return function(choice_);
case MetadataType: return function(metadata_);
case RepeatType: return function(repeat_);
case SeqType: return function(seq_);
default: return function(blank_);
}
}

27
src/compiler/rules/choice.cc
View file

@ -4,33 +4,6 @@
namespace tree_sitter {
namespace rules {
static inline void add_choice_element(std::vector<Rule> *elements, const Rule &new_rule) {
new_rule.match(
[elements](Choice choice) {
for (auto &element : choice.elements) {
add_choice_element(elements, element);
}
},
[elements](auto rule) {
for (auto &element : *elements) {
if (element == rule) return;
}
elements->push_back(rule);
}
);
}
std::shared_ptr<Rule> Choice::build(const std::vector<Rule> &rules) {
std::vector<Rule> elements;
for (auto &element : rules) {
add_choice_element(&elements, element);
}
return std::make_shared<Rule>(
(elements.size() == 1) ? elements.front() : Choice{elements}
);
}
bool Choice::operator==(const Choice &other) const {
return elements == other.elements;
}

1
src/compiler/rules/choice.h
View file

@ -12,7 +12,6 @@ struct Rule;
struct Choice {
std::vector<Rule> elements;
static std::shared_ptr<Rule> build(const std::vector<Rule> &rules);
bool operator==(const Choice &other) const;
};

6
src/compiler/rules/repeat.cc
View file

@ -11,11 +11,5 @@ bool Repeat::operator==(const Repeat &other) const {
return rule->operator==(*other.rule);
}
std::shared_ptr<Rule> Repeat::build(const Rule &rule) {
return std::make_shared<Rule>(
rule.is<Repeat>() ? rule : Repeat{rule}
);
}
} // namespace rules
} // namespace tree_sitter

1
src/compiler/rules/repeat.h
View file

@ -12,7 +12,6 @@ struct Repeat {
std::shared_ptr<Rule> rule;
explicit Repeat(const Rule &rule);
static std::shared_ptr<Rule> build(const Rule &rule);
bool operator==(const Repeat &other) const;
};

22
src/compiler/rules/seq.cc
View file

@ -8,28 +8,6 @@ Seq::Seq(const Rule &left, const Rule &right) :
left(std::make_shared<Rule>(left)),
right(std::make_shared<Rule>(right)) {}
std::shared_ptr<Rule> Seq::build(const std::vector<Rule> &rules) {
Rule result;
for (const auto &rule : rules) {
rule.match(
[](Blank) {},
[&](Metadata metadata) {
if (!metadata.rule->is<Blank>()) {
result = Seq{result, rule};
}
},
[&](auto) {
if (result.is<Blank>()) {
result = rule;
} else {
result = Seq{result, rule};
}
}
);
}
return std::make_shared<Rule>(result);
}
bool Seq::operator==(const Seq &other) const {
return left->operator==(*other.left) && right->operator==(*other.right);
}

1
src/compiler/rules/seq.h
View file

@ -14,7 +14,6 @@ struct Seq {
std::shared_ptr<Rule> right;
Seq(const Rule &left, const Rule &right);
static std::shared_ptr<Rule> build(const std::vector<Rule> &rules);
bool operator==(const Seq &other) const;
};

54
test/compiler/build_tables/lex_item_test.cc
View file

@ -86,7 +86,7 @@ describe("LexItemSet::transitions()", [&]() {
it("handles sequences", [&]() {
LexItemSet item_set({
LexItem(Symbol::non_terminal(1), Seq::build({
LexItem(Symbol::non_terminal(1), Rule::seq({
CharacterSet{{ 'w' }},
CharacterSet{{ 'x' }},
CharacterSet{{ 'y' }},
@ -101,7 +101,7 @@ describe("LexItemSet::transitions()", [&]() {
CharacterSet().include('w'),
Transition{
LexItemSet({
LexItem(Symbol::non_terminal(1), Seq::build({
LexItem(Symbol::non_terminal(1), Rule::seq({
CharacterSet{{ 'x' }},
CharacterSet{{ 'y' }},
CharacterSet{{ 'z' }},
@ -116,10 +116,10 @@ describe("LexItemSet::transitions()", [&]() {
it("handles sequences with nested precedence", [&]() {
LexItemSet item_set({
LexItem(Symbol::non_terminal(1), Seq::build({
Metadata::prec(3, Seq::build({
LexItem(Symbol::non_terminal(1), Rule::seq({
Metadata::prec(3, Rule::seq({
CharacterSet{{ 'v' }},
Metadata::prec(4, Seq::build({
Metadata::prec(4, Rule::seq({
CharacterSet{{ 'w' }},
CharacterSet{{ 'x' }} })),
CharacterSet{{ 'y' }} })),
@ -138,9 +138,9 @@ describe("LexItemSet::transitions()", [&]() {
// The outer precedence is now 'active', because we are within its
// contained rule.
LexItemSet({
LexItem(Symbol::non_terminal(1), Seq::build({
Metadata::active_prec(3, Seq::build({
Metadata::prec(4, Seq::build({
LexItem(Symbol::non_terminal(1), Rule::seq({
Metadata::active_prec(3, Rule::seq({
Metadata::prec(4, Rule::seq({
CharacterSet{{ 'w' }},
CharacterSet{{ 'x' }}
})),
@ -168,8 +168,8 @@ describe("LexItemSet::transitions()", [&]() {
Transition{
// The inner precedence is now 'active'
LexItemSet({
LexItem(Symbol::non_terminal(1), Seq::build({
Metadata::active_prec(3, Seq::build({
LexItem(Symbol::non_terminal(1), Rule::seq({
Metadata::active_prec(3, Rule::seq({
Metadata::active_prec(4, CharacterSet{{'x'}}),
CharacterSet{{'y'}}
})),
@ -194,7 +194,7 @@ describe("LexItemSet::transitions()", [&]() {
CharacterSet().include('x'),
Transition{
LexItemSet({
LexItem(Symbol::non_terminal(1), Seq::build({
LexItem(Symbol::non_terminal(1), Rule::seq({
Metadata::active_prec(3, CharacterSet{{'y'}}),
CharacterSet{{'z'}},
})),
@ -228,8 +228,8 @@ describe("LexItemSet::transitions()", [&]() {
it("handles sequences where the left hand side can be blank", [&]() {
LexItemSet item_set({
LexItem(Symbol::non_terminal(1), Seq::build({
Choice::build({
LexItem(Symbol::non_terminal(1), Rule::seq({
Rule::choice({
CharacterSet{{ 'x' }},
Blank{},
}),
@ -245,7 +245,7 @@ describe("LexItemSet::transitions()", [&]() {
CharacterSet().include('x'),
Transition{
LexItemSet({
LexItem(Symbol::non_terminal(1), Seq::build({
LexItem(Symbol::non_terminal(1), Rule::seq({
CharacterSet{{ 'y' }},
CharacterSet{{ 'z' }},
})),
@ -277,7 +277,7 @@ describe("LexItemSet::transitions()", [&]() {
it("handles repeats", [&]() {
LexItemSet item_set({
LexItem(Symbol::non_terminal(1), Repeat{Seq::build({
LexItem(Symbol::non_terminal(1), Repeat{Rule::seq({
CharacterSet{{ 'a' }},
CharacterSet{{ 'b' }},
})}),
@ -291,9 +291,9 @@ describe("LexItemSet::transitions()", [&]() {
CharacterSet().include('a'),
Transition{
LexItemSet({
LexItem(Symbol::non_terminal(1), Seq::build({
LexItem(Symbol::non_terminal(1), Rule::seq({
CharacterSet{{ 'b' }},
Repeat{Seq::build({
Repeat{Rule::seq({
CharacterSet{{ 'a' }},
CharacterSet{{ 'b' }},
})}
@ -342,12 +342,12 @@ describe("LexItemSet::transitions()", [&]() {
it("handles choices between overlapping character sets", [&]() {
LexItemSet item_set({
LexItem(Symbol::non_terminal(1), Choice::build({
Metadata::active_prec(2, Seq::build({
LexItem(Symbol::non_terminal(1), Rule::choice({
Metadata::active_prec(2, Rule::seq({
CharacterSet{{ 'a', 'b', 'c', 'd' }},
CharacterSet{{ 'x' }},
})),
Metadata::active_prec(3, Seq::build({
Metadata::active_prec(3, Rule::seq({
CharacterSet{{ 'c', 'd', 'e', 'f' }},
CharacterSet{{ 'y' }},
})),
@ -393,12 +393,12 @@ describe("LexItemSet::transitions()", [&]() {
it("handles choices between a subset and a superset of characters", [&]() {
LexItemSet item_set({
LexItem(Symbol::non_terminal(1), Choice::build({
Seq::build({
LexItem(Symbol::non_terminal(1), Rule::choice({
Rule::seq({
CharacterSet{{ 'b', 'c', 'd' }},
CharacterSet{{ 'x' }},
}),
Seq::build({
Rule::seq({
CharacterSet{{ 'a', 'b', 'c', 'd', 'e', 'f' }},
CharacterSet{{ 'y' }},
}),
@ -434,10 +434,10 @@ describe("LexItemSet::transitions()", [&]() {
it("handles choices between whitelisted and blacklisted character sets", [&]() {
LexItemSet item_set({
LexItem(Symbol::non_terminal(1), Seq::build({
Choice::build({
LexItem(Symbol::non_terminal(1), Rule::seq({
Rule::choice({
CharacterSet().include_all().exclude('/'),
Seq::build({
Rule::seq({
CharacterSet{{ '\\' }},
CharacterSet{{ '/' }},
}),
@ -464,7 +464,7 @@ describe("LexItemSet::transitions()", [&]() {
Transition{
LexItemSet({
LexItem(Symbol::non_terminal(1), CharacterSet{{ '/' }}),
LexItem(Symbol::non_terminal(1), Seq::build({ CharacterSet{{ '/' }}, CharacterSet{{ '/' }} })),
LexItem(Symbol::non_terminal(1), Rule::seq({ CharacterSet{{ '/' }}, CharacterSet{{ '/' }} })),
}),
PrecedenceRange(),
false

12
test/compiler/build_tables/rule_can_be_blank_test.cc
View file

@ -24,24 +24,24 @@ describe("rule_can_be_blank", [&]() {
});
it("returns true for choices iff one or more sides can be blank", [&]() {
rule = Choice::build({ CharacterSet{{'x'}}, Blank{} });
rule = Rule::choice({ CharacterSet{{'x'}}, Blank{} });
AssertThat(rule_can_be_blank(rule), IsTrue());
rule = Choice::build({ Blank{}, CharacterSet{{'x'}} });
rule = Rule::choice({ Blank{}, CharacterSet{{'x'}} });
AssertThat(rule_can_be_blank(rule), IsTrue());
rule = Choice::build({ CharacterSet{{'x'}}, CharacterSet{{'y'}} });
rule = Rule::choice({ CharacterSet{{'x'}}, CharacterSet{{'y'}} });
AssertThat(rule_can_be_blank(rule), IsFalse());
});
it("returns true for sequences iff both sides can be blank", [&]() {
rule = Seq::build({ Blank{}, CharacterSet{{'x'}} });
rule = Rule::seq({ Blank{}, CharacterSet{{'x'}} });
AssertThat(rule_can_be_blank(rule), IsFalse());
rule = Seq::build({ CharacterSet{{'x'}}, Blank{} });
rule = Rule::seq({ CharacterSet{{'x'}}, Blank{} });
AssertThat(rule_can_be_blank(rule), IsFalse());
rule = Seq::build({ Blank{}, Choice::build({ CharacterSet{{'x'}}, Blank{} }) });
rule = Rule::seq({ Blank{}, Rule::choice({ CharacterSet{{'x'}}, Blank{} }) });
AssertThat(rule_can_be_blank(rule), IsTrue());
});

60
test/compiler/prepare_grammar/expand_repeats_test.cc
View file

@ -27,8 +27,8 @@ describe("expand_repeats", []() {
AssertThat(result.variables, Equals(vector<Variable>{
Variable{"rule0", VariableTypeNamed, Symbol::non_terminal(1)},
Variable{"rule0_repeat1", VariableTypeAuxiliary, Choice::build({
Seq::build({ Symbol::non_terminal(1), Symbol::terminal(0) }),
Variable{"rule0_repeat1", VariableTypeAuxiliary, Rule::choice({
Rule::seq({ Symbol::non_terminal(1), Symbol::terminal(0) }),
Symbol::terminal(0),
})},
}));
@ -37,7 +37,7 @@ describe("expand_repeats", []() {
it("replaces repeats inside of sequences", [&]() {
InitialSyntaxGrammar grammar{
{
Variable{"rule0", VariableTypeNamed, Seq::build({
Variable{"rule0", VariableTypeNamed, Rule::seq({
Symbol::terminal(10),
Repeat{Symbol::terminal(11)},
})},
@ -48,12 +48,12 @@ describe("expand_repeats", []() {
auto result = expand_repeats(grammar);
AssertThat(result.variables, Equals(vector<Variable>{
Variable{"rule0", VariableTypeNamed, Seq::build({
Variable{"rule0", VariableTypeNamed, Rule::seq({
Symbol::terminal(10),
Symbol::non_terminal(1),
})},
Variable{"rule0_repeat1", VariableTypeAuxiliary, Choice::build({
Seq::build({ Symbol::non_terminal(1), Symbol::terminal(11) }),
Variable{"rule0_repeat1", VariableTypeAuxiliary, Rule::choice({
Rule::seq({ Symbol::non_terminal(1), Symbol::terminal(11) }),
Symbol::terminal(11)
})},
}));
@ -62,7 +62,7 @@ describe("expand_repeats", []() {
it("replaces repeats inside of choices", [&]() {
InitialSyntaxGrammar grammar{
{
Variable{"rule0", VariableTypeNamed, Choice::build({
Variable{"rule0", VariableTypeNamed, Rule::choice({
Symbol::terminal(10),
Repeat{Symbol::terminal(11)}
})},
@ -73,12 +73,12 @@ describe("expand_repeats", []() {
auto result = expand_repeats(grammar);
AssertThat(result.variables, Equals(vector<Variable>{
Variable{"rule0", VariableTypeNamed, Choice::build({
Variable{"rule0", VariableTypeNamed, Rule::choice({
Symbol::terminal(10),
Symbol::non_terminal(1),
})},
Variable{"rule0_repeat1", VariableTypeAuxiliary, Choice::build({
Seq::build({ Symbol::non_terminal(1), Symbol::terminal(11) }),
Variable{"rule0_repeat1", VariableTypeAuxiliary, Rule::choice({
Rule::seq({ Symbol::non_terminal(1), Symbol::terminal(11) }),
Symbol::terminal(11),
})},
}));
@ -87,11 +87,11 @@ describe("expand_repeats", []() {
it("does not create redundant auxiliary rules", [&]() {
InitialSyntaxGrammar grammar{
{
Variable{"rule0", VariableTypeNamed, Choice::build({
Seq::build({ Symbol::terminal(1), Repeat{Symbol::terminal(4)} }),
Seq::build({ Symbol::terminal(2), Repeat{Symbol::terminal(4)} }),
Variable{"rule0", VariableTypeNamed, Rule::choice({
Rule::seq({ Symbol::terminal(1), Repeat{Symbol::terminal(4)} }),
Rule::seq({ Symbol::terminal(2), Repeat{Symbol::terminal(4)} }),
})},
Variable{"rule1", VariableTypeNamed, Seq::build({
Variable{"rule1", VariableTypeNamed, Rule::seq({
Symbol::terminal(3),
Repeat{Symbol::terminal(4)}
})},
@ -102,16 +102,16 @@ describe("expand_repeats", []() {
auto result = expand_repeats(grammar);
AssertThat(result.variables, Equals(vector<Variable>{
Variable{"rule0", VariableTypeNamed, Choice::build({
Seq::build({ Symbol::terminal(1), Symbol::non_terminal(2) }),
Seq::build({ Symbol::terminal(2), Symbol::non_terminal(2) }),
Variable{"rule0", VariableTypeNamed, Rule::choice({
Rule::seq({ Symbol::terminal(1), Symbol::non_terminal(2) }),
Rule::seq({ Symbol::terminal(2), Symbol::non_terminal(2) }),
})},
Variable{"rule1", VariableTypeNamed, Seq::build({
Variable{"rule1", VariableTypeNamed, Rule::seq({
Symbol::terminal(3),
Symbol::non_terminal(2),
})},
Variable{"rule0_repeat1", VariableTypeAuxiliary, Choice::build({
Seq::build({ Symbol::non_terminal(2), Symbol::terminal(4) }),
Variable{"rule0_repeat1", VariableTypeAuxiliary, Rule::choice({
Rule::seq({ Symbol::non_terminal(2), Symbol::terminal(4) }),
Symbol::terminal(4),
})},
}));
@ -120,7 +120,7 @@ describe("expand_repeats", []() {
it("can replace multiple repeats in the same rule", [&]() {
InitialSyntaxGrammar grammar{
{
Variable{"rule0", VariableTypeNamed, Seq::build({
Variable{"rule0", VariableTypeNamed, Rule::seq({
Repeat{Symbol::terminal(10)},
Repeat{Symbol::terminal(11)},
})},
@ -131,16 +131,16 @@ describe("expand_repeats", []() {
auto result = expand_repeats(grammar);
AssertThat(result.variables, Equals(vector<Variable>{
Variable{"rule0", VariableTypeNamed, Seq::build({
Variable{"rule0", VariableTypeNamed, Rule::seq({
Symbol::non_terminal(1),
Symbol::non_terminal(2),
})},
Variable{"rule0_repeat1", VariableTypeAuxiliary, Choice::build({
Seq::build({ Symbol::non_terminal(1), Symbol::terminal(10) }),
Variable{"rule0_repeat1", VariableTypeAuxiliary, Rule::choice({
Rule::seq({ Symbol::non_terminal(1), Symbol::terminal(10) }),
Symbol::terminal(10),
})},
Variable{"rule0_repeat2", VariableTypeAuxiliary, Choice::build({
Seq::build({ Symbol::non_terminal(2), Symbol::terminal(11) }),
Variable{"rule0_repeat2", VariableTypeAuxiliary, Rule::choice({
Rule::seq({ Symbol::non_terminal(2), Symbol::terminal(11) }),
Symbol::terminal(11),
})},
}));
@ -160,12 +160,12 @@ describe("expand_repeats", []() {
AssertThat(result.variables, Equals(vector<Variable>{
Variable{"rule0", VariableTypeNamed, Symbol::non_terminal(2)},
Variable{"rule1", VariableTypeNamed, Symbol::non_terminal(3)},
Variable{"rule0_repeat1", VariableTypeAuxiliary, Choice::build({
Seq::build({ Symbol::non_terminal(2), Symbol::terminal(10) }),
Variable{"rule0_repeat1", VariableTypeAuxiliary, Rule::choice({
Rule::seq({ Symbol::non_terminal(2), Symbol::terminal(10) }),
Symbol::terminal(10),
})},
Variable{"rule1_repeat1", VariableTypeAuxiliary, Choice::build({
Seq::build({ Symbol::non_terminal(3), Symbol::terminal(11) }),
Variable{"rule1_repeat1", VariableTypeAuxiliary, Rule::choice({
Rule::seq({ Symbol::non_terminal(3), Symbol::terminal(11) }),
Symbol::terminal(11),
})},
}));

14
test/compiler/prepare_grammar/expand_tokens_test.cc
View file

@ -17,14 +17,14 @@ describe("expand_tokens", []() {
describe("string rules", [&]() {
it("replaces strings with sequences of character sets", [&]() {
AssertThat(
expand_token(Seq::build({
expand_token(Rule::seq({
String{"a"},
String{"bcd"},
String{"e"}
})).rule,
Equals(*Seq::build({
Equals(Rule::seq({
CharacterSet{{ 'a' }},
Seq::build({
Rule::seq({
CharacterSet{{ 'b' }},
CharacterSet{{ 'c' }},
CharacterSet{{ 'd' }},
@ -36,7 +36,7 @@ describe("expand_tokens", []() {
it("handles strings containing non-ASCII UTF8 characters", [&]() {
AssertThat(
expand_token(String{"\u03B1 \u03B2"}).rule,
Equals(*Seq::build({
Equals(Rule::seq({
CharacterSet{{ 945 }},
CharacterSet{{ ' ' }},
CharacterSet{{ 946 }},
@ -48,12 +48,12 @@ describe("expand_tokens", []() {
describe("regexp rules", [&]() {
it("replaces regexps with the equivalent rule tree", [&]() {
AssertThat(
expand_token(Seq::build({
expand_token(Rule::seq({
String{"a"},
Pattern{"x+"},
String{"b"},
})).rule,
Equals(*Seq::build({
Equals(Rule::seq({
CharacterSet{{'a'}},
Repeat{CharacterSet{{ 'x' }}},
CharacterSet{{'b'}},
@ -72,7 +72,7 @@ describe("expand_tokens", []() {
it("returns an error when the grammar contains an invalid regex", [&]() {
AssertThat(
expand_token(Seq::build({
expand_token(Rule::seq({
Pattern{"("},
String{"xyz"},
Pattern{"["},

22
test/compiler/prepare_grammar/extract_choices_test.cc
View file

@ -9,9 +9,9 @@ using prepare_grammar::extract_choices;
describe("extract_choices", []() {
it("expands rules containing choices into multiple rules", [&]() {
auto rule = Seq::build({
auto rule = Rule::seq({
Symbol::terminal(1),
Choice::build({
Rule::choice({
Symbol::terminal(2),
Symbol::terminal(3),
Symbol::terminal(4)
@ -22,14 +22,14 @@ describe("extract_choices", []() {
auto result = extract_choices(rule);
AssertThat(result, Equals(vector<Rule>({
Seq::build({Symbol::terminal(1), Symbol::terminal(2), Symbol::terminal(5)}),
Seq::build({Symbol::terminal(1), Symbol::terminal(3), Symbol::terminal(5)}),
Seq::build({Symbol::terminal(1), Symbol::terminal(4), Symbol::terminal(5)}),
Rule::seq({Symbol::terminal(1), Symbol::terminal(2), Symbol::terminal(5)}),
Rule::seq({Symbol::terminal(1), Symbol::terminal(3), Symbol::terminal(5)}),
Rule::seq({Symbol::terminal(1), Symbol::terminal(4), Symbol::terminal(5)}),
})));
});
it("handles metadata rules", [&]() {
auto rule = Metadata::prec(5, Choice::build({
auto rule = Metadata::prec(5, Rule::choice({
Symbol::terminal(2),
Symbol::terminal(3),
Symbol::terminal(4)
@ -43,9 +43,9 @@ describe("extract_choices", []() {
});
it("handles nested choices", [&]() {
auto rule = Choice::build({
Seq::build({
Choice::build({
auto rule = Rule::choice({
Rule::seq({
Rule::choice({
Symbol::terminal(1),
Symbol::terminal(2)
}),
@ -55,8 +55,8 @@ describe("extract_choices", []() {
});
AssertThat(extract_choices(rule), Equals(vector<Rule>({
Seq::build({Symbol::terminal(1), Symbol::terminal(3)}),
Seq::build({Symbol::terminal(2), Symbol::terminal(3)}),
Rule::seq({Symbol::terminal(1), Symbol::terminal(3)}),
Rule::seq({Symbol::terminal(2), Symbol::terminal(3)}),
Symbol::terminal(4),
})));
});

40
test/compiler/prepare_grammar/extract_tokens_test.cc
View file

@ -21,13 +21,13 @@ describe("extract_tokens", []() {
InternedVariable{
"rule_A",
VariableTypeNamed,
Repeat{Seq::build({
Repeat{Rule::seq({
String{"ab"},
Pattern{"cd+"},
Choice::build({
Rule::choice({
Symbol::non_terminal(1),
Symbol::non_terminal(2),
Metadata::token(Repeat{Choice::build({
Metadata::token(Repeat{Rule::choice({
String{"ef"},
String{"g"}
})}),
@ -42,7 +42,7 @@ describe("extract_tokens", []() {
InternedVariable{
"rule_C",
VariableTypeNamed,
Choice::build({ String{"i"}, Blank{} })
Rule::choice({ String{"i"}, Blank{} })
},
InternedVariable{
"rule_D",
@ -65,7 +65,7 @@ describe("extract_tokens", []() {
InitialSyntaxVariable{
"rule_A",
VariableTypeNamed,
Repeat{Seq::build({
Repeat{Rule::seq({
// This string is now the first token in the lexical grammar.
Symbol::terminal(0),
@ -73,7 +73,7 @@ describe("extract_tokens", []() {
// This pattern is now the second rule in the lexical grammar.
Symbol::terminal(1),
Choice::build({
Rule::choice({
// Rule 1, which this symbol pointed to, has been moved to the
// lexical grammar.
Symbol::terminal(3),
@ -91,7 +91,7 @@ describe("extract_tokens", []() {
InitialSyntaxVariable{
"rule_C",
VariableTypeNamed,
Choice::build({Symbol::terminal(4), Blank{}})
Rule::choice({Symbol::terminal(4), Blank{}})
},
InitialSyntaxVariable{
@ -122,7 +122,7 @@ describe("extract_tokens", []() {
LexicalVariable{
"/(ef|g)+/",
VariableTypeAuxiliary,
Repeat{Choice::build({
Repeat{Rule::choice({
Seq{CharacterSet{{'e'}}, CharacterSet{{'f'}}},
CharacterSet{{'g'}},
})},
@ -153,7 +153,7 @@ describe("extract_tokens", []() {
{
"rule_A",
VariableTypeNamed,
Seq::build({
Rule::seq({
String{"ab"},
Symbol::non_terminal(1),
String{"ab"},
@ -172,7 +172,7 @@ describe("extract_tokens", []() {
InitialSyntaxVariable{
"rule_A",
VariableTypeNamed,
Seq::build({
Rule::seq({
Symbol::terminal(0),
Symbol::non_terminal(1),
Symbol::terminal(0)
@ -195,7 +195,7 @@ describe("extract_tokens", []() {
InternedVariable{
"rule_A",
VariableTypeNamed,
Seq::build({ Symbol::non_terminal(1), String{"ab"} })
Rule::seq({ Symbol::non_terminal(1), String{"ab"} })
},
InternedVariable{
"rule_B",
@ -205,7 +205,7 @@ describe("extract_tokens", []() {
InternedVariable{
"rule_C",
VariableTypeNamed,
Seq::build({ String{"ef"}, String{"cd"} })
Rule::seq({ String{"ef"}, String{"cd"} })
},
}, {}, {}, {}});
@ -216,7 +216,7 @@ describe("extract_tokens", []() {
InitialSyntaxVariable{
"rule_A",
VariableTypeNamed,
Seq::build({ Symbol::non_terminal(1), Symbol::terminal(0) })
Rule::seq({ Symbol::non_terminal(1), Symbol::terminal(0) })
},
InitialSyntaxVariable{
"rule_B",
@ -226,7 +226,7 @@ describe("extract_tokens", []() {
InitialSyntaxVariable{
"rule_C",
VariableTypeNamed,
Seq::build({ Symbol::terminal(2), Symbol::terminal(1) })
Rule::seq({ Symbol::terminal(2), Symbol::terminal(1) })
},
})));
@ -335,7 +335,7 @@ describe("extract_tokens", []() {
InternedVariable{
"rule_A",
VariableTypeNamed,
Seq::build({ String{"w"}, String{"x"}, Symbol::non_terminal(1) })
Rule::seq({ String{"w"}, String{"x"}, Symbol::non_terminal(1) })
},
InternedVariable{
"rule_B",
@ -370,12 +370,12 @@ describe("extract_tokens", []() {
InternedVariable{
"rule_A",
VariableTypeNamed,
Seq::build({ String{"x"}, Symbol::non_terminal(1) })
Rule::seq({ String{"x"}, Symbol::non_terminal(1) })
},
InternedVariable{
"rule_B",
VariableTypeNamed,
Seq::build({ String{"y"}, String{"z"} })
Rule::seq({ String{"y"}, String{"z"} })
},
},
{
@ -398,7 +398,7 @@ describe("extract_tokens", []() {
{"rule_B", VariableTypeNamed, String{"y"}},
},
{
Choice::build({ Symbol::non_terminal(1), Blank{} })
Rule::choice({ Symbol::non_terminal(1), Blank{} })
},
{},
{}
@ -417,12 +417,12 @@ describe("extract_tokens", []() {
{
"rule_A",
VariableTypeNamed,
Seq::build({ String{"x"}, Symbol::non_terminal(1) })
Rule::seq({ String{"x"}, Symbol::non_terminal(1) })
},
{
"rule_B",
VariableTypeNamed,
Seq::build({ String{"y"}, String{"z"} })
Rule::seq({ String{"y"}, String{"z"} })
},
},
{},

12
test/compiler/prepare_grammar/flatten_grammar_test.cc
View file

@ -14,12 +14,12 @@ describe("flatten_grammar", []() {
SyntaxVariable result = flatten_rule({
"test",
VariableTypeNamed,
Seq::build({
Rule::seq({
Symbol::non_terminal(1),
Metadata::prec_left(101, Seq::build({
Metadata::prec_left(101, Rule::seq({
Symbol::non_terminal(2),
Choice::build({
Metadata::prec_right(102, Seq::build({
Rule::choice({
Metadata::prec_right(102, Rule::seq({
Symbol::non_terminal(3),
Symbol::non_terminal(4)
})),
@ -56,7 +56,7 @@ describe("flatten_grammar", []() {
SyntaxVariable result = flatten_rule({
"test1",
VariableTypeNamed,
Metadata::prec_left(101, Seq::build({
Metadata::prec_left(101, Rule::seq({
Symbol::non_terminal(1),
Symbol::non_terminal(2),
}))
@ -72,7 +72,7 @@ describe("flatten_grammar", []() {
result = flatten_rule({
"test2",
VariableTypeNamed,
Metadata::prec_left(101, Seq::build({
Metadata::prec_left(101, Rule::seq({
Symbol::non_terminal(1),
}))
});

8
test/compiler/prepare_grammar/intern_symbols_test.cc
View file

@ -13,7 +13,7 @@ describe("intern_symbols", []() {
it("replaces named symbols with numerically-indexed symbols", [&]() {
InputGrammar grammar{
{
{"x", VariableTypeNamed, Choice::build({ NamedSymbol{"y"}, NamedSymbol{"_z"} })},
{"x", VariableTypeNamed, Rule::choice({ NamedSymbol{"y"}, NamedSymbol{"_z"} })},
{"y", VariableTypeNamed, NamedSymbol{"_z"}},
{"_z", VariableTypeNamed, String{"stuff"}}
}, {}, {}, {}
@ -23,7 +23,7 @@ describe("intern_symbols", []() {
AssertThat(result.second, Equals(CompileError::none()));
AssertThat(result.first.variables, Equals(vector<prepare_grammar::InternedGrammar::Variable>{
{"x", VariableTypeNamed, Choice::build({ Symbol::non_terminal(1), Symbol::non_terminal(2) })},
{"x", VariableTypeNamed, Rule::choice({ Symbol::non_terminal(1), Symbol::non_terminal(2) })},
{"y", VariableTypeNamed, Symbol::non_terminal(2)},
{"_z", VariableTypeHidden, String{"stuff"}},
}));
@ -47,7 +47,7 @@ describe("intern_symbols", []() {
it("translates the grammar's optional 'extra_tokens' to numerical symbols", [&]() {
InputGrammar grammar{
{
{"x", VariableTypeNamed, Choice::build({ NamedSymbol{"y"}, NamedSymbol{"z"} })},
{"x", VariableTypeNamed, Rule::choice({ NamedSymbol{"y"}, NamedSymbol{"z"} })},
{"y", VariableTypeNamed, NamedSymbol{"z"}},
{"z", VariableTypeNamed, String{"stuff"}}
},
@ -67,7 +67,7 @@ describe("intern_symbols", []() {
it("records any rule names that match external token names", [&]() {
InputGrammar grammar{
{
{"x", VariableTypeNamed, Choice::build({ NamedSymbol{"y"}, NamedSymbol{"z"} })},
{"x", VariableTypeNamed, Rule::choice({ NamedSymbol{"y"}, NamedSymbol{"z"} })},
{"y", VariableTypeNamed, NamedSymbol{"z"}},
{"z", VariableTypeNamed, String{"stuff"}},
},

52
test/compiler/prepare_grammar/parse_regex_test.cc
View file

@ -29,7 +29,7 @@ describe("parse_regex", []() {
{
"character classes",
"\\w-\\d-\\s-\\W-\\D-\\S",
Seq::build({
Rule::seq({
CharacterSet{{
'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm',
'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z',
@ -60,7 +60,7 @@ describe("parse_regex", []() {
{
"choices",
"ab|cd|ef",
Choice::build({
Rule::choice({
Seq{
CharacterSet{{'a'}},
CharacterSet{{'b'}}
@ -79,7 +79,7 @@ describe("parse_regex", []() {
{
"simple sequences",
"abc",
Seq::build({
Rule::seq({
CharacterSet{{'a'}},
CharacterSet{{'b'}},
CharacterSet{{'c'}}
@ -113,12 +113,12 @@ describe("parse_regex", []() {
{
"character groups in sequences",
"x([^x]|\\\\x)*x",
Seq::build({
Rule::seq({
CharacterSet{{'x'}},
Choice::build({
Repeat{Choice::build({
Rule::choice({
Repeat{Rule::choice({
CharacterSet().include_all().exclude('x'),
Seq::build({
Rule::seq({
CharacterSet{{'\\'}},
CharacterSet{{'x'}}
})
@ -132,8 +132,8 @@ describe("parse_regex", []() {
{
"choices in sequences",
"(a|b)cd",
Seq::build({
Choice::build({
Rule::seq({
Rule::choice({
CharacterSet{{'a'}},
CharacterSet{{'b'}} }),
CharacterSet{{'c'}},
@ -143,7 +143,7 @@ describe("parse_regex", []() {
{
"escaped parentheses",
"a\\(b",
Seq::build({
Rule::seq({
CharacterSet{{'a'}},
CharacterSet{{'('}},
CharacterSet{{'b'}},
@ -153,7 +153,7 @@ describe("parse_regex", []() {
{
"escaped periods",
"a\\.",
Seq::build({
Rule::seq({
CharacterSet{{'a'}},
CharacterSet{{'.'}},
})
@ -162,7 +162,7 @@ describe("parse_regex", []() {
{
"escaped characters",
"\\t\\n\\r",
Seq::build({
Rule::seq({
CharacterSet{{'\t'}},
CharacterSet{{'\n'}},
CharacterSet{{'\r'}},
@ -172,22 +172,22 @@ describe("parse_regex", []() {
{
"plus repeats",
"(ab)+(cd)+",
Seq::build({
Repeat{Seq::build({ CharacterSet{{'a'}}, CharacterSet{{'b'}} })},
Repeat{Seq::build({ CharacterSet{{'c'}}, CharacterSet{{'d'}} })},
Rule::seq({
Repeat{Rule::seq({ CharacterSet{{'a'}}, CharacterSet{{'b'}} })},
Repeat{Rule::seq({ CharacterSet{{'c'}}, CharacterSet{{'d'}} })},
})
},
{
"asterix repeats",
"(ab)*(cd)*",
Seq::build({
Choice::build({
Repeat{Seq::build({ CharacterSet{{'a'}}, CharacterSet{{'b'}} })},
Rule::seq({
Rule::choice({
Repeat{Rule::seq({ CharacterSet{{'a'}}, CharacterSet{{'b'}} })},
Blank{},
}),
Choice::build({
Repeat{Seq::build({ CharacterSet{{'c'}}, CharacterSet{{'d'}} })},
Rule::choice({
Repeat{Rule::seq({ CharacterSet{{'c'}}, CharacterSet{{'d'}} })},
Blank{},
}),
})
@ -196,10 +196,10 @@ describe("parse_regex", []() {
{
"optional rules",
"a(bc)?",
Seq::build({
Rule::seq({
CharacterSet{{'a'}},
Choice::build({
Seq::build({
Rule::choice({
Rule::seq({
CharacterSet{{'b'}},
CharacterSet{{'c'}},
}),
@ -211,11 +211,11 @@ describe("parse_regex", []() {
{
"choices containing negated character classes",
"/([^/]|(\\\\/))+/",
Seq::build({
Rule::seq({
CharacterSet{{'/'}},
Repeat{Choice::build({
Repeat{Rule::choice({
CharacterSet().include_all().exclude('/'),
Seq::build({
Rule::seq({
CharacterSet{{'\\'}},
CharacterSet{{'/'}},
}),

28
test/compiler/rules/choice_test.cc
View file

@ -8,38 +8,38 @@ START_TEST
describe("Choice", []() {
describe("constructing choices", [&]() {
it("eliminates duplicate members", [&]() {
Rule rule = Choice::build({
Seq::build({ NamedSymbol{"one"}, NamedSymbol{"two"} }),
Rule rule = Rule::choice({
Rule::seq({ NamedSymbol{"one"}, NamedSymbol{"two"} }),
NamedSymbol{"three"},
Seq::build({ NamedSymbol{"one"}, NamedSymbol{"two"} })
Rule::seq({ NamedSymbol{"one"}, NamedSymbol{"two"} })
});
AssertThat(rule, Equals(Rule(Choice{{
Seq::build({ NamedSymbol{"one"}, NamedSymbol{"two"} }),
Rule::seq({ NamedSymbol{"one"}, NamedSymbol{"two"} }),
NamedSymbol{"three"},
}})));
rule = Choice::build({
rule = Rule::choice({
Blank{},
Blank{},
Choice::build({
Rule::choice({
Blank{},
NamedSymbol{"four"}
})
});
AssertThat(rule, Equals(*Choice::build({Blank{}, NamedSymbol{"four"}})));
AssertThat(rule, Equals(Rule::choice({Blank{}, NamedSymbol{"four"}})));
});
it("eliminates duplicates within nested choices", [&]() {
Rule rule = Choice::build({
Seq::build({
Rule rule = Rule::choice({
Rule::seq({
NamedSymbol{"one"},
NamedSymbol{"two"}
}),
Choice::build({
Rule::choice({
NamedSymbol{"three"},
Seq::build({
Rule::seq({
NamedSymbol{"one"},
NamedSymbol{"two"}
})
@ -47,7 +47,7 @@ describe("Choice", []() {
});
AssertThat(rule, Equals(Rule(Choice{{
Seq::build({
Rule::seq({
NamedSymbol{"one"},
NamedSymbol{"two"},
}),
@ -56,9 +56,9 @@ describe("Choice", []() {
});
it("doesn't construct a choice if there's only one unique member", [&]() {
Rule rule = Choice::build({
Rule rule = Rule::choice({
NamedSymbol{"one"},
Choice::build({
Rule::choice({
NamedSymbol{"one"},
})
});

4
test/compiler/rules/repeat_test.cc
View file

@ -9,8 +9,8 @@ describe("Repeat", []() {
describe("constructing repeats", [&]() {
it("doesn't create redundant repeats", [&]() {
Rule symbol = Symbol::non_terminal(1);
Rule repeat = Repeat::build(Rule(symbol));
Rule outer_repeat = Repeat::build(Rule(repeat));
Rule repeat = Rule::repeat(Rule(symbol));
Rule outer_repeat = Rule::repeat(Rule(repeat));
AssertThat(repeat, !Equals(symbol));
AssertThat(outer_repeat, Equals(repeat));