traxys/tree-sitter
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Avoid creating duplicate metadata rules

Browse source
This commit is contained in:
Max Brunsfeld 2018-03-21 11:37:08 -07:00
parent 997913aa82
commit 43e14332ed
12 changed files with 169 additions and 101 deletions
Download patch file Download diff file Expand all files Collapse all files

3
src/compiler/build_tables/lex_item_transitions.cc
View file

@ -12,6 +12,7 @@ namespace build_tables {
using std::function;
using std::map;
using std::move;
using std::pair;
using std::vector;
using rules::CharacterSet;
@ -157,7 +158,7 @@ class TransitionBuilder {
add_transition(
transitions, pair.first,
transform_transition(pair.second, [&params](Rule rule) {
return rules::Metadata{rule, params};
return rules::Metadata::merge(move(rule), params);
})
);
}

7
src/compiler/build_tables/lex_table_builder.cc
View file

@ -19,6 +19,7 @@ namespace build_tables {
using std::iswalpha;
using std::map;
using std::move;
using std::pair;
using std::set;
using std::string;
@ -465,7 +466,7 @@ class LexTableBuilderImpl : public LexTableBuilder {
LexItemSet result;
terminals.for_each([&](Symbol symbol) {
if (symbol.is_terminal()) {
for (const auto &rule : rules_for_symbol(symbol)) {
for (auto &&rule : rules_for_symbol(symbol)) {
if (with_separators) {
for (const auto &separator_rule : separator_rules) {
result.entries.insert(LexItem(
@ -473,13 +474,13 @@ class LexTableBuilderImpl : public LexTableBuilder {
Metadata::separator(
Rule::seq({
separator_rule,
Metadata::main_token(rule)
Metadata::main_token(move(rule))
})
)
));
}
} else {
result.entries.insert(LexItem(symbol, Metadata::main_token(rule)));
result.entries.insert(LexItem(symbol, Metadata::main_token(move(rule))));
}
}
}

13
src/compiler/parse_grammar.cc
View file

@ -8,6 +8,7 @@
namespace tree_sitter {
using std::move;
using std::string;
using std::vector;
using std::unordered_set;
@ -112,7 +113,7 @@ ParseRuleResult parse_rule(json_value *rule_json) {
if (!result.error_message.empty()) {
return "Invalid token content: " + result.error_message;
}
return Rule(Metadata::token(result.rule));
return Rule(Metadata::token(move(result.rule)));
}
if (type == "PATTERN") {
@ -153,7 +154,7 @@ ParseRuleResult parse_rule(json_value *rule_json) {
if (!result.error_message.empty()) {
return "Invalid precedence content: " + result.error_message;
}
return Rule(Metadata::prec(precedence_json.u.integer, result.rule));
return Rule(Metadata::prec(precedence_json.u.integer, move(result.rule)));
}
if (type == "PREC_LEFT") {
@ -167,7 +168,7 @@ ParseRuleResult parse_rule(json_value *rule_json) {
if (!result.error_message.empty()) {
return "Invalid precedence content: " + result.error_message;
}
return Rule(Metadata::prec_left(precedence_json.u.integer, result.rule));
return Rule(Metadata::prec_left(precedence_json.u.integer, move(result.rule)));
}
if (type == "PREC_RIGHT") {
@ -181,7 +182,7 @@ ParseRuleResult parse_rule(json_value *rule_json) {
if (!result.error_message.empty()) {
return "Invalid precedence content: " + result.error_message;
}
return Rule(Metadata::prec_right(precedence_json.u.integer, result.rule));
return Rule(Metadata::prec_right(precedence_json.u.integer, move(result.rule)));
}
if (type == "PREC_DYNAMIC") {
@ -195,7 +196,7 @@ ParseRuleResult parse_rule(json_value *rule_json) {
if (!result.error_message.empty()) {
return "Invalid precedence content: " + result.error_message;
}
return Rule(Metadata::prec_dynamic(precedence_json.u.integer, result.rule));
return Rule(Metadata::prec_dynamic(precedence_json.u.integer, move(result.rule)));
}
if (type == "ALIAS") {
@ -217,7 +218,7 @@ ParseRuleResult parse_rule(json_value *rule_json) {
return Rule(Metadata::alias(
string(value_json.u.string.ptr),
is_named_json.u.boolean,
result.rule
move(result.rule)
));
}

8
src/compiler/prepare_grammar/extract_tokens.cc
View file

@ -114,7 +114,13 @@ class TokenExtractor {
[this](const rules::Metadata &rule) -> Rule {
if (rule.params.is_token) {
return extract_token(*rule.rule, VariableTypeAuxiliary);
rules::Metadata metadata{*rule.rule, rule.params};
metadata.params.is_token = false;
if (metadata.params == rules::MetadataParams{}) {
return extract_token(*metadata.rule, VariableTypeAuxiliary);
} else {
return extract_token(metadata, VariableTypeAuxiliary);
}
} else {
return rules::Metadata{apply(*rule.rule), rule.params};
}

8
src/compiler/rule.cc
View file

@ -138,9 +138,15 @@ bool Rule::is<Symbol>() const { return type == SymbolType; }
template <>
bool Rule::is<Repeat>() const { return type == RepeatType; }
template <>
bool Rule::is<Metadata>() const { return type == MetadataType; }
template <>
const Symbol & Rule::get_unchecked<Symbol>() const { return symbol_; }
template <>
const Metadata & Rule::get_unchecked<Metadata>() const { return metadata_; }
static inline void add_choice_element(std::vector<Rule> *elements, const Rule &new_rule) {
new_rule.match(
[elements](Choice choice) {
@ -284,4 +290,4 @@ size_t hash<Rule>::operator()(const Rule &rule) const {
}
}
} // namespace std
} // namespace std

153
src/compiler/rules/metadata.cc
View file

@ -30,72 +30,125 @@ bool Metadata::operator==(const Metadata &other) const {
return rule->operator==(*other.rule) && params == other.params;
}
Metadata Metadata::token(const Rule &rule) {
MetadataParams params;
params.is_token = true;
return Metadata{rule, params};
template <typename T>
static Metadata add_metadata(Rule &&rule, T &&callback) {
if (rule.is<Metadata>()) {
Metadata metadata = rule.get_unchecked<Metadata>();
callback(metadata.params);
return metadata;
} else {
MetadataParams params;
callback(params);
return Metadata{move(rule), params};
}
}
Metadata Metadata::active_prec(int precedence, const Rule &rule) {
MetadataParams params;
params.has_precedence = true;
params.precedence = precedence;
params.is_active = true;
return Metadata{rule, params};
Metadata Metadata::merge(Rule &&rule, MetadataParams new_params) {
return add_metadata(move(rule), [&](MetadataParams &params) {
if (new_params.has_precedence && !params.has_precedence) {
params.has_precedence = true;
params.precedence = new_params.precedence;
}
if (new_params.has_associativity && !params.has_associativity) {
params.has_associativity = true;
params.associativity = new_params.associativity;
}
if (new_params.dynamic_precedence != 0) {
params.dynamic_precedence = new_params.dynamic_precedence;
}
if (new_params.is_string) params.is_string = true;
if (new_params.is_active) params.is_active = true;
if (new_params.is_main_token) params.is_main_token = true;
if (!new_params.alias.value.empty()) {
params.alias = new_params.alias;
}
});
}
Metadata Metadata::prec(int precedence, const Rule &rule) {
MetadataParams params;
params.has_precedence = true;
params.precedence = precedence;
return Metadata{rule, params};
Metadata Metadata::token(Rule &&rule) {
return add_metadata(move(rule), [](MetadataParams &params) {
params.is_token = true;
});
}
Metadata Metadata::prec_left(int precedence, const Rule &rule) {
MetadataParams params;
params.has_precedence = true;
params.precedence = precedence;
params.has_associativity = true;
params.associativity = AssociativityLeft;
return Metadata{rule, params};
Metadata Metadata::active_prec(int precedence, Rule &&rule) {
return add_metadata(move(rule), [&](MetadataParams &params) {
params.has_precedence = true;
params.precedence = precedence;
params.is_active = true;
});
}
Metadata Metadata::prec_right(int precedence, const Rule &rule) {
MetadataParams params;
params.has_precedence = true;
params.precedence = precedence;
params.has_associativity = true;
params.associativity = AssociativityRight;
return Metadata{rule, params};
Metadata Metadata::prec(int precedence, Rule &&rule) {
return add_metadata(move(rule), [&](MetadataParams &params) {
if (!params.has_precedence) {
params.has_precedence = true;
params.precedence = precedence;
}
});
}
Metadata Metadata::prec_dynamic(int dynamic_precedence, const Rule &rule) {
MetadataParams params;
params.dynamic_precedence = dynamic_precedence;
return Metadata{rule, params};
Metadata Metadata::prec_left(int precedence, Rule &&rule) {
return add_metadata(move(rule), [&](MetadataParams &params) {
if (!params.has_precedence) {
params.has_precedence = true;
params.precedence = precedence;
}
if (!params.has_associativity) {
params.has_associativity = true;
params.associativity = AssociativityLeft;
}
});
}
Metadata Metadata::separator(const Rule &rule) {
MetadataParams params;
params.has_precedence = true;
params.precedence = INT_MIN;
params.is_active = true;
return Metadata{rule, params};
Metadata Metadata::prec_right(int precedence, Rule &&rule) {
return add_metadata(move(rule), [&](MetadataParams &params) {
if (!params.has_precedence) {
params.has_precedence = true;
params.precedence = precedence;
}
if (!params.has_associativity) {
params.has_associativity = true;
params.associativity = AssociativityRight;
}
});
}
Metadata Metadata::main_token(const Rule &rule) {
MetadataParams params;
params.has_precedence = true;
params.precedence = 0;
params.is_main_token = true;
return Metadata{rule, params};
Metadata Metadata::prec_dynamic(int dynamic_precedence, Rule &&rule) {
return add_metadata(move(rule), [&](MetadataParams &params) {
params.dynamic_precedence = dynamic_precedence;
});
}
Metadata Metadata::alias(string &&value, bool is_named, const Rule &rule) {
MetadataParams params;
params.alias.value = move(value);
params.alias.is_named = is_named;
return Metadata{rule, params};
Metadata Metadata::separator(Rule &&rule) {
return add_metadata(move(rule), [&](MetadataParams &params) {
if (!params.has_precedence) {
params.has_precedence = true;
params.precedence = INT_MIN;
}
params.is_active = true;
});
}
Metadata Metadata::main_token(Rule &&rule) {
return add_metadata(move(rule), [&](MetadataParams &params) {
if (!params.has_precedence) {
params.has_precedence = true;
params.precedence = 0;
}
params.is_main_token = true;
});
}
Metadata Metadata::alias(string &&value, bool is_named, Rule &&rule) {
return add_metadata(move(rule), [&](MetadataParams &params) {
params.alias.value = move(value);
params.alias.is_named = is_named;
});
}
} // namespace rules

19
src/compiler/rules/metadata.h
View file

@ -62,15 +62,16 @@ struct Metadata {
Metadata(const Rule &rule, MetadataParams params);
static Metadata token(const Rule &rule);
static Metadata active_prec(int precedence, const Rule &rule);
static Metadata prec(int precedence, const Rule &rule);
static Metadata prec_left(int precedence, const Rule &rule);
static Metadata prec_right(int precedence, const Rule &rule);
static Metadata prec_dynamic(int precedence, const Rule &rule);
static Metadata separator(const Rule &rule);
static Metadata main_token(const Rule &rule);
static Metadata alias(std::string &&value, bool is_named, const Rule &rule);
static Metadata merge(Rule &&rule, MetadataParams params);
static Metadata token(Rule &&rule);
static Metadata active_prec(int precedence, Rule &&rule);
static Metadata prec(int precedence, Rule &&rule);
static Metadata prec_left(int precedence, Rule &&rule);
static Metadata prec_right(int precedence, Rule &&rule);
static Metadata prec_dynamic(int precedence, Rule &&rule);
static Metadata separator(Rule &&rule);
static Metadata main_token(Rule &&rule);
static Metadata alias(std::string &&value, bool is_named, Rule &&rule);
bool operator==(const Metadata &other) const;
};

7
test/compiler/build_tables/lex_item_test.cc
View file

@ -61,11 +61,8 @@ describe("LexItemSet::transitions()", [&]() {
});
it("marks transitions that are within the main token (as opposed to separators)", [&]() {
MetadataParams params;
params.is_main_token = true;
LexItemSet item_set({
LexItem(Symbol::non_terminal(1), Metadata{CharacterSet{{'x'}}, params}),
LexItem(Symbol::non_terminal(1), Metadata::main_token(CharacterSet{{'x'}})),
});
AssertThat(
@ -75,7 +72,7 @@ describe("LexItemSet::transitions()", [&]() {
CharacterSet({'x'}),
Transition{
LexItemSet({
LexItem(Symbol::non_terminal(1), Metadata{Blank{}, params}),
LexItem(Symbol::non_terminal(1), Metadata::active_prec(0, Metadata::main_token(Blank{}))),
}),
PrecedenceRange(),
true

21
test/compiler/rules/repeat_test.cc
View file

@ -1,21 +0,0 @@
#include "test_helper.h"
#include "compiler/rule.h"
using namespace rules;
START_TEST
describe("Repeat", []() {
describe("constructing repeats", [&]() {
it("doesn't create redundant repeats", [&]() {
Rule symbol = Symbol::non_terminal(1);
Rule repeat = Rule::repeat(Rule(symbol));
Rule outer_repeat = Rule::repeat(Rule(repeat));
AssertThat(repeat, !Equals(symbol));
AssertThat(outer_repeat, Equals(repeat));
});
});
});
END_TEST

21
test/compiler/rules/choice_test.cc → test/compiler/rules/rule_test.cc
View file

@ -5,7 +5,26 @@ using namespace rules;
START_TEST
describe("Choice", []() {
describe("Repeat", []() {
describe("constructing repeats", [&]() {
it("doesn't create redundant repeats", [&]() {
Rule symbol = Symbol::non_terminal(1);
Rule repeat = Rule::repeat(Rule(symbol));
Rule outer_repeat = Rule::repeat(Rule(repeat));
AssertThat(repeat, !Equals(symbol));
AssertThat(outer_repeat, Equals(repeat));
});
});
describe("adding metadata to rules", [&]() {
it("doesn't create redundant metadata rules", [&]() {
Rule symbol = Symbol::non_terminal(1);
Rule outer_rule = Metadata::prec(2, Metadata::prec(1, Rule(symbol)));
AssertThat(outer_rule, Equals(Rule(Metadata::prec(1, Rule(symbol)))));
});
});
describe("constructing choices", [&]() {
it("eliminates duplicate members", [&]() {
Rule rule = Rule::choice({

7
test/helpers/stream_methods.cc
View file

@ -111,7 +111,12 @@ ostream &operator<<(ostream &stream, const Repeat &rule) {
}
ostream &operator<<(ostream &stream, const Metadata &rule) {
return stream << "(Metadata " << *rule.rule << ")";
stream << "(Metadata";
if (rule.params.has_precedence) stream << " prec=" << to_string(rule.params.precedence);
if (rule.params.has_associativity) stream << " assoc=" << rule.params.associativity;
if (rule.params.is_token) stream << " token";
if (rule.params.is_main_token) stream << " main";
return stream << " " << *rule.rule << ")";
}
ostream &operator<<(ostream &stream, const Rule &rule) {

3
tests.gyp
View file

@ -49,8 +49,7 @@
'test/compiler/prepare_grammar/intern_symbols_test.cc',
'test/compiler/prepare_grammar/parse_regex_test.cc',
'test/compiler/rules/character_set_test.cc',
'test/compiler/rules/choice_test.cc',
'test/compiler/rules/repeat_test.cc',
'test/compiler/rules/rule_test.cc',
'test/compiler/util/string_helpers_test.cc',
'test/helpers/encoding_helpers.cc',
'test/helpers/file_helpers.cc',