diff --git a/src/compiler/build_tables/lex_item_transitions.cc b/src/compiler/build_tables/lex_item_transitions.cc index c4e05420..d30bf011 100644 --- a/src/compiler/build_tables/lex_item_transitions.cc +++ b/src/compiler/build_tables/lex_item_transitions.cc @@ -12,6 +12,7 @@ namespace build_tables { using std::function; using std::map; +using std::move; using std::pair; using std::vector; using rules::CharacterSet; @@ -157,7 +158,7 @@ class TransitionBuilder { add_transition( transitions, pair.first, transform_transition(pair.second, [¶ms](Rule rule) { - return rules::Metadata{rule, params}; + return rules::Metadata::merge(move(rule), params); }) ); } diff --git a/src/compiler/build_tables/lex_table_builder.cc b/src/compiler/build_tables/lex_table_builder.cc index 2c0d8e10..fa8b86f3 100644 --- a/src/compiler/build_tables/lex_table_builder.cc +++ b/src/compiler/build_tables/lex_table_builder.cc @@ -19,6 +19,7 @@ namespace build_tables { using std::iswalpha; using std::map; +using std::move; using std::pair; using std::set; using std::string; @@ -465,7 +466,7 @@ class LexTableBuilderImpl : public LexTableBuilder { LexItemSet result; terminals.for_each([&](Symbol symbol) { if (symbol.is_terminal()) { - for (const auto &rule : rules_for_symbol(symbol)) { + for (auto &&rule : rules_for_symbol(symbol)) { if (with_separators) { for (const auto &separator_rule : separator_rules) { result.entries.insert(LexItem( @@ -473,13 +474,13 @@ class LexTableBuilderImpl : public LexTableBuilder { Metadata::separator( Rule::seq({ separator_rule, - Metadata::main_token(rule) + Metadata::main_token(move(rule)) }) ) )); } } else { - result.entries.insert(LexItem(symbol, Metadata::main_token(rule))); + result.entries.insert(LexItem(symbol, Metadata::main_token(move(rule)))); } } } diff --git a/src/compiler/parse_grammar.cc b/src/compiler/parse_grammar.cc index 345fb94f..e233cfe0 100644 --- a/src/compiler/parse_grammar.cc +++ b/src/compiler/parse_grammar.cc @@ -8,6 +8,7 @@ namespace tree_sitter { +using std::move; using std::string; using std::vector; using std::unordered_set; @@ -112,7 +113,7 @@ ParseRuleResult parse_rule(json_value *rule_json) { if (!result.error_message.empty()) { return "Invalid token content: " + result.error_message; } - return Rule(Metadata::token(result.rule)); + return Rule(Metadata::token(move(result.rule))); } if (type == "PATTERN") { @@ -153,7 +154,7 @@ ParseRuleResult parse_rule(json_value *rule_json) { if (!result.error_message.empty()) { return "Invalid precedence content: " + result.error_message; } - return Rule(Metadata::prec(precedence_json.u.integer, result.rule)); + return Rule(Metadata::prec(precedence_json.u.integer, move(result.rule))); } if (type == "PREC_LEFT") { @@ -167,7 +168,7 @@ ParseRuleResult parse_rule(json_value *rule_json) { if (!result.error_message.empty()) { return "Invalid precedence content: " + result.error_message; } - return Rule(Metadata::prec_left(precedence_json.u.integer, result.rule)); + return Rule(Metadata::prec_left(precedence_json.u.integer, move(result.rule))); } if (type == "PREC_RIGHT") { @@ -181,7 +182,7 @@ ParseRuleResult parse_rule(json_value *rule_json) { if (!result.error_message.empty()) { return "Invalid precedence content: " + result.error_message; } - return Rule(Metadata::prec_right(precedence_json.u.integer, result.rule)); + return Rule(Metadata::prec_right(precedence_json.u.integer, move(result.rule))); } if (type == "PREC_DYNAMIC") { @@ -195,7 +196,7 @@ ParseRuleResult parse_rule(json_value *rule_json) { if (!result.error_message.empty()) { return "Invalid precedence content: " + result.error_message; } - return Rule(Metadata::prec_dynamic(precedence_json.u.integer, result.rule)); + return Rule(Metadata::prec_dynamic(precedence_json.u.integer, move(result.rule))); } if (type == "ALIAS") { @@ -217,7 +218,7 @@ ParseRuleResult parse_rule(json_value *rule_json) { return Rule(Metadata::alias( string(value_json.u.string.ptr), is_named_json.u.boolean, - result.rule + move(result.rule) )); } diff --git a/src/compiler/prepare_grammar/extract_tokens.cc b/src/compiler/prepare_grammar/extract_tokens.cc index 61016687..2e5cf1d9 100644 --- a/src/compiler/prepare_grammar/extract_tokens.cc +++ b/src/compiler/prepare_grammar/extract_tokens.cc @@ -114,7 +114,13 @@ class TokenExtractor { [this](const rules::Metadata &rule) -> Rule { if (rule.params.is_token) { - return extract_token(*rule.rule, VariableTypeAuxiliary); + rules::Metadata metadata{*rule.rule, rule.params}; + metadata.params.is_token = false; + if (metadata.params == rules::MetadataParams{}) { + return extract_token(*metadata.rule, VariableTypeAuxiliary); + } else { + return extract_token(metadata, VariableTypeAuxiliary); + } } else { return rules::Metadata{apply(*rule.rule), rule.params}; } diff --git a/src/compiler/rule.cc b/src/compiler/rule.cc index f802f3fa..29ee1793 100644 --- a/src/compiler/rule.cc +++ b/src/compiler/rule.cc @@ -138,9 +138,15 @@ bool Rule::is() const { return type == SymbolType; } template <> bool Rule::is() const { return type == RepeatType; } +template <> +bool Rule::is() const { return type == MetadataType; } + template <> const Symbol & Rule::get_unchecked() const { return symbol_; } +template <> +const Metadata & Rule::get_unchecked() const { return metadata_; } + static inline void add_choice_element(std::vector *elements, const Rule &new_rule) { new_rule.match( [elements](Choice choice) { @@ -284,4 +290,4 @@ size_t hash::operator()(const Rule &rule) const { } } -} // namespace std \ No newline at end of file +} // namespace std diff --git a/src/compiler/rules/metadata.cc b/src/compiler/rules/metadata.cc index 95822c5e..40dcb21e 100644 --- a/src/compiler/rules/metadata.cc +++ b/src/compiler/rules/metadata.cc @@ -30,72 +30,125 @@ bool Metadata::operator==(const Metadata &other) const { return rule->operator==(*other.rule) && params == other.params; } -Metadata Metadata::token(const Rule &rule) { - MetadataParams params; - params.is_token = true; - return Metadata{rule, params}; +template +static Metadata add_metadata(Rule &&rule, T &&callback) { + if (rule.is()) { + Metadata metadata = rule.get_unchecked(); + callback(metadata.params); + return metadata; + } else { + MetadataParams params; + callback(params); + return Metadata{move(rule), params}; + } } -Metadata Metadata::active_prec(int precedence, const Rule &rule) { - MetadataParams params; - params.has_precedence = true; - params.precedence = precedence; - params.is_active = true; - return Metadata{rule, params}; +Metadata Metadata::merge(Rule &&rule, MetadataParams new_params) { + return add_metadata(move(rule), [&](MetadataParams ¶ms) { + if (new_params.has_precedence && !params.has_precedence) { + params.has_precedence = true; + params.precedence = new_params.precedence; + } + + if (new_params.has_associativity && !params.has_associativity) { + params.has_associativity = true; + params.associativity = new_params.associativity; + } + + if (new_params.dynamic_precedence != 0) { + params.dynamic_precedence = new_params.dynamic_precedence; + } + + if (new_params.is_string) params.is_string = true; + if (new_params.is_active) params.is_active = true; + if (new_params.is_main_token) params.is_main_token = true; + + if (!new_params.alias.value.empty()) { + params.alias = new_params.alias; + } + }); } -Metadata Metadata::prec(int precedence, const Rule &rule) { - MetadataParams params; - params.has_precedence = true; - params.precedence = precedence; - return Metadata{rule, params}; +Metadata Metadata::token(Rule &&rule) { + return add_metadata(move(rule), [](MetadataParams ¶ms) { + params.is_token = true; + }); } -Metadata Metadata::prec_left(int precedence, const Rule &rule) { - MetadataParams params; - params.has_precedence = true; - params.precedence = precedence; - params.has_associativity = true; - params.associativity = AssociativityLeft; - return Metadata{rule, params}; +Metadata Metadata::active_prec(int precedence, Rule &&rule) { + return add_metadata(move(rule), [&](MetadataParams ¶ms) { + params.has_precedence = true; + params.precedence = precedence; + params.is_active = true; + }); } -Metadata Metadata::prec_right(int precedence, const Rule &rule) { - MetadataParams params; - params.has_precedence = true; - params.precedence = precedence; - params.has_associativity = true; - params.associativity = AssociativityRight; - return Metadata{rule, params}; +Metadata Metadata::prec(int precedence, Rule &&rule) { + return add_metadata(move(rule), [&](MetadataParams ¶ms) { + if (!params.has_precedence) { + params.has_precedence = true; + params.precedence = precedence; + } + }); } -Metadata Metadata::prec_dynamic(int dynamic_precedence, const Rule &rule) { - MetadataParams params; - params.dynamic_precedence = dynamic_precedence; - return Metadata{rule, params}; +Metadata Metadata::prec_left(int precedence, Rule &&rule) { + return add_metadata(move(rule), [&](MetadataParams ¶ms) { + if (!params.has_precedence) { + params.has_precedence = true; + params.precedence = precedence; + } + if (!params.has_associativity) { + params.has_associativity = true; + params.associativity = AssociativityLeft; + } + }); } -Metadata Metadata::separator(const Rule &rule) { - MetadataParams params; - params.has_precedence = true; - params.precedence = INT_MIN; - params.is_active = true; - return Metadata{rule, params}; +Metadata Metadata::prec_right(int precedence, Rule &&rule) { + return add_metadata(move(rule), [&](MetadataParams ¶ms) { + if (!params.has_precedence) { + params.has_precedence = true; + params.precedence = precedence; + } + if (!params.has_associativity) { + params.has_associativity = true; + params.associativity = AssociativityRight; + } + }); } -Metadata Metadata::main_token(const Rule &rule) { - MetadataParams params; - params.has_precedence = true; - params.precedence = 0; - params.is_main_token = true; - return Metadata{rule, params}; +Metadata Metadata::prec_dynamic(int dynamic_precedence, Rule &&rule) { + return add_metadata(move(rule), [&](MetadataParams ¶ms) { + params.dynamic_precedence = dynamic_precedence; + }); } -Metadata Metadata::alias(string &&value, bool is_named, const Rule &rule) { - MetadataParams params; - params.alias.value = move(value); - params.alias.is_named = is_named; - return Metadata{rule, params}; +Metadata Metadata::separator(Rule &&rule) { + return add_metadata(move(rule), [&](MetadataParams ¶ms) { + if (!params.has_precedence) { + params.has_precedence = true; + params.precedence = INT_MIN; + } + params.is_active = true; + }); +} + +Metadata Metadata::main_token(Rule &&rule) { + return add_metadata(move(rule), [&](MetadataParams ¶ms) { + if (!params.has_precedence) { + params.has_precedence = true; + params.precedence = 0; + } + params.is_main_token = true; + }); +} + +Metadata Metadata::alias(string &&value, bool is_named, Rule &&rule) { + return add_metadata(move(rule), [&](MetadataParams ¶ms) { + params.alias.value = move(value); + params.alias.is_named = is_named; + }); } } // namespace rules diff --git a/src/compiler/rules/metadata.h b/src/compiler/rules/metadata.h index 15d32c00..73a4a66d 100644 --- a/src/compiler/rules/metadata.h +++ b/src/compiler/rules/metadata.h @@ -62,15 +62,16 @@ struct Metadata { Metadata(const Rule &rule, MetadataParams params); - static Metadata token(const Rule &rule); - static Metadata active_prec(int precedence, const Rule &rule); - static Metadata prec(int precedence, const Rule &rule); - static Metadata prec_left(int precedence, const Rule &rule); - static Metadata prec_right(int precedence, const Rule &rule); - static Metadata prec_dynamic(int precedence, const Rule &rule); - static Metadata separator(const Rule &rule); - static Metadata main_token(const Rule &rule); - static Metadata alias(std::string &&value, bool is_named, const Rule &rule); + static Metadata merge(Rule &&rule, MetadataParams params); + static Metadata token(Rule &&rule); + static Metadata active_prec(int precedence, Rule &&rule); + static Metadata prec(int precedence, Rule &&rule); + static Metadata prec_left(int precedence, Rule &&rule); + static Metadata prec_right(int precedence, Rule &&rule); + static Metadata prec_dynamic(int precedence, Rule &&rule); + static Metadata separator(Rule &&rule); + static Metadata main_token(Rule &&rule); + static Metadata alias(std::string &&value, bool is_named, Rule &&rule); bool operator==(const Metadata &other) const; }; diff --git a/test/compiler/build_tables/lex_item_test.cc b/test/compiler/build_tables/lex_item_test.cc index 1979772d..71d5555f 100644 --- a/test/compiler/build_tables/lex_item_test.cc +++ b/test/compiler/build_tables/lex_item_test.cc @@ -61,11 +61,8 @@ describe("LexItemSet::transitions()", [&]() { }); it("marks transitions that are within the main token (as opposed to separators)", [&]() { - MetadataParams params; - params.is_main_token = true; - LexItemSet item_set({ - LexItem(Symbol::non_terminal(1), Metadata{CharacterSet{{'x'}}, params}), + LexItem(Symbol::non_terminal(1), Metadata::main_token(CharacterSet{{'x'}})), }); AssertThat( @@ -75,7 +72,7 @@ describe("LexItemSet::transitions()", [&]() { CharacterSet({'x'}), Transition{ LexItemSet({ - LexItem(Symbol::non_terminal(1), Metadata{Blank{}, params}), + LexItem(Symbol::non_terminal(1), Metadata::active_prec(0, Metadata::main_token(Blank{}))), }), PrecedenceRange(), true diff --git a/test/compiler/rules/repeat_test.cc b/test/compiler/rules/repeat_test.cc deleted file mode 100644 index 05b2f117..00000000 --- a/test/compiler/rules/repeat_test.cc +++ /dev/null @@ -1,21 +0,0 @@ -#include "test_helper.h" -#include "compiler/rule.h" - -using namespace rules; - -START_TEST - -describe("Repeat", []() { - describe("constructing repeats", [&]() { - it("doesn't create redundant repeats", [&]() { - Rule symbol = Symbol::non_terminal(1); - Rule repeat = Rule::repeat(Rule(symbol)); - Rule outer_repeat = Rule::repeat(Rule(repeat)); - - AssertThat(repeat, !Equals(symbol)); - AssertThat(outer_repeat, Equals(repeat)); - }); - }); -}); - -END_TEST diff --git a/test/compiler/rules/choice_test.cc b/test/compiler/rules/rule_test.cc similarity index 70% rename from test/compiler/rules/choice_test.cc rename to test/compiler/rules/rule_test.cc index fb2bbbaf..389cb1dd 100644 --- a/test/compiler/rules/choice_test.cc +++ b/test/compiler/rules/rule_test.cc @@ -5,7 +5,26 @@ using namespace rules; START_TEST -describe("Choice", []() { +describe("Repeat", []() { + describe("constructing repeats", [&]() { + it("doesn't create redundant repeats", [&]() { + Rule symbol = Symbol::non_terminal(1); + Rule repeat = Rule::repeat(Rule(symbol)); + Rule outer_repeat = Rule::repeat(Rule(repeat)); + + AssertThat(repeat, !Equals(symbol)); + AssertThat(outer_repeat, Equals(repeat)); + }); + }); + + describe("adding metadata to rules", [&]() { + it("doesn't create redundant metadata rules", [&]() { + Rule symbol = Symbol::non_terminal(1); + Rule outer_rule = Metadata::prec(2, Metadata::prec(1, Rule(symbol))); + AssertThat(outer_rule, Equals(Rule(Metadata::prec(1, Rule(symbol))))); + }); + }); + describe("constructing choices", [&]() { it("eliminates duplicate members", [&]() { Rule rule = Rule::choice({ diff --git a/test/helpers/stream_methods.cc b/test/helpers/stream_methods.cc index 9b13303c..5e20eb34 100644 --- a/test/helpers/stream_methods.cc +++ b/test/helpers/stream_methods.cc @@ -111,7 +111,12 @@ ostream &operator<<(ostream &stream, const Repeat &rule) { } ostream &operator<<(ostream &stream, const Metadata &rule) { - return stream << "(Metadata " << *rule.rule << ")"; + stream << "(Metadata"; + if (rule.params.has_precedence) stream << " prec=" << to_string(rule.params.precedence); + if (rule.params.has_associativity) stream << " assoc=" << rule.params.associativity; + if (rule.params.is_token) stream << " token"; + if (rule.params.is_main_token) stream << " main"; + return stream << " " << *rule.rule << ")"; } ostream &operator<<(ostream &stream, const Rule &rule) { diff --git a/tests.gyp b/tests.gyp index 02012a0a..56ef89e7 100644 --- a/tests.gyp +++ b/tests.gyp @@ -49,8 +49,7 @@ 'test/compiler/prepare_grammar/intern_symbols_test.cc', 'test/compiler/prepare_grammar/parse_regex_test.cc', 'test/compiler/rules/character_set_test.cc', - 'test/compiler/rules/choice_test.cc', - 'test/compiler/rules/repeat_test.cc', + 'test/compiler/rules/rule_test.cc', 'test/compiler/util/string_helpers_test.cc', 'test/helpers/encoding_helpers.cc', 'test/helpers/file_helpers.cc',