diff --git a/examples/parsers/arithmetic.c b/examples/parsers/arithmetic.c index 278a4336..ca43988a 100644 --- a/examples/parsers/arithmetic.c +++ b/examples/parsers/arithmetic.c @@ -548,7 +548,7 @@ PARSE_TABLE = { [1] = { [ts_aux_sym_token3] = SHIFT(2), [ts_aux_sym_token4] = SHIFT(100), - [ts_builtin_sym_end] = REDUCE(ts_sym_difference, 1), + [ts_builtin_sym_end] = REDUCE(ts_sym_sum, 1), }, [2] = { [ts_sym__operand1] = SHIFT(3), @@ -616,7 +616,7 @@ PARSE_TABLE = { [ts_builtin_sym_error] = SHIFT(78), }, [12] = { - [ts_aux_sym_token2] = REDUCE(ts_sym_difference, 1), + [ts_aux_sym_token2] = REDUCE(ts_sym_sum, 1), [ts_aux_sym_token3] = SHIFT(13), [ts_aux_sym_token4] = SHIFT(76), }, @@ -1207,8 +1207,8 @@ PARSE_TABLE = { [ts_builtin_sym_end] = REDUCE(ts_sym_difference, 3), }, [102] = { - [ts_aux_sym_token3] = REDUCE(ts_sym_product, 1), - [ts_aux_sym_token4] = REDUCE(ts_sym_product, 1), + [ts_aux_sym_token3] = REDUCE(ts_sym_quotient, 1), + [ts_aux_sym_token4] = REDUCE(ts_sym_quotient, 1), [ts_aux_sym_token5] = SHIFT(103), [ts_aux_sym_token6] = SHIFT(117), [ts_builtin_sym_end] = REDUCE(ts_sym_quotient, 1), diff --git a/include/tree_sitter/parser.h b/include/tree_sitter/parser.h index 1d02addb..1bef260a 100644 --- a/include/tree_sitter/parser.h +++ b/include/tree_sitter/parser.h @@ -53,7 +53,7 @@ static ts_tree * ts_lex(ts_lexer *lexer, state_id lex_state) char lookahead; \ next_state: \ lookahead = ts_lexer_lookahead_char(lexer); - + #define START_TOKEN() \ ts_lexer_start_token(lexer); @@ -164,7 +164,7 @@ static void ts_lexer_advance(ts_lexer *lexer) { static void ts_lexer_start_token(ts_lexer *lexer) { lexer->token_start_position = ts_lexer_position(lexer); -} +} static ts_tree * ts_lexer_build_node(ts_lexer *lexer, ts_symbol symbol) { size_t current_position = ts_lexer_position(lexer); diff --git a/spec/compiler/build_tables/check_metadata_spec.cc b/spec/compiler/build_tables/check_metadata_spec.cc index 7032f004..060fcf9f 100644 --- a/spec/compiler/build_tables/check_metadata_spec.cc +++ b/spec/compiler/build_tables/check_metadata_spec.cc @@ -19,12 +19,12 @@ describe("checking if rules have metadata", []() { auto rule = make_shared(sym("x"), MetadataValue(1 << 2)); AssertThat(check_metadata(rule, value), IsFalse()); }); - + it("returns false for a non-metadata rule", [&]() { auto rule = sym("x"); AssertThat(check_metadata(rule, value), IsFalse()); }); - + it("returns true for a compatible metadata rule preceded by rules that can be blank", [&]() { auto rule = seq({ repeat(sym("x")), @@ -33,13 +33,13 @@ describe("checking if rules have metadata", []() { AssertThat(check_metadata(rule, value), IsTrue()); }); - + it("returns true for a choice including a compatible metadata rule", [&]() { auto rule = choice({ sym("x"), make_shared(sym("x"), MetadataValue(value | 1)), }); - + AssertThat(check_metadata(rule, value), IsTrue()); }); diff --git a/spec/compiler/build_tables/first_set_spec.cc b/spec/compiler/build_tables/first_set_spec.cc index dbe06512..30b07a49 100644 --- a/spec/compiler/build_tables/first_set_spec.cc +++ b/spec/compiler/build_tables/first_set_spec.cc @@ -84,10 +84,10 @@ describe("computing FIRST sets", []() { }))); }); }); - + it("ignores metadata rules", [&]() { auto rule = make_shared(sym("x"), MetadataValue(1)); - + AssertThat(first_set(rule, null_grammar), Equals(set({ Symbol("x"), }))); diff --git a/spec/compiler/build_tables/item_set_transitions_spec.cc b/spec/compiler/build_tables/item_set_transitions_spec.cc index 85cf5542..bfa13a26 100644 --- a/spec/compiler/build_tables/item_set_transitions_spec.cc +++ b/spec/compiler/build_tables/item_set_transitions_spec.cc @@ -15,7 +15,7 @@ describe("item set transitions", []() { LexItemSet set1({ LexItem(Symbol("A"), pattern("[a-f]")), LexItem(Symbol("B"), pattern("[e-x]")) }); - + AssertThat(char_transitions(set1, grammar), Equals(map({ { CharacterSet({ {'a', 'd'} }), LexItemSet({ LexItem(Symbol("A"), blank()) }) }, diff --git a/spec/compiler/build_tables/merge_transitions_spec.cc b/spec/compiler/build_tables/merge_transitions_spec.cc index daef24a2..75434c84 100644 --- a/spec/compiler/build_tables/merge_transitions_spec.cc +++ b/spec/compiler/build_tables/merge_transitions_spec.cc @@ -8,11 +8,11 @@ START_TEST describe("merging character set transitions", []() { typedef map int_map; - + auto bitwise = [](int l, int r) -> int { return l | r; }; - + describe("when none of the transitions intersect", [&]() { it("returns the union of the two sets of transitions", [&]() { int_map map1({ @@ -20,12 +20,12 @@ describe("merging character set transitions", []() { { CharacterSet({ 'x', 'y' }), 2 }, { CharacterSet({ '1', '9' }), 4 }, }); - + int_map map2({ { CharacterSet({ ' ' }), 8 }, { CharacterSet({ '\t' }), 16 }, }); - + AssertThat(merge_char_transitions(map1, map2, bitwise), Equals(int_map({ { CharacterSet({ 'a', 'c' }), 1 }, { CharacterSet({ 'x', 'y' }), 2 }, @@ -33,50 +33,50 @@ describe("merging character set transitions", []() { { CharacterSet({ ' ' }), 8 }, { CharacterSet({ '\t' }), 16 }, }))); - + AssertThat(merge_char_transitions(map2, map1, bitwise), Equals(merge_char_transitions(map1, map2, bitwise))); }); }); - + describe("when transitions intersect", [&]() { it("merges the intersecting transitions using the provided function", [&]() { int_map map1({ { CharacterSet({ {'a', 'f'}, {'A', 'F'} }), 1 }, { CharacterSet({ {'0', '9'} }), 2 }, }); - + int_map map2({ { CharacterSet({ 'c' }), 4 }, { CharacterSet({ '3' }), 8 }, }); - + AssertThat(merge_char_transitions(map1, map2, bitwise), Equals(int_map({ { CharacterSet({ {'a', 'b'}, {'d', 'f'}, {'A', 'F'} }), 1 }, { CharacterSet({ {'c'} }), 5 }, { CharacterSet({ {'0', '2'}, {'4', '9'} }), 2 }, { CharacterSet({ '3' }), 10 }, }))); - + AssertThat(merge_char_transitions(map2, map1, bitwise), Equals(merge_char_transitions(map1, map2, bitwise))); }); }); - + describe("when two of the right transitions intersect the same left transition", [&]() { it("splits the left-hand transition correctly", [&]() { int_map map1({ { CharacterSet({ 'a', 'c' }), 1 }, }); - + int_map map2({ { CharacterSet({ 'a' }), 2 }, { CharacterSet({ 'c' }), 4 }, }); - + AssertThat(merge_char_transitions(map1, map2, bitwise), Equals(int_map({ { CharacterSet({ 'a' }), 3 }, { CharacterSet({ 'c' }), 5 }, }))); - + AssertThat(merge_char_transitions(map2, map1, bitwise), Equals(merge_char_transitions(map1, map2, bitwise))); }); }); diff --git a/spec/compiler/build_tables/rule_can_be_blank_spec.cc b/spec/compiler/build_tables/rule_can_be_blank_spec.cc index 7762b5fe..f5cefc5a 100644 --- a/spec/compiler/build_tables/rule_can_be_blank_spec.cc +++ b/spec/compiler/build_tables/rule_can_be_blank_spec.cc @@ -46,7 +46,7 @@ describe("checking if rules can be blank", [&]() { rule = seq({ blank(), choice({ sym("x"), blank() }) }); AssertThat(rule_can_be_blank(rule), IsTrue()); }); - + it("ignores metadata rules", [&]() { rule = make_shared(blank(), rules::MetadataValue(0)); AssertThat(rule_can_be_blank(rule), IsTrue()); diff --git a/spec/compiler/build_tables/rule_transitions_spec.cc b/spec/compiler/build_tables/rule_transitions_spec.cc index 7b202443..9e5e60f4 100644 --- a/spec/compiler/build_tables/rule_transitions_spec.cc +++ b/spec/compiler/build_tables/rule_transitions_spec.cc @@ -142,7 +142,7 @@ describe("rule transitions", []() { { CharacterSet({ {'a', 'c'} }), choice({ sym("x"), sym("y") }) }, { CharacterSet({ {'d', 'z'} }), sym("y") }, }))); - + AssertThat( char_transitions(choice({ seq({ diff --git a/spec/compiler/rules/character_set_spec.cc b/spec/compiler/rules/character_set_spec.cc index 911d9e41..54c1b1b9 100644 --- a/spec/compiler/rules/character_set_spec.cc +++ b/spec/compiler/rules/character_set_spec.cc @@ -41,7 +41,7 @@ describe("character sets", []() { set.add_set(CharacterSet({ CharacterRange('s', 'z') })); AssertThat(set, Equals(CharacterSet({ {'a', 'z'} }))); }); - + it("becomes the complete set when the complement is added", [&]() { CharacterSet set({ 'c' }); auto complement = set.complement(); @@ -64,7 +64,7 @@ describe("character sets", []() { describe("subtracting sets", []() { CharacterSet intersection; - + it("works for disjoint sets", [&]() { CharacterSet set1({ {'a', 'z'} }); intersection = set1.remove_set(CharacterSet({ {'A', 'Z'} })); @@ -85,7 +85,7 @@ describe("character sets", []() { AssertThat(set1, Equals(CharacterSet({ {'a', 'l'} }))); AssertThat(intersection, Equals(CharacterSet({ {'m', 's'} }))); }); - + it("works for a set that overlaps the left side", [&]() { CharacterSet set2({ {'m','z'} }); intersection = set2.remove_set(CharacterSet({ {'a', 's'} })); diff --git a/src/compiler/build_tables/build_tables.cc b/src/compiler/build_tables/build_tables.cc index 1e3d7a51..9eed8c39 100644 --- a/src/compiler/build_tables/build_tables.cc +++ b/src/compiler/build_tables/build_tables.cc @@ -47,7 +47,7 @@ namespace tree_sitter { lex_table.add_action(state_id, rule, LexAction::Advance(new_state_id)); } } - + void add_token_start(const LexItemSet &item_set, LexStateId state_id) { for (auto &item : item_set) if (item.has_metadata(rules::START_TOKEN)) @@ -78,14 +78,14 @@ namespace tree_sitter { } } } - + rules::rule_ptr after_separators(rules::rule_ptr rule) { return rules::Seq::Build({ make_shared(CharacterSet({ ' ', '\t', '\n', '\r' }).copy()), make_shared(rule, rules::START_TOKEN) }); } - + LexItemSet lex_item_set_for_parse_state(const ParseState &state) { LexItemSet result; for (auto &symbol : state.expected_inputs()) { @@ -101,7 +101,7 @@ namespace tree_sitter { ParseState &state = parse_table.states[state_id]; state.lex_state_id = add_lex_state(lex_item_set_for_parse_state(state)); } - + LexStateId add_lex_state(const LexItemSet &item_set) { auto pair = lex_state_ids.find(item_set); if (pair == lex_state_ids.end()) { diff --git a/src/compiler/build_tables/check_metadata.cc b/src/compiler/build_tables/check_metadata.cc index 5f0c5ae6..c279f50c 100644 --- a/src/compiler/build_tables/check_metadata.cc +++ b/src/compiler/build_tables/check_metadata.cc @@ -11,7 +11,7 @@ namespace tree_sitter { rules::MetadataValue metadata_value; public: HasMetadata(rules::MetadataValue value) : metadata_value(value) {} - + void visit(const rules::Choice *rule) { value = apply(rule->left) || apply(rule->right); } @@ -26,12 +26,12 @@ namespace tree_sitter { result = result || apply(rule->right); value = result; } - + void visit(const rules::Metadata *rule) { value = rule->value & metadata_value; } }; - + bool check_metadata(const rules::rule_ptr &rule, rules::MetadataValue value) { return HasMetadata(value).apply(rule); } diff --git a/src/compiler/build_tables/first_set.cc b/src/compiler/build_tables/first_set.cc index 6b946fbb..1593bfaa 100644 --- a/src/compiler/build_tables/first_set.cc +++ b/src/compiler/build_tables/first_set.cc @@ -35,7 +35,7 @@ namespace tree_sitter { } } } - + void visit(const rules::Metadata *rule) { value = apply(rule->rule); } diff --git a/src/compiler/build_tables/item.cc b/src/compiler/build_tables/item.cc index 64794886..106d2b98 100644 --- a/src/compiler/build_tables/item.cc +++ b/src/compiler/build_tables/item.cc @@ -19,11 +19,11 @@ namespace tree_sitter { bool Item::is_done() const { return rule_can_be_blank(rule); } - + bool Item::has_metadata(rules::MetadataValue value) const { return check_metadata(rule, value); } - + ostream& operator<<(ostream &stream, const LexItem &item) { return stream << string("# pairs_to_insert; - + auto iter = result.begin(); while (iter != result.end()) { rules::CharacterSet char_set = iter->first; T value = iter->second; - + rules::CharacterSet intersection = char_set.remove_set(new_char_set); if (!intersection.is_empty()) { new_char_set.remove_set(intersection); @@ -71,9 +71,9 @@ namespace tree_sitter { ++iter; } } - + result.insert(pairs_to_insert.begin(), pairs_to_insert.end()); - + if (!new_char_set.is_empty()) result.insert({ new_char_set, new_pair.second }); } diff --git a/src/compiler/build_tables/rule_can_be_blank.cc b/src/compiler/build_tables/rule_can_be_blank.cc index 42147bd8..2b836dc2 100644 --- a/src/compiler/build_tables/rule_can_be_blank.cc +++ b/src/compiler/build_tables/rule_can_be_blank.cc @@ -34,7 +34,7 @@ namespace tree_sitter { void visit(const rules::Seq *rule) { value = apply(rule->left) && apply(rule->right); } - + void visit(const rules::Metadata *rule) { value = apply(rule->rule); } diff --git a/src/compiler/build_tables/rule_transitions.cc b/src/compiler/build_tables/rule_transitions.cc index ca49ac6d..f2670bde 100644 --- a/src/compiler/build_tables/rule_transitions.cc +++ b/src/compiler/build_tables/rule_transitions.cc @@ -88,7 +88,7 @@ namespace tree_sitter { return rules::Seq::Build({ value, rule->copy() }); }); } - + void visit(const rules::Metadata *rule) { this->value = this->apply(rule->rule); } diff --git a/src/compiler/prepare_grammar/extract_tokens.cc b/src/compiler/prepare_grammar/extract_tokens.cc index 997a0db6..a1899e8f 100644 --- a/src/compiler/prepare_grammar/extract_tokens.cc +++ b/src/compiler/prepare_grammar/extract_tokens.cc @@ -62,7 +62,7 @@ namespace tree_sitter { vector> rules, tokens, aux_rules, aux_tokens; TokenExtractor extractor; - for (auto &pair : input_grammar.rules) { + for (auto &pair : input_grammar.rules) { string name = pair.first; rule_ptr rule = pair.second; if (IsToken().apply(rule)) diff --git a/src/compiler/rules/metadata.cc b/src/compiler/rules/metadata.cc index 92860d76..bad7e2fd 100644 --- a/src/compiler/rules/metadata.cc +++ b/src/compiler/rules/metadata.cc @@ -6,27 +6,27 @@ namespace tree_sitter { using std::hash; using std::make_shared; - + namespace rules { Metadata::Metadata(rule_ptr rule, MetadataValue value) : rule(rule), value(value) {} - + bool Metadata::operator==(const Rule &rule) const { auto other = dynamic_cast(&rule); return other && other->value == value && other->rule->operator==(*this->rule); } - + size_t Metadata::hash_code() const { return hash()(value); } - + rule_ptr Metadata::copy() const { return make_shared(rule, value); } - + std::string Metadata::to_string() const { return "#to_string() + ">"; } - + void Metadata::accept(Visitor *visitor) const { visitor->visit(this); } diff --git a/src/compiler/rules/metadata.h b/src/compiler/rules/metadata.h index bd3dae5f..8a4608da 100644 --- a/src/compiler/rules/metadata.h +++ b/src/compiler/rules/metadata.h @@ -14,13 +14,13 @@ namespace tree_sitter { class Metadata : public Rule { public: Metadata(rule_ptr rule, MetadataValue value); - + bool operator==(const Rule& other) const; size_t hash_code() const; rule_ptr copy() const; std::string to_string() const; void accept(Visitor *visitor) const; - + const rule_ptr rule; const MetadataValue value; }; diff --git a/src/compiler/rules/visitor.cc b/src/compiler/rules/visitor.cc index 5eb19066..f3620cac 100644 --- a/src/compiler/rules/visitor.cc +++ b/src/compiler/rules/visitor.cc @@ -22,11 +22,11 @@ namespace tree_sitter { void Visitor::visit(const Seq *rule) { default_visit(rule); } void Visitor::visit(const String *rule) { default_visit(rule); } void Visitor::visit(const Symbol *rule) { default_visit(rule); } - + void IdentityRuleFn::default_visit(const Rule *rule) { value = rule->copy(); } - + void IdentityRuleFn::visit(const Choice *rule) { value = Choice::Build({ apply(rule->left), apply(rule->right) }); } diff --git a/src/compiler/rules/visitor.h b/src/compiler/rules/visitor.h index ec9d237d..371654fe 100644 --- a/src/compiler/rules/visitor.h +++ b/src/compiler/rules/visitor.h @@ -40,7 +40,7 @@ namespace tree_sitter { return value; } }; - + class IdentityRuleFn : public RuleFn { virtual void default_visit(const Rule *rule); virtual void visit(const Choice *rule);