From 39aa0ccc91dfa511aa61d3ad9595b74808265d56 Mon Sep 17 00:00:00 2001 From: Max Brunsfeld Date: Sun, 9 Mar 2014 19:49:35 -0700 Subject: [PATCH] Add script to trim whitespace --- include/tree_sitter/compiler.h | 6 +-- script/trim.sh | 3 ++ .../build_tables/build_tables_spec.cpp | 14 +++--- spec/compiler/build_tables/first_set_spec.cpp | 16 +++---- .../build_tables/follow_sets_spec.cpp | 12 ++--- .../build_tables/item_set_closure_spec.cpp | 2 +- .../build_tables/rule_can_be_blank_spec.cpp | 2 +- .../build_tables/rule_transitions_spec.cpp | 26 +++++------ .../helpers/character_set_helpers.cpp | 2 +- spec/compiler/helpers/equals_pointer.h | 10 ++--- spec/compiler/helpers/stream_methods.h | 6 +-- spec/compiler/prepare_grammar_spec.cpp | 18 ++++---- spec/compiler/rules/character_set_spec.cpp | 22 +++++----- spec/compiler/rules/pattern_spec.cpp | 16 +++---- spec/compiler/rules/rules_spec.cpp | 4 +- spec/runtime/arithmetic_spec.cpp | 16 +++---- spec/runtime/json_spec.cpp | 22 +++++----- src/compiler/build_tables/build_tables.cpp | 28 ++++++------ src/compiler/build_tables/build_tables.h | 2 +- src/compiler/build_tables/first_set.cpp | 10 ++--- src/compiler/build_tables/first_set.h | 2 +- src/compiler/build_tables/follow_sets.cpp | 2 +- src/compiler/build_tables/follow_sets.h | 4 +- src/compiler/build_tables/item.cpp | 12 ++--- src/compiler/build_tables/item.h | 10 ++--- .../build_tables/item_set_closure.cpp | 4 +- src/compiler/build_tables/item_set_closure.h | 2 +- .../build_tables/item_set_transitions.cpp | 8 ++-- .../build_tables/item_set_transitions.h | 2 +- src/compiler/build_tables/merge_transitions.h | 4 +- .../build_tables/rule_can_be_blank.cpp | 12 ++--- src/compiler/build_tables/rule_can_be_blank.h | 2 +- .../build_tables/rule_transitions.cpp | 14 +++--- src/compiler/generate_code/c_code.cpp | 44 +++++++++---------- src/compiler/generate_code/helpers.cpp | 8 ++-- src/compiler/grammar.cpp | 10 ++--- src/compiler/lex_table.cpp | 26 +++++------ src/compiler/lex_table.h | 12 ++--- src/compiler/parse_table.cpp | 26 +++++------ src/compiler/parse_table.h | 14 +++--- .../prepare_grammar/expand_repeats.cpp | 8 ++-- src/compiler/prepare_grammar/expand_repeats.h | 2 +- .../prepare_grammar/extract_tokens.cpp | 22 +++++----- src/compiler/prepare_grammar/extract_tokens.h | 2 +- .../prepare_grammar/prepare_grammar.cpp | 2 +- .../prepare_grammar/prepare_grammar.h | 2 +- src/compiler/prepared_grammar.cpp | 12 ++--- src/compiler/prepared_grammar.h | 8 ++-- src/compiler/rules/blank.cpp | 10 ++--- src/compiler/rules/blank.h | 2 +- src/compiler/rules/character_range.cpp | 12 ++--- src/compiler/rules/character_set.cpp | 30 ++++++------- src/compiler/rules/character_set.h | 6 +-- src/compiler/rules/choice.cpp | 10 ++--- src/compiler/rules/pattern.cpp | 40 ++++++++--------- src/compiler/rules/pattern.h | 2 +- src/compiler/rules/repeat.cpp | 10 ++--- src/compiler/rules/rule.cpp | 2 +- src/compiler/rules/rule.h | 2 +- src/compiler/rules/rules.cpp | 12 ++--- src/compiler/rules/seq.cpp | 8 ++-- src/compiler/rules/string.cpp | 12 ++--- src/compiler/rules/symbol.cpp | 12 ++--- src/compiler/rules/symbol.h | 6 +-- src/compiler/rules/visitor.h | 4 +- src/runtime/tree.c | 4 +- 66 files changed, 350 insertions(+), 347 deletions(-) create mode 100755 script/trim.sh diff --git a/include/tree_sitter/compiler.h b/include/tree_sitter/compiler.h index 8f317a4c..718a4f63 100644 --- a/include/tree_sitter/compiler.h +++ b/include/tree_sitter/compiler.h @@ -10,9 +10,9 @@ namespace tree_sitter { namespace rules { class Rule; typedef std::shared_ptr rule_ptr; - + std::ostream& operator<<(std::ostream& stream, const rule_ptr &rule); - + rule_ptr blank(); rule_ptr choice(const std::vector &rules); rule_ptr repeat(const rule_ptr &content); @@ -31,7 +31,7 @@ namespace tree_sitter { const std::string start_rule_name; const std::map rules; }; - + std::ostream& operator<<(std::ostream &stream, const Grammar &grammar); std::string compile(const Grammar &grammar, std::string name); diff --git a/script/trim.sh b/script/trim.sh new file mode 100755 index 00000000..19df440c --- /dev/null +++ b/script/trim.sh @@ -0,0 +1,3 @@ +#!/usr/bin/env bash + +find src spec include -type f | xargs perl -pi -e 's/ +$//' diff --git a/spec/compiler/build_tables/build_tables_spec.cpp b/spec/compiler/build_tables/build_tables_spec.cpp index 866aa90c..eba1b5ed 100644 --- a/spec/compiler/build_tables/build_tables_spec.cpp +++ b/spec/compiler/build_tables/build_tables_spec.cpp @@ -36,7 +36,7 @@ describe("building parse and lex tables", []() { sym("right-paren") }) }) } }, {}); - + PreparedGrammar lex_grammar("", { { "plus", str("+") }, { "variable", pattern("\\w+") }, @@ -44,25 +44,25 @@ describe("building parse and lex tables", []() { { "left-paren", str("(") }, { "right-paren", str(")") } }, {}); - + ParseTable table; LexTable lex_table; - + before_each([&]() { pair tables = build_tables::build_tables(grammar, lex_grammar); table = tables.first; lex_table = tables.second; }); - + function parse_state = [&](size_t index) { return table.states[index]; }; - + function lex_state = [&](size_t parse_state_index) { long index = table.states[parse_state_index].lex_state_id; return lex_table.states[index]; }; - + it("has the right starting state", [&]() { AssertThat(keys(parse_state(0).actions), Equals(set({ Symbol("expression"), @@ -71,7 +71,7 @@ describe("building parse and lex tables", []() { Symbol("variable"), Symbol("left-paren"), }))); - + AssertThat(lex_state(0).expected_inputs(), Equals(set({ CharacterSet({ '(' }), CharacterSet({ CharacterRange('0', '9') }), diff --git a/spec/compiler/build_tables/first_set_spec.cpp b/spec/compiler/build_tables/first_set_spec.cpp index 778bf3e1..2d29dac3 100644 --- a/spec/compiler/build_tables/first_set_spec.cpp +++ b/spec/compiler/build_tables/first_set_spec.cpp @@ -14,7 +14,7 @@ describe("computing FIRST sets", []() { describe("for a sequence AB", [&]() { it("ignores B when A cannot be blank", [&]() { auto rule = seq({ sym("x"), sym("y") }); - + AssertThat(first_set(rule, null_grammar), Equals(set({ Symbol("x"), }))); @@ -26,13 +26,13 @@ describe("computing FIRST sets", []() { sym("x"), blank() }), sym("y") }); - + AssertThat(first_set(rule, null_grammar), Equals(set({ Symbol("x"), Symbol("y") }))); }); - + it("includes FIRST(A's right hand side) when A is a non-terminal", [&]() { auto rule = choice({ seq({ @@ -40,7 +40,7 @@ describe("computing FIRST sets", []() { sym("x"), sym("A") }), sym("A") }); - + Grammar grammar("A", { { "A", choice({ seq({ @@ -49,19 +49,19 @@ describe("computing FIRST sets", []() { sym("y") }), sym("y") }) } }); - + AssertThat(first_set(rule, grammar), Equals(set({ Symbol("y") }))); }); - + it("includes FIRST(B) when A is a non-terminal and its expansion can be blank", [&]() { Grammar grammar("A", {{ "A", choice({ sym("x"), blank() }) }}); - + auto rule = seq({ sym("A"), sym("y") }); - + AssertThat(first_set(rule, grammar), Equals(set({ Symbol("x"), Symbol("y") diff --git a/spec/compiler/build_tables/follow_sets_spec.cpp b/spec/compiler/build_tables/follow_sets_spec.cpp index ba67078f..6adea9a1 100644 --- a/spec/compiler/build_tables/follow_sets_spec.cpp +++ b/spec/compiler/build_tables/follow_sets_spec.cpp @@ -13,35 +13,35 @@ describe("computing FOLLOW sets", []() { { "A", sym("a") }, { "B", sym("b") }, }, {}); - + it("all of the starting non-terminals for the item, and their following terminals", [&]() { ParseItem item(Symbol("C"), choice({ seq({ sym("A"), choice({ sym("x"), sym("y") }) }), seq({ sym("B"), sym("z") }), }), {}, Symbol("w")); - + AssertThat(follow_sets(item, grammar), Equals(map>({ { Symbol("A"), set({ Symbol("x"), Symbol("y") }) }, { Symbol("B"), set({ Symbol("z") }) }, }))); }); - + it("does not include terminals at the beginning of the item", [&]() { ParseItem item(Symbol("C"), choice({ seq({ sym("A"), choice({ sym("x"), sym("y") }) }), seq({ sym("x"), sym("y") }), }), {}, Symbol("w")); - + AssertThat(follow_sets(item, grammar), Equals(map>({ { Symbol("A"), set({ Symbol("x"), Symbol("y") }) }, }))); }); - + it("includes the item's lookahead terminal if the rule after the non-terminal might be blank", [&]() { ParseItem item(Symbol("C"), choice({ seq({ sym("A"), choice({ sym("x"), blank() }) }), }), {}, Symbol("w")); - + AssertThat(follow_sets(item, grammar), Equals(map>({ { Symbol("A"), set({ Symbol("x"), Symbol("w") }) }, }))); diff --git a/spec/compiler/build_tables/item_set_closure_spec.cpp b/spec/compiler/build_tables/item_set_closure_spec.cpp index 19e7e9b0..9bee8609 100644 --- a/spec/compiler/build_tables/item_set_closure_spec.cpp +++ b/spec/compiler/build_tables/item_set_closure_spec.cpp @@ -26,7 +26,7 @@ describe("computing closures of item sets", []() { sym("v"), sym("n") }) } }, {}); - + it("computes the item set closure", [&]() { ParseItemSet item_set = item_set_closure(ParseItemSet({ ParseItem(Symbol("E"), grammar.rule(Symbol("E")), {}, Symbol("__END__")) diff --git a/spec/compiler/build_tables/rule_can_be_blank_spec.cpp b/spec/compiler/build_tables/rule_can_be_blank_spec.cpp index deffa12f..684a6719 100644 --- a/spec/compiler/build_tables/rule_can_be_blank_spec.cpp +++ b/spec/compiler/build_tables/rule_can_be_blank_spec.cpp @@ -15,7 +15,7 @@ describe("checking if rules can be blank", [&]() { }), str("y"), }); - + AssertThat(rule_can_be_blank(rule), Equals(false)); }); }); diff --git a/spec/compiler/build_tables/rule_transitions_spec.cpp b/spec/compiler/build_tables/rule_transitions_spec.cpp index f65264b0..78d72d16 100644 --- a/spec/compiler/build_tables/rule_transitions_spec.cpp +++ b/spec/compiler/build_tables/rule_transitions_spec.cpp @@ -16,7 +16,7 @@ public: } return true; } - + rule_map(const initializer_list> &list) : map(list) {} }; @@ -30,7 +30,7 @@ describe("rule transitions", []() { { Symbol("1"), blank() } }))); }); - + it("handles choices", [&]() { AssertThat( sym_transitions(choice({ sym("1"), sym("2") })), @@ -39,7 +39,7 @@ describe("rule transitions", []() { { Symbol("2"), blank() } }))); }); - + it("handles sequences", [&]() { AssertThat( sym_transitions(seq({ sym("1"), sym("2") })), @@ -47,7 +47,7 @@ describe("rule transitions", []() { { Symbol("1"), sym("2") } }))); }); - + it("handles long sequences", [&]() { AssertThat( sym_transitions(seq({ @@ -60,7 +60,7 @@ describe("rule transitions", []() { { Symbol("1"), seq({ sym("2"), sym("3"), sym("4") }) } }))); }); - + it("handles sequences whose left sides can be blank", [&]() { AssertThat( sym_transitions(seq({ @@ -76,7 +76,7 @@ describe("rule transitions", []() { { Symbol("1"), choice({ seq({ sym("1"), sym("2") }), sym("2"), }) } }))); }); - + it("handles choices with common starting symbols", [&]() { AssertThat( sym_transitions( @@ -87,7 +87,7 @@ describe("rule transitions", []() { { Symbol("1"), choice({ sym("2"), sym("3") }) } }))); }); - + it("handles characters", [&]() { AssertThat( char_transitions(character({ '1' })), @@ -95,7 +95,7 @@ describe("rule transitions", []() { { CharacterSet({ '1' }), blank() } }))); }); - + it("handles strings", [&]() { AssertThat( char_transitions(str("bad")), @@ -103,7 +103,7 @@ describe("rule transitions", []() { { CharacterSet({ 'b' }), seq({ character({ 'a' }), character({ 'd' }) }) } }))); }); - + it("handles patterns", [&]() { AssertThat( char_transitions(pattern("a|b")), @@ -112,8 +112,8 @@ describe("rule transitions", []() { { CharacterSet({ 'b' }), blank() } }))); }); - - + + it("handles choices between overlapping character sets", [&]() { AssertThat( char_transitions(choice({ @@ -145,7 +145,7 @@ describe("rule transitions", []() { }) }) }}))); - + rule = repeat(str("a")); AssertThat( char_transitions(rule), @@ -168,7 +168,7 @@ describe("rule transitions", []() { }), character({ '"' }), }); - + AssertThat(char_transitions(rule), Equals(rule_map({ { CharacterSet({ '"' }).complement(), seq({ choice({ diff --git a/spec/compiler/helpers/character_set_helpers.cpp b/spec/compiler/helpers/character_set_helpers.cpp index 9f0a5d35..b72592a0 100644 --- a/spec/compiler/helpers/character_set_helpers.cpp +++ b/spec/compiler/helpers/character_set_helpers.cpp @@ -4,7 +4,7 @@ namespace tree_sitter { using std::make_shared; using std::set; - + namespace rules { rule_ptr character(const set &ranges) { return make_shared(ranges); diff --git a/spec/compiler/helpers/equals_pointer.h b/spec/compiler/helpers/equals_pointer.h index c5a4b8cd..348f7fe6 100644 --- a/spec/compiler/helpers/equals_pointer.h +++ b/spec/compiler/helpers/equals_pointer.h @@ -5,20 +5,20 @@ namespace snowhouse { using namespace std; - + template struct EqualsPointerConstraint : Expression> { EqualsPointerConstraint(const ExpectedType& expected) : expected(expected) {} - + template bool operator()(const ActualType& actual) const { return *expected == *actual; } - + ExpectedType expected; }; - + template struct Stringizer> { @@ -28,7 +28,7 @@ namespace snowhouse { return builder.str(); } }; - + template inline EqualsPointerConstraint EqualsPointer(const ExpectedType& expected) { return EqualsPointerConstraint(expected); diff --git a/spec/compiler/helpers/stream_methods.h b/spec/compiler/helpers/stream_methods.h index 0a90080e..7a5a7bdd 100644 --- a/spec/compiler/helpers/stream_methods.h +++ b/spec/compiler/helpers/stream_methods.h @@ -8,7 +8,7 @@ using std::cout; namespace std { - + template inline std::ostream& operator<<(std::ostream &stream, const std::vector &vector) { stream << std::string("#"; } - + template inline std::ostream& operator<<(std::ostream &stream, const std::set &set) { stream << std::string("#"; } - + template inline std::ostream& operator<<(std::ostream &stream, const std::map &map) { stream << std::string("#("token1", SymbolTypeAuxiliary), @@ -28,36 +28,36 @@ describe("preparing a grammar", []() { sym("rule3") }), make_shared("token1", SymbolTypeAuxiliary) }) } }, {}))); - + AssertThat(result.second, Equals(PreparedGrammar("", {}, { { "token1", str("ab") }, }))); }); - + it("moves entire rules into the lexical grammar when possible, preserving their names", [&]() { auto result = prepare_grammar(Grammar("rule1", { { "rule1", sym("rule2") }, { "rule2", pattern("a|b") } })); - + AssertThat(result.first, Equals(PreparedGrammar("rule1", { { "rule1", sym("rule2") } }, {}))); - + AssertThat(result.second, Equals(PreparedGrammar("", { { "rule2", pattern("a|b") }, }, {}))); }); - + it("does not extract blanks into tokens", [&]() { pair result = prepare_grammar(Grammar("rule1", { { "rule1", choice({ sym("rule2"), blank() }) }, })); - + AssertThat(result.first, Equals(PreparedGrammar("rule1", { { "rule1", choice({ sym("rule2"), blank() }) }, }, {}))); - + AssertThat(result.second, Equals(PreparedGrammar("", {}, {}))); }); }); @@ -71,7 +71,7 @@ describe("preparing a grammar", []() { sym("y") }) }, })).first; - + AssertThat(result, Equals(PreparedGrammar("rule1", { { "rule1", seq({ sym("x"), diff --git a/spec/compiler/rules/character_set_spec.cpp b/spec/compiler/rules/character_set_spec.cpp index 4d90e4a6..0afc2a5e 100644 --- a/spec/compiler/rules/character_set_spec.cpp +++ b/spec/compiler/rules/character_set_spec.cpp @@ -7,7 +7,7 @@ START_TEST describe("character sets", []() { char max_char = 255; - + describe("computing the complement", [&]() { it("works for the set containing only the null character", [&]() { CharacterSet set1({ '\0' }); @@ -28,14 +28,14 @@ describe("character sets", []() { AssertThat(set2.complement(), Equals(set1)); }); }); - + describe("computing unions", [&]() { it("works for disjoint sets", [&]() { CharacterSet set({ {'a', 'z'} }); set.add_set(CharacterSet({ {'A', 'Z'} })); AssertThat(set, Equals(CharacterSet({ {'a', 'z'}, {'A', 'Z'} }))); }); - + it("works for sets with adjacent ranges", [&]() { CharacterSet set({ CharacterRange('a', 'r') }); set.add_set(CharacterSet({ CharacterRange('s', 'z') })); @@ -46,33 +46,33 @@ describe("character sets", []() { set.add_set(c); AssertThat(set, Equals(CharacterSet({ {0, max_char} }))); }); - + it("works when the result becomes a continuous range", []() { CharacterSet set({ {'a', 'd'}, {'f', 'z'} }); set.add_set(CharacterSet({ {'c', 'g'} })); AssertThat(set, Equals(CharacterSet({ {'a', 'z'} }))); }); - + it("does nothing for the set of all characters", [&]() { CharacterSet set({ 'a' }); set.add_set(set.complement()); AssertThat(set, Equals(CharacterSet({ {'\0', max_char} }))); }); }); - + describe("computing differences", []() { it("works for disjoint sets", []() { CharacterSet set1({ {'a','z'} }); set1.remove_set(CharacterSet({ {'A','Z'} })); AssertThat(set1, Equals(CharacterSet({ {'a', 'z'} }))); }); - + it("works when one set spans the other", []() { CharacterSet set1({ {'a','z'} }); set1.remove_set(CharacterSet({ {'d','s'} })); AssertThat(set1, Equals(CharacterSet({ {'a', 'c'}, {'t', 'z'} }))); }); - + it("works for sets that overlap", []() { CharacterSet set1({ {'a','s'} }); set1.remove_set(CharacterSet({ {'m','z'} })); @@ -82,21 +82,21 @@ describe("character sets", []() { set2.remove_set(CharacterSet({ {'a','s'} })); AssertThat(set2, Equals(CharacterSet({ {'t', 'z'} }))); }); - + it("works for sets with multiple ranges", []() { CharacterSet set1({ {'a','d'}, {'m', 'z'} }); set1.remove_set(CharacterSet({ {'c','o'}, {'s','x'} })); AssertThat(set1, Equals(CharacterSet({ {'a', 'b'}, {'p','r'}, {'y','z'} }))); }); }); - + describe("computing intersections", []() { it("returns an empty set for disjoint sets", []() { CharacterSet set1({ {'a','d'} }); CharacterSet set2({ {'e','x'} }); AssertThat(set1.intersect(set2), Equals(CharacterSet())); }); - + it("works for sets with a single overlapping range", []() { CharacterSet set1({ {'a','e'} }); CharacterSet set2({ {'c','x'} }); diff --git a/spec/compiler/rules/pattern_spec.cpp b/spec/compiler/rules/pattern_spec.cpp index 5291acac..5fa90e2e 100644 --- a/spec/compiler/rules/pattern_spec.cpp +++ b/spec/compiler/rules/pattern_spec.cpp @@ -48,35 +48,35 @@ describe("parsing pattern rules", []() { }) }))); }); - + it("parses character sets", []() { Pattern rule("[aAeE]"); AssertThat( rule.to_rule_tree(), EqualsPointer(character({ 'a', 'A', 'e', 'E' }))); }); - + it("parses character ranges", []() { Pattern rule("[12a-dA-D3]"); AssertThat( rule.to_rule_tree(), EqualsPointer(character({ {'1', '3'}, {'a', 'd'}, { 'A', 'D' }, }))); }); - + it("parses negated characters", []() { Pattern rule("[^a\\d]"); AssertThat( rule.to_rule_tree(), EqualsPointer(character({ {'a'}, {'0', '9'} }, false))); }); - + it("parses backslashes", []() { Pattern rule("\\\\"); AssertThat( rule.to_rule_tree(), EqualsPointer(character({ '\\' }))); }); - + it("parses character groups in sequences", []() { Pattern rule("\"([^\"]|\\\\\")+\""); AssertThat( @@ -90,7 +90,7 @@ describe("parsing pattern rules", []() { character({ '"' }) }))); }); - + it("parses choices in sequences", []() { Pattern rule("(a|b)cd"); AssertThat( @@ -104,7 +104,7 @@ describe("parsing pattern rules", []() { character({ 'd' }) }))); }); - + it("parses special characters when they are escaped", []() { Pattern rule("a\\(b"); AssertThat( @@ -115,7 +115,7 @@ describe("parsing pattern rules", []() { character({ 'b' }) }))); }); - + it("parses repeating rules", []() { Pattern rule("(ab)+(cd)+"); AssertThat( diff --git a/spec/compiler/rules/rules_spec.cpp b/spec/compiler/rules/rules_spec.cpp index c0c6ea90..9d9700a2 100644 --- a/spec/compiler/rules/rules_spec.cpp +++ b/spec/compiler/rules/rules_spec.cpp @@ -9,12 +9,12 @@ describe("constructing rules", []() { rule_ptr symbol1 = sym("1"); rule_ptr symbol2 = sym("2"); rule_ptr symbol3 = sym("3"); - + it("constructs binary trees", [&]() { AssertThat( seq({ symbol1, symbol2, symbol3 }), EqualsPointer(seq({ seq({ symbol1, symbol2 }), symbol3 }))); - + AssertThat( choice({ symbol1, symbol2, symbol3 }), EqualsPointer(choice({ choice({ symbol1, symbol2 }), symbol3 }))); diff --git a/spec/runtime/arithmetic_spec.cpp b/spec/runtime/arithmetic_spec.cpp index 09e56272..240d18b7 100644 --- a/spec/runtime/arithmetic_spec.cpp +++ b/spec/runtime/arithmetic_spec.cpp @@ -6,22 +6,22 @@ START_TEST describe("arithmetic", []() { ts_document *doc; - + before_each([&]() { doc = ts_document_make(); ts_document_set_parser(doc, ts_parse_config_arithmetic); }); - + after_each([&]() { ts_document_free(doc); }); - + it("parses variables", [&]() { ts_document_set_input_string(doc, "x"); AssertThat(string(ts_document_string(doc)), Equals( "(expression (term (factor (variable))))")); }); - + it("parses numbers", [&]() { ts_document_set_input_string(doc, "5"); AssertThat(string(ts_document_string(doc)), Equals( @@ -32,12 +32,12 @@ describe("arithmetic", []() { ts_document_set_input_string(doc, "x + y"); AssertThat(string(ts_document_string(doc)), Equals( "(expression (term (factor (variable))) (plus) (term (factor (variable))))")); - + ts_document_set_input_string(doc, "x * y"); AssertThat(string(ts_document_string(doc)), Equals( "(expression (term (factor (variable)) (times) (factor (variable))))")); }); - + it("parses complex trees", [&]() { ts_document_set_input_string(doc, "x * y + z * a"); AssertThat(string(ts_document_string(doc)), Equals( @@ -47,13 +47,13 @@ describe("arithmetic", []() { AssertThat(string(ts_document_string(doc)), Equals( "(expression (term (factor (variable)) (times) (factor (expression (term (factor (variable))) (plus) (term (factor (variable)))))))")); }); - + describe("error recovery", [&]() { it("recovers from errors at the top level", [&]() { ts_document_set_input_string(doc, "x * * y"); AssertThat(string(ts_document_string(doc)), Equals("(ERROR)")); }); - + it("recovers from errors in parenthesized expressions", [&]() { ts_document_set_input_string(doc, "x + (y * + z) * 5"); AssertThat(string(ts_document_string(doc)), Equals( diff --git a/spec/runtime/json_spec.cpp b/spec/runtime/json_spec.cpp index 2d585f9a..d2ba792e 100644 --- a/spec/runtime/json_spec.cpp +++ b/spec/runtime/json_spec.cpp @@ -6,16 +6,16 @@ START_TEST describe("json", []() { ts_document *doc; - + before_each([&]() { doc = ts_document_make(); ts_document_set_parser(doc, ts_parse_config_json); }); - + after_each([&]() { ts_document_free(doc); }); - + it("parses strings", [&]() { ts_document_set_input_string(doc, "\"\""); AssertThat(string(ts_document_string(doc)), Equals("(value (string))")); @@ -26,7 +26,7 @@ describe("json", []() { ts_document_set_input_string(doc, "\"this is a \\\"string\\\" within a string\""); AssertThat(string(ts_document_string(doc)), Equals("(value (string))")); }); - + it("parses objects", [&]() { ts_document_set_input_string(doc, "{}"); AssertThat(string(ts_document_string(doc)), Equals("(value (object))")); @@ -37,7 +37,7 @@ describe("json", []() { ts_document_set_input_string(doc, "{\"key1\": 1, \"key2\": 2 }"); AssertThat(string(ts_document_string(doc)), Equals("(value (object (string) (value (number)) (string) (value (number))))")); }); - + it("parses arrays", [&]() { ts_document_set_input_string(doc, "[]"); AssertThat(string(ts_document_string(doc)), Equals("(value (array))")); @@ -48,11 +48,11 @@ describe("json", []() { ts_document_set_input_string(doc, "[1, 2, 3]"); AssertThat(string(ts_document_string(doc)), Equals("(value (array (value (number)) (value (number)) (value (number))))")); }); - + describe("tracking the positions of AST nodes", [&]() { it("records the widths and offsets of nodes", [&]() { ts_document_set_input_string(doc, " [12, 5]"); - + const ts_tree *tree = ts_document_tree(doc); const ts_tree *array = ts_tree_children(tree)[0]; const ts_tree *number1 = ts_tree_children(array)[0]; @@ -66,18 +66,18 @@ describe("json", []() { AssertThat(array->offset, Equals(2)); AssertThat(array->size, Equals(7)); - + AssertThat(tree->offset, Equals(2)); AssertThat(tree->size, Equals(7)); }); }); - + describe("errors", [&]() { it("reports errors in the top-level node", [&]() { ts_document_set_input_string(doc, "["); AssertThat(string(ts_document_string(doc)), Equals("(ERROR)")); }); - + it("reports errors inside of arrays and objects", [&]() { ts_document_set_input_string(doc, "{ \"key1\": 1, 5 }"); AssertThat(string(ts_document_string(doc)), Equals("(value (object (string) (value (number)) (ERROR)))")); @@ -85,7 +85,7 @@ describe("json", []() { ts_document_set_input_string(doc, "[1,,2]"); AssertThat(string(ts_document_string(doc)), Equals("(value (array (value (number)) (ERROR) (value (number))))")); }); - + it("reports errors in nested objects", [&]() { ts_document_set_input_string(doc, "{ \"key1\": { \"key2\": 1, 2 }, [, \"key3\": 3 }"); AssertThat(string(ts_document_string(doc)), Equals("(value (object (string) (value (object (string) (value (number)) (ERROR))) (ERROR) (string) (value (number))))")); diff --git a/src/compiler/build_tables/build_tables.cpp b/src/compiler/build_tables/build_tables.cpp index ff9a1409..b19454d4 100644 --- a/src/compiler/build_tables/build_tables.cpp +++ b/src/compiler/build_tables/build_tables.cpp @@ -25,7 +25,7 @@ namespace tree_sitter { map lex_state_ids; ParseTable parse_table; LexTable lex_table; - + long parse_state_id_for_item_set(const ParseItemSet &item_set) const { auto entry = parse_state_ids.find(item_set); return (entry == parse_state_ids.end()) ? NOT_FOUND : entry->second; @@ -35,20 +35,20 @@ namespace tree_sitter { auto entry = lex_state_ids.find(item_set); return (entry == lex_state_ids.end()) ? NOT_FOUND : entry->second; } - + void add_shift_actions(const ParseItemSet &item_set, ParseStateId state_id) { for (auto transition : sym_transitions(item_set, grammar)) { Symbol symbol = transition.first; ParseItemSet item_set = transition.second; ParseStateId new_state_id = add_parse_state(item_set); parse_table.add_action(state_id, symbol, ParseAction::Shift(new_state_id)); - + if (symbol == rules::ERROR) { parse_table.error_table.insert({ state_id, { new_state_id, first_set(transition.second, grammar) } }); } } } - + void add_advance_actions(const LexItemSet &item_set, LexStateId state_id) { for (auto transition : char_transitions(item_set, grammar)) { CharacterSet rule = transition.first; @@ -57,7 +57,7 @@ namespace tree_sitter { lex_table.add_action(state_id, rule, LexAction::Advance(new_state_id)); } } - + void add_accept_token_actions(const LexItemSet &item_set, LexStateId state_id) { for (LexItem item : item_set) { if (item.is_done()) { @@ -65,7 +65,7 @@ namespace tree_sitter { } } } - + void add_reduce_actions(const ParseItemSet &item_set, ParseStateId state_id) { for (ParseItem item : item_set) { if (item.is_done()) { @@ -76,7 +76,7 @@ namespace tree_sitter { } } } - + void assign_lex_state(ParseStateId state_id) { ParseState &state = parse_table.states[state_id]; LexItemSet item_set; @@ -87,7 +87,7 @@ namespace tree_sitter { state.lex_state_id = add_lex_state(item_set); } - + LexStateId add_lex_state(const LexItemSet &item_set) { auto state_id = lex_state_id_for_item_set(item_set); if (state_id == NOT_FOUND) { @@ -98,7 +98,7 @@ namespace tree_sitter { } return state_id; } - + ParseStateId add_parse_state(const ParseItemSet &item_set) { auto state_id = parse_state_id_for_item_set(item_set); if (state_id == NOT_FOUND) { @@ -111,7 +111,7 @@ namespace tree_sitter { } return state_id; } - + void add_error_lex_state() { LexItemSet error_item_set; for (auto &pair : lex_grammar.rules) @@ -126,7 +126,7 @@ namespace tree_sitter { // std::vector item_sets(parse_state_ids.size()); // for (auto &pair : parse_state_ids) // item_sets[pair.second] = &pair.first; -// +// // for (int i = 0; i < item_sets.size(); i++) { // std:cout << "\n\n" << i; // for (auto &item : *item_sets[i]) { @@ -136,9 +136,9 @@ namespace tree_sitter { // } // } // } - + public: - + TableBuilder(const PreparedGrammar &grammar, const PreparedGrammar &lex_grammar) : grammar(grammar), lex_grammar(lex_grammar) {}; @@ -151,7 +151,7 @@ namespace tree_sitter { return pair(parse_table, lex_table); } }; - + pair build_tables(const PreparedGrammar &grammar, const PreparedGrammar &lex_grammar) { return TableBuilder(grammar, lex_grammar).build(); } diff --git a/src/compiler/build_tables/build_tables.h b/src/compiler/build_tables/build_tables.h index 9dd01502..8939ba18 100644 --- a/src/compiler/build_tables/build_tables.h +++ b/src/compiler/build_tables/build_tables.h @@ -6,7 +6,7 @@ namespace tree_sitter { class PreparedGrammar; - + namespace build_tables { std::pair build_tables(const PreparedGrammar &grammar, const PreparedGrammar &lex_grammar); } diff --git a/src/compiler/build_tables/first_set.cpp b/src/compiler/build_tables/first_set.cpp index 5d4df4b6..29e019e1 100644 --- a/src/compiler/build_tables/first_set.cpp +++ b/src/compiler/build_tables/first_set.cpp @@ -21,7 +21,7 @@ namespace tree_sitter { const PreparedGrammar grammar; public: FirstSet(const PreparedGrammar &grammar) : grammar(grammar) {} - + void visit(const Symbol *rule) { if (grammar.has_definition(*rule)) { value = apply(grammar.rule(*rule)); @@ -29,11 +29,11 @@ namespace tree_sitter { value = set({ *rule }); } } - + void visit(const Choice *rule) { value = set_union(apply(rule->left), apply(rule->right)); } - + void visit(const Seq *rule) { auto result = apply(rule->left); if (rule_can_be_blank(rule->left, grammar)) { @@ -42,11 +42,11 @@ namespace tree_sitter { value = result; } }; - + set first_set(const rule_ptr &rule, const PreparedGrammar &grammar) { return FirstSet(grammar).apply(rule); } - + set first_set(const ParseItemSet &item_set, const PreparedGrammar &grammar) { set result; for (auto &item : item_set) { diff --git a/src/compiler/build_tables/first_set.h b/src/compiler/build_tables/first_set.h index cab3cbbf..418cd706 100644 --- a/src/compiler/build_tables/first_set.h +++ b/src/compiler/build_tables/first_set.h @@ -7,7 +7,7 @@ namespace tree_sitter { class PreparedGrammar; - + namespace build_tables { /* diff --git a/src/compiler/build_tables/follow_sets.cpp b/src/compiler/build_tables/follow_sets.cpp index 1e3f48b3..af885ea1 100644 --- a/src/compiler/build_tables/follow_sets.cpp +++ b/src/compiler/build_tables/follow_sets.cpp @@ -24,7 +24,7 @@ namespace tree_sitter { result.insert({ symbol, following_terminals }); } } - + return result; } } diff --git a/src/compiler/build_tables/follow_sets.h b/src/compiler/build_tables/follow_sets.h index e3b46970..b61ec8a4 100644 --- a/src/compiler/build_tables/follow_sets.h +++ b/src/compiler/build_tables/follow_sets.h @@ -8,9 +8,9 @@ namespace tree_sitter { class PreparedGrammar; - + namespace build_tables { - + /* * Returns a map of non-terminal symbols to sets of terminal symbols. * The keys are the non-terminals which may appear first in the given diff --git a/src/compiler/build_tables/item.cpp b/src/compiler/build_tables/item.cpp index 7242d3e1..08efd941 100644 --- a/src/compiler/build_tables/item.cpp +++ b/src/compiler/build_tables/item.cpp @@ -14,11 +14,11 @@ namespace tree_sitter { Item::Item(const Symbol &lhs, const rule_ptr rule) : lhs(lhs), rule(rule) {}; - + bool Item::is_done() const { return rule_can_be_blank(rule); } - + ostream& operator<<(ostream &stream, const LexItem &item) { return stream << string("#"); } - + ostream& operator<<(ostream &stream, const ParseItem &item) { return stream << string("#"); } - + bool LexItem::operator<(const LexItem &other) const { if (lhs < other.lhs) return true; if (other.lhs < lhs) return false; @@ -60,13 +60,13 @@ namespace tree_sitter { } LexItem::LexItem(const Symbol &lhs, const rule_ptr rule) : Item(lhs, rule) {} - + bool LexItem::operator==(const LexItem &other) const { bool lhs_eq = other.lhs == lhs; bool rules_eq = (*other.rule == *rule); return lhs_eq && rules_eq; } - + ParseItem::ParseItem(const Symbol &lhs, const rule_ptr rule, const vector &consumed_symbols, const Symbol &lookahead_sym) : Item(lhs, rule), consumed_symbols(consumed_symbols), diff --git a/src/compiler/build_tables/item.h b/src/compiler/build_tables/item.h index 8826cec2..9a32231d 100644 --- a/src/compiler/build_tables/item.h +++ b/src/compiler/build_tables/item.h @@ -8,7 +8,7 @@ namespace tree_sitter { class Grammar; - + namespace build_tables { class Item { public: @@ -38,9 +38,9 @@ namespace tree_sitter { typedef std::set ParseItemSet; typedef std::set LexItemSet; - - std::ostream& operator<<(std::ostream &stream, const LexItem &item); - std::ostream& operator<<(std::ostream &stream, const ParseItem &item); + + std::ostream& operator<<(std::ostream &stream, const LexItem &item); + std::ostream& operator<<(std::ostream &stream, const ParseItem &item); } } @@ -64,7 +64,7 @@ namespace std { hash()(item.lookahead_sym.name); } }; - + template struct hash> { size_t operator()(const set &set) const { diff --git a/src/compiler/build_tables/item_set_closure.cpp b/src/compiler/build_tables/item_set_closure.cpp index 5fc39333..d6986f10 100644 --- a/src/compiler/build_tables/item_set_closure.cpp +++ b/src/compiler/build_tables/item_set_closure.cpp @@ -8,12 +8,12 @@ namespace tree_sitter { using std::set; using rules::Symbol; - + namespace build_tables { static bool contains(const ParseItemSet &items, const ParseItem &item) { return items.size() > 0 && (std::find(items.begin(), items.end(), item) != items.end()); } - + static void add_item(ParseItemSet &item_set, const ParseItem &item, const PreparedGrammar &grammar) { if (!contains(item_set, item)) { item_set.insert(item); diff --git a/src/compiler/build_tables/item_set_closure.h b/src/compiler/build_tables/item_set_closure.h index 714bdd4a..a4f306e5 100644 --- a/src/compiler/build_tables/item_set_closure.h +++ b/src/compiler/build_tables/item_set_closure.h @@ -5,7 +5,7 @@ namespace tree_sitter { class PreparedGrammar; - + namespace build_tables { const ParseItemSet item_set_closure(const ParseItemSet &item_set, const PreparedGrammar &grammar); } diff --git a/src/compiler/build_tables/item_set_transitions.cpp b/src/compiler/build_tables/item_set_transitions.cpp index 1d2f9661..eeace711 100644 --- a/src/compiler/build_tables/item_set_transitions.cpp +++ b/src/compiler/build_tables/item_set_transitions.cpp @@ -8,7 +8,7 @@ namespace tree_sitter { using std::set; using rules::CharacterSet; using rules::Symbol; - + namespace build_tables { map char_transitions(const LexItem &item) { map result; @@ -18,7 +18,7 @@ namespace tree_sitter { } return result; } - + map sym_transitions(const ParseItem &item, const PreparedGrammar &grammar) { map result; for (auto transition : sym_transitions(item.rule)) { @@ -30,14 +30,14 @@ namespace tree_sitter { } return result; } - + template static set merge_sets(const set &left, const set &right) { set result = left; result.insert(right.begin(), right.end()); return result; } - + map char_transitions(const LexItemSet &item_set, const PreparedGrammar &grammar) { map result; for (const LexItem &item : item_set) { diff --git a/src/compiler/build_tables/item_set_transitions.h b/src/compiler/build_tables/item_set_transitions.h index 18cd0b5c..f4795945 100644 --- a/src/compiler/build_tables/item_set_transitions.h +++ b/src/compiler/build_tables/item_set_transitions.h @@ -10,7 +10,7 @@ namespace tree_sitter { class CharacterSet; class Symbol; } - + namespace build_tables { std::map char_transitions(const LexItemSet &item_set, const PreparedGrammar &grammar); std::map sym_transitions(const ParseItemSet &item_set, const PreparedGrammar &grammar); diff --git a/src/compiler/build_tables/merge_transitions.h b/src/compiler/build_tables/merge_transitions.h index 26d8bdf8..121e0a65 100644 --- a/src/compiler/build_tables/merge_transitions.h +++ b/src/compiler/build_tables/merge_transitions.h @@ -7,7 +7,7 @@ namespace tree_sitter { namespace build_tables { - + /* * Merges two transition maps with symbol keys. If both maps * contain values for the same symbol, the new value for that @@ -36,7 +36,7 @@ namespace tree_sitter { } return result; } - + /* * Merges two transition maps with character set keys. If the * two maps contain values for overlapping character sets, the diff --git a/src/compiler/build_tables/rule_can_be_blank.cpp b/src/compiler/build_tables/rule_can_be_blank.cpp index 343651f5..8ec22768 100644 --- a/src/compiler/build_tables/rule_can_be_blank.cpp +++ b/src/compiler/build_tables/rule_can_be_blank.cpp @@ -16,11 +16,11 @@ namespace tree_sitter { void default_visit(const Rule *) { value = false; } - + virtual void visit(const Blank *) { value = true; } - + virtual void visit(const Repeat *rule) { value = true; } @@ -28,12 +28,12 @@ namespace tree_sitter { virtual void visit(const Choice *rule) { value = apply(rule->left) || apply(rule->right); } - + virtual void visit(const Seq *rule) { value = apply(rule->left) && apply(rule->right); } }; - + class CanBeBlankRecursive : public CanBeBlank { const PreparedGrammar grammar; using CanBeBlank::visit; @@ -45,11 +45,11 @@ namespace tree_sitter { value = grammar.has_definition(*rule) && apply(grammar.rule(*rule)); } }; - + bool rule_can_be_blank(const rule_ptr &rule) { return CanBeBlank().apply(rule); } - + bool rule_can_be_blank(const rule_ptr &rule, const PreparedGrammar &grammar) { return CanBeBlankRecursive(grammar).apply(rule); } diff --git a/src/compiler/build_tables/rule_can_be_blank.h b/src/compiler/build_tables/rule_can_be_blank.h index b0145b46..2a95e3a6 100644 --- a/src/compiler/build_tables/rule_can_be_blank.h +++ b/src/compiler/build_tables/rule_can_be_blank.h @@ -5,7 +5,7 @@ namespace tree_sitter { class PreparedGrammar; - + namespace build_tables { bool rule_can_be_blank(const rules::rule_ptr &rule); bool rule_can_be_blank(const rules::rule_ptr &rule, const PreparedGrammar &grammar); diff --git a/src/compiler/build_tables/rule_transitions.cpp b/src/compiler/build_tables/rule_transitions.cpp index b1db4612..b66a076d 100644 --- a/src/compiler/build_tables/rule_transitions.cpp +++ b/src/compiler/build_tables/rule_transitions.cpp @@ -19,7 +19,7 @@ namespace tree_sitter { namespace build_tables { template map merge_transitions(const map &left, const map &right); - + template<> map merge_transitions(const map &left, const map &right) { auto transitions = merge_char_transitions(left, right, [](rule_ptr left, rule_ptr right) { @@ -35,7 +35,7 @@ namespace tree_sitter { }); return *static_cast *>(&transitions); } - + template map map_transitions(const map &initial, std::function map_fn) { map result; @@ -55,7 +55,7 @@ namespace tree_sitter { void visit(const CharacterSet *rule) { visit_atom(rule); } - + void visit(const Symbol *rule) { visit_atom(rule); } @@ -73,25 +73,25 @@ namespace tree_sitter { result = merge_transitions(result, this->apply(rule->right)); this->value = result; } - + void visit(const Repeat *rule) { this->value = map_transitions(this->apply(rule->content), [&](const rule_ptr &value) { return Seq::Build({ value, make_shared(rule->copy(), make_shared()) }); }); } - + void visit(const String *rule) { rule_ptr result = make_shared(); for (char val : rule->value) result = Seq::Build({ result, make_shared(set({ val })) }); this->value = this->apply(result); } - + void visit(const Pattern *rule) { this->value = this->apply(rule->to_rule_tree()); } }; - + map char_transitions(const rule_ptr &rule) { return RuleTransitions().apply(rule); } diff --git a/src/compiler/generate_code/c_code.cpp b/src/compiler/generate_code/c_code.cpp index 3d9017d9..a2b7ea17 100644 --- a/src/compiler/generate_code/c_code.cpp +++ b/src/compiler/generate_code/c_code.cpp @@ -12,7 +12,7 @@ namespace tree_sitter { using std::vector; using std::set; using std::pair; - + namespace generate_code { string _switch(string condition, string body) { return join({ @@ -21,7 +21,7 @@ namespace tree_sitter { "}" }); } - + string _case(string value, string body) { return join({ "case " + value + ":", @@ -35,14 +35,14 @@ namespace tree_sitter { indent(body) }); } - + string _if(string condition, string body) { return join({ "if (" + condition + ")", indent(body), "" }); } - + class CCodeGenerator { const string name; const ParseTable parse_table; @@ -53,7 +53,7 @@ namespace tree_sitter { parse_table(parse_table), lex_table(lex_table) {} - + string symbol_id(rules::Symbol symbol) { if (symbol.is_built_in()) { if (symbol == rules::ERROR) @@ -65,7 +65,7 @@ namespace tree_sitter { else return "ts_sym_" + symbol.name; } - + string character_code(char character) { switch (character) { case '\0': @@ -78,17 +78,17 @@ namespace tree_sitter { return string() + character; } } - + string condition_for_character_range(const rules::CharacterRange &range) { string lookahead("LOOKAHEAD_CHAR()"); if (range.min == range.max) { return lookahead + " == '" + character_code(range.min) + "'"; } else { - return string("'") + character_code(range.min) + string("' <= ") + lookahead + + return string("'") + character_code(range.min) + string("' <= ") + lookahead + " && " + lookahead + " <= '" + character_code(range.max) + "'"; } } - + string condition_for_character_set(const rules::CharacterSet &set) { vector parts; if (set.ranges.size() == 1) { @@ -99,7 +99,7 @@ namespace tree_sitter { return join(parts, " ||\n "); } } - + string condition_for_character_rule(const rules::CharacterSet &rule) { vector parts; pair representation = rule.most_compact_representation(); @@ -108,7 +108,7 @@ namespace tree_sitter { else return "!(" + condition_for_character_set(rule.complement()) + ")"; } - + string collapse_flags(vector flags) { string result; bool started = false; @@ -119,7 +119,7 @@ namespace tree_sitter { } return result; } - + string code_for_parse_actions(const set &actions, const set &expected_inputs) { auto action = actions.begin(); switch (action->type) { @@ -133,7 +133,7 @@ namespace tree_sitter { return ""; } } - + string parse_error_call(const set &expected_inputs) { string result = "PARSE_ERROR(" + to_string(expected_inputs.size()) + ", EXPECT({"; bool started = false; @@ -198,7 +198,7 @@ namespace tree_sitter { body += _default("LEX_PANIC();"); return _switch("LEX_STATE()", body); } - + string symbol_enum() { string result = "enum {\n"; for (auto symbol : parse_table.symbols) @@ -218,7 +218,7 @@ namespace tree_sitter { string includes() { return "#include \"tree_sitter/parser.h\""; } - + string recover_case(ParseStateId state, set symbols) { string result = "RECOVER(" + to_string(state) + ", " + to_string(symbols.size()) + ", EXPECT({"; bool started = false; @@ -231,7 +231,7 @@ namespace tree_sitter { } return result + "}));"; } - + string recover_function() { string cases; for (auto &pair : parse_table.error_table) { @@ -239,7 +239,7 @@ namespace tree_sitter { cases += _case(to_string(pair.first), recover_case(pair_for_state.first, pair_for_state.second)); } cases += _default(recover_case(0, set())); - + string body = _switch("state", cases); return join({ "static const ts_symbol * ts_recover(ts_state state, ts_state *to_state, size_t *count) {", @@ -247,7 +247,7 @@ namespace tree_sitter { "}" }); } - + string lex_function() { return join({ "LEX_FN() {", @@ -256,7 +256,7 @@ namespace tree_sitter { "}" }); } - + string parse_function() { return join({ "PARSE_FN() {", @@ -266,11 +266,11 @@ namespace tree_sitter { "}" }); } - + string parse_config_struct() { return "EXPORT_PARSER(ts_parse_config_" + name + ");"; } - + string code() { return join({ includes(), @@ -283,7 +283,7 @@ namespace tree_sitter { }, "\n\n") + "\n"; } }; - + string c_code(string name, const ParseTable &parse_table, const LexTable &lex_table) { return CCodeGenerator(name, parse_table, lex_table).code(); } diff --git a/src/compiler/generate_code/helpers.cpp b/src/compiler/generate_code/helpers.cpp index 805d5d00..5c994ee7 100644 --- a/src/compiler/generate_code/helpers.cpp +++ b/src/compiler/generate_code/helpers.cpp @@ -12,7 +12,7 @@ namespace tree_sitter { pos += replace.length(); } } - + string join(vector lines, string separator) { string result; bool started = false; @@ -23,17 +23,17 @@ namespace tree_sitter { } return result; } - + string join(vector lines) { return join(lines, "\n"); } - + string indent(string input) { string tab = " "; str_replace(input, "\n", "\n" + tab); return tab + input; } - + string escape_string(string input) { str_replace(input, "\"", "\\\""); return input; diff --git a/src/compiler/grammar.cpp b/src/compiler/grammar.cpp index 62114b58..8e1679a2 100644 --- a/src/compiler/grammar.cpp +++ b/src/compiler/grammar.cpp @@ -6,11 +6,11 @@ namespace tree_sitter { using std::map; using std::ostream; using rules::rule_ptr; - + Grammar::Grammar(std::string start_rule_name, const map &rules) : - start_rule_name(start_rule_name), + start_rule_name(start_rule_name), rules(rules) {} - + bool Grammar::operator==(const Grammar &other) const { if (other.start_rule_name != start_rule_name) return false; if (other.rules.size() != rules.size()) return false; @@ -20,10 +20,10 @@ namespace tree_sitter { if (other_pair == other.rules.end()) return false; if (!other_pair->second->operator==(*pair.second)) return false; } - + return true; } - + ostream& operator<<(ostream &stream, const Grammar &grammar) { stream << string("# other.type) return false; @@ -39,7 +39,7 @@ namespace tree_sitter { if (state_index > other.state_index) return false; return (symbol < other.symbol); } - + std::ostream& operator<<(std::ostream &stream, const LexAction &action) { switch (action.type) { case LexActionTypeError: @@ -50,33 +50,33 @@ namespace tree_sitter { return stream << string("#"; } } - + set LexState::expected_inputs() const { set result; for (auto &pair : actions) result.insert(pair.first); return result; } - + LexStateId LexTable::add_state() { states.push_back(LexState()); return states.size() - 1; } - + LexState & state(LexTable *table, LexStateId id) { if (id < 0) return table->error_state; else return table->states[id]; } - + void LexTable::add_action(LexStateId id, CharacterSet match, LexAction action) { state(this, id).actions[match].insert(action); } - + void LexTable::add_default_action(LexStateId id, LexAction action) { state(this, id).default_actions.insert(action); } - + const LexStateId LexTable::ERROR_STATE_ID = -1; } \ No newline at end of file diff --git a/src/compiler/lex_table.h b/src/compiler/lex_table.h index 6c637d74..104cb1f0 100644 --- a/src/compiler/lex_table.h +++ b/src/compiler/lex_table.h @@ -14,7 +14,7 @@ namespace tree_sitter { LexActionTypeError, LexActionTypeAdvance } LexActionType; - + class LexAction { LexAction(LexActionType type, size_t state_index, rules::Symbol symbol); public: @@ -23,12 +23,12 @@ namespace tree_sitter { static LexAction Advance(size_t state_index); bool operator==(const LexAction &action) const; bool operator<(const LexAction &action) const; - + LexActionType type; rules::Symbol symbol; size_t state_index; }; - + std::ostream& operator<<(std::ostream &stream, const LexAction &item); } @@ -50,16 +50,16 @@ namespace tree_sitter { std::set default_actions; std::set expected_inputs() const; }; - + typedef long int LexStateId; - + class LexTable { public: static const LexStateId ERROR_STATE_ID; LexStateId add_state(); void add_action(LexStateId state_id, rules::CharacterSet rule, LexAction action); void add_default_action(LexStateId state_id, LexAction action); - + std::vector states; LexState error_state; }; diff --git a/src/compiler/parse_table.cpp b/src/compiler/parse_table.cpp index 77ec0df0..02a43a1a 100644 --- a/src/compiler/parse_table.cpp +++ b/src/compiler/parse_table.cpp @@ -7,36 +7,36 @@ namespace tree_sitter { using std::set; using std::vector; using rules::Symbol; - + ParseAction::ParseAction(ParseActionType type, size_t state_index, Symbol symbol, const vector &child_flags) : type(type), symbol(symbol), state_index(state_index), child_flags(child_flags) {}; - + ParseAction ParseAction::Error() { return ParseAction(ParseActionTypeError, -1, Symbol(""), {}); } - + ParseAction ParseAction::Accept() { return ParseAction(ParseActionTypeAccept, -1, Symbol(""), {}); } - + ParseAction ParseAction::Shift(size_t state_index) { return ParseAction(ParseActionTypeShift, state_index, Symbol(""), {}); } - + ParseAction ParseAction::Reduce(Symbol symbol, const vector &child_flags) { return ParseAction(ParseActionTypeReduce, -1, symbol, child_flags); } - + bool ParseAction::operator==(const ParseAction &other) const { bool types_eq = type == other.type; bool state_indices_eq = state_index == other.state_index; bool child_flags_eq = child_flags == other.child_flags; return types_eq && state_indices_eq && child_flags_eq; } - + bool ParseAction::operator<(const ParseAction &other) const { if (type < other.type) return true; if (type > other.type) return false; @@ -44,7 +44,7 @@ namespace tree_sitter { if (state_index > other.state_index) return false; return (child_flags < other.child_flags); } - + ostream& operator<<(ostream &stream, const ParseAction &action) { switch (action.type) { case ParseActionTypeError: @@ -57,16 +57,16 @@ namespace tree_sitter { return stream << (string("#"); } } - + ParseState::ParseState() : lex_state_id(-1) {} - + set ParseState::expected_inputs() const { set result; for (auto pair : actions) result.insert(pair.first); return result; } - + ostream& operator<<(ostream &stream, const ParseState &state) { stream << string("#"); return stream; } - + ParseStateId ParseTable::add_state() { states.push_back(ParseState()); return states.size() - 1; } - + void ParseTable::add_action(ParseStateId id, Symbol symbol, ParseAction action) { symbols.insert(symbol); states[id].actions[symbol].insert(action); diff --git a/src/compiler/parse_table.h b/src/compiler/parse_table.h index 05218439..4dbb2cfe 100644 --- a/src/compiler/parse_table.h +++ b/src/compiler/parse_table.h @@ -14,7 +14,7 @@ namespace tree_sitter { ParseActionTypeShift, ParseActionTypeReduce, } ParseActionType; - + class ParseAction { ParseAction(ParseActionType type, size_t state_index, rules::Symbol symbol, const std::vector &child_flags); public: @@ -24,7 +24,7 @@ namespace tree_sitter { static ParseAction Reduce(rules::Symbol symbol, const std::vector &child_flags); bool operator==(const ParseAction &action) const; bool operator<(const ParseAction &action) const; - + ParseActionType type; rules::Symbol symbol; size_t state_index; @@ -33,7 +33,7 @@ namespace tree_sitter { std::ostream& operator<<(std::ostream &stream, const ParseAction &item); } - + namespace std { template<> struct hash { @@ -55,16 +55,16 @@ namespace tree_sitter { std::set expected_inputs() const; LexStateId lex_state_id; }; - + typedef unsigned long int ParseStateId; - + std::ostream& operator<<(std::ostream &stream, const ParseState &state); - + class ParseTable { public: size_t add_state(); void add_action(ParseStateId state_id, rules::Symbol symbol, ParseAction action); - + std::vector states; std::set symbols; std::map>> error_table; diff --git a/src/compiler/prepare_grammar/expand_repeats.cpp b/src/compiler/prepare_grammar/expand_repeats.cpp index 421859b0..5d1670e4 100644 --- a/src/compiler/prepare_grammar/expand_repeats.cpp +++ b/src/compiler/prepare_grammar/expand_repeats.cpp @@ -22,14 +22,14 @@ namespace tree_sitter { Seq::Build({ rule, make_shared(name, SymbolTypeAuxiliary) }), make_shared() }); } - + void visit(const Repeat *rule) { rule_ptr inner_rule = apply(rule->content); string helper_rule_name = string("repeat_helper") + to_string(aux_rules.size() + 1); aux_rules.insert({ helper_rule_name, make_repeat_helper(helper_rule_name, inner_rule) }); value = make_shared(helper_rule_name, SymbolTypeAuxiliary); } - + void visit(const Seq *rule) { value = Seq::Build({ apply(rule->left), apply(rule->right) }); } @@ -45,14 +45,14 @@ namespace tree_sitter { public: map aux_rules; }; - + PreparedGrammar expand_repeats(const PreparedGrammar &grammar) { map rules, aux_rules(grammar.aux_rules); ExpandRepeats expander; for (auto &pair : grammar.rules) rules.insert({ pair.first, expander.apply(pair.second) }); - + aux_rules.insert(expander.aux_rules.begin(), expander.aux_rules.end()); return PreparedGrammar(grammar.start_rule_name, rules, aux_rules); diff --git a/src/compiler/prepare_grammar/expand_repeats.h b/src/compiler/prepare_grammar/expand_repeats.h index dab4018c..7d1b73e2 100644 --- a/src/compiler/prepare_grammar/expand_repeats.h +++ b/src/compiler/prepare_grammar/expand_repeats.h @@ -5,7 +5,7 @@ namespace tree_sitter { class PreparedGrammar; - + namespace prepare_grammar { PreparedGrammar expand_repeats(const PreparedGrammar &); } diff --git a/src/compiler/prepare_grammar/extract_tokens.cpp b/src/compiler/prepare_grammar/extract_tokens.cpp index 1461f817..7db2371d 100644 --- a/src/compiler/prepare_grammar/extract_tokens.cpp +++ b/src/compiler/prepare_grammar/extract_tokens.cpp @@ -18,13 +18,13 @@ namespace tree_sitter { using std::map; using std::make_shared; using namespace rules; - + namespace prepare_grammar { class IsToken : public RuleFn { void default_visit(const Rule *rule) { value = false; } - + void visit(const String *rule) { value = true; } @@ -43,7 +43,7 @@ namespace tree_sitter { tokens.insert({ name, rule }); return name; } - + void default_visit(const Rule *rule) { auto result = rule->copy(); if (IsToken().apply(result)) { @@ -52,15 +52,15 @@ namespace tree_sitter { value = result; } } - + void visit(const Choice *rule) { value = Choice::Build({ apply(rule->left), apply(rule->right) }); } - + void visit(const Seq *rule) { value = Seq::Build({ apply(rule->left), apply(rule->right) }); } - + void visit(const Repeat *rule) { value = make_shared(apply(rule->content)); } @@ -68,11 +68,11 @@ namespace tree_sitter { public: map tokens; }; - + pair extract_tokens(const PreparedGrammar &input_grammar) { map rules, tokens, aux_rules, aux_tokens; TokenExtractor extractor; - + for (auto &pair : input_grammar.rules) { string name = pair.first; rule_ptr rule = pair.second; @@ -90,10 +90,10 @@ namespace tree_sitter { else aux_rules.insert({ name, extractor.apply(rule) }); } - + aux_tokens.insert(extractor.tokens.begin(), extractor.tokens.end()); - - return { + + return { PreparedGrammar(input_grammar.start_rule_name, rules, aux_rules), PreparedGrammar("", tokens, aux_tokens) }; diff --git a/src/compiler/prepare_grammar/extract_tokens.h b/src/compiler/prepare_grammar/extract_tokens.h index e45efda5..70acf48e 100644 --- a/src/compiler/prepare_grammar/extract_tokens.h +++ b/src/compiler/prepare_grammar/extract_tokens.h @@ -5,7 +5,7 @@ namespace tree_sitter { class PreparedGrammar; - + namespace prepare_grammar { std::pair extract_tokens(const PreparedGrammar &); } diff --git a/src/compiler/prepare_grammar/prepare_grammar.cpp b/src/compiler/prepare_grammar/prepare_grammar.cpp index 69a6bfdd..2c828be6 100644 --- a/src/compiler/prepare_grammar/prepare_grammar.cpp +++ b/src/compiler/prepare_grammar/prepare_grammar.cpp @@ -5,7 +5,7 @@ namespace tree_sitter { using std::pair; - + namespace prepare_grammar { pair prepare_grammar(const Grammar &input_grammar) { auto grammars = prepare_grammar::extract_tokens(input_grammar); diff --git a/src/compiler/prepare_grammar/prepare_grammar.h b/src/compiler/prepare_grammar/prepare_grammar.h index d65cc45d..1c3f80fc 100644 --- a/src/compiler/prepare_grammar/prepare_grammar.h +++ b/src/compiler/prepare_grammar/prepare_grammar.h @@ -6,7 +6,7 @@ namespace tree_sitter { class Grammar; class PreparedGrammar; - + namespace prepare_grammar { std::pair prepare_grammar(const Grammar &); } diff --git a/src/compiler/prepared_grammar.cpp b/src/compiler/prepared_grammar.cpp index 2140f5ec..fe25ffa4 100644 --- a/src/compiler/prepared_grammar.cpp +++ b/src/compiler/prepared_grammar.cpp @@ -15,7 +15,7 @@ namespace tree_sitter { const map &aux_rules) : Grammar(start_rule_name, rules), aux_rules(aux_rules) {} - + PreparedGrammar::PreparedGrammar(std::string start_rule_name, const initializer_list> &rules, const initializer_list> &aux_rules) : @@ -45,14 +45,14 @@ namespace tree_sitter { } return true; } - + bool PreparedGrammar::has_definition(const Symbol &symbol) const { return rule(symbol).get() != nullptr; } - + ostream& operator<<(ostream &stream, const PreparedGrammar &grammar) { stream << string("#"); } } \ No newline at end of file diff --git a/src/compiler/prepared_grammar.h b/src/compiler/prepared_grammar.h index 31522f5a..5164fb15 100644 --- a/src/compiler/prepared_grammar.h +++ b/src/compiler/prepared_grammar.h @@ -14,16 +14,16 @@ namespace tree_sitter { const std::initializer_list> &rules, const std::initializer_list> &aux_rules); PreparedGrammar(const Grammar &grammar); - + bool operator==(const PreparedGrammar &other) const; bool has_definition(const rules::Symbol &symbol) const; const rules::rule_ptr rule(const rules::Symbol &symbol) const; - + const std::map aux_rules; }; - + std::ostream& operator<<(std::ostream &stream, const PreparedGrammar &grammar); - + std::string compile(const Grammar &grammar, std::string name); } diff --git a/src/compiler/rules/blank.cpp b/src/compiler/rules/blank.cpp index 9a4a9162..e8a88dfa 100644 --- a/src/compiler/rules/blank.cpp +++ b/src/compiler/rules/blank.cpp @@ -4,23 +4,23 @@ namespace tree_sitter { namespace rules { Blank::Blank() {} - + bool Blank::operator==(const Rule &rule) const { return dynamic_cast(&rule) != nullptr; } - + size_t Blank::hash_code() const { return 0; } - + rule_ptr Blank::copy() const { return std::make_shared(); } - + std::string Blank::to_string() const { return "#"; } - + void Blank::accept(Visitor &visitor) const { visitor.visit(this); } diff --git a/src/compiler/rules/blank.h b/src/compiler/rules/blank.h index b32e2c4b..92c60a87 100644 --- a/src/compiler/rules/blank.h +++ b/src/compiler/rules/blank.h @@ -8,7 +8,7 @@ namespace tree_sitter { class Blank : public Rule { public: Blank(); - + bool operator==(const Rule& other) const; size_t hash_code() const; rule_ptr copy() const; diff --git a/src/compiler/rules/character_range.cpp b/src/compiler/rules/character_range.cpp index b654ad90..094b233a 100644 --- a/src/compiler/rules/character_range.cpp +++ b/src/compiler/rules/character_range.cpp @@ -3,24 +3,24 @@ namespace tree_sitter { using std::string; - + namespace rules { static const char MAX_CHAR = '\xff'; - + CharacterRange::CharacterRange(char value) : min(value), max(value) {} CharacterRange::CharacterRange(char min, char max) : min(min), max(max) {} - + bool CharacterRange::operator==(const CharacterRange &other) const { return min == other.min && max == other.max; } - + bool CharacterRange::operator<(const CharacterRange &other) const { if (min < other.min) return true; if (min > other.min) return false; if (max < other.max) return true; return false; } - + string escape_character(char input) { switch (input) { case '\0': @@ -31,7 +31,7 @@ namespace tree_sitter { return string() + input; } } - + string CharacterRange::to_string() const { if (min == 0 && max == MAX_CHAR) return ""; diff --git a/src/compiler/rules/character_set.cpp b/src/compiler/rules/character_set.cpp index 468349ee..5ebeaa70 100644 --- a/src/compiler/rules/character_set.cpp +++ b/src/compiler/rules/character_set.cpp @@ -14,24 +14,24 @@ namespace tree_sitter { int max_int(const CharacterRange &range) { return range.max == MAX_CHAR ? 255 : (int)range.max; } - + int min_int(const CharacterRange &range) { return (int)range.min; } - + CharacterSet::CharacterSet() : ranges({}) {} CharacterSet::CharacterSet(const set &ranges) : ranges(ranges) {} CharacterSet::CharacterSet(const initializer_list &ranges) : ranges(ranges) {} - + bool CharacterSet::operator==(const Rule &rule) const { const CharacterSet *other = dynamic_cast(&rule); return other && (ranges == other->ranges); } - + bool CharacterSet::operator<(const CharacterSet &other) const { return ranges < other.ranges; } - + size_t CharacterSet::hash_code() const { size_t result = std::hash()(ranges.size()); for (auto &range : ranges) { @@ -51,13 +51,13 @@ namespace tree_sitter { result += " " + range.to_string(); return result + " }>"; } - + CharacterSet CharacterSet::complement() const { CharacterSet result({ {0, MAX_CHAR} }); result.remove_set(*this); return result; } - + std::pair CharacterSet::most_compact_representation() const { auto first_range = *ranges.begin(); if (first_range.min == 0 && first_range.max > 0) { @@ -66,7 +66,7 @@ namespace tree_sitter { return { *this, true }; } } - + void add_range(CharacterSet *self, CharacterRange new_range) { set new_ranges; @@ -87,7 +87,7 @@ namespace tree_sitter { new_range.max = range.max; } } - + if (!is_adjacent) { new_ranges.insert(range); } @@ -95,7 +95,7 @@ namespace tree_sitter { new_ranges.insert(new_range); self->ranges = new_ranges; } - + CharacterSet remove_range(CharacterSet *self, CharacterRange new_range) { CharacterSet removed_set; set new_ranges; @@ -126,17 +126,17 @@ namespace tree_sitter { self->ranges = new_ranges; return removed_set; } - + bool CharacterSet::is_empty() const { return ranges.empty(); } - + void CharacterSet::add_set(const CharacterSet &other) { for (auto &other_range : other.ranges) { add_range(this, other_range); } } - + CharacterSet CharacterSet::remove_set(const CharacterSet &other) { CharacterSet result; for (auto &other_range : other.ranges) { @@ -145,12 +145,12 @@ namespace tree_sitter { } return result; } - + CharacterSet CharacterSet::intersect(const CharacterSet &set) const { CharacterSet copy = *this; return copy.remove_set(set); } - + void CharacterSet::accept(Visitor &visitor) const { visitor.visit(this); } diff --git a/src/compiler/rules/character_set.h b/src/compiler/rules/character_set.h index 975df757..83902616 100644 --- a/src/compiler/rules/character_set.h +++ b/src/compiler/rules/character_set.h @@ -13,21 +13,21 @@ namespace tree_sitter { CharacterSet(); CharacterSet(const std::set &ranges); CharacterSet(const std::initializer_list &ranges); - + bool operator==(const Rule& other) const; bool operator<(const CharacterSet &) const; size_t hash_code() const; rule_ptr copy() const; std::string to_string() const; void accept(Visitor &visitor) const; - + void add_set(const CharacterSet &other); CharacterSet remove_set(const CharacterSet &other); CharacterSet complement() const; CharacterSet intersect(const CharacterSet &) const; std::pair most_compact_representation() const; bool is_empty() const; - + std::set ranges; }; } diff --git a/src/compiler/rules/choice.cpp b/src/compiler/rules/choice.cpp index e7a372ff..64b67db4 100644 --- a/src/compiler/rules/choice.cpp +++ b/src/compiler/rules/choice.cpp @@ -5,17 +5,17 @@ namespace tree_sitter { using std::string; using std::make_shared; using std::vector; - + namespace rules { Choice::Choice(rule_ptr left, rule_ptr right) : left(left), right(right) {}; - + rule_ptr Choice::Build(const vector &rules) { rule_ptr result; for (auto rule : rules) result = result.get() ? make_shared(result, rule) : rule; return result; } - + bool Choice::operator==(const Rule &rule) const { const Choice *other = dynamic_cast(&rule); return other && (*other->left == *left) && (*other->right == *right); @@ -24,7 +24,7 @@ namespace tree_sitter { size_t Choice::hash_code() const { return left->hash_code() ^ right->hash_code(); } - + rule_ptr Choice::copy() const { return std::make_shared(*this); } @@ -32,7 +32,7 @@ namespace tree_sitter { string Choice::to_string() const { return string("#to_string() + " " + right->to_string() + ">"; } - + void Choice::accept(Visitor &visitor) const { visitor.visit(this); } diff --git a/src/compiler/rules/pattern.cpp b/src/compiler/rules/pattern.cpp index 06d07e9d..d4e440c7 100644 --- a/src/compiler/rules/pattern.cpp +++ b/src/compiler/rules/pattern.cpp @@ -12,14 +12,14 @@ namespace tree_sitter { using std::hash; using std::make_shared; using std::set; - + class PatternParser { public: PatternParser(const string &input) : input(input), length(input.length()), position(0) {} - + rule_ptr rule() { auto result = term(); while (has_more_input() && peek() == '|') { @@ -28,7 +28,7 @@ namespace tree_sitter { } return result; } - + private: rule_ptr term() { rule_ptr result = factor(); @@ -36,7 +36,7 @@ namespace tree_sitter { result = Seq::Build({ result, factor() }); return result; } - + rule_ptr factor() { rule_ptr result = atom(); if (has_more_input() && (peek() == '+')) { @@ -45,7 +45,7 @@ namespace tree_sitter { } return result; } - + rule_ptr atom() { rule_ptr result; switch (peek()) { @@ -77,7 +77,7 @@ namespace tree_sitter { } return result; } - + CharacterSet char_set() { bool is_affirmative = true; if (peek() == '^') { @@ -89,7 +89,7 @@ namespace tree_sitter { result.add_set(single_char()); return is_affirmative ? result : result.complement(); } - + CharacterSet single_char() { CharacterSet value; switch (peek()) { @@ -112,7 +112,7 @@ namespace tree_sitter { } return value; } - + CharacterSet escaped_char(char value) { switch (value) { case '\\': @@ -128,52 +128,52 @@ namespace tree_sitter { return CharacterSet(); } } - + void next() { position++; } - + char peek() { return input[position]; } - + bool has_more_input() { return position < length; } - + bool has_error() { return error != ""; } - + string error; const string input; const size_t length; size_t position; }; - + Pattern::Pattern(const string &string) : value(string) {}; - + bool Pattern::operator==(tree_sitter::rules::Rule const &other) const { auto pattern = dynamic_cast(&other); return pattern && (pattern->value == value); } - + size_t Pattern::hash_code() const { return hash()(value); } - + rule_ptr Pattern::copy() const { return std::make_shared(*this); } - + string Pattern::to_string() const { return string("#"; } - + void Pattern::accept(Visitor &visitor) const { visitor.visit(this); } - + rule_ptr Pattern::to_rule_tree() const { return PatternParser(value).rule(); } diff --git a/src/compiler/rules/pattern.h b/src/compiler/rules/pattern.h index a19ec576..7d8fa66b 100644 --- a/src/compiler/rules/pattern.h +++ b/src/compiler/rules/pattern.h @@ -15,7 +15,7 @@ namespace tree_sitter { rule_ptr copy() const; std::string to_string() const; void accept(Visitor &visitor) const; - + rule_ptr to_rule_tree() const; }; } diff --git a/src/compiler/rules/repeat.cpp b/src/compiler/rules/repeat.cpp index 76c1c3cc..36ac5663 100644 --- a/src/compiler/rules/repeat.cpp +++ b/src/compiler/rules/repeat.cpp @@ -6,24 +6,24 @@ using std::string; namespace tree_sitter { namespace rules { Repeat::Repeat(const rule_ptr content) : content(content) {} - + bool Repeat::operator==(const Rule &rule) const { const Repeat *other = dynamic_cast(&rule); return other && (*other->content == *content); } - + size_t Repeat::hash_code() const { return content->hash_code(); } - + rule_ptr Repeat::copy() const { return std::make_shared(*this); } - + string Repeat::to_string() const { return string("#to_string() + ">"; } - + void Repeat::accept(Visitor &visitor) const { visitor.visit(this); } diff --git a/src/compiler/rules/rule.cpp b/src/compiler/rules/rule.cpp index d7b4b875..43bc1a93 100644 --- a/src/compiler/rules/rule.cpp +++ b/src/compiler/rules/rule.cpp @@ -9,7 +9,7 @@ namespace tree_sitter { bool Rule::operator!=(const Rule &other) const { return !this->operator==(other); } - + ostream& operator<<(ostream& stream, const Rule &rule) { return stream << rule.to_string(); } diff --git a/src/compiler/rules/rule.h b/src/compiler/rules/rule.h index c10901f2..f21d8f45 100644 --- a/src/compiler/rules/rule.h +++ b/src/compiler/rules/rule.h @@ -20,7 +20,7 @@ namespace tree_sitter { virtual std::string to_string() const = 0; virtual void accept(Visitor &visitor) const = 0; }; - + std::ostream& operator<<(std::ostream& stream, const Rule &rule); std::ostream& operator<<(std::ostream& stream, const rule_ptr &rule); } diff --git a/src/compiler/rules/rules.cpp b/src/compiler/rules/rules.cpp index f1a363d9..328144c7 100644 --- a/src/compiler/rules/rules.cpp +++ b/src/compiler/rules/rules.cpp @@ -15,20 +15,20 @@ namespace tree_sitter { using std::string; using std::set; using std::vector; - + namespace rules { rule_ptr blank() { return make_shared(); } - + rule_ptr choice(const vector &rules) { return Choice::Build(rules); } - + rule_ptr repeat(const rule_ptr &content) { return std::make_shared(content); } - + rule_ptr seq(const vector &rules) { return Seq::Build(rules); } @@ -44,11 +44,11 @@ namespace tree_sitter { rule_ptr pattern(const string &value) { return make_shared(value); } - + rule_ptr str(const string &value) { return make_shared(value); } - + rule_ptr err(const rule_ptr &rule) { return choice({ rule, ERROR.copy() }); } diff --git a/src/compiler/rules/seq.cpp b/src/compiler/rules/seq.cpp index 29e2878c..70cddcdb 100644 --- a/src/compiler/rules/seq.cpp +++ b/src/compiler/rules/seq.cpp @@ -21,19 +21,19 @@ namespace tree_sitter { const Seq *other = dynamic_cast(&rule); return other && (*other->left == *left) && (*other->right == *right); } - + size_t Seq::hash_code() const { return left->hash_code() ^ right->hash_code(); } - + rule_ptr Seq::copy() const { return std::make_shared(*this); } - + string Seq::to_string() const { return string("#to_string() + " " + right->to_string() + ">"; } - + void Seq::accept(Visitor &visitor) const { visitor.visit(this); } diff --git a/src/compiler/rules/string.cpp b/src/compiler/rules/string.cpp index c4e5ead5..161c538c 100644 --- a/src/compiler/rules/string.cpp +++ b/src/compiler/rules/string.cpp @@ -7,24 +7,24 @@ using std::hash; namespace tree_sitter { namespace rules { String::String(string value) : value(value) {}; - + bool String::operator==(const Rule &rule) const { const String *other = dynamic_cast(&rule); return other && (other->value == value); } - + size_t String::hash_code() const { return hash()(value); } - + rule_ptr String::copy() const { return std::make_shared(*this); } - + string String::to_string() const { return string("#"; - } - + } + void String::accept(Visitor &visitor) const { visitor.visit(this); } diff --git a/src/compiler/rules/symbol.cpp b/src/compiler/rules/symbol.cpp index 310945be..941580b0 100644 --- a/src/compiler/rules/symbol.cpp +++ b/src/compiler/rules/symbol.cpp @@ -22,11 +22,11 @@ namespace tree_sitter { size_t Symbol::hash_code() const { return hash()(name) ^ hash()(type); } - + rule_ptr Symbol::copy() const { return std::make_shared(*this); } - + string Symbol::to_string() const { switch (type) { case SymbolTypeNormal: @@ -39,17 +39,17 @@ namespace tree_sitter { return string("#"; } } - + bool Symbol::operator<(const Symbol &other) const { if (type < other.type) return true; if (type > other.type) return false; return (name < other.name); } - + bool Symbol::is_built_in() const { return type == SymbolTypeBuiltIn; } - + bool Symbol::is_auxiliary() const { return type == SymbolTypeAuxiliary; } @@ -57,7 +57,7 @@ namespace tree_sitter { bool Symbol::is_hidden() const { return (type == SymbolTypeHidden || type == SymbolTypeAuxiliary); } - + void Symbol::accept(Visitor &visitor) const { visitor.visit(this); } diff --git a/src/compiler/rules/symbol.h b/src/compiler/rules/symbol.h index cd0c9b72..28c53dd1 100644 --- a/src/compiler/rules/symbol.h +++ b/src/compiler/rules/symbol.h @@ -11,8 +11,8 @@ namespace tree_sitter { SymbolTypeHidden, SymbolTypeAuxiliary, SymbolTypeBuiltIn - } SymbolType; - + } SymbolType; + class Symbol : public Rule { public: Symbol(const std::string &name); @@ -20,7 +20,7 @@ namespace tree_sitter { bool operator==(const Rule& other) const; bool operator==(const Symbol &other) const; - + size_t hash_code() const; rule_ptr copy() const; std::string to_string() const; diff --git a/src/compiler/rules/visitor.h b/src/compiler/rules/visitor.h index e48a207c..29520e08 100644 --- a/src/compiler/rules/visitor.h +++ b/src/compiler/rules/visitor.h @@ -13,7 +13,7 @@ namespace tree_sitter { class Seq; class String; class Pattern; - + class Visitor { public: virtual void default_visit(const Rule *rule); @@ -26,7 +26,7 @@ namespace tree_sitter { virtual void visit(const String *rule); virtual void visit(const Pattern *rule); }; - + template class RuleFn : public Visitor { protected: diff --git a/src/runtime/tree.c b/src/runtime/tree.c index 046db2fb..c73f6144 100644 --- a/src/runtime/tree.c +++ b/src/runtime/tree.c @@ -87,7 +87,7 @@ static size_t tree_write_to_string(const ts_tree *tree, const char **symbol_name return snprintf(string, limit, "%s", NULL_TREE_STRING); if (tree->symbol == ts_builtin_sym_error) return snprintf(string, limit, "%s", ERROR_TREE_STRING); - + size_t result = snprintf(string, limit, "(%s", symbol_names[tree->symbol]); char *cursor = string + result; for (size_t i = 0; i < tree->data.children.count; i++) { @@ -96,7 +96,7 @@ static size_t tree_write_to_string(const ts_tree *tree, const char **symbol_name result += tree_write_to_string(child, symbol_names, cursor + 1, limit); cursor = (limit > 0) ? string + result : string; } - + return result + snprintf(cursor, limit, ")"); }