traxys/tree-sitter
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Add script to trim whitespace

Browse source
This commit is contained in:
Max Brunsfeld 2014-03-09 19:49:35 -07:00
parent e681a63552
commit 39aa0ccc91
66 changed files with 350 additions and 347 deletions
Download patch file Download diff file Expand all files Collapse all files

14
spec/compiler/build_tables/build_tables_spec.cpp
View file

@ -36,7 +36,7 @@ describe("building parse and lex tables", []() {
sym("right-paren")
}) }) }
}, {});
PreparedGrammar lex_grammar("", {
{ "plus", str("+") },
{ "variable", pattern("\\w+") },
@ -44,25 +44,25 @@ describe("building parse and lex tables", []() {
{ "left-paren", str("(") },
{ "right-paren", str(")") }
}, {});
ParseTable table;
LexTable lex_table;
before_each([&]() {
pair<ParseTable, LexTable> tables = build_tables::build_tables(grammar, lex_grammar);
table = tables.first;
lex_table = tables.second;
});
function<ParseState(size_t)> parse_state = [&](size_t index) {
return table.states[index];
};
function<LexState(size_t)> lex_state = [&](size_t parse_state_index) {
long index = table.states[parse_state_index].lex_state_id;
return lex_table.states[index];
};
it("has the right starting state", [&]() {
AssertThat(keys(parse_state(0).actions), Equals(set<Symbol>({
Symbol("expression"),
@ -71,7 +71,7 @@ describe("building parse and lex tables", []() {
Symbol("variable"),
Symbol("left-paren"),
})));
AssertThat(lex_state(0).expected_inputs(), Equals(set<CharacterSet>({
CharacterSet({ '(' }),
CharacterSet({ CharacterRange('0', '9') }),

16
spec/compiler/build_tables/first_set_spec.cpp
View file

@ -14,7 +14,7 @@ describe("computing FIRST sets", []() {
describe("for a sequence AB", [&]() {
it("ignores B when A cannot be blank", [&]() {
auto rule = seq({ sym("x"), sym("y") });
AssertThat(first_set(rule, null_grammar), Equals(set<Symbol>({
Symbol("x"),
})));
@ -26,13 +26,13 @@ describe("computing FIRST sets", []() {
sym("x"),
blank() }),
sym("y") });
AssertThat(first_set(rule, null_grammar), Equals(set<Symbol>({
Symbol("x"),
Symbol("y")
})));
});
it("includes FIRST(A's right hand side) when A is a non-terminal", [&]() {
auto rule = choice({
seq({
@ -40,7 +40,7 @@ describe("computing FIRST sets", []() {
sym("x"),
sym("A") }),
sym("A") });
Grammar grammar("A", {
{ "A", choice({
seq({
@ -49,19 +49,19 @@ describe("computing FIRST sets", []() {
sym("y") }),
sym("y") }) }
});
AssertThat(first_set(rule, grammar), Equals(set<Symbol>({
Symbol("y")
})));
});
it("includes FIRST(B) when A is a non-terminal and its expansion can be blank", [&]() {
Grammar grammar("A", {{ "A", choice({ sym("x"), blank() }) }});
auto rule = seq({
sym("A"),
sym("y") });
AssertThat(first_set(rule, grammar), Equals(set<Symbol>({
Symbol("x"),
Symbol("y")

12
spec/compiler/build_tables/follow_sets_spec.cpp
View file

@ -13,35 +13,35 @@ describe("computing FOLLOW sets", []() {
{ "A", sym("a") },
{ "B", sym("b") },
}, {});
it("all of the starting non-terminals for the item, and their following terminals", [&]() {
ParseItem item(Symbol("C"), choice({
seq({ sym("A"), choice({ sym("x"), sym("y") }) }),
seq({ sym("B"), sym("z") }),
}), {}, Symbol("w"));
AssertThat(follow_sets(item, grammar), Equals(map<Symbol, set<Symbol>>({
{ Symbol("A"), set<Symbol>({ Symbol("x"), Symbol("y") }) },
{ Symbol("B"), set<Symbol>({ Symbol("z") }) },
})));
});
it("does not include terminals at the beginning of the item", [&]() {
ParseItem item(Symbol("C"), choice({
seq({ sym("A"), choice({ sym("x"), sym("y") }) }),
seq({ sym("x"), sym("y") }),
}), {}, Symbol("w"));
AssertThat(follow_sets(item, grammar), Equals(map<Symbol, set<Symbol>>({
{ Symbol("A"), set<Symbol>({ Symbol("x"), Symbol("y") }) },
})));
});
it("includes the item's lookahead terminal if the rule after the non-terminal might be blank", [&]() {
ParseItem item(Symbol("C"), choice({
seq({ sym("A"), choice({ sym("x"), blank() }) }),
}), {}, Symbol("w"));
AssertThat(follow_sets(item, grammar), Equals(map<Symbol, set<Symbol>>({
{ Symbol("A"), set<Symbol>({ Symbol("x"), Symbol("w") }) },
})));

2
spec/compiler/build_tables/item_set_closure_spec.cpp
View file

@ -26,7 +26,7 @@ describe("computing closures of item sets", []() {
sym("v"),
sym("n") }) }
}, {});
it("computes the item set closure", [&]() {
ParseItemSet item_set = item_set_closure(ParseItemSet({
ParseItem(Symbol("E"), grammar.rule(Symbol("E")), {}, Symbol("__END__"))

2
spec/compiler/build_tables/rule_can_be_blank_spec.cpp
View file

@ -15,7 +15,7 @@ describe("checking if rules can be blank", [&]() {
}),
str("y"),
});
AssertThat(rule_can_be_blank(rule), Equals(false));
});
});

26
spec/compiler/build_tables/rule_transitions_spec.cpp
View file

@ -16,7 +16,7 @@ public:
}
return true;
}
rule_map(const initializer_list<pair<const K, rule_ptr>> &list) : map<K, rule_ptr>(list) {}
};
@ -30,7 +30,7 @@ describe("rule transitions", []() {
{ Symbol("1"), blank() }
})));
});
it("handles choices", [&]() {
AssertThat(
sym_transitions(choice({ sym("1"), sym("2") })),
@ -39,7 +39,7 @@ describe("rule transitions", []() {
{ Symbol("2"), blank() }
})));
});
it("handles sequences", [&]() {
AssertThat(
sym_transitions(seq({ sym("1"), sym("2") })),
@ -47,7 +47,7 @@ describe("rule transitions", []() {
{ Symbol("1"), sym("2") }
})));
});
it("handles long sequences", [&]() {
AssertThat(
sym_transitions(seq({
@ -60,7 +60,7 @@ describe("rule transitions", []() {
{ Symbol("1"), seq({ sym("2"), sym("3"), sym("4") }) }
})));
});
it("handles sequences whose left sides can be blank", [&]() {
AssertThat(
sym_transitions(seq({
@ -76,7 +76,7 @@ describe("rule transitions", []() {
{ Symbol("1"), choice({ seq({ sym("1"), sym("2") }), sym("2"), }) }
})));
});
it("handles choices with common starting symbols", [&]() {
AssertThat(
sym_transitions(
@ -87,7 +87,7 @@ describe("rule transitions", []() {
{ Symbol("1"), choice({ sym("2"), sym("3") }) }
})));
});
it("handles characters", [&]() {
AssertThat(
char_transitions(character({ '1' })),
@ -95,7 +95,7 @@ describe("rule transitions", []() {
{ CharacterSet({ '1' }), blank() }
})));
});
it("handles strings", [&]() {
AssertThat(
char_transitions(str("bad")),
@ -103,7 +103,7 @@ describe("rule transitions", []() {
{ CharacterSet({ 'b' }), seq({ character({ 'a' }), character({ 'd' }) }) }
})));
});
it("handles patterns", [&]() {
AssertThat(
char_transitions(pattern("a|b")),
@ -112,8 +112,8 @@ describe("rule transitions", []() {
{ CharacterSet({ 'b' }), blank() }
})));
});
it("handles choices between overlapping character sets", [&]() {
AssertThat(
char_transitions(choice({
@ -145,7 +145,7 @@ describe("rule transitions", []() {
})
})
}})));
rule = repeat(str("a"));
AssertThat(
char_transitions(rule),
@ -168,7 +168,7 @@ describe("rule transitions", []() {
}),
character({ '"' }),
});
AssertThat(char_transitions(rule), Equals(rule_map<CharacterSet>({
{ CharacterSet({ '"' }).complement(), seq({
choice({

2
spec/compiler/helpers/character_set_helpers.cpp
View file

@ -4,7 +4,7 @@
namespace tree_sitter {
using std::make_shared;
using std::set;
namespace rules {
rule_ptr character(const set<CharacterRange> &ranges) {
return make_shared<CharacterSet>(ranges);

10
spec/compiler/helpers/equals_pointer.h
View file

@ -5,20 +5,20 @@
namespace snowhouse {
using namespace std;
template<typename ExpectedType>
struct EqualsPointerConstraint : Expression<EqualsPointerConstraint<ExpectedType>>
{
EqualsPointerConstraint(const ExpectedType& expected) : expected(expected) {}
template<typename ActualType>
bool operator()(const ActualType& actual) const {
return *expected == *actual;
}
ExpectedType expected;
};
template<typename ExpectedType>
struct Stringizer<EqualsPointerConstraint<ExpectedType>>
{
@ -28,7 +28,7 @@ namespace snowhouse {
return builder.str();
}
};
template<typename ExpectedType>
inline EqualsPointerConstraint<ExpectedType> EqualsPointer(const ExpectedType& expected) {
return EqualsPointerConstraint<ExpectedType>(expected);

6
spec/compiler/helpers/stream_methods.h
View file

@ -8,7 +8,7 @@
using std::cout;
namespace std {
template<typename T>
inline std::ostream& operator<<(std::ostream &stream, const std::vector<T> &vector) {
stream << std::string("#<vector: ");
@ -20,7 +20,7 @@ namespace std {
}
return stream << ">";
}
template<typename T>
inline std::ostream& operator<<(std::ostream &stream, const std::set<T> &set) {
stream << std::string("#<set: ");
@ -32,7 +32,7 @@ namespace std {
}
return stream << ">";
}
template<typename TKey, typename TValue>
inline std::ostream& operator<<(std::ostream &stream, const std::map<TKey, TValue> &map) {
stream << std::string("#<map: ");

18
spec/compiler/prepare_grammar_spec.cpp
View file

@ -19,7 +19,7 @@ describe("preparing a grammar", []() {
sym("rule3") }),
str("ab") }) }
}));
AssertThat(result.first, Equals(PreparedGrammar("rule1", {
{ "rule1", seq({
make_shared<Symbol>("token1", SymbolTypeAuxiliary),
@ -28,36 +28,36 @@ describe("preparing a grammar", []() {
sym("rule3") }),
make_shared<Symbol>("token1", SymbolTypeAuxiliary) }) }
}, {})));
AssertThat(result.second, Equals(PreparedGrammar("", {}, {
{ "token1", str("ab") },
})));
});
it("moves entire rules into the lexical grammar when possible, preserving their names", [&]() {
auto result = prepare_grammar(Grammar("rule1", {
{ "rule1", sym("rule2") },
{ "rule2", pattern("a|b") }
}));
AssertThat(result.first, Equals(PreparedGrammar("rule1", {
{ "rule1", sym("rule2") }
}, {})));
AssertThat(result.second, Equals(PreparedGrammar("", {
{ "rule2", pattern("a|b") },
}, {})));
});
it("does not extract blanks into tokens", [&]() {
pair<PreparedGrammar, PreparedGrammar> result = prepare_grammar(Grammar("rule1", {
{ "rule1", choice({ sym("rule2"), blank() }) },
}));
AssertThat(result.first, Equals(PreparedGrammar("rule1", {
{ "rule1", choice({ sym("rule2"), blank() }) },
}, {})));
AssertThat(result.second, Equals(PreparedGrammar("", {}, {})));
});
});
@ -71,7 +71,7 @@ describe("preparing a grammar", []() {
sym("y")
}) },
})).first;
AssertThat(result, Equals(PreparedGrammar("rule1", {
{ "rule1", seq({
sym("x"),

22
spec/compiler/rules/character_set_spec.cpp
View file

@ -7,7 +7,7 @@ START_TEST
describe("character sets", []() {
char max_char = 255;
describe("computing the complement", [&]() {
it("works for the set containing only the null character", [&]() {
CharacterSet set1({ '\0' });
@ -28,14 +28,14 @@ describe("character sets", []() {
AssertThat(set2.complement(), Equals(set1));
});
});
describe("computing unions", [&]() {
it("works for disjoint sets", [&]() {
CharacterSet set({ {'a', 'z'} });
set.add_set(CharacterSet({ {'A', 'Z'} }));
AssertThat(set, Equals(CharacterSet({ {'a', 'z'}, {'A', 'Z'} })));
});
it("works for sets with adjacent ranges", [&]() {
CharacterSet set({ CharacterRange('a', 'r') });
set.add_set(CharacterSet({ CharacterRange('s', 'z') }));
@ -46,33 +46,33 @@ describe("character sets", []() {
set.add_set(c);
AssertThat(set, Equals(CharacterSet({ {0, max_char} })));
});
it("works when the result becomes a continuous range", []() {
CharacterSet set({ {'a', 'd'}, {'f', 'z'} });
set.add_set(CharacterSet({ {'c', 'g'} }));
AssertThat(set, Equals(CharacterSet({ {'a', 'z'} })));
});
it("does nothing for the set of all characters", [&]() {
CharacterSet set({ 'a' });
set.add_set(set.complement());
AssertThat(set, Equals(CharacterSet({ {'\0', max_char} })));
});
});
describe("computing differences", []() {
it("works for disjoint sets", []() {
CharacterSet set1({ {'a','z'} });
set1.remove_set(CharacterSet({ {'A','Z'} }));
AssertThat(set1, Equals(CharacterSet({ {'a', 'z'} })));
});
it("works when one set spans the other", []() {
CharacterSet set1({ {'a','z'} });
set1.remove_set(CharacterSet({ {'d','s'} }));
AssertThat(set1, Equals(CharacterSet({ {'a', 'c'}, {'t', 'z'} })));
});
it("works for sets that overlap", []() {
CharacterSet set1({ {'a','s'} });
set1.remove_set(CharacterSet({ {'m','z'} }));
@ -82,21 +82,21 @@ describe("character sets", []() {
set2.remove_set(CharacterSet({ {'a','s'} }));
AssertThat(set2, Equals(CharacterSet({ {'t', 'z'} })));
});
it("works for sets with multiple ranges", []() {
CharacterSet set1({ {'a','d'}, {'m', 'z'} });
set1.remove_set(CharacterSet({ {'c','o'}, {'s','x'} }));
AssertThat(set1, Equals(CharacterSet({ {'a', 'b'}, {'p','r'}, {'y','z'} })));
});
});
describe("computing intersections", []() {
it("returns an empty set for disjoint sets", []() {
CharacterSet set1({ {'a','d'} });
CharacterSet set2({ {'e','x'} });
AssertThat(set1.intersect(set2), Equals(CharacterSet()));
});
it("works for sets with a single overlapping range", []() {
CharacterSet set1({ {'a','e'} });
CharacterSet set2({ {'c','x'} });

16
spec/compiler/rules/pattern_spec.cpp
View file

@ -48,35 +48,35 @@ describe("parsing pattern rules", []() {
})
})));
});
it("parses character sets", []() {
Pattern rule("[aAeE]");
AssertThat(
rule.to_rule_tree(),
EqualsPointer(character({ 'a', 'A', 'e', 'E' })));
});
it("parses character ranges", []() {
Pattern rule("[12a-dA-D3]");
AssertThat(
rule.to_rule_tree(),
EqualsPointer(character({ {'1', '3'}, {'a', 'd'}, { 'A', 'D' }, })));
});
it("parses negated characters", []() {
Pattern rule("[^a\\d]");
AssertThat(
rule.to_rule_tree(),
EqualsPointer(character({ {'a'}, {'0', '9'} }, false)));
});
it("parses backslashes", []() {
Pattern rule("\\\\");
AssertThat(
rule.to_rule_tree(),
EqualsPointer(character({ '\\' })));
});
it("parses character groups in sequences", []() {
Pattern rule("\"([^\"]|\\\\\")+\"");
AssertThat(
@ -90,7 +90,7 @@ describe("parsing pattern rules", []() {
character({ '"' })
})));
});
it("parses choices in sequences", []() {
Pattern rule("(a|b)cd");
AssertThat(
@ -104,7 +104,7 @@ describe("parsing pattern rules", []() {
character({ 'd' })
})));
});
it("parses special characters when they are escaped", []() {
Pattern rule("a\\(b");
AssertThat(
@ -115,7 +115,7 @@ describe("parsing pattern rules", []() {
character({ 'b' })
})));
});
it("parses repeating rules", []() {
Pattern rule("(ab)+(cd)+");
AssertThat(

4
spec/compiler/rules/rules_spec.cpp
View file

@ -9,12 +9,12 @@ describe("constructing rules", []() {
rule_ptr symbol1 = sym("1");
rule_ptr symbol2 = sym("2");
rule_ptr symbol3 = sym("3");
it("constructs binary trees", [&]() {
AssertThat(
seq({ symbol1, symbol2, symbol3 }),
EqualsPointer(seq({ seq({ symbol1, symbol2 }), symbol3 })));
AssertThat(
choice({ symbol1, symbol2, symbol3 }),
EqualsPointer(choice({ choice({ symbol1, symbol2 }), symbol3 })));

16
spec/runtime/arithmetic_spec.cpp
View file

@ -6,22 +6,22 @@ START_TEST
describe("arithmetic", []() {
ts_document *doc;
before_each([&]() {
doc = ts_document_make();
ts_document_set_parser(doc, ts_parse_config_arithmetic);
});
after_each([&]() {
ts_document_free(doc);
});
it("parses variables", [&]() {
ts_document_set_input_string(doc, "x");
AssertThat(string(ts_document_string(doc)), Equals(
"(expression (term (factor (variable))))"));
});
it("parses numbers", [&]() {
ts_document_set_input_string(doc, "5");
AssertThat(string(ts_document_string(doc)), Equals(
@ -32,12 +32,12 @@ describe("arithmetic", []() {
ts_document_set_input_string(doc, "x + y");
AssertThat(string(ts_document_string(doc)), Equals(
"(expression (term (factor (variable))) (plus) (term (factor (variable))))"));
ts_document_set_input_string(doc, "x * y");
AssertThat(string(ts_document_string(doc)), Equals(
"(expression (term (factor (variable)) (times) (factor (variable))))"));
});
it("parses complex trees", [&]() {
ts_document_set_input_string(doc, "x * y + z * a");
AssertThat(string(ts_document_string(doc)), Equals(
@ -47,13 +47,13 @@ describe("arithmetic", []() {
AssertThat(string(ts_document_string(doc)), Equals(
"(expression (term (factor (variable)) (times) (factor (expression (term (factor (variable))) (plus) (term (factor (variable)))))))"));
});
describe("error recovery", [&]() {
it("recovers from errors at the top level", [&]() {
ts_document_set_input_string(doc, "x * * y");
AssertThat(string(ts_document_string(doc)), Equals("(ERROR)"));
});
it("recovers from errors in parenthesized expressions", [&]() {
ts_document_set_input_string(doc, "x + (y * + z) * 5");
AssertThat(string(ts_document_string(doc)), Equals(

22
spec/runtime/json_spec.cpp
View file

@ -6,16 +6,16 @@ START_TEST
describe("json", []() {
ts_document *doc;
before_each([&]() {
doc = ts_document_make();
ts_document_set_parser(doc, ts_parse_config_json);
});
after_each([&]() {
ts_document_free(doc);
});
it("parses strings", [&]() {
ts_document_set_input_string(doc, "\"\"");
AssertThat(string(ts_document_string(doc)), Equals("(value (string))"));
@ -26,7 +26,7 @@ describe("json", []() {
ts_document_set_input_string(doc, "\"this is a \\\"string\\\" within a string\"");
AssertThat(string(ts_document_string(doc)), Equals("(value (string))"));
});
it("parses objects", [&]() {
ts_document_set_input_string(doc, "{}");
AssertThat(string(ts_document_string(doc)), Equals("(value (object))"));
@ -37,7 +37,7 @@ describe("json", []() {
ts_document_set_input_string(doc, "{\"key1\": 1, \"key2\": 2 }");
AssertThat(string(ts_document_string(doc)), Equals("(value (object (string) (value (number)) (string) (value (number))))"));
});
it("parses arrays", [&]() {
ts_document_set_input_string(doc, "[]");
AssertThat(string(ts_document_string(doc)), Equals("(value (array))"));
@ -48,11 +48,11 @@ describe("json", []() {
ts_document_set_input_string(doc, "[1, 2, 3]");
AssertThat(string(ts_document_string(doc)), Equals("(value (array (value (number)) (value (number)) (value (number))))"));
});
describe("tracking the positions of AST nodes", [&]() {
it("records the widths and offsets of nodes", [&]() {
ts_document_set_input_string(doc, " [12, 5]");
const ts_tree *tree = ts_document_tree(doc);
const ts_tree *array = ts_tree_children(tree)[0];
const ts_tree *number1 = ts_tree_children(array)[0];
@ -66,18 +66,18 @@ describe("json", []() {
AssertThat(array->offset, Equals(2));
AssertThat(array->size, Equals(7));
AssertThat(tree->offset, Equals(2));
AssertThat(tree->size, Equals(7));
});
});
describe("errors", [&]() {
it("reports errors in the top-level node", [&]() {
ts_document_set_input_string(doc, "[");
AssertThat(string(ts_document_string(doc)), Equals("(ERROR)"));
});
it("reports errors inside of arrays and objects", [&]() {
ts_document_set_input_string(doc, "{ \"key1\": 1, 5 }");
AssertThat(string(ts_document_string(doc)), Equals("(value (object (string) (value (number)) (ERROR)))"));
@ -85,7 +85,7 @@ describe("json", []() {
ts_document_set_input_string(doc, "[1,,2]");
AssertThat(string(ts_document_string(doc)), Equals("(value (array (value (number)) (ERROR) (value (number))))"));
});
it("reports errors in nested objects", [&]() {
ts_document_set_input_string(doc, "{ \"key1\": { \"key2\": 1, 2 }, [, \"key3\": 3 }");
AssertThat(string(ts_document_string(doc)), Equals("(value (object (string) (value (object (string) (value (number)) (ERROR))) (ERROR) (string) (value (number))))"));