diff --git a/include/tree_sitter/runtime.h b/include/tree_sitter/runtime.h index d197c118..93ea8351 100644 --- a/include/tree_sitter/runtime.h +++ b/include/tree_sitter/runtime.h @@ -72,8 +72,7 @@ TSNode ts_document_root_node(const TSDocument *); #define ts_builtin_sym_error 0 #define ts_builtin_sym_end 1 -#define ts_builtin_sym_document 2 -#define ts_builtin_sym_start 3 +#define ts_builtin_sym_start 2 #ifdef __cplusplus } diff --git a/spec/fixtures/parsers/arithmetic.c b/spec/fixtures/parsers/arithmetic.c index 3cc03384..2a48fd4b 100644 --- a/spec/fixtures/parsers/arithmetic.c +++ b/spec/fixtures/parsers/arithmetic.c @@ -1,7 +1,7 @@ #include "tree_sitter/parser.h" #define STATE_COUNT 32 -#define SYMBOL_COUNT 20 +#define SYMBOL_COUNT 19 enum { sym_expression = ts_builtin_sym_start, @@ -24,7 +24,6 @@ enum { }; static const char *ts_symbol_names[] = { - [ts_builtin_sym_document] = "DOCUMENT", [sym_expression] = "expression", [sym_sum] = "sum", [sym_difference] = "difference", diff --git a/spec/fixtures/parsers/c.c b/spec/fixtures/parsers/c.c index 02b39a1c..7ff5205f 100644 --- a/spec/fixtures/parsers/c.c +++ b/spec/fixtures/parsers/c.c @@ -1,7 +1,7 @@ #include "tree_sitter/parser.h" #define STATE_COUNT 268 -#define SYMBOL_COUNT 65 +#define SYMBOL_COUNT 64 enum { sym_program = ts_builtin_sym_start, @@ -69,7 +69,6 @@ enum { }; static const char *ts_symbol_names[] = { - [ts_builtin_sym_document] = "DOCUMENT", [sym_program] = "program", [sym_function_definition] = "function_definition", [sym_declaration_specifiers] = "declaration_specifiers", diff --git a/spec/fixtures/parsers/golang.c b/spec/fixtures/parsers/golang.c index cd95b73e..fedcf49c 100644 --- a/spec/fixtures/parsers/golang.c +++ b/spec/fixtures/parsers/golang.c @@ -1,7 +1,7 @@ #include "tree_sitter/parser.h" #define STATE_COUNT 431 -#define SYMBOL_COUNT 85 +#define SYMBOL_COUNT 84 enum { sym_program = ts_builtin_sym_start, @@ -89,7 +89,6 @@ enum { }; static const char *ts_symbol_names[] = { - [ts_builtin_sym_document] = "DOCUMENT", [sym_program] = "program", [sym_package_directive] = "package_directive", [sym_imports_block] = "imports_block", diff --git a/spec/fixtures/parsers/javascript.c b/spec/fixtures/parsers/javascript.c index 3e9bd2cf..8468a6ca 100644 --- a/spec/fixtures/parsers/javascript.c +++ b/spec/fixtures/parsers/javascript.c @@ -1,7 +1,7 @@ #include "tree_sitter/parser.h" #define STATE_COUNT 1564 -#define SYMBOL_COUNT 107 +#define SYMBOL_COUNT 106 enum { sym_program = ts_builtin_sym_start, @@ -111,7 +111,6 @@ enum { }; static const char *ts_symbol_names[] = { - [ts_builtin_sym_document] = "DOCUMENT", [sym_program] = "program", [sym_statement] = "statement", [sym_expression_statement] = "expression_statement", diff --git a/spec/fixtures/parsers/json.c b/spec/fixtures/parsers/json.c index a4431586..d1beaa1d 100644 --- a/spec/fixtures/parsers/json.c +++ b/spec/fixtures/parsers/json.c @@ -1,7 +1,7 @@ #include "tree_sitter/parser.h" #define STATE_COUNT 69 -#define SYMBOL_COUNT 19 +#define SYMBOL_COUNT 18 enum { sym_value = ts_builtin_sym_start, @@ -23,7 +23,6 @@ enum { }; static const char *ts_symbol_names[] = { - [ts_builtin_sym_document] = "DOCUMENT", [sym_value] = "value", [sym_object] = "object", [sym_array] = "array", diff --git a/spec/runtime/document_spec.cc b/spec/runtime/document_spec.cc index 6a13d00c..7cca92d7 100644 --- a/spec/runtime/document_spec.cc +++ b/spec/runtime/document_spec.cc @@ -27,7 +27,7 @@ describe("Document", [&]() { ts_document_set_input_string(doc, "{ \"key\": [1, 2] }"); AssertThat(ts_node_string(ts_document_root_node(doc), doc), Equals( - "(DOCUMENT (object (string) (array (number) (number))))")); + "(object (string) (array (number) (number)))")); }); }); @@ -58,7 +58,7 @@ describe("Document", [&]() { ts_document_set_language(doc, ts_language_json()); AssertThat(ts_node_string(ts_document_root_node(doc), doc), Equals( - "(DOCUMENT (object (string) (array (number) (number))))")); + "(object (string) (array (number) (number)))")); }); }); }); diff --git a/spec/runtime/helpers/tree_helpers.cc b/spec/runtime/helpers/tree_helpers.cc index 767d36a3..92705179 100644 --- a/spec/runtime/helpers/tree_helpers.cc +++ b/spec/runtime/helpers/tree_helpers.cc @@ -1,8 +1,8 @@ #include "runtime/helpers/tree_helpers.h" const char *symbol_names[12] = { - "ERROR", "END", "DOCUMENT", "AMBIGUITY", - "zero", "one", "two", "three", "four", "five", "six", "seven", + "ERROR", "END", "two", "three", "four", "five", "six", "seven", "eight", + "nine", "ten", "eleven", }; TSTree ** tree_array(std::vector trees) { diff --git a/spec/runtime/languages/arithmetic/errors.txt b/spec/runtime/languages/arithmetic/errors.txt index c8675ba4..cbb75418 100644 --- a/spec/runtime/languages/arithmetic/errors.txt +++ b/spec/runtime/languages/arithmetic/errors.txt @@ -6,7 +6,7 @@ x * * y --- -(variable) (ERROR (UNEXPECTED '*') (variable)) +(ERROR (variable) (UNEXPECTED '*') (variable)) ===================================================== errors inside parenthesized expressions diff --git a/spec/runtime/languages/javascript/literals.txt b/spec/runtime/languages/javascript/literals.txt index 6aff2d35..bde21155 100644 --- a/spec/runtime/languages/javascript/literals.txt +++ b/spec/runtime/languages/javascript/literals.txt @@ -68,18 +68,20 @@ var thing = { }; --- -(comment) -(comment) -(var_declaration (var_assignment - (identifier) - (object - (comment) - (comment) - (pair (identifier) (function_expression - (formal_parameters (identifier) (comment)) - (statement_block - (comment) - (expression_statement (function_call (identifier))))))))) + +(program + (comment) + (comment) + (var_declaration (var_assignment + (identifier) + (object + (comment) + (comment) + (pair (identifier) (function_expression + (formal_parameters (identifier) (comment)) + (statement_block + (comment) + (expression_statement (function_call (identifier)))))))))) ========================================== comments within expressions diff --git a/spec/runtime/languages/language_specs.cc b/spec/runtime/languages/language_specs.cc index edc1be15..02662477 100644 --- a/spec/runtime/languages/language_specs.cc +++ b/spec/runtime/languages/language_specs.cc @@ -10,6 +10,12 @@ extern "C" const TSLanguage *ts_language_arithmetic(); extern "C" const TSLanguage *ts_language_golang(); extern "C" const TSLanguage *ts_language_c(); +static string trim_newlines(const string &input) { + size_t start = input.find_first_not_of("\n"); + size_t last = input.find_last_not_of("\n"); + return input.substr(start, last + 1); +} + START_TEST describe("Languages", [&]() { @@ -31,9 +37,11 @@ describe("Languages", [&]() { }); for (auto &entry : test_entries_for_language(language_name)) { + entry.input = trim_newlines(entry.input); + auto expect_the_correct_tree = [&]() { const char *node_string = ts_node_string(ts_document_root_node(doc), doc); - AssertThat(node_string, Equals(("(DOCUMENT " + entry.tree_string + ")").c_str())); + AssertThat(node_string, Equals(entry.tree_string.c_str())); free((void *)node_string); }; diff --git a/spec/runtime/node_spec.cc b/spec/runtime/node_spec.cc index a8e48d46..ef4e4f4d 100644 --- a/spec/runtime/node_spec.cc +++ b/spec/runtime/node_spec.cc @@ -6,18 +6,18 @@ START_TEST describe("Node", []() { TSDocument *document; - TSNode root; + TSNode array_node; before_each([&]() { document = ts_document_make(); ts_document_set_language(document, ts_language_json()); ts_document_set_input_string(document, " [123, false, {\"x\": null}]"); - root = ts_document_root_node(document); - AssertThat(ts_node_string(root, document), Equals( - "(DOCUMENT (array " + array_node = ts_document_root_node(document); + AssertThat(ts_node_string(array_node, document), Equals( + "(array " "(number) " "(false) " - "(object (string) (null))))")); + "(object (string) (null)))")); }); after_each([&]() { @@ -26,25 +26,23 @@ describe("Node", []() { describe("child_count()", [&]() { it("returns the number of visible child nodes", [&]() { - TSNode array = ts_node_child(root, 0); - AssertThat(ts_node_child_count(array), Equals(3)); + AssertThat(ts_node_child_count(array_node), Equals(3)); }); }); describe("child(i)", [&]() { it("returns the child node at the given index", [&]() { - TSNode parent = ts_node_child(root, 0); - TSNode child1 = ts_node_child(parent, 0); - TSNode child2 = ts_node_child(parent, 1); - TSNode child3 = ts_node_child(parent, 2); + TSNode child1 = ts_node_child(array_node, 0); + TSNode child2 = ts_node_child(array_node, 1); + TSNode child3 = ts_node_child(array_node, 2); - AssertThat(ts_node_name(parent, document), Equals("array")); + AssertThat(ts_node_name(array_node, document), Equals("array")); AssertThat(ts_node_name(child1, document), Equals("number")); AssertThat(ts_node_name(child2, document), Equals("false")); AssertThat(ts_node_name(child3, document), Equals("object")); - AssertThat(ts_node_pos(parent).bytes, Equals(2)); - AssertThat(ts_node_size(parent).bytes, Equals(25)); + AssertThat(ts_node_pos(array_node).bytes, Equals(2)); + AssertThat(ts_node_size(array_node).bytes, Equals(25)); AssertThat(ts_node_pos(child1).bytes, Equals(3)); AssertThat(ts_node_size(child1).bytes, Equals(3)); @@ -55,17 +53,15 @@ describe("Node", []() { AssertThat(ts_node_pos(child3).bytes, Equals(15)); AssertThat(ts_node_size(child3).bytes, Equals(11)); - AssertThat(ts_node_parent(child1), Equals(parent)); - AssertThat(ts_node_parent(child2), Equals(parent)); - AssertThat(ts_node_parent(child3), Equals(parent)); - AssertThat(ts_node_parent(parent), Equals(root)); - AssertThat(ts_node_parent(root).data, Equals(nullptr)); + AssertThat(ts_node_parent(child1), Equals(array_node)); + AssertThat(ts_node_parent(child2), Equals(array_node)); + AssertThat(ts_node_parent(child3), Equals(array_node)); + AssertThat(ts_node_parent(array_node).data, Equals(nullptr)); }); }); describe("next_sibling() and prev_sibling()", [&]() { it("returns the node's next and previous siblings", [&]() { - TSNode array_node = ts_node_child(root, 0); TSNode number_node = ts_node_child(array_node, 0); TSNode false_node = ts_node_child(array_node, 1); TSNode object_node = ts_node_child(array_node, 2); @@ -81,9 +77,8 @@ describe("Node", []() { }); it("returns null when the node has no parent", [&]() { - TSNode array_node = ts_node_child(root, 0); - AssertThat(ts_node_next_sibling(root).data, Equals(nullptr)); - AssertThat(ts_node_prev_sibling(root).data, Equals(nullptr)); + AssertThat(ts_node_next_sibling(array_node).data, Equals(nullptr)); + AssertThat(ts_node_prev_sibling(array_node).data, Equals(nullptr)); AssertThat(ts_node_next_sibling(array_node).data, Equals(nullptr)); AssertThat(ts_node_prev_sibling(array_node).data, Equals(nullptr)); }); @@ -92,12 +87,12 @@ describe("Node", []() { describe("find_for_range(start, end)", [&]() { describe("when there is a leaf node that spans the given range exactly", [&]() { it("returns that leaf node", [&]() { - TSNode leaf = ts_node_find_for_range(root, 16, 18); + TSNode leaf = ts_node_find_for_range(array_node, 16, 18); AssertThat(ts_node_name(leaf, document), Equals("string")); AssertThat(ts_node_size(leaf).bytes, Equals(3)); AssertThat(ts_node_pos(leaf).bytes, Equals(16)); - leaf = ts_node_find_for_range(root, 3, 5); + leaf = ts_node_find_for_range(array_node, 3, 5); AssertThat(ts_node_name(leaf, document), Equals("number")); AssertThat(ts_node_size(leaf).bytes, Equals(3)); AssertThat(ts_node_pos(leaf).bytes, Equals(3)); @@ -106,12 +101,12 @@ describe("Node", []() { describe("when there is a leaf node that extends beyond the given range", [&]() { it("returns that leaf node", [&]() { - TSNode leaf = ts_node_find_for_range(root, 16, 17); + TSNode leaf = ts_node_find_for_range(array_node, 16, 17); AssertThat(ts_node_name(leaf, document), Equals("string")); AssertThat(ts_node_size(leaf).bytes, Equals(3)); AssertThat(ts_node_pos(leaf).bytes, Equals(16)); - leaf = ts_node_find_for_range(root, 17, 18); + leaf = ts_node_find_for_range(array_node, 17, 18); AssertThat(ts_node_name(leaf, document), Equals("string")); AssertThat(ts_node_size(leaf).bytes, Equals(3)); AssertThat(ts_node_pos(leaf).bytes, Equals(16)); @@ -120,14 +115,14 @@ describe("Node", []() { describe("when there is no leaf node that spans the given range", [&]() { it("returns the smallest node that does span the range", [&]() { - TSNode node = ts_node_find_for_range(root, 16, 19); + TSNode node = ts_node_find_for_range(array_node, 16, 19); AssertThat(ts_node_name(node, document), Equals("object")); AssertThat(ts_node_size(node).bytes, Equals(11)); AssertThat(ts_node_pos(node).bytes, Equals(15)); }); it("does not return invisible nodes (repeats)", [&]() { - TSNode node = ts_node_find_for_range(root, 6, 7); + TSNode node = ts_node_find_for_range(array_node, 6, 7); AssertThat(ts_node_name(node, document), Equals("array")); AssertThat(ts_node_size(node).bytes, Equals(25)); AssertThat(ts_node_pos(node).bytes, Equals(2)); @@ -137,7 +132,7 @@ describe("Node", []() { describe("find_for_pos(position)", [&]() { it("finds the smallest node that spans the given position", [&]() { - TSNode node = ts_node_find_for_pos(root, 10); + TSNode node = ts_node_find_for_pos(array_node, 10); AssertThat(ts_node_name(node, document), Equals("false")); AssertThat(ts_node_pos(node).bytes, Equals(8)); AssertThat(ts_node_size(node).bytes, Equals(5)); diff --git a/spec/runtime/parser_spec.cc b/spec/runtime/parser_spec.cc index e49d4a8e..00963f8e 100644 --- a/spec/runtime/parser_spec.cc +++ b/spec/runtime/parser_spec.cc @@ -76,11 +76,10 @@ describe("Parser", [&]() { set_text(" [123, @@@@@, true]"); AssertThat(ts_node_string(root, doc), Equals( - "(DOCUMENT (array (number) (ERROR (UNEXPECTED '@')) (true)))")); + "(array (number) (ERROR (UNEXPECTED '@')) (true))")); - TSNode array = ts_node_child(root, 0); - TSNode error = ts_node_child(array, 1); - TSNode last = ts_node_child(array, 2); + TSNode error = ts_node_child(root, 1); + TSNode last = ts_node_child(root, 2); AssertThat(ts_node_name(error, doc), Equals("ERROR")); AssertThat(ts_node_pos(error).bytes, Equals(strlen(" [123, "))) @@ -96,11 +95,10 @@ describe("Parser", [&]() { set_text(" [123, faaaaalse, true]"); AssertThat(ts_node_string(root, doc), Equals( - "(DOCUMENT (array (number) (ERROR (UNEXPECTED 'a')) (true)))")); + "(array (number) (ERROR (UNEXPECTED 'a')) (true))")); - TSNode array = ts_node_child(root, 0); - TSNode error = ts_node_child(array, 1); - TSNode last = ts_node_child(array, 2); + TSNode error = ts_node_child(root, 1); + TSNode last = ts_node_child(root, 2); AssertThat(ts_node_name(error, doc), Equals("ERROR")); AssertThat(ts_node_pos(error).bytes, Equals(strlen(" [123, "))) @@ -116,11 +114,10 @@ describe("Parser", [&]() { set_text(" [123, true false, true]"); AssertThat(ts_node_string(root, doc), Equals( - "(DOCUMENT (array (number) (ERROR (true) (UNEXPECTED 'f') (false)) (true)))")); + "(array (number) (ERROR (true) (UNEXPECTED 'f') (false)) (true))")); - TSNode array = ts_node_child(root, 0); - TSNode error = ts_node_child(array, 1); - TSNode last = ts_node_child(array, 2); + TSNode error = ts_node_child(root, 1); + TSNode last = ts_node_child(root, 2); AssertThat(ts_node_name(error, doc), Equals("ERROR")); AssertThat(ts_node_pos(error).bytes, Equals(strlen(" [123, "))); @@ -136,11 +133,10 @@ describe("Parser", [&]() { set_text(" [123, , true]"); AssertThat(ts_node_string(root, doc), Equals( - "(DOCUMENT (array (number) (ERROR (UNEXPECTED ',')) (true)))")); + "(array (number) (ERROR (UNEXPECTED ',')) (true))")); - TSNode array = ts_node_child(root, 0); - TSNode error = ts_node_child(array, 1); - TSNode last = ts_node_child(array, 2); + TSNode error = ts_node_child(root, 1); + TSNode last = ts_node_child(root, 2); AssertThat(ts_node_name(error, doc), Equals("ERROR")); AssertThat(ts_node_pos(error).bytes, Equals(strlen(" [123, "))); @@ -164,7 +160,7 @@ describe("Parser", [&]() { set_text("fn()\n"); AssertThat(ts_node_string(root, doc), Equals( - "(DOCUMENT (expression_statement (function_call (identifier))))")); + "(expression_statement (function_call (identifier)))")); }); }); @@ -175,9 +171,8 @@ describe("Parser", [&]() { " .otherFn();"); AssertThat(ts_node_string(root, doc), Equals( - "(DOCUMENT " - "(expression_statement (function_call " - "(member_access (function_call (identifier)) (identifier)))))")); + "(expression_statement (function_call " + "(member_access (function_call (identifier)) (identifier))))")); }); }); @@ -190,11 +185,10 @@ describe("Parser", [&]() { ".otherFn();"); AssertThat(ts_node_string(root, doc), Equals( - "(DOCUMENT " - "(expression_statement (function_call " - "(member_access (function_call (identifier)) " - "(comment) " - "(identifier)))))")); + "(expression_statement (function_call " + "(member_access (function_call (identifier)) " + "(comment) " + "(identifier))))")); }); }); }); @@ -210,18 +204,18 @@ describe("Parser", [&]() { set_text("x ^ (100 + abc)"); AssertThat(ts_node_string(root, doc), Equals( - "(DOCUMENT (exponent " + "(exponent " "(variable) " - "(group (sum (number) (variable)))))")); + "(group (sum (number) (variable))))")); insert_text(strlen("x ^ (100 + abc"), " * 5"); }); it("updates the parse tree", [&]() { AssertThat(ts_node_string(root, doc), Equals( - "(DOCUMENT (exponent " + "(exponent " "(variable) " - "(group (sum (number) (product (variable) (number))))))")); + "(group (sum (number) (product (variable) (number)))))")); }); it("re-reads only the changed portion of the input", [&]() { @@ -236,20 +230,20 @@ describe("Parser", [&]() { set_text("123 * 456 ^ (10 + x)"); AssertThat(ts_node_string(root, doc), Equals( - "(DOCUMENT (product " + "(product " "(number) " - "(exponent (number) (group (sum (number) (variable))))))")); + "(exponent (number) (group (sum (number) (variable)))))")); insert_text(strlen("123"), " + 5"); }); it("updates the parse tree", [&]() { AssertThat(ts_node_string(root, doc), Equals( - "(DOCUMENT (sum " + "(sum " "(number) " "(product " "(number) " - "(exponent (number) (group (sum (number) (variable)))))))")); + "(exponent (number) (group (sum (number) (variable))))))")); }); it("re-reads only the changed portion of the input", [&]() { @@ -264,19 +258,19 @@ describe("Parser", [&]() { set_text("var x = y;"); AssertThat(ts_node_string(root, doc), Equals( - "(DOCUMENT (var_declaration (var_assignment " - "(identifier) (identifier))))")); + "(var_declaration (var_assignment " + "(identifier) (identifier)))")); insert_text(strlen("var x = y"), " *"); AssertThat(ts_node_string(root, doc), Equals( - "(DOCUMENT (var_declaration (ERROR (identifier) (identifier) (UNEXPECTED ';'))))")); + "(var_declaration (ERROR (identifier) (identifier) (UNEXPECTED ';')))")); insert_text(strlen("var x = y *"), " z"); AssertThat(ts_node_string(root, doc), Equals( - "(DOCUMENT (var_declaration (var_assignment " - "(identifier) (math_op (identifier) (identifier)))))")); + "(var_declaration (var_assignment " + "(identifier) (math_op (identifier) (identifier))))")); }); }); @@ -285,14 +279,14 @@ describe("Parser", [&]() { set_text("abc * 123"); AssertThat(ts_node_string(root, doc), Equals( - "(DOCUMENT (product (variable) (number)))")); + "(product (variable) (number))")); insert_text(strlen("ab"), "XYZ"); }); it("updates the parse tree", [&]() { AssertThat(ts_node_string(root, doc), Equals( - "(DOCUMENT (product (variable) (number)))")); + "(product (variable) (number))")); TSNode node = ts_node_find_for_pos(root, 1); AssertThat(ts_node_name(node, doc), Equals("variable")); @@ -305,14 +299,14 @@ describe("Parser", [&]() { set_text("abc * 123"); AssertThat(ts_node_string(root, doc), Equals( - "(DOCUMENT (product (variable) (number)))")); + "(product (variable) (number))")); insert_text(strlen("abc"), "XYZ"); }); it("updates the parse tree", [&]() { AssertThat(ts_node_string(root, doc), Equals( - "(DOCUMENT (product (variable) (number)))")); + "(product (variable) (number))")); TSNode node = ts_node_find_for_pos(root, 1); AssertThat(ts_node_name(node, doc), Equals("variable")); @@ -326,7 +320,7 @@ describe("Parser", [&]() { set_text("\u03b1\u03b2\u03b4 + 1"); AssertThat(ts_node_string(root, doc), Equals( - "(DOCUMENT (sum (variable) (number)))")); + "(sum (variable) (number))")); // αβδ + ψ1 insert_text(strlen("abd + "), "\u03c8"); @@ -334,7 +328,7 @@ describe("Parser", [&]() { it("inserts the text according to the UTF8 character index", [&]() { AssertThat(ts_node_string(root, doc), Equals( - "(DOCUMENT (sum (variable) (variable)))")); + "(sum (variable) (variable))")); }); }); @@ -345,7 +339,7 @@ describe("Parser", [&]() { "abc"); AssertThat(ts_node_string(root, doc), Equals( - "(DOCUMENT (product (number) (comment) (variable)))")); + "(product (number) (comment) (variable))")); insert_text( strlen("123 *\n" @@ -356,7 +350,7 @@ describe("Parser", [&]() { it("updates the parse tree", [&]() { AssertThat(ts_node_string(root, doc), Equals( - "(DOCUMENT (product (number) (comment) (variable)))")); + "(product (number) (comment) (variable))")); }); }); }); @@ -367,14 +361,14 @@ describe("Parser", [&]() { set_text("123 * 456"); AssertThat(ts_node_string(root, doc), Equals( - "(DOCUMENT (product (number) (number)))")); + "(product (number) (number))")); delete_text(strlen("123 "), 2); }); it("updates the parse tree, creating an error", [&]() { AssertThat(ts_node_string(root, doc), Equals( - "(DOCUMENT (number) (ERROR (UNEXPECTED '4') (number)))")); + "(ERROR (number) (UNEXPECTED '4') (number))")); }); }); }); @@ -386,14 +380,14 @@ describe("Parser", [&]() { set_text("{ x: (b.c) };"); AssertThat(ts_node_string(root, doc), Equals( - "(DOCUMENT (expression_statement (object (pair " - "(identifier) (expression (member_access (identifier) (identifier)))))))")); + "(expression_statement (object (pair " + "(identifier) (expression (member_access (identifier) (identifier))))))")); replace_text(strlen("{ x: "), strlen("(b.c)"), "b.c"); AssertThat(ts_node_string(root, doc), Equals( - "(DOCUMENT (expression_statement (object (pair " - "(identifier) (member_access (identifier) (identifier))))))")); + "(expression_statement (object (pair " + "(identifier) (member_access (identifier) (identifier)))))")); }); }); }); @@ -407,11 +401,10 @@ describe("Parser", [&]() { it("terminates them at the end of the document", [&]() { set_text("x # this is a comment"); - AssertThat(ts_node_string(root, doc), Equals("(DOCUMENT " - "(expression (variable) (comment)))")); + AssertThat(ts_node_string(root, doc), Equals( + "(expression (variable) (comment))")); - TSNode expression = ts_node_child(root, 0); - TSNode comment = ts_node_child(expression, 1); + TSNode comment = ts_node_child(root, 1); AssertThat(ts_node_size(comment).bytes, Equals(strlen("# this is a comment"))); }); @@ -421,8 +414,8 @@ describe("Parser", [&]() { // x # ΩΩΩ — ΔΔ set_text("x # \u03A9\u03A9\u03A9 \u2014 \u0394\u0394"); - AssertThat(ts_node_string(root, doc), Equals("(DOCUMENT " - "(expression (variable) (comment)))")); + AssertThat(ts_node_string(root, doc), Equals( + "(expression (variable) (comment))")); AssertThat(ts_node_size(root).chars, Equals(strlen("x # OOO - DD"))); AssertThat(ts_node_size(root).bytes, Equals(strlen("x # \u03A9\u03A9\u03A9 \u2014 \u0394\u0394"))); diff --git a/spec/runtime/tree_spec.cc b/spec/runtime/tree_spec.cc index 186e4608..17b99b6d 100644 --- a/spec/runtime/tree_spec.cc +++ b/spec/runtime/tree_spec.cc @@ -11,7 +11,6 @@ enum { }; static const char *names[] = { - "DOCUMENT", "ERROR", "END", "cat", diff --git a/src/compiler/build_tables/build_parse_table.cc b/src/compiler/build_tables/build_parse_table.cc index 21e8f332..1259b054 100644 --- a/src/compiler/build_tables/build_parse_table.cc +++ b/src/compiler/build_tables/build_parse_table.cc @@ -73,7 +73,6 @@ class ParseTableBuilder { add_reduce_extra_actions(state); parse_table.symbols.insert(rules::ERROR()); - parse_table.symbols.insert(rules::DOCUMENT()); return { parse_table, nullptr }; } diff --git a/src/compiler/generate_code/c_code.cc b/src/compiler/generate_code/c_code.cc index 50204c25..25631ecc 100644 --- a/src/compiler/generate_code/c_code.cc +++ b/src/compiler/generate_code/c_code.cc @@ -325,8 +325,6 @@ class CCodeGenerator { return "ts_builtin_sym_error"; else if (symbol == rules::END_OF_INPUT()) return "ts_builtin_sym_end"; - else if (symbol == rules::DOCUMENT()) - return "ts_builtin_sym_document"; else return ""; } else { @@ -344,8 +342,6 @@ class CCodeGenerator { return "ERROR"; else if (symbol == rules::END_OF_INPUT()) return "END"; - else if (symbol == rules::DOCUMENT()) - return "DOCUMENT"; else return ""; } else { diff --git a/src/compiler/rules/built_in_symbols.cc b/src/compiler/rules/built_in_symbols.cc index 92c14866..9d1e924b 100644 --- a/src/compiler/rules/built_in_symbols.cc +++ b/src/compiler/rules/built_in_symbols.cc @@ -15,9 +15,5 @@ Symbol START() { return Symbol(-3); } -Symbol DOCUMENT() { - return Symbol(-4); -} - } // namespace rules } // namespace tree_sitter diff --git a/src/compiler/rules/built_in_symbols.h b/src/compiler/rules/built_in_symbols.h index 63ad3df4..efefb479 100644 --- a/src/compiler/rules/built_in_symbols.h +++ b/src/compiler/rules/built_in_symbols.h @@ -7,9 +7,8 @@ namespace tree_sitter { namespace rules { Symbol ERROR(); -Symbol START(); Symbol END_OF_INPUT(); -Symbol DOCUMENT(); +Symbol START(); } // namespace rules } // namespace tree_sitter diff --git a/src/runtime/document.c b/src/runtime/document.c index d097b65f..b7dbeea9 100644 --- a/src/runtime/document.c +++ b/src/runtime/document.c @@ -1,5 +1,6 @@ #include "tree_sitter/parser.h" #include "runtime/node.h" +#include "runtime/tree.h" #include "runtime/length.h" #include "runtime/parser.h" #include "runtime/string_input.h" @@ -62,5 +63,8 @@ void ts_document_set_input_string(TSDocument *document, const char *text) { } TSNode ts_document_root_node(const TSDocument *document) { - return ts_node_make(document->tree, ts_length_zero()); + TSTree *tree = document->tree; + while (tree && ts_tree_is_singleton(tree)) + tree = tree->children[0]; + return ts_node_make(tree, ts_length_zero()); } diff --git a/src/runtime/parser.c b/src/runtime/parser.c index 9a65f048..501ae03a 100644 --- a/src/runtime/parser.c +++ b/src/runtime/parser.c @@ -305,7 +305,17 @@ static bool ts_parser__handle_error(TSParser *parser, int head) { } static TSTree *ts_parser__finish(TSParser *parser) { - return ts_parser__reduce(parser, 0, ts_builtin_sym_document, -1, false, true); + ParseStackPopResult pop_result = + ts_parse_stack_pop(parser->stack, 0, -1, true).contents[0]; + + TSTree **trees = pop_result.trees; + size_t extra_count = pop_result.tree_count - 1; + TSTree *root = trees[extra_count]; + assert(root->child_count > 0); + + ts_tree_prepend_children(root, extra_count, trees); + ts_parse_stack_push(parser->stack, 0, 0, root); + return root; } typedef enum { diff --git a/src/runtime/tree.c b/src/runtime/tree.c index 4af731d5..63257b68 100644 --- a/src/runtime/tree.c +++ b/src/runtime/tree.c @@ -67,8 +67,8 @@ TSTree *ts_tree_make_node(TSSymbol symbol, size_t child_count, } else { if (is_hidden) options |= TSTreeOptionsHidden; - if (child_count == 1 && symbol != ts_builtin_sym_document && - (ts_tree_is_visible(children[0]) || ts_tree_is_wrapper(children[0]))) + if (child_count == 1 && + (ts_tree_is_visible(children[0]) || ts_tree_is_singleton(children[0]))) options |= (TSTreeOptionsSingleton | TSTreeOptionsHidden); if (child_count > 0) { if (ts_tree_is_fragile_left(children[0])) @@ -183,3 +183,32 @@ char *ts_tree_string(const TSTree *tree, const char **symbol_names) { ts_tree__write_to_string(tree, symbol_names, result, size, 1); return result; } + +void ts_tree_prepend_children(TSTree *tree, size_t count, TSTree **children) { + if (count == 0) + return; + + tree->size = ts_length_add(tree->size, tree->padding); + + size_t visible_count = 0; + for (size_t i = 0; i < count; i++) { + if (i == 0) + tree->padding = children[i]->padding; + else + tree->size = ts_length_add(tree->size, children[i]->padding); + tree->size = ts_length_add(tree->size, children[i]->size); + if (ts_tree_is_visible(children[i])) + visible_count++; + } + + size_t new_child_count = count + tree->child_count; + TSTree **new_children = realloc(children, new_child_count * sizeof(TSTree *)); + memcpy(new_children + count, tree->children, + tree->child_count * sizeof(TSTree *)); + free(tree->children); + + ts_tree_unset_singleton(tree); + tree->children = new_children; + tree->visible_child_count += visible_count; + tree->child_count += count; +} diff --git a/src/runtime/tree.h b/src/runtime/tree.h index a3491d05..27abbf99 100644 --- a/src/runtime/tree.h +++ b/src/runtime/tree.h @@ -47,10 +47,14 @@ static inline bool ts_tree_is_visible(const TSTree *tree) { return !(tree->options & TSTreeOptionsHidden); } -static inline bool ts_tree_is_wrapper(const TSTree *tree) { +static inline bool ts_tree_is_singleton(const TSTree *tree) { return !!(tree->options & TSTreeOptionsSingleton); } +static inline void ts_tree_unset_singleton(TSTree *tree) { + tree->options = (TSTreeOptions)(tree->options & ~TSTreeOptionsSingleton); +} + static inline void ts_tree_set_options(TSTree *tree, TSTreeOptions options) { tree->options = (TSTreeOptions)(tree->options | options); } @@ -84,6 +88,7 @@ bool ts_tree_eq(const TSTree *tree1, const TSTree *tree2); char *ts_tree_string(const TSTree *tree, const char **names); char *ts_tree_error_string(const TSTree *tree, const char **names); TSLength ts_tree_total_size(const TSTree *tree); +void ts_tree_prepend_children(TSTree *, size_t, TSTree **); static inline bool ts_tree_is_empty(TSTree *tree) { return ts_tree_total_size(tree).bytes == 0; diff --git a/todo.md b/todo.md index a2c2f3ce..25509208 100644 --- a/todo.md +++ b/todo.md @@ -7,8 +7,8 @@ TODO * Fix memory leaks in the graph-structured parse stack. ### Runtime System -* Remove 'document' wrapper node. * Make separate symbol for unexpected characters than for interior error nodes. +* Make anonymous tokens visible via separate API methods ### Testing / Quality * Start running the clang-analyzer on the codebase on travis-CI.