Remove 'document' wrapper node

include/tree_sitter/runtime.h

@@ -72,8 +72,7 @@ TSNode ts_document_root_node(const TSDocument *);
 
 #define ts_builtin_sym_error 0
 #define ts_builtin_sym_end 1
-#define ts_builtin_sym_document 2
-#define ts_builtin_sym_start 3
+#define ts_builtin_sym_start 2
 
 #ifdef __cplusplus
 }

spec/fixtures/parsers/arithmetic.c

@@ -1,7 +1,7 @@
 #include "tree_sitter/parser.h"
 
 #define STATE_COUNT 32
-#define SYMBOL_COUNT 20
+#define SYMBOL_COUNT 19
 
 enum {
     sym_expression = ts_builtin_sym_start,
@@ -24,7 +24,6 @@ enum {
 };
 
 static const char *ts_symbol_names[] = {
-    [ts_builtin_sym_document] = "DOCUMENT",
     [sym_expression] = "expression",
     [sym_sum] = "sum",
     [sym_difference] = "difference",

spec/fixtures/parsers/c.c

@@ -1,7 +1,7 @@
 #include "tree_sitter/parser.h"
 
 #define STATE_COUNT 268
-#define SYMBOL_COUNT 65
+#define SYMBOL_COUNT 64
 
 enum {
     sym_program = ts_builtin_sym_start,
@@ -69,7 +69,6 @@ enum {
 };
 
 static const char *ts_symbol_names[] = {
-    [ts_builtin_sym_document] = "DOCUMENT",
     [sym_program] = "program",
     [sym_function_definition] = "function_definition",
     [sym_declaration_specifiers] = "declaration_specifiers",

spec/fixtures/parsers/golang.c

@@ -1,7 +1,7 @@
 #include "tree_sitter/parser.h"
 
 #define STATE_COUNT 431
-#define SYMBOL_COUNT 85
+#define SYMBOL_COUNT 84
 
 enum {
     sym_program = ts_builtin_sym_start,
@@ -89,7 +89,6 @@ enum {
 };
 
 static const char *ts_symbol_names[] = {
-    [ts_builtin_sym_document] = "DOCUMENT",
     [sym_program] = "program",
     [sym_package_directive] = "package_directive",
     [sym_imports_block] = "imports_block",

spec/fixtures/parsers/javascript.c

@@ -1,7 +1,7 @@
 #include "tree_sitter/parser.h"
 
 #define STATE_COUNT 1564
-#define SYMBOL_COUNT 107
+#define SYMBOL_COUNT 106
 
 enum {
     sym_program = ts_builtin_sym_start,
@@ -111,7 +111,6 @@ enum {
 };
 
 static const char *ts_symbol_names[] = {
-    [ts_builtin_sym_document] = "DOCUMENT",
     [sym_program] = "program",
     [sym_statement] = "statement",
     [sym_expression_statement] = "expression_statement",

spec/fixtures/parsers/json.c

@@ -1,7 +1,7 @@
 #include "tree_sitter/parser.h"
 
 #define STATE_COUNT 69
-#define SYMBOL_COUNT 19
+#define SYMBOL_COUNT 18
 
 enum {
     sym_value = ts_builtin_sym_start,
@@ -23,7 +23,6 @@ enum {
 };
 
 static const char *ts_symbol_names[] = {
-    [ts_builtin_sym_document] = "DOCUMENT",
     [sym_value] = "value",
     [sym_object] = "object",
     [sym_array] = "array",

spec/runtime/document_spec.cc

@@ -27,7 +27,7 @@ describe("Document", [&]() {
         ts_document_set_input_string(doc, "{ \"key\": [1, 2] }");
 
         AssertThat(ts_node_string(ts_document_root_node(doc), doc), Equals(
-          "(DOCUMENT (object (string) (array (number) (number))))"));
+          "(object (string) (array (number) (number)))"));
       });
     });
 
@@ -58,7 +58,7 @@ describe("Document", [&]() {
         ts_document_set_language(doc, ts_language_json());
 
         AssertThat(ts_node_string(ts_document_root_node(doc), doc), Equals(
-          "(DOCUMENT (object (string) (array (number) (number))))"));
+          "(object (string) (array (number) (number)))"));
       });
     });
   });

spec/runtime/helpers/tree_helpers.cc

@@ -1,8 +1,8 @@
 #include "runtime/helpers/tree_helpers.h"
 
 const char *symbol_names[12] = {
-  "ERROR", "END", "DOCUMENT", "AMBIGUITY",
-  "zero", "one", "two", "three", "four", "five", "six", "seven",
+  "ERROR", "END",  "two", "three", "four", "five", "six", "seven", "eight",
+  "nine", "ten", "eleven",
 };
 
 TSTree ** tree_array(std::vector<TSTree *> trees) {

spec/runtime/languages/arithmetic/errors.txt

@@ -6,7 +6,7 @@ x * * y
 
 ---
 
-(variable) (ERROR (UNEXPECTED '*') (variable))
+(ERROR (variable) (UNEXPECTED '*') (variable))
 
 =====================================================
 errors inside parenthesized expressions

spec/runtime/languages/javascript/literals.txt

@@ -68,18 +68,20 @@ var thing = {
 };
 
 ---
-(comment)
-(comment)
-(var_declaration (var_assignment
-    (identifier)
-    (object
-        (comment)
-        (comment)
-        (pair (identifier) (function_expression
-            (formal_parameters (identifier) (comment))
-            (statement_block
-                (comment)
-                (expression_statement (function_call (identifier)))))))))
+
+(program
+  (comment)
+  (comment)
+  (var_declaration (var_assignment
+      (identifier)
+      (object
+          (comment)
+          (comment)
+          (pair (identifier) (function_expression
+              (formal_parameters (identifier) (comment))
+              (statement_block
+                  (comment)
+                  (expression_statement (function_call (identifier))))))))))
 
 ==========================================
 comments within expressions

spec/runtime/languages/language_specs.cc

@@ -10,6 +10,12 @@ extern "C" const TSLanguage *ts_language_arithmetic();
 extern "C" const TSLanguage *ts_language_golang();
 extern "C" const TSLanguage *ts_language_c();
 
+static string trim_newlines(const string &input) {
+  size_t start = input.find_first_not_of("\n");
+  size_t last = input.find_last_not_of("\n");
+  return input.substr(start, last + 1);
+}
+
 START_TEST
 
 describe("Languages", [&]() {
@@ -31,9 +37,11 @@ describe("Languages", [&]() {
       });
 
       for (auto &entry : test_entries_for_language(language_name)) {
+        entry.input = trim_newlines(entry.input);
+
         auto expect_the_correct_tree = [&]() {
           const char *node_string = ts_node_string(ts_document_root_node(doc), doc);
-          AssertThat(node_string, Equals(("(DOCUMENT " + entry.tree_string + ")").c_str()));
+          AssertThat(node_string, Equals(entry.tree_string.c_str()));
           free((void *)node_string);
         };
 

spec/runtime/node_spec.cc

@@ -6,18 +6,18 @@ START_TEST
 
 describe("Node", []() {
   TSDocument *document;
-  TSNode root;
+  TSNode array_node;
 
   before_each([&]() {
     document = ts_document_make();
     ts_document_set_language(document, ts_language_json());
     ts_document_set_input_string(document, "  [123, false, {\"x\": null}]");
-    root = ts_document_root_node(document);
-    AssertThat(ts_node_string(root, document), Equals(
-      "(DOCUMENT (array "
+    array_node = ts_document_root_node(document);
+    AssertThat(ts_node_string(array_node, document), Equals(
+      "(array "
         "(number) "
         "(false) "
-        "(object (string) (null))))"));
+        "(object (string) (null)))"));
   });
 
   after_each([&]() {
@@ -26,25 +26,23 @@ describe("Node", []() {
 
   describe("child_count()", [&]() {
     it("returns the number of visible child nodes", [&]() {
-      TSNode array = ts_node_child(root, 0);
-      AssertThat(ts_node_child_count(array), Equals<size_t>(3));
+      AssertThat(ts_node_child_count(array_node), Equals<size_t>(3));
     });
   });
 
   describe("child(i)", [&]() {
     it("returns the child node at the given index", [&]() {
-      TSNode parent = ts_node_child(root, 0);
-      TSNode child1 = ts_node_child(parent, 0);
-      TSNode child2 = ts_node_child(parent, 1);
-      TSNode child3 = ts_node_child(parent, 2);
+      TSNode child1 = ts_node_child(array_node, 0);
+      TSNode child2 = ts_node_child(array_node, 1);
+      TSNode child3 = ts_node_child(array_node, 2);
 
-      AssertThat(ts_node_name(parent, document), Equals("array"));
+      AssertThat(ts_node_name(array_node, document), Equals("array"));
       AssertThat(ts_node_name(child1, document), Equals("number"));
       AssertThat(ts_node_name(child2, document), Equals("false"));
       AssertThat(ts_node_name(child3, document), Equals("object"));
 
-      AssertThat(ts_node_pos(parent).bytes, Equals<size_t>(2));
-      AssertThat(ts_node_size(parent).bytes, Equals<size_t>(25));
+      AssertThat(ts_node_pos(array_node).bytes, Equals<size_t>(2));
+      AssertThat(ts_node_size(array_node).bytes, Equals<size_t>(25));
 
       AssertThat(ts_node_pos(child1).bytes, Equals<size_t>(3));
       AssertThat(ts_node_size(child1).bytes, Equals<size_t>(3));
@@ -55,17 +53,15 @@ describe("Node", []() {
       AssertThat(ts_node_pos(child3).bytes, Equals<size_t>(15));
       AssertThat(ts_node_size(child3).bytes, Equals<size_t>(11));
 
-      AssertThat(ts_node_parent(child1), Equals(parent));
-      AssertThat(ts_node_parent(child2), Equals(parent));
-      AssertThat(ts_node_parent(child3), Equals(parent));
-      AssertThat(ts_node_parent(parent), Equals(root));
-      AssertThat(ts_node_parent(root).data, Equals<void *>(nullptr));
+      AssertThat(ts_node_parent(child1), Equals(array_node));
+      AssertThat(ts_node_parent(child2), Equals(array_node));
+      AssertThat(ts_node_parent(child3), Equals(array_node));
+      AssertThat(ts_node_parent(array_node).data, Equals<void *>(nullptr));
     });
   });
 
   describe("next_sibling() and prev_sibling()", [&]() {
     it("returns the node's next and previous siblings", [&]() {
-      TSNode array_node = ts_node_child(root, 0);
       TSNode number_node = ts_node_child(array_node, 0);
       TSNode false_node = ts_node_child(array_node, 1);
       TSNode object_node = ts_node_child(array_node, 2);
@@ -81,9 +77,8 @@ describe("Node", []() {
     });
 
     it("returns null when the node has no parent", [&]() {
-      TSNode array_node = ts_node_child(root, 0);
-      AssertThat(ts_node_next_sibling(root).data, Equals<void *>(nullptr));
-      AssertThat(ts_node_prev_sibling(root).data, Equals<void *>(nullptr));
+      AssertThat(ts_node_next_sibling(array_node).data, Equals<void *>(nullptr));
+      AssertThat(ts_node_prev_sibling(array_node).data, Equals<void *>(nullptr));
       AssertThat(ts_node_next_sibling(array_node).data, Equals<void *>(nullptr));
       AssertThat(ts_node_prev_sibling(array_node).data, Equals<void *>(nullptr));
     });
@@ -92,12 +87,12 @@ describe("Node", []() {
   describe("find_for_range(start, end)", [&]() {
     describe("when there is a leaf node that spans the given range exactly", [&]() {
       it("returns that leaf node", [&]() {
-        TSNode leaf = ts_node_find_for_range(root, 16, 18);
+        TSNode leaf = ts_node_find_for_range(array_node, 16, 18);
         AssertThat(ts_node_name(leaf, document), Equals("string"));
         AssertThat(ts_node_size(leaf).bytes, Equals<size_t>(3));
         AssertThat(ts_node_pos(leaf).bytes, Equals<size_t>(16));
 
-        leaf = ts_node_find_for_range(root, 3, 5);
+        leaf = ts_node_find_for_range(array_node, 3, 5);
         AssertThat(ts_node_name(leaf, document), Equals("number"));
         AssertThat(ts_node_size(leaf).bytes, Equals<size_t>(3));
         AssertThat(ts_node_pos(leaf).bytes, Equals<size_t>(3));
@@ -106,12 +101,12 @@ describe("Node", []() {
 
     describe("when there is a leaf node that extends beyond the given range", [&]() {
       it("returns that leaf node", [&]() {
-        TSNode leaf = ts_node_find_for_range(root, 16, 17);
+        TSNode leaf = ts_node_find_for_range(array_node, 16, 17);
         AssertThat(ts_node_name(leaf, document), Equals("string"));
         AssertThat(ts_node_size(leaf).bytes, Equals<size_t>(3));
         AssertThat(ts_node_pos(leaf).bytes, Equals<size_t>(16));
 
-        leaf = ts_node_find_for_range(root, 17, 18);
+        leaf = ts_node_find_for_range(array_node, 17, 18);
         AssertThat(ts_node_name(leaf, document), Equals("string"));
         AssertThat(ts_node_size(leaf).bytes, Equals<size_t>(3));
         AssertThat(ts_node_pos(leaf).bytes, Equals<size_t>(16));
@@ -120,14 +115,14 @@ describe("Node", []() {
 
     describe("when there is no leaf node that spans the given range", [&]() {
       it("returns the smallest node that does span the range", [&]() {
-        TSNode node = ts_node_find_for_range(root, 16, 19);
+        TSNode node = ts_node_find_for_range(array_node, 16, 19);
         AssertThat(ts_node_name(node, document), Equals("object"));
         AssertThat(ts_node_size(node).bytes, Equals<size_t>(11));
         AssertThat(ts_node_pos(node).bytes, Equals<size_t>(15));
       });
 
       it("does not return invisible nodes (repeats)", [&]() {
-        TSNode node = ts_node_find_for_range(root, 6, 7);
+        TSNode node = ts_node_find_for_range(array_node, 6, 7);
         AssertThat(ts_node_name(node, document), Equals("array"));
         AssertThat(ts_node_size(node).bytes, Equals<size_t>(25));
         AssertThat(ts_node_pos(node).bytes, Equals<size_t>(2));
@@ -137,7 +132,7 @@ describe("Node", []() {
 
   describe("find_for_pos(position)", [&]() {
     it("finds the smallest node that spans the given position", [&]() {
-      TSNode node = ts_node_find_for_pos(root, 10);
+      TSNode node = ts_node_find_for_pos(array_node, 10);
       AssertThat(ts_node_name(node, document), Equals("false"));
       AssertThat(ts_node_pos(node).bytes, Equals<size_t>(8));
       AssertThat(ts_node_size(node).bytes, Equals<size_t>(5));

spec/runtime/parser_spec.cc

@@ -76,11 +76,10 @@ describe("Parser", [&]() {
         set_text("  [123,  @@@@@,   true]");
 
         AssertThat(ts_node_string(root, doc), Equals(
-          "(DOCUMENT (array (number) (ERROR (UNEXPECTED '@')) (true)))"));
+          "(array (number) (ERROR (UNEXPECTED '@')) (true))"));
 
-        TSNode array = ts_node_child(root, 0);
-        TSNode error = ts_node_child(array, 1);
-        TSNode last = ts_node_child(array, 2);
+        TSNode error = ts_node_child(root, 1);
+        TSNode last = ts_node_child(root, 2);
 
         AssertThat(ts_node_name(error, doc), Equals("ERROR"));
         AssertThat(ts_node_pos(error).bytes, Equals(strlen("  [123,  ")))
@@ -96,11 +95,10 @@ describe("Parser", [&]() {
         set_text("  [123, faaaaalse, true]");
 
         AssertThat(ts_node_string(root, doc), Equals(
-          "(DOCUMENT (array (number) (ERROR (UNEXPECTED 'a')) (true)))"));
+          "(array (number) (ERROR (UNEXPECTED 'a')) (true))"));
 
-        TSNode array = ts_node_child(root, 0);
-        TSNode error = ts_node_child(array, 1);
-        TSNode last = ts_node_child(array, 2);
+        TSNode error = ts_node_child(root, 1);
+        TSNode last = ts_node_child(root, 2);
 
         AssertThat(ts_node_name(error, doc), Equals("ERROR"));
         AssertThat(ts_node_pos(error).bytes, Equals(strlen("  [123, ")))
@@ -116,11 +114,10 @@ describe("Parser", [&]() {
         set_text("  [123, true false, true]");
 
         AssertThat(ts_node_string(root, doc), Equals(
-          "(DOCUMENT (array (number) (ERROR (true) (UNEXPECTED 'f') (false)) (true)))"));
+          "(array (number) (ERROR (true) (UNEXPECTED 'f') (false)) (true))"));
 
-        TSNode array = ts_node_child(root, 0);
-        TSNode error = ts_node_child(array, 1);
-        TSNode last = ts_node_child(array, 2);
+        TSNode error = ts_node_child(root, 1);
+        TSNode last = ts_node_child(root, 2);
 
         AssertThat(ts_node_name(error, doc), Equals("ERROR"));
         AssertThat(ts_node_pos(error).bytes, Equals(strlen("  [123, ")));
@@ -136,11 +133,10 @@ describe("Parser", [&]() {
         set_text("  [123, , true]");
 
         AssertThat(ts_node_string(root, doc), Equals(
-          "(DOCUMENT (array (number) (ERROR (UNEXPECTED ',')) (true)))"));
+          "(array (number) (ERROR (UNEXPECTED ',')) (true))"));
 
-        TSNode array = ts_node_child(root, 0);
-        TSNode error = ts_node_child(array, 1);
-        TSNode last = ts_node_child(array, 2);
+        TSNode error = ts_node_child(root, 1);
+        TSNode last = ts_node_child(root, 2);
 
         AssertThat(ts_node_name(error, doc), Equals("ERROR"));
         AssertThat(ts_node_pos(error).bytes, Equals(strlen("  [123, ")));
@@ -164,7 +160,7 @@ describe("Parser", [&]() {
         set_text("fn()\n");
 
         AssertThat(ts_node_string(root, doc), Equals(
-          "(DOCUMENT (expression_statement (function_call (identifier))))"));
+          "(expression_statement (function_call (identifier)))"));
       });
     });
 
@@ -175,9 +171,8 @@ describe("Parser", [&]() {
           "  .otherFn();");
 
         AssertThat(ts_node_string(root, doc), Equals(
-          "(DOCUMENT "
-            "(expression_statement (function_call "
-              "(member_access (function_call (identifier)) (identifier)))))"));
+          "(expression_statement (function_call "
+            "(member_access (function_call (identifier)) (identifier))))"));
       });
     });
 
@@ -190,11 +185,10 @@ describe("Parser", [&]() {
           ".otherFn();");
 
         AssertThat(ts_node_string(root, doc), Equals(
-          "(DOCUMENT "
-            "(expression_statement (function_call "
-              "(member_access (function_call (identifier)) "
-                "(comment) "
-                "(identifier)))))"));
+          "(expression_statement (function_call "
+            "(member_access (function_call (identifier)) "
+              "(comment) "
+              "(identifier))))"));
       });
     });
   });
@@ -210,18 +204,18 @@ describe("Parser", [&]() {
           set_text("x ^ (100 + abc)");
 
           AssertThat(ts_node_string(root, doc), Equals(
-            "(DOCUMENT (exponent "
+            "(exponent "
               "(variable) "
-              "(group (sum (number) (variable)))))"));
+              "(group (sum (number) (variable))))"));
 
           insert_text(strlen("x ^ (100 + abc"), " * 5");
         });
 
         it("updates the parse tree", [&]() {
           AssertThat(ts_node_string(root, doc), Equals(
-            "(DOCUMENT (exponent "
+            "(exponent "
               "(variable) "
-              "(group (sum (number) (product (variable) (number))))))"));
+              "(group (sum (number) (product (variable) (number)))))"));
         });
 
         it("re-reads only the changed portion of the input", [&]() {
@@ -236,20 +230,20 @@ describe("Parser", [&]() {
           set_text("123 * 456 ^ (10 + x)");
 
           AssertThat(ts_node_string(root, doc), Equals(
-            "(DOCUMENT (product "
+            "(product "
               "(number) "
-              "(exponent (number) (group (sum (number) (variable))))))"));
+              "(exponent (number) (group (sum (number) (variable)))))"));
 
           insert_text(strlen("123"), " + 5");
         });
 
         it("updates the parse tree", [&]() {
           AssertThat(ts_node_string(root, doc), Equals(
-            "(DOCUMENT (sum "
+            "(sum "
               "(number) "
               "(product "
                 "(number) "
-                "(exponent (number) (group (sum (number) (variable)))))))"));
+                "(exponent (number) (group (sum (number) (variable))))))"));
         });
 
         it("re-reads only the changed portion of the input", [&]() {
@@ -264,19 +258,19 @@ describe("Parser", [&]() {
           set_text("var x = y;");
 
           AssertThat(ts_node_string(root, doc), Equals(
-            "(DOCUMENT (var_declaration (var_assignment "
-              "(identifier) (identifier))))"));
+            "(var_declaration (var_assignment "
+              "(identifier) (identifier)))"));
 
           insert_text(strlen("var x = y"), " *");
 
           AssertThat(ts_node_string(root, doc), Equals(
-              "(DOCUMENT (var_declaration (ERROR (identifier) (identifier) (UNEXPECTED ';'))))"));
+              "(var_declaration (ERROR (identifier) (identifier) (UNEXPECTED ';')))"));
 
           insert_text(strlen("var x = y *"), " z");
 
           AssertThat(ts_node_string(root, doc), Equals(
-            "(DOCUMENT (var_declaration (var_assignment "
-              "(identifier) (math_op (identifier) (identifier)))))"));
+            "(var_declaration (var_assignment "
+              "(identifier) (math_op (identifier) (identifier))))"));
         });
       });
 
@@ -285,14 +279,14 @@ describe("Parser", [&]() {
           set_text("abc * 123");
 
           AssertThat(ts_node_string(root, doc), Equals(
-            "(DOCUMENT (product (variable) (number)))"));
+            "(product (variable) (number))"));
 
           insert_text(strlen("ab"), "XYZ");
         });
 
         it("updates the parse tree", [&]() {
           AssertThat(ts_node_string(root, doc), Equals(
-            "(DOCUMENT (product (variable) (number)))"));
+            "(product (variable) (number))"));
 
           TSNode node = ts_node_find_for_pos(root, 1);
           AssertThat(ts_node_name(node, doc), Equals("variable"));
@@ -305,14 +299,14 @@ describe("Parser", [&]() {
           set_text("abc * 123");
 
           AssertThat(ts_node_string(root, doc), Equals(
-            "(DOCUMENT (product (variable) (number)))"));
+            "(product (variable) (number))"));
 
           insert_text(strlen("abc"), "XYZ");
         });
 
         it("updates the parse tree", [&]() {
           AssertThat(ts_node_string(root, doc), Equals(
-            "(DOCUMENT (product (variable) (number)))"));
+            "(product (variable) (number))"));
 
           TSNode node = ts_node_find_for_pos(root, 1);
           AssertThat(ts_node_name(node, doc), Equals("variable"));
@@ -326,7 +320,7 @@ describe("Parser", [&]() {
           set_text("\u03b1\u03b2\u03b4 + 1");
 
           AssertThat(ts_node_string(root, doc), Equals(
-            "(DOCUMENT (sum (variable) (number)))"));
+            "(sum (variable) (number))"));
 
           // αβδ + ψ1
           insert_text(strlen("abd + "), "\u03c8");
@@ -334,7 +328,7 @@ describe("Parser", [&]() {
 
         it("inserts the text according to the UTF8 character index", [&]() {
           AssertThat(ts_node_string(root, doc), Equals(
-            "(DOCUMENT (sum (variable) (variable)))"));
+            "(sum (variable) (variable))"));
         });
       });
 
@@ -345,7 +339,7 @@ describe("Parser", [&]() {
             "abc");
 
           AssertThat(ts_node_string(root, doc), Equals(
-            "(DOCUMENT (product (number) (comment) (variable)))"));
+            "(product (number) (comment) (variable))"));
 
           insert_text(
             strlen("123 *\n"
@@ -356,7 +350,7 @@ describe("Parser", [&]() {
 
         it("updates the parse tree", [&]() {
           AssertThat(ts_node_string(root, doc), Equals(
-            "(DOCUMENT (product (number) (comment) (variable)))"));
+            "(product (number) (comment) (variable))"));
         });
       });
     });
@@ -367,14 +361,14 @@ describe("Parser", [&]() {
           set_text("123 * 456");
 
           AssertThat(ts_node_string(root, doc), Equals(
-            "(DOCUMENT (product (number) (number)))"));
+            "(product (number) (number))"));
 
           delete_text(strlen("123 "), 2);
         });
 
         it("updates the parse tree, creating an error", [&]() {
           AssertThat(ts_node_string(root, doc), Equals(
-            "(DOCUMENT (number) (ERROR (UNEXPECTED '4') (number)))"));
+            "(ERROR (number) (UNEXPECTED '4') (number))"));
         });
       });
     });
@@ -386,14 +380,14 @@ describe("Parser", [&]() {
         set_text("{ x: (b.c) };");
 
         AssertThat(ts_node_string(root, doc), Equals(
-          "(DOCUMENT (expression_statement (object (pair "
-            "(identifier) (expression (member_access (identifier) (identifier)))))))"));
+          "(expression_statement (object (pair "
+            "(identifier) (expression (member_access (identifier) (identifier))))))"));
 
         replace_text(strlen("{ x: "), strlen("(b.c)"), "b.c");
 
         AssertThat(ts_node_string(root, doc), Equals(
-          "(DOCUMENT (expression_statement (object (pair "
-            "(identifier) (member_access (identifier) (identifier))))))"));
+          "(expression_statement (object (pair "
+            "(identifier) (member_access (identifier) (identifier)))))"));
       });
     });
   });
@@ -407,11 +401,10 @@ describe("Parser", [&]() {
       it("terminates them at the end of the document", [&]() {
         set_text("x # this is a comment");
 
-        AssertThat(ts_node_string(root, doc), Equals("(DOCUMENT "
-          "(expression (variable) (comment)))"));
+        AssertThat(ts_node_string(root, doc), Equals(
+          "(expression (variable) (comment))"));
 
-        TSNode expression = ts_node_child(root, 0);
-        TSNode comment = ts_node_child(expression, 1);
+        TSNode comment = ts_node_child(root, 1);
 
         AssertThat(ts_node_size(comment).bytes, Equals(strlen("# this is a comment")));
       });
@@ -421,8 +414,8 @@ describe("Parser", [&]() {
       // x # ΩΩΩ — ΔΔ
       set_text("x # \u03A9\u03A9\u03A9 \u2014 \u0394\u0394");
 
-      AssertThat(ts_node_string(root, doc), Equals("(DOCUMENT "
-        "(expression (variable) (comment)))"));
+      AssertThat(ts_node_string(root, doc), Equals(
+        "(expression (variable) (comment))"));
 
       AssertThat(ts_node_size(root).chars, Equals(strlen("x # OOO - DD")));
       AssertThat(ts_node_size(root).bytes, Equals(strlen("x # \u03A9\u03A9\u03A9 \u2014 \u0394\u0394")));

spec/runtime/tree_spec.cc

@@ -11,7 +11,6 @@ enum {
 };
 
 static const char *names[] = {
-  "DOCUMENT",
   "ERROR",
   "END",
   "cat",

src/compiler/build_tables/build_parse_table.cc

@@ -73,7 +73,6 @@ class ParseTableBuilder {
       add_reduce_extra_actions(state);
 
     parse_table.symbols.insert(rules::ERROR());
-    parse_table.symbols.insert(rules::DOCUMENT());
 
     return { parse_table, nullptr };
   }

src/compiler/generate_code/c_code.cc

@@ -325,8 +325,6 @@ class CCodeGenerator {
         return "ts_builtin_sym_error";
       else if (symbol == rules::END_OF_INPUT())
         return "ts_builtin_sym_end";
-      else if (symbol == rules::DOCUMENT())
-        return "ts_builtin_sym_document";
       else
         return "";
     } else {
@@ -344,8 +342,6 @@ class CCodeGenerator {
         return "ERROR";
       else if (symbol == rules::END_OF_INPUT())
         return "END";
-      else if (symbol == rules::DOCUMENT())
-        return "DOCUMENT";
       else
         return "";
     } else {

src/compiler/rules/built_in_symbols.cc

@@ -15,9 +15,5 @@ Symbol START() {
   return Symbol(-3);
 }
 
-Symbol DOCUMENT() {
-  return Symbol(-4);
-}
-
 }  // namespace rules
 }  // namespace tree_sitter

src/compiler/rules/built_in_symbols.h

@@ -7,9 +7,8 @@ namespace tree_sitter {
 namespace rules {
 
 Symbol ERROR();
-Symbol START();
 Symbol END_OF_INPUT();
-Symbol DOCUMENT();
+Symbol START();
 
 }  // namespace rules
 }  // namespace tree_sitter

src/runtime/document.c

@@ -1,5 +1,6 @@
 #include "tree_sitter/parser.h"
 #include "runtime/node.h"
+#include "runtime/tree.h"
 #include "runtime/length.h"
 #include "runtime/parser.h"
 #include "runtime/string_input.h"
@@ -62,5 +63,8 @@ void ts_document_set_input_string(TSDocument *document, const char *text) {
 }
 
 TSNode ts_document_root_node(const TSDocument *document) {
-  return ts_node_make(document->tree, ts_length_zero());
+  TSTree *tree = document->tree;
+  while (tree && ts_tree_is_singleton(tree))
+    tree = tree->children[0];
+  return ts_node_make(tree, ts_length_zero());
 }

src/runtime/parser.c

@@ -305,7 +305,17 @@ static bool ts_parser__handle_error(TSParser *parser, int head) {
 }
 
 static TSTree *ts_parser__finish(TSParser *parser) {
-  return ts_parser__reduce(parser, 0, ts_builtin_sym_document, -1, false, true);
+  ParseStackPopResult pop_result =
+    ts_parse_stack_pop(parser->stack, 0, -1, true).contents[0];
+
+  TSTree **trees = pop_result.trees;
+  size_t extra_count = pop_result.tree_count - 1;
+  TSTree *root = trees[extra_count];
+  assert(root->child_count > 0);
+
+  ts_tree_prepend_children(root, extra_count, trees);
+  ts_parse_stack_push(parser->stack, 0, 0, root);
+  return root;
 }
 
 typedef enum {

src/runtime/tree.c

@@ -67,8 +67,8 @@ TSTree *ts_tree_make_node(TSSymbol symbol, size_t child_count,
   } else {
     if (is_hidden)
       options |= TSTreeOptionsHidden;
-    if (child_count == 1 && symbol != ts_builtin_sym_document &&
-        (ts_tree_is_visible(children[0]) || ts_tree_is_wrapper(children[0])))
+    if (child_count == 1 &&
+        (ts_tree_is_visible(children[0]) || ts_tree_is_singleton(children[0])))
       options |= (TSTreeOptionsSingleton | TSTreeOptionsHidden);
     if (child_count > 0) {
       if (ts_tree_is_fragile_left(children[0]))
@@ -183,3 +183,32 @@ char *ts_tree_string(const TSTree *tree, const char **symbol_names) {
   ts_tree__write_to_string(tree, symbol_names, result, size, 1);
   return result;
 }
+
+void ts_tree_prepend_children(TSTree *tree, size_t count, TSTree **children) {
+  if (count == 0)
+    return;
+
+  tree->size = ts_length_add(tree->size, tree->padding);
+
+  size_t visible_count = 0;
+  for (size_t i = 0; i < count; i++) {
+    if (i == 0)
+      tree->padding = children[i]->padding;
+    else
+      tree->size = ts_length_add(tree->size, children[i]->padding);
+    tree->size = ts_length_add(tree->size, children[i]->size);
+    if (ts_tree_is_visible(children[i]))
+      visible_count++;
+  }
+
+  size_t new_child_count = count + tree->child_count;
+  TSTree **new_children = realloc(children, new_child_count * sizeof(TSTree *));
+  memcpy(new_children + count, tree->children,
+         tree->child_count * sizeof(TSTree *));
+  free(tree->children);
+
+  ts_tree_unset_singleton(tree);
+  tree->children = new_children;
+  tree->visible_child_count += visible_count;
+  tree->child_count += count;
+}

src/runtime/tree.h

@@ -47,10 +47,14 @@ static inline bool ts_tree_is_visible(const TSTree *tree) {
   return !(tree->options & TSTreeOptionsHidden);
 }
 
-static inline bool ts_tree_is_wrapper(const TSTree *tree) {
+static inline bool ts_tree_is_singleton(const TSTree *tree) {
   return !!(tree->options & TSTreeOptionsSingleton);
 }
 
+static inline void ts_tree_unset_singleton(TSTree *tree) {
+  tree->options = (TSTreeOptions)(tree->options & ~TSTreeOptionsSingleton);
+}
+
 static inline void ts_tree_set_options(TSTree *tree, TSTreeOptions options) {
   tree->options = (TSTreeOptions)(tree->options | options);
 }
@@ -84,6 +88,7 @@ bool ts_tree_eq(const TSTree *tree1, const TSTree *tree2);
 char *ts_tree_string(const TSTree *tree, const char **names);
 char *ts_tree_error_string(const TSTree *tree, const char **names);
 TSLength ts_tree_total_size(const TSTree *tree);
+void ts_tree_prepend_children(TSTree *, size_t, TSTree **);
 
 static inline bool ts_tree_is_empty(TSTree *tree) {
   return ts_tree_total_size(tree).bytes == 0;

todo.md

@@ -7,8 +7,8 @@ TODO
 * Fix memory leaks in the graph-structured parse stack.
 
 ### Runtime System
-* Remove 'document' wrapper node.
 * Make separate symbol for unexpected characters than for interior error nodes.
+* Make anonymous tokens visible via separate API methods
 
 ### Testing / Quality
 * Start running the clang-analyzer on the codebase on travis-CI.