Start work on error recovery

- In runtime, make parse errors part of the parse tree
- Add error state to lexers in which they can accept any token

src/runtime/document.cpp

@@ -3,8 +3,9 @@
 struct ts_document {
     ts_parse_fn *parse_fn;
     const char **symbol_names;
-    ts_error error;
-    ts_tree *tree;
+    const ts_tree *tree;
+    size_t error_count;
+    ts_tree **errors;
 };
 
 ts_document * ts_document_make() {
@@ -21,18 +22,18 @@ void ts_document_set_parser(ts_document *document, ts_parse_config config) {
 }
 
 void ts_document_set_text(ts_document *document, const char *text) {
-    ts_parse_result result = document->parse_fn(text);
-    document->tree = result.tree;
-    document->error = result.error;
+    const ts_tree * result = document->parse_fn(text);
+    document->tree = result;
+    document->errors = NULL;
 }
 
-ts_tree * ts_document_tree(const ts_document *document) {
+const ts_tree * ts_document_tree(const ts_document *document) {
     return document->tree;
 }
 
 const char * ts_document_string(const ts_document *document) {
-    if (document->error.expected_inputs != NULL) {
-        return ts_error_string(&document->error);
+    if (document->error_count > 0) {
+        return ts_tree_error_string(document->errors[0], document->symbol_names);
     } else {
         return ts_tree_string(document->tree, document->symbol_names);
     }

src/runtime/error.cpp

@@ -1,16 +0,0 @@
-#include "tree_sitter/runtime.h"
-#include <string>
-#include "string.h"
-
-using std::string;
-
-const char * ts_error_string(const ts_error *error) {
-    string result = string("Unexpected character '") + error->lookahead_char + "'. Expected:";
-    for (int i = 0; i < error->expected_input_count; i++) {
-        result += string(" ") + error->expected_inputs[i];
-    }
-
-    char *stuff = (char *)malloc(result.size() * sizeof(char));
-    strcpy(stuff, result.c_str());
-    return stuff;
-}

src/runtime/tree.cpp

@@ -3,18 +3,43 @@
 #include <string.h>
 
 using std::string;
+using std::to_string;
 
-ts_tree * ts_tree_make(ts_symbol value, size_t child_count, ts_tree **children) {
+const ts_symbol ts_symbol_error = -1;
+
+ts_tree * ts_tree_make_leaf(ts_symbol symbol) {
     ts_tree *result = new ts_tree();
-    result->value = value;
-    result->child_count = child_count;
-    result->children = children;
     result->ref_count = 0;
+    result->symbol = symbol;
+    result->data.children = {
+        .count = 0,
+        .contents = NULL
+    };
+    return result;
+}
+
+ts_tree * ts_tree_make_node(ts_symbol symbol, size_t child_count, ts_tree **children) {
+    ts_tree *result = new ts_tree();
+    result->ref_count = 0;
+    result->symbol = symbol;
+    result->data.children = {
+        .count = child_count,
+        .contents = children
+    };
     for (int i = 0; i < child_count; i++)
         ts_tree_retain(children[i]);
     return result;
 }
 
+ts_tree * ts_tree_make_error(char lookahead_char, size_t expected_input_count, const ts_symbol *expected_inputs) {
+    ts_tree *result = new ts_tree();
+    result->data.error = {
+        .lookahead_char = lookahead_char,
+        .expected_input_count = expected_input_count,
+    };
+    return result;
+}
+
 void ts_tree_retain(ts_tree *tree) {
     tree->ref_count++;
 }
@@ -22,28 +47,46 @@ void ts_tree_retain(ts_tree *tree) {
 void ts_tree_release(ts_tree *tree) {
     tree->ref_count--;
     if (tree->ref_count == 0) {
-        for (int i = 0; i < tree->child_count; i++)
-            ts_tree_release(tree->children[i]);
+        ts_tree **children = tree->data.children.contents;
+        for (int i = 0; i < ts_tree_child_count(tree); i++)
+            ts_tree_release(children[i]);
+//        free(children);
         free(tree);
     }
 }
 
 int ts_tree_equals(const ts_tree *node1, const ts_tree *node2) {
-    if (node1->value != node2->value) return 0;
-    if (node1->child_count != node2->child_count) return 0;
-    for (int i = 0; i < node1->child_count; i++) {
-        ts_tree *child1 = node1->children[i];
-        ts_tree *child2 = node2->children[i];
-        if (!ts_tree_equals(child1, child2)) return 0;
+    if (node1->symbol != node2->symbol) return 0;
+    if (node1->symbol == ts_symbol_error) {
+        // check error equality
+    } else {
+        if (node1->data.children.count != node2->data.children.count)
+            return 0;
+        for (int i = 0; i < node1->data.children.count; i++) {
+            ts_tree *child1 = node1->data.children.contents[i];
+            ts_tree *child2 = node2->data.children.contents[i];
+            if (!ts_tree_equals(child1, child2))
+                return 0;
+        }
     }
     return 1;
 }
 
+ts_tree ** ts_tree_children(const ts_tree *tree) {
+    if (tree->symbol == ts_symbol_error) return NULL;
+    return tree->data.children.contents;
+}
+
+size_t ts_tree_child_count(const ts_tree *tree) {
+    if (tree->symbol == ts_symbol_error) return 0;
+    return tree->data.children.count;
+}
+
 static string __tree_to_string(const ts_tree *tree, const char **symbol_names) {
     if (!tree) return "#<null-tree>";
-    string result = string("(") + symbol_names[tree->value];
-    for (int i = 0; i < tree->child_count; i++)
-        result += " " + __tree_to_string(tree->children[i], symbol_names);
+    string result = string("(") + symbol_names[tree->symbol];
+    for (int i = 0; i < tree->data.children.count; i++)
+        result += " " + __tree_to_string(tree->data.children.contents[i], symbol_names);
     return result + ")";
 }
 
@@ -53,3 +96,15 @@ char * ts_tree_string(const ts_tree *tree, const char **symbol_names) {
     strcpy(result, value.c_str());
     return result;
 }
+
+char * ts_tree_error_string(const ts_tree *tree, const char **symbol_names) {
+    string result = string("Unexpected character '") + tree->data.error.lookahead_char + "'. Expected:";
+    for (int i = 0; i < tree->data.error.expected_input_count; i++) {
+        ts_symbol symbol = tree->data.error.expected_inputs[i];
+        result += string(" ") + symbol_names[symbol];
+    }
+    
+    char *stuff = (char *)malloc(result.size() * sizeof(char));
+    strcpy(stuff, result.c_str());
+    return stuff;
+}