Add serialize, deserialize and reset callbacks to external scanners

Signed-off-by: Nathan Sobo <nathan@github.com>

include/tree_sitter/parser.h

@@ -12,6 +12,8 @@ extern "C" {
 typedef unsigned short TSSymbol;
 typedef unsigned short TSStateId;
 
+typedef uint8_t TSExternalTokenState[16];
+
 #define ts_builtin_sym_error ((TSSymbol)-1)
 #define ts_builtin_sym_end 0
 
@@ -75,8 +77,11 @@ typedef struct TSLanguage {
   const TSSymbol *external_token_symbol_map;
   const bool *external_token_lists;
   struct {
-    void * (*create)();
+    void *(*create)();
     bool (*scan)(void *, TSLexer *, const bool *symbol_whitelist);
+    void (*reset)(void *);
+    bool (*serialize)(void *, TSExternalTokenState);
+    void (*deserialize)(void *, TSExternalTokenState);
     void (*destroy)(void *);
   } external_scanner;
 } TSLanguage;

spec/fixtures/external_scanners/extra_external_tokens.c

@@ -8,7 +8,14 @@ void *ts_language_extra_external_tokens_external_scanner_create() {
   return NULL;
 }
 
-void ts_language_extra_external_tokens_external_scanner_destroy(void *payload) {
+void ts_language_extra_external_tokens_external_scanner_reset(void *payload) {
+}
+
+bool ts_language_extra_external_tokens_external_scanner_serialize(void *payload, TSExternalTokenState state) {
+  return true;
+}
+
+void ts_language_extra_external_tokens_external_scanner_deserialize(void *payload, TSExternalTokenState state) {
 }
 
 bool ts_language_extra_external_tokens_external_scanner_scan(
@@ -30,3 +37,6 @@ bool ts_language_extra_external_tokens_external_scanner_scan(
 
   return false;
 }
+
+void ts_language_extra_external_tokens_external_scanner_destroy(void *payload) {
+}

spec/fixtures/external_scanners/percent_strings.c

@@ -103,6 +103,16 @@ bool ts_language_external_scanner_example_external_scanner_scan(
   return false;
 }
 
+void ts_language_external_scanner_example_external_scanner_reset(void *payload) {
+}
+
+bool ts_language_external_scanner_example_external_scanner_serialize(void *payload, TSExternalTokenState state) {
+  return true;
+}
+
+void ts_language_external_scanner_example_external_scanner_deserialize(void *payload, TSExternalTokenState state) {
+}
+
 void ts_language_external_scanner_example_external_scanner_destroy(void *payload) {
   free(payload);
 }

spec/fixtures/external_scanners/shared_external_tokens.c

@@ -10,7 +10,14 @@ void *ts_language_shared_external_tokens_external_scanner_create() {
   return NULL;
 }
 
-void ts_language_shared_external_tokens_external_scanner_destroy(void *payload) {
+void ts_language_shared_external_tokens_external_scanner_reset(void *payload) {
+}
+
+bool ts_language_shared_external_tokens_external_scanner_serialize(void *payload, TSExternalTokenState state) {
+  return true;
+}
+
+void ts_language_shared_external_tokens_external_scanner_deserialize(void *payload, TSExternalTokenState state) {
 }
 
 bool ts_language_shared_external_tokens_external_scanner_scan(
@@ -51,3 +58,6 @@ bool ts_language_shared_external_tokens_external_scanner_scan(
 
   return false;
 }
+
+void ts_language_shared_external_tokens_external_scanner_destroy(void *payload) {
+}

src/compiler/generate_code/c_code.cc

@@ -356,6 +356,9 @@ class CCodeGenerator {
 
       line("void *" + external_scanner_name + "_create();");
       line("bool " + external_scanner_name + "_scan(void *, TSLexer *, const bool *);");
+      line("void " + external_scanner_name + "_reset(void *);");
+      line("bool " + external_scanner_name + "_serialize(void *, TSExternalTokenState);");
+      line("void " + external_scanner_name + "_deserialize(void *, TSExternalTokenState);");
       line("void " + external_scanner_name + "_destroy();");
       line();
 
@@ -366,6 +369,9 @@ class CCodeGenerator {
           indent([&]() {
             line(external_scanner_name + "_create,");
             line(external_scanner_name + "_scan,");
+            line(external_scanner_name + "_reset,");
+            line(external_scanner_name + "_serialize,");
+            line(external_scanner_name + "_deserialize,");
             line(external_scanner_name + "_destroy,");
           });
           line(");");

src/runtime/node.c

@@ -39,7 +39,15 @@ static inline bool ts_node__is_relevant(TSNode self, bool include_anonymous) {
 static inline uint32_t ts_node__relevant_child_count(TSNode self,
                                                    bool include_anonymous) {
   const Tree *tree = ts_node__tree(self);
-  return include_anonymous ? tree->visible_child_count : tree->named_child_count;
+  if (tree->child_count > 0) {
+    if (include_anonymous) {
+      return tree->visible_child_count;
+    } else {
+      return tree->named_child_count;
+    }
+  } else {
+    return 0;
+  }
 }
 
 static inline TSNode ts_node__direct_parent(TSNode self, uint32_t *index) {
@@ -324,11 +332,21 @@ TSNode ts_node_named_child(TSNode self, uint32_t child_index) {
 }
 
 uint32_t ts_node_child_count(TSNode self) {
-  return ts_node__tree(self)->visible_child_count;
+  const Tree *tree = ts_node__tree(self);
+  if (tree->child_count > 0) {
+    return tree->visible_child_count;
+  } else {
+    return 0;
+  }
 }
 
 uint32_t ts_node_named_child_count(TSNode self) {
-  return ts_node__tree(self)->named_child_count;
+  const Tree *tree = ts_node__tree(self);
+  if (tree->child_count > 0) {
+    return tree->named_child_count;
+  } else {
+    return 0;
+  }
 }
 
 TSNode ts_node_next_sibling(TSNode self) {

src/runtime/parser.c

@@ -752,8 +752,9 @@ static void parser__start(Parser *self, TSInput input, Tree *previous_tree) {
     LOG("new_parse");
   }
 
-  if (self->language->external_scanner.create)
-    self->language->external_scanner.create();
+  if (self->language->external_scanner.reset) {
+    self->language->external_scanner.reset(self->external_scanner_payload);
+  }
 
   ts_lexer_set_input(&self->lexer, input);
   ts_stack_clear(self->stack);

src/runtime/tree.h

@@ -22,10 +22,13 @@ typedef struct Tree {
   } context;
 
   uint32_t child_count;
-  uint32_t visible_child_count;
-  uint32_t named_child_count;
   union {
-    struct Tree **children;
+    struct {
+      uint32_t visible_child_count;
+      uint32_t named_child_count;
+      struct Tree **children;
+    };
+    TSExternalTokenState external_token_state;
     int32_t lookahead_char;
   };