diff --git a/include/tree_sitter/parser.h b/include/tree_sitter/parser.h index 27113e32..90247719 100644 --- a/include/tree_sitter/parser.h +++ b/include/tree_sitter/parser.h @@ -12,6 +12,8 @@ extern "C" { typedef unsigned short TSSymbol; typedef unsigned short TSStateId; +typedef uint8_t TSExternalTokenState[16]; + #define ts_builtin_sym_error ((TSSymbol)-1) #define ts_builtin_sym_end 0 @@ -75,8 +77,11 @@ typedef struct TSLanguage { const TSSymbol *external_token_symbol_map; const bool *external_token_lists; struct { - void * (*create)(); + void *(*create)(); bool (*scan)(void *, TSLexer *, const bool *symbol_whitelist); + void (*reset)(void *); + bool (*serialize)(void *, TSExternalTokenState); + void (*deserialize)(void *, TSExternalTokenState); void (*destroy)(void *); } external_scanner; } TSLanguage; diff --git a/spec/fixtures/external_scanners/extra_external_tokens.c b/spec/fixtures/external_scanners/extra_external_tokens.c index 45803213..ba3338af 100644 --- a/spec/fixtures/external_scanners/extra_external_tokens.c +++ b/spec/fixtures/external_scanners/extra_external_tokens.c @@ -8,7 +8,14 @@ void *ts_language_extra_external_tokens_external_scanner_create() { return NULL; } -void ts_language_extra_external_tokens_external_scanner_destroy(void *payload) { +void ts_language_extra_external_tokens_external_scanner_reset(void *payload) { +} + +bool ts_language_extra_external_tokens_external_scanner_serialize(void *payload, TSExternalTokenState state) { + return true; +} + +void ts_language_extra_external_tokens_external_scanner_deserialize(void *payload, TSExternalTokenState state) { } bool ts_language_extra_external_tokens_external_scanner_scan( @@ -30,3 +37,6 @@ bool ts_language_extra_external_tokens_external_scanner_scan( return false; } + +void ts_language_extra_external_tokens_external_scanner_destroy(void *payload) { +} diff --git a/spec/fixtures/external_scanners/percent_strings.c b/spec/fixtures/external_scanners/percent_strings.c index 9e7a7d12..56c12e81 100644 --- a/spec/fixtures/external_scanners/percent_strings.c +++ b/spec/fixtures/external_scanners/percent_strings.c @@ -103,6 +103,16 @@ bool ts_language_external_scanner_example_external_scanner_scan( return false; } +void ts_language_external_scanner_example_external_scanner_reset(void *payload) { +} + +bool ts_language_external_scanner_example_external_scanner_serialize(void *payload, TSExternalTokenState state) { + return true; +} + +void ts_language_external_scanner_example_external_scanner_deserialize(void *payload, TSExternalTokenState state) { +} + void ts_language_external_scanner_example_external_scanner_destroy(void *payload) { free(payload); } diff --git a/spec/fixtures/external_scanners/shared_external_tokens.c b/spec/fixtures/external_scanners/shared_external_tokens.c index eb63a37f..3be1a848 100644 --- a/spec/fixtures/external_scanners/shared_external_tokens.c +++ b/spec/fixtures/external_scanners/shared_external_tokens.c @@ -10,7 +10,14 @@ void *ts_language_shared_external_tokens_external_scanner_create() { return NULL; } -void ts_language_shared_external_tokens_external_scanner_destroy(void *payload) { +void ts_language_shared_external_tokens_external_scanner_reset(void *payload) { +} + +bool ts_language_shared_external_tokens_external_scanner_serialize(void *payload, TSExternalTokenState state) { + return true; +} + +void ts_language_shared_external_tokens_external_scanner_deserialize(void *payload, TSExternalTokenState state) { } bool ts_language_shared_external_tokens_external_scanner_scan( @@ -51,3 +58,6 @@ bool ts_language_shared_external_tokens_external_scanner_scan( return false; } + +void ts_language_shared_external_tokens_external_scanner_destroy(void *payload) { +} diff --git a/src/compiler/generate_code/c_code.cc b/src/compiler/generate_code/c_code.cc index 5df39413..ed034c1b 100644 --- a/src/compiler/generate_code/c_code.cc +++ b/src/compiler/generate_code/c_code.cc @@ -356,6 +356,9 @@ class CCodeGenerator { line("void *" + external_scanner_name + "_create();"); line("bool " + external_scanner_name + "_scan(void *, TSLexer *, const bool *);"); + line("void " + external_scanner_name + "_reset(void *);"); + line("bool " + external_scanner_name + "_serialize(void *, TSExternalTokenState);"); + line("void " + external_scanner_name + "_deserialize(void *, TSExternalTokenState);"); line("void " + external_scanner_name + "_destroy();"); line(); @@ -366,6 +369,9 @@ class CCodeGenerator { indent([&]() { line(external_scanner_name + "_create,"); line(external_scanner_name + "_scan,"); + line(external_scanner_name + "_reset,"); + line(external_scanner_name + "_serialize,"); + line(external_scanner_name + "_deserialize,"); line(external_scanner_name + "_destroy,"); }); line(");"); diff --git a/src/runtime/node.c b/src/runtime/node.c index d5bcb1a0..15e2d5cf 100644 --- a/src/runtime/node.c +++ b/src/runtime/node.c @@ -39,7 +39,15 @@ static inline bool ts_node__is_relevant(TSNode self, bool include_anonymous) { static inline uint32_t ts_node__relevant_child_count(TSNode self, bool include_anonymous) { const Tree *tree = ts_node__tree(self); - return include_anonymous ? tree->visible_child_count : tree->named_child_count; + if (tree->child_count > 0) { + if (include_anonymous) { + return tree->visible_child_count; + } else { + return tree->named_child_count; + } + } else { + return 0; + } } static inline TSNode ts_node__direct_parent(TSNode self, uint32_t *index) { @@ -324,11 +332,21 @@ TSNode ts_node_named_child(TSNode self, uint32_t child_index) { } uint32_t ts_node_child_count(TSNode self) { - return ts_node__tree(self)->visible_child_count; + const Tree *tree = ts_node__tree(self); + if (tree->child_count > 0) { + return tree->visible_child_count; + } else { + return 0; + } } uint32_t ts_node_named_child_count(TSNode self) { - return ts_node__tree(self)->named_child_count; + const Tree *tree = ts_node__tree(self); + if (tree->child_count > 0) { + return tree->named_child_count; + } else { + return 0; + } } TSNode ts_node_next_sibling(TSNode self) { diff --git a/src/runtime/parser.c b/src/runtime/parser.c index be827250..e81b73a6 100644 --- a/src/runtime/parser.c +++ b/src/runtime/parser.c @@ -752,8 +752,9 @@ static void parser__start(Parser *self, TSInput input, Tree *previous_tree) { LOG("new_parse"); } - if (self->language->external_scanner.create) - self->language->external_scanner.create(); + if (self->language->external_scanner.reset) { + self->language->external_scanner.reset(self->external_scanner_payload); + } ts_lexer_set_input(&self->lexer, input); ts_stack_clear(self->stack); diff --git a/src/runtime/tree.h b/src/runtime/tree.h index c37d61ab..7aea708f 100644 --- a/src/runtime/tree.h +++ b/src/runtime/tree.h @@ -22,10 +22,13 @@ typedef struct Tree { } context; uint32_t child_count; - uint32_t visible_child_count; - uint32_t named_child_count; union { - struct Tree **children; + struct { + uint32_t visible_child_count; + uint32_t named_child_count; + struct Tree **children; + }; + TSExternalTokenState external_token_state; int32_t lookahead_char; };