diff --git a/include/tree_sitter/parser.h b/include/tree_sitter/parser.h index d99655f8..35e77a87 100644 --- a/include/tree_sitter/parser.h +++ b/include/tree_sitter/parser.h @@ -9,13 +9,14 @@ extern "C" { #include #include -typedef uint16_t TSSymbol; -typedef uint16_t TSStateId; - #define ts_builtin_sym_error ((TSSymbol)-1) #define ts_builtin_sym_end 0 #define TREE_SITTER_SERIALIZATION_BUFFER_SIZE 1024 +typedef uint16_t TSSymbol; + +typedef uint16_t TSStateId; + typedef struct { bool visible : 1; bool named : 1; @@ -129,6 +130,7 @@ typedef struct TSLanguage { */ #define STATE(id) id + #define ACTIONS(id) id #define SHIFT(state_value) \ diff --git a/src/runtime/parser.c b/src/runtime/parser.c index 02e96857..2115435e 100644 --- a/src/runtime/parser.c +++ b/src/runtime/parser.c @@ -247,8 +247,8 @@ static void ts_parser__restore_external_scanner(TSParser *self, Subtree *externa if (external_token) { self->language->external_scanner.deserialize( self->external_scanner_payload, - ts_external_token_state_data(&external_token->external_token_state), - external_token->external_token_state.length + ts_external_scanner_state_data(&external_token->external_scanner_state), + external_token->external_scanner_state.length ); } else { self->language->external_scanner.deserialize(self->external_scanner_payload, NULL, 0); @@ -391,7 +391,7 @@ static Subtree *ts_parser__lex(TSParser *self, StackVersion version, TSStateId p self->external_scanner_payload, self->lexer.debug_buffer ); - ts_external_token_state_init(&result->external_token_state, self->lexer.debug_buffer, length); + ts_external_scanner_state_init(&result->external_scanner_state, self->lexer.debug_buffer, length); } } @@ -408,7 +408,7 @@ static Subtree *ts_parser__get_cached_token(TSParser *self, size_t byte_index, TokenCache *cache = &self->token_cache; if (cache->token && cache->byte_index == byte_index && - ts_subtree_external_token_state_eq(cache->last_external_token, last_external_token)) { + ts_subtree_external_scanner_state_eq(cache->last_external_token, last_external_token)) { return cache->token; } else { return NULL; @@ -464,7 +464,7 @@ static Subtree *ts_parser__get_lookahead(TSParser *self, StackVersion version, T continue; } - if (!ts_subtree_external_token_state_eq(reusable_node->last_external_token, last_external_token)) { + if (!ts_subtree_external_scanner_state_eq(reusable_node->last_external_token, last_external_token)) { LOG("reusable_node_has_different_external_scanner_state symbol:%s", SYM_NAME(result->symbol)); reusable_node_advance(reusable_node); continue; diff --git a/src/runtime/stack.c b/src/runtime/stack.c index c8af7a58..21468e49 100644 --- a/src/runtime/stack.c +++ b/src/runtime/stack.c @@ -164,7 +164,7 @@ static bool stack__subtree_is_equivalent(const Subtree *left, const Subtree *rig left->padding.bytes == right->padding.bytes && left->size.bytes == right->size.bytes && left->extra == right->extra && - ts_subtree_external_token_state_eq(left, right)))); + ts_subtree_external_scanner_state_eq(left, right)))); } static void stack_node_add_link(StackNode *self, StackLink link) { @@ -605,7 +605,7 @@ bool ts_stack_can_merge(Stack *self, StackVersion version1, StackVersion version head1->node->state == head2->node->state && head1->node->position.bytes == head2->node->position.bytes && head1->node->error_cost == head2->node->error_cost && - ts_subtree_external_token_state_eq(head1->last_external_token, head2->last_external_token); + ts_subtree_external_scanner_state_eq(head1->last_external_token, head2->last_external_token); } void ts_stack_halt(Stack *self, StackVersion version) { @@ -684,9 +684,9 @@ bool ts_stack_print_dot_graph(Stack *self, const TSLanguage *language, FILE *f) ); if (head->last_external_token) { - TSExternalTokenState *state = &head->last_external_token->external_token_state; - const char *data = ts_external_token_state_data(state); - fprintf(f, "\nexternal_token_state:"); + ExternalScannerState *state = &head->last_external_token->external_scanner_state; + const char *data = ts_external_scanner_state_data(state); + fprintf(f, "\nexternal_scanner_state:"); for (uint32_t j = 0; j < state->length; j++) fprintf(f, " %2X", data[j]); } diff --git a/src/runtime/subtree.c b/src/runtime/subtree.c index aff2a6fc..c3da95c5 100644 --- a/src/runtime/subtree.c +++ b/src/runtime/subtree.c @@ -21,27 +21,27 @@ TSStateId TS_TREE_STATE_NONE = USHRT_MAX; static const uint32_t MAX_TREE_POOL_SIZE = 1024; -static const TSExternalTokenState empty_state = {.length = 0, .short_data = {0}}; +static const ExternalScannerState empty_state = {.length = 0, .short_data = {0}}; -// ExternalTokenState +// ExternalScannerState -void ts_external_token_state_init(TSExternalTokenState *self, const char *content, unsigned length) { +void ts_external_scanner_state_init(ExternalScannerState *self, const char *data, unsigned length) { self->length = length; if (length > sizeof(self->short_data)) { self->long_data = ts_malloc(length); - memcpy(self->long_data, content, length); + memcpy(self->long_data, data, length); } else { - memcpy(self->short_data, content, length); + memcpy(self->short_data, data, length); } } -void ts_external_token_state_delete(TSExternalTokenState *self) { +void ts_external_scanner_state_delete(ExternalScannerState *self) { if (self->length > sizeof(self->short_data)) { ts_free(self->long_data); } } -const char *ts_external_token_state_data(const TSExternalTokenState *self) { +const char *ts_external_scanner_state_data(const ExternalScannerState *self) { if (self->length > sizeof(self->short_data)) { return self->long_data; } else { @@ -49,10 +49,11 @@ const char *ts_external_token_state_data(const TSExternalTokenState *self) { } } -bool ts_external_token_state_eq(const TSExternalTokenState *a, const TSExternalTokenState *b) { - return a == b || - (a->length == b->length && - memcmp(ts_external_token_state_data(a), ts_external_token_state_data(b), a->length) == 0); +bool ts_external_scanner_state_eq(const ExternalScannerState *a, const ExternalScannerState *b) { + return a == b || ( + a->length == b->length && + !memcmp(ts_external_scanner_state_data(a), ts_external_scanner_state_data(b), a->length) + ); } // SubtreeArray @@ -403,7 +404,7 @@ void ts_subtree_release(SubtreePool *pool, Subtree *self) { } array_delete(&tree->children); } else if (tree->has_external_tokens) { - ts_external_token_state_delete(&tree->external_token_state); + ts_external_scanner_state_delete(&tree->external_scanner_state); } ts_subtree_pool_free(pool, tree); } @@ -694,10 +695,10 @@ void ts_subtree_print_dot_graph(const Subtree *self, const TSLanguage *language, fprintf(f, "}\n"); } -bool ts_subtree_external_token_state_eq(const Subtree *self, const Subtree *other) { - const TSExternalTokenState *state1 = &empty_state; - const TSExternalTokenState *state2 = &empty_state; - if (self && self->has_external_tokens) state1 = &self->external_token_state; - if (other && other->has_external_tokens) state2 = &other->external_token_state; - return ts_external_token_state_eq(state1, state2); +bool ts_subtree_external_scanner_state_eq(const Subtree *self, const Subtree *other) { + const ExternalScannerState *state1 = &empty_state; + const ExternalScannerState *state2 = &empty_state; + if (self && self->has_external_tokens) state1 = &self->external_scanner_state; + if (other && other->has_external_tokens) state2 = &other->external_scanner_state; + return ts_external_scanner_state_eq(state1, state2); } diff --git a/src/runtime/subtree.h b/src/runtime/subtree.h index fd1fcaff..2f7d60b0 100644 --- a/src/runtime/subtree.h +++ b/src/runtime/subtree.h @@ -20,7 +20,7 @@ typedef struct { char short_data[sizeof(char *) + sizeof(uint32_t)]; }; uint32_t length; -} TSExternalTokenState; +} ExternalScannerState; typedef struct Subtree Subtree; @@ -61,7 +61,7 @@ struct Subtree { }; struct { uint32_t _2; - TSExternalTokenState external_token_state; + ExternalScannerState external_scanner_state; }; struct { uint32_t _1; @@ -75,8 +75,8 @@ typedef struct { SubtreeArray tree_stack; } SubtreePool; -void ts_external_token_state_init(TSExternalTokenState *, const char *, unsigned); -const char *ts_external_token_state_data(const TSExternalTokenState *); +void ts_external_scanner_state_init(ExternalScannerState *, const char *, unsigned); +const char *ts_external_scanner_state_data(const ExternalScannerState *); bool ts_subtree_array_copy(SubtreeArray, SubtreeArray *); void ts_subtree_array_delete(SubtreePool *, SubtreeArray *); @@ -104,7 +104,7 @@ Subtree *ts_subtree_edit(Subtree *, const TSInputEdit *edit, SubtreePool *); char *ts_subtree_string(const Subtree *, const TSLanguage *, bool include_all); void ts_subtree_print_dot_graph(const Subtree *, const TSLanguage *, FILE *); Subtree *ts_subtree_last_external_token(Subtree *); -bool ts_subtree_external_token_state_eq(const Subtree *, const Subtree *); +bool ts_subtree_external_scanner_state_eq(const Subtree *, const Subtree *); static inline uint32_t ts_subtree_total_bytes(const Subtree *self) { return self->padding.bytes + self->size.bytes; diff --git a/test/runtime/stack_test.cc b/test/runtime/stack_test.cc index ae5b1589..a6ec09c4 100644 --- a/test/runtime/stack_test.cc +++ b/test/runtime/stack_test.cc @@ -528,8 +528,8 @@ describe("Stack", [&]() { before_each([&]() { subtrees[1]->has_external_tokens = true; subtrees[2]->has_external_tokens = true; - ts_external_token_state_init(&subtrees[1]->external_token_state, NULL, 0); - ts_external_token_state_init(&subtrees[2]->external_token_state, NULL, 0); + ts_external_scanner_state_init(&subtrees[1]->external_scanner_state, NULL, 0); + ts_external_scanner_state_init(&subtrees[2]->external_scanner_state, NULL, 0); }); it("allows the state to be retrieved", [&]() { @@ -546,8 +546,8 @@ describe("Stack", [&]() { }); it("does not merge stack versions with different external token states", [&]() { - ts_external_token_state_init(&subtrees[1]->external_token_state, "abcd", 2); - ts_external_token_state_init(&subtrees[2]->external_token_state, "ABCD", 2); + ts_external_scanner_state_init(&subtrees[1]->external_scanner_state, "abcd", 2); + ts_external_scanner_state_init(&subtrees[2]->external_scanner_state, "ABCD", 2); ts_stack_copy_version(stack, 0); push(0, subtrees[0], 5); @@ -560,8 +560,8 @@ describe("Stack", [&]() { }); it("merges stack versions with identical external token states", [&]() { - ts_external_token_state_init(&subtrees[1]->external_token_state, "abcd", 2); - ts_external_token_state_init(&subtrees[2]->external_token_state, "abcd", 2); + ts_external_scanner_state_init(&subtrees[1]->external_scanner_state, "abcd", 2); + ts_external_scanner_state_init(&subtrees[2]->external_scanner_state, "abcd", 2); ts_stack_copy_version(stack, 0); push(0, subtrees[0], 5);