traxys/tree-sitter
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Rename TSExternalTokenState -> ExternalScannerState

Browse source
This commit is contained in:
Max Brunsfeld 2018-05-11 12:57:41 -07:00
parent f0c7295d27
commit bf1bb1604f
6 changed files with 45 additions and 42 deletions
Download patch file Download diff file Expand all files Collapse all files

8
include/tree_sitter/parser.h
View file

@ -9,13 +9,14 @@ extern "C" {
#include <stdint.h>
#include <stdlib.h>
typedef uint16_t TSSymbol;
typedef uint16_t TSStateId;
#define ts_builtin_sym_error ((TSSymbol)-1)
#define ts_builtin_sym_end 0
#define TREE_SITTER_SERIALIZATION_BUFFER_SIZE 1024
typedef uint16_t TSSymbol;
typedef uint16_t TSStateId;
typedef struct {
bool visible : 1;
bool named : 1;
@ -129,6 +130,7 @@ typedef struct TSLanguage {
*/
#define STATE(id) id
#define ACTIONS(id) id
#define SHIFT(state_value) \

10
src/runtime/parser.c
View file

@ -247,8 +247,8 @@ static void ts_parser__restore_external_scanner(TSParser *self, Subtree *externa
if (external_token) {
self->language->external_scanner.deserialize(
self->external_scanner_payload,
ts_external_token_state_data(&external_token->external_token_state),
external_token->external_token_state.length
ts_external_scanner_state_data(&external_token->external_scanner_state),
external_token->external_scanner_state.length
);
} else {
self->language->external_scanner.deserialize(self->external_scanner_payload, NULL, 0);
@ -391,7 +391,7 @@ static Subtree *ts_parser__lex(TSParser *self, StackVersion version, TSStateId p
self->external_scanner_payload,
self->lexer.debug_buffer
);
ts_external_token_state_init(&result->external_token_state, self->lexer.debug_buffer, length);
ts_external_scanner_state_init(&result->external_scanner_state, self->lexer.debug_buffer, length);
}
}
@ -408,7 +408,7 @@ static Subtree *ts_parser__get_cached_token(TSParser *self, size_t byte_index,
TokenCache *cache = &self->token_cache;
if (cache->token &&
cache->byte_index == byte_index &&
ts_subtree_external_token_state_eq(cache->last_external_token, last_external_token)) {
ts_subtree_external_scanner_state_eq(cache->last_external_token, last_external_token)) {
return cache->token;
} else {
return NULL;
@ -464,7 +464,7 @@ static Subtree *ts_parser__get_lookahead(TSParser *self, StackVersion version, T
continue;
}
if (!ts_subtree_external_token_state_eq(reusable_node->last_external_token, last_external_token)) {
if (!ts_subtree_external_scanner_state_eq(reusable_node->last_external_token, last_external_token)) {
LOG("reusable_node_has_different_external_scanner_state symbol:%s", SYM_NAME(result->symbol));
reusable_node_advance(reusable_node);
continue;

10
src/runtime/stack.c
View file

@ -164,7 +164,7 @@ static bool stack__subtree_is_equivalent(const Subtree *left, const Subtree *rig
left->padding.bytes == right->padding.bytes &&
left->size.bytes == right->size.bytes &&
left->extra == right->extra &&
ts_subtree_external_token_state_eq(left, right))));
ts_subtree_external_scanner_state_eq(left, right))));
}
static void stack_node_add_link(StackNode *self, StackLink link) {
@ -605,7 +605,7 @@ bool ts_stack_can_merge(Stack *self, StackVersion version1, StackVersion version
head1->node->state == head2->node->state &&
head1->node->position.bytes == head2->node->position.bytes &&
head1->node->error_cost == head2->node->error_cost &&
ts_subtree_external_token_state_eq(head1->last_external_token, head2->last_external_token);
ts_subtree_external_scanner_state_eq(head1->last_external_token, head2->last_external_token);
}
void ts_stack_halt(Stack *self, StackVersion version) {
@ -684,9 +684,9 @@ bool ts_stack_print_dot_graph(Stack *self, const TSLanguage *language, FILE *f)
);
if (head->last_external_token) {
TSExternalTokenState *state = &head->last_external_token->external_token_state;
const char *data = ts_external_token_state_data(state);
fprintf(f, "\nexternal_token_state:");
ExternalScannerState *state = &head->last_external_token->external_scanner_state;
const char *data = ts_external_scanner_state_data(state);
fprintf(f, "\nexternal_scanner_state:");
for (uint32_t j = 0; j < state->length; j++) fprintf(f, " %2X", data[j]);
}

37
src/runtime/subtree.c
View file

@ -21,27 +21,27 @@ TSStateId TS_TREE_STATE_NONE = USHRT_MAX;
static const uint32_t MAX_TREE_POOL_SIZE = 1024;
static const TSExternalTokenState empty_state = {.length = 0, .short_data = {0}};
static const ExternalScannerState empty_state = {.length = 0, .short_data = {0}};
// ExternalTokenState
// ExternalScannerState
void ts_external_token_state_init(TSExternalTokenState *self, const char *content, unsigned length) {
void ts_external_scanner_state_init(ExternalScannerState *self, const char *data, unsigned length) {
self->length = length;
if (length > sizeof(self->short_data)) {
self->long_data = ts_malloc(length);
memcpy(self->long_data, content, length);
memcpy(self->long_data, data, length);
} else {
memcpy(self->short_data, content, length);
memcpy(self->short_data, data, length);
}
}
void ts_external_token_state_delete(TSExternalTokenState *self) {
void ts_external_scanner_state_delete(ExternalScannerState *self) {
if (self->length > sizeof(self->short_data)) {
ts_free(self->long_data);
}
}
const char *ts_external_token_state_data(const TSExternalTokenState *self) {
const char *ts_external_scanner_state_data(const ExternalScannerState *self) {
if (self->length > sizeof(self->short_data)) {
return self->long_data;
} else {
@ -49,10 +49,11 @@ const char *ts_external_token_state_data(const TSExternalTokenState *self) {
}
}
bool ts_external_token_state_eq(const TSExternalTokenState *a, const TSExternalTokenState *b) {
return a == b ||
(a->length == b->length &&
memcmp(ts_external_token_state_data(a), ts_external_token_state_data(b), a->length) == 0);
bool ts_external_scanner_state_eq(const ExternalScannerState *a, const ExternalScannerState *b) {
return a == b || (
a->length == b->length &&
!memcmp(ts_external_scanner_state_data(a), ts_external_scanner_state_data(b), a->length)
);
}
// SubtreeArray
@ -403,7 +404,7 @@ void ts_subtree_release(SubtreePool *pool, Subtree *self) {
}
array_delete(&tree->children);
} else if (tree->has_external_tokens) {
ts_external_token_state_delete(&tree->external_token_state);
ts_external_scanner_state_delete(&tree->external_scanner_state);
}
ts_subtree_pool_free(pool, tree);
}
@ -694,10 +695,10 @@ void ts_subtree_print_dot_graph(const Subtree *self, const TSLanguage *language,
fprintf(f, "}\n");
}
bool ts_subtree_external_token_state_eq(const Subtree *self, const Subtree *other) {
const TSExternalTokenState *state1 = &empty_state;
const TSExternalTokenState *state2 = &empty_state;
if (self && self->has_external_tokens) state1 = &self->external_token_state;
if (other && other->has_external_tokens) state2 = &other->external_token_state;
return ts_external_token_state_eq(state1, state2);
bool ts_subtree_external_scanner_state_eq(const Subtree *self, const Subtree *other) {
const ExternalScannerState *state1 = &empty_state;
const ExternalScannerState *state2 = &empty_state;
if (self && self->has_external_tokens) state1 = &self->external_scanner_state;
if (other && other->has_external_tokens) state2 = &other->external_scanner_state;
return ts_external_scanner_state_eq(state1, state2);
}

10
src/runtime/subtree.h
View file

@ -20,7 +20,7 @@ typedef struct {
char short_data[sizeof(char *) + sizeof(uint32_t)];
};
uint32_t length;
} TSExternalTokenState;
} ExternalScannerState;
typedef struct Subtree Subtree;
@ -61,7 +61,7 @@ struct Subtree {
};
struct {
uint32_t _2;
TSExternalTokenState external_token_state;
ExternalScannerState external_scanner_state;
};
struct {
uint32_t _1;
@ -75,8 +75,8 @@ typedef struct {
SubtreeArray tree_stack;
} SubtreePool;
void ts_external_token_state_init(TSExternalTokenState *, const char *, unsigned);
const char *ts_external_token_state_data(const TSExternalTokenState *);
void ts_external_scanner_state_init(ExternalScannerState *, const char *, unsigned);
const char *ts_external_scanner_state_data(const ExternalScannerState *);
bool ts_subtree_array_copy(SubtreeArray, SubtreeArray *);
void ts_subtree_array_delete(SubtreePool *, SubtreeArray *);
@ -104,7 +104,7 @@ Subtree *ts_subtree_edit(Subtree *, const TSInputEdit *edit, SubtreePool *);
char *ts_subtree_string(const Subtree *, const TSLanguage *, bool include_all);
void ts_subtree_print_dot_graph(const Subtree *, const TSLanguage *, FILE *);
Subtree *ts_subtree_last_external_token(Subtree *);
bool ts_subtree_external_token_state_eq(const Subtree *, const Subtree *);
bool ts_subtree_external_scanner_state_eq(const Subtree *, const Subtree *);
static inline uint32_t ts_subtree_total_bytes(const Subtree *self) {
return self->padding.bytes + self->size.bytes;

12
test/runtime/stack_test.cc
View file

@ -528,8 +528,8 @@ describe("Stack", [&]() {
before_each([&]() {
subtrees[1]->has_external_tokens = true;
subtrees[2]->has_external_tokens = true;
ts_external_token_state_init(&subtrees[1]->external_token_state, NULL, 0);
ts_external_token_state_init(&subtrees[2]->external_token_state, NULL, 0);
ts_external_scanner_state_init(&subtrees[1]->external_scanner_state, NULL, 0);
ts_external_scanner_state_init(&subtrees[2]->external_scanner_state, NULL, 0);
});
it("allows the state to be retrieved", [&]() {
@ -546,8 +546,8 @@ describe("Stack", [&]() {
});
it("does not merge stack versions with different external token states", [&]() {
ts_external_token_state_init(&subtrees[1]->external_token_state, "abcd", 2);
ts_external_token_state_init(&subtrees[2]->external_token_state, "ABCD", 2);
ts_external_scanner_state_init(&subtrees[1]->external_scanner_state, "abcd", 2);
ts_external_scanner_state_init(&subtrees[2]->external_scanner_state, "ABCD", 2);
ts_stack_copy_version(stack, 0);
push(0, subtrees[0], 5);
@ -560,8 +560,8 @@ describe("Stack", [&]() {
});
it("merges stack versions with identical external token states", [&]() {
ts_external_token_state_init(&subtrees[1]->external_token_state, "abcd", 2);
ts_external_token_state_init(&subtrees[2]->external_token_state, "abcd", 2);
ts_external_scanner_state_init(&subtrees[1]->external_scanner_state, "abcd", 2);
ts_external_scanner_state_init(&subtrees[2]->external_scanner_state, "abcd", 2);
ts_stack_copy_version(stack, 0);
push(0, subtrees[0], 5);