traxys/tree-sitter
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Take more parameters in ts_subtree_new_leaf

Browse source
This commit is contained in:
Max Brunsfeld 2018-09-15 00:08:47 -07:00
parent c828ca308f
commit e00c3bbdb9
5 changed files with 104 additions and 68 deletions
Download patch file Download diff file Expand all files Collapse all files

39
src/runtime/parser.c
View file

@ -402,11 +402,15 @@ static const Subtree *ts_parser__lex(TSParser *self, StackVersion version, TSSta
last_byte_scanned = self->lexer.current_position.bytes;
}
uint32_t bytes_scanned = last_byte_scanned - start_position.bytes + 1;
Subtree *result;
if (skipped_error) {
Length padding = length_sub(error_start_position, start_position);
Length size = length_sub(error_end_position, error_start_position);
result = ts_subtree_new_error(&self->tree_pool, size, padding, first_error_character, self->language);
result->parse_state = parse_state;
result->bytes_scanned = bytes_scanned;
} else {
if (self->lexer.token_end_position.bytes < self->lexer.token_start_position.bytes) {
self->lexer.token_start_position = self->lexer.token_end_position;
@ -433,22 +437,31 @@ static const Subtree *ts_parser__lex(TSParser *self, StackVersion version, TSSta
}
}
result = ts_subtree_new_leaf(&self->tree_pool, symbol, padding, size, !found_external_token, self->language);
result->is_keyword = is_keyword;
result = ts_subtree_new_leaf(
&self->tree_pool,
symbol,
padding,
size,
bytes_scanned,
parse_state,
found_external_token,
is_keyword,
self->language
);
if (found_external_token) {
result->has_external_tokens = true;
unsigned length = self->language->external_scanner.serialize(
self->external_scanner_payload,
self->lexer.debug_buffer
);
ts_external_scanner_state_init(&result->external_scanner_state, self->lexer.debug_buffer, length);
ts_external_scanner_state_init(
&result->external_scanner_state,
self->lexer.debug_buffer,
length
);
}
}
result->bytes_scanned = last_byte_scanned - start_position.bytes + 1;
result->parse_state = parse_state;
LOG("lexed_lookahead sym:%s, size:%u", SYM_NAME(result->symbol), result->size.bytes);
return result;
}
@ -948,7 +961,17 @@ static void ts_parser__halt_parse(TSParser *self) {
Subtree *root_error = ts_subtree_new_error_node(&self->tree_pool, &children, self->language);
ts_stack_push(self->stack, 0, root_error, false, 0);
Subtree *eof = ts_subtree_new_leaf(&self->tree_pool, ts_builtin_sym_end, length_zero(), length_zero(), true, self->language);
Subtree *eof = ts_subtree_new_leaf(
&self->tree_pool,
ts_builtin_sym_end,
length_zero(),
length_zero(),
0,
0,
false,
false,
self->language
);
ts_parser__accept(self, 0, eof);
}

46
src/runtime/subtree.c
View file

@ -162,7 +162,35 @@ void ts_subtree_pool_free(SubtreePool *self, Subtree *tree) {
// Subtree
Subtree *ts_subtree_new_leaf(SubtreePool *pool, TSSymbol symbol, Length padding, Length size,
Subtree *ts_subtree_new_leaf(
SubtreePool *pool, TSSymbol symbol, Length padding, Length size, uint32_t bytes_scanned,
TSStateId parse_state, bool has_external_tokens, bool is_keyword, const TSLanguage *language
) {
TSSymbolMetadata metadata = ts_language_symbol_metadata(language, symbol);
bool is_small = !has_external_tokens;
Subtree *result = ts_subtree_pool_allocate(pool, is_small);
result->ref_count = 1;
result->padding = padding;
result->size = size;
result->bytes_scanned = bytes_scanned;
result->error_cost = 0;
result->child_count = 0;
result->symbol = symbol;
result->parse_state = parse_state;
result->is_small = is_small;
result->visible = metadata.visible;
result->named = metadata.named;
result->extra = symbol == ts_builtin_sym_end;
result->fragile_left = false;
result->fragile_right = false;
result->has_changes = false;
result->has_external_tokens = has_external_tokens;
result->is_missing = false;
result->is_keyword = is_keyword;
return result;
}
Subtree *ts_subtree__new(SubtreePool *pool, TSSymbol symbol, Length padding, Length size,
bool is_small, const TSLanguage *language) {
TSSymbolMetadata metadata = ts_language_symbol_metadata(language, symbol);
Subtree *result = ts_subtree_pool_allocate(pool, is_small);
@ -189,7 +217,7 @@ Subtree *ts_subtree_new_leaf(SubtreePool *pool, TSSymbol symbol, Length padding,
Subtree *ts_subtree_new_error(SubtreePool *pool, Length size, Length padding,
int32_t lookahead_char, const TSLanguage *language) {
Subtree *result = ts_subtree_new_leaf(pool, ts_builtin_sym_error, padding, size, false, language);
Subtree *result = ts_subtree__new(pool, ts_builtin_sym_error, padding, size, false, language);
result->fragile_left = true;
result->fragile_right = true;
result->lookahead_char = lookahead_char;
@ -434,7 +462,7 @@ Subtree *ts_subtree_new_error_node(SubtreePool *pool, SubtreeArray *children,
Subtree *ts_subtree_new_missing_leaf(SubtreePool *pool, TSSymbol symbol, Length padding,
const TSLanguage *language) {
Subtree *result = ts_subtree_new_leaf(pool, symbol, padding, length_zero(), true, language);
Subtree *result = ts_subtree__new(pool, symbol, padding, length_zero(), true, language);
result->is_missing = true;
result->error_cost = ERROR_COST_PER_MISSING_TREE + ERROR_COST_PER_RECOVERY;
return result;
@ -485,12 +513,14 @@ bool ts_subtree_eq(const Subtree *self, const Subtree *other) {
if (self->size.bytes != other->size.bytes) return false;
if (self->symbol == ts_builtin_sym_error) return self->lookahead_char == other->lookahead_char;
if (self->child_count != other->child_count) return false;
if (self->visible_child_count != other->visible_child_count) return false;
if (self->named_child_count != other->named_child_count) return false;
if (self->child_count > 0) {
if (self->visible_child_count != other->visible_child_count) return false;
if (self->named_child_count != other->named_child_count) return false;
for (uint32_t i = 0; i < self->child_count; i++) {
if (!ts_subtree_eq(self->children[i], other->children[i])) {
return false;
for (uint32_t i = 0; i < self->child_count; i++) {
if (!ts_subtree_eq(self->children[i], other->children[i])) {
return false;
}
}
}
return true;

5
src/runtime/subtree.h
View file

@ -91,7 +91,10 @@ void ts_subtree_pool_delete(SubtreePool *);
Subtree *ts_subtree_pool_allocate(SubtreePool *, bool);
void ts_subtree_pool_free(SubtreePool *, Subtree *);
Subtree *ts_subtree_new_leaf(SubtreePool *, TSSymbol, Length, Length, bool, const TSLanguage *);
Subtree *ts_subtree_new_leaf(
SubtreePool *, TSSymbol, Length, Length, uint32_t,
TSStateId, bool, bool, const TSLanguage *
);
Subtree *ts_subtree_new_node(SubtreePool *, TSSymbol, SubtreeArray *, unsigned, const TSLanguage *);
Subtree *ts_subtree_new_copy(SubtreePool *, const Subtree *);
Subtree *ts_subtree_new_error_node(SubtreePool *, SubtreeArray *, const TSLanguage *);