traxys/tree-sitter
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Cram terminal subtree data into a 64-bit integer when possible

Browse source
This commit is contained in:
Max Brunsfeld 2018-09-17 13:12:13 -07:00
parent e00c3bbdb9
commit b29d0f622f
21 changed files with 1258 additions and 1007 deletions
Download patch file Download diff file Expand all files Collapse all files

2
project.gyp
View file

@ -127,7 +127,7 @@
},
},
'Test': {
'defines': ['TREE_SITTER_WRAP_MALLOC=true'],
'defines': ['TREE_SITTER_TEST=true'],
'cflags': [ '-g' ],
'ldflags': [ '-g' ],
'xcode_settings': {

2
src/runtime/alloc.h
View file

@ -9,7 +9,7 @@ extern "C" {
#include <stdbool.h>
#include <stdio.h>
#if defined(TREE_SITTER_WRAP_MALLOC)
#if defined(TREE_SITTER_TEST)
void *ts_record_malloc(size_t);
void *ts_record_calloc(size_t, size_t);

67
src/runtime/get_changed_ranges.c
View file

@ -57,32 +57,35 @@ Length iterator_start_position(Iterator *self) {
if (self->in_padding) {
return entry.position;
} else {
return length_add(entry.position, entry.subtree->padding);
return length_add(entry.position, ts_subtree_padding(*entry.subtree));
}
}
Length iterator_end_position(Iterator *self) {
TreeCursorEntry entry = *array_back(&self->cursor.stack);
Length result = length_add(entry.position, entry.subtree->padding);
Length result = length_add(entry.position, ts_subtree_padding(*entry.subtree));
if (self->in_padding) {
return result;
} else {
return length_add(result, entry.subtree->size);
return length_add(result, ts_subtree_size(*entry.subtree));
}
}
static bool iterator_tree_is_visible(const Iterator *self) {
TreeCursorEntry entry = *array_back(&self->cursor.stack);
if (entry.subtree->visible) return true;
if (ts_subtree_visible(*entry.subtree)) return true;
if (self->cursor.stack.size > 1) {
const Subtree *parent = self->cursor.stack.contents[self->cursor.stack.size - 2].subtree;
const TSSymbol *alias_sequence = ts_language_alias_sequence(self->language, parent->alias_sequence_id);
Subtree parent = *self->cursor.stack.contents[self->cursor.stack.size - 2].subtree;
const TSSymbol *alias_sequence = ts_language_alias_sequence(
self->language,
parent.ptr->alias_sequence_id
);
return alias_sequence && alias_sequence[entry.structural_child_index] != 0;
}
return false;
}
static void iterator_get_visible_state(const Iterator *self, const Subtree **tree,
static void iterator_get_visible_state(const Iterator *self, Subtree *tree,
TSSymbol *alias_symbol, uint32_t *start_byte) {
uint32_t i = self->cursor.stack.size - 1;
@ -98,15 +101,15 @@ static void iterator_get_visible_state(const Iterator *self, const Subtree **tre
const Subtree *parent = self->cursor.stack.contents[i - 1].subtree;
const TSSymbol *alias_sequence = ts_language_alias_sequence(
self->language,
parent->alias_sequence_id
parent->ptr->alias_sequence_id
);
if (alias_sequence) {
*alias_symbol = alias_sequence[entry.structural_child_index];
}
}
if (entry.subtree->visible || *alias_symbol) {
*tree = entry.subtree;
if (ts_subtree_visible(*entry.subtree) || *alias_symbol) {
*tree = *entry.subtree;
*start_byte = entry.position.bytes;
break;
}
@ -129,10 +132,10 @@ static bool iterator_descend(Iterator *self, uint32_t goal_position) {
TreeCursorEntry entry = *array_back(&self->cursor.stack);
Length position = entry.position;
uint32_t structural_child_index = 0;
for (uint32_t i = 0; i < entry.subtree->child_count; i++) {
const Subtree *child = entry.subtree->children[i];
Length child_left = length_add(position, child->padding);
Length child_right = length_add(child_left, child->size);
for (uint32_t i = 0, n = ts_subtree_child_count(*entry.subtree); i < n; i++) {
const Subtree *child = &entry.subtree->ptr->children[i];
Length child_left = length_add(position, ts_subtree_padding(*child));
Length child_right = length_add(child_left, ts_subtree_size(*child));
if (child_right.bytes > goal_position) {
array_push(&self->cursor.stack, ((TreeCursorEntry){
@ -156,7 +159,7 @@ static bool iterator_descend(Iterator *self, uint32_t goal_position) {
}
position = child_right;
if (!child->extra) structural_child_index++;
if (!ts_subtree_extra(*child)) structural_child_index++;
}
} while (did_descend);
@ -181,11 +184,11 @@ static void iterator_advance(Iterator *self) {
const Subtree *parent = array_back(&self->cursor.stack)->subtree;
uint32_t child_index = entry.child_index + 1;
if (parent->child_count > child_index) {
Length position = length_add(entry.position, ts_subtree_total_size(entry.subtree));
if (ts_subtree_child_count(*parent) > child_index) {
Length position = length_add(entry.position, ts_subtree_total_size(*entry.subtree));
uint32_t structural_child_index = entry.structural_child_index;
if (!entry.subtree->extra) structural_child_index++;
const Subtree *next_child = parent->children[child_index];
if (!ts_subtree_extra(*entry.subtree)) structural_child_index++;
const Subtree *next_child = &parent->ptr->children[child_index];
array_push(&self->cursor.stack, ((TreeCursorEntry){
.subtree = next_child,
@ -195,7 +198,7 @@ static void iterator_advance(Iterator *self) {
}));
if (iterator_tree_is_visible(self)) {
if (next_child->padding.bytes > 0) {
if (ts_subtree_padding(*next_child).bytes > 0) {
self->in_padding = true;
} else {
self->visible_depth++;
@ -215,30 +218,30 @@ typedef enum {
} IteratorComparison;
IteratorComparison iterator_compare(const Iterator *old_iter, const Iterator *new_iter) {
const Subtree *old_tree = NULL, *new_tree = NULL;
Subtree old_tree = {.ptr = NULL}, new_tree = {.ptr = NULL};
uint32_t old_start = 0, new_start = 0;
TSSymbol old_alias_symbol = 0, new_alias_symbol = 0;
iterator_get_visible_state(old_iter, &old_tree, &old_alias_symbol, &old_start);
iterator_get_visible_state(new_iter, &new_tree, &new_alias_symbol, &new_start);
if (!old_tree && !new_tree) return IteratorMatches;
if (!old_tree || !new_tree) return IteratorDiffers;
if (!old_tree.ptr && !new_tree.ptr) return IteratorMatches;
if (!old_tree.ptr || !new_tree.ptr) return IteratorDiffers;
if (old_alias_symbol == new_alias_symbol) {
if (old_start == new_start) {
if (!old_tree->has_changes &&
old_tree->symbol == new_tree->symbol &&
old_tree->symbol != ts_builtin_sym_error &&
old_tree->size.bytes == new_tree->size.bytes &&
old_tree->parse_state != TS_TREE_STATE_NONE &&
new_tree->parse_state != TS_TREE_STATE_NONE &&
(old_tree->parse_state == ERROR_STATE) ==
(new_tree->parse_state == ERROR_STATE)) {
if (!ts_subtree_has_changes(old_tree) &&
ts_subtree_symbol(old_tree) == ts_subtree_symbol(new_tree) &&
ts_subtree_symbol(old_tree) != ts_builtin_sym_error &&
ts_subtree_size(old_tree).bytes == ts_subtree_size(new_tree).bytes &&
ts_subtree_parse_state(old_tree) != TS_TREE_STATE_NONE &&
ts_subtree_parse_state(new_tree) != TS_TREE_STATE_NONE &&
(ts_subtree_parse_state(old_tree) == ERROR_STATE) ==
(ts_subtree_parse_state(new_tree) == ERROR_STATE)) {
return IteratorMatches;
}
}
if (old_tree->symbol == new_tree->symbol) {
if (ts_subtree_symbol(old_tree) == ts_subtree_symbol(new_tree)) {
return IteratorMayDiffer;
}
}

126
src/runtime/node.c
View file

@ -4,7 +4,7 @@
#include "runtime/language.h"
typedef struct {
const Subtree *parent;
Subtree parent;
const TSTree *tree;
Length position;
uint32_t child_index;
@ -40,17 +40,20 @@ static inline uint32_t ts_node__alias(const TSNode *self) {
return self->context[3];
}
static inline const Subtree *ts_node__subtree(TSNode self) {
return self.id;
static inline Subtree ts_node__subtree(TSNode self) {
return *(const Subtree *)self.id;
}
// ChildIterator
static inline ChildIterator ts_node_iterate_children(const TSNode *node) {
const Subtree *subtree = ts_node__subtree(*node);
Subtree subtree = ts_node__subtree(*node);
if (ts_subtree_child_count(subtree) == 0) {
return (ChildIterator) {NULL_SUBTREE, node->tree, length_zero(), 0, 0, NULL};
}
const TSSymbol *alias_sequence = ts_language_alias_sequence(
node->tree->language,
subtree->alias_sequence_id
subtree.ptr->alias_sequence_id
);
return (ChildIterator) {
.tree = node->tree,
@ -63,17 +66,17 @@ static inline ChildIterator ts_node_iterate_children(const TSNode *node) {
}
static inline bool ts_node_child_iterator_next(ChildIterator *self, TSNode *result) {
if (self->child_index == self->parent->child_count) return false;
const Subtree *child = self->parent->children[self->child_index];
if (!self->parent.ptr || self->child_index == self->parent.ptr->child_count) return false;
const Subtree *child = &self->parent.ptr->children[self->child_index];
TSSymbol alias_symbol = 0;
if (!child->extra) {
if (!ts_subtree_extra(*child)) {
if (self->alias_sequence) {
alias_symbol = self->alias_sequence[self->structural_child_index];
}
self->structural_child_index++;
}
if (self->child_index > 0) {
self->position = length_add(self->position, child->padding);
self->position = length_add(self->position, ts_subtree_padding(*child));
}
*result = ts_node_new(
self->tree,
@ -81,7 +84,7 @@ static inline bool ts_node_child_iterator_next(ChildIterator *self, TSNode *resu
self->position,
alias_symbol
);
self->position = length_add(self->position, child->size);
self->position = length_add(self->position, ts_subtree_size(*child));
self->child_index++;
return true;
}
@ -89,30 +92,25 @@ static inline bool ts_node_child_iterator_next(ChildIterator *self, TSNode *resu
// TSNode - private
static inline bool ts_node__is_relevant(TSNode self, bool include_anonymous) {
const Subtree *tree = ts_node__subtree(self);
Subtree tree = ts_node__subtree(self);
if (include_anonymous) {
return tree->visible || ts_node__alias(&self);
return ts_subtree_visible(tree) || ts_node__alias(&self);
} else {
return (
(tree->visible && tree->named) ||
(
ts_node__alias(&self) &&
ts_language_symbol_metadata(
self.tree->language,
ts_node__alias(&self)
).named
)
);
return
(ts_subtree_visible(tree) &&
ts_subtree_named(tree)) ||
(ts_node__alias(&self) &&
ts_language_symbol_metadata(self.tree->language, ts_node__alias(&self)).named);
}
}
static inline uint32_t ts_node__relevant_child_count(TSNode self, bool include_anonymous) {
const Subtree *tree = ts_node__subtree(self);
if (tree->child_count > 0) {
Subtree tree = ts_node__subtree(self);
if (ts_subtree_child_count(tree) > 0) {
if (include_anonymous) {
return tree->visible_child_count;
return tree.ptr->visible_child_count;
} else {
return tree->named_child_count;
return tree.ptr->named_child_count;
}
} else {
return 0;
@ -153,25 +151,27 @@ static inline TSNode ts_node__child(TSNode self, uint32_t child_index, bool incl
return ts_node__null();
}
static bool ts_subtree_has_trailing_empty_descendant(const Subtree *self, const Subtree *other) {
for (unsigned i = self->child_count - 1; i + 1 > 0; i--) {
const Subtree *child = self->children[i];
if (child->size.bytes > 0 || child->padding.bytes > 0) break;
if (child == other || ts_subtree_has_trailing_empty_descendant(child, other)) return true;
static bool ts_subtree_has_trailing_empty_descendant(Subtree self, Subtree other) {
for (unsigned i = ts_subtree_child_count(self) - 1; i + 1 > 0; i--) {
Subtree child = self.ptr->children[i];
if (ts_subtree_total_bytes(child) > 0) break;
if (child.ptr == other.ptr || ts_subtree_has_trailing_empty_descendant(child, other)) {
return true;
}
}
return false;
}
static inline TSNode ts_node__prev_sibling(TSNode self, bool include_anonymous) {
const Subtree *self_subtree = ts_node__subtree(self);
bool self_is_empty = self_subtree->size.bytes == 0 && self_subtree->padding.bytes == 0;
Subtree self_subtree = ts_node__subtree(self);
bool self_is_empty = ts_subtree_total_bytes(self_subtree) == 0;
uint32_t target_end_byte = ts_node_end_byte(self);
TSNode node = ts_node_parent(self);
TSNode earlier_node = ts_node__null();
bool earlier_node_is_relevant = false;
while (ts_node__subtree(node)) {
while (!ts_node_is_null(node)) {
TSNode earlier_child = ts_node__null();
bool earlier_child_is_relevant = false;
bool found_child_containing_target = false;
@ -180,11 +180,14 @@ static inline TSNode ts_node__prev_sibling(TSNode self, bool include_anonymous)
ChildIterator iterator = ts_node_iterate_children(&node);
while (ts_node_child_iterator_next(&iterator, &child)) {
if (child.id == self.id) break;
if (iterator.position.bytes > target_end_byte) {
found_child_containing_target = true;
break;
}
if (iterator.position.bytes > target_end_byte || (
iterator.position.bytes == target_end_byte && (
!self_is_empty ||
ts_subtree_has_trailing_empty_descendant(ts_node__subtree(child), self_subtree)))) {
if (iterator.position.bytes == target_end_byte &&
(!self_is_empty ||
ts_subtree_has_trailing_empty_descendant(ts_node__subtree(child), self_subtree))) {
found_child_containing_target = true;
break;
}
@ -225,7 +228,7 @@ static inline TSNode ts_node__next_sibling(TSNode self, bool include_anonymous)
TSNode later_node = ts_node__null();
bool later_node_is_relevant = false;
while (ts_node__subtree(node)) {
while (!ts_node_is_null(node)) {
TSNode later_child = ts_node__null();
bool later_child_is_relevant = false;
TSNode child_containing_target = ts_node__null();
@ -235,7 +238,7 @@ static inline TSNode ts_node__next_sibling(TSNode self, bool include_anonymous)
while (ts_node_child_iterator_next(&iterator, &child)) {
if (iterator.position.bytes < target_end_byte) continue;
if (ts_node_start_byte(child) <= ts_node_start_byte(self)) {
if (ts_node__subtree(child) != ts_node__subtree(self)) {
if (ts_node__subtree(child).ptr != ts_node__subtree(self).ptr) {
child_containing_target = child;
}
} else if (ts_node__is_relevant(child, include_anonymous)) {
@ -360,16 +363,17 @@ static inline TSNode ts_node__descendant_for_point_range(TSNode self, TSPoint mi
// TSNode - public
uint32_t ts_node_end_byte(TSNode self) {
return ts_node_start_byte(self) + ts_node__subtree(self)->size.bytes;
return ts_node_start_byte(self) + ts_subtree_size(ts_node__subtree(self)).bytes;
}
TSPoint ts_node_end_point(TSNode self) {
return point_add(ts_node_start_point(self), ts_node__subtree(self)->size.extent);
return point_add(ts_node_start_point(self), ts_subtree_size(ts_node__subtree(self)).extent);
}
TSSymbol ts_node_symbol(TSNode self) {
const Subtree *tree = ts_node__subtree(self);
return ts_node__alias(&self) ? ts_node__alias(&self) : tree->symbol;
return ts_node__alias(&self)
? ts_node__alias(&self)
: ts_subtree_symbol(ts_node__subtree(self));
}
const char *ts_node_type(TSNode self) {
@ -385,27 +389,25 @@ bool ts_node_eq(TSNode self, TSNode other) {
}
bool ts_node_is_null(TSNode self) {
return ts_node__subtree(self) == NULL;
return self.id == 0;
}
bool ts_node_is_named(TSNode self) {
const Subtree *tree = ts_node__subtree(self);
return ts_node__alias(&self)
? ts_language_symbol_metadata(self.tree->language, ts_node__alias(&self)).named
: tree->named;
: ts_subtree_named(ts_node__subtree(self));
}
bool ts_node_is_missing(TSNode self) {
const Subtree *tree = ts_node__subtree(self);
return tree->is_missing;
return ts_subtree_missing(ts_node__subtree(self));
}
bool ts_node_has_changes(TSNode self) {
return ts_node__subtree(self)->has_changes;
return ts_subtree_has_changes(ts_node__subtree(self));
}
bool ts_node_has_error(TSNode self) {
return ts_node__subtree(self)->error_cost > 0;
return ts_subtree_error_cost(ts_node__subtree(self)) > 0;
}
TSNode ts_node_parent(TSNode self) {
@ -414,7 +416,7 @@ TSNode ts_node_parent(TSNode self) {
node = ts_tree_root_node(self.tree);
uint32_t end_byte = ts_node_end_byte(self);
if (ts_node__subtree(node) == ts_node__subtree(self)) return ts_node__null();
if (node.id == self.id) return ts_node__null();
TSNode last_visible_node = node;
bool did_descend = true;
@ -426,7 +428,7 @@ TSNode ts_node_parent(TSNode self) {
while (ts_node_child_iterator_next(&iterator, &child)) {
if (
ts_node_start_byte(child) > ts_node_start_byte(self) ||
ts_node__subtree(child) == ts_node__subtree(self)
child.id == self.id
) break;
if (iterator.position.bytes >= end_byte) {
node = child;
@ -452,18 +454,18 @@ TSNode ts_node_named_child(TSNode self, uint32_t child_index) {
}
uint32_t ts_node_child_count(TSNode self) {
const Subtree *tree = ts_node__subtree(self);
if (tree->child_count > 0) {
return tree->visible_child_count;
Subtree tree = ts_node__subtree(self);
if (ts_subtree_child_count(tree) > 0) {
return tree.ptr->visible_child_count;
} else {
return 0;
}
}
uint32_t ts_node_named_child_count(TSNode self) {
const Subtree *tree = ts_node__subtree(self);
if (tree->child_count > 0) {
return tree->named_child_count;
Subtree tree = ts_node__subtree(self);
if (ts_subtree_child_count(tree) > 0) {
return tree.ptr->named_child_count;
} else {
return 0;
}
@ -497,8 +499,7 @@ TSNode ts_node_descendant_for_byte_range(TSNode self, uint32_t min, uint32_t max
return ts_node__descendant_for_byte_range(self, min, max, true);
}
TSNode ts_node_named_descendant_for_byte_range(TSNode self, uint32_t min,
uint32_t max) {
TSNode ts_node_named_descendant_for_byte_range(TSNode self, uint32_t min, uint32_t max) {
return ts_node__descendant_for_byte_range(self, min, max, false);
}
@ -506,8 +507,7 @@ TSNode ts_node_descendant_for_point_range(TSNode self, TSPoint min, TSPoint max)
return ts_node__descendant_for_point_range(self, min, max, true);
}
TSNode ts_node_named_descendant_for_point_range(TSNode self, TSPoint min,
TSPoint max) {
TSNode ts_node_named_descendant_for_point_range(TSNode self, TSPoint min, TSPoint max) {
return ts_node__descendant_for_point_range(self, min, max, false);
}

402
src/runtime/parser.c
View file

@ -35,14 +35,16 @@
#define SYM_NAME(symbol) ts_language_symbol_name(self->language, symbol)
#define TREE_NAME(tree) SYM_NAME(ts_subtree_symbol(tree))
static const unsigned MAX_VERSION_COUNT = 6;
static const unsigned MAX_VERSION_COUNT_OVERFLOW = 4;
static const unsigned MAX_SUMMARY_DEPTH = 16;
static const unsigned MAX_COST_DIFFERENCE = 16 * ERROR_COST_PER_SKIPPED_TREE;
typedef struct {
const Subtree *token;
const Subtree *last_external_token;
Subtree token;
Subtree last_external_token;
uint32_t byte_index;
} TokenCache;
@ -52,8 +54,9 @@ struct TSParser {
SubtreePool tree_pool;
const TSLanguage *language;
ReduceActionSet reduce_actions;
const Subtree *finished_tree;
Subtree scratch_tree;
Subtree finished_tree;
SubtreeHeapData scratch_tree_data;
MutableSubtree scratch_tree;
TokenCache token_cache;
ReusableNode reusable_node;
void *external_scanner_payload;
@ -62,7 +65,7 @@ struct TSParser {
size_t operation_limit;
volatile bool enabled;
bool halt_on_error;
const Subtree *old_tree;
Subtree old_tree;
};
typedef struct {
@ -132,16 +135,16 @@ static bool ts_parser__breakdown_top_of_stack(TSParser *self, StackVersion versi
for (uint32_t i = 0; i < pop.size; i++) {
StackSlice slice = pop.contents[i];
TSStateId state = ts_stack_state(self->stack, slice.version);
const Subtree *parent = *array_front(&slice.subtrees);
Subtree parent = *array_front(&slice.subtrees);
for (uint32_t j = 0; j < parent->child_count; j++) {
const Subtree *child = parent->children[j];
pending = child->child_count > 0;
for (uint32_t j = 0, n = ts_subtree_child_count(parent); j < n; j++) {
Subtree child = parent.ptr->children[j];
pending = ts_subtree_child_count(child) > 0;
if (child->symbol == ts_builtin_sym_error) {
if (ts_subtree_is_error(child)) {
state = ERROR_STATE;
} else if (!child->extra) {
state = ts_language_next_state(self->language, state, child->symbol);
} else if (!ts_subtree_extra(child)) {
state = ts_language_next_state(self->language, state, ts_subtree_symbol(child));
}
ts_subtree_retain(child);
@ -149,14 +152,14 @@ static bool ts_parser__breakdown_top_of_stack(TSParser *self, StackVersion versi
}
for (uint32_t j = 1; j < slice.subtrees.size; j++) {
const Subtree *tree = slice.subtrees.contents[j];
Subtree tree = slice.subtrees.contents[j];
ts_stack_push(self->stack, slice.version, tree, false, state);
}
ts_subtree_release(&self->tree_pool, parent);
array_delete(&slice.subtrees);
LOG("breakdown_top_of_stack tree:%s", SYM_NAME(parent->symbol));
LOG("breakdown_top_of_stack tree:%s", TREE_NAME(parent));
LOG_STACK();
}
} while (pending);
@ -164,12 +167,12 @@ static bool ts_parser__breakdown_top_of_stack(TSParser *self, StackVersion versi
return did_break_down;
}
static void ts_parser__breakdown_lookahead(TSParser *self, const Subtree **lookahead,
static void ts_parser__breakdown_lookahead(TSParser *self, Subtree *lookahead,
TSStateId state, ReusableNode *reusable_node) {
bool did_descend = false;
const Subtree *tree = reusable_node_tree(reusable_node);
while (tree->child_count > 0 && tree->parse_state != state) {
LOG("state_mismatch sym:%s", SYM_NAME(tree->symbol));
Subtree tree = reusable_node_tree(reusable_node);
while (ts_subtree_child_count(tree) > 0 && ts_subtree_parse_state(tree) != state) {
LOG("state_mismatch sym:%s", TREE_NAME(tree));
reusable_node_descend(reusable_node);
tree = reusable_node_tree(reusable_node);
did_descend = true;
@ -233,8 +236,10 @@ static ErrorStatus ts_parser__version_status(TSParser *self, StackVersion versio
}
static bool ts_parser__better_version_exists(TSParser *self, StackVersion version,
bool is_in_error, unsigned cost) {
if (self->finished_tree && self->finished_tree->error_cost <= cost) return true;
bool is_in_error, unsigned cost) {
if (self->finished_tree.ptr && ts_subtree_error_cost(self->finished_tree) <= cost) {
return true;
}
Length position = ts_stack_position(self->stack, version);
ErrorStatus status = {
@ -262,50 +267,41 @@ static bool ts_parser__better_version_exists(TSParser *self, StackVersion versio
return false;
}
static void ts_parser__restore_external_scanner(TSParser *self, const Subtree *external_token) {
if (external_token) {
static void ts_parser__restore_external_scanner(TSParser *self, Subtree external_token) {
if (external_token.ptr) {
self->language->external_scanner.deserialize(
self->external_scanner_payload,
ts_external_scanner_state_data(&external_token->external_scanner_state),
external_token->external_scanner_state.length
ts_external_scanner_state_data(&external_token.ptr->external_scanner_state),
external_token.ptr->external_scanner_state.length
);
} else {
self->language->external_scanner.deserialize(self->external_scanner_payload, NULL, 0);
}
}
static bool ts_parser__can_reuse_first_leaf(TSParser *self, TSStateId state, const Subtree *tree,
static bool ts_parser__can_reuse_first_leaf(TSParser *self, TSStateId state, Subtree tree,
TableEntry *table_entry) {
TSLexMode current_lex_mode = self->language->lex_modes[state];
TSStateId leaf_state;
TSSymbol leaf_symbol;
if (tree->child_count > 0) {
leaf_state = tree->first_leaf.parse_state;
leaf_symbol = tree->first_leaf.symbol;
} else {
leaf_state = tree->parse_state;
leaf_symbol = tree->symbol;
}
TSSymbol leaf_symbol = ts_subtree_leaf_symbol(tree);
TSStateId leaf_state = ts_subtree_leaf_parse_state(tree);
TSLexMode leaf_lex_mode = self->language->lex_modes[leaf_state];
// If the token was created in a state with the same set of lookaheads, it is reusable.
if (memcmp(&leaf_lex_mode, &current_lex_mode, sizeof(TSLexMode)) == 0 &&
(leaf_symbol != self->language->keyword_capture_token ||
(!tree->is_keyword && tree->parse_state == state))) return true;
(!ts_subtree_is_keyword(tree) && ts_subtree_parse_state(tree) == state))) return true;
// Empty tokens are not reusable in states with different lookaheads.
if (tree->size.bytes == 0 && tree->symbol != ts_builtin_sym_end) return false;
if (ts_subtree_size(tree).bytes == 0 && leaf_symbol != ts_builtin_sym_end) return false;
// If the current state allows external tokens or other tokens that conflict with this
// token, this token is not reusable.
return current_lex_mode.external_lex_state == 0 && table_entry->is_reusable;
}
static const Subtree *ts_parser__lex(TSParser *self, StackVersion version, TSStateId parse_state) {
static Subtree ts_parser__lex(TSParser *self, StackVersion version, TSStateId parse_state) {
Length start_position = ts_stack_position(self->stack, version);
const Subtree *external_token = ts_stack_last_external_token(self->stack, version);
Subtree external_token = ts_stack_last_external_token(self->stack, version);
TSLexMode lex_mode = self->language->lex_modes[parse_state];
const bool *valid_external_tokens = ts_language_enabled_external_tokens(
self->language,
@ -404,13 +400,19 @@ static const Subtree *ts_parser__lex(TSParser *self, StackVersion version, TSSta
uint32_t bytes_scanned = last_byte_scanned - start_position.bytes + 1;
Subtree *result;
Subtree result;
if (skipped_error) {
Length padding = length_sub(error_start_position, start_position);
Length size = length_sub(error_end_position, error_start_position);
result = ts_subtree_new_error(&self->tree_pool, size, padding, first_error_character, self->language);
result->parse_state = parse_state;
result->bytes_scanned = bytes_scanned;
result = ts_subtree_new_error(
&self->tree_pool,
first_error_character,
padding,
size,
bytes_scanned,
parse_state,
self->language
);
} else {
if (self->lexer.token_end_position.bytes < self->lexer.token_start_position.bytes) {
self->lexer.token_start_position = self->lexer.token_end_position;
@ -455,85 +457,87 @@ static const Subtree *ts_parser__lex(TSParser *self, StackVersion version, TSSta
self->lexer.debug_buffer
);
ts_external_scanner_state_init(
&result->external_scanner_state,
&((SubtreeHeapData *)result.ptr)->external_scanner_state,
self->lexer.debug_buffer,
length
);
}
}
LOG("lexed_lookahead sym:%s, size:%u", SYM_NAME(result->symbol), result->size.bytes);
LOG(
"lexed_lookahead sym:%s, size:%u",
SYM_NAME(ts_subtree_symbol(result)),
ts_subtree_size(result).bytes
);
return result;
}
static const Subtree *ts_parser__get_cached_token(TSParser *self, TSStateId state,
size_t position,
const Subtree *last_external_token,
TableEntry *table_entry) {
static Subtree ts_parser__get_cached_token(TSParser *self, TSStateId state,
size_t position, Subtree last_external_token,
TableEntry *table_entry) {
TokenCache *cache = &self->token_cache;
if (
cache->token && cache->byte_index == position &&
cache->token.ptr && cache->byte_index == position &&
ts_subtree_external_scanner_state_eq(cache->last_external_token, last_external_token)
) {
ts_language_table_entry(self->language, state, cache->token->symbol, table_entry);
ts_language_table_entry(self->language, state, ts_subtree_symbol(cache->token), table_entry);
if (ts_parser__can_reuse_first_leaf(self, state, cache->token, table_entry)) {
ts_subtree_retain(cache->token);
return cache->token;
}
}
return NULL;
return NULL_SUBTREE;
}
static void ts_parser__set_cached_token(TSParser *self, size_t byte_index,
const Subtree *last_external_token,
const Subtree *token) {
Subtree last_external_token,
Subtree token) {
TokenCache *cache = &self->token_cache;
if (token) ts_subtree_retain(token);
if (last_external_token) ts_subtree_retain(last_external_token);
if (cache->token) ts_subtree_release(&self->tree_pool, cache->token);
if (cache->last_external_token) ts_subtree_release(&self->tree_pool, cache->last_external_token);
if (token.ptr) ts_subtree_retain(token);
if (last_external_token.ptr) ts_subtree_retain(last_external_token);
if (cache->token.ptr) ts_subtree_release(&self->tree_pool, cache->token);
if (cache->last_external_token.ptr) ts_subtree_release(&self->tree_pool, cache->last_external_token);
cache->token = token;
cache->byte_index = byte_index;
cache->last_external_token = last_external_token;
}
static const Subtree *ts_parser__reuse_node(TSParser *self, StackVersion version,
TSStateId *state, uint32_t position,
const Subtree *last_external_token,
TableEntry *table_entry) {
const Subtree *result;
while ((result = reusable_node_tree(&self->reusable_node))) {
static Subtree ts_parser__reuse_node(TSParser *self, StackVersion version,
TSStateId *state, uint32_t position,
Subtree last_external_token, TableEntry *table_entry) {
Subtree result;
while ((result = reusable_node_tree(&self->reusable_node)).ptr) {
uint32_t byte_offset = reusable_node_byte_offset(&self->reusable_node);
if (byte_offset > position) {
LOG("before_reusable_node symbol:%s", SYM_NAME(result->symbol));
LOG("before_reusable_node symbol:%s", TREE_NAME(result));
break;
}
if (byte_offset < position) {
LOG("past_reusable_node symbol:%s", SYM_NAME(result->symbol));
LOG("past_reusable_node symbol:%s", TREE_NAME(result));
reusable_node_advance(&self->reusable_node);
continue;
}
if (!ts_subtree_external_scanner_state_eq(self->reusable_node.last_external_token, last_external_token)) {
LOG("reusable_node_has_different_external_scanner_state symbol:%s", SYM_NAME(result->symbol));
LOG("reusable_node_has_different_external_scanner_state symbol:%s", TREE_NAME(result));
reusable_node_advance(&self->reusable_node);
continue;
}
const char *reason = NULL;
if (result->has_changes) {
if (ts_subtree_has_changes(result)) {
reason = "has_changes";
} else if (result->symbol == ts_builtin_sym_error) {
} else if (ts_subtree_is_error(result)) {
reason = "is_error";
} else if (result->is_missing) {
} else if (ts_subtree_missing(result)) {
reason = "is_missing";
} else if (result->fragile_left || result->fragile_right) {
} else if (ts_subtree_is_fragile(result)) {
reason = "is_fragile";
}
if (reason) {
LOG("cant_reuse_node_%s tree:%s", reason, SYM_NAME(result->symbol));
LOG("cant_reuse_node_%s tree:%s", reason, TREE_NAME(result));
if (!reusable_node_descend(&self->reusable_node)) {
reusable_node_advance(&self->reusable_node);
ts_parser__breakdown_top_of_stack(self, version);
@ -547,97 +551,92 @@ static const Subtree *ts_parser__reuse_node(TSParser *self, StackVersion version
if (!ts_parser__can_reuse_first_leaf(self, *state, result, table_entry)) {
LOG(
"cant_reuse_node symbol:%s, first_leaf_symbol:%s",
SYM_NAME(result->symbol),
TREE_NAME(result),
SYM_NAME(leaf_symbol)
);
reusable_node_advance_past_leaf(&self->reusable_node);
break;
}
LOG("reuse_node symbol:%s", SYM_NAME(result->symbol));
LOG("reuse_node symbol:%s", TREE_NAME(result));
ts_subtree_retain(result);
return result;
}
return NULL;
return NULL_SUBTREE;
}
static bool ts_parser__select_tree(TSParser *self, const Subtree *left, const Subtree *right) {
if (!left) return true;
if (!right) return false;
static bool ts_parser__select_tree(TSParser *self, Subtree left, Subtree right) {
if (!left.ptr) return true;
if (!right.ptr) return false;
if (right->error_cost < left->error_cost) {
LOG("select_smaller_error symbol:%s, over_symbol:%s",
SYM_NAME(right->symbol), SYM_NAME(left->symbol));
if (ts_subtree_error_cost(right) < ts_subtree_error_cost(left)) {
LOG("select_smaller_error symbol:%s, over_symbol:%s", TREE_NAME(right), TREE_NAME(left));
return true;
}
if (left->error_cost < right->error_cost) {
LOG("select_smaller_error symbol:%s, over_symbol:%s",
SYM_NAME(left->symbol), SYM_NAME(right->symbol));
if (ts_subtree_error_cost(left) < ts_subtree_error_cost(right)) {
LOG("select_smaller_error symbol:%s, over_symbol:%s", TREE_NAME(left), TREE_NAME(right));
return false;
}
if (right->dynamic_precedence > left->dynamic_precedence) {
if (ts_subtree_dynamic_precedence(right) > ts_subtree_dynamic_precedence(left)) {
LOG("select_higher_precedence symbol:%s, prec:%u, over_symbol:%s, other_prec:%u",
SYM_NAME(right->symbol), right->dynamic_precedence, SYM_NAME(left->symbol),
left->dynamic_precedence);
TREE_NAME(right), ts_subtree_dynamic_precedence(right), TREE_NAME(left),
ts_subtree_dynamic_precedence(left));
return true;
}
if (left->dynamic_precedence > right->dynamic_precedence) {
if (ts_subtree_dynamic_precedence(left) > ts_subtree_dynamic_precedence(right)) {
LOG("select_higher_precedence symbol:%s, prec:%u, over_symbol:%s, other_prec:%u",
SYM_NAME(left->symbol), left->dynamic_precedence, SYM_NAME(right->symbol),
right->dynamic_precedence);
TREE_NAME(left), ts_subtree_dynamic_precedence(left), TREE_NAME(right),
ts_subtree_dynamic_precedence(right));
return false;
}
if (left->error_cost > 0) return true;
if (ts_subtree_error_cost(left) > 0) return true;
int comparison = ts_subtree_compare(left, right);
switch (comparison) {
case -1:
LOG("select_earlier symbol:%s, over_symbol:%s", SYM_NAME(left->symbol),
SYM_NAME(right->symbol));
LOG("select_earlier symbol:%s, over_symbol:%s", TREE_NAME(left), TREE_NAME(right));
return false;
break;
case 1:
LOG("select_earlier symbol:%s, over_symbol:%s", SYM_NAME(right->symbol),
SYM_NAME(left->symbol));
LOG("select_earlier symbol:%s, over_symbol:%s", TREE_NAME(right), TREE_NAME(left));
return true;
default:
LOG("select_existing symbol:%s, over_symbol:%s", SYM_NAME(left->symbol),
SYM_NAME(right->symbol));
LOG("select_existing symbol:%s, over_symbol:%s", TREE_NAME(left), TREE_NAME(right));
return false;
}
}
static void ts_parser__shift(TSParser *self, StackVersion version, TSStateId state,
const Subtree *lookahead, bool extra) {
const Subtree *subtree_to_push;
if (extra != lookahead->extra) {
Subtree *result = ts_subtree_make_mut(&self->tree_pool, lookahead);
result->extra = extra;
subtree_to_push = result;
Subtree lookahead, bool extra) {
Subtree subtree_to_push;
if (extra != ts_subtree_extra(lookahead)) {
MutableSubtree result = ts_subtree_make_mut(&self->tree_pool, lookahead);
ts_subtree_set_extra(&result);
subtree_to_push = ts_subtree_from_mut(result);
} else {
subtree_to_push = lookahead;
}
bool is_pending = subtree_to_push->child_count > 0;
bool is_pending = ts_subtree_child_count(subtree_to_push) > 0;
ts_stack_push(self->stack, version, subtree_to_push, is_pending, state);
if (subtree_to_push->has_external_tokens) {
if (ts_subtree_has_external_tokens(subtree_to_push)) {
ts_stack_set_last_external_token(
self->stack, version, ts_subtree_last_external_token(subtree_to_push)
);
}
}
static bool ts_parser__replace_children(TSParser *self, Subtree *tree, SubtreeArray *children) {
self->scratch_tree = *tree;
self->scratch_tree.child_count = 0;
ts_subtree_set_children(&self->scratch_tree, children->contents, children->size, self->language);
if (ts_parser__select_tree(self, tree, &self->scratch_tree)) {
*tree = self->scratch_tree;
static bool ts_parser__replace_children(TSParser *self, MutableSubtree *tree, SubtreeArray *children) {
*self->scratch_tree.ptr = *tree->ptr;
self->scratch_tree.ptr->child_count = 0;
ts_subtree_set_children(self->scratch_tree, children->contents, children->size, self->language);
if (ts_parser__select_tree(self, ts_subtree_from_mut(*tree), ts_subtree_from_mut(self->scratch_tree))) {
*tree->ptr = *self->scratch_tree.ptr;
return true;
} else {
return false;
@ -675,11 +674,11 @@ static StackVersion ts_parser__reduce(TSParser *self, StackVersion version, TSSy
// node. They will be re-pushed onto the stack after the parent node is
// created and pushed.
SubtreeArray children = slice.subtrees;
while (children.size > 0 && children.contents[children.size - 1]->extra) {
while (children.size > 0 && ts_subtree_extra(children.contents[children.size - 1])) {
children.size--;
}
Subtree *parent = ts_subtree_new_node(&self->tree_pool,
MutableSubtree parent = ts_subtree_new_node(&self->tree_pool,
symbol, &children, alias_sequence_id, self->language
);
@ -693,11 +692,11 @@ static StackVersion ts_parser__reduce(TSParser *self, StackVersion version, TSSy
i++;
SubtreeArray children = next_slice.subtrees;
while (children.size > 0 && children.contents[children.size - 1]->extra) {
while (children.size > 0 && ts_subtree_extra(children.contents[children.size - 1])) {
children.size--;
}
if (ts_parser__replace_children(self, parent, &children)) {
if (ts_parser__replace_children(self, &parent, &children)) {
ts_subtree_array_delete(&self->tree_pool, &slice.subtrees);
slice = next_slice;
} else {
@ -705,23 +704,23 @@ static StackVersion ts_parser__reduce(TSParser *self, StackVersion version, TSSy
}
}
parent->dynamic_precedence += dynamic_precedence;
parent->alias_sequence_id = alias_sequence_id;
parent.ptr->dynamic_precedence += dynamic_precedence;
parent.ptr->alias_sequence_id = alias_sequence_id;
TSStateId state = ts_stack_state(self->stack, slice_version);
TSStateId next_state = ts_language_next_state(self->language, state, symbol);
if (fragile || pop.size > 1 || initial_version_count > 1) {
parent->fragile_left = true;
parent->fragile_right = true;
parent->parse_state = TS_TREE_STATE_NONE;
parent.ptr->fragile_left = true;
parent.ptr->fragile_right = true;
parent.ptr->parse_state = TS_TREE_STATE_NONE;
} else {
parent->parse_state = state;
parent.ptr->parse_state = state;
}
// Push the parent node onto the stack, along with any extra tokens that
// were previously on top of the stack.
ts_stack_push(self->stack, slice_version, parent, false, next_state);
for (uint32_t j = parent->child_count; j < slice.subtrees.size; j++) {
ts_stack_push(self->stack, slice_version, ts_subtree_from_mut(parent), false, next_state);
for (uint32_t j = parent.ptr->child_count; j < slice.subtrees.size; j++) {
ts_stack_push(self->stack, slice_version, slice.subtrees.contents[j], false, next_state);
}
@ -740,35 +739,40 @@ static StackVersion ts_parser__reduce(TSParser *self, StackVersion version, TSSy
: STACK_VERSION_NONE;
}
static void ts_parser__accept(TSParser *self, StackVersion version, const Subtree *lookahead) {
assert(lookahead->symbol == ts_builtin_sym_end);
static void ts_parser__accept(TSParser *self, StackVersion version, Subtree lookahead) {
assert(ts_subtree_is_eof(lookahead));
ts_stack_push(self->stack, version, lookahead, false, 1);
StackSliceArray pop = ts_stack_pop_all(self->stack, version);
for (uint32_t i = 0; i < pop.size; i++) {
SubtreeArray trees = pop.contents[i].subtrees;
const Subtree *root = NULL;
Subtree root = NULL_SUBTREE;
for (uint32_t j = trees.size - 1; j + 1 > 0; j--) {
const Subtree *child = trees.contents[j];
if (!child->extra) {
for (uint32_t k = 0; k < child->child_count; k++) {
ts_subtree_retain(child->children[k]);
Subtree child = trees.contents[j];
if (!ts_subtree_extra(child)) {
assert(!child.data.is_inline);
uint32_t child_count = ts_subtree_child_count(child);
for (uint32_t k = 0; k < child_count; k++) {
ts_subtree_retain(child.ptr->children[k]);
}
array_splice(&trees, j, 1, child->child_count, child->children);
root = ts_subtree_new_node(
&self->tree_pool, child->symbol, &trees,
child->alias_sequence_id, self->language
);
array_splice(&trees, j, 1, child_count, child.ptr->children);
root = ts_subtree_from_mut(ts_subtree_new_node(
&self->tree_pool,
ts_subtree_symbol(child),
&trees,
child.ptr->alias_sequence_id,
self->language
));
ts_subtree_release(&self->tree_pool, child);
break;
}
}
assert(root && root->ref_count > 0);
assert(root.ptr);
self->accept_count++;
if (self->finished_tree) {
if (self->finished_tree.ptr) {
if (ts_parser__select_tree(self, self->finished_tree, root)) {
ts_subtree_release(&self->tree_pool, self->finished_tree);
self->finished_tree = root;
@ -906,7 +910,7 @@ static void ts_parser__handle_error(TSParser *self, StackVersion version,
Length padding = length_sub(self->lexer.token_end_position, position);
StackVersion version_with_missing_tree = ts_stack_copy_version(self->stack, v);
const Subtree *missing_tree = ts_subtree_new_missing_leaf(
Subtree missing_tree = ts_subtree_new_missing_leaf(
&self->tree_pool, missing_symbol, padding, self->language
);
ts_stack_push(
@ -931,7 +935,7 @@ static void ts_parser__handle_error(TSParser *self, StackVersion version,
}
}
ts_stack_push(self->stack, v, NULL, false, ERROR_STATE);
ts_stack_push(self->stack, v, NULL_SUBTREE, false, ERROR_STATE);
v = (v == version) ? previous_version_count : v + 1;
}
@ -953,15 +957,23 @@ static void ts_parser__halt_parse(TSParser *self) {
ts_stack_position(self->stack, 0)
);
Subtree *filler_node = ts_subtree_new_error(&self->tree_pool, remaining_length, length_zero(), 0, self->language);
filler_node->visible = false;
Subtree filler_node = ts_subtree_new_error(
&self->tree_pool,
0,
length_zero(),
remaining_length,
remaining_length.bytes,
0,
self->language
);
ts_subtree_to_mut_unsafe(filler_node).ptr->visible = false;
ts_stack_push(self->stack, 0, filler_node, false, 0);
SubtreeArray children = array_new();
Subtree *root_error = ts_subtree_new_error_node(&self->tree_pool, &children, self->language);
Subtree root_error = ts_subtree_new_error_node(&self->tree_pool, &children, false, self->language);
ts_stack_push(self->stack, 0, root_error, false, 0);
Subtree *eof = ts_subtree_new_leaf(
Subtree eof = ts_subtree_new_leaf(
&self->tree_pool,
ts_builtin_sym_end,
length_zero(),
@ -999,10 +1011,13 @@ static bool ts_parser__recover_to_state(TSParser *self, StackVersion version, un
SubtreeArray error_trees = ts_stack_pop_error(self->stack, slice.version);
if (error_trees.size > 0) {
assert(error_trees.size == 1);
const Subtree *error_tree = error_trees.contents[0];
array_splice(&slice.subtrees, 0, 0, error_tree->child_count, error_tree->children);
for (unsigned j = 0; j < error_tree->child_count; j++) {
ts_subtree_retain(slice.subtrees.contents[j]);
Subtree error_tree = error_trees.contents[0];
uint32_t error_child_count = ts_subtree_child_count(error_tree);
if (error_child_count > 0) {
array_splice(&slice.subtrees, 0, 0, error_child_count, error_tree.ptr->children);
for (unsigned j = 0; j < error_child_count; j++) {
ts_subtree_retain(slice.subtrees.contents[j]);
}
}
ts_subtree_array_delete(&self->tree_pool, &error_trees);
}
@ -1010,15 +1025,14 @@ static bool ts_parser__recover_to_state(TSParser *self, StackVersion version, un
SubtreeArray trailing_extras = ts_subtree_array_remove_trailing_extras(&slice.subtrees);
if (slice.subtrees.size > 0) {
Subtree *error = ts_subtree_new_error_node(&self->tree_pool, &slice.subtrees, self->language);
error->extra = true;
Subtree error = ts_subtree_new_error_node(&self->tree_pool, &slice.subtrees, true, self->language);
ts_stack_push(self->stack, slice.version, error, false, goal_state);
} else {
array_delete(&slice.subtrees);
}
for (unsigned j = 0; j < trailing_extras.size; j++) {
const Subtree *tree = trailing_extras.contents[j];
Subtree tree = trailing_extras.contents[j];
ts_stack_push(self->stack, slice.version, tree, false, goal_state);
}
@ -1029,7 +1043,7 @@ static bool ts_parser__recover_to_state(TSParser *self, StackVersion version, un
return previous_version != STACK_VERSION_NONE;
}
static void ts_parser__recover(TSParser *self, StackVersion version, const Subtree *lookahead) {
static void ts_parser__recover(TSParser *self, StackVersion version, Subtree lookahead) {
bool did_recover = false;
unsigned previous_version_count = ts_stack_version_count(self->stack);
Length position = ts_stack_position(self->stack, version);
@ -1037,7 +1051,7 @@ static void ts_parser__recover(TSParser *self, StackVersion version, const Subtr
unsigned node_count_since_error = ts_stack_node_count_since_error(self->stack, version);
unsigned current_error_cost = ts_stack_error_cost(self->stack, version);
if (summary && lookahead->symbol != ts_builtin_sym_error) {
if (summary && !ts_subtree_is_error(lookahead)) {
for (unsigned i = 0; i < summary->size; i++) {
StackSummaryEntry entry = summary->contents[i];
@ -1066,7 +1080,7 @@ static void ts_parser__recover(TSParser *self, StackVersion version, const Subtr
(position.extent.row - entry.position.extent.row) * ERROR_COST_PER_SKIPPED_LINE;
if (ts_parser__better_version_exists(self, version, false, new_cost)) break;
if (ts_language_has_actions(self->language, entry.state, lookahead->symbol)) {
if (ts_language_has_actions(self->language, entry.state, ts_subtree_symbol(lookahead))) {
if (ts_parser__recover_to_state(self, version, depth, entry.state)) {
did_recover = true;
LOG("recover_to_previous state:%u, depth:%u", entry.state, depth);
@ -1089,10 +1103,10 @@ static void ts_parser__recover(TSParser *self, StackVersion version, const Subtr
return;
}
if (lookahead->symbol == ts_builtin_sym_end) {
if (ts_subtree_is_eof(lookahead)) {
LOG("recover_eof");
SubtreeArray children = array_new();
const Subtree *parent = ts_subtree_new_error_node(&self->tree_pool, &children, self->language);
Subtree parent = ts_subtree_new_error_node(&self->tree_pool, &children, false, self->language);
ts_stack_push(self->stack, version, parent, false, 1);
ts_parser__accept(self, version, lookahead);
return;
@ -1110,18 +1124,18 @@ static void ts_parser__recover(TSParser *self, StackVersion version, const Subtr
}
unsigned n;
const TSParseAction *actions = ts_language_actions(self->language, 1, lookahead->symbol, &n);
const TSParseAction *actions = ts_language_actions(self->language, 1, ts_subtree_symbol(lookahead), &n);
if (n > 0 && actions[n - 1].type == TSParseActionTypeShift && actions[n - 1].params.extra) {
Subtree *mutable_lookahead = ts_subtree_make_mut(&self->tree_pool, lookahead);
mutable_lookahead->extra = true;
lookahead = mutable_lookahead;
MutableSubtree mutable_lookahead = ts_subtree_make_mut(&self->tree_pool, lookahead);
ts_subtree_set_extra(&mutable_lookahead);
lookahead = ts_subtree_from_mut(mutable_lookahead);
}
LOG("skip_token symbol:%s", SYM_NAME(lookahead->symbol));
LOG("skip_token symbol:%s", TREE_NAME(lookahead));
SubtreeArray children = array_new();
array_reserve(&children, 1);
array_push(&children, lookahead);
const Subtree *error_repeat = ts_subtree_new_node(
MutableSubtree error_repeat = ts_subtree_new_node(
&self->tree_pool,
ts_builtin_sym_error_repeat,
&children,
@ -1134,7 +1148,7 @@ static void ts_parser__recover(TSParser *self, StackVersion version, const Subtr
assert(pop.size == 1);
assert(pop.contents[0].subtrees.size == 1);
ts_stack_renumber_version(self->stack, pop.contents[0].version, version);
array_push(&pop.contents[0].subtrees, error_repeat);
array_push(&pop.contents[0].subtrees, ts_subtree_from_mut(error_repeat));
error_repeat = ts_subtree_new_node(
&self->tree_pool,
ts_builtin_sym_error_repeat,
@ -1144,9 +1158,9 @@ static void ts_parser__recover(TSParser *self, StackVersion version, const Subtr
);
}
ts_stack_push(self->stack, version, error_repeat, false, ERROR_STATE);
ts_stack_push(self->stack, version, ts_subtree_from_mut(error_repeat), false, ERROR_STATE);
if (lookahead->has_external_tokens) {
if (ts_subtree_has_external_tokens(lookahead)) {
ts_stack_set_last_external_token(
self->stack, version, ts_subtree_last_external_token(lookahead)
);
@ -1156,11 +1170,11 @@ static void ts_parser__recover(TSParser *self, StackVersion version, const Subtr
static void ts_parser__advance(TSParser *self, StackVersion version, bool allow_node_reuse) {
TSStateId state = ts_stack_state(self->stack, version);
uint32_t position = ts_stack_position(self->stack, version).bytes;
const Subtree *last_external_token = ts_stack_last_external_token(self->stack, version);
Subtree last_external_token = ts_stack_last_external_token(self->stack, version);
bool did_reuse = true;
const Subtree *lookahead = NULL;
TableEntry table_entry;
Subtree lookahead = NULL_SUBTREE;
TableEntry table_entry = {.action_count = 0};
// If possible, reuse a node from the previous syntax tree.
if (allow_node_reuse) {
@ -1170,7 +1184,7 @@ static void ts_parser__advance(TSParser *self, StackVersion version, bool allow_
}
// Otherwise, try to reuse the token previously returned by the lexer.
if (!lookahead) {
if (!lookahead.ptr) {
did_reuse = false;
lookahead = ts_parser__get_cached_token(
self, state, position, last_external_token, &table_entry
@ -1178,10 +1192,10 @@ static void ts_parser__advance(TSParser *self, StackVersion version, bool allow_
}
// Otherwise, re-run the lexer.
if (!lookahead) {
if (!lookahead.ptr) {
lookahead = ts_parser__lex(self, version, state);
ts_parser__set_cached_token(self, position, last_external_token, lookahead);
ts_language_table_entry(self->language, state, lookahead->symbol, &table_entry);
ts_language_table_entry(self->language, state, ts_subtree_symbol(lookahead), &table_entry);
}
for (;;) {
@ -1206,9 +1220,9 @@ static void ts_parser__advance(TSParser *self, StackVersion version, bool allow_
LOG("shift state:%u", next_state);
}
if (lookahead->child_count > 0) {
if (ts_subtree_child_count(lookahead) > 0) {
ts_parser__breakdown_lookahead(self, &lookahead, state, &self->reusable_node);
next_state = ts_language_next_state(self->language, state, lookahead->symbol);
next_state = ts_language_next_state(self->language, state, ts_subtree_symbol(lookahead));
}
ts_parser__shift(self, version, next_state, lookahead, action.params.extra);
@ -1237,7 +1251,7 @@ static void ts_parser__advance(TSParser *self, StackVersion version, bool allow_
}
case TSParseActionTypeRecover: {
if (lookahead->child_count > 0) {
if (ts_subtree_child_count(lookahead) > 0) {
ts_parser__breakdown_lookahead(self, &lookahead, ERROR_STATE, &self->reusable_node);
}
@ -1261,18 +1275,21 @@ static void ts_parser__advance(TSParser *self, StackVersion version, bool allow_
continue;
}
if (lookahead->is_keyword && lookahead->symbol != self->language->keyword_capture_token) {
if (
ts_subtree_is_keyword(lookahead) &&
ts_subtree_symbol(lookahead) != self->language->keyword_capture_token
) {
ts_language_table_entry(self->language, state, self->language->keyword_capture_token, &table_entry);
if (table_entry.action_count > 0) {
LOG(
"switch from_keyword:%s, to_word_token:%s",
SYM_NAME(lookahead->symbol),
TREE_NAME(lookahead),
SYM_NAME(self->language->keyword_capture_token)
);
Subtree *mutable_lookahead = ts_subtree_make_mut(&self->tree_pool, lookahead);
mutable_lookahead->symbol = self->language->keyword_capture_token;
lookahead = mutable_lookahead;
MutableSubtree mutable_lookahead = ts_subtree_make_mut(&self->tree_pool, lookahead);
ts_subtree_set_symbol(&mutable_lookahead, self->language->keyword_capture_token);
lookahead = ts_subtree_from_mut(mutable_lookahead);
continue;
}
}
@ -1395,14 +1412,15 @@ TSParser *ts_parser_new() {
array_reserve(&self->reduce_actions, 4);
self->tree_pool = ts_subtree_pool_new(32);
self->stack = ts_stack_new(&self->tree_pool);
self->finished_tree = NULL;
self->finished_tree = NULL_SUBTREE;
self->reusable_node = reusable_node_new();
self->dot_graph_file = NULL;
self->halt_on_error = false;
self->enabled = true;
self->operation_limit = SIZE_MAX;
self->old_tree = NULL;
ts_parser__set_cached_token(self, 0, NULL, NULL);
self->old_tree = NULL_SUBTREE;
self->scratch_tree.ptr = &self->scratch_tree_data;
ts_parser__set_cached_token(self, 0, NULL_SUBTREE, NULL_SUBTREE);
return self;
}
@ -1411,12 +1429,12 @@ void ts_parser_delete(TSParser *self) {
if (self->reduce_actions.contents) {
array_delete(&self->reduce_actions);
}
if (self->old_tree) {
if (self->old_tree.ptr) {
ts_subtree_release(&self->tree_pool, self->old_tree);
self->old_tree = NULL;
self->old_tree = NULL_SUBTREE;
}
ts_lexer_delete(&self->lexer);
ts_parser__set_cached_token(self, 0, NULL, NULL);
ts_parser__set_cached_token(self, 0, NULL_SUBTREE, NULL_SUBTREE);
ts_subtree_pool_delete(&self->tree_pool);
reusable_node_delete(&self->reusable_node);
ts_parser_set_language(self, NULL);
@ -1489,18 +1507,18 @@ void ts_parser_reset(TSParser *self) {
self->language->external_scanner.deserialize(self->external_scanner_payload, NULL, 0);
}
if (self->old_tree) {
if (self->old_tree.ptr) {
ts_subtree_release(&self->tree_pool, self->old_tree);
self->old_tree = NULL;
self->old_tree = NULL_SUBTREE;
}
reusable_node_clear(&self->reusable_node);
ts_lexer_reset(&self->lexer, length_zero());
ts_stack_clear(self->stack);
ts_parser__set_cached_token(self, 0, NULL, NULL);
if (self->finished_tree) {
ts_parser__set_cached_token(self, 0, NULL_SUBTREE, NULL_SUBTREE);
if (self->finished_tree.ptr) {
ts_subtree_release(&self->tree_pool, self->finished_tree);
self->finished_tree = NULL;
self->finished_tree = NULL_SUBTREE;
}
self->accept_count = 0;
}
@ -1552,7 +1570,7 @@ TSTree *ts_parser_parse(TSParser *self, const TSTree *old_tree, TSInput input) {
}
unsigned min_error_cost = ts_parser__condense_stack(self);
if (self->finished_tree && self->finished_tree->error_cost < min_error_cost) {
if (self->finished_tree.ptr && ts_subtree_error_cost(self->finished_tree) < min_error_cost) {
break;
} else if (self->halt_on_error && min_error_cost > 0) {
ts_parser__halt_parse(self);
@ -1565,7 +1583,7 @@ TSTree *ts_parser_parse(TSParser *self, const TSTree *old_tree, TSInput input) {
LOG_TREE();
TSTree *result = ts_tree_new(self->finished_tree, self->language);
self->finished_tree = NULL;
self->finished_tree = NULL_SUBTREE;
ts_parser_reset(self);
return result;
}

26
src/runtime/reusable_node.h
View file

@ -1,26 +1,26 @@
#include "runtime/subtree.h"
typedef struct {
const Subtree *tree;
Subtree tree;
uint32_t child_index;
uint32_t byte_offset;
} StackEntry;
typedef struct {
Array(StackEntry) stack;
const Subtree *last_external_token;
Subtree last_external_token;
} ReusableNode;
static inline ReusableNode reusable_node_new() {
return (ReusableNode) {array_new(), NULL};
return (ReusableNode) {array_new(), (Subtree) {.ptr = NULL}};
}
static inline void reusable_node_clear(ReusableNode *self) {
array_clear(&self->stack);
self->last_external_token = NULL;
self->last_external_token.ptr = NULL;
}
static inline void reusable_node_reset(ReusableNode *self, const Subtree *tree) {
static inline void reusable_node_reset(ReusableNode *self, Subtree tree) {
reusable_node_clear(self);
array_push(&self->stack, ((StackEntry) {
.tree = tree,
@ -29,10 +29,10 @@ static inline void reusable_node_reset(ReusableNode *self, const Subtree *tree)
}));
}
static inline const Subtree *reusable_node_tree(ReusableNode *self) {
static inline Subtree reusable_node_tree(ReusableNode *self) {
return self->stack.size > 0
? self->stack.contents[self->stack.size - 1].tree
: NULL;
: (Subtree) { .ptr = NULL };
}
static inline uint32_t reusable_node_byte_offset(ReusableNode *self) {
@ -48,21 +48,21 @@ static inline void reusable_node_delete(ReusableNode *self) {
static inline void reusable_node_advance(ReusableNode *self) {
StackEntry last_entry = *array_back(&self->stack);
uint32_t byte_offset = last_entry.byte_offset + ts_subtree_total_bytes(last_entry.tree);
if (last_entry.tree->has_external_tokens) {
if (ts_subtree_has_external_tokens(last_entry.tree)) {
self->last_external_token = ts_subtree_last_external_token(last_entry.tree);
}
const Subtree *tree;
Subtree tree;
uint32_t next_index;
do {
StackEntry popped_entry = array_pop(&self->stack);
next_index = popped_entry.child_index + 1;
if (self->stack.size == 0) return;
tree = array_back(&self->stack)->tree;
} while (tree->child_count <= next_index);
} while (ts_subtree_child_count(tree) <= next_index);
array_push(&self->stack, ((StackEntry) {
.tree = tree->children[next_index],
.tree = tree.ptr->children[next_index],
.child_index = next_index,
.byte_offset = byte_offset,
}));
@ -70,9 +70,9 @@ static inline void reusable_node_advance(ReusableNode *self) {
static inline bool reusable_node_descend(ReusableNode *self) {
StackEntry last_entry = *array_back(&self->stack);
if (last_entry.tree->child_count > 0) {
if (ts_subtree_child_count(last_entry.tree) > 0) {
array_push(&self->stack, ((StackEntry) {
.tree = last_entry.tree->children[0],
.tree = last_entry.tree.ptr->children[0],
.child_index = 0,
.byte_offset = last_entry.byte_offset,
}));

140
src/runtime/stack.c
View file

@ -21,7 +21,7 @@ typedef struct StackNode StackNode;
typedef struct {
StackNode *node;
const Subtree *subtree;
Subtree subtree;
bool is_pending;
} StackLink;
@ -58,7 +58,7 @@ typedef enum {
typedef struct {
StackNode *node;
const Subtree *last_external_token;
Subtree last_external_token;
StackSummary *summary;
unsigned node_count_at_last_error;
TSSymbol lookahead_when_paused;
@ -100,10 +100,12 @@ recur:
StackNode *first_predecessor = NULL;
if (self->link_count > 0) {
for (unsigned i = self->link_count - 1; i > 0; i--) {
if (self->links[i].subtree) ts_subtree_release(subtree_pool, self->links[i].subtree);
stack_node_release(self->links[i].node, pool, subtree_pool);
StackLink link = self->links[i];
if (link.subtree.ptr) ts_subtree_release(subtree_pool, link.subtree);
stack_node_release(link.node, pool, subtree_pool);
}
if (self->links[0].subtree) ts_subtree_release(subtree_pool, self->links[0].subtree);
StackLink link = self->links[0];
if (link.subtree.ptr) ts_subtree_release(subtree_pool, link.subtree);
first_predecessor = self->links[0].node;
}
@ -119,7 +121,7 @@ recur:
}
}
static StackNode *stack_node_new(StackNode *previous_node, const Subtree *subtree,
static StackNode *stack_node_new(StackNode *previous_node, Subtree subtree,
bool is_pending, TSStateId state, StackNodeArray *pool) {
StackNode *node = pool->size > 0 ?
array_pop(pool) :
@ -139,15 +141,11 @@ static StackNode *stack_node_new(StackNode *previous_node, const Subtree *subtre
node->dynamic_precedence = previous_node->dynamic_precedence;
node->node_count = previous_node->node_count;
if (subtree) {
node->error_cost += subtree->error_cost;
if (subtree.ptr) {
node->error_cost += ts_subtree_error_cost(subtree);
node->position = length_add(node->position, ts_subtree_total_size(subtree));
if (subtree->child_count) {
node->node_count += subtree->node_count;
node->dynamic_precedence += subtree->dynamic_precedence;
} else {
node->node_count++;
}
node->node_count += ts_subtree_node_count(subtree);
node->dynamic_precedence += ts_subtree_dynamic_precedence(subtree);
}
} else {
node->position = length_zero();
@ -157,17 +155,16 @@ static StackNode *stack_node_new(StackNode *previous_node, const Subtree *subtre
return node;
}
static bool stack__subtree_is_equivalent(const Subtree *left, const Subtree *right) {
static bool stack__subtree_is_equivalent(Subtree left, Subtree right) {
return
left == right ||
(left &&
right &&
left->symbol == right->symbol &&
((left->error_cost > 0 && right->error_cost > 0) ||
(left->padding.bytes == right->padding.bytes &&
left->size.bytes == right->size.bytes &&
left->child_count == right->child_count &&
left->extra == right->extra &&
left.ptr == right.ptr ||
(left.ptr && right.ptr &&
ts_subtree_symbol(left) == ts_subtree_symbol(right) &&
((ts_subtree_error_cost(left) > 0 && ts_subtree_error_cost(right) > 0) ||
(ts_subtree_padding(left).bytes == ts_subtree_padding(right).bytes &&
ts_subtree_size(left).bytes == ts_subtree_size(right).bytes &&
ts_subtree_child_count(left) == ts_subtree_child_count(right) &&
ts_subtree_extra(left) == ts_subtree_extra(right) &&
ts_subtree_external_scanner_state_eq(left, right))));
}
@ -183,13 +180,14 @@ static void stack_node_add_link(StackNode *self, StackLink link, SubtreePool *su
// we can safely remove the ambiguity ahead of time without changing behavior.
if (existing_link->node == link.node) {
if (
link.subtree->child_count > 0 &&
link.subtree->dynamic_precedence > existing_link->subtree->dynamic_precedence
ts_subtree_dynamic_precedence(link.subtree) >
ts_subtree_dynamic_precedence(existing_link->subtree)
) {
ts_subtree_retain(link.subtree);
ts_subtree_release(subtree_pool, existing_link->subtree);
existing_link->subtree = link.subtree;
self->dynamic_precedence = link.node->dynamic_precedence + link.subtree->dynamic_precedence;
self->dynamic_precedence =
link.node->dynamic_precedence + ts_subtree_dynamic_precedence(link.subtree);
}
return;
}
@ -200,8 +198,10 @@ static void stack_node_add_link(StackNode *self, StackLink link, SubtreePool *su
for (int j = 0; j < link.node->link_count; j++) {
stack_node_add_link(existing_link->node, link.node->links[j], subtree_pool);
}
int dynamic_precedence = link.node->dynamic_precedence;
if (link.subtree) dynamic_precedence += link.subtree->dynamic_precedence;
int32_t dynamic_precedence = link.node->dynamic_precedence;
if (link.subtree.ptr) {
dynamic_precedence += ts_subtree_dynamic_precedence(link.subtree);
}
if (dynamic_precedence > self->dynamic_precedence) {
self->dynamic_precedence = dynamic_precedence;
}
@ -217,14 +217,10 @@ static void stack_node_add_link(StackNode *self, StackLink link, SubtreePool *su
int dynamic_precedence = link.node->dynamic_precedence;
self->links[self->link_count++] = link;
if (link.subtree) {
if (link.subtree.ptr) {
ts_subtree_retain(link.subtree);
if (link.subtree->child_count > 0) {
node_count += link.subtree->node_count;
dynamic_precedence += link.subtree->dynamic_precedence;
} else {
node_count++;
}
node_count += ts_subtree_node_count(link.subtree);
dynamic_precedence += ts_subtree_dynamic_precedence(link.subtree);
}
if (node_count > self->node_count) self->node_count = node_count;
@ -233,7 +229,7 @@ static void stack_node_add_link(StackNode *self, StackLink link, SubtreePool *su
static void stack_head_delete(StackHead *self, StackNodeArray *pool, SubtreePool *subtree_pool) {
if (self->node) {
if (self->last_external_token) {
if (self->last_external_token.ptr) {
ts_subtree_release(subtree_pool, self->last_external_token);
}
if (self->summary) {
@ -255,7 +251,7 @@ static StackVersion ts_stack__add_version(Stack *self, StackVersion original_ver
};
array_push(&self->heads, head);
stack_node_retain(node);
if (head.last_external_token) ts_subtree_retain(head.last_external_token);
if (head.last_external_token.ptr) ts_subtree_retain(head.last_external_token);
return (StackVersion)(self->heads.size - 1);
}
@ -343,13 +339,13 @@ inline StackSliceArray stack__iter(Stack *self, StackVersion version,
}
next_iterator->node = link.node;
if (link.subtree) {
if (link.subtree.ptr) {
if (include_subtrees) {
array_push(&next_iterator->subtrees, link.subtree);
ts_subtree_retain(link.subtree);
}
if (!link.subtree->extra) {
if (!ts_subtree_extra(link.subtree)) {
next_iterator->subtree_count++;
if (!link.is_pending) {
next_iterator->is_pending = false;
@ -379,7 +375,7 @@ Stack *ts_stack_new(SubtreePool *subtree_pool) {
array_reserve(&self->node_pool, MAX_NODE_POOL_SIZE);
self->subtree_pool = subtree_pool;
self->base_node = stack_node_new(NULL, NULL, false, 1, &self->node_pool);
self->base_node = stack_node_new(NULL, (Subtree) {.ptr = NULL}, false, 1, &self->node_pool);
ts_stack_clear(self);
return self;
@ -416,14 +412,14 @@ Length ts_stack_position(const Stack *self, StackVersion version) {
return array_get(&self->heads, version)->node->position;
}
const Subtree *ts_stack_last_external_token(const Stack *self, StackVersion version) {
Subtree ts_stack_last_external_token(const Stack *self, StackVersion version) {
return array_get(&self->heads, version)->last_external_token;
}
void ts_stack_set_last_external_token(Stack *self, StackVersion version, const Subtree *token) {
void ts_stack_set_last_external_token(Stack *self, StackVersion version, Subtree token) {
StackHead *head = array_get(&self->heads, version);
if (token) ts_subtree_retain(token);
if (head->last_external_token) ts_subtree_release(self->subtree_pool, head->last_external_token);
if (token.ptr) ts_subtree_retain(token);
if (head->last_external_token.ptr) ts_subtree_release(self->subtree_pool, head->last_external_token);
head->last_external_token = token;
}
@ -432,7 +428,7 @@ unsigned ts_stack_error_cost(const Stack *self, StackVersion version) {
unsigned result = head->node->error_cost;
if (
head->status == StackStatusPaused ||
(head->node->state == ERROR_STATE && !head->node->links[0].subtree)) {
(head->node->state == ERROR_STATE && !head->node->links[0].subtree.ptr)) {
result += ERROR_COST_PER_RECOVERY;
}
return result;
@ -446,11 +442,11 @@ unsigned ts_stack_node_count_since_error(const Stack *self, StackVersion version
return head->node->node_count - head->node_count_at_last_error;
}
void ts_stack_push(Stack *self, StackVersion version, const Subtree *subtree,
void ts_stack_push(Stack *self, StackVersion version, Subtree subtree,
bool pending, TSStateId state) {
StackHead *head = array_get(&self->heads, version);
StackNode *new_node = stack_node_new(head->node, subtree, pending, state, &self->node_pool);
if (!subtree) head->node_count_at_last_error = new_node->node_count;
if (!subtree.ptr) head->node_count_at_last_error = new_node->node_count;
head->node = new_node;
}
@ -507,7 +503,7 @@ StackSliceArray ts_stack_pop_pending(Stack *self, StackVersion version) {
inline StackAction pop_error_callback(void *payload, const Iterator *iterator) {
if (iterator->subtrees.size > 0) {
bool *found_error = payload;
if (!*found_error && iterator->subtrees.contents[0]->symbol == ts_builtin_sym_error) {
if (!*found_error && ts_subtree_is_error(iterator->subtrees.contents[0])) {
*found_error = true;
return StackActionPop | StackActionStop;
} else {
@ -521,7 +517,7 @@ inline StackAction pop_error_callback(void *payload, const Iterator *iterator) {
SubtreeArray ts_stack_pop_error(Stack *self, StackVersion version) {
StackNode *node = array_get(&self->heads, version)->node;
for (unsigned i = 0; i < node->link_count; i++) {
if (node->links[i].subtree && node->links[i].subtree->symbol == ts_builtin_sym_error) {
if (node->links[i].subtree.ptr && ts_subtree_is_error(node->links[i].subtree)) {
bool found_error = false;
StackSliceArray pop = stack__iter(self, version, pop_error_callback, &found_error, 1);
if (pop.size > 0) {
@ -615,7 +611,7 @@ StackVersion ts_stack_copy_version(Stack *self, StackVersion version) {
array_push(&self->heads, self->heads.contents[version]);
StackHead *head = array_back(&self->heads);
stack_node_retain(head->node);
if (head->last_external_token) ts_subtree_retain(head->last_external_token);
if (head->last_external_token.ptr) ts_subtree_retain(head->last_external_token);
head->summary = NULL;
return self->heads.size - 1;
}
@ -686,7 +682,7 @@ void ts_stack_clear(Stack *self) {
array_clear(&self->heads);
array_push(&self->heads, ((StackHead){
.node = self->base_node,
.last_external_token = NULL,
.last_external_token.ptr = NULL,
.status = StackStatusActive,
.lookahead_when_paused = 0,
}));
@ -721,8 +717,8 @@ bool ts_stack_print_dot_graph(Stack *self, const TSLanguage *language, FILE *f)
ts_stack_error_cost(self, i)
);
if (head->last_external_token) {
const ExternalScannerState *state = &head->last_external_token->external_scanner_state;
if (head->last_external_token.ptr) {
const ExternalScannerState *state = &head->last_external_token.ptr->external_scanner_state;
const char *data = ts_external_scanner_state_data(state);
fprintf(f, "\nexternal_scanner_state:");
for (uint32_t j = 0; j < state->length; j++) fprintf(f, " %2X", data[j]);
@ -751,13 +747,17 @@ bool ts_stack_print_dot_graph(Stack *self, const TSLanguage *language, FILE *f)
all_iterators_done = false;
fprintf(f, "node_%p [", node);
if (node->state == ERROR_STATE)
if (node->state == ERROR_STATE) {
fprintf(f, "label=\"?\"");
else if (node->link_count == 1 && node->links[0].subtree &&
node->links[0].subtree->extra)
} else if (
node->link_count == 1 &&
node->links[0].subtree.ptr &&
ts_subtree_extra(node->links[0].subtree)
) {
fprintf(f, "shape=point margin=0 label=\"\"");
else
} else {
fprintf(f, "label=\"%d\"", node->state);
}
fprintf(
f,
@ -772,26 +772,28 @@ bool ts_stack_print_dot_graph(Stack *self, const TSLanguage *language, FILE *f)
for (int j = 0; j < node->link_count; j++) {
StackLink link = node->links[j];
fprintf(f, "node_%p -> node_%p [", node, link.node);
if (link.is_pending)
fprintf(f, "style=dashed ");
if (link.subtree && link.subtree->extra)
fprintf(f, "fontcolor=gray ");
if (link.is_pending) fprintf(f, "style=dashed ");
if (link.subtree.ptr && ts_subtree_extra(link.subtree)) fprintf(f, "fontcolor=gray ");
if (!link.subtree) {
if (!link.subtree.ptr) {
fprintf(f, "color=red");
} else {
fprintf(f, "label=\"");
if (link.subtree->visible && !link.subtree->named) fprintf(f, "'");
const char *name = ts_language_symbol_name(language, link.subtree->symbol);
bool quoted = ts_subtree_visible(link.subtree) && !ts_subtree_named(link.subtree);
if (quoted) fprintf(f, "'");
const char *name = ts_language_symbol_name(language, ts_subtree_symbol(link.subtree));
for (const char *c = name; *c; c++) {
if (*c == '\"' || *c == '\\') fprintf(f, "\\");
fprintf(f, "%c", *c);
}
if (link.subtree->visible && !link.subtree->named) fprintf(f, "'");
if (quoted) fprintf(f, "'");
fprintf(f, "\"");
fprintf(f, "labeltooltip=\"error_cost: %u\ndynamic_precedence: %u\"",
link.subtree->error_cost,
link.subtree->dynamic_precedence);
fprintf(
f,
"labeltooltip=\"error_cost: %u\ndynamic_precedence: %u\"",
ts_subtree_error_cost(link.subtree),
ts_subtree_dynamic_precedence(link.subtree)
);
}
fprintf(f, "];\n");

6
src/runtime/stack.h
View file

@ -42,10 +42,10 @@ uint32_t ts_stack_version_count(const Stack *);
TSStateId ts_stack_state(const Stack *, StackVersion);
// Get the last external token associated with a given version of the stack.
const Subtree *ts_stack_last_external_token(const Stack *, StackVersion);
Subtree ts_stack_last_external_token(const Stack *, StackVersion);
// Set the last external token associated with a given version of the stack.
void ts_stack_set_last_external_token(Stack *, StackVersion, const Subtree *);
void ts_stack_set_last_external_token(Stack *, StackVersion, Subtree );
// Get the position of the given version of the stack within the document.
Length ts_stack_position(const Stack *, StackVersion);
@ -55,7 +55,7 @@ Length ts_stack_position(const Stack *, StackVersion);
// This transfers ownership of the tree to the Stack. Callers that
// need to retain ownership of the tree for their own purposes should
// first retain the tree.
void ts_stack_push(Stack *, StackVersion, const Subtree *, bool, TSStateId);
void ts_stack_push(Stack *, StackVersion, Subtree , bool, TSStateId);
// Pop the given number of entries from the given version of the stack. This
// operation can increase the number of stack versions by revealing multiple

800
src/runtime/subtree.c
View file

File diff suppressed because it is too large Load diff

229
src/runtime/subtree.h
View file

@ -9,10 +9,15 @@ extern "C" {
#include <stdio.h>
#include "runtime/length.h"
#include "runtime/array.h"
#include "runtime/error_costs.h"
#include "tree_sitter/runtime.h"
#include "tree_sitter/parser.h"
extern TSStateId TS_TREE_STATE_NONE;
#define NULL_SUBTREE ((Subtree) {.ptr = NULL})
typedef union Subtree Subtree;
typedef union MutableSubtree MutableSubtree;
typedef struct {
union {
@ -22,9 +27,24 @@ typedef struct {
uint32_t length;
} ExternalScannerState;
typedef struct Subtree Subtree;
typedef struct {
bool is_inline : 1;
bool visible : 1;
bool named : 1;
bool extra : 1;
bool has_changes : 1;
bool is_missing : 1;
bool is_keyword : 1;
uint8_t symbol;
uint8_t padding_bytes;
uint8_t size_bytes;
uint8_t padding_columns;
uint8_t padding_rows : 4;
uint8_t additional_bytes_scanned : 4;
uint16_t parse_state;
} SubtreeInlineData;
struct Subtree {
typedef struct {
volatile uint32_t ref_count;
Length padding;
Length size;
@ -34,7 +54,6 @@ struct Subtree {
TSSymbol symbol;
TSStateId parse_state;
bool is_small : 1;
bool visible : 1;
bool named : 1;
bool extra : 1;
@ -48,7 +67,7 @@ struct Subtree {
union {
// Non-terminal subtrees (`child_count > 0`)
struct {
const Subtree **children;
Subtree *children;
uint32_t visible_child_count;
uint32_t named_child_count;
uint32_t node_count;
@ -67,14 +86,23 @@ struct Subtree {
// Error terminal subtrees (`child_count == 0 && symbol == ts_builtin_sym_error`)
int32_t lookahead_char;
};
} SubtreeHeapData;
union Subtree {
SubtreeInlineData data;
const SubtreeHeapData *ptr;
};
typedef Array(const Subtree *) SubtreeArray;
typedef Array(Subtree *) MutableSubtreeArray;
union MutableSubtree {
SubtreeInlineData data;
SubtreeHeapData *ptr;
};
typedef Array(Subtree) SubtreeArray;
typedef Array(MutableSubtree) MutableSubtreeArray;
typedef struct {
MutableSubtreeArray free_trees;
MutableSubtreeArray free_small_trees;
MutableSubtreeArray tree_stack;
} SubtreePool;
@ -88,41 +116,176 @@ void ts_subtree_array_reverse(SubtreeArray *);
SubtreePool ts_subtree_pool_new(uint32_t capacity);
void ts_subtree_pool_delete(SubtreePool *);
Subtree *ts_subtree_pool_allocate(SubtreePool *, bool);
void ts_subtree_pool_free(SubtreePool *, Subtree *);
Subtree *ts_subtree_new_leaf(
Subtree ts_subtree_new_leaf(
SubtreePool *, TSSymbol, Length, Length, uint32_t,
TSStateId, bool, bool, const TSLanguage *
);
Subtree *ts_subtree_new_node(SubtreePool *, TSSymbol, SubtreeArray *, unsigned, const TSLanguage *);
Subtree *ts_subtree_new_copy(SubtreePool *, const Subtree *);
Subtree *ts_subtree_new_error_node(SubtreePool *, SubtreeArray *, const TSLanguage *);
Subtree *ts_subtree_new_error(SubtreePool *, Length, Length, int32_t, const TSLanguage *);
Subtree *ts_subtree_new_missing_leaf(SubtreePool *, TSSymbol, Length, const TSLanguage *);
Subtree *ts_subtree_make_mut(SubtreePool *, const Subtree *);
void ts_subtree_retain(const Subtree *tree);
void ts_subtree_release(SubtreePool *, const Subtree *tree);
bool ts_subtree_eq(const Subtree *tree1, const Subtree *tree2);
int ts_subtree_compare(const Subtree *tree1, const Subtree *tree2);
void ts_subtree_set_children(Subtree *, const Subtree **, uint32_t, const TSLanguage *);
void ts_subtree_balance(const Subtree *, SubtreePool *, const TSLanguage *);
const Subtree *ts_subtree_edit(const Subtree *, const TSInputEdit *edit, SubtreePool *);
char *ts_subtree_string(const Subtree *, const TSLanguage *, bool include_all);
void ts_subtree_print_dot_graph(const Subtree *, const TSLanguage *, FILE *);
const Subtree *ts_subtree_last_external_token(const Subtree *);
bool ts_subtree_external_scanner_state_eq(const Subtree *, const Subtree *);
Subtree ts_subtree_new_error(
SubtreePool *, int32_t, Length, Length, uint32_t, TSStateId, const TSLanguage *
);
MutableSubtree ts_subtree_new_node(SubtreePool *, TSSymbol, SubtreeArray *, unsigned, const TSLanguage *);
Subtree ts_subtree_new_error_node(SubtreePool *, SubtreeArray *, bool, const TSLanguage *);
Subtree ts_subtree_new_missing_leaf(SubtreePool *, TSSymbol, Length, const TSLanguage *);
MutableSubtree ts_subtree_make_mut(SubtreePool *, Subtree);
void ts_subtree_retain(Subtree);
void ts_subtree_release(SubtreePool *, Subtree);
bool ts_subtree_eq(Subtree, Subtree);
int ts_subtree_compare(Subtree, Subtree);
void ts_subtree_set_children(MutableSubtree, Subtree *, uint32_t, const TSLanguage *);
void ts_subtree_balance(Subtree, SubtreePool *, const TSLanguage *);
Subtree ts_subtree_edit(Subtree, const TSInputEdit *edit, SubtreePool *);
char *ts_subtree_string(Subtree, const TSLanguage *, bool include_all);
void ts_subtree_print_dot_graph(Subtree, const TSLanguage *, FILE *);
Subtree ts_subtree_last_external_token(Subtree);
bool ts_subtree_external_scanner_state_eq(Subtree, Subtree);
static inline uint32_t ts_subtree_total_bytes(const Subtree *self) {
return self->padding.bytes + self->size.bytes;
#define SUBTREE_GET(self, name) (self.data.is_inline ? self.data.name : self.ptr->name)
static inline TSSymbol ts_subtree_symbol(Subtree self) { return SUBTREE_GET(self, symbol); }
static inline bool ts_subtree_visible(Subtree self) { return SUBTREE_GET(self, visible); }
static inline bool ts_subtree_named(Subtree self) { return SUBTREE_GET(self, named); }
static inline bool ts_subtree_extra(Subtree self) { return SUBTREE_GET(self, extra); }
static inline bool ts_subtree_has_changes(Subtree self) { return SUBTREE_GET(self, has_changes); }
static inline bool ts_subtree_missing(Subtree self) { return SUBTREE_GET(self, is_missing); }
static inline bool ts_subtree_is_keyword(Subtree self) { return SUBTREE_GET(self, is_keyword); }
static inline TSStateId ts_subtree_parse_state(Subtree self) { return SUBTREE_GET(self, parse_state); }
#undef SUBTREE_GET
static inline void ts_subtree_set_symbol(MutableSubtree *self, TSSymbol symbol) {
if (self->data.is_inline) {
assert(symbol < UINT8_MAX);
self->data.symbol = symbol;
} else {
self->ptr->symbol = symbol;
}
}
static inline Length ts_subtree_total_size(const Subtree *self) {
return length_add(self->padding, self->size);
static inline void ts_subtree_set_extra(MutableSubtree *self) {
if (self->data.is_inline) {
self->data.extra = true;
} else {
self->ptr->extra = true;
}
}
static inline TSSymbol ts_subtree_leaf_symbol(const Subtree *self) {
return self->child_count > 0 ? self->first_leaf.symbol : self->symbol;
static inline TSSymbol ts_subtree_leaf_symbol(Subtree self) {
if (self.data.is_inline) return self.data.symbol;
if (self.ptr->child_count == 0) return self.ptr->symbol;
return self.ptr->first_leaf.symbol;
}
static inline TSStateId ts_subtree_leaf_parse_state(Subtree self) {
if (self.data.is_inline) return self.data.parse_state;
if (self.ptr->child_count == 0) return self.ptr->parse_state;
return self.ptr->first_leaf.parse_state;
}
static inline Length ts_subtree_padding(Subtree self) {
if (self.data.is_inline) {
Length result = {self.data.padding_bytes, {self.data.padding_rows, self.data.padding_columns}};
return result;
} else {
return self.ptr->padding;
}
}
static inline Length ts_subtree_size(Subtree self) {
if (self.data.is_inline) {
Length result = {self.data.size_bytes, {0, self.data.size_bytes}};
return result;
} else {
return self.ptr->size;
}
}
static inline Length ts_subtree_total_size(Subtree self) {
return length_add(ts_subtree_padding(self), ts_subtree_size(self));
}
static inline uint32_t ts_subtree_total_bytes(Subtree self) {
return ts_subtree_total_size(self).bytes;
}
static inline uint32_t ts_subtree_bytes_scanned(Subtree self) {
return self.data.is_inline
? (uint32_t)self.data.padding_bytes +
(uint32_t)self.data.size_bytes +
(uint32_t)self.data.additional_bytes_scanned
: self.ptr->bytes_scanned;
}
static inline uint32_t ts_subtree_child_count(Subtree self) {
return self.data.is_inline ? 0 : self.ptr->child_count;
}
static inline uint32_t ts_subtree_node_count(Subtree self) {
return (self.data.is_inline || self.ptr->child_count == 0) ? 1 : self.ptr->node_count;
}
static inline uint32_t ts_subtree_visible_child_count(Subtree self) {
if (ts_subtree_child_count(self) > 0) {
return self.ptr->visible_child_count;
} else {
return 0;
}
}
static inline uint32_t ts_subtree_error_cost(Subtree self) {
if (ts_subtree_missing(self)) {
return ERROR_COST_PER_MISSING_TREE + ERROR_COST_PER_RECOVERY;
} else {
return self.data.is_inline ? 0 : self.ptr->error_cost;
}
}
static inline int32_t ts_subtree_dynamic_precedence(Subtree self) {
return (self.data.is_inline || self.ptr->child_count == 0) ? 0 : self.ptr->dynamic_precedence;
}
static inline uint16_t ts_subtree_alias_sequence_id(Subtree self) {
if (ts_subtree_child_count(self) > 0) {
return self.ptr->alias_sequence_id;
} else {
return 0;
}
}
static inline bool ts_subtree_fragile_left(Subtree self) {
return self.data.is_inline ? false : self.ptr->fragile_left;
}
static inline bool ts_subtree_fragile_right(Subtree self) {
return self.data.is_inline ? false : self.ptr->fragile_right;
}
static inline bool ts_subtree_has_external_tokens(Subtree self) {
return self.data.is_inline ? false : self.ptr->has_external_tokens;
}
static inline bool ts_subtree_is_fragile(Subtree self) {
return self.data.is_inline ? false : (self.ptr->fragile_left || self.ptr->fragile_right);
}
static inline bool ts_subtree_is_error(Subtree self) {
return ts_subtree_symbol(self) == ts_builtin_sym_error;
}
static inline bool ts_subtree_is_eof(Subtree self) {
return ts_subtree_symbol(self) == ts_builtin_sym_end;
}
static inline Subtree ts_subtree_from_mut(MutableSubtree self) {
Subtree result;
result.data = self.data;
return result;
}
static inline MutableSubtree ts_subtree_to_mut_unsafe(Subtree self) {
MutableSubtree result;
result.data = self.data;
return result;
}
#ifdef __cplusplus

8
src/runtime/tree.c
View file

@ -7,7 +7,7 @@
static const unsigned PARENT_CACHE_CAPACITY = 32;
TSTree *ts_tree_new(const Subtree *root, const TSLanguage *language) {
TSTree *ts_tree_new(Subtree root, const TSLanguage *language) {
TSTree *result = ts_malloc(sizeof(TSTree));
result->root = root;
result->language = language;
@ -31,7 +31,7 @@ void ts_tree_delete(TSTree *self) {
}
TSNode ts_tree_root_node(const TSTree *self) {
return ts_node_new(self, self->root, self->root->padding, 0);
return ts_node_new(self, &self->root, ts_subtree_padding(self->root), 0);
}
const TSLanguage *ts_tree_language(const TSTree *self) {
@ -53,7 +53,7 @@ TSRange *ts_tree_get_changed_ranges(const TSTree *self, const TSTree *other, uin
ts_tree_cursor_init(&cursor1, root);
ts_tree_cursor_init(&cursor2, root);
*count = ts_subtree_get_changed_ranges(
self->root, other->root, &cursor1, &cursor2,
&self->root, &other->root, &cursor1, &cursor2,
self->language, &result
);
array_delete(&cursor1.stack);
@ -85,7 +85,7 @@ void ts_tree_set_cached_parent(const TSTree *_self, const TSNode *node, const TS
uint32_t index = (self->parent_cache_start + self->parent_cache_size) % PARENT_CACHE_CAPACITY;
self->parent_cache[index] = (ParentCacheEntry) {
.child = node->id,
.parent = parent->id,
.parent = (const Subtree *)parent->id,
.position = {
parent->context[0],
{parent->context[1], parent->context[2]}

4
src/runtime/tree.h
View file

@ -13,14 +13,14 @@ typedef struct {
} ParentCacheEntry;
struct TSTree {
const Subtree *root;
Subtree root;
const TSLanguage *language;
ParentCacheEntry *parent_cache;
uint32_t parent_cache_start;
uint32_t parent_cache_size;
};
TSTree *ts_tree_new(const Subtree *root, const TSLanguage *language);
TSTree *ts_tree_new(Subtree root, const TSLanguage *language);
TSNode ts_node_new(const TSTree *, const Subtree *, Length, TSSymbol);
TSNode ts_tree_get_cached_parent(const TSTree *, const TSNode *);
void ts_tree_set_cached_parent(const TSTree *, const TSNode *, const TSNode *);

46
src/runtime/tree_cursor.c
View file

@ -5,7 +5,7 @@
#include "runtime/tree.h"
typedef struct {
const Subtree *parent;
Subtree parent;
const TSTree *tree;
Length position;
uint32_t child_index;
@ -17,13 +17,16 @@ typedef struct {
static inline ChildIterator ts_tree_cursor_iterate_children(const TreeCursor *self) {
TreeCursorEntry *last_entry = array_back(&self->stack);
if (ts_subtree_child_count(*last_entry->subtree) == 0) {
return (ChildIterator) {NULL_SUBTREE, self->tree, length_zero(), 0, 0, NULL};
}
const TSSymbol *alias_sequence = ts_language_alias_sequence(
self->tree->language,
last_entry->subtree->alias_sequence_id
last_entry->subtree->ptr->alias_sequence_id
);
return (ChildIterator) {
.tree = self->tree,
.parent = last_entry->subtree,
.parent = *last_entry->subtree,
.position = last_entry->position,
.child_index = 0,
.structural_child_index = 0,
@ -34,26 +37,27 @@ static inline ChildIterator ts_tree_cursor_iterate_children(const TreeCursor *se
static inline bool ts_tree_cursor_child_iterator_next(ChildIterator *self,
TreeCursorEntry *result,
bool *visible) {
if (self->child_index == self->parent->child_count) return false;
const Subtree *child = self->parent->children[self->child_index];
if (!self->parent.ptr || self->child_index == self->parent.ptr->child_count) return false;
const Subtree *child = &self->parent.ptr->children[self->child_index];
*result = (TreeCursorEntry) {
.subtree = child,
.position = self->position,
.child_index = self->child_index,
.structural_child_index = self->structural_child_index,
};
*visible = child->visible;
if (!child->extra && self->alias_sequence) {
*visible = ts_subtree_visible(*child);
bool extra = ts_subtree_extra(*child);
if (!extra && self->alias_sequence) {
*visible |= self->alias_sequence[self->structural_child_index];
}
self->position = length_add(self->position, child->size);
self->position = length_add(self->position, ts_subtree_size(*child));
self->child_index++;
if (!child->extra) self->structural_child_index++;
if (!extra) self->structural_child_index++;
if (self->child_index < self->parent->child_count) {
const Subtree *child = self->parent->children[self->child_index];
self->position = length_add(self->position, child->padding);
if (self->child_index < self->parent.ptr->child_count) {
Subtree next_child = self->parent.ptr->children[self->child_index];
self->position = length_add(self->position, ts_subtree_padding(next_child));
}
return true;
@ -104,7 +108,7 @@ bool ts_tree_cursor_goto_first_child(TSTreeCursor *_self) {
return true;
}
if (entry.subtree->child_count > 0 && entry.subtree->visible_child_count > 0) {
if (ts_subtree_visible_child_count(*entry.subtree) > 0) {
array_push(&self->stack, entry);
did_descend = true;
break;
@ -128,11 +132,9 @@ int64_t ts_tree_cursor_goto_first_child_for_byte(TSTreeCursor *_self, uint32_t g
TreeCursorEntry entry;
ChildIterator iterator = ts_tree_cursor_iterate_children(self);
while (ts_tree_cursor_child_iterator_next(&iterator, &entry, &visible)) {
uint32_t end_byte = entry.position.bytes + entry.subtree->size.bytes;
uint32_t end_byte = entry.position.bytes + ts_subtree_size(*entry.subtree).bytes;
bool at_goal = end_byte > goal_byte;
uint32_t visible_child_count = entry.subtree->child_count > 0
? entry.subtree->visible_child_count
: 0;
uint32_t visible_child_count = ts_subtree_visible_child_count(*entry.subtree);
if (at_goal) {
if (visible) {
@ -183,7 +185,7 @@ bool ts_tree_cursor_goto_next_sibling(TSTreeCursor *_self) {
return true;
}
if (entry.subtree->child_count > 0 && entry.subtree->visible_child_count > 0) {
if (ts_subtree_visible_child_count(*entry.subtree)) {
array_push(&self->stack, entry);
ts_tree_cursor_goto_first_child(_self);
return true;
@ -204,11 +206,11 @@ bool ts_tree_cursor_goto_parent(TSTreeCursor *_self) {
TreeCursorEntry *parent_entry = &self->stack.contents[i - 1];
const TSSymbol *alias_sequence = ts_language_alias_sequence(
self->tree->language,
parent_entry->subtree->alias_sequence_id
parent_entry->subtree->ptr->alias_sequence_id
);
is_aliased = alias_sequence && alias_sequence[entry->structural_child_index];
}
if (entry->subtree->visible || is_aliased) {
if (ts_subtree_visible(*entry->subtree) || is_aliased) {
self->stack.size = i + 1;
return true;
}
@ -224,9 +226,9 @@ TSNode ts_tree_cursor_current_node(const TSTreeCursor *_self) {
TreeCursorEntry *parent_entry = &self->stack.contents[self->stack.size - 2];
const TSSymbol *alias_sequence = ts_language_alias_sequence(
self->tree->language,
parent_entry->subtree->alias_sequence_id
parent_entry->subtree->ptr->alias_sequence_id
);
if (alias_sequence && !last_entry->subtree->extra) {
if (alias_sequence && !ts_subtree_extra(*last_entry->subtree)) {
alias_symbol = alias_sequence[last_entry->structural_child_index];
}
}

8
test/helpers/point_helpers.cc
View file

@ -35,6 +35,14 @@ bool operator>(const TSPoint &left, const TSPoint &right) {
return right < left;
}
Length operator*(const Length &length, uint32_t factor) {
return {length.bytes * factor, {0, length.extent.column * factor}};
}
Length operator+(const Length &left, const Length &right) {
return length_add(left, right);
}
std::ostream &operator<<(std::ostream &stream, const TSPoint &point) {
return stream << "{" << point.row << ", " << point.column << "}";
}

4
test/helpers/point_helpers.h
View file

@ -14,6 +14,10 @@ bool operator==(const TSRange &left, const TSRange &right);
bool operator==(const Length &left, const Length &right);
Length operator*(const Length &length, uint32_t factor);
Length operator+(const Length &left, const Length &right);
std::ostream &operator<<(std::ostream &stream, const TSPoint &point);
std::ostream &operator<<(std::ostream &stream, const TSRange &range);

17
test/helpers/tree_helpers.cc
View file

@ -14,18 +14,18 @@ const char *symbol_names[24] = {
"twenty-two", "twenty-three"
};
SubtreeArray *tree_array(std::vector<const Subtree *> trees) {
SubtreeArray *tree_array(std::vector<Subtree> trees) {
static SubtreeArray result;
result.capacity = trees.size();
result.size = trees.size();
result.contents = (const Subtree **)calloc(trees.size(), sizeof(Subtree *));
result.contents = (Subtree *)calloc(trees.size(), sizeof(Subtree));
for (size_t i = 0; i < trees.size(); i++) {
result.contents[i] = trees[i];
}
return &result;
}
ostream &operator<<(std::ostream &stream, const Subtree *tree) {
ostream &operator<<(std::ostream &stream, Subtree tree) {
static TSLanguage DUMMY_LANGUAGE = {};
DUMMY_LANGUAGE.symbol_names = symbol_names;
char *string = ts_subtree_string(tree, &DUMMY_LANGUAGE, false);
@ -52,13 +52,10 @@ bool operator==(const TSNode &left, const TSNode &right) {
ts_node_start_point(left) == ts_node_start_point(right);
}
bool operator==(const std::vector<const Subtree *> &vec, const SubtreeArray &array) {
if (vec.size() != array.size)
return false;
for (size_t i = 0; i < array.size; i++)
if (array.contents[i] != vec[i])
return false;
return true;
bool operator==(const std::vector<Subtree> &vec, const SubtreeArray &array) {
return
vec.size() == array.size &&
std::memcmp(vec.data(), array.contents, array.size * sizeof(Subtree)) == 0;
}
void assert_consistent_tree_sizes(TSNode node) {

6
test/helpers/tree_helpers.h
View file

@ -6,12 +6,12 @@
#include <string>
extern const char *symbol_names[24];
SubtreeArray *tree_array(std::vector<const Subtree *> trees);
SubtreeArray *tree_array(std::vector<Subtree> trees);
std::ostream &operator<<(std::ostream &stream, const Subtree *tree);
std::ostream &operator<<(std::ostream &stream, Subtree tree);
std::ostream &operator<<(std::ostream &stream, const TSNode &node);
bool operator==(const TSNode &left, const TSNode &right);
bool operator==(const std::vector<const Subtree *> &right, const SubtreeArray &array);
bool operator==(const std::vector<Subtree> &right, const SubtreeArray &array);
void assert_consistent_tree_sizes(TSNode node);

2
test/runtime/parser_test.cc
View file

@ -839,7 +839,6 @@ describe("Parser", [&]() {
AssertThat(ts_node_type(interpolation_node1), Equals("template_substitution"));
AssertThat(ts_node_type(interpolation_node2), Equals("template_substitution"));
AssertThat(ts_node_type(close_quote_node), Equals("`"));
ts_tree_delete(js_tree);
TSRange included_ranges[] = {
{
@ -863,6 +862,7 @@ describe("Parser", [&]() {
};
ts_parser_set_included_ranges(parser, included_ranges, 3);
ts_tree_delete(js_tree);
ts_parser_set_language(parser, load_real_language("html"));
tree = ts_parser_parse_string(parser, nullptr, source_code.c_str(), source_code.size());

44
test/runtime/stack_test.cc
View file

@ -19,10 +19,6 @@ enum {
symbol9, symbol10
};
Length operator*(const Length &length, uint32_t factor) {
return {length.bytes * factor, {0, length.extent.column * factor}};
}
void free_slice_array(SubtreePool *pool, StackSliceArray *slices) {
for (size_t i = 0; i < slices->size; i++) {
StackSlice slice = slices->contents[i];
@ -44,8 +40,8 @@ void free_slice_array(SubtreePool *pool, StackSliceArray *slices) {
}
}
Subtree *mutate(const Subtree *subtree) {
return (Subtree *)subtree;
SubtreeHeapData *mutate(Subtree subtree) {
return ts_subtree_to_mut_unsafe(subtree).ptr;
}
struct StackEntry {
@ -73,7 +69,7 @@ START_TEST
describe("Stack", [&]() {
Stack *stack;
const size_t subtree_count = 11;
const Subtree *subtrees[subtree_count];
Subtree subtrees[subtree_count];
Length tree_len = {3, {0, 3}};
SubtreePool pool;
@ -92,7 +88,7 @@ describe("Stack", [&]() {
&pool, i + 1, length_zero(), tree_len, 0,
TS_TREE_STATE_NONE, true, false, &dummy_language
);
ts_external_scanner_state_init(&((Subtree *)subtrees[i])->external_scanner_state, nullptr, 0);
ts_external_scanner_state_init(&mutate(subtrees[i])->external_scanner_state, nullptr, 0);
}
});
@ -107,7 +103,7 @@ describe("Stack", [&]() {
AssertThat(record_alloc::outstanding_allocation_indices(), IsEmpty());
});
auto push = [&](StackVersion version, const Subtree *tree, TSStateId state) {
auto push = [&](StackVersion version, Subtree tree, TSStateId state) {
ts_subtree_retain(tree);
ts_stack_push(stack, version, tree, false, state);
};
@ -269,7 +265,7 @@ describe("Stack", [&]() {
StackSlice slice = pop.contents[0];
AssertThat(slice.version, Equals<StackVersion>(1));
AssertThat(slice.subtrees, Equals(vector<const Subtree *>({ subtrees[1], subtrees[2] })));
AssertThat(slice.subtrees, Equals(vector<Subtree>({ subtrees[1], subtrees[2] })));
AssertThat(ts_stack_state(stack, 1), Equals(stateA));
free_slice_array(&pool,&pop);
@ -285,7 +281,7 @@ describe("Stack", [&]() {
AssertThat(pop.size, Equals<size_t>(1));
StackSlice slice = pop.contents[0];
AssertThat(slice.subtrees, Equals(vector<const Subtree *>({ subtrees[0], subtrees[1], subtrees[2] })));
AssertThat(slice.subtrees, Equals(vector<Subtree>({ subtrees[0], subtrees[1], subtrees[2] })));
AssertThat(ts_stack_state(stack, 1), Equals(1));
free_slice_array(&pool,&pop);
@ -330,11 +326,11 @@ describe("Stack", [&]() {
StackSlice slice1 = pop.contents[0];
AssertThat(slice1.version, Equals<StackVersion>(1));
AssertThat(slice1.subtrees, Equals(vector<const Subtree *>({ subtrees[2], subtrees[3], subtrees[10] })));
AssertThat(slice1.subtrees, Equals(vector<Subtree>({ subtrees[2], subtrees[3], subtrees[10] })));
StackSlice slice2 = pop.contents[1];
AssertThat(slice2.version, Equals<StackVersion>(2));
AssertThat(slice2.subtrees, Equals(vector<const Subtree *>({ subtrees[5], subtrees[6], subtrees[10] })));
AssertThat(slice2.subtrees, Equals(vector<Subtree>({ subtrees[5], subtrees[6], subtrees[10] })));
AssertThat(ts_stack_version_count(stack), Equals<size_t>(3));
AssertThat(get_stack_entries(stack, 0), Equals(vector<StackEntry>({
@ -374,7 +370,7 @@ describe("Stack", [&]() {
StackSlice slice1 = pop.contents[0];
AssertThat(slice1.version, Equals<StackVersion>(1));
AssertThat(slice1.subtrees, Equals(vector<const Subtree *>({ subtrees[10] })));
AssertThat(slice1.subtrees, Equals(vector<Subtree>({ subtrees[10] })));
AssertThat(ts_stack_version_count(stack), Equals<size_t>(2));
AssertThat(ts_stack_state(stack, 0), Equals(stateI));
@ -396,11 +392,11 @@ describe("Stack", [&]() {
StackSlice slice1 = pop.contents[0];
AssertThat(slice1.version, Equals<StackVersion>(1));
AssertThat(slice1.subtrees, Equals(vector<const Subtree *>({ subtrees[1], subtrees[2], subtrees[3], subtrees[10] })));
AssertThat(slice1.subtrees, Equals(vector<Subtree>({ subtrees[1], subtrees[2], subtrees[3], subtrees[10] })));
StackSlice slice2 = pop.contents[1];
AssertThat(slice2.version, Equals<StackVersion>(1));
AssertThat(slice2.subtrees, Equals(vector<const Subtree *>({ subtrees[4], subtrees[5], subtrees[6], subtrees[10] })));
AssertThat(slice2.subtrees, Equals(vector<Subtree>({ subtrees[4], subtrees[5], subtrees[6], subtrees[10] })));
AssertThat(ts_stack_version_count(stack), Equals<size_t>(2));
AssertThat(ts_stack_state(stack, 0), Equals(stateI));
@ -451,15 +447,15 @@ describe("Stack", [&]() {
StackSlice slice1 = pop.contents[0];
AssertThat(slice1.version, Equals<StackVersion>(1));
AssertThat(slice1.subtrees, Equals(vector<const Subtree *>({ subtrees[3], subtrees[10] })));
AssertThat(slice1.subtrees, Equals(vector<Subtree>({ subtrees[3], subtrees[10] })));
StackSlice slice2 = pop.contents[1];
AssertThat(slice2.version, Equals<StackVersion>(2));
AssertThat(slice2.subtrees, Equals(vector<const Subtree *>({ subtrees[6], subtrees[10] })));
AssertThat(slice2.subtrees, Equals(vector<Subtree>({ subtrees[6], subtrees[10] })));
StackSlice slice3 = pop.contents[2];
AssertThat(slice3.version, Equals<StackVersion>(3));
AssertThat(slice3.subtrees, Equals(vector<const Subtree *>({ subtrees[9], subtrees[10] })));
AssertThat(slice3.subtrees, Equals(vector<Subtree>({ subtrees[9], subtrees[10] })));
AssertThat(ts_stack_version_count(stack), Equals<size_t>(4));
AssertThat(ts_stack_state(stack, 0), Equals(stateI));
@ -506,7 +502,7 @@ describe("Stack", [&]() {
StackSliceArray pop = ts_stack_pop_pending(stack, 0);
AssertThat(pop.size, Equals<size_t>(1));
AssertThat(pop.contents[0].subtrees, Equals(vector<const Subtree *>({ subtrees[1], subtrees[2], subtrees[3] })));
AssertThat(pop.contents[0].subtrees, Equals(vector<Subtree>({ subtrees[1], subtrees[2], subtrees[3] })));
AssertThat(get_stack_entries(stack, 0), Equals(vector<StackEntry>({
{stateA, 0},
@ -541,16 +537,16 @@ describe("Stack", [&]() {
});
it("allows the state to be retrieved", [&]() {
AssertThat(ts_stack_last_external_token(stack, 0), Equals<Subtree *>(nullptr));
AssertThat(ts_stack_last_external_token(stack, 0).ptr, Equals<const SubtreeHeapData *>(nullptr));
ts_stack_set_last_external_token(stack, 0, subtrees[1]);
AssertThat(ts_stack_last_external_token(stack, 0), Equals(subtrees[1]));
AssertThat(ts_stack_last_external_token(stack, 0).ptr, Equals<const SubtreeHeapData *>(subtrees[1].ptr));
ts_stack_copy_version(stack, 0);
AssertThat(ts_stack_last_external_token(stack, 1), Equals(subtrees[1]));
AssertThat(ts_stack_last_external_token(stack, 1).ptr, Equals<const SubtreeHeapData *>(subtrees[1].ptr));
ts_stack_set_last_external_token(stack, 0, subtrees[2]);
AssertThat(ts_stack_last_external_token(stack, 0), Equals(subtrees[2]));
AssertThat(ts_stack_last_external_token(stack, 0).ptr, Equals<const SubtreeHeapData *>(subtrees[2].ptr));
});
it("does not merge stack versions with different external token states", [&]() {

324
test/runtime/subtree_test.cc
View file

@ -4,13 +4,13 @@
#include "runtime/subtree.h"
#include "runtime/length.h"
void assert_consistent(const Subtree *tree) {
if (tree->child_count == 0) return;
AssertThat(tree->children[0]->padding, Equals<Length>(tree->padding));
void assert_consistent(Subtree tree) {
if (ts_subtree_child_count(tree) == 0) return;
AssertThat(tree.ptr->children[0].ptr->padding, Equals<Length>(tree.ptr->padding));
Length total_children_size = length_zero();
for (size_t i = 0; i < tree->child_count; i++) {
const Subtree *child = tree->children[i];
for (size_t i = 0; i < tree.ptr->child_count; i++) {
Subtree child = tree.ptr->children[i];
assert_consistent(child);
total_children_size = length_add(total_children_size, ts_subtree_total_size(child));
}
@ -54,124 +54,101 @@ describe("Subtree", []() {
);
};
auto new_node = [&](TSSymbol symbol, vector<Subtree> children) {
return ts_subtree_from_mut(ts_subtree_new_node(
&pool, symbol, tree_array(children), 0, &language
));
};
describe("new_node", [&]() {
const Subtree *tree1, *tree2, *parent1;
Subtree tree1, tree2;
before_each([&]() {
tree1 = new_leaf(symbol1, {2, {0, 1}}, {5, {0, 4}}, 0);
tree2 = new_leaf(symbol2, {1, {0, 1}}, {3, {0, 3}}, 0);
ts_subtree_retain(tree1);
ts_subtree_retain(tree2);
parent1 = ts_subtree_new_node(&pool, symbol3, tree_array({
tree1,
tree2,
}), 0, &language);
});
after_each([&]() {
ts_subtree_release(&pool, tree1);
ts_subtree_release(&pool, tree2);
ts_subtree_release(&pool, parent1);
});
it("computes its size and padding based on its child nodes", [&]() {
AssertThat(parent1->size.bytes, Equals<size_t>(
tree1->size.bytes + tree2->padding.bytes + tree2->size.bytes
));
AssertThat(parent1->padding.bytes, Equals<size_t>(tree1->padding.bytes));
ts_subtree_retain(tree1);
ts_subtree_retain(tree2);
Subtree parent = new_node(symbol3, {tree1, tree2});
AssertThat(
ts_subtree_size(parent),
Equals<Length>(
ts_subtree_size(tree1) + ts_subtree_padding(tree2) + ts_subtree_size(tree2)
));
AssertThat(ts_subtree_padding(parent), Equals<Length>(ts_subtree_padding(tree1)));
ts_subtree_release(&pool, parent);
});
describe("when the first node is fragile on the left side", [&]() {
const Subtree *parent;
before_each([&]() {
Subtree *mutable_tree1 = (Subtree *)tree1;
mutable_tree1->fragile_left = true;
mutable_tree1->extra = true;
it("records that it is fragile on the left side", [&]() {
MutableSubtree mutable_tree1 = ts_subtree_to_mut_unsafe(tree1);
mutable_tree1.ptr->fragile_left = true;
mutable_tree1.ptr->extra = true;
ts_subtree_retain(tree1);
ts_subtree_retain(tree2);
parent = ts_subtree_new_node(&pool, symbol3, tree_array({
tree1,
tree2,
}), 0, &language);
});
Subtree parent = new_node(symbol3, {tree1, tree2});
after_each([&]() {
AssertThat(ts_subtree_fragile_left(parent), IsTrue());
AssertThat(ts_subtree_fragile_right(parent), IsFalse());
ts_subtree_release(&pool, parent);
});
it("records that it is fragile on the left side", [&]() {
AssertThat(parent->fragile_left, IsTrue());
});
});
describe("when the last node is fragile on the right side", [&]() {
const Subtree *parent;
before_each([&]() {
Subtree *mutable_tree2 = (Subtree *)tree2;
mutable_tree2->fragile_right = true;
mutable_tree2->extra = true;
it("records that it is fragile on the right side", [&]() {
MutableSubtree mutable_tree2 = ts_subtree_to_mut_unsafe(tree2);
mutable_tree2.ptr->fragile_right = true;
mutable_tree2.ptr->extra = true;
ts_subtree_retain(tree1);
ts_subtree_retain(tree2);
parent = ts_subtree_new_node(&pool, symbol3, tree_array({
tree1,
tree2,
}), 0, &language);
});
Subtree parent = new_node(symbol3, {tree1, tree2});
after_each([&]() {
AssertThat(ts_subtree_fragile_left(parent), IsFalse());
AssertThat(ts_subtree_fragile_right(parent), IsTrue());
ts_subtree_release(&pool, parent);
});
it("records that it is fragile on the right side", [&]() {
AssertThat(parent->fragile_right, IsTrue());
});
});
describe("when the outer nodes aren't fragile on their outer side", [&]() {
const Subtree *parent;
before_each([&]() {
Subtree *mutable_tree1 = (Subtree *)tree1;
Subtree *mutable_tree2 = (Subtree *)tree2;
mutable_tree1->fragile_right = true;
mutable_tree2->fragile_left = true;
it("records that it is not fragile", [&]() {
MutableSubtree mutable_tree1 = ts_subtree_to_mut_unsafe(tree1);
MutableSubtree mutable_tree2 = ts_subtree_to_mut_unsafe(tree2);
mutable_tree1.ptr->fragile_right = true;
mutable_tree2.ptr->fragile_left = true;
ts_subtree_retain(tree1);
ts_subtree_retain(tree2);
parent = ts_subtree_new_node(&pool, symbol3, tree_array({
tree1,
tree2,
}), 0, &language);
});
Subtree parent = new_node(symbol3, {tree1, tree2});
after_each([&]() {
AssertThat(ts_subtree_fragile_left(parent), IsFalse());
AssertThat(ts_subtree_fragile_right(parent), IsFalse());
ts_subtree_release(&pool, parent);
});
it("records that it is not fragile", [&]() {
AssertThat(parent->fragile_left, IsFalse());
AssertThat(parent->fragile_right, IsFalse());
});
});
});
describe("edit", [&]() {
const Subtree *tree;
Subtree tree;
before_each([&]() {
tree = ts_subtree_new_node(&pool, symbol1, tree_array({
tree = new_node(symbol1, {
new_leaf(symbol2, {2, {0, 2}}, {3, {0, 3}}, 0),
new_leaf(symbol3, {2, {0, 2}}, {3, {0, 3}}, 0),
new_leaf(symbol4, {2, {0, 2}}, {3, {0, 3}}, 0),
}), 0, &language);
});
AssertThat(tree->padding, Equals<Length>({2, {0, 2}}));
AssertThat(tree->size, Equals<Length>({13, {0, 13}}));
AssertThat(tree.ptr->padding, Equals<Length>({2, {0, 2}}));
AssertThat(tree.ptr->size, Equals<Length>({13, {0, 13}}));
});
after_each([&]() {
@ -188,21 +165,21 @@ describe("Subtree", []() {
edit.new_end_point = {0, 2};
ts_subtree_retain(tree);
const Subtree *new_tree = ts_subtree_edit(tree, &edit, &pool);
Subtree new_tree = ts_subtree_edit(tree, &edit, &pool);
assert_consistent(tree);
assert_consistent(new_tree);
AssertThat(tree->has_changes, IsFalse());
AssertThat(tree->padding, Equals<Length>({2, {0, 2}}));
AssertThat(tree->size, Equals<Length>({13, {0, 13}}));
AssertThat(ts_subtree_has_changes(tree), IsFalse());
AssertThat(ts_subtree_padding(tree), Equals<Length>({2, {0, 2}}));
AssertThat(ts_subtree_size(tree), Equals<Length>({13, {0, 13}}));
AssertThat(tree->children[0]->has_changes, IsFalse());
AssertThat(tree->children[0]->padding, Equals<Length>({2, {0, 2}}));
AssertThat(tree->children[0]->size, Equals<Length>({3, {0, 3}}));
AssertThat(ts_subtree_has_changes(tree.ptr->children[0]), IsFalse());
AssertThat(ts_subtree_padding(tree.ptr->children[0]), Equals<Length>({2, {0, 2}}));
AssertThat(ts_subtree_size(tree.ptr->children[0]), Equals<Length>({3, {0, 3}}));
AssertThat(tree->children[1]->has_changes, IsFalse());
AssertThat(tree->children[1]->padding, Equals<Length>({2, {0, 2}}));
AssertThat(tree->children[1]->size, Equals<Length>({3, {0, 3}}));
AssertThat(ts_subtree_has_changes(tree.ptr->children[1]), IsFalse());
AssertThat(ts_subtree_padding(tree.ptr->children[1]), Equals<Length>({2, {0, 2}}));
AssertThat(ts_subtree_size(tree.ptr->children[1]), Equals<Length>({3, {0, 3}}));
ts_subtree_release(&pool, new_tree);
});
@ -220,17 +197,17 @@ describe("Subtree", []() {
tree = ts_subtree_edit(tree, &edit, &pool);
assert_consistent(tree);
AssertThat(tree->has_changes, IsTrue());
AssertThat(tree->padding, Equals<Length>({3, {0, 3}}));
AssertThat(tree->size, Equals<Length>({13, {0, 13}}));
AssertThat(ts_subtree_has_changes(tree), IsTrue());
AssertThat(ts_subtree_padding(tree), Equals<Length>({3, {0, 3}}));
AssertThat(ts_subtree_size(tree), Equals<Length>({13, {0, 13}}));
AssertThat(tree->children[0]->has_changes, IsTrue());
AssertThat(tree->children[0]->padding, Equals<Length>({3, {0, 3}}));
AssertThat(tree->children[0]->size, Equals<Length>({3, {0, 3}}));
AssertThat(ts_subtree_has_changes(tree.ptr->children[0]), IsTrue());
AssertThat(ts_subtree_padding(tree.ptr->children[0]), Equals<Length>({3, {0, 3}}));
AssertThat(ts_subtree_size(tree.ptr->children[0]), Equals<Length>({3, {0, 3}}));
AssertThat(tree->children[1]->has_changes, IsFalse());
AssertThat(tree->children[1]->padding, Equals<Length>({2, {0, 2}}));
AssertThat(tree->children[1]->size, Equals<Length>({3, {0, 3}}));
AssertThat(ts_subtree_has_changes(tree.ptr->children[1]), IsFalse());
AssertThat(ts_subtree_padding(tree.ptr->children[1]), Equals<Length>({2, {0, 2}}));
AssertThat(ts_subtree_size(tree.ptr->children[1]), Equals<Length>({3, {0, 3}}));
});
});
@ -247,13 +224,13 @@ describe("Subtree", []() {
tree = ts_subtree_edit(tree, &edit, &pool);
assert_consistent(tree);
AssertThat(tree->has_changes, IsTrue());
AssertThat(tree->padding, Equals<Length>({5, {0, 5}}));
AssertThat(tree->size, Equals<Length>({11, {0, 11}}));
AssertThat(ts_subtree_has_changes(tree), IsTrue());
AssertThat(ts_subtree_padding(tree), Equals<Length>({5, {0, 5}}));
AssertThat(ts_subtree_size(tree), Equals<Length>({11, {0, 11}}));
AssertThat(tree->children[0]->has_changes, IsTrue());
AssertThat(tree->children[0]->padding, Equals<Length>({5, {0, 5}}));
AssertThat(tree->children[0]->size, Equals<Length>({1, {0, 1}}));
AssertThat(ts_subtree_has_changes(tree.ptr->children[0]), IsTrue());
AssertThat(ts_subtree_padding(tree.ptr->children[0]), Equals<Length>({5, {0, 5}}));
AssertThat(ts_subtree_size(tree.ptr->children[0]), Equals<Length>({1, {0, 1}}));
});
});
@ -270,15 +247,15 @@ describe("Subtree", []() {
tree = ts_subtree_edit(tree, &edit, &pool);
assert_consistent(tree);
AssertThat(tree->has_changes, IsTrue());
AssertThat(tree->padding, Equals<Length>({4, {0, 4}}));
AssertThat(tree->size, Equals<Length>({13, {0, 13}}));
AssertThat(ts_subtree_has_changes(tree), IsTrue());
AssertThat(ts_subtree_padding(tree), Equals<Length>({4, {0, 4}}));
AssertThat(ts_subtree_size(tree), Equals<Length>({13, {0, 13}}));
AssertThat(tree->children[0]->has_changes, IsTrue());
AssertThat(tree->children[0]->padding, Equals<Length>({4, {0, 4}}));
AssertThat(tree->children[0]->size, Equals<Length>({3, {0, 3}}));
AssertThat(ts_subtree_has_changes(tree.ptr->children[0]), IsTrue());
AssertThat(ts_subtree_padding(tree.ptr->children[0]), Equals<Length>({4, {0, 4}}));
AssertThat(ts_subtree_size(tree.ptr->children[0]), Equals<Length>({3, {0, 3}}));
AssertThat(tree->children[1]->has_changes, IsFalse());
AssertThat(ts_subtree_has_changes(tree.ptr->children[1]), IsFalse());
});
});
@ -295,15 +272,15 @@ describe("Subtree", []() {
tree = ts_subtree_edit(tree, &edit, &pool);
assert_consistent(tree);
AssertThat(tree->has_changes, IsTrue());
AssertThat(tree->padding, Equals<Length>({2, {0, 2}}));
AssertThat(tree->size, Equals<Length>({16, {0, 16}}));
AssertThat(ts_subtree_has_changes(tree), IsTrue());
AssertThat(ts_subtree_padding(tree), Equals<Length>({2, {0, 2}}));
AssertThat(ts_subtree_size(tree), Equals<Length>({16, {0, 16}}));
AssertThat(tree->children[0]->has_changes, IsTrue());
AssertThat(tree->children[0]->padding, Equals<Length>({2, {0, 2}}));
AssertThat(tree->children[0]->size, Equals<Length>({6, {0, 6}}));
AssertThat(ts_subtree_has_changes(tree.ptr->children[0]), IsTrue());
AssertThat(ts_subtree_padding(tree.ptr->children[0]), Equals<Length>({2, {0, 2}}));
AssertThat(ts_subtree_size(tree.ptr->children[0]), Equals<Length>({6, {0, 6}}));
AssertThat(tree->children[1]->has_changes, IsFalse());
AssertThat(ts_subtree_has_changes(tree.ptr->children[1]), IsFalse());
});
});
@ -320,28 +297,28 @@ describe("Subtree", []() {
tree = ts_subtree_edit(tree, &edit, &pool);
assert_consistent(tree);
AssertThat(tree->has_changes, IsTrue());
AssertThat(tree->padding, Equals<Length>({4, {0, 4}}));
AssertThat(tree->size, Equals<Length>({4, {0, 4}}));
AssertThat(ts_subtree_has_changes(tree), IsTrue());
AssertThat(ts_subtree_padding(tree), Equals<Length>({4, {0, 4}}));
AssertThat(ts_subtree_size(tree), Equals<Length>({4, {0, 4}}));
AssertThat(tree->children[0]->has_changes, IsTrue());
AssertThat(tree->children[0]->padding, Equals<Length>({4, {0, 4}}));
AssertThat(tree->children[0]->size, Equals<Length>({0, {0, 0}}));
AssertThat(ts_subtree_has_changes(tree.ptr->children[0]), IsTrue());
AssertThat(ts_subtree_padding(tree.ptr->children[0]), Equals<Length>({4, {0, 4}}));
AssertThat(ts_subtree_size(tree.ptr->children[0]), Equals<Length>({0, {0, 0}}));
AssertThat(tree->children[1]->has_changes, IsTrue());
AssertThat(tree->children[1]->padding, Equals<Length>({0, {0, 0}}));
AssertThat(tree->children[1]->size, Equals<Length>({0, {0, 0}}));
AssertThat(ts_subtree_has_changes(tree.ptr->children[1]), IsTrue());
AssertThat(ts_subtree_padding(tree.ptr->children[1]), Equals<Length>({0, {0, 0}}));
AssertThat(ts_subtree_size(tree.ptr->children[1]), Equals<Length>({0, {0, 0}}));
AssertThat(tree->children[2]->has_changes, IsTrue());
AssertThat(tree->children[2]->padding, Equals<Length>({1, {0, 1}}));
AssertThat(tree->children[2]->size, Equals<Length>({3, {0, 3}}));
AssertThat(ts_subtree_has_changes(tree.ptr->children[2]), IsTrue());
AssertThat(ts_subtree_padding(tree.ptr->children[2]), Equals<Length>({1, {0, 1}}));
AssertThat(ts_subtree_size(tree.ptr->children[2]), Equals<Length>({3, {0, 3}}));
});
});
describe("edits within a tree's range of scanned bytes", [&]() {
it("marks preceding trees as changed", [&]() {
Subtree *mutable_child = (Subtree *)tree->children[0];
mutable_child->bytes_scanned = 7;
MutableSubtree mutable_child = ts_subtree_to_mut_unsafe(tree.ptr->children[0]);
mutable_child.ptr->bytes_scanned = 7;
TSInputEdit edit;
edit.start_byte = 6;
@ -354,7 +331,7 @@ describe("Subtree", []() {
tree = ts_subtree_edit(tree, &edit, &pool);
assert_consistent(tree);
AssertThat(tree->children[0]->has_changes, IsTrue());
AssertThat(ts_subtree_has_changes(tree.ptr->children[0]), IsTrue());
});
});
@ -371,9 +348,9 @@ describe("Subtree", []() {
tree = ts_subtree_edit(tree, &edit, &pool);
assert_consistent(tree);
AssertThat(tree->size.bytes, Equals(14u));
AssertThat(tree->children[2]->has_changes, IsTrue());
AssertThat(tree->children[2]->size.bytes, Equals(4u));
AssertThat(ts_subtree_size(tree).bytes, Equals(14u));
AssertThat(ts_subtree_has_changes(tree.ptr->children[2]), IsTrue());
AssertThat(ts_subtree_size(tree.ptr->children[2]).bytes, Equals(4u));
});
});
@ -390,14 +367,14 @@ describe("Subtree", []() {
tree = ts_subtree_edit(tree, &edit, &pool);
assert_consistent(tree);
AssertThat(tree->size.bytes, Equals(13u));
AssertThat(tree->children[2]->size.bytes, Equals(3u));
AssertThat(ts_subtree_size(tree).bytes, Equals(13u));
AssertThat(ts_subtree_size(tree.ptr->children[2]).bytes, Equals(3u));
});
});
});
describe("eq", [&]() {
const Subtree *leaf;
Subtree leaf;
before_each([&]() {
leaf = new_leaf(symbol1, {2, {1, 1}}, {5, {1, 4}}, 0);
@ -408,20 +385,14 @@ describe("Subtree", []() {
});
it("returns true for identical trees", [&]() {
const Subtree *leaf_copy = new_leaf(symbol1, {2, {1, 1}}, {5, {1, 4}}, 0);
Subtree leaf_copy = new_leaf(symbol1, {2, {1, 1}}, {5, {1, 4}}, 0);
AssertThat(ts_subtree_eq(leaf, leaf_copy), IsTrue());
const Subtree *parent = ts_subtree_new_node(&pool, symbol2, tree_array({
leaf,
leaf_copy,
}), 0, &language);
Subtree parent = new_node(symbol2, {leaf, leaf_copy});
ts_subtree_retain(leaf);
ts_subtree_retain(leaf_copy);
const Subtree *parent_copy = ts_subtree_new_node(&pool, symbol2, tree_array({
leaf,
leaf_copy,
}), 0, &language);
Subtree parent_copy = new_node(symbol2, {leaf, leaf_copy});
ts_subtree_retain(leaf);
ts_subtree_retain(leaf_copy);
@ -433,11 +404,11 @@ describe("Subtree", []() {
});
it("returns false for trees with different symbols", [&]() {
const Subtree *different_leaf = new_leaf(
leaf->symbol + 1,
leaf->padding,
leaf->size,
leaf->bytes_scanned
Subtree different_leaf = new_leaf(
ts_subtree_symbol(leaf) + 1,
ts_subtree_padding(leaf),
ts_subtree_size(leaf),
ts_subtree_bytes_scanned(leaf)
);
AssertThat(ts_subtree_eq(leaf, different_leaf), IsFalse());
@ -445,40 +416,39 @@ describe("Subtree", []() {
});
it("returns false for trees with different options", [&]() {
const Subtree *different_leaf = new_leaf(
leaf->symbol, leaf->padding, leaf->size, leaf->bytes_scanned
Subtree different_leaf = new_leaf(
ts_subtree_symbol(leaf),
ts_subtree_padding(leaf),
ts_subtree_size(leaf),
ts_subtree_bytes_scanned(leaf)
);
((Subtree *)different_leaf)->visible = !leaf->visible;
ts_subtree_to_mut_unsafe(different_leaf).ptr->visible = !ts_subtree_visible(leaf);
AssertThat(ts_subtree_eq(leaf, different_leaf), IsFalse());
ts_subtree_release(&pool, different_leaf);
});
it("returns false for trees with different paddings or sizes", [&]() {
const Subtree *different_leaf = new_leaf(
leaf->symbol, {}, leaf->size, leaf->bytes_scanned
Subtree different_leaf = new_leaf(
ts_subtree_symbol(leaf),
{},
ts_subtree_size(leaf),
ts_subtree_bytes_scanned(leaf)
);
AssertThat(ts_subtree_eq(leaf, different_leaf), IsFalse());
ts_subtree_release(&pool, different_leaf);
different_leaf = new_leaf(symbol1, leaf->padding, {}, leaf->bytes_scanned);
different_leaf = new_leaf(symbol1, ts_subtree_padding(leaf), {}, ts_subtree_bytes_scanned(leaf));
AssertThat(ts_subtree_eq(leaf, different_leaf), IsFalse());
ts_subtree_release(&pool, different_leaf);
});
it("returns false for trees with different children", [&]() {
const Subtree *leaf2 = new_leaf(symbol2, {1, {0, 1}}, {3, {0, 3}}, 0);
const Subtree *parent = ts_subtree_new_node(&pool, symbol2, tree_array({
leaf,
leaf2,
}), 0, &language);
Subtree leaf2 = new_leaf(symbol2, {1, {0, 1}}, {3, {0, 3}}, 0);
Subtree parent = new_node(symbol2, {leaf, leaf2});
ts_subtree_retain(leaf);
ts_subtree_retain(leaf2);
const Subtree *different_parent = ts_subtree_new_node(&pool, symbol2, tree_array({
leaf2,
leaf,
}), 0, &language);
Subtree different_parent = new_node(symbol2, {leaf2, leaf});
ts_subtree_retain(leaf2);
ts_subtree_retain(leaf);
@ -495,31 +465,33 @@ describe("Subtree", []() {
Length padding = {1, {0, 1}};
Length size = {2, {0, 2}};
auto make_external = [](const Subtree *_tree) {
Subtree *tree = (Subtree *)_tree;
ts_external_scanner_state_init(&tree->external_scanner_state, NULL, 0);
auto make_external = [](Subtree tree) {
ts_external_scanner_state_init(
&ts_subtree_to_mut_unsafe(tree).ptr->external_scanner_state,
NULL, 0
);
return tree;
};
it("returns the last serialized external token state in the given tree", [&]() {
const Subtree *tree1, *tree2, *tree3, *tree4, *tree5, *tree6, *tree7, *tree8, *tree9;
Subtree tree1, tree2, tree3, tree4, tree5, tree6, tree7, tree8, tree9;
tree1 = ts_subtree_new_node(&pool, symbol1, tree_array({
(tree2 = ts_subtree_new_node(&pool, symbol2, tree_array({
tree1 = new_node(symbol1, {
(tree2 = new_node(symbol2, {
(tree3 = make_external(ts_subtree_new_leaf(&pool, symbol3, padding, size, 0, 0, true, false, &language))),
(tree4 = new_leaf(symbol4, padding, size, 0)),
(tree5 = new_leaf(symbol5, padding, size, 0)),
}), 0, &language)),
(tree6 = ts_subtree_new_node(&pool, symbol6, tree_array({
(tree7 = ts_subtree_new_node(&pool, symbol7, tree_array({
})),
(tree6 = new_node(symbol6, {
(tree7 = new_node(symbol7, {
(tree8 = new_leaf(symbol8, padding, size, 0)),
}), 0, &language)),
})),
(tree9 = new_leaf(symbol9, padding, size, 0)),
}), 0, &language)),
}), 0, &language);
})),
});
auto token = ts_subtree_last_external_token(tree1);
AssertThat(token, Equals(tree3));
AssertThat(token.ptr, Equals(tree3.ptr));
ts_subtree_release(&pool, tree1);
});

2
test/test_helper.h
View file

@ -16,6 +16,6 @@ using namespace tree_sitter;
#define START_TEST go_bandit([]() {
#define END_TEST });
#define TREE_SITTER_WRAP_MALLOC
#define TREE_SITTER_TEST
#endif // TEST_HELPER_