traxys/tree-sitter
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Merge pull request #155 from tree-sitter/error-recovery-improvements

Browse source 
Fix some error recovery problems
This commit is contained in:
Max Brunsfeld 2018-04-09 12:44:02 -07:00 • committed by GitHub
commit ff70f8d881
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
22 changed files with 745 additions and 578 deletions
Download patch file Download diff file Expand all files Collapse all files

2
script/test
View file

@ -132,7 +132,7 @@ case ${mode} in
if [[ -n $line_count ]]; then
head -n $line_count $dot_file | dot -Tsvg >> $html_file
else
cat $dot_file | dot -Tsvg >> $html_file
cat $dot_file | grep -v 'Assertion' | dot -Tsvg >> $html_file
fi
rm $dot_file
echo "Wrote $html_file - $line_count"

8
src/compiler/build_tables/parse_table_builder.cc
View file

@ -150,6 +150,8 @@ class ParseTableBuilderImpl : public ParseTableBuilder {
MatchesLongerStringWithValidNextChar
);
parse_table.states[state_id].terminal_entries.clear();
// Add all the tokens that have no conflict with other tokens.
LookaheadSet non_conflicting_tokens;
for (unsigned i = 0; i < lexical_grammar.variables.size(); i++) {
@ -186,12 +188,6 @@ class ParseTableBuilderImpl : public ParseTableBuilder {
}
}
for (const Symbol &symbol : grammar.extra_tokens) {
if (!parse_table.states[state_id].terminal_entries.count(symbol)) {
parse_table.add_terminal_action(state_id, symbol, ParseAction::ShiftExtra());
}
}
for (size_t i = 0; i < grammar.external_tokens.size(); i++) {
if (grammar.external_tokens[i].corresponding_internal_token == rules::NONE()) {
parse_table.states[state_id].terminal_entries[Symbol::external(i)].actions.push_back(ParseAction::Recover());

16
src/runtime/array.h
View file

@ -12,18 +12,18 @@ extern "C" {
#include <stdbool.h>
#include "runtime/alloc.h"
#define Array(T) \
struct { \
T *contents; \
#define Array(T) \
struct { \
uint32_t size; \
uint32_t capacity; \
T *contents; \
}
#define array_init(self) \
((self)->size = 0, (self)->capacity = 0, (self)->contents = NULL)
#define array_new() \
{ NULL, 0, 0 }
{ 0, 0, NULL }
#define array_get(self, index) \
(assert((uint32_t)index < (self)->size), &(self)->contents[index])
@ -47,14 +47,14 @@ extern "C" {
(self)->contents[(self)->size++] = (element))
#define array_push_all(self, other) \
array_splice((self), (self)->size, 0, (other)->size, (other)->contents)
array_splice((self), (self)->size, 0, (other))
#define array_splice(self, index, old_count, new_count, new_elements) \
#define array_splice(self, index, old_count, new_array) \
array__splice((VoidArray *)(self), array__elem_size(self), index, old_count, \
new_count, (new_elements))
(new_array)->size, (new_array)->contents)
#define array_insert(self, index, element) \
array_splice(self, index, 0, 1, &(element))
array__splice((VoidArray *)(self), array__elem_size(self), index, 0, 1, &element)
#define array_pop(self) ((self)->contents[--(self)->size])

3
src/runtime/error_costs.h
View file

@ -2,7 +2,8 @@
#define RUNTIME_ERROR_COSTS_H_
#define ERROR_STATE 0
#define ERROR_COST_PER_MISSING_TREE 150
#define ERROR_COST_PER_RECOVERY 500
#define ERROR_COST_PER_MISSING_TREE 110
#define ERROR_COST_PER_SKIPPED_TREE 100
#define ERROR_COST_PER_SKIPPED_LINE 30
#define ERROR_COST_PER_SKIPPED_CHAR 1

8
src/runtime/get_changed_ranges.c
View file

@ -127,8 +127,8 @@ static bool iterator_descend(Iterator *self, uint32_t goal_position) {
TreePathEntry entry = *array_back(&self->path);
Length position = entry.position;
uint32_t structural_child_index = 0;
for (uint32_t i = 0; i < entry.tree->child_count; i++) {
Tree *child = entry.tree->children[i];
for (uint32_t i = 0; i < entry.tree->children.size; i++) {
Tree *child = entry.tree->children.contents[i];
Length child_left = length_add(position, child->padding);
Length child_right = length_add(child_left, child->size);
@ -179,11 +179,11 @@ static void iterator_advance(Iterator *self) {
Tree *parent = array_back(&self->path)->tree;
uint32_t child_index = entry.child_index + 1;
if (parent->child_count > child_index) {
if (parent->children.size > child_index) {
Length position = length_add(entry.position, ts_tree_total_size(entry.tree));
uint32_t structural_child_index = entry.structural_child_index;
if (!entry.tree->extra) structural_child_index++;
Tree *next_child = parent->children[child_index];
Tree *next_child = parent->children.contents[child_index];
array_push(&self->path, ((TreePathEntry){
.tree = next_child,

12
src/runtime/language.c
View file

@ -4,11 +4,10 @@
void ts_language_table_entry(const TSLanguage *self, TSStateId state,
TSSymbol symbol, TableEntry *result) {
if (symbol == ts_builtin_sym_error) {
if (symbol == ts_builtin_sym_error || symbol == ts_builtin_sym_error_repeat) {
result->action_count = 0;
result->is_reusable = false;
result->actions = NULL;
return;
} else {
assert(symbol < self->token_count);
uint32_t action_index = self->parse_table[state * self->symbol_count + symbol];
@ -27,10 +26,11 @@ uint32_t ts_language_version(const TSLanguage *language) {
return language->version;
}
TSSymbolMetadata ts_language_symbol_metadata(const TSLanguage *language,
TSSymbol symbol) {
if (symbol == ts_builtin_sym_error) {
TSSymbolMetadata ts_language_symbol_metadata(const TSLanguage *language, TSSymbol symbol) {
if (symbol == ts_builtin_sym_error) {
return (TSSymbolMetadata){.visible = true, .named = true};
} else if (symbol == ts_builtin_sym_error_repeat) {
return (TSSymbolMetadata){.visible = false, .named = false};
} else {
return language->symbol_metadata[symbol];
}
@ -39,6 +39,8 @@ TSSymbolMetadata ts_language_symbol_metadata(const TSLanguage *language,
const char *ts_language_symbol_name(const TSLanguage *language, TSSymbol symbol) {
if (symbol == ts_builtin_sym_error) {
return "ERROR";
} else if (symbol == ts_builtin_sym_error_repeat) {
return "_ERROR";
} else {
return language->symbol_names[symbol];
}

4
src/runtime/language.h
View file

@ -8,6 +8,8 @@ extern "C" {
#include "tree_sitter/parser.h"
#include "runtime/tree.h"
#define ts_builtin_sym_error_repeat (ts_builtin_sym_error - 1)
typedef struct {
const TSParseAction *actions;
uint32_t action_count;
@ -51,7 +53,7 @@ static inline bool ts_language_has_reduce_action(const TSLanguage *self,
static inline TSStateId ts_language_next_state(const TSLanguage *self,
TSStateId state,
TSSymbol symbol) {
if (symbol == ts_builtin_sym_error) {
if (symbol == ts_builtin_sym_error || symbol == ts_builtin_sym_error_repeat) {
return 0;
} else if (symbol < self->token_count) {
uint32_t count;

20
src/runtime/node.c
View file

@ -39,7 +39,7 @@ static inline bool ts_node__is_relevant(TSNode self, bool include_anonymous) {
static inline uint32_t ts_node__relevant_child_count(TSNode self,
bool include_anonymous) {
const Tree *tree = ts_node__tree(self);
if (tree->child_count > 0) {
if (tree->children.size > 0) {
if (include_anonymous) {
return tree->visible_child_count;
} else {
@ -61,7 +61,7 @@ static inline TSNode ts_node__direct_parent(TSNode self, uint32_t *index) {
}
static inline TSNode ts_node__direct_child(TSNode self, uint32_t i) {
const Tree *child_tree = ts_node__tree(self)->children[i];
const Tree *child_tree = ts_node__tree(self)->children.contents[i];
return ts_node_make(
child_tree,
ts_node__offset_byte(self) + child_tree->context.offset.bytes,
@ -78,7 +78,7 @@ static inline TSNode ts_node__child(TSNode self, uint32_t child_index,
did_descend = false;
uint32_t index = 0;
for (uint32_t i = 0; i < ts_node__tree(result)->child_count; i++) {
for (uint32_t i = 0; i < ts_node__tree(result)->children.size; i++) {
TSNode child = ts_node__direct_child(result, i);
if (ts_node__is_relevant(child, include_anonymous)) {
if (index == child_index)
@ -134,7 +134,7 @@ static inline TSNode ts_node__next_sibling(TSNode self, bool include_anonymous)
if (!result.data)
break;
for (uint32_t i = index + 1; i < ts_node__tree(result)->child_count; i++) {
for (uint32_t i = index + 1; i < ts_node__tree(result)->children.size; i++) {
TSNode child = ts_node__direct_child(result, i);
if (ts_node__is_relevant(child, include_anonymous))
return child;
@ -160,7 +160,7 @@ static inline TSNode ts_node__first_child_for_byte(TSNode self, uint32_t goal,
while (did_descend) {
did_descend = false;
for (uint32_t i = 0; i < ts_node__tree(node)->child_count; i++) {
for (uint32_t i = 0; i < ts_node__tree(node)->children.size; i++) {
TSNode child = ts_node__direct_child(node, i);
if (ts_node_end_byte(child) > goal) {
if (ts_node__is_relevant(child, include_anonymous)) {
@ -187,7 +187,7 @@ static inline TSNode ts_node__descendant_for_byte_range(TSNode self, uint32_t mi
while (did_descend) {
did_descend = false;
for (uint32_t i = 0, n = ts_node__tree(node)->child_count; i < n; i++) {
for (uint32_t i = 0, n = ts_node__tree(node)->children.size; i < n; i++) {
TSNode child = ts_node__direct_child(node, i);
if (ts_node_end_byte(child) > max) {
if (ts_node_start_byte(child) > min) break;
@ -214,7 +214,7 @@ static inline TSNode ts_node__descendant_for_point_range(TSNode self, TSPoint mi
while (did_descend) {
did_descend = false;
for (uint32_t i = 0, n = ts_node__tree(node)->child_count; i < n; i++) {
for (uint32_t i = 0, n = ts_node__tree(node)->children.size; i < n; i++) {
TSNode child = ts_node__direct_child(node, i);
const Tree *child_tree = ts_node__tree(child);
if (i > 0) start_position = point_add(start_position, child_tree->padding.extent);
@ -318,7 +318,7 @@ uint32_t ts_node_child_index(TSNode self) {
uint32_t index = tree->context.index;
if (!parent) return UINT32_MAX;
for (uint32_t i = 0; i < index; i++) {
Tree *child = parent->children[i];
Tree *child = parent->children.contents[i];
result += child->visible ? 1 : child->visible_child_count;
}
if (parent->visible) break;
@ -338,7 +338,7 @@ TSNode ts_node_named_child(TSNode self, uint32_t child_index) {
uint32_t ts_node_child_count(TSNode self) {
const Tree *tree = ts_node__tree(self);
if (tree->child_count > 0) {
if (tree->children.size > 0) {
return tree->visible_child_count;
} else {
return 0;
@ -347,7 +347,7 @@ uint32_t ts_node_child_count(TSNode self) {
uint32_t ts_node_named_child_count(TSNode self) {
const Tree *tree = ts_node__tree(self);
if (tree->child_count > 0) {
if (tree->children.size > 0) {
return tree->named_child_count;
} else {
return 0;

514
src/runtime/parser.c
View file

@ -19,10 +19,10 @@
parser__log(self); \
}
#define LOG_STACK() \
if (self->print_debugging_graphs) { \
ts_stack_print_dot_graph(self->stack, self->language->symbol_names, stderr); \
fputs("\n\n", stderr); \
#define LOG_STACK() \
if (self->print_debugging_graphs) { \
ts_stack_print_dot_graph(self->stack, self->language, stderr); \
fputs("\n\n", stderr); \
}
#define LOG_TREE() \
@ -39,7 +39,7 @@ static const unsigned MAX_COST_DIFFERENCE = 16 * ERROR_COST_PER_SKIPPED_TREE;
typedef struct {
unsigned cost;
unsigned push_count;
unsigned node_count;
int dynamic_precedence;
bool is_in_error;
} ErrorStatus;
@ -86,9 +86,9 @@ static bool parser__breakdown_top_of_stack(Parser *self, StackVersion version) {
TSStateId state = ts_stack_state(self->stack, slice.version);
Tree *parent = *array_front(&slice.trees);
for (uint32_t j = 0; j < parent->child_count; j++) {
Tree *child = parent->children[j];
pending = child->child_count > 0;
for (uint32_t j = 0; j < parent->children.size; j++) {
Tree *child = parent->children.contents[j];
pending = child->children.size > 0;
if (child->symbol == ts_builtin_sym_error) {
state = ERROR_STATE;
@ -105,7 +105,6 @@ static bool parser__breakdown_top_of_stack(Parser *self, StackVersion version) {
ts_stack_push(self->stack, slice.version, tree, false, state);
}
ts_stack_decrease_push_count(self->stack, slice.version, parent->child_count + 1);
ts_tree_release(&self->tree_pool, parent);
array_delete(&slice.trees);
@ -121,7 +120,7 @@ static void parser__breakdown_lookahead(Parser *self, Tree **lookahead,
TSStateId state,
ReusableNode *reusable_node) {
bool did_break_down = false;
while (reusable_node->tree->child_count > 0 && reusable_node->tree->parse_state != state) {
while (reusable_node->tree->children.size > 0 && reusable_node->tree->parse_state != state) {
LOG("state_mismatch sym:%s", SYM_NAME(reusable_node->tree->symbol));
reusable_node_breakdown(reusable_node);
did_break_down = true;
@ -151,7 +150,7 @@ static ErrorComparison parser__compare_versions(Parser *self, ErrorStatus a, Err
}
if (a.cost < b.cost) {
if ((b.cost - a.cost) * (1 + a.push_count) > MAX_COST_DIFFERENCE) {
if ((b.cost - a.cost) * (1 + a.node_count) > MAX_COST_DIFFERENCE) {
return ErrorComparisonTakeLeft;
} else {
return ErrorComparisonPreferLeft;
@ -159,7 +158,7 @@ static ErrorComparison parser__compare_versions(Parser *self, ErrorStatus a, Err
}
if (b.cost < a.cost) {
if ((a.cost - b.cost) * (1 + b.push_count) > MAX_COST_DIFFERENCE) {
if ((a.cost - b.cost) * (1 + b.node_count) > MAX_COST_DIFFERENCE) {
return ErrorComparisonTakeRight;
} else {
return ErrorComparisonPreferRight;
@ -171,6 +170,18 @@ static ErrorComparison parser__compare_versions(Parser *self, ErrorStatus a, Err
return ErrorComparisonNone;
}
static ErrorStatus parser__version_status(Parser *self, StackVersion version) {
unsigned cost = ts_stack_error_cost(self->stack, version);
bool is_paused = ts_stack_is_paused(self->stack, version);
if (is_paused) cost += ERROR_COST_PER_SKIPPED_TREE;
return (ErrorStatus) {
.cost = cost,
.node_count = ts_stack_node_count_since_error(self->stack, version),
.dynamic_precedence = ts_stack_dynamic_precedence(self->stack, version),
.is_in_error = is_paused || ts_stack_state(self->stack, version) == ERROR_STATE
};
}
static bool parser__better_version_exists(Parser *self, StackVersion version,
bool is_in_error, unsigned cost) {
if (self->finished_tree && self->finished_tree->error_cost <= cost) return true;
@ -180,19 +191,14 @@ static bool parser__better_version_exists(Parser *self, StackVersion version,
.cost = cost,
.is_in_error = is_in_error,
.dynamic_precedence = ts_stack_dynamic_precedence(self->stack, version),
.push_count = 0,
.node_count = ts_stack_node_count_since_error(self->stack, version),
};
for (StackVersion i = 0, n = ts_stack_version_count(self->stack); i < n; i++) {
if (i == version ||
ts_stack_is_halted(self->stack, i) ||
!ts_stack_is_active(self->stack, i) ||
ts_stack_position(self->stack, i).bytes < position.bytes) continue;
ErrorStatus status_i = {
.cost = ts_stack_error_cost(self->stack, i),
.is_in_error = ts_stack_state(self->stack, i) == ERROR_STATE,
.dynamic_precedence = ts_stack_dynamic_precedence(self->stack, i),
.push_count = ts_stack_push_count(self->stack, i)
};
ErrorStatus status_i = parser__version_status(self, i);
switch (parser__compare_versions(self, status, status_i)) {
case ErrorComparisonTakeRight:
return true;
@ -206,83 +212,6 @@ static bool parser__better_version_exists(Parser *self, StackVersion version,
return false;
}
static unsigned parser__condense_stack(Parser *self) {
bool made_changes = false;
unsigned min_error_cost = UINT_MAX;
for (StackVersion i = 0; i < ts_stack_version_count(self->stack); i++) {
if (ts_stack_is_halted(self->stack, i)) {
ts_stack_remove_version(self->stack, i);
i--;
continue;
}
ErrorStatus status_i = {
.cost = ts_stack_error_cost(self->stack, i),
.push_count = ts_stack_push_count(self->stack, i),
.dynamic_precedence = ts_stack_dynamic_precedence(self->stack, i),
.is_in_error = ts_stack_state(self->stack, i) == ERROR_STATE,
};
if (!status_i.is_in_error && status_i.cost < min_error_cost) {
min_error_cost = status_i.cost;
}
for (StackVersion j = 0; j < i; j++) {
ErrorStatus status_j = {
.cost = ts_stack_error_cost(self->stack, j),
.push_count = ts_stack_push_count(self->stack, j),
.dynamic_precedence = ts_stack_dynamic_precedence(self->stack, j),
.is_in_error = ts_stack_state(self->stack, j) == ERROR_STATE,
};
bool can_merge = ts_stack_can_merge(self->stack, j, i);
switch (parser__compare_versions(self, status_j, status_i)) {
case ErrorComparisonTakeLeft:
made_changes = true;
ts_stack_remove_version(self->stack, i);
i--;
j = i;
break;
case ErrorComparisonPreferLeft:
case ErrorComparisonNone:
if (can_merge) {
made_changes = true;
ts_stack_force_merge(self->stack, j, i);
i--;
j = i;
}
break;
case ErrorComparisonPreferRight:
made_changes = true;
ts_stack_swap_versions(self->stack, i, j);
if (can_merge) {
ts_stack_force_merge(self->stack, j, i);
i--;
j = i;
}
break;
case ErrorComparisonTakeRight:
made_changes = true;
ts_stack_remove_version(self->stack, j);
i--;
j--;
break;
}
}
}
while (ts_stack_version_count(self->stack) > MAX_VERSION_COUNT) {
ts_stack_remove_version(self->stack, MAX_VERSION_COUNT);
made_changes = true;
}
if (made_changes) {
LOG("condense");
LOG_STACK();
}
return min_error_cost;
}
static void parser__restore_external_scanner(Parser *self, Tree *external_token) {
if (external_token) {
self->language->external_scanner.deserialize(
@ -334,9 +263,7 @@ static Tree *parser__lex(Parser *self, StackVersion version, TSStateId parse_sta
self->lexer.token_end_position = self->lexer.current_position;
}
if (error_mode && self->lexer.token_end_position.bytes <= current_position.bytes) {
LOG("disregard_empty_token");
} else {
if (!error_mode || self->lexer.token_end_position.bytes > current_position.bytes) {
found_external_token = true;
break;
}
@ -360,7 +287,6 @@ static Tree *parser__lex(Parser *self, StackVersion version, TSStateId parse_sta
}
if (!error_mode) {
LOG("retry_in_error_mode");
error_mode = true;
lex_mode = self->language->lex_modes[ERROR_STATE];
valid_external_tokens = ts_language_enabled_external_tokens(
@ -521,7 +447,7 @@ static Tree *parser__get_lookahead(Parser *self, StackVersion version, TSStateId
reason = "is_missing";
} else if (result->fragile_left || result->fragile_right) {
reason = "is_fragile";
} else if (self->in_ambiguity && result->child_count) {
} else if (self->in_ambiguity && result->children.size) {
reason = "in_ambiguity";
}
@ -629,7 +555,7 @@ static void parser__shift(Parser *self, StackVersion version, TSStateId state,
ts_tree_retain(lookahead);
}
bool is_pending = lookahead->child_count > 0;
bool is_pending = lookahead->children.size > 0;
ts_stack_push(self->stack, version, lookahead, is_pending, state);
if (lookahead->has_external_tokens) {
ts_stack_set_last_external_token(
@ -638,10 +564,10 @@ static void parser__shift(Parser *self, StackVersion version, TSStateId state,
}
}
static bool parser__replace_children(Parser *self, Tree *tree, Tree **children, uint32_t count) {
static bool parser__replace_children(Parser *self, Tree *tree, TreeArray *children) {
self->scratch_tree = *tree;
self->scratch_tree.child_count = 0;
ts_tree_set_children(&self->scratch_tree, count, children, self->language);
self->scratch_tree.children.size = 0;
ts_tree_set_children(&self->scratch_tree, children, self->language);
if (parser__select_tree(self, tree, &self->scratch_tree)) {
*tree = self->scratch_tree;
return true;
@ -663,13 +589,13 @@ static StackSliceArray parser__reduce(Parser *self, StackVersion version, TSSymb
// Extra tokens on top of the stack should not be included in this new parent
// node. They will be re-pushed onto the stack after the parent node is
// created and pushed.
uint32_t child_count = slice.trees.size;
while (child_count > 0 && slice.trees.contents[child_count - 1]->extra) {
child_count--;
TreeArray children = slice.trees;
while (children.size > 0 && children.contents[children.size - 1]->extra) {
children.size--;
}
Tree *parent = ts_tree_make_node(&self->tree_pool,
symbol, child_count, slice.trees.contents, alias_sequence_id, self->language
symbol, &children, alias_sequence_id, self->language
);
// This pop operation may have caused multiple stack versions to collapse
@ -681,12 +607,12 @@ static StackSliceArray parser__reduce(Parser *self, StackVersion version, TSSymb
if (next_slice.version != slice.version) break;
i++;
uint32_t child_count = next_slice.trees.size;
while (child_count > 0 && next_slice.trees.contents[child_count - 1]->extra) {
child_count--;
TreeArray children = next_slice.trees;
while (children.size > 0 && children.contents[children.size - 1]->extra) {
children.size--;
}
if (parser__replace_children(self, parent, next_slice.trees.contents, child_count)) {
if (parser__replace_children(self, parent, &children)) {
ts_tree_array_delete(&self->tree_pool, &slice.trees);
slice = next_slice;
} else {
@ -710,9 +636,22 @@ static StackSliceArray parser__reduce(Parser *self, StackVersion version, TSSymb
// Push the parent node onto the stack, along with any extra tokens that
// were previously on top of the stack.
ts_stack_push(self->stack, slice.version, parent, false, next_state);
for (uint32_t j = parent->child_count; j < slice.trees.size; j++) {
Tree *tree = slice.trees.contents[j];
ts_stack_push(self->stack, slice.version, tree, false, next_state);
for (uint32_t j = parent->children.size; j < slice.trees.size; j++) {
ts_stack_push(self->stack, slice.version, slice.trees.contents[j], false, next_state);
}
if (ts_stack_version_count(self->stack) > MAX_VERSION_COUNT) {
i++;
while (i < pop.size) {
StackSlice slice = pop.contents[i];
ts_tree_array_delete(&self->tree_pool, &slice.trees);
ts_stack_halt(self->stack, slice.version);
i++;
}
while (ts_stack_version_count(self->stack) > slice.version + 1) {
ts_stack_remove_version(self->stack, slice.version + 1);
}
break;
}
}
@ -747,29 +686,28 @@ static void parser__start(Parser *self, TSInput input, Tree *previous_tree) {
self->in_ambiguity = false;
}
static void parser__accept(Parser *self, StackVersion version,
Tree *lookahead) {
static void parser__accept(Parser *self, StackVersion version, Tree *lookahead) {
lookahead->extra = true;
assert(lookahead->symbol == ts_builtin_sym_end);
ts_tree_retain(lookahead);
ts_stack_push(self->stack, version, lookahead, false, 1);
StackSliceArray pop = ts_stack_pop_all(self->stack, version);
StackSliceArray pop = ts_stack_pop_all(self->stack, version);
for (uint32_t i = 0; i < pop.size; i++) {
StackSlice slice = pop.contents[i];
TreeArray trees = slice.trees;
TreeArray trees = pop.contents[i].trees;
Tree *root = NULL;
for (uint32_t j = trees.size - 1; j + 1 > 0; j--) {
Tree *child = trees.contents[j];
if (!child->extra) {
root = ts_tree_make_copy(&self->tree_pool, child);
root->child_count = 0;
for (uint32_t k = 0; k < child->child_count; k++) {
ts_tree_retain(child->children[k]);
for (uint32_t k = 0; k < child->children.size; k++) {
ts_tree_retain(child->children.contents[k]);
}
array_splice(&trees, j, 1, child->child_count, child->children);
ts_tree_set_children(root, trees.size, trees.contents, self->language);
array_splice(&trees, j, 1, &child->children);
root = ts_tree_make_node(
&self->tree_pool, child->symbol, &trees,
child->alias_sequence_id, self->language
);
ts_tree_release(&self->tree_pool, child);
break;
}
@ -796,11 +734,23 @@ static void parser__accept(Parser *self, StackVersion version,
static bool parser__do_all_potential_reductions(Parser *self, StackVersion starting_version,
TSSymbol lookahead_symbol) {
bool result = false;
for (StackVersion version = starting_version;;) {
uint32_t initial_version_count = ts_stack_version_count(self->stack);
bool can_shift_lookahead_symbol = false;
StackVersion version = starting_version;
for (unsigned i = 0; true; i++) {
uint32_t version_count = ts_stack_version_count(self->stack);
if (version >= version_count) break;
bool merged = false;
for (StackVersion i = initial_version_count; i < version; i++) {
if (ts_stack_merge(self->stack, i, version)) {
merged = true;
break;
}
}
if (merged) continue;
TSStateId state = ts_stack_state(self->stack, version);
bool has_shift_action = false;
array_clear(&self->reduce_actions);
@ -822,7 +772,7 @@ static bool parser__do_all_potential_reductions(Parser *self, StackVersion start
switch (action.type) {
case TSParseActionTypeShift:
case TSParseActionTypeRecover:
if (!action.params.extra) has_shift_action = true;
if (!action.params.extra && !action.params.repetition) has_shift_action = true;
break;
case TSParseActionTypeReduce:
if (action.params.child_count > 0)
@ -838,9 +788,9 @@ static bool parser__do_all_potential_reductions(Parser *self, StackVersion start
}
}
bool has_reduce_action = self->reduce_actions.size > 0;
for (uint32_t i = 0; i < self->reduce_actions.size; i++) {
ReduceAction action = self->reduce_actions.contents[i];
parser__reduce(
self, version, action.symbol, action.count,
action.dynamic_precedence, action.alias_sequence_id,
@ -849,14 +799,12 @@ static bool parser__do_all_potential_reductions(Parser *self, StackVersion start
}
if (has_shift_action) {
result = true;
} else {
if (has_reduce_action) {
ts_stack_renumber_version(self->stack, version_count, version);
continue;
} else if (lookahead_symbol != 0) {
ts_stack_remove_version(self->stack, version);
}
can_shift_lookahead_symbol = true;
} else if (self->reduce_actions.size > 0 && i < MAX_VERSION_COUNT) {
ts_stack_renumber_version(self->stack, version_count, version);
continue;
} else if (lookahead_symbol != 0) {
ts_stack_remove_version(self->stack, version);
}
if (version == starting_version) {
@ -865,28 +813,12 @@ static bool parser__do_all_potential_reductions(Parser *self, StackVersion start
version++;
}
}
return result;
return can_shift_lookahead_symbol;
}
static void parser__handle_error(Parser *self, StackVersion version, TSSymbol lookahead_symbol) {
// If enough parse versions have already completed, just halt this version.
if (self->accept_count > MAX_VERSION_COUNT) {
ts_stack_halt(self->stack, version);
LOG("bail_after_too_many_tries");
return;
}
// If there are other in-progress versions that are clearly better than this one,
// just halt this version.
unsigned new_cost = ts_stack_error_cost(self->stack, version) + ERROR_COST_PER_SKIPPED_TREE;
if (parser__better_version_exists(self, version, true, new_cost)) {
ts_stack_halt(self->stack, version);
LOG("bail_on_error");
return;
}
// Perform any reductions that could have happened in this state, regardless of the lookahead.
LOG("handle_error");
uint32_t previous_version_count = ts_stack_version_count(self->stack);
parser__do_all_potential_reductions(self, version, 0);
uint32_t version_count = ts_stack_version_count(self->stack);
@ -922,8 +854,11 @@ static void parser__handle_error(Parser *self, StackVersion version, TSSymbol lo
self, version_with_missing_tree,
lookahead_symbol
)) {
LOG("recover_with_missing symbol:%s, state:%u", SYM_NAME(missing_symbol), state_after_missing_symbol);
LOG_STACK();
LOG(
"recover_with_missing symbol:%s, state:%u",
SYM_NAME(missing_symbol),
ts_stack_state(self->stack, version_with_missing_tree)
);
did_insert_missing_token = true;
break;
}
@ -936,7 +871,7 @@ static void parser__handle_error(Parser *self, StackVersion version, TSSymbol lo
}
for (unsigned i = previous_version_count; i < version_count; i++) {
ts_stack_force_merge(self->stack, version, previous_version_count);
assert(ts_stack_merge(self->stack, version, previous_version_count));
}
ts_stack_record_summary(self->stack, version, MAX_SUMMARY_DEPTH);
@ -987,13 +922,14 @@ static bool parser__recover_to_state(Parser *self, StackVersion version, unsigne
continue;
}
StackSliceArray error_pop = ts_stack_pop_error(self->stack, slice.version);
if (error_pop.size > 0) {
StackSlice error_slice = error_pop.contents[0];
array_push_all(&error_slice.trees, &slice.trees);
array_delete(&slice.trees);
slice.trees = error_slice.trees;
ts_stack_renumber_version(self->stack, error_slice.version, slice.version);
TreeArray error_trees = ts_stack_pop_error(self->stack, slice.version);
if (error_trees.size > 0) {
assert(error_trees.size == 1);
array_splice(&slice.trees, 0, 0, &error_trees.contents[0]->children);
for (unsigned j = 0; j < error_trees.contents[0]->children.size; j++) {
ts_tree_retain(slice.trees.contents[j]);
}
ts_tree_array_delete(&self->tree_pool, &error_trees);
}
TreeArray trailing_extras = ts_tree_array_remove_trailing_extras(&slice.trees);
@ -1023,39 +959,52 @@ static void parser__recover(Parser *self, StackVersion version, Tree *lookahead)
unsigned previous_version_count = ts_stack_version_count(self->stack);
Length position = ts_stack_position(self->stack, version);
StackSummary *summary = ts_stack_get_summary(self->stack, version);
unsigned node_count_since_error = ts_stack_node_count_since_error(self->stack, version);
unsigned current_error_cost = ts_stack_error_cost(self->stack, version);
for (unsigned i = 0; i < summary->size; i++) {
StackSummaryEntry entry = summary->contents[i];
if (summary && lookahead->symbol != ts_builtin_sym_error) {
for (unsigned i = 0; i < summary->size; i++) {
StackSummaryEntry entry = summary->contents[i];
if (entry.state == ERROR_STATE) continue;
if (entry.position.bytes == position.bytes) continue;
unsigned depth = entry.depth + ts_stack_depth_since_error(self->stack, version);
if (entry.state == ERROR_STATE) continue;
if (entry.position.bytes == position.bytes) continue;
unsigned depth = entry.depth;
if (node_count_since_error > 0) depth++;
unsigned new_cost =
depth * ERROR_COST_PER_SKIPPED_TREE +
(position.bytes - entry.position.bytes) * ERROR_COST_PER_SKIPPED_CHAR +
(position.extent.row - entry.position.extent.row) * ERROR_COST_PER_SKIPPED_LINE;
if (parser__better_version_exists(self, version, false, new_cost)) break;
bool would_merge = false;
for (unsigned j = 0; j < previous_version_count; j++) {
if (
ts_stack_state(self->stack, j) == entry.state &&
ts_stack_position(self->stack, j).bytes == position.bytes
) {
would_merge = true;
break;
}
}
if (ts_language_has_actions(self->language, entry.state, lookahead->symbol)) {
if (parser__recover_to_state(self, version, depth, entry.state)) {
did_recover = true;
LOG("recover state:%u, depth:%u", entry.state, depth);
break;
if (would_merge) continue;
unsigned new_cost =
current_error_cost +
entry.depth * ERROR_COST_PER_SKIPPED_TREE +
(position.bytes - entry.position.bytes) * ERROR_COST_PER_SKIPPED_CHAR +
(position.extent.row - entry.position.extent.row) * ERROR_COST_PER_SKIPPED_LINE;
if (parser__better_version_exists(self, version, false, new_cost)) break;
if (ts_language_has_actions(self->language, entry.state, lookahead->symbol)) {
if (parser__recover_to_state(self, version, depth, entry.state)) {
did_recover = true;
LOG("recover_to_previous state:%u, depth:%u", entry.state, depth);
LOG_STACK();
break;
}
}
}
}
for (unsigned i = previous_version_count; i < ts_stack_version_count(self->stack); i++) {
if (ts_stack_is_halted(self->stack, i)) {
if (!ts_stack_is_active(self->stack, i)) {
ts_stack_remove_version(self->stack, i--);
} else {
for (unsigned j = 0; j < i; j++) {
if (ts_stack_can_merge(self->stack, j, i)) {
ts_stack_remove_version(self->stack, i--);
break;
}
}
}
}
@ -1073,15 +1022,56 @@ static void parser__recover(Parser *self, StackVersion version, Tree *lookahead)
return;
}
unsigned new_cost =
current_error_cost + ERROR_COST_PER_SKIPPED_TREE +
ts_tree_total_bytes(lookahead) * ERROR_COST_PER_SKIPPED_CHAR +
ts_tree_total_size(lookahead).extent.row * ERROR_COST_PER_SKIPPED_LINE;
if (parser__better_version_exists(self, version, false, new_cost)) {
ts_stack_halt(self->stack, version);
return;
}
unsigned n;
const TSParseAction *actions = ts_language_actions(self->language, 1, lookahead->symbol, &n);
bool extra = n > 0 && actions[n - 1].type == TSParseActionTypeShift && actions[n - 1].params.extra;
parser__shift(self, version, ERROR_STATE, lookahead, extra);
if (n > 0 && actions[n - 1].type == TSParseActionTypeShift && actions[n - 1].params.extra) {
lookahead->extra = true;
}
if (parser__better_version_exists(self, version, true, ts_stack_error_cost(self->stack, version))) {
ts_stack_halt(self->stack, version);
} else {
LOG("skip_token symbol:%s", SYM_NAME(lookahead->symbol));
LOG("skip_token symbol:%s", SYM_NAME(lookahead->symbol));
ts_tree_retain(lookahead);
TreeArray children = array_new();
array_grow(&children, 1);
array_push(&children, lookahead);
Tree *error_repeat = ts_tree_make_node(
&self->tree_pool,
ts_builtin_sym_error_repeat,
&children,
0,
self->language
);
if (node_count_since_error > 0) {
StackSliceArray pop = ts_stack_pop_count(self->stack, version, 1);
assert(pop.size == 1);
assert(pop.contents[0].trees.size == 1);
ts_stack_renumber_version(self->stack, pop.contents[0].version, version);
array_push(&pop.contents[0].trees, error_repeat);
error_repeat = ts_tree_make_node(
&self->tree_pool,
ts_builtin_sym_error_repeat,
&pop.contents[0].trees,
0,
self->language
);
}
ts_stack_push(self->stack, version, error_repeat, false, ERROR_STATE);
if (lookahead->has_external_tokens) {
ts_stack_set_last_external_token(
self->stack, version, ts_tree_last_external_token(lookahead)
);
}
}
@ -1101,6 +1091,10 @@ static void parser__advance(Parser *self, StackVersion version, ReusableNode *re
if (action.params.repetition) break;
TSStateId next_state;
if (action.params.extra) {
// TODO remove when TREE_SITTER_LANGUAGE_VERSION 9 is out.
if (state == ERROR_STATE) continue;
next_state = state;
LOG("shift_extra");
} else {
@ -1108,7 +1102,7 @@ static void parser__advance(Parser *self, StackVersion version, ReusableNode *re
LOG("shift state:%u", next_state);
}
if (lookahead->child_count > 0) {
if (lookahead->children.size > 0) {
parser__breakdown_lookahead(self, &lookahead, state, reusable_node);
next_state = ts_language_next_state(self->language, state, lookahead->symbol);
}
@ -1140,7 +1134,7 @@ static void parser__advance(Parser *self, StackVersion version, ReusableNode *re
}
case TSParseActionTypeRecover: {
while (lookahead->child_count > 0) {
while (lookahead->children.size > 0) {
parser__breakdown_lookahead(self, &lookahead, state, reusable_node);
}
parser__recover(self, version, lookahead);
@ -1154,21 +1148,15 @@ static void parser__advance(Parser *self, StackVersion version, ReusableNode *re
if (last_reduction_version != STACK_VERSION_NONE) {
ts_stack_renumber_version(self->stack, last_reduction_version, version);
LOG_STACK();
} else if (state == ERROR_STATE) {
parser__recover(self, version, lookahead);
ts_tree_release(&self->tree_pool, lookahead);
return;
} else if (!parser__breakdown_top_of_stack(self, version)) {
if (state == ERROR_STATE) {
ts_stack_push(self->stack, version, lookahead, false, ERROR_STATE);
return;
}
parser__handle_error(self, version, lookahead->first_leaf.symbol);
if (ts_stack_is_halted(self->stack, version)) {
ts_tree_release(&self->tree_pool, lookahead);
return;
} else if (lookahead->size.bytes == 0) {
ts_tree_release(&self->tree_pool, lookahead);
state = ts_stack_state(self->stack, version);
lookahead = parser__get_lookahead(self, version, &state, reusable_node, &table_entry);
}
LOG("detect_error");
ts_stack_pause(self->stack, version, lookahead->first_leaf.symbol);
ts_tree_release(&self->tree_pool, lookahead);
return;
}
state = ts_stack_state(self->stack, version);
@ -1176,6 +1164,92 @@ static void parser__advance(Parser *self, StackVersion version, ReusableNode *re
}
}
static unsigned parser__condense_stack(Parser *self) {
bool made_changes = false;
unsigned min_error_cost = UINT_MAX;
for (StackVersion i = 0; i < ts_stack_version_count(self->stack); i++) {
if (ts_stack_is_halted(self->stack, i)) {
ts_stack_remove_version(self->stack, i);
i--;
continue;
}
ErrorStatus status_i = parser__version_status(self, i);
if (!status_i.is_in_error && status_i.cost < min_error_cost) {
min_error_cost = status_i.cost;
}
for (StackVersion j = 0; j < i; j++) {
ErrorStatus status_j = parser__version_status(self, j);
switch (parser__compare_versions(self, status_j, status_i)) {
case ErrorComparisonTakeLeft:
made_changes = true;
ts_stack_remove_version(self->stack, i);
i--;
j = i;
break;
case ErrorComparisonPreferLeft:
case ErrorComparisonNone:
if (ts_stack_merge(self->stack, j, i)) {
made_changes = true;
i--;
j = i;
}
break;
case ErrorComparisonPreferRight:
made_changes = true;
if (ts_stack_merge(self->stack, j, i)) {
i--;
j = i;
} else {
ts_stack_swap_versions(self->stack, i, j);
}
break;
case ErrorComparisonTakeRight:
made_changes = true;
ts_stack_remove_version(self->stack, j);
i--;
j--;
break;
}
}
}
while (ts_stack_version_count(self->stack) > MAX_VERSION_COUNT) {
ts_stack_remove_version(self->stack, MAX_VERSION_COUNT);
made_changes = true;
}
if (ts_stack_version_count(self->stack) > 0) {
bool has_unpaused_version = false;
for (StackVersion i = 0, n = ts_stack_version_count(self->stack); i < n; i++) {
if (ts_stack_is_paused(self->stack, i)) {
if (!has_unpaused_version && self->accept_count < MAX_VERSION_COUNT) {
LOG("resume version:%u", i);
min_error_cost = ts_stack_error_cost(self->stack, i);
TSSymbol lookahead_symbol = ts_stack_resume(self->stack, i);
parser__handle_error(self, i, lookahead_symbol);
has_unpaused_version = true;
} else {
ts_stack_remove_version(self->stack, i);
i--;
n--;
}
} else {
has_unpaused_version = true;
}
}
}
if (made_changes) {
LOG("condense");
LOG_STACK();
}
return min_error_cost;
}
bool parser_init(Parser *self) {
ts_lexer_init(&self->lexer);
array_init(&self->reduce_actions);
@ -1219,13 +1293,7 @@ Tree *parser_parse(Parser *self, TSInput input, Tree *old_tree, bool halt_on_err
for (version = 0; version < ts_stack_version_count(self->stack); version++) {
reusable_node = self->reusable_node;
while (!ts_stack_is_halted(self->stack, version)) {
position = ts_stack_position(self->stack, version).bytes;
if (position > last_position || (version > 0 && position == last_position)) {
last_position = position;
break;
}
while (ts_stack_is_active(self->stack, version)) {
LOG("process version:%d, version_count:%u, state:%d, row:%u, col:%u",
version, ts_stack_version_count(self->stack),
ts_stack_state(self->stack, version),
@ -1234,6 +1302,12 @@ Tree *parser_parse(Parser *self, TSInput input, Tree *old_tree, bool halt_on_err
parser__advance(self, version, &reusable_node);
LOG_STACK();
position = ts_stack_position(self->stack, version).bytes;
if (position > last_position || (version > 0 && position == last_position)) {
last_position = position;
break;
}
}
}

12
src/runtime/reusable_node.h
View file

@ -20,8 +20,8 @@ static inline void reusable_node_pop(ReusableNode *self) {
while (self->tree) {
Tree *parent = self->tree->context.parent;
uint32_t next_index = self->tree->context.index + 1;
if (parent && parent->child_count > next_index) {
self->tree = parent->children[next_index];
if (parent && parent->children.size > next_index) {
self->tree = parent->children.contents[next_index];
return;
}
self->tree = parent;
@ -30,17 +30,17 @@ static inline void reusable_node_pop(ReusableNode *self) {
static inline ReusableNode reusable_node_after_leaf(const ReusableNode *self) {
ReusableNode result = *self;
while (result.tree->child_count > 0)
result.tree = result.tree->children[0];
while (result.tree->children.size > 0)
result.tree = result.tree->children.contents[0];
reusable_node_pop(&result);
return result;
}
static inline bool reusable_node_breakdown(ReusableNode *self) {
if (self->tree->child_count == 0) {
if (self->tree->children.size == 0) {
return false;
} else {
self->tree = self->tree->children[0];
self->tree = self->tree->children.contents[0];
return true;
}
}

227
src/runtime/stack.c
View file

@ -1,4 +1,5 @@
#include "runtime/alloc.h"
#include "runtime/language.h"
#include "runtime/tree.h"
#include "runtime/array.h"
#include "runtime/stack.h"
@ -31,7 +32,7 @@ struct StackNode {
short unsigned int link_count;
uint32_t ref_count;
unsigned error_cost;
unsigned depth;
unsigned node_count;
int dynamic_precedence;
};
@ -49,12 +50,19 @@ typedef struct {
typedef Array(StackNode *) StackNodeArray;
typedef enum {
StackStatusActive,
StackStatusPaused,
StackStatusHalted,
} StackStatus;
typedef struct {
StackNode *node;
Tree *last_external_token;
uint32_t push_count;
bool is_halted;
StackSummary *summary;
unsigned node_count_at_last_error;
TSSymbol lookahead_when_paused;
StackStatus status;
} StackHead;
struct Stack {
@ -116,11 +124,9 @@ static StackNode *stack_node_new(StackNode *previous_node, Tree *tree, bool is_p
StackNode *node = pool->size > 0 ?
array_pop(pool) :
ts_malloc(sizeof(StackNode));
*node = (StackNode){.ref_count = 1, .link_count = 0, .state = state, .depth = 0};
*node = (StackNode){.ref_count = 1, .link_count = 0, .state = state};
if (previous_node) {
stack_node_retain(previous_node);
node->link_count = 1;
node->links[0] = (StackLink){
.node = previous_node,
@ -131,24 +137,13 @@ static StackNode *stack_node_new(StackNode *previous_node, Tree *tree, bool is_p
node->position = previous_node->position;
node->error_cost = previous_node->error_cost;
node->dynamic_precedence = previous_node->dynamic_precedence;
node->node_count = previous_node->node_count;
if (tree) {
node->depth = previous_node->depth;
if (!tree->extra) node->depth++;
node->error_cost += tree->error_cost;
node->position = length_add(node->position, ts_tree_total_size(tree));
node->dynamic_precedence += tree->dynamic_precedence;
if (state == ERROR_STATE && !tree->extra) {
node->error_cost +=
ERROR_COST_PER_SKIPPED_TREE * ((tree->visible || tree->child_count == 0) ? 1 : tree->visible_child_count) +
ERROR_COST_PER_SKIPPED_CHAR * tree->size.bytes +
ERROR_COST_PER_SKIPPED_LINE * tree->size.extent.row;
if (previous_node->links[0].tree) {
node->error_cost +=
ERROR_COST_PER_SKIPPED_CHAR * tree->padding.bytes +
ERROR_COST_PER_SKIPPED_LINE * tree->padding.extent.row;
}
}
if (!tree->extra) node->node_count += tree->node_count;
}
} else {
node->position = length_zero();
@ -165,7 +160,7 @@ static bool stack__tree_is_equivalent(const Tree *left, const Tree *right) {
right &&
left->symbol == right->symbol &&
((left->error_cost > 0 && right->error_cost > 0) ||
(left->child_count == 0 && right->child_count == 0 &&
(left->children.size == 0 && right->children.size == 0 &&
left->padding.bytes == right->padding.bytes &&
left->size.bytes == right->size.bytes &&
left->extra == right->extra &&
@ -189,11 +184,15 @@ static void stack_node_add_link(StackNode *self, StackLink link) {
}
}
if (self->link_count < MAX_LINK_COUNT) {
stack_node_retain(link.node);
if (link.tree) ts_tree_retain(link.tree);
self->links[self->link_count++] = link;
}
if (self->link_count == MAX_LINK_COUNT) return;
stack_node_retain(link.node);
if (link.tree) ts_tree_retain(link.tree);
self->links[self->link_count++] = link;
unsigned node_count = link.node->node_count;
if (link.tree) node_count += link.tree->node_count;
if (node_count > self->node_count) self->node_count = node_count;
}
static void stack_head_delete(StackHead *self, StackNodeArray *pool, TreePool *tree_pool) {
@ -210,21 +209,22 @@ static void stack_head_delete(StackHead *self, StackNodeArray *pool, TreePool *t
}
static StackVersion ts_stack__add_version(Stack *self, StackVersion original_version,
StackNode *node, Tree *last_external_token) {
StackNode *node) {
StackHead head = {
.node = node,
.push_count = self->heads.contents[original_version].push_count,
.last_external_token = last_external_token,
.is_halted = false,
.node_count_at_last_error = self->heads.contents[original_version].node_count_at_last_error,
.last_external_token = self->heads.contents[original_version].last_external_token,
.status = StackStatusActive,
.lookahead_when_paused = 0,
};
array_push(&self->heads, head);
stack_node_retain(node);
if (last_external_token) ts_tree_retain(last_external_token);
if (head.last_external_token) ts_tree_retain(head.last_external_token);
return (StackVersion)(self->heads.size - 1);
}
static void ts_stack__add_slice(Stack *self, StackVersion original_version, StackNode *node,
TreeArray *trees, Tree *last_external_token) {
static void ts_stack__add_slice(Stack *self, StackVersion original_version,
StackNode *node, TreeArray *trees) {
for (uint32_t i = self->slices.size - 1; i + 1 > 0; i--) {
StackVersion version = self->slices.contents[i].version;
if (self->heads.contents[version].node == node) {
@ -234,7 +234,7 @@ static void ts_stack__add_slice(Stack *self, StackVersion original_version, Stac
}
}
StackVersion version = ts_stack__add_version(self, original_version, node, last_external_token);
StackVersion version = ts_stack__add_version(self, original_version, node);
StackSlice slice = { *trees, version };
array_push(&self->slices, slice);
}
@ -246,7 +246,6 @@ inline StackSliceArray stack__iter(Stack *self, StackVersion version,
array_clear(&self->iterators);
StackHead *head = array_get(&self->heads, version);
Tree *last_external_token = head->last_external_token;
Iterator iterator = {
.node = head->node,
.trees = array_new(),
@ -273,8 +272,7 @@ inline StackSliceArray stack__iter(Stack *self, StackVersion version,
self,
version,
node,
&trees,
last_external_token
&trees
);
}
@ -375,14 +373,6 @@ Length ts_stack_position(const Stack *self, StackVersion version) {
return array_get(&self->heads, version)->node->position;
}
unsigned ts_stack_push_count(const Stack *self, StackVersion version) {
return array_get(&self->heads, version)->push_count;
}
void ts_stack_decrease_push_count(Stack *self, StackVersion version, unsigned decrement) {
array_get(&self->heads, version)->push_count -= decrement;
}
Tree *ts_stack_last_external_token(const Stack *self, StackVersion version) {
return array_get(&self->heads, version)->last_external_token;
}
@ -396,18 +386,27 @@ void ts_stack_set_last_external_token(Stack *self, StackVersion version, Tree *t
unsigned ts_stack_error_cost(const Stack *self, StackVersion version) {
StackHead *head = array_get(&self->heads, version);
return head->node->error_cost;
unsigned result = head->node->error_cost;
if (
head->status == StackStatusPaused ||
(head->node->state == ERROR_STATE && !head->node->links[0].tree)) {
result += ERROR_COST_PER_RECOVERY;
}
return result;
}
unsigned ts_stack_node_count_since_error(const Stack *self, StackVersion version) {
StackHead *head = array_get(&self->heads, version);
if (head->node->node_count < head->node_count_at_last_error) {
head->node_count_at_last_error = head->node->node_count;
}
return head->node->node_count - head->node_count_at_last_error;
}
void ts_stack_push(Stack *self, StackVersion version, Tree *tree, bool pending, TSStateId state) {
StackHead *head = array_get(&self->heads, version);
StackNode *new_node = stack_node_new(head->node, tree, pending, state, &self->node_pool);
if (state == ERROR_STATE) {
head->push_count = 0;
} else if (!tree->extra) {
head->push_count++;
}
stack_node_release(head->node, &self->node_pool, self->tree_pool);
if (!tree) head->node_count_at_last_error = new_node->node_count;
head->node = new_node;
}
@ -475,15 +474,21 @@ inline StackAction pop_error_callback(void *payload, const Iterator *iterator) {
}
}
StackSliceArray ts_stack_pop_error(Stack *self, StackVersion version) {
TreeArray ts_stack_pop_error(Stack *self, StackVersion version) {
StackNode *node = array_get(&self->heads, version)->node;
for (unsigned i = 0; i < node->link_count; i++) {
if (node->links[i].tree && node->links[i].tree->symbol == ts_builtin_sym_error) {
bool found_error = false;
return stack__iter(self, version, pop_error_callback, &found_error, true);
StackSliceArray pop = stack__iter(self, version, pop_error_callback, &found_error, true);
if (pop.size > 0) {
assert(pop.size == 1);
ts_stack_renumber_version(self, pop.contents[0].version, version);
return pop.contents[0].trees;
}
break;
}
}
return (StackSliceArray){.size = 0};
return (TreeArray){.size = 0};
}
inline StackAction pop_all_callback(void *payload, const Iterator *iterator) {
@ -531,10 +536,6 @@ StackSummary *ts_stack_get_summary(Stack *self, StackVersion version) {
return array_get(&self->heads, version)->summary;
}
unsigned ts_stack_depth_since_error(Stack *self, StackVersion version) {
return array_get(&self->heads, version)->node->depth;
}
int ts_stack_dynamic_precedence(Stack *self, StackVersion version) {
return array_get(&self->heads, version)->node->dynamic_precedence;
}
@ -547,8 +548,14 @@ void ts_stack_remove_version(Stack *self, StackVersion version) {
void ts_stack_renumber_version(Stack *self, StackVersion v1, StackVersion v2) {
assert(v2 < v1);
assert((uint32_t)v1 < self->heads.size);
stack_head_delete(&self->heads.contents[v2], &self->node_pool, self->tree_pool);
self->heads.contents[v2] = self->heads.contents[v1];
StackHead *source_head = &self->heads.contents[v1];
StackHead *target_head = &self->heads.contents[v2];
if (target_head->summary && !source_head->summary) {
source_head->summary = target_head->summary;
target_head->summary = NULL;
}
stack_head_delete(target_head, &self->node_pool, self->tree_pool);
*target_head = *source_head;
array_erase(&self->heads, v1);
}
@ -569,40 +576,61 @@ StackVersion ts_stack_copy_version(Stack *self, StackVersion version) {
}
bool ts_stack_merge(Stack *self, StackVersion version1, StackVersion version2) {
if (ts_stack_can_merge(self, version1, version2)) {
ts_stack_force_merge(self, version1, version2);
return true;
} else {
return false;
if (!ts_stack_can_merge(self, version1, version2)) return false;
StackHead *head1 = &self->heads.contents[version1];
StackHead *head2 = &self->heads.contents[version2];
for (uint32_t i = 0; i < head2->node->link_count; i++) {
stack_node_add_link(head1->node, head2->node->links[i]);
}
if (head1->node->state == ERROR_STATE) {
head1->node_count_at_last_error = head1->node->node_count;
}
ts_stack_remove_version(self, version2);
return true;
}
bool ts_stack_can_merge(Stack *self, StackVersion version1, StackVersion version2) {
StackHead *head1 = &self->heads.contents[version1];
StackHead *head2 = &self->heads.contents[version2];
return
!head1->is_halted && !head2->is_halted &&
head1->status == StackStatusActive &&
head2->status == StackStatusActive &&
head1->node->state == head2->node->state &&
head1->node->position.bytes == head2->node->position.bytes &&
head1->node->depth == head2->node->depth &&
head1->node->error_cost == head2->node->error_cost &&
ts_tree_external_token_state_eq(head1->last_external_token, head2->last_external_token);
}
void ts_stack_force_merge(Stack *self, StackVersion version1, StackVersion version2) {
StackHead *head1 = &self->heads.contents[version1];
StackHead *head2 = &self->heads.contents[version2];
for (uint32_t i = 0; i < head2->node->link_count; i++) {
stack_node_add_link(head1->node, head2->node->links[i]);
}
ts_stack_remove_version(self, version2);
}
void ts_stack_halt(Stack *self, StackVersion version) {
array_get(&self->heads, version)->is_halted = true;
array_get(&self->heads, version)->status = StackStatusHalted;
}
bool ts_stack_is_halted(Stack *self, StackVersion version) {
return array_get(&self->heads, version)->is_halted;
void ts_stack_pause(Stack *self, StackVersion version, TSSymbol lookahead) {
StackHead *head = array_get(&self->heads, version);
head->status = StackStatusPaused;
head->lookahead_when_paused = lookahead;
head->node_count_at_last_error = head->node->node_count;
}
bool ts_stack_is_active(const Stack *self, StackVersion version) {
return array_get(&self->heads, version)->status == StackStatusActive;
}
bool ts_stack_is_halted(const Stack *self, StackVersion version) {
return array_get(&self->heads, version)->status == StackStatusHalted;
}
bool ts_stack_is_paused(const Stack *self, StackVersion version) {
return array_get(&self->heads, version)->status == StackStatusPaused;
}
TSSymbol ts_stack_resume(Stack *self, StackVersion version) {
StackHead *head = array_get(&self->heads, version);
assert(head->status == StackStatusPaused);
TSSymbol result = head->lookahead_when_paused;
head->status = StackStatusActive;
head->lookahead_when_paused = 0;
return result;
}
void ts_stack_clear(Stack *self) {
@ -614,11 +642,12 @@ void ts_stack_clear(Stack *self) {
array_push(&self->heads, ((StackHead){
.node = self->base_node,
.last_external_token = NULL,
.is_halted = false,
.status = StackStatusActive,
.lookahead_when_paused = 0,
}));
}
bool ts_stack_print_dot_graph(Stack *self, const char **symbol_names, FILE *f) {
bool ts_stack_print_dot_graph(Stack *self, const TSLanguage *language, FILE *f) {
bool was_recording_allocations = ts_toggle_allocation_recording(false);
if (!f)
f = stderr;
@ -631,13 +660,20 @@ bool ts_stack_print_dot_graph(Stack *self, const char **symbol_names, FILE *f) {
array_clear(&self->iterators);
for (uint32_t i = 0; i < self->heads.size; i++) {
if (ts_stack_is_halted(self, i)) continue;
StackHead *head = &self->heads.contents[i];
if (head->status == StackStatusHalted) continue;
fprintf(f, "node_head_%u [shape=none, label=\"\"]\n", i);
fprintf(
f,
"node_head_%u -> node_%p [label=%u, fontcolor=blue, weight=10000, "
"labeltooltip=\"push_count: %u\ndepth: %u", i, head->node, i, head->push_count, head->node->depth
fprintf(f, "node_head_%u -> node_%p [", i, head->node);
if (head->status == StackStatusPaused) {
fprintf(f, "color=red ");
}
fprintf(f,
"label=%u, fontcolor=blue, weight=10000, labeltooltip=\"node_count: %u\nerror_cost: %u",
i,
ts_stack_node_count_since_error(self, i),
ts_stack_error_cost(self, i)
);
if (head->last_external_token) {
@ -680,9 +716,10 @@ bool ts_stack_print_dot_graph(Stack *self, const char **symbol_names, FILE *f) {
fprintf(
f,
" tooltip=\"position: %u,%u\nerror_cost: %u\ndynamic_precedence: %d\"];\n",
" tooltip=\"position: %u,%u\nnode_count:%u\nerror_cost: %u\ndynamic_precedence: %d\"];\n",
node->position.extent.row,
node->position.extent.column,
node->node_count,
node->error_cost,
node->dynamic_precedence
);
@ -697,21 +734,17 @@ bool ts_stack_print_dot_graph(Stack *self, const char **symbol_names, FILE *f) {
if (!link.tree) {
fprintf(f, "color=red");
} else if (link.tree->symbol == ts_builtin_sym_error) {
fprintf(f, "label=\"ERROR\"");
} else {
fprintf(f, "label=\"");
if (!link.tree->named)
fprintf(f, "'");
const char *name = symbol_names[link.tree->symbol];
if (link.tree->visible && !link.tree->named) fprintf(f, "'");
const char *name = ts_language_symbol_name(language, link.tree->symbol);
for (const char *c = name; *c; c++) {
if (*c == '\"' || *c == '\\')
fprintf(f, "\\");
if (*c == '\"' || *c == '\\') fprintf(f, "\\");
fprintf(f, "%c", *c);
}
if (!link.tree->named)
fprintf(f, "'");
fprintf(f, "\" labeltooltip=\"error_cost: %u\ndynamic_precedence: %u\"",
if (link.tree->visible && !link.tree->named) fprintf(f, "'");
fprintf(f, "\"");
fprintf(f, "labeltooltip=\"error_cost: %u\ndynamic_precedence: %u\"",
link.tree->error_cost,
link.tree->dynamic_precedence);
}

31
src/runtime/stack.h
View file

@ -41,15 +41,6 @@ uint32_t ts_stack_version_count(const Stack *);
// empty, this returns the initial state, 0.
TSStateId ts_stack_state(const Stack *, StackVersion);
// Get the number of trees that have been pushed to a given version of
// the stack.
unsigned ts_stack_push_count(const Stack *, StackVersion);
// In the event that trees were permanently removed from some version
// of the stack, decrease the version's push count to account for the
// removal.
void ts_stack_decrease_push_count(Stack *, StackVersion, unsigned);
// Get the last external token associated with a given version of the stack.
Tree *ts_stack_last_external_token(const Stack *, StackVersion);
@ -74,7 +65,7 @@ void ts_stack_push(Stack *, StackVersion, Tree *, bool, TSStateId);
StackSliceArray ts_stack_pop_count(Stack *, StackVersion, uint32_t count);
// Remove an error at the top of the given version of the stack.
StackSliceArray ts_stack_pop_error(Stack *, StackVersion);
TreeArray ts_stack_pop_error(Stack *, StackVersion);
// Remove any pending trees from the top of the given version of the stack.
StackSliceArray ts_stack_pop_pending(Stack *, StackVersion);
@ -82,7 +73,9 @@ StackSliceArray ts_stack_pop_pending(Stack *, StackVersion);
// Remove any all trees from the given version of the stack.
StackSliceArray ts_stack_pop_all(Stack *, StackVersion);
unsigned ts_stack_depth_since_error(Stack *, StackVersion);
// Get the maximum number of tree nodes reachable from this version of the stack
// since the last error was detected.
unsigned ts_stack_node_count_since_error(const Stack *, StackVersion);
int ts_stack_dynamic_precedence(Stack *, StackVersion);
@ -94,17 +87,27 @@ void ts_stack_record_summary(Stack *, StackVersion, unsigned max_depth);
// given version of the stack.
StackSummary *ts_stack_get_summary(Stack *, StackVersion);
// Get the total cost of all errors on the given version of the stack.
unsigned ts_stack_error_cost(const Stack *, StackVersion version);
// Determine whether the given two stack versions can be merged.
bool ts_stack_merge(Stack *, StackVersion, StackVersion);
// Merge the given two stack versions if possible, returning true
// if they were successfully merged and false otherwise.
bool ts_stack_can_merge(Stack *, StackVersion, StackVersion);
void ts_stack_force_merge(Stack *, StackVersion, StackVersion);
TSSymbol ts_stack_resume(Stack *, StackVersion);
void ts_stack_pause(Stack *, StackVersion, TSSymbol);
void ts_stack_halt(Stack *, StackVersion);
bool ts_stack_is_halted(Stack *, StackVersion);
bool ts_stack_is_active(const Stack *, StackVersion);
bool ts_stack_is_paused(const Stack *, StackVersion);
bool ts_stack_is_halted(const Stack *, StackVersion);
void ts_stack_renumber_version(Stack *, StackVersion, StackVersion);
@ -117,7 +120,7 @@ void ts_stack_remove_version(Stack *, StackVersion);
void ts_stack_clear(Stack *);
bool ts_stack_print_dot_graph(Stack *, const char **, FILE *);
bool ts_stack_print_dot_graph(Stack *, const TSLanguage *, FILE *);
typedef void (*StackIterateCallback)(void *, TSStateId, uint32_t);

205
src/runtime/tree.c
View file

@ -68,16 +68,6 @@ void ts_tree_array_delete(TreePool *pool, TreeArray *self) {
array_delete(self);
}
uint32_t ts_tree_array_essential_count(const TreeArray *self) {
uint32_t result = 0;
for (uint32_t i = 0; i < self->size; i++) {
Tree *tree = self->contents[i];
if (!tree->extra && tree->symbol != ts_builtin_sym_error)
result++;
}
return result;
}
TreeArray ts_tree_array_remove_last_n(TreeArray *self, uint32_t remove_count) {
TreeArray result = array_new();
if (self->size == 0 || remove_count == 0) return result;
@ -169,14 +159,13 @@ Tree *ts_tree_make_leaf(TreePool *pool, TSSymbol symbol, Length padding, Length
.ref_count = 1,
.symbol = symbol,
.size = size,
.child_count = 0,
.children = NULL,
.visible_child_count = 0,
.named_child_count = 0,
.alias_sequence_id = 0,
.padding = padding,
.visible = metadata.visible,
.named = metadata.named,
.node_count = 1,
.has_changes = false,
.first_leaf = {
.symbol = symbol,
@ -206,50 +195,50 @@ Tree *ts_tree_make_copy(TreePool *pool, Tree *self) {
static void ts_tree__compress(Tree *self, unsigned count, const TSLanguage *language) {
Tree *tree = self;
for (unsigned i = 0; i < count; i++) {
if (tree->ref_count > 1 || tree->child_count != 2) break;
if (tree->ref_count > 1 || tree->children.size != 2) break;
Tree *child = tree->children[0];
Tree *child = tree->children.contents[0];
if (
child->ref_count > 1 ||
child->child_count != 2 ||
child->children.size != 2 ||
child->symbol != tree->symbol
) break;
Tree *grandchild = child->children[0];
Tree *grandchild = child->children.contents[0];
if (
grandchild->ref_count > 1 ||
grandchild->child_count != 2 ||
grandchild->children.size != 2 ||
grandchild->symbol != tree->symbol
) break;
tree->children[0] = grandchild;
tree->children.contents[0] = grandchild;
grandchild->context.parent = tree;
grandchild->context.index = -1;
child->children[0] = grandchild->children[1];
child->children[0]->context.parent = child;
child->children[0]->context.index = -1;
child->children.contents[0] = grandchild->children.contents[1];
child->children.contents[0]->context.parent = child;
child->children.contents[0]->context.index = -1;
grandchild->children[1] = child;
grandchild->children[1]->context.parent = grandchild;
grandchild->children[1]->context.index = -1;
grandchild->children.contents[1] = child;
grandchild->children.contents[1]->context.parent = grandchild;
grandchild->children.contents[1]->context.index = -1;
tree = grandchild;
}
while (tree != self) {
tree = tree->context.parent;
Tree *child = tree->children[0];
Tree *grandchild = child->children[1];
ts_tree_set_children(grandchild, 2, grandchild->children, language);
ts_tree_set_children(child, 2, child->children, language);
ts_tree_set_children(tree, 2, tree->children, language);
Tree *child = tree->children.contents[0];
Tree *grandchild = child->children.contents[1];
ts_tree_set_children(grandchild, &grandchild->children, language);
ts_tree_set_children(child, &child->children, language);
ts_tree_set_children(tree, &tree->children, language);
}
}
void ts_tree__balance(Tree *self, const TSLanguage *language) {
if (self->children[0]->repeat_depth > self->children[1]->repeat_depth) {
unsigned n = self->children[0]->repeat_depth - self->children[1]->repeat_depth;
if (self->children.contents[0]->repeat_depth > self->children.contents[1]->repeat_depth) {
unsigned n = self->children.contents[0]->repeat_depth - self->children.contents[1]->repeat_depth;
for (unsigned i = n / 2; i > 0; i /= 2) {
ts_tree__compress(self, i, language);
n -= i;
@ -272,8 +261,8 @@ void ts_tree_assign_parents(Tree *self, TreePool *pool, const TSLanguage *langua
const TSSymbol *alias_sequence = ts_language_alias_sequence(language, tree->alias_sequence_id);
uint32_t non_extra_index = 0;
bool earlier_child_was_changed = false;
for (uint32_t i = 0; i < tree->child_count; i++) {
Tree *child = tree->children[i];
for (uint32_t i = 0; i < tree->children.size; i++) {
Tree *child = tree->children.contents[i];
if (earlier_child_was_changed || child->context.parent != tree || child->context.index != i) {
earlier_child_was_changed = true;
child->context.parent = tree;
@ -295,24 +284,25 @@ void ts_tree_assign_parents(Tree *self, TreePool *pool, const TSLanguage *langua
}
}
void ts_tree_set_children(Tree *self, uint32_t child_count, Tree **children,
const TSLanguage *language) {
if (self->child_count > 0 && children != self->children) ts_free(self->children);
void ts_tree_set_children(Tree *self, TreeArray *children, const TSLanguage *language) {
if (self->children.size > 0 && children->contents != self->children.contents) {
array_delete(&self->children);
}
self->children = children;
self->child_count = child_count;
self->children = *children;
self->named_child_count = 0;
self->visible_child_count = 0;
self->error_cost = 0;
self->repeat_depth = 0;
self->node_count = 1;
self->has_external_tokens = false;
self->dynamic_precedence = 0;
uint32_t non_extra_index = 0;
const TSSymbol *alias_sequence = ts_language_alias_sequence(language, self->alias_sequence_id);
for (uint32_t i = 0; i < child_count; i++) {
Tree *child = children[i];
for (uint32_t i = 0; i < self->children.size; i++) {
Tree *child = self->children.contents[i];
if (i == 0) {
self->padding = child->padding;
@ -324,8 +314,11 @@ void ts_tree_set_children(Tree *self, uint32_t child_count, Tree **children,
self->size = length_add(self->size, ts_tree_total_size(child));
}
self->error_cost += child->error_cost;
if (child->symbol != ts_builtin_sym_error_repeat) {
self->error_cost += child->error_cost;
}
self->dynamic_precedence += child->dynamic_precedence;
self->node_count += child->node_count;
if (alias_sequence && alias_sequence[non_extra_index] != 0 && !child->extra) {
self->visible_child_count++;
@ -335,7 +328,7 @@ void ts_tree_set_children(Tree *self, uint32_t child_count, Tree **children,
} else if (child->visible) {
self->visible_child_count++;
if (child->named) self->named_child_count++;
} else if (child->child_count > 0) {
} else if (child->children.size > 0) {
self->visible_child_count += child->visible_child_count;
self->named_child_count += child->named_child_count;
}
@ -350,72 +343,63 @@ void ts_tree_set_children(Tree *self, uint32_t child_count, Tree **children,
if (!child->extra) non_extra_index++;
}
if (self->symbol == ts_builtin_sym_error) {
self->error_cost += ERROR_COST_PER_SKIPPED_CHAR * self->size.bytes +
if (self->symbol == ts_builtin_sym_error || self->symbol == ts_builtin_sym_error_repeat) {
self->error_cost += ERROR_COST_PER_RECOVERY +
ERROR_COST_PER_SKIPPED_CHAR * self->size.bytes +
ERROR_COST_PER_SKIPPED_LINE * self->size.extent.row;
for (uint32_t i = 0; i < child_count; i++)
if (!self->children[i]->extra)
for (uint32_t i = 0; i < self->children.size; i++) {
Tree *child = self->children.contents[i];
if (child->extra) continue;
if (child->symbol == ts_builtin_sym_error && child->children.size == 0) continue;
if (child->visible) {
self->error_cost += ERROR_COST_PER_SKIPPED_TREE;
} else {
self->error_cost += ERROR_COST_PER_SKIPPED_TREE * child->visible_child_count;
}
}
}
if (child_count > 0) {
self->first_leaf = children[0]->first_leaf;
if (children[0]->fragile_left) {
self->fragile_left = true;
}
if (children[child_count - 1]->fragile_right) {
self->fragile_right = true;
}
if (self->children.size > 0) {
Tree *first_child = self->children.contents[0];
Tree *last_child = self->children.contents[self->children.size - 1];
self->first_leaf = first_child->first_leaf;
if (first_child->fragile_left) self->fragile_left = true;
if (last_child->fragile_right) self->fragile_right = true;
if (
self->child_count == 2 &&
self->children.size == 2 &&
!self->visible && !self->named &&
self->children[0]->symbol == self->symbol &&
self->children[1]->symbol == self->symbol
first_child->symbol == self->symbol &&
last_child->symbol == self->symbol
) {
if (self->children[0]->repeat_depth > self->children[1]->repeat_depth) {
self->repeat_depth = self->children[0]->repeat_depth + 1;
if (first_child->repeat_depth > last_child->repeat_depth) {
self->repeat_depth = first_child->repeat_depth + 1;
} else {
self->repeat_depth = self->children[1]->repeat_depth + 1;
self->repeat_depth = last_child->repeat_depth + 1;
}
}
}
}
Tree *ts_tree_make_node(TreePool *pool, TSSymbol symbol, uint32_t child_count, Tree **children,
Tree *ts_tree_make_node(TreePool *pool, TSSymbol symbol, TreeArray *children,
unsigned alias_sequence_id, const TSLanguage *language) {
Tree *result = ts_tree_make_leaf(pool, symbol, length_zero(), length_zero(), language);
result->alias_sequence_id = alias_sequence_id;
ts_tree_set_children(result, child_count, children, language);
if (symbol == ts_builtin_sym_error || symbol == ts_builtin_sym_error_repeat) {
result->fragile_left = true;
result->fragile_right = true;
}
ts_tree_set_children(result, children, language);
return result;
}
Tree *ts_tree_make_error_node(TreePool *pool, TreeArray *children, const TSLanguage *language) {
for (uint32_t i = 0; i < children->size; i++) {
Tree *child = children->contents[i];
if (child->symbol == ts_builtin_sym_error && child->child_count > 0) {
array_splice(children, i, 1, child->child_count, child->children);
i += child->child_count - 1;
for (uint32_t j = 0; j < child->child_count; j++)
ts_tree_retain(child->children[j]);
ts_tree_release(pool, child);
}
}
Tree *result = ts_tree_make_node(
pool, ts_builtin_sym_error,
children->size, children->contents,
0, language
);
result->fragile_left = true;
result->fragile_right = true;
return result;
return ts_tree_make_node(pool, ts_builtin_sym_error, children, 0, language);
}
Tree *ts_tree_make_missing_leaf(TreePool *pool, TSSymbol symbol, const TSLanguage *language) {
Tree *result = ts_tree_make_leaf(pool, symbol, length_zero(), length_zero(), language);
result->is_missing = true;
result->error_cost = ERROR_COST_PER_MISSING_TREE;
result->error_cost = ERROR_COST_PER_MISSING_TREE + ERROR_COST_PER_RECOVERY;
return result;
}
@ -433,11 +417,11 @@ void ts_tree_release(TreePool *pool, Tree *self) {
assert(tree->ref_count > 0);
tree->ref_count--;
if (tree->ref_count == 0) {
if (tree->child_count > 0) {
for (uint32_t i = 0; i < tree->child_count; i++) {
array_push(&pool->tree_stack, tree->children[i]);
if (tree->children.size > 0) {
for (uint32_t i = 0; i < tree->children.size; i++) {
array_push(&pool->tree_stack, tree->children.contents[i]);
}
ts_free(tree->children);
array_delete(&tree->children);
} else if (tree->has_external_tokens) {
ts_external_token_state_delete(&tree->external_token_state);
}
@ -478,12 +462,12 @@ bool ts_tree_eq(const Tree *self, const Tree *other) {
if (self->padding.bytes != other->padding.bytes) return false;
if (self->size.bytes != other->size.bytes) return false;
if (self->symbol == ts_builtin_sym_error) return self->lookahead_char == other->lookahead_char;
if (self->child_count != other->child_count) return false;
if (self->children.size != other->children.size) return false;
if (self->visible_child_count != other->visible_child_count) return false;
if (self->named_child_count != other->named_child_count) return false;
for (uint32_t i = 0; i < self->child_count; i++) {
if (!ts_tree_eq(self->children[i], other->children[i])) {
for (uint32_t i = 0; i < self->children.size; i++) {
if (!ts_tree_eq(self->children.contents[i], other->children.contents[i])) {
return false;
}
}
@ -495,13 +479,13 @@ int ts_tree_compare(const Tree *left, const Tree *right) {
return -1;
if (right->symbol < left->symbol)
return 1;
if (left->child_count < right->child_count)
if (left->children.size < right->children.size)
return -1;
if (right->child_count < left->child_count)
if (right->children.size < left->children.size)
return 1;
for (uint32_t i = 0; i < left->child_count; i++) {
Tree *left_child = left->children[i];
Tree *right_child = right->children[i];
for (uint32_t i = 0; i < left->children.size; i++) {
Tree *left_child = left->children.contents[i];
Tree *right_child = right->children.contents[i];
switch (ts_tree_compare(left_child, right_child)) {
case -1:
return -1;
@ -521,10 +505,10 @@ static inline long min_byte(long a, long b) {
bool ts_tree_invalidate_lookahead(Tree *self, uint32_t edit_byte_offset) {
if (edit_byte_offset >= self->bytes_scanned) return false;
self->has_changes = true;
if (self->child_count > 0) {
if (self->children.size > 0) {
uint32_t child_start_byte = 0;
for (uint32_t i = 0; i < self->child_count; i++) {
Tree *child = self->children[i];
for (uint32_t i = 0; i < self->children.size; i++) {
Tree *child = self->children.contents[i];
if (child_start_byte > edit_byte_offset) break;
ts_tree_invalidate_lookahead(child, edit_byte_offset - child_start_byte);
child_start_byte += ts_tree_total_bytes(child);
@ -575,8 +559,8 @@ void ts_tree_edit(Tree *self, const TSInputEdit *edit) {
long remaining_bytes_to_delete = 0;
TSPoint remaining_extent_to_delete = {0, 0};
Length child_left, child_right = length_zero();
for (uint32_t i = 0; i < self->child_count; i++) {
Tree *child = self->children[i];
for (uint32_t i = 0; i < self->children.size; i++) {
Tree *child = self->children.contents[i];
child_left = child_right;
child_right = length_add(child_left, ts_tree_total_size(child));
@ -622,9 +606,9 @@ void ts_tree_edit(Tree *self, const TSInputEdit *edit) {
Tree *ts_tree_last_external_token(Tree *tree) {
if (!tree->has_external_tokens) return NULL;
while (tree->child_count > 0) {
for (uint32_t i = tree->child_count - 1; i + 1 > 0; i--) {
Tree *child = tree->children[i];
while (tree->children.size > 0) {
for (uint32_t i = tree->children.size - 1; i + 1 > 0; i--) {
Tree *child = tree->children.contents[i];
if (child->has_external_tokens) {
tree = child;
break;
@ -670,7 +654,7 @@ static size_t ts_tree__write_to_string(const Tree *self, const TSLanguage *langu
}
if (visible) {
if (self->symbol == ts_builtin_sym_error && self->child_count == 0 && self->size.bytes > 0) {
if (self->symbol == ts_builtin_sym_error && self->children.size == 0 && self->size.bytes > 0) {
cursor += snprintf(*writer, limit, "(UNEXPECTED ");
cursor += ts_tree__write_char_to_string(*writer, limit, self->lookahead_char);
} else if (self->is_missing) {
@ -682,8 +666,8 @@ static size_t ts_tree__write_to_string(const Tree *self, const TSLanguage *langu
}
}
for (uint32_t i = 0; i < self->child_count; i++) {
Tree *child = self->children[i];
for (uint32_t i = 0; i < self->children.size; i++) {
Tree *child = self->children.contents[i];
cursor += ts_tree__write_to_string(child, language, *writer, limit, false, include_all);
}
@ -705,7 +689,7 @@ void ts_tree__print_dot_graph(const Tree *self, uint32_t byte_offset,
TSSymbol symbol = self->context.alias_symbol ? self->context.alias_symbol : self->symbol;
fprintf(f, "tree_%p [label=\"%s\"", self, ts_language_symbol_name(language, symbol));
if (self->child_count == 0)
if (self->children.size == 0)
fprintf(f, ", shape=plaintext");
if (self->extra)
fprintf(f, ", fontcolor=gray");
@ -713,16 +697,15 @@ void ts_tree__print_dot_graph(const Tree *self, uint32_t byte_offset,
fprintf(f, ", tooltip=\"address:%p\nrange:%u - %u\nstate:%d\nerror-cost:%u\nrepeat-depth:%u\"]\n",
self, byte_offset, byte_offset + ts_tree_total_bytes(self), self->parse_state,
self->error_cost, self->repeat_depth);
for (uint32_t i = 0; i < self->child_count; i++) {
const Tree *child = self->children[i];
for (uint32_t i = 0; i < self->children.size; i++) {
const Tree *child = self->children.contents[i];
ts_tree__print_dot_graph(child, byte_offset, language, f);
fprintf(f, "tree_%p -> tree_%p [tooltip=%u]\n", self, child, i);
byte_offset += ts_tree_total_bytes(child);
}
}
void ts_tree_print_dot_graph(const Tree *self, const TSLanguage *language,
FILE *f) {
void ts_tree_print_dot_graph(const Tree *self, const TSLanguage *language, FILE *f) {
fprintf(f, "digraph tree {\n");
fprintf(f, "edge [arrowhead=none]\n");
ts_tree__print_dot_graph(self, 0, language, f);

72
src/runtime/tree.h
View file

@ -17,12 +17,16 @@ extern TSStateId TS_TREE_STATE_NONE;
typedef struct {
union {
char *long_data;
char short_data[sizeof(char *) + sizeof(unsigned)];
char short_data[sizeof(char *) + sizeof(uint32_t)];
};
unsigned length;
uint32_t length;
} TSExternalTokenState;
typedef struct Tree {
typedef struct Tree Tree;
typedef Array(Tree *) TreeArray;
struct Tree {
struct {
struct Tree *parent;
uint32_t index;
@ -31,34 +35,16 @@ typedef struct Tree {
bool alias_is_named : 1;
} context;
uint32_t child_count;
union {
struct {
struct Tree **children;
uint32_t visible_child_count;
uint32_t named_child_count;
unsigned short alias_sequence_id;
};
TSExternalTokenState external_token_state;
int32_t lookahead_char;
};
Length padding;
Length size;
uint32_t bytes_scanned;
TSSymbol symbol;
TSStateId parse_state;
unsigned error_cost;
unsigned repeat_depth;
struct {
TSSymbol symbol;
TSLexMode lex_mode;
} first_leaf;
uint32_t ref_count;
int dynamic_precedence;
uint32_t bytes_scanned;
uint32_t error_cost;
uint32_t node_count;
uint32_t repeat_depth;
uint32_t child_count;
int32_t dynamic_precedence;
bool visible : 1;
bool named : 1;
bool extra : 1;
@ -67,9 +53,30 @@ typedef struct Tree {
bool has_changes : 1;
bool has_external_tokens : 1;
bool is_missing : 1;
} Tree;
TSSymbol symbol;
TSStateId parse_state;
struct {
TSSymbol symbol;
TSLexMode lex_mode;
} first_leaf;
typedef Array(Tree *) TreeArray;
union {
struct {
TreeArray children;
uint32_t visible_child_count;
uint32_t named_child_count;
uint16_t alias_sequence_id;
};
struct {
uint32_t _2;
TSExternalTokenState external_token_state;
};
struct {
uint32_t _1;
int32_t lookahead_char;
};
};
};
typedef struct {
TreeArray free_trees;
@ -81,7 +88,6 @@ const char *ts_external_token_state_data(const TSExternalTokenState *);
bool ts_tree_array_copy(TreeArray, TreeArray *);
void ts_tree_array_delete(TreePool *, TreeArray *);
uint32_t ts_tree_array_essential_count(const TreeArray *);
TreeArray ts_tree_array_remove_last_n(TreeArray *, uint32_t);
TreeArray ts_tree_array_remove_trailing_extras(TreeArray *);
void ts_tree_array_reverse(TreeArray *);
@ -92,7 +98,7 @@ Tree *ts_tree_pool_allocate(TreePool *);
void ts_tree_pool_free(TreePool *, Tree *);
Tree *ts_tree_make_leaf(TreePool *, TSSymbol, Length, Length, const TSLanguage *);
Tree *ts_tree_make_node(TreePool *, TSSymbol, uint32_t, Tree **, unsigned, const TSLanguage *);
Tree *ts_tree_make_node(TreePool *, TSSymbol, TreeArray *, unsigned, const TSLanguage *);
Tree *ts_tree_make_copy(TreePool *, Tree *child);
Tree *ts_tree_make_error_node(TreePool *, TreeArray *, const TSLanguage *);
Tree *ts_tree_make_error(TreePool *, Length, Length, int32_t, const TSLanguage *);
@ -103,7 +109,7 @@ bool ts_tree_eq(const Tree *tree1, const Tree *tree2);
int ts_tree_compare(const Tree *tree1, const Tree *tree2);
uint32_t ts_tree_start_column(const Tree *self);
uint32_t ts_tree_end_column(const Tree *self);
void ts_tree_set_children(Tree *, uint32_t, Tree **, const TSLanguage *);
void ts_tree_set_children(Tree *, TreeArray *, const TSLanguage *);
void ts_tree_assign_parents(Tree *, TreePool *, const TSLanguage *);
void ts_tree_edit(Tree *, const TSInputEdit *edit);
char *ts_tree_string(const Tree *, const TSLanguage *, bool include_all);

25
test/fixtures/error_corpus/c_errors.txt vendored
View file

@ -81,7 +81,9 @@ int main() {
(function_declarator (identifier) (parameter_list))
(compound_statement
(if_statement
(field_expression (identifier) (MISSING))
(field_expression
(identifier)
(MISSING))
(compound_statement
(expression_statement (call_expression (identifier) (argument_list)))
(expression_statement (call_expression (identifier) (argument_list)))
@ -141,3 +143,24 @@ int y = 5;
(translation_unit
(declaration (primitive_type) (ERROR (identifier)) (identifier))
(declaration (primitive_type) (init_declarator (identifier) (number_literal))))
==========================================
Declarations with missing variable names
==========================================
int a() {
struct x = 1;
int = 2;
}
---
(translation_unit
(function_definition
(primitive_type)
(function_declarator (identifier) (parameter_list))
(compound_statement
(struct_specifier (type_identifier))
(ERROR (number_literal))
(primitive_type)
(ERROR (number_literal)))))

44
test/fixtures/error_corpus/javascript_errors.txt vendored
View file

@ -77,14 +77,12 @@ if ({a: 'b'} {c: 'd'}) {
(ERROR (object (pair (property_identifier) (string))))
(object (pair (property_identifier) (string))))
(statement_block
(expression_statement (assignment_expression
(identifier)
(ERROR (function
(formal_parameters (identifier))
(statement_block (expression_statement (identifier)))))
(function
(formal_parameters (identifier))
(statement_block (expression_statement (identifier)))))))))
(expression_statement
(assignment_expression
(identifier)
(function (formal_parameters (identifier)) (statement_block (expression_statement (identifier)))))
(MISSING))
(function (formal_parameters (identifier)) (statement_block (expression_statement (identifier)))))))
===================================================
Extra tokens at the end of the file
@ -149,3 +147,33 @@ const a = `b c ${d +} f g`
(variable_declarator
(identifier)
(template_string (template_substitution (identifier) (ERROR))))))
=========================================================
Long sequences of invalid tokens
=========================================================
function main(x) {
console.log('a');
what??????????????????????????????????????????????????
console.log('b');
return {};
}
---
(program
(function
(identifier)
(formal_parameters (identifier))
(statement_block
(expression_statement
(call_expression
(member_expression (identifier) (property_identifier))
(arguments (string))))
(expression_statement
(identifier)
(ERROR
(call_expression
(member_expression (identifier) (property_identifier))
(arguments (string)))))
(return_statement (object)))))

11
test/helpers/tree_helpers.cc
View file

@ -16,11 +16,14 @@ const char *symbol_names[24] = {
"twenty-two", "twenty-three"
};
Tree ** tree_array(std::vector<Tree *> trees) {
Tree ** result = (Tree **)calloc(trees.size(), sizeof(Tree *));
TreeArray *tree_array(std::vector<Tree *> trees) {
static TreeArray result;
result.capacity = trees.size();
result.size = trees.size();
result.contents = (Tree **)calloc(trees.size(), sizeof(Tree *));
for (size_t i = 0; i < trees.size(); i++)
result[i] = trees[i];
return result;
result.contents[i] = trees[i];
return &result;
}
ostream &operator<<(std::ostream &stream, const Tree *tree) {

2
test/helpers/tree_helpers.h
View file

@ -6,7 +6,7 @@
#include <string>
extern const char *symbol_names[24];
Tree ** tree_array(std::vector<Tree *> trees);
TreeArray *tree_array(std::vector<Tree *> trees);
std::ostream &operator<<(std::ostream &stream, const Tree *tree);
std::ostream &operator<<(std::ostream &stream, const TSNode &node);

5
test/integration/fuzzing-examples.cc
View file

@ -30,7 +30,10 @@ describe("examples found via fuzzing", [&]() {
it(("parses example number " + to_string(i)).c_str(), [&]() {
TSDocument *document = ts_document_new();
// ts_document_print_debugging_graphs(document, true);
if (getenv("TREE_SITTER_ENABLE_DEBUG_GRAPHS")) {
ts_document_print_debugging_graphs(document, true);
}
const string &language_name = examples[i].first;
ts_document_set_language(document, load_real_language(language_name));

8
test/integration/real_grammars.cc
View file

@ -32,6 +32,7 @@ vector<string> test_languages({
for (auto &language_name : test_languages) {
describe(("the " + language_name + " language").c_str(), [&]() {
TSDocument *document;
const bool debug_graphs_enabled = getenv("TREE_SITTER_ENABLE_DEBUG_GRAPHS");
before_each([&]() {
record_alloc::start();
@ -39,7 +40,7 @@ for (auto &language_name : test_languages) {
ts_document_set_language(document, load_real_language(language_name));
// ts_document_set_logger(document, stderr_logger_new(true));
if (getenv("TREE_SITTER_ENABLE_DEBUG_GRAPHS")) {
if (debug_graphs_enabled) {
ts_document_print_debugging_graphs(document, true);
}
});
@ -55,6 +56,7 @@ for (auto &language_name : test_languages) {
auto it_handles_edit_sequence = [&](string name, std::function<void()> edit_sequence){
it(("parses " + entry.description + ": " + name).c_str(), [&]() {
input = new SpyInput(entry.input, 3);
if (debug_graphs_enabled) printf("%s\n\n", input->content.c_str());
ts_document_set_input(document, input->input());
edit_sequence();
@ -88,9 +90,11 @@ for (auto &language_name : test_languages) {
ts_document_edit(document, input->replace(edit_position, 0, inserted_text));
ts_document_parse(document);
assert_correct_tree_size(document, input->content);
if (debug_graphs_enabled) printf("%s\n\n", input->content.c_str());
ts_document_edit(document, input->undo());
assert_correct_tree_size(document, input->content);
if (debug_graphs_enabled) printf("%s\n\n", input->content.c_str());
TSRange *ranges;
uint32_t range_count;
@ -112,9 +116,11 @@ for (auto &language_name : test_languages) {
ts_document_edit(document, input->replace(edit_position, deletion_size, ""));
ts_document_parse(document);
assert_correct_tree_size(document, input->content);
if (debug_graphs_enabled) printf("%s\n\n", input->content.c_str());
ts_document_edit(document, input->undo());
assert_correct_tree_size(document, input->content);
if (debug_graphs_enabled) printf("%s\n\n", input->content.c_str());
TSRange *ranges;
uint32_t range_count;

6
test/runtime/document_test.cc
View file

@ -22,6 +22,10 @@ describe("Document", [&]() {
before_each([&]() {
record_alloc::start();
document = ts_document_new();
if (getenv("TREE_SITTER_ENABLE_DEBUG_GRAPHS")) {
ts_document_print_debugging_graphs(document, true);
}
});
after_each([&]() {
@ -434,7 +438,7 @@ describe("Document", [&]() {
root = ts_document_root_node(document);
assert_node_string_equals(
root,
"(ERROR (number) (null) (UNEXPECTED 'e'))");
"(ERROR (number) (null))");
AssertThat(ts_node_end_byte(root), Equals(input_string.size()));
});

88
test/runtime/tree_test.cc
View file

@ -7,11 +7,11 @@
void assert_consistent(const Tree *tree) {
if (tree->child_count == 0)
return;
AssertThat(tree->children[0]->padding, Equals<Length>(tree->padding));
AssertThat(tree->children.contents[0]->padding, Equals<Length>(tree->padding));
Length total_children_size = length_zero();
for (size_t i = 0; i < tree->child_count; i++) {
Tree *child = tree->children[i];
for (size_t i = 0; i < tree->children.size; i++) {
Tree *child = tree->children.contents[i];
AssertThat(child->context.offset, Equals(total_children_size));
assert_consistent(child);
total_children_size = length_add(total_children_size, ts_tree_total_size(child));
@ -86,7 +86,7 @@ describe("Tree", []() {
ts_tree_retain(tree1);
ts_tree_retain(tree2);
parent1 = ts_tree_make_node(&pool, symbol3, 2, tree_array({
parent1 = ts_tree_make_node(&pool, symbol3, tree_array({
tree1,
tree2,
}), 0, &language);
@ -114,7 +114,7 @@ describe("Tree", []() {
ts_tree_retain(tree1);
ts_tree_retain(tree2);
parent = ts_tree_make_node(&pool, symbol3, 2, tree_array({
parent = ts_tree_make_node(&pool, symbol3, tree_array({
tree1,
tree2,
}), 0, &language);
@ -138,7 +138,7 @@ describe("Tree", []() {
ts_tree_retain(tree1);
ts_tree_retain(tree2);
parent = ts_tree_make_node(&pool, symbol3, 2, tree_array({
parent = ts_tree_make_node(&pool, symbol3, tree_array({
tree1,
tree2,
}), 0, &language);
@ -162,7 +162,7 @@ describe("Tree", []() {
ts_tree_retain(tree1);
ts_tree_retain(tree2);
parent = ts_tree_make_node(&pool, symbol3, 2, tree_array({
parent = ts_tree_make_node(&pool, symbol3, tree_array({
tree1,
tree2,
}), 0, &language);
@ -183,7 +183,7 @@ describe("Tree", []() {
Tree *tree = nullptr;
before_each([&]() {
tree = ts_tree_make_node(&pool, symbol1, 3, tree_array({
tree = ts_tree_make_node(&pool, symbol1, tree_array({
ts_tree_make_leaf(&pool, symbol2, {2, {0, 2}}, {3, {0, 3}}, &language),
ts_tree_make_leaf(&pool, symbol3, {2, {0, 2}}, {3, {0, 3}}, &language),
ts_tree_make_leaf(&pool, symbol4, {2, {0, 2}}, {3, {0, 3}}, &language),
@ -213,13 +213,13 @@ describe("Tree", []() {
AssertThat(tree->padding, Equals<Length>({3, {0, 3}}));
AssertThat(tree->size, Equals<Length>({13, {0, 13}}));
AssertThat(tree->children[0]->has_changes, IsTrue());
AssertThat(tree->children[0]->padding, Equals<Length>({3, {0, 3}}));
AssertThat(tree->children[0]->size, Equals<Length>({3, {0, 3}}));
AssertThat(tree->children.contents[0]->has_changes, IsTrue());
AssertThat(tree->children.contents[0]->padding, Equals<Length>({3, {0, 3}}));
AssertThat(tree->children.contents[0]->size, Equals<Length>({3, {0, 3}}));
AssertThat(tree->children[1]->has_changes, IsFalse());
AssertThat(tree->children[1]->padding, Equals<Length>({2, {0, 2}}));
AssertThat(tree->children[1]->size, Equals<Length>({3, {0, 3}}));
AssertThat(tree->children.contents[1]->has_changes, IsFalse());
AssertThat(tree->children.contents[1]->padding, Equals<Length>({2, {0, 2}}));
AssertThat(tree->children.contents[1]->size, Equals<Length>({3, {0, 3}}));
});
});
@ -239,9 +239,9 @@ describe("Tree", []() {
AssertThat(tree->padding, Equals<Length>({5, {0, 5}}));
AssertThat(tree->size, Equals<Length>({11, {0, 11}}));
AssertThat(tree->children[0]->has_changes, IsTrue());
AssertThat(tree->children[0]->padding, Equals<Length>({5, {0, 5}}));
AssertThat(tree->children[0]->size, Equals<Length>({1, {0, 1}}));
AssertThat(tree->children.contents[0]->has_changes, IsTrue());
AssertThat(tree->children.contents[0]->padding, Equals<Length>({5, {0, 5}}));
AssertThat(tree->children.contents[0]->size, Equals<Length>({1, {0, 1}}));
});
});
@ -263,11 +263,11 @@ describe("Tree", []() {
AssertThat(tree->padding, Equals<Length>({4, {0, 4}}));
AssertThat(tree->size, Equals<Length>({13, {0, 13}}));
AssertThat(tree->children[0]->has_changes, IsTrue());
AssertThat(tree->children[0]->padding, Equals<Length>({4, {0, 4}}));
AssertThat(tree->children[0]->size, Equals<Length>({3, {0, 3}}));
AssertThat(tree->children.contents[0]->has_changes, IsTrue());
AssertThat(tree->children.contents[0]->padding, Equals<Length>({4, {0, 4}}));
AssertThat(tree->children.contents[0]->size, Equals<Length>({3, {0, 3}}));
AssertThat(tree->children[1]->has_changes, IsFalse());
AssertThat(tree->children.contents[1]->has_changes, IsFalse());
});
});
@ -287,11 +287,11 @@ describe("Tree", []() {
AssertThat(tree->padding, Equals<Length>({2, {0, 2}}));
AssertThat(tree->size, Equals<Length>({16, {0, 16}}));
AssertThat(tree->children[0]->has_changes, IsTrue());
AssertThat(tree->children[0]->padding, Equals<Length>({2, {0, 2}}));
AssertThat(tree->children[0]->size, Equals<Length>({6, {0, 6}}));
AssertThat(tree->children.contents[0]->has_changes, IsTrue());
AssertThat(tree->children.contents[0]->padding, Equals<Length>({2, {0, 2}}));
AssertThat(tree->children.contents[0]->size, Equals<Length>({6, {0, 6}}));
AssertThat(tree->children[1]->has_changes, IsFalse());
AssertThat(tree->children.contents[1]->has_changes, IsFalse());
});
});
@ -313,23 +313,23 @@ describe("Tree", []() {
AssertThat(tree->padding, Equals<Length>({4, {0, 4}}));
AssertThat(tree->size, Equals<Length>({4, {0, 4}}));
AssertThat(tree->children[0]->has_changes, IsTrue());
AssertThat(tree->children[0]->padding, Equals<Length>({4, {0, 4}}));
AssertThat(tree->children[0]->size, Equals<Length>({0, {0, 0}}));
AssertThat(tree->children.contents[0]->has_changes, IsTrue());
AssertThat(tree->children.contents[0]->padding, Equals<Length>({4, {0, 4}}));
AssertThat(tree->children.contents[0]->size, Equals<Length>({0, {0, 0}}));
AssertThat(tree->children[1]->has_changes, IsTrue());
AssertThat(tree->children[1]->padding, Equals<Length>({0, {0, 0}}));
AssertThat(tree->children[1]->size, Equals<Length>({0, {0, 0}}));
AssertThat(tree->children.contents[1]->has_changes, IsTrue());
AssertThat(tree->children.contents[1]->padding, Equals<Length>({0, {0, 0}}));
AssertThat(tree->children.contents[1]->size, Equals<Length>({0, {0, 0}}));
AssertThat(tree->children[2]->has_changes, IsTrue());
AssertThat(tree->children[2]->padding, Equals<Length>({1, {0, 1}}));
AssertThat(tree->children[2]->size, Equals<Length>({3, {0, 3}}));
AssertThat(tree->children.contents[2]->has_changes, IsTrue());
AssertThat(tree->children.contents[2]->padding, Equals<Length>({1, {0, 1}}));
AssertThat(tree->children.contents[2]->size, Equals<Length>({3, {0, 3}}));
});
});
describe("edits within a tree's range of scanned bytes", [&]() {
it("marks preceding trees as changed", [&]() {
tree->children[0]->bytes_scanned = 7;
tree->children.contents[0]->bytes_scanned = 7;
TSInputEdit edit;
edit.start_byte = 6;
@ -341,7 +341,7 @@ describe("Tree", []() {
ts_tree_edit(tree, &edit);
assert_consistent(tree);
AssertThat(tree->children[0]->has_changes, IsTrue());
AssertThat(tree->children.contents[0]->has_changes, IsTrue());
});
});
});
@ -361,14 +361,14 @@ describe("Tree", []() {
Tree *leaf_copy = ts_tree_make_leaf(&pool, symbol1, {2, {1, 1}}, {5, {1, 4}}, &language);
AssertThat(ts_tree_eq(leaf, leaf_copy), IsTrue());
Tree *parent = ts_tree_make_node(&pool, symbol2, 2, tree_array({
Tree *parent = ts_tree_make_node(&pool, symbol2, tree_array({
leaf,
leaf_copy,
}), 0, &language);
ts_tree_retain(leaf);
ts_tree_retain(leaf_copy);
Tree *parent_copy = ts_tree_make_node(&pool, symbol2, 2, tree_array({
Tree *parent_copy = ts_tree_make_node(&pool, symbol2, tree_array({
leaf,
leaf_copy,
}), 0, &language);
@ -415,14 +415,14 @@ describe("Tree", []() {
it("returns false for trees with different children", [&]() {
Tree *leaf2 = ts_tree_make_leaf(&pool, symbol2, {1, {0, 1}}, {3, {0, 3}}, &language);
Tree *parent = ts_tree_make_node(&pool, symbol2, 2, tree_array({
Tree *parent = ts_tree_make_node(&pool, symbol2, tree_array({
leaf,
leaf2,
}), 0, &language);
ts_tree_retain(leaf);
ts_tree_retain(leaf2);
Tree *different_parent = ts_tree_make_node(&pool, symbol2, 2, tree_array({
Tree *different_parent = ts_tree_make_node(&pool, symbol2, tree_array({
leaf2,
leaf,
}), 0, &language);
@ -450,14 +450,14 @@ describe("Tree", []() {
it("returns the last serialized external token state in the given tree", [&]() {
Tree *tree1, *tree2, *tree3, *tree4, *tree5, *tree6, *tree7, *tree8, *tree9;
tree1 = ts_tree_make_node(&pool, symbol1, 2, tree_array({
(tree2 = ts_tree_make_node(&pool, symbol2, 3, tree_array({
tree1 = ts_tree_make_node(&pool, symbol1, tree_array({
(tree2 = ts_tree_make_node(&pool, symbol2, tree_array({
(tree3 = make_external(ts_tree_make_leaf(&pool, symbol3, padding, size, &language))),
(tree4 = ts_tree_make_leaf(&pool, symbol4, padding, size, &language)),
(tree5 = ts_tree_make_leaf(&pool, symbol5, padding, size, &language)),
}), 0, &language)),
(tree6 = ts_tree_make_node(&pool, symbol6, 2, tree_array({
(tree7 = ts_tree_make_node(&pool, symbol7, 1, tree_array({
(tree6 = ts_tree_make_node(&pool, symbol6, tree_array({
(tree7 = ts_tree_make_node(&pool, symbol7, tree_array({
(tree8 = ts_tree_make_leaf(&pool, symbol8, padding, size, &language)),
}), 0, &language)),
(tree9 = ts_tree_make_leaf(&pool, symbol9, padding, size, &language)),