tree-sitter/src/runtime/stack.c
Max Brunsfeld e59558c83b Allow stack versions to be temporarily paused
This way, when detecting an error, we can defer the decision about
whether to bail or recover until all stack versions are processed.
2018-04-06 13:26:18 -07:00

778 lines
24 KiB
C

#include "runtime/alloc.h"
#include "runtime/tree.h"
#include "runtime/array.h"
#include "runtime/stack.h"
#include "runtime/length.h"
#include <assert.h>
#include <stdio.h>
#define MAX_LINK_COUNT 8
#define MAX_NODE_POOL_SIZE 50
#define MAX_ITERATOR_COUNT 64
#ifdef _WIN32
#define inline __forceinline
#else
#define inline static inline __attribute__((always_inline))
#endif
typedef struct StackNode StackNode;
typedef struct {
StackNode *node;
Tree *tree;
bool is_pending;
} StackLink;
struct StackNode {
TSStateId state;
Length position;
StackLink links[MAX_LINK_COUNT];
short unsigned int link_count;
uint32_t ref_count;
unsigned error_cost;
unsigned depth;
int dynamic_precedence;
};
typedef struct {
StackNode *node;
TreeArray trees;
uint32_t tree_count;
bool is_pending;
} Iterator;
typedef struct {
void *payload;
StackIterateCallback callback;
} StackIterateSession;
typedef Array(StackNode *) StackNodeArray;
typedef enum {
StackStatusActive,
StackStatusPaused,
StackStatusHalted,
} StackStatus;
typedef struct {
StackNode *node;
Tree *last_external_token;
StackSummary *summary;
uint32_t push_count;
TSSymbol lookahead_when_paused;
StackStatus status;
} StackHead;
struct Stack {
Array(StackHead) heads;
StackSliceArray slices;
Array(Iterator) iterators;
StackNodeArray node_pool;
StackNode *base_node;
TreePool *tree_pool;
};
typedef unsigned StackAction;
enum {
StackActionNone,
StackActionStop = 1,
StackActionPop = 2,
};
typedef StackAction (*StackCallback)(void *, const Iterator *);
static void stack_node_retain(StackNode *self) {
if (!self)
return;
assert(self->ref_count > 0);
self->ref_count++;
assert(self->ref_count != 0);
}
static void stack_node_release(StackNode *self, StackNodeArray *pool, TreePool *tree_pool) {
recur:
assert(self->ref_count != 0);
self->ref_count--;
if (self->ref_count > 0) return;
StackNode *first_predecessor = NULL;
if (self->link_count > 0) {
for (unsigned i = self->link_count - 1; i > 0; i--) {
if (self->links[i].tree) ts_tree_release(tree_pool, self->links[i].tree);
stack_node_release(self->links[i].node, pool, tree_pool);
}
if (self->links[0].tree) ts_tree_release(tree_pool, self->links[0].tree);
first_predecessor = self->links[0].node;
}
if (pool->size < MAX_NODE_POOL_SIZE) {
array_push(pool, self);
} else {
ts_free(self);
}
if (first_predecessor) {
self = first_predecessor;
goto recur;
}
}
static StackNode *stack_node_new(StackNode *previous_node, Tree *tree, bool is_pending,
TSStateId state, StackNodeArray *pool) {
StackNode *node = pool->size > 0 ?
array_pop(pool) :
ts_malloc(sizeof(StackNode));
*node = (StackNode){.ref_count = 1, .link_count = 0, .state = state, .depth = 0};
if (previous_node) {
stack_node_retain(previous_node);
node->link_count = 1;
node->links[0] = (StackLink){
.node = previous_node,
.tree = tree,
.is_pending = is_pending,
};
node->position = previous_node->position;
node->error_cost = previous_node->error_cost;
node->dynamic_precedence = previous_node->dynamic_precedence;
if (tree) {
node->depth = previous_node->depth;
if (!tree->extra) node->depth++;
node->error_cost += tree->error_cost;
node->position = length_add(node->position, ts_tree_total_size(tree));
node->dynamic_precedence += tree->dynamic_precedence;
if (state == ERROR_STATE && !tree->extra) {
node->error_cost +=
ERROR_COST_PER_SKIPPED_TREE * ((tree->visible || tree->child_count == 0) ? 1 : tree->visible_child_count) +
ERROR_COST_PER_SKIPPED_CHAR * tree->size.bytes +
ERROR_COST_PER_SKIPPED_LINE * tree->size.extent.row;
if (previous_node->links[0].tree) {
node->error_cost +=
ERROR_COST_PER_SKIPPED_CHAR * tree->padding.bytes +
ERROR_COST_PER_SKIPPED_LINE * tree->padding.extent.row;
}
}
}
} else {
node->position = length_zero();
node->error_cost = 0;
}
return node;
}
static bool stack__tree_is_equivalent(const Tree *left, const Tree *right) {
return
left == right ||
(left &&
right &&
left->symbol == right->symbol &&
((left->error_cost > 0 && right->error_cost > 0) ||
(left->child_count == 0 && right->child_count == 0 &&
left->padding.bytes == right->padding.bytes &&
left->size.bytes == right->size.bytes &&
left->extra == right->extra &&
ts_tree_external_token_state_eq(left, right))));
}
static void stack_node_add_link(StackNode *self, StackLink link) {
if (link.node == self) return;
for (int i = 0; i < self->link_count; i++) {
StackLink existing_link = self->links[i];
if (stack__tree_is_equivalent(existing_link.tree, link.tree)) {
if (existing_link.node == link.node) return;
if (existing_link.node->state == link.node->state &&
existing_link.node->position.bytes == link.node->position.bytes) {
for (int j = 0; j < link.node->link_count; j++) {
stack_node_add_link(existing_link.node, link.node->links[j]);
}
return;
}
}
}
if (self->link_count < MAX_LINK_COUNT) {
stack_node_retain(link.node);
if (link.tree) ts_tree_retain(link.tree);
self->links[self->link_count++] = link;
}
}
static void stack_head_delete(StackHead *self, StackNodeArray *pool, TreePool *tree_pool) {
if (self->node) {
if (self->last_external_token) {
ts_tree_release(tree_pool, self->last_external_token);
}
if (self->summary) {
array_delete(self->summary);
ts_free(self->summary);
}
stack_node_release(self->node, pool, tree_pool);
}
}
static StackVersion ts_stack__add_version(Stack *self, StackVersion original_version,
StackNode *node, Tree *last_external_token) {
StackHead head = {
.node = node,
.push_count = self->heads.contents[original_version].push_count,
.last_external_token = last_external_token,
.status = StackStatusActive,
.lookahead_when_paused = 0,
};
array_push(&self->heads, head);
stack_node_retain(node);
if (last_external_token) ts_tree_retain(last_external_token);
return (StackVersion)(self->heads.size - 1);
}
static void ts_stack__add_slice(Stack *self, StackVersion original_version, StackNode *node,
TreeArray *trees, Tree *last_external_token) {
for (uint32_t i = self->slices.size - 1; i + 1 > 0; i--) {
StackVersion version = self->slices.contents[i].version;
if (self->heads.contents[version].node == node) {
StackSlice slice = {*trees, version};
array_insert(&self->slices, i + 1, slice);
return;
}
}
StackVersion version = ts_stack__add_version(self, original_version, node, last_external_token);
StackSlice slice = { *trees, version };
array_push(&self->slices, slice);
}
inline StackSliceArray stack__iter(Stack *self, StackVersion version,
StackCallback callback, void *payload,
bool include_trees) {
array_clear(&self->slices);
array_clear(&self->iterators);
StackHead *head = array_get(&self->heads, version);
Tree *last_external_token = head->last_external_token;
Iterator iterator = {
.node = head->node,
.trees = array_new(),
.tree_count = 0,
.is_pending = true,
};
array_push(&self->iterators, iterator);
while (self->iterators.size > 0) {
for (uint32_t i = 0, size = self->iterators.size; i < size; i++) {
Iterator *iterator = &self->iterators.contents[i];
StackNode *node = iterator->node;
StackAction action = callback(payload, iterator);
bool should_pop = action & StackActionPop;
bool should_stop = action & StackActionStop || node->link_count == 0;
if (should_pop) {
TreeArray trees = iterator->trees;
if (!should_stop)
ts_tree_array_copy(trees, &trees);
ts_tree_array_reverse(&trees);
ts_stack__add_slice(
self,
version,
node,
&trees,
last_external_token
);
}
if (should_stop) {
if (!should_pop)
ts_tree_array_delete(self->tree_pool, &iterator->trees);
array_erase(&self->iterators, i);
i--, size--;
continue;
}
for (uint32_t j = 1; j <= node->link_count; j++) {
Iterator *next_iterator;
StackLink link;
if (j == node->link_count) {
link = node->links[0];
next_iterator = &self->iterators.contents[i];
} else {
if (self->iterators.size >= MAX_ITERATOR_COUNT) continue;
link = node->links[j];
Iterator current_iterator = self->iterators.contents[i];
array_push(&self->iterators, current_iterator);
next_iterator = array_back(&self->iterators);
ts_tree_array_copy(next_iterator->trees, &next_iterator->trees);
}
next_iterator->node = link.node;
if (link.tree) {
if (include_trees) {
array_push(&next_iterator->trees, link.tree);
ts_tree_retain(link.tree);
}
if (!link.tree->extra) {
next_iterator->tree_count++;
if (!link.is_pending) {
next_iterator->is_pending = false;
}
}
} else {
next_iterator->tree_count++;
next_iterator->is_pending = false;
}
}
}
}
return self->slices;
}
Stack *ts_stack_new(TreePool *tree_pool) {
Stack *self = ts_calloc(1, sizeof(Stack));
array_init(&self->heads);
array_init(&self->slices);
array_init(&self->iterators);
array_init(&self->node_pool);
array_grow(&self->heads, 4);
array_grow(&self->slices, 4);
array_grow(&self->iterators, 4);
array_grow(&self->node_pool, MAX_NODE_POOL_SIZE);
self->tree_pool = tree_pool;
self->base_node = stack_node_new(NULL, NULL, false, 1, &self->node_pool);
ts_stack_clear(self);
return self;
}
void ts_stack_delete(Stack *self) {
if (self->slices.contents)
array_delete(&self->slices);
if (self->iterators.contents)
array_delete(&self->iterators);
stack_node_release(self->base_node, &self->node_pool, self->tree_pool);
for (uint32_t i = 0; i < self->heads.size; i++) {
stack_head_delete(&self->heads.contents[i], &self->node_pool, self->tree_pool);
}
array_clear(&self->heads);
if (self->node_pool.contents) {
for (uint32_t i = 0; i < self->node_pool.size; i++)
ts_free(self->node_pool.contents[i]);
array_delete(&self->node_pool);
}
array_delete(&self->heads);
ts_free(self);
}
uint32_t ts_stack_version_count(const Stack *self) {
return self->heads.size;
}
TSStateId ts_stack_state(const Stack *self, StackVersion version) {
return array_get(&self->heads, version)->node->state;
}
Length ts_stack_position(const Stack *self, StackVersion version) {
return array_get(&self->heads, version)->node->position;
}
unsigned ts_stack_push_count(const Stack *self, StackVersion version) {
return array_get(&self->heads, version)->push_count;
}
void ts_stack_decrease_push_count(Stack *self, StackVersion version, unsigned decrement) {
array_get(&self->heads, version)->push_count -= decrement;
}
Tree *ts_stack_last_external_token(const Stack *self, StackVersion version) {
return array_get(&self->heads, version)->last_external_token;
}
void ts_stack_set_last_external_token(Stack *self, StackVersion version, Tree *token) {
StackHead *head = array_get(&self->heads, version);
if (token) ts_tree_retain(token);
if (head->last_external_token) ts_tree_release(self->tree_pool, head->last_external_token);
head->last_external_token = token;
}
unsigned ts_stack_error_cost(const Stack *self, StackVersion version) {
StackHead *head = array_get(&self->heads, version);
return head->node->error_cost;
}
void ts_stack_push(Stack *self, StackVersion version, Tree *tree, bool pending, TSStateId state) {
StackHead *head = array_get(&self->heads, version);
StackNode *new_node = stack_node_new(head->node, tree, pending, state, &self->node_pool);
if (state == ERROR_STATE) {
head->push_count = 0;
} else if (!tree->extra) {
head->push_count++;
}
stack_node_release(head->node, &self->node_pool, self->tree_pool);
head->node = new_node;
}
inline StackAction iterate_callback(void *payload, const Iterator *iterator) {
StackIterateSession *session = payload;
session->callback(
session->payload,
iterator->node->state,
iterator->tree_count
);
return StackActionNone;
}
void ts_stack_iterate(Stack *self, StackVersion version,
StackIterateCallback callback, void *payload) {
StackIterateSession session = {payload, callback};
stack__iter(self, version, iterate_callback, &session, true);
}
inline StackAction pop_count_callback(void *payload, const Iterator *iterator) {
unsigned *goal_tree_count = payload;
if (iterator->tree_count == *goal_tree_count) {
return StackActionPop | StackActionStop;
} else {
return StackActionNone;
}
}
StackSliceArray ts_stack_pop_count(Stack *self, StackVersion version, uint32_t count) {
return stack__iter(self, version, pop_count_callback, &count, true);
}
inline StackAction pop_pending_callback(void *payload, const Iterator *iterator) {
if (iterator->tree_count >= 1) {
if (iterator->is_pending) {
return StackActionPop | StackActionStop;
} else {
return StackActionStop;
}
} else {
return StackActionNone;
}
}
StackSliceArray ts_stack_pop_pending(Stack *self, StackVersion version) {
StackSliceArray pop = stack__iter(self, version, pop_pending_callback, NULL, true);
if (pop.size > 0) {
ts_stack_renumber_version(self, pop.contents[0].version, version);
pop.contents[0].version = version;
}
return pop;
}
inline StackAction pop_error_callback(void *payload, const Iterator *iterator) {
if (iterator->trees.size > 0) {
bool *found_error = payload;
if (!*found_error && iterator->trees.contents[0]->symbol == ts_builtin_sym_error) {
*found_error = true;
return StackActionPop | StackActionStop;
} else {
return StackActionStop;
}
} else {
return StackActionNone;
}
}
StackSliceArray ts_stack_pop_error(Stack *self, StackVersion version) {
StackNode *node = array_get(&self->heads, version)->node;
for (unsigned i = 0; i < node->link_count; i++) {
if (node->links[i].tree && node->links[i].tree->symbol == ts_builtin_sym_error) {
bool found_error = false;
return stack__iter(self, version, pop_error_callback, &found_error, true);
}
}
return (StackSliceArray){.size = 0};
}
inline StackAction pop_all_callback(void *payload, const Iterator *iterator) {
return iterator->node->link_count == 0 ? StackActionPop : StackActionNone;
}
StackSliceArray ts_stack_pop_all(Stack *self, StackVersion version) {
return stack__iter(self, version, pop_all_callback, NULL, true);
}
typedef struct {
StackSummary *summary;
unsigned max_depth;
} SummarizeStackSession;
inline StackAction summarize_stack_callback(void *payload, const Iterator *iterator) {
SummarizeStackSession *session = payload;
TSStateId state = iterator->node->state;
unsigned depth = iterator->tree_count;
if (depth > session->max_depth) return StackActionStop;
for (unsigned i = session->summary->size - 1; i + 1 > 0; i--) {
StackSummaryEntry entry = session->summary->contents[i];
if (entry.depth < depth) break;
if (entry.depth == depth && entry.state == state) return StackActionNone;
}
array_push(session->summary, ((StackSummaryEntry){
.position = iterator->node->position,
.depth = depth,
.state = state,
}));
return StackActionNone;
}
void ts_stack_record_summary(Stack *self, StackVersion version, unsigned max_depth) {
SummarizeStackSession session = {
.summary = ts_malloc(sizeof(StackSummary)),
.max_depth = max_depth
};
array_init(session.summary);
stack__iter(self, version, summarize_stack_callback, &session, false);
self->heads.contents[version].summary = session.summary;
}
StackSummary *ts_stack_get_summary(Stack *self, StackVersion version) {
return array_get(&self->heads, version)->summary;
}
unsigned ts_stack_depth_since_error(Stack *self, StackVersion version) {
return array_get(&self->heads, version)->node->depth;
}
int ts_stack_dynamic_precedence(Stack *self, StackVersion version) {
return array_get(&self->heads, version)->node->dynamic_precedence;
}
void ts_stack_remove_version(Stack *self, StackVersion version) {
stack_head_delete(array_get(&self->heads, version), &self->node_pool, self->tree_pool);
array_erase(&self->heads, version);
}
void ts_stack_renumber_version(Stack *self, StackVersion v1, StackVersion v2) {
assert(v2 < v1);
assert((uint32_t)v1 < self->heads.size);
stack_head_delete(&self->heads.contents[v2], &self->node_pool, self->tree_pool);
self->heads.contents[v2] = self->heads.contents[v1];
array_erase(&self->heads, v1);
}
void ts_stack_swap_versions(Stack *self, StackVersion v1, StackVersion v2) {
StackHead temporary_head = self->heads.contents[v1];
self->heads.contents[v1] = self->heads.contents[v2];
self->heads.contents[v2] = temporary_head;
}
StackVersion ts_stack_copy_version(Stack *self, StackVersion version) {
assert(version < self->heads.size);
array_push(&self->heads, self->heads.contents[version]);
StackHead *head = array_back(&self->heads);
stack_node_retain(head->node);
if (head->last_external_token) ts_tree_retain(head->last_external_token);
head->summary = NULL;
return self->heads.size - 1;
}
bool ts_stack_merge(Stack *self, StackVersion version1, StackVersion version2) {
if (ts_stack_can_merge(self, version1, version2)) {
ts_stack_force_merge(self, version1, version2);
return true;
} else {
return false;
}
}
bool ts_stack_can_merge(Stack *self, StackVersion version1, StackVersion version2) {
StackHead *head1 = &self->heads.contents[version1];
StackHead *head2 = &self->heads.contents[version2];
return
head1->status == StackStatusActive &&
head2->status == StackStatusActive &&
head1->node->state == head2->node->state &&
head1->node->position.bytes == head2->node->position.bytes &&
head1->node->depth == head2->node->depth &&
ts_tree_external_token_state_eq(head1->last_external_token, head2->last_external_token);
}
void ts_stack_force_merge(Stack *self, StackVersion version1, StackVersion version2) {
StackHead *head1 = &self->heads.contents[version1];
StackHead *head2 = &self->heads.contents[version2];
for (uint32_t i = 0; i < head2->node->link_count; i++) {
stack_node_add_link(head1->node, head2->node->links[i]);
}
ts_stack_remove_version(self, version2);
}
void ts_stack_halt(Stack *self, StackVersion version) {
array_get(&self->heads, version)->status = StackStatusHalted;
}
void ts_stack_pause(Stack *self, StackVersion version, TSSymbol lookahead) {
StackHead *head = array_get(&self->heads, version);
head->status = StackStatusPaused;
head->lookahead_when_paused = lookahead;
}
bool ts_stack_is_active(const Stack *self, StackVersion version) {
return array_get(&self->heads, version)->status == StackStatusActive;
}
bool ts_stack_is_halted(const Stack *self, StackVersion version) {
return array_get(&self->heads, version)->status == StackStatusHalted;
}
bool ts_stack_is_paused(const Stack *self, StackVersion version) {
return array_get(&self->heads, version)->status == StackStatusPaused;
}
TSSymbol ts_stack_resume(Stack *self, StackVersion version) {
StackHead *head = array_get(&self->heads, version);
assert(head->status == StackStatusPaused);
TSSymbol result = head->lookahead_when_paused;
head->status = StackStatusActive;
head->lookahead_when_paused = 0;
return result;
}
void ts_stack_clear(Stack *self) {
stack_node_retain(self->base_node);
for (uint32_t i = 0; i < self->heads.size; i++) {
stack_head_delete(&self->heads.contents[i], &self->node_pool, self->tree_pool);
}
array_clear(&self->heads);
array_push(&self->heads, ((StackHead){
.node = self->base_node,
.last_external_token = NULL,
.status = StackStatusActive,
.lookahead_when_paused = 0,
}));
}
bool ts_stack_print_dot_graph(Stack *self, const char **symbol_names, FILE *f) {
bool was_recording_allocations = ts_toggle_allocation_recording(false);
if (!f)
f = stderr;
fprintf(f, "digraph stack {\n");
fprintf(f, "rankdir=\"RL\";\n");
fprintf(f, "edge [arrowhead=none]\n");
Array(StackNode *) visited_nodes = array_new();
array_clear(&self->iterators);
for (uint32_t i = 0; i < self->heads.size; i++) {
StackHead *head = &self->heads.contents[i];
if (head->status == StackStatusHalted) continue;
fprintf(f, "node_head_%u [shape=none, label=\"\"]\n", i);
fprintf(f, "node_head_%u -> node_%p [", i, head->node);
if (head->status == StackStatusPaused) {
fprintf(f, "color=red ");
}
fprintf(f,
"label=%u, fontcolor=blue, weight=10000, labeltooltip=\"push_count: %u\ndepth: %u",
i, head->push_count, head->node->depth
);
if (head->last_external_token) {
TSExternalTokenState *state = &head->last_external_token->external_token_state;
const char *data = ts_external_token_state_data(state);
fprintf(f, "\nexternal_token_state:");
for (uint32_t j = 0; j < state->length; j++) fprintf(f, " %2X", data[j]);
}
fprintf(f, "\"]\n");
array_push(&self->iterators, ((Iterator){.node = head->node }));
}
bool all_iterators_done = false;
while (!all_iterators_done) {
all_iterators_done = true;
for (uint32_t i = 0; i < self->iterators.size; i++) {
Iterator iterator = self->iterators.contents[i];
StackNode *node = iterator.node;
for (uint32_t j = 0; j < visited_nodes.size; j++) {
if (visited_nodes.contents[j] == node) {
node = NULL;
break;
}
}
if (!node) continue;
all_iterators_done = false;
fprintf(f, "node_%p [", node);
if (node->state == ERROR_STATE)
fprintf(f, "label=\"?\"");
else if (node->link_count == 1 && node->links[0].tree &&
node->links[0].tree->extra)
fprintf(f, "shape=point margin=0 label=\"\"");
else
fprintf(f, "label=\"%d\"", node->state);
fprintf(
f,
" tooltip=\"position: %u,%u\nerror_cost: %u\ndynamic_precedence: %d\"];\n",
node->position.extent.row,
node->position.extent.column,
node->error_cost,
node->dynamic_precedence
);
for (int j = 0; j < node->link_count; j++) {
StackLink link = node->links[j];
fprintf(f, "node_%p -> node_%p [", node, link.node);
if (link.is_pending)
fprintf(f, "style=dashed ");
if (link.tree && link.tree->extra)
fprintf(f, "fontcolor=gray ");
if (!link.tree) {
fprintf(f, "color=red");
} else if (link.tree->symbol == ts_builtin_sym_error) {
fprintf(f, "label=\"ERROR\"");
} else {
fprintf(f, "label=\"");
if (!link.tree->named)
fprintf(f, "'");
const char *name = symbol_names[link.tree->symbol];
for (const char *c = name; *c; c++) {
if (*c == '\"' || *c == '\\')
fprintf(f, "\\");
fprintf(f, "%c", *c);
}
if (!link.tree->named)
fprintf(f, "'");
fprintf(f, "\" labeltooltip=\"error_cost: %u\ndynamic_precedence: %u\"",
link.tree->error_cost,
link.tree->dynamic_precedence);
}
fprintf(f, "];\n");
Iterator *next_iterator;
if (j == 0) {
next_iterator = &self->iterators.contents[i];
} else {
array_push(&self->iterators, iterator);
next_iterator = array_back(&self->iterators);
}
next_iterator->node = link.node;
}
array_push(&visited_nodes, node);
}
}
fprintf(f, "}\n");
array_delete(&visited_nodes);
ts_toggle_allocation_recording(was_recording_allocations);
return true;
}