traxys/tree-sitter
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Merge branch 'master' into node-fields

Browse source
This commit is contained in:
Max Brunsfeld 2019-03-26 11:58:21 -07:00
commit 5035e194ff
34 changed files with 1178 additions and 240 deletions
Download patch file Download diff file Expand all files Collapse all files

95
lib/src/parser.c
View file

@ -1,19 +1,22 @@
#include <time.h>
#include <assert.h>
#include <stdio.h>
#include <limits.h>
#include <stdbool.h>
#include "tree_sitter/api.h"
#include "./subtree.h"
#include "./lexer.h"
#include "./length.h"
#include "./array.h"
#include "./language.h"
#include "./alloc.h"
#include "./stack.h"
#include "./reusable_node.h"
#include "./reduce_action.h"
#include "./array.h"
#include "./atomic.h"
#include "./clock.h"
#include "./error_costs.h"
#include "./get_changed_ranges.h"
#include "./language.h"
#include "./length.h"
#include "./lexer.h"
#include "./reduce_action.h"
#include "./reusable_node.h"
#include "./stack.h"
#include "./subtree.h"
#include "./tree.h"
#define LOG(...) \
@ -42,6 +45,7 @@ static const unsigned MAX_VERSION_COUNT = 6;
static const unsigned MAX_VERSION_COUNT_OVERFLOW = 4;
static const unsigned MAX_SUMMARY_DEPTH = 16;
static const unsigned MAX_COST_DIFFERENCE = 16 * ERROR_COST_PER_SKIPPED_TREE;
static const unsigned OP_COUNT_PER_TIMEOUT_CHECK = 100;
typedef struct {
Subtree token;
@ -62,9 +66,11 @@ struct TSParser {
ReusableNode reusable_node;
void *external_scanner_payload;
FILE *dot_graph_file;
TSClock end_clock;
TSDuration timeout_duration;
unsigned accept_count;
size_t operation_limit;
volatile bool enabled;
unsigned operation_count;
const volatile size_t *cancellation_flag;
bool halt_on_error;
Subtree old_tree;
TSRangeArray included_range_differences;
@ -327,7 +333,7 @@ static Subtree ts_parser__lex(TSParser *self, StackVersion version, TSStateId pa
LOG(
"lex_external state:%d, row:%u, column:%u",
lex_mode.external_lex_state,
current_position.extent.row,
current_position.extent.row + 1,
current_position.extent.column
);
ts_lexer_start(&self->lexer);
@ -365,7 +371,7 @@ static Subtree ts_parser__lex(TSParser *self, StackVersion version, TSStateId pa
LOG(
"lex_internal state:%d, row:%u, column:%u",
lex_mode.lex_state,
current_position.extent.row,
current_position.extent.row + 1,
current_position.extent.column
);
ts_lexer_start(&self->lexer);
@ -1242,7 +1248,11 @@ static void ts_parser__recover(TSParser *self, StackVersion version, Subtree loo
}
}
static void ts_parser__advance(TSParser *self, StackVersion version, bool allow_node_reuse) {
static bool ts_parser__advance(
TSParser *self,
StackVersion version,
bool allow_node_reuse
) {
TSStateId state = ts_stack_state(self->stack, version);
uint32_t position = ts_stack_position(self->stack, version).bytes;
Subtree last_external_token = ts_stack_last_external_token(self->stack, version);
@ -1274,6 +1284,17 @@ static void ts_parser__advance(TSParser *self, StackVersion version, bool allow_
}
for (;;) {
if (++self->operation_count == OP_COUNT_PER_TIMEOUT_CHECK) {
self->operation_count = 0;
if (
(self->cancellation_flag && atomic_load(self->cancellation_flag)) ||
(!clock_is_null(self->end_clock) && clock_is_gt(clock_now(), self->end_clock))
) {
ts_subtree_release(&self->tree_pool, lookahead);
return false;
}
}
StackVersion last_reduction_version = STACK_VERSION_NONE;
for (uint32_t i = 0; i < table_entry.action_count; i++) {
@ -1302,7 +1323,7 @@ static void ts_parser__advance(TSParser *self, StackVersion version, bool allow_
ts_parser__shift(self, version, next_state, lookahead, action.params.extra);
if (did_reuse) reusable_node_advance(&self->reusable_node);
return;
return true;
}
case TSParseActionTypeReduce: {
@ -1322,7 +1343,7 @@ static void ts_parser__advance(TSParser *self, StackVersion version, bool allow_
case TSParseActionTypeAccept: {
LOG("accept");
ts_parser__accept(self, version, lookahead);
return;
return true;
}
case TSParseActionTypeRecover: {
@ -1332,7 +1353,7 @@ static void ts_parser__advance(TSParser *self, StackVersion version, bool allow_
ts_parser__recover(self, version, lookahead);
if (did_reuse) reusable_node_advance(&self->reusable_node);
return;
return true;
}
}
}
@ -1371,7 +1392,7 @@ static void ts_parser__advance(TSParser *self, StackVersion version, bool allow_
if (state == ERROR_STATE) {
ts_parser__recover(self, version, lookahead);
return;
return true;
}
if (ts_parser__breakdown_top_of_stack(self, version)) {
@ -1381,7 +1402,7 @@ static void ts_parser__advance(TSParser *self, StackVersion version, bool allow_
LOG("detect_error");
ts_stack_pause(self->stack, version, ts_subtree_leaf_symbol(lookahead));
ts_subtree_release(&self->tree_pool, lookahead);
return;
return true;
}
}
@ -1480,7 +1501,7 @@ static bool ts_parser_has_outstanding_parse(TSParser *self) {
// Parser - Public
TSParser *ts_parser_new() {
TSParser *ts_parser_new(void) {
TSParser *self = ts_calloc(1, sizeof(TSParser));
ts_lexer_init(&self->lexer);
array_init(&self->reduce_actions);
@ -1491,8 +1512,10 @@ TSParser *ts_parser_new() {
self->reusable_node = reusable_node_new();
self->dot_graph_file = NULL;
self->halt_on_error = false;
self->enabled = true;
self->operation_limit = SIZE_MAX;
self->cancellation_flag = NULL;
self->timeout_duration = 0;
self->end_clock = clock_null();
self->operation_count = 0;
self->old_tree = NULL_SUBTREE;
self->scratch_tree.ptr = &self->scratch_tree_data;
self->included_range_differences = (TSRangeArray) array_new();
@ -1569,20 +1592,20 @@ void ts_parser_halt_on_error(TSParser *self, bool should_halt_on_error) {
self->halt_on_error = should_halt_on_error;
}
bool ts_parser_enabled(const TSParser *self) {
return self->enabled;
const size_t *ts_parser_cancellation_flag(const TSParser *self) {
return (const size_t *)self->cancellation_flag;
}
void ts_parser_set_enabled(TSParser *self, bool enabled) {
self->enabled = enabled;
void ts_parser_set_cancellation_flag(TSParser *self, const size_t *flag) {
self->cancellation_flag = (const volatile size_t *)flag;
}
size_t ts_parser_operation_limit(const TSParser *self) {
return self->operation_limit;
uint64_t ts_parser_timeout_micros(const TSParser *self) {
return duration_to_micros(self->timeout_duration);
}
void ts_parser_set_operation_limit(TSParser *self, size_t limit) {
self->operation_limit = limit;
void ts_parser_set_timeout_micros(TSParser *self, uint64_t timeout_micros) {
self->timeout_duration = duration_from_micros(timeout_micros);
}
void ts_parser_set_included_ranges(TSParser *self, const TSRange *ranges, uint32_t count) {
@ -1645,24 +1668,26 @@ TSTree *ts_parser_parse(TSParser *self, const TSTree *old_tree, TSInput input) {
}
uint32_t position = 0, last_position = 0, version_count = 0;
size_t operation_count = 0;
self->operation_count = 0;
if (self->timeout_duration) {
self->end_clock = clock_after(clock_now(), self->timeout_duration);
} else {
self->end_clock = clock_null();
}
do {
for (StackVersion version = 0;
version_count = ts_stack_version_count(self->stack), version < version_count;
version++) {
if (operation_count > self->operation_limit || !self->enabled) return NULL;
operation_count++;
bool allow_node_reuse = version_count == 1;
while (ts_stack_is_active(self->stack, version)) {
LOG("process version:%d, version_count:%u, state:%d, row:%u, col:%u",
version, ts_stack_version_count(self->stack),
ts_stack_state(self->stack, version),
ts_stack_position(self->stack, version).extent.row,
ts_stack_position(self->stack, version).extent.row + 1,
ts_stack_position(self->stack, version).extent.column);
ts_parser__advance(self, version, allow_node_reuse);
if (!ts_parser__advance(self, version, allow_node_reuse)) return NULL;
LOG_STACK();
position = ts_stack_position(self->stack, version).bytes;