From 345e344377ee49d23ba843e9c1cf74d6a8ed49c4 Mon Sep 17 00:00:00 2001 From: Axel Hecht Date: Thu, 5 Apr 2018 14:39:25 +0200 Subject: [PATCH 01/90] Tests for issue 158 --- .../prepare_grammar/parse_regex_test.cc | 21 +++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/test/compiler/prepare_grammar/parse_regex_test.cc b/test/compiler/prepare_grammar/parse_regex_test.cc index a252654b..ef4f4857 100644 --- a/test/compiler/prepare_grammar/parse_regex_test.cc +++ b/test/compiler/prepare_grammar/parse_regex_test.cc @@ -150,6 +150,27 @@ describe("parse_regex", []() { }) }, + { + "escaped brackets", + "\\[\\]", + Rule::seq({ + CharacterSet{{'['}}, + CharacterSet{{']'}}, + }) + }, + + { + "escaped brackets in choice", + "[\\[\\]]", + CharacterSet{{'[', ']'}} + }, + + { + "escaped brackets in range", + "[\\[-\\]]", + CharacterSet{{'[', '\\', ']'}} + }, + { "escaped periods", "a\\.", From 65e654ea9bc07d0e0d7c5a04a4b940974a1c1dee Mon Sep 17 00:00:00 2001 From: Max Brunsfeld Date: Thu, 5 Apr 2018 13:25:16 -0700 Subject: [PATCH 02/90] Remove overly conservative check for the validity of keyword capture tokens --- src/compiler/build_tables/lex_table_builder.cc | 1 - 1 file changed, 1 deletion(-) diff --git a/src/compiler/build_tables/lex_table_builder.cc b/src/compiler/build_tables/lex_table_builder.cc index 250f6cde..066ddf1c 100644 --- a/src/compiler/build_tables/lex_table_builder.cc +++ b/src/compiler/build_tables/lex_table_builder.cc @@ -182,7 +182,6 @@ class LexTableBuilderImpl : public LexTableBuilder { // Don't use a token to capture keywords if it overlaps with separator characters. AllCharacterAggregator capture_aggregator; capture_aggregator.apply(grammar.variables[i].rule); - if (capture_aggregator.result.includes_all) continue; if (capture_aggregator.result.intersects(separator_start_characters)) continue; // Don't use a token to capture keywords if it conflicts with other tokens From 1ca261c79be3048a5c9d735c2bcf26e00be25163 Mon Sep 17 00:00:00 2001 From: Max Brunsfeld Date: Fri, 6 Apr 2018 12:46:06 -0700 Subject: [PATCH 03/90] Fix some regex parsing bugs * Allow escape sequences to be used in ranges * Don't give special meaning to dashes outside of character classes --- src/compiler/prepare_grammar/parse_regex.cc | 61 +++++++++++-------- .../prepare_grammar/parse_regex_test.cc | 30 +++++++++ 2 files changed, 64 insertions(+), 27 deletions(-) diff --git a/src/compiler/prepare_grammar/parse_regex.cc b/src/compiler/prepare_grammar/parse_regex.cc index 021b7e4d..4117af0a 100644 --- a/src/compiler/prepare_grammar/parse_regex.cc +++ b/src/compiler/prepare_grammar/parse_regex.cc @@ -136,10 +136,7 @@ class PatternParser { } default: { - auto pair = single_char(); - if (pair.second.type) - return { Blank{}, pair.second }; - return {pair.first, CompileError::none()}; + return {single_char(), CompileError::none()}; } } } @@ -154,38 +151,46 @@ class PatternParser { } while (has_more_input() && (peek() != ']')) { - auto pair = single_char(); - if (pair.second.type) - return { CharacterSet(), pair.second }; + auto characters = single_char(); + + if (peek() == '-') { + next(); + if (!characters.includes_all && characters.included_chars.size() == 1 && peek() != ']') { + auto next_characters = single_char(); + if (!next_characters.includes_all && next_characters.included_chars.size() == 1) { + characters.include( + *characters.included_chars.begin(), + *next_characters.included_chars.begin() + ); + } else { + characters.include('-'); + characters.add_set(next_characters); + } + } else { + characters.include('-'); + } + } + if (is_affirmative) - result.add_set(pair.first); + result.add_set(characters); else - result.remove_set(pair.first); + result.remove_set(characters); } return { result, CompileError::none() }; } - pair single_char() { + CharacterSet single_char() { CharacterSet value; - switch (peek()) { - case '\\': - next(); - value = escaped_char(peek()); - next(); - break; - default: - uint32_t first_char = peek(); - next(); - if (peek() == '-') { - next(); - value = CharacterSet().include(first_char, peek()); - next(); - } else { - value = CharacterSet().include(first_char); - } + if (peek() == '\\') { + next(); + value = escaped_char(peek()); + next(); + } else { + value = CharacterSet().include(peek()); + next(); } - return { value, CompileError::none() }; + return value; } CharacterSet escaped_char(uint32_t value) { @@ -220,6 +225,8 @@ class PatternParser { .exclude('\t') .exclude('\n') .exclude('\r'); + case '0': + return CharacterSet().include('\0'); case 't': return CharacterSet().include('\t'); case 'n': diff --git a/test/compiler/prepare_grammar/parse_regex_test.cc b/test/compiler/prepare_grammar/parse_regex_test.cc index ef4f4857..8a3ab54e 100644 --- a/test/compiler/prepare_grammar/parse_regex_test.cc +++ b/test/compiler/prepare_grammar/parse_regex_test.cc @@ -110,6 +110,30 @@ describe("parse_regex", []() { CharacterSet{{'\\'}} }, + { + "dashes", + "a-b", + Rule::seq({ + CharacterSet{{'a'}}, + CharacterSet{{'-'}}, + CharacterSet{{'b'}} + }) + }, + + { + "literal dashes in character classes", + "[a-][\\d-a][\\S-a]", + Rule::seq({ + CharacterSet{{'a', '-'}}, + CharacterSet().include('0', '9').include('-').include('a'), + CharacterSet().include_all() + .exclude(' ') + .exclude('\t') + .exclude('\r') + .exclude('\n') + }) + }, + { "character groups in sequences", "x([^x]|\\\\x)*x", @@ -171,6 +195,12 @@ describe("parse_regex", []() { CharacterSet{{'[', '\\', ']'}} }, + { + "escaped characters in ranges", + "[\\0-\\n]", + CharacterSet().include(0, '\n') + }, + { "escaped periods", "a\\.", From e59558c83b24ce07f142506add1763413d1fcecc Mon Sep 17 00:00:00 2001 From: Max Brunsfeld Date: Mon, 2 Apr 2018 09:47:01 -0700 Subject: [PATCH 04/90] Allow stack versions to be temporarily paused This way, when detecting an error, we can defer the decision about whether to bail or recover until all stack versions are processed. --- src/runtime/parser.c | 257 ++++++++++++------------ src/runtime/stack.c | 64 ++++-- src/runtime/stack.h | 10 +- test/fixtures/error_corpus/c_errors.txt | 21 ++ 4 files changed, 206 insertions(+), 146 deletions(-) diff --git a/src/runtime/parser.c b/src/runtime/parser.c index 34b4506b..a6dfa53a 100644 --- a/src/runtime/parser.c +++ b/src/runtime/parser.c @@ -171,6 +171,18 @@ static ErrorComparison parser__compare_versions(Parser *self, ErrorStatus a, Err return ErrorComparisonNone; } +static ErrorStatus parser__version_status(Parser *self, StackVersion version) { + unsigned cost = ts_stack_error_cost(self->stack, version); + bool is_paused = ts_stack_is_paused(self->stack, version); + if (is_paused) cost += ERROR_COST_PER_SKIPPED_TREE; + return (ErrorStatus) { + .cost = cost, + .push_count = ts_stack_push_count(self->stack, version), + .dynamic_precedence = ts_stack_dynamic_precedence(self->stack, version), + .is_in_error = is_paused || ts_stack_state(self->stack, version) == ERROR_STATE + }; +} + static bool parser__better_version_exists(Parser *self, StackVersion version, bool is_in_error, unsigned cost) { if (self->finished_tree && self->finished_tree->error_cost <= cost) return true; @@ -185,14 +197,9 @@ static bool parser__better_version_exists(Parser *self, StackVersion version, for (StackVersion i = 0, n = ts_stack_version_count(self->stack); i < n; i++) { if (i == version || - ts_stack_is_halted(self->stack, i) || + !ts_stack_is_active(self->stack, i) || ts_stack_position(self->stack, i).bytes < position.bytes) continue; - ErrorStatus status_i = { - .cost = ts_stack_error_cost(self->stack, i), - .is_in_error = ts_stack_state(self->stack, i) == ERROR_STATE, - .dynamic_precedence = ts_stack_dynamic_precedence(self->stack, i), - .push_count = ts_stack_push_count(self->stack, i) - }; + ErrorStatus status_i = parser__version_status(self, i); switch (parser__compare_versions(self, status, status_i)) { case ErrorComparisonTakeRight: return true; @@ -206,83 +213,6 @@ static bool parser__better_version_exists(Parser *self, StackVersion version, return false; } -static unsigned parser__condense_stack(Parser *self) { - bool made_changes = false; - unsigned min_error_cost = UINT_MAX; - for (StackVersion i = 0; i < ts_stack_version_count(self->stack); i++) { - if (ts_stack_is_halted(self->stack, i)) { - ts_stack_remove_version(self->stack, i); - i--; - continue; - } - - ErrorStatus status_i = { - .cost = ts_stack_error_cost(self->stack, i), - .push_count = ts_stack_push_count(self->stack, i), - .dynamic_precedence = ts_stack_dynamic_precedence(self->stack, i), - .is_in_error = ts_stack_state(self->stack, i) == ERROR_STATE, - }; - if (!status_i.is_in_error && status_i.cost < min_error_cost) { - min_error_cost = status_i.cost; - } - - for (StackVersion j = 0; j < i; j++) { - ErrorStatus status_j = { - .cost = ts_stack_error_cost(self->stack, j), - .push_count = ts_stack_push_count(self->stack, j), - .dynamic_precedence = ts_stack_dynamic_precedence(self->stack, j), - .is_in_error = ts_stack_state(self->stack, j) == ERROR_STATE, - }; - - bool can_merge = ts_stack_can_merge(self->stack, j, i); - switch (parser__compare_versions(self, status_j, status_i)) { - case ErrorComparisonTakeLeft: - made_changes = true; - ts_stack_remove_version(self->stack, i); - i--; - j = i; - break; - case ErrorComparisonPreferLeft: - case ErrorComparisonNone: - if (can_merge) { - made_changes = true; - ts_stack_force_merge(self->stack, j, i); - i--; - j = i; - } - break; - case ErrorComparisonPreferRight: - made_changes = true; - ts_stack_swap_versions(self->stack, i, j); - if (can_merge) { - ts_stack_force_merge(self->stack, j, i); - i--; - j = i; - } - break; - case ErrorComparisonTakeRight: - made_changes = true; - ts_stack_remove_version(self->stack, j); - i--; - j--; - break; - } - } - } - - while (ts_stack_version_count(self->stack) > MAX_VERSION_COUNT) { - ts_stack_remove_version(self->stack, MAX_VERSION_COUNT); - made_changes = true; - } - - if (made_changes) { - LOG("condense"); - LOG_STACK(); - } - - return min_error_cost; -} - static void parser__restore_external_scanner(Parser *self, Tree *external_token) { if (external_token) { self->language->external_scanner.deserialize( @@ -334,9 +264,7 @@ static Tree *parser__lex(Parser *self, StackVersion version, TSStateId parse_sta self->lexer.token_end_position = self->lexer.current_position; } - if (error_mode && self->lexer.token_end_position.bytes <= current_position.bytes) { - LOG("disregard_empty_token"); - } else { + if (!error_mode || self->lexer.token_end_position.bytes > current_position.bytes) { found_external_token = true; break; } @@ -360,7 +288,6 @@ static Tree *parser__lex(Parser *self, StackVersion version, TSStateId parse_sta } if (!error_mode) { - LOG("retry_in_error_mode"); error_mode = true; lex_mode = self->language->lex_modes[ERROR_STATE]; valid_external_tokens = ts_language_enabled_external_tokens( @@ -797,7 +724,8 @@ static void parser__accept(Parser *self, StackVersion version, static bool parser__do_all_potential_reductions(Parser *self, StackVersion starting_version, TSSymbol lookahead_symbol) { bool result = false; - for (StackVersion version = starting_version;;) { + for (StackVersion version = starting_version; + ts_stack_version_count(self->stack) < MAX_VERSION_COUNT;) { uint32_t version_count = ts_stack_version_count(self->stack); if (version >= version_count) break; @@ -869,24 +797,7 @@ static bool parser__do_all_potential_reductions(Parser *self, StackVersion start } static void parser__handle_error(Parser *self, StackVersion version, TSSymbol lookahead_symbol) { - // If enough parse versions have already completed, just halt this version. - if (self->accept_count > MAX_VERSION_COUNT) { - ts_stack_halt(self->stack, version); - LOG("bail_after_too_many_tries"); - return; - } - - // If there are other in-progress versions that are clearly better than this one, - // just halt this version. - unsigned new_cost = ts_stack_error_cost(self->stack, version) + ERROR_COST_PER_SKIPPED_TREE; - if (parser__better_version_exists(self, version, true, new_cost)) { - ts_stack_halt(self->stack, version); - LOG("bail_on_error"); - return; - } - // Perform any reductions that could have happened in this state, regardless of the lookahead. - LOG("handle_error"); uint32_t previous_version_count = ts_stack_version_count(self->stack); parser__do_all_potential_reductions(self, version, 0); uint32_t version_count = ts_stack_version_count(self->stack); @@ -923,7 +834,6 @@ static void parser__handle_error(Parser *self, StackVersion version, TSSymbol lo lookahead_symbol )) { LOG("recover_with_missing symbol:%s, state:%u", SYM_NAME(missing_symbol), state_after_missing_symbol); - LOG_STACK(); did_insert_missing_token = true; break; } @@ -1023,13 +933,15 @@ static void parser__recover(Parser *self, StackVersion version, Tree *lookahead) unsigned previous_version_count = ts_stack_version_count(self->stack); Length position = ts_stack_position(self->stack, version); StackSummary *summary = ts_stack_get_summary(self->stack, version); + unsigned depth_since_error = ts_stack_depth_since_error(self->stack, version); for (unsigned i = 0; i < summary->size; i++) { StackSummaryEntry entry = summary->contents[i]; if (entry.state == ERROR_STATE) continue; if (entry.position.bytes == position.bytes) continue; - unsigned depth = entry.depth + ts_stack_depth_since_error(self->stack, version); + unsigned depth = entry.depth + depth_since_error; + if (depth > MAX_SUMMARY_DEPTH) break; unsigned new_cost = depth * ERROR_COST_PER_SKIPPED_TREE + @@ -1041,21 +953,22 @@ static void parser__recover(Parser *self, StackVersion version, Tree *lookahead) if (parser__recover_to_state(self, version, depth, entry.state)) { did_recover = true; LOG("recover state:%u, depth:%u", entry.state, depth); + LOG_STACK(); break; } } } for (unsigned i = previous_version_count; i < ts_stack_version_count(self->stack); i++) { - if (ts_stack_is_halted(self->stack, i)) { - ts_stack_remove_version(self->stack, i--); - } else { + if (ts_stack_is_active(self->stack, i)) { for (unsigned j = 0; j < i; j++) { if (ts_stack_can_merge(self->stack, j, i)) { ts_stack_remove_version(self->stack, i--); break; } } + } else { + ts_stack_remove_version(self->stack, i--); } } @@ -1154,21 +1067,14 @@ static void parser__advance(Parser *self, StackVersion version, ReusableNode *re if (last_reduction_version != STACK_VERSION_NONE) { ts_stack_renumber_version(self->stack, last_reduction_version, version); LOG_STACK(); + } else if (state == ERROR_STATE) { + ts_stack_push(self->stack, version, lookahead, false, ERROR_STATE); + return; } else if (!parser__breakdown_top_of_stack(self, version)) { - if (state == ERROR_STATE) { - ts_stack_push(self->stack, version, lookahead, false, ERROR_STATE); - return; - } - - parser__handle_error(self, version, lookahead->first_leaf.symbol); - if (ts_stack_is_halted(self->stack, version)) { - ts_tree_release(&self->tree_pool, lookahead); - return; - } else if (lookahead->size.bytes == 0) { - ts_tree_release(&self->tree_pool, lookahead); - state = ts_stack_state(self->stack, version); - lookahead = parser__get_lookahead(self, version, &state, reusable_node, &table_entry); - } + LOG("detect_error"); + ts_stack_pause(self->stack, version, lookahead->first_leaf.symbol); + ts_tree_release(&self->tree_pool, lookahead); + return; } state = ts_stack_state(self->stack, version); @@ -1176,6 +1082,93 @@ static void parser__advance(Parser *self, StackVersion version, ReusableNode *re } } +static unsigned parser__condense_stack(Parser *self) { + bool made_changes = false; + unsigned min_error_cost = UINT_MAX; + for (StackVersion i = 0; i < ts_stack_version_count(self->stack); i++) { + if (ts_stack_is_halted(self->stack, i)) { + ts_stack_remove_version(self->stack, i); + i--; + continue; + } + + ErrorStatus status_i = parser__version_status(self, i); + if (!status_i.is_in_error && status_i.cost < min_error_cost) { + min_error_cost = status_i.cost; + } + + for (StackVersion j = 0; j < i; j++) { + ErrorStatus status_j = parser__version_status(self, j); + + bool can_merge = ts_stack_can_merge(self->stack, j, i); + switch (parser__compare_versions(self, status_j, status_i)) { + case ErrorComparisonTakeLeft: + made_changes = true; + ts_stack_remove_version(self->stack, i); + i--; + j = i; + break; + case ErrorComparisonPreferLeft: + case ErrorComparisonNone: + if (can_merge) { + made_changes = true; + ts_stack_force_merge(self->stack, j, i); + i--; + j = i; + } + break; + case ErrorComparisonPreferRight: + made_changes = true; + ts_stack_swap_versions(self->stack, i, j); + if (can_merge) { + ts_stack_force_merge(self->stack, j, i); + i--; + j = i; + } + break; + case ErrorComparisonTakeRight: + made_changes = true; + ts_stack_remove_version(self->stack, j); + i--; + j--; + break; + } + } + } + + while (ts_stack_version_count(self->stack) > MAX_VERSION_COUNT) { + ts_stack_remove_version(self->stack, MAX_VERSION_COUNT); + made_changes = true; + } + + if (ts_stack_version_count(self->stack) > 0) { + bool has_unpaused_version = false; + for (StackVersion i = 0, n = ts_stack_version_count(self->stack); i < n; i++) { + if (ts_stack_is_paused(self->stack, i)) { + if (!has_unpaused_version) { + LOG("resume version:%u", i); + TSSymbol lookahead_symbol = ts_stack_resume(self->stack, i); + parser__handle_error(self, i, lookahead_symbol); + has_unpaused_version = true; + } else { + ts_stack_remove_version(self->stack, i); + i--; + n--; + } + } else { + has_unpaused_version = true; + } + } + } + + if (made_changes) { + LOG("condense"); + LOG_STACK(); + } + + return min_error_cost; +} + bool parser_init(Parser *self) { ts_lexer_init(&self->lexer); array_init(&self->reduce_actions); @@ -1219,13 +1212,7 @@ Tree *parser_parse(Parser *self, TSInput input, Tree *old_tree, bool halt_on_err for (version = 0; version < ts_stack_version_count(self->stack); version++) { reusable_node = self->reusable_node; - while (!ts_stack_is_halted(self->stack, version)) { - position = ts_stack_position(self->stack, version).bytes; - if (position > last_position || (version > 0 && position == last_position)) { - last_position = position; - break; - } - + while (ts_stack_is_active(self->stack, version)) { LOG("process version:%d, version_count:%u, state:%d, row:%u, col:%u", version, ts_stack_version_count(self->stack), ts_stack_state(self->stack, version), @@ -1234,6 +1221,12 @@ Tree *parser_parse(Parser *self, TSInput input, Tree *old_tree, bool halt_on_err parser__advance(self, version, &reusable_node); LOG_STACK(); + + position = ts_stack_position(self->stack, version).bytes; + if (position > last_position || (version > 0 && position == last_position)) { + last_position = position; + break; + } } } diff --git a/src/runtime/stack.c b/src/runtime/stack.c index bd60a9ec..1ea8a181 100644 --- a/src/runtime/stack.c +++ b/src/runtime/stack.c @@ -49,12 +49,19 @@ typedef struct { typedef Array(StackNode *) StackNodeArray; +typedef enum { + StackStatusActive, + StackStatusPaused, + StackStatusHalted, +} StackStatus; + typedef struct { StackNode *node; Tree *last_external_token; - uint32_t push_count; - bool is_halted; StackSummary *summary; + uint32_t push_count; + TSSymbol lookahead_when_paused; + StackStatus status; } StackHead; struct Stack { @@ -215,7 +222,8 @@ static StackVersion ts_stack__add_version(Stack *self, StackVersion original_ver .node = node, .push_count = self->heads.contents[original_version].push_count, .last_external_token = last_external_token, - .is_halted = false, + .status = StackStatusActive, + .lookahead_when_paused = 0, }; array_push(&self->heads, head); stack_node_retain(node); @@ -581,7 +589,8 @@ bool ts_stack_can_merge(Stack *self, StackVersion version1, StackVersion version StackHead *head1 = &self->heads.contents[version1]; StackHead *head2 = &self->heads.contents[version2]; return - !head1->is_halted && !head2->is_halted && + head1->status == StackStatusActive && + head2->status == StackStatusActive && head1->node->state == head2->node->state && head1->node->position.bytes == head2->node->position.bytes && head1->node->depth == head2->node->depth && @@ -598,11 +607,34 @@ void ts_stack_force_merge(Stack *self, StackVersion version1, StackVersion versi } void ts_stack_halt(Stack *self, StackVersion version) { - array_get(&self->heads, version)->is_halted = true; + array_get(&self->heads, version)->status = StackStatusHalted; } -bool ts_stack_is_halted(Stack *self, StackVersion version) { - return array_get(&self->heads, version)->is_halted; +void ts_stack_pause(Stack *self, StackVersion version, TSSymbol lookahead) { + StackHead *head = array_get(&self->heads, version); + head->status = StackStatusPaused; + head->lookahead_when_paused = lookahead; +} + +bool ts_stack_is_active(const Stack *self, StackVersion version) { + return array_get(&self->heads, version)->status == StackStatusActive; +} + +bool ts_stack_is_halted(const Stack *self, StackVersion version) { + return array_get(&self->heads, version)->status == StackStatusHalted; +} + +bool ts_stack_is_paused(const Stack *self, StackVersion version) { + return array_get(&self->heads, version)->status == StackStatusPaused; +} + +TSSymbol ts_stack_resume(Stack *self, StackVersion version) { + StackHead *head = array_get(&self->heads, version); + assert(head->status == StackStatusPaused); + TSSymbol result = head->lookahead_when_paused; + head->status = StackStatusActive; + head->lookahead_when_paused = 0; + return result; } void ts_stack_clear(Stack *self) { @@ -614,7 +646,8 @@ void ts_stack_clear(Stack *self) { array_push(&self->heads, ((StackHead){ .node = self->base_node, .last_external_token = NULL, - .is_halted = false, + .status = StackStatusActive, + .lookahead_when_paused = 0, })); } @@ -631,13 +664,18 @@ bool ts_stack_print_dot_graph(Stack *self, const char **symbol_names, FILE *f) { array_clear(&self->iterators); for (uint32_t i = 0; i < self->heads.size; i++) { - if (ts_stack_is_halted(self, i)) continue; StackHead *head = &self->heads.contents[i]; + if (head->status == StackStatusHalted) continue; + fprintf(f, "node_head_%u [shape=none, label=\"\"]\n", i); - fprintf( - f, - "node_head_%u -> node_%p [label=%u, fontcolor=blue, weight=10000, " - "labeltooltip=\"push_count: %u\ndepth: %u", i, head->node, i, head->push_count, head->node->depth + fprintf(f, "node_head_%u -> node_%p [", i, head->node); + + if (head->status == StackStatusPaused) { + fprintf(f, "color=red "); + } + fprintf(f, + "label=%u, fontcolor=blue, weight=10000, labeltooltip=\"push_count: %u\ndepth: %u", + i, head->push_count, head->node->depth ); if (head->last_external_token) { diff --git a/src/runtime/stack.h b/src/runtime/stack.h index 14f0f2a7..0be0a4dd 100644 --- a/src/runtime/stack.h +++ b/src/runtime/stack.h @@ -102,9 +102,17 @@ bool ts_stack_can_merge(Stack *, StackVersion, StackVersion); void ts_stack_force_merge(Stack *, StackVersion, StackVersion); +TSSymbol ts_stack_resume(Stack *, StackVersion); + +void ts_stack_pause(Stack *, StackVersion, TSSymbol); + void ts_stack_halt(Stack *, StackVersion); -bool ts_stack_is_halted(Stack *, StackVersion); +bool ts_stack_is_active(const Stack *, StackVersion); + +bool ts_stack_is_paused(const Stack *, StackVersion); + +bool ts_stack_is_halted(const Stack *, StackVersion); void ts_stack_renumber_version(Stack *, StackVersion, StackVersion); diff --git a/test/fixtures/error_corpus/c_errors.txt b/test/fixtures/error_corpus/c_errors.txt index d1d76d78..c5833156 100644 --- a/test/fixtures/error_corpus/c_errors.txt +++ b/test/fixtures/error_corpus/c_errors.txt @@ -141,3 +141,24 @@ int y = 5; (translation_unit (declaration (primitive_type) (ERROR (identifier)) (identifier)) (declaration (primitive_type) (init_declarator (identifier) (number_literal)))) + +========================================== +Declarations with missing variable names +========================================== + +int a() { + struct x = 1; + int = 2; +} + +--- + +(translation_unit + (function_definition + (primitive_type) + (function_declarator (identifier) (parameter_list)) + (compound_statement + (struct_specifier (type_identifier)) + (ERROR (number_literal)) + (primitive_type) + (ERROR (number_literal))))) From f3bbf045b704a3b250ea60210b89ce9db759a8ad Mon Sep 17 00:00:00 2001 From: Max Brunsfeld Date: Mon, 2 Apr 2018 10:05:12 -0700 Subject: [PATCH 05/90] Avoid unnecessary stack node retain and release on every push --- src/runtime/stack.c | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/runtime/stack.c b/src/runtime/stack.c index 1ea8a181..60ed6575 100644 --- a/src/runtime/stack.c +++ b/src/runtime/stack.c @@ -126,8 +126,6 @@ static StackNode *stack_node_new(StackNode *previous_node, Tree *tree, bool is_p *node = (StackNode){.ref_count = 1, .link_count = 0, .state = state, .depth = 0}; if (previous_node) { - stack_node_retain(previous_node); - node->link_count = 1; node->links[0] = (StackLink){ .node = previous_node, @@ -415,7 +413,6 @@ void ts_stack_push(Stack *self, StackVersion version, Tree *tree, bool pending, } else if (!tree->extra) { head->push_count++; } - stack_node_release(head->node, &self->node_pool, self->tree_pool); head->node = new_node; } From 80f856cef56f5c0a350625a0483211b8559a8892 Mon Sep 17 00:00:00 2001 From: Max Brunsfeld Date: Mon, 2 Apr 2018 10:57:44 -0700 Subject: [PATCH 06/90] Maintain a total node count on every tree This simplifies (and fixes bugs in) the parse stack's tracking of its total node count since the last error, which is needed for error recovery. --- src/runtime/error_costs.h | 2 +- src/runtime/parser.c | 14 ++-- src/runtime/stack.c | 83 +++++++++---------- src/runtime/stack.h | 12 +-- src/runtime/tree.c | 3 + src/runtime/tree.h | 1 + .../error_corpus/javascript_errors.txt | 13 +-- test/integration/real_grammars.cc | 9 +- 8 files changed, 69 insertions(+), 68 deletions(-) diff --git a/src/runtime/error_costs.h b/src/runtime/error_costs.h index 5ba1fc62..f543b3ff 100644 --- a/src/runtime/error_costs.h +++ b/src/runtime/error_costs.h @@ -2,7 +2,7 @@ #define RUNTIME_ERROR_COSTS_H_ #define ERROR_STATE 0 -#define ERROR_COST_PER_MISSING_TREE 150 +#define ERROR_COST_PER_MISSING_TREE 110 #define ERROR_COST_PER_SKIPPED_TREE 100 #define ERROR_COST_PER_SKIPPED_LINE 30 #define ERROR_COST_PER_SKIPPED_CHAR 1 diff --git a/src/runtime/parser.c b/src/runtime/parser.c index a6dfa53a..76bcb4b9 100644 --- a/src/runtime/parser.c +++ b/src/runtime/parser.c @@ -39,7 +39,7 @@ static const unsigned MAX_COST_DIFFERENCE = 16 * ERROR_COST_PER_SKIPPED_TREE; typedef struct { unsigned cost; - unsigned push_count; + unsigned node_count; int dynamic_precedence; bool is_in_error; } ErrorStatus; @@ -105,7 +105,6 @@ static bool parser__breakdown_top_of_stack(Parser *self, StackVersion version) { ts_stack_push(self->stack, slice.version, tree, false, state); } - ts_stack_decrease_push_count(self->stack, slice.version, parent->child_count + 1); ts_tree_release(&self->tree_pool, parent); array_delete(&slice.trees); @@ -151,7 +150,7 @@ static ErrorComparison parser__compare_versions(Parser *self, ErrorStatus a, Err } if (a.cost < b.cost) { - if ((b.cost - a.cost) * (1 + a.push_count) > MAX_COST_DIFFERENCE) { + if ((b.cost - a.cost) * (1 + a.node_count) > MAX_COST_DIFFERENCE) { return ErrorComparisonTakeLeft; } else { return ErrorComparisonPreferLeft; @@ -159,7 +158,7 @@ static ErrorComparison parser__compare_versions(Parser *self, ErrorStatus a, Err } if (b.cost < a.cost) { - if ((a.cost - b.cost) * (1 + b.push_count) > MAX_COST_DIFFERENCE) { + if ((a.cost - b.cost) * (1 + b.node_count) > MAX_COST_DIFFERENCE) { return ErrorComparisonTakeRight; } else { return ErrorComparisonPreferRight; @@ -177,7 +176,7 @@ static ErrorStatus parser__version_status(Parser *self, StackVersion version) { if (is_paused) cost += ERROR_COST_PER_SKIPPED_TREE; return (ErrorStatus) { .cost = cost, - .push_count = ts_stack_push_count(self->stack, version), + .node_count = ts_stack_node_count_since_error(self->stack, version), .dynamic_precedence = ts_stack_dynamic_precedence(self->stack, version), .is_in_error = is_paused || ts_stack_state(self->stack, version) == ERROR_STATE }; @@ -192,7 +191,7 @@ static bool parser__better_version_exists(Parser *self, StackVersion version, .cost = cost, .is_in_error = is_in_error, .dynamic_precedence = ts_stack_dynamic_precedence(self->stack, version), - .push_count = 0, + .node_count = ts_stack_node_count_since_error(self->stack, version), }; for (StackVersion i = 0, n = ts_stack_version_count(self->stack); i < n; i++) { @@ -933,7 +932,7 @@ static void parser__recover(Parser *self, StackVersion version, Tree *lookahead) unsigned previous_version_count = ts_stack_version_count(self->stack); Length position = ts_stack_position(self->stack, version); StackSummary *summary = ts_stack_get_summary(self->stack, version); - unsigned depth_since_error = ts_stack_depth_since_error(self->stack, version); + unsigned depth_since_error = ts_stack_node_count_since_error(self->stack, version); for (unsigned i = 0; i < summary->size; i++) { StackSummaryEntry entry = summary->contents[i]; @@ -1150,6 +1149,7 @@ static unsigned parser__condense_stack(Parser *self) { TSSymbol lookahead_symbol = ts_stack_resume(self->stack, i); parser__handle_error(self, i, lookahead_symbol); has_unpaused_version = true; + min_error_cost = ts_stack_error_cost(self->stack, i); } else { ts_stack_remove_version(self->stack, i); i--; diff --git a/src/runtime/stack.c b/src/runtime/stack.c index 60ed6575..f993f59f 100644 --- a/src/runtime/stack.c +++ b/src/runtime/stack.c @@ -31,7 +31,7 @@ struct StackNode { short unsigned int link_count; uint32_t ref_count; unsigned error_cost; - unsigned depth; + unsigned node_count; int dynamic_precedence; }; @@ -59,7 +59,7 @@ typedef struct { StackNode *node; Tree *last_external_token; StackSummary *summary; - uint32_t push_count; + unsigned node_count_at_last_error; TSSymbol lookahead_when_paused; StackStatus status; } StackHead; @@ -123,7 +123,7 @@ static StackNode *stack_node_new(StackNode *previous_node, Tree *tree, bool is_p StackNode *node = pool->size > 0 ? array_pop(pool) : ts_malloc(sizeof(StackNode)); - *node = (StackNode){.ref_count = 1, .link_count = 0, .state = state, .depth = 0}; + *node = (StackNode){.ref_count = 1, .link_count = 0, .state = state}; if (previous_node) { node->link_count = 1; @@ -136,22 +136,25 @@ static StackNode *stack_node_new(StackNode *previous_node, Tree *tree, bool is_p node->position = previous_node->position; node->error_cost = previous_node->error_cost; node->dynamic_precedence = previous_node->dynamic_precedence; + node->node_count = previous_node->node_count; if (tree) { - node->depth = previous_node->depth; - if (!tree->extra) node->depth++; node->error_cost += tree->error_cost; node->position = length_add(node->position, ts_tree_total_size(tree)); node->dynamic_precedence += tree->dynamic_precedence; - if (state == ERROR_STATE && !tree->extra) { - node->error_cost += - ERROR_COST_PER_SKIPPED_TREE * ((tree->visible || tree->child_count == 0) ? 1 : tree->visible_child_count) + - ERROR_COST_PER_SKIPPED_CHAR * tree->size.bytes + - ERROR_COST_PER_SKIPPED_LINE * tree->size.extent.row; - if (previous_node->links[0].tree) { + if (!tree->extra) { + node->node_count += tree->node_count; + + if (state == ERROR_STATE) { node->error_cost += - ERROR_COST_PER_SKIPPED_CHAR * tree->padding.bytes + - ERROR_COST_PER_SKIPPED_LINE * tree->padding.extent.row; + ERROR_COST_PER_SKIPPED_TREE * ((tree->visible || tree->child_count == 0) ? 1 : tree->visible_child_count) + + ERROR_COST_PER_SKIPPED_CHAR * tree->size.bytes + + ERROR_COST_PER_SKIPPED_LINE * tree->size.extent.row; + if (previous_node->links[0].tree) { + node->error_cost += + ERROR_COST_PER_SKIPPED_CHAR * tree->padding.bytes + + ERROR_COST_PER_SKIPPED_LINE * tree->padding.extent.row; + } } } } @@ -198,6 +201,10 @@ static void stack_node_add_link(StackNode *self, StackLink link) { stack_node_retain(link.node); if (link.tree) ts_tree_retain(link.tree); self->links[self->link_count++] = link; + + unsigned node_count = link.node->node_count; + if (link.tree) node_count += link.tree->node_count; + if (node_count > self->node_count) self->node_count = node_count; } } @@ -215,22 +222,22 @@ static void stack_head_delete(StackHead *self, StackNodeArray *pool, TreePool *t } static StackVersion ts_stack__add_version(Stack *self, StackVersion original_version, - StackNode *node, Tree *last_external_token) { + StackNode *node) { StackHead head = { .node = node, - .push_count = self->heads.contents[original_version].push_count, - .last_external_token = last_external_token, + .node_count_at_last_error = self->heads.contents[original_version].node_count_at_last_error, + .last_external_token = self->heads.contents[original_version].last_external_token, .status = StackStatusActive, .lookahead_when_paused = 0, }; array_push(&self->heads, head); stack_node_retain(node); - if (last_external_token) ts_tree_retain(last_external_token); + if (head.last_external_token) ts_tree_retain(head.last_external_token); return (StackVersion)(self->heads.size - 1); } -static void ts_stack__add_slice(Stack *self, StackVersion original_version, StackNode *node, - TreeArray *trees, Tree *last_external_token) { +static void ts_stack__add_slice(Stack *self, StackVersion original_version, + StackNode *node, TreeArray *trees) { for (uint32_t i = self->slices.size - 1; i + 1 > 0; i--) { StackVersion version = self->slices.contents[i].version; if (self->heads.contents[version].node == node) { @@ -240,7 +247,7 @@ static void ts_stack__add_slice(Stack *self, StackVersion original_version, Stac } } - StackVersion version = ts_stack__add_version(self, original_version, node, last_external_token); + StackVersion version = ts_stack__add_version(self, original_version, node); StackSlice slice = { *trees, version }; array_push(&self->slices, slice); } @@ -252,7 +259,6 @@ inline StackSliceArray stack__iter(Stack *self, StackVersion version, array_clear(&self->iterators); StackHead *head = array_get(&self->heads, version); - Tree *last_external_token = head->last_external_token; Iterator iterator = { .node = head->node, .trees = array_new(), @@ -279,8 +285,7 @@ inline StackSliceArray stack__iter(Stack *self, StackVersion version, self, version, node, - &trees, - last_external_token + &trees ); } @@ -381,14 +386,6 @@ Length ts_stack_position(const Stack *self, StackVersion version) { return array_get(&self->heads, version)->node->position; } -unsigned ts_stack_push_count(const Stack *self, StackVersion version) { - return array_get(&self->heads, version)->push_count; -} - -void ts_stack_decrease_push_count(Stack *self, StackVersion version, unsigned decrement) { - array_get(&self->heads, version)->push_count -= decrement; -} - Tree *ts_stack_last_external_token(const Stack *self, StackVersion version) { return array_get(&self->heads, version)->last_external_token; } @@ -405,14 +402,15 @@ unsigned ts_stack_error_cost(const Stack *self, StackVersion version) { return head->node->error_cost; } +unsigned ts_stack_node_count_since_error(const Stack *self, StackVersion version) { + StackHead *head = array_get(&self->heads, version); + return head->node->node_count - head->node_count_at_last_error; +} + void ts_stack_push(Stack *self, StackVersion version, Tree *tree, bool pending, TSStateId state) { StackHead *head = array_get(&self->heads, version); StackNode *new_node = stack_node_new(head->node, tree, pending, state, &self->node_pool); - if (state == ERROR_STATE) { - head->push_count = 0; - } else if (!tree->extra) { - head->push_count++; - } + if (!tree) head->node_count_at_last_error = new_node->node_count; head->node = new_node; } @@ -536,10 +534,6 @@ StackSummary *ts_stack_get_summary(Stack *self, StackVersion version) { return array_get(&self->heads, version)->summary; } -unsigned ts_stack_depth_since_error(Stack *self, StackVersion version) { - return array_get(&self->heads, version)->node->depth; -} - int ts_stack_dynamic_precedence(Stack *self, StackVersion version) { return array_get(&self->heads, version)->node->dynamic_precedence; } @@ -590,7 +584,6 @@ bool ts_stack_can_merge(Stack *self, StackVersion version1, StackVersion version head2->status == StackStatusActive && head1->node->state == head2->node->state && head1->node->position.bytes == head2->node->position.bytes && - head1->node->depth == head2->node->depth && ts_tree_external_token_state_eq(head1->last_external_token, head2->last_external_token); } @@ -600,6 +593,9 @@ void ts_stack_force_merge(Stack *self, StackVersion version1, StackVersion versi for (uint32_t i = 0; i < head2->node->link_count; i++) { stack_node_add_link(head1->node, head2->node->links[i]); } + if (head2->node_count_at_last_error > head1->node_count_at_last_error) { + head1->node_count_at_last_error = head2->node_count_at_last_error; + } ts_stack_remove_version(self, version2); } @@ -611,6 +607,7 @@ void ts_stack_pause(Stack *self, StackVersion version, TSSymbol lookahead) { StackHead *head = array_get(&self->heads, version); head->status = StackStatusPaused; head->lookahead_when_paused = lookahead; + head->node_count_at_last_error = head->node->node_count; } bool ts_stack_is_active(const Stack *self, StackVersion version) { @@ -671,8 +668,8 @@ bool ts_stack_print_dot_graph(Stack *self, const char **symbol_names, FILE *f) { fprintf(f, "color=red "); } fprintf(f, - "label=%u, fontcolor=blue, weight=10000, labeltooltip=\"push_count: %u\ndepth: %u", - i, head->push_count, head->node->depth + "label=%u, fontcolor=blue, weight=10000, labeltooltip=\"node_count: %u", + i, head->node->node_count - head->node_count_at_last_error ); if (head->last_external_token) { diff --git a/src/runtime/stack.h b/src/runtime/stack.h index 0be0a4dd..4704d90e 100644 --- a/src/runtime/stack.h +++ b/src/runtime/stack.h @@ -41,15 +41,6 @@ uint32_t ts_stack_version_count(const Stack *); // empty, this returns the initial state, 0. TSStateId ts_stack_state(const Stack *, StackVersion); -// Get the number of trees that have been pushed to a given version of -// the stack. -unsigned ts_stack_push_count(const Stack *, StackVersion); - -// In the event that trees were permanently removed from some version -// of the stack, decrease the version's push count to account for the -// removal. -void ts_stack_decrease_push_count(Stack *, StackVersion, unsigned); - // Get the last external token associated with a given version of the stack. Tree *ts_stack_last_external_token(const Stack *, StackVersion); @@ -82,7 +73,8 @@ StackSliceArray ts_stack_pop_pending(Stack *, StackVersion); // Remove any all trees from the given version of the stack. StackSliceArray ts_stack_pop_all(Stack *, StackVersion); -unsigned ts_stack_depth_since_error(Stack *, StackVersion); +// Get the number of tree nodes on the stack since the most recent error. +unsigned ts_stack_node_count_since_error(const Stack *, StackVersion); int ts_stack_dynamic_precedence(Stack *, StackVersion); diff --git a/src/runtime/tree.c b/src/runtime/tree.c index ae35e1f8..c58c987a 100644 --- a/src/runtime/tree.c +++ b/src/runtime/tree.c @@ -177,6 +177,7 @@ Tree *ts_tree_make_leaf(TreePool *pool, TSSymbol symbol, Length padding, Length .padding = padding, .visible = metadata.visible, .named = metadata.named, + .node_count = 1, .has_changes = false, .first_leaf = { .symbol = symbol, @@ -305,6 +306,7 @@ void ts_tree_set_children(Tree *self, uint32_t child_count, Tree **children, self->visible_child_count = 0; self->error_cost = 0; self->repeat_depth = 0; + self->node_count = 1; self->has_external_tokens = false; self->dynamic_precedence = 0; @@ -326,6 +328,7 @@ void ts_tree_set_children(Tree *self, uint32_t child_count, Tree **children, self->error_cost += child->error_cost; self->dynamic_precedence += child->dynamic_precedence; + self->node_count += child->node_count; if (alias_sequence && alias_sequence[non_extra_index] != 0 && !child->extra) { self->visible_child_count++; diff --git a/src/runtime/tree.h b/src/runtime/tree.h index 0e3c2880..ad104894 100644 --- a/src/runtime/tree.h +++ b/src/runtime/tree.h @@ -50,6 +50,7 @@ typedef struct Tree { TSSymbol symbol; TSStateId parse_state; unsigned error_cost; + unsigned node_count; unsigned repeat_depth; struct { diff --git a/test/fixtures/error_corpus/javascript_errors.txt b/test/fixtures/error_corpus/javascript_errors.txt index 250f13b6..d435ba86 100644 --- a/test/fixtures/error_corpus/javascript_errors.txt +++ b/test/fixtures/error_corpus/javascript_errors.txt @@ -79,12 +79,13 @@ if ({a: 'b'} {c: 'd'}) { (statement_block (expression_statement (assignment_expression (identifier) - (ERROR (function - (formal_parameters (identifier)) - (statement_block (expression_statement (identifier))))) - (function - (formal_parameters (identifier)) - (statement_block (expression_statement (identifier))))))))) + (call_expression + (function (formal_parameters (identifier)) (statement_block (expression_statement (identifier)))) + (ERROR) + (arguments (identifier)))) + (MISSING)) + (statement_block + (expression_statement (identifier)))))) =================================================== Extra tokens at the end of the file diff --git a/test/integration/real_grammars.cc b/test/integration/real_grammars.cc index 6d5ac27e..6cee7870 100644 --- a/test/integration/real_grammars.cc +++ b/test/integration/real_grammars.cc @@ -32,6 +32,8 @@ vector test_languages({ for (auto &language_name : test_languages) { describe(("the " + language_name + " language").c_str(), [&]() { TSDocument *document; + const bool debug_graphs_enabled = getenv("TREE_SITTER_ENABLE_DEBUG_GRAPHS"); + before_each([&]() { record_alloc::start(); @@ -39,7 +41,7 @@ for (auto &language_name : test_languages) { ts_document_set_language(document, load_real_language(language_name)); // ts_document_set_logger(document, stderr_logger_new(true)); - if (getenv("TREE_SITTER_ENABLE_DEBUG_GRAPHS")) { + if (debug_graphs_enabled) { ts_document_print_debugging_graphs(document, true); } }); @@ -55,6 +57,7 @@ for (auto &language_name : test_languages) { auto it_handles_edit_sequence = [&](string name, std::function edit_sequence){ it(("parses " + entry.description + ": " + name).c_str(), [&]() { input = new SpyInput(entry.input, 3); + if (debug_graphs_enabled) printf("%s\n\n", input->content.c_str()); ts_document_set_input(document, input->input()); edit_sequence(); @@ -88,9 +91,11 @@ for (auto &language_name : test_languages) { ts_document_edit(document, input->replace(edit_position, 0, inserted_text)); ts_document_parse(document); assert_correct_tree_size(document, input->content); + if (debug_graphs_enabled) printf("%s\n\n", input->content.c_str()); ts_document_edit(document, input->undo()); assert_correct_tree_size(document, input->content); + if (debug_graphs_enabled) printf("%s\n\n", input->content.c_str()); TSRange *ranges; uint32_t range_count; @@ -112,9 +117,11 @@ for (auto &language_name : test_languages) { ts_document_edit(document, input->replace(edit_position, deletion_size, "")); ts_document_parse(document); assert_correct_tree_size(document, input->content); + if (debug_graphs_enabled) printf("%s\n\n", input->content.c_str()); ts_document_edit(document, input->undo()); assert_correct_tree_size(document, input->content); + if (debug_graphs_enabled) printf("%s\n\n", input->content.c_str()); TSRange *ranges; uint32_t range_count; From 1d9d6f37ad770df8eed4c8ff80dc8863616f6ef3 Mon Sep 17 00:00:00 2001 From: Max Brunsfeld Date: Mon, 2 Apr 2018 11:52:34 -0700 Subject: [PATCH 07/90] Introduce an error cost per error instance to favor fewer errors --- src/runtime/error_costs.h | 1 + src/runtime/stack.c | 10 +++++++--- src/runtime/tree.c | 11 +++++++---- 3 files changed, 15 insertions(+), 7 deletions(-) diff --git a/src/runtime/error_costs.h b/src/runtime/error_costs.h index f543b3ff..d6420488 100644 --- a/src/runtime/error_costs.h +++ b/src/runtime/error_costs.h @@ -2,6 +2,7 @@ #define RUNTIME_ERROR_COSTS_H_ #define ERROR_STATE 0 +#define ERROR_COST_PER_RECOVERY 500 #define ERROR_COST_PER_MISSING_TREE 110 #define ERROR_COST_PER_SKIPPED_TREE 100 #define ERROR_COST_PER_SKIPPED_LINE 30 diff --git a/src/runtime/stack.c b/src/runtime/stack.c index f993f59f..17a301fe 100644 --- a/src/runtime/stack.c +++ b/src/runtime/stack.c @@ -399,7 +399,9 @@ void ts_stack_set_last_external_token(Stack *self, StackVersion version, Tree *t unsigned ts_stack_error_cost(const Stack *self, StackVersion version) { StackHead *head = array_get(&self->heads, version); - return head->node->error_cost; + unsigned result = head->node->error_cost; + if (head->node->state == ERROR_STATE) result += ERROR_COST_PER_RECOVERY; + return result; } unsigned ts_stack_node_count_since_error(const Stack *self, StackVersion version) { @@ -668,8 +670,10 @@ bool ts_stack_print_dot_graph(Stack *self, const char **symbol_names, FILE *f) { fprintf(f, "color=red "); } fprintf(f, - "label=%u, fontcolor=blue, weight=10000, labeltooltip=\"node_count: %u", - i, head->node->node_count - head->node_count_at_last_error + "label=%u, fontcolor=blue, weight=10000, labeltooltip=\"node_count: %u\nerror_cost: %u", + i, + ts_stack_node_count_since_error(self, i), + ts_stack_error_cost(self, i) ); if (head->last_external_token) { diff --git a/src/runtime/tree.c b/src/runtime/tree.c index c58c987a..f88895d2 100644 --- a/src/runtime/tree.c +++ b/src/runtime/tree.c @@ -354,11 +354,14 @@ void ts_tree_set_children(Tree *self, uint32_t child_count, Tree **children, } if (self->symbol == ts_builtin_sym_error) { - self->error_cost += ERROR_COST_PER_SKIPPED_CHAR * self->size.bytes + + self->error_cost += ERROR_COST_PER_RECOVERY + + ERROR_COST_PER_SKIPPED_CHAR * self->size.bytes + ERROR_COST_PER_SKIPPED_LINE * self->size.extent.row; - for (uint32_t i = 0; i < child_count; i++) - if (!self->children[i]->extra) + for (uint32_t i = 0; i < child_count; i++) { + if (!self->children[i]->extra) { self->error_cost += ERROR_COST_PER_SKIPPED_TREE; + } + } } if (child_count > 0) { @@ -418,7 +421,7 @@ Tree *ts_tree_make_error_node(TreePool *pool, TreeArray *children, const TSLangu Tree *ts_tree_make_missing_leaf(TreePool *pool, TSSymbol symbol, const TSLanguage *language) { Tree *result = ts_tree_make_leaf(pool, symbol, length_zero(), length_zero(), language); result->is_missing = true; - result->error_cost = ERROR_COST_PER_MISSING_TREE; + result->error_cost = ERROR_COST_PER_MISSING_TREE + ERROR_COST_PER_RECOVERY; return result; } From 0ec7e5ce4261da16b68c0405aa30997a9be33d9e Mon Sep 17 00:00:00 2001 From: Max Brunsfeld Date: Mon, 2 Apr 2018 11:57:26 -0700 Subject: [PATCH 08/90] Remove ts_stack_force_merge function --- src/runtime/parser.c | 12 +++++------- src/runtime/stack.c | 43 ++++++++++++++++++------------------------- src/runtime/stack.h | 9 ++++++--- 3 files changed, 29 insertions(+), 35 deletions(-) diff --git a/src/runtime/parser.c b/src/runtime/parser.c index 76bcb4b9..96f98cdf 100644 --- a/src/runtime/parser.c +++ b/src/runtime/parser.c @@ -845,7 +845,7 @@ static void parser__handle_error(Parser *self, StackVersion version, TSSymbol lo } for (unsigned i = previous_version_count; i < version_count; i++) { - ts_stack_force_merge(self->stack, version, previous_version_count); + assert(ts_stack_merge(self->stack, version, previous_version_count)); } ts_stack_record_summary(self->stack, version, MAX_SUMMARY_DEPTH); @@ -1099,7 +1099,6 @@ static unsigned parser__condense_stack(Parser *self) { for (StackVersion j = 0; j < i; j++) { ErrorStatus status_j = parser__version_status(self, j); - bool can_merge = ts_stack_can_merge(self->stack, j, i); switch (parser__compare_versions(self, status_j, status_i)) { case ErrorComparisonTakeLeft: made_changes = true; @@ -1109,20 +1108,19 @@ static unsigned parser__condense_stack(Parser *self) { break; case ErrorComparisonPreferLeft: case ErrorComparisonNone: - if (can_merge) { + if (ts_stack_merge(self->stack, j, i)) { made_changes = true; - ts_stack_force_merge(self->stack, j, i); i--; j = i; } break; case ErrorComparisonPreferRight: made_changes = true; - ts_stack_swap_versions(self->stack, i, j); - if (can_merge) { - ts_stack_force_merge(self->stack, j, i); + if (ts_stack_merge(self->stack, j, i)) { i--; j = i; + } else { + ts_stack_swap_versions(self->stack, i, j); } break; case ErrorComparisonTakeRight: diff --git a/src/runtime/stack.c b/src/runtime/stack.c index 17a301fe..9186397f 100644 --- a/src/runtime/stack.c +++ b/src/runtime/stack.c @@ -197,15 +197,15 @@ static void stack_node_add_link(StackNode *self, StackLink link) { } } - if (self->link_count < MAX_LINK_COUNT) { - stack_node_retain(link.node); - if (link.tree) ts_tree_retain(link.tree); - self->links[self->link_count++] = link; + if (self->link_count == MAX_LINK_COUNT) return; - unsigned node_count = link.node->node_count; - if (link.tree) node_count += link.tree->node_count; - if (node_count > self->node_count) self->node_count = node_count; - } + stack_node_retain(link.node); + if (link.tree) ts_tree_retain(link.tree); + self->links[self->link_count++] = link; + + unsigned node_count = link.node->node_count; + if (link.tree) node_count += link.tree->node_count; + if (node_count > self->node_count) self->node_count = node_count; } static void stack_head_delete(StackHead *self, StackNodeArray *pool, TreePool *tree_pool) { @@ -570,12 +570,17 @@ StackVersion ts_stack_copy_version(Stack *self, StackVersion version) { } bool ts_stack_merge(Stack *self, StackVersion version1, StackVersion version2) { - if (ts_stack_can_merge(self, version1, version2)) { - ts_stack_force_merge(self, version1, version2); - return true; - } else { - return false; + if (!ts_stack_can_merge(self, version1, version2)) return false; + StackHead *head1 = &self->heads.contents[version1]; + StackHead *head2 = &self->heads.contents[version2]; + for (uint32_t i = 0; i < head2->node->link_count; i++) { + stack_node_add_link(head1->node, head2->node->links[i]); } + if (head2->node_count_at_last_error > head1->node_count_at_last_error) { + head1->node_count_at_last_error = head2->node_count_at_last_error; + } + ts_stack_remove_version(self, version2); + return true; } bool ts_stack_can_merge(Stack *self, StackVersion version1, StackVersion version2) { @@ -589,18 +594,6 @@ bool ts_stack_can_merge(Stack *self, StackVersion version1, StackVersion version ts_tree_external_token_state_eq(head1->last_external_token, head2->last_external_token); } -void ts_stack_force_merge(Stack *self, StackVersion version1, StackVersion version2) { - StackHead *head1 = &self->heads.contents[version1]; - StackHead *head2 = &self->heads.contents[version2]; - for (uint32_t i = 0; i < head2->node->link_count; i++) { - stack_node_add_link(head1->node, head2->node->links[i]); - } - if (head2->node_count_at_last_error > head1->node_count_at_last_error) { - head1->node_count_at_last_error = head2->node_count_at_last_error; - } - ts_stack_remove_version(self, version2); -} - void ts_stack_halt(Stack *self, StackVersion version) { array_get(&self->heads, version)->status = StackStatusHalted; } diff --git a/src/runtime/stack.h b/src/runtime/stack.h index 4704d90e..92a09b69 100644 --- a/src/runtime/stack.h +++ b/src/runtime/stack.h @@ -73,7 +73,8 @@ StackSliceArray ts_stack_pop_pending(Stack *, StackVersion); // Remove any all trees from the given version of the stack. StackSliceArray ts_stack_pop_all(Stack *, StackVersion); -// Get the number of tree nodes on the stack since the most recent error. +// Get the maximum number of tree nodes reachable from this version of the stack +// since the last error was detected. unsigned ts_stack_node_count_since_error(const Stack *, StackVersion); int ts_stack_dynamic_precedence(Stack *, StackVersion); @@ -86,14 +87,16 @@ void ts_stack_record_summary(Stack *, StackVersion, unsigned max_depth); // given version of the stack. StackSummary *ts_stack_get_summary(Stack *, StackVersion); +// Get the total cost of all errors on the given version of the stack. unsigned ts_stack_error_cost(const Stack *, StackVersion version); +// Determine whether the given two stack versions can be merged. bool ts_stack_merge(Stack *, StackVersion, StackVersion); +// Merge the given two stack versions if possible, returning true +// if they were successfully merged and false otherwise. bool ts_stack_can_merge(Stack *, StackVersion, StackVersion); -void ts_stack_force_merge(Stack *, StackVersion, StackVersion); - TSSymbol ts_stack_resume(Stack *, StackVersion); void ts_stack_pause(Stack *, StackVersion, TSSymbol); From 34349f9cbb19870f3fd1f4e76a11a8c38368b207 Mon Sep 17 00:00:00 2001 From: Max Brunsfeld Date: Mon, 2 Apr 2018 12:07:14 -0700 Subject: [PATCH 09/90] Put back limit on error handling based on accepted tree count --- src/runtime/parser.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/runtime/parser.c b/src/runtime/parser.c index 96f98cdf..9baf4ce2 100644 --- a/src/runtime/parser.c +++ b/src/runtime/parser.c @@ -1142,12 +1142,12 @@ static unsigned parser__condense_stack(Parser *self) { bool has_unpaused_version = false; for (StackVersion i = 0, n = ts_stack_version_count(self->stack); i < n; i++) { if (ts_stack_is_paused(self->stack, i)) { - if (!has_unpaused_version) { + if (!has_unpaused_version && self->accept_count < MAX_VERSION_COUNT) { LOG("resume version:%u", i); + min_error_cost = ts_stack_error_cost(self->stack, i); TSSymbol lookahead_symbol = ts_stack_resume(self->stack, i); parser__handle_error(self, i, lookahead_symbol); has_unpaused_version = true; - min_error_cost = ts_stack_error_cost(self->stack, i); } else { ts_stack_remove_version(self->stack, i); i--; From a6cf2e87e7b146fff6f326e624fad118ea5a5a81 Mon Sep 17 00:00:00 2001 From: Max Brunsfeld Date: Mon, 2 Apr 2018 13:58:20 -0700 Subject: [PATCH 10/90] Fix halt_on_error tests --- src/runtime/stack.c | 4 +++- test/integration/real_grammars.cc | 1 - test/runtime/document_test.cc | 6 +++++- 3 files changed, 8 insertions(+), 3 deletions(-) diff --git a/src/runtime/stack.c b/src/runtime/stack.c index 9186397f..885083e4 100644 --- a/src/runtime/stack.c +++ b/src/runtime/stack.c @@ -400,7 +400,9 @@ void ts_stack_set_last_external_token(Stack *self, StackVersion version, Tree *t unsigned ts_stack_error_cost(const Stack *self, StackVersion version) { StackHead *head = array_get(&self->heads, version); unsigned result = head->node->error_cost; - if (head->node->state == ERROR_STATE) result += ERROR_COST_PER_RECOVERY; + if (head->node->state == ERROR_STATE || head->status == StackStatusPaused) { + result += ERROR_COST_PER_RECOVERY; + } return result; } diff --git a/test/integration/real_grammars.cc b/test/integration/real_grammars.cc index 6cee7870..37465add 100644 --- a/test/integration/real_grammars.cc +++ b/test/integration/real_grammars.cc @@ -34,7 +34,6 @@ for (auto &language_name : test_languages) { TSDocument *document; const bool debug_graphs_enabled = getenv("TREE_SITTER_ENABLE_DEBUG_GRAPHS"); - before_each([&]() { record_alloc::start(); document = ts_document_new(); diff --git a/test/runtime/document_test.cc b/test/runtime/document_test.cc index 7c4bad6c..0be03657 100644 --- a/test/runtime/document_test.cc +++ b/test/runtime/document_test.cc @@ -22,6 +22,10 @@ describe("Document", [&]() { before_each([&]() { record_alloc::start(); document = ts_document_new(); + + if (getenv("TREE_SITTER_ENABLE_DEBUG_GRAPHS")) { + ts_document_print_debugging_graphs(document, true); + } }); after_each([&]() { @@ -434,7 +438,7 @@ describe("Document", [&]() { root = ts_document_root_node(document); assert_node_string_equals( root, - "(ERROR (number) (null) (UNEXPECTED 'e'))"); + "(ERROR (number) (null))"); AssertThat(ts_node_end_byte(root), Equals(input_string.size())); }); From 09be0b6ef5902ce114494b994a9118880e4f4762 Mon Sep 17 00:00:00 2001 From: Max Brunsfeld Date: Mon, 2 Apr 2018 18:04:26 -0700 Subject: [PATCH 11/90] Store trees' children in TreeArrays, not w/ separate pointer and length --- src/runtime/array.h | 8 +- src/runtime/get_changed_ranges.c | 8 +- src/runtime/node.c | 20 ++-- src/runtime/parser.c | 53 +++++----- src/runtime/reusable_node.h | 12 +-- src/runtime/stack.c | 4 +- src/runtime/tree.c | 163 +++++++++++++++---------------- src/runtime/tree.h | 72 +++++++------- test/helpers/tree_helpers.cc | 11 ++- test/helpers/tree_helpers.h | 2 +- test/runtime/tree_test.cc | 88 ++++++++--------- 11 files changed, 220 insertions(+), 221 deletions(-) diff --git a/src/runtime/array.h b/src/runtime/array.h index 0fa08d5a..e4e7ff0f 100644 --- a/src/runtime/array.h +++ b/src/runtime/array.h @@ -12,18 +12,18 @@ extern "C" { #include #include "runtime/alloc.h" -#define Array(T) \ - struct { \ - T *contents; \ +#define Array(T) \ + struct { \ uint32_t size; \ uint32_t capacity; \ + T *contents; \ } #define array_init(self) \ ((self)->size = 0, (self)->capacity = 0, (self)->contents = NULL) #define array_new() \ - { NULL, 0, 0 } + { 0, 0, NULL } #define array_get(self, index) \ (assert((uint32_t)index < (self)->size), &(self)->contents[index]) diff --git a/src/runtime/get_changed_ranges.c b/src/runtime/get_changed_ranges.c index 41fc1877..26211613 100644 --- a/src/runtime/get_changed_ranges.c +++ b/src/runtime/get_changed_ranges.c @@ -127,8 +127,8 @@ static bool iterator_descend(Iterator *self, uint32_t goal_position) { TreePathEntry entry = *array_back(&self->path); Length position = entry.position; uint32_t structural_child_index = 0; - for (uint32_t i = 0; i < entry.tree->child_count; i++) { - Tree *child = entry.tree->children[i]; + for (uint32_t i = 0; i < entry.tree->children.size; i++) { + Tree *child = entry.tree->children.contents[i]; Length child_left = length_add(position, child->padding); Length child_right = length_add(child_left, child->size); @@ -179,11 +179,11 @@ static void iterator_advance(Iterator *self) { Tree *parent = array_back(&self->path)->tree; uint32_t child_index = entry.child_index + 1; - if (parent->child_count > child_index) { + if (parent->children.size > child_index) { Length position = length_add(entry.position, ts_tree_total_size(entry.tree)); uint32_t structural_child_index = entry.structural_child_index; if (!entry.tree->extra) structural_child_index++; - Tree *next_child = parent->children[child_index]; + Tree *next_child = parent->children.contents[child_index]; array_push(&self->path, ((TreePathEntry){ .tree = next_child, diff --git a/src/runtime/node.c b/src/runtime/node.c index 1f7487cb..c825a104 100644 --- a/src/runtime/node.c +++ b/src/runtime/node.c @@ -39,7 +39,7 @@ static inline bool ts_node__is_relevant(TSNode self, bool include_anonymous) { static inline uint32_t ts_node__relevant_child_count(TSNode self, bool include_anonymous) { const Tree *tree = ts_node__tree(self); - if (tree->child_count > 0) { + if (tree->children.size > 0) { if (include_anonymous) { return tree->visible_child_count; } else { @@ -61,7 +61,7 @@ static inline TSNode ts_node__direct_parent(TSNode self, uint32_t *index) { } static inline TSNode ts_node__direct_child(TSNode self, uint32_t i) { - const Tree *child_tree = ts_node__tree(self)->children[i]; + const Tree *child_tree = ts_node__tree(self)->children.contents[i]; return ts_node_make( child_tree, ts_node__offset_byte(self) + child_tree->context.offset.bytes, @@ -78,7 +78,7 @@ static inline TSNode ts_node__child(TSNode self, uint32_t child_index, did_descend = false; uint32_t index = 0; - for (uint32_t i = 0; i < ts_node__tree(result)->child_count; i++) { + for (uint32_t i = 0; i < ts_node__tree(result)->children.size; i++) { TSNode child = ts_node__direct_child(result, i); if (ts_node__is_relevant(child, include_anonymous)) { if (index == child_index) @@ -134,7 +134,7 @@ static inline TSNode ts_node__next_sibling(TSNode self, bool include_anonymous) if (!result.data) break; - for (uint32_t i = index + 1; i < ts_node__tree(result)->child_count; i++) { + for (uint32_t i = index + 1; i < ts_node__tree(result)->children.size; i++) { TSNode child = ts_node__direct_child(result, i); if (ts_node__is_relevant(child, include_anonymous)) return child; @@ -160,7 +160,7 @@ static inline TSNode ts_node__first_child_for_byte(TSNode self, uint32_t goal, while (did_descend) { did_descend = false; - for (uint32_t i = 0; i < ts_node__tree(node)->child_count; i++) { + for (uint32_t i = 0; i < ts_node__tree(node)->children.size; i++) { TSNode child = ts_node__direct_child(node, i); if (ts_node_end_byte(child) > goal) { if (ts_node__is_relevant(child, include_anonymous)) { @@ -187,7 +187,7 @@ static inline TSNode ts_node__descendant_for_byte_range(TSNode self, uint32_t mi while (did_descend) { did_descend = false; - for (uint32_t i = 0, n = ts_node__tree(node)->child_count; i < n; i++) { + for (uint32_t i = 0, n = ts_node__tree(node)->children.size; i < n; i++) { TSNode child = ts_node__direct_child(node, i); if (ts_node_end_byte(child) > max) { if (ts_node_start_byte(child) > min) break; @@ -214,7 +214,7 @@ static inline TSNode ts_node__descendant_for_point_range(TSNode self, TSPoint mi while (did_descend) { did_descend = false; - for (uint32_t i = 0, n = ts_node__tree(node)->child_count; i < n; i++) { + for (uint32_t i = 0, n = ts_node__tree(node)->children.size; i < n; i++) { TSNode child = ts_node__direct_child(node, i); const Tree *child_tree = ts_node__tree(child); if (i > 0) start_position = point_add(start_position, child_tree->padding.extent); @@ -318,7 +318,7 @@ uint32_t ts_node_child_index(TSNode self) { uint32_t index = tree->context.index; if (!parent) return UINT32_MAX; for (uint32_t i = 0; i < index; i++) { - Tree *child = parent->children[i]; + Tree *child = parent->children.contents[i]; result += child->visible ? 1 : child->visible_child_count; } if (parent->visible) break; @@ -338,7 +338,7 @@ TSNode ts_node_named_child(TSNode self, uint32_t child_index) { uint32_t ts_node_child_count(TSNode self) { const Tree *tree = ts_node__tree(self); - if (tree->child_count > 0) { + if (tree->children.size > 0) { return tree->visible_child_count; } else { return 0; @@ -347,7 +347,7 @@ uint32_t ts_node_child_count(TSNode self) { uint32_t ts_node_named_child_count(TSNode self) { const Tree *tree = ts_node__tree(self); - if (tree->child_count > 0) { + if (tree->children.size > 0) { return tree->named_child_count; } else { return 0; diff --git a/src/runtime/parser.c b/src/runtime/parser.c index 9baf4ce2..a377c42a 100644 --- a/src/runtime/parser.c +++ b/src/runtime/parser.c @@ -86,9 +86,9 @@ static bool parser__breakdown_top_of_stack(Parser *self, StackVersion version) { TSStateId state = ts_stack_state(self->stack, slice.version); Tree *parent = *array_front(&slice.trees); - for (uint32_t j = 0; j < parent->child_count; j++) { - Tree *child = parent->children[j]; - pending = child->child_count > 0; + for (uint32_t j = 0; j < parent->children.size; j++) { + Tree *child = parent->children.contents[j]; + pending = child->children.size > 0; if (child->symbol == ts_builtin_sym_error) { state = ERROR_STATE; @@ -120,7 +120,7 @@ static void parser__breakdown_lookahead(Parser *self, Tree **lookahead, TSStateId state, ReusableNode *reusable_node) { bool did_break_down = false; - while (reusable_node->tree->child_count > 0 && reusable_node->tree->parse_state != state) { + while (reusable_node->tree->children.size > 0 && reusable_node->tree->parse_state != state) { LOG("state_mismatch sym:%s", SYM_NAME(reusable_node->tree->symbol)); reusable_node_breakdown(reusable_node); did_break_down = true; @@ -447,7 +447,7 @@ static Tree *parser__get_lookahead(Parser *self, StackVersion version, TSStateId reason = "is_missing"; } else if (result->fragile_left || result->fragile_right) { reason = "is_fragile"; - } else if (self->in_ambiguity && result->child_count) { + } else if (self->in_ambiguity && result->children.size) { reason = "in_ambiguity"; } @@ -555,7 +555,7 @@ static void parser__shift(Parser *self, StackVersion version, TSStateId state, ts_tree_retain(lookahead); } - bool is_pending = lookahead->child_count > 0; + bool is_pending = lookahead->children.size > 0; ts_stack_push(self->stack, version, lookahead, is_pending, state); if (lookahead->has_external_tokens) { ts_stack_set_last_external_token( @@ -564,10 +564,10 @@ static void parser__shift(Parser *self, StackVersion version, TSStateId state, } } -static bool parser__replace_children(Parser *self, Tree *tree, Tree **children, uint32_t count) { +static bool parser__replace_children(Parser *self, Tree *tree, TreeArray *children) { self->scratch_tree = *tree; - self->scratch_tree.child_count = 0; - ts_tree_set_children(&self->scratch_tree, count, children, self->language); + self->scratch_tree.children.size = 0; + ts_tree_set_children(&self->scratch_tree, children, self->language); if (parser__select_tree(self, tree, &self->scratch_tree)) { *tree = self->scratch_tree; return true; @@ -589,13 +589,13 @@ static StackSliceArray parser__reduce(Parser *self, StackVersion version, TSSymb // Extra tokens on top of the stack should not be included in this new parent // node. They will be re-pushed onto the stack after the parent node is // created and pushed. - uint32_t child_count = slice.trees.size; - while (child_count > 0 && slice.trees.contents[child_count - 1]->extra) { - child_count--; + TreeArray children = slice.trees; + while (children.size > 0 && children.contents[children.size - 1]->extra) { + children.size--; } Tree *parent = ts_tree_make_node(&self->tree_pool, - symbol, child_count, slice.trees.contents, alias_sequence_id, self->language + symbol, &children, alias_sequence_id, self->language ); // This pop operation may have caused multiple stack versions to collapse @@ -607,12 +607,12 @@ static StackSliceArray parser__reduce(Parser *self, StackVersion version, TSSymb if (next_slice.version != slice.version) break; i++; - uint32_t child_count = next_slice.trees.size; - while (child_count > 0 && next_slice.trees.contents[child_count - 1]->extra) { - child_count--; + TreeArray children = next_slice.trees; + while (children.size > 0 && children.contents[children.size - 1]->extra) { + children.size--; } - if (parser__replace_children(self, parent, next_slice.trees.contents, child_count)) { + if (parser__replace_children(self, parent, &children)) { ts_tree_array_delete(&self->tree_pool, &slice.trees); slice = next_slice; } else { @@ -636,9 +636,8 @@ static StackSliceArray parser__reduce(Parser *self, StackVersion version, TSSymb // Push the parent node onto the stack, along with any extra tokens that // were previously on top of the stack. ts_stack_push(self->stack, slice.version, parent, false, next_state); - for (uint32_t j = parent->child_count; j < slice.trees.size; j++) { - Tree *tree = slice.trees.contents[j]; - ts_stack_push(self->stack, slice.version, tree, false, next_state); + for (uint32_t j = parent->children.size; j < slice.trees.size; j++) { + ts_stack_push(self->stack, slice.version, slice.trees.contents[j], false, next_state); } } @@ -690,12 +689,12 @@ static void parser__accept(Parser *self, StackVersion version, Tree *child = trees.contents[j]; if (!child->extra) { root = ts_tree_make_copy(&self->tree_pool, child); - root->child_count = 0; - for (uint32_t k = 0; k < child->child_count; k++) { - ts_tree_retain(child->children[k]); + root->children.size = 0; + for (uint32_t k = 0; k < child->children.size; k++) { + ts_tree_retain(child->children.contents[k]); } - array_splice(&trees, j, 1, child->child_count, child->children); - ts_tree_set_children(root, trees.size, trees.contents, self->language); + array_splice(&trees, j, 1, child->children.size, child->children.contents); + ts_tree_set_children(root, &trees, self->language); ts_tree_release(&self->tree_pool, child); break; } @@ -1020,7 +1019,7 @@ static void parser__advance(Parser *self, StackVersion version, ReusableNode *re LOG("shift state:%u", next_state); } - if (lookahead->child_count > 0) { + if (lookahead->children.size > 0) { parser__breakdown_lookahead(self, &lookahead, state, reusable_node); next_state = ts_language_next_state(self->language, state, lookahead->symbol); } @@ -1052,7 +1051,7 @@ static void parser__advance(Parser *self, StackVersion version, ReusableNode *re } case TSParseActionTypeRecover: { - while (lookahead->child_count > 0) { + while (lookahead->children.size > 0) { parser__breakdown_lookahead(self, &lookahead, state, reusable_node); } parser__recover(self, version, lookahead); diff --git a/src/runtime/reusable_node.h b/src/runtime/reusable_node.h index 87d20dbf..04b9af7e 100644 --- a/src/runtime/reusable_node.h +++ b/src/runtime/reusable_node.h @@ -20,8 +20,8 @@ static inline void reusable_node_pop(ReusableNode *self) { while (self->tree) { Tree *parent = self->tree->context.parent; uint32_t next_index = self->tree->context.index + 1; - if (parent && parent->child_count > next_index) { - self->tree = parent->children[next_index]; + if (parent && parent->children.size > next_index) { + self->tree = parent->children.contents[next_index]; return; } self->tree = parent; @@ -30,17 +30,17 @@ static inline void reusable_node_pop(ReusableNode *self) { static inline ReusableNode reusable_node_after_leaf(const ReusableNode *self) { ReusableNode result = *self; - while (result.tree->child_count > 0) - result.tree = result.tree->children[0]; + while (result.tree->children.size > 0) + result.tree = result.tree->children.contents[0]; reusable_node_pop(&result); return result; } static inline bool reusable_node_breakdown(ReusableNode *self) { - if (self->tree->child_count == 0) { + if (self->tree->children.size == 0) { return false; } else { - self->tree = self->tree->children[0]; + self->tree = self->tree->children.contents[0]; return true; } } diff --git a/src/runtime/stack.c b/src/runtime/stack.c index 885083e4..aa2eaed4 100644 --- a/src/runtime/stack.c +++ b/src/runtime/stack.c @@ -147,7 +147,7 @@ static StackNode *stack_node_new(StackNode *previous_node, Tree *tree, bool is_p if (state == ERROR_STATE) { node->error_cost += - ERROR_COST_PER_SKIPPED_TREE * ((tree->visible || tree->child_count == 0) ? 1 : tree->visible_child_count) + + ERROR_COST_PER_SKIPPED_TREE * ((tree->visible || tree->children.size == 0) ? 1 : tree->visible_child_count) + ERROR_COST_PER_SKIPPED_CHAR * tree->size.bytes + ERROR_COST_PER_SKIPPED_LINE * tree->size.extent.row; if (previous_node->links[0].tree) { @@ -173,7 +173,7 @@ static bool stack__tree_is_equivalent(const Tree *left, const Tree *right) { right && left->symbol == right->symbol && ((left->error_cost > 0 && right->error_cost > 0) || - (left->child_count == 0 && right->child_count == 0 && + (left->children.size == 0 && right->children.size == 0 && left->padding.bytes == right->padding.bytes && left->size.bytes == right->size.bytes && left->extra == right->extra && diff --git a/src/runtime/tree.c b/src/runtime/tree.c index f88895d2..06828f3b 100644 --- a/src/runtime/tree.c +++ b/src/runtime/tree.c @@ -169,8 +169,6 @@ Tree *ts_tree_make_leaf(TreePool *pool, TSSymbol symbol, Length padding, Length .ref_count = 1, .symbol = symbol, .size = size, - .child_count = 0, - .children = NULL, .visible_child_count = 0, .named_child_count = 0, .alias_sequence_id = 0, @@ -207,50 +205,50 @@ Tree *ts_tree_make_copy(TreePool *pool, Tree *self) { static void ts_tree__compress(Tree *self, unsigned count, const TSLanguage *language) { Tree *tree = self; for (unsigned i = 0; i < count; i++) { - if (tree->ref_count > 1 || tree->child_count != 2) break; + if (tree->ref_count > 1 || tree->children.size != 2) break; - Tree *child = tree->children[0]; + Tree *child = tree->children.contents[0]; if ( child->ref_count > 1 || - child->child_count != 2 || + child->children.size != 2 || child->symbol != tree->symbol ) break; - Tree *grandchild = child->children[0]; + Tree *grandchild = child->children.contents[0]; if ( grandchild->ref_count > 1 || - grandchild->child_count != 2 || + grandchild->children.size != 2 || grandchild->symbol != tree->symbol ) break; - tree->children[0] = grandchild; + tree->children.contents[0] = grandchild; grandchild->context.parent = tree; grandchild->context.index = -1; - child->children[0] = grandchild->children[1]; - child->children[0]->context.parent = child; - child->children[0]->context.index = -1; + child->children.contents[0] = grandchild->children.contents[1]; + child->children.contents[0]->context.parent = child; + child->children.contents[0]->context.index = -1; - grandchild->children[1] = child; - grandchild->children[1]->context.parent = grandchild; - grandchild->children[1]->context.index = -1; + grandchild->children.contents[1] = child; + grandchild->children.contents[1]->context.parent = grandchild; + grandchild->children.contents[1]->context.index = -1; tree = grandchild; } while (tree != self) { tree = tree->context.parent; - Tree *child = tree->children[0]; - Tree *grandchild = child->children[1]; - ts_tree_set_children(grandchild, 2, grandchild->children, language); - ts_tree_set_children(child, 2, child->children, language); - ts_tree_set_children(tree, 2, tree->children, language); + Tree *child = tree->children.contents[0]; + Tree *grandchild = child->children.contents[1]; + ts_tree_set_children(grandchild, &grandchild->children, language); + ts_tree_set_children(child, &child->children, language); + ts_tree_set_children(tree, &tree->children, language); } } void ts_tree__balance(Tree *self, const TSLanguage *language) { - if (self->children[0]->repeat_depth > self->children[1]->repeat_depth) { - unsigned n = self->children[0]->repeat_depth - self->children[1]->repeat_depth; + if (self->children.contents[0]->repeat_depth > self->children.contents[1]->repeat_depth) { + unsigned n = self->children.contents[0]->repeat_depth - self->children.contents[1]->repeat_depth; for (unsigned i = n / 2; i > 0; i /= 2) { ts_tree__compress(self, i, language); n -= i; @@ -273,8 +271,8 @@ void ts_tree_assign_parents(Tree *self, TreePool *pool, const TSLanguage *langua const TSSymbol *alias_sequence = ts_language_alias_sequence(language, tree->alias_sequence_id); uint32_t non_extra_index = 0; bool earlier_child_was_changed = false; - for (uint32_t i = 0; i < tree->child_count; i++) { - Tree *child = tree->children[i]; + for (uint32_t i = 0; i < tree->children.size; i++) { + Tree *child = tree->children.contents[i]; if (earlier_child_was_changed || child->context.parent != tree || child->context.index != i) { earlier_child_was_changed = true; child->context.parent = tree; @@ -296,12 +294,12 @@ void ts_tree_assign_parents(Tree *self, TreePool *pool, const TSLanguage *langua } } -void ts_tree_set_children(Tree *self, uint32_t child_count, Tree **children, - const TSLanguage *language) { - if (self->child_count > 0 && children != self->children) ts_free(self->children); +void ts_tree_set_children(Tree *self, TreeArray *children, const TSLanguage *language) { + if (self->children.size > 0 && children->contents != self->children.contents) { + array_delete(&self->children); + } - self->children = children; - self->child_count = child_count; + self->children = *children; self->named_child_count = 0; self->visible_child_count = 0; self->error_cost = 0; @@ -313,8 +311,8 @@ void ts_tree_set_children(Tree *self, uint32_t child_count, Tree **children, uint32_t non_extra_index = 0; const TSSymbol *alias_sequence = ts_language_alias_sequence(language, self->alias_sequence_id); - for (uint32_t i = 0; i < child_count; i++) { - Tree *child = children[i]; + for (uint32_t i = 0; i < self->children.size; i++) { + Tree *child = self->children.contents[i]; if (i == 0) { self->padding = child->padding; @@ -338,7 +336,7 @@ void ts_tree_set_children(Tree *self, uint32_t child_count, Tree **children, } else if (child->visible) { self->visible_child_count++; if (child->named) self->named_child_count++; - } else if (child->child_count > 0) { + } else if (child->children.size > 0) { self->visible_child_count += child->visible_child_count; self->named_child_count += child->named_child_count; } @@ -357,62 +355,55 @@ void ts_tree_set_children(Tree *self, uint32_t child_count, Tree **children, self->error_cost += ERROR_COST_PER_RECOVERY + ERROR_COST_PER_SKIPPED_CHAR * self->size.bytes + ERROR_COST_PER_SKIPPED_LINE * self->size.extent.row; - for (uint32_t i = 0; i < child_count; i++) { - if (!self->children[i]->extra) { + for (uint32_t i = 0; i < self->children.size; i++) { + if (!self->children.contents[i]->extra) { self->error_cost += ERROR_COST_PER_SKIPPED_TREE; } } } - if (child_count > 0) { - self->first_leaf = children[0]->first_leaf; - if (children[0]->fragile_left) { - self->fragile_left = true; - } - if (children[child_count - 1]->fragile_right) { - self->fragile_right = true; - } + if (self->children.size > 0) { + Tree *first_child = self->children.contents[0]; + Tree *last_child = self->children.contents[self->children.size - 1]; + self->first_leaf = first_child->first_leaf; + if (first_child->fragile_left) self->fragile_left = true; + if (last_child->fragile_right) self->fragile_right = true; if ( - self->child_count == 2 && + self->children.size == 2 && !self->visible && !self->named && - self->children[0]->symbol == self->symbol && - self->children[1]->symbol == self->symbol + first_child->symbol == self->symbol && + last_child->symbol == self->symbol ) { - if (self->children[0]->repeat_depth > self->children[1]->repeat_depth) { - self->repeat_depth = self->children[0]->repeat_depth + 1; + if (first_child->repeat_depth > last_child->repeat_depth) { + self->repeat_depth = first_child->repeat_depth + 1; } else { - self->repeat_depth = self->children[1]->repeat_depth + 1; + self->repeat_depth = last_child->repeat_depth + 1; } } } } -Tree *ts_tree_make_node(TreePool *pool, TSSymbol symbol, uint32_t child_count, Tree **children, +Tree *ts_tree_make_node(TreePool *pool, TSSymbol symbol, TreeArray *children, unsigned alias_sequence_id, const TSLanguage *language) { Tree *result = ts_tree_make_leaf(pool, symbol, length_zero(), length_zero(), language); result->alias_sequence_id = alias_sequence_id; - ts_tree_set_children(result, child_count, children, language); + ts_tree_set_children(result, children, language); return result; } Tree *ts_tree_make_error_node(TreePool *pool, TreeArray *children, const TSLanguage *language) { for (uint32_t i = 0; i < children->size; i++) { Tree *child = children->contents[i]; - if (child->symbol == ts_builtin_sym_error && child->child_count > 0) { - array_splice(children, i, 1, child->child_count, child->children); - i += child->child_count - 1; - for (uint32_t j = 0; j < child->child_count; j++) - ts_tree_retain(child->children[j]); + if (child->symbol == ts_builtin_sym_error && child->children.size > 0) { + array_splice(children, i, 1, child->children.size, child->children.contents); + i += child->children.size - 1; + for (uint32_t j = 0; j < child->children.size; j++) + ts_tree_retain(child->children.contents[j]); ts_tree_release(pool, child); } } - Tree *result = ts_tree_make_node( - pool, ts_builtin_sym_error, - children->size, children->contents, - 0, language - ); - + Tree *result = ts_tree_make_node(pool, ts_builtin_sym_error, children, 0, language); result->fragile_left = true; result->fragile_right = true; return result; @@ -439,11 +430,11 @@ void ts_tree_release(TreePool *pool, Tree *self) { assert(tree->ref_count > 0); tree->ref_count--; if (tree->ref_count == 0) { - if (tree->child_count > 0) { - for (uint32_t i = 0; i < tree->child_count; i++) { - array_push(&pool->tree_stack, tree->children[i]); + if (tree->children.size > 0) { + for (uint32_t i = 0; i < tree->children.size; i++) { + array_push(&pool->tree_stack, tree->children.contents[i]); } - ts_free(tree->children); + array_delete(&tree->children); } else if (tree->has_external_tokens) { ts_external_token_state_delete(&tree->external_token_state); } @@ -484,12 +475,12 @@ bool ts_tree_eq(const Tree *self, const Tree *other) { if (self->padding.bytes != other->padding.bytes) return false; if (self->size.bytes != other->size.bytes) return false; if (self->symbol == ts_builtin_sym_error) return self->lookahead_char == other->lookahead_char; - if (self->child_count != other->child_count) return false; + if (self->children.size != other->children.size) return false; if (self->visible_child_count != other->visible_child_count) return false; if (self->named_child_count != other->named_child_count) return false; - for (uint32_t i = 0; i < self->child_count; i++) { - if (!ts_tree_eq(self->children[i], other->children[i])) { + for (uint32_t i = 0; i < self->children.size; i++) { + if (!ts_tree_eq(self->children.contents[i], other->children.contents[i])) { return false; } } @@ -501,13 +492,13 @@ int ts_tree_compare(const Tree *left, const Tree *right) { return -1; if (right->symbol < left->symbol) return 1; - if (left->child_count < right->child_count) + if (left->children.size < right->children.size) return -1; - if (right->child_count < left->child_count) + if (right->children.size < left->children.size) return 1; - for (uint32_t i = 0; i < left->child_count; i++) { - Tree *left_child = left->children[i]; - Tree *right_child = right->children[i]; + for (uint32_t i = 0; i < left->children.size; i++) { + Tree *left_child = left->children.contents[i]; + Tree *right_child = right->children.contents[i]; switch (ts_tree_compare(left_child, right_child)) { case -1: return -1; @@ -527,10 +518,10 @@ static inline long min_byte(long a, long b) { bool ts_tree_invalidate_lookahead(Tree *self, uint32_t edit_byte_offset) { if (edit_byte_offset >= self->bytes_scanned) return false; self->has_changes = true; - if (self->child_count > 0) { + if (self->children.size > 0) { uint32_t child_start_byte = 0; - for (uint32_t i = 0; i < self->child_count; i++) { - Tree *child = self->children[i]; + for (uint32_t i = 0; i < self->children.size; i++) { + Tree *child = self->children.contents[i]; if (child_start_byte > edit_byte_offset) break; ts_tree_invalidate_lookahead(child, edit_byte_offset - child_start_byte); child_start_byte += ts_tree_total_bytes(child); @@ -581,8 +572,8 @@ void ts_tree_edit(Tree *self, const TSInputEdit *edit) { long remaining_bytes_to_delete = 0; TSPoint remaining_extent_to_delete = {0, 0}; Length child_left, child_right = length_zero(); - for (uint32_t i = 0; i < self->child_count; i++) { - Tree *child = self->children[i]; + for (uint32_t i = 0; i < self->children.size; i++) { + Tree *child = self->children.contents[i]; child_left = child_right; child_right = length_add(child_left, ts_tree_total_size(child)); @@ -628,9 +619,9 @@ void ts_tree_edit(Tree *self, const TSInputEdit *edit) { Tree *ts_tree_last_external_token(Tree *tree) { if (!tree->has_external_tokens) return NULL; - while (tree->child_count > 0) { - for (uint32_t i = tree->child_count - 1; i + 1 > 0; i--) { - Tree *child = tree->children[i]; + while (tree->children.size > 0) { + for (uint32_t i = tree->children.size - 1; i + 1 > 0; i--) { + Tree *child = tree->children.contents[i]; if (child->has_external_tokens) { tree = child; break; @@ -676,7 +667,7 @@ static size_t ts_tree__write_to_string(const Tree *self, const TSLanguage *langu } if (visible) { - if (self->symbol == ts_builtin_sym_error && self->child_count == 0 && self->size.bytes > 0) { + if (self->symbol == ts_builtin_sym_error && self->children.size == 0 && self->size.bytes > 0) { cursor += snprintf(*writer, limit, "(UNEXPECTED "); cursor += ts_tree__write_char_to_string(*writer, limit, self->lookahead_char); } else if (self->is_missing) { @@ -688,8 +679,8 @@ static size_t ts_tree__write_to_string(const Tree *self, const TSLanguage *langu } } - for (uint32_t i = 0; i < self->child_count; i++) { - Tree *child = self->children[i]; + for (uint32_t i = 0; i < self->children.size; i++) { + Tree *child = self->children.contents[i]; cursor += ts_tree__write_to_string(child, language, *writer, limit, false, include_all); } @@ -711,7 +702,7 @@ void ts_tree__print_dot_graph(const Tree *self, uint32_t byte_offset, TSSymbol symbol = self->context.alias_symbol ? self->context.alias_symbol : self->symbol; fprintf(f, "tree_%p [label=\"%s\"", self, ts_language_symbol_name(language, symbol)); - if (self->child_count == 0) + if (self->children.size == 0) fprintf(f, ", shape=plaintext"); if (self->extra) fprintf(f, ", fontcolor=gray"); @@ -719,8 +710,8 @@ void ts_tree__print_dot_graph(const Tree *self, uint32_t byte_offset, fprintf(f, ", tooltip=\"address:%p\nrange:%u - %u\nstate:%d\nerror-cost:%u\nrepeat-depth:%u\"]\n", self, byte_offset, byte_offset + ts_tree_total_bytes(self), self->parse_state, self->error_cost, self->repeat_depth); - for (uint32_t i = 0; i < self->child_count; i++) { - const Tree *child = self->children[i]; + for (uint32_t i = 0; i < self->children.size; i++) { + const Tree *child = self->children.contents[i]; ts_tree__print_dot_graph(child, byte_offset, language, f); fprintf(f, "tree_%p -> tree_%p [tooltip=%u]\n", self, child, i); byte_offset += ts_tree_total_bytes(child); diff --git a/src/runtime/tree.h b/src/runtime/tree.h index ad104894..09dae3dd 100644 --- a/src/runtime/tree.h +++ b/src/runtime/tree.h @@ -17,12 +17,16 @@ extern TSStateId TS_TREE_STATE_NONE; typedef struct { union { char *long_data; - char short_data[sizeof(char *) + sizeof(unsigned)]; + char short_data[sizeof(char *) + sizeof(uint32_t)]; }; - unsigned length; + uint32_t length; } TSExternalTokenState; -typedef struct Tree { +typedef struct Tree Tree; + +typedef Array(Tree *) TreeArray; + +struct Tree { struct { struct Tree *parent; uint32_t index; @@ -31,35 +35,16 @@ typedef struct Tree { bool alias_is_named : 1; } context; - uint32_t child_count; - union { - struct { - struct Tree **children; - uint32_t visible_child_count; - uint32_t named_child_count; - unsigned short alias_sequence_id; - }; - TSExternalTokenState external_token_state; - int32_t lookahead_char; - }; - Length padding; Length size; - uint32_t bytes_scanned; - - TSSymbol symbol; - TSStateId parse_state; - unsigned error_cost; - unsigned node_count; - unsigned repeat_depth; - - struct { - TSSymbol symbol; - TSLexMode lex_mode; - } first_leaf; - uint32_t ref_count; - int dynamic_precedence; + uint32_t bytes_scanned; + uint32_t error_cost; + uint32_t node_count; + uint32_t repeat_depth; + uint32_t child_count; + int32_t dynamic_precedence; + bool visible : 1; bool named : 1; bool extra : 1; @@ -68,9 +53,30 @@ typedef struct Tree { bool has_changes : 1; bool has_external_tokens : 1; bool is_missing : 1; -} Tree; + TSSymbol symbol; + TSStateId parse_state; + struct { + TSSymbol symbol; + TSLexMode lex_mode; + } first_leaf; -typedef Array(Tree *) TreeArray; + union { + struct { + TreeArray children; + uint32_t visible_child_count; + uint32_t named_child_count; + uint16_t alias_sequence_id; + }; + struct { + uint32_t _2; + TSExternalTokenState external_token_state; + }; + struct { + uint32_t _1; + int32_t lookahead_char; + }; + }; +}; typedef struct { TreeArray free_trees; @@ -93,7 +99,7 @@ Tree *ts_tree_pool_allocate(TreePool *); void ts_tree_pool_free(TreePool *, Tree *); Tree *ts_tree_make_leaf(TreePool *, TSSymbol, Length, Length, const TSLanguage *); -Tree *ts_tree_make_node(TreePool *, TSSymbol, uint32_t, Tree **, unsigned, const TSLanguage *); +Tree *ts_tree_make_node(TreePool *, TSSymbol, TreeArray *, unsigned, const TSLanguage *); Tree *ts_tree_make_copy(TreePool *, Tree *child); Tree *ts_tree_make_error_node(TreePool *, TreeArray *, const TSLanguage *); Tree *ts_tree_make_error(TreePool *, Length, Length, int32_t, const TSLanguage *); @@ -104,7 +110,7 @@ bool ts_tree_eq(const Tree *tree1, const Tree *tree2); int ts_tree_compare(const Tree *tree1, const Tree *tree2); uint32_t ts_tree_start_column(const Tree *self); uint32_t ts_tree_end_column(const Tree *self); -void ts_tree_set_children(Tree *, uint32_t, Tree **, const TSLanguage *); +void ts_tree_set_children(Tree *, TreeArray *, const TSLanguage *); void ts_tree_assign_parents(Tree *, TreePool *, const TSLanguage *); void ts_tree_edit(Tree *, const TSInputEdit *edit); char *ts_tree_string(const Tree *, const TSLanguage *, bool include_all); diff --git a/test/helpers/tree_helpers.cc b/test/helpers/tree_helpers.cc index abd6bc10..5142ca8d 100644 --- a/test/helpers/tree_helpers.cc +++ b/test/helpers/tree_helpers.cc @@ -16,11 +16,14 @@ const char *symbol_names[24] = { "twenty-two", "twenty-three" }; -Tree ** tree_array(std::vector trees) { - Tree ** result = (Tree **)calloc(trees.size(), sizeof(Tree *)); +TreeArray *tree_array(std::vector trees) { + static TreeArray result; + result.capacity = trees.size(); + result.size = trees.size(); + result.contents = (Tree **)calloc(trees.size(), sizeof(Tree *)); for (size_t i = 0; i < trees.size(); i++) - result[i] = trees[i]; - return result; + result.contents[i] = trees[i]; + return &result; } ostream &operator<<(std::ostream &stream, const Tree *tree) { diff --git a/test/helpers/tree_helpers.h b/test/helpers/tree_helpers.h index 19ae2c70..c28dcd98 100644 --- a/test/helpers/tree_helpers.h +++ b/test/helpers/tree_helpers.h @@ -6,7 +6,7 @@ #include extern const char *symbol_names[24]; -Tree ** tree_array(std::vector trees); +TreeArray *tree_array(std::vector trees); std::ostream &operator<<(std::ostream &stream, const Tree *tree); std::ostream &operator<<(std::ostream &stream, const TSNode &node); diff --git a/test/runtime/tree_test.cc b/test/runtime/tree_test.cc index 96f485f7..8669b6c1 100644 --- a/test/runtime/tree_test.cc +++ b/test/runtime/tree_test.cc @@ -7,11 +7,11 @@ void assert_consistent(const Tree *tree) { if (tree->child_count == 0) return; - AssertThat(tree->children[0]->padding, Equals(tree->padding)); + AssertThat(tree->children.contents[0]->padding, Equals(tree->padding)); Length total_children_size = length_zero(); - for (size_t i = 0; i < tree->child_count; i++) { - Tree *child = tree->children[i]; + for (size_t i = 0; i < tree->children.size; i++) { + Tree *child = tree->children.contents[i]; AssertThat(child->context.offset, Equals(total_children_size)); assert_consistent(child); total_children_size = length_add(total_children_size, ts_tree_total_size(child)); @@ -86,7 +86,7 @@ describe("Tree", []() { ts_tree_retain(tree1); ts_tree_retain(tree2); - parent1 = ts_tree_make_node(&pool, symbol3, 2, tree_array({ + parent1 = ts_tree_make_node(&pool, symbol3, tree_array({ tree1, tree2, }), 0, &language); @@ -114,7 +114,7 @@ describe("Tree", []() { ts_tree_retain(tree1); ts_tree_retain(tree2); - parent = ts_tree_make_node(&pool, symbol3, 2, tree_array({ + parent = ts_tree_make_node(&pool, symbol3, tree_array({ tree1, tree2, }), 0, &language); @@ -138,7 +138,7 @@ describe("Tree", []() { ts_tree_retain(tree1); ts_tree_retain(tree2); - parent = ts_tree_make_node(&pool, symbol3, 2, tree_array({ + parent = ts_tree_make_node(&pool, symbol3, tree_array({ tree1, tree2, }), 0, &language); @@ -162,7 +162,7 @@ describe("Tree", []() { ts_tree_retain(tree1); ts_tree_retain(tree2); - parent = ts_tree_make_node(&pool, symbol3, 2, tree_array({ + parent = ts_tree_make_node(&pool, symbol3, tree_array({ tree1, tree2, }), 0, &language); @@ -183,7 +183,7 @@ describe("Tree", []() { Tree *tree = nullptr; before_each([&]() { - tree = ts_tree_make_node(&pool, symbol1, 3, tree_array({ + tree = ts_tree_make_node(&pool, symbol1, tree_array({ ts_tree_make_leaf(&pool, symbol2, {2, {0, 2}}, {3, {0, 3}}, &language), ts_tree_make_leaf(&pool, symbol3, {2, {0, 2}}, {3, {0, 3}}, &language), ts_tree_make_leaf(&pool, symbol4, {2, {0, 2}}, {3, {0, 3}}, &language), @@ -213,13 +213,13 @@ describe("Tree", []() { AssertThat(tree->padding, Equals({3, {0, 3}})); AssertThat(tree->size, Equals({13, {0, 13}})); - AssertThat(tree->children[0]->has_changes, IsTrue()); - AssertThat(tree->children[0]->padding, Equals({3, {0, 3}})); - AssertThat(tree->children[0]->size, Equals({3, {0, 3}})); + AssertThat(tree->children.contents[0]->has_changes, IsTrue()); + AssertThat(tree->children.contents[0]->padding, Equals({3, {0, 3}})); + AssertThat(tree->children.contents[0]->size, Equals({3, {0, 3}})); - AssertThat(tree->children[1]->has_changes, IsFalse()); - AssertThat(tree->children[1]->padding, Equals({2, {0, 2}})); - AssertThat(tree->children[1]->size, Equals({3, {0, 3}})); + AssertThat(tree->children.contents[1]->has_changes, IsFalse()); + AssertThat(tree->children.contents[1]->padding, Equals({2, {0, 2}})); + AssertThat(tree->children.contents[1]->size, Equals({3, {0, 3}})); }); }); @@ -239,9 +239,9 @@ describe("Tree", []() { AssertThat(tree->padding, Equals({5, {0, 5}})); AssertThat(tree->size, Equals({11, {0, 11}})); - AssertThat(tree->children[0]->has_changes, IsTrue()); - AssertThat(tree->children[0]->padding, Equals({5, {0, 5}})); - AssertThat(tree->children[0]->size, Equals({1, {0, 1}})); + AssertThat(tree->children.contents[0]->has_changes, IsTrue()); + AssertThat(tree->children.contents[0]->padding, Equals({5, {0, 5}})); + AssertThat(tree->children.contents[0]->size, Equals({1, {0, 1}})); }); }); @@ -263,11 +263,11 @@ describe("Tree", []() { AssertThat(tree->padding, Equals({4, {0, 4}})); AssertThat(tree->size, Equals({13, {0, 13}})); - AssertThat(tree->children[0]->has_changes, IsTrue()); - AssertThat(tree->children[0]->padding, Equals({4, {0, 4}})); - AssertThat(tree->children[0]->size, Equals({3, {0, 3}})); + AssertThat(tree->children.contents[0]->has_changes, IsTrue()); + AssertThat(tree->children.contents[0]->padding, Equals({4, {0, 4}})); + AssertThat(tree->children.contents[0]->size, Equals({3, {0, 3}})); - AssertThat(tree->children[1]->has_changes, IsFalse()); + AssertThat(tree->children.contents[1]->has_changes, IsFalse()); }); }); @@ -287,11 +287,11 @@ describe("Tree", []() { AssertThat(tree->padding, Equals({2, {0, 2}})); AssertThat(tree->size, Equals({16, {0, 16}})); - AssertThat(tree->children[0]->has_changes, IsTrue()); - AssertThat(tree->children[0]->padding, Equals({2, {0, 2}})); - AssertThat(tree->children[0]->size, Equals({6, {0, 6}})); + AssertThat(tree->children.contents[0]->has_changes, IsTrue()); + AssertThat(tree->children.contents[0]->padding, Equals({2, {0, 2}})); + AssertThat(tree->children.contents[0]->size, Equals({6, {0, 6}})); - AssertThat(tree->children[1]->has_changes, IsFalse()); + AssertThat(tree->children.contents[1]->has_changes, IsFalse()); }); }); @@ -313,23 +313,23 @@ describe("Tree", []() { AssertThat(tree->padding, Equals({4, {0, 4}})); AssertThat(tree->size, Equals({4, {0, 4}})); - AssertThat(tree->children[0]->has_changes, IsTrue()); - AssertThat(tree->children[0]->padding, Equals({4, {0, 4}})); - AssertThat(tree->children[0]->size, Equals({0, {0, 0}})); + AssertThat(tree->children.contents[0]->has_changes, IsTrue()); + AssertThat(tree->children.contents[0]->padding, Equals({4, {0, 4}})); + AssertThat(tree->children.contents[0]->size, Equals({0, {0, 0}})); - AssertThat(tree->children[1]->has_changes, IsTrue()); - AssertThat(tree->children[1]->padding, Equals({0, {0, 0}})); - AssertThat(tree->children[1]->size, Equals({0, {0, 0}})); + AssertThat(tree->children.contents[1]->has_changes, IsTrue()); + AssertThat(tree->children.contents[1]->padding, Equals({0, {0, 0}})); + AssertThat(tree->children.contents[1]->size, Equals({0, {0, 0}})); - AssertThat(tree->children[2]->has_changes, IsTrue()); - AssertThat(tree->children[2]->padding, Equals({1, {0, 1}})); - AssertThat(tree->children[2]->size, Equals({3, {0, 3}})); + AssertThat(tree->children.contents[2]->has_changes, IsTrue()); + AssertThat(tree->children.contents[2]->padding, Equals({1, {0, 1}})); + AssertThat(tree->children.contents[2]->size, Equals({3, {0, 3}})); }); }); describe("edits within a tree's range of scanned bytes", [&]() { it("marks preceding trees as changed", [&]() { - tree->children[0]->bytes_scanned = 7; + tree->children.contents[0]->bytes_scanned = 7; TSInputEdit edit; edit.start_byte = 6; @@ -341,7 +341,7 @@ describe("Tree", []() { ts_tree_edit(tree, &edit); assert_consistent(tree); - AssertThat(tree->children[0]->has_changes, IsTrue()); + AssertThat(tree->children.contents[0]->has_changes, IsTrue()); }); }); }); @@ -361,14 +361,14 @@ describe("Tree", []() { Tree *leaf_copy = ts_tree_make_leaf(&pool, symbol1, {2, {1, 1}}, {5, {1, 4}}, &language); AssertThat(ts_tree_eq(leaf, leaf_copy), IsTrue()); - Tree *parent = ts_tree_make_node(&pool, symbol2, 2, tree_array({ + Tree *parent = ts_tree_make_node(&pool, symbol2, tree_array({ leaf, leaf_copy, }), 0, &language); ts_tree_retain(leaf); ts_tree_retain(leaf_copy); - Tree *parent_copy = ts_tree_make_node(&pool, symbol2, 2, tree_array({ + Tree *parent_copy = ts_tree_make_node(&pool, symbol2, tree_array({ leaf, leaf_copy, }), 0, &language); @@ -415,14 +415,14 @@ describe("Tree", []() { it("returns false for trees with different children", [&]() { Tree *leaf2 = ts_tree_make_leaf(&pool, symbol2, {1, {0, 1}}, {3, {0, 3}}, &language); - Tree *parent = ts_tree_make_node(&pool, symbol2, 2, tree_array({ + Tree *parent = ts_tree_make_node(&pool, symbol2, tree_array({ leaf, leaf2, }), 0, &language); ts_tree_retain(leaf); ts_tree_retain(leaf2); - Tree *different_parent = ts_tree_make_node(&pool, symbol2, 2, tree_array({ + Tree *different_parent = ts_tree_make_node(&pool, symbol2, tree_array({ leaf2, leaf, }), 0, &language); @@ -450,14 +450,14 @@ describe("Tree", []() { it("returns the last serialized external token state in the given tree", [&]() { Tree *tree1, *tree2, *tree3, *tree4, *tree5, *tree6, *tree7, *tree8, *tree9; - tree1 = ts_tree_make_node(&pool, symbol1, 2, tree_array({ - (tree2 = ts_tree_make_node(&pool, symbol2, 3, tree_array({ + tree1 = ts_tree_make_node(&pool, symbol1, tree_array({ + (tree2 = ts_tree_make_node(&pool, symbol2, tree_array({ (tree3 = make_external(ts_tree_make_leaf(&pool, symbol3, padding, size, &language))), (tree4 = ts_tree_make_leaf(&pool, symbol4, padding, size, &language)), (tree5 = ts_tree_make_leaf(&pool, symbol5, padding, size, &language)), }), 0, &language)), - (tree6 = ts_tree_make_node(&pool, symbol6, 2, tree_array({ - (tree7 = ts_tree_make_node(&pool, symbol7, 1, tree_array({ + (tree6 = ts_tree_make_node(&pool, symbol6, tree_array({ + (tree7 = ts_tree_make_node(&pool, symbol7, tree_array({ (tree8 = ts_tree_make_leaf(&pool, symbol8, padding, size, &language)), }), 0, &language)), (tree9 = ts_tree_make_leaf(&pool, symbol9, padding, size, &language)), From 94ed1b696488e76a2b1380361372bd2a5de12553 Mon Sep 17 00:00:00 2001 From: Max Brunsfeld Date: Fri, 6 Apr 2018 13:28:32 -0700 Subject: [PATCH 12/90] Make array_splice take an array, not a pointer and length --- src/runtime/array.h | 8 ++++---- src/runtime/parser.c | 17 ++++++++--------- src/runtime/tree.c | 2 +- 3 files changed, 13 insertions(+), 14 deletions(-) diff --git a/src/runtime/array.h b/src/runtime/array.h index e4e7ff0f..a0492526 100644 --- a/src/runtime/array.h +++ b/src/runtime/array.h @@ -47,14 +47,14 @@ extern "C" { (self)->contents[(self)->size++] = (element)) #define array_push_all(self, other) \ - array_splice((self), (self)->size, 0, (other)->size, (other)->contents) + array_splice((self), (self)->size, 0, (other)) -#define array_splice(self, index, old_count, new_count, new_elements) \ +#define array_splice(self, index, old_count, new_array) \ array__splice((VoidArray *)(self), array__elem_size(self), index, old_count, \ - new_count, (new_elements)) + (new_array)->size, (new_array)->contents) #define array_insert(self, index, element) \ - array_splice(self, index, 0, 1, &(element)) + array__splice((VoidArray *)(self), array__elem_size(self), index, 0, 1, &element) #define array_pop(self) ((self)->contents[--(self)->size]) diff --git a/src/runtime/parser.c b/src/runtime/parser.c index a377c42a..0977a077 100644 --- a/src/runtime/parser.c +++ b/src/runtime/parser.c @@ -672,29 +672,28 @@ static void parser__start(Parser *self, TSInput input, Tree *previous_tree) { self->in_ambiguity = false; } -static void parser__accept(Parser *self, StackVersion version, - Tree *lookahead) { +static void parser__accept(Parser *self, StackVersion version, Tree *lookahead) { lookahead->extra = true; assert(lookahead->symbol == ts_builtin_sym_end); ts_tree_retain(lookahead); ts_stack_push(self->stack, version, lookahead, false, 1); - StackSliceArray pop = ts_stack_pop_all(self->stack, version); + StackSliceArray pop = ts_stack_pop_all(self->stack, version); for (uint32_t i = 0; i < pop.size; i++) { - StackSlice slice = pop.contents[i]; - TreeArray trees = slice.trees; + TreeArray trees = pop.contents[i].trees; Tree *root = NULL; for (uint32_t j = trees.size - 1; j + 1 > 0; j--) { Tree *child = trees.contents[j]; if (!child->extra) { - root = ts_tree_make_copy(&self->tree_pool, child); - root->children.size = 0; for (uint32_t k = 0; k < child->children.size; k++) { ts_tree_retain(child->children.contents[k]); } - array_splice(&trees, j, 1, child->children.size, child->children.contents); - ts_tree_set_children(root, &trees, self->language); + array_splice(&trees, j, 1, &child->children); + root = ts_tree_make_node( + &self->tree_pool, child->symbol, &trees, + child->alias_sequence_id, self->language + ); ts_tree_release(&self->tree_pool, child); break; } diff --git a/src/runtime/tree.c b/src/runtime/tree.c index 06828f3b..f6dc542f 100644 --- a/src/runtime/tree.c +++ b/src/runtime/tree.c @@ -395,7 +395,7 @@ Tree *ts_tree_make_error_node(TreePool *pool, TreeArray *children, const TSLangu for (uint32_t i = 0; i < children->size; i++) { Tree *child = children->contents[i]; if (child->symbol == ts_builtin_sym_error && child->children.size > 0) { - array_splice(children, i, 1, child->children.size, child->children.contents); + array_splice(children, i, 1, &child->children); i += child->children.size - 1; for (uint32_t j = 0; j < child->children.size; j++) ts_tree_retain(child->children.contents[j]); From 87098760de8b77a167d95943ba164a0a3c8fed8b Mon Sep 17 00:00:00 2001 From: Max Brunsfeld Date: Fri, 6 Apr 2018 09:35:17 -0700 Subject: [PATCH 13/90] :art: --- src/runtime/language.c | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/runtime/language.c b/src/runtime/language.c index 7cdc71fc..7ef941ae 100644 --- a/src/runtime/language.c +++ b/src/runtime/language.c @@ -8,7 +8,6 @@ void ts_language_table_entry(const TSLanguage *self, TSStateId state, result->action_count = 0; result->is_reusable = false; result->actions = NULL; - return; } else { assert(symbol < self->token_count); uint32_t action_index = self->parse_table[state * self->symbol_count + symbol]; @@ -27,8 +26,7 @@ uint32_t ts_language_version(const TSLanguage *language) { return language->version; } -TSSymbolMetadata ts_language_symbol_metadata(const TSLanguage *language, - TSSymbol symbol) { +TSSymbolMetadata ts_language_symbol_metadata(const TSLanguage *language, TSSymbol symbol) { if (symbol == ts_builtin_sym_error) { return (TSSymbolMetadata){.visible = true, .named = true}; } else { From 3c737d82950337aea7a9d2ffb6d98bbb80957433 Mon Sep 17 00:00:00 2001 From: Max Brunsfeld Date: Fri, 6 Apr 2018 09:35:17 -0700 Subject: [PATCH 14/90] Respect -D flag in fuzzing examples --- test/integration/fuzzing-examples.cc | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/test/integration/fuzzing-examples.cc b/test/integration/fuzzing-examples.cc index c962440e..46cac15a 100644 --- a/test/integration/fuzzing-examples.cc +++ b/test/integration/fuzzing-examples.cc @@ -30,7 +30,10 @@ describe("examples found via fuzzing", [&]() { it(("parses example number " + to_string(i)).c_str(), [&]() { TSDocument *document = ts_document_new(); - // ts_document_print_debugging_graphs(document, true); + + if (getenv("TREE_SITTER_ENABLE_DEBUG_GRAPHS")) { + ts_document_print_debugging_graphs(document, true); + } const string &language_name = examples[i].first; ts_document_set_language(document, load_real_language(language_name)); From a79a8c216ff2d56402f910f7c7f9beb578c9c2d4 Mon Sep 17 00:00:00 2001 From: Max Brunsfeld Date: Fri, 6 Apr 2018 09:35:17 -0700 Subject: [PATCH 15/90] Include each node count and error cost in stack dot graphs --- src/runtime/stack.c | 29 +++++++++++++++-------------- 1 file changed, 15 insertions(+), 14 deletions(-) diff --git a/src/runtime/stack.c b/src/runtime/stack.c index aa2eaed4..e05d6a06 100644 --- a/src/runtime/stack.c +++ b/src/runtime/stack.c @@ -711,9 +711,10 @@ bool ts_stack_print_dot_graph(Stack *self, const char **symbol_names, FILE *f) { fprintf( f, - " tooltip=\"position: %u,%u\nerror_cost: %u\ndynamic_precedence: %d\"];\n", + " tooltip=\"position: %u,%u\nnode_count:%u\nerror_cost: %u\ndynamic_precedence: %d\"];\n", node->position.extent.row, node->position.extent.column, + node->node_count, node->error_cost, node->dynamic_precedence ); @@ -728,21 +729,21 @@ bool ts_stack_print_dot_graph(Stack *self, const char **symbol_names, FILE *f) { if (!link.tree) { fprintf(f, "color=red"); - } else if (link.tree->symbol == ts_builtin_sym_error) { - fprintf(f, "label=\"ERROR\""); } else { - fprintf(f, "label=\""); - if (!link.tree->named) - fprintf(f, "'"); - const char *name = symbol_names[link.tree->symbol]; - for (const char *c = name; *c; c++) { - if (*c == '\"' || *c == '\\') - fprintf(f, "\\"); - fprintf(f, "%c", *c); + if (link.tree->symbol == ts_builtin_sym_error) { + fprintf(f, "label=\"ERROR\""); + } else { + fprintf(f, "label=\""); + if (!link.tree->named) fprintf(f, "'"); + const char *name = symbol_names[link.tree->symbol]; + for (const char *c = name; *c; c++) { + if (*c == '\"' || *c == '\\') fprintf(f, "\\"); + fprintf(f, "%c", *c); + } + if (!link.tree->named) fprintf(f, "'"); + fprintf(f, "\""); } - if (!link.tree->named) - fprintf(f, "'"); - fprintf(f, "\" labeltooltip=\"error_cost: %u\ndynamic_precedence: %u\"", + fprintf(f, "labeltooltip=\"error_cost: %u\ndynamic_precedence: %u\"", link.tree->error_cost, link.tree->dynamic_precedence); } From 1eafcf0ba74bc38820943f0b8a3936882930135c Mon Sep 17 00:00:00 2001 From: Max Brunsfeld Date: Fri, 6 Apr 2018 09:35:17 -0700 Subject: [PATCH 16/90] Make test script generate debug graphs even when assertions fail --- script/test | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/script/test b/script/test index 34e66241..74025499 100755 --- a/script/test +++ b/script/test @@ -132,7 +132,7 @@ case ${mode} in if [[ -n $line_count ]]; then head -n $line_count $dot_file | dot -Tsvg >> $html_file else - cat $dot_file | dot -Tsvg >> $html_file + cat $dot_file | grep -v 'Assertion' | dot -Tsvg >> $html_file fi rm $dot_file echo "Wrote $html_file - $line_count" From 379a2fd1214960d1896915ba88df84a0a9b3ced1 Mon Sep 17 00:00:00 2001 From: Max Brunsfeld Date: Fri, 6 Apr 2018 09:35:17 -0700 Subject: [PATCH 17/90] Incrementally build a tree of skipped tokens Rather than pushing them to the stack individually --- .../build_tables/parse_table_builder.cc | 8 +- src/runtime/language.c | 8 +- src/runtime/language.h | 4 +- src/runtime/parser.c | 195 +++++++++++++----- src/runtime/stack.c | 50 ++--- src/runtime/stack.h | 2 +- src/runtime/tree.c | 33 ++- test/fixtures/error_corpus/c_errors.txt | 4 +- .../error_corpus/javascript_errors.txt | 45 +++- 9 files changed, 233 insertions(+), 116 deletions(-) diff --git a/src/compiler/build_tables/parse_table_builder.cc b/src/compiler/build_tables/parse_table_builder.cc index 51ed5cc4..6218fec6 100644 --- a/src/compiler/build_tables/parse_table_builder.cc +++ b/src/compiler/build_tables/parse_table_builder.cc @@ -150,6 +150,8 @@ class ParseTableBuilderImpl : public ParseTableBuilder { MatchesLongerStringWithValidNextChar ); + parse_table.states[state_id].terminal_entries.clear(); + // Add all the tokens that have no conflict with other tokens. LookaheadSet non_conflicting_tokens; for (unsigned i = 0; i < lexical_grammar.variables.size(); i++) { @@ -186,12 +188,6 @@ class ParseTableBuilderImpl : public ParseTableBuilder { } } - for (const Symbol &symbol : grammar.extra_tokens) { - if (!parse_table.states[state_id].terminal_entries.count(symbol)) { - parse_table.add_terminal_action(state_id, symbol, ParseAction::ShiftExtra()); - } - } - for (size_t i = 0; i < grammar.external_tokens.size(); i++) { if (grammar.external_tokens[i].corresponding_internal_token == rules::NONE()) { parse_table.states[state_id].terminal_entries[Symbol::external(i)].actions.push_back(ParseAction::Recover()); diff --git a/src/runtime/language.c b/src/runtime/language.c index 7ef941ae..9bf1fc63 100644 --- a/src/runtime/language.c +++ b/src/runtime/language.c @@ -4,7 +4,7 @@ void ts_language_table_entry(const TSLanguage *self, TSStateId state, TSSymbol symbol, TableEntry *result) { - if (symbol == ts_builtin_sym_error) { + if (symbol == ts_builtin_sym_error || symbol == ts_builtin_sym_error_repeat) { result->action_count = 0; result->is_reusable = false; result->actions = NULL; @@ -27,8 +27,10 @@ uint32_t ts_language_version(const TSLanguage *language) { } TSSymbolMetadata ts_language_symbol_metadata(const TSLanguage *language, TSSymbol symbol) { - if (symbol == ts_builtin_sym_error) { + if (symbol == ts_builtin_sym_error) { return (TSSymbolMetadata){.visible = true, .named = true}; + } else if (symbol == ts_builtin_sym_error_repeat) { + return (TSSymbolMetadata){.visible = false, .named = false}; } else { return language->symbol_metadata[symbol]; } @@ -37,6 +39,8 @@ TSSymbolMetadata ts_language_symbol_metadata(const TSLanguage *language, TSSymbo const char *ts_language_symbol_name(const TSLanguage *language, TSSymbol symbol) { if (symbol == ts_builtin_sym_error) { return "ERROR"; + } else if (symbol == ts_builtin_sym_error_repeat) { + return "_ERROR"; } else { return language->symbol_names[symbol]; } diff --git a/src/runtime/language.h b/src/runtime/language.h index 966d15df..64733242 100644 --- a/src/runtime/language.h +++ b/src/runtime/language.h @@ -8,6 +8,8 @@ extern "C" { #include "tree_sitter/parser.h" #include "runtime/tree.h" +#define ts_builtin_sym_error_repeat (ts_builtin_sym_error - 1) + typedef struct { const TSParseAction *actions; uint32_t action_count; @@ -51,7 +53,7 @@ static inline bool ts_language_has_reduce_action(const TSLanguage *self, static inline TSStateId ts_language_next_state(const TSLanguage *self, TSStateId state, TSSymbol symbol) { - if (symbol == ts_builtin_sym_error) { + if (symbol == ts_builtin_sym_error || symbol == ts_builtin_sym_error_repeat) { return 0; } else if (symbol < self->token_count) { uint32_t count; diff --git a/src/runtime/parser.c b/src/runtime/parser.c index 0977a077..23ce42b1 100644 --- a/src/runtime/parser.c +++ b/src/runtime/parser.c @@ -639,6 +639,20 @@ static StackSliceArray parser__reduce(Parser *self, StackVersion version, TSSymb for (uint32_t j = parent->children.size; j < slice.trees.size; j++) { ts_stack_push(self->stack, slice.version, slice.trees.contents[j], false, next_state); } + + if (ts_stack_version_count(self->stack) > MAX_VERSION_COUNT) { + i++; + while (i < pop.size) { + StackSlice slice = pop.contents[i]; + ts_tree_array_delete(&self->tree_pool, &slice.trees); + ts_stack_halt(self->stack, slice.version); + i++; + } + while (ts_stack_version_count(self->stack) > slice.version + 1) { + ts_stack_remove_version(self->stack, slice.version + 1); + } + break; + } } for (StackVersion i = initial_version_count; i < ts_stack_version_count(self->stack); i++) { @@ -720,12 +734,23 @@ static void parser__accept(Parser *self, StackVersion version, Tree *lookahead) static bool parser__do_all_potential_reductions(Parser *self, StackVersion starting_version, TSSymbol lookahead_symbol) { - bool result = false; - for (StackVersion version = starting_version; - ts_stack_version_count(self->stack) < MAX_VERSION_COUNT;) { + uint32_t initial_version_count = ts_stack_version_count(self->stack); + + bool can_shift_lookahead_symbol = false; + StackVersion version = starting_version; + for (unsigned i = 0; true; i++) { uint32_t version_count = ts_stack_version_count(self->stack); if (version >= version_count) break; + bool merged = false; + for (StackVersion i = initial_version_count; i < version; i++) { + if (ts_stack_merge(self->stack, i, version)) { + merged = true; + break; + } + } + if (merged) continue; + TSStateId state = ts_stack_state(self->stack, version); bool has_shift_action = false; array_clear(&self->reduce_actions); @@ -747,7 +772,7 @@ static bool parser__do_all_potential_reductions(Parser *self, StackVersion start switch (action.type) { case TSParseActionTypeShift: case TSParseActionTypeRecover: - if (!action.params.extra) has_shift_action = true; + if (!action.params.extra && !action.params.repetition) has_shift_action = true; break; case TSParseActionTypeReduce: if (action.params.child_count > 0) @@ -763,9 +788,9 @@ static bool parser__do_all_potential_reductions(Parser *self, StackVersion start } } - bool has_reduce_action = self->reduce_actions.size > 0; for (uint32_t i = 0; i < self->reduce_actions.size; i++) { ReduceAction action = self->reduce_actions.contents[i]; + parser__reduce( self, version, action.symbol, action.count, action.dynamic_precedence, action.alias_sequence_id, @@ -774,14 +799,12 @@ static bool parser__do_all_potential_reductions(Parser *self, StackVersion start } if (has_shift_action) { - result = true; - } else { - if (has_reduce_action) { - ts_stack_renumber_version(self->stack, version_count, version); - continue; - } else if (lookahead_symbol != 0) { - ts_stack_remove_version(self->stack, version); - } + can_shift_lookahead_symbol = true; + } else if (self->reduce_actions.size > 0 && i < MAX_VERSION_COUNT) { + ts_stack_renumber_version(self->stack, version_count, version); + continue; + } else if (lookahead_symbol != 0) { + ts_stack_remove_version(self->stack, version); } if (version == starting_version) { @@ -790,7 +813,8 @@ static bool parser__do_all_potential_reductions(Parser *self, StackVersion start version++; } } - return result; + + return can_shift_lookahead_symbol; } static void parser__handle_error(Parser *self, StackVersion version, TSSymbol lookahead_symbol) { @@ -830,7 +854,11 @@ static void parser__handle_error(Parser *self, StackVersion version, TSSymbol lo self, version_with_missing_tree, lookahead_symbol )) { - LOG("recover_with_missing symbol:%s, state:%u", SYM_NAME(missing_symbol), state_after_missing_symbol); + LOG( + "recover_with_missing symbol:%s, state:%u", + SYM_NAME(missing_symbol), + ts_stack_state(self->stack, version_with_missing_tree) + ); did_insert_missing_token = true; break; } @@ -894,13 +922,14 @@ static bool parser__recover_to_state(Parser *self, StackVersion version, unsigne continue; } - StackSliceArray error_pop = ts_stack_pop_error(self->stack, slice.version); - if (error_pop.size > 0) { - StackSlice error_slice = error_pop.contents[0]; - array_push_all(&error_slice.trees, &slice.trees); - array_delete(&slice.trees); - slice.trees = error_slice.trees; - ts_stack_renumber_version(self->stack, error_slice.version, slice.version); + TreeArray error_trees = ts_stack_pop_error(self->stack, slice.version); + if (error_trees.size > 0) { + assert(error_trees.size == 1); + array_splice(&slice.trees, 0, 0, &error_trees.contents[0]->children); + for (unsigned j = 0; j < error_trees.contents[0]->children.size; j++) { + ts_tree_retain(slice.trees.contents[j]); + } + ts_tree_array_delete(&self->tree_pool, &error_trees); } TreeArray trailing_extras = ts_tree_array_remove_trailing_extras(&slice.trees); @@ -930,41 +959,51 @@ static void parser__recover(Parser *self, StackVersion version, Tree *lookahead) unsigned previous_version_count = ts_stack_version_count(self->stack); Length position = ts_stack_position(self->stack, version); StackSummary *summary = ts_stack_get_summary(self->stack, version); - unsigned depth_since_error = ts_stack_node_count_since_error(self->stack, version); + unsigned node_count_since_error = ts_stack_node_count_since_error(self->stack, version); + unsigned current_error_cost = ts_stack_error_cost(self->stack, version); - for (unsigned i = 0; i < summary->size; i++) { - StackSummaryEntry entry = summary->contents[i]; + if (summary && lookahead->symbol != ts_builtin_sym_error) { + for (unsigned i = 0; i < summary->size; i++) { + StackSummaryEntry entry = summary->contents[i]; - if (entry.state == ERROR_STATE) continue; - if (entry.position.bytes == position.bytes) continue; - unsigned depth = entry.depth + depth_since_error; - if (depth > MAX_SUMMARY_DEPTH) break; + if (entry.state == ERROR_STATE) continue; + if (entry.position.bytes == position.bytes) continue; + unsigned depth = entry.depth; + if (node_count_since_error > 0) depth++; - unsigned new_cost = - depth * ERROR_COST_PER_SKIPPED_TREE + - (position.bytes - entry.position.bytes) * ERROR_COST_PER_SKIPPED_CHAR + - (position.extent.row - entry.position.extent.row) * ERROR_COST_PER_SKIPPED_LINE; - if (parser__better_version_exists(self, version, false, new_cost)) break; + bool would_merge = false; + for (unsigned j = 0; j < previous_version_count; j++) { + if ( + ts_stack_state(self->stack, j) == entry.state && + ts_stack_position(self->stack, j).bytes == position.bytes + ) { + would_merge = true; + break; + } + } - if (ts_language_has_actions(self->language, entry.state, lookahead->symbol)) { - if (parser__recover_to_state(self, version, depth, entry.state)) { - did_recover = true; - LOG("recover state:%u, depth:%u", entry.state, depth); - LOG_STACK(); - break; + if (would_merge) continue; + + unsigned new_cost = + current_error_cost + + entry.depth * ERROR_COST_PER_SKIPPED_TREE + + (position.bytes - entry.position.bytes) * ERROR_COST_PER_SKIPPED_CHAR + + (position.extent.row - entry.position.extent.row) * ERROR_COST_PER_SKIPPED_LINE; + if (parser__better_version_exists(self, version, false, new_cost)) break; + + if (ts_language_has_actions(self->language, entry.state, lookahead->symbol)) { + if (parser__recover_to_state(self, version, depth, entry.state)) { + did_recover = true; + LOG("recover_to_previous state:%u, depth:%u", entry.state, depth); + LOG_STACK(); + break; + } } } } for (unsigned i = previous_version_count; i < ts_stack_version_count(self->stack); i++) { - if (ts_stack_is_active(self->stack, i)) { - for (unsigned j = 0; j < i; j++) { - if (ts_stack_can_merge(self->stack, j, i)) { - ts_stack_remove_version(self->stack, i--); - break; - } - } - } else { + if (!ts_stack_is_active(self->stack, i)) { ts_stack_remove_version(self->stack, i--); } } @@ -983,15 +1022,56 @@ static void parser__recover(Parser *self, StackVersion version, Tree *lookahead) return; } + unsigned new_cost = + current_error_cost + ERROR_COST_PER_SKIPPED_TREE + + ts_tree_total_bytes(lookahead) * ERROR_COST_PER_SKIPPED_CHAR + + ts_tree_total_size(lookahead).extent.row * ERROR_COST_PER_SKIPPED_LINE; + + if (parser__better_version_exists(self, version, false, new_cost)) { + ts_stack_halt(self->stack, version); + return; + } + unsigned n; const TSParseAction *actions = ts_language_actions(self->language, 1, lookahead->symbol, &n); - bool extra = n > 0 && actions[n - 1].type == TSParseActionTypeShift && actions[n - 1].params.extra; - parser__shift(self, version, ERROR_STATE, lookahead, extra); + if (n > 0 && actions[n - 1].type == TSParseActionTypeShift && actions[n - 1].params.extra) { + lookahead->extra = true; + } - if (parser__better_version_exists(self, version, true, ts_stack_error_cost(self->stack, version))) { - ts_stack_halt(self->stack, version); - } else { - LOG("skip_token symbol:%s", SYM_NAME(lookahead->symbol)); + LOG("skip_token symbol:%s", SYM_NAME(lookahead->symbol)); + ts_tree_retain(lookahead); + TreeArray children = array_new(); + array_grow(&children, 1); + array_push(&children, lookahead); + Tree *error_repeat = ts_tree_make_node( + &self->tree_pool, + ts_builtin_sym_error_repeat, + &children, + 0, + self->language + ); + + if (node_count_since_error > 0) { + StackSliceArray pop = ts_stack_pop_count(self->stack, version, 1); + assert(pop.size == 1); + assert(pop.contents[0].trees.size == 1); + ts_stack_renumber_version(self->stack, pop.contents[0].version, version); + array_push(&pop.contents[0].trees, error_repeat); + error_repeat = ts_tree_make_node( + &self->tree_pool, + ts_builtin_sym_error_repeat, + &pop.contents[0].trees, + 0, + self->language + ); + } + + ts_stack_push(self->stack, version, error_repeat, false, ERROR_STATE); + + if (lookahead->has_external_tokens) { + ts_stack_set_last_external_token( + self->stack, version, ts_tree_last_external_token(lookahead) + ); } } @@ -1011,6 +1091,10 @@ static void parser__advance(Parser *self, StackVersion version, ReusableNode *re if (action.params.repetition) break; TSStateId next_state; if (action.params.extra) { + + // TODO remove when TREE_SITTER_LANGUAGE_VERSION 9 is out. + if (state == ERROR_STATE) continue; + next_state = state; LOG("shift_extra"); } else { @@ -1065,7 +1149,8 @@ static void parser__advance(Parser *self, StackVersion version, ReusableNode *re ts_stack_renumber_version(self->stack, last_reduction_version, version); LOG_STACK(); } else if (state == ERROR_STATE) { - ts_stack_push(self->stack, version, lookahead, false, ERROR_STATE); + parser__recover(self, version, lookahead); + ts_tree_release(&self->tree_pool, lookahead); return; } else if (!parser__breakdown_top_of_stack(self, version)) { LOG("detect_error"); diff --git a/src/runtime/stack.c b/src/runtime/stack.c index e05d6a06..4d9f5efe 100644 --- a/src/runtime/stack.c +++ b/src/runtime/stack.c @@ -142,21 +142,7 @@ static StackNode *stack_node_new(StackNode *previous_node, Tree *tree, bool is_p node->error_cost += tree->error_cost; node->position = length_add(node->position, ts_tree_total_size(tree)); node->dynamic_precedence += tree->dynamic_precedence; - if (!tree->extra) { - node->node_count += tree->node_count; - - if (state == ERROR_STATE) { - node->error_cost += - ERROR_COST_PER_SKIPPED_TREE * ((tree->visible || tree->children.size == 0) ? 1 : tree->visible_child_count) + - ERROR_COST_PER_SKIPPED_CHAR * tree->size.bytes + - ERROR_COST_PER_SKIPPED_LINE * tree->size.extent.row; - if (previous_node->links[0].tree) { - node->error_cost += - ERROR_COST_PER_SKIPPED_CHAR * tree->padding.bytes + - ERROR_COST_PER_SKIPPED_LINE * tree->padding.extent.row; - } - } - } + if (!tree->extra) node->node_count += tree->node_count; } } else { node->position = length_zero(); @@ -400,7 +386,9 @@ void ts_stack_set_last_external_token(Stack *self, StackVersion version, Tree *t unsigned ts_stack_error_cost(const Stack *self, StackVersion version) { StackHead *head = array_get(&self->heads, version); unsigned result = head->node->error_cost; - if (head->node->state == ERROR_STATE || head->status == StackStatusPaused) { + if ( + head->status == StackStatusPaused || + (head->node->state == ERROR_STATE && !head->node->links[0].tree)) { result += ERROR_COST_PER_RECOVERY; } return result; @@ -408,6 +396,9 @@ unsigned ts_stack_error_cost(const Stack *self, StackVersion version) { unsigned ts_stack_node_count_since_error(const Stack *self, StackVersion version) { StackHead *head = array_get(&self->heads, version); + if (head->node->node_count < head->node_count_at_last_error) { + head->node_count_at_last_error = head->node->node_count; + } return head->node->node_count - head->node_count_at_last_error; } @@ -482,15 +473,21 @@ inline StackAction pop_error_callback(void *payload, const Iterator *iterator) { } } -StackSliceArray ts_stack_pop_error(Stack *self, StackVersion version) { +TreeArray ts_stack_pop_error(Stack *self, StackVersion version) { StackNode *node = array_get(&self->heads, version)->node; for (unsigned i = 0; i < node->link_count; i++) { if (node->links[i].tree && node->links[i].tree->symbol == ts_builtin_sym_error) { bool found_error = false; - return stack__iter(self, version, pop_error_callback, &found_error, true); + StackSliceArray pop = stack__iter(self, version, pop_error_callback, &found_error, true); + if (pop.size > 0) { + assert(pop.size == 1); + ts_stack_renumber_version(self, pop.contents[0].version, version); + return pop.contents[0].trees; + } + break; } } - return (StackSliceArray){.size = 0}; + return (TreeArray){.size = 0}; } inline StackAction pop_all_callback(void *payload, const Iterator *iterator) { @@ -550,8 +547,14 @@ void ts_stack_remove_version(Stack *self, StackVersion version) { void ts_stack_renumber_version(Stack *self, StackVersion v1, StackVersion v2) { assert(v2 < v1); assert((uint32_t)v1 < self->heads.size); - stack_head_delete(&self->heads.contents[v2], &self->node_pool, self->tree_pool); - self->heads.contents[v2] = self->heads.contents[v1]; + StackHead *source_head = &self->heads.contents[v1]; + StackHead *target_head = &self->heads.contents[v2]; + if (target_head->summary && !source_head->summary) { + source_head->summary = target_head->summary; + target_head->summary = NULL; + } + stack_head_delete(target_head, &self->node_pool, self->tree_pool); + *target_head = *source_head; array_erase(&self->heads, v1); } @@ -578,8 +581,8 @@ bool ts_stack_merge(Stack *self, StackVersion version1, StackVersion version2) { for (uint32_t i = 0; i < head2->node->link_count; i++) { stack_node_add_link(head1->node, head2->node->links[i]); } - if (head2->node_count_at_last_error > head1->node_count_at_last_error) { - head1->node_count_at_last_error = head2->node_count_at_last_error; + if (head1->node->state == ERROR_STATE) { + head1->node_count_at_last_error = head1->node->node_count; } ts_stack_remove_version(self, version2); return true; @@ -593,6 +596,7 @@ bool ts_stack_can_merge(Stack *self, StackVersion version1, StackVersion version head2->status == StackStatusActive && head1->node->state == head2->node->state && head1->node->position.bytes == head2->node->position.bytes && + head1->node->error_cost == head2->node->error_cost && ts_tree_external_token_state_eq(head1->last_external_token, head2->last_external_token); } diff --git a/src/runtime/stack.h b/src/runtime/stack.h index 92a09b69..32e68e6e 100644 --- a/src/runtime/stack.h +++ b/src/runtime/stack.h @@ -65,7 +65,7 @@ void ts_stack_push(Stack *, StackVersion, Tree *, bool, TSStateId); StackSliceArray ts_stack_pop_count(Stack *, StackVersion, uint32_t count); // Remove an error at the top of the given version of the stack. -StackSliceArray ts_stack_pop_error(Stack *, StackVersion); +TreeArray ts_stack_pop_error(Stack *, StackVersion); // Remove any pending trees from the top of the given version of the stack. StackSliceArray ts_stack_pop_pending(Stack *, StackVersion); diff --git a/src/runtime/tree.c b/src/runtime/tree.c index f6dc542f..f0ffd1b4 100644 --- a/src/runtime/tree.c +++ b/src/runtime/tree.c @@ -324,7 +324,9 @@ void ts_tree_set_children(Tree *self, TreeArray *children, const TSLanguage *lan self->size = length_add(self->size, ts_tree_total_size(child)); } - self->error_cost += child->error_cost; + if (child->symbol != ts_builtin_sym_error_repeat) { + self->error_cost += child->error_cost; + } self->dynamic_precedence += child->dynamic_precedence; self->node_count += child->node_count; @@ -351,13 +353,18 @@ void ts_tree_set_children(Tree *self, TreeArray *children, const TSLanguage *lan if (!child->extra) non_extra_index++; } - if (self->symbol == ts_builtin_sym_error) { + if (self->symbol == ts_builtin_sym_error || self->symbol == ts_builtin_sym_error_repeat) { self->error_cost += ERROR_COST_PER_RECOVERY + ERROR_COST_PER_SKIPPED_CHAR * self->size.bytes + ERROR_COST_PER_SKIPPED_LINE * self->size.extent.row; for (uint32_t i = 0; i < self->children.size; i++) { - if (!self->children.contents[i]->extra) { + Tree *child = self->children.contents[i]; + if (child->extra) continue; + if (child->symbol == ts_builtin_sym_error && child->children.size == 0) continue; + if (child->visible) { self->error_cost += ERROR_COST_PER_SKIPPED_TREE; + } else { + self->error_cost += ERROR_COST_PER_SKIPPED_TREE * child->visible_child_count; } } } @@ -387,26 +394,16 @@ Tree *ts_tree_make_node(TreePool *pool, TSSymbol symbol, TreeArray *children, unsigned alias_sequence_id, const TSLanguage *language) { Tree *result = ts_tree_make_leaf(pool, symbol, length_zero(), length_zero(), language); result->alias_sequence_id = alias_sequence_id; + if (symbol == ts_builtin_sym_error || symbol == ts_builtin_sym_error_repeat) { + result->fragile_left = true; + result->fragile_right = true; + } ts_tree_set_children(result, children, language); return result; } Tree *ts_tree_make_error_node(TreePool *pool, TreeArray *children, const TSLanguage *language) { - for (uint32_t i = 0; i < children->size; i++) { - Tree *child = children->contents[i]; - if (child->symbol == ts_builtin_sym_error && child->children.size > 0) { - array_splice(children, i, 1, &child->children); - i += child->children.size - 1; - for (uint32_t j = 0; j < child->children.size; j++) - ts_tree_retain(child->children.contents[j]); - ts_tree_release(pool, child); - } - } - - Tree *result = ts_tree_make_node(pool, ts_builtin_sym_error, children, 0, language); - result->fragile_left = true; - result->fragile_right = true; - return result; + return ts_tree_make_node(pool, ts_builtin_sym_error, children, 0, language); } Tree *ts_tree_make_missing_leaf(TreePool *pool, TSSymbol symbol, const TSLanguage *language) { diff --git a/test/fixtures/error_corpus/c_errors.txt b/test/fixtures/error_corpus/c_errors.txt index c5833156..b2931b7d 100644 --- a/test/fixtures/error_corpus/c_errors.txt +++ b/test/fixtures/error_corpus/c_errors.txt @@ -81,7 +81,9 @@ int main() { (function_declarator (identifier) (parameter_list)) (compound_statement (if_statement - (field_expression (identifier) (MISSING)) + (field_expression + (identifier) + (MISSING)) (compound_statement (expression_statement (call_expression (identifier) (argument_list))) (expression_statement (call_expression (identifier) (argument_list))) diff --git a/test/fixtures/error_corpus/javascript_errors.txt b/test/fixtures/error_corpus/javascript_errors.txt index d435ba86..fb31d9e2 100644 --- a/test/fixtures/error_corpus/javascript_errors.txt +++ b/test/fixtures/error_corpus/javascript_errors.txt @@ -77,15 +77,12 @@ if ({a: 'b'} {c: 'd'}) { (ERROR (object (pair (property_identifier) (string)))) (object (pair (property_identifier) (string)))) (statement_block - (expression_statement (assignment_expression - (identifier) - (call_expression - (function (formal_parameters (identifier)) (statement_block (expression_statement (identifier)))) - (ERROR) - (arguments (identifier)))) - (MISSING)) - (statement_block - (expression_statement (identifier)))))) + (expression_statement + (assignment_expression + (identifier) + (function (formal_parameters (identifier)) (statement_block (expression_statement (identifier))))) + (MISSING)) + (function (formal_parameters (identifier)) (statement_block (expression_statement (identifier))))))) =================================================== Extra tokens at the end of the file @@ -150,3 +147,33 @@ const a = `b c ${d +} f g` (variable_declarator (identifier) (template_string (template_substitution (identifier) (ERROR)))))) + +========================================================= +Long sequences of invalid tokens +========================================================= + +function main(x) { + console.log('a'); + what?????????????????????????????????????????????????? + console.log('b'); + return {}; +} + +--- + +(program + (function + (identifier) + (formal_parameters (identifier)) + (statement_block + (expression_statement + (call_expression + (member_expression (identifier) (property_identifier)) + (arguments (string)))) + (expression_statement + (identifier) + (ERROR + (call_expression + (member_expression (identifier) (property_identifier)) + (arguments (string))))) + (return_statement (object))))) From 33820253e8463b27776eeaad87c792848d26508a Mon Sep 17 00:00:00 2001 From: Max Brunsfeld Date: Sun, 8 Apr 2018 13:49:20 -0700 Subject: [PATCH 18/90] Make stack_print_dot_graph function take a language as an argument --- src/runtime/parser.c | 8 ++++---- src/runtime/stack.c | 23 ++++++++++------------- src/runtime/stack.h | 2 +- src/runtime/tree.c | 3 +-- 4 files changed, 16 insertions(+), 20 deletions(-) diff --git a/src/runtime/parser.c b/src/runtime/parser.c index 23ce42b1..bfb0ab37 100644 --- a/src/runtime/parser.c +++ b/src/runtime/parser.c @@ -19,10 +19,10 @@ parser__log(self); \ } -#define LOG_STACK() \ - if (self->print_debugging_graphs) { \ - ts_stack_print_dot_graph(self->stack, self->language->symbol_names, stderr); \ - fputs("\n\n", stderr); \ +#define LOG_STACK() \ + if (self->print_debugging_graphs) { \ + ts_stack_print_dot_graph(self->stack, self->language, stderr); \ + fputs("\n\n", stderr); \ } #define LOG_TREE() \ diff --git a/src/runtime/stack.c b/src/runtime/stack.c index 4d9f5efe..76a6e89c 100644 --- a/src/runtime/stack.c +++ b/src/runtime/stack.c @@ -1,4 +1,5 @@ #include "runtime/alloc.h" +#include "runtime/language.h" #include "runtime/tree.h" #include "runtime/array.h" #include "runtime/stack.h" @@ -646,7 +647,7 @@ void ts_stack_clear(Stack *self) { })); } -bool ts_stack_print_dot_graph(Stack *self, const char **symbol_names, FILE *f) { +bool ts_stack_print_dot_graph(Stack *self, const TSLanguage *language, FILE *f) { bool was_recording_allocations = ts_toggle_allocation_recording(false); if (!f) f = stderr; @@ -734,19 +735,15 @@ bool ts_stack_print_dot_graph(Stack *self, const char **symbol_names, FILE *f) { if (!link.tree) { fprintf(f, "color=red"); } else { - if (link.tree->symbol == ts_builtin_sym_error) { - fprintf(f, "label=\"ERROR\""); - } else { - fprintf(f, "label=\""); - if (!link.tree->named) fprintf(f, "'"); - const char *name = symbol_names[link.tree->symbol]; - for (const char *c = name; *c; c++) { - if (*c == '\"' || *c == '\\') fprintf(f, "\\"); - fprintf(f, "%c", *c); - } - if (!link.tree->named) fprintf(f, "'"); - fprintf(f, "\""); + fprintf(f, "label=\""); + if (link.tree->visible && !link.tree->named) fprintf(f, "'"); + const char *name = ts_language_symbol_name(language, link.tree->symbol); + for (const char *c = name; *c; c++) { + if (*c == '\"' || *c == '\\') fprintf(f, "\\"); + fprintf(f, "%c", *c); } + if (link.tree->visible && !link.tree->named) fprintf(f, "'"); + fprintf(f, "\""); fprintf(f, "labeltooltip=\"error_cost: %u\ndynamic_precedence: %u\"", link.tree->error_cost, link.tree->dynamic_precedence); diff --git a/src/runtime/stack.h b/src/runtime/stack.h index 32e68e6e..4a552323 100644 --- a/src/runtime/stack.h +++ b/src/runtime/stack.h @@ -120,7 +120,7 @@ void ts_stack_remove_version(Stack *, StackVersion); void ts_stack_clear(Stack *); -bool ts_stack_print_dot_graph(Stack *, const char **, FILE *); +bool ts_stack_print_dot_graph(Stack *, const TSLanguage *, FILE *); typedef void (*StackIterateCallback)(void *, TSStateId, uint32_t); diff --git a/src/runtime/tree.c b/src/runtime/tree.c index f0ffd1b4..b6dd7a9c 100644 --- a/src/runtime/tree.c +++ b/src/runtime/tree.c @@ -715,8 +715,7 @@ void ts_tree__print_dot_graph(const Tree *self, uint32_t byte_offset, } } -void ts_tree_print_dot_graph(const Tree *self, const TSLanguage *language, - FILE *f) { +void ts_tree_print_dot_graph(const Tree *self, const TSLanguage *language, FILE *f) { fprintf(f, "digraph tree {\n"); fprintf(f, "edge [arrowhead=none]\n"); ts_tree__print_dot_graph(self, 0, language, f); From 3672a8ad879261df5d8d3241772174b691ac2026 Mon Sep 17 00:00:00 2001 From: Max Brunsfeld Date: Sun, 8 Apr 2018 13:49:32 -0700 Subject: [PATCH 19/90] Remove unused tree method --- src/runtime/tree.c | 10 ---------- src/runtime/tree.h | 1 - 2 files changed, 11 deletions(-) diff --git a/src/runtime/tree.c b/src/runtime/tree.c index b6dd7a9c..ad58bfd3 100644 --- a/src/runtime/tree.c +++ b/src/runtime/tree.c @@ -68,16 +68,6 @@ void ts_tree_array_delete(TreePool *pool, TreeArray *self) { array_delete(self); } -uint32_t ts_tree_array_essential_count(const TreeArray *self) { - uint32_t result = 0; - for (uint32_t i = 0; i < self->size; i++) { - Tree *tree = self->contents[i]; - if (!tree->extra && tree->symbol != ts_builtin_sym_error) - result++; - } - return result; -} - TreeArray ts_tree_array_remove_last_n(TreeArray *self, uint32_t remove_count) { TreeArray result = array_new(); if (self->size == 0 || remove_count == 0) return result; diff --git a/src/runtime/tree.h b/src/runtime/tree.h index 09dae3dd..5b655afb 100644 --- a/src/runtime/tree.h +++ b/src/runtime/tree.h @@ -88,7 +88,6 @@ const char *ts_external_token_state_data(const TSExternalTokenState *); bool ts_tree_array_copy(TreeArray, TreeArray *); void ts_tree_array_delete(TreePool *, TreeArray *); -uint32_t ts_tree_array_essential_count(const TreeArray *); TreeArray ts_tree_array_remove_last_n(TreeArray *, uint32_t); TreeArray ts_tree_array_remove_trailing_extras(TreeArray *); void ts_tree_array_reverse(TreeArray *); From b0b8279c14213e67914c99ccfe7772075ce049a7 Mon Sep 17 00:00:00 2001 From: Max Brunsfeld Date: Mon, 9 Apr 2018 18:09:54 -0700 Subject: [PATCH 20/90] Preallocate tree arrays when popping from the stack --- src/runtime/array.h | 30 +++++++++++++++++------------- src/runtime/parser.c | 4 ++-- src/runtime/stack.c | 29 ++++++++++++++++++----------- src/runtime/tree.c | 2 +- 4 files changed, 38 insertions(+), 27 deletions(-) diff --git a/src/runtime/array.h b/src/runtime/array.h index a0492526..4d2f478b 100644 --- a/src/runtime/array.h +++ b/src/runtime/array.h @@ -34,16 +34,16 @@ extern "C" { #define array_clear(self) ((self)->size = 0) -#define array_grow(self, new_capacity) \ - array__grow((VoidArray *)(self), array__elem_size(self), new_capacity) +#define array_reserve(self, new_capacity) \ + array__reserve((VoidArray *)(self), array__elem_size(self), new_capacity) #define array_erase(self, index) \ array__erase((VoidArray *)(self), array__elem_size(self), index) #define array_delete(self) array__delete((VoidArray *)self) -#define array_push(self, element) \ - (array_grow((self), (self)->size + 1), \ +#define array_push(self, element) \ + (array__grow((VoidArray *)(self), array__elem_size(self)), \ (self)->contents[(self)->size++] = (element)) #define array_push_all(self, other) \ @@ -80,21 +80,25 @@ static inline void array__erase(VoidArray *self, size_t element_size, self->size--; } -static inline void array__grow(VoidArray *self, size_t element_size, - uint32_t new_capacity) { +static inline void array__reserve(VoidArray *self, size_t element_size, uint32_t new_capacity) { if (new_capacity > self->capacity) { - if (new_capacity < 2 * self->capacity) - new_capacity = 2 * self->capacity; - if (new_capacity < 8) - new_capacity = 8; - if (self->contents) + if (self->contents) { self->contents = ts_realloc(self->contents, new_capacity * element_size); - else + } else { self->contents = ts_calloc(new_capacity, element_size); + } self->capacity = new_capacity; } } +static inline void array__grow(VoidArray *self, size_t element_size) { + if (self->size == self->capacity) { + size_t new_capacity = self->capacity * 2; + if (new_capacity < 8) new_capacity = 8; + array__reserve(self, element_size, new_capacity); + } +} + static inline void array__splice(VoidArray *self, size_t element_size, uint32_t index, uint32_t old_count, uint32_t new_count, void *elements) { @@ -103,7 +107,7 @@ static inline void array__splice(VoidArray *self, size_t element_size, uint32_t new_end = index + new_count; assert(old_end <= self->size); - array__grow(self, element_size, new_size); + array__reserve(self, element_size, new_size); char *contents = (char *)self->contents; if (self->size > old_end) diff --git a/src/runtime/parser.c b/src/runtime/parser.c index bfb0ab37..265354a2 100644 --- a/src/runtime/parser.c +++ b/src/runtime/parser.c @@ -1041,7 +1041,7 @@ static void parser__recover(Parser *self, StackVersion version, Tree *lookahead) LOG("skip_token symbol:%s", SYM_NAME(lookahead->symbol)); ts_tree_retain(lookahead); TreeArray children = array_new(); - array_grow(&children, 1); + array_reserve(&children, 1); array_push(&children, lookahead); Tree *error_repeat = ts_tree_make_node( &self->tree_pool, @@ -1253,7 +1253,7 @@ static unsigned parser__condense_stack(Parser *self) { bool parser_init(Parser *self) { ts_lexer_init(&self->lexer); array_init(&self->reduce_actions); - array_grow(&self->reduce_actions, 4); + array_reserve(&self->reduce_actions, 4); ts_tree_pool_init(&self->tree_pool); self->stack = ts_stack_new(&self->tree_pool); self->finished_tree = NULL; diff --git a/src/runtime/stack.c b/src/runtime/stack.c index 76a6e89c..f7383846 100644 --- a/src/runtime/stack.c +++ b/src/runtime/stack.c @@ -241,7 +241,7 @@ static void ts_stack__add_slice(Stack *self, StackVersion original_version, inline StackSliceArray stack__iter(Stack *self, StackVersion version, StackCallback callback, void *payload, - bool include_trees) { + int goal_tree_count) { array_clear(&self->slices); array_clear(&self->iterators); @@ -252,6 +252,13 @@ inline StackSliceArray stack__iter(Stack *self, StackVersion version, .tree_count = 0, .is_pending = true, }; + + bool include_trees = false; + if (goal_tree_count >= 0) { + include_trees = true; + array_reserve(&iterator.trees, goal_tree_count); + } + array_push(&self->iterators, iterator); while (self->iterators.size > 0) { @@ -330,10 +337,10 @@ Stack *ts_stack_new(TreePool *tree_pool) { array_init(&self->slices); array_init(&self->iterators); array_init(&self->node_pool); - array_grow(&self->heads, 4); - array_grow(&self->slices, 4); - array_grow(&self->iterators, 4); - array_grow(&self->node_pool, MAX_NODE_POOL_SIZE); + array_reserve(&self->heads, 4); + array_reserve(&self->slices, 4); + array_reserve(&self->iterators, 4); + array_reserve(&self->node_pool, MAX_NODE_POOL_SIZE); self->tree_pool = tree_pool; self->base_node = stack_node_new(NULL, NULL, false, 1, &self->node_pool); @@ -423,7 +430,7 @@ inline StackAction iterate_callback(void *payload, const Iterator *iterator) { void ts_stack_iterate(Stack *self, StackVersion version, StackIterateCallback callback, void *payload) { StackIterateSession session = {payload, callback}; - stack__iter(self, version, iterate_callback, &session, true); + stack__iter(self, version, iterate_callback, &session, -1); } inline StackAction pop_count_callback(void *payload, const Iterator *iterator) { @@ -436,7 +443,7 @@ inline StackAction pop_count_callback(void *payload, const Iterator *iterator) { } StackSliceArray ts_stack_pop_count(Stack *self, StackVersion version, uint32_t count) { - return stack__iter(self, version, pop_count_callback, &count, true); + return stack__iter(self, version, pop_count_callback, &count, count); } inline StackAction pop_pending_callback(void *payload, const Iterator *iterator) { @@ -452,7 +459,7 @@ inline StackAction pop_pending_callback(void *payload, const Iterator *iterator) } StackSliceArray ts_stack_pop_pending(Stack *self, StackVersion version) { - StackSliceArray pop = stack__iter(self, version, pop_pending_callback, NULL, true); + StackSliceArray pop = stack__iter(self, version, pop_pending_callback, NULL, 0); if (pop.size > 0) { ts_stack_renumber_version(self, pop.contents[0].version, version); pop.contents[0].version = version; @@ -479,7 +486,7 @@ TreeArray ts_stack_pop_error(Stack *self, StackVersion version) { for (unsigned i = 0; i < node->link_count; i++) { if (node->links[i].tree && node->links[i].tree->symbol == ts_builtin_sym_error) { bool found_error = false; - StackSliceArray pop = stack__iter(self, version, pop_error_callback, &found_error, true); + StackSliceArray pop = stack__iter(self, version, pop_error_callback, &found_error, 1); if (pop.size > 0) { assert(pop.size == 1); ts_stack_renumber_version(self, pop.contents[0].version, version); @@ -496,7 +503,7 @@ inline StackAction pop_all_callback(void *payload, const Iterator *iterator) { } StackSliceArray ts_stack_pop_all(Stack *self, StackVersion version) { - return stack__iter(self, version, pop_all_callback, NULL, true); + return stack__iter(self, version, pop_all_callback, NULL, 0); } typedef struct { @@ -528,7 +535,7 @@ void ts_stack_record_summary(Stack *self, StackVersion version, unsigned max_dep .max_depth = max_depth }; array_init(session.summary); - stack__iter(self, version, summarize_stack_callback, &session, false); + stack__iter(self, version, summarize_stack_callback, &session, -1); self->heads.contents[version].summary = session.summary; } diff --git a/src/runtime/tree.c b/src/runtime/tree.c index ad58bfd3..24e62466 100644 --- a/src/runtime/tree.c +++ b/src/runtime/tree.c @@ -82,7 +82,7 @@ TreeArray ts_tree_array_remove_last_n(TreeArray *self, uint32_t remove_count) { } } - array_grow(&result, self->size - split_index); + array_reserve(&result, self->size - split_index); for (uint32_t i = split_index; i < self->size; i++) { array_push(&result, self->contents[i]); } From f00d2ade4666be0746a2c92efb3a590bed40ccb1 Mon Sep 17 00:00:00 2001 From: Max Brunsfeld Date: Mon, 9 Apr 2018 19:37:53 -0700 Subject: [PATCH 21/90] Remove unused function --- src/runtime/tree.c | 23 ----------------------- src/runtime/tree.h | 1 - 2 files changed, 24 deletions(-) diff --git a/src/runtime/tree.c b/src/runtime/tree.c index 24e62466..c1ada974 100644 --- a/src/runtime/tree.c +++ b/src/runtime/tree.c @@ -68,29 +68,6 @@ void ts_tree_array_delete(TreePool *pool, TreeArray *self) { array_delete(self); } -TreeArray ts_tree_array_remove_last_n(TreeArray *self, uint32_t remove_count) { - TreeArray result = array_new(); - if (self->size == 0 || remove_count == 0) return result; - - uint32_t count = 0; - uint32_t split_index = self->size - 1; - for (; split_index + 1 > 0; split_index--) { - Tree *tree = self->contents[split_index]; - if (!tree->extra) { - count++; - if (count == remove_count) break; - } - } - - array_reserve(&result, self->size - split_index); - for (uint32_t i = split_index; i < self->size; i++) { - array_push(&result, self->contents[i]); - } - - self->size = split_index; - return result; -} - TreeArray ts_tree_array_remove_trailing_extras(TreeArray *self) { TreeArray result = array_new(); diff --git a/src/runtime/tree.h b/src/runtime/tree.h index 5b655afb..f416516d 100644 --- a/src/runtime/tree.h +++ b/src/runtime/tree.h @@ -88,7 +88,6 @@ const char *ts_external_token_state_data(const TSExternalTokenState *); bool ts_tree_array_copy(TreeArray, TreeArray *); void ts_tree_array_delete(TreePool *, TreeArray *); -TreeArray ts_tree_array_remove_last_n(TreeArray *, uint32_t); TreeArray ts_tree_array_remove_trailing_extras(TreeArray *); void ts_tree_array_reverse(TreeArray *); From e8cfb9ced079a43da0badf04e874926c248966e0 Mon Sep 17 00:00:00 2001 From: Max Brunsfeld Date: Tue, 17 Apr 2018 17:13:22 -0700 Subject: [PATCH 22/90] Remove incorrect return statement This prevented conflicts between some tokens from being recorded properly. In the case of JavaScript, it prevented tree-sitter from recognizing the conflict between the forward slash operator and the regex token, allowing regexes to be merged into parse states containing '/' incorrectly. Refs tree-sitter/tree-sitter-javascript#71 --- src/compiler/build_tables/lex_table_builder.cc | 1 - 1 file changed, 1 deletion(-) diff --git a/src/compiler/build_tables/lex_table_builder.cc b/src/compiler/build_tables/lex_table_builder.cc index 066ddf1c..e0cab5e2 100644 --- a/src/compiler/build_tables/lex_table_builder.cc +++ b/src/compiler/build_tables/lex_table_builder.cc @@ -310,7 +310,6 @@ class LexTableBuilderImpl : public LexTableBuilder { } else { record_conflict(accept_action.symbol, advance_symbol, MatchesLongerString); } - return; } } From d5cfc06fa2f03d68b1d2f307519200d35b960af2 Mon Sep 17 00:00:00 2001 From: Max Brunsfeld Date: Tue, 17 Apr 2018 17:33:45 -0700 Subject: [PATCH 23/90] Fix unit test for invalid utf8 at EOF --- test/runtime/parser_test.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/runtime/parser_test.cc b/test/runtime/parser_test.cc index 4c0d1287..4481ece7 100644 --- a/test/runtime/parser_test.cc +++ b/test/runtime/parser_test.cc @@ -196,13 +196,13 @@ describe("Parser", [&]() { char *string = (char *)malloc(1); string[0] = '\xdf'; - ts_document_set_language(document, load_real_language("javascript")); + ts_document_set_language(document, load_real_language("json")); ts_document_set_input_string_with_length(document, string, 1); ts_document_parse(document); free(string); - assert_root_node("(program (ERROR (UNEXPECTED INVALID)))"); + assert_root_node("(ERROR (UNEXPECTED INVALID))"); }); }); From f6208435915dc776f79809cc4ab5f538e3dcfa32 Mon Sep 17 00:00:00 2001 From: Max Brunsfeld Date: Tue, 8 May 2018 11:42:15 -0700 Subject: [PATCH 24/90] Remove unused ReusableNode function --- src/runtime/parser.c | 10 ++++------ src/runtime/reusable_node.h | 11 +++++------ 2 files changed, 9 insertions(+), 12 deletions(-) diff --git a/src/runtime/parser.c b/src/runtime/parser.c index 265354a2..ff194936 100644 --- a/src/runtime/parser.c +++ b/src/runtime/parser.c @@ -396,8 +396,7 @@ static void parser__set_cached_token(Parser *self, size_t byte_index, Tree *last } static bool parser__can_reuse_first_leaf(Parser *self, TSStateId state, Tree *tree, - TableEntry *table_entry, - ReusableNode *next_reusable_node) { + TableEntry *table_entry) { TSLexMode current_lex_mode = self->language->lex_modes[state]; // If the token was created in a state with the same set of lookaheads, it is reusable. @@ -462,14 +461,13 @@ static Tree *parser__get_lookahead(Parser *self, StackVersion version, TSStateId } ts_language_table_entry(self->language, *state, result->first_leaf.symbol, table_entry); - ReusableNode next_reusable_node = reusable_node_after_leaf(reusable_node); - if (!parser__can_reuse_first_leaf(self, *state, result, table_entry, &next_reusable_node)) { + if (!parser__can_reuse_first_leaf(self, *state, result, table_entry)) { LOG( "cant_reuse_node symbol:%s, first_leaf_symbol:%s", SYM_NAME(result->symbol), SYM_NAME(result->first_leaf.symbol) ); - *reusable_node = next_reusable_node; + reusable_node_pop_leaf(reusable_node); break; } @@ -480,7 +478,7 @@ static Tree *parser__get_lookahead(Parser *self, StackVersion version, TSStateId if ((result = parser__get_cached_token(self, position.bytes, last_external_token))) { ts_language_table_entry(self->language, *state, result->first_leaf.symbol, table_entry); - if (parser__can_reuse_first_leaf(self, *state, result, table_entry, NULL)) { + if (parser__can_reuse_first_leaf(self, *state, result, table_entry)) { ts_tree_retain(result); return result; } diff --git a/src/runtime/reusable_node.h b/src/runtime/reusable_node.h index 04b9af7e..4a6290f4 100644 --- a/src/runtime/reusable_node.h +++ b/src/runtime/reusable_node.h @@ -28,12 +28,11 @@ static inline void reusable_node_pop(ReusableNode *self) { } } -static inline ReusableNode reusable_node_after_leaf(const ReusableNode *self) { - ReusableNode result = *self; - while (result.tree->children.size > 0) - result.tree = result.tree->children.contents[0]; - reusable_node_pop(&result); - return result; +static inline void reusable_node_pop_leaf(ReusableNode *self) { + while (self->tree->children.size > 0) { + self->tree = self->tree->children.contents[0]; + } + reusable_node_pop(self); } static inline bool reusable_node_breakdown(ReusableNode *self) { From 8300f24fec6cd8e1a09b7ee21cac8cce5a29b25e Mon Sep 17 00:00:00 2001 From: Max Brunsfeld Date: Fri, 20 Apr 2018 10:00:00 -0700 Subject: [PATCH 25/90] Avoid slow test setup if seed flag is set to -1 --- test/integration/real_grammars.cc | 2 ++ test/integration/test_grammars.cc | 2 ++ test/test_helper.h | 2 ++ test/tests.cc | 11 ++++++----- 4 files changed, 12 insertions(+), 5 deletions(-) diff --git a/test/integration/real_grammars.cc b/test/integration/real_grammars.cc index 37465add..e842ee0c 100644 --- a/test/integration/real_grammars.cc +++ b/test/integration/real_grammars.cc @@ -20,6 +20,8 @@ static void assert_correct_tree_size(TSDocument *document, string content) { START_TEST +if (TREE_SITTER_SEED == -1) return; + vector test_languages({ "javascript", "json", diff --git a/test/integration/test_grammars.cc b/test/integration/test_grammars.cc index 9dbefcd8..f4ba6d92 100644 --- a/test/integration/test_grammars.cc +++ b/test/integration/test_grammars.cc @@ -9,6 +9,8 @@ START_TEST +if (TREE_SITTER_SEED == -1) return; + string grammars_dir_path = join_path({"test", "fixtures", "test_grammars"}); vector test_languages = list_directory(grammars_dir_path); diff --git a/test/test_helper.h b/test/test_helper.h index 6947452b..0cb2d41c 100644 --- a/test/test_helper.h +++ b/test/test_helper.h @@ -5,6 +5,8 @@ #include "tree_sitter/compiler.h" #include "tree_sitter/runtime.h" +extern int TREE_SITTER_SEED; + namespace tree_sitter {} using namespace std; diff --git a/test/tests.cc b/test/tests.cc index cb9d6595..303f9059 100644 --- a/test/tests.cc +++ b/test/tests.cc @@ -1,17 +1,18 @@ #include "test_helper.h" #include "helpers/random_helpers.h" +int TREE_SITTER_SEED = 0; + int main(int argc, char *argv[]) { - int seed; const char *seed_env = getenv("TREE_SITTER_SEED"); if (seed_env) { - seed = atoi(seed_env); + TREE_SITTER_SEED = atoi(seed_env); } else { - seed = get_time_as_seed(); + TREE_SITTER_SEED = get_time_as_seed(); } - printf("Random seed: %d\n", seed); - random_reseed(seed); + printf("Random seed: %d\n", TREE_SITTER_SEED); + random_reseed(TREE_SITTER_SEED); return bandit::run(argc, argv); } From 973e4a44f0ade679598bc7499be2d2820bcfd355 Mon Sep 17 00:00:00 2001 From: Max Brunsfeld Date: Wed, 9 May 2018 10:16:10 -0700 Subject: [PATCH 26/90] Start work on removing parent pointers Co-Authored-By: Rick Winfrey --- include/tree_sitter/runtime.h | 8 +- src/runtime/array.h | 9 + src/runtime/document.c | 8 +- src/runtime/node.c | 307 +++++++++++------- src/runtime/node.h | 8 - src/runtime/parser.c | 54 +-- src/runtime/reusable_node.h | 95 ++++-- src/runtime/tree.c | 138 ++++---- src/runtime/tree.h | 14 +- .../prepare_grammar/extract_tokens_test.cc | 22 +- test/helpers/tree_helpers.cc | 25 +- test/runtime/document_test.cc | 2 +- test/runtime/node_test.cc | 40 +-- test/runtime/tree_test.cc | 4 +- 14 files changed, 410 insertions(+), 324 deletions(-) delete mode 100644 src/runtime/node.h diff --git a/include/tree_sitter/runtime.h b/include/tree_sitter/runtime.h index c3bef506..c4bc8ab5 100644 --- a/include/tree_sitter/runtime.h +++ b/include/tree_sitter/runtime.h @@ -63,8 +63,11 @@ typedef struct { } TSRange; typedef struct { - const void *data; - uint32_t offset[2]; + const void *subtree; + const TSDocument *document; + TSPoint position; + uint32_t byte; + TSSymbol alias_symbol; } TSNode; uint32_t ts_node_start_byte(TSNode); @@ -84,7 +87,6 @@ TSNode ts_node_child(TSNode, uint32_t); TSNode ts_node_named_child(TSNode, uint32_t); uint32_t ts_node_child_count(TSNode); uint32_t ts_node_named_child_count(TSNode); -uint32_t ts_node_child_index(TSNode); TSNode ts_node_next_sibling(TSNode); TSNode ts_node_next_named_sibling(TSNode); TSNode ts_node_prev_sibling(TSNode); diff --git a/src/runtime/array.h b/src/runtime/array.h index 4d2f478b..45b3adaa 100644 --- a/src/runtime/array.h +++ b/src/runtime/array.h @@ -58,6 +58,9 @@ extern "C" { #define array_pop(self) ((self)->contents[--(self)->size]) +#define array_assign(self, other) \ + array__assign((VoidArray *)(self), (const VoidArray *)(other), array__elem_size(self)) + // Private typedef Array(void) VoidArray; @@ -91,6 +94,12 @@ static inline void array__reserve(VoidArray *self, size_t element_size, uint32_t } } +static inline void array__assign(VoidArray *self, const VoidArray *other, size_t element_size) { + array__reserve(self, element_size, other->size); + self->size = other->size; + memcpy(self->contents, other->contents, self->size * element_size); +} + static inline void array__grow(VoidArray *self, size_t element_size) { if (self->size == self->capacity) { size_t new_capacity = self->capacity * 2; diff --git a/src/runtime/document.c b/src/runtime/document.c index d611d989..ddd89b43 100644 --- a/src/runtime/document.c +++ b/src/runtime/document.c @@ -1,5 +1,4 @@ #include "runtime/alloc.h" -#include "runtime/node.h" #include "runtime/tree.h" #include "runtime/parser.h" #include "runtime/string_input.h" @@ -171,7 +170,12 @@ void ts_document_invalidate(TSDocument *self) { } TSNode ts_document_root_node(const TSDocument *self) { - return ts_node_make(self->tree, 0, 0); + return (TSNode) { + .subtree = self->tree, + .document = self, + .position = {0, 0}, + .byte = 0, + }; } uint32_t ts_document_parse_count(const TSDocument *self) { diff --git a/src/runtime/node.c b/src/runtime/node.c index c825a104..246cd290 100644 --- a/src/runtime/node.c +++ b/src/runtime/node.c @@ -1,43 +1,91 @@ #include -#include "runtime/node.h" #include "runtime/tree.h" #include "runtime/document.h" +#include "runtime/language.h" -TSNode ts_node_make(const Tree *tree, uint32_t byte, uint32_t row) { - return (TSNode){.data = tree, .offset = { byte, row } }; -} +// NodeChildIterator -/* - * Private - */ +typedef struct { + const Tree *parent; + const TSDocument *document; + Length position; + uint32_t child_index; + uint32_t structural_child_index; + const TSSymbol *alias_sequence; +} NodeChildIterator; + +// TSNode - Private static inline TSNode ts_node__null() { - return ts_node_make(NULL, 0, 0); + return (TSNode) { + .subtree = NULL, + .document = NULL, + .position = {0, 0}, + .byte = 0, + }; } static inline const Tree *ts_node__tree(TSNode self) { - return self.data; + return self.subtree; } -static inline uint32_t ts_node__offset_byte(TSNode self) { - return self.offset[0]; +static inline NodeChildIterator ts_node_child_iterator_begin(const TSNode *node) { + const Tree *tree = ts_node__tree(*node); + const TSSymbol *alias_sequence = ts_language_alias_sequence( + node->document->parser.language, + tree->alias_sequence_id + ); + return (NodeChildIterator) { + .parent = tree, + .document = node->document, + .position = {node->byte, node->position}, + .child_index = 0, + .structural_child_index = 0, + .alias_sequence = alias_sequence, + }; } -static inline uint32_t ts_node__offset_row(TSNode self) { - return self.offset[1]; +static inline bool ts_node_child_iterator_next(NodeChildIterator *self, TSNode *result) { + if (self->child_index == self->parent->children.size) return false; + Tree *child = self->parent->children.contents[self->child_index]; + TSSymbol alias_symbol = 0; + if (!child->extra) { + if (self->alias_sequence) { + alias_symbol = self->alias_sequence[self->structural_child_index]; + } + self->structural_child_index++; + } + *result = (TSNode) { + .subtree = child, + .document = self->document, + .position = self->position.extent, + .byte = self->position.bytes, + .alias_symbol = alias_symbol, + }; + self->position = length_add(self->position, ts_tree_total_size(child)); + self->child_index++; + return true; } static inline bool ts_node__is_relevant(TSNode self, bool include_anonymous) { const Tree *tree = ts_node__tree(self); if (include_anonymous) { - return tree->context.alias_symbol || tree->visible; + return tree->visible || self.alias_symbol; } else { - return tree->context.alias_is_named || (tree->visible && tree->named); + return ( + (tree->visible && tree->named) || + ( + self.alias_symbol && + ts_language_symbol_metadata( + self.document->parser.language, + self.alias_symbol + ).named + ) + ); } } -static inline uint32_t ts_node__relevant_child_count(TSNode self, - bool include_anonymous) { +static inline uint32_t ts_node__relevant_child_count(TSNode self, bool include_anonymous) { const Tree *tree = ts_node__tree(self); if (tree->children.size > 0) { if (include_anonymous) { @@ -50,44 +98,23 @@ static inline uint32_t ts_node__relevant_child_count(TSNode self, } } -static inline TSNode ts_node__direct_parent(TSNode self, uint32_t *index) { - const Tree *tree = ts_node__tree(self); - *index = tree->context.index; - return ts_node_make( - tree->context.parent, - ts_node__offset_byte(self) - tree->context.offset.bytes, - ts_node__offset_row(self) - tree->context.offset.extent.row - ); -} - -static inline TSNode ts_node__direct_child(TSNode self, uint32_t i) { - const Tree *child_tree = ts_node__tree(self)->children.contents[i]; - return ts_node_make( - child_tree, - ts_node__offset_byte(self) + child_tree->context.offset.bytes, - ts_node__offset_row(self) + child_tree->context.offset.extent.row - ); -} - -static inline TSNode ts_node__child(TSNode self, uint32_t child_index, - bool include_anonymous) { +static inline TSNode ts_node__child(TSNode self, uint32_t child_index, bool include_anonymous) { TSNode result = self; bool did_descend = true; while (did_descend) { did_descend = false; + TSNode child; uint32_t index = 0; - for (uint32_t i = 0; i < ts_node__tree(result)->children.size; i++) { - TSNode child = ts_node__direct_child(result, i); + NodeChildIterator iterator = ts_node_child_iterator_begin(&result); + while (ts_node_child_iterator_next(&iterator, &child)) { if (ts_node__is_relevant(child, include_anonymous)) { - if (index == child_index) - return child; + if (index == child_index) return child; index++; } else { uint32_t grandchild_index = child_index - index; - uint32_t grandchild_count = - ts_node__relevant_child_count(child, include_anonymous); + uint32_t grandchild_count = ts_node__relevant_child_count(child, include_anonymous); if (grandchild_index < grandchild_count) { did_descend = true; result = child; @@ -102,48 +129,80 @@ static inline TSNode ts_node__child(TSNode self, uint32_t child_index, return ts_node__null(); } -static inline TSNode ts_node__prev_sibling(TSNode self, bool include_anonymous) { - TSNode result = self; - - do { - uint32_t index; - result = ts_node__direct_parent(result, &index); - if (!result.data) +static inline bool ts_node__last_child_before(TSNode self, TSNode target, + bool include_anonymous, TSNode *result) { + TSNode child; + TSNode earlier_child = ts_node__null(); + bool earlier_child_is_relevant = false; + bool found_child_containing_target = false; + NodeChildIterator iterator = ts_node_child_iterator_begin(&self); + uint32_t target_end_byte = ts_node_end_byte(target); + while (ts_node_child_iterator_next(&iterator, &child)) { + if (iterator.position.bytes >= target_end_byte) { + found_child_containing_target = true; break; - - for (uint32_t i = index - 1; i + 1 > 0; i--) { - TSNode child = ts_node__direct_child(result, i); - if (ts_node__is_relevant(child, include_anonymous)) - return child; - uint32_t grandchild_count = - ts_node__relevant_child_count(child, include_anonymous); - if (grandchild_count > 0) - return ts_node__child(child, grandchild_count - 1, include_anonymous); } - } while (!ts_node__tree(result)->visible); - return ts_node__null(); + if (ts_node__is_relevant(child, include_anonymous)) { + earlier_child = child; + earlier_child_is_relevant = true; + } else if (ts_node__relevant_child_count(child, include_anonymous) > 0) { + earlier_child = child; + earlier_child_is_relevant = false; + } + } + + if (found_child_containing_target && child.subtree != target.subtree) { + if (ts_node__last_child_before(child, target, include_anonymous, result)) { + return true; + } + } + + if (earlier_child_is_relevant) { + *result = earlier_child; + return true; + } + + if (earlier_child.subtree) { + return ts_node__last_child_before(earlier_child, target, include_anonymous, result); + } + + return false; +} + +static inline TSNode ts_node__prev_sibling(TSNode self, bool include_anonymous) { + TSNode result = ts_node__null(); + TSNode parent = ts_node_parent(self); + if (parent.subtree) { + ts_node__last_child_before(parent, self, include_anonymous, &result); + } + return result; } static inline TSNode ts_node__next_sibling(TSNode self, bool include_anonymous) { - TSNode result = self; + TSNode node = ts_node_parent(self); + if (!node.subtree) return ts_node__null(); + uint32_t end_byte = ts_node_end_byte(self); - do { - uint32_t index; - result = ts_node__direct_parent(result, &index); - if (!result.data) - break; + bool did_descend = true; + while (did_descend) { + did_descend = false; - for (uint32_t i = index + 1; i < ts_node__tree(result)->children.size; i++) { - TSNode child = ts_node__direct_child(result, i); - if (ts_node__is_relevant(child, include_anonymous)) - return child; - uint32_t grandchild_count = - ts_node__relevant_child_count(child, include_anonymous); - if (grandchild_count > 0) - return ts_node__child(child, 0, include_anonymous); + TSNode child; + NodeChildIterator iterator = ts_node_child_iterator_begin(&node); + while (ts_node_child_iterator_next(&iterator, &child)) { + if (iterator.position.bytes > end_byte && child.subtree != self.subtree) { + if (ts_node__is_relevant(child, include_anonymous)) { + return child; + } + if (ts_node__relevant_child_count(child, include_anonymous) > 0) { + node = child; + did_descend = true; + break; + } + } } - } while (!ts_node__tree(result)->visible); + } return ts_node__null(); } @@ -160,8 +219,9 @@ static inline TSNode ts_node__first_child_for_byte(TSNode self, uint32_t goal, while (did_descend) { did_descend = false; - for (uint32_t i = 0; i < ts_node__tree(node)->children.size; i++) { - TSNode child = ts_node__direct_child(node, i); + TSNode child; + NodeChildIterator iterator = ts_node_child_iterator_begin(&node); + while (ts_node_child_iterator_next(&iterator, &child)) { if (ts_node_end_byte(child) > goal) { if (ts_node__is_relevant(child, include_anonymous)) { return child; @@ -187,10 +247,11 @@ static inline TSNode ts_node__descendant_for_byte_range(TSNode self, uint32_t mi while (did_descend) { did_descend = false; - for (uint32_t i = 0, n = ts_node__tree(node)->children.size; i < n; i++) { - TSNode child = ts_node__direct_child(node, i); - if (ts_node_end_byte(child) > max) { - if (ts_node_start_byte(child) > min) break; + TSNode child; + NodeChildIterator iterator = ts_node_child_iterator_begin(&node); + while (ts_node_child_iterator_next(&iterator, &child)) { + if (iterator.position.bytes > max) { + if (child.byte > min) break; node = child; if (ts_node__is_relevant(node, include_anonymous)) last_visible_node = node; did_descend = true; @@ -214,10 +275,13 @@ static inline TSNode ts_node__descendant_for_point_range(TSNode self, TSPoint mi while (did_descend) { did_descend = false; - for (uint32_t i = 0, n = ts_node__tree(node)->children.size; i < n; i++) { - TSNode child = ts_node__direct_child(node, i); + TSNode child; + NodeChildIterator iterator = ts_node_child_iterator_begin(&node); + while (ts_node_child_iterator_next(&iterator, &child)) { const Tree *child_tree = ts_node__tree(child); - if (i > 0) start_position = point_add(start_position, child_tree->padding.extent); + if (iterator.child_index != 1) { + start_position = point_add(start_position, child_tree->padding.extent); + } end_position = point_add(start_position, child_tree->size.extent); if (point_gt(end_position, max)) { if (point_gt(start_position, min)) break; @@ -233,12 +297,10 @@ static inline TSNode ts_node__descendant_for_point_range(TSNode self, TSPoint mi return last_visible_node; } -/* - * Public - */ +// TSNode - Public uint32_t ts_node_start_byte(TSNode self) { - return ts_node__offset_byte(self) + ts_node__tree(self)->padding.bytes; + return self.byte + ts_node__tree(self)->padding.bytes; } uint32_t ts_node_end_byte(TSNode self) { @@ -246,21 +308,16 @@ uint32_t ts_node_end_byte(TSNode self) { } TSPoint ts_node_start_point(TSNode self) { - const Tree *tree = ts_node__tree(self); - return (TSPoint){ ts_node__offset_row(self) + tree->padding.extent.row, - ts_tree_start_column(tree) }; + return point_add(self.position, ts_node__tree(self)->padding.extent); } TSPoint ts_node_end_point(TSNode self) { - const Tree *tree = ts_node__tree(self); - return (TSPoint){ ts_node__offset_row(self) + tree->padding.extent.row + - tree->size.extent.row, - ts_tree_end_column(tree) }; + return point_add(ts_node_start_point(self), ts_node__tree(self)->size.extent); } TSSymbol ts_node_symbol(TSNode self) { const Tree *tree = ts_node__tree(self); - return tree->context.alias_symbol ? tree->context.alias_symbol : tree->symbol; + return self.alias_symbol ? self.alias_symbol : tree->symbol; } const char *ts_node_type(TSNode self, const TSDocument *document) { @@ -272,15 +329,17 @@ char *ts_node_string(TSNode self, const TSDocument *document) { } bool ts_node_eq(TSNode self, TSNode other) { - return + return ( ts_tree_eq(ts_node__tree(self), ts_node__tree(other)) && - self.offset[0] == other.offset[0] && - self.offset[1] == other.offset[1]; + self.byte == other.byte + ); } bool ts_node_is_named(TSNode self) { const Tree *tree = ts_node__tree(self); - return tree->context.alias_symbol ? tree->context.alias_is_named : tree->named; + return self.alias_symbol + ? ts_language_symbol_metadata(self.document->parser.language, self.alias_symbol).named + : tree->named; } bool ts_node_is_missing(TSNode self) { @@ -297,35 +356,31 @@ bool ts_node_has_error(TSNode self) { } TSNode ts_node_parent(TSNode self) { - TSNode result = self; - uint32_t index; + TSNode node = ts_document_root_node(self.document); + uint32_t end_byte = ts_node_end_byte(self); + if (node.subtree == self.subtree) return ts_node__null(); - do { - result = ts_node__direct_parent(result, &index); - if (!result.data) - return ts_node__null(); - } while (!ts_node__tree(result)->visible); + TSNode last_visible_node = node; + bool did_descend = true; + while (did_descend) { + did_descend = false; - return result; -} - -uint32_t ts_node_child_index(TSNode self) { - const Tree *tree = ts_node__tree(self); - uint32_t result = 0; - - for (;;) { - const Tree *parent = tree->context.parent; - uint32_t index = tree->context.index; - if (!parent) return UINT32_MAX; - for (uint32_t i = 0; i < index; i++) { - Tree *child = parent->children.contents[i]; - result += child->visible ? 1 : child->visible_child_count; + TSNode child; + NodeChildIterator iterator = ts_node_child_iterator_begin(&node); + while (ts_node_child_iterator_next(&iterator, &child)) { + if (child.byte > self.byte || child.subtree == self.subtree) break; + if (iterator.position.bytes >= end_byte) { + node = child; + if (ts_node__is_relevant(child, true)) { + last_visible_node = node; + } + did_descend = true; + break; + } } - if (parent->visible) break; - tree = parent; } - return result; + return last_visible_node; } TSNode ts_node_child(TSNode self, uint32_t child_index) { diff --git a/src/runtime/node.h b/src/runtime/node.h deleted file mode 100644 index ee184c9a..00000000 --- a/src/runtime/node.h +++ /dev/null @@ -1,8 +0,0 @@ -#ifndef RUNTIME_NODE_H_ -#define RUNTIME_NODE_H_ - -#include "runtime/tree.h" - -TSNode ts_node_make(const Tree *, uint32_t byte, uint32_t row); - -#endif diff --git a/src/runtime/parser.c b/src/runtime/parser.c index ff194936..cac6c9e1 100644 --- a/src/runtime/parser.c +++ b/src/runtime/parser.c @@ -119,16 +119,19 @@ static bool parser__breakdown_top_of_stack(Parser *self, StackVersion version) { static void parser__breakdown_lookahead(Parser *self, Tree **lookahead, TSStateId state, ReusableNode *reusable_node) { - bool did_break_down = false; - while (reusable_node->tree->children.size > 0 && reusable_node->tree->parse_state != state) { - LOG("state_mismatch sym:%s", SYM_NAME(reusable_node->tree->symbol)); - reusable_node_breakdown(reusable_node); - did_break_down = true; + bool did_descend = false; + Tree *tree = reusable_node_tree(reusable_node); + while (tree->children.size > 0 && tree->parse_state != state) { + LOG("state_mismatch sym:%s", SYM_NAME(tree->symbol)); + reusable_node_descend(reusable_node); + tree = reusable_node_tree(reusable_node); + did_descend = true; } - if (did_break_down) { + if (did_descend) { ts_tree_release(&self->tree_pool, *lookahead); - ts_tree_retain(*lookahead = reusable_node->tree); + *lookahead = tree; + ts_tree_retain(*lookahead); } } @@ -419,21 +422,22 @@ static Tree *parser__get_lookahead(Parser *self, StackVersion version, TSStateId Tree *last_external_token = ts_stack_last_external_token(self->stack, version); Tree *result; - while ((result = reusable_node->tree)) { - if (reusable_node->byte_index > position.bytes) { + while ((result = reusable_node_tree(reusable_node))) { + uint32_t byte_offset = reusable_node_byte_offset(reusable_node); + if (byte_offset > position.bytes) { LOG("before_reusable_node symbol:%s", SYM_NAME(result->symbol)); break; } - if (reusable_node->byte_index < position.bytes) { + if (byte_offset < position.bytes) { LOG("past_reusable_node symbol:%s", SYM_NAME(result->symbol)); - reusable_node_pop(reusable_node); + reusable_node_advance(reusable_node); continue; } if (!ts_tree_external_token_state_eq(reusable_node->last_external_token, last_external_token)) { LOG("reusable_node_has_different_external_scanner_state symbol:%s", SYM_NAME(result->symbol)); - reusable_node_pop(reusable_node); + reusable_node_advance(reusable_node); continue; } @@ -452,8 +456,8 @@ static Tree *parser__get_lookahead(Parser *self, StackVersion version, TSStateId if (reason) { LOG("cant_reuse_node_%s tree:%s", reason, SYM_NAME(result->symbol)); - if (!reusable_node_breakdown(reusable_node)) { - reusable_node_pop(reusable_node); + if (!reusable_node_descend(reusable_node)) { + reusable_node_advance(reusable_node); parser__breakdown_top_of_stack(self, version); *state = ts_stack_state(self->stack, version); } @@ -467,7 +471,7 @@ static Tree *parser__get_lookahead(Parser *self, StackVersion version, TSStateId SYM_NAME(result->symbol), SYM_NAME(result->first_leaf.symbol) ); - reusable_node_pop_leaf(reusable_node); + reusable_node_advance_past_leaf(reusable_node); break; } @@ -678,7 +682,7 @@ static void parser__start(Parser *self, TSInput input, Tree *previous_tree) { ts_lexer_set_input(&self->lexer, input); ts_stack_clear(self->stack); - self->reusable_node = reusable_node_new(previous_tree); + reusable_node_reset(&self->reusable_node, previous_tree); self->finished_tree = NULL; self->accept_count = 0; self->in_ambiguity = false; @@ -1106,7 +1110,9 @@ static void parser__advance(Parser *self, StackVersion version, ReusableNode *re } parser__shift(self, version, next_state, lookahead, action.params.extra); - if (lookahead == reusable_node->tree) reusable_node_pop(reusable_node); + if (lookahead == reusable_node_tree(reusable_node)) { + reusable_node_advance(reusable_node); + } ts_tree_release(&self->tree_pool, lookahead); return; } @@ -1136,7 +1142,9 @@ static void parser__advance(Parser *self, StackVersion version, ReusableNode *re parser__breakdown_lookahead(self, &lookahead, state, reusable_node); } parser__recover(self, version, lookahead); - if (lookahead == reusable_node->tree) reusable_node_pop(reusable_node); + if (lookahead == reusable_node_tree(reusable_node)) { + reusable_node_advance(reusable_node); + } ts_tree_release(&self->tree_pool, lookahead); return; } @@ -1255,6 +1263,7 @@ bool parser_init(Parser *self) { ts_tree_pool_init(&self->tree_pool); self->stack = ts_stack_new(&self->tree_pool); self->finished_tree = NULL; + self->reusable_node = reusable_node_new(); parser__set_cached_token(self, 0, NULL, NULL); return true; } @@ -1277,6 +1286,7 @@ void parser_destroy(Parser *self) { if (self->reduce_actions.contents) array_delete(&self->reduce_actions); ts_tree_pool_delete(&self->tree_pool); + reusable_node_delete(&self->reusable_node); parser_set_language(self, NULL); } @@ -1285,11 +1295,12 @@ Tree *parser_parse(Parser *self, TSInput input, Tree *old_tree, bool halt_on_err StackVersion version = STACK_VERSION_NONE; uint32_t position = 0, last_position = 0; - ReusableNode reusable_node; + ReusableNode reusable_node = reusable_node_new(); + reusable_node_assign(&reusable_node, &self->reusable_node); do { for (version = 0; version < ts_stack_version_count(self->stack); version++) { - reusable_node = self->reusable_node; + reusable_node_assign(&reusable_node, &self->reusable_node); while (ts_stack_is_active(self->stack, version)) { LOG("process version:%d, version_count:%u, state:%d, row:%u, col:%u", @@ -1309,7 +1320,7 @@ Tree *parser_parse(Parser *self, TSInput input, Tree *old_tree, bool halt_on_err } } - self->reusable_node = reusable_node; + reusable_node_assign(&self->reusable_node, &reusable_node); unsigned min_error_cost = parser__condense_stack(self); if (self->finished_tree && self->finished_tree->error_cost < min_error_cost) { @@ -1322,6 +1333,7 @@ Tree *parser_parse(Parser *self, TSInput input, Tree *old_tree, bool halt_on_err self->in_ambiguity = version > 1; } while (version != 0); + reusable_node_delete(&reusable_node); ts_stack_clear(self->stack); parser__set_cached_token(self, 0, NULL, NULL); ts_tree_assign_parents(self->finished_tree, &self->tree_pool, self->language); diff --git a/src/runtime/reusable_node.h b/src/runtime/reusable_node.h index 4a6290f4..803304de 100644 --- a/src/runtime/reusable_node.h +++ b/src/runtime/reusable_node.h @@ -2,44 +2,81 @@ typedef struct { Tree *tree; - uint32_t byte_index; + uint32_t child_index; + uint32_t byte_offset; +} StackEntry; + +typedef struct { + Array(StackEntry) stack; Tree *last_external_token; } ReusableNode; -static inline ReusableNode reusable_node_new(Tree *tree) { - ReusableNode result = {tree, 0, NULL}; - return result; +static inline ReusableNode reusable_node_new() { + return (ReusableNode) {array_new(), NULL}; } -static inline void reusable_node_pop(ReusableNode *self) { - self->byte_index += ts_tree_total_bytes(self->tree); - if (self->tree->has_external_tokens) { - self->last_external_token = ts_tree_last_external_token(self->tree); - } - - while (self->tree) { - Tree *parent = self->tree->context.parent; - uint32_t next_index = self->tree->context.index + 1; - if (parent && parent->children.size > next_index) { - self->tree = parent->children.contents[next_index]; - return; - } - self->tree = parent; - } +static inline void reusable_node_reset(ReusableNode *self, Tree *tree) { + array_clear(&self->stack); + array_push(&self->stack, ((StackEntry) { + .tree = tree, + .child_index = 0, + .byte_offset = 0, + })); } -static inline void reusable_node_pop_leaf(ReusableNode *self) { - while (self->tree->children.size > 0) { - self->tree = self->tree->children.contents[0]; - } - reusable_node_pop(self); +static inline Tree *reusable_node_tree(ReusableNode *self) { + return array_back(&self->stack)->tree; } -static inline bool reusable_node_breakdown(ReusableNode *self) { - if (self->tree->children.size == 0) { - return false; - } else { - self->tree = self->tree->children.contents[0]; +static inline uint32_t reusable_node_byte_offset(ReusableNode *self) { + return array_back(&self->stack)->byte_offset; +} + +static inline void reusable_node_delete(ReusableNode *self) { + array_delete(&self->stack); +} + +static inline void reusable_node_assign(ReusableNode *self, const ReusableNode *other) { + array_assign(&self->stack, &other->stack); +} + +static inline void reusable_node_advance(ReusableNode *self) { + StackEntry last_entry = *array_back(&self->stack); + uint32_t byte_offset = last_entry.byte_offset + ts_tree_total_bytes(last_entry.tree); + if (last_entry.tree->has_external_tokens) { + self->last_external_token = ts_tree_last_external_token(last_entry.tree); + } + + Tree *tree; + uint32_t next_index; + do { + StackEntry popped_entry = array_pop(&self->stack); + next_index = popped_entry.child_index + 1; + tree = array_back(&self->stack)->tree; + } while (tree->children.size <= next_index); + + array_push(&self->stack, ((StackEntry) { + .tree = tree->children.contents[next_index], + .child_index = next_index, + .byte_offset = byte_offset, + })); +} + +static inline bool reusable_node_descend(ReusableNode *self) { + StackEntry last_entry = *array_back(&self->stack); + if (last_entry.tree->children.size > 0) { + array_push(&self->stack, ((StackEntry) { + .tree = last_entry.tree->children.contents[0], + .child_index = 0, + .byte_offset = last_entry.byte_offset, + })); return true; + } else { + return false; } } + +static inline void reusable_node_advance_past_leaf(ReusableNode *self) { + while (reusable_node_descend(self)) {} + reusable_node_advance(self); +} diff --git a/src/runtime/tree.c b/src/runtime/tree.c index c1ada974..fcb534dd 100644 --- a/src/runtime/tree.c +++ b/src/runtime/tree.c @@ -169,7 +169,9 @@ Tree *ts_tree_make_copy(TreePool *pool, Tree *self) { return result; } -static void ts_tree__compress(Tree *self, unsigned count, const TSLanguage *language) { +static void ts_tree__compress(Tree *self, unsigned count, const TSLanguage *language, TreeArray *stack) { + unsigned initial_stack_size = stack->size; + Tree *tree = self; for (unsigned i = 0; i < count; i++) { if (tree->ref_count > 1 || tree->children.size != 2) break; @@ -189,22 +191,14 @@ static void ts_tree__compress(Tree *self, unsigned count, const TSLanguage *lang ) break; tree->children.contents[0] = grandchild; - grandchild->context.parent = tree; - grandchild->context.index = -1; - child->children.contents[0] = grandchild->children.contents[1]; - child->children.contents[0]->context.parent = child; - child->children.contents[0]->context.index = -1; - grandchild->children.contents[1] = child; - grandchild->children.contents[1]->context.parent = grandchild; - grandchild->children.contents[1]->context.index = -1; - + array_push(stack, tree); tree = grandchild; } - while (tree != self) { - tree = tree->context.parent; + while (stack->size > initial_stack_size) { + tree = array_pop(stack); Tree *child = tree->children.contents[0]; Tree *grandchild = child->children.contents[1]; ts_tree_set_children(grandchild, &grandchild->children, language); @@ -213,50 +207,30 @@ static void ts_tree__compress(Tree *self, unsigned count, const TSLanguage *lang } } -void ts_tree__balance(Tree *self, const TSLanguage *language) { - if (self->children.contents[0]->repeat_depth > self->children.contents[1]->repeat_depth) { - unsigned n = self->children.contents[0]->repeat_depth - self->children.contents[1]->repeat_depth; - for (unsigned i = n / 2; i > 0; i /= 2) { - ts_tree__compress(self, i, language); - n -= i; - } - } -} - void ts_tree_assign_parents(Tree *self, TreePool *pool, const TSLanguage *language) { - self->context.parent = NULL; array_clear(&pool->tree_stack); array_push(&pool->tree_stack, self); while (pool->tree_stack.size > 0) { Tree *tree = array_pop(&pool->tree_stack); if (tree->repeat_depth > 0) { - ts_tree__balance(tree, language); + if (tree->children.contents[0]->repeat_depth > tree->children.contents[1]->repeat_depth) { + unsigned n = ( + tree->children.contents[0]->repeat_depth - + tree->children.contents[1]->repeat_depth + ); + for (unsigned i = n / 2; i > 0; i /= 2) { + ts_tree__compress(tree, i, language, &pool->tree_stack); + n -= i; + } + } } - Length offset = length_zero(); - const TSSymbol *alias_sequence = ts_language_alias_sequence(language, tree->alias_sequence_id); - uint32_t non_extra_index = 0; - bool earlier_child_was_changed = false; for (uint32_t i = 0; i < tree->children.size; i++) { Tree *child = tree->children.contents[i]; - if (earlier_child_was_changed || child->context.parent != tree || child->context.index != i) { - earlier_child_was_changed = true; - child->context.parent = tree; - child->context.index = i; - child->context.offset = offset; - if (!child->extra && alias_sequence && alias_sequence[non_extra_index] != 0) { - TSSymbolMetadata metadata = ts_language_symbol_metadata(language, alias_sequence[non_extra_index]); - child->context.alias_symbol = alias_sequence[non_extra_index]; - child->context.alias_is_named = metadata.named; - } else { - child->context.alias_symbol = 0; - child->context.alias_is_named = false; - } + if (child->ref_count == 1) { array_push(&pool->tree_stack, child); } - offset = length_add(offset, ts_tree_total_size(child)); - if (!child->extra) non_extra_index++; } } } @@ -407,25 +381,6 @@ void ts_tree_release(TreePool *pool, Tree *self) { } } -uint32_t ts_tree_start_column(const Tree *self) { - uint32_t column = self->padding.extent.column; - if (self->padding.extent.row > 0) - return column; - for (const Tree *tree = self; tree != NULL; tree = tree->context.parent) { - column += tree->context.offset.extent.column; - if (tree->context.offset.extent.row > 0) - break; - } - return column; -} - -uint32_t ts_tree_end_column(const Tree *self) { - uint32_t result = self->size.extent.column; - if (self->size.extent.row == 0) - result += ts_tree_start_column(self); - return result; -} - bool ts_tree_eq(const Tree *self, const Tree *other) { if (self) { if (!other) return false; @@ -577,7 +532,6 @@ void ts_tree_edit(Tree *self, const TSInputEdit *edit) { } child_right = length_add(child_left, ts_tree_total_size(child)); - child->context.offset = child_left; } } @@ -612,9 +566,10 @@ static size_t ts_tree__write_char_to_string(char *s, size_t n, int32_t c) { return snprintf(s, n, "%d", c); } -static size_t ts_tree__write_to_string(const Tree *self, const TSLanguage *language, - char *string, size_t limit, bool is_root, - bool include_all) { +static size_t ts_tree__write_to_string(const Tree *self, char *string, size_t limit, + const TSLanguage *language, bool is_root, + bool include_all, TSSymbol alias_symbol, + bool alias_is_named) { if (!self) return snprintf(string, limit, "(NULL)"); char *cursor = string; @@ -624,7 +579,7 @@ static size_t ts_tree__write_to_string(const Tree *self, const TSLanguage *langu is_root || self->is_missing || (self->visible && self->named) || - self->context.alias_is_named; + alias_is_named; if (visible && !is_root) { cursor += snprintf(*writer, limit, " "); @@ -637,15 +592,35 @@ static size_t ts_tree__write_to_string(const Tree *self, const TSLanguage *langu } else if (self->is_missing) { cursor += snprintf(*writer, limit, "(MISSING"); } else { - TSSymbol symbol = self->context.alias_symbol ? self->context.alias_symbol : self->symbol; + TSSymbol symbol = alias_symbol ? alias_symbol : self->symbol; const char *symbol_name = ts_language_symbol_name(language, symbol); cursor += snprintf(*writer, limit, "(%s", symbol_name); } } + const TSSymbol *alias_sequence = ts_language_alias_sequence(language, self->alias_sequence_id); + uint32_t structural_child_index = 0; for (uint32_t i = 0; i < self->children.size; i++) { Tree *child = self->children.contents[i]; - cursor += ts_tree__write_to_string(child, language, *writer, limit, false, include_all); + if (child->extra) { + cursor += ts_tree__write_to_string( + child, *writer, limit, + language, false, include_all, + 0, false + ); + } else { + cursor += ts_tree__write_to_string( + child, *writer, limit, + language, false, include_all, + alias_sequence + ? alias_sequence[structural_child_index] + : 0, + alias_sequence + ? ts_language_symbol_metadata(language, alias_sequence[structural_child_index]).named + : false + ); + structural_child_index++; + } } if (visible) cursor += snprintf(*writer, limit, ")"); @@ -655,15 +630,19 @@ static size_t ts_tree__write_to_string(const Tree *self, const TSLanguage *langu char *ts_tree_string(const Tree *self, const TSLanguage *language, bool include_all) { char scratch_string[1]; - size_t size = ts_tree__write_to_string(self, language, scratch_string, 0, true, include_all) + 1; + size_t size = ts_tree__write_to_string( + self, scratch_string, 0, + language, true, + include_all, 0, false + ) + 1; char *result = ts_malloc(size * sizeof(char)); - ts_tree__write_to_string(self, language, result, size, true, include_all); + ts_tree__write_to_string(self, result, size, language, true, include_all, 0, false); return result; } void ts_tree__print_dot_graph(const Tree *self, uint32_t byte_offset, - const TSLanguage *language, FILE *f) { - TSSymbol symbol = self->context.alias_symbol ? self->context.alias_symbol : self->symbol; + const TSLanguage *language, TSSymbol alias_symbol, FILE *f) { + TSSymbol symbol = alias_symbol ? alias_symbol : self->symbol; fprintf(f, "tree_%p [label=\"%s\"", self, ts_language_symbol_name(language, symbol)); if (self->children.size == 0) @@ -674,9 +653,18 @@ void ts_tree__print_dot_graph(const Tree *self, uint32_t byte_offset, fprintf(f, ", tooltip=\"address:%p\nrange:%u - %u\nstate:%d\nerror-cost:%u\nrepeat-depth:%u\"]\n", self, byte_offset, byte_offset + ts_tree_total_bytes(self), self->parse_state, self->error_cost, self->repeat_depth); + + const TSSymbol *alias_sequence = ts_language_alias_sequence(language, self->alias_sequence_id); + uint32_t structural_child_index = 0; for (uint32_t i = 0; i < self->children.size; i++) { const Tree *child = self->children.contents[i]; - ts_tree__print_dot_graph(child, byte_offset, language, f); + if (child->extra) { + ts_tree__print_dot_graph(child, byte_offset, language, 0, f); + } else { + TSSymbol alias_symbol = alias_sequence ? alias_sequence[structural_child_index] : 0; + ts_tree__print_dot_graph(child, byte_offset, language, alias_symbol, f); + structural_child_index++; + } fprintf(f, "tree_%p -> tree_%p [tooltip=%u]\n", self, child, i); byte_offset += ts_tree_total_bytes(child); } @@ -685,7 +673,7 @@ void ts_tree__print_dot_graph(const Tree *self, uint32_t byte_offset, void ts_tree_print_dot_graph(const Tree *self, const TSLanguage *language, FILE *f) { fprintf(f, "digraph tree {\n"); fprintf(f, "edge [arrowhead=none]\n"); - ts_tree__print_dot_graph(self, 0, language, f); + ts_tree__print_dot_graph(self, 0, language, 0, f); fprintf(f, "}\n"); } diff --git a/src/runtime/tree.h b/src/runtime/tree.h index f416516d..da5f1e90 100644 --- a/src/runtime/tree.h +++ b/src/runtime/tree.h @@ -27,14 +27,6 @@ typedef struct Tree Tree; typedef Array(Tree *) TreeArray; struct Tree { - struct { - struct Tree *parent; - uint32_t index; - Length offset; - TSSymbol alias_symbol : 15; - bool alias_is_named : 1; - } context; - Length padding; Length size; uint32_t ref_count; @@ -106,8 +98,6 @@ void ts_tree_retain(Tree *tree); void ts_tree_release(TreePool *, Tree *tree); bool ts_tree_eq(const Tree *tree1, const Tree *tree2); int ts_tree_compare(const Tree *tree1, const Tree *tree2); -uint32_t ts_tree_start_column(const Tree *self); -uint32_t ts_tree_end_column(const Tree *self); void ts_tree_set_children(Tree *, TreeArray *, const TSLanguage *); void ts_tree_assign_parents(Tree *, TreePool *, const TSLanguage *); void ts_tree_edit(Tree *, const TSInputEdit *edit); @@ -120,6 +110,10 @@ static inline uint32_t ts_tree_total_bytes(const Tree *self) { return self->padding.bytes + self->size.bytes; } +static inline uint32_t ts_tree_total_rows(const Tree *self) { + return self->padding.extent.row + self->size.extent.row; +} + static inline Length ts_tree_total_size(const Tree *self) { return length_add(self->padding, self->size); } diff --git a/test/compiler/prepare_grammar/extract_tokens_test.cc b/test/compiler/prepare_grammar/extract_tokens_test.cc index 0f9be780..da2ae60a 100644 --- a/test/compiler/prepare_grammar/extract_tokens_test.cc +++ b/test/compiler/prepare_grammar/extract_tokens_test.cc @@ -48,7 +48,7 @@ describe("extract_tokens", []() { Repeat{Symbol::non_terminal(3)} }, }, - {}, {}, {}, {} + {}, {}, {}, {}, {} }); InitialSyntaxGrammar &syntax_grammar = get<0>(result); @@ -156,7 +156,7 @@ describe("extract_tokens", []() { }) }, }, - {}, {}, {}, {} + {}, {}, {}, {}, {} }); InitialSyntaxGrammar &syntax_grammar = get<0>(result); @@ -203,7 +203,7 @@ describe("extract_tokens", []() { Rule::seq({ String{"ef"}, String{"cd"} }) }, }, - {}, {}, {}, {} + {}, {}, {}, {}, {} }); InitialSyntaxGrammar &syntax_grammar = get<0>(result); @@ -258,7 +258,7 @@ describe("extract_tokens", []() { String{"a"} }, }, - {}, {}, {}, {} + {}, {}, {}, {}, {} }); InitialSyntaxGrammar &syntax_grammar = get<0>(result); @@ -298,7 +298,7 @@ describe("extract_tokens", []() { { { Symbol::non_terminal(2), Symbol::non_terminal(3) } }, - {}, {} + {}, {}, {} }); InitialSyntaxGrammar &syntax_grammar = get<0>(result); @@ -319,7 +319,7 @@ describe("extract_tokens", []() { String{"y"}, Pattern{" "}, }, - {}, {}, {} + {}, {}, {}, {} }); AssertThat(get<2>(result), Equals(CompileError::none())); @@ -340,7 +340,7 @@ describe("extract_tokens", []() { { String{"y"}, }, - {}, {}, {} + {}, {}, {}, {} }); AssertThat(get<2>(result), Equals(CompileError::none())); @@ -370,7 +370,7 @@ describe("extract_tokens", []() { { Symbol::non_terminal(2), }, - {}, {}, {} + {}, {}, {}, {} }); AssertThat(get<2>(result), Equals(CompileError::none())); @@ -399,7 +399,7 @@ describe("extract_tokens", []() { { Symbol::non_terminal(1) }, - {}, {}, {} + {}, {}, {}, {} }); AssertThat(get<2>(result), Equals(CompileError( @@ -417,7 +417,7 @@ describe("extract_tokens", []() { { Rule::choice({ Symbol::non_terminal(1), Blank{} }) }, - {}, {}, {} + {}, {}, {}, {} }); AssertThat(get<2>(result), Equals(CompileError( @@ -446,7 +446,7 @@ describe("extract_tokens", []() { { Variable{"rule_A", VariableTypeNamed, Symbol::non_terminal(0)} }, - {} + {}, {} }); AssertThat(get<2>(result), Equals(CompileError( diff --git a/test/helpers/tree_helpers.cc b/test/helpers/tree_helpers.cc index 5142ca8d..8fedb560 100644 --- a/test/helpers/tree_helpers.cc +++ b/test/helpers/tree_helpers.cc @@ -1,8 +1,7 @@ -#include "bandit/bandit.h" +#include "test_helper.h" #include "helpers/tree_helpers.h" #include "helpers/point_helpers.h" #include "runtime/document.h" -#include "runtime/node.h" #include using std::string; @@ -21,24 +20,30 @@ TreeArray *tree_array(std::vector trees) { result.capacity = trees.size(); result.size = trees.size(); result.contents = (Tree **)calloc(trees.size(), sizeof(Tree *)); - for (size_t i = 0; i < trees.size(); i++) + for (size_t i = 0; i < trees.size(); i++) { result.contents[i] = trees[i]; + } return &result; } ostream &operator<<(std::ostream &stream, const Tree *tree) { static TSLanguage DUMMY_LANGUAGE = {}; - static TSDocument DUMMY_DOCUMENT = {}; - DUMMY_DOCUMENT.parser.language = &DUMMY_LANGUAGE; DUMMY_LANGUAGE.symbol_names = symbol_names; - TSNode node; - node.data = tree; - return stream << string(ts_node_string(node, &DUMMY_DOCUMENT)); + char *string = ts_tree_string(tree, &DUMMY_LANGUAGE, false); + stream << string; + ts_free(string); + return stream; } ostream &operator<<(ostream &stream, const TSNode &node) { - return stream << string("{") << (const Tree *)node.data << - string(", ") << to_string(ts_node_start_byte(node)) << string("}"); + if (node.subtree) { + char *string = ts_node_string(node, node.document); + stream << "{" << string << ", " << to_string(ts_node_start_byte(node)) << "}"; + ts_free(string); + return stream; + } else { + return stream << "NULL"; + } } bool operator==(const TSNode &left, const TSNode &right) { diff --git a/test/runtime/document_test.cc b/test/runtime/document_test.cc index 0be03657..97aa4fe2 100644 --- a/test/runtime/document_test.cc +++ b/test/runtime/document_test.cc @@ -182,7 +182,7 @@ describe("Document", [&]() { ts_document_parse(document); ts_document_set_language(document, load_real_language("javascript")); - AssertThat(ts_document_root_node(document).data, Equals(nullptr)); + AssertThat(ts_document_root_node(document).subtree, Equals(nullptr)); ts_document_parse(document); root = ts_document_root_node(document); diff --git a/test/runtime/node_test.cc b/test/runtime/node_test.cc index ff12169f..22196ec5 100644 --- a/test/runtime/node_test.cc +++ b/test/runtime/node_test.cc @@ -71,6 +71,7 @@ describe("Node", [&]() { document = ts_document_new(); ts_document_set_language(document, load_real_language("json")); ts_document_set_input_string(document, json_string.c_str()); + // ts_document_print_debugging_graphs(document, true); ts_document_parse(document); root_node = ts_node_child(ts_document_root_node(document), 0); }); @@ -157,7 +158,7 @@ describe("Node", [&]() { AssertThat(ts_node_parent(number_node), Equals(root_node)); AssertThat(ts_node_parent(false_node), Equals(root_node)); AssertThat(ts_node_parent(object_node), Equals(root_node)); - AssertThat(ts_node_parent(ts_document_root_node(document)).data, Equals(nullptr)); + AssertThat(ts_node_parent(ts_document_root_node(document)).subtree, Equals(nullptr)); }); it("works correctly when the node contains aliased children and extras", [&]() { @@ -239,19 +240,7 @@ describe("Node", [&]() { child = ts_node_first_named_child_for_byte(root_node, object_index + 1); AssertThat(ts_node_type(child, document), Equals("object")); child = ts_node_first_named_child_for_byte(root_node, object_end_index); - AssertThat(child.data, Equals(nullptr)); - }); - }); - - describe("child_index()", [&]() { - it("returns the index of the node within its parent", [&]() { - AssertThat(ts_node_child_index(ts_node_child(root_node, 0)), Equals(0u)); - AssertThat(ts_node_child_index(ts_node_child(root_node, 1)), Equals(1u)); - AssertThat(ts_node_child_index(ts_node_child(root_node, 2)), Equals(2u)); - AssertThat(ts_node_child_index(ts_node_child(root_node, 3)), Equals(3u)); - AssertThat(ts_node_child_index(ts_node_child(root_node, 4)), Equals(4u)); - AssertThat(ts_node_child_index(ts_node_child(root_node, 5)), Equals(5u)); - AssertThat(ts_node_child_index(ts_node_child(root_node, 6)), Equals(6u)); + AssertThat(child.subtree, Equals(nullptr)); }); }); @@ -335,7 +324,7 @@ describe("Node", [&]() { AssertThat(ts_node_parent(child5), Equals(root_node)); AssertThat(ts_node_parent(child6), Equals(root_node)); AssertThat(ts_node_parent(child7), Equals(root_node)); - AssertThat(ts_node_parent(ts_document_root_node(document)).data, Equals(nullptr)); + AssertThat(ts_node_parent(ts_document_root_node(document)).subtree, Equals(nullptr)); }); }); @@ -355,15 +344,16 @@ describe("Node", [&]() { TSNode brace_node2 = ts_node_child(object_node, 2); TSNode bracket_node2 = ts_node_child(root_node, 6); + AssertThat(ts_node_parent(bracket_node1), Equals(root_node)); AssertThat(ts_node_next_sibling(bracket_node1), Equals(number_node)); AssertThat(ts_node_next_sibling(number_node), Equals(array_comma_node1)); AssertThat(ts_node_next_sibling(array_comma_node1), Equals(false_node)); AssertThat(ts_node_next_sibling(false_node), Equals(array_comma_node2)); AssertThat(ts_node_next_sibling(array_comma_node2), Equals(object_node)); AssertThat(ts_node_next_sibling(object_node), Equals(bracket_node2)); - AssertThat(ts_node_next_sibling(bracket_node2).data, Equals(nullptr)); + AssertThat(ts_node_next_sibling(bracket_node2).subtree, Equals(nullptr)); - AssertThat(ts_node_prev_sibling(bracket_node1).data, Equals(nullptr)); + AssertThat(ts_node_prev_sibling(bracket_node1).subtree, Equals(nullptr)); AssertThat(ts_node_prev_sibling(number_node), Equals(bracket_node1)); AssertThat(ts_node_prev_sibling(array_comma_node1), Equals(number_node)); AssertThat(ts_node_prev_sibling(false_node), Equals(array_comma_node1)); @@ -373,24 +363,24 @@ describe("Node", [&]() { AssertThat(ts_node_next_sibling(brace_node1), Equals(pair_node)); AssertThat(ts_node_next_sibling(pair_node), Equals(brace_node2)); - AssertThat(ts_node_next_sibling(brace_node2).data, Equals(nullptr)); + AssertThat(ts_node_next_sibling(brace_node2).subtree, Equals(nullptr)); - AssertThat(ts_node_prev_sibling(brace_node1).data, Equals(nullptr)); + AssertThat(ts_node_prev_sibling(brace_node1).subtree, Equals(nullptr)); AssertThat(ts_node_prev_sibling(pair_node), Equals(brace_node1)); AssertThat(ts_node_prev_sibling(brace_node2), Equals(pair_node)); AssertThat(ts_node_next_sibling(string_node), Equals(colon_node)); AssertThat(ts_node_next_sibling(colon_node), Equals(null_node)); - AssertThat(ts_node_next_sibling(null_node).data, Equals(nullptr)); + AssertThat(ts_node_next_sibling(null_node).subtree, Equals(nullptr)); - AssertThat(ts_node_prev_sibling(string_node).data, Equals(nullptr)); + AssertThat(ts_node_prev_sibling(string_node).subtree, Equals(nullptr)); AssertThat(ts_node_prev_sibling(colon_node), Equals(string_node)); AssertThat(ts_node_prev_sibling(null_node), Equals(colon_node)); }); it("returns null when the node has no parent", [&]() { - AssertThat(ts_node_next_named_sibling(root_node).data, Equals(nullptr)); - AssertThat(ts_node_prev_named_sibling(root_node).data, Equals(nullptr)); + AssertThat(ts_node_next_named_sibling(root_node).subtree, Equals(nullptr)); + AssertThat(ts_node_prev_named_sibling(root_node).subtree, Equals(nullptr)); }); }); @@ -412,8 +402,8 @@ describe("Node", [&]() { }); it("returns null when the node has no parent", [&]() { - AssertThat(ts_node_next_named_sibling(root_node).data, Equals(nullptr)); - AssertThat(ts_node_prev_named_sibling(root_node).data, Equals(nullptr)); + AssertThat(ts_node_next_named_sibling(root_node).subtree, Equals(nullptr)); + AssertThat(ts_node_prev_named_sibling(root_node).subtree, Equals(nullptr)); }); }); diff --git a/test/runtime/tree_test.cc b/test/runtime/tree_test.cc index 8669b6c1..262572b1 100644 --- a/test/runtime/tree_test.cc +++ b/test/runtime/tree_test.cc @@ -5,14 +5,12 @@ #include "runtime/length.h" void assert_consistent(const Tree *tree) { - if (tree->child_count == 0) - return; + if (tree->child_count == 0) return; AssertThat(tree->children.contents[0]->padding, Equals(tree->padding)); Length total_children_size = length_zero(); for (size_t i = 0; i < tree->children.size; i++) { Tree *child = tree->children.contents[i]; - AssertThat(child->context.offset, Equals(total_children_size)); assert_consistent(child); total_children_size = length_add(total_children_size, ts_tree_total_size(child)); } From f857d64d54621f0ef74c92f1d235b8e93f2c2cd1 Mon Sep 17 00:00:00 2001 From: Max Brunsfeld Date: Wed, 9 May 2018 13:23:47 -0700 Subject: [PATCH 27/90] Avoid recursion in ts_node__prev_sibling --- src/runtime/node.c | 91 +++++++++++++++++++++++----------------------- 1 file changed, 45 insertions(+), 46 deletions(-) diff --git a/src/runtime/node.c b/src/runtime/node.c index 246cd290..2230d1af 100644 --- a/src/runtime/node.c +++ b/src/runtime/node.c @@ -129,54 +129,53 @@ static inline TSNode ts_node__child(TSNode self, uint32_t child_index, bool incl return ts_node__null(); } -static inline bool ts_node__last_child_before(TSNode self, TSNode target, - bool include_anonymous, TSNode *result) { - TSNode child; - TSNode earlier_child = ts_node__null(); - bool earlier_child_is_relevant = false; - bool found_child_containing_target = false; - NodeChildIterator iterator = ts_node_child_iterator_begin(&self); - uint32_t target_end_byte = ts_node_end_byte(target); - while (ts_node_child_iterator_next(&iterator, &child)) { - if (iterator.position.bytes >= target_end_byte) { - found_child_containing_target = true; - break; - } - - if (ts_node__is_relevant(child, include_anonymous)) { - earlier_child = child; - earlier_child_is_relevant = true; - } else if (ts_node__relevant_child_count(child, include_anonymous) > 0) { - earlier_child = child; - earlier_child_is_relevant = false; - } - } - - if (found_child_containing_target && child.subtree != target.subtree) { - if (ts_node__last_child_before(child, target, include_anonymous, result)) { - return true; - } - } - - if (earlier_child_is_relevant) { - *result = earlier_child; - return true; - } - - if (earlier_child.subtree) { - return ts_node__last_child_before(earlier_child, target, include_anonymous, result); - } - - return false; -} - static inline TSNode ts_node__prev_sibling(TSNode self, bool include_anonymous) { - TSNode result = ts_node__null(); - TSNode parent = ts_node_parent(self); - if (parent.subtree) { - ts_node__last_child_before(parent, self, include_anonymous, &result); + uint32_t target_end_byte = ts_node_end_byte(self); + + TSNode node = ts_node_parent(self); + TSNode earlier_node = ts_node__null(); + bool earlier_node_is_relevant = false; + + while (node.subtree) { + TSNode earlier_child = ts_node__null(); + bool earlier_child_is_relevant = false; + bool found_child_containing_target = false; + + TSNode child; + NodeChildIterator iterator = ts_node_child_iterator_begin(&node); + while (ts_node_child_iterator_next(&iterator, &child)) { + if (iterator.position.bytes >= target_end_byte) { + found_child_containing_target = child.subtree != self.subtree; + break; + } + + if (ts_node__is_relevant(child, include_anonymous)) { + earlier_child = child; + earlier_child_is_relevant = true; + } else if (ts_node__relevant_child_count(child, include_anonymous) > 0) { + earlier_child = child; + earlier_child_is_relevant = false; + } + } + + if (found_child_containing_target) { + if (earlier_child.subtree) { + earlier_node = earlier_child; + earlier_node_is_relevant = earlier_child_is_relevant; + } + node = child; + } else if (earlier_child_is_relevant) { + return earlier_child; + } else if (earlier_child.subtree) { + node = earlier_child; + } else if (earlier_node_is_relevant) { + return earlier_node; + } else { + node = earlier_node; + } } - return result; + + return ts_node__null(); } static inline TSNode ts_node__next_sibling(TSNode self, bool include_anonymous) { From 19e3750f13634d2820f5852fb135c0c0f1e20e0d Mon Sep 17 00:00:00 2001 From: Max Brunsfeld Date: Wed, 9 May 2018 13:46:46 -0700 Subject: [PATCH 28/90] Make ts_node_next_sibling work more like ts_node_prev_sibling Co-Authored-By: Rick Winfrey --- src/runtime/node.c | 51 +++++++++++++++++++++++++++++++++------------- 1 file changed, 37 insertions(+), 14 deletions(-) diff --git a/src/runtime/node.c b/src/runtime/node.c index 2230d1af..99b6256d 100644 --- a/src/runtime/node.c +++ b/src/runtime/node.c @@ -179,28 +179,51 @@ static inline TSNode ts_node__prev_sibling(TSNode self, bool include_anonymous) } static inline TSNode ts_node__next_sibling(TSNode self, bool include_anonymous) { - TSNode node = ts_node_parent(self); - if (!node.subtree) return ts_node__null(); - uint32_t end_byte = ts_node_end_byte(self); + uint32_t target_end_byte = ts_node_end_byte(self); - bool did_descend = true; - while (did_descend) { - did_descend = false; + TSNode node = ts_node_parent(self); + TSNode later_node = ts_node__null(); + bool later_node_is_relevant = false; + + while (node.subtree) { + TSNode later_child = ts_node__null(); + bool later_child_is_relevant = false; + TSNode child_containing_target = ts_node__null(); TSNode child; NodeChildIterator iterator = ts_node_child_iterator_begin(&node); while (ts_node_child_iterator_next(&iterator, &child)) { - if (iterator.position.bytes > end_byte && child.subtree != self.subtree) { - if (ts_node__is_relevant(child, include_anonymous)) { - return child; - } - if (ts_node__relevant_child_count(child, include_anonymous) > 0) { - node = child; - did_descend = true; - break; + if (iterator.position.bytes < target_end_byte) continue; + if (child.byte <= self.byte) { + if (child.subtree != self.subtree) { + child_containing_target = child; } + } else if (ts_node__is_relevant(child, include_anonymous)) { + later_child = child; + later_child_is_relevant = true; + break; + } else if (ts_node__relevant_child_count(child, include_anonymous) > 0) { + later_child = child; + later_child_is_relevant = false; + break; } } + + if (child_containing_target.subtree) { + if (later_child.subtree) { + later_node = later_child; + later_node_is_relevant = later_child_is_relevant; + } + node = child_containing_target; + } else if (later_child_is_relevant) { + return later_child; + } else if (later_child.subtree) { + node = later_child; + } else if (later_node_is_relevant) { + return later_node; + } else { + node = later_node; + } } return ts_node__null(); From b06747b6ca01a868102af30e5db8dc76ac8620cf Mon Sep 17 00:00:00 2001 From: Max Brunsfeld Date: Wed, 9 May 2018 14:14:42 -0700 Subject: [PATCH 29/90] Remove stale unit tests Co-Authored-By: Rick Winfrey --- test/runtime/parser_test.cc | 47 ------------------------------------- 1 file changed, 47 deletions(-) diff --git a/test/runtime/parser_test.cc b/test/runtime/parser_test.cc index 4481ece7..204eedbd 100644 --- a/test/runtime/parser_test.cc +++ b/test/runtime/parser_test.cc @@ -206,53 +206,6 @@ describe("Parser", [&]() { }); }); - describe("handling extra tokens", [&]() { - describe("when the token appears as part of a grammar rule", [&]() { - it("incorporates it into the tree", [&]() { - ts_document_set_language(document, load_real_language("javascript")); - set_text("fn()\n"); - - assert_root_node( - "(program (expression_statement (call_expression (identifier) (arguments))))"); - }); - }); - - describe("when the token appears somewhere else", [&]() { - it("incorporates it into the tree", [&]() { - ts_document_set_language(document, load_real_language("javascript")); - set_text( - "fn()\n" - " .otherFn();"); - - assert_root_node( - "(program (expression_statement (call_expression " - "(member_expression " - "(call_expression (identifier) (arguments)) " - "(property_identifier)) " - "(arguments))))"); - }); - }); - - describe("when several extra tokens appear in a row", [&]() { - it("incorporates them into the tree", [&]() { - ts_document_set_language(document, load_real_language("javascript")); - set_text( - "fn()\n\n" - "// This is a comment" - "\n\n" - ".otherFn();"); - - assert_root_node( - "(program (expression_statement (call_expression " - "(member_expression " - "(call_expression (identifier) (arguments)) " - "(comment) " - "(property_identifier)) " - "(arguments))))"); - }); - }); - }); - describe("editing", [&]() { describe("creating new tokens near the end of the input", [&]() { it("updates the parse tree and re-reads only the changed portion of the text", [&]() { From 5fa6d395785129c65e271f8b321846f239e9b4e4 Mon Sep 17 00:00:00 2001 From: Max Brunsfeld Date: Wed, 9 May 2018 14:14:56 -0700 Subject: [PATCH 30/90] Fix alias handling in ts_tree_string Co-Authored-By: Rick Winfrey --- src/runtime/tree.c | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/src/runtime/tree.c b/src/runtime/tree.c index fcb534dd..0174d3d8 100644 --- a/src/runtime/tree.c +++ b/src/runtime/tree.c @@ -609,15 +609,12 @@ static size_t ts_tree__write_to_string(const Tree *self, char *string, size_t li 0, false ); } else { + TSSymbol alias_symbol = alias_sequence ? alias_sequence[structural_child_index] : 0; cursor += ts_tree__write_to_string( child, *writer, limit, language, false, include_all, - alias_sequence - ? alias_sequence[structural_child_index] - : 0, - alias_sequence - ? ts_language_symbol_metadata(language, alias_sequence[structural_child_index]).named - : false + alias_symbol, + alias_symbol ? ts_language_symbol_metadata(language, alias_symbol).named : false ); structural_child_index++; } From 8d805feab16423e0ee51b818eef32c93b924182b Mon Sep 17 00:00:00 2001 From: Max Brunsfeld Date: Wed, 9 May 2018 14:15:11 -0700 Subject: [PATCH 31/90] Allow ReusableNode to advance off the end of the tree Co-Authored-By: Rick Winfrey --- src/runtime/reusable_node.h | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/src/runtime/reusable_node.h b/src/runtime/reusable_node.h index 803304de..71d56b68 100644 --- a/src/runtime/reusable_node.h +++ b/src/runtime/reusable_node.h @@ -25,11 +25,15 @@ static inline void reusable_node_reset(ReusableNode *self, Tree *tree) { } static inline Tree *reusable_node_tree(ReusableNode *self) { - return array_back(&self->stack)->tree; + return self->stack.size > 0 + ? self->stack.contents[self->stack.size - 1].tree + : NULL; } static inline uint32_t reusable_node_byte_offset(ReusableNode *self) { - return array_back(&self->stack)->byte_offset; + return self->stack.size > 0 + ? self->stack.contents[self->stack.size - 1].byte_offset + : UINT32_MAX; } static inline void reusable_node_delete(ReusableNode *self) { @@ -52,6 +56,7 @@ static inline void reusable_node_advance(ReusableNode *self) { do { StackEntry popped_entry = array_pop(&self->stack); next_index = popped_entry.child_index + 1; + if (self->stack.size == 0) return; tree = array_back(&self->stack)->tree; } while (tree->children.size <= next_index); From 92255bbfdda33970594b2f6a6f573221e333063c Mon Sep 17 00:00:00 2001 From: Max Brunsfeld Date: Wed, 9 May 2018 15:28:28 -0700 Subject: [PATCH 32/90] Remove document parameter from ts_node_type, ts_node_string Co-Authored-By: Rick Winfrey --- include/tree_sitter/runtime.h | 4 +- src/runtime/node.c | 8 +- test/helpers/scope_sequence.cc | 2 +- test/helpers/tree_helpers.cc | 2 +- test/integration/real_grammars.cc | 2 +- test/integration/test_grammars.cc | 2 +- test/runtime/document_test.cc | 2 +- test/runtime/language_test.cc | 4 +- test/runtime/node_test.cc | 125 +++++++++++++++--------------- test/runtime/parser_test.cc | 24 +++--- 10 files changed, 87 insertions(+), 88 deletions(-) diff --git a/include/tree_sitter/runtime.h b/include/tree_sitter/runtime.h index c4bc8ab5..934d5320 100644 --- a/include/tree_sitter/runtime.h +++ b/include/tree_sitter/runtime.h @@ -75,8 +75,8 @@ TSPoint ts_node_start_point(TSNode); uint32_t ts_node_end_byte(TSNode); TSPoint ts_node_end_point(TSNode); TSSymbol ts_node_symbol(TSNode); -const char *ts_node_type(TSNode, const TSDocument *); -char *ts_node_string(TSNode, const TSDocument *); +const char *ts_node_type(TSNode); +char *ts_node_string(TSNode); bool ts_node_eq(TSNode, TSNode); bool ts_node_is_named(TSNode); bool ts_node_is_missing(TSNode); diff --git a/src/runtime/node.c b/src/runtime/node.c index 99b6256d..77d913f3 100644 --- a/src/runtime/node.c +++ b/src/runtime/node.c @@ -342,12 +342,12 @@ TSSymbol ts_node_symbol(TSNode self) { return self.alias_symbol ? self.alias_symbol : tree->symbol; } -const char *ts_node_type(TSNode self, const TSDocument *document) { - return ts_language_symbol_name(document->parser.language, ts_node_symbol(self)); +const char *ts_node_type(TSNode self) { + return ts_language_symbol_name(self.document->parser.language, ts_node_symbol(self)); } -char *ts_node_string(TSNode self, const TSDocument *document) { - return ts_tree_string(ts_node__tree(self), document->parser.language, false); +char *ts_node_string(TSNode self) { + return ts_tree_string(ts_node__tree(self), self.document->parser.language, false); } bool ts_node_eq(TSNode self, TSNode other) { diff --git a/test/helpers/scope_sequence.cc b/test/helpers/scope_sequence.cc index d6e2e3b1..c3db70ac 100644 --- a/test/helpers/scope_sequence.cc +++ b/test/helpers/scope_sequence.cc @@ -27,7 +27,7 @@ static void append_to_scope_sequence(ScopeSequence *sequence, sequence, current_scopes, text, ts_node_start_byte(node) - sequence->size() ); - current_scopes->push_back(ts_node_type(node, document)); + current_scopes->push_back(ts_node_type(node)); for (size_t i = 0, n = ts_node_child_count(node); i < n; i++) { TSNode child = ts_node_child(node, i); diff --git a/test/helpers/tree_helpers.cc b/test/helpers/tree_helpers.cc index 8fedb560..553f8b49 100644 --- a/test/helpers/tree_helpers.cc +++ b/test/helpers/tree_helpers.cc @@ -37,7 +37,7 @@ ostream &operator<<(std::ostream &stream, const Tree *tree) { ostream &operator<<(ostream &stream, const TSNode &node) { if (node.subtree) { - char *string = ts_node_string(node, node.document); + char *string = ts_node_string(node); stream << "{" << string << ", " << to_string(ts_node_start_byte(node)) << "}"; ts_free(string); return stream; diff --git a/test/integration/real_grammars.cc b/test/integration/real_grammars.cc index e842ee0c..608ded82 100644 --- a/test/integration/real_grammars.cc +++ b/test/integration/real_grammars.cc @@ -63,7 +63,7 @@ for (auto &language_name : test_languages) { edit_sequence(); TSNode root_node = ts_document_root_node(document); - const char *node_string = ts_node_string(root_node, document); + const char *node_string = ts_node_string(root_node); string result(node_string); ts_free((void *)node_string); AssertThat(result, Equals(entry.tree_string)); diff --git a/test/integration/test_grammars.cc b/test/integration/test_grammars.cc index f4ba6d92..f5324579 100644 --- a/test/integration/test_grammars.cc +++ b/test/integration/test_grammars.cc @@ -66,7 +66,7 @@ for (auto &language_name : test_languages) { TSNode root_node = ts_document_root_node(document); AssertThat(ts_node_end_byte(root_node), Equals(entry.input.size())); assert_consistent_tree_sizes(root_node); - const char *node_string = ts_node_string(root_node, document); + const char *node_string = ts_node_string(root_node); string result(node_string); ts_free((void *)node_string); ts_document_free(document); diff --git a/test/runtime/document_test.cc b/test/runtime/document_test.cc index 97aa4fe2..5f140251 100644 --- a/test/runtime/document_test.cc +++ b/test/runtime/document_test.cc @@ -35,7 +35,7 @@ describe("Document", [&]() { }); auto assert_node_string_equals = [&](TSNode node, const string &expected) { - char *str = ts_node_string(node, document); + char *str = ts_node_string(node); string actual(str); ts_free(str); AssertThat(actual, Equals(expected)); diff --git a/test/runtime/language_test.cc b/test/runtime/language_test.cc index 7d9d51cc..4726c4ba 100644 --- a/test/runtime/language_test.cc +++ b/test/runtime/language_test.cc @@ -35,11 +35,11 @@ describe("Language", []() { ts_document_parse(document); TSNode root_node = ts_document_root_node(document); - char *string = ts_node_string(root_node, document); + char *string = ts_node_string(root_node); AssertThat(string, Equals("(a (c))")); TSNode aliased_node = ts_node_child(root_node, 0); - AssertThat(ts_node_type(aliased_node, document), Equals("c")); + AssertThat(ts_node_type(aliased_node), Equals("c")); TSSymbol aliased_symbol = ts_node_symbol(aliased_node); AssertThat(ts_language_symbol_count(language), IsGreaterThan(aliased_symbol)); diff --git a/test/runtime/node_test.cc b/test/runtime/node_test.cc index 22196ec5..91717ed2 100644 --- a/test/runtime/node_test.cc +++ b/test/runtime/node_test.cc @@ -71,7 +71,6 @@ describe("Node", [&]() { document = ts_document_new(); ts_document_set_language(document, load_real_language("json")); ts_document_set_input_string(document, json_string.c_str()); - // ts_document_print_debugging_graphs(document, true); ts_document_parse(document); root_node = ts_node_child(ts_document_root_node(document), 0); }); @@ -84,7 +83,7 @@ describe("Node", [&]() { }); it("parses the example as expected (precondition)", [&]() { - char *node_string = ts_node_string(root_node, document); + char *node_string = ts_node_string(root_node); AssertThat(node_string, Equals( "(array " "(number) " @@ -95,7 +94,7 @@ describe("Node", [&]() { describe("named_child_count(), named_child(i)", [&]() { it("returns the named child node at the given index", [&]() { - AssertThat(ts_node_type(root_node, document), Equals("array")); + AssertThat(ts_node_type(root_node), Equals("array")); AssertThat(ts_node_named_child_count(root_node), Equals(3)); AssertThat(ts_node_start_byte(root_node), Equals(array_index)); @@ -107,9 +106,9 @@ describe("Node", [&]() { TSNode false_node = ts_node_named_child(root_node, 1); TSNode object_node = ts_node_named_child(root_node, 2); - AssertThat(ts_node_type(number_node, document), Equals("number")); - AssertThat(ts_node_type(false_node, document), Equals("false")); - AssertThat(ts_node_type(object_node, document), Equals("object")); + AssertThat(ts_node_type(number_node), Equals("number")); + AssertThat(ts_node_type(false_node), Equals("false")); + AssertThat(ts_node_type(object_node), Equals("object")); AssertThat(ts_node_start_byte(number_node), Equals(number_index)); AssertThat(ts_node_end_byte(number_node), Equals(number_end_index)); @@ -129,7 +128,7 @@ describe("Node", [&]() { TSNode pair_node = ts_node_named_child(object_node, 0); - AssertThat(ts_node_type(pair_node, document), Equals("pair")); + AssertThat(ts_node_type(pair_node), Equals("pair")); AssertThat(ts_node_start_byte(pair_node), Equals(string_index)); AssertThat(ts_node_end_byte(pair_node), Equals(null_end_index)); AssertThat(ts_node_start_point(pair_node), Equals({ 6, 4 })); @@ -139,8 +138,8 @@ describe("Node", [&]() { TSNode string_node = ts_node_named_child(pair_node, 0); TSNode null_node = ts_node_named_child(pair_node, 1); - AssertThat(ts_node_type(string_node, document), Equals("string")); - AssertThat(ts_node_type(null_node, document), Equals("null")); + AssertThat(ts_node_type(string_node), Equals("string")); + AssertThat(ts_node_type(null_node), Equals("null")); AssertThat(ts_node_start_byte(string_node), Equals(string_index)); AssertThat(ts_node_end_byte(string_node), Equals(string_end_index)); @@ -169,16 +168,16 @@ describe("Node", [&]() { ts_document_parse(document); root_node = ts_document_root_node(document); - char *node_string = ts_node_string(root_node, document); + char *node_string = ts_node_string(root_node); AssertThat(node_string, Equals("(a (b) (comment) (B) (comment) (b))")); ts_free(node_string); AssertThat(ts_node_named_child_count(root_node), Equals(5u)); - AssertThat(ts_node_type(ts_node_named_child(root_node, 0), document), Equals("b")); - AssertThat(ts_node_type(ts_node_named_child(root_node, 1), document), Equals("comment")); - AssertThat(ts_node_type(ts_node_named_child(root_node, 2), document), Equals("B")); - AssertThat(ts_node_type(ts_node_named_child(root_node, 3), document), Equals("comment")); - AssertThat(ts_node_type(ts_node_named_child(root_node, 4), document), Equals("b")); + AssertThat(ts_node_type(ts_node_named_child(root_node, 0)), Equals("b")); + AssertThat(ts_node_type(ts_node_named_child(root_node, 1)), Equals("comment")); + AssertThat(ts_node_type(ts_node_named_child(root_node, 2)), Equals("B")); + AssertThat(ts_node_type(ts_node_named_child(root_node, 3)), Equals("comment")); + AssertThat(ts_node_type(ts_node_named_child(root_node, 4)), Equals("b")); AssertThat(ts_node_symbol(ts_node_named_child(root_node, 0)), !Equals(ts_node_symbol(ts_node_named_child(root_node, 2)))); }); @@ -189,29 +188,29 @@ describe("Node", [&]() { TSNode child; child = ts_node_first_child_for_byte(root_node, array_index); - AssertThat(ts_node_type(child, document), Equals("[")); + AssertThat(ts_node_type(child), Equals("[")); child = ts_node_first_child_for_byte(root_node, number_index); - AssertThat(ts_node_type(child, document), Equals("number")); + AssertThat(ts_node_type(child), Equals("number")); child = ts_node_first_child_for_byte(root_node, number_end_index); - AssertThat(ts_node_type(child, document), Equals(",")); + AssertThat(ts_node_type(child), Equals(",")); child = ts_node_first_child_for_byte(root_node, number_end_index + 1); - AssertThat(ts_node_type(child, document), Equals("false")); + AssertThat(ts_node_type(child), Equals("false")); child = ts_node_first_child_for_byte(root_node, false_index - 1); - AssertThat(ts_node_type(child, document), Equals("false")); + AssertThat(ts_node_type(child), Equals("false")); child = ts_node_first_child_for_byte(root_node, false_index); - AssertThat(ts_node_type(child, document), Equals("false")); + AssertThat(ts_node_type(child), Equals("false")); child = ts_node_first_child_for_byte(root_node, false_index + 1); - AssertThat(ts_node_type(child, document), Equals("false")); + AssertThat(ts_node_type(child), Equals("false")); child = ts_node_first_child_for_byte(root_node, false_end_index); - AssertThat(ts_node_type(child, document), Equals(",")); + AssertThat(ts_node_type(child), Equals(",")); child = ts_node_first_child_for_byte(root_node, false_end_index); - AssertThat(ts_node_type(child, document), Equals(",")); + AssertThat(ts_node_type(child), Equals(",")); child = ts_node_first_child_for_byte(root_node, object_index); - AssertThat(ts_node_type(child, document), Equals("object")); + AssertThat(ts_node_type(child), Equals("object")); child = ts_node_first_child_for_byte(root_node, object_index + 1); - AssertThat(ts_node_type(child, document), Equals("object")); + AssertThat(ts_node_type(child), Equals("object")); child = ts_node_first_child_for_byte(root_node, object_end_index); - AssertThat(ts_node_type(child, document), Equals("]")); + AssertThat(ts_node_type(child), Equals("]")); }); }); @@ -220,25 +219,25 @@ describe("Node", [&]() { TSNode child; child = ts_node_first_named_child_for_byte(root_node, array_index); - AssertThat(ts_node_type(child, document), Equals("number")); + AssertThat(ts_node_type(child), Equals("number")); child = ts_node_first_named_child_for_byte(root_node, number_index); - AssertThat(ts_node_type(child, document), Equals("number")); + AssertThat(ts_node_type(child), Equals("number")); child = ts_node_first_named_child_for_byte(root_node, number_end_index); - AssertThat(ts_node_type(child, document), Equals("false")); + AssertThat(ts_node_type(child), Equals("false")); child = ts_node_first_named_child_for_byte(root_node, number_end_index + 1); - AssertThat(ts_node_type(child, document), Equals("false")); + AssertThat(ts_node_type(child), Equals("false")); child = ts_node_first_named_child_for_byte(root_node, false_index - 1); - AssertThat(ts_node_type(child, document), Equals("false")); + AssertThat(ts_node_type(child), Equals("false")); child = ts_node_first_named_child_for_byte(root_node, false_index); - AssertThat(ts_node_type(child, document), Equals("false")); + AssertThat(ts_node_type(child), Equals("false")); child = ts_node_first_named_child_for_byte(root_node, false_index + 1); - AssertThat(ts_node_type(child, document), Equals("false")); + AssertThat(ts_node_type(child), Equals("false")); child = ts_node_first_named_child_for_byte(root_node, false_end_index); - AssertThat(ts_node_type(child, document), Equals("object")); + AssertThat(ts_node_type(child), Equals("object")); child = ts_node_first_named_child_for_byte(root_node, object_index); - AssertThat(ts_node_type(child, document), Equals("object")); + AssertThat(ts_node_type(child), Equals("object")); child = ts_node_first_named_child_for_byte(root_node, object_index + 1); - AssertThat(ts_node_type(child, document), Equals("object")); + AssertThat(ts_node_type(child), Equals("object")); child = ts_node_first_named_child_for_byte(root_node, object_end_index); AssertThat(child.subtree, Equals(nullptr)); }); @@ -255,14 +254,14 @@ describe("Node", [&]() { TSNode child6 = ts_node_child(root_node, 5); TSNode child7 = ts_node_child(root_node, 6); - AssertThat(ts_node_type(root_node, document), Equals("array")); - AssertThat(ts_node_type(child1, document), Equals("[")); - AssertThat(ts_node_type(child2, document), Equals("number")); - AssertThat(ts_node_type(child3, document), Equals(",")); - AssertThat(ts_node_type(child4, document), Equals("false")); - AssertThat(ts_node_type(child5, document), Equals(",")); - AssertThat(ts_node_type(child6, document), Equals("object")); - AssertThat(ts_node_type(child7, document), Equals("]")); + AssertThat(ts_node_type(root_node), Equals("array")); + AssertThat(ts_node_type(child1), Equals("[")); + AssertThat(ts_node_type(child2), Equals("number")); + AssertThat(ts_node_type(child3), Equals(",")); + AssertThat(ts_node_type(child4), Equals("false")); + AssertThat(ts_node_type(child5), Equals(",")); + AssertThat(ts_node_type(child6), Equals("object")); + AssertThat(ts_node_type(child7), Equals("]")); AssertThat(ts_node_is_named(root_node), IsTrue()); AssertThat(ts_node_is_named(child1), IsFalse()); @@ -303,13 +302,13 @@ describe("Node", [&]() { TSNode grandchild3 = ts_node_child(pair, 1); TSNode grandchild4 = ts_node_child(pair, 2); - AssertThat(ts_node_type(left_brace, document), Equals("{")); - AssertThat(ts_node_type(pair, document), Equals("pair")); - AssertThat(ts_node_type(right_brace, document), Equals("}")); + AssertThat(ts_node_type(left_brace), Equals("{")); + AssertThat(ts_node_type(pair), Equals("pair")); + AssertThat(ts_node_type(right_brace), Equals("}")); - AssertThat(ts_node_type(grandchild2, document), Equals("string")); - AssertThat(ts_node_type(grandchild3, document), Equals(":")); - AssertThat(ts_node_type(grandchild4, document), Equals("null")); + AssertThat(ts_node_type(grandchild2), Equals("string")); + AssertThat(ts_node_type(grandchild3), Equals(":")); + AssertThat(ts_node_type(grandchild4), Equals("null")); AssertThat(ts_node_parent(grandchild2), Equals(pair)); AssertThat(ts_node_parent(grandchild3), Equals(pair)); @@ -411,14 +410,14 @@ describe("Node", [&]() { describe("when there is a leaf node that spans the given range exactly", [&]() { it("returns that leaf node", [&]() { TSNode leaf = ts_node_named_descendant_for_byte_range(root_node, string_index, string_end_index - 1); - AssertThat(ts_node_type(leaf, document), Equals("string")); + AssertThat(ts_node_type(leaf), Equals("string")); AssertThat(ts_node_start_byte(leaf), Equals(string_index)); AssertThat(ts_node_end_byte(leaf), Equals(string_end_index)); AssertThat(ts_node_start_point(leaf), Equals({ 6, 4 })); AssertThat(ts_node_end_point(leaf), Equals({ 6, 7 })); leaf = ts_node_named_descendant_for_byte_range(root_node, number_index, number_end_index - 1); - AssertThat(ts_node_type(leaf, document), Equals("number")); + AssertThat(ts_node_type(leaf), Equals("number")); AssertThat(ts_node_start_byte(leaf), Equals(number_index)); AssertThat(ts_node_end_byte(leaf), Equals(number_end_index)); AssertThat(ts_node_start_point(leaf), Equals({ 3, 2 })); @@ -429,14 +428,14 @@ describe("Node", [&]() { describe("when there is a leaf node that extends beyond the given range", [&]() { it("returns that leaf node", [&]() { TSNode leaf = ts_node_named_descendant_for_byte_range(root_node, string_index, string_index + 1); - AssertThat(ts_node_type(leaf, document), Equals("string")); + AssertThat(ts_node_type(leaf), Equals("string")); AssertThat(ts_node_start_byte(leaf), Equals(string_index)); AssertThat(ts_node_end_byte(leaf), Equals(string_end_index)); AssertThat(ts_node_start_point(leaf), Equals({ 6, 4 })); AssertThat(ts_node_end_point(leaf), Equals({ 6, 7 })); leaf = ts_node_named_descendant_for_byte_range(root_node, string_index + 1, string_index + 2); - AssertThat(ts_node_type(leaf, document), Equals("string")); + AssertThat(ts_node_type(leaf), Equals("string")); AssertThat(ts_node_start_byte(leaf), Equals(string_index)); AssertThat(ts_node_end_byte(leaf), Equals(string_end_index)); AssertThat(ts_node_start_point(leaf), Equals({ 6, 4 })); @@ -447,7 +446,7 @@ describe("Node", [&]() { describe("when there is no leaf node that spans the given range", [&]() { it("returns the smallest node that does span the range", [&]() { TSNode pair_node = ts_node_named_descendant_for_byte_range(root_node, string_index, string_index + 3); - AssertThat(ts_node_type(pair_node, document), Equals("pair")); + AssertThat(ts_node_type(pair_node), Equals("pair")); AssertThat(ts_node_start_byte(pair_node), Equals(string_index)); AssertThat(ts_node_end_byte(pair_node), Equals(null_end_index)); AssertThat(ts_node_start_point(pair_node), Equals({ 6, 4 })); @@ -456,7 +455,7 @@ describe("Node", [&]() { it("does not return invisible nodes (repeats)", [&]() { TSNode node = ts_node_named_descendant_for_byte_range(root_node, number_end_index, number_end_index + 1); - AssertThat(ts_node_type(node, document), Equals("array")); + AssertThat(ts_node_type(node), Equals("array")); AssertThat(ts_node_start_byte(node), Equals(array_index)); AssertThat(ts_node_end_byte(node), Equals(array_end_index)); AssertThat(ts_node_start_point(node), Equals({ 2, 0 })); @@ -468,14 +467,14 @@ describe("Node", [&]() { describe("descendant_for_byte_range(start, end)", [&]() { it("returns the smallest node that spans the given byte offsets", [&]() { TSNode node1 = ts_node_descendant_for_byte_range(root_node, colon_index, colon_index); - AssertThat(ts_node_type(node1, document), Equals(":")); + AssertThat(ts_node_type(node1), Equals(":")); AssertThat(ts_node_start_byte(node1), Equals(colon_index)); AssertThat(ts_node_end_byte(node1), Equals(colon_index + 1)); AssertThat(ts_node_start_point(node1), Equals({ 6, 7 })); AssertThat(ts_node_end_point(node1), Equals({ 6, 8 })); TSNode node2 = ts_node_descendant_for_byte_range(root_node, string_index + 2, string_index + 4); - AssertThat(ts_node_type(node2, document), Equals("pair")); + AssertThat(ts_node_type(node2), Equals("pair")); AssertThat(ts_node_start_byte(node2), Equals(string_index)); AssertThat(ts_node_end_byte(node2), Equals(null_end_index)); AssertThat(ts_node_start_point(node2), Equals({ 6, 4 })); @@ -490,10 +489,10 @@ describe("Node", [&]() { uint32_t comma_position = input_string.find(","); TSNode node1 = ts_node_descendant_for_byte_range(root_node, comma_position, comma_position); - AssertThat(ts_node_type(node1, document), Equals(",")); + AssertThat(ts_node_type(node1), Equals(",")); TSNode node2 = ts_node_descendant_for_byte_range(root_node, 6, 10); - AssertThat(ts_node_type(node2, document), Equals("string")); + AssertThat(ts_node_type(node2), Equals("string")); AssertThat(ts_node_start_byte(node2), Equals(1)); AssertThat(ts_node_end_byte(node2), Equals(11)); }); @@ -502,14 +501,14 @@ describe("Node", [&]() { describe("descendant_for_point_range(start, end)", [&]() { it("returns the smallest concrete node that spans the given range", [&]() { TSNode node1 = ts_node_descendant_for_point_range(root_node, {6, 7}, {6, 7}); - AssertThat(ts_node_type(node1, document), Equals(":")); + AssertThat(ts_node_type(node1), Equals(":")); AssertThat(ts_node_start_byte(node1), Equals(colon_index)); AssertThat(ts_node_end_byte(node1), Equals(colon_index + 1)); AssertThat(ts_node_start_point(node1), Equals({ 6, 7 })); AssertThat(ts_node_end_point(node1), Equals({ 6, 8 })); TSNode node2 = ts_node_descendant_for_point_range(root_node, {6, 6}, {6, 8}); - AssertThat(ts_node_type(node2, document), Equals("pair")); + AssertThat(ts_node_type(node2), Equals("pair")); AssertThat(ts_node_start_byte(node2), Equals(string_index)); AssertThat(ts_node_end_byte(node2), Equals(null_end_index)); AssertThat(ts_node_start_point(node2), Equals({ 6, 4 })); diff --git a/test/runtime/parser_test.cc b/test/runtime/parser_test.cc index 204eedbd..2cf70b69 100644 --- a/test/runtime/parser_test.cc +++ b/test/runtime/parser_test.cc @@ -71,7 +71,7 @@ describe("Parser", [&]() { auto assert_root_node = [&](const string &expected) { TSNode node = ts_document_root_node(document); - char *node_string = ts_node_string(node, document); + char *node_string = ts_node_string(node); string actual(node_string); ts_free(node_string); AssertThat(actual, Equals(expected)); @@ -93,7 +93,7 @@ describe("Parser", [&]() { "(value (array (number) (ERROR (UNEXPECTED '@')) (true)))"); TSNode error = ts_node_named_child(ts_node_child(root, 0), 1); - AssertThat(ts_node_type(error, document), Equals("ERROR")); + AssertThat(ts_node_type(error), Equals("ERROR")); AssertThat(get_node_text(error), Equals("@@@@@,")); AssertThat(ts_node_child_count(error), Equals(2)); @@ -104,7 +104,7 @@ describe("Parser", [&]() { AssertThat(get_node_text(comma), Equals(",")); TSNode node_after_error = ts_node_next_named_sibling(error); - AssertThat(ts_node_type(node_after_error, document), Equals("true")); + AssertThat(ts_node_type(node_after_error), Equals("true")); AssertThat(get_node_text(node_after_error), Equals("true")); }); }); @@ -118,20 +118,20 @@ describe("Parser", [&]() { "(value (array (number) (ERROR (UNEXPECTED 'a')) (true)))"); TSNode error = ts_node_named_child(ts_node_child(root, 0), 1); - AssertThat(ts_node_type(error, document), Equals("ERROR")); + AssertThat(ts_node_type(error), Equals("ERROR")); AssertThat(get_node_text(error), Equals("faaaaalse,")); AssertThat(ts_node_child_count(error), Equals(2)); TSNode garbage = ts_node_child(error, 0); - AssertThat(ts_node_type(garbage, document), Equals("ERROR")); + AssertThat(ts_node_type(garbage), Equals("ERROR")); AssertThat(get_node_text(garbage), Equals("faaaaalse")); TSNode comma = ts_node_child(error, 1); - AssertThat(ts_node_type(comma, document), Equals(",")); + AssertThat(ts_node_type(comma), Equals(",")); AssertThat(get_node_text(comma), Equals(",")); TSNode last = ts_node_next_named_sibling(error); - AssertThat(ts_node_type(last, document), Equals("true")); + AssertThat(ts_node_type(last), Equals("true")); AssertThat(ts_node_start_byte(last), Equals(strlen(" [123, faaaaalse, "))); }); }); @@ -145,12 +145,12 @@ describe("Parser", [&]() { "(value (array (number) (true) (ERROR (false)) (true)))"); TSNode error = ts_node_named_child(ts_node_child(root, 0), 2); - AssertThat(ts_node_type(error, document), Equals("ERROR")); + AssertThat(ts_node_type(error), Equals("ERROR")); AssertThat(get_node_text(error), Equals("false")); AssertThat(ts_node_child_count(error), Equals(1)); TSNode last = ts_node_next_named_sibling(error); - AssertThat(ts_node_type(last, document), Equals("true")); + AssertThat(ts_node_type(last), Equals("true")); AssertThat(get_node_text(last), Equals("true")); }); }); @@ -186,7 +186,7 @@ describe("Parser", [&]() { ); TSNode error = ts_node_named_child(root, 1); - AssertThat(ts_node_type(error, document), Equals("ERROR")); + AssertThat(ts_node_type(error), Equals("ERROR")); AssertThat(ts_node_start_point(error), Equals({2, 0})); AssertThat(ts_node_end_point(error), Equals({2, 2})); }); @@ -303,7 +303,7 @@ describe("Parser", [&]() { "(program (expression_statement (binary_expression (identifier) (number))))"); TSNode node = ts_node_named_descendant_for_byte_range(root, 1, 1); - AssertThat(ts_node_type(node, document), Equals("identifier")); + AssertThat(ts_node_type(node), Equals("identifier")); AssertThat(ts_node_end_byte(node), Equals(strlen("abXYZc"))); }); }); @@ -322,7 +322,7 @@ describe("Parser", [&]() { "(program (expression_statement (binary_expression (identifier) (number))))"); TSNode node = ts_node_named_descendant_for_byte_range(root, 1, 1); - AssertThat(ts_node_type(node, document), Equals("identifier")); + AssertThat(ts_node_type(node), Equals("identifier")); AssertThat(ts_node_end_byte(node), Equals(strlen("abcXYZ"))); }); }); From 666dfb76d21166e315ddbf34049980e5081832b9 Mon Sep 17 00:00:00 2001 From: Max Brunsfeld Date: Wed, 9 May 2018 16:38:56 -0700 Subject: [PATCH 33/90] Remove document parameter from ts_node_type, ts_node_string Co-Authored-By: Rick Winfrey --- include/tree_sitter/runtime.h | 22 +++-- project.gyp | 1 + src/runtime/document.c | 15 ++-- src/runtime/document.h | 6 +- src/runtime/get_changed_ranges.c | 55 ++++++------- src/runtime/get_changed_ranges.h | 11 +-- src/runtime/tree_cursor.c | 134 +++++++++++++++++++++++++++++++ src/runtime/tree_cursor.h | 20 +++++ test/runtime/node_test.cc | 128 +++++++++++++++++++++++++++++ 9 files changed, 339 insertions(+), 53 deletions(-) create mode 100644 src/runtime/tree_cursor.c create mode 100644 src/runtime/tree_cursor.h diff --git a/include/tree_sitter/runtime.h b/include/tree_sitter/runtime.h index 934d5320..d3e282fb 100644 --- a/include/tree_sitter/runtime.h +++ b/include/tree_sitter/runtime.h @@ -14,6 +14,7 @@ extern "C" { typedef unsigned short TSSymbol; typedef struct TSLanguage TSLanguage; typedef struct TSDocument TSDocument; +typedef struct TSTreeCursor TSTreeCursor; typedef enum { TSInputEncodingUTF8, @@ -70,6 +71,12 @@ typedef struct { TSSymbol alias_symbol; } TSNode; +typedef struct { + TSRange **changed_ranges; + uint32_t *changed_range_count; + bool halt_on_error; +} TSParseOptions; + uint32_t ts_node_start_byte(TSNode); TSPoint ts_node_start_point(TSNode); uint32_t ts_node_end_byte(TSNode); @@ -112,19 +119,18 @@ void ts_document_print_debugging_graphs(TSDocument *, bool); void ts_document_edit(TSDocument *, TSInputEdit); void ts_document_parse(TSDocument *); void ts_document_parse_and_get_changed_ranges(TSDocument *, TSRange **, uint32_t *); - -typedef struct { - TSRange **changed_ranges; - uint32_t *changed_range_count; - bool halt_on_error; -} TSParseOptions; - void ts_document_parse_with_options(TSDocument *, TSParseOptions); - void ts_document_invalidate(TSDocument *); TSNode ts_document_root_node(const TSDocument *); +TSTreeCursor *ts_document_tree_cursor(const TSDocument *); uint32_t ts_document_parse_count(const TSDocument *); +void ts_tree_cursor_delete(TSTreeCursor *); +bool ts_tree_cursor_goto_first_child(TSTreeCursor *); +bool ts_tree_cursor_goto_next_sibling(TSTreeCursor *); +bool ts_tree_cursor_goto_parent(TSTreeCursor *); +TSNode ts_tree_cursor_current_node(TSTreeCursor *); + uint32_t ts_language_symbol_count(const TSLanguage *); const char *ts_language_symbol_name(const TSLanguage *, TSSymbol); TSSymbolType ts_language_symbol_type(const TSLanguage *, TSSymbol); diff --git a/project.gyp b/project.gyp index b968a171..f59c3c91 100644 --- a/project.gyp +++ b/project.gyp @@ -96,6 +96,7 @@ 'src/runtime/parser.c', 'src/runtime/string_input.c', 'src/runtime/tree.c', + 'src/runtime/tree_cursor.c', 'src/runtime/utf16.c', 'externals/utf8proc/utf8proc.c', ], diff --git a/src/runtime/document.c b/src/runtime/document.c index ddd89b43..ab10ae49 100644 --- a/src/runtime/document.c +++ b/src/runtime/document.c @@ -3,6 +3,7 @@ #include "runtime/parser.h" #include "runtime/string_input.h" #include "runtime/document.h" +#include "runtime/tree_cursor.h" #include "runtime/get_changed_ranges.h" #define LOG(...) \ @@ -12,15 +13,15 @@ TSDocument *ts_document_new() { TSDocument *self = ts_calloc(1, sizeof(TSDocument)); parser_init(&self->parser); - array_init(&self->tree_path1); - array_init(&self->tree_path2); + array_init(&self->cursor1.stack); + array_init(&self->cursor2.stack); return self; } void ts_document_free(TSDocument *self) { if (self->tree) ts_tree_release(&self->parser.tree_pool, self->tree); - if (self->tree_path1.contents) array_delete(&self->tree_path1); - if (self->tree_path2.contents) array_delete(&self->tree_path2); + if (self->cursor1.stack.contents) array_delete(&self->cursor1.stack); + if (self->cursor2.stack.contents) array_delete(&self->cursor2.stack); parser_destroy(&self->parser); ts_document_set_input(self, (TSInput){ NULL, @@ -141,7 +142,7 @@ void ts_document_parse_with_options(TSDocument *self, TSParseOptions options) { if (options.changed_ranges && options.changed_range_count) { *options.changed_range_count = ts_tree_get_changed_ranges( - old_tree, tree, &self->tree_path1, &self->tree_path2, + old_tree, tree, &self->cursor1, &self->cursor2, self->parser.language, options.changed_ranges ); @@ -181,3 +182,7 @@ TSNode ts_document_root_node(const TSDocument *self) { uint32_t ts_document_parse_count(const TSDocument *self) { return self->parse_count; } + +TSTreeCursor *ts_document_tree_cursor(const TSDocument *self) { + return ts_tree_cursor_new(self); +} diff --git a/src/runtime/document.h b/src/runtime/document.h index 0be05f14..986fcec9 100644 --- a/src/runtime/document.h +++ b/src/runtime/document.h @@ -3,15 +3,15 @@ #include "runtime/parser.h" #include "runtime/tree.h" -#include "runtime/get_changed_ranges.h" +#include "runtime/tree_cursor.h" #include struct TSDocument { Parser parser; TSInput input; Tree *tree; - TreePath tree_path1; - TreePath tree_path2; + TSTreeCursor cursor1; + TSTreeCursor cursor2; size_t parse_count; bool valid; bool owns_input; diff --git a/src/runtime/get_changed_ranges.c b/src/runtime/get_changed_ranges.c index 26211613..7632af24 100644 --- a/src/runtime/get_changed_ranges.c +++ b/src/runtime/get_changed_ranges.c @@ -2,6 +2,7 @@ #include "runtime/tree.h" #include "runtime/language.h" #include "runtime/error_costs.h" +#include "runtime/tree_cursor.h" #include // #define DEBUG_GET_CHANGED_RANGES @@ -24,22 +25,22 @@ static void range_array_add(RangeArray *results, TSPoint start, TSPoint end) { } typedef struct { - TreePath path; + TSTreeCursor cursor; const TSLanguage *language; unsigned visible_depth; bool in_padding; } Iterator; -static Iterator iterator_new(TreePath *path, Tree *tree, const TSLanguage *language) { - array_clear(path); - array_push(path, ((TreePathEntry){ +static Iterator iterator_new(TSTreeCursor *cursor, Tree *tree, const TSLanguage *language) { + array_clear(&cursor->stack); + array_push(&cursor->stack, ((TreeCursorEntry){ .tree = tree, .position = length_zero(), .child_index = 0, .structural_child_index = 0, })); return (Iterator) { - .path = *path, + .cursor = *cursor, .language = language, .visible_depth = 1, .in_padding = false, @@ -47,11 +48,11 @@ static Iterator iterator_new(TreePath *path, Tree *tree, const TSLanguage *langu } static bool iterator_done(Iterator *self) { - return self->path.size == 0; + return self->cursor.stack.size == 0; } Length iterator_start_position(Iterator *self) { - TreePathEntry entry = *array_back(&self->path); + TreeCursorEntry entry = *array_back(&self->cursor.stack); if (self->in_padding) { return entry.position; } else { @@ -60,7 +61,7 @@ Length iterator_start_position(Iterator *self) { } Length iterator_end_position(Iterator *self) { - TreePathEntry entry = *array_back(&self->path); + TreeCursorEntry entry = *array_back(&self->cursor.stack); Length result = length_add(entry.position, entry.tree->padding); if (self->in_padding) { return result; @@ -70,10 +71,10 @@ Length iterator_end_position(Iterator *self) { } static bool iterator_tree_is_visible(const Iterator *self) { - TreePathEntry entry = *array_back(&self->path); + TreeCursorEntry entry = *array_back(&self->cursor.stack); if (entry.tree->visible) return true; - if (self->path.size > 1) { - Tree *parent = self->path.contents[self->path.size - 2].tree; + if (self->cursor.stack.size > 1) { + Tree *parent = self->cursor.stack.contents[self->cursor.stack.size - 2].tree; const TSSymbol *alias_sequence = ts_language_alias_sequence(self->language, parent->alias_sequence_id); return alias_sequence && alias_sequence[entry.structural_child_index] != 0; } @@ -82,7 +83,7 @@ static bool iterator_tree_is_visible(const Iterator *self) { static void iterator_get_visible_state(const Iterator *self, Tree **tree, TSSymbol *alias_symbol, uint32_t *start_byte) { - uint32_t i = self->path.size - 1; + uint32_t i = self->cursor.stack.size - 1; if (self->in_padding) { if (i == 0) return; @@ -90,10 +91,10 @@ static void iterator_get_visible_state(const Iterator *self, Tree **tree, } for (; i + 1 > 0; i--) { - TreePathEntry entry = self->path.contents[i]; + TreeCursorEntry entry = self->cursor.stack.contents[i]; if (i > 0) { - Tree *parent = self->path.contents[i - 1].tree; + Tree *parent = self->cursor.stack.contents[i - 1].tree; const TSSymbol *alias_sequence = ts_language_alias_sequence( self->language, parent->alias_sequence_id @@ -114,8 +115,8 @@ static void iterator_get_visible_state(const Iterator *self, Tree **tree, static void iterator_ascend(Iterator *self) { if (iterator_done(self)) return; if (iterator_tree_is_visible(self) && !self->in_padding) self->visible_depth--; - if (array_back(&self->path)->child_index > 0) self->in_padding = false; - self->path.size--; + if (array_back(&self->cursor.stack)->child_index > 0) self->in_padding = false; + self->cursor.stack.size--; } static bool iterator_descend(Iterator *self, uint32_t goal_position) { @@ -124,7 +125,7 @@ static bool iterator_descend(Iterator *self, uint32_t goal_position) { bool did_descend; do { did_descend = false; - TreePathEntry entry = *array_back(&self->path); + TreeCursorEntry entry = *array_back(&self->cursor.stack); Length position = entry.position; uint32_t structural_child_index = 0; for (uint32_t i = 0; i < entry.tree->children.size; i++) { @@ -133,7 +134,7 @@ static bool iterator_descend(Iterator *self, uint32_t goal_position) { Length child_right = length_add(child_left, child->size); if (child_right.bytes > goal_position) { - array_push(&self->path, ((TreePathEntry){ + array_push(&self->cursor.stack, ((TreeCursorEntry){ .tree = child, .position = position, .child_index = i, @@ -174,10 +175,10 @@ static void iterator_advance(Iterator *self) { for (;;) { if (iterator_tree_is_visible(self)) self->visible_depth--; - TreePathEntry entry = array_pop(&self->path); + TreeCursorEntry entry = array_pop(&self->cursor.stack); if (iterator_done(self)) return; - Tree *parent = array_back(&self->path)->tree; + Tree *parent = array_back(&self->cursor.stack)->tree; uint32_t child_index = entry.child_index + 1; if (parent->children.size > child_index) { Length position = length_add(entry.position, ts_tree_total_size(entry.tree)); @@ -185,7 +186,7 @@ static void iterator_advance(Iterator *self) { if (!entry.tree->extra) structural_child_index++; Tree *next_child = parent->children.contents[child_index]; - array_push(&self->path, ((TreePathEntry){ + array_push(&self->cursor.stack, ((TreeCursorEntry){ .tree = next_child, .position = position, .child_index = child_index, @@ -246,7 +247,7 @@ IteratorComparison iterator_compare(const Iterator *old_iter, const Iterator *ne #ifdef DEBUG_GET_CHANGED_RANGES static inline void iterator_print_state(Iterator *self) { - TreePathEntry entry = *array_back(&self->path); + TreeCursorEntry entry = *array_back(&self->cursor.stack); TSPoint start = iterator_start_position(self).extent; TSPoint end = iterator_end_position(self).extent; const char *name = ts_language_symbol_name(self->language, entry.tree->symbol); @@ -261,12 +262,12 @@ static inline void iterator_print_state(Iterator *self) { #endif unsigned ts_tree_get_changed_ranges(Tree *old_tree, Tree *new_tree, - TreePath *path1, TreePath *path2, + TSTreeCursor *cursor1, TSTreeCursor *cursor2, const TSLanguage *language, TSRange **ranges) { RangeArray results = array_new(); - Iterator old_iter = iterator_new(path1, old_tree, language); - Iterator new_iter = iterator_new(path2, new_tree, language); + Iterator old_iter = iterator_new(cursor1, old_tree, language); + Iterator new_iter = iterator_new(cursor2, new_tree, language); Length position = iterator_start_position(&old_iter); Length next_position = iterator_start_position(&new_iter); @@ -348,8 +349,8 @@ unsigned ts_tree_get_changed_ranges(Tree *old_tree, Tree *new_tree, position = next_position; } while (!iterator_done(&old_iter) && !iterator_done(&new_iter)); - *path1 = old_iter.path; - *path2 = new_iter.path; + *cursor1 = old_iter.cursor; + *cursor2 = new_iter.cursor; *ranges = results.contents; return results.size; } diff --git a/src/runtime/get_changed_ranges.h b/src/runtime/get_changed_ranges.h index 360cdbd4..fa408aaa 100644 --- a/src/runtime/get_changed_ranges.h +++ b/src/runtime/get_changed_ranges.h @@ -3,17 +3,8 @@ #include "runtime/tree.h" -typedef struct { - Tree *tree; - Length position; - uint32_t child_index; - uint32_t structural_child_index; -} TreePathEntry; - -typedef Array(TreePathEntry) TreePath; - unsigned ts_tree_get_changed_ranges( - Tree *old_tree, Tree *new_tree, TreePath *path1, TreePath *path2, + Tree *old_tree, Tree *new_tree, TSTreeCursor *cursor1, TSTreeCursor *cursor2, const TSLanguage *language, TSRange **ranges ); diff --git a/src/runtime/tree_cursor.c b/src/runtime/tree_cursor.c new file mode 100644 index 00000000..db8298d2 --- /dev/null +++ b/src/runtime/tree_cursor.c @@ -0,0 +1,134 @@ +#include "tree_sitter/runtime.h" +#include "runtime/alloc.h" +#include "runtime/tree_cursor.h" +#include "runtime/document.h" +#include "runtime/language.h" + +TSTreeCursor *ts_tree_cursor_new(const TSDocument *document) { + TSTreeCursor *self = ts_malloc(sizeof(TSTreeCursor)); + self->document = document; + array_init(&self->stack); + array_push(&self->stack, ((TreeCursorEntry) { + .tree = document->tree, + .position = length_zero(), + .child_index = 0, + .structural_child_index = 0, + })); + return self; +} + +void ts_tree_cursor_delete(TSTreeCursor *self) { + array_delete(&self->stack); + ts_free(self); +} + +bool ts_tree_cursor_goto_first_child(TSTreeCursor *self) { + TreeCursorEntry *last_entry = array_back(&self->stack); + Tree *tree = last_entry->tree; + Length position = last_entry->position; + + bool did_descend; + do { + did_descend = false; + + uint32_t structural_child_index = 0; + for (uint32_t i = 0; i < tree->children.size; i++) { + Tree *child = tree->children.contents[i]; + if (child->visible || child->visible_child_count > 0) { + array_push(&self->stack, ((TreeCursorEntry) { + .tree = child, + .child_index = i, + .structural_child_index = structural_child_index, + .position = position, + })); + + if (child->visible) { + return true; + } else { + tree = child; + did_descend = true; + break; + } + } + if (!child->extra) structural_child_index++; + position = length_add(position, ts_tree_total_size(child)); + } + } while (did_descend); + + return false; +} + +bool ts_tree_cursor_goto_next_sibling(TSTreeCursor *self) { + TreeCursorEntry *child_entry = array_back(&self->stack); + + for (unsigned i = self->stack.size - 2; i + 1 > 0; i--) { + TreeCursorEntry *parent_entry = &self->stack.contents[i]; + + Tree *parent = parent_entry->tree; + uint32_t child_index = child_entry->child_index; + uint32_t structural_child_index = child_entry->structural_child_index; + Length position = child_entry->position; + Tree *child = parent->children.contents[child_index]; + + while (++child_index < parent->children.size) { + if (!child->extra) structural_child_index++; + position = length_add(position, ts_tree_total_size(child)); + child = parent->children.contents[child_index]; + + if (child->visible || child->visible_child_count > 0) { + self->stack.contents[i + 1] = (TreeCursorEntry) { + .tree = child, + .child_index = child_index, + .structural_child_index = structural_child_index, + .position = position, + }; + self->stack.size = i + 2; + + if (child->visible) { + return true; + } else { + ts_tree_cursor_goto_first_child(self); + return true; + } + } + } + + child_entry = parent_entry; + if (parent->visible) break; + } + + return false; +} + +bool ts_tree_cursor_goto_parent(TSTreeCursor *self) { + for (unsigned i = self->stack.size - 2; i + 1 > 0; i--) { + TreeCursorEntry *entry = &self->stack.contents[i]; + if (entry->tree->visible) { + self->stack.size = i + 1; + return true; + } + } + return false; +} + +TSNode ts_tree_cursor_current_node(TSTreeCursor *self) { + TreeCursorEntry *last_entry = array_back(&self->stack); + TSSymbol alias_symbol = 0; + if (self->stack.size > 1) { + TreeCursorEntry *parent_entry = &self->stack.contents[self->stack.size - 2]; + const TSSymbol *alias_sequence = ts_language_alias_sequence( + self->document->parser.language, + parent_entry->tree->alias_sequence_id + ); + if (alias_sequence) { + alias_symbol = alias_sequence[last_entry->structural_child_index]; + } + } + return (TSNode) { + .document = self->document, + .subtree = last_entry->tree, + .position = last_entry->position.extent, + .byte = last_entry->position.bytes, + .alias_symbol = alias_symbol, + }; +} diff --git a/src/runtime/tree_cursor.h b/src/runtime/tree_cursor.h new file mode 100644 index 00000000..33e1d802 --- /dev/null +++ b/src/runtime/tree_cursor.h @@ -0,0 +1,20 @@ +#ifndef RUNTIME_TREE_CURSOR_H_ +#define RUNTIME_TREE_CURSOR_H_ + +#include "runtime/tree.h" + +typedef struct { + Tree *tree; + Length position; + uint32_t child_index; + uint32_t structural_child_index; +} TreeCursorEntry; + +struct TSTreeCursor { + const TSDocument *document; + Array(TreeCursorEntry) stack; +}; + +TSTreeCursor *ts_tree_cursor_new(const TSDocument *); + +#endif // RUNTIME_TREE_CURSOR_H_ diff --git a/test/runtime/node_test.cc b/test/runtime/node_test.cc index 91717ed2..808084be 100644 --- a/test/runtime/node_test.cc +++ b/test/runtime/node_test.cc @@ -517,4 +517,132 @@ describe("Node", [&]() { }); }); +describe("TreeCursor", [&]() { + TSDocument *document; + TSTreeCursor *cursor; + + before_each([&]() { + record_alloc::start(); + + document = ts_document_new(); + ts_document_set_language(document, load_real_language("json")); + ts_document_set_input_string(document, json_string.c_str()); + ts_document_parse(document); + + cursor = ts_document_tree_cursor(document); + }); + + after_each([&]() { + ts_tree_cursor_delete(cursor); + ts_document_free(document); + + record_alloc::stop(); + AssertThat(record_alloc::outstanding_allocation_indices(), IsEmpty()); + }); + + it("can walk the tree", [&]() { + TSNode node = ts_tree_cursor_current_node(cursor); + AssertThat(ts_node_type(node), Equals("value")); + AssertThat(ts_node_start_byte(node), Equals(array_index)); + + AssertThat(ts_tree_cursor_goto_first_child(cursor), IsTrue()); + node = ts_tree_cursor_current_node(cursor); + AssertThat(ts_node_type(node), Equals("array")); + AssertThat(ts_node_start_byte(node), Equals(array_index)); + + AssertThat(ts_tree_cursor_goto_first_child(cursor), IsTrue()); + node = ts_tree_cursor_current_node(cursor); + AssertThat(ts_node_type(node), Equals("[")); + AssertThat(ts_node_start_byte(node), Equals(array_index)); + + // Cannot descend into a node with no children + AssertThat(ts_tree_cursor_goto_first_child(cursor), IsFalse()); + node = ts_tree_cursor_current_node(cursor); + AssertThat(ts_node_type(node), Equals("[")); + AssertThat(ts_node_start_byte(node), Equals(array_index)); + + AssertThat(ts_tree_cursor_goto_next_sibling(cursor), IsTrue()); + node = ts_tree_cursor_current_node(cursor); + AssertThat(ts_node_type(node), Equals("number")); + AssertThat(ts_node_start_byte(node), Equals(number_index)); + + AssertThat(ts_tree_cursor_goto_next_sibling(cursor), IsTrue()); + node = ts_tree_cursor_current_node(cursor); + AssertThat(ts_node_type(node), Equals(",")); + AssertThat(ts_node_start_byte(node), Equals(number_end_index)); + + AssertThat(ts_tree_cursor_goto_next_sibling(cursor), IsTrue()); + node = ts_tree_cursor_current_node(cursor); + AssertThat(ts_node_type(node), Equals("false")); + AssertThat(ts_node_start_byte(node), Equals(false_index)); + + AssertThat(ts_tree_cursor_goto_next_sibling(cursor), IsTrue()); + node = ts_tree_cursor_current_node(cursor); + AssertThat(ts_node_type(node), Equals(",")); + AssertThat(ts_node_start_byte(node), Equals(false_end_index)); + + AssertThat(ts_tree_cursor_goto_next_sibling(cursor), IsTrue()); + node = ts_tree_cursor_current_node(cursor); + AssertThat(ts_node_type(node), Equals("object")); + AssertThat(ts_node_start_byte(node), Equals(object_index)); + + AssertThat(ts_tree_cursor_goto_first_child(cursor), IsTrue()); + node = ts_tree_cursor_current_node(cursor); + AssertThat(ts_node_type(node), Equals("{")); + AssertThat(ts_node_start_byte(node), Equals(object_index)); + + AssertThat(ts_tree_cursor_goto_next_sibling(cursor), IsTrue()); + node = ts_tree_cursor_current_node(cursor); + AssertThat(ts_node_type(node), Equals("pair")); + AssertThat(ts_node_start_byte(node), Equals(string_index)); + + AssertThat(ts_tree_cursor_goto_first_child(cursor), IsTrue()); + node = ts_tree_cursor_current_node(cursor); + AssertThat(ts_node_type(node), Equals("string")); + AssertThat(ts_node_start_byte(node), Equals(string_index)); + + AssertThat(ts_tree_cursor_goto_next_sibling(cursor), IsTrue()); + node = ts_tree_cursor_current_node(cursor); + AssertThat(ts_node_type(node), Equals(":")); + AssertThat(ts_node_start_byte(node), Equals(string_end_index)); + + AssertThat(ts_tree_cursor_goto_next_sibling(cursor), IsTrue()); + node = ts_tree_cursor_current_node(cursor); + AssertThat(ts_node_type(node), Equals("null")); + AssertThat(ts_node_start_byte(node), Equals(null_index)); + + // Cannot move beyond a node with no next sibling + AssertThat(ts_tree_cursor_goto_next_sibling(cursor), IsFalse()); + node = ts_tree_cursor_current_node(cursor); + AssertThat(ts_node_type(node), Equals("null")); + AssertThat(ts_node_start_byte(node), Equals(null_index)); + + AssertThat(ts_tree_cursor_goto_parent(cursor), IsTrue()); + node = ts_tree_cursor_current_node(cursor); + AssertThat(ts_node_type(node), Equals("pair")); + AssertThat(ts_node_start_byte(node), Equals(string_index)); + + AssertThat(ts_tree_cursor_goto_parent(cursor), IsTrue()); + node = ts_tree_cursor_current_node(cursor); + AssertThat(ts_node_type(node), Equals("object")); + AssertThat(ts_node_start_byte(node), Equals(object_index)); + + AssertThat(ts_tree_cursor_goto_parent(cursor), IsTrue()); + node = ts_tree_cursor_current_node(cursor); + AssertThat(ts_node_type(node), Equals("array")); + AssertThat(ts_node_start_byte(node), Equals(array_index)); + + AssertThat(ts_tree_cursor_goto_parent(cursor), IsTrue()); + node = ts_tree_cursor_current_node(cursor); + AssertThat(ts_node_type(node), Equals("value")); + AssertThat(ts_node_start_byte(node), Equals(array_index)); + + // The root node doesn't have a parent. + AssertThat(ts_tree_cursor_goto_parent(cursor), IsFalse()); + node = ts_tree_cursor_current_node(cursor); + AssertThat(ts_node_type(node), Equals("value")); + AssertThat(ts_node_start_byte(node), Equals(array_index)); + }); +}); + END_TEST From 78d158899ec0b14fa796430f8830834b037df7c4 Mon Sep 17 00:00:00 2001 From: Max Brunsfeld Date: Wed, 9 May 2018 17:30:13 -0700 Subject: [PATCH 34/90] Use atomic operations for updating syntax tree reference counts Co-Authored-By: Rick Winfrey --- src/runtime/atomic.h | 30 ++++++++++++++++++++++++++++++ src/runtime/tree.c | 6 +++--- src/runtime/tree.h | 2 +- 3 files changed, 34 insertions(+), 4 deletions(-) create mode 100644 src/runtime/atomic.h diff --git a/src/runtime/atomic.h b/src/runtime/atomic.h new file mode 100644 index 00000000..d1ab1f23 --- /dev/null +++ b/src/runtime/atomic.h @@ -0,0 +1,30 @@ +#ifndef RUNTIME_ATOMIC_H_ +#define RUNTIME_ATOMIC_H_ + +#include + +#ifdef _WIN32 + +#include + +static inline uint32_t atomic_inc(volatile uint32_t *p) { + return InterlockedIncrement(p); +} + +static inline uint32_t atomic_dec(volatile uint32_t *p) { + return InterlockedDecrement(p); +} + +#else + +static inline uint32_t atomic_inc(volatile uint32_t *p) { + return __sync_add_and_fetch(p, 1u); +} + +static inline uint32_t atomic_dec(volatile uint32_t *p) { + return __sync_sub_and_fetch(p, 1u); +} + +#endif + +#endif // RUNTIME_ATOMIC_H_ diff --git a/src/runtime/tree.c b/src/runtime/tree.c index 0174d3d8..85707c1b 100644 --- a/src/runtime/tree.c +++ b/src/runtime/tree.c @@ -5,6 +5,7 @@ #include #include #include "runtime/alloc.h" +#include "runtime/atomic.h" #include "runtime/tree.h" #include "runtime/length.h" #include "runtime/language.h" @@ -356,7 +357,7 @@ Tree *ts_tree_make_missing_leaf(TreePool *pool, TSSymbol symbol, const TSLanguag void ts_tree_retain(Tree *self) { assert(self->ref_count > 0); - self->ref_count++; + atomic_inc(&self->ref_count); assert(self->ref_count != 0); } @@ -366,8 +367,7 @@ void ts_tree_release(TreePool *pool, Tree *self) { while (pool->tree_stack.size > 0) { Tree *tree = array_pop(&pool->tree_stack); assert(tree->ref_count > 0); - tree->ref_count--; - if (tree->ref_count == 0) { + if (atomic_dec(&tree->ref_count) == 0) { if (tree->children.size > 0) { for (uint32_t i = 0; i < tree->children.size; i++) { array_push(&pool->tree_stack, tree->children.contents[i]); diff --git a/src/runtime/tree.h b/src/runtime/tree.h index da5f1e90..0eb6f929 100644 --- a/src/runtime/tree.h +++ b/src/runtime/tree.h @@ -29,7 +29,7 @@ typedef Array(Tree *) TreeArray; struct Tree { Length padding; Length size; - uint32_t ref_count; + volatile uint32_t ref_count; uint32_t bytes_scanned; uint32_t error_cost; uint32_t node_count; From a53d0b43a1cc721c419abbf6d85fb32a9b4a0612 Mon Sep 17 00:00:00 2001 From: Max Brunsfeld Date: Wed, 9 May 2018 22:55:54 -0700 Subject: [PATCH 35/90] Remove unnecessary include --- test/helpers/tree_helpers.cc | 1 - 1 file changed, 1 deletion(-) diff --git a/test/helpers/tree_helpers.cc b/test/helpers/tree_helpers.cc index 553f8b49..9a86b364 100644 --- a/test/helpers/tree_helpers.cc +++ b/test/helpers/tree_helpers.cc @@ -1,7 +1,6 @@ #include "test_helper.h" #include "helpers/tree_helpers.h" #include "helpers/point_helpers.h" -#include "runtime/document.h" #include using std::string; From 59694e60faebfbf8f19f0c31f7d751f02c5debf0 Mon Sep 17 00:00:00 2001 From: Max Brunsfeld Date: Thu, 10 May 2018 09:13:46 -0700 Subject: [PATCH 36/90] Rename ts_tree_assign_parents -> ts_tree_balance --- src/runtime/parser.c | 2 +- src/runtime/tree.c | 3 ++- src/runtime/tree.h | 2 +- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/src/runtime/parser.c b/src/runtime/parser.c index cac6c9e1..f1ba0d16 100644 --- a/src/runtime/parser.c +++ b/src/runtime/parser.c @@ -1336,7 +1336,7 @@ Tree *parser_parse(Parser *self, TSInput input, Tree *old_tree, bool halt_on_err reusable_node_delete(&reusable_node); ts_stack_clear(self->stack); parser__set_cached_token(self, 0, NULL, NULL); - ts_tree_assign_parents(self->finished_tree, &self->tree_pool, self->language); + ts_tree_balance(self->finished_tree, &self->tree_pool, self->language); LOG("done"); LOG_TREE(); diff --git a/src/runtime/tree.c b/src/runtime/tree.c index 85707c1b..e80a1f68 100644 --- a/src/runtime/tree.c +++ b/src/runtime/tree.c @@ -208,11 +208,12 @@ static void ts_tree__compress(Tree *self, unsigned count, const TSLanguage *lang } } -void ts_tree_assign_parents(Tree *self, TreePool *pool, const TSLanguage *language) { +void ts_tree_balance(Tree *self, TreePool *pool, const TSLanguage *language) { array_clear(&pool->tree_stack); array_push(&pool->tree_stack, self); while (pool->tree_stack.size > 0) { Tree *tree = array_pop(&pool->tree_stack); + assert(tree); if (tree->repeat_depth > 0) { if (tree->children.contents[0]->repeat_depth > tree->children.contents[1]->repeat_depth) { diff --git a/src/runtime/tree.h b/src/runtime/tree.h index 0eb6f929..ce96e41d 100644 --- a/src/runtime/tree.h +++ b/src/runtime/tree.h @@ -99,7 +99,7 @@ void ts_tree_release(TreePool *, Tree *tree); bool ts_tree_eq(const Tree *tree1, const Tree *tree2); int ts_tree_compare(const Tree *tree1, const Tree *tree2); void ts_tree_set_children(Tree *, TreeArray *, const TSLanguage *); -void ts_tree_assign_parents(Tree *, TreePool *, const TSLanguage *); +void ts_tree_balance(Tree *, TreePool *, const TSLanguage *); void ts_tree_edit(Tree *, const TSInputEdit *edit); char *ts_tree_string(const Tree *, const TSLanguage *, bool include_all); void ts_tree_print_dot_graph(const Tree *, const TSLanguage *, FILE *); From eff7283325d1fa9bfe0d394a894bd825fca73b8a Mon Sep 17 00:00:00 2001 From: Max Brunsfeld Date: Thu, 10 May 2018 09:48:50 -0700 Subject: [PATCH 37/90] Add assertion to satisfy clang static analyzer Co-Authored-By: Rick Winfrey --- src/runtime/tree.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/runtime/tree.c b/src/runtime/tree.c index e80a1f68..a7e9fb99 100644 --- a/src/runtime/tree.c +++ b/src/runtime/tree.c @@ -200,6 +200,7 @@ static void ts_tree__compress(Tree *self, unsigned count, const TSLanguage *lang while (stack->size > initial_stack_size) { tree = array_pop(stack); + assert(tree); Tree *child = tree->children.contents[0]; Tree *grandchild = child->children.contents[1]; ts_tree_set_children(grandchild, &grandchild->children, language); From df79ff59971465fc58f670ef40781a67aae21693 Mon Sep 17 00:00:00 2001 From: Max Brunsfeld Date: Thu, 10 May 2018 12:04:18 -0700 Subject: [PATCH 38/90] Refactor ts_tree_edit Co-Authored-By: Rick Winfrey --- src/runtime/tree.c | 142 +++++++++++++++++--------------------- test/runtime/tree_test.cc | 4 -- 2 files changed, 65 insertions(+), 81 deletions(-) diff --git a/src/runtime/tree.c b/src/runtime/tree.c index a7e9fb99..13a2f08d 100644 --- a/src/runtime/tree.c +++ b/src/runtime/tree.c @@ -11,6 +11,12 @@ #include "runtime/language.h" #include "runtime/error_costs.h" +typedef struct { + Length start; + Length added; + Length removed; +} Edit; + TSStateId TS_TREE_STATE_NONE = USHRT_MAX; // ExternalTokenState @@ -432,12 +438,8 @@ int ts_tree_compare(const Tree *left, const Tree *right) { return 0; } -static inline long min_byte(long a, long b) { - return a <= b ? a : b; -} - -bool ts_tree_invalidate_lookahead(Tree *self, uint32_t edit_byte_offset) { - if (edit_byte_offset >= self->bytes_scanned) return false; +void ts_tree_invalidate_lookahead(Tree *self, uint32_t edit_byte_offset) { + if (edit_byte_offset >= self->bytes_scanned) return; self->has_changes = true; if (self->children.size > 0) { uint32_t child_start_byte = 0; @@ -448,95 +450,70 @@ bool ts_tree_invalidate_lookahead(Tree *self, uint32_t edit_byte_offset) { child_start_byte += ts_tree_total_bytes(child); } } - return true; } -static inline TSPoint ts_tree_total_extent(const Tree *self) { - return point_add(self->padding.extent, self->size.extent); -} - -void ts_tree_edit(Tree *self, const TSInputEdit *edit) { - uint32_t old_end_byte = edit->start_byte + edit->bytes_removed; - uint32_t new_end_byte = edit->start_byte + edit->bytes_added; - TSPoint old_end_point = point_add(edit->start_point, edit->extent_removed); - TSPoint new_end_point = point_add(edit->start_point, edit->extent_added); - - assert(old_end_byte <= ts_tree_total_bytes(self)); +void ts_tree__edit(Tree *self, Edit edit) { + Length new_end = length_add(edit.start, edit.added); + Length old_end = length_add(edit.start, edit.removed); self->has_changes = true; - if (edit->start_byte < self->padding.bytes) { - if (self->padding.bytes >= old_end_byte) { - uint32_t trailing_padding_bytes = self->padding.bytes - old_end_byte; - TSPoint trailing_padding_extent = point_sub(self->padding.extent, old_end_point); - self->padding.bytes = new_end_byte + trailing_padding_bytes; - self->padding.extent = point_add(new_end_point, trailing_padding_extent); - } else { - uint32_t removed_content_bytes = old_end_byte - self->padding.bytes; - TSPoint removed_content_extent = point_sub(old_end_point, self->padding.extent); - self->size.bytes = self->size.bytes - removed_content_bytes; - self->size.extent = point_sub(self->size.extent, removed_content_extent); - self->padding.bytes = new_end_byte; - self->padding.extent = new_end_point; - } - } else if (edit->start_byte == self->padding.bytes && edit->bytes_removed == 0) { - self->padding.bytes = self->padding.bytes + edit->bytes_added; - self->padding.extent = point_add(self->padding.extent, edit->extent_added); + if (old_end.bytes <= self->padding.bytes) { + self->padding = length_add(new_end, length_sub(self->padding, old_end)); + } else if (edit.start.bytes < self->padding.bytes) { + self->size = length_sub(self->size, length_sub(old_end, self->padding)); + self->padding = new_end; + } else if (edit.start.bytes == self->padding.bytes && edit.removed.bytes == 0) { + self->padding = length_add(self->padding, edit.added); } else { - uint32_t trailing_content_bytes = ts_tree_total_bytes(self) - old_end_byte; - TSPoint trailing_content_extent = point_sub(ts_tree_total_extent(self), old_end_point); - self->size.bytes = new_end_byte + trailing_content_bytes - self->padding.bytes; - self->size.extent = point_sub(point_add(new_end_point, trailing_content_extent), self->padding.extent); + Length new_total_size = length_add(new_end, length_sub(ts_tree_total_size(self), old_end)); + self->size = length_sub(new_total_size, self->padding); } - bool found_first_child = false; - long remaining_bytes_to_delete = 0; - TSPoint remaining_extent_to_delete = {0, 0}; Length child_left, child_right = length_zero(); for (uint32_t i = 0; i < self->children.size; i++) { Tree *child = self->children.contents[i]; + Length child_size = ts_tree_total_size(child); child_left = child_right; - child_right = length_add(child_left, ts_tree_total_size(child)); + child_right = length_add(child_left, child_size); - if (!found_first_child && child_right.bytes >= edit->start_byte) { - found_first_child = true; - TSInputEdit child_edit = { - .start_byte = edit->start_byte - child_left.bytes, - .bytes_added = edit->bytes_added, - .bytes_removed = edit->bytes_removed, - .start_point = point_sub(edit->start_point, child_left.extent), - .extent_added = edit->extent_added, - .extent_removed = edit->extent_removed, + if (child_left.bytes > old_end.bytes || + (child_left.bytes == old_end.bytes && child_size.bytes > 0 && i > 0)) break; + + if (child_right.bytes > edit.start.bytes || + (child_right.bytes == edit.start.bytes && edit.removed.bytes == 0)) { + Edit child_edit = { + .start = length_sub(edit.start, child_left), + .added = edit.added, + .removed = edit.removed, }; - if (old_end_byte > child_right.bytes) { - child_edit.bytes_removed = child_right.bytes - edit->start_byte; - child_edit.extent_removed = point_sub(child_right.extent, edit->start_point); - remaining_bytes_to_delete = old_end_byte - child_right.bytes; - remaining_extent_to_delete = point_sub(old_end_point, child_right.extent); + if (edit.start.bytes < child_left.bytes) { + child_edit.start = length_zero(); } - ts_tree_edit(child, &child_edit); - } else if (remaining_bytes_to_delete > 0) { - TSInputEdit child_edit = { - .start_byte = 0, - .bytes_added = 0, - .bytes_removed = min_byte(remaining_bytes_to_delete, ts_tree_total_bytes(child)), - .start_point = {0, 0}, - .extent_added = {0, 0}, - .extent_removed = point_min(remaining_extent_to_delete, ts_tree_total_size(child).extent), - }; - remaining_bytes_to_delete -= child_edit.bytes_removed; - remaining_extent_to_delete = point_sub(remaining_extent_to_delete, child_edit.extent_removed); - ts_tree_edit(child, &child_edit); - } else { - ts_tree_invalidate_lookahead(child, edit->start_byte - child_left.bytes); - } + if (old_end.bytes > child_right.bytes) { + child_edit.removed = length_sub(child_size, child_edit.start); + } - child_right = length_add(child_left, ts_tree_total_size(child)); + edit.added = length_zero(); + edit.removed = length_sub(edit.removed, child_edit.removed); + + ts_tree__edit(child, child_edit); + } else if (child_left.bytes <= edit.start.bytes) { + ts_tree_invalidate_lookahead(child, edit.start.bytes - child_left.bytes); + } } } +void ts_tree_edit(Tree *self, const TSInputEdit *edit) { + ts_tree__edit(self, (Edit) { + .start = {edit->start_byte, edit->start_point}, + .added = {edit->bytes_added, edit->extent_added}, + .removed = {edit->bytes_removed, edit->extent_removed}, + }); +} + Tree *ts_tree_last_external_token(Tree *tree) { if (!tree->has_external_tokens) return NULL; while (tree->children.size > 0) { @@ -649,9 +626,20 @@ void ts_tree__print_dot_graph(const Tree *self, uint32_t byte_offset, if (self->extra) fprintf(f, ", fontcolor=gray"); - fprintf(f, ", tooltip=\"address:%p\nrange:%u - %u\nstate:%d\nerror-cost:%u\nrepeat-depth:%u\"]\n", - self, byte_offset, byte_offset + ts_tree_total_bytes(self), self->parse_state, - self->error_cost, self->repeat_depth); + fprintf(f, ", tooltip=\"" + "address:%p\n" + "range:%u - %u\n" + "state:%d\n" + "error-cost:%u\n" + "repeat-depth:%u\n" + "bytes-scanned:%u\"]\n", + self, + byte_offset, byte_offset + ts_tree_total_bytes(self), + self->parse_state, + self->error_cost, + self->repeat_depth, + self->bytes_scanned + ); const TSSymbol *alias_sequence = ts_language_alias_sequence(language, self->alias_sequence_id); uint32_t structural_child_index = 0; diff --git a/test/runtime/tree_test.cc b/test/runtime/tree_test.cc index 262572b1..e80743e7 100644 --- a/test/runtime/tree_test.cc +++ b/test/runtime/tree_test.cc @@ -255,8 +255,6 @@ describe("Tree", []() { ts_tree_edit(tree, &edit); assert_consistent(tree); - assert_consistent(tree); - AssertThat(tree->has_changes, IsTrue()); AssertThat(tree->padding, Equals({4, {0, 4}})); AssertThat(tree->size, Equals({13, {0, 13}})); @@ -305,8 +303,6 @@ describe("Tree", []() { ts_tree_edit(tree, &edit); assert_consistent(tree); - assert_consistent(tree); - AssertThat(tree->has_changes, IsTrue()); AssertThat(tree->padding, Equals({4, {0, 4}})); AssertThat(tree->size, Equals({4, {0, 4}})); From 09e663c7d1118da4aaf5b8832e54e32bd0258dc2 Mon Sep 17 00:00:00 2001 From: Max Brunsfeld Date: Thu, 10 May 2018 12:23:05 -0700 Subject: [PATCH 39/90] Make ts_tree_edit return a new tree rather than mutating its argument Co-Authored-By: Rick Winfrey --- src/runtime/document.c | 2 +- src/runtime/tree.c | 78 +++++++++++++++++++++++++-------------- src/runtime/tree.h | 2 +- test/runtime/tree_test.cc | 14 +++---- 4 files changed, 59 insertions(+), 37 deletions(-) diff --git a/src/runtime/document.c b/src/runtime/document.c index ab10ae49..9a13ca30 100644 --- a/src/runtime/document.c +++ b/src/runtime/document.c @@ -97,7 +97,7 @@ void ts_document_edit(TSDocument *self, TSInputEdit edit) { if (edit.bytes_removed > max_bytes - edit.start_byte) edit.bytes_removed = max_bytes - edit.start_byte; - ts_tree_edit(self->tree, &edit); + self->tree = ts_tree_edit(self->tree, &edit, &self->parser.tree_pool); if (self->parser.print_debugging_graphs) { ts_tree_print_dot_graph(self->tree, self->parser.language, stderr); diff --git a/src/runtime/tree.c b/src/runtime/tree.c index 13a2f08d..602392f4 100644 --- a/src/runtime/tree.c +++ b/src/runtime/tree.c @@ -58,8 +58,9 @@ bool ts_tree_array_copy(TreeArray self, TreeArray *dest) { if (self.capacity > 0) { contents = ts_calloc(self.capacity, sizeof(Tree *)); memcpy(contents, self.contents, self.size * sizeof(Tree *)); - for (uint32_t i = 0; i < self.size; i++) + for (uint32_t i = 0; i < self.size; i++) { ts_tree_retain(contents[i]); + } } dest->size = self.size; @@ -172,10 +173,23 @@ Tree *ts_tree_make_error(TreePool *pool, Length size, Length padding, int32_t lo Tree *ts_tree_make_copy(TreePool *pool, Tree *self) { Tree *result = ts_tree_pool_allocate(pool); *result = *self; + if (result->children.size > 0) { + ts_tree_array_copy(self->children, &result->children); + } result->ref_count = 1; return result; } +static Tree *ts_tree_make_mut(TreePool *pool, Tree *self) { + if (self->ref_count == 1) { + return self; + } else { + Tree *result = ts_tree_make_copy(pool, self); + ts_tree_release(pool, self); + return result; + } +} + static void ts_tree__compress(Tree *self, unsigned count, const TSLanguage *language, TreeArray *stack) { unsigned initial_stack_size = stack->size; @@ -438,42 +452,48 @@ int ts_tree_compare(const Tree *left, const Tree *right) { return 0; } -void ts_tree_invalidate_lookahead(Tree *self, uint32_t edit_byte_offset) { - if (edit_byte_offset >= self->bytes_scanned) return; - self->has_changes = true; - if (self->children.size > 0) { +Tree *ts_tree_invalidate_lookahead(Tree *self, uint32_t edit_byte_offset, TreePool *pool) { + if (edit_byte_offset >= self->bytes_scanned) return self; + + Tree *result = ts_tree_make_mut(pool, self); + result->has_changes = true; + + if (result->children.size > 0) { uint32_t child_start_byte = 0; - for (uint32_t i = 0; i < self->children.size; i++) { - Tree *child = self->children.contents[i]; + for (uint32_t i = 0; i < result->children.size; i++) { + Tree **child = &result->children.contents[i]; if (child_start_byte > edit_byte_offset) break; - ts_tree_invalidate_lookahead(child, edit_byte_offset - child_start_byte); - child_start_byte += ts_tree_total_bytes(child); + *child = ts_tree_invalidate_lookahead(*child, edit_byte_offset - child_start_byte, pool); + child_start_byte += ts_tree_total_bytes(*child); } } + + return result; } -void ts_tree__edit(Tree *self, Edit edit) { +Tree *ts_tree__edit(Tree *self, Edit edit, TreePool *pool) { Length new_end = length_add(edit.start, edit.added); Length old_end = length_add(edit.start, edit.removed); - self->has_changes = true; + Tree *result = ts_tree_make_mut(pool, self); + result->has_changes = true; - if (old_end.bytes <= self->padding.bytes) { - self->padding = length_add(new_end, length_sub(self->padding, old_end)); - } else if (edit.start.bytes < self->padding.bytes) { - self->size = length_sub(self->size, length_sub(old_end, self->padding)); - self->padding = new_end; - } else if (edit.start.bytes == self->padding.bytes && edit.removed.bytes == 0) { - self->padding = length_add(self->padding, edit.added); + if (old_end.bytes <= result->padding.bytes) { + result->padding = length_add(new_end, length_sub(result->padding, old_end)); + } else if (edit.start.bytes < result->padding.bytes) { + result->size = length_sub(result->size, length_sub(old_end, result->padding)); + result->padding = new_end; + } else if (edit.start.bytes == result->padding.bytes && edit.removed.bytes == 0) { + result->padding = length_add(result->padding, edit.added); } else { - Length new_total_size = length_add(new_end, length_sub(ts_tree_total_size(self), old_end)); - self->size = length_sub(new_total_size, self->padding); + Length new_total_size = length_add(new_end, length_sub(ts_tree_total_size(result), old_end)); + result->size = length_sub(new_total_size, result->padding); } Length child_left, child_right = length_zero(); - for (uint32_t i = 0; i < self->children.size; i++) { - Tree *child = self->children.contents[i]; - Length child_size = ts_tree_total_size(child); + for (uint32_t i = 0; i < result->children.size; i++) { + Tree **child = &result->children.contents[i]; + Length child_size = ts_tree_total_size(*child); child_left = child_right; child_right = length_add(child_left, child_size); @@ -499,19 +519,21 @@ void ts_tree__edit(Tree *self, Edit edit) { edit.added = length_zero(); edit.removed = length_sub(edit.removed, child_edit.removed); - ts_tree__edit(child, child_edit); + *child = ts_tree__edit(*child, child_edit, pool); } else if (child_left.bytes <= edit.start.bytes) { - ts_tree_invalidate_lookahead(child, edit.start.bytes - child_left.bytes); + *child = ts_tree_invalidate_lookahead(*child, edit.start.bytes - child_left.bytes, pool); } } + + return result; } -void ts_tree_edit(Tree *self, const TSInputEdit *edit) { - ts_tree__edit(self, (Edit) { +Tree *ts_tree_edit(Tree *self, const TSInputEdit *edit, TreePool *pool) { + return ts_tree__edit(self, (Edit) { .start = {edit->start_byte, edit->start_point}, .added = {edit->bytes_added, edit->extent_added}, .removed = {edit->bytes_removed, edit->extent_removed}, - }); + }, pool); } Tree *ts_tree_last_external_token(Tree *tree) { diff --git a/src/runtime/tree.h b/src/runtime/tree.h index ce96e41d..4fa95522 100644 --- a/src/runtime/tree.h +++ b/src/runtime/tree.h @@ -100,7 +100,7 @@ bool ts_tree_eq(const Tree *tree1, const Tree *tree2); int ts_tree_compare(const Tree *tree1, const Tree *tree2); void ts_tree_set_children(Tree *, TreeArray *, const TSLanguage *); void ts_tree_balance(Tree *, TreePool *, const TSLanguage *); -void ts_tree_edit(Tree *, const TSInputEdit *edit); +Tree *ts_tree_edit(Tree *, const TSInputEdit *edit, TreePool *); char *ts_tree_string(const Tree *, const TSLanguage *, bool include_all); void ts_tree_print_dot_graph(const Tree *, const TSLanguage *, FILE *); Tree *ts_tree_last_external_token(Tree *); diff --git a/test/runtime/tree_test.cc b/test/runtime/tree_test.cc index e80743e7..2d26f021 100644 --- a/test/runtime/tree_test.cc +++ b/test/runtime/tree_test.cc @@ -178,7 +178,7 @@ describe("Tree", []() { }); describe("edit", [&]() { - Tree *tree = nullptr; + Tree *tree; before_each([&]() { tree = ts_tree_make_node(&pool, symbol1, tree_array({ @@ -204,7 +204,7 @@ describe("Tree", []() { edit.start_point = {0, 1}; edit.extent_removed = {0, 0}; edit.extent_added = {0, 1}; - ts_tree_edit(tree, &edit); + tree = ts_tree_edit(tree, &edit, &pool); assert_consistent(tree); AssertThat(tree->has_changes, IsTrue()); @@ -230,7 +230,7 @@ describe("Tree", []() { edit.start_point = {0, 1}; edit.extent_removed = {0, 3}; edit.extent_added = {0, 4}; - ts_tree_edit(tree, &edit); + tree = ts_tree_edit(tree, &edit, &pool); assert_consistent(tree); AssertThat(tree->has_changes, IsTrue()); @@ -252,7 +252,7 @@ describe("Tree", []() { edit.start_point = {0, 2}; edit.extent_removed = {0, 0}; edit.extent_added = {0, 2}; - ts_tree_edit(tree, &edit); + tree = ts_tree_edit(tree, &edit, &pool); assert_consistent(tree); AssertThat(tree->has_changes, IsTrue()); @@ -276,7 +276,7 @@ describe("Tree", []() { edit.start_point = {0, 2}; edit.extent_removed = {0, 2}; edit.extent_added = {0, 5}; - ts_tree_edit(tree, &edit); + tree = ts_tree_edit(tree, &edit, &pool); assert_consistent(tree); AssertThat(tree->has_changes, IsTrue()); @@ -300,7 +300,7 @@ describe("Tree", []() { edit.start_point = {0, 1}; edit.extent_removed = {0, 10}; edit.extent_added = {0, 3}; - ts_tree_edit(tree, &edit); + tree = ts_tree_edit(tree, &edit, &pool); assert_consistent(tree); AssertThat(tree->has_changes, IsTrue()); @@ -332,7 +332,7 @@ describe("Tree", []() { edit.start_point = {0, 6}; edit.extent_removed = {0, 1}; edit.extent_added = {0, 1}; - ts_tree_edit(tree, &edit); + tree = ts_tree_edit(tree, &edit, &pool); assert_consistent(tree); AssertThat(tree->children.contents[0]->has_changes, IsTrue()); From 61327b627a575e2d47968fc2ac68a2073c4407f1 Mon Sep 17 00:00:00 2001 From: Max Brunsfeld Date: Thu, 10 May 2018 12:28:16 -0700 Subject: [PATCH 40/90] Add a unit test asserting that ts_tree_edit doesn't mutate the tree Co-Authored-By: Rick Winfrey --- test/runtime/tree_test.cc | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/test/runtime/tree_test.cc b/test/runtime/tree_test.cc index 2d26f021..1613b930 100644 --- a/test/runtime/tree_test.cc +++ b/test/runtime/tree_test.cc @@ -195,6 +195,35 @@ describe("Tree", []() { ts_tree_release(&pool, tree); }); + it("does not mutate the argument", [&]() { + TSInputEdit edit; + edit.start_byte = 1; + edit.bytes_removed = 0; + edit.bytes_added = 1; + edit.start_point = {0, 1}; + edit.extent_removed = {0, 0}; + edit.extent_added = {0, 1}; + + ts_tree_retain(tree); + Tree *new_tree = ts_tree_edit(tree, &edit, &pool); + assert_consistent(tree); + assert_consistent(new_tree); + + AssertThat(tree->has_changes, IsFalse()); + AssertThat(tree->padding, Equals({2, {0, 2}})); + AssertThat(tree->size, Equals({13, {0, 13}})); + + AssertThat(tree->children.contents[0]->has_changes, IsFalse()); + AssertThat(tree->children.contents[0]->padding, Equals({2, {0, 2}})); + AssertThat(tree->children.contents[0]->size, Equals({3, {0, 3}})); + + AssertThat(tree->children.contents[1]->has_changes, IsFalse()); + AssertThat(tree->children.contents[1]->padding, Equals({2, {0, 2}})); + AssertThat(tree->children.contents[1]->size, Equals({3, {0, 3}})); + + ts_tree_release(&pool, new_tree); + }); + describe("edits within a tree's padding", [&]() { it("resizes the padding of the tree and its leftmost descendants", [&]() { TSInputEdit edit; From 35510a612de0a458c2cca3640294eecb6fcb552c Mon Sep 17 00:00:00 2001 From: Max Brunsfeld Date: Thu, 10 May 2018 15:11:14 -0700 Subject: [PATCH 41/90] Rename Tree -> Subtree --- project.gyp | 2 +- src/runtime/document.c | 22 +- src/runtime/document.h | 4 +- src/runtime/get_changed_ranges.c | 22 +- src/runtime/get_changed_ranges.h | 6 +- src/runtime/language.c | 2 +- src/runtime/language.h | 2 +- src/runtime/lexer.c | 2 +- src/runtime/lexer.h | 2 +- src/runtime/node.c | 32 +-- src/runtime/parser.c | 204 ++++++++--------- src/runtime/parser.h | 14 +- src/runtime/reusable_node.h | 16 +- src/runtime/stack.c | 172 +++++++------- src/runtime/stack.h | 14 +- src/runtime/{tree.c => subtree.c} | 215 ++++++++--------- src/runtime/subtree.h | 125 ++++++++++ src/runtime/tree.h | 125 ---------- src/runtime/tree_cursor.c | 12 +- src/runtime/tree_cursor.h | 4 +- test/helpers/tree_helpers.cc | 12 +- test/helpers/tree_helpers.h | 8 +- test/runtime/stack_test.cc | 216 +++++++++--------- .../runtime/{tree_test.cc => subtree_test.cc} | 208 ++++++++--------- tests.gyp | 2 +- 25 files changed, 724 insertions(+), 719 deletions(-) rename src/runtime/{tree.c => subtree.c} (70%) create mode 100644 src/runtime/subtree.h delete mode 100644 src/runtime/tree.h rename test/runtime/{tree_test.cc => subtree_test.cc} (65%) diff --git a/project.gyp b/project.gyp index f59c3c91..cfc1776b 100644 --- a/project.gyp +++ b/project.gyp @@ -95,7 +95,7 @@ 'src/runtime/stack.c', 'src/runtime/parser.c', 'src/runtime/string_input.c', - 'src/runtime/tree.c', + 'src/runtime/subtree.c', 'src/runtime/tree_cursor.c', 'src/runtime/utf16.c', 'externals/utf8proc/utf8proc.c', diff --git a/src/runtime/document.c b/src/runtime/document.c index 9a13ca30..4298025a 100644 --- a/src/runtime/document.c +++ b/src/runtime/document.c @@ -1,5 +1,5 @@ #include "runtime/alloc.h" -#include "runtime/tree.h" +#include "runtime/subtree.h" #include "runtime/parser.h" #include "runtime/string_input.h" #include "runtime/document.h" @@ -19,7 +19,7 @@ TSDocument *ts_document_new() { } void ts_document_free(TSDocument *self) { - if (self->tree) ts_tree_release(&self->parser.tree_pool, self->tree); + if (self->tree) ts_subtree_release(&self->parser.tree_pool, self->tree); if (self->cursor1.stack.contents) array_delete(&self->cursor1.stack); if (self->cursor2.stack.contents) array_delete(&self->cursor2.stack); parser_destroy(&self->parser); @@ -41,7 +41,7 @@ void ts_document_set_language(TSDocument *self, const TSLanguage *language) { ts_document_invalidate(self); parser_set_language(&self->parser, language); if (self->tree) { - ts_tree_release(&self->parser.tree_pool, self->tree); + ts_subtree_release(&self->parser.tree_pool, self->tree); self->tree = NULL; } } @@ -91,16 +91,16 @@ void ts_document_edit(TSDocument *self, TSInputEdit edit) { if (!self->tree) return; - uint32_t max_bytes = ts_tree_total_bytes(self->tree); + uint32_t max_bytes = ts_subtree_total_bytes(self->tree); if (edit.start_byte > max_bytes) return; if (edit.bytes_removed > max_bytes - edit.start_byte) edit.bytes_removed = max_bytes - edit.start_byte; - self->tree = ts_tree_edit(self->tree, &edit, &self->parser.tree_pool); + self->tree = ts_subtree_edit(self->tree, &edit, &self->parser.tree_pool); if (self->parser.print_debugging_graphs) { - ts_tree_print_dot_graph(self->tree, self->parser.language, stderr); + ts_subtree_print_dot_graph(self->tree, self->parser.language, stderr); } } @@ -130,18 +130,18 @@ void ts_document_parse_with_options(TSDocument *self, TSParseOptions options) { if (!self->input.read || !self->parser.language) return; - Tree *reusable_tree = self->valid ? self->tree : NULL; + Subtree *reusable_tree = self->valid ? self->tree : NULL; if (reusable_tree && !reusable_tree->has_changes) return; - Tree *tree = parser_parse(&self->parser, self->input, reusable_tree, options.halt_on_error); + Subtree *tree = parser_parse(&self->parser, self->input, reusable_tree, options.halt_on_error); if (self->tree) { - Tree *old_tree = self->tree; + Subtree *old_tree = self->tree; self->tree = tree; if (options.changed_ranges && options.changed_range_count) { - *options.changed_range_count = ts_tree_get_changed_ranges( + *options.changed_range_count = ts_subtree_get_changed_ranges( old_tree, tree, &self->cursor1, &self->cursor2, self->parser.language, options.changed_ranges ); @@ -158,7 +158,7 @@ void ts_document_parse_with_options(TSDocument *self, TSParseOptions options) { } } - ts_tree_release(&self->parser.tree_pool, old_tree); + ts_subtree_release(&self->parser.tree_pool, old_tree); } self->tree = tree; diff --git a/src/runtime/document.h b/src/runtime/document.h index 986fcec9..9e53963b 100644 --- a/src/runtime/document.h +++ b/src/runtime/document.h @@ -2,14 +2,14 @@ #define RUNTIME_DOCUMENT_H_ #include "runtime/parser.h" -#include "runtime/tree.h" +#include "runtime/subtree.h" #include "runtime/tree_cursor.h" #include struct TSDocument { Parser parser; TSInput input; - Tree *tree; + Subtree *tree; TSTreeCursor cursor1; TSTreeCursor cursor2; size_t parse_count; diff --git a/src/runtime/get_changed_ranges.c b/src/runtime/get_changed_ranges.c index 7632af24..76e01f29 100644 --- a/src/runtime/get_changed_ranges.c +++ b/src/runtime/get_changed_ranges.c @@ -1,5 +1,5 @@ #include "runtime/get_changed_ranges.h" -#include "runtime/tree.h" +#include "runtime/subtree.h" #include "runtime/language.h" #include "runtime/error_costs.h" #include "runtime/tree_cursor.h" @@ -31,7 +31,7 @@ typedef struct { bool in_padding; } Iterator; -static Iterator iterator_new(TSTreeCursor *cursor, Tree *tree, const TSLanguage *language) { +static Iterator iterator_new(TSTreeCursor *cursor, Subtree *tree, const TSLanguage *language) { array_clear(&cursor->stack); array_push(&cursor->stack, ((TreeCursorEntry){ .tree = tree, @@ -74,14 +74,14 @@ static bool iterator_tree_is_visible(const Iterator *self) { TreeCursorEntry entry = *array_back(&self->cursor.stack); if (entry.tree->visible) return true; if (self->cursor.stack.size > 1) { - Tree *parent = self->cursor.stack.contents[self->cursor.stack.size - 2].tree; + Subtree *parent = self->cursor.stack.contents[self->cursor.stack.size - 2].tree; const TSSymbol *alias_sequence = ts_language_alias_sequence(self->language, parent->alias_sequence_id); return alias_sequence && alias_sequence[entry.structural_child_index] != 0; } return false; } -static void iterator_get_visible_state(const Iterator *self, Tree **tree, +static void iterator_get_visible_state(const Iterator *self, Subtree **tree, TSSymbol *alias_symbol, uint32_t *start_byte) { uint32_t i = self->cursor.stack.size - 1; @@ -94,7 +94,7 @@ static void iterator_get_visible_state(const Iterator *self, Tree **tree, TreeCursorEntry entry = self->cursor.stack.contents[i]; if (i > 0) { - Tree *parent = self->cursor.stack.contents[i - 1].tree; + Subtree *parent = self->cursor.stack.contents[i - 1].tree; const TSSymbol *alias_sequence = ts_language_alias_sequence( self->language, parent->alias_sequence_id @@ -129,7 +129,7 @@ static bool iterator_descend(Iterator *self, uint32_t goal_position) { Length position = entry.position; uint32_t structural_child_index = 0; for (uint32_t i = 0; i < entry.tree->children.size; i++) { - Tree *child = entry.tree->children.contents[i]; + Subtree *child = entry.tree->children.contents[i]; Length child_left = length_add(position, child->padding); Length child_right = length_add(child_left, child->size); @@ -178,13 +178,13 @@ static void iterator_advance(Iterator *self) { TreeCursorEntry entry = array_pop(&self->cursor.stack); if (iterator_done(self)) return; - Tree *parent = array_back(&self->cursor.stack)->tree; + Subtree *parent = array_back(&self->cursor.stack)->tree; uint32_t child_index = entry.child_index + 1; if (parent->children.size > child_index) { - Length position = length_add(entry.position, ts_tree_total_size(entry.tree)); + Length position = length_add(entry.position, ts_subtree_total_size(entry.tree)); uint32_t structural_child_index = entry.structural_child_index; if (!entry.tree->extra) structural_child_index++; - Tree *next_child = parent->children.contents[child_index]; + Subtree *next_child = parent->children.contents[child_index]; array_push(&self->cursor.stack, ((TreeCursorEntry){ .tree = next_child, @@ -214,7 +214,7 @@ typedef enum { } IteratorComparison; IteratorComparison iterator_compare(const Iterator *old_iter, const Iterator *new_iter) { - Tree *old_tree = NULL, *new_tree = NULL; + Subtree *old_tree = NULL, *new_tree = NULL; uint32_t old_start = 0, new_start = 0; TSSymbol old_alias_symbol = 0, new_alias_symbol = 0; iterator_get_visible_state(old_iter, &old_tree, &old_alias_symbol, &old_start); @@ -261,7 +261,7 @@ static inline void iterator_print_state(Iterator *self) { } #endif -unsigned ts_tree_get_changed_ranges(Tree *old_tree, Tree *new_tree, +unsigned ts_subtree_get_changed_ranges(Subtree *old_tree, Subtree *new_tree, TSTreeCursor *cursor1, TSTreeCursor *cursor2, const TSLanguage *language, TSRange **ranges) { RangeArray results = array_new(); diff --git a/src/runtime/get_changed_ranges.h b/src/runtime/get_changed_ranges.h index fa408aaa..bf138a21 100644 --- a/src/runtime/get_changed_ranges.h +++ b/src/runtime/get_changed_ranges.h @@ -1,10 +1,10 @@ #ifndef RUNTIME_GET_CHANGED_RANGES_H_ #define RUNTIME_GET_CHANGED_RANGES_H_ -#include "runtime/tree.h" +#include "runtime/subtree.h" -unsigned ts_tree_get_changed_ranges( - Tree *old_tree, Tree *new_tree, TSTreeCursor *cursor1, TSTreeCursor *cursor2, +unsigned ts_subtree_get_changed_ranges( + Subtree *old_tree, Subtree *new_tree, TSTreeCursor *cursor1, TSTreeCursor *cursor2, const TSLanguage *language, TSRange **ranges ); diff --git a/src/runtime/language.c b/src/runtime/language.c index 9bf1fc63..75e7da8b 100644 --- a/src/runtime/language.c +++ b/src/runtime/language.c @@ -1,5 +1,5 @@ #include "runtime/language.h" -#include "runtime/tree.h" +#include "runtime/subtree.h" #include "runtime/error_costs.h" void ts_language_table_entry(const TSLanguage *self, TSStateId state, diff --git a/src/runtime/language.h b/src/runtime/language.h index 64733242..879a5b5b 100644 --- a/src/runtime/language.h +++ b/src/runtime/language.h @@ -6,7 +6,7 @@ extern "C" { #endif #include "tree_sitter/parser.h" -#include "runtime/tree.h" +#include "runtime/subtree.h" #define ts_builtin_sym_error_repeat (ts_builtin_sym_error - 1) diff --git a/src/runtime/lexer.c b/src/runtime/lexer.c index 157276a1..e6e5aa58 100644 --- a/src/runtime/lexer.c +++ b/src/runtime/lexer.c @@ -1,6 +1,6 @@ #include #include "runtime/lexer.h" -#include "runtime/tree.h" +#include "runtime/subtree.h" #include "runtime/length.h" #include "runtime/utf16.h" #include "utf8proc.h" diff --git a/src/runtime/lexer.h b/src/runtime/lexer.h index 08a71a11..90be55a7 100644 --- a/src/runtime/lexer.h +++ b/src/runtime/lexer.h @@ -8,7 +8,7 @@ extern "C" { #include "tree_sitter/parser.h" #include "tree_sitter/runtime.h" #include "runtime/length.h" -#include "runtime/tree.h" +#include "runtime/subtree.h" typedef struct { TSLexer data; diff --git a/src/runtime/node.c b/src/runtime/node.c index 77d913f3..4df3400a 100644 --- a/src/runtime/node.c +++ b/src/runtime/node.c @@ -1,12 +1,12 @@ #include -#include "runtime/tree.h" +#include "runtime/subtree.h" #include "runtime/document.h" #include "runtime/language.h" // NodeChildIterator typedef struct { - const Tree *parent; + const Subtree *parent; const TSDocument *document; Length position; uint32_t child_index; @@ -25,12 +25,12 @@ static inline TSNode ts_node__null() { }; } -static inline const Tree *ts_node__tree(TSNode self) { +static inline const Subtree *ts_node__tree(TSNode self) { return self.subtree; } static inline NodeChildIterator ts_node_child_iterator_begin(const TSNode *node) { - const Tree *tree = ts_node__tree(*node); + const Subtree *tree = ts_node__tree(*node); const TSSymbol *alias_sequence = ts_language_alias_sequence( node->document->parser.language, tree->alias_sequence_id @@ -47,7 +47,7 @@ static inline NodeChildIterator ts_node_child_iterator_begin(const TSNode *node) static inline bool ts_node_child_iterator_next(NodeChildIterator *self, TSNode *result) { if (self->child_index == self->parent->children.size) return false; - Tree *child = self->parent->children.contents[self->child_index]; + Subtree *child = self->parent->children.contents[self->child_index]; TSSymbol alias_symbol = 0; if (!child->extra) { if (self->alias_sequence) { @@ -62,13 +62,13 @@ static inline bool ts_node_child_iterator_next(NodeChildIterator *self, TSNode * .byte = self->position.bytes, .alias_symbol = alias_symbol, }; - self->position = length_add(self->position, ts_tree_total_size(child)); + self->position = length_add(self->position, ts_subtree_total_size(child)); self->child_index++; return true; } static inline bool ts_node__is_relevant(TSNode self, bool include_anonymous) { - const Tree *tree = ts_node__tree(self); + const Subtree *tree = ts_node__tree(self); if (include_anonymous) { return tree->visible || self.alias_symbol; } else { @@ -86,7 +86,7 @@ static inline bool ts_node__is_relevant(TSNode self, bool include_anonymous) { } static inline uint32_t ts_node__relevant_child_count(TSNode self, bool include_anonymous) { - const Tree *tree = ts_node__tree(self); + const Subtree *tree = ts_node__tree(self); if (tree->children.size > 0) { if (include_anonymous) { return tree->visible_child_count; @@ -300,7 +300,7 @@ static inline TSNode ts_node__descendant_for_point_range(TSNode self, TSPoint mi TSNode child; NodeChildIterator iterator = ts_node_child_iterator_begin(&node); while (ts_node_child_iterator_next(&iterator, &child)) { - const Tree *child_tree = ts_node__tree(child); + const Subtree *child_tree = ts_node__tree(child); if (iterator.child_index != 1) { start_position = point_add(start_position, child_tree->padding.extent); } @@ -338,7 +338,7 @@ TSPoint ts_node_end_point(TSNode self) { } TSSymbol ts_node_symbol(TSNode self) { - const Tree *tree = ts_node__tree(self); + const Subtree *tree = ts_node__tree(self); return self.alias_symbol ? self.alias_symbol : tree->symbol; } @@ -347,25 +347,25 @@ const char *ts_node_type(TSNode self) { } char *ts_node_string(TSNode self) { - return ts_tree_string(ts_node__tree(self), self.document->parser.language, false); + return ts_subtree_string(ts_node__tree(self), self.document->parser.language, false); } bool ts_node_eq(TSNode self, TSNode other) { return ( - ts_tree_eq(ts_node__tree(self), ts_node__tree(other)) && + ts_subtree_eq(ts_node__tree(self), ts_node__tree(other)) && self.byte == other.byte ); } bool ts_node_is_named(TSNode self) { - const Tree *tree = ts_node__tree(self); + const Subtree *tree = ts_node__tree(self); return self.alias_symbol ? ts_language_symbol_metadata(self.document->parser.language, self.alias_symbol).named : tree->named; } bool ts_node_is_missing(TSNode self) { - const Tree *tree = ts_node__tree(self); + const Subtree *tree = ts_node__tree(self); return tree->is_missing; } @@ -414,7 +414,7 @@ TSNode ts_node_named_child(TSNode self, uint32_t child_index) { } uint32_t ts_node_child_count(TSNode self) { - const Tree *tree = ts_node__tree(self); + const Subtree *tree = ts_node__tree(self); if (tree->children.size > 0) { return tree->visible_child_count; } else { @@ -423,7 +423,7 @@ uint32_t ts_node_child_count(TSNode self) { } uint32_t ts_node_named_child_count(TSNode self) { - const Tree *tree = ts_node__tree(self); + const Subtree *tree = ts_node__tree(self); if (tree->children.size > 0) { return tree->named_child_count; } else { diff --git a/src/runtime/parser.c b/src/runtime/parser.c index f1ba0d16..8bb7ae31 100644 --- a/src/runtime/parser.c +++ b/src/runtime/parser.c @@ -4,7 +4,7 @@ #include #include #include "tree_sitter/runtime.h" -#include "runtime/tree.h" +#include "runtime/subtree.h" #include "runtime/lexer.h" #include "runtime/length.h" #include "runtime/array.h" @@ -27,7 +27,7 @@ #define LOG_TREE() \ if (self->print_debugging_graphs) { \ - ts_tree_print_dot_graph(self->finished_tree, self->language, stderr); \ + ts_subtree_print_dot_graph(self->finished_tree, self->language, stderr); \ fputs("\n", stderr); \ } @@ -84,10 +84,10 @@ static bool parser__breakdown_top_of_stack(Parser *self, StackVersion version) { for (uint32_t i = 0; i < pop.size; i++) { StackSlice slice = pop.contents[i]; TSStateId state = ts_stack_state(self->stack, slice.version); - Tree *parent = *array_front(&slice.trees); + Subtree *parent = *array_front(&slice.subtrees); for (uint32_t j = 0; j < parent->children.size; j++) { - Tree *child = parent->children.contents[j]; + Subtree *child = parent->children.contents[j]; pending = child->children.size > 0; if (child->symbol == ts_builtin_sym_error) { @@ -96,17 +96,17 @@ static bool parser__breakdown_top_of_stack(Parser *self, StackVersion version) { state = ts_language_next_state(self->language, state, child->symbol); } - ts_tree_retain(child); + ts_subtree_retain(child); ts_stack_push(self->stack, slice.version, child, pending, state); } - for (uint32_t j = 1; j < slice.trees.size; j++) { - Tree *tree = slice.trees.contents[j]; + for (uint32_t j = 1; j < slice.subtrees.size; j++) { + Subtree *tree = slice.subtrees.contents[j]; ts_stack_push(self->stack, slice.version, tree, false, state); } - ts_tree_release(&self->tree_pool, parent); - array_delete(&slice.trees); + ts_subtree_release(&self->tree_pool, parent); + array_delete(&slice.subtrees); LOG("breakdown_top_of_stack tree:%s", SYM_NAME(parent->symbol)); LOG_STACK(); @@ -116,11 +116,11 @@ static bool parser__breakdown_top_of_stack(Parser *self, StackVersion version) { return did_break_down; } -static void parser__breakdown_lookahead(Parser *self, Tree **lookahead, +static void parser__breakdown_lookahead(Parser *self, Subtree **lookahead, TSStateId state, ReusableNode *reusable_node) { bool did_descend = false; - Tree *tree = reusable_node_tree(reusable_node); + Subtree *tree = reusable_node_tree(reusable_node); while (tree->children.size > 0 && tree->parse_state != state) { LOG("state_mismatch sym:%s", SYM_NAME(tree->symbol)); reusable_node_descend(reusable_node); @@ -129,9 +129,9 @@ static void parser__breakdown_lookahead(Parser *self, Tree **lookahead, } if (did_descend) { - ts_tree_release(&self->tree_pool, *lookahead); + ts_subtree_release(&self->tree_pool, *lookahead); *lookahead = tree; - ts_tree_retain(*lookahead); + ts_subtree_retain(*lookahead); } } @@ -215,7 +215,7 @@ static bool parser__better_version_exists(Parser *self, StackVersion version, return false; } -static void parser__restore_external_scanner(Parser *self, Tree *external_token) { +static void parser__restore_external_scanner(Parser *self, Subtree *external_token) { if (external_token) { self->language->external_scanner.deserialize( self->external_scanner_payload, @@ -227,9 +227,9 @@ static void parser__restore_external_scanner(Parser *self, Tree *external_token) } } -static Tree *parser__lex(Parser *self, StackVersion version, TSStateId parse_state) { +static Subtree *parser__lex(Parser *self, StackVersion version, TSStateId parse_state) { Length start_position = ts_stack_position(self->stack, version); - Tree *external_token = ts_stack_last_external_token(self->stack, version); + Subtree *external_token = ts_stack_last_external_token(self->stack, version); TSLexMode lex_mode = self->language->lex_modes[parse_state]; const bool *valid_external_tokens = ts_language_enabled_external_tokens( self->language, @@ -326,11 +326,11 @@ static Tree *parser__lex(Parser *self, StackVersion version, TSStateId parse_sta last_byte_scanned = self->lexer.current_position.bytes; } - Tree *result; + Subtree *result; if (skipped_error) { Length padding = length_sub(error_start_position, start_position); Length size = length_sub(error_end_position, error_start_position); - result = ts_tree_make_error(&self->tree_pool, size, padding, first_error_character, self->language); + result = ts_subtree_make_error(&self->tree_pool, size, padding, first_error_character, self->language); } else { if (self->lexer.token_end_position.bytes < self->lexer.token_start_position.bytes) { self->lexer.token_start_position = self->lexer.token_end_position; @@ -355,7 +355,7 @@ static Tree *parser__lex(Parser *self, StackVersion version, TSStateId parse_sta } } - result = ts_tree_make_leaf(&self->tree_pool, symbol, padding, size, self->language); + result = ts_subtree_make_leaf(&self->tree_pool, symbol, padding, size, self->language); if (found_external_token) { result->has_external_tokens = true; @@ -375,30 +375,30 @@ static Tree *parser__lex(Parser *self, StackVersion version, TSStateId parse_sta return result; } -static Tree *parser__get_cached_token(Parser *self, size_t byte_index, Tree *last_external_token) { +static Subtree *parser__get_cached_token(Parser *self, size_t byte_index, Subtree *last_external_token) { TokenCache *cache = &self->token_cache; if (cache->token && cache->byte_index == byte_index && - ts_tree_external_token_state_eq(cache->last_external_token, last_external_token)) { + ts_subtree_external_token_state_eq(cache->last_external_token, last_external_token)) { return cache->token; } else { return NULL; } } -static void parser__set_cached_token(Parser *self, size_t byte_index, Tree *last_external_token, - Tree *token) { +static void parser__set_cached_token(Parser *self, size_t byte_index, Subtree *last_external_token, + Subtree *token) { TokenCache *cache = &self->token_cache; - if (token) ts_tree_retain(token); - if (last_external_token) ts_tree_retain(last_external_token); - if (cache->token) ts_tree_release(&self->tree_pool, cache->token); - if (cache->last_external_token) ts_tree_release(&self->tree_pool, cache->last_external_token); + if (token) ts_subtree_retain(token); + if (last_external_token) ts_subtree_retain(last_external_token); + if (cache->token) ts_subtree_release(&self->tree_pool, cache->token); + if (cache->last_external_token) ts_subtree_release(&self->tree_pool, cache->last_external_token); cache->token = token; cache->byte_index = byte_index; cache->last_external_token = last_external_token; } -static bool parser__can_reuse_first_leaf(Parser *self, TSStateId state, Tree *tree, +static bool parser__can_reuse_first_leaf(Parser *self, TSStateId state, Subtree *tree, TableEntry *table_entry) { TSLexMode current_lex_mode = self->language->lex_modes[state]; @@ -416,12 +416,12 @@ static bool parser__can_reuse_first_leaf(Parser *self, TSStateId state, Tree *tr return current_lex_mode.external_lex_state == 0 && table_entry->is_reusable; } -static Tree *parser__get_lookahead(Parser *self, StackVersion version, TSStateId *state, +static Subtree *parser__get_lookahead(Parser *self, StackVersion version, TSStateId *state, ReusableNode *reusable_node, TableEntry *table_entry) { Length position = ts_stack_position(self->stack, version); - Tree *last_external_token = ts_stack_last_external_token(self->stack, version); + Subtree *last_external_token = ts_stack_last_external_token(self->stack, version); - Tree *result; + Subtree *result; while ((result = reusable_node_tree(reusable_node))) { uint32_t byte_offset = reusable_node_byte_offset(reusable_node); if (byte_offset > position.bytes) { @@ -435,7 +435,7 @@ static Tree *parser__get_lookahead(Parser *self, StackVersion version, TSStateId continue; } - if (!ts_tree_external_token_state_eq(reusable_node->last_external_token, last_external_token)) { + if (!ts_subtree_external_token_state_eq(reusable_node->last_external_token, last_external_token)) { LOG("reusable_node_has_different_external_scanner_state symbol:%s", SYM_NAME(result->symbol)); reusable_node_advance(reusable_node); continue; @@ -476,14 +476,14 @@ static Tree *parser__get_lookahead(Parser *self, StackVersion version, TSStateId } LOG("reuse_node symbol:%s", SYM_NAME(result->symbol)); - ts_tree_retain(result); + ts_subtree_retain(result); return result; } if ((result = parser__get_cached_token(self, position.bytes, last_external_token))) { ts_language_table_entry(self->language, *state, result->first_leaf.symbol, table_entry); if (parser__can_reuse_first_leaf(self, *state, result, table_entry)) { - ts_tree_retain(result); + ts_subtree_retain(result); return result; } } @@ -494,7 +494,7 @@ static Tree *parser__get_lookahead(Parser *self, StackVersion version, TSStateId return result; } -static bool parser__select_tree(Parser *self, Tree *left, Tree *right) { +static bool parser__select_tree(Parser *self, Subtree *left, Subtree *right) { if (!left) return true; if (!right) return false; @@ -526,7 +526,7 @@ static bool parser__select_tree(Parser *self, Tree *left, Tree *right) { if (left->error_cost > 0) return true; - int comparison = ts_tree_compare(left, right); + int comparison = ts_subtree_compare(left, right); switch (comparison) { case -1: LOG("select_earlier symbol:%s, over_symbol:%s", SYM_NAME(left->symbol), @@ -545,31 +545,31 @@ static bool parser__select_tree(Parser *self, Tree *left, Tree *right) { } static void parser__shift(Parser *self, StackVersion version, TSStateId state, - Tree *lookahead, bool extra) { + Subtree *lookahead, bool extra) { if (extra != lookahead->extra) { if (ts_stack_version_count(self->stack) > 1) { - lookahead = ts_tree_make_copy(&self->tree_pool, lookahead); + lookahead = ts_subtree_make_copy(&self->tree_pool, lookahead); } else { - ts_tree_retain(lookahead); + ts_subtree_retain(lookahead); } lookahead->extra = extra; } else { - ts_tree_retain(lookahead); + ts_subtree_retain(lookahead); } bool is_pending = lookahead->children.size > 0; ts_stack_push(self->stack, version, lookahead, is_pending, state); if (lookahead->has_external_tokens) { ts_stack_set_last_external_token( - self->stack, version, ts_tree_last_external_token(lookahead) + self->stack, version, ts_subtree_last_external_token(lookahead) ); } } -static bool parser__replace_children(Parser *self, Tree *tree, TreeArray *children) { +static bool parser__replace_children(Parser *self, Subtree *tree, SubtreeArray *children) { self->scratch_tree = *tree; self->scratch_tree.children.size = 0; - ts_tree_set_children(&self->scratch_tree, children, self->language); + ts_subtree_set_children(&self->scratch_tree, children, self->language); if (parser__select_tree(self, tree, &self->scratch_tree)) { *tree = self->scratch_tree; return true; @@ -591,12 +591,12 @@ static StackSliceArray parser__reduce(Parser *self, StackVersion version, TSSymb // Extra tokens on top of the stack should not be included in this new parent // node. They will be re-pushed onto the stack after the parent node is // created and pushed. - TreeArray children = slice.trees; + SubtreeArray children = slice.subtrees; while (children.size > 0 && children.contents[children.size - 1]->extra) { children.size--; } - Tree *parent = ts_tree_make_node(&self->tree_pool, + Subtree *parent = ts_subtree_make_node(&self->tree_pool, symbol, &children, alias_sequence_id, self->language ); @@ -609,16 +609,16 @@ static StackSliceArray parser__reduce(Parser *self, StackVersion version, TSSymb if (next_slice.version != slice.version) break; i++; - TreeArray children = next_slice.trees; + SubtreeArray children = next_slice.subtrees; while (children.size > 0 && children.contents[children.size - 1]->extra) { children.size--; } if (parser__replace_children(self, parent, &children)) { - ts_tree_array_delete(&self->tree_pool, &slice.trees); + ts_subtree_array_delete(&self->tree_pool, &slice.subtrees); slice = next_slice; } else { - ts_tree_array_delete(&self->tree_pool, &next_slice.trees); + ts_subtree_array_delete(&self->tree_pool, &next_slice.subtrees); } } @@ -638,15 +638,15 @@ static StackSliceArray parser__reduce(Parser *self, StackVersion version, TSSymb // Push the parent node onto the stack, along with any extra tokens that // were previously on top of the stack. ts_stack_push(self->stack, slice.version, parent, false, next_state); - for (uint32_t j = parent->children.size; j < slice.trees.size; j++) { - ts_stack_push(self->stack, slice.version, slice.trees.contents[j], false, next_state); + for (uint32_t j = parent->children.size; j < slice.subtrees.size; j++) { + ts_stack_push(self->stack, slice.version, slice.subtrees.contents[j], false, next_state); } if (ts_stack_version_count(self->stack) > MAX_VERSION_COUNT) { i++; while (i < pop.size) { StackSlice slice = pop.contents[i]; - ts_tree_array_delete(&self->tree_pool, &slice.trees); + ts_subtree_array_delete(&self->tree_pool, &slice.subtrees); ts_stack_halt(self->stack, slice.version); i++; } @@ -669,7 +669,7 @@ static StackSliceArray parser__reduce(Parser *self, StackVersion version, TSSymb return pop; } -static void parser__start(Parser *self, TSInput input, Tree *previous_tree) { +static void parser__start(Parser *self, TSInput input, Subtree *previous_tree) { if (previous_tree) { LOG("parse_after_edit"); } else { @@ -688,29 +688,29 @@ static void parser__start(Parser *self, TSInput input, Tree *previous_tree) { self->in_ambiguity = false; } -static void parser__accept(Parser *self, StackVersion version, Tree *lookahead) { +static void parser__accept(Parser *self, StackVersion version, Subtree *lookahead) { lookahead->extra = true; assert(lookahead->symbol == ts_builtin_sym_end); - ts_tree_retain(lookahead); + ts_subtree_retain(lookahead); ts_stack_push(self->stack, version, lookahead, false, 1); StackSliceArray pop = ts_stack_pop_all(self->stack, version); for (uint32_t i = 0; i < pop.size; i++) { - TreeArray trees = pop.contents[i].trees; + SubtreeArray trees = pop.contents[i].subtrees; - Tree *root = NULL; + Subtree *root = NULL; for (uint32_t j = trees.size - 1; j + 1 > 0; j--) { - Tree *child = trees.contents[j]; + Subtree *child = trees.contents[j]; if (!child->extra) { for (uint32_t k = 0; k < child->children.size; k++) { - ts_tree_retain(child->children.contents[k]); + ts_subtree_retain(child->children.contents[k]); } array_splice(&trees, j, 1, &child->children); - root = ts_tree_make_node( + root = ts_subtree_make_node( &self->tree_pool, child->symbol, &trees, child->alias_sequence_id, self->language ); - ts_tree_release(&self->tree_pool, child); + ts_subtree_release(&self->tree_pool, child); break; } } @@ -720,10 +720,10 @@ static void parser__accept(Parser *self, StackVersion version, Tree *lookahead) if (self->finished_tree) { if (parser__select_tree(self, self->finished_tree, root)) { - ts_tree_release(&self->tree_pool, self->finished_tree); + ts_subtree_release(&self->tree_pool, self->finished_tree); self->finished_tree = root; } else { - ts_tree_release(&self->tree_pool, root); + ts_subtree_release(&self->tree_pool, root); } } else { self->finished_tree = root; @@ -845,7 +845,7 @@ static void parser__handle_error(Parser *self, StackVersion version, TSSymbol lo lookahead_symbol )) { StackVersion version_with_missing_tree = ts_stack_copy_version(self->stack, v); - Tree *missing_tree = ts_tree_make_missing_leaf(&self->tree_pool, missing_symbol, self->language); + Subtree *missing_tree = ts_subtree_make_missing_leaf(&self->tree_pool, missing_symbol, self->language); ts_stack_push( self->stack, version_with_missing_tree, missing_tree, false, @@ -890,17 +890,17 @@ static void parser__halt_parse(Parser *self) { ts_stack_position(self->stack, 0) ); - Tree *filler_node = ts_tree_make_error(&self->tree_pool, remaining_length, length_zero(), 0, self->language); + Subtree *filler_node = ts_subtree_make_error(&self->tree_pool, remaining_length, length_zero(), 0, self->language); filler_node->visible = false; ts_stack_push(self->stack, 0, filler_node, false, 0); - TreeArray children = array_new(); - Tree *root_error = ts_tree_make_error_node(&self->tree_pool, &children, self->language); + SubtreeArray children = array_new(); + Subtree *root_error = ts_subtree_make_error_node(&self->tree_pool, &children, self->language); ts_stack_push(self->stack, 0, root_error, false, 0); - Tree *eof = ts_tree_make_leaf(&self->tree_pool, ts_builtin_sym_end, length_zero(), length_zero(), self->language); + Subtree *eof = ts_subtree_make_leaf(&self->tree_pool, ts_builtin_sym_end, length_zero(), length_zero(), self->language); parser__accept(self, 0, eof); - ts_tree_release(&self->tree_pool, eof); + ts_subtree_release(&self->tree_pool, eof); } static bool parser__recover_to_state(Parser *self, StackVersion version, unsigned depth, @@ -912,40 +912,40 @@ static bool parser__recover_to_state(Parser *self, StackVersion version, unsigne StackSlice slice = pop.contents[i]; if (slice.version == previous_version) { - ts_tree_array_delete(&self->tree_pool, &slice.trees); + ts_subtree_array_delete(&self->tree_pool, &slice.subtrees); array_erase(&pop, i--); continue; } if (ts_stack_state(self->stack, slice.version) != goal_state) { ts_stack_halt(self->stack, slice.version); - ts_tree_array_delete(&self->tree_pool, &slice.trees); + ts_subtree_array_delete(&self->tree_pool, &slice.subtrees); array_erase(&pop, i--); continue; } - TreeArray error_trees = ts_stack_pop_error(self->stack, slice.version); + SubtreeArray error_trees = ts_stack_pop_error(self->stack, slice.version); if (error_trees.size > 0) { assert(error_trees.size == 1); - array_splice(&slice.trees, 0, 0, &error_trees.contents[0]->children); + array_splice(&slice.subtrees, 0, 0, &error_trees.contents[0]->children); for (unsigned j = 0; j < error_trees.contents[0]->children.size; j++) { - ts_tree_retain(slice.trees.contents[j]); + ts_subtree_retain(slice.subtrees.contents[j]); } - ts_tree_array_delete(&self->tree_pool, &error_trees); + ts_subtree_array_delete(&self->tree_pool, &error_trees); } - TreeArray trailing_extras = ts_tree_array_remove_trailing_extras(&slice.trees); + SubtreeArray trailing_extras = ts_subtree_array_remove_trailing_extras(&slice.subtrees); - if (slice.trees.size > 0) { - Tree *error = ts_tree_make_error_node(&self->tree_pool, &slice.trees, self->language); + if (slice.subtrees.size > 0) { + Subtree *error = ts_subtree_make_error_node(&self->tree_pool, &slice.subtrees, self->language); error->extra = true; ts_stack_push(self->stack, slice.version, error, false, goal_state); } else { - array_delete(&slice.trees); + array_delete(&slice.subtrees); } for (unsigned j = 0; j < trailing_extras.size; j++) { - Tree *tree = trailing_extras.contents[j]; + Subtree *tree = trailing_extras.contents[j]; ts_stack_push(self->stack, slice.version, tree, false, goal_state); } @@ -956,7 +956,7 @@ static bool parser__recover_to_state(Parser *self, StackVersion version, unsigne return previous_version != STACK_VERSION_NONE; } -static void parser__recover(Parser *self, StackVersion version, Tree *lookahead) { +static void parser__recover(Parser *self, StackVersion version, Subtree *lookahead) { bool did_recover = false; unsigned previous_version_count = ts_stack_version_count(self->stack); Length position = ts_stack_position(self->stack, version); @@ -1017,8 +1017,8 @@ static void parser__recover(Parser *self, StackVersion version, Tree *lookahead) if (lookahead->symbol == ts_builtin_sym_end) { LOG("recover_eof"); - TreeArray children = array_new(); - Tree *parent = ts_tree_make_error_node(&self->tree_pool, &children, self->language); + SubtreeArray children = array_new(); + Subtree *parent = ts_subtree_make_error_node(&self->tree_pool, &children, self->language); ts_stack_push(self->stack, version, parent, false, 1); parser__accept(self, version, lookahead); return; @@ -1026,8 +1026,8 @@ static void parser__recover(Parser *self, StackVersion version, Tree *lookahead) unsigned new_cost = current_error_cost + ERROR_COST_PER_SKIPPED_TREE + - ts_tree_total_bytes(lookahead) * ERROR_COST_PER_SKIPPED_CHAR + - ts_tree_total_size(lookahead).extent.row * ERROR_COST_PER_SKIPPED_LINE; + ts_subtree_total_bytes(lookahead) * ERROR_COST_PER_SKIPPED_CHAR + + ts_subtree_total_size(lookahead).extent.row * ERROR_COST_PER_SKIPPED_LINE; if (parser__better_version_exists(self, version, false, new_cost)) { ts_stack_halt(self->stack, version); @@ -1041,11 +1041,11 @@ static void parser__recover(Parser *self, StackVersion version, Tree *lookahead) } LOG("skip_token symbol:%s", SYM_NAME(lookahead->symbol)); - ts_tree_retain(lookahead); - TreeArray children = array_new(); + ts_subtree_retain(lookahead); + SubtreeArray children = array_new(); array_reserve(&children, 1); array_push(&children, lookahead); - Tree *error_repeat = ts_tree_make_node( + Subtree *error_repeat = ts_subtree_make_node( &self->tree_pool, ts_builtin_sym_error_repeat, &children, @@ -1056,13 +1056,13 @@ static void parser__recover(Parser *self, StackVersion version, Tree *lookahead) if (node_count_since_error > 0) { StackSliceArray pop = ts_stack_pop_count(self->stack, version, 1); assert(pop.size == 1); - assert(pop.contents[0].trees.size == 1); + assert(pop.contents[0].subtrees.size == 1); ts_stack_renumber_version(self->stack, pop.contents[0].version, version); - array_push(&pop.contents[0].trees, error_repeat); - error_repeat = ts_tree_make_node( + array_push(&pop.contents[0].subtrees, error_repeat); + error_repeat = ts_subtree_make_node( &self->tree_pool, ts_builtin_sym_error_repeat, - &pop.contents[0].trees, + &pop.contents[0].subtrees, 0, self->language ); @@ -1072,7 +1072,7 @@ static void parser__recover(Parser *self, StackVersion version, Tree *lookahead) if (lookahead->has_external_tokens) { ts_stack_set_last_external_token( - self->stack, version, ts_tree_last_external_token(lookahead) + self->stack, version, ts_subtree_last_external_token(lookahead) ); } } @@ -1080,7 +1080,7 @@ static void parser__recover(Parser *self, StackVersion version, Tree *lookahead) static void parser__advance(Parser *self, StackVersion version, ReusableNode *reusable_node) { TSStateId state = ts_stack_state(self->stack, version); TableEntry table_entry; - Tree *lookahead = parser__get_lookahead(self, version, &state, reusable_node, &table_entry); + Subtree *lookahead = parser__get_lookahead(self, version, &state, reusable_node, &table_entry); for (;;) { StackVersion last_reduction_version = STACK_VERSION_NONE; @@ -1113,7 +1113,7 @@ static void parser__advance(Parser *self, StackVersion version, ReusableNode *re if (lookahead == reusable_node_tree(reusable_node)) { reusable_node_advance(reusable_node); } - ts_tree_release(&self->tree_pool, lookahead); + ts_subtree_release(&self->tree_pool, lookahead); return; } @@ -1133,7 +1133,7 @@ static void parser__advance(Parser *self, StackVersion version, ReusableNode *re case TSParseActionTypeAccept: { LOG("accept"); parser__accept(self, version, lookahead); - ts_tree_release(&self->tree_pool, lookahead); + ts_subtree_release(&self->tree_pool, lookahead); return; } @@ -1145,7 +1145,7 @@ static void parser__advance(Parser *self, StackVersion version, ReusableNode *re if (lookahead == reusable_node_tree(reusable_node)) { reusable_node_advance(reusable_node); } - ts_tree_release(&self->tree_pool, lookahead); + ts_subtree_release(&self->tree_pool, lookahead); return; } } @@ -1156,12 +1156,12 @@ static void parser__advance(Parser *self, StackVersion version, ReusableNode *re LOG_STACK(); } else if (state == ERROR_STATE) { parser__recover(self, version, lookahead); - ts_tree_release(&self->tree_pool, lookahead); + ts_subtree_release(&self->tree_pool, lookahead); return; } else if (!parser__breakdown_top_of_stack(self, version)) { LOG("detect_error"); ts_stack_pause(self->stack, version, lookahead->first_leaf.symbol); - ts_tree_release(&self->tree_pool, lookahead); + ts_subtree_release(&self->tree_pool, lookahead); return; } @@ -1260,7 +1260,7 @@ bool parser_init(Parser *self) { ts_lexer_init(&self->lexer); array_init(&self->reduce_actions); array_reserve(&self->reduce_actions, 4); - ts_tree_pool_init(&self->tree_pool); + ts_subtree_pool_init(&self->tree_pool); self->stack = ts_stack_new(&self->tree_pool); self->finished_tree = NULL; self->reusable_node = reusable_node_new(); @@ -1285,12 +1285,12 @@ void parser_destroy(Parser *self) { ts_stack_delete(self->stack); if (self->reduce_actions.contents) array_delete(&self->reduce_actions); - ts_tree_pool_delete(&self->tree_pool); + ts_subtree_pool_delete(&self->tree_pool); reusable_node_delete(&self->reusable_node); parser_set_language(self, NULL); } -Tree *parser_parse(Parser *self, TSInput input, Tree *old_tree, bool halt_on_error) { +Subtree *parser_parse(Parser *self, TSInput input, Subtree *old_tree, bool halt_on_error) { parser__start(self, input, old_tree); StackVersion version = STACK_VERSION_NONE; @@ -1336,7 +1336,7 @@ Tree *parser_parse(Parser *self, TSInput input, Tree *old_tree, bool halt_on_err reusable_node_delete(&reusable_node); ts_stack_clear(self->stack); parser__set_cached_token(self, 0, NULL, NULL); - ts_tree_balance(self->finished_tree, &self->tree_pool, self->language); + ts_subtree_balance(self->finished_tree, &self->tree_pool, self->language); LOG("done"); LOG_TREE(); diff --git a/src/runtime/parser.h b/src/runtime/parser.h index dab58dc9..ff776bc2 100644 --- a/src/runtime/parser.h +++ b/src/runtime/parser.h @@ -10,22 +10,22 @@ extern "C" { #include "runtime/lexer.h" #include "runtime/reusable_node.h" #include "runtime/reduce_action.h" -#include "runtime/tree.h" +#include "runtime/subtree.h" typedef struct { - Tree *token; - Tree *last_external_token; + Subtree *token; + Subtree *last_external_token; uint32_t byte_index; } TokenCache; typedef struct { Lexer lexer; Stack *stack; - TreePool tree_pool; + SubtreePool tree_pool; const TSLanguage *language; ReduceActionSet reduce_actions; - Tree *finished_tree; - Tree scratch_tree; + Subtree *finished_tree; + Subtree scratch_tree; TokenCache token_cache; ReusableNode reusable_node; void *external_scanner_payload; @@ -36,7 +36,7 @@ typedef struct { bool parser_init(Parser *); void parser_destroy(Parser *); -Tree *parser_parse(Parser *, TSInput, Tree *, bool halt_on_error); +Subtree *parser_parse(Parser *, TSInput, Subtree *, bool halt_on_error); void parser_set_language(Parser *, const TSLanguage *); #ifdef __cplusplus diff --git a/src/runtime/reusable_node.h b/src/runtime/reusable_node.h index 71d56b68..5f1884d5 100644 --- a/src/runtime/reusable_node.h +++ b/src/runtime/reusable_node.h @@ -1,21 +1,21 @@ -#include "runtime/tree.h" +#include "runtime/subtree.h" typedef struct { - Tree *tree; + Subtree *tree; uint32_t child_index; uint32_t byte_offset; } StackEntry; typedef struct { Array(StackEntry) stack; - Tree *last_external_token; + Subtree *last_external_token; } ReusableNode; static inline ReusableNode reusable_node_new() { return (ReusableNode) {array_new(), NULL}; } -static inline void reusable_node_reset(ReusableNode *self, Tree *tree) { +static inline void reusable_node_reset(ReusableNode *self, Subtree *tree) { array_clear(&self->stack); array_push(&self->stack, ((StackEntry) { .tree = tree, @@ -24,7 +24,7 @@ static inline void reusable_node_reset(ReusableNode *self, Tree *tree) { })); } -static inline Tree *reusable_node_tree(ReusableNode *self) { +static inline Subtree *reusable_node_tree(ReusableNode *self) { return self->stack.size > 0 ? self->stack.contents[self->stack.size - 1].tree : NULL; @@ -46,12 +46,12 @@ static inline void reusable_node_assign(ReusableNode *self, const ReusableNode * static inline void reusable_node_advance(ReusableNode *self) { StackEntry last_entry = *array_back(&self->stack); - uint32_t byte_offset = last_entry.byte_offset + ts_tree_total_bytes(last_entry.tree); + uint32_t byte_offset = last_entry.byte_offset + ts_subtree_total_bytes(last_entry.tree); if (last_entry.tree->has_external_tokens) { - self->last_external_token = ts_tree_last_external_token(last_entry.tree); + self->last_external_token = ts_subtree_last_external_token(last_entry.tree); } - Tree *tree; + Subtree *tree; uint32_t next_index; do { StackEntry popped_entry = array_pop(&self->stack); diff --git a/src/runtime/stack.c b/src/runtime/stack.c index f7383846..c8af7a58 100644 --- a/src/runtime/stack.c +++ b/src/runtime/stack.c @@ -1,6 +1,6 @@ #include "runtime/alloc.h" #include "runtime/language.h" -#include "runtime/tree.h" +#include "runtime/subtree.h" #include "runtime/array.h" #include "runtime/stack.h" #include "runtime/length.h" @@ -21,7 +21,7 @@ typedef struct StackNode StackNode; typedef struct { StackNode *node; - Tree *tree; + Subtree *subtree; bool is_pending; } StackLink; @@ -38,8 +38,8 @@ struct StackNode { typedef struct { StackNode *node; - TreeArray trees; - uint32_t tree_count; + SubtreeArray subtrees; + uint32_t subtree_count; bool is_pending; } Iterator; @@ -58,7 +58,7 @@ typedef enum { typedef struct { StackNode *node; - Tree *last_external_token; + Subtree *last_external_token; StackSummary *summary; unsigned node_count_at_last_error; TSSymbol lookahead_when_paused; @@ -71,7 +71,7 @@ struct Stack { Array(Iterator) iterators; StackNodeArray node_pool; StackNode *base_node; - TreePool *tree_pool; + SubtreePool *subtree_pool; }; typedef unsigned StackAction; @@ -91,7 +91,7 @@ static void stack_node_retain(StackNode *self) { assert(self->ref_count != 0); } -static void stack_node_release(StackNode *self, StackNodeArray *pool, TreePool *tree_pool) { +static void stack_node_release(StackNode *self, StackNodeArray *pool, SubtreePool *subtree_pool) { recur: assert(self->ref_count != 0); self->ref_count--; @@ -100,10 +100,10 @@ recur: StackNode *first_predecessor = NULL; if (self->link_count > 0) { for (unsigned i = self->link_count - 1; i > 0; i--) { - if (self->links[i].tree) ts_tree_release(tree_pool, self->links[i].tree); - stack_node_release(self->links[i].node, pool, tree_pool); + if (self->links[i].subtree) ts_subtree_release(subtree_pool, self->links[i].subtree); + stack_node_release(self->links[i].node, pool, subtree_pool); } - if (self->links[0].tree) ts_tree_release(tree_pool, self->links[0].tree); + if (self->links[0].subtree) ts_subtree_release(subtree_pool, self->links[0].subtree); first_predecessor = self->links[0].node; } @@ -119,7 +119,7 @@ recur: } } -static StackNode *stack_node_new(StackNode *previous_node, Tree *tree, bool is_pending, +static StackNode *stack_node_new(StackNode *previous_node, Subtree *subtree, bool is_pending, TSStateId state, StackNodeArray *pool) { StackNode *node = pool->size > 0 ? array_pop(pool) : @@ -130,7 +130,7 @@ static StackNode *stack_node_new(StackNode *previous_node, Tree *tree, bool is_p node->link_count = 1; node->links[0] = (StackLink){ .node = previous_node, - .tree = tree, + .subtree = subtree, .is_pending = is_pending, }; @@ -139,11 +139,11 @@ static StackNode *stack_node_new(StackNode *previous_node, Tree *tree, bool is_p node->dynamic_precedence = previous_node->dynamic_precedence; node->node_count = previous_node->node_count; - if (tree) { - node->error_cost += tree->error_cost; - node->position = length_add(node->position, ts_tree_total_size(tree)); - node->dynamic_precedence += tree->dynamic_precedence; - if (!tree->extra) node->node_count += tree->node_count; + if (subtree) { + node->error_cost += subtree->error_cost; + node->position = length_add(node->position, ts_subtree_total_size(subtree)); + node->dynamic_precedence += subtree->dynamic_precedence; + if (!subtree->extra) node->node_count += subtree->node_count; } } else { node->position = length_zero(); @@ -153,7 +153,7 @@ static StackNode *stack_node_new(StackNode *previous_node, Tree *tree, bool is_p return node; } -static bool stack__tree_is_equivalent(const Tree *left, const Tree *right) { +static bool stack__subtree_is_equivalent(const Subtree *left, const Subtree *right) { return left == right || (left && @@ -164,7 +164,7 @@ static bool stack__tree_is_equivalent(const Tree *left, const Tree *right) { left->padding.bytes == right->padding.bytes && left->size.bytes == right->size.bytes && left->extra == right->extra && - ts_tree_external_token_state_eq(left, right)))); + ts_subtree_external_token_state_eq(left, right)))); } static void stack_node_add_link(StackNode *self, StackLink link) { @@ -172,7 +172,7 @@ static void stack_node_add_link(StackNode *self, StackLink link) { for (int i = 0; i < self->link_count; i++) { StackLink existing_link = self->links[i]; - if (stack__tree_is_equivalent(existing_link.tree, link.tree)) { + if (stack__subtree_is_equivalent(existing_link.subtree, link.subtree)) { if (existing_link.node == link.node) return; if (existing_link.node->state == link.node->state && existing_link.node->position.bytes == link.node->position.bytes) { @@ -187,24 +187,24 @@ static void stack_node_add_link(StackNode *self, StackLink link) { if (self->link_count == MAX_LINK_COUNT) return; stack_node_retain(link.node); - if (link.tree) ts_tree_retain(link.tree); + if (link.subtree) ts_subtree_retain(link.subtree); self->links[self->link_count++] = link; unsigned node_count = link.node->node_count; - if (link.tree) node_count += link.tree->node_count; + if (link.subtree) node_count += link.subtree->node_count; if (node_count > self->node_count) self->node_count = node_count; } -static void stack_head_delete(StackHead *self, StackNodeArray *pool, TreePool *tree_pool) { +static void stack_head_delete(StackHead *self, StackNodeArray *pool, SubtreePool *subtree_pool) { if (self->node) { if (self->last_external_token) { - ts_tree_release(tree_pool, self->last_external_token); + ts_subtree_release(subtree_pool, self->last_external_token); } if (self->summary) { array_delete(self->summary); ts_free(self->summary); } - stack_node_release(self->node, pool, tree_pool); + stack_node_release(self->node, pool, subtree_pool); } } @@ -219,44 +219,44 @@ static StackVersion ts_stack__add_version(Stack *self, StackVersion original_ver }; array_push(&self->heads, head); stack_node_retain(node); - if (head.last_external_token) ts_tree_retain(head.last_external_token); + if (head.last_external_token) ts_subtree_retain(head.last_external_token); return (StackVersion)(self->heads.size - 1); } static void ts_stack__add_slice(Stack *self, StackVersion original_version, - StackNode *node, TreeArray *trees) { + StackNode *node, SubtreeArray *subtrees) { for (uint32_t i = self->slices.size - 1; i + 1 > 0; i--) { StackVersion version = self->slices.contents[i].version; if (self->heads.contents[version].node == node) { - StackSlice slice = {*trees, version}; + StackSlice slice = {*subtrees, version}; array_insert(&self->slices, i + 1, slice); return; } } StackVersion version = ts_stack__add_version(self, original_version, node); - StackSlice slice = { *trees, version }; + StackSlice slice = { *subtrees, version }; array_push(&self->slices, slice); } inline StackSliceArray stack__iter(Stack *self, StackVersion version, StackCallback callback, void *payload, - int goal_tree_count) { + int goal_subtree_count) { array_clear(&self->slices); array_clear(&self->iterators); StackHead *head = array_get(&self->heads, version); Iterator iterator = { .node = head->node, - .trees = array_new(), - .tree_count = 0, + .subtrees = array_new(), + .subtree_count = 0, .is_pending = true, }; - bool include_trees = false; - if (goal_tree_count >= 0) { - include_trees = true; - array_reserve(&iterator.trees, goal_tree_count); + bool include_subtrees = false; + if (goal_subtree_count >= 0) { + include_subtrees = true; + array_reserve(&iterator.subtrees, goal_subtree_count); } array_push(&self->iterators, iterator); @@ -271,21 +271,21 @@ inline StackSliceArray stack__iter(Stack *self, StackVersion version, bool should_stop = action & StackActionStop || node->link_count == 0; if (should_pop) { - TreeArray trees = iterator->trees; + SubtreeArray subtrees = iterator->subtrees; if (!should_stop) - ts_tree_array_copy(trees, &trees); - ts_tree_array_reverse(&trees); + ts_subtree_array_copy(subtrees, &subtrees); + ts_subtree_array_reverse(&subtrees); ts_stack__add_slice( self, version, node, - &trees + &subtrees ); } if (should_stop) { if (!should_pop) - ts_tree_array_delete(self->tree_pool, &iterator->trees); + ts_subtree_array_delete(self->subtree_pool, &iterator->subtrees); array_erase(&self->iterators, i); i--, size--; continue; @@ -303,24 +303,24 @@ inline StackSliceArray stack__iter(Stack *self, StackVersion version, Iterator current_iterator = self->iterators.contents[i]; array_push(&self->iterators, current_iterator); next_iterator = array_back(&self->iterators); - ts_tree_array_copy(next_iterator->trees, &next_iterator->trees); + ts_subtree_array_copy(next_iterator->subtrees, &next_iterator->subtrees); } next_iterator->node = link.node; - if (link.tree) { - if (include_trees) { - array_push(&next_iterator->trees, link.tree); - ts_tree_retain(link.tree); + if (link.subtree) { + if (include_subtrees) { + array_push(&next_iterator->subtrees, link.subtree); + ts_subtree_retain(link.subtree); } - if (!link.tree->extra) { - next_iterator->tree_count++; + if (!link.subtree->extra) { + next_iterator->subtree_count++; if (!link.is_pending) { next_iterator->is_pending = false; } } } else { - next_iterator->tree_count++; + next_iterator->subtree_count++; next_iterator->is_pending = false; } } @@ -330,7 +330,7 @@ inline StackSliceArray stack__iter(Stack *self, StackVersion version, return self->slices; } -Stack *ts_stack_new(TreePool *tree_pool) { +Stack *ts_stack_new(SubtreePool *subtree_pool) { Stack *self = ts_calloc(1, sizeof(Stack)); array_init(&self->heads); @@ -342,7 +342,7 @@ Stack *ts_stack_new(TreePool *tree_pool) { array_reserve(&self->iterators, 4); array_reserve(&self->node_pool, MAX_NODE_POOL_SIZE); - self->tree_pool = tree_pool; + self->subtree_pool = subtree_pool; self->base_node = stack_node_new(NULL, NULL, false, 1, &self->node_pool); ts_stack_clear(self); @@ -354,9 +354,9 @@ void ts_stack_delete(Stack *self) { array_delete(&self->slices); if (self->iterators.contents) array_delete(&self->iterators); - stack_node_release(self->base_node, &self->node_pool, self->tree_pool); + stack_node_release(self->base_node, &self->node_pool, self->subtree_pool); for (uint32_t i = 0; i < self->heads.size; i++) { - stack_head_delete(&self->heads.contents[i], &self->node_pool, self->tree_pool); + stack_head_delete(&self->heads.contents[i], &self->node_pool, self->subtree_pool); } array_clear(&self->heads); if (self->node_pool.contents) { @@ -380,14 +380,14 @@ Length ts_stack_position(const Stack *self, StackVersion version) { return array_get(&self->heads, version)->node->position; } -Tree *ts_stack_last_external_token(const Stack *self, StackVersion version) { +Subtree *ts_stack_last_external_token(const Stack *self, StackVersion version) { return array_get(&self->heads, version)->last_external_token; } -void ts_stack_set_last_external_token(Stack *self, StackVersion version, Tree *token) { +void ts_stack_set_last_external_token(Stack *self, StackVersion version, Subtree *token) { StackHead *head = array_get(&self->heads, version); - if (token) ts_tree_retain(token); - if (head->last_external_token) ts_tree_release(self->tree_pool, head->last_external_token); + if (token) ts_subtree_retain(token); + if (head->last_external_token) ts_subtree_release(self->subtree_pool, head->last_external_token); head->last_external_token = token; } @@ -396,7 +396,7 @@ unsigned ts_stack_error_cost(const Stack *self, StackVersion version) { unsigned result = head->node->error_cost; if ( head->status == StackStatusPaused || - (head->node->state == ERROR_STATE && !head->node->links[0].tree)) { + (head->node->state == ERROR_STATE && !head->node->links[0].subtree)) { result += ERROR_COST_PER_RECOVERY; } return result; @@ -410,10 +410,10 @@ unsigned ts_stack_node_count_since_error(const Stack *self, StackVersion version return head->node->node_count - head->node_count_at_last_error; } -void ts_stack_push(Stack *self, StackVersion version, Tree *tree, bool pending, TSStateId state) { +void ts_stack_push(Stack *self, StackVersion version, Subtree *subtree, bool pending, TSStateId state) { StackHead *head = array_get(&self->heads, version); - StackNode *new_node = stack_node_new(head->node, tree, pending, state, &self->node_pool); - if (!tree) head->node_count_at_last_error = new_node->node_count; + StackNode *new_node = stack_node_new(head->node, subtree, pending, state, &self->node_pool); + if (!subtree) head->node_count_at_last_error = new_node->node_count; head->node = new_node; } @@ -422,7 +422,7 @@ inline StackAction iterate_callback(void *payload, const Iterator *iterator) { session->callback( session->payload, iterator->node->state, - iterator->tree_count + iterator->subtree_count ); return StackActionNone; } @@ -434,8 +434,8 @@ void ts_stack_iterate(Stack *self, StackVersion version, } inline StackAction pop_count_callback(void *payload, const Iterator *iterator) { - unsigned *goal_tree_count = payload; - if (iterator->tree_count == *goal_tree_count) { + unsigned *goal_subtree_count = payload; + if (iterator->subtree_count == *goal_subtree_count) { return StackActionPop | StackActionStop; } else { return StackActionNone; @@ -447,7 +447,7 @@ StackSliceArray ts_stack_pop_count(Stack *self, StackVersion version, uint32_t c } inline StackAction pop_pending_callback(void *payload, const Iterator *iterator) { - if (iterator->tree_count >= 1) { + if (iterator->subtree_count >= 1) { if (iterator->is_pending) { return StackActionPop | StackActionStop; } else { @@ -468,9 +468,9 @@ StackSliceArray ts_stack_pop_pending(Stack *self, StackVersion version) { } inline StackAction pop_error_callback(void *payload, const Iterator *iterator) { - if (iterator->trees.size > 0) { + if (iterator->subtrees.size > 0) { bool *found_error = payload; - if (!*found_error && iterator->trees.contents[0]->symbol == ts_builtin_sym_error) { + if (!*found_error && iterator->subtrees.contents[0]->symbol == ts_builtin_sym_error) { *found_error = true; return StackActionPop | StackActionStop; } else { @@ -481,21 +481,21 @@ inline StackAction pop_error_callback(void *payload, const Iterator *iterator) { } } -TreeArray ts_stack_pop_error(Stack *self, StackVersion version) { +SubtreeArray ts_stack_pop_error(Stack *self, StackVersion version) { StackNode *node = array_get(&self->heads, version)->node; for (unsigned i = 0; i < node->link_count; i++) { - if (node->links[i].tree && node->links[i].tree->symbol == ts_builtin_sym_error) { + if (node->links[i].subtree && node->links[i].subtree->symbol == ts_builtin_sym_error) { bool found_error = false; StackSliceArray pop = stack__iter(self, version, pop_error_callback, &found_error, 1); if (pop.size > 0) { assert(pop.size == 1); ts_stack_renumber_version(self, pop.contents[0].version, version); - return pop.contents[0].trees; + return pop.contents[0].subtrees; } break; } } - return (TreeArray){.size = 0}; + return (SubtreeArray){.size = 0}; } inline StackAction pop_all_callback(void *payload, const Iterator *iterator) { @@ -514,7 +514,7 @@ typedef struct { inline StackAction summarize_stack_callback(void *payload, const Iterator *iterator) { SummarizeStackSession *session = payload; TSStateId state = iterator->node->state; - unsigned depth = iterator->tree_count; + unsigned depth = iterator->subtree_count; if (depth > session->max_depth) return StackActionStop; for (unsigned i = session->summary->size - 1; i + 1 > 0; i--) { StackSummaryEntry entry = session->summary->contents[i]; @@ -548,7 +548,7 @@ int ts_stack_dynamic_precedence(Stack *self, StackVersion version) { } void ts_stack_remove_version(Stack *self, StackVersion version) { - stack_head_delete(array_get(&self->heads, version), &self->node_pool, self->tree_pool); + stack_head_delete(array_get(&self->heads, version), &self->node_pool, self->subtree_pool); array_erase(&self->heads, version); } @@ -561,7 +561,7 @@ void ts_stack_renumber_version(Stack *self, StackVersion v1, StackVersion v2) { source_head->summary = target_head->summary; target_head->summary = NULL; } - stack_head_delete(target_head, &self->node_pool, self->tree_pool); + stack_head_delete(target_head, &self->node_pool, self->subtree_pool); *target_head = *source_head; array_erase(&self->heads, v1); } @@ -577,7 +577,7 @@ StackVersion ts_stack_copy_version(Stack *self, StackVersion version) { array_push(&self->heads, self->heads.contents[version]); StackHead *head = array_back(&self->heads); stack_node_retain(head->node); - if (head->last_external_token) ts_tree_retain(head->last_external_token); + if (head->last_external_token) ts_subtree_retain(head->last_external_token); head->summary = NULL; return self->heads.size - 1; } @@ -605,7 +605,7 @@ bool ts_stack_can_merge(Stack *self, StackVersion version1, StackVersion version head1->node->state == head2->node->state && head1->node->position.bytes == head2->node->position.bytes && head1->node->error_cost == head2->node->error_cost && - ts_tree_external_token_state_eq(head1->last_external_token, head2->last_external_token); + ts_subtree_external_token_state_eq(head1->last_external_token, head2->last_external_token); } void ts_stack_halt(Stack *self, StackVersion version) { @@ -643,7 +643,7 @@ TSSymbol ts_stack_resume(Stack *self, StackVersion version) { void ts_stack_clear(Stack *self) { stack_node_retain(self->base_node); for (uint32_t i = 0; i < self->heads.size; i++) { - stack_head_delete(&self->heads.contents[i], &self->node_pool, self->tree_pool); + stack_head_delete(&self->heads.contents[i], &self->node_pool, self->subtree_pool); } array_clear(&self->heads); array_push(&self->heads, ((StackHead){ @@ -715,8 +715,8 @@ bool ts_stack_print_dot_graph(Stack *self, const TSLanguage *language, FILE *f) fprintf(f, "node_%p [", node); if (node->state == ERROR_STATE) fprintf(f, "label=\"?\""); - else if (node->link_count == 1 && node->links[0].tree && - node->links[0].tree->extra) + else if (node->link_count == 1 && node->links[0].subtree && + node->links[0].subtree->extra) fprintf(f, "shape=point margin=0 label=\"\""); else fprintf(f, "label=\"%d\"", node->state); @@ -736,24 +736,24 @@ bool ts_stack_print_dot_graph(Stack *self, const TSLanguage *language, FILE *f) fprintf(f, "node_%p -> node_%p [", node, link.node); if (link.is_pending) fprintf(f, "style=dashed "); - if (link.tree && link.tree->extra) + if (link.subtree && link.subtree->extra) fprintf(f, "fontcolor=gray "); - if (!link.tree) { + if (!link.subtree) { fprintf(f, "color=red"); } else { fprintf(f, "label=\""); - if (link.tree->visible && !link.tree->named) fprintf(f, "'"); - const char *name = ts_language_symbol_name(language, link.tree->symbol); + if (link.subtree->visible && !link.subtree->named) fprintf(f, "'"); + const char *name = ts_language_symbol_name(language, link.subtree->symbol); for (const char *c = name; *c; c++) { if (*c == '\"' || *c == '\\') fprintf(f, "\\"); fprintf(f, "%c", *c); } - if (link.tree->visible && !link.tree->named) fprintf(f, "'"); + if (link.subtree->visible && !link.subtree->named) fprintf(f, "'"); fprintf(f, "\""); fprintf(f, "labeltooltip=\"error_cost: %u\ndynamic_precedence: %u\"", - link.tree->error_cost, - link.tree->dynamic_precedence); + link.subtree->error_cost, + link.subtree->dynamic_precedence); } fprintf(f, "];\n"); diff --git a/src/runtime/stack.h b/src/runtime/stack.h index 4a552323..88c06bab 100644 --- a/src/runtime/stack.h +++ b/src/runtime/stack.h @@ -6,7 +6,7 @@ extern "C" { #endif #include "runtime/array.h" -#include "runtime/tree.h" +#include "runtime/subtree.h" #include "runtime/error_costs.h" #include @@ -16,7 +16,7 @@ typedef unsigned StackVersion; #define STACK_VERSION_NONE ((StackVersion)-1) typedef struct { - TreeArray trees; + SubtreeArray subtrees; StackVersion version; } StackSlice; typedef Array(StackSlice) StackSliceArray; @@ -29,7 +29,7 @@ typedef struct { typedef Array(StackSummaryEntry) StackSummary; // Create a stack. -Stack *ts_stack_new(TreePool *); +Stack *ts_stack_new(SubtreePool *); // Release the memory reserved for a given stack. void ts_stack_delete(Stack *); @@ -42,10 +42,10 @@ uint32_t ts_stack_version_count(const Stack *); TSStateId ts_stack_state(const Stack *, StackVersion); // Get the last external token associated with a given version of the stack. -Tree *ts_stack_last_external_token(const Stack *, StackVersion); +Subtree *ts_stack_last_external_token(const Stack *, StackVersion); // Set the last external token associated with a given version of the stack. -void ts_stack_set_last_external_token(Stack *, StackVersion, Tree *); +void ts_stack_set_last_external_token(Stack *, StackVersion, Subtree *); // Get the position of the given version of the stack within the document. Length ts_stack_position(const Stack *, StackVersion); @@ -55,7 +55,7 @@ Length ts_stack_position(const Stack *, StackVersion); // This transfers ownership of the tree to the Stack. Callers that // need to retain ownership of the tree for their own purposes should // first retain the tree. -void ts_stack_push(Stack *, StackVersion, Tree *, bool, TSStateId); +void ts_stack_push(Stack *, StackVersion, Subtree *, bool, TSStateId); // Pop the given number of entries from the given version of the stack. This // operation can increase the number of stack versions by revealing multiple @@ -65,7 +65,7 @@ void ts_stack_push(Stack *, StackVersion, Tree *, bool, TSStateId); StackSliceArray ts_stack_pop_count(Stack *, StackVersion, uint32_t count); // Remove an error at the top of the given version of the stack. -TreeArray ts_stack_pop_error(Stack *, StackVersion); +SubtreeArray ts_stack_pop_error(Stack *, StackVersion); // Remove any pending trees from the top of the given version of the stack. StackSliceArray ts_stack_pop_pending(Stack *, StackVersion); diff --git a/src/runtime/tree.c b/src/runtime/subtree.c similarity index 70% rename from src/runtime/tree.c rename to src/runtime/subtree.c index 602392f4..1808ee6d 100644 --- a/src/runtime/tree.c +++ b/src/runtime/subtree.c @@ -6,7 +6,7 @@ #include #include "runtime/alloc.h" #include "runtime/atomic.h" -#include "runtime/tree.h" +#include "runtime/subtree.h" #include "runtime/length.h" #include "runtime/language.h" #include "runtime/error_costs.h" @@ -51,15 +51,15 @@ bool ts_external_token_state_eq(const TSExternalTokenState *a, const TSExternalT memcmp(ts_external_token_state_data(a), ts_external_token_state_data(b), a->length) == 0); } -// TreeArray +// SubtreeArray -bool ts_tree_array_copy(TreeArray self, TreeArray *dest) { - Tree **contents = NULL; +bool ts_subtree_array_copy(SubtreeArray self, SubtreeArray *dest) { + Subtree **contents = NULL; if (self.capacity > 0) { - contents = ts_calloc(self.capacity, sizeof(Tree *)); - memcpy(contents, self.contents, self.size * sizeof(Tree *)); + contents = ts_calloc(self.capacity, sizeof(Subtree *)); + memcpy(contents, self.contents, self.size * sizeof(Subtree *)); for (uint32_t i = 0; i < self.size; i++) { - ts_tree_retain(contents[i]); + ts_subtree_retain(contents[i]); } } @@ -69,47 +69,47 @@ bool ts_tree_array_copy(TreeArray self, TreeArray *dest) { return true; } -void ts_tree_array_delete(TreePool *pool, TreeArray *self) { +void ts_subtree_array_delete(SubtreePool *pool, SubtreeArray *self) { for (uint32_t i = 0; i < self->size; i++) { - ts_tree_release(pool, self->contents[i]); + ts_subtree_release(pool, self->contents[i]); } array_delete(self); } -TreeArray ts_tree_array_remove_trailing_extras(TreeArray *self) { - TreeArray result = array_new(); +SubtreeArray ts_subtree_array_remove_trailing_extras(SubtreeArray *self) { + SubtreeArray result = array_new(); uint32_t i = self->size - 1; for (; i + 1 > 0; i--) { - Tree *child = self->contents[i]; + Subtree *child = self->contents[i]; if (!child->extra) break; array_push(&result, child); } self->size = i + 1; - ts_tree_array_reverse(&result); + ts_subtree_array_reverse(&result); return result; } -void ts_tree_array_reverse(TreeArray *self) { +void ts_subtree_array_reverse(SubtreeArray *self) { for (uint32_t i = 0, limit = self->size / 2; i < limit; i++) { size_t reverse_index = self->size - 1 - i; - Tree *swap = self->contents[i]; + Subtree *swap = self->contents[i]; self->contents[i] = self->contents[reverse_index]; self->contents[reverse_index] = swap; } } -// TreePool +// SubtreePool static const uint32_t MAX_TREE_POOL_SIZE = 1024; -void ts_tree_pool_init(TreePool *self) { +void ts_subtree_pool_init(SubtreePool *self) { array_init(&self->free_trees); array_init(&self->tree_stack); } -void ts_tree_pool_delete(TreePool *self) { +void ts_subtree_pool_delete(SubtreePool *self) { if (self->free_trees.contents) { for (unsigned i = 0; i < self->free_trees.size; i++) { ts_free(self->free_trees.contents[i]); @@ -119,15 +119,15 @@ void ts_tree_pool_delete(TreePool *self) { if (self->tree_stack.contents) array_delete(&self->tree_stack); } -Tree *ts_tree_pool_allocate(TreePool *self) { +Subtree *ts_subtree_pool_allocate(SubtreePool *self) { if (self->free_trees.size > 0) { return array_pop(&self->free_trees); } else { - return ts_malloc(sizeof(Tree)); + return ts_malloc(sizeof(Subtree)); } } -void ts_tree_pool_free(TreePool *self, Tree *tree) { +void ts_subtree_pool_free(SubtreePool *self, Subtree *tree) { if (self->free_trees.size < MAX_TREE_POOL_SIZE) { array_push(&self->free_trees, tree); } else { @@ -135,12 +135,13 @@ void ts_tree_pool_free(TreePool *self, Tree *tree) { } } -// Tree +// Subtree -Tree *ts_tree_make_leaf(TreePool *pool, TSSymbol symbol, Length padding, Length size, const TSLanguage *language) { +Subtree *ts_subtree_make_leaf(SubtreePool *pool, TSSymbol symbol, Length padding, Length size, + const TSLanguage *language) { TSSymbolMetadata metadata = ts_language_symbol_metadata(language, symbol); - Tree *result = ts_tree_pool_allocate(pool); - *result = (Tree){ + Subtree *result = ts_subtree_pool_allocate(pool); + *result = (Subtree){ .ref_count = 1, .symbol = symbol, .size = size, @@ -161,50 +162,51 @@ Tree *ts_tree_make_leaf(TreePool *pool, TSSymbol symbol, Length padding, Length return result; } -Tree *ts_tree_make_error(TreePool *pool, Length size, Length padding, int32_t lookahead_char, - const TSLanguage *language) { - Tree *result = ts_tree_make_leaf(pool, ts_builtin_sym_error, padding, size, language); +Subtree *ts_subtree_make_error(SubtreePool *pool, Length size, Length padding, + int32_t lookahead_char, const TSLanguage *language) { + Subtree *result = ts_subtree_make_leaf(pool, ts_builtin_sym_error, padding, size, language); result->fragile_left = true; result->fragile_right = true; result->lookahead_char = lookahead_char; return result; } -Tree *ts_tree_make_copy(TreePool *pool, Tree *self) { - Tree *result = ts_tree_pool_allocate(pool); +Subtree *ts_subtree_make_copy(SubtreePool *pool, Subtree *self) { + Subtree *result = ts_subtree_pool_allocate(pool); *result = *self; if (result->children.size > 0) { - ts_tree_array_copy(self->children, &result->children); + ts_subtree_array_copy(self->children, &result->children); } result->ref_count = 1; return result; } -static Tree *ts_tree_make_mut(TreePool *pool, Tree *self) { +static Subtree *ts_subtree_make_mut(SubtreePool *pool, Subtree *self) { if (self->ref_count == 1) { return self; } else { - Tree *result = ts_tree_make_copy(pool, self); - ts_tree_release(pool, self); + Subtree *result = ts_subtree_make_copy(pool, self); + ts_subtree_release(pool, self); return result; } } -static void ts_tree__compress(Tree *self, unsigned count, const TSLanguage *language, TreeArray *stack) { +static void ts_subtree__compress(Subtree *self, unsigned count, const TSLanguage *language, + SubtreeArray *stack) { unsigned initial_stack_size = stack->size; - Tree *tree = self; + Subtree *tree = self; for (unsigned i = 0; i < count; i++) { if (tree->ref_count > 1 || tree->children.size != 2) break; - Tree *child = tree->children.contents[0]; + Subtree *child = tree->children.contents[0]; if ( child->ref_count > 1 || child->children.size != 2 || child->symbol != tree->symbol ) break; - Tree *grandchild = child->children.contents[0]; + Subtree *grandchild = child->children.contents[0]; if ( grandchild->ref_count > 1 || grandchild->children.size != 2 || @@ -221,19 +223,19 @@ static void ts_tree__compress(Tree *self, unsigned count, const TSLanguage *lang while (stack->size > initial_stack_size) { tree = array_pop(stack); assert(tree); - Tree *child = tree->children.contents[0]; - Tree *grandchild = child->children.contents[1]; - ts_tree_set_children(grandchild, &grandchild->children, language); - ts_tree_set_children(child, &child->children, language); - ts_tree_set_children(tree, &tree->children, language); + Subtree *child = tree->children.contents[0]; + Subtree *grandchild = child->children.contents[1]; + ts_subtree_set_children(grandchild, &grandchild->children, language); + ts_subtree_set_children(child, &child->children, language); + ts_subtree_set_children(tree, &tree->children, language); } } -void ts_tree_balance(Tree *self, TreePool *pool, const TSLanguage *language) { +void ts_subtree_balance(Subtree *self, SubtreePool *pool, const TSLanguage *language) { array_clear(&pool->tree_stack); array_push(&pool->tree_stack, self); while (pool->tree_stack.size > 0) { - Tree *tree = array_pop(&pool->tree_stack); + Subtree *tree = array_pop(&pool->tree_stack); assert(tree); if (tree->repeat_depth > 0) { @@ -243,14 +245,14 @@ void ts_tree_balance(Tree *self, TreePool *pool, const TSLanguage *language) { tree->children.contents[1]->repeat_depth ); for (unsigned i = n / 2; i > 0; i /= 2) { - ts_tree__compress(tree, i, language, &pool->tree_stack); + ts_subtree__compress(tree, i, language, &pool->tree_stack); n -= i; } } } for (uint32_t i = 0; i < tree->children.size; i++) { - Tree *child = tree->children.contents[i]; + Subtree *child = tree->children.contents[i]; if (child->ref_count == 1) { array_push(&pool->tree_stack, child); } @@ -258,7 +260,7 @@ void ts_tree_balance(Tree *self, TreePool *pool, const TSLanguage *language) { } } -void ts_tree_set_children(Tree *self, TreeArray *children, const TSLanguage *language) { +void ts_subtree_set_children(Subtree *self, SubtreeArray *children, const TSLanguage *language) { if (self->children.size > 0 && children->contents != self->children.contents) { array_delete(&self->children); } @@ -276,16 +278,16 @@ void ts_tree_set_children(Tree *self, TreeArray *children, const TSLanguage *lan const TSSymbol *alias_sequence = ts_language_alias_sequence(language, self->alias_sequence_id); for (uint32_t i = 0; i < self->children.size; i++) { - Tree *child = self->children.contents[i]; + Subtree *child = self->children.contents[i]; if (i == 0) { self->padding = child->padding; self->size = child->size; self->bytes_scanned = child->bytes_scanned; } else { - uint32_t bytes_scanned = ts_tree_total_bytes(self) + child->bytes_scanned; + uint32_t bytes_scanned = ts_subtree_total_bytes(self) + child->bytes_scanned; if (bytes_scanned > self->bytes_scanned) self->bytes_scanned = bytes_scanned; - self->size = length_add(self->size, ts_tree_total_size(child)); + self->size = length_add(self->size, ts_subtree_total_size(child)); } if (child->symbol != ts_builtin_sym_error_repeat) { @@ -322,7 +324,7 @@ void ts_tree_set_children(Tree *self, TreeArray *children, const TSLanguage *lan ERROR_COST_PER_SKIPPED_CHAR * self->size.bytes + ERROR_COST_PER_SKIPPED_LINE * self->size.extent.row; for (uint32_t i = 0; i < self->children.size; i++) { - Tree *child = self->children.contents[i]; + Subtree *child = self->children.contents[i]; if (child->extra) continue; if (child->symbol == ts_builtin_sym_error && child->children.size == 0) continue; if (child->visible) { @@ -334,8 +336,8 @@ void ts_tree_set_children(Tree *self, TreeArray *children, const TSLanguage *lan } if (self->children.size > 0) { - Tree *first_child = self->children.contents[0]; - Tree *last_child = self->children.contents[self->children.size - 1]; + Subtree *first_child = self->children.contents[0]; + Subtree *last_child = self->children.contents[self->children.size - 1]; self->first_leaf = first_child->first_leaf; if (first_child->fragile_left) self->fragile_left = true; if (last_child->fragile_right) self->fragile_right = true; @@ -354,40 +356,42 @@ void ts_tree_set_children(Tree *self, TreeArray *children, const TSLanguage *lan } } -Tree *ts_tree_make_node(TreePool *pool, TSSymbol symbol, TreeArray *children, +Subtree *ts_subtree_make_node(SubtreePool *pool, TSSymbol symbol, SubtreeArray *children, unsigned alias_sequence_id, const TSLanguage *language) { - Tree *result = ts_tree_make_leaf(pool, symbol, length_zero(), length_zero(), language); + Subtree *result = ts_subtree_make_leaf(pool, symbol, length_zero(), length_zero(), language); result->alias_sequence_id = alias_sequence_id; if (symbol == ts_builtin_sym_error || symbol == ts_builtin_sym_error_repeat) { result->fragile_left = true; result->fragile_right = true; } - ts_tree_set_children(result, children, language); + ts_subtree_set_children(result, children, language); return result; } -Tree *ts_tree_make_error_node(TreePool *pool, TreeArray *children, const TSLanguage *language) { - return ts_tree_make_node(pool, ts_builtin_sym_error, children, 0, language); +Subtree *ts_subtree_make_error_node(SubtreePool *pool, SubtreeArray *children, + const TSLanguage *language) { + return ts_subtree_make_node(pool, ts_builtin_sym_error, children, 0, language); } -Tree *ts_tree_make_missing_leaf(TreePool *pool, TSSymbol symbol, const TSLanguage *language) { - Tree *result = ts_tree_make_leaf(pool, symbol, length_zero(), length_zero(), language); +Subtree *ts_subtree_make_missing_leaf(SubtreePool *pool, TSSymbol symbol, + const TSLanguage *language) { + Subtree *result = ts_subtree_make_leaf(pool, symbol, length_zero(), length_zero(), language); result->is_missing = true; result->error_cost = ERROR_COST_PER_MISSING_TREE + ERROR_COST_PER_RECOVERY; return result; } -void ts_tree_retain(Tree *self) { +void ts_subtree_retain(Subtree *self) { assert(self->ref_count > 0); atomic_inc(&self->ref_count); assert(self->ref_count != 0); } -void ts_tree_release(TreePool *pool, Tree *self) { +void ts_subtree_release(SubtreePool *pool, Subtree *self) { array_clear(&pool->tree_stack); array_push(&pool->tree_stack, self); while (pool->tree_stack.size > 0) { - Tree *tree = array_pop(&pool->tree_stack); + Subtree *tree = array_pop(&pool->tree_stack); assert(tree->ref_count > 0); if (atomic_dec(&tree->ref_count) == 0) { if (tree->children.size > 0) { @@ -398,12 +402,12 @@ void ts_tree_release(TreePool *pool, Tree *self) { } else if (tree->has_external_tokens) { ts_external_token_state_delete(&tree->external_token_state); } - ts_tree_pool_free(pool, tree); + ts_subtree_pool_free(pool, tree); } } } -bool ts_tree_eq(const Tree *self, const Tree *other) { +bool ts_subtree_eq(const Subtree *self, const Subtree *other) { if (self) { if (!other) return false; } else { @@ -421,14 +425,14 @@ bool ts_tree_eq(const Tree *self, const Tree *other) { if (self->named_child_count != other->named_child_count) return false; for (uint32_t i = 0; i < self->children.size; i++) { - if (!ts_tree_eq(self->children.contents[i], other->children.contents[i])) { + if (!ts_subtree_eq(self->children.contents[i], other->children.contents[i])) { return false; } } return true; } -int ts_tree_compare(const Tree *left, const Tree *right) { +int ts_subtree_compare(const Subtree *left, const Subtree *right) { if (left->symbol < right->symbol) return -1; if (right->symbol < left->symbol) @@ -438,9 +442,9 @@ int ts_tree_compare(const Tree *left, const Tree *right) { if (right->children.size < left->children.size) return 1; for (uint32_t i = 0; i < left->children.size; i++) { - Tree *left_child = left->children.contents[i]; - Tree *right_child = right->children.contents[i]; - switch (ts_tree_compare(left_child, right_child)) { + Subtree *left_child = left->children.contents[i]; + Subtree *right_child = right->children.contents[i]; + switch (ts_subtree_compare(left_child, right_child)) { case -1: return -1; case 1: @@ -452,30 +456,31 @@ int ts_tree_compare(const Tree *left, const Tree *right) { return 0; } -Tree *ts_tree_invalidate_lookahead(Tree *self, uint32_t edit_byte_offset, TreePool *pool) { +Subtree *ts_subtree_invalidate_lookahead(Subtree *self, uint32_t edit_byte_offset, + SubtreePool *pool) { if (edit_byte_offset >= self->bytes_scanned) return self; - Tree *result = ts_tree_make_mut(pool, self); + Subtree *result = ts_subtree_make_mut(pool, self); result->has_changes = true; if (result->children.size > 0) { uint32_t child_start_byte = 0; for (uint32_t i = 0; i < result->children.size; i++) { - Tree **child = &result->children.contents[i]; + Subtree **child = &result->children.contents[i]; if (child_start_byte > edit_byte_offset) break; - *child = ts_tree_invalidate_lookahead(*child, edit_byte_offset - child_start_byte, pool); - child_start_byte += ts_tree_total_bytes(*child); + *child = ts_subtree_invalidate_lookahead(*child, edit_byte_offset - child_start_byte, pool); + child_start_byte += ts_subtree_total_bytes(*child); } } return result; } -Tree *ts_tree__edit(Tree *self, Edit edit, TreePool *pool) { +Subtree *ts_subtree__edit(Subtree *self, Edit edit, SubtreePool *pool) { Length new_end = length_add(edit.start, edit.added); Length old_end = length_add(edit.start, edit.removed); - Tree *result = ts_tree_make_mut(pool, self); + Subtree *result = ts_subtree_make_mut(pool, self); result->has_changes = true; if (old_end.bytes <= result->padding.bytes) { @@ -486,14 +491,14 @@ Tree *ts_tree__edit(Tree *self, Edit edit, TreePool *pool) { } else if (edit.start.bytes == result->padding.bytes && edit.removed.bytes == 0) { result->padding = length_add(result->padding, edit.added); } else { - Length new_total_size = length_add(new_end, length_sub(ts_tree_total_size(result), old_end)); + Length new_total_size = length_add(new_end, length_sub(ts_subtree_total_size(result), old_end)); result->size = length_sub(new_total_size, result->padding); } Length child_left, child_right = length_zero(); for (uint32_t i = 0; i < result->children.size; i++) { - Tree **child = &result->children.contents[i]; - Length child_size = ts_tree_total_size(*child); + Subtree **child = &result->children.contents[i]; + Length child_size = ts_subtree_total_size(*child); child_left = child_right; child_right = length_add(child_left, child_size); @@ -519,28 +524,28 @@ Tree *ts_tree__edit(Tree *self, Edit edit, TreePool *pool) { edit.added = length_zero(); edit.removed = length_sub(edit.removed, child_edit.removed); - *child = ts_tree__edit(*child, child_edit, pool); + *child = ts_subtree__edit(*child, child_edit, pool); } else if (child_left.bytes <= edit.start.bytes) { - *child = ts_tree_invalidate_lookahead(*child, edit.start.bytes - child_left.bytes, pool); + *child = ts_subtree_invalidate_lookahead(*child, edit.start.bytes - child_left.bytes, pool); } } return result; } -Tree *ts_tree_edit(Tree *self, const TSInputEdit *edit, TreePool *pool) { - return ts_tree__edit(self, (Edit) { +Subtree *ts_subtree_edit(Subtree *self, const TSInputEdit *edit, SubtreePool *pool) { + return ts_subtree__edit(self, (Edit) { .start = {edit->start_byte, edit->start_point}, .added = {edit->bytes_added, edit->extent_added}, .removed = {edit->bytes_removed, edit->extent_removed}, }, pool); } -Tree *ts_tree_last_external_token(Tree *tree) { +Subtree *ts_subtree_last_external_token(Subtree *tree) { if (!tree->has_external_tokens) return NULL; while (tree->children.size > 0) { for (uint32_t i = tree->children.size - 1; i + 1 > 0; i--) { - Tree *child = tree->children.contents[i]; + Subtree *child = tree->children.contents[i]; if (child->has_external_tokens) { tree = child; break; @@ -550,7 +555,7 @@ Tree *ts_tree_last_external_token(Tree *tree) { return tree; } -static size_t ts_tree__write_char_to_string(char *s, size_t n, int32_t c) { +static size_t ts_subtree__write_char_to_string(char *s, size_t n, int32_t c) { if (c == 0) return snprintf(s, n, "EOF"); if (c == -1) @@ -567,7 +572,7 @@ static size_t ts_tree__write_char_to_string(char *s, size_t n, int32_t c) { return snprintf(s, n, "%d", c); } -static size_t ts_tree__write_to_string(const Tree *self, char *string, size_t limit, +static size_t ts_subtree__write_to_string(const Subtree *self, char *string, size_t limit, const TSLanguage *language, bool is_root, bool include_all, TSSymbol alias_symbol, bool alias_is_named) { @@ -589,7 +594,7 @@ static size_t ts_tree__write_to_string(const Tree *self, char *string, size_t li if (visible) { if (self->symbol == ts_builtin_sym_error && self->children.size == 0 && self->size.bytes > 0) { cursor += snprintf(*writer, limit, "(UNEXPECTED "); - cursor += ts_tree__write_char_to_string(*writer, limit, self->lookahead_char); + cursor += ts_subtree__write_char_to_string(*writer, limit, self->lookahead_char); } else if (self->is_missing) { cursor += snprintf(*writer, limit, "(MISSING"); } else { @@ -602,16 +607,16 @@ static size_t ts_tree__write_to_string(const Tree *self, char *string, size_t li const TSSymbol *alias_sequence = ts_language_alias_sequence(language, self->alias_sequence_id); uint32_t structural_child_index = 0; for (uint32_t i = 0; i < self->children.size; i++) { - Tree *child = self->children.contents[i]; + Subtree *child = self->children.contents[i]; if (child->extra) { - cursor += ts_tree__write_to_string( + cursor += ts_subtree__write_to_string( child, *writer, limit, language, false, include_all, 0, false ); } else { TSSymbol alias_symbol = alias_sequence ? alias_sequence[structural_child_index] : 0; - cursor += ts_tree__write_to_string( + cursor += ts_subtree__write_to_string( child, *writer, limit, language, false, include_all, alias_symbol, @@ -626,19 +631,19 @@ static size_t ts_tree__write_to_string(const Tree *self, char *string, size_t li return cursor - string; } -char *ts_tree_string(const Tree *self, const TSLanguage *language, bool include_all) { +char *ts_subtree_string(const Subtree *self, const TSLanguage *language, bool include_all) { char scratch_string[1]; - size_t size = ts_tree__write_to_string( + size_t size = ts_subtree__write_to_string( self, scratch_string, 0, language, true, include_all, 0, false ) + 1; char *result = ts_malloc(size * sizeof(char)); - ts_tree__write_to_string(self, result, size, language, true, include_all, 0, false); + ts_subtree__write_to_string(self, result, size, language, true, include_all, 0, false); return result; } -void ts_tree__print_dot_graph(const Tree *self, uint32_t byte_offset, +void ts_subtree__print_dot_graph(const Subtree *self, uint32_t byte_offset, const TSLanguage *language, TSSymbol alias_symbol, FILE *f) { TSSymbol symbol = alias_symbol ? alias_symbol : self->symbol; fprintf(f, "tree_%p [label=\"%s\"", self, ts_language_symbol_name(language, symbol)); @@ -656,7 +661,7 @@ void ts_tree__print_dot_graph(const Tree *self, uint32_t byte_offset, "repeat-depth:%u\n" "bytes-scanned:%u\"]\n", self, - byte_offset, byte_offset + ts_tree_total_bytes(self), + byte_offset, byte_offset + ts_subtree_total_bytes(self), self->parse_state, self->error_cost, self->repeat_depth, @@ -666,29 +671,29 @@ void ts_tree__print_dot_graph(const Tree *self, uint32_t byte_offset, const TSSymbol *alias_sequence = ts_language_alias_sequence(language, self->alias_sequence_id); uint32_t structural_child_index = 0; for (uint32_t i = 0; i < self->children.size; i++) { - const Tree *child = self->children.contents[i]; + const Subtree *child = self->children.contents[i]; if (child->extra) { - ts_tree__print_dot_graph(child, byte_offset, language, 0, f); + ts_subtree__print_dot_graph(child, byte_offset, language, 0, f); } else { TSSymbol alias_symbol = alias_sequence ? alias_sequence[structural_child_index] : 0; - ts_tree__print_dot_graph(child, byte_offset, language, alias_symbol, f); + ts_subtree__print_dot_graph(child, byte_offset, language, alias_symbol, f); structural_child_index++; } fprintf(f, "tree_%p -> tree_%p [tooltip=%u]\n", self, child, i); - byte_offset += ts_tree_total_bytes(child); + byte_offset += ts_subtree_total_bytes(child); } } -void ts_tree_print_dot_graph(const Tree *self, const TSLanguage *language, FILE *f) { +void ts_subtree_print_dot_graph(const Subtree *self, const TSLanguage *language, FILE *f) { fprintf(f, "digraph tree {\n"); fprintf(f, "edge [arrowhead=none]\n"); - ts_tree__print_dot_graph(self, 0, language, 0, f); + ts_subtree__print_dot_graph(self, 0, language, 0, f); fprintf(f, "}\n"); } static const TSExternalTokenState empty_state = {.length = 0, .short_data = {0}}; -bool ts_tree_external_token_state_eq(const Tree *self, const Tree *other) { +bool ts_subtree_external_token_state_eq(const Subtree *self, const Subtree *other) { const TSExternalTokenState *state1 = &empty_state; const TSExternalTokenState *state2 = &empty_state; if (self && self->has_external_tokens) state1 = &self->external_token_state; diff --git a/src/runtime/subtree.h b/src/runtime/subtree.h new file mode 100644 index 00000000..5168ac38 --- /dev/null +++ b/src/runtime/subtree.h @@ -0,0 +1,125 @@ +#ifndef RUNTIME_TREE_H_ +#define RUNTIME_TREE_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +#include +#include "tree_sitter/parser.h" +#include "tree_sitter/runtime.h" +#include "runtime/length.h" +#include "runtime/array.h" +#include + +extern TSStateId TS_TREE_STATE_NONE; + +typedef struct { + union { + char *long_data; + char short_data[sizeof(char *) + sizeof(uint32_t)]; + }; + uint32_t length; +} TSExternalTokenState; + +typedef struct Subtree Subtree; + +typedef Array(Subtree *) SubtreeArray; + +struct Subtree { + Length padding; + Length size; + volatile uint32_t ref_count; + uint32_t bytes_scanned; + uint32_t error_cost; + uint32_t node_count; + uint32_t repeat_depth; + uint32_t child_count; + int32_t dynamic_precedence; + + bool visible : 1; + bool named : 1; + bool extra : 1; + bool fragile_left : 1; + bool fragile_right : 1; + bool has_changes : 1; + bool has_external_tokens : 1; + bool is_missing : 1; + TSSymbol symbol; + TSStateId parse_state; + struct { + TSSymbol symbol; + TSLexMode lex_mode; + } first_leaf; + + union { + struct { + SubtreeArray children; + uint32_t visible_child_count; + uint32_t named_child_count; + uint16_t alias_sequence_id; + }; + struct { + uint32_t _2; + TSExternalTokenState external_token_state; + }; + struct { + uint32_t _1; + int32_t lookahead_char; + }; + }; +}; + +typedef struct { + SubtreeArray free_trees; + SubtreeArray tree_stack; +} SubtreePool; + +void ts_external_token_state_init(TSExternalTokenState *, const char *, unsigned); +const char *ts_external_token_state_data(const TSExternalTokenState *); + +bool ts_subtree_array_copy(SubtreeArray, SubtreeArray *); +void ts_subtree_array_delete(SubtreePool *, SubtreeArray *); +SubtreeArray ts_subtree_array_remove_trailing_extras(SubtreeArray *); +void ts_subtree_array_reverse(SubtreeArray *); + +void ts_subtree_pool_init(SubtreePool *); +void ts_subtree_pool_delete(SubtreePool *); +Subtree *ts_subtree_pool_allocate(SubtreePool *); +void ts_subtree_pool_free(SubtreePool *, Subtree *); + +Subtree *ts_subtree_make_leaf(SubtreePool *, TSSymbol, Length, Length, const TSLanguage *); +Subtree *ts_subtree_make_node(SubtreePool *, TSSymbol, SubtreeArray *, unsigned, const TSLanguage *); +Subtree *ts_subtree_make_copy(SubtreePool *, Subtree *child); +Subtree *ts_subtree_make_error_node(SubtreePool *, SubtreeArray *, const TSLanguage *); +Subtree *ts_subtree_make_error(SubtreePool *, Length, Length, int32_t, const TSLanguage *); +Subtree *ts_subtree_make_missing_leaf(SubtreePool *, TSSymbol, const TSLanguage *); +void ts_subtree_retain(Subtree *tree); +void ts_subtree_release(SubtreePool *, Subtree *tree); +bool ts_subtree_eq(const Subtree *tree1, const Subtree *tree2); +int ts_subtree_compare(const Subtree *tree1, const Subtree *tree2); +void ts_subtree_set_children(Subtree *, SubtreeArray *, const TSLanguage *); +void ts_subtree_balance(Subtree *, SubtreePool *, const TSLanguage *); +Subtree *ts_subtree_edit(Subtree *, const TSInputEdit *edit, SubtreePool *); +char *ts_subtree_string(const Subtree *, const TSLanguage *, bool include_all); +void ts_subtree_print_dot_graph(const Subtree *, const TSLanguage *, FILE *); +Subtree *ts_subtree_last_external_token(Subtree *); +bool ts_subtree_external_token_state_eq(const Subtree *, const Subtree *); + +static inline uint32_t ts_subtree_total_bytes(const Subtree *self) { + return self->padding.bytes + self->size.bytes; +} + +static inline uint32_t ts_subtree_total_rows(const Subtree *self) { + return self->padding.extent.row + self->size.extent.row; +} + +static inline Length ts_subtree_total_size(const Subtree *self) { + return length_add(self->padding, self->size); +} + +#ifdef __cplusplus +} +#endif + +#endif // RUNTIME_TREE_H_ diff --git a/src/runtime/tree.h b/src/runtime/tree.h deleted file mode 100644 index 4fa95522..00000000 --- a/src/runtime/tree.h +++ /dev/null @@ -1,125 +0,0 @@ -#ifndef RUNTIME_TREE_H_ -#define RUNTIME_TREE_H_ - -#ifdef __cplusplus -extern "C" { -#endif - -#include -#include "tree_sitter/parser.h" -#include "tree_sitter/runtime.h" -#include "runtime/length.h" -#include "runtime/array.h" -#include - -extern TSStateId TS_TREE_STATE_NONE; - -typedef struct { - union { - char *long_data; - char short_data[sizeof(char *) + sizeof(uint32_t)]; - }; - uint32_t length; -} TSExternalTokenState; - -typedef struct Tree Tree; - -typedef Array(Tree *) TreeArray; - -struct Tree { - Length padding; - Length size; - volatile uint32_t ref_count; - uint32_t bytes_scanned; - uint32_t error_cost; - uint32_t node_count; - uint32_t repeat_depth; - uint32_t child_count; - int32_t dynamic_precedence; - - bool visible : 1; - bool named : 1; - bool extra : 1; - bool fragile_left : 1; - bool fragile_right : 1; - bool has_changes : 1; - bool has_external_tokens : 1; - bool is_missing : 1; - TSSymbol symbol; - TSStateId parse_state; - struct { - TSSymbol symbol; - TSLexMode lex_mode; - } first_leaf; - - union { - struct { - TreeArray children; - uint32_t visible_child_count; - uint32_t named_child_count; - uint16_t alias_sequence_id; - }; - struct { - uint32_t _2; - TSExternalTokenState external_token_state; - }; - struct { - uint32_t _1; - int32_t lookahead_char; - }; - }; -}; - -typedef struct { - TreeArray free_trees; - TreeArray tree_stack; -} TreePool; - -void ts_external_token_state_init(TSExternalTokenState *, const char *, unsigned); -const char *ts_external_token_state_data(const TSExternalTokenState *); - -bool ts_tree_array_copy(TreeArray, TreeArray *); -void ts_tree_array_delete(TreePool *, TreeArray *); -TreeArray ts_tree_array_remove_trailing_extras(TreeArray *); -void ts_tree_array_reverse(TreeArray *); - -void ts_tree_pool_init(TreePool *); -void ts_tree_pool_delete(TreePool *); -Tree *ts_tree_pool_allocate(TreePool *); -void ts_tree_pool_free(TreePool *, Tree *); - -Tree *ts_tree_make_leaf(TreePool *, TSSymbol, Length, Length, const TSLanguage *); -Tree *ts_tree_make_node(TreePool *, TSSymbol, TreeArray *, unsigned, const TSLanguage *); -Tree *ts_tree_make_copy(TreePool *, Tree *child); -Tree *ts_tree_make_error_node(TreePool *, TreeArray *, const TSLanguage *); -Tree *ts_tree_make_error(TreePool *, Length, Length, int32_t, const TSLanguage *); -Tree *ts_tree_make_missing_leaf(TreePool *, TSSymbol, const TSLanguage *); -void ts_tree_retain(Tree *tree); -void ts_tree_release(TreePool *, Tree *tree); -bool ts_tree_eq(const Tree *tree1, const Tree *tree2); -int ts_tree_compare(const Tree *tree1, const Tree *tree2); -void ts_tree_set_children(Tree *, TreeArray *, const TSLanguage *); -void ts_tree_balance(Tree *, TreePool *, const TSLanguage *); -Tree *ts_tree_edit(Tree *, const TSInputEdit *edit, TreePool *); -char *ts_tree_string(const Tree *, const TSLanguage *, bool include_all); -void ts_tree_print_dot_graph(const Tree *, const TSLanguage *, FILE *); -Tree *ts_tree_last_external_token(Tree *); -bool ts_tree_external_token_state_eq(const Tree *, const Tree *); - -static inline uint32_t ts_tree_total_bytes(const Tree *self) { - return self->padding.bytes + self->size.bytes; -} - -static inline uint32_t ts_tree_total_rows(const Tree *self) { - return self->padding.extent.row + self->size.extent.row; -} - -static inline Length ts_tree_total_size(const Tree *self) { - return length_add(self->padding, self->size); -} - -#ifdef __cplusplus -} -#endif - -#endif // RUNTIME_TREE_H_ diff --git a/src/runtime/tree_cursor.c b/src/runtime/tree_cursor.c index db8298d2..730dacf9 100644 --- a/src/runtime/tree_cursor.c +++ b/src/runtime/tree_cursor.c @@ -24,7 +24,7 @@ void ts_tree_cursor_delete(TSTreeCursor *self) { bool ts_tree_cursor_goto_first_child(TSTreeCursor *self) { TreeCursorEntry *last_entry = array_back(&self->stack); - Tree *tree = last_entry->tree; + Subtree *tree = last_entry->tree; Length position = last_entry->position; bool did_descend; @@ -33,7 +33,7 @@ bool ts_tree_cursor_goto_first_child(TSTreeCursor *self) { uint32_t structural_child_index = 0; for (uint32_t i = 0; i < tree->children.size; i++) { - Tree *child = tree->children.contents[i]; + Subtree *child = tree->children.contents[i]; if (child->visible || child->visible_child_count > 0) { array_push(&self->stack, ((TreeCursorEntry) { .tree = child, @@ -51,7 +51,7 @@ bool ts_tree_cursor_goto_first_child(TSTreeCursor *self) { } } if (!child->extra) structural_child_index++; - position = length_add(position, ts_tree_total_size(child)); + position = length_add(position, ts_subtree_total_size(child)); } } while (did_descend); @@ -64,15 +64,15 @@ bool ts_tree_cursor_goto_next_sibling(TSTreeCursor *self) { for (unsigned i = self->stack.size - 2; i + 1 > 0; i--) { TreeCursorEntry *parent_entry = &self->stack.contents[i]; - Tree *parent = parent_entry->tree; + Subtree *parent = parent_entry->tree; uint32_t child_index = child_entry->child_index; uint32_t structural_child_index = child_entry->structural_child_index; Length position = child_entry->position; - Tree *child = parent->children.contents[child_index]; + Subtree *child = parent->children.contents[child_index]; while (++child_index < parent->children.size) { if (!child->extra) structural_child_index++; - position = length_add(position, ts_tree_total_size(child)); + position = length_add(position, ts_subtree_total_size(child)); child = parent->children.contents[child_index]; if (child->visible || child->visible_child_count > 0) { diff --git a/src/runtime/tree_cursor.h b/src/runtime/tree_cursor.h index 33e1d802..5e571fba 100644 --- a/src/runtime/tree_cursor.h +++ b/src/runtime/tree_cursor.h @@ -1,10 +1,10 @@ #ifndef RUNTIME_TREE_CURSOR_H_ #define RUNTIME_TREE_CURSOR_H_ -#include "runtime/tree.h" +#include "runtime/subtree.h" typedef struct { - Tree *tree; + Subtree *tree; Length position; uint32_t child_index; uint32_t structural_child_index; diff --git a/test/helpers/tree_helpers.cc b/test/helpers/tree_helpers.cc index 9a86b364..7fd015a3 100644 --- a/test/helpers/tree_helpers.cc +++ b/test/helpers/tree_helpers.cc @@ -14,21 +14,21 @@ const char *symbol_names[24] = { "twenty-two", "twenty-three" }; -TreeArray *tree_array(std::vector trees) { - static TreeArray result; +SubtreeArray *tree_array(std::vector trees) { + static SubtreeArray result; result.capacity = trees.size(); result.size = trees.size(); - result.contents = (Tree **)calloc(trees.size(), sizeof(Tree *)); + result.contents = (Subtree **)calloc(trees.size(), sizeof(Subtree *)); for (size_t i = 0; i < trees.size(); i++) { result.contents[i] = trees[i]; } return &result; } -ostream &operator<<(std::ostream &stream, const Tree *tree) { +ostream &operator<<(std::ostream &stream, const Subtree *tree) { static TSLanguage DUMMY_LANGUAGE = {}; DUMMY_LANGUAGE.symbol_names = symbol_names; - char *string = ts_tree_string(tree, &DUMMY_LANGUAGE, false); + char *string = ts_subtree_string(tree, &DUMMY_LANGUAGE, false); stream << string; ts_free(string); return stream; @@ -49,7 +49,7 @@ bool operator==(const TSNode &left, const TSNode &right) { return ts_node_eq(left, right); } -bool operator==(const std::vector &vec, const TreeArray &array) { +bool operator==(const std::vector &vec, const SubtreeArray &array) { if (vec.size() != array.size) return false; for (size_t i = 0; i < array.size; i++) diff --git a/test/helpers/tree_helpers.h b/test/helpers/tree_helpers.h index c28dcd98..bd08aad7 100644 --- a/test/helpers/tree_helpers.h +++ b/test/helpers/tree_helpers.h @@ -1,17 +1,17 @@ #ifndef HELPERS_TREE_HELPERS_H_ #define HELPERS_TREE_HELPERS_H_ -#include "runtime/tree.h" +#include "runtime/subtree.h" #include #include extern const char *symbol_names[24]; -TreeArray *tree_array(std::vector trees); +SubtreeArray *tree_array(std::vector trees); -std::ostream &operator<<(std::ostream &stream, const Tree *tree); +std::ostream &operator<<(std::ostream &stream, const Subtree *tree); std::ostream &operator<<(std::ostream &stream, const TSNode &node); bool operator==(const TSNode &left, const TSNode &right); -bool operator==(const std::vector &right, const TreeArray &array); +bool operator==(const std::vector &right, const SubtreeArray &array); void assert_consistent_tree_sizes(TSNode node); diff --git a/test/runtime/stack_test.cc b/test/runtime/stack_test.cc index d6cbcea0..99a979d1 100644 --- a/test/runtime/stack_test.cc +++ b/test/runtime/stack_test.cc @@ -4,7 +4,7 @@ #include "helpers/record_alloc.h" #include "helpers/stream_methods.h" #include "runtime/stack.h" -#include "runtime/tree.h" +#include "runtime/subtree.h" #include "runtime/length.h" #include "runtime/alloc.h" @@ -23,23 +23,23 @@ Length operator*(const Length &length, uint32_t factor) { return {length.bytes * factor, {0, length.extent.column * factor}}; } -void free_slice_array(TreePool *pool, StackSliceArray *slices) { +void free_slice_array(SubtreePool *pool, StackSliceArray *slices) { for (size_t i = 0; i < slices->size; i++) { StackSlice slice = slices->contents[i]; bool matches_prior_trees = false; for (size_t j = 0; j < i; j++) { StackSlice prior_slice = slices->contents[j]; - if (slice.trees.contents == prior_slice.trees.contents) { + if (slice.subtrees.contents == prior_slice.subtrees.contents) { matches_prior_trees = true; break; } } if (!matches_prior_trees) { - for (size_t j = 0; j < slice.trees.size; j++) - ts_tree_release(pool, slice.trees.contents[j]); - array_delete(&slice.trees); + for (size_t j = 0; j < slice.subtrees.size; j++) + ts_subtree_release(pool, slice.subtrees.contents[j]); + array_delete(&slice.subtrees); } } } @@ -54,9 +54,9 @@ vector get_stack_entries(Stack *stack, StackVersion version) { ts_stack_iterate( stack, version, - [](void *payload, TSStateId state, uint32_t tree_count) { + [](void *payload, TSStateId state, uint32_t subtree_count) { auto entries = static_cast *>(payload); - StackEntry entry = {state, tree_count}; + StackEntry entry = {state, subtree_count}; if (find(entries->begin(), entries->end(), entry) == entries->end()) { entries->push_back(entry); } @@ -68,39 +68,39 @@ START_TEST describe("Stack", [&]() { Stack *stack; - const size_t tree_count = 11; - Tree *trees[tree_count]; + const size_t subtree_count = 11; + Subtree *subtrees[subtree_count]; Length tree_len = {3, {0, 3}}; - TreePool pool; + SubtreePool pool; before_each([&]() { record_alloc::start(); - ts_tree_pool_init(&pool); + ts_subtree_pool_init(&pool); stack = ts_stack_new(&pool); TSLanguage dummy_language; TSSymbolMetadata symbol_metadata[50] = {}; dummy_language.symbol_metadata = symbol_metadata; - for (size_t i = 0; i < tree_count; i++) { - trees[i] = ts_tree_make_leaf(&pool, i, length_zero(), tree_len, &dummy_language); + for (size_t i = 0; i < subtree_count; i++) { + subtrees[i] = ts_subtree_make_leaf(&pool, i, length_zero(), tree_len, &dummy_language); } }); after_each([&]() { ts_stack_delete(stack); - for (size_t i = 0; i < tree_count; i++) { - ts_tree_release(&pool, trees[i]); + for (size_t i = 0; i < subtree_count; i++) { + ts_subtree_release(&pool, subtrees[i]); } - ts_tree_pool_delete(&pool); + ts_subtree_pool_delete(&pool); record_alloc::stop(); AssertThat(record_alloc::outstanding_allocation_indices(), IsEmpty()); }); - auto push = [&](StackVersion version, Tree *tree, TSStateId state) { - ts_tree_retain(tree); + auto push = [&](StackVersion version, Subtree *tree, TSStateId state) { + ts_subtree_retain(tree); ts_stack_push(stack, version, tree, false, state); }; @@ -111,17 +111,17 @@ describe("Stack", [&]() { AssertThat(ts_stack_position(stack, 0), Equals(length_zero())); // . <──0── A* - push(0, trees[0], stateA); + push(0, subtrees[0], stateA); AssertThat(ts_stack_state(stack, 0), Equals(stateA)); AssertThat(ts_stack_position(stack, 0), Equals(tree_len)); // . <──0── A <──1── B* - push(0, trees[1], stateB); + push(0, subtrees[1], stateB); AssertThat(ts_stack_state(stack, 0), Equals(stateB)); AssertThat(ts_stack_position(stack, 0), Equals(tree_len * 2)); // . <──0── A <──1── B <──2── C* - push(0, trees[2], stateC); + push(0, subtrees[2], stateC); AssertThat(ts_stack_state(stack, 0), Equals(stateC)); AssertThat(ts_stack_position(stack, 0), Equals(tree_len * 3)); @@ -139,7 +139,7 @@ describe("Stack", [&]() { // . <──0── A <─* // ↑ // └───* - push(0, trees[0], stateA); + push(0, subtrees[0], stateA); ts_stack_copy_version(stack, 0); }); @@ -147,10 +147,10 @@ describe("Stack", [&]() { // . <──0── A <──1── B <──3── D* // ↑ // └───2─── C <──4── D* - push(0, trees[1], stateB); - push(1, trees[2], stateC); - push(0, trees[3], stateD); - push(1, trees[4], stateD); + push(0, subtrees[1], stateB); + push(1, subtrees[2], stateC); + push(0, subtrees[3], stateD); + push(1, subtrees[4], stateD); // . <──0── A <──1── B <──3── D* // ↑ | @@ -170,8 +170,8 @@ describe("Stack", [&]() { // . <──0── A <──1── B* // ↑ // └───2─── C* - push(0, trees[1], stateB); - push(1, trees[2], stateC); + push(0, subtrees[1], stateB); + push(1, subtrees[2], stateC); AssertThat(ts_stack_merge(stack, 0, 1), IsFalse()); AssertThat(ts_stack_version_count(stack), Equals(2)); @@ -181,11 +181,11 @@ describe("Stack", [&]() { // . <──0── A <──1── B <────3──── D* // ↑ // └───2─── C <──4── D* - trees[3]->size = tree_len * 3; - push(0, trees[1], stateB); - push(1, trees[2], stateC); - push(0, trees[3], stateD); - push(1, trees[4], stateD); + subtrees[3]->size = tree_len * 3; + push(0, subtrees[1], stateB); + push(1, subtrees[2], stateC); + push(0, subtrees[3], stateD); + push(1, subtrees[4], stateD); AssertThat(ts_stack_merge(stack, 0, 1), IsFalse()); AssertThat(ts_stack_version_count(stack), Equals(2)); @@ -196,12 +196,12 @@ describe("Stack", [&]() { // . <──0── A <──1── B <──3── D <──5── E* // ↑ // └───2─── C <──4── D <──5── E* - push(0, trees[1], stateB); - push(1, trees[2], stateC); - push(0, trees[3], stateD); - push(1, trees[4], stateD); - push(0, trees[5], stateE); - push(1, trees[5], stateE); + push(0, subtrees[1], stateB); + push(1, subtrees[2], stateC); + push(0, subtrees[3], stateD); + push(1, subtrees[4], stateD); + push(0, subtrees[5], stateE); + push(1, subtrees[5], stateE); // . <──0── A <──1── B <──3── D <──5── E* // ↑ | @@ -224,12 +224,12 @@ describe("Stack", [&]() { // . <──0── A <────1──── B* // ↑ // └2─ A <──1── B* - trees[2]->extra = true; - trees[2]->size = tree_len * 0; + subtrees[2]->extra = true; + subtrees[2]->size = tree_len * 0; - push(0, trees[1], stateB); - push(1, trees[2], stateA); - push(1, trees[1], stateB); + push(0, subtrees[1], stateB); + push(1, subtrees[2], stateA); + push(1, subtrees[1], stateB); // . <──0── A <──1── B* AssertThat(ts_stack_merge(stack, 0, 1), IsTrue()); @@ -246,9 +246,9 @@ describe("Stack", [&]() { describe("pop_count(version, count)", [&]() { before_each([&]() { // . <──0── A <──1── B <──2── C* - push(0, trees[0], stateA); - push(0, trees[1], stateB); - push(0, trees[2], stateC); + push(0, subtrees[0], stateA); + push(0, subtrees[1], stateB); + push(0, subtrees[2], stateC); }); it("creates a new version with the given number of entries removed", [&]() { @@ -261,14 +261,14 @@ describe("Stack", [&]() { StackSlice slice = pop.contents[0]; AssertThat(slice.version, Equals(1)); - AssertThat(slice.trees, Equals(vector({ trees[1], trees[2] }))); + AssertThat(slice.subtrees, Equals(vector({ subtrees[1], subtrees[2] }))); AssertThat(ts_stack_state(stack, 1), Equals(stateA)); free_slice_array(&pool,&pop); }); - it("does not count 'extra' trees toward the given count", [&]() { - trees[1]->extra = true; + it("does not count 'extra' subtrees toward the given count", [&]() { + subtrees[1]->extra = true; // . <──0── A <──1── B <──2── C* // ↑ @@ -277,7 +277,7 @@ describe("Stack", [&]() { AssertThat(pop.size, Equals(1)); StackSlice slice = pop.contents[0]; - AssertThat(slice.trees, Equals(vector({ trees[0], trees[1], trees[2] }))); + AssertThat(slice.subtrees, Equals(vector({ subtrees[0], subtrees[1], subtrees[2] }))); AssertThat(ts_stack_state(stack, 1), Equals(1)); free_slice_array(&pool,&pop); @@ -288,14 +288,14 @@ describe("Stack", [&]() { // . <──0── A <──1── B <──2── C <──3── D <──10── I* // ↑ | // └───4─── E <──5── F <──6───┘ - push(0, trees[3], stateD); + push(0, subtrees[3], stateD); StackSliceArray pop = ts_stack_pop_count(stack, 0, 3); free_slice_array(&pool,&pop); - push(1, trees[4], stateE); - push(1, trees[5], stateF); - push(1, trees[6], stateD); + push(1, subtrees[4], stateE); + push(1, subtrees[5], stateF); + push(1, subtrees[6], stateD); ts_stack_merge(stack, 0, 1); - push(0, trees[10], stateI); + push(0, subtrees[10], stateI); AssertThat(ts_stack_version_count(stack), Equals(1)); AssertThat(get_stack_entries(stack, 0), Equals(vector({ @@ -322,11 +322,11 @@ describe("Stack", [&]() { StackSlice slice1 = pop.contents[0]; AssertThat(slice1.version, Equals(1)); - AssertThat(slice1.trees, Equals(vector({ trees[2], trees[3], trees[10] }))); + AssertThat(slice1.subtrees, Equals(vector({ subtrees[2], subtrees[3], subtrees[10] }))); StackSlice slice2 = pop.contents[1]; AssertThat(slice2.version, Equals(2)); - AssertThat(slice2.trees, Equals(vector({ trees[5], trees[6], trees[10] }))); + AssertThat(slice2.subtrees, Equals(vector({ subtrees[5], subtrees[6], subtrees[10] }))); AssertThat(ts_stack_version_count(stack), Equals(3)); AssertThat(get_stack_entries(stack, 0), Equals(vector({ @@ -366,7 +366,7 @@ describe("Stack", [&]() { StackSlice slice1 = pop.contents[0]; AssertThat(slice1.version, Equals(1)); - AssertThat(slice1.trees, Equals(vector({ trees[10] }))); + AssertThat(slice1.subtrees, Equals(vector({ subtrees[10] }))); AssertThat(ts_stack_version_count(stack), Equals(2)); AssertThat(ts_stack_state(stack, 0), Equals(stateI)); @@ -388,11 +388,11 @@ describe("Stack", [&]() { StackSlice slice1 = pop.contents[0]; AssertThat(slice1.version, Equals(1)); - AssertThat(slice1.trees, Equals(vector({ trees[1], trees[2], trees[3], trees[10] }))); + AssertThat(slice1.subtrees, Equals(vector({ subtrees[1], subtrees[2], subtrees[3], subtrees[10] }))); StackSlice slice2 = pop.contents[1]; AssertThat(slice2.version, Equals(1)); - AssertThat(slice2.trees, Equals(vector({ trees[4], trees[5], trees[6], trees[10] }))); + AssertThat(slice2.subtrees, Equals(vector({ subtrees[4], subtrees[5], subtrees[6], subtrees[10] }))); AssertThat(ts_stack_version_count(stack), Equals(2)); AssertThat(ts_stack_state(stack, 0), Equals(stateI)); @@ -403,7 +403,7 @@ describe("Stack", [&]() { }); describe("when there are three paths that lead to three different versions", [&]() { - it("returns three entries with different arrays of trees", [&]() { + it("returns three entries with different arrays of subtrees", [&]() { // . <──0── A <──1── B <──2── C <──3── D <──10── I* // ↑ | // ├───4─── E <──5── F <──6───┘ @@ -411,10 +411,10 @@ describe("Stack", [&]() { // └───7─── G <──8── H <──9───┘ StackSliceArray pop = ts_stack_pop_count(stack, 0, 4); free_slice_array(&pool,&pop); - push(1, trees[7], stateG); - push(1, trees[8], stateH); - push(1, trees[9], stateD); - push(1, trees[10], stateI); + push(1, subtrees[7], stateG); + push(1, subtrees[8], stateH); + push(1, subtrees[9], stateD); + push(1, subtrees[10], stateI); ts_stack_merge(stack, 0, 1); AssertThat(ts_stack_version_count(stack), Equals(1)); @@ -443,15 +443,15 @@ describe("Stack", [&]() { StackSlice slice1 = pop.contents[0]; AssertThat(slice1.version, Equals(1)); - AssertThat(slice1.trees, Equals(vector({ trees[3], trees[10] }))); + AssertThat(slice1.subtrees, Equals(vector({ subtrees[3], subtrees[10] }))); StackSlice slice2 = pop.contents[1]; AssertThat(slice2.version, Equals(2)); - AssertThat(slice2.trees, Equals(vector({ trees[6], trees[10] }))); + AssertThat(slice2.subtrees, Equals(vector({ subtrees[6], subtrees[10] }))); StackSlice slice3 = pop.contents[2]; AssertThat(slice3.version, Equals(3)); - AssertThat(slice3.trees, Equals(vector({ trees[9], trees[10] }))); + AssertThat(slice3.subtrees, Equals(vector({ subtrees[9], subtrees[10] }))); AssertThat(ts_stack_version_count(stack), Equals(4)); AssertThat(ts_stack_state(stack, 0), Equals(stateI)); @@ -467,12 +467,12 @@ describe("Stack", [&]() { describe("pop_pending(version)", [&]() { before_each([&]() { - push(0, trees[0], stateA); + push(0, subtrees[0], stateA); }); it("removes the top node from the stack if it was pushed in pending mode", [&]() { - ts_stack_push(stack, 0, trees[1], true, stateB); - ts_tree_retain(trees[1]); + ts_stack_push(stack, 0, subtrees[1], true, stateB); + ts_subtree_retain(subtrees[1]); StackSliceArray pop = ts_stack_pop_pending(stack, 0); AssertThat(pop.size, Equals(1)); @@ -485,20 +485,20 @@ describe("Stack", [&]() { free_slice_array(&pool,&pop); }); - it("skips entries whose trees are extra", [&]() { - ts_stack_push(stack, 0, trees[1], true, stateB); - ts_tree_retain(trees[1]); + it("skips entries whose subtrees are extra", [&]() { + ts_stack_push(stack, 0, subtrees[1], true, stateB); + ts_subtree_retain(subtrees[1]); - trees[2]->extra = true; - trees[3]->extra = true; + subtrees[2]->extra = true; + subtrees[3]->extra = true; - push(0, trees[2], stateB); - push(0, trees[3], stateB); + push(0, subtrees[2], stateB); + push(0, subtrees[3], stateB); StackSliceArray pop = ts_stack_pop_pending(stack, 0); AssertThat(pop.size, Equals(1)); - AssertThat(pop.contents[0].trees, Equals(vector({ trees[1], trees[2], trees[3] }))); + AssertThat(pop.contents[0].subtrees, Equals(vector({ subtrees[1], subtrees[2], subtrees[3] }))); AssertThat(get_stack_entries(stack, 0), Equals(vector({ {stateA, 0}, @@ -509,7 +509,7 @@ describe("Stack", [&]() { }); it("does nothing if the top node was not pushed in pending mode", [&]() { - push(0, trees[1], stateB); + push(0, subtrees[1], stateB); StackSliceArray pop = ts_stack_pop_pending(stack, 0); AssertThat(pop.size, Equals(0)); @@ -526,59 +526,59 @@ describe("Stack", [&]() { describe("setting external token state", [&]() { before_each([&]() { - trees[1]->has_external_tokens = true; - trees[2]->has_external_tokens = true; - ts_external_token_state_init(&trees[1]->external_token_state, NULL, 0); - ts_external_token_state_init(&trees[2]->external_token_state, NULL, 0); + subtrees[1]->has_external_tokens = true; + subtrees[2]->has_external_tokens = true; + ts_external_token_state_init(&subtrees[1]->external_token_state, NULL, 0); + ts_external_token_state_init(&subtrees[2]->external_token_state, NULL, 0); }); it("allows the state to be retrieved", [&]() { - AssertThat(ts_stack_last_external_token(stack, 0), Equals(nullptr)); + AssertThat(ts_stack_last_external_token(stack, 0), Equals(nullptr)); - ts_stack_set_last_external_token(stack, 0, trees[1]); - AssertThat(ts_stack_last_external_token(stack, 0), Equals(trees[1])); + ts_stack_set_last_external_token(stack, 0, subtrees[1]); + AssertThat(ts_stack_last_external_token(stack, 0), Equals(subtrees[1])); ts_stack_copy_version(stack, 0); - AssertThat(ts_stack_last_external_token(stack, 1), Equals(trees[1])); + AssertThat(ts_stack_last_external_token(stack, 1), Equals(subtrees[1])); - ts_stack_set_last_external_token(stack, 0, trees[2]); - AssertThat(ts_stack_last_external_token(stack, 0), Equals(trees[2])); + ts_stack_set_last_external_token(stack, 0, subtrees[2]); + AssertThat(ts_stack_last_external_token(stack, 0), Equals(subtrees[2])); }); it("does not merge stack versions with different external token states", [&]() { - ts_external_token_state_init(&trees[1]->external_token_state, "abcd", 2); - ts_external_token_state_init(&trees[2]->external_token_state, "ABCD", 2); + ts_external_token_state_init(&subtrees[1]->external_token_state, "abcd", 2); + ts_external_token_state_init(&subtrees[2]->external_token_state, "ABCD", 2); ts_stack_copy_version(stack, 0); - push(0, trees[0], 5); - push(1, trees[0], 5); + push(0, subtrees[0], 5); + push(1, subtrees[0], 5); - ts_stack_set_last_external_token(stack, 0, trees[1]); - ts_stack_set_last_external_token(stack, 1, trees[2]); + ts_stack_set_last_external_token(stack, 0, subtrees[1]); + ts_stack_set_last_external_token(stack, 1, subtrees[2]); AssertThat(ts_stack_merge(stack, 0, 1), IsFalse()); }); it("merges stack versions with identical external token states", [&]() { - ts_external_token_state_init(&trees[1]->external_token_state, "abcd", 2); - ts_external_token_state_init(&trees[2]->external_token_state, "abcd", 2); + ts_external_token_state_init(&subtrees[1]->external_token_state, "abcd", 2); + ts_external_token_state_init(&subtrees[2]->external_token_state, "abcd", 2); ts_stack_copy_version(stack, 0); - push(0, trees[0], 5); - push(1, trees[0], 5); + push(0, subtrees[0], 5); + push(1, subtrees[0], 5); - ts_stack_set_last_external_token(stack, 0, trees[1]); - ts_stack_set_last_external_token(stack, 1, trees[2]); + ts_stack_set_last_external_token(stack, 0, subtrees[1]); + ts_stack_set_last_external_token(stack, 1, subtrees[2]); AssertThat(ts_stack_merge(stack, 0, 1), IsTrue()); }); it("does not distinguish between an *empty* external token state and *no* external token state", [&]() { ts_stack_copy_version(stack, 0); - push(0, trees[0], 5); - push(1, trees[0], 5); + push(0, subtrees[0], 5); + push(1, subtrees[0], 5); - ts_stack_set_last_external_token(stack, 0, trees[1]); + ts_stack_set_last_external_token(stack, 0, subtrees[1]); AssertThat(ts_stack_merge(stack, 0, 1), IsTrue()); }); @@ -595,7 +595,7 @@ std::ostream &operator<<(std::ostream &stream, const StackEntry &entry) { return stream << "{" << entry.state << ", " << entry.depth << "}"; } -std::ostream &operator<<(std::ostream &stream, const TreeArray &array) { +std::ostream &operator<<(std::ostream &stream, const SubtreeArray &array) { stream << "["; bool first = true; for (size_t i = 0; i < array.size; i++) { diff --git a/test/runtime/tree_test.cc b/test/runtime/subtree_test.cc similarity index 65% rename from test/runtime/tree_test.cc rename to test/runtime/subtree_test.cc index 1613b930..d56be01d 100644 --- a/test/runtime/tree_test.cc +++ b/test/runtime/subtree_test.cc @@ -1,26 +1,26 @@ #include "test_helper.h" #include "helpers/tree_helpers.h" #include "helpers/point_helpers.h" -#include "runtime/tree.h" +#include "runtime/subtree.h" #include "runtime/length.h" -void assert_consistent(const Tree *tree) { +void assert_consistent(const Subtree *tree) { if (tree->child_count == 0) return; AssertThat(tree->children.contents[0]->padding, Equals(tree->padding)); Length total_children_size = length_zero(); for (size_t i = 0; i < tree->children.size; i++) { - Tree *child = tree->children.contents[i]; + Subtree *child = tree->children.contents[i]; assert_consistent(child); - total_children_size = length_add(total_children_size, ts_tree_total_size(child)); + total_children_size = length_add(total_children_size, ts_subtree_total_size(child)); } - AssertThat(total_children_size, Equals(ts_tree_total_size(tree))); + AssertThat(total_children_size, Equals(ts_subtree_total_size(tree))); }; START_TEST -describe("Tree", []() { +describe("Subtree", []() { enum { symbol1 = 1, symbol2, @@ -38,29 +38,29 @@ describe("Tree", []() { TSLanguage language; language.symbol_metadata = metadata_list; - TreePool pool; + SubtreePool pool; before_each([&]() { - ts_tree_pool_init(&pool); + ts_subtree_pool_init(&pool); }); after_each([&]() { - ts_tree_pool_delete(&pool); + ts_subtree_pool_delete(&pool); }); describe("make_leaf", [&]() { it("does not mark the tree as fragile", [&]() { - Tree *tree = ts_tree_make_leaf(&pool, symbol1, {2, {0, 1}}, {5, {0, 4}}, &language); + Subtree *tree = ts_subtree_make_leaf(&pool, symbol1, {2, {0, 1}}, {5, {0, 4}}, &language); AssertThat(tree->fragile_left, IsFalse()); AssertThat(tree->fragile_right, IsFalse()); - ts_tree_release(&pool, tree); + ts_subtree_release(&pool, tree); }); }); describe("make_error", [&]() { it("marks the tree as fragile", [&]() { - Tree *error_tree = ts_tree_make_error( + Subtree *error_tree = ts_subtree_make_error( &pool, length_zero(), length_zero(), @@ -71,29 +71,29 @@ describe("Tree", []() { AssertThat(error_tree->fragile_left, IsTrue()); AssertThat(error_tree->fragile_right, IsTrue()); - ts_tree_release(&pool, error_tree); + ts_subtree_release(&pool, error_tree); }); }); describe("make_node", [&]() { - Tree *tree1, *tree2, *parent1; + Subtree *tree1, *tree2, *parent1; before_each([&]() { - tree1 = ts_tree_make_leaf(&pool, symbol1, {2, {0, 1}}, {5, {0, 4}}, &language); - tree2 = ts_tree_make_leaf(&pool, symbol2, {1, {0, 1}}, {3, {0, 3}}, &language); + tree1 = ts_subtree_make_leaf(&pool, symbol1, {2, {0, 1}}, {5, {0, 4}}, &language); + tree2 = ts_subtree_make_leaf(&pool, symbol2, {1, {0, 1}}, {3, {0, 3}}, &language); - ts_tree_retain(tree1); - ts_tree_retain(tree2); - parent1 = ts_tree_make_node(&pool, symbol3, tree_array({ + ts_subtree_retain(tree1); + ts_subtree_retain(tree2); + parent1 = ts_subtree_make_node(&pool, symbol3, tree_array({ tree1, tree2, }), 0, &language); }); after_each([&]() { - ts_tree_release(&pool, tree1); - ts_tree_release(&pool, tree2); - ts_tree_release(&pool, parent1); + ts_subtree_release(&pool, tree1); + ts_subtree_release(&pool, tree2); + ts_subtree_release(&pool, parent1); }); it("computes its size and padding based on its child nodes", [&]() { @@ -104,22 +104,22 @@ describe("Tree", []() { }); describe("when the first node is fragile on the left side", [&]() { - Tree *parent; + Subtree *parent; before_each([&]() { tree1->fragile_left = true; tree1->extra = true; - ts_tree_retain(tree1); - ts_tree_retain(tree2); - parent = ts_tree_make_node(&pool, symbol3, tree_array({ + ts_subtree_retain(tree1); + ts_subtree_retain(tree2); + parent = ts_subtree_make_node(&pool, symbol3, tree_array({ tree1, tree2, }), 0, &language); }); after_each([&]() { - ts_tree_release(&pool, parent); + ts_subtree_release(&pool, parent); }); it("records that it is fragile on the left side", [&]() { @@ -128,22 +128,22 @@ describe("Tree", []() { }); describe("when the last node is fragile on the right side", [&]() { - Tree *parent; + Subtree *parent; before_each([&]() { tree2->fragile_right = true; tree2->extra = true; - ts_tree_retain(tree1); - ts_tree_retain(tree2); - parent = ts_tree_make_node(&pool, symbol3, tree_array({ + ts_subtree_retain(tree1); + ts_subtree_retain(tree2); + parent = ts_subtree_make_node(&pool, symbol3, tree_array({ tree1, tree2, }), 0, &language); }); after_each([&]() { - ts_tree_release(&pool, parent); + ts_subtree_release(&pool, parent); }); it("records that it is fragile on the right side", [&]() { @@ -152,22 +152,22 @@ describe("Tree", []() { }); describe("when the outer nodes aren't fragile on their outer side", [&]() { - Tree *parent; + Subtree *parent; before_each([&]() { tree1->fragile_right = true; tree2->fragile_left = true; - ts_tree_retain(tree1); - ts_tree_retain(tree2); - parent = ts_tree_make_node(&pool, symbol3, tree_array({ + ts_subtree_retain(tree1); + ts_subtree_retain(tree2); + parent = ts_subtree_make_node(&pool, symbol3, tree_array({ tree1, tree2, }), 0, &language); }); after_each([&]() { - ts_tree_release(&pool, parent); + ts_subtree_release(&pool, parent); }); it("records that it is not fragile", [&]() { @@ -178,13 +178,13 @@ describe("Tree", []() { }); describe("edit", [&]() { - Tree *tree; + Subtree *tree; before_each([&]() { - tree = ts_tree_make_node(&pool, symbol1, tree_array({ - ts_tree_make_leaf(&pool, symbol2, {2, {0, 2}}, {3, {0, 3}}, &language), - ts_tree_make_leaf(&pool, symbol3, {2, {0, 2}}, {3, {0, 3}}, &language), - ts_tree_make_leaf(&pool, symbol4, {2, {0, 2}}, {3, {0, 3}}, &language), + tree = ts_subtree_make_node(&pool, symbol1, tree_array({ + ts_subtree_make_leaf(&pool, symbol2, {2, {0, 2}}, {3, {0, 3}}, &language), + ts_subtree_make_leaf(&pool, symbol3, {2, {0, 2}}, {3, {0, 3}}, &language), + ts_subtree_make_leaf(&pool, symbol4, {2, {0, 2}}, {3, {0, 3}}, &language), }), 0, &language); AssertThat(tree->padding, Equals({2, {0, 2}})); @@ -192,7 +192,7 @@ describe("Tree", []() { }); after_each([&]() { - ts_tree_release(&pool, tree); + ts_subtree_release(&pool, tree); }); it("does not mutate the argument", [&]() { @@ -204,8 +204,8 @@ describe("Tree", []() { edit.extent_removed = {0, 0}; edit.extent_added = {0, 1}; - ts_tree_retain(tree); - Tree *new_tree = ts_tree_edit(tree, &edit, &pool); + ts_subtree_retain(tree); + Subtree *new_tree = ts_subtree_edit(tree, &edit, &pool); assert_consistent(tree); assert_consistent(new_tree); @@ -221,7 +221,7 @@ describe("Tree", []() { AssertThat(tree->children.contents[1]->padding, Equals({2, {0, 2}})); AssertThat(tree->children.contents[1]->size, Equals({3, {0, 3}})); - ts_tree_release(&pool, new_tree); + ts_subtree_release(&pool, new_tree); }); describe("edits within a tree's padding", [&]() { @@ -233,7 +233,7 @@ describe("Tree", []() { edit.start_point = {0, 1}; edit.extent_removed = {0, 0}; edit.extent_added = {0, 1}; - tree = ts_tree_edit(tree, &edit, &pool); + tree = ts_subtree_edit(tree, &edit, &pool); assert_consistent(tree); AssertThat(tree->has_changes, IsTrue()); @@ -259,7 +259,7 @@ describe("Tree", []() { edit.start_point = {0, 1}; edit.extent_removed = {0, 3}; edit.extent_added = {0, 4}; - tree = ts_tree_edit(tree, &edit, &pool); + tree = ts_subtree_edit(tree, &edit, &pool); assert_consistent(tree); AssertThat(tree->has_changes, IsTrue()); @@ -281,7 +281,7 @@ describe("Tree", []() { edit.start_point = {0, 2}; edit.extent_removed = {0, 0}; edit.extent_added = {0, 2}; - tree = ts_tree_edit(tree, &edit, &pool); + tree = ts_subtree_edit(tree, &edit, &pool); assert_consistent(tree); AssertThat(tree->has_changes, IsTrue()); @@ -305,7 +305,7 @@ describe("Tree", []() { edit.start_point = {0, 2}; edit.extent_removed = {0, 2}; edit.extent_added = {0, 5}; - tree = ts_tree_edit(tree, &edit, &pool); + tree = ts_subtree_edit(tree, &edit, &pool); assert_consistent(tree); AssertThat(tree->has_changes, IsTrue()); @@ -329,7 +329,7 @@ describe("Tree", []() { edit.start_point = {0, 1}; edit.extent_removed = {0, 10}; edit.extent_added = {0, 3}; - tree = ts_tree_edit(tree, &edit, &pool); + tree = ts_subtree_edit(tree, &edit, &pool); assert_consistent(tree); AssertThat(tree->has_changes, IsTrue()); @@ -361,7 +361,7 @@ describe("Tree", []() { edit.start_point = {0, 6}; edit.extent_removed = {0, 1}; edit.extent_added = {0, 1}; - tree = ts_tree_edit(tree, &edit, &pool); + tree = ts_subtree_edit(tree, &edit, &pool); assert_consistent(tree); AssertThat(tree->children.contents[0]->has_changes, IsTrue()); @@ -370,43 +370,43 @@ describe("Tree", []() { }); describe("eq", [&]() { - Tree *leaf; + Subtree *leaf; before_each([&]() { - leaf = ts_tree_make_leaf(&pool, symbol1, {2, {0, 1}}, {5, {0, 4}}, &language); + leaf = ts_subtree_make_leaf(&pool, symbol1, {2, {0, 1}}, {5, {0, 4}}, &language); }); after_each([&]() { - ts_tree_release(&pool, leaf); + ts_subtree_release(&pool, leaf); }); it("returns true for identical trees", [&]() { - Tree *leaf_copy = ts_tree_make_leaf(&pool, symbol1, {2, {1, 1}}, {5, {1, 4}}, &language); - AssertThat(ts_tree_eq(leaf, leaf_copy), IsTrue()); + Subtree *leaf_copy = ts_subtree_make_leaf(&pool, symbol1, {2, {1, 1}}, {5, {1, 4}}, &language); + AssertThat(ts_subtree_eq(leaf, leaf_copy), IsTrue()); - Tree *parent = ts_tree_make_node(&pool, symbol2, tree_array({ + Subtree *parent = ts_subtree_make_node(&pool, symbol2, tree_array({ leaf, leaf_copy, }), 0, &language); - ts_tree_retain(leaf); - ts_tree_retain(leaf_copy); + ts_subtree_retain(leaf); + ts_subtree_retain(leaf_copy); - Tree *parent_copy = ts_tree_make_node(&pool, symbol2, tree_array({ + Subtree *parent_copy = ts_subtree_make_node(&pool, symbol2, tree_array({ leaf, leaf_copy, }), 0, &language); - ts_tree_retain(leaf); - ts_tree_retain(leaf_copy); + ts_subtree_retain(leaf); + ts_subtree_retain(leaf_copy); - AssertThat(ts_tree_eq(parent, parent_copy), IsTrue()); + AssertThat(ts_subtree_eq(parent, parent_copy), IsTrue()); - ts_tree_release(&pool, leaf_copy); - ts_tree_release(&pool, parent); - ts_tree_release(&pool, parent_copy); + ts_subtree_release(&pool, leaf_copy); + ts_subtree_release(&pool, parent); + ts_subtree_release(&pool, parent_copy); }); it("returns false for trees with different symbols", [&]() { - Tree *different_leaf = ts_tree_make_leaf( + Subtree *different_leaf = ts_subtree_make_leaf( &pool, leaf->symbol + 1, leaf->padding, @@ -414,50 +414,50 @@ describe("Tree", []() { &language ); - AssertThat(ts_tree_eq(leaf, different_leaf), IsFalse()); - ts_tree_release(&pool, different_leaf); + AssertThat(ts_subtree_eq(leaf, different_leaf), IsFalse()); + ts_subtree_release(&pool, different_leaf); }); it("returns false for trees with different options", [&]() { - Tree *different_leaf = ts_tree_make_leaf(&pool, leaf->symbol, leaf->padding, leaf->size, &language); + Subtree *different_leaf = ts_subtree_make_leaf(&pool, leaf->symbol, leaf->padding, leaf->size, &language); different_leaf->visible = !leaf->visible; - AssertThat(ts_tree_eq(leaf, different_leaf), IsFalse()); - ts_tree_release(&pool, different_leaf); + AssertThat(ts_subtree_eq(leaf, different_leaf), IsFalse()); + ts_subtree_release(&pool, different_leaf); }); it("returns false for trees with different paddings or sizes", [&]() { - Tree *different_leaf = ts_tree_make_leaf(&pool, leaf->symbol, {}, leaf->size, &language); - AssertThat(ts_tree_eq(leaf, different_leaf), IsFalse()); - ts_tree_release(&pool, different_leaf); + Subtree *different_leaf = ts_subtree_make_leaf(&pool, leaf->symbol, {}, leaf->size, &language); + AssertThat(ts_subtree_eq(leaf, different_leaf), IsFalse()); + ts_subtree_release(&pool, different_leaf); - different_leaf = ts_tree_make_leaf(&pool, symbol1, leaf->padding, {}, &language); - AssertThat(ts_tree_eq(leaf, different_leaf), IsFalse()); - ts_tree_release(&pool, different_leaf); + different_leaf = ts_subtree_make_leaf(&pool, symbol1, leaf->padding, {}, &language); + AssertThat(ts_subtree_eq(leaf, different_leaf), IsFalse()); + ts_subtree_release(&pool, different_leaf); }); it("returns false for trees with different children", [&]() { - Tree *leaf2 = ts_tree_make_leaf(&pool, symbol2, {1, {0, 1}}, {3, {0, 3}}, &language); + Subtree *leaf2 = ts_subtree_make_leaf(&pool, symbol2, {1, {0, 1}}, {3, {0, 3}}, &language); - Tree *parent = ts_tree_make_node(&pool, symbol2, tree_array({ + Subtree *parent = ts_subtree_make_node(&pool, symbol2, tree_array({ leaf, leaf2, }), 0, &language); - ts_tree_retain(leaf); - ts_tree_retain(leaf2); + ts_subtree_retain(leaf); + ts_subtree_retain(leaf2); - Tree *different_parent = ts_tree_make_node(&pool, symbol2, tree_array({ + Subtree *different_parent = ts_subtree_make_node(&pool, symbol2, tree_array({ leaf2, leaf, }), 0, &language); - ts_tree_retain(leaf2); - ts_tree_retain(leaf); + ts_subtree_retain(leaf2); + ts_subtree_retain(leaf); - AssertThat(ts_tree_eq(different_parent, parent), IsFalse()); - AssertThat(ts_tree_eq(parent, different_parent), IsFalse()); + AssertThat(ts_subtree_eq(different_parent, parent), IsFalse()); + AssertThat(ts_subtree_eq(parent, different_parent), IsFalse()); - ts_tree_release(&pool, leaf2); - ts_tree_release(&pool, parent); - ts_tree_release(&pool, different_parent); + ts_subtree_release(&pool, leaf2); + ts_subtree_release(&pool, parent); + ts_subtree_release(&pool, different_parent); }); }); @@ -465,32 +465,32 @@ describe("Tree", []() { Length padding = {1, {0, 1}}; Length size = {2, {0, 2}}; - auto make_external = [](Tree *tree) { + auto make_external = [](Subtree *tree) { tree->has_external_tokens = true; return tree; }; it("returns the last serialized external token state in the given tree", [&]() { - Tree *tree1, *tree2, *tree3, *tree4, *tree5, *tree6, *tree7, *tree8, *tree9; + Subtree *tree1, *tree2, *tree3, *tree4, *tree5, *tree6, *tree7, *tree8, *tree9; - tree1 = ts_tree_make_node(&pool, symbol1, tree_array({ - (tree2 = ts_tree_make_node(&pool, symbol2, tree_array({ - (tree3 = make_external(ts_tree_make_leaf(&pool, symbol3, padding, size, &language))), - (tree4 = ts_tree_make_leaf(&pool, symbol4, padding, size, &language)), - (tree5 = ts_tree_make_leaf(&pool, symbol5, padding, size, &language)), + tree1 = ts_subtree_make_node(&pool, symbol1, tree_array({ + (tree2 = ts_subtree_make_node(&pool, symbol2, tree_array({ + (tree3 = make_external(ts_subtree_make_leaf(&pool, symbol3, padding, size, &language))), + (tree4 = ts_subtree_make_leaf(&pool, symbol4, padding, size, &language)), + (tree5 = ts_subtree_make_leaf(&pool, symbol5, padding, size, &language)), }), 0, &language)), - (tree6 = ts_tree_make_node(&pool, symbol6, tree_array({ - (tree7 = ts_tree_make_node(&pool, symbol7, tree_array({ - (tree8 = ts_tree_make_leaf(&pool, symbol8, padding, size, &language)), + (tree6 = ts_subtree_make_node(&pool, symbol6, tree_array({ + (tree7 = ts_subtree_make_node(&pool, symbol7, tree_array({ + (tree8 = ts_subtree_make_leaf(&pool, symbol8, padding, size, &language)), }), 0, &language)), - (tree9 = ts_tree_make_leaf(&pool, symbol9, padding, size, &language)), + (tree9 = ts_subtree_make_leaf(&pool, symbol9, padding, size, &language)), }), 0, &language)), }), 0, &language); - auto token = ts_tree_last_external_token(tree1); + auto token = ts_subtree_last_external_token(tree1); AssertThat(token, Equals(tree3)); - ts_tree_release(&pool, tree1); + ts_subtree_release(&pool, tree1); }); }); }); diff --git a/tests.gyp b/tests.gyp index eefdf2bf..b964a507 100644 --- a/tests.gyp +++ b/tests.gyp @@ -71,7 +71,7 @@ 'test/runtime/node_test.cc', 'test/runtime/parser_test.cc', 'test/runtime/stack_test.cc', - 'test/runtime/tree_test.cc', + 'test/runtime/subtree_test.cc', 'test/tests.cc', ], 'cflags': [ From 33f76430402f59cccafdd5052b92344e9a070564 Mon Sep 17 00:00:00 2001 From: Max Brunsfeld Date: Thu, 10 May 2018 15:16:24 -0700 Subject: [PATCH 42/90] Rename Parser -> TSParser Co-Authored-By: Rick Winfrey --- src/runtime/document.c | 8 +-- src/runtime/document.h | 2 +- src/runtime/parser.c | 148 +++++++++++++++++++++-------------------- src/runtime/parser.h | 10 +-- 4 files changed, 85 insertions(+), 83 deletions(-) diff --git a/src/runtime/document.c b/src/runtime/document.c index 4298025a..7d175b6a 100644 --- a/src/runtime/document.c +++ b/src/runtime/document.c @@ -12,7 +12,7 @@ TSDocument *ts_document_new() { TSDocument *self = ts_calloc(1, sizeof(TSDocument)); - parser_init(&self->parser); + ts_parser_init(&self->parser); array_init(&self->cursor1.stack); array_init(&self->cursor2.stack); return self; @@ -22,7 +22,7 @@ void ts_document_free(TSDocument *self) { if (self->tree) ts_subtree_release(&self->parser.tree_pool, self->tree); if (self->cursor1.stack.contents) array_delete(&self->cursor1.stack); if (self->cursor2.stack.contents) array_delete(&self->cursor2.stack); - parser_destroy(&self->parser); + ts_parser_destroy(&self->parser); ts_document_set_input(self, (TSInput){ NULL, NULL, @@ -39,7 +39,7 @@ const TSLanguage *ts_document_language(TSDocument *self) { void ts_document_set_language(TSDocument *self, const TSLanguage *language) { if (language->version != TREE_SITTER_LANGUAGE_VERSION) return; ts_document_invalidate(self); - parser_set_language(&self->parser, language); + ts_parser_set_language(&self->parser, language); if (self->tree) { ts_subtree_release(&self->parser.tree_pool, self->tree); self->tree = NULL; @@ -134,7 +134,7 @@ void ts_document_parse_with_options(TSDocument *self, TSParseOptions options) { if (reusable_tree && !reusable_tree->has_changes) return; - Subtree *tree = parser_parse(&self->parser, self->input, reusable_tree, options.halt_on_error); + Subtree *tree = ts_parser_parse(&self->parser, self->input, reusable_tree, options.halt_on_error); if (self->tree) { Subtree *old_tree = self->tree; diff --git a/src/runtime/document.h b/src/runtime/document.h index 9e53963b..a7476ebf 100644 --- a/src/runtime/document.h +++ b/src/runtime/document.h @@ -7,7 +7,7 @@ #include struct TSDocument { - Parser parser; + TSParser parser; TSInput input; Subtree *tree; TSTreeCursor cursor1; diff --git a/src/runtime/parser.c b/src/runtime/parser.c index 8bb7ae31..f3f6fd64 100644 --- a/src/runtime/parser.c +++ b/src/runtime/parser.c @@ -16,7 +16,7 @@ #define LOG(...) \ if (self->lexer.logger.log || self->print_debugging_graphs) { \ snprintf(self->lexer.debug_buffer, TREE_SITTER_SERIALIZATION_BUFFER_SIZE, __VA_ARGS__); \ - parser__log(self); \ + ts_parser__log(self); \ } #define LOG_STACK() \ @@ -25,10 +25,10 @@ fputs("\n\n", stderr); \ } -#define LOG_TREE() \ - if (self->print_debugging_graphs) { \ +#define LOG_TREE() \ + if (self->print_debugging_graphs) { \ ts_subtree_print_dot_graph(self->finished_tree, self->language, stderr); \ - fputs("\n", stderr); \ + fputs("\n", stderr); \ } #define SYM_NAME(symbol) ts_language_symbol_name(self->language, symbol) @@ -52,7 +52,7 @@ typedef enum { ErrorComparisonTakeRight, } ErrorComparison; -static void parser__log(Parser *self) { +static void ts_parser__log(TSParser *self) { if (self->lexer.logger.log) { self->lexer.logger.log( self->lexer.logger.payload, @@ -71,7 +71,7 @@ static void parser__log(Parser *self) { } } -static bool parser__breakdown_top_of_stack(Parser *self, StackVersion version) { +static bool ts_parser__breakdown_top_of_stack(TSParser *self, StackVersion version) { bool did_break_down = false; bool pending = false; @@ -116,7 +116,7 @@ static bool parser__breakdown_top_of_stack(Parser *self, StackVersion version) { return did_break_down; } -static void parser__breakdown_lookahead(Parser *self, Subtree **lookahead, +static void ts_parser__breakdown_lookahead(TSParser *self, Subtree **lookahead, TSStateId state, ReusableNode *reusable_node) { bool did_descend = false; @@ -135,7 +135,7 @@ static void parser__breakdown_lookahead(Parser *self, Subtree **lookahead, } } -static ErrorComparison parser__compare_versions(Parser *self, ErrorStatus a, ErrorStatus b) { +static ErrorComparison ts_parser__compare_versions(TSParser *self, ErrorStatus a, ErrorStatus b) { if (!a.is_in_error && b.is_in_error) { if (a.cost < b.cost) { return ErrorComparisonTakeLeft; @@ -173,7 +173,7 @@ static ErrorComparison parser__compare_versions(Parser *self, ErrorStatus a, Err return ErrorComparisonNone; } -static ErrorStatus parser__version_status(Parser *self, StackVersion version) { +static ErrorStatus ts_parser__version_status(TSParser *self, StackVersion version) { unsigned cost = ts_stack_error_cost(self->stack, version); bool is_paused = ts_stack_is_paused(self->stack, version); if (is_paused) cost += ERROR_COST_PER_SKIPPED_TREE; @@ -185,7 +185,7 @@ static ErrorStatus parser__version_status(Parser *self, StackVersion version) { }; } -static bool parser__better_version_exists(Parser *self, StackVersion version, +static bool ts_parser__better_version_exists(TSParser *self, StackVersion version, bool is_in_error, unsigned cost) { if (self->finished_tree && self->finished_tree->error_cost <= cost) return true; @@ -201,8 +201,8 @@ static bool parser__better_version_exists(Parser *self, StackVersion version, if (i == version || !ts_stack_is_active(self->stack, i) || ts_stack_position(self->stack, i).bytes < position.bytes) continue; - ErrorStatus status_i = parser__version_status(self, i); - switch (parser__compare_versions(self, status, status_i)) { + ErrorStatus status_i = ts_parser__version_status(self, i); + switch (ts_parser__compare_versions(self, status, status_i)) { case ErrorComparisonTakeRight: return true; case ErrorComparisonPreferRight: @@ -215,7 +215,7 @@ static bool parser__better_version_exists(Parser *self, StackVersion version, return false; } -static void parser__restore_external_scanner(Parser *self, Subtree *external_token) { +static void ts_parser__restore_external_scanner(TSParser *self, Subtree *external_token) { if (external_token) { self->language->external_scanner.deserialize( self->external_scanner_payload, @@ -227,7 +227,7 @@ static void parser__restore_external_scanner(Parser *self, Subtree *external_tok } } -static Subtree *parser__lex(Parser *self, StackVersion version, TSStateId parse_state) { +static Subtree *ts_parser__lex(TSParser *self, StackVersion version, TSStateId parse_state) { Length start_position = ts_stack_position(self->stack, version); Subtree *external_token = ts_stack_last_external_token(self->stack, version); TSLexMode lex_mode = self->language->lex_modes[parse_state]; @@ -256,7 +256,7 @@ static Subtree *parser__lex(Parser *self, StackVersion version, TSStateId parse_ current_position.extent.column ); ts_lexer_start(&self->lexer); - parser__restore_external_scanner(self, external_token); + ts_parser__restore_external_scanner(self, external_token); if (self->language->external_scanner.scan( self->external_scanner_payload, &self->lexer.data, @@ -375,7 +375,8 @@ static Subtree *parser__lex(Parser *self, StackVersion version, TSStateId parse_ return result; } -static Subtree *parser__get_cached_token(Parser *self, size_t byte_index, Subtree *last_external_token) { +static Subtree *ts_parser__get_cached_token(TSParser *self, size_t byte_index, + Subtree *last_external_token) { TokenCache *cache = &self->token_cache; if (cache->token && cache->byte_index == byte_index && @@ -386,7 +387,7 @@ static Subtree *parser__get_cached_token(Parser *self, size_t byte_index, Subtre } } -static void parser__set_cached_token(Parser *self, size_t byte_index, Subtree *last_external_token, +static void ts_parser__set_cached_token(TSParser *self, size_t byte_index, Subtree *last_external_token, Subtree *token) { TokenCache *cache = &self->token_cache; if (token) ts_subtree_retain(token); @@ -398,7 +399,7 @@ static void parser__set_cached_token(Parser *self, size_t byte_index, Subtree *l cache->last_external_token = last_external_token; } -static bool parser__can_reuse_first_leaf(Parser *self, TSStateId state, Subtree *tree, +static bool ts_parser__can_reuse_first_leaf(TSParser *self, TSStateId state, Subtree *tree, TableEntry *table_entry) { TSLexMode current_lex_mode = self->language->lex_modes[state]; @@ -416,7 +417,7 @@ static bool parser__can_reuse_first_leaf(Parser *self, TSStateId state, Subtree return current_lex_mode.external_lex_state == 0 && table_entry->is_reusable; } -static Subtree *parser__get_lookahead(Parser *self, StackVersion version, TSStateId *state, +static Subtree *ts_parser__get_lookahead(TSParser *self, StackVersion version, TSStateId *state, ReusableNode *reusable_node, TableEntry *table_entry) { Length position = ts_stack_position(self->stack, version); Subtree *last_external_token = ts_stack_last_external_token(self->stack, version); @@ -458,14 +459,14 @@ static Subtree *parser__get_lookahead(Parser *self, StackVersion version, TSStat LOG("cant_reuse_node_%s tree:%s", reason, SYM_NAME(result->symbol)); if (!reusable_node_descend(reusable_node)) { reusable_node_advance(reusable_node); - parser__breakdown_top_of_stack(self, version); + ts_parser__breakdown_top_of_stack(self, version); *state = ts_stack_state(self->stack, version); } continue; } ts_language_table_entry(self->language, *state, result->first_leaf.symbol, table_entry); - if (!parser__can_reuse_first_leaf(self, *state, result, table_entry)) { + if (!ts_parser__can_reuse_first_leaf(self, *state, result, table_entry)) { LOG( "cant_reuse_node symbol:%s, first_leaf_symbol:%s", SYM_NAME(result->symbol), @@ -480,21 +481,21 @@ static Subtree *parser__get_lookahead(Parser *self, StackVersion version, TSStat return result; } - if ((result = parser__get_cached_token(self, position.bytes, last_external_token))) { + if ((result = ts_parser__get_cached_token(self, position.bytes, last_external_token))) { ts_language_table_entry(self->language, *state, result->first_leaf.symbol, table_entry); - if (parser__can_reuse_first_leaf(self, *state, result, table_entry)) { + if (ts_parser__can_reuse_first_leaf(self, *state, result, table_entry)) { ts_subtree_retain(result); return result; } } - result = parser__lex(self, version, *state); - parser__set_cached_token(self, position.bytes, last_external_token, result); + result = ts_parser__lex(self, version, *state); + ts_parser__set_cached_token(self, position.bytes, last_external_token, result); ts_language_table_entry(self->language, *state, result->symbol, table_entry); return result; } -static bool parser__select_tree(Parser *self, Subtree *left, Subtree *right) { +static bool ts_parser__select_tree(TSParser *self, Subtree *left, Subtree *right) { if (!left) return true; if (!right) return false; @@ -544,7 +545,7 @@ static bool parser__select_tree(Parser *self, Subtree *left, Subtree *right) { } } -static void parser__shift(Parser *self, StackVersion version, TSStateId state, +static void ts_parser__shift(TSParser *self, StackVersion version, TSStateId state, Subtree *lookahead, bool extra) { if (extra != lookahead->extra) { if (ts_stack_version_count(self->stack) > 1) { @@ -566,11 +567,11 @@ static void parser__shift(Parser *self, StackVersion version, TSStateId state, } } -static bool parser__replace_children(Parser *self, Subtree *tree, SubtreeArray *children) { +static bool ts_parser__replace_children(TSParser *self, Subtree *tree, SubtreeArray *children) { self->scratch_tree = *tree; self->scratch_tree.children.size = 0; ts_subtree_set_children(&self->scratch_tree, children, self->language); - if (parser__select_tree(self, tree, &self->scratch_tree)) { + if (ts_parser__select_tree(self, tree, &self->scratch_tree)) { *tree = self->scratch_tree; return true; } else { @@ -578,7 +579,7 @@ static bool parser__replace_children(Parser *self, Subtree *tree, SubtreeArray * } } -static StackSliceArray parser__reduce(Parser *self, StackVersion version, TSSymbol symbol, +static StackSliceArray ts_parser__reduce(TSParser *self, StackVersion version, TSSymbol symbol, uint32_t count, int dynamic_precedence, uint16_t alias_sequence_id, bool fragile) { uint32_t initial_version_count = ts_stack_version_count(self->stack); @@ -614,7 +615,7 @@ static StackSliceArray parser__reduce(Parser *self, StackVersion version, TSSymb children.size--; } - if (parser__replace_children(self, parent, &children)) { + if (ts_parser__replace_children(self, parent, &children)) { ts_subtree_array_delete(&self->tree_pool, &slice.subtrees); slice = next_slice; } else { @@ -669,7 +670,7 @@ static StackSliceArray parser__reduce(Parser *self, StackVersion version, TSSymb return pop; } -static void parser__start(Parser *self, TSInput input, Subtree *previous_tree) { +static void ts_parser__start(TSParser *self, TSInput input, Subtree *previous_tree) { if (previous_tree) { LOG("parse_after_edit"); } else { @@ -688,7 +689,7 @@ static void parser__start(Parser *self, TSInput input, Subtree *previous_tree) { self->in_ambiguity = false; } -static void parser__accept(Parser *self, StackVersion version, Subtree *lookahead) { +static void ts_parser__accept(TSParser *self, StackVersion version, Subtree *lookahead) { lookahead->extra = true; assert(lookahead->symbol == ts_builtin_sym_end); ts_subtree_retain(lookahead); @@ -719,7 +720,7 @@ static void parser__accept(Parser *self, StackVersion version, Subtree *lookahea self->accept_count++; if (self->finished_tree) { - if (parser__select_tree(self, self->finished_tree, root)) { + if (ts_parser__select_tree(self, self->finished_tree, root)) { ts_subtree_release(&self->tree_pool, self->finished_tree); self->finished_tree = root; } else { @@ -734,7 +735,7 @@ static void parser__accept(Parser *self, StackVersion version, Subtree *lookahea ts_stack_halt(self->stack, version); } -static bool parser__do_all_potential_reductions(Parser *self, StackVersion starting_version, +static bool ts_parser__do_all_potential_reductions(TSParser *self, StackVersion starting_version, TSSymbol lookahead_symbol) { uint32_t initial_version_count = ts_stack_version_count(self->stack); @@ -793,7 +794,7 @@ static bool parser__do_all_potential_reductions(Parser *self, StackVersion start for (uint32_t i = 0; i < self->reduce_actions.size; i++) { ReduceAction action = self->reduce_actions.contents[i]; - parser__reduce( + ts_parser__reduce( self, version, action.symbol, action.count, action.dynamic_precedence, action.alias_sequence_id, true @@ -819,10 +820,11 @@ static bool parser__do_all_potential_reductions(Parser *self, StackVersion start return can_shift_lookahead_symbol; } -static void parser__handle_error(Parser *self, StackVersion version, TSSymbol lookahead_symbol) { +static void ts_parser__handle_error(TSParser *self, StackVersion version, + TSSymbol lookahead_symbol) { // Perform any reductions that could have happened in this state, regardless of the lookahead. uint32_t previous_version_count = ts_stack_version_count(self->stack); - parser__do_all_potential_reductions(self, version, 0); + ts_parser__do_all_potential_reductions(self, version, 0); uint32_t version_count = ts_stack_version_count(self->stack); // Push a discontinuity onto the stack. Merge all of the stack versions that @@ -852,7 +854,7 @@ static void parser__handle_error(Parser *self, StackVersion version, TSSymbol lo state_after_missing_symbol ); - if (parser__do_all_potential_reductions( + if (ts_parser__do_all_potential_reductions( self, version_with_missing_tree, lookahead_symbol )) { @@ -880,7 +882,7 @@ static void parser__handle_error(Parser *self, StackVersion version, TSSymbol lo LOG_STACK(); } -static void parser__halt_parse(Parser *self) { +static void ts_parser__halt_parse(TSParser *self) { LOG("halting_parse"); LOG_STACK(); @@ -899,11 +901,11 @@ static void parser__halt_parse(Parser *self) { ts_stack_push(self->stack, 0, root_error, false, 0); Subtree *eof = ts_subtree_make_leaf(&self->tree_pool, ts_builtin_sym_end, length_zero(), length_zero(), self->language); - parser__accept(self, 0, eof); + ts_parser__accept(self, 0, eof); ts_subtree_release(&self->tree_pool, eof); } -static bool parser__recover_to_state(Parser *self, StackVersion version, unsigned depth, +static bool ts_parser__recover_to_state(TSParser *self, StackVersion version, unsigned depth, TSStateId goal_state) { StackSliceArray pop = ts_stack_pop_count(self->stack, version, depth); StackVersion previous_version = STACK_VERSION_NONE; @@ -956,7 +958,7 @@ static bool parser__recover_to_state(Parser *self, StackVersion version, unsigne return previous_version != STACK_VERSION_NONE; } -static void parser__recover(Parser *self, StackVersion version, Subtree *lookahead) { +static void ts_parser__recover(TSParser *self, StackVersion version, Subtree *lookahead) { bool did_recover = false; unsigned previous_version_count = ts_stack_version_count(self->stack); Length position = ts_stack_position(self->stack, version); @@ -991,10 +993,10 @@ static void parser__recover(Parser *self, StackVersion version, Subtree *lookahe entry.depth * ERROR_COST_PER_SKIPPED_TREE + (position.bytes - entry.position.bytes) * ERROR_COST_PER_SKIPPED_CHAR + (position.extent.row - entry.position.extent.row) * ERROR_COST_PER_SKIPPED_LINE; - if (parser__better_version_exists(self, version, false, new_cost)) break; + if (ts_parser__better_version_exists(self, version, false, new_cost)) break; if (ts_language_has_actions(self->language, entry.state, lookahead->symbol)) { - if (parser__recover_to_state(self, version, depth, entry.state)) { + if (ts_parser__recover_to_state(self, version, depth, entry.state)) { did_recover = true; LOG("recover_to_previous state:%u, depth:%u", entry.state, depth); LOG_STACK(); @@ -1020,7 +1022,7 @@ static void parser__recover(Parser *self, StackVersion version, Subtree *lookahe SubtreeArray children = array_new(); Subtree *parent = ts_subtree_make_error_node(&self->tree_pool, &children, self->language); ts_stack_push(self->stack, version, parent, false, 1); - parser__accept(self, version, lookahead); + ts_parser__accept(self, version, lookahead); return; } @@ -1029,7 +1031,7 @@ static void parser__recover(Parser *self, StackVersion version, Subtree *lookahe ts_subtree_total_bytes(lookahead) * ERROR_COST_PER_SKIPPED_CHAR + ts_subtree_total_size(lookahead).extent.row * ERROR_COST_PER_SKIPPED_LINE; - if (parser__better_version_exists(self, version, false, new_cost)) { + if (ts_parser__better_version_exists(self, version, false, new_cost)) { ts_stack_halt(self->stack, version); return; } @@ -1077,10 +1079,10 @@ static void parser__recover(Parser *self, StackVersion version, Subtree *lookahe } } -static void parser__advance(Parser *self, StackVersion version, ReusableNode *reusable_node) { +static void ts_parser__advance(TSParser *self, StackVersion version, ReusableNode *reusable_node) { TSStateId state = ts_stack_state(self->stack, version); TableEntry table_entry; - Subtree *lookahead = parser__get_lookahead(self, version, &state, reusable_node, &table_entry); + Subtree *lookahead = ts_parser__get_lookahead(self, version, &state, reusable_node, &table_entry); for (;;) { StackVersion last_reduction_version = STACK_VERSION_NONE; @@ -1105,11 +1107,11 @@ static void parser__advance(Parser *self, StackVersion version, ReusableNode *re } if (lookahead->children.size > 0) { - parser__breakdown_lookahead(self, &lookahead, state, reusable_node); + ts_parser__breakdown_lookahead(self, &lookahead, state, reusable_node); next_state = ts_language_next_state(self->language, state, lookahead->symbol); } - parser__shift(self, version, next_state, lookahead, action.params.extra); + ts_parser__shift(self, version, next_state, lookahead, action.params.extra); if (lookahead == reusable_node_tree(reusable_node)) { reusable_node_advance(reusable_node); } @@ -1120,7 +1122,7 @@ static void parser__advance(Parser *self, StackVersion version, ReusableNode *re case TSParseActionTypeReduce: { bool is_fragile = table_entry.action_count > 1; LOG("reduce sym:%s, child_count:%u", SYM_NAME(action.params.symbol), action.params.child_count); - StackSliceArray reduction = parser__reduce( + StackSliceArray reduction = ts_parser__reduce( self, version, action.params.symbol, action.params.child_count, action.params.dynamic_precedence, action.params.alias_sequence_id, is_fragile @@ -1132,16 +1134,16 @@ static void parser__advance(Parser *self, StackVersion version, ReusableNode *re case TSParseActionTypeAccept: { LOG("accept"); - parser__accept(self, version, lookahead); + ts_parser__accept(self, version, lookahead); ts_subtree_release(&self->tree_pool, lookahead); return; } case TSParseActionTypeRecover: { while (lookahead->children.size > 0) { - parser__breakdown_lookahead(self, &lookahead, state, reusable_node); + ts_parser__breakdown_lookahead(self, &lookahead, state, reusable_node); } - parser__recover(self, version, lookahead); + ts_parser__recover(self, version, lookahead); if (lookahead == reusable_node_tree(reusable_node)) { reusable_node_advance(reusable_node); } @@ -1155,10 +1157,10 @@ static void parser__advance(Parser *self, StackVersion version, ReusableNode *re ts_stack_renumber_version(self->stack, last_reduction_version, version); LOG_STACK(); } else if (state == ERROR_STATE) { - parser__recover(self, version, lookahead); + ts_parser__recover(self, version, lookahead); ts_subtree_release(&self->tree_pool, lookahead); return; - } else if (!parser__breakdown_top_of_stack(self, version)) { + } else if (!ts_parser__breakdown_top_of_stack(self, version)) { LOG("detect_error"); ts_stack_pause(self->stack, version, lookahead->first_leaf.symbol); ts_subtree_release(&self->tree_pool, lookahead); @@ -1170,7 +1172,7 @@ static void parser__advance(Parser *self, StackVersion version, ReusableNode *re } } -static unsigned parser__condense_stack(Parser *self) { +static unsigned ts_parser__condense_stack(TSParser *self) { bool made_changes = false; unsigned min_error_cost = UINT_MAX; for (StackVersion i = 0; i < ts_stack_version_count(self->stack); i++) { @@ -1180,15 +1182,15 @@ static unsigned parser__condense_stack(Parser *self) { continue; } - ErrorStatus status_i = parser__version_status(self, i); + ErrorStatus status_i = ts_parser__version_status(self, i); if (!status_i.is_in_error && status_i.cost < min_error_cost) { min_error_cost = status_i.cost; } for (StackVersion j = 0; j < i; j++) { - ErrorStatus status_j = parser__version_status(self, j); + ErrorStatus status_j = ts_parser__version_status(self, j); - switch (parser__compare_versions(self, status_j, status_i)) { + switch (ts_parser__compare_versions(self, status_j, status_i)) { case ErrorComparisonTakeLeft: made_changes = true; ts_stack_remove_version(self->stack, i); @@ -1235,7 +1237,7 @@ static unsigned parser__condense_stack(Parser *self) { LOG("resume version:%u", i); min_error_cost = ts_stack_error_cost(self->stack, i); TSSymbol lookahead_symbol = ts_stack_resume(self->stack, i); - parser__handle_error(self, i, lookahead_symbol); + ts_parser__handle_error(self, i, lookahead_symbol); has_unpaused_version = true; } else { ts_stack_remove_version(self->stack, i); @@ -1256,7 +1258,7 @@ static unsigned parser__condense_stack(Parser *self) { return min_error_cost; } -bool parser_init(Parser *self) { +bool ts_parser_init(TSParser *self) { ts_lexer_init(&self->lexer); array_init(&self->reduce_actions); array_reserve(&self->reduce_actions, 4); @@ -1264,11 +1266,11 @@ bool parser_init(Parser *self) { self->stack = ts_stack_new(&self->tree_pool); self->finished_tree = NULL; self->reusable_node = reusable_node_new(); - parser__set_cached_token(self, 0, NULL, NULL); + ts_parser__set_cached_token(self, 0, NULL, NULL); return true; } -void parser_set_language(Parser *self, const TSLanguage *language) { +void ts_parser_set_language(TSParser *self, const TSLanguage *language) { if (self->external_scanner_payload && self->language->external_scanner.destroy) self->language->external_scanner.destroy(self->external_scanner_payload); @@ -1280,18 +1282,18 @@ void parser_set_language(Parser *self, const TSLanguage *language) { self->language = language; } -void parser_destroy(Parser *self) { +void ts_parser_destroy(TSParser *self) { if (self->stack) ts_stack_delete(self->stack); if (self->reduce_actions.contents) array_delete(&self->reduce_actions); ts_subtree_pool_delete(&self->tree_pool); reusable_node_delete(&self->reusable_node); - parser_set_language(self, NULL); + ts_parser_set_language(self, NULL); } -Subtree *parser_parse(Parser *self, TSInput input, Subtree *old_tree, bool halt_on_error) { - parser__start(self, input, old_tree); +Subtree *ts_parser_parse(TSParser *self, TSInput input, Subtree *old_tree, bool halt_on_error) { + ts_parser__start(self, input, old_tree); StackVersion version = STACK_VERSION_NONE; uint32_t position = 0, last_position = 0; @@ -1309,7 +1311,7 @@ Subtree *parser_parse(Parser *self, TSInput input, Subtree *old_tree, bool halt_ ts_stack_position(self->stack, version).extent.row, ts_stack_position(self->stack, version).extent.column); - parser__advance(self, version, &reusable_node); + ts_parser__advance(self, version, &reusable_node); LOG_STACK(); position = ts_stack_position(self->stack, version).bytes; @@ -1322,11 +1324,11 @@ Subtree *parser_parse(Parser *self, TSInput input, Subtree *old_tree, bool halt_ reusable_node_assign(&self->reusable_node, &reusable_node); - unsigned min_error_cost = parser__condense_stack(self); + unsigned min_error_cost = ts_parser__condense_stack(self); if (self->finished_tree && self->finished_tree->error_cost < min_error_cost) { break; } else if (halt_on_error && min_error_cost > 0) { - parser__halt_parse(self); + ts_parser__halt_parse(self); break; } @@ -1335,7 +1337,7 @@ Subtree *parser_parse(Parser *self, TSInput input, Subtree *old_tree, bool halt_ reusable_node_delete(&reusable_node); ts_stack_clear(self->stack); - parser__set_cached_token(self, 0, NULL, NULL); + ts_parser__set_cached_token(self, 0, NULL, NULL); ts_subtree_balance(self->finished_tree, &self->tree_pool, self->language); LOG("done"); diff --git a/src/runtime/parser.h b/src/runtime/parser.h index ff776bc2..46cc7489 100644 --- a/src/runtime/parser.h +++ b/src/runtime/parser.h @@ -32,12 +32,12 @@ typedef struct { bool in_ambiguity; bool print_debugging_graphs; unsigned accept_count; -} Parser; +} TSParser; -bool parser_init(Parser *); -void parser_destroy(Parser *); -Subtree *parser_parse(Parser *, TSInput, Subtree *, bool halt_on_error); -void parser_set_language(Parser *, const TSLanguage *); +bool ts_parser_init(TSParser *); +void ts_parser_destroy(TSParser *); +Subtree *ts_parser_parse(TSParser *, TSInput, Subtree *, bool halt_on_error); +void ts_parser_set_language(TSParser *, const TSLanguage *); #ifdef __cplusplus } From e75ecd1bb125deea42ac1567b6ed442bcdf8bbfb Mon Sep 17 00:00:00 2001 From: Max Brunsfeld Date: Thu, 10 May 2018 22:22:37 -0700 Subject: [PATCH 43/90] Rework API completely --- include/tree_sitter/runtime.h | 49 ++- project.gyp | 2 +- src/runtime/document.c | 188 ---------- src/runtime/document.h | 20 -- src/runtime/get_changed_ranges.c | 34 +- src/runtime/node.c | 22 +- src/runtime/parser.c | 118 +++++-- src/runtime/parser.h | 46 --- src/runtime/reusable_node.h | 2 +- src/runtime/string_input.c | 25 +- src/runtime/string_input.h | 10 +- src/runtime/subtree.c | 17 +- src/runtime/subtree.h | 8 +- src/runtime/tree.c | 59 ++++ src/runtime/tree.h | 19 ++ src/runtime/tree_cursor.c | 32 +- src/runtime/tree_cursor.h | 6 +- test/benchmarks.cc | 39 +-- test/helpers/scope_sequence.cc | 11 +- test/helpers/scope_sequence.h | 2 +- test/integration/fuzzing-examples.cc | 20 +- test/integration/real_grammars.cc | 139 +++++--- test/integration/test_grammars.cc | 18 +- test/runtime/document_test.cc | 490 --------------------------- test/runtime/language_test.cc | 12 +- test/runtime/node_test.cc | 56 +-- test/runtime/parser_test.cc | 266 +++++++++++---- test/runtime/stack_test.cc | 2 +- test/runtime/subtree_test.cc | 2 +- test/runtime/tree_test.cc | 200 +++++++++++ tests.gyp | 2 +- 31 files changed, 841 insertions(+), 1075 deletions(-) delete mode 100644 src/runtime/document.c delete mode 100644 src/runtime/document.h delete mode 100644 src/runtime/parser.h create mode 100644 src/runtime/tree.c create mode 100644 src/runtime/tree.h delete mode 100644 test/runtime/document_test.cc create mode 100644 test/runtime/tree_test.cc diff --git a/include/tree_sitter/runtime.h b/include/tree_sitter/runtime.h index d3e282fb..fa6fd919 100644 --- a/include/tree_sitter/runtime.h +++ b/include/tree_sitter/runtime.h @@ -5,6 +5,7 @@ extern "C" { #endif +#include #include #include #include @@ -13,7 +14,8 @@ extern "C" { typedef unsigned short TSSymbol; typedef struct TSLanguage TSLanguage; -typedef struct TSDocument TSDocument; +typedef struct TSParser TSParser; +typedef struct TSTree TSTree; typedef struct TSTreeCursor TSTreeCursor; typedef enum { @@ -65,17 +67,29 @@ typedef struct { typedef struct { const void *subtree; - const TSDocument *document; + const TSTree *tree; TSPoint position; uint32_t byte; TSSymbol alias_symbol; } TSNode; -typedef struct { - TSRange **changed_ranges; - uint32_t *changed_range_count; - bool halt_on_error; -} TSParseOptions; +TSParser *ts_parser_new(); +void ts_parser_delete(TSParser *); +const TSLanguage *ts_parser_language(const TSParser *); +bool ts_parser_set_language(TSParser *, const TSLanguage *); +TSLogger ts_parser_logger(const TSParser *); +void ts_parser_set_logger(TSParser *, TSLogger); +void ts_parser_print_debugging_graphs(TSParser *, bool); +void ts_parser_halt_on_error(TSParser *, bool); +TSTree *ts_parser_parse(TSParser *, const TSTree *, TSInput); +TSTree *ts_parser_parse_string(TSParser *, const TSTree *, const char *, uint32_t); + +TSTree *ts_tree_copy(const TSTree *); +void ts_tree_delete(const TSTree *); +TSNode ts_tree_root_node(const TSTree *); +void ts_tree_edit(TSTree *, const TSInputEdit *); +TSRange *ts_tree_get_changed_ranges(const TSTree *, const TSTree *, uint32_t *); +void ts_tree_print_dot_graph(const TSTree *, FILE *); uint32_t ts_node_start_byte(TSNode); TSPoint ts_node_start_point(TSNode); @@ -105,26 +119,7 @@ TSNode ts_node_named_descendant_for_byte_range(TSNode, uint32_t, uint32_t); TSNode ts_node_descendant_for_point_range(TSNode, TSPoint, TSPoint); TSNode ts_node_named_descendant_for_point_range(TSNode, TSPoint, TSPoint); -TSDocument *ts_document_new(); -void ts_document_free(TSDocument *); -const TSLanguage *ts_document_language(TSDocument *); -void ts_document_set_language(TSDocument *, const TSLanguage *); -TSInput ts_document_input(TSDocument *); -void ts_document_set_input(TSDocument *, TSInput); -void ts_document_set_input_string(TSDocument *, const char *); -void ts_document_set_input_string_with_length(TSDocument *, const char *, uint32_t); -TSLogger ts_document_logger(const TSDocument *); -void ts_document_set_logger(TSDocument *, TSLogger); -void ts_document_print_debugging_graphs(TSDocument *, bool); -void ts_document_edit(TSDocument *, TSInputEdit); -void ts_document_parse(TSDocument *); -void ts_document_parse_and_get_changed_ranges(TSDocument *, TSRange **, uint32_t *); -void ts_document_parse_with_options(TSDocument *, TSParseOptions); -void ts_document_invalidate(TSDocument *); -TSNode ts_document_root_node(const TSDocument *); -TSTreeCursor *ts_document_tree_cursor(const TSDocument *); -uint32_t ts_document_parse_count(const TSDocument *); - +TSTreeCursor *ts_tree_cursor_new(const TSTree *); void ts_tree_cursor_delete(TSTreeCursor *); bool ts_tree_cursor_goto_first_child(TSTreeCursor *); bool ts_tree_cursor_goto_next_sibling(TSTreeCursor *); diff --git a/project.gyp b/project.gyp index cfc1776b..11174724 100644 --- a/project.gyp +++ b/project.gyp @@ -87,7 +87,6 @@ 'externals/utf8proc', ], 'sources': [ - 'src/runtime/document.c', 'src/runtime/get_changed_ranges.c', 'src/runtime/language.c', 'src/runtime/lexer.c', @@ -96,6 +95,7 @@ 'src/runtime/parser.c', 'src/runtime/string_input.c', 'src/runtime/subtree.c', + 'src/runtime/tree.c', 'src/runtime/tree_cursor.c', 'src/runtime/utf16.c', 'externals/utf8proc/utf8proc.c', diff --git a/src/runtime/document.c b/src/runtime/document.c deleted file mode 100644 index 7d175b6a..00000000 --- a/src/runtime/document.c +++ /dev/null @@ -1,188 +0,0 @@ -#include "runtime/alloc.h" -#include "runtime/subtree.h" -#include "runtime/parser.h" -#include "runtime/string_input.h" -#include "runtime/document.h" -#include "runtime/tree_cursor.h" -#include "runtime/get_changed_ranges.h" - -#define LOG(...) \ - snprintf(self->parser.lexer.debug_buffer, TREE_SITTER_SERIALIZATION_BUFFER_SIZE, __VA_ARGS__); \ - self->parser.lexer.logger.log(self->parser.lexer.logger.payload, TSLogTypeLex, self->parser.lexer.debug_buffer); \ - -TSDocument *ts_document_new() { - TSDocument *self = ts_calloc(1, sizeof(TSDocument)); - ts_parser_init(&self->parser); - array_init(&self->cursor1.stack); - array_init(&self->cursor2.stack); - return self; -} - -void ts_document_free(TSDocument *self) { - if (self->tree) ts_subtree_release(&self->parser.tree_pool, self->tree); - if (self->cursor1.stack.contents) array_delete(&self->cursor1.stack); - if (self->cursor2.stack.contents) array_delete(&self->cursor2.stack); - ts_parser_destroy(&self->parser); - ts_document_set_input(self, (TSInput){ - NULL, - NULL, - NULL, - TSInputEncodingUTF8, - }); - ts_free(self); -} - -const TSLanguage *ts_document_language(TSDocument *self) { - return self->parser.language; -} - -void ts_document_set_language(TSDocument *self, const TSLanguage *language) { - if (language->version != TREE_SITTER_LANGUAGE_VERSION) return; - ts_document_invalidate(self); - ts_parser_set_language(&self->parser, language); - if (self->tree) { - ts_subtree_release(&self->parser.tree_pool, self->tree); - self->tree = NULL; - } -} - -TSLogger ts_document_logger(const TSDocument *self) { - return self->parser.lexer.logger; -} - -void ts_document_set_logger(TSDocument *self, TSLogger logger) { - self->parser.lexer.logger = logger; -} - -void ts_document_print_debugging_graphs(TSDocument *self, bool should_print) { - self->parser.print_debugging_graphs = should_print; -} - -TSInput ts_document_input(TSDocument *self) { - return self->input; -} - -void ts_document_set_input(TSDocument *self, TSInput input) { - if (self->owns_input) - ts_free(self->input.payload); - self->input = input; - self->owns_input = false; -} - -void ts_document_set_input_string(TSDocument *self, const char *text) { - ts_document_invalidate(self); - TSInput input = ts_string_input_make(text); - ts_document_set_input(self, input); - if (input.payload) { - self->owns_input = true; - } -} - -void ts_document_set_input_string_with_length(TSDocument *self, const char *text, uint32_t length) { - ts_document_invalidate(self); - TSInput input = ts_string_input_make_with_length(text, length); - ts_document_set_input(self, input); - if (input.payload) { - self->owns_input = true; - } -} - -void ts_document_edit(TSDocument *self, TSInputEdit edit) { - if (!self->tree) - return; - - uint32_t max_bytes = ts_subtree_total_bytes(self->tree); - if (edit.start_byte > max_bytes) - return; - if (edit.bytes_removed > max_bytes - edit.start_byte) - edit.bytes_removed = max_bytes - edit.start_byte; - - self->tree = ts_subtree_edit(self->tree, &edit, &self->parser.tree_pool); - - if (self->parser.print_debugging_graphs) { - ts_subtree_print_dot_graph(self->tree, self->parser.language, stderr); - } -} - -void ts_document_parse(TSDocument *self) { - ts_document_parse_with_options(self, (TSParseOptions){ - .halt_on_error = false, - .changed_ranges = NULL, - .changed_range_count = NULL, - }); -} - -void ts_document_parse_and_get_changed_ranges(TSDocument *self, TSRange **ranges, - uint32_t *range_count) { - ts_document_parse_with_options(self, (TSParseOptions){ - .halt_on_error = false, - .changed_ranges = ranges, - .changed_range_count = range_count, - }); -} - -void ts_document_parse_with_options(TSDocument *self, TSParseOptions options) { - if (options.changed_ranges && options.changed_range_count) { - *options.changed_ranges = NULL; - *options.changed_range_count = 0; - } - - if (!self->input.read || !self->parser.language) - return; - - Subtree *reusable_tree = self->valid ? self->tree : NULL; - if (reusable_tree && !reusable_tree->has_changes) - return; - - Subtree *tree = ts_parser_parse(&self->parser, self->input, reusable_tree, options.halt_on_error); - - if (self->tree) { - Subtree *old_tree = self->tree; - self->tree = tree; - - if (options.changed_ranges && options.changed_range_count) { - *options.changed_range_count = ts_subtree_get_changed_ranges( - old_tree, tree, &self->cursor1, &self->cursor2, - self->parser.language, options.changed_ranges - ); - - if (self->parser.lexer.logger.log) { - for (unsigned i = 0; i < *options.changed_range_count; i++) { - TSRange range = (*options.changed_ranges)[i]; - LOG( - "changed_range start:[%u %u], end:[%u %u]", - range.start.row, range.start.column, - range.end.row, range.end.column - ); - } - } - } - - ts_subtree_release(&self->parser.tree_pool, old_tree); - } - - self->tree = tree; - self->parse_count++; - self->valid = true; -} - -void ts_document_invalidate(TSDocument *self) { - self->valid = false; -} - -TSNode ts_document_root_node(const TSDocument *self) { - return (TSNode) { - .subtree = self->tree, - .document = self, - .position = {0, 0}, - .byte = 0, - }; -} - -uint32_t ts_document_parse_count(const TSDocument *self) { - return self->parse_count; -} - -TSTreeCursor *ts_document_tree_cursor(const TSDocument *self) { - return ts_tree_cursor_new(self); -} diff --git a/src/runtime/document.h b/src/runtime/document.h deleted file mode 100644 index a7476ebf..00000000 --- a/src/runtime/document.h +++ /dev/null @@ -1,20 +0,0 @@ -#ifndef RUNTIME_DOCUMENT_H_ -#define RUNTIME_DOCUMENT_H_ - -#include "runtime/parser.h" -#include "runtime/subtree.h" -#include "runtime/tree_cursor.h" -#include - -struct TSDocument { - TSParser parser; - TSInput input; - Subtree *tree; - TSTreeCursor cursor1; - TSTreeCursor cursor2; - size_t parse_count; - bool valid; - bool owns_input; -}; - -#endif diff --git a/src/runtime/get_changed_ranges.c b/src/runtime/get_changed_ranges.c index 76e01f29..032fdaab 100644 --- a/src/runtime/get_changed_ranges.c +++ b/src/runtime/get_changed_ranges.c @@ -34,7 +34,7 @@ typedef struct { static Iterator iterator_new(TSTreeCursor *cursor, Subtree *tree, const TSLanguage *language) { array_clear(&cursor->stack); array_push(&cursor->stack, ((TreeCursorEntry){ - .tree = tree, + .subtree = tree, .position = length_zero(), .child_index = 0, .structural_child_index = 0, @@ -56,25 +56,25 @@ Length iterator_start_position(Iterator *self) { if (self->in_padding) { return entry.position; } else { - return length_add(entry.position, entry.tree->padding); + return length_add(entry.position, entry.subtree->padding); } } Length iterator_end_position(Iterator *self) { TreeCursorEntry entry = *array_back(&self->cursor.stack); - Length result = length_add(entry.position, entry.tree->padding); + Length result = length_add(entry.position, entry.subtree->padding); if (self->in_padding) { return result; } else { - return length_add(result, entry.tree->size); + return length_add(result, entry.subtree->size); } } static bool iterator_tree_is_visible(const Iterator *self) { TreeCursorEntry entry = *array_back(&self->cursor.stack); - if (entry.tree->visible) return true; + if (entry.subtree->visible) return true; if (self->cursor.stack.size > 1) { - Subtree *parent = self->cursor.stack.contents[self->cursor.stack.size - 2].tree; + Subtree *parent = self->cursor.stack.contents[self->cursor.stack.size - 2].subtree; const TSSymbol *alias_sequence = ts_language_alias_sequence(self->language, parent->alias_sequence_id); return alias_sequence && alias_sequence[entry.structural_child_index] != 0; } @@ -94,7 +94,7 @@ static void iterator_get_visible_state(const Iterator *self, Subtree **tree, TreeCursorEntry entry = self->cursor.stack.contents[i]; if (i > 0) { - Subtree *parent = self->cursor.stack.contents[i - 1].tree; + Subtree *parent = self->cursor.stack.contents[i - 1].subtree; const TSSymbol *alias_sequence = ts_language_alias_sequence( self->language, parent->alias_sequence_id @@ -104,8 +104,8 @@ static void iterator_get_visible_state(const Iterator *self, Subtree **tree, } } - if (entry.tree->visible || *alias_symbol) { - *tree = entry.tree; + if (entry.subtree->visible || *alias_symbol) { + *tree = entry.subtree; *start_byte = entry.position.bytes; break; } @@ -128,14 +128,14 @@ static bool iterator_descend(Iterator *self, uint32_t goal_position) { TreeCursorEntry entry = *array_back(&self->cursor.stack); Length position = entry.position; uint32_t structural_child_index = 0; - for (uint32_t i = 0; i < entry.tree->children.size; i++) { - Subtree *child = entry.tree->children.contents[i]; + for (uint32_t i = 0; i < entry.subtree->children.size; i++) { + Subtree *child = entry.subtree->children.contents[i]; Length child_left = length_add(position, child->padding); Length child_right = length_add(child_left, child->size); if (child_right.bytes > goal_position) { array_push(&self->cursor.stack, ((TreeCursorEntry){ - .tree = child, + .subtree = child, .position = position, .child_index = i, .structural_child_index = structural_child_index, @@ -178,16 +178,16 @@ static void iterator_advance(Iterator *self) { TreeCursorEntry entry = array_pop(&self->cursor.stack); if (iterator_done(self)) return; - Subtree *parent = array_back(&self->cursor.stack)->tree; + Subtree *parent = array_back(&self->cursor.stack)->subtree; uint32_t child_index = entry.child_index + 1; if (parent->children.size > child_index) { - Length position = length_add(entry.position, ts_subtree_total_size(entry.tree)); + Length position = length_add(entry.position, ts_subtree_total_size(entry.subtree)); uint32_t structural_child_index = entry.structural_child_index; - if (!entry.tree->extra) structural_child_index++; + if (!entry.subtree->extra) structural_child_index++; Subtree *next_child = parent->children.contents[child_index]; array_push(&self->cursor.stack, ((TreeCursorEntry){ - .tree = next_child, + .subtree = next_child, .position = position, .child_index = child_index, .structural_child_index = structural_child_index, @@ -250,7 +250,7 @@ static inline void iterator_print_state(Iterator *self) { TreeCursorEntry entry = *array_back(&self->cursor.stack); TSPoint start = iterator_start_position(self).extent; TSPoint end = iterator_end_position(self).extent; - const char *name = ts_language_symbol_name(self->language, entry.tree->symbol); + const char *name = ts_language_symbol_name(self->language, entry.subtree->symbol); printf( "(%-25s %s\t depth:%u [%u, %u] - [%u, %u])", name, self->in_padding ? "(p)" : " ", diff --git a/src/runtime/node.c b/src/runtime/node.c index 4df3400a..e260f13b 100644 --- a/src/runtime/node.c +++ b/src/runtime/node.c @@ -1,13 +1,13 @@ #include #include "runtime/subtree.h" -#include "runtime/document.h" +#include "runtime/tree.h" #include "runtime/language.h" // NodeChildIterator typedef struct { const Subtree *parent; - const TSDocument *document; + const TSTree *tree; Length position; uint32_t child_index; uint32_t structural_child_index; @@ -19,7 +19,7 @@ typedef struct { static inline TSNode ts_node__null() { return (TSNode) { .subtree = NULL, - .document = NULL, + .tree = NULL, .position = {0, 0}, .byte = 0, }; @@ -32,12 +32,12 @@ static inline const Subtree *ts_node__tree(TSNode self) { static inline NodeChildIterator ts_node_child_iterator_begin(const TSNode *node) { const Subtree *tree = ts_node__tree(*node); const TSSymbol *alias_sequence = ts_language_alias_sequence( - node->document->parser.language, + node->tree->language, tree->alias_sequence_id ); return (NodeChildIterator) { .parent = tree, - .document = node->document, + .tree = node->tree, .position = {node->byte, node->position}, .child_index = 0, .structural_child_index = 0, @@ -57,7 +57,7 @@ static inline bool ts_node_child_iterator_next(NodeChildIterator *self, TSNode * } *result = (TSNode) { .subtree = child, - .document = self->document, + .tree = self->tree, .position = self->position.extent, .byte = self->position.bytes, .alias_symbol = alias_symbol, @@ -77,7 +77,7 @@ static inline bool ts_node__is_relevant(TSNode self, bool include_anonymous) { ( self.alias_symbol && ts_language_symbol_metadata( - self.document->parser.language, + self.tree->language, self.alias_symbol ).named ) @@ -343,11 +343,11 @@ TSSymbol ts_node_symbol(TSNode self) { } const char *ts_node_type(TSNode self) { - return ts_language_symbol_name(self.document->parser.language, ts_node_symbol(self)); + return ts_language_symbol_name(self.tree->language, ts_node_symbol(self)); } char *ts_node_string(TSNode self) { - return ts_subtree_string(ts_node__tree(self), self.document->parser.language, false); + return ts_subtree_string(ts_node__tree(self), self.tree->language, false); } bool ts_node_eq(TSNode self, TSNode other) { @@ -360,7 +360,7 @@ bool ts_node_eq(TSNode self, TSNode other) { bool ts_node_is_named(TSNode self) { const Subtree *tree = ts_node__tree(self); return self.alias_symbol - ? ts_language_symbol_metadata(self.document->parser.language, self.alias_symbol).named + ? ts_language_symbol_metadata(self.tree->language, self.alias_symbol).named : tree->named; } @@ -378,7 +378,7 @@ bool ts_node_has_error(TSNode self) { } TSNode ts_node_parent(TSNode self) { - TSNode node = ts_document_root_node(self.document); + TSNode node = ts_tree_root_node(self.tree); uint32_t end_byte = ts_node_end_byte(self); if (node.subtree == self.subtree) return ts_node__null(); diff --git a/src/runtime/parser.c b/src/runtime/parser.c index f3f6fd64..1c7da6b1 100644 --- a/src/runtime/parser.c +++ b/src/runtime/parser.c @@ -1,4 +1,3 @@ -#include "runtime/parser.h" #include #include #include @@ -10,8 +9,12 @@ #include "runtime/array.h" #include "runtime/language.h" #include "runtime/alloc.h" +#include "runtime/stack.h" +#include "runtime/reusable_node.h" #include "runtime/reduce_action.h" #include "runtime/error_costs.h" +#include "runtime/string_input.h" +#include "runtime/tree.h" #define LOG(...) \ if (self->lexer.logger.log || self->print_debugging_graphs) { \ @@ -37,6 +40,29 @@ static const unsigned MAX_VERSION_COUNT = 6; static const unsigned MAX_SUMMARY_DEPTH = 16; static const unsigned MAX_COST_DIFFERENCE = 16 * ERROR_COST_PER_SKIPPED_TREE; +typedef struct { + Subtree *token; + Subtree *last_external_token; + uint32_t byte_index; +} TokenCache; + +struct TSParser { + Lexer lexer; + Stack *stack; + SubtreePool tree_pool; + const TSLanguage *language; + ReduceActionSet reduce_actions; + Subtree *finished_tree; + Subtree scratch_tree; + TokenCache token_cache; + ReusableNode reusable_node; + void *external_scanner_payload; + bool in_ambiguity; + bool print_debugging_graphs; + bool halt_on_error; + unsigned accept_count; +}; + typedef struct { unsigned cost; unsigned node_count; @@ -52,6 +78,8 @@ typedef enum { ErrorComparisonTakeRight, } ErrorComparison; +// Parser - Private + static void ts_parser__log(TSParser *self) { if (self->lexer.logger.log) { self->lexer.logger.log( @@ -670,7 +698,7 @@ static StackSliceArray ts_parser__reduce(TSParser *self, StackVersion version, T return pop; } -static void ts_parser__start(TSParser *self, TSInput input, Subtree *previous_tree) { +static void ts_parser__start(TSParser *self, TSInput input, const Subtree *previous_tree) { if (previous_tree) { LOG("parse_after_edit"); } else { @@ -1258,42 +1286,76 @@ static unsigned ts_parser__condense_stack(TSParser *self) { return min_error_cost; } -bool ts_parser_init(TSParser *self) { +// Parser - Public + +TSParser *ts_parser_new() { + TSParser *self = ts_calloc(1, sizeof(TSParser)); ts_lexer_init(&self->lexer); array_init(&self->reduce_actions); array_reserve(&self->reduce_actions, 4); - ts_subtree_pool_init(&self->tree_pool); + self->tree_pool = ts_subtree_pool_new(32); self->stack = ts_stack_new(&self->tree_pool); self->finished_tree = NULL; self->reusable_node = reusable_node_new(); + self->print_debugging_graphs = false; + self->halt_on_error = false; ts_parser__set_cached_token(self, 0, NULL, NULL); - return true; + return self; } -void ts_parser_set_language(TSParser *self, const TSLanguage *language) { - if (self->external_scanner_payload && self->language->external_scanner.destroy) - self->language->external_scanner.destroy(self->external_scanner_payload); - - if (language && language->external_scanner.create) - self->external_scanner_payload = language->external_scanner.create(); - else - self->external_scanner_payload = NULL; - - self->language = language; -} - -void ts_parser_destroy(TSParser *self) { - if (self->stack) +void ts_parser_delete(TSParser *self) { + if (self->stack) { ts_stack_delete(self->stack); - if (self->reduce_actions.contents) + } + if (self->reduce_actions.contents) { array_delete(&self->reduce_actions); + } ts_subtree_pool_delete(&self->tree_pool); reusable_node_delete(&self->reusable_node); ts_parser_set_language(self, NULL); + ts_free(self); } -Subtree *ts_parser_parse(TSParser *self, TSInput input, Subtree *old_tree, bool halt_on_error) { - ts_parser__start(self, input, old_tree); +const TSLanguage *ts_parser_language(const TSParser *self) { + return self->language; +} + +bool ts_parser_set_language(TSParser *self, const TSLanguage *language) { + if (language && language->version != TREE_SITTER_LANGUAGE_VERSION) return false; + + if (self->external_scanner_payload && self->language->external_scanner.destroy) { + self->language->external_scanner.destroy(self->external_scanner_payload); + } + + if (language && language->external_scanner.create) { + self->external_scanner_payload = language->external_scanner.create(); + } else { + self->external_scanner_payload = NULL; + } + + self->language = language; + return true; +} + +TSLogger ts_parser_logger(const TSParser *self) { + return self->lexer.logger; +} + +void ts_parser_set_logger(TSParser *self, TSLogger logger) { + self->lexer.logger = logger; +} + +void ts_parser_print_debugging_graphs(TSParser *self, bool should_print) { + self->print_debugging_graphs = should_print; +} + +void ts_parser_halt_on_error(TSParser *self, bool should_halt_on_error) { + self->halt_on_error = should_halt_on_error; +} + +TSTree *ts_parser_parse(TSParser *self, const TSTree *old_tree, TSInput input) { + if (!self->language) return NULL; + ts_parser__start(self, input, old_tree ? old_tree->root : NULL); StackVersion version = STACK_VERSION_NONE; uint32_t position = 0, last_position = 0; @@ -1327,7 +1389,7 @@ Subtree *ts_parser_parse(TSParser *self, TSInput input, Subtree *old_tree, bool unsigned min_error_cost = ts_parser__condense_stack(self); if (self->finished_tree && self->finished_tree->error_cost < min_error_cost) { break; - } else if (halt_on_error && min_error_cost > 0) { + } else if (self->halt_on_error && min_error_cost > 0) { ts_parser__halt_parse(self); break; } @@ -1342,5 +1404,13 @@ Subtree *ts_parser_parse(TSParser *self, TSInput input, Subtree *old_tree, bool LOG("done"); LOG_TREE(); - return self->finished_tree; + + return ts_tree_new(self->finished_tree, self->language); +} + +TSTree *ts_parser_parse_string(TSParser *self, const TSTree *old_tree, + const char *string, uint32_t length) { + TSStringInput input; + ts_string_input_init(&input, string, length); + return ts_parser_parse(self, old_tree, input.input); } diff --git a/src/runtime/parser.h b/src/runtime/parser.h deleted file mode 100644 index 46cc7489..00000000 --- a/src/runtime/parser.h +++ /dev/null @@ -1,46 +0,0 @@ -#ifndef RUNTIME_PARSER_H_ -#define RUNTIME_PARSER_H_ - -#ifdef __cplusplus -extern "C" { -#endif - -#include "runtime/stack.h" -#include "runtime/array.h" -#include "runtime/lexer.h" -#include "runtime/reusable_node.h" -#include "runtime/reduce_action.h" -#include "runtime/subtree.h" - -typedef struct { - Subtree *token; - Subtree *last_external_token; - uint32_t byte_index; -} TokenCache; - -typedef struct { - Lexer lexer; - Stack *stack; - SubtreePool tree_pool; - const TSLanguage *language; - ReduceActionSet reduce_actions; - Subtree *finished_tree; - Subtree scratch_tree; - TokenCache token_cache; - ReusableNode reusable_node; - void *external_scanner_payload; - bool in_ambiguity; - bool print_debugging_graphs; - unsigned accept_count; -} TSParser; - -bool ts_parser_init(TSParser *); -void ts_parser_destroy(TSParser *); -Subtree *ts_parser_parse(TSParser *, TSInput, Subtree *, bool halt_on_error); -void ts_parser_set_language(TSParser *, const TSLanguage *); - -#ifdef __cplusplus -} -#endif - -#endif // RUNTIME_PARSER_H_ diff --git a/src/runtime/reusable_node.h b/src/runtime/reusable_node.h index 5f1884d5..c1d4f06b 100644 --- a/src/runtime/reusable_node.h +++ b/src/runtime/reusable_node.h @@ -24,7 +24,7 @@ static inline void reusable_node_reset(ReusableNode *self, Subtree *tree) { })); } -static inline Subtree *reusable_node_tree(ReusableNode *self) { +static inline const Subtree *reusable_node_tree(ReusableNode *self) { return self->stack.size > 0 ? self->stack.contents[self->stack.size - 1].tree : NULL; diff --git a/src/runtime/string_input.c b/src/runtime/string_input.c index 53f69ee5..c4e13e0c 100644 --- a/src/runtime/string_input.c +++ b/src/runtime/string_input.c @@ -1,13 +1,7 @@ +#include "tree_sitter/runtime.h" #include "runtime/string_input.h" -#include "runtime/alloc.h" #include -typedef struct { - const char *string; - uint32_t position; - uint32_t length; -} TSStringInput; - static const char *ts_string_input__read(void *payload, uint32_t *bytes_read) { TSStringInput *input = (TSStringInput *)payload; if (input->position >= input->length) { @@ -26,17 +20,12 @@ static int ts_string_input__seek(void *payload, uint32_t byte, TSPoint _) { return (byte < input->length); } -TSInput ts_string_input_make(const char *string) { - return ts_string_input_make_with_length(string, strlen(string)); -} - -TSInput ts_string_input_make_with_length(const char *string, uint32_t length) { - TSStringInput *input = ts_malloc(sizeof(TSStringInput)); - input->string = string; - input->position = 0; - input->length = length; - return (TSInput){ - .payload = input, +void ts_string_input_init(TSStringInput *self, const char *string, uint32_t length) { + self->string = string; + self->position = 0; + self->length = length; + self->input = (TSInput) { + .payload = self, .read = ts_string_input__read, .seek = ts_string_input__seek, .encoding = TSInputEncodingUTF8, diff --git a/src/runtime/string_input.h b/src/runtime/string_input.h index c96cd416..19171e4f 100644 --- a/src/runtime/string_input.h +++ b/src/runtime/string_input.h @@ -7,8 +7,14 @@ extern "C" { #include "tree_sitter/runtime.h" -TSInput ts_string_input_make(const char *); -TSInput ts_string_input_make_with_length(const char *, uint32_t); +typedef struct { + const char *string; + uint32_t position; + uint32_t length; + TSInput input; +} TSStringInput; + +void ts_string_input_init(TSStringInput *, const char *, uint32_t); #ifdef __cplusplus } diff --git a/src/runtime/subtree.c b/src/runtime/subtree.c index 1808ee6d..aff2a6fc 100644 --- a/src/runtime/subtree.c +++ b/src/runtime/subtree.c @@ -19,6 +19,10 @@ typedef struct { TSStateId TS_TREE_STATE_NONE = USHRT_MAX; +static const uint32_t MAX_TREE_POOL_SIZE = 1024; + +static const TSExternalTokenState empty_state = {.length = 0, .short_data = {0}}; + // ExternalTokenState void ts_external_token_state_init(TSExternalTokenState *self, const char *content, unsigned length) { @@ -102,11 +106,10 @@ void ts_subtree_array_reverse(SubtreeArray *self) { // SubtreePool -static const uint32_t MAX_TREE_POOL_SIZE = 1024; - -void ts_subtree_pool_init(SubtreePool *self) { - array_init(&self->free_trees); - array_init(&self->tree_stack); +SubtreePool ts_subtree_pool_new(uint32_t capacity) { + SubtreePool self = {array_new(), array_new()}; + array_reserve(&self.free_trees, capacity); + return self; } void ts_subtree_pool_delete(SubtreePool *self) { @@ -128,7 +131,7 @@ Subtree *ts_subtree_pool_allocate(SubtreePool *self) { } void ts_subtree_pool_free(SubtreePool *self, Subtree *tree) { - if (self->free_trees.size < MAX_TREE_POOL_SIZE) { + if (self->free_trees.capacity > 0 && self->free_trees.size < MAX_TREE_POOL_SIZE) { array_push(&self->free_trees, tree); } else { ts_free(tree); @@ -691,8 +694,6 @@ void ts_subtree_print_dot_graph(const Subtree *self, const TSLanguage *language, fprintf(f, "}\n"); } -static const TSExternalTokenState empty_state = {.length = 0, .short_data = {0}}; - bool ts_subtree_external_token_state_eq(const Subtree *self, const Subtree *other) { const TSExternalTokenState *state1 = &empty_state; const TSExternalTokenState *state2 = &empty_state; diff --git a/src/runtime/subtree.h b/src/runtime/subtree.h index 5168ac38..fd1fcaff 100644 --- a/src/runtime/subtree.h +++ b/src/runtime/subtree.h @@ -1,5 +1,5 @@ -#ifndef RUNTIME_TREE_H_ -#define RUNTIME_TREE_H_ +#ifndef RUNTIME_SUBTREE_H_ +#define RUNTIME_SUBTREE_H_ #ifdef __cplusplus extern "C" { @@ -83,7 +83,7 @@ void ts_subtree_array_delete(SubtreePool *, SubtreeArray *); SubtreeArray ts_subtree_array_remove_trailing_extras(SubtreeArray *); void ts_subtree_array_reverse(SubtreeArray *); -void ts_subtree_pool_init(SubtreePool *); +SubtreePool ts_subtree_pool_new(uint32_t capacity); void ts_subtree_pool_delete(SubtreePool *); Subtree *ts_subtree_pool_allocate(SubtreePool *); void ts_subtree_pool_free(SubtreePool *, Subtree *); @@ -122,4 +122,4 @@ static inline Length ts_subtree_total_size(const Subtree *self) { } #endif -#endif // RUNTIME_TREE_H_ +#endif // RUNTIME_SUBTREE_H_ diff --git a/src/runtime/tree.c b/src/runtime/tree.c new file mode 100644 index 00000000..2c365644 --- /dev/null +++ b/src/runtime/tree.c @@ -0,0 +1,59 @@ +#include "tree_sitter/runtime.h" +#include "runtime/array.h" +#include "runtime/get_changed_ranges.h" +#include "runtime/subtree.h" +#include "runtime/tree_cursor.h" +#include "runtime/tree.h" + +TSTree *ts_tree_new(const Subtree *root, const TSLanguage *language) { + TSTree *result = ts_malloc(sizeof(TSTree)); + result->root = root; + result->language = language; + return result; +} + +TSTree *ts_tree_copy(const TSTree *self) { + ts_subtree_retain(self->root); + return ts_tree_new(self->root, self->language); +} + +void ts_tree_delete(const TSTree *self) { + SubtreePool pool = ts_subtree_pool_new(0); + ts_subtree_release(&pool, self->root); + ts_subtree_pool_delete(&pool); + ts_free(self); +} + +TSNode ts_tree_root_node(const TSTree *self) { + return (TSNode) { + .subtree = self->root, + .tree = self, + .position = {0, 0}, + .byte = 0, + .alias_symbol = 0, + }; +} + +void ts_tree_edit(TSTree *self, const TSInputEdit *edit) { + SubtreePool pool = ts_subtree_pool_new(0); + self->root = ts_subtree_edit(self->root, edit, &pool); + assert(pool.tree_stack.capacity == 0 && pool.free_trees.capacity == 0); +} + +TSRange *ts_tree_get_changed_ranges(const TSTree *self, const TSTree *other, uint32_t *count) { + TSRange *result; + TSTreeCursor cursor1, cursor2; + ts_tree_cursor_init(&cursor1, self); + ts_tree_cursor_init(&cursor2, self); + *count = ts_subtree_get_changed_ranges( + self->root, other->root, &cursor1, &cursor2, + self->language, &result + ); + array_delete(&cursor1.stack); + array_delete(&cursor2.stack); + return result; +} + +void ts_tree_print_dot_graph(const TSTree *self, FILE *file) { + ts_subtree_print_dot_graph(self->root, self->language, file); +} diff --git a/src/runtime/tree.h b/src/runtime/tree.h new file mode 100644 index 00000000..7429e06c --- /dev/null +++ b/src/runtime/tree.h @@ -0,0 +1,19 @@ +#ifndef RUNTIME_TREE_H_ +#define RUNTIME_TREE_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +struct TSTree { + const Subtree *root; + const TSLanguage *language; +}; + +TSTree *ts_tree_new(const Subtree *root, const TSLanguage *language); + +#ifdef __cplusplus +} +#endif + +#endif // RUNTIME_TREE_H_ diff --git a/src/runtime/tree_cursor.c b/src/runtime/tree_cursor.c index 730dacf9..c7aa9691 100644 --- a/src/runtime/tree_cursor.c +++ b/src/runtime/tree_cursor.c @@ -1,20 +1,24 @@ #include "tree_sitter/runtime.h" #include "runtime/alloc.h" #include "runtime/tree_cursor.h" -#include "runtime/document.h" #include "runtime/language.h" +#include "runtime/tree.h" -TSTreeCursor *ts_tree_cursor_new(const TSDocument *document) { +TSTreeCursor *ts_tree_cursor_new(const TSTree *tree) { TSTreeCursor *self = ts_malloc(sizeof(TSTreeCursor)); - self->document = document; + ts_tree_cursor_init(self, tree); + return self; +} + +void ts_tree_cursor_init(TSTreeCursor *self, const TSTree *tree) { + self->tree = tree; array_init(&self->stack); array_push(&self->stack, ((TreeCursorEntry) { - .tree = document->tree, + .subtree = tree->root, .position = length_zero(), .child_index = 0, .structural_child_index = 0, })); - return self; } void ts_tree_cursor_delete(TSTreeCursor *self) { @@ -24,7 +28,7 @@ void ts_tree_cursor_delete(TSTreeCursor *self) { bool ts_tree_cursor_goto_first_child(TSTreeCursor *self) { TreeCursorEntry *last_entry = array_back(&self->stack); - Subtree *tree = last_entry->tree; + Subtree *tree = last_entry->subtree; Length position = last_entry->position; bool did_descend; @@ -36,7 +40,7 @@ bool ts_tree_cursor_goto_first_child(TSTreeCursor *self) { Subtree *child = tree->children.contents[i]; if (child->visible || child->visible_child_count > 0) { array_push(&self->stack, ((TreeCursorEntry) { - .tree = child, + .subtree = child, .child_index = i, .structural_child_index = structural_child_index, .position = position, @@ -64,7 +68,7 @@ bool ts_tree_cursor_goto_next_sibling(TSTreeCursor *self) { for (unsigned i = self->stack.size - 2; i + 1 > 0; i--) { TreeCursorEntry *parent_entry = &self->stack.contents[i]; - Subtree *parent = parent_entry->tree; + Subtree *parent = parent_entry->subtree; uint32_t child_index = child_entry->child_index; uint32_t structural_child_index = child_entry->structural_child_index; Length position = child_entry->position; @@ -77,7 +81,7 @@ bool ts_tree_cursor_goto_next_sibling(TSTreeCursor *self) { if (child->visible || child->visible_child_count > 0) { self->stack.contents[i + 1] = (TreeCursorEntry) { - .tree = child, + .subtree = child, .child_index = child_index, .structural_child_index = structural_child_index, .position = position, @@ -103,7 +107,7 @@ bool ts_tree_cursor_goto_next_sibling(TSTreeCursor *self) { bool ts_tree_cursor_goto_parent(TSTreeCursor *self) { for (unsigned i = self->stack.size - 2; i + 1 > 0; i--) { TreeCursorEntry *entry = &self->stack.contents[i]; - if (entry->tree->visible) { + if (entry->subtree->visible) { self->stack.size = i + 1; return true; } @@ -117,16 +121,16 @@ TSNode ts_tree_cursor_current_node(TSTreeCursor *self) { if (self->stack.size > 1) { TreeCursorEntry *parent_entry = &self->stack.contents[self->stack.size - 2]; const TSSymbol *alias_sequence = ts_language_alias_sequence( - self->document->parser.language, - parent_entry->tree->alias_sequence_id + self->tree->language, + parent_entry->subtree->alias_sequence_id ); if (alias_sequence) { alias_symbol = alias_sequence[last_entry->structural_child_index]; } } return (TSNode) { - .document = self->document, - .subtree = last_entry->tree, + .tree = self->tree, + .subtree = last_entry->subtree, .position = last_entry->position.extent, .byte = last_entry->position.bytes, .alias_symbol = alias_symbol, diff --git a/src/runtime/tree_cursor.h b/src/runtime/tree_cursor.h index 5e571fba..c016b276 100644 --- a/src/runtime/tree_cursor.h +++ b/src/runtime/tree_cursor.h @@ -4,17 +4,17 @@ #include "runtime/subtree.h" typedef struct { - Subtree *tree; + Subtree *subtree; Length position; uint32_t child_index; uint32_t structural_child_index; } TreeCursorEntry; struct TSTreeCursor { - const TSDocument *document; + const TSTree *tree; Array(TreeCursorEntry) stack; }; -TSTreeCursor *ts_tree_cursor_new(const TSDocument *); +void ts_tree_cursor_init(TSTreeCursor *, const TSTree *); #endif // RUNTIME_TREE_CURSOR_H_ diff --git a/test/benchmarks.cc b/test/benchmarks.cc index 616d2de6..d4f475b5 100644 --- a/test/benchmarks.cc +++ b/test/benchmarks.cc @@ -43,12 +43,12 @@ int main(int argc, char *arg[]) { vector error_speeds; vector non_error_speeds; - auto document = ts_document_new(); + TSParser *parser = ts_parser_new(); if (getenv("TREE_SITTER_BENCHMARK_SVG")) { - ts_document_print_debugging_graphs(document, true); + ts_parser_print_debugging_graphs(parser, true); } else if (getenv("TREE_SITTER_BENCHMARK_LOG")) { - ts_document_set_logger(document, stderr_logger_new(false)); + ts_parser_set_logger(parser, stderr_logger_new(false)); } auto language_filter = getenv("TREE_SITTER_BENCHMARK_LANGUAGE"); @@ -61,7 +61,7 @@ int main(int argc, char *arg[]) { for (auto &language_name : language_names) { if (language_filter && language_name != language_filter) continue; - ts_document_set_language(document, load_real_language(language_name)); + ts_parser_set_language(parser, load_real_language(language_name)); printf("%s\n", language_name.c_str()); @@ -69,20 +69,16 @@ int main(int argc, char *arg[]) { if (file_name_filter && example.file_name != file_name_filter) continue; if (example.input.size() < 256) continue; - ts_document_invalidate(document); - ts_document_set_input_string(document, ""); - ts_document_parse(document); - - ts_document_invalidate(document); - ts_document_set_input_string(document, example.input.c_str()); - clock_t start_time = clock(); - ts_document_parse(document); + TSTree *tree = ts_parser_parse_string(parser, nullptr, example.input.c_str(), example.input.size()); clock_t end_time = clock(); - unsigned duration = (end_time - start_time) * 1000 / CLOCKS_PER_SEC; - assert(!ts_node_has_error(ts_document_root_node(document))); + + assert(!ts_node_has_error(ts_tree_root_node(tree))); + ts_tree_delete(tree); + + size_t duration = (end_time - start_time) * 1000 / CLOCKS_PER_SEC; size_t speed = static_cast(example.input.size()) / duration; - printf(" %-30s\t%u ms\t\t%lu bytes/ms\n", example.file_name.c_str(), duration, speed); + printf(" %-30s\t%lu ms\t\t%lu bytes/ms\n", example.file_name.c_str(), duration, speed); if (speed != 0) non_error_speeds.push_back(speed); } @@ -93,15 +89,15 @@ int main(int argc, char *arg[]) { if (file_name_filter && example.file_name != file_name_filter) continue; if (example.input.size() < 256) continue; - ts_document_invalidate(document); - ts_document_set_input_string(document, example.input.c_str()); - clock_t start_time = clock(); - ts_document_parse(document); + TSTree *tree = ts_parser_parse_string(parser, nullptr, example.input.c_str(), example.input.size()); clock_t end_time = clock(); - unsigned duration = (end_time - start_time) * 1000 / CLOCKS_PER_SEC; + + ts_tree_delete(tree); + + size_t duration = (end_time - start_time) * 1000 / CLOCKS_PER_SEC; size_t speed = static_cast(example.input.size()) / duration; - printf(" %-30s\t%u ms\t\t%lu bytes/ms\n", example.file_name.c_str(), duration, speed); + printf(" %-30s\t%lu ms\t\t%lu bytes/ms\n", example.file_name.c_str(), duration, speed); if (speed != 0) error_speeds.push_back(speed); } } @@ -118,5 +114,6 @@ int main(int argc, char *arg[]) { printf(" %-30s\t%lu bytes/ms\n", "average speed", mean(error_speeds)); printf(" %-30s\t%lu bytes/ms\n", "worst speed", min(error_speeds)); + ts_parser_delete(parser); return 0; } diff --git a/test/helpers/scope_sequence.cc b/test/helpers/scope_sequence.cc index c3db70ac..8851f0c4 100644 --- a/test/helpers/scope_sequence.cc +++ b/test/helpers/scope_sequence.cc @@ -21,8 +21,7 @@ static void append_text_to_scope_sequence(ScopeSequence *sequence, static void append_to_scope_sequence(ScopeSequence *sequence, ScopeStack *current_scopes, - TSNode node, TSDocument *document, - const std::string &text) { + TSNode node, const std::string &text) { append_text_to_scope_sequence( sequence, current_scopes, text, ts_node_start_byte(node) - sequence->size() ); @@ -31,7 +30,7 @@ static void append_to_scope_sequence(ScopeSequence *sequence, for (size_t i = 0, n = ts_node_child_count(node); i < n; i++) { TSNode child = ts_node_child(node, i); - append_to_scope_sequence(sequence, current_scopes, child, document, text); + append_to_scope_sequence(sequence, current_scopes, child, text); } append_text_to_scope_sequence( @@ -41,11 +40,11 @@ static void append_to_scope_sequence(ScopeSequence *sequence, current_scopes->pop_back(); } -ScopeSequence build_scope_sequence(TSDocument *document, const std::string &text) { +ScopeSequence build_scope_sequence(TSTree *tree, const std::string &text) { ScopeSequence sequence; ScopeStack current_scopes; - TSNode node = ts_document_root_node(document); - append_to_scope_sequence(&sequence, ¤t_scopes, node, document, text); + TSNode node = ts_tree_root_node(tree); + append_to_scope_sequence(&sequence, ¤t_scopes, node, text); return sequence; } diff --git a/test/helpers/scope_sequence.h b/test/helpers/scope_sequence.h index c83ad597..2ad15117 100644 --- a/test/helpers/scope_sequence.h +++ b/test/helpers/scope_sequence.h @@ -9,7 +9,7 @@ typedef std::string Scope; typedef std::vector ScopeStack; typedef std::vector ScopeSequence; -ScopeSequence build_scope_sequence(TSDocument *document, const std::string &text); +ScopeSequence build_scope_sequence(TSTree *tree, const std::string &text); void verify_changed_ranges(const ScopeSequence &old, const ScopeSequence &new_sequence, const std::string &text, TSRange *ranges, size_t range_count); diff --git a/test/integration/fuzzing-examples.cc b/test/integration/fuzzing-examples.cc index 46cac15a..92682d4c 100644 --- a/test/integration/fuzzing-examples.cc +++ b/test/integration/fuzzing-examples.cc @@ -29,14 +29,14 @@ describe("examples found via fuzzing", [&]() { for (unsigned i = 0, n = examples.size(); i < n; i++) { it(("parses example number " + to_string(i)).c_str(), [&]() { - TSDocument *document = ts_document_new(); + TSParser *parser = ts_parser_new(); if (getenv("TREE_SITTER_ENABLE_DEBUG_GRAPHS")) { - ts_document_print_debugging_graphs(document, true); + ts_parser_print_debugging_graphs(parser, true); } const string &language_name = examples[i].first; - ts_document_set_language(document, load_real_language(language_name)); + ts_parser_set_language(parser, load_real_language(language_name)); string input; const string &base64_input = examples[i].second; @@ -47,18 +47,12 @@ describe("examples found via fuzzing", [&]() { base64_input.size() )); - ts_document_set_input_string_with_length( - document, - input.c_str(), - input.size() - ); - - ts_document_parse(document); - - TSNode node = ts_document_root_node(document); + TSTree *tree = ts_parser_parse_string(parser, nullptr, input.c_str(), input.size()); + TSNode node = ts_tree_root_node(tree); assert_consistent_tree_sizes(node); - ts_document_free(document); + ts_tree_delete(tree); + ts_parser_delete(parser); }); } diff --git a/test/integration/real_grammars.cc b/test/integration/real_grammars.cc index 608ded82..0d594ce5 100644 --- a/test/integration/real_grammars.cc +++ b/test/integration/real_grammars.cc @@ -12,8 +12,8 @@ #include "helpers/tree_helpers.h" #include -static void assert_correct_tree_size(TSDocument *document, string content) { - TSNode root_node = ts_document_root_node(document); +static void assert_correct_tree_size(TSTree *tree, string content) { + TSNode root_node = ts_tree_root_node(tree); AssertThat(ts_node_end_byte(root_node), Equals(content.size())); assert_consistent_tree_sizes(root_node); } @@ -33,48 +33,43 @@ vector test_languages({ for (auto &language_name : test_languages) { describe(("the " + language_name + " language").c_str(), [&]() { - TSDocument *document; + TSParser *parser; const bool debug_graphs_enabled = getenv("TREE_SITTER_ENABLE_DEBUG_GRAPHS"); before_each([&]() { record_alloc::start(); - document = ts_document_new(); - ts_document_set_language(document, load_real_language(language_name)); + parser = ts_parser_new(); + ts_parser_set_language(parser, load_real_language(language_name)); - // ts_document_set_logger(document, stderr_logger_new(true)); + // ts_parser_set_logger(parser, stderr_logger_new(true)); if (debug_graphs_enabled) { - ts_document_print_debugging_graphs(document, true); + ts_parser_print_debugging_graphs(parser, true); } }); after_each([&]() { - ts_document_free(document); + ts_parser_delete(parser); AssertThat(record_alloc::outstanding_allocation_indices(), IsEmpty()); }); for (auto &entry : read_real_language_corpus(language_name)) { SpyInput *input; - auto it_handles_edit_sequence = [&](string name, std::function edit_sequence){ - it(("parses " + entry.description + ": " + name).c_str(), [&]() { - input = new SpyInput(entry.input, 3); - if (debug_graphs_enabled) printf("%s\n\n", input->content.c_str()); - ts_document_set_input(document, input->input()); - edit_sequence(); + it(("parses " + entry.description + ": initial parse").c_str(), [&]() { + input = new SpyInput(entry.input, 3); + if (debug_graphs_enabled) printf("%s\n\n", input->content.c_str()); - TSNode root_node = ts_document_root_node(document); - const char *node_string = ts_node_string(root_node); - string result(node_string); - ts_free((void *)node_string); - AssertThat(result, Equals(entry.tree_string)); + TSTree *tree = ts_parser_parse(parser, nullptr, input->input()); + assert_correct_tree_size(tree, input->content); - assert_correct_tree_size(document, input->content); - delete input; - }); - }; + TSNode root_node = ts_tree_root_node(tree); + const char *node_string = ts_node_string(root_node); + string result(node_string); + ts_free((void *)node_string); + AssertThat(result, Equals(entry.tree_string)); - it_handles_edit_sequence("initial parse", [&]() { - ts_document_parse(document); + ts_tree_delete(tree); + delete input; }); set> deletions; @@ -86,54 +81,88 @@ for (auto &language_name : test_languages) { string inserted_text = random_words(random_unsigned(4) + 1); if (insertions.insert({edit_position, inserted_text}).second) { - string description = "\"" + inserted_text + "\" at " + to_string(edit_position); - - it_handles_edit_sequence("repairing an insertion of " + description, [&]() { - ts_document_edit(document, input->replace(edit_position, 0, inserted_text)); - ts_document_parse(document); - assert_correct_tree_size(document, input->content); + it(("parses " + entry.description + + ": repairing an insertion of \"" + inserted_text + "\"" + + " at " + to_string(edit_position)).c_str(), [&]() { + input = new SpyInput(entry.input, 3); if (debug_graphs_enabled) printf("%s\n\n", input->content.c_str()); - ts_document_edit(document, input->undo()); - assert_correct_tree_size(document, input->content); + input->replace(edit_position, 0, inserted_text); + TSTree *tree = ts_parser_parse(parser, nullptr, input->input()); + assert_correct_tree_size(tree, input->content); if (debug_graphs_enabled) printf("%s\n\n", input->content.c_str()); - TSRange *ranges; + TSInputEdit edit = input->undo(); + ts_tree_edit(tree, &edit); + assert_correct_tree_size(tree, input->content); + if (debug_graphs_enabled) printf("%s\n\n", input->content.c_str()); + + TSTree *new_tree = ts_parser_parse(parser, tree, input->input()); + assert_correct_tree_size(new_tree, input->content); + uint32_t range_count; - ScopeSequence old_scope_sequence = build_scope_sequence(document, input->content); - ts_document_parse_and_get_changed_ranges(document, &ranges, &range_count); - assert_correct_tree_size(document, input->content); + TSRange *ranges = ts_tree_get_changed_ranges(tree, new_tree, &range_count); - ScopeSequence new_scope_sequence = build_scope_sequence(document, input->content); - verify_changed_ranges(old_scope_sequence, new_scope_sequence, - input->content, ranges, range_count); + ScopeSequence old_scope_sequence = build_scope_sequence(tree, input->content); + ScopeSequence new_scope_sequence = build_scope_sequence(new_tree, input->content); + verify_changed_ranges( + old_scope_sequence, new_scope_sequence, + input->content, ranges, range_count + ); ts_free(ranges); + + TSNode root_node = ts_tree_root_node(new_tree); + const char *node_string = ts_node_string(root_node); + string result(node_string); + ts_free((void *)node_string); + AssertThat(result, Equals(entry.tree_string)); + + ts_tree_delete(tree); + ts_tree_delete(new_tree); + delete input; }); } if (deletions.insert({edit_position, deletion_size}).second) { - string desription = to_string(edit_position) + "-" + to_string(edit_position + deletion_size); - - it_handles_edit_sequence("repairing a deletion of " + desription, [&]() { - ts_document_edit(document, input->replace(edit_position, deletion_size, "")); - ts_document_parse(document); - assert_correct_tree_size(document, input->content); + it(("parses " + entry.description + + ": repairing a deletion of " + + to_string(edit_position) + "-" + to_string(edit_position + deletion_size)).c_str(), [&]() { + input = new SpyInput(entry.input, 3); if (debug_graphs_enabled) printf("%s\n\n", input->content.c_str()); - ts_document_edit(document, input->undo()); - assert_correct_tree_size(document, input->content); + input->replace(edit_position, deletion_size, ""); + TSTree *tree = ts_parser_parse(parser, nullptr, input->input()); + assert_correct_tree_size(tree, input->content); if (debug_graphs_enabled) printf("%s\n\n", input->content.c_str()); - TSRange *ranges; + TSInputEdit edit = input->undo(); + ts_tree_edit(tree, &edit); + assert_correct_tree_size(tree, input->content); + if (debug_graphs_enabled) printf("%s\n\n", input->content.c_str()); + + TSTree *new_tree = ts_parser_parse(parser, tree, input->input()); + assert_correct_tree_size(new_tree, input->content); + uint32_t range_count; - ScopeSequence old_scope_sequence = build_scope_sequence(document, input->content); - ts_document_parse_and_get_changed_ranges(document, &ranges, &range_count); - assert_correct_tree_size(document, input->content); + TSRange *ranges = ts_tree_get_changed_ranges(tree, new_tree, &range_count); - ScopeSequence new_scope_sequence = build_scope_sequence(document, input->content); - verify_changed_ranges(old_scope_sequence, new_scope_sequence, - input->content, ranges, range_count); + ScopeSequence old_scope_sequence = build_scope_sequence(tree, input->content); + ScopeSequence new_scope_sequence = build_scope_sequence(new_tree, input->content); + verify_changed_ranges( + old_scope_sequence, new_scope_sequence, + input->content, ranges, range_count + ); ts_free(ranges); + + TSNode root_node = ts_tree_root_node(new_tree); + const char *node_string = ts_node_string(root_node); + string result(node_string); + ts_free((void *)node_string); + AssertThat(result, Equals(entry.tree_string)); + + ts_tree_delete(tree); + ts_tree_delete(new_tree); + delete input; }); } } diff --git a/test/integration/test_grammars.cc b/test/integration/test_grammars.cc index f5324579..bba40ace 100644 --- a/test/integration/test_grammars.cc +++ b/test/integration/test_grammars.cc @@ -52,26 +52,26 @@ for (auto &language_name : test_languages) { ); } - TSDocument *document = ts_document_new(); - ts_document_set_language(document, language); - ts_document_set_input_string_with_length(document, entry.input.c_str(), entry.input.size()); + TSParser *parser = ts_parser_new(); + ts_parser_set_language(parser, language); - // ts_document_print_debugging_graphs(document, true); if (getenv("TREE_SITTER_ENABLE_DEBUG_GRAPHS")) { - ts_document_print_debugging_graphs(document, true); + ts_parser_print_debugging_graphs(parser, true); } - ts_document_parse(document); + TSTree *tree = ts_parser_parse_string(parser, nullptr, entry.input.c_str(), entry.input.size()); - TSNode root_node = ts_document_root_node(document); + TSNode root_node = ts_tree_root_node(tree); AssertThat(ts_node_end_byte(root_node), Equals(entry.input.size())); assert_consistent_tree_sizes(root_node); + const char *node_string = ts_node_string(root_node); string result(node_string); ts_free((void *)node_string); - ts_document_free(document); - AssertThat(result, Equals(entry.tree_string)); + + ts_tree_delete(tree); + ts_parser_delete(parser); AssertThat(record_alloc::outstanding_allocation_indices(), IsEmpty()); }); } diff --git a/test/runtime/document_test.cc b/test/runtime/document_test.cc deleted file mode 100644 index 5f140251..00000000 --- a/test/runtime/document_test.cc +++ /dev/null @@ -1,490 +0,0 @@ -#include "test_helper.h" -#include "runtime/alloc.h" -#include "helpers/record_alloc.h" -#include "helpers/stream_methods.h" -#include "helpers/tree_helpers.h" -#include "helpers/point_helpers.h" -#include "helpers/spy_logger.h" -#include "helpers/stderr_logger.h" -#include "helpers/spy_input.h" -#include "helpers/load_language.h" - -TSPoint point(size_t row, size_t column) { - return TSPoint{static_cast(row), static_cast(column)}; -} - -START_TEST - -describe("Document", [&]() { - TSDocument *document; - TSNode root; - - before_each([&]() { - record_alloc::start(); - document = ts_document_new(); - - if (getenv("TREE_SITTER_ENABLE_DEBUG_GRAPHS")) { - ts_document_print_debugging_graphs(document, true); - } - }); - - after_each([&]() { - ts_document_free(document); - record_alloc::stop(); - AssertThat(record_alloc::outstanding_allocation_indices(), IsEmpty()); - }); - - auto assert_node_string_equals = [&](TSNode node, const string &expected) { - char *str = ts_node_string(node); - string actual(str); - ts_free(str); - AssertThat(actual, Equals(expected)); - }; - - describe("set_input(input)", [&]() { - SpyInput *spy_input; - - before_each([&]() { - spy_input = new SpyInput("{\"key\": [null, 2]}", 3); - - ts_document_set_language(document, load_real_language("json")); - ts_document_set_input_string(document, "{\"key\": [1, 2]}"); - ts_document_parse(document); - - root = ts_document_root_node(document); - assert_node_string_equals( - root, - "(value (object (pair (string) (array (number) (number)))))"); - }); - - after_each([&]() { - delete spy_input; - }); - - it("handles both UTF8 and UTF16 encodings", [&]() { - const char16_t content[] = u"[true, false]"; - spy_input->content = string((const char *)content, sizeof(content)); - spy_input->encoding = TSInputEncodingUTF16; - - ts_document_set_input(document, spy_input->input()); - ts_document_invalidate(document); - ts_document_parse(document); - - root = ts_document_root_node(document); - assert_node_string_equals( - root, - "(value (array (true) (false)))"); - }); - - it("handles truncated UTF16 data", [&]() { - const char content[1] = { '\0' }; - spy_input->content = string(content, sizeof(content)); - spy_input->encoding = TSInputEncodingUTF16; - - ts_document_set_input(document, spy_input->input()); - ts_document_invalidate(document); - ts_document_parse(document); - }); - - it("measures columns in bytes", [&]() { - const char16_t content[] = u"[true, false]"; - spy_input->content = string((const char *)content, sizeof(content)); - spy_input->encoding = TSInputEncodingUTF16; - TSInput input = spy_input->input(); - - ts_document_set_input(document, input); - ts_document_invalidate(document); - ts_document_parse(document); - root = ts_document_root_node(document); - AssertThat(ts_node_end_point(root), Equals({0, 28})); - }); - - it("allows the input to be retrieved later", [&]() { - ts_document_set_input(document, spy_input->input()); - AssertThat(ts_document_input(document).payload, Equals(spy_input)); - AssertThat(ts_document_input(document).read, Equals(spy_input->input().read)); - AssertThat(ts_document_input(document).seek, Equals(spy_input->input().seek)); - }); - - it("does not assume that the document's text has changed", [&]() { - ts_document_set_input(document, spy_input->input()); - AssertThat(ts_document_root_node(document), Equals(root)); - AssertThat(ts_node_has_changes(root), IsFalse()); - AssertThat(spy_input->strings_read(), IsEmpty()); - }); - - it("reads text from the new input for future parses", [&]() { - ts_document_set_input(document, spy_input->input()); - - // Insert 'null', delete '1'. - TSInputEdit edit = {}; - edit.start_point.column = edit.start_byte = strlen("{\"key\": ["); - edit.extent_added.column = edit.bytes_added = 4; - edit.extent_removed.column = edit.bytes_removed = 1; - - ts_document_edit(document, edit); - ts_document_parse(document); - - TSNode new_root = ts_document_root_node(document); - assert_node_string_equals( - new_root, - "(value (object (pair (string) (array (null) (number)))))"); - AssertThat(spy_input->strings_read(), Equals(vector({" [null, 2" }))); - }); - - it("allows setting input string with length", [&]() { - const char content[] = { '1' }; - ts_document_set_input_string_with_length(document, content, 1); - ts_document_parse(document); - TSNode new_root = ts_document_root_node(document); - AssertThat(ts_node_end_byte(new_root), Equals(1)); - assert_node_string_equals( - new_root, - "(value (number))"); - }); - - it("reads from the new input correctly when the old input was blank", [&]() { - ts_document_set_input_string(document, ""); - ts_document_parse(document); - TSNode new_root = ts_document_root_node(document); - AssertThat(ts_node_end_byte(new_root), Equals(0)); - assert_node_string_equals( - new_root, - "(ERROR)"); - - ts_document_set_input_string(document, "1"); - ts_document_parse(document); - new_root = ts_document_root_node(document); - AssertThat(ts_node_end_byte(new_root), Equals(1)); - assert_node_string_equals( - new_root, - "(value (number))"); - }); - }); - - describe("set_language(language)", [&]() { - before_each([&]() { - ts_document_set_input_string(document, "{\"key\": [1, 2]}\n"); - }); - - it("uses the given language for future parses", [&]() { - ts_document_set_language(document, load_real_language("json")); - ts_document_parse(document); - - root = ts_document_root_node(document); - assert_node_string_equals( - root, - "(value (object (pair (string) (array (number) (number)))))"); - }); - - it("clears out any previous tree", [&]() { - ts_document_set_language(document, load_real_language("json")); - ts_document_parse(document); - - ts_document_set_language(document, load_real_language("javascript")); - AssertThat(ts_document_root_node(document).subtree, Equals(nullptr)); - - ts_document_parse(document); - root = ts_document_root_node(document); - assert_node_string_equals( - root, - "(program (expression_statement " - "(object (pair (string) (array (number) (number))))))"); - }); - - it("does not allow setting a language with a different version number", [&]() { - TSLanguage language = *load_real_language("json"); - AssertThat(ts_language_version(&language), Equals(TREE_SITTER_LANGUAGE_VERSION)); - - language.version++; - AssertThat(ts_language_version(&language), !Equals(TREE_SITTER_LANGUAGE_VERSION)); - - ts_document_set_language(document, &language); - AssertThat(ts_document_language(document), Equals(nullptr)); - }); - }); - - describe("set_logger(TSLogger)", [&]() { - SpyLogger *logger; - - before_each([&]() { - logger = new SpyLogger(); - ts_document_set_language(document, load_real_language("json")); - ts_document_set_input_string(document, "[1, 2]"); - }); - - after_each([&]() { - delete logger; - }); - - it("calls the debugger with a message for each parse action", [&]() { - ts_document_set_logger(document, logger->logger()); - ts_document_parse(document); - - AssertThat(logger->messages, Contains("new_parse")); - AssertThat(logger->messages, Contains("skip character:' '")); - AssertThat(logger->messages, Contains("consume character:'['")); - AssertThat(logger->messages, Contains("consume character:'1'")); - AssertThat(logger->messages, Contains("reduce sym:array, child_count:4")); - AssertThat(logger->messages, Contains("accept")); - }); - - it("allows the debugger to be retrieved later", [&]() { - ts_document_set_logger(document, logger->logger()); - AssertThat(ts_document_logger(document).payload, Equals(logger)); - }); - - describe("disabling debugging", [&]() { - before_each([&]() { - ts_document_set_logger(document, logger->logger()); - ts_document_set_logger(document, {NULL, NULL}); - }); - - it("does not call the debugger any more", [&]() { - ts_document_parse(document); - AssertThat(logger->messages, IsEmpty()); - }); - }); - }); - - describe("parse_and_get_changed_ranges()", [&]() { - SpyInput *input; - - before_each([&]() { - ts_document_set_language(document, load_real_language("javascript")); - input = new SpyInput("{a: null};\n", 3); - ts_document_set_input(document, input->input()); - ts_document_parse(document); - assert_node_string_equals( - ts_document_root_node(document), - "(program (expression_statement (object (pair (property_identifier) (null)))))"); - }); - - after_each([&]() { - delete input; - }); - - auto get_invalidated_ranges_for_edit = [&](std::function callback) -> vector { - TSInputEdit edit = callback(); - ts_document_edit(document, edit); - - TSRange *ranges; - uint32_t range_count = 0; - ts_document_parse_and_get_changed_ranges(document, &ranges, &range_count); - - vector result; - for (size_t i = 0; i < range_count; i++) { - result.push_back(ranges[i]); - } - ts_free(ranges); - return result; - }; - - it("reports changes when one token has been updated", [&]() { - // Replace `null` with `nothing` - auto ranges = get_invalidated_ranges_for_edit([&]() { - return input->replace(input->content.find("ull"), 1, "othing"); - }); - - AssertThat(ranges, Equals(vector({ - TSRange{ - point(0, input->content.find("nothing")), - point(0, input->content.find("}")) - }, - }))); - - // Replace `nothing` with `null` again - ranges = get_invalidated_ranges_for_edit([&]() { - return input->undo(); - }); - - AssertThat(ranges, Equals(vector({ - TSRange{ - point(0, input->content.find("null")), - point(0, input->content.find("}")) - }, - }))); - }); - - it("reports no changes when leading whitespace has changed (regression)", [&]() { - input->chars_per_chunk = 80; - - // Insert leading whitespace - auto ranges = get_invalidated_ranges_for_edit([&]() { - return input->replace(0, 0, "\n"); - }); - assert_node_string_equals( - ts_document_root_node(document), - "(program (expression_statement (object (pair (property_identifier) (null)))))"); - AssertThat(ranges, Equals(vector({}))); - - // Remove leading whitespace - ranges = get_invalidated_ranges_for_edit([&]() { - return input->undo(); - }); - assert_node_string_equals( - ts_document_root_node(document), - "(program (expression_statement (object (pair (property_identifier) (null)))))"); - AssertThat(ranges, Equals(vector({}))); - - // Insert leading whitespace again - ranges = get_invalidated_ranges_for_edit([&]() { - return input->replace(0, 0, "\n"); - }); - assert_node_string_equals( - ts_document_root_node(document), - "(program (expression_statement (object (pair (property_identifier) (null)))))"); - AssertThat(ranges, Equals(vector({}))); - }); - - it("reports changes when tokens have been appended", [&]() { - // Add a second key-value pair - auto ranges = get_invalidated_ranges_for_edit([&]() { - return input->replace(input->content.find("}"), 0, ", b: false"); - }); - - AssertThat(ranges, Equals(vector({ - TSRange{ - point(0, input->content.find(",")), - point(0, input->content.find("}")) - }, - }))); - - // Add a third key-value pair in between the first two - ranges = get_invalidated_ranges_for_edit([&]() { - return input->replace(input->content.find(", b"), 0, ", c: 1"); - }); - - assert_node_string_equals( - ts_document_root_node(document), - "(program (expression_statement (object " - "(pair (property_identifier) (null)) " - "(pair (property_identifier) (number)) " - "(pair (property_identifier) (false)))))"); - - AssertThat(ranges, Equals(vector({ - TSRange{ - point(0, input->content.find(", c")), - point(0, input->content.find(", b")) - }, - }))); - - // Delete the middle pair. - ranges = get_invalidated_ranges_for_edit([&]() { - return input->undo(); - }); - - assert_node_string_equals( - ts_document_root_node(document), - "(program (expression_statement (object " - "(pair (property_identifier) (null)) " - "(pair (property_identifier) (false)))))"); - - AssertThat(ranges, IsEmpty()); - - // Delete the second pair. - ranges = get_invalidated_ranges_for_edit([&]() { - return input->undo(); - }); - - assert_node_string_equals( - ts_document_root_node(document), - "(program (expression_statement (object " - "(pair (property_identifier) (null)))))"); - - AssertThat(ranges, IsEmpty()); - }); - - it("reports changes when trees have been wrapped", [&]() { - // Wrap the object in an assignment expression. - auto ranges = get_invalidated_ranges_for_edit([&]() { - return input->replace(input->content.find("null"), 0, "b === "); - }); - - assert_node_string_equals( - ts_document_root_node(document), - "(program (expression_statement (object " - "(pair (property_identifier) (binary_expression (identifier) (null))))))"); - - AssertThat(ranges, Equals(vector({ - TSRange{ - point(0, input->content.find("b ===")), - point(0, input->content.find("}")) - }, - }))); - }); - }); - - describe("parse_with_options(options)", [&]() { - it("halts as soon as an error is found if the halt_on_error flag is set", [&]() { - string input_string = "[1, null, error, 3]"; - ts_document_set_language(document, load_real_language("json")); - ts_document_set_input_string(document, input_string.c_str()); - - TSParseOptions options = {}; - options.changed_ranges = nullptr; - - options.halt_on_error = false; - ts_document_parse_with_options(document, options); - root = ts_document_root_node(document); - assert_node_string_equals( - root, - "(value (array (number) (null) (ERROR (UNEXPECTED 'e')) (number)))"); - - ts_document_invalidate(document); - - options.halt_on_error = true; - ts_document_parse_with_options(document, options); - root = ts_document_root_node(document); - assert_node_string_equals( - root, - "(ERROR (number) (null))"); - - AssertThat(ts_node_end_byte(root), Equals(input_string.size())); - }); - - it("does not insert missing tokens if the halt_on_error flag is set", [&]() { - string input_string = "[1, null, 3"; - ts_document_set_language(document, load_real_language("json")); - ts_document_set_input_string(document, input_string.c_str()); - - TSParseOptions options = {}; - options.changed_ranges = nullptr; - - options.halt_on_error = false; - ts_document_parse_with_options(document, options); - root = ts_document_root_node(document); - assert_node_string_equals( - root, - "(value (array (number) (null) (number) (MISSING)))"); - - ts_document_invalidate(document); - - options.halt_on_error = true; - ts_document_parse_with_options(document, options); - root = ts_document_root_node(document); - assert_node_string_equals( - root, - "(ERROR (number) (null) (number))"); - - AssertThat(ts_node_end_byte(root), Equals(input_string.size())); - }); - - it("can parse valid code with the halt_on_error flag set", [&]() { - string input_string = "[1, null, 3]"; - ts_document_set_language(document, load_real_language("json")); - ts_document_set_input_string(document, input_string.c_str()); - - TSParseOptions options = {}; - options.changed_ranges = nullptr; - options.halt_on_error = true; - ts_document_parse_with_options(document, options); - root = ts_document_root_node(document); - assert_node_string_equals( - root, - "(value (array (number) (null) (number)))"); - }); - }); -}); - -END_TEST diff --git a/test/runtime/language_test.cc b/test/runtime/language_test.cc index 4726c4ba..747327c0 100644 --- a/test/runtime/language_test.cc +++ b/test/runtime/language_test.cc @@ -28,13 +28,12 @@ describe("Language", []() { } })JSON"); - TSDocument *document = ts_document_new(); + TSParser *parser = ts_parser_new(); const TSLanguage *language = load_test_language("aliased_rules", compile_result); - ts_document_set_language(document, language); - ts_document_set_input_string(document, "b"); - ts_document_parse(document); + ts_parser_set_language(parser, language); + TSTree *tree = ts_parser_parse_string(parser, nullptr, "b", 1); - TSNode root_node = ts_document_root_node(document); + TSNode root_node = ts_tree_root_node(tree); char *string = ts_node_string(root_node); AssertThat(string, Equals("(a (c))")); @@ -47,7 +46,8 @@ describe("Language", []() { AssertThat(ts_language_symbol_type(language, aliased_symbol), Equals(TSSymbolTypeRegular)); ts_free(string); - ts_document_free(document); + ts_parser_delete(parser); + ts_tree_delete(tree); }); }); }); diff --git a/test/runtime/node_test.cc b/test/runtime/node_test.cc index 808084be..4c306897 100644 --- a/test/runtime/node_test.cc +++ b/test/runtime/node_test.cc @@ -62,21 +62,22 @@ string grammar_with_aliases_and_extras = R"JSON({ })JSON"; describe("Node", [&]() { - TSDocument *document; + TSParser *parser; + TSTree *tree; TSNode root_node; before_each([&]() { record_alloc::start(); - document = ts_document_new(); - ts_document_set_language(document, load_real_language("json")); - ts_document_set_input_string(document, json_string.c_str()); - ts_document_parse(document); - root_node = ts_node_child(ts_document_root_node(document), 0); + parser = ts_parser_new(); + ts_parser_set_language(parser, load_real_language("json")); + tree = ts_parser_parse_string(parser, nullptr, json_string.c_str(), json_string.size()); + root_node = ts_node_child(ts_tree_root_node(tree), 0); }); after_each([&]() { - ts_document_free(document); + ts_parser_delete(parser); + ts_tree_delete(tree); record_alloc::stop(); AssertThat(record_alloc::outstanding_allocation_indices(), IsEmpty()); @@ -157,16 +158,17 @@ describe("Node", [&]() { AssertThat(ts_node_parent(number_node), Equals(root_node)); AssertThat(ts_node_parent(false_node), Equals(root_node)); AssertThat(ts_node_parent(object_node), Equals(root_node)); - AssertThat(ts_node_parent(ts_document_root_node(document)).subtree, Equals(nullptr)); + AssertThat(ts_node_parent(ts_tree_root_node(tree)).subtree, Equals(nullptr)); }); it("works correctly when the node contains aliased children and extras", [&]() { TSCompileResult compile_result = ts_compile_grammar(grammar_with_aliases_and_extras.c_str()); const TSLanguage *language = load_test_language("aliases_and_extras", compile_result); - ts_document_set_language(document, language); - ts_document_set_input_string(document, "b ... b ... b"); - ts_document_parse(document); - root_node = ts_document_root_node(document); + ts_parser_set_language(parser, language); + + ts_tree_delete(tree); + tree = ts_parser_parse_string(parser, nullptr, "b ... b ... b", 13); + root_node = ts_tree_root_node(tree); char *node_string = ts_node_string(root_node); AssertThat(node_string, Equals("(a (b) (comment) (B) (comment) (b))")); @@ -179,7 +181,10 @@ describe("Node", [&]() { AssertThat(ts_node_type(ts_node_named_child(root_node, 3)), Equals("comment")); AssertThat(ts_node_type(ts_node_named_child(root_node, 4)), Equals("b")); - AssertThat(ts_node_symbol(ts_node_named_child(root_node, 0)), !Equals(ts_node_symbol(ts_node_named_child(root_node, 2)))); + AssertThat( + ts_node_symbol(ts_node_named_child(root_node, 0)), + !Equals(ts_node_symbol(ts_node_named_child(root_node, 2))) + ); }); }); @@ -323,7 +328,7 @@ describe("Node", [&]() { AssertThat(ts_node_parent(child5), Equals(root_node)); AssertThat(ts_node_parent(child6), Equals(root_node)); AssertThat(ts_node_parent(child7), Equals(root_node)); - AssertThat(ts_node_parent(ts_document_root_node(document)).subtree, Equals(nullptr)); + AssertThat(ts_node_parent(ts_tree_root_node(tree)).subtree, Equals(nullptr)); }); }); @@ -483,9 +488,10 @@ describe("Node", [&]() { it("works in the presence of multi-byte characters", [&]() { string input_string = "[\"αβγδ\", \"αβγδ\"]"; - ts_document_set_input_string(document, input_string.c_str()); - ts_document_parse(document); - TSNode root_node = ts_document_root_node(document); + + ts_tree_delete(tree); + tree = ts_parser_parse_string(parser, nullptr, input_string.c_str(), input_string.size()); + TSNode root_node = ts_tree_root_node(tree); uint32_t comma_position = input_string.find(","); TSNode node1 = ts_node_descendant_for_byte_range(root_node, comma_position, comma_position); @@ -518,23 +524,23 @@ describe("Node", [&]() { }); describe("TreeCursor", [&]() { - TSDocument *document; + TSParser *parser; + TSTree *tree; TSTreeCursor *cursor; before_each([&]() { record_alloc::start(); - document = ts_document_new(); - ts_document_set_language(document, load_real_language("json")); - ts_document_set_input_string(document, json_string.c_str()); - ts_document_parse(document); - - cursor = ts_document_tree_cursor(document); + parser = ts_parser_new(); + ts_parser_set_language(parser, load_real_language("json")); + tree = ts_parser_parse_string(parser, nullptr, json_string.c_str(), json_string.size()); + cursor = ts_tree_cursor_new(tree); }); after_each([&]() { + ts_tree_delete(tree); ts_tree_cursor_delete(cursor); - ts_document_free(document); + ts_parser_delete(parser); record_alloc::stop(); AssertThat(record_alloc::outstanding_allocation_indices(), IsEmpty()); diff --git a/test/runtime/parser_test.cc b/test/runtime/parser_test.cc index 2cf70b69..e9cfbe72 100644 --- a/test/runtime/parser_test.cc +++ b/test/runtime/parser_test.cc @@ -1,17 +1,20 @@ #include "test_helper.h" #include "runtime/alloc.h" +#include "runtime/language.h" #include "helpers/record_alloc.h" #include "helpers/spy_input.h" #include "helpers/load_language.h" #include "helpers/record_alloc.h" #include "helpers/point_helpers.h" +#include "helpers/spy_logger.h" #include "helpers/stderr_logger.h" #include "helpers/dedent.h" START_TEST describe("Parser", [&]() { - TSDocument *document; + TSParser *parser; + TSTree *tree; SpyInput *input; TSNode root; size_t chunk_size; @@ -21,14 +24,16 @@ describe("Parser", [&]() { chunk_size = 3; input = nullptr; - document = ts_document_new(); + tree = nullptr; + parser = ts_parser_new(); if (getenv("TREE_SITTER_ENABLE_DEBUG_GRAPHS")) { - ts_document_print_debugging_graphs(document, true); + ts_parser_print_debugging_graphs(parser, true); } }); after_each([&]() { - if (document) ts_document_free(document); + if (parser) ts_parser_delete(parser); + if (tree) ts_tree_delete(tree); if (input) delete input; record_alloc::stop(); @@ -37,10 +42,8 @@ describe("Parser", [&]() { auto set_text = [&](string text) { input = new SpyInput(text, chunk_size); - ts_document_set_input(document, input->input()); - ts_document_parse(document); - - root = ts_document_root_node(document); + tree = ts_parser_parse(parser, nullptr, input->input()); + root = ts_tree_root_node(tree); AssertThat(ts_node_end_byte(root), Equals(text.size())); input->clear(); }; @@ -48,10 +51,13 @@ describe("Parser", [&]() { auto replace_text = [&](size_t position, size_t length, string new_text) { size_t prev_size = ts_node_end_byte(root); - ts_document_edit(document, input->replace(position, length, new_text)); - ts_document_parse(document); + TSInputEdit edit = input->replace(position, length, new_text); + ts_tree_edit(tree, &edit); + TSTree *new_tree = ts_parser_parse(parser, tree, input->input()); + ts_tree_delete(tree); + tree = new_tree; - root = ts_document_root_node(document); + root = ts_tree_root_node(tree); size_t new_size = ts_node_end_byte(root); AssertThat(new_size, Equals(prev_size - length + new_text.size())); }; @@ -65,12 +71,15 @@ describe("Parser", [&]() { }; auto undo = [&]() { - ts_document_edit(document, input->undo()); - ts_document_parse(document); + TSInputEdit edit = input->undo(); + ts_tree_edit(tree, &edit); + TSTree *new_tree = ts_parser_parse(parser, tree, input->input()); + ts_tree_delete(tree); + tree = new_tree; }; auto assert_root_node = [&](const string &expected) { - TSNode node = ts_document_root_node(document); + TSNode node = ts_tree_root_node(tree); char *node_string = ts_node_string(node); string actual(node_string); ts_free(node_string); @@ -86,11 +95,9 @@ describe("Parser", [&]() { describe("handling errors", [&]() { describe("when there is an invalid substring right before a valid token", [&]() { it("computes the error node's size and position correctly", [&]() { - ts_document_set_language(document, load_real_language("json")); + ts_parser_set_language(parser, load_real_language("json")); set_text(" [123, @@@@@, true]"); - - assert_root_node( - "(value (array (number) (ERROR (UNEXPECTED '@')) (true)))"); + assert_root_node("(value (array (number) (ERROR (UNEXPECTED '@')) (true)))"); TSNode error = ts_node_named_child(ts_node_child(root, 0), 1); AssertThat(ts_node_type(error), Equals("ERROR")); @@ -111,7 +118,7 @@ describe("Parser", [&]() { describe("when there is an unexpected string in the middle of a token", [&]() { it("computes the error node's size and position correctly", [&]() { - ts_document_set_language(document, load_real_language("json")); + ts_parser_set_language(parser, load_real_language("json")); set_text(" [123, faaaaalse, true]"); assert_root_node( @@ -138,11 +145,10 @@ describe("Parser", [&]() { describe("when there is one unexpected token between two valid tokens", [&]() { it("computes the error node's size and position correctly", [&]() { - ts_document_set_language(document, load_real_language("json")); + ts_parser_set_language(parser, load_real_language("json")); set_text(" [123, true false, true]"); - assert_root_node( - "(value (array (number) (true) (ERROR (false)) (true)))"); + assert_root_node("(value (array (number) (true) (ERROR (false)) (true)))"); TSNode error = ts_node_named_child(ts_node_child(root, 0), 2); AssertThat(ts_node_type(error), Equals("ERROR")); @@ -157,26 +163,23 @@ describe("Parser", [&]() { describe("when there is an unexpected string at the end of a token", [&]() { it("computes the error's size and position correctly", [&]() { - ts_document_set_language(document, load_real_language("json")); + ts_parser_set_language(parser, load_real_language("json")); set_text(" [123, \"hi\n, true]"); - - assert_root_node( - "(value (array (number) (ERROR (UNEXPECTED '\\n')) (true)))"); + assert_root_node("(value (array (number) (ERROR (UNEXPECTED '\\n')) (true)))"); }); }); describe("when there is an unterminated error", [&]() { it("maintains a consistent tree", [&]() { - ts_document_set_language(document, load_real_language("javascript")); + ts_parser_set_language(parser, load_real_language("javascript")); set_text("a; ' this string never ends"); - assert_root_node( - "(program (expression_statement (identifier)) (ERROR (UNEXPECTED EOF)))"); + assert_root_node("(program (expression_statement (identifier)) (ERROR (UNEXPECTED EOF)))"); }); }); describe("when there are extra tokens at the end of the viable prefix", [&]() { it("does not include them in the error node", [&]() { - ts_document_set_language(document, load_real_language("javascript")); + ts_parser_set_language(parser, load_real_language("javascript")); set_text( "var x;\n" "\n" @@ -196,20 +199,64 @@ describe("Parser", [&]() { char *string = (char *)malloc(1); string[0] = '\xdf'; - ts_document_set_language(document, load_real_language("json")); - ts_document_set_input_string_with_length(document, string, 1); - ts_document_parse(document); + ts_parser_set_language(parser, load_real_language("json")); + tree = ts_parser_parse_string(parser, nullptr, string, 1); free(string); - assert_root_node("(ERROR (UNEXPECTED INVALID))"); }); + + describe("when halt_on_error is set to true", [&]() { + it("halts as soon as an error is found if the halt_on_error flag is set", [&]() { + string input_string = "[1, null, error, 3]"; + ts_parser_set_language(parser, load_real_language("json")); + + tree = ts_parser_parse_string(parser, nullptr, input_string.c_str(), input_string.size()); + root = ts_tree_root_node(tree); + assert_root_node("(value (array (number) (null) (ERROR (UNEXPECTED 'e')) (number)))"); + + ts_parser_halt_on_error(parser, true); + + ts_tree_delete(tree); + tree = ts_parser_parse_string(parser, nullptr, input_string.c_str(), input_string.size()); + root = ts_tree_root_node(tree); + assert_root_node("(ERROR (number) (null))"); + AssertThat(ts_node_end_byte(root), Equals(input_string.size())); + }); + + it("does not insert missing tokens if the halt_on_error flag is set", [&]() { + string input_string = "[1, null, 3"; + ts_parser_set_language(parser, load_real_language("json")); + + tree = ts_parser_parse_string(parser, nullptr, input_string.c_str(), input_string.size()); + root = ts_tree_root_node(tree); + assert_root_node("(value (array (number) (null) (number) (MISSING)))"); + + ts_parser_halt_on_error(parser, true); + + ts_tree_delete(tree); + tree = ts_parser_parse_string(parser, nullptr, input_string.c_str(), input_string.size()); + root = ts_tree_root_node(tree); + assert_root_node("(ERROR (number) (null) (number))"); + AssertThat(ts_node_end_byte(root), Equals(input_string.size())); + }); + + it("can parse valid code with the halt_on_error flag set", [&]() { + string input_string = "[1, null, 3]"; + ts_parser_set_language(parser, load_real_language("json")); + + ts_parser_halt_on_error(parser, true); + tree = ts_parser_parse_string(parser, nullptr, input_string.c_str(), input_string.size()); + root = ts_tree_root_node(tree); + assert_root_node("(value (array (number) (null) (number)))"); + }); + }); }); describe("editing", [&]() { describe("creating new tokens near the end of the input", [&]() { it("updates the parse tree and re-reads only the changed portion of the text", [&]() { - ts_document_set_language(document, load_real_language("javascript")); + ts_parser_set_language(parser, load_real_language("javascript")); set_text("x * (100 + abc);"); assert_root_node( @@ -242,7 +289,7 @@ describe("Parser", [&]() { it("updates the parse tree and re-reads only the changed portion of the input", [&]() { chunk_size = 2; - ts_document_set_language(document, load_real_language("javascript")); + ts_parser_set_language(parser, load_real_language("javascript")); set_text("123 + 456 * (10 + x);"); assert_root_node( @@ -268,7 +315,7 @@ describe("Parser", [&]() { describe("introducing an error", [&]() { it("gives the error the right size", [&]() { - ts_document_set_language(document, load_real_language("javascript")); + ts_parser_set_language(parser, load_real_language("javascript")); set_text("var x = y;"); assert_root_node( @@ -291,7 +338,7 @@ describe("Parser", [&]() { describe("into the middle of an existing token", [&]() { it("updates the parse tree", [&]() { - ts_document_set_language(document, load_real_language("javascript")); + ts_parser_set_language(parser, load_real_language("javascript")); set_text("abc * 123;"); assert_root_node( @@ -310,7 +357,7 @@ describe("Parser", [&]() { describe("at the end of an existing token", [&]() { it("updates the parse tree", [&]() { - ts_document_set_language(document, load_real_language("javascript")); + ts_parser_set_language(parser, load_real_language("javascript")); set_text("abc * 123;"); assert_root_node( @@ -329,7 +376,7 @@ describe("Parser", [&]() { describe("inserting text into a node containing a extra token", [&]() { it("updates the parse tree", [&]() { - ts_document_set_language(document, load_real_language("javascript")); + ts_parser_set_language(parser, load_real_language("javascript")); set_text("123 *\n" "// a-comment\n" "abc;"); @@ -356,7 +403,7 @@ describe("Parser", [&]() { describe("when a critical token is removed", [&]() { it("updates the parse tree, creating an error", [&]() { - ts_document_set_language(document, load_real_language("javascript")); + ts_parser_set_language(parser, load_real_language("javascript")); set_text("123 * 456; 789 * 123;"); assert_root_node( @@ -376,7 +423,7 @@ describe("Parser", [&]() { describe("with external tokens", [&]() { it("maintains the external scanner's state during incremental parsing", [&]() { - ts_document_set_language(document, load_real_language("python")); + ts_parser_set_language(parser, load_real_language("python")); string text = dedent(R"PYTHON( if a: print b @@ -404,7 +451,7 @@ describe("Parser", [&]() { }); it("does not try to reuse nodes that are within the edited region", [&]() { - ts_document_set_language(document, load_real_language("javascript")); + ts_parser_set_language(parser, load_real_language("javascript")); set_text("{ x: (b.c) };"); assert_root_node( @@ -417,23 +464,12 @@ describe("Parser", [&]() { "(program (expression_statement (object (pair " "(property_identifier) (member_expression (identifier) (property_identifier))))))"); }); - - it("updates the document's parse count", [&]() { - ts_document_set_language(document, load_real_language("javascript")); - AssertThat(ts_document_parse_count(document), Equals(0)); - - set_text("{ x: (b.c) };"); - AssertThat(ts_document_parse_count(document), Equals(1)); - - insert_text(strlen("{ x"), "yz"); - AssertThat(ts_document_parse_count(document), Equals(2)); - }); }); describe("lexing", [&]() { describe("handling tokens containing wildcard patterns (e.g. comments)", [&]() { - it("terminates them at the end of the document", [&]() { - ts_document_set_language(document, load_real_language("javascript")); + it("terminates them at the end of the string", [&]() { + ts_parser_set_language(parser, load_real_language("javascript")); set_text("x; // this is a comment"); assert_root_node( @@ -448,7 +484,7 @@ describe("Parser", [&]() { it("recognizes UTF8 characters as single characters", [&]() { // 'ΩΩΩ — ΔΔ'; - ts_document_set_language(document, load_real_language("javascript")); + ts_parser_set_language(parser, load_real_language("javascript")); set_text("'\u03A9\u03A9\u03A9 \u2014 \u0394\u0394';"); assert_root_node( @@ -460,14 +496,120 @@ describe("Parser", [&]() { it("handles non-UTF8 characters", [&]() { const char *string = "cons\xeb\x00e=ls\x83l6hi');\x0a"; - ts_document_set_language(document, load_real_language("javascript")); - ts_document_set_input_string(document, string); - ts_document_parse(document); - - TSNode root = ts_document_root_node(document); + ts_parser_set_language(parser, load_real_language("javascript")); + tree = ts_parser_parse_string(parser, nullptr, string, strlen(string)); + TSNode root = ts_tree_root_node(tree); AssertThat(ts_node_end_byte(root), Equals(strlen(string))); }); }); + + describe("handling TSInputs", [&]() { + SpyInput *spy_input; + + before_each([&]() { + spy_input = new SpyInput("{\"key\": [null, 2]}", 3); + ts_parser_set_language(parser, load_real_language("json")); + }); + + after_each([&]() { + delete spy_input; + }); + + it("handles UTF16 encodings", [&]() { + const char16_t content[] = u"[true, false]"; + spy_input->content = string((const char *)content, sizeof(content)); + spy_input->encoding = TSInputEncodingUTF16; + + tree = ts_parser_parse(parser, nullptr, spy_input->input()); + root = ts_tree_root_node(tree); + assert_root_node( + "(value (array (true) (false)))"); + }); + + it("handles truncated UTF16 data", [&]() { + const char content[1] = { '\0' }; + spy_input->content = string(content, sizeof(content)); + spy_input->encoding = TSInputEncodingUTF16; + + tree = ts_parser_parse(parser, nullptr, spy_input->input()); + }); + + it("measures columns in bytes", [&]() { + const char16_t content[] = u"[true, false]"; + spy_input->content = string((const char *)content, sizeof(content)); + spy_input->encoding = TSInputEncodingUTF16; + + tree = ts_parser_parse(parser, nullptr, spy_input->input()); + root = ts_tree_root_node(tree); + AssertThat(ts_node_end_point(root), Equals({0, 28})); + }); + }); + + describe("set_language(language)", [&]() { + string input_string = "{\"key\": [1, 2]}\n"; + + it("uses the given language for future parses", [&]() { + ts_parser_set_language(parser, load_real_language("json")); + tree = ts_parser_parse_string(parser, nullptr, input_string.c_str(), input_string.size()); + + root = ts_tree_root_node(tree); + assert_root_node( + "(value (object (pair (string) (array (number) (number)))))"); + }); + + it("does not allow setting a language with a different version number", [&]() { + TSLanguage language = *load_real_language("json"); + AssertThat(ts_language_version(&language), Equals(TREE_SITTER_LANGUAGE_VERSION)); + + language.version++; + AssertThat(ts_language_version(&language), !Equals(TREE_SITTER_LANGUAGE_VERSION)); + + AssertThat(ts_parser_set_language(parser, &language), IsFalse()); + AssertThat(ts_parser_language(parser), Equals(nullptr)); + }); + }); + + describe("set_logger(TSLogger)", [&]() { + SpyLogger *logger; + + before_each([&]() { + logger = new SpyLogger(); + ts_parser_set_language(parser, load_real_language("json")); + }); + + after_each([&]() { + delete logger; + }); + + it("calls the debugger with a message for each parse action", [&]() { + ts_parser_set_logger(parser, logger->logger()); + tree = ts_parser_parse_string(parser, nullptr, "[ 1, 2, 3 ]", 11); + + AssertThat(logger->messages, Contains("new_parse")); + AssertThat(logger->messages, Contains("skip character:' '")); + AssertThat(logger->messages, Contains("consume character:'['")); + AssertThat(logger->messages, Contains("consume character:'1'")); + AssertThat(logger->messages, Contains("reduce sym:array, child_count:4")); + AssertThat(logger->messages, Contains("accept")); + }); + + it("allows the debugger to be retrieved later", [&]() { + ts_parser_set_logger(parser, logger->logger()); + AssertThat(ts_parser_logger(parser).payload, Equals(logger)); + }); + + describe("disabling debugging", [&]() { + before_each([&]() { + ts_parser_set_logger(parser, logger->logger()); + ts_parser_set_logger(parser, {NULL, NULL}); + }); + + it("does not call the debugger any more", [&]() { + tree = ts_parser_parse_string(parser, nullptr, "{}", 2); + AssertThat(logger->messages, IsEmpty()); + }); + }); + }); }); END_TEST diff --git a/test/runtime/stack_test.cc b/test/runtime/stack_test.cc index 99a979d1..ae5b1589 100644 --- a/test/runtime/stack_test.cc +++ b/test/runtime/stack_test.cc @@ -76,7 +76,7 @@ describe("Stack", [&]() { before_each([&]() { record_alloc::start(); - ts_subtree_pool_init(&pool); + pool = ts_subtree_pool_new(10); stack = ts_stack_new(&pool); TSLanguage dummy_language; diff --git a/test/runtime/subtree_test.cc b/test/runtime/subtree_test.cc index d56be01d..093e68db 100644 --- a/test/runtime/subtree_test.cc +++ b/test/runtime/subtree_test.cc @@ -41,7 +41,7 @@ describe("Subtree", []() { SubtreePool pool; before_each([&]() { - ts_subtree_pool_init(&pool); + pool = ts_subtree_pool_new(10); }); after_each([&]() { diff --git a/test/runtime/tree_test.cc b/test/runtime/tree_test.cc new file mode 100644 index 00000000..a87c754b --- /dev/null +++ b/test/runtime/tree_test.cc @@ -0,0 +1,200 @@ +#include "test_helper.h" +#include "runtime/alloc.h" +#include "helpers/record_alloc.h" +#include "helpers/stream_methods.h" +#include "helpers/tree_helpers.h" +#include "helpers/point_helpers.h" +#include "helpers/spy_logger.h" +#include "helpers/stderr_logger.h" +#include "helpers/spy_input.h" +#include "helpers/load_language.h" + +TSPoint point(uint32_t row, uint32_t column) { + TSPoint result = {row, column}; + return result; +} + +START_TEST + +describe("Tree", [&]() { + TSParser *parser; + SpyInput *input; + TSTree *tree; + + before_each([&]() { + parser = ts_parser_new(); + }); + + after_each([&]() { + ts_parser_delete(parser); + }); + + auto assert_root_node = [&](const string &expected) { + TSNode node = ts_tree_root_node(tree); + char *node_string = ts_node_string(node); + string actual(node_string); + ts_free(node_string); + AssertThat(actual, Equals(expected)); + }; + + describe("get_changed_ranges()", [&]() { + before_each([&]() { + ts_parser_set_language(parser, load_real_language("javascript")); + input = new SpyInput("{a: null};\n", 3); + tree = ts_parser_parse(parser, nullptr, input->input()); + + assert_root_node( + "(program (expression_statement (object (pair (property_identifier) (null)))))" + ); + }); + + after_each([&]() { + ts_tree_delete(tree); + delete input; + }); + + auto get_changed_ranges_for_edit = [&](function fn) -> vector { + TSInputEdit edit = fn(); + ts_tree_edit(tree, &edit); + + uint32_t range_count = 0; + TSTree *new_tree = ts_parser_parse(parser, tree, input->input()); + TSRange *ranges = ts_tree_get_changed_ranges(tree, new_tree, &range_count); + ts_tree_delete(tree); + tree = new_tree; + + vector result; + for (size_t i = 0; i < range_count; i++) { + result.push_back(ranges[i]); + } + + ts_free(ranges); + return result; + }; + + it("reports changes when one token has been updated", [&]() { + // Replace `null` with `nothing` + auto ranges = get_changed_ranges_for_edit([&]() { + return input->replace(input->content.find("ull"), 1, "othing"); + }); + AssertThat(ranges, Equals(vector({ + TSRange{ + point(0, input->content.find("nothing")), + point(0, input->content.find("}")) + }, + }))); + + // Replace `nothing` with `null` again + ranges = get_changed_ranges_for_edit([&]() { + return input->undo(); + }); + AssertThat(ranges, Equals(vector({ + TSRange{ + point(0, input->content.find("null")), + point(0, input->content.find("}")) + }, + }))); + }); + + it("reports no changes when leading whitespace has changed (regression)", [&]() { + input->chars_per_chunk = 80; + + // Insert leading whitespace + auto ranges = get_changed_ranges_for_edit([&]() { + return input->replace(0, 0, "\n"); + }); + assert_root_node( + "(program (expression_statement (object (pair (property_identifier) (null)))))" + ); + AssertThat(ranges, IsEmpty()); + + // Remove leading whitespace + ranges = get_changed_ranges_for_edit([&]() { + return input->undo(); + }); + assert_root_node( + "(program (expression_statement (object (pair (property_identifier) (null)))))" + ); + AssertThat(ranges, IsEmpty()); + + // Insert leading whitespace again + ranges = get_changed_ranges_for_edit([&]() { + return input->replace(0, 0, "\n"); + }); + assert_root_node( + "(program (expression_statement (object (pair (property_identifier) (null)))))" + ); + AssertThat(ranges, IsEmpty()); + }); + + it("reports changes when tokens have been appended", [&]() { + // Add a second key-value pair + auto ranges = get_changed_ranges_for_edit([&]() { + return input->replace(input->content.find("}"), 0, ", b: false"); + }); + AssertThat(ranges, Equals(vector({ + TSRange{ + point(0, input->content.find(",")), + point(0, input->content.find("}")) + }, + }))); + + // Add a third key-value pair in between the first two + ranges = get_changed_ranges_for_edit([&]() { + return input->replace(input->content.find(", b"), 0, ", c: 1"); + }); + assert_root_node( + "(program (expression_statement (object " + "(pair (property_identifier) (null)) " + "(pair (property_identifier) (number)) " + "(pair (property_identifier) (false)))))" + ); + AssertThat(ranges, Equals(vector({ + TSRange{ + point(0, input->content.find(", c")), + point(0, input->content.find(", b")) + }, + }))); + + // Delete the middle pair. + ranges = get_changed_ranges_for_edit([&]() { + return input->undo(); + }); + assert_root_node( + "(program (expression_statement (object " + "(pair (property_identifier) (null)) " + "(pair (property_identifier) (false)))))" + ); + AssertThat(ranges, IsEmpty()); + + // Delete the second pair. + ranges = get_changed_ranges_for_edit([&]() { + return input->undo(); + }); + assert_root_node( + "(program (expression_statement (object " + "(pair (property_identifier) (null)))))" + ); + AssertThat(ranges, IsEmpty()); + }); + + it("reports changes when trees have been wrapped", [&]() { + // Wrap the object in an assignment expression. + auto ranges = get_changed_ranges_for_edit([&]() { + return input->replace(input->content.find("null"), 0, "b === "); + }); + assert_root_node( + "(program (expression_statement (object " + "(pair (property_identifier) (binary_expression (identifier) (null))))))" + ); + AssertThat(ranges, Equals(vector({ + TSRange{ + point(0, input->content.find("b ===")), + point(0, input->content.find("}")) + }, + }))); + }); + }); +}); + +END_TEST diff --git a/tests.gyp b/tests.gyp index b964a507..ab0de485 100644 --- a/tests.gyp +++ b/tests.gyp @@ -66,12 +66,12 @@ 'test/integration/fuzzing-examples.cc', 'test/integration/real_grammars.cc', 'test/integration/test_grammars.cc', - 'test/runtime/document_test.cc', 'test/runtime/language_test.cc', 'test/runtime/node_test.cc', 'test/runtime/parser_test.cc', 'test/runtime/stack_test.cc', 'test/runtime/subtree_test.cc', + 'test/runtime/tree_test.cc', 'test/tests.cc', ], 'cflags': [ From 199a94cc26640c3add7b23757a1de510d7acf058 Mon Sep 17 00:00:00 2001 From: Max Brunsfeld Date: Fri, 11 May 2018 12:43:04 -0700 Subject: [PATCH 44/90] Allow the parser to print dot graphs to any file --- include/tree_sitter/runtime.h | 12 +++++----- src/runtime/parser.c | 36 ++++++++++++++-------------- test/benchmarks.cc | 2 +- test/integration/fuzzing-examples.cc | 2 +- test/integration/real_grammars.cc | 2 +- test/integration/test_grammars.cc | 2 +- test/runtime/parser_test.cc | 2 +- 7 files changed, 29 insertions(+), 29 deletions(-) diff --git a/include/tree_sitter/runtime.h b/include/tree_sitter/runtime.h index fa6fd919..acd2c3b3 100644 --- a/include/tree_sitter/runtime.h +++ b/include/tree_sitter/runtime.h @@ -34,6 +34,11 @@ typedef struct { uint32_t column; } TSPoint; +typedef struct { + TSPoint start; + TSPoint end; +} TSRange; + typedef struct { void *payload; const char *(*read)(void *payload, uint32_t *bytes_read); @@ -60,11 +65,6 @@ typedef struct { TSPoint extent_added; } TSInputEdit; -typedef struct { - TSPoint start; - TSPoint end; -} TSRange; - typedef struct { const void *subtree; const TSTree *tree; @@ -79,7 +79,7 @@ const TSLanguage *ts_parser_language(const TSParser *); bool ts_parser_set_language(TSParser *, const TSLanguage *); TSLogger ts_parser_logger(const TSParser *); void ts_parser_set_logger(TSParser *, TSLogger); -void ts_parser_print_debugging_graphs(TSParser *, bool); +void ts_parser_print_dot_graphs(TSParser *, FILE *); void ts_parser_halt_on_error(TSParser *, bool); TSTree *ts_parser_parse(TSParser *, const TSTree *, TSInput); TSTree *ts_parser_parse_string(TSParser *, const TSTree *, const char *, uint32_t); diff --git a/src/runtime/parser.c b/src/runtime/parser.c index 1c7da6b1..02e96857 100644 --- a/src/runtime/parser.c +++ b/src/runtime/parser.c @@ -17,21 +17,21 @@ #include "runtime/tree.h" #define LOG(...) \ - if (self->lexer.logger.log || self->print_debugging_graphs) { \ + if (self->lexer.logger.log || self->dot_graph_file) { \ snprintf(self->lexer.debug_buffer, TREE_SITTER_SERIALIZATION_BUFFER_SIZE, __VA_ARGS__); \ ts_parser__log(self); \ } -#define LOG_STACK() \ - if (self->print_debugging_graphs) { \ - ts_stack_print_dot_graph(self->stack, self->language, stderr); \ - fputs("\n\n", stderr); \ +#define LOG_STACK() \ + if (self->dot_graph_file) { \ + ts_stack_print_dot_graph(self->stack, self->language, self->dot_graph_file); \ + fputs("\n\n", self->dot_graph_file); \ } -#define LOG_TREE() \ - if (self->print_debugging_graphs) { \ - ts_subtree_print_dot_graph(self->finished_tree, self->language, stderr); \ - fputs("\n", stderr); \ +#define LOG_TREE() \ + if (self->dot_graph_file) { \ + ts_subtree_print_dot_graph(self->finished_tree, self->language, self->dot_graph_file); \ + fputs("\n", self->dot_graph_file); \ } #define SYM_NAME(symbol) ts_language_symbol_name(self->language, symbol) @@ -58,7 +58,7 @@ struct TSParser { ReusableNode reusable_node; void *external_scanner_payload; bool in_ambiguity; - bool print_debugging_graphs; + FILE *dot_graph_file; bool halt_on_error; unsigned accept_count; }; @@ -89,13 +89,13 @@ static void ts_parser__log(TSParser *self) { ); } - if (self->print_debugging_graphs) { - fprintf(stderr, "graph {\nlabel=\""); + if (self->dot_graph_file) { + fprintf(self->dot_graph_file, "graph {\nlabel=\""); for (char *c = &self->lexer.debug_buffer[0]; *c != 0; c++) { - if (*c == '"') fputc('\\', stderr); - fputc(*c, stderr); + if (*c == '"') fputc('\\', self->dot_graph_file); + fputc(*c, self->dot_graph_file); } - fprintf(stderr, "\"\n}\n\n"); + fprintf(self->dot_graph_file, "\"\n}\n\n"); } } @@ -1297,7 +1297,7 @@ TSParser *ts_parser_new() { self->stack = ts_stack_new(&self->tree_pool); self->finished_tree = NULL; self->reusable_node = reusable_node_new(); - self->print_debugging_graphs = false; + self->dot_graph_file = NULL; self->halt_on_error = false; ts_parser__set_cached_token(self, 0, NULL, NULL); return self; @@ -1345,8 +1345,8 @@ void ts_parser_set_logger(TSParser *self, TSLogger logger) { self->lexer.logger = logger; } -void ts_parser_print_debugging_graphs(TSParser *self, bool should_print) { - self->print_debugging_graphs = should_print; +void ts_parser_print_dot_graphs(TSParser *self, FILE *file) { + self->dot_graph_file = file; } void ts_parser_halt_on_error(TSParser *self, bool should_halt_on_error) { diff --git a/test/benchmarks.cc b/test/benchmarks.cc index d4f475b5..6612444e 100644 --- a/test/benchmarks.cc +++ b/test/benchmarks.cc @@ -46,7 +46,7 @@ int main(int argc, char *arg[]) { TSParser *parser = ts_parser_new(); if (getenv("TREE_SITTER_BENCHMARK_SVG")) { - ts_parser_print_debugging_graphs(parser, true); + ts_parser_print_dot_graphs(parser, stderr); } else if (getenv("TREE_SITTER_BENCHMARK_LOG")) { ts_parser_set_logger(parser, stderr_logger_new(false)); } diff --git a/test/integration/fuzzing-examples.cc b/test/integration/fuzzing-examples.cc index 92682d4c..aca95840 100644 --- a/test/integration/fuzzing-examples.cc +++ b/test/integration/fuzzing-examples.cc @@ -32,7 +32,7 @@ describe("examples found via fuzzing", [&]() { TSParser *parser = ts_parser_new(); if (getenv("TREE_SITTER_ENABLE_DEBUG_GRAPHS")) { - ts_parser_print_debugging_graphs(parser, true); + ts_parser_print_dot_graphs(parser, stderr); } const string &language_name = examples[i].first; diff --git a/test/integration/real_grammars.cc b/test/integration/real_grammars.cc index 0d594ce5..a628177f 100644 --- a/test/integration/real_grammars.cc +++ b/test/integration/real_grammars.cc @@ -43,7 +43,7 @@ for (auto &language_name : test_languages) { // ts_parser_set_logger(parser, stderr_logger_new(true)); if (debug_graphs_enabled) { - ts_parser_print_debugging_graphs(parser, true); + ts_parser_print_dot_graphs(parser, stderr); } }); diff --git a/test/integration/test_grammars.cc b/test/integration/test_grammars.cc index bba40ace..62174855 100644 --- a/test/integration/test_grammars.cc +++ b/test/integration/test_grammars.cc @@ -56,7 +56,7 @@ for (auto &language_name : test_languages) { ts_parser_set_language(parser, language); if (getenv("TREE_SITTER_ENABLE_DEBUG_GRAPHS")) { - ts_parser_print_debugging_graphs(parser, true); + ts_parser_print_dot_graphs(parser, stderr); } TSTree *tree = ts_parser_parse_string(parser, nullptr, entry.input.c_str(), entry.input.size()); diff --git a/test/runtime/parser_test.cc b/test/runtime/parser_test.cc index e9cfbe72..7d0b2d1d 100644 --- a/test/runtime/parser_test.cc +++ b/test/runtime/parser_test.cc @@ -27,7 +27,7 @@ describe("Parser", [&]() { tree = nullptr; parser = ts_parser_new(); if (getenv("TREE_SITTER_ENABLE_DEBUG_GRAPHS")) { - ts_parser_print_debugging_graphs(parser, true); + ts_parser_print_dot_graphs(parser, stderr); } }); From cd55d5275d1cce5129d62486ec445ad1760f1a3c Mon Sep 17 00:00:00 2001 From: Max Brunsfeld Date: Fri, 11 May 2018 12:43:16 -0700 Subject: [PATCH 45/90] Update the readme to reflect the new API --- README.md | 23 +++++++++++++---------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/README.md b/README.md index 2feacc25..330dcf86 100644 --- a/README.md +++ b/README.md @@ -179,25 +179,28 @@ tokens, like `(` and `+`. This is useful when analyzing the meaning of a documen TSLanguage *tree_sitter_arithmetic(); int main() { - TSDocument *document = ts_document_new(); - ts_document_set_language(document, tree_sitter_arithmetic()); - ts_document_set_input_string(document, "a + b * 5"); - ts_document_parse(document); + TSParser *parser = ts_parser_new(); + ts_parser_set_language(parser, tree_sitter_arithmetic()); - TSNode root_node = ts_document_root_node(document); - assert(!strcmp(ts_node_type(root_node, document), "expression")); + const char *source_code = "a + b * 5"; + TSTree *tree = ts_parser_parse(parser, NULL, source_code, strlen(source_code)); + + TSNode root_node = ts_tree_root_node(tree); + assert(!strcmp(ts_node_type(root_node), "expression")); assert(ts_node_named_child_count(root_node) == 1); TSNode sum_node = ts_node_named_child(root_node, 0); - assert(!strcmp(ts_node_type(sum_node, document), "sum")); + assert(!strcmp(ts_node_type(sum_node), "sum")); assert(ts_node_named_child_count(sum_node) == 2); TSNode product_node = ts_node_child(ts_node_named_child(sum_node, 1), 0); - assert(!strcmp(ts_node_type(product_node, document), "product")); + assert(!strcmp(ts_node_type(product_node), "product")); assert(ts_node_named_child_count(product_node) == 2); - printf("Syntax tree: %s\n", ts_node_string(root_node, document)); - ts_document_free(document); + printf("Syntax tree: %s\n", ts_node_string(root_node)); + + ts_tree_delete(tree); + ts_parser_delete(parser); return 0; } ``` From f0c7295d272f383e04b4f418385106269683cb2a Mon Sep 17 00:00:00 2001 From: Max Brunsfeld Date: Fri, 11 May 2018 12:47:49 -0700 Subject: [PATCH 46/90] Update fuzz driver to use new API --- test/fuzz/fuzzer.cc | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/test/fuzz/fuzzer.cc b/test/fuzz/fuzzer.cc index 7b1aea5b..570dfa7e 100644 --- a/test/fuzz/fuzzer.cc +++ b/test/fuzz/fuzzer.cc @@ -12,16 +12,15 @@ extern "C" const TSLanguage *TS_LANG(); extern "C" int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) { const char *str = reinterpret_cast(data); - TSDocument *document = ts_document_new(); - ts_document_set_language(document, TS_LANG()); - ts_document_set_input_string_with_length(document, str, size); + TSParser *parser = ts_parser_new(); + ts_parser_set_language(parser, TS_LANG()); + ts_parser_halt_on_error(parser, TS_HALT_ON_ERROR); - TSParseOptions options = {}; - options.halt_on_error = TS_HALT_ON_ERROR; - ts_document_parse_with_options(document, options); + TSTree *tree = ts_parser_parse_string(parser, NULL, str, size); + TSNode root_node = ts_document_root_node(tree); - TSNode root_node = ts_document_root_node(document); - ts_document_free(document); + ts_tree_delete(tree); + ts_parser_delete(parser); return 0; } From bf1bb1604fd1e667a35026cc803486299008aac4 Mon Sep 17 00:00:00 2001 From: Max Brunsfeld Date: Fri, 11 May 2018 12:57:41 -0700 Subject: [PATCH 47/90] Rename TSExternalTokenState -> ExternalScannerState --- include/tree_sitter/parser.h | 8 +++++--- src/runtime/parser.c | 10 +++++----- src/runtime/stack.c | 10 +++++----- src/runtime/subtree.c | 37 ++++++++++++++++++------------------ src/runtime/subtree.h | 10 +++++----- test/runtime/stack_test.cc | 12 ++++++------ 6 files changed, 45 insertions(+), 42 deletions(-) diff --git a/include/tree_sitter/parser.h b/include/tree_sitter/parser.h index d99655f8..35e77a87 100644 --- a/include/tree_sitter/parser.h +++ b/include/tree_sitter/parser.h @@ -9,13 +9,14 @@ extern "C" { #include #include -typedef uint16_t TSSymbol; -typedef uint16_t TSStateId; - #define ts_builtin_sym_error ((TSSymbol)-1) #define ts_builtin_sym_end 0 #define TREE_SITTER_SERIALIZATION_BUFFER_SIZE 1024 +typedef uint16_t TSSymbol; + +typedef uint16_t TSStateId; + typedef struct { bool visible : 1; bool named : 1; @@ -129,6 +130,7 @@ typedef struct TSLanguage { */ #define STATE(id) id + #define ACTIONS(id) id #define SHIFT(state_value) \ diff --git a/src/runtime/parser.c b/src/runtime/parser.c index 02e96857..2115435e 100644 --- a/src/runtime/parser.c +++ b/src/runtime/parser.c @@ -247,8 +247,8 @@ static void ts_parser__restore_external_scanner(TSParser *self, Subtree *externa if (external_token) { self->language->external_scanner.deserialize( self->external_scanner_payload, - ts_external_token_state_data(&external_token->external_token_state), - external_token->external_token_state.length + ts_external_scanner_state_data(&external_token->external_scanner_state), + external_token->external_scanner_state.length ); } else { self->language->external_scanner.deserialize(self->external_scanner_payload, NULL, 0); @@ -391,7 +391,7 @@ static Subtree *ts_parser__lex(TSParser *self, StackVersion version, TSStateId p self->external_scanner_payload, self->lexer.debug_buffer ); - ts_external_token_state_init(&result->external_token_state, self->lexer.debug_buffer, length); + ts_external_scanner_state_init(&result->external_scanner_state, self->lexer.debug_buffer, length); } } @@ -408,7 +408,7 @@ static Subtree *ts_parser__get_cached_token(TSParser *self, size_t byte_index, TokenCache *cache = &self->token_cache; if (cache->token && cache->byte_index == byte_index && - ts_subtree_external_token_state_eq(cache->last_external_token, last_external_token)) { + ts_subtree_external_scanner_state_eq(cache->last_external_token, last_external_token)) { return cache->token; } else { return NULL; @@ -464,7 +464,7 @@ static Subtree *ts_parser__get_lookahead(TSParser *self, StackVersion version, T continue; } - if (!ts_subtree_external_token_state_eq(reusable_node->last_external_token, last_external_token)) { + if (!ts_subtree_external_scanner_state_eq(reusable_node->last_external_token, last_external_token)) { LOG("reusable_node_has_different_external_scanner_state symbol:%s", SYM_NAME(result->symbol)); reusable_node_advance(reusable_node); continue; diff --git a/src/runtime/stack.c b/src/runtime/stack.c index c8af7a58..21468e49 100644 --- a/src/runtime/stack.c +++ b/src/runtime/stack.c @@ -164,7 +164,7 @@ static bool stack__subtree_is_equivalent(const Subtree *left, const Subtree *rig left->padding.bytes == right->padding.bytes && left->size.bytes == right->size.bytes && left->extra == right->extra && - ts_subtree_external_token_state_eq(left, right)))); + ts_subtree_external_scanner_state_eq(left, right)))); } static void stack_node_add_link(StackNode *self, StackLink link) { @@ -605,7 +605,7 @@ bool ts_stack_can_merge(Stack *self, StackVersion version1, StackVersion version head1->node->state == head2->node->state && head1->node->position.bytes == head2->node->position.bytes && head1->node->error_cost == head2->node->error_cost && - ts_subtree_external_token_state_eq(head1->last_external_token, head2->last_external_token); + ts_subtree_external_scanner_state_eq(head1->last_external_token, head2->last_external_token); } void ts_stack_halt(Stack *self, StackVersion version) { @@ -684,9 +684,9 @@ bool ts_stack_print_dot_graph(Stack *self, const TSLanguage *language, FILE *f) ); if (head->last_external_token) { - TSExternalTokenState *state = &head->last_external_token->external_token_state; - const char *data = ts_external_token_state_data(state); - fprintf(f, "\nexternal_token_state:"); + ExternalScannerState *state = &head->last_external_token->external_scanner_state; + const char *data = ts_external_scanner_state_data(state); + fprintf(f, "\nexternal_scanner_state:"); for (uint32_t j = 0; j < state->length; j++) fprintf(f, " %2X", data[j]); } diff --git a/src/runtime/subtree.c b/src/runtime/subtree.c index aff2a6fc..c3da95c5 100644 --- a/src/runtime/subtree.c +++ b/src/runtime/subtree.c @@ -21,27 +21,27 @@ TSStateId TS_TREE_STATE_NONE = USHRT_MAX; static const uint32_t MAX_TREE_POOL_SIZE = 1024; -static const TSExternalTokenState empty_state = {.length = 0, .short_data = {0}}; +static const ExternalScannerState empty_state = {.length = 0, .short_data = {0}}; -// ExternalTokenState +// ExternalScannerState -void ts_external_token_state_init(TSExternalTokenState *self, const char *content, unsigned length) { +void ts_external_scanner_state_init(ExternalScannerState *self, const char *data, unsigned length) { self->length = length; if (length > sizeof(self->short_data)) { self->long_data = ts_malloc(length); - memcpy(self->long_data, content, length); + memcpy(self->long_data, data, length); } else { - memcpy(self->short_data, content, length); + memcpy(self->short_data, data, length); } } -void ts_external_token_state_delete(TSExternalTokenState *self) { +void ts_external_scanner_state_delete(ExternalScannerState *self) { if (self->length > sizeof(self->short_data)) { ts_free(self->long_data); } } -const char *ts_external_token_state_data(const TSExternalTokenState *self) { +const char *ts_external_scanner_state_data(const ExternalScannerState *self) { if (self->length > sizeof(self->short_data)) { return self->long_data; } else { @@ -49,10 +49,11 @@ const char *ts_external_token_state_data(const TSExternalTokenState *self) { } } -bool ts_external_token_state_eq(const TSExternalTokenState *a, const TSExternalTokenState *b) { - return a == b || - (a->length == b->length && - memcmp(ts_external_token_state_data(a), ts_external_token_state_data(b), a->length) == 0); +bool ts_external_scanner_state_eq(const ExternalScannerState *a, const ExternalScannerState *b) { + return a == b || ( + a->length == b->length && + !memcmp(ts_external_scanner_state_data(a), ts_external_scanner_state_data(b), a->length) + ); } // SubtreeArray @@ -403,7 +404,7 @@ void ts_subtree_release(SubtreePool *pool, Subtree *self) { } array_delete(&tree->children); } else if (tree->has_external_tokens) { - ts_external_token_state_delete(&tree->external_token_state); + ts_external_scanner_state_delete(&tree->external_scanner_state); } ts_subtree_pool_free(pool, tree); } @@ -694,10 +695,10 @@ void ts_subtree_print_dot_graph(const Subtree *self, const TSLanguage *language, fprintf(f, "}\n"); } -bool ts_subtree_external_token_state_eq(const Subtree *self, const Subtree *other) { - const TSExternalTokenState *state1 = &empty_state; - const TSExternalTokenState *state2 = &empty_state; - if (self && self->has_external_tokens) state1 = &self->external_token_state; - if (other && other->has_external_tokens) state2 = &other->external_token_state; - return ts_external_token_state_eq(state1, state2); +bool ts_subtree_external_scanner_state_eq(const Subtree *self, const Subtree *other) { + const ExternalScannerState *state1 = &empty_state; + const ExternalScannerState *state2 = &empty_state; + if (self && self->has_external_tokens) state1 = &self->external_scanner_state; + if (other && other->has_external_tokens) state2 = &other->external_scanner_state; + return ts_external_scanner_state_eq(state1, state2); } diff --git a/src/runtime/subtree.h b/src/runtime/subtree.h index fd1fcaff..2f7d60b0 100644 --- a/src/runtime/subtree.h +++ b/src/runtime/subtree.h @@ -20,7 +20,7 @@ typedef struct { char short_data[sizeof(char *) + sizeof(uint32_t)]; }; uint32_t length; -} TSExternalTokenState; +} ExternalScannerState; typedef struct Subtree Subtree; @@ -61,7 +61,7 @@ struct Subtree { }; struct { uint32_t _2; - TSExternalTokenState external_token_state; + ExternalScannerState external_scanner_state; }; struct { uint32_t _1; @@ -75,8 +75,8 @@ typedef struct { SubtreeArray tree_stack; } SubtreePool; -void ts_external_token_state_init(TSExternalTokenState *, const char *, unsigned); -const char *ts_external_token_state_data(const TSExternalTokenState *); +void ts_external_scanner_state_init(ExternalScannerState *, const char *, unsigned); +const char *ts_external_scanner_state_data(const ExternalScannerState *); bool ts_subtree_array_copy(SubtreeArray, SubtreeArray *); void ts_subtree_array_delete(SubtreePool *, SubtreeArray *); @@ -104,7 +104,7 @@ Subtree *ts_subtree_edit(Subtree *, const TSInputEdit *edit, SubtreePool *); char *ts_subtree_string(const Subtree *, const TSLanguage *, bool include_all); void ts_subtree_print_dot_graph(const Subtree *, const TSLanguage *, FILE *); Subtree *ts_subtree_last_external_token(Subtree *); -bool ts_subtree_external_token_state_eq(const Subtree *, const Subtree *); +bool ts_subtree_external_scanner_state_eq(const Subtree *, const Subtree *); static inline uint32_t ts_subtree_total_bytes(const Subtree *self) { return self->padding.bytes + self->size.bytes; diff --git a/test/runtime/stack_test.cc b/test/runtime/stack_test.cc index ae5b1589..a6ec09c4 100644 --- a/test/runtime/stack_test.cc +++ b/test/runtime/stack_test.cc @@ -528,8 +528,8 @@ describe("Stack", [&]() { before_each([&]() { subtrees[1]->has_external_tokens = true; subtrees[2]->has_external_tokens = true; - ts_external_token_state_init(&subtrees[1]->external_token_state, NULL, 0); - ts_external_token_state_init(&subtrees[2]->external_token_state, NULL, 0); + ts_external_scanner_state_init(&subtrees[1]->external_scanner_state, NULL, 0); + ts_external_scanner_state_init(&subtrees[2]->external_scanner_state, NULL, 0); }); it("allows the state to be retrieved", [&]() { @@ -546,8 +546,8 @@ describe("Stack", [&]() { }); it("does not merge stack versions with different external token states", [&]() { - ts_external_token_state_init(&subtrees[1]->external_token_state, "abcd", 2); - ts_external_token_state_init(&subtrees[2]->external_token_state, "ABCD", 2); + ts_external_scanner_state_init(&subtrees[1]->external_scanner_state, "abcd", 2); + ts_external_scanner_state_init(&subtrees[2]->external_scanner_state, "ABCD", 2); ts_stack_copy_version(stack, 0); push(0, subtrees[0], 5); @@ -560,8 +560,8 @@ describe("Stack", [&]() { }); it("merges stack versions with identical external token states", [&]() { - ts_external_token_state_init(&subtrees[1]->external_token_state, "abcd", 2); - ts_external_token_state_init(&subtrees[2]->external_token_state, "abcd", 2); + ts_external_scanner_state_init(&subtrees[1]->external_scanner_state, "abcd", 2); + ts_external_scanner_state_init(&subtrees[2]->external_scanner_state, "abcd", 2); ts_stack_copy_version(stack, 0); push(0, subtrees[0], 5); From 20c183b7cd7f33a61923b647ddd238f8721db748 Mon Sep 17 00:00:00 2001 From: Max Brunsfeld Date: Fri, 11 May 2018 13:02:12 -0700 Subject: [PATCH 48/90] Rename ts_subtree_make_* -> ts_subtree_new_* --- src/runtime/parser.c | 26 +++++++-------- src/runtime/subtree.c | 28 ++++++++-------- src/runtime/subtree.h | 16 ++++----- test/runtime/stack_test.cc | 2 +- test/runtime/subtree_test.cc | 64 ++++++++++++++++++------------------ 5 files changed, 66 insertions(+), 70 deletions(-) diff --git a/src/runtime/parser.c b/src/runtime/parser.c index 2115435e..8332501b 100644 --- a/src/runtime/parser.c +++ b/src/runtime/parser.c @@ -358,7 +358,7 @@ static Subtree *ts_parser__lex(TSParser *self, StackVersion version, TSStateId p if (skipped_error) { Length padding = length_sub(error_start_position, start_position); Length size = length_sub(error_end_position, error_start_position); - result = ts_subtree_make_error(&self->tree_pool, size, padding, first_error_character, self->language); + result = ts_subtree_new_error(&self->tree_pool, size, padding, first_error_character, self->language); } else { if (self->lexer.token_end_position.bytes < self->lexer.token_start_position.bytes) { self->lexer.token_start_position = self->lexer.token_end_position; @@ -383,7 +383,7 @@ static Subtree *ts_parser__lex(TSParser *self, StackVersion version, TSStateId p } } - result = ts_subtree_make_leaf(&self->tree_pool, symbol, padding, size, self->language); + result = ts_subtree_new_leaf(&self->tree_pool, symbol, padding, size, self->language); if (found_external_token) { result->has_external_tokens = true; @@ -577,7 +577,7 @@ static void ts_parser__shift(TSParser *self, StackVersion version, TSStateId sta Subtree *lookahead, bool extra) { if (extra != lookahead->extra) { if (ts_stack_version_count(self->stack) > 1) { - lookahead = ts_subtree_make_copy(&self->tree_pool, lookahead); + lookahead = ts_subtree_new_copy(&self->tree_pool, lookahead); } else { ts_subtree_retain(lookahead); } @@ -625,7 +625,7 @@ static StackSliceArray ts_parser__reduce(TSParser *self, StackVersion version, T children.size--; } - Subtree *parent = ts_subtree_make_node(&self->tree_pool, + Subtree *parent = ts_subtree_new_node(&self->tree_pool, symbol, &children, alias_sequence_id, self->language ); @@ -735,7 +735,7 @@ static void ts_parser__accept(TSParser *self, StackVersion version, Subtree *loo ts_subtree_retain(child->children.contents[k]); } array_splice(&trees, j, 1, &child->children); - root = ts_subtree_make_node( + root = ts_subtree_new_node( &self->tree_pool, child->symbol, &trees, child->alias_sequence_id, self->language ); @@ -875,7 +875,7 @@ static void ts_parser__handle_error(TSParser *self, StackVersion version, lookahead_symbol )) { StackVersion version_with_missing_tree = ts_stack_copy_version(self->stack, v); - Subtree *missing_tree = ts_subtree_make_missing_leaf(&self->tree_pool, missing_symbol, self->language); + Subtree *missing_tree = ts_subtree_new_missing_leaf(&self->tree_pool, missing_symbol, self->language); ts_stack_push( self->stack, version_with_missing_tree, missing_tree, false, @@ -920,15 +920,15 @@ static void ts_parser__halt_parse(TSParser *self) { ts_stack_position(self->stack, 0) ); - Subtree *filler_node = ts_subtree_make_error(&self->tree_pool, remaining_length, length_zero(), 0, self->language); + Subtree *filler_node = ts_subtree_new_error(&self->tree_pool, remaining_length, length_zero(), 0, self->language); filler_node->visible = false; ts_stack_push(self->stack, 0, filler_node, false, 0); SubtreeArray children = array_new(); - Subtree *root_error = ts_subtree_make_error_node(&self->tree_pool, &children, self->language); + Subtree *root_error = ts_subtree_new_error_node(&self->tree_pool, &children, self->language); ts_stack_push(self->stack, 0, root_error, false, 0); - Subtree *eof = ts_subtree_make_leaf(&self->tree_pool, ts_builtin_sym_end, length_zero(), length_zero(), self->language); + Subtree *eof = ts_subtree_new_leaf(&self->tree_pool, ts_builtin_sym_end, length_zero(), length_zero(), self->language); ts_parser__accept(self, 0, eof); ts_subtree_release(&self->tree_pool, eof); } @@ -967,7 +967,7 @@ static bool ts_parser__recover_to_state(TSParser *self, StackVersion version, un SubtreeArray trailing_extras = ts_subtree_array_remove_trailing_extras(&slice.subtrees); if (slice.subtrees.size > 0) { - Subtree *error = ts_subtree_make_error_node(&self->tree_pool, &slice.subtrees, self->language); + Subtree *error = ts_subtree_new_error_node(&self->tree_pool, &slice.subtrees, self->language); error->extra = true; ts_stack_push(self->stack, slice.version, error, false, goal_state); } else { @@ -1048,7 +1048,7 @@ static void ts_parser__recover(TSParser *self, StackVersion version, Subtree *lo if (lookahead->symbol == ts_builtin_sym_end) { LOG("recover_eof"); SubtreeArray children = array_new(); - Subtree *parent = ts_subtree_make_error_node(&self->tree_pool, &children, self->language); + Subtree *parent = ts_subtree_new_error_node(&self->tree_pool, &children, self->language); ts_stack_push(self->stack, version, parent, false, 1); ts_parser__accept(self, version, lookahead); return; @@ -1075,7 +1075,7 @@ static void ts_parser__recover(TSParser *self, StackVersion version, Subtree *lo SubtreeArray children = array_new(); array_reserve(&children, 1); array_push(&children, lookahead); - Subtree *error_repeat = ts_subtree_make_node( + Subtree *error_repeat = ts_subtree_new_node( &self->tree_pool, ts_builtin_sym_error_repeat, &children, @@ -1089,7 +1089,7 @@ static void ts_parser__recover(TSParser *self, StackVersion version, Subtree *lo assert(pop.contents[0].subtrees.size == 1); ts_stack_renumber_version(self->stack, pop.contents[0].version, version); array_push(&pop.contents[0].subtrees, error_repeat); - error_repeat = ts_subtree_make_node( + error_repeat = ts_subtree_new_node( &self->tree_pool, ts_builtin_sym_error_repeat, &pop.contents[0].subtrees, diff --git a/src/runtime/subtree.c b/src/runtime/subtree.c index c3da95c5..0cd125bf 100644 --- a/src/runtime/subtree.c +++ b/src/runtime/subtree.c @@ -141,7 +141,7 @@ void ts_subtree_pool_free(SubtreePool *self, Subtree *tree) { // Subtree -Subtree *ts_subtree_make_leaf(SubtreePool *pool, TSSymbol symbol, Length padding, Length size, +Subtree *ts_subtree_new_leaf(SubtreePool *pool, TSSymbol symbol, Length padding, Length size, const TSLanguage *language) { TSSymbolMetadata metadata = ts_language_symbol_metadata(language, symbol); Subtree *result = ts_subtree_pool_allocate(pool); @@ -166,16 +166,16 @@ Subtree *ts_subtree_make_leaf(SubtreePool *pool, TSSymbol symbol, Length padding return result; } -Subtree *ts_subtree_make_error(SubtreePool *pool, Length size, Length padding, +Subtree *ts_subtree_new_error(SubtreePool *pool, Length size, Length padding, int32_t lookahead_char, const TSLanguage *language) { - Subtree *result = ts_subtree_make_leaf(pool, ts_builtin_sym_error, padding, size, language); + Subtree *result = ts_subtree_new_leaf(pool, ts_builtin_sym_error, padding, size, language); result->fragile_left = true; result->fragile_right = true; result->lookahead_char = lookahead_char; return result; } -Subtree *ts_subtree_make_copy(SubtreePool *pool, Subtree *self) { +Subtree *ts_subtree_new_copy(SubtreePool *pool, Subtree *self) { Subtree *result = ts_subtree_pool_allocate(pool); *result = *self; if (result->children.size > 0) { @@ -185,11 +185,11 @@ Subtree *ts_subtree_make_copy(SubtreePool *pool, Subtree *self) { return result; } -static Subtree *ts_subtree_make_mut(SubtreePool *pool, Subtree *self) { +static Subtree *ts_subtree_new_mut(SubtreePool *pool, Subtree *self) { if (self->ref_count == 1) { return self; } else { - Subtree *result = ts_subtree_make_copy(pool, self); + Subtree *result = ts_subtree_new_copy(pool, self); ts_subtree_release(pool, self); return result; } @@ -360,9 +360,9 @@ void ts_subtree_set_children(Subtree *self, SubtreeArray *children, const TSLang } } -Subtree *ts_subtree_make_node(SubtreePool *pool, TSSymbol symbol, SubtreeArray *children, +Subtree *ts_subtree_new_node(SubtreePool *pool, TSSymbol symbol, SubtreeArray *children, unsigned alias_sequence_id, const TSLanguage *language) { - Subtree *result = ts_subtree_make_leaf(pool, symbol, length_zero(), length_zero(), language); + Subtree *result = ts_subtree_new_leaf(pool, symbol, length_zero(), length_zero(), language); result->alias_sequence_id = alias_sequence_id; if (symbol == ts_builtin_sym_error || symbol == ts_builtin_sym_error_repeat) { result->fragile_left = true; @@ -372,14 +372,14 @@ Subtree *ts_subtree_make_node(SubtreePool *pool, TSSymbol symbol, SubtreeArray * return result; } -Subtree *ts_subtree_make_error_node(SubtreePool *pool, SubtreeArray *children, +Subtree *ts_subtree_new_error_node(SubtreePool *pool, SubtreeArray *children, const TSLanguage *language) { - return ts_subtree_make_node(pool, ts_builtin_sym_error, children, 0, language); + return ts_subtree_new_node(pool, ts_builtin_sym_error, children, 0, language); } -Subtree *ts_subtree_make_missing_leaf(SubtreePool *pool, TSSymbol symbol, +Subtree *ts_subtree_new_missing_leaf(SubtreePool *pool, TSSymbol symbol, const TSLanguage *language) { - Subtree *result = ts_subtree_make_leaf(pool, symbol, length_zero(), length_zero(), language); + Subtree *result = ts_subtree_new_leaf(pool, symbol, length_zero(), length_zero(), language); result->is_missing = true; result->error_cost = ERROR_COST_PER_MISSING_TREE + ERROR_COST_PER_RECOVERY; return result; @@ -464,7 +464,7 @@ Subtree *ts_subtree_invalidate_lookahead(Subtree *self, uint32_t edit_byte_offse SubtreePool *pool) { if (edit_byte_offset >= self->bytes_scanned) return self; - Subtree *result = ts_subtree_make_mut(pool, self); + Subtree *result = ts_subtree_new_mut(pool, self); result->has_changes = true; if (result->children.size > 0) { @@ -484,7 +484,7 @@ Subtree *ts_subtree__edit(Subtree *self, Edit edit, SubtreePool *pool) { Length new_end = length_add(edit.start, edit.added); Length old_end = length_add(edit.start, edit.removed); - Subtree *result = ts_subtree_make_mut(pool, self); + Subtree *result = ts_subtree_new_mut(pool, self); result->has_changes = true; if (old_end.bytes <= result->padding.bytes) { diff --git a/src/runtime/subtree.h b/src/runtime/subtree.h index 2f7d60b0..da69383d 100644 --- a/src/runtime/subtree.h +++ b/src/runtime/subtree.h @@ -88,12 +88,12 @@ void ts_subtree_pool_delete(SubtreePool *); Subtree *ts_subtree_pool_allocate(SubtreePool *); void ts_subtree_pool_free(SubtreePool *, Subtree *); -Subtree *ts_subtree_make_leaf(SubtreePool *, TSSymbol, Length, Length, const TSLanguage *); -Subtree *ts_subtree_make_node(SubtreePool *, TSSymbol, SubtreeArray *, unsigned, const TSLanguage *); -Subtree *ts_subtree_make_copy(SubtreePool *, Subtree *child); -Subtree *ts_subtree_make_error_node(SubtreePool *, SubtreeArray *, const TSLanguage *); -Subtree *ts_subtree_make_error(SubtreePool *, Length, Length, int32_t, const TSLanguage *); -Subtree *ts_subtree_make_missing_leaf(SubtreePool *, TSSymbol, const TSLanguage *); +Subtree *ts_subtree_new_leaf(SubtreePool *, TSSymbol, Length, Length, const TSLanguage *); +Subtree *ts_subtree_new_node(SubtreePool *, TSSymbol, SubtreeArray *, unsigned, const TSLanguage *); +Subtree *ts_subtree_new_copy(SubtreePool *, Subtree *child); +Subtree *ts_subtree_new_error_node(SubtreePool *, SubtreeArray *, const TSLanguage *); +Subtree *ts_subtree_new_error(SubtreePool *, Length, Length, int32_t, const TSLanguage *); +Subtree *ts_subtree_new_missing_leaf(SubtreePool *, TSSymbol, const TSLanguage *); void ts_subtree_retain(Subtree *tree); void ts_subtree_release(SubtreePool *, Subtree *tree); bool ts_subtree_eq(const Subtree *tree1, const Subtree *tree2); @@ -110,10 +110,6 @@ static inline uint32_t ts_subtree_total_bytes(const Subtree *self) { return self->padding.bytes + self->size.bytes; } -static inline uint32_t ts_subtree_total_rows(const Subtree *self) { - return self->padding.extent.row + self->size.extent.row; -} - static inline Length ts_subtree_total_size(const Subtree *self) { return length_add(self->padding, self->size); } diff --git a/test/runtime/stack_test.cc b/test/runtime/stack_test.cc index a6ec09c4..67812a07 100644 --- a/test/runtime/stack_test.cc +++ b/test/runtime/stack_test.cc @@ -84,7 +84,7 @@ describe("Stack", [&]() { dummy_language.symbol_metadata = symbol_metadata; for (size_t i = 0; i < subtree_count; i++) { - subtrees[i] = ts_subtree_make_leaf(&pool, i, length_zero(), tree_len, &dummy_language); + subtrees[i] = ts_subtree_new_leaf(&pool, i, length_zero(), tree_len, &dummy_language); } }); diff --git a/test/runtime/subtree_test.cc b/test/runtime/subtree_test.cc index 093e68db..4d32ec32 100644 --- a/test/runtime/subtree_test.cc +++ b/test/runtime/subtree_test.cc @@ -50,7 +50,7 @@ describe("Subtree", []() { describe("make_leaf", [&]() { it("does not mark the tree as fragile", [&]() { - Subtree *tree = ts_subtree_make_leaf(&pool, symbol1, {2, {0, 1}}, {5, {0, 4}}, &language); + Subtree *tree = ts_subtree_new_leaf(&pool, symbol1, {2, {0, 1}}, {5, {0, 4}}, &language); AssertThat(tree->fragile_left, IsFalse()); AssertThat(tree->fragile_right, IsFalse()); @@ -60,7 +60,7 @@ describe("Subtree", []() { describe("make_error", [&]() { it("marks the tree as fragile", [&]() { - Subtree *error_tree = ts_subtree_make_error( + Subtree *error_tree = ts_subtree_new_error( &pool, length_zero(), length_zero(), @@ -79,12 +79,12 @@ describe("Subtree", []() { Subtree *tree1, *tree2, *parent1; before_each([&]() { - tree1 = ts_subtree_make_leaf(&pool, symbol1, {2, {0, 1}}, {5, {0, 4}}, &language); - tree2 = ts_subtree_make_leaf(&pool, symbol2, {1, {0, 1}}, {3, {0, 3}}, &language); + tree1 = ts_subtree_new_leaf(&pool, symbol1, {2, {0, 1}}, {5, {0, 4}}, &language); + tree2 = ts_subtree_new_leaf(&pool, symbol2, {1, {0, 1}}, {3, {0, 3}}, &language); ts_subtree_retain(tree1); ts_subtree_retain(tree2); - parent1 = ts_subtree_make_node(&pool, symbol3, tree_array({ + parent1 = ts_subtree_new_node(&pool, symbol3, tree_array({ tree1, tree2, }), 0, &language); @@ -112,7 +112,7 @@ describe("Subtree", []() { ts_subtree_retain(tree1); ts_subtree_retain(tree2); - parent = ts_subtree_make_node(&pool, symbol3, tree_array({ + parent = ts_subtree_new_node(&pool, symbol3, tree_array({ tree1, tree2, }), 0, &language); @@ -136,7 +136,7 @@ describe("Subtree", []() { ts_subtree_retain(tree1); ts_subtree_retain(tree2); - parent = ts_subtree_make_node(&pool, symbol3, tree_array({ + parent = ts_subtree_new_node(&pool, symbol3, tree_array({ tree1, tree2, }), 0, &language); @@ -160,7 +160,7 @@ describe("Subtree", []() { ts_subtree_retain(tree1); ts_subtree_retain(tree2); - parent = ts_subtree_make_node(&pool, symbol3, tree_array({ + parent = ts_subtree_new_node(&pool, symbol3, tree_array({ tree1, tree2, }), 0, &language); @@ -181,10 +181,10 @@ describe("Subtree", []() { Subtree *tree; before_each([&]() { - tree = ts_subtree_make_node(&pool, symbol1, tree_array({ - ts_subtree_make_leaf(&pool, symbol2, {2, {0, 2}}, {3, {0, 3}}, &language), - ts_subtree_make_leaf(&pool, symbol3, {2, {0, 2}}, {3, {0, 3}}, &language), - ts_subtree_make_leaf(&pool, symbol4, {2, {0, 2}}, {3, {0, 3}}, &language), + tree = ts_subtree_new_node(&pool, symbol1, tree_array({ + ts_subtree_new_leaf(&pool, symbol2, {2, {0, 2}}, {3, {0, 3}}, &language), + ts_subtree_new_leaf(&pool, symbol3, {2, {0, 2}}, {3, {0, 3}}, &language), + ts_subtree_new_leaf(&pool, symbol4, {2, {0, 2}}, {3, {0, 3}}, &language), }), 0, &language); AssertThat(tree->padding, Equals({2, {0, 2}})); @@ -373,7 +373,7 @@ describe("Subtree", []() { Subtree *leaf; before_each([&]() { - leaf = ts_subtree_make_leaf(&pool, symbol1, {2, {0, 1}}, {5, {0, 4}}, &language); + leaf = ts_subtree_new_leaf(&pool, symbol1, {2, {0, 1}}, {5, {0, 4}}, &language); }); after_each([&]() { @@ -381,17 +381,17 @@ describe("Subtree", []() { }); it("returns true for identical trees", [&]() { - Subtree *leaf_copy = ts_subtree_make_leaf(&pool, symbol1, {2, {1, 1}}, {5, {1, 4}}, &language); + Subtree *leaf_copy = ts_subtree_new_leaf(&pool, symbol1, {2, {1, 1}}, {5, {1, 4}}, &language); AssertThat(ts_subtree_eq(leaf, leaf_copy), IsTrue()); - Subtree *parent = ts_subtree_make_node(&pool, symbol2, tree_array({ + Subtree *parent = ts_subtree_new_node(&pool, symbol2, tree_array({ leaf, leaf_copy, }), 0, &language); ts_subtree_retain(leaf); ts_subtree_retain(leaf_copy); - Subtree *parent_copy = ts_subtree_make_node(&pool, symbol2, tree_array({ + Subtree *parent_copy = ts_subtree_new_node(&pool, symbol2, tree_array({ leaf, leaf_copy, }), 0, &language); @@ -406,7 +406,7 @@ describe("Subtree", []() { }); it("returns false for trees with different symbols", [&]() { - Subtree *different_leaf = ts_subtree_make_leaf( + Subtree *different_leaf = ts_subtree_new_leaf( &pool, leaf->symbol + 1, leaf->padding, @@ -419,33 +419,33 @@ describe("Subtree", []() { }); it("returns false for trees with different options", [&]() { - Subtree *different_leaf = ts_subtree_make_leaf(&pool, leaf->symbol, leaf->padding, leaf->size, &language); + Subtree *different_leaf = ts_subtree_new_leaf(&pool, leaf->symbol, leaf->padding, leaf->size, &language); different_leaf->visible = !leaf->visible; AssertThat(ts_subtree_eq(leaf, different_leaf), IsFalse()); ts_subtree_release(&pool, different_leaf); }); it("returns false for trees with different paddings or sizes", [&]() { - Subtree *different_leaf = ts_subtree_make_leaf(&pool, leaf->symbol, {}, leaf->size, &language); + Subtree *different_leaf = ts_subtree_new_leaf(&pool, leaf->symbol, {}, leaf->size, &language); AssertThat(ts_subtree_eq(leaf, different_leaf), IsFalse()); ts_subtree_release(&pool, different_leaf); - different_leaf = ts_subtree_make_leaf(&pool, symbol1, leaf->padding, {}, &language); + different_leaf = ts_subtree_new_leaf(&pool, symbol1, leaf->padding, {}, &language); AssertThat(ts_subtree_eq(leaf, different_leaf), IsFalse()); ts_subtree_release(&pool, different_leaf); }); it("returns false for trees with different children", [&]() { - Subtree *leaf2 = ts_subtree_make_leaf(&pool, symbol2, {1, {0, 1}}, {3, {0, 3}}, &language); + Subtree *leaf2 = ts_subtree_new_leaf(&pool, symbol2, {1, {0, 1}}, {3, {0, 3}}, &language); - Subtree *parent = ts_subtree_make_node(&pool, symbol2, tree_array({ + Subtree *parent = ts_subtree_new_node(&pool, symbol2, tree_array({ leaf, leaf2, }), 0, &language); ts_subtree_retain(leaf); ts_subtree_retain(leaf2); - Subtree *different_parent = ts_subtree_make_node(&pool, symbol2, tree_array({ + Subtree *different_parent = ts_subtree_new_node(&pool, symbol2, tree_array({ leaf2, leaf, }), 0, &language); @@ -473,17 +473,17 @@ describe("Subtree", []() { it("returns the last serialized external token state in the given tree", [&]() { Subtree *tree1, *tree2, *tree3, *tree4, *tree5, *tree6, *tree7, *tree8, *tree9; - tree1 = ts_subtree_make_node(&pool, symbol1, tree_array({ - (tree2 = ts_subtree_make_node(&pool, symbol2, tree_array({ - (tree3 = make_external(ts_subtree_make_leaf(&pool, symbol3, padding, size, &language))), - (tree4 = ts_subtree_make_leaf(&pool, symbol4, padding, size, &language)), - (tree5 = ts_subtree_make_leaf(&pool, symbol5, padding, size, &language)), + tree1 = ts_subtree_new_node(&pool, symbol1, tree_array({ + (tree2 = ts_subtree_new_node(&pool, symbol2, tree_array({ + (tree3 = make_external(ts_subtree_new_leaf(&pool, symbol3, padding, size, &language))), + (tree4 = ts_subtree_new_leaf(&pool, symbol4, padding, size, &language)), + (tree5 = ts_subtree_new_leaf(&pool, symbol5, padding, size, &language)), }), 0, &language)), - (tree6 = ts_subtree_make_node(&pool, symbol6, tree_array({ - (tree7 = ts_subtree_make_node(&pool, symbol7, tree_array({ - (tree8 = ts_subtree_make_leaf(&pool, symbol8, padding, size, &language)), + (tree6 = ts_subtree_new_node(&pool, symbol6, tree_array({ + (tree7 = ts_subtree_new_node(&pool, symbol7, tree_array({ + (tree8 = ts_subtree_new_leaf(&pool, symbol8, padding, size, &language)), }), 0, &language)), - (tree9 = ts_subtree_make_leaf(&pool, symbol9, padding, size, &language)), + (tree9 = ts_subtree_new_leaf(&pool, symbol9, padding, size, &language)), }), 0, &language)), }), 0, &language); From 6bb63f549f87183a8538aa7ca4224379c9d25b09 Mon Sep 17 00:00:00 2001 From: Max Brunsfeld Date: Fri, 11 May 2018 14:59:59 -0700 Subject: [PATCH 49/90] Fix unused lambda captures --- src/compiler/build_tables/lex_item_transitions.cc | 2 +- src/compiler/prepare_grammar/extract_tokens.cc | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/compiler/build_tables/lex_item_transitions.cc b/src/compiler/build_tables/lex_item_transitions.cc index d30bf011..7b4eb611 100644 --- a/src/compiler/build_tables/lex_item_transitions.cc +++ b/src/compiler/build_tables/lex_item_transitions.cc @@ -80,7 +80,7 @@ class TransitionBuilder { public: void apply(const Rule &rule) { rule.match( - [this](const rules::Blank &) {}, + [](const rules::Blank &) {}, [this](const rules::CharacterSet &character_set) { PrecedenceRange precedence; diff --git a/src/compiler/prepare_grammar/extract_tokens.cc b/src/compiler/prepare_grammar/extract_tokens.cc index 2e5cf1d9..93b06be2 100644 --- a/src/compiler/prepare_grammar/extract_tokens.cc +++ b/src/compiler/prepare_grammar/extract_tokens.cc @@ -30,7 +30,7 @@ class SymbolReplacer { Rule apply(const Rule &rule) { return rule.match( - [this](const rules::Blank &blank) -> Rule { + [](const rules::Blank &blank) -> Rule { return blank; }, @@ -110,7 +110,7 @@ class TokenExtractor { public: Rule apply(const rules::Rule &rule) { return rule.match( - [this](const rules::Blank &blank) -> Rule { return blank; }, + [](const rules::Blank &blank) -> Rule { return blank; }, [this](const rules::Metadata &rule) -> Rule { if (rule.params.is_token) { From fe5350617546e26403a6e6b8f8da5068b22959a2 Mon Sep 17 00:00:00 2001 From: Max Brunsfeld Date: Fri, 11 May 2018 15:06:13 -0700 Subject: [PATCH 50/90] Declare subtrees as const wherever possible Co-Authored-By: Rick Winfrey --- include/tree_sitter/runtime.h | 2 +- src/runtime/get_changed_ranges.c | 22 +++--- src/runtime/get_changed_ranges.h | 3 +- src/runtime/node.c | 2 +- src/runtime/parser.c | 108 +++++++++++++------------- src/runtime/reusable_node.h | 8 +- src/runtime/stack.c | 17 ++-- src/runtime/stack.h | 6 +- src/runtime/subtree.c | 129 +++++++++++++++++-------------- src/runtime/subtree.h | 20 ++--- src/runtime/tree.c | 2 +- src/runtime/tree_cursor.c | 8 +- src/runtime/tree_cursor.h | 2 +- test/helpers/tree_helpers.cc | 6 +- test/helpers/tree_helpers.h | 4 +- test/runtime/stack_test.cc | 60 +++++++------- test/runtime/subtree_test.cc | 67 +++++++++------- 17 files changed, 244 insertions(+), 222 deletions(-) diff --git a/include/tree_sitter/runtime.h b/include/tree_sitter/runtime.h index acd2c3b3..8f379718 100644 --- a/include/tree_sitter/runtime.h +++ b/include/tree_sitter/runtime.h @@ -85,7 +85,7 @@ TSTree *ts_parser_parse(TSParser *, const TSTree *, TSInput); TSTree *ts_parser_parse_string(TSParser *, const TSTree *, const char *, uint32_t); TSTree *ts_tree_copy(const TSTree *); -void ts_tree_delete(const TSTree *); +void ts_tree_delete(TSTree *); TSNode ts_tree_root_node(const TSTree *); void ts_tree_edit(TSTree *, const TSInputEdit *); TSRange *ts_tree_get_changed_ranges(const TSTree *, const TSTree *, uint32_t *); diff --git a/src/runtime/get_changed_ranges.c b/src/runtime/get_changed_ranges.c index 032fdaab..2b015d27 100644 --- a/src/runtime/get_changed_ranges.c +++ b/src/runtime/get_changed_ranges.c @@ -31,7 +31,7 @@ typedef struct { bool in_padding; } Iterator; -static Iterator iterator_new(TSTreeCursor *cursor, Subtree *tree, const TSLanguage *language) { +static Iterator iterator_new(TSTreeCursor *cursor, const Subtree *tree, const TSLanguage *language) { array_clear(&cursor->stack); array_push(&cursor->stack, ((TreeCursorEntry){ .subtree = tree, @@ -74,14 +74,14 @@ static bool iterator_tree_is_visible(const Iterator *self) { TreeCursorEntry entry = *array_back(&self->cursor.stack); if (entry.subtree->visible) return true; if (self->cursor.stack.size > 1) { - Subtree *parent = self->cursor.stack.contents[self->cursor.stack.size - 2].subtree; + const Subtree *parent = self->cursor.stack.contents[self->cursor.stack.size - 2].subtree; const TSSymbol *alias_sequence = ts_language_alias_sequence(self->language, parent->alias_sequence_id); return alias_sequence && alias_sequence[entry.structural_child_index] != 0; } return false; } -static void iterator_get_visible_state(const Iterator *self, Subtree **tree, +static void iterator_get_visible_state(const Iterator *self, const Subtree **tree, TSSymbol *alias_symbol, uint32_t *start_byte) { uint32_t i = self->cursor.stack.size - 1; @@ -94,7 +94,7 @@ static void iterator_get_visible_state(const Iterator *self, Subtree **tree, TreeCursorEntry entry = self->cursor.stack.contents[i]; if (i > 0) { - Subtree *parent = self->cursor.stack.contents[i - 1].subtree; + const Subtree *parent = self->cursor.stack.contents[i - 1].subtree; const TSSymbol *alias_sequence = ts_language_alias_sequence( self->language, parent->alias_sequence_id @@ -129,7 +129,7 @@ static bool iterator_descend(Iterator *self, uint32_t goal_position) { Length position = entry.position; uint32_t structural_child_index = 0; for (uint32_t i = 0; i < entry.subtree->children.size; i++) { - Subtree *child = entry.subtree->children.contents[i]; + const Subtree *child = entry.subtree->children.contents[i]; Length child_left = length_add(position, child->padding); Length child_right = length_add(child_left, child->size); @@ -178,13 +178,13 @@ static void iterator_advance(Iterator *self) { TreeCursorEntry entry = array_pop(&self->cursor.stack); if (iterator_done(self)) return; - Subtree *parent = array_back(&self->cursor.stack)->subtree; + const Subtree *parent = array_back(&self->cursor.stack)->subtree; uint32_t child_index = entry.child_index + 1; if (parent->children.size > child_index) { Length position = length_add(entry.position, ts_subtree_total_size(entry.subtree)); uint32_t structural_child_index = entry.structural_child_index; if (!entry.subtree->extra) structural_child_index++; - Subtree *next_child = parent->children.contents[child_index]; + const Subtree *next_child = parent->children.contents[child_index]; array_push(&self->cursor.stack, ((TreeCursorEntry){ .subtree = next_child, @@ -214,7 +214,7 @@ typedef enum { } IteratorComparison; IteratorComparison iterator_compare(const Iterator *old_iter, const Iterator *new_iter) { - Subtree *old_tree = NULL, *new_tree = NULL; + const Subtree *old_tree = NULL, *new_tree = NULL; uint32_t old_start = 0, new_start = 0; TSSymbol old_alias_symbol = 0, new_alias_symbol = 0; iterator_get_visible_state(old_iter, &old_tree, &old_alias_symbol, &old_start); @@ -261,9 +261,9 @@ static inline void iterator_print_state(Iterator *self) { } #endif -unsigned ts_subtree_get_changed_ranges(Subtree *old_tree, Subtree *new_tree, - TSTreeCursor *cursor1, TSTreeCursor *cursor2, - const TSLanguage *language, TSRange **ranges) { +unsigned ts_subtree_get_changed_ranges(const Subtree *old_tree, const Subtree *new_tree, + TSTreeCursor *cursor1, TSTreeCursor *cursor2, + const TSLanguage *language, TSRange **ranges) { RangeArray results = array_new(); Iterator old_iter = iterator_new(cursor1, old_tree, language); diff --git a/src/runtime/get_changed_ranges.h b/src/runtime/get_changed_ranges.h index bf138a21..fbe9cc23 100644 --- a/src/runtime/get_changed_ranges.h +++ b/src/runtime/get_changed_ranges.h @@ -4,7 +4,8 @@ #include "runtime/subtree.h" unsigned ts_subtree_get_changed_ranges( - Subtree *old_tree, Subtree *new_tree, TSTreeCursor *cursor1, TSTreeCursor *cursor2, + const Subtree *old_tree, const Subtree *new_tree, + TSTreeCursor *cursor1, TSTreeCursor *cursor2, const TSLanguage *language, TSRange **ranges ); diff --git a/src/runtime/node.c b/src/runtime/node.c index e260f13b..1f221d12 100644 --- a/src/runtime/node.c +++ b/src/runtime/node.c @@ -47,7 +47,7 @@ static inline NodeChildIterator ts_node_child_iterator_begin(const TSNode *node) static inline bool ts_node_child_iterator_next(NodeChildIterator *self, TSNode *result) { if (self->child_index == self->parent->children.size) return false; - Subtree *child = self->parent->children.contents[self->child_index]; + const Subtree *child = self->parent->children.contents[self->child_index]; TSSymbol alias_symbol = 0; if (!child->extra) { if (self->alias_sequence) { diff --git a/src/runtime/parser.c b/src/runtime/parser.c index 8332501b..dde46f80 100644 --- a/src/runtime/parser.c +++ b/src/runtime/parser.c @@ -41,8 +41,8 @@ static const unsigned MAX_SUMMARY_DEPTH = 16; static const unsigned MAX_COST_DIFFERENCE = 16 * ERROR_COST_PER_SKIPPED_TREE; typedef struct { - Subtree *token; - Subtree *last_external_token; + const Subtree *token; + const Subtree *last_external_token; uint32_t byte_index; } TokenCache; @@ -52,7 +52,7 @@ struct TSParser { SubtreePool tree_pool; const TSLanguage *language; ReduceActionSet reduce_actions; - Subtree *finished_tree; + const Subtree *finished_tree; Subtree scratch_tree; TokenCache token_cache; ReusableNode reusable_node; @@ -112,10 +112,10 @@ static bool ts_parser__breakdown_top_of_stack(TSParser *self, StackVersion versi for (uint32_t i = 0; i < pop.size; i++) { StackSlice slice = pop.contents[i]; TSStateId state = ts_stack_state(self->stack, slice.version); - Subtree *parent = *array_front(&slice.subtrees); + const Subtree *parent = *array_front(&slice.subtrees); for (uint32_t j = 0; j < parent->children.size; j++) { - Subtree *child = parent->children.contents[j]; + const Subtree *child = parent->children.contents[j]; pending = child->children.size > 0; if (child->symbol == ts_builtin_sym_error) { @@ -129,7 +129,7 @@ static bool ts_parser__breakdown_top_of_stack(TSParser *self, StackVersion versi } for (uint32_t j = 1; j < slice.subtrees.size; j++) { - Subtree *tree = slice.subtrees.contents[j]; + const Subtree *tree = slice.subtrees.contents[j]; ts_stack_push(self->stack, slice.version, tree, false, state); } @@ -144,11 +144,10 @@ static bool ts_parser__breakdown_top_of_stack(TSParser *self, StackVersion versi return did_break_down; } -static void ts_parser__breakdown_lookahead(TSParser *self, Subtree **lookahead, - TSStateId state, - ReusableNode *reusable_node) { +static void ts_parser__breakdown_lookahead(TSParser *self, const Subtree **lookahead, + TSStateId state, ReusableNode *reusable_node) { bool did_descend = false; - Subtree *tree = reusable_node_tree(reusable_node); + const Subtree *tree = reusable_node_tree(reusable_node); while (tree->children.size > 0 && tree->parse_state != state) { LOG("state_mismatch sym:%s", SYM_NAME(tree->symbol)); reusable_node_descend(reusable_node); @@ -243,7 +242,7 @@ static bool ts_parser__better_version_exists(TSParser *self, StackVersion versio return false; } -static void ts_parser__restore_external_scanner(TSParser *self, Subtree *external_token) { +static void ts_parser__restore_external_scanner(TSParser *self, const Subtree *external_token) { if (external_token) { self->language->external_scanner.deserialize( self->external_scanner_payload, @@ -255,9 +254,9 @@ static void ts_parser__restore_external_scanner(TSParser *self, Subtree *externa } } -static Subtree *ts_parser__lex(TSParser *self, StackVersion version, TSStateId parse_state) { +static const Subtree *ts_parser__lex(TSParser *self, StackVersion version, TSStateId parse_state) { Length start_position = ts_stack_position(self->stack, version); - Subtree *external_token = ts_stack_last_external_token(self->stack, version); + const Subtree *external_token = ts_stack_last_external_token(self->stack, version); TSLexMode lex_mode = self->language->lex_modes[parse_state]; const bool *valid_external_tokens = ts_language_enabled_external_tokens( self->language, @@ -403,8 +402,8 @@ static Subtree *ts_parser__lex(TSParser *self, StackVersion version, TSStateId p return result; } -static Subtree *ts_parser__get_cached_token(TSParser *self, size_t byte_index, - Subtree *last_external_token) { +static const Subtree *ts_parser__get_cached_token(TSParser *self, size_t byte_index, + const Subtree *last_external_token) { TokenCache *cache = &self->token_cache; if (cache->token && cache->byte_index == byte_index && @@ -415,8 +414,8 @@ static Subtree *ts_parser__get_cached_token(TSParser *self, size_t byte_index, } } -static void ts_parser__set_cached_token(TSParser *self, size_t byte_index, Subtree *last_external_token, - Subtree *token) { +static void ts_parser__set_cached_token(TSParser *self, size_t byte_index, + const Subtree *last_external_token, const Subtree *token) { TokenCache *cache = &self->token_cache; if (token) ts_subtree_retain(token); if (last_external_token) ts_subtree_retain(last_external_token); @@ -427,8 +426,8 @@ static void ts_parser__set_cached_token(TSParser *self, size_t byte_index, Subtr cache->last_external_token = last_external_token; } -static bool ts_parser__can_reuse_first_leaf(TSParser *self, TSStateId state, Subtree *tree, - TableEntry *table_entry) { +static bool ts_parser__can_reuse_first_leaf(TSParser *self, TSStateId state, const Subtree *tree, + TableEntry *table_entry) { TSLexMode current_lex_mode = self->language->lex_modes[state]; // If the token was created in a state with the same set of lookaheads, it is reusable. @@ -445,12 +444,13 @@ static bool ts_parser__can_reuse_first_leaf(TSParser *self, TSStateId state, Sub return current_lex_mode.external_lex_state == 0 && table_entry->is_reusable; } -static Subtree *ts_parser__get_lookahead(TSParser *self, StackVersion version, TSStateId *state, - ReusableNode *reusable_node, TableEntry *table_entry) { +static const Subtree *ts_parser__get_lookahead(TSParser *self, StackVersion version, + TSStateId *state, ReusableNode *reusable_node, + TableEntry *table_entry) { Length position = ts_stack_position(self->stack, version); - Subtree *last_external_token = ts_stack_last_external_token(self->stack, version); + const Subtree *last_external_token = ts_stack_last_external_token(self->stack, version); - Subtree *result; + const Subtree *result; while ((result = reusable_node_tree(reusable_node))) { uint32_t byte_offset = reusable_node_byte_offset(reusable_node); if (byte_offset > position.bytes) { @@ -523,7 +523,7 @@ static Subtree *ts_parser__get_lookahead(TSParser *self, StackVersion version, T return result; } -static bool ts_parser__select_tree(TSParser *self, Subtree *left, Subtree *right) { +static bool ts_parser__select_tree(TSParser *self, const Subtree *left, const Subtree *right) { if (!left) return true; if (!right) return false; @@ -574,23 +574,21 @@ static bool ts_parser__select_tree(TSParser *self, Subtree *left, Subtree *right } static void ts_parser__shift(TSParser *self, StackVersion version, TSStateId state, - Subtree *lookahead, bool extra) { + const Subtree *lookahead, bool extra) { + const Subtree *subtree_to_push; if (extra != lookahead->extra) { - if (ts_stack_version_count(self->stack) > 1) { - lookahead = ts_subtree_new_copy(&self->tree_pool, lookahead); - } else { - ts_subtree_retain(lookahead); - } - lookahead->extra = extra; + Subtree *result = ts_subtree_make_mut(&self->tree_pool, lookahead); + result->extra = extra; + subtree_to_push = result; } else { - ts_subtree_retain(lookahead); + subtree_to_push = lookahead; } - bool is_pending = lookahead->children.size > 0; - ts_stack_push(self->stack, version, lookahead, is_pending, state); - if (lookahead->has_external_tokens) { + bool is_pending = subtree_to_push->children.size > 0; + ts_stack_push(self->stack, version, subtree_to_push, is_pending, state); + if (subtree_to_push->has_external_tokens) { ts_stack_set_last_external_token( - self->stack, version, ts_subtree_last_external_token(lookahead) + self->stack, version, ts_subtree_last_external_token(subtree_to_push) ); } } @@ -717,19 +715,17 @@ static void ts_parser__start(TSParser *self, TSInput input, const Subtree *previ self->in_ambiguity = false; } -static void ts_parser__accept(TSParser *self, StackVersion version, Subtree *lookahead) { - lookahead->extra = true; +static void ts_parser__accept(TSParser *self, StackVersion version, const Subtree *lookahead) { assert(lookahead->symbol == ts_builtin_sym_end); - ts_subtree_retain(lookahead); ts_stack_push(self->stack, version, lookahead, false, 1); StackSliceArray pop = ts_stack_pop_all(self->stack, version); for (uint32_t i = 0; i < pop.size; i++) { SubtreeArray trees = pop.contents[i].subtrees; - Subtree *root = NULL; + const Subtree *root = NULL; for (uint32_t j = trees.size - 1; j + 1 > 0; j--) { - Subtree *child = trees.contents[j]; + const Subtree *child = trees.contents[j]; if (!child->extra) { for (uint32_t k = 0; k < child->children.size; k++) { ts_subtree_retain(child->children.contents[k]); @@ -875,7 +871,9 @@ static void ts_parser__handle_error(TSParser *self, StackVersion version, lookahead_symbol )) { StackVersion version_with_missing_tree = ts_stack_copy_version(self->stack, v); - Subtree *missing_tree = ts_subtree_new_missing_leaf(&self->tree_pool, missing_symbol, self->language); + const Subtree *missing_tree = ts_subtree_new_missing_leaf( + &self->tree_pool, missing_symbol, self->language + ); ts_stack_push( self->stack, version_with_missing_tree, missing_tree, false, @@ -930,11 +928,10 @@ static void ts_parser__halt_parse(TSParser *self) { Subtree *eof = ts_subtree_new_leaf(&self->tree_pool, ts_builtin_sym_end, length_zero(), length_zero(), self->language); ts_parser__accept(self, 0, eof); - ts_subtree_release(&self->tree_pool, eof); } static bool ts_parser__recover_to_state(TSParser *self, StackVersion version, unsigned depth, - TSStateId goal_state) { + TSStateId goal_state) { StackSliceArray pop = ts_stack_pop_count(self->stack, version, depth); StackVersion previous_version = STACK_VERSION_NONE; @@ -975,7 +972,7 @@ static bool ts_parser__recover_to_state(TSParser *self, StackVersion version, un } for (unsigned j = 0; j < trailing_extras.size; j++) { - Subtree *tree = trailing_extras.contents[j]; + const Subtree *tree = trailing_extras.contents[j]; ts_stack_push(self->stack, slice.version, tree, false, goal_state); } @@ -986,7 +983,7 @@ static bool ts_parser__recover_to_state(TSParser *self, StackVersion version, un return previous_version != STACK_VERSION_NONE; } -static void ts_parser__recover(TSParser *self, StackVersion version, Subtree *lookahead) { +static void ts_parser__recover(TSParser *self, StackVersion version, const Subtree *lookahead) { bool did_recover = false; unsigned previous_version_count = ts_stack_version_count(self->stack); Length position = ts_stack_position(self->stack, version); @@ -1042,13 +1039,14 @@ static void ts_parser__recover(TSParser *self, StackVersion version, Subtree *lo if (did_recover && ts_stack_version_count(self->stack) > MAX_VERSION_COUNT) { ts_stack_halt(self->stack, version); + ts_subtree_release(&self->tree_pool, lookahead); return; } if (lookahead->symbol == ts_builtin_sym_end) { LOG("recover_eof"); SubtreeArray children = array_new(); - Subtree *parent = ts_subtree_new_error_node(&self->tree_pool, &children, self->language); + const Subtree *parent = ts_subtree_new_error_node(&self->tree_pool, &children, self->language); ts_stack_push(self->stack, version, parent, false, 1); ts_parser__accept(self, version, lookahead); return; @@ -1061,21 +1059,23 @@ static void ts_parser__recover(TSParser *self, StackVersion version, Subtree *lo if (ts_parser__better_version_exists(self, version, false, new_cost)) { ts_stack_halt(self->stack, version); + ts_subtree_release(&self->tree_pool, lookahead); return; } unsigned n; const TSParseAction *actions = ts_language_actions(self->language, 1, lookahead->symbol, &n); if (n > 0 && actions[n - 1].type == TSParseActionTypeShift && actions[n - 1].params.extra) { - lookahead->extra = true; + Subtree *mutable_lookahead = ts_subtree_make_mut(&self->tree_pool, lookahead); + mutable_lookahead->extra = true; + lookahead = mutable_lookahead; } LOG("skip_token symbol:%s", SYM_NAME(lookahead->symbol)); - ts_subtree_retain(lookahead); SubtreeArray children = array_new(); array_reserve(&children, 1); array_push(&children, lookahead); - Subtree *error_repeat = ts_subtree_new_node( + const Subtree *error_repeat = ts_subtree_new_node( &self->tree_pool, ts_builtin_sym_error_repeat, &children, @@ -1110,7 +1110,9 @@ static void ts_parser__recover(TSParser *self, StackVersion version, Subtree *lo static void ts_parser__advance(TSParser *self, StackVersion version, ReusableNode *reusable_node) { TSStateId state = ts_stack_state(self->stack, version); TableEntry table_entry; - Subtree *lookahead = ts_parser__get_lookahead(self, version, &state, reusable_node, &table_entry); + const Subtree *lookahead = ts_parser__get_lookahead( + self, version, &state, reusable_node, &table_entry + ); for (;;) { StackVersion last_reduction_version = STACK_VERSION_NONE; @@ -1143,7 +1145,6 @@ static void ts_parser__advance(TSParser *self, StackVersion version, ReusableNod if (lookahead == reusable_node_tree(reusable_node)) { reusable_node_advance(reusable_node); } - ts_subtree_release(&self->tree_pool, lookahead); return; } @@ -1163,7 +1164,6 @@ static void ts_parser__advance(TSParser *self, StackVersion version, ReusableNod case TSParseActionTypeAccept: { LOG("accept"); ts_parser__accept(self, version, lookahead); - ts_subtree_release(&self->tree_pool, lookahead); return; } @@ -1175,7 +1175,6 @@ static void ts_parser__advance(TSParser *self, StackVersion version, ReusableNod if (lookahead == reusable_node_tree(reusable_node)) { reusable_node_advance(reusable_node); } - ts_subtree_release(&self->tree_pool, lookahead); return; } } @@ -1186,7 +1185,6 @@ static void ts_parser__advance(TSParser *self, StackVersion version, ReusableNod LOG_STACK(); } else if (state == ERROR_STATE) { ts_parser__recover(self, version, lookahead); - ts_subtree_release(&self->tree_pool, lookahead); return; } else if (!ts_parser__breakdown_top_of_stack(self, version)) { LOG("detect_error"); diff --git a/src/runtime/reusable_node.h b/src/runtime/reusable_node.h index c1d4f06b..42ae6f1e 100644 --- a/src/runtime/reusable_node.h +++ b/src/runtime/reusable_node.h @@ -1,21 +1,21 @@ #include "runtime/subtree.h" typedef struct { - Subtree *tree; + const Subtree *tree; uint32_t child_index; uint32_t byte_offset; } StackEntry; typedef struct { Array(StackEntry) stack; - Subtree *last_external_token; + const Subtree *last_external_token; } ReusableNode; static inline ReusableNode reusable_node_new() { return (ReusableNode) {array_new(), NULL}; } -static inline void reusable_node_reset(ReusableNode *self, Subtree *tree) { +static inline void reusable_node_reset(ReusableNode *self, const Subtree *tree) { array_clear(&self->stack); array_push(&self->stack, ((StackEntry) { .tree = tree, @@ -51,7 +51,7 @@ static inline void reusable_node_advance(ReusableNode *self) { self->last_external_token = ts_subtree_last_external_token(last_entry.tree); } - Subtree *tree; + const Subtree *tree; uint32_t next_index; do { StackEntry popped_entry = array_pop(&self->stack); diff --git a/src/runtime/stack.c b/src/runtime/stack.c index 21468e49..8921d559 100644 --- a/src/runtime/stack.c +++ b/src/runtime/stack.c @@ -21,7 +21,7 @@ typedef struct StackNode StackNode; typedef struct { StackNode *node; - Subtree *subtree; + const Subtree *subtree; bool is_pending; } StackLink; @@ -58,7 +58,7 @@ typedef enum { typedef struct { StackNode *node; - Subtree *last_external_token; + const Subtree *last_external_token; StackSummary *summary; unsigned node_count_at_last_error; TSSymbol lookahead_when_paused; @@ -119,8 +119,8 @@ recur: } } -static StackNode *stack_node_new(StackNode *previous_node, Subtree *subtree, bool is_pending, - TSStateId state, StackNodeArray *pool) { +static StackNode *stack_node_new(StackNode *previous_node, const Subtree *subtree, + bool is_pending, TSStateId state, StackNodeArray *pool) { StackNode *node = pool->size > 0 ? array_pop(pool) : ts_malloc(sizeof(StackNode)); @@ -380,11 +380,11 @@ Length ts_stack_position(const Stack *self, StackVersion version) { return array_get(&self->heads, version)->node->position; } -Subtree *ts_stack_last_external_token(const Stack *self, StackVersion version) { +const Subtree *ts_stack_last_external_token(const Stack *self, StackVersion version) { return array_get(&self->heads, version)->last_external_token; } -void ts_stack_set_last_external_token(Stack *self, StackVersion version, Subtree *token) { +void ts_stack_set_last_external_token(Stack *self, StackVersion version, const Subtree *token) { StackHead *head = array_get(&self->heads, version); if (token) ts_subtree_retain(token); if (head->last_external_token) ts_subtree_release(self->subtree_pool, head->last_external_token); @@ -410,7 +410,8 @@ unsigned ts_stack_node_count_since_error(const Stack *self, StackVersion version return head->node->node_count - head->node_count_at_last_error; } -void ts_stack_push(Stack *self, StackVersion version, Subtree *subtree, bool pending, TSStateId state) { +void ts_stack_push(Stack *self, StackVersion version, const Subtree *subtree, + bool pending, TSStateId state) { StackHead *head = array_get(&self->heads, version); StackNode *new_node = stack_node_new(head->node, subtree, pending, state, &self->node_pool); if (!subtree) head->node_count_at_last_error = new_node->node_count; @@ -684,7 +685,7 @@ bool ts_stack_print_dot_graph(Stack *self, const TSLanguage *language, FILE *f) ); if (head->last_external_token) { - ExternalScannerState *state = &head->last_external_token->external_scanner_state; + const ExternalScannerState *state = &head->last_external_token->external_scanner_state; const char *data = ts_external_scanner_state_data(state); fprintf(f, "\nexternal_scanner_state:"); for (uint32_t j = 0; j < state->length; j++) fprintf(f, " %2X", data[j]); diff --git a/src/runtime/stack.h b/src/runtime/stack.h index 88c06bab..b7dba342 100644 --- a/src/runtime/stack.h +++ b/src/runtime/stack.h @@ -42,10 +42,10 @@ uint32_t ts_stack_version_count(const Stack *); TSStateId ts_stack_state(const Stack *, StackVersion); // Get the last external token associated with a given version of the stack. -Subtree *ts_stack_last_external_token(const Stack *, StackVersion); +const Subtree *ts_stack_last_external_token(const Stack *, StackVersion); // Set the last external token associated with a given version of the stack. -void ts_stack_set_last_external_token(Stack *, StackVersion, Subtree *); +void ts_stack_set_last_external_token(Stack *, StackVersion, const Subtree *); // Get the position of the given version of the stack within the document. Length ts_stack_position(const Stack *, StackVersion); @@ -55,7 +55,7 @@ Length ts_stack_position(const Stack *, StackVersion); // This transfers ownership of the tree to the Stack. Callers that // need to retain ownership of the tree for their own purposes should // first retain the tree. -void ts_stack_push(Stack *, StackVersion, Subtree *, bool, TSStateId); +void ts_stack_push(Stack *, StackVersion, const Subtree *, bool, TSStateId); // Pop the given number of entries from the given version of the stack. This // operation can increase the number of stack versions by revealing multiple diff --git a/src/runtime/subtree.c b/src/runtime/subtree.c index 0cd125bf..0991b67c 100644 --- a/src/runtime/subtree.c +++ b/src/runtime/subtree.c @@ -59,7 +59,7 @@ bool ts_external_scanner_state_eq(const ExternalScannerState *a, const ExternalS // SubtreeArray bool ts_subtree_array_copy(SubtreeArray self, SubtreeArray *dest) { - Subtree **contents = NULL; + const Subtree **contents = NULL; if (self.capacity > 0) { contents = ts_calloc(self.capacity, sizeof(Subtree *)); memcpy(contents, self.contents, self.size * sizeof(Subtree *)); @@ -86,7 +86,7 @@ SubtreeArray ts_subtree_array_remove_trailing_extras(SubtreeArray *self) { uint32_t i = self->size - 1; for (; i + 1 > 0; i--) { - Subtree *child = self->contents[i]; + const Subtree *child = self->contents[i]; if (!child->extra) break; array_push(&result, child); } @@ -99,7 +99,7 @@ SubtreeArray ts_subtree_array_remove_trailing_extras(SubtreeArray *self) { void ts_subtree_array_reverse(SubtreeArray *self) { for (uint32_t i = 0, limit = self->size / 2; i < limit; i++) { size_t reverse_index = self->size - 1 - i; - Subtree *swap = self->contents[i]; + const Subtree *swap = self->contents[i]; self->contents[i] = self->contents[reverse_index]; self->contents[reverse_index] = swap; } @@ -142,7 +142,7 @@ void ts_subtree_pool_free(SubtreePool *self, Subtree *tree) { // Subtree Subtree *ts_subtree_new_leaf(SubtreePool *pool, TSSymbol symbol, Length padding, Length size, - const TSLanguage *language) { + const TSLanguage *language) { TSSymbolMetadata metadata = ts_language_symbol_metadata(language, symbol); Subtree *result = ts_subtree_pool_allocate(pool); *result = (Subtree){ @@ -163,11 +163,12 @@ Subtree *ts_subtree_new_leaf(SubtreePool *pool, TSSymbol symbol, Length padding, }, .has_external_tokens = false, }; + if (symbol == ts_builtin_sym_end) result->extra = true; return result; } Subtree *ts_subtree_new_error(SubtreePool *pool, Length size, Length padding, - int32_t lookahead_char, const TSLanguage *language) { + int32_t lookahead_char, const TSLanguage *language) { Subtree *result = ts_subtree_new_leaf(pool, ts_builtin_sym_error, padding, size, language); result->fragile_left = true; result->fragile_right = true; @@ -175,7 +176,7 @@ Subtree *ts_subtree_new_error(SubtreePool *pool, Length size, Length padding, return result; } -Subtree *ts_subtree_new_copy(SubtreePool *pool, Subtree *self) { +Subtree *ts_subtree_new_copy(SubtreePool *pool, const Subtree *self) { Subtree *result = ts_subtree_pool_allocate(pool); *result = *self; if (result->children.size > 0) { @@ -185,9 +186,9 @@ Subtree *ts_subtree_new_copy(SubtreePool *pool, Subtree *self) { return result; } -static Subtree *ts_subtree_new_mut(SubtreePool *pool, Subtree *self) { +Subtree *ts_subtree_make_mut(SubtreePool *pool, const Subtree *self) { if (self->ref_count == 1) { - return self; + return (Subtree *)self; } else { Subtree *result = ts_subtree_new_copy(pool, self); ts_subtree_release(pool, self); @@ -196,21 +197,21 @@ static Subtree *ts_subtree_new_mut(SubtreePool *pool, Subtree *self) { } static void ts_subtree__compress(Subtree *self, unsigned count, const TSLanguage *language, - SubtreeArray *stack) { + MutableSubtreeArray *stack) { unsigned initial_stack_size = stack->size; Subtree *tree = self; for (unsigned i = 0; i < count; i++) { if (tree->ref_count > 1 || tree->children.size != 2) break; - Subtree *child = tree->children.contents[0]; + Subtree *child = (Subtree *)tree->children.contents[0]; if ( child->ref_count > 1 || child->children.size != 2 || child->symbol != tree->symbol ) break; - Subtree *grandchild = child->children.contents[0]; + Subtree *grandchild = (Subtree *)child->children.contents[0]; if ( grandchild->ref_count > 1 || grandchild->children.size != 2 || @@ -227,38 +228,41 @@ static void ts_subtree__compress(Subtree *self, unsigned count, const TSLanguage while (stack->size > initial_stack_size) { tree = array_pop(stack); assert(tree); - Subtree *child = tree->children.contents[0]; - Subtree *grandchild = child->children.contents[1]; + Subtree *child = (Subtree *)tree->children.contents[0]; + Subtree *grandchild = (Subtree *)child->children.contents[1]; ts_subtree_set_children(grandchild, &grandchild->children, language); ts_subtree_set_children(child, &child->children, language); ts_subtree_set_children(tree, &tree->children, language); } } -void ts_subtree_balance(Subtree *self, SubtreePool *pool, const TSLanguage *language) { +void ts_subtree_balance(const Subtree *self, SubtreePool *pool, const TSLanguage *language) { array_clear(&pool->tree_stack); - array_push(&pool->tree_stack, self); + + if (self->ref_count == 1) { + array_push(&pool->tree_stack, (Subtree *)self); + } + while (pool->tree_stack.size > 0) { Subtree *tree = array_pop(&pool->tree_stack); assert(tree); - if (tree->repeat_depth > 0) { - if (tree->children.contents[0]->repeat_depth > tree->children.contents[1]->repeat_depth) { - unsigned n = ( - tree->children.contents[0]->repeat_depth - - tree->children.contents[1]->repeat_depth - ); - for (unsigned i = n / 2; i > 0; i /= 2) { - ts_subtree__compress(tree, i, language, &pool->tree_stack); - n -= i; - } + if (tree->repeat_depth > 0 && + tree->children.contents[0]->repeat_depth > tree->children.contents[1]->repeat_depth) { + unsigned n = ( + tree->children.contents[0]->repeat_depth - + tree->children.contents[1]->repeat_depth + ); + for (unsigned i = n / 2; i > 0; i /= 2) { + ts_subtree__compress(tree, i, language, &pool->tree_stack); + n -= i; } } for (uint32_t i = 0; i < tree->children.size; i++) { - Subtree *child = tree->children.contents[i]; + const Subtree *child = tree->children.contents[i]; if (child->ref_count == 1) { - array_push(&pool->tree_stack, child); + array_push(&pool->tree_stack, (Subtree *)child); } } } @@ -282,7 +286,7 @@ void ts_subtree_set_children(Subtree *self, SubtreeArray *children, const TSLang const TSSymbol *alias_sequence = ts_language_alias_sequence(language, self->alias_sequence_id); for (uint32_t i = 0; i < self->children.size; i++) { - Subtree *child = self->children.contents[i]; + const Subtree *child = self->children.contents[i]; if (i == 0) { self->padding = child->padding; @@ -328,7 +332,7 @@ void ts_subtree_set_children(Subtree *self, SubtreeArray *children, const TSLang ERROR_COST_PER_SKIPPED_CHAR * self->size.bytes + ERROR_COST_PER_SKIPPED_LINE * self->size.extent.row; for (uint32_t i = 0; i < self->children.size; i++) { - Subtree *child = self->children.contents[i]; + const Subtree *child = self->children.contents[i]; if (child->extra) continue; if (child->symbol == ts_builtin_sym_error && child->children.size == 0) continue; if (child->visible) { @@ -340,8 +344,8 @@ void ts_subtree_set_children(Subtree *self, SubtreeArray *children, const TSLang } if (self->children.size > 0) { - Subtree *first_child = self->children.contents[0]; - Subtree *last_child = self->children.contents[self->children.size - 1]; + const Subtree *first_child = self->children.contents[0]; + const Subtree *last_child = self->children.contents[self->children.size - 1]; self->first_leaf = first_child->first_leaf; if (first_child->fragile_left) self->fragile_left = true; if (last_child->fragile_right) self->fragile_right = true; @@ -361,7 +365,7 @@ void ts_subtree_set_children(Subtree *self, SubtreeArray *children, const TSLang } Subtree *ts_subtree_new_node(SubtreePool *pool, TSSymbol symbol, SubtreeArray *children, - unsigned alias_sequence_id, const TSLanguage *language) { + unsigned alias_sequence_id, const TSLanguage *language) { Subtree *result = ts_subtree_new_leaf(pool, symbol, length_zero(), length_zero(), language); result->alias_sequence_id = alias_sequence_id; if (symbol == ts_builtin_sym_error || symbol == ts_builtin_sym_error_repeat) { @@ -373,41 +377,46 @@ Subtree *ts_subtree_new_node(SubtreePool *pool, TSSymbol symbol, SubtreeArray *c } Subtree *ts_subtree_new_error_node(SubtreePool *pool, SubtreeArray *children, - const TSLanguage *language) { + const TSLanguage *language) { return ts_subtree_new_node(pool, ts_builtin_sym_error, children, 0, language); } Subtree *ts_subtree_new_missing_leaf(SubtreePool *pool, TSSymbol symbol, - const TSLanguage *language) { + const TSLanguage *language) { Subtree *result = ts_subtree_new_leaf(pool, symbol, length_zero(), length_zero(), language); result->is_missing = true; result->error_cost = ERROR_COST_PER_MISSING_TREE + ERROR_COST_PER_RECOVERY; return result; } -void ts_subtree_retain(Subtree *self) { +void ts_subtree_retain(const Subtree *self) { assert(self->ref_count > 0); - atomic_inc(&self->ref_count); + atomic_inc((volatile uint32_t *)&self->ref_count); assert(self->ref_count != 0); } -void ts_subtree_release(SubtreePool *pool, Subtree *self) { +void ts_subtree_release(SubtreePool *pool, const Subtree *self) { array_clear(&pool->tree_stack); - array_push(&pool->tree_stack, self); + + assert(self->ref_count > 0); + if (atomic_dec((volatile uint32_t *)&self->ref_count) == 0) { + array_push(&pool->tree_stack, (Subtree *)self); + } + while (pool->tree_stack.size > 0) { Subtree *tree = array_pop(&pool->tree_stack); - assert(tree->ref_count > 0); - if (atomic_dec(&tree->ref_count) == 0) { - if (tree->children.size > 0) { - for (uint32_t i = 0; i < tree->children.size; i++) { - array_push(&pool->tree_stack, tree->children.contents[i]); + if (tree->children.size > 0) { + for (uint32_t i = 0; i < tree->children.size; i++) { + const Subtree *child = tree->children.contents[i]; + if (atomic_dec((volatile uint32_t *)&child->ref_count) == 0) { + array_push(&pool->tree_stack, (Subtree *)child); } - array_delete(&tree->children); - } else if (tree->has_external_tokens) { - ts_external_scanner_state_delete(&tree->external_scanner_state); } - ts_subtree_pool_free(pool, tree); + array_delete(&tree->children); + } else if (tree->has_external_tokens) { + ts_external_scanner_state_delete(&tree->external_scanner_state); } + ts_subtree_pool_free(pool, tree); } } @@ -446,8 +455,8 @@ int ts_subtree_compare(const Subtree *left, const Subtree *right) { if (right->children.size < left->children.size) return 1; for (uint32_t i = 0; i < left->children.size; i++) { - Subtree *left_child = left->children.contents[i]; - Subtree *right_child = right->children.contents[i]; + const Subtree *left_child = left->children.contents[i]; + const Subtree *right_child = right->children.contents[i]; switch (ts_subtree_compare(left_child, right_child)) { case -1: return -1; @@ -460,17 +469,17 @@ int ts_subtree_compare(const Subtree *left, const Subtree *right) { return 0; } -Subtree *ts_subtree_invalidate_lookahead(Subtree *self, uint32_t edit_byte_offset, - SubtreePool *pool) { +const Subtree *ts_subtree_invalidate_lookahead(const Subtree *self, uint32_t edit_byte_offset, + SubtreePool *pool) { if (edit_byte_offset >= self->bytes_scanned) return self; - Subtree *result = ts_subtree_new_mut(pool, self); + Subtree *result = ts_subtree_make_mut(pool, self); result->has_changes = true; if (result->children.size > 0) { uint32_t child_start_byte = 0; for (uint32_t i = 0; i < result->children.size; i++) { - Subtree **child = &result->children.contents[i]; + const Subtree **child = &result->children.contents[i]; if (child_start_byte > edit_byte_offset) break; *child = ts_subtree_invalidate_lookahead(*child, edit_byte_offset - child_start_byte, pool); child_start_byte += ts_subtree_total_bytes(*child); @@ -480,11 +489,11 @@ Subtree *ts_subtree_invalidate_lookahead(Subtree *self, uint32_t edit_byte_offse return result; } -Subtree *ts_subtree__edit(Subtree *self, Edit edit, SubtreePool *pool) { +const Subtree *ts_subtree__edit(const Subtree *self, Edit edit, SubtreePool *pool) { Length new_end = length_add(edit.start, edit.added); Length old_end = length_add(edit.start, edit.removed); - Subtree *result = ts_subtree_new_mut(pool, self); + Subtree *result = ts_subtree_make_mut(pool, self); result->has_changes = true; if (old_end.bytes <= result->padding.bytes) { @@ -501,7 +510,7 @@ Subtree *ts_subtree__edit(Subtree *self, Edit edit, SubtreePool *pool) { Length child_left, child_right = length_zero(); for (uint32_t i = 0; i < result->children.size; i++) { - Subtree **child = &result->children.contents[i]; + const Subtree **child = &result->children.contents[i]; Length child_size = ts_subtree_total_size(*child); child_left = child_right; child_right = length_add(child_left, child_size); @@ -537,7 +546,7 @@ Subtree *ts_subtree__edit(Subtree *self, Edit edit, SubtreePool *pool) { return result; } -Subtree *ts_subtree_edit(Subtree *self, const TSInputEdit *edit, SubtreePool *pool) { +const Subtree *ts_subtree_edit(const Subtree *self, const TSInputEdit *edit, SubtreePool *pool) { return ts_subtree__edit(self, (Edit) { .start = {edit->start_byte, edit->start_point}, .added = {edit->bytes_added, edit->extent_added}, @@ -545,11 +554,11 @@ Subtree *ts_subtree_edit(Subtree *self, const TSInputEdit *edit, SubtreePool *po }, pool); } -Subtree *ts_subtree_last_external_token(Subtree *tree) { +const Subtree *ts_subtree_last_external_token(const Subtree *tree) { if (!tree->has_external_tokens) return NULL; while (tree->children.size > 0) { for (uint32_t i = tree->children.size - 1; i + 1 > 0; i--) { - Subtree *child = tree->children.contents[i]; + const Subtree *child = tree->children.contents[i]; if (child->has_external_tokens) { tree = child; break; @@ -611,7 +620,7 @@ static size_t ts_subtree__write_to_string(const Subtree *self, char *string, siz const TSSymbol *alias_sequence = ts_language_alias_sequence(language, self->alias_sequence_id); uint32_t structural_child_index = 0; for (uint32_t i = 0; i < self->children.size; i++) { - Subtree *child = self->children.contents[i]; + const Subtree *child = self->children.contents[i]; if (child->extra) { cursor += ts_subtree__write_to_string( child, *writer, limit, diff --git a/src/runtime/subtree.h b/src/runtime/subtree.h index da69383d..cf59db43 100644 --- a/src/runtime/subtree.h +++ b/src/runtime/subtree.h @@ -24,7 +24,8 @@ typedef struct { typedef struct Subtree Subtree; -typedef Array(Subtree *) SubtreeArray; +typedef Array(const Subtree *) SubtreeArray; +typedef Array(Subtree *) MutableSubtreeArray; struct Subtree { Length padding; @@ -71,8 +72,8 @@ struct Subtree { }; typedef struct { - SubtreeArray free_trees; - SubtreeArray tree_stack; + MutableSubtreeArray free_trees; + MutableSubtreeArray tree_stack; } SubtreePool; void ts_external_scanner_state_init(ExternalScannerState *, const char *, unsigned); @@ -90,20 +91,21 @@ void ts_subtree_pool_free(SubtreePool *, Subtree *); Subtree *ts_subtree_new_leaf(SubtreePool *, TSSymbol, Length, Length, const TSLanguage *); Subtree *ts_subtree_new_node(SubtreePool *, TSSymbol, SubtreeArray *, unsigned, const TSLanguage *); -Subtree *ts_subtree_new_copy(SubtreePool *, Subtree *child); +Subtree *ts_subtree_new_copy(SubtreePool *, const Subtree *); Subtree *ts_subtree_new_error_node(SubtreePool *, SubtreeArray *, const TSLanguage *); Subtree *ts_subtree_new_error(SubtreePool *, Length, Length, int32_t, const TSLanguage *); Subtree *ts_subtree_new_missing_leaf(SubtreePool *, TSSymbol, const TSLanguage *); -void ts_subtree_retain(Subtree *tree); -void ts_subtree_release(SubtreePool *, Subtree *tree); +Subtree *ts_subtree_make_mut(SubtreePool *, const Subtree *); +void ts_subtree_retain(const Subtree *tree); +void ts_subtree_release(SubtreePool *, const Subtree *tree); bool ts_subtree_eq(const Subtree *tree1, const Subtree *tree2); int ts_subtree_compare(const Subtree *tree1, const Subtree *tree2); void ts_subtree_set_children(Subtree *, SubtreeArray *, const TSLanguage *); -void ts_subtree_balance(Subtree *, SubtreePool *, const TSLanguage *); -Subtree *ts_subtree_edit(Subtree *, const TSInputEdit *edit, SubtreePool *); +void ts_subtree_balance(const Subtree *, SubtreePool *, const TSLanguage *); +const Subtree *ts_subtree_edit(const Subtree *, const TSInputEdit *edit, SubtreePool *); char *ts_subtree_string(const Subtree *, const TSLanguage *, bool include_all); void ts_subtree_print_dot_graph(const Subtree *, const TSLanguage *, FILE *); -Subtree *ts_subtree_last_external_token(Subtree *); +const Subtree *ts_subtree_last_external_token(const Subtree *); bool ts_subtree_external_scanner_state_eq(const Subtree *, const Subtree *); static inline uint32_t ts_subtree_total_bytes(const Subtree *self) { diff --git a/src/runtime/tree.c b/src/runtime/tree.c index 2c365644..3e8a145e 100644 --- a/src/runtime/tree.c +++ b/src/runtime/tree.c @@ -17,7 +17,7 @@ TSTree *ts_tree_copy(const TSTree *self) { return ts_tree_new(self->root, self->language); } -void ts_tree_delete(const TSTree *self) { +void ts_tree_delete(TSTree *self) { SubtreePool pool = ts_subtree_pool_new(0); ts_subtree_release(&pool, self->root); ts_subtree_pool_delete(&pool); diff --git a/src/runtime/tree_cursor.c b/src/runtime/tree_cursor.c index c7aa9691..31eee8b4 100644 --- a/src/runtime/tree_cursor.c +++ b/src/runtime/tree_cursor.c @@ -28,7 +28,7 @@ void ts_tree_cursor_delete(TSTreeCursor *self) { bool ts_tree_cursor_goto_first_child(TSTreeCursor *self) { TreeCursorEntry *last_entry = array_back(&self->stack); - Subtree *tree = last_entry->subtree; + const Subtree *tree = last_entry->subtree; Length position = last_entry->position; bool did_descend; @@ -37,7 +37,7 @@ bool ts_tree_cursor_goto_first_child(TSTreeCursor *self) { uint32_t structural_child_index = 0; for (uint32_t i = 0; i < tree->children.size; i++) { - Subtree *child = tree->children.contents[i]; + const Subtree *child = tree->children.contents[i]; if (child->visible || child->visible_child_count > 0) { array_push(&self->stack, ((TreeCursorEntry) { .subtree = child, @@ -68,11 +68,11 @@ bool ts_tree_cursor_goto_next_sibling(TSTreeCursor *self) { for (unsigned i = self->stack.size - 2; i + 1 > 0; i--) { TreeCursorEntry *parent_entry = &self->stack.contents[i]; - Subtree *parent = parent_entry->subtree; + const Subtree *parent = parent_entry->subtree; uint32_t child_index = child_entry->child_index; uint32_t structural_child_index = child_entry->structural_child_index; Length position = child_entry->position; - Subtree *child = parent->children.contents[child_index]; + const Subtree *child = parent->children.contents[child_index]; while (++child_index < parent->children.size) { if (!child->extra) structural_child_index++; diff --git a/src/runtime/tree_cursor.h b/src/runtime/tree_cursor.h index c016b276..615e7e06 100644 --- a/src/runtime/tree_cursor.h +++ b/src/runtime/tree_cursor.h @@ -4,7 +4,7 @@ #include "runtime/subtree.h" typedef struct { - Subtree *subtree; + const Subtree *subtree; Length position; uint32_t child_index; uint32_t structural_child_index; diff --git a/test/helpers/tree_helpers.cc b/test/helpers/tree_helpers.cc index 7fd015a3..0e63a19f 100644 --- a/test/helpers/tree_helpers.cc +++ b/test/helpers/tree_helpers.cc @@ -14,11 +14,11 @@ const char *symbol_names[24] = { "twenty-two", "twenty-three" }; -SubtreeArray *tree_array(std::vector trees) { +SubtreeArray *tree_array(std::vector trees) { static SubtreeArray result; result.capacity = trees.size(); result.size = trees.size(); - result.contents = (Subtree **)calloc(trees.size(), sizeof(Subtree *)); + result.contents = (const Subtree **)calloc(trees.size(), sizeof(Subtree *)); for (size_t i = 0; i < trees.size(); i++) { result.contents[i] = trees[i]; } @@ -49,7 +49,7 @@ bool operator==(const TSNode &left, const TSNode &right) { return ts_node_eq(left, right); } -bool operator==(const std::vector &vec, const SubtreeArray &array) { +bool operator==(const std::vector &vec, const SubtreeArray &array) { if (vec.size() != array.size) return false; for (size_t i = 0; i < array.size; i++) diff --git a/test/helpers/tree_helpers.h b/test/helpers/tree_helpers.h index bd08aad7..00740faf 100644 --- a/test/helpers/tree_helpers.h +++ b/test/helpers/tree_helpers.h @@ -6,12 +6,12 @@ #include extern const char *symbol_names[24]; -SubtreeArray *tree_array(std::vector trees); +SubtreeArray *tree_array(std::vector trees); std::ostream &operator<<(std::ostream &stream, const Subtree *tree); std::ostream &operator<<(std::ostream &stream, const TSNode &node); bool operator==(const TSNode &left, const TSNode &right); -bool operator==(const std::vector &right, const SubtreeArray &array); +bool operator==(const std::vector &right, const SubtreeArray &array); void assert_consistent_tree_sizes(TSNode node); diff --git a/test/runtime/stack_test.cc b/test/runtime/stack_test.cc index 67812a07..68104935 100644 --- a/test/runtime/stack_test.cc +++ b/test/runtime/stack_test.cc @@ -44,6 +44,10 @@ void free_slice_array(SubtreePool *pool, StackSliceArray *slices) { } } +Subtree *mutate(const Subtree *subtree) { + return (Subtree *)subtree; +} + struct StackEntry { TSStateId state; size_t depth; @@ -69,7 +73,7 @@ START_TEST describe("Stack", [&]() { Stack *stack; const size_t subtree_count = 11; - Subtree *subtrees[subtree_count]; + const Subtree *subtrees[subtree_count]; Length tree_len = {3, {0, 3}}; SubtreePool pool; @@ -84,7 +88,7 @@ describe("Stack", [&]() { dummy_language.symbol_metadata = symbol_metadata; for (size_t i = 0; i < subtree_count; i++) { - subtrees[i] = ts_subtree_new_leaf(&pool, i, length_zero(), tree_len, &dummy_language); + subtrees[i] = ts_subtree_new_leaf(&pool, i + 1, length_zero(), tree_len, &dummy_language); } }); @@ -99,7 +103,7 @@ describe("Stack", [&]() { AssertThat(record_alloc::outstanding_allocation_indices(), IsEmpty()); }); - auto push = [&](StackVersion version, Subtree *tree, TSStateId state) { + auto push = [&](StackVersion version, const Subtree *tree, TSStateId state) { ts_subtree_retain(tree); ts_stack_push(stack, version, tree, false, state); }; @@ -181,7 +185,7 @@ describe("Stack", [&]() { // . <──0── A <──1── B <────3──── D* // ↑ // └───2─── C <──4── D* - subtrees[3]->size = tree_len * 3; + mutate(subtrees[3])->size = tree_len * 3; push(0, subtrees[1], stateB); push(1, subtrees[2], stateC); push(0, subtrees[3], stateD); @@ -224,8 +228,8 @@ describe("Stack", [&]() { // . <──0── A <────1──── B* // ↑ // └2─ A <──1── B* - subtrees[2]->extra = true; - subtrees[2]->size = tree_len * 0; + mutate(subtrees[2])->extra = true; + mutate(subtrees[2])->size = tree_len * 0; push(0, subtrees[1], stateB); push(1, subtrees[2], stateA); @@ -261,14 +265,14 @@ describe("Stack", [&]() { StackSlice slice = pop.contents[0]; AssertThat(slice.version, Equals(1)); - AssertThat(slice.subtrees, Equals(vector({ subtrees[1], subtrees[2] }))); + AssertThat(slice.subtrees, Equals(vector({ subtrees[1], subtrees[2] }))); AssertThat(ts_stack_state(stack, 1), Equals(stateA)); free_slice_array(&pool,&pop); }); it("does not count 'extra' subtrees toward the given count", [&]() { - subtrees[1]->extra = true; + mutate(subtrees[1])->extra = true; // . <──0── A <──1── B <──2── C* // ↑ @@ -277,7 +281,7 @@ describe("Stack", [&]() { AssertThat(pop.size, Equals(1)); StackSlice slice = pop.contents[0]; - AssertThat(slice.subtrees, Equals(vector({ subtrees[0], subtrees[1], subtrees[2] }))); + AssertThat(slice.subtrees, Equals(vector({ subtrees[0], subtrees[1], subtrees[2] }))); AssertThat(ts_stack_state(stack, 1), Equals(1)); free_slice_array(&pool,&pop); @@ -322,11 +326,11 @@ describe("Stack", [&]() { StackSlice slice1 = pop.contents[0]; AssertThat(slice1.version, Equals(1)); - AssertThat(slice1.subtrees, Equals(vector({ subtrees[2], subtrees[3], subtrees[10] }))); + AssertThat(slice1.subtrees, Equals(vector({ subtrees[2], subtrees[3], subtrees[10] }))); StackSlice slice2 = pop.contents[1]; AssertThat(slice2.version, Equals(2)); - AssertThat(slice2.subtrees, Equals(vector({ subtrees[5], subtrees[6], subtrees[10] }))); + AssertThat(slice2.subtrees, Equals(vector({ subtrees[5], subtrees[6], subtrees[10] }))); AssertThat(ts_stack_version_count(stack), Equals(3)); AssertThat(get_stack_entries(stack, 0), Equals(vector({ @@ -366,7 +370,7 @@ describe("Stack", [&]() { StackSlice slice1 = pop.contents[0]; AssertThat(slice1.version, Equals(1)); - AssertThat(slice1.subtrees, Equals(vector({ subtrees[10] }))); + AssertThat(slice1.subtrees, Equals(vector({ subtrees[10] }))); AssertThat(ts_stack_version_count(stack), Equals(2)); AssertThat(ts_stack_state(stack, 0), Equals(stateI)); @@ -388,11 +392,11 @@ describe("Stack", [&]() { StackSlice slice1 = pop.contents[0]; AssertThat(slice1.version, Equals(1)); - AssertThat(slice1.subtrees, Equals(vector({ subtrees[1], subtrees[2], subtrees[3], subtrees[10] }))); + AssertThat(slice1.subtrees, Equals(vector({ subtrees[1], subtrees[2], subtrees[3], subtrees[10] }))); StackSlice slice2 = pop.contents[1]; AssertThat(slice2.version, Equals(1)); - AssertThat(slice2.subtrees, Equals(vector({ subtrees[4], subtrees[5], subtrees[6], subtrees[10] }))); + AssertThat(slice2.subtrees, Equals(vector({ subtrees[4], subtrees[5], subtrees[6], subtrees[10] }))); AssertThat(ts_stack_version_count(stack), Equals(2)); AssertThat(ts_stack_state(stack, 0), Equals(stateI)); @@ -443,15 +447,15 @@ describe("Stack", [&]() { StackSlice slice1 = pop.contents[0]; AssertThat(slice1.version, Equals(1)); - AssertThat(slice1.subtrees, Equals(vector({ subtrees[3], subtrees[10] }))); + AssertThat(slice1.subtrees, Equals(vector({ subtrees[3], subtrees[10] }))); StackSlice slice2 = pop.contents[1]; AssertThat(slice2.version, Equals(2)); - AssertThat(slice2.subtrees, Equals(vector({ subtrees[6], subtrees[10] }))); + AssertThat(slice2.subtrees, Equals(vector({ subtrees[6], subtrees[10] }))); StackSlice slice3 = pop.contents[2]; AssertThat(slice3.version, Equals(3)); - AssertThat(slice3.subtrees, Equals(vector({ subtrees[9], subtrees[10] }))); + AssertThat(slice3.subtrees, Equals(vector({ subtrees[9], subtrees[10] }))); AssertThat(ts_stack_version_count(stack), Equals(4)); AssertThat(ts_stack_state(stack, 0), Equals(stateI)); @@ -489,8 +493,8 @@ describe("Stack", [&]() { ts_stack_push(stack, 0, subtrees[1], true, stateB); ts_subtree_retain(subtrees[1]); - subtrees[2]->extra = true; - subtrees[3]->extra = true; + mutate(subtrees[2])->extra = true; + mutate(subtrees[3])->extra = true; push(0, subtrees[2], stateB); push(0, subtrees[3], stateB); @@ -498,7 +502,7 @@ describe("Stack", [&]() { StackSliceArray pop = ts_stack_pop_pending(stack, 0); AssertThat(pop.size, Equals(1)); - AssertThat(pop.contents[0].subtrees, Equals(vector({ subtrees[1], subtrees[2], subtrees[3] }))); + AssertThat(pop.contents[0].subtrees, Equals(vector({ subtrees[1], subtrees[2], subtrees[3] }))); AssertThat(get_stack_entries(stack, 0), Equals(vector({ {stateA, 0}, @@ -526,10 +530,10 @@ describe("Stack", [&]() { describe("setting external token state", [&]() { before_each([&]() { - subtrees[1]->has_external_tokens = true; - subtrees[2]->has_external_tokens = true; - ts_external_scanner_state_init(&subtrees[1]->external_scanner_state, NULL, 0); - ts_external_scanner_state_init(&subtrees[2]->external_scanner_state, NULL, 0); + mutate(subtrees[1])->has_external_tokens = true; + mutate(subtrees[2])->has_external_tokens = true; + ts_external_scanner_state_init(&mutate(subtrees[1])->external_scanner_state, NULL, 0); + ts_external_scanner_state_init(&mutate(subtrees[2])->external_scanner_state, NULL, 0); }); it("allows the state to be retrieved", [&]() { @@ -546,8 +550,8 @@ describe("Stack", [&]() { }); it("does not merge stack versions with different external token states", [&]() { - ts_external_scanner_state_init(&subtrees[1]->external_scanner_state, "abcd", 2); - ts_external_scanner_state_init(&subtrees[2]->external_scanner_state, "ABCD", 2); + ts_external_scanner_state_init(&mutate(subtrees[1])->external_scanner_state, "abcd", 2); + ts_external_scanner_state_init(&mutate(subtrees[2])->external_scanner_state, "ABCD", 2); ts_stack_copy_version(stack, 0); push(0, subtrees[0], 5); @@ -560,8 +564,8 @@ describe("Stack", [&]() { }); it("merges stack versions with identical external token states", [&]() { - ts_external_scanner_state_init(&subtrees[1]->external_scanner_state, "abcd", 2); - ts_external_scanner_state_init(&subtrees[2]->external_scanner_state, "abcd", 2); + ts_external_scanner_state_init(&mutate(subtrees[1])->external_scanner_state, "abcd", 2); + ts_external_scanner_state_init(&mutate(subtrees[2])->external_scanner_state, "abcd", 2); ts_stack_copy_version(stack, 0); push(0, subtrees[0], 5); diff --git a/test/runtime/subtree_test.cc b/test/runtime/subtree_test.cc index 4d32ec32..3c1c9ad2 100644 --- a/test/runtime/subtree_test.cc +++ b/test/runtime/subtree_test.cc @@ -10,7 +10,7 @@ void assert_consistent(const Subtree *tree) { Length total_children_size = length_zero(); for (size_t i = 0; i < tree->children.size; i++) { - Subtree *child = tree->children.contents[i]; + const Subtree *child = tree->children.contents[i]; assert_consistent(child); total_children_size = length_add(total_children_size, ts_subtree_total_size(child)); } @@ -50,7 +50,7 @@ describe("Subtree", []() { describe("make_leaf", [&]() { it("does not mark the tree as fragile", [&]() { - Subtree *tree = ts_subtree_new_leaf(&pool, symbol1, {2, {0, 1}}, {5, {0, 4}}, &language); + const Subtree *tree = ts_subtree_new_leaf(&pool, symbol1, {2, {0, 1}}, {5, {0, 4}}, &language); AssertThat(tree->fragile_left, IsFalse()); AssertThat(tree->fragile_right, IsFalse()); @@ -60,7 +60,7 @@ describe("Subtree", []() { describe("make_error", [&]() { it("marks the tree as fragile", [&]() { - Subtree *error_tree = ts_subtree_new_error( + const Subtree *error_tree = ts_subtree_new_error( &pool, length_zero(), length_zero(), @@ -76,7 +76,7 @@ describe("Subtree", []() { }); describe("make_node", [&]() { - Subtree *tree1, *tree2, *parent1; + const Subtree *tree1, *tree2, *parent1; before_each([&]() { tree1 = ts_subtree_new_leaf(&pool, symbol1, {2, {0, 1}}, {5, {0, 4}}, &language); @@ -104,11 +104,12 @@ describe("Subtree", []() { }); describe("when the first node is fragile on the left side", [&]() { - Subtree *parent; + const Subtree *parent; before_each([&]() { - tree1->fragile_left = true; - tree1->extra = true; + Subtree *mutable_tree1 = (Subtree *)tree1; + mutable_tree1->fragile_left = true; + mutable_tree1->extra = true; ts_subtree_retain(tree1); ts_subtree_retain(tree2); @@ -128,11 +129,12 @@ describe("Subtree", []() { }); describe("when the last node is fragile on the right side", [&]() { - Subtree *parent; + const Subtree *parent; before_each([&]() { - tree2->fragile_right = true; - tree2->extra = true; + Subtree *mutable_tree2 = (Subtree *)tree2; + mutable_tree2->fragile_right = true; + mutable_tree2->extra = true; ts_subtree_retain(tree1); ts_subtree_retain(tree2); @@ -152,11 +154,13 @@ describe("Subtree", []() { }); describe("when the outer nodes aren't fragile on their outer side", [&]() { - Subtree *parent; + const Subtree *parent; before_each([&]() { - tree1->fragile_right = true; - tree2->fragile_left = true; + Subtree *mutable_tree1 = (Subtree *)tree1; + Subtree *mutable_tree2 = (Subtree *)tree2; + mutable_tree1->fragile_right = true; + mutable_tree2->fragile_left = true; ts_subtree_retain(tree1); ts_subtree_retain(tree2); @@ -178,7 +182,7 @@ describe("Subtree", []() { }); describe("edit", [&]() { - Subtree *tree; + const Subtree *tree; before_each([&]() { tree = ts_subtree_new_node(&pool, symbol1, tree_array({ @@ -205,7 +209,7 @@ describe("Subtree", []() { edit.extent_added = {0, 1}; ts_subtree_retain(tree); - Subtree *new_tree = ts_subtree_edit(tree, &edit, &pool); + const Subtree *new_tree = ts_subtree_edit(tree, &edit, &pool); assert_consistent(tree); assert_consistent(new_tree); @@ -352,7 +356,8 @@ describe("Subtree", []() { describe("edits within a tree's range of scanned bytes", [&]() { it("marks preceding trees as changed", [&]() { - tree->children.contents[0]->bytes_scanned = 7; + Subtree *mutable_child = (Subtree *)tree->children.contents[0]; + mutable_child->bytes_scanned = 7; TSInputEdit edit; edit.start_byte = 6; @@ -370,7 +375,7 @@ describe("Subtree", []() { }); describe("eq", [&]() { - Subtree *leaf; + const Subtree *leaf; before_each([&]() { leaf = ts_subtree_new_leaf(&pool, symbol1, {2, {0, 1}}, {5, {0, 4}}, &language); @@ -381,17 +386,17 @@ describe("Subtree", []() { }); it("returns true for identical trees", [&]() { - Subtree *leaf_copy = ts_subtree_new_leaf(&pool, symbol1, {2, {1, 1}}, {5, {1, 4}}, &language); + const Subtree *leaf_copy = ts_subtree_new_leaf(&pool, symbol1, {2, {1, 1}}, {5, {1, 4}}, &language); AssertThat(ts_subtree_eq(leaf, leaf_copy), IsTrue()); - Subtree *parent = ts_subtree_new_node(&pool, symbol2, tree_array({ + const Subtree *parent = ts_subtree_new_node(&pool, symbol2, tree_array({ leaf, leaf_copy, }), 0, &language); ts_subtree_retain(leaf); ts_subtree_retain(leaf_copy); - Subtree *parent_copy = ts_subtree_new_node(&pool, symbol2, tree_array({ + const Subtree *parent_copy = ts_subtree_new_node(&pool, symbol2, tree_array({ leaf, leaf_copy, }), 0, &language); @@ -406,7 +411,7 @@ describe("Subtree", []() { }); it("returns false for trees with different symbols", [&]() { - Subtree *different_leaf = ts_subtree_new_leaf( + const Subtree *different_leaf = ts_subtree_new_leaf( &pool, leaf->symbol + 1, leaf->padding, @@ -419,14 +424,16 @@ describe("Subtree", []() { }); it("returns false for trees with different options", [&]() { - Subtree *different_leaf = ts_subtree_new_leaf(&pool, leaf->symbol, leaf->padding, leaf->size, &language); - different_leaf->visible = !leaf->visible; + const Subtree *different_leaf = ts_subtree_new_leaf( + &pool, leaf->symbol, leaf->padding, leaf->size, &language + ); + ((Subtree *)different_leaf)->visible = !leaf->visible; AssertThat(ts_subtree_eq(leaf, different_leaf), IsFalse()); ts_subtree_release(&pool, different_leaf); }); it("returns false for trees with different paddings or sizes", [&]() { - Subtree *different_leaf = ts_subtree_new_leaf(&pool, leaf->symbol, {}, leaf->size, &language); + const Subtree *different_leaf = ts_subtree_new_leaf(&pool, leaf->symbol, {}, leaf->size, &language); AssertThat(ts_subtree_eq(leaf, different_leaf), IsFalse()); ts_subtree_release(&pool, different_leaf); @@ -436,16 +443,16 @@ describe("Subtree", []() { }); it("returns false for trees with different children", [&]() { - Subtree *leaf2 = ts_subtree_new_leaf(&pool, symbol2, {1, {0, 1}}, {3, {0, 3}}, &language); + const Subtree *leaf2 = ts_subtree_new_leaf(&pool, symbol2, {1, {0, 1}}, {3, {0, 3}}, &language); - Subtree *parent = ts_subtree_new_node(&pool, symbol2, tree_array({ + const Subtree *parent = ts_subtree_new_node(&pool, symbol2, tree_array({ leaf, leaf2, }), 0, &language); ts_subtree_retain(leaf); ts_subtree_retain(leaf2); - Subtree *different_parent = ts_subtree_new_node(&pool, symbol2, tree_array({ + const Subtree *different_parent = ts_subtree_new_node(&pool, symbol2, tree_array({ leaf2, leaf, }), 0, &language); @@ -465,13 +472,13 @@ describe("Subtree", []() { Length padding = {1, {0, 1}}; Length size = {2, {0, 2}}; - auto make_external = [](Subtree *tree) { - tree->has_external_tokens = true; + auto make_external = [](const Subtree *tree) { + ((Subtree *)tree)->has_external_tokens = true; return tree; }; it("returns the last serialized external token state in the given tree", [&]() { - Subtree *tree1, *tree2, *tree3, *tree4, *tree5, *tree6, *tree7, *tree8, *tree9; + const Subtree *tree1, *tree2, *tree3, *tree4, *tree5, *tree6, *tree7, *tree8, *tree9; tree1 = ts_subtree_new_node(&pool, symbol1, tree_array({ (tree2 = ts_subtree_new_node(&pool, symbol2, tree_array({ From a3e08e7c315d35b0936673f650b64406263b9a10 Mon Sep 17 00:00:00 2001 From: Max Brunsfeld Date: Fri, 11 May 2018 16:10:36 -0700 Subject: [PATCH 51/90] Add randomized multi-threaded tests on parse trees Co-Authored-By: Rick Winfrey --- test/helpers/random_helpers.cc | 37 +++++++++----------- test/helpers/random_helpers.h | 23 ++++++++++--- test/integration/real_grammars.cc | 6 ++-- test/runtime/tree_test.cc | 57 +++++++++++++++++++++++++++++++ test/tests.cc | 2 +- 5 files changed, 96 insertions(+), 29 deletions(-) diff --git a/test/helpers/random_helpers.cc b/test/helpers/random_helpers.cc index d79475c7..b3de98e2 100644 --- a/test/helpers/random_helpers.cc +++ b/test/helpers/random_helpers.cc @@ -1,3 +1,4 @@ +#include "helpers/random_helpers.h" #include #include #include @@ -6,54 +7,50 @@ using std::string; using std::vector; -static std::default_random_engine engine; +Generator default_generator(0); unsigned get_time_as_seed() { return time(nullptr); } -void random_reseed(unsigned seed) { +void Generator::reseed(unsigned seed) { engine.seed(seed); } -unsigned random_unsigned() { - return std::uniform_int_distribution()(engine); +unsigned Generator::operator()() { + return distribution(engine); } -unsigned random_unsigned(unsigned max) { - return std::uniform_int_distribution(0, max - 1)(engine); +unsigned Generator::operator()(unsigned max) { + return distribution(engine) % max; } -static string random_string(char min, char max) { +string Generator::str(char min, char max) { string result; - size_t length = random_unsigned(12); + size_t length = operator()(12); for (size_t i = 0; i < length; i++) { - result += (min + random_unsigned(max - min)); + result += (min + operator()(max - min)); } return result; } -static string random_char(string characters) { - size_t index = random_unsigned(characters.size()); - return string() + characters[index]; -} - -string random_words(size_t count) { +string Generator::words(size_t count) { string result; bool just_inserted_word = false; for (size_t i = 0; i < count; i++) { - if (random_unsigned(10) < 6) { - result += random_char("!(){}[]<>+-="); + if (operator()(10) < 6) { + const char *operator_characters = "!(){}[]<>+-="; + result += operator_characters[operator()(strlen(operator_characters))]; } else { if (just_inserted_word) result += " "; - result += random_string('a', 'z'); + result += str('a', 'z'); just_inserted_word = true; } } return result; } -string select_random(const vector &list) { - return list[random_unsigned(list.size())]; +string Generator::select(const vector &list) { + return list[operator()(list.size())]; } diff --git a/test/helpers/random_helpers.h b/test/helpers/random_helpers.h index b66c4aee..d415aafe 100644 --- a/test/helpers/random_helpers.h +++ b/test/helpers/random_helpers.h @@ -3,12 +3,25 @@ #include #include +#include unsigned get_time_as_seed(); -void random_reseed(unsigned); -unsigned random_unsigned(); -unsigned random_unsigned(unsigned max); -std::string random_words(size_t count); -std::string select_random(const std::vector &); + +class Generator { + std::default_random_engine engine; + std::uniform_int_distribution distribution; + +public: + Generator(uint32_t seed) : engine{seed} {} + + void reseed(unsigned); + unsigned operator()(); + unsigned operator()(unsigned max); + std::string words(size_t count); + std::string str(char min, char max); + std::string select(const std::vector &); +}; + +extern Generator default_generator; #endif // HELPERS_RANDOM_HELPERS_H_ diff --git a/test/integration/real_grammars.cc b/test/integration/real_grammars.cc index a628177f..02752a34 100644 --- a/test/integration/real_grammars.cc +++ b/test/integration/real_grammars.cc @@ -76,9 +76,9 @@ for (auto &language_name : test_languages) { set> insertions; for (size_t i = 0; i < 60; i++) { - size_t edit_position = random_unsigned(utf8_char_count(entry.input)); - size_t deletion_size = random_unsigned(utf8_char_count(entry.input) - edit_position); - string inserted_text = random_words(random_unsigned(4) + 1); + size_t edit_position = default_generator(utf8_char_count(entry.input)); + size_t deletion_size = default_generator(utf8_char_count(entry.input) - edit_position); + string inserted_text = default_generator.words(default_generator(4) + 1); if (insertions.insert({edit_position, inserted_text}).second) { it(("parses " + entry.description + diff --git a/test/runtime/tree_test.cc b/test/runtime/tree_test.cc index a87c754b..6d038220 100644 --- a/test/runtime/tree_test.cc +++ b/test/runtime/tree_test.cc @@ -1,4 +1,5 @@ #include "test_helper.h" +#include #include "runtime/alloc.h" #include "helpers/record_alloc.h" #include "helpers/stream_methods.h" @@ -8,6 +9,11 @@ #include "helpers/stderr_logger.h" #include "helpers/spy_input.h" #include "helpers/load_language.h" +#include "helpers/random_helpers.h" +#include "helpers/read_test_entries.h" +#include "helpers/encoding_helpers.h" +#include "helpers/tree_helpers.h" +#include TSPoint point(uint32_t row, uint32_t column) { TSPoint result = {row, column}; @@ -37,6 +43,57 @@ describe("Tree", [&]() { AssertThat(actual, Equals(expected)); }; + describe("copy()", [&]() { + it("returns a tree that can be safely used while the current tree is edited", [&]() { + const TSLanguage *language = load_real_language("javascript"); + ts_parser_set_language(parser, language); + string source_code = examples_for_language("javascript")[0].input; + + input = new SpyInput(source_code, 32); + TSTree *original_tree = ts_parser_parse(parser, nullptr, input->input()); + + vector> new_trees; + for (unsigned i = 0; i < 8; i++) { + TSTree *tree_copy = ts_tree_copy(original_tree); + new_trees.push_back(std::async([i, tree_copy, &source_code, language]() { + Generator random(TREE_SITTER_SEED + i); + + TSTree *tree = tree_copy; + TSParser *parser = ts_parser_new(); + ts_parser_set_language(parser, language); + SpyInput *input = new SpyInput(source_code, 32); + + for (unsigned j = 0; j < 30; j++) { + usleep(random(200)); + + size_t edit_position = random(utf8_char_count(input->content)); + size_t deletion_size = random(utf8_char_count(input->content) - edit_position); + string inserted_text = random.words(random(4) + 1); + + TSInputEdit edit = input->replace(edit_position, deletion_size, inserted_text); + ts_tree_edit(tree, &edit); + + TSTree *new_tree = ts_parser_parse(parser, tree, input->input()); + ts_tree_delete(tree); + tree = new_tree; + } + + ts_parser_delete(parser); + delete input; + + return tree; + })); + } + + for (auto &future : new_trees) { + future.wait(); + TSTree *new_tree = future.get(); + assert_consistent_tree_sizes(ts_tree_root_node(new_tree)); + ts_tree_delete(new_tree); + } + }); + }); + describe("get_changed_ranges()", [&]() { before_each([&]() { ts_parser_set_language(parser, load_real_language("javascript")); diff --git a/test/tests.cc b/test/tests.cc index 303f9059..0d8c23e7 100644 --- a/test/tests.cc +++ b/test/tests.cc @@ -12,7 +12,7 @@ int main(int argc, char *argv[]) { } printf("Random seed: %d\n", TREE_SITTER_SEED); - random_reseed(TREE_SITTER_SEED); + default_generator.reseed(TREE_SITTER_SEED); return bandit::run(argc, argv); } From 043a2fc0d99defac87b21a696b842ad8350dca0e Mon Sep 17 00:00:00 2001 From: Max Brunsfeld Date: Fri, 11 May 2018 16:53:47 -0700 Subject: [PATCH 52/90] Assert absence of memory leaks in randomized multi-threaded tree test --- src/runtime/tree.c | 2 +- test/helpers/record_alloc.cc | 16 ++++++++++++++-- test/helpers/record_alloc.h | 2 +- test/runtime/tree_test.cc | 13 ++++++++----- 4 files changed, 24 insertions(+), 9 deletions(-) diff --git a/src/runtime/tree.c b/src/runtime/tree.c index 3e8a145e..ce1e2b98 100644 --- a/src/runtime/tree.c +++ b/src/runtime/tree.c @@ -37,7 +37,7 @@ TSNode ts_tree_root_node(const TSTree *self) { void ts_tree_edit(TSTree *self, const TSInputEdit *edit) { SubtreePool pool = ts_subtree_pool_new(0); self->root = ts_subtree_edit(self->root, edit, &pool); - assert(pool.tree_stack.capacity == 0 && pool.free_trees.capacity == 0); + ts_subtree_pool_delete(&pool); } TSRange *ts_tree_get_changed_ranges(const TSTree *self, const TSTree *other, uint32_t *count) { diff --git a/test/helpers/record_alloc.cc b/test/helpers/record_alloc.cc index 43e11abe..2e2ea648 100644 --- a/test/helpers/record_alloc.cc +++ b/test/helpers/record_alloc.cc @@ -1,6 +1,7 @@ #include #include #include +#include using std::map; using std::vector; @@ -8,13 +9,16 @@ using std::vector; static bool _enabled = false; static size_t _allocation_count = 0; static map _outstanding_allocations; +static std::mutex _outstanding_allocations_mutex; +static bool _multi_threaded_mode = false; namespace record_alloc { -void start() { +void start(bool multi_threaded_mode) { _enabled = true; _allocation_count = 0; _outstanding_allocations.clear(); + _multi_threaded_mode = multi_threaded_mode; } void stop() { @@ -30,7 +34,11 @@ vector outstanding_allocation_indices() { } size_t allocation_count() { - return _allocation_count; + size_t result; + _outstanding_allocations_mutex.lock(); + result = _allocation_count; + _outstanding_allocations_mutex.unlock(); + return result; } } // namespace record_alloc @@ -39,16 +47,20 @@ extern "C" { static void *record_allocation(void *result) { if (!_enabled) return result; + if (_multi_threaded_mode) _outstanding_allocations_mutex.lock(); _outstanding_allocations[result] = _allocation_count; _allocation_count++; + if (_multi_threaded_mode) _outstanding_allocations_mutex.unlock(); return result; } static void record_deallocation(void *pointer) { + if (_multi_threaded_mode) _outstanding_allocations_mutex.lock(); auto entry = _outstanding_allocations.find(pointer); if (entry != _outstanding_allocations.end()) { _outstanding_allocations.erase(entry); } + if (_multi_threaded_mode) _outstanding_allocations_mutex.unlock(); } void *ts_record_malloc(size_t size) { diff --git a/test/helpers/record_alloc.h b/test/helpers/record_alloc.h index 1f5968ac..f21876b4 100644 --- a/test/helpers/record_alloc.h +++ b/test/helpers/record_alloc.h @@ -5,7 +5,7 @@ namespace record_alloc { -void start(); +void start(bool multi_threaded_mode = false); void stop(); void fail_at_allocation_index(size_t failure_index); std::vector outstanding_allocation_indices(); diff --git a/test/runtime/tree_test.cc b/test/runtime/tree_test.cc index 6d038220..ad9d3204 100644 --- a/test/runtime/tree_test.cc +++ b/test/runtime/tree_test.cc @@ -28,11 +28,17 @@ describe("Tree", [&]() { TSTree *tree; before_each([&]() { + record_alloc::start(true); parser = ts_parser_new(); + tree = nullptr; + input = nullptr; }); after_each([&]() { + if (tree) ts_tree_delete(tree); + if (input) delete input; ts_parser_delete(parser); + AssertThat(record_alloc::outstanding_allocation_indices(), IsEmpty()); }); auto assert_root_node = [&](const string &expected) { @@ -85,6 +91,8 @@ describe("Tree", [&]() { })); } + ts_tree_delete(original_tree); + for (auto &future : new_trees) { future.wait(); TSTree *new_tree = future.get(); @@ -105,11 +113,6 @@ describe("Tree", [&]() { ); }); - after_each([&]() { - ts_tree_delete(tree); - delete input; - }); - auto get_changed_ranges_for_edit = [&](function fn) -> vector { TSInputEdit edit = fn(); ts_tree_edit(tree, &edit); From 32c06b9b5988849eb2041cb51771c235ef48d3d9 Mon Sep 17 00:00:00 2001 From: Max Brunsfeld Date: Fri, 11 May 2018 17:08:46 -0700 Subject: [PATCH 53/90] Make multi-threaded test work on windows Co-Authored-By: Rick Winfrey --- test/helpers/random_helpers.cc | 23 +++++++++++++++++++++-- test/helpers/random_helpers.h | 1 + test/runtime/tree_test.cc | 7 +++---- 3 files changed, 25 insertions(+), 6 deletions(-) diff --git a/test/helpers/random_helpers.cc b/test/helpers/random_helpers.cc index b3de98e2..b0200e51 100644 --- a/test/helpers/random_helpers.cc +++ b/test/helpers/random_helpers.cc @@ -34,13 +34,14 @@ string Generator::str(char min, char max) { return result; } +static string operator_characters = "!(){}[]<>+-="; + string Generator::words(size_t count) { string result; bool just_inserted_word = false; for (size_t i = 0; i < count; i++) { if (operator()(10) < 6) { - const char *operator_characters = "!(){}[]<>+-="; - result += operator_characters[operator()(strlen(operator_characters))]; + result += operator_characters[operator()(operator_characters.size())]; } else { if (just_inserted_word) result += " "; @@ -54,3 +55,21 @@ string Generator::words(size_t count) { string Generator::select(const vector &list) { return list[operator()(list.size())]; } + +#ifdef _WIN32 + +#include + +void Generator::sleep_some() { + Sleep(operator()(5)); +} + +#else + +#include + +void Generator::sleep_some() { + usleep(operator()(5 * 1000)); +} + +#endif diff --git a/test/helpers/random_helpers.h b/test/helpers/random_helpers.h index d415aafe..7dd471fd 100644 --- a/test/helpers/random_helpers.h +++ b/test/helpers/random_helpers.h @@ -20,6 +20,7 @@ public: std::string words(size_t count); std::string str(char min, char max); std::string select(const std::vector &); + void sleep_some(); }; extern Generator default_generator; diff --git a/test/runtime/tree_test.cc b/test/runtime/tree_test.cc index ad9d3204..b599f568 100644 --- a/test/runtime/tree_test.cc +++ b/test/runtime/tree_test.cc @@ -13,7 +13,6 @@ #include "helpers/read_test_entries.h" #include "helpers/encoding_helpers.h" #include "helpers/tree_helpers.h" -#include TSPoint point(uint32_t row, uint32_t column) { TSPoint result = {row, column}; @@ -67,10 +66,10 @@ describe("Tree", [&]() { TSTree *tree = tree_copy; TSParser *parser = ts_parser_new(); ts_parser_set_language(parser, language); - SpyInput *input = new SpyInput(source_code, 32); + SpyInput *input = new SpyInput(source_code, 1024); - for (unsigned j = 0; j < 30; j++) { - usleep(random(200)); + for (unsigned j = 0; j < 10; j++) { + random.sleep_some(); size_t edit_position = random(utf8_char_count(input->content)); size_t deletion_size = random(utf8_char_count(input->content) - edit_position); From b2c5741deddbe5b428242f367f90395faa503234 Mon Sep 17 00:00:00 2001 From: Max Brunsfeld Date: Fri, 11 May 2018 17:19:16 -0700 Subject: [PATCH 54/90] Link tests against libpthread on linux --- tests.gyp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests.gyp b/tests.gyp index ab0de485..80e8d618 100644 --- a/tests.gyp +++ b/tests.gyp @@ -101,7 +101,7 @@ 'cflags_cc': ['-std=c++14'], 'conditions': [ ['OS=="linux"', { - 'libraries': ['-ldl'], + 'libraries': ['-ldl', '-lpthread'], }] ], 'xcode_settings': { From ebddb1a0b5c9b05b8ea67643a9dfc7c5d46a549a Mon Sep 17 00:00:00 2001 From: Max Brunsfeld Date: Wed, 16 May 2018 13:50:53 -0700 Subject: [PATCH 55/90] Add ts_tree_cursor_goto_first_child_for_byte method Atom needs this for efficiently seeking to the leaf node at a given position, visiting all of its ancestors along the way. --- include/tree_sitter/runtime.h | 1 + src/runtime/tree_cursor.c | 51 +++++++++++++++++++++++++++++++++++ test/runtime/node_test.cc | 50 ++++++++++++++++++++++++++++++++++ 3 files changed, 102 insertions(+) diff --git a/include/tree_sitter/runtime.h b/include/tree_sitter/runtime.h index 8f379718..5601c875 100644 --- a/include/tree_sitter/runtime.h +++ b/include/tree_sitter/runtime.h @@ -122,6 +122,7 @@ TSNode ts_node_named_descendant_for_point_range(TSNode, TSPoint, TSPoint); TSTreeCursor *ts_tree_cursor_new(const TSTree *); void ts_tree_cursor_delete(TSTreeCursor *); bool ts_tree_cursor_goto_first_child(TSTreeCursor *); +int64_t ts_tree_cursor_goto_first_child_for_byte(TSTreeCursor *, uint32_t); bool ts_tree_cursor_goto_next_sibling(TSTreeCursor *); bool ts_tree_cursor_goto_parent(TSTreeCursor *); TSNode ts_tree_cursor_current_node(TSTreeCursor *); diff --git a/src/runtime/tree_cursor.c b/src/runtime/tree_cursor.c index 31eee8b4..5541cd65 100644 --- a/src/runtime/tree_cursor.c +++ b/src/runtime/tree_cursor.c @@ -62,6 +62,57 @@ bool ts_tree_cursor_goto_first_child(TSTreeCursor *self) { return false; } +int64_t ts_tree_cursor_goto_first_child_for_byte(TSTreeCursor *self, uint32_t goal_byte) { + uint32_t initial_size = self->stack.size; + TreeCursorEntry *last_entry = array_back(&self->stack); + const Subtree *tree = last_entry->subtree; + Length position = last_entry->position; + uint32_t visible_child_index = 0; + + bool did_descend; + do { + did_descend = false; + + uint32_t structural_child_index = 0; + for (uint32_t i = 0; i < tree->children.size; i++) { + const Subtree *child = tree->children.contents[i]; + Length next_position = length_add(position, ts_subtree_total_size(child)); + bool at_goal = next_position.bytes > goal_byte; + + if (at_goal) { + if (child->visible || child->visible_child_count > 0) { + array_push(&self->stack, ((TreeCursorEntry) { + .subtree = child, + .child_index = i, + .structural_child_index = structural_child_index, + .position = position, + })); + + if (child->visible) { + return visible_child_index; + } else { + tree = child; + did_descend = true; + break; + } + } + } else { + if (child->visible) { + visible_child_index++; + } else { + visible_child_index += child->visible_child_count; + } + } + + if (!child->extra) structural_child_index++; + position = next_position; + } + } while (did_descend); + + self->stack.size = initial_size; + return -1; +} + bool ts_tree_cursor_goto_next_sibling(TSTreeCursor *self) { TreeCursorEntry *child_entry = array_back(&self->stack); diff --git a/test/runtime/node_test.cc b/test/runtime/node_test.cc index 4c306897..555dee5b 100644 --- a/test/runtime/node_test.cc +++ b/test/runtime/node_test.cc @@ -649,6 +649,56 @@ describe("TreeCursor", [&]() { AssertThat(ts_node_type(node), Equals("value")); AssertThat(ts_node_start_byte(node), Equals(array_index)); }); + + it("can find the first child of a given node which spans the given byte offset", [&]() { + int64_t child_index = ts_tree_cursor_goto_first_child_for_byte(cursor, 1); + TSNode node = ts_tree_cursor_current_node(cursor); + AssertThat(ts_node_type(node), Equals("array")); + AssertThat(ts_node_start_byte(node), Equals(array_index)); + AssertThat(child_index, Equals(0)); + + child_index = ts_tree_cursor_goto_first_child_for_byte(cursor, array_index); + node = ts_tree_cursor_current_node(cursor); + AssertThat(ts_node_type(node), Equals("[")); + AssertThat(ts_node_start_byte(node), Equals(array_index)); + AssertThat(child_index, Equals(0)); + + ts_tree_cursor_goto_parent(cursor); + child_index = ts_tree_cursor_goto_first_child_for_byte(cursor, array_index + 1); + node = ts_tree_cursor_current_node(cursor); + AssertThat(ts_node_type(node), Equals("number")); + AssertThat(ts_node_start_byte(node), Equals(number_index)); + AssertThat(child_index, Equals(1)); + + ts_tree_cursor_goto_parent(cursor); + child_index = ts_tree_cursor_goto_first_child_for_byte(cursor, number_index + 1); + node = ts_tree_cursor_current_node(cursor); + AssertThat(ts_node_type(node), Equals("number")); + AssertThat(ts_node_start_byte(node), Equals(number_index)); + AssertThat(child_index, Equals(1)); + + ts_tree_cursor_goto_parent(cursor); + child_index = ts_tree_cursor_goto_first_child_for_byte(cursor, false_index - 1); + node = ts_tree_cursor_current_node(cursor); + AssertThat(ts_node_type(node), Equals("false")); + AssertThat(ts_node_start_byte(node), Equals(false_index)); + AssertThat(child_index, Equals(3)); + + ts_tree_cursor_goto_parent(cursor); + child_index = ts_tree_cursor_goto_first_child_for_byte(cursor, object_end_index - 1); + node = ts_tree_cursor_current_node(cursor); + AssertThat(ts_node_type(node), Equals("object")); + AssertThat(ts_node_start_byte(node), Equals(object_index)); + AssertThat(child_index, Equals(5)); + + // There is no child past the end of the array + ts_tree_cursor_goto_parent(cursor); + child_index = ts_tree_cursor_goto_first_child_for_byte(cursor, array_end_index); + node = ts_tree_cursor_current_node(cursor); + AssertThat(ts_node_type(node), Equals("array")); + AssertThat(ts_node_start_byte(node), Equals(array_index)); + AssertThat(child_index, Equals(-1)); + }); }); END_TEST From 6fc8d9871c2a5b9ba11847271f3c2ec69154c0db Mon Sep 17 00:00:00 2001 From: Max Brunsfeld Date: Wed, 16 May 2018 15:44:04 -0700 Subject: [PATCH 56/90] Hide the details of TSNode's fields in the public API --- include/tree_sitter/runtime.h | 8 +- src/runtime/node.c | 162 ++++++++++++++++++++-------------- src/runtime/tree.c | 8 +- src/runtime/tree.h | 1 + src/runtime/tree_cursor.c | 8 +- test/helpers/tree_helpers.cc | 6 +- test/runtime/node_test.cc | 27 +++--- 7 files changed, 120 insertions(+), 100 deletions(-) diff --git a/include/tree_sitter/runtime.h b/include/tree_sitter/runtime.h index 5601c875..42fc7ea4 100644 --- a/include/tree_sitter/runtime.h +++ b/include/tree_sitter/runtime.h @@ -66,11 +66,8 @@ typedef struct { } TSInputEdit; typedef struct { - const void *subtree; - const TSTree *tree; - TSPoint position; - uint32_t byte; - TSSymbol alias_symbol; + const void *ref[2]; + uint32_t context[4]; } TSNode; TSParser *ts_parser_new(); @@ -99,6 +96,7 @@ TSSymbol ts_node_symbol(TSNode); const char *ts_node_type(TSNode); char *ts_node_string(TSNode); bool ts_node_eq(TSNode, TSNode); +bool ts_node_is_null(TSNode); bool ts_node_is_named(TSNode); bool ts_node_is_missing(TSNode); bool ts_node_has_changes(TSNode); diff --git a/src/runtime/node.c b/src/runtime/node.c index 1f221d12..24ce7b13 100644 --- a/src/runtime/node.c +++ b/src/runtime/node.c @@ -3,8 +3,6 @@ #include "runtime/tree.h" #include "runtime/language.h" -// NodeChildIterator - typedef struct { const Subtree *parent; const TSTree *tree; @@ -14,31 +12,54 @@ typedef struct { const TSSymbol *alias_sequence; } NodeChildIterator; -// TSNode - Private +// TSNode - constructors -static inline TSNode ts_node__null() { +TSNode ts_node_new(const TSTree *tree, const Subtree *subtree, Length position, TSSymbol alias) { return (TSNode) { - .subtree = NULL, - .tree = NULL, - .position = {0, 0}, - .byte = 0, + {tree, subtree}, + {position.bytes, position.extent.row, position.extent.column, alias} }; } -static inline const Subtree *ts_node__tree(TSNode self) { - return self.subtree; +static inline TSNode ts_node__null() { + return (TSNode) {{NULL, NULL}, {0, 0, 0, 0}}; } +// TSNode - accessors + +static inline const TSTree *ts_node__tree(const TSNode *self) { + return self->ref[0]; +} + +static inline const Subtree *ts_node__subtree(TSNode self) { + return self.ref[1]; +} + +static inline uint32_t ts_node__byte(const TSNode *self) { + return self->context[0]; +} + +static inline TSPoint ts_node__position(const TSNode *self) { + return (TSPoint) {self->context[1], self->context[2]}; +} + +static inline uint32_t ts_node__alias(const TSNode *self) { + return self->context[3]; +} + +// NodeChildIterator + static inline NodeChildIterator ts_node_child_iterator_begin(const TSNode *node) { - const Subtree *tree = ts_node__tree(*node); + const TSTree *tree = ts_node__tree(node); + const Subtree *subtree = ts_node__subtree(*node); const TSSymbol *alias_sequence = ts_language_alias_sequence( - node->tree->language, - tree->alias_sequence_id + tree->language, + subtree->alias_sequence_id ); return (NodeChildIterator) { - .parent = tree, - .tree = node->tree, - .position = {node->byte, node->position}, + .tree = tree, + .parent = subtree, + .position = {ts_node__byte(node), ts_node__position(node)}, .child_index = 0, .structural_child_index = 0, .alias_sequence = alias_sequence, @@ -55,30 +76,31 @@ static inline bool ts_node_child_iterator_next(NodeChildIterator *self, TSNode * } self->structural_child_index++; } - *result = (TSNode) { - .subtree = child, - .tree = self->tree, - .position = self->position.extent, - .byte = self->position.bytes, - .alias_symbol = alias_symbol, - }; + *result = ts_node_new( + self->tree, + child, + self->position, + alias_symbol + ); self->position = length_add(self->position, ts_subtree_total_size(child)); self->child_index++; return true; } +// TSNode - private + static inline bool ts_node__is_relevant(TSNode self, bool include_anonymous) { - const Subtree *tree = ts_node__tree(self); + const Subtree *tree = ts_node__subtree(self); if (include_anonymous) { - return tree->visible || self.alias_symbol; + return tree->visible || ts_node__alias(&self); } else { return ( (tree->visible && tree->named) || ( - self.alias_symbol && + ts_node__alias(&self) && ts_language_symbol_metadata( - self.tree->language, - self.alias_symbol + ts_node__tree(&self)->language, + ts_node__alias(&self) ).named ) ); @@ -86,7 +108,7 @@ static inline bool ts_node__is_relevant(TSNode self, bool include_anonymous) { } static inline uint32_t ts_node__relevant_child_count(TSNode self, bool include_anonymous) { - const Subtree *tree = ts_node__tree(self); + const Subtree *tree = ts_node__subtree(self); if (tree->children.size > 0) { if (include_anonymous) { return tree->visible_child_count; @@ -136,7 +158,7 @@ static inline TSNode ts_node__prev_sibling(TSNode self, bool include_anonymous) TSNode earlier_node = ts_node__null(); bool earlier_node_is_relevant = false; - while (node.subtree) { + while (ts_node__subtree(node)) { TSNode earlier_child = ts_node__null(); bool earlier_child_is_relevant = false; bool found_child_containing_target = false; @@ -145,7 +167,7 @@ static inline TSNode ts_node__prev_sibling(TSNode self, bool include_anonymous) NodeChildIterator iterator = ts_node_child_iterator_begin(&node); while (ts_node_child_iterator_next(&iterator, &child)) { if (iterator.position.bytes >= target_end_byte) { - found_child_containing_target = child.subtree != self.subtree; + found_child_containing_target = ts_node__subtree(child) != ts_node__subtree(self); break; } @@ -159,14 +181,14 @@ static inline TSNode ts_node__prev_sibling(TSNode self, bool include_anonymous) } if (found_child_containing_target) { - if (earlier_child.subtree) { + if (!ts_node_is_null(earlier_child)) { earlier_node = earlier_child; earlier_node_is_relevant = earlier_child_is_relevant; } node = child; } else if (earlier_child_is_relevant) { return earlier_child; - } else if (earlier_child.subtree) { + } else if (!ts_node_is_null(earlier_child)) { node = earlier_child; } else if (earlier_node_is_relevant) { return earlier_node; @@ -185,7 +207,7 @@ static inline TSNode ts_node__next_sibling(TSNode self, bool include_anonymous) TSNode later_node = ts_node__null(); bool later_node_is_relevant = false; - while (node.subtree) { + while (ts_node__subtree(node)) { TSNode later_child = ts_node__null(); bool later_child_is_relevant = false; TSNode child_containing_target = ts_node__null(); @@ -194,8 +216,8 @@ static inline TSNode ts_node__next_sibling(TSNode self, bool include_anonymous) NodeChildIterator iterator = ts_node_child_iterator_begin(&node); while (ts_node_child_iterator_next(&iterator, &child)) { if (iterator.position.bytes < target_end_byte) continue; - if (child.byte <= self.byte) { - if (child.subtree != self.subtree) { + if (ts_node__byte(&child) <= ts_node__byte(&self)) { + if (ts_node__subtree(child) != ts_node__subtree(self)) { child_containing_target = child; } } else if (ts_node__is_relevant(child, include_anonymous)) { @@ -209,15 +231,15 @@ static inline TSNode ts_node__next_sibling(TSNode self, bool include_anonymous) } } - if (child_containing_target.subtree) { - if (later_child.subtree) { + if (!ts_node_is_null(child_containing_target)) { + if (!ts_node_is_null(later_child)) { later_node = later_child; later_node_is_relevant = later_child_is_relevant; } node = child_containing_target; } else if (later_child_is_relevant) { return later_child; - } else if (later_child.subtree) { + } else if (!ts_node_is_null(later_child)) { node = later_child; } else if (later_node_is_relevant) { return later_node; @@ -229,8 +251,8 @@ static inline TSNode ts_node__next_sibling(TSNode self, bool include_anonymous) return ts_node__null(); } -static inline bool point_gt(TSPoint a, TSPoint b) { - return a.row > b.row || (a.row == b.row && a.column > b.column); +static inline bool point_gt(TSPoint self, TSPoint other) { + return self.row > other.row || (self.row == other.row && self.column > other.column); } static inline TSNode ts_node__first_child_for_byte(TSNode self, uint32_t goal, @@ -273,7 +295,7 @@ static inline TSNode ts_node__descendant_for_byte_range(TSNode self, uint32_t mi NodeChildIterator iterator = ts_node_child_iterator_begin(&node); while (ts_node_child_iterator_next(&iterator, &child)) { if (iterator.position.bytes > max) { - if (child.byte > min) break; + if (ts_node__byte(&child) > min) break; node = child; if (ts_node__is_relevant(node, include_anonymous)) last_visible_node = node; did_descend = true; @@ -300,7 +322,7 @@ static inline TSNode ts_node__descendant_for_point_range(TSNode self, TSPoint mi TSNode child; NodeChildIterator iterator = ts_node_child_iterator_begin(&node); while (ts_node_child_iterator_next(&iterator, &child)) { - const Subtree *child_tree = ts_node__tree(child); + const Subtree *child_tree = ts_node__subtree(child); if (iterator.child_index != 1) { start_position = point_add(start_position, child_tree->padding.extent); } @@ -319,68 +341,75 @@ static inline TSNode ts_node__descendant_for_point_range(TSNode self, TSPoint mi return last_visible_node; } -// TSNode - Public +// TSNode - public uint32_t ts_node_start_byte(TSNode self) { - return self.byte + ts_node__tree(self)->padding.bytes; + return ts_node__byte(&self) + ts_node__subtree(self)->padding.bytes; } uint32_t ts_node_end_byte(TSNode self) { - return ts_node_start_byte(self) + ts_node__tree(self)->size.bytes; + return ts_node_start_byte(self) + ts_node__subtree(self)->size.bytes; } TSPoint ts_node_start_point(TSNode self) { - return point_add(self.position, ts_node__tree(self)->padding.extent); + return point_add( + ts_node__position(&self), + ts_node__subtree(self)->padding.extent + ); } TSPoint ts_node_end_point(TSNode self) { - return point_add(ts_node_start_point(self), ts_node__tree(self)->size.extent); + return point_add(ts_node_start_point(self), ts_node__subtree(self)->size.extent); } TSSymbol ts_node_symbol(TSNode self) { - const Subtree *tree = ts_node__tree(self); - return self.alias_symbol ? self.alias_symbol : tree->symbol; + const Subtree *tree = ts_node__subtree(self); + return ts_node__alias(&self) ? ts_node__alias(&self) : tree->symbol; } const char *ts_node_type(TSNode self) { - return ts_language_symbol_name(self.tree->language, ts_node_symbol(self)); + return ts_language_symbol_name(ts_node__tree(&self)->language, ts_node_symbol(self)); } char *ts_node_string(TSNode self) { - return ts_subtree_string(ts_node__tree(self), self.tree->language, false); + return ts_subtree_string(ts_node__subtree(self), ts_node__tree(&self)->language, false); } bool ts_node_eq(TSNode self, TSNode other) { return ( - ts_subtree_eq(ts_node__tree(self), ts_node__tree(other)) && - self.byte == other.byte + ts_subtree_eq(ts_node__subtree(self), ts_node__subtree(other)) && + ts_node__byte(&self) == ts_node__byte(&other) ); } +bool ts_node_is_null(TSNode self) { + return ts_node__subtree(self) == NULL; +} + bool ts_node_is_named(TSNode self) { - const Subtree *tree = ts_node__tree(self); - return self.alias_symbol - ? ts_language_symbol_metadata(self.tree->language, self.alias_symbol).named + const Subtree *tree = ts_node__subtree(self); + return ts_node__alias(&self) + ? ts_language_symbol_metadata(ts_node__tree(&self)->language, ts_node__alias(&self)).named : tree->named; } bool ts_node_is_missing(TSNode self) { - const Subtree *tree = ts_node__tree(self); + const Subtree *tree = ts_node__subtree(self); return tree->is_missing; } bool ts_node_has_changes(TSNode self) { - return ts_node__tree(self)->has_changes; + return ts_node__subtree(self)->has_changes; } bool ts_node_has_error(TSNode self) { - return ts_node__tree(self)->error_cost > 0; + return ts_node__subtree(self)->error_cost > 0; } TSNode ts_node_parent(TSNode self) { - TSNode node = ts_tree_root_node(self.tree); + TSNode node = ts_tree_root_node(ts_node__tree(&self)); uint32_t end_byte = ts_node_end_byte(self); - if (node.subtree == self.subtree) return ts_node__null(); + if (ts_node__subtree(node) == ts_node__subtree(self)) return ts_node__null(); TSNode last_visible_node = node; bool did_descend = true; @@ -390,7 +419,10 @@ TSNode ts_node_parent(TSNode self) { TSNode child; NodeChildIterator iterator = ts_node_child_iterator_begin(&node); while (ts_node_child_iterator_next(&iterator, &child)) { - if (child.byte > self.byte || child.subtree == self.subtree) break; + if ( + ts_node__byte(&child) > ts_node__byte(&self) || + ts_node__subtree(child) == ts_node__subtree(self) + ) break; if (iterator.position.bytes >= end_byte) { node = child; if (ts_node__is_relevant(child, true)) { @@ -414,7 +446,7 @@ TSNode ts_node_named_child(TSNode self, uint32_t child_index) { } uint32_t ts_node_child_count(TSNode self) { - const Subtree *tree = ts_node__tree(self); + const Subtree *tree = ts_node__subtree(self); if (tree->children.size > 0) { return tree->visible_child_count; } else { @@ -423,7 +455,7 @@ uint32_t ts_node_child_count(TSNode self) { } uint32_t ts_node_named_child_count(TSNode self) { - const Subtree *tree = ts_node__tree(self); + const Subtree *tree = ts_node__subtree(self); if (tree->children.size > 0) { return tree->named_child_count; } else { diff --git a/src/runtime/tree.c b/src/runtime/tree.c index ce1e2b98..4a64483b 100644 --- a/src/runtime/tree.c +++ b/src/runtime/tree.c @@ -25,13 +25,7 @@ void ts_tree_delete(TSTree *self) { } TSNode ts_tree_root_node(const TSTree *self) { - return (TSNode) { - .subtree = self->root, - .tree = self, - .position = {0, 0}, - .byte = 0, - .alias_symbol = 0, - }; + return ts_node_new(self, self->root, length_zero(), 0); } void ts_tree_edit(TSTree *self, const TSInputEdit *edit) { diff --git a/src/runtime/tree.h b/src/runtime/tree.h index 7429e06c..50ae8490 100644 --- a/src/runtime/tree.h +++ b/src/runtime/tree.h @@ -11,6 +11,7 @@ struct TSTree { }; TSTree *ts_tree_new(const Subtree *root, const TSLanguage *language); +TSNode ts_node_new(const TSTree *, const Subtree *, Length, TSSymbol); #ifdef __cplusplus } diff --git a/src/runtime/tree_cursor.c b/src/runtime/tree_cursor.c index 5541cd65..5300f3d3 100644 --- a/src/runtime/tree_cursor.c +++ b/src/runtime/tree_cursor.c @@ -179,11 +179,5 @@ TSNode ts_tree_cursor_current_node(TSTreeCursor *self) { alias_symbol = alias_sequence[last_entry->structural_child_index]; } } - return (TSNode) { - .tree = self->tree, - .subtree = last_entry->subtree, - .position = last_entry->position.extent, - .byte = last_entry->position.bytes, - .alias_symbol = alias_symbol, - }; + return ts_node_new(self->tree, last_entry->subtree, last_entry->position, alias_symbol); } diff --git a/test/helpers/tree_helpers.cc b/test/helpers/tree_helpers.cc index 0e63a19f..c7196713 100644 --- a/test/helpers/tree_helpers.cc +++ b/test/helpers/tree_helpers.cc @@ -35,13 +35,13 @@ ostream &operator<<(std::ostream &stream, const Subtree *tree) { } ostream &operator<<(ostream &stream, const TSNode &node) { - if (node.subtree) { + if (ts_node_is_null(node)) { + return stream << "NULL"; + } else { char *string = ts_node_string(node); stream << "{" << string << ", " << to_string(ts_node_start_byte(node)) << "}"; ts_free(string); return stream; - } else { - return stream << "NULL"; } } diff --git a/test/runtime/node_test.cc b/test/runtime/node_test.cc index 555dee5b..68e79467 100644 --- a/test/runtime/node_test.cc +++ b/test/runtime/node_test.cc @@ -65,6 +65,7 @@ describe("Node", [&]() { TSParser *parser; TSTree *tree; TSNode root_node; + TSNode NULL_NODE = {}; before_each([&]() { record_alloc::start(); @@ -158,7 +159,7 @@ describe("Node", [&]() { AssertThat(ts_node_parent(number_node), Equals(root_node)); AssertThat(ts_node_parent(false_node), Equals(root_node)); AssertThat(ts_node_parent(object_node), Equals(root_node)); - AssertThat(ts_node_parent(ts_tree_root_node(tree)).subtree, Equals(nullptr)); + AssertThat(ts_node_parent(ts_tree_root_node(tree)), Equals(NULL_NODE)); }); it("works correctly when the node contains aliased children and extras", [&]() { @@ -244,7 +245,7 @@ describe("Node", [&]() { child = ts_node_first_named_child_for_byte(root_node, object_index + 1); AssertThat(ts_node_type(child), Equals("object")); child = ts_node_first_named_child_for_byte(root_node, object_end_index); - AssertThat(child.subtree, Equals(nullptr)); + AssertThat(child, Equals(NULL_NODE)); }); }); @@ -328,7 +329,7 @@ describe("Node", [&]() { AssertThat(ts_node_parent(child5), Equals(root_node)); AssertThat(ts_node_parent(child6), Equals(root_node)); AssertThat(ts_node_parent(child7), Equals(root_node)); - AssertThat(ts_node_parent(ts_tree_root_node(tree)).subtree, Equals(nullptr)); + AssertThat(ts_node_parent(ts_tree_root_node(tree)), Equals(NULL_NODE)); }); }); @@ -355,9 +356,9 @@ describe("Node", [&]() { AssertThat(ts_node_next_sibling(false_node), Equals(array_comma_node2)); AssertThat(ts_node_next_sibling(array_comma_node2), Equals(object_node)); AssertThat(ts_node_next_sibling(object_node), Equals(bracket_node2)); - AssertThat(ts_node_next_sibling(bracket_node2).subtree, Equals(nullptr)); + AssertThat(ts_node_next_sibling(bracket_node2), Equals(NULL_NODE)); - AssertThat(ts_node_prev_sibling(bracket_node1).subtree, Equals(nullptr)); + AssertThat(ts_node_prev_sibling(bracket_node1), Equals(NULL_NODE)); AssertThat(ts_node_prev_sibling(number_node), Equals(bracket_node1)); AssertThat(ts_node_prev_sibling(array_comma_node1), Equals(number_node)); AssertThat(ts_node_prev_sibling(false_node), Equals(array_comma_node1)); @@ -367,24 +368,24 @@ describe("Node", [&]() { AssertThat(ts_node_next_sibling(brace_node1), Equals(pair_node)); AssertThat(ts_node_next_sibling(pair_node), Equals(brace_node2)); - AssertThat(ts_node_next_sibling(brace_node2).subtree, Equals(nullptr)); + AssertThat(ts_node_next_sibling(brace_node2), Equals(NULL_NODE)); - AssertThat(ts_node_prev_sibling(brace_node1).subtree, Equals(nullptr)); + AssertThat(ts_node_prev_sibling(brace_node1), Equals(NULL_NODE)); AssertThat(ts_node_prev_sibling(pair_node), Equals(brace_node1)); AssertThat(ts_node_prev_sibling(brace_node2), Equals(pair_node)); AssertThat(ts_node_next_sibling(string_node), Equals(colon_node)); AssertThat(ts_node_next_sibling(colon_node), Equals(null_node)); - AssertThat(ts_node_next_sibling(null_node).subtree, Equals(nullptr)); + AssertThat(ts_node_next_sibling(null_node), Equals(NULL_NODE)); - AssertThat(ts_node_prev_sibling(string_node).subtree, Equals(nullptr)); + AssertThat(ts_node_prev_sibling(string_node), Equals(NULL_NODE)); AssertThat(ts_node_prev_sibling(colon_node), Equals(string_node)); AssertThat(ts_node_prev_sibling(null_node), Equals(colon_node)); }); it("returns null when the node has no parent", [&]() { - AssertThat(ts_node_next_named_sibling(root_node).subtree, Equals(nullptr)); - AssertThat(ts_node_prev_named_sibling(root_node).subtree, Equals(nullptr)); + AssertThat(ts_node_next_named_sibling(root_node), Equals(NULL_NODE)); + AssertThat(ts_node_prev_named_sibling(root_node), Equals(NULL_NODE)); }); }); @@ -406,8 +407,8 @@ describe("Node", [&]() { }); it("returns null when the node has no parent", [&]() { - AssertThat(ts_node_next_named_sibling(root_node).subtree, Equals(nullptr)); - AssertThat(ts_node_prev_named_sibling(root_node).subtree, Equals(nullptr)); + AssertThat(ts_node_next_named_sibling(root_node), Equals(NULL_NODE)); + AssertThat(ts_node_prev_named_sibling(root_node), Equals(NULL_NODE)); }); }); From e3670be42fdbee93baef062ad5b466bd486b9f8c Mon Sep 17 00:00:00 2001 From: Max Brunsfeld Date: Wed, 16 May 2018 16:05:08 -0700 Subject: [PATCH 57/90] Avoid one heap allocation when instantiating a TSTreeCursor --- include/tree_sitter/runtime.h | 10 ++- src/runtime/get_changed_ranges.c | 6 +- src/runtime/get_changed_ranges.h | 3 +- src/runtime/node.c | 4 +- src/runtime/tree.c | 2 +- src/runtime/tree_cursor.c | 29 ++++---- src/runtime/tree_cursor.h | 8 +-- test/runtime/node_test.cc | 120 +++++++++++++++---------------- 8 files changed, 96 insertions(+), 86 deletions(-) diff --git a/include/tree_sitter/runtime.h b/include/tree_sitter/runtime.h index 42fc7ea4..71c8b5b3 100644 --- a/include/tree_sitter/runtime.h +++ b/include/tree_sitter/runtime.h @@ -16,7 +16,6 @@ typedef unsigned short TSSymbol; typedef struct TSLanguage TSLanguage; typedef struct TSParser TSParser; typedef struct TSTree TSTree; -typedef struct TSTreeCursor TSTreeCursor; typedef enum { TSInputEncodingUTF8, @@ -66,10 +65,15 @@ typedef struct { } TSInputEdit; typedef struct { - const void *ref[2]; uint32_t context[4]; + const void *ref[2]; } TSNode; +typedef struct { + uint32_t context[2]; + void *ref[2]; +} TSTreeCursor; + TSParser *ts_parser_new(); void ts_parser_delete(TSParser *); const TSLanguage *ts_parser_language(const TSParser *); @@ -117,7 +121,7 @@ TSNode ts_node_named_descendant_for_byte_range(TSNode, uint32_t, uint32_t); TSNode ts_node_descendant_for_point_range(TSNode, TSPoint, TSPoint); TSNode ts_node_named_descendant_for_point_range(TSNode, TSPoint, TSPoint); -TSTreeCursor *ts_tree_cursor_new(const TSTree *); +TSTreeCursor ts_tree_cursor_new(const TSTree *); void ts_tree_cursor_delete(TSTreeCursor *); bool ts_tree_cursor_goto_first_child(TSTreeCursor *); int64_t ts_tree_cursor_goto_first_child_for_byte(TSTreeCursor *, uint32_t); diff --git a/src/runtime/get_changed_ranges.c b/src/runtime/get_changed_ranges.c index 2b015d27..d46bf704 100644 --- a/src/runtime/get_changed_ranges.c +++ b/src/runtime/get_changed_ranges.c @@ -25,13 +25,13 @@ static void range_array_add(RangeArray *results, TSPoint start, TSPoint end) { } typedef struct { - TSTreeCursor cursor; + TreeCursor cursor; const TSLanguage *language; unsigned visible_depth; bool in_padding; } Iterator; -static Iterator iterator_new(TSTreeCursor *cursor, const Subtree *tree, const TSLanguage *language) { +static Iterator iterator_new(TreeCursor *cursor, const Subtree *tree, const TSLanguage *language) { array_clear(&cursor->stack); array_push(&cursor->stack, ((TreeCursorEntry){ .subtree = tree, @@ -262,7 +262,7 @@ static inline void iterator_print_state(Iterator *self) { #endif unsigned ts_subtree_get_changed_ranges(const Subtree *old_tree, const Subtree *new_tree, - TSTreeCursor *cursor1, TSTreeCursor *cursor2, + TreeCursor *cursor1, TreeCursor *cursor2, const TSLanguage *language, TSRange **ranges) { RangeArray results = array_new(); diff --git a/src/runtime/get_changed_ranges.h b/src/runtime/get_changed_ranges.h index fbe9cc23..9daeb919 100644 --- a/src/runtime/get_changed_ranges.h +++ b/src/runtime/get_changed_ranges.h @@ -1,11 +1,12 @@ #ifndef RUNTIME_GET_CHANGED_RANGES_H_ #define RUNTIME_GET_CHANGED_RANGES_H_ +#include "runtime/tree_cursor.h" #include "runtime/subtree.h" unsigned ts_subtree_get_changed_ranges( const Subtree *old_tree, const Subtree *new_tree, - TSTreeCursor *cursor1, TSTreeCursor *cursor2, + TreeCursor *cursor1, TreeCursor *cursor2, const TSLanguage *language, TSRange **ranges ); diff --git a/src/runtime/node.c b/src/runtime/node.c index 24ce7b13..8eaaf220 100644 --- a/src/runtime/node.c +++ b/src/runtime/node.c @@ -16,13 +16,13 @@ typedef struct { TSNode ts_node_new(const TSTree *tree, const Subtree *subtree, Length position, TSSymbol alias) { return (TSNode) { + {position.bytes, position.extent.row, position.extent.column, alias}, {tree, subtree}, - {position.bytes, position.extent.row, position.extent.column, alias} }; } static inline TSNode ts_node__null() { - return (TSNode) {{NULL, NULL}, {0, 0, 0, 0}}; + return ts_node_new(NULL, NULL, length_zero(), 0); } // TSNode - accessors diff --git a/src/runtime/tree.c b/src/runtime/tree.c index 4a64483b..5e82df89 100644 --- a/src/runtime/tree.c +++ b/src/runtime/tree.c @@ -36,7 +36,7 @@ void ts_tree_edit(TSTree *self, const TSInputEdit *edit) { TSRange *ts_tree_get_changed_ranges(const TSTree *self, const TSTree *other, uint32_t *count) { TSRange *result; - TSTreeCursor cursor1, cursor2; + TreeCursor cursor1, cursor2; ts_tree_cursor_init(&cursor1, self); ts_tree_cursor_init(&cursor2, self); *count = ts_subtree_get_changed_ranges( diff --git a/src/runtime/tree_cursor.c b/src/runtime/tree_cursor.c index 5300f3d3..d7695634 100644 --- a/src/runtime/tree_cursor.c +++ b/src/runtime/tree_cursor.c @@ -4,13 +4,13 @@ #include "runtime/language.h" #include "runtime/tree.h" -TSTreeCursor *ts_tree_cursor_new(const TSTree *tree) { - TSTreeCursor *self = ts_malloc(sizeof(TSTreeCursor)); - ts_tree_cursor_init(self, tree); +TSTreeCursor ts_tree_cursor_new(const TSTree *tree) { + TSTreeCursor self; + ts_tree_cursor_init((TreeCursor *)&self, tree); return self; } -void ts_tree_cursor_init(TSTreeCursor *self, const TSTree *tree) { +void ts_tree_cursor_init(TreeCursor *self, const TSTree *tree) { self->tree = tree; array_init(&self->stack); array_push(&self->stack, ((TreeCursorEntry) { @@ -21,12 +21,13 @@ void ts_tree_cursor_init(TSTreeCursor *self, const TSTree *tree) { })); } -void ts_tree_cursor_delete(TSTreeCursor *self) { +void ts_tree_cursor_delete(TSTreeCursor *_self) { + TreeCursor *self = (TreeCursor *)_self; array_delete(&self->stack); - ts_free(self); } -bool ts_tree_cursor_goto_first_child(TSTreeCursor *self) { +bool ts_tree_cursor_goto_first_child(TSTreeCursor *_self) { + TreeCursor *self = (TreeCursor *)_self; TreeCursorEntry *last_entry = array_back(&self->stack); const Subtree *tree = last_entry->subtree; Length position = last_entry->position; @@ -62,7 +63,8 @@ bool ts_tree_cursor_goto_first_child(TSTreeCursor *self) { return false; } -int64_t ts_tree_cursor_goto_first_child_for_byte(TSTreeCursor *self, uint32_t goal_byte) { +int64_t ts_tree_cursor_goto_first_child_for_byte(TSTreeCursor *_self, uint32_t goal_byte) { + TreeCursor *self = (TreeCursor *)_self; uint32_t initial_size = self->stack.size; TreeCursorEntry *last_entry = array_back(&self->stack); const Subtree *tree = last_entry->subtree; @@ -113,7 +115,8 @@ int64_t ts_tree_cursor_goto_first_child_for_byte(TSTreeCursor *self, uint32_t go return -1; } -bool ts_tree_cursor_goto_next_sibling(TSTreeCursor *self) { +bool ts_tree_cursor_goto_next_sibling(TSTreeCursor *_self) { + TreeCursor *self = (TreeCursor *)_self; TreeCursorEntry *child_entry = array_back(&self->stack); for (unsigned i = self->stack.size - 2; i + 1 > 0; i--) { @@ -142,7 +145,7 @@ bool ts_tree_cursor_goto_next_sibling(TSTreeCursor *self) { if (child->visible) { return true; } else { - ts_tree_cursor_goto_first_child(self); + ts_tree_cursor_goto_first_child(_self); return true; } } @@ -155,7 +158,8 @@ bool ts_tree_cursor_goto_next_sibling(TSTreeCursor *self) { return false; } -bool ts_tree_cursor_goto_parent(TSTreeCursor *self) { +bool ts_tree_cursor_goto_parent(TSTreeCursor *_self) { + TreeCursor *self = (TreeCursor *)_self; for (unsigned i = self->stack.size - 2; i + 1 > 0; i--) { TreeCursorEntry *entry = &self->stack.contents[i]; if (entry->subtree->visible) { @@ -166,7 +170,8 @@ bool ts_tree_cursor_goto_parent(TSTreeCursor *self) { return false; } -TSNode ts_tree_cursor_current_node(TSTreeCursor *self) { +TSNode ts_tree_cursor_current_node(TSTreeCursor *_self) { + TreeCursor *self = (TreeCursor *)_self; TreeCursorEntry *last_entry = array_back(&self->stack); TSSymbol alias_symbol = 0; if (self->stack.size > 1) { diff --git a/src/runtime/tree_cursor.h b/src/runtime/tree_cursor.h index 615e7e06..0fa1b167 100644 --- a/src/runtime/tree_cursor.h +++ b/src/runtime/tree_cursor.h @@ -10,11 +10,11 @@ typedef struct { uint32_t structural_child_index; } TreeCursorEntry; -struct TSTreeCursor { - const TSTree *tree; +typedef struct { Array(TreeCursorEntry) stack; -}; + const TSTree *tree; +} TreeCursor; -void ts_tree_cursor_init(TSTreeCursor *, const TSTree *); +void ts_tree_cursor_init(TreeCursor *, const TSTree *); #endif // RUNTIME_TREE_CURSOR_H_ diff --git a/test/runtime/node_test.cc b/test/runtime/node_test.cc index 68e79467..3cbd48c8 100644 --- a/test/runtime/node_test.cc +++ b/test/runtime/node_test.cc @@ -527,7 +527,7 @@ describe("Node", [&]() { describe("TreeCursor", [&]() { TSParser *parser; TSTree *tree; - TSTreeCursor *cursor; + TSTreeCursor cursor; before_each([&]() { record_alloc::start(); @@ -540,7 +540,7 @@ describe("TreeCursor", [&]() { after_each([&]() { ts_tree_delete(tree); - ts_tree_cursor_delete(cursor); + ts_tree_cursor_delete(&cursor); ts_parser_delete(parser); record_alloc::stop(); @@ -548,154 +548,154 @@ describe("TreeCursor", [&]() { }); it("can walk the tree", [&]() { - TSNode node = ts_tree_cursor_current_node(cursor); + TSNode node = ts_tree_cursor_current_node(&cursor); AssertThat(ts_node_type(node), Equals("value")); AssertThat(ts_node_start_byte(node), Equals(array_index)); - AssertThat(ts_tree_cursor_goto_first_child(cursor), IsTrue()); - node = ts_tree_cursor_current_node(cursor); + AssertThat(ts_tree_cursor_goto_first_child(&cursor), IsTrue()); + node = ts_tree_cursor_current_node(&cursor); AssertThat(ts_node_type(node), Equals("array")); AssertThat(ts_node_start_byte(node), Equals(array_index)); - AssertThat(ts_tree_cursor_goto_first_child(cursor), IsTrue()); - node = ts_tree_cursor_current_node(cursor); + AssertThat(ts_tree_cursor_goto_first_child(&cursor), IsTrue()); + node = ts_tree_cursor_current_node(&cursor); AssertThat(ts_node_type(node), Equals("[")); AssertThat(ts_node_start_byte(node), Equals(array_index)); // Cannot descend into a node with no children - AssertThat(ts_tree_cursor_goto_first_child(cursor), IsFalse()); - node = ts_tree_cursor_current_node(cursor); + AssertThat(ts_tree_cursor_goto_first_child(&cursor), IsFalse()); + node = ts_tree_cursor_current_node(&cursor); AssertThat(ts_node_type(node), Equals("[")); AssertThat(ts_node_start_byte(node), Equals(array_index)); - AssertThat(ts_tree_cursor_goto_next_sibling(cursor), IsTrue()); - node = ts_tree_cursor_current_node(cursor); + AssertThat(ts_tree_cursor_goto_next_sibling(&cursor), IsTrue()); + node = ts_tree_cursor_current_node(&cursor); AssertThat(ts_node_type(node), Equals("number")); AssertThat(ts_node_start_byte(node), Equals(number_index)); - AssertThat(ts_tree_cursor_goto_next_sibling(cursor), IsTrue()); - node = ts_tree_cursor_current_node(cursor); + AssertThat(ts_tree_cursor_goto_next_sibling(&cursor), IsTrue()); + node = ts_tree_cursor_current_node(&cursor); AssertThat(ts_node_type(node), Equals(",")); AssertThat(ts_node_start_byte(node), Equals(number_end_index)); - AssertThat(ts_tree_cursor_goto_next_sibling(cursor), IsTrue()); - node = ts_tree_cursor_current_node(cursor); + AssertThat(ts_tree_cursor_goto_next_sibling(&cursor), IsTrue()); + node = ts_tree_cursor_current_node(&cursor); AssertThat(ts_node_type(node), Equals("false")); AssertThat(ts_node_start_byte(node), Equals(false_index)); - AssertThat(ts_tree_cursor_goto_next_sibling(cursor), IsTrue()); - node = ts_tree_cursor_current_node(cursor); + AssertThat(ts_tree_cursor_goto_next_sibling(&cursor), IsTrue()); + node = ts_tree_cursor_current_node(&cursor); AssertThat(ts_node_type(node), Equals(",")); AssertThat(ts_node_start_byte(node), Equals(false_end_index)); - AssertThat(ts_tree_cursor_goto_next_sibling(cursor), IsTrue()); - node = ts_tree_cursor_current_node(cursor); + AssertThat(ts_tree_cursor_goto_next_sibling(&cursor), IsTrue()); + node = ts_tree_cursor_current_node(&cursor); AssertThat(ts_node_type(node), Equals("object")); AssertThat(ts_node_start_byte(node), Equals(object_index)); - AssertThat(ts_tree_cursor_goto_first_child(cursor), IsTrue()); - node = ts_tree_cursor_current_node(cursor); + AssertThat(ts_tree_cursor_goto_first_child(&cursor), IsTrue()); + node = ts_tree_cursor_current_node(&cursor); AssertThat(ts_node_type(node), Equals("{")); AssertThat(ts_node_start_byte(node), Equals(object_index)); - AssertThat(ts_tree_cursor_goto_next_sibling(cursor), IsTrue()); - node = ts_tree_cursor_current_node(cursor); + AssertThat(ts_tree_cursor_goto_next_sibling(&cursor), IsTrue()); + node = ts_tree_cursor_current_node(&cursor); AssertThat(ts_node_type(node), Equals("pair")); AssertThat(ts_node_start_byte(node), Equals(string_index)); - AssertThat(ts_tree_cursor_goto_first_child(cursor), IsTrue()); - node = ts_tree_cursor_current_node(cursor); + AssertThat(ts_tree_cursor_goto_first_child(&cursor), IsTrue()); + node = ts_tree_cursor_current_node(&cursor); AssertThat(ts_node_type(node), Equals("string")); AssertThat(ts_node_start_byte(node), Equals(string_index)); - AssertThat(ts_tree_cursor_goto_next_sibling(cursor), IsTrue()); - node = ts_tree_cursor_current_node(cursor); + AssertThat(ts_tree_cursor_goto_next_sibling(&cursor), IsTrue()); + node = ts_tree_cursor_current_node(&cursor); AssertThat(ts_node_type(node), Equals(":")); AssertThat(ts_node_start_byte(node), Equals(string_end_index)); - AssertThat(ts_tree_cursor_goto_next_sibling(cursor), IsTrue()); - node = ts_tree_cursor_current_node(cursor); + AssertThat(ts_tree_cursor_goto_next_sibling(&cursor), IsTrue()); + node = ts_tree_cursor_current_node(&cursor); AssertThat(ts_node_type(node), Equals("null")); AssertThat(ts_node_start_byte(node), Equals(null_index)); // Cannot move beyond a node with no next sibling - AssertThat(ts_tree_cursor_goto_next_sibling(cursor), IsFalse()); - node = ts_tree_cursor_current_node(cursor); + AssertThat(ts_tree_cursor_goto_next_sibling(&cursor), IsFalse()); + node = ts_tree_cursor_current_node(&cursor); AssertThat(ts_node_type(node), Equals("null")); AssertThat(ts_node_start_byte(node), Equals(null_index)); - AssertThat(ts_tree_cursor_goto_parent(cursor), IsTrue()); - node = ts_tree_cursor_current_node(cursor); + AssertThat(ts_tree_cursor_goto_parent(&cursor), IsTrue()); + node = ts_tree_cursor_current_node(&cursor); AssertThat(ts_node_type(node), Equals("pair")); AssertThat(ts_node_start_byte(node), Equals(string_index)); - AssertThat(ts_tree_cursor_goto_parent(cursor), IsTrue()); - node = ts_tree_cursor_current_node(cursor); + AssertThat(ts_tree_cursor_goto_parent(&cursor), IsTrue()); + node = ts_tree_cursor_current_node(&cursor); AssertThat(ts_node_type(node), Equals("object")); AssertThat(ts_node_start_byte(node), Equals(object_index)); - AssertThat(ts_tree_cursor_goto_parent(cursor), IsTrue()); - node = ts_tree_cursor_current_node(cursor); + AssertThat(ts_tree_cursor_goto_parent(&cursor), IsTrue()); + node = ts_tree_cursor_current_node(&cursor); AssertThat(ts_node_type(node), Equals("array")); AssertThat(ts_node_start_byte(node), Equals(array_index)); - AssertThat(ts_tree_cursor_goto_parent(cursor), IsTrue()); - node = ts_tree_cursor_current_node(cursor); + AssertThat(ts_tree_cursor_goto_parent(&cursor), IsTrue()); + node = ts_tree_cursor_current_node(&cursor); AssertThat(ts_node_type(node), Equals("value")); AssertThat(ts_node_start_byte(node), Equals(array_index)); // The root node doesn't have a parent. - AssertThat(ts_tree_cursor_goto_parent(cursor), IsFalse()); - node = ts_tree_cursor_current_node(cursor); + AssertThat(ts_tree_cursor_goto_parent(&cursor), IsFalse()); + node = ts_tree_cursor_current_node(&cursor); AssertThat(ts_node_type(node), Equals("value")); AssertThat(ts_node_start_byte(node), Equals(array_index)); }); it("can find the first child of a given node which spans the given byte offset", [&]() { - int64_t child_index = ts_tree_cursor_goto_first_child_for_byte(cursor, 1); - TSNode node = ts_tree_cursor_current_node(cursor); + int64_t child_index = ts_tree_cursor_goto_first_child_for_byte(&cursor, 1); + TSNode node = ts_tree_cursor_current_node(&cursor); AssertThat(ts_node_type(node), Equals("array")); AssertThat(ts_node_start_byte(node), Equals(array_index)); AssertThat(child_index, Equals(0)); - child_index = ts_tree_cursor_goto_first_child_for_byte(cursor, array_index); - node = ts_tree_cursor_current_node(cursor); + child_index = ts_tree_cursor_goto_first_child_for_byte(&cursor, array_index); + node = ts_tree_cursor_current_node(&cursor); AssertThat(ts_node_type(node), Equals("[")); AssertThat(ts_node_start_byte(node), Equals(array_index)); AssertThat(child_index, Equals(0)); - ts_tree_cursor_goto_parent(cursor); - child_index = ts_tree_cursor_goto_first_child_for_byte(cursor, array_index + 1); - node = ts_tree_cursor_current_node(cursor); + ts_tree_cursor_goto_parent(&cursor); + child_index = ts_tree_cursor_goto_first_child_for_byte(&cursor, array_index + 1); + node = ts_tree_cursor_current_node(&cursor); AssertThat(ts_node_type(node), Equals("number")); AssertThat(ts_node_start_byte(node), Equals(number_index)); AssertThat(child_index, Equals(1)); - ts_tree_cursor_goto_parent(cursor); - child_index = ts_tree_cursor_goto_first_child_for_byte(cursor, number_index + 1); - node = ts_tree_cursor_current_node(cursor); + ts_tree_cursor_goto_parent(&cursor); + child_index = ts_tree_cursor_goto_first_child_for_byte(&cursor, number_index + 1); + node = ts_tree_cursor_current_node(&cursor); AssertThat(ts_node_type(node), Equals("number")); AssertThat(ts_node_start_byte(node), Equals(number_index)); AssertThat(child_index, Equals(1)); - ts_tree_cursor_goto_parent(cursor); - child_index = ts_tree_cursor_goto_first_child_for_byte(cursor, false_index - 1); - node = ts_tree_cursor_current_node(cursor); + ts_tree_cursor_goto_parent(&cursor); + child_index = ts_tree_cursor_goto_first_child_for_byte(&cursor, false_index - 1); + node = ts_tree_cursor_current_node(&cursor); AssertThat(ts_node_type(node), Equals("false")); AssertThat(ts_node_start_byte(node), Equals(false_index)); AssertThat(child_index, Equals(3)); - ts_tree_cursor_goto_parent(cursor); - child_index = ts_tree_cursor_goto_first_child_for_byte(cursor, object_end_index - 1); - node = ts_tree_cursor_current_node(cursor); + ts_tree_cursor_goto_parent(&cursor); + child_index = ts_tree_cursor_goto_first_child_for_byte(&cursor, object_end_index - 1); + node = ts_tree_cursor_current_node(&cursor); AssertThat(ts_node_type(node), Equals("object")); AssertThat(ts_node_start_byte(node), Equals(object_index)); AssertThat(child_index, Equals(5)); // There is no child past the end of the array - ts_tree_cursor_goto_parent(cursor); - child_index = ts_tree_cursor_goto_first_child_for_byte(cursor, array_end_index); - node = ts_tree_cursor_current_node(cursor); + ts_tree_cursor_goto_parent(&cursor); + child_index = ts_tree_cursor_goto_first_child_for_byte(&cursor, array_end_index); + node = ts_tree_cursor_current_node(&cursor); AssertThat(ts_node_type(node), Equals("array")); AssertThat(ts_node_start_byte(node), Equals(array_index)); AssertThat(child_index, Equals(-1)); From 95be6e3bee4eaebf5825c8c187c960037cea66f4 Mon Sep 17 00:00:00 2001 From: Max Brunsfeld Date: Wed, 16 May 2018 16:20:33 -0700 Subject: [PATCH 58/90] Make it clear which field of TSNode can be used as a unique id --- include/tree_sitter/runtime.h | 6 ++++-- src/runtime/node.c | 19 ++++++++++--------- 2 files changed, 14 insertions(+), 11 deletions(-) diff --git a/include/tree_sitter/runtime.h b/include/tree_sitter/runtime.h index 71c8b5b3..ea52d4c2 100644 --- a/include/tree_sitter/runtime.h +++ b/include/tree_sitter/runtime.h @@ -66,12 +66,14 @@ typedef struct { typedef struct { uint32_t context[4]; - const void *ref[2]; + const void *id; + const void *tree; } TSNode; typedef struct { uint32_t context[2]; - void *ref[2]; + const void *id; + const void *tree; } TSTreeCursor; TSParser *ts_parser_new(); diff --git a/src/runtime/node.c b/src/runtime/node.c index 8eaaf220..31843e2a 100644 --- a/src/runtime/node.c +++ b/src/runtime/node.c @@ -17,7 +17,8 @@ typedef struct { TSNode ts_node_new(const TSTree *tree, const Subtree *subtree, Length position, TSSymbol alias) { return (TSNode) { {position.bytes, position.extent.row, position.extent.column, alias}, - {tree, subtree}, + subtree, + tree, }; } @@ -27,14 +28,6 @@ static inline TSNode ts_node__null() { // TSNode - accessors -static inline const TSTree *ts_node__tree(const TSNode *self) { - return self->ref[0]; -} - -static inline const Subtree *ts_node__subtree(TSNode self) { - return self.ref[1]; -} - static inline uint32_t ts_node__byte(const TSNode *self) { return self->context[0]; } @@ -47,6 +40,14 @@ static inline uint32_t ts_node__alias(const TSNode *self) { return self->context[3]; } +static inline const Subtree *ts_node__subtree(TSNode self) { + return self.id; +} + +static inline const TSTree *ts_node__tree(const TSNode *self) { + return self->tree; +} + // NodeChildIterator static inline NodeChildIterator ts_node_child_iterator_begin(const TSNode *node) { From 074c0510949f4afde3d65c5a0ae85f8a8c7a1b1c Mon Sep 17 00:00:00 2001 From: Max Brunsfeld Date: Thu, 17 May 2018 11:14:51 -0700 Subject: [PATCH 59/90] Change the TSInputEdit struct to work with old/new start and end positions --- include/tree_sitter/runtime.h | 8 ++--- src/runtime/subtree.c | 55 +++++++++++++++---------------- test/helpers/spy_input.cc | 25 +++++++++----- test/runtime/subtree_test.cc | 62 +++++++++++++++++++---------------- 4 files changed, 81 insertions(+), 69 deletions(-) diff --git a/include/tree_sitter/runtime.h b/include/tree_sitter/runtime.h index ea52d4c2..6a6767e9 100644 --- a/include/tree_sitter/runtime.h +++ b/include/tree_sitter/runtime.h @@ -57,11 +57,11 @@ typedef struct { typedef struct { uint32_t start_byte; - uint32_t bytes_removed; - uint32_t bytes_added; + uint32_t old_end_byte; + uint32_t new_end_byte; TSPoint start_point; - TSPoint extent_removed; - TSPoint extent_added; + TSPoint old_end_point; + TSPoint new_end_point; } TSInputEdit; typedef struct { diff --git a/src/runtime/subtree.c b/src/runtime/subtree.c index 0991b67c..9b2d954f 100644 --- a/src/runtime/subtree.c +++ b/src/runtime/subtree.c @@ -13,8 +13,8 @@ typedef struct { Length start; - Length added; - Length removed; + Length old_end; + Length new_end; } Edit; TSStateId TS_TREE_STATE_NONE = USHRT_MAX; @@ -490,22 +490,23 @@ const Subtree *ts_subtree_invalidate_lookahead(const Subtree *self, uint32_t edi } const Subtree *ts_subtree__edit(const Subtree *self, Edit edit, SubtreePool *pool) { - Length new_end = length_add(edit.start, edit.added); - Length old_end = length_add(edit.start, edit.removed); - Subtree *result = ts_subtree_make_mut(pool, self); result->has_changes = true; - if (old_end.bytes <= result->padding.bytes) { - result->padding = length_add(new_end, length_sub(result->padding, old_end)); + bool pure_insertion = edit.old_end.bytes == edit.start.bytes; + + if (edit.old_end.bytes <= result->padding.bytes) { + result->padding = length_add(edit.new_end, length_sub(result->padding, edit.old_end)); } else if (edit.start.bytes < result->padding.bytes) { - result->size = length_sub(result->size, length_sub(old_end, result->padding)); - result->padding = new_end; - } else if (edit.start.bytes == result->padding.bytes && edit.removed.bytes == 0) { - result->padding = length_add(result->padding, edit.added); + result->size = length_sub(result->size, length_sub(edit.old_end, result->padding)); + result->padding = edit.new_end; + } else if (edit.start.bytes == result->padding.bytes && pure_insertion) { + result->padding = edit.new_end; } else { - Length new_total_size = length_add(new_end, length_sub(ts_subtree_total_size(result), old_end)); - result->size = length_sub(new_total_size, result->padding); + result->size = length_add( + length_sub(edit.new_end, result->padding), + length_sub(result->size, length_sub(edit.old_end, result->padding)) + ); } Length child_left, child_right = length_zero(); @@ -515,27 +516,23 @@ const Subtree *ts_subtree__edit(const Subtree *self, Edit edit, SubtreePool *poo child_left = child_right; child_right = length_add(child_left, child_size); - if (child_left.bytes > old_end.bytes || - (child_left.bytes == old_end.bytes && child_size.bytes > 0 && i > 0)) break; + if (child_left.bytes > edit.old_end.bytes || + (child_left.bytes == edit.old_end.bytes && child_size.bytes > 0 && i > 0)) break; if (child_right.bytes > edit.start.bytes || - (child_right.bytes == edit.start.bytes && edit.removed.bytes == 0)) { + (child_right.bytes == edit.start.bytes && pure_insertion)) { Edit child_edit = { .start = length_sub(edit.start, child_left), - .added = edit.added, - .removed = edit.removed, + .old_end = length_sub(edit.old_end, child_left), + .new_end = length_sub(edit.new_end, child_left), }; - if (edit.start.bytes < child_left.bytes) { - child_edit.start = length_zero(); - } + if (edit.start.bytes < child_left.bytes) child_edit.start = length_zero(); + if (edit.old_end.bytes < child_left.bytes) child_edit.old_end = length_zero(); + if (edit.new_end.bytes < child_left.bytes) child_edit.new_end = length_zero(); + if (edit.old_end.bytes > child_right.bytes) child_edit.old_end = child_size; - if (old_end.bytes > child_right.bytes) { - child_edit.removed = length_sub(child_size, child_edit.start); - } - - edit.added = length_zero(); - edit.removed = length_sub(edit.removed, child_edit.removed); + edit.new_end = edit.start; *child = ts_subtree__edit(*child, child_edit, pool); } else if (child_left.bytes <= edit.start.bytes) { @@ -549,8 +546,8 @@ const Subtree *ts_subtree__edit(const Subtree *self, Edit edit, SubtreePool *poo const Subtree *ts_subtree_edit(const Subtree *self, const TSInputEdit *edit, SubtreePool *pool) { return ts_subtree__edit(self, (Edit) { .start = {edit->start_byte, edit->start_point}, - .added = {edit->bytes_added, edit->extent_added}, - .removed = {edit->bytes_removed, edit->extent_removed}, + .old_end = {edit->old_end_byte, edit->old_end_point}, + .new_end = {edit->new_end_byte, edit->new_end_point}, }, pool); } diff --git a/test/helpers/spy_input.cc b/test/helpers/spy_input.cc index 889dcdf0..6126f101 100644 --- a/test/helpers/spy_input.cc +++ b/test/helpers/spy_input.cc @@ -1,5 +1,6 @@ #include "helpers/spy_input.h" #include "helpers/encoding_helpers.h" +#include "runtime/point.h" #include #include #include @@ -20,6 +21,14 @@ SpyInput::~SpyInput() { delete[] buffer; } +static TSPoint operator+(TSPoint a, TSPoint b) { + if (b.row > 0) { + return TSPoint {a.row + b.row, b.column}; + } else { + return TSPoint {a.row, a.column + b.column}; + } +} + static void add_byte_range(vector> *ranges, uint32_t start, uint32_t count) { uint32_t end = start + count; @@ -112,11 +121,11 @@ TSInputEdit SpyInput::replace(size_t start_byte, size_t bytes_removed, string te undo_stack.push_back(SpyInputEdit{start_byte, bytes_added, swap.first}); TSInputEdit result = {}; result.start_byte = start_byte; - result.bytes_added = bytes_added; - result.bytes_removed = bytes_removed; + result.old_end_byte = start_byte + bytes_removed; + result.new_end_byte = start_byte + bytes_added; result.start_point = swap.second; - result.extent_removed = get_extent(swap.first); - result.extent_added = get_extent(text); + result.old_end_point = result.start_point + get_extent(swap.first); + result.new_end_point = result.start_point + get_extent(text); return result; } @@ -126,11 +135,11 @@ TSInputEdit SpyInput::undo() { auto swap = swap_substr(entry.start_byte, entry.bytes_removed, entry.text_inserted); TSInputEdit result; result.start_byte = entry.start_byte; - result.bytes_removed = entry.bytes_removed; - result.bytes_added = entry.text_inserted.size(); + result.old_end_byte = entry.start_byte + entry.bytes_removed; + result.new_end_byte = entry.start_byte + entry.text_inserted.size(); result.start_point = swap.second; - result.extent_removed = get_extent(swap.first); - result.extent_added = get_extent(entry.text_inserted); + result.old_end_point = result.start_point + get_extent(swap.first); + result.new_end_point = result.start_point + get_extent(entry.text_inserted); return result; } diff --git a/test/runtime/subtree_test.cc b/test/runtime/subtree_test.cc index 3c1c9ad2..246f985e 100644 --- a/test/runtime/subtree_test.cc +++ b/test/runtime/subtree_test.cc @@ -202,11 +202,11 @@ describe("Subtree", []() { it("does not mutate the argument", [&]() { TSInputEdit edit; edit.start_byte = 1; - edit.bytes_removed = 0; - edit.bytes_added = 1; + edit.old_end_byte = 1; + edit.new_end_byte = 2; edit.start_point = {0, 1}; - edit.extent_removed = {0, 0}; - edit.extent_added = {0, 1}; + edit.old_end_point = {0, 1}; + edit.new_end_point = {0, 2}; ts_subtree_retain(tree); const Subtree *new_tree = ts_subtree_edit(tree, &edit, &pool); @@ -232,11 +232,12 @@ describe("Subtree", []() { it("resizes the padding of the tree and its leftmost descendants", [&]() { TSInputEdit edit; edit.start_byte = 1; - edit.bytes_removed = 0; - edit.bytes_added = 1; + edit.old_end_byte = 1; + edit.new_end_byte = 2; edit.start_point = {0, 1}; - edit.extent_removed = {0, 0}; - edit.extent_added = {0, 1}; + edit.old_end_point = {0, 1}; + edit.new_end_point = {0, 2}; + tree = ts_subtree_edit(tree, &edit, &pool); assert_consistent(tree); @@ -258,11 +259,12 @@ describe("Subtree", []() { it("shrinks the content to compensate for the expanded padding", [&]() { TSInputEdit edit; edit.start_byte = 1; - edit.bytes_removed = 3; - edit.bytes_added = 4; + edit.old_end_byte = 4; + edit.new_end_byte = 5; edit.start_point = {0, 1}; - edit.extent_removed = {0, 3}; - edit.extent_added = {0, 4}; + edit.old_end_point = {0, 4}; + edit.new_end_point = {0, 5}; + tree = ts_subtree_edit(tree, &edit, &pool); assert_consistent(tree); @@ -280,11 +282,12 @@ describe("Subtree", []() { it("expands the tree's padding", [&]() { TSInputEdit edit; edit.start_byte = 2; - edit.bytes_removed = 0; - edit.bytes_added = 2; + edit.old_end_byte = 2; + edit.new_end_byte = 4; edit.start_point = {0, 2}; - edit.extent_removed = {0, 0}; - edit.extent_added = {0, 2}; + edit.old_end_point = {0, 2}; + edit.new_end_point = {0, 4}; + tree = ts_subtree_edit(tree, &edit, &pool); assert_consistent(tree); @@ -304,11 +307,12 @@ describe("Subtree", []() { it("resizes the content and not the padding", [&]() { TSInputEdit edit; edit.start_byte = 2; - edit.bytes_removed = 2; - edit.bytes_added = 5; + edit.old_end_byte = 4; + edit.new_end_byte = 7; edit.start_point = {0, 2}; - edit.extent_removed = {0, 2}; - edit.extent_added = {0, 5}; + edit.old_end_point = {0, 4}; + edit.new_end_point = {0, 7}; + tree = ts_subtree_edit(tree, &edit, &pool); assert_consistent(tree); @@ -328,11 +332,12 @@ describe("Subtree", []() { it("shrinks subsequent child nodes", [&]() { TSInputEdit edit; edit.start_byte = 1; - edit.bytes_removed = 10; - edit.bytes_added = 3; + edit.old_end_byte = 11; + edit.new_end_byte = 4; edit.start_point = {0, 1}; - edit.extent_removed = {0, 10}; - edit.extent_added = {0, 3}; + edit.old_end_point = {0, 11}; + edit.new_end_point = {0, 4}; + tree = ts_subtree_edit(tree, &edit, &pool); assert_consistent(tree); @@ -361,11 +366,12 @@ describe("Subtree", []() { TSInputEdit edit; edit.start_byte = 6; - edit.bytes_removed = 1; - edit.bytes_added = 1; + edit.old_end_byte = 7; + edit.new_end_byte = 7; edit.start_point = {0, 6}; - edit.extent_removed = {0, 1}; - edit.extent_added = {0, 1}; + edit.old_end_point = {0, 7}; + edit.new_end_point = {0, 7}; + tree = ts_subtree_edit(tree, &edit, &pool); assert_consistent(tree); From 5ec3769cb4c9acfda64f80d7c14abce939e8b4c5 Mon Sep 17 00:00:00 2001 From: Max Brunsfeld Date: Thu, 17 May 2018 14:24:32 -0700 Subject: [PATCH 60/90] Make ts_tree_cursor_current_node take the cursor as const --- include/tree_sitter/runtime.h | 2 +- src/runtime/tree_cursor.c | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/include/tree_sitter/runtime.h b/include/tree_sitter/runtime.h index 6a6767e9..e0badebd 100644 --- a/include/tree_sitter/runtime.h +++ b/include/tree_sitter/runtime.h @@ -129,7 +129,7 @@ bool ts_tree_cursor_goto_first_child(TSTreeCursor *); int64_t ts_tree_cursor_goto_first_child_for_byte(TSTreeCursor *, uint32_t); bool ts_tree_cursor_goto_next_sibling(TSTreeCursor *); bool ts_tree_cursor_goto_parent(TSTreeCursor *); -TSNode ts_tree_cursor_current_node(TSTreeCursor *); +TSNode ts_tree_cursor_current_node(const TSTreeCursor *); uint32_t ts_language_symbol_count(const TSLanguage *); const char *ts_language_symbol_name(const TSLanguage *, TSSymbol); diff --git a/src/runtime/tree_cursor.c b/src/runtime/tree_cursor.c index d7695634..092e31f9 100644 --- a/src/runtime/tree_cursor.c +++ b/src/runtime/tree_cursor.c @@ -170,8 +170,8 @@ bool ts_tree_cursor_goto_parent(TSTreeCursor *_self) { return false; } -TSNode ts_tree_cursor_current_node(TSTreeCursor *_self) { - TreeCursor *self = (TreeCursor *)_self; +TSNode ts_tree_cursor_current_node(const TSTreeCursor *_self) { + const TreeCursor *self = (const TreeCursor *)_self; TreeCursorEntry *last_entry = array_back(&self->stack); TSSymbol alias_symbol = 0; if (self->stack.size > 1) { From 3c01382b95364ce40f0cf9856865a30af77f9690 Mon Sep 17 00:00:00 2001 From: Max Brunsfeld Date: Thu, 17 May 2018 17:59:50 -0700 Subject: [PATCH 61/90] Avoid warnings about repeated typedefs --- include/tree_sitter/parser.h | 7 +++++-- include/tree_sitter/runtime.h | 2 +- src/runtime/language.h | 2 +- src/runtime/lexer.h | 4 ++-- src/runtime/subtree.h | 6 +++--- 5 files changed, 12 insertions(+), 9 deletions(-) diff --git a/include/tree_sitter/parser.h b/include/tree_sitter/parser.h index 35e77a87..1b6c2b13 100644 --- a/include/tree_sitter/parser.h +++ b/include/tree_sitter/parser.h @@ -13,7 +13,10 @@ extern "C" { #define ts_builtin_sym_end 0 #define TREE_SITTER_SERIALIZATION_BUFFER_SIZE 1024 +#ifndef TREE_SITTER_RUNTIME_H_ typedef uint16_t TSSymbol; +typedef struct TSLanguage TSLanguage; +#endif typedef uint16_t TSStateId; @@ -67,7 +70,7 @@ typedef union { }; } TSParseActionEntry; -typedef struct TSLanguage { +struct TSLanguage { uint32_t version; uint32_t symbol_count; uint32_t alias_count; @@ -92,7 +95,7 @@ typedef struct TSLanguage { unsigned (*serialize)(void *, char *); void (*deserialize)(void *, const char *, unsigned); } external_scanner; -} TSLanguage; +}; /* * Lexer Macros diff --git a/include/tree_sitter/runtime.h b/include/tree_sitter/runtime.h index e0badebd..b483ba7f 100644 --- a/include/tree_sitter/runtime.h +++ b/include/tree_sitter/runtime.h @@ -12,7 +12,7 @@ extern "C" { #define TREE_SITTER_LANGUAGE_VERSION 8 -typedef unsigned short TSSymbol; +typedef uint16_t TSSymbol; typedef struct TSLanguage TSLanguage; typedef struct TSParser TSParser; typedef struct TSTree TSTree; diff --git a/src/runtime/language.h b/src/runtime/language.h index 879a5b5b..8386a054 100644 --- a/src/runtime/language.h +++ b/src/runtime/language.h @@ -5,8 +5,8 @@ extern "C" { #endif -#include "tree_sitter/parser.h" #include "runtime/subtree.h" +#include "tree_sitter/parser.h" #define ts_builtin_sym_error_repeat (ts_builtin_sym_error - 1) diff --git a/src/runtime/lexer.h b/src/runtime/lexer.h index 90be55a7..d6cf6279 100644 --- a/src/runtime/lexer.h +++ b/src/runtime/lexer.h @@ -5,10 +5,10 @@ extern "C" { #endif -#include "tree_sitter/parser.h" -#include "tree_sitter/runtime.h" #include "runtime/length.h" #include "runtime/subtree.h" +#include "tree_sitter/runtime.h" +#include "tree_sitter/parser.h" typedef struct { TSLexer data; diff --git a/src/runtime/subtree.h b/src/runtime/subtree.h index cf59db43..ec8e3032 100644 --- a/src/runtime/subtree.h +++ b/src/runtime/subtree.h @@ -6,11 +6,11 @@ extern "C" { #endif #include -#include "tree_sitter/parser.h" -#include "tree_sitter/runtime.h" +#include #include "runtime/length.h" #include "runtime/array.h" -#include +#include "tree_sitter/runtime.h" +#include "tree_sitter/parser.h" extern TSStateId TS_TREE_STATE_NONE; From 9c1e82a7eac97767cee0469faa2722fd5753b065 Mon Sep 17 00:00:00 2001 From: Max Brunsfeld Date: Fri, 18 May 2018 10:25:55 -0700 Subject: [PATCH 62/90] :arrow_up: utf8proc, use new UTF8PROC_STATIC macro --- externals/utf8proc | 2 +- project.gyp | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/externals/utf8proc b/externals/utf8proc index 40e60595..d81308fa 160000 --- a/externals/utf8proc +++ b/externals/utf8proc @@ -1 +1 @@ -Subproject commit 40e605959eb5cb90b2587fa88e3b661558fbc55a +Subproject commit d81308faba0cfb3fccf8c3b12446863c7b76ae32 diff --git a/project.gyp b/project.gyp index 11174724..b88a6c65 100644 --- a/project.gyp +++ b/project.gyp @@ -151,7 +151,7 @@ '-Wno-unused-parameter' ], - 'defines': ['UTF8PROC_EXPORTS'], + 'defines': ['UTF8PROC_STATIC'], 'xcode_settings': { 'ALWAYS_SEARCH_USER_PATHS': 'NO', From 78f28b14ce519ba085ab7886c2fc19739f7f7da0 Mon Sep 17 00:00:00 2001 From: Max Brunsfeld Date: Fri, 18 May 2018 14:27:52 -0700 Subject: [PATCH 63/90] Remove unused field --- src/runtime/subtree.h | 1 - test/runtime/subtree_test.cc | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/src/runtime/subtree.h b/src/runtime/subtree.h index ec8e3032..0fd3311a 100644 --- a/src/runtime/subtree.h +++ b/src/runtime/subtree.h @@ -35,7 +35,6 @@ struct Subtree { uint32_t error_cost; uint32_t node_count; uint32_t repeat_depth; - uint32_t child_count; int32_t dynamic_precedence; bool visible : 1; diff --git a/test/runtime/subtree_test.cc b/test/runtime/subtree_test.cc index 246f985e..1ff7356f 100644 --- a/test/runtime/subtree_test.cc +++ b/test/runtime/subtree_test.cc @@ -5,7 +5,7 @@ #include "runtime/length.h" void assert_consistent(const Subtree *tree) { - if (tree->child_count == 0) return; + if (tree->children.size == 0) return; AssertThat(tree->children.contents[0]->padding, Equals(tree->padding)); Length total_children_size = length_zero(); From 95fbc23fd66d043ba7a2d72901b6a8794411d586 Mon Sep 17 00:00:00 2001 From: Max Brunsfeld Date: Mon, 21 May 2018 14:09:09 -0700 Subject: [PATCH 64/90] Add support for running focused tests via windows test script --- script/test.cmd | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/script/test.cmd b/script/test.cmd index 31907ab6..f2d97303 100644 --- a/script/test.cmd +++ b/script/test.cmd @@ -1,2 +1,9 @@ +@echo off msbuild /p:Configuration=Test tests.vcxproj -.\test\tests.exe --reporter=singleline --no-color \ No newline at end of file + +set only_arg= +IF not "%~1"=="" ( + set only_arg=--only=%1 +) + +.\test\tests.exe --reporter=singleline --no-color %only_arg% From 39c3de3fc8f1baf8a5504559d98e55c4dabb32d2 Mon Sep 17 00:00:00 2001 From: Max Brunsfeld Date: Fri, 18 May 2018 18:04:42 -0700 Subject: [PATCH 65/90] Don't reuse nodes within ambiguities --- src/runtime/parser.c | 176 ++++++++++++++++++------------------ src/runtime/reusable_node.h | 5 +- 2 files changed, 91 insertions(+), 90 deletions(-) diff --git a/src/runtime/parser.c b/src/runtime/parser.c index dde46f80..3ec5b969 100644 --- a/src/runtime/parser.c +++ b/src/runtime/parser.c @@ -57,7 +57,6 @@ struct TSParser { TokenCache token_cache; ReusableNode reusable_node; void *external_scanner_payload; - bool in_ambiguity; FILE *dot_graph_file; bool halt_on_error; unsigned accept_count; @@ -254,6 +253,24 @@ static void ts_parser__restore_external_scanner(TSParser *self, const Subtree *e } } +static bool ts_parser__can_reuse_first_leaf(TSParser *self, TSStateId state, const Subtree *tree, + TableEntry *table_entry) { + TSLexMode current_lex_mode = self->language->lex_modes[state]; + + // If the token was created in a state with the same set of lookaheads, it is reusable. + if (tree->first_leaf.lex_mode.lex_state == current_lex_mode.lex_state && + tree->first_leaf.lex_mode.external_lex_state == current_lex_mode.external_lex_state && + (tree->first_leaf.symbol != self->language->keyword_capture_token || + tree->parse_state == state)) return true; + + // Empty tokens are not reusable in states with different lookaheads. + if (tree->size.bytes == 0 && tree->symbol != ts_builtin_sym_end) return false; + + // If the current state allows external tokens or other tokens that conflict with this + // token, this token is not reusable. + return current_lex_mode.external_lex_state == 0 && table_entry->is_reusable; +} + static const Subtree *ts_parser__lex(TSParser *self, StackVersion version, TSStateId parse_state) { Length start_position = ts_stack_position(self->stack, version); const Subtree *external_token = ts_stack_last_external_token(self->stack, version); @@ -402,20 +419,27 @@ static const Subtree *ts_parser__lex(TSParser *self, StackVersion version, TSSta return result; } -static const Subtree *ts_parser__get_cached_token(TSParser *self, size_t byte_index, - const Subtree *last_external_token) { +static const Subtree *ts_parser__get_cached_token(TSParser *self, TSStateId state, + size_t position, + const Subtree *last_external_token, + TableEntry *table_entry) { TokenCache *cache = &self->token_cache; - if (cache->token && - cache->byte_index == byte_index && - ts_subtree_external_scanner_state_eq(cache->last_external_token, last_external_token)) { - return cache->token; - } else { - return NULL; + if ( + cache->token && cache->byte_index == position && + ts_subtree_external_scanner_state_eq(cache->last_external_token, last_external_token) + ) { + ts_language_table_entry(self->language, state, cache->token->first_leaf.symbol, table_entry); + if (ts_parser__can_reuse_first_leaf(self, state, cache->token, table_entry)) { + ts_subtree_retain(cache->token); + return cache->token; + } } + return NULL; } static void ts_parser__set_cached_token(TSParser *self, size_t byte_index, - const Subtree *last_external_token, const Subtree *token) { + const Subtree *last_external_token, + const Subtree *token) { TokenCache *cache = &self->token_cache; if (token) ts_subtree_retain(token); if (last_external_token) ts_subtree_retain(last_external_token); @@ -426,47 +450,27 @@ static void ts_parser__set_cached_token(TSParser *self, size_t byte_index, cache->last_external_token = last_external_token; } -static bool ts_parser__can_reuse_first_leaf(TSParser *self, TSStateId state, const Subtree *tree, +static const Subtree *ts_parser__reuse_node(TSParser *self, StackVersion version, + TSStateId *state, uint32_t position, + const Subtree *last_external_token, TableEntry *table_entry) { - TSLexMode current_lex_mode = self->language->lex_modes[state]; - - // If the token was created in a state with the same set of lookaheads, it is reusable. - if (tree->first_leaf.lex_mode.lex_state == current_lex_mode.lex_state && - tree->first_leaf.lex_mode.external_lex_state == current_lex_mode.external_lex_state && - (tree->first_leaf.symbol != self->language->keyword_capture_token || - tree->parse_state == state)) return true; - - // Empty tokens are not reusable in states with different lookaheads. - if (tree->size.bytes == 0 && tree->symbol != ts_builtin_sym_end) return false; - - // If the current state allows external tokens or other tokens that conflict with this - // token, this token is not reusable. - return current_lex_mode.external_lex_state == 0 && table_entry->is_reusable; -} - -static const Subtree *ts_parser__get_lookahead(TSParser *self, StackVersion version, - TSStateId *state, ReusableNode *reusable_node, - TableEntry *table_entry) { - Length position = ts_stack_position(self->stack, version); - const Subtree *last_external_token = ts_stack_last_external_token(self->stack, version); - const Subtree *result; - while ((result = reusable_node_tree(reusable_node))) { - uint32_t byte_offset = reusable_node_byte_offset(reusable_node); - if (byte_offset > position.bytes) { + while ((result = reusable_node_tree(&self->reusable_node))) { + uint32_t byte_offset = reusable_node_byte_offset(&self->reusable_node); + if (byte_offset > position) { LOG("before_reusable_node symbol:%s", SYM_NAME(result->symbol)); break; } - if (byte_offset < position.bytes) { + if (byte_offset < position) { LOG("past_reusable_node symbol:%s", SYM_NAME(result->symbol)); - reusable_node_advance(reusable_node); + reusable_node_advance(&self->reusable_node); continue; } - if (!ts_subtree_external_scanner_state_eq(reusable_node->last_external_token, last_external_token)) { + if (!ts_subtree_external_scanner_state_eq(self->reusable_node.last_external_token, last_external_token)) { LOG("reusable_node_has_different_external_scanner_state symbol:%s", SYM_NAME(result->symbol)); - reusable_node_advance(reusable_node); + reusable_node_advance(&self->reusable_node); continue; } @@ -479,14 +483,12 @@ static const Subtree *ts_parser__get_lookahead(TSParser *self, StackVersion vers reason = "is_missing"; } else if (result->fragile_left || result->fragile_right) { reason = "is_fragile"; - } else if (self->in_ambiguity && result->children.size) { - reason = "in_ambiguity"; } if (reason) { LOG("cant_reuse_node_%s tree:%s", reason, SYM_NAME(result->symbol)); - if (!reusable_node_descend(reusable_node)) { - reusable_node_advance(reusable_node); + if (!reusable_node_descend(&self->reusable_node)) { + reusable_node_advance(&self->reusable_node); ts_parser__breakdown_top_of_stack(self, version); *state = ts_stack_state(self->stack, version); } @@ -500,7 +502,7 @@ static const Subtree *ts_parser__get_lookahead(TSParser *self, StackVersion vers SYM_NAME(result->symbol), SYM_NAME(result->first_leaf.symbol) ); - reusable_node_advance_past_leaf(reusable_node); + reusable_node_advance_past_leaf(&self->reusable_node); break; } @@ -509,18 +511,7 @@ static const Subtree *ts_parser__get_lookahead(TSParser *self, StackVersion vers return result; } - if ((result = ts_parser__get_cached_token(self, position.bytes, last_external_token))) { - ts_language_table_entry(self->language, *state, result->first_leaf.symbol, table_entry); - if (ts_parser__can_reuse_first_leaf(self, *state, result, table_entry)) { - ts_subtree_retain(result); - return result; - } - } - - result = ts_parser__lex(self, version, *state); - ts_parser__set_cached_token(self, position.bytes, last_external_token, result); - ts_language_table_entry(self->language, *state, result->symbol, table_entry); - return result; + return NULL; } static bool ts_parser__select_tree(TSParser *self, const Subtree *left, const Subtree *right) { @@ -654,7 +645,7 @@ static StackSliceArray ts_parser__reduce(TSParser *self, StackVersion version, T TSStateId state = ts_stack_state(self->stack, slice.version); TSStateId next_state = ts_language_next_state(self->language, state, symbol); - if (fragile || self->in_ambiguity || pop.size > 1 || initial_version_count > 1) { + if (fragile || pop.size > 1 || initial_version_count > 1) { parent->fragile_left = true; parent->fragile_right = true; parent->parse_state = TS_TREE_STATE_NONE; @@ -712,7 +703,6 @@ static void ts_parser__start(TSParser *self, TSInput input, const Subtree *previ reusable_node_reset(&self->reusable_node, previous_tree); self->finished_tree = NULL; self->accept_count = 0; - self->in_ambiguity = false; } static void ts_parser__accept(TSParser *self, StackVersion version, const Subtree *lookahead) { @@ -1107,12 +1097,36 @@ static void ts_parser__recover(TSParser *self, StackVersion version, const Subtr } } -static void ts_parser__advance(TSParser *self, StackVersion version, ReusableNode *reusable_node) { +static void ts_parser__advance(TSParser *self, StackVersion version, bool allow_node_reuse) { TSStateId state = ts_stack_state(self->stack, version); + uint32_t position = ts_stack_position(self->stack, version).bytes; + const Subtree *last_external_token = ts_stack_last_external_token(self->stack, version); + + bool did_reuse = true; + const Subtree *lookahead = NULL; TableEntry table_entry; - const Subtree *lookahead = ts_parser__get_lookahead( - self, version, &state, reusable_node, &table_entry - ); + + // If possible, reuse a node from the previous syntax tree. + if (allow_node_reuse) { + lookahead = ts_parser__reuse_node( + self, version, &state, position, last_external_token, &table_entry + ); + } + + // Otherwise, try to reuse the token previously returned by the lexer. + if (!lookahead) { + did_reuse = false; + lookahead = ts_parser__get_cached_token( + self, state, position, last_external_token, &table_entry + ); + } + + // Otherwise, re-run the lexer. + if (!lookahead) { + lookahead = ts_parser__lex(self, version, state); + ts_parser__set_cached_token(self, position, last_external_token, lookahead); + ts_language_table_entry(self->language, state, lookahead->symbol, &table_entry); + } for (;;) { StackVersion last_reduction_version = STACK_VERSION_NONE; @@ -1137,14 +1151,12 @@ static void ts_parser__advance(TSParser *self, StackVersion version, ReusableNod } if (lookahead->children.size > 0) { - ts_parser__breakdown_lookahead(self, &lookahead, state, reusable_node); + ts_parser__breakdown_lookahead(self, &lookahead, state, &self->reusable_node); next_state = ts_language_next_state(self->language, state, lookahead->symbol); } ts_parser__shift(self, version, next_state, lookahead, action.params.extra); - if (lookahead == reusable_node_tree(reusable_node)) { - reusable_node_advance(reusable_node); - } + if (did_reuse) reusable_node_advance(&self->reusable_node); return; } @@ -1168,13 +1180,12 @@ static void ts_parser__advance(TSParser *self, StackVersion version, ReusableNod } case TSParseActionTypeRecover: { - while (lookahead->children.size > 0) { - ts_parser__breakdown_lookahead(self, &lookahead, state, reusable_node); + if (lookahead->children.size > 0) { + ts_parser__breakdown_lookahead(self, &lookahead, ERROR_STATE, &self->reusable_node); } + ts_parser__recover(self, version, lookahead); - if (lookahead == reusable_node_tree(reusable_node)) { - reusable_node_advance(reusable_node); - } + if (did_reuse) reusable_node_advance(&self->reusable_node); return; } } @@ -1355,15 +1366,13 @@ TSTree *ts_parser_parse(TSParser *self, const TSTree *old_tree, TSInput input) { if (!self->language) return NULL; ts_parser__start(self, input, old_tree ? old_tree->root : NULL); - StackVersion version = STACK_VERSION_NONE; - uint32_t position = 0, last_position = 0; - ReusableNode reusable_node = reusable_node_new(); - reusable_node_assign(&reusable_node, &self->reusable_node); + uint32_t position = 0, last_position = 0, version_count = 0; do { - for (version = 0; version < ts_stack_version_count(self->stack); version++) { - reusable_node_assign(&reusable_node, &self->reusable_node); - + for (StackVersion version = 0; + version_count = ts_stack_version_count(self->stack), version < version_count; + version++) { + bool allow_node_reuse = version_count == 1; while (ts_stack_is_active(self->stack, version)) { LOG("process version:%d, version_count:%u, state:%d, row:%u, col:%u", version, ts_stack_version_count(self->stack), @@ -1371,7 +1380,7 @@ TSTree *ts_parser_parse(TSParser *self, const TSTree *old_tree, TSInput input) { ts_stack_position(self->stack, version).extent.row, ts_stack_position(self->stack, version).extent.column); - ts_parser__advance(self, version, &reusable_node); + ts_parser__advance(self, version, allow_node_reuse); LOG_STACK(); position = ts_stack_position(self->stack, version).bytes; @@ -1382,8 +1391,6 @@ TSTree *ts_parser_parse(TSParser *self, const TSTree *old_tree, TSInput input) { } } - reusable_node_assign(&self->reusable_node, &reusable_node); - unsigned min_error_cost = ts_parser__condense_stack(self); if (self->finished_tree && self->finished_tree->error_cost < min_error_cost) { break; @@ -1391,11 +1398,8 @@ TSTree *ts_parser_parse(TSParser *self, const TSTree *old_tree, TSInput input) { ts_parser__halt_parse(self); break; } + } while (version_count != 0); - self->in_ambiguity = version > 1; - } while (version != 0); - - reusable_node_delete(&reusable_node); ts_stack_clear(self->stack); ts_parser__set_cached_token(self, 0, NULL, NULL); ts_subtree_balance(self->finished_tree, &self->tree_pool, self->language); diff --git a/src/runtime/reusable_node.h b/src/runtime/reusable_node.h index 42ae6f1e..5575012b 100644 --- a/src/runtime/reusable_node.h +++ b/src/runtime/reusable_node.h @@ -22,6 +22,7 @@ static inline void reusable_node_reset(ReusableNode *self, const Subtree *tree) .child_index = 0, .byte_offset = 0, })); + self->last_external_token = NULL; } static inline const Subtree *reusable_node_tree(ReusableNode *self) { @@ -40,10 +41,6 @@ static inline void reusable_node_delete(ReusableNode *self) { array_delete(&self->stack); } -static inline void reusable_node_assign(ReusableNode *self, const ReusableNode *other) { - array_assign(&self->stack, &other->stack); -} - static inline void reusable_node_advance(ReusableNode *self) { StackEntry last_entry = *array_back(&self->stack); uint32_t byte_offset = last_entry.byte_offset + ts_subtree_total_bytes(last_entry.tree); From 1fece241aa6a8e7c07e8f4a1c2b80bfdac0a0c60 Mon Sep 17 00:00:00 2001 From: Max Brunsfeld Date: Wed, 16 May 2018 17:42:38 -0700 Subject: [PATCH 66/90] Add ts_parser_set_enabled API --- include/tree_sitter/runtime.h | 2 ++ src/runtime/parser.c | 29 +++++++++++++++++++++- test/runtime/parser_test.cc | 46 +++++++++++++++++++++++++++++++++++ 3 files changed, 76 insertions(+), 1 deletion(-) diff --git a/include/tree_sitter/runtime.h b/include/tree_sitter/runtime.h index b483ba7f..e21571e8 100644 --- a/include/tree_sitter/runtime.h +++ b/include/tree_sitter/runtime.h @@ -86,6 +86,8 @@ void ts_parser_print_dot_graphs(TSParser *, FILE *); void ts_parser_halt_on_error(TSParser *, bool); TSTree *ts_parser_parse(TSParser *, const TSTree *, TSInput); TSTree *ts_parser_parse_string(TSParser *, const TSTree *, const char *, uint32_t); +bool ts_parser_enabled(TSParser *); +void ts_parser_set_enabled(TSParser *, bool); TSTree *ts_tree_copy(const TSTree *); void ts_tree_delete(TSTree *); diff --git a/src/runtime/parser.c b/src/runtime/parser.c index 3ec5b969..26a377d7 100644 --- a/src/runtime/parser.c +++ b/src/runtime/parser.c @@ -60,6 +60,7 @@ struct TSParser { FILE *dot_graph_file; bool halt_on_error; unsigned accept_count; + volatile bool enabled; }; typedef struct { @@ -705,6 +706,15 @@ static void ts_parser__start(TSParser *self, TSInput input, const Subtree *previ self->accept_count = 0; } +static void ts_parser__stop(TSParser *self) { + ts_stack_clear(self->stack); + ts_parser__set_cached_token(self, 0, NULL, NULL); + if (self->finished_tree) { + ts_subtree_release(&self->tree_pool, self->finished_tree); + self->finished_tree = NULL; + } +} + static void ts_parser__accept(TSParser *self, StackVersion version, const Subtree *lookahead) { assert(lookahead->symbol == ts_builtin_sym_end); ts_stack_push(self->stack, version, lookahead, false, 1); @@ -1308,6 +1318,7 @@ TSParser *ts_parser_new() { self->reusable_node = reusable_node_new(); self->dot_graph_file = NULL; self->halt_on_error = false; + self->enabled = true; ts_parser__set_cached_token(self, 0, NULL, NULL); return self; } @@ -1362,6 +1373,14 @@ void ts_parser_halt_on_error(TSParser *self, bool should_halt_on_error) { self->halt_on_error = should_halt_on_error; } +bool ts_parser_enabled(TSParser *self) { + return self->enabled; +} + +void ts_parser_set_enabled(TSParser *self, bool enabled) { + self->enabled = enabled; +} + TSTree *ts_parser_parse(TSParser *self, const TSTree *old_tree, TSInput input) { if (!self->language) return NULL; ts_parser__start(self, input, old_tree ? old_tree->root : NULL); @@ -1372,6 +1391,11 @@ TSTree *ts_parser_parse(TSParser *self, const TSTree *old_tree, TSInput input) { for (StackVersion version = 0; version_count = ts_stack_version_count(self->stack), version < version_count; version++) { + if (!self->enabled) { + ts_parser__stop(self); + return NULL; + } + bool allow_node_reuse = version_count == 1; while (ts_stack_is_active(self->stack, version)) { LOG("process version:%d, version_count:%u, state:%d, row:%u, col:%u", @@ -1404,10 +1428,13 @@ TSTree *ts_parser_parse(TSParser *self, const TSTree *old_tree, TSInput input) { ts_parser__set_cached_token(self, 0, NULL, NULL); ts_subtree_balance(self->finished_tree, &self->tree_pool, self->language); + TSTree *result = ts_tree_new(self->finished_tree, self->language); LOG("done"); LOG_TREE(); + self->finished_tree = NULL; - return ts_tree_new(self->finished_tree, self->language); + ts_parser__stop(self); + return result; } TSTree *ts_parser_parse_string(TSParser *self, const TSTree *old_tree, diff --git a/test/runtime/parser_test.cc b/test/runtime/parser_test.cc index 7d0b2d1d..4e787c76 100644 --- a/test/runtime/parser_test.cc +++ b/test/runtime/parser_test.cc @@ -1,4 +1,5 @@ #include "test_helper.h" +#include #include "runtime/alloc.h" #include "runtime/language.h" #include "helpers/record_alloc.h" @@ -610,6 +611,51 @@ describe("Parser", [&]() { }); }); }); + + describe("set_enabled(enabled)", [&]() { + it("stops the in-progress parse if false is passed", [&]() { + ts_parser_set_language(parser, load_real_language("json")); + AssertThat(ts_parser_enabled(parser), IsTrue()); + + auto tree_future = std::async([parser]() { + size_t read_count = 0; + TSInput infinite_input = { + &read_count, + [](void *payload, uint32_t *bytes_read) { + size_t *read_count = static_cast(payload); + assert((*read_count)++ < 100000); + *bytes_read = 1; + return "["; + }, + [](void *payload, unsigned byte, TSPoint position) -> int { + return true; + }, + TSInputEncodingUTF8 + }; + + return ts_parser_parse(parser, nullptr, infinite_input); + }); + + auto cancel_future = std::async([parser]() { + ts_parser_set_enabled(parser, false); + }); + + cancel_future.wait(); + tree_future.wait(); + AssertThat(ts_parser_enabled(parser), IsFalse()); + AssertThat(tree_future.get(), Equals(nullptr)); + + TSTree *tree = ts_parser_parse_string(parser, nullptr, "[]", 2); + AssertThat(ts_parser_enabled(parser), IsFalse()); + AssertThat(tree, Equals(nullptr)); + + ts_parser_set_enabled(parser, true); + AssertThat(ts_parser_enabled(parser), IsTrue()); + tree = ts_parser_parse_string(parser, nullptr, "[]", 2); + AssertThat(tree, !Equals(nullptr)); + ts_tree_delete(tree); + }); + }); }); END_TEST From c56c4affb25a3ea4797e0ff29ad3c81718d8ef9e Mon Sep 17 00:00:00 2001 From: Phil Turnbull Date: Mon, 21 May 2018 10:48:19 -0700 Subject: [PATCH 67/90] Use ts_tree_root_node in fuzz driver --- test/fuzz/fuzzer.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/fuzz/fuzzer.cc b/test/fuzz/fuzzer.cc index 570dfa7e..6fe1e4cb 100644 --- a/test/fuzz/fuzzer.cc +++ b/test/fuzz/fuzzer.cc @@ -17,7 +17,7 @@ extern "C" int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) { ts_parser_halt_on_error(parser, TS_HALT_ON_ERROR); TSTree *tree = ts_parser_parse_string(parser, NULL, str, size); - TSNode root_node = ts_document_root_node(tree); + TSNode root_node = ts_tree_root_node(tree); ts_tree_delete(tree); ts_parser_delete(parser); From c41841f4e95a0769f688f6b715cdc7db2fbad039 Mon Sep 17 00:00:00 2001 From: Phil Turnbull Date: Mon, 21 May 2018 12:49:23 -0700 Subject: [PATCH 68/90] Assert ts_parser_set_language in fuzz driver --- test/fuzz/fuzzer.cc | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/test/fuzz/fuzzer.cc b/test/fuzz/fuzzer.cc index 6fe1e4cb..5c247556 100644 --- a/test/fuzz/fuzzer.cc +++ b/test/fuzz/fuzzer.cc @@ -1,4 +1,5 @@ #include +#include #include "tree_sitter/runtime.h" void test_log(void *payload, TSLogType type, const char *string) { } @@ -13,7 +14,11 @@ extern "C" int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) { const char *str = reinterpret_cast(data); TSParser *parser = ts_parser_new(); - ts_parser_set_language(parser, TS_LANG()); + + // This can fail if the language version doesn't match the runtime version + bool language_ok = ts_parser_set_language(parser, TS_LANG()); + assert(language_ok); + ts_parser_halt_on_error(parser, TS_HALT_ON_ERROR); TSTree *tree = ts_parser_parse_string(parser, NULL, str, size); From 80630ce50404feaf4662558aea5ed68fbf6df263 Mon Sep 17 00:00:00 2001 From: Max Brunsfeld Date: Tue, 22 May 2018 08:50:04 -0700 Subject: [PATCH 69/90] Store nodes' public-facing positions, not pre-padding positions --- src/runtime/node.c | 37 +++++++++++++------------------------ src/runtime/tree.c | 2 +- src/runtime/tree_cursor.c | 7 ++++++- 3 files changed, 20 insertions(+), 26 deletions(-) diff --git a/src/runtime/node.c b/src/runtime/node.c index 31843e2a..15a1572a 100644 --- a/src/runtime/node.c +++ b/src/runtime/node.c @@ -28,12 +28,12 @@ static inline TSNode ts_node__null() { // TSNode - accessors -static inline uint32_t ts_node__byte(const TSNode *self) { - return self->context[0]; +uint32_t ts_node_start_byte(const TSNode self) { + return self.context[0]; } -static inline TSPoint ts_node__position(const TSNode *self) { - return (TSPoint) {self->context[1], self->context[2]}; +TSPoint ts_node_start_point(const TSNode self) { + return (TSPoint) {self.context[1], self.context[2]}; } static inline uint32_t ts_node__alias(const TSNode *self) { @@ -60,7 +60,7 @@ static inline NodeChildIterator ts_node_child_iterator_begin(const TSNode *node) return (NodeChildIterator) { .tree = tree, .parent = subtree, - .position = {ts_node__byte(node), ts_node__position(node)}, + .position = {ts_node_start_byte(*node), ts_node_start_point(*node)}, .child_index = 0, .structural_child_index = 0, .alias_sequence = alias_sequence, @@ -77,13 +77,16 @@ static inline bool ts_node_child_iterator_next(NodeChildIterator *self, TSNode * } self->structural_child_index++; } + if (self->child_index > 0) { + self->position = length_add(self->position, child->padding); + } *result = ts_node_new( self->tree, child, self->position, alias_symbol ); - self->position = length_add(self->position, ts_subtree_total_size(child)); + self->position = length_add(self->position, child->size); self->child_index++; return true; } @@ -217,7 +220,7 @@ static inline TSNode ts_node__next_sibling(TSNode self, bool include_anonymous) NodeChildIterator iterator = ts_node_child_iterator_begin(&node); while (ts_node_child_iterator_next(&iterator, &child)) { if (iterator.position.bytes < target_end_byte) continue; - if (ts_node__byte(&child) <= ts_node__byte(&self)) { + if (ts_node_start_byte(child) <= ts_node_start_byte(self)) { if (ts_node__subtree(child) != ts_node__subtree(self)) { child_containing_target = child; } @@ -296,7 +299,7 @@ static inline TSNode ts_node__descendant_for_byte_range(TSNode self, uint32_t mi NodeChildIterator iterator = ts_node_child_iterator_begin(&node); while (ts_node_child_iterator_next(&iterator, &child)) { if (iterator.position.bytes > max) { - if (ts_node__byte(&child) > min) break; + if (ts_node_start_byte(child) > min) break; node = child; if (ts_node__is_relevant(node, include_anonymous)) last_visible_node = node; did_descend = true; @@ -344,21 +347,10 @@ static inline TSNode ts_node__descendant_for_point_range(TSNode self, TSPoint mi // TSNode - public -uint32_t ts_node_start_byte(TSNode self) { - return ts_node__byte(&self) + ts_node__subtree(self)->padding.bytes; -} - uint32_t ts_node_end_byte(TSNode self) { return ts_node_start_byte(self) + ts_node__subtree(self)->size.bytes; } -TSPoint ts_node_start_point(TSNode self) { - return point_add( - ts_node__position(&self), - ts_node__subtree(self)->padding.extent - ); -} - TSPoint ts_node_end_point(TSNode self) { return point_add(ts_node_start_point(self), ts_node__subtree(self)->size.extent); } @@ -377,10 +369,7 @@ char *ts_node_string(TSNode self) { } bool ts_node_eq(TSNode self, TSNode other) { - return ( - ts_subtree_eq(ts_node__subtree(self), ts_node__subtree(other)) && - ts_node__byte(&self) == ts_node__byte(&other) - ); + return self.tree == other.tree && self.id == other.id; } bool ts_node_is_null(TSNode self) { @@ -421,7 +410,7 @@ TSNode ts_node_parent(TSNode self) { NodeChildIterator iterator = ts_node_child_iterator_begin(&node); while (ts_node_child_iterator_next(&iterator, &child)) { if ( - ts_node__byte(&child) > ts_node__byte(&self) || + ts_node_start_byte(child) > ts_node_start_byte(self) || ts_node__subtree(child) == ts_node__subtree(self) ) break; if (iterator.position.bytes >= end_byte) { diff --git a/src/runtime/tree.c b/src/runtime/tree.c index 5e82df89..9d7c36fc 100644 --- a/src/runtime/tree.c +++ b/src/runtime/tree.c @@ -25,7 +25,7 @@ void ts_tree_delete(TSTree *self) { } TSNode ts_tree_root_node(const TSTree *self) { - return ts_node_new(self, self->root, length_zero(), 0); + return ts_node_new(self, self->root, self->root->padding, 0); } void ts_tree_edit(TSTree *self, const TSInputEdit *edit) { diff --git a/src/runtime/tree_cursor.c b/src/runtime/tree_cursor.c index 092e31f9..4f7e0078 100644 --- a/src/runtime/tree_cursor.c +++ b/src/runtime/tree_cursor.c @@ -184,5 +184,10 @@ TSNode ts_tree_cursor_current_node(const TSTreeCursor *_self) { alias_symbol = alias_sequence[last_entry->structural_child_index]; } } - return ts_node_new(self->tree, last_entry->subtree, last_entry->position, alias_symbol); + return ts_node_new( + self->tree, + last_entry->subtree, + length_add(last_entry->position, last_entry->subtree->padding), + alias_symbol + ); } From 8f31a5f02aa10f817a0dbd0fbfd50c0683c73819 Mon Sep 17 00:00:00 2001 From: Phil Turnbull Date: Tue, 22 May 2018 09:01:59 -0700 Subject: [PATCH 70/90] Remove unused import --- test/fuzz/fuzzer.cc | 1 - 1 file changed, 1 deletion(-) diff --git a/test/fuzz/fuzzer.cc b/test/fuzz/fuzzer.cc index 5c247556..3727c763 100644 --- a/test/fuzz/fuzzer.cc +++ b/test/fuzz/fuzzer.cc @@ -1,4 +1,3 @@ -#include #include #include "tree_sitter/runtime.h" From d7bea0a6f63ec5c66ad7a9f434a9b6dfdf8d9bc5 Mon Sep 17 00:00:00 2001 From: Phil Turnbull Date: Tue, 22 May 2018 09:05:02 -0700 Subject: [PATCH 71/90] Remove unused logger --- test/fuzz/fuzzer.cc | 6 ------ 1 file changed, 6 deletions(-) diff --git a/test/fuzz/fuzzer.cc b/test/fuzz/fuzzer.cc index 3727c763..2ed7683f 100644 --- a/test/fuzz/fuzzer.cc +++ b/test/fuzz/fuzzer.cc @@ -1,12 +1,6 @@ #include #include "tree_sitter/runtime.h" -void test_log(void *payload, TSLogType type, const char *string) { } - -TSLogger logger = { - .log = test_log, -}; - extern "C" const TSLanguage *TS_LANG(); extern "C" int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) { From c0763c69c4278277937048aa45f9c7094fd7f86c Mon Sep 17 00:00:00 2001 From: Max Brunsfeld Date: Tue, 22 May 2018 13:30:10 -0700 Subject: [PATCH 72/90] Fix handling of aliases in TreeCursor --- src/runtime/tree_cursor.c | 64 ++++++++++++++++++++++++++++----------- test/runtime/node_test.cc | 62 ++++++++++++++++++++++++++++++++----- 2 files changed, 101 insertions(+), 25 deletions(-) diff --git a/src/runtime/tree_cursor.c b/src/runtime/tree_cursor.c index 4f7e0078..c7c106b3 100644 --- a/src/runtime/tree_cursor.c +++ b/src/runtime/tree_cursor.c @@ -29,17 +29,27 @@ void ts_tree_cursor_delete(TSTreeCursor *_self) { bool ts_tree_cursor_goto_first_child(TSTreeCursor *_self) { TreeCursor *self = (TreeCursor *)_self; TreeCursorEntry *last_entry = array_back(&self->stack); - const Subtree *tree = last_entry->subtree; + const Subtree *subtree = last_entry->subtree; + if (subtree->children.size == 0 || subtree->visible_child_count == 0) return false; Length position = last_entry->position; bool did_descend; do { did_descend = false; + const TSSymbol *alias_sequence = ts_language_alias_sequence( + self->tree->language, + subtree->alias_sequence_id + ); uint32_t structural_child_index = 0; - for (uint32_t i = 0; i < tree->children.size; i++) { - const Subtree *child = tree->children.contents[i]; - if (child->visible || child->visible_child_count > 0) { + for (uint32_t i = 0; i < subtree->children.size; i++) { + const Subtree *child = subtree->children.contents[i]; + bool visible = child->visible; + if (!child->extra) { + visible |= alias_sequence && alias_sequence[structural_child_index]; + } + + if (visible || (child->children.size > 0 && child->visible_child_count > 0)) { array_push(&self->stack, ((TreeCursorEntry) { .subtree = child, .child_index = i, @@ -47,10 +57,10 @@ bool ts_tree_cursor_goto_first_child(TSTreeCursor *_self) { .position = position, })); - if (child->visible) { + if (visible) { return true; } else { - tree = child; + subtree = child; did_descend = true; break; } @@ -67,7 +77,7 @@ int64_t ts_tree_cursor_goto_first_child_for_byte(TSTreeCursor *_self, uint32_t g TreeCursor *self = (TreeCursor *)_self; uint32_t initial_size = self->stack.size; TreeCursorEntry *last_entry = array_back(&self->stack); - const Subtree *tree = last_entry->subtree; + const Subtree *subtree = last_entry->subtree; Length position = last_entry->position; uint32_t visible_child_index = 0; @@ -75,14 +85,25 @@ int64_t ts_tree_cursor_goto_first_child_for_byte(TSTreeCursor *_self, uint32_t g do { did_descend = false; + const TSSymbol *alias_sequence = ts_language_alias_sequence( + self->tree->language, + subtree->alias_sequence_id + ); uint32_t structural_child_index = 0; - for (uint32_t i = 0; i < tree->children.size; i++) { - const Subtree *child = tree->children.contents[i]; + for (uint32_t i = 0; i < subtree->children.size; i++) { + const Subtree *child = subtree->children.contents[i]; Length next_position = length_add(position, ts_subtree_total_size(child)); bool at_goal = next_position.bytes > goal_byte; + bool visible = child->visible; + if (!child->extra) { + visible |= alias_sequence && alias_sequence[structural_child_index]; + } + + uint32_t visible_child_count = child->children.size > 0 ? child->visible_child_count : 0; + if (at_goal) { - if (child->visible || child->visible_child_count > 0) { + if (visible || visible_child_count > 0) { array_push(&self->stack, ((TreeCursorEntry) { .subtree = child, .child_index = i, @@ -90,19 +111,19 @@ int64_t ts_tree_cursor_goto_first_child_for_byte(TSTreeCursor *_self, uint32_t g .position = position, })); - if (child->visible) { + if (visible) { return visible_child_index; } else { - tree = child; + subtree = child; did_descend = true; break; } } } else { - if (child->visible) { + if (visible) { visible_child_index++; } else { - visible_child_index += child->visible_child_count; + visible_child_index += visible_child_count; } } @@ -127,13 +148,22 @@ bool ts_tree_cursor_goto_next_sibling(TSTreeCursor *_self) { uint32_t structural_child_index = child_entry->structural_child_index; Length position = child_entry->position; const Subtree *child = parent->children.contents[child_index]; + const TSSymbol *alias_sequence = ts_language_alias_sequence( + self->tree->language, + parent->alias_sequence_id + ); while (++child_index < parent->children.size) { if (!child->extra) structural_child_index++; position = length_add(position, ts_subtree_total_size(child)); child = parent->children.contents[child_index]; - if (child->visible || child->visible_child_count > 0) { + bool visible = child->visible; + if (!child->extra) { + visible |= alias_sequence && alias_sequence[structural_child_index]; + } + + if (visible || (child->children.size > 0 && child->visible_child_count > 0)) { self->stack.contents[i + 1] = (TreeCursorEntry) { .subtree = child, .child_index = child_index, @@ -142,7 +172,7 @@ bool ts_tree_cursor_goto_next_sibling(TSTreeCursor *_self) { }; self->stack.size = i + 2; - if (child->visible) { + if (visible) { return true; } else { ts_tree_cursor_goto_first_child(_self); @@ -180,7 +210,7 @@ TSNode ts_tree_cursor_current_node(const TSTreeCursor *_self) { self->tree->language, parent_entry->subtree->alias_sequence_id ); - if (alias_sequence) { + if (alias_sequence && !last_entry->subtree->extra) { alias_symbol = alias_sequence[last_entry->structural_child_index]; } } diff --git a/test/runtime/node_test.cc b/test/runtime/node_test.cc index 3cbd48c8..313bec40 100644 --- a/test/runtime/node_test.cc +++ b/test/runtime/node_test.cc @@ -52,15 +52,28 @@ string grammar_with_aliases_and_extras = R"JSON({ "named": true, "content": {"type": "SYMBOL", "name": "b"} }, - {"type": "SYMBOL", "name": "b"} + { + "type": "ALIAS", + "value": "C", + "named": true, + "content": {"type": "SYMBOL", "name": "_c"} + } ] }, "b": {"type": "STRING", "value": "b"}, + + "_c": {"type": "STRING", "value": "c"}, + "comment": {"type": "STRING", "value": "..."} } })JSON"; +const TSLanguage *language_with_aliases_and_extras = load_test_language( + "aliases_and_extras", + ts_compile_grammar(grammar_with_aliases_and_extras.c_str()) +); + describe("Node", [&]() { TSParser *parser; TSTree *tree; @@ -163,16 +176,13 @@ describe("Node", [&]() { }); it("works correctly when the node contains aliased children and extras", [&]() { - TSCompileResult compile_result = ts_compile_grammar(grammar_with_aliases_and_extras.c_str()); - const TSLanguage *language = load_test_language("aliases_and_extras", compile_result); - ts_parser_set_language(parser, language); - + ts_parser_set_language(parser, language_with_aliases_and_extras); ts_tree_delete(tree); - tree = ts_parser_parse_string(parser, nullptr, "b ... b ... b", 13); + tree = ts_parser_parse_string(parser, nullptr, "b ... b ... c", 13); root_node = ts_tree_root_node(tree); char *node_string = ts_node_string(root_node); - AssertThat(node_string, Equals("(a (b) (comment) (B) (comment) (b))")); + AssertThat(node_string, Equals("(a (b) (comment) (B) (comment) (C))")); ts_free(node_string); AssertThat(ts_node_named_child_count(root_node), Equals(5u)); @@ -180,7 +190,7 @@ describe("Node", [&]() { AssertThat(ts_node_type(ts_node_named_child(root_node, 1)), Equals("comment")); AssertThat(ts_node_type(ts_node_named_child(root_node, 2)), Equals("B")); AssertThat(ts_node_type(ts_node_named_child(root_node, 3)), Equals("comment")); - AssertThat(ts_node_type(ts_node_named_child(root_node, 4)), Equals("b")); + AssertThat(ts_node_type(ts_node_named_child(root_node, 4)), Equals("C")); AssertThat( ts_node_symbol(ts_node_named_child(root_node, 0)), @@ -700,6 +710,42 @@ describe("TreeCursor", [&]() { AssertThat(ts_node_start_byte(node), Equals(array_index)); AssertThat(child_index, Equals(-1)); }); + + it("walks the tree correctly when the node contains aliased children and extras", [&]() { + ts_parser_set_language(parser, language_with_aliases_and_extras); + ts_tree_cursor_delete(&cursor); + ts_tree_delete(tree); + + tree = ts_parser_parse_string(parser, nullptr, "b ... b ... c", 13); + cursor = ts_tree_cursor_new(tree); + + TSNode node = ts_tree_cursor_current_node(&cursor); + AssertThat(ts_node_type(node), Equals("a")); + + AssertThat(ts_tree_cursor_goto_first_child(&cursor), IsTrue()); + node = ts_tree_cursor_current_node(&cursor); + AssertThat(ts_node_type(node), Equals("b")); + + AssertThat(ts_tree_cursor_goto_next_sibling(&cursor), IsTrue()); + node = ts_tree_cursor_current_node(&cursor); + AssertThat(ts_node_type(node), Equals("comment")); + + AssertThat(ts_tree_cursor_goto_next_sibling(&cursor), IsTrue()); + node = ts_tree_cursor_current_node(&cursor); + AssertThat(ts_node_type(node), Equals("B")); + + AssertThat(ts_tree_cursor_goto_next_sibling(&cursor), IsTrue()); + node = ts_tree_cursor_current_node(&cursor); + AssertThat(ts_node_type(node), Equals("comment")); + + AssertThat(ts_tree_cursor_goto_next_sibling(&cursor), IsTrue()); + node = ts_tree_cursor_current_node(&cursor); + AssertThat(ts_node_type(node), Equals("C")); + + AssertThat(ts_tree_cursor_goto_next_sibling(&cursor), IsFalse()); + AssertThat(ts_tree_cursor_goto_parent(&cursor), IsTrue()); + AssertThat(ts_tree_cursor_goto_first_child_for_byte(&cursor, 0), Equals(0)); + }); }); END_TEST From babb8261a3c43b028a84b222737094e17d45dca6 Mon Sep 17 00:00:00 2001 From: Max Brunsfeld Date: Tue, 22 May 2018 14:42:57 -0700 Subject: [PATCH 73/90] Extract an iterator from TreeCursor methods --- src/runtime/node.c | 24 ++--- src/runtime/tree_cursor.c | 213 ++++++++++++++++++-------------------- 2 files changed, 114 insertions(+), 123 deletions(-) diff --git a/src/runtime/node.c b/src/runtime/node.c index 15a1572a..3c858b92 100644 --- a/src/runtime/node.c +++ b/src/runtime/node.c @@ -10,7 +10,7 @@ typedef struct { uint32_t child_index; uint32_t structural_child_index; const TSSymbol *alias_sequence; -} NodeChildIterator; +} ChildIterator; // TSNode - constructors @@ -48,16 +48,16 @@ static inline const TSTree *ts_node__tree(const TSNode *self) { return self->tree; } -// NodeChildIterator +// ChildIterator -static inline NodeChildIterator ts_node_child_iterator_begin(const TSNode *node) { +static inline ChildIterator ts_node_iterate_children(const TSNode *node) { const TSTree *tree = ts_node__tree(node); const Subtree *subtree = ts_node__subtree(*node); const TSSymbol *alias_sequence = ts_language_alias_sequence( tree->language, subtree->alias_sequence_id ); - return (NodeChildIterator) { + return (ChildIterator) { .tree = tree, .parent = subtree, .position = {ts_node_start_byte(*node), ts_node_start_point(*node)}, @@ -67,7 +67,7 @@ static inline NodeChildIterator ts_node_child_iterator_begin(const TSNode *node) }; } -static inline bool ts_node_child_iterator_next(NodeChildIterator *self, TSNode *result) { +static inline bool ts_node_child_iterator_next(ChildIterator *self, TSNode *result) { if (self->child_index == self->parent->children.size) return false; const Subtree *child = self->parent->children.contents[self->child_index]; TSSymbol alias_symbol = 0; @@ -133,7 +133,7 @@ static inline TSNode ts_node__child(TSNode self, uint32_t child_index, bool incl TSNode child; uint32_t index = 0; - NodeChildIterator iterator = ts_node_child_iterator_begin(&result); + ChildIterator iterator = ts_node_iterate_children(&result); while (ts_node_child_iterator_next(&iterator, &child)) { if (ts_node__is_relevant(child, include_anonymous)) { if (index == child_index) return child; @@ -168,7 +168,7 @@ static inline TSNode ts_node__prev_sibling(TSNode self, bool include_anonymous) bool found_child_containing_target = false; TSNode child; - NodeChildIterator iterator = ts_node_child_iterator_begin(&node); + ChildIterator iterator = ts_node_iterate_children(&node); while (ts_node_child_iterator_next(&iterator, &child)) { if (iterator.position.bytes >= target_end_byte) { found_child_containing_target = ts_node__subtree(child) != ts_node__subtree(self); @@ -217,7 +217,7 @@ static inline TSNode ts_node__next_sibling(TSNode self, bool include_anonymous) TSNode child_containing_target = ts_node__null(); TSNode child; - NodeChildIterator iterator = ts_node_child_iterator_begin(&node); + ChildIterator iterator = ts_node_iterate_children(&node); while (ts_node_child_iterator_next(&iterator, &child)) { if (iterator.position.bytes < target_end_byte) continue; if (ts_node_start_byte(child) <= ts_node_start_byte(self)) { @@ -268,7 +268,7 @@ static inline TSNode ts_node__first_child_for_byte(TSNode self, uint32_t goal, did_descend = false; TSNode child; - NodeChildIterator iterator = ts_node_child_iterator_begin(&node); + ChildIterator iterator = ts_node_iterate_children(&node); while (ts_node_child_iterator_next(&iterator, &child)) { if (ts_node_end_byte(child) > goal) { if (ts_node__is_relevant(child, include_anonymous)) { @@ -296,7 +296,7 @@ static inline TSNode ts_node__descendant_for_byte_range(TSNode self, uint32_t mi did_descend = false; TSNode child; - NodeChildIterator iterator = ts_node_child_iterator_begin(&node); + ChildIterator iterator = ts_node_iterate_children(&node); while (ts_node_child_iterator_next(&iterator, &child)) { if (iterator.position.bytes > max) { if (ts_node_start_byte(child) > min) break; @@ -324,7 +324,7 @@ static inline TSNode ts_node__descendant_for_point_range(TSNode self, TSPoint mi did_descend = false; TSNode child; - NodeChildIterator iterator = ts_node_child_iterator_begin(&node); + ChildIterator iterator = ts_node_iterate_children(&node); while (ts_node_child_iterator_next(&iterator, &child)) { const Subtree *child_tree = ts_node__subtree(child); if (iterator.child_index != 1) { @@ -407,7 +407,7 @@ TSNode ts_node_parent(TSNode self) { did_descend = false; TSNode child; - NodeChildIterator iterator = ts_node_child_iterator_begin(&node); + ChildIterator iterator = ts_node_iterate_children(&node); while (ts_node_child_iterator_next(&iterator, &child)) { if ( ts_node_start_byte(child) > ts_node_start_byte(self) || diff --git a/src/runtime/tree_cursor.c b/src/runtime/tree_cursor.c index c7c106b3..f2d9279f 100644 --- a/src/runtime/tree_cursor.c +++ b/src/runtime/tree_cursor.c @@ -4,6 +4,56 @@ #include "runtime/language.h" #include "runtime/tree.h" +typedef struct { + const Subtree *parent; + const TSTree *tree; + Length position; + uint32_t child_index; + uint32_t structural_child_index; + const TSSymbol *alias_sequence; +} ChildIterator; + +// ChildIterator + +static inline ChildIterator ts_tree_cursor_iterate_children(const TreeCursor *self) { + TreeCursorEntry *last_entry = array_back(&self->stack); + const TSSymbol *alias_sequence = ts_language_alias_sequence( + self->tree->language, + last_entry->subtree->alias_sequence_id + ); + return (ChildIterator) { + .tree = self->tree, + .parent = last_entry->subtree, + .position = last_entry->position, + .child_index = 0, + .structural_child_index = 0, + .alias_sequence = alias_sequence, + }; +} + +static inline bool ts_tree_cursor_child_iterator_next(ChildIterator *self, + TreeCursorEntry *result, + bool *visible) { + if (self->child_index == self->parent->children.size) return false; + const Subtree *child = self->parent->children.contents[self->child_index]; + *result = (TreeCursorEntry) { + .subtree = child, + .position = self->position, + .child_index = self->child_index, + .structural_child_index = self->structural_child_index, + }; + *visible = child->visible; + if (!child->extra && self->alias_sequence) { + *visible |= self->alias_sequence[self->structural_child_index]; + } + self->position = length_add(self->position, ts_subtree_total_size(child)); + self->child_index++; + if (!child->extra) self->structural_child_index++; + return true; +} + +// TSTreeCursor - lifecycle + TSTreeCursor ts_tree_cursor_new(const TSTree *tree) { TSTreeCursor self; ts_tree_cursor_init((TreeCursor *)&self, tree); @@ -26,47 +76,29 @@ void ts_tree_cursor_delete(TSTreeCursor *_self) { array_delete(&self->stack); } +// TSTreeCursor - walking the tree + bool ts_tree_cursor_goto_first_child(TSTreeCursor *_self) { TreeCursor *self = (TreeCursor *)_self; - TreeCursorEntry *last_entry = array_back(&self->stack); - const Subtree *subtree = last_entry->subtree; - if (subtree->children.size == 0 || subtree->visible_child_count == 0) return false; - Length position = last_entry->position; bool did_descend; do { did_descend = false; - const TSSymbol *alias_sequence = ts_language_alias_sequence( - self->tree->language, - subtree->alias_sequence_id - ); - uint32_t structural_child_index = 0; - for (uint32_t i = 0; i < subtree->children.size; i++) { - const Subtree *child = subtree->children.contents[i]; - bool visible = child->visible; - if (!child->extra) { - visible |= alias_sequence && alias_sequence[structural_child_index]; + bool visible; + TreeCursorEntry entry; + ChildIterator iterator = ts_tree_cursor_iterate_children(self); + while (ts_tree_cursor_child_iterator_next(&iterator, &entry, &visible)) { + if (visible) { + array_push(&self->stack, entry); + return true; } - if (visible || (child->children.size > 0 && child->visible_child_count > 0)) { - array_push(&self->stack, ((TreeCursorEntry) { - .subtree = child, - .child_index = i, - .structural_child_index = structural_child_index, - .position = position, - })); - - if (visible) { - return true; - } else { - subtree = child; - did_descend = true; - break; - } + if (entry.subtree->children.size > 0 && entry.subtree->visible_child_count > 0) { + array_push(&self->stack, entry); + did_descend = true; + break; } - if (!child->extra) structural_child_index++; - position = length_add(position, ts_subtree_total_size(child)); } } while (did_descend); @@ -76,59 +108,37 @@ bool ts_tree_cursor_goto_first_child(TSTreeCursor *_self) { int64_t ts_tree_cursor_goto_first_child_for_byte(TSTreeCursor *_self, uint32_t goal_byte) { TreeCursor *self = (TreeCursor *)_self; uint32_t initial_size = self->stack.size; - TreeCursorEntry *last_entry = array_back(&self->stack); - const Subtree *subtree = last_entry->subtree; - Length position = last_entry->position; uint32_t visible_child_index = 0; bool did_descend; do { did_descend = false; - const TSSymbol *alias_sequence = ts_language_alias_sequence( - self->tree->language, - subtree->alias_sequence_id - ); - uint32_t structural_child_index = 0; - for (uint32_t i = 0; i < subtree->children.size; i++) { - const Subtree *child = subtree->children.contents[i]; - Length next_position = length_add(position, ts_subtree_total_size(child)); - bool at_goal = next_position.bytes > goal_byte; - - bool visible = child->visible; - if (!child->extra) { - visible |= alias_sequence && alias_sequence[structural_child_index]; - } - - uint32_t visible_child_count = child->children.size > 0 ? child->visible_child_count : 0; + bool visible; + TreeCursorEntry entry; + ChildIterator iterator = ts_tree_cursor_iterate_children(self); + while (ts_tree_cursor_child_iterator_next(&iterator, &entry, &visible)) { + bool at_goal = iterator.position.bytes > goal_byte; + uint32_t visible_child_count = entry.subtree->children.size > 0 + ? entry.subtree->visible_child_count + : 0; if (at_goal) { - if (visible || visible_child_count > 0) { - array_push(&self->stack, ((TreeCursorEntry) { - .subtree = child, - .child_index = i, - .structural_child_index = structural_child_index, - .position = position, - })); - - if (visible) { - return visible_child_index; - } else { - subtree = child; - did_descend = true; - break; - } - } - } else { if (visible) { - visible_child_index++; - } else { - visible_child_index += visible_child_count; + array_push(&self->stack, entry); + return visible_child_index; } - } - if (!child->extra) structural_child_index++; - position = next_position; + if (visible_child_count > 0) { + array_push(&self->stack, entry); + did_descend = true; + break; + } + } else if (visible) { + visible_child_index++; + } else { + visible_child_index += visible_child_count; + } } } while (did_descend); @@ -138,53 +148,34 @@ int64_t ts_tree_cursor_goto_first_child_for_byte(TSTreeCursor *_self, uint32_t g bool ts_tree_cursor_goto_next_sibling(TSTreeCursor *_self) { TreeCursor *self = (TreeCursor *)_self; - TreeCursorEntry *child_entry = array_back(&self->stack); + uint32_t initial_size = self->stack.size; - for (unsigned i = self->stack.size - 2; i + 1 > 0; i--) { - TreeCursorEntry *parent_entry = &self->stack.contents[i]; + while (self->stack.size > 1) { + bool visible; + TreeCursorEntry entry = array_pop(&self->stack); + ChildIterator iterator = ts_tree_cursor_iterate_children(self); + iterator.child_index = entry.child_index; + iterator.structural_child_index = entry.structural_child_index; + iterator.position = entry.position; - const Subtree *parent = parent_entry->subtree; - uint32_t child_index = child_entry->child_index; - uint32_t structural_child_index = child_entry->structural_child_index; - Length position = child_entry->position; - const Subtree *child = parent->children.contents[child_index]; - const TSSymbol *alias_sequence = ts_language_alias_sequence( - self->tree->language, - parent->alias_sequence_id - ); + ts_tree_cursor_child_iterator_next(&iterator, &entry, &visible); + if (visible && self->stack.size + 1 < initial_size) break; - while (++child_index < parent->children.size) { - if (!child->extra) structural_child_index++; - position = length_add(position, ts_subtree_total_size(child)); - child = parent->children.contents[child_index]; - - bool visible = child->visible; - if (!child->extra) { - visible |= alias_sequence && alias_sequence[structural_child_index]; + while (ts_tree_cursor_child_iterator_next(&iterator, &entry, &visible)) { + if (visible) { + array_push(&self->stack, entry); + return true; } - if (visible || (child->children.size > 0 && child->visible_child_count > 0)) { - self->stack.contents[i + 1] = (TreeCursorEntry) { - .subtree = child, - .child_index = child_index, - .structural_child_index = structural_child_index, - .position = position, - }; - self->stack.size = i + 2; - - if (visible) { - return true; - } else { - ts_tree_cursor_goto_first_child(_self); - return true; - } + if (entry.subtree->children.size > 0 && entry.subtree->visible_child_count > 0) { + array_push(&self->stack, entry); + ts_tree_cursor_goto_first_child(_self); + return true; } } - - child_entry = parent_entry; - if (parent->visible) break; } + self->stack.size = initial_size; return false; } From e16f0338d67954edd0653e772ec25e33c6ac4851 Mon Sep 17 00:00:00 2001 From: Max Brunsfeld Date: Wed, 23 May 2018 14:30:23 -0700 Subject: [PATCH 74/90] Add APIs for pausing a parse after N operations and resuming later --- include/tree_sitter/runtime.h | 5 ++- src/runtime/parser.c | 51 +++++++++++++++++------------- test/runtime/parser_test.cc | 58 +++++++++++++++++++++++++++++++++++ 3 files changed, 91 insertions(+), 23 deletions(-) diff --git a/include/tree_sitter/runtime.h b/include/tree_sitter/runtime.h index e21571e8..73cf4fd1 100644 --- a/include/tree_sitter/runtime.h +++ b/include/tree_sitter/runtime.h @@ -86,8 +86,11 @@ void ts_parser_print_dot_graphs(TSParser *, FILE *); void ts_parser_halt_on_error(TSParser *, bool); TSTree *ts_parser_parse(TSParser *, const TSTree *, TSInput); TSTree *ts_parser_parse_string(TSParser *, const TSTree *, const char *, uint32_t); -bool ts_parser_enabled(TSParser *); +bool ts_parser_enabled(const TSParser *); void ts_parser_set_enabled(TSParser *, bool); +size_t ts_parser_operation_limit(const TSParser *); +void ts_parser_set_operation_limit(TSParser *, size_t); +TSTree *ts_parser_resume(TSParser *); TSTree *ts_tree_copy(const TSTree *); void ts_tree_delete(TSTree *); diff --git a/src/runtime/parser.c b/src/runtime/parser.c index 26a377d7..efde7830 100644 --- a/src/runtime/parser.c +++ b/src/runtime/parser.c @@ -58,9 +58,10 @@ struct TSParser { ReusableNode reusable_node; void *external_scanner_payload; FILE *dot_graph_file; - bool halt_on_error; unsigned accept_count; + size_t operation_limit; volatile bool enabled; + bool halt_on_error; }; typedef struct { @@ -700,19 +701,14 @@ static void ts_parser__start(TSParser *self, TSInput input, const Subtree *previ } ts_lexer_set_input(&self->lexer, input); - ts_stack_clear(self->stack); - reusable_node_reset(&self->reusable_node, previous_tree); - self->finished_tree = NULL; - self->accept_count = 0; -} - -static void ts_parser__stop(TSParser *self) { ts_stack_clear(self->stack); ts_parser__set_cached_token(self, 0, NULL, NULL); + reusable_node_reset(&self->reusable_node, previous_tree); if (self->finished_tree) { ts_subtree_release(&self->tree_pool, self->finished_tree); self->finished_tree = NULL; } + self->accept_count = 0; } static void ts_parser__accept(TSParser *self, StackVersion version, const Subtree *lookahead) { @@ -1319,6 +1315,7 @@ TSParser *ts_parser_new() { self->dot_graph_file = NULL; self->halt_on_error = false; self->enabled = true; + self->operation_limit = SIZE_MAX; ts_parser__set_cached_token(self, 0, NULL, NULL); return self; } @@ -1373,7 +1370,7 @@ void ts_parser_halt_on_error(TSParser *self, bool should_halt_on_error) { self->halt_on_error = should_halt_on_error; } -bool ts_parser_enabled(TSParser *self) { +bool ts_parser_enabled(const TSParser *self) { return self->enabled; } @@ -1381,20 +1378,26 @@ void ts_parser_set_enabled(TSParser *self, bool enabled) { self->enabled = enabled; } -TSTree *ts_parser_parse(TSParser *self, const TSTree *old_tree, TSInput input) { - if (!self->language) return NULL; - ts_parser__start(self, input, old_tree ? old_tree->root : NULL); +size_t ts_parser_operation_limit(const TSParser *self) { + return self->operation_limit; +} + +void ts_parser_set_operation_limit(TSParser *self, size_t limit) { + self->operation_limit = limit; +} + +TSTree *ts_parser_resume(TSParser *self) { + if (!self->language || !self->lexer.input.read) return NULL; uint32_t position = 0, last_position = 0, version_count = 0; + size_t operation_count = 0; do { for (StackVersion version = 0; version_count = ts_stack_version_count(self->stack), version < version_count; version++) { - if (!self->enabled) { - ts_parser__stop(self); - return NULL; - } + operation_count++; + if (operation_count > self->operation_limit || !self->enabled) return NULL; bool allow_node_reuse = version_count == 1; while (ts_stack_is_active(self->stack, version)) { @@ -1424,19 +1427,23 @@ TSTree *ts_parser_parse(TSParser *self, const TSTree *old_tree, TSInput input) { } } while (version_count != 0); - ts_stack_clear(self->stack); - ts_parser__set_cached_token(self, 0, NULL, NULL); ts_subtree_balance(self->finished_tree, &self->tree_pool, self->language); - - TSTree *result = ts_tree_new(self->finished_tree, self->language); LOG("done"); LOG_TREE(); - self->finished_tree = NULL; - ts_parser__stop(self); + TSTree *result = ts_tree_new(self->finished_tree, self->language); + self->finished_tree = NULL; + ts_stack_clear(self->stack); + ts_parser__set_cached_token(self, 0, NULL, NULL); + ts_lexer_set_input(&self->lexer, (TSInput) { NULL, NULL, NULL, 0 }); return result; } +TSTree *ts_parser_parse(TSParser *self, const TSTree *old_tree, TSInput input) { + ts_parser__start(self, input, old_tree ? old_tree->root : NULL); + return ts_parser_resume(self); +} + TSTree *ts_parser_parse_string(TSParser *self, const TSTree *old_tree, const char *string, uint32_t length) { TSStringInput input; diff --git a/test/runtime/parser_test.cc b/test/runtime/parser_test.cc index 4e787c76..9599563d 100644 --- a/test/runtime/parser_test.cc +++ b/test/runtime/parser_test.cc @@ -656,6 +656,64 @@ describe("Parser", [&]() { ts_tree_delete(tree); }); }); + + describe("set_operation_limit(limit)", [&]() { + it("limits the amount of work the parser does on any given call to parse()", [&]() { + ts_parser_set_language(parser, load_real_language("json")); + + struct InputState { + const char *string; + size_t read_count; + }; + + InputState state = {"[", 0}; + + // An input that repeats the given string forever, counting how many times + // it has been read. + TSInput infinite_input = { + &state, + [](void *payload, uint32_t *bytes_read) { + InputState *state = static_cast(payload); + assert(state->read_count++ <= 10); + *bytes_read = strlen(state->string); + return state->string; + }, + [](void *payload, unsigned byte, TSPoint position) -> int { + return true; + }, + TSInputEncodingUTF8 + }; + + ts_parser_set_operation_limit(parser, 10); + TSTree *tree = ts_parser_parse(parser, nullptr, infinite_input); + AssertThat(tree, Equals(nullptr)); + + state.read_count = 0; + state.string = ""; + + tree = ts_parser_resume(parser); + AssertThat(tree, !Equals(nullptr)); + ts_tree_delete(tree); + }); + }); + + describe("resume()", [&]() { + it("does nothing unless parsing was previously halted", [&]() { + ts_parser_set_language(parser, load_real_language("json")); + + TSTree *tree = ts_parser_resume(parser); + AssertThat(tree, Equals(nullptr)); + tree = ts_parser_resume(parser); + AssertThat(tree, Equals(nullptr)); + + tree = ts_parser_parse_string(parser, nullptr, "true", 4); + AssertThat(tree, !Equals(nullptr)); + ts_tree_delete(tree); + + tree = ts_parser_resume(parser); + AssertThat(tree, Equals(nullptr)); + }); + }); }); END_TEST From 69d8c6f5e6d1541357d7e829b2b5a248b4ddc3e5 Mon Sep 17 00:00:00 2001 From: Max Brunsfeld Date: Wed, 23 May 2018 15:41:16 -0700 Subject: [PATCH 75/90] Check that language is present in both parse() and resume() --- src/runtime/parser.c | 1 + test/runtime/parser_test.cc | 9 +++++++++ 2 files changed, 10 insertions(+) diff --git a/src/runtime/parser.c b/src/runtime/parser.c index efde7830..dcb55a62 100644 --- a/src/runtime/parser.c +++ b/src/runtime/parser.c @@ -1440,6 +1440,7 @@ TSTree *ts_parser_resume(TSParser *self) { } TSTree *ts_parser_parse(TSParser *self, const TSTree *old_tree, TSInput input) { + if (!self->language) return NULL; ts_parser__start(self, input, old_tree ? old_tree->root : NULL); return ts_parser_resume(self); } diff --git a/test/runtime/parser_test.cc b/test/runtime/parser_test.cc index 9599563d..7c9e786f 100644 --- a/test/runtime/parser_test.cc +++ b/test/runtime/parser_test.cc @@ -568,6 +568,15 @@ describe("Parser", [&]() { AssertThat(ts_parser_set_language(parser, &language), IsFalse()); AssertThat(ts_parser_language(parser), Equals(nullptr)); }); + + it("does nothing when parse is called while the language is null", [&]() { + tree = ts_parser_parse_string(parser, nullptr, "{}", 2); + AssertThat(tree, Equals(nullptr)); + + ts_parser_set_language(parser, nullptr); + tree = ts_parser_parse_string(parser, nullptr, "{}", 2); + AssertThat(tree, Equals(nullptr)); + }); }); describe("set_logger(TSLogger)", [&]() { From 6fca8f2f4d9f46f0384b300e06148023d3ac9330 Mon Sep 17 00:00:00 2001 From: Max Brunsfeld Date: Thu, 24 May 2018 16:01:14 -0700 Subject: [PATCH 76/90] Make ts_compile_grammar take an optional log file, start logging to it --- include/tree_sitter/compiler.h | 4 +- project.gyp | 1 + .../build_tables/lex_table_builder.cc | 55 +++++++++++++++++-- .../build_tables/parse_table_builder.cc | 33 ++++++++--- src/compiler/compile.cc | 9 ++- src/compiler/log.cc | 31 +++++++++++ src/compiler/log.h | 38 +++++++++++++ src/compiler/rules/character_set.cc | 5 ++ src/compiler/rules/character_set.h | 3 +- test/helpers/load_language.cc | 2 +- test/integration/test_grammars.cc | 4 +- test/runtime/language_test.cc | 2 +- test/runtime/node_test.cc | 2 +- 13 files changed, 164 insertions(+), 25 deletions(-) create mode 100644 src/compiler/log.cc create mode 100644 src/compiler/log.h diff --git a/include/tree_sitter/compiler.h b/include/tree_sitter/compiler.h index 30f4e99c..ca2a28f7 100644 --- a/include/tree_sitter/compiler.h +++ b/include/tree_sitter/compiler.h @@ -5,6 +5,8 @@ extern "C" { #endif +#include + typedef enum { TSCompileErrorTypeNone, TSCompileErrorTypeInvalidGrammar, @@ -25,7 +27,7 @@ typedef struct { TSCompileErrorType error_type; } TSCompileResult; -TSCompileResult ts_compile_grammar(const char *input); +TSCompileResult ts_compile_grammar(const char *input, FILE *log_file); #ifdef __cplusplus } diff --git a/project.gyp b/project.gyp index b88a6c65..0006a18a 100644 --- a/project.gyp +++ b/project.gyp @@ -22,6 +22,7 @@ 'src/compiler/compile.cc', 'src/compiler/generate_code/c_code.cc', 'src/compiler/lex_table.cc', + 'src/compiler/log.cc', 'src/compiler/parse_grammar.cc', 'src/compiler/parse_table.cc', 'src/compiler/precedence_range.cc', diff --git a/src/compiler/build_tables/lex_table_builder.cc b/src/compiler/build_tables/lex_table_builder.cc index e0cab5e2..54957efe 100644 --- a/src/compiler/build_tables/lex_table_builder.cc +++ b/src/compiler/build_tables/lex_table_builder.cc @@ -9,9 +9,11 @@ #include #include "compiler/build_tables/lex_item.h" #include "compiler/build_tables/lookahead_set.h" -#include "compiler/parse_table.h" #include "compiler/lexical_grammar.h" +#include "compiler/log.h" +#include "compiler/parse_table.h" #include "compiler/rule.h" +#include "utf8proc.h" namespace tree_sitter { namespace build_tables { @@ -78,6 +80,7 @@ class LexTableBuilderImpl : public LexTableBuilder { bool conflict_detection_mode; LookaheadSet keyword_symbols; Symbol keyword_capture_token; + char encoding_buffer[8]; public: LexTableBuilderImpl(const SyntaxGrammar &syntax_grammar, @@ -151,6 +154,7 @@ class LexTableBuilderImpl : public LexTableBuilder { // For each pair of tokens, generate a lex table for just those two tokens and record what // conflicts arise. + LOG_START("detecting conflicts between tokens"); conflict_detection_mode = true; for (Symbol::Index i = 0, n = grammar.variables.size(); i < n; i++) { for (Symbol::Index j = 0; j < i; j++) { @@ -165,6 +169,7 @@ class LexTableBuilderImpl : public LexTableBuilder { } } } + LOG_END(); // Find a 'keyword capture token' that matches all of the indentified keywords. for (Symbol::Index i = 0, n = grammar.variables.size(); i < n; i++) { @@ -304,9 +309,33 @@ class LexTableBuilderImpl : public LexTableBuilder { if (prefer_advancing && !next_item_set_can_yield_this_token) { auto advance_symbol = transition.destination.entries.begin()->lhs; - if (characters.intersects(following_characters_by_token[accept_action.symbol.index]) || - characters.intersects(separator_start_characters)) { - record_conflict(accept_action.symbol, advance_symbol, MatchesLongerStringWithValidNextChar); + auto &following_chars = following_characters_by_token[accept_action.symbol.index]; + CharacterSet conflicting_following_chars = characters.intersection(following_chars); + CharacterSet conflicting_sep_chars = characters.intersection(separator_start_characters); + if (!conflicting_following_chars.is_empty()) { + LOG( + "%s shadows %s followed by '%s'", + token_name(advance_symbol).c_str(), + token_name(accept_action.symbol).c_str(), + log_char(*conflicting_following_chars.included_chars.begin()) + ); + record_conflict( + accept_action.symbol, + advance_symbol, + MatchesLongerStringWithValidNextChar + ); + } else if (!conflicting_sep_chars.is_empty()) { + LOG( + "%s shadows %s followed by '%s'", + token_name(advance_symbol).c_str(), + token_name(accept_action.symbol).c_str(), + log_char(*conflicting_sep_chars.included_chars.begin()) + ); + record_conflict( + accept_action.symbol, + advance_symbol, + MatchesLongerStringWithValidNextChar + ); } else { record_conflict(accept_action.symbol, advance_symbol, MatchesLongerString); } @@ -508,8 +537,22 @@ class LexTableBuilderImpl : public LexTableBuilder { main_lex_state_ids.clear(); } - const string &token_name(rules::Symbol &symbol) { - return grammar.variables[symbol.index].name; + string token_name(rules::Symbol &symbol) { + const LexicalVariable &variable = grammar.variables[symbol.index]; + if (variable.type == VariableTypeNamed) { + return variable.name; + } else { + return "'" + variable.name + "'"; + } + } + + const char *log_char(int32_t character) { + uint32_t count = utf8proc_encode_char( + character, + reinterpret_cast(encoding_buffer) + ); + encoding_buffer[count] = 0; + return encoding_buffer; } }; diff --git a/src/compiler/build_tables/parse_table_builder.cc b/src/compiler/build_tables/parse_table_builder.cc index 6218fec6..c19d5a2b 100644 --- a/src/compiler/build_tables/parse_table_builder.cc +++ b/src/compiler/build_tables/parse_table_builder.cc @@ -6,6 +6,7 @@ #include #include #include +#include "compiler/log.h" #include "compiler/parse_table.h" #include "compiler/build_tables/parse_item.h" #include "compiler/build_tables/parse_item_set_builder.h" @@ -152,8 +153,8 @@ class ParseTableBuilderImpl : public ParseTableBuilder { parse_table.states[state_id].terminal_entries.clear(); - // Add all the tokens that have no conflict with other tokens. - LookaheadSet non_conflicting_tokens; + // First, identify the conflict-free tokens. + LookaheadSet conflict_free_tokens; for (unsigned i = 0; i < lexical_grammar.variables.size(); i++) { Symbol token = Symbol::terminal(i); bool conflicts_with_other_tokens = false; @@ -166,27 +167,41 @@ class ParseTableBuilderImpl : public ParseTableBuilder { break; } } - if (!conflicts_with_other_tokens) non_conflicting_tokens.insert(token); + if (!conflicts_with_other_tokens) conflict_free_tokens.insert(token); } + // Include in the error recover state all of the tokens that are either + // conflict-free themselves, or have no conflicts with any conflict-free + // tokens. + LOG_START("finding non-conflicting tokens for error recovery"); LookaheadSet tokens; for (unsigned i = 0; i < lexical_grammar.variables.size(); i++) { Symbol token = Symbol::terminal(i); - bool conflicts_with_other_tokens = false; - if (!non_conflicting_tokens.contains(token)) { - non_conflicting_tokens.for_each([&](Symbol other_token) { + if (conflict_free_tokens.contains(token)) { + LOG("include %s", symbol_name(token).c_str()); + parse_table.add_terminal_action(state_id, token, ParseAction::Recover()); + } else { + bool conflicts_with_other_tokens = false; + conflict_free_tokens.for_each([&](Symbol other_token) { if (!coincident_tokens_by_token[token.index].contains(other_token) && (lex_table_builder->get_conflict_status(other_token, token) & CannotMerge)) { + LOG( + "exclude %s: conflicts with %s", + symbol_name(token).c_str(), + symbol_name(other_token).c_str() + ); conflicts_with_other_tokens = true; return false; } return true; }); - } - if (!conflicts_with_other_tokens) { - parse_table.add_terminal_action(state_id, token, ParseAction::Recover()); + if (!conflicts_with_other_tokens) { + LOG("include %s", symbol_name(token).c_str()); + parse_table.add_terminal_action(state_id, token, ParseAction::Recover()); + } } } + LOG_END(); for (size_t i = 0; i < grammar.external_tokens.size(); i++) { if (grammar.external_tokens[i].corresponding_internal_token == rules::NONE()) { diff --git a/src/compiler/compile.cc b/src/compiler/compile.cc index 56af3aed..49a3468b 100644 --- a/src/compiler/compile.cc +++ b/src/compiler/compile.cc @@ -3,6 +3,7 @@ #include "compiler/build_tables/parse_table_builder.h" #include "compiler/generate_code/c_code.h" #include "compiler/syntax_grammar.h" +#include "compiler/log.h" #include "compiler/lexical_grammar.h" #include "compiler/parse_grammar.h" #include "json.h" @@ -16,7 +17,9 @@ using std::vector; using std::get; using std::make_tuple; -extern "C" TSCompileResult ts_compile_grammar(const char *input) { +extern "C" TSCompileResult ts_compile_grammar(const char *input, FILE *log_file) { + set_log_file(log_file); + ParseGrammarResult parse_result = parse_grammar(string(input)); if (!parse_result.error_message.empty()) { return { nullptr, strdup(parse_result.error_message.c_str()), @@ -48,8 +51,8 @@ extern "C" TSCompileResult ts_compile_grammar(const char *input) { move(lexical_grammar) ); - return { - strdup(code.c_str()), nullptr, TSCompileErrorTypeNone }; + set_log_file(nullptr); + return { strdup(code.c_str()), nullptr, TSCompileErrorTypeNone }; } } // namespace tree_sitter diff --git a/src/compiler/log.cc b/src/compiler/log.cc new file mode 100644 index 00000000..5502c7d6 --- /dev/null +++ b/src/compiler/log.cc @@ -0,0 +1,31 @@ +#include "compiler/log.h" + +static const char *SPACES = " "; + +namespace tree_sitter { + +thread_local unsigned _indent_level = 0; +thread_local FILE *_log_file = nullptr; + +void set_log_file(FILE *file) { + _log_file = file; + _indent_level = 0; +} + +FILE *get_log_file() { + return _log_file; +} + +void _indent_logs() { + _indent_level++; +} + +void _outdent_logs() { + _indent_level--; +} + +void _print_indent() { + fwrite(SPACES, 1, _indent_level * 2, _log_file); +} + +} diff --git a/src/compiler/log.h b/src/compiler/log.h new file mode 100644 index 00000000..c848f970 --- /dev/null +++ b/src/compiler/log.h @@ -0,0 +1,38 @@ +#ifndef COMPILER_LOG_H_ +#define COMPILER_LOG_H_ + +#include + +namespace tree_sitter { + +void set_log_file(FILE *); +FILE *get_log_file(); +void _indent_logs(); +void _outdent_logs(); +void _print_indent(); + +#define LOG_START(...) \ + do { \ + LOG(__VA_ARGS__); \ + _indent_logs(); \ + } while (0) + +#define LOG_END(...) \ + do { \ + _outdent_logs(); \ + LOG(""); \ + } while (0) + +#define LOG(...) \ + do { \ + FILE *f = get_log_file(); \ + if (f) { \ + _print_indent(); \ + fprintf(f, __VA_ARGS__); \ + fputs("\n", f); \ + } \ + } while (0) + +} // namespace tree_sitter + +#endif // COMPILER_LOG_H_ diff --git a/src/compiler/rules/character_set.cc b/src/compiler/rules/character_set.cc index c199368d..b0064cbb 100644 --- a/src/compiler/rules/character_set.cc +++ b/src/compiler/rules/character_set.cc @@ -159,6 +159,11 @@ bool CharacterSet::intersects(const CharacterSet &other) const { return !copy.remove_set(other).is_empty(); } +CharacterSet CharacterSet::intersection(const CharacterSet &other) const { + CharacterSet copy(*this); + return copy.remove_set(other); +} + vector CharacterSet::included_ranges() const { return consolidate_ranges(included_chars); } diff --git a/src/compiler/rules/character_set.h b/src/compiler/rules/character_set.h index 0c991c43..c49b0d1d 100644 --- a/src/compiler/rules/character_set.h +++ b/src/compiler/rules/character_set.h @@ -35,6 +35,7 @@ struct CharacterSet { void add_set(const CharacterSet &other); CharacterSet remove_set(const CharacterSet &other); + CharacterSet intersection(const CharacterSet &other) const; bool intersects(const CharacterSet &other) const; bool is_empty() const; @@ -49,4 +50,4 @@ struct CharacterSet { } // namespace rules } // namespace tree_sitter -#endif // COMPILER_RULES_CHARACTER_SET_H_ \ No newline at end of file +#endif // COMPILER_RULES_CHARACTER_SET_H_ diff --git a/test/helpers/load_language.cc b/test/helpers/load_language.cc index f01a2184..efec4371 100644 --- a/test/helpers/load_language.cc +++ b/test/helpers/load_language.cc @@ -223,7 +223,7 @@ const TSLanguage *load_real_language(const string &language_name) { printf("\n" "Regenerating the %s parser...\n", language_name.c_str()); string grammar_json = read_file(grammar_filename); - TSCompileResult result = ts_compile_grammar(grammar_json.c_str()); + TSCompileResult result = ts_compile_grammar(grammar_json.c_str(), nullptr); if (result.error_type != TSCompileErrorTypeNone) { fprintf(stderr, "Failed to compile %s grammar: %s\n", language_name.c_str(), result.error_message); return nullptr; diff --git a/test/integration/test_grammars.cc b/test/integration/test_grammars.cc index 62174855..3741a3c9 100644 --- a/test/integration/test_grammars.cc +++ b/test/integration/test_grammars.cc @@ -27,7 +27,7 @@ for (auto &language_name : test_languages) { if (file_exists(expected_error_path)) { it("fails with the correct error message", [&]() { - TSCompileResult compile_result = ts_compile_grammar(grammar_json.c_str()); + TSCompileResult compile_result = ts_compile_grammar(grammar_json.c_str(), nullptr); string expected_error = read_file(expected_error_path); AssertThat((void *)compile_result.error_message, !Equals(nullptr)); AssertThat(compile_result.error_message, Equals(expected_error)); @@ -43,7 +43,7 @@ for (auto &language_name : test_languages) { string external_scanner_path = join_path({directory_path, "scanner.c"}); if (!file_exists(external_scanner_path)) external_scanner_path = ""; - TSCompileResult compile_result = ts_compile_grammar(grammar_json.c_str()); + TSCompileResult compile_result = ts_compile_grammar(grammar_json.c_str(), nullptr); language = load_test_language( language_name, diff --git a/test/runtime/language_test.cc b/test/runtime/language_test.cc index 747327c0..1c460347 100644 --- a/test/runtime/language_test.cc +++ b/test/runtime/language_test.cc @@ -26,7 +26,7 @@ describe("Language", []() { "value": "b" } } - })JSON"); + })JSON", nullptr); TSParser *parser = ts_parser_new(); const TSLanguage *language = load_test_language("aliased_rules", compile_result); diff --git a/test/runtime/node_test.cc b/test/runtime/node_test.cc index 313bec40..8683a9ef 100644 --- a/test/runtime/node_test.cc +++ b/test/runtime/node_test.cc @@ -71,7 +71,7 @@ string grammar_with_aliases_and_extras = R"JSON({ const TSLanguage *language_with_aliases_and_extras = load_test_language( "aliases_and_extras", - ts_compile_grammar(grammar_with_aliases_and_extras.c_str()) + ts_compile_grammar(grammar_with_aliases_and_extras.c_str(), nullptr) ); describe("Node", [&]() { From 915978aa9d3a3717b517f21074c0cb4e461aa057 Mon Sep 17 00:00:00 2001 From: Max Brunsfeld Date: Thu, 24 May 2018 16:22:16 -0700 Subject: [PATCH 77/90] Avoid redundant logging of conflicting tokens --- .../build_tables/lex_table_builder.cc | 38 ++++++++++--------- 1 file changed, 21 insertions(+), 17 deletions(-) diff --git a/src/compiler/build_tables/lex_table_builder.cc b/src/compiler/build_tables/lex_table_builder.cc index 54957efe..001dbc81 100644 --- a/src/compiler/build_tables/lex_table_builder.cc +++ b/src/compiler/build_tables/lex_table_builder.cc @@ -265,9 +265,11 @@ class LexTableBuilderImpl : public LexTableBuilder { } private: - void record_conflict(Symbol shadowed_token, Symbol other_token, ConflictStatus status) { + bool record_conflict(Symbol shadowed_token, Symbol other_token, ConflictStatus status) { unsigned index = shadowed_token.index * grammar.variables.size() + other_token.index; + bool old_value = conflict_matrix[index] & status; conflict_matrix[index] = static_cast(conflict_matrix[index] | status); + return old_value; } LexStateId add_lex_state(LexTable &lex_table, const LexItemSet &item_set) { @@ -313,29 +315,31 @@ class LexTableBuilderImpl : public LexTableBuilder { CharacterSet conflicting_following_chars = characters.intersection(following_chars); CharacterSet conflicting_sep_chars = characters.intersection(separator_start_characters); if (!conflicting_following_chars.is_empty()) { - LOG( - "%s shadows %s followed by '%s'", - token_name(advance_symbol).c_str(), - token_name(accept_action.symbol).c_str(), - log_char(*conflicting_following_chars.included_chars.begin()) - ); - record_conflict( + if (record_conflict( accept_action.symbol, advance_symbol, MatchesLongerStringWithValidNextChar - ); + )) { + LOG( + "%s shadows %s followed by '%s'", + token_name(advance_symbol).c_str(), + token_name(accept_action.symbol).c_str(), + log_char(*conflicting_following_chars.included_chars.begin()) + ); + } } else if (!conflicting_sep_chars.is_empty()) { - LOG( - "%s shadows %s followed by '%s'", - token_name(advance_symbol).c_str(), - token_name(accept_action.symbol).c_str(), - log_char(*conflicting_sep_chars.included_chars.begin()) - ); - record_conflict( + if (record_conflict( accept_action.symbol, advance_symbol, MatchesLongerStringWithValidNextChar - ); + )) { + LOG( + "%s shadows %s followed by '%s'", + token_name(advance_symbol).c_str(), + token_name(accept_action.symbol).c_str(), + log_char(*conflicting_sep_chars.included_chars.begin()) + ); + } } else { record_conflict(accept_action.symbol, advance_symbol, MatchesLongerString); } From 406a85a1662e924c8c4612c35f5cde9326874a49 Mon Sep 17 00:00:00 2001 From: Max Brunsfeld Date: Thu, 24 May 2018 16:46:43 -0700 Subject: [PATCH 78/90] Log when lexical conflicts prevents parse state merging --- src/compiler/build_tables/parse_table_builder.cc | 8 ++++++++ src/compiler/log.cc | 2 +- src/compiler/log.h | 5 ++++- 3 files changed, 13 insertions(+), 2 deletions(-) diff --git a/src/compiler/build_tables/parse_table_builder.cc b/src/compiler/build_tables/parse_table_builder.cc index c19d5a2b..127b866c 100644 --- a/src/compiler/build_tables/parse_table_builder.cc +++ b/src/compiler/build_tables/parse_table_builder.cc @@ -53,6 +53,7 @@ class ParseTableBuilderImpl : public ParseTableBuilder { unique_ptr lex_table_builder; unordered_map following_tokens_by_token; vector coincident_tokens_by_token; + set> logged_conflict_tokens; public: ParseTableBuilderImpl(const SyntaxGrammar &syntax_grammar, const LexicalGrammar &lexical_grammar) @@ -367,6 +368,7 @@ class ParseTableBuilderImpl : public ParseTableBuilder { } void remove_duplicate_parse_states() { + LOG_START("removing duplicate parse states"); unordered_map> state_indices_by_signature; for (auto &pair : state_ids_by_item_set) { @@ -528,6 +530,12 @@ class ParseTableBuilderImpl : public ParseTableBuilder { if (!new_token.is_built_in()) { for (const auto &entry : state.terminal_entries) { if (lex_table_builder->get_conflict_status(entry.first, new_token) & CannotDistinguish) { + LOG_IF( + logged_conflict_tokens.insert({entry.first, new_token}).second, + "cannot merge parse states due to token conflict: %s and %s", + symbol_name(entry.first).c_str(), + symbol_name(new_token).c_str() + ); return false; } } diff --git a/src/compiler/log.cc b/src/compiler/log.cc index 5502c7d6..c0f3a03c 100644 --- a/src/compiler/log.cc +++ b/src/compiler/log.cc @@ -25,7 +25,7 @@ void _outdent_logs() { } void _print_indent() { - fwrite(SPACES, 1, _indent_level * 2, _log_file); + fwrite(SPACES, 1, _indent_level * 4, _log_file); } } diff --git a/src/compiler/log.h b/src/compiler/log.h index c848f970..d781cb41 100644 --- a/src/compiler/log.h +++ b/src/compiler/log.h @@ -24,9 +24,12 @@ void _print_indent(); } while (0) #define LOG(...) \ + LOG_IF(true, __VA_ARGS__) + +#define LOG_IF(condition, ...) \ do { \ FILE *f = get_log_file(); \ - if (f) { \ + if (f && condition) { \ _print_indent(); \ fprintf(f, __VA_ARGS__); \ fputs("\n", f); \ From 356d5e02218db3b3951f6f979edf47c263c0911b Mon Sep 17 00:00:00 2001 From: Max Brunsfeld Date: Fri, 25 May 2018 15:29:15 -0700 Subject: [PATCH 79/90] Generalize logic for finding a keyword capture token --- .../build_tables/lex_table_builder.cc | 268 +++++++++++------- src/compiler/build_tables/lex_table_builder.h | 30 +- src/compiler/build_tables/lookahead_set.cc | 26 ++ src/compiler/build_tables/lookahead_set.h | 2 + .../build_tables/parse_table_builder.cc | 34 +-- src/compiler/parse_table.cc | 4 + src/compiler/parse_table.h | 1 + 7 files changed, 244 insertions(+), 121 deletions(-) diff --git a/src/compiler/build_tables/lex_table_builder.cc b/src/compiler/build_tables/lex_table_builder.cc index 001dbc81..4a507b81 100644 --- a/src/compiler/build_tables/lex_table_builder.cc +++ b/src/compiler/build_tables/lex_table_builder.cc @@ -15,6 +15,19 @@ #include "compiler/rule.h" #include "utf8proc.h" +namespace std { + +using tree_sitter::rules::Symbol; + +size_t hash>::operator()( + const pair &p +) const { + hash hasher; + return hasher(p.first) ^ hasher(p.second); +} + +} // namespace std + namespace tree_sitter { namespace build_tables { @@ -36,8 +49,24 @@ using rules::Symbol; using rules::Metadata; using rules::Seq; +static const std::unordered_set EMPTY; + +bool CoincidentTokenIndex::contains(Symbol a, Symbol b) const { + return a == b || !states_with(a, b).empty(); +} + +const std::unordered_set &CoincidentTokenIndex::states_with(Symbol a, Symbol b) const { + if (a.index > b.index) std::swap(a, b); + auto iter = entries.find({a.index, b.index}); + if (iter == entries.end()) { + return EMPTY; + } else { + return iter->second; + } +} + template -class StartOrEndCharacterAggregator { +class CharacterAggregator { public: void apply(const Rule &rule) { rule.match( @@ -62,8 +91,8 @@ class StartOrEndCharacterAggregator { CharacterSet result; }; -using StartingCharacterAggregator = StartOrEndCharacterAggregator; -using AllCharacterAggregator = StartOrEndCharacterAggregator; +using StartingCharacterAggregator = CharacterAggregator; +using AllCharacterAggregator = CharacterAggregator; class LexTableBuilderImpl : public LexTableBuilder { LexTable main_lex_table; @@ -75,7 +104,8 @@ class LexTableBuilderImpl : public LexTableBuilder { CharacterSet separator_start_characters; vector starting_characters_by_token; vector following_characters_by_token; - const vector &coincident_tokens_by_token; + const CoincidentTokenIndex &coincident_token_index; + ParseTable *parse_table; vector conflict_matrix; bool conflict_detection_mode; LookaheadSet keyword_symbols; @@ -86,11 +116,13 @@ class LexTableBuilderImpl : public LexTableBuilder { LexTableBuilderImpl(const SyntaxGrammar &syntax_grammar, const LexicalGrammar &lexical_grammar, const unordered_map &following_tokens_by_token, - const vector &coincident_tokens) + const CoincidentTokenIndex &coincident_token_index, + ParseTable *parse_table) : grammar(lexical_grammar), starting_characters_by_token(lexical_grammar.variables.size()), following_characters_by_token(lexical_grammar.variables.size()), - coincident_tokens_by_token(coincident_tokens), + coincident_token_index(coincident_token_index), + parse_table(parse_table), conflict_matrix(lexical_grammar.variables.size() * lexical_grammar.variables.size(), DoesNotMatch), conflict_detection_mode(false), keyword_capture_token(rules::NONE()) { @@ -106,51 +138,41 @@ class LexTableBuilderImpl : public LexTableBuilder { separator_start_characters = separator_character_aggregator.result; // Compute the set of characters that each token can start with and the set of non-separator - // characters that can follow each token. + // characters that can follow each token. Also identify all of the tokens that consist + // entirely of letters, and can be considered 'keywords'. + LOG_START("characterizing tokens"); + LookaheadSet potential_keyword_symbols; for (unsigned i = 0, n = grammar.variables.size(); i < n; i++) { + Symbol token = Symbol::terminal(i); + StartingCharacterAggregator starting_character_aggregator; starting_character_aggregator.apply(grammar.variables[i].rule); starting_characters_by_token[i] = starting_character_aggregator.result; StartingCharacterAggregator following_character_aggregator; - const auto &following_tokens = following_tokens_by_token.find(Symbol::terminal(i)); + const auto &following_tokens = following_tokens_by_token.find(token); if (following_tokens != following_tokens_by_token.end()) { following_tokens->second.for_each([&](Symbol following_token) { following_character_aggregator.apply(grammar.variables[following_token.index].rule); return true; }); } + following_characters_by_token[i] = following_character_aggregator.result; - if (grammar.variables[i].is_string) { - AllCharacterAggregator aggregator; - aggregator.apply(grammar.variables[i].rule); - bool all_alpha = true, all_lower = true; - for (auto character : aggregator.result.included_chars) { - if (!iswalpha(character) && character != '_') all_alpha = false; - if (!iswlower(character)) all_lower = false; - } - - if (all_lower) { - keyword_symbols.insert(Symbol::terminal(i)); - } - - // TODO - Refactor this. In general, a keyword token cannot be followed immediately - // by another alphanumeric character. But this requirement is currently not expressed - // anywhere in the grammar. So without this hack, we would be overly conservative about - // merging parse states because we would often consider `identifier` tokens to *conflict* - // with keyword tokens. - if (all_alpha) { - following_character_aggregator.result - .exclude('a', 'z') - .exclude('A', 'Z') - .exclude('0', '9') - .exclude('_') - .exclude('$'); + AllCharacterAggregator aggregator; + aggregator.apply(grammar.variables[i].rule); + bool all_alpha = true; + for (auto character : aggregator.result.included_chars) { + if (!iswalpha(character) && character != '_') { + all_alpha = false; } } - - following_characters_by_token[i] = following_character_aggregator.result; + if (all_alpha) { + LOG("potential keyword: %s", token_name(token).c_str()); + potential_keyword_symbols.insert(token); + } } + LOG_END(); // For each pair of tokens, generate a lex table for just those two tokens and record what // conflicts arise. @@ -171,50 +193,102 @@ class LexTableBuilderImpl : public LexTableBuilder { } LOG_END(); - // Find a 'keyword capture token' that matches all of the indentified keywords. + LOG_START("finding keyword capture token"); for (Symbol::Index i = 0, n = grammar.variables.size(); i < n; i++) { - Symbol symbol = Symbol::terminal(i); - bool matches_all_keywords = true; - keyword_symbols.for_each([&](Symbol keyword_symbol) { - if (!(get_conflict_status(symbol, keyword_symbol) & MatchesSameString)) { - matches_all_keywords = false; + Symbol candidate = Symbol::terminal(i); + + LookaheadSet homonyms; + potential_keyword_symbols.for_each([&](Symbol other_token) { + if (get_conflict_status(other_token, candidate) & MatchesShorterStringWithinSeparators) { + homonyms.clear(); return false; } + if (get_conflict_status(candidate, other_token) == MatchesSameString) { + homonyms.insert(other_token); + } return true; }); - if (!matches_all_keywords) continue; + if (homonyms.empty()) continue; - // Don't use a token to capture keywords if it overlaps with separator characters. - AllCharacterAggregator capture_aggregator; - capture_aggregator.apply(grammar.variables[i].rule); - if (capture_aggregator.result.intersects(separator_start_characters)) continue; + LOG_START( + "keyword capture token candidate: %s, homonym count: %lu", + token_name(candidate).c_str(), + homonyms.size() + ); + + homonyms.for_each([&](Symbol homonym1) { + homonyms.for_each([&](Symbol homonym2) { + if (get_conflict_status(homonym1, homonym2) & MatchesSameString) { + LOG( + "conflict between homonyms %s %s", + token_name(homonym1).c_str(), + token_name(homonym2).c_str() + ); + homonyms.remove(homonym1); + } + return false; + }); + return true; + }); - // Don't use a token to capture keywords if it conflicts with other tokens - // that occur in the same state as a keyword. - bool shadows_other_tokens = false; for (Symbol::Index j = 0; j < n; j++) { - Symbol other_symbol = Symbol::terminal(j); - if ((get_conflict_status(other_symbol, symbol) & (MatchesShorterStringWithinSeparators|MatchesLongerStringWithValidNextChar)) && - !keyword_symbols.contains(other_symbol) && - keyword_symbols.intersects(coincident_tokens_by_token[j])) { - shadows_other_tokens = true; - break; + Symbol other_token = Symbol::terminal(j); + if (other_token == candidate || homonyms.contains(other_token)) continue; + bool candidate_shadows_other = get_conflict_status(other_token, candidate); + bool other_shadows_candidate = get_conflict_status(candidate, other_token); + + if (candidate_shadows_other || other_shadows_candidate) { + homonyms.for_each([&](Symbol homonym) { + bool other_shadows_homonym = get_conflict_status(homonym, other_token); + + bool candidate_was_already_present = true; + for (ParseStateId state_id : coincident_token_index.states_with(homonym, other_token)) { + if (!parse_table->states[state_id].has_terminal_entry(candidate)) { + candidate_was_already_present = false; + break; + } + } + + if (!candidate_was_already_present) { + if (candidate_shadows_other) { + homonyms.remove(homonym); + LOG( + "remove %s because candidate would shadow %s", + token_name(homonym).c_str(), + token_name(other_token).c_str() + ); + } else if (other_shadows_candidate != other_shadows_homonym) { + homonyms.remove(homonym); + LOG( + "remove %s because %s would shadow candidate", + token_name(homonym).c_str(), + token_name(other_token).c_str() + ); + } + } + return true; + }); } } - if (shadows_other_tokens) continue; - // If multiple keyword capture tokens are found, don't bother extracting - // the keywords into their own function. - if (keyword_capture_token == rules::NONE()) { - keyword_capture_token = symbol; - } else { - keyword_capture_token = rules::NONE(); - break; + if (homonyms.size() > keyword_symbols.size()) { + LOG_START("found capture token. homonyms:"); + homonyms.for_each([&](Symbol homonym) { + LOG("%s", token_name(homonym).c_str()); + return true; + }); + LOG_END(); + keyword_symbols = homonyms; + keyword_capture_token = candidate; } + + LOG_END(); } + + LOG_END(); } - BuildResult build(ParseTable *parse_table) { + BuildResult build() { clear(); conflict_detection_mode = false; vector>> starting_token_sets; @@ -250,8 +324,8 @@ class LexTableBuilderImpl : public LexTableBuilder { add_lex_state(keyword_lex_table, item_set_for_terminals(keyword_symbols, false)); - mark_fragile_tokens(parse_table); - remove_duplicate_lex_states(main_lex_table, parse_table); + mark_fragile_tokens(); + remove_duplicate_lex_states(main_lex_table); return {main_lex_table, keyword_lex_table, keyword_capture_token}; } @@ -266,10 +340,11 @@ class LexTableBuilderImpl : public LexTableBuilder { private: bool record_conflict(Symbol shadowed_token, Symbol other_token, ConflictStatus status) { + if (!conflict_detection_mode) return false; unsigned index = shadowed_token.index * grammar.variables.size() + other_token.index; - bool old_value = conflict_matrix[index] & status; + bool was_set = conflict_matrix[index] & status; conflict_matrix[index] = static_cast(conflict_matrix[index] | status); - return old_value; + return !was_set; } LexStateId add_lex_state(LexTable &lex_table, const LexItemSet &item_set) { @@ -313,8 +388,12 @@ class LexTableBuilderImpl : public LexTableBuilder { auto advance_symbol = transition.destination.entries.begin()->lhs; auto &following_chars = following_characters_by_token[accept_action.symbol.index]; CharacterSet conflicting_following_chars = characters.intersection(following_chars); - CharacterSet conflicting_sep_chars = characters.intersection(separator_start_characters); - if (!conflicting_following_chars.is_empty()) { + if (conflicting_following_chars.is_empty()) { + conflicting_following_chars = characters.intersection(separator_start_characters); + } + if (conflicting_following_chars.is_empty()) { + record_conflict(accept_action.symbol, advance_symbol, MatchesLongerString); + } else { if (record_conflict( accept_action.symbol, advance_symbol, @@ -327,21 +406,6 @@ class LexTableBuilderImpl : public LexTableBuilder { log_char(*conflicting_following_chars.included_chars.begin()) ); } - } else if (!conflicting_sep_chars.is_empty()) { - if (record_conflict( - accept_action.symbol, - advance_symbol, - MatchesLongerStringWithValidNextChar - )) { - LOG( - "%s shadows %s followed by '%s'", - token_name(advance_symbol).c_str(), - token_name(accept_action.symbol).c_str(), - log_char(*conflicting_sep_chars.included_chars.begin()) - ); - } - } else { - record_conflict(accept_action.symbol, advance_symbol, MatchesLongerString); } } } @@ -364,9 +428,21 @@ class LexTableBuilderImpl : public LexTableBuilder { AcceptTokenAction &existing_action = lex_table.states[state_id].accept_action; if (existing_action.is_present()) { if (should_replace_accept_action(existing_action, action)) { - record_conflict(existing_action.symbol, action.symbol, MatchesSameString); + if (record_conflict(existing_action.symbol, action.symbol, MatchesSameString)) { + LOG( + "%s shadows %s - same length", + token_name(action.symbol).c_str(), + token_name(existing_action.symbol).c_str() + ); + } } else { - record_conflict(action.symbol, existing_action.symbol, MatchesSameString); + if (record_conflict(action.symbol, existing_action.symbol, MatchesSameString)) { + LOG( + "%s shadows %s - same length", + token_name(existing_action.symbol).c_str(), + token_name(action.symbol).c_str() + ); + } continue; } } @@ -375,7 +451,7 @@ class LexTableBuilderImpl : public LexTableBuilder { } } - void mark_fragile_tokens(ParseTable *parse_table) { + void mark_fragile_tokens() { for (ParseState &state : parse_table->states) { for (auto &entry : state.terminal_entries) { Symbol token = entry.first; @@ -401,7 +477,7 @@ class LexTableBuilderImpl : public LexTableBuilder { const LookaheadSet &existing_set = in_left ? right : *left; existing_set.for_each([&](Symbol existing_symbol) { if ((get_conflict_status(existing_symbol, different_symbol) & CannotDistinguish) || - !coincident_tokens_by_token[different_symbol.index].contains(existing_symbol)) { + !coincident_token_index.contains(different_symbol, existing_symbol)) { is_compatible = false; return false; } @@ -417,7 +493,7 @@ class LexTableBuilderImpl : public LexTableBuilder { return is_compatible; } - void remove_duplicate_lex_states(LexTable &lex_table, ParseTable *parse_table) { + void remove_duplicate_lex_states(LexTable &lex_table) { for (LexState &state : lex_table.states) { state.accept_action.is_string = false; state.accept_action.precedence = 0; @@ -541,7 +617,7 @@ class LexTableBuilderImpl : public LexTableBuilder { main_lex_state_ids.clear(); } - string token_name(rules::Symbol &symbol) { + string token_name(const rules::Symbol &symbol) { const LexicalVariable &variable = grammar.variables[symbol.index]; if (variable.type == VariableTypeNamed) { return variable.name; @@ -563,17 +639,19 @@ class LexTableBuilderImpl : public LexTableBuilder { unique_ptr LexTableBuilder::create(const SyntaxGrammar &syntax_grammar, const LexicalGrammar &lexical_grammar, const unordered_map &following_tokens, - const vector &coincident_tokens) { + const CoincidentTokenIndex &coincident_tokens, + ParseTable *parse_table) { return unique_ptr(new LexTableBuilderImpl( syntax_grammar, lexical_grammar, following_tokens, - coincident_tokens + coincident_tokens, + parse_table )); } -LexTableBuilder::BuildResult LexTableBuilder::build(ParseTable *parse_table) { - return static_cast(this)->build(parse_table); +LexTableBuilder::BuildResult LexTableBuilder::build() { + return static_cast(this)->build(); } ConflictStatus LexTableBuilder::get_conflict_status(Symbol a, Symbol b) const { diff --git a/src/compiler/build_tables/lex_table_builder.h b/src/compiler/build_tables/lex_table_builder.h index 2a1051aa..4ec4f22b 100644 --- a/src/compiler/build_tables/lex_table_builder.h +++ b/src/compiler/build_tables/lex_table_builder.h @@ -4,9 +4,22 @@ #include #include #include -#include +#include +#include +#include "compiler/parse_table.h" #include "compiler/lex_table.h" +namespace std { + +using tree_sitter::rules::Symbol; + +template <> +struct hash> { + size_t operator()(const pair &) const; +}; + +} // namespace std + namespace tree_sitter { struct ParseTable; @@ -30,12 +43,23 @@ enum ConflictStatus { ), }; +struct CoincidentTokenIndex { + std::unordered_map< + std::pair, + std::unordered_set + > entries; + + bool contains(rules::Symbol, rules::Symbol) const; + const std::unordered_set &states_with(rules::Symbol, rules::Symbol) const; +}; + class LexTableBuilder { public: static std::unique_ptr create(const SyntaxGrammar &, const LexicalGrammar &, const std::unordered_map &, - const std::vector &); + const CoincidentTokenIndex &, + ParseTable *); struct BuildResult { LexTable main_table; @@ -43,7 +67,7 @@ class LexTableBuilder { rules::Symbol keyword_capture_token; }; - BuildResult build(ParseTable *); + BuildResult build(); ConflictStatus get_conflict_status(rules::Symbol, rules::Symbol) const; diff --git a/src/compiler/build_tables/lookahead_set.cc b/src/compiler/build_tables/lookahead_set.cc index 80ec58e1..6e5f73b5 100644 --- a/src/compiler/build_tables/lookahead_set.cc +++ b/src/compiler/build_tables/lookahead_set.cc @@ -117,5 +117,31 @@ bool LookaheadSet::insert(const Symbol &symbol) { return false; } +bool LookaheadSet::remove(const Symbol &symbol) { + if (symbol == rules::END_OF_INPUT()) { + if (eof) { + eof = false; + return true; + } + return false; + } + + auto &bits = symbol.is_external() ? external_bits : terminal_bits; + if (bits.size() > static_cast(symbol.index)) { + if (bits[symbol.index]) { + bits[symbol.index] = false; + return true; + } + } + + return false; +} + +void LookaheadSet::clear() { + eof = false; + terminal_bits.clear(); + external_bits.clear(); +} + } // namespace build_tables } // namespace tree_sitter diff --git a/src/compiler/build_tables/lookahead_set.h b/src/compiler/build_tables/lookahead_set.h index bb9eeff9..6445969d 100644 --- a/src/compiler/build_tables/lookahead_set.h +++ b/src/compiler/build_tables/lookahead_set.h @@ -22,6 +22,8 @@ class LookaheadSet { bool contains(const rules::Symbol &) const; bool insert_all(const LookaheadSet &); bool insert(const rules::Symbol &); + bool remove(const rules::Symbol &); + void clear(); bool intersects(const LookaheadSet &) const; template diff --git a/src/compiler/build_tables/parse_table_builder.cc b/src/compiler/build_tables/parse_table_builder.cc index 127b866c..0e6b4247 100644 --- a/src/compiler/build_tables/parse_table_builder.cc +++ b/src/compiler/build_tables/parse_table_builder.cc @@ -52,28 +52,14 @@ class ParseTableBuilderImpl : public ParseTableBuilder { ParseItemSetBuilder item_set_builder; unique_ptr lex_table_builder; unordered_map following_tokens_by_token; - vector coincident_tokens_by_token; + CoincidentTokenIndex coincident_token_index; set> logged_conflict_tokens; public: ParseTableBuilderImpl(const SyntaxGrammar &syntax_grammar, const LexicalGrammar &lexical_grammar) : grammar(syntax_grammar), lexical_grammar(lexical_grammar), - item_set_builder(syntax_grammar, lexical_grammar), - coincident_tokens_by_token(lexical_grammar.variables.size()) { - - for (unsigned i = 0, n = lexical_grammar.variables.size(); i < n; i++) { - coincident_tokens_by_token[i].insert(rules::END_OF_INPUT()); - if (lexical_grammar.variables[i].is_string) { - for (unsigned j = 0; j < i; j++) { - if (lexical_grammar.variables[j].is_string) { - coincident_tokens_by_token[i].insert(Symbol::terminal(j)); - coincident_tokens_by_token[j].insert(Symbol::terminal(i)); - } - } - } - } - } + item_set_builder(syntax_grammar, lexical_grammar) {} BuildResult build() { // Ensure that the empty rename sequence has index 0. @@ -106,7 +92,8 @@ class ParseTableBuilderImpl : public ParseTableBuilder { grammar, lexical_grammar, following_tokens_by_token, - coincident_tokens_by_token + coincident_token_index, + &parse_table ); build_error_parse_state(error_state_id); @@ -115,7 +102,7 @@ class ParseTableBuilderImpl : public ParseTableBuilder { eliminate_unit_reductions(); populate_used_terminals(); - auto lex_table_result = lex_table_builder->build(&parse_table); + auto lex_table_result = lex_table_builder->build(); return { parse_table, lex_table_result.main_table, @@ -161,8 +148,7 @@ class ParseTableBuilderImpl : public ParseTableBuilder { bool conflicts_with_other_tokens = false; for (unsigned j = 0; j < lexical_grammar.variables.size(); j++) { Symbol other_token = Symbol::terminal(j); - if (j != i && - !coincident_tokens_by_token[token.index].contains(other_token) && + if (!coincident_token_index.contains(token, other_token) && (lex_table_builder->get_conflict_status(other_token, token) & CannotMerge)) { conflicts_with_other_tokens = true; break; @@ -184,7 +170,7 @@ class ParseTableBuilderImpl : public ParseTableBuilder { } else { bool conflicts_with_other_tokens = false; conflict_free_tokens.for_each([&](Symbol other_token) { - if (!coincident_tokens_by_token[token.index].contains(other_token) && + if (!coincident_token_index.contains(token, other_token) && (lex_table_builder->get_conflict_status(other_token, token) & CannotMerge)) { LOG( "exclude %s: conflicts with %s", @@ -332,8 +318,10 @@ class ParseTableBuilderImpl : public ParseTableBuilder { if (iter->first.is_built_in() || iter->first.is_external()) continue; for (auto other_iter = terminals.begin(); other_iter != iter; ++other_iter) { if (other_iter->first.is_built_in() || other_iter->first.is_external()) continue; - coincident_tokens_by_token[iter->first.index].insert(other_iter->first); - coincident_tokens_by_token[other_iter->first.index].insert(iter->first); + coincident_token_index.entries[{ + other_iter->first.index, + iter->first.index + }].insert(state_id); } } diff --git a/src/compiler/parse_table.cc b/src/compiler/parse_table.cc index ed41d473..252185f4 100644 --- a/src/compiler/parse_table.cc +++ b/src/compiler/parse_table.cc @@ -123,6 +123,10 @@ bool ParseState::has_shift_action() const { return (!nonterminal_entries.empty()); } +bool ParseState::has_terminal_entry(rules::Symbol symbol) const { + return terminal_entries.find(symbol) != terminal_entries.end(); +} + void ParseState::each_referenced_state(function fn) { for (auto &entry : terminal_entries) for (ParseAction &action : entry.second.actions) diff --git a/src/compiler/parse_table.h b/src/compiler/parse_table.h index 770deafb..bf85c4b7 100644 --- a/src/compiler/parse_table.h +++ b/src/compiler/parse_table.h @@ -65,6 +65,7 @@ struct ParseState { bool merge(const ParseState &); void each_referenced_state(std::function); bool has_shift_action() const; + bool has_terminal_entry(rules::Symbol) const; std::map terminal_entries; std::map nonterminal_entries; From 45c52f94596ac5b590c3299c86e26378edb3157f Mon Sep 17 00:00:00 2001 From: Max Brunsfeld Date: Fri, 25 May 2018 21:24:53 -0700 Subject: [PATCH 80/90] Allow keywords to contain numbers, as long as they start w/ a letter --- .../build_tables/lex_table_builder.cc | 67 +++++++++++-------- 1 file changed, 39 insertions(+), 28 deletions(-) diff --git a/src/compiler/build_tables/lex_table_builder.cc b/src/compiler/build_tables/lex_table_builder.cc index 4a507b81..178cfb75 100644 --- a/src/compiler/build_tables/lex_table_builder.cc +++ b/src/compiler/build_tables/lex_table_builder.cc @@ -138,8 +138,8 @@ class LexTableBuilderImpl : public LexTableBuilder { separator_start_characters = separator_character_aggregator.result; // Compute the set of characters that each token can start with and the set of non-separator - // characters that can follow each token. Also identify all of the tokens that consist - // entirely of letters, and can be considered 'keywords'. + // characters that can follow each token. Also identify all of the tokens that can be + // considered 'keywords'. LOG_START("characterizing tokens"); LookaheadSet potential_keyword_symbols; for (unsigned i = 0, n = grammar.variables.size(); i < n; i++) { @@ -159,18 +159,30 @@ class LexTableBuilderImpl : public LexTableBuilder { } following_characters_by_token[i] = following_character_aggregator.result; - AllCharacterAggregator aggregator; - aggregator.apply(grammar.variables[i].rule); - bool all_alpha = true; - for (auto character : aggregator.result.included_chars) { - if (!iswalpha(character) && character != '_') { - all_alpha = false; + AllCharacterAggregator all_character_aggregator; + all_character_aggregator.apply(grammar.variables[i].rule); + + if ( + !starting_character_aggregator.result.includes_all && + !all_character_aggregator.result.includes_all + ) { + bool starts_alpha = true, all_alnum = true; + for (auto character : starting_character_aggregator.result.included_chars) { + if (!iswalpha(character) && character != '_') { + starts_alpha = false; + } + } + for (auto character : all_character_aggregator.result.included_chars) { + if (!iswalnum(character) && character != '_') { + all_alnum = false; + } + } + if (starts_alpha && all_alnum) { + LOG("potential keyword: %s", token_name(token).c_str()); + potential_keyword_symbols.insert(token); } } - if (all_alpha) { - LOG("potential keyword: %s", token_name(token).c_str()); - potential_keyword_symbols.insert(token); - } + } LOG_END(); @@ -248,23 +260,22 @@ class LexTableBuilderImpl : public LexTableBuilder { break; } } + if (candidate_was_already_present) return true; - if (!candidate_was_already_present) { - if (candidate_shadows_other) { - homonyms.remove(homonym); - LOG( - "remove %s because candidate would shadow %s", - token_name(homonym).c_str(), - token_name(other_token).c_str() - ); - } else if (other_shadows_candidate != other_shadows_homonym) { - homonyms.remove(homonym); - LOG( - "remove %s because %s would shadow candidate", - token_name(homonym).c_str(), - token_name(other_token).c_str() - ); - } + if (candidate_shadows_other) { + homonyms.remove(homonym); + LOG( + "remove %s because candidate would shadow %s", + token_name(homonym).c_str(), + token_name(other_token).c_str() + ); + } else if (other_shadows_candidate && !other_shadows_homonym) { + homonyms.remove(homonym); + LOG( + "remove %s because %s would shadow candidate", + token_name(homonym).c_str(), + token_name(other_token).c_str() + ); } return true; }); From 8120e61d8d9035cf4d9c0a53c8127f50bf82c085 Mon Sep 17 00:00:00 2001 From: Max Brunsfeld Date: Fri, 25 May 2018 21:37:25 -0700 Subject: [PATCH 81/90] Remove blank lines from log messages --- src/compiler/log.h | 1 - 1 file changed, 1 deletion(-) diff --git a/src/compiler/log.h b/src/compiler/log.h index d781cb41..2f7ad3e2 100644 --- a/src/compiler/log.h +++ b/src/compiler/log.h @@ -20,7 +20,6 @@ void _print_indent(); #define LOG_END(...) \ do { \ _outdent_logs(); \ - LOG(""); \ } while (0) #define LOG(...) \ From f3014cb7678bf3b95cbe4140253858295cdaa706 Mon Sep 17 00:00:00 2001 From: Max Brunsfeld Date: Tue, 29 May 2018 16:00:32 -0700 Subject: [PATCH 82/90] Cache parent nodes to speed up ts_node_parent in common cases --- src/runtime/node.c | 37 ++++++++++++++++++++++------------- src/runtime/tree.c | 41 +++++++++++++++++++++++++++++++++++++++ src/runtime/tree.h | 12 ++++++++++++ test/runtime/node_test.cc | 11 +++++++++++ 4 files changed, 87 insertions(+), 14 deletions(-) diff --git a/src/runtime/node.c b/src/runtime/node.c index 3c858b92..607cf9de 100644 --- a/src/runtime/node.c +++ b/src/runtime/node.c @@ -125,6 +125,7 @@ static inline uint32_t ts_node__relevant_child_count(TSNode self, bool include_a } static inline TSNode ts_node__child(TSNode self, uint32_t child_index, bool include_anonymous) { + const TSTree *tree = ts_node__tree(&self); TSNode result = self; bool did_descend = true; @@ -136,7 +137,10 @@ static inline TSNode ts_node__child(TSNode self, uint32_t child_index, bool incl ChildIterator iterator = ts_node_iterate_children(&result); while (ts_node_child_iterator_next(&iterator, &child)) { if (ts_node__is_relevant(child, include_anonymous)) { - if (index == child_index) return child; + if (index == child_index) { + ts_tree_set_cached_parent(tree, &child, &self); + return child; + } index++; } else { uint32_t grandchild_index = child_index - index; @@ -290,6 +294,7 @@ static inline TSNode ts_node__descendant_for_byte_range(TSNode self, uint32_t mi bool include_anonymous) { TSNode node = self; TSNode last_visible_node = self; + const TSTree *tree = ts_node__tree(&self); bool did_descend = true; while (did_descend) { @@ -301,7 +306,10 @@ static inline TSNode ts_node__descendant_for_byte_range(TSNode self, uint32_t mi if (iterator.position.bytes > max) { if (ts_node_start_byte(child) > min) break; node = child; - if (ts_node__is_relevant(node, include_anonymous)) last_visible_node = node; + if (ts_node__is_relevant(node, include_anonymous)) { + ts_tree_set_cached_parent(tree, &child, &last_visible_node); + last_visible_node = node; + } did_descend = true; break; } @@ -316,8 +324,7 @@ static inline TSNode ts_node__descendant_for_point_range(TSNode self, TSPoint mi bool include_anonymous) { TSNode node = self; TSNode last_visible_node = self; - TSPoint start_position = ts_node_start_point(self); - TSPoint end_position = ts_node_end_point(self); + const TSTree *tree = ts_node__tree(&self); bool did_descend = true; while (did_descend) { @@ -326,19 +333,16 @@ static inline TSNode ts_node__descendant_for_point_range(TSNode self, TSPoint mi TSNode child; ChildIterator iterator = ts_node_iterate_children(&node); while (ts_node_child_iterator_next(&iterator, &child)) { - const Subtree *child_tree = ts_node__subtree(child); - if (iterator.child_index != 1) { - start_position = point_add(start_position, child_tree->padding.extent); - } - end_position = point_add(start_position, child_tree->size.extent); - if (point_gt(end_position, max)) { - if (point_gt(start_position, min)) break; + if (point_gt(iterator.position.extent, max)) { + if (point_gt(ts_node_start_point(child), min)) break; node = child; - if (ts_node__is_relevant(node, include_anonymous)) last_visible_node = node; + if (ts_node__is_relevant(node, include_anonymous)) { + ts_tree_set_cached_parent(tree, &child, &last_visible_node); + last_visible_node = node; + } did_descend = true; break; } - start_position = end_position; } } @@ -397,7 +401,11 @@ bool ts_node_has_error(TSNode self) { } TSNode ts_node_parent(TSNode self) { - TSNode node = ts_tree_root_node(ts_node__tree(&self)); + const TSTree *tree = ts_node__tree(&self); + TSNode node = ts_tree_get_cached_parent(tree, &self); + if (node.id) return node; + + node = ts_tree_root_node(tree); uint32_t end_byte = ts_node_end_byte(self); if (ts_node__subtree(node) == ts_node__subtree(self)) return ts_node__null(); @@ -416,6 +424,7 @@ TSNode ts_node_parent(TSNode self) { if (iterator.position.bytes >= end_byte) { node = child; if (ts_node__is_relevant(child, true)) { + ts_tree_set_cached_parent(tree, &node, &last_visible_node); last_visible_node = node; } did_descend = true; diff --git a/src/runtime/tree.c b/src/runtime/tree.c index 9d7c36fc..72fba7a8 100644 --- a/src/runtime/tree.c +++ b/src/runtime/tree.c @@ -5,10 +5,15 @@ #include "runtime/tree_cursor.h" #include "runtime/tree.h" +static const unsigned PARENT_CACHE_CAPACITY = 32; + TSTree *ts_tree_new(const Subtree *root, const TSLanguage *language) { TSTree *result = ts_malloc(sizeof(TSTree)); result->root = root; result->language = language; + result->parent_cache = NULL; + result->parent_cache_start = 0; + result->parent_cache_size = 0; return result; } @@ -21,6 +26,7 @@ void ts_tree_delete(TSTree *self) { SubtreePool pool = ts_subtree_pool_new(0); ts_subtree_release(&pool, self->root); ts_subtree_pool_delete(&pool); + if (self->parent_cache) ts_free(self->parent_cache); ts_free(self); } @@ -51,3 +57,38 @@ TSRange *ts_tree_get_changed_ranges(const TSTree *self, const TSTree *other, uin void ts_tree_print_dot_graph(const TSTree *self, FILE *file) { ts_subtree_print_dot_graph(self->root, self->language, file); } + +TSNode ts_tree_get_cached_parent(const TSTree *self, const TSNode *node) { + for (uint32_t i = 0; i < self->parent_cache_size; i++) { + uint32_t index = (self->parent_cache_start + i) % PARENT_CACHE_CAPACITY; + ParentCacheEntry *entry = &self->parent_cache[index]; + if (entry->child == node->id) { + return ts_node_new(self, entry->parent, entry->position, entry->alias_symbol); + } + } + return ts_node_new(NULL, NULL, length_zero(), 0); +} + +void ts_tree_set_cached_parent(const TSTree *_self, const TSNode *node, const TSNode *parent) { + TSTree *self = (TSTree *)_self; + if (!self->parent_cache) { + self->parent_cache = ts_calloc(PARENT_CACHE_CAPACITY, sizeof(ParentCacheEntry)); + } + + uint32_t index = (self->parent_cache_start + self->parent_cache_size) % PARENT_CACHE_CAPACITY; + self->parent_cache[index] = (ParentCacheEntry) { + .child = node->id, + .parent = parent->id, + .position = { + parent->context[0], + {parent->context[1], parent->context[2]} + }, + .alias_symbol = parent->context[3], + }; + + if (self->parent_cache_size == PARENT_CACHE_CAPACITY) { + self->parent_cache_start++; + } else { + self->parent_cache_size++; + } +} diff --git a/src/runtime/tree.h b/src/runtime/tree.h index 50ae8490..99481d88 100644 --- a/src/runtime/tree.h +++ b/src/runtime/tree.h @@ -5,13 +5,25 @@ extern "C" { #endif +typedef struct { + const Subtree *child; + const Subtree *parent; + Length position; + TSSymbol alias_symbol; +} ParentCacheEntry; + struct TSTree { const Subtree *root; const TSLanguage *language; + ParentCacheEntry *parent_cache; + uint32_t parent_cache_start; + uint32_t parent_cache_size; }; TSTree *ts_tree_new(const Subtree *root, const TSLanguage *language); TSNode ts_node_new(const TSTree *, const Subtree *, Length, TSSymbol); +TSNode ts_tree_get_cached_parent(const TSTree *, const TSNode *); +void ts_tree_set_cached_parent(const TSTree *, const TSNode *, const TSNode *); #ifdef __cplusplus } diff --git a/test/runtime/node_test.cc b/test/runtime/node_test.cc index 313bec40..1f36f3a1 100644 --- a/test/runtime/node_test.cc +++ b/test/runtime/node_test.cc @@ -438,6 +438,13 @@ describe("Node", [&]() { AssertThat(ts_node_end_byte(leaf), Equals(number_end_index)); AssertThat(ts_node_start_point(leaf), Equals({ 3, 2 })); AssertThat(ts_node_end_point(leaf), Equals({ 3, 5 })); + + TSNode parent = ts_node_parent(leaf); + AssertThat(ts_node_type(parent), Equals("array")); + AssertThat(ts_node_start_byte(parent), Equals(array_index)); + parent = ts_node_parent(parent); + AssertThat(ts_node_type(parent), Equals("value")); + AssertThat(ts_node_start_byte(parent), Equals(array_index)); }); }); @@ -495,6 +502,8 @@ describe("Node", [&]() { AssertThat(ts_node_end_byte(node2), Equals(null_end_index)); AssertThat(ts_node_start_point(node2), Equals({ 6, 4 })); AssertThat(ts_node_end_point(node2), Equals({ 6, 13 })); + + AssertThat(ts_node_parent(node1), Equals(node2)); }); it("works in the presence of multi-byte characters", [&]() { @@ -530,6 +539,8 @@ describe("Node", [&]() { AssertThat(ts_node_end_byte(node2), Equals(null_end_index)); AssertThat(ts_node_start_point(node2), Equals({ 6, 4 })); AssertThat(ts_node_end_point(node2), Equals({ 6, 13 })); + + AssertThat(ts_node_parent(node1), Equals(node2)); }); }); }); From 7ad50f27311cda9fafbf2308a025f5aea4ac7c20 Mon Sep 17 00:00:00 2001 From: Max Brunsfeld Date: Sun, 10 Jun 2018 09:54:59 -0700 Subject: [PATCH 83/90] Start fleshing out docs site --- docs/_config.yml | 1 + docs/_layouts/default.html | 123 +++++++++++ docs/_layouts/table-of-contents.html | 74 ------- docs/assets/css/style.scss | 39 ++++ docs/assets/images/tree-sitter-small.png | Bin 0 -> 115131 bytes docs/css/style.css | 13 -- docs/index.md | 48 ++++- docs/section-2-architecture.md | 18 ++ ...rsers.md => section-3-creating-parsers.md} | 191 ++++++++++-------- docs/section-4-using-parsers.md | 8 + 10 files changed, 339 insertions(+), 176 deletions(-) create mode 100644 docs/_layouts/default.html delete mode 100644 docs/_layouts/table-of-contents.html create mode 100644 docs/assets/css/style.scss create mode 100644 docs/assets/images/tree-sitter-small.png delete mode 100644 docs/css/style.css create mode 100644 docs/section-2-architecture.md rename docs/{creating-parsers.md => section-3-creating-parsers.md} (91%) create mode 100644 docs/section-4-using-parsers.md diff --git a/docs/_config.yml b/docs/_config.yml index 10e6731a..891551df 100644 --- a/docs/_config.yml +++ b/docs/_config.yml @@ -1 +1,2 @@ markdown: kramdown +theme: jekyll-theme-cayman diff --git a/docs/_layouts/default.html b/docs/_layouts/default.html new file mode 100644 index 00000000..1bee1a8a --- /dev/null +++ b/docs/_layouts/default.html @@ -0,0 +1,123 @@ + + + + + + + + + {{ page.title }} + + + +
+
+
+
+ +
+
+
+
+ +
+
+
+
+ +
+
+ {{ content }} +
+
+
+
+ + + + + + + + diff --git a/docs/_layouts/table-of-contents.html b/docs/_layouts/table-of-contents.html deleted file mode 100644 index 0e2dc7f6..00000000 --- a/docs/_layouts/table-of-contents.html +++ /dev/null @@ -1,74 +0,0 @@ - - - - - - - - - {{ page.title }} - - - - - -
-
-
-
- -
-
-
-
- - -
-
-
-
- -
-
- {{ content }} -
-
-
-
- - - - - - - - diff --git a/docs/assets/css/style.scss b/docs/assets/css/style.scss new file mode 100644 index 00000000..2fe29224 --- /dev/null +++ b/docs/assets/css/style.scss @@ -0,0 +1,39 @@ +--- +--- + +@import 'jekyll-theme-cayman'; + +#main-content, #table-of-contents { + padding-top: 20px; +} + +#table-of-contents { + border-right: 1px solid #ddd; +} + +.nav-link.active { + text-decoration: underline; +} + +.logo { + padding: 20px; + padding-top: 0; + display: block; +} + +.toc-section, .logo { + border-bottom: 1px solid #ccc; +} + +.toc-section.active { + background-color: #edffcb; +} + +li { + display: block; +} + +body { + overflow-y: scroll; + padding-bottom: 100px; +} diff --git a/docs/assets/images/tree-sitter-small.png b/docs/assets/images/tree-sitter-small.png new file mode 100644 index 0000000000000000000000000000000000000000..73f7f163fa251bd66267c7abeed610ac805cbed3 GIT binary patch literal 115131 zcmeAS@N?(olHy`uVBq!ia0y~yV4MKL9Bd2>3=R9u&M+`AFct^7J29*~C-ahlfq}EY zBeIx*fm;}a85w5Hkzin8jLZy)D2ed(u}aR*)k{ptPfFFR$Sq(10h8?tx|+m6SjhDZl{4oh6xR2%GYXq25Z)&(#OX=o{)8 z=)=th84nV*@-NCv%_~U+nQv!k16GM52DQgVA7nX_V<0{Siw3#4*>TzEgTn+AK6YF_ zhtwk(7#tWpT^vIy7~jOQCxnDvvon6#`h`nTpz8fzWe%>62Bk8ASdop_uhxeaT;2M5 zowk(!KP5?{`lYS5w^UcVhPkqxE;nPgj1QSM0x6sp^vA z1s=r&v)R4f;`(M;zV|mKx?Wf`^RD>AD$%}Xi%$h94=wJjpL~Dy?rZfo1)n#4fAeFt zx-63wO9{sdfec514N4MiKR++uUL?4)>>h9M{hZVZe;Jx3+d16>KkBdF+Wh^@-`^$b zg_6zA{t!JE6fX!VIOH+^HOZ)zy>T~R^PK6U1NUoWRODXWTf&*NFL&aM{QbuhSbn$myRiDBr_&yJ zop78`AhCX?a?#hv{+*lKw=X&B>gDLyHz7je?Hm1!{+m2aK>-Tq6!fn1*UdYBa^;gp z1+3-C4v)4jlTZA)C#;zHyve%O$S+Hr9oid2SxT6aSb6?#IjOjy_qgTU!%goj3a#>V zqFnyA&Sn15_0iZ`vi5fAG|$aeowM8fpQJ7I4oqy6=qY1ziBEm}cJ3V2e^=rv6VB;B zaq3b0w=Vd+TFT$9@~M3(cQ)j`?a#HgQV2V;);s%TLH1Mer;=~#3hwNv{CQ#jgcTZ2 z!KtoQY_d$cEKH&2cZo4^?49k)$Jpok`HLrW+@F8f_sbpI>GUb_t;@3b_lnMGIwIlu zMbd#4DFH5vm+6{`ih3oqzx(u~$8}Tr_Nr>be|j@*@*I|%O8<{){QU1#-b?$F<>y}f zyl^SBeOh#}_R`HS!mU`M)L2TGE*b6L?fZ4swNOJt!!pk0-}dawx_hoQV@~CtEq=#d zzmrp*=a_pmCPd|@Wuz06L*C+Tor`qT!^?lZI{S6gtLC@M?%h39XIJp$%Y2RZf-(W~ zR;2KK67}@iw!vRosoF%JKIdvs`C#2dHc(% zmv>93|9dOu>D{xk{L#UEv%7sRt@Ay(%k!nlvpcuWTNGamF`Rv7l0>15SWHM~!`B9< zx_6JHQ;r?dsC>I|YnyWY+|+;D3+|XNNq^9(_;|TYZRW>E&da~O{muG4s9?pX51`Dd z%fzH=ywdec*!z@bhXpehFIdt3_+*uj;6kmOJt-%Jn8RK#+xIr+_j~Kfe{J8jKiHw5 z`~R<=LQ!zD0DoB6l=#eZ#fMb9f}Vzt252qv_8C z!Ookf9eHBgtt7YopSj${ga2Rg44zp=jxVi0>~R$8bUgA1iph#`=L-A$zvX;)*h3{m+AYH@X(3ikdEcypx?zh^x=? zF!$->9`ilTHS&~;?u%9C9aP_xbKR)Zy3XNev4VfIO0eU;Id{bM_{yFydA;J~r0G)w z-&UQFoS9*klP=;eZl3w{zLa0r&rf$x_n-Kbx=P~cOa`|;k;@PJ{9IP?#@Z$qw>0=U z%z7N2eaP&4)@!R1S64s&Yt1J!qc?5BWCg>EK8rmR)=XUXwf3y9c;XAa+z<|dwKS%Vt^`32~&`2#}}TEY1>PEXH>ja_TDWd z)xW=z*MislSYHrJv(my(L4k>i9bDoMc&u5EF>hSurCL1EMc!KL^1B^@FP*$>v-89m z7}xH2ux$}z^O>0-uUrwZ(D&Z4V)x4Ob?XF9obhy6DPkjw`Pp{pb_xvu);STRTM=#zKmAkz)J)w6)%Bj;H;{5j+BFsb^ z1x)j0UcY;+|NTpymr=L#{4HN~PW`{nFD=isBd@IM)(gwZLvNa``Q@MOQ?0tBSa8w7 zA$hz0EWVerE6X`Dl8?N3tX3VgX-WCS>8mt}jwNJGbD%mg`qf zhnv;?ovc$^wcGT=$+d!IvqEo0&e1=xQLKI5D*nbvAH)BKf0@(E+W2hmn`|A8tq-d1 zMb~7zm(<(x8F#qb|G72SW|k7S+FtF`@_*ZZv>YfpCD)_#MbXsv?A(eAm(FNwxbUs{ zy-nw%g2suv#dRO2yI0HXKR4U(F!!#-v(8yF-&^SC>bz{vI@{mvO*xhe^UG|yZl7pe z$|mmGC|~n6de?nNNB-<@N$=k% zl^vZOebMOcvWNPUl{Xf*YL-m9xks#xopn|o%l=uBC6yP96@INZe6;?4`#YPN1{{)m z3RU6^3m@>WzW%mvOWna$Hg22&3zOH(4}TnUnBC<@O?>R*aK*xD{fjqk3kzdCt+hU8 zU$EAygQAn$TGsw6J$|C_@u6GgFD)({PuOJ4);jCbG@VFE%h!|sz25gTm;d+L4zBoX z!IvE!qz!zpGwn^A-F#rlL|gk`rJonhzG&cTA#v|dpE{f8{nP2I3tm*-lnkg+u47-* zCv!K-GtA3IgkkLOQ6`*kURSLypi2|+#C_8!*`;qa3l9hx7r(oN!Z*G-$dckd>b z+19U0T^)Xvl)T^jKjqoMqFYN9BffHI=eqqc&5iq9c;LiI^Z#wLZBJLGTfK{Hs(Se3 zp5@;E|K6|X^T~Uw@zUR}>ha|AKc6qxMp-`H6a7l;(?YA$=Xz>3eD42Wy1`tebc?y< zOZ zlw4N1KL6kM*WM>@F8bzvFgDm>frM2_sp&k68M$75fsfYQU}a|C7GIe0UHEy?)AuKm zb-!(tHqX109=oe#XX?#OsjSnIwEtwkxV<=d|GwJYcYYidw~uW*W7}2ysrgg#XOCAR z3cL44e~)@Ds}TS9*Xx6ukAByF!eVmB;O(S+xqNdw_Wu2Hvt{;2zi63T)<3WN$bUcZ zu9m%Tx8&#dH|wn)8`)?Z*2~%KnR;-4taY<(&SK+Wq15}=Hoq}HTAya$5z4sccF~z< z9@&T5)2p&vI9{+U5p2B6v&yQ;?9%na2K=w?Y0Fmy1}*kGY%QbxDlJd->WvNRfeX`r zZGWFXUDUgN^SXI+I@%m6=NC_i{5pp|C|L#~t=-m;V2GWyL9N z<2KWJ`EmEty@%tH!j`efFj~yGZc(@7#J<=L-TaMRX|aM9R#TrE6s%XVE&6<7r+(d@ z%F2(tCeE&s+m7+(=%0SRP~cCgvAWHVquXD{A1!t5d_VhrWWnb<|8;&>R2Hfg>59J0 z3yJ@D=fjyzfj1gIna$Z0#Tzp1*_w!&?mgQ@wog>+d-Lt%uCwmXAB9|9n)j++N|fnZ z^O8XAYucM7ITE&3Pk#M&$em+LBD z{*!N!nAy4Uz^$)xii(W;IQa9YOK|*f_v!5E(@uuZKX`wy#o0ObK574QTXyWIPJDih>F2Mx z_9;)Jr8{eR|5lal%znS{vCD2LvuDw54&@*F&ri`Ww3BOJnH_G(H^<1;RpLYL?ZB8o z|2jW2mnoi`%Ds2E%=4LifB(${ua4~nj0>VxA6$En-8}Hik`SXWudfzXyqY<=f9}Z% zD=U&dd=>qCD7iyfjJJ)owqgdgQxmL>c^S{r{iasAOk=;H{#Zk7}BP(ag z6VdkSg&G&6!*Y&qUo+YL*}9zz)xf+B)m-S(3#MsFntAouJM6geY0}{McjSfy%T0o$v#t$>j z6n}|!&61PU=+Ez)o?gk_o)}R-@&E5<_b-U<^hhges$gk3v8180BE_@Z#=tS{?_af( zUcK!6OYg4zasA@W*k#+EvQ6t!wrYsKvOrjXY01W5Q@ikFbI(U*@VjHs5N| zBHqi*^Vzq*_U-MQQdZl3%J#p^@~i2ut{e`(_b+o@OUYZ+oslz_cCZ`IY`74+cOk2T z6wA72t5_F%cd)-NR6qLU#AKs?rylNEa$@CY`{(hto#ID5*Isk9dG=@*N0t6#UH=$$ z$&-OW9aHzuoxE5=R=178c;zv#~2fAvp3{x3S=(5Y;cne)Hz{ii)^ zmnSCNzZYA_`tgUl-r2psI}$&#e}4LZUoEqY>%Nrb>W42J3Z6IrzshFs?~J*rH)q{W zd1ZS!u)Jr+`PUz=e(w4wA}Vrf@_J2E-jCt;S-Hh_G*rCYu|}!E^g-=h=1hkNX3dLu zm;28z635wpZt5hc6sW5 z2KCQ{GYTX$%c8$Lnj6uh(q!;KOo~ytTmIf{J^4LbyndAmR(?FsuVSkhC8#lV;jH>M zvU3!}yeDL{+pK=ItL)g04f^};3b3wQd23qY)iqw{!&q5LR>iV5=si2WVUl6a&->Z4 zDpxEleQvO(?5kqn?fi3Bs{$gje|~#2JBWd4!d$k=3$q0HuT7slRV{!0%=Hs?+Lqa= zv658D%a=DOP;qx3W-HcntP=AU6-SVe>0e@ zN~Xpf;aF|L_{=lC|J07myQ1xf^8?}*F+ZNL`kAM8IrqXJX=mSguiyDhYGLxC&E4Ld ztwjgq^!`jU^$N&%p0@17jG0^Zgq~!XIMwF&Po7g3_QC7ndgI-P7TRbo3yGTd>aW&;g}u)dWA{{- z@^74Pvnqdf-_`2S^RA#OGPR%ib5C^K0-;G=VV7^c_j6o%TWDr=MX7<;^=$hW+Jp-FZhf2K z5z~C+U*H+Dj5*y^*|)yd_N-qMt!K-&HF%N1+W#tYB9?1ozpeiAcD8XK3fq`>6u#k;w42kt;8f$WJ4=N>U!I?$P#0W2&tm)YjFOA5&i+pB zdsJmx8*$;=tZ9+E+QbT26IacPvSPX8+GN1YE`LWX&(FWZRAeU4qnEk;o&2u)0wR`^ z_uSF(Z2kMJ(f(5E{ddNnKL3tCaZ>j_)5Dvd1;_amw6+SnZYjBOebs*zvpeamCnRU= zn53lRA@@yiQ*JHG=b!q~-wxaV*()P!7x(qJ`i_0#?a6g1I|?s|COaO;Iwm-QVaJwF zQztm+&;0yCCgR<}NZGR+)p%sYJD2x0CihNj48NnnDo`6~pT2n2Y>zpzGIvvK&s1z* zYk&4SkNJ6t>gcqzw1d~?YHu!o!=bPA$o)$O!?H_De6Anwd;7P)*P8E-xTU01s zI@5JiFSQ8Qi%-5Y9b#Z|7k=`>x!!YoVQ@m@wAjT(YmH}fbsaf++5b`h`^nMA_GkV* z@vG+i@%1{tj-Tb<_cQpT@3+l5hJoMPzGwcuckQp?_D7bTZ6Dvxf9Bcl`1r~0>HDUd z#|O1BK1)4+zp&CL??RM-YVQUS@zwt`1T@4QGx)#WpT^pF@Nd<FMlB;LO^Vb)Sb&$-&3wSo+SiAq^@AH38$STf^Sa^4&{xw=gQgkvC!=3g{rmG$PQK)|}nIlfuYxCal5~if;`%40&Tp~A$W}houz`@S8CwY6`KfmLB zs)0M@UOnea-=HtvHGRHz*hJQ?LWL$??O(iBwJ_{kt8iq!#i5%Q8LMyA|31dQ>HT@# zB8ST619ShZT32-b^>@WpA9hwLDTSJ?oF(btv|(n0(Tj$Oj-H2BEM-*?H*sSwF!@!c z_ifFd|7Wsp{&*ayl=b;;`F)H3z24mrVte(^B)~vm%L~!y!#ZO-hk56|0ZhheU z#fTJ^L=BucF|$L74{@0?XLM2;UiRi~`i6((14OV9kX%$jb! zRjrekF4ldrUP`03ZWvw5=gHnlcR)=T?LJxjVp?qSi8EX)wv z8E*BZOqY7>!fH)TPh9@^s!Z{pd9J;PtxW2#=&9aewrUqnYnuOyiM#)9!=>8NySIhP zA1TN@(lQj^s^wt6RhPp+Ys2Q<%PqD#9T3`@W1&ASyl~wD`Gqeo-8<>}Mm9F(_4W7@ z4X5?DKiRr=UC3_JD|>HP?>u$pf~(q^t{z*bJ9ZW_Q%&VrXH@ZV8t>cnOMZidH{2mPLH-Cv0pZjm!v?e@BNttE@WkGE`J-`*+z{_n%xirXeNYi1X| zNPWDdSU=xlV&s$KH($Oi`WDf6MCF;6RLaR}bNQ(Ug@2tkEstF!vgmB!lE7k*Sebl* zo0dsO-@JOtd}O(9)#{7p>!!|L_~O*AbScKlic2;(w};yrHqSP@^ZnLZp&KjN9~3Hn z+Fm#Rd0Iyfr?YOP15-(+O~29k`H>s>YTEy=y?@mDOS_?%yPC#{(-WJex1YSm>;Cll z`o65G5*re)D@8cyU450qxBr?!2H)GOw9h52I};b#TPG*x zB_93gApK>_oWgdQo09*S)A`>7tl4cV%R1XKX{G9mSx>WLG^^R#Zp<*+Q|Pc#aKh_b z=Y`+ASaGIqzpuj^-4`}p4C1O)S#y?6iRQlT-fi(J>&=G0ADbMvpOS3jWv<+n@3#Nr z`t`SdTc0yM6qF~qt!T4RYX90Tf%)#X4h)mqm@jQgaA;|B((v4QdVyHjoP|fX8a$4% zNmV(%dWK(*koV5@em1WjDw_leKCze+b2oa;jf--RO&?xKbhY95q`tpSz>5F2f?Q6& zn9}@b$4vq%-_1Uraw9|hQIc@S@u)v-iH7c$GtR2dZMgAAebs!nB!^3t&XFhQ+hv=* z{Z+Pe$Ipn@=g(h1EGl(YEmJA8=}pXIpL5SIoX?SE^2-aDwf#cMksE(sY!nZDZ-4*e ziZ2p3{CotWO--K%ek+c67OwauMDEj<|5x7i#5}kd^|Wfs)~uzWH<@NXh*DSZW%#me zg%Iv{Vl!dx*I%~{aBtH-mtcB(pJq|Yp+KZb6og%?$Lp8=DY4^ zS8qIN-o?eGV)<;%n!5_0USu0Dom!tQT7OziI{J=A(I3{IH}6RFuq??4*}vgljmzhI zk95p_Cx7PM5_!UOf7?uP{jfw?w?!}23q|(+`?3Dhfd=&rOrHX8Pc`DwpS$(Tl{1PA z^ZAP==;}@kE^9pbsek^vN$K|`br+}gYWMF!wWB7z1%+kOi9xFmm8ja zdRxMDDP~{D-)+TV4JYz7F3HSP-<0rW^0M#q|2g%&v1Hyjb#GVc(EYrVD4&qgSC z_aUjDYd@!~pQxR8@@n9=&gQT{8y*Y&pM0tBAMXnD_S_SGC^l+pCU4ob%zZT__5bJp zzhTV&d~$)$(ZEk~=e&cS`zif(E1M=5T-C+DXmy;-$sW(yoyyLI5n_5rF4g*FbtliY zEu-QQ)Z zc3FPe%2H_Be$4E<8Iz{uG@VGNra&i=ZU29MRamgB;rjd?_xGjmcl`NyS^W0$6F;&a zt@(B8?<`5rEm5a9pKNuHiz=AnRDSgPY2DN)Fu(%!E>fpdI_?}WS=ZJx z`iFH!2_fPZLcdllsQP(ByRpyzi}b=B%MFDm?k#0f+_F!ncHNniI_H1JRaREcueMKF zs-`{Huk_d9qB(c&B<4Jb&6~MYI#D`2XR|Vs`mRH#*164lvF+j?vCCXmqG`)iPTrZg ze$#oGJ-0U;kO^(Pa^Ts6psKQ4F`xD3vBlc0^FMK3`nz#~nV5vuL~jK%z4y`u3KOFj zX!ahukvr$f>=!mWxvsk;Jux!LYkIG}rt)Zu%KzQYUyj^xxz^Ic(xSNkXY}MF4=Uvt*D^Bm zE-|g#o%!j~3r9h_i0?04mSpa8OxiY^peOV7F!N-@Etc8ST?=R&-08n$*+TSs1r1=cZH4eV#`P z-yFA%?#w*!!pu##SzLGiOmVr>>@cVN{5;#+dN0q!^XJ_toOI-H#sANf zQ(i1loHS8HKeKQm*F&RS8vnIjy!%V@BD*5&%J}#?=C*n)u~A@CGFsn%r0Pb!oAJ{? zj*CJ&<+6_~w@}z}^xjQLwf?xAD$lEt5l5dYGHHFjFtO)Xch3@*RH>H_Uak6dsJHjh zqqR$$Ro91CF8Zdh{lZJF&tX#*>RjBUmLs}1c=~UXr>cRf57ocB@MmOKTw9v+_wFvo zhWqyz7k&xfv!&}u^W+l(LJvNkU%|4}WBq*3(CunruiwtnJ-jM($-I>^94-&yqGgW# zo-d`utS8;N_lWYV`Cnh9>Guomx9Oi{H1rCa&Hne(SHqv*{;p+J{=Mz?-pczk zAF9|`EShNlp|AR=LCUk(8^+dATNmq0a-KKoh@-#CnRx4*M~@j?&gnY5+!8i@>dvEl zcF}LkPHggI(DIJ5UNN0PbaK-ios&$@1Lx^Eztk5H?#eym;}`O1^{G?qUP!+Fk|(@s zLZNHrsom>$XbQM@X|0&v;Jrmvn!|dhN|mzl#0MMu+B~@8B*M?F-D48Zyj9-A`O4Di zk)|v^Y$YNJkH4~iy7PDVN7f?w5WQcIB$qAVvOeyTrH1(&%?J0zpD&C3F573* zk>yivZ+GBEc1e8B!kNni*mpAXsr@-v-=nDDTqP(hIIF-{kVW3wVcOY_oz>=Zrpyn2 zJH10VYwDz{ssUy){nImbsvig#`t5(pm3dc@!SV5dycLe8M3+Zi`!At+zjnvs1Ghu} zd7Yh>c;fu~bH1x&=T~O$IJn$*Mv&U$UAH{9O7Hxc&7b7{$8efR6Hy*S3ae=}BD|q%=E*2=L>`k`|_ji$DV?626vp!Jv+S83P?*&?pIB2{V%42yO zDj4>0_13vRx40ad*1wqJeZ(K5bM~8`ESzyc$f&J$;ZqSyjn_t7Hoba0Jy*d|`Qf4r z?_9l5dDhpdA<|4D(S_?LHA|bF$nIY^>4=Nr_g5D))4$JC6b@T=XGUw&DZWWr)9;E% zmI{U)cDh^ep&%urQ1sct@HnND6PJr#TA=SLb|h3}cf^xFVoZnLTF!lVWcd#MimOGJ z#GgL*R1^rlez4By lAUVgs*bnntfT#p>}T;^3)Z`pXbL2!~lVa!jX2p3zIcWSLR zn}hx(nLR#WclXfsTx%_XM9J`VF`d2gE`D!sYyC^So8{wR(sDAsNS*(Aaqy^Y>4eOy!O4ydkK4J|+V782K5x6gf?tT+IU@V~P3^Fp69*+S z!;Ul;XxCpe6?2Jpuzh>$mul8spYHB9y@O7hJ3nObhJ;K=k8C|u=&Pt%AZsbPa*~Lo zUZFa_Hq+tnoedo;&K;dQbz%5Dt?%~c6K|ZTd-&M-$ogD%&mN0~CBjj$ufz{em5CpB{BIN$#i@qp0e^D?-p}ixgdLoN%s%_N?@#fU~*~lqYeFDA{_2K z7OP~scW}H9Ro|HVB+V#DsPRcQ|KzS$N^4HstYX{7DD?S4r@L!IR>SX>y-oJV&Wf-9 zucU9gq3iR-E6dZ5@3;ElJ^kv2z9ZlBr(KCrSKu{BSkt9rQ)-r3{&DNwMcVrV=CDqb z;_R&dz3!3Z1+kVE!xa|G95fcpc-D36u6jU|)whPGxjX*VC_P+wyyUptzEA&t-Cp{A zgP+&SH775ff6h{Ra9#FA_ivs?I$@Rj4NRY@3kU>%>AW0r==aBaSHpu{o<*5IIn@-R zv}ujpp_FZh_T*fNp03EjG->Wyf#gDFy&sB~mT;`p+Ixysf7(vRRy)m5*R=g_+L!XZ zi3$(kI5cgc$PKZJp)(I@?F`}W{=;VQpMCqochU@x*<&ZhasH5gGI8!j$2)tQo?dm; zKdr7`d)z{Gj*4q+b;+y}qk}bKM~=$5o2vUYn{2$+dM)+awOyB{eNqY&6q4 zZTj^6cdDmul)uG&_2lBGYnR7!yE`&>&U)PXsfY2^C7oB7?(A{2>DqaJ*OaCEqiZ<= zQjY$7@Z_E1d7t^(v+F{|8jf(@2|HN$^pvTlg#1nmu0n+yFO!rzJ-=pW{F*a$_HL8f z-+I%;{@mE?{bI?HCtD^Q-mP@x_SyRv)`d?Hx?j)`==tu|%hfxCW<_snl9}_nAY@v{ zj4g{7)#!YGUcTL9iRv1^stwaG{}aFew@&ucZ0jxJkKcNyIX2Gic^YtbV$9Fds9Q&> zw7ql6&5p!giT@DeFUApY=Aggh-|)K@n=c7G{T|>qA+%@ZrTmxSTbmkwwHcVUms{?C zD6hTk#Od$*lz)Gl-l1ywpz_WYONYGg#Z$9?eScrZsUliyo_9xLQ~K#$Ey>HzrCnR~ zPOZXo{@z}O5(T!$XE&?;uKo0O{l^u3I<19AEc|zVZr$%`^+$;7(w znd%>`+S%KL0vDJzhRzY$5wa#{_3R9>cYQ{0WnR_Sgl^~6)tuivSz4@#ujtMB`A_#{ zPg%5Q&%4jeQT_s69BafsOZ(0F@$%ETxyzpfZfRZOX>>|^wdebnEK39r)*t4bUS0F; z=K23_Q$H6J1W2sgw{v>W?3==;^2|@R2i-~zUfs?170+s#yyx2NDp^*+ z&a{w0@v6Dd^19qIwjI19y949#U%B!5?kkIM4ubG~#b6-VSGc3vf&BVR+Vlxgb|KPmZby@#zo2ipD z{jWQ3mRiFxx4}drLtkgc91ekPFVChw>yP@<&%eXwGRbUQta>Duc_Z?8hBVyM98uHNBV^=Q;#va)U_T>TU#<~P5z=#{so&ll{m~F z^{*;XIHkdG>Oo11%IIaelKG~4Dgd%^#sKW%w0T&dCwQdjuObnKP*qK!-DI{l63?)TYq{jxKU zvOYUIQ=X#y!lUdwWi4Mrr)#mtmu98R^p^h}@a?VO%kBlL#qVY@{_R?x3+h4oz-RS>e6h*=Wn~lJ-5N+pSa=~OM|{gFASc)>*Nz& zwk`B`(25IU8>TF*_t>j)q9;aC%(Jg+!h^iMs-~}c`i{J2-LNf?Ly&(WU$f1pJvob9 z<(a*#9BftzPuO%}LMP3hs>W`X*Hp12$#D9&dH=Gl1q>8kq~?Wnec6|yB^|oq+wb{SpIr{i z+H9Y?WOka#q3H*f=xv>N<@ss3x0SD(#3j6r@a3#(VUwF`Xl}b^&iSIt=R-3OMNd`} zJ{ceu9qzts)|HjdbA31_d=8kzC(3$2bm8rVC6jWs?@w8JULk~UUY&WV)Ew#V1xL1; zeaSpy_Nw$M3)8e#$9+)|pY~okvgXv=N7+FYw$HC}de|~=@hR-OeW1Z$`^WPAS;W0L=9w^c6RG+i;cU-2*=x@NN@{lbH}?T`K) z*y$#kk(GSkOd{^V_Me{L4UGk#9T0zbee>0OcYbo5UFiSH>hL~S5eMyg2m0BBze`t86a4UcI-<@+>#g zl~NVg<)_-uub(baARV>oo8RSkAE!qNR89KT6zR#fP}0Wf$w&V9BgZDI#hSY@+zi;X zxp>O{x;X7_@xM#{IPNOTu>SGlr^~lx&*Fw1;&2fX`1cU#kJza%58zFjTa~LF>AGj_{zKmReojbQckR>+4`Nc(DBEdwvIp9nM?m|Hxn>;dU&G5kA$%A z$A4HA9-nVi_^=IC$b|ICQbD_xH&dIrVHf}OFuPB` z{^p;4nS4@>-@S8=UG|+7huf7;*xX9FVoLRtHL>oyEaK*+FZPSZu?*G5G^O~+$H@B+-vu!kuDrPMSMr8b$K8e1-ufl%Is)HwtTg=f zZX7w)YkqF#4wLgw{_2(SJyp7&c9hSGJ;*H<~wESKCb9kJLtY7i9nSF>1eNnESlWeQm@Xbs~micM-vR$ zcZT`l2_Mg(haWrxB*f-l@0oA%a9U^4|H;fpC8sPhdaAZltWVw6RF~xkuf{g3UFK2B za(^_egSejrR?Ic$zH>)M>Sn|In+^Rh5=yU3U2ebs!{J+uF0QhC9XleQA36A}<3Wq+ zyTnqxK>Y*+T-e?6~Q-_qR|zsE~K&`2j@!-0c${rtr2^{w{FC!SgOdqHYz zdwqas1k>BUQvaC(MHE!5R34hvh1M@jb!c!=WM5bi^^;}Vt+gJnZF>)Pc5eG8qg)}pWXZ#gy?|GY9U)~NcSg`~+I7a5T~rR7I9SERhVXPF{fC;#sLe)slAQXSJz z9riuiFa7`cGO5cFchxt!8{IE0eSh=+-(Lr2ueiM5XK(v)p0@_cm%jzyS=ayJptjh3 zKHd0Fox*a08-L%D-tazdUvk}*4TIycd#P9@h25$>d$~(JwER^MC*QeO7YE@(+`A9ag9c3h=&| zyt`##0zwn&_; zG2bk4#*F7nQ&Vp(ioYhv)KhRe*K_A0b@t5Foh;2KBDkk`_HAcq6FYx*tJgWzD4lf{ z8#)^+KFnc1%J;#4s=9rpK=t`8&MYr<3Qm7j?pe?=wfgRCF0KuGv|MVZt#nmlPn2e| zvT~TtK2dL7sZ$ew;}OHOz49FUlP}FI_?i1u@aFE!UP;Bfx(*_qJ6^BbDPRBdp!O2>Ke|0OJjYaK z`LUGPUnlRXeKxuz@^D3mf?#0LGYgBI9kv{uelB_~7WXl?fXN$vRy zs%K8}5z3r%^V{Y6Up%vm{7ah!Vj?_k?_QjcU@Mxw`{xI&C>TiDETD7*`(?g>4mfY^JsB6X7{J$1EIrOsX#J!mtleqS&!s>f} zqVCluD$9R#mdM-p?bB z`}8y9q#x_Dy%*lz{xZ^*nJXs=NiP)GRbEit z+Tq&}?5xtKEW~Zit}g0w`GD`S4LcH^G}Ux0Jrk3WP!mzSc2A$>HurrIRX(n{Pcxn| zyuaoAWas<%X-a#hPoKK*-sZRkGq?i9?e<%5JABsDd)ki=%AamEYd^i!dN=ip@Ap*# z>RQdOG+cs?GTQIlp(PoA)FI%AvVMH)(~avE|0rB)FQ(K~@a0!u=hQ=;{5w|^6qJbm z>ORx7-y_Y?d{f%h*$XG9>vSbe)8fu-;1M@vWZc8Ue2ew&&+9WS@_h1NUATI?;(GAn zEf&8;nLuo;~lF`TVT<>3wIeG6{cQaL#qcr0?<8D$Dy6 zV>&};73J#heA4yp5tFT(=akrUDNBlP%$%ToSjyi?+v(t3-jl9^KW{P%3f?R@6uq7! zQvbKRi$`GZuhjn=%v0cP*xCHT{>39TxjILO4`KJ$ zE3NpjTdcpcYtsiiLFWCFt#`Kbt#6#u*I@7QDst!Jovx3bJ>)&Nsyf?D3Bpl4RoV zJuZ4bac=oRmbxF$;`J)tajHFjslPtRD=OR7Vq)c=mCjj8@gIV8^CviORQdO5gM!+( z<5h=*Evs1D6prl4yA{b}@YPvw;+;gkC%>W|m1jK_X1M(3!6#1LhG>=}v)dhJyfav? zDjq1QXm~?XMPbeCPg?1ALe;s3i>^j~>P{~!P+PLq&rRvk%B7KyR!)pg%#dMk`JaDx z?fjIbDpU3Z-4)i_o;mgC+Jck4&l-cbbr^e3PjKHjIgPdB+oZ+Y;ucIf7yDORJ)-~R z$|?bchh<+sw10kR^YxHvcuz(0(O0_P6Yku294#_?Z|M3pyPj=|dv#^2&g^RW14|CY z2=>2T*WSn6S7E9s_eNl1uJ2^72^}Z5zF5e8zvjVx=c}hCF48{h)Vfo{YSHq-S~cYf zlb<$x5 zZkLo<^F)?gochA@>Y{=kOU-F-9@j@s^*pZkGXq%zm3AED5}xlIfBe&)gSx-AtJSSxR4O>|WX$_wg!D%{MqJV^dXbBsr#AK$EayE$9RK3J9q zJ#lQcy}dv+?(>7q-L?~@mu~p<%KF%b)vXG0!QH=oIMWs{o5guodTZv^^*qbAADi-H z#lnxCn% zwliOIw@_8H@kJkNYuix4S$p2_6zH$?I_Q&h*tN;cCqhqLJE~6<4Ip(iS@~%CR(l%q&;c+<4!}7($reH>z zpWL2=|AL0!->9~&&Xd*aocJ(n>$QmH4&mO_SMD9Q@^twjH`AbcLQqgPN6X}6R}Y^y zdf0ftvtY-rHGUP}f4yT4-zRc;2czW5sX4Xb7kW+?GZcxZuH!nw@Z9Wtuz;=e+RPNxy zjH>CEQnPmi{chdYvCCFK=(m!1H)ok)kJtX|f7bW#eG|H|Fxb;wuDbH8P|TmOqQw*a zw4O^!uN1tx#!XsPRngVMqT)gATe*7Q&FcK$f0ms+x4Q7YO-d5KXJ222(2+Ao9nQ^H$Z}T(_k~qGi^34ad5t`irtRIoXc_zs=2x zPv*JWS+PLrTeFOz=zCGsZS#{=LA`ulim58Up{axAVb^H9^ zJz3=UBka5Fy;-p;a?6uV)RUQOB&S32GG=WL~ox>x#_C}b5>37JnlpfxK=a!udWz#6}wHCZ)VxO~rV&~a&s*&27I zv*W6uUw-M#>05G?d?Lkm3LToZ$lE=l_@>Ywk24lxT0UI?qKQ-e?=SWG6dK|1ls{zV zist<-p|P(X{8mvA-|eTe<+=jzioV-*zqW|#z50JR!*t5$+d3{0-+g925|DcG@%xD# zU!wDNem=VG1^Y%Z#-q%qD3VFRO1fA1oyqF)uN8mpylIoW>4@m; z{MmZX&z5yZI~{RU)v`V?b^7Y*^Mxu4FZdi2Qw2PV+ zxbak3_Uyz$*I7Il6`WU_Oj{hAd~necxgL|Ewm0VcL(fiPEz5hCx8YP%k(3p)@1E4< zZ;vkQeKJe*j>^uRo0fO1SSI5**+gt=(Ve(Aos*g$rBppV{C-oxy%pM4v(*%4Y-x*L zKU?I_vFY~ij0SV|o7e}O{#N$j&acKFOzTe`awONFxuQf(;PzHg=huGB4}5F5 zIBuHdvKB4mSbU#j!{lG3B`@_(rXQYq+-jm&c;Hq~mp5KZ8jE^4B+q_+^(st>E3;=Inp@dhw-7 z2MMNA2YZATF-QBv+`N42sGwhn%eyJxg#v$WeZ5Gh@YCkY2W<~tVo6foI!SAqZ?r!L zx1Y;@jUpx^_1%7vyKUDL%Lrt(23Bh}B}qoE>^x<)Vw=lKC5cV)A}0%W$IhBsbXi-- zCE7FO$Fn8s-RS|=2c#XDvmHIwB*xvXo0zfv6Zf(2Z&+s9?K$`{x4Lh>TD$o_h9GK@&t74E<$G?~(%gQiPcfO1QLe3(|LfETeE+oW`Bffio^UDbQ0s(? z%Z|EhZjQLO{j1BP3G+l&pUDs{R5+yRW;MlpvP8rV;U9ty`UeZ%*}N$A2@dG}pz+!D z+3|%Of~OrWh3(w3FulZII82Xs>Iu)K$K16phh1%NTeV`=(;}Y;HWB*(UPW zkLyKzAJ0s26?w4k;Id7Re$>8L`o6zoD^H@+M}>Yb;f_A*Z!+l%oU@(0&K)*i!tCJu zIqtO6f=fkT+Dn@m_SNTFoZA(8>qCCA&rT(a*SkUm*R0VvalBH!lYJ+DOt_)NPp`kK z(JBf*p0Qm{y{s00DvwW2vF+W5r6-P`+G*9?SeX<0Jn&w6@LSeHC(oa3YOk`Ha-(yD z+{OYw6~0>wJDd#7ot7+%{TzP!aD0(cjb9<}*&^?QKD@fziJI$mzZUj?HGGj`b);P+ z*R)iL{o#{GirZ~0JpSF9sN}DEOYyb+JqoUWfweI%U|+tOu8&pZt}G}v*(xYxsKkz;K$b-_p0&6)&0oPPv24- zs4ifmSCUhIO4Xer>F8s2i{zSV?U$x=c0 zZEvR4ba+Ovxhpc~w|$e+acjG-;k?A-&fPsbIhV=GaB0S6TJ&vL!msnMLbW3=3L4X3d}fdTnIn&zlnad;3^&y*m3iJ#y07m6A@KJMpfexHv<@N3oH? zfJ^1qkGjG#(*um!9&e8?d?XT>!rY$_e*RLm+=H~uK3tuQ9}QO4IoD*rO~}y|Qe32` zKe57K&z@=1=P)R54>F9OX1(~{p-n-DZ!j~io_q9S>)Hk1%{b#P-4uTG< z@_&m{H@6+z?9?<_rX%}!UF=hrABIJOx+VV(|w;DJGS2aJHwMz z*d_UXWkTznZ-;n#4D-Eb2t}N@_~RhwFVRDjlbnUBg$j;MDsqf5d=+Q%V>Q>>e*(v{ z!tGX9|H}O9-0*qvo`wluq*s_qlX`5WJE84ux(y!cl3`H>u1-h76=K$0i&hNM4S&!(@y2wU0++kh+a|^DV>{pPm1%HZ z?%&7u6AOjspPKM-Z|WVXZvR`i!cAR{Ush z(>QIBciYQv0?Q05P40^e{Q1j@wVwScC~{2d;FGGpuw;6{^tLxW{VtoV6kOw+SA7D_Oq2=Pv@RreD&?xXuG$&>@GB0 zIG4RqNbk=H>z}jn(C6ICyz};ia7E9#z2<)A%jYIbQ}?KDWnyc3^lHY&89Pk1dj;kw z))ju-a^&Hrd+Vki={%wP@Qb0q>`yyq|7y_fj0jL?(OlhsXjLRz_#)PhiC-@C+H3Ay z+u9Qqw`^HkjgD-I?oUt7zyD?`?=;2b-?Xl^)Z03{rro<8dgxbQaGWP= zq=2Qn>64PX^~^wfs(SmMZ4kye}xT;{F81z`~d2b9?>g zH?!Z`boR_##;V@WCRMCkr!qF*QRq83OHjdjxfa3(Jl}HFq4V`rj@; zv&k|cc+q){c)2_$*UM>L$E7x(&&l`pyQ98zi^sM%;p)sM^n2I${db#Jye{C4<6V*G zEOG^QrccY8#r$@d9++G_UC|)B?Z=K&4+E9m<9_|Ub=Kte-VMRMKQ3}ERP$7>-d>md zxpBts082&(t}O8`)dzc?@AiD{*Q_FaF>&+dor!#N*Xb1LoVqIc&)I*fvxB0yPg_{s zjnmQwt3AZlKly$^|Ea>C9Zm&z`dd~f`ba44{&Cq|r)tKj)vGTCzp|YZW2?imgzf&N zOUrlhhFeAK`~E?F)Ax>4`wLR0d*+;Z5;4OhfALIX*LCrmTwc~moYDXG!*S^m)$NXa zx2$*Q9*MS_HvP+fGgih|;$Er}_7e&wEMD?-j^M)dEvyd7Lc+q|GEaK?PnK@q8*pil zU!&0BXmPx7-d@d){s9cego>WwZ%;}`aa4z7=62|O0hQrO)U!6j)ovDh()$;|vykmElO zmlX%Tw{DKkdL#MO_f@yB;>AT%C5~o^+z)ZNvqf%GT-C+k7|;K`yHvi}uaFI!byY#( zNt11p%btf3{JWGpiu5!kl|G()KD|aoW}4}$6L+e2pE$Ey+@XKUvV^%pvVl2!Q``DZ zGJepRRQ$nT$w_JBwaK}T3l{x|JM~L&m!4yae31rQN5f8u2Yh|JCubJ;-Aev_tHXBx z>DT>hgAaTvoF05nt|+*oCnx#twh2`Q5j1Pho_B&k;VfriN5pvX=$?N;(Y1L00JY3#YUh!1>lk51gE!FL5=DB|v26`+^ zO6D)bZ+X6)Au>H()9HWqV}T1|u{mn42hwzQ?6}YI-ul4CDOS^dJ@Yuh8+d;A@8?@T z9ccAQcwI(NQ`7~|)s-CV>dhhF! zo)i{{o)gGSRML8`ak}71*4$?szO+B&nviz#iQW0ni=Ivmn_JB`<3`xr^}+1k6V7e2 zWaKMI<@n39?upM951|*IV*PukdFRdHd&FI+{XgWcTwh&h<8HI@phfwC#_*nf+wqc5usew$IC9XW}_k!FS}*nSXD)LZ7=RhCjM%#dGSR zf@{dD#aneIAL$5-zxh&g(liO?qkl7H1gD){`=G_caMm`aHS+TX-|X@Deb-CqYe7NG zdILt83&|ZVTg5A%y}fAkur0HzSb0uT-<#H*CS_k1zu0PFcy8uu^DlC{4liJv)cxqX zQJr9+uhT^Gl9eY}Rb4tXpz@ z?fa`$w@=dwYM-o9UsB2l`(@_iK-xWp{05tvwDS)4KZs1j@P zjB`bKAM!tNzg@J0_3PdTGuPId|GTCdxaR0o@v}XryG}{m+Sl-V*OkLb%G}x0PEF!3 zQBe`P)>*e^P1It>H(A#YzFqqLh@{pI^-lLgCl|jG?S1gEX_uOttDg3}&3e)H`S%1K zd{0y9*j1bSYlo#`TRczMv1;*;PZkLT-e!IJ?9?RlHm?l!B8#N5FBzXXIi^4I$YISY zoz4}qSe?iBP>!Z$>x_9ujPC#5?{~SeGdcL+wYj%fCX}xI!<4Y-_4+v*KW*HaR(o1B za{oN7c7{&&?;AWit+`bC-pXy9ULGZ5){}KNa_@;yanX*RvNw<3txP!}^P|M$Nq(Z^ zIwg&3+8<73uelez^_KJ9k_)R}x`gIxF5}4g-KPJ2YEaAcTt@+);&HnE0lSlh6r)V3#jL%}QS8=?Oqre}Hho#k+QnOs;C(i|ma zlD$}9!Xr6}ym^+N4;=UxEAw5(rY%#e&V6G>^!{Hxf4)3*{P}Tn?WSLs_X^ejn6l;l zJEogE7VYA_Rk`=WXMqK!0e)_iROb9Vz3GCwp2|)0UCkOIhl);kM~bgBH~4T;b47t% z$M&NWl)!3P+|E8wPsCl6~r)pBLURll!9R)n&7A z%^nG(*0j@=8$Lzvel;cN*krS5(~CYHnD4^G?WIpl2&+ORbvtt7HqNkF*Y$b(jnZh^!Uwmeh zWxy(rt{V#czLz97{ycCiU*(&`*GJPdZi(*od)$5Dovi97j~$GRB2QmM1_#I2Kb|YU zFgmO;UAt<%;X88^M#Bx0H8tI19zWi^?%h?>C-XT^zKn0G=y87DBO=bTf6d}EdsjDq zk-r;0)6r;_j^?g;QtV8XmM?F(edmxdzkJZOB5~{6?y035YkGHS>rFT^=bw1Y+2uzj z8eE9JyD5CvwpIE}iGg)$Kk{7e?Tno1FpK9>#Le;}#zET8G^I(H#)nnf(4xC`<4W9RhNAU1B!hOViHPXtZ>cil|!!2470{pJh2cF~)Bf9sb-N$!}GIsdXJ&$+YF z^JvH7b3zfL!p=S@GW->abim(M61ILh;P zsl~n-7T>O%YyYwI(Z>zp>;D3{0H2 zO?jE3rSl5zjh!L)Sz2V;A2##HE#7@?Yc5mNU4z%((%m1;2{7uN^B`Tu(vD;IZpB%e zVF{gr>Y_m#T6^1b9%WxiR^GvtWKr5Rg=>S+^RoR6Wh+J3&kkR(qj~lMjaH5s9{1%I ziLzU`r!H|jxvf-i)AREdo!Z;xK2A`~yuliH&QC@~?VH8FuRl$=_Z|p8*PL4;`0cR9 zSKrXROJ6^`ebqCo^;`x^+L{LX-jw~R&&stv+Ko=kWBOC_w9YB$KSqd z@%Ndc(3OqtmlzGjX3p-HpOq6N!^P<0ZZb>v^~#yf49Vx^?kMk1m)qf4mn;`!ypolF zcR6FMf%ox+kCz@^uyGRe>YgVj3JOk5%(@fCEj7dWC^z5c_Qgw<-#%fmN}MU_Y+=fr z7-LN%qqoWoss>hy&W#)YD;S=s|0BiR{3Uh1chGEk!3-Ccsw@c(fram4%Y_X5rZi4I z-l=V~@bjik8?8=-J`(OL;43q_7IU=txUKNI$q(;rf7P5^^kYY>nn4A}^+2B^Gkk98 z+ir|*cUmYa)3rfUAU`pw&{*|Ib-*K)kOhZU%;`V5K5qS0`Q8Z=gcaN$UOE-Mr^(6T z!voHqsz`ffzST8r7@f6W%WSzPrOtBk`cv~*rQm0`PEOBXH`R9cjFq3JFum4(R2>pH zX^&&W1dDZ+<+u3XyxCs$y>m&XeSPJTmcte`J*#a#-J85Y?aLIO0xvZdudNDGxed5Z zUkom8VK`Qr=%=@Xr#r%{)N$hqiCl^9YuBIeZvS1;-_FhBXT>M0zFgs$l6q{YQen~8 zFU}!tch6+KxpMFSh4|y%HcG3i_BzfN|Gw>8X+n{KOXnd&<*MzKsizxXuy=VswtX!7 zz3YoiHSeLRM>M&-jx1Z=*3NKz)k=-{c`;jJnf5L$aT1v6$GodM-C>22Sje1W)_8`= zFW>YO3zcWB&iUt-azE4i_LlV_Gt1dIJIudC{QvdqoDkEY_ot@{iF42K(TV<9a3y)V z<|(dscev;H?OUb(TE^`3snh_$-YZk%CD(5Fp?7%Go*zdzU87DpzdksjxV22r?78d2 z7FFSy7JT>Z%Qcvi4%{f;_xD@&oawsJKSkyI!)A80H}~|GPi~7?U8V>aEV)p7ea)qWZ4&C@rSIj`ZM@i1&k*@n9qDC+t}%*kqmR=HPc7_HbnjS_x#&dg?DI$G zP2IEV&i>WWf}aW>?bMi@r1jcZM@jLFt*?DhXkg9!g@N;SSGNanE%1$YQJ;9hphfE= zb2cM`!1K=={3bMc@+EBjK4HSTps7xQy3ZSTMjtmT?w{8a*H=8%>g$*3?D2bK0^4s# z9+AGq{5QOvLE%7a*rAn+g{mJpES)4F#Q*O7=d#Vd`xn0?WyxQSV!8cn9@{y>+_oHQ%40g^^6t{~ z*`*J|c8Q9FKg}&NJ#}c+jLD(N-MuT)9{paz^CWzW%Ok@Q6%MTiheeKsWj9^kHM@mb zZ;zVSBPp>h@$$RNPIKk?j604u-?HXeS8>z!yhl~d5(; z7|yRw622sNtVX&iySab+4EFd-M-6J`?B7wjBl&3O+u+hg3mL9WseiFYtn$NzJzu?Y zI2kiLz7+&~c);c@`@71?=USK(lkCFijgxoWsNwwJKk>l^2UVl&j$he9YDXh(8f>mA z_1tmk|AerT?EY=d785F__)hquDeJ*iv*Vmw!;g9Vt(&qQ9clc&E_(a#H%}(+ofomQC-n|u5(tX1J(wEI<&YwGJO>-FO}1YQQ$$W;GnI5OXgi7BXx*Z*>YUeSvg z)vqdc{rP#=(X!6zV6(F++wp3PN!CA31+v@ke|tT)JX}o7;HggK6>Hvq%XhK32=aBT zuG#;$?6`l1C?~u4s&9XHciq{O!WK3+(`)C-*A`NLW{0k3>Yh9+^54b{6(?iXZhtk6 zGsw_&#fhgGna6lEZdo6)T>q9q;YE?bmT4O&)@km$^&@%8pKrxmgl@X+S0eJ zJ=;h#WX@z|PJ7Sb)m_r!3PxOYzV$}OH^1+h_%3;wgZ|{ekKDf|i*Ymd)mu#{+J64S zT3#zs@$DjOk{QBvM5v$#w&QH5j{&vCfxc^1hXKP58EdOXeW0e@= z&fjg)HurwK*=&8Gy}E$qvB8(O`fZ7m*}bOUYy2Q^E@1MyKX$LmrG=SPWWM>W<2Wg& zd-}dbZ|#)C(pI(!x1t}On56bG(s{|r;M$ZYsdGb4utiM_SaNjk&y33BQ+LiX^K}T2 zd9dT_d-d%eMkV$+%<2bTBre^_FVv?~AQL&I*m1vS+Aa2HyEiCJE8cNDZNAmXy+&7G zpU^Z27Wk?7iML`-^BjZ4^GY}(9339p1TFIOzVh&{7kg7<(p2Wz(^8+DSZMF(xbN_q4V=;-k_HWs{ ziBTG=_qS*&ZQuVh|4g^>gM=5F@poUA?#uc6=20_4C%d@b#^c|bE8ZO0vESgCXm#$9 zCkqa1^9B?doliW-dS7*N>wNPElCJURIL;iqQT(!VrFtU)~TARtQw9F(JFI1r`lgv zR#0UNoqcZIBY6px+`}?QPs=k4&%2Z|-#h1er*5OBWKqrK@9a;OzkhJNF|Al7Z0d~j z;?H9IEMgjc?iN@iw4VDhz3&{`%!flf)`#?@&o7%S@b#LBuTI6IkgxZ|e zaC{Ja!{C$uO2frYk3_PKd&~N?^kSb}6!}?r)v(iYwVXs0`<5ybHO0qEk2QT_RhF82 z?T^(yXXRy&1GLv1pZ~sV?$uoltzIp~5_=3TUUbTNyJAs*Sy=Yhs#C2D;d&ZE45l8{)dM`g@Oe`+s_Ru3I;sSHg$en%=PKg)0ypk(_XNgdbV-jvU$l57}$Kb zyx`d3bzP`yQeN|IkBIXMEw<+;J;|8D`6$RtaQ?6MZ@SxUQ$JrjZpv$w@$2`QwXai; zsCMi(+q=K$OwyGE;Ui2g@AM|v=KuN?E->lUb&o0m*6wY=g5vugv)`)tUhKT&{uE2C zNrh4}$25*z4nEz;@bzHK4cVJNnrC${U1S{5cu4@b$5DC(>?jBwCduQd#C?@k9|IGruAaUvvUJCs$`#ke$Mv$oZ7PctJiOBzZ3rN zkAZ$LhjiSMvo}4}JC5@2`*izG$BiBH<<{rdnEOW5r@H)j|M#QY^f@!1RaqPg^g8kF zSk~O&ztc3&315CDvCPK$?B0meF3Ja(dO~~pZgPvwTC^+tAP;Bqq1p*-3|~^N-*?O)AFO%l<{UcKXx`+uute&?81O6jjxo?IxV|ZpQZd) zn6G3`$n7=nza3~dG&&WqL!vnTgTM3TPT7~2{x06V>drY`{hjM(*>V*c=^KcvU&vA4 zSM%{)r9IP{<ZEa|l9 zJvl3<=;{r|na`prvLBZ7-Yl>5vrsX<*IrqekadWtA z&Ecnd5vIBhFSmbx^u1HLAyp)4!?ee|dH1iCg~vs%ynL}LwtbV6r$S%kKCfvpj|-fp z90|U3`*dJH$S$4mPPL-%>f$?hw6IJKRbpahUb;~;CR^3mKhfiWRl1^L`?X1iOOD@~ zsqyvem)+rYJCD0+)Lor)RkF)SCiP;_--z6Oi^bnF^Jo+=SRZ`+-?p8fYQL`As(*Fu z&b;2};;%t->(8z1w@2AIfrR?uPN-VWbFA^J>kBylh#?QUt5-Lb7}mlck$wu z$W!f~PAu+Fm)kY-?%V12mM&i%ocsEMn*XdwMe+Mm#!MS7)I17)bf+zP(p}TexH&85 zp3f_^{}DI6L@RkB`zjGhlby@w%Le!Sy|GvO&f4hv>AN;EIGar3d>y8AWuER6$D?+a zB=+b1brXECar@>wcWeHCEt{k7w?QIwW0`;3V}bakY0G!_ey_Q5&Dx4@#Ywx$yT3IL z_4xdW7vFz>rLw_HQ@J_$?uQEt-xgX~9ke!`{55JvQuFPfUk*3r3rAcWF>8;+{?xF%nXyB<^?G1uRC`HUGqOdflkuog^DTEyaf;FtQTbIZRqo!*um zDzMH!`)?g9^OWAY?718AW}RH`f1>*1u^zVuR$raf=bo`GuA7#>>rn02Q+DNV)*jA# z{%qF7&4+d$$;_!NeWdm3R95Vxv)NxKG*4a9{W3U&DO!5dtuEhCNuk#pHk_My((W@3w&s}1C#N87X32v}mW_o?;*K2Y0 zcP#iOuq}GxvUcSXpZ6-p=N>)eXuELaqD+qHTITzF!Ar}e7hJhf&%%*z(#p^OzIRp5 ziyw2cB=>xKwr7KM-u%@CuiqZ4yfR&W!lsWqjz}-u%BbZ(NkB?}qubgu*S_uU`m29> zX6|&6cW<@S5BYdb^#0s=DKpO3wEas>yTzQWjeo7YrCk;te3GEh#Q4dvB3jevZ#gExr`LQmYI56erY>(sq(@_PbrC9qn|2#Uy^i}6kPXWxik4-ci?Kl zwBIY8pC3Ld!kN@KfvrSoy5b6lYPP5Q8JfJ#2XAoTt@v;+eX{qb+{q3R4c(c~ln*Ge z9bgh}R}^qkP}}$C=k`0t|1G(p8!=0*D{G!u`D%?NKLl0(pNanbZ9k_$f{#LxN5me} zFuBWF+|!r#p0k=>diH2ZAxjPmOS?ix`%;@XlLT1vE<-mBn=7e=I9^a;yBFo$YY$)+)KLC-3a=tP`x=>#LXlQH^oG@wt<8 zKPDP2`C%5Y%|eQguWa}Hzn}SL7FCOd^9p@tXWFy#=C17(6P;>K$R{1Yxbbko^SNBJ z%1-a@iFFS#nW$C3ZS}9(9Tl>yX*~h6h+K^N8-Fmy9 zLavJpj+fa*w(yDYeW|6i+%l~7mCZ1)F zc~ZTr^w7gCGWJQ!W!l`2t!SKAdd9)ND?tC|#^oY)a_TGYhPAT!`^?Y1lD~rcb&Fnh zgCyr4IRQZK!H>;Jibqw7tjbd3F&&+=vGcm0H!OF>u^L8uV z*Zj4vxSFr3vi?`_^KF_LcXZ#MFXdbrsOKm$EvIIt+Erb7p4?TRbrA zc~Z&#(${q{yEj_zHrW17`u-12hD&|&(?5zR>Dw85Z4o*d7_-ytujng{qMa>MOv?AH zagG|uI$e?4_?Zc@~|WBVMemfc`@AkBT|V?pn< zn6K6Kf45|d&zl*|@#v#VghI`u*hY(AReQn&A_JV?8a)`X2k|&+h|5P`-u6*~!A?=9T* zifLlY%bIsLR3@GYshTF7q;9u-mIkxm>61%#FJ9`#z2!6yx00x!;3{^O!b9z?tL?s@ zJh|es%OusJ6_X$NMKlz?e4SVP_Uu$AEnTNAca4-yOlO#0{Uaapy#L=%-|hbk`V~8E z82=@BE{M$VzhL_A{KU_5^&c&AI8&May7rGp%i%i{-=5m6;mXpb?)qvh3)b5H)> zVr^s;^?aTAj^k^ zdtkS~f!Fi=Kb#{)B4PR-`OgC^s49L z3d6mR-5U0C8D3m`_=uYKf#xS`ZtB_A&pT+%!MCGtlB|Ab0sq zIaTfe<417|l$3<7WqWvqA5G+Hx_YpI-_T>pxl{A@zTm1}|0H4W>A8zDHa1x$d|2gt zo8^RCsl=9-Aws#`9Q{)_)$X&H$g|a(*P@q`kHN)%XRTAi-nqR(vjtCVxav~WBpoiR`S>710xT{c>}^2A$IqX$yUvD)o8(r{f|I#f2Tr z)SYEgzA{ORy=F0qY*;aA`@Hp~=WVMGT+M9cd>?W8k6?7jiej!M=G(;B&+0$D(8eao z$G7U#b+D~}TjZJ82>$KkLlg0N|=#u5Czi;20 z{qLPzudHW$(dM=PhT|Q{Q;%MGsj{Vo}lJ%z< zLawX{IeDl%X2};Jix}IVALLG&YTePTKj4|y(yk$TL8p0!-CE8rTh-kcY6LrMi?vSq zsm^NYF*5J$3=%2CvTZ+W?e*d$?cJ%Ojd#?$NM|DPY9H)T)GS9nG1 zSLwxvp2u#wp&L6nW08c+FOFN!Eq4k?pXiSFzrkj6apH5wi9e^9G^bAEn5T01-R$Pz$_Gcp$Jk9`|D|LU}p z6OOs`$~=@};&1-#G~aWIc+LIY)m8u7TaR|k6*Ib?D0goW=k*A8>7R|t!O_W^m9FsZ zZf;+7ccJ+&p2iF)*NV%q2TlurQ-68QmOY;x@9i^e5_lIX@%I1c>Bn!ZemMWni|XffpY7}8 zG=H!w8>W8pxbI-5$ggtUzvh8$_`j2f_g6{gsnopo{@WZKFEm|2Az;e;JMaGmZd`F( z^w6feYQfF76$)=jpI};Llk2yTL8vQAM`#z%!Wo~!1O#iQ+>fFwy(vM=I&c$pz9Qf|QwY}>eO_*}LX==yvMW!vRLd8#CFTTvz>~^ZFG|nE@In^RiXs@~>#p3JVl&GLz0f^PDx zz88{A(t9Pmiu@#8wsQwD)=k~>v#Vy=DOP>acWinNAA}eEetU^EDdoz2Wv)1dhWbbI zoSkHzsQR+3=b3b;^!n4x($d`Wwyov+t3S^^zAiNGOo_)~&rZFZlc!b`eBsJevS#o7 z^rkd=zNG=nS5LX094+ZjkGCH^Ig9&D&l_HyUcIG@SDt!%^PcsUi@6MJT&8PwSX}*k z@P6F-j{R4kbug#$mwiiVmEn|3mf_NM^9&8&ce-Kc)iSRC4kZkiJ0~3sxoRHkVj38E z=d#o!y|jrw8G$N1rxQ1O@y%9^vayS`zd1YkeK6k~VXl%R?SJ>ao%N$nP{qBT<*2m#=+_@8YW!`WEKeD%4zt48>aU;e~*K6;dy)xo{c*tU+g42R`m!6C4 zoUufNS>fA*NBg;pS++c=z7?h07vto%&4OQ%!;JBQ@5%1k(DW9GECzi~v&DO_#soim zyZpf2%|BZo{o5U1d39>|%%@>1U&RTmu)pwW@^&}dITkCE=lFOi9_F|CaA3{7PtWV$ zXY=R%TvT_Sa}VPU-nN>5D_G?EYZ@;eGfz`_KWWbi{Wt4n9CFOQtUSN_gX`=2+RJxl zRa-iBsK?o9ZuIng*IEC+ardit(yh9j>hjOD`1TasV)*Uhef|M!jE~nfrcRqryfb(@ ze|1alL}+%21~OZr>-zKW^=r zdToV}+TOWLMN5ND38z{*9mrj(n=tR3%E6^3A=`JQw>o^9R#m6?`1C>6=QE%0-5vk; zoLY9X;--(1=P%yB_oFv{UtIF^`2TT?EQ-#(g-`Fv&T1FlF8t`c3*!_MMIru=PMz~+ zyq9_V?`WEDGRur`4&`_)X2u@L+TfptkArMEkKbS_Og>P={qg@zzZE=LUWe~E-?vK$ zt$(O2o!>LF{BUv8_XFQ1T$-XC+1eVcaDTRWKHqw~4V7OPMwLJPoxiW>=DJ*cefvuf zE{Uh?`gK*>>|I6i@qg)`J{On0zqh$OGI@9Oho9&FCWg632S3|()?dA}Y|9d^Hs14k zQ5UwP{M_{`|KIWc^7{85kIZtZ@O#Q8hfnv1@3?Oh`?@IAeq+JS8-7=v-i`&C z>(*|taR2vH@$u}uJ6lp}KCU#{EaJ#wo|~G<^K)NEz_I=tH-DbJVQilEWu*0-91 zDVOI8*ULrT?s%sb>nZnB=J`#!f5JfVf%+pY8INb(@qd(T7hMiMc}YY2(0OzF%X=%f+{?YPdO^~y z)Tb|Q2X$3U^Zy&W@la*Tsy7@z-QG<7;i6`nZZPwmvv6{k0pslg*2XtEt3_u=E%ppz zK5E+1<&oyP{PdaNS?OneV{gXIJTF`Rz4oD>b@Dx@6CF{FNsN*+KC4A|Pk!UB{Ozcl z0^gCKk0)0;Ssjl(`EG9BzgMSQ1a7&DxobElEoWo8n?7U0OSL7byXF7wPrq|-?zy#* zdhSYvZ&j{sIkNAS^sh+<+kdrXpB0(OTDxzl-OOb3Q2F9W*VD06EcP8dFk|VHXS@F_ zjr+6c74z;Rp+cJ;rQG(Mv!FqA(OQY4)?VjMPF8d`TD-JcB6CiCa`C>isRh-N`Cn>< z&q@|^PWsN%+H`-`AF0ff8=h@kyDrRQq1WvgAvUJ$K89}#%D4L5INo7<*o9}Y_%SW+ z#oWf5MP(X1#11g__0{|latN#0bLe=uKugS&=C5pT6kepa^Q}C@B>F^f`jl7yE_Kg$ zWla1WeD|2L@8R{U_GRk2Y%PlFG#2vB=2KdjRJ(?;rkUqKm`33F%46&<2g(aJ?RoM& zKP~-v{g;f|qs6~sAKd@E|LNgp>-OH3eBgU(&q21vv)6A%RW}LUs!v~b|3L0p-QwlE z^YmImm*l$&uxV*Ix_(O$@dyvrI$t^W_u4-Sf)h@{{jt_%lMn{s@m5Y%)ngB6?aTI zx$f(}mBz>H*xQb(s7PK~bnf|lkx8G0j@s95xw&`E`^@|cnfnD+eDSHdIPGEfwX^Zt zYJXomUuN;$@Gsx}iltNZoHPTrv>e>=>fF|#L&t8ZY`-?OgX_9OOZKE?H-*2cpJ=)% z=)9O$WZ(Gj36I-&* z>e}OP?^jOyxBP7A6^re8&BgD3PVh-eol&r@(U@EIT+Zpk`YYGZV(P8W37S(=6BJwI zw&Y0i%0(-Zj%|2u*6{qM{jsf2e<&)tt@Ze7sHhtMdFH!K&tru+n#><)eczM#+2-!L z%3TWAgsycKELh~ADwcBgL*wICrPRaGH7a6t-rdFAHEO<#_s%joeY1K& z=fyculUwxe=vW%Jt=$>9_e#a12DhF~hrEsME;U~}*<*>Xu$b<(HNHx1cbyEBK2K%2 z&b#0y|KH8F;u1_t_!W2=M8 zR&I`eX0EWWn`noVjE~43jogKfOSu>)2&vTF{u7_SE8Zq9?{#}xXBcN6^UHM+%E$NZ zoR`lkQae-N?6<#r-l|TUvSzmEvU@v!|No?Rbo;7pN2Mxi!uDGz8fk@XoX_&&%)8hr z!XGaSoVyy zYt!)jKJPEKjUKv;3G)Od&dKIbd9?5MJCB+d3*$Wu9^HBSo^iGJw0n#EwPTvZXXL+Z zW7=MD)`mIiN%EwHhaNJqmA0&6ZFc$RU9&X$jj;{Q{3qxQjew8Z#M6E-E2&l=3@Y5v!ntdPUnz^i zGX6TXXjRb_2_9k-F(i+JofAlR*SQI5x-ka%JWwj{QtI7s? zCG$mhd3Q~l8nP~s_r%6(E3La9KRJG%-2T?I`1m(LhH?j~3&-W^56u-87FL;)8u!44 z^L3ta#~o#rrqxz#|tnCMz8Ui@o`=r3)9i=4DIy5!h)~j>-`g1 z98NqJRu|XYn3mjnrlEA-iIplGi{AY7zN4=H+kRc{qb3GdxdmV5F;{Jg)-jFQx97#G z@bc+$9Dn3~S}hmwSP+^xUF`Xv-><8)|2ynIyndG#yTsF~15!4MOLSDHGORx~`LkEi zGqs&-ntUQvRXzRYyinxx=XoZcenmyrwZ3u9#2+!XMuP8h6dh8QuQ;Ptd1kc_uVRwB z0E>gNL?#QD^k-K)W;1i3nG3ikgf2aw{`F_%h2Lzao9gF#UvIxMW9KiIb@5N< zJo+o?FxBFc5Q9_#|Alv=f+zBw<{vN>P-V~(OgOO5_=!S9cVLUslpD3*Ebq7;nBa8a zhW7W8zsvI&Cfu>U)16<%!Lo!=Lc!!$>HgX8xTY8WS$T%DRq}UYdd&;dOB{1KRupgh zXnA9AYWweh&0qZjbMG$y|1Op*C35+vnb*5BHwiYJRxwB~cR$P-b0p=x&vUyuA4HonNiV|A$X-!BRiFLyfp+4`vX@#UA#Tc?Pp)bC^c=(6TsL}1b`*LpeU zxx&{1+%7VGU%S8N!ro1hRSgci{`WuIaXPPhJL`eBQ$rONE_MhJ(30Rt^qHb?Xxq0F zdEZ<6b06wvgfpB|Tf1=j6}|&EEqfjGLVy2@p8xX~PwFX$my?{-UyCy)Dl#$b>lAw7 z!Vvl3Oy=GY^V)X@p9}F^du2DLb^aflfA25<^}TCQYrTZUUm!_4PWtJ!*uV#yxHt0@ zDm+%1n-lm$T5k1*b=%m!U(0sC8@fFQW4Fb`Z@;2$ZrN-XnX~4^Ox4-dd*|)0II7us++m5# z;f6xzn4()}q>3B&9Gy}1JMY=&qHXO=4fld#-)a6=kM0pR^QskPs{FEh_T7!lvUzWp zC2dcu-Nsiud+px`mexs4d}0jKSPm;5<4kelsOsU^_s@P_i95duuWsCo?|%*+o@&0r z>b9)ogOWpHd*+&Fhy2{HbEY)htaJx!7oUBMUi0Eld%L9eL{+i3o@mH0KOzvp9n)X4 z<)viy@3-~Q(?hj>oL6!@Rf4=+68vAap`mvvyvOYKNGBGGF+V=!9I>{pWtY zy7Ket*{P{|`zz+=eu!XvapX*n_4QrxEKF6~W~xO?UuwRPQ{%dUWmeYSp7q?96VBaO zXS>J$(T0@3X^i>jKT4|q?b86RjRGzqD+}k_h zyq`zfp7hlhgReAiWLJ;hq~|=h{+IOPVE+A|5BT5xwfff`_y528V-MPR8ne_p$faqn ztWwlFrp!Of+192x`LJi@$D)~U>_a85-MC`*zEl3s+w)>;Z-nfx{G3+&ZT6y{`R^Z} zuX$#C@&B*&|Eq5Ax~l!|=Dx>kzppfH__v`%Fz847nXBp{2MWxpn@qa)nm-A;d+T*+ zSc{mUu=p#c8Ee++^aSfBZ`yQz=kHsFF0vM(-o`cuUTfboV*M3Tn{rPmnET8g>5odY zyI(Nm`lgC89h=cr*t($0>+Ia;UFLW5&P6?QSmxToj`PY=Qu{n1xoxCQH&GW9)?6Gcd zjf~-|C8d5>`U=0>H1s|^^>TS+rasF|`(uw+$$g%j$lUl~_ru0XhW+PH${*-$yQHjs zwXq`EVZH;G+*ao0t-(n(U3~0QW_WeqoL4<(V{F@>{-6JEsn<39mz?UM@m}xp^E>Ve zSt^SD4722~ef|DrA@jp*{f~U6haUfo@eWwlaU(2l^Qk?@W}cXwd!u;m-{(%%5l?(g zf3uz_{Sa1jtHb|@&n-5G+!a-qzoa(Wnq(SwxL^J}ZAYZh@24}O&NOgV@~Ylb7EFo` zUb)V{Ms`Nu{7b8(??3Mfi+@7`TwA~CPg%?6{AsJVxu4Q0a(L`_ zf^Wjx)(KTh=06CqzrX3qd*OnM-0bToxC&V1~4$q6tjsz~`)DQqtL;gDrY`@s^UckfdH=e67T@`uy7A(@5M|3< zO;3CqJQ`A4_MeR3btqL#uP`u*`7m?KlEj6ptbfT(+9dQ*?3i$Yd0Nz}ZHkLFUC>aF zc$)l}U45}prb39xv*$wnGdp~~vv`F}yBBWB*yR1b|LvM``#a|ztI1vWV1H9YYHj01N-l2PW@3Gsvo;}dhK5b=imr3mefB0ec170PsS6ddJ&_R8>4}@9UDSZ;XmW1O8uHy=3_dlU2G?PFlX^wdL26 zdGY?*vWXh1LC=?M>6rB{@3LZ8_t)8<=ddnaK6#6$lcSu-$z89q))sM|+{xAvmUE(@ z{(rdpUD4_CrSJbXm(PqWOZz8eIaAKm_~Aznmeu>_^#yJJ-Wrs9<@g;L*>dahr&IMT z&EsD)vaS;;XJTPsXxNckGbi}Zsjt2jb9N=OyBKyA^ExT5QLVR~d4g>tZ{xKlA&=($ zos+(=YpQ&8=UpU|ogwGRHWi12IjN2MA|A}ef#;XAzAzH&^bL6z^8Gced=?!celfQj?U0{7C=B3&G&M@8t)p=Kc&UtXiT9jd8VQ_k@!6V(Ws1m29pX+!J z=>7I^y6+qzI9EkdFqxxSS@Ducx9&yLle@I19NE3`)cfVjz8OtC>9^LW&}$x42Y)=a;(c=wVEK36X;V`XAk zS*Db_XUd}^Lfsqzy=^#y+w1>sk!YXB{sPtI)`@907j8elRc%w)61YrCA zJg)S@Y-dZ459524{>vN%0*}o7SC;oEKJuLq7{5Hd#2~a$AW=Ey0%USJaBhPH<_a`adskrt*^RK$uy%TS||1s}F ztXlEi+4b`K`vMP?CcW0bH#3XHK~AH7cJ_BauRl%94tks?t={^dI8wZ!$@KdX^FW!_ z6)oS_De^F8TF;M}+-%4kV$IND{EoBQ$b`job?DNa+~EPfI_nvJ%Q%oV=hIabx)K+#Oh2n=W)-IHt(LaaqV@sD zq>>*Y)0j+y-hEvBJJnXL?02qDhqc8lQ@2xY8vkCZO|X=4+$` z^zhqj0`8YuY5y}W7L|W}MPv2$wZ_jqjw&kQO* zE%(k>?70^a`?_rJ#n(kf!m(j-OEl`f1kI_o&vsEsIrzTiXYNAJB_=GQ3tXfhunDwo zP+84qx+DMZXT|Dg{})G|4p-eV+kQf3gS=%;x%=cDNprciRLlg!AMZcC{l0z&@2aP> z_wn$n@8m82vu?^0NwNKFe;x3<+&^cBY?HV3=IcGe8y^>m>u&Cu)56l}6IVib^`{~cfKXG&iXMI2u*SGu6#SVzv|KCr}+3UM}dXQZ5 zle05BtU9YMJrl{waWC}`(mNqIREskR4BHreiP*Vxv1pF1*zN; zPpLD`F)mB4tv&mFgNC0@%z`_Ac-!mV@DzNnEB5>4+cEL4)QPC4NtWyGOFX>yyE

4sY_XW&vt!IxwT5bBC4QeCtKo%2UgM2cRpTfe%G<1Pdx6k zn5Kf3#m2%HSJmyq7o0fJs)0wGCm zeTq`E!>4*jipkCYT4vbE95!d|si4O^%cc4<=Bj>b-~Lbh=7KG8*V)+~^vw9!CMuqO z?ET&7|MwobR?6R~Uh(Og!%~Hwg+4FemQ0x2Dt%J;(t)JTpDzEEW9O+l3&}aYxO?IM z$`!jN_M|^#ayk~@ys4|$+f3(zrC`L?IjyJ8Pu8kB>Y1>~O4}@IQ)y?b(t>}6ovwmC zTRYrk`M#8U{<*9;pEL5jN9CLT*FL*A=N(GYw#(dRdFJmo_1$;1_vcD-Mp(+_{P1YZ zbhg#F+Wh*W?~#zYIQ#QIzWv}(F~7UB*IaG)PwnR!y6Uq+1NN8b#a?>&MN2(51np2u2&1=q?EvF1u)i>># z!rR`i&-G|)f-UFO-|v4;vy}40XzcW$Qj^}V${)h8|?T4FhgT=TvHT?e| z%%O5c<=64ps-@+34y3*hyD^bfdg<)f8+KXF+Oc+_O9Stz@=v9DGgeM{ux{`B`kCjL z7ev;6SSz^n+m+mpeW!gV9FC8=cSlfXa8>rWc?1`Qxwbgf%mhHw2gby8kn8s%f2!-&6DCt1+R^B7gPH ze*W>P(Y@LAv2W{s?!B(~(bdR$XJNHw#r+@O)4uG@Obu*|`=hPc-~V!Ur2M4q6%}<0 zi$6c(TG6rexexETuL6yU5v+n!`2*5IUf%vAlpXCiZ&A!s-M?igFW-A+IAcSVu7^*6 z=bE#fIe!@>l~e4lKNd>v=Cu)X2wwBvFn`mz50^c95wb&{KVtTE$=$SxIMwe@S&x@PfkY~Hx=nxLra(`Tm+X?s6) z*qGhy>7>ISGc!x=@>I#6osusMR&3t6?9K!sHnxsa zJTC=0{J%EE1U(ly`6y>wcA(w+4{C=eB)^jVeOP)z!jpr=ucYP+R6jJCzIj0ehnZWY zh?qtYqvz3&0o;!rAH`Zs-g2`wXoY&Li=3I(J^N#=FIXfLe{`6=_&CG+n)@oJI+=>E zk?-1L%a7i2pI`lyclWjZ4<}DnO|h=rq`N`mdhFL{7sEVz?w)e^w&8YOY2wjeqR%Cd zWPG;Jbl_`L&-5wt%r_OBpL=a@t?t7;z1x;ek~C+~ zUcKd}x9Pe^DoH^{Pi8#SXx$Z7u%TNxlzOap)`O#|uICCv z*sZLuF;(rn_s+3Aqk6Ssdj z`{ufR=f)y2eepvt#Y*RDOO`JAb#YaCk&#G(%+HMA%^NLcbR!$A78f*DDlpq@tYG?a zT=jI3!){>*E?2$@^1gD<4t+H6R9dMM>5>@KvsvmVgVvP0>kM=rGx9jxJGCfht1PR9 z4EM8r5n-;!FPZPTS-Ph1nvC&n*|p!Vtb1W>#m{$RQ`t)1hl?ckyx0|SgV!rRB<=i% z+><(!CdKU%*!ty)*1m>&Z4n7MQCX|~X58-FC8>EiTZz&B(}l#0H64?>X4PMqb*tS~ z_osg=bE2q`@qEjW$N$<6%{c#b^_7?DMT4g$?*3`WyzBHe>AG;bAz_C%kd{lO$YyoOwM8Q zT{Jz@HcoMg_WB&f18168(0ZC>=2l`68y!ik}m0+e6* zBn2I@QCqi&Q~XlRCHbX2du}SvKDPTa-@o-oZ-*2INLM{eo-oDIe@f1t-^Z56m3|Ii z-eJYJ`GV4n6W`e9)JG(%d)u@x^7A+w8nasC`r4x$;& z2c?O7?(;W^WMpP&Y}lp#nt!{ohk(E83C;EXB2k-l=0>g3=GnoN;3)BNrPZeh5x+G8 zf)1?~o+pDgF}Ixccd9O&b|`9c{{P44ceT~mo=m;JEozbTp53JvW_^)7c*lOSB8ksX5Y`DQLJGi z!0}!0ib}VC?hL&^+eI}u44)TDO-PQNqO&Y5OMs#E^0kO%8)y7faCy;j(<4wi;LalZ zJN)~$UcdY0-qLrIPET6Gb9TE$=v~?4sanzNZMuxrPM%3Q+}Aoi@Hx|-4xfvcChmXr z@wCMp^wWqn0=iiOJRy%dwQq95{;_XZAuD=Ldo3o|X zX|{)mrp5a$$L}8O(#<=pR;Vy(qHgRo9n}rxJ8oP^SpPalx};R}@47Whj!xb4b=L03 zi=~g4KKQ|r<-A6@UaE87t|fCiFaPa0VPJf4Ywmh4pZAKN@9kSB7yk2)-}MdgUH`1s zKKv7DWasdt`H4fL%EDhxJ}>Nlu`DR4X+C^#Lc@oThDQHg=CCr&zW47#pMk^qiGe#V z9LiJBbAH}5gKMKqk{#33OY%PRS~o2Txqhn=h`}p2IgF30!rDyNQTz_Qr{7H|+;<^2M4+rmF$yWWhp!LHHbFC=Ojm8T7 zd9JHn;~z2~5r~(atZSv`s_y=6$2F;q8@zgjn*TjGSey6vAV>N8=l8S?S>J3wHXXqOa(Z3OUJSxA&hQ8iwJo!J%A>O#_SE@fO_|qii zuaWy=)@d>OEtR(q+CMqj-Ctk+Hgx;F|GBa2{DdzqO{@GJ`Ma&GE5pP^-R}+Cl4XHCOXk9*{h$qMH4{=CuAR^`o7$Kh1S|d(8TaJcsP- zfo(rS>1Mz6ioX&SN^>9LH*Y6{`a%|L(5FUy}v8wC+ExepErA&f6b+$ zV#|taJb`C3-|am9y{9t$_37qyDeo8Oy?k1HXVz(>n5cI;PeW~1-VkBbEMeLEt0eo! zQvQH?C8dwr6C_tMWY)T_Wiraj?fU1w#cKTt?W$!SOMFkTzb`Y>=-kSm6s6F#GSFon zM|F5uX1`Lajops}%=|eQp9?DXo9FGV^Zohw{Jxz|-l}sfIs|_H>sl%DkyYs8O~+{y zL>Y=E@^4SdEbOp#=dH~%oLtg67`5Ua zUk(U++V)uVnr+Nh?I?rSWjl_4KC37{rS662V_ieVYY7E4KmI?sa$K_`;7}pgyp9=b zoV_XqZl`n_TdmoDQd5sxnW@k?Zo>(K$14=%GNcy^X+L=w-H^DJZPVk(VAqwbGA=q9 z3~sfu`5Xq3f5m^*9^;;M;S~xA}k1Uap(=*!AVRZ}V-wChgsS{V#isar|Yk zRSJKCzu%kn=j{JFr!5PM!}9|?CNlX)@^R;fOueSexgg>5+rL|W&N&(-KIiBqH>Ir8 z!WA|5>OB8sGw*!0%ktOSmE|d{!acqdg1pT;6rZY# z3C%l{b4zaSwd+CU2cAW4cNcP)dVisU$Tdy|d*k|*4J*%;2yFPh+JB!-&@~rv^VrvS zW<^h%SscCpQ2=jO(pS+-41U2O-rwpoZtLG|%)K{ja>%i3(hkxWUmZMb{M^MX?%CJ; zn%fd;o&ie&%HElu@3s5;q5b~U4}ahPUlx4kC-FB{!07%ETvt`H=n)QoxMRKNm(Or(WNDy1>BwY=;-W^kS;yWH|vqS zaCF~eRmLN;S9`gp$e9Z+mA`i6);d;(N9t~2RgH%>WH$*UemX0)O5av%`H8g7VJI|AL`Y+YmZi_c9{Nqc;D{DwmnIc z&-Lw#oqybgp|$>FcYI16^M$+89`!$4Y}Pa~xujc`{dmgtRbXEX2k-hKfsSjPH~%uX z{MfMWqOeJ~R`ag6lRWf7l^9ypE;W5L^Eokf{$feRYYTF`zsnVU4ijOQ*(tzYQT92f z@4*qZ&2twr9!e2z*D>zpIaeaKu9a;`MBe2;Q+gHnS1qkfI1=W%wjOnuI4V+{qXr5RzZX)N&M<^4lHS z((4M}ZJ&MisLG<9yW~86t(wsOW5G$`*BvE4MbEvxvv$*#_;8w%LO5>_Z!R#Wt14MG*6uVgzHUI-?bfQR6orXxZZFzxPQ?LHs*Ar z&2@}s3|S2|%g%J1JpJT*$(HCIE$#x#HO7yQGA{L8={t92@v|=BlO}ve*BlBl|M@}a zc+ZVjfs-|+w@eLfw^QJeue`x*D#^?_tM6;Xv{&wN{0|?z-Kki4z&y(KyF;5 zIsPx1XNLJ>7sgXY+Vb-|4u0#-w`KmMaIq6tFN!W{UblV{(*0J?B5yhlG}_n z73c`>OrPPP^63VbZfyD19aEjo?lu!&W5w~*$>R*WV&t^k82z8M>otvMuj21odwnLi zm9M^**TH{GLX2iYVWzfS?)Ps;s=mrS@$}68D;n!ITZ#ot%k5q2B+zw4Qc2RVi9L6- z-5ztI^VaLKVms=<+|lf#nqk9kBgapdM%_l$u)cK>VT>F zQ&;SnsJ<|kLw;xI^i}%~++o|#D*oxVZdrR+>)j27KmXj{x8?VYb=zNWPP*tMK6RO? z@E^Ss>K2|!kL|v+em#~s<^8=^v-)301Pf+<)O1~!@N%i!Q)xq?KS3PVBZ{Zmu5##& zz14D1z#&<3^1K)8{=N3k(#!IZyj!*B(wf~qns0+SS2b4!URb>Nz)G> z_BYKoco+G3`U~AN-nQ|=^AngNH>hW@zi(!_C6yiP9lEn!V1dVRRpvIQGL7q+m)$L< z^_s2H?by&KZ<%1H_gGfxxM=yo|1AcuMU&J^tXbTXZ(bB^ITLY6Q|)x{|Hs$svlnf= za87FRs~xWU=U-``YHX8t@>N*I$vCm-&yuzK?5}FSub%%!DM(P`&cr8AR+(=2xV-9p zO=aKJiCR2FU3zJZ_|4qbHt#%#mDNTx{}AL*%Mhl@|?*It39aMV6!8&c|mP9 zSMj+Qi7^jtnv}OQvx~3qDc{N69lN+8)A_1k=hHpEf;Mx9=*#`eIs8(6NBar8RvpRz ztMa~tZQD7?eA~`BuHC(pJ*F8esO`VI=*7lOOcuowp=&Jc&UAbIZ% z2-zPDEikKbewirjpBU;|7yM?`-#488`bzIyt0srdoX7O$Sb zTF*Y;^{&xFAa&M1XAe!Swi{8#(u)g2_xai%HOz?dH)Y{|>^-G9!@9NTO@&Km^W&On z^}oCGeXhByO^!WQ^zhg5Xu03-PWf)X8^7wc;TQJ(%O@WSTXUkX`(6K^6R&@-{uij< z_3Ts7u7sqz#v?qx96A^3q$F>;_cO+WlffvxusUA1&0?^K*Q5M`d4i>zv23!!MnwKz-R$?}Ibn#{Dhr{e3n# zH1DEPr_N@jn6{j~pSx972|wjH%B65?e|^NS$yVQ=y^c~4R(WxH>hug*&&0RUfj@6b z1b@j0crsb_(*?C-(bXsH6sn#2Wv=bvT;8c&zOE*Q@d2~l8wtCl3?F&JeN!hL+2ySt zZXMD-f5qn=TbkSZf0W0#E_-s~`km72(e^)2`m=SXE#DUqb$3tPVMFbxT&9hId5dn$ zG}XQu7s}z-C;U!!{$8(V+auSdJWpe6aMKp(-dwqQo#}(#o3BbHt5;3h^#9@Cb;dEv zc0bNtw{lPZ4beMwD^Kq(YRwW@bWw@lSaM?4<>XbSdJf`;4?WOGjedHQXOV>dyHB^> z_cGM^mqzBl2{TD4m?pHp|X{%4Xg^P4K! z`gx1KX~y&DOA(1D-A+{T2rOFi=h=y<$1Kioxw+#X_t|r#pKWlkn7lNSh4XQL$AX=^ zLj1OU6=HtA+dE8Uc43qF50Q02uj3Ewj=VN=ecsCwf7?y{vu+9r)_>Uje*e?k|Np+2 zd^xT$$|8zA@o+o;{%KnZ9=1&nyo`vipJzGdgd zq;+rKx-iDUbIE&?pq!*WZ?Ch`feikSuWD|t>dMo#a-2JlZ|q&Yqm~prd)t+o$j)v@hq`-2h&Niy3Y5qyK)auCJ zZpZu+o#l7DeU)$j=LYxu3$~8l*|s5bmp)=Dbo_kvxz)?cFV~iCzxO(e?|o@$s&#L- zs&J{x(I-1vU#~NLz!Wggk7)w`SDxL+=D#bC-CUBg&c5Mb>xvTvttpbTL?3d!zxjYG zOmF)~DM8Vdeo6Ze8g9|PQsrMg`^bXy`}|xgZH^A*y9K6qJf7n%mbq5=k*nIv(4}hY zPR8vxG9kF&`?=ZKU)MkTU^iR2WWtO7M?pbr>pSXvw`a{xcqEs2^_u2s^J@1w@^chV zuq^lXZelKI>EJC)NY$-u<6fNIW!U!N2&ZXa(zE>)@Bbf;iM2j)XA75`*!v^C77y3G z;*dV))%9o6gyZ*LXhv6P*xzo|zi{aMiSIjjIXL&!F^UEFwB*b4v-oNJwAsU$6!Jp8 zKlPf>yVsJeGZvN|cjEIA=4c5r>t3|;+k(LUw0o1}TKX>uTeU!d9VNnG~f=Bv7 zei@dIjD@eJ{$Ff!E_B~I#YHFcbEWQbH%jd>``{d;;2iKp((mWFuW{4DO#iOf*72=Z z^CACAxuD+fssAsw@H-s2sCM=Atn{_dSN~;nKf2I&y6&HLQP2aWSK&t+< z{T*8~51598cN}gDny3*RVslW0v2UB&mdu5RbZ;4SKb7%&p(hk{S;cU|o%ofpZ&t~2 z>NK;SGfuzbpC)#8mtV-1B<`k_zK$*oM_v~gO=A(}SmE($&D%FcXY7yh=COKP`y8_W zXP;jh|8w*4gVM!%(X)Q@D=G{4MjbhBUsrvnncprf?z-88MNC`1#N3cA&ydaf?0F>Z zQ$mT1(f@5N8X^t{pF2Hq5HOcWN?EqxP1^+T>RSG<`xWQD+-KtY@S%{mNyuA)fXPbH z1r3jmdutsNPyD>N=HK^{*+;57GhL%PUW7~CnYcJPn;~Lzw>IMJ?R+P7O}Lz(cW)+H@3qOPpFelOr}oLF}zFaO6s-uHS%Tb<+{ zZ4j8cGqy~>=JdA1_3O0lnz=XKX%1iS@uz&=zZcJE+k5ue|N62uJoopf{`&aohq+A{ z4%mDtWOnMP7}IygpciVHG_!Ie?iemrT39u3jG``cE=%{md&_=u zQ)mZ+Gf#uE;;H6|%nxG)X8znI{wLVrqu*7>FsYlHue_QWs{C~u-=|q?>pG)?lHN?- zoF088I?KOS-!e3>V!d%RGbXMMEC!14sQ2b5yrD}ZMPib-LTHj+Hs$cIHbgAs~$A{ z+{z-g=Q>mN)y{15$v5s!k_?p>3SaQmHA?5nW|g};nNmEw3S=s|_vByE&VO^=*w$U- zif$O=IrZ?LLM8o1ypbEeJ4oCTXPn^?IITw2)?Q_9;iihM+u{pnihR-+D1HC?&);8nqo%yN7gPOnD*OL0r|r#L7F8#5 zr92S%>)dc>=T7bCH$SeOYZaVobLsQxWxcvvQ@NzyA8q>de*T-=8s&;D5A`~ir))}N z@X$}N+>vrqDm*&&nUr+Nw6;LkI8TO+7Lo72Y%;jF%*8seO*!sm=9?1V*{5UX@0(up zhi$8$l9;cDdvLSHS2abhLT5)#KUM}V*0cW_F8NK5{J5or)A8xttKMIwo5jyNUS?in z@w4(_^aF<03tR6CI5SyGsDHDsb)Mg{()O1okKUILEKdw4?^D=zr{cSMi^D$U7jjLE z%cYhU@=O1F$Uom|mH4HK%%@p1?YlghQq9jt=7!WSWG`;b@i`MM$QskZxoV&P5k|u$ z7TwlraSM-&F3+^;EPS1pFA_Rsm4y?BTiUz{f!D*|ZDDFkI536v7}qAZ<7(0>OM8}l zadK9ODpqH>-pO@%f`aPq6Ddp2Ze5%AK5p;w9tVFR5%%P-XFh)`esW*E%l4f^_9aiL z#L|N&H{@OLox+j%$D|>o;$>EC*c#CZqCGD}4lc@gqQ0fS?s-?oyvgddO?EH5R`q*w zDmv6JIkfs$+1f{}N_8rB<)0i&l+O4lp5Li@MRv{%`Pbd264y?am~Nxk<#cs&zyrqG z&RKW8rGCw2RC)e%OZZ=*t47|OOV8#dznrWezdBy)WZ{>_$$@hU<=_8)6BT%-a9zsr z%T-Kmvn?E#Y}LvO*fu9HU<;FsvuA&@KP?BaGL=$`Z4;1$gb z2HUm@KKjKGV*OG)$@|)Jr3o$5r*uA^XOjMaaccgpEid=!MsIU?&#OOuDaQ$BS@w44 zgOx1Gy$?Rs@R|R($6C7ft^bjhW3$qx2TxJ2^_X_!tXklXii>Ucf*V3+&-pp2B*2`n z^zuWO1l@;8Rl8q_NP8{W6|^Fo)q#Vf+xG!SNNb{gX6fOTf}b?{%4c#d0U9VA%GE^V2vNMra7_y{xGqdq-{;qs0f)NnUDHArGxXeA#(!V9P;<)40H0|+ z&KKK8vVH{xXtf~#2ogdZNBB_E;@R8m+-uG zoO&^P*VK#GkHuZQeot>^Quo~1tx6}jm z&8tRv+bpd*aWGt!5YL$KJ!Z%NO#%0~iD$1yQvG&0acBh>1Be6>C3cKv2ISUth zPFv=r5cFf=7wK2oHh=O=b2cRObw}(ww0O#m4h=s4HkBsxpEqCJxE;v;YwoQ{QIY3G zcAa<_$#r~V(8eWwl1iLntv=VxN|^4wdn4`|sl{?ht?-h~-7iz7PTtG@sNvVEAcjrX zehD#8=_y}v;_xZakOymcS6up_5oogXuHLH&Z@#VXGg+l?nZ564{+8DT8dD}MlV6w` zwOpZRhs#fn6XFckr4x={xBokfIWpz7vdjY(ara3XeC5~G`Y$(pcyPe!mRF7K-z}T! z_ujSiobPriR%=DZzU99!*cH`A>zEMQyjCJKLuPr9WHvVS(Href@!29nl|5t0TZp?7j zl+RB;F~7r42Vc#^w;G%~dZM2B zo26-b=ZZp`wx_N>kC$3BFn!AT*{YGxuU_eZlc6ZP9Qs;4~DJtWf8 zoFBqi5W)~)+*$j`m;a%T(1x=@<(s_Lu+PbDw}@F~qiM*LusG7|g>%E>$p@|W&&g9{ zoxq;E!{A7h& zKW18PGna(rf(?cYJioF^UrJy8c<0~LU%HVy?lbKF^znYxz3Zk5a~peXkJ(F`ClwrQ zS+mf{^Q!-e+Fkhv3TJS5@pfOZX}Nylf5Lyk>$g@)M4XV4OnxfaGhO`e`S`M#;W~vE zH9y{R50uJZ8~pU4$Gx_$y1NuPLGLL!93efQ#~dd zHoTer#v{q4@`b~)81~2P=~nxXO5Dg$>zh%^PZYYG!Ec*Koa+tjLp5 z5b)||E#s4e|Cnap(rjstxOy*d<{i$8ra3u(ZhVyTKPIx^IrE{JJ-HPTTlii?W;wL1 zSXR1Tu4JO@#XN-=0pCoIG0=G73+&s3*`CVtb z^qqx=FGaqvH7!?htW5SfBFG{jZCC2YR(jA=Kl}T((5ib6UtifjJB(}Lk&ed~B4%-T zN;`=PnUIkN?Q9=L$T+F>#FGOuT`6>zBY(_-c{!+ ze^~2R)pV;p4u8ET{pe0)R9q;m!dg7Y~@rX1N+7Gk<{U3=V=E{{8t zu7y4GJstaw_jK;YfNLq7hSz`2n6drAlY@e9-8*HZTMdk#>uuwZ*!bv()$Ei%Ok0cE zgj|%T<-L`43uV#&Dx5v#$Eh^s!m@YW>y|uQWy@8pd_#O+!9MmQ8}bgGoXkH*xmf&8 zc%87D^u~j&^In)UJXw3zdgF<%>xL({j;KlQ_qUvANS)&7vHOE({~)LG##@b>xtokzchx9^s|eolD% z6?Kb+UQ>T5tA@W5K6G1i_b(0?j$QhKOu{l|25I=-(e-5>Y<)1M8uKAkxHWR;N7 z%A%j#qD*d&8V=RCEmGji{ciW; z*X-VPAE)oK-r`*JDmm%>{o3v7(;Pba3Zw*FSMI2pb!B3Cqv^5~cQziLy#bRMt9n(c z8B?O7tzOi9oBz~pqWE*mnW;kOoRd}Zd?sFwJ!xmjEc1yWq;R9i&dS$sEzb2l6MC%Y z_uS&q=wun_{s5;8U7|FO>Z|y{rucGYMQ!Y%Bk#OIbdwT`{@g&ioKY6Rx zB{fzvK1*Xc!8YCYs_E;Vq91O1Zmq8JKl-V^Xam#XKXMB9)mxl?UW|Ht+T{2Pe}j1r zd$v{YFOgn#`D$P8Q<=YejG{X>I{a)^5$tZ1c%qe3kk2wRL&oFE;bNiWWekV)+|NJS zT6b^t-{f6Vg&O)r%bq{iA(y>%jwV*mMyS#`>d4IY*6Tm$QFC4HXz`&kHw%j`>!_HSRK)iu+4PQZr; z>ulTCO)Win;jn$tLk-E#TO6N>cka4U!)$OYKjw?d5xFLmN0~@C8m)YsEHLWj6ToiT|fNExt)s~k>g*UbT@u@QV zyyU`brN+)=;guc-PR{&%RnC>=TFRFf8aKl0Vjpd@xtsKnBVyXxLu`V#_Sgx?#pg@# zH+3k8Y&^{>v-s*grnh$&@z{zb^hf_>d2i(s+np|J-Ws@ah=!I zstdOq-#A}fek=Xd1`!6nWyu$L1rGWMB`Ho?~?SF!ISk1+}}+9uI(uE_ws+ge{J8VeEXVJlPBa)bw56|?-wxIaVzjrtX@jd-T%A(@0z{u*WKs) z7BY3+a#{HHw)XOMs*6}6=KOP8*0tcs9Pjx#-0^d^=kEE~p{DV>vYhi#%VCexi4UvJ zrfWFpeAyl8@Z>>VZ$-fQ8wLy^3^sQTe35c9Pe{zmY}q5L=Kq)`bNSv4ohB!Pdne!LhIocNoLbhpZ<^kfTPImg zWY)is{bG19hU-MjzYX)wtd9H4eSE$)pQ?Yd?DXXxy{p7;{Nq1w_5FSQ{^PZuKVB79 z5-^uoCsE3BQO4To%x=~lUuKy;v#*YN((yKb>lud6Gk6sL7(5VuV38&(S(I5@xK7*W zK%9g_*e>PIuCYH^i|;;U-hHGulPM&><%>d~Riu%ra%^JyEB6UZSB35y&Jj4n(J-`1)@~O+6&--4>|NHlS|9wd#Ce8w-ylZn-&;NO4`K9TvJ)#?B75Q@+%w!fx zPB@gpz<8jE!+qgD-o^#?j$2&o%=&x76BGm9{hV^+Lcql2Uk4st+wtNAi}j=J4}|y& z%&u07WR)(}DCl)xpL9of*DT?m74>hf&AX@Zp>7f1;{MLy72&)eE;dUgpPaONlJ(u| z@;9<%1fCk7Va)6dRag{R^w+9$qspyMii>P#{*V_*tlx2_y!}>BR@%RS{)E^q&MZ$7 z_bywv`|7U~w#S|>>%1*=EGzZs1^1`6x}K>Etv~#Bxq0>JEN;fa-Rxx{v6I)HoV=CM zmSfRY89~;mqOba2Sw<$U^PC~|y)&a>wbrFM^&z=VHMbY?EZy`}Km7NHV#U3O+7Dh@ z9>9A}q>%Nwm3!_vf#Y^<-V5~`H%gyhxF!2-qHNfbstfi%iwYYi&)>l}-MVt|o~`am zui4~QMJ&Czr}6f^=`Ilka}qlYlGRTdKVyB*&T!5>#98Y2i!R6M7nV5+UV1U*nRMZe ziir}hwZgoXr)uj>W}VTY_1BfF;DE1thC}+v35$2_I@F*p(^h$Ad*VK=B??PCDuiBd z{^b>^an`sbNa@*L9=QYCyr(!$Iuq~A`boGX_lek7WwWexdYh|ut7bWFoU0?8p>tgA zYAlP_0^UNaa|<@Rx-oW1b#>-V*)c;yu)08O#ViQ|Hu4@;LNQ%KRiC`{C|<)+FAz)O5qN@l5mdSk=|mjJ%KH zS=V0YJpE~YbH<7O`_dh(tMw*uUVmP^=^yJ`!(Qv^Iu*wsnhTa*KA4ktv#8SDUM(d^ zjdg_?uiD~IQx=D)aZg2%QAf`uDC)O??K z7y$mUtRb&V}HSgwKLam-Dh~{?G4KpB8s zS7%A?RR6?i*5mEG>5kBuj#tyK_N(hkPJei0{*_!SzCe%7SI)n=7ja#&FInS`cD>Cb z4n4*M*J#FXD(f6On4N3&g4^Yua(I5}+WVTVz}$4_w|{Bj|8Bos_k;PpnJh<)+kO$Z zzq7tr@JcD(6J&N=+k9bHs@RMfGi76*_N?>GQH@j*^R!#!GNny!*|!&UXI)qAm{_y+ zFXu6n6>eufJdEKudT2+)@1;DCRotx3|Fp7r{fPUJ3~!gy!S`Ja6FMG0*mbO8A^X%? zE4^IVlarqRJnnTjTfeH{$=##x_Ql)(|6{jy{Y!a`rPCXnPiMreQLBgT6`EF*+U`}S!{!!kalEnUGwQjQhB=rb0>#&3_+SXxNP%7n}D(k8L+UsDA?%B}U7TM~ae?zysCs(h``z9avMdjQ|b9cwv zDO>#G?gj?UJI%Z|ydlD5-Soi!?W+3SN>e^c1f@UO82iPOS4vaGW&$Szw}oim*`(4N zmd|@!+Xeh~TJ)7&Rqk7?e4wzW`sot;zhBSaV6s@qV|A!N%^uf6Z=Slp*0861cKx46GiC(;Tf=@`zaovjxck`0Nzc~D9N?aK zFokoz=ptTcMYR_Nv#zcV-x+(mtY4wuUuU9(>!V$Le2m5nn=jp1`}FSYbM?%pZar?1 z5}f|NZfE=>714s)u5)uWuT@>V)oiU`<+SnGqHS*1ynlUBf3>~gv(7}z0>MxxHT|wz z&truaZT(uM_Oj9NNyn-Wp^Wny6a==OoBMv|2c`#}3KREMhb(`^=_aCJ{%!9J5B7^3 z>b8N$I}ats%wTH`{N_3@hWpu0Q!V9n0UklcdZw+`KbB3{Jdag$8;ls*H+F?EShc?Ju2PN9k$cAmv`Nl8+D4jYYSZ+I4(!NHoP0u zmEO8@%GaM(CSL5DG95l1Y18WWKffve*}|9q)8E@Z(B8eO`v1~<)zjUiroJ?u_sA?S zIVU-$fFU@?hf_sG#o>gyz(2*Q-^)Ku`Yv=y+f#w_>_Nk0$5?r8uzk1sc4OYXtEYRa z^24j{-?#hj5k5=%zvA~ZrPTCVq~br}$Fp3b zd;}eAX3zap@;<|Yb6wDmMK5d8_peKzRP~F~Z>7K+tAFOQC)Fo*Dt(=NZ6ZgPL*mW5 znq8ZXA8zkBS@Y~nVqE05Xs%-xU7ugc*N2@7V>jG>r|7i$yIrsK%D45tJJY;qkNLA7 zucP{Y?u;%eUeftvo5ewekG((UnrAtk4fTA!X-Vb7O#jw}s(;_sTj%ZnvC{P1KKu1K z(v6QSjAgg~H?+B;ek0+!$oCFirCgg=dS`BlFkF=7IJ`)Lt+@WZz5k7ee?NckIVDv0 z{o{lrpS5h)uRP%{Janv$(NpGU3!lE0Ug0Hop?Ud?YX6_Tont3)J7MaJ`&$ocKN4gO zHec*=-F}U?VVTin+f$~OeD)m`=qnQRSPhdXdx3VfF zNv}yE@BcI}{!HyjPRd6fg>v}c4_tHbhn$AfEVhr~hi|UeQml246#&$W<=oufM0Ij5zqUF%rsVZ-visD}c_uPUwmQ7!X1#qz)M z)3f`}ox5DdmCdtqjmf!_g)g6+pAq`-^cFjDPMy=er#pP^Z*gE)D(tR*)la|AW8ag9 zwsZe!o5r8s$n0`ZZ_AT4I?LIL_8Bfb(sJfcFheTS9p^e_t~$0e?l#N3FaAj8XqaQt za;}c!r02xaQjRd2F!5EQ=iR}L7x?uz9|_ZO`@8kXqPXeTP4(|S zNn5;mt=v|YUDfj9875Hy75i zn?9%bcK^$W(!vp zwJ@hM0p1KsT9qvg@4d4MP6T|P=44Zo>HJUJ;_|J2KIPa6&vio;UTCjOp5H#@=uYmF ztJc2qm%q|iQS#=dfYi(&?wZoc_a?q>cDNSQ6{tEjf~_*X=Al*%zf|sj$Ia}Hhf3Yg ztUD{cCHv)UCY32yQhrHvD<~Gd%?w%Nb}BJ`$;OknQ}kOco+?##IvntbN>xpEsLWOP z9UPFEzR>M%%Ctl z<96-wJ^KQy(+;cKwTj-Ewj0WEu*1lw=qFW@lBu z@PC`;*=uo|@1ETon6t9<*8+zlZvT(oT>ezZw9%$zN_&CVthGr+`Y&WxZpo-Vzf!Gc zLsa6OPaP|#Hox+y=JJU0;J>wGntzws_NPCzO|QiK{Q2|T*O$U+k-WAOHKg+U-3{PhQJkyI~gZ#&cfI&nh<>=2(36YvA3&7_%t4Z*yZ!LH}e6ndkC} z#}Cff?a1D5`Cn$LcIeE-{>f7gM9kq-;5ZlX-Q2=7O(`k!r2@aeClOVf&qpiYXY6TJ zSjriYaz|O=xS&8jSH~x-XCgfdotJz%R`%E8`khbK{-KP^7o03^36x!C7i@V;Uf}xs z=Zn`Y*_HJ^L*ut-4oHGf}f%S+Y(l4)`(s zz)vfYOwFrrKIUpC>K5p6c7&(x%!^McTXsdcbHdr6qkJ0|s(h}HFFfz}>i@Okr`Ox% zUG0VXuczni%3dvqmMe&{_D zV?(r(n#!#0e))S6gwLO4iMlZ>o9Sz#UcCFdqw5b(=Cf;Td8j(`*Xrvv0)`(eH2XKm z+)}TtuKiHBb}h5u%M}9WDmJApVPz>}uktTAwxVkL;eVl)83A+DEln8T9e2)i+QqCK zasI`O%aICsc|5gGEo9>NY<3oJYHQdqQEv6Y$;HQ&95`Zyme29de)!`(i(uHNf4#Q@ zT|+BZ$q1U%|9AW=VR%6@LfN`B$oNRq(*q%rTll*9zd1L3-}1aMa(DHzJ?WQ755kUJtY=bJNAkvZ7!8wwfc+?$03GmX-8%--E90W<0z6Sn4xFk z+41|~G0!8$cWU3iomrpirNgxH#DVTxz0js%+UC-jvPaA5-cI$Vn z7I@k$@o3laN_~yC)|A-{+viPE(o0ATxEB;?xyZffXCcRqy($;}bU9xwF8J>)P&xC- zAp=9P$pW7bm9IPOpu#`(-GePh7Iw8TZZ(cOcxA-`vyIBEo}x+}mlc>e1J-{~^<5tG z%KZM7@X~PmJ#h3$&8H3G@kZ1?eiry0yns9Sz->nJFGq0@bl2-jU!JW^f&~)yX z-xb#NC5&3P=D(_o`ko+IU(<5+_zU%bZ{7lbr=R%$=T%(wao5TfAt|h7&r;-49g>qT zMrLjD-YC$&PUK?sp86AoA3uj4RG0En?p~s7V#e5My7bJO>t>%hWhWh-91^N^E8PC< zY<2#t{tJt}Iih9$Ykdlh)00|fwD4l;(|ML_nxwh{R``5P*U0Gkl4B=e{6T;H$M)44 z%dTGwos?gu(ZZzWRKj>eX(6jT|0J9JX73E*Gr!3~5@d zzdhorSji8?(sy>D?S^iqcPgx00(D(vZ`tupJW}MT6fbaOVY^p&*<2)$v%>b^uLFjBE1kcdRk`AL?3t1TLr9bW|A%vPDqHM6sA&FNcOvS|8B3X2 zWrp{!_B(z0G?`2Lua4U030oGv{j_B6iCdDBT(870U3W=t)g1l#e=o-UerA7vZKhU7 zZ0wxB|F3Q>`~UUr_WN>&O{-^Cto&hZJ9}B4>z7oaM}bH6> zlf2&7UiqvLE~#_GNoPpwj50goo<2C z3;GZI|57*i=k5LSFPF=wJ$?7s`2LFwWh;Y)hnJUoE}P%2Jw?t!xG#Fw?Y62ud)vDQ zX5L(p7;^CDW2u^}gv-9y~GVbM#k3AXetFIR8cADqqi zeA5HXIwhrSiN%)%v{(K(rKsWb>{Yg6ydc+9YvvQ~Kd!mIdXSmiv5bc=Z3*wc4CTq& zCjT-_{NBx45qO{Z+r>`}u_Y_d4H*x#Xa~!@*Am|I36}-$tCW znxkj^DPKu$?(*AfwtW-0mOdxi&Qtw>tH{1`+u5fhe^@1JzDd1h?%GPR>l z>vewA$av~;zq6Uj`+j0$U7!AnRGz0#wtq1aIpp%-?el=b<3c;5wg0I%H?NH9-x`*tuW|zl@L{&XhVCj8l6Pzm=weiAFkt+L1?_ce$eIHL*`s|@P0iC5&k)$MXW!CIGxY8E zYM*rIe#(=y%|Ug_`g{61{tOp83SL+VyWCV)Yn$LFbmPLDl;WI85s`O3cV-lPeC?|k zaW1P!=1J-H2O{5tuY@S+3zynXu0 zalekzB4x9=Z>wyUA6l9JBKZ3E8m@yccl*h#X`k4^*tuMEMr}&7xxR~m$aBw0ZO%DmWKu;-J&+TgicznK0! zd~mz;hBc2*DYf&KI*iFQ3c)syVtZc!oeh@L|sE z?3Lv{-QTPxtukKo;gi%a`Io(HdTj+O*4;Wip?U6?AD%CCo-VWvHr@I|EYt4E+EqpK zZrl=|=w5d6mhT+*{!$6P9PYOp&x9{hnaSeW`DZfoBJQ0+bE-e@-7a%}%ks~6_TJz4 z|3}wyf%IQj4O#wwddPqOi)jbD_B6|V3?Bn#Jzeq6`~B|1=jHrz*}FRbaEYv2wDRUL zez`3-cKyxSFkRuEcxCq_%f~8$E6e}vT*D~j{%l9m+Ht)Y>9ZV;TsSj_Wk)XG%fu-y3{D5* z_dN0tKIp@%YdkS2P5|p8KDL&M!Wntng)xRjH0J*LAY$w^q_R z?DI+RjAW>F<)f^|-J2cNBsiAM*->72C5TtnChOaMb%j@3zI&ZonZlQH`MF_?m->vm z{P*r2{W~>fwdJ#8Q#`NUI@6}{cQ>Qn-IY8i{fd=?0_=`EoO0ih*1Sq~&93Sr1-cxs z)~a1=`Q)7?wRye#+%4>JN9QO1pE1?(M%dJ%RZk2wT6#pS&rW~#$5Jq=;hFpA%qwXJ zznLY*6m&6_J-Tc6aP~~r_zS;_7EM-;`sUJjRnE#ILZJV4sF2`+Ej=cu+c(>-W@+hq zX)~v4&QHIsW=Xu$Q$ET_PpF7&3@z}OVzYLupSZWp6}6Q!Q(RT+H55-Q%zL&}N=o&F zwoQJhs@mrt^D_P&w$RjA8RDCHb7x+^XNP^%)M-xBUOX_k8m)GG){2W;tY#FYj+~G6t(E42ke7;dWg_=FAf6vU?bh$UJ{-+5`OX068VrJFl4JTey z|DV%uxhXDavgvJ8wO=73AC@G16_1{FX?LaYzMp~DC+cmuZ*%{Ru)O*1$H6zhrEiF8 z^?3gOcjD&rb-D@mb(1Q+>ofQ3G}}!HUF0B?t+67CXSTGQWXMSa2ZiNk$5&U_FaMYI zuG~a+%R@1-_iS2~O3b%m?yI=@?W zRgYs)NX#TTG3F|FMIVuAF30#L%{eFbF~ByKA$-SrZvC%Q&TrrQK3Dw3zHc^PBsCj~ zrn8wu*)DgwDQ@|5>Gyr#AMM}!{hs)}&yBP7_W$@NZ&R!56WZ%FfiFh;j(fze=glRT zyOygKm-tpGGW%vJhWwhg!|Y!5qnOg4E(h!FzXZFuEDS7Z6%sF8wdvQzdp_k|=*rcx95$ z?}wN^<1*paodUc?%gMg7ycB@jJv?Jy*}t{L`;?7wThAgZlAQo z!0p`bpB{c+awAvcFx#)=Gk(APcI9N9b(sJspOet>(0jzxo2dn?HOs^p{3wUWhnhzj&#najeY3 zNs}(kb+7z&F6s2sB}W@O7Wf(c=#z)Py@{G)!L5S2=Z-=A}JzFEajoGG)r{ z&DWN?PEfhjyk$zq7sdON3VnW*$n9J*iO0u3c)_2)D>&R8t(0_j*Mq=5$rRW%OZv?pZT)R`&+il1@p;~H`w!mxORnZSIvnts$o!(J^5KJn z#{}74Pp$v?YP%f6fA53e^44{(UcX)9ZM}tPP(kBw)odp2gWcNKo-PkA34hCP;Ueym zG0~@NDW~u9Jxi5NpPKJBqweO!I<5XWN@ut&jcYUol+(92`7BXk-7|aUT37TwQH>32d$pEfG261X2{mP%?Iof6bgKGfzFOS&cKjE_ zc`Wsr-lD|{J%ua7W)$4I7L=!E?8S1=Fy)E772m{d$%ZeF>^reSQMg5$FGOoi+_USy zl4fm~(bJR}`uPKv>c8_JXg0hZ61h|DLZCoflygXZ`8o`8)r)6Hv;5TMQz99DJ0DECcYfvmRs8|N#}C~|JLbO1 zF0waMer17r@pP|@H&d=(2}+Q(VSBRV6j0%FNhJI1jQfu})d^b^H9XxEFN^hc6VlPN-V& ztkXk9bJM-V11S;{+;}!!iBYK#^P!^0BCX_V#7EQFg|t zb;pvnC_N9axg}3M4+#kgzA&DaV83$4lbRoIv)?wR^WE6z_Wou}y{nCgl=CXbcv(@s zwfz;%Js~?(e2QK<*!!?a=sZu7TN82P%4PG^Wm4Yr&*pzHpKCWO->B;Hzlln>xn>F} zpO(hxk{q3hd85f6Eo~)I;cI!G<;@6J$Al8m!(H&m| z`6kYrdHT~G&xYCSk0+aOa(7F}gkG2%V%2PplPS=x?~@HB?ub9a^^D?5yg_ACu08%-a0N)s4aY&#!7lh5rj*&c0W- z=A)9`{HICJba@tBeahZ(oF!0Nm~+q4uWxefw{RSKTdxzcE850!kFK7cPyzR}7N--t zmh9tlxB0VYYvIhssUb0YzDzG-N)F~;7sZmY)WJ?=rQ{Jurf~m_t9fpEu6F->@MKCd zXW8YYuXB8zciVo7dERrG$N9C&6Tw^Sm6VjG*_{iBw|L|vpSAWC7wgwX{&n}Sq$eFn zbItvFx#jQsgD zV}2*LGi*J7q+-?I%Zr=KPF8YWYg)1OyX&F73Qk9y`K~!+h%PYG)jh^_FcVE$S zvp8|pfz-c3$?V_XZsc@Yb0qa>^hsXnhp&ETw44e*zn163ks1MQ9ZzMytt`tOI3f@2 zO*f9a{B+fG5!35O_a!Y5Sr+?jrJ$I^kfFG;`0S9F@WqJci*) z5<%-Vvo_rNS0&7UHI_X=^~9cFEW*XnV#d}(ztnxL@dN6EV{k}{9i$h5M2?K3;GJJHFdL}Z(ijNrvTQ=JzavCt3M zY4q7A*S(E-g|nDme%MFB|1xi+w=wHibVq!h+Sb^xy)Qa>4s)lAT-}ws`&jVf%>SOh6_i#z?AX{NVO4Rj zDCa_0{^_)$#-Bgn&hqvR5JSC;#ecsLu4L8pH5op`ed^|MfdTyG0 z7OzQHWlQm<%Ud!R$4RkYYn-}X>d5;;Z@Nx#o^IZtaivR1{QKii_gjwNpEv!?JKH5s z&zuex*wFo$X&d*x>{3-BjoN^y1E<$rS?O}_Ol0NjvllY=1O&@6E{yFuB+bb zd6U>88Ai>%=!;n#j}(L%cCXneXlQ%+<>g0VkqbIMXys^hKmO_%^KBRJ?Pf-+(9Kg> zXI5y>%Cnyq=ozW7GJa1^SH+r~lX356O`arR#kpV4Q@PdPgjKx_-`oZHSB#Fe?LK!~ zVB^J^GD>V070RZncDg8dx``h?SM}}JO?c@;BX#ela`PfU%< zlGzfkY8+cwvRQD!yzGh;F3r#>^O9AcE1lJ{n|`iNK=9*Ln^oR|&pWshEBzYdO1B8O zUoib%F3i`cQTWpG#*22Ha~r1iJxa5gS2kOJ;&v-BISa{C6$@VO5a)8seLw5S$8M{% z<0&@|1oYqCwEE6G`z!hS@-H_nRAAF)nAs-8zp|oFt~oicNc7T!<68>JHAxBIlae_{P?e7037QWSSWt{)kfhjcZ=s4I*J`> zUex>Y(WADDws}g<_0Ol6REUUO5e*7nRX1z z@?gK_gd>bhi1xk@J(gSY4{j{@Aq`xC_Yoo0>jwpO|#?dcMW#BZu?1 z*&S8jTKIkAazSBc@nwER|8MTU%#<`iCAa1FYmT$(v)(@!+Q<=JZ||ev|Y;8w}q`>K_%XxQJO}Qp8A;03xtUlgI;f^&dOEiydG+uT%i1U5JQX5&PAN^-p zx9w9p`S0^_nE>Zh*MzHP@=FT2h4}AU9aZIVT5Y>0{Q8R-N>zq7az45TX13cTmhAmF zD|a)0r8tsA1ykiRbtI%^zDk==DP>K$TJkjS85v1-1ee|M?IE> zbJ?Vfx98VQldyMvx3v4gg_-)YFV@ZEp2cq0@#95ANaL#mW|P-y%5VI7#`2ubp*2$u zOgQpWO7iVS^?6EXt}6CtX!x&st)Q~|!Q57Rlc($lBaI#v6wkU}tkcM9Jo)&hYde{w zwECwk=-Mh@JN>ozJJsT@YY~?fgwCHac68e)t*@!u{?NiY?zwOQ3;VbIC-#LJRm)93 zFy-LFHAkGHHeBDsvQWtBh4w`c>4oq2OYZaT4GmM-l3=*??qtzlOYIBE5@z~1cP&z+D{bED{T(A#43>9>|;S)3F& zqQ#_R>L;M~@4EW>zpDGv_kUj1?X^kh?Kc0qKh5!Rp%WOhrkvlQ{Jg5#?RARga>F2t zm7RfB+BS0^)>X{lNbs7)xY{59Lqr%P4CGR&~P20ck z;+taeg?33gs;L?Riw;aT+7qAtFZ5o^bEoV6b61v|act{(BqMxqoYq`tFDSwidL!IojFTEwpih ziK3IjOGmjIPQRSy{J1I+R^hjHapm)nZXX{{K_vk%b<_SiS6vjAC<$+`)MkO%Ga_RCM@hI6j1xk_PZ?MguwUa`Fjg~eXcL~UB3TU@Bg27 zk6o_4{=w$&L+k&)HkIdZ*Uw#jZrAC0$N8`H6_XTqbuM&Mzt(OS|HFD&muIBd;fP&X zpPF>KO0Hg=`S)&jTDGb~f&Qq7}T$sUO;alP9Pv9EHUWJy)Z z{QI(eN4ogGxvgqWiX-Hgk5`tzlNS*7%9$BOkziceLh-rV*maP|AVEpoSH>}_Wm)>j|@EZ?~Q zf3e!}5dN6zi@RRk+NGR*ukg0`sS_zF$17Xi6*p$h+GW$0tQvEK!O3`G+oJqnH_qI? z#zu`UHeIC|8;)G6$-Z^{+&`wr2P%A>0=NRy>zt0TJX$Bk`Ov#oRndJN=mt;QJ zG54)}A;FwDF>#6MJgwQ{CihL`Tbq;@Dr+5nU-nfc_~NV`8SADNUrhNPduifjK;VL@0Ab6WkA2j~BuxZk0(G4}RaD zKj}p*-}N`2I$DEwvxu*I%yg7*XAgf{=pJYD$i94j^@2B!1)G=izj@v<;hx*t z%XjY`JtDPX(kCJDV~(@eb2G6R7_Plh_3L(SG~=bTt?D;DWLIwMD)?bAf7VI=%@>wG z<7~gObAz0c%FKj!lXV3cZXK}OtgOTRRPa~K?5|cEPL_JSXpX$Ed1~gBE-{nBtL*c| zKUG?a2cKW?lJD2g)nBsKFVXq7aWNau={ZS(pEVR;+|cTO{k2g1*2i^!jhFI!-QAL^ zUM!!cIca0}uCROEw>sYCWo_Q^^=FH^W~`w4LDjI2Yp*|l(b%W9(I;^u*70DSLxh>x4ZMRxjzD7j|CjShdyX^&cj1 zt+dRrLu1XaPRiMvotQ`!>&C-H}kVs&X=8W#B+mi zanM#lSG7z`A zaV)!onSPbojU^)GJPv;FIKI-HrX=Mt;eG=Uzj zxrP(dm;23Av6p!$!o_gmUtC^XXKC^+GoIF|{!UiA9+f;=>2o3{_v5FTr_0aTht$X% zKk#u)g!axgT+hyG3O)Dvcdfa2G5Zk*?LU84=C!k{Z=U=%#r*S^Z`W`4On>+B`knh5 z7c3F`cmLIErS-Pk`SO2bO%ek{f;>e9-p{-)u zl~2CAd3lYyU9*7t{E4Sz#a7!YD@G}r7%tTkcb2UF(C+u5V9uV?lXo3qn{>x+>X~fz zD)t-7l20UVNZtEfnZ1;Ue{yHV_eT*Oxd+ZP^3At-wfmL-*5i!xgFdd{zq&5tprPc! z>849vI2L&O*?!fUucVM6o69`CD{=46%qQPZy;-EX^v>U?3!9&)*vlOcnRVx0>CT(C zq`qlvV3U-7pKM%z#M9nrU69=7%&oIl+G-tjEjeFmBx!KvYQF7CtNE^~l?(Q6?fWM7 zx@CID!+$E179P`8-+pFQ_O%yk-_vv@{y){QyBjI?B5<<)xr2*7ym6b>yJ%PbUZFRP zEA21eobxV+?cf3XxWc19f6LW8dD`)t-DG2tg4VJFG1jZP_FuSlH$HgpEZeYWJNEB? zIQ>rdsfQW5b_`$Vy?B3r{_EiG^uxO%c0XgbEchJsa)VrnG0%&A0zFC@S_K7po<`E& zEzW;AwJ3Fx>nq(u2J7`+uU)uLBAvlt_uI|f7iWdlmM%Sf)^J(EazpD4=O4OGzWi|> zvus%03^#^@E<5}L1=oM{TkE=D&z>fc+i?>!k~te^%vx0#&Aso0!`vmlMS__tCqKEn z_WrIGgJ>xm!>6CT-tP>l+c?8-sd(6f`4?Y$O?WbeY0;$TuS~AZyW8|2UrONkO{K@S zR#(!0DrOvfC9vw(mNh!Eesx#=+J$+?tkk;Yal2+lcHNKjchzfWckJ0#t-n8G^{ZZk zH+}B*C)MvhS<;(jC#WQ&C}*uW%>6S6sJ68z=uL`TI-TS-arfk5H<8d-u9_un@ zFWK^^CtVzpgyG zVA|VVimgW(k}mgh9Pn2+jQ?#Qf0Nn#rE%i zUc8>ytQQ%-Z|f^}|Gia8^J=d>$eO)>PwTVWfAaPJg#7g1_k3>n|I4-Ze{L?{Y^3w; z(%hi)2Rsk1__5&UwFvJRd6kJC4{oh4sMyWYSL}1^SuuZ$O3asFi{mpQI7I(R?=tr( zZQpU^D|bzk(xHX_PMqm4epced5#)a4*wULX+ZcDh=5tc6SbydRx7?QgJ-MP*rB?)> z$nyz3Oz1B+jh3F_#d2V6uDO<+QLyZjga2MUzIU6a|7Zh`SAf+iCgWt`rb7@Z>;K0pN?s7Ub;zA zH(uf2ZP(eVo=1gSOYh7}mb>`Y%s@0EKo7c^zax^Ar*~h2csdLKZetulJ@A}dW$@hK= z-mCNY@6fZ=RP9!v;QoW2`)0pQz4yR5OFDaxR7z@EVO8py+TS83pjis>E5q>Hnt1?O`qTAv)1-l@+G_a+s&t9g8u}rd%e>nNARGRx5Aa_ z8)^&T~h?#tg&FstM8MPZzgkmcO5u zUB88G(v7+A>`W8m|G%1Xx9awWsred;JqtQsHn?s7A|;wEB<8` zlVtEd6-i#k#4eebh$9bm#Sfj`&LR1kCw8~Z^k9d5M(eM!t-JP9ASSJ<%`I6`AnnPO zon2hgmVx%n6I-f-h1#CAobqI>&hCHqL~-+bM~<2uE&9)$b&AZZCbi9BSQ>TN&M)ee z#VW~P-8NbeesFV2z0hekc)EJE^hVqD30EEKv!dGsH^#EgST(ml(cMsRLU8chEkfP1 z+@H_jcCo0r**DFFH!3AhyTixi`qS4+p;Ml9>4{qeim+aA4|=pDG{iXX4DZ{QDvzUu z*_{m!#NFqAKWEGC*sm(rD<&MOx%h92^R0hpC-^J!o-g80@mDyhqw5fOCbdCDWmbCi z{FDb#YgT_|sr8VQ>elgleOm89^{(4XruO%_n*?m(YBXGU%wYPJKT^)it-3Zp6qwk| zUTwW*L(%G|)2}N8R@QFfxmu&mUFGmufiM5w#)FO@f|;Ms%h@DC3d?gZq?L~ z#}m1wL=T?Ob9s8eZ(H=27a>P@k8J)}^XI~Zy+1aWtx-zbW*cLDRejUEUk`)V-`F}m zZxf@-oPt;*rs=WMwq2_dn)UYJblWEqv*LLkD@-_?ZvHf@KPJqFvoclZQQ~zyL96R- zX+HJ&`>lI4CY>#5Z+aN;__JmDLzCJ1Pgd8Mq%QRm+T=d_klLo_wRLmYJiM~oGM+Tq z9CZBl@2UVZN7sav_ohyq;TxZ#{D`mm!J3CcZt?pRDA!jIexamws z?PYdB;cHew9{tkSTUsP`e~{`i{Pu8Xz4Nxe8RuJSU+;`N?b*}%fL%hJi-S?IN2Vh~ zN!r?(OQK`%k5ye;_4GHz39XnfdUspumPW^n!b(T)m@jO$uV1g(ysyt{Jl4*$8(lZ^Q4mqU)#KV4lU^#kD56GMew*;pOY@ui7SfLrCx2 z$Lj9}TyHKey=7l!cwLNJ+s|Re1(m?c*4KH~^|5gePrF;KvkQD3su;Wmb{2r{qR$V@ zqF>vpOqk5C&a)|?XVu$+BeQaf1l};MoO|c|v5$#HN|S?>82j{+-ZzJb$!rlQIThvo zwx#xa{ec$^Vr|lrcKwb|_~j2yne_gB<^2VrdCNAkXirJ)2|DT&=ypKly#1AAr44qM zf|k27>0ep6c)dl!RM)EtN%Pa^)dkOfy_iMdPDo%@AP2 zxi^17{=VuetG{OZ;uwV|?mVi@lw{N-cG|D^>2i^eA00d5U6i(*FFh|PE|ghw&s3x^ z;%gMg6}GM31_!y7w1SlRc*~;;4+zWdKDpawiT#wY_ZBgytr8!dj-u_CN^Zbe7>$M_*-h4I%CxbQ^c|A>d5&Am$jnaf` z-sk%xPR=U!|8=3xfw|*@I)BykAa4DZM{KRcN4IL92@c>k`D&?svuFN7 zwM|K7vG@6_m>1ZJM{=)XU&PwmdCW~ba;Nbb*}%tb-wGD73X1S{|59ClVe;fHX=$t1 zr_7zsnIxBd=5E~i8=uzAW?_5!bl&q797@?D3sU8zr{|?gn+a;}KC^OD3RlfK_w~Jt zR<6ITtt9k!MgHsWw*CAu+IqZ|VU5X|LeI_qTotchQPny5#hNo)%N=cYDag&**4-qk zuHf_TMt|*_UsLjHU+-SvA%4|SwCCrM<)7ECP`X|v@^AK0fj-S5zq%FiEKygc$1lC% zpPYDkm&^LY^13Z{SHiyEd%d#%nH1yyeRfwD9h)4NnkYB-nA7|^srM%z{p&h7<^JiK zEACb=0!vy$k4A1wE!`Qt#;Jf&%COV_pn=u~!RL<`R+{qmbr&i%tH*ZUexIS5RG7y9 z@Ze?TXAXx+0y1q`mp6mFZ((F9tUq7qe)%<-@KAxe~TeA%n zT4a7rIwig?PG?2VUiF&}^Dk|YeRj%m&u$JU-Sw5M%VKl>o;cJPT=>!Vh=ytGX11o& zGmfgKoSVSqely_wvy9OHHe5@KbFWNSZQ=+EUvRrhtccD`5eLG?rS+TfYX)5Idr{tVxxTvz|L?SDw}-<4I>xyxx>kDC(Jc#m6c`+h+8SP_%NZ0o{^V;f$zOgP;5?3zW^ zjAGu9^~t}R7df{~oqf0KsE5qvZVy(?+|}of3AOiLF_62m`SR=Sk3_%nRh^sOmvxct zA9K`^hZoa$W8oOH)+d+7Q^zo zdy_9%KF&HXbE)n`kjAECrTdO9%v@y`Tp4%2Qp;iGMAgZj4VNs$3quPQZZzsyB_mY+ z@!qo;&sAoNyEyy_SGDA7yJ_OpalZW7Z7F-#x4w^OoE2HKP^DH!y`y5uG4

btP*9 z8zhc?nPa3KTW*)^!mnKF(!FPq#=oi@j~a*PNy~JC;xx_%j3lA4KsY1b;_^I+$?H!(DYqppZ>yS z3zgT{m`_c1+`{*I>mT29{4NXcJzuhK^Qo`TWl-xaqE z-KZO*zO%;T;pwP#uO9GbuC93*)Yls%q!BjTr+ukau-q;E531b{vWx314xI?LukvE*rmx~wr*$gV=(nHp zD)ySraq+yeVbJZ;Z@YB(UVpReZ4I5g+3U)`pr6aS4g6M=tU7alm*n4_?ixw?d%s@0 z|MysI`CsP!rYeEwC#mYiURAfbuwutr>G1sf)hqrjUNU3H2cbR3^<@gX-Jds>n`pn% z*_ysE`{HS%qn+ElBYr&0-n6Ma^!uh+g}a{w<$V2qfi02ez;vyYvNg^RpKiW-ykS+P zlT$ZiZvoqkmo`(T&JI_PQe&KdB$F$Ur?29<`EJbxEe#=lw=ClQB&Oa=74yiwa`j@i zUg|9`Mt+}~tbdF7ywjKy*H>Gn+dHZ7|Y z1x`8DY}k;Af6SQEP^*L{7? z*>tyTRoVK|t!5kfe$U-yZ@KnDSpLnn_kj~_i{BgmzR3J0ymGqo zn&GQ{2I+Iw>rWkFsXX+}_Hw@K>Vi1t)k_2$4D{kMT;FmhWL`~G|2FIIkBK|o&$)-y zr5?Hc{*D*VMqcR~G0!Z5eFAFF7x&h2ZM<|mF`Ic#{p$OiljZykgqo+@?r;9woqw_U zkN2;eYwQ1*%g=twc{=~EeY;Oij+ye$ zq*G031=htH=COnJoQ}fmbvzXppU!zq3JUNntKBN zd^~yX%-0Uj4Azh%Mh-HqGV+!dn+v;7gqO-lzumBHvaG<4>oS`rKTdQCGW#82CEtF= zxlg4+-py+<>xMXCBrYiEa{}L_2MY) zGUrlx*_W=Y(-W-o1>#q+uIsdGn3X;&`0~v&ul0Sl&B~oz_J8i*Ye#ncIxF%};Ls`O z%z53D7AN_v;+me! zj$e}>eh|oAxuD86>hp=hjxhV*&o`CK;@y;%l_|CHl*r7kHcx}!t&#g%1(Gbh%of&6 zWxXZc>&&}c;NhAj#s98^T$13}_^NtB;9TRsPW*qqyIO2{bgeKm=S%Ek)u~fD+Ur*D z@erHX(bmE5W}r7&WXrt$`!#0=sNIb?eDvnVXDZgVhACTDES%SRu%b62*Ya9ocGt~@ z<5s$+rpLmHH&hj#6X;8b(thb8*SDE_yMx0w$0JRTPH6~bhP_jJHa~0U9tGn^ijmKc znz&8bcle z<$G4{d7{^q^3FZhBftE#R@^+FvkweZ_GB1l5<3|f4?(#Ai8CjJPJfKFXf-dk_Q(?4 z;t_M5hvD?<{WYQ&deQ~|O$et6cT=TaQQyn)+i1ZbqB8~z9Vggc8i0Z zF1h`kYJ0jf@A~$8Csv2sd|0ayYq~@C*VA{JoVlX;50$I`%l;2;aagd^ME=aznAE2t z>yzVEZ?4?>?DXTapL{Bk=KQcb{P(ZGIh)mYsw^u^PX<}O`}(i`NBQpeb=QNcj(s%$ zzajnp#q*#0{@6_0${jYP!JzR-gt4)R)hvFUb!mFe-%F0o*O+lb^8J3nx{QNM(pcX7 z;?~>nZ$=`Ab_=_TRfEXleMQb^SOQnH@0%y~e203wRr)io@0R<-_IKvYjduKG@Y(UZ zn}92qnR?{oje^!!B^1g+xaVl@s+)eErVqYxk{p zxGZt4J@)HE-TDn1InAywT{$GJi9VKDQ@iVhN{`QsA ztgGfaZ#ZQTVaYrH{bZ#k^_NKk{tqR}Vh*%#tF4|CTjJQo%U2qmD|;!9S?z)QnY^DZ zH|uS3zVYm5W!xdun;sXZSnT9-bNBu^^^b4tmT?MQwA12hLjvQhV!x>;?p%}dJE@^$ zkYVPRBdPFBB-?f)`^pTli_yzYt;}h)i=61DA1piFT3S#*(odts|L8Msoz@!}D{ub% zmS6v+wE9>2I+L>vKfcG;f3&ZCe_X$2-g_VJ`FCUX-8!?`WqKTkZb4wrl5-cI$<3;o zW$3>z%jkSNZa+_%$9v*z-S+xU*vqm;dxnY! z-^`O6r?EuaOuLl+GWz(#4^BTH8t-=Q_OhK)R(c-wg_S6!AcE zPT%syC6h1MH@C`uoE297QejcwuBzwD75nNGSJ*~nKMhSi6T;QabtI!FqQ;T;_{*|Q zJN;U(`kV^;IjQZ=p9k{}UE)_*@jyPkFlJ`tD|=SwCA~tIesQ+WYxt~mc2dg{E6+=U zNurD9C0#gq`St3TT;0pG4m^zi_d5M;f8Kxn?p)PhZ@)ZVv2Ue&jNFs8RzI%4Ia8Cg z=r@;wrP5?wo~Em9p{ct9QbUhLd%vDh`|6m^Z< zf14%&cdx2tTi?Fjn%w?cX5*AwO1jY=0{a=w7GLgAS&^}EV}j4y=$?gQ(vM<#&&?4# ztsz_ZZ1cpgu?Je)9}2jzJ2~~S+f}X@zu_2UuB<-s`R;hVnB2w1V%C~4v&mn{=tt=PSpNhSWG z{oI6|AOEa=^Z);1m)}3q{~wc;Te>car^EWPiTGXN_qB6gFM1HcVt-B{PGu&OpxOK# znfEG6TR!{9DBDd_nJVYkdq({)$HhsvbCs8e9y`dP;j{byy^u8TKT1bz9Fwhf`z004 z@7wZlW-r%7Ugsxmzjp2Pd-(HC+h?hLf%?Ki|2N!huiJQ_Y~O3HORvB9uebdbqc_28 z&!;`U_TfjIYsD3ol&2NWG<|bBxBk;l+g(q;uUdBde`CcT#h|Vwe74bs5hw3V{l32N zZ?=_Uo89lFr5pFk9-nbXIA@9R#2GygZ?OfIdgQatSRwlQ$k7xVOS$CauQFR&Y}38c zHZ2us(QG?vapIm^HUIJxrM*%7(hHgA_$@a+wYAcB3H$9r$30W8X-pHmq30xU{q9Yp z#7_6emK|P?w@MWk`3ux+{qxCX>lWa?qS)m zGv`cqoSw8pMRA2r`Nd_j))U+ge3%quvi{C;4uevGMu9|0sUx%Ad{Mdg?UMXwy(1UD zGM^8WUwo_Ewf&5o%)@)7f8IQu|0jC2)qAerCrrh6o?zUx{Bw9_#%))PlKDEaFM_Mp zC&`w?yRCD|KH@&|o7Ja}|4;1XwT`)8qUY~E zXvyCud*`T#kXo)%%gd+D=WpGwUv8Hpze4`1{RPw0O$_lL*Lm&!bT@R`wCEgWmaRFq zAB}iYo;?5FvhtAR_9Gia*4|6oxmslB^^@i4ODc`;UWgHMo_S>B5!v43Z|<$Xeruk% z-_PKpPnSNQdjDE}_K(wh+h@&NeD~j9(YH)z|B5n7NteI&4Zr{WuDsp1^?#kkc7H#q zysgY#@6nrvr-xm)CvTPPYdB%4JMFNHT0rU<_NwxeKQi|*g*}-0YEE2q?<&TwU0H{u z{Ny*Nm(7;VYG2Hu>EG1!|M#=mhLU#!8Oz&yx4SgCHdwsbII&@Gfk>a6PwCB{n$w&t zo|~QfHX-?kyr0nk2Uu?RgV^H0@@_XU^F!Mib^X z-S*>7(@k?JK6sY%-OJs#Zxz<>IA8p-OW4v`X1b`jVVL`2tzB!^+)7cq)gGLkew}3! zchEUbBgd2<>)Gw{`QOyuu={qh|3>=12Ykx&mTW%e_55eSx-7yrCY+nD`~I{5if|NmbJ!!@e)qKO)?F^5z|ve&~p6tiRF|q`l@=%fnu$oBGMm zPKTaMJ?EZ#W_?_G{D1rM+l%?HXqNjjm;TM(UH*QbfPnR)z6)n8v>&W*PSx)@#&vsZ z?T6U#e>eZzd^jTi=YeAPPl>dZnKzX?4k%1FDY=$soR@ZS*8jh@%07HaPdbirFjn4Y zdmzVN{cu%cdBENM3msDN6Ho+JN;_)61B5kt2bVg@L(|Pi}s())w^|%i^WWrzME6O8KBE%C##G z&y{<6*?nGdwe^+l=bhhDBGd(%k6beCFptpNx>;TEqmOs{E^n@fJF2e)>D@eYy?MXN z#2*pyEXUsJ_hiidsB}K$`6DHtT<;$nnSWO~y)N1AbI5l4jicdvPw%TZEBj*B<2Brm zTgp1#Yi>xDPjd4sn^^mN^Y)wHPOn#!NU>{=+Zy?I;`Fx%tLJU_{jT~_tI@~s&0)JM z7|u2;?+91cOua!Mgxb$A}%B(leo+goQ7k+pO=-=PN$d>ugdCRnoFRtsZ zPEq-FeV%QO{ja_6Z+Aw8_jdj%vA%gP|NocS@;^`1?~^;?yfwm*>CvncGVK+A1n=iR zU3KqoP4yjK`3D;F_ldc?w6Po97ZKuMRNS=A%va!?&iTZRiD}A{j}+E@FJrmm=+l)n zyRmz_z!PcC7MmdL_M92tIA=adH)2?^AzAx{hFI&L_*Lh+v6DRdZzAMZNczvR5r^LfrMoO}-I-g2Jx)54oNO>N37zpL!cSntuIAYhUD_Wb|7wxv&YPYW)b_%yL=Pu+3TZC^K)-@5*8 z_nnoSRCan~T?`4A{kre|&0FOQBf{gKigTVgBEh@7x-e(Y^aY(Y?F|}C@4Xs+eTtAx zGFo8PGHsFN;$QU-_1CEumGxOCKYF5VU+5D1Mdy;)MAst<`--ks>GV4Yd~&Pb+$51P z_wmG@_`~A&e)c{vOb-q^&cA-&$7#`>1$JNh%kL~q6v)sxrl~nAf0G8+b<=-x!F5Z{$S9gkJ{_XYWHtowz z{`Secd749P*mL$f$11n{Y_InJ&2_f)P3)A#^2g^CSJnt~h@U}l4k?AE!&`U=Bv{lqqXY9ws#pV19=s`&AhR{b`Qp|Wv5uTa9-m*Y?owfO@9FP1 zC4W7)uM728topj~*CqDr&;Ri4d}_;e??Suu><7R7R(y)Trp{^fDJ)WK;}x|MuA8$3 zw)k!NuFn70{hf8@`9B|A=U1$#p0`fp)Q_Ut==;qA{JfnCH>&UTZ~j@nl(R$1V&}F^ z3QvDN-)PhnES20G5z~2v@3U@#e5Kah1HM}KKJN*?A2F%@I@A00YVBenPD&s9w``VT zVto6C>GM}_k54Q1?b_qG>hWUR0<#>E`1#%+wifo?f4epQ&eSa17ilKa=OcCmPJOob zO2L}Lr&g&sOj3C0YGHUTcFCH{CHHLXep$9oJ8pR3|JTiz%j}O|@jJXE_P~2tY0I2! z)7fiREWczq%Usbhu>1FokmHh*Gnd$!Zw(hN-S$L@$>`E`4#k6KW+tbqGx_cjkeUDB z(%0|}pDUF1PRJ_1T=?Y^bJ)hOQQ0@YPkB0ZR%oL@`KOp-`#$@}GeR7HTi977{Ep|i z;d#aR$-6mX+8sN#F1yuH(i6pdH9+k7mW=gjBK9){J6brp)~st?uvKW%$IT5giN|b2 zLXtlgODdXLUhR9-vgqZjlQnVAl+>d2)@@s}DCl)wZkkZtQEBf#+E#jD3%wK4L^urO{Gt;rMZ~?2$?q~h5Gl6yc>L1$cVEBj54mG{_4d6h zVc}WTVjm~`dlX!s&;D(9+G^!w`)>D!&EV()}csHSaRa@|x;dCt2T^Csua z|M%A9_s4SfH5}ECw(eAxd7*N*NFY*grN{)O`m}lajOZ7yH*J#Idgl($ z??>##Zx2i?tFG`rE;6aQbBVs}lYTvgM)f)8`UKl27~0qSsHs3x}GvcE`J#nJ~h*NTj;I7{rghZ8|9}pt351?J^Sb4Q-zSMi@g3t zI}iB^N zlfvV7KV4taaIf6M@`y=vX3%9douySLPMz?sR_m0oR#8uND0!s0|MD$G9bH|S&wpfA z#o2vl+Z9u1yWCw-;gIEN_D+G8`F0-{U&!)L)%$dzb^h7<`yW@`l)d-1h$ALbb)Q+z z8}aquYQnFm#cpcZ{XSY|lj`yr91pbl^I8~$D`%;_{JUk1QHRJxE)M4ZX`6RV(lVL* z`Str>`{Osa*FUp5?%S(hmven>{J&?D%TKQ|z47d+>;sNtiThNveH~(w_g6k=&SPJ< z&~LKF;a@Ljr8FJpEuJt%Y~GQ=$TZbu8D2%qafNfv#+OQ56tGZm-S}^B+MM9M=JDtE z9h}#>;J9%AtEO&8uM54$>vrAO+pTbZ7kkp0-OUdd|8I5{C_UZ6!~TmYto%)f=+tQn zGj;UUHTm{!e?29({AJYc4a?5YyLmsE-T%Cr0C)I(PrXHRW8^(Dw|@VhIIs5oOvc;a zi*_eA^=tKTn;y;*sPa3h@qD(n%r(47}8 zr{>!Vs+1&t3at)Z^o-5q>a5*6<32__?_Ir8yUieIr>EBBi5>}R#~!YGZDbcCw&h%N z{9m7qlZ~pZ{jC=1v>&`wI@Ms|bG?rOf*oxeUhYkK^sx5l=kGT!DxBZ8?7+M;>u>(L zz4B`Qa`X8i`<(io%bbxpC%Qsz?&Jw#!kee>b$+gDcn$Fq(e6@G&{(mR^>z++a=eu`o^Nd*eeP^Ti_Pc!uJ<~8TCg{d~uk)H# z<+5y&3M;Rv-FST2?tT*AsZ0AZirmDGJl?YF!0=p$I&x-9)YaZ2an{#L=9 z4@wSvd7e3Y`WL>sb(!y~P-3CNv*=GhOr~tuSYK$SUbgA|qP@(PY|EB86h3 z5YWbA=P{MlK#Q+PQSRNjX8YQ&k?$1NN1XAMw>sZ8@z1YauVfyEBt@$JycYBBZ1J1( z4BKuV{<}8f{?XokJp97F%2&eZd&x=Rmy+y3R&(oj3UdFPTf@0;W=ZL_S}bM|J5tGC0SozJeP zPJ8gGZrysl-(MaI`#$yheDnVOMlFq-eedg79`hAGQ)shb>NYybw2;rRu5JD2wC3Gr73;Lhq%c%U`SO z?iRS;qp)5%_Sf9|U#7qxeZ87?{mzEdeR8+b&wh>dyJ}#Z z`egr>X(IFfJxToIs@18`BbA`}Pl>Di^DXW3H@|V~tN+T|Xli;b()ljqH!hVZwoFtplrNi&hk~W z&gFG1F73Bg-`sz%O8G>_&uh+1eg^8ZvwAmVdN&2Ube3!_>iyH=YS>)yXQ$lTDtE^> zecR)6dfVgD`oDY;ahm4&HE0Ic)Qv$3P77wlGvA%V^D5%O%Sp@XwHLXx^KRx}cEXX# zJ>p=hk(G zU*|u+Tv{)5Epk>#@)1q0<=<83Zoc)(N_TyaDlXm?m;=id_v!i3xFOmOic0byGH225apC36l-~T&ZeQmYL zL9;x~qpS38u8f%Gr1O-&eAUv~2|a5&ek%TwIu*;~F)Ju$#jnZ7Poy)cA2PbV=`Ghc zMFk%ZVJElPxw>VSLq*G8s(o?&-0rZihjA35e-o|`hH$s>5>cVX2zY0o|CeD7&*HrcB-0vyv z*$MY`zzlsuHtd*Z}*Ojh?#1& zacv&Y@(mKt&5iWbCtsIzvFXTo$Y*`&kYdlIjqZC=F1T>JtD7nw-P`VW#)GT)<=1tu zy8kUH*qYSadxN!N(e;IHnM#v2=xwvSDq~0XmJq-J{@1Ad!`nubod~aI!T(^|` zb@B!KxNW+)uFO05&oNu&P}b93S?zBYKeQ9A*t~o5RloVC(hl#MaQ2nT>d5V#mqmZS zIk7N2(^@g`#)f@G(F;WtRPX%X6eTOiW^mFsDcB?^!2WAOmdb>MC(7(*it%}`x?j>M zllf6~f^NH=RrZ>|v(tGaX3g2(`|+E{);-r;1+K~+-4(SxPkFhQ6GtRN(EJlu$`TgL z-|#=8oAtipZPmt4vPVPSCBEdI#xUzw65HROFZUTwXS}{+zD@aysD1HgOn)wq-zg)% zfyesOpZ*!$wZ_kOzi>O;xIFi8>U%|1{p^KvnHSVarA++(y~M(MW5%j2S%3Xf)^q+o zc0KCru8w1O_4i(}agmmt*Q0iR^Urgy9UabP-2d?4!IX{r%p9K{T*Z4UA*J8=(r*qzhiL~F*`Xa^4{`t|`uI6warM?Yi zX|wD$Yqb?jnanOc>Dh`iOh=C-uDyKt=YyF)kNo2*G`pOgm98FligQN6GWNO0uk^J# zyp@-g`fg#f@3p9s+lGpp8;%x!w%tEDv;0rBe%#r-wdYRgO>xOSQ1kZKM6=E7{}s%S z-MRD1wY(GCr#(yiW6PnqpQSCZw*1ipPHl~6IfCaN^|~odWUst?Qu$p^kBHwIX?D$J zP8?==cK4&C{2%4^M=^dqyla-oZ>Q9yvbV3NEx%y zRjb6$8=U-GW-E8;?JIv)-h0}?W(tjaCz}RL*z)*KfR)~+^1phueaq(dZ4`A@3Cu4o zm|{{ucOcS=(hU%H*`-v}R}bhHYD{Tb_NhUzRNtl))1JbonFG zYY#nqR(E=AxW4zr zg8BZcPhXoW%$D8am+kGU$fz*>bjs4x@NSuXok#D5Hxj=u>@NRzt^D2efN#eb3@5KQ z_lf(>J>ODr&5CrS>bmqyYBfNt^9OP__|ot%XRaG`vk3P8=Do)*hCD-b&-@}*An=dSR|Nh&jxE-p-!5a7DI@+7O+Fo-g zhD0v;I&uFkqrS?`_aZqSJ>onmcqcw6>T39_ZFQE}O?r7ZzkPaFmQC8e z2hMuj8^35|pS6OzppXz3Jd~g3|BfFrEkKP1cHXxf$tk@uzlleETuy zfKpuKwu$;`?If%63h8u66hLT&EqozVyv8uU#p(?9|jp zb`hepCU_|?vA?4cdbs1U$|R0XtxiR~IB)kkb5*3)9Q1f?y{Gls%I(7PVrEHyKc!#y zm)ml8x!VIXm9Gt&VhsHqzERgT7d>uT6OtD$Eq(guo9G9nZ?2~*$z^VF$RiZzx$SO3#yon%*Jbb0N%y>%BK%SSE06#DA%OKYx zlHa~xOEzx`vVP9AHZsX8>hC{wj;C=sH)USVni%bMze1_|S*w4*qC~dmTIJtrryBA- zT59O} zef>JSt+m&0>;Kc-+F@O{L-_TAQk_W+q6VC?*Yy@vIrZ%N?P*4C|L>-rf1kfI zf7!}$q_ezvM|8IOnzdriOO|=QR(^Lufv@DR)mpm_-^UTWlOm3A3Kh;gv**p>zoGwR z_B-zTBs6V`u-?>i|DUYKl4EbK>ds~_P3FJuXWkRcF3Tg@!0{qSgn#7@^UV2uOnxQL zYn(YI6>s>p;B}hsVZlf3-3zqhZH1Sw@y@+2_2sHsgtQxLoq2ii{oV7ea=+UKW?wbE zIwyMGlXIW@w1VS=0^hur-S)U$%HZhR^1u6L>{;?IJJdUEgd|cI)lFAh+-H`42A@=9&2%OqNZvclBO+KHXD$ z_1imIn>XbwH7SImnW&sphfo^KSGSST?=VbzOQr+EF0X5{~C zsVu#?{Z3WGZo9bakH2Mz&1KZjaW9=Z+y3t|`}4E>{qJAiUVbMg*>&n`!|T(JSGv2L zxUqV^L|0h0)u&0a6~7q$Y-?oH*4=%}c(T!dm1=8+V8_W%on5;(9y`}>Nc^XhLYPCYrh?8f>xH>=<5G#C09F#Gw!?}DW#Ckfkr zyOBIuVB!t-cl*`%)o1)^eq(#+#ee&6>zFrmX=Ts<{^?Tn`z4MVoO?B8Gn-bd5YZ0X zFg-T>`nngZcK?pPxpm{Nq{B;BG_YE)Xt^k{TCOuhdg5-tf&P$UwS5Nq}!HE6m73M^nwr0jn`5%K9<(|BF zvg7p$uFgI8^WrX?u5R#1I-hKwzlEFit>Csl7w%qyHcV&GPUSwijR`Oy?r{#(j7vNT6}j!?h;=+;o`-umo8uSRNA$}^Si=k zmtWVsRBuh2r1VKz;>+C15m!5c$ z_H)a9+Zmdlqjv0#op0xR^w7+=!V(6Y?*1wF2=ro?Usux9%%jvIM4VniOuD`gH+<_jW5)__Wjt;B55rX zd2yFzNiA?f=;CV~29Fo9ZE=_ocZ1We=I_uFCbd(<$0vc8G;gx`Cp* zj@aWx+pALibhJ)Q`2L%2c!BS- z$cZ1d6CGBX{>_PU&#p86c4yXEd(*1_+rAgaD5Nd6Xo&n!J7Ia@CTDr(50<8m8ywe8 zl3Uqzwrxoo_hr4-DNIwJ>?@65kT0s+mwUT9%F6CrtOINJS20o5$`5@`w?EwT;D0lB z@6zWRI)BuClzsSpCBxOLA)SSS3)rUE%}_O|{4KTA>+(7C?do|R^*7k8tJ7Z!nR9GW zj=00@^yEJO9;MlJq2(WiEj!*$>sz+#(yu#u5*=F&+p^cqyz~FH=;~|Fr+Tf|w>xXZ zBO|C&%O2vC8_3WsvuWGUTblffvV6W@+0OPQyJZ4T^O>V9GLK9S7=NBW=a8#ii|9CuPav*#mY(AuMA z^H;nTJGOXL){o6~ym#z(PhHtRWm4DV)s9}DbvkP-E_8cmFA;g<8|OB2$&#o%<+XgA zEm^ER%|{%o(;QSdvJ@?9l{j=}S8_Q_d2grm>C>ZLkA>0-cdJ{I66Mo6J^GG3-!ArJ z-Nau^ZO={C#C>PInC7!}axmMvryid7`d{p9UchH{toP)e(+|128JU|JTN7W1=Wa2{ z*KR)11F~b(u4YKyhIp#-~*;k|s9MgTYfXUrgs<~`Z z#I0L4H_G(x>9Z)_Py4v%5rewd5$@;u`+kR=wA;h~%}e5lf3nfVC9J&mH~DzCoJ@=L zWqcH}cH)C|dkv>4?!4qts^p;E{du+Fx}u2+tJ&4+wnkibRZCZwZ2z|EjzQ@R)(ow; zZlMq3te;FVyj7{Zz1`YIL$JwCLq?dl`sN-r$9wgKoL6T{>HTZ2&Ogj{HuU($>Kos- zzYN&0w`^ViS^2sjp`Pm%4o^)`*xlv5wC?M5NjIr>{k`j~m-Oo#nWY&L`NgAwm&3um zw3#Ww!1DEC#tHAX)Yz(2%*ib-d-9H#nekM}v~bO8mMbq3I5e93BLxknq*h*1Fj700 zArQzcByc3&K&g1IY?^oBr%QjWe_HIT$;+567%7%5V!R;m_YVHtz?drK>t2z@j@}cf!BV zwQBD?r|aF_Bgp>h$xXgYjpb9q8eHrCF+S*%YPj`Op={bwwp35S)2&ZrtR3b~l93fy zae*gu!MaTm51n^Ut?rZUJ$P$g`u>@X#@Y9)EZ5i1*)^qe(**^t;&sy#?3e7HZX|d6 zUb_<4{jFP{3Q5VWwmE6S&Ud>g{nw0(t2)_Oc~UCscg0Lb&D=H7hZb%r%*w0>yGQP!j@0A9z0ce z%zCx!9@DB5k_-QRSKUpRk{q%01 zR%J$qb+3Q;?yqOwe{}kJ_U$_{oxL&4N%430JwC95#pj2x+%5gT?AB6u_xxF!-p?hF zxbFUekXbSf$}9;C%!Y?5NJNxsLN7fhd=I@Ro&LBW-Ub?ht>K2AQJ*W*IYFwfsx)N9{S`{vlzRHL_T3Yz+- zCN&sTSR7rdAEzTmUquRxyTOd4mid}atHtJ-x5vHJLCzi(hoKj8W3TZs@u*@+9A zy$^&}_ou|4ziB0@dX@Li=XrV)*6=-D?cOQ8PnqrG=Y#I&ZD;xhD;-SNOv~Qsdi5&r zs_A!fuLYGg4by<6R_Xt1jNpyY{{a@<>&^PBCFU$t{Z(v#oXC$%T&zJBnsJ z&*N>rp$heu&*Jw5zl!8xU&S-4nQNcG6xEO$3-=iExF_d7VOpOd;3(nE-5=!|6vD&v zVX<0)!Nqq8>`ITH$u>&nP2BUPWZP4v3@yG_&+TiH-&fD8pMLqIYWpPZ*Zl8KnY5LD zpPuF1m!#CXvtv%zvcx|HHLGs&bJ|EwzZlZq7^YtL@b8nwKW|@cSF`3$J+!p>!OyLd zo6S}{yHK=$!n;Y|@78_2bL2_auIuq@Hq16S)E*fvB$3LRWxV2qI&)C&sFpMl2U+f9p-Cq5{M$+iYp^7OY&D$m^xHf9!-w0Fx|JTg-*!K@pCN}>%-4}a% z%JUg-|1PgO^F2H)X4$q2Cmae@0`qQh&OW7OP_1*&%hUWw#B)2-rwTLJb#5}O;y--h z#QefFOt+h)FMkoVePVR--apYRRr_XyIk6!r0B~{FRZJ@g(aZi_Ny8 zSLdwyQT}h?sx!RIiqawT_w>E*ydcv1M*6n{+hNaMrU#8$cm6s=WgV6@?KJ$>{i!0q zy67V>yYS`qXz%1hj~Sl@uIYHUGfAPwY<}ysLx0PJb7o0y@V?M0Q4uDuz212$59i~P z-S^TRohp0!Yi7*QFIIdbwLSOP%(K5QL{GKuzHZ91V%EpmCQS?8cDnZXu}UwCt++S! z^!|Tu4eBJz-d1&eEveYRH~HIDe(i9xhRQuwTl`L^s}wJsy~cN0{{$_<8JNut>q1sZw`Kp%dyFi*yi)(-2Mo0TdCIzvYJk7g_T@Zp3KrFKb!fH&qtZJ z2JiMw{B-Z|?WrY=zq1XZH^=vu3M()B%5*p>z^Qx<<4yf)h04gr<}L4bZ!Il5X)XNZ zQ8m+o@ZJ`Q-ZeG>&4tI_K6%ufJ~Q>=Yxauntn1C2+vC5T|9-vERwJOqL;7is_}){0 zPDC=M@Th!{y0~GE>++k~Q&Iy?ti7KwwZuk3Z$>ZMp5tN{Z&kjF-SAaDB~fmhOUD_G zz(cAg97ot*?RfuPG*N9kZ{@+=UWe8>PvFq`e&br{s^dBNrxxm{toqb8_prFd_C41h zzi(f)%z*!=aZG50@Hs2f1ua4sW!K)Hrl-F~yfUw|wl?eN z)jElP^UNhAnW`0Zc+!^eb@(5;vY976h`)A6{27OvpZ14`-rUH{^l_iSv$-DZR~(G^ zoKC!daG}7!&i$x~hGb;NT&*I%e+51k4V}-Xrf94@_igS*##<2!vbwY`>hI6n7AaHx z$IS56)V+_B4f>@G&1Bn-D%Wfl)jI8EkeArZd@s7jHR|E!4a;RWXqblz8Ll{7#nZlP zR+h`Nl=W>{t9&-;oPYA7p7G3M&&`!z=RQ6+zxaRue%UI4=L)-5|BLi8{kPgsB=`L4 z=y|^{_xzTxtbFO8Aos`5+GSq0R6wzUAA@91iKgRP$Ui_WL zQbEg=nspyKN*gboW;9;-sA0m1Jv;24KYedMXC~A22PygGp1K;2YxcbO5Li8_FRml6 zAfWi)ndzqwD%NcX{LQkur8sNLWw!6znm+X{A&Id^8a()lj#SA$-~OeqJ93A%qF_{V=fPv?*MC-b#RjN!SiX=E z3-UgeSg|wW3hSP8ZzXg5ePphxzq&oYT6~rpU^YzzkVr^dF)zz}nazXUtU%Fy>$6wbz6td2nu>VQg zQxCgIX}2$Yd6sAY&{D`cyxjdG?*~R_w`0z?-5&cD1+UwdmhqM=RmR;pw_%&ms?FPE z*EQ{35_BxCXy%>z=Q*73%wpV*{Pei>ux5o>n6Zux$mE}Z!nELvfev9aBRzF(o%LWzsK(^ zf05k3_t$jgrAB|3h(>T9ao<_L>BZmv=U*f(w>HQ#z6$PXpMGQ; z9#ija-nE$js@%eW-kS&K8u;_&YERCue-g_Ta`Wb8_5C0IZ3^X&DZM%4x&6PKv(C3% zXWOOwa$Y?_Gb6d_!tkQupC& zlQuMXtFu=ydZ*aiT=>OyLgv(mOqI9H%52P~%~KwAp7%6eHFLX5rfab1r%V-}mAez# z167{h`22pK1n_C2hESaKno_`@KcCx#vqB>kG}lpY~^A2UFig)*E?8(#rn_ zCtk{ZQE}3G635g_`d&fdOK&~kSNGcJtmYN4-D0NgpVX^OJB!y>l^tyAmp{zEZFX(g zeVejBeLJo`{{85Cxy*w1bJhq&=_K#G^YOosr7WL?bjHeyb5u=sE@NvbWlMCcicNNV zbgM3Vd)gPLofl^3&wH2_o?l>eO}aOF+k1_>Taxql^{;+2(Z!S}n9W$o+Iul~4GBw6v>*ZA)Pk4KF2 z!aiQBpZ+n~@2!-`mQMj`X?lHIw@EGId42Qg>aW$?AIqGGZ9XT~uFkSn{#IO@v#WFG z;*~B7CHGa%^I4=BAhqC>mZl(&hR`LirHf{JSt%B7`gkdD`>}FoCvBhmcV78Unl`Pg zHO)cla?YCUylBJCo@2NUB}ox}L}`ore!;cvCp1YFwkaoYYPrzdni_Y6#1^<=5+6rq+)>sU>nyBxljcsS_p z&7aCpPP+Mg0KS=+9B$(yo}#aVFK&DYzW22Pt4c=yoH^5b_3&hNi>c) zE}wrGXS8kF2}Apex4X>rPP}+`M9o>TwqW6nm6zgu)B8kZHoc0zreO3{sp46K>ST)v znm3FD{BE-Gx9mD17$dRvL!sG;h6{C@vd?cbUZlRWn z9GN~fYsyhGo+nL>iI=n`w6A8~$zoigbbD^zgnt_wjk3?hr+%0r`RNJoUzJ_iv-8Dk66X}|F-aBj&CCAwe5u85 z8R_FIiwY`huJT&2lsA13)@1XDyA`*_R4ZVr@L4J6z{yiCS1wxh_lNzIO+9Y>FRdz= zVs7`JNG>+tIKeXP$wp?S^PhkG*!{(Ln?yj%m2WQT_G*ns!%m;Q6)xnR8@zm0jJ={) zz%-Ze12&U3NEbeTbF)tQcX?+w=Opf3Y$uj5-rf8A-%Eev73zB>&OA2w=G$pgpMAzz zFz?T@b62McelRPY|IOx~i{#OOoOPyq+vOV+jg3FEzIlJx&9gOi$44o1xy}4+AKI^Q z9)5i9aGjZ0Y1N}Uu{ShYXPP^-EqplF;n{^d$Ff6w6N~yZwr`m4P#3j%R{Gil%WpQl z(%qxH;GYx=n`@HeGshE$AD!QN^0tbF7LO#?#$$)8?iFqQTJM`7o*U!I&ZlgVtoDjo z(Vq3qW ziay^l@09%_>qk?5p6L3n^Oyfy**(r@2Txvm`E9+Z?u#F;Yv&gnfA3NKlCgu^V``Ge zHs4j7>q@pgKljw|zHw4tui?}8x;tzCJ<<(-SZ-ssN?}s_kJJ7Bdp=!hT6x`OYRi46 zV_%+3_UBrvF#S=kv0P2&QKdO%MoUh1bXpu*el+yBl&or@n(njXb0;xu4Ux(YY%u1! z;*$DG#6!Vy&eupYo#O%(zTNdpmfB@UZcUEeckqPpKA~eTRzFtJH+K)3ylHAnQuMo$ zzLS4mc#258Vsf@V)Y351WP4eQkcX6f$k!Ge6IdAFX=d3(E8bJLU8YgNvkypgn4NOx)IK4-sO z=PVre<~_S|+b-hMQRx-)8_X9iNlw21Psun>R#WlrsN4zg`~rRQ{9L+{Ql}^je2TL~P-DziXMp?RIt> z&Rs4pr~R!rJvlvf|EYUZ)(VLjI^3)Hk+eqN$>3bsF^3&`C%H32)S`3e_{Fm+9ZRUgeC>8KWn$4ILoVZPW!K|oD%D=A1@P5c4*mt z@Unf3&7C=_T4Grz@-8O4I8oP`v4vjUEONkvY-3raI8>E-Z8awcIZmCukkF842%AF+}!xnT!$g0 z;*apvl2_L?dIOrcx<4J_D%julRXIf==9+>0#oo)4%}yGBkJV;3){0J1==pc6`~ZiC ziRmUi#dtdwX3Y~G0`DJKRDM1G`0B37-;O(EO<-0H(%bp=zKDjUn%vQEcUC69|2}oe zk-m?vC*9M|SlnM;@_zmH^|y4(AEbLsp7qn{hTUb=&Fm+aXeX&YFKAjKm$6qR>2>MX zPkZ&b#W;c-quS0c%0JS`x4fYYk&N^SCopysy*gr!@Q5 z?)l}feU0O17GM5ip0~U3*&oYdz2yf#I_7NFld`v9tG+)^IPG71)&$L9J&$AgO@^86 zM<<{D^!Y}-m+Tha0I3`7qBD3upKw^Yd!uaXOZ|m&{#}2)KS8vFC9G(-rZvy`|8Wc6 zwq|Wz+}S$mno8`NiyUGc4;CGn@If!&c41Q0pLA!_h=L5B8BxjWo}KVxJSAwq((m%m zmBzXM+26DG3F>h~&i5@0IlJRwyOI;+^+lFyD-NZKm=;E?OFTfefNI#vE=pLO^_ zX5nFz_tWqHT_f>6!f?vAtX0R3n6M>Z?(?;fYdBl+dj9#zc8eDa)@ylB+V+p3n$z#m zRgVIBOdg{0Uh#RrJNt{}vJo zk2fUBZHr6DQaaRsH>zz*1oPqlwq^HyugO2VyIWU#_Rb%D#{YvV+}<89=Xtj9Zm?zU zp9%H9txku$w!NkDT0qaD}ATPd(|s`(o6;E#)-=OBFX>@?$&j+~#hGUNzS3Bi|d9khN3OIG^*PKt^zOS6P!Kh)y)XELdIoIpN#C4S4 zSazm)$JRd8rMDjWduiM$u#vdE{Kl*MRWHqdR^7B~Vdt74cUZr1)duk#CAm|{{S)~t zEEgWgc>hFSaU=86>mN>Ud~n&>uIi_@5kH^ww_-)Jxs?o@%bCyCaxM?ND*p7eaQOb?8|#pARqH#1_xvZcPc>)-yj%(ecD`K6z~)@x|YI9_Y4 zFZ<7I7kg@xQPib^lfBo2la-kMn|=MWFVW)KjP_mp`=uFY9Cx4IBVVBH7u26t_pXRUv&ovaRX4zR)e)D^d+U_imv+E*zJR6sOwzC?$o*>wWXHrS{~#-TQCo{atqK z^gC<$if@NcUbo+6Y^dtZCwp#*JD>c0ozE9;ZT<9n?f=T(@3$|%`{{yo{F6m*uFvL^ zN{csR5iHPsS>$;CJcos~wezaNz|yISX|Er@Z#HEP6)G`I+x+yMm(aq%x!YLs7A}lE z5HsO@YnER1EzfNV6{%ZK3D*_;kDF;$*7oFS>f)p7Z|+HE{?-g-iplp4`!b{cl9*n6 zaEEQYgLP)Y>AekC91i{e6F&dU|AbSb-`BXF?3lmr+^WOJzTNigxhejwe{s|GpnXwB z^&h6p{Qe~3b;kEv*V(sqSFGQ9cbi{+#PUN7Y1>2k17t56y!#A!kInunR-4}4 zxJ~xxHr#8zm*L>-ZPJO!mWzq3c;iU%p`+aA6$cAKh{@(Ot{`|b4Grv!&r`y=( z-@ByV>7FCowZSa4Dtf!#NAu~ok6xL@D_VBWvYL%|*VJqe5fk0jbFY72-p&{u`2I`Q zh4+^XFR)28{Blw*DpqdVW4?&<+1d7qqDC4_#((Y`t|?r-_oPbpwWX`#6t7wtc{<%X za9Qa3Uhd?oty83%Rla6)$@tp0yj_rzA~)^L9UhgHr>0JtRJ4-e@d`%eML%15Q?h3K zeLd}jZvP(Dl`k`cVq$VOc5Y$3x5eG)Y;lpK)XAxn7!3Mdgu-RADxSXNW6an-X~vqj z(;1o5p4I*PyRLV2-Hn~6PuZCCJ6=3h*3nnm;P$Uie=YMDzx<@-#~D;7PC3)m`eKEm z>H1as4D8wO1)XuLsES>_b~dB%Rd0&^I#_-Q^mni~wUhMUr zI@7H(^U(ssxWC6L(+-?zaq$tMh$SIYrEB<;>SyI_H@$J;beS5@mb)77d?K#= zzLX~Mz~}R=>UWy8udWrEFA(g?eX=Y}dD&D~u8iEn9@k$da7bD$TfWOwo&Bpr^EuYc zJ!$;X+%cs%>~Z-&;`cZgg4BkH~ZKwTpdrE681%sQsyD%GR0U)Aly+&G}a-8oeq95Z$KDqmH{X7|FD`rc}zAr*=<(W4uo&C0nIp+F;mHdZd0_RVkn^p4t!-PeL zl2Sj|P1OsSP*AYQS2EE@A@a%d^wV=rZu~SqlwD=1nv>o)?#b)zV|C8oGxIs+#g+Qt z!NU*7ta@|>yEzuCXjoiHs&*9hskDB4{=J7~j=Ucq+s@aG&!5Kg`v~~!+plyfTCy#M zohfu`&9BXeI9!*981S#jyW)7<;k6+{V=TKY^IRt8XSd(ge2~ue@8P!d6wI&J7Y$f% zy0^Hbfc^IdpJOwZUO0c+==dq^b*(8$A(^!^BQNBvo7VNINK@y8bJJ?B9pA5cWM+0w znVNWbAJ@js99vJkef{j6?!kLy1%`ai>n3vbFL42exTu;gVpisE7gfd4;*Xx$^US|<6AWkIvno^UOANY*iN(W zMP7D(k>k%~xgg)Atk#p2!)9LBT6Unw)F+)o?`hkL_1hNQ4d-sQDQ>uukh<{V1-@yK z3UBlB#BAqH{gSxj?Z)!DZQG*L6}IUI*xRY{yx@5<<-!v2R~ixq5^2|3f9)(vt%|kd zx!R~0w)==?Ndq_A+Zjuau<1-9Se-_PDi(ALrX??w4Nc^|-43y?(Pyz#r$Ig3C9WdB|JI{@9!GxB1?B{yFw9y5FC< zyYJDf>^hhKKJ7xELNX=?pA}Bpq<%5qM(}z}iSF{V2_h$cHYW9XT}>5WXE*0;$vqsF zrI@#Nmi|*ApJp4A*ns+v#o00Tr{{Sm{9Y}&f;-DDr@ivR#qb52`K`9_vPo&@-CoA~ zE4Y+rjwX+TMnK}e{tZq(TV9|2TI%-fuU`7A!;>3MvaEEvc=Z`q6orVyUD0*rdmcJ(J$fc}oX^zMRlNV3^By?DDOW``N*4D?#Zt~vg zU-w(F`EfGex?bqj`qjyb?Ups0!R__GOS2y-ZZQ^}o%nH^(vxo=#Ae9*adaHf>pOS6 zp6^z!*kX%ZiLZ~|z2g%V{jZp+mG=(T)|tey*t0!(dt&gR*nrjFBt@T zIA|!zn6b~&V6<#|^cCzQ;tS5)I>_{>Dv$nEi25>N!M}tn&%Pu(TvftMLoZ-UV6D;?X5-j z+aD)8d_VN+$NPZy2R}+M#F`vVJoUj;@9u<1jnb>3p=EY;j6rkaFKMJ(T~9CROgS-G zc(*6(Z^c)7IcG(emK=B05LOM{;!|zl~z}uKIBr$Kf^DYIhtYf{$oE+q}<*YnA!x~=oO~K?G-VK z^125EZ?bS^nR32c?c8-p>IYl#YF7*4Hl2%gt8I&x3(Lq>&UcKdxw1$^b>|B1+%Ly_ z0@TE&Et6WXuin7Acj1=6)nAU6h48%4TGRLI*_X2UT5OA?xSX;wQg$)iG?AKBe%MrL zUFXSro^jFJd?OqqTqIQZ7b|R;rt>pR+iBDEZhnd8dI#H2v!w;s9%=E|eKm+@bFJIf zz&AZ&{E_DV@}l~u-w3MA|GOdD?bvT^=bGppUxK)jRN}0h-n6l?M2=r&#j-T9j)1>U#07Nv_w0;2cBF&nmD;DH#@{`k zEV}=sSAG4o*3V)m&5wtd-BzJH=@?w`|UV8`>CarpV3)~MfeEk*1ywUuY4^w7m zB&*R%33mSnYv!KhYq@vL;LwD3YBwh>dF0w@d`)at;*W%hSKh8x*>T-tWlQSOh-udy z3Qenb9W1Ev5dI)s$g=mG{jSJeygMS261h*FJsFr;Jljam?B$y#MvE2U3tyacD9it` z&hyjc&A&_bemmFBc(>!;&wDHHEj}i}z%B5yyLXjlbX=n1pRVmDyB1%IT48@KYvINh zdqX29YP;?DV79lJXJw8g{5b|!2WTfBGmuKv8kU(Q(F^!lcDb0O-TJmcY26((?`Cy+NP~9H_0A!{=>}F zH@V6$;IQQ05awqY{N-t^ZB@OEb1tU8nEJQSV8QAQ|194B;=g|R)~z#aqP^xiiXr!J zX#LWi#BASq?BNaHYfFM=7o-a4O?RwiJ-qn{&+Ztl?$03}BI@b>9wIz%`S^9D)m(OS zTwASO8d(~;H00owhvKrziiWjM=FMI=snc0m)!ux&Yv`uxyW8h(T(@tR;8K%q4rVhCCkHJ#Vczb# zX(O+UGRLtS?n!L1DzB&SRk7Z7XxYSg750NNd|e$!qneu43T|6W{5>y5<6h$3r?W** zr(XR2JN4mZV?n>YBC9qO>;I9ykhm_g(4i~VDJ^YR+k7?M>-DcBEKEU8xbk7i^j8fM z2Dff&|7M^4?#&zKt%0%1oJS7%=A;UFs|$W}VdHtN85-C6$vLa>3-5jpyV-Z7{(k&t+!8KAgTb;34AwMaqndMQ-mTe{LHW`LTJiDvAV&jZu*A8Bc?CLI7U6ftEa@Ce4=RTN* zD5Y?2monh8t_YbqC0x{Y^^Tbj);)dntm~8LrM2mMG9)<<`=k~~=KVg;Ygu>HHJT&& zvHAt8Gq0AG@-Hi#~pqjVOV~177NpELkSKQ0Ue`! z1~!Kr&Jzw9-xX$WXxGx#KYqwz&IE-npC+D`Zm!wP>QnAdSAX z+I&`7U!RwW#rM7i%U3%J@yms}hP)77^L0h8BBSW_wh6^Q6IxkcpFekE!=39)n*+tq zKa?-xW%meUSlST%<=WnZ&p&+pwb)9Mt%U8;$+#8w=DJA)Xh~T_H%>b&-zB*u=Yfk` zuCmR_9cfByyj4QtLyIituXJTtv7FEQa;D4nD&xp=(f>bL#`xS&R&#kf`CF8V_?$fR z3mb&y+HAU|*Tw7nd`JI>i?CO`9&tI8K&Y7kvcz%ujhV<+sJFB++n{%$c@!8ZJ zbJx$W>ThV2G$c}F@4tTWM$40(&*Q{}zyy)9ppY5L$?QflT)Z!~Ca46281{6g`nn!|t2Wuw=yG32 zeL@FUdJv!fnv=qh4j#Bw_P70-!?x>u-SVatzs}BIv^BYQee!~>5Z}l#V9ly<&zIlGCVA-5jg9{8xp0|k|>b%f!E@aF8 z9QNgahqZ-dZ@OYvMs8DqE~UQ*Rnjbv(8=-r~smUKDyk*A@+gEv3UnY^6+FZj4a zFwYB~C7Nzl5A%iE?;X~2jkDqY@@D0?6s3nZ;*MU^Tq*I5Fd=F-=7f7}%?0qG8U@)}r9}F4`f*yxVZj zah^>P;pvv=b=pp?W8CDO*|JSAZ&p;c1AqJtuT@Vow3sZKE*50#JGeLGIZOTWlV_GK z;h7`GR>D>kmGOFp`Po^uAKf!{-ffFN^<ka!I5Qu9z}SpIL7}ZY4;{06({zVlFkFGJ{=a4WwYjrDfd-#&*955R+3%U zJbCkuUAsD$FJE5q`$xy}>Dzbd^Tb88vPxdsJ-t69wAeC<`C!=f+n@MnM!Kj>N{*CD zoj<*Nf;RVUf9BglmcKSfw=&-4ULE@4)rM9_<1@z`n|J9u@yi_#`+tvnbHF+TGk_ z>Z?|E{L!x0=cg<0uAg*pvHo=FYIbk?-#Ra|Pfce0R%vv_G(KtdcaJ@Z^Lak|OzT|u zvMBX_&CcwcrF*KoPfS1WH+jw$=1(7IpFiQQT|art6%_;dC&zNx1K5{6f4kI2{#5Uo zv!@qpU#Z*0a*6MruX{FA`h2TD8x1~eUEX|V-74Yrr#@vJe&JnpVq){rwnK-aCS9+y zWljyvo0(D`FhxVD!XWj(x9ZcYb%!6<_SCJL##)n_Y8lkW)5H_#w7~K1?dy_kd0Vdu z2)V8e3B7XSSo6`MVEGlR720L*Z4P|ZHCa_?Wlrpy;3FZ8rHYDboE%1q>vwwA)bu`j znknXzFV6C=HC+5|YofN|`~$_CzU5E6H#tx6x9t5ByQTzLuCFim(CT?~``8w#{9DXg z+f(o7sWTge2mb+KZ@*eyAifgODhFz0p z{^QzGn@3L8~HxqalDy(f!dEW74sr;lffA@Dq2jpvNx2@tz^xe|?&?oCa z`9i@-Q4^9fwoaMg{Exx7^NEV8mt$DhnH^7ye%4(U{MgjH!w=Zv= z#aef-$fBo@o#A)ik9%9|Ui>cZXS}Wd;*Nc6eNuG7feWT5-pbm{aGZ69d!DYCaL36v zv!@#Kr7w7fC=VKX2XsH)(s$!T93~ zmb=ZWG5;T?9and>=$?nWt6Z`7lk?WsYNZ;ymwV|2ZpmdZPP=;S<@4(Ef6xE!-tVNH zAv;alEl??8kJbf`70VwSy}e-HrKqd}ReZb}rayj0Td*IB(^PX}b2ygV_O@yDsn9Dz zQx-2dcxFpTxa@XEsl*4d_qvaF@7+8*MAUH#V}yzv_vB0SZkttaVU)Ri=voN3lX?EV zpE`YZ_e-C6*nOMtJNLqR?Qc0cs=`Ne3)VFEHM&V;d}mn{nEh2gU4+@W>U-R-!t3i! z|NV2M;<0UDP*{k{`u!fi7H-lBbdqFwG_h-DX7z$9^;f^Ucl<~f<2YhgS2LyM$BYem zi-XSoIJ@9U`?n>DiSphN4BjS7vez4E2X3E}JmuE^=pqld;~iffNtHF|?%RJ2X_Q{j+ZFfE_xl?^O z#c+4|yQZUjDlyZ{dXW<(kVwwQuoTt&zCBeMJJ3_Dfm4qNh(jt5|Exy8MdR zYHFHVR8cP{<`T|3ThLx@r->P!#>JADC-Ez&V|L0vEn|B-d8 zK+sM->lI(MBe*p0*L<6+6ES(wR+b1M)!LY%g9|FP>+k##soTDL;{wyL`}3=LZq%eb z|M&2h@NGHf#q8zPub2C>Ki{Nkysns;(e&glQ`3O#sfA)*yw^i=vNvpX|6wjwYHjw| zK<9?E-1Ex`Cfjp!SnsS4;(va)pyrKEuAE@3^Oarv`x_EIe*AcN&7GF5(te*#ZomEX z#VwY~)QgKBy%N-$zutfPQttT4`)YS}weOAG)wa81`*!(P#+p~fay3o*+Dkhp?fkw+ zp}K$B3FUq}#ryj43c~sl+O@mey)!dEZfxJaS@}`ZH_5cz5|{roGYgOZQ{MDEpzY75 ze+@f1W;vM5e4LmZ^L@r|;cx2eveRQP%)aaY<4|Yxs(9OI34@-u_lxDUw%^|Dp7MX0(;2tkqUX$wNpm+`*N^@6`u;+_ zO%XHBhiUt*jT6#6ywzAeuIumQJ*fre;+FfDZ=a(0-mQ2W>#_xFSFUk;Avld~L0i?D z%Mw;raff!htiJkqPOY$o@b>1A!0xS5%;%-ootr8meyYiR`AV0dGYKbbUapz^^K9v+!3*S(7)nPZrb z80Nh>!=v)rivRel+7C^9>PH(MPTalj`>w}**)N`6%XCOrnENbrUYxsuvQtOj-uI8+ zrR01)9WJKSnys31EYj!+)7Ge&E=lM9->XxRtCriY+Hj=%=V8_R`S!jWZ!5Zm{wyjVagE|&V-j z84H!J^@-aX6*<+Q?N(LH7VA5iyq8q^I3`-ptAG3Vj`_U^7yYR-Z+-3CcU{uMy}o(> z=D%UiTh7d|DbiS##MSewboE2uCr`Bhvd=tvJkYTBmv(;TJM)~JUTGoS%AJyDz2tVRJD0w6x$~(9`XxMhTk8(a6OwE`5>RkBOKW36ZHsV|vY1GU zXGTi-hE-Lbw`S|;1!T_c5e&%PT2|V$V^_7eteF10oAZ~3M5bJLD;je7g3T=(3v-LW z;LOLG8YiY6&!2nZU4YQXPAThcThg!Vea}kf=oT!ka(W}fyui)#UH+1e3vtH}9>}#T zSbFlvaj)_N%70tS4@@sJo3*Fps(#Fx4_0Q6x1D{OX8mK&tk>L6uPsmdY0)Qq{iQhX z|L&;~Tdwwe?u{>Bde$NO%;k;OY;N2xkr7TvHq~fXnK(T-D{_D1WYYr958Neza~_D03w)H)bAW~IQ*m3z^1YKoripf`Z%Oh|a$R;s zuJKIMx4zuy>)yXEDzB<9NzUW!&^Nj`BS8P>5BckxB#yTvzF(-UpFH3CXVVV}0oPYd z{q8z`|1L5-KKgE@Ve*L#$@;!Bx2Cq)2MU{+?&Q|3?w0-7f6l?`QM9#nu31aluKkUx z_&5KmKRoq%#-~Y_ntUY9Cu&|fy64U5QuCyG#s11iGEEEfW3D9F^jl4-TJzE1*w@$W zCz)gxvL>h<+>_J&TvYA+h3CrEVcP>b-`i3uRefZ>WX$tP$vm~ZefUe@~sW)7o90H^rH~ zEWy`9)TCfZ^;u_LmF=HCam}p%pCW0l5wlp{Zgt(Ay-yz%Hs9;%<$6?|w0!H@pIuhp zBbjq#<>E!2{%go9OAS}|>7E!c<2b{!g!t#V%FFKsE~qR0V1J9@#e=&SX4kVBB=~u2 zpS=I^?C(?84zk|3Z7!Xtt}$(&^V~>VPNwB6R;nDYL5;GwlDU#~=-JQtg4?*gq&8l> zmsC(N^~w`lAsN9Qg>KOaNq&D;UWpW5*M87ofBA8?kFDw9tnZ99w#a5jS6d}n2>Y*3 zFc4hz`OOB6O)=lz-&9LgJ?i!MT1>QN=Y^LCBu~oa@6Bqmv1Z?eKP@X4@h@)k9C>E^_L~*^?75mrkaStz8ROkM+8Y|Co!;C3qkHw_ zs-VB@^Vj~D=J1s5I2#aSDu&c6DD3g@$%vFqErc$mk06+%fE;8EXWMJegBMKL}1#) zAe-91DUWw;-7tHHvlA0@;79))@9m3|C#-*0Si7N4RqT~!_hg1#OZIQ?L%I!hw@rL@ z`*fYC*puWiS>5-4-+Jv#^@>k!SgO(asONfoeAR<*v)BFGz5Sp-&%3&s(JPxp$Y>5zeLhyK1_=$ll@S{8KpJMqmJMnqqs{4t~yG~i>-%;J{c1-Bx5^cu(KjjzW8>i0Ix2?IV zajig4O{T^#`rzHm?0z@4s6TeLiLVvk{x5Y?h)TWk>}zVp9TMyJmVL6{5a06l2(uuo z71s-%C5!j0EWYAr)Y>1G`(V?{!W#=t3Kf59%#ygm`{C58>w7kxnSQ=@-l{IcJ$-L| z8h>qAv8}7~W1)NWIyn=CJXWpePp%b&-d7N?Sg~Zq{ zooD@zS!`MLrbo@Qecr*lm${dp*;M?QZR@2yeIIW*Mr2AG985lW@Sxzavh>2jLg8@p z@`^mm#AA6A9egx|ucXg-b7Oi_wj3|_mo4dAw%dL@`RBc(;>1T!pYnUQD7I(H7f!pf zm|a!b_GybxORDbsC4NSi`cAlt^G*40Q#UilXxZ{*k4pZ2{w%+~@yU;0XJ#7zth;Lc ztadIZ+i}*rAJ?Zw?@OG2sM+x1%<_dH*Hlh+OlebfM<3KbDH+}Y3;5$_M>s$jL57b zOw0ZFO){I$JU+mme`ofTkOB$Q+6|T2heEF1U}rn=qQd@l)4|8>xuK<+Lbk1*@9Uf= zzUXY(;CIv5PW#XYUQTfBf|)k>(GdJ|25jyz0xu$IANd-`?HSov-`P zWnJ|P=AEw&et-JwxcKSM^8Z)Wba#I;ezJI}_LD{3`crTA>jgP-uY8nsKH;FC)0QtC z%%8smn(=MBo>IgVY|-;pLnwdljn`9rW(jsn9+;&4YkB{B?fLtvR)66OzQj>#zFjwV z*Oru9TMW}v_H0qFxo5li{v2P%gny_{Gw*U9GZF%(h*WUh1@85e!9B6;HwdUun-xr>#Z=Cs?U+D0P&bY1}Ilp)M z)kQ8(N-b3TYGt^L$&!dxZb{Hm*6bF;6bPFF&b%|NRvHdlhbKezO#;x8+T{{l3~MEBNzK+29Ow zmuIT-50A-c%~@K^x=UDWhKqH0w|~ISsHbxG^4Ik2dB?i*`?APMp+UOlv36dpK{gZb zE(`6o^=>uw-*ouo@1;9dF!&r6>RzyIo6wn^-7T%|j%QgjeY7i@$`YIyP9AAsnAZ2{ z==t9h#p|SxckK-C-{AW_yuu+{@qBk{c!+=rxafwz1&hr^k1A(qo_M z$NjT>^ys0o{(s-r+UM+v5ue@#A2iOd(c(V7_Lsv4otj{_&%gig$iDDq|G(evcl_^H z1ZZsJK01GO`VpnjHJe4mo;$ufzF+OpU;En23=H<0JzX3_Ty(?ERR3=EDDipo{*?JU z4$&*-@ijWP%M((6D@+uymzaM1{M0(}`?`{|?WXR%Z_7SCP2OOG(aKetjrZ>JSo276 z?cTgLr;*d5L+_2^fg@Ag?~9l3mU?ORDDCH$UTalj+qXNuE$)6Rv_nqiqeIgscFDDt z!LJtFDk}WeAZ;?|%K@1g!b?xS&e*1(qVg@^XYS2{h<{&?vQ1Mw^=ki*7e5HqWHu=2gU_LJ5B-yYJudcOI__XqvwxOTBzioE|O^IJC$ zOUo~{JOAW!O3r3ph-GPB6<;1L@+l#up{?RXllUgqk|`{T;xCV~Kbbb$xM01m@6n}c zj?(367G-~BxsU$u3fgntV$1H`JRL=%{QCY&XD@HLSn|wIUpsGMV&Efri8WV3&)Pi| z@eeB6`D-D6YL0nJK0EUVCzj~~5wUFXKU*rwJ16TuJ1ZnUC-}~!(x~YBLX1n@^S>Kj z_nyyLocis`L!-<~7B0WP2DbeP&I#JBF|F#$h9}PxU2SumS#HfZ5ySMy`N-wl?k6gy z8aTB%)n_^sTNpOqwm6iwtD-+yrLx9pJx6TfC#^Yi_)SOEv%3}n zZr{1tCK_poN38OmWyBvA!QXVnIIrP{GSBamPCX`B4&3%4f=5iPZ=E{m;C1SUbISYg z=bdsz6<(}Mk}9~gNH36H<8MAw(b?)ozQeZV}?&3h8xT`AbX`@HeOl*D7% zz05glFO-FMf7#T?5%hv*<-&I?+NW~gTi)EbEo`w^L{jpKXZlC~?BMD&*kN}_@EgDB z@|=ZAtBQ9t`e{AbC~I?oC*XHT!qsi__piUQ@e&_P%7h+qF%coXZ)(TgE(zc7Z*K0M zGELOm>yqBhz1(L~;+d9Al`-IFTT-&2AZqr%M|T(Rld4EJ%_z~h_NCqFP1>8aGdAyB zC@WX=XHn$z#pjQbjM$celBySyS;IG7(y zf6u$2`G;5FyTnfUMc3L2s$?%|-`OH`UNP~)yu*8~-icrDo$~nMjTa9}llAw0R_IGC z5-CnuF#F;8>8;H#Zmhn!cG~%7!zDkqUiy}J$X@wQ$Llj&g@l9}OSxm$yxXZ!z}67w z%$z(&^znktf}QG11#Ve*2=sE?&%1izK<{{H2iKU4w3W=CiiTLpS{YSj*ludw2WM2EQi>k9Q|(h_Jht z8`QY3X+Amq$&tGge9TVWYsn2_NINv)`?c%qZ~3^zM#kyA|NmA?QgHpT-Zx^EzhBJ^ zR*ycu{F=Gy%6YG5T+OrUt~PB8{BL^W-jiabq&rXk-<#dBU~%53^S7U!ez@31``F3F zRq@pq&#-6mKe2TQUe3mAICt0N`X%QIm&wa!ba@tnGF zE-^65TqN_G0&BYS?sD3Rq_6?+%;*RNo5A{({zJ3>j(8jZ`j+M z{y>oJv#Ri~_BDUDStVO@r|ncx<@2X!FYlkuKfPA}y^U$? z%8DH?)-7%d@?7|H_x8M|S^cjQKddvTF?pV=RrX<9;k^6{lhl_=X=%Hwm%kLf9=`MI zt$li{4murHe)8}5v(x)xo|;VD`*8aI-pa&(>g^9-Wk+^g;@M=dY|BZ`<9W8n!;O!g z`=Yte!tOa+&ikl-UVYtJTJsncKb^ZTcKVm8>z+Ibaoy|xCM*%-n%7VY2Ol)T@SK+npkeXd8zWdOY5m~e)`W3_x9UO?UIyRpDpik`JY7E z`B`NTe|Jx>KU#H8{%x9!m5hq)tDaQ(DW@hr;hI^!-99bh#DjK$XY%_eYp>M2nr2^q z`QM#Mx906w{``vJ-*Pf)1R!+9aF03$EieGx~JURtbgj# z_V}ra%5A4v|2B&*Ui@+AW9EXmXnprN##3yM$22c&Ub~MipLg}mpS#!X;ybw`>&iVI zwZvl^xzk?6?mng+Yx(*7=h!4R=Be`{FMV45+-)ndu#o9d!B8;gO>`Cb#g@+cUo2liPiim)H@Xz^zhj4m2#$8$FJ?Gd++z#EYI1s z{`fPwyLV4leBX01iU0k9)cn6n*(YRj!=+Y09sU?7xcB_8+kqcyuc{e64)I z-!++w??vtT^L%H{<7E9Krx(?~pSI-is-w5w?631_v3YKvax8ep$CWd}W}MAD<0krK z{>sPf=YKx>?YsW*1Y`T1-`MIUT8oby){lsD*b;YWzTD@=@Gv`xMt-NrC!a;^)4c5O zZrGWWy!V)+VV_ZMjQx@--9xiFm>NGh&*3@Ol__#t+-qz4I*V#&lY^5p^cGY)&)#}# zc5>Af^YtnO$l7+srEngq%FMDJ4NmU=GqV$~~y+1pJ-Oxh=3W;=GU zy-H%Y$DZ2;D_1PNb?Q>z=8xO?j;vd>$K_r1Pj}6;{!>2x`a0>Z`>H4FB>$#9nPBI1 zEYWQ8x``2T*&Mt-1p{NB9*T|=Do)*I^W}u_qz^}D7d`DwP~dR7dGi2AeGivM+tk<{ z0T;4XUUCbWd@k$E=83xm^(X%4kTF*C*izE7Vx0zSvR+f{S%pl^3N|H{aOGKrm4R}{ zBKn>z>9VwS7c9KDDrCWp?4Gv$#fO<}%6#TpRsNp+KX%R)}}3*a3BU8|j*(rTy;?^OiW7ervX?e>JC~W=Z9zm%Ak= z%~`bS5&MeT9{DqK4gbAk_1>?4Xy@*YXOD;VcPsqtefUqES+O)_<(&sJ-RB%RzMn_C z@2$ejBPTYM^PM>8>>(}9mu8$g&K?&O^>>wNZoi82uQ#aynZ(CqG3p;^9u+AoPG{p(voeptu%29*cS zdd$Og;MI$xY3-~XeH}&#{U(S1oK2rD{^aCD+t(@Ye<^?ZDL$>Ueb>oNX`L>+Rvy{( zc;4yuJ)dtFzuzY#4cXzbq<2O90Pm#-+^6~4`aBtVZ zg*iVA{ip5O?YO#)`S5kE$_S}`;Xl1|Pezq5Pmz&gnv&)8d*OR)ezpTPMi*x%9%QRj zJgg(0XRq*e=XzO_IIF{Pdixb6jCX0Zyjrp0_C{Y%9+@ZC8!BTgzVCV|um2-qrnsTt zBmHAL`+2AQIiuovH+_e(!J+3gGVe)enQ=0n++${ z_#NZTTC&dwF%Ms;Bn9xv%=wBlpm33(Hrw|1bBh zKl9(DPk7NCMO&x3**4ifu1%j`o%Q`3@9MgvmbaKpZ+G6Q{eG9v)U?%q@6TuK^#@Ih zbKWaFuC3qA@M?S1NvkM@c-^ow&863Q-iTRLo<68k)t&K8+(_{7p32WWulL8^kv+cn z*_oX-X9ChoB0k(t-TmP=bIzXYw-jYc3m%xSbyh!q;Ed+sxadz;e_3DfHJ!cd-!z88 z&!@sF&&$r)qc~A!QgbI4r-+$Jr_0=mFY^m0P3Mc%Z`(85e?^MQ(v$A*JOBLT)O)nr zY2AcBZf6=;*n;+LO8l@p-*jTLqwSqS6SMl24u5;E$ggr;{@*>Dd+$0Ot7?H1Gl~7# zt5>Ponuf~#f6um3%SX_CJ;&DtNA>S8aJD!_es(yiYP4(hov#oEx&Hc_xlTd;}~b3`RKc0 zuEp{`uBqmW)qN_{A0Aq7^nT~(TjJ06e|tDT(5+TGzou~QlqYH4nnlt!weRy^rQNn) zkrC2+bC2Bl@X~(STYo=ZmHrl7l*#w*py#b%E8Z;!FT6I{@mfRn;=PZi9uq&RZ24rE zvE$(%!#tyc4(1sSb~Qh1mssqNE6Z8!bY>Bobla@9`&Iqoc4xSb>UcD~m9-Jvb;q&p zpOSl9@7E~@^ima4dbcu2EzIL>lHsWndLp29uS{@4n9P)0uSz&4eXBXU!h2;y%Qhk7 zTeGqZ_P#3ekiOcH&iay*rQ=AyrEXce#MAcw`R~0L7Oy(@)jry2axH%(=S>C9>ZTm7 zO`Y5UN0MT-qSkJU`g(Qi)mzv1)|Oqoer;*g#;okGon5Z3fi3|zSyJPjm|EVX#H-y_ znYrh^wXwO2hf^3w1;5X`xz^9;6~C)L|MUOn_qU54I&d&OdatD5Z@a}|jljZykgJl5 z)IN2!@;!Oi{$FM0&(Na(wcowI-kYBK?9S?6jt&aXm__p%_?GTj;&F6Il2&A|RI$6? zT}Ow61ue=<8`YV_K3Q5x$LHU3%zrJUEUmb9WuMEAnGQutZ-eICGc#5AD!QEMMnTM% zz=a$=EIT;e_@B1AZpxSEF#NRliTs&q%vj}2eF%N@apJ_sO1XZojK*&-u3TgEv1`@%h~JEp$f z5WAfB(3YN=2ag?QKfTd%;`2m-((eIvk+LStSZ;>?TcBP1j~jM z77H7@dH?S!dwJ>V>F_lRifXSs`&xFgQT%$joJZp%BldgOc#9s&)X&_n;urS!bwi=P zWZsnYe?hEy$r55g8=V4l&t7HPqk8?mXuIo7<|U0@Qk_BN8;rku`tzMIlz43!`Y1;* zDZlX!BgX<(mX-ELnkyYvyx4aAno^GkXNk1viO0QjHEx+Hoa?yR6Kk71Z9#_f(&ZKx zzG*6-v!13~Sa?$4up`@j*`^ZnBUiR>z4~O1U+(KE$E;%xToAb-~UD}%Dn(<#g?)$c^&kx;$RTLj9iTZr$-Jc(lFLUIv+&%6`j(jcGdyJQN z#_X^-Yw>PUzSp^PXFomgmEXWsqBJdP-W-2d#)mak3)a8fJSU0Y-y&i8<39&CX*b*d zE69EIZ_%y59U}W)JnH!n`g;5Qu3{}|(P`0NeDgiMK2K$zZdjt_ z&T!z?s;t6<=N#J$Ty-lc)C6}9}**9=aEDxP>*wtm1@)qVB>G883v9(GwsUv_I;O=p9#P9He6-qs&D7r4Q%+j7AF#;F71!%B z?(^cEx`Z?EkvNxZ^1HaozwaYA)aYEZn4@%MFOU59YdzcN{dRrsaPDGxN2^20xgSAO zhJU1@B!>@}w!cN|aZdld&SeWWAg zW_EIiX5j<7E{A}a{h9SUFTXYZ=%CEGja&7Ia>%TNbvKzZn{L@=^?v`fqC`p|t)1^( zN@`f`Ue#hxv9e(7^T#+KJGa(f!)$_f=RL=#M-*oZn-SGb5yV)(P7C6261G`DE|3 zZcr9a2rqagVO_a}aj%Pn%hO+8j%#9j_I+ShbU*HCbcE}8%7K`~%t8mxetyd-4Fd5q z4f)?6=keRIhb7tk)UNvVlTyFyGOcW%tZgQ-_Cva5)q+*Z9~sK+E*a$ATiP~jl9P?G zxue_8%5Rr%1y7cJZ1r@}Zin3!cCw{RMqloyxxZ_QVcl;nd4JW)ockF(^@rYWj}zXX ze0jOyZ+=mqHJQ_vNWZV{5uPp)7=5s7Z=ajnjSUip5fdNZd6Foq?=3CFYAIN>Nc2R{ zxp;>pwzEPfStY7d`?69O6g0B>z5KXs<)idZtnCf5+4a)L{(k*hT4a$u{}OYAZOz9A zbA_3&3zz=eu6!t+x4veD)$Q_T~=T`p+ROMPGvMZ@u3!Ma#kA$~k$i_dZw3 zJ5M%LI8@3y*j(9fVs^#Lc1Mp}!0faa2^kMk*$h8^+2Gk=&ZKwf>fO$}1;w?>7y7=v zx)8zNEnGG4ZhW3=-ITx&Yrek^%D;U0Syk-WyqqZwHQiVEns1gaIH~&S_21+#yYBzH z=b&v6wR5AvuFap~86D^B&e>RXUSO5hp#-Nb4|(6`Ogg<@QunbMoA;L!8m~_zd?~jih36?1Q-M;>({r~It?>D?QKkS&*bnWKl`ueHMyMzx;EWMh=aB#v(;{`vw zxeR`?DCwBX{G9blTjBrOW9L>kTZrvtZP_tFbz;)-OMy98KUT$`cdtGlrkoSHXz66D z*UpQ@URZDX)EA(@qQWrY;Lf$8b;==H+KZ(fnjcDBJ$Lzy5%bf&NxRQY=01F6b<^P^ z5l>#9wGDLcJkSurm-^1FW;v&U%?7QM!wkWZ9|^YbbUDtuB*VbHh=JSbc=h2`Ya_i9KW65){kdm5M|kmz ze?3W((+;3RMjx&*6nzaYyZDK zoxWAxl)rM{6l2lT-mj|z`m7iwMK*lf6DYvm`evu~L(5K9^^Z?3UR)Xd{aJdCx3^!O zj^&hDkyyer566#yh!VewqVemixJ9C zS*v544;^|SRrs9a)lY}t!NOgi3i@3-zxlA~H6FgfpeWMCowZP7y?aEVa1N`D?cN2e zH+J4U-W4M=FX;orLxzhB^f}9)_gT%DbFFd5m5Zy^h!$>@nkqDdb8<|@;nt0seUgS+ zTr5i-X#1ay_gwMq?N1@aTkR`rUYe{gRd^=My>_S2p_Nfg-Q_3#5EkCj`61K5Nj>--=lQyJi4C2GKN=F6Exuom zU(&5OcjDi@z`P z-Kp{UW_rA)#KZg(b1c3tx|_f5=a!DwbH5j!eY{wO`%j&osQFIY>1{Rd`b8&CY?B)BcU6FNr{{&b5xSbi4|Ng+*o$g0BsOepfWIl99{bc{V8w$Kn+n-M64?FcI z`~CFxcmCquOI)3j79Ut!(R9M$9^b~P-JcI?Kl!hBBt#}z`p|XpJ$wCcX*qsaQ?c?v z4v)(rTNSoBDk=fV$9&s_WjHP}OntRVYl2OIq{4!;0%vz}ykFV7`=FlLa@SL9OE1~P z965YC*}>+z|J%23VmAK#p&XpLB1H7*$$4MpPww^)<1~FL+x5ug!`#Uiwr;lgm|W^L z<>k};`tkp+v`)Xr;;tQfVqr$rSz*)m4>Qj-^1M5J@r(EzkGIdepT`S)yt(|gQ9KPXS&X%OJ(%KFyGxAS$#|AIWWUgN{7PBq2au3arLZ_&1y z`&4`8E_o-hVdBqRvM>U?+FQsvTyX5e9G~i!!kQR&0YE# z-NIkb$7FU-$oyyga4}<`z!|sQ!6C&#f*X0iS!BO0J9&F+xYByh_Ed&|T`Rv|-pR|- z;4=NYRf5&;#j~#|${aJ(aJfI_==MK;b$cu#vlf^$=^pWJnkpJEcyGq`qyC4Q+C8IN ze4CyvJu&}%wUb?Ky?%|usf96fHTPFp#pc{ge8BW0>UY?b6YSfoDk6_Eyix&KXt1}MT}jY_31iJ<~KcV1_2KX3OpJQ9;}sa`K>3kde^R~lPyc9OFldL z#wtW7Z!gg*Xw&fb+F?Ag~M z-LL5FQ^L1Iels_V(@UKTEn?iub8l`+-KMC+F)d&flN2jh@97 zrVG5%^lSNADJQD8EU{2*^7i@#nidS6WskBRzU*C7_G7BoyDla3^zM~Uuk}CL)~Nqx zy>*IwikznAJ>jK{*2R2xQZ27mr`QH4Kd+cszgwm+u2JyB&WF59`sT*Rce$0sJUkhD z;vS=(nwFWPsko7P+@V*GV=MJ1@Hg*H<#sX5wG(JP`1p*EEVszLxVn8ofk6s&jSG$H zUupbI+U%I7yJaQEktb@ug0{@v$s88B;c-yy?>>hMYXv>{Uvo@3aZ95x%;Lzrc+I|| zmYM8~DO-|-gx6kBz8x2?s`&8m2Fb#gSzGQ`6))L;T2SM8@4Ic8_ujmeRH&@H859_) zH1WopA4=JZuS>U8sN-~4 z_wwzk6*=-)-{Sp_8#`FqeO_8@P&8h#&_>`5$CXz{x^vAhGA#V?JzrzTC;tB-AD5}k zHK|)WZxC)<*VQLHFXNmuLxC~b`B$sVO=VY2m$>iuKg^T4`_*pBoixdL zR^fF&Q>?!D96a^PD=;!ZU$v_7M%R@oEFE4y`tO~b_*&g(dbfCe^dF`P=BgHf|GvHB z%`y&%K6c{hiHY}G&prO)Uz5eF6un4P*h0p*XoB@C1uIL2=O_QF2EEg0o+0eQx7~2< z8g{3uB};nWZ(eS7)c0w}?TPcJw?1doJka*z$)2w~+`@J`VT+Opae%+;iPb?FSnZwTg-+skDAoIgT3!djj6=nOs|2oC(^*Z&@ zoJ?i++AW%s`J6uf*q~>BC(o(0qsJ>ViO0<&+V}|nVz;ZE)AdshI@mnXixTMf)HS;D z?sU{cCq5^u8(oW-XH>jcGy70ogx|aN>$eRqwQv@z{N`5h^|?Oz^@nPgt12^=Jo9L% zU%}Gu-npLtbHeU9N2{7NV)lOjrZ2ep_dyGe7ePNHdNM(H0l z^tHWREJrw&RG&A^KiR08f1=}xr;(mnnA&{lfc>{XDO4 zetO%Rvj!O(PWSD-`Dyli8>M?E!)6*OMegEE#;EA!Kw-=3`}8Q0iu z+_uVF`|0ED{3mby-;u#4*A`q~?&b7`ot^LbI(cieKa*9x^YRyT+|gxSnDThdl=T<2 zUrE+%l<3^LZ<6|hX-mDEwfffVsCp`{q_O5VGM(gxgT?etuQ+9Js7Oe8x zbTH8*ak)ojM!WTH3)Xoyx{eRlK9QMZq&NAGv-N@{x?!)^cB@^@|93+$e0pmA(p)bS zx1O1*ag2^Zp>xz*Bj?XIa(ndQ!K@O^Q)h+rJ9K6_{;V zR@86!5qk62y&_L3<3PL0_okIuUtbqp&Alu*@oU9)j`^9j`SCyecYa?t`^ohGe>aFc zm!0%uisz?CkN@l5ulym&+M>N)Wao=ltwLh+xgsr`rina#eD{m*muspahV^H{B5L1m z^}aa$_fpXqe$LZ7I5!-7wUncqWdWa9#_#X-e9avGlD0K_uJms8NVXMud{cLmb+z!B zmS%rO+uw;dO3zJfSfFrJSF5T(D#y9+$hBvRp^Y2*oG&mIn8cgU#D>KYxC6G z>-zUkQJ25F)ONKB(g%;4!?HGP4>n-d2M z&9wECmsPMz_jT7r^* znxwkMH15>q&r5Fg8K`GDvz%7$4rNL_CLtlS>JP8Qm3=O)rHbMOGB;d}yW=I5E`)_m zJIK#uogpC5*BEW^u5Ie(c?X@9+KzW?9u}8e&*pb0@iX_7_H}=j|9NL_Kk??~OBda{ z&g{-v>C*Rk>P_xs#b-_7mK>6TMd^<;@+*?MF&nxem+|5t_jwf}A&a}y0lcFQR>Efxql~IRb8v7r6u6uX5<9RIhC{926V&hA% zo@P2Y3(3op&)^3u9zX>#7j*r~@ncYR0)K@sA6yILAqvymA9?qVgz}?krW2-`p@07YeOtcaDNaCbxBG9invRDsry1 zm-`tL(lPn;iSxE>#usC1+B*%(&WW5|5o;_qc_r(nqRrEPRQ&Ar_iB+it|YqZcH{cw z>c5ue&fx)(1wMVboKw~3{aF1Z=&o0v!|Y^%^z6!*_O?@&T>RoKBVEW%yeb+byIrUnOv?c1Je&U4gUQ-|_V;{DKFs^?;o;OT5|dkwx@fc% zrW{u{_^y-heuFt*_Nk(>y8eMh?Pu-9B1%2yT~XMTQ=H7YPO(hHM`NwsVHKt%0p;@A z1yW~k9F)7gTkz||>^+C2pT-@%T`zIQOTWRM?GM|A!^`8D$}e0wT{11VpDXL%It50S zlXsPlFD#oIapTQ;R<7vhAK5m_ub=eGDPp?GmKA?w{T2&*u9|R#eYc%Jk_lR-l0#QY~p11&~!_CnX z-txb6V)#E#(P__VIq|-W;>&i{*F1=gG^+2sQ~zDpD<|#ir)m85G5Yc~JHoc*L@95S z>;%*86V3eB{2)BQ-PQ6XPB*9jVL@s296 zQ&`HpOLD@ub6LtWGHT?67+x9NGF`pfY@g(}_38f&p7ZPHskIw!J*;q&O-NYqgWCm; z0*MU~GwZhANGQB=z(%NJ+9_A}ut^WOUrm(%ePbW5*Yb-C=gQ6BCGcY9Rmq5crB@ps zatfIxt$Cyq(aka2I=QdWS}^Krf#9}Xq8ryFA89Ia_)?s5T*_2$Er0Vx{v#Ya`4gsf z?L54S>)^*5GdbF>R|icwBpG*3O#4tuLfo{cV%@djD`#}i4vF}d#{b4b(5_|jy=R?o zDlcsOaUlCTlXZoV3cJkfQ(9gEMSGoW`F(w5bqxgDX67uZ-@@YJ?%_Nsa?hb$fzHMW zO|un?68FE!Hz;3{b><&q(eXpmMK$)HuD0%5 zWfgp2^Suw7vlqTzlyNI`$L4K64+TqZoMcz`QuF6?`fC6T@TD*cO|$}g|^zLfWNUAw5#T|{EX zjh~A+E5we6eqL_(ozOuXG|L+|c8=jbL zJuoAErVK;*BiY4UKDAzy*}O-=-K+0y%OU~Yxok}$x`mr_+&T*9Hzam6We3(4u=}PQ z{U*tI?N#;PEx1dT#CBgvc!&~nRjyZGnwZa5^T&jnnmJt z?)Ww)fBf>zDZw%RV<*RiYYKbhE^Tvt*pNgbarPdy~^NM;6Vtl+)CtD<}yHT(wOvZG=3X>*@2@Q!{JJOi@<{Y1M zK7#$k4iDd{`%lB@u>NM)795Z9|`wxMs)|UaGZ2V zT6oB*p!|OGA=&+V7x{epR=+AM*uC-R`O*i!5*inHww}lpc-(TVPwf8&k3C8P4F3+C zs)(0zOmOX*yyL}hWp4Fq??YEu@{XM5KOqrwDnYRS+<_w>KTDpSC#I^DeE;iJj`%o6 z>Hn+PGB+Q#XSkSlQaP$7H>ok>ln@isqtgEGG|)-pB)rf6FJe)#2O)o+3Omw9xZ=33C4F=fg{NssfVV>FlNpL=8C`S63(#-9_iRq9^G zKKxu{I78sVBpH+I;+NU3hc_-b5WVgw-{ePZd$N!G+2o++c_`5(C24(tMxXDEvqrd8BDFZb|K-~A?Lz5H5sD!=G5xO?z&R)^vj8x*Z+Pv|NZf6)jiMj zScDi4{5lq(QugE6cEO4dP2LSBCW~xb(dyTzB=7fXU)wB$=0~>|O6c0{k>#AeLA$hh z&Xz`&qo>Pfu(TX^UM9E7C0_QZ$GgIBvlBAjX1ME>oSM|B=pk}GEU<-T?rIzEG?(K+ znrRE!1B<60nfzojU!{VCMEdIHBf`5cD6Y^^b>!>lXq{QS?oRpNEYoj1e`?e2b{o~b zS~}sBgz81>5H+RlyRG8v^UCZpTUvMxzrITlbNiNiBTV-wJHPXi8#>C5+`H--$^vaQ zI>dEce^7UB*hLP`=PGmMR&Ecr(~i*a`M-K<*w)4-&mRJ}Tr!fa#rx-M z7g=28DB|xlX?OKvo3G7pqc$9$%z5&l{huGNcFX;kfBo90$h$wpen&siw)2&o#KN*j zqidN(Mc2|P8Cmb9ESa)3degF{YHmw!b#tpdy)|{Kh>d>8L!T0*O)ZIUjxx4gn8D)N zWA0Qb!}yV-NH4*B$)C7gM=a?mbD#L!;p_eDWfo%YsjH{d7X-{obamHWBgmK zSUmOdb)A{pe_b`V|MTWnw)+0>r?jKJx5(c)-dB2c>1E%=li$hjSzi%w)R1B3mhNSC zvA=8nRkeA-^TBMf5x$)jYnr+!Iw{5JkzpNdXpCHkvj5JDy#Lx;V#~gi>sU; zOcN9kVqW52)i3h$fcCYp?kPtP>^S&dWRKw$xlnn3e#Hf6U zP;l#b?w!&cuDbR|4xNjCIAn?S`)`+2o-8DocU$dba`O~lTUTYTkGwbKwYRn(IIuIW zgoB5TC1%;#I|dFW>_WHnLekpM3Q}I*c(b)^HpPiz={i@bW{<<&=@#D2C zJq@$Jr9WNk@6CO8Lw>ww?zOA=={*e%RTpiZYI{y8IQwDo>*aTT{jSg7%fA1h*s8q~ zgf>k$Y;b0dn$h!Lo{{MQU8O0_UvIzUdc*RNzscX|k6~5fx$0eIU+=fev8H3Wb){;p=8ah25UD@5!Z=6JML< z%;P#d$M8kSZLWzkEFZUb8zwq6DcTr$xbV948M>ykPQ&MdU=IfmdFniq9?PirCvd z^HwaLKet|B!Bc5Y&Qi50PoAVcJ3Vji)k!;3_ov8*`nk+e-I|%1d9nKYyS48PQh%}Y z6bWZM)P5;3bD8Op$o)^YpRaZCURkLA)ghLBgLk8zcvjFmcqY|vnp_vd-_eYsdy}v_oH|F{hHWA!KdTTUA-~s zpUtXAW(wJ*=XPr7O7i;t|EGKU(!XtgmS%Fiaph~?)+MztsY4(*WYKmOclF4i%K{?$ za?^r31YNW>wUjJ7EpK{yd|Bk|(0<^6gF__euWg;@&v;1gJMMgnYvS*_iPPUWS08wI z_@8)DBro{N_tK|op&g{MB^0q!WmiNp_fp-ip z&CS_cc5f>E7`Ms4>Z0NT=Y$7}44REwemGvURrP3l-rDulLgshY+f!%eBwklN5ZqO> z`lvLUR);~;1*5#@%Npgo?me<(WBm2rq~yS5-w)hRmA}kv(21?u=zs0tA@k&S^q){Nys>7h(?Q0c4wiphe4eo#S@q?>F}}XJD^^^4Ia7Mpy~_b@3?E9H=T;Sm ze=Mv}n9^YJ+r{Bd*T?;f8Y6!M>`vo3I6t~>a)yLy?!HNjZ!KK2_`9idsi~sigs>7m z=jN8WD~bi*6c!w6DR{X{_uonf^}>rKPpv*$RKHKt_;%*!#SHt0;St#zHTaI$)>hd+ z;oyIIqdo2DhrjLm6Ykf3i0yRxu&?Wo%!$)QqJoJW+;=yw`@Z4o`wNN%Y6=dIMcVoU z<>$P1ZFiV1&=9xF?WtYK)ryC$;vNnm35WFFzkJ1Bpm45!{m&n6f7VI;yq$0Fe#(DY zXz9kiPmI3E`h;J&y#BSo1;qu<5AHNEe5}06$<}hpaOt`i#_UGf*VbI?)|2wh|LD{7 z=-%sljy-d$M9W+6e+4C`20lhvrswP|9uA-QR;4RG)VMlt)p-^c-YsWmem=X`V%ZMq z?B7qEwIWNxZt(K|UEu5h3i8iPOl&NDcWbx&4eWcsV{-I@x!|;GJT8$TyFII_esx=H zk^NC(?-YOhi$=ukYi)H`6ruVS@-nI?E!^pS`-I2kx(9VfHNTwX^3LXTUtt@ z?(xFIrE90M9`_OR*PimC#N>K6$aPjMB^+~33nbLZ|9X?DYPqO%b=kfA^|qf3EiEnE zSFc*N&Uvw8ba<+CnvQ44L;kj`mY}x}zb=O9Ea7K#V_&&`<&^&WXOmqY+sD;EewTXs zokP&GGVOwS-_~AG;$~Tr`TCZ)z5{n#))W7#*FVpzmK^cu*d%a%a;7FyAgQp-&|CbW zLnu*jrn&F@x~qmVQ-3YGFVn#`@p)X;YQ;0>b5>**owF(1ajxLWY28P%5A+Ww2^t-I z_4d@og}XxPr<{BrK4)6N%$vU#%)02?V7lOyyDSsfZ>}5{Ob<7Gxf`FOH|0(3Qr(A* za@Mb;qURr;z~;h!>0ic+j!SFi>(5eH7Ha)u;$7>dso&4NRS3Avt1OuGu`63iMXAu{ z)91F84xLwC8P}UHuHW|XFpGJZeEn1Zp!nEMs{*Gtt;NEYvUf8#{w;l>4zluvzzcg8 z2ZtKxLxP>-@=$|!a1!@X@@6rk!5=#Q(jGL{Z_Uu{uC8Dq5XGQl( z=7aU2K5XH(f7ipJYQSt5DjlD6MBFcUJuRu`;K{(%OFV{QG-@ zm!0`AN&4O2HWQ^w>$`7!S)jhR3ua*nCnLYUeB}R6t3UnX3ZJ^v`#JmmZ?}G%9@yxm zX%xZY;;^vihrx5p&#e~uyD|$M3a>SOcys1V;)J-!1)nx3Ijq0gqOf;@ooT3f$N9PU z_vt>~xjim+LVS=h<6+g-;27_ySu>iHnvPl#k5Xe!eGL{q!5w zjh8N6xuB}>MZSdN1vq~$crK7|ujJKI`G{?8=9cQq+xBlL{r~a(qdQAa96Px{P~h8* zjSlWkNeBGHYpoowPU(1kHT}KszGvHxPh0%1RrdCaH-C=z`%Oz!R$8)kXXJAmE9MDx zzU|;#T-(qfIP;iYmAd=Af7ku;p8ZTLI@ez8uf-dING%jsi< z$1br3D_^hMQt6i3taAU?+|{)_e#?*d{hcXzd)wQ{uKRPv&vBZuwd`E8hbQPgmps4V zbiG4L642bxxX;l+(z3GZ)UxkolWrThwEq0Jbb-s#rwYsk&nzeGvtipI_F2E~Be%M1 z!mYpF&zJpPAMu+tSR(7Xf}rrvtM`|CUs=JkQs+#o#_GVA?pvzAz%%O?9;PiBTYs{j zknR3=x9)JyMb3@V-%`Xgck#b%Uw6dw)_dodD|)x;8hQk&pN}iKetP}-U0lpO$J!Se zrYG%@n5@3^WG9?8#BSlg$zo!hR)yDCx0W{O;tJhUJ=C&KdL`$Jg)o&R-zl7FV; z%9+0mnHJyNzv;1UrHoXN-OpbW{`Iw+Wd548bh`iU#G|i#&nF&;s*ADPV5!IUEzN$X zhmw0=TfS$f5xbe8Kc`C!xB_%&Kak+iuyXfHw+mj&Gj<6dxO$)?#mCL~sR~<5#iH}l zb-z!2Jt*P%{ { - source_file: $ => repeat($._definition), +{ + // ... - _definition: $ => choice( - $.function_definition - // TODO: other kinds of definitions - ), + rules: $ => { + source_file: $ => repeat($._definition), - function_definition: $ => seq( - 'func', - $.identifier, - $.parameter_list, - $._type, - $.block - ), + _definition: $ => choice( + $.function_definition + // TODO: other kinds of definitions + ), - parameter_list: $ => seq( - '(', - // TODO: parameters - ')' - ), + function_definition: $ => seq( + 'func', + $.identifier, + $.parameter_list, + $._type, + $.block + ), - _type: $ => choice( - 'bool' - // TODO: other kinds of types - ), + parameter_list: $ => seq( + '(', + // TODO: parameters + ')' + ), - block: $ => seq( - '{', - repeat($._statement), - '}' - ), + _type: $ => choice( + 'bool' + // TODO: other kinds of types + ), - _statement: $ => choice( - $.return_statement - // TODO: other kinds of statements - ), + block: $ => seq( + '{', + repeat($._statement), + '}' + ), - return_statement: $ => seq( - 'return', - $._expression, - ';' - ), + _statement: $ => choice( + $.return_statement + // TODO: other kinds of statements + ), - _expression: $ => choice( - $.identifier, - $.number - // TODO: other kinds of expressions - ), + return_statement: $ => seq( + 'return', + $._expression, + ';' + ), - identifier: $ => /[a-z]+/, + _expression: $ => choice( + $.identifier, + $.number + // TODO: other kinds of expressions + ), - number: $ => /\d+/ + identifier: $ => /[a-z]+/, + + number: $ => /\d+/ + } } ``` @@ -118,27 +123,31 @@ Some of the details of this grammar will be explained in more depth later on, bu With this structure in place, you can now freely decide what part of the grammar to flesh out next. For example, you might decide to start with *types*. One-by-one, you could define the rules for writing basic types and composing them into more complex types: ```js -_type: $ => choice( - $.primitive_type, - $.array_type, - $.pointer_type -), +{ + // ... -primitive_type: $ => choice( - 'bool', - 'int' -), + _type: $ => choice( + $.primitive_type, + $.array_type, + $.pointer_type + ), -array_type: $ => seq( - '[', - ']', - $._type -), + primitive_type: $ => choice( + 'bool', + 'int' + ), -pointer_type: $ => seq( - '*', - $._type -), + array_type: $ => seq( + '[', + ']', + $._type + ), + + pointer_type: $ => seq( + '*', + $._type + ) +} ``` After developing the *type* sublanguage a bit further, you might decide to switch to working on *statements* or *expressions* instead. It's often useful to check your progress by trying to parse some real code using `tree-sitter parse`. @@ -250,24 +259,28 @@ The language spec encodes the 20 precedence levels of JavaScript expressions usi To produce a readable syntax tree, we'd like to model JavaScript expressions using a much flatter structure like this: ```js -_expression: $ => choice( - $.identifier, - $.unary_expression, - $.binary_expression, +{ // ... -), -unary_expression: $ => choice( - seq('-', $._expression), - seq('!', $._expression), - // ... -), + _expression: $ => choice( + $.identifier, + $.unary_expression, + $.binary_expression, + // ... + ), -binary_expression: $ => choice( - seq($._expression, '*', $._expression), - seq($._expression, '+', $._expression), - // ... -), + unary_expression: $ => choice( + seq('-', $._expression), + seq('!', $._expression), + // ... + ), + + binary_expression: $ => choice( + seq($._expression, '*', $._expression), + seq($._expression, '+', $._expression), + // ... + ), +} ``` Of course, this flat structure is highly ambiguous. If we try to generate a parser, Tree-sitter gives us an error message: @@ -293,11 +306,15 @@ Possible resolutions: For an expression like `-a * b`, it's not clear whether the `-` operator applies to the `a * b` or just to the `a`. This is where the `prec` function described above comes into play. By wrapping a rule with `prec`, we can indicate that certain sequence of symbols should *bind to each other more tightly* than others. For example, the `'-', $._expression` sequence in `unary_expression` should bind more tightly than the `$._expression, '+', $._expression` sequence in `binary_expression`: ```js -unary_expression: $ => prec(2, choice( - seq('-', $._expression), - seq('!', $._expression), +{ // ... -)) + + unary_expression: $ => prec(2, choice( + seq('-', $._expression), + seq('!', $._expression), + // ... + )) +} ``` ### Using associativity @@ -323,11 +340,15 @@ Possible resolutions: For an expression like `a * b * c`, it's not clear whether we mean `a * (b * c)` or `(a * b) * c`. This is where `prec.left` and `prec.right` come into use. We want to select the second interpretation, so we use `prec.left`. ```js -binary_expression: $ => choice( - prec.left(2, seq($._expression, '*', $._expression)), - prec.left(1, seq($._expression, '+', $._expression)), +{ // ... -), + + binary_expression: $ => choice( + prec.left(2, seq($._expression, '*', $._expression)), + prec.left(1, seq($._expression, '+', $._expression)), + // ... + ), +} ``` ### Hiding rules diff --git a/docs/section-4-using-parsers.md b/docs/section-4-using-parsers.md new file mode 100644 index 00000000..903b4527 --- /dev/null +++ b/docs/section-4-using-parsers.md @@ -0,0 +1,8 @@ +--- +title: Using Parsers +permalink: using-parsers +--- + +# Using Parsers + +WIP From d1665da21c2d8350622b34935aab1b4a442a971a Mon Sep 17 00:00:00 2001 From: Max Brunsfeld Date: Mon, 11 Jun 2018 19:17:10 -0700 Subject: [PATCH 84/90] Add docs --- docs/assets/css/style.scss | 5 +- docs/index.md | 17 +++++-- docs/section-2-architecture.md | 8 +++- docs/section-3-creating-parsers.md | 2 + docs/section-4-using-parsers.md | 75 +++++++++++++++++++++++++++++- 5 files changed, 100 insertions(+), 7 deletions(-) diff --git a/docs/assets/css/style.scss b/docs/assets/css/style.scss index 2fe29224..75ec2f4f 100644 --- a/docs/assets/css/style.scss +++ b/docs/assets/css/style.scss @@ -8,7 +8,8 @@ } #table-of-contents { - border-right: 1px solid #ddd; + border-right: 1px solid #ccc; + border-bottom: 1px solid #ccc; } .nav-link.active { @@ -21,7 +22,7 @@ display: block; } -.toc-section, .logo { +.toc-section:not(:last-child), .logo { border-bottom: 1px solid #ccc; } diff --git a/docs/index.md b/docs/index.md index d11d34dd..9e8a720c 100644 --- a/docs/index.md +++ b/docs/index.md @@ -4,11 +4,11 @@ title: Introduction # Introduction -Tree-sitter is a library for parsing source code. It aims to be: +Tree-sitter is an incremental parsing library. It can be used to build a concrete syntax tree for a source file and to efficiently update the syntax tree as the source file is edited. Tree-sitter aims to be: -* **Fast** and incremental so that it can be used in a text editor -* **Robust** enough to provide useful results even in the presence of syntax errors * **General** enough to parse any programming language +* **Fast** enough to parse on every keystroke in a text editor +* **Robust** enough to provide useful results even in the presence of syntax errors, * **Dependency-free** (and written in pure C) so that it can be embedded in any application ### Language Bindings @@ -48,3 +48,14 @@ There are parsers in development for these languages: * [FOSDEM 2018](https://www.youtube.com/watch?v=0CGzC_iss-8) * [GitHub Universe 2017](https://www.youtube.com/watch?v=a1rC79DHpmY) + +### Underlying Research + +The design of Tree-sitter was greatly influenced by the following research papers: + +- [Practical Algorithms for Incremental Software Development Environments](https://www2.eecs.berkeley.edu/Pubs/TechRpts/1997/CSD-97-946.pdf) +- [Context Aware Scanning for Parsing Extensible Languages](http://www.umsec.umn.edu/publications/Context-Aware-Scanning-Parsing-Extensible) +- [Efficient and Flexible Incremental Parsing](http://ftp.cs.berkeley.edu/sggs/toplas-parsing.ps) +- [Incremental Analysis of Real Programming Languages](https://pdfs.semanticscholar.org/ca69/018c29cc415820ed207d7e1d391e2da1656f.pdf) +- [Error Detection and Recovery in LR Parsers](http://what-when-how.com/compiler-writing/bottom-up-parsing-compiler-writing-part-13) +- [Error Recovery for LR Parsers](http://www.dtic.mil/dtic/tr/fulltext/u2/a043470.pdf) diff --git a/docs/section-2-architecture.md b/docs/section-2-architecture.md index ad007cce..a1101d44 100644 --- a/docs/section-2-architecture.md +++ b/docs/section-2-architecture.md @@ -9,10 +9,16 @@ Tree-sitter consists of two separate libraries, both of which expose C APIs. The first library, `libcompiler`, is used to generate a parser for a language by supplying a [context-free grammar](https://en.wikipedia.org/wiki/Context-free_grammar) describing the -language. `libcompiler` is a build tool; once the parser has been generated, it is no longer needed. Its public interface is specified in the header file [`compiler.h`](https://github.com/tree-sitter/tree-sitter/blob/master/include/tree_sitter/compiler.h). +language. `libcompiler` is a build tool; it is no longer needed once a parser has been generated. Its public interface is specified in the header file [`compiler.h`](https://github.com/tree-sitter/tree-sitter/blob/master/include/tree_sitter/compiler.h). The second library, `libruntime`, is used in combination with the parsers generated by `libcompiler`, to produce syntax trees from source code and keep the syntax trees up-to-date as the source code changes. `libruntime` is designed to be embedded in applications. Its interface is specified in the header file [`runtime.h`](https://github.com/tree-sitter/tree-sitter/blob/master/include/tree_sitter/runtime.h). ## The Compiler + +WIP + +## The Runtime + +WIP diff --git a/docs/section-3-creating-parsers.md b/docs/section-3-creating-parsers.md index 1de02833..5d09670e 100644 --- a/docs/section-3-creating-parsers.md +++ b/docs/section-3-creating-parsers.md @@ -357,6 +357,8 @@ You may have noticed in the above examples that some of the grammar rule name li ## Dealing with LR conflicts +TODO + [cst]: https://en.wikipedia.org/wiki/Parse_tree [non-terminal]: https://en.wikipedia.org/wiki/Terminal_and_nonterminal_symbols [language-spec]: https://en.wikipedia.org/wiki/Programming_language_specification diff --git a/docs/section-4-using-parsers.md b/docs/section-4-using-parsers.md index 903b4527..054e0e2e 100644 --- a/docs/section-4-using-parsers.md +++ b/docs/section-4-using-parsers.md @@ -5,4 +5,77 @@ permalink: using-parsers # Using Parsers -WIP +A Tree-sitter parser consists of a single C source file which exports one function with the naming scheme `tree_sitter_${LANGUAGE_NAME}`. This function returns a pointer to a `TSLanguage` struct, which can be used in conjunction with a `TSParser` to produce a syntax trees. + +## The Raw C API + +Here's an example of a simple C program that uses the Tree-sitter [JSON parser](https://github.com/tree-sitter/tree-sitter-json). + +```c +// Filename - test-json-parser.c + +#include +#include +#include +#include "tree_sitter/runtime.h" + +TSLanguage *tree_sitter_json(); + +int main() { + // Create a parser with the JSON language. + TSParser *parser = ts_parser_new(); + ts_parser_set_language(parser, tree_sitter_json()); + + // Parse some source code. + const char *source_code = "[1, null]"; + TSTree *tree = ts_parser_parse_string(parser, NULL, source_code, strlen(source_code)); + + // Find some syntax tree nodes. + TSNode root_node = ts_tree_root_node(tree); + TSNode array_node = ts_node_named_child(root_node, 0); + TSNode number_node = ts_node_named_child(array_node, 0); + + // Check that the nodes have the expected types. + assert(!strcmp(ts_node_type(root_node), "value")); + assert(!strcmp(ts_node_type(array_node), "array")); + assert(!strcmp(ts_node_type(number_node), "number")); + + // Check that the nodes have the expected child counts. + assert(ts_node_child_count(root_node) == 1); + assert(ts_node_child_count(array_node) == 4); + assert(ts_node_named_child_count(array_node) == 2); + assert(ts_node_child_count(number_node) == 0); + + // Print the syntax tree as an S-expression. + char *string = ts_node_string(root_node); + printf("Syntax tree: %s\n", string); + + // Free all of the heap allocations. + free(string); + ts_tree_delete(tree); + ts_parser_delete(parser); + return 0; +} +``` + +This program uses the Tree-sitter C API, which is declared in the header file `tree_sitter/runtime.h`, so we need to add the `tree_sitter/include` directory to the include path. We also need to link `libruntime.a` into the binary. + +```sh +clang \ + -I tree-sitter/include \ + test-json-parser.c \ + tree-sitter-json/src/parser.c \ + tree-sitter/out/Release/libruntime.a \ + -o test-json-parser + +./test-json-parser +``` + +### Providing the text to parse + +Text input is provided to a tree-sitter parser via a `TSInput` struct, which contains function pointers for seeking to positions in the text, and for reading chunks of text. The text can be encoded in either UTF8 or UTF16. This interface allows you to efficiently parse text that is stored in your own data structure. + +### Querying the syntax tree + +Tree-sitter provides a DOM-style interface for inspecting syntax trees. Functions like `ts_node_child(node, index)` and `ts_node_next_sibling(node)` expose every node in the concrete syntax tree. This is useful for operations like syntax-highlighting, which operate on a token-by-token basis. You can also traverse the tree in a more abstract way by using functions like +`ts_node_named_child(node, index)` and `ts_node_next_named_sibling(node)`. These functions don't expose nodes that were specified in the grammar as anonymous tokens, like `:` and `{`. This is useful when analyzing the meaning of a document. From a7ffbd022fc41e5ebb3509907708a3f4dc65419e Mon Sep 17 00:00:00 2001 From: Max Brunsfeld Date: Tue, 12 Jun 2018 17:37:46 -0700 Subject: [PATCH 85/90] Make docs page readable on mobile --- docs/_layouts/default.html | 151 +++++++++++++++++++------------------ docs/assets/css/style.scss | 115 +++++++++++++++++++++++----- docs/index.md | 5 +- 3 files changed, 176 insertions(+), 95 deletions(-) diff --git a/docs/_layouts/default.html b/docs/_layouts/default.html index 1bee1a8a..3e5c82ae 100644 --- a/docs/_layouts/default.html +++ b/docs/_layouts/default.html @@ -4,99 +4,93 @@ - {{ page.title }} - -

-
-
-
- -
-
+ {% endfor %} + {% endcapture %} + {{ my_toc | strip | markdownify | strip }} + + + {% else %} +
  • + + {{ other_page.title }} + +
    • + {% endif %} + {% endfor %} +
    + + ☰ + +
    + {{ content }} +
    - -
    -
    -
    -
    - -
    -
    - {{ content }} -
    -
    -
    -
    @@ -110,6 +104,15 @@