diff --git a/src/runtime/parser.c b/src/runtime/parser.c index 495b4135..92077144 100644 --- a/src/runtime/parser.c +++ b/src/runtime/parser.c @@ -402,11 +402,15 @@ static const Subtree *ts_parser__lex(TSParser *self, StackVersion version, TSSta last_byte_scanned = self->lexer.current_position.bytes; } + uint32_t bytes_scanned = last_byte_scanned - start_position.bytes + 1; + Subtree *result; if (skipped_error) { Length padding = length_sub(error_start_position, start_position); Length size = length_sub(error_end_position, error_start_position); result = ts_subtree_new_error(&self->tree_pool, size, padding, first_error_character, self->language); + result->parse_state = parse_state; + result->bytes_scanned = bytes_scanned; } else { if (self->lexer.token_end_position.bytes < self->lexer.token_start_position.bytes) { self->lexer.token_start_position = self->lexer.token_end_position; @@ -433,22 +437,31 @@ static const Subtree *ts_parser__lex(TSParser *self, StackVersion version, TSSta } } - result = ts_subtree_new_leaf(&self->tree_pool, symbol, padding, size, !found_external_token, self->language); - result->is_keyword = is_keyword; + result = ts_subtree_new_leaf( + &self->tree_pool, + symbol, + padding, + size, + bytes_scanned, + parse_state, + found_external_token, + is_keyword, + self->language + ); if (found_external_token) { - result->has_external_tokens = true; unsigned length = self->language->external_scanner.serialize( self->external_scanner_payload, self->lexer.debug_buffer ); - ts_external_scanner_state_init(&result->external_scanner_state, self->lexer.debug_buffer, length); + ts_external_scanner_state_init( + &result->external_scanner_state, + self->lexer.debug_buffer, + length + ); } } - result->bytes_scanned = last_byte_scanned - start_position.bytes + 1; - result->parse_state = parse_state; - LOG("lexed_lookahead sym:%s, size:%u", SYM_NAME(result->symbol), result->size.bytes); return result; } @@ -948,7 +961,17 @@ static void ts_parser__halt_parse(TSParser *self) { Subtree *root_error = ts_subtree_new_error_node(&self->tree_pool, &children, self->language); ts_stack_push(self->stack, 0, root_error, false, 0); - Subtree *eof = ts_subtree_new_leaf(&self->tree_pool, ts_builtin_sym_end, length_zero(), length_zero(), true, self->language); + Subtree *eof = ts_subtree_new_leaf( + &self->tree_pool, + ts_builtin_sym_end, + length_zero(), + length_zero(), + 0, + 0, + false, + false, + self->language + ); ts_parser__accept(self, 0, eof); } diff --git a/src/runtime/subtree.c b/src/runtime/subtree.c index e6a213ec..9b8967fc 100644 --- a/src/runtime/subtree.c +++ b/src/runtime/subtree.c @@ -162,7 +162,35 @@ void ts_subtree_pool_free(SubtreePool *self, Subtree *tree) { // Subtree -Subtree *ts_subtree_new_leaf(SubtreePool *pool, TSSymbol symbol, Length padding, Length size, +Subtree *ts_subtree_new_leaf( + SubtreePool *pool, TSSymbol symbol, Length padding, Length size, uint32_t bytes_scanned, + TSStateId parse_state, bool has_external_tokens, bool is_keyword, const TSLanguage *language +) { + TSSymbolMetadata metadata = ts_language_symbol_metadata(language, symbol); + bool is_small = !has_external_tokens; + Subtree *result = ts_subtree_pool_allocate(pool, is_small); + result->ref_count = 1; + result->padding = padding; + result->size = size; + result->bytes_scanned = bytes_scanned; + result->error_cost = 0; + result->child_count = 0; + result->symbol = symbol; + result->parse_state = parse_state; + result->is_small = is_small; + result->visible = metadata.visible; + result->named = metadata.named; + result->extra = symbol == ts_builtin_sym_end; + result->fragile_left = false; + result->fragile_right = false; + result->has_changes = false; + result->has_external_tokens = has_external_tokens; + result->is_missing = false; + result->is_keyword = is_keyword; + return result; +} + +Subtree *ts_subtree__new(SubtreePool *pool, TSSymbol symbol, Length padding, Length size, bool is_small, const TSLanguage *language) { TSSymbolMetadata metadata = ts_language_symbol_metadata(language, symbol); Subtree *result = ts_subtree_pool_allocate(pool, is_small); @@ -189,7 +217,7 @@ Subtree *ts_subtree_new_leaf(SubtreePool *pool, TSSymbol symbol, Length padding, Subtree *ts_subtree_new_error(SubtreePool *pool, Length size, Length padding, int32_t lookahead_char, const TSLanguage *language) { - Subtree *result = ts_subtree_new_leaf(pool, ts_builtin_sym_error, padding, size, false, language); + Subtree *result = ts_subtree__new(pool, ts_builtin_sym_error, padding, size, false, language); result->fragile_left = true; result->fragile_right = true; result->lookahead_char = lookahead_char; @@ -434,7 +462,7 @@ Subtree *ts_subtree_new_error_node(SubtreePool *pool, SubtreeArray *children, Subtree *ts_subtree_new_missing_leaf(SubtreePool *pool, TSSymbol symbol, Length padding, const TSLanguage *language) { - Subtree *result = ts_subtree_new_leaf(pool, symbol, padding, length_zero(), true, language); + Subtree *result = ts_subtree__new(pool, symbol, padding, length_zero(), true, language); result->is_missing = true; result->error_cost = ERROR_COST_PER_MISSING_TREE + ERROR_COST_PER_RECOVERY; return result; @@ -485,12 +513,14 @@ bool ts_subtree_eq(const Subtree *self, const Subtree *other) { if (self->size.bytes != other->size.bytes) return false; if (self->symbol == ts_builtin_sym_error) return self->lookahead_char == other->lookahead_char; if (self->child_count != other->child_count) return false; - if (self->visible_child_count != other->visible_child_count) return false; - if (self->named_child_count != other->named_child_count) return false; + if (self->child_count > 0) { + if (self->visible_child_count != other->visible_child_count) return false; + if (self->named_child_count != other->named_child_count) return false; - for (uint32_t i = 0; i < self->child_count; i++) { - if (!ts_subtree_eq(self->children[i], other->children[i])) { - return false; + for (uint32_t i = 0; i < self->child_count; i++) { + if (!ts_subtree_eq(self->children[i], other->children[i])) { + return false; + } } } return true; diff --git a/src/runtime/subtree.h b/src/runtime/subtree.h index c6887385..52b5c2ff 100644 --- a/src/runtime/subtree.h +++ b/src/runtime/subtree.h @@ -91,7 +91,10 @@ void ts_subtree_pool_delete(SubtreePool *); Subtree *ts_subtree_pool_allocate(SubtreePool *, bool); void ts_subtree_pool_free(SubtreePool *, Subtree *); -Subtree *ts_subtree_new_leaf(SubtreePool *, TSSymbol, Length, Length, bool, const TSLanguage *); +Subtree *ts_subtree_new_leaf( + SubtreePool *, TSSymbol, Length, Length, uint32_t, + TSStateId, bool, bool, const TSLanguage * +); Subtree *ts_subtree_new_node(SubtreePool *, TSSymbol, SubtreeArray *, unsigned, const TSLanguage *); Subtree *ts_subtree_new_copy(SubtreePool *, const Subtree *); Subtree *ts_subtree_new_error_node(SubtreePool *, SubtreeArray *, const TSLanguage *); diff --git a/test/runtime/stack_test.cc b/test/runtime/stack_test.cc index 9cadf347..0330f318 100644 --- a/test/runtime/stack_test.cc +++ b/test/runtime/stack_test.cc @@ -88,7 +88,11 @@ describe("Stack", [&]() { dummy_language.symbol_metadata = symbol_metadata; for (size_t i = 0; i < subtree_count; i++) { - subtrees[i] = ts_subtree_new_leaf(&pool, i + 1, length_zero(), tree_len, false, &dummy_language); + subtrees[i] = ts_subtree_new_leaf( + &pool, i + 1, length_zero(), tree_len, 0, + TS_TREE_STATE_NONE, true, false, &dummy_language + ); + ts_external_scanner_state_init(&((Subtree *)subtrees[i])->external_scanner_state, nullptr, 0); } }); diff --git a/test/runtime/subtree_test.cc b/test/runtime/subtree_test.cc index 54cd88b6..dd238f3e 100644 --- a/test/runtime/subtree_test.cc +++ b/test/runtime/subtree_test.cc @@ -48,39 +48,18 @@ describe("Subtree", []() { ts_subtree_pool_delete(&pool); }); - describe("make_leaf", [&]() { - it("does not mark the tree as fragile", [&]() { - const Subtree *tree = ts_subtree_new_leaf(&pool, symbol1, {2, {0, 1}}, {5, {0, 4}}, false, &language); - AssertThat(tree->fragile_left, IsFalse()); - AssertThat(tree->fragile_right, IsFalse()); + auto new_leaf = [&](TSSymbol symbol, Length padding, Length size, uint32_t bytes_scanned) { + return ts_subtree_new_leaf( + &pool, symbol, padding, size, bytes_scanned, 0, false, false, &language + ); + }; - ts_subtree_release(&pool, tree); - }); - }); - - describe("make_error", [&]() { - it("marks the tree as fragile", [&]() { - const Subtree *error_tree = ts_subtree_new_error( - &pool, - length_zero(), - length_zero(), - 'z', - &language - ); - - AssertThat(error_tree->fragile_left, IsTrue()); - AssertThat(error_tree->fragile_right, IsTrue()); - - ts_subtree_release(&pool, error_tree); - }); - }); - - describe("make_node", [&]() { + describe("new_node", [&]() { const Subtree *tree1, *tree2, *parent1; before_each([&]() { - tree1 = ts_subtree_new_leaf(&pool, symbol1, {2, {0, 1}}, {5, {0, 4}}, false, &language); - tree2 = ts_subtree_new_leaf(&pool, symbol2, {1, {0, 1}}, {3, {0, 3}}, false, &language); + tree1 = new_leaf(symbol1, {2, {0, 1}}, {5, {0, 4}}, 0); + tree2 = new_leaf(symbol2, {1, {0, 1}}, {3, {0, 3}}, 0); ts_subtree_retain(tree1); ts_subtree_retain(tree2); @@ -186,9 +165,9 @@ describe("Subtree", []() { before_each([&]() { tree = ts_subtree_new_node(&pool, symbol1, tree_array({ - ts_subtree_new_leaf(&pool, symbol2, {2, {0, 2}}, {3, {0, 3}}, false, &language), - ts_subtree_new_leaf(&pool, symbol3, {2, {0, 2}}, {3, {0, 3}}, false, &language), - ts_subtree_new_leaf(&pool, symbol4, {2, {0, 2}}, {3, {0, 3}}, false, &language), + new_leaf(symbol2, {2, {0, 2}}, {3, {0, 3}}, 0), + new_leaf(symbol3, {2, {0, 2}}, {3, {0, 3}}, 0), + new_leaf(symbol4, {2, {0, 2}}, {3, {0, 3}}, 0), }), 0, &language); AssertThat(tree->padding, Equals({2, {0, 2}})); @@ -421,7 +400,7 @@ describe("Subtree", []() { const Subtree *leaf; before_each([&]() { - leaf = ts_subtree_new_leaf(&pool, symbol1, {2, {0, 1}}, {5, {0, 4}}, false, &language); + leaf = new_leaf(symbol1, {2, {1, 1}}, {5, {1, 4}}, 0); }); after_each([&]() { @@ -429,7 +408,7 @@ describe("Subtree", []() { }); it("returns true for identical trees", [&]() { - const Subtree *leaf_copy = ts_subtree_new_leaf(&pool, symbol1, {2, {1, 1}}, {5, {1, 4}}, false, &language); + const Subtree *leaf_copy = new_leaf(symbol1, {2, {1, 1}}, {5, {1, 4}}, 0); AssertThat(ts_subtree_eq(leaf, leaf_copy), IsTrue()); const Subtree *parent = ts_subtree_new_node(&pool, symbol2, tree_array({ @@ -454,13 +433,11 @@ describe("Subtree", []() { }); it("returns false for trees with different symbols", [&]() { - const Subtree *different_leaf = ts_subtree_new_leaf( - &pool, + const Subtree *different_leaf = new_leaf( leaf->symbol + 1, leaf->padding, leaf->size, - false, - &language + leaf->bytes_scanned ); AssertThat(ts_subtree_eq(leaf, different_leaf), IsFalse()); @@ -468,8 +445,8 @@ describe("Subtree", []() { }); it("returns false for trees with different options", [&]() { - const Subtree *different_leaf = ts_subtree_new_leaf( - &pool, leaf->symbol, leaf->padding, leaf->size, false, &language + const Subtree *different_leaf = new_leaf( + leaf->symbol, leaf->padding, leaf->size, leaf->bytes_scanned ); ((Subtree *)different_leaf)->visible = !leaf->visible; AssertThat(ts_subtree_eq(leaf, different_leaf), IsFalse()); @@ -477,19 +454,19 @@ describe("Subtree", []() { }); it("returns false for trees with different paddings or sizes", [&]() { - const Subtree *different_leaf = ts_subtree_new_leaf( - &pool, leaf->symbol, {}, leaf->size, false, &language + const Subtree *different_leaf = new_leaf( + leaf->symbol, {}, leaf->size, leaf->bytes_scanned ); AssertThat(ts_subtree_eq(leaf, different_leaf), IsFalse()); ts_subtree_release(&pool, different_leaf); - different_leaf = ts_subtree_new_leaf(&pool, symbol1, leaf->padding, {}, false, &language); + different_leaf = new_leaf(symbol1, leaf->padding, {}, leaf->bytes_scanned); AssertThat(ts_subtree_eq(leaf, different_leaf), IsFalse()); ts_subtree_release(&pool, different_leaf); }); it("returns false for trees with different children", [&]() { - const Subtree *leaf2 = ts_subtree_new_leaf(&pool, symbol2, {1, {0, 1}}, {3, {0, 3}}, false, &language); + const Subtree *leaf2 = new_leaf(symbol2, {1, {0, 1}}, {3, {0, 3}}, 0); const Subtree *parent = ts_subtree_new_node(&pool, symbol2, tree_array({ leaf, @@ -520,7 +497,6 @@ describe("Subtree", []() { auto make_external = [](const Subtree *_tree) { Subtree *tree = (Subtree *)_tree; - tree->has_external_tokens = true; ts_external_scanner_state_init(&tree->external_scanner_state, NULL, 0); return tree; }; @@ -530,15 +506,15 @@ describe("Subtree", []() { tree1 = ts_subtree_new_node(&pool, symbol1, tree_array({ (tree2 = ts_subtree_new_node(&pool, symbol2, tree_array({ - (tree3 = make_external(ts_subtree_new_leaf(&pool, symbol3, padding, size, false, &language))), - (tree4 = ts_subtree_new_leaf(&pool, symbol4, padding, size, false, &language)), - (tree5 = ts_subtree_new_leaf(&pool, symbol5, padding, size, false, &language)), + (tree3 = make_external(ts_subtree_new_leaf(&pool, symbol3, padding, size, 0, 0, true, false, &language))), + (tree4 = new_leaf(symbol4, padding, size, 0)), + (tree5 = new_leaf(symbol5, padding, size, 0)), }), 0, &language)), (tree6 = ts_subtree_new_node(&pool, symbol6, tree_array({ (tree7 = ts_subtree_new_node(&pool, symbol7, tree_array({ - (tree8 = ts_subtree_new_leaf(&pool, symbol8, padding, size, false, &language)), + (tree8 = new_leaf(symbol8, padding, size, 0)), }), 0, &language)), - (tree9 = ts_subtree_new_leaf(&pool, symbol9, padding, size, false, &language)), + (tree9 = new_leaf(symbol9, padding, size, 0)), }), 0, &language)), }), 0, &language);