traxys/tree-sitter
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Take more parameters in ts_subtree_new_leaf

Browse source
This commit is contained in:
Max Brunsfeld 2018-09-15 00:08:47 -07:00
parent c828ca308f
commit e00c3bbdb9
5 changed files with 104 additions and 68 deletions
Download patch file Download diff file Expand all files Collapse all files

39
src/runtime/parser.c
View file

@ -402,11 +402,15 @@ static const Subtree *ts_parser__lex(TSParser *self, StackVersion version, TSSta
last_byte_scanned = self->lexer.current_position.bytes;
}
uint32_t bytes_scanned = last_byte_scanned - start_position.bytes + 1;
Subtree *result;
if (skipped_error) {
Length padding = length_sub(error_start_position, start_position);
Length size = length_sub(error_end_position, error_start_position);
result = ts_subtree_new_error(&self->tree_pool, size, padding, first_error_character, self->language);
result->parse_state = parse_state;
result->bytes_scanned = bytes_scanned;
} else {
if (self->lexer.token_end_position.bytes < self->lexer.token_start_position.bytes) {
self->lexer.token_start_position = self->lexer.token_end_position;
@ -433,22 +437,31 @@ static const Subtree *ts_parser__lex(TSParser *self, StackVersion version, TSSta
}
}
result = ts_subtree_new_leaf(&self->tree_pool, symbol, padding, size, !found_external_token, self->language);
result->is_keyword = is_keyword;
result = ts_subtree_new_leaf(
&self->tree_pool,
symbol,
padding,
size,
bytes_scanned,
parse_state,
found_external_token,
is_keyword,
self->language
);
if (found_external_token) {
result->has_external_tokens = true;
unsigned length = self->language->external_scanner.serialize(
self->external_scanner_payload,
self->lexer.debug_buffer
);
ts_external_scanner_state_init(&result->external_scanner_state, self->lexer.debug_buffer, length);
ts_external_scanner_state_init(
&result->external_scanner_state,
self->lexer.debug_buffer,
length
);
}
}
result->bytes_scanned = last_byte_scanned - start_position.bytes + 1;
result->parse_state = parse_state;
LOG("lexed_lookahead sym:%s, size:%u", SYM_NAME(result->symbol), result->size.bytes);
return result;
}
@ -948,7 +961,17 @@ static void ts_parser__halt_parse(TSParser *self) {
Subtree *root_error = ts_subtree_new_error_node(&self->tree_pool, &children, self->language);
ts_stack_push(self->stack, 0, root_error, false, 0);
Subtree *eof = ts_subtree_new_leaf(&self->tree_pool, ts_builtin_sym_end, length_zero(), length_zero(), true, self->language);
Subtree *eof = ts_subtree_new_leaf(
&self->tree_pool,
ts_builtin_sym_end,
length_zero(),
length_zero(),
0,
0,
false,
false,
self->language
);
ts_parser__accept(self, 0, eof);
}

46
src/runtime/subtree.c
View file

@ -162,7 +162,35 @@ void ts_subtree_pool_free(SubtreePool *self, Subtree *tree) {
// Subtree
Subtree *ts_subtree_new_leaf(SubtreePool *pool, TSSymbol symbol, Length padding, Length size,
Subtree *ts_subtree_new_leaf(
SubtreePool *pool, TSSymbol symbol, Length padding, Length size, uint32_t bytes_scanned,
TSStateId parse_state, bool has_external_tokens, bool is_keyword, const TSLanguage *language
) {
TSSymbolMetadata metadata = ts_language_symbol_metadata(language, symbol);
bool is_small = !has_external_tokens;
Subtree *result = ts_subtree_pool_allocate(pool, is_small);
result->ref_count = 1;
result->padding = padding;
result->size = size;
result->bytes_scanned = bytes_scanned;
result->error_cost = 0;
result->child_count = 0;
result->symbol = symbol;
result->parse_state = parse_state;
result->is_small = is_small;
result->visible = metadata.visible;
result->named = metadata.named;
result->extra = symbol == ts_builtin_sym_end;
result->fragile_left = false;
result->fragile_right = false;
result->has_changes = false;
result->has_external_tokens = has_external_tokens;
result->is_missing = false;
result->is_keyword = is_keyword;
return result;
}
Subtree *ts_subtree__new(SubtreePool *pool, TSSymbol symbol, Length padding, Length size,
bool is_small, const TSLanguage *language) {
TSSymbolMetadata metadata = ts_language_symbol_metadata(language, symbol);
Subtree *result = ts_subtree_pool_allocate(pool, is_small);
@ -189,7 +217,7 @@ Subtree *ts_subtree_new_leaf(SubtreePool *pool, TSSymbol symbol, Length padding,
Subtree *ts_subtree_new_error(SubtreePool *pool, Length size, Length padding,
int32_t lookahead_char, const TSLanguage *language) {
Subtree *result = ts_subtree_new_leaf(pool, ts_builtin_sym_error, padding, size, false, language);
Subtree *result = ts_subtree__new(pool, ts_builtin_sym_error, padding, size, false, language);
result->fragile_left = true;
result->fragile_right = true;
result->lookahead_char = lookahead_char;
@ -434,7 +462,7 @@ Subtree *ts_subtree_new_error_node(SubtreePool *pool, SubtreeArray *children,
Subtree *ts_subtree_new_missing_leaf(SubtreePool *pool, TSSymbol symbol, Length padding,
const TSLanguage *language) {
Subtree *result = ts_subtree_new_leaf(pool, symbol, padding, length_zero(), true, language);
Subtree *result = ts_subtree__new(pool, symbol, padding, length_zero(), true, language);
result->is_missing = true;
result->error_cost = ERROR_COST_PER_MISSING_TREE + ERROR_COST_PER_RECOVERY;
return result;
@ -485,12 +513,14 @@ bool ts_subtree_eq(const Subtree *self, const Subtree *other) {
if (self->size.bytes != other->size.bytes) return false;
if (self->symbol == ts_builtin_sym_error) return self->lookahead_char == other->lookahead_char;
if (self->child_count != other->child_count) return false;
if (self->visible_child_count != other->visible_child_count) return false;
if (self->named_child_count != other->named_child_count) return false;
if (self->child_count > 0) {
if (self->visible_child_count != other->visible_child_count) return false;
if (self->named_child_count != other->named_child_count) return false;
for (uint32_t i = 0; i < self->child_count; i++) {
if (!ts_subtree_eq(self->children[i], other->children[i])) {
return false;
for (uint32_t i = 0; i < self->child_count; i++) {
if (!ts_subtree_eq(self->children[i], other->children[i])) {
return false;
}
}
}
return true;

5
src/runtime/subtree.h
View file

@ -91,7 +91,10 @@ void ts_subtree_pool_delete(SubtreePool *);
Subtree *ts_subtree_pool_allocate(SubtreePool *, bool);
void ts_subtree_pool_free(SubtreePool *, Subtree *);
Subtree *ts_subtree_new_leaf(SubtreePool *, TSSymbol, Length, Length, bool, const TSLanguage *);
Subtree *ts_subtree_new_leaf(
SubtreePool *, TSSymbol, Length, Length, uint32_t,
TSStateId, bool, bool, const TSLanguage *
);
Subtree *ts_subtree_new_node(SubtreePool *, TSSymbol, SubtreeArray *, unsigned, const TSLanguage *);
Subtree *ts_subtree_new_copy(SubtreePool *, const Subtree *);
Subtree *ts_subtree_new_error_node(SubtreePool *, SubtreeArray *, const TSLanguage *);

6
test/runtime/stack_test.cc
View file

@ -88,7 +88,11 @@ describe("Stack", [&]() {
dummy_language.symbol_metadata = symbol_metadata;
for (size_t i = 0; i < subtree_count; i++) {
subtrees[i] = ts_subtree_new_leaf(&pool, i + 1, length_zero(), tree_len, false, &dummy_language);
subtrees[i] = ts_subtree_new_leaf(
&pool, i + 1, length_zero(), tree_len, 0,
TS_TREE_STATE_NONE, true, false, &dummy_language
);
ts_external_scanner_state_init(&((Subtree *)subtrees[i])->external_scanner_state, nullptr, 0);
}
});

76
test/runtime/subtree_test.cc
View file

@ -48,39 +48,18 @@ describe("Subtree", []() {
ts_subtree_pool_delete(&pool);
});
describe("make_leaf", [&]() {
it("does not mark the tree as fragile", [&]() {
const Subtree *tree = ts_subtree_new_leaf(&pool, symbol1, {2, {0, 1}}, {5, {0, 4}}, false, &language);
AssertThat(tree->fragile_left, IsFalse());
AssertThat(tree->fragile_right, IsFalse());
auto new_leaf = [&](TSSymbol symbol, Length padding, Length size, uint32_t bytes_scanned) {
return ts_subtree_new_leaf(
&pool, symbol, padding, size, bytes_scanned, 0, false, false, &language
);
};
ts_subtree_release(&pool, tree);
});
});
describe("make_error", [&]() {
it("marks the tree as fragile", [&]() {
const Subtree *error_tree = ts_subtree_new_error(
&pool,
length_zero(),
length_zero(),
'z',
&language
);
AssertThat(error_tree->fragile_left, IsTrue());
AssertThat(error_tree->fragile_right, IsTrue());
ts_subtree_release(&pool, error_tree);
});
});
describe("make_node", [&]() {
describe("new_node", [&]() {
const Subtree *tree1, *tree2, *parent1;
before_each([&]() {
tree1 = ts_subtree_new_leaf(&pool, symbol1, {2, {0, 1}}, {5, {0, 4}}, false, &language);
tree2 = ts_subtree_new_leaf(&pool, symbol2, {1, {0, 1}}, {3, {0, 3}}, false, &language);
tree1 = new_leaf(symbol1, {2, {0, 1}}, {5, {0, 4}}, 0);
tree2 = new_leaf(symbol2, {1, {0, 1}}, {3, {0, 3}}, 0);
ts_subtree_retain(tree1);
ts_subtree_retain(tree2);
@ -186,9 +165,9 @@ describe("Subtree", []() {
before_each([&]() {
tree = ts_subtree_new_node(&pool, symbol1, tree_array({
ts_subtree_new_leaf(&pool, symbol2, {2, {0, 2}}, {3, {0, 3}}, false, &language),
ts_subtree_new_leaf(&pool, symbol3, {2, {0, 2}}, {3, {0, 3}}, false, &language),
ts_subtree_new_leaf(&pool, symbol4, {2, {0, 2}}, {3, {0, 3}}, false, &language),
new_leaf(symbol2, {2, {0, 2}}, {3, {0, 3}}, 0),
new_leaf(symbol3, {2, {0, 2}}, {3, {0, 3}}, 0),
new_leaf(symbol4, {2, {0, 2}}, {3, {0, 3}}, 0),
}), 0, &language);
AssertThat(tree->padding, Equals<Length>({2, {0, 2}}));
@ -421,7 +400,7 @@ describe("Subtree", []() {
const Subtree *leaf;
before_each([&]() {
leaf = ts_subtree_new_leaf(&pool, symbol1, {2, {0, 1}}, {5, {0, 4}}, false, &language);
leaf = new_leaf(symbol1, {2, {1, 1}}, {5, {1, 4}}, 0);
});
after_each([&]() {
@ -429,7 +408,7 @@ describe("Subtree", []() {
});
it("returns true for identical trees", [&]() {
const Subtree *leaf_copy = ts_subtree_new_leaf(&pool, symbol1, {2, {1, 1}}, {5, {1, 4}}, false, &language);
const Subtree *leaf_copy = new_leaf(symbol1, {2, {1, 1}}, {5, {1, 4}}, 0);
AssertThat(ts_subtree_eq(leaf, leaf_copy), IsTrue());
const Subtree *parent = ts_subtree_new_node(&pool, symbol2, tree_array({
@ -454,13 +433,11 @@ describe("Subtree", []() {
});
it("returns false for trees with different symbols", [&]() {
const Subtree *different_leaf = ts_subtree_new_leaf(
&pool,
const Subtree *different_leaf = new_leaf(
leaf->symbol + 1,
leaf->padding,
leaf->size,
false,
&language
leaf->bytes_scanned
);
AssertThat(ts_subtree_eq(leaf, different_leaf), IsFalse());
@ -468,8 +445,8 @@ describe("Subtree", []() {
});
it("returns false for trees with different options", [&]() {
const Subtree *different_leaf = ts_subtree_new_leaf(
&pool, leaf->symbol, leaf->padding, leaf->size, false, &language
const Subtree *different_leaf = new_leaf(
leaf->symbol, leaf->padding, leaf->size, leaf->bytes_scanned
);
((Subtree *)different_leaf)->visible = !leaf->visible;
AssertThat(ts_subtree_eq(leaf, different_leaf), IsFalse());
@ -477,19 +454,19 @@ describe("Subtree", []() {
});
it("returns false for trees with different paddings or sizes", [&]() {
const Subtree *different_leaf = ts_subtree_new_leaf(
&pool, leaf->symbol, {}, leaf->size, false, &language
const Subtree *different_leaf = new_leaf(
leaf->symbol, {}, leaf->size, leaf->bytes_scanned
);
AssertThat(ts_subtree_eq(leaf, different_leaf), IsFalse());
ts_subtree_release(&pool, different_leaf);
different_leaf = ts_subtree_new_leaf(&pool, symbol1, leaf->padding, {}, false, &language);
different_leaf = new_leaf(symbol1, leaf->padding, {}, leaf->bytes_scanned);
AssertThat(ts_subtree_eq(leaf, different_leaf), IsFalse());
ts_subtree_release(&pool, different_leaf);
});
it("returns false for trees with different children", [&]() {
const Subtree *leaf2 = ts_subtree_new_leaf(&pool, symbol2, {1, {0, 1}}, {3, {0, 3}}, false, &language);
const Subtree *leaf2 = new_leaf(symbol2, {1, {0, 1}}, {3, {0, 3}}, 0);
const Subtree *parent = ts_subtree_new_node(&pool, symbol2, tree_array({
leaf,
@ -520,7 +497,6 @@ describe("Subtree", []() {
auto make_external = [](const Subtree *_tree) {
Subtree *tree = (Subtree *)_tree;
tree->has_external_tokens = true;
ts_external_scanner_state_init(&tree->external_scanner_state, NULL, 0);
return tree;
};
@ -530,15 +506,15 @@ describe("Subtree", []() {
tree1 = ts_subtree_new_node(&pool, symbol1, tree_array({
(tree2 = ts_subtree_new_node(&pool, symbol2, tree_array({
(tree3 = make_external(ts_subtree_new_leaf(&pool, symbol3, padding, size, false, &language))),
(tree4 = ts_subtree_new_leaf(&pool, symbol4, padding, size, false, &language)),
(tree5 = ts_subtree_new_leaf(&pool, symbol5, padding, size, false, &language)),
(tree3 = make_external(ts_subtree_new_leaf(&pool, symbol3, padding, size, 0, 0, true, false, &language))),
(tree4 = new_leaf(symbol4, padding, size, 0)),
(tree5 = new_leaf(symbol5, padding, size, 0)),
}), 0, &language)),
(tree6 = ts_subtree_new_node(&pool, symbol6, tree_array({
(tree7 = ts_subtree_new_node(&pool, symbol7, tree_array({
(tree8 = ts_subtree_new_leaf(&pool, symbol8, padding, size, false, &language)),
(tree8 = new_leaf(symbol8, padding, size, 0)),
}), 0, &language)),
(tree9 = ts_subtree_new_leaf(&pool, symbol9, padding, size, false, &language)),
(tree9 = new_leaf(symbol9, padding, size, 0)),
}), 0, &language)),
}), 0, &language);