traxys/tree-sitter
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Support anonymous tokens inside of RENAME rules

Browse source
This commit is contained in:
Max Brunsfeld 2017-07-14 10:19:58 -07:00
parent b3a72954ff
commit 4b40a1ed6c
14 changed files with 230 additions and 38 deletions
Download patch file Download diff file Expand all files Collapse all files

7
src/runtime/language.h
View file

@ -63,6 +63,13 @@ ts_language_enabled_external_tokens(const TSLanguage *self,
}
}
static inline const TSSymbol *
ts_language_rename_sequence(const TSLanguage *self, unsigned id) {
return id > 0 ?
self->rename_sequences + id * self->max_rename_sequence_length :
NULL;
}
#ifdef __cplusplus
}
#endif

3
src/runtime/node.c
View file

@ -304,7 +304,8 @@ bool ts_node_eq(TSNode self, TSNode other) {
}
bool ts_node_is_named(TSNode self) {
return ts_node__tree(self)->named;
const Tree *tree = ts_node__tree(self);
return tree->named || tree->context.rename_symbol != 0;
}
bool ts_node_has_changes(TSNode self) {

24
src/runtime/parser.c
View file

@ -538,10 +538,11 @@ static void parser__shift(Parser *self, StackVersion version, TSStateId state,
}
static bool parser__switch_children(Parser *self, Tree *tree,
Tree **children, uint32_t count) {
Tree **children, uint32_t count,
const TSSymbol *rename_sequence) {
self->scratch_tree.symbol = tree->symbol;
self->scratch_tree.child_count = 0;
ts_tree_set_children(&self->scratch_tree, count, children);
ts_tree_set_children(&self->scratch_tree, count, children, rename_sequence);
if (parser__select_tree(self, tree, &self->scratch_tree)) {
tree->size = self->scratch_tree.size;
tree->padding = self->scratch_tree.padding;
@ -568,6 +569,7 @@ static StackPopResult parser__reduce(Parser *self, StackVersion version,
const TSLanguage *language = self->language;
TSSymbolMetadata metadata = ts_language_symbol_metadata(language, symbol);
const TSSymbol *rename_sequence = ts_language_rename_sequence(language, rename_sequence_id);
for (uint32_t i = 0; i < pop.slices.size; i++) {
StackSlice slice = pop.slices.contents[i];
@ -579,7 +581,7 @@ static StackPopResult parser__reduce(Parser *self, StackVersion version,
while (child_count > 0 && slice.trees.contents[child_count - 1]->extra)
child_count--;
Tree *parent = ts_tree_make_node(symbol, child_count, slice.trees.contents, metadata);
Tree *parent = ts_tree_make_node(symbol, child_count, slice.trees.contents, metadata, rename_sequence);
// This pop operation may have caused multiple stack versions to collapse
// into one, because they all diverged from a common state. In that case,
@ -594,7 +596,7 @@ static StackPopResult parser__reduce(Parser *self, StackVersion version,
while (child_count > 0 && next_slice.trees.contents[child_count - 1]->extra)
child_count--;
if (parser__switch_children(self, parent, next_slice.trees.contents, child_count)) {
if (parser__switch_children(self, parent, next_slice.trees.contents, child_count, rename_sequence)) {
ts_tree_array_delete(&slice.trees);
slice = next_slice;
} else {
@ -839,9 +841,11 @@ static bool parser__repair_error(Parser *self, StackSlice slice,
array_push(&children, slice.trees.contents[i]);
array_delete(&slice.trees);
Tree *parent =
ts_tree_make_node(symbol, children.size, children.contents,
ts_language_symbol_metadata(self->language, symbol));
Tree *parent = ts_tree_make_node(
symbol, children.size, children.contents,
ts_language_symbol_metadata(self->language, symbol),
NULL
);
parser__push(self, slice.version, parent, next_state);
ts_stack_decrease_push_count(self->stack, slice.version, error->child_count);
@ -899,7 +903,11 @@ static void parser__accept(Parser *self, StackVersion version,
for (uint32_t k = 0; k < child->child_count; k++)
ts_tree_retain(child->children[k]);
array_splice(&trees, j, 1, child->child_count, child->children);
ts_tree_set_children(root, trees.size, trees.contents);
const TSSymbol *rename_sequence = ts_language_rename_sequence(
self->language,
root->rename_sequence_id
);
ts_tree_set_children(root, trees.size, trees.contents, rename_sequence);
ts_tree_release(child);
break;
}

28
src/runtime/tree.c
View file

@ -7,6 +7,7 @@
#include "runtime/alloc.h"
#include "runtime/tree.h"
#include "runtime/length.h"
#include "runtime/language.h"
#include "runtime/error_costs.h"
TSStateId TS_TREE_STATE_NONE = USHRT_MAX;
@ -127,16 +128,15 @@ void ts_tree_assign_parents(Tree *self, TreePath *path, const TSLanguage *langua
while (path->size > 0) {
Tree *tree = array_pop(path).tree;
Length offset = length_zero();
const TSSymbol *rename_symbols = language->rename_sequences +
tree->rename_sequence_id * language->max_rename_sequence_length;
const TSSymbol *rename_sequence = ts_language_rename_sequence(language, tree->rename_sequence_id);
for (uint32_t i = 0; i < tree->child_count; i++) {
Tree *child = tree->children[i];
if (child->context.parent != tree || child->context.index != i) {
child->context.parent = tree;
child->context.index = i;
child->context.offset = offset;
if (tree->rename_sequence_id && rename_symbols[i] != 0) {
child->context.rename_symbol = rename_symbols[i];
if (rename_sequence && rename_sequence[i] != 0) {
child->context.rename_symbol = rename_sequence[i];
}
array_push(path, ((TreePathEntry){child, length_zero(), 0}));
}
@ -146,7 +146,8 @@ void ts_tree_assign_parents(Tree *self, TreePath *path, const TSLanguage *langua
}
void ts_tree_set_children(Tree *self, uint32_t child_count, Tree **children) {
void ts_tree_set_children(Tree *self, uint32_t child_count, Tree **children,
const TSSymbol *rename_sequence) {
if (self->child_count > 0)
ts_free(self->children);
@ -176,8 +177,9 @@ void ts_tree_set_children(Tree *self, uint32_t child_count, Tree **children) {
if (child->visible) {
self->visible_child_count++;
if (child->named)
if (child->named || (rename_sequence && rename_sequence[i] != 0)) {
self->named_child_count++;
}
} else if (child->child_count > 0) {
self->visible_child_count += child->visible_child_count;
self->named_child_count += child->named_child_count;
@ -208,11 +210,11 @@ void ts_tree_set_children(Tree *self, uint32_t child_count, Tree **children) {
}
}
Tree *ts_tree_make_node(TSSymbol symbol, uint32_t child_count,
Tree **children, TSSymbolMetadata metadata) {
Tree *ts_tree_make_node(TSSymbol symbol, uint32_t child_count, Tree **children,
TSSymbolMetadata metadata, const TSSymbol *rename_sequence) {
Tree *result =
ts_tree_make_leaf(symbol, length_zero(), length_zero(), metadata);
ts_tree_set_children(result, child_count, children);
ts_tree_set_children(result, child_count, children, rename_sequence);
return result;
}
@ -230,7 +232,7 @@ Tree *ts_tree_make_error_node(TreeArray *children) {
Tree *result = ts_tree_make_node(
ts_builtin_sym_error, children->size, children->contents,
(TSSymbolMetadata){.extra = false, .visible = true, .named = true });
(TSSymbolMetadata){.extra = false, .visible = true, .named = true }, NULL);
result->fragile_left = true;
result->fragile_right = true;
@ -482,7 +484,11 @@ static size_t ts_tree__write_to_string(const Tree *self,
char *cursor = string;
char **writer = (limit > 0) ? &cursor : &string;
bool visible = include_all || is_root || (self->visible && self->named);
bool visible =
include_all ||
is_root ||
(self->visible && self->named) ||
self->context.rename_symbol != 0;
if (visible && !is_root) {
cursor += snprintf(*writer, limit, " ");

4
src/runtime/tree.h
View file

@ -75,7 +75,7 @@ TreeArray ts_tree_array_remove_last_n(TreeArray *, uint32_t);
TreeArray ts_tree_array_remove_trailing_extras(TreeArray *);
Tree *ts_tree_make_leaf(TSSymbol, Length, Length, TSSymbolMetadata);
Tree *ts_tree_make_node(TSSymbol, uint32_t, Tree **, TSSymbolMetadata);
Tree *ts_tree_make_node(TSSymbol, uint32_t, Tree **, TSSymbolMetadata, const TSSymbol *);
Tree *ts_tree_make_copy(Tree *child);
Tree *ts_tree_make_error_node(TreeArray *);
Tree *ts_tree_make_error(Length, Length, int32_t);
@ -86,7 +86,7 @@ int ts_tree_compare(const Tree *tree1, const Tree *tree2);
uint32_t ts_tree_start_column(const Tree *self);
uint32_t ts_tree_end_column(const Tree *self);
void ts_tree_set_children(Tree *, uint32_t, Tree **);
void ts_tree_set_children(Tree *, uint32_t, Tree **, const TSSymbol *);
void ts_tree_assign_parents(Tree *, TreePath *, const TSLanguage *);
void ts_tree_edit(Tree *, const TSInputEdit *edit);
char *ts_tree_string(const Tree *, const TSLanguage *, bool include_all);

18
test/fixtures/test_grammars/inlined_renamed_rules/corpus.txt vendored Normal file
View file

@ -0,0 +1,18 @@
======================================
Method calls
======================================
a.b(c(d.e));
---
(statement
(call_expression
(member_expression
(variable_name)
(property_name))
(call_expression
(variable_name)
(member_expression
(variable_name)
(property_name)))))

73
test/fixtures/test_grammars/inlined_renamed_rules/grammar.json vendored Normal file
View file

@ -0,0 +1,73 @@
{
"name": "inlined_renamed_rules",
"extras": [
{"type": "PATTERN", "value": "\\s"}
],
"inline": [
"expression"
],
"rules": {
"statement": {
"type": "SEQ",
"members": [
{"type": "SYMBOL", "name": "expression"},
{"type": "STRING", "value": ";"}
]
},
"expression": {
"type": "CHOICE",
"members": [
{"type": "SYMBOL", "name": "call_expression"},
{"type": "SYMBOL", "name": "member_expression"},
{
"type": "RENAME",
"value": "variable_name",
"content": {
"type": "SYMBOL",
"name": "identifier"
}
}
]
},
"call_expression": {
"type": "PREC_LEFT",
"value": 0,
"content": {
"type": "SEQ",
"members": [
{"type": "SYMBOL", "name": "expression"},
{"type": "STRING", "value": "("},
{"type": "SYMBOL", "name": "expression"},
{"type": "STRING", "value": ")"},
]
}
},
"member_expression": {
"type": "PREC_LEFT",
"value": 1,
"content": {
"type": "SEQ",
"members": [
{"type": "SYMBOL", "name": "expression"},
{"type": "STRING", "value": "."},
{
"type": "RENAME",
"value": "property_name",
"content": {
"type": "SYMBOL",
"name": "identifier"
}
}
]
}
},
"identifier": {"type": "PATTERN", "value": "\\a+"}
}
}

1
test/fixtures/test_grammars/inlined_renamed_rules/readme.md vendored Normal file
View file

@ -0,0 +1 @@
This grammar shows that a rule marked as `inline` can *contain* a `RENAME` rule.

13
test/fixtures/test_grammars/renamed_inlined_rules/corpus.txt vendored Normal file
View file

@ -0,0 +1,13 @@
=========================
OK
=========================
a.b.c
---
(expression (member_expression
(expression (member_expression
(expression (variable_name))
(property_name)))
(property_name)))

57
test/fixtures/test_grammars/renamed_inlined_rules/grammar.json vendored Normal file
View file

@ -0,0 +1,57 @@
{
"name": "renamed_inlined_rules",
"extras": [
{"type": "PATTERN", "value": "\\s"}
],
"inline": [
"identifier"
],
"rules": {
"expression": {
"type": "CHOICE",
"members": [
{"type": "SYMBOL", "name": "member_expression"},
{
"type": "RENAME",
"value": "variable_name",
"content": {
"type": "SYMBOL",
"name": "identifier"
}
}
]
},
"member_expression": {
"type": "PREC_LEFT",
"value": 1,
"content": {
"type": "SEQ",
"members": [
{"type": "SYMBOL", "name": "expression"},
{"type": "STRING", "value": "."},
{
"type": "RENAME",
"value": "property_name",
"content": {
"type": "SYMBOL",
"name": "identifier"
}
}
]
}
},
"identifier": {
"type": "CHOICE",
"members": [
{"type": "STRING", "value": "a"},
{"type": "STRING", "value": "b"},
{"type": "STRING", "value": "c"}
]
}
}
}

1
test/fixtures/test_grammars/renamed_inlined_rules/readme.md vendored Normal file
View file

@ -0,0 +1 @@
This grammar shows that `RENAME` rules can *contain* a rule that is marked as `inline`. It also shows that you can rename a rule that would otherwise be anonymous, and it will then appear as a named node.

11
test/helpers/tree_helpers.cc
View file

@ -53,11 +53,11 @@ bool operator==(const std::vector<Tree *> &vec, const TreeArray &array) {
void assert_consistent_tree_sizes(TSNode node) {
size_t child_count = ts_node_child_count(node);
size_t named_child_count = ts_node_named_child_count(node);
size_t start_byte = ts_node_start_byte(node);
size_t end_byte = ts_node_end_byte(node);
TSPoint start_point = ts_node_start_point(node);
TSPoint end_point = ts_node_end_point(node);
bool some_child_has_changes = false;
AssertThat(start_byte, !IsGreaterThan(end_byte));
AssertThat(start_point, !IsGreaterThan(end_point));
@ -65,6 +65,8 @@ void assert_consistent_tree_sizes(TSNode node) {
size_t last_child_end_byte = start_byte;
TSPoint last_child_end_point = start_point;
bool some_child_has_changes = false;
size_t actual_named_child_count = 0;
for (size_t i = 0; i < child_count; i++) {
TSNode child = ts_node_child(node, i);
size_t child_start_byte = ts_node_start_byte(child);
@ -73,13 +75,16 @@ void assert_consistent_tree_sizes(TSNode node) {
AssertThat(child_start_byte, !IsLessThan(last_child_end_byte));
AssertThat(child_start_point, !IsLessThan(last_child_end_point));
assert_consistent_tree_sizes(child);
if (ts_node_has_changes(child))
some_child_has_changes = true;
if (ts_node_has_changes(child)) some_child_has_changes = true;
if (ts_node_is_named(child)) actual_named_child_count++;
last_child_end_byte = ts_node_end_byte(child);
last_child_end_point = ts_node_end_point(child);
}
AssertThat(actual_named_child_count, Equals(named_child_count));
if (child_count > 0) {
AssertThat(end_byte, !IsLessThan(last_child_end_byte));
AssertThat(end_point, !IsLessThan(last_child_end_point));

2
test/integration/test_grammars.cc
View file

@ -3,6 +3,7 @@
#include "helpers/load_language.h"
#include "helpers/stderr_logger.h"
#include "helpers/file_helpers.h"
#include "helpers/tree_helpers.h"
#include "runtime/alloc.h"
START_TEST
@ -54,6 +55,7 @@ for (auto &language_name : test_languages) {
ts_document_parse(document);
TSNode root_node = ts_document_root_node(document);
assert_consistent_tree_sizes(root_node);
const char *node_string = ts_node_string(root_node, document);
string result(node_string);
ts_free((void *)node_string);

26
test/runtime/tree_test.cc
View file

@ -72,7 +72,7 @@ describe("Tree", []() {
parent1 = ts_tree_make_node(symbol3, 2, tree_array({
tree1,
tree2,
}), visible);
}), visible, nullptr);
});
after_each([&]() {
@ -103,7 +103,7 @@ describe("Tree", []() {
parent = ts_tree_make_node(symbol3, 2, tree_array({
tree1,
tree2,
}), visible);
}), visible, nullptr);
});
after_each([&]() {
@ -127,7 +127,7 @@ describe("Tree", []() {
parent = ts_tree_make_node(symbol3, 2, tree_array({
tree1,
tree2,
}), visible);
}), visible, nullptr);
});
after_each([&]() {
@ -151,7 +151,7 @@ describe("Tree", []() {
parent = ts_tree_make_node(symbol3, 2, tree_array({
tree1,
tree2,
}), visible);
}), visible, nullptr);
});
after_each([&]() {
@ -173,7 +173,7 @@ describe("Tree", []() {
ts_tree_make_leaf(symbol2, {2, 2, {0, 2}}, {3, 3, {0, 3}}, visible),
ts_tree_make_leaf(symbol3, {2, 2, {0, 2}}, {3, 3, {0, 3}}, visible),
ts_tree_make_leaf(symbol4, {2, 2, {0, 2}}, {3, 3, {0, 3}}, visible),
}), visible);
}), visible, nullptr);
AssertThat(tree->padding, Equals<Length>({2, 2, {0, 2}}));
AssertThat(tree->size, Equals<Length>({13, 13, {0, 13}}));
@ -350,14 +350,14 @@ describe("Tree", []() {
Tree *parent = ts_tree_make_node(symbol2, 2, tree_array({
leaf,
leaf_copy,
}), visible);
}), visible, nullptr);
ts_tree_retain(leaf);
ts_tree_retain(leaf_copy);
Tree *parent_copy = ts_tree_make_node(symbol2, 2, tree_array({
leaf,
leaf_copy,
}), visible);
}), visible, nullptr);
ts_tree_retain(leaf);
ts_tree_retain(leaf_copy);
@ -401,14 +401,14 @@ describe("Tree", []() {
Tree *parent = ts_tree_make_node(symbol2, 2, tree_array({
leaf,
leaf2,
}), visible);
}), visible, nullptr);
ts_tree_retain(leaf);
ts_tree_retain(leaf2);
Tree *different_parent = ts_tree_make_node(symbol2, 2, tree_array({
leaf2,
leaf,
}), visible);
}), visible, nullptr);
ts_tree_retain(leaf2);
ts_tree_retain(leaf);
@ -438,14 +438,14 @@ describe("Tree", []() {
(tree3 = make_external(ts_tree_make_leaf(symbol3, padding, size, visible))),
(tree4 = ts_tree_make_leaf(symbol4, padding, size, visible)),
(tree5 = ts_tree_make_leaf(symbol5, padding, size, visible)),
}), visible)),
}), visible, nullptr)),
(tree6 = ts_tree_make_node(symbol6, 2, tree_array({
(tree7 = ts_tree_make_node(symbol7, 1, tree_array({
(tree8 = ts_tree_make_leaf(symbol8, padding, size, visible)),
}), visible)),
}), visible, nullptr)),
(tree9 = ts_tree_make_leaf(symbol9, padding, size, visible)),
}), visible)),
}), visible);
}), visible, nullptr)),
}), visible, nullptr);
auto token = ts_tree_last_external_token(tree1);
AssertThat(token, Equals(tree3));