traxys/tree-sitter
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Support anonymous tokens inside of RENAME rules

Browse source
This commit is contained in:
Max Brunsfeld 2017-07-14 10:19:58 -07:00
parent b3a72954ff
commit 4b40a1ed6c
14 changed files with 230 additions and 38 deletions
Download patch file Download diff file Expand all files Collapse all files

7
src/runtime/language.h
View file

@ -63,6 +63,13 @@ ts_language_enabled_external_tokens(const TSLanguage *self,
}
}
static inline const TSSymbol *
ts_language_rename_sequence(const TSLanguage *self, unsigned id) {
return id > 0 ?
self->rename_sequences + id * self->max_rename_sequence_length :
NULL;
}
#ifdef __cplusplus
}
#endif

3
src/runtime/node.c
View file

@ -304,7 +304,8 @@ bool ts_node_eq(TSNode self, TSNode other) {
}
bool ts_node_is_named(TSNode self) {
return ts_node__tree(self)->named;
const Tree *tree = ts_node__tree(self);
return tree->named || tree->context.rename_symbol != 0;
}
bool ts_node_has_changes(TSNode self) {

24
src/runtime/parser.c
View file

@ -538,10 +538,11 @@ static void parser__shift(Parser *self, StackVersion version, TSStateId state,
}
static bool parser__switch_children(Parser *self, Tree *tree,
Tree **children, uint32_t count) {
Tree **children, uint32_t count,
const TSSymbol *rename_sequence) {
self->scratch_tree.symbol = tree->symbol;
self->scratch_tree.child_count = 0;
ts_tree_set_children(&self->scratch_tree, count, children);
ts_tree_set_children(&self->scratch_tree, count, children, rename_sequence);
if (parser__select_tree(self, tree, &self->scratch_tree)) {
tree->size = self->scratch_tree.size;
tree->padding = self->scratch_tree.padding;
@ -568,6 +569,7 @@ static StackPopResult parser__reduce(Parser *self, StackVersion version,
const TSLanguage *language = self->language;
TSSymbolMetadata metadata = ts_language_symbol_metadata(language, symbol);
const TSSymbol *rename_sequence = ts_language_rename_sequence(language, rename_sequence_id);
for (uint32_t i = 0; i < pop.slices.size; i++) {
StackSlice slice = pop.slices.contents[i];
@ -579,7 +581,7 @@ static StackPopResult parser__reduce(Parser *self, StackVersion version,
while (child_count > 0 && slice.trees.contents[child_count - 1]->extra)
child_count--;
Tree *parent = ts_tree_make_node(symbol, child_count, slice.trees.contents, metadata);
Tree *parent = ts_tree_make_node(symbol, child_count, slice.trees.contents, metadata, rename_sequence);
// This pop operation may have caused multiple stack versions to collapse
// into one, because they all diverged from a common state. In that case,
@ -594,7 +596,7 @@ static StackPopResult parser__reduce(Parser *self, StackVersion version,
while (child_count > 0 && next_slice.trees.contents[child_count - 1]->extra)
child_count--;
if (parser__switch_children(self, parent, next_slice.trees.contents, child_count)) {
if (parser__switch_children(self, parent, next_slice.trees.contents, child_count, rename_sequence)) {
ts_tree_array_delete(&slice.trees);
slice = next_slice;
} else {
@ -839,9 +841,11 @@ static bool parser__repair_error(Parser *self, StackSlice slice,
array_push(&children, slice.trees.contents[i]);
array_delete(&slice.trees);
Tree *parent =
ts_tree_make_node(symbol, children.size, children.contents,
ts_language_symbol_metadata(self->language, symbol));
Tree *parent = ts_tree_make_node(
symbol, children.size, children.contents,
ts_language_symbol_metadata(self->language, symbol),
NULL
);
parser__push(self, slice.version, parent, next_state);
ts_stack_decrease_push_count(self->stack, slice.version, error->child_count);
@ -899,7 +903,11 @@ static void parser__accept(Parser *self, StackVersion version,
for (uint32_t k = 0; k < child->child_count; k++)
ts_tree_retain(child->children[k]);
array_splice(&trees, j, 1, child->child_count, child->children);
ts_tree_set_children(root, trees.size, trees.contents);
const TSSymbol *rename_sequence = ts_language_rename_sequence(
self->language,
root->rename_sequence_id
);
ts_tree_set_children(root, trees.size, trees.contents, rename_sequence);
ts_tree_release(child);
break;
}

28
src/runtime/tree.c
View file

@ -7,6 +7,7 @@
#include "runtime/alloc.h"
#include "runtime/tree.h"
#include "runtime/length.h"
#include "runtime/language.h"
#include "runtime/error_costs.h"
TSStateId TS_TREE_STATE_NONE = USHRT_MAX;
@ -127,16 +128,15 @@ void ts_tree_assign_parents(Tree *self, TreePath *path, const TSLanguage *langua
while (path->size > 0) {
Tree *tree = array_pop(path).tree;
Length offset = length_zero();
const TSSymbol *rename_symbols = language->rename_sequences +
tree->rename_sequence_id * language->max_rename_sequence_length;
const TSSymbol *rename_sequence = ts_language_rename_sequence(language, tree->rename_sequence_id);
for (uint32_t i = 0; i < tree->child_count; i++) {
Tree *child = tree->children[i];
if (child->context.parent != tree || child->context.index != i) {
child->context.parent = tree;
child->context.index = i;
child->context.offset = offset;
if (tree->rename_sequence_id && rename_symbols[i] != 0) {
child->context.rename_symbol = rename_symbols[i];
if (rename_sequence && rename_sequence[i] != 0) {
child->context.rename_symbol = rename_sequence[i];
}
array_push(path, ((TreePathEntry){child, length_zero(), 0}));
}
@ -146,7 +146,8 @@ void ts_tree_assign_parents(Tree *self, TreePath *path, const TSLanguage *langua
}
void ts_tree_set_children(Tree *self, uint32_t child_count, Tree **children) {
void ts_tree_set_children(Tree *self, uint32_t child_count, Tree **children,
const TSSymbol *rename_sequence) {
if (self->child_count > 0)
ts_free(self->children);
@ -176,8 +177,9 @@ void ts_tree_set_children(Tree *self, uint32_t child_count, Tree **children) {
if (child->visible) {
self->visible_child_count++;
if (child->named)
if (child->named || (rename_sequence && rename_sequence[i] != 0)) {
self->named_child_count++;
}
} else if (child->child_count > 0) {
self->visible_child_count += child->visible_child_count;
self->named_child_count += child->named_child_count;
@ -208,11 +210,11 @@ void ts_tree_set_children(Tree *self, uint32_t child_count, Tree **children) {
}
}
Tree *ts_tree_make_node(TSSymbol symbol, uint32_t child_count,
Tree **children, TSSymbolMetadata metadata) {
Tree *ts_tree_make_node(TSSymbol symbol, uint32_t child_count, Tree **children,
TSSymbolMetadata metadata, const TSSymbol *rename_sequence) {
Tree *result =
ts_tree_make_leaf(symbol, length_zero(), length_zero(), metadata);
ts_tree_set_children(result, child_count, children);
ts_tree_set_children(result, child_count, children, rename_sequence);
return result;
}
@ -230,7 +232,7 @@ Tree *ts_tree_make_error_node(TreeArray *children) {
Tree *result = ts_tree_make_node(
ts_builtin_sym_error, children->size, children->contents,
(TSSymbolMetadata){.extra = false, .visible = true, .named = true });
(TSSymbolMetadata){.extra = false, .visible = true, .named = true }, NULL);
result->fragile_left = true;
result->fragile_right = true;
@ -482,7 +484,11 @@ static size_t ts_tree__write_to_string(const Tree *self,
char *cursor = string;
char **writer = (limit > 0) ? &cursor : &string;
bool visible = include_all || is_root || (self->visible && self->named);
bool visible =
include_all ||
is_root ||
(self->visible && self->named) ||
self->context.rename_symbol != 0;
if (visible && !is_root) {
cursor += snprintf(*writer, limit, " ");

4
src/runtime/tree.h
View file

@ -75,7 +75,7 @@ TreeArray ts_tree_array_remove_last_n(TreeArray *, uint32_t);
TreeArray ts_tree_array_remove_trailing_extras(TreeArray *);
Tree *ts_tree_make_leaf(TSSymbol, Length, Length, TSSymbolMetadata);
Tree *ts_tree_make_node(TSSymbol, uint32_t, Tree **, TSSymbolMetadata);
Tree *ts_tree_make_node(TSSymbol, uint32_t, Tree **, TSSymbolMetadata, const TSSymbol *);
Tree *ts_tree_make_copy(Tree *child);
Tree *ts_tree_make_error_node(TreeArray *);
Tree *ts_tree_make_error(Length, Length, int32_t);
@ -86,7 +86,7 @@ int ts_tree_compare(const Tree *tree1, const Tree *tree2);
uint32_t ts_tree_start_column(const Tree *self);
uint32_t ts_tree_end_column(const Tree *self);
void ts_tree_set_children(Tree *, uint32_t, Tree **);
void ts_tree_set_children(Tree *, uint32_t, Tree **, const TSSymbol *);
void ts_tree_assign_parents(Tree *, TreePath *, const TSLanguage *);
void ts_tree_edit(Tree *, const TSInputEdit *edit);
char *ts_tree_string(const Tree *, const TSLanguage *, bool include_all);