traxys/tree-sitter
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Rename RENAME rule to ALIAS, allow it to create anonymous nodes

Browse source
This commit is contained in:
Max Brunsfeld 2017-07-31 11:45:24 -07:00
parent b5f421cafb
commit cb5fe80348
28 changed files with 304 additions and 270 deletions
Download patch file Download diff file Expand all files Collapse all files

2
src/runtime/language.c
View file

@ -32,7 +32,7 @@ void ts_language_table_entry(const TSLanguage *self, TSStateId state,
}
uint32_t ts_language_symbol_count(const TSLanguage *language) {
return language->symbol_count + language->rename_symbol_count;
return language->symbol_count + language->alias_count;
}
uint32_t ts_language_version(const TSLanguage *language) {

4
src/runtime/language.h
View file

@ -64,9 +64,9 @@ ts_language_enabled_external_tokens(const TSLanguage *self,
}
static inline const TSSymbol *
ts_language_rename_sequence(const TSLanguage *self, unsigned id) {
ts_language_alias_sequence(const TSLanguage *self, unsigned id) {
return id > 0 ?
self->rename_sequences + id * self->max_rename_sequence_length :
self->alias_sequences + id * self->max_alias_sequence_length :
NULL;
}

6
src/runtime/node.c
View file

@ -33,7 +33,7 @@ static inline uint32_t ts_node__offset_row(TSNode self) {
static inline bool ts_node__is_relevant(TSNode self, bool include_anonymous) {
const Tree *tree = ts_node__tree(self);
if (tree->context.rename_symbol > 0) {
if (tree->context.alias_symbol > 0) {
return true;
} else {
return include_anonymous ? tree->visible : tree->visible && tree->named;
@ -269,7 +269,7 @@ TSPoint ts_node_end_point(TSNode self) {
TSSymbol ts_node_symbol(TSNode self) {
const Tree *tree = ts_node__tree(self);
return tree->context.rename_symbol ? tree->context.rename_symbol : tree->symbol;
return tree->context.alias_symbol ? tree->context.alias_symbol : tree->symbol;
}
TSSymbolIterator ts_node_symbols(TSNode self) {
@ -308,7 +308,7 @@ bool ts_node_eq(TSNode self, TSNode other) {
bool ts_node_is_named(TSNode self) {
const Tree *tree = ts_node__tree(self);
return tree->named || tree->context.rename_symbol != 0;
return tree->context.alias_symbol ? tree->context.alias_is_named : tree->named;
}
bool ts_node_has_changes(TSNode self) {

49
src/runtime/parser.c
View file

@ -338,7 +338,7 @@ static Tree *parser__lex(Parser *self, StackVersion version) {
if (skipped_error) {
Length padding = length_sub(error_start_position, start_position);
Length size = length_sub(error_end_position, error_start_position);
result = ts_tree_make_error(size, padding, first_error_character);
result = ts_tree_make_error(size, padding, first_error_character, self->language);
} else {
TSSymbol symbol = self->lexer.data.result_symbol;
if (found_external_token) {
@ -347,8 +347,7 @@ static Tree *parser__lex(Parser *self, StackVersion version) {
Length padding = length_sub(self->lexer.token_start_position, start_position);
Length size = length_sub(self->lexer.token_end_position, self->lexer.token_start_position);
TSSymbolMetadata metadata = ts_language_symbol_metadata(self->language, symbol);
result = ts_tree_make_leaf(symbol, padding, size, metadata);
result = ts_tree_make_leaf(symbol, padding, size, self->language);
if (found_external_token) {
result->has_external_tokens = true;
@ -544,12 +543,7 @@ static void parser__shift(Parser *self, StackVersion version, TSStateId state,
static bool parser__replace_children(Parser *self, Tree *tree, Tree **children, uint32_t count) {
self->scratch_tree.symbol = tree->symbol;
self->scratch_tree.child_count = 0;
ts_tree_set_children(
&self->scratch_tree,
count,
children,
ts_language_rename_sequence(self->language, tree->rename_sequence_id)
);
ts_tree_set_children(&self->scratch_tree, count, children, self->language);
if (parser__select_tree(self, tree, &self->scratch_tree)) {
tree->size = self->scratch_tree.size;
tree->padding = self->scratch_tree.padding;
@ -566,16 +560,13 @@ static bool parser__replace_children(Parser *self, Tree *tree, Tree **children,
static StackPopResult parser__reduce(Parser *self, StackVersion version,
TSSymbol symbol, uint32_t count,
int dynamic_precedence, uint16_t rename_sequence_id,
int dynamic_precedence, uint16_t alias_sequence_id,
bool fragile, bool allow_skipping) {
uint32_t initial_version_count = ts_stack_version_count(self->stack);
StackPopResult pop = ts_stack_pop_count(self->stack, version, count);
if (pop.stopped_at_error) return pop;
TSSymbolMetadata metadata = ts_language_symbol_metadata(self->language, symbol);
const TSSymbol *rename_sequence = ts_language_rename_sequence(self->language, rename_sequence_id);
for (uint32_t i = 0; i < pop.slices.size; i++) {
StackSlice slice = pop.slices.contents[i];
@ -588,7 +579,7 @@ static StackPopResult parser__reduce(Parser *self, StackVersion version,
}
Tree *parent = ts_tree_make_node(
symbol, child_count, slice.trees.contents, metadata, rename_sequence
symbol, child_count, slice.trees.contents, alias_sequence_id, self->language
);
// This pop operation may have caused multiple stack versions to collapse
@ -614,7 +605,7 @@ static StackPopResult parser__reduce(Parser *self, StackVersion version,
}
parent->dynamic_precedence += dynamic_precedence;
parent->rename_sequence_id = rename_sequence_id;
parent->alias_sequence_id = alias_sequence_id;
TSStateId state = ts_stack_top_state(self->stack, slice.version);
TSStateId next_state = ts_language_next_state(self->language, state, symbol);
@ -840,7 +831,7 @@ static bool parser__repair_error(Parser *self, StackSlice slice,
TreeArray skipped_children = ts_tree_array_remove_last_n(&children, skip_count);
TreeArray trailing_extras = ts_tree_array_remove_trailing_extras(&skipped_children);
Tree *error = ts_tree_make_error_node(&skipped_children);
Tree *error = ts_tree_make_error_node(&skipped_children, self->language);
array_push(&children, error);
array_push_all(&children, &trailing_extras);
trailing_extras.size = 0;
@ -852,8 +843,7 @@ static bool parser__repair_error(Parser *self, StackSlice slice,
Tree *parent = ts_tree_make_node(
symbol, children.size, children.contents,
ts_language_symbol_metadata(self->language, symbol),
NULL
0, self->language
);
parser__push(self, slice.version, parent, next_state);
ts_stack_decrease_push_count(self->stack, slice.version, error->child_count);
@ -912,11 +902,7 @@ static void parser__accept(Parser *self, StackVersion version,
for (uint32_t k = 0; k < child->child_count; k++)
ts_tree_retain(child->children[k]);
array_splice(&trees, j, 1, child->child_count, child->children);
const TSSymbol *rename_sequence = ts_language_rename_sequence(
self->language,
root->rename_sequence_id
);
ts_tree_set_children(root, trees.size, trees.contents, rename_sequence);
ts_tree_set_children(root, trees.size, trees.contents, self->language);
ts_tree_release(child);
break;
}
@ -965,7 +951,7 @@ static bool parser__do_potential_reductions(Parser *self, StackVersion version)
.symbol = action.params.symbol,
.count = action.params.child_count,
.dynamic_precedence = action.params.dynamic_precedence,
.rename_sequence_id = action.params.rename_sequence_id,
.alias_sequence_id = action.params.alias_sequence_id,
});
default:
break;
@ -978,7 +964,7 @@ static bool parser__do_potential_reductions(Parser *self, StackVersion version)
ReduceAction action = self->reduce_actions.contents[i];
StackPopResult reduction = parser__reduce(
self, version, action.symbol, action.count,
action.dynamic_precedence, action.rename_sequence_id,
action.dynamic_precedence, action.alias_sequence_id,
true, false
);
if (reduction.stopped_at_error) {
@ -1039,7 +1025,7 @@ static bool parser__skip_preceding_trees(Parser *self, StackVersion version,
}
previous_version = slice.version;
Tree *error = ts_tree_make_error_node(&slice.trees);
Tree *error = ts_tree_make_error_node(&slice.trees, self->language);
error->extra = true;
TSStateId state = ts_stack_top_state(self->stack, slice.version);
parser__push(self, slice.version, error, state);
@ -1104,16 +1090,15 @@ static void parser__halt_parse(Parser *self) {
ts_stack_top_position(self->stack, 0)
);
Tree *filler_node = ts_tree_make_error(remaining_length, length_zero(), 0);
Tree *filler_node = ts_tree_make_error(remaining_length, length_zero(), 0, self->language);
filler_node->visible = false;
parser__push(self, 0, filler_node, 0);
TreeArray children = array_new();
Tree *root_error = ts_tree_make_error_node(&children);
Tree *root_error = ts_tree_make_error_node(&children, self->language);
parser__push(self, 0, root_error, 0);
TSSymbolMetadata metadata = ts_language_symbol_metadata(self->language, ts_builtin_sym_end);
Tree *eof = ts_tree_make_leaf(ts_builtin_sym_end, length_zero(), length_zero(), metadata);
Tree *eof = ts_tree_make_leaf(ts_builtin_sym_end, length_zero(), length_zero(), self->language);
parser__accept(self, 0, eof);
ts_tree_release(eof);
}
@ -1123,7 +1108,7 @@ static void parser__recover(Parser *self, StackVersion version, TSStateId state,
if (lookahead->symbol == ts_builtin_sym_end) {
LOG("recover_eof");
TreeArray children = array_new();
Tree *parent = ts_tree_make_error_node(&children);
Tree *parent = ts_tree_make_error_node(&children, self->language);
parser__push(self, version, parent, 1);
parser__accept(self, version, lookahead);
return;
@ -1218,7 +1203,7 @@ static void parser__advance(Parser *self, StackVersion version,
LOG("reduce sym:%s, child_count:%u", SYM_NAME(action.params.symbol), action.params.child_count);
StackPopResult reduction = parser__reduce(
self, version, action.params.symbol, action.params.child_count,
action.params.dynamic_precedence, action.params.rename_sequence_id,
action.params.dynamic_precedence, action.params.alias_sequence_id,
action.params.fragile, true
);
StackSlice slice = *array_front(&reduction.slices);

2
src/runtime/reduce_action.h
View file

@ -12,7 +12,7 @@ typedef struct {
uint32_t count;
TSSymbol symbol;
int dynamic_precedence;
unsigned short rename_sequence_id;
unsigned short alias_sequence_id;
} ReduceAction;
typedef Array(ReduceAction) ReduceActionSet;

94
src/runtime/tree.c
View file

@ -42,23 +42,23 @@ bool ts_external_token_state_eq(const TSExternalTokenState *a, const TSExternalT
memcmp(ts_external_token_state_data(a), ts_external_token_state_data(b), a->length) == 0);
}
Tree *ts_tree_make_leaf(TSSymbol sym, Length padding, Length size,
TSSymbolMetadata metadata) {
Tree *ts_tree_make_leaf(TSSymbol symbol, Length padding, Length size, const TSLanguage *language) {
TSSymbolMetadata metadata = ts_language_symbol_metadata(language, symbol);
Tree *result = ts_malloc(sizeof(Tree));
*result = (Tree){
.ref_count = 1,
.symbol = sym,
.symbol = symbol,
.size = size,
.child_count = 0,
.visible_child_count = 0,
.named_child_count = 0,
.children = NULL,
.rename_sequence_id = 0,
.alias_sequence_id = 0,
.padding = padding,
.visible = metadata.visible,
.named = metadata.named,
.has_changes = false,
.first_leaf.symbol = sym,
.first_leaf.symbol = symbol,
.has_external_tokens = false,
};
return result;
@ -134,11 +134,9 @@ TreeArray ts_tree_array_remove_trailing_extras(TreeArray *self) {
return result;
}
Tree *ts_tree_make_error(Length size, Length padding, int32_t lookahead_char) {
Tree *result = ts_tree_make_leaf(ts_builtin_sym_error, padding, size,
(TSSymbolMetadata){
.visible = true, .named = true,
});
Tree *ts_tree_make_error(Length size, Length padding, int32_t lookahead_char,
const TSLanguage *language) {
Tree *result = ts_tree_make_leaf(ts_builtin_sym_error, padding, size, language);
result->fragile_left = true;
result->fragile_right = true;
result->lookahead_char = lookahead_char;
@ -158,7 +156,7 @@ void ts_tree_assign_parents(Tree *self, TreePath *path, const TSLanguage *langua
while (path->size > 0) {
Tree *tree = array_pop(path).tree;
Length offset = length_zero();
const TSSymbol *rename_sequence = ts_language_rename_sequence(language, tree->rename_sequence_id);
const TSSymbol *alias_sequence = ts_language_alias_sequence(language, tree->alias_sequence_id);
uint32_t non_extra_index = 0;
for (uint32_t i = 0; i < tree->child_count; i++) {
Tree *child = tree->children[i];
@ -166,10 +164,12 @@ void ts_tree_assign_parents(Tree *self, TreePath *path, const TSLanguage *langua
child->context.parent = tree;
child->context.index = i;
child->context.offset = offset;
if (!child->extra && rename_sequence && rename_sequence[non_extra_index] != 0) {
child->context.rename_symbol = rename_sequence[non_extra_index];
if (!child->extra && alias_sequence && alias_sequence[non_extra_index] != 0) {
TSSymbolMetadata metadata = ts_language_symbol_metadata(language, alias_sequence[non_extra_index]);
child->context.alias_symbol = alias_sequence[non_extra_index];
child->context.alias_is_named = metadata.named;
} else {
child->context.rename_symbol = 0;
child->context.alias_symbol = 0;
}
array_push(path, ((TreePathEntry){child, length_zero(), 0}));
}
@ -181,9 +181,8 @@ void ts_tree_assign_parents(Tree *self, TreePath *path, const TSLanguage *langua
void ts_tree_set_children(Tree *self, uint32_t child_count, Tree **children,
const TSSymbol *rename_sequence) {
if (self->child_count > 0)
ts_free(self->children);
const TSLanguage *language) {
if (self->child_count > 0) ts_free(self->children);
self->children = children;
self->child_count = child_count;
@ -194,6 +193,7 @@ void ts_tree_set_children(Tree *self, uint32_t child_count, Tree **children,
self->dynamic_precedence = 0;
uint32_t non_extra_index = 0;
const TSSymbol *alias_sequence = ts_language_alias_sequence(language, self->alias_sequence_id);
for (uint32_t i = 0; i < child_count; i++) {
Tree *child = children[i];
@ -211,9 +211,11 @@ void ts_tree_set_children(Tree *self, uint32_t child_count, Tree **children,
self->error_cost += child->error_cost;
self->dynamic_precedence += child->dynamic_precedence;
if (rename_sequence && rename_sequence[non_extra_index] != 0 && !child->extra) {
if (alias_sequence && alias_sequence[non_extra_index] != 0 && !child->extra) {
self->visible_child_count++;
self->named_child_count++;
if (ts_language_symbol_metadata(language, alias_sequence[non_extra_index]).named) {
self->named_child_count++;
}
} else if (child->visible) {
self->visible_child_count++;
if (child->named) self->named_child_count++;
@ -250,14 +252,14 @@ void ts_tree_set_children(Tree *self, uint32_t child_count, Tree **children,
}
Tree *ts_tree_make_node(TSSymbol symbol, uint32_t child_count, Tree **children,
TSSymbolMetadata metadata, const TSSymbol *rename_sequence) {
Tree *result =
ts_tree_make_leaf(symbol, length_zero(), length_zero(), metadata);
ts_tree_set_children(result, child_count, children, rename_sequence);
unsigned alias_sequence_id, const TSLanguage *language) {
Tree *result = ts_tree_make_leaf(symbol, length_zero(), length_zero(), language);
result->alias_sequence_id = alias_sequence_id;
ts_tree_set_children(result, child_count, children, language);
return result;
}
Tree *ts_tree_make_error_node(TreeArray *children) {
Tree *ts_tree_make_error_node(TreeArray *children, const TSLanguage *language) {
for (uint32_t i = 0; i < children->size; i++) {
Tree *child = children->contents[i];
if (child->symbol == ts_builtin_sym_error && child->child_count > 0) {
@ -269,9 +271,8 @@ Tree *ts_tree_make_error_node(TreeArray *children) {
}
}
Tree *result = ts_tree_make_node(
ts_builtin_sym_error, children->size, children->contents,
(TSSymbolMetadata){.extra = false, .visible = true, .named = true }, NULL);
Tree *result =
ts_tree_make_node(ts_builtin_sym_error, children->size, children->contents, 0, language);
result->fragile_left = true;
result->fragile_right = true;
@ -328,29 +329,26 @@ uint32_t ts_tree_end_column(const Tree *self) {
bool ts_tree_eq(const Tree *self, const Tree *other) {
if (self) {
if (!other)
return false;
if (!other) return false;
} else {
return !other;
}
if (self->symbol != other->symbol)
return false;
if (self->visible != other->visible)
return false;
if (self->named != other->named)
return false;
if (self->symbol == ts_builtin_sym_error)
return self->lookahead_char == other->lookahead_char;
if (self->child_count != other->child_count)
return false;
if (self->visible_child_count != other->visible_child_count)
return false;
if (self->named_child_count != other->named_child_count)
return false;
for (uint32_t i = 0; i < self->child_count; i++)
if (!ts_tree_eq(self->children[i], other->children[i]))
if (self->symbol != other->symbol) return false;
if (self->visible != other->visible) return false;
if (self->named != other->named) return false;
if (self->padding.bytes != other->padding.bytes) return false;
if (self->size.bytes != other->size.bytes) return false;
if (self->symbol == ts_builtin_sym_error) return self->lookahead_char == other->lookahead_char;
if (self->child_count != other->child_count) return false;
if (self->visible_child_count != other->visible_child_count) return false;
if (self->named_child_count != other->named_child_count) return false;
for (uint32_t i = 0; i < self->child_count; i++) {
if (!ts_tree_eq(self->children[i], other->children[i])) {
return false;
}
}
return true;
}
@ -528,7 +526,7 @@ static size_t ts_tree__write_to_string(const Tree *self,
include_all ||
is_root ||
(self->visible && self->named) ||
self->context.rename_symbol != 0;
self->context.alias_is_named;
if (visible && !is_root) {
cursor += snprintf(*writer, limit, " ");
@ -539,7 +537,7 @@ static size_t ts_tree__write_to_string(const Tree *self,
cursor += snprintf(*writer, limit, "(UNEXPECTED ");
cursor += ts_tree__write_char_to_string(*writer, limit, self->lookahead_char);
} else {
TSSymbol symbol = self->context.rename_symbol ? self->context.rename_symbol : self->symbol;
TSSymbol symbol = self->context.alias_symbol ? self->context.alias_symbol : self->symbol;
cursor += snprintf(*writer, limit, "(%s", ts_language_symbol_name(language, symbol));
}
}
@ -566,7 +564,7 @@ char *ts_tree_string(const Tree *self, const TSLanguage *language,
void ts_tree__print_dot_graph(const Tree *self, uint32_t byte_offset,
const TSLanguage *language, FILE *f) {
TSSymbol symbol = self->context.rename_symbol ? self->context.rename_symbol : self->symbol;
TSSymbol symbol = self->context.alias_symbol ? self->context.alias_symbol : self->symbol;
fprintf(f, "tree_%p [label=\"%s\"", self, ts_language_symbol_name(language, symbol));
if (self->child_count == 0)

15
src/runtime/tree.h
View file

@ -30,7 +30,8 @@ typedef struct Tree {
struct Tree *parent;
uint32_t index;
Length offset;
TSSymbol rename_symbol;
TSSymbol alias_symbol : 15;
bool alias_is_named : 1;
} context;
uint32_t child_count;
@ -39,7 +40,7 @@ typedef struct Tree {
struct Tree **children;
uint32_t visible_child_count;
uint32_t named_child_count;
unsigned short rename_sequence_id;
unsigned short alias_sequence_id;
};
TSExternalTokenState external_token_state;
int32_t lookahead_char;
@ -85,11 +86,11 @@ uint32_t ts_tree_array_essential_count(const TreeArray *);
TreeArray ts_tree_array_remove_last_n(TreeArray *, uint32_t);
TreeArray ts_tree_array_remove_trailing_extras(TreeArray *);
Tree *ts_tree_make_leaf(TSSymbol, Length, Length, TSSymbolMetadata);
Tree *ts_tree_make_node(TSSymbol, uint32_t, Tree **, TSSymbolMetadata, const TSSymbol *);
Tree *ts_tree_make_leaf(TSSymbol, Length, Length, const TSLanguage *);
Tree *ts_tree_make_node(TSSymbol, uint32_t, Tree **, unsigned, const TSLanguage *);
Tree *ts_tree_make_copy(Tree *child);
Tree *ts_tree_make_error_node(TreeArray *);
Tree *ts_tree_make_error(Length, Length, int32_t);
Tree *ts_tree_make_error_node(TreeArray *, const TSLanguage *);
Tree *ts_tree_make_error(Length, Length, int32_t, const TSLanguage *);
void ts_tree_retain(Tree *tree);
void ts_tree_release(Tree *tree);
bool ts_tree_eq(const Tree *tree1, const Tree *tree2);
@ -97,7 +98,7 @@ int ts_tree_compare(const Tree *tree1, const Tree *tree2);
uint32_t ts_tree_start_column(const Tree *self);
uint32_t ts_tree_end_column(const Tree *self);
void ts_tree_set_children(Tree *, uint32_t, Tree **, const TSSymbol *);
void ts_tree_set_children(Tree *, uint32_t, Tree **, const TSLanguage *);
void ts_tree_assign_parents(Tree *, TreePath *, const TSLanguage *);
void ts_tree_edit(Tree *, const TSInputEdit *edit);
char *ts_tree_string(const Tree *, const TSLanguage *, bool include_all);

22
src/runtime/tree_path.h
View file

@ -133,19 +133,19 @@ TreePathComparison tree_path_compare(const TreePath *old_path,
const TreePath *new_path,
const TSLanguage *language) {
Tree *old_tree = NULL;
TSSymbol old_rename_symbol = 0;
TSSymbol old_alias_symbol = 0;
Length old_start = length_zero();
for (uint32_t i = old_path->size - 1; i + 1 > 0; i--) {
old_tree = old_path->contents[i].tree;
if (old_tree->visible) {
old_start = old_path->contents[i].position;
if (i > 0) {
const TSSymbol *rename_sequence = ts_language_rename_sequence(
const TSSymbol *alias_sequence = ts_language_alias_sequence(
language,
old_path->contents[i - 1].tree->rename_sequence_id
old_path->contents[i - 1].tree->alias_sequence_id
);
if (rename_sequence) {
old_rename_symbol = rename_sequence[old_path->contents[i].child_index];
if (alias_sequence) {
old_alias_symbol = alias_sequence[old_path->contents[i].child_index];
}
}
break;
@ -153,26 +153,26 @@ TreePathComparison tree_path_compare(const TreePath *old_path,
}
Tree *new_tree = NULL;
TSSymbol new_rename_symbol = 0;
TSSymbol new_alias_symbol = 0;
Length new_start = length_zero();
for (uint32_t i = new_path->size - 1; i + 1 > 0; i--) {
new_tree = new_path->contents[i].tree;
if (new_tree->visible) {
new_start = old_path->contents[i].position;
if (i > 0) {
const TSSymbol *rename_sequence = ts_language_rename_sequence(
const TSSymbol *alias_sequence = ts_language_alias_sequence(
language,
new_path->contents[i - 1].tree->rename_sequence_id
new_path->contents[i - 1].tree->alias_sequence_id
);
if (rename_sequence) {
new_rename_symbol = rename_sequence[new_path->contents[i].child_index];
if (alias_sequence) {
new_alias_symbol = alias_sequence[new_path->contents[i].child_index];
}
}
break;
}
}
if (old_rename_symbol == new_rename_symbol) {
if (old_alias_symbol == new_alias_symbol) {
if (old_start.bytes == new_start.bytes) {
if (!old_tree->has_changes &&
old_tree->symbol == new_tree->symbol &&