traxys/tree-sitter
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Change edit API to be byte-based

Browse source
This commit is contained in:
Max Brunsfeld 2016-09-13 13:08:52 -07:00
parent cc62fe0375
commit 00528e50ce
11 changed files with 261 additions and 133 deletions
Download patch file Download diff file Expand all files Collapse all files

49
src/runtime/document.c
View file

@ -80,13 +80,13 @@ void ts_document_edit(TSDocument *self, TSInputEdit edit) {
if (!self->tree)
return;
size_t max_chars = ts_tree_total_chars(self->tree);
if (edit.position > max_chars)
edit.position = max_chars;
if (edit.chars_removed > max_chars - edit.position)
edit.chars_removed = max_chars - edit.position;
size_t max_bytes = ts_tree_total_bytes(self->tree);
if (edit.start_byte > max_bytes)
edit.start_byte = max_bytes;
if (edit.bytes_removed > max_bytes - edit.start_byte)
edit.bytes_removed = max_bytes - edit.start_byte;
ts_tree_edit(self->tree, edit);
ts_tree_edit(self->tree, &edit);
}
typedef Array(TSRange) RangeArray;
@ -107,18 +107,19 @@ static bool push_diff(RangeArray *results, TSNode *node, bool *extend_last_chang
return array_push(results, ((TSRange){start, end}));
}
static bool ts_tree_diff(TSDocument *doc, TSTree *old, TSNode *new_node,
size_t depth, RangeArray *results, bool *extend_last_change) {
static bool ts_tree_get_changes(TSDocument *doc, TSTree *old, TSNode *new_node,
size_t depth, RangeArray *results,
bool *extend_last_change) {
TSTree *new = (TSTree *)(new_node->data);
PRINT("At %lu, ('%s', %lu) vs ('%s', %lu) {",
ts_node_start_char(*new_node),
NAME(old), old->size.chars,
NAME(new), new->size.chars);
ts_node_start_byte(*new_node),
NAME(old), old->size.bytes,
NAME(new), new->size.bytes);
if (old->visible) {
if (old == new || (old->symbol == new->symbol &&
old->size.chars == new->size.chars && !old->has_changes)) {
old->size.bytes == new->size.bytes && !old->has_changes)) {
*extend_last_change = false;
PRINT("}", NULL);
return true;
@ -140,21 +141,21 @@ static bool ts_tree_diff(TSDocument *doc, TSTree *old, TSNode *new_node,
depth++;
size_t old_child_start;
size_t old_child_end = ts_node_start_char(*new_node) - old->padding.chars;
size_t old_child_end = ts_node_start_byte(*new_node) - old->padding.bytes;
for (size_t j = 0; j < old->child_count; j++) {
TSTree *old_child = old->children[j];
if (old_child->padding.chars == 0 && old_child->size.chars == 0)
if (old_child->padding.bytes == 0 && old_child->size.bytes == 0)
continue;
old_child_start = old_child_end + old_child->padding.chars;
old_child_end = old_child_start + old_child->size.chars;
old_child_start = old_child_end + old_child->padding.bytes;
old_child_end = old_child_start + old_child->size.bytes;
while (true) {
size_t new_child_start = ts_node_start_char(*new_node);
size_t new_child_start = ts_node_start_byte(*new_node);
if (new_child_start < old_child_start) {
PRINT("skip new:('%s', %lu), old:('%s', %lu), old_parent:%s",
NAME(new_node->data), ts_node_start_char(*new_node), NAME(old_child),
NAME(new_node->data), ts_node_start_byte(*new_node), NAME(old_child),
old_child_start, NAME(old));
if (!push_diff(results, new_node, extend_last_change))
@ -163,23 +164,23 @@ static bool ts_tree_diff(TSDocument *doc, TSTree *old, TSNode *new_node,
TSNode next = ts_node_next_sibling(*new_node);
if (next.data) {
PRINT("advance before diff ('%s', %lu) -> ('%s', %lu)",
NAME(new_node->data), ts_node_start_char(*new_node), NAME(next.data),
ts_node_start_char(next));
NAME(new_node->data), ts_node_start_byte(*new_node), NAME(next.data),
ts_node_start_byte(next));
*new_node = next;
} else {
break;
}
} else if (new_child_start == old_child_start) {
if (!ts_tree_diff(doc, old_child, new_node, depth, results, extend_last_change))
if (!ts_tree_get_changes(doc, old_child, new_node, depth, results, extend_last_change))
return false;
if (old_child->visible) {
TSNode next = ts_node_next_sibling(*new_node);
if (next.data) {
PRINT("advance after diff ('%s', %lu) -> ('%s', %lu)",
NAME(new_node->data), ts_node_start_char(*new_node), NAME(next.data),
ts_node_start_char(next));
NAME(new_node->data), ts_node_start_byte(*new_node), NAME(next.data),
ts_node_start_byte(next));
*new_node = next;
}
}
@ -225,7 +226,7 @@ int ts_document_parse_and_get_changed_ranges(TSDocument *self, TSRange **ranges,
if (ranges && range_count) {
bool extend_last_change = false;
RangeArray result = {0, 0, 0};
if (!ts_tree_diff(self, old_tree, &new_root, 0, &result, &extend_last_change))
if (!ts_tree_get_changes(self, old_tree, &new_root, 0, &result, &extend_last_change))
return -1;
*ranges = result.contents;
*range_count = result.size;

29
src/runtime/length.h
View file

@ -18,13 +18,19 @@ static inline TSPoint ts_point_sub(TSPoint a, TSPoint b) {
return (TSPoint){0, a.column - b.column};
}
static inline TSPoint ts_point_min(TSPoint a, TSPoint b) {
if (a.row < b.row || (a.row == b.row && a.column < b.column))
return a;
else
return b;
}
static inline bool ts_length_is_unknown(TSLength self) {
return self.chars > 0 && self.bytes == 0;
return self.bytes > 0 && self.chars == 0;
}
static inline void ts_length_set_unknown(TSLength *self) {
self->bytes = 0;
self->extent = (TSPoint){0, 0};
self->chars = 0;
}
static inline TSLength ts_length_min(TSLength len1, TSLength len2) {
@ -34,13 +40,13 @@ static inline TSLength ts_length_min(TSLength len1, TSLength len2) {
static inline TSLength ts_length_add(TSLength len1, TSLength len2) {
TSLength result;
result.chars = len1.chars + len2.chars;
result.bytes = len1.bytes + len2.bytes;
result.extent = ts_point_add(len1.extent, len2.extent);
if (ts_length_is_unknown(len1) || ts_length_is_unknown(len2)) {
result.bytes = 0;
result.extent = (TSPoint){0, result.chars};
result.chars = 0;
} else {
result.bytes = len1.bytes + len2.bytes;
result.extent = ts_point_add(len1.extent, len2.extent);
result.chars = len1.chars + len2.chars;
}
return result;
@ -48,14 +54,13 @@ static inline TSLength ts_length_add(TSLength len1, TSLength len2) {
static inline TSLength ts_length_sub(TSLength len1, TSLength len2) {
TSLength result;
result.chars = len1.chars - len2.chars;
result.bytes = len1.bytes - len2.bytes;
result.extent = ts_point_sub(len1.extent, len2.extent);
if (ts_length_is_unknown(len1) || ts_length_is_unknown(len2)) {
result.bytes = 0;
result.extent = (TSPoint){0, result.chars};
result.chars = 0;
} else {
result.bytes = len1.bytes - len2.bytes;
result.extent = ts_point_sub(len1.extent, len2.extent);
result.chars = len1.chars - len2.chars;
}
return result;

20
src/runtime/parser.c
View file

@ -144,7 +144,7 @@ error:
}
static void parser__pop_reusable_node(ReusableNode *reusable_node) {
reusable_node->char_index += ts_tree_total_chars(reusable_node->tree);
reusable_node->byte_index += ts_tree_total_bytes(reusable_node->tree);
while (reusable_node->tree) {
TSTree *parent = reusable_node->tree->context.parent;
size_t next_index = reusable_node->tree->context.index + 1;
@ -270,7 +270,7 @@ static TSTree *parser__lex(Parser *self, TSStateId parse_state) {
break;
}
if (self->lexer.current_position.chars == position.chars) {
if (self->lexer.current_position.bytes == position.bytes) {
if (!skipped_error) {
error_start_position = self->lexer.current_position;
first_error_character = self->lexer.lookahead;
@ -317,15 +317,15 @@ static TSTree *parser__get_lookahead(Parser *self, StackVersion version,
TSLength position = ts_stack_top_position(self->stack, version);
while (reusable_node->tree) {
if (reusable_node->char_index > position.chars) {
if (reusable_node->byte_index > position.bytes) {
LOG("before_reusable sym:%s, pos:%lu",
SYM_NAME(reusable_node->tree->symbol), reusable_node->char_index);
SYM_NAME(reusable_node->tree->symbol), reusable_node->byte_index);
break;
}
if (reusable_node->char_index < position.chars) {
if (reusable_node->byte_index < position.bytes) {
LOG("past_reusable sym:%s, pos:%lu",
SYM_NAME(reusable_node->tree->symbol), reusable_node->char_index);
SYM_NAME(reusable_node->tree->symbol), reusable_node->byte_index);
parser__pop_reusable_node(reusable_node);
continue;
}
@ -333,7 +333,7 @@ static TSTree *parser__get_lookahead(Parser *self, StackVersion version,
if (reusable_node->tree->has_changes) {
LOG("cant_reuse_changed tree:%s, size:%lu",
SYM_NAME(reusable_node->tree->symbol),
reusable_node->tree->size.chars);
reusable_node->tree->size.bytes);
if (!parser__breakdown_reusable_node(reusable_node)) {
parser__pop_reusable_node(reusable_node);
CHECK(parser__breakdown_top_of_stack(self, version));
@ -344,7 +344,7 @@ static TSTree *parser__get_lookahead(Parser *self, StackVersion version,
if (reusable_node->tree->symbol == ts_builtin_sym_error) {
LOG("cant_reuse_error tree:%s, size:%lu",
SYM_NAME(reusable_node->tree->symbol),
reusable_node->tree->size.chars);
reusable_node->tree->size.bytes);
if (!parser__breakdown_reusable_node(reusable_node)) {
parser__pop_reusable_node(reusable_node);
CHECK(parser__breakdown_top_of_stack(self, version));
@ -357,7 +357,7 @@ static TSTree *parser__get_lookahead(Parser *self, StackVersion version,
return result;
}
if (self->cached_token && position.chars == self->cached_token_char_index) {
if (self->cached_token && position.bytes == self->cached_token_byte_index) {
ts_tree_retain(self->cached_token);
return self->cached_token;
}
@ -1073,7 +1073,7 @@ static bool parser__advance(Parser *self, StackVersion version,
validated_lookahead = true;
LOG("lookahead sym:%s, size:%lu", SYM_NAME(lookahead->symbol),
lookahead->size.chars);
lookahead->size.bytes);
}
bool reduction_stopped_at_error = false;

4
src/runtime/parser.h
View file

@ -11,7 +11,7 @@ extern "C" {
typedef struct {
TSTree *tree;
size_t char_index;
size_t byte_index;
} ReusableNode;
typedef struct {
@ -24,7 +24,7 @@ typedef struct {
bool print_debugging_graphs;
TSTree scratch_tree;
TSTree *cached_token;
size_t cached_token_char_index;
size_t cached_token_byte_index;
ReusableNode reusable_node;
} Parser;

95
src/runtime/tree.c
View file

@ -307,34 +307,48 @@ static inline long min(long a, long b) {
return a <= b ? a : b;
}
void ts_tree_edit(TSTree *self, TSInputEdit edit) {
size_t start = edit.position;
size_t new_end = edit.position + edit.chars_inserted;
size_t old_end = edit.position + edit.chars_removed;
assert(old_end <= ts_tree_total_chars(self));
void ts_tree_edit(TSTree *self, const TSInputEdit *edit) {
size_t old_end_byte = edit->start_byte + edit->bytes_removed;
size_t new_end_byte = edit->start_byte + edit->bytes_added;
TSPoint old_end_point = ts_point_add(edit->start_point, edit->extent_removed);
TSPoint new_end_point = ts_point_add(edit->start_point, edit->extent_added);
assert(old_end_byte <= ts_tree_total_bytes(self));
self->has_changes = true;
if (start < self->padding.chars) {
if (edit->start_byte < self->padding.bytes) {
ts_length_set_unknown(&self->padding);
long remaining_padding = self->padding.chars - old_end;
if (remaining_padding >= 0) {
self->padding.chars = new_end + remaining_padding;
if (self->padding.bytes >= old_end_byte) {
size_t trailing_padding_bytes = self->padding.bytes - old_end_byte;
TSPoint trailing_padding_extent = ts_point_sub(self->padding.extent, old_end_point);
self->padding.bytes = new_end_byte + trailing_padding_bytes;
self->padding.extent = ts_point_add(new_end_point, trailing_padding_extent);
} else {
self->padding.chars = new_end;
self->size.chars += remaining_padding;
ts_length_set_unknown(&self->size);
size_t removed_content_bytes = old_end_byte - self->padding.bytes;
TSPoint removed_content_extent = ts_point_sub(old_end_point, self->padding.extent);
self->size.bytes = self->size.bytes - removed_content_bytes;
self->size.extent = ts_point_sub(self->size.extent, removed_content_extent);
self->padding.bytes = new_end_byte;
self->padding.extent = new_end_point;
}
} else if (start == self->padding.chars && edit.chars_removed == 0) {
self->padding.chars += edit.chars_inserted;
} else if (edit->start_byte == self->padding.bytes && edit->bytes_removed == 0) {
ts_length_set_unknown(&self->padding);
self->padding.bytes = self->padding.bytes + edit->bytes_added;
self->padding.extent = ts_point_add(self->padding.extent, edit->extent_added);
} else {
self->size.chars += (edit.chars_inserted - edit.chars_removed);
ts_length_set_unknown(&self->size);
size_t trailing_content_bytes = ts_tree_total_bytes(self) - old_end_byte;
TSPoint trailing_content_extent = ts_point_sub(ts_tree_total_extent(self), old_end_point);
self->size.bytes = new_end_byte + trailing_content_bytes - self->padding.bytes;
self->size.extent = ts_point_sub(ts_point_add(new_end_point, trailing_content_extent), self->padding.extent);
}
bool found_first_child = false;
long remainder_to_delete = edit.chars_removed - edit.chars_inserted;
long remaining_bytes_to_delete = 0;
TSPoint remaining_extent_to_delete = {0, 0};
TSLength child_left, child_right = ts_length_zero();
for (size_t i = 0; i < self->child_count; i++) {
TSTree *child = self->children[i];
@ -342,30 +356,41 @@ void ts_tree_edit(TSTree *self, TSInputEdit edit) {
if (!found_first_child) {
child_right = ts_length_add(child_left, ts_tree_total_size(child));
if (child_right.chars >= start) {
if (child_right.bytes >= edit->start_byte) {
found_first_child = true;
size_t chars_removed = min(edit.chars_removed, child_right.chars - start);
remainder_to_delete -= (chars_removed - edit.chars_inserted);
ts_tree_edit(child, (TSInputEdit){
.position = start - child_left.chars,
.chars_inserted = edit.chars_inserted,
.chars_removed = chars_removed,
});
child_right = ts_length_add(child_left, ts_tree_total_size(child));
TSInputEdit child_edit = {
.start_byte = edit->start_byte - child_left.bytes,
.bytes_added = edit->bytes_added,
.bytes_removed = edit->bytes_removed,
.start_point = ts_point_sub(edit->start_point, child_left.extent),
.extent_added = edit->extent_added,
.extent_removed = edit->extent_removed,
};
if (old_end_byte > child_right.bytes) {
child_edit.bytes_removed = child_right.bytes - edit->start_byte;
child_edit.extent_removed = ts_point_sub(child_right.extent, edit->start_point);
remaining_bytes_to_delete = old_end_byte - child_right.bytes;
remaining_extent_to_delete = ts_point_sub(old_end_point, child_right.extent);
}
ts_tree_edit(child, &child_edit);
}
} else {
if (remainder_to_delete > 0) {
size_t chars_removed = min(remainder_to_delete, ts_tree_total_chars(child));
remainder_to_delete -= chars_removed;
ts_tree_edit(
child,
(TSInputEdit){
.position = 0, .chars_inserted = 0, .chars_removed = chars_removed,
});
}
child_right = ts_length_add(child_right, ts_tree_total_size(child));
} else if (remaining_bytes_to_delete > 0) {
TSInputEdit child_edit = {
.start_byte = 0,
.bytes_added = 0,
.bytes_removed = min(remaining_bytes_to_delete, ts_tree_total_bytes(child)),
.start_point = {0, 0},
.extent_added = {0, 0},
.extent_removed = ts_point_min(remaining_extent_to_delete, ts_tree_total_size(child).extent),
};
remaining_bytes_to_delete -= child_edit.bytes_removed;
remaining_extent_to_delete = ts_point_sub(remaining_extent_to_delete, child_edit.extent_removed);
ts_tree_edit(child, &child_edit);
}
child_right = ts_length_add(child_left, ts_tree_total_size(child));
child->context.offset = child_left;
}
}

10
src/runtime/tree.h
View file

@ -68,7 +68,7 @@ size_t ts_tree_start_column(const TSTree *self);
size_t ts_tree_end_column(const TSTree *self);
void ts_tree_set_children(TSTree *, size_t, TSTree **);
void ts_tree_assign_parents(TSTree *);
void ts_tree_edit(TSTree *, TSInputEdit);
void ts_tree_edit(TSTree *, const TSInputEdit *edit);
char *ts_tree_string(const TSTree *, const TSLanguage *, bool include_all);
void ts_tree_print_dot_graph(const TSTree *, const TSLanguage *, FILE *);
@ -76,10 +76,18 @@ static inline size_t ts_tree_total_chars(const TSTree *self) {
return self->padding.chars + self->size.chars;
}
static inline size_t ts_tree_total_bytes(const TSTree *self) {
return self->padding.bytes + self->size.bytes;
}
static inline TSLength ts_tree_total_size(const TSTree *self) {
return ts_length_add(self->padding, self->size);
}
static inline TSPoint ts_tree_total_extent(const TSTree *self) {
return ts_point_add(self->padding.extent, self->size.extent);
}
static inline bool ts_tree_is_fragile(const TSTree *tree) {
return tree->fragile_left || tree->fragile_right ||
ts_tree_total_chars(tree) == 0;