Merge pull request #9 from joshvera/line-numbers

Track row/column ranges in TSNodes
This commit is contained in:
Max Brunsfeld 2015-12-02 14:54:47 -08:00
commit b7decfa52e
18 changed files with 348 additions and 76 deletions

View file

@ -38,6 +38,10 @@ typedef struct TSLexer {
TSLength token_end_position;
TSLength token_start_position;
TSPoint current_point;
TSPoint token_end_point;
TSPoint token_start_point;
size_t lookahead_size;
int32_t lookahead;

View file

@ -35,9 +35,15 @@ typedef struct {
size_t chars_removed;
} TSInputEdit;
typedef struct {
size_t row;
size_t column;
} TSPoint;
typedef struct {
const void *data;
TSLength offset;
size_t row;
} TSNode;
typedef unsigned short TSSymbol;
@ -46,6 +52,9 @@ typedef struct TSDocument TSDocument;
TSLength ts_node_pos(TSNode);
TSLength ts_node_size(TSNode);
TSPoint ts_node_size_point(TSNode);
TSPoint ts_node_start_point(TSNode);
TSPoint ts_node_end_point(TSNode);
TSSymbol ts_node_symbol(TSNode);
const char *ts_node_name(TSNode, const TSDocument *);
const char *ts_node_string(TSNode, const TSDocument *);

View file

@ -68,10 +68,10 @@ function run_tests {
;;
debug)
if which -s gdb; then
gdb $cmd -- "${args[@]}"
elif which -s lldb; then
if which -s lldb; then
lldb $cmd -- "${args[@]}"
elif which -s gdb; then
gdb $cmd -- "${args[@]}"
else
echo "No debugger found"
exit 1

View file

@ -0,0 +1,24 @@
#include <string>
#include <ostream>
#include "runtime/length.h"
using namespace std;
bool operator==(const TSPoint &left, const TSPoint &right) {
return left.row == right.row && left.column == right.column;
}
std::ostream &operator<<(std::ostream &stream, const TSPoint &point) {
return stream << "{" << point.row << ", " << point.column << "}";
}
bool operator<(const TSPoint &left, const TSPoint &right) {
if (left.row < right.row) return true;
if (left.row > right.row) return false;
return left.column < right.column;
}
bool operator>(const TSPoint &left, const TSPoint &right) {
return right < left;
}

View file

@ -0,0 +1,8 @@
bool operator==(const TSPoint &left, const TSPoint &right);
bool operator<(const TSPoint &left, const TSPoint &right);
bool operator>(const TSPoint &left, const TSPoint &right);
std::ostream &operator<<(std::ostream &stream, const TSPoint &point);

View file

@ -4,6 +4,7 @@
#include "runtime/helpers/read_test_entries.h"
#include "runtime/helpers/spy_input.h"
#include "runtime/helpers/log_debugger.h"
#include "runtime/helpers/point_helpers.h"
extern "C" const TSLanguage *ts_language_javascript();
extern "C" const TSLanguage *ts_language_json();
@ -23,6 +24,9 @@ void expect_a_consistent_tree(TSNode node, TSDocument *doc) {
TSLength end = ts_length_add(start, ts_node_size(node));
size_t child_count = ts_node_child_count(node);
TSPoint start_point = ts_node_start_point(node);
TSPoint end_point = ts_node_end_point(node);
bool has_changes = ts_node_has_changes(node);
bool some_child_has_changes = false;
@ -31,8 +35,15 @@ void expect_a_consistent_tree(TSNode node, TSDocument *doc) {
TSLength child_start = ts_node_pos(child);
TSLength child_end = ts_length_add(child_start, ts_node_size(child));
TSPoint child_start_point = ts_node_start_point(child);
TSPoint child_end_point = ts_node_end_point(child);
AssertThat(child_start.chars, IsGreaterThan(start.chars) || Equals(start.chars));
AssertThat(child_end.chars, IsLessThan(end.chars) || Equals(end.chars));
AssertThat(child_start_point, IsGreaterThan(start_point) || Equals(start_point));
AssertThat(child_end_point, IsLessThan(end_point) || Equals(end_point));
if (ts_node_has_changes(child))
some_child_has_changes = true;
}

View file

@ -1,5 +1,6 @@
#include "runtime/runtime_spec_helper.h"
#include "runtime/helpers/tree_helpers.h"
#include "runtime/helpers/point_helpers.h"
extern "C" TSLanguage * ts_language_json();
@ -8,11 +9,23 @@ START_TEST
describe("Node", []() {
TSDocument *document;
TSNode array_node;
string input_string;
before_each([&]() {
document = ts_document_make();
ts_document_set_language(document, ts_language_json());
ts_document_set_input_string(document, " [123, false, {\"x\": null}]");
input_string = "\n"
"\n"
"[\n"
" 123,\n"
" false,\n"
" {\n"
" \"x\": null\n"
" }\n"
"]";
ts_document_set_input_string(document, input_string.c_str());
ts_document_parse(document);
array_node = ts_document_root_node(document);
@ -27,7 +40,7 @@ describe("Node", []() {
ts_document_free(document);
});
describe("child_count(), child(i)", [&]() {
describe("named_child_count(), named_child(i)", [&]() {
it("returns the named child node at the given index", [&]() {
AssertThat(ts_node_named_child_count(array_node), Equals<size_t>(3));
@ -41,16 +54,28 @@ describe("Node", []() {
AssertThat(ts_node_name(child3, document), Equals("object"));
AssertThat(ts_node_pos(array_node).bytes, Equals<size_t>(2));
AssertThat(ts_node_size(array_node).bytes, Equals<size_t>(25));
AssertThat(ts_node_size(array_node).bytes, Equals<size_t>(41));
AssertThat(ts_node_pos(child1).bytes, Equals<size_t>(3));
AssertThat(ts_node_start_point(array_node), Equals<TSPoint>({ 2, 0 }));
AssertThat(ts_node_end_point(array_node), Equals<TSPoint>({ 8, 1 }));
AssertThat(ts_node_pos(child1).bytes, Equals(input_string.find("123")));
AssertThat(ts_node_size(child1).bytes, Equals<size_t>(3));
AssertThat(ts_node_pos(child2).bytes, Equals<size_t>(8));
AssertThat(ts_node_start_point(child1), Equals<TSPoint>({ 3, 2 }));
AssertThat(ts_node_end_point(child1), Equals<TSPoint>({ 3, 5 }));
AssertThat(ts_node_pos(child2).bytes, Equals(input_string.find("false")));
AssertThat(ts_node_size(child2).bytes, Equals<size_t>(5));
AssertThat(ts_node_pos(child3).bytes, Equals<size_t>(15));
AssertThat(ts_node_size(child3).bytes, Equals<size_t>(11));
AssertThat(ts_node_start_point(child2), Equals<TSPoint>({ 4, 2 }));
AssertThat(ts_node_end_point(child2), Equals<TSPoint>({ 4, 7 }));
AssertThat(ts_node_pos(child3).bytes, Equals(input_string.find("{")));
AssertThat(ts_node_size(child3).bytes, Equals<size_t>(19));
AssertThat(ts_node_start_point(child3), Equals<TSPoint>({ 5, 2 }));
AssertThat(ts_node_end_point(child3), Equals<TSPoint>({ 7, 3 }));
AssertThat(ts_node_named_child_count(child3), Equals<size_t>(2));
@ -60,6 +85,9 @@ describe("Node", []() {
AssertThat(ts_node_name(grandchild1, document), Equals("string"));
AssertThat(ts_node_name(grandchild2, document), Equals("null"));
AssertThat(ts_node_start_point(grandchild1), Equals<TSPoint>({ 6, 4 }));
AssertThat(ts_node_end_point(grandchild1), Equals<TSPoint>({ 6, 7 }));
AssertThat(ts_node_parent(child1), Equals(array_node));
AssertThat(ts_node_parent(child2), Equals(array_node));
AssertThat(ts_node_parent(child3), Equals(array_node));
@ -67,7 +95,7 @@ describe("Node", []() {
});
});
describe("concrete_child_count(), concrete_child(i)", [&]() {
describe("child_count(), child(i)", [&]() {
it("returns the child node at the given index, including anonymous nodes", [&]() {
AssertThat(ts_node_child_count(array_node), Equals<size_t>(7));
TSNode child1 = ts_node_child(array_node, 0);
@ -99,15 +127,27 @@ describe("Node", []() {
AssertThat(ts_node_pos(child1).bytes, Equals<size_t>(2));
AssertThat(ts_node_size(child1).bytes, Equals<size_t>(1));
AssertThat(ts_node_pos(child3).bytes, Equals<size_t>(6));
AssertThat(ts_node_start_point(child1), Equals<TSPoint>({ 2, 0 }));
AssertThat(ts_node_end_point(child1), Equals<TSPoint>({ 2, 1 }));
AssertThat(ts_node_pos(child3).bytes, Equals<size_t>(9));
AssertThat(ts_node_size(child3).bytes, Equals<size_t>(1));
AssertThat(ts_node_pos(child5).bytes, Equals<size_t>(13));
AssertThat(ts_node_start_point(child3), Equals<TSPoint>({ 3, 5 }));
AssertThat(ts_node_end_point(child3), Equals<TSPoint>({ 3, 6 }));
AssertThat(ts_node_pos(child5).bytes, Equals<size_t>(18));
AssertThat(ts_node_size(child5).bytes, Equals<size_t>(1));
AssertThat(ts_node_pos(child7).bytes, Equals<size_t>(26));
AssertThat(ts_node_start_point(child5), Equals<TSPoint>({ 4, 7 }));
AssertThat(ts_node_end_point(child5), Equals<TSPoint>({ 4, 8 }));
AssertThat(ts_node_pos(child7).bytes, Equals<size_t>(42));
AssertThat(ts_node_size(child7).bytes, Equals<size_t>(1));
AssertThat(ts_node_start_point(child7), Equals<TSPoint>({ 8, 0 }));
AssertThat(ts_node_end_point(child7), Equals<TSPoint>({ 8, 1 }));
AssertThat(ts_node_child_count(child6), Equals<size_t>(5))
TSNode grandchild1 = ts_node_child(child6, 0);
@ -133,7 +173,7 @@ describe("Node", []() {
});
});
describe("next_concrete_sibling(), prev_concrete_sibling()", [&]() {
describe("next_sibling(), prev_sibling()", [&]() {
it("returns the node's next and previous sibling, including anonymous nodes", [&]() {
TSNode bracket_node1 = ts_node_child(array_node, 0);
TSNode number_node = ts_node_child(array_node, 1);
@ -185,7 +225,7 @@ describe("Node", []() {
});
});
describe("next_concrete_sibling(), prev_concrete_sibling()", [&]() {
describe("next_named_sibling(), prev_concrete_sibling()", [&]() {
it("returns the node's next and previous siblings", [&]() {
TSNode number_node = ts_node_named_child(array_node, 0);
TSNode false_node = ts_node_named_child(array_node, 1);
@ -212,60 +252,89 @@ describe("Node", []() {
describe("find_for_range(start, end)", [&]() {
describe("when there is a leaf node that spans the given range exactly", [&]() {
it("returns that leaf node", [&]() {
TSNode leaf = ts_node_named_descendent_for_range(array_node, 16, 18);
TSNode leaf = ts_node_named_descendent_for_range(array_node, 28, 30);
AssertThat(ts_node_name(leaf, document), Equals("string"));
AssertThat(ts_node_size(leaf).bytes, Equals<size_t>(3));
AssertThat(ts_node_pos(leaf).bytes, Equals<size_t>(16));
AssertThat(ts_node_pos(leaf).bytes, Equals<size_t>(28));
leaf = ts_node_named_descendent_for_range(array_node, 3, 5);
size_t index = input_string.find("123");
leaf = ts_node_named_descendent_for_range(array_node, index, index + 2);
AssertThat(ts_node_name(leaf, document), Equals("number"));
AssertThat(ts_node_pos(leaf).bytes, Equals(index));
AssertThat(ts_node_size(leaf).bytes, Equals<size_t>(3));
AssertThat(ts_node_pos(leaf).bytes, Equals<size_t>(3));
AssertThat(ts_node_start_point(leaf), Equals<TSPoint>({ 3, 2 }));
AssertThat(ts_node_end_point(leaf), Equals<TSPoint>({ 3, 5 }));
});
});
describe("when there is a leaf node that extends beyond the given range", [&]() {
it("returns that leaf node", [&]() {
TSNode leaf = ts_node_named_descendent_for_range(array_node, 16, 17);
size_t index = input_string.find("\"x\"");
TSNode leaf = ts_node_named_descendent_for_range(array_node, index, index + 1);
AssertThat(ts_node_name(leaf, document), Equals("string"));
AssertThat(ts_node_pos(leaf).bytes, Equals(index));
AssertThat(ts_node_size(leaf).bytes, Equals<size_t>(3));
AssertThat(ts_node_pos(leaf).bytes, Equals<size_t>(16));
leaf = ts_node_named_descendent_for_range(array_node, 17, 18);
AssertThat(ts_node_start_point(leaf), Equals<TSPoint>({ 6, 4 }));
AssertThat(ts_node_end_point(leaf), Equals<TSPoint>({ 6, 7 }));
leaf = ts_node_named_descendent_for_range(array_node, index + 1, index + 2);
AssertThat(ts_node_name(leaf, document), Equals("string"));
AssertThat(ts_node_pos(leaf).bytes, Equals<size_t>(index));
AssertThat(ts_node_size(leaf).bytes, Equals<size_t>(3));
AssertThat(ts_node_pos(leaf).bytes, Equals<size_t>(16));
AssertThat(ts_node_start_point(leaf), Equals<TSPoint>({ 6, 4 }));
AssertThat(ts_node_end_point(leaf), Equals<TSPoint>({ 6, 7 }));
});
});
describe("when there is no leaf node that spans the given range", [&]() {
it("returns the smallest node that does span the range", [&]() {
TSNode node = ts_node_named_descendent_for_range(array_node, 16, 19);
size_t index = input_string.find("\"x\"");
TSNode node = ts_node_named_descendent_for_range(array_node, index, index + 3);
AssertThat(ts_node_name(node, document), Equals("object"));
AssertThat(ts_node_size(node).bytes, Equals<size_t>(11));
AssertThat(ts_node_pos(node).bytes, Equals<size_t>(15));
size_t object_index = input_string.find("{");
AssertThat(ts_node_pos(node).bytes, Equals<size_t>(object_index));
AssertThat(ts_node_size(node).bytes, Equals<size_t>(19));
AssertThat(ts_node_start_point(node), Equals<TSPoint>({ 5, 2 }));
AssertThat(ts_node_end_point(node), Equals<TSPoint>({ 7, 3 }));
});
it("does not return invisible nodes (repeats)", [&]() {
TSNode node = ts_node_named_descendent_for_range(array_node, 6, 7);
size_t comma_index = input_string.find(",");
TSNode node = ts_node_named_descendent_for_range(array_node, comma_index, comma_index + 1);
AssertThat(ts_node_name(node, document), Equals("array"));
AssertThat(ts_node_size(node).bytes, Equals<size_t>(25));
AssertThat(ts_node_pos(node).bytes, Equals<size_t>(2));
size_t array_index = input_string.find("[");
AssertThat(ts_node_pos(node).bytes, Equals<size_t>(array_index));
AssertThat(ts_node_size(node).bytes, Equals<size_t>(41));
AssertThat(ts_node_start_point(node), Equals<TSPoint>({ 2, 0 }));
AssertThat(ts_node_end_point(node), Equals<TSPoint>({ 8, 1 }));
});
});
});
describe("find_concrete_for_range(start, end)", [&]() {
describe("descendent_for_range(start, end)", [&]() {
it("returns the smallest concrete node that spans the given range", [&]() {
TSNode node1 = ts_node_descendent_for_range(array_node, 19, 19);
size_t colon_index = input_string.find(":");
TSNode node1 = ts_node_descendent_for_range(array_node, colon_index, colon_index);
AssertThat(ts_node_name(node1, document), Equals(":"));
AssertThat(ts_node_pos(node1).bytes, Equals<size_t>(19));
AssertThat(ts_node_pos(node1).bytes, Equals<size_t>(colon_index));
AssertThat(ts_node_size(node1).bytes, Equals<size_t>(1));
AssertThat(ts_node_start_point(node1), Equals<TSPoint>({ 6, 7 }));
AssertThat(ts_node_end_point(node1), Equals<TSPoint>({ 6, 8 }));
TSNode node2 = ts_node_descendent_for_range(array_node, 18, 20);
size_t index = input_string.find("\":");
TSNode node2 = ts_node_descendent_for_range(array_node, index, index + 2);
AssertThat(ts_node_name(node2, document), Equals("object"));
AssertThat(ts_node_pos(node2).bytes, Equals<size_t>(15));
AssertThat(ts_node_size(node2).bytes, Equals<size_t>(11));
size_t object_index = input_string.find("{");
AssertThat(ts_node_pos(node2).bytes, Equals<size_t>(object_index));
AssertThat(ts_node_size(node2).bytes, Equals<size_t>(19));
AssertThat(ts_node_start_point(node2), Equals<TSPoint>({ 5, 2 }));
AssertThat(ts_node_end_point(node2), Equals<TSPoint>({ 7, 3 }));
});
});
});

View file

@ -44,7 +44,7 @@ describe("Stack", [&]() {
TSLength len = ts_length_make(2, 2);
for (size_t i = 0; i < tree_count; i++)
trees[i] = ts_tree_make_leaf(i, len, len, TSNodeTypeNamed);
trees[i] = ts_tree_make_leaf(i, len, len, {1, 1}, {1, 2}, TSNodeTypeNamed);
});
after_each([&]() {

View file

@ -29,8 +29,8 @@ describe("Tree", []() {
TSTree *tree1, *tree2, *parent1;
before_each([&]() {
tree1 = ts_tree_make_leaf(cat, {2, 1}, {5, 4}, TSNodeTypeNamed);
tree2 = ts_tree_make_leaf(cat, {1, 1}, {3, 3}, TSNodeTypeNamed);
tree1 = ts_tree_make_leaf(cat, {2, 1}, {5, 4}, ts_point_zero(), ts_point_zero(), TSNodeTypeNamed);
tree2 = ts_tree_make_leaf(cat, {1, 1}, {3, 3}, ts_point_zero(), ts_point_zero(), TSNodeTypeNamed);
parent1 = ts_tree_make_node(dog, 2, tree_array({
tree1,
tree2,
@ -55,6 +55,8 @@ describe("Tree", []() {
TSTree *error_tree = ts_tree_make_error(
ts_length_zero(),
ts_length_zero(),
ts_point_zero(),
ts_point_zero(),
'z');
AssertThat(ts_tree_is_fragile_left(error_tree), IsTrue());
@ -145,9 +147,9 @@ describe("Tree", []() {
before_each([&]() {
tree = ts_tree_make_node(cat, 3, tree_array({
ts_tree_make_leaf(dog, {2, 2}, {3, 3}, TSNodeTypeNamed),
ts_tree_make_leaf(eel, {2, 2}, {3, 3}, TSNodeTypeNamed),
ts_tree_make_leaf(fox, {2, 2}, {3, 3}, TSNodeTypeNamed),
ts_tree_make_leaf(dog, {2, 2}, {3, 3}, {1, 2}, {1, 3}, TSNodeTypeNamed),
ts_tree_make_leaf(eel, {2, 2}, {3, 3}, {1, 2}, {1, 3}, TSNodeTypeNamed),
ts_tree_make_leaf(fox, {2, 2}, {3, 3}, {1, 2}, {1, 3}, TSNodeTypeNamed),
}), TSNodeTypeNamed);
AssertThat(tree->padding, Equals<TSLength>({2, 2}));
@ -266,10 +268,10 @@ describe("Tree", []() {
describe("equality", [&]() {
it("returns true for identical trees", [&]() {
TSTree *tree1_copy = ts_tree_make_leaf(cat, {2, 1}, {5, 4}, TSNodeTypeNamed);
TSTree *tree1_copy = ts_tree_make_leaf(cat, {2, 1}, {5, 4}, {1, 1}, {1, 4}, TSNodeTypeNamed);
AssertThat(ts_tree_eq(tree1, tree1_copy), IsTrue());
TSTree *tree2_copy = ts_tree_make_leaf(cat, {1, 1}, {3, 3}, TSNodeTypeNamed);
TSTree *tree2_copy = ts_tree_make_leaf(cat, {1, 1}, {3, 3}, {1, 1}, {1, 3}, TSNodeTypeNamed);
AssertThat(ts_tree_eq(tree2, tree2_copy), IsTrue());
TSTree *parent2 = ts_tree_make_node(dog, 2, tree_array({
@ -289,6 +291,8 @@ describe("Tree", []() {
tree1->symbol + 1,
tree1->padding,
tree1->size,
tree1->padding_point,
tree1->size_point,
TSNodeTypeNamed);
AssertThat(ts_tree_eq(tree1, different_tree), IsFalse());
@ -300,6 +304,8 @@ describe("Tree", []() {
tree1->symbol + 1,
tree1->padding,
tree1->size,
tree1->padding_point,
tree1->size_point,
TSNodeTypeNamed);
TSTree *different_parent = ts_tree_make_node(dog, 2, tree_array({

View file

@ -85,7 +85,7 @@ void ts_document_invalidate(TSDocument *self) {
}
TSNode ts_document_root_node(const TSDocument *self) {
TSNode result = ts_node_make(self->tree, ts_length_zero());
TSNode result = ts_node_make(self->tree, ts_length_zero(), 0);
while (result.data && !ts_tree_is_visible(result.data))
result = ts_node_named_child(result, 0);
return result;

View file

@ -26,6 +26,39 @@ static inline TSLength ts_length_sub(TSLength len1, TSLength len2) {
return result;
}
static inline TSPoint ts_point_make(size_t row, size_t column) {
TSPoint point;
point.row = row;
point.column = column;
return point;
}
static inline TSPoint ts_point_add(TSPoint point1, TSPoint point2) {
size_t row = point1.row + point2.row;
size_t column;
if (point2.row == 0) {
column = point1.column + point2.column;
} else {
column = point2.column;
}
return ts_point_make(row, column);
}
static inline TSPoint ts_point_sub(TSPoint point1, TSPoint point2) {
size_t row, column;
if (point1.row == point2.row) {
row = 0;
column = point1.column - point2.column;
} else {
row = point1.row - point2.row;
column = point1.column;
}
return ts_point_make(row, column);
}
static inline TSLength ts_length_zero() {
TSLength result;
result.bytes = result.chars = 0;
@ -36,6 +69,12 @@ static inline bool ts_length_eq(TSLength len1, TSLength len2) {
return len1.bytes == len2.bytes && len1.chars == len2.chars;
}
static inline TSPoint ts_point_zero() {
TSPoint point;
point.row = point.column = 0;
return point;
}
static inline TSLength ts_length_make(size_t bytes, size_t chars) {
TSLength result;
result.bytes = bytes;

View file

@ -51,8 +51,10 @@ static void ts_lexer__start(TSLexer *self, TSStateId lex_state) {
}
static void ts_lexer__start_token(TSLexer *self) {
LOG("start_token chars:%lu", self->current_position.chars);
LOG("start_token chars:%lu, rows:%lu, columns:%lu", self->current_position.chars, self->current_point.row, self->current_point.column);
self->token_start_position = self->current_position;
self->token_start_point = self->current_point;
}
static bool ts_lexer__advance(TSLexer *self, TSStateId state) {
@ -64,6 +66,13 @@ static bool ts_lexer__advance(TSLexer *self, TSStateId state) {
if (self->lookahead_size) {
self->current_position.bytes += self->lookahead_size;
self->current_position.chars += 1;
if (self->lookahead == '\n') {
self->current_point.row += 1;
self->current_point.column = 0;
} else {
self->current_point.column += 1;
}
}
if (self->current_position.bytes >= self->chunk_start + self->chunk_size)
@ -81,12 +90,16 @@ static TSTree *ts_lexer__accept(TSLexer *self, TSSymbol symbol,
ts_length_sub(self->token_start_position, self->token_end_position);
self->token_end_position = self->current_position;
TSPoint size_point = ts_point_sub(self->current_point, self ->token_start_point);
TSPoint padding_point = ts_point_sub(self->token_start_point, self->token_end_point);
self->token_end_point = self->current_point;
if (symbol == ts_builtin_sym_error) {
LOG("error_char");
return ts_tree_make_error(size, padding, self->lookahead);
return ts_tree_make_error(size, padding, size_point, padding_point, self->lookahead);
} else {
LOG("accept_token sym:%s", symbol_name);
return ts_tree_make_leaf(symbol, padding, size, node_type);
return ts_tree_make_leaf(symbol, padding, size, padding_point, size_point, node_type);
}
}
@ -105,17 +118,22 @@ TSLexer ts_lexer_make() {
.chunk_start = 0,
.debugger = ts_debugger_null(),
};
ts_lexer_reset(&result, ts_length_zero());
ts_lexer_reset(&result, ts_length_zero(), ts_point_zero());
return result;
}
void ts_lexer_reset(TSLexer *self, TSLength position) {
void ts_lexer_reset(TSLexer *self, TSLength position, TSPoint point) {
if (ts_length_eq(position, self->current_position))
return;
self->token_start_position = position;
self->token_end_position = position;
self->current_position = position;
self->token_start_point = point;
self->token_end_point = point;
self->current_point = point;
self->chunk = 0;
self->chunk_start = 0;
self->chunk_size = 0;

View file

@ -8,7 +8,7 @@ extern "C" {
#include "tree_sitter/parser.h"
TSLexer ts_lexer_make();
void ts_lexer_reset(TSLexer *, TSLength);
void ts_lexer_reset(TSLexer *, TSLength, TSPoint);
#ifdef __cplusplus
}

View file

@ -4,8 +4,8 @@
#include "runtime/tree.h"
#include "runtime/document.h"
TSNode ts_node_make(const TSTree *tree, TSLength offset) {
return (TSNode){.data = tree, .offset = offset };
TSNode ts_node_make(const TSTree *tree, TSLength offset, size_t row) {
return (TSNode){.data = tree, .offset = offset, .row = row };
}
/*
@ -13,7 +13,7 @@ TSNode ts_node_make(const TSTree *tree, TSLength offset) {
*/
static inline TSNode ts_node__null() {
return ts_node_make(NULL, ts_length_zero());
return ts_node_make(NULL, ts_length_zero(), 0);
}
static inline const TSTree *ts_node__tree(TSNode self) {
@ -24,10 +24,12 @@ static inline TSLength ts_node__offset(TSNode self) {
return self.offset;
}
static inline TSNode ts_node__child(TSNode self, size_t child_index,
TSNodeType type) {
const TSTree *tree = ts_node__tree(self);
TSLength position = ts_node__offset(self);
size_t offset_row = self.row;
bool did_descend = true;
while (did_descend) {
@ -38,7 +40,7 @@ static inline TSNode ts_node__child(TSNode self, size_t child_index,
TSTree *child = tree->children[i];
if (child->options.type >= type) {
if (index == child_index)
return ts_node_make(child, position);
return ts_node_make(child, position, offset_row);
index++;
} else {
size_t grandchild_index = child_index - index;
@ -54,6 +56,7 @@ static inline TSNode ts_node__child(TSNode self, size_t child_index,
index += grandchild_count;
}
position = ts_length_add(position, ts_tree_total_size(child));
offset_row += child->padding_point.row + child->size_point.row;
}
}
@ -62,25 +65,28 @@ static inline TSNode ts_node__child(TSNode self, size_t child_index,
static inline TSNode ts_node__prev_sibling(TSNode self, TSNodeType type) {
const TSTree *tree = ts_node__tree(self);
TSLength position = ts_node__offset(self);
TSLength offset = ts_node__offset(self);
size_t offset_row = self.row;
do {
size_t index = tree->context.index;
position = ts_length_sub(position, tree->context.offset);
offset = ts_length_sub(offset, tree->context.offset);
offset_row -= tree->context.offset_point.row;
tree = tree->context.parent;
if (!tree)
break;
for (size_t i = index - 1; i + 1 > 0; i--) {
const TSTree *child = tree->children[i];
TSLength child_position = ts_length_add(position, child->context.offset);
TSLength child_offset = ts_length_add(offset, child->context.offset);
size_t child_offset_row = offset_row + child->context.offset_point.row;
if (child->options.type >= type)
return ts_node_make(child, child_position);
return ts_node_make(child, child_offset, child_offset_row);
size_t grandchild_count = (type == TSNodeTypeNamed)
? child->named_child_count
: child->visible_child_count;
if (grandchild_count > 0)
return ts_node__child(ts_node_make(child, child_position),
return ts_node__child(ts_node_make(child, child_offset, child_offset_row),
grandchild_count - 1, type);
}
} while (!ts_tree_is_visible(tree));
@ -90,25 +96,28 @@ static inline TSNode ts_node__prev_sibling(TSNode self, TSNodeType type) {
static inline TSNode ts_node__next_sibling(TSNode self, TSNodeType type) {
const TSTree *tree = ts_node__tree(self);
TSLength position = ts_node__offset(self);
TSLength offset = ts_node__offset(self);
size_t offset_row = self.row;
do {
size_t index = tree->context.index;
position = ts_length_sub(position, tree->context.offset);
offset = ts_length_sub(offset, tree->context.offset);
offset_row -= tree->context.offset_point.row;
tree = tree->context.parent;
if (!tree)
break;
for (size_t i = index + 1; i < tree->child_count; i++) {
const TSTree *child = tree->children[i];
TSLength child_position = ts_length_add(position, child->context.offset);
TSLength child_offset = ts_length_add(offset, child->context.offset);
size_t child_offset_row = offset_row + child->context.offset_point.row;
if (child->options.type >= type)
return ts_node_make(child, child_position);
return ts_node_make(child, child_offset, child_offset_row);
size_t grandchild_count = (type == TSNodeTypeNamed)
? child->named_child_count
: child->visible_child_count;
if (grandchild_count > 0)
return ts_node__child(ts_node_make(child, child_position), 0, type);
return ts_node__child(ts_node_make(child, child_offset, child_offset_row), 0, type);
}
} while (!ts_tree_is_visible(tree));
@ -119,6 +128,7 @@ static inline TSNode ts_node__descendent_for_range(TSNode self, size_t min,
size_t max, TSNodeType type) {
const TSTree *tree = ts_node__tree(self), *last_visible_tree = tree;
TSLength position = ts_node__offset(self), last_visible_position = position;
size_t offset_row = self.row, last_visible_row = offset_row;
bool did_descend = true;
while (did_descend) {
@ -133,15 +143,17 @@ static inline TSNode ts_node__descendent_for_range(TSNode self, size_t min,
if (child->options.type >= type) {
last_visible_tree = tree;
last_visible_position = position;
last_visible_row = offset_row;
}
did_descend = true;
break;
}
position = ts_length_add(position, ts_tree_total_size(child));
offset_row += child->padding_point.row + child->size_point.row;
}
}
return ts_node_make(last_visible_tree, last_visible_position);
return ts_node_make(last_visible_tree, last_visible_position, last_visible_row);
}
/*
@ -156,6 +168,19 @@ TSLength ts_node_size(TSNode self) {
return ts_node__tree(self)->size;
}
TSPoint ts_node_size_point(TSNode self) {
return ts_node__tree(self)->size_point;
}
TSPoint ts_node_start_point(TSNode self) {
const TSTree *tree = ts_node__tree(self);
return ts_point_make(self.row + tree->padding_point.row, ts_tree_offset_column(tree));
}
TSPoint ts_node_end_point(TSNode self) {
return ts_point_add(ts_node_start_point(self), ts_node_size_point(self));
}
TSSymbol ts_node_symbol(TSNode self) {
return ts_node__tree(self)->symbol;
}
@ -171,7 +196,8 @@ const char *ts_node_string(TSNode self, const TSDocument *document) {
bool ts_node_eq(TSNode self, TSNode other) {
return ts_tree_eq(ts_node__tree(self), ts_node__tree(other)) &&
ts_length_eq(ts_node__offset(self), ts_node__offset(other));
ts_length_eq(ts_node__offset(self), ts_node__offset(other)) &&
self.row == other.row;
}
bool ts_node_is_named(TSNode self) {
@ -185,15 +211,18 @@ bool ts_node_has_changes(TSNode self) {
TSNode ts_node_parent(TSNode self) {
const TSTree *tree = ts_node__tree(self);
TSLength position = ts_node__offset(self);
size_t offset_row = self.row;
do {
position = ts_length_sub(position, tree->context.offset);
offset_row -= tree->context.offset_point.row;
tree = tree->context.parent;
if (!tree)
return ts_node__null();
} while (!ts_tree_is_visible(tree));
return ts_node_make(tree, position);
return ts_node_make(tree, position, offset_row);
}
TSNode ts_node_child(TSNode self, size_t child_index) {

View file

@ -5,6 +5,6 @@
#include "runtime/length.h"
#include "runtime/tree.h"
TSNode ts_node_make(const TSTree *, TSLength);
TSNode ts_node_make(const TSTree *, TSLength, size_t);
#endif

View file

@ -27,6 +27,7 @@ typedef struct {
TSTree *reusable_subtree;
size_t reusable_subtree_pos;
TSLength position;
TSPoint offset_point;
} LookaheadState;
typedef enum {
@ -150,6 +151,7 @@ static ConsumeResult ts_parser__shift(TSParser *self, int head,
LookaheadState *head_state = vector_get(&self->lookahead_states, head);
head_state->position =
ts_length_add(head_state->position, ts_tree_total_size(lookahead));
head_state->offset_point = ts_point_add(head_state->offset_point, ts_tree_total_size_point(lookahead));
if (ts_stack_push(self->stack, head, parse_state, lookahead)) {
LOG("merge head:%d", head);
vector_erase(&self->lookahead_states, head);
@ -271,6 +273,7 @@ static void ts_parser__reduce_error(TSParser *self, int head,
child_count, false, true);
reduced->size = ts_length_add(reduced->size, lookahead->padding);
head_state->position = ts_length_add(head_state->position, lookahead->padding);
head_state->offset_point = ts_point_add(head_state->offset_point, lookahead->padding_point);
lookahead->padding = ts_length_zero();
ts_tree_set_fragile_left(reduced);
ts_tree_set_fragile_right(reduced);
@ -340,7 +343,7 @@ static void ts_parser__start(TSParser *self, TSInput input,
}
self->lexer.input = input;
ts_lexer_reset(&self->lexer, ts_length_zero());
ts_lexer_reset(&self->lexer, ts_length_zero(), ts_point_zero());
ts_stack_clear(self->stack);
LookaheadState head_state = {
@ -482,6 +485,7 @@ TSTree *ts_parser_parse(TSParser *self, TSInput input, TSTree *previous_tree) {
for (;;) {
TSTree *lookahead = NULL;
TSLength position = ts_length_zero();
TSPoint offset_point = ts_point_zero();
for (int head = 0; head < ts_stack_head_count(self->stack);) {
LookaheadState *state = vector_get(&self->lookahead_states, head);
@ -497,7 +501,8 @@ TSTree *ts_parser_parse(TSParser *self, TSInput input, TSTree *previous_tree) {
lookahead = reused_lookahead;
} else {
position = state->position;
ts_lexer_reset(&self->lexer, position);
offset_point = state->offset_point;
ts_lexer_reset(&self->lexer, position, offset_point);
TSStateId parse_state = ts_stack_top_state(self->stack, head);
TSStateId lex_state = self->language->lex_states[parse_state];
lookahead = self->language->lex_fn(&self->lexer, lex_state);

View file

@ -7,6 +7,8 @@
#include "runtime/length.h"
TSTree *ts_tree_make_leaf(TSSymbol sym, TSLength padding, TSLength size,
TSPoint padding_point,
TSPoint size_point,
TSNodeType node_type) {
TSTree *result = malloc(sizeof(TSTree));
*result = (TSTree){
@ -18,6 +20,8 @@ TSTree *ts_tree_make_leaf(TSSymbol sym, TSLength padding, TSLength size,
.named_child_count = 0,
.children = NULL,
.padding = padding,
.padding_point = padding_point,
.size_point = size_point,
.options = {.type = node_type },
};
@ -29,24 +33,31 @@ TSTree *ts_tree_make_leaf(TSSymbol sym, TSLength padding, TSLength size,
return result;
}
TSTree *ts_tree_make_error(TSLength size, TSLength padding, char lookahead_char) {
TSTree *ts_tree_make_error(TSLength size, TSLength padding,
TSPoint size_point,
TSPoint padding_point,
char lookahead_char) {
TSTree *result =
ts_tree_make_leaf(ts_builtin_sym_error, padding, size, TSNodeTypeNamed);
ts_tree_make_leaf(ts_builtin_sym_error, padding, size, padding_point,
size_point, TSNodeTypeNamed);
result->lookahead_char = lookahead_char;
return result;
}
void ts_tree_assign_parents(TSTree *self) {
TSLength offset = ts_length_zero();
TSPoint offset_point = ts_point_zero();
for (size_t i = 0; i < self->child_count; i++) {
TSTree *child = self->children[i];
if (child->context.parent != self) {
child->context.parent = self;
child->context.index = i;
child->context.offset = offset;
child->context.offset_point = offset_point;
ts_tree_assign_parents(child);
}
offset = ts_length_add(offset, ts_tree_total_size(child));
offset_point = ts_point_add(offset_point, ts_tree_total_size_point(child));
}
}
@ -62,9 +73,12 @@ static void ts_tree__set_children(TSTree *self, TSTree **children,
if (i == 0) {
self->padding = child->padding;
self->size = child->size;
self->padding_point = child->padding_point;
self->size_point = child->size_point;
} else {
self->size =
ts_length_add(ts_length_add(self->size, child->padding), child->size);
self->size_point = ts_point_add(ts_point_add(self->size_point, child->padding_point), child->size_point);
}
switch (child->options.type) {
@ -93,7 +107,7 @@ static void ts_tree__set_children(TSTree *self, TSTree **children,
TSTree *ts_tree_make_node(TSSymbol symbol, size_t child_count,
TSTree **children, TSNodeType node_type) {
TSTree *result =
ts_tree_make_leaf(symbol, ts_length_zero(), ts_length_zero(), node_type);
ts_tree_make_leaf(symbol, ts_length_zero(), ts_length_zero(), ts_point_zero(), ts_point_zero(), node_type);
ts_tree__set_children(result, children, child_count);
return result;
}
@ -115,10 +129,34 @@ void ts_tree_release(TSTree *self) {
}
}
size_t ts_tree_offset_column(const TSTree *self) {
size_t column = self->padding_point.column;
if (self->padding_point.row > 0) {
return column;
}
const TSTree *parent = self;
TSPoint offset_point;
do {
offset_point = parent->context.offset_point;
column += offset_point.column;
parent = parent->context.parent;
if (!parent) break;
} while (offset_point.row == 0);
return column;
}
TSLength ts_tree_total_size(const TSTree *self) {
return ts_length_add(self->padding, self->size);
}
TSPoint ts_tree_total_size_point(const TSTree *self) {
return ts_point_add(self->padding_point, self->size_point);
}
bool ts_tree_eq(const TSTree *self, const TSTree *other) {
if (self) {
if (!other)

View file

@ -13,6 +13,7 @@ struct TSTree {
struct TSTree *parent;
size_t index;
TSLength offset;
TSPoint offset_point;
} context;
size_t child_count;
size_t visible_child_count;
@ -23,7 +24,12 @@ struct TSTree {
};
TSLength padding;
TSLength size;
TSPoint padding_point;
TSPoint size_point;
TSSymbol symbol;
struct {
TSNodeType type : 2;
bool extra : 1;
@ -34,16 +40,22 @@ struct TSTree {
unsigned short int ref_count;
};
TSTree *ts_tree_make_leaf(TSSymbol, TSLength, TSLength, TSNodeType);
TSTree *ts_tree_make_leaf(TSSymbol, TSLength, TSLength, TSPoint,
TSPoint, TSNodeType);
TSTree *ts_tree_make_node(TSSymbol, size_t, TSTree **, TSNodeType);
TSTree *ts_tree_make_error(TSLength size, TSLength padding, char lookahead_char);
TSTree *ts_tree_make_error(TSLength size, TSLength padding,
TSPoint padding_point,
TSPoint size_point, char lookahead_char);
void ts_tree_retain(TSTree *tree);
void ts_tree_release(TSTree *tree);
bool ts_tree_eq(const TSTree *tree1, const TSTree *tree2);
int ts_tree_compare(const TSTree *tree1, const TSTree *tree2);
char *ts_tree_string(const TSTree *tree, const char **names,
bool include_anonymous);
size_t ts_tree_offset_column(const TSTree *self);
TSLength ts_tree_total_size(const TSTree *tree);
TSPoint ts_tree_total_size_point(const TSTree *self);
void ts_tree_prepend_children(TSTree *, size_t, TSTree **);
void ts_tree_assign_parents(TSTree *);
void ts_tree_edit(TSTree *, TSInputEdit);