diff --git a/include/tree_sitter/parser.h b/include/tree_sitter/parser.h index bb29a627..722c5208 100644 --- a/include/tree_sitter/parser.h +++ b/include/tree_sitter/parser.h @@ -38,6 +38,10 @@ typedef struct TSLexer { TSLength token_end_position; TSLength token_start_position; + TSPoint current_point; + TSPoint token_end_point; + TSPoint token_start_point; + size_t lookahead_size; int32_t lookahead; diff --git a/include/tree_sitter/runtime.h b/include/tree_sitter/runtime.h index 1143ad0e..86678caf 100644 --- a/include/tree_sitter/runtime.h +++ b/include/tree_sitter/runtime.h @@ -35,9 +35,15 @@ typedef struct { size_t chars_removed; } TSInputEdit; +typedef struct { + size_t row; + size_t column; +} TSPoint; + typedef struct { const void *data; TSLength offset; + size_t row; } TSNode; typedef unsigned short TSSymbol; @@ -46,6 +52,9 @@ typedef struct TSDocument TSDocument; TSLength ts_node_pos(TSNode); TSLength ts_node_size(TSNode); +TSPoint ts_node_size_point(TSNode); +TSPoint ts_node_start_point(TSNode); +TSPoint ts_node_end_point(TSNode); TSSymbol ts_node_symbol(TSNode); const char *ts_node_name(TSNode, const TSDocument *); const char *ts_node_string(TSNode, const TSDocument *); diff --git a/script/util/run_tests.sh b/script/util/run_tests.sh index 4e3759e4..df1f29db 100644 --- a/script/util/run_tests.sh +++ b/script/util/run_tests.sh @@ -68,10 +68,10 @@ function run_tests { ;; debug) - if which -s gdb; then - gdb $cmd -- "${args[@]}" - elif which -s lldb; then + if which -s lldb; then lldb $cmd -- "${args[@]}" + elif which -s gdb; then + gdb $cmd -- "${args[@]}" else echo "No debugger found" exit 1 diff --git a/spec/runtime/helpers/point_helpers.cc b/spec/runtime/helpers/point_helpers.cc new file mode 100644 index 00000000..97a444a9 --- /dev/null +++ b/spec/runtime/helpers/point_helpers.cc @@ -0,0 +1,24 @@ +#include +#include +#include "runtime/length.h" + +using namespace std; + +bool operator==(const TSPoint &left, const TSPoint &right) { + return left.row == right.row && left.column == right.column; +} + +std::ostream &operator<<(std::ostream &stream, const TSPoint &point) { + return stream << "{" << point.row << ", " << point.column << "}"; +} + +bool operator<(const TSPoint &left, const TSPoint &right) { + if (left.row < right.row) return true; + if (left.row > right.row) return false; + + return left.column < right.column; +} + +bool operator>(const TSPoint &left, const TSPoint &right) { + return right < left; +} diff --git a/spec/runtime/helpers/point_helpers.h b/spec/runtime/helpers/point_helpers.h new file mode 100644 index 00000000..671fd4dc --- /dev/null +++ b/spec/runtime/helpers/point_helpers.h @@ -0,0 +1,8 @@ + +bool operator==(const TSPoint &left, const TSPoint &right); + +bool operator<(const TSPoint &left, const TSPoint &right); + +bool operator>(const TSPoint &left, const TSPoint &right); + +std::ostream &operator<<(std::ostream &stream, const TSPoint &point); diff --git a/spec/runtime/language_specs.cc b/spec/runtime/language_specs.cc index 81bd54fa..57e9e49b 100644 --- a/spec/runtime/language_specs.cc +++ b/spec/runtime/language_specs.cc @@ -4,6 +4,7 @@ #include "runtime/helpers/read_test_entries.h" #include "runtime/helpers/spy_input.h" #include "runtime/helpers/log_debugger.h" +#include "runtime/helpers/point_helpers.h" extern "C" const TSLanguage *ts_language_javascript(); extern "C" const TSLanguage *ts_language_json(); @@ -23,6 +24,9 @@ void expect_a_consistent_tree(TSNode node, TSDocument *doc) { TSLength end = ts_length_add(start, ts_node_size(node)); size_t child_count = ts_node_child_count(node); + TSPoint start_point = ts_node_start_point(node); + TSPoint end_point = ts_node_end_point(node); + bool has_changes = ts_node_has_changes(node); bool some_child_has_changes = false; @@ -31,8 +35,15 @@ void expect_a_consistent_tree(TSNode node, TSDocument *doc) { TSLength child_start = ts_node_pos(child); TSLength child_end = ts_length_add(child_start, ts_node_size(child)); + TSPoint child_start_point = ts_node_start_point(child); + TSPoint child_end_point = ts_node_end_point(child); + AssertThat(child_start.chars, IsGreaterThan(start.chars) || Equals(start.chars)); AssertThat(child_end.chars, IsLessThan(end.chars) || Equals(end.chars)); + + AssertThat(child_start_point, IsGreaterThan(start_point) || Equals(start_point)); + AssertThat(child_end_point, IsLessThan(end_point) || Equals(end_point)); + if (ts_node_has_changes(child)) some_child_has_changes = true; } diff --git a/spec/runtime/node_spec.cc b/spec/runtime/node_spec.cc index e58317c9..ce5eb772 100644 --- a/spec/runtime/node_spec.cc +++ b/spec/runtime/node_spec.cc @@ -1,5 +1,6 @@ #include "runtime/runtime_spec_helper.h" #include "runtime/helpers/tree_helpers.h" +#include "runtime/helpers/point_helpers.h" extern "C" TSLanguage * ts_language_json(); @@ -8,11 +9,23 @@ START_TEST describe("Node", []() { TSDocument *document; TSNode array_node; + string input_string; before_each([&]() { document = ts_document_make(); ts_document_set_language(document, ts_language_json()); - ts_document_set_input_string(document, " [123, false, {\"x\": null}]"); + + input_string = "\n" + "\n" + "[\n" + " 123,\n" + " false,\n" + " {\n" + " \"x\": null\n" + " }\n" + "]"; + + ts_document_set_input_string(document, input_string.c_str()); ts_document_parse(document); array_node = ts_document_root_node(document); @@ -27,7 +40,7 @@ describe("Node", []() { ts_document_free(document); }); - describe("child_count(), child(i)", [&]() { + describe("named_child_count(), named_child(i)", [&]() { it("returns the named child node at the given index", [&]() { AssertThat(ts_node_named_child_count(array_node), Equals(3)); @@ -41,16 +54,28 @@ describe("Node", []() { AssertThat(ts_node_name(child3, document), Equals("object")); AssertThat(ts_node_pos(array_node).bytes, Equals(2)); - AssertThat(ts_node_size(array_node).bytes, Equals(25)); + AssertThat(ts_node_size(array_node).bytes, Equals(41)); - AssertThat(ts_node_pos(child1).bytes, Equals(3)); + AssertThat(ts_node_start_point(array_node), Equals({ 2, 0 })); + AssertThat(ts_node_end_point(array_node), Equals({ 8, 1 })); + + AssertThat(ts_node_pos(child1).bytes, Equals(input_string.find("123"))); AssertThat(ts_node_size(child1).bytes, Equals(3)); - AssertThat(ts_node_pos(child2).bytes, Equals(8)); + AssertThat(ts_node_start_point(child1), Equals({ 3, 2 })); + AssertThat(ts_node_end_point(child1), Equals({ 3, 5 })); + + AssertThat(ts_node_pos(child2).bytes, Equals(input_string.find("false"))); AssertThat(ts_node_size(child2).bytes, Equals(5)); - AssertThat(ts_node_pos(child3).bytes, Equals(15)); - AssertThat(ts_node_size(child3).bytes, Equals(11)); + AssertThat(ts_node_start_point(child2), Equals({ 4, 2 })); + AssertThat(ts_node_end_point(child2), Equals({ 4, 7 })); + + AssertThat(ts_node_pos(child3).bytes, Equals(input_string.find("{"))); + AssertThat(ts_node_size(child3).bytes, Equals(19)); + + AssertThat(ts_node_start_point(child3), Equals({ 5, 2 })); + AssertThat(ts_node_end_point(child3), Equals({ 7, 3 })); AssertThat(ts_node_named_child_count(child3), Equals(2)); @@ -60,6 +85,9 @@ describe("Node", []() { AssertThat(ts_node_name(grandchild1, document), Equals("string")); AssertThat(ts_node_name(grandchild2, document), Equals("null")); + AssertThat(ts_node_start_point(grandchild1), Equals({ 6, 4 })); + AssertThat(ts_node_end_point(grandchild1), Equals({ 6, 7 })); + AssertThat(ts_node_parent(child1), Equals(array_node)); AssertThat(ts_node_parent(child2), Equals(array_node)); AssertThat(ts_node_parent(child3), Equals(array_node)); @@ -67,7 +95,7 @@ describe("Node", []() { }); }); - describe("concrete_child_count(), concrete_child(i)", [&]() { + describe("child_count(), child(i)", [&]() { it("returns the child node at the given index, including anonymous nodes", [&]() { AssertThat(ts_node_child_count(array_node), Equals(7)); TSNode child1 = ts_node_child(array_node, 0); @@ -99,15 +127,27 @@ describe("Node", []() { AssertThat(ts_node_pos(child1).bytes, Equals(2)); AssertThat(ts_node_size(child1).bytes, Equals(1)); - AssertThat(ts_node_pos(child3).bytes, Equals(6)); + AssertThat(ts_node_start_point(child1), Equals({ 2, 0 })); + AssertThat(ts_node_end_point(child1), Equals({ 2, 1 })); + + AssertThat(ts_node_pos(child3).bytes, Equals(9)); AssertThat(ts_node_size(child3).bytes, Equals(1)); - AssertThat(ts_node_pos(child5).bytes, Equals(13)); + AssertThat(ts_node_start_point(child3), Equals({ 3, 5 })); + AssertThat(ts_node_end_point(child3), Equals({ 3, 6 })); + + AssertThat(ts_node_pos(child5).bytes, Equals(18)); AssertThat(ts_node_size(child5).bytes, Equals(1)); - AssertThat(ts_node_pos(child7).bytes, Equals(26)); + AssertThat(ts_node_start_point(child5), Equals({ 4, 7 })); + AssertThat(ts_node_end_point(child5), Equals({ 4, 8 })); + + AssertThat(ts_node_pos(child7).bytes, Equals(42)); AssertThat(ts_node_size(child7).bytes, Equals(1)); + AssertThat(ts_node_start_point(child7), Equals({ 8, 0 })); + AssertThat(ts_node_end_point(child7), Equals({ 8, 1 })); + AssertThat(ts_node_child_count(child6), Equals(5)) TSNode grandchild1 = ts_node_child(child6, 0); @@ -133,7 +173,7 @@ describe("Node", []() { }); }); - describe("next_concrete_sibling(), prev_concrete_sibling()", [&]() { + describe("next_sibling(), prev_sibling()", [&]() { it("returns the node's next and previous sibling, including anonymous nodes", [&]() { TSNode bracket_node1 = ts_node_child(array_node, 0); TSNode number_node = ts_node_child(array_node, 1); @@ -185,7 +225,7 @@ describe("Node", []() { }); }); - describe("next_concrete_sibling(), prev_concrete_sibling()", [&]() { + describe("next_named_sibling(), prev_concrete_sibling()", [&]() { it("returns the node's next and previous siblings", [&]() { TSNode number_node = ts_node_named_child(array_node, 0); TSNode false_node = ts_node_named_child(array_node, 1); @@ -212,60 +252,89 @@ describe("Node", []() { describe("find_for_range(start, end)", [&]() { describe("when there is a leaf node that spans the given range exactly", [&]() { it("returns that leaf node", [&]() { - TSNode leaf = ts_node_named_descendent_for_range(array_node, 16, 18); + TSNode leaf = ts_node_named_descendent_for_range(array_node, 28, 30); AssertThat(ts_node_name(leaf, document), Equals("string")); AssertThat(ts_node_size(leaf).bytes, Equals(3)); - AssertThat(ts_node_pos(leaf).bytes, Equals(16)); + AssertThat(ts_node_pos(leaf).bytes, Equals(28)); - leaf = ts_node_named_descendent_for_range(array_node, 3, 5); + size_t index = input_string.find("123"); + leaf = ts_node_named_descendent_for_range(array_node, index, index + 2); AssertThat(ts_node_name(leaf, document), Equals("number")); + AssertThat(ts_node_pos(leaf).bytes, Equals(index)); AssertThat(ts_node_size(leaf).bytes, Equals(3)); - AssertThat(ts_node_pos(leaf).bytes, Equals(3)); + AssertThat(ts_node_start_point(leaf), Equals({ 3, 2 })); + AssertThat(ts_node_end_point(leaf), Equals({ 3, 5 })); }); }); describe("when there is a leaf node that extends beyond the given range", [&]() { it("returns that leaf node", [&]() { - TSNode leaf = ts_node_named_descendent_for_range(array_node, 16, 17); + size_t index = input_string.find("\"x\""); + TSNode leaf = ts_node_named_descendent_for_range(array_node, index, index + 1); AssertThat(ts_node_name(leaf, document), Equals("string")); + AssertThat(ts_node_pos(leaf).bytes, Equals(index)); AssertThat(ts_node_size(leaf).bytes, Equals(3)); - AssertThat(ts_node_pos(leaf).bytes, Equals(16)); - leaf = ts_node_named_descendent_for_range(array_node, 17, 18); + AssertThat(ts_node_start_point(leaf), Equals({ 6, 4 })); + AssertThat(ts_node_end_point(leaf), Equals({ 6, 7 })); + + leaf = ts_node_named_descendent_for_range(array_node, index + 1, index + 2); AssertThat(ts_node_name(leaf, document), Equals("string")); + AssertThat(ts_node_pos(leaf).bytes, Equals(index)); AssertThat(ts_node_size(leaf).bytes, Equals(3)); - AssertThat(ts_node_pos(leaf).bytes, Equals(16)); + + AssertThat(ts_node_start_point(leaf), Equals({ 6, 4 })); + AssertThat(ts_node_end_point(leaf), Equals({ 6, 7 })); }); }); describe("when there is no leaf node that spans the given range", [&]() { it("returns the smallest node that does span the range", [&]() { - TSNode node = ts_node_named_descendent_for_range(array_node, 16, 19); + size_t index = input_string.find("\"x\""); + TSNode node = ts_node_named_descendent_for_range(array_node, index, index + 3); AssertThat(ts_node_name(node, document), Equals("object")); - AssertThat(ts_node_size(node).bytes, Equals(11)); - AssertThat(ts_node_pos(node).bytes, Equals(15)); + + size_t object_index = input_string.find("{"); + AssertThat(ts_node_pos(node).bytes, Equals(object_index)); + AssertThat(ts_node_size(node).bytes, Equals(19)); + + AssertThat(ts_node_start_point(node), Equals({ 5, 2 })); + AssertThat(ts_node_end_point(node), Equals({ 7, 3 })); }); it("does not return invisible nodes (repeats)", [&]() { - TSNode node = ts_node_named_descendent_for_range(array_node, 6, 7); + size_t comma_index = input_string.find(","); + TSNode node = ts_node_named_descendent_for_range(array_node, comma_index, comma_index + 1); AssertThat(ts_node_name(node, document), Equals("array")); - AssertThat(ts_node_size(node).bytes, Equals(25)); - AssertThat(ts_node_pos(node).bytes, Equals(2)); + size_t array_index = input_string.find("["); + AssertThat(ts_node_pos(node).bytes, Equals(array_index)); + AssertThat(ts_node_size(node).bytes, Equals(41)); + + AssertThat(ts_node_start_point(node), Equals({ 2, 0 })); + AssertThat(ts_node_end_point(node), Equals({ 8, 1 })); }); }); }); - describe("find_concrete_for_range(start, end)", [&]() { + describe("descendent_for_range(start, end)", [&]() { it("returns the smallest concrete node that spans the given range", [&]() { - TSNode node1 = ts_node_descendent_for_range(array_node, 19, 19); + size_t colon_index = input_string.find(":"); + TSNode node1 = ts_node_descendent_for_range(array_node, colon_index, colon_index); AssertThat(ts_node_name(node1, document), Equals(":")); - AssertThat(ts_node_pos(node1).bytes, Equals(19)); + AssertThat(ts_node_pos(node1).bytes, Equals(colon_index)); AssertThat(ts_node_size(node1).bytes, Equals(1)); + AssertThat(ts_node_start_point(node1), Equals({ 6, 7 })); + AssertThat(ts_node_end_point(node1), Equals({ 6, 8 })); - TSNode node2 = ts_node_descendent_for_range(array_node, 18, 20); + size_t index = input_string.find("\":"); + TSNode node2 = ts_node_descendent_for_range(array_node, index, index + 2); AssertThat(ts_node_name(node2, document), Equals("object")); - AssertThat(ts_node_pos(node2).bytes, Equals(15)); - AssertThat(ts_node_size(node2).bytes, Equals(11)); + + size_t object_index = input_string.find("{"); + AssertThat(ts_node_pos(node2).bytes, Equals(object_index)); + AssertThat(ts_node_size(node2).bytes, Equals(19)); + AssertThat(ts_node_start_point(node2), Equals({ 5, 2 })); + AssertThat(ts_node_end_point(node2), Equals({ 7, 3 })); }); }); }); diff --git a/spec/runtime/stack_spec.cc b/spec/runtime/stack_spec.cc index 7c3cc1c7..7c825642 100644 --- a/spec/runtime/stack_spec.cc +++ b/spec/runtime/stack_spec.cc @@ -44,7 +44,7 @@ describe("Stack", [&]() { TSLength len = ts_length_make(2, 2); for (size_t i = 0; i < tree_count; i++) - trees[i] = ts_tree_make_leaf(i, len, len, TSNodeTypeNamed); + trees[i] = ts_tree_make_leaf(i, len, len, {1, 1}, {1, 2}, TSNodeTypeNamed); }); after_each([&]() { diff --git a/spec/runtime/tree_spec.cc b/spec/runtime/tree_spec.cc index c2629fb3..99125dbe 100644 --- a/spec/runtime/tree_spec.cc +++ b/spec/runtime/tree_spec.cc @@ -29,8 +29,8 @@ describe("Tree", []() { TSTree *tree1, *tree2, *parent1; before_each([&]() { - tree1 = ts_tree_make_leaf(cat, {2, 1}, {5, 4}, TSNodeTypeNamed); - tree2 = ts_tree_make_leaf(cat, {1, 1}, {3, 3}, TSNodeTypeNamed); + tree1 = ts_tree_make_leaf(cat, {2, 1}, {5, 4}, ts_point_zero(), ts_point_zero(), TSNodeTypeNamed); + tree2 = ts_tree_make_leaf(cat, {1, 1}, {3, 3}, ts_point_zero(), ts_point_zero(), TSNodeTypeNamed); parent1 = ts_tree_make_node(dog, 2, tree_array({ tree1, tree2, @@ -55,6 +55,8 @@ describe("Tree", []() { TSTree *error_tree = ts_tree_make_error( ts_length_zero(), ts_length_zero(), + ts_point_zero(), + ts_point_zero(), 'z'); AssertThat(ts_tree_is_fragile_left(error_tree), IsTrue()); @@ -145,9 +147,9 @@ describe("Tree", []() { before_each([&]() { tree = ts_tree_make_node(cat, 3, tree_array({ - ts_tree_make_leaf(dog, {2, 2}, {3, 3}, TSNodeTypeNamed), - ts_tree_make_leaf(eel, {2, 2}, {3, 3}, TSNodeTypeNamed), - ts_tree_make_leaf(fox, {2, 2}, {3, 3}, TSNodeTypeNamed), + ts_tree_make_leaf(dog, {2, 2}, {3, 3}, {1, 2}, {1, 3}, TSNodeTypeNamed), + ts_tree_make_leaf(eel, {2, 2}, {3, 3}, {1, 2}, {1, 3}, TSNodeTypeNamed), + ts_tree_make_leaf(fox, {2, 2}, {3, 3}, {1, 2}, {1, 3}, TSNodeTypeNamed), }), TSNodeTypeNamed); AssertThat(tree->padding, Equals({2, 2})); @@ -266,10 +268,10 @@ describe("Tree", []() { describe("equality", [&]() { it("returns true for identical trees", [&]() { - TSTree *tree1_copy = ts_tree_make_leaf(cat, {2, 1}, {5, 4}, TSNodeTypeNamed); + TSTree *tree1_copy = ts_tree_make_leaf(cat, {2, 1}, {5, 4}, {1, 1}, {1, 4}, TSNodeTypeNamed); AssertThat(ts_tree_eq(tree1, tree1_copy), IsTrue()); - TSTree *tree2_copy = ts_tree_make_leaf(cat, {1, 1}, {3, 3}, TSNodeTypeNamed); + TSTree *tree2_copy = ts_tree_make_leaf(cat, {1, 1}, {3, 3}, {1, 1}, {1, 3}, TSNodeTypeNamed); AssertThat(ts_tree_eq(tree2, tree2_copy), IsTrue()); TSTree *parent2 = ts_tree_make_node(dog, 2, tree_array({ @@ -289,6 +291,8 @@ describe("Tree", []() { tree1->symbol + 1, tree1->padding, tree1->size, + tree1->padding_point, + tree1->size_point, TSNodeTypeNamed); AssertThat(ts_tree_eq(tree1, different_tree), IsFalse()); @@ -300,6 +304,8 @@ describe("Tree", []() { tree1->symbol + 1, tree1->padding, tree1->size, + tree1->padding_point, + tree1->size_point, TSNodeTypeNamed); TSTree *different_parent = ts_tree_make_node(dog, 2, tree_array({ diff --git a/src/runtime/document.c b/src/runtime/document.c index 091bda23..1afcb2f3 100644 --- a/src/runtime/document.c +++ b/src/runtime/document.c @@ -85,7 +85,7 @@ void ts_document_invalidate(TSDocument *self) { } TSNode ts_document_root_node(const TSDocument *self) { - TSNode result = ts_node_make(self->tree, ts_length_zero()); + TSNode result = ts_node_make(self->tree, ts_length_zero(), 0); while (result.data && !ts_tree_is_visible(result.data)) result = ts_node_named_child(result, 0); return result; diff --git a/src/runtime/length.h b/src/runtime/length.h index 58e078e4..96051869 100644 --- a/src/runtime/length.h +++ b/src/runtime/length.h @@ -26,6 +26,39 @@ static inline TSLength ts_length_sub(TSLength len1, TSLength len2) { return result; } +static inline TSPoint ts_point_make(size_t row, size_t column) { + TSPoint point; + point.row = row; + point.column = column; + return point; +} + +static inline TSPoint ts_point_add(TSPoint point1, TSPoint point2) { + size_t row = point1.row + point2.row; + + size_t column; + if (point2.row == 0) { + column = point1.column + point2.column; + } else { + column = point2.column; + } + + return ts_point_make(row, column); +} + +static inline TSPoint ts_point_sub(TSPoint point1, TSPoint point2) { + size_t row, column; + if (point1.row == point2.row) { + row = 0; + column = point1.column - point2.column; + } else { + row = point1.row - point2.row; + column = point1.column; + } + + return ts_point_make(row, column); +} + static inline TSLength ts_length_zero() { TSLength result; result.bytes = result.chars = 0; @@ -36,6 +69,12 @@ static inline bool ts_length_eq(TSLength len1, TSLength len2) { return len1.bytes == len2.bytes && len1.chars == len2.chars; } +static inline TSPoint ts_point_zero() { + TSPoint point; + point.row = point.column = 0; + return point; +} + static inline TSLength ts_length_make(size_t bytes, size_t chars) { TSLength result; result.bytes = bytes; diff --git a/src/runtime/lexer.c b/src/runtime/lexer.c index de4c9bf0..91af5222 100644 --- a/src/runtime/lexer.c +++ b/src/runtime/lexer.c @@ -51,8 +51,10 @@ static void ts_lexer__start(TSLexer *self, TSStateId lex_state) { } static void ts_lexer__start_token(TSLexer *self) { - LOG("start_token chars:%lu", self->current_position.chars); + LOG("start_token chars:%lu, rows:%lu, columns:%lu", self->current_position.chars, self->current_point.row, self->current_point.column); + self->token_start_position = self->current_position; + self->token_start_point = self->current_point; } static bool ts_lexer__advance(TSLexer *self, TSStateId state) { @@ -64,6 +66,13 @@ static bool ts_lexer__advance(TSLexer *self, TSStateId state) { if (self->lookahead_size) { self->current_position.bytes += self->lookahead_size; self->current_position.chars += 1; + + if (self->lookahead == '\n') { + self->current_point.row += 1; + self->current_point.column = 0; + } else { + self->current_point.column += 1; + } } if (self->current_position.bytes >= self->chunk_start + self->chunk_size) @@ -81,12 +90,16 @@ static TSTree *ts_lexer__accept(TSLexer *self, TSSymbol symbol, ts_length_sub(self->token_start_position, self->token_end_position); self->token_end_position = self->current_position; + TSPoint size_point = ts_point_sub(self->current_point, self ->token_start_point); + TSPoint padding_point = ts_point_sub(self->token_start_point, self->token_end_point); + self->token_end_point = self->current_point; + if (symbol == ts_builtin_sym_error) { LOG("error_char"); - return ts_tree_make_error(size, padding, self->lookahead); + return ts_tree_make_error(size, padding, size_point, padding_point, self->lookahead); } else { LOG("accept_token sym:%s", symbol_name); - return ts_tree_make_leaf(symbol, padding, size, node_type); + return ts_tree_make_leaf(symbol, padding, size, padding_point, size_point, node_type); } } @@ -105,17 +118,22 @@ TSLexer ts_lexer_make() { .chunk_start = 0, .debugger = ts_debugger_null(), }; - ts_lexer_reset(&result, ts_length_zero()); + ts_lexer_reset(&result, ts_length_zero(), ts_point_zero()); return result; } -void ts_lexer_reset(TSLexer *self, TSLength position) { +void ts_lexer_reset(TSLexer *self, TSLength position, TSPoint point) { if (ts_length_eq(position, self->current_position)) return; self->token_start_position = position; self->token_end_position = position; self->current_position = position; + + self->token_start_point = point; + self->token_end_point = point; + self->current_point = point; + self->chunk = 0; self->chunk_start = 0; self->chunk_size = 0; diff --git a/src/runtime/lexer.h b/src/runtime/lexer.h index b127d483..2c4b5812 100644 --- a/src/runtime/lexer.h +++ b/src/runtime/lexer.h @@ -8,7 +8,7 @@ extern "C" { #include "tree_sitter/parser.h" TSLexer ts_lexer_make(); -void ts_lexer_reset(TSLexer *, TSLength); +void ts_lexer_reset(TSLexer *, TSLength, TSPoint); #ifdef __cplusplus } diff --git a/src/runtime/node.c b/src/runtime/node.c index f6919d2a..cbc720ce 100644 --- a/src/runtime/node.c +++ b/src/runtime/node.c @@ -4,8 +4,8 @@ #include "runtime/tree.h" #include "runtime/document.h" -TSNode ts_node_make(const TSTree *tree, TSLength offset) { - return (TSNode){.data = tree, .offset = offset }; +TSNode ts_node_make(const TSTree *tree, TSLength offset, size_t row) { + return (TSNode){.data = tree, .offset = offset, .row = row }; } /* @@ -13,7 +13,7 @@ TSNode ts_node_make(const TSTree *tree, TSLength offset) { */ static inline TSNode ts_node__null() { - return ts_node_make(NULL, ts_length_zero()); + return ts_node_make(NULL, ts_length_zero(), 0); } static inline const TSTree *ts_node__tree(TSNode self) { @@ -24,10 +24,12 @@ static inline TSLength ts_node__offset(TSNode self) { return self.offset; } + static inline TSNode ts_node__child(TSNode self, size_t child_index, TSNodeType type) { const TSTree *tree = ts_node__tree(self); TSLength position = ts_node__offset(self); + size_t offset_row = self.row; bool did_descend = true; while (did_descend) { @@ -38,7 +40,7 @@ static inline TSNode ts_node__child(TSNode self, size_t child_index, TSTree *child = tree->children[i]; if (child->options.type >= type) { if (index == child_index) - return ts_node_make(child, position); + return ts_node_make(child, position, offset_row); index++; } else { size_t grandchild_index = child_index - index; @@ -54,6 +56,7 @@ static inline TSNode ts_node__child(TSNode self, size_t child_index, index += grandchild_count; } position = ts_length_add(position, ts_tree_total_size(child)); + offset_row += child->padding_point.row + child->size_point.row; } } @@ -62,25 +65,28 @@ static inline TSNode ts_node__child(TSNode self, size_t child_index, static inline TSNode ts_node__prev_sibling(TSNode self, TSNodeType type) { const TSTree *tree = ts_node__tree(self); - TSLength position = ts_node__offset(self); + TSLength offset = ts_node__offset(self); + size_t offset_row = self.row; do { size_t index = tree->context.index; - position = ts_length_sub(position, tree->context.offset); + offset = ts_length_sub(offset, tree->context.offset); + offset_row -= tree->context.offset_point.row; tree = tree->context.parent; if (!tree) break; for (size_t i = index - 1; i + 1 > 0; i--) { const TSTree *child = tree->children[i]; - TSLength child_position = ts_length_add(position, child->context.offset); + TSLength child_offset = ts_length_add(offset, child->context.offset); + size_t child_offset_row = offset_row + child->context.offset_point.row; if (child->options.type >= type) - return ts_node_make(child, child_position); + return ts_node_make(child, child_offset, child_offset_row); size_t grandchild_count = (type == TSNodeTypeNamed) ? child->named_child_count : child->visible_child_count; if (grandchild_count > 0) - return ts_node__child(ts_node_make(child, child_position), + return ts_node__child(ts_node_make(child, child_offset, child_offset_row), grandchild_count - 1, type); } } while (!ts_tree_is_visible(tree)); @@ -90,25 +96,28 @@ static inline TSNode ts_node__prev_sibling(TSNode self, TSNodeType type) { static inline TSNode ts_node__next_sibling(TSNode self, TSNodeType type) { const TSTree *tree = ts_node__tree(self); - TSLength position = ts_node__offset(self); + TSLength offset = ts_node__offset(self); + size_t offset_row = self.row; do { size_t index = tree->context.index; - position = ts_length_sub(position, tree->context.offset); + offset = ts_length_sub(offset, tree->context.offset); + offset_row -= tree->context.offset_point.row; tree = tree->context.parent; if (!tree) break; for (size_t i = index + 1; i < tree->child_count; i++) { const TSTree *child = tree->children[i]; - TSLength child_position = ts_length_add(position, child->context.offset); + TSLength child_offset = ts_length_add(offset, child->context.offset); + size_t child_offset_row = offset_row + child->context.offset_point.row; if (child->options.type >= type) - return ts_node_make(child, child_position); + return ts_node_make(child, child_offset, child_offset_row); size_t grandchild_count = (type == TSNodeTypeNamed) ? child->named_child_count : child->visible_child_count; if (grandchild_count > 0) - return ts_node__child(ts_node_make(child, child_position), 0, type); + return ts_node__child(ts_node_make(child, child_offset, child_offset_row), 0, type); } } while (!ts_tree_is_visible(tree)); @@ -119,6 +128,7 @@ static inline TSNode ts_node__descendent_for_range(TSNode self, size_t min, size_t max, TSNodeType type) { const TSTree *tree = ts_node__tree(self), *last_visible_tree = tree; TSLength position = ts_node__offset(self), last_visible_position = position; + size_t offset_row = self.row, last_visible_row = offset_row; bool did_descend = true; while (did_descend) { @@ -133,15 +143,17 @@ static inline TSNode ts_node__descendent_for_range(TSNode self, size_t min, if (child->options.type >= type) { last_visible_tree = tree; last_visible_position = position; + last_visible_row = offset_row; } did_descend = true; break; } position = ts_length_add(position, ts_tree_total_size(child)); + offset_row += child->padding_point.row + child->size_point.row; } } - return ts_node_make(last_visible_tree, last_visible_position); + return ts_node_make(last_visible_tree, last_visible_position, last_visible_row); } /* @@ -156,6 +168,19 @@ TSLength ts_node_size(TSNode self) { return ts_node__tree(self)->size; } +TSPoint ts_node_size_point(TSNode self) { + return ts_node__tree(self)->size_point; +} + +TSPoint ts_node_start_point(TSNode self) { + const TSTree *tree = ts_node__tree(self); + return ts_point_make(self.row + tree->padding_point.row, ts_tree_offset_column(tree)); +} + +TSPoint ts_node_end_point(TSNode self) { + return ts_point_add(ts_node_start_point(self), ts_node_size_point(self)); +} + TSSymbol ts_node_symbol(TSNode self) { return ts_node__tree(self)->symbol; } @@ -171,7 +196,8 @@ const char *ts_node_string(TSNode self, const TSDocument *document) { bool ts_node_eq(TSNode self, TSNode other) { return ts_tree_eq(ts_node__tree(self), ts_node__tree(other)) && - ts_length_eq(ts_node__offset(self), ts_node__offset(other)); + ts_length_eq(ts_node__offset(self), ts_node__offset(other)) && + self.row == other.row; } bool ts_node_is_named(TSNode self) { @@ -185,15 +211,18 @@ bool ts_node_has_changes(TSNode self) { TSNode ts_node_parent(TSNode self) { const TSTree *tree = ts_node__tree(self); TSLength position = ts_node__offset(self); + size_t offset_row = self.row; do { position = ts_length_sub(position, tree->context.offset); + offset_row -= tree->context.offset_point.row; + tree = tree->context.parent; if (!tree) return ts_node__null(); } while (!ts_tree_is_visible(tree)); - return ts_node_make(tree, position); + return ts_node_make(tree, position, offset_row); } TSNode ts_node_child(TSNode self, size_t child_index) { diff --git a/src/runtime/node.h b/src/runtime/node.h index f1ea67b3..2a169df8 100644 --- a/src/runtime/node.h +++ b/src/runtime/node.h @@ -5,6 +5,6 @@ #include "runtime/length.h" #include "runtime/tree.h" -TSNode ts_node_make(const TSTree *, TSLength); +TSNode ts_node_make(const TSTree *, TSLength, size_t); #endif diff --git a/src/runtime/parser.c b/src/runtime/parser.c index eec1a31c..71d254c1 100644 --- a/src/runtime/parser.c +++ b/src/runtime/parser.c @@ -27,6 +27,7 @@ typedef struct { TSTree *reusable_subtree; size_t reusable_subtree_pos; TSLength position; + TSPoint offset_point; } LookaheadState; typedef enum { @@ -150,6 +151,7 @@ static ConsumeResult ts_parser__shift(TSParser *self, int head, LookaheadState *head_state = vector_get(&self->lookahead_states, head); head_state->position = ts_length_add(head_state->position, ts_tree_total_size(lookahead)); + head_state->offset_point = ts_point_add(head_state->offset_point, ts_tree_total_size_point(lookahead)); if (ts_stack_push(self->stack, head, parse_state, lookahead)) { LOG("merge head:%d", head); vector_erase(&self->lookahead_states, head); @@ -271,6 +273,7 @@ static void ts_parser__reduce_error(TSParser *self, int head, child_count, false, true); reduced->size = ts_length_add(reduced->size, lookahead->padding); head_state->position = ts_length_add(head_state->position, lookahead->padding); + head_state->offset_point = ts_point_add(head_state->offset_point, lookahead->padding_point); lookahead->padding = ts_length_zero(); ts_tree_set_fragile_left(reduced); ts_tree_set_fragile_right(reduced); @@ -340,7 +343,7 @@ static void ts_parser__start(TSParser *self, TSInput input, } self->lexer.input = input; - ts_lexer_reset(&self->lexer, ts_length_zero()); + ts_lexer_reset(&self->lexer, ts_length_zero(), ts_point_zero()); ts_stack_clear(self->stack); LookaheadState head_state = { @@ -482,6 +485,7 @@ TSTree *ts_parser_parse(TSParser *self, TSInput input, TSTree *previous_tree) { for (;;) { TSTree *lookahead = NULL; TSLength position = ts_length_zero(); + TSPoint offset_point = ts_point_zero(); for (int head = 0; head < ts_stack_head_count(self->stack);) { LookaheadState *state = vector_get(&self->lookahead_states, head); @@ -497,7 +501,8 @@ TSTree *ts_parser_parse(TSParser *self, TSInput input, TSTree *previous_tree) { lookahead = reused_lookahead; } else { position = state->position; - ts_lexer_reset(&self->lexer, position); + offset_point = state->offset_point; + ts_lexer_reset(&self->lexer, position, offset_point); TSStateId parse_state = ts_stack_top_state(self->stack, head); TSStateId lex_state = self->language->lex_states[parse_state]; lookahead = self->language->lex_fn(&self->lexer, lex_state); diff --git a/src/runtime/tree.c b/src/runtime/tree.c index 0591ba03..c78f8f84 100644 --- a/src/runtime/tree.c +++ b/src/runtime/tree.c @@ -7,6 +7,8 @@ #include "runtime/length.h" TSTree *ts_tree_make_leaf(TSSymbol sym, TSLength padding, TSLength size, + TSPoint padding_point, + TSPoint size_point, TSNodeType node_type) { TSTree *result = malloc(sizeof(TSTree)); *result = (TSTree){ @@ -18,6 +20,8 @@ TSTree *ts_tree_make_leaf(TSSymbol sym, TSLength padding, TSLength size, .named_child_count = 0, .children = NULL, .padding = padding, + .padding_point = padding_point, + .size_point = size_point, .options = {.type = node_type }, }; @@ -29,24 +33,31 @@ TSTree *ts_tree_make_leaf(TSSymbol sym, TSLength padding, TSLength size, return result; } -TSTree *ts_tree_make_error(TSLength size, TSLength padding, char lookahead_char) { +TSTree *ts_tree_make_error(TSLength size, TSLength padding, + TSPoint size_point, + TSPoint padding_point, + char lookahead_char) { TSTree *result = - ts_tree_make_leaf(ts_builtin_sym_error, padding, size, TSNodeTypeNamed); + ts_tree_make_leaf(ts_builtin_sym_error, padding, size, padding_point, + size_point, TSNodeTypeNamed); result->lookahead_char = lookahead_char; return result; } void ts_tree_assign_parents(TSTree *self) { TSLength offset = ts_length_zero(); + TSPoint offset_point = ts_point_zero(); for (size_t i = 0; i < self->child_count; i++) { TSTree *child = self->children[i]; if (child->context.parent != self) { child->context.parent = self; child->context.index = i; child->context.offset = offset; + child->context.offset_point = offset_point; ts_tree_assign_parents(child); } offset = ts_length_add(offset, ts_tree_total_size(child)); + offset_point = ts_point_add(offset_point, ts_tree_total_size_point(child)); } } @@ -62,9 +73,12 @@ static void ts_tree__set_children(TSTree *self, TSTree **children, if (i == 0) { self->padding = child->padding; self->size = child->size; + self->padding_point = child->padding_point; + self->size_point = child->size_point; } else { self->size = ts_length_add(ts_length_add(self->size, child->padding), child->size); + self->size_point = ts_point_add(ts_point_add(self->size_point, child->padding_point), child->size_point); } switch (child->options.type) { @@ -93,7 +107,7 @@ static void ts_tree__set_children(TSTree *self, TSTree **children, TSTree *ts_tree_make_node(TSSymbol symbol, size_t child_count, TSTree **children, TSNodeType node_type) { TSTree *result = - ts_tree_make_leaf(symbol, ts_length_zero(), ts_length_zero(), node_type); + ts_tree_make_leaf(symbol, ts_length_zero(), ts_length_zero(), ts_point_zero(), ts_point_zero(), node_type); ts_tree__set_children(result, children, child_count); return result; } @@ -115,10 +129,34 @@ void ts_tree_release(TSTree *self) { } } +size_t ts_tree_offset_column(const TSTree *self) { + size_t column = self->padding_point.column; + + if (self->padding_point.row > 0) { + return column; + } + + const TSTree *parent = self; + TSPoint offset_point; + do { + offset_point = parent->context.offset_point; + column += offset_point.column; + + parent = parent->context.parent; + if (!parent) break; + } while (offset_point.row == 0); + + return column; +} + TSLength ts_tree_total_size(const TSTree *self) { return ts_length_add(self->padding, self->size); } +TSPoint ts_tree_total_size_point(const TSTree *self) { + return ts_point_add(self->padding_point, self->size_point); +} + bool ts_tree_eq(const TSTree *self, const TSTree *other) { if (self) { if (!other) diff --git a/src/runtime/tree.h b/src/runtime/tree.h index 93bfdecf..817ee64a 100644 --- a/src/runtime/tree.h +++ b/src/runtime/tree.h @@ -13,6 +13,7 @@ struct TSTree { struct TSTree *parent; size_t index; TSLength offset; + TSPoint offset_point; } context; size_t child_count; size_t visible_child_count; @@ -23,7 +24,12 @@ struct TSTree { }; TSLength padding; TSLength size; + + TSPoint padding_point; + TSPoint size_point; + TSSymbol symbol; + struct { TSNodeType type : 2; bool extra : 1; @@ -34,16 +40,22 @@ struct TSTree { unsigned short int ref_count; }; -TSTree *ts_tree_make_leaf(TSSymbol, TSLength, TSLength, TSNodeType); +TSTree *ts_tree_make_leaf(TSSymbol, TSLength, TSLength, TSPoint, + TSPoint, TSNodeType); TSTree *ts_tree_make_node(TSSymbol, size_t, TSTree **, TSNodeType); -TSTree *ts_tree_make_error(TSLength size, TSLength padding, char lookahead_char); +TSTree *ts_tree_make_error(TSLength size, TSLength padding, + TSPoint padding_point, + TSPoint size_point, char lookahead_char); void ts_tree_retain(TSTree *tree); void ts_tree_release(TSTree *tree); bool ts_tree_eq(const TSTree *tree1, const TSTree *tree2); int ts_tree_compare(const TSTree *tree1, const TSTree *tree2); char *ts_tree_string(const TSTree *tree, const char **names, bool include_anonymous); + +size_t ts_tree_offset_column(const TSTree *self); TSLength ts_tree_total_size(const TSTree *tree); +TSPoint ts_tree_total_size_point(const TSTree *self); void ts_tree_prepend_children(TSTree *, size_t, TSTree **); void ts_tree_assign_parents(TSTree *); void ts_tree_edit(TSTree *, TSInputEdit);