diff --git a/include/tree_sitter/parser.h b/include/tree_sitter/parser.h index 553e6e40..4d8cf0fb 100644 --- a/include/tree_sitter/parser.h +++ b/include/tree_sitter/parser.h @@ -8,6 +8,8 @@ extern "C" { #include #include "tree_sitter/runtime.h" +typedef struct TSTree TSTree; + #define ts_lex_state_error 0 typedef struct { diff --git a/include/tree_sitter/runtime.h b/include/tree_sitter/runtime.h index 203998e0..3adc0af8 100644 --- a/include/tree_sitter/runtime.h +++ b/include/tree_sitter/runtime.h @@ -7,30 +7,6 @@ extern "C" { #include -typedef unsigned short TSSymbol; -#define ts_builtin_sym_error 0 -#define ts_builtin_sym_end 1 -#define ts_start_sym 2 - -typedef struct TSTree TSTree; - -typedef enum { - TSTreeOptionsHidden = 1, - TSTreeOptionsExtra = 2, - TSTreeOptionsWrapper = 4, -} TSTreeOptions; - -TSTree * ts_tree_make_leaf(TSSymbol symbol, size_t size, size_t offset, int is_hidden); -TSTree * ts_tree_make_node(TSSymbol symbol, size_t child_count, TSTree **children, int is_hidden); -TSTree * ts_tree_make_error(char lookahead_char, size_t expected_input_count, const TSSymbol *expected_inputs, size_t size, size_t offset); -void ts_tree_retain(TSTree *tree); -void ts_tree_release(TSTree *tree); -int ts_tree_equals(const TSTree *tree1, const TSTree *tree2); -char * ts_tree_string(const TSTree *tree, const char **names); -char * ts_tree_error_string(const TSTree *tree, const char **names); -TSTree ** ts_tree_children(const TSTree *tree, size_t *count); -size_t ts_tree_total_size(const TSTree *tree); - typedef struct { void *data; const char * (* read_fn)(void *data, size_t *bytes_read); @@ -44,18 +20,38 @@ typedef struct { size_t bytes_removed; } TSInputEdit; -typedef struct TSParser TSParser; +typedef unsigned short TSSymbol; +typedef struct TSNode TSNode; +typedef struct TSParser TSParser; typedef struct TSDocument TSDocument; + +size_t ts_node_pos(const TSNode *); +size_t ts_node_size(const TSNode *); +TSSymbol ts_node_sym(const TSNode *); +const char * ts_node_name(const TSNode *); +TSNode * ts_node_child(TSNode *, size_t); +TSNode * ts_node_leaf_at_pos(TSNode *, size_t); +TSNode * ts_node_parent(TSNode *node); +TSNode * ts_node_next(TSNode *node); +TSNode * ts_node_prev(TSNode *node); +void ts_node_retain(TSNode *node); +void ts_node_release(TSNode *node); +const char * ts_node_string(const TSNode *); + TSDocument * ts_document_make(); void ts_document_free(TSDocument *doc); void ts_document_set_parser(TSDocument *doc, TSParser *parser); void ts_document_set_input(TSDocument *doc, TSInput input); void ts_document_set_input_string(TSDocument *doc, const char *text); void ts_document_edit(TSDocument *doc, TSInputEdit edit); -const TSTree * ts_document_tree(const TSDocument *doc); const char * ts_document_string(const TSDocument *doc); -const char * ts_document_symbol_name(const TSDocument *document, const TSTree *tree); +TSNode * ts_document_get_node(const TSDocument *document, size_t position); +TSNode * ts_document_root_node(const TSDocument *document); + +#define ts_builtin_sym_error 0 +#define ts_builtin_sym_end 1 +#define ts_start_sym 2 #ifdef __cplusplus } diff --git a/spec/runtime/document_spec.cc b/spec/runtime/document_spec.cc index 88650c1e..e0c204a2 100644 --- a/spec/runtime/document_spec.cc +++ b/spec/runtime/document_spec.cc @@ -1,5 +1,6 @@ #include "runtime/runtime_spec_helper.h" #include "runtime/helpers/spy_reader.h" +#include "runtime/tree.h" extern "C" TSParser * ts_parser_json(); @@ -7,72 +8,109 @@ START_TEST describe("incremental parsing", [&]() { TSDocument *doc; - SpyReader *reader; before_each([&]() { doc = ts_document_make(); ts_document_set_parser(doc, ts_parser_json()); - - reader = new SpyReader("{ \"key\": [1, 2] }", 5); - ts_document_set_input(doc, reader->input); }); after_each([&]() { ts_document_free(doc); - delete reader; }); - it("parses the input", [&]() { - AssertThat(string(ts_document_string(doc)), Equals( - "(object (string) (array (number) (number)))")); - }); + describe("incremental parsing", [&]() { + SpyReader *reader; - it("reads the entire input", [&]() { - AssertThat(reader->strings_read, Equals(vector({ - "{ \"key\": [1, 2] }" - }))); - }); - - describe("modifying the end of the input", [&]() { before_each([&]() { - size_t position(string("{ \"key\": [1, 2]").length()); - string inserted_text(", \"key2\": 4"); - - reader->content.insert(position, inserted_text); - ts_document_edit(doc, { position, 0, inserted_text.length() }); + reader = new SpyReader("{ \"key\": [1, 2] }", 5); + ts_document_set_input(doc, reader->input); }); - it("updates the parse tree", [&]() { + after_each([&]() { + delete reader; + }); + + it("parses the input", [&]() { AssertThat(string(ts_document_string(doc)), Equals( - "(object (string) (array (number) (number)) (string) (number))")); + "(object (string) (array (number) (number)))")); }); - it("re-reads only the changed portion of the input", [&]() { - AssertThat(reader->strings_read.size(), Equals(2)); - AssertThat(reader->strings_read[1], Equals(", \"key2\": 4 }")); + it("reads the entire input", [&]() { + AssertThat(reader->strings_read, Equals(vector({ + "{ \"key\": [1, 2] }" + }))); + }); + + describe("modifying the end of the input", [&]() { + before_each([&]() { + size_t position(string("{ \"key\": [1, 2]").length()); + string inserted_text(", \"key2\": 4"); + + reader->content.insert(position, inserted_text); + ts_document_edit(doc, { position, 0, inserted_text.length() }); + }); + + it("updates the parse tree", [&]() { + AssertThat(string(ts_document_string(doc)), Equals( + "(object (string) (array (number) (number)) (string) (number))")); + }); + + it("re-reads only the changed portion of the input", [&]() { + AssertThat(reader->strings_read.size(), Equals(2)); + AssertThat(reader->strings_read[1], Equals(", \"key2\": 4 }")); + }); + }); + + describe("modifying the beginning of the input", [&]() { + before_each([&]() { + size_t position(string("{ ").length()); + string inserted_text("\"key2\": 4, "); + + reader->content.insert(position, inserted_text); + ts_document_edit(doc, { position, 0, inserted_text.length() }); + }); + + it("updates the parse tree", [&]() { + AssertThat(string(ts_document_string(doc)), Equals( + "(object (string) (number) (string) (array (number) (number)))")); + }); + + it_skip("re-reads only the changed portion of the input", [&]() { + AssertThat(reader->strings_read.size(), Equals(2)); + AssertThat(reader->strings_read[1], Equals("\"key2\": 4, ")); + }); }); }); - describe("modifying the beginning of the input", [&]() { - before_each([&]() { - size_t position(string("{ ").length()); - string inserted_text("\"key2\": 4, "); + it("records the widths and offsets of nodes", [&]() { + ts_document_set_input_string(doc, " [12, 5, 345]"); - reader->content.insert(position, inserted_text); - ts_document_edit(doc, { position, 0, inserted_text.length() }); - }); + TSNode *array = ts_document_root_node(doc); + TSNode *number1 = ts_node_child(array, 0); + TSNode *number2 = ts_node_child(array, 1); + TSNode *number3 = ts_node_child(array, 2); - it("2 updates the parse tree", [&]() { - AssertThat(string(ts_document_string(doc)), Equals( - "(object (string) (number) (string) (array (number) (number)))")); - }); + AssertThat(ts_node_name(array), Equals("array")); + AssertThat(ts_node_name(number1), Equals("number")); + AssertThat(ts_node_name(number2), Equals("number")); - it_skip("re-reads only the changed portion of the input", [&]() { - AssertThat(reader->strings_read.size(), Equals(2)); - AssertThat(reader->strings_read[1], Equals("\"key2\": 4, ")); - }); + AssertThat(ts_node_pos(array), Equals(2)); + AssertThat(ts_node_size(array), Equals(12)); + + AssertThat(ts_node_pos(number1), Equals(3)); + AssertThat(ts_node_size(number1), Equals(2)); + + AssertThat(ts_node_pos(number2), Equals(7)); + AssertThat(ts_node_size(number2), Equals(1)); + + AssertThat(ts_node_pos(number3), Equals(10)); + AssertThat(ts_node_size(number3), Equals(3)); + + ts_node_release(array); + ts_node_release(number1); + ts_node_release(number2); + ts_node_release(number3); }); - }); END_TEST diff --git a/spec/runtime/helpers/tree_helpers.h b/spec/runtime/helpers/tree_helpers.h index 0a9c036a..34fbdc92 100644 --- a/spec/runtime/helpers/tree_helpers.h +++ b/spec/runtime/helpers/tree_helpers.h @@ -1,10 +1,9 @@ #ifndef HELPERS_TREE_HELPERS_H_ #define HELPERS_TREE_HELPERS_H_ -#include "tree_sitter/runtime.h" - +#include "tree_sitter/parser.h" #include TSTree ** tree_array(std::vector trees); -#endif // HELPERS_TREE_HELPERS_H_ \ No newline at end of file +#endif // HELPERS_TREE_HELPERS_H_ diff --git a/spec/runtime/language_specs.cc b/spec/runtime/languages/language_specs.cc similarity index 100% rename from spec/runtime/language_specs.cc rename to spec/runtime/languages/language_specs.cc diff --git a/spec/runtime/node_position_spec.cc b/spec/runtime/node_position_spec.cc deleted file mode 100644 index e60586cc..00000000 --- a/spec/runtime/node_position_spec.cc +++ /dev/null @@ -1,46 +0,0 @@ -#include "runtime/runtime_spec_helper.h" -#include "runtime/tree.h" - -extern "C" TSParser * ts_parser_json(); - -START_TEST - -describe("tracking the positions of AST nodes", []() { - TSDocument *doc; - - before_each([&]() { - doc = ts_document_make(); - ts_document_set_parser(doc, ts_parser_json()); - }); - - after_each([&]() { - ts_document_free(doc); - }); - - it("records the widths and offsets of nodes", [&]() { - ts_document_set_input_string(doc, " [12, 5]"); - - const TSTree *tree = ts_document_tree(doc); - const TSTree *array = tree->children[0]; - const TSTree *number1 = array->children[1]->children[0]; - const TSTree *number2 = array->children[2]->children[1]->children[0]; - - AssertThat(ts_document_symbol_name(doc, array), Equals("array")); - AssertThat(ts_document_symbol_name(doc, number1), Equals("number")); - AssertThat(ts_document_symbol_name(doc, number2), Equals("number")); - - AssertThat(number1->offset, Equals(0)); - AssertThat(number1->size, Equals(2)); - - AssertThat(number2->offset, Equals(1)); - AssertThat(number2->size, Equals(1)); - - AssertThat(array->offset, Equals(2)); - AssertThat(array->size, Equals(7)); - - AssertThat(tree->offset, Equals(2)); - AssertThat(tree->size, Equals(7)); - }); -}); - -END_TEST diff --git a/spec/runtime/node_spec.cc b/spec/runtime/node_spec.cc new file mode 100644 index 00000000..611396b9 --- /dev/null +++ b/spec/runtime/node_spec.cc @@ -0,0 +1,53 @@ +#include "runtime/runtime_spec_helper.h" + +extern "C" TSParser * ts_parser_arithmetic(); + +START_TEST + +describe("Node", []() { + TSDocument *document; + + before_each([&]() { + document = ts_document_make(); + ts_document_set_parser(document, ts_parser_arithmetic()); + }); + + after_each([&]() { + ts_document_free(document); + }); + + describe("getting the nth child node", [&]() { + TSNode *root; + + describe("when the child has more than n visible children", [&]() { + before_each([&]() { + ts_document_set_input_string(document, "x + 1"); + root = ts_document_root_node(document); + + AssertThat(ts_node_name(root), Equals("sum")); + AssertThat(ts_node_string(root), Equals("(sum (variable) (number))")); + }); + + after_each([&]() { + ts_node_release(root); + }); + + it("returns the nth child", [&]() { + TSNode *child1 = ts_node_child(root, 0); + AssertThat(ts_node_name(child1), Equals("variable")); + + TSNode *child2 = ts_node_child(root, 1); + AssertThat(ts_node_name(child2), Equals("number")); + + ts_node_release(child1); + ts_node_release(child2); + }); + }); + }); + + it("gets the first token", [&]() { + // ts_document_get_node(document, 0); + }); +}); + +END_TEST diff --git a/src/runtime/document.c b/src/runtime/document.c index 4a246b4d..cdfa8c86 100644 --- a/src/runtime/document.c +++ b/src/runtime/document.c @@ -1,6 +1,7 @@ #include "tree_sitter/runtime.h" #include "tree_sitter/parser.h" #include "runtime/tree.h" +#include "runtime/node.h" #include struct TSDocument { @@ -11,11 +12,16 @@ struct TSDocument { }; TSDocument * ts_document_make() { - return malloc(sizeof(TSDocument)); + TSDocument *document = malloc(sizeof(TSDocument)); + *document = (TSDocument) { + .input = (TSInput) {} + }; + return document; } void ts_document_free(TSDocument *document) { - ts_parser_free(document->parser); + if (document->parser) + ts_parser_free(document->parser); if (document->input.release_fn) document->input.release_fn(document->input.data); free(document); @@ -87,3 +93,29 @@ TSInput ts_string_input_make(const char *string) { void ts_document_set_input_string(TSDocument *document, const char *text) { ts_document_set_input(document, ts_string_input_make(text)); } + +TSNode * ts_document_root_node(const TSDocument *document) { + const TSTree *tree = document->tree; + size_t position = 0; + while (ts_tree_is_wrapper(tree)) { + position = tree->offset; + tree = tree->children[0]; + } + + TSNode *result = malloc(sizeof(TSNode)); + *result = (TSNode) { + .ref_count = 1, + .position = position, + .content = tree, + .parent = NULL, + .config = &document->parser->config, + }; + return result; +} + +TSNode * ts_document_get_node(const TSDocument *document, size_t pos) { + TSNode *root = ts_document_root_node(document); + TSNode *result = ts_node_leaf_at_pos(root, pos); + ts_node_release(root); + return result; +} diff --git a/src/runtime/lexer.c b/src/runtime/lexer.c index 95291616..a659058e 100644 --- a/src/runtime/lexer.c +++ b/src/runtime/lexer.c @@ -1,5 +1,5 @@ -#include "tree_sitter/runtime.h" #include "tree_sitter/parser.h" +#include "runtime/tree.h" TSLexer ts_lexer_make() { return (TSLexer) { diff --git a/src/runtime/node.c b/src/runtime/node.c new file mode 100644 index 00000000..23716023 --- /dev/null +++ b/src/runtime/node.c @@ -0,0 +1,95 @@ +#include "runtime/node.h" +#include "tree_sitter/parser.h" +#include "runtime/tree.h" + +TSNode * ts_node_make(TSTree *tree, TSNode *parent, size_t position, TSParserConfig *config) { + if (parent) ts_node_retain(parent); + TSNode *result = malloc(sizeof(TSNode)); + *result = (TSNode) { + .ref_count = 1, + .parent = parent, + .content = tree, + .position = position, + .config = config + }; + return result; +} + +void ts_node_retain(TSNode *node) { + node->ref_count++; +} + +void ts_node_release(TSNode *node) { + node->ref_count--; + if (node->ref_count == 0) { + if (node->parent) ts_node_release(node->parent); + free(node); + } +} + +size_t ts_node_pos(const TSNode *node) { + return node->position; +} + +size_t ts_node_size(const TSNode *node) { + return node->content->size; +} + +const char * ts_node_name(const TSNode *node) { + return node->config->symbol_names[node->content->symbol]; +} + +TSTree * ts_tree_child(const TSTree *tree, size_t goal_index, size_t *offset, size_t *children_seen) { + *offset = 0; + *children_seen = 0; + size_t child_count; + TSTree **children = ts_tree_children(tree, &child_count); + + if (!children) { + *offset = tree->offset + tree->size; + } + + for (size_t i = 0; i < child_count; i++) { + size_t delta_index = 0, delta_offset = 0; + TSTree *child = children[i]; + TSTree *result = NULL; + + if (ts_tree_is_visible(child)) { + delta_offset = child->offset; + if (*children_seen == goal_index) { + result = child; + } else { + delta_index = 1; + delta_offset += child->size; + } + } else { + result = ts_tree_child(child, (goal_index - *children_seen), &delta_offset, &delta_index); + } + + *offset += delta_offset; + *children_seen += delta_index; + + if (result) { + return result; + } + } + + return NULL; +} + +const char * ts_node_string(const TSNode *node) { + return ts_tree_string(node->content, node->config->symbol_names); +} + +TSNode * ts_node_child(TSNode *parent, size_t goal_index) { + size_t offset, index; + TSTree *child = ts_tree_child(parent->content, goal_index, &offset, &index); + if (child) + return ts_node_make(child, parent, offset, parent->config); + else + return NULL; +} + +TSNode * ts_node_leaf_at_pos(TSNode *parent, size_t child_index) { + return NULL; +} diff --git a/src/runtime/node.h b/src/runtime/node.h new file mode 100644 index 00000000..607e3236 --- /dev/null +++ b/src/runtime/node.h @@ -0,0 +1,17 @@ +#ifndef RUNTIME_NODE_H_ +#define RUNTIME_NODE_H_ + +#include "tree_sitter/parser.h" + +struct TSNode { + size_t ref_count; + size_t position; + size_t index; + const TSTree *content; + struct TSNode *parent; + TSParserConfig *config; +}; + +TSNode * ts_node_make(TSTree *tree, TSNode *parent, size_t position, TSParserConfig *config); + +#endif diff --git a/src/runtime/tree.c b/src/runtime/tree.c index 0f4d84b6..8f273b8e 100644 --- a/src/runtime/tree.c +++ b/src/runtime/tree.c @@ -1,6 +1,6 @@ -#include "tree_sitter/runtime.h" #include #include +#include "tree_sitter/parser.h" #include "runtime/tree.h" static TSTree * ts_tree_make(TSSymbol symbol, size_t size, size_t offset, int is_hidden) { diff --git a/src/runtime/tree.h b/src/runtime/tree.h index 41de2791..e9d8695c 100644 --- a/src/runtime/tree.h +++ b/src/runtime/tree.h @@ -1,8 +1,18 @@ #ifndef RUNTIME_TREE_H_ #define RUNTIME_TREE_H_ +#ifdef __cplusplus +extern "C" { +#endif + #include "tree_sitter/runtime.h" +typedef enum { + TSTreeOptionsHidden = 1, + TSTreeOptionsExtra = 2, + TSTreeOptionsWrapper = 4, +} TSTreeOptions; + struct TSTree { TSSymbol symbol; TSTreeOptions options; @@ -38,4 +48,19 @@ static inline int ts_tree_is_wrapper(const TSTree *tree) { return (tree->options & TSTreeOptionsWrapper); } +TSTree * ts_tree_make_leaf(TSSymbol symbol, size_t size, size_t offset, int is_hidden); +TSTree * ts_tree_make_node(TSSymbol symbol, size_t child_count, TSTree **children, int is_hidden); +TSTree * ts_tree_make_error(char lookahead_char, size_t expected_input_count, const TSSymbol *expected_inputs, size_t size, size_t offset); +void ts_tree_retain(TSTree *tree); +void ts_tree_release(TSTree *tree); +int ts_tree_equals(const TSTree *tree1, const TSTree *tree2); +char * ts_tree_string(const TSTree *tree, const char **names); +char * ts_tree_error_string(const TSTree *tree, const char **names); +TSTree ** ts_tree_children(const TSTree *tree, size_t *count); +size_t ts_tree_total_size(const TSTree *tree); + +#ifdef __cplusplus +} +#endif + #endif // RUNTIME_TREE_H_