diff --git a/spec/runtime/parser_spec.cc b/spec/runtime/parser_spec.cc index 2e4c9d20..c06f5322 100644 --- a/spec/runtime/parser_spec.cc +++ b/spec/runtime/parser_spec.cc @@ -20,16 +20,12 @@ describe("Parser", [&]() { chunk_size = 3; input = nullptr; - document = ts_document_new(); }); after_each([&]() { - if (document) - ts_document_free(document); - - if (input) - delete input; + if (document) ts_document_free(document); + if (input) delete input; record_alloc::stop(); AssertThat(record_alloc::outstanding_allocation_indices(), IsEmpty()); @@ -45,26 +41,6 @@ describe("Parser", [&]() { input->clear(); }; - auto insert_text = [&](size_t position, string text) { - size_t prev_size = ts_node_end_byte(root); - ts_document_edit(document, input->replace(position, 0, text)); - ts_document_parse(document); - - root = ts_document_root_node(document); - size_t new_size = ts_node_end_byte(root); - AssertThat(new_size, Equals(prev_size + text.size())); - }; - - auto delete_text = [&](size_t position, size_t length) { - size_t prev_size = ts_node_end_byte(root); - ts_document_edit(document, input->replace(position, length, "")); - ts_document_parse(document); - - root = ts_document_root_node(document); - size_t new_size = ts_node_end_byte(root); - AssertThat(new_size, Equals(prev_size - length)); - }; - auto replace_text = [&](size_t position, size_t length, string new_text) { size_t prev_size = ts_node_end_byte(root); @@ -76,21 +52,29 @@ describe("Parser", [&]() { AssertThat(new_size, Equals(prev_size - length + new_text.size())); }; + auto insert_text = [&](size_t position, string text) { + replace_text(position, 0, text); + }; + + auto delete_text = [&](size_t position, size_t length) { + replace_text(position, length, ""); + }; + auto assert_root_node = [&](const string &expected) { TSNode node = ts_document_root_node(document); - char *str = ts_node_string(node, document); - string actual(str); - ts_free(str); + char *node_string = ts_node_string(node, document); + string actual(node_string); + ts_free(node_string); AssertThat(actual, Equals(expected)); }; - describe("handling errors", [&]() { - auto get_node_text = [&](TSNode node) { - size_t start = ts_node_start_byte(node); - size_t end = ts_node_end_byte(node); - return input->content.substr(start, end - start); - }; + auto get_node_text = [&](TSNode node) { + size_t start = ts_node_start_byte(node); + size_t end = ts_node_end_byte(node); + return input->content.substr(start, end - start); + }; + describe("handling errors", [&]() { describe("when there is an invalid substring right before a valid token", [&]() { it("computes the error node's size and position correctly", [&]() { ts_document_set_language(document, get_test_language("json")); @@ -183,7 +167,7 @@ describe("Parser", [&]() { describe("handling extra tokens", [&]() { describe("when the token appears as part of a grammar rule", [&]() { - it("is incorporated into the tree", [&]() { + it("incorporates it into the tree", [&]() { ts_document_set_language(document, get_test_language("javascript")); set_text("fn()\n"); @@ -193,7 +177,7 @@ describe("Parser", [&]() { }); describe("when the token appears somewhere else", [&]() { - it("is incorporated into the tree", [&]() { + it("incorporates it into the tree", [&]() { ts_document_set_language(document, get_test_language("javascript")); set_text( "fn()\n" @@ -209,7 +193,7 @@ describe("Parser", [&]() { }); describe("when several extra tokens appear in a row", [&]() { - it("is incorporated into the tree", [&]() { + it("incorporates them into the tree", [&]() { ts_document_set_language(document, get_test_language("javascript")); set_text( "fn()\n\n" @@ -229,177 +213,156 @@ describe("Parser", [&]() { }); describe("editing", [&]() { - describe("inserting text", [&]() { - describe("creating new tokens near the end of the input", [&]() { - it("updates the parse tree and re-reads only the changed portion of the text", [&]() { - ts_document_set_language(document, get_test_language("javascript")); - set_text("x * (100 + abc);"); - - assert_root_node( - "(program (expression_statement (math_op " - "(identifier) " - "(math_op (number) (identifier)))))"); - - insert_text(strlen("x * (100 + abc"), ".d"); - - assert_root_node( - "(program (expression_statement (math_op " - "(identifier) " - "(math_op (number) (member_access (identifier) (identifier))))))"); - - AssertThat(input->strings_read, Equals(vector({ " + abc.d)", "" }))); - }); - }); - - describe("creating new tokens near the beginning of the input", [&]() { - it("updates the parse tree and re-reads only the changed portion of the input", [&]() { - chunk_size = 2; - - ts_document_set_language(document, get_test_language("javascript")); - set_text("123 + 456 * (10 + x);"); - - assert_root_node( - "(program (expression_statement (math_op " - "(number) " - "(math_op (number) (math_op (number) (identifier))))))"); - - insert_text(strlen("123"), " || 5"); - - assert_root_node( - "(program (expression_statement (bool_op " - "(number) " - "(math_op " - "(number) " - "(math_op (number) (math_op (number) (identifier)))))))"); - - AssertThat(input->strings_read, Equals(vector({ "123 || 5 +", "" }))); - }); - }); - - describe("introducing an error", [&]() { - it("gives the error the right size", [&]() { - ts_document_set_language(document, get_test_language("javascript")); - set_text("var x = y;"); - - assert_root_node( - "(program (var_declaration (var_assignment " - "(identifier) (identifier))))"); - - insert_text(strlen("var x = y"), " *"); - - assert_root_node( - "(program (var_declaration (var_assignment " - "(identifier) (identifier)) (ERROR)))"); - - insert_text(strlen("var x = y *"), " z"); - - assert_root_node( - "(program (var_declaration (var_assignment " - "(identifier) (math_op (identifier) (identifier)))))"); - }); - }); - - describe("into the middle of an existing token", [&]() { - it("updates the parse tree", [&]() { - ts_document_set_language(document, get_test_language("javascript")); - set_text("abc * 123;"); - - assert_root_node( - "(program (expression_statement (math_op (identifier) (number))))"); - - insert_text(strlen("ab"), "XYZ"); - - assert_root_node( - "(program (expression_statement (math_op (identifier) (number))))"); - - TSNode node = ts_node_named_descendant_for_char_range(root, 1, 1); - AssertThat(ts_node_type(node, document), Equals("identifier")); - AssertThat(ts_node_end_byte(node), Equals(strlen("abXYZc"))); - }); - }); - - describe("at the end of an existing token", [&]() { - it("updates the parse tree", [&]() { - ts_document_set_language(document, get_test_language("javascript")); - set_text("abc * 123;"); - - assert_root_node( - "(program (expression_statement (math_op (identifier) (number))))"); - - insert_text(strlen("abc"), "XYZ"); - - assert_root_node( - "(program (expression_statement (math_op (identifier) (number))))"); - - TSNode node = ts_node_named_descendant_for_char_range(root, 1, 1); - AssertThat(ts_node_type(node, document), Equals("identifier")); - AssertThat(ts_node_end_byte(node), Equals(strlen("abcXYZ"))); - }); - }); - - describe("into a node containing a extra token", [&]() { - it("updates the parse tree", [&]() { - ts_document_set_language(document, get_test_language("javascript")); - set_text("123 *\n" - "// a-comment\n" - "abc;"); - - assert_root_node( - "(program (expression_statement (math_op " - "(number) " - "(comment) " - "(identifier))))"); - - insert_text( - strlen("123 *\n" - "// a-comment\n" - "abc"), - "XYZ"); - - assert_root_node( - "(program (expression_statement (math_op " - "(number) " - "(comment) " - "(identifier))))"); - }); - }); - }); - - describe("deleting text", [&]() { - describe("when a critical token is removed", [&]() { - it("updates the parse tree, creating an error", [&]() { - ts_document_set_language(document, get_test_language("javascript")); - set_text("123 * 456; 789 * 123;"); - - assert_root_node( - "(program " - "(expression_statement (math_op (number) (number))) " - "(expression_statement (math_op (number) (number))))"); - - delete_text(strlen("123 "), 2); - - assert_root_node( - "(program " - "(expression_statement (number) (ERROR (number))) " - "(expression_statement (math_op (number) (number))))"); - }); - }); - }); - - describe("replacing text", [&]() { - it("does not try to re-use nodes that are within the edited region", [&]() { + describe("creating new tokens near the end of the input", [&]() { + it("updates the parse tree and re-reads only the changed portion of the text", [&]() { ts_document_set_language(document, get_test_language("javascript")); - set_text("{ x: (b.c) };"); + set_text("x * (100 + abc);"); assert_root_node( - "(program (expression_statement (object (pair " - "(identifier) (member_access (identifier) (identifier))))))"); + "(program (expression_statement (math_op " + "(identifier) " + "(math_op (number) (identifier)))))"); - replace_text(strlen("{ x: "), strlen("(b.c)"), "b.c"); + insert_text(strlen("x * (100 + abc"), ".d"); assert_root_node( - "(program (expression_statement (object (pair " - "(identifier) (member_access (identifier) (identifier))))))"); + "(program (expression_statement (math_op " + "(identifier) " + "(math_op (number) (member_access (identifier) (identifier))))))"); + + AssertThat(input->strings_read, Equals(vector({ " + abc.d)", "" }))); + }); + }); + + describe("creating new tokens near the beginning of the input", [&]() { + it("updates the parse tree and re-reads only the changed portion of the input", [&]() { + chunk_size = 2; + + ts_document_set_language(document, get_test_language("javascript")); + set_text("123 + 456 * (10 + x);"); + + assert_root_node( + "(program (expression_statement (math_op " + "(number) " + "(math_op (number) (math_op (number) (identifier))))))"); + + insert_text(strlen("123"), " || 5"); + + assert_root_node( + "(program (expression_statement (bool_op " + "(number) " + "(math_op " + "(number) " + "(math_op (number) (math_op (number) (identifier)))))))"); + + AssertThat(input->strings_read, Equals(vector({ "123 || 5 +", "" }))); + }); + }); + + describe("introducing an error", [&]() { + it("gives the error the right size", [&]() { + ts_document_set_language(document, get_test_language("javascript")); + set_text("var x = y;"); + + assert_root_node( + "(program (var_declaration (var_assignment " + "(identifier) (identifier))))"); + + insert_text(strlen("var x = y"), " *"); + + assert_root_node( + "(program (var_declaration (var_assignment " + "(identifier) (identifier)) (ERROR)))"); + + insert_text(strlen("var x = y *"), " z"); + + assert_root_node( + "(program (var_declaration (var_assignment " + "(identifier) (math_op (identifier) (identifier)))))"); + }); + }); + + describe("into the middle of an existing token", [&]() { + it("updates the parse tree", [&]() { + ts_document_set_language(document, get_test_language("javascript")); + set_text("abc * 123;"); + + assert_root_node( + "(program (expression_statement (math_op (identifier) (number))))"); + + insert_text(strlen("ab"), "XYZ"); + + assert_root_node( + "(program (expression_statement (math_op (identifier) (number))))"); + + TSNode node = ts_node_named_descendant_for_char_range(root, 1, 1); + AssertThat(ts_node_type(node, document), Equals("identifier")); + AssertThat(ts_node_end_byte(node), Equals(strlen("abXYZc"))); + }); + }); + + describe("at the end of an existing token", [&]() { + it("updates the parse tree", [&]() { + ts_document_set_language(document, get_test_language("javascript")); + set_text("abc * 123;"); + + assert_root_node( + "(program (expression_statement (math_op (identifier) (number))))"); + + insert_text(strlen("abc"), "XYZ"); + + assert_root_node( + "(program (expression_statement (math_op (identifier) (number))))"); + + TSNode node = ts_node_named_descendant_for_char_range(root, 1, 1); + AssertThat(ts_node_type(node, document), Equals("identifier")); + AssertThat(ts_node_end_byte(node), Equals(strlen("abcXYZ"))); + }); + }); + + describe("inserting text into a node containing a extra token", [&]() { + it("updates the parse tree", [&]() { + ts_document_set_language(document, get_test_language("javascript")); + set_text("123 *\n" + "// a-comment\n" + "abc;"); + + assert_root_node( + "(program (expression_statement (math_op " + "(number) " + "(comment) " + "(identifier))))"); + + insert_text( + strlen("123 *\n" + "// a-comment\n" + "abc"), + "XYZ"); + + assert_root_node( + "(program (expression_statement (math_op " + "(number) " + "(comment) " + "(identifier))))"); + }); + }); + + describe("when a critical token is removed", [&]() { + it("updates the parse tree, creating an error", [&]() { + ts_document_set_language(document, get_test_language("javascript")); + set_text("123 * 456; 789 * 123;"); + + assert_root_node( + "(program " + "(expression_statement (math_op (number) (number))) " + "(expression_statement (math_op (number) (number))))"); + + delete_text(strlen("123 "), 2); + + assert_root_node( + "(program " + "(expression_statement (number) (ERROR (number))) " + "(expression_statement (math_op (number) (number))))"); }); }); @@ -426,6 +389,21 @@ describe("Parser", [&]() { }); }); + it("does not try to re-use nodes that are within the edited region", [&]() { + ts_document_set_language(document, get_test_language("javascript")); + set_text("{ x: (b.c) };"); + + assert_root_node( + "(program (expression_statement (object (pair " + "(identifier) (member_access (identifier) (identifier))))))"); + + replace_text(strlen("{ x: "), strlen("(b.c)"), "b.c"); + + assert_root_node( + "(program (expression_statement (object (pair " + "(identifier) (member_access (identifier) (identifier))))))"); + }); + it("updates the document's parse count", [&]() { ts_document_set_language(document, get_test_language("javascript")); AssertThat(ts_document_parse_count(document), Equals(0));