diff --git a/spec/helpers/dedent.h b/spec/helpers/dedent.h new file mode 100644 index 00000000..1387acf9 --- /dev/null +++ b/spec/helpers/dedent.h @@ -0,0 +1,12 @@ +#include "compiler/util/string_helpers.h" +#include + +static std::string dedent(std::string input) { + size_t indent_level = input.find_first_not_of("\n ") - input.find_first_not_of("\n"); + std::string whitespace = "\n" + std::string(indent_level, ' '); + tree_sitter::util::str_replace(&input, whitespace, "\n"); + return input.substr( + input.find_first_not_of("\n "), + input.find_last_not_of("\n ") + 1 + ); +} diff --git a/spec/integration/compile_grammar_spec.cc b/spec/integration/compile_grammar_spec.cc index 2c7560c0..ed2109c2 100644 --- a/spec/integration/compile_grammar_spec.cc +++ b/spec/integration/compile_grammar_spec.cc @@ -2,19 +2,10 @@ #include "runtime/alloc.h" #include "helpers/load_language.h" #include "helpers/stderr_logger.h" +#include "helpers/dedent.h" #include "compiler/util/string_helpers.h" #include -static string dedent(string input) { - size_t indent_level = input.find_first_not_of("\n ") - input.find_first_not_of("\n"); - string whitespace = "\n" + string(indent_level, ' '); - util::str_replace(&input, whitespace, "\n"); - return input.substr( - input.find_first_not_of("\n "), - input.find_last_not_of("\n ") + 1 - ); -} - static string fill_template(string input, map parameters) { string result = input; for (const auto &pair : parameters) { diff --git a/spec/runtime/parser_spec.cc b/spec/runtime/parser_spec.cc index a14fa68e..969ac078 100644 --- a/spec/runtime/parser_spec.cc +++ b/spec/runtime/parser_spec.cc @@ -4,6 +4,8 @@ #include "helpers/spy_input.h" #include "helpers/load_language.h" #include "helpers/record_alloc.h" +#include "helpers/stderr_logger.h" +#include "helpers/dedent.h" START_TEST @@ -33,13 +35,13 @@ describe("Parser", [&]() { AssertThat(record_alloc::outstanding_allocation_indices(), IsEmpty()); }); - auto set_text = [&](const char *text) { + auto set_text = [&](string text) { input = new SpyInput(text, chunk_size); ts_document_set_input(doc, input->input()); ts_document_parse(doc); root = ts_document_root_node(doc); - AssertThat(ts_node_end_byte(root), Equals(strlen(text))); + AssertThat(ts_node_end_byte(root), Equals(text.size())); input->clear(); }; @@ -404,6 +406,28 @@ describe("Parser", [&]() { }); }); + describe("with external tokens", [&]() { + before_each([&]() { + ts_document_set_language(doc, get_test_language("python")); + }); + + it("maintains the external scanner's state during incremental parsing", [&]() { + string text = dedent(R"PYTHON( + if a: + print b + + return c + )PYTHON"); + + set_text(text); + + assert_root_node("(module " + "(if_statement (identifier) " + "(print_statement (identifier))) " + "(return_statement (expression_list (identifier))))"); + }); + }); + it("updates the document's parse count", [&]() { ts_document_set_language(doc, get_test_language("javascript")); AssertThat(ts_document_parse_count(doc), Equals(0));