Start work on unit test that edits python code

Signed-off-by: Nathan Sobo <nathan@github.com>

spec/helpers/dedent.h

@@ -0,0 +1,12 @@
+#include "compiler/util/string_helpers.h"
+#include <string>
+
+static std::string dedent(std::string input) {
+  size_t indent_level = input.find_first_not_of("\n ") - input.find_first_not_of("\n");
+  std::string whitespace = "\n" + std::string(indent_level, ' ');
+  tree_sitter::util::str_replace(&input, whitespace, "\n");
+  return input.substr(
+    input.find_first_not_of("\n "),
+    input.find_last_not_of("\n ") + 1
+  );
+}

spec/integration/compile_grammar_spec.cc

@@ -2,19 +2,10 @@
 #include "runtime/alloc.h"
 #include "helpers/load_language.h"
 #include "helpers/stderr_logger.h"
+#include "helpers/dedent.h"
 #include "compiler/util/string_helpers.h"
 #include <map>
 
-static string dedent(string input) {
-  size_t indent_level = input.find_first_not_of("\n ") - input.find_first_not_of("\n");
-  string whitespace = "\n" + string(indent_level, ' ');
-  util::str_replace(&input, whitespace, "\n");
-  return input.substr(
-    input.find_first_not_of("\n "),
-    input.find_last_not_of("\n ") + 1
-  );
-}
-
 static string fill_template(string input, map<string, string> parameters) {
   string result = input;
   for (const auto &pair : parameters) {

spec/runtime/parser_spec.cc

@@ -4,6 +4,8 @@
 #include "helpers/spy_input.h"
 #include "helpers/load_language.h"
 #include "helpers/record_alloc.h"
+#include "helpers/stderr_logger.h"
+#include "helpers/dedent.h"
 
 START_TEST
 
@@ -33,13 +35,13 @@ describe("Parser", [&]() {
     AssertThat(record_alloc::outstanding_allocation_indices(), IsEmpty());
   });
 
-  auto set_text = [&](const char *text) {
+  auto set_text = [&](string text) {
     input = new SpyInput(text, chunk_size);
     ts_document_set_input(doc, input->input());
     ts_document_parse(doc);
 
     root = ts_document_root_node(doc);
-    AssertThat(ts_node_end_byte(root), Equals(strlen(text)));
+    AssertThat(ts_node_end_byte(root), Equals(text.size()));
     input->clear();
   };
 
@@ -404,6 +406,28 @@ describe("Parser", [&]() {
       });
     });
 
+    describe("with external tokens", [&]() {
+      before_each([&]() {
+        ts_document_set_language(doc, get_test_language("python"));
+      });
+
+      it("maintains the external scanner's state during incremental parsing", [&]() {
+        string text = dedent(R"PYTHON(
+          if a:
+              print b
+
+          return c
+        )PYTHON");
+
+        set_text(text);
+
+        assert_root_node("(module "
+          "(if_statement (identifier) "
+            "(print_statement (identifier))) "
+          "(return_statement (expression_list (identifier))))");
+      });
+    });
+
     it("updates the document's parse count", [&]() {
       ts_document_set_language(doc, get_test_language("javascript"));
       AssertThat(ts_document_parse_count(doc), Equals<size_t>(0));