Add TSInput option to measure columns in bytes not characters

include/tree_sitter/runtime.h

@@ -31,6 +31,7 @@ typedef struct {
   const char *(*read)(void *payload, uint32_t *bytes_read);
   int (*seek)(void *payload, uint32_t character_index, uint32_t byte_index);
   TSInputEncoding encoding;
+  bool measure_columns_in_bytes;
 } TSInput;
 
 typedef enum {

src/runtime/document.c

@@ -26,8 +26,13 @@ void ts_document_free(TSDocument *self) {
   parser_destroy(&self->parser);
   if (self->tree)
     ts_tree_release(self->tree);
-  ts_document_set_input(self,
-                        (TSInput){ NULL, NULL, NULL, TSInputEncodingUTF8 });
+  ts_document_set_input(self, (TSInput){
+    NULL,
+    NULL,
+    NULL,
+    TSInputEncodingUTF8,
+    false
+  });
   ts_free(self);
 }
 

src/runtime/lexer.c

@@ -60,6 +60,8 @@ static void ts_lexer__advance(void *payload, bool skip) {
     if (self->data.lookahead == '\n') {
       self->current_position.extent.row++;
       self->current_position.extent.column = 0;
+    } else if (self->input.measure_columns_in_bytes) {
+      self->current_position.extent.column += self->lookahead_size;
     } else {
       self->current_position.extent.column++;
     }

src/runtime/string_input.c

@@ -43,8 +43,9 @@ TSInput ts_string_input_make_with_length(const char *string, uint32_t length) {
     .read = ts_string_input_read,
     .seek = ts_string_input_seek,
     .encoding = TSInputEncodingUTF8,
+    .measure_columns_in_bytes = false,
   };
 
 error:
-  return (TSInput){ NULL, NULL, NULL, TSInputEncodingUTF8 };
+  return (TSInput){ NULL, NULL, NULL, TSInputEncodingUTF8, false };
 }

test/helpers/spy_input.cc

@@ -86,6 +86,7 @@ TSInput SpyInput::input() {
   result.encoding = encoding;
   result.seek = seek;
   result.read = read;
+  result.measure_columns_in_bytes = true;
   return result;
 }
 

test/runtime/document_test.cc

@@ -76,11 +76,22 @@ describe("Document", [&]() {
       const char16_t content[] = u"[true, false]";
       spy_input->content = string((const char *)content, sizeof(content));
       spy_input->encoding = TSInputEncodingUTF16;
-      // spy_input->measure_columns_in_bytes
+      TSInput input = spy_input->input();
 
-      ts_document_set_input(document, spy_input->input());
+      input.measure_columns_in_bytes = false;
+      ts_document_set_input(document, input);
       ts_document_invalidate(document);
       ts_document_parse(document);
+
+      TSNode root = ts_document_root_node(document);
+      AssertThat(ts_node_end_point(root), Equals<TSPoint>({0, 13}));
+
+      input.measure_columns_in_bytes = true;
+      ts_document_set_input(document, input);
+      ts_document_invalidate(document);
+      ts_document_parse(document);
+      root = ts_document_root_node(document);
+      AssertThat(ts_node_end_point(root), Equals<TSPoint>({0, 26}));
     });
 
     it("allows the input to be retrieved later", [&]() {