Support UTF16 directly

This makes the API easier to use from javascript

src/runtime/lexer.c

@@ -4,6 +4,7 @@
 #include "runtime/tree.h"
 #include "runtime/length.h"
 #include "runtime/debugger.h"
+#include "runtime/utf16.h"
 #include "utf8proc.h"
 
 #define LOG(...)                                                     \
@@ -18,7 +19,7 @@
                                                      : "lookahead char:%d",  \
       self->lookahead);
 
-static const char *empty_chunk = "";
+static const char empty_chunk[2] = { 0, 0 };
 
 static void ts_lexer__get_chunk(TSLexer *self) {
   TSInput input = self->input;
@@ -35,9 +36,14 @@ static void ts_lexer__get_chunk(TSLexer *self) {
 
 static void ts_lexer__get_lookahead(TSLexer *self) {
   size_t position_in_chunk = self->current_position.bytes - self->chunk_start;
-  self->lookahead_size = utf8proc_iterate(
-    (const uint8_t *)self->chunk + position_in_chunk,
-    self->chunk_size - position_in_chunk + 1, &self->lookahead);
+  const uint8_t *chunk = (const uint8_t *)self->chunk + position_in_chunk;
+  size_t size = self->chunk_size - position_in_chunk + 1;
+
+  if (self->input.encoding == TSInputEncodingUTF8)
+    self->lookahead_size = utf8proc_iterate(chunk, size, &self->lookahead);
+  else
+    self->lookahead_size = utf16_iterate(chunk, size, &self->lookahead);
+
   LOG_LOOKAHEAD();
 }
 

src/runtime/utf16.c

@@ -0,0 +1,24 @@
+#include "runtime/utf16.h"
+
+int utf16_iterate(const uint8_t *string, size_t length, int32_t *code_point) {
+  uint16_t *units = (uint16_t *)string;
+  uint16_t unit = units[0];
+
+  if (unit < 0xd800 || unit >= 0xe000) {
+    *code_point = unit;
+    return 2;
+  }
+
+  if (unit < 0xdc00) {
+    if (length >= 4) {
+      uint16_t next_unit = units[1];
+      if (next_unit >= 0xdc00 && next_unit < 0xe000) {
+        *code_point = 0x10000 + ((unit - 0xd800) << 10) + (next_unit - 0xdc00);
+        return 4;
+      }
+    }
+  }
+
+  *code_point = -1;
+  return 2;
+}

src/runtime/utf16.h

@@ -0,0 +1,20 @@
+#ifndef RUNTIME_UTF16_H_
+#define RUNTIME_UTF16_H_
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include <stdint.h>
+#include <stdlib.h>
+
+// Analogous to utf8proc's utf8proc_iterate function. Reads one code point from
+// the given string and stores it in the location pointed to by `code_point`.
+// Returns the number of bytes in `string` that were read.
+int utf16_iterate(const uint8_t *string, size_t length, int32_t *code_point);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif  // RUNTIME_UTF16_H_