15

lib/include/tree_sitter/api.h

@@ -11,9 +11,9 @@
 extern "C" {
 #endif
 
-#include <stdlib.h>
-#include <stdint.h>
 #include <stdbool.h>
+#include <stdint.h>
+#include <stdlib.h>
 
 /****************************/
 /* Section - ABI Versioning */
@@ -26,7 +26,7 @@ extern "C" {
  * The Tree-sitter library is generally backwards-compatible with languages
  * generated using older CLI versions, but is not forwards-compatible.
  */
-#define TREE_SITTER_LANGUAGE_VERSION 14
+#define TREE_SITTER_LANGUAGE_VERSION 15
 
 /**
  * The earliest ABI version that is supported by the current version of the

lib/src/lexer.c

@@ -155,8 +155,12 @@ static void ts_lexer_goto(Lexer *self, Length position) {
   }
 }
 
+static void ts_lexer__mark_begin(Lexer *self) {
+  self->token_start_position = self->current_position;
+}
+
 // Intended to be called only from functions that control logging.
-static void ts_lexer__do_advance(Lexer *self, bool skip) {
+static void ts_lexer__do_advance(Lexer *self) {
   if (self->lookahead_size) {
     self->current_position.bytes += self->lookahead_size;
     if (self->data.lookahead == '\n') {
@@ -187,7 +191,7 @@ static void ts_lexer__do_advance(Lexer *self, bool skip) {
     }
   }
 
-  if (skip) self->token_start_position = self->current_position;
+  self->token_start_position = self->current_position;
 
   if (current_range) {
     if (
@@ -206,17 +210,20 @@ static void ts_lexer__do_advance(Lexer *self, bool skip) {
 
 // Advance to the next character in the source code, retrieving a new
 // chunk of source code if needed.
-static void ts_lexer__advance(TSLexer *_self, bool skip) {
+static void ts_lexer__advance(TSLexer *_self) {
   Lexer *self = (Lexer *)_self;
   if (!self->chunk) return;
 
-  if (skip) {
-    LOG("skip", self->data.lookahead)
-  } else {
-    LOG("consume", self->data.lookahead)
-  }
+  LOG("consume", self->data.lookahead)
 
-  ts_lexer__do_advance(self, skip);
+#if TREE_SITTER_LANGUAGE_VERSION <= 14
+  if (!self->called_mark_begin) {
+    ts_lexer__mark_begin(self);
+    self->called_mark_begin = true;
+  }
+#endif
+
+  ts_lexer__do_advance(self);
 }
 
 // Mark that a token match has completed. This can be called multiple
@@ -263,7 +270,7 @@ static uint32_t ts_lexer__get_column(TSLexer *_self) {
     ts_lexer__get_lookahead(self);
     while (self->current_position.bytes < goal_byte && self->chunk) {
       result++;
-      ts_lexer__do_advance(self, false);
+      ts_lexer__do_advance(self);
       if (ts_lexer__eof(_self)) break;
     }
   }
@@ -342,7 +349,7 @@ void ts_lexer_start(Lexer *self) {
     if (
       self->current_position.bytes == 0 &&
       self->data.lookahead == BYTE_ORDER_MARK
-    ) ts_lexer__advance(&self->data, true);
+    ) ts_lexer__advance(&self->data);
   }
 }
 
@@ -375,10 +382,14 @@ void ts_lexer_finish(Lexer *self, uint32_t *lookahead_end_byte) {
 
 void ts_lexer_advance_to_end(Lexer *self) {
   while (self->chunk) {
-    ts_lexer__advance(&self->data, false);
+    ts_lexer__advance(&self->data);
   }
 }
 
+void ts_lexer_mark_begin(Lexer *self) {
+  ts_lexer__mark_begin(self);
+}
+
 void ts_lexer_mark_end(Lexer *self) {
   ts_lexer__mark_end(&self->data);
 }

lib/src/lexer.h

@@ -27,6 +27,7 @@ typedef struct {
   uint32_t chunk_size;
   uint32_t lookahead_size;
   bool did_get_column;
+  bool called_mark_begin;
 
   char debug_buffer[TREE_SITTER_SERIALIZATION_BUFFER_SIZE];
 } Lexer;
@@ -38,6 +39,7 @@ void ts_lexer_reset(Lexer *, Length);
 void ts_lexer_start(Lexer *);
 void ts_lexer_finish(Lexer *, uint32_t *);
 void ts_lexer_advance_to_end(Lexer *);
+void ts_lexer_mark_begin(Lexer *);
 void ts_lexer_mark_end(Lexer *);
 bool ts_lexer_set_included_ranges(Lexer *self, const TSRange *ranges, uint32_t count);
 TSRange *ts_lexer_included_ranges(const Lexer *self, uint32_t *count);

lib/src/parser.c

@@ -613,7 +613,7 @@ static Subtree ts_parser__lex(
         self->lexer.data.result_symbol = ts_builtin_sym_error;
         break;
       }
-      self->lexer.data.advance(&self->lexer.data, false);
+      self->lexer.data.advance(&self->lexer.data);
     }
 
     error_end_position = self->lexer.current_position;

lib/src/parser.h

@@ -42,7 +42,11 @@ typedef struct TSLexer TSLexer;
 struct TSLexer {
   int32_t lookahead;
   TSSymbol result_symbol;
+#if TREE_SITTER_LANGUAGE_VERSION >= 15
+  void (*advance)(TSLexer *);
+#else
   void (*advance)(TSLexer *, bool);
+#endif
   void (*mark_end)(TSLexer *);
   uint32_t (*get_column)(TSLexer *);
   bool (*is_at_included_range_start)(const TSLexer *);