Call external scanner functions when lexing

src/runtime/document.c

@@ -37,7 +37,7 @@ const TSLanguage *ts_document_language(TSDocument *self) {
 
 void ts_document_set_language(TSDocument *self, const TSLanguage *language) {
   ts_document_invalidate(self);
-  self->parser.language = language;
+  parser_set_language(&self->parser, language);
   if (self->tree) {
     ts_tree_release(self->tree);
     self->tree = NULL;

src/runtime/language.h

@@ -49,6 +49,12 @@ static inline TSStateId ts_language_next_state(const TSLanguage *self,
   }
 }
 
+static inline const bool *
+ts_language_enabled_external_tokens(const TSLanguage *self,
+                                    unsigned external_scanner_state) {
+  return self->external_token_lists + self->external_token_count * external_scanner_state;
+}
+
 #ifdef __cplusplus
 }
 #endif

src/runtime/lexer.c

@@ -123,9 +123,7 @@ void ts_lexer_reset(Lexer *self, Length position) {
   return;
 }
 
-void ts_lexer_start(Lexer *self, TSStateId lex_state) {
-  LOG("start_lex state:%d, pos:%u", lex_state, self->current_position.chars);
-
+void ts_lexer_start(Lexer *self) {
   self->token_start_position = self->current_position;
   self->data.result_symbol = 0;
 

src/runtime/lexer.h

@@ -30,7 +30,7 @@ typedef struct {
 void ts_lexer_init(Lexer *);
 void ts_lexer_set_input(Lexer *, TSInput);
 void ts_lexer_reset(Lexer *, Length);
-void ts_lexer_start(Lexer *, TSStateId);
+void ts_lexer_start(Lexer *);
 
 #ifdef __cplusplus
 }

src/runtime/parser.c

@@ -209,23 +209,43 @@ static bool parser__condense_stack(Parser *self) {
 }
 
 static Tree *parser__lex(Parser *self, TSStateId parse_state) {
+  Length start_position = self->lexer.current_position;
+  ts_lexer_start(&self->lexer);
+
+  TSLexMode lex_mode = self->language->lex_modes[parse_state];
+  if (lex_mode.external_tokens) {
+    const bool *external_tokens = ts_language_enabled_external_tokens(self->language, lex_mode.external_tokens);
+    if (self->language->external_scanner.scan(
+      self->external_scanner_payload,
+      &self->lexer.data,
+      external_tokens
+    )) {
+      TSSymbol symbol = self->language->external_token_symbol_map[self->lexer.data.result_symbol];
+      Length padding = length_sub(self->lexer.token_start_position, start_position);
+      Length size = length_sub(self->lexer.current_position, self->lexer.token_start_position);
+      TSSymbolMetadata metadata = ts_language_symbol_metadata(self->language, symbol);
+      Tree *result = ts_tree_make_leaf(symbol, padding, size, metadata);
+      result->parse_state = parse_state;
+      return result;
+    } else {
+      ts_lexer_reset(&self->lexer, start_position);
+    }
+  }
+
   TSStateId start_state = self->language->lex_modes[parse_state].lex_state;
   TSStateId current_state = start_state;
-  Length start_position = self->lexer.current_position;
   LOG("lex state:%d", start_state);
 
   bool skipped_error = false;
   int32_t first_error_character = 0;
   Length error_start_position, error_end_position;
 
-  ts_lexer_start(&self->lexer, start_state);
-
   while (!self->language->lex_fn(&self->lexer.data, current_state)) {
     if (current_state != ERROR_STATE) {
       LOG("retry_in_error_mode");
       current_state = ERROR_STATE;
       ts_lexer_reset(&self->lexer, start_position);
-      ts_lexer_start(&self->lexer, current_state);
+      ts_lexer_start(&self->lexer);
       continue;
     }
 
@@ -247,7 +267,6 @@ static Tree *parser__lex(Parser *self, TSStateId parse_state) {
   }
 
   Tree *result;
-
   if (skipped_error) {
     Length padding = length_sub(error_start_position, start_position);
     Length size = length_sub(error_end_position, error_start_position);
@@ -255,18 +274,12 @@ static Tree *parser__lex(Parser *self, TSStateId parse_state) {
     result = ts_tree_make_error(size, padding, first_error_character);
   } else {
     TSSymbol symbol = self->lexer.data.result_symbol;
-    Length padding =
-      length_sub(self->lexer.token_start_position, start_position);
-    Length size = length_sub(self->lexer.current_position,
-                                  self->lexer.token_start_position);
-    result =
-      ts_tree_make_leaf(symbol, padding, size,
-                        ts_language_symbol_metadata(self->language, symbol));
+    Length padding = length_sub(self->lexer.token_start_position, start_position);
+    Length size = length_sub(self->lexer.current_position, self->lexer.token_start_position);
+    TSSymbolMetadata metadata = ts_language_symbol_metadata(self->language, symbol);
+    result = ts_tree_make_leaf(symbol, padding, size, metadata);
   }
 
-  if (!result)
-    return NULL;
-
   result->parse_state = parse_state;
   result->first_leaf.lex_state = start_state;
   return result;
@@ -1106,6 +1119,15 @@ bool parser_init(Parser *self) {
   return true;
 }
 
+void parser_set_language(Parser *self, const TSLanguage *language) {
+  self->language = language;
+  if (language->external_scanner.create) {
+    self->external_scanner_payload = language->external_scanner.create();
+  } else {
+    self->external_scanner_payload = NULL;
+  }
+}
+
 void parser_destroy(Parser *self) {
   if (self->stack)
     ts_stack_delete(self->stack);

src/runtime/parser.h

@@ -29,11 +29,13 @@ typedef struct {
   ReusableNode reusable_node;
   TreePath tree_path1;
   TreePath tree_path2;
+  void *external_scanner_payload;
 } Parser;
 
 bool parser_init(Parser *);
 void parser_destroy(Parser *);
 Tree *parser_parse(Parser *, TSInput, Tree *);
+void parser_set_language(Parser *, const TSLanguage *);
 
 #ifdef __cplusplus
 }