Merge remote-tracking branch 'origin/master' into update-fixture-grammars

src/runtime/document.c

@@ -99,10 +99,28 @@ void ts_document_edit(TSDocument *self, TSInputEdit edit) {
   ts_tree_edit(self->tree, &edit);
 }
 
+void ts_document_parse(TSDocument *self) {
+  return ts_document_parse_with_options(self, (TSParseOptions){
+    .halt_on_error = false,
+    .changed_ranges = NULL,
+    .changed_range_count = NULL,
+  });
+}
+
 void ts_document_parse_and_get_changed_ranges(TSDocument *self, TSRange **ranges,
                                               uint32_t *range_count) {
-  if (ranges) *ranges = NULL;
-  if (range_count) *range_count = 0;
+  return ts_document_parse_with_options(self, (TSParseOptions){
+    .halt_on_error = false,
+    .changed_ranges = ranges,
+    .changed_range_count = range_count,
+  });
+}
+
+void ts_document_parse_with_options(TSDocument *self, TSParseOptions options) {
+  if (options.changed_ranges && options.changed_range_count) {
+    *options.changed_ranges = NULL;
+    *options.changed_range_count = 0;
+  }
 
   if (!self->input.read || !self->parser.language)
     return;
@@ -111,17 +129,17 @@ void ts_document_parse_and_get_changed_ranges(TSDocument *self, TSRange **ranges
   if (reusable_tree && !reusable_tree->has_changes)
     return;
 
-  Tree *tree = parser_parse(&self->parser, self->input, reusable_tree);
+  Tree *tree = parser_parse(&self->parser, self->input, reusable_tree, options.halt_on_error);
 
   if (self->tree) {
     Tree *old_tree = self->tree;
     self->tree = tree;
 
-    if (ranges && range_count) {
+    if (options.changed_ranges && options.changed_range_count) {
       tree_path_init(&self->parser.tree_path1, old_tree);
       tree_path_init(&self->parser.tree_path2, tree);
       tree_path_get_changes(&self->parser.tree_path1, &self->parser.tree_path2,
-                            ranges, range_count);
+                            options.changed_ranges, options.changed_range_count);
     }
 
     ts_tree_release(old_tree);
@@ -132,10 +150,6 @@ void ts_document_parse_and_get_changed_ranges(TSDocument *self, TSRange **ranges
   self->valid = true;
 }
 
-void ts_document_parse(TSDocument *self) {
-  ts_document_parse_and_get_changed_ranges(self, NULL, NULL);
-}
-
 void ts_document_invalidate(TSDocument *self) {
   self->valid = false;
 }

src/runtime/language.c

@@ -54,3 +54,14 @@ const char *ts_language_symbol_name(const TSLanguage *language, TSSymbol symbol)
   else
     return language->symbol_names[symbol];
 }
+
+TSSymbolType ts_language_symbol_type(const TSLanguage *language, TSSymbol symbol) {
+  TSSymbolMetadata metadata = ts_language_symbol_metadata(language, symbol);
+  if (metadata.named) {
+    return TSSymbolTypeRegular;
+  } else if (metadata.visible) {
+    return TSSymbolTypeAnonymous;
+  } else {
+    return TSSymbolTypeAuxiliary;
+  }
+}

src/runtime/lexer.c

@@ -145,3 +145,7 @@ void ts_lexer_start(Lexer *self) {
   if (!self->lookahead_size)
     ts_lexer__get_lookahead(self);
 }
+
+void ts_lexer_advance_to_end(Lexer *self) {
+  while (self->data.lookahead != 0) ts_lexer__advance(self, false);
+}

src/runtime/lexer.h

@@ -32,6 +32,7 @@ void ts_lexer_init(Lexer *);
 void ts_lexer_set_input(Lexer *, TSInput);
 void ts_lexer_reset(Lexer *, Length);
 void ts_lexer_start(Lexer *);
+void ts_lexer_advance_to_end(Lexer *);
 
 #ifdef __cplusplus
 }

src/runtime/parser.c

@@ -151,21 +151,28 @@ static bool parser__can_reuse(Parser *self, TSStateId state, Tree *tree,
   return tree->child_count > 1 && tree->error_cost == 0;
 }
 
-static bool parser__condense_stack(Parser *self) {
-  bool result = false;
+typedef int CondenseResult;
+static int CondenseResultMadeChange = 1;
+static int CondenseResultAllVersionsHadError = 2;
+
+static CondenseResult parser__condense_stack(Parser *self) {
+  CondenseResult result = 0;
+  bool has_version_without_errors = false;
+
   for (StackVersion i = 0; i < ts_stack_version_count(self->stack); i++) {
     if (ts_stack_is_halted(self->stack, i)) {
       ts_stack_remove_version(self->stack, i);
-      result = true;
+      result |= CondenseResultMadeChange;
       i--;
       continue;
     }
 
     ErrorStatus error_status = ts_stack_error_status(self->stack, i);
+    if (error_status.count == 0) has_version_without_errors = true;
 
     for (StackVersion j = 0; j < i; j++) {
       if (ts_stack_merge(self->stack, j, i)) {
-        result = true;
+        result |= CondenseResultMadeChange;
         i--;
         break;
       }
@@ -174,18 +181,23 @@ static bool parser__condense_stack(Parser *self) {
                                    ts_stack_error_status(self->stack, j))) {
         case -1:
           ts_stack_remove_version(self->stack, j);
-          result = true;
+          result |= CondenseResultMadeChange;
           i--;
           j--;
           break;
         case 1:
           ts_stack_remove_version(self->stack, i);
-          result = true;
+          result |= CondenseResultMadeChange;
           i--;
           break;
       }
     }
   }
+
+  if (!has_version_without_errors && ts_stack_version_count(self->stack) > 0) {
+    result |= CondenseResultAllVersionsHadError;
+  }
+
   return result;
 }
 
@@ -990,6 +1002,30 @@ static void parser__handle_error(Parser *self, StackVersion version,
   }
 }
 
+static void parser__halt_parse(Parser *self) {
+  LOG("halting_parse");
+  LOG_STACK();
+
+  ts_lexer_advance_to_end(&self->lexer);
+  Length remaining_length = length_sub(
+    self->lexer.current_position,
+    ts_stack_top_position(self->stack, 0)
+  );
+
+  Tree *filler_node = ts_tree_make_error(remaining_length, length_zero(), 0);
+  filler_node->visible = false;
+  parser__push(self, 0, filler_node, 0);
+
+  TreeArray children = array_new();
+  Tree *root_error = ts_tree_make_error_node(&children);
+  parser__push(self, 0, root_error, 0);
+
+  TSSymbolMetadata metadata = ts_language_symbol_metadata(self->language, ts_builtin_sym_end);
+  Tree *eof = ts_tree_make_leaf(ts_builtin_sym_end, length_zero(), length_zero(), metadata);
+  parser__accept(self, 0, eof);
+  ts_tree_release(eof);
+}
+
 static void parser__recover(Parser *self, StackVersion version, TSStateId state,
                             Tree *lookahead) {
   if (lookahead->symbol == ts_builtin_sym_end) {
@@ -1194,7 +1230,7 @@ void parser_destroy(Parser *self) {
   parser_set_language(self, NULL);
 }
 
-Tree *parser_parse(Parser *self, TSInput input, Tree *old_tree) {
+Tree *parser_parse(Parser *self, TSInput input, Tree *old_tree, bool halt_on_error) {
   parser__start(self, input, old_tree);
 
   StackVersion version = STACK_VERSION_NONE;
@@ -1224,7 +1260,13 @@ Tree *parser_parse(Parser *self, TSInput input, Tree *old_tree) {
 
     self->reusable_node = reusable_node;
 
-    if (parser__condense_stack(self)) {
+    CondenseResult condense_result = parser__condense_stack(self);
+    if (halt_on_error && (condense_result & CondenseResultAllVersionsHadError)) {
+      parser__halt_parse(self);
+      break;
+    }
+
+    if (condense_result & CondenseResultMadeChange) {
       LOG("condense");
       LOG_STACK();
     }

src/runtime/parser.h

@@ -31,7 +31,7 @@ typedef struct {
 
 bool parser_init(Parser *);
 void parser_destroy(Parser *);
-Tree *parser_parse(Parser *, TSInput, Tree *);
+Tree *parser_parse(Parser *, TSInput, Tree *, bool halt_on_error);
 void parser_set_language(Parser *, const TSLanguage *);
 
 #ifdef __cplusplus

src/runtime/stack.c

@@ -504,7 +504,7 @@ bool ts_stack_merge(Stack *self, StackVersion version, StackVersion new_version)
       new_node->position.chars == node->position.chars &&
       new_node->error_count == node->error_count &&
       new_node->error_cost == node->error_cost &&
-      new_head->external_token_state == head->external_token_state) {
+      ts_external_token_state_eq(new_head->external_token_state, head->external_token_state)) {
     for (uint32_t j = 0; j < new_node->link_count; j++)
       stack_node_add_link(node, new_node->links[j]);
     if (new_head->push_count > head->push_count)