Merge remote-tracking branch 'origin/master' into update-fixture-grammars

LICENSE

@@ -0,0 +1,7 @@
+Copyright 2014 Max Brunsfeld
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

include/tree_sitter/runtime.h

@@ -20,6 +20,12 @@ typedef enum {
   TSInputEncodingUTF16,
 } TSInputEncoding;
 
+typedef enum {
+  TSSymbolTypeRegular,
+  TSSymbolTypeAnonymous,
+  TSSymbolTypeAuxiliary,
+} TSSymbolType;
+
 typedef struct {
   void *payload;
   const char *(*read)(void *payload, uint32_t *bytes_read);
@@ -111,12 +117,22 @@ void ts_document_print_debugging_graphs(TSDocument *, bool);
 void ts_document_edit(TSDocument *, TSInputEdit);
 void ts_document_parse(TSDocument *);
 void ts_document_parse_and_get_changed_ranges(TSDocument *, TSRange **, uint32_t *);
+
+typedef struct {
+  TSRange **changed_ranges;
+  uint32_t *changed_range_count;
+  bool halt_on_error;
+} TSParseOptions;
+
+void ts_document_parse_with_options(TSDocument *, TSParseOptions);
+
 void ts_document_invalidate(TSDocument *);
 TSNode ts_document_root_node(const TSDocument *);
 uint32_t ts_document_parse_count(const TSDocument *);
 
 uint32_t ts_language_symbol_count(const TSLanguage *);
 const char *ts_language_symbol_name(const TSLanguage *, TSSymbol);
+TSSymbolType ts_language_symbol_type(const TSLanguage *, TSSymbol);
 uint32_t ts_language_version(const TSLanguage *);
 
 #ifdef __cplusplus

src/compiler/generate_code/c_code.cc

@@ -295,7 +295,7 @@ class CCodeGenerator {
   }
 
   void add_external_scanner_symbol_map() {
-    line("TSSymbol ts_external_scanner_symbol_map[EXTERNAL_TOKEN_COUNT] = {");
+    line("static TSSymbol ts_external_scanner_symbol_map[EXTERNAL_TOKEN_COUNT] = {");
     indent([&]() {
       for (size_t i = 0; i < syntax_grammar.external_tokens.size(); i++) {
         line("[" + external_token_id(i) + "] = " + symbol_id(Symbol::external(i)) + ",");

src/compiler/parse_table.cc

@@ -151,7 +151,7 @@ bool ParseState::has_shift_action() const {
 void ParseState::each_referenced_state(function<void(ParseStateId *)> fn) {
   for (auto &entry : terminal_entries)
     for (ParseAction &action : entry.second.actions)
-      if (action.type == ParseActionTypeShift || ParseActionTypeRecover)
+      if ((action.type == ParseActionTypeShift && !action.extra) || action.type == ParseActionTypeRecover)
         fn(&action.state_index);
   for (auto &entry : nonterminal_entries)
     fn(&entry.second);

src/runtime/document.c

@@ -99,10 +99,28 @@ void ts_document_edit(TSDocument *self, TSInputEdit edit) {
   ts_tree_edit(self->tree, &edit);
 }
 
+void ts_document_parse(TSDocument *self) {
+  return ts_document_parse_with_options(self, (TSParseOptions){
+    .halt_on_error = false,
+    .changed_ranges = NULL,
+    .changed_range_count = NULL,
+  });
+}
+
 void ts_document_parse_and_get_changed_ranges(TSDocument *self, TSRange **ranges,
                                               uint32_t *range_count) {
-  if (ranges) *ranges = NULL;
-  if (range_count) *range_count = 0;
+  return ts_document_parse_with_options(self, (TSParseOptions){
+    .halt_on_error = false,
+    .changed_ranges = ranges,
+    .changed_range_count = range_count,
+  });
+}
+
+void ts_document_parse_with_options(TSDocument *self, TSParseOptions options) {
+  if (options.changed_ranges && options.changed_range_count) {
+    *options.changed_ranges = NULL;
+    *options.changed_range_count = 0;
+  }
 
   if (!self->input.read || !self->parser.language)
     return;
@@ -111,17 +129,17 @@ void ts_document_parse_and_get_changed_ranges(TSDocument *self, TSRange **ranges
   if (reusable_tree && !reusable_tree->has_changes)
     return;
 
-  Tree *tree = parser_parse(&self->parser, self->input, reusable_tree);
+  Tree *tree = parser_parse(&self->parser, self->input, reusable_tree, options.halt_on_error);
 
   if (self->tree) {
     Tree *old_tree = self->tree;
     self->tree = tree;
 
-    if (ranges && range_count) {
+    if (options.changed_ranges && options.changed_range_count) {
       tree_path_init(&self->parser.tree_path1, old_tree);
       tree_path_init(&self->parser.tree_path2, tree);
       tree_path_get_changes(&self->parser.tree_path1, &self->parser.tree_path2,
-                            ranges, range_count);
+                            options.changed_ranges, options.changed_range_count);
     }
 
     ts_tree_release(old_tree);
@@ -132,10 +150,6 @@ void ts_document_parse_and_get_changed_ranges(TSDocument *self, TSRange **ranges
   self->valid = true;
 }
 
-void ts_document_parse(TSDocument *self) {
-  ts_document_parse_and_get_changed_ranges(self, NULL, NULL);
-}
-
 void ts_document_invalidate(TSDocument *self) {
   self->valid = false;
 }

src/runtime/language.c

@@ -54,3 +54,14 @@ const char *ts_language_symbol_name(const TSLanguage *language, TSSymbol symbol)
   else
     return language->symbol_names[symbol];
 }
+
+TSSymbolType ts_language_symbol_type(const TSLanguage *language, TSSymbol symbol) {
+  TSSymbolMetadata metadata = ts_language_symbol_metadata(language, symbol);
+  if (metadata.named) {
+    return TSSymbolTypeRegular;
+  } else if (metadata.visible) {
+    return TSSymbolTypeAnonymous;
+  } else {
+    return TSSymbolTypeAuxiliary;
+  }
+}

src/runtime/lexer.c

@@ -145,3 +145,7 @@ void ts_lexer_start(Lexer *self) {
   if (!self->lookahead_size)
     ts_lexer__get_lookahead(self);
 }
+
+void ts_lexer_advance_to_end(Lexer *self) {
+  while (self->data.lookahead != 0) ts_lexer__advance(self, false);
+}

src/runtime/lexer.h

@@ -32,6 +32,7 @@ void ts_lexer_init(Lexer *);
 void ts_lexer_set_input(Lexer *, TSInput);
 void ts_lexer_reset(Lexer *, Length);
 void ts_lexer_start(Lexer *);
+void ts_lexer_advance_to_end(Lexer *);
 
 #ifdef __cplusplus
 }

src/runtime/parser.c

@@ -151,21 +151,28 @@ static bool parser__can_reuse(Parser *self, TSStateId state, Tree *tree,
   return tree->child_count > 1 && tree->error_cost == 0;
 }
 
-static bool parser__condense_stack(Parser *self) {
-  bool result = false;
+typedef int CondenseResult;
+static int CondenseResultMadeChange = 1;
+static int CondenseResultAllVersionsHadError = 2;
+
+static CondenseResult parser__condense_stack(Parser *self) {
+  CondenseResult result = 0;
+  bool has_version_without_errors = false;
+
   for (StackVersion i = 0; i < ts_stack_version_count(self->stack); i++) {
     if (ts_stack_is_halted(self->stack, i)) {
       ts_stack_remove_version(self->stack, i);
-      result = true;
+      result |= CondenseResultMadeChange;
       i--;
       continue;
     }
 
     ErrorStatus error_status = ts_stack_error_status(self->stack, i);
+    if (error_status.count == 0) has_version_without_errors = true;
 
     for (StackVersion j = 0; j < i; j++) {
       if (ts_stack_merge(self->stack, j, i)) {
-        result = true;
+        result |= CondenseResultMadeChange;
         i--;
         break;
       }
@@ -174,18 +181,23 @@ static bool parser__condense_stack(Parser *self) {
                                    ts_stack_error_status(self->stack, j))) {
         case -1:
           ts_stack_remove_version(self->stack, j);
-          result = true;
+          result |= CondenseResultMadeChange;
           i--;
           j--;
           break;
         case 1:
           ts_stack_remove_version(self->stack, i);
-          result = true;
+          result |= CondenseResultMadeChange;
           i--;
           break;
       }
     }
   }
+
+  if (!has_version_without_errors && ts_stack_version_count(self->stack) > 0) {
+    result |= CondenseResultAllVersionsHadError;
+  }
+
   return result;
 }
 
@@ -990,6 +1002,30 @@ static void parser__handle_error(Parser *self, StackVersion version,
   }
 }
 
+static void parser__halt_parse(Parser *self) {
+  LOG("halting_parse");
+  LOG_STACK();
+
+  ts_lexer_advance_to_end(&self->lexer);
+  Length remaining_length = length_sub(
+    self->lexer.current_position,
+    ts_stack_top_position(self->stack, 0)
+  );
+
+  Tree *filler_node = ts_tree_make_error(remaining_length, length_zero(), 0);
+  filler_node->visible = false;
+  parser__push(self, 0, filler_node, 0);
+
+  TreeArray children = array_new();
+  Tree *root_error = ts_tree_make_error_node(&children);
+  parser__push(self, 0, root_error, 0);
+
+  TSSymbolMetadata metadata = ts_language_symbol_metadata(self->language, ts_builtin_sym_end);
+  Tree *eof = ts_tree_make_leaf(ts_builtin_sym_end, length_zero(), length_zero(), metadata);
+  parser__accept(self, 0, eof);
+  ts_tree_release(eof);
+}
+
 static void parser__recover(Parser *self, StackVersion version, TSStateId state,
                             Tree *lookahead) {
   if (lookahead->symbol == ts_builtin_sym_end) {
@@ -1194,7 +1230,7 @@ void parser_destroy(Parser *self) {
   parser_set_language(self, NULL);
 }
 
-Tree *parser_parse(Parser *self, TSInput input, Tree *old_tree) {
+Tree *parser_parse(Parser *self, TSInput input, Tree *old_tree, bool halt_on_error) {
   parser__start(self, input, old_tree);
 
   StackVersion version = STACK_VERSION_NONE;
@@ -1224,7 +1260,13 @@ Tree *parser_parse(Parser *self, TSInput input, Tree *old_tree) {
 
     self->reusable_node = reusable_node;
 
-    if (parser__condense_stack(self)) {
+    CondenseResult condense_result = parser__condense_stack(self);
+    if (halt_on_error && (condense_result & CondenseResultAllVersionsHadError)) {
+      parser__halt_parse(self);
+      break;
+    }
+
+    if (condense_result & CondenseResultMadeChange) {
       LOG("condense");
       LOG_STACK();
     }

src/runtime/parser.h

@@ -31,7 +31,7 @@ typedef struct {
 
 bool parser_init(Parser *);
 void parser_destroy(Parser *);
-Tree *parser_parse(Parser *, TSInput, Tree *);
+Tree *parser_parse(Parser *, TSInput, Tree *, bool halt_on_error);
 void parser_set_language(Parser *, const TSLanguage *);
 
 #ifdef __cplusplus

src/runtime/stack.c

@@ -504,7 +504,7 @@ bool ts_stack_merge(Stack *self, StackVersion version, StackVersion new_version)
       new_node->position.chars == node->position.chars &&
       new_node->error_count == node->error_count &&
       new_node->error_cost == node->error_cost &&
-      new_head->external_token_state == head->external_token_state) {
+      ts_external_token_state_eq(new_head->external_token_state, head->external_token_state)) {
     for (uint32_t j = 0; j < new_node->link_count; j++)
       stack_node_add_link(node, new_node->links[j]);
     if (new_head->push_count > head->push_count)

test/runtime/document_test.cc

@@ -367,6 +367,51 @@ describe("Document", [&]() {
       })));
     });
   });
+
+  describe("parse_with_options(options)", [&]() {
+    it("halts as soon as an error is found if the halt_on_error flag is set", [&]() {
+      string input_string = "[1, null, error, 3]";
+      ts_document_set_language(document, load_real_language("json"));
+      ts_document_set_input_string(document, input_string.c_str());
+
+      TSParseOptions options;
+      options.changed_ranges = nullptr;
+
+      options.halt_on_error = false;
+      ts_document_parse_with_options(document, options);
+      root = ts_document_root_node(document);
+      assert_node_string_equals(
+        root,
+        "(array (number) (null) (ERROR (UNEXPECTED 'e')) (number))");
+
+      ts_document_invalidate(document);
+
+      options.halt_on_error = true;
+      ts_document_parse_with_options(document, options);
+      root = ts_document_root_node(document);
+      assert_node_string_equals(
+        root,
+        "(ERROR (number) (null) (UNEXPECTED 'e'))");
+
+      AssertThat(ts_node_end_char(root), Equals(input_string.size()));
+      AssertThat(ts_node_end_byte(root), Equals(input_string.size()));
+    });
+
+    it("can parse valid code with the halt_on_error flag set", [&]() {
+      string input_string = "[1, null, 3]";
+      ts_document_set_language(document, load_real_language("json"));
+      ts_document_set_input_string(document, input_string.c_str());
+
+      TSParseOptions options;
+      options.changed_ranges = nullptr;
+      options.halt_on_error = true;
+      ts_document_parse_with_options(document, options);
+      root = ts_document_root_node(document);
+      assert_node_string_equals(
+        root,
+        "(array (number) (null) (number))");
+    });
+  });
 });
 
 END_TEST