Merge remote-tracking branch 'origin/master' into line-numbers

.gitattributes

@@ -0,0 +1 @@
+/spec/fixtures/parsers/** linguist-vendored

README.md

@@ -171,3 +171,4 @@ clang                                                        \
 - [Context Aware Scanning for Parsing Extensible Languages](http://www.umsec.umn.edu/publications/Context-Aware-Scanning-Parsing-Extensible)
 - [LR(1) Parser Generation System](http://arxiv.org/pdf/1010.1234.pdf)  (Error recovery scheme)
 - [Efficient and Flexible Incremental Parsing](http://harmonia.cs.berkeley.edu/papers/twagner-parsing.ps.gz)
+- [Incremental Analysis of Real Programming Languages](http://harmonia.cs.berkeley.edu/papers/twagner-glr.pdf)

spec/fixtures/corpus/c/declarations.txt

@@ -55,10 +55,8 @@ ambiguous expressions
 ==========================================
 
 int main() {
-  // cast
+  // cast vs parenthesized product
   a((B *)c);
-
-  // parenthesized product
   d((e * f));
 }
 
@@ -71,8 +69,7 @@ int main() {
     (comment)
     (expression_statement (call_expression
       (identifier)
-      (cast_expression (type_name (identifier) (abstract_pointer_declarator)) (identifier)))
-      (comment))
+      (cast_expression (type_name (identifier) (abstract_pointer_declarator)) (identifier))))
     (expression_statement (call_expression
       (identifier)
       (math_expression (identifier) (identifier)))))))
@@ -114,5 +111,5 @@ int main() {
     (function_declarator (identifier))
     (compound_statement
       (comment)
-      (expression_statement (call_expression (identifier) (identifier)))
+      (declaration (identifier) (identifier))
       (declaration (identifier) (identifier)))))

spec/fixtures/corpus/c/errors.txt

@@ -1,3 +1,16 @@
+==========================================
+errors in top-level declarations
+==========================================
+
+int int int;
+int y;
+
+---
+
+(translation_unit
+  (declaration (ERROR (identifier) (identifier) (UNEXPECTED 'i') (identifier)))
+  (declaration (identifier) (identifier)))
+
 ==========================================
 errors in compound statements
 ==========================================

spec/fixtures/corpus/cpp/statements.txt

@@ -69,7 +69,7 @@ int main() {
   (declarator (direct_declarator (direct_declarator (identifier))))
   (function_body (compound_statement
     (simple_declaration
-      (type_specifier (template_call (identifier) (identifier)))
+      (type_specifier (template_call (identifier) (type_id (type_specifier (identifier)))))
       (init_declarator
         (declarator (direct_declarator (identifier)))
         (initializer (initializer_clause (number)))))))))

spec/fixtures/grammars/c.cc

@@ -25,7 +25,6 @@ extern const Grammar c = Grammar({
     optional(sym("declaration_specifiers")),
     sym("_type_specifier"),
     sym("_declarator"),
-    repeat(sym("declaration")),
     sym("compound_statement") }) },
 
   { "declaration_specifiers", repeat1(choice({
@@ -53,12 +52,6 @@ extern const Grammar c = Grammar({
       str("short") })),
     sym("identifier") }) },
 
-  { "macro_type", seq({
-    sym("identifier"),
-    str("("),
-    sym("_type_specifier"),
-    str(")") }) },
-
   { "struct_specifier", seq({
     str("struct"),
     optional(sym("identifier")),
@@ -82,9 +75,10 @@ extern const Grammar c = Grammar({
     sym("_declarator") }) },
 
   { "declaration", seq({
-    optional(sym("declaration_specifiers")),
-    sym("_type_specifier"),
-    comma_sep(sym("_init_declarator")),
+    err(seq({
+      optional(sym("declaration_specifiers")),
+      sym("_type_specifier"),
+      comma_sep1(sym("_init_declarator")) })),
     str(";") }) },
 
   { "_init_declarator", choice({
@@ -220,6 +214,12 @@ extern const Grammar c = Grammar({
 
   { "number", pattern("\\d+(\\.\\d+)?") },
 
+  { "macro_type", seq({
+    sym("identifier"),
+    str("("),
+    sym("_type_specifier"),
+    str(")") }) },
+
   { "comment", token(choice({
     pattern("//[^\n]*"),
     seq({

spec/runtime/stack_spec.cc

@@ -275,6 +275,30 @@ describe("Stack", [&]() {
         AssertThat(ts_stack_entry_next_count(next), Equals(2));
       });
     });
+
+    describe("when the first head is only one node deep", [&]() {
+      it("adds it as an additional successor node to The Null node", [&]() {
+        /*
+         *    .__A0.
+         *  B1.__/
+         */
+        ts_stack_clear(stack);
+        ts_stack_split(stack, 0);
+        ts_stack_push(stack, 0, stateA, trees[0]);
+        bool merged = ts_stack_push(stack, 1, stateB, trees[1]);
+        AssertThat(merged, IsFalse());
+        merged = ts_stack_push(stack, 1, stateA, trees[0]);
+        AssertThat(merged, IsTrue());
+
+        AssertThat(ts_stack_head_count(stack), Equals(1));
+        StackEntry *head = ts_stack_head(stack, 0);
+        AssertThat(*head, Equals<StackEntry>({trees[0], stateA}));
+
+        AssertThat(ts_stack_entry_next_count(head), Equals(2));
+        AssertThat(ts_stack_entry_next(head, 0), Equals<StackEntry *>(nullptr));
+        AssertThat(*ts_stack_entry_next(head, 1), Equals<StackEntry>({trees[1], stateB}));
+      });
+    });
   });
 
   describe("popping from a stack head that has been merged", [&]() {

src/runtime/parser.c

@@ -41,9 +41,12 @@ static const TSParseAction *ts_language__actions(const TSLanguage *language,
   return actions ? actions : ERROR_ACTIONS;
 }
 
-static TSParseAction ts_language__action(const TSLanguage *language,
-                                         TSStateId state, TSSymbol sym) {
-  return ts_language__actions(language, state, sym)[0];
+static TSParseAction ts_language__last_action(const TSLanguage *language,
+                                              TSStateId state, TSSymbol sym) {
+  const TSParseAction *action = ts_language__actions(language, state, sym);
+  while ((action + 1)->type)
+    action++;
+  return *action;
 }
 
 /*
@@ -107,7 +110,7 @@ static void ts_parser__get_next_lookahead(TSParser *self) {
 
     TSStateId top_state = ts_stack_top_state(self->stack, 0);
     TSSymbol symbol = self->reusable_subtree->symbol;
-    if (ts_language__action(self->language, top_state, symbol).type ==
+    if (ts_language__last_action(self->language, top_state, symbol).type ==
         TSParseActionTypeError) {
       DEBUG("cant_reuse sym:%s", SYM_NAME(self->reusable_subtree->symbol));
       ts_parser__pop_reusable_subtree(self);
@@ -145,7 +148,8 @@ static void ts_parser__get_next_lookahead(TSParser *self) {
  *  Parse Actions
  */
 
-static ConsumeResult ts_parser__shift(TSParser *self, int head, TSStateId parse_state) {
+static ConsumeResult ts_parser__shift(TSParser *self, int head,
+                                      TSStateId parse_state) {
   if (ts_stack_push(self->stack, head, parse_state, self->lookahead))
     return ConsumeResultRemoved;
   else
@@ -186,8 +190,8 @@ static TSTree *ts_parser__reduce(TSParser *self, int head, TSSymbol symbol,
         ts_tree_set_extra(parent);
         state = top_state;
       } else {
-        state =
-          ts_language__action(self->language, top_state, symbol).data.to_state;
+        state = ts_language__last_action(self->language, top_state, symbol)
+                  .data.to_state;
       }
 
       ts_stack_push(self->stack, pop_result.index, state, parent);
@@ -229,15 +233,15 @@ static bool ts_parser__handle_error(TSParser *self, int head) {
      *  expected and the current lookahead token is expected afterwards.
      */
     int i = -1;
-    for (StackEntry *entry = entry_before_error; entry != NULL;
+    for (StackEntry *entry = entry_before_error; true;
          entry = ts_stack_entry_next(entry, head), i++) {
-      TSStateId stack_state = entry->state;
-      TSParseAction action_on_error =
-        ts_language__action(self->language, stack_state, ts_builtin_sym_error);
+      TSStateId stack_state = entry ? entry->state : 0;
+      TSParseAction action_on_error = ts_language__last_action(
+        self->language, stack_state, ts_builtin_sym_error);
 
       if (action_on_error.type == TSParseActionTypeShift) {
         TSStateId state_after_error = action_on_error.data.to_state;
-        TSParseAction action_after_error = ts_language__action(
+        TSParseAction action_after_error = ts_language__last_action(
           self->language, state_after_error, self->lookahead->symbol);
 
         if (action_after_error.type != TSParseActionTypeError) {
@@ -247,6 +251,9 @@ static bool ts_parser__handle_error(TSParser *self, int head) {
           return true;
         }
       }
+
+      if (!entry)
+        break;
     }
 
     /*
@@ -382,8 +389,31 @@ static ConsumeResult ts_parser__consume_lookahead(TSParser *self, int head) {
   }
 }
 
+static int ts_tree__compare(TSTree *left, TSTree *right) {
+  if (left->symbol < right->symbol) return -1;
+  if (right->symbol < left->symbol) return 1;
+  if (left->child_count < right->child_count) return -1;
+  if (right->child_count < left->child_count) return 1;
+  for (size_t i = 0; i < left->child_count; i++) {
+    TSTree *left_child = left->children[i];
+    TSTree *right_child = right->children[i];
+    switch (ts_tree__compare(left_child, right_child)) {
+      case -1:
+        return -1;
+      case 1:
+        return 1;
+      default:
+        break;
+    }
+  }
+  return 0;
+}
+
 static TSTree *ts_parser__select_tree(void *data, TSTree *left, TSTree *right) {
-  return right;
+  if (ts_tree__compare(left, right) <= 0)
+    return left;
+  else
+    return right;
 }
 
 /*

src/runtime/stack.c

@@ -124,8 +124,11 @@ static void ts_stack__add_node_successor(Stack *self, StackNode *node,
                                          StackNode *new_successor) {
   for (int i = 0; i < node->successor_count; i++) {
     StackNode *successor = node->successors[i];
+    if (!successor)
+      continue;
     if (successor == new_successor)
       return;
+
     if (successor->entry.state == new_successor->entry.state) {
       if (successor->entry.tree != new_successor->entry.tree) {
         successor->entry.tree = self->tree_selection_callback.callback(