From d72b49316b6fb5a74b7c444cbc3b3dbb25e06c0c Mon Sep 17 00:00:00 2001
From: Max Brunsfeld <maxbrunsfeld@gmail.com>
Date: Sun, 4 Dec 2016 10:40:32 -0800
Subject: [PATCH] Handle external tokens in apply_transitive_closure

---
 spec/integration/compile_grammar_spec.cc      | 58 +++++++++++--------
 .../build_tables/parse_item_set_builder.cc    |  5 ++
 2 files changed, 39 insertions(+), 24 deletions(-)

diff --git a/spec/integration/compile_grammar_spec.cc b/spec/integration/compile_grammar_spec.cc
index 934b428c..ea0696ec 100644
--- a/spec/integration/compile_grammar_spec.cc
+++ b/spec/integration/compile_grammar_spec.cc
@@ -518,29 +518,43 @@ describe("compile_grammar", []() {
           "percent_string_end"
         ],
 
+        "extras": [
+          {"type": "PATTERN", "value": "\\s"}
+        ],
+
         "rules": {
+          "expression": {
+            "type": "CHOICE",
+            "members": [
+              {"type": "SYMBOL", "name": "string"},
+              {"type": "SYMBOL", "name": "sum"},
+              {"type": "SYMBOL", "name": "identifier"}
+            ]
+          },
+
+          "sum": {
+            "type": "PREC_LEFT",
+            "value": 0,
+            "content": {
+              "type": "SEQ",
+              "members": [
+                {"type": "SYMBOL", "name": "expression"},
+                {"type": "STRING", "value": "+"},
+                {"type": "SYMBOL", "name": "expression"}
+              ]
+            }
+          },
+
           "string": {
             "type": "CHOICE",
             "members": [
-              {
-                "type": "EXTERNAL_TOKEN",
-                "name": "percent_string"
-              },
+              {"type": "EXTERNAL_TOKEN", "name": "percent_string"},
               {
                 "type": "SEQ",
                 "members": [
-                  {
-                    "type": "EXTERNAL_TOKEN",
-                    "name": "percent_string_start"
-                  },
-                  {
-                    "type": "SYMBOL",
-                    "name": "identifier"
-                  },
-                  {
-                    "type": "EXTERNAL_TOKEN",
-                    "name": "percent_string_end"
-                  }
+                  {"type": "EXTERNAL_TOKEN", "name": "percent_string_start"},
+                  {"type": "SYMBOL", "name": "expression"},
+                  {"type": "EXTERNAL_TOKEN", "name": "percent_string_end"}
                 ]
               },
             ]
@@ -562,17 +576,13 @@ describe("compile_grammar", []() {
         "spec/fixtures/external_scanners/external_scan.c"
       ));
 
-      ts_document_set_input_string(document, "%(sup (external) scanner?)");
+      ts_document_set_input_string(document, "x + %(sup (external) scanner?)");
       ts_document_parse(document);
-      assert_root_node("(string)");
+      assert_root_node("(expression (sum (expression (identifier)) (expression (string))))");
 
-      ts_document_set_input_string(document, "%{sup {} external {} scanner?}");
+      ts_document_set_input_string(document, "%{sup {} #{x + y} {} scanner?}");
       ts_document_parse(document);
-      assert_root_node("(string)");
-
-      ts_document_set_input_string(document, "%(1 #{two} three)");
-      ts_document_parse(document);
-      assert_root_node("(string (identifier))");
+      assert_root_node("(expression (string (expression (sum (expression (identifier)) (expression (identifier))))))");
     });
   });
 
diff --git a/src/compiler/build_tables/parse_item_set_builder.cc b/src/compiler/build_tables/parse_item_set_builder.cc
index 7e29efdf..0a2039d3 100644
--- a/src/compiler/build_tables/parse_item_set_builder.cc
+++ b/src/compiler/build_tables/parse_item_set_builder.cc
@@ -31,6 +31,11 @@ ParseItemSetBuilder::ParseItemSetBuilder(const SyntaxGrammar &grammar,
     first_sets.insert({symbol, LookaheadSet({ symbol })});
   }
 
+  for (size_t i = 0, n = grammar.external_tokens.size(); i < n; i++) {
+    Symbol symbol(i, Symbol::External);
+    first_sets.insert({symbol, LookaheadSet({ symbol })});
+  }
+
   for (size_t i = 0, n = grammar.variables.size(); i < n; i++) {
     Symbol symbol(i, Symbol::NonTerminal);
     LookaheadSet first_set;