From b61b27f22fdd0529803ae542b2a726f4b1a8000e Mon Sep 17 00:00:00 2001
From: Max Brunsfeld <maxbrunsfeld@gmail.com>
Date: Mon, 26 Oct 2015 17:19:37 -0700
Subject: [PATCH] Handle inline ubiquitous that are used elsewhere in the
 grammar

---
 .../compiler/prepare_grammar/extract_tokens_spec.cc | 13 +++++++++++++
 src/compiler/prepare_grammar/extract_tokens.cc      | 13 +++++++++++++
 2 files changed, 26 insertions(+)

diff --git a/spec/compiler/prepare_grammar/extract_tokens_spec.cc b/spec/compiler/prepare_grammar/extract_tokens_spec.cc
index 14e6ba56..365f0713 100644
--- a/spec/compiler/prepare_grammar/extract_tokens_spec.cc
+++ b/spec/compiler/prepare_grammar/extract_tokens_spec.cc
@@ -159,6 +159,19 @@ describe("extract_tokens", []() {
       AssertThat(get<0>(result).ubiquitous_tokens, IsEmpty());
     });
 
+    it("handles inline ubiquitous tokens that match tokens in the grammar", [&]() {
+      auto result = extract_tokens(InternedGrammar{{
+        Variable("rule_A", VariableTypeNamed, str("x")),
+        Variable("rule_B", VariableTypeNamed, str("y")),
+      }, {
+        str("y"),
+      }, {}});
+
+      AssertThat(get<2>(result), Equals<const GrammarError *>(nullptr));
+      AssertThat(get<1>(result).separators.size(), Equals<size_t>(0));
+      AssertThat(get<0>(result).ubiquitous_tokens, Equals(set<Symbol>({ Symbol(1, true) })));
+    });
+
     it("updates ubiquitous symbols according to the new symbol numbers", [&]() {
       auto result = extract_tokens(InternedGrammar{{
         Variable("rule_A", VariableTypeNamed, seq({ str("w"), str("x"), i_sym(1) })),
diff --git a/src/compiler/prepare_grammar/extract_tokens.cc b/src/compiler/prepare_grammar/extract_tokens.cc
index c63cb968..ecc1295f 100644
--- a/src/compiler/prepare_grammar/extract_tokens.cc
+++ b/src/compiler/prepare_grammar/extract_tokens.cc
@@ -154,6 +154,19 @@ tuple<InitialSyntaxGrammar, LexicalGrammar, const GrammarError *> extract_tokens
    *  lexical grammar's separator rules.
    */
   for (const rule_ptr &rule : grammar.ubiquitous_tokens) {
+    int i = 0;
+    bool used_elsewhere_in_grammar = false;
+    for (const Variable &variable : lexical_grammar.variables) {
+      if (variable.rule->operator==(*rule)) {
+        syntax_grammar.ubiquitous_tokens.insert(Symbol(i, true));
+        used_elsewhere_in_grammar = true;
+      }
+      i++;
+    }
+
+    if (used_elsewhere_in_grammar)
+      continue;
+
     if (is_token(rule)) {
       lexical_grammar.separators.push_back(rule);
       continue;