Make grammars' separator characters configurable

2014-06-26 07:31:08 -07:00 · 2014-06-26 07:31:08 -07:00 · a9dff20658
commit a9dff20658
parent 8aea89750d
5 changed files with 24 additions and 5 deletions
--- a/src/compiler/prepare_grammar/extract_tokens.cc
+++ b/src/compiler/prepare_grammar/extract_tokens.cc
@ -124,7 +124,7 @@ namespace tree_sitter {

            return {
                SyntaxGrammar(rules, aux_rules, ubiquitous_tokens),
-                LexicalGrammar(tokens, aux_tokens, {}),
+                LexicalGrammar(tokens, aux_tokens, input_grammar.separators),
            };
        }
    }