diff --git a/spec/integration/compile_grammar_spec.cc b/spec/integration/compile_grammar_spec.cc index 14d026d0..743ea286 100644 --- a/spec/integration/compile_grammar_spec.cc +++ b/spec/integration/compile_grammar_spec.cc @@ -513,9 +513,9 @@ describe("compile_grammar", []() { "name": "external_scanner_example", "externals": [ - "percent_string", - "percent_string_start", - "percent_string_end" + "_percent_string", + "_percent_string_start", + "_percent_string_end" ], "extras": [ @@ -548,13 +548,13 @@ describe("compile_grammar", []() { "string": { "type": "CHOICE", "members": [ - {"type": "SYMBOL", "name": "percent_string"}, + {"type": "SYMBOL", "name": "_percent_string"}, { "type": "SEQ", "members": [ - {"type": "SYMBOL", "name": "percent_string_start"}, + {"type": "SYMBOL", "name": "_percent_string_start"}, {"type": "SYMBOL", "name": "expression"}, - {"type": "SYMBOL", "name": "percent_string_end"} + {"type": "SYMBOL", "name": "_percent_string_end"} ] }, ] diff --git a/src/compiler/build_tables/build_parse_table.cc b/src/compiler/build_tables/build_parse_table.cc index 4c3ba8c6..bdaac037 100644 --- a/src/compiler/build_tables/build_parse_table.cc +++ b/src/compiler/build_tables/build_parse_table.cc @@ -584,7 +584,7 @@ class ParseTableBuilder { return grammar.variables[symbol.index].name; } case Symbol::External: { - return grammar.external_tokens[symbol.index]; + return grammar.external_tokens[symbol.index].name; } } } diff --git a/src/compiler/generate_code/c_code.cc b/src/compiler/generate_code/c_code.cc index db8756e6..e5272595 100644 --- a/src/compiler/generate_code/c_code.cc +++ b/src/compiler/generate_code/c_code.cc @@ -485,7 +485,7 @@ class CCodeGenerator { // Helper functions string external_token_id(Symbol::Index index) { - return "ts_external_token_" + syntax_grammar.external_tokens[index]; + return "ts_external_token_" + syntax_grammar.external_tokens[index].name; } string symbol_id(const Symbol &symbol) { @@ -528,7 +528,8 @@ class CCodeGenerator { return { variable.name, variable.type }; } case Symbol::External: { - return { syntax_grammar.external_tokens[symbol.index], VariableTypeNamed }; + const ExternalToken &token = syntax_grammar.external_tokens[symbol.index]; + return { token.name, token.type }; } } } diff --git a/src/compiler/prepare_grammar/initial_syntax_grammar.h b/src/compiler/prepare_grammar/initial_syntax_grammar.h index d4b1c8d5..1ac319cb 100644 --- a/src/compiler/prepare_grammar/initial_syntax_grammar.h +++ b/src/compiler/prepare_grammar/initial_syntax_grammar.h @@ -15,7 +15,7 @@ struct InitialSyntaxGrammar { std::vector variables; std::set extra_tokens; std::set expected_conflicts; - std::vector external_tokens; + std::vector external_tokens; }; } // namespace prepare_grammar diff --git a/src/compiler/prepare_grammar/intern_symbols.cc b/src/compiler/prepare_grammar/intern_symbols.cc index efe4b37f..daad9d2e 100644 --- a/src/compiler/prepare_grammar/intern_symbols.cc +++ b/src/compiler/prepare_grammar/intern_symbols.cc @@ -54,7 +54,14 @@ CompileError missing_rule_error(string rule_name) { pair intern_symbols(const Grammar &grammar) { InternedGrammar result; - result.external_tokens = grammar.external_tokens; + + for (auto &external_token_name : grammar.external_tokens) { + result.external_tokens.push_back(ExternalToken{ + external_token_name, + external_token_name[0] == '_' ? VariableTypeHidden : VariableTypeNamed + }); + } + InternSymbols interner(grammar); for (auto &pair : grammar.rules) { @@ -62,9 +69,11 @@ pair intern_symbols(const Grammar &grammar) { if (!interner.missing_rule_name.empty()) return { result, missing_rule_error(interner.missing_rule_name) }; - result.variables.push_back(Variable( - pair.first, pair.first[0] == '_' ? VariableTypeHidden : VariableTypeNamed, - new_rule)); + result.variables.push_back(Variable{ + pair.first, + pair.first[0] == '_' ? VariableTypeHidden : VariableTypeNamed, + new_rule + }); } for (auto &rule : grammar.extra_tokens) { diff --git a/src/compiler/prepare_grammar/interned_grammar.h b/src/compiler/prepare_grammar/interned_grammar.h index 7b425c3a..c8a14647 100644 --- a/src/compiler/prepare_grammar/interned_grammar.h +++ b/src/compiler/prepare_grammar/interned_grammar.h @@ -15,7 +15,7 @@ struct InternedGrammar { std::vector variables; std::vector extra_tokens; std::set expected_conflicts; - std::vector external_tokens; + std::vector external_tokens; }; } // namespace prepare_grammar diff --git a/src/compiler/syntax_grammar.h b/src/compiler/syntax_grammar.h index e34ddbbe..3d001b61 100644 --- a/src/compiler/syntax_grammar.h +++ b/src/compiler/syntax_grammar.h @@ -10,6 +10,11 @@ namespace tree_sitter { +struct ExternalToken { + std::string name; + VariableType type; +}; + struct ProductionStep { ProductionStep(const rules::Symbol &, int, rules::Associativity); bool operator==(const ProductionStep &) const; @@ -36,7 +41,7 @@ struct SyntaxGrammar { std::vector variables; std::set extra_tokens; std::set expected_conflicts; - std::vector external_tokens; + std::vector external_tokens; }; } // namespace tree_sitter