traxys/tree-sitter
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Call the external scanner whenever an external token is valid

Browse source 
For some reason, there was previously some extra logic that prevented
the external scanner from being invoked if the only valid external
token also had an internal definition.

It's surprising to not call the external scanner if an external
token is valid.
This commit is contained in:
Max Brunsfeld 2017-07-15 23:10:32 -07:00
parent 1a195d44bb
commit 66dc12587a
1 changed files with 1 additions and 3 deletions
Download patch file Download diff file Expand all files Collapse all files

4
src/compiler/generate_code/c_code.cc
View file

@ -303,12 +303,10 @@ class CCodeGenerator {
line("[" + to_string(state_id++) + "] = {.lex_state = ");
add(to_string(state.lex_state_id));
bool needs_external_scanner = false;
set<Symbol::Index> external_token_indices;
for (const auto &pair : state.terminal_entries) {
Symbol symbol = pair.first;
if (symbol.is_external()) {
needs_external_scanner = true;
external_token_indices.insert(symbol.index);
} else if (symbol.is_terminal()) {
auto corresponding_external_token =
@ -319,7 +317,7 @@ class CCodeGenerator {
}
}
if (needs_external_scanner) {
if (!external_token_indices.empty()) {
add(", .external_lex_state = " + add_external_scanner_state(external_token_indices));
}