traxys/tree-sitter
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

cli: Compute fragile tokens *after* merging compatibile parse states

Browse source 
Previously, we failed to mark as fragile some tokens that *should* be 
fragile because of tokens that were introduced during parse state 
merging.
This commit is contained in:
Max Brunsfeld 2019-02-06 16:18:49 -08:00
parent af694b4c13
commit 9b8bf8dfe2
1 changed files with 4 additions and 4 deletions
Download patch file Download diff file Expand all files Collapse all files

8
cli/src/generate/build_tables/mod.rs
View file

@ -47,7 +47,6 @@ pub(crate) fn build_tables(
&keywords,
);
populate_used_symbols(&mut parse_table, syntax_grammar, lexical_grammar);
mark_fragile_tokens(&mut parse_table, lexical_grammar, &token_conflict_map);
if minimize {
minimize_parse_table(
&mut parse_table,
@ -67,6 +66,7 @@ pub(crate) fn build_tables(
&token_conflict_map,
minimize,
);
mark_fragile_tokens(&mut parse_table, lexical_grammar, &token_conflict_map);
Ok((
parse_table,
main_lex_table,
@ -324,9 +324,9 @@ fn mark_fragile_tokens(
}
for (token, entry) in state.terminal_entries.iter_mut() {
if token.is_terminal() {
for i in 0..n {
if token_conflict_map.does_overlap(i, token.index) {
if valid_tokens_mask[i] {
for (i, is_valid) in valid_tokens_mask.iter().enumerate() {
if *is_valid {
if token_conflict_map.does_overlap(i, token.index) {
entry.reusable = false;
break;
}