traxys/tree-sitter
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Fix word token index issue in a different way

Browse source 
Refs https://github.com/tree-sitter/tree-sitter/issues/258
This commit is contained in:
Max Brunsfeld 2019-01-17 12:50:30 -08:00
parent 8f4096e5cb
commit c27f776d41
2 changed files with 11 additions and 11 deletions
Download patch file Download diff file Expand all files Collapse all files

12
cli/src/generate/build_tables/mod.rs
View file

@ -175,7 +175,17 @@ fn populate_used_symbols(
parse_table.symbols.push(Symbol::end());
for (i, value) in terminal_usages.into_iter().enumerate() {
if value {
parse_table.symbols.push(Symbol::terminal(i));
// Assign the grammar's word token a low numerical index. This ensures that
// it can be stored in a subtree with no heap allocations, even for grammars with
// very large numbers of tokens. This is an optimization, but it's also important to
// ensure that a subtree's symbol can be successfully reassigned to the word token
// without having to move the subtree to the heap.
// See https://github.com/tree-sitter/tree-sitter/issues/258
if syntax_grammar.word_token.map_or(false, |t| t.index == i) {
parse_table.symbols.insert(1, Symbol::terminal(i));
} else {
parse_table.symbols.push(Symbol::terminal(i));
}
}
}
for (i, value) in external_usages.into_iter().enumerate() {

10
cli/src/generate/prepare_grammar/extract_tokens.rs
View file

@ -15,16 +15,6 @@ pub(super) fn extract_tokens(
extracted_usage_counts: Vec::new(),
};
// Extract the word token first to give it a low numerical index. This ensure that
// it can be stored in a subtree with no heap allocations, even for grammars with
// very large numbers of tokens. This is an optimization, but also important to
// ensure that a subtree's symbol can be successfully reassigned to the word token
// without having to move the subtree to the heap.
// See https://github.com/tree-sitter/tree-sitter/issues/258
if let Some(token) = grammar.word_token {
extractor.extract_tokens_in_variable(&mut grammar.variables[token.index]);
}
for mut variable in grammar.variables.iter_mut() {
extractor.extract_tokens_in_variable(&mut variable);
}