traxys/tree-sitter
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Rename extra_tokens -> extra_symbols

Browse source
This commit is contained in:
Max Brunsfeld 2019-10-21 17:26:01 -07:00
parent fcaabea0cf
commit 0cceca7b4e
11 changed files with 33 additions and 33 deletions
Download patch file Download diff file Expand all files Collapse all files

2
cli/src/generate/prepare_grammar/expand_repeats.rs
View file

@ -283,7 +283,7 @@ mod tests {
fn build_grammar(variables: Vec<Variable>) -> ExtractedSyntaxGrammar {
ExtractedSyntaxGrammar {
variables,
extra_tokens: Vec::new(),
extra_symbols: Vec::new(),
external_tokens: Vec::new(),
expected_conflicts: Vec::new(),
variables_to_inline: Vec::new(),

2
cli/src/generate/prepare_grammar/extract_simple_aliases.rs
View file

@ -146,7 +146,7 @@ mod tests {
}],
},
],
extra_tokens: Vec::new(),
extra_symbols: Vec::new(),
expected_conflicts: Vec::new(),
variables_to_inline: Vec::new(),
supertype_symbols: Vec::new(),

18
cli/src/generate/prepare_grammar/extract_tokens.rs
View file

@ -90,13 +90,13 @@ pub(super) fn extract_tokens(
.collect();
let mut separators = Vec::new();
let mut extra_tokens = Vec::new();
for rule in grammar.extra_tokens {
let mut extra_symbols = Vec::new();
for rule in grammar.extra_symbols {
if let Rule::Symbol(symbol) = rule {
extra_tokens.push(symbol_replacer.replace_symbol(symbol));
extra_symbols.push(symbol_replacer.replace_symbol(symbol));
} else {
if let Some(index) = lexical_variables.iter().position(|v| v.rule == rule) {
extra_tokens.push(Symbol::terminal(index));
extra_symbols.push(Symbol::terminal(index));
} else {
separators.push(rule);
}
@ -150,7 +150,7 @@ pub(super) fn extract_tokens(
ExtractedSyntaxGrammar {
variables,
expected_conflicts,
extra_tokens,
extra_symbols,
variables_to_inline,
supertype_symbols,
external_tokens,
@ -407,15 +407,15 @@ mod test {
}
#[test]
fn test_extracting_extra_tokens() {
fn test_extracting_extra_symbols() {
let mut grammar = build_grammar(vec![
Variable::named("rule_0", Rule::string("x")),
Variable::named("comment", Rule::pattern("//.*")),
]);
grammar.extra_tokens = vec![Rule::string(" "), Rule::non_terminal(1)];
grammar.extra_symbols = vec![Rule::string(" "), Rule::non_terminal(1)];
let (syntax_grammar, lexical_grammar) = extract_tokens(grammar).unwrap();
assert_eq!(syntax_grammar.extra_tokens, vec![Symbol::terminal(1),]);
assert_eq!(syntax_grammar.extra_symbols, vec![Symbol::terminal(1),]);
assert_eq!(lexical_grammar.separators, vec![Rule::string(" "),]);
}
@ -492,7 +492,7 @@ mod test {
fn build_grammar(variables: Vec<Variable>) -> InternedGrammar {
InternedGrammar {
variables,
extra_tokens: Vec::new(),
extra_symbols: Vec::new(),
external_tokens: Vec::new(),
expected_conflicts: Vec::new(),
variables_to_inline: Vec::new(),

2
cli/src/generate/prepare_grammar/flatten_grammar.rs
View file

@ -199,7 +199,7 @@ unless they are used only as the grammar's start rule.
}
}
Ok(SyntaxGrammar {
extra_tokens: grammar.extra_tokens,
extra_symbols: grammar.extra_symbols,
expected_conflicts: grammar.expected_conflicts,
variables_to_inline: grammar.variables_to_inline,
external_tokens: grammar.external_tokens,

10
cli/src/generate/prepare_grammar/intern_symbols.rs
View file

@ -30,9 +30,9 @@ pub(super) fn intern_symbols(grammar: &InputGrammar) -> Result<InternedGrammar>
external_tokens.push(Variable { name, kind, rule });
}
let mut extra_tokens = Vec::with_capacity(grammar.extra_tokens.len());
for extra_token in grammar.extra_tokens.iter() {
extra_tokens.push(interner.intern_rule(extra_token)?);
let mut extra_symbols = Vec::with_capacity(grammar.extra_symbols.len());
for extra_token in grammar.extra_symbols.iter() {
extra_symbols.push(interner.intern_rule(extra_token)?);
}
let mut supertype_symbols = Vec::with_capacity(grammar.supertype_symbols.len());
@ -76,7 +76,7 @@ pub(super) fn intern_symbols(grammar: &InputGrammar) -> Result<InternedGrammar>
Ok(InternedGrammar {
variables,
external_tokens,
extra_tokens,
extra_symbols,
expected_conflicts,
variables_to_inline,
supertype_symbols,
@ -236,7 +236,7 @@ mod tests {
InputGrammar {
variables,
name: "the_language".to_string(),
extra_tokens: Vec::new(),
extra_symbols: Vec::new(),
external_tokens: Vec::new(),
expected_conflicts: Vec::new(),
variables_to_inline: Vec::new(),

2
cli/src/generate/prepare_grammar/mod.rs
View file

@ -21,7 +21,7 @@ use crate::generate::rules::{AliasMap, Rule, Symbol};
pub(crate) struct IntermediateGrammar<T, U> {
variables: Vec<Variable>,
extra_tokens: Vec<T>,
extra_symbols: Vec<T>,
expected_conflicts: Vec<Vec<Symbol>>,
external_tokens: Vec<U>,
variables_to_inline: Vec<Symbol>,

6
cli/src/generate/prepare_grammar/process_inlines.rs
View file

@ -196,7 +196,7 @@ mod tests {
fn test_basic_inlining() {
let grammar = SyntaxGrammar {
expected_conflicts: Vec::new(),
extra_tokens: Vec::new(),
extra_symbols: Vec::new(),
external_tokens: Vec::new(),
supertype_symbols: Vec::new(),
word_token: None,
@ -327,7 +327,7 @@ mod tests {
Symbol::non_terminal(3),
],
expected_conflicts: Vec::new(),
extra_tokens: Vec::new(),
extra_symbols: Vec::new(),
external_tokens: Vec::new(),
supertype_symbols: Vec::new(),
word_token: None,
@ -429,7 +429,7 @@ mod tests {
},
],
expected_conflicts: Vec::new(),
extra_tokens: Vec::new(),
extra_symbols: Vec::new(),
external_tokens: Vec::new(),
supertype_symbols: Vec::new(),
word_token: None,