From 0cceca7b4e9bad7080bffafb3f2be8080a56acc4 Mon Sep 17 00:00:00 2001 From: Max Brunsfeld Date: Mon, 21 Oct 2019 17:26:01 -0700 Subject: [PATCH] Rename extra_tokens -> extra_symbols --- .../generate/build_tables/build_parse_table.rs | 6 +++--- cli/src/generate/grammars.rs | 4 ++-- cli/src/generate/node_types.rs | 10 +++++----- cli/src/generate/parse_grammar.rs | 4 ++-- .../generate/prepare_grammar/expand_repeats.rs | 2 +- .../prepare_grammar/extract_simple_aliases.rs | 2 +- .../generate/prepare_grammar/extract_tokens.rs | 18 +++++++++--------- .../prepare_grammar/flatten_grammar.rs | 2 +- .../generate/prepare_grammar/intern_symbols.rs | 10 +++++----- cli/src/generate/prepare_grammar/mod.rs | 2 +- .../prepare_grammar/process_inlines.rs | 6 +++--- 11 files changed, 33 insertions(+), 33 deletions(-) diff --git a/cli/src/generate/build_tables/build_parse_table.rs b/cli/src/generate/build_tables/build_parse_table.rs index 8a59c977..c63701ee 100644 --- a/cli/src/generate/build_tables/build_parse_table.rs +++ b/cli/src/generate/build_tables/build_parse_table.rs @@ -76,7 +76,7 @@ impl<'a> ParseTableBuilder<'a> { let mut non_terminal_extra_item_sets_by_first_terminal = BTreeMap::new(); for extra_non_terminal in self .syntax_grammar - .extra_tokens + .extra_symbols .iter() .filter(|s| s.is_non_terminal()) { @@ -336,7 +336,7 @@ impl<'a> ParseTableBuilder<'a> { // are added to every state except for those at the ends of non-terminal // extras. if !is_end_of_non_terminal_extra { - for extra_token in &self.syntax_grammar.extra_tokens { + for extra_token in &self.syntax_grammar.extra_symbols { if extra_token.is_non_terminal() { state .nonterminal_entries @@ -843,7 +843,7 @@ fn populate_following_tokens( } } } - for extra in &grammar.extra_tokens { + for extra in &grammar.extra_symbols { if extra.is_terminal() { for entry in result.iter_mut() { entry.insert(*extra); diff --git a/cli/src/generate/grammars.rs b/cli/src/generate/grammars.rs index 0b42c4c3..6cf325dd 100644 --- a/cli/src/generate/grammars.rs +++ b/cli/src/generate/grammars.rs @@ -23,7 +23,7 @@ pub(crate) struct Variable { pub(crate) struct InputGrammar { pub name: String, pub variables: Vec, - pub extra_tokens: Vec, + pub extra_symbols: Vec, pub expected_conflicts: Vec>, pub external_tokens: Vec, pub variables_to_inline: Vec, @@ -87,7 +87,7 @@ pub(crate) struct ExternalToken { #[derive(Debug, Default)] pub(crate) struct SyntaxGrammar { pub variables: Vec, - pub extra_tokens: Vec, + pub extra_symbols: Vec, pub expected_conflicts: Vec>, pub external_tokens: Vec, pub supertype_symbols: Vec, diff --git a/cli/src/generate/node_types.rs b/cli/src/generate/node_types.rs index 2b88e54f..59594d89 100644 --- a/cli/src/generate/node_types.rs +++ b/cli/src/generate/node_types.rs @@ -689,7 +689,7 @@ mod tests { fn test_node_types_simple() { let node_types = get_node_types(InputGrammar { name: String::new(), - extra_tokens: Vec::new(), + extra_symbols: Vec::new(), external_tokens: Vec::new(), expected_conflicts: Vec::new(), variables_to_inline: Vec::new(), @@ -775,7 +775,7 @@ mod tests { fn test_node_types_with_supertypes() { let node_types = get_node_types(InputGrammar { name: String::new(), - extra_tokens: Vec::new(), + extra_symbols: Vec::new(), external_tokens: Vec::new(), expected_conflicts: Vec::new(), variables_to_inline: Vec::new(), @@ -862,7 +862,7 @@ mod tests { fn test_node_types_for_children_without_fields() { let node_types = get_node_types(InputGrammar { name: String::new(), - extra_tokens: Vec::new(), + extra_symbols: Vec::new(), external_tokens: Vec::new(), expected_conflicts: Vec::new(), variables_to_inline: Vec::new(), @@ -960,7 +960,7 @@ mod tests { fn test_node_types_for_aliased_nodes() { let node_types = get_node_types(InputGrammar { name: String::new(), - extra_tokens: Vec::new(), + extra_symbols: Vec::new(), external_tokens: Vec::new(), expected_conflicts: Vec::new(), variables_to_inline: Vec::new(), @@ -1036,7 +1036,7 @@ mod tests { fn test_node_types_with_multiple_valued_fields() { let node_types = get_node_types(InputGrammar { name: String::new(), - extra_tokens: Vec::new(), + extra_symbols: Vec::new(), external_tokens: Vec::new(), expected_conflicts: Vec::new(), variables_to_inline: Vec::new(), diff --git a/cli/src/generate/parse_grammar.rs b/cli/src/generate/parse_grammar.rs index feb560a9..c01dbd99 100644 --- a/cli/src/generate/parse_grammar.rs +++ b/cli/src/generate/parse_grammar.rs @@ -87,7 +87,7 @@ pub(crate) fn parse_grammar(input: &str) -> Result { }) } - let extra_tokens = grammar_json + let extra_symbols = grammar_json .extras .unwrap_or(Vec::new()) .into_iter() @@ -107,7 +107,7 @@ pub(crate) fn parse_grammar(input: &str) -> Result { name: grammar_json.name, word_token: grammar_json.word, variables, - extra_tokens, + extra_symbols, expected_conflicts, external_tokens, supertype_symbols, diff --git a/cli/src/generate/prepare_grammar/expand_repeats.rs b/cli/src/generate/prepare_grammar/expand_repeats.rs index ccc83d97..0660f06e 100644 --- a/cli/src/generate/prepare_grammar/expand_repeats.rs +++ b/cli/src/generate/prepare_grammar/expand_repeats.rs @@ -283,7 +283,7 @@ mod tests { fn build_grammar(variables: Vec) -> ExtractedSyntaxGrammar { ExtractedSyntaxGrammar { variables, - extra_tokens: Vec::new(), + extra_symbols: Vec::new(), external_tokens: Vec::new(), expected_conflicts: Vec::new(), variables_to_inline: Vec::new(), diff --git a/cli/src/generate/prepare_grammar/extract_simple_aliases.rs b/cli/src/generate/prepare_grammar/extract_simple_aliases.rs index 9a0b7fbb..6da009d5 100644 --- a/cli/src/generate/prepare_grammar/extract_simple_aliases.rs +++ b/cli/src/generate/prepare_grammar/extract_simple_aliases.rs @@ -146,7 +146,7 @@ mod tests { }], }, ], - extra_tokens: Vec::new(), + extra_symbols: Vec::new(), expected_conflicts: Vec::new(), variables_to_inline: Vec::new(), supertype_symbols: Vec::new(), diff --git a/cli/src/generate/prepare_grammar/extract_tokens.rs b/cli/src/generate/prepare_grammar/extract_tokens.rs index de37b1f9..ae6e7244 100644 --- a/cli/src/generate/prepare_grammar/extract_tokens.rs +++ b/cli/src/generate/prepare_grammar/extract_tokens.rs @@ -90,13 +90,13 @@ pub(super) fn extract_tokens( .collect(); let mut separators = Vec::new(); - let mut extra_tokens = Vec::new(); - for rule in grammar.extra_tokens { + let mut extra_symbols = Vec::new(); + for rule in grammar.extra_symbols { if let Rule::Symbol(symbol) = rule { - extra_tokens.push(symbol_replacer.replace_symbol(symbol)); + extra_symbols.push(symbol_replacer.replace_symbol(symbol)); } else { if let Some(index) = lexical_variables.iter().position(|v| v.rule == rule) { - extra_tokens.push(Symbol::terminal(index)); + extra_symbols.push(Symbol::terminal(index)); } else { separators.push(rule); } @@ -150,7 +150,7 @@ pub(super) fn extract_tokens( ExtractedSyntaxGrammar { variables, expected_conflicts, - extra_tokens, + extra_symbols, variables_to_inline, supertype_symbols, external_tokens, @@ -407,15 +407,15 @@ mod test { } #[test] - fn test_extracting_extra_tokens() { + fn test_extracting_extra_symbols() { let mut grammar = build_grammar(vec![ Variable::named("rule_0", Rule::string("x")), Variable::named("comment", Rule::pattern("//.*")), ]); - grammar.extra_tokens = vec![Rule::string(" "), Rule::non_terminal(1)]; + grammar.extra_symbols = vec![Rule::string(" "), Rule::non_terminal(1)]; let (syntax_grammar, lexical_grammar) = extract_tokens(grammar).unwrap(); - assert_eq!(syntax_grammar.extra_tokens, vec![Symbol::terminal(1),]); + assert_eq!(syntax_grammar.extra_symbols, vec![Symbol::terminal(1),]); assert_eq!(lexical_grammar.separators, vec![Rule::string(" "),]); } @@ -492,7 +492,7 @@ mod test { fn build_grammar(variables: Vec) -> InternedGrammar { InternedGrammar { variables, - extra_tokens: Vec::new(), + extra_symbols: Vec::new(), external_tokens: Vec::new(), expected_conflicts: Vec::new(), variables_to_inline: Vec::new(), diff --git a/cli/src/generate/prepare_grammar/flatten_grammar.rs b/cli/src/generate/prepare_grammar/flatten_grammar.rs index e325776c..f2b43a04 100644 --- a/cli/src/generate/prepare_grammar/flatten_grammar.rs +++ b/cli/src/generate/prepare_grammar/flatten_grammar.rs @@ -199,7 +199,7 @@ unless they are used only as the grammar's start rule. } } Ok(SyntaxGrammar { - extra_tokens: grammar.extra_tokens, + extra_symbols: grammar.extra_symbols, expected_conflicts: grammar.expected_conflicts, variables_to_inline: grammar.variables_to_inline, external_tokens: grammar.external_tokens, diff --git a/cli/src/generate/prepare_grammar/intern_symbols.rs b/cli/src/generate/prepare_grammar/intern_symbols.rs index 4c0fc5c7..7cd411ef 100644 --- a/cli/src/generate/prepare_grammar/intern_symbols.rs +++ b/cli/src/generate/prepare_grammar/intern_symbols.rs @@ -30,9 +30,9 @@ pub(super) fn intern_symbols(grammar: &InputGrammar) -> Result external_tokens.push(Variable { name, kind, rule }); } - let mut extra_tokens = Vec::with_capacity(grammar.extra_tokens.len()); - for extra_token in grammar.extra_tokens.iter() { - extra_tokens.push(interner.intern_rule(extra_token)?); + let mut extra_symbols = Vec::with_capacity(grammar.extra_symbols.len()); + for extra_token in grammar.extra_symbols.iter() { + extra_symbols.push(interner.intern_rule(extra_token)?); } let mut supertype_symbols = Vec::with_capacity(grammar.supertype_symbols.len()); @@ -76,7 +76,7 @@ pub(super) fn intern_symbols(grammar: &InputGrammar) -> Result Ok(InternedGrammar { variables, external_tokens, - extra_tokens, + extra_symbols, expected_conflicts, variables_to_inline, supertype_symbols, @@ -236,7 +236,7 @@ mod tests { InputGrammar { variables, name: "the_language".to_string(), - extra_tokens: Vec::new(), + extra_symbols: Vec::new(), external_tokens: Vec::new(), expected_conflicts: Vec::new(), variables_to_inline: Vec::new(), diff --git a/cli/src/generate/prepare_grammar/mod.rs b/cli/src/generate/prepare_grammar/mod.rs index a574aefb..029483d3 100644 --- a/cli/src/generate/prepare_grammar/mod.rs +++ b/cli/src/generate/prepare_grammar/mod.rs @@ -21,7 +21,7 @@ use crate::generate::rules::{AliasMap, Rule, Symbol}; pub(crate) struct IntermediateGrammar { variables: Vec, - extra_tokens: Vec, + extra_symbols: Vec, expected_conflicts: Vec>, external_tokens: Vec, variables_to_inline: Vec, diff --git a/cli/src/generate/prepare_grammar/process_inlines.rs b/cli/src/generate/prepare_grammar/process_inlines.rs index 68568419..9ef89d75 100644 --- a/cli/src/generate/prepare_grammar/process_inlines.rs +++ b/cli/src/generate/prepare_grammar/process_inlines.rs @@ -196,7 +196,7 @@ mod tests { fn test_basic_inlining() { let grammar = SyntaxGrammar { expected_conflicts: Vec::new(), - extra_tokens: Vec::new(), + extra_symbols: Vec::new(), external_tokens: Vec::new(), supertype_symbols: Vec::new(), word_token: None, @@ -327,7 +327,7 @@ mod tests { Symbol::non_terminal(3), ], expected_conflicts: Vec::new(), - extra_tokens: Vec::new(), + extra_symbols: Vec::new(), external_tokens: Vec::new(), supertype_symbols: Vec::new(), word_token: None, @@ -429,7 +429,7 @@ mod tests { }, ], expected_conflicts: Vec::new(), - extra_tokens: Vec::new(), + extra_symbols: Vec::new(), external_tokens: Vec::new(), supertype_symbols: Vec::new(), word_token: None,